ChrisDryden:fix-split-number-memory-issue - Branch - uutils/coreutils

split: stream data using copy to not overwhelm memory with large files

#10251

Comparing

ChrisDryden:fix-split-number-memory-issue

(

da74374

) with

main

(

f7b5f8f

)

-21%

Regressions: 1

Untouched: 140

Skipped: 38

No successful run was found on main ( 450e7cf) during the generation of this report, so f7b5f8f was used instead as the comparison base. There might be some changes unrelated to this pull request in this report.

Benchmarks

Skipped (38)

Failed

split_number_chunks

src/uu/split/benches/split_bench.rs

Regression

-21%

289.5 µs364.5 µs

Passed

cksum_crc32b

src/uu/cksum/benches/cksum_bench.rs

13.2 ms13.2 ms

dd_copy_1m_blocks

src/uu/dd/benches/dd_bench.rs

1.5 ms1.5 ms

dd_copy_partial

src/uu/dd/benches/dd_bench.rs

572.1 µs570.5 µs

file_tz_abbreviations

src/uu/date/benches/date_bench.rs

18.5 ms18.4 ms

cksum_multiple_files

src/uu/cksum/benches/cksum_bench.rs

25.2 ms25.1 ms

du_summarize_balanced_tree[(5, 4, 10)]

src/uu/du/benches/du_bench.rs

8.5 ms8.5 ms

du_max_depth_balanced_tree[(6, 4, 10)]

src/uu/du/benches/du_bench.rs

33.1 ms33 ms

cp_preserve_metadata[(5, 4, 10)]

src/uu/cp/benches/cp_bench.rs

74.9 ms74.8 ms

cksum_sysv

src/uu/cksum/benches/cksum_bench.rs

48.2 ms48.2 ms

dd_copy_64k_blocks

src/uu/dd/benches/dd_bench.rs

617.9 µs617.4 µs

df_deep_directory

src/uu/df/benches/df_bench.rs

382.7 µs382.4 µs

cksum_raw_output

src/uu/cksum/benches/cksum_bench.rs

16.7 ms16.7 ms

cksum_default

src/uu/cksum/benches/cksum_bench.rs

16.7 ms16.7 ms

cksum_crc

src/uu/cksum/benches/cksum_bench.rs

16.7 ms16.7 ms

cksum_blake3

src/uu/cksum/benches/cksum_bench.rs

214.6 µs214.5 µs

sort_long_line[160000]

src/uu/sort/benches/sort_bench.rs

1.7 ms1.7 ms

cp_archive_balanced_tree[(5, 4, 10)]

src/uu/cp/benches/cp_bench.rs

78 ms77.9 ms

single_date_now

src/uu/date/benches/date_bench.rs

190.6 µs190.5 µs

sort_unique_utf8_locale

src/uu/sort/benches/sort_locale_utf8_bench.rs

38.1 ms38.1 ms

cksum_bsd

src/uu/cksum/benches/cksum_bench.rs

128.3 ms128.2 ms

split_bytes

src/uu/split/benches/split_bench.rs

515.3 µs515.2 µs

cksum_blake2b

src/uu/cksum/benches/cksum_bench.rs

192.7 ms192.7 ms

sort_mixed_utf8_locale

src/uu/sort/benches/sort_locale_utf8_bench.rs

37.8 ms37.8 ms

du_wide_tree[(5000, 500)]

src/uu/du/benches/du_bench.rs

11.3 ms11.3 ms

complex_relative_date

src/uu/date/benches/date_bench.rs

245.5 µs245.5 µs

mv_single_file

src/uu/mv/benches/mv_bench.rs

141.1 ms141.1 ms

tsort_tree_dag[(10, 3)]

src/uu/tsort/benches/tsort_bench.rs

34.6 ms34.6 ms

cp_recursive_balanced_tree[(5, 4, 10)]

src/uu/cp/benches/cp_bench.rs

72.3 ms72.3 ms

tsort_wide_dag[100000]

src/uu/tsort/benches/tsort_bench.rs

131.5 ms131.5 ms

mv_force_overwrite

src/uu/mv/benches/mv_bench.rs

149.7 ms149.7 ms

ls_recursive_balanced_tree[(6, 4, 15)]

src/uu/ls/benches/ls_bench.rs

55.3 ms55.3 ms

wc_bytes_lines_synthetic[2000]

src/uu/wc/benches/wc_bench.rs

120.2 ms120.2 ms

uniq_case_insensitive[10000]

src/uu/uniq/benches/uniq_bench.rs

1.7 ms1.7 ms

factor_multiple_u64s[2]

src/uu/factor/benches/factor_bench.rs

224.9 ms224.9 ms

sort_mixed_c_locale

src/uu/sort/benches/sort_locale_c_bench.rs

37.3 ms37.3 ms

wc_lines_variable_length[(50, 500)]

src/uu/wc/benches/wc_bench.rs

3.5 ms3.5 ms

wc_lines_large_line_count[500000]

src/uu/wc/benches/wc_bench.rs

2.9 ms2.9 ms

sort_numeric_utf8_locale

src/uu/sort/benches/sort_locale_utf8_bench.rs

22.5 ms22.5 ms

mv_multiple_to_dir

src/uu/mv/benches/mv_bench.rs

12.9 ms12.9 ms

sort_key_field[500000]

src/uu/sort/benches/sort_bench.rs

692 ms692 ms

sort_case_sensitive[500000]

src/uu/sort/benches/sort_bench.rs

166.1 ms166.1 ms

uniq_heavy_duplicates[10000]

src/uu/uniq/benches/uniq_bench.rs

1.9 ms1.9 ms

tsort_linear_chain[1000000]

src/uu/tsort/benches/tsort_bench.rs

1.6 s1.6 s

uniq_with_count[10000]

src/uu/uniq/benches/uniq_bench.rs

1.8 ms1.8 ms

cksum_sm3

src/uu/cksum/benches/cksum_bench.rs

1.6 s1.6 s

df_with_path

src/uu/df/benches/df_bench.rs

408.1 µs408.1 µs

shuf_input_range[1000000]

src/uu/shuf/benches/shuf_bench.rs

163.5 ms163.5 ms

join_custom_separator

src/uu/join/benches/join_bench.rs

12.1 ms12.1 ms

ls_recursive_long_all_balanced_tree[(6, 4, 15)]

src/uu/ls/benches/ls_bench.rs

130.2 ms130.2 ms

cksum_md5

src/uu/cksum/benches/cksum_bench.rs

317.9 ms317.9 ms

sort_ascii_utf8_locale

src/uu/sort/benches/sort_locale_utf8_bench.rs

20.6 ms20.6 ms

cut_characters

src/uu/cut/benches/cut_bench.rs

7.4 ms7.4 ms

cksum_sha384

src/uu/cksum/benches/cksum_bench.rs

1.1 s1.1 s

cksum_sha512

src/uu/cksum/benches/cksum_bench.rs

1.1 s1.1 s

cut_bytes

src/uu/cut/benches/cut_bench.rs

8.4 ms8.4 ms

cksum_sha1

src/uu/cksum/benches/cksum_bench.rs

723 ms723 ms

mv_directory

src/uu/mv/benches/mv_bench.rs

3.5 ms3.5 ms

cksum_sha256

src/uu/cksum/benches/cksum_bench.rs

1.7 s1.7 s

cksum_sha224

src/uu/cksum/benches/cksum_bench.rs

1.7 s1.7 s

cut_fields_tab

src/uu/cut/benches/cut_bench.rs

34.1 ms34.1 ms

cut_fields_custom_delim

src/uu/cut/benches/cut_bench.rs

40.8 ms40.8 ms

join_full_match

src/uu/join/benches/join_bench.rs

12.8 ms12.8 ms

expand_custom_tabstops[50000]

src/uu/expand/benches/expand_bench.rs

36.9 ms36.9 ms

wc_words_synthetic[2000]

src/uu/wc/benches/wc_bench.rs

11.4 s11.4 s

unexpand_large_file[10]

src/uu/unexpand/benches/unexpand_bench.rs

529 ms529 ms

cksum_shake128

src/uu/cksum/benches/cksum_bench.rs

1.5 s1.5 s

cksum_shake256

src/uu/cksum/benches/cksum_bench.rs

1.9 s1.9 s

expand_many_lines[100000]

src/uu/expand/benches/expand_bench.rs

148.6 ms148.6 ms

nl_many_lines[100000]

src/uu/nl/benches/nl_bench.rs

44.9 ms44.9 ms

unexpand_many_lines[100000]

src/uu/unexpand/benches/unexpand_bench.rs

252.3 ms252.3 ms

nl_large_file[10]

src/uu/nl/benches/nl_bench.rs

57.2 ms57.2 ms

sort_mixed_data[500000]

src/uu/sort/benches/sort_bench.rs

319.1 ms319.1 ms

fold_many_lines[100000]

src/uu/fold/benches/fold_bench.rs

80.4 ms80.4 ms

seq_formatted

src/uu/seq/benches/seq_bench.rs

96.1 ms96.1 ms

numfmt_padding[(10000, 50)]

src/uu/numfmt/benches/numfmt_bench.rs

48.6 ms48.6 ms

numfmt_round_modes[("down", 10000)]

src/uu/numfmt/benches/numfmt_bench.rs

34.8 ms34.8 ms

cksum_sha3

src/uu/cksum/benches/cksum_bench.rs

1.8 s1.8 s

cksum_sha2

src/uu/cksum/benches/cksum_bench.rs

1.7 s1.7 s

wc_words_large_line_count[100000]

src/uu/wc/benches/wc_bench.rs

48 ms48 ms

numfmt_round_modes[("up", 10000)]

src/uu/numfmt/benches/numfmt_bench.rs

34.8 ms34.8 ms

fold_custom_width[50000]

src/uu/fold/benches/fold_bench.rs

33.1 ms33.1 ms

numfmt_round_modes[("towards-zero", 10000)]

src/uu/numfmt/benches/numfmt_bench.rs

34.7 ms34.7 ms

numfmt_to_si[10000]

src/uu/numfmt/benches/numfmt_bench.rs

34.1 ms34.1 ms

wc_default_large_line_count[100000]

src/uu/wc/benches/wc_bench.rs

50.4 ms50.4 ms

numfmt_to_iec[10000]

src/uu/numfmt/benches/numfmt_bench.rs

34.3 ms34.3 ms

seq_custom_separator

src/uu/seq/benches/seq_bench.rs

28.7 ms28.7 ms

numfmt_large_numbers_si[10000]

src/uu/numfmt/benches/numfmt_bench.rs

32.7 ms32.7 ms

sort_numeric[500000]

src/uu/sort/benches/sort_bench.rs

1.2 s1.2 s

numfmt_to_si_precision[10000]

src/uu/numfmt/benches/numfmt_bench.rs

35.7 ms35.7 ms

file_iso_dates

src/uu/date/benches/date_bench.rs

13.8 ms13.8 ms

numfmt_from_si[10000]

src/uu/numfmt/benches/numfmt_bench.rs

32.6 ms32.6 ms

sort_reverse_locale[500000]

src/uu/sort/benches/sort_bench.rs

352.3 ms352.3 ms

seq_integers

src/uu/seq/benches/seq_bench.rs

26.1 ms26.1 ms

join_partial_overlap

src/uu/join/benches/join_bench.rs

10.5 ms10.5 ms

seq_with_step

src/uu/seq/benches/seq_bench.rs

13.3 ms13.3 ms

ls_recursive_wide_tree[(10000, 1000)]

src/uu/ls/benches/ls_bench.rs

51.7 ms51.7 ms

rm_recursive_tree

src/uu/rm/benches/rm_bench.rs

11.8 ms11.8 ms

wc_lines_extreme_line_lengths[(100000, 200)]

src/uu/wc/benches/wc_bench.rs

1.6 ms1.6 ms

sort_accented_data[500000]

src/uu/sort/benches/sort_bench.rs

353.2 ms353.2 ms

join_french_locale

src/uu/join/benches/join_bench.rs

12.8 ms12.8 ms

sort_dictionary_order[500000]

src/uu/sort/benches/sort_bench.rs

1.1 s1.1 s

sort_ascii_c_locale

src/uu/sort/benches/sort_locale_c_bench.rs

20.5 ms20.5 ms

wc_chars_large_line_count[100000]

src/uu/wc/benches/wc_bench.rs

1 ms1 ms

ls_recursive_long_all_wide_tree[(15000, 1500)]

src/uu/ls/benches/ls_bench.rs

140.2 ms140.2 ms

seq_large_integers

src/uu/seq/benches/seq_bench.rs

2.2 ms2.2 ms

dd_copy_with_skip

src/uu/dd/benches/dd_bench.rs

2.3 ms2.3 ms

tsort_complex_dag[50000]

src/uu/tsort/benches/tsort_bench.rs

86.1 ms86.1 ms

ls_recursive_long_all_mixed_tree

src/uu/ls/benches/ls_bench.rs

2.8 ms2.8 ms

ls_recursive_long_all_deep_tree[(100, 4)]

src/uu/ls/benches/ls_bench.rs

3.2 ms3.2 ms

sort_case_insensitive[500000]

src/uu/sort/benches/sort_bench.rs

267.2 ms267.3 ms

rm_force_files

src/uu/rm/benches/rm_bench.rs

2 ms2 ms

sort_general_numeric[200000]

src/uu/sort/benches/sort_bench.rs

728.8 ms728.9 ms

rm_multiple_files

src/uu/rm/benches/rm_bench.rs

2.2 ms2.2 ms

sort_reverse_utf8_locale

src/uu/sort/benches/sort_locale_utf8_bench.rs

37.2 ms37.2 ms

ls_recursive_deep_tree[(200, 2)]

src/uu/ls/benches/ls_bench.rs

2.2 ms2.2 ms

sort_german_de_locale

src/uu/sort/benches/sort_locale_de_bench.rs

38 ms38 ms

shuf_repeat_sampling[50000]

src/uu/shuf/benches/shuf_bench.rs

5.2 ms5.2 ms

ls_recursive_mixed_tree

src/uu/ls/benches/ls_bench.rs

1.5 ms1.5 ms

b64_encode_synthetic

src/uu/base64/benches/base64_bench.rs

168.9 µs168.9 µs

cp_recursive_wide_tree[(6000, 800)]

src/uu/cp/benches/cp_bench.rs

189.4 ms189.4 ms

wc_bytes_synthetic[500]

src/uu/wc/benches/wc_bench.rs

191.6 µs191.7 µs

shuf_lines[100000]

src/uu/shuf/benches/shuf_bench.rs

31.6 ms31.6 ms

cp_recursive_deep_tree[(120, 4)]

src/uu/cp/benches/cp_bench.rs

13.6 ms13.6 ms

rm_single_file

src/uu/rm/benches/rm_bench.rs

124 ms124 ms

dd_copy_separate_blocks

src/uu/dd/benches/dd_bench.rs

44.1 ms44.1 ms

b64_decode_ignore_garbage_synthetic

src/uu/base64/benches/base64_bench.rs

169.2 µs169.3 µs

du_all_wide_tree[(5000, 500)]

src/uu/du/benches/du_bench.rs

18.1 ms18.1 ms

sort_german_c_locale

src/uu/sort/benches/sort_locale_c_bench.rs

38 ms38 ms

split_lines

src/uu/split/benches/split_bench.rs

5.7 ms5.7 ms

b64_decode_synthetic

src/uu/base64/benches/base64_bench.rs

171.3 µs171.4 µs

cp_large_file[16]

src/uu/cp/benches/cp_bench.rs

373.3 µs373.6 µs

split_numeric_suffix

src/uu/split/benches/split_bench.rs

5.9 ms5.9 ms

dd_copy_4k_blocks

src/uu/dd/benches/dd_bench.rs

1.3 ms1.4 ms

sort_unique_locale[500000]

src/uu/sort/benches/sort_bench.rs

484.6 ms485.3 ms

dd_copy_default

src/uu/dd/benches/dd_bench.rs

13.9 ms14 ms

file_custom_format

src/uu/date/benches/date_bench.rs

13.2 ms13.2 ms

dd_copy_with_seek

src/uu/dd/benches/dd_bench.rs

2.3 ms2.3 ms

du_deep_tree[(100, 3)]

src/uu/du/benches/du_bench.rs

1.2 ms1.2 ms

sort_ascii_only[500000]

src/uu/sort/benches/sort_bench.rs

-1%

344.4 ms346.3 ms

dd_copy_8k_blocks

src/uu/dd/benches/dd_bench.rs

-1%

1 ms1 ms

Commits

Click on a commit to change the comparison range

Base

main

f7b5f8f

-20.58%

split: stream data in --number mode to avoid loading entire chunks into memory

86e6a79

24 hours ago

by ChrisDryden

split: increase memory limit in streaming test for cross-compiled platforms

da74374

20 hours ago

by ChrisDryden

Home Terms Privacy Docs