naoNao89:fix/sort-benchmark-variance - Branch - uutils/coreutils

Reduce variance in sort benchmarks by reusing temp file

#8939Merged

Comparing

naoNao89:fix/sort-benchmark-variance

(

607383a

) with

main

(

0258583

)

-3%

Regressions: 2

Untouched: 102

Archived: 75

Benchmarks

75 archived benchmarks were run. If they were deleted in another branch, consider rebasing to remove them from the report. .

Failed

du_balanced_tree[(5, 4, 10)]

src/uu/du/benches/du_bench.rs

Regression

-2%

9.1 ms9.3 ms

du_human_balanced_tree[(5, 4, 10)]

src/uu/du/benches/du_bench.rs

Regression

-3%

10.1 ms10.5 ms

Passed

ls_recursive_wide_tree[(10000, 1000)]

src/uu/ls/benches/ls_bench.rs

+1%

52.1 ms51.6 ms

du_deep_tree[(100, 3)]

src/uu/du/benches/du_bench.rs

+1%

1.1 ms1.1 ms

du_wide_tree[(5000, 500)]

src/uu/du/benches/du_bench.rs

+1%

11.3 ms11.2 ms

ls_recursive_long_all_wide_tree[(15000, 1500)]

src/uu/ls/benches/ls_bench.rs

+1%

146.4 ms145.4 ms

du_all_wide_tree[(5000, 500)]

src/uu/du/benches/du_bench.rs

+1%

18.2 ms18.1 ms

ls_recursive_balanced_tree[(6, 4, 15)]

src/uu/ls/benches/ls_bench.rs

+1%

56.1 ms55.8 ms

ls_recursive_long_all_deep_tree[(100, 4)]

src/uu/ls/benches/ls_bench.rs

3.2 ms3.1 ms

rm_single_file

src/uu/rm/benches/rm_bench.rs

100.4 ms100.1 ms

du_all_balanced_tree[(4, 3, 10)]

src/uu/du/benches/du_bench.rs

1.6 ms1.6 ms

sort_unique_locale[500000]

src/uu/sort/benches/sort_bench.rs

1.1 s1.1 s

sort_numeric[500000]

src/uu/sort/benches/sort_bench.rs

1.2 s1.2 s

cp_recursive_balanced_tree[(5, 4, 10)]

src/uu/cp/benches/cp_bench.rs

71.9 ms71.8 ms

cp_recursive_deep_tree[(120, 4)]

src/uu/cp/benches/cp_bench.rs

13.3 ms13.3 ms

tsort_complex_dag[50000]

src/uu/tsort/benches/tsort_bench.rs

84 ms84 ms

sort_ascii_c_locale

src/uu/sort/benches/sort_locale_bench.rs

27.6 ms27.6 ms

cp_preserve_metadata[(5, 4, 10)]

src/uu/cp/benches/cp_bench.rs

74.4 ms74.3 ms

sort_key_field[500000]

src/uu/sort/benches/sort_bench.rs

688.8 ms688.8 ms

tsort_linear_chain[1000000]

src/uu/tsort/benches/tsort_bench.rs

1.5 s1.5 s

tsort_wide_dag[100000]

src/uu/tsort/benches/tsort_bench.rs

135.5 ms135.5 ms

rm_force_files

src/uu/rm/benches/rm_bench.rs

2 ms2 ms

sort_case_insensitive[500000]

src/uu/sort/benches/sort_bench.rs

417.2 ms417.1 ms

sort_mixed_data[500000]

src/uu/sort/benches/sort_bench.rs

729.6 ms729.6 ms

fold_custom_width[50000]

src/uu/fold/benches/fold_bench.rs

46.6 ms46.6 ms

uniq_heavy_duplicates[10000]

src/uu/uniq/benches/uniq_bench.rs

1.9 ms1.9 ms

seq_formatted

src/uu/seq/benches/seq_bench.rs

99.7 ms99.7 ms

fold_many_lines[100000]

src/uu/fold/benches/fold_bench.rs

125.3 ms125.3 ms

numfmt_to_iec[10000]

src/uu/numfmt/benches/numfmt_bench.rs

34 ms34 ms

cut_characters

src/uu/cut/benches/cut_bench.rs

7.5 ms7.5 ms

expand_many_lines[100000]

src/uu/expand/benches/expand_bench.rs

100.9 ms100.9 ms

unexpand_large_file[10]

src/uu/unexpand/benches/unexpand_bench.rs

548 ms548 ms

wc_words_synthetic[2000]

src/uu/wc/benches/wc_bench.rs

10.3 s10.3 s

numfmt_to_si[10000]

src/uu/numfmt/benches/numfmt_bench.rs

33.8 ms33.8 ms

unexpand_many_lines[100000]

src/uu/unexpand/benches/unexpand_bench.rs

261.4 ms261.4 ms

nl_large_file[10]

src/uu/nl/benches/nl_bench.rs

100.6 ms100.6 ms

wc_bytes_lines_synthetic[2000]

src/uu/wc/benches/wc_bench.rs

120.2 ms120.2 ms

nl_many_lines[100000]

src/uu/nl/benches/nl_bench.rs

79.5 ms79.5 ms

sort_dictionary_order[500000]

src/uu/sort/benches/sort_bench.rs

1.1 s1.1 s

numfmt_padding[(10000, 50)]

src/uu/numfmt/benches/numfmt_bench.rs

48.4 ms48.4 ms

wc_default_large_line_count[100000]

src/uu/wc/benches/wc_bench.rs

48.9 ms48.9 ms

hashsum_sha256

src/uu/hashsum/benches/hashsum_bench.rs

173.6 ms173.6 ms

expand_custom_tabstops[50000]

src/uu/expand/benches/expand_bench.rs

25.9 ms25.9 ms

cut_fields_custom_delim

src/uu/cut/benches/cut_bench.rs

40.5 ms40.5 ms

numfmt_round_modes[("down", 10000)]

src/uu/numfmt/benches/numfmt_bench.rs

34.5 ms34.5 ms

seq_integers

src/uu/seq/benches/seq_bench.rs

26.7 ms26.7 ms

wc_words_large_line_count[100000]

src/uu/wc/benches/wc_bench.rs

43.8 ms43.8 ms

hashsum_sha512

src/uu/hashsum/benches/hashsum_bench.rs

111.7 ms111.7 ms

hashsum_sha256_check

src/uu/hashsum/benches/hashsum_bench.rs

173.6 ms173.6 ms

seq_custom_separator

src/uu/seq/benches/seq_bench.rs

29.2 ms29.2 ms

numfmt_from_si[10000]

src/uu/numfmt/benches/numfmt_bench.rs

33.1 ms33.1 ms

hashsum_sha1

src/uu/hashsum/benches/hashsum_bench.rs

72.5 ms72.5 ms

numfmt_round_modes[("up", 10000)]

src/uu/numfmt/benches/numfmt_bench.rs

32.7 ms32.7 ms

uniq_with_count[10000]

src/uu/uniq/benches/uniq_bench.rs

1.8 ms1.8 ms

hashsum_md5

src/uu/hashsum/benches/hashsum_bench.rs

32 ms32 ms

numfmt_to_si_precision[10000]

src/uu/numfmt/benches/numfmt_bench.rs

35.3 ms35.3 ms

numfmt_round_modes[("towards-zero", 10000)]

src/uu/numfmt/benches/numfmt_bench.rs

34.5 ms34.5 ms

cut_fields_tab

src/uu/cut/benches/cut_bench.rs

33.9 ms33.9 ms

hashsum_md5_check

src/uu/hashsum/benches/hashsum_bench.rs

32 ms32 ms

numfmt_large_numbers_si[10000]

src/uu/numfmt/benches/numfmt_bench.rs

34.5 ms34.5 ms

wc_lines_variable_length[(50, 500)]

src/uu/wc/benches/wc_bench.rs

3.4 ms3.4 ms

sort_reverse_mixed

src/uu/sort/benches/sort_locale_bench.rs

93.2 ms93.2 ms

seq_with_step

src/uu/seq/benches/seq_bench.rs

13.6 ms13.6 ms

sort_reverse_locale[500000]

src/uu/sort/benches/sort_bench.rs

820.7 ms820.7 ms

cp_large_file[16]

src/uu/cp/benches/cp_bench.rs

293.7 µs293.7 µs

wc_lines_large_line_count[500000]

src/uu/wc/benches/wc_bench.rs

2.9 ms2.9 ms

sort_accented_data[500000]

src/uu/sort/benches/sort_bench.rs

819.3 ms819.3 ms

cut_bytes

src/uu/cut/benches/cut_bench.rs

8.4 ms8.4 ms

wc_lines_extreme_line_lengths[(100000, 200)]

src/uu/wc/benches/wc_bench.rs

1.6 ms1.6 ms

sort_german_locale

src/uu/sort/benches/sort_locale_bench.rs

94.5 ms94.5 ms

uniq_case_insensitive[10000]

src/uu/uniq/benches/uniq_bench.rs

1.7 ms1.7 ms

wc_chars_large_line_count[100000]

src/uu/wc/benches/wc_bench.rs

990 µs990.1 µs

sort_case_sensitive[500000]

src/uu/sort/benches/sort_bench.rs

327.1 ms327.1 ms

mv_directory

src/uu/mv/benches/mv_bench.rs

3.1 ms3.1 ms

rm_multiple_files

src/uu/rm/benches/rm_bench.rs

2.2 ms2.2 ms

sort_numeric

src/uu/sort/benches/sort_locale_bench.rs

21.8 ms21.8 ms

split_lines

src/uu/split/benches/split_bench.rs

5.9 ms5.9 ms

tsort_tree_dag[(10, 3)]

src/uu/tsort/benches/tsort_bench.rs

34.1 ms34.1 ms

sort_long_line[160000]

src/uu/sort/benches/sort_bench.rs

2 ms2 ms

sort_mixed_c_locale

src/uu/sort/benches/sort_locale_bench.rs

93.8 ms93.8 ms

wc_bytes_synthetic[500]

src/uu/wc/benches/wc_bench.rs

162.7 µs162.7 µs

split_numeric_suffix

src/uu/split/benches/split_bench.rs

6.1 ms6.1 ms

b64_encode_synthetic

src/uu/base64/benches/base64_bench.rs

141.6 µs141.6 µs

cp_recursive_wide_tree[(6000, 800)]

src/uu/cp/benches/cp_bench.rs

188.5 ms188.5 ms

mv_single_file

src/uu/mv/benches/mv_bench.rs

120.5 ms120.5 ms

mv_force_overwrite

src/uu/mv/benches/mv_bench.rs

123.8 ms123.9 ms

mv_multiple_to_dir

src/uu/mv/benches/mv_bench.rs

13 ms13 ms

sort_ascii_utf8_locale

src/uu/sort/benches/sort_locale_bench.rs

55.5 ms55.6 ms

sort_german_c_locale

src/uu/sort/benches/sort_locale_bench.rs

94.5 ms94.5 ms

cp_archive_balanced_tree[(5, 4, 10)]

src/uu/cp/benches/cp_bench.rs

77.1 ms77.1 ms

b64_decode_synthetic

src/uu/base64/benches/base64_bench.rs

145.6 µs145.7 µs

split_bytes

src/uu/split/benches/split_bench.rs

482.5 µs482.8 µs

sort_unique_mixed

src/uu/sort/benches/sort_locale_bench.rs

87.6 ms87.7 ms

b64_decode_ignore_garbage_synthetic

src/uu/base64/benches/base64_bench.rs

145.1 µs145.2 µs

split_number_chunks

src/uu/split/benches/split_bench.rs

270.4 µs270.7 µs

sort_ascii_only[500000]

src/uu/sort/benches/sort_bench.rs

798 ms799.1 ms

ls_recursive_long_all_mixed_tree

src/uu/ls/benches/ls_bench.rs

4 ms4 ms

ls_recursive_deep_tree[(200, 2)]

src/uu/ls/benches/ls_bench.rs

2.1 ms2.1 ms

sort_mixed_utf8_locale

src/uu/sort/benches/sort_locale_bench.rs

93.8 ms94.1 ms

du_max_depth_balanced_tree[(6, 4, 10)]

src/uu/du/benches/du_bench.rs

32.8 ms32.9 ms

rm_recursive_tree

src/uu/rm/benches/rm_bench.rs

14 ms14 ms

ls_recursive_mixed_tree

src/uu/ls/benches/ls_bench.rs

1.4 ms1.4 ms

ls_recursive_long_all_balanced_tree[(6, 4, 15)]

src/uu/ls/benches/ls_bench.rs

-1%

136.8 ms138.4 ms

du_summarize_balanced_tree[(5, 4, 10)]

src/uu/du/benches/du_bench.rs

-1%

8.4 ms8.5 ms

Commits

Click on a commit to change the comparison range

Base

main

0258583

-3.36%

bench(sort): reduce variance by reusing output files across iterations Locale benchmarks were creating temp files inside the bench loop (from d1cd9998be), causing filesystem noise and false CodSpeed regressions. The same commit's sort_bench.rs got it right with file creation outside the loop. This fix aligns with that pattern.

607383a

1 month ago

by naoNao89

Home Terms Privacy Docs