mattsu2020:fmt_compatibility - Branch - uutils/coreutils

fmt : handle invalid UTF-8 input by replacing malformed sequences

#9329

Comparing

mattsu2020:fmt_compatibility

(

e36cf9d

) with

main

(

cc103ec

)

Untouched: 127

Skipped: 6

Benchmarks

Skipped (6)

Passed

tsort_input_parsing_heavy[5000]

src/uu/tsort/benches/tsort_bench.rs

+1%

84.1 ms*82.9 ms

du_wide_tree[(5000, 500)]

src/uu/du/benches/du_bench.rs

+1%

11.4 ms11.3 ms

cp_large_file[16]

src/uu/cp/benches/cp_bench.rs

+1%

363.3 µs361.1 µs

du_summarize_balanced_tree[(5, 4, 10)]

src/uu/du/benches/du_bench.rs

8.5 ms8.5 ms

du_max_depth_balanced_tree[(6, 4, 10)]

src/uu/du/benches/du_bench.rs

33.2 ms33.1 ms

du_deep_tree[(100, 3)]

src/uu/du/benches/du_bench.rs

1.2 ms1.2 ms

sort_mixed_data[500000]

src/uu/sort/benches/sort_bench.rs

328.4 ms*328.1 ms

cp_archive_balanced_tree[(5, 4, 10)]

src/uu/cp/benches/cp_bench.rs

78 ms78 ms

b64_decode_synthetic

src/uu/base64/benches/base64_bench.rs

166.3 µs*166.1 µs

ls_recursive_mixed_tree

src/uu/ls/benches/ls_bench.rs

1.5 ms*1.5 ms

sort_unique_mixed

src/uu/sort/benches/sort_locale_bench.rs

39.4 ms*39.4 ms

ls_recursive_long_all_balanced_tree[(6, 4, 15)]

src/uu/ls/benches/ls_bench.rs

134.9 ms*134.8 ms

ls_recursive_balanced_tree[(6, 4, 15)]

src/uu/ls/benches/ls_bench.rs

56.1 ms*56.1 ms

sort_german_locale

src/uu/sort/benches/sort_locale_bench.rs

39.3 ms*39.3 ms

ls_recursive_wide_tree[(10000, 1000)]

src/uu/ls/benches/ls_bench.rs

52 ms*51.9 ms

b64_decode_ignore_garbage_synthetic

src/uu/base64/benches/base64_bench.rs

163.2 µs*163.1 µs

ls_recursive_long_all_deep_tree[(100, 4)]

src/uu/ls/benches/ls_bench.rs

3.3 ms*3.3 ms

b64_encode_synthetic

src/uu/base64/benches/base64_bench.rs

161.8 µs*161.8 µs

cp_preserve_metadata[(5, 4, 10)]

src/uu/cp/benches/cp_bench.rs

75 ms75 ms

cp_recursive_balanced_tree[(5, 4, 10)]

src/uu/cp/benches/cp_bench.rs

72.3 ms72.3 ms

sort_reverse_locale[500000]

src/uu/sort/benches/sort_bench.rs

361.8 ms*361.8 ms

wc_bytes_synthetic[500]

src/uu/wc/benches/wc_bench.rs

181.6 µs*181.5 µs

shuf_input_range[1000000]

src/uu/shuf/benches/shuf_bench.rs

229.4 ms229.4 ms

factor_multiple_u64s[2]

src/uu/factor/benches/factor_bench.rs

212.4 ms212.4 ms

hashsum_sha512

src/uu/hashsum/benches/hashsum_bench.rs

110.8 ms*110.8 ms

mv_single_file

src/uu/mv/benches/mv_bench.rs

135.6 ms135.5 ms

du_all_wide_tree[(5000, 500)]

src/uu/du/benches/du_bench.rs

18.1 ms18.1 ms

sort_case_insensitive[500000]

src/uu/sort/benches/sort_bench.rs

276.7 ms*276.7 ms

split_lines

src/uu/split/benches/split_bench.rs

5.7 ms5.7 ms

mv_force_overwrite

src/uu/mv/benches/mv_bench.rs

144.3 ms144.3 ms

cksum_sha512

src/uu/cksum/benches/cksum_bench.rs

1.1 s*1.1 s

cksum_sha384

src/uu/cksum/benches/cksum_bench.rs

1.1 s*1.1 s

sort_unique_locale[500000]

src/uu/sort/benches/sort_bench.rs

494.2 ms*494.1 ms

sort_ascii_utf8_locale

src/uu/sort/benches/sort_locale_bench.rs

43.1 ms*43.1 ms

wc_chars_large_line_count[100000]

src/uu/wc/benches/wc_bench.rs

1 ms*1 ms

uniq_case_insensitive[10000]

src/uu/uniq/benches/uniq_bench.rs

1.8 ms1.8 ms

rm_single_file

src/uu/rm/benches/rm_bench.rs

119.4 ms119.4 ms

shuf_repeat_sampling[50000]

src/uu/shuf/benches/shuf_bench.rs

5.1 ms5.1 ms

cksum_sha2

src/uu/cksum/benches/cksum_bench.rs

1.7 s*1.7 s

rm_multiple_files

src/uu/rm/benches/rm_bench.rs

2.2 ms2.2 ms

cksum_sha3

src/uu/cksum/benches/cksum_bench.rs

1.8 s*1.8 s

rm_force_files

src/uu/rm/benches/rm_bench.rs

2 ms2 ms

seq_large_integers

src/uu/seq/benches/seq_bench.rs

2.2 ms2.2 ms

mv_directory

src/uu/mv/benches/mv_bench.rs

3.5 ms3.5 ms

wc_lines_large_line_count[500000]

src/uu/wc/benches/wc_bench.rs

2.9 ms*2.9 ms

cut_bytes

src/uu/cut/benches/cut_bench.rs

8.4 ms8.4 ms

wc_lines_variable_length[(50, 500)]

src/uu/wc/benches/wc_bench.rs

3.4 ms*3.4 ms

uniq_with_count[10000]

src/uu/uniq/benches/uniq_bench.rs

1.9 ms1.9 ms

sort_mixed_c_locale

src/uu/sort/benches/sort_locale_bench.rs

38.3 ms*38.3 ms

cut_fields_tab

src/uu/cut/benches/cut_bench.rs

34.1 ms34.1 ms

numfmt_round_modes[("up", 10000)]

src/uu/numfmt/benches/numfmt_bench.rs

34.3 ms34.3 ms

numfmt_to_si[10000]

src/uu/numfmt/benches/numfmt_bench.rs

33.8 ms33.8 ms

uniq_heavy_duplicates[10000]

src/uu/uniq/benches/uniq_bench.rs

2 ms2 ms

numfmt_round_modes[("down", 10000)]

src/uu/numfmt/benches/numfmt_bench.rs

34.3 ms34.3 ms

rm_recursive_tree

src/uu/rm/benches/rm_bench.rs

13.6 ms13.6 ms

numfmt_from_si[10000]

src/uu/numfmt/benches/numfmt_bench.rs

32.7 ms32.7 ms

numfmt_to_si_precision[10000]

src/uu/numfmt/benches/numfmt_bench.rs

35.3 ms35.3 ms

seq_with_step

src/uu/seq/benches/seq_bench.rs

13.3 ms13.3 ms

numfmt_round_modes[("towards-zero", 10000)]

src/uu/numfmt/benches/numfmt_bench.rs

34.3 ms34.3 ms

nl_many_lines[100000]

src/uu/nl/benches/nl_bench.rs

44.9 ms44.9 ms

numfmt_to_iec[10000]

src/uu/numfmt/benches/numfmt_bench.rs

33.8 ms33.8 ms

wc_words_large_line_count[100000]

src/uu/wc/benches/wc_bench.rs

48 ms*47.9 ms

numfmt_large_numbers_si[10000]

src/uu/numfmt/benches/numfmt_bench.rs

32.5 ms32.5 ms

nl_large_file[10]

src/uu/nl/benches/nl_bench.rs

57.1 ms57.1 ms

numfmt_padding[(10000, 50)]

src/uu/numfmt/benches/numfmt_bench.rs

48 ms48 ms

sort_dictionary_order[500000]

src/uu/sort/benches/sort_bench.rs

1.1 s*1.1 s

seq_integers

src/uu/seq/benches/seq_bench.rs

26.1 ms26.1 ms

wc_default_large_line_count[100000]

src/uu/wc/benches/wc_bench.rs

50.4 ms*50.4 ms

seq_custom_separator

src/uu/seq/benches/seq_bench.rs

28.7 ms28.7 ms

hashsum_sha256_check

src/uu/hashsum/benches/hashsum_bench.rs

172.9 ms*172.9 ms

hashsum_md5

src/uu/hashsum/benches/hashsum_bench.rs

32 ms*32 ms

expand_many_lines[100000]

src/uu/expand/benches/expand_bench.rs

149.7 ms149.7 ms

hashsum_md5_check

src/uu/hashsum/benches/hashsum_bench.rs

32 ms*32 ms

seq_formatted

src/uu/seq/benches/seq_bench.rs

96.1 ms96.1 ms

hashsum_sha1

src/uu/hashsum/benches/hashsum_bench.rs

72.5 ms*72.5 ms

cksum_shake256

src/uu/cksum/benches/cksum_bench.rs

1.9 s*1.9 s

cksum_shake128

src/uu/cksum/benches/cksum_bench.rs

1.5 s*1.5 s

wc_words_synthetic[2000]

src/uu/wc/benches/wc_bench.rs

11.4 s*11.4 s

unexpand_large_file[10]

src/uu/unexpand/benches/unexpand_bench.rs

567.6 ms567.6 ms

unexpand_many_lines[100000]

src/uu/unexpand/benches/unexpand_bench.rs

270.7 ms270.7 ms

hashsum_sha256

src/uu/hashsum/benches/hashsum_bench.rs

173 ms*173 ms

wc_bytes_lines_synthetic[2000]

src/uu/wc/benches/wc_bench.rs

120.2 ms*120.2 ms

fold_many_lines[100000]

src/uu/fold/benches/fold_bench.rs

80.4 ms80.4 ms

sort_reverse_mixed

src/uu/sort/benches/sort_locale_bench.rs

39.1 ms*39.1 ms

fold_custom_width[50000]

src/uu/fold/benches/fold_bench.rs

33 ms33 ms

cut_fields_custom_delim

src/uu/cut/benches/cut_bench.rs

40.8 ms40.8 ms

expand_custom_tabstops[50000]

src/uu/expand/benches/expand_bench.rs

37.1 ms37.1 ms

sort_numeric[500000]

src/uu/sort/benches/sort_bench.rs

1.2 s*1.2 s

sort_ascii_only[500000]

src/uu/sort/benches/sort_bench.rs

353.8 ms*353.8 ms

cksum_sha256

src/uu/cksum/benches/cksum_bench.rs

1.7 s*1.7 s

cksum_sm3

src/uu/cksum/benches/cksum_bench.rs

1.6 s*1.6 s

ls_recursive_long_all_wide_tree[(15000, 1500)]

src/uu/ls/benches/ls_bench.rs

143.2 ms*143.2 ms

cut_characters

src/uu/cut/benches/cut_bench.rs

7.4 ms7.4 ms

wc_lines_extreme_line_lengths[(100000, 200)]

src/uu/wc/benches/wc_bench.rs

1.6 ms*1.6 ms

sort_ascii_c_locale

src/uu/sort/benches/sort_locale_bench.rs

21.5 ms*21.5 ms

cksum_md5

src/uu/cksum/benches/cksum_bench.rs

317.9 ms*317.9 ms

sort_accented_data[500000]

src/uu/sort/benches/sort_bench.rs

362.6 ms*362.7 ms

cksum_sha224

src/uu/cksum/benches/cksum_bench.rs

1.7 s*1.7 s

cksum_blake2b

src/uu/cksum/benches/cksum_bench.rs

192.7 ms*192.8 ms

ls_recursive_long_all_mixed_tree

src/uu/ls/benches/ls_bench.rs

2.9 ms*2.9 ms

shuf_lines[100000]

src/uu/shuf/benches/shuf_bench.rs

31.6 ms31.6 ms

cksum_sha1

src/uu/cksum/benches/cksum_bench.rs

723.1 ms*723.2 ms

cp_recursive_deep_tree[(120, 4)]

src/uu/cp/benches/cp_bench.rs

13.6 ms13.6 ms

sort_numeric

src/uu/sort/benches/sort_locale_bench.rs

23.9 ms*23.9 ms

split_numeric_suffix

src/uu/split/benches/split_bench.rs

5.9 ms5.9 ms

split_bytes

src/uu/split/benches/split_bench.rs

510.1 µs510.2 µs

split_number_chunks

src/uu/split/benches/split_bench.rs

294.5 µs294.6 µs

cksum_bsd

src/uu/cksum/benches/cksum_bench.rs

128.3 ms*128.4 ms

sort_mixed_utf8_locale

src/uu/sort/benches/sort_locale_bench.rs

38.8 ms*38.8 ms

sort_german_c_locale

src/uu/sort/benches/sort_locale_bench.rs

38.4 ms*38.4 ms

ls_recursive_deep_tree[(200, 2)]

src/uu/ls/benches/ls_bench.rs

2.2 ms*2.2 ms

mv_multiple_to_dir

src/uu/mv/benches/mv_bench.rs

12.8 ms12.9 ms

cksum_blake3

src/uu/cksum/benches/cksum_bench.rs

208 µs*208.1 µs

sort_long_line[160000]

src/uu/sort/benches/sort_bench.rs

1.7 ms*1.7 ms

cp_recursive_wide_tree[(6000, 800)]

src/uu/cp/benches/cp_bench.rs

189.5 ms189.6 ms

sort_case_sensitive[500000]

src/uu/sort/benches/sort_bench.rs

173.8 ms*174 ms

sort_key_field[500000]

src/uu/sort/benches/sort_bench.rs

702.4 ms*703.1 ms

cksum_sysv

src/uu/cksum/benches/cksum_bench.rs

48.2 ms*48.3 ms

tsort_complex_dag[50000]

src/uu/tsort/benches/tsort_bench.rs

84 ms*84.1 ms

tsort_linear_chain[1000000]

src/uu/tsort/benches/tsort_bench.rs

1.5 s*1.5 s

cksum_multiple_files

src/uu/cksum/benches/cksum_bench.rs

25.3 ms*25.4 ms

cksum_crc

src/uu/cksum/benches/cksum_bench.rs

16.9 ms*17 ms

cksum_default

src/uu/cksum/benches/cksum_bench.rs

16.9 ms*17 ms

tsort_wide_dag[100000]

src/uu/tsort/benches/tsort_bench.rs

133.2 ms*133.6 ms

cksum_crc32b

src/uu/cksum/benches/cksum_bench.rs

13.3 ms*13.4 ms

tsort_tree_dag[(10, 3)]

src/uu/tsort/benches/tsort_bench.rs

34.1 ms*34.2 ms

cksum_raw_output

src/uu/cksum/benches/cksum_bench.rs

16.9 ms*17 ms

Commits

Click on a commit to change the comparison range

Base

main

cc103ec

chore: add FFFD to spell-checker ignore list in fmt test.

2c617d4

1 month ago

by mattsu2020

-0.01%

refactor(uu/fmt): switch string outputs to byte slices for efficiency - Changed `indent_str` field in `BreakArgs` to `indent: &[u8]` to avoid repeated UTF-8 conversions. - Updated `write_all` calls to pass `&s` instead of `s.as_bytes()` in fmt.rs and similar string/byteslicing in linebreak.rs. - Modified method signatures in parasplit.rs to accept `&[u8]` instead of `&str` for prefix matching, ensuring consistent byte-level operations without assuming valid UTF-8.

db77543

1 month ago

by mattsu2020

refactor(fmt): replace Option::map_or(false, f) with is_some_and(f) - Updated indentation calculation in FileLines to use is_some_and for tab and character checks, avoiding unnecessary computations and improving code flow. - Changed punctuation checks in WordSplit iterator to use is_some_and for cleaner, more idiomatic Rust code. - This refactor enhances readability and leverages short-circuiting behavior.

4445c5a

1 month ago

by mattsu2020

+2.07%

style(fmt): compact whitespace check in WordSplit iterator to single line Refactored the is_whitespace assignment by combining chained method calls on one line for improved conciseness and readability.

c59f1bc

1 month ago

by mattsu2020

-2.09%

fix(test_fmt): align invalid UTF-8 handling with GNU-compatible passthrough Updated test_fmt_invalid_utf8 to expect raw byte (\xA0) passthrough instead of replacement character (\u{FFFD}) for invalid UTF-8 input, ensuring GNU-compatible behavior in fmt. This fixes the test expectation to match actual output, avoiding lossy conversion.

6a313a4

1 month ago

by mattsu2020

+0.05%

Merge branch 'main' into fmt_compatibility

fee09e6

9 days ago

by mattsu2020

-0.01%

Merge branch 'main' into fmt_compatibility

e36cf9d

7 hours ago

by mattsu2020

Home Terms Privacy Docs