fix/huggingface-multi-split-fallback - Branch - Eventual-Inc/Daft

fix(io): load all splits in read_huggingface fallback path

#5757Merged

Comparing

fix/huggingface-multi-split-fallback

(

1e49b66

) with

main

(

75caaad

)

Untouched: 24

Ignored: 4

Benchmarks

Passed

test_tpch[1-in-memory-3]

tests/benchmarks/test_local_tpch.py

+3%

128.3 ms124.6 ms

test_show[1 Small File]

tests/benchmarks/test_interactive_reads.py

+2%

13.4 ms13.1 ms

test_tpch_sql[1-in-memory-2]

tests/benchmarks/test_local_tpch.py

+1%

167.9 ms165.5 ms

test_tpch[1-in-memory-4]

tests/benchmarks/test_local_tpch.py

+1%

89.6 ms88.6 ms

test_explain[100 Small Files]

tests/benchmarks/test_interactive_reads.py

+1%

12.1 ms12 ms

test_tpch[1-in-memory-7]

tests/benchmarks/test_local_tpch.py

+1%

128.4 ms127.2 ms

test_tpch[1-in-memory-8]

tests/benchmarks/test_local_tpch.py

+1%

151.8 ms150.9 ms

test_tpch[1-in-memory-5]

tests/benchmarks/test_local_tpch.py

131.8 ms131.2 ms

test_tpch_sql[1-in-memory-7]

tests/benchmarks/test_local_tpch.py

120.7 ms120.3 ms

test_tpch[1-in-memory-1]

tests/benchmarks/test_local_tpch.py

403.6 ms402.7 ms

test_iter_rows_first_row[1 Small File]

tests/benchmarks/test_interactive_reads.py

35.8 ms35.8 ms

test_tpch_sql[1-in-memory-8]

tests/benchmarks/test_local_tpch.py

137.4 ms137.1 ms

test_tpch[1-in-memory-6]

tests/benchmarks/test_local_tpch.py

28.8 ms28.9 ms

test_tpch_sql[1-in-memory-9]

tests/benchmarks/test_local_tpch.py

265 ms265.3 ms

test_tpch_sql[1-in-memory-4]

tests/benchmarks/test_local_tpch.py

89.1 ms89.2 ms

test_count[100 Small Files]

tests/benchmarks/test_interactive_reads.py

57.6 ms57.7 ms

test_tpch_sql[1-in-memory-5]

tests/benchmarks/test_local_tpch.py

119.7 ms119.9 ms

test_tpch[1-in-memory-2]

tests/benchmarks/test_local_tpch.py

59.3 ms59.5 ms

test_tpch[1-in-memory-10]

tests/benchmarks/test_local_tpch.py

199.5 ms200.1 ms

test_tpch_sql[1-in-memory-3]

tests/benchmarks/test_local_tpch.py

117.7 ms118.3 ms

test_tpch_sql[1-in-memory-10]

tests/benchmarks/test_local_tpch.py

-1%

190.3 ms191.8 ms

test_tpch_sql[1-in-memory-6]

tests/benchmarks/test_local_tpch.py

-1%

29.5 ms29.9 ms

test_tpch_sql[1-in-memory-1]

tests/benchmarks/test_local_tpch.py

-1%

401.9 ms407.3 ms

test_tpch[1-in-memory-9]

tests/benchmarks/test_local_tpch.py

-2%

276.2 ms281.1 ms

Ignored

test_count[1 Small File]

tests/benchmarks/test_interactive_reads.py

Ignored

4.1 ms4.1 ms

test_explain[1 Small File]

tests/benchmarks/test_interactive_reads.py

Ignored

+2%

2.1 ms2.1 ms

test_show[100 Small Files]

tests/benchmarks/test_interactive_reads.py

Ignored

+33%

31.5 ms23.7 ms

test_iter_rows_first_row[100 Small Files]

tests/benchmarks/test_interactive_reads.py

Ignored

-42%

140.8 ms241 ms

Commits

Click on a commit to change the comparison range

Base

main

75caaad

+0.21%

fix(io): load all splits in read_huggingface fallback path The fallback path previously hardcoded split='train', which failed for datasets that don't have a train split or have multiple splits. Now we load all splits using load_dataset() without a split parameter and concatenate them using concatenate_datasets(), matching the behavior of the main parquet path which reads all splits.

1e49b66

15 hours ago

by ykdojo

Home Terms Privacy Docs