perf(shuffle): Write one shuffle file per task instead of N partition files
#6948Comparing
colin/flight-shuffle-perf
(878485c
) withmain
(9beb5f2
)0%
- Untouched
- 40
- Ignored
- 4
- Skipped
- 10
Benchmarks
54 total
test_clickbench_sql[2]
tests/benchmarks/test_clickbench.py+4%
21.5 ms20.7 ms
test_tpch_sql[1-in-memory-4]
tests/benchmarks/test_local_tpch.py+2%
86.6 ms85 ms
test_tpch_sql[1-in-memory-3]
tests/benchmarks/test_local_tpch.py+2%
122.4 ms120.2 ms
test_iter_rows_first_row[1 Small File]
tests/benchmarks/test_interactive_reads.py+2%
34.5 ms33.9 ms
test_tpch_sql[1-in-memory-1]
tests/benchmarks/test_local_tpch.py+2%
422.3 ms415.3 ms
test_clickbench_sql[4]
tests/benchmarks/test_clickbench.py+1%
80.2 ms79.5 ms
test_clickbench_sql[37]
tests/benchmarks/test_clickbench.py+1%
183.5 ms182.5 ms
test_read_jsonl_boolean
tests/benchmarks/test_json_reads.py0%
357.3 ms355.7 ms
test_tpch_sql[1-in-memory-5]
tests/benchmarks/test_local_tpch.py0%
113.6 ms113.1 ms
test_read_jsonl_float64
tests/benchmarks/test_json_reads.py0%
281.9 ms281.5 ms
test_read_jsonl_int64
tests/benchmarks/test_json_reads.py0%
283.8 ms283.8 ms
test_tpch_sql[1-in-memory-6]
tests/benchmarks/test_local_tpch.py0%
33.1 ms33.1 ms
test_tpch_sql[1-in-memory-7]
tests/benchmarks/test_local_tpch.py0%
118.9 ms119.1 ms
test_tpch_sql[1-in-memory-10]
tests/benchmarks/test_local_tpch.py0%
169.8 ms170.2 ms
test_explain[100 Small Files]
tests/benchmarks/test_interactive_reads.py0%
10.3 ms10.3 ms
test_read_jsonl_nested_list
tests/benchmarks/test_json_reads.py0%
445.6 ms446.8 ms
test_read_jsonl_wide_table
tests/benchmarks/test_json_reads.py0%
1.7 s1.7 s
test_read_jsonl_with_column_projection
tests/benchmarks/test_json_reads.py0%
1.3 s1.3 s
test_clickbench_sql[19]
tests/benchmarks/test_clickbench.py0%
11.1 ms11.2 ms
test_clickbench_sql[42]
tests/benchmarks/test_clickbench.py0%
83.3 ms83.7 ms
test_tpch_sql[1-in-memory-8]
tests/benchmarks/test_local_tpch.py0%
132.7 ms133.3 ms
test_read_jsonl_with_explicit_schema
tests/benchmarks/test_json_reads.py0%
1.7 s1.7 s
test_read_json_array_format
tests/benchmarks/test_json_reads.py-1%
1.8 s1.8 s
test_read_jsonl_with_limit
tests/benchmarks/test_json_reads.py-1%
68.2 ms68.7 ms
test_tpch_sql[1-in-memory-9]
tests/benchmarks/test_local_tpch.py-1%
248.3 ms250.1 ms
Commits
Click on a commit to change the comparison rangeBase
main
9beb5f2
-0.18%
perf(shuffle): spawn-free oneshot writer + read-side server concat
d68ea49
3 days ago
by colin-ho-0.22%
refactor(shuffle): simplify oneshot writer and read-side concat
feb7d58
2 days ago
by colin-ho0%
perf(shuffle): bump repartition buffer threshold 64 MiB -> 256 MiB
3ea0999
4 hours ago
by colin-ho+0.17%
chore(shuffle): strip oneshot writer profiling instrumentation
9f5dc34
4 hours ago
by colin-ho-11.4%
refactor(shuffle): drop env-var override for chunk target bytes
7c45b6a
3 hours ago
by colin-ho+11.29%
simplify server
878485c
1 hour ago
by colin-ho