feat: implement ray-based pre-shuffle-merge for flight shuffle
#6306Comparing
oh/flight_shuffle_dev
(3920728
) withmain
(5437e2d
)0%
- Untouched
- 36
- Ignored
- 4
Benchmarks
40 total
test_tpch[1-in-memory-10]
tests/benchmarks/test_local_tpch.py+2%
169.8 ms166.1 ms
test_tpch_sql[1-in-memory-1]
tests/benchmarks/test_local_tpch.py+2%
553 ms541.4 ms
test_count[100 Small Files]
tests/benchmarks/test_interactive_reads.py+2%
57.6 ms56.7 ms
test_read_json_array_format
tests/benchmarks/test_json_reads.py+2%
1.8 s1.8 s
test_read_jsonl_with_limit
tests/benchmarks/test_json_reads.py+1%
68.1 ms67.5 ms
test_tpch_sql[1-in-memory-10]
tests/benchmarks/test_local_tpch.py+1%
169.5 ms168.1 ms
test_tpch_sql[1-in-memory-2]
tests/benchmarks/test_local_tpch.py+1%
154.6 ms153.8 ms
test_tpch[1-in-memory-7]
tests/benchmarks/test_local_tpch.py0%
115.7 ms115.2 ms
test_read_jsonl_with_explicit_schema
tests/benchmarks/test_json_reads.py0%
1.6 s1.6 s
test_read_jsonl_string
tests/benchmarks/test_json_reads.py0%
326.1 ms324.9 ms
test_tpch_sql[1-in-memory-3]
tests/benchmarks/test_local_tpch.py0%
122.5 ms122 ms
test_tpch_sql[1-in-memory-5]
tests/benchmarks/test_local_tpch.py0%
115.6 ms115.3 ms
test_tpch[1-in-memory-6]
tests/benchmarks/test_local_tpch.py0%
32.2 ms32.2 ms
test_read_jsonl_boolean
tests/benchmarks/test_json_reads.py0%
356.6 ms356.2 ms
test_read_jsonl_high_nulls
tests/benchmarks/test_json_reads.py0%
402 ms401.6 ms
test_tpch[1-in-memory-9]
tests/benchmarks/test_local_tpch.py0%
255.1 ms255.1 ms
test_tpch[1-in-memory-5]
tests/benchmarks/test_local_tpch.py0%
115.4 ms115.5 ms
test_read_jsonl_with_column_projection
tests/benchmarks/test_json_reads.py0%
1.3 s1.3 s
test_read_jsonl_wide_table
tests/benchmarks/test_json_reads.py0%
1.6 s1.6 s
test_tpch_sql[1-in-memory-6]
tests/benchmarks/test_local_tpch.py0%
32.8 ms32.9 ms
test_read_jsonl_int64
tests/benchmarks/test_json_reads.py0%
280.6 ms281.6 ms
test_tpch[1-in-memory-4]
tests/benchmarks/test_local_tpch.py0%
83.6 ms83.9 ms
test_read_jsonl_nested_list
tests/benchmarks/test_json_reads.py0%
441.8 ms443.7 ms
test_read_jsonl_float64
tests/benchmarks/test_json_reads.py-1%
277.3 ms279.1 ms
test_tpch[1-in-memory-3]
tests/benchmarks/test_local_tpch.py-1%
120.6 ms121.4 ms
Commits
Click on a commit to change the comparison rangeBase
main
5437e2d
-0.05%
fix flight pre shuffle merge implementation
e9411b8
6 hours ago
by ohbh-0.18%
don't use threshold in distributed operator
3920728
4 hours ago
by ohbh