perf(parquet): column parallelism + S3 byte range prefetching for arrow-rs reader
#6353Comparing
desmond/intra-rg-col-parallelism
(f7fc593
) withmain
(24e4c1d
)0%
- Untouched
- 36
- Ignored
- 4
Benchmarks
40 total
test_tpch_sql[1-in-memory-4]
tests/benchmarks/test_local_tpch.py+2%
86.3 ms84.4 ms
test_tpch_sql[1-in-memory-7]
tests/benchmarks/test_local_tpch.py+1%
121.5 ms119.9 ms
test_tpch_sql[1-in-memory-9]
tests/benchmarks/test_local_tpch.py+1%
257.1 ms254.5 ms
test_tpch[1-in-memory-8]
tests/benchmarks/test_local_tpch.py+1%
129.7 ms128.8 ms
test_tpch_sql[1-in-memory-8]
tests/benchmarks/test_local_tpch.py+1%
137.6 ms136.8 ms
test_read_jsonl_nested_struct
tests/benchmarks/test_json_reads.py+1%
502.1 ms499.1 ms
test_read_jsonl_string
tests/benchmarks/test_json_reads.py+1%
327.5 ms325.5 ms
test_tpch[1-in-memory-9]
tests/benchmarks/test_local_tpch.py0%
252.8 ms251.6 ms
test_read_jsonl_nested_list
tests/benchmarks/test_json_reads.py0%
442.6 ms440.6 ms
test_tpch[1-in-memory-1]
tests/benchmarks/test_local_tpch.py0%
550.9 ms548.5 ms
test_read_jsonl_float64
tests/benchmarks/test_json_reads.py0%
279.1 ms277.9 ms
test_read_json_array_format
tests/benchmarks/test_json_reads.py0%
1.8 s1.8 s
test_tpch[1-in-memory-5]
tests/benchmarks/test_local_tpch.py0%
115.1 ms114.8 ms
test_tpch_sql[1-in-memory-1]
tests/benchmarks/test_local_tpch.py0%
544.6 ms543.7 ms
test_read_jsonl_high_nulls
tests/benchmarks/test_json_reads.py0%
404.1 ms403.5 ms
test_read_jsonl_with_limit
tests/benchmarks/test_json_reads.py0%
68.3 ms68.3 ms
test_read_jsonl_with_column_projection
tests/benchmarks/test_json_reads.py0%
1.3 s1.3 s
test_read_jsonl_with_explicit_schema
tests/benchmarks/test_json_reads.py0%
1.6 s1.6 s
test_explain[100 Small Files]
tests/benchmarks/test_interactive_reads.py0%
11 ms11.1 ms
test_tpch_sql[1-in-memory-10]
tests/benchmarks/test_local_tpch.py0%
168.9 ms169 ms
test_read_jsonl_wide_table
tests/benchmarks/test_json_reads.py0%
1.6 s1.6 s
test_tpch[1-in-memory-6]
tests/benchmarks/test_local_tpch.py0%
32.1 ms32.1 ms
test_iter_rows_first_row[1 Small File]
tests/benchmarks/test_interactive_reads.py0%
33.7 ms33.8 ms
test_read_jsonl_int64
tests/benchmarks/test_json_reads.py0%
281.6 ms282.6 ms
test_tpch_sql[1-in-memory-3]
tests/benchmarks/test_local_tpch.py0%
120.3 ms120.7 ms
Commits
Click on a commit to change the comparison rangeBase
main
24e4c1d
+0.04%
perf(parquet): tune column parallelism thresholds and use per-file-handle I/O
193b2ab
1 month ago
by desmondcheongzx-0.18%
perf(parquet): pre-fetch S3 byte ranges for cross-column/RG coalescing
893bbed
1 month ago
by desmondcheongzx+0.17%
perf(parquet): use full-RG batch size in prefetched async decode
6caeb31
1 month ago
by desmondcheongzx-0.26%
perf(parquet): spawn column decode on compute runtime (DAFTCPU)
7311336
30 days ago
by desmondcheongzx-0.17%
perf(parquet): per-(RG, col) parallelism with per-column concat
3e6fbc3
30 days ago
by desmondcheongzx+0.29%
refactor(parquet): extract spawn_column_decode helper, remove duplication
f7fc593
30 days ago
by desmondcheongzx