mkitti:mkitti-get-chunk-slice-vectorization - Branch - zarr-developers/zarr-python

perf: Vectorize get_chunk_slice for faster sharded writes

#3713

Comparing

mkitti:mkitti-get-chunk-slice-vectorization

(

157d283

) with

main

(

36caf1f

)

×7.2

Improvement: 4

Untouched: 52

Skipped: 6

Benchmarks

62 total

test_morton_order_iter[(32, 32, 32)]

tests/benchmarks/test_indexing.py

×7.2

502.6 ms70.2 ms

test_morton_order_iter[(16, 16, 16)]

tests/benchmarks/test_indexing.py

×6.9

56.2 ms8.1 ms

test_sharded_morton_write_single_chunk[(32, 32, 32)-memory]

tests/benchmarks/test_indexing.py

×5.8

953.3 ms164.4 ms

test_morton_order_iter[(8, 8, 8)]

tests/benchmarks/test_indexing.py

×5.5

6.2 ms1.1 ms

test_write_array[local-Layout(shape=(1000000,), chunks=(1000,), shards=None)-gzip]

tests/benchmarks/test_e2e.py

+4%

1.7 s1.6 s

test_write_array[memory-Layout(shape=(1000000,), chunks=(100,), shards=(1000000,))-None]

tests/benchmarks/test_e2e.py

+4%

5.5 s5.3 s

test_write_array[local-Layout(shape=(1000000,), chunks=(100,), shards=(1000000,))-None]

tests/benchmarks/test_e2e.py

+4%

5.5 s5.3 s

test_read_array[local-Layout(shape=(1000000,), chunks=(1000,), shards=(1000,))-None]

tests/benchmarks/test_e2e.py

+3%

1.4 s1.4 s

test_write_array[memory-Layout(shape=(1000000,), chunks=(100,), shards=(1000000,))-gzip]

tests/benchmarks/test_e2e.py

+2%

9.7 s9.5 s

test_write_array[local-Layout(shape=(1000000,), chunks=(1000,), shards=None)-None]

tests/benchmarks/test_e2e.py

+2%

1.2 s1.2 s

test_read_array[memory-Layout(shape=(1000000,), chunks=(1000,), shards=None)-None]

tests/benchmarks/test_e2e.py

+2%

282 ms275.9 ms

test_slice_indexing[None-(slice(None, None, None), slice(None, None, None), slice(None, None, None))-memory]

tests/benchmarks/test_indexing.py

+2%

384 ms376.4 ms

test_slice_indexing[None-(0, 0, 0)-memory]

tests/benchmarks/test_indexing.py

+2%

860.1 µs843.3 µs

test_write_array[local-Layout(shape=(1000000,), chunks=(100,), shards=(1000000,))-gzip]

tests/benchmarks/test_e2e.py

+2%

9.7 s9.5 s

test_slice_indexing[None-(slice(None, None, None), slice(None, None, None), slice(None, None, None))-memory_get_latency]

tests/benchmarks/test_indexing.py

+2%

436.1 ms428.3 ms

test_slice_indexing[None-(slice(10, -10, 4), slice(10, -10, 4), slice(10, -10, 4))-memory_get_latency]

tests/benchmarks/test_indexing.py

+2%

236.7 ms232.5 ms

test_sharded_morton_indexing_large[(32, 32, 32)-memory]

tests/benchmarks/test_indexing.py

+2%

9.5 s9.3 s

test_write_array[memory-Layout(shape=(1000000,), chunks=(1000,), shards=None)-None]

tests/benchmarks/test_e2e.py

+2%

561.6 ms552.6 ms

test_slice_indexing[None-(slice(10, -10, 4), slice(10, -10, 4), slice(10, -10, 4))-memory]

tests/benchmarks/test_indexing.py

+2%

208.1 ms204.8 ms

test_read_array[local-Layout(shape=(1000000,), chunks=(1000,), shards=(1000,))-gzip]

tests/benchmarks/test_e2e.py

+2%

1.8 s1.7 s

test_slice_indexing[None-(slice(0, None, 4), slice(0, None, 4), slice(0, None, 4))-memory_get_latency]

tests/benchmarks/test_indexing.py

+2%

433.5 ms426.8 ms

test_read_array[local-Layout(shape=(1000000,), chunks=(100,), shards=(1000000,))-None]

tests/benchmarks/test_e2e.py

+2%

2.9 s2.9 s

test_sharded_morton_indexing[(32, 32, 32)-memory]

tests/benchmarks/test_indexing.py

+2%

1.2 s1.2 s

test_read_array[memory-Layout(shape=(1000000,), chunks=(1000,), shards=(1000,))-None]

tests/benchmarks/test_e2e.py

+1%

987.7 ms973.3 ms

test_read_array[local-Layout(shape=(1000000,), chunks=(1000,), shards=None)-gzip]

tests/benchmarks/test_e2e.py

+1%

943.1 ms929.6 ms

Commits

Click on a commit to change the comparison range

Base

main

36caf1f

×7.2

Merge branch 'main' into mkitti-get-chunk-slice-vectorization

157d283

12 hours ago

by d-v-b

Home Terms Privacy Docs