aldenks:coalesce-shard-reads - Branch - zarr-developers/zarr-python

Coalesce and parallelize partial shard reads

#3004

Comparing

aldenks:coalesce-shard-reads

(

6437fb6

) with

main

(

c06923d

)

-1%

Untouched: 30

Benchmarks

30 total

test_write_array[local-Layout(shape=(1000000,), chunks=(1000,), shards=(1000,))-gzip]

tests/benchmarks/test_e2e.py

+1%

3.3 s3.2 s

test_write_array[memory-Layout(shape=(1000000,), chunks=(1000,), shards=(1000,))-gzip]

tests/benchmarks/test_e2e.py

+1%

2.2 s2.2 s

test_write_array[local-Layout(shape=(1000000,), chunks=(1000,), shards=(1000,))-None]

tests/benchmarks/test_e2e.py

+1%

2.9 s2.8 s

test_write_array[local-Layout(shape=(1000000,), chunks=(100,), shards=(1000000,))-None]

tests/benchmarks/test_e2e.py

11.2 s11.2 s

test_read_array[local-Layout(shape=(1000000,), chunks=(1000,), shards=(1000,))-None]

tests/benchmarks/test_e2e.py

1.6 s1.6 s

test_write_array[local-Layout(shape=(1000000,), chunks=(100,), shards=(1000000,))-gzip]

tests/benchmarks/test_e2e.py

15.4 s15.4 s

test_write_array[memory-Layout(shape=(1000000,), chunks=(1000,), shards=None)-gzip]

tests/benchmarks/test_e2e.py

992.2 ms992.7 ms

test_read_array[local-Layout(shape=(1000000,), chunks=(100,), shards=(1000000,))-None]

tests/benchmarks/test_e2e.py

3.8 s3.8 s

test_slice_indexing[(slice(0, None, 4), slice(0, None, 4), slice(0, None, 4))-memory]

tests/benchmarks/test_indexing.py

493.9 ms494.7 ms

test_slice_indexing[(slice(10, -10, 4), slice(10, -10, 4), slice(10, -10, 4))-memory]

tests/benchmarks/test_indexing.py

270.7 ms271.2 ms

test_read_array[memory-Layout(shape=(1000000,), chunks=(1000,), shards=None)-None]

tests/benchmarks/test_e2e.py

365.9 ms366.9 ms

test_slice_indexing[(slice(None, None, None), slice(0, 3, 2), slice(0, 10, None))-memory]

tests/benchmarks/test_indexing.py

4.7 ms4.7 ms

test_slice_indexing[(0, 0, 0)-memory]

tests/benchmarks/test_indexing.py

925 µs927.6 µs

test_write_array[memory-Layout(shape=(1000000,), chunks=(100,), shards=(1000000,))-None]

tests/benchmarks/test_e2e.py

11.2 s11.2 s

test_read_array[memory-Layout(shape=(1000000,), chunks=(1000,), shards=(1000,))-gzip]

tests/benchmarks/test_e2e.py

1.5 s1.5 s

test_read_array[memory-Layout(shape=(1000000,), chunks=(1000,), shards=(1000,))-None]

tests/benchmarks/test_e2e.py

1.2 s1.2 s

test_read_array[memory-Layout(shape=(1000000,), chunks=(100,), shards=(1000000,))-None]

tests/benchmarks/test_e2e.py

-1%

3.8 s3.8 s

test_write_array[memory-Layout(shape=(1000000,), chunks=(1000,), shards=(1000,))-None]

tests/benchmarks/test_e2e.py

-1%

1.7 s1.7 s

test_write_array[memory-Layout(shape=(1000000,), chunks=(1000,), shards=None)-None]

tests/benchmarks/test_e2e.py

-1%

543.1 ms546.4 ms

test_read_array[local-Layout(shape=(1000000,), chunks=(1000,), shards=(1000,))-gzip]

tests/benchmarks/test_e2e.py

-1%

1.9 s1.9 s

test_slice_indexing[(slice(None, None, None), slice(None, None, None), slice(None, None, None))-memory]

tests/benchmarks/test_indexing.py

-1%

496.5 ms500.2 ms

test_read_array[memory-Layout(shape=(1000000,), chunks=(1000,), shards=None)-gzip]

tests/benchmarks/test_e2e.py

-1%

669.9 ms675.1 ms

test_read_array[memory-Layout(shape=(1000000,), chunks=(100,), shards=(1000000,))-gzip]

tests/benchmarks/test_e2e.py

-1%

6.8 s6.9 s

test_read_array[local-Layout(shape=(1000000,), chunks=(100,), shards=(1000000,))-gzip]

tests/benchmarks/test_e2e.py

-1%

6.8 s6.8 s

test_read_array[local-Layout(shape=(1000000,), chunks=(1000,), shards=None)-None]

tests/benchmarks/test_e2e.py

-1%

703.2 ms712.8 ms

Commits

Click on a commit to change the comparison range

Base

main

c06923d

-0.41%

Merge branch 'main' into coalesce-shard-reads

ba77e87

4 days ago

by aldenks

+0.3%

Ensure no change in behavior when ByteGetter.get returns None + comment

b71302f

4 days ago

by aldenks

-0.68%

Add test_sharing_unit.py, focusing on coallesce behavior, but with basic tests for other components

24f6f1c

4 days ago

by aldenks

+0.21%

Fix typing errors in test_sharing_unit.py

6437fb6

4 days ago

by aldenks

Home Terms Privacy Docs