diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 70812439c..22bba5ed7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -10,8 +10,10 @@ repos: rev: 'v0.4.3' hooks: - id: ruff - args: ["--fix", "--show-fixes"] + args: ["--fix", "--show-fixes", --select, I] + types_or: [ python, pyi, jupyter ] - id: ruff-format + types_or: [ python, pyi, jupyter ] - repo: https://github.com/codespell-project/codespell rev: v2.2.6 hooks: diff --git a/bench/compress_normal.py b/bench/compress_normal.py index 803d54b76..38969b010 100644 --- a/bench/compress_normal.py +++ b/bench/compress_normal.py @@ -1,9 +1,8 @@ import sys import timeit -import numpy as np - import line_profiler +import numpy as np import zarr from zarr import blosc diff --git a/notebooks/advanced_indexing.ipynb b/notebooks/advanced_indexing.ipynb index eba6b5880..dc0e50b0a 100644 --- a/notebooks/advanced_indexing.ipynb +++ b/notebooks/advanced_indexing.ipynb @@ -25,11 +25,14 @@ ], "source": [ "import sys\n", - "sys.path.insert(0, '..')\n", - "import zarr\n", + "\n", + "sys.path.insert(0, \"..\")\n", "import numpy as np\n", + "import zarr\n", + "\n", "np.random.seed(42)\n", "import cProfile\n", + "\n", "zarr.__version__" ] }, @@ -57,7 +60,7 @@ "source": [ "a = np.arange(10)\n", "za = zarr.array(a, chunks=2)\n", - "ix = [False, True, False, True, False, True, False, True, False, True]" + "ix = [False, True, False, True, False, True, False, True, False, True]" ] }, { @@ -547,7 +550,7 @@ ], "source": [ "# combine with slice\n", - "za.oindex[[1, 3], :]" + "za.oindex[[1, 3], :]" ] }, { @@ -973,10 +976,10 @@ } ], "source": [ - "a = np.array([(b'aaa', 1, 4.2),\n", - " (b'bbb', 2, 8.4),\n", - " (b'ccc', 3, 12.6)], \n", - " dtype=[('foo', 'S3'), ('bar', 'i4'), ('baz', 'f8')])\n", + "a = np.array(\n", + " [(b\"aaa\", 1, 4.2), (b\"bbb\", 2, 8.4), (b\"ccc\", 3, 12.6)],\n", + " dtype=[(\"foo\", \"S3\"), (\"bar\", \"i4\"), (\"baz\", \"f8\")],\n", + ")\n", "za = zarr.array(a, chunks=2, fill_value=None)\n", "za[:]" ] @@ -999,7 +1002,7 @@ } ], "source": [ - "za['foo']" + "za[\"foo\"]" ] }, { @@ -1020,7 +1023,7 @@ } ], "source": [ - "za['foo', 'baz']" + "za[\"foo\", \"baz\"]" ] }, { @@ -1041,7 +1044,7 @@ } ], "source": [ - "za[:2, 'foo']" + "za[:2, \"foo\"]" ] }, { @@ -1062,7 +1065,7 @@ } ], "source": [ - "za[:2, 'foo', 'baz']" + "za[:2, \"foo\", \"baz\"]" ] }, { @@ -1083,7 +1086,7 @@ } ], "source": [ - "za.oindex[[0, 2], 'foo']" + "za.oindex[[0, 2], \"foo\"]" ] }, { @@ -1104,7 +1107,7 @@ } ], "source": [ - "za.vindex[[0, 2], 'foo']" + "za.vindex[[0, 2], \"foo\"]" ] }, { @@ -1125,7 +1128,7 @@ } ], "source": [ - "za['bar'] = 42\n", + "za[\"bar\"] = 42\n", "za[:]" ] }, @@ -1147,7 +1150,7 @@ } ], "source": [ - "za[:2, 'bar'] = 84\n", + "za[:2, \"bar\"] = 84\n", "za[:]" ] }, @@ -1176,7 +1179,7 @@ } ], "source": [ - "a['foo', 'baz']" + "a[\"foo\", \"baz\"]" ] }, { @@ -1197,7 +1200,7 @@ } ], "source": [ - "a[['foo', 'baz']]" + "a[[\"foo\", \"baz\"]]" ] }, { @@ -1218,7 +1221,7 @@ } ], "source": [ - "za['foo', 'baz']" + "za[\"foo\", \"baz\"]" ] }, { @@ -1243,7 +1246,7 @@ } ], "source": [ - "za[['foo', 'baz']]" + "za[[\"foo\", \"baz\"]]" ] }, { @@ -1437,14 +1440,15 @@ "metadata": {}, "outputs": [], "source": [ - "import tempfile\n", "import cProfile\n", "import pstats\n", + "import tempfile\n", + "\n", "\n", - "def profile(statement, sort='time', restrictions=(7,)):\n", + "def profile(statement, sort=\"time\", restrictions=(7,)):\n", " with tempfile.NamedTemporaryFile() as f:\n", " cProfile.run(statement, filename=f.name)\n", - " pstats.Stats(f.name).sort_stats(sort).print_stats(*restrictions)\n" + " pstats.Stats(f.name).sort_stats(sort).print_stats(*restrictions)" ] }, { @@ -1477,7 +1481,7 @@ } ], "source": [ - "profile('zc.oindex[ix_dense_bool]')" + "profile(\"zc.oindex[ix_dense_bool]\")" ] }, { @@ -1517,7 +1521,7 @@ } ], "source": [ - "profile('zc.vindex[ix_dense_bool]')" + "profile(\"zc.vindex[ix_dense_bool]\")" ] }, { @@ -1551,7 +1555,7 @@ } ], "source": [ - "ix_dense_int = np.random.choice(c.shape[0], size=c.shape[0]//10, replace=True)\n", + "ix_dense_int = np.random.choice(c.shape[0], size=c.shape[0] // 10, replace=True)\n", "ix_dense_int_sorted = ix_dense_int.copy()\n", "ix_dense_int_sorted.sort()\n", "len(ix_dense_int)" @@ -1689,7 +1693,7 @@ } ], "source": [ - "profile('zc.oindex[ix_dense_int_sorted]')" + "profile(\"zc.oindex[ix_dense_int_sorted]\")" ] }, { @@ -1722,7 +1726,7 @@ } ], "source": [ - "profile('zc.vindex[ix_dense_int_sorted]')" + "profile(\"zc.vindex[ix_dense_int_sorted]\")" ] }, { @@ -1755,7 +1759,7 @@ } ], "source": [ - "profile('zc.oindex[ix_dense_int]')" + "profile(\"zc.oindex[ix_dense_int]\")" ] }, { @@ -1788,7 +1792,7 @@ } ], "source": [ - "profile('zc.vindex[ix_dense_int]')" + "profile(\"zc.vindex[ix_dense_int]\")" ] }, { @@ -1908,7 +1912,7 @@ } ], "source": [ - "profile('zc.oindex[ix_sparse_bool]')" + "profile(\"zc.oindex[ix_sparse_bool]\")" ] }, { @@ -1941,7 +1945,7 @@ } ], "source": [ - "profile('zc.vindex[ix_sparse_bool]')" + "profile(\"zc.vindex[ix_sparse_bool]\")" ] }, { @@ -1968,7 +1972,7 @@ } ], "source": [ - "ix_sparse_int = np.random.choice(c.shape[0], size=c.shape[0]//10000, replace=True)\n", + "ix_sparse_int = np.random.choice(c.shape[0], size=c.shape[0] // 10000, replace=True)\n", "ix_sparse_int_sorted = ix_sparse_int.copy()\n", "ix_sparse_int_sorted.sort()\n", "len(ix_sparse_int)" @@ -2106,7 +2110,7 @@ } ], "source": [ - "profile('zc.oindex[ix_sparse_int]')" + "profile(\"zc.oindex[ix_sparse_int]\")" ] }, { @@ -2139,7 +2143,7 @@ } ], "source": [ - "profile('zc.vindex[ix_sparse_int]')" + "profile(\"zc.vindex[ix_sparse_int]\")" ] }, { @@ -2330,7 +2334,7 @@ } ], "source": [ - "profile('zc[::2]')" + "profile(\"zc[::2]\")" ] }, { @@ -2480,8 +2484,8 @@ "metadata": {}, "outputs": [], "source": [ - "ix0 = np.random.choice(d.shape[0], size=int(d.shape[0] * .5), replace=True)\n", - "ix1 = np.random.choice(d.shape[1], size=int(d.shape[1] * .5), replace=True)" + "ix0 = np.random.choice(d.shape[0], size=int(d.shape[0] * 0.5), replace=True)\n", + "ix1 = np.random.choice(d.shape[1], size=int(d.shape[1] * 0.5), replace=True)" ] }, { @@ -2542,7 +2546,7 @@ } ], "source": [ - "n = int(d.size * .1)\n", + "n = int(d.size * 0.1)\n", "ix0 = np.random.choice(d.shape[0], size=n, replace=True)\n", "ix1 = np.random.choice(d.shape[1], size=n, replace=True)\n", "n" @@ -2612,7 +2616,7 @@ } ], "source": [ - "profile('zd.vindex[ix0, ix1]')" + "profile(\"zd.vindex[ix0, ix1]\")" ] }, { @@ -2637,8 +2641,9 @@ "metadata": {}, "outputs": [], "source": [ - "import h5py\n", - "import tempfile" + "import tempfile\n", + "\n", + "import h5py" ] }, { @@ -2647,7 +2652,7 @@ "metadata": {}, "outputs": [], "source": [ - "h5f = h5py.File(tempfile.mktemp(), driver='core', backing_store=False)" + "h5f = h5py.File(tempfile.mktemp(), driver=\"core\", backing_store=False)" ] }, { @@ -2667,7 +2672,9 @@ } ], "source": [ - "hc = h5f.create_dataset('c', data=c, compression='gzip', compression_opts=1, chunks=zc.chunks, shuffle=True)\n", + "hc = h5f.create_dataset(\n", + " \"c\", data=c, compression=\"gzip\", compression_opts=1, chunks=zc.chunks, shuffle=True\n", + ")\n", "hc" ] }, @@ -2733,7 +2740,7 @@ "metadata": {}, "outputs": [], "source": [ - "# # this is pathological, takes minutes \n", + "# # this is pathological, takes minutes\n", "# %time hc[ix_dense_bool]" ] }, diff --git a/notebooks/blosc_microbench.ipynb b/notebooks/blosc_microbench.ipynb index 9361d8e95..a9c5571d4 100644 --- a/notebooks/blosc_microbench.ipynb +++ b/notebooks/blosc_microbench.ipynb @@ -21,6 +21,7 @@ "source": [ "import numpy as np\n", "import zarr\n", + "\n", "zarr.__version__" ] }, @@ -45,8 +46,8 @@ } ], "source": [ - "z = zarr.empty(shape=100000000, chunks=200000, dtype='i8')\n", - "data = np.arange(100000000, dtype='i8')\n", + "z = zarr.empty(shape=100000000, chunks=200000, dtype=\"i8\")\n", + "data = np.arange(100000000, dtype=\"i8\")\n", "%timeit z[:] = data\n", "%timeit z[:]\n", "print(z)\n", @@ -74,7 +75,7 @@ } ], "source": [ - "z = zarr.empty(shape=100000000, chunks=200000, dtype='f8')\n", + "z = zarr.empty(shape=100000000, chunks=200000, dtype=\"f8\")\n", "data = np.random.normal(size=100000000)\n", "%timeit z[:] = data\n", "%timeit z[:]\n", @@ -101,10 +102,13 @@ } ], "source": [ - "import numpy as np\n", "import sys\n", - "sys.path.insert(0, '..')\n", + "\n", + "import numpy as np\n", + "\n", + "sys.path.insert(0, \"..\")\n", "import zarr\n", + "\n", "zarr.__version__" ] }, @@ -129,8 +133,8 @@ } ], "source": [ - "z = zarr.empty(shape=100000000, chunks=200000, dtype='i8')\n", - "data = np.arange(100000000, dtype='i8')\n", + "z = zarr.empty(shape=100000000, chunks=200000, dtype=\"i8\")\n", + "data = np.arange(100000000, dtype=\"i8\")\n", "%timeit z[:] = data\n", "%timeit z[:]\n", "print(z)\n", @@ -158,7 +162,7 @@ } ], "source": [ - "z = zarr.empty(shape=100000000, chunks=200000, dtype='f8')\n", + "z = zarr.empty(shape=100000000, chunks=200000, dtype=\"f8\")\n", "data = np.random.normal(size=100000000)\n", "%timeit z[:] = data\n", "%timeit z[:]\n", diff --git a/notebooks/dask_2d_subset.ipynb b/notebooks/dask_2d_subset.ipynb index 6e88b510d..b9b1c1822 100644 --- a/notebooks/dask_2d_subset.ipynb +++ b/notebooks/dask_2d_subset.ipynb @@ -24,8 +24,12 @@ } ], "source": [ - "import zarr; print('zarr', zarr.__version__)\n", - "import dask; print('dask', dask.__version__)\n", + "import zarr\n", + "\n", + "print(\"zarr\", zarr.__version__)\n", + "import dask\n", + "\n", + "print(\"dask\", dask.__version__)\n", "import dask.array as da\n", "import numpy as np" ] @@ -60,8 +64,10 @@ ], "source": [ "# here's the real data\n", - "callset = zarr.open_group('/kwiat/2/coluzzi/ag1000g/data/phase1/release/AR3.1/variation/main/zarr2/zstd/ag1000g.phase1.ar3',\n", - " mode='r')\n", + "callset = zarr.open_group(\n", + " \"/kwiat/2/coluzzi/ag1000g/data/phase1/release/AR3.1/variation/main/zarr2/zstd/ag1000g.phase1.ar3\",\n", + " mode=\"r\",\n", + ")\n", "callset" ] }, @@ -88,7 +94,7 @@ ], "source": [ "# here's the array we're going to work with\n", - "g = callset['3R/calldata/genotype']\n", + "g = callset[\"3R/calldata/genotype\"]\n", "g" ] }, @@ -144,7 +150,7 @@ ], "source": [ "# load condition used to make selection on first axis\n", - "dim0_condition = callset['3R/variants/FILTER_PASS'][:]\n", + "dim0_condition = callset[\"3R/variants/FILTER_PASS\"][:]\n", "dim0_condition.shape, dim0_condition.dtype, np.count_nonzero(dim0_condition)" ] }, @@ -367,7 +373,8 @@ "source": [ "# what's taking so long?\n", "import cProfile\n", - "cProfile.run('gd[dim0_condition][:, dim1_indices]', sort='time')" + "\n", + "cProfile.run(\"gd[dim0_condition][:, dim1_indices]\", sort=\"time\")" ] }, { @@ -465,7 +472,7 @@ } ], "source": [ - "cProfile.run('gd[dim0_condition][:, dim1_indices]', sort='cumtime')" + "cProfile.run(\"gd[dim0_condition][:, dim1_indices]\", sort=\"cumtime\")" ] }, { @@ -498,8 +505,11 @@ ], "source": [ "# create a synthetic dataset for profiling\n", - "a = zarr.array(np.random.randint(-1, 4, size=(20000000, 200, 2), dtype='i1'),\n", - " chunks=(10000, 100, 2), compressor=zarr.Blosc(cname='zstd', clevel=1, shuffle=2))\n", + "a = zarr.array(\n", + " np.random.randint(-1, 4, size=(20000000, 200, 2), dtype=\"i1\"),\n", + " chunks=(10000, 100, 2),\n", + " compressor=zarr.Blosc(cname=\"zstd\", clevel=1, shuffle=2),\n", + ")\n", "a" ] }, @@ -673,7 +683,7 @@ } ], "source": [ - "cProfile.run('d[c][:, s]', sort='time')" + "cProfile.run(\"d[c][:, s]\", sort=\"time\")" ] }, { @@ -841,7 +851,7 @@ ], "source": [ "# problem is in fact just the dim0 selection\n", - "cProfile.run('d[c]', sort='time')" + "cProfile.run(\"d[c]\", sort=\"time\")" ] } ], diff --git a/notebooks/dask_copy.ipynb b/notebooks/dask_copy.ipynb index ba4391737..ca28315f5 100644 --- a/notebooks/dask_copy.ipynb +++ b/notebooks/dask_copy.ipynb @@ -134,24 +134,29 @@ ], "source": [ "import sys\n", - "sys.path.insert(0, '..')\n", + "\n", + "sys.path.insert(0, \"..\")\n", "import zarr\n", - "print('zarr', zarr.__version__)\n", - "from zarr import blosc\n", - "import numpy as np\n", - "import h5py\n", + "\n", + "print(\"zarr\", zarr.__version__)\n", "import bcolz\n", - "# don't let bcolz use multiple threads internally, we want to \n", + "import h5py\n", + "import numpy as np\n", + "from zarr import blosc\n", + "\n", + "# don't let bcolz use multiple threads internally, we want to\n", "# see whether dask can make good use of multiple CPUs\n", "bcolz.set_nthreads(1)\n", "import multiprocessing\n", + "\n", + "import bokeh\n", "import dask\n", "import dask.array as da\n", - "from dask.diagnostics import Profiler, ResourceProfiler, CacheProfiler\n", - "from dask.diagnostics.profile_visualize import visualize\n", - "from cachey import nbytes\n", - "import bokeh\n", "from bokeh.io import output_notebook\n", + "from cachey import nbytes\n", + "from dask.diagnostics import CacheProfiler, Profiler, ResourceProfiler\n", + "from dask.diagnostics.profile_visualize import visualize\n", + "\n", "output_notebook()" ] }, @@ -163,9 +168,10 @@ }, "outputs": [], "source": [ - "import tempfile\n", "import operator\n", + "import tempfile\n", "from functools import reduce\n", + "\n", "from zarr.util import human_readable_size\n", "\n", "\n", @@ -176,9 +182,9 @@ " fn = tempfile.mktemp()\n", "\n", " # file creation args\n", - " kwargs['mode'] = 'w'\n", - " kwargs['driver'] = 'core'\n", - " kwargs['backing_store'] = False\n", + " kwargs[\"mode\"] = \"w\"\n", + " kwargs[\"driver\"] = \"core\"\n", + " kwargs[\"backing_store\"] = False\n", "\n", " # open HDF5 file\n", " h5f = h5py.File(fn, **kwargs)\n", @@ -188,7 +194,7 @@ "\n", "def h5d_diagnostics(d):\n", " \"\"\"Print some diagnostics on an HDF5 dataset.\"\"\"\n", - " \n", + "\n", " print(d)\n", " nbytes = reduce(operator.mul, d.shape) * d.dtype.itemsize\n", " cbytes = d._id.get_storage_size()\n", @@ -196,15 +202,14 @@ " ratio = nbytes / cbytes\n", " else:\n", " ratio = np.inf\n", - " r = ' compression: %s' % d.compression\n", - " r += '; compression_opts: %s' % d.compression_opts\n", - " r += '; shuffle: %s' % d.shuffle\n", - " r += '\\n nbytes: %s' % human_readable_size(nbytes)\n", - " r += '; nbytes_stored: %s' % human_readable_size(cbytes)\n", - " r += '; ratio: %.1f' % ratio\n", - " r += '; chunks: %s' % str(d.chunks)\n", - " print(r)\n", - " " + " r = \" compression: %s\" % d.compression\n", + " r += \"; compression_opts: %s\" % d.compression_opts\n", + " r += \"; shuffle: %s\" % d.shuffle\n", + " r += \"\\n nbytes: %s\" % human_readable_size(nbytes)\n", + " r += \"; nbytes_stored: %s\" % human_readable_size(cbytes)\n", + " r += \"; ratio: %.1f\" % ratio\n", + " r += \"; chunks: %s\" % str(d.chunks)\n", + " print(r)" ] }, { @@ -219,8 +224,7 @@ " dsrc = da.from_array(src, chunks=chunks)\n", " with Profiler() as prof, ResourceProfiler(dt=dt) as rprof:\n", " da.store(dsrc, dst, num_workers=num_workers, lock=lock)\n", - " visualize([prof, rprof], min_border_top=60, min_border_bottom=60)\n", - " " + " visualize([prof, rprof], min_border_top=60, min_border_bottom=60)" ] }, { @@ -250,7 +254,7 @@ ], "source": [ "# a1 = np.arange(400000000, dtype='i4')\n", - "a1 = np.random.normal(2000, 1000, size=200000000).astype('u2')\n", + "a1 = np.random.normal(2000, 1000, size=200000000).astype(\"u2\")\n", "a1" ] }, @@ -295,7 +299,7 @@ }, "outputs": [], "source": [ - "chunks = 2**20, # 4M" + "chunks = (2**20,) # 4M" ] }, { @@ -425,7 +429,7 @@ } ], "source": [ - "profile_dask_copy(a1, a2, chunks, lock=True, dt=.01)" + "profile_dask_copy(a1, a2, chunks, lock=True, dt=0.01)" ] }, { @@ -535,7 +539,7 @@ } ], "source": [ - "profile_dask_copy(a1, a2, chunks, lock=False, dt=.01)" + "profile_dask_copy(a1, a2, chunks, lock=False, dt=0.01)" ] }, { @@ -567,8 +571,9 @@ } ], "source": [ - "z1 = zarr.array(a1, chunks=chunks, compression='blosc', \n", - " compression_opts=dict(cname='lz4', clevel=1, shuffle=2))\n", + "z1 = zarr.array(\n", + " a1, chunks=chunks, compression=\"blosc\", compression_opts=dict(cname=\"lz4\", clevel=1, shuffle=2)\n", + ")\n", "z1" ] }, @@ -705,7 +710,7 @@ } ], "source": [ - "profile_dask_copy(z1, z2, chunks, lock=True, dt=.02)" + "profile_dask_copy(z1, z2, chunks, lock=True, dt=0.02)" ] }, { @@ -912,7 +917,7 @@ } ], "source": [ - "h1 = h5f.create_dataset('h1', data=a1, chunks=chunks, compression='lzf', shuffle=True)\n", + "h1 = h5f.create_dataset(\"h1\", data=a1, chunks=chunks, compression=\"lzf\", shuffle=True)\n", "h5d_diagnostics(h1)" ] }, @@ -934,9 +939,14 @@ } ], "source": [ - "h2 = h5f.create_dataset('h2', shape=h1.shape, chunks=h1.chunks, \n", - " compression=h1.compression, compression_opts=h1.compression_opts, \n", - " shuffle=h1.shuffle)\n", + "h2 = h5f.create_dataset(\n", + " \"h2\",\n", + " shape=h1.shape,\n", + " chunks=h1.chunks,\n", + " compression=h1.compression,\n", + " compression_opts=h1.compression_opts,\n", + " shuffle=h1.shuffle,\n", + ")\n", "h5d_diagnostics(h2)" ] }, @@ -1190,8 +1200,7 @@ } ], "source": [ - "c1 = bcolz.carray(a1, chunklen=chunks[0],\n", - " cparams=bcolz.cparams(cname='lz4', clevel=1, shuffle=2))\n", + "c1 = bcolz.carray(a1, chunklen=chunks[0], cparams=bcolz.cparams(cname=\"lz4\", clevel=1, shuffle=2))\n", "c1" ] }, @@ -1218,8 +1227,12 @@ } ], "source": [ - "c2 = bcolz.zeros(a1.shape, chunklen=chunks[0], dtype=a1.dtype, \n", - " cparams=bcolz.cparams(cname='lz4', clevel=1, shuffle=2))\n", + "c2 = bcolz.zeros(\n", + " a1.shape,\n", + " chunklen=chunks[0],\n", + " dtype=a1.dtype,\n", + " cparams=bcolz.cparams(cname=\"lz4\", clevel=1, shuffle=2),\n", + ")\n", "c2" ] }, diff --git a/notebooks/dask_count_alleles.ipynb b/notebooks/dask_count_alleles.ipynb index 8b9b7cec6..66c93b771 100644 --- a/notebooks/dask_count_alleles.ipynb +++ b/notebooks/dask_count_alleles.ipynb @@ -127,23 +127,28 @@ ], "source": [ "import sys\n", - "sys.path.insert(0, '..')\n", + "\n", + "sys.path.insert(0, \"..\")\n", "import zarr\n", - "print('zarr', zarr.__version__)\n", - "from zarr import blosc\n", - "import numpy as np\n", - "import h5py\n", + "\n", + "print(\"zarr\", zarr.__version__)\n", "import multiprocessing\n", + "\n", + "import bokeh\n", "import dask\n", "import dask.array as da\n", - "from dask.diagnostics import Profiler, ResourceProfiler, CacheProfiler\n", - "from dask.diagnostics.profile_visualize import visualize\n", - "from cachey import nbytes\n", - "import bokeh\n", + "import h5py\n", + "import numpy as np\n", "from bokeh.io import output_notebook\n", + "from cachey import nbytes\n", + "from dask.diagnostics import CacheProfiler, Profiler, ResourceProfiler\n", + "from dask.diagnostics.profile_visualize import visualize\n", + "from zarr import blosc\n", + "\n", "output_notebook()\n", - "from functools import reduce\n", "import operator\n", + "from functools import reduce\n", + "\n", "import allel" ] }, @@ -166,9 +171,11 @@ } ], "source": [ - "callset = h5py.File('/data/coluzzi/ag1000g/data/phase1/release/AR3/variation/main/hdf5/ag1000g.phase1.ar3.pass.h5',\n", - " mode='r')\n", - "genotype = callset['3R/calldata/genotype']\n", + "callset = h5py.File(\n", + " \"/data/coluzzi/ag1000g/data/phase1/release/AR3/variation/main/hdf5/ag1000g.phase1.ar3.pass.h5\",\n", + " mode=\"r\",\n", + ")\n", + "genotype = callset[\"3R/calldata/genotype\"]\n", "genotype" ] }, @@ -197,8 +204,12 @@ "# copy into a zarr array\n", "# N.B., chunks in HDF5 are too small really, use something bigger\n", "chunks = (genotype.chunks[0], genotype.chunks[1] * 20, genotype.chunks[2])\n", - "genotype_zarr = zarr.array(genotype, chunks=chunks, compression='blosc',\n", - " compression_opts=dict(cname='lz4', clevel=1, shuffle=2))\n", + "genotype_zarr = zarr.array(\n", + " genotype,\n", + " chunks=chunks,\n", + " compression=\"blosc\",\n", + " compression_opts=dict(cname=\"lz4\", clevel=1, shuffle=2),\n", + ")\n", "genotype_zarr" ] }, diff --git a/notebooks/genotype_benchmark_compressors.ipynb b/notebooks/genotype_benchmark_compressors.ipynb index b262e63fa..a9a0f9b94 100644 --- a/notebooks/genotype_benchmark_compressors.ipynb +++ b/notebooks/genotype_benchmark_compressors.ipynb @@ -18,15 +18,20 @@ ], "source": [ "import sys\n", - "sys.path.insert(0, '..')\n", + "\n", + "sys.path.insert(0, \"..\")\n", "import functools\n", "import timeit\n", + "\n", "import zarr\n", - "print('zarr', zarr.__version__)\n", + "\n", + "print(\"zarr\", zarr.__version__)\n", "from zarr import blosc\n", - "print('blosc', blosc.version())\n", - "import numpy as np\n", + "\n", + "print(\"blosc\", blosc.version())\n", "import h5py\n", + "import numpy as np\n", + "\n", "%matplotlib inline\n", "import matplotlib.pyplot as plt" ] @@ -50,9 +55,11 @@ } ], "source": [ - "callset = h5py.File('/data/coluzzi/ag1000g/data/phase1/release/AR3/variation/main/hdf5/ag1000g.phase1.ar3.pass.h5',\n", - " mode='r')\n", - "genotype = callset['3R/calldata/genotype']\n", + "callset = h5py.File(\n", + " \"/data/coluzzi/ag1000g/data/phase1/release/AR3/variation/main/hdf5/ag1000g.phase1.ar3.pass.h5\",\n", + " mode=\"r\",\n", + ")\n", + "genotype = callset[\"3R/calldata/genotype\"]\n", "genotype" ] }, @@ -86,7 +93,7 @@ } ], "source": [ - "genotype_sample = genotype[1000000:1000000+n_variants, ...]\n", + "genotype_sample = genotype[1000000 : 1000000 + n_variants, ...]\n", "genotype_sample.shape" ] }, @@ -133,9 +140,11 @@ ], "source": [ "# 1M chunks of first dimension\n", - "chunks = (int(2**20 / (genotype_sample.shape[1] * genotype_sample.shape[2])), \n", - " genotype_sample.shape[1], \n", - " genotype_sample.shape[2])\n", + "chunks = (\n", + " int(2**20 / (genotype_sample.shape[1] * genotype_sample.shape[2])),\n", + " genotype_sample.shape[1],\n", + " genotype_sample.shape[2],\n", + ")\n", "chunks" ] }, @@ -183,7 +192,7 @@ } ], "source": [ - "zarr.array(genotype_sample, chunks=chunks, compression_opts=dict(cname='lz4', clevel=1, shuffle=2))" + "zarr.array(genotype_sample, chunks=chunks, compression_opts=dict(cname=\"lz4\", clevel=1, shuffle=2))" ] }, { @@ -208,7 +217,7 @@ } ], "source": [ - "zarr.array(genotype_sample, chunks=chunks, compression_opts=dict(cname='zstd', clevel=1, shuffle=2))" + "zarr.array(genotype_sample, chunks=chunks, compression_opts=dict(cname=\"zstd\", clevel=1, shuffle=2))" ] }, { @@ -221,41 +230,41 @@ "source": [ "compression_configs = (\n", " (None, None),\n", - " ('zlib', 1),\n", - " ('bz2', 1),\n", - " ('lzma', dict(preset=1)),\n", - " ('blosc', dict(cname='snappy', clevel=0, shuffle=0)),\n", - " ('blosc', dict(cname='snappy', clevel=0, shuffle=2)),\n", - " ('blosc', dict(cname='snappy', clevel=9, shuffle=0)),\n", - " ('blosc', dict(cname='snappy', clevel=9, shuffle=2)),\n", - " ('blosc', dict(cname='blosclz', clevel=1, shuffle=0)),\n", - " ('blosc', dict(cname='blosclz', clevel=1, shuffle=2)),\n", - " ('blosc', dict(cname='blosclz', clevel=5, shuffle=0)),\n", - " ('blosc', dict(cname='blosclz', clevel=5, shuffle=2)),\n", - " ('blosc', dict(cname='blosclz', clevel=9, shuffle=0)),\n", - " ('blosc', dict(cname='blosclz', clevel=9, shuffle=2)),\n", - " ('blosc', dict(cname='lz4', clevel=1, shuffle=0)),\n", - " ('blosc', dict(cname='lz4', clevel=1, shuffle=2)),\n", - " ('blosc', dict(cname='lz4', clevel=5, shuffle=0)),\n", - " ('blosc', dict(cname='lz4', clevel=5, shuffle=2)),\n", - " ('blosc', dict(cname='lz4', clevel=9, shuffle=0)),\n", - " ('blosc', dict(cname='lz4', clevel=9, shuffle=2)),\n", - " ('blosc', dict(cname='lz4hc', clevel=1, shuffle=0)),\n", - " ('blosc', dict(cname='lz4hc', clevel=1, shuffle=2)),\n", - " ('blosc', dict(cname='lz4hc', clevel=3, shuffle=0)),\n", - " ('blosc', dict(cname='lz4hc', clevel=3, shuffle=2)),\n", - " ('blosc', dict(cname='zstd', clevel=1, shuffle=0)),\n", - " ('blosc', dict(cname='zstd', clevel=1, shuffle=2)),\n", - " ('blosc', dict(cname='zstd', clevel=3, shuffle=0)),\n", - " ('blosc', dict(cname='zstd', clevel=3, shuffle=2)),\n", - " ('blosc', dict(cname='zstd', clevel=5, shuffle=0)),\n", - " ('blosc', dict(cname='zstd', clevel=5, shuffle=2)),\n", - " ('blosc', dict(cname='zlib', clevel=1, shuffle=0)),\n", - " ('blosc', dict(cname='zlib', clevel=1, shuffle=2)),\n", - " ('blosc', dict(cname='zlib', clevel=3, shuffle=0)),\n", - " ('blosc', dict(cname='zlib', clevel=3, shuffle=2)),\n", - " ('blosc', dict(cname='zlib', clevel=5, shuffle=0)),\n", - " ('blosc', dict(cname='zlib', clevel=5, shuffle=2)),\n", + " (\"zlib\", 1),\n", + " (\"bz2\", 1),\n", + " (\"lzma\", dict(preset=1)),\n", + " (\"blosc\", dict(cname=\"snappy\", clevel=0, shuffle=0)),\n", + " (\"blosc\", dict(cname=\"snappy\", clevel=0, shuffle=2)),\n", + " (\"blosc\", dict(cname=\"snappy\", clevel=9, shuffle=0)),\n", + " (\"blosc\", dict(cname=\"snappy\", clevel=9, shuffle=2)),\n", + " (\"blosc\", dict(cname=\"blosclz\", clevel=1, shuffle=0)),\n", + " (\"blosc\", dict(cname=\"blosclz\", clevel=1, shuffle=2)),\n", + " (\"blosc\", dict(cname=\"blosclz\", clevel=5, shuffle=0)),\n", + " (\"blosc\", dict(cname=\"blosclz\", clevel=5, shuffle=2)),\n", + " (\"blosc\", dict(cname=\"blosclz\", clevel=9, shuffle=0)),\n", + " (\"blosc\", dict(cname=\"blosclz\", clevel=9, shuffle=2)),\n", + " (\"blosc\", dict(cname=\"lz4\", clevel=1, shuffle=0)),\n", + " (\"blosc\", dict(cname=\"lz4\", clevel=1, shuffle=2)),\n", + " (\"blosc\", dict(cname=\"lz4\", clevel=5, shuffle=0)),\n", + " (\"blosc\", dict(cname=\"lz4\", clevel=5, shuffle=2)),\n", + " (\"blosc\", dict(cname=\"lz4\", clevel=9, shuffle=0)),\n", + " (\"blosc\", dict(cname=\"lz4\", clevel=9, shuffle=2)),\n", + " (\"blosc\", dict(cname=\"lz4hc\", clevel=1, shuffle=0)),\n", + " (\"blosc\", dict(cname=\"lz4hc\", clevel=1, shuffle=2)),\n", + " (\"blosc\", dict(cname=\"lz4hc\", clevel=3, shuffle=0)),\n", + " (\"blosc\", dict(cname=\"lz4hc\", clevel=3, shuffle=2)),\n", + " (\"blosc\", dict(cname=\"zstd\", clevel=1, shuffle=0)),\n", + " (\"blosc\", dict(cname=\"zstd\", clevel=1, shuffle=2)),\n", + " (\"blosc\", dict(cname=\"zstd\", clevel=3, shuffle=0)),\n", + " (\"blosc\", dict(cname=\"zstd\", clevel=3, shuffle=2)),\n", + " (\"blosc\", dict(cname=\"zstd\", clevel=5, shuffle=0)),\n", + " (\"blosc\", dict(cname=\"zstd\", clevel=5, shuffle=2)),\n", + " (\"blosc\", dict(cname=\"zlib\", clevel=1, shuffle=0)),\n", + " (\"blosc\", dict(cname=\"zlib\", clevel=1, shuffle=2)),\n", + " (\"blosc\", dict(cname=\"zlib\", clevel=3, shuffle=0)),\n", + " (\"blosc\", dict(cname=\"zlib\", clevel=3, shuffle=2)),\n", + " (\"blosc\", dict(cname=\"zlib\", clevel=5, shuffle=0)),\n", + " (\"blosc\", dict(cname=\"zlib\", clevel=5, shuffle=2)),\n", ")" ] }, @@ -284,12 +293,16 @@ "def compression_ratios():\n", " x = list()\n", " for compression, compression_opts in compression_configs:\n", - " z = zarr.array(genotype_sample, chunks=chunks, compression=compression, \n", - " compression_opts=compression_opts)\n", + " z = zarr.array(\n", + " genotype_sample,\n", + " chunks=chunks,\n", + " compression=compression,\n", + " compression_opts=compression_opts,\n", + " )\n", " ratio = z.nbytes / z.nbytes_stored\n", " x.append(ratio)\n", " log(compression, compression_opts, ratio)\n", - " return x\n" + " return x" ] }, { @@ -311,32 +324,31 @@ } ], "source": [ - "ratios = compression_ratios() \n", - "labels = ['%s - %s' % (c, o)\n", - " for c, o in compression_configs]\n", + "ratios = compression_ratios()\n", + "labels = [\"%s - %s\" % (c, o) for c, o in compression_configs]\n", "\n", - "fig = plt.figure(figsize=(12, len(compression_configs)*.3))\n", - "fig.suptitle('Compression ratio', fontsize=14, y=1.01)\n", + "fig = plt.figure(figsize=(12, len(compression_configs) * 0.3))\n", + "fig.suptitle(\"Compression ratio\", fontsize=14, y=1.01)\n", "ax = fig.add_subplot(1, 1, 1)\n", "\n", - "y = [i for i, (c, o) in enumerate(compression_configs) if c == 'blosc' and o['shuffle'] == 2]\n", + "y = [i for i, (c, o) in enumerate(compression_configs) if c == \"blosc\" and o[\"shuffle\"] == 2]\n", "x = [ratios[i] for i in y]\n", - "ax.barh(bottom=np.array(y)+.2, width=np.array(x), height=.6, label='bit shuffle', color='b')\n", + "ax.barh(bottom=np.array(y) + 0.2, width=np.array(x), height=0.6, label=\"bit shuffle\", color=\"b\")\n", "\n", - "y = [i for i, (c, o) in enumerate(compression_configs) if c != 'blosc' or o['shuffle'] == 0]\n", + "y = [i for i, (c, o) in enumerate(compression_configs) if c != \"blosc\" or o[\"shuffle\"] == 0]\n", "x = [ratios[i] for i in y]\n", - "ax.barh(bottom=np.array(y)+.2, width=np.array(x), height=.6, label='no shuffle', color='g')\n", + "ax.barh(bottom=np.array(y) + 0.2, width=np.array(x), height=0.6, label=\"no shuffle\", color=\"g\")\n", "\n", - "ax.set_yticks(np.arange(len(labels))+.5)\n", + "ax.set_yticks(np.arange(len(labels)) + 0.5)\n", "ax.set_yticklabels(labels, rotation=0)\n", "\n", - "ax.set_xlim(0, max(ratios)+3)\n", + "ax.set_xlim(0, max(ratios) + 3)\n", "ax.set_ylim(0, len(ratios))\n", - "ax.set_xlabel('compression ratio')\n", - "ax.grid(axis='x')\n", - "ax.legend(loc='upper right')\n", + "ax.set_xlabel(\"compression ratio\")\n", + "ax.grid(axis=\"x\")\n", + "ax.legend(loc=\"upper right\")\n", "\n", - "fig.tight_layout();\n" + "fig.tight_layout();" ] }, { @@ -352,29 +364,36 @@ " c = list()\n", " d = list()\n", " for compression, compression_opts in compression_configs:\n", - " \n", + "\n", " def compress():\n", - " zarr.array(genotype_sample, chunks=chunks, compression=compression, \n", - " compression_opts=compression_opts)\n", - " \n", + " zarr.array(\n", + " genotype_sample,\n", + " chunks=chunks,\n", + " compression=compression,\n", + " compression_opts=compression_opts,\n", + " )\n", + "\n", " t = timeit.Timer(stmt=compress, globals=locals())\n", " compress_times = t.repeat(repeat=repeat, number=number)\n", " c.append(compress_times)\n", - " \n", - " z = zarr.array(genotype_sample, chunks=chunks, compression=compression, \n", - " compression_opts=compression_opts)\n", - " \n", + "\n", + " z = zarr.array(\n", + " genotype_sample,\n", + " chunks=chunks,\n", + " compression=compression,\n", + " compression_opts=compression_opts,\n", + " )\n", + "\n", " def decompress():\n", " z[:]\n", - " \n", + "\n", " t = timeit.Timer(stmt=decompress, globals=locals())\n", " decompress_times = t.repeat(repeat=repeat, number=number)\n", " d.append(decompress_times)\n", - " \n", + "\n", " log(compression, compression_opts, compress_times, decompress_times)\n", - " \n", - " return c, d\n", - " " + "\n", + " return c, d" ] }, { @@ -398,29 +417,29 @@ "source": [ "co_t, de_t = compression_decompression_times()\n", "\n", - "fig = plt.figure(figsize=(12, len(compression_configs)*.3))\n", - "fig.suptitle('Compression speed', fontsize=14, y=1.01)\n", + "fig = plt.figure(figsize=(12, len(compression_configs) * 0.3))\n", + "fig.suptitle(\"Compression speed\", fontsize=14, y=1.01)\n", "\n", "\n", "ax = fig.add_subplot(1, 1, 1)\n", "\n", - "y = [i for i, (c, o) in enumerate(compression_configs) if c == 'blosc' and o['shuffle'] == 2]\n", + "y = [i for i, (c, o) in enumerate(compression_configs) if c == \"blosc\" and o[\"shuffle\"] == 2]\n", "x = (nbytes / 1000000) / np.array([co_t[i] for i in y])\n", - "ax.barh(bottom=np.array(y)+.2, width=x.max(axis=1), height=.6, label='bit shuffle', color='b')\n", + "ax.barh(bottom=np.array(y) + 0.2, width=x.max(axis=1), height=0.6, label=\"bit shuffle\", color=\"b\")\n", "\n", - "y = [i for i, (c, o) in enumerate(compression_configs) if c != 'blosc' or o['shuffle'] == 0]\n", + "y = [i for i, (c, o) in enumerate(compression_configs) if c != \"blosc\" or o[\"shuffle\"] == 0]\n", "x = (nbytes / 1000000) / np.array([co_t[i] for i in y])\n", - "ax.barh(bottom=np.array(y)+.2, width=x.max(axis=1), height=.6, label='no shuffle', color='g')\n", + "ax.barh(bottom=np.array(y) + 0.2, width=x.max(axis=1), height=0.6, label=\"no shuffle\", color=\"g\")\n", "\n", - "ax.set_yticks(np.arange(len(labels))+.5)\n", + "ax.set_yticks(np.arange(len(labels)) + 0.5)\n", "ax.set_yticklabels(labels, rotation=0)\n", "\n", "xlim = (0, np.max((nbytes / 1000000) / np.array(co_t)) + 100)\n", "ax.set_xlim(*xlim)\n", "ax.set_ylim(0, len(co_t))\n", - "ax.set_xlabel('speed (Mb/s)')\n", - "ax.grid(axis='x')\n", - "ax.legend(loc='upper right')\n", + "ax.set_xlabel(\"speed (Mb/s)\")\n", + "ax.grid(axis=\"x\")\n", + "ax.legend(loc=\"upper right\")\n", "\n", "fig.tight_layout();" ] @@ -446,29 +465,29 @@ "source": [ "co_t, de_t = compression_decompression_times()\n", "\n", - "fig = plt.figure(figsize=(12, len(compression_configs)*.3))\n", - "fig.suptitle('Decompression speed', fontsize=14, y=1.01)\n", + "fig = plt.figure(figsize=(12, len(compression_configs) * 0.3))\n", + "fig.suptitle(\"Decompression speed\", fontsize=14, y=1.01)\n", "\n", "\n", "ax = fig.add_subplot(1, 1, 1)\n", "\n", - "y = [i for i, (c, o) in enumerate(compression_configs) if c == 'blosc' and o['shuffle'] == 2]\n", + "y = [i for i, (c, o) in enumerate(compression_configs) if c == \"blosc\" and o[\"shuffle\"] == 2]\n", "x = (nbytes / 1000000) / np.array([de_t[i] for i in y])\n", - "ax.barh(bottom=np.array(y)+.2, width=x.max(axis=1), height=.6, label='bit shuffle', color='b')\n", + "ax.barh(bottom=np.array(y) + 0.2, width=x.max(axis=1), height=0.6, label=\"bit shuffle\", color=\"b\")\n", "\n", - "y = [i for i, (c, o) in enumerate(compression_configs) if c != 'blosc' or o['shuffle'] == 0]\n", + "y = [i for i, (c, o) in enumerate(compression_configs) if c != \"blosc\" or o[\"shuffle\"] == 0]\n", "x = (nbytes / 1000000) / np.array([de_t[i] for i in y])\n", - "ax.barh(bottom=np.array(y)+.2, width=x.max(axis=1), height=.6, label='no shuffle', color='g')\n", + "ax.barh(bottom=np.array(y) + 0.2, width=x.max(axis=1), height=0.6, label=\"no shuffle\", color=\"g\")\n", "\n", - "ax.set_yticks(np.arange(len(labels))+.5)\n", + "ax.set_yticks(np.arange(len(labels)) + 0.5)\n", "ax.set_yticklabels(labels, rotation=0)\n", "\n", "xlim = (0, np.max((nbytes / 1000000) / np.array(de_t)) + 100)\n", "ax.set_xlim(*xlim)\n", "ax.set_ylim(0, len(de_t))\n", - "ax.set_xlabel('speed (Mb/s)')\n", - "ax.grid(axis='x')\n", - "ax.legend(loc='upper right')\n", + "ax.set_xlabel(\"speed (Mb/s)\")\n", + "ax.grid(axis=\"x\")\n", + "ax.legend(loc=\"upper right\")\n", "\n", "fig.tight_layout();" ] diff --git a/notebooks/object_arrays.ipynb b/notebooks/object_arrays.ipynb index 714d02490..c4379bdd2 100644 --- a/notebooks/object_arrays.ipynb +++ b/notebooks/object_arrays.ipynb @@ -36,6 +36,7 @@ ], "source": [ "import zarr\n", + "\n", "zarr.__version__" ] }, @@ -57,6 +58,7 @@ ], "source": [ "import numcodecs\n", + "\n", "numcodecs.__version__" ] }, @@ -155,11 +157,11 @@ } ], "source": [ - "z[0] = 'foo'\n", - "z[1] = b'bar' # msgpack doesn't support bytes objects correctly\n", + "z[0] = \"foo\"\n", + "z[1] = b\"bar\" # msgpack doesn't support bytes objects correctly\n", "z[2] = 1\n", - "z[3] = [2, 4, 6, 'baz']\n", - "z[4] = {'a': 'b', 'c': 'd'}\n", + "z[3] = [2, 4, 6, \"baz\"]\n", + "z[4] = {\"a\": \"b\", \"c\": \"d\"}\n", "a = z[:]\n", "a" ] @@ -263,7 +265,7 @@ } ], "source": [ - "z[0] = 'foo'" + "z[0] = \"foo\"" ] }, { @@ -293,6 +295,7 @@ ], "source": [ "from numcodecs.tests.common import greetings\n", + "\n", "z = zarr.array(greetings, chunks=5, dtype=object, object_codec=numcodecs.MsgPack())\n", "z[:]" ] diff --git a/notebooks/repr_info.ipynb b/notebooks/repr_info.ipynb index 487a4175b..ab5896efb 100644 --- a/notebooks/repr_info.ipynb +++ b/notebooks/repr_info.ipynb @@ -65,7 +65,7 @@ "metadata": {}, "outputs": [], "source": [ - "z = root.zeros('foo/bar/baz', shape=1000000)" + "z = root.zeros(\"foo/bar/baz\", shape=1000000)" ] }, { @@ -330,7 +330,7 @@ } ], "source": [ - "root['foo/bar'].info" + "root[\"foo/bar\"].info" ] }, { diff --git a/notebooks/repr_tree.ipynb b/notebooks/repr_tree.ipynb index dde8bbb9a..d1bc2e01c 100644 --- a/notebooks/repr_tree.ipynb +++ b/notebooks/repr_tree.ipynb @@ -18,6 +18,7 @@ ], "source": [ "import zarr\n", + "\n", "zarr.__version__" ] }, @@ -28,15 +29,15 @@ "outputs": [], "source": [ "g1 = zarr.group()\n", - "g2 = g1.create_group('foo')\n", - "g3 = g1.create_group('bar')\n", - "g3.create_group('baz')\n", - "g3.create_dataset('xxx', shape=100)\n", - "g3.create_dataset('yyy', shape=(100, 100), dtype='i4')\n", - "g5 = g3.create_group('quux')\n", - "g5.create_dataset('aaa', shape=100)\n", - "g5.create_dataset('bbb', shape=(100, 100), dtype='i4')\n", - "g7 = g3.create_group('zoo')" + "g2 = g1.create_group(\"foo\")\n", + "g3 = g1.create_group(\"bar\")\n", + "g3.create_group(\"baz\")\n", + "g3.create_dataset(\"xxx\", shape=100)\n", + "g3.create_dataset(\"yyy\", shape=(100, 100), dtype=\"i4\")\n", + "g5 = g3.create_group(\"quux\")\n", + "g5.create_dataset(\"aaa\", shape=100)\n", + "g5.create_dataset(\"bbb\", shape=(100, 100), dtype=\"i4\")\n", + "g7 = g3.create_group(\"zoo\")" ] }, { diff --git a/notebooks/store_benchmark.ipynb b/notebooks/store_benchmark.ipynb index 869e7df60..36681df88 100644 --- a/notebooks/store_benchmark.ipynb +++ b/notebooks/store_benchmark.ipynb @@ -32,6 +32,7 @@ ], "source": [ "import zarr\n", + "\n", "zarr.__version__" ] }, @@ -53,6 +54,7 @@ ], "source": [ "import bsddb3\n", + "\n", "bsddb3.__version__" ] }, @@ -74,6 +76,7 @@ ], "source": [ "import lmdb\n", + "\n", "lmdb.__version__" ] }, @@ -104,7 +107,8 @@ "source": [ "import os\n", "import shutil\n", - "bench_dir = '../data/bench'\n", + "\n", + "bench_dir = \"../data/bench\"\n", "\n", "\n", "def clean():\n", @@ -112,20 +116,24 @@ " shutil.rmtree(bench_dir)\n", " os.makedirs(bench_dir)\n", "\n", - " \n", - "def setup(a, name='foo/bar'):\n", + "\n", + "def setup(a, name=\"foo/bar\"):\n", " global fdict_z, hdict_z, lmdb_z, gdbm_z, ndbm_z, bdbm_btree_z, bdbm_hash_z, zip_z, dir_z\n", - " \n", + "\n", " clean()\n", " fdict_root = zarr.group(store=dict())\n", " hdict_root = zarr.group(store=zarr.DictStore())\n", - " lmdb_root = zarr.group(store=zarr.LMDBStore(os.path.join(bench_dir, 'lmdb')))\n", - " gdbm_root = zarr.group(store=zarr.DBMStore(os.path.join(bench_dir, 'gdbm'), open=dbm.gnu.open))\n", - " ndbm_root = zarr.group(store=zarr.DBMStore(os.path.join(bench_dir, 'ndbm'), open=dbm.ndbm.open))\n", - " bdbm_btree_root = zarr.group(store=zarr.DBMStore(os.path.join(bench_dir, 'bdbm_btree'), open=bsddb3.btopen))\n", - " bdbm_hash_root = zarr.group(store=zarr.DBMStore(os.path.join(bench_dir, 'bdbm_hash'), open=bsddb3.hashopen))\n", - " zip_root = zarr.group(store=zarr.ZipStore(os.path.join(bench_dir, 'zip'), mode='w'))\n", - " dir_root = zarr.group(store=zarr.DirectoryStore(os.path.join(bench_dir, 'dir')))\n", + " lmdb_root = zarr.group(store=zarr.LMDBStore(os.path.join(bench_dir, \"lmdb\")))\n", + " gdbm_root = zarr.group(store=zarr.DBMStore(os.path.join(bench_dir, \"gdbm\"), open=dbm.gnu.open))\n", + " ndbm_root = zarr.group(store=zarr.DBMStore(os.path.join(bench_dir, \"ndbm\"), open=dbm.ndbm.open))\n", + " bdbm_btree_root = zarr.group(\n", + " store=zarr.DBMStore(os.path.join(bench_dir, \"bdbm_btree\"), open=bsddb3.btopen)\n", + " )\n", + " bdbm_hash_root = zarr.group(\n", + " store=zarr.DBMStore(os.path.join(bench_dir, \"bdbm_hash\"), open=bsddb3.hashopen)\n", + " )\n", + " zip_root = zarr.group(store=zarr.ZipStore(os.path.join(bench_dir, \"zip\"), mode=\"w\"))\n", + " dir_root = zarr.group(store=zarr.DirectoryStore(os.path.join(bench_dir, \"dir\")))\n", "\n", " fdict_z = fdict_root.empty_like(name, a)\n", " hdict_z = hdict_root.empty_like(name, a)\n", @@ -139,9 +147,7 @@ "\n", " # check compression ratio\n", " fdict_z[:] = a\n", - " return fdict_z.info\n", - " \n", - " " + " return fdict_z.info" ] }, { @@ -162,12 +168,12 @@ " # needed for zip benchmarks to avoid duplicate entries\n", " z.store.clear()\n", " z[:] = a\n", - " if hasattr(z.store, 'flush'):\n", + " if hasattr(z.store, \"flush\"):\n", " z.store.flush()\n", - " \n", - " \n", + "\n", + "\n", "def load(z, a):\n", - " z.get_basic_selection(out=a)\n" + " z.get_basic_selection(out=a)" ] }, { @@ -932,9 +938,8 @@ " d = da.from_array(source, chunks=chunks, asarray=False, fancy=False, lock=False)\n", " result = (d // 2) * 2\n", " da.store(result, sink, lock=False)\n", - " if hasattr(sink.store, 'flush'):\n", - " sink.store.flush()\n", - " " + " if hasattr(sink.store, \"flush\"):\n", + " sink.store.flush()" ] }, { diff --git a/notebooks/zip_benchmark.ipynb b/notebooks/zip_benchmark.ipynb index 680555242..c23c887fa 100644 --- a/notebooks/zip_benchmark.ipynb +++ b/notebooks/zip_benchmark.ipynb @@ -20,8 +20,10 @@ ], "source": [ "import sys\n", - "sys.path.insert(0, '..')\n", + "\n", + "sys.path.insert(0, \"..\")\n", "import zarr\n", + "\n", "zarr.__version__" ] }, @@ -47,10 +49,12 @@ } ], "source": [ - "store = zarr.ZipStore('/data/coluzzi/ag1000g/data/phase1/release/AR3.1/haplotypes/main/zarr2/zstd/ag1000g.phase1.ar3.1.haplotypes.zip',\n", - " mode='r')\n", + "store = zarr.ZipStore(\n", + " \"/data/coluzzi/ag1000g/data/phase1/release/AR3.1/haplotypes/main/zarr2/zstd/ag1000g.phase1.ar3.1.haplotypes.zip\",\n", + " mode=\"r\",\n", + ")\n", "grp = zarr.Group(store)\n", - "z = grp['3L/calldata/genotype']\n", + "z = grp[\"3L/calldata/genotype\"]\n", "z" ] }, @@ -135,7 +139,8 @@ ], "source": [ "import cProfile\n", - "cProfile.run('z[:10]', sort='cumtime')" + "\n", + "cProfile.run(\"z[:10]\", sort=\"cumtime\")" ] }, { @@ -159,6 +164,7 @@ "source": [ "import dask\n", "import dask.array as da\n", + "\n", "dask.__version__" ] }, @@ -244,9 +250,11 @@ ], "source": [ "# compare with same data via directory store\n", - "store_dir = zarr.DirectoryStore('/data/coluzzi/ag1000g/data/phase1/release/AR3.1/haplotypes/main/zarr2/zstd/ag1000g.phase1.ar3.1.haplotypes')\n", + "store_dir = zarr.DirectoryStore(\n", + " \"/data/coluzzi/ag1000g/data/phase1/release/AR3.1/haplotypes/main/zarr2/zstd/ag1000g.phase1.ar3.1.haplotypes\"\n", + ")\n", "grp_dir = zarr.Group(store_dir)\n", - "z_dir = grp_dir['3L/calldata/genotype']\n", + "z_dir = grp_dir[\"3L/calldata/genotype\"]\n", "z_dir" ] }, diff --git a/src/zarr/__init__.py b/src/zarr/__init__.py index a8aff30b5..1fa76e56c 100644 --- a/src/zarr/__init__.py +++ b/src/zarr/__init__.py @@ -3,6 +3,7 @@ from typing import Union import zarr.codecs # noqa: F401 +from zarr._version import version as __version__ from zarr.array import Array, AsyncArray # noqa: F401 from zarr.array_v2 import ArrayV2 from zarr.config import config # noqa: F401 @@ -12,7 +13,6 @@ make_store_path, ) from zarr.sync import sync as _sync -from zarr._version import version as __version__ # in case setuptools scm screw up and find version to be 0.0.0 assert not __version__.startswith("0.0.0") diff --git a/src/zarr/abc/codec.py b/src/zarr/abc/codec.py index 8897cced8..1fdc7c196 100644 --- a/src/zarr/abc/codec.py +++ b/src/zarr/abc/codec.py @@ -4,14 +4,14 @@ from typing import TYPE_CHECKING, Optional import numpy as np -from zarr.abc.metadata import Metadata +from zarr.abc.metadata import Metadata from zarr.common import ArraySpec from zarr.store import StorePath - if TYPE_CHECKING: from typing_extensions import Self + from zarr.common import BytesLike, SliceSelection from zarr.metadata import ArrayMetadata diff --git a/src/zarr/abc/metadata.py b/src/zarr/abc/metadata.py index f27b37cba..5f86471b9 100644 --- a/src/zarr/abc/metadata.py +++ b/src/zarr/abc/metadata.py @@ -1,11 +1,13 @@ from __future__ import annotations + from typing import TYPE_CHECKING, Sequence if TYPE_CHECKING: from typing import Dict + from typing_extensions import Self -from dataclasses import fields, dataclass +from dataclasses import dataclass, fields from zarr.common import JSON diff --git a/src/zarr/abc/store.py b/src/zarr/abc/store.py index d92f8d4e2..f3b04e41f 100644 --- a/src/zarr/abc/store.py +++ b/src/zarr/abc/store.py @@ -1,7 +1,6 @@ -from abc import abstractmethod, ABC - +from abc import ABC, abstractmethod from collections.abc import AsyncGenerator -from typing import List, Tuple, Optional +from typing import List, Optional, Tuple class Store(ABC): diff --git a/src/zarr/array.py b/src/zarr/array.py index 9f4ec911b..6faed1de3 100644 --- a/src/zarr/array.py +++ b/src/zarr/array.py @@ -9,14 +9,15 @@ from __future__ import annotations -from dataclasses import dataclass, replace - import json +from dataclasses import dataclass, replace from typing import Any, Dict, Iterable, Literal, Optional, Tuple, Union import numpy as np -from zarr.abc.codec import Codec +from zarr.abc.codec import Codec +from zarr.chunk_grids import RegularChunkGrid +from zarr.chunk_key_encodings import DefaultChunkKeyEncoding, V2ChunkKeyEncoding # from zarr.array_v2 import ArrayV2 from zarr.codecs import BytesCodec @@ -29,10 +30,7 @@ concurrent_map, ) from zarr.config import config - from zarr.indexing import BasicIndexer, all_chunk_coords, is_total_slice -from zarr.chunk_grids import RegularChunkGrid -from zarr.chunk_key_encodings import DefaultChunkKeyEncoding, V2ChunkKeyEncoding from zarr.metadata import ArrayMetadata, parse_indexing_order from zarr.store import StoreLike, StorePath, make_store_path from zarr.sync import sync diff --git a/src/zarr/array_v2.py b/src/zarr/array_v2.py index 18251e7db..24ec36355 100644 --- a/src/zarr/array_v2.py +++ b/src/zarr/array_v2.py @@ -1,13 +1,12 @@ from __future__ import annotations import asyncio -from dataclasses import dataclass, replace import json +from dataclasses import dataclass, replace from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional, Union import numcodecs import numpy as np - from numcodecs.compat import ensure_bytes, ensure_ndarray from zarr.common import ( @@ -430,11 +429,8 @@ async def convert_to_v3_async(self) -> Array: from zarr.abc.codec import Codec from zarr.array import Array - from zarr.common import ZARR_JSON from zarr.chunk_grids import RegularChunkGrid from zarr.chunk_key_encodings import V2ChunkKeyEncoding - from zarr.metadata import ArrayMetadata, DataType - from zarr.codecs import ( BloscCodec, BloscShuffle, @@ -442,6 +438,8 @@ async def convert_to_v3_async(self) -> Array: GzipCodec, TransposeCodec, ) + from zarr.common import ZARR_JSON + from zarr.metadata import ArrayMetadata, DataType data_type = DataType.from_dtype(self.metadata.dtype) endian: Literal["little", "big"] diff --git a/src/zarr/attributes.py b/src/zarr/attributes.py index 8086e18d7..c12578205 100644 --- a/src/zarr/attributes.py +++ b/src/zarr/attributes.py @@ -1,10 +1,11 @@ from __future__ import annotations + from collections.abc import MutableMapping from typing import TYPE_CHECKING, Any, Union if TYPE_CHECKING: - from zarr.group import Group from zarr.array import Array + from zarr.group import Group class Attributes(MutableMapping[str, Any]): diff --git a/src/zarr/chunk_grids.py b/src/zarr/chunk_grids.py index 73557f6e4..b27c09b2b 100644 --- a/src/zarr/chunk_grids.py +++ b/src/zarr/chunk_grids.py @@ -1,8 +1,9 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, Dict + from dataclasses import dataclass -from zarr.abc.metadata import Metadata +from typing import TYPE_CHECKING, Any, Dict +from zarr.abc.metadata import Metadata from zarr.common import ( JSON, ChunkCoords, diff --git a/src/zarr/chunk_key_encodings.py b/src/zarr/chunk_key_encodings.py index ebc7654dd..ee0a051c6 100644 --- a/src/zarr/chunk_key_encodings.py +++ b/src/zarr/chunk_key_encodings.py @@ -1,9 +1,10 @@ from __future__ import annotations + from abc import abstractmethod -from typing import TYPE_CHECKING, Dict, Literal, cast from dataclasses import dataclass -from zarr.abc.metadata import Metadata +from typing import TYPE_CHECKING, Dict, Literal, cast +from zarr.abc.metadata import Metadata from zarr.common import ( JSON, ChunkCoords, diff --git a/src/zarr/codecs/__init__.py b/src/zarr/codecs/__init__.py index 8fa0c9f7b..08b97b932 100644 --- a/src/zarr/codecs/__init__.py +++ b/src/zarr/codecs/__init__.py @@ -1,6 +1,6 @@ from __future__ import annotations -from zarr.codecs.blosc import BloscCodec, BloscCname, BloscShuffle # noqa: F401 +from zarr.codecs.blosc import BloscCname, BloscCodec, BloscShuffle # noqa: F401 from zarr.codecs.bytes import BytesCodec, Endian # noqa: F401 from zarr.codecs.crc32c_ import Crc32cCodec # noqa: F401 from zarr.codecs.gzip import GzipCodec # noqa: F401 diff --git a/src/zarr/codecs/blosc.py b/src/zarr/codecs/blosc.py index 5ee2b7640..41da37b44 100644 --- a/src/zarr/codecs/blosc.py +++ b/src/zarr/codecs/blosc.py @@ -1,8 +1,8 @@ from __future__ import annotations + from dataclasses import dataclass, replace from enum import Enum from functools import cached_property - from typing import TYPE_CHECKING, Union import numcodecs @@ -15,7 +15,9 @@ if TYPE_CHECKING: from typing import Dict, Optional + from typing_extensions import Self + from zarr.common import JSON, ArraySpec, BytesLike diff --git a/src/zarr/codecs/bytes.py b/src/zarr/codecs/bytes.py index 566b3a8df..bb8395c17 100644 --- a/src/zarr/codecs/bytes.py +++ b/src/zarr/codecs/bytes.py @@ -1,8 +1,8 @@ from __future__ import annotations + +import sys from dataclasses import dataclass, replace from enum import Enum -import sys - from typing import TYPE_CHECKING, Dict, Optional, Union import numpy as np @@ -12,9 +12,10 @@ from zarr.common import parse_enum, parse_named_configuration if TYPE_CHECKING: - from zarr.common import JSON, ArraySpec, BytesLike from typing_extensions import Self + from zarr.common import JSON, ArraySpec, BytesLike + class Endian(Enum): big = "big" diff --git a/src/zarr/codecs/crc32c_.py b/src/zarr/codecs/crc32c_.py index dd61b3425..e4abe537c 100644 --- a/src/zarr/codecs/crc32c_.py +++ b/src/zarr/codecs/crc32c_.py @@ -1,10 +1,9 @@ from __future__ import annotations -from dataclasses import dataclass +from dataclasses import dataclass from typing import TYPE_CHECKING import numpy as np - from crc32c import crc32c from zarr.abc.codec import BytesBytesCodec @@ -13,8 +12,10 @@ if TYPE_CHECKING: from typing import Dict, Optional + from typing_extensions import Self - from zarr.common import JSON, BytesLike, ArraySpec + + from zarr.common import JSON, ArraySpec, BytesLike @dataclass(frozen=True) diff --git a/src/zarr/codecs/gzip.py b/src/zarr/codecs/gzip.py index 71dcaa6bb..89f0b8730 100644 --- a/src/zarr/codecs/gzip.py +++ b/src/zarr/codecs/gzip.py @@ -1,16 +1,19 @@ from __future__ import annotations -from dataclasses import dataclass +from dataclasses import dataclass from typing import TYPE_CHECKING from numcodecs.gzip import GZip + from zarr.abc.codec import BytesBytesCodec from zarr.codecs.registry import register_codec from zarr.common import parse_named_configuration, to_thread if TYPE_CHECKING: - from typing import Optional, Dict + from typing import Dict, Optional + from typing_extensions import Self + from zarr.common import JSON, ArraySpec, BytesLike diff --git a/src/zarr/codecs/pipeline.py b/src/zarr/codecs/pipeline.py index c2338f717..53cc29c42 100644 --- a/src/zarr/codecs/pipeline.py +++ b/src/zarr/codecs/pipeline.py @@ -1,10 +1,11 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Iterable -import numpy as np from dataclasses import dataclass +from typing import TYPE_CHECKING, Iterable from warnings import warn +import numpy as np + from zarr.abc.codec import ( ArrayArrayCodec, ArrayBytesCodec, @@ -19,9 +20,10 @@ if TYPE_CHECKING: from typing import Iterator, List, Optional, Tuple, Union - from zarr.store import StorePath - from zarr.metadata import ArrayMetadata + from zarr.common import JSON, ArraySpec, BytesLike, SliceSelection + from zarr.metadata import ArrayMetadata + from zarr.store import StorePath @dataclass(frozen=True) diff --git a/src/zarr/codecs/registry.py b/src/zarr/codecs/registry.py index 7d4604125..3a432101b 100644 --- a/src/zarr/codecs/registry.py +++ b/src/zarr/codecs/registry.py @@ -1,12 +1,14 @@ from __future__ import annotations + from typing import TYPE_CHECKING if TYPE_CHECKING: from typing import Dict, Type - from zarr.abc.codec import Codec -from importlib.metadata import EntryPoint, entry_points as get_entry_points + from zarr.abc.codec import Codec +from importlib.metadata import EntryPoint +from importlib.metadata import entry_points as get_entry_points __codec_registry: Dict[str, Type[Codec]] = {} __lazy_load_codecs: Dict[str, EntryPoint] = {} diff --git a/src/zarr/codecs/sharding.py b/src/zarr/codecs/sharding.py index e94074e63..9dd7c2347 100644 --- a/src/zarr/codecs/sharding.py +++ b/src/zarr/codecs/sharding.py @@ -1,17 +1,19 @@ from __future__ import annotations -from enum import Enum -from typing import TYPE_CHECKING, Iterable, Mapping, NamedTuple, Union + from dataclasses import dataclass, replace +from enum import Enum from functools import lru_cache - +from typing import TYPE_CHECKING, Iterable, Mapping, NamedTuple, Union import numpy as np + from zarr.abc.codec import ( - Codec, ArrayBytesCodec, ArrayBytesCodecPartialDecodeMixin, ArrayBytesCodecPartialEncodeMixin, + Codec, ) +from zarr.chunk_grids import RegularChunkGrid from zarr.codecs.bytes import BytesCodec from zarr.codecs.crc32c_ import Crc32cCodec from zarr.codecs.pipeline import CodecPipeline @@ -26,7 +28,6 @@ product, ) from zarr.config import config -from zarr.chunk_grids import RegularChunkGrid from zarr.indexing import ( BasicIndexer, c_order_iter, @@ -40,15 +41,16 @@ if TYPE_CHECKING: from typing import Awaitable, Callable, Dict, Iterator, List, Optional, Set, Tuple + from typing_extensions import Self - from zarr.store import StorePath from zarr.common import ( JSON, - ChunkCoords, BytesLike, + ChunkCoords, SliceSelection, ) + from zarr.store import StorePath MAX_UINT_64 = 2**64 - 1 diff --git a/src/zarr/codecs/transpose.py b/src/zarr/codecs/transpose.py index a13708955..70d773553 100644 --- a/src/zarr/codecs/transpose.py +++ b/src/zarr/codecs/transpose.py @@ -1,12 +1,13 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Dict, Iterable, Union, cast from dataclasses import dataclass, replace +from typing import TYPE_CHECKING, Dict, Iterable, Union, cast from zarr.common import JSON, ArraySpec, ChunkCoordsLike, parse_named_configuration if TYPE_CHECKING: from typing import TYPE_CHECKING, Optional, Tuple + from typing_extensions import Self import numpy as np diff --git a/src/zarr/codecs/zstd.py b/src/zarr/codecs/zstd.py index ad10a7fdb..7c57c3092 100644 --- a/src/zarr/codecs/zstd.py +++ b/src/zarr/codecs/zstd.py @@ -1,7 +1,7 @@ from __future__ import annotations -from typing import TYPE_CHECKING -from dataclasses import dataclass +from dataclasses import dataclass +from typing import TYPE_CHECKING from zstandard import ZstdCompressor, ZstdDecompressor @@ -11,8 +11,10 @@ if TYPE_CHECKING: from typing import Dict, Optional + from typing_extensions import Self - from zarr.common import BytesLike, JSON, ArraySpec + + from zarr.common import JSON, ArraySpec, BytesLike def parse_zstd_level(data: JSON) -> int: diff --git a/src/zarr/common.py b/src/zarr/common.py index 7ef2fc9a6..6bf960493 100644 --- a/src/zarr/common.py +++ b/src/zarr/common.py @@ -1,21 +1,22 @@ from __future__ import annotations + +import asyncio +import contextvars +import functools +from dataclasses import dataclass +from enum import Enum from typing import ( TYPE_CHECKING, - Literal, - Union, - Tuple, - Iterable, + Any, Dict, + Iterable, List, + Literal, + Tuple, TypeVar, + Union, overload, - Any, ) -import asyncio -import contextvars -from dataclasses import dataclass -from enum import Enum -import functools if TYPE_CHECKING: from typing import Any, Awaitable, Callable, Iterator, Optional, Type diff --git a/src/zarr/group.py b/src/zarr/group.py index c71860b1b..16215b87f 100644 --- a/src/zarr/group.py +++ b/src/zarr/group.py @@ -1,21 +1,20 @@ from __future__ import annotations -from typing import TYPE_CHECKING -from dataclasses import asdict, dataclass, field, replace import asyncio import json import logging +from dataclasses import asdict, dataclass, field, replace +from typing import TYPE_CHECKING if TYPE_CHECKING: from typing import ( Any, AsyncGenerator, - Literal, AsyncIterator, + Literal, ) from zarr.abc.metadata import Metadata - -from zarr.array import AsyncArray, Array +from zarr.array import Array, AsyncArray from zarr.attributes import Attributes from zarr.common import ZARR_JSON, ZARRAY_JSON, ZATTRS_JSON, ZGROUP_JSON from zarr.store import StoreLike, StorePath, make_store_path diff --git a/src/zarr/metadata.py b/src/zarr/metadata.py index 3903bacd4..9f0fcfefa 100644 --- a/src/zarr/metadata.py +++ b/src/zarr/metadata.py @@ -1,23 +1,24 @@ from __future__ import annotations -from enum import Enum -from typing import TYPE_CHECKING, cast, Dict, Iterable, Any -from dataclasses import dataclass, field + import json +from dataclasses import dataclass, field +from enum import Enum +from typing import TYPE_CHECKING, Any, Dict, Iterable, cast + import numpy as np import numpy.typing as npt from zarr.chunk_grids import ChunkGrid, RegularChunkGrid from zarr.chunk_key_encodings import ChunkKeyEncoding, parse_separator - if TYPE_CHECKING: - from typing import Literal, Union, List, Optional, Tuple + from typing import List, Literal, Optional, Tuple, Union + from zarr.codecs.pipeline import CodecPipeline from zarr.abc.codec import Codec from zarr.abc.metadata import Metadata - from zarr.common import ( JSON, ArraySpec, @@ -28,7 +29,6 @@ ) from zarr.config import parse_indexing_order - # For type checking _bool = bool diff --git a/src/zarr/store/core.py b/src/zarr/store/core.py index 29506aa61..0d74f3a85 100644 --- a/src/zarr/store/core.py +++ b/src/zarr/store/core.py @@ -3,8 +3,8 @@ from pathlib import Path from typing import Any, Optional, Tuple, Union -from zarr.common import BytesLike from zarr.abc.store import Store +from zarr.common import BytesLike from zarr.store.local import LocalStore diff --git a/src/zarr/store/local.py b/src/zarr/store/local.py index dde28d521..53c67698c 100644 --- a/src/zarr/store/local.py +++ b/src/zarr/store/local.py @@ -4,7 +4,7 @@ import shutil from collections.abc import AsyncGenerator from pathlib import Path -from typing import Union, Optional, List, Tuple +from typing import List, Optional, Tuple, Union from zarr.abc.store import Store from zarr.common import BytesLike, concurrent_map, to_thread diff --git a/src/zarr/store/memory.py b/src/zarr/store/memory.py index 2a09bc2dd..152b52c2a 100644 --- a/src/zarr/store/memory.py +++ b/src/zarr/store/memory.py @@ -1,10 +1,10 @@ from __future__ import annotations from collections.abc import AsyncGenerator -from typing import Optional, MutableMapping, List, Tuple +from typing import List, MutableMapping, Optional, Tuple -from zarr.common import BytesLike, concurrent_map from zarr.abc.store import Store +from zarr.common import BytesLike, concurrent_map # TODO: this store could easily be extended to wrap any MutuableMapping store from v2 diff --git a/src/zarr/store/remote.py b/src/zarr/store/remote.py index c42cf3f56..264021889 100644 --- a/src/zarr/store/remote.py +++ b/src/zarr/store/remote.py @@ -3,13 +3,12 @@ from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, Union from zarr.abc.store import Store -from zarr.store.core import _dereference_path from zarr.common import BytesLike - +from zarr.store.core import _dereference_path if TYPE_CHECKING: - from upath import UPath from fsspec.asyn import AsyncFileSystem + from upath import UPath class RemoteStore(Store): @@ -20,8 +19,8 @@ class RemoteStore(Store): root: UPath def __init__(self, url: Union[UPath, str], **storage_options: Dict[str, Any]): - from upath import UPath import fsspec + from upath import UPath if isinstance(url, str): self.root = UPath(url, **storage_options) diff --git a/src/zarr/sync.py b/src/zarr/sync.py index 649db0be7..174680cd1 100644 --- a/src/zarr/sync.py +++ b/src/zarr/sync.py @@ -1,12 +1,13 @@ from __future__ import annotations + from typing import TYPE_CHECKING, TypeVar if TYPE_CHECKING: from typing import Any, AsyncIterator, Coroutine import asyncio -from concurrent.futures import wait import threading +from concurrent.futures import wait from typing_extensions import ParamSpec diff --git a/src/zarr/v2/_storage/absstore.py b/src/zarr/v2/_storage/absstore.py index ee03d44bd..d1c45136d 100644 --- a/src/zarr/v2/_storage/absstore.py +++ b/src/zarr/v2/_storage/absstore.py @@ -1,9 +1,11 @@ """This module contains storage classes related to Azure Blob Storage (ABS)""" import warnings + from numcodecs.compat import ensure_bytes -from zarr.v2.util import normalize_storage_path + from zarr.v2._storage.store import Store +from zarr.v2.util import normalize_storage_path __doctest_requires__ = { ("ABSStore", "ABSStore.*"): ["azure.storage.blob"], diff --git a/src/zarr/v2/_storage/store.py b/src/zarr/v2/_storage/store.py index ec1dbf056..d9f986c04 100644 --- a/src/zarr/v2/_storage/store.py +++ b/src/zarr/v2/_storage/store.py @@ -1,10 +1,9 @@ from collections.abc import MutableMapping from typing import Any, List, Mapping, Optional, Sequence, Union +from zarr.v2.context import Context from zarr.v2.meta import Metadata2 from zarr.v2.util import normalize_storage_path -from zarr.v2.context import Context - # v2 store keys array_meta_key = ".zarray" diff --git a/src/zarr/v2/attrs.py b/src/zarr/v2/attrs.py index af23d43b9..e7567e2b6 100644 --- a/src/zarr/v2/attrs.py +++ b/src/zarr/v2/attrs.py @@ -1,6 +1,6 @@ -from typing import Any import warnings from collections.abc import MutableMapping +from typing import Any from zarr.v2._storage.store import Store from zarr.v2.util import json_dumps diff --git a/src/zarr/v2/convenience.py b/src/zarr/v2/convenience.py index 6355a11af..a8792557b 100644 --- a/src/zarr/v2/convenience.py +++ b/src/zarr/v2/convenience.py @@ -4,25 +4,24 @@ import os import re from collections.abc import Mapping, MutableMapping +from typing import Any, Union + from zarr.v2.core import Array from zarr.v2.creation import array as _create_array from zarr.v2.creation import open_array from zarr.v2.errors import CopyError, PathNotFoundError -from zarr.v2.hierarchy import Group +from zarr.v2.hierarchy import Group, open_group from zarr.v2.hierarchy import group as _create_group -from zarr.v2.hierarchy import open_group from zarr.v2.meta import json_dumps, json_loads from zarr.v2.storage import ( + BaseStore, + ConsolidatedMetadataStore, contains_array, contains_group, normalize_store_arg, - BaseStore, - ConsolidatedMetadataStore, ) from zarr.v2.util import TreeViewer, buffer_size, normalize_storage_path -from typing import Any, Union - StoreLike = Union[BaseStore, MutableMapping[str, Any], str, None] _builtin_open = open # builtin open is later shadowed by a local open function diff --git a/src/zarr/v2/core.py b/src/zarr/v2/core.py index 273d2857e..84f6afed4 100644 --- a/src/zarr/v2/core.py +++ b/src/zarr/v2/core.py @@ -14,17 +14,17 @@ from zarr.v2.attrs import Attributes from zarr.v2.codecs import AsType, get_codec from zarr.v2.context import Context -from zarr.v2.errors import ArrayNotFoundError, ReadOnlyError, ArrayIndexError +from zarr.v2.errors import ArrayIndexError, ArrayNotFoundError, ReadOnlyError from zarr.v2.indexing import ( BasicIndexer, + BlockIndex, + BlockIndexer, CoordinateIndexer, MaskIndexer, OIndex, OrthogonalIndexer, - VIndex, - BlockIndex, - BlockIndexer, PartialChunkIterator, + VIndex, check_fields, check_no_multi_fields, ensure_tuple, @@ -36,18 +36,20 @@ pop_fields, ) from zarr.v2.storage import ( - _prefix_to_array_key, KVStore, + _prefix_to_array_key, getsize, listdir, normalize_store_arg, ) from zarr.v2.util import ( ConstantMap, + InfoReporter, + PartialReadBuffer, UncompressedPartialReadBufferV3, all_equal, - InfoReporter, check_array_shape, + ensure_ndarray_like, human_readable_size, is_total_slice, nolock, @@ -55,8 +57,6 @@ normalize_resize_args, normalize_shape, normalize_storage_path, - PartialReadBuffer, - ensure_ndarray_like, ) __all__ = ["Array"] diff --git a/src/zarr/v2/hierarchy.py b/src/zarr/v2/hierarchy.py index acd65750e..9250d947f 100644 --- a/src/zarr/v2/hierarchy.py +++ b/src/zarr/v2/hierarchy.py @@ -25,20 +25,19 @@ ReadOnlyError, ) from zarr.v2.storage import ( - _prefix_to_group_key, BaseStore, MemoryStore, - group_meta_key, + _prefix_to_group_key, attrs_key, contains_array, contains_group, + group_meta_key, init_group, listdir, normalize_store_arg, rename, rmdir, ) - from zarr.v2.util import ( InfoReporter, TreeViewer, diff --git a/src/zarr/v2/indexing.py b/src/zarr/v2/indexing.py index 1c11409d0..268b4f87c 100644 --- a/src/zarr/v2/indexing.py +++ b/src/zarr/v2/indexing.py @@ -5,13 +5,12 @@ import numpy as np - from zarr.v2.errors import ( ArrayIndexError, + BoundsCheckError, NegativeStepError, - err_too_many_indices, VindexInvalidSelectionError, - BoundsCheckError, + err_too_many_indices, ) diff --git a/src/zarr/v2/meta.py b/src/zarr/v2/meta.py index 2f2f3b948..ff552d952 100644 --- a/src/zarr/v2/meta.py +++ b/src/zarr/v2/meta.py @@ -1,14 +1,14 @@ import base64 import itertools from collections.abc import Mapping +from typing import TYPE_CHECKING, Any, List, Union, cast +from typing import Mapping as MappingType import numpy as np from zarr.v2.errors import MetadataError from zarr.v2.util import json_dumps, json_loads -from typing import cast, Union, Any, List, Mapping as MappingType, TYPE_CHECKING - if TYPE_CHECKING: # pragma: no cover pass diff --git a/src/zarr/v2/n5.py b/src/zarr/v2/n5.py index 92b0f3792..ab49d7781 100644 --- a/src/zarr/v2/n5.py +++ b/src/zarr/v2/n5.py @@ -3,8 +3,8 @@ import os import struct import sys -from typing import Any, Dict, Optional, cast import warnings +from typing import Any, Dict, Optional, cast import numpy as np from numcodecs.abc import Codec @@ -12,8 +12,13 @@ from numcodecs.registry import get_codec, register_codec from zarr.v2.meta import ZARR_FORMAT, json_dumps, json_loads -from zarr.v2.storage import FSStore -from zarr.v2.storage import NestedDirectoryStore, _prog_ckey, _prog_number, normalize_storage_path +from zarr.v2.storage import ( + FSStore, + NestedDirectoryStore, + _prog_ckey, + _prog_number, + normalize_storage_path, +) from zarr.v2.storage import array_meta_key as zarr_array_meta_key from zarr.v2.storage import attrs_key as zarr_attrs_key from zarr.v2.storage import group_meta_key as zarr_group_meta_key diff --git a/src/zarr/v2/storage.py b/src/zarr/v2/storage.py index dd0b090a8..07290370a 100644 --- a/src/zarr/v2/storage.py +++ b/src/zarr/v2/storage.py @@ -25,6 +25,8 @@ import shutil import sys import tempfile +import time +import uuid import warnings import zipfile from collections import OrderedDict @@ -32,25 +34,39 @@ from os import scandir from pickle import PicklingError from threading import Lock, RLock -from typing import Sequence, Mapping, Optional, Union, List, Tuple, Dict, Any -import uuid -import time +from typing import Any, Dict, List, Mapping, Optional, Sequence, Tuple, Union -from numcodecs.compat import ensure_bytes, ensure_text, ensure_contiguous_ndarray_like +from numcodecs.compat import ensure_bytes, ensure_contiguous_ndarray_like, ensure_text from numcodecs.registry import codec_registry -from zarr.v2.context import Context +from zarr.v2._storage.absstore import ABSStore # noqa: F401 +from zarr.v2._storage.store import ( # noqa: F401 + DEFAULT_ZARR_VERSION, + BaseStore, + Store, + _listdir_from_keys, + _path_to_prefix, + _prefix_to_array_key, + _prefix_to_group_key, + _rename_from_keys, + _rmdir_from_keys, + array_meta_key, + attrs_key, + group_meta_key, +) +from zarr.v2.context import Context from zarr.v2.errors import ( - MetadataError, BadCompressorError, ContainsArrayError, ContainsGroupError, FSPathExistNotDir, + MetadataError, ReadOnlyError, ) from zarr.v2.meta import encode_array_metadata, encode_group_metadata from zarr.v2.util import ( buffer_size, + ensure_contiguous_ndarray_or_bytes, json_loads, nolock, normalize_chunks, @@ -61,23 +77,6 @@ normalize_shape, normalize_storage_path, retry_call, - ensure_contiguous_ndarray_or_bytes, -) - -from zarr.v2._storage.absstore import ABSStore # noqa: F401 -from zarr.v2._storage.store import ( # noqa: F401 - _listdir_from_keys, - _rename_from_keys, - _rmdir_from_keys, - _path_to_prefix, - _prefix_to_array_key, - _prefix_to_group_key, - array_meta_key, - attrs_key, - group_meta_key, - DEFAULT_ZARR_VERSION, - BaseStore, - Store, ) __doctest_requires__ = { diff --git a/src/zarr/v2/util.py b/src/zarr/v2/util.py index 8751b39cd..a1276bed3 100644 --- a/src/zarr/v2/util.py +++ b/src/zarr/v2/util.py @@ -1,21 +1,21 @@ import inspect import json import math -import numbers -from textwrap import TextWrapper import mmap +import numbers import time +from textwrap import TextWrapper from typing import ( Any, Callable, Dict, + Iterable, Iterator, Mapping, Optional, Tuple, TypeVar, Union, - Iterable, cast, ) @@ -23,15 +23,15 @@ import numpy.typing as npt from asciitree import BoxStyle, LeftAligned from asciitree.traversal import Traversal +from numcodecs.blosc import cbuffer_metainfo, cbuffer_sizes from numcodecs.compat import ( - ensure_text, - ensure_ndarray_like, ensure_bytes, ensure_contiguous_ndarray_like, + ensure_ndarray_like, + ensure_text, ) from numcodecs.ndarray_like import NDArrayLike from numcodecs.registry import codec_registry -from numcodecs.blosc import cbuffer_sizes, cbuffer_metainfo KeyType = TypeVar("KeyType") ValueType = TypeVar("ValueType") diff --git a/tests/v2/conftest.py b/tests/v2/conftest.py index c84cdfa43..a7a445c64 100644 --- a/tests/v2/conftest.py +++ b/tests/v2/conftest.py @@ -1,4 +1,5 @@ import pathlib + import pytest diff --git a/tests/v2/test_attrs.py b/tests/v2/test_attrs.py index b477b8bef..1c6984858 100644 --- a/tests/v2/test_attrs.py +++ b/tests/v2/test_attrs.py @@ -1,11 +1,11 @@ import json import pytest - from zarr.v2.attrs import Attributes -from zarr.v2.storage import KVStore, DirectoryStore -from .util import CountingDict from zarr.v2.hierarchy import group +from zarr.v2.storage import DirectoryStore, KVStore + +from .util import CountingDict def _init_store(): diff --git a/tests/v2/test_convenience.py b/tests/v2/test_convenience.py index f558a8800..881943756 100644 --- a/tests/v2/test_convenience.py +++ b/tests/v2/test_convenience.py @@ -5,21 +5,20 @@ import numpy as np import pytest +import zarr.v2 as zarr from numcodecs import Adler32, Zlib from numpy.testing import assert_array_equal - -import zarr.v2 as zarr from zarr.v2.convenience import ( consolidate_metadata, copy, + copy_all, copy_store, load, open, open_consolidated, save, - save_group, save_array, - copy_all, + save_group, ) from zarr.v2.core import Array from zarr.v2.errors import CopyError diff --git a/tests/v2/test_core.py b/tests/v2/test_core.py index 197461d12..d80383b2f 100644 --- a/tests/v2/test_core.py +++ b/tests/v2/test_core.py @@ -1,15 +1,17 @@ import atexit import os -import sys import pickle import shutil -from typing import Any, Literal, Optional, Tuple, Union +import sys import unittest from itertools import zip_longest from tempfile import mkdtemp +from typing import Any, Literal, Optional, Tuple, Union + import numpy as np import packaging.version import pytest +import zarr.v2 from numcodecs import ( BZ2, JSON, @@ -29,15 +31,12 @@ from numcodecs.compat import ensure_bytes, ensure_ndarray from numcodecs.tests.common import greetings from numpy.testing import assert_array_almost_equal, assert_array_equal - -import zarr.v2 from zarr.v2._storage.store import ( BaseStore, ) - from zarr.v2.core import Array from zarr.v2.meta import json_loads -from zarr.v2.n5 import N5Store, N5FSStore, n5_keywords +from zarr.v2.n5 import N5FSStore, N5Store, n5_keywords from zarr.v2.storage import ( ABSStore, DBMStore, @@ -54,9 +53,9 @@ init_group, normalize_store_arg, ) - from zarr.v2.util import buffer_size -from .util import abs_container, skip_test_env_var, have_fsspec, mktemp + +from .util import abs_container, have_fsspec, mktemp, skip_test_env_var # noinspection PyMethodMayBeStatic diff --git a/tests/v2/test_creation.py b/tests/v2/test_creation.py index 08073a8ac..8b27d2bd1 100644 --- a/tests/v2/test_creation.py +++ b/tests/v2/test_creation.py @@ -6,7 +6,6 @@ import numpy as np import pytest from numpy.testing import assert_array_equal - from zarr.v2.codecs import Zlib from zarr.v2.core import Array from zarr.v2.creation import ( @@ -27,8 +26,8 @@ from zarr.v2.n5 import N5Store from zarr.v2.storage import DirectoryStore, KVStore from zarr.v2.sync import ThreadSynchronizer -from .util import mktemp, have_fsspec +from .util import have_fsspec, mktemp _VERSIONS = (None, 2) _VERSIONS2 = (2,) diff --git a/tests/v2/test_dim_separator.py b/tests/v2/test_dim_separator.py index b0e9d0ecc..5d4ccb9ba 100644 --- a/tests/v2/test_dim_separator.py +++ b/tests/v2/test_dim_separator.py @@ -1,14 +1,13 @@ import pathlib - -import pytest -from numpy.testing import assert_array_equal from functools import partial +import pytest import zarr.v2 +from numpy.testing import assert_array_equal from zarr.v2.core import Array -from zarr.v2.storage import DirectoryStore, NestedDirectoryStore, FSStore -from .util import have_fsspec +from zarr.v2.storage import DirectoryStore, FSStore, NestedDirectoryStore +from .util import have_fsspec needs_fsspec = pytest.mark.skipif(not have_fsspec, reason="needs fsspec") diff --git a/tests/v2/test_filters.py b/tests/v2/test_filters.py index fe21dc775..d5f411cfb 100644 --- a/tests/v2/test_filters.py +++ b/tests/v2/test_filters.py @@ -11,7 +11,6 @@ Zlib, ) from numpy.testing import assert_array_almost_equal, assert_array_equal - from zarr.v2.creation import array compressors = [ diff --git a/tests/v2/test_hierarchy.py b/tests/v2/test_hierarchy.py index 23c5a56ed..e8d57afe0 100644 --- a/tests/v2/test_hierarchy.py +++ b/tests/v2/test_hierarchy.py @@ -1,8 +1,8 @@ import atexit import os -import sys import pickle import shutil +import sys import tempfile import textwrap import unittest @@ -17,7 +17,6 @@ from numcodecs import Zlib from numpy.testing import assert_array_equal - from zarr.v2.attrs import Attributes from zarr.v2.core import Array from zarr.v2.creation import open_array @@ -25,9 +24,9 @@ from zarr.v2.storage import ( ABSStore, DBMStore, - KVStore, DirectoryStore, FSStore, + KVStore, LMDBStore, LRUStoreCache, MemoryStore, @@ -41,9 +40,9 @@ init_array, init_group, ) - from zarr.v2.util import InfoReporter -from .util import skip_test_env_var, have_fsspec, abs_container, mktemp + +from .util import abs_container, have_fsspec, mktemp, skip_test_env_var # noinspection PyStatementEffect diff --git a/tests/v2/test_indexing.py b/tests/v2/test_indexing.py index 13fbc878e..927b21c3e 100644 --- a/tests/v2/test_indexing.py +++ b/tests/v2/test_indexing.py @@ -1,16 +1,15 @@ import numpy import numpy as np import pytest -from numpy.testing import assert_array_equal - import zarr.v2 +from numpy.testing import assert_array_equal from zarr.v2.indexing import ( + PartialChunkIterator, make_slice_selection, normalize_integer_selection, oindex, oindex_set, replace_ellipsis, - PartialChunkIterator, ) from .util import CountingDict diff --git a/tests/v2/test_info.py b/tests/v2/test_info.py index 9f6811929..b95c3dc4b 100644 --- a/tests/v2/test_info.py +++ b/tests/v2/test_info.py @@ -1,6 +1,5 @@ import numcodecs import pytest - import zarr.v2 from zarr.v2.util import InfoReporter diff --git a/tests/v2/test_meta.py b/tests/v2/test_meta.py index b7c00ec64..ee55c2f37 100644 --- a/tests/v2/test_meta.py +++ b/tests/v2/test_meta.py @@ -3,18 +3,17 @@ import numpy as np import pytest - from zarr.v2.codecs import Blosc, Delta, Pickle, Zlib from zarr.v2.errors import MetadataError from zarr.v2.meta import ( ZARR_FORMAT, decode_array_metadata, decode_dtype, + decode_fill_value, decode_group_metadata, encode_array_metadata, encode_dtype, encode_fill_value, - decode_fill_value, ) from zarr.v2.util import normalize_dtype, normalize_fill_value diff --git a/tests/v2/test_meta_array.py b/tests/v2/test_meta_array.py index c47673fb0..44b19c8c3 100644 --- a/tests/v2/test_meta_array.py +++ b/tests/v2/test_meta_array.py @@ -1,15 +1,14 @@ from typing import Optional + import numpy as np import pytest - +import zarr.v2.codecs from numcodecs.abc import Codec from numcodecs.compat import ensure_contiguous_ndarray_like from numcodecs.registry import get_codec, register_codec - -import zarr.v2.codecs from zarr.v2.core import Array from zarr.v2.creation import array, empty, full, ones, open_array, zeros -from zarr.v2.hierarchy import open_group, group +from zarr.v2.hierarchy import group, open_group from zarr.v2.storage import DirectoryStore, MemoryStore, Store, ZipStore diff --git a/tests/v2/test_n5.py b/tests/v2/test_n5.py index 238e9b2c6..2ab95a329 100644 --- a/tests/v2/test_n5.py +++ b/tests/v2/test_n5.py @@ -1,13 +1,13 @@ -import pytest +import atexit +import json +from typing import Tuple -from zarr.v2.n5 import N5ChunkWrapper, N5FSStore +import numpy as np +import pytest +from numcodecs import GZip from zarr.v2.creation import create +from zarr.v2.n5 import N5ChunkWrapper, N5FSStore from zarr.v2.storage import atexit_rmtree -from numcodecs import GZip -import numpy as np -from typing import Tuple -import json -import atexit from .util import have_fsspec diff --git a/tests/v2/test_storage.py b/tests/v2/test_storage.py index 17b80e6a5..1400d20b1 100644 --- a/tests/v2/test_storage.py +++ b/tests/v2/test_storage.py @@ -3,9 +3,9 @@ import json import os import pathlib -import sys import pickle import shutil +import sys import tempfile from contextlib import contextmanager from pickle import PicklingError @@ -13,11 +13,9 @@ import numpy as np import pytest -from numpy.testing import assert_array_almost_equal, assert_array_equal - -from numcodecs.compat import ensure_bytes - import zarr.v2 +from numcodecs.compat import ensure_bytes +from numpy.testing import assert_array_almost_equal, assert_array_equal from zarr.v2 import meta_v1 from zarr.v2.codecs import BZ2, AsType, Blosc, Zlib from zarr.v2.context import Context @@ -25,13 +23,14 @@ from zarr.v2.errors import ContainsArrayError, ContainsGroupError, MetadataError from zarr.v2.hierarchy import group from zarr.v2.meta import ZARR_FORMAT, decode_array_metadata -from zarr.v2.n5 import N5Store, N5FSStore, N5_FORMAT, n5_attrs_key +from zarr.v2.n5 import N5_FORMAT, N5FSStore, N5Store, n5_attrs_key from zarr.v2.storage import ( ABSStore, ConsolidatedMetadataStore, DBMStore, DictStore, DirectoryStore, + FSStore, KVStore, LMDBStore, LRUStoreCache, @@ -52,13 +51,15 @@ group_meta_key, init_array, init_group, + listdir, migrate_1to2, normalize_store_arg, + rename, ) -from zarr.v2.storage import FSStore, rename, listdir -from .util import CountingDict, have_fsspec, skip_test_env_var, abs_container, mktemp from zarr.v2.util import ConstantMap, json_dumps +from .util import CountingDict, abs_container, have_fsspec, mktemp, skip_test_env_var + @contextmanager def does_not_raise(): diff --git a/tests/v2/test_sync.py b/tests/v2/test_sync.py index 8bf1304dc..cd2bf97bd 100644 --- a/tests/v2/test_sync.py +++ b/tests/v2/test_sync.py @@ -8,7 +8,6 @@ import numpy as np from numpy.testing import assert_array_equal - from zarr.v2.attrs import Attributes from zarr.v2.core import Array from zarr.v2.hierarchy import Group diff --git a/tests/v2/test_util.py b/tests/v2/test_util.py index 35c355693..610d6ead2 100644 --- a/tests/v2/test_util.py +++ b/tests/v2/test_util.py @@ -3,7 +3,6 @@ import numpy as np import pytest - from zarr.v2.core import Array from zarr.v2.util import ( ConstantMap, @@ -23,8 +22,8 @@ normalize_shape, retry_call, tree_array_icon, - tree_group_icon, tree_get_icon, + tree_group_icon, tree_widget, ) diff --git a/tests/v2/util.py b/tests/v2/util.py index 12c5e379f..80dcbdb93 100644 --- a/tests/v2/util.py +++ b/tests/v2/util.py @@ -2,11 +2,10 @@ import os import tempfile from typing import Any, Mapping, Sequence -from zarr.v2.context import Context - -from zarr.v2.storage import Store import pytest +from zarr.v2.context import Context +from zarr.v2.storage import Store class CountingDict(Store): @@ -65,8 +64,8 @@ def skip_test_env_var(name): def abs_container(): - from azure.core.exceptions import ResourceExistsError import azure.storage.blob as asb + from azure.core.exceptions import ResourceExistsError URL = "http://127.0.0.1:10000" ACCOUNT_NAME = "devstoreaccount1" diff --git a/tests/v3/conftest.py b/tests/v3/conftest.py index 3dc55c029..1161c870a 100644 --- a/tests/v3/conftest.py +++ b/tests/v3/conftest.py @@ -1,7 +1,7 @@ import pathlib -import pytest -from zarr.store import LocalStore, StorePath, MemoryStore, RemoteStore +import pytest +from zarr.store import LocalStore, MemoryStore, RemoteStore, StorePath @pytest.fixture(params=[str, pathlib.Path]) diff --git a/tests/v3/test_codec_entrypoints.py b/tests/v3/test_codec_entrypoints.py index 8fbf76b83..b36919dcd 100644 --- a/tests/v3/test_codec_entrypoints.py +++ b/tests/v3/test_codec_entrypoints.py @@ -2,10 +2,8 @@ import sys import pytest - import zarr.codecs.registry - here = os.path.abspath(os.path.dirname(__file__)) diff --git a/tests/v3/test_codecs.py b/tests/v3/test_codecs.py index e042c7f27..54dc6a34b 100644 --- a/tests/v3/test_codecs.py +++ b/tests/v3/test_codecs.py @@ -1,29 +1,27 @@ from __future__ import annotations -from dataclasses import dataclass import json +from dataclasses import dataclass from typing import Iterator, List, Literal, Optional, Tuple - import numpy as np import pytest import zarr.v2 from zarr.abc.codec import Codec +from zarr.abc.store import Store from zarr.array import Array, AsyncArray -from zarr.common import Selection -from zarr.indexing import morton_order_iter from zarr.codecs import ( - ShardingCodec, - ShardingCodecIndexLocation, BloscCodec, BytesCodec, GzipCodec, + ShardingCodec, + ShardingCodecIndexLocation, TransposeCodec, ZstdCodec, ) - -from zarr.abc.store import Store +from zarr.common import Selection from zarr.config import config +from zarr.indexing import morton_order_iter from zarr.store import MemoryStore, StorePath diff --git a/tests/v3/test_common.py b/tests/v3/test_common.py index b5690d0d7..e0c78385f 100644 --- a/tests/v3/test_common.py +++ b/tests/v3/test_common.py @@ -1,14 +1,14 @@ from __future__ import annotations + from typing import TYPE_CHECKING, Iterable if TYPE_CHECKING: - from typing import Literal, Any, Tuple + from typing import Any, Literal, Tuple import numpy as np -from zarr.config import parse_indexing_order -from zarr.common import parse_shapelike -from zarr.common import parse_name, product import pytest +from zarr.common import parse_name, parse_shapelike, product +from zarr.config import parse_indexing_order @pytest.mark.parametrize("data", [(0, 0, 0, 0), (1, 3, 4, 5, 6), (2, 4)]) diff --git a/tests/v3/test_group.py b/tests/v3/test_group.py index 11400ef80..acaf92053 100644 --- a/tests/v3/test_group.py +++ b/tests/v3/test_group.py @@ -1,13 +1,13 @@ from __future__ import annotations + from typing import TYPE_CHECKING from zarr.sync import sync if TYPE_CHECKING: - from zarr.store import MemoryStore, LocalStore -import pytest + from zarr.store import LocalStore, MemoryStore import numpy as np - +import pytest from zarr.group import AsyncGroup, Group, GroupMetadata from zarr.store import LocalStore, StorePath diff --git a/tests/v3/test_metadata.py b/tests/v3/test_metadata.py index 33df4a643..2546974c0 100644 --- a/tests/v3/test_metadata.py +++ b/tests/v3/test_metadata.py @@ -1,9 +1,11 @@ from __future__ import annotations -import pytest + from typing import TYPE_CHECKING +import pytest + if TYPE_CHECKING: - from typing import Sequence, Any + from typing import Any, Sequence from zarr.metadata import parse_dimension_names, parse_zarr_format_v2, parse_zarr_format_v3 diff --git a/tests/v3/test_storage.py b/tests/v3/test_storage.py index 2761e608e..e97950678 100644 --- a/tests/v3/test_storage.py +++ b/tests/v3/test_storage.py @@ -1,8 +1,7 @@ import pytest - -from zarr.testing.store import StoreTests from zarr.store.local import LocalStore from zarr.store.memory import MemoryStore +from zarr.testing.store import StoreTests class TestMemoryStore(StoreTests): diff --git a/tests/v3/test_sync.py b/tests/v3/test_sync.py index ba262f521..13f7eb43d 100644 --- a/tests/v3/test_sync.py +++ b/tests/v3/test_sync.py @@ -1,11 +1,10 @@ -from collections.abc import AsyncGenerator import asyncio import time -from unittest.mock import patch, AsyncMock - -from zarr.sync import sync, _get_loop, _get_lock, SyncError, SyncMixin +from collections.abc import AsyncGenerator +from unittest.mock import AsyncMock, patch import pytest +from zarr.sync import SyncError, SyncMixin, _get_lock, _get_loop, sync @pytest.fixture(params=[True, False])