diff --git a/bench/compress_normal.py b/bench/compress_normal.py index 803d54b76..38969b010 100644 --- a/bench/compress_normal.py +++ b/bench/compress_normal.py @@ -1,9 +1,8 @@ import sys import timeit -import numpy as np - import line_profiler +import numpy as np import zarr from zarr import blosc diff --git a/pyproject.toml b/pyproject.toml index fca263db9..98949f7a6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -160,7 +160,15 @@ extend-exclude = [ "build", "dist", "venv", - "docs" + "docs", + "src/zarr/v2/", + "tests/v2/" +] +[tool.ruff.lint] +extend-select = [ + "B", # flake8-bugbear + "I", # isort + "UP", # pyupgrade ] [tool.ruff.lint] diff --git a/src/zarr/__init__.py b/src/zarr/__init__.py index 65daae8f6..8856ce25b 100644 --- a/src/zarr/__init__.py +++ b/src/zarr/__init__.py @@ -1,9 +1,9 @@ from __future__ import annotations -from typing import Union - import zarr.codecs # noqa: F401 +from zarr._version import version as __version__ from zarr.array import Array, AsyncArray + from zarr.array_v2 import ArrayV2 from zarr.config import config # noqa: F401 from zarr.group import AsyncGroup, Group @@ -12,7 +12,6 @@ make_store_path, ) from zarr.sync import sync as _sync -from zarr._version import version as __version__ # in case setuptools scm screw up and find version to be 0.0.0 assert not __version__.startswith("0.0.0") @@ -20,7 +19,7 @@ async def open_auto_async( store: StoreLike, -) -> Union[AsyncArray, AsyncGroup]: +) -> AsyncArray | AsyncGroup: store_path = make_store_path(store) try: return await AsyncArray.open(store_path) @@ -30,7 +29,7 @@ async def open_auto_async( def open_auto( store: StoreLike, -) -> Union[Array, ArrayV2, Group]: +) -> Array | ArrayV2 | Group: object = _sync( open_auto_async(store), ) diff --git a/src/zarr/abc/codec.py b/src/zarr/abc/codec.py index 8897cced8..9ba6ca025 100644 --- a/src/zarr/abc/codec.py +++ b/src/zarr/abc/codec.py @@ -1,17 +1,17 @@ from __future__ import annotations from abc import abstractmethod -from typing import TYPE_CHECKING, Optional +from typing import TYPE_CHECKING import numpy as np -from zarr.abc.metadata import Metadata +from zarr.abc.metadata import Metadata from zarr.common import ArraySpec from zarr.store import StorePath - if TYPE_CHECKING: from typing_extensions import Self + from zarr.common import BytesLike, SliceSelection from zarr.metadata import ArrayMetadata @@ -47,7 +47,7 @@ async def encode( self, chunk_array: np.ndarray, chunk_spec: ArraySpec, - ) -> Optional[np.ndarray]: + ) -> np.ndarray | None: pass @@ -65,7 +65,7 @@ async def encode( self, chunk_array: np.ndarray, chunk_spec: ArraySpec, - ) -> Optional[BytesLike]: + ) -> BytesLike | None: pass @@ -76,7 +76,7 @@ async def decode_partial( store_path: StorePath, selection: SliceSelection, chunk_spec: ArraySpec, - ) -> Optional[np.ndarray]: + ) -> np.ndarray | None: pass @@ -106,5 +106,5 @@ async def encode( self, chunk_array: BytesLike, chunk_spec: ArraySpec, - ) -> Optional[BytesLike]: + ) -> BytesLike | None: pass diff --git a/src/zarr/abc/metadata.py b/src/zarr/abc/metadata.py index f27b37cba..36edf6953 100644 --- a/src/zarr/abc/metadata.py +++ b/src/zarr/abc/metadata.py @@ -1,11 +1,12 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Sequence + +from collections.abc import Sequence +from typing import TYPE_CHECKING if TYPE_CHECKING: - from typing import Dict from typing_extensions import Self -from dataclasses import fields, dataclass +from dataclasses import dataclass, fields from zarr.common import JSON @@ -36,7 +37,7 @@ def to_dict(self) -> JSON: return out_dict @classmethod - def from_dict(cls, data: Dict[str, JSON]) -> Self: + def from_dict(cls, data: dict[str, JSON]) -> Self: """ Create an instance of the model from a dictionary """ diff --git a/src/zarr/abc/store.py b/src/zarr/abc/store.py index d92f8d4e2..3b7ac6907 100644 --- a/src/zarr/abc/store.py +++ b/src/zarr/abc/store.py @@ -1,14 +1,10 @@ -from abc import abstractmethod, ABC - +from abc import ABC, abstractmethod from collections.abc import AsyncGenerator -from typing import List, Tuple, Optional class Store(ABC): @abstractmethod - async def get( - self, key: str, byte_range: Optional[Tuple[int, Optional[int]]] = None - ) -> Optional[bytes]: + async def get(self, key: str, byte_range: tuple[int, int | None] | None = None) -> bytes | None: """Retrieve the value associated with a given key. Parameters @@ -24,8 +20,8 @@ async def get( @abstractmethod async def get_partial_values( - self, key_ranges: List[Tuple[str, Tuple[int, int]]] - ) -> List[Optional[bytes]]: + self, key_ranges: list[tuple[str, tuple[int, int]]] + ) -> list[bytes | None]: """Retrieve possibly partial values from given key_ranges. Parameters @@ -88,7 +84,7 @@ def supports_partial_writes(self) -> bool: ... @abstractmethod - async def set_partial_values(self, key_start_values: List[Tuple[str, int, bytes]]) -> None: + async def set_partial_values(self, key_start_values: list[tuple[str, int, bytes]]) -> None: """Store values at a given key, starting at byte range_start. Parameters diff --git a/src/zarr/array.py b/src/zarr/array.py index 128d7c58e..b444b1ec5 100644 --- a/src/zarr/array.py +++ b/src/zarr/array.py @@ -9,15 +9,17 @@ from __future__ import annotations -from dataclasses import dataclass, replace - import json -from typing import Any, Dict, Iterable, Literal, Optional, Tuple, Union +from collections.abc import Iterable +from dataclasses import dataclass, replace +from typing import Any, Literal import numpy as np import numpy.typing as npt -from zarr.abc.codec import Codec +from zarr.abc.codec import Codec +from zarr.chunk_grids import RegularChunkGrid +from zarr.chunk_key_encodings import DefaultChunkKeyEncoding, V2ChunkKeyEncoding # from zarr.array_v2 import ArrayV2 from zarr.codecs import BytesCodec @@ -31,10 +33,7 @@ concurrent_map, ) from zarr.config import config - from zarr.indexing import BasicIndexer, all_chunk_coords, is_total_slice -from zarr.chunk_grids import RegularChunkGrid -from zarr.chunk_key_encodings import DefaultChunkKeyEncoding, V2ChunkKeyEncoding from zarr.metadata import ArrayMetadata, parse_indexing_order from zarr.store import StoreLike, StorePath, make_store_path from zarr.sync import sync @@ -80,14 +79,12 @@ async def create( shape: ChunkCoords, dtype: npt.DTypeLike, chunk_shape: ChunkCoords, - fill_value: Optional[Any] = None, - chunk_key_encoding: Union[ - Tuple[Literal["default"], Literal[".", "/"]], - Tuple[Literal["v2"], Literal[".", "/"]], - ] = ("default", "/"), - codecs: Optional[Iterable[Union[Codec, Dict[str, Any]]]] = None, - dimension_names: Optional[Iterable[str]] = None, - attributes: Optional[Dict[str, Any]] = None, + fill_value: Any | None = None, + chunk_key_encoding: tuple[Literal["default"], Literal[".", "/"]] + | tuple[Literal["v2"], Literal[".", "/"]] = ("default", "/"), + codecs: Iterable[Codec | dict[str, Any]] | None = None, + dimension_names: Iterable[str] | None = None, + attributes: dict[str, Any] | None = None, exists_ok: bool = False, ) -> AsyncArray: store_path = make_store_path(store) @@ -129,7 +126,7 @@ async def create( def from_dict( cls, store_path: StorePath, - data: Dict[str, Any], + data: dict[str, Any], ) -> AsyncArray: metadata = ArrayMetadata.from_dict(data) async_array = cls(metadata=metadata, store_path=store_path) @@ -380,7 +377,7 @@ async def _delete_key(key: str) -> None: await (self.store_path / ZARR_JSON).set(new_metadata.to_bytes()) return replace(self, metadata=new_metadata) - async def update_attributes(self, new_attributes: Dict[str, Any]) -> AsyncArray: + async def update_attributes(self, new_attributes: dict[str, Any]) -> AsyncArray: new_metadata = replace(self.metadata, attributes=new_attributes) # Write new metadata @@ -406,14 +403,12 @@ def create( shape: ChunkCoords, dtype: npt.DTypeLike, chunk_shape: ChunkCoords, - fill_value: Optional[Any] = None, - chunk_key_encoding: Union[ - Tuple[Literal["default"], Literal[".", "/"]], - Tuple[Literal["v2"], Literal[".", "/"]], - ] = ("default", "/"), - codecs: Optional[Iterable[Union[Codec, Dict[str, Any]]]] = None, - dimension_names: Optional[Iterable[str]] = None, - attributes: Optional[Dict[str, Any]] = None, + fill_value: Any | None = None, + chunk_key_encoding: tuple[Literal["default"], Literal[".", "/"]] + | tuple[Literal["v2"], Literal[".", "/"]] = ("default", "/"), + codecs: Iterable[Codec | dict[str, Any]] | None = None, + dimension_names: Iterable[str] | None = None, + attributes: dict[str, Any] | None = None, exists_ok: bool = False, ) -> Array: async_array = sync( @@ -436,7 +431,7 @@ def create( def from_dict( cls, store_path: StorePath, - data: Dict[str, Any], + data: dict[str, Any], ) -> Array: async_array = AsyncArray.from_dict(store_path=store_path, data=data) return cls(async_array) @@ -508,7 +503,7 @@ def resize(self, new_shape: ChunkCoords) -> Array: ) ) - def update_attributes(self, new_attributes: Dict[str, Any]) -> Array: + def update_attributes(self, new_attributes: dict[str, Any]) -> Array: return type(self)( sync( self._async_array.update_attributes(new_attributes), diff --git a/src/zarr/array_v2.py b/src/zarr/array_v2.py index 18251e7db..a2c5bb0d4 100644 --- a/src/zarr/array_v2.py +++ b/src/zarr/array_v2.py @@ -1,13 +1,12 @@ from __future__ import annotations import asyncio -from dataclasses import dataclass, replace import json -from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional, Union +from dataclasses import dataclass, replace +from typing import TYPE_CHECKING, Any, Literal import numcodecs import numpy as np - from numcodecs.compat import ensure_bytes, ensure_ndarray from zarr.common import ( @@ -52,7 +51,7 @@ async def set(self, value: np.ndarray): @dataclass(frozen=True) class ArrayV2: metadata: ArrayV2Metadata - attributes: Optional[Dict[str, Any]] + attributes: dict[str, Any] | None store_path: StorePath @classmethod @@ -64,11 +63,11 @@ async def create_async( dtype: np.dtype, chunks: ChunkCoords, dimension_separator: Literal[".", "/"] = ".", - fill_value: Optional[Union[None, int, float]] = None, + fill_value: None | int | float | None = None, order: Literal["C", "F"] = "C", - filters: Optional[List[Dict[str, Any]]] = None, - compressor: Optional[Dict[str, Any]] = None, - attributes: Optional[Dict[str, Any]] = None, + filters: list[dict[str, Any]] | None = None, + compressor: dict[str, Any] | None = None, + attributes: dict[str, Any] | None = None, exists_ok: bool = False, ) -> ArrayV2: store_path = make_store_path(store) @@ -108,11 +107,11 @@ def create( dtype: np.dtype, chunks: ChunkCoords, dimension_separator: Literal[".", "/"] = ".", - fill_value: Optional[Union[None, int, float]] = None, + fill_value: None | int | float | None = None, order: Literal["C", "F"] = "C", - filters: Optional[List[Dict[str, Any]]] = None, - compressor: Optional[Dict[str, Any]] = None, - attributes: Optional[Dict[str, Any]] = None, + filters: list[dict[str, Any]] | None = None, + compressor: dict[str, Any] | None = None, + attributes: dict[str, Any] | None = None, exists_ok: bool = False, ) -> ArrayV2: return sync( @@ -162,7 +161,7 @@ def from_dict( cls, store_path: StorePath, zarray_json: Any, - zattrs_json: Optional[Any], + zattrs_json: Any | None, ) -> ArrayV2: metadata = ArrayV2Metadata.from_dict(zarray_json) out = cls( @@ -252,7 +251,7 @@ async def _read_chunk( else: out[out_selection] = self.metadata.fill_value - async def _decode_chunk(self, chunk_bytes: Optional[BytesLike]) -> Optional[np.ndarray]: + async def _decode_chunk(self, chunk_bytes: BytesLike | None) -> np.ndarray | None: if chunk_bytes is None: return None @@ -365,7 +364,7 @@ async def _write_chunk( await self._write_chunk_to_store(store_path, chunk_array) async def _write_chunk_to_store(self, store_path: StorePath, chunk_array: np.ndarray): - chunk_bytes: Optional[BytesLike] + chunk_bytes: BytesLike | None if np.all(chunk_array == self.metadata.fill_value): # chunks that only contain fill_value will be removed await store_path.delete() @@ -376,7 +375,7 @@ async def _write_chunk_to_store(self, store_path: StorePath, chunk_array: np.nda else: await store_path.set(chunk_bytes) - async def _encode_chunk(self, chunk_array: np.ndarray) -> Optional[BytesLike]: + async def _encode_chunk(self, chunk_array: np.ndarray) -> BytesLike | None: chunk_array = chunk_array.ravel(order=self.metadata.order) if self.metadata.filters is not None: @@ -430,11 +429,8 @@ async def convert_to_v3_async(self) -> Array: from zarr.abc.codec import Codec from zarr.array import Array - from zarr.common import ZARR_JSON from zarr.chunk_grids import RegularChunkGrid from zarr.chunk_key_encodings import V2ChunkKeyEncoding - from zarr.metadata import ArrayMetadata, DataType - from zarr.codecs import ( BloscCodec, BloscShuffle, @@ -442,6 +438,8 @@ async def convert_to_v3_async(self) -> Array: GzipCodec, TransposeCodec, ) + from zarr.common import ZARR_JSON + from zarr.metadata import ArrayMetadata, DataType data_type = DataType.from_dtype(self.metadata.dtype) endian: Literal["little", "big"] @@ -456,7 +454,7 @@ async def convert_to_v3_async(self) -> Array: self.metadata.filters is None or len(self.metadata.filters) == 0 ), "Filters are not supported by v3." - codecs: List[Codec] = [] + codecs: list[Codec] = [] if self.metadata.order == "F": codecs.append(TransposeCodec(order=tuple(reversed(range(self.metadata.ndim))))) @@ -500,11 +498,11 @@ async def convert_to_v3_async(self) -> Array: data=json.loads(new_metadata_bytes), ) - async def update_attributes_async(self, new_attributes: Dict[str, Any]) -> ArrayV2: + async def update_attributes_async(self, new_attributes: dict[str, Any]) -> ArrayV2: await (self.store_path / ZATTRS_JSON).set(json.dumps(new_attributes).encode()) return replace(self, attributes=new_attributes) - def update_attributes(self, new_attributes: Dict[str, Any]) -> ArrayV2: + def update_attributes(self, new_attributes: dict[str, Any]) -> ArrayV2: return sync( self.update_attributes_async(new_attributes), ) diff --git a/src/zarr/attributes.py b/src/zarr/attributes.py index 18f6a63a5..b5123df98 100644 --- a/src/zarr/attributes.py +++ b/src/zarr/attributes.py @@ -1,14 +1,15 @@ from __future__ import annotations + from collections.abc import MutableMapping -from typing import TYPE_CHECKING, Any, Iterator, Union +from typing import TYPE_CHECKING, Any, Iterator if TYPE_CHECKING: - from zarr.group import Group from zarr.array import Array + from zarr.group import Group class Attributes(MutableMapping[str, Any]): - def __init__(self, obj: Union[Array, Group]): + def __init__(self, obj: Array | Group): # key=".zattrs", read_only=False, cache=True, synchronizer=None self._obj = obj diff --git a/src/zarr/chunk_grids.py b/src/zarr/chunk_grids.py index 73557f6e4..5a301e457 100644 --- a/src/zarr/chunk_grids.py +++ b/src/zarr/chunk_grids.py @@ -1,8 +1,9 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, Dict + from dataclasses import dataclass -from zarr.abc.metadata import Metadata +from typing import TYPE_CHECKING, Any +from zarr.abc.metadata import Metadata from zarr.common import ( JSON, ChunkCoords, @@ -18,7 +19,7 @@ @dataclass(frozen=True) class ChunkGrid(Metadata): @classmethod - def from_dict(cls, data: Dict[str, JSON]) -> ChunkGrid: + def from_dict(cls, data: dict[str, JSON]) -> ChunkGrid: if isinstance(data, ChunkGrid): return data @@ -38,10 +39,10 @@ def __init__(self, *, chunk_shape: ChunkCoordsLike) -> None: object.__setattr__(self, "chunk_shape", chunk_shape_parsed) @classmethod - def from_dict(cls, data: Dict[str, Any]) -> Self: + def from_dict(cls, data: dict[str, Any]) -> Self: _, configuration_parsed = parse_named_configuration(data, "regular") return cls(**configuration_parsed) # type: ignore[arg-type] - def to_dict(self) -> Dict[str, JSON]: + def to_dict(self) -> dict[str, JSON]: return {"name": "regular", "configuration": {"chunk_shape": list(self.chunk_shape)}} diff --git a/src/zarr/chunk_key_encodings.py b/src/zarr/chunk_key_encodings.py index ebc7654dd..5ecb98ef6 100644 --- a/src/zarr/chunk_key_encodings.py +++ b/src/zarr/chunk_key_encodings.py @@ -1,9 +1,10 @@ from __future__ import annotations + from abc import abstractmethod -from typing import TYPE_CHECKING, Dict, Literal, cast from dataclasses import dataclass -from zarr.abc.metadata import Metadata +from typing import TYPE_CHECKING, Literal, cast +from zarr.abc.metadata import Metadata from zarr.common import ( JSON, ChunkCoords, @@ -33,7 +34,7 @@ def __init__(self, *, separator: SeparatorLiteral) -> None: object.__setattr__(self, "separator", separator_parsed) @classmethod - def from_dict(cls, data: Dict[str, JSON]) -> ChunkKeyEncoding: + def from_dict(cls, data: dict[str, JSON]) -> ChunkKeyEncoding: if isinstance(data, ChunkKeyEncoding): return data @@ -44,7 +45,7 @@ def from_dict(cls, data: Dict[str, JSON]) -> ChunkKeyEncoding: return V2ChunkKeyEncoding(**configuration_parsed) # type: ignore[arg-type] raise ValueError(f"Unknown chunk key encoding. Got {name_parsed}.") - def to_dict(self) -> Dict[str, JSON]: + def to_dict(self) -> dict[str, JSON]: return {"name": self.name, "configuration": {"separator": self.separator}} @abstractmethod diff --git a/src/zarr/codecs/__init__.py b/src/zarr/codecs/__init__.py index 8fa0c9f7b..08b97b932 100644 --- a/src/zarr/codecs/__init__.py +++ b/src/zarr/codecs/__init__.py @@ -1,6 +1,6 @@ from __future__ import annotations -from zarr.codecs.blosc import BloscCodec, BloscCname, BloscShuffle # noqa: F401 +from zarr.codecs.blosc import BloscCname, BloscCodec, BloscShuffle # noqa: F401 from zarr.codecs.bytes import BytesCodec, Endian # noqa: F401 from zarr.codecs.crc32c_ import Crc32cCodec # noqa: F401 from zarr.codecs.gzip import GzipCodec # noqa: F401 diff --git a/src/zarr/codecs/blosc.py b/src/zarr/codecs/blosc.py index 5ee2b7640..6b0cd4705 100644 --- a/src/zarr/codecs/blosc.py +++ b/src/zarr/codecs/blosc.py @@ -1,9 +1,9 @@ from __future__ import annotations + from dataclasses import dataclass, replace from enum import Enum from functools import cached_property - -from typing import TYPE_CHECKING, Union +from typing import TYPE_CHECKING import numcodecs import numpy as np @@ -14,8 +14,8 @@ from zarr.common import parse_enum, parse_named_configuration, to_thread if TYPE_CHECKING: - from typing import Dict, Optional from typing_extensions import Self + from zarr.common import JSON, ArraySpec, BytesLike @@ -86,10 +86,10 @@ class BloscCodec(BytesBytesCodec): def __init__( self, *, - typesize: Optional[int] = None, - cname: Union[BloscCname, str] = BloscCname.zstd, + typesize: int | None = None, + cname: BloscCname | str = BloscCname.zstd, clevel: int = 5, - shuffle: Union[BloscShuffle, str, None] = None, + shuffle: BloscShuffle | str | None = None, blocksize: int = 0, ) -> None: typesize_parsed = parse_typesize(typesize) if typesize is not None else None @@ -105,11 +105,11 @@ def __init__( object.__setattr__(self, "blocksize", blocksize_parsed) @classmethod - def from_dict(cls, data: Dict[str, JSON]) -> Self: + def from_dict(cls, data: dict[str, JSON]) -> Self: _, configuration_parsed = parse_named_configuration(data, "blosc") return cls(**configuration_parsed) # type: ignore[arg-type] - def to_dict(self) -> Dict[str, JSON]: + def to_dict(self) -> dict[str, JSON]: if self.typesize is None: raise ValueError("`typesize` needs to be set for serialization.") if self.shuffle is None: @@ -169,7 +169,7 @@ async def encode( self, chunk_bytes: bytes, chunk_spec: ArraySpec, - ) -> Optional[BytesLike]: + ) -> BytesLike | None: chunk_array = np.frombuffer(chunk_bytes, dtype=chunk_spec.dtype) return await to_thread(self._blosc_codec.encode, chunk_array) diff --git a/src/zarr/codecs/bytes.py b/src/zarr/codecs/bytes.py index 566b3a8df..0533ad9af 100644 --- a/src/zarr/codecs/bytes.py +++ b/src/zarr/codecs/bytes.py @@ -1,9 +1,9 @@ from __future__ import annotations + +import sys from dataclasses import dataclass, replace from enum import Enum -import sys - -from typing import TYPE_CHECKING, Dict, Optional, Union +from typing import TYPE_CHECKING import numpy as np @@ -12,9 +12,10 @@ from zarr.common import parse_enum, parse_named_configuration if TYPE_CHECKING: - from zarr.common import JSON, ArraySpec, BytesLike from typing_extensions import Self + from zarr.common import JSON, ArraySpec, BytesLike + class Endian(Enum): big = "big" @@ -28,22 +29,22 @@ class Endian(Enum): class BytesCodec(ArrayBytesCodec): is_fixed_size = True - endian: Optional[Endian] + endian: Endian | None - def __init__(self, *, endian: Union[Endian, str, None] = default_system_endian) -> None: + def __init__(self, *, endian: Endian | str | None = default_system_endian) -> None: endian_parsed = None if endian is None else parse_enum(endian, Endian) object.__setattr__(self, "endian", endian_parsed) @classmethod - def from_dict(cls, data: Dict[str, JSON]) -> Self: + def from_dict(cls, data: dict[str, JSON]) -> Self: _, configuration_parsed = parse_named_configuration( data, "bytes", require_configuration=False ) configuration_parsed = configuration_parsed or {} return cls(**configuration_parsed) # type: ignore[arg-type] - def to_dict(self) -> Dict[str, JSON]: + def to_dict(self) -> dict[str, JSON]: if self.endian is None: return {"name": "bytes"} else: @@ -93,7 +94,7 @@ async def encode( self, chunk_array: np.ndarray, _chunk_spec: ArraySpec, - ) -> Optional[BytesLike]: + ) -> BytesLike | None: if chunk_array.dtype.itemsize > 1: byteorder = self._get_byteorder(chunk_array) if self.endian is not None and self.endian != byteorder: diff --git a/src/zarr/codecs/crc32c_.py b/src/zarr/codecs/crc32c_.py index dd61b3425..377ba688f 100644 --- a/src/zarr/codecs/crc32c_.py +++ b/src/zarr/codecs/crc32c_.py @@ -1,10 +1,9 @@ from __future__ import annotations -from dataclasses import dataclass +from dataclasses import dataclass from typing import TYPE_CHECKING import numpy as np - from crc32c import crc32c from zarr.abc.codec import BytesBytesCodec @@ -12,9 +11,9 @@ from zarr.common import parse_named_configuration if TYPE_CHECKING: - from typing import Dict, Optional from typing_extensions import Self - from zarr.common import JSON, BytesLike, ArraySpec + + from zarr.common import JSON, ArraySpec, BytesLike @dataclass(frozen=True) @@ -22,11 +21,11 @@ class Crc32cCodec(BytesBytesCodec): is_fixed_size = True @classmethod - def from_dict(cls, data: Dict[str, JSON]) -> Self: + def from_dict(cls, data: dict[str, JSON]) -> Self: parse_named_configuration(data, "crc32c", require_configuration=False) return cls() - def to_dict(self) -> Dict[str, JSON]: + def to_dict(self) -> dict[str, JSON]: return {"name": "crc32c"} async def decode( @@ -50,7 +49,7 @@ async def encode( self, chunk_bytes: bytes, _chunk_spec: ArraySpec, - ) -> Optional[BytesLike]: + ) -> BytesLike | None: return chunk_bytes + np.uint32(crc32c(chunk_bytes)).tobytes() def compute_encoded_size(self, input_byte_length: int, _chunk_spec: ArraySpec) -> int: diff --git a/src/zarr/codecs/gzip.py b/src/zarr/codecs/gzip.py index 71dcaa6bb..a492852b4 100644 --- a/src/zarr/codecs/gzip.py +++ b/src/zarr/codecs/gzip.py @@ -1,16 +1,17 @@ from __future__ import annotations -from dataclasses import dataclass +from dataclasses import dataclass from typing import TYPE_CHECKING from numcodecs.gzip import GZip + from zarr.abc.codec import BytesBytesCodec from zarr.codecs.registry import register_codec from zarr.common import parse_named_configuration, to_thread if TYPE_CHECKING: - from typing import Optional, Dict from typing_extensions import Self + from zarr.common import JSON, ArraySpec, BytesLike @@ -36,11 +37,11 @@ def __init__(self, *, level: int = 5) -> None: object.__setattr__(self, "level", level_parsed) @classmethod - def from_dict(cls, data: Dict[str, JSON]) -> Self: + def from_dict(cls, data: dict[str, JSON]) -> Self: _, configuration_parsed = parse_named_configuration(data, "gzip") return cls(**configuration_parsed) # type: ignore[arg-type] - def to_dict(self) -> Dict[str, JSON]: + def to_dict(self) -> dict[str, JSON]: return {"name": "gzip", "configuration": {"level": self.level}} async def decode( @@ -54,7 +55,7 @@ async def encode( self, chunk_bytes: bytes, _chunk_spec: ArraySpec, - ) -> Optional[BytesLike]: + ) -> BytesLike | None: return await to_thread(GZip(self.level).encode, chunk_bytes) def compute_encoded_size( diff --git a/src/zarr/codecs/pipeline.py b/src/zarr/codecs/pipeline.py index da131868c..c2ce7b193 100644 --- a/src/zarr/codecs/pipeline.py +++ b/src/zarr/codecs/pipeline.py @@ -1,10 +1,12 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Iterable -import numpy as np +from collections.abc import Iterable from dataclasses import dataclass +from typing import TYPE_CHECKING from warnings import warn +import numpy as np + from zarr.abc.codec import ( ArrayArrayCodec, ArrayBytesCodec, @@ -18,21 +20,22 @@ from zarr.common import parse_named_configuration if TYPE_CHECKING: - from typing import Iterator, List, Optional, Tuple, Union - from zarr.store import StorePath - from zarr.metadata import ArrayMetadata + from collections.abc import Iterator + from zarr.common import JSON, ArraySpec, BytesLike, SliceSelection + from zarr.metadata import ArrayMetadata + from zarr.store import StorePath @dataclass(frozen=True) class CodecPipeline(Metadata): - array_array_codecs: Tuple[ArrayArrayCodec, ...] + array_array_codecs: tuple[ArrayArrayCodec, ...] array_bytes_codec: ArrayBytesCodec - bytes_bytes_codecs: Tuple[BytesBytesCodec, ...] + bytes_bytes_codecs: tuple[BytesBytesCodec, ...] @classmethod - def from_dict(cls, data: Iterable[Union[JSON, Codec]]) -> CodecPipeline: - out: List[Codec] = [] + def from_dict(cls, data: Iterable[JSON | Codec]) -> CodecPipeline: + out: list[Codec] = [] if not isinstance(data, Iterable): raise TypeError(f"Expected iterable, got {type(data)}") @@ -51,13 +54,13 @@ def evolve(self, array_spec: ArraySpec) -> CodecPipeline: return CodecPipeline.from_list([c.evolve(array_spec) for c in self]) @classmethod - def from_list(cls, codecs: List[Codec]) -> CodecPipeline: + def from_list(cls, codecs: list[Codec]) -> CodecPipeline: from zarr.codecs.sharding import ShardingCodec if not any(isinstance(codec, ArrayBytesCodec) for codec in codecs): raise ValueError("Exactly one array-to-bytes codec is required.") - prev_codec: Optional[Codec] = None + prev_codec: Codec | None = None for codec in codecs: if prev_codec is not None: if isinstance(codec, ArrayBytesCodec) and isinstance(prev_codec, ArrayBytesCodec): @@ -86,7 +89,8 @@ def from_list(cls, codecs: List[Codec]) -> CodecPipeline: if any(isinstance(codec, ShardingCodec) for codec in codecs) and len(codecs) > 1: warn( "Combining a `sharding_indexed` codec disables partial reads and " - + "writes, which may lead to inefficient performance." + + "writes, which may lead to inefficient performance.", + stacklevel=3, ) return CodecPipeline( @@ -112,13 +116,9 @@ def supports_partial_encode(self) -> bool: ) def __iter__(self) -> Iterator[Codec]: - for aa_codec in self.array_array_codecs: - yield aa_codec - + yield from self.array_array_codecs yield self.array_bytes_codec - - for bb_codec in self.bytes_bytes_codecs: - yield bb_codec + yield from self.bytes_bytes_codecs def validate(self, array_metadata: ArrayMetadata) -> None: for codec in self: @@ -126,12 +126,12 @@ def validate(self, array_metadata: ArrayMetadata) -> None: def _codecs_with_resolved_metadata( self, array_spec: ArraySpec - ) -> Tuple[ - List[Tuple[ArrayArrayCodec, ArraySpec]], - Tuple[ArrayBytesCodec, ArraySpec], - List[Tuple[BytesBytesCodec, ArraySpec]], + ) -> tuple[ + list[tuple[ArrayArrayCodec, ArraySpec]], + tuple[ArrayBytesCodec, ArraySpec], + list[tuple[BytesBytesCodec, ArraySpec]], ]: - aa_codecs_with_spec: List[Tuple[ArrayArrayCodec, ArraySpec]] = [] + aa_codecs_with_spec: list[tuple[ArrayArrayCodec, ArraySpec]] = [] for aa_codec in self.array_array_codecs: aa_codecs_with_spec.append((aa_codec, array_spec)) array_spec = aa_codec.resolve_metadata(array_spec) @@ -139,7 +139,7 @@ def _codecs_with_resolved_metadata( ab_codec_with_spec = (self.array_bytes_codec, array_spec) array_spec = self.array_bytes_codec.resolve_metadata(array_spec) - bb_codecs_with_spec: List[Tuple[BytesBytesCodec, ArraySpec]] = [] + bb_codecs_with_spec: list[tuple[BytesBytesCodec, ArraySpec]] = [] for bb_codec in self.bytes_bytes_codecs: bb_codecs_with_spec.append((bb_codec, array_spec)) array_spec = bb_codec.resolve_metadata(array_spec) @@ -173,7 +173,7 @@ async def decode_partial( store_path: StorePath, selection: SliceSelection, chunk_spec: ArraySpec, - ) -> Optional[np.ndarray]: + ) -> np.ndarray | None: assert self.supports_partial_decode assert isinstance(self.array_bytes_codec, ArrayBytesCodecPartialDecodeMixin) return await self.array_bytes_codec.decode_partial(store_path, selection, chunk_spec) @@ -182,7 +182,7 @@ async def encode( self, chunk_array: np.ndarray, array_spec: ArraySpec, - ) -> Optional[BytesLike]: + ) -> BytesLike | None: ( aa_codecs_with_spec, ab_codec_with_spec, diff --git a/src/zarr/codecs/registry.py b/src/zarr/codecs/registry.py index 7d4604125..2f2b09499 100644 --- a/src/zarr/codecs/registry.py +++ b/src/zarr/codecs/registry.py @@ -1,29 +1,29 @@ from __future__ import annotations + from typing import TYPE_CHECKING if TYPE_CHECKING: - from typing import Dict, Type from zarr.abc.codec import Codec -from importlib.metadata import EntryPoint, entry_points as get_entry_points - +from importlib.metadata import EntryPoint +from importlib.metadata import entry_points as get_entry_points -__codec_registry: Dict[str, Type[Codec]] = {} -__lazy_load_codecs: Dict[str, EntryPoint] = {} +__codec_registry: dict[str, type[Codec]] = {} +__lazy_load_codecs: dict[str, EntryPoint] = {} -def _collect_entrypoints() -> Dict[str, EntryPoint]: +def _collect_entrypoints() -> dict[str, EntryPoint]: entry_points = get_entry_points() for e in entry_points.select(group="zarr.codecs"): __lazy_load_codecs[e.name] = e return __lazy_load_codecs -def register_codec(key: str, codec_cls: Type[Codec]) -> None: +def register_codec(key: str, codec_cls: type[Codec]) -> None: __codec_registry[key] = codec_cls -def get_codec_class(key: str) -> Type[Codec]: +def get_codec_class(key: str) -> type[Codec]: item = __codec_registry.get(key) if item is None: if key in __lazy_load_codecs: diff --git a/src/zarr/codecs/sharding.py b/src/zarr/codecs/sharding.py index e94074e63..86d139576 100644 --- a/src/zarr/codecs/sharding.py +++ b/src/zarr/codecs/sharding.py @@ -1,17 +1,20 @@ from __future__ import annotations -from enum import Enum -from typing import TYPE_CHECKING, Iterable, Mapping, NamedTuple, Union + +from collections.abc import Iterable, Mapping from dataclasses import dataclass, replace +from enum import Enum from functools import lru_cache - +from typing import TYPE_CHECKING, NamedTuple import numpy as np + from zarr.abc.codec import ( - Codec, ArrayBytesCodec, ArrayBytesCodecPartialDecodeMixin, ArrayBytesCodecPartialEncodeMixin, + Codec, ) +from zarr.chunk_grids import RegularChunkGrid from zarr.codecs.bytes import BytesCodec from zarr.codecs.crc32c_ import Crc32cCodec from zarr.codecs.pipeline import CodecPipeline @@ -26,7 +29,6 @@ product, ) from zarr.config import config -from zarr.chunk_grids import RegularChunkGrid from zarr.indexing import ( BasicIndexer, c_order_iter, @@ -39,16 +41,17 @@ ) if TYPE_CHECKING: - from typing import Awaitable, Callable, Dict, Iterator, List, Optional, Set, Tuple + from collections.abc import Awaitable, Callable, Iterator + from typing_extensions import Self - from zarr.store import StorePath from zarr.common import ( JSON, - ChunkCoords, BytesLike, + ChunkCoords, SliceSelection, ) + from zarr.store import StorePath MAX_UINT_64 = 2**64 - 1 @@ -73,13 +76,13 @@ def chunks_per_shard(self) -> ChunkCoords: def _localize_chunk(self, chunk_coords: ChunkCoords) -> ChunkCoords: return tuple( chunk_i % shard_i - for chunk_i, shard_i in zip(chunk_coords, self.offsets_and_lengths.shape) + for chunk_i, shard_i in zip(chunk_coords, self.offsets_and_lengths.shape, strict=False) ) def is_all_empty(self) -> bool: return bool(np.array_equiv(self.offsets_and_lengths, MAX_UINT_64)) - def get_chunk_slice(self, chunk_coords: ChunkCoords) -> Optional[Tuple[int, int]]: + def get_chunk_slice(self, chunk_coords: ChunkCoords) -> tuple[int, int] | None: localized_chunk = self._localize_chunk(chunk_coords) chunk_start, chunk_len = self.offsets_and_lengths[localized_chunk] if (chunk_start, chunk_len) == (MAX_UINT_64, MAX_UINT_64): @@ -87,7 +90,7 @@ def get_chunk_slice(self, chunk_coords: ChunkCoords) -> Optional[Tuple[int, int] else: return (int(chunk_start), int(chunk_start) + int(chunk_len)) - def set_chunk_slice(self, chunk_coords: ChunkCoords, chunk_slice: Optional[slice]) -> None: + def set_chunk_slice(self, chunk_coords: ChunkCoords, chunk_slice: slice | None) -> None: localized_chunk = self._localize_chunk(chunk_coords) if chunk_slice is None: self.offsets_and_lengths[localized_chunk] = (MAX_UINT_64, MAX_UINT_64) @@ -152,7 +155,7 @@ def create_empty(cls, chunks_per_shard: ChunkCoords) -> _ShardProxy: obj.index = index return obj - def __getitem__(self, chunk_coords: ChunkCoords) -> Optional[BytesLike]: + def __getitem__(self, chunk_coords: ChunkCoords) -> BytesLike | None: chunk_byte_slice = self.index.get_chunk_slice(chunk_coords) if chunk_byte_slice: return self.buf[chunk_byte_slice[0] : chunk_byte_slice[1]] @@ -173,7 +176,7 @@ class _ShardBuilder(_ShardProxy): def merge_with_morton_order( cls, chunks_per_shard: ChunkCoords, - tombstones: Set[ChunkCoords], + tombstones: set[ChunkCoords], *shard_dicts: Mapping[ChunkCoords, BytesLike], ) -> _ShardBuilder: obj = cls.create_empty(chunks_per_shard) @@ -230,9 +233,9 @@ def __init__( self, *, chunk_shape: ChunkCoordsLike, - codecs: Optional[Iterable[Union[Codec, JSON]]] = None, - index_codecs: Optional[Iterable[Union[Codec, JSON]]] = None, - index_location: Optional[ShardingCodecIndexLocation] = ShardingCodecIndexLocation.end, + codecs: Iterable[Codec | JSON] | None = None, + index_codecs: Iterable[Codec | JSON] | None = None, + index_location: ShardingCodecIndexLocation | None = ShardingCodecIndexLocation.end, ) -> None: chunk_shape_parsed = parse_shapelike(chunk_shape) codecs_parsed = ( @@ -255,11 +258,11 @@ def __init__( object.__setattr__(self, "index_location", index_location_parsed) @classmethod - def from_dict(cls, data: Dict[str, JSON]) -> Self: + def from_dict(cls, data: dict[str, JSON]) -> Self: _, configuration_parsed = parse_named_configuration(data, "sharding_indexed") return cls(**configuration_parsed) # type: ignore[arg-type] - def to_dict(self) -> Dict[str, JSON]: + def to_dict(self) -> dict[str, JSON]: return { "name": "sharding_indexed", "configuration": { @@ -290,6 +293,7 @@ def validate(self, array_metadata: ArrayMetadata) -> None: for s, c in zip( array_metadata.chunk_grid.chunk_shape, self.chunk_shape, + strict=False, ) ): raise ValueError( @@ -349,7 +353,7 @@ async def decode_partial( store_path: StorePath, selection: SliceSelection, shard_spec: ArraySpec, - ) -> Optional[np.ndarray]: + ) -> np.ndarray | None: shard_shape = shard_spec.shape chunk_shape = self.chunk_shape chunks_per_shard = self._get_chunks_per_shard(shard_spec) @@ -412,7 +416,7 @@ async def decode_partial( async def _read_chunk( self, - shard_dict: Mapping[ChunkCoords, Optional[BytesLike]], + shard_dict: Mapping[ChunkCoords, BytesLike | None], chunk_coords: ChunkCoords, chunk_selection: SliceSelection, out_selection: SliceSelection, @@ -432,7 +436,7 @@ async def encode( self, shard_array: np.ndarray, shard_spec: ArraySpec, - ) -> Optional[BytesLike]: + ) -> BytesLike | None: shard_shape = shard_spec.shape chunk_shape = self.chunk_shape chunks_per_shard = self._get_chunks_per_shard(shard_spec) @@ -450,7 +454,7 @@ async def _write_chunk( chunk_coords: ChunkCoords, chunk_selection: SliceSelection, out_selection: SliceSelection, - ) -> Tuple[ChunkCoords, Optional[BytesLike]]: + ) -> tuple[ChunkCoords, BytesLike | None]: if is_total_slice(chunk_selection, chunk_shape): chunk_array = shard_array[out_selection] else: @@ -470,7 +474,7 @@ async def _write_chunk( return (chunk_coords, None) # assembling and encoding chunks within the shard - encoded_chunks: List[Tuple[ChunkCoords, Optional[BytesLike]]] = await concurrent_map( + encoded_chunks: list[tuple[ChunkCoords, BytesLike | None]] = await concurrent_map( [ (shard_array, chunk_coords, chunk_selection, out_selection) for chunk_coords, chunk_selection, out_selection in indexer @@ -505,7 +509,7 @@ async def encode_partial( await self._load_full_shard_maybe(store_path, chunks_per_shard) ) or _ShardProxy.create_empty(chunks_per_shard) new_shard_builder = _ShardBuilder.create_empty(chunks_per_shard) - tombstones: Set[ChunkCoords] = set() + tombstones: set[ChunkCoords] = set() indexer = list( BasicIndexer( @@ -519,7 +523,7 @@ async def _write_chunk( chunk_coords: ChunkCoords, chunk_selection: SliceSelection, out_selection: SliceSelection, - ) -> Tuple[ChunkCoords, Optional[BytesLike]]: + ) -> tuple[ChunkCoords, BytesLike | None]: chunk_array = None if is_total_slice(chunk_selection, self.chunk_shape): chunk_array = shard_array[out_selection] @@ -549,7 +553,7 @@ async def _write_chunk( else: return (chunk_coords, None) - encoded_chunks: List[Tuple[ChunkCoords, Optional[BytesLike]]] = await concurrent_map( + encoded_chunks: list[tuple[ChunkCoords, BytesLike | None]] = await concurrent_map( [ ( chunk_coords, @@ -586,7 +590,7 @@ async def _write_chunk( ) def _is_total_shard( - self, all_chunk_coords: Set[ChunkCoords], chunks_per_shard: ChunkCoords + self, all_chunk_coords: set[ChunkCoords], chunks_per_shard: ChunkCoords ) -> bool: return len(all_chunk_coords) == product(chunks_per_shard) and all( chunk_coords in all_chunk_coords for chunk_coords in c_order_iter(chunks_per_shard) @@ -615,7 +619,7 @@ def _shard_index_size(self, chunks_per_shard: ChunkCoords) -> int: 16 * product(chunks_per_shard), self._get_index_chunk_spec(chunks_per_shard) ) - @lru_cache + @lru_cache # noqa: B019 def _get_index_chunk_spec(self, chunks_per_shard: ChunkCoords) -> ArraySpec: return ArraySpec( shape=chunks_per_shard + (2,), @@ -624,7 +628,7 @@ def _get_index_chunk_spec(self, chunks_per_shard: ChunkCoords) -> ArraySpec: order="C", # Note: this is hard-coded for simplicity -- it is not surfaced into user code ) - @lru_cache + @lru_cache # noqa: B019 def _get_chunk_spec(self, shard_spec: ArraySpec) -> ArraySpec: return ArraySpec( shape=self.chunk_shape, @@ -633,19 +637,20 @@ def _get_chunk_spec(self, shard_spec: ArraySpec) -> ArraySpec: order=shard_spec.order, ) - @lru_cache + @lru_cache # noqa: B019 def _get_chunks_per_shard(self, shard_spec: ArraySpec) -> ChunkCoords: return tuple( s // c for s, c in zip( shard_spec.shape, self.chunk_shape, + strict=False, ) ) async def _load_shard_index_maybe( self, store_path: StorePath, chunks_per_shard: ChunkCoords - ) -> Optional[_ShardIndex]: + ) -> _ShardIndex | None: shard_index_size = self._shard_index_size(chunks_per_shard) if self.index_location == ShardingCodecIndexLocation.start: index_bytes = await store_path.get((0, shard_index_size)) @@ -664,7 +669,7 @@ async def _load_shard_index( async def _load_full_shard_maybe( self, store_path: StorePath, chunks_per_shard: ChunkCoords - ) -> Optional[_ShardProxy]: + ) -> _ShardProxy | None: shard_bytes = await store_path.get() return ( diff --git a/src/zarr/codecs/transpose.py b/src/zarr/codecs/transpose.py index a13708955..d9f873329 100644 --- a/src/zarr/codecs/transpose.py +++ b/src/zarr/codecs/transpose.py @@ -1,12 +1,14 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Dict, Iterable, Union, cast +from collections.abc import Iterable from dataclasses import dataclass, replace +from typing import TYPE_CHECKING, cast from zarr.common import JSON, ArraySpec, ChunkCoordsLike, parse_named_configuration if TYPE_CHECKING: - from typing import TYPE_CHECKING, Optional, Tuple + from typing import TYPE_CHECKING + from typing_extensions import Self import numpy as np @@ -15,7 +17,7 @@ from zarr.codecs.registry import register_codec -def parse_transpose_order(data: Union[JSON, Iterable[int]]) -> Tuple[int, ...]: +def parse_transpose_order(data: JSON | Iterable[int]) -> tuple[int, ...]: if not isinstance(data, Iterable): raise TypeError(f"Expected an iterable. Got {data} instead.") if not all(isinstance(a, int) for a in data): @@ -27,7 +29,7 @@ def parse_transpose_order(data: Union[JSON, Iterable[int]]) -> Tuple[int, ...]: class TransposeCodec(ArrayArrayCodec): is_fixed_size = True - order: Tuple[int, ...] + order: tuple[int, ...] def __init__(self, *, order: ChunkCoordsLike) -> None: order_parsed = parse_transpose_order(order) @@ -35,11 +37,11 @@ def __init__(self, *, order: ChunkCoordsLike) -> None: object.__setattr__(self, "order", order_parsed) @classmethod - def from_dict(cls, data: Dict[str, JSON]) -> Self: + def from_dict(cls, data: dict[str, JSON]) -> Self: _, configuration_parsed = parse_named_configuration(data, "transpose") return cls(**configuration_parsed) # type: ignore[arg-type] - def to_dict(self) -> Dict[str, JSON]: + def to_dict(self) -> dict[str, JSON]: return {"name": "transpose", "configuration": {"order": list(self.order)}} def evolve(self, array_spec: ArraySpec) -> Self: @@ -88,7 +90,7 @@ async def encode( self, chunk_array: np.ndarray, chunk_spec: ArraySpec, - ) -> Optional[np.ndarray]: + ) -> np.ndarray | None: chunk_array = chunk_array.transpose(self.order) return chunk_array diff --git a/src/zarr/codecs/zstd.py b/src/zarr/codecs/zstd.py index ad10a7fdb..a2ff69099 100644 --- a/src/zarr/codecs/zstd.py +++ b/src/zarr/codecs/zstd.py @@ -1,7 +1,7 @@ from __future__ import annotations -from typing import TYPE_CHECKING -from dataclasses import dataclass +from dataclasses import dataclass +from typing import TYPE_CHECKING from zstandard import ZstdCompressor, ZstdDecompressor @@ -10,9 +10,9 @@ from zarr.common import parse_named_configuration, to_thread if TYPE_CHECKING: - from typing import Dict, Optional from typing_extensions import Self - from zarr.common import BytesLike, JSON, ArraySpec + + from zarr.common import JSON, ArraySpec, BytesLike def parse_zstd_level(data: JSON) -> int: @@ -44,11 +44,11 @@ def __init__(self, *, level: int = 0, checksum: bool = False) -> None: object.__setattr__(self, "checksum", checksum_parsed) @classmethod - def from_dict(cls, data: Dict[str, JSON]) -> Self: + def from_dict(cls, data: dict[str, JSON]) -> Self: _, configuration_parsed = parse_named_configuration(data, "zstd") return cls(**configuration_parsed) # type: ignore[arg-type] - def to_dict(self) -> Dict[str, JSON]: + def to_dict(self) -> dict[str, JSON]: return {"name": "zstd", "configuration": {"level": self.level, "checksum": self.checksum}} def _compress(self, data: bytes) -> bytes: @@ -70,7 +70,7 @@ async def encode( self, chunk_bytes: bytes, _chunk_spec: ArraySpec, - ) -> Optional[BytesLike]: + ) -> BytesLike | None: return await to_thread(self._compress, chunk_bytes) def compute_encoded_size(self, _input_byte_length: int, _chunk_spec: ArraySpec) -> int: diff --git a/src/zarr/common.py b/src/zarr/common.py index ea26cae7b..371f6784b 100644 --- a/src/zarr/common.py +++ b/src/zarr/common.py @@ -1,24 +1,22 @@ from __future__ import annotations + +import asyncio +import contextvars +import functools +from collections.abc import Iterable +from dataclasses import dataclass +from enum import Enum from typing import ( TYPE_CHECKING, + Any, Literal, - Union, - Tuple, - Iterable, - Dict, - List, TypeVar, overload, - Any, ) -import asyncio -import contextvars -from dataclasses import dataclass -from enum import Enum -import functools if TYPE_CHECKING: - from typing import Any, Awaitable, Callable, Iterator, Optional, Type + from collections.abc import Awaitable, Callable, Iterator + from typing import Any import numpy as np @@ -27,25 +25,25 @@ ZGROUP_JSON = ".zgroup" ZATTRS_JSON = ".zattrs" -BytesLike = Union[bytes, bytearray, memoryview] -ChunkCoords = Tuple[int, ...] +BytesLike = bytes | bytearray | memoryview +ChunkCoords = tuple[int, ...] ChunkCoordsLike = Iterable[int] -SliceSelection = Tuple[slice, ...] -Selection = Union[slice, SliceSelection] -JSON = Union[str, None, int, float, Enum, Dict[str, "JSON"], List["JSON"], Tuple["JSON", ...]] +SliceSelection = tuple[slice, ...] +Selection = slice | SliceSelection +JSON = str | None | int | float | Enum | dict[str, "JSON"] | list["JSON"] | tuple["JSON", ...] def product(tup: ChunkCoords) -> int: return functools.reduce(lambda x, y: x * y, tup, 1) -T = TypeVar("T", bound=Tuple[Any, ...]) +T = TypeVar("T", bound=tuple[Any, ...]) V = TypeVar("V") async def concurrent_map( - items: List[T], func: Callable[..., Awaitable[V]], limit: Optional[int] = None -) -> List[V]: + items: list[T], func: Callable[..., Awaitable[V]], limit: int | None = None +) -> list[V]: if limit is None: return await asyncio.gather(*[func(*item) for item in items]) @@ -69,12 +67,12 @@ async def to_thread(func: Callable[..., V], /, *args: Any, **kwargs: Any) -> V: E = TypeVar("E", bound=Enum) -def enum_names(enum: Type[E]) -> Iterator[str]: +def enum_names(enum: type[E]) -> Iterator[str]: for item in enum: yield item.name -def parse_enum(data: JSON, cls: Type[E]) -> E: +def parse_enum(data: JSON, cls: type[E]) -> E: if isinstance(data, cls): return data if not isinstance(data, str): @@ -109,7 +107,7 @@ def ndim(self) -> int: return len(self.shape) -def parse_name(data: JSON, expected: Optional[str] = None) -> str: +def parse_name(data: JSON, expected: str | None = None) -> str: if isinstance(data, str): if expected is None or data == expected: return data @@ -126,19 +124,19 @@ def parse_configuration(data: JSON) -> JSON: @overload def parse_named_configuration( - data: JSON, expected_name: Optional[str] = None -) -> Tuple[str, Dict[str, JSON]]: ... + data: JSON, expected_name: str | None = None +) -> tuple[str, dict[str, JSON]]: ... @overload def parse_named_configuration( - data: JSON, expected_name: Optional[str] = None, *, require_configuration: bool = True -) -> Tuple[str, Optional[Dict[str, JSON]]]: ... + data: JSON, expected_name: str | None = None, *, require_configuration: bool = True +) -> tuple[str, dict[str, JSON] | None]: ... def parse_named_configuration( - data: JSON, expected_name: Optional[str] = None, *, require_configuration: bool = True -) -> Tuple[str, Optional[JSON]]: + data: JSON, expected_name: str | None = None, *, require_configuration: bool = True +) -> tuple[str, JSON | None]: if not isinstance(data, dict): raise TypeError(f"Expected dict, got {type(data)}") if "name" not in data: @@ -153,7 +151,7 @@ def parse_named_configuration( return name_parsed, configuration_parsed -def parse_shapelike(data: Any) -> Tuple[int, ...]: +def parse_shapelike(data: Any) -> tuple[int, ...]: if not isinstance(data, Iterable): raise TypeError(f"Expected an iterable. Got {data} instead.") data_tuple = tuple(data) diff --git a/src/zarr/group.py b/src/zarr/group.py index c71860b1b..a7c554572 100644 --- a/src/zarr/group.py +++ b/src/zarr/group.py @@ -1,21 +1,19 @@ from __future__ import annotations -from typing import TYPE_CHECKING -from dataclasses import asdict, dataclass, field, replace import asyncio import json import logging +from dataclasses import asdict, dataclass, field, replace +from typing import TYPE_CHECKING if TYPE_CHECKING: + from collections.abc import AsyncGenerator, AsyncIterator from typing import ( Any, - AsyncGenerator, Literal, - AsyncIterator, ) from zarr.abc.metadata import Metadata - -from zarr.array import AsyncArray, Array +from zarr.array import Array, AsyncArray from zarr.attributes import Attributes from zarr.common import ZARR_JSON, ZARRAY_JSON, ZATTRS_JSON, ZGROUP_JSON from zarr.store import StoreLike, StorePath, make_store_path @@ -83,7 +81,7 @@ async def create( cls, store: StoreLike, *, - attributes: dict[str, Any] = {}, + attributes: dict[str, Any] = {}, # noqa: B006, FIXME exists_ok: bool = False, zarr_format: Literal[2, 3] = 3, ) -> AsyncGroup: @@ -337,7 +335,7 @@ async def group_keys(self) -> AsyncGenerator[str, None]: # todo: decide if this method should be separate from `group_keys` async def groups(self) -> AsyncGenerator[AsyncGroup, None]: - async for key, value in self.members(): + async for _, value in self.members(): if isinstance(value, AsyncGroup): yield value @@ -349,7 +347,7 @@ async def array_keys(self) -> AsyncGenerator[str, None]: # todo: decide if this method should be separate from `array_keys` async def arrays(self) -> AsyncIterator[AsyncArray]: - async for key, value in self.members(): + async for _, value in self.members(): if isinstance(value, AsyncArray): yield value @@ -393,7 +391,7 @@ def create( cls, store: StoreLike, *, - attributes: dict[str, Any] = {}, + attributes: dict[str, Any] = {}, # noqa: B006, FIXME exists_ok: bool = False, ) -> Group: obj = sync( diff --git a/src/zarr/indexing.py b/src/zarr/indexing.py index 9f324eb5e..af82b6bb5 100644 --- a/src/zarr/indexing.py +++ b/src/zarr/indexing.py @@ -2,7 +2,8 @@ import itertools import math -from typing import Iterator, List, NamedTuple, Optional, Tuple +from collections.abc import Iterator +from typing import NamedTuple from zarr.common import ChunkCoords, Selection, SliceSelection, product @@ -14,9 +15,7 @@ def _ensure_tuple(v: Selection) -> SliceSelection: def _err_too_many_indices(selection: SliceSelection, shape: ChunkCoords) -> None: - raise IndexError( - "too many indices for array; expected {}, got {}".format(len(shape), len(selection)) - ) + raise IndexError(f"too many indices for array; expected {len(shape)}, got {len(selection)}") def _err_negative_step() -> None: @@ -47,7 +46,7 @@ def _ensure_selection( class _ChunkDimProjection(NamedTuple): dim_chunk_ix: int dim_chunk_sel: slice - dim_out_sel: Optional[slice] + dim_out_sel: slice | None def _ceildiv(a: float, b: float) -> int: @@ -124,20 +123,20 @@ class _ChunkProjection(NamedTuple): class BasicIndexer: - dim_indexers: List[_SliceDimIndexer] + dim_indexers: list[_SliceDimIndexer] shape: ChunkCoords def __init__( self, selection: Selection, - shape: Tuple[int, ...], - chunk_shape: Tuple[int, ...], + shape: tuple[int, ...], + chunk_shape: tuple[int, ...], ): # setup per-dimension indexers self.dim_indexers = [ _SliceDimIndexer(dim_sel, dim_len, dim_chunk_len) for dim_sel, dim_len, dim_chunk_len in zip( - _ensure_selection(selection, shape), shape, chunk_shape + _ensure_selection(selection, shape), shape, chunk_shape, strict=False ) ] self.shape = tuple(s.nitems for s in self.dim_indexers) @@ -198,11 +197,13 @@ def is_total_slice(item: Selection, shape: ChunkCoords) -> bool: or ((dim_sel.stop - dim_sel.start == dim_len) and (dim_sel.step in [1, None])) ) ) - for dim_sel, dim_len in zip(item, shape) + for dim_sel, dim_len in zip(item, shape, strict=False) ) else: - raise TypeError("expected slice or tuple of slices, found %r" % item) + raise TypeError("expected slice or tuple of slices, found {item!r}") def all_chunk_coords(shape: ChunkCoords, chunk_shape: ChunkCoords) -> Iterator[ChunkCoords]: - return itertools.product(*(range(0, _ceildiv(s, c)) for s, c in zip(shape, chunk_shape))) + return itertools.product( + *(range(0, _ceildiv(s, c)) for s, c in zip(shape, chunk_shape, strict=False)) + ) diff --git a/src/zarr/metadata.py b/src/zarr/metadata.py index 3903bacd4..a7d0ffbe3 100644 --- a/src/zarr/metadata.py +++ b/src/zarr/metadata.py @@ -1,23 +1,25 @@ from __future__ import annotations -from enum import Enum -from typing import TYPE_CHECKING, cast, Dict, Iterable, Any -from dataclasses import dataclass, field + import json +from collections.abc import Iterable +from dataclasses import dataclass, field +from enum import Enum +from typing import TYPE_CHECKING, Any, cast + import numpy as np import numpy.typing as npt from zarr.chunk_grids import ChunkGrid, RegularChunkGrid from zarr.chunk_key_encodings import ChunkKeyEncoding, parse_separator - if TYPE_CHECKING: - from typing import Literal, Union, List, Optional, Tuple + from typing import Literal + from zarr.codecs.pipeline import CodecPipeline from zarr.abc.codec import Codec from zarr.abc.metadata import Metadata - from zarr.common import ( JSON, ArraySpec, @@ -28,7 +30,6 @@ ) from zarr.config import parse_indexing_order - # For type checking _bool = bool @@ -111,8 +112,8 @@ class ArrayMetadata(Metadata): chunk_key_encoding: ChunkKeyEncoding fill_value: Any codecs: CodecPipeline - attributes: Dict[str, Any] = field(default_factory=dict) - dimension_names: Optional[Tuple[str, ...]] = None + attributes: dict[str, Any] = field(default_factory=dict) + dimension_names: tuple[str, ...] | None = None zarr_format: Literal[3] = field(default=3, init=False) node_type: Literal["array"] = field(default="array", init=False) @@ -210,7 +211,7 @@ def _json_convert(o): ).encode() @classmethod - def from_dict(cls, data: Dict[str, Any]) -> ArrayMetadata: + def from_dict(cls, data: dict[str, Any]) -> ArrayMetadata: # check that the zarr_format attribute is correct _ = parse_zarr_format_v3(data.pop("zarr_format")) # check that the node_type attribute is correct @@ -220,7 +221,7 @@ def from_dict(cls, data: Dict[str, Any]) -> ArrayMetadata: return cls(**data, dimension_names=dimension_names) - def to_dict(self) -> Dict[str, Any]: + def to_dict(self) -> dict[str, Any]: out_dict = super().to_dict() if not isinstance(out_dict, dict): @@ -238,12 +239,12 @@ class ArrayV2Metadata(Metadata): shape: ChunkCoords chunks: ChunkCoords dtype: np.dtype[Any] - fill_value: Union[None, int, float] = 0 + fill_value: None | int | float = 0 order: Literal["C", "F"] = "C" - filters: Optional[List[Dict[str, Any]]] = None + filters: list[dict[str, Any]] | None = None dimension_separator: Literal[".", "/"] = "." - compressor: Optional[Dict[str, Any]] = None - attributes: Optional[Dict[str, Any]] = cast(Dict[str, Any], field(default_factory=dict)) + compressor: dict[str, Any] | None = None + attributes: dict[str, Any] | None = cast(dict[str, Any], field(default_factory=dict)) zarr_format: Literal[2] = field(init=False, default=2) def __init__( @@ -255,9 +256,9 @@ def __init__( fill_value: Any, order: Literal["C", "F"], dimension_separator: Literal[".", "/"] = ".", - compressor: Optional[Dict[str, Any]] = None, - filters: Optional[List[Dict[str, Any]]] = None, - attributes: Optional[Dict[str, JSON]] = None, + compressor: dict[str, Any] | None = None, + filters: list[dict[str, Any]] | None = None, + attributes: dict[str, JSON] | None = None, ): """ Metadata for a Zarr version 2 array. @@ -301,13 +302,13 @@ def _json_convert(o): return json.dumps(self.to_dict(), default=_json_convert).encode() @classmethod - def from_dict(cls, data: Dict[str, Any]) -> ArrayV2Metadata: + def from_dict(cls, data: dict[str, Any]) -> ArrayV2Metadata: # check that the zarr_format attribute is correct _ = parse_zarr_format_v2(data.pop("zarr_format")) return cls(**data) -def parse_dimension_names(data: Any) -> Tuple[str, ...] | None: +def parse_dimension_names(data: Any) -> tuple[str, ...] | None: if data is None: return data if isinstance(data, Iterable) and all([isinstance(x, str) for x in data]): @@ -317,11 +318,11 @@ def parse_dimension_names(data: Any) -> Tuple[str, ...] | None: # todo: real validation -def parse_attributes(data: Any) -> Dict[str, JSON]: +def parse_attributes(data: Any) -> dict[str, JSON]: if data is None: return {} - data_json = cast(Dict[str, JSON], data) + data_json = cast(dict[str, JSON], data) return data_json @@ -348,7 +349,7 @@ def parse_node_type_array(data: Any) -> Literal["array"]: # todo: real validation -def parse_filters(data: Any) -> List[Codec]: +def parse_filters(data: Any) -> list[Codec]: return data @@ -367,7 +368,7 @@ def parse_v2_metadata(data: ArrayV2Metadata) -> ArrayV2Metadata: return data -def parse_codecs(data: Iterable[Union[Codec, JSON]]) -> CodecPipeline: +def parse_codecs(data: Iterable[Codec | JSON]) -> CodecPipeline: from zarr.codecs.pipeline import CodecPipeline if not isinstance(data, Iterable): diff --git a/src/zarr/store/core.py b/src/zarr/store/core.py index cc017ec98..9e2da48c7 100644 --- a/src/zarr/store/core.py +++ b/src/zarr/store/core.py @@ -1,10 +1,10 @@ from __future__ import annotations from pathlib import Path -from typing import Any, Optional, Tuple, Union +from typing import Any -from zarr.common import BytesLike from zarr.abc.store import Store +from zarr.common import BytesLike from zarr.store.local import LocalStore @@ -21,16 +21,14 @@ class StorePath: store: Store path: str - def __init__(self, store: Store, path: Optional[str] = None): + def __init__(self, store: Store, path: str | None = None): self.store = store self.path = path or "" - async def get( - self, byte_range: Optional[Tuple[int, Optional[int]]] = None - ) -> Optional[BytesLike]: + async def get(self, byte_range: tuple[int, int | None] | None = None) -> BytesLike | None: return await self.store.get(self.path, byte_range) - async def set(self, value: BytesLike, byte_range: Optional[Tuple[int, int]] = None) -> None: + async def set(self, value: BytesLike, byte_range: tuple[int, int] | None = None) -> None: if byte_range is not None: raise NotImplementedError("Store.set does not have partial writes yet") await self.store.set(self.path, value) @@ -59,7 +57,7 @@ def __eq__(self, other: Any) -> bool: return False -StoreLike = Union[Store, StorePath, Path, str] +StoreLike = Store | StorePath | Path | str def make_store_path(store_like: StoreLike) -> StorePath: diff --git a/src/zarr/store/local.py b/src/zarr/store/local.py index e5021b648..158c41034 100644 --- a/src/zarr/store/local.py +++ b/src/zarr/store/local.py @@ -4,13 +4,12 @@ import shutil from collections.abc import AsyncGenerator from pathlib import Path -from typing import Union, Optional, List, Tuple from zarr.abc.store import Store from zarr.common import BytesLike, concurrent_map, to_thread -def _get(path: Path, byte_range: Optional[Tuple[int, Optional[int]]] = None) -> bytes: +def _get(path: Path, byte_range: tuple[int, int | None] | None = None) -> bytes: """ Fetch a contiguous region of bytes from a file. Parameters @@ -49,7 +48,7 @@ def _get(path: Path, byte_range: Optional[Tuple[int, Optional[int]]] = None) -> def _put( path: Path, value: BytesLike, - start: Optional[int] = None, + start: int | None = None, auto_mkdir: bool = True, ) -> int | None: if auto_mkdir: @@ -71,7 +70,7 @@ class LocalStore(Store): root: Path auto_mkdir: bool - def __init__(self, root: Union[Path, str], auto_mkdir: bool = True): + def __init__(self, root: Path | str, auto_mkdir: bool = True): if isinstance(root, str): root = Path(root) assert isinstance(root, Path) @@ -88,9 +87,7 @@ def __repr__(self) -> str: def __eq__(self, other: object) -> bool: return isinstance(other, type(self)) and self.root == other.root - async def get( - self, key: str, byte_range: Optional[Tuple[int, Optional[int]]] = None - ) -> Optional[bytes]: + async def get(self, key: str, byte_range: tuple[int, int | None] | None = None) -> bytes | None: assert isinstance(key, str) path = self.root / key @@ -100,8 +97,8 @@ async def get( return None async def get_partial_values( - self, key_ranges: List[Tuple[str, Tuple[int, int]]] - ) -> List[Optional[bytes]]: + self, key_ranges: list[tuple[str, tuple[int, int]]] + ) -> list[bytes | None]: """ Read byte ranges from multiple keys. Parameters @@ -124,7 +121,7 @@ async def set(self, key: str, value: BytesLike) -> None: path = self.root / key await to_thread(_put, path, value, auto_mkdir=self.auto_mkdir) - async def set_partial_values(self, key_start_values: List[Tuple[str, int, bytes]]) -> None: + async def set_partial_values(self, key_start_values: list[tuple[str, int, bytes]]) -> None: args = [] for key, start, value in key_start_values: assert isinstance(key, str) diff --git a/src/zarr/store/memory.py b/src/zarr/store/memory.py index 9f10ed22a..02a42ec74 100644 --- a/src/zarr/store/memory.py +++ b/src/zarr/store/memory.py @@ -1,10 +1,9 @@ from __future__ import annotations -from collections.abc import AsyncGenerator -from typing import Optional, MutableMapping, List, Tuple +from collections.abc import AsyncGenerator, MutableMapping -from zarr.common import BytesLike, concurrent_map from zarr.abc.store import Store +from zarr.common import BytesLike, concurrent_map # TODO: this store could easily be extended to wrap any MutuableMapping store from v2 @@ -16,7 +15,7 @@ class MemoryStore(Store): _store_dict: MutableMapping[str, bytes] - def __init__(self, store_dict: Optional[MutableMapping[str, bytes]] = None): + def __init__(self, store_dict: MutableMapping[str, bytes] | None = None): self._store_dict = store_dict or {} def __str__(self) -> str: @@ -26,8 +25,8 @@ def __repr__(self) -> str: return f"MemoryStore({str(self)!r})" async def get( - self, key: str, byte_range: Optional[Tuple[int, Optional[int]]] = None - ) -> Optional[BytesLike]: + self, key: str, byte_range: tuple[int, int | None] | None = None + ) -> BytesLike | None: assert isinstance(key, str) try: value = self._store_dict[key] @@ -38,8 +37,8 @@ async def get( return None async def get_partial_values( - self, key_ranges: List[Tuple[str, Tuple[int, int]]] - ) -> List[Optional[BytesLike]]: + self, key_ranges: list[tuple[str, tuple[int, int]]] + ) -> list[BytesLike | None]: vals = await concurrent_map(key_ranges, self.get, limit=None) return vals @@ -47,10 +46,10 @@ async def exists(self, key: str) -> bool: return key in self._store_dict async def set( - self, key: str, value: BytesLike, byte_range: Optional[Tuple[int, int]] = None + self, key: str, value: BytesLike, byte_range: tuple[int, int] | None = None ) -> None: assert isinstance(key, str) - if not isinstance(value, (bytes, bytearray, memoryview)): + if not isinstance(value, (bytes | bytearray | memoryview)): raise TypeError(f"Expected BytesLike. Got {type(value)}.") if byte_range is not None: @@ -66,7 +65,7 @@ async def delete(self, key: str) -> None: except KeyError: pass # Q(JH): why not raise? - async def set_partial_values(self, key_start_values: List[Tuple[str, int, bytes]]) -> None: + async def set_partial_values(self, key_start_values: list[tuple[str, int, bytes]]) -> None: raise NotImplementedError async def list(self) -> AsyncGenerator[str, None]: diff --git a/src/zarr/store/remote.py b/src/zarr/store/remote.py index fa6cd2167..cbdaa1fe1 100644 --- a/src/zarr/store/remote.py +++ b/src/zarr/store/remote.py @@ -1,15 +1,14 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, Union +from typing import TYPE_CHECKING, Any from zarr.abc.store import Store -from zarr.store.core import _dereference_path from zarr.common import BytesLike - +from zarr.store.core import _dereference_path if TYPE_CHECKING: - from upath import UPath from fsspec.asyn import AsyncFileSystem + from upath import UPath class RemoteStore(Store): @@ -19,9 +18,9 @@ class RemoteStore(Store): root: UPath - def __init__(self, url: Union[UPath, str], **storage_options: Dict[str, Any]): - from upath import UPath + def __init__(self, url: UPath | str, **storage_options: dict[str, Any]): import fsspec + from upath import UPath if isinstance(url, str): self.root = UPath(url, **storage_options) @@ -41,7 +40,7 @@ def __str__(self) -> str: def __repr__(self) -> str: return f"RemoteStore({str(self)!r})" - def _make_fs(self) -> Tuple[AsyncFileSystem, str]: + def _make_fs(self) -> tuple[AsyncFileSystem, str]: import fsspec storage_options = self.root._kwargs.copy() @@ -51,8 +50,8 @@ def _make_fs(self) -> Tuple[AsyncFileSystem, str]: return fs, root async def get( - self, key: str, byte_range: Optional[Tuple[int, Optional[int]]] = None - ) -> Optional[BytesLike]: + self, key: str, byte_range: tuple[int, int | None] | None = None + ) -> BytesLike | None: assert isinstance(key, str) fs, root = self._make_fs() path = _dereference_path(root, key) @@ -69,7 +68,7 @@ async def get( return value async def set( - self, key: str, value: BytesLike, byte_range: Optional[Tuple[int, int]] = None + self, key: str, value: BytesLike, byte_range: tuple[int, int] | None = None ) -> None: assert isinstance(key, str) fs, root = self._make_fs() diff --git a/src/zarr/sync.py b/src/zarr/sync.py index ea765077c..8af14f602 100644 --- a/src/zarr/sync.py +++ b/src/zarr/sync.py @@ -1,12 +1,14 @@ from __future__ import annotations + from typing import TYPE_CHECKING, TypeVar if TYPE_CHECKING: - from typing import Any, AsyncIterator, Coroutine + from collections.abc import AsyncIterator, Coroutine + from typing import Any import asyncio -from concurrent.futures import wait import threading +from concurrent.futures import wait from typing_extensions import ParamSpec diff --git a/src/zarr/testing/__init__.py b/src/zarr/testing/__init__.py index 9b622b43c..35b91f916 100644 --- a/src/zarr/testing/__init__.py +++ b/src/zarr/testing/__init__.py @@ -4,6 +4,6 @@ if importlib.util.find_spec("pytest") is not None: from zarr.testing.store import StoreTests else: - warnings.warn("pytest not installed, skipping test suite") + warnings.warn("pytest not installed, skipping test suite", stacklevel=2) __all__ = ["StoreTests"] diff --git a/tests/v2/conftest.py b/tests/v2/conftest.py index c84cdfa43..a7a445c64 100644 --- a/tests/v2/conftest.py +++ b/tests/v2/conftest.py @@ -1,4 +1,5 @@ import pathlib + import pytest diff --git a/tests/v2/test_attrs.py b/tests/v2/test_attrs.py index b477b8bef..1c6984858 100644 --- a/tests/v2/test_attrs.py +++ b/tests/v2/test_attrs.py @@ -1,11 +1,11 @@ import json import pytest - from zarr.v2.attrs import Attributes -from zarr.v2.storage import KVStore, DirectoryStore -from .util import CountingDict from zarr.v2.hierarchy import group +from zarr.v2.storage import DirectoryStore, KVStore + +from .util import CountingDict def _init_store(): diff --git a/tests/v2/test_convenience.py b/tests/v2/test_convenience.py index f558a8800..881943756 100644 --- a/tests/v2/test_convenience.py +++ b/tests/v2/test_convenience.py @@ -5,21 +5,20 @@ import numpy as np import pytest +import zarr.v2 as zarr from numcodecs import Adler32, Zlib from numpy.testing import assert_array_equal - -import zarr.v2 as zarr from zarr.v2.convenience import ( consolidate_metadata, copy, + copy_all, copy_store, load, open, open_consolidated, save, - save_group, save_array, - copy_all, + save_group, ) from zarr.v2.core import Array from zarr.v2.errors import CopyError diff --git a/tests/v2/test_core.py b/tests/v2/test_core.py index 197461d12..fb9e302c2 100644 --- a/tests/v2/test_core.py +++ b/tests/v2/test_core.py @@ -1,15 +1,17 @@ import atexit import os -import sys import pickle import shutil -from typing import Any, Literal, Optional, Tuple, Union +import sys import unittest from itertools import zip_longest from tempfile import mkdtemp +from typing import Any, Literal + import numpy as np import packaging.version import pytest +import zarr.v2 from numcodecs import ( BZ2, JSON, @@ -29,15 +31,12 @@ from numcodecs.compat import ensure_bytes, ensure_ndarray from numcodecs.tests.common import greetings from numpy.testing import assert_array_almost_equal, assert_array_equal - -import zarr.v2 from zarr.v2._storage.store import ( BaseStore, ) - from zarr.v2.core import Array from zarr.v2.meta import json_loads -from zarr.v2.n5 import N5Store, N5FSStore, n5_keywords +from zarr.v2.n5 import N5FSStore, N5Store, n5_keywords from zarr.v2.storage import ( ABSStore, DBMStore, @@ -54,9 +53,9 @@ init_group, normalize_store_arg, ) - from zarr.v2.util import buffer_size -from .util import abs_container, skip_test_env_var, have_fsspec, mktemp + +from .util import abs_container, have_fsspec, mktemp, skip_test_env_var # noinspection PyMethodMayBeStatic @@ -72,22 +71,22 @@ class TestArray: partial_decompress: bool = False write_empty_chunks = True read_only = False - storage_transformers: Tuple[Any, ...] = () + storage_transformers: tuple[Any, ...] = () def create_store(self) -> BaseStore: return KVStore(dict()) # used by child classes - def create_chunk_store(self) -> Optional[BaseStore]: + def create_chunk_store(self) -> BaseStore | None: return None - def create_storage_transformers(self, shape: Union[int, Tuple[int, ...]]) -> Tuple[Any, ...]: + def create_storage_transformers(self, shape: int | tuple[int, ...]) -> tuple[Any, ...]: return () - def create_filters(self, dtype: Optional[str]) -> Tuple[Any, ...]: + def create_filters(self, dtype: str | None) -> tuple[Any, ...]: return () - def create_array(self, shape: Union[int, Tuple[int, ...]], **kwargs): + def create_array(self, shape: int | tuple[int, ...], **kwargs): store = self.create_store() chunk_store = self.create_chunk_store() # keyword arguments for array initialization @@ -1170,7 +1169,7 @@ def test_dtypes(self): # datetime, timedelta for base_type in "Mm": for resolution in "D", "us", "ns": - dtype = "{}8[{}]".format(base_type, resolution) + dtype = f"{base_type}8[{resolution}]" z = self.create_array(shape=100, dtype=dtype, fill_value=0) assert z.dtype == np.dtype(dtype) a = np.random.randint( @@ -1342,7 +1341,7 @@ def compare_arrays(expected, actual, item_dtype): assert isinstance(actual, np.ndarray) assert actual.dtype == object assert actual.shape == expected.shape - for ev, av in zip(expected.flat, actual.flat): + for ev, av in zip(expected.flat, actual.flat, strict=False): assert isinstance(av, np.ndarray) assert_array_equal(ev, av) assert av.dtype == item_dtype @@ -1360,7 +1359,7 @@ def compare_arrays(expected, actual, item_dtype): # convenience API for item_type in "int", " Tuple[Any, ...]: + def create_filters(self, dtype: str | None) -> tuple[Any, ...]: return ( Delta(dtype=dtype), FixedScaleOffset(dtype=dtype, scale=1, offset=0), diff --git a/tests/v2/test_creation.py b/tests/v2/test_creation.py index 08073a8ac..8b27d2bd1 100644 --- a/tests/v2/test_creation.py +++ b/tests/v2/test_creation.py @@ -6,7 +6,6 @@ import numpy as np import pytest from numpy.testing import assert_array_equal - from zarr.v2.codecs import Zlib from zarr.v2.core import Array from zarr.v2.creation import ( @@ -27,8 +26,8 @@ from zarr.v2.n5 import N5Store from zarr.v2.storage import DirectoryStore, KVStore from zarr.v2.sync import ThreadSynchronizer -from .util import mktemp, have_fsspec +from .util import have_fsspec, mktemp _VERSIONS = (None, 2) _VERSIONS2 = (2,) diff --git a/tests/v2/test_dim_separator.py b/tests/v2/test_dim_separator.py index b0e9d0ecc..5d4ccb9ba 100644 --- a/tests/v2/test_dim_separator.py +++ b/tests/v2/test_dim_separator.py @@ -1,14 +1,13 @@ import pathlib - -import pytest -from numpy.testing import assert_array_equal from functools import partial +import pytest import zarr.v2 +from numpy.testing import assert_array_equal from zarr.v2.core import Array -from zarr.v2.storage import DirectoryStore, NestedDirectoryStore, FSStore -from .util import have_fsspec +from zarr.v2.storage import DirectoryStore, FSStore, NestedDirectoryStore +from .util import have_fsspec needs_fsspec = pytest.mark.skipif(not have_fsspec, reason="needs fsspec") diff --git a/tests/v2/test_filters.py b/tests/v2/test_filters.py index fe21dc775..d5f411cfb 100644 --- a/tests/v2/test_filters.py +++ b/tests/v2/test_filters.py @@ -11,7 +11,6 @@ Zlib, ) from numpy.testing import assert_array_almost_equal, assert_array_equal - from zarr.v2.creation import array compressors = [ diff --git a/tests/v2/test_hierarchy.py b/tests/v2/test_hierarchy.py index 23c5a56ed..e8d57afe0 100644 --- a/tests/v2/test_hierarchy.py +++ b/tests/v2/test_hierarchy.py @@ -1,8 +1,8 @@ import atexit import os -import sys import pickle import shutil +import sys import tempfile import textwrap import unittest @@ -17,7 +17,6 @@ from numcodecs import Zlib from numpy.testing import assert_array_equal - from zarr.v2.attrs import Attributes from zarr.v2.core import Array from zarr.v2.creation import open_array @@ -25,9 +24,9 @@ from zarr.v2.storage import ( ABSStore, DBMStore, - KVStore, DirectoryStore, FSStore, + KVStore, LMDBStore, LRUStoreCache, MemoryStore, @@ -41,9 +40,9 @@ init_array, init_group, ) - from zarr.v2.util import InfoReporter -from .util import skip_test_env_var, have_fsspec, abs_container, mktemp + +from .util import abs_container, have_fsspec, mktemp, skip_test_env_var # noinspection PyStatementEffect diff --git a/tests/v2/test_indexing.py b/tests/v2/test_indexing.py index 13fbc878e..c552544ef 100644 --- a/tests/v2/test_indexing.py +++ b/tests/v2/test_indexing.py @@ -1,16 +1,15 @@ import numpy import numpy as np import pytest -from numpy.testing import assert_array_equal - import zarr.v2 +from numpy.testing import assert_array_equal from zarr.v2.indexing import ( + PartialChunkIterator, make_slice_selection, normalize_integer_selection, oindex, oindex_set, replace_ellipsis, - PartialChunkIterator, ) from .util import CountingDict @@ -1092,7 +1091,9 @@ def test_get_block_selection_1d(): z = zarr.v2.create(shape=a.shape, chunks=100, dtype=a.dtype) z[:] = a - for selection, expected_idx in zip(block_selections_1d, block_selections_1d_array_projection): + for selection, expected_idx in zip( + block_selections_1d, block_selections_1d_array_projection, strict=False + ): _test_get_block_selection(a, z, selection, expected_idx) bad_selections = block_selections_1d_bad + [ @@ -1144,7 +1145,9 @@ def test_get_block_selection_2d(): z = zarr.v2.create(shape=a.shape, chunks=(300, 3), dtype=a.dtype) z[:] = a - for selection, expected_idx in zip(block_selections_2d, block_selections_2d_array_projection): + for selection, expected_idx in zip( + block_selections_2d, block_selections_2d_array_projection, strict=False + ): _test_get_block_selection(a, z, selection, expected_idx) with pytest.raises(IndexError): @@ -1181,7 +1184,9 @@ def test_set_block_selection_1d(): a = np.empty(v.shape, dtype=v.dtype) z = zarr.v2.create(shape=a.shape, chunks=100, dtype=a.dtype) - for selection, expected_idx in zip(block_selections_1d, block_selections_1d_array_projection): + for selection, expected_idx in zip( + block_selections_1d, block_selections_1d_array_projection, strict=False + ): _test_set_block_selection(v, a, z, selection, expected_idx) for selection in block_selections_1d_bad: @@ -1197,7 +1202,9 @@ def test_set_block_selection_2d(): a = np.empty(v.shape, dtype=v.dtype) z = zarr.v2.create(shape=a.shape, chunks=(300, 3), dtype=a.dtype) - for selection, expected_idx in zip(block_selections_2d, block_selections_2d_array_projection): + for selection, expected_idx in zip( + block_selections_2d, block_selections_2d_array_projection, strict=False + ): _test_set_block_selection(v, a, z, selection, expected_idx) with pytest.raises(IndexError): @@ -1634,7 +1641,7 @@ def test_set_selections_with_fields(): ), ( (slice(0, 10, 1),), - np.arange(0, 10).reshape((10)), + np.arange(0, 10).reshape(10), [(0, 10, (slice(0, 10, 1),))], ), ((0,), np.arange(0, 100).reshape((10, 10)), [(0, 10, (slice(0, 1, 1),))]), @@ -1646,7 +1653,7 @@ def test_set_selections_with_fields(): np.arange(0, 100).reshape((10, 10)), [(0, 1, (slice(0, 1, 1), slice(0, 1, 1)))], ), - ((0,), np.arange(0, 10).reshape((10)), [(0, 1, (slice(0, 1, 1),))]), + ((0,), np.arange(0, 10).reshape(10), [(0, 1, (slice(0, 1, 1),))]), pytest.param( (slice(5, 8, 1), slice(2, 4, 1), slice(0, 5, 1)), np.arange(2, 100002).reshape((10, 1, 10000)), @@ -1722,7 +1729,7 @@ def test_accessed_chunks(shape, chunks, ops): for ii, (optype, slices) in enumerate(ops): # Resolve the slices into the accessed chunks for each dimension chunks_per_dim = [] - for N, C, sl in zip(shape, chunks, slices): + for N, C, sl in zip(shape, chunks, slices, strict=False): chunk_ind = np.arange(N, dtype=int)[sl] // C chunks_per_dim.append(np.unique(chunk_ind)) diff --git a/tests/v2/test_info.py b/tests/v2/test_info.py index 9f6811929..b95c3dc4b 100644 --- a/tests/v2/test_info.py +++ b/tests/v2/test_info.py @@ -1,6 +1,5 @@ import numcodecs import pytest - import zarr.v2 from zarr.v2.util import InfoReporter diff --git a/tests/v2/test_meta.py b/tests/v2/test_meta.py index b7c00ec64..ee55c2f37 100644 --- a/tests/v2/test_meta.py +++ b/tests/v2/test_meta.py @@ -3,18 +3,17 @@ import numpy as np import pytest - from zarr.v2.codecs import Blosc, Delta, Pickle, Zlib from zarr.v2.errors import MetadataError from zarr.v2.meta import ( ZARR_FORMAT, decode_array_metadata, decode_dtype, + decode_fill_value, decode_group_metadata, encode_array_metadata, encode_dtype, encode_fill_value, - decode_fill_value, ) from zarr.v2.util import normalize_dtype, normalize_fill_value diff --git a/tests/v2/test_meta_array.py b/tests/v2/test_meta_array.py index c47673fb0..cae21c20b 100644 --- a/tests/v2/test_meta_array.py +++ b/tests/v2/test_meta_array.py @@ -1,15 +1,12 @@ -from typing import Optional import numpy as np import pytest - +import zarr.v2.codecs from numcodecs.abc import Codec from numcodecs.compat import ensure_contiguous_ndarray_like from numcodecs.registry import get_codec, register_codec - -import zarr.v2.codecs from zarr.v2.core import Array from zarr.v2.creation import array, empty, full, ones, open_array, zeros -from zarr.v2.hierarchy import open_group, group +from zarr.v2.hierarchy import group, open_group from zarr.v2.storage import DirectoryStore, MemoryStore, Store, ZipStore @@ -96,7 +93,7 @@ def init_compressor(compressor) -> CuPyCPUCompressor: return CuPyCPUCompressor(compressor) -def init_store(tmp_path, store_type) -> Optional[Store]: +def init_store(tmp_path, store_type) -> Store | None: if store_type is DirectoryStore: return store_type(str(tmp_path / "store")) if store_type is MemoryStore: diff --git a/tests/v2/test_n5.py b/tests/v2/test_n5.py index 238e9b2c6..0cac21b81 100644 --- a/tests/v2/test_n5.py +++ b/tests/v2/test_n5.py @@ -1,13 +1,12 @@ -import pytest +import atexit +import json -from zarr.v2.n5 import N5ChunkWrapper, N5FSStore +import numpy as np +import pytest +from numcodecs import GZip from zarr.v2.creation import create +from zarr.v2.n5 import N5ChunkWrapper, N5FSStore from zarr.v2.storage import atexit_rmtree -from numcodecs import GZip -import numpy as np -from typing import Tuple -import json -import atexit from .util import have_fsspec @@ -28,7 +27,7 @@ def test_make_n5_chunk_wrapper(): @pytest.mark.parametrize("chunk_shape", ((2,), (4, 4), (8, 8, 8))) -def test_partial_chunk_decode(chunk_shape: Tuple[int, ...]): +def test_partial_chunk_decode(chunk_shape: tuple[int, ...]): # Test that the N5Chunk wrapper can handle fractional chunks that # may be generated by other N5 implementations dtype = "uint8" diff --git a/tests/v2/test_storage.py b/tests/v2/test_storage.py index 17b80e6a5..e7f1d7fac 100644 --- a/tests/v2/test_storage.py +++ b/tests/v2/test_storage.py @@ -3,9 +3,9 @@ import json import os import pathlib -import sys import pickle import shutil +import sys import tempfile from contextlib import contextmanager from pickle import PicklingError @@ -13,11 +13,9 @@ import numpy as np import pytest -from numpy.testing import assert_array_almost_equal, assert_array_equal - -from numcodecs.compat import ensure_bytes - import zarr.v2 +from numcodecs.compat import ensure_bytes +from numpy.testing import assert_array_almost_equal, assert_array_equal from zarr.v2 import meta_v1 from zarr.v2.codecs import BZ2, AsType, Blosc, Zlib from zarr.v2.context import Context @@ -25,13 +23,14 @@ from zarr.v2.errors import ContainsArrayError, ContainsGroupError, MetadataError from zarr.v2.hierarchy import group from zarr.v2.meta import ZARR_FORMAT, decode_array_metadata -from zarr.v2.n5 import N5Store, N5FSStore, N5_FORMAT, n5_attrs_key +from zarr.v2.n5 import N5_FORMAT, N5FSStore, N5Store, n5_attrs_key from zarr.v2.storage import ( ABSStore, ConsolidatedMetadataStore, DBMStore, DictStore, DirectoryStore, + FSStore, KVStore, LMDBStore, LRUStoreCache, @@ -52,13 +51,15 @@ group_meta_key, init_array, init_group, + listdir, migrate_1to2, normalize_store_arg, + rename, ) -from zarr.v2.storage import FSStore, rename, listdir -from .util import CountingDict, have_fsspec, skip_test_env_var, abs_container, mktemp from zarr.v2.util import ConstantMap, json_dumps +from .util import CountingDict, abs_container, have_fsspec, mktemp, skip_test_env_var + @contextmanager def does_not_raise(): @@ -1503,9 +1504,10 @@ def test_filters(self): (None, does_not_raise()), ([], does_not_raise()), ([AsType("f4", "f8")], pytest.raises(ValueError)), - ] + ], + strict=False, ) - for filters, error in zip(all_filters, all_errors): + for filters, error in zip(all_filters, all_errors, strict=False): store = self.create_store() with error: init_array(store, shape=1000, chunks=100, filters=filters) @@ -1642,9 +1644,10 @@ def test_filters(self): (None, does_not_raise()), ([], does_not_raise()), ([AsType("f4", "f8")], pytest.raises(ValueError)), - ] + ], + strict=False, ) - for filters, error in zip(all_filters, all_errors): + for filters, error in zip(all_filters, all_errors, strict=False): store = self.create_store() with error: init_array(store, shape=1000, chunks=100, filters=filters) @@ -2274,7 +2277,7 @@ def test_format_compatibility(): order = "C" for j, compressor in enumerate(compressors): - path = "{}/{}".format(i, j) + path = f"{i}/{j}" if path not in fixture: # pragma: no cover # store the data - should be one-time operation @@ -2353,7 +2356,7 @@ def test_iterators_with_prefix(self): assert 4 == len(store) keys = [prefix + "a", prefix + "b", prefix + "c/d", prefix + "c/e/f"] values = [b"aaa", b"bbb", b"ddd", b"fff"] - items = list(zip(keys, values)) + items = list(zip(keys, values, strict=False)) assert set(keys) == set(store) assert set(keys) == set(store.keys()) assert set(values) == set(store.values()) diff --git a/tests/v2/test_sync.py b/tests/v2/test_sync.py index ea6fd0523..685f463dc 100644 --- a/tests/v2/test_sync.py +++ b/tests/v2/test_sync.py @@ -8,7 +8,6 @@ import numpy as np from numpy.testing import assert_array_equal - from zarr.v2.attrs import Attributes from zarr.v2.core import Array from zarr.v2.hierarchy import Group @@ -70,7 +69,7 @@ def test_parallel_setitem(self): pool = self.create_pool() # parallel setitem - results = pool.map(_set_arange, zip([arr] * n, range(n)), chunksize=1) + results = pool.map(_set_arange, zip([arr] * n, range(n), strict=False), chunksize=1) results = sorted(results) assert list(range(n)) == results @@ -87,7 +86,7 @@ def test_parallel_append(self): pool = self.create_pool() # parallel append - results = pool.map(_append, zip([arr] * n, range(n)), chunksize=1) + results = pool.map(_append, zip([arr] * n, range(n), strict=False), chunksize=1) results = sorted(results) assert [((i + 2) * 1000,) for i in range(n)] == results @@ -230,7 +229,9 @@ def test_parallel_create_group(self): # parallel create group n = 100 results = list( - pool.map(_create_group, zip([g] * n, [str(i) for i in range(n)]), chunksize=1) + pool.map( + _create_group, zip([g] * n, [str(i) for i in range(n)], strict=False), chunksize=1 + ) ) assert n == len(results) pool.close() @@ -248,7 +249,11 @@ def test_parallel_require_group(self): # parallel require group n = 100 results = list( - pool.map(_require_group, zip([g] * n, [str(i // 10) for i in range(n)]), chunksize=1) + pool.map( + _require_group, + zip([g] * n, [str(i // 10) for i in range(n)], strict=False), + chunksize=1, + ) ) assert n == len(results) pool.close() diff --git a/tests/v2/test_util.py b/tests/v2/test_util.py index 35c355693..7ce5b041f 100644 --- a/tests/v2/test_util.py +++ b/tests/v2/test_util.py @@ -3,7 +3,6 @@ import numpy as np import pytest - from zarr.v2.core import Array from zarr.v2.util import ( ConstantMap, @@ -23,8 +22,8 @@ normalize_shape, retry_call, tree_array_icon, - tree_group_icon, tree_get_icon, + tree_group_icon, tree_widget, ) @@ -161,7 +160,7 @@ def test_guess_chunks(): assert isinstance(chunks, tuple) assert len(chunks) == len(shape) # doesn't make any sense to allow chunks to have zero length dimension - assert all(0 < c <= max(s, 1) for c, s in zip(chunks, shape)) + assert all(0 < c <= max(s, 1) for c, s in zip(chunks, shape, strict=False)) # ludicrous itemsize chunks = guess_chunks((1000000,), 40000000000) diff --git a/tests/v2/util.py b/tests/v2/util.py index 12c5e379f..7d2e7a1f4 100644 --- a/tests/v2/util.py +++ b/tests/v2/util.py @@ -1,12 +1,12 @@ import collections import os import tempfile -from typing import Any, Mapping, Sequence -from zarr.v2.context import Context - -from zarr.v2.storage import Store +from collections.abc import Mapping, Sequence +from typing import Any import pytest +from zarr.v2.context import Context +from zarr.v2.storage import Store class CountingDict(Store): @@ -65,8 +65,8 @@ def skip_test_env_var(name): def abs_container(): - from azure.core.exceptions import ResourceExistsError import azure.storage.blob as asb + from azure.core.exceptions import ResourceExistsError URL = "http://127.0.0.1:10000" ACCOUNT_NAME = "devstoreaccount1" diff --git a/tests/v3/conftest.py b/tests/v3/conftest.py index 3dc55c029..1161c870a 100644 --- a/tests/v3/conftest.py +++ b/tests/v3/conftest.py @@ -1,7 +1,7 @@ import pathlib -import pytest -from zarr.store import LocalStore, StorePath, MemoryStore, RemoteStore +import pytest +from zarr.store import LocalStore, MemoryStore, RemoteStore, StorePath @pytest.fixture(params=[str, pathlib.Path]) diff --git a/tests/v3/test_codec_entrypoints.py b/tests/v3/test_codec_entrypoints.py index 8fbf76b83..b36919dcd 100644 --- a/tests/v3/test_codec_entrypoints.py +++ b/tests/v3/test_codec_entrypoints.py @@ -2,10 +2,8 @@ import sys import pytest - import zarr.codecs.registry - here = os.path.abspath(os.path.dirname(__file__)) diff --git a/tests/v3/test_codecs.py b/tests/v3/test_codecs.py index e042c7f27..6a328fd75 100644 --- a/tests/v3/test_codecs.py +++ b/tests/v3/test_codecs.py @@ -1,29 +1,28 @@ from __future__ import annotations -from dataclasses import dataclass import json -from typing import Iterator, List, Literal, Optional, Tuple - +from collections.abc import Iterator +from dataclasses import dataclass +from typing import Literal import numpy as np import pytest import zarr.v2 from zarr.abc.codec import Codec +from zarr.abc.store import Store from zarr.array import Array, AsyncArray -from zarr.common import Selection -from zarr.indexing import morton_order_iter from zarr.codecs import ( - ShardingCodec, - ShardingCodecIndexLocation, BloscCodec, BytesCodec, GzipCodec, + ShardingCodec, + ShardingCodecIndexLocation, TransposeCodec, ZstdCodec, ) - -from zarr.abc.store import Store +from zarr.common import Selection from zarr.config import config +from zarr.indexing import morton_order_iter from zarr.store import MemoryStore, StorePath @@ -57,7 +56,7 @@ def sample_data() -> np.ndarray: return np.arange(0, 128 * 128 * 128, dtype="uint16").reshape((128, 128, 128), order="F") -def order_from_dim(order: Literal["F", "C"], ndim: int) -> Tuple[int, ...]: +def order_from_dim(order: Literal["F", "C"], ndim: int) -> tuple[int, ...]: if order == "F": return tuple(ndim - x - 1 for x in range(ndim)) else: @@ -244,7 +243,7 @@ async def test_order( ): data = np.arange(0, 256, dtype="uint16").reshape((32, 8), order=input_order) - codecs_: List[Codec] = ( + codecs_: list[Codec] = ( [ ShardingCodec( chunk_shape=(16, 8), @@ -311,7 +310,7 @@ def test_order_implicit( ): data = np.arange(0, 256, dtype="uint16").reshape((16, 16), order=input_order) - codecs_: Optional[List[Codec]] = [ShardingCodec(chunk_shape=(8, 8))] if with_sharding else None + codecs_: list[Codec] | None = [ShardingCodec(chunk_shape=(8, 8))] if with_sharding else None with config.set({"array.order": runtime_write_order}): a = Array.create( @@ -354,7 +353,7 @@ async def test_transpose( ): data = np.arange(0, 256, dtype="uint16").reshape((1, 32, 8), order=input_order) - codecs_: List[Codec] = ( + codecs_: list[Codec] = ( [ ShardingCodec( chunk_shape=(1, 16, 8), diff --git a/tests/v3/test_common.py b/tests/v3/test_common.py index b5690d0d7..37db1f31f 100644 --- a/tests/v3/test_common.py +++ b/tests/v3/test_common.py @@ -1,18 +1,19 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Iterable + +from collections.abc import Iterable +from typing import TYPE_CHECKING if TYPE_CHECKING: - from typing import Literal, Any, Tuple + from typing import Any, Literal import numpy as np -from zarr.config import parse_indexing_order -from zarr.common import parse_shapelike -from zarr.common import parse_name, product import pytest +from zarr.common import parse_name, parse_shapelike, product +from zarr.config import parse_indexing_order @pytest.mark.parametrize("data", [(0, 0, 0, 0), (1, 3, 4, 5, 6), (2, 4)]) -def test_product(data: Tuple[int, ...]): +def test_product(data: tuple[int, ...]): assert product(data) == np.prod(data) @@ -33,7 +34,7 @@ def test_parse_enum(): ... @pytest.mark.parametrize("data", [("foo", "bar"), (10, 11)]) -def test_parse_name_invalid(data: Tuple[Any, Any]): +def test_parse_name_invalid(data: tuple[Any, Any]): observed, expected = data if isinstance(observed, str): with pytest.raises(ValueError, match=f"Expected '{expected}'. Got {observed} instead."): @@ -46,7 +47,7 @@ def test_parse_name_invalid(data: Tuple[Any, Any]): @pytest.mark.parametrize("data", [("foo", "foo"), ("10", "10")]) -def test_parse_name_valid(data: Tuple[Any, Any]): +def test_parse_name_valid(data: tuple[Any, Any]): observed, expected = data assert parse_name(observed, expected) == observed @@ -83,7 +84,7 @@ def test_parse_shapelike_valid(data: Iterable[Any]): # todo: more dtypes @pytest.mark.parametrize("data", [("uint8", np.uint8), ("float64", np.float64)]) -def parse_dtype(data: Tuple[str, np.dtype]): +def parse_dtype(data: tuple[str, np.dtype]): unparsed, parsed = data assert parse_dtype(unparsed) == parsed diff --git a/tests/v3/test_group.py b/tests/v3/test_group.py index 11400ef80..acaf92053 100644 --- a/tests/v3/test_group.py +++ b/tests/v3/test_group.py @@ -1,13 +1,13 @@ from __future__ import annotations + from typing import TYPE_CHECKING from zarr.sync import sync if TYPE_CHECKING: - from zarr.store import MemoryStore, LocalStore -import pytest + from zarr.store import LocalStore, MemoryStore import numpy as np - +import pytest from zarr.group import AsyncGroup, Group, GroupMetadata from zarr.store import LocalStore, StorePath diff --git a/tests/v3/test_metadata.py b/tests/v3/test_metadata.py index 33df4a643..65297c52d 100644 --- a/tests/v3/test_metadata.py +++ b/tests/v3/test_metadata.py @@ -1,9 +1,12 @@ from __future__ import annotations -import pytest + from typing import TYPE_CHECKING +import pytest + if TYPE_CHECKING: - from typing import Sequence, Any + from collections.abc import Sequence + from typing import Any from zarr.metadata import parse_dimension_names, parse_zarr_format_v2, parse_zarr_format_v3 diff --git a/tests/v3/test_storage.py b/tests/v3/test_storage.py index 2761e608e..e97950678 100644 --- a/tests/v3/test_storage.py +++ b/tests/v3/test_storage.py @@ -1,8 +1,7 @@ import pytest - -from zarr.testing.store import StoreTests from zarr.store.local import LocalStore from zarr.store.memory import MemoryStore +from zarr.testing.store import StoreTests class TestMemoryStore(StoreTests): diff --git a/tests/v3/test_sync.py b/tests/v3/test_sync.py index ba262f521..13f7eb43d 100644 --- a/tests/v3/test_sync.py +++ b/tests/v3/test_sync.py @@ -1,11 +1,10 @@ -from collections.abc import AsyncGenerator import asyncio import time -from unittest.mock import patch, AsyncMock - -from zarr.sync import sync, _get_loop, _get_lock, SyncError, SyncMixin +from collections.abc import AsyncGenerator +from unittest.mock import AsyncMock, patch import pytest +from zarr.sync import SyncError, SyncMixin, _get_lock, _get_loop, sync @pytest.fixture(params=[True, False])