From 7b84806aa645a130efbae486b2ed2d58fa86f7ad Mon Sep 17 00:00:00 2001 From: Joseph Hamman Date: Sat, 19 Oct 2024 21:14:56 -0700 Subject: [PATCH 1/4] test(ci): add test environment for upstream dependencies --- .github/workflows/test.yml | 27 +++++++++++++++++++++++++++ pyproject.toml | 29 +++++++++++++++++++++++++++++ src/zarr/codecs/zstd.py | 7 ++++--- 3 files changed, 60 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3bd622692..e09975e44 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -43,3 +43,30 @@ jobs: - name: Run Tests run: | hatch env run --env test.py${{ matrix.python-version }}-${{ matrix.numpy-version }}-${{ matrix.dependency-set }} run + + upstream: + name: py=${{ matrix.python-version }}-upstream + + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ['3.13'] + + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + cache: 'pip' + - name: Install Hatch + run: | + python -m pip install --upgrade pip + pip install hatch + - name: Set Up Hatch Env + run: | + hatch env create upstream + hatch env run -e upstream list-env + - name: Run Tests + run: | + hatch env run --env upstream run diff --git a/pyproject.toml b/pyproject.toml index 574b09b07..9268a873f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -183,6 +183,35 @@ features = ['docs'] build = "cd docs && make html" serve = "sphinx-autobuild docs docs/_build --host 0.0.0.0" +[tool.hatch.envs.upstream] +dependencies = [ + 'numpy>=2.2.0.dev0', + 'numcodecs @ git+https://github.com/zarr-developers/numcodecs', + 'fsspec @ git+https://github.com/fsspec/filesystem_spec', + 's3fs @ git+https://github.com/fsspec/s3fs', + 'universal_pathlib @ git+https://github.com/fsspec/universal_pathlib', + 'crc32c @ git+https://github.com/ICRAR/crc32c', + 'typing_extensions @ git+https://github.com/python/typing_extensions', + 'donfig @ git+https://github.com/pytroll/donfig', + # test deps + 'hypothesis', + 'pytest', + 'pytest-cov', + 'pytest-asyncio', + 'moto[s3]', +] + +# help! this is not working for numpy :( +[tool.hatch.envs.upstream.env-vars] +PIP_EXTRA_INDEX_URL = "https://pypi.anaconda.org/scientific-python-nightly-wheels/simple" +PIP_PRE = "1" + +[tool.hatch.envs.upstream.scripts] +run = "pytest --verbose" +run-mypy = "mypy src" +run-hypothesis = "pytest --hypothesis-profile ci tests/test_properties.py tests/test_store/test_stateful*" +list-env = "pip list" + [tool.ruff] line-length = 100 force-exclude = true diff --git a/src/zarr/codecs/zstd.py b/src/zarr/codecs/zstd.py index 949f762b2..b4a4a13c2 100644 --- a/src/zarr/codecs/zstd.py +++ b/src/zarr/codecs/zstd.py @@ -3,10 +3,11 @@ import asyncio from dataclasses import dataclass from functools import cached_property -from importlib.metadata import version from typing import TYPE_CHECKING +import numcodecs from numcodecs.zstd import Zstd +from packaging.version import Version from zarr.abc.codec import BytesBytesCodec from zarr.core.buffer.cpu import as_numpy_array_wrapper @@ -43,8 +44,8 @@ class ZstdCodec(BytesBytesCodec): def __init__(self, *, level: int = 0, checksum: bool = False) -> None: # numcodecs 0.13.0 introduces the checksum attribute for the zstd codec - _numcodecs_version = tuple(map(int, version("numcodecs").split("."))) - if _numcodecs_version < (0, 13, 0): # pragma: no cover + _numcodecs_version = Version(numcodecs.__version__) + if _numcodecs_version < Version("0.13.0"): raise RuntimeError( "numcodecs version >= 0.13.0 is required to use the zstd codec. " f"Version {_numcodecs_version} is currently installed." From 4c9619b29a95adf5fafcfefcaa8d48cb11ddf267 Mon Sep 17 00:00:00 2001 From: Joseph Hamman Date: Sun, 20 Oct 2024 08:24:37 -0700 Subject: [PATCH 2/4] try PIP_INDEX_URL again --- pyproject.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 9268a873f..b14381217 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -185,7 +185,7 @@ serve = "sphinx-autobuild docs docs/_build --host 0.0.0.0" [tool.hatch.envs.upstream] dependencies = [ - 'numpy>=2.2.0.dev0', + 'numpy', # from scientific-python-nightly-wheels 'numcodecs @ git+https://github.com/zarr-developers/numcodecs', 'fsspec @ git+https://github.com/fsspec/filesystem_spec', 's3fs @ git+https://github.com/fsspec/s3fs', @@ -201,9 +201,9 @@ dependencies = [ 'moto[s3]', ] -# help! this is not working for numpy :( [tool.hatch.envs.upstream.env-vars] -PIP_EXTRA_INDEX_URL = "https://pypi.anaconda.org/scientific-python-nightly-wheels/simple" +PIP_INDEX_URL = "https://pypi.anaconda.org/scientific-python-nightly-wheels/simple/" +PIP_EXTRA_INDEX_URL = "https://pypi.org/simple/" PIP_PRE = "1" [tool.hatch.envs.upstream.scripts] From 163de4dd145fa43c55a2f864b26a2672ea8e7dc4 Mon Sep 17 00:00:00 2001 From: Joseph Hamman Date: Sun, 20 Oct 2024 21:37:39 -0700 Subject: [PATCH 3/4] test(ci): add test environment for oldest supported dependency versions --- .github/workflows/test.yml | 27 ++++++++++++++++++++++++ pyproject.toml | 43 ++++++++++++++++++++++++++++++++------ src/zarr/storage/remote.py | 10 ++++++--- 3 files changed, 71 insertions(+), 9 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e09975e44..95a1a182e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -70,3 +70,30 @@ jobs: - name: Run Tests run: | hatch env run --env upstream run + + min_deps: + name: py=${{ matrix.python-version }}-min_deps + + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ['3.11'] + + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + cache: 'pip' + - name: Install Hatch + run: | + python -m pip install --upgrade pip + pip install hatch + - name: Set Up Hatch Env + run: | + hatch env create min_deps + hatch env run -e min_deps list-env + - name: Run Tests + run: | + hatch env run --env min_deps run diff --git a/pyproject.toml b/pyproject.toml index e3d8a310b..ef2ccd946 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,12 +28,13 @@ requires-python = ">=3.11" dependencies = [ 'asciitree', 'numpy>=1.25', - 'numcodecs>=0.10.2', - 'fsspec>2024', - 'crc32c', - 'typing_extensions', - 'donfig', + 'numcodecs>=0.13', + 'fsspec>=2022.10.0', + 'crc32c>=2.3', + 'typing_extensions>=4.6', + 'donfig>=0.8', ] + dynamic = [ "version", ] @@ -98,7 +99,7 @@ extra = [ ] optional = [ 'lmdb', - 'universal-pathlib', + 'universal-pathlib>=0.0.22', ] [project.urls] @@ -184,6 +185,7 @@ build = "cd docs && make html" serve = "sphinx-autobuild docs docs/_build --host 0.0.0.0" [tool.hatch.envs.upstream] +python = "3.13" dependencies = [ 'numpy', # from scientific-python-nightly-wheels 'numcodecs @ git+https://github.com/zarr-developers/numcodecs', @@ -212,6 +214,35 @@ run-mypy = "mypy src" run-hypothesis = "pytest --hypothesis-profile ci tests/test_properties.py tests/test_store/test_stateful*" list-env = "pip list" +[tool.hatch.envs.min_deps] +description = """Test environment for minimum supported dependencies + +See Spec 0000 for details and drop schedule: https://scientific-python.org/specs/spec-0000/ +""" +python = "3.11" +dependencies = [ + 'numpy==1.25.*', + 'numcodecs==0.13.*', # 0.13 needed for? (should be 0.11) + 'fsspec==2022.10.0', + 's3fs==2022.10.0', + 'universal_pathlib==0.0.22', + 'crc32c==2.3.*', + 'typing_extensions==4.6.*', # 4.5 needed for @deprecated, 4.6 for Buffer + 'donfig==0.8.*', + # test deps + 'hypothesis', + 'pytest', + 'pytest-cov', + 'pytest-asyncio', + 'moto[s3]', +] + +[tool.hatch.envs.min_deps.scripts] +run = "pytest --verbose" +run-hypothesis = "pytest --hypothesis-profile ci tests/test_properties.py tests/test_store/test_stateful*" +list-env = "pip list" + + [tool.ruff] line-length = 100 force-exclude = true diff --git a/src/zarr/storage/remote.py b/src/zarr/storage/remote.py index 0a0ec7f7c..812b1e24f 100644 --- a/src/zarr/storage/remote.py +++ b/src/zarr/storage/remote.py @@ -2,8 +2,6 @@ from typing import TYPE_CHECKING, Any, Self -import fsspec - from zarr.abc.store import ByteRangeRequest, Store from zarr.storage.common import _dereference_path @@ -130,7 +128,13 @@ def from_url( ------- RemoteStore """ - fs, path = fsspec.url_to_fs(url, **storage_options) + try: + from fsspec import url_to_fs + except ImportError: + # before fsspec==2024.3.1 + from fsspec.core import url_to_fs + + fs, path = url_to_fs(url, **storage_options) return cls(fs=fs, path=path, mode=mode, allowed_exceptions=allowed_exceptions) async def clear(self) -> None: From d343d34fdafb697f1cca242b6496abc1dc41d25f Mon Sep 17 00:00:00 2001 From: Joseph Hamman Date: Sun, 20 Oct 2024 21:52:50 -0700 Subject: [PATCH 4/4] use a matrix again --- .github/workflows/test.yml | 46 ++++++++++---------------------------- 1 file changed, 12 insertions(+), 34 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 95a1a182e..d32f6f793 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -44,41 +44,19 @@ jobs: run: | hatch env run --env test.py${{ matrix.python-version }}-${{ matrix.numpy-version }}-${{ matrix.dependency-set }} run - upstream: - name: py=${{ matrix.python-version }}-upstream + test-upstream-and-min-deps: + name: py=${{ matrix.python-version }}-${{ matrix.dependency-set }} runs-on: ubuntu-latest strategy: matrix: - python-version: ['3.13'] - - steps: - - uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - cache: 'pip' - - name: Install Hatch - run: | - python -m pip install --upgrade pip - pip install hatch - - name: Set Up Hatch Env - run: | - hatch env create upstream - hatch env run -e upstream list-env - - name: Run Tests - run: | - hatch env run --env upstream run - - min_deps: - name: py=${{ matrix.python-version }}-min_deps - - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ['3.11'] - + python-version: ['3.11', "3.13"] + dependency-set: ["upstream", "min_deps"] + exclude: + - python-version: "3.13" + dependency-set: min_deps + - python-version: "3.11" + dependency-set: upstream steps: - uses: actions/checkout@v4 - name: Set up Python @@ -92,8 +70,8 @@ jobs: pip install hatch - name: Set Up Hatch Env run: | - hatch env create min_deps - hatch env run -e min_deps list-env + hatch env create ${{ matrix.dependency-set }} + hatch env run -e ${{ matrix.dependency-set }} list-env - name: Run Tests run: | - hatch env run --env min_deps run + hatch env run --env ${{ matrix.dependency-set }} run