Make zarr ArrayWriter customizable when writing dataset to zarr #9471
846 fail, 1 633 skipped, 17 738 pass in 1h 19m 20s
10 files 10 suites 1h 19m 20s ⏱️
20 217 tests 17 738 ✅ 1 633 💤 846 ❌
158 201 runs 132 232 ✅ 20 048 💤 5 921 ❌
Results for commit 01b2e73.
Annotations
Check warning on line 0 in xarray.tests.test_backends.TestZarrWriteEmpty
github-actions / Test Results
6 out of 9 runs failed: test_write_empty[True-True] (xarray.tests.test_backends.TestZarrWriteEmpty)
artifacts/Test results for Linux-3.10 min-all-deps/pytest.xml [took 0s]
artifacts/Test results for Linux-3.10/pytest.xml [took 0s]
artifacts/Test results for Linux-3.11 all-but-dask/pytest.xml [took 0s]
artifacts/Test results for Linux-3.12/pytest.xml [took 0s]
artifacts/Test results for macOS-3.10/pytest.xml [took 0s]
artifacts/Test results for macOS-3.12/pytest.xml [took 0s]
Raw output
AttributeError: 'NoneType' object has no attribute 'sync'
self = <xarray.tests.test_backends.TestZarrWriteEmpty object at 0x7fab263d7310>
consolidated = True, write_empty = True
@pytest.mark.parametrize("consolidated", [True, False, None])
@pytest.mark.parametrize("write_empty", [True, False, None])
def test_write_empty(
self, consolidated: bool | None, write_empty: bool | None
) -> None:
if write_empty is False:
expected = ["0.1.0", "1.1.0"]
else:
expected = [
"0.0.0",
"0.0.1",
"0.1.0",
"0.1.1",
"1.0.0",
"1.0.1",
"1.1.0",
"1.1.1",
]
ds = xr.Dataset(
data_vars={
"test": (
("Z", "Y", "X"),
np.array([np.nan, np.nan, 1.0, np.nan]).reshape((1, 2, 2)),
)
}
)
if has_dask:
ds["test"] = ds["test"].chunk(1)
encoding = None
else:
encoding = {"test": {"chunks": (1, 1, 1)}}
with self.temp_dir() as (d, store):
> ds.to_zarr(
store,
mode="w",
encoding=encoding,
write_empty_chunks=write_empty,
)
#x1B[1m#x1B[31m/home/runner/work/xarray/xarray/xarray/tests/test_backends.py#x1B[0m:3388:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
#x1B[1m#x1B[31m/home/runner/work/xarray/xarray/xarray/core/dataset.py#x1B[0m:2566: in to_zarr
return to_zarr( # type: ignore[call-overload,misc]
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
dataset = <xarray.Dataset> Size: 32B
Dimensions: (Z: 1, Y: 2, X: 2)
Dimensions without coordinates: Z, Y, X
Data variables:
test (Z, Y, X) float64 32B dask.array<chunksize=(1, 1, 1), meta=np.ndarray>
store = '/tmp/tmpwy3ouxr5/test.zarr', chunk_store = None, mode = 'w'
synchronizer = None, group = None, encoding = {}
def to_zarr(
dataset: Dataset,
store: MutableMapping | str | os.PathLike[str] | None = None,
chunk_store: MutableMapping | str | os.PathLike | None = None,
mode: ZarrWriteModes | None = None,
synchronizer=None,
group: str | None = None,
encoding: Mapping | None = None,
*,
compute: bool = True,
consolidated: bool | None = None,
append_dim: Hashable | None = None,
region: Mapping[str, slice | Literal["auto"]] | Literal["auto"] | None = None,
safe_chunks: bool = True,
storage_options: dict[str, str] | None = None,
zarr_version: int | None = None,
write_empty_chunks: bool | None = None,
chunkmanager_store_kwargs: dict[str, Any] | None = None,
writer: ArrayWriter | None = None,
) -> backends.ZarrStore | Delayed:
"""This function creates an appropriate datastore for writing a dataset to
a zarr ztore
See `Dataset.to_zarr` for full API docs.
"""
# Load empty arrays to avoid bug saving zero length dimensions (Issue #5741)
for v in dataset.variables.values():
if v.size == 0:
v.load()
# expand str and path-like arguments
store = _normalize_path(store)
chunk_store = _normalize_path(chunk_store)
if storage_options is None:
mapper = store
chunk_mapper = chunk_store
else:
from fsspec import get_mapper
if not isinstance(store, str):
raise ValueError(
f"store must be a string to use storage_options. Got {type(store)}"
)
mapper = get_mapper(store, **storage_options)
if chunk_store is not None:
chunk_mapper = get_mapper(chunk_store, **storage_options)
else:
chunk_mapper = chunk_store
if encoding is None:
encoding = {}
if mode is None:
if append_dim is not None:
mode = "a"
elif region is not None:
mode = "r+"
else:
mode = "w-"
if mode not in ["a", "a-"] and append_dim is not None:
raise ValueError("cannot set append_dim unless mode='a' or mode=None")
if mode not in ["a", "a-", "r+"] and region is not None:
raise ValueError(
"cannot set region unless mode='a', mode='a-', mode='r+' or mode=None"
)
if mode not in ["w", "w-", "a", "a-", "r+"]:
raise ValueError(
"The only supported options for mode are 'w', "
f"'w-', 'a', 'a-', and 'r+', but mode={mode!r}"
)
# validate Dataset keys, DataArray names
_validate_dataset_names(dataset)
if zarr_version is None:
# default to 2 if store doesn't specify it's version (e.g. a path)
zarr_version = int(getattr(store, "_store_version", 2))
if consolidated is None and zarr_version > 2:
consolidated = False
if mode == "r+":
already_consolidated = consolidated
consolidate_on_close = False
else:
already_consolidated = False
consolidate_on_close = consolidated or consolidated is None
zstore = backends.ZarrStore.open_group(
store=mapper,
mode=mode,
synchronizer=synchronizer,
group=group,
consolidated=already_consolidated,
consolidate_on_close=consolidate_on_close,
chunk_store=chunk_mapper,
append_dim=append_dim,
write_region=region,
safe_chunks=safe_chunks,
stacklevel=4, # for Dataset.to_zarr()
zarr_version=zarr_version,
write_empty=write_empty_chunks,
)
if region is not None:
zstore._validate_and_autodetect_region(dataset)
# can't modify indexes with region writes
dataset = dataset.drop_vars(dataset.indexes)
if append_dim is not None and append_dim in region:
raise ValueError(
f"cannot list the same dimension in both ``append_dim`` and "
f"``region`` with to_zarr(), got {append_dim} in both"
)
if encoding and mode in ["a", "a-", "r+"]:
existing_var_names = set(zstore.zarr_group.array_keys())
for var_name in existing_var_names:
if var_name in encoding:
raise ValueError(
f"variable {var_name!r} already exists, but encoding was provided"
)
# TODO: figure out how to properly handle unlimited_dims
dump_to_store(dataset, zstore, writer, encoding=encoding)
> writes = writer.sync(
compute=compute, chunkmanager_store_kwargs=chunkmanager_store_kwargs
)
#x1B[1m#x1B[31mE AttributeError: 'NoneType' object has no attribute 'sync'#x1B[0m
#x1B[1m#x1B[31m/home/runner/work/xarray/xarray/xarray/backends/api.py#x1B[0m:1783: AttributeError
Check warning on line 0 in xarray.tests.test_backends.TestZarrWriteEmpty
github-actions / Test Results
6 out of 9 runs failed: test_write_empty[True-False] (xarray.tests.test_backends.TestZarrWriteEmpty)
artifacts/Test results for Linux-3.10 min-all-deps/pytest.xml [took 0s]
artifacts/Test results for Linux-3.10/pytest.xml [took 0s]
artifacts/Test results for Linux-3.11 all-but-dask/pytest.xml [took 0s]
artifacts/Test results for Linux-3.12/pytest.xml [took 0s]
artifacts/Test results for macOS-3.10/pytest.xml [took 0s]
artifacts/Test results for macOS-3.12/pytest.xml [took 0s]
Raw output
AttributeError: 'NoneType' object has no attribute 'sync'
self = <xarray.tests.test_backends.TestZarrWriteEmpty object at 0x7fab263d7370>
consolidated = False, write_empty = True
@pytest.mark.parametrize("consolidated", [True, False, None])
@pytest.mark.parametrize("write_empty", [True, False, None])
def test_write_empty(
self, consolidated: bool | None, write_empty: bool | None
) -> None:
if write_empty is False:
expected = ["0.1.0", "1.1.0"]
else:
expected = [
"0.0.0",
"0.0.1",
"0.1.0",
"0.1.1",
"1.0.0",
"1.0.1",
"1.1.0",
"1.1.1",
]
ds = xr.Dataset(
data_vars={
"test": (
("Z", "Y", "X"),
np.array([np.nan, np.nan, 1.0, np.nan]).reshape((1, 2, 2)),
)
}
)
if has_dask:
ds["test"] = ds["test"].chunk(1)
encoding = None
else:
encoding = {"test": {"chunks": (1, 1, 1)}}
with self.temp_dir() as (d, store):
> ds.to_zarr(
store,
mode="w",
encoding=encoding,
write_empty_chunks=write_empty,
)
#x1B[1m#x1B[31m/home/runner/work/xarray/xarray/xarray/tests/test_backends.py#x1B[0m:3388:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
#x1B[1m#x1B[31m/home/runner/work/xarray/xarray/xarray/core/dataset.py#x1B[0m:2566: in to_zarr
return to_zarr( # type: ignore[call-overload,misc]
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
dataset = <xarray.Dataset> Size: 32B
Dimensions: (Z: 1, Y: 2, X: 2)
Dimensions without coordinates: Z, Y, X
Data variables:
test (Z, Y, X) float64 32B dask.array<chunksize=(1, 1, 1), meta=np.ndarray>
store = '/tmp/tmp32vx4nrv/test.zarr', chunk_store = None, mode = 'w'
synchronizer = None, group = None, encoding = {}
def to_zarr(
dataset: Dataset,
store: MutableMapping | str | os.PathLike[str] | None = None,
chunk_store: MutableMapping | str | os.PathLike | None = None,
mode: ZarrWriteModes | None = None,
synchronizer=None,
group: str | None = None,
encoding: Mapping | None = None,
*,
compute: bool = True,
consolidated: bool | None = None,
append_dim: Hashable | None = None,
region: Mapping[str, slice | Literal["auto"]] | Literal["auto"] | None = None,
safe_chunks: bool = True,
storage_options: dict[str, str] | None = None,
zarr_version: int | None = None,
write_empty_chunks: bool | None = None,
chunkmanager_store_kwargs: dict[str, Any] | None = None,
writer: ArrayWriter | None = None,
) -> backends.ZarrStore | Delayed:
"""This function creates an appropriate datastore for writing a dataset to
a zarr ztore
See `Dataset.to_zarr` for full API docs.
"""
# Load empty arrays to avoid bug saving zero length dimensions (Issue #5741)
for v in dataset.variables.values():
if v.size == 0:
v.load()
# expand str and path-like arguments
store = _normalize_path(store)
chunk_store = _normalize_path(chunk_store)
if storage_options is None:
mapper = store
chunk_mapper = chunk_store
else:
from fsspec import get_mapper
if not isinstance(store, str):
raise ValueError(
f"store must be a string to use storage_options. Got {type(store)}"
)
mapper = get_mapper(store, **storage_options)
if chunk_store is not None:
chunk_mapper = get_mapper(chunk_store, **storage_options)
else:
chunk_mapper = chunk_store
if encoding is None:
encoding = {}
if mode is None:
if append_dim is not None:
mode = "a"
elif region is not None:
mode = "r+"
else:
mode = "w-"
if mode not in ["a", "a-"] and append_dim is not None:
raise ValueError("cannot set append_dim unless mode='a' or mode=None")
if mode not in ["a", "a-", "r+"] and region is not None:
raise ValueError(
"cannot set region unless mode='a', mode='a-', mode='r+' or mode=None"
)
if mode not in ["w", "w-", "a", "a-", "r+"]:
raise ValueError(
"The only supported options for mode are 'w', "
f"'w-', 'a', 'a-', and 'r+', but mode={mode!r}"
)
# validate Dataset keys, DataArray names
_validate_dataset_names(dataset)
if zarr_version is None:
# default to 2 if store doesn't specify it's version (e.g. a path)
zarr_version = int(getattr(store, "_store_version", 2))
if consolidated is None and zarr_version > 2:
consolidated = False
if mode == "r+":
already_consolidated = consolidated
consolidate_on_close = False
else:
already_consolidated = False
consolidate_on_close = consolidated or consolidated is None
zstore = backends.ZarrStore.open_group(
store=mapper,
mode=mode,
synchronizer=synchronizer,
group=group,
consolidated=already_consolidated,
consolidate_on_close=consolidate_on_close,
chunk_store=chunk_mapper,
append_dim=append_dim,
write_region=region,
safe_chunks=safe_chunks,
stacklevel=4, # for Dataset.to_zarr()
zarr_version=zarr_version,
write_empty=write_empty_chunks,
)
if region is not None:
zstore._validate_and_autodetect_region(dataset)
# can't modify indexes with region writes
dataset = dataset.drop_vars(dataset.indexes)
if append_dim is not None and append_dim in region:
raise ValueError(
f"cannot list the same dimension in both ``append_dim`` and "
f"``region`` with to_zarr(), got {append_dim} in both"
)
if encoding and mode in ["a", "a-", "r+"]:
existing_var_names = set(zstore.zarr_group.array_keys())
for var_name in existing_var_names:
if var_name in encoding:
raise ValueError(
f"variable {var_name!r} already exists, but encoding was provided"
)
# TODO: figure out how to properly handle unlimited_dims
dump_to_store(dataset, zstore, writer, encoding=encoding)
> writes = writer.sync(
compute=compute, chunkmanager_store_kwargs=chunkmanager_store_kwargs
)
#x1B[1m#x1B[31mE AttributeError: 'NoneType' object has no attribute 'sync'#x1B[0m
#x1B[1m#x1B[31m/home/runner/work/xarray/xarray/xarray/backends/api.py#x1B[0m:1783: AttributeError
Check warning on line 0 in xarray.tests.test_backends.TestZarrWriteEmpty
github-actions / Test Results
6 out of 9 runs failed: test_write_empty[True-None] (xarray.tests.test_backends.TestZarrWriteEmpty)
artifacts/Test results for Linux-3.10 min-all-deps/pytest.xml [took 0s]
artifacts/Test results for Linux-3.10/pytest.xml [took 0s]
artifacts/Test results for Linux-3.11 all-but-dask/pytest.xml [took 0s]
artifacts/Test results for Linux-3.12/pytest.xml [took 0s]
artifacts/Test results for macOS-3.10/pytest.xml [took 0s]
artifacts/Test results for macOS-3.12/pytest.xml [took 0s]
Raw output
AttributeError: 'NoneType' object has no attribute 'sync'
self = <xarray.tests.test_backends.TestZarrWriteEmpty object at 0x7fab263d73d0>
consolidated = None, write_empty = True
@pytest.mark.parametrize("consolidated", [True, False, None])
@pytest.mark.parametrize("write_empty", [True, False, None])
def test_write_empty(
self, consolidated: bool | None, write_empty: bool | None
) -> None:
if write_empty is False:
expected = ["0.1.0", "1.1.0"]
else:
expected = [
"0.0.0",
"0.0.1",
"0.1.0",
"0.1.1",
"1.0.0",
"1.0.1",
"1.1.0",
"1.1.1",
]
ds = xr.Dataset(
data_vars={
"test": (
("Z", "Y", "X"),
np.array([np.nan, np.nan, 1.0, np.nan]).reshape((1, 2, 2)),
)
}
)
if has_dask:
ds["test"] = ds["test"].chunk(1)
encoding = None
else:
encoding = {"test": {"chunks": (1, 1, 1)}}
with self.temp_dir() as (d, store):
> ds.to_zarr(
store,
mode="w",
encoding=encoding,
write_empty_chunks=write_empty,
)
#x1B[1m#x1B[31m/home/runner/work/xarray/xarray/xarray/tests/test_backends.py#x1B[0m:3388:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
#x1B[1m#x1B[31m/home/runner/work/xarray/xarray/xarray/core/dataset.py#x1B[0m:2566: in to_zarr
return to_zarr( # type: ignore[call-overload,misc]
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
dataset = <xarray.Dataset> Size: 32B
Dimensions: (Z: 1, Y: 2, X: 2)
Dimensions without coordinates: Z, Y, X
Data variables:
test (Z, Y, X) float64 32B dask.array<chunksize=(1, 1, 1), meta=np.ndarray>
store = '/tmp/tmp7g5imzau/test.zarr', chunk_store = None, mode = 'w'
synchronizer = None, group = None, encoding = {}
def to_zarr(
dataset: Dataset,
store: MutableMapping | str | os.PathLike[str] | None = None,
chunk_store: MutableMapping | str | os.PathLike | None = None,
mode: ZarrWriteModes | None = None,
synchronizer=None,
group: str | None = None,
encoding: Mapping | None = None,
*,
compute: bool = True,
consolidated: bool | None = None,
append_dim: Hashable | None = None,
region: Mapping[str, slice | Literal["auto"]] | Literal["auto"] | None = None,
safe_chunks: bool = True,
storage_options: dict[str, str] | None = None,
zarr_version: int | None = None,
write_empty_chunks: bool | None = None,
chunkmanager_store_kwargs: dict[str, Any] | None = None,
writer: ArrayWriter | None = None,
) -> backends.ZarrStore | Delayed:
"""This function creates an appropriate datastore for writing a dataset to
a zarr ztore
See `Dataset.to_zarr` for full API docs.
"""
# Load empty arrays to avoid bug saving zero length dimensions (Issue #5741)
for v in dataset.variables.values():
if v.size == 0:
v.load()
# expand str and path-like arguments
store = _normalize_path(store)
chunk_store = _normalize_path(chunk_store)
if storage_options is None:
mapper = store
chunk_mapper = chunk_store
else:
from fsspec import get_mapper
if not isinstance(store, str):
raise ValueError(
f"store must be a string to use storage_options. Got {type(store)}"
)
mapper = get_mapper(store, **storage_options)
if chunk_store is not None:
chunk_mapper = get_mapper(chunk_store, **storage_options)
else:
chunk_mapper = chunk_store
if encoding is None:
encoding = {}
if mode is None:
if append_dim is not None:
mode = "a"
elif region is not None:
mode = "r+"
else:
mode = "w-"
if mode not in ["a", "a-"] and append_dim is not None:
raise ValueError("cannot set append_dim unless mode='a' or mode=None")
if mode not in ["a", "a-", "r+"] and region is not None:
raise ValueError(
"cannot set region unless mode='a', mode='a-', mode='r+' or mode=None"
)
if mode not in ["w", "w-", "a", "a-", "r+"]:
raise ValueError(
"The only supported options for mode are 'w', "
f"'w-', 'a', 'a-', and 'r+', but mode={mode!r}"
)
# validate Dataset keys, DataArray names
_validate_dataset_names(dataset)
if zarr_version is None:
# default to 2 if store doesn't specify it's version (e.g. a path)
zarr_version = int(getattr(store, "_store_version", 2))
if consolidated is None and zarr_version > 2:
consolidated = False
if mode == "r+":
already_consolidated = consolidated
consolidate_on_close = False
else:
already_consolidated = False
consolidate_on_close = consolidated or consolidated is None
zstore = backends.ZarrStore.open_group(
store=mapper,
mode=mode,
synchronizer=synchronizer,
group=group,
consolidated=already_consolidated,
consolidate_on_close=consolidate_on_close,
chunk_store=chunk_mapper,
append_dim=append_dim,
write_region=region,
safe_chunks=safe_chunks,
stacklevel=4, # for Dataset.to_zarr()
zarr_version=zarr_version,
write_empty=write_empty_chunks,
)
if region is not None:
zstore._validate_and_autodetect_region(dataset)
# can't modify indexes with region writes
dataset = dataset.drop_vars(dataset.indexes)
if append_dim is not None and append_dim in region:
raise ValueError(
f"cannot list the same dimension in both ``append_dim`` and "
f"``region`` with to_zarr(), got {append_dim} in both"
)
if encoding and mode in ["a", "a-", "r+"]:
existing_var_names = set(zstore.zarr_group.array_keys())
for var_name in existing_var_names:
if var_name in encoding:
raise ValueError(
f"variable {var_name!r} already exists, but encoding was provided"
)
# TODO: figure out how to properly handle unlimited_dims
dump_to_store(dataset, zstore, writer, encoding=encoding)
> writes = writer.sync(
compute=compute, chunkmanager_store_kwargs=chunkmanager_store_kwargs
)
#x1B[1m#x1B[31mE AttributeError: 'NoneType' object has no attribute 'sync'#x1B[0m
#x1B[1m#x1B[31m/home/runner/work/xarray/xarray/xarray/backends/api.py#x1B[0m:1783: AttributeError
Check warning on line 0 in xarray.tests.test_backends.TestZarrWriteEmpty
github-actions / Test Results
6 out of 9 runs failed: test_write_empty[False-True] (xarray.tests.test_backends.TestZarrWriteEmpty)
artifacts/Test results for Linux-3.10 min-all-deps/pytest.xml [took 0s]
artifacts/Test results for Linux-3.10/pytest.xml [took 0s]
artifacts/Test results for Linux-3.11 all-but-dask/pytest.xml [took 0s]
artifacts/Test results for Linux-3.12/pytest.xml [took 0s]
artifacts/Test results for macOS-3.10/pytest.xml [took 0s]
artifacts/Test results for macOS-3.12/pytest.xml [took 0s]
Raw output
AttributeError: 'NoneType' object has no attribute 'sync'
self = <xarray.tests.test_backends.TestZarrWriteEmpty object at 0x7fab263d7ca0>
consolidated = True, write_empty = False
@pytest.mark.parametrize("consolidated", [True, False, None])
@pytest.mark.parametrize("write_empty", [True, False, None])
def test_write_empty(
self, consolidated: bool | None, write_empty: bool | None
) -> None:
if write_empty is False:
expected = ["0.1.0", "1.1.0"]
else:
expected = [
"0.0.0",
"0.0.1",
"0.1.0",
"0.1.1",
"1.0.0",
"1.0.1",
"1.1.0",
"1.1.1",
]
ds = xr.Dataset(
data_vars={
"test": (
("Z", "Y", "X"),
np.array([np.nan, np.nan, 1.0, np.nan]).reshape((1, 2, 2)),
)
}
)
if has_dask:
ds["test"] = ds["test"].chunk(1)
encoding = None
else:
encoding = {"test": {"chunks": (1, 1, 1)}}
with self.temp_dir() as (d, store):
> ds.to_zarr(
store,
mode="w",
encoding=encoding,
write_empty_chunks=write_empty,
)
#x1B[1m#x1B[31m/home/runner/work/xarray/xarray/xarray/tests/test_backends.py#x1B[0m:3388:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
#x1B[1m#x1B[31m/home/runner/work/xarray/xarray/xarray/core/dataset.py#x1B[0m:2566: in to_zarr
return to_zarr( # type: ignore[call-overload,misc]
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
dataset = <xarray.Dataset> Size: 32B
Dimensions: (Z: 1, Y: 2, X: 2)
Dimensions without coordinates: Z, Y, X
Data variables:
test (Z, Y, X) float64 32B dask.array<chunksize=(1, 1, 1), meta=np.ndarray>
store = '/tmp/tmp1qd4v7tf/test.zarr', chunk_store = None, mode = 'w'
synchronizer = None, group = None, encoding = {}
def to_zarr(
dataset: Dataset,
store: MutableMapping | str | os.PathLike[str] | None = None,
chunk_store: MutableMapping | str | os.PathLike | None = None,
mode: ZarrWriteModes | None = None,
synchronizer=None,
group: str | None = None,
encoding: Mapping | None = None,
*,
compute: bool = True,
consolidated: bool | None = None,
append_dim: Hashable | None = None,
region: Mapping[str, slice | Literal["auto"]] | Literal["auto"] | None = None,
safe_chunks: bool = True,
storage_options: dict[str, str] | None = None,
zarr_version: int | None = None,
write_empty_chunks: bool | None = None,
chunkmanager_store_kwargs: dict[str, Any] | None = None,
writer: ArrayWriter | None = None,
) -> backends.ZarrStore | Delayed:
"""This function creates an appropriate datastore for writing a dataset to
a zarr ztore
See `Dataset.to_zarr` for full API docs.
"""
# Load empty arrays to avoid bug saving zero length dimensions (Issue #5741)
for v in dataset.variables.values():
if v.size == 0:
v.load()
# expand str and path-like arguments
store = _normalize_path(store)
chunk_store = _normalize_path(chunk_store)
if storage_options is None:
mapper = store
chunk_mapper = chunk_store
else:
from fsspec import get_mapper
if not isinstance(store, str):
raise ValueError(
f"store must be a string to use storage_options. Got {type(store)}"
)
mapper = get_mapper(store, **storage_options)
if chunk_store is not None:
chunk_mapper = get_mapper(chunk_store, **storage_options)
else:
chunk_mapper = chunk_store
if encoding is None:
encoding = {}
if mode is None:
if append_dim is not None:
mode = "a"
elif region is not None:
mode = "r+"
else:
mode = "w-"
if mode not in ["a", "a-"] and append_dim is not None:
raise ValueError("cannot set append_dim unless mode='a' or mode=None")
if mode not in ["a", "a-", "r+"] and region is not None:
raise ValueError(
"cannot set region unless mode='a', mode='a-', mode='r+' or mode=None"
)
if mode not in ["w", "w-", "a", "a-", "r+"]:
raise ValueError(
"The only supported options for mode are 'w', "
f"'w-', 'a', 'a-', and 'r+', but mode={mode!r}"
)
# validate Dataset keys, DataArray names
_validate_dataset_names(dataset)
if zarr_version is None:
# default to 2 if store doesn't specify it's version (e.g. a path)
zarr_version = int(getattr(store, "_store_version", 2))
if consolidated is None and zarr_version > 2:
consolidated = False
if mode == "r+":
already_consolidated = consolidated
consolidate_on_close = False
else:
already_consolidated = False
consolidate_on_close = consolidated or consolidated is None
zstore = backends.ZarrStore.open_group(
store=mapper,
mode=mode,
synchronizer=synchronizer,
group=group,
consolidated=already_consolidated,
consolidate_on_close=consolidate_on_close,
chunk_store=chunk_mapper,
append_dim=append_dim,
write_region=region,
safe_chunks=safe_chunks,
stacklevel=4, # for Dataset.to_zarr()
zarr_version=zarr_version,
write_empty=write_empty_chunks,
)
if region is not None:
zstore._validate_and_autodetect_region(dataset)
# can't modify indexes with region writes
dataset = dataset.drop_vars(dataset.indexes)
if append_dim is not None and append_dim in region:
raise ValueError(
f"cannot list the same dimension in both ``append_dim`` and "
f"``region`` with to_zarr(), got {append_dim} in both"
)
if encoding and mode in ["a", "a-", "r+"]:
existing_var_names = set(zstore.zarr_group.array_keys())
for var_name in existing_var_names:
if var_name in encoding:
raise ValueError(
f"variable {var_name!r} already exists, but encoding was provided"
)
# TODO: figure out how to properly handle unlimited_dims
dump_to_store(dataset, zstore, writer, encoding=encoding)
> writes = writer.sync(
compute=compute, chunkmanager_store_kwargs=chunkmanager_store_kwargs
)
#x1B[1m#x1B[31mE AttributeError: 'NoneType' object has no attribute 'sync'#x1B[0m
#x1B[1m#x1B[31m/home/runner/work/xarray/xarray/xarray/backends/api.py#x1B[0m:1783: AttributeError
Check warning on line 0 in xarray.tests.test_backends.TestZarrWriteEmpty
github-actions / Test Results
6 out of 9 runs failed: test_write_empty[False-False] (xarray.tests.test_backends.TestZarrWriteEmpty)
artifacts/Test results for Linux-3.10 min-all-deps/pytest.xml [took 0s]
artifacts/Test results for Linux-3.10/pytest.xml [took 0s]
artifacts/Test results for Linux-3.11 all-but-dask/pytest.xml [took 0s]
artifacts/Test results for Linux-3.12/pytest.xml [took 0s]
artifacts/Test results for macOS-3.10/pytest.xml [took 0s]
artifacts/Test results for macOS-3.12/pytest.xml [took 0s]
Raw output
AttributeError: 'NoneType' object has no attribute 'sync'
self = <xarray.tests.test_backends.TestZarrWriteEmpty object at 0x7fab263d7970>
consolidated = False, write_empty = False
@pytest.mark.parametrize("consolidated", [True, False, None])
@pytest.mark.parametrize("write_empty", [True, False, None])
def test_write_empty(
self, consolidated: bool | None, write_empty: bool | None
) -> None:
if write_empty is False:
expected = ["0.1.0", "1.1.0"]
else:
expected = [
"0.0.0",
"0.0.1",
"0.1.0",
"0.1.1",
"1.0.0",
"1.0.1",
"1.1.0",
"1.1.1",
]
ds = xr.Dataset(
data_vars={
"test": (
("Z", "Y", "X"),
np.array([np.nan, np.nan, 1.0, np.nan]).reshape((1, 2, 2)),
)
}
)
if has_dask:
ds["test"] = ds["test"].chunk(1)
encoding = None
else:
encoding = {"test": {"chunks": (1, 1, 1)}}
with self.temp_dir() as (d, store):
> ds.to_zarr(
store,
mode="w",
encoding=encoding,
write_empty_chunks=write_empty,
)
#x1B[1m#x1B[31m/home/runner/work/xarray/xarray/xarray/tests/test_backends.py#x1B[0m:3388:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
#x1B[1m#x1B[31m/home/runner/work/xarray/xarray/xarray/core/dataset.py#x1B[0m:2566: in to_zarr
return to_zarr( # type: ignore[call-overload,misc]
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
dataset = <xarray.Dataset> Size: 32B
Dimensions: (Z: 1, Y: 2, X: 2)
Dimensions without coordinates: Z, Y, X
Data variables:
test (Z, Y, X) float64 32B dask.array<chunksize=(1, 1, 1), meta=np.ndarray>
store = '/tmp/tmp3dvgdz_j/test.zarr', chunk_store = None, mode = 'w'
synchronizer = None, group = None, encoding = {}
def to_zarr(
dataset: Dataset,
store: MutableMapping | str | os.PathLike[str] | None = None,
chunk_store: MutableMapping | str | os.PathLike | None = None,
mode: ZarrWriteModes | None = None,
synchronizer=None,
group: str | None = None,
encoding: Mapping | None = None,
*,
compute: bool = True,
consolidated: bool | None = None,
append_dim: Hashable | None = None,
region: Mapping[str, slice | Literal["auto"]] | Literal["auto"] | None = None,
safe_chunks: bool = True,
storage_options: dict[str, str] | None = None,
zarr_version: int | None = None,
write_empty_chunks: bool | None = None,
chunkmanager_store_kwargs: dict[str, Any] | None = None,
writer: ArrayWriter | None = None,
) -> backends.ZarrStore | Delayed:
"""This function creates an appropriate datastore for writing a dataset to
a zarr ztore
See `Dataset.to_zarr` for full API docs.
"""
# Load empty arrays to avoid bug saving zero length dimensions (Issue #5741)
for v in dataset.variables.values():
if v.size == 0:
v.load()
# expand str and path-like arguments
store = _normalize_path(store)
chunk_store = _normalize_path(chunk_store)
if storage_options is None:
mapper = store
chunk_mapper = chunk_store
else:
from fsspec import get_mapper
if not isinstance(store, str):
raise ValueError(
f"store must be a string to use storage_options. Got {type(store)}"
)
mapper = get_mapper(store, **storage_options)
if chunk_store is not None:
chunk_mapper = get_mapper(chunk_store, **storage_options)
else:
chunk_mapper = chunk_store
if encoding is None:
encoding = {}
if mode is None:
if append_dim is not None:
mode = "a"
elif region is not None:
mode = "r+"
else:
mode = "w-"
if mode not in ["a", "a-"] and append_dim is not None:
raise ValueError("cannot set append_dim unless mode='a' or mode=None")
if mode not in ["a", "a-", "r+"] and region is not None:
raise ValueError(
"cannot set region unless mode='a', mode='a-', mode='r+' or mode=None"
)
if mode not in ["w", "w-", "a", "a-", "r+"]:
raise ValueError(
"The only supported options for mode are 'w', "
f"'w-', 'a', 'a-', and 'r+', but mode={mode!r}"
)
# validate Dataset keys, DataArray names
_validate_dataset_names(dataset)
if zarr_version is None:
# default to 2 if store doesn't specify it's version (e.g. a path)
zarr_version = int(getattr(store, "_store_version", 2))
if consolidated is None and zarr_version > 2:
consolidated = False
if mode == "r+":
already_consolidated = consolidated
consolidate_on_close = False
else:
already_consolidated = False
consolidate_on_close = consolidated or consolidated is None
zstore = backends.ZarrStore.open_group(
store=mapper,
mode=mode,
synchronizer=synchronizer,
group=group,
consolidated=already_consolidated,
consolidate_on_close=consolidate_on_close,
chunk_store=chunk_mapper,
append_dim=append_dim,
write_region=region,
safe_chunks=safe_chunks,
stacklevel=4, # for Dataset.to_zarr()
zarr_version=zarr_version,
write_empty=write_empty_chunks,
)
if region is not None:
zstore._validate_and_autodetect_region(dataset)
# can't modify indexes with region writes
dataset = dataset.drop_vars(dataset.indexes)
if append_dim is not None and append_dim in region:
raise ValueError(
f"cannot list the same dimension in both ``append_dim`` and "
f"``region`` with to_zarr(), got {append_dim} in both"
)
if encoding and mode in ["a", "a-", "r+"]:
existing_var_names = set(zstore.zarr_group.array_keys())
for var_name in existing_var_names:
if var_name in encoding:
raise ValueError(
f"variable {var_name!r} already exists, but encoding was provided"
)
# TODO: figure out how to properly handle unlimited_dims
dump_to_store(dataset, zstore, writer, encoding=encoding)
> writes = writer.sync(
compute=compute, chunkmanager_store_kwargs=chunkmanager_store_kwargs
)
#x1B[1m#x1B[31mE AttributeError: 'NoneType' object has no attribute 'sync'#x1B[0m
#x1B[1m#x1B[31m/home/runner/work/xarray/xarray/xarray/backends/api.py#x1B[0m:1783: AttributeError
Check warning on line 0 in xarray.tests.test_backends.TestZarrWriteEmpty
github-actions / Test Results
6 out of 9 runs failed: test_write_empty[False-None] (xarray.tests.test_backends.TestZarrWriteEmpty)
artifacts/Test results for Linux-3.10 min-all-deps/pytest.xml [took 0s]
artifacts/Test results for Linux-3.10/pytest.xml [took 0s]
artifacts/Test results for Linux-3.11 all-but-dask/pytest.xml [took 0s]
artifacts/Test results for Linux-3.12/pytest.xml [took 0s]
artifacts/Test results for macOS-3.10/pytest.xml [took 0s]
artifacts/Test results for macOS-3.12/pytest.xml [took 0s]
Raw output
AttributeError: 'NoneType' object has no attribute 'sync'
self = <xarray.tests.test_backends.TestZarrWriteEmpty object at 0x7fab263d7eb0>
consolidated = None, write_empty = False
@pytest.mark.parametrize("consolidated", [True, False, None])
@pytest.mark.parametrize("write_empty", [True, False, None])
def test_write_empty(
self, consolidated: bool | None, write_empty: bool | None
) -> None:
if write_empty is False:
expected = ["0.1.0", "1.1.0"]
else:
expected = [
"0.0.0",
"0.0.1",
"0.1.0",
"0.1.1",
"1.0.0",
"1.0.1",
"1.1.0",
"1.1.1",
]
ds = xr.Dataset(
data_vars={
"test": (
("Z", "Y", "X"),
np.array([np.nan, np.nan, 1.0, np.nan]).reshape((1, 2, 2)),
)
}
)
if has_dask:
ds["test"] = ds["test"].chunk(1)
encoding = None
else:
encoding = {"test": {"chunks": (1, 1, 1)}}
with self.temp_dir() as (d, store):
> ds.to_zarr(
store,
mode="w",
encoding=encoding,
write_empty_chunks=write_empty,
)
#x1B[1m#x1B[31m/home/runner/work/xarray/xarray/xarray/tests/test_backends.py#x1B[0m:3388:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
#x1B[1m#x1B[31m/home/runner/work/xarray/xarray/xarray/core/dataset.py#x1B[0m:2566: in to_zarr
return to_zarr( # type: ignore[call-overload,misc]
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
dataset = <xarray.Dataset> Size: 32B
Dimensions: (Z: 1, Y: 2, X: 2)
Dimensions without coordinates: Z, Y, X
Data variables:
test (Z, Y, X) float64 32B dask.array<chunksize=(1, 1, 1), meta=np.ndarray>
store = '/tmp/tmpsd5jfi9a/test.zarr', chunk_store = None, mode = 'w'
synchronizer = None, group = None, encoding = {}
def to_zarr(
dataset: Dataset,
store: MutableMapping | str | os.PathLike[str] | None = None,
chunk_store: MutableMapping | str | os.PathLike | None = None,
mode: ZarrWriteModes | None = None,
synchronizer=None,
group: str | None = None,
encoding: Mapping | None = None,
*,
compute: bool = True,
consolidated: bool | None = None,
append_dim: Hashable | None = None,
region: Mapping[str, slice | Literal["auto"]] | Literal["auto"] | None = None,
safe_chunks: bool = True,
storage_options: dict[str, str] | None = None,
zarr_version: int | None = None,
write_empty_chunks: bool | None = None,
chunkmanager_store_kwargs: dict[str, Any] | None = None,
writer: ArrayWriter | None = None,
) -> backends.ZarrStore | Delayed:
"""This function creates an appropriate datastore for writing a dataset to
a zarr ztore
See `Dataset.to_zarr` for full API docs.
"""
# Load empty arrays to avoid bug saving zero length dimensions (Issue #5741)
for v in dataset.variables.values():
if v.size == 0:
v.load()
# expand str and path-like arguments
store = _normalize_path(store)
chunk_store = _normalize_path(chunk_store)
if storage_options is None:
mapper = store
chunk_mapper = chunk_store
else:
from fsspec import get_mapper
if not isinstance(store, str):
raise ValueError(
f"store must be a string to use storage_options. Got {type(store)}"
)
mapper = get_mapper(store, **storage_options)
if chunk_store is not None:
chunk_mapper = get_mapper(chunk_store, **storage_options)
else:
chunk_mapper = chunk_store
if encoding is None:
encoding = {}
if mode is None:
if append_dim is not None:
mode = "a"
elif region is not None:
mode = "r+"
else:
mode = "w-"
if mode not in ["a", "a-"] and append_dim is not None:
raise ValueError("cannot set append_dim unless mode='a' or mode=None")
if mode not in ["a", "a-", "r+"] and region is not None:
raise ValueError(
"cannot set region unless mode='a', mode='a-', mode='r+' or mode=None"
)
if mode not in ["w", "w-", "a", "a-", "r+"]:
raise ValueError(
"The only supported options for mode are 'w', "
f"'w-', 'a', 'a-', and 'r+', but mode={mode!r}"
)
# validate Dataset keys, DataArray names
_validate_dataset_names(dataset)
if zarr_version is None:
# default to 2 if store doesn't specify it's version (e.g. a path)
zarr_version = int(getattr(store, "_store_version", 2))
if consolidated is None and zarr_version > 2:
consolidated = False
if mode == "r+":
already_consolidated = consolidated
consolidate_on_close = False
else:
already_consolidated = False
consolidate_on_close = consolidated or consolidated is None
zstore = backends.ZarrStore.open_group(
store=mapper,
mode=mode,
synchronizer=synchronizer,
group=group,
consolidated=already_consolidated,
consolidate_on_close=consolidate_on_close,
chunk_store=chunk_mapper,
append_dim=append_dim,
write_region=region,
safe_chunks=safe_chunks,
stacklevel=4, # for Dataset.to_zarr()
zarr_version=zarr_version,
write_empty=write_empty_chunks,
)
if region is not None:
zstore._validate_and_autodetect_region(dataset)
# can't modify indexes with region writes
dataset = dataset.drop_vars(dataset.indexes)
if append_dim is not None and append_dim in region:
raise ValueError(
f"cannot list the same dimension in both ``append_dim`` and "
f"``region`` with to_zarr(), got {append_dim} in both"
)
if encoding and mode in ["a", "a-", "r+"]:
existing_var_names = set(zstore.zarr_group.array_keys())
for var_name in existing_var_names:
if var_name in encoding:
raise ValueError(
f"variable {var_name!r} already exists, but encoding was provided"
)
# TODO: figure out how to properly handle unlimited_dims
dump_to_store(dataset, zstore, writer, encoding=encoding)
> writes = writer.sync(
compute=compute, chunkmanager_store_kwargs=chunkmanager_store_kwargs
)
#x1B[1m#x1B[31mE AttributeError: 'NoneType' object has no attribute 'sync'#x1B[0m
#x1B[1m#x1B[31m/home/runner/work/xarray/xarray/xarray/backends/api.py#x1B[0m:1783: AttributeError
Check warning on line 0 in xarray.tests.test_backends.TestZarrWriteEmpty
github-actions / Test Results
6 out of 9 runs failed: test_write_empty[None-True] (xarray.tests.test_backends.TestZarrWriteEmpty)
artifacts/Test results for Linux-3.10 min-all-deps/pytest.xml [took 0s]
artifacts/Test results for Linux-3.10/pytest.xml [took 0s]
artifacts/Test results for Linux-3.11 all-but-dask/pytest.xml [took 0s]
artifacts/Test results for Linux-3.12/pytest.xml [took 0s]
artifacts/Test results for macOS-3.10/pytest.xml [took 0s]
artifacts/Test results for macOS-3.12/pytest.xml [took 0s]
Raw output
AttributeError: 'NoneType' object has no attribute 'sync'
self = <xarray.tests.test_backends.TestZarrWriteEmpty object at 0x7fab263d7f10>
consolidated = True, write_empty = None
@pytest.mark.parametrize("consolidated", [True, False, None])
@pytest.mark.parametrize("write_empty", [True, False, None])
def test_write_empty(
self, consolidated: bool | None, write_empty: bool | None
) -> None:
if write_empty is False:
expected = ["0.1.0", "1.1.0"]
else:
expected = [
"0.0.0",
"0.0.1",
"0.1.0",
"0.1.1",
"1.0.0",
"1.0.1",
"1.1.0",
"1.1.1",
]
ds = xr.Dataset(
data_vars={
"test": (
("Z", "Y", "X"),
np.array([np.nan, np.nan, 1.0, np.nan]).reshape((1, 2, 2)),
)
}
)
if has_dask:
ds["test"] = ds["test"].chunk(1)
encoding = None
else:
encoding = {"test": {"chunks": (1, 1, 1)}}
with self.temp_dir() as (d, store):
> ds.to_zarr(
store,
mode="w",
encoding=encoding,
write_empty_chunks=write_empty,
)
#x1B[1m#x1B[31m/home/runner/work/xarray/xarray/xarray/tests/test_backends.py#x1B[0m:3388:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
#x1B[1m#x1B[31m/home/runner/work/xarray/xarray/xarray/core/dataset.py#x1B[0m:2566: in to_zarr
return to_zarr( # type: ignore[call-overload,misc]
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
dataset = <xarray.Dataset> Size: 32B
Dimensions: (Z: 1, Y: 2, X: 2)
Dimensions without coordinates: Z, Y, X
Data variables:
test (Z, Y, X) float64 32B dask.array<chunksize=(1, 1, 1), meta=np.ndarray>
store = '/tmp/tmpled4r_lh/test.zarr', chunk_store = None, mode = 'w'
synchronizer = None, group = None, encoding = {}
def to_zarr(
dataset: Dataset,
store: MutableMapping | str | os.PathLike[str] | None = None,
chunk_store: MutableMapping | str | os.PathLike | None = None,
mode: ZarrWriteModes | None = None,
synchronizer=None,
group: str | None = None,
encoding: Mapping | None = None,
*,
compute: bool = True,
consolidated: bool | None = None,
append_dim: Hashable | None = None,
region: Mapping[str, slice | Literal["auto"]] | Literal["auto"] | None = None,
safe_chunks: bool = True,
storage_options: dict[str, str] | None = None,
zarr_version: int | None = None,
write_empty_chunks: bool | None = None,
chunkmanager_store_kwargs: dict[str, Any] | None = None,
writer: ArrayWriter | None = None,
) -> backends.ZarrStore | Delayed:
"""This function creates an appropriate datastore for writing a dataset to
a zarr ztore
See `Dataset.to_zarr` for full API docs.
"""
# Load empty arrays to avoid bug saving zero length dimensions (Issue #5741)
for v in dataset.variables.values():
if v.size == 0:
v.load()
# expand str and path-like arguments
store = _normalize_path(store)
chunk_store = _normalize_path(chunk_store)
if storage_options is None:
mapper = store
chunk_mapper = chunk_store
else:
from fsspec import get_mapper
if not isinstance(store, str):
raise ValueError(
f"store must be a string to use storage_options. Got {type(store)}"
)
mapper = get_mapper(store, **storage_options)
if chunk_store is not None:
chunk_mapper = get_mapper(chunk_store, **storage_options)
else:
chunk_mapper = chunk_store
if encoding is None:
encoding = {}
if mode is None:
if append_dim is not None:
mode = "a"
elif region is not None:
mode = "r+"
else:
mode = "w-"
if mode not in ["a", "a-"] and append_dim is not None:
raise ValueError("cannot set append_dim unless mode='a' or mode=None")
if mode not in ["a", "a-", "r+"] and region is not None:
raise ValueError(
"cannot set region unless mode='a', mode='a-', mode='r+' or mode=None"
)
if mode not in ["w", "w-", "a", "a-", "r+"]:
raise ValueError(
"The only supported options for mode are 'w', "
f"'w-', 'a', 'a-', and 'r+', but mode={mode!r}"
)
# validate Dataset keys, DataArray names
_validate_dataset_names(dataset)
if zarr_version is None:
# default to 2 if store doesn't specify it's version (e.g. a path)
zarr_version = int(getattr(store, "_store_version", 2))
if consolidated is None and zarr_version > 2:
consolidated = False
if mode == "r+":
already_consolidated = consolidated
consolidate_on_close = False
else:
already_consolidated = False
consolidate_on_close = consolidated or consolidated is None
zstore = backends.ZarrStore.open_group(
store=mapper,
mode=mode,
synchronizer=synchronizer,
group=group,
consolidated=already_consolidated,
consolidate_on_close=consolidate_on_close,
chunk_store=chunk_mapper,
append_dim=append_dim,
write_region=region,
safe_chunks=safe_chunks,
stacklevel=4, # for Dataset.to_zarr()
zarr_version=zarr_version,
write_empty=write_empty_chunks,
)
if region is not None:
zstore._validate_and_autodetect_region(dataset)
# can't modify indexes with region writes
dataset = dataset.drop_vars(dataset.indexes)
if append_dim is not None and append_dim in region:
raise ValueError(
f"cannot list the same dimension in both ``append_dim`` and "
f"``region`` with to_zarr(), got {append_dim} in both"
)
if encoding and mode in ["a", "a-", "r+"]:
existing_var_names = set(zstore.zarr_group.array_keys())
for var_name in existing_var_names:
if var_name in encoding:
raise ValueError(
f"variable {var_name!r} already exists, but encoding was provided"
)
# TODO: figure out how to properly handle unlimited_dims
dump_to_store(dataset, zstore, writer, encoding=encoding)
> writes = writer.sync(
compute=compute, chunkmanager_store_kwargs=chunkmanager_store_kwargs
)
#x1B[1m#x1B[31mE AttributeError: 'NoneType' object has no attribute 'sync'#x1B[0m
#x1B[1m#x1B[31m/home/runner/work/xarray/xarray/xarray/backends/api.py#x1B[0m:1783: AttributeError
Check warning on line 0 in xarray.tests.test_backends.TestZarrWriteEmpty
github-actions / Test Results
6 out of 9 runs failed: test_write_empty[None-False] (xarray.tests.test_backends.TestZarrWriteEmpty)
artifacts/Test results for Linux-3.10 min-all-deps/pytest.xml [took 0s]
artifacts/Test results for Linux-3.10/pytest.xml [took 0s]
artifacts/Test results for Linux-3.11 all-but-dask/pytest.xml [took 0s]
artifacts/Test results for Linux-3.12/pytest.xml [took 0s]
artifacts/Test results for macOS-3.10/pytest.xml [took 0s]
artifacts/Test results for macOS-3.12/pytest.xml [took 0s]
Raw output
AttributeError: 'NoneType' object has no attribute 'sync'
self = <xarray.tests.test_backends.TestZarrWriteEmpty object at 0x7fab263d5c00>
consolidated = False, write_empty = None
@pytest.mark.parametrize("consolidated", [True, False, None])
@pytest.mark.parametrize("write_empty", [True, False, None])
def test_write_empty(
self, consolidated: bool | None, write_empty: bool | None
) -> None:
if write_empty is False:
expected = ["0.1.0", "1.1.0"]
else:
expected = [
"0.0.0",
"0.0.1",
"0.1.0",
"0.1.1",
"1.0.0",
"1.0.1",
"1.1.0",
"1.1.1",
]
ds = xr.Dataset(
data_vars={
"test": (
("Z", "Y", "X"),
np.array([np.nan, np.nan, 1.0, np.nan]).reshape((1, 2, 2)),
)
}
)
if has_dask:
ds["test"] = ds["test"].chunk(1)
encoding = None
else:
encoding = {"test": {"chunks": (1, 1, 1)}}
with self.temp_dir() as (d, store):
> ds.to_zarr(
store,
mode="w",
encoding=encoding,
write_empty_chunks=write_empty,
)
#x1B[1m#x1B[31m/home/runner/work/xarray/xarray/xarray/tests/test_backends.py#x1B[0m:3388:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
#x1B[1m#x1B[31m/home/runner/work/xarray/xarray/xarray/core/dataset.py#x1B[0m:2566: in to_zarr
return to_zarr( # type: ignore[call-overload,misc]
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
dataset = <xarray.Dataset> Size: 32B
Dimensions: (Z: 1, Y: 2, X: 2)
Dimensions without coordinates: Z, Y, X
Data variables:
test (Z, Y, X) float64 32B dask.array<chunksize=(1, 1, 1), meta=np.ndarray>
store = '/tmp/tmp74wdkejr/test.zarr', chunk_store = None, mode = 'w'
synchronizer = None, group = None, encoding = {}
def to_zarr(
dataset: Dataset,
store: MutableMapping | str | os.PathLike[str] | None = None,
chunk_store: MutableMapping | str | os.PathLike | None = None,
mode: ZarrWriteModes | None = None,
synchronizer=None,
group: str | None = None,
encoding: Mapping | None = None,
*,
compute: bool = True,
consolidated: bool | None = None,
append_dim: Hashable | None = None,
region: Mapping[str, slice | Literal["auto"]] | Literal["auto"] | None = None,
safe_chunks: bool = True,
storage_options: dict[str, str] | None = None,
zarr_version: int | None = None,
write_empty_chunks: bool | None = None,
chunkmanager_store_kwargs: dict[str, Any] | None = None,
writer: ArrayWriter | None = None,
) -> backends.ZarrStore | Delayed:
"""This function creates an appropriate datastore for writing a dataset to
a zarr ztore
See `Dataset.to_zarr` for full API docs.
"""
# Load empty arrays to avoid bug saving zero length dimensions (Issue #5741)
for v in dataset.variables.values():
if v.size == 0:
v.load()
# expand str and path-like arguments
store = _normalize_path(store)
chunk_store = _normalize_path(chunk_store)
if storage_options is None:
mapper = store
chunk_mapper = chunk_store
else:
from fsspec import get_mapper
if not isinstance(store, str):
raise ValueError(
f"store must be a string to use storage_options. Got {type(store)}"
)
mapper = get_mapper(store, **storage_options)
if chunk_store is not None:
chunk_mapper = get_mapper(chunk_store, **storage_options)
else:
chunk_mapper = chunk_store
if encoding is None:
encoding = {}
if mode is None:
if append_dim is not None:
mode = "a"
elif region is not None:
mode = "r+"
else:
mode = "w-"
if mode not in ["a", "a-"] and append_dim is not None:
raise ValueError("cannot set append_dim unless mode='a' or mode=None")
if mode not in ["a", "a-", "r+"] and region is not None:
raise ValueError(
"cannot set region unless mode='a', mode='a-', mode='r+' or mode=None"
)
if mode not in ["w", "w-", "a", "a-", "r+"]:
raise ValueError(
"The only supported options for mode are 'w', "
f"'w-', 'a', 'a-', and 'r+', but mode={mode!r}"
)
# validate Dataset keys, DataArray names
_validate_dataset_names(dataset)
if zarr_version is None:
# default to 2 if store doesn't specify it's version (e.g. a path)
zarr_version = int(getattr(store, "_store_version", 2))
if consolidated is None and zarr_version > 2:
consolidated = False
if mode == "r+":
already_consolidated = consolidated
consolidate_on_close = False
else:
already_consolidated = False
consolidate_on_close = consolidated or consolidated is None
zstore = backends.ZarrStore.open_group(
store=mapper,
mode=mode,
synchronizer=synchronizer,
group=group,
consolidated=already_consolidated,
consolidate_on_close=consolidate_on_close,
chunk_store=chunk_mapper,
append_dim=append_dim,
write_region=region,
safe_chunks=safe_chunks,
stacklevel=4, # for Dataset.to_zarr()
zarr_version=zarr_version,
write_empty=write_empty_chunks,
)
if region is not None:
zstore._validate_and_autodetect_region(dataset)
# can't modify indexes with region writes
dataset = dataset.drop_vars(dataset.indexes)
if append_dim is not None and append_dim in region:
raise ValueError(
f"cannot list the same dimension in both ``append_dim`` and "
f"``region`` with to_zarr(), got {append_dim} in both"
)
if encoding and mode in ["a", "a-", "r+"]:
existing_var_names = set(zstore.zarr_group.array_keys())
for var_name in existing_var_names:
if var_name in encoding:
raise ValueError(
f"variable {var_name!r} already exists, but encoding was provided"
)
# TODO: figure out how to properly handle unlimited_dims
dump_to_store(dataset, zstore, writer, encoding=encoding)
> writes = writer.sync(
compute=compute, chunkmanager_store_kwargs=chunkmanager_store_kwargs
)
#x1B[1m#x1B[31mE AttributeError: 'NoneType' object has no attribute 'sync'#x1B[0m
#x1B[1m#x1B[31m/home/runner/work/xarray/xarray/xarray/backends/api.py#x1B[0m:1783: AttributeError
Check warning on line 0 in xarray.tests.test_backends.TestZarrWriteEmpty
github-actions / Test Results
6 out of 9 runs failed: test_write_empty[None-None] (xarray.tests.test_backends.TestZarrWriteEmpty)
artifacts/Test results for Linux-3.10 min-all-deps/pytest.xml [took 0s]
artifacts/Test results for Linux-3.10/pytest.xml [took 0s]
artifacts/Test results for Linux-3.11 all-but-dask/pytest.xml [took 0s]
artifacts/Test results for Linux-3.12/pytest.xml [took 0s]
artifacts/Test results for macOS-3.10/pytest.xml [took 0s]
artifacts/Test results for macOS-3.12/pytest.xml [took 0s]
Raw output
AttributeError: 'NoneType' object has no attribute 'sync'
self = <xarray.tests.test_backends.TestZarrWriteEmpty object at 0x7fab263d6380>
consolidated = None, write_empty = None
@pytest.mark.parametrize("consolidated", [True, False, None])
@pytest.mark.parametrize("write_empty", [True, False, None])
def test_write_empty(
self, consolidated: bool | None, write_empty: bool | None
) -> None:
if write_empty is False:
expected = ["0.1.0", "1.1.0"]
else:
expected = [
"0.0.0",
"0.0.1",
"0.1.0",
"0.1.1",
"1.0.0",
"1.0.1",
"1.1.0",
"1.1.1",
]
ds = xr.Dataset(
data_vars={
"test": (
("Z", "Y", "X"),
np.array([np.nan, np.nan, 1.0, np.nan]).reshape((1, 2, 2)),
)
}
)
if has_dask:
ds["test"] = ds["test"].chunk(1)
encoding = None
else:
encoding = {"test": {"chunks": (1, 1, 1)}}
with self.temp_dir() as (d, store):
> ds.to_zarr(
store,
mode="w",
encoding=encoding,
write_empty_chunks=write_empty,
)
#x1B[1m#x1B[31m/home/runner/work/xarray/xarray/xarray/tests/test_backends.py#x1B[0m:3388:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
#x1B[1m#x1B[31m/home/runner/work/xarray/xarray/xarray/core/dataset.py#x1B[0m:2566: in to_zarr
return to_zarr( # type: ignore[call-overload,misc]
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
dataset = <xarray.Dataset> Size: 32B
Dimensions: (Z: 1, Y: 2, X: 2)
Dimensions without coordinates: Z, Y, X
Data variables:
test (Z, Y, X) float64 32B dask.array<chunksize=(1, 1, 1), meta=np.ndarray>
store = '/tmp/tmp74q8k_kh/test.zarr', chunk_store = None, mode = 'w'
synchronizer = None, group = None, encoding = {}
def to_zarr(
dataset: Dataset,
store: MutableMapping | str | os.PathLike[str] | None = None,
chunk_store: MutableMapping | str | os.PathLike | None = None,
mode: ZarrWriteModes | None = None,
synchronizer=None,
group: str | None = None,
encoding: Mapping | None = None,
*,
compute: bool = True,
consolidated: bool | None = None,
append_dim: Hashable | None = None,
region: Mapping[str, slice | Literal["auto"]] | Literal["auto"] | None = None,
safe_chunks: bool = True,
storage_options: dict[str, str] | None = None,
zarr_version: int | None = None,
write_empty_chunks: bool | None = None,
chunkmanager_store_kwargs: dict[str, Any] | None = None,
writer: ArrayWriter | None = None,
) -> backends.ZarrStore | Delayed:
"""This function creates an appropriate datastore for writing a dataset to
a zarr ztore
See `Dataset.to_zarr` for full API docs.
"""
# Load empty arrays to avoid bug saving zero length dimensions (Issue #5741)
for v in dataset.variables.values():
if v.size == 0:
v.load()
# expand str and path-like arguments
store = _normalize_path(store)
chunk_store = _normalize_path(chunk_store)
if storage_options is None:
mapper = store
chunk_mapper = chunk_store
else:
from fsspec import get_mapper
if not isinstance(store, str):
raise ValueError(
f"store must be a string to use storage_options. Got {type(store)}"
)
mapper = get_mapper(store, **storage_options)
if chunk_store is not None:
chunk_mapper = get_mapper(chunk_store, **storage_options)
else:
chunk_mapper = chunk_store
if encoding is None:
encoding = {}
if mode is None:
if append_dim is not None:
mode = "a"
elif region is not None:
mode = "r+"
else:
mode = "w-"
if mode not in ["a", "a-"] and append_dim is not None:
raise ValueError("cannot set append_dim unless mode='a' or mode=None")
if mode not in ["a", "a-", "r+"] and region is not None:
raise ValueError(
"cannot set region unless mode='a', mode='a-', mode='r+' or mode=None"
)
if mode not in ["w", "w-", "a", "a-", "r+"]:
raise ValueError(
"The only supported options for mode are 'w', "
f"'w-', 'a', 'a-', and 'r+', but mode={mode!r}"
)
# validate Dataset keys, DataArray names
_validate_dataset_names(dataset)
if zarr_version is None:
# default to 2 if store doesn't specify it's version (e.g. a path)
zarr_version = int(getattr(store, "_store_version", 2))
if consolidated is None and zarr_version > 2:
consolidated = False
if mode == "r+":
already_consolidated = consolidated
consolidate_on_close = False
else:
already_consolidated = False
consolidate_on_close = consolidated or consolidated is None
zstore = backends.ZarrStore.open_group(
store=mapper,
mode=mode,
synchronizer=synchronizer,
group=group,
consolidated=already_consolidated,
consolidate_on_close=consolidate_on_close,
chunk_store=chunk_mapper,
append_dim=append_dim,
write_region=region,
safe_chunks=safe_chunks,
stacklevel=4, # for Dataset.to_zarr()
zarr_version=zarr_version,
write_empty=write_empty_chunks,
)
if region is not None:
zstore._validate_and_autodetect_region(dataset)
# can't modify indexes with region writes
dataset = dataset.drop_vars(dataset.indexes)
if append_dim is not None and append_dim in region:
raise ValueError(
f"cannot list the same dimension in both ``append_dim`` and "
f"``region`` with to_zarr(), got {append_dim} in both"
)
if encoding and mode in ["a", "a-", "r+"]:
existing_var_names = set(zstore.zarr_group.array_keys())
for var_name in existing_var_names:
if var_name in encoding:
raise ValueError(
f"variable {var_name!r} already exists, but encoding was provided"
)
# TODO: figure out how to properly handle unlimited_dims
dump_to_store(dataset, zstore, writer, encoding=encoding)
> writes = writer.sync(
compute=compute, chunkmanager_store_kwargs=chunkmanager_store_kwargs
)
#x1B[1m#x1B[31mE AttributeError: 'NoneType' object has no attribute 'sync'#x1B[0m
#x1B[1m#x1B[31m/home/runner/work/xarray/xarray/xarray/backends/api.py#x1B[0m:1783: AttributeError
Check warning on line 0 in xarray.tests.test_backends.TestDataArrayToZarr
github-actions / Test Results
8 out of 9 runs failed: test_dataarray_to_zarr_no_name[tmp_path] (xarray.tests.test_backends.TestDataArrayToZarr)
artifacts/Test results for Linux-3.10 min-all-deps/pytest.xml [took 0s]
artifacts/Test results for Linux-3.10/pytest.xml [took 0s]
artifacts/Test results for Linux-3.11 all-but-dask/pytest.xml [took 0s]
artifacts/Test results for Linux-3.12/pytest.xml [took 0s]
artifacts/Test results for Windows-3.10/pytest.xml [took 0s]
artifacts/Test results for Windows-3.12/pytest.xml [took 0s]
artifacts/Test results for macOS-3.10/pytest.xml [took 0s]
artifacts/Test results for macOS-3.12/pytest.xml [took 0s]
Raw output
AttributeError: 'NoneType' object has no attribute 'sync'
self = <xarray.tests.test_backends.TestDataArrayToZarr object at 0x7f3ff49c5300>
tmp_store = PosixPath('/tmp/pytest-of-runner/pytest-0/popen-gw2/test_dataarray_to_zarr_no_name0')
def test_dataarray_to_zarr_no_name(self, tmp_store) -> None:
original_da = DataArray(np.arange(12).reshape((3, 4)))
> original_da.to_zarr(tmp_store)
#x1B[1m#x1B[31m/home/runner/work/xarray/xarray/xarray/tests/test_backends.py#x1B[0m:5223:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
#x1B[1m#x1B[31m/home/runner/work/xarray/xarray/xarray/core/dataarray.py#x1B[0m:4355: in to_zarr
return to_zarr( # type: ignore[call-overload,misc]
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
dataset = <xarray.Dataset> Size: 96B
Dimensions: (dim_0: 3, dim_1: 4)
Dimensions without coordinates: dim_0, dim_1
Data variables:
__xarray_dataarray_variable__ (dim_0, dim_1) int64 96B 0 1 2 3 ... 9 10 11
store = '/tmp/pytest-of-runner/pytest-0/popen-gw2/test_dataarray_to_zarr_no_name0'
chunk_store = None, mode = 'w-', synchronizer = None, group = None
encoding = {}
def to_zarr(
dataset: Dataset,
store: MutableMapping | str | os.PathLike[str] | None = None,
chunk_store: MutableMapping | str | os.PathLike | None = None,
mode: ZarrWriteModes | None = None,
synchronizer=None,
group: str | None = None,
encoding: Mapping | None = None,
*,
compute: bool = True,
consolidated: bool | None = None,
append_dim: Hashable | None = None,
region: Mapping[str, slice | Literal["auto"]] | Literal["auto"] | None = None,
safe_chunks: bool = True,
storage_options: dict[str, str] | None = None,
zarr_version: int | None = None,
write_empty_chunks: bool | None = None,
chunkmanager_store_kwargs: dict[str, Any] | None = None,
writer: ArrayWriter | None = None,
) -> backends.ZarrStore | Delayed:
"""This function creates an appropriate datastore for writing a dataset to
a zarr ztore
See `Dataset.to_zarr` for full API docs.
"""
# Load empty arrays to avoid bug saving zero length dimensions (Issue #5741)
for v in dataset.variables.values():
if v.size == 0:
v.load()
# expand str and path-like arguments
store = _normalize_path(store)
chunk_store = _normalize_path(chunk_store)
if storage_options is None:
mapper = store
chunk_mapper = chunk_store
else:
from fsspec import get_mapper
if not isinstance(store, str):
raise ValueError(
f"store must be a string to use storage_options. Got {type(store)}"
)
mapper = get_mapper(store, **storage_options)
if chunk_store is not None:
chunk_mapper = get_mapper(chunk_store, **storage_options)
else:
chunk_mapper = chunk_store
if encoding is None:
encoding = {}
if mode is None:
if append_dim is not None:
mode = "a"
elif region is not None:
mode = "r+"
else:
mode = "w-"
if mode not in ["a", "a-"] and append_dim is not None:
raise ValueError("cannot set append_dim unless mode='a' or mode=None")
if mode not in ["a", "a-", "r+"] and region is not None:
raise ValueError(
"cannot set region unless mode='a', mode='a-', mode='r+' or mode=None"
)
if mode not in ["w", "w-", "a", "a-", "r+"]:
raise ValueError(
"The only supported options for mode are 'w', "
f"'w-', 'a', 'a-', and 'r+', but mode={mode!r}"
)
# validate Dataset keys, DataArray names
_validate_dataset_names(dataset)
if zarr_version is None:
# default to 2 if store doesn't specify it's version (e.g. a path)
zarr_version = int(getattr(store, "_store_version", 2))
if consolidated is None and zarr_version > 2:
consolidated = False
if mode == "r+":
already_consolidated = consolidated
consolidate_on_close = False
else:
already_consolidated = False
consolidate_on_close = consolidated or consolidated is None
zstore = backends.ZarrStore.open_group(
store=mapper,
mode=mode,
synchronizer=synchronizer,
group=group,
consolidated=already_consolidated,
consolidate_on_close=consolidate_on_close,
chunk_store=chunk_mapper,
append_dim=append_dim,
write_region=region,
safe_chunks=safe_chunks,
stacklevel=4, # for Dataset.to_zarr()
zarr_version=zarr_version,
write_empty=write_empty_chunks,
)
if region is not None:
zstore._validate_and_autodetect_region(dataset)
# can't modify indexes with region writes
dataset = dataset.drop_vars(dataset.indexes)
if append_dim is not None and append_dim in region:
raise ValueError(
f"cannot list the same dimension in both ``append_dim`` and "
f"``region`` with to_zarr(), got {append_dim} in both"
)
if encoding and mode in ["a", "a-", "r+"]:
existing_var_names = set(zstore.zarr_group.array_keys())
for var_name in existing_var_names:
if var_name in encoding:
raise ValueError(
f"variable {var_name!r} already exists, but encoding was provided"
)
# TODO: figure out how to properly handle unlimited_dims
dump_to_store(dataset, zstore, writer, encoding=encoding)
> writes = writer.sync(
compute=compute, chunkmanager_store_kwargs=chunkmanager_store_kwargs
)
#x1B[1m#x1B[31mE AttributeError: 'NoneType' object has no attribute 'sync'#x1B[0m
#x1B[1m#x1B[31m/home/runner/work/xarray/xarray/xarray/backends/api.py#x1B[0m:1783: AttributeError
Check warning on line 0 in xarray.tests.test_backends.TestZarrWriteEmpty
github-actions / Test Results
6 out of 9 runs failed: test_avoid_excess_metadata_calls (xarray.tests.test_backends.TestZarrWriteEmpty)
artifacts/Test results for Linux-3.10 min-all-deps/pytest.xml [took 0s]
artifacts/Test results for Linux-3.10/pytest.xml [took 0s]
artifacts/Test results for Linux-3.11 all-but-dask/pytest.xml [took 0s]
artifacts/Test results for Linux-3.12/pytest.xml [took 0s]
artifacts/Test results for macOS-3.10/pytest.xml [took 0s]
artifacts/Test results for macOS-3.12/pytest.xml [took 0s]
Raw output
AttributeError: 'NoneType' object has no attribute 'sync'
self = <xarray.tests.test_backends.TestZarrWriteEmpty object at 0x7fab263d5a20>
def test_avoid_excess_metadata_calls(self) -> None:
"""Test that chunk requests do not trigger redundant metadata requests.
This test targets logic in backends.zarr.ZarrArrayWrapper, asserting that calls
to retrieve chunk data after initialization do not trigger additional
metadata requests.
https://github.com/pydata/xarray/issues/8290
"""
import zarr
ds = xr.Dataset(data_vars={"test": (("Z",), np.array([123]).reshape(1))})
# The call to retrieve metadata performs a group lookup. We patch Group.__getitem__
# so that we can inspect calls to this method - specifically count of calls.
# Use of side_effect means that calls are passed through to the original method
# rather than a mocked method.
Group = zarr.hierarchy.Group
with (
self.create_zarr_target() as store,
patch.object(
Group, "__getitem__", side_effect=Group.__getitem__, autospec=True
) as mock,
):
> ds.to_zarr(store, mode="w")
#x1B[1m#x1B[31m/home/runner/work/xarray/xarray/xarray/tests/test_backends.py#x1B[0m:3432:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
#x1B[1m#x1B[31m/home/runner/work/xarray/xarray/xarray/core/dataset.py#x1B[0m:2566: in to_zarr
return to_zarr( # type: ignore[call-overload,misc]
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
dataset = <xarray.Dataset> Size: 8B
Dimensions: (Z: 1)
Dimensions without coordinates: Z
Data variables:
test (Z) int64 8B 123
store = '/tmp/tmpi4ubm0j6/temp-0.zarr', chunk_store = None, mode = 'w'
synchronizer = None, group = None, encoding = {}
def to_zarr(
dataset: Dataset,
store: MutableMapping | str | os.PathLike[str] | None = None,
chunk_store: MutableMapping | str | os.PathLike | None = None,
mode: ZarrWriteModes | None = None,
synchronizer=None,
group: str | None = None,
encoding: Mapping | None = None,
*,
compute: bool = True,
consolidated: bool | None = None,
append_dim: Hashable | None = None,
region: Mapping[str, slice | Literal["auto"]] | Literal["auto"] | None = None,
safe_chunks: bool = True,
storage_options: dict[str, str] | None = None,
zarr_version: int | None = None,
write_empty_chunks: bool | None = None,
chunkmanager_store_kwargs: dict[str, Any] | None = None,
writer: ArrayWriter | None = None,
) -> backends.ZarrStore | Delayed:
"""This function creates an appropriate datastore for writing a dataset to
a zarr ztore
See `Dataset.to_zarr` for full API docs.
"""
# Load empty arrays to avoid bug saving zero length dimensions (Issue #5741)
for v in dataset.variables.values():
if v.size == 0:
v.load()
# expand str and path-like arguments
store = _normalize_path(store)
chunk_store = _normalize_path(chunk_store)
if storage_options is None:
mapper = store
chunk_mapper = chunk_store
else:
from fsspec import get_mapper
if not isinstance(store, str):
raise ValueError(
f"store must be a string to use storage_options. Got {type(store)}"
)
mapper = get_mapper(store, **storage_options)
if chunk_store is not None:
chunk_mapper = get_mapper(chunk_store, **storage_options)
else:
chunk_mapper = chunk_store
if encoding is None:
encoding = {}
if mode is None:
if append_dim is not None:
mode = "a"
elif region is not None:
mode = "r+"
else:
mode = "w-"
if mode not in ["a", "a-"] and append_dim is not None:
raise ValueError("cannot set append_dim unless mode='a' or mode=None")
if mode not in ["a", "a-", "r+"] and region is not None:
raise ValueError(
"cannot set region unless mode='a', mode='a-', mode='r+' or mode=None"
)
if mode not in ["w", "w-", "a", "a-", "r+"]:
raise ValueError(
"The only supported options for mode are 'w', "
f"'w-', 'a', 'a-', and 'r+', but mode={mode!r}"
)
# validate Dataset keys, DataArray names
_validate_dataset_names(dataset)
if zarr_version is None:
# default to 2 if store doesn't specify it's version (e.g. a path)
zarr_version = int(getattr(store, "_store_version", 2))
if consolidated is None and zarr_version > 2:
consolidated = False
if mode == "r+":
already_consolidated = consolidated
consolidate_on_close = False
else:
already_consolidated = False
consolidate_on_close = consolidated or consolidated is None
zstore = backends.ZarrStore.open_group(
store=mapper,
mode=mode,
synchronizer=synchronizer,
group=group,
consolidated=already_consolidated,
consolidate_on_close=consolidate_on_close,
chunk_store=chunk_mapper,
append_dim=append_dim,
write_region=region,
safe_chunks=safe_chunks,
stacklevel=4, # for Dataset.to_zarr()
zarr_version=zarr_version,
write_empty=write_empty_chunks,
)
if region is not None:
zstore._validate_and_autodetect_region(dataset)
# can't modify indexes with region writes
dataset = dataset.drop_vars(dataset.indexes)
if append_dim is not None and append_dim in region:
raise ValueError(
f"cannot list the same dimension in both ``append_dim`` and "
f"``region`` with to_zarr(), got {append_dim} in both"
)
if encoding and mode in ["a", "a-", "r+"]:
existing_var_names = set(zstore.zarr_group.array_keys())
for var_name in existing_var_names:
if var_name in encoding:
raise ValueError(
f"variable {var_name!r} already exists, but encoding was provided"
)
# TODO: figure out how to properly handle unlimited_dims
dump_to_store(dataset, zstore, writer, encoding=encoding)
> writes = writer.sync(
compute=compute, chunkmanager_store_kwargs=chunkmanager_store_kwargs
)
#x1B[1m#x1B[31mE AttributeError: 'NoneType' object has no attribute 'sync'#x1B[0m
#x1B[1m#x1B[31m/home/runner/work/xarray/xarray/xarray/backends/api.py#x1B[0m:1783: AttributeError
Check warning on line 0 in xarray.tests.test_backends.TestDataArrayToZarr
github-actions / Test Results
8 out of 9 runs failed: test_dataarray_to_zarr_no_name[ZipStore] (xarray.tests.test_backends.TestDataArrayToZarr)
artifacts/Test results for Linux-3.10 min-all-deps/pytest.xml [took 0s]
artifacts/Test results for Linux-3.10/pytest.xml [took 0s]
artifacts/Test results for Linux-3.11 all-but-dask/pytest.xml [took 0s]
artifacts/Test results for Linux-3.12/pytest.xml [took 0s]
artifacts/Test results for Windows-3.10/pytest.xml [took 0s]
artifacts/Test results for Windows-3.12/pytest.xml [took 0s]
artifacts/Test results for macOS-3.10/pytest.xml [took 0s]
artifacts/Test results for macOS-3.12/pytest.xml [took 0s]
Raw output
AttributeError: 'NoneType' object has no attribute 'sync'
self = <xarray.tests.test_backends.TestDataArrayToZarr object at 0x7f3ff49c50f0>
tmp_store = <zarr.storage.ZipStore object at 0x7f3fda0e1240>
def test_dataarray_to_zarr_no_name(self, tmp_store) -> None:
original_da = DataArray(np.arange(12).reshape((3, 4)))
> original_da.to_zarr(tmp_store)
#x1B[1m#x1B[31m/home/runner/work/xarray/xarray/xarray/tests/test_backends.py#x1B[0m:5223:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
#x1B[1m#x1B[31m/home/runner/work/xarray/xarray/xarray/core/dataarray.py#x1B[0m:4355: in to_zarr
return to_zarr( # type: ignore[call-overload,misc]
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
dataset = <xarray.Dataset> Size: 96B
Dimensions: (dim_0: 3, dim_1: 4)
Dimensions without coordinates: dim_0, dim_1
Data variables:
__xarray_dataarray_variable__ (dim_0, dim_1) int64 96B 0 1 2 3 ... 9 10 11
store = <zarr.storage.ZipStore object at 0x7f3fda0e1240>, chunk_store = None
mode = 'w-', synchronizer = None, group = None, encoding = {}
def to_zarr(
dataset: Dataset,
store: MutableMapping | str | os.PathLike[str] | None = None,
chunk_store: MutableMapping | str | os.PathLike | None = None,
mode: ZarrWriteModes | None = None,
synchronizer=None,
group: str | None = None,
encoding: Mapping | None = None,
*,
compute: bool = True,
consolidated: bool | None = None,
append_dim: Hashable | None = None,
region: Mapping[str, slice | Literal["auto"]] | Literal["auto"] | None = None,
safe_chunks: bool = True,
storage_options: dict[str, str] | None = None,
zarr_version: int | None = None,
write_empty_chunks: bool | None = None,
chunkmanager_store_kwargs: dict[str, Any] | None = None,
writer: ArrayWriter | None = None,
) -> backends.ZarrStore | Delayed:
"""This function creates an appropriate datastore for writing a dataset to
a zarr ztore
See `Dataset.to_zarr` for full API docs.
"""
# Load empty arrays to avoid bug saving zero length dimensions (Issue #5741)
for v in dataset.variables.values():
if v.size == 0:
v.load()
# expand str and path-like arguments
store = _normalize_path(store)
chunk_store = _normalize_path(chunk_store)
if storage_options is None:
mapper = store
chunk_mapper = chunk_store
else:
from fsspec import get_mapper
if not isinstance(store, str):
raise ValueError(
f"store must be a string to use storage_options. Got {type(store)}"
)
mapper = get_mapper(store, **storage_options)
if chunk_store is not None:
chunk_mapper = get_mapper(chunk_store, **storage_options)
else:
chunk_mapper = chunk_store
if encoding is None:
encoding = {}
if mode is None:
if append_dim is not None:
mode = "a"
elif region is not None:
mode = "r+"
else:
mode = "w-"
if mode not in ["a", "a-"] and append_dim is not None:
raise ValueError("cannot set append_dim unless mode='a' or mode=None")
if mode not in ["a", "a-", "r+"] and region is not None:
raise ValueError(
"cannot set region unless mode='a', mode='a-', mode='r+' or mode=None"
)
if mode not in ["w", "w-", "a", "a-", "r+"]:
raise ValueError(
"The only supported options for mode are 'w', "
f"'w-', 'a', 'a-', and 'r+', but mode={mode!r}"
)
# validate Dataset keys, DataArray names
_validate_dataset_names(dataset)
if zarr_version is None:
# default to 2 if store doesn't specify it's version (e.g. a path)
zarr_version = int(getattr(store, "_store_version", 2))
if consolidated is None and zarr_version > 2:
consolidated = False
if mode == "r+":
already_consolidated = consolidated
consolidate_on_close = False
else:
already_consolidated = False
consolidate_on_close = consolidated or consolidated is None
zstore = backends.ZarrStore.open_group(
store=mapper,
mode=mode,
synchronizer=synchronizer,
group=group,
consolidated=already_consolidated,
consolidate_on_close=consolidate_on_close,
chunk_store=chunk_mapper,
append_dim=append_dim,
write_region=region,
safe_chunks=safe_chunks,
stacklevel=4, # for Dataset.to_zarr()
zarr_version=zarr_version,
write_empty=write_empty_chunks,
)
if region is not None:
zstore._validate_and_autodetect_region(dataset)
# can't modify indexes with region writes
dataset = dataset.drop_vars(dataset.indexes)
if append_dim is not None and append_dim in region:
raise ValueError(
f"cannot list the same dimension in both ``append_dim`` and "
f"``region`` with to_zarr(), got {append_dim} in both"
)
if encoding and mode in ["a", "a-", "r+"]:
existing_var_names = set(zstore.zarr_group.array_keys())
for var_name in existing_var_names:
if var_name in encoding:
raise ValueError(
f"variable {var_name!r} already exists, but encoding was provided"
)
# TODO: figure out how to properly handle unlimited_dims
dump_to_store(dataset, zstore, writer, encoding=encoding)
> writes = writer.sync(
compute=compute, chunkmanager_store_kwargs=chunkmanager_store_kwargs
)
#x1B[1m#x1B[31mE AttributeError: 'NoneType' object has no attribute 'sync'#x1B[0m
#x1B[1m#x1B[31m/home/runner/work/xarray/xarray/xarray/backends/api.py#x1B[0m:1783: AttributeError
Check warning on line 0 in xarray.tests.test_backends.TestDataArrayToZarr
github-actions / Test Results
8 out of 9 runs failed: test_dataarray_to_zarr_no_name[Dict] (xarray.tests.test_backends.TestDataArrayToZarr)
artifacts/Test results for Linux-3.10 min-all-deps/pytest.xml [took 0s]
artifacts/Test results for Linux-3.10/pytest.xml [took 0s]
artifacts/Test results for Linux-3.11 all-but-dask/pytest.xml [took 0s]
artifacts/Test results for Linux-3.12/pytest.xml [took 0s]
artifacts/Test results for Windows-3.10/pytest.xml [took 0s]
artifacts/Test results for Windows-3.12/pytest.xml [took 0s]
artifacts/Test results for macOS-3.10/pytest.xml [took 0s]
artifacts/Test results for macOS-3.12/pytest.xml [took 0s]
Raw output
AttributeError: 'NoneType' object has no attribute 'sync'
self = <xarray.tests.test_backends.TestDataArrayToZarr object at 0x7f3ff49c41f0>
tmp_store = {'.zattrs': b'{}', '.zgroup': b'{\n "zarr_format": 2\n}', '.zmetadata': b'{\n "metadata": {\n ".zattrs": ...n "filters": null,\n "order": "C",\n "shape": [\n 3,\n 4\n ],\n "zarr_format": 2\n}', ...}
def test_dataarray_to_zarr_no_name(self, tmp_store) -> None:
original_da = DataArray(np.arange(12).reshape((3, 4)))
> original_da.to_zarr(tmp_store)
#x1B[1m#x1B[31m/home/runner/work/xarray/xarray/xarray/tests/test_backends.py#x1B[0m:5223:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
#x1B[1m#x1B[31m/home/runner/work/xarray/xarray/xarray/core/dataarray.py#x1B[0m:4355: in to_zarr
return to_zarr( # type: ignore[call-overload,misc]
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
dataset = <xarray.Dataset> Size: 96B
Dimensions: (dim_0: 3, dim_1: 4)
Dimensions without coordinates: dim_0, dim_1
Data variables:
__xarray_dataarray_variable__ (dim_0, dim_1) int64 96B 0 1 2 3 ... 9 10 11
store = {'.zattrs': b'{}', '.zgroup': b'{\n "zarr_format": 2\n}', '.zmetadata': b'{\n "metadata": {\n ".zattrs": ...n "filters": null,\n "order": "C",\n "shape": [\n 3,\n 4\n ],\n "zarr_format": 2\n}', ...}
chunk_store = None, mode = 'w-', synchronizer = None, group = None
encoding = {}
def to_zarr(
dataset: Dataset,
store: MutableMapping | str | os.PathLike[str] | None = None,
chunk_store: MutableMapping | str | os.PathLike | None = None,
mode: ZarrWriteModes | None = None,
synchronizer=None,
group: str | None = None,
encoding: Mapping | None = None,
*,
compute: bool = True,
consolidated: bool | None = None,
append_dim: Hashable | None = None,
region: Mapping[str, slice | Literal["auto"]] | Literal["auto"] | None = None,
safe_chunks: bool = True,
storage_options: dict[str, str] | None = None,
zarr_version: int | None = None,
write_empty_chunks: bool | None = None,
chunkmanager_store_kwargs: dict[str, Any] | None = None,
writer: ArrayWriter | None = None,
) -> backends.ZarrStore | Delayed:
"""This function creates an appropriate datastore for writing a dataset to
a zarr ztore
See `Dataset.to_zarr` for full API docs.
"""
# Load empty arrays to avoid bug saving zero length dimensions (Issue #5741)
for v in dataset.variables.values():
if v.size == 0:
v.load()
# expand str and path-like arguments
store = _normalize_path(store)
chunk_store = _normalize_path(chunk_store)
if storage_options is None:
mapper = store
chunk_mapper = chunk_store
else:
from fsspec import get_mapper
if not isinstance(store, str):
raise ValueError(
f"store must be a string to use storage_options. Got {type(store)}"
)
mapper = get_mapper(store, **storage_options)
if chunk_store is not None:
chunk_mapper = get_mapper(chunk_store, **storage_options)
else:
chunk_mapper = chunk_store
if encoding is None:
encoding = {}
if mode is None:
if append_dim is not None:
mode = "a"
elif region is not None:
mode = "r+"
else:
mode = "w-"
if mode not in ["a", "a-"] and append_dim is not None:
raise ValueError("cannot set append_dim unless mode='a' or mode=None")
if mode not in ["a", "a-", "r+"] and region is not None:
raise ValueError(
"cannot set region unless mode='a', mode='a-', mode='r+' or mode=None"
)
if mode not in ["w", "w-", "a", "a-", "r+"]:
raise ValueError(
"The only supported options for mode are 'w', "
f"'w-', 'a', 'a-', and 'r+', but mode={mode!r}"
)
# validate Dataset keys, DataArray names
_validate_dataset_names(dataset)
if zarr_version is None:
# default to 2 if store doesn't specify it's version (e.g. a path)
zarr_version = int(getattr(store, "_store_version", 2))
if consolidated is None and zarr_version > 2:
consolidated = False
if mode == "r+":
already_consolidated = consolidated
consolidate_on_close = False
else:
already_consolidated = False
consolidate_on_close = consolidated or consolidated is None
zstore = backends.ZarrStore.open_group(
store=mapper,
mode=mode,
synchronizer=synchronizer,
group=group,
consolidated=already_consolidated,
consolidate_on_close=consolidate_on_close,
chunk_store=chunk_mapper,
append_dim=append_dim,
write_region=region,
safe_chunks=safe_chunks,
stacklevel=4, # for Dataset.to_zarr()
zarr_version=zarr_version,
write_empty=write_empty_chunks,
)
if region is not None:
zstore._validate_and_autodetect_region(dataset)
# can't modify indexes with region writes
dataset = dataset.drop_vars(dataset.indexes)
if append_dim is not None and append_dim in region:
raise ValueError(
f"cannot list the same dimension in both ``append_dim`` and "
f"``region`` with to_zarr(), got {append_dim} in both"
)
if encoding and mode in ["a", "a-", "r+"]:
existing_var_names = set(zstore.zarr_group.array_keys())
for var_name in existing_var_names:
if var_name in encoding:
raise ValueError(
f"variable {var_name!r} already exists, but encoding was provided"
)
# TODO: figure out how to properly handle unlimited_dims
dump_to_store(dataset, zstore, writer, encoding=encoding)
> writes = writer.sync(
compute=compute, chunkmanager_store_kwargs=chunkmanager_store_kwargs
)
#x1B[1m#x1B[31mE AttributeError: 'NoneType' object has no attribute 'sync'#x1B[0m
#x1B[1m#x1B[31m/home/runner/work/xarray/xarray/xarray/backends/api.py#x1B[0m:1783: AttributeError
Check warning on line 0 in xarray.tests.test_backends.TestZarrKVStoreV3
github-actions / Test Results
8 out of 9 runs failed: test_zero_dimensional_variable (xarray.tests.test_backends.TestZarrKVStoreV3)
artifacts/Test results for Linux-3.10 min-all-deps/pytest.xml [took 0s]
artifacts/Test results for Linux-3.10/pytest.xml [took 0s]
artifacts/Test results for Linux-3.11 all-but-dask/pytest.xml [took 0s]
artifacts/Test results for Linux-3.12/pytest.xml [took 0s]
artifacts/Test results for Windows-3.10/pytest.xml [took 0s]
artifacts/Test results for Windows-3.12/pytest.xml [took 0s]
artifacts/Test results for macOS-3.10/pytest.xml [took 0s]
artifacts/Test results for macOS-3.12/pytest.xml [took 0s]
Raw output
AttributeError: 'NoneType' object has no attribute 'sync'
self = <xarray.tests.test_backends.TestZarrKVStoreV3 object at 0x7fab2498c640>
def test_zero_dimensional_variable(self) -> None:
expected = create_test_data()
expected["float_var"] = ([], 1.0e9, {"units": "units of awesome"})
expected["bytes_var"] = ([], b"foobar")
expected["string_var"] = ([], "foobar")
> with self.roundtrip(expected) as actual:
#x1B[1m#x1B[31m/home/runner/work/xarray/xarray/xarray/tests/test_backends.py#x1B[0m:382:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
#x1B[1m#x1B[31m/home/runner/micromamba/envs/xarray-tests/lib/python3.10/contextlib.py#x1B[0m:135: in __enter__
return next(self.gen)
#x1B[1m#x1B[31m/home/runner/work/xarray/xarray/xarray/tests/test_backends.py#x1B[0m:2238: in roundtrip
self.save(data, store_target, **save_kwargs)
#x1B[1m#x1B[31m/home/runner/work/xarray/xarray/xarray/tests/test_backends.py#x1B[0m:2220: in save
return dataset.to_zarr(store=store_target, **kwargs, **self.version_kwargs)
#x1B[1m#x1B[31m/home/runner/work/xarray/xarray/xarray/core/dataset.py#x1B[0m:2566: in to_zarr
return to_zarr( # type: ignore[call-overload,misc]
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
dataset = <xarray.Dataset> Size: 2kB
Dimensions: (dim2: 9, dim3: 10, time: 20, dim1: 8)
Coordinates:
* dim2 (dim2) ... ... -0.5001 -0.4647
float_var float64 8B 1e+09
bytes_var |S6 6B b'foobar'
string_var <U6 24B 'foobar'
store = <KVStoreV3:
{'zarr.json': b'{\n "extensions": [],\n "metadata_encoding": "https://purl.org/zarr/spec/protocol/c...\xd2)\xd7\xbfc/\xa9*\x89I\xef?\nF\xe6K\x002\xe5?\x84Kz=\xf6\x00\xe0\xbf\x0fcw\xc5\xd5\xbc\xdd\xbf"}
at 0x7fab24d0ff40>
chunk_store = None, mode = 'w-', synchronizer = None, group = None
encoding = {}
def to_zarr(
dataset: Dataset,
store: MutableMapping | str | os.PathLike[str] | None = None,
chunk_store: MutableMapping | str | os.PathLike | None = None,
mode: ZarrWriteModes | None = None,
synchronizer=None,
group: str | None = None,
encoding: Mapping | None = None,
*,
compute: bool = True,
consolidated: bool | None = None,
append_dim: Hashable | None = None,
region: Mapping[str, slice | Literal["auto"]] | Literal["auto"] | None = None,
safe_chunks: bool = True,
storage_options: dict[str, str] | None = None,
zarr_version: int | None = None,
write_empty_chunks: bool | None = None,
chunkmanager_store_kwargs: dict[str, Any] | None = None,
writer: ArrayWriter | None = None,
) -> backends.ZarrStore | Delayed:
"""This function creates an appropriate datastore for writing a dataset to
a zarr ztore
See `Dataset.to_zarr` for full API docs.
"""
# Load empty arrays to avoid bug saving zero length dimensions (Issue #5741)
for v in dataset.variables.values():
if v.size == 0:
v.load()
# expand str and path-like arguments
store = _normalize_path(store)
chunk_store = _normalize_path(chunk_store)
if storage_options is None:
mapper = store
chunk_mapper = chunk_store
else:
from fsspec import get_mapper
if not isinstance(store, str):
raise ValueError(
f"store must be a string to use storage_options. Got {type(store)}"
)
mapper = get_mapper(store, **storage_options)
if chunk_store is not None:
chunk_mapper = get_mapper(chunk_store, **storage_options)
else:
chunk_mapper = chunk_store
if encoding is None:
encoding = {}
if mode is None:
if append_dim is not None:
mode = "a"
elif region is not None:
mode = "r+"
else:
mode = "w-"
if mode not in ["a", "a-"] and append_dim is not None:
raise ValueError("cannot set append_dim unless mode='a' or mode=None")
if mode not in ["a", "a-", "r+"] and region is not None:
raise ValueError(
"cannot set region unless mode='a', mode='a-', mode='r+' or mode=None"
)
if mode not in ["w", "w-", "a", "a-", "r+"]:
raise ValueError(
"The only supported options for mode are 'w', "
f"'w-', 'a', 'a-', and 'r+', but mode={mode!r}"
)
# validate Dataset keys, DataArray names
_validate_dataset_names(dataset)
if zarr_version is None:
# default to 2 if store doesn't specify it's version (e.g. a path)
zarr_version = int(getattr(store, "_store_version", 2))
if consolidated is None and zarr_version > 2:
consolidated = False
if mode == "r+":
already_consolidated = consolidated
consolidate_on_close = False
else:
already_consolidated = False
consolidate_on_close = consolidated or consolidated is None
zstore = backends.ZarrStore.open_group(
store=mapper,
mode=mode,
synchronizer=synchronizer,
group=group,
consolidated=already_consolidated,
consolidate_on_close=consolidate_on_close,
chunk_store=chunk_mapper,
append_dim=append_dim,
write_region=region,
safe_chunks=safe_chunks,
stacklevel=4, # for Dataset.to_zarr()
zarr_version=zarr_version,
write_empty=write_empty_chunks,
)
if region is not None:
zstore._validate_and_autodetect_region(dataset)
# can't modify indexes with region writes
dataset = dataset.drop_vars(dataset.indexes)
if append_dim is not None and append_dim in region:
raise ValueError(
f"cannot list the same dimension in both ``append_dim`` and "
f"``region`` with to_zarr(), got {append_dim} in both"
)
if encoding and mode in ["a", "a-", "r+"]:
existing_var_names = set(zstore.zarr_group.array_keys())
for var_name in existing_var_names:
if var_name in encoding:
raise ValueError(
f"variable {var_name!r} already exists, but encoding was provided"
)
# TODO: figure out how to properly handle unlimited_dims
dump_to_store(dataset, zstore, writer, encoding=encoding)
> writes = writer.sync(
compute=compute, chunkmanager_store_kwargs=chunkmanager_store_kwargs
)
#x1B[1m#x1B[31mE AttributeError: 'NoneType' object has no attribute 'sync'#x1B[0m
#x1B[1m#x1B[31m/home/runner/work/xarray/xarray/xarray/backends/api.py#x1B[0m:1783: AttributeError
Check warning on line 0 in xarray.tests.test_backends.TestDataArrayToZarr
github-actions / Test Results
8 out of 9 runs failed: test_dataarray_to_zarr_with_name[tmp_path] (xarray.tests.test_backends.TestDataArrayToZarr)
artifacts/Test results for Linux-3.10 min-all-deps/pytest.xml [took 0s]
artifacts/Test results for Linux-3.10/pytest.xml [took 0s]
artifacts/Test results for Linux-3.11 all-but-dask/pytest.xml [took 0s]
artifacts/Test results for Linux-3.12/pytest.xml [took 0s]
artifacts/Test results for Windows-3.10/pytest.xml [took 0s]
artifacts/Test results for Windows-3.12/pytest.xml [took 0s]
artifacts/Test results for macOS-3.10/pytest.xml [took 0s]
artifacts/Test results for macOS-3.12/pytest.xml [took 0s]
Raw output
AttributeError: 'NoneType' object has no attribute 'sync'
self = <xarray.tests.test_backends.TestDataArrayToZarr object at 0x7f3ff49c6fb0>
tmp_store = PosixPath('/tmp/pytest-of-runner/pytest-0/popen-gw2/test_dataarray_to_zarr_with_na0')
def test_dataarray_to_zarr_with_name(self, tmp_store) -> None:
original_da = DataArray(np.arange(12).reshape((3, 4)), name="test")
> original_da.to_zarr(tmp_store)
#x1B[1m#x1B[31m/home/runner/work/xarray/xarray/xarray/tests/test_backends.py#x1B[0m:5231:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
#x1B[1m#x1B[31m/home/runner/work/xarray/xarray/xarray/core/dataarray.py#x1B[0m:4355: in to_zarr
return to_zarr( # type: ignore[call-overload,misc]
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
dataset = <xarray.Dataset> Size: 96B
Dimensions: (dim_0: 3, dim_1: 4)
Dimensions without coordinates: dim_0, dim_1
Data variables:
test (dim_0, dim_1) int64 96B 0 1 2 3 4 5 6 7 8 9 10 11
store = '/tmp/pytest-of-runner/pytest-0/popen-gw2/test_dataarray_to_zarr_with_na0'
chunk_store = None, mode = 'w-', synchronizer = None, group = None
encoding = {}
def to_zarr(
dataset: Dataset,
store: MutableMapping | str | os.PathLike[str] | None = None,
chunk_store: MutableMapping | str | os.PathLike | None = None,
mode: ZarrWriteModes | None = None,
synchronizer=None,
group: str | None = None,
encoding: Mapping | None = None,
*,
compute: bool = True,
consolidated: bool | None = None,
append_dim: Hashable | None = None,
region: Mapping[str, slice | Literal["auto"]] | Literal["auto"] | None = None,
safe_chunks: bool = True,
storage_options: dict[str, str] | None = None,
zarr_version: int | None = None,
write_empty_chunks: bool | None = None,
chunkmanager_store_kwargs: dict[str, Any] | None = None,
writer: ArrayWriter | None = None,
) -> backends.ZarrStore | Delayed:
"""This function creates an appropriate datastore for writing a dataset to
a zarr ztore
See `Dataset.to_zarr` for full API docs.
"""
# Load empty arrays to avoid bug saving zero length dimensions (Issue #5741)
for v in dataset.variables.values():
if v.size == 0:
v.load()
# expand str and path-like arguments
store = _normalize_path(store)
chunk_store = _normalize_path(chunk_store)
if storage_options is None:
mapper = store
chunk_mapper = chunk_store
else:
from fsspec import get_mapper
if not isinstance(store, str):
raise ValueError(
f"store must be a string to use storage_options. Got {type(store)}"
)
mapper = get_mapper(store, **storage_options)
if chunk_store is not None:
chunk_mapper = get_mapper(chunk_store, **storage_options)
else:
chunk_mapper = chunk_store
if encoding is None:
encoding = {}
if mode is None:
if append_dim is not None:
mode = "a"
elif region is not None:
mode = "r+"
else:
mode = "w-"
if mode not in ["a", "a-"] and append_dim is not None:
raise ValueError("cannot set append_dim unless mode='a' or mode=None")
if mode not in ["a", "a-", "r+"] and region is not None:
raise ValueError(
"cannot set region unless mode='a', mode='a-', mode='r+' or mode=None"
)
if mode not in ["w", "w-", "a", "a-", "r+"]:
raise ValueError(
"The only supported options for mode are 'w', "
f"'w-', 'a', 'a-', and 'r+', but mode={mode!r}"
)
# validate Dataset keys, DataArray names
_validate_dataset_names(dataset)
if zarr_version is None:
# default to 2 if store doesn't specify it's version (e.g. a path)
zarr_version = int(getattr(store, "_store_version", 2))
if consolidated is None and zarr_version > 2:
consolidated = False
if mode == "r+":
already_consolidated = consolidated
consolidate_on_close = False
else:
already_consolidated = False
consolidate_on_close = consolidated or consolidated is None
zstore = backends.ZarrStore.open_group(
store=mapper,
mode=mode,
synchronizer=synchronizer,
group=group,
consolidated=already_consolidated,
consolidate_on_close=consolidate_on_close,
chunk_store=chunk_mapper,
append_dim=append_dim,
write_region=region,
safe_chunks=safe_chunks,
stacklevel=4, # for Dataset.to_zarr()
zarr_version=zarr_version,
write_empty=write_empty_chunks,
)
if region is not None:
zstore._validate_and_autodetect_region(dataset)
# can't modify indexes with region writes
dataset = dataset.drop_vars(dataset.indexes)
if append_dim is not None and append_dim in region:
raise ValueError(
f"cannot list the same dimension in both ``append_dim`` and "
f"``region`` with to_zarr(), got {append_dim} in both"
)
if encoding and mode in ["a", "a-", "r+"]:
existing_var_names = set(zstore.zarr_group.array_keys())
for var_name in existing_var_names:
if var_name in encoding:
raise ValueError(
f"variable {var_name!r} already exists, but encoding was provided"
)
# TODO: figure out how to properly handle unlimited_dims
dump_to_store(dataset, zstore, writer, encoding=encoding)
> writes = writer.sync(
compute=compute, chunkmanager_store_kwargs=chunkmanager_store_kwargs
)
#x1B[1m#x1B[31mE AttributeError: 'NoneType' object has no attribute 'sync'#x1B[0m
#x1B[1m#x1B[31m/home/runner/work/xarray/xarray/xarray/backends/api.py#x1B[0m:1783: AttributeError
Check warning on line 0 in xarray.tests.test_backends.TestDataArrayToZarr
github-actions / Test Results
8 out of 9 runs failed: test_dataarray_to_zarr_with_name[ZipStore] (xarray.tests.test_backends.TestDataArrayToZarr)
artifacts/Test results for Linux-3.10 min-all-deps/pytest.xml [took 0s]
artifacts/Test results for Linux-3.10/pytest.xml [took 0s]
artifacts/Test results for Linux-3.11 all-but-dask/pytest.xml [took 0s]
artifacts/Test results for Linux-3.12/pytest.xml [took 0s]
artifacts/Test results for Windows-3.10/pytest.xml [took 0s]
artifacts/Test results for Windows-3.12/pytest.xml [took 0s]
artifacts/Test results for macOS-3.10/pytest.xml [took 0s]
artifacts/Test results for macOS-3.12/pytest.xml [took 0s]
Raw output
AttributeError: 'NoneType' object has no attribute 'sync'
self = <xarray.tests.test_backends.TestDataArrayToZarr object at 0x7f3ff49c6fe0>
tmp_store = <zarr.storage.ZipStore object at 0x7f3ff52bf820>
def test_dataarray_to_zarr_with_name(self, tmp_store) -> None:
original_da = DataArray(np.arange(12).reshape((3, 4)), name="test")
> original_da.to_zarr(tmp_store)
#x1B[1m#x1B[31m/home/runner/work/xarray/xarray/xarray/tests/test_backends.py#x1B[0m:5231:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
#x1B[1m#x1B[31m/home/runner/work/xarray/xarray/xarray/core/dataarray.py#x1B[0m:4355: in to_zarr
return to_zarr( # type: ignore[call-overload,misc]
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
dataset = <xarray.Dataset> Size: 96B
Dimensions: (dim_0: 3, dim_1: 4)
Dimensions without coordinates: dim_0, dim_1
Data variables:
test (dim_0, dim_1) int64 96B 0 1 2 3 4 5 6 7 8 9 10 11
store = <zarr.storage.ZipStore object at 0x7f3ff52bf820>, chunk_store = None
mode = 'w-', synchronizer = None, group = None, encoding = {}
def to_zarr(
dataset: Dataset,
store: MutableMapping | str | os.PathLike[str] | None = None,
chunk_store: MutableMapping | str | os.PathLike | None = None,
mode: ZarrWriteModes | None = None,
synchronizer=None,
group: str | None = None,
encoding: Mapping | None = None,
*,
compute: bool = True,
consolidated: bool | None = None,
append_dim: Hashable | None = None,
region: Mapping[str, slice | Literal["auto"]] | Literal["auto"] | None = None,
safe_chunks: bool = True,
storage_options: dict[str, str] | None = None,
zarr_version: int | None = None,
write_empty_chunks: bool | None = None,
chunkmanager_store_kwargs: dict[str, Any] | None = None,
writer: ArrayWriter | None = None,
) -> backends.ZarrStore | Delayed:
"""This function creates an appropriate datastore for writing a dataset to
a zarr ztore
See `Dataset.to_zarr` for full API docs.
"""
# Load empty arrays to avoid bug saving zero length dimensions (Issue #5741)
for v in dataset.variables.values():
if v.size == 0:
v.load()
# expand str and path-like arguments
store = _normalize_path(store)
chunk_store = _normalize_path(chunk_store)
if storage_options is None:
mapper = store
chunk_mapper = chunk_store
else:
from fsspec import get_mapper
if not isinstance(store, str):
raise ValueError(
f"store must be a string to use storage_options. Got {type(store)}"
)
mapper = get_mapper(store, **storage_options)
if chunk_store is not None:
chunk_mapper = get_mapper(chunk_store, **storage_options)
else:
chunk_mapper = chunk_store
if encoding is None:
encoding = {}
if mode is None:
if append_dim is not None:
mode = "a"
elif region is not None:
mode = "r+"
else:
mode = "w-"
if mode not in ["a", "a-"] and append_dim is not None:
raise ValueError("cannot set append_dim unless mode='a' or mode=None")
if mode not in ["a", "a-", "r+"] and region is not None:
raise ValueError(
"cannot set region unless mode='a', mode='a-', mode='r+' or mode=None"
)
if mode not in ["w", "w-", "a", "a-", "r+"]:
raise ValueError(
"The only supported options for mode are 'w', "
f"'w-', 'a', 'a-', and 'r+', but mode={mode!r}"
)
# validate Dataset keys, DataArray names
_validate_dataset_names(dataset)
if zarr_version is None:
# default to 2 if store doesn't specify it's version (e.g. a path)
zarr_version = int(getattr(store, "_store_version", 2))
if consolidated is None and zarr_version > 2:
consolidated = False
if mode == "r+":
already_consolidated = consolidated
consolidate_on_close = False
else:
already_consolidated = False
consolidate_on_close = consolidated or consolidated is None
zstore = backends.ZarrStore.open_group(
store=mapper,
mode=mode,
synchronizer=synchronizer,
group=group,
consolidated=already_consolidated,
consolidate_on_close=consolidate_on_close,
chunk_store=chunk_mapper,
append_dim=append_dim,
write_region=region,
safe_chunks=safe_chunks,
stacklevel=4, # for Dataset.to_zarr()
zarr_version=zarr_version,
write_empty=write_empty_chunks,
)
if region is not None:
zstore._validate_and_autodetect_region(dataset)
# can't modify indexes with region writes
dataset = dataset.drop_vars(dataset.indexes)
if append_dim is not None and append_dim in region:
raise ValueError(
f"cannot list the same dimension in both ``append_dim`` and "
f"``region`` with to_zarr(), got {append_dim} in both"
)
if encoding and mode in ["a", "a-", "r+"]:
existing_var_names = set(zstore.zarr_group.array_keys())
for var_name in existing_var_names:
if var_name in encoding:
raise ValueError(
f"variable {var_name!r} already exists, but encoding was provided"
)
# TODO: figure out how to properly handle unlimited_dims
dump_to_store(dataset, zstore, writer, encoding=encoding)
> writes = writer.sync(
compute=compute, chunkmanager_store_kwargs=chunkmanager_store_kwargs
)
#x1B[1m#x1B[31mE AttributeError: 'NoneType' object has no attribute 'sync'#x1B[0m
#x1B[1m#x1B[31m/home/runner/work/xarray/xarray/xarray/backends/api.py#x1B[0m:1783: AttributeError
Check warning on line 0 in xarray.tests.test_backends.TestZarrKVStoreV3
github-actions / Test Results
8 out of 9 runs failed: test_roundtrip_test_data (xarray.tests.test_backends.TestZarrKVStoreV3)
artifacts/Test results for Linux-3.10 min-all-deps/pytest.xml [took 0s]
artifacts/Test results for Linux-3.10/pytest.xml [took 0s]
artifacts/Test results for Linux-3.11 all-but-dask/pytest.xml [took 0s]
artifacts/Test results for Linux-3.12/pytest.xml [took 0s]
artifacts/Test results for Windows-3.10/pytest.xml [took 0s]
artifacts/Test results for Windows-3.12/pytest.xml [took 0s]
artifacts/Test results for macOS-3.10/pytest.xml [took 0s]
artifacts/Test results for macOS-3.12/pytest.xml [took 0s]
Raw output
AttributeError: 'NoneType' object has no attribute 'sync'
self = <xarray.tests.test_backends.TestZarrKVStoreV3 object at 0x7fab2498cd90>
def test_roundtrip_test_data(self) -> None:
expected = create_test_data()
> with self.roundtrip(expected) as actual:
#x1B[1m#x1B[31m/home/runner/work/xarray/xarray/xarray/tests/test_backends.py#x1B[0m:415:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
#x1B[1m#x1B[31m/home/runner/micromamba/envs/xarray-tests/lib/python3.10/contextlib.py#x1B[0m:135: in __enter__
return next(self.gen)
#x1B[1m#x1B[31m/home/runner/work/xarray/xarray/xarray/tests/test_backends.py#x1B[0m:2238: in roundtrip
self.save(data, store_target, **save_kwargs)
#x1B[1m#x1B[31m/home/runner/work/xarray/xarray/xarray/tests/test_backends.py#x1B[0m:2220: in save
return dataset.to_zarr(store=store_target, **kwargs, **self.version_kwargs)
#x1B[1m#x1B[31m/home/runner/work/xarray/xarray/xarray/core/dataset.py#x1B[0m:2566: in to_zarr
return to_zarr( # type: ignore[call-overload,misc]
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
dataset = <xarray.Dataset> Size: 2kB
Dimensions: (dim2: 9, dim3: 10, time: 20, dim1: 8)
Coordinates:
* dim2 (dim2) float6... 1.155 0.3591 -0.2959 ... -0.3348 -0.4515
var3 (dim3, dim1) float64 640B -0.6651 1.141 -0.3281 ... -1.787 1.225
store = <KVStoreV3:
{'zarr.json': b'{\n "extensions": [],\n "metadata_encoding": "https://purl.org/zarr/spec/protocol/c...87\xb3N\x05\xd4?\x1d\x82\xea\x85z~\xef\xbf\xc9\xdf\xb5<\xd4l\xd5\xbf\xe1\xab\x1dK\xa6\xe5\xdc\xbf"}
at 0x7fab0a72dfc0>
chunk_store = None, mode = 'w-', synchronizer = None, group = None
encoding = {}
def to_zarr(
dataset: Dataset,
store: MutableMapping | str | os.PathLike[str] | None = None,
chunk_store: MutableMapping | str | os.PathLike | None = None,
mode: ZarrWriteModes | None = None,
synchronizer=None,
group: str | None = None,
encoding: Mapping | None = None,
*,
compute: bool = True,
consolidated: bool | None = None,
append_dim: Hashable | None = None,
region: Mapping[str, slice | Literal["auto"]] | Literal["auto"] | None = None,
safe_chunks: bool = True,
storage_options: dict[str, str] | None = None,
zarr_version: int | None = None,
write_empty_chunks: bool | None = None,
chunkmanager_store_kwargs: dict[str, Any] | None = None,
writer: ArrayWriter | None = None,
) -> backends.ZarrStore | Delayed:
"""This function creates an appropriate datastore for writing a dataset to
a zarr ztore
See `Dataset.to_zarr` for full API docs.
"""
# Load empty arrays to avoid bug saving zero length dimensions (Issue #5741)
for v in dataset.variables.values():
if v.size == 0:
v.load()
# expand str and path-like arguments
store = _normalize_path(store)
chunk_store = _normalize_path(chunk_store)
if storage_options is None:
mapper = store
chunk_mapper = chunk_store
else:
from fsspec import get_mapper
if not isinstance(store, str):
raise ValueError(
f"store must be a string to use storage_options. Got {type(store)}"
)
mapper = get_mapper(store, **storage_options)
if chunk_store is not None:
chunk_mapper = get_mapper(chunk_store, **storage_options)
else:
chunk_mapper = chunk_store
if encoding is None:
encoding = {}
if mode is None:
if append_dim is not None:
mode = "a"
elif region is not None:
mode = "r+"
else:
mode = "w-"
if mode not in ["a", "a-"] and append_dim is not None:
raise ValueError("cannot set append_dim unless mode='a' or mode=None")
if mode not in ["a", "a-", "r+"] and region is not None:
raise ValueError(
"cannot set region unless mode='a', mode='a-', mode='r+' or mode=None"
)
if mode not in ["w", "w-", "a", "a-", "r+"]:
raise ValueError(
"The only supported options for mode are 'w', "
f"'w-', 'a', 'a-', and 'r+', but mode={mode!r}"
)
# validate Dataset keys, DataArray names
_validate_dataset_names(dataset)
if zarr_version is None:
# default to 2 if store doesn't specify it's version (e.g. a path)
zarr_version = int(getattr(store, "_store_version", 2))
if consolidated is None and zarr_version > 2:
consolidated = False
if mode == "r+":
already_consolidated = consolidated
consolidate_on_close = False
else:
already_consolidated = False
consolidate_on_close = consolidated or consolidated is None
zstore = backends.ZarrStore.open_group(
store=mapper,
mode=mode,
synchronizer=synchronizer,
group=group,
consolidated=already_consolidated,
consolidate_on_close=consolidate_on_close,
chunk_store=chunk_mapper,
append_dim=append_dim,
write_region=region,
safe_chunks=safe_chunks,
stacklevel=4, # for Dataset.to_zarr()
zarr_version=zarr_version,
write_empty=write_empty_chunks,
)
if region is not None:
zstore._validate_and_autodetect_region(dataset)
# can't modify indexes with region writes
dataset = dataset.drop_vars(dataset.indexes)
if append_dim is not None and append_dim in region:
raise ValueError(
f"cannot list the same dimension in both ``append_dim`` and "
f"``region`` with to_zarr(), got {append_dim} in both"
)
if encoding and mode in ["a", "a-", "r+"]:
existing_var_names = set(zstore.zarr_group.array_keys())
for var_name in existing_var_names:
if var_name in encoding:
raise ValueError(
f"variable {var_name!r} already exists, but encoding was provided"
)
# TODO: figure out how to properly handle unlimited_dims
dump_to_store(dataset, zstore, writer, encoding=encoding)
> writes = writer.sync(
compute=compute, chunkmanager_store_kwargs=chunkmanager_store_kwargs
)
#x1B[1m#x1B[31mE AttributeError: 'NoneType' object has no attribute 'sync'#x1B[0m
#x1B[1m#x1B[31m/home/runner/work/xarray/xarray/xarray/backends/api.py#x1B[0m:1783: AttributeError
Check warning on line 0 in xarray.tests.test_backends.TestDataArrayToZarr
github-actions / Test Results
8 out of 9 runs failed: test_dataarray_to_zarr_with_name[Dict] (xarray.tests.test_backends.TestDataArrayToZarr)
artifacts/Test results for Linux-3.10 min-all-deps/pytest.xml [took 0s]
artifacts/Test results for Linux-3.10/pytest.xml [took 0s]
artifacts/Test results for Linux-3.11 all-but-dask/pytest.xml [took 0s]
artifacts/Test results for Linux-3.12/pytest.xml [took 0s]
artifacts/Test results for Windows-3.10/pytest.xml [took 0s]
artifacts/Test results for Windows-3.12/pytest.xml [took 0s]
artifacts/Test results for macOS-3.10/pytest.xml [took 0s]
artifacts/Test results for macOS-3.12/pytest.xml [took 0s]
Raw output
AttributeError: 'NoneType' object has no attribute 'sync'
self = <xarray.tests.test_backends.TestDataArrayToZarr object at 0x7f3ff49c71c0>
tmp_store = {'.zattrs': b'{}', '.zgroup': b'{\n "zarr_format": 2\n}', '.zmetadata': b'{\n "metadata": {\n ".zattrs": ...n "filters": null,\n "order": "C",\n "shape": [\n 3,\n 4\n ],\n "zarr_format": 2\n}', ...}
def test_dataarray_to_zarr_with_name(self, tmp_store) -> None:
original_da = DataArray(np.arange(12).reshape((3, 4)), name="test")
> original_da.to_zarr(tmp_store)
#x1B[1m#x1B[31m/home/runner/work/xarray/xarray/xarray/tests/test_backends.py#x1B[0m:5231:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
#x1B[1m#x1B[31m/home/runner/work/xarray/xarray/xarray/core/dataarray.py#x1B[0m:4355: in to_zarr
return to_zarr( # type: ignore[call-overload,misc]
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
dataset = <xarray.Dataset> Size: 96B
Dimensions: (dim_0: 3, dim_1: 4)
Dimensions without coordinates: dim_0, dim_1
Data variables:
test (dim_0, dim_1) int64 96B 0 1 2 3 4 5 6 7 8 9 10 11
store = {'.zattrs': b'{}', '.zgroup': b'{\n "zarr_format": 2\n}', '.zmetadata': b'{\n "metadata": {\n ".zattrs": ...n "filters": null,\n "order": "C",\n "shape": [\n 3,\n 4\n ],\n "zarr_format": 2\n}', ...}
chunk_store = None, mode = 'w-', synchronizer = None, group = None
encoding = {}
def to_zarr(
dataset: Dataset,
store: MutableMapping | str | os.PathLike[str] | None = None,
chunk_store: MutableMapping | str | os.PathLike | None = None,
mode: ZarrWriteModes | None = None,
synchronizer=None,
group: str | None = None,
encoding: Mapping | None = None,
*,
compute: bool = True,
consolidated: bool | None = None,
append_dim: Hashable | None = None,
region: Mapping[str, slice | Literal["auto"]] | Literal["auto"] | None = None,
safe_chunks: bool = True,
storage_options: dict[str, str] | None = None,
zarr_version: int | None = None,
write_empty_chunks: bool | None = None,
chunkmanager_store_kwargs: dict[str, Any] | None = None,
writer: ArrayWriter | None = None,
) -> backends.ZarrStore | Delayed:
"""This function creates an appropriate datastore for writing a dataset to
a zarr ztore
See `Dataset.to_zarr` for full API docs.
"""
# Load empty arrays to avoid bug saving zero length dimensions (Issue #5741)
for v in dataset.variables.values():
if v.size == 0:
v.load()
# expand str and path-like arguments
store = _normalize_path(store)
chunk_store = _normalize_path(chunk_store)
if storage_options is None:
mapper = store
chunk_mapper = chunk_store
else:
from fsspec import get_mapper
if not isinstance(store, str):
raise ValueError(
f"store must be a string to use storage_options. Got {type(store)}"
)
mapper = get_mapper(store, **storage_options)
if chunk_store is not None:
chunk_mapper = get_mapper(chunk_store, **storage_options)
else:
chunk_mapper = chunk_store
if encoding is None:
encoding = {}
if mode is None:
if append_dim is not None:
mode = "a"
elif region is not None:
mode = "r+"
else:
mode = "w-"
if mode not in ["a", "a-"] and append_dim is not None:
raise ValueError("cannot set append_dim unless mode='a' or mode=None")
if mode not in ["a", "a-", "r+"] and region is not None:
raise ValueError(
"cannot set region unless mode='a', mode='a-', mode='r+' or mode=None"
)
if mode not in ["w", "w-", "a", "a-", "r+"]:
raise ValueError(
"The only supported options for mode are 'w', "
f"'w-', 'a', 'a-', and 'r+', but mode={mode!r}"
)
# validate Dataset keys, DataArray names
_validate_dataset_names(dataset)
if zarr_version is None:
# default to 2 if store doesn't specify it's version (e.g. a path)
zarr_version = int(getattr(store, "_store_version", 2))
if consolidated is None and zarr_version > 2:
consolidated = False
if mode == "r+":
already_consolidated = consolidated
consolidate_on_close = False
else:
already_consolidated = False
consolidate_on_close = consolidated or consolidated is None
zstore = backends.ZarrStore.open_group(
store=mapper,
mode=mode,
synchronizer=synchronizer,
group=group,
consolidated=already_consolidated,
consolidate_on_close=consolidate_on_close,
chunk_store=chunk_mapper,
append_dim=append_dim,
write_region=region,
safe_chunks=safe_chunks,
stacklevel=4, # for Dataset.to_zarr()
zarr_version=zarr_version,
write_empty=write_empty_chunks,
)
if region is not None:
zstore._validate_and_autodetect_region(dataset)
# can't modify indexes with region writes
dataset = dataset.drop_vars(dataset.indexes)
if append_dim is not None and append_dim in region:
raise ValueError(
f"cannot list the same dimension in both ``append_dim`` and "
f"``region`` with to_zarr(), got {append_dim} in both"
)
if encoding and mode in ["a", "a-", "r+"]:
existing_var_names = set(zstore.zarr_group.array_keys())
for var_name in existing_var_names:
if var_name in encoding:
raise ValueError(
f"variable {var_name!r} already exists, but encoding was provided"
)
# TODO: figure out how to properly handle unlimited_dims
dump_to_store(dataset, zstore, writer, encoding=encoding)
> writes = writer.sync(
compute=compute, chunkmanager_store_kwargs=chunkmanager_store_kwargs
)
#x1B[1m#x1B[31mE AttributeError: 'NoneType' object has no attribute 'sync'#x1B[0m
#x1B[1m#x1B[31m/home/runner/work/xarray/xarray/xarray/backends/api.py#x1B[0m:1783: AttributeError
Check warning on line 0 in xarray.tests.test_backends.TestDataArrayToZarr
github-actions / Test Results
8 out of 9 runs failed: test_dataarray_to_zarr_coord_name_clash[tmp_path] (xarray.tests.test_backends.TestDataArrayToZarr)
artifacts/Test results for Linux-3.10 min-all-deps/pytest.xml [took 0s]
artifacts/Test results for Linux-3.10/pytest.xml [took 0s]
artifacts/Test results for Linux-3.11 all-but-dask/pytest.xml [took 0s]
artifacts/Test results for Linux-3.12/pytest.xml [took 0s]
artifacts/Test results for Windows-3.10/pytest.xml [took 0s]
artifacts/Test results for Windows-3.12/pytest.xml [took 0s]
artifacts/Test results for macOS-3.10/pytest.xml [took 0s]
artifacts/Test results for macOS-3.12/pytest.xml [took 0s]
Raw output
AttributeError: 'NoneType' object has no attribute 'sync'
self = <xarray.tests.test_backends.TestDataArrayToZarr object at 0x7f3ff49c74c0>
tmp_store = PosixPath('/tmp/pytest-of-runner/pytest-0/popen-gw2/test_dataarray_to_zarr_coord_n0')
def test_dataarray_to_zarr_coord_name_clash(self, tmp_store) -> None:
original_da = DataArray(
np.arange(12).reshape((3, 4)), dims=["x", "y"], name="x"
)
> original_da.to_zarr(tmp_store)
#x1B[1m#x1B[31m/home/runner/work/xarray/xarray/xarray/tests/test_backends.py#x1B[0m:5241:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
#x1B[1m#x1B[31m/home/runner/work/xarray/xarray/xarray/core/dataarray.py#x1B[0m:4355: in to_zarr
return to_zarr( # type: ignore[call-overload,misc]
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
dataset = <xarray.Dataset> Size: 96B
Dimensions: (x: 3, y: 4)
Dimensions without coordinates: x, y
Data v...__xarray_dataarray_variable__ (x, y) int64 96B 0 1 2 3 4 5 6 7 8 9 10 11
Attributes:
__xarray_dataarray_name__: x
store = '/tmp/pytest-of-runner/pytest-0/popen-gw2/test_dataarray_to_zarr_coord_n0'
chunk_store = None, mode = 'w-', synchronizer = None, group = None
encoding = {}
def to_zarr(
dataset: Dataset,
store: MutableMapping | str | os.PathLike[str] | None = None,
chunk_store: MutableMapping | str | os.PathLike | None = None,
mode: ZarrWriteModes | None = None,
synchronizer=None,
group: str | None = None,
encoding: Mapping | None = None,
*,
compute: bool = True,
consolidated: bool | None = None,
append_dim: Hashable | None = None,
region: Mapping[str, slice | Literal["auto"]] | Literal["auto"] | None = None,
safe_chunks: bool = True,
storage_options: dict[str, str] | None = None,
zarr_version: int | None = None,
write_empty_chunks: bool | None = None,
chunkmanager_store_kwargs: dict[str, Any] | None = None,
writer: ArrayWriter | None = None,
) -> backends.ZarrStore | Delayed:
"""This function creates an appropriate datastore for writing a dataset to
a zarr ztore
See `Dataset.to_zarr` for full API docs.
"""
# Load empty arrays to avoid bug saving zero length dimensions (Issue #5741)
for v in dataset.variables.values():
if v.size == 0:
v.load()
# expand str and path-like arguments
store = _normalize_path(store)
chunk_store = _normalize_path(chunk_store)
if storage_options is None:
mapper = store
chunk_mapper = chunk_store
else:
from fsspec import get_mapper
if not isinstance(store, str):
raise ValueError(
f"store must be a string to use storage_options. Got {type(store)}"
)
mapper = get_mapper(store, **storage_options)
if chunk_store is not None:
chunk_mapper = get_mapper(chunk_store, **storage_options)
else:
chunk_mapper = chunk_store
if encoding is None:
encoding = {}
if mode is None:
if append_dim is not None:
mode = "a"
elif region is not None:
mode = "r+"
else:
mode = "w-"
if mode not in ["a", "a-"] and append_dim is not None:
raise ValueError("cannot set append_dim unless mode='a' or mode=None")
if mode not in ["a", "a-", "r+"] and region is not None:
raise ValueError(
"cannot set region unless mode='a', mode='a-', mode='r+' or mode=None"
)
if mode not in ["w", "w-", "a", "a-", "r+"]:
raise ValueError(
"The only supported options for mode are 'w', "
f"'w-', 'a', 'a-', and 'r+', but mode={mode!r}"
)
# validate Dataset keys, DataArray names
_validate_dataset_names(dataset)
if zarr_version is None:
# default to 2 if store doesn't specify it's version (e.g. a path)
zarr_version = int(getattr(store, "_store_version", 2))
if consolidated is None and zarr_version > 2:
consolidated = False
if mode == "r+":
already_consolidated = consolidated
consolidate_on_close = False
else:
already_consolidated = False
consolidate_on_close = consolidated or consolidated is None
zstore = backends.ZarrStore.open_group(
store=mapper,
mode=mode,
synchronizer=synchronizer,
group=group,
consolidated=already_consolidated,
consolidate_on_close=consolidate_on_close,
chunk_store=chunk_mapper,
append_dim=append_dim,
write_region=region,
safe_chunks=safe_chunks,
stacklevel=4, # for Dataset.to_zarr()
zarr_version=zarr_version,
write_empty=write_empty_chunks,
)
if region is not None:
zstore._validate_and_autodetect_region(dataset)
# can't modify indexes with region writes
dataset = dataset.drop_vars(dataset.indexes)
if append_dim is not None and append_dim in region:
raise ValueError(
f"cannot list the same dimension in both ``append_dim`` and "
f"``region`` with to_zarr(), got {append_dim} in both"
)
if encoding and mode in ["a", "a-", "r+"]:
existing_var_names = set(zstore.zarr_group.array_keys())
for var_name in existing_var_names:
if var_name in encoding:
raise ValueError(
f"variable {var_name!r} already exists, but encoding was provided"
)
# TODO: figure out how to properly handle unlimited_dims
dump_to_store(dataset, zstore, writer, encoding=encoding)
> writes = writer.sync(
compute=compute, chunkmanager_store_kwargs=chunkmanager_store_kwargs
)
#x1B[1m#x1B[31mE AttributeError: 'NoneType' object has no attribute 'sync'#x1B[0m
#x1B[1m#x1B[31m/home/runner/work/xarray/xarray/xarray/backends/api.py#x1B[0m:1783: AttributeError
Check warning on line 0 in xarray.tests.test_backends.TestZarrKVStoreV3
github-actions / Test Results
8 out of 9 runs failed: test_load (xarray.tests.test_backends.TestZarrKVStoreV3)
artifacts/Test results for Linux-3.10 min-all-deps/pytest.xml [took 0s]
artifacts/Test results for Linux-3.10/pytest.xml [took 0s]
artifacts/Test results for Linux-3.11 all-but-dask/pytest.xml [took 0s]
artifacts/Test results for Linux-3.12/pytest.xml [took 0s]
artifacts/Test results for Windows-3.10/pytest.xml [took 0s]
artifacts/Test results for Windows-3.12/pytest.xml [took 0s]
artifacts/Test results for macOS-3.10/pytest.xml [took 0s]
artifacts/Test results for macOS-3.12/pytest.xml [took 0s]
Raw output
AttributeError: 'NoneType' object has no attribute 'sync'
self = <xarray.tests.test_backends.TestZarrKVStoreV3 object at 0x7fab2498d000>
def test_load(self) -> None:
expected = create_test_data()
@contextlib.contextmanager
def assert_loads(vars=None):
if vars is None:
vars = expected
with self.roundtrip(expected) as actual:
for k, v in actual.variables.items():
# IndexVariables are eagerly loaded into memory
assert v._in_memory == (k in actual.dims)
yield actual
for k, v in actual.variables.items():
if k in vars:
assert v._in_memory
assert_identical(expected, actual)
with pytest.raises(AssertionError):
# make sure the contextmanager works!
> with assert_loads() as ds:
#x1B[1m#x1B[31m/home/runner/work/xarray/xarray/xarray/tests/test_backends.py#x1B[0m:438:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
#x1B[1m#x1B[31m/home/runner/micromamba/envs/xarray-tests/lib/python3.10/contextlib.py#x1B[0m:135: in __enter__
return next(self.gen)
#x1B[1m#x1B[31m/home/runner/work/xarray/xarray/xarray/tests/test_backends.py#x1B[0m:426: in assert_loads
with self.roundtrip(expected) as actual:
#x1B[1m#x1B[31m/home/runner/micromamba/envs/xarray-tests/lib/python3.10/contextlib.py#x1B[0m:135: in __enter__
return next(self.gen)
#x1B[1m#x1B[31m/home/runner/work/xarray/xarray/xarray/tests/test_backends.py#x1B[0m:2238: in roundtrip
self.save(data, store_target, **save_kwargs)
#x1B[1m#x1B[31m/home/runner/work/xarray/xarray/xarray/tests/test_backends.py#x1B[0m:2220: in save
return dataset.to_zarr(store=store_target, **kwargs, **self.version_kwargs)
#x1B[1m#x1B[31m/home/runner/work/xarray/xarray/xarray/core/dataset.py#x1B[0m:2566: in to_zarr
return to_zarr( # type: ignore[call-overload,misc]
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
dataset = <xarray.Dataset> Size: 2kB
Dimensions: (dim2: 9, dim3: 10, time: 20, dim1: 8)
Coordinates:
* dim2 (dim2) float6...B 0.1435 1.298 -0.622 ... 0.9467 0.5783
var3 (dim3, dim1) float64 640B -0.503 -0.4269 -0.01691 ... -0.03325 2.0
store = <KVStoreV3:
{'zarr.json': b'{\n "extensions": [],\n "metadata_encoding": "https://purl.org/zarr/spec/protocol/c...6\xbf\xfb\xdfe\x9eL\xa3\xce?P\x9fo\xa9\xd0|\xf8\xbfA\x04\xa6L\x94K\xee?\xc71\xc5\xd2\xd6\x81\xe2?'}
at 0x7fab12d727a0>
chunk_store = None, mode = 'w-', synchronizer = None, group = None
encoding = {}
def to_zarr(
dataset: Dataset,
store: MutableMapping | str | os.PathLike[str] | None = None,
chunk_store: MutableMapping | str | os.PathLike | None = None,
mode: ZarrWriteModes | None = None,
synchronizer=None,
group: str | None = None,
encoding: Mapping | None = None,
*,
compute: bool = True,
consolidated: bool | None = None,
append_dim: Hashable | None = None,
region: Mapping[str, slice | Literal["auto"]] | Literal["auto"] | None = None,
safe_chunks: bool = True,
storage_options: dict[str, str] | None = None,
zarr_version: int | None = None,
write_empty_chunks: bool | None = None,
chunkmanager_store_kwargs: dict[str, Any] | None = None,
writer: ArrayWriter | None = None,
) -> backends.ZarrStore | Delayed:
"""This function creates an appropriate datastore for writing a dataset to
a zarr ztore
See `Dataset.to_zarr` for full API docs.
"""
# Load empty arrays to avoid bug saving zero length dimensions (Issue #5741)
for v in dataset.variables.values():
if v.size == 0:
v.load()
# expand str and path-like arguments
store = _normalize_path(store)
chunk_store = _normalize_path(chunk_store)
if storage_options is None:
mapper = store
chunk_mapper = chunk_store
else:
from fsspec import get_mapper
if not isinstance(store, str):
raise ValueError(
f"store must be a string to use storage_options. Got {type(store)}"
)
mapper = get_mapper(store, **storage_options)
if chunk_store is not None:
chunk_mapper = get_mapper(chunk_store, **storage_options)
else:
chunk_mapper = chunk_store
if encoding is None:
encoding = {}
if mode is None:
if append_dim is not None:
mode = "a"
elif region is not None:
mode = "r+"
else:
mode = "w-"
if mode not in ["a", "a-"] and append_dim is not None:
raise ValueError("cannot set append_dim unless mode='a' or mode=None")
if mode not in ["a", "a-", "r+"] and region is not None:
raise ValueError(
"cannot set region unless mode='a', mode='a-', mode='r+' or mode=None"
)
if mode not in ["w", "w-", "a", "a-", "r+"]:
raise ValueError(
"The only supported options for mode are 'w', "
f"'w-', 'a', 'a-', and 'r+', but mode={mode!r}"
)
# validate Dataset keys, DataArray names
_validate_dataset_names(dataset)
if zarr_version is None:
# default to 2 if store doesn't specify it's version (e.g. a path)
zarr_version = int(getattr(store, "_store_version", 2))
if consolidated is None and zarr_version > 2:
consolidated = False
if mode == "r+":
already_consolidated = consolidated
consolidate_on_close = False
else:
already_consolidated = False
consolidate_on_close = consolidated or consolidated is None
zstore = backends.ZarrStore.open_group(
store=mapper,
mode=mode,
synchronizer=synchronizer,
group=group,
consolidated=already_consolidated,
consolidate_on_close=consolidate_on_close,
chunk_store=chunk_mapper,
append_dim=append_dim,
write_region=region,
safe_chunks=safe_chunks,
stacklevel=4, # for Dataset.to_zarr()
zarr_version=zarr_version,
write_empty=write_empty_chunks,
)
if region is not None:
zstore._validate_and_autodetect_region(dataset)
# can't modify indexes with region writes
dataset = dataset.drop_vars(dataset.indexes)
if append_dim is not None and append_dim in region:
raise ValueError(
f"cannot list the same dimension in both ``append_dim`` and "
f"``region`` with to_zarr(), got {append_dim} in both"
)
if encoding and mode in ["a", "a-", "r+"]:
existing_var_names = set(zstore.zarr_group.array_keys())
for var_name in existing_var_names:
if var_name in encoding:
raise ValueError(
f"variable {var_name!r} already exists, but encoding was provided"
)
# TODO: figure out how to properly handle unlimited_dims
dump_to_store(dataset, zstore, writer, encoding=encoding)
> writes = writer.sync(
compute=compute, chunkmanager_store_kwargs=chunkmanager_store_kwargs
)
#x1B[1m#x1B[31mE AttributeError: 'NoneType' object has no attribute 'sync'#x1B[0m
#x1B[1m#x1B[31m/home/runner/work/xarray/xarray/xarray/backends/api.py#x1B[0m:1783: AttributeError
Check warning on line 0 in xarray.tests.test_backends.TestDataArrayToZarr
github-actions / Test Results
8 out of 9 runs failed: test_dataarray_to_zarr_coord_name_clash[ZipStore] (xarray.tests.test_backends.TestDataArrayToZarr)
artifacts/Test results for Linux-3.10 min-all-deps/pytest.xml [took 0s]
artifacts/Test results for Linux-3.10/pytest.xml [took 0s]
artifacts/Test results for Linux-3.11 all-but-dask/pytest.xml [took 0s]
artifacts/Test results for Linux-3.12/pytest.xml [took 0s]
artifacts/Test results for Windows-3.10/pytest.xml [took 0s]
artifacts/Test results for Windows-3.12/pytest.xml [took 0s]
artifacts/Test results for macOS-3.10/pytest.xml [took 0s]
artifacts/Test results for macOS-3.12/pytest.xml [took 0s]
Raw output
AttributeError: 'NoneType' object has no attribute 'sync'
self = <xarray.tests.test_backends.TestDataArrayToZarr object at 0x7f3ff49c74f0>
tmp_store = <zarr.storage.ZipStore object at 0x7f3fd970b0a0>
def test_dataarray_to_zarr_coord_name_clash(self, tmp_store) -> None:
original_da = DataArray(
np.arange(12).reshape((3, 4)), dims=["x", "y"], name="x"
)
> original_da.to_zarr(tmp_store)
#x1B[1m#x1B[31m/home/runner/work/xarray/xarray/xarray/tests/test_backends.py#x1B[0m:5241:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
#x1B[1m#x1B[31m/home/runner/work/xarray/xarray/xarray/core/dataarray.py#x1B[0m:4355: in to_zarr
return to_zarr( # type: ignore[call-overload,misc]
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
dataset = <xarray.Dataset> Size: 96B
Dimensions: (x: 3, y: 4)
Dimensions without coordinates: x, y
Data v...__xarray_dataarray_variable__ (x, y) int64 96B 0 1 2 3 4 5 6 7 8 9 10 11
Attributes:
__xarray_dataarray_name__: x
store = <zarr.storage.ZipStore object at 0x7f3fd970b0a0>, chunk_store = None
mode = 'w-', synchronizer = None, group = None, encoding = {}
def to_zarr(
dataset: Dataset,
store: MutableMapping | str | os.PathLike[str] | None = None,
chunk_store: MutableMapping | str | os.PathLike | None = None,
mode: ZarrWriteModes | None = None,
synchronizer=None,
group: str | None = None,
encoding: Mapping | None = None,
*,
compute: bool = True,
consolidated: bool | None = None,
append_dim: Hashable | None = None,
region: Mapping[str, slice | Literal["auto"]] | Literal["auto"] | None = None,
safe_chunks: bool = True,
storage_options: dict[str, str] | None = None,
zarr_version: int | None = None,
write_empty_chunks: bool | None = None,
chunkmanager_store_kwargs: dict[str, Any] | None = None,
writer: ArrayWriter | None = None,
) -> backends.ZarrStore | Delayed:
"""This function creates an appropriate datastore for writing a dataset to
a zarr ztore
See `Dataset.to_zarr` for full API docs.
"""
# Load empty arrays to avoid bug saving zero length dimensions (Issue #5741)
for v in dataset.variables.values():
if v.size == 0:
v.load()
# expand str and path-like arguments
store = _normalize_path(store)
chunk_store = _normalize_path(chunk_store)
if storage_options is None:
mapper = store
chunk_mapper = chunk_store
else:
from fsspec import get_mapper
if not isinstance(store, str):
raise ValueError(
f"store must be a string to use storage_options. Got {type(store)}"
)
mapper = get_mapper(store, **storage_options)
if chunk_store is not None:
chunk_mapper = get_mapper(chunk_store, **storage_options)
else:
chunk_mapper = chunk_store
if encoding is None:
encoding = {}
if mode is None:
if append_dim is not None:
mode = "a"
elif region is not None:
mode = "r+"
else:
mode = "w-"
if mode not in ["a", "a-"] and append_dim is not None:
raise ValueError("cannot set append_dim unless mode='a' or mode=None")
if mode not in ["a", "a-", "r+"] and region is not None:
raise ValueError(
"cannot set region unless mode='a', mode='a-', mode='r+' or mode=None"
)
if mode not in ["w", "w-", "a", "a-", "r+"]:
raise ValueError(
"The only supported options for mode are 'w', "
f"'w-', 'a', 'a-', and 'r+', but mode={mode!r}"
)
# validate Dataset keys, DataArray names
_validate_dataset_names(dataset)
if zarr_version is None:
# default to 2 if store doesn't specify it's version (e.g. a path)
zarr_version = int(getattr(store, "_store_version", 2))
if consolidated is None and zarr_version > 2:
consolidated = False
if mode == "r+":
already_consolidated = consolidated
consolidate_on_close = False
else:
already_consolidated = False
consolidate_on_close = consolidated or consolidated is None
zstore = backends.ZarrStore.open_group(
store=mapper,
mode=mode,
synchronizer=synchronizer,
group=group,
consolidated=already_consolidated,
consolidate_on_close=consolidate_on_close,
chunk_store=chunk_mapper,
append_dim=append_dim,
write_region=region,
safe_chunks=safe_chunks,
stacklevel=4, # for Dataset.to_zarr()
zarr_version=zarr_version,
write_empty=write_empty_chunks,
)
if region is not None:
zstore._validate_and_autodetect_region(dataset)
# can't modify indexes with region writes
dataset = dataset.drop_vars(dataset.indexes)
if append_dim is not None and append_dim in region:
raise ValueError(
f"cannot list the same dimension in both ``append_dim`` and "
f"``region`` with to_zarr(), got {append_dim} in both"
)
if encoding and mode in ["a", "a-", "r+"]:
existing_var_names = set(zstore.zarr_group.array_keys())
for var_name in existing_var_names:
if var_name in encoding:
raise ValueError(
f"variable {var_name!r} already exists, but encoding was provided"
)
# TODO: figure out how to properly handle unlimited_dims
dump_to_store(dataset, zstore, writer, encoding=encoding)
> writes = writer.sync(
compute=compute, chunkmanager_store_kwargs=chunkmanager_store_kwargs
)
#x1B[1m#x1B[31mE AttributeError: 'NoneType' object has no attribute 'sync'#x1B[0m
#x1B[1m#x1B[31m/home/runner/work/xarray/xarray/xarray/backends/api.py#x1B[0m:1783: AttributeError
Check warning on line 0 in xarray.tests.test_backends.TestDataArrayToZarr
github-actions / Test Results
8 out of 9 runs failed: test_dataarray_to_zarr_coord_name_clash[Dict] (xarray.tests.test_backends.TestDataArrayToZarr)
artifacts/Test results for Linux-3.10 min-all-deps/pytest.xml [took 0s]
artifacts/Test results for Linux-3.10/pytest.xml [took 0s]
artifacts/Test results for Linux-3.11 all-but-dask/pytest.xml [took 0s]
artifacts/Test results for Linux-3.12/pytest.xml [took 0s]
artifacts/Test results for Windows-3.10/pytest.xml [took 0s]
artifacts/Test results for Windows-3.12/pytest.xml [took 0s]
artifacts/Test results for macOS-3.10/pytest.xml [took 0s]
artifacts/Test results for macOS-3.12/pytest.xml [took 0s]
Raw output
AttributeError: 'NoneType' object has no attribute 'sync'
self = <xarray.tests.test_backends.TestDataArrayToZarr object at 0x7f3ff49c76d0>
tmp_store = {'.zattrs': b'{\n "__xarray_dataarray_name__": "x"\n}', '.zgroup': b'{\n "zarr_format": 2\n}', '.zmetadata': b'{...n "filters": null,\n "order": "C",\n "shape": [\n 3,\n 4\n ],\n "zarr_format": 2\n}', ...}
def test_dataarray_to_zarr_coord_name_clash(self, tmp_store) -> None:
original_da = DataArray(
np.arange(12).reshape((3, 4)), dims=["x", "y"], name="x"
)
> original_da.to_zarr(tmp_store)
#x1B[1m#x1B[31m/home/runner/work/xarray/xarray/xarray/tests/test_backends.py#x1B[0m:5241:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
#x1B[1m#x1B[31m/home/runner/work/xarray/xarray/xarray/core/dataarray.py#x1B[0m:4355: in to_zarr
return to_zarr( # type: ignore[call-overload,misc]
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
dataset = <xarray.Dataset> Size: 96B
Dimensions: (x: 3, y: 4)
Dimensions without coordinates: x, y
Data v...__xarray_dataarray_variable__ (x, y) int64 96B 0 1 2 3 4 5 6 7 8 9 10 11
Attributes:
__xarray_dataarray_name__: x
store = {'.zattrs': b'{\n "__xarray_dataarray_name__": "x"\n}', '.zgroup': b'{\n "zarr_format": 2\n}', '.zmetadata': b'{...n "filters": null,\n "order": "C",\n "shape": [\n 3,\n 4\n ],\n "zarr_format": 2\n}', ...}
chunk_store = None, mode = 'w-', synchronizer = None, group = None
encoding = {}
def to_zarr(
dataset: Dataset,
store: MutableMapping | str | os.PathLike[str] | None = None,
chunk_store: MutableMapping | str | os.PathLike | None = None,
mode: ZarrWriteModes | None = None,
synchronizer=None,
group: str | None = None,
encoding: Mapping | None = None,
*,
compute: bool = True,
consolidated: bool | None = None,
append_dim: Hashable | None = None,
region: Mapping[str, slice | Literal["auto"]] | Literal["auto"] | None = None,
safe_chunks: bool = True,
storage_options: dict[str, str] | None = None,
zarr_version: int | None = None,
write_empty_chunks: bool | None = None,
chunkmanager_store_kwargs: dict[str, Any] | None = None,
writer: ArrayWriter | None = None,
) -> backends.ZarrStore | Delayed:
"""This function creates an appropriate datastore for writing a dataset to
a zarr ztore
See `Dataset.to_zarr` for full API docs.
"""
# Load empty arrays to avoid bug saving zero length dimensions (Issue #5741)
for v in dataset.variables.values():
if v.size == 0:
v.load()
# expand str and path-like arguments
store = _normalize_path(store)
chunk_store = _normalize_path(chunk_store)
if storage_options is None:
mapper = store
chunk_mapper = chunk_store
else:
from fsspec import get_mapper
if not isinstance(store, str):
raise ValueError(
f"store must be a string to use storage_options. Got {type(store)}"
)
mapper = get_mapper(store, **storage_options)
if chunk_store is not None:
chunk_mapper = get_mapper(chunk_store, **storage_options)
else:
chunk_mapper = chunk_store
if encoding is None:
encoding = {}
if mode is None:
if append_dim is not None:
mode = "a"
elif region is not None:
mode = "r+"
else:
mode = "w-"
if mode not in ["a", "a-"] and append_dim is not None:
raise ValueError("cannot set append_dim unless mode='a' or mode=None")
if mode not in ["a", "a-", "r+"] and region is not None:
raise ValueError(
"cannot set region unless mode='a', mode='a-', mode='r+' or mode=None"
)
if mode not in ["w", "w-", "a", "a-", "r+"]:
raise ValueError(
"The only supported options for mode are 'w', "
f"'w-', 'a', 'a-', and 'r+', but mode={mode!r}"
)
# validate Dataset keys, DataArray names
_validate_dataset_names(dataset)
if zarr_version is None:
# default to 2 if store doesn't specify it's version (e.g. a path)
zarr_version = int(getattr(store, "_store_version", 2))
if consolidated is None and zarr_version > 2:
consolidated = False
if mode == "r+":
already_consolidated = consolidated
consolidate_on_close = False
else:
already_consolidated = False
consolidate_on_close = consolidated or consolidated is None
zstore = backends.ZarrStore.open_group(
store=mapper,
mode=mode,
synchronizer=synchronizer,
group=group,
consolidated=already_consolidated,
consolidate_on_close=consolidate_on_close,
chunk_store=chunk_mapper,
append_dim=append_dim,
write_region=region,
safe_chunks=safe_chunks,
stacklevel=4, # for Dataset.to_zarr()
zarr_version=zarr_version,
write_empty=write_empty_chunks,
)
if region is not None:
zstore._validate_and_autodetect_region(dataset)
# can't modify indexes with region writes
dataset = dataset.drop_vars(dataset.indexes)
if append_dim is not None and append_dim in region:
raise ValueError(
f"cannot list the same dimension in both ``append_dim`` and "
f"``region`` with to_zarr(), got {append_dim} in both"
)
if encoding and mode in ["a", "a-", "r+"]:
existing_var_names = set(zstore.zarr_group.array_keys())
for var_name in existing_var_names:
if var_name in encoding:
raise ValueError(
f"variable {var_name!r} already exists, but encoding was provided"
)
# TODO: figure out how to properly handle unlimited_dims
dump_to_store(dataset, zstore, writer, encoding=encoding)
> writes = writer.sync(
compute=compute, chunkmanager_store_kwargs=chunkmanager_store_kwargs
)
#x1B[1m#x1B[31mE AttributeError: 'NoneType' object has no attribute 'sync'#x1B[0m
#x1B[1m#x1B[31m/home/runner/work/xarray/xarray/xarray/backends/api.py#x1B[0m:1783: AttributeError
Check warning on line 0 in xarray.tests.test_backends.TestZarrKVStoreV3
github-actions / Test Results
8 out of 9 runs failed: test_dataset_compute (xarray.tests.test_backends.TestZarrKVStoreV3)
artifacts/Test results for Linux-3.10 min-all-deps/pytest.xml [took 0s]
artifacts/Test results for Linux-3.10/pytest.xml [took 0s]
artifacts/Test results for Linux-3.11 all-but-dask/pytest.xml [took 0s]
artifacts/Test results for Linux-3.12/pytest.xml [took 0s]
artifacts/Test results for Windows-3.10/pytest.xml [took 0s]
artifacts/Test results for Windows-3.12/pytest.xml [took 0s]
artifacts/Test results for macOS-3.10/pytest.xml [took 0s]
artifacts/Test results for macOS-3.12/pytest.xml [took 0s]
Raw output
AttributeError: 'NoneType' object has no attribute 'sync'
self = <xarray.tests.test_backends.TestZarrKVStoreV3 object at 0x7fab2498d270>
def test_dataset_compute(self) -> None:
expected = create_test_data()
> with self.roundtrip(expected) as actual:
#x1B[1m#x1B[31m/home/runner/work/xarray/xarray/xarray/tests/test_backends.py#x1B[0m:455:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
#x1B[1m#x1B[31m/home/runner/micromamba/envs/xarray-tests/lib/python3.10/contextlib.py#x1B[0m:135: in __enter__
return next(self.gen)
#x1B[1m#x1B[31m/home/runner/work/xarray/xarray/xarray/tests/test_backends.py#x1B[0m:2238: in roundtrip
self.save(data, store_target, **save_kwargs)
#x1B[1m#x1B[31m/home/runner/work/xarray/xarray/xarray/tests/test_backends.py#x1B[0m:2220: in save
return dataset.to_zarr(store=store_target, **kwargs, **self.version_kwargs)
#x1B[1m#x1B[31m/home/runner/work/xarray/xarray/xarray/core/dataset.py#x1B[0m:2566: in to_zarr
return to_zarr( # type: ignore[call-overload,misc]
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
dataset = <xarray.Dataset> Size: 2kB
Dimensions: (dim2: 9, dim3: 10, time: 20, dim1: 8)
Coordinates:
* dim2 (dim2) float6...6B 1.906 -1.21 0.7257 ... 0.4253 0.5991
var3 (dim3, dim1) float64 640B 0.7302 0.4752 -0.08174 ... 0.7798 0.8699
store = <KVStoreV3:
{'zarr.json': b'{\n "extensions": [],\n "metadata_encoding": "https://purl.org/zarr/spec/protocol/c...\xc1\xf7\xdb?8\x12\x8e\xe29\x0e\xf6\xbfg\x04\x14_\xac\x9c\xc7?\xa2w\\\xd6\x817\xdb?uc\xd3x*,\xe3?"}
at 0x7fab27667280>
chunk_store = None, mode = 'w-', synchronizer = None, group = None
encoding = {}
def to_zarr(
dataset: Dataset,
store: MutableMapping | str | os.PathLike[str] | None = None,
chunk_store: MutableMapping | str | os.PathLike | None = None,
mode: ZarrWriteModes | None = None,
synchronizer=None,
group: str | None = None,
encoding: Mapping | None = None,
*,
compute: bool = True,
consolidated: bool | None = None,
append_dim: Hashable | None = None,
region: Mapping[str, slice | Literal["auto"]] | Literal["auto"] | None = None,
safe_chunks: bool = True,
storage_options: dict[str, str] | None = None,
zarr_version: int | None = None,
write_empty_chunks: bool | None = None,
chunkmanager_store_kwargs: dict[str, Any] | None = None,
writer: ArrayWriter | None = None,
) -> backends.ZarrStore | Delayed:
"""This function creates an appropriate datastore for writing a dataset to
a zarr ztore
See `Dataset.to_zarr` for full API docs.
"""
# Load empty arrays to avoid bug saving zero length dimensions (Issue #5741)
for v in dataset.variables.values():
if v.size == 0:
v.load()
# expand str and path-like arguments
store = _normalize_path(store)
chunk_store = _normalize_path(chunk_store)
if storage_options is None:
mapper = store
chunk_mapper = chunk_store
else:
from fsspec import get_mapper
if not isinstance(store, str):
raise ValueError(
f"store must be a string to use storage_options. Got {type(store)}"
)
mapper = get_mapper(store, **storage_options)
if chunk_store is not None:
chunk_mapper = get_mapper(chunk_store, **storage_options)
else:
chunk_mapper = chunk_store
if encoding is None:
encoding = {}
if mode is None:
if append_dim is not None:
mode = "a"
elif region is not None:
mode = "r+"
else:
mode = "w-"
if mode not in ["a", "a-"] and append_dim is not None:
raise ValueError("cannot set append_dim unless mode='a' or mode=None")
if mode not in ["a", "a-", "r+"] and region is not None:
raise ValueError(
"cannot set region unless mode='a', mode='a-', mode='r+' or mode=None"
)
if mode not in ["w", "w-", "a", "a-", "r+"]:
raise ValueError(
"The only supported options for mode are 'w', "
f"'w-', 'a', 'a-', and 'r+', but mode={mode!r}"
)
# validate Dataset keys, DataArray names
_validate_dataset_names(dataset)
if zarr_version is None:
# default to 2 if store doesn't specify it's version (e.g. a path)
zarr_version = int(getattr(store, "_store_version", 2))
if consolidated is None and zarr_version > 2:
consolidated = False
if mode == "r+":
already_consolidated = consolidated
consolidate_on_close = False
else:
already_consolidated = False
consolidate_on_close = consolidated or consolidated is None
zstore = backends.ZarrStore.open_group(
store=mapper,
mode=mode,
synchronizer=synchronizer,
group=group,
consolidated=already_consolidated,
consolidate_on_close=consolidate_on_close,
chunk_store=chunk_mapper,
append_dim=append_dim,
write_region=region,
safe_chunks=safe_chunks,
stacklevel=4, # for Dataset.to_zarr()
zarr_version=zarr_version,
write_empty=write_empty_chunks,
)
if region is not None:
zstore._validate_and_autodetect_region(dataset)
# can't modify indexes with region writes
dataset = dataset.drop_vars(dataset.indexes)
if append_dim is not None and append_dim in region:
raise ValueError(
f"cannot list the same dimension in both ``append_dim`` and "
f"``region`` with to_zarr(), got {append_dim} in both"
)
if encoding and mode in ["a", "a-", "r+"]:
existing_var_names = set(zstore.zarr_group.array_keys())
for var_name in existing_var_names:
if var_name in encoding:
raise ValueError(
f"variable {var_name!r} already exists, but encoding was provided"
)
# TODO: figure out how to properly handle unlimited_dims
dump_to_store(dataset, zstore, writer, encoding=encoding)
> writes = writer.sync(
compute=compute, chunkmanager_store_kwargs=chunkmanager_store_kwargs
)
#x1B[1m#x1B[31mE AttributeError: 'NoneType' object has no attribute 'sync'#x1B[0m
#x1B[1m#x1B[31m/home/runner/work/xarray/xarray/xarray/backends/api.py#x1B[0m:1783: AttributeError
Check warning on line 0 in xarray.tests.test_backends.TestDataArrayToZarr
github-actions / Test Results
8 out of 9 runs failed: test_open_dataarray_options[tmp_path] (xarray.tests.test_backends.TestDataArrayToZarr)
artifacts/Test results for Linux-3.10 min-all-deps/pytest.xml [took 0s]
artifacts/Test results for Linux-3.10/pytest.xml [took 0s]
artifacts/Test results for Linux-3.11 all-but-dask/pytest.xml [took 0s]
artifacts/Test results for Linux-3.12/pytest.xml [took 0s]
artifacts/Test results for Windows-3.10/pytest.xml [took 0s]
artifacts/Test results for Windows-3.12/pytest.xml [took 0s]
artifacts/Test results for macOS-3.10/pytest.xml [took 0s]
artifacts/Test results for macOS-3.12/pytest.xml [took 0s]
Raw output
AttributeError: 'NoneType' object has no attribute 'sync'
self = <xarray.tests.test_backends.TestDataArrayToZarr object at 0x7f3ff49c79d0>
tmp_store = PosixPath('/tmp/pytest-of-runner/pytest-0/popen-gw2/test_open_dataarray_options_tm0')
def test_open_dataarray_options(self, tmp_store) -> None:
data = DataArray(np.arange(5), coords={"y": ("x", range(5))}, dims=["x"])
> data.to_zarr(tmp_store)
#x1B[1m#x1B[31m/home/runner/work/xarray/xarray/xarray/tests/test_backends.py#x1B[0m:5249:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
#x1B[1m#x1B[31m/home/runner/work/xarray/xarray/xarray/core/dataarray.py#x1B[0m:4355: in to_zarr
return to_zarr( # type: ignore[call-overload,misc]
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
dataset = <xarray.Dataset> Size: 80B
Dimensions: (x: 5)
Coordinates:
y (... 0 1 2 3 4
Dimensions without coordinates: x
Data variables:
__xarray_dataarray_variable__ (x) int64 40B 0 1 2 3 4
store = '/tmp/pytest-of-runner/pytest-0/popen-gw2/test_open_dataarray_options_tm0'
chunk_store = None, mode = 'w-', synchronizer = None, group = None
encoding = {}
def to_zarr(
dataset: Dataset,
store: MutableMapping | str | os.PathLike[str] | None = None,
chunk_store: MutableMapping | str | os.PathLike | None = None,
mode: ZarrWriteModes | None = None,
synchronizer=None,
group: str | None = None,
encoding: Mapping | None = None,
*,
compute: bool = True,
consolidated: bool | None = None,
append_dim: Hashable | None = None,
region: Mapping[str, slice | Literal["auto"]] | Literal["auto"] | None = None,
safe_chunks: bool = True,
storage_options: dict[str, str] | None = None,
zarr_version: int | None = None,
write_empty_chunks: bool | None = None,
chunkmanager_store_kwargs: dict[str, Any] | None = None,
writer: ArrayWriter | None = None,
) -> backends.ZarrStore | Delayed:
"""This function creates an appropriate datastore for writing a dataset to
a zarr ztore
See `Dataset.to_zarr` for full API docs.
"""
# Load empty arrays to avoid bug saving zero length dimensions (Issue #5741)
for v in dataset.variables.values():
if v.size == 0:
v.load()
# expand str and path-like arguments
store = _normalize_path(store)
chunk_store = _normalize_path(chunk_store)
if storage_options is None:
mapper = store
chunk_mapper = chunk_store
else:
from fsspec import get_mapper
if not isinstance(store, str):
raise ValueError(
f"store must be a string to use storage_options. Got {type(store)}"
)
mapper = get_mapper(store, **storage_options)
if chunk_store is not None:
chunk_mapper = get_mapper(chunk_store, **storage_options)
else:
chunk_mapper = chunk_store
if encoding is None:
encoding = {}
if mode is None:
if append_dim is not None:
mode = "a"
elif region is not None:
mode = "r+"
else:
mode = "w-"
if mode not in ["a", "a-"] and append_dim is not None:
raise ValueError("cannot set append_dim unless mode='a' or mode=None")
if mode not in ["a", "a-", "r+"] and region is not None:
raise ValueError(
"cannot set region unless mode='a', mode='a-', mode='r+' or mode=None"
)
if mode not in ["w", "w-", "a", "a-", "r+"]:
raise ValueError(
"The only supported options for mode are 'w', "
f"'w-', 'a', 'a-', and 'r+', but mode={mode!r}"
)
# validate Dataset keys, DataArray names
_validate_dataset_names(dataset)
if zarr_version is None:
# default to 2 if store doesn't specify it's version (e.g. a path)
zarr_version = int(getattr(store, "_store_version", 2))
if consolidated is None and zarr_version > 2:
consolidated = False
if mode == "r+":
already_consolidated = consolidated
consolidate_on_close = False
else:
already_consolidated = False
consolidate_on_close = consolidated or consolidated is None
zstore = backends.ZarrStore.open_group(
store=mapper,
mode=mode,
synchronizer=synchronizer,
group=group,
consolidated=already_consolidated,
consolidate_on_close=consolidate_on_close,
chunk_store=chunk_mapper,
append_dim=append_dim,
write_region=region,
safe_chunks=safe_chunks,
stacklevel=4, # for Dataset.to_zarr()
zarr_version=zarr_version,
write_empty=write_empty_chunks,
)
if region is not None:
zstore._validate_and_autodetect_region(dataset)
# can't modify indexes with region writes
dataset = dataset.drop_vars(dataset.indexes)
if append_dim is not None and append_dim in region:
raise ValueError(
f"cannot list the same dimension in both ``append_dim`` and "
f"``region`` with to_zarr(), got {append_dim} in both"
)
if encoding and mode in ["a", "a-", "r+"]:
existing_var_names = set(zstore.zarr_group.array_keys())
for var_name in existing_var_names:
if var_name in encoding:
raise ValueError(
f"variable {var_name!r} already exists, but encoding was provided"
)
# TODO: figure out how to properly handle unlimited_dims
dump_to_store(dataset, zstore, writer, encoding=encoding)
> writes = writer.sync(
compute=compute, chunkmanager_store_kwargs=chunkmanager_store_kwargs
)
#x1B[1m#x1B[31mE AttributeError: 'NoneType' object has no attribute 'sync'#x1B[0m
#x1B[1m#x1B[31m/home/runner/work/xarray/xarray/xarray/backends/api.py#x1B[0m:1783: AttributeError
Check warning on line 0 in xarray.tests.test_backends.TestDataArrayToZarr
github-actions / Test Results
8 out of 9 runs failed: test_open_dataarray_options[ZipStore] (xarray.tests.test_backends.TestDataArrayToZarr)
artifacts/Test results for Linux-3.10 min-all-deps/pytest.xml [took 0s]
artifacts/Test results for Linux-3.10/pytest.xml [took 0s]
artifacts/Test results for Linux-3.11 all-but-dask/pytest.xml [took 0s]
artifacts/Test results for Linux-3.12/pytest.xml [took 0s]
artifacts/Test results for Windows-3.10/pytest.xml [took 0s]
artifacts/Test results for Windows-3.12/pytest.xml [took 0s]
artifacts/Test results for macOS-3.10/pytest.xml [took 0s]
artifacts/Test results for macOS-3.12/pytest.xml [took 0s]
Raw output
AttributeError: 'NoneType' object has no attribute 'sync'
self = <xarray.tests.test_backends.TestDataArrayToZarr object at 0x7f3ff49c7a00>
tmp_store = <zarr.storage.ZipStore object at 0x7f3ff73e14b0>
def test_open_dataarray_options(self, tmp_store) -> None:
data = DataArray(np.arange(5), coords={"y": ("x", range(5))}, dims=["x"])
> data.to_zarr(tmp_store)
#x1B[1m#x1B[31m/home/runner/work/xarray/xarray/xarray/tests/test_backends.py#x1B[0m:5249:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
#x1B[1m#x1B[31m/home/runner/work/xarray/xarray/xarray/core/dataarray.py#x1B[0m:4355: in to_zarr
return to_zarr( # type: ignore[call-overload,misc]
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
dataset = <xarray.Dataset> Size: 80B
Dimensions: (x: 5)
Coordinates:
y (... 0 1 2 3 4
Dimensions without coordinates: x
Data variables:
__xarray_dataarray_variable__ (x) int64 40B 0 1 2 3 4
store = <zarr.storage.ZipStore object at 0x7f3ff73e14b0>, chunk_store = None
mode = 'w-', synchronizer = None, group = None, encoding = {}
def to_zarr(
dataset: Dataset,
store: MutableMapping | str | os.PathLike[str] | None = None,
chunk_store: MutableMapping | str | os.PathLike | None = None,
mode: ZarrWriteModes | None = None,
synchronizer=None,
group: str | None = None,
encoding: Mapping | None = None,
*,
compute: bool = True,
consolidated: bool | None = None,
append_dim: Hashable | None = None,
region: Mapping[str, slice | Literal["auto"]] | Literal["auto"] | None = None,
safe_chunks: bool = True,
storage_options: dict[str, str] | None = None,
zarr_version: int | None = None,
write_empty_chunks: bool | None = None,
chunkmanager_store_kwargs: dict[str, Any] | None = None,
writer: ArrayWriter | None = None,
) -> backends.ZarrStore | Delayed:
"""This function creates an appropriate datastore for writing a dataset to
a zarr ztore
See `Dataset.to_zarr` for full API docs.
"""
# Load empty arrays to avoid bug saving zero length dimensions (Issue #5741)
for v in dataset.variables.values():
if v.size == 0:
v.load()
# expand str and path-like arguments
store = _normalize_path(store)
chunk_store = _normalize_path(chunk_store)
if storage_options is None:
mapper = store
chunk_mapper = chunk_store
else:
from fsspec import get_mapper
if not isinstance(store, str):
raise ValueError(
f"store must be a string to use storage_options. Got {type(store)}"
)
mapper = get_mapper(store, **storage_options)
if chunk_store is not None:
chunk_mapper = get_mapper(chunk_store, **storage_options)
else:
chunk_mapper = chunk_store
if encoding is None:
encoding = {}
if mode is None:
if append_dim is not None:
mode = "a"
elif region is not None:
mode = "r+"
else:
mode = "w-"
if mode not in ["a", "a-"] and append_dim is not None:
raise ValueError("cannot set append_dim unless mode='a' or mode=None")
if mode not in ["a", "a-", "r+"] and region is not None:
raise ValueError(
"cannot set region unless mode='a', mode='a-', mode='r+' or mode=None"
)
if mode not in ["w", "w-", "a", "a-", "r+"]:
raise ValueError(
"The only supported options for mode are 'w', "
f"'w-', 'a', 'a-', and 'r+', but mode={mode!r}"
)
# validate Dataset keys, DataArray names
_validate_dataset_names(dataset)
if zarr_version is None:
# default to 2 if store doesn't specify it's version (e.g. a path)
zarr_version = int(getattr(store, "_store_version", 2))
if consolidated is None and zarr_version > 2:
consolidated = False
if mode == "r+":
already_consolidated = consolidated
consolidate_on_close = False
else:
already_consolidated = False
consolidate_on_close = consolidated or consolidated is None
zstore = backends.ZarrStore.open_group(
store=mapper,
mode=mode,
synchronizer=synchronizer,
group=group,
consolidated=already_consolidated,
consolidate_on_close=consolidate_on_close,
chunk_store=chunk_mapper,
append_dim=append_dim,
write_region=region,
safe_chunks=safe_chunks,
stacklevel=4, # for Dataset.to_zarr()
zarr_version=zarr_version,
write_empty=write_empty_chunks,
)
if region is not None:
zstore._validate_and_autodetect_region(dataset)
# can't modify indexes with region writes
dataset = dataset.drop_vars(dataset.indexes)
if append_dim is not None and append_dim in region:
raise ValueError(
f"cannot list the same dimension in both ``append_dim`` and "
f"``region`` with to_zarr(), got {append_dim} in both"
)
if encoding and mode in ["a", "a-", "r+"]:
existing_var_names = set(zstore.zarr_group.array_keys())
for var_name in existing_var_names:
if var_name in encoding:
raise ValueError(
f"variable {var_name!r} already exists, but encoding was provided"
)
# TODO: figure out how to properly handle unlimited_dims
dump_to_store(dataset, zstore, writer, encoding=encoding)
> writes = writer.sync(
compute=compute, chunkmanager_store_kwargs=chunkmanager_store_kwargs
)
#x1B[1m#x1B[31mE AttributeError: 'NoneType' object has no attribute 'sync'#x1B[0m
#x1B[1m#x1B[31m/home/runner/work/xarray/xarray/xarray/backends/api.py#x1B[0m:1783: AttributeError