def consolidate_metadata(store: StoreLike, metadata_key=".zmetadata"):
    """
    Consolidate all metadata for groups and arrays within the given store
    into a single resource and put it under the given key.

    This produces a single object in the backend store, containing all the
    metadata read from all the zarr-related keys that can be found. After
    metadata have been consolidated, use :func:`open_consolidated` to open
    the root group in optimised, read-only mode, using the consolidated
    metadata to reduce the number of read operations on the backend store.

    Note, that if the metadata in the store is changed after this
    consolidation, then the metadata read by :func:`open_consolidated`
    would be incorrect unless this function is called again.

    .. note:: This is an experimental feature.

    Parameters
    ----------
    store : MutableMapping or string
        Store or path to directory in file system or name of zip file.
    metadata_key : str
        Key to put the consolidated metadata under.

    Returns
    -------
    g : :class:`zarr.hierarchy.Group`
        Group instance, opened with the new consolidated metadata.

    See Also
    --------
    open_consolidated

    """
    store = normalize_store_arg(store, mode="w")

    def is_zarr_key(key):
        return (key.endswith('.zarray') or key.endswith('.zgroup') or
                key.endswith('.zattrs'))

    out = {
        'zarr_consolidated_format': 1,
        'metadata': {
            key: json_loads(store[key])
            for key in store if is_zarr_key(key)
        }
    }
    store[metadata_key] = json_dumps(out)
    return open_consolidated(store, metadata_key=metadata_key) class MetadataError(Exception):
    pass def group(store=None, overwrite=False, chunk_store=None,
          cache_attrs=True, synchronizer=None, path=None):
    """Create a group.

    Parameters
    ----------
    store : MutableMapping or string, optional
        Store or path to directory in file system.
    overwrite : bool, optional
        If True, delete any pre-existing data in `store` at `path` before
        creating the group.
    chunk_store : MutableMapping, optional
        Separate storage for chunks. If not provided, `store` will be used
        for storage of both chunks and metadata.
    cache_attrs : bool, optional
        If True (default), user attributes will be cached for attribute read
        operations. If False, user attributes are reloaded from the store prior
        to all attribute read operations.
    synchronizer : object, optional
        Array synchronizer.
    path : string, optional
        Group path within store.

    Returns
    -------
    g : zarr.hierarchy.Group

    Examples
    --------
    Create a group in memory::

        >>> import zarr
        >>> g = zarr.group()
        >>> g
        <zarr.hierarchy.Group '/'>

    Create a group with a different store::

        >>> store = zarr.DirectoryStore('data/example.zarr')
        >>> g = zarr.group(store=store, overwrite=True)
        >>> g
        <zarr.hierarchy.Group '/'>

    """

    # handle polymorphic store arg
    store = _normalize_store_arg(store)
    path = normalize_storage_path(path)

    # require group
    if overwrite or not contains_group(store):
        init_group(store, overwrite=overwrite, chunk_store=chunk_store,
                   path=path)

    return Group(store, read_only=False, chunk_store=chunk_store,
                 cache_attrs=cache_attrs, synchronizer=synchronizer, path=path) ZARR_FORMAT = 2 @classmethod
def decode_array_metadata(cls, s: Union[MappingType, str]) -> MappingType[str, Any]:
    meta = cls.parse_metadata(s)

    # check metadata format
    zarr_format = meta.get("zarr_format", None)
    if zarr_format != cls.ZARR_FORMAT:
        raise MetadataError("unsupported zarr format: %s" % zarr_format)

    # extract array metadata fields
    try:
        dtype = cls.decode_dtype(meta["dtype"])
        if dtype.hasobject:
            import numcodecs
            object_codec = numcodecs.get_codec(meta['filters'][0])
        else:
            object_codec = None

        dimension_separator = meta.get("dimension_separator", None)
        fill_value = cls.decode_fill_value(meta['fill_value'], dtype, object_codec)
        meta = dict(
            zarr_format=meta["zarr_format"],
            shape=tuple(meta["shape"]),
            chunks=tuple(meta["chunks"]),
            dtype=dtype,
            compressor=meta["compressor"],
            fill_value=fill_value,
            order=meta["order"],
            filters=meta["filters"],
        )
        if dimension_separator:
            meta['dimension_separator'] = dimension_separator
    except Exception as e:
        raise MetadataError("error decoding metadata") from e
    else:
        return meta class N5Store(NestedDirectoryStore):
    """Storage class using directories and files on a standard file system,
    following the N5 format (https://github.com/saalfeldlab/n5).

    Parameters
    ----------
    path : string
        Location of directory to use as the root of the storage hierarchy.
    normalize_keys : bool, optional
        If True, all store keys will be normalized to use lower case characters
        (e.g. 'foo' and 'FOO' will be treated as equivalent). This can be
        useful to avoid potential discrepancies between case-sensitive and
        case-insensitive file system. Default value is False.

    Examples
    --------
    Store a single array::

        >>> import zarr
        >>> store = zarr.N5Store('data/array.n5')
        >>> z = zarr.zeros((10, 10), chunks=(5, 5), store=store, overwrite=True)
        >>> z[...] = 42

    Store a group::

        >>> store = zarr.N5Store('data/group.n5')
        >>> root = zarr.group(store=store, overwrite=True)
        >>> foo = root.create_group('foo')
        >>> bar = foo.zeros('bar', shape=(10, 10), chunks=(5, 5))
        >>> bar[...] = 42

    Notes
    -----

    This is an experimental feature.

    Safe to write in multiple threads or processes.

    """

    def __getitem__(self, key):

        if key.endswith(zarr_group_meta_key):

            key = key.replace(zarr_group_meta_key, n5_attrs_key)
            value = group_metadata_to_zarr(self._load_n5_attrs(key))

            return json_dumps(value)

        elif key.endswith(zarr_array_meta_key):

            key = key.replace(zarr_array_meta_key, n5_attrs_key)
            value = array_metadata_to_zarr(self._load_n5_attrs(key))

            return json_dumps(value)

        elif key.endswith(zarr_attrs_key):

            key = key.replace(zarr_attrs_key, n5_attrs_key)
            value = attrs_to_zarr(self._load_n5_attrs(key))

            if len(value) == 0:
                raise KeyError(key)
            else:
                return json_dumps(value)

        elif is_chunk_key(key):

            key = invert_chunk_coords(key)

        return super().__getitem__(key)

    def __setitem__(self, key, value):

        if key.endswith(zarr_group_meta_key):

            key = key.replace(zarr_group_meta_key, n5_attrs_key)

            n5_attrs = self._load_n5_attrs(key)
            n5_attrs.update(**group_metadata_to_n5(json_loads(value)))

            value = json_dumps(n5_attrs)

        elif key.endswith(zarr_array_meta_key):

            key = key.replace(zarr_array_meta_key, n5_attrs_key)

            n5_attrs = self._load_n5_attrs(key)
            n5_attrs.update(**array_metadata_to_n5(json_loads(value)))

            value = json_dumps(n5_attrs)

        elif key.endswith(zarr_attrs_key):

            key = key.replace(zarr_attrs_key, n5_attrs_key)

            n5_attrs = self._load_n5_attrs(key)
            zarr_attrs = json_loads(value)

            for k in n5_keywords:
                if k in zarr_attrs.keys():
                    warnings.warn("attribute %s is a reserved N5 keyword" % k, UserWarning)

            # replace previous user attributes
            for k in list(n5_attrs.keys()):
                if k not in n5_keywords:
                    del n5_attrs[k]

            # add new user attributes
            n5_attrs.update(**zarr_attrs)

            value = json_dumps(n5_attrs)

        elif is_chunk_key(key):

            key = invert_chunk_coords(key)

        super().__setitem__(key, value)

    def __delitem__(self, key):

        if key.endswith(zarr_group_meta_key):  # pragma: no cover
            key = key.replace(zarr_group_meta_key, n5_attrs_key)
        elif key.endswith(zarr_array_meta_key):  # pragma: no cover
            key = key.replace(zarr_array_meta_key, n5_attrs_key)
        elif key.endswith(zarr_attrs_key):  # pragma: no cover
            key = key.replace(zarr_attrs_key, n5_attrs_key)
        elif is_chunk_key(key):
            key = invert_chunk_coords(key)

        super().__delitem__(key)

    def __contains__(self, key):

        if key.endswith(zarr_group_meta_key):

            key = key.replace(zarr_group_meta_key, n5_attrs_key)
            if key not in self:
                return False
            # group if not a dataset (attributes do not contain 'dimensions')
            return 'dimensions' not in self._load_n5_attrs(key)

        elif key.endswith(zarr_array_meta_key):

            key = key.replace(zarr_array_meta_key, n5_attrs_key)
            # array if attributes contain 'dimensions'
            return 'dimensions' in self._load_n5_attrs(key)

        elif key.endswith(zarr_attrs_key):

            key = key.replace(zarr_attrs_key, n5_attrs_key)
            return self._contains_attrs(key)

        elif is_chunk_key(key):

            key = invert_chunk_coords(key)

        return super().__contains__(key)

    def __eq__(self, other):
        return (
            isinstance(other, N5Store) and
            self.path == other.path
        )

    def listdir(self, path=None):

        if path is not None:
            path = invert_chunk_coords(path)

        # We can't use NestedDirectoryStore's listdir, as it requires
        # array_meta_key to be present in array directories, which this store
        # doesn't provide.
        children = super().listdir(path=path)

        if self._is_array(path):

            # replace n5 attribute file with respective zarr attribute files
            children.remove(n5_attrs_key)
            children.append(zarr_array_meta_key)
            if self._contains_attrs(path):
                children.append(zarr_attrs_key)

            # special handling of directories containing an array to map
            # inverted nested chunk keys back to standard chunk keys
            new_children = []
            root_path = self.dir_path(path)
            for entry in children:
                entry_path = os.path.join(root_path, entry)
                if _prog_number.match(entry) and os.path.isdir(entry_path):
                    for dir_path, _, file_names in os.walk(entry_path):
                        for file_name in file_names:
                            file_path = os.path.join(dir_path, file_name)
                            rel_path = file_path.split(root_path + os.path.sep)[1]
                            new_child = rel_path.replace(os.path.sep, '.')
                            new_children.append(invert_chunk_coords(new_child))
                else:
                    new_children.append(entry)

            return sorted(new_children)

        elif self._is_group(path):

            # replace n5 attribute file with respective zarr attribute files
            children.remove(n5_attrs_key)
            children.append(zarr_group_meta_key)
            if self._contains_attrs(path):  # pragma: no cover
                children.append(zarr_attrs_key)

            return sorted(children)

        else:

            return children

    def _load_n5_attrs(self, path):
        try:
            s = super().__getitem__(path)
            return json_loads(s)
        except KeyError:
            return {}

    def _is_group(self, path):

        if path is None:
            attrs_key = n5_attrs_key
        else:
            attrs_key = os.path.join(path, n5_attrs_key)

        n5_attrs = self._load_n5_attrs(attrs_key)
        return len(n5_attrs) > 0 and 'dimensions' not in n5_attrs

    def _is_array(self, path):

        if path is None:
            attrs_key = n5_attrs_key
        else:
            attrs_key = os.path.join(path, n5_attrs_key)

        return 'dimensions' in self._load_n5_attrs(attrs_key)

    def _contains_attrs(self, path):

        if path is None:
            attrs_key = n5_attrs_key
        else:
            if not path.endswith(n5_attrs_key):
                attrs_key = os.path.join(path, n5_attrs_key)
            else:  # pragma: no cover
                attrs_key = path

        attrs = attrs_to_zarr(self._load_n5_attrs(attrs_key))
        return len(attrs) > 0 class N5FSStore(FSStore):
    """Implementation of the N5 format (https://github.com/saalfeldlab/n5)
    using `fsspec`, which allows storage on a variety of filesystems. Based
    on `zarr.N5Store`.
    Parameters
    ----------
    path : string
        Location of directory to use as the root of the storage hierarchy.
    normalize_keys : bool, optional
        If True, all store keys will be normalized to use lower case characters
        (e.g. 'foo' and 'FOO' will be treated as equivalent). This can be
        useful to avoid potential discrepancies between case-sensitive and
        case-insensitive file system. Default value is False.

    Examples
    --------
    Store a single array::

        >>> import zarr
        >>> store = zarr.N5FSStore('data/array.n5', auto_mkdir=True)
        >>> z = zarr.zeros((10, 10), chunks=(5, 5), store=store, overwrite=True)
        >>> z[...] = 42

    Store a group::

        >>> store = zarr.N5FSStore('data/group.n5', auto_mkdir=True)
        >>> root = zarr.group(store=store, overwrite=True)
        >>> foo = root.create_group('foo')
        >>> bar = foo.zeros('bar', shape=(10, 10), chunks=(5, 5))
        >>> bar[...] = 42

    Notes
    -----
    This is an experimental feature.
    Safe to write in multiple threads or processes.

    Be advised that the `_dimension_separator` property of this store
    (and arrays it creates) is ".", but chunks saved by this store will
    in fact be "/" separated, as proscribed by the N5 format.

    This is counter-intuitive (to say the least), but not arbitrary.
    Chunks in N5 format are stored with reversed dimension order
    relative to Zarr chunks: a chunk of a 3D Zarr array would be stored
    on a file system as `/0/1/2`, but in N5 the same chunk would be
    stored as `/2/1/0`. Therefore, stores targeting N5 must intercept
    chunk keys and flip the order of the dimensions before writing to
    storage, and this procedure requires chunk keys with "." separated
    dimensions, hence the Zarr arrays targeting N5 have the deceptive
    "." dimension separator.
    """
    _array_meta_key = 'attributes.json'
    _group_meta_key = 'attributes.json'
    _attrs_key = 'attributes.json'

    def __init__(self, *args, **kwargs):
        if 'dimension_separator' in kwargs:
            kwargs.pop('dimension_separator')
            warnings.warn('Keyword argument `dimension_separator` will be ignored')
        dimension_separator = "."
        super().__init__(*args, dimension_separator=dimension_separator, **kwargs)

    @staticmethod
    def _swap_separator(key):
        segments = list(key.split('/'))
        if segments:
            last_segment = segments[-1]
            if _prog_ckey.match(last_segment):
                coords = list(last_segment.split('.'))
                last_segment = '/'.join(coords[::-1])
                segments = segments[:-1] + [last_segment]
                key = '/'.join(segments)
        return key

    def _normalize_key(self, key):
        if is_chunk_key(key):
            key = invert_chunk_coords(key)

        key = normalize_storage_path(key).lstrip("/")
        if key:
            *bits, end = key.split("/")

            if end not in (self._array_meta_key, self._group_meta_key, self._attrs_key):
                end = end.replace(".", "/")
                key = "/".join(bits + [end])
        return key.lower() if self.normalize_keys else key

    def __getitem__(self, key):
        if key.endswith(zarr_group_meta_key):

            key = key.replace(zarr_group_meta_key, self._group_meta_key)
            value = group_metadata_to_zarr(self._load_n5_attrs(key))

            return json_dumps(value)

        elif key.endswith(zarr_array_meta_key):

            key = key.replace(zarr_array_meta_key, self._array_meta_key)
            value = array_metadata_to_zarr(self._load_n5_attrs(key))

            return json_dumps(value)

        elif key.endswith(zarr_attrs_key):

            key = key.replace(zarr_attrs_key, self._attrs_key)
            value = attrs_to_zarr(self._load_n5_attrs(key))

            if len(value) == 0:
                raise KeyError(key)
            else:
                return json_dumps(value)

        elif is_chunk_key(key):
            key = self._swap_separator(key)

        return super().__getitem__(key)

    def __setitem__(self, key, value):
        if key.endswith(zarr_group_meta_key):

            key = key.replace(zarr_group_meta_key, self._group_meta_key)

            n5_attrs = self._load_n5_attrs(key)
            n5_attrs.update(**group_metadata_to_n5(json_loads(value)))

            value = json_dumps(n5_attrs)

        elif key.endswith(zarr_array_meta_key):

            key = key.replace(zarr_array_meta_key, self._array_meta_key)

            n5_attrs = self._load_n5_attrs(key)
            n5_attrs.update(**array_metadata_to_n5(json_loads(value)))

            value = json_dumps(n5_attrs)

        elif key.endswith(zarr_attrs_key):

            key = key.replace(zarr_attrs_key, self._attrs_key)

            n5_attrs = self._load_n5_attrs(key)
            zarr_attrs = json_loads(value)

            for k in n5_keywords:
                if k in zarr_attrs.keys():
                    warnings.warn("attribute %s is a reserved N5 keyword" % k, UserWarning)

            # replace previous user attributes
            for k in list(n5_attrs.keys()):
                if k not in n5_keywords:
                    del n5_attrs[k]

            # add new user attributes
            n5_attrs.update(**zarr_attrs)

            value = json_dumps(n5_attrs)

        elif is_chunk_key(key):
            key = self._swap_separator(key)

        super().__setitem__(key, value)

    def __delitem__(self, key):

        if key.endswith(zarr_group_meta_key):  # pragma: no cover
            key = key.replace(zarr_group_meta_key, self._group_meta_key)
        elif key.endswith(zarr_array_meta_key):  # pragma: no cover
            key = key.replace(zarr_array_meta_key, self._array_meta_key)
        elif key.endswith(zarr_attrs_key):  # pragma: no cover
            key = key.replace(zarr_attrs_key, self._attrs_key)
        elif is_chunk_key(key):
            key = self._swap_separator(key)

        super().__delitem__(key)

    def __contains__(self, key):
        if key.endswith(zarr_group_meta_key):

            key = key.replace(zarr_group_meta_key, self._group_meta_key)
            if key not in self:
                return False
            # group if not a dataset (attributes do not contain 'dimensions')
            return "dimensions" not in self._load_n5_attrs(key)

        elif key.endswith(zarr_array_meta_key):

            key = key.replace(zarr_array_meta_key, self._array_meta_key)
            # array if attributes contain 'dimensions'
            return "dimensions" in self._load_n5_attrs(key)

        elif key.endswith(zarr_attrs_key):

            key = key.replace(zarr_attrs_key, self._attrs_key)
            return self._contains_attrs(key)

        elif is_chunk_key(key):
            key = self._swap_separator(key)

        return super().__contains__(key)

    def __eq__(self, other):
        return isinstance(other, N5FSStore) and self.path == other.path

    def listdir(self, path=None):
        if path is not None:
            path = invert_chunk_coords(path)

        # We can't use NestedDirectoryStore's listdir, as it requires
        # array_meta_key to be present in array directories, which this store
        # doesn't provide.
        children = super().listdir(path=path)
        if self._is_array(path):

            # replace n5 attribute file with respective zarr attribute files
            children.remove(self._array_meta_key)
            children.append(zarr_array_meta_key)
            if self._contains_attrs(path):
                children.append(zarr_attrs_key)

            # special handling of directories containing an array to map
            # inverted nested chunk keys back to standard chunk keys
            new_children = []
            root_path = self.dir_path(path)
            for entry in children:
                entry_path = os.path.join(root_path, entry)
                if _prog_number.match(entry) and self.fs.isdir(entry_path):
                    for file_name in self.fs.find(entry_path):
                        file_path = os.path.join(root_path, file_name)
                        rel_path = file_path.split(root_path)[1]
                        new_child = rel_path.lstrip('/').replace('/', ".")
                        new_children.append(invert_chunk_coords(new_child))
                else:
                    new_children.append(entry)
            return sorted(new_children)

        elif self._is_group(path):

            # replace n5 attribute file with respective zarr attribute files
            children.remove(self._group_meta_key)
            children.append(zarr_group_meta_key)
            if self._contains_attrs(path):  # pragma: no cover
                children.append(zarr_attrs_key)
            return sorted(children)
        else:
            return children

    def _load_n5_attrs(self, path):
        try:
            s = super().__getitem__(path)
            return json_loads(s)
        except KeyError:
            return {}

    def _is_group(self, path):

        if path is None:
            attrs_key = self._attrs_key
        else:
            attrs_key = os.path.join(path, self._attrs_key)

        n5_attrs = self._load_n5_attrs(attrs_key)
        return len(n5_attrs) > 0 and "dimensions" not in n5_attrs

    def _is_array(self, path):

        if path is None:
            attrs_key = self._attrs_key
        else:
            attrs_key = os.path.join(path, self._attrs_key)

        return "dimensions" in self._load_n5_attrs(attrs_key)

    def _contains_attrs(self, path):

        if path is None:
            attrs_key = self._attrs_key
        else:
            if not path.endswith(self._attrs_key):
                attrs_key = os.path.join(path, self._attrs_key)
            else:  # pragma: no cover
                attrs_key = path

        attrs = attrs_to_zarr(self._load_n5_attrs(attrs_key))
        return len(attrs) > 0 def contains_array(store: StoreLike, path: Path = None) -> bool:
def contains_group(store: StoreLike, path: Path = None) -> bool:
def normalize_store_arg(store: Any, storage_options=None, mode="r") -> BaseStore:
def rmdir(store: StoreLike, path: Path = None):
def rename(store: BaseStore, src_path: Path, dst_path: Path):
def listdir(store: BaseStore, path: Path = None):
def getsize(store: BaseStore, path: Path = None) -> int:
def _require_parent_group(
    path: Optional[str],
    store: StoreLike,
    chunk_store: Optional[StoreLike],
    overwrite: bool,
):
def init_array(
    store: StoreLike,
    shape: Tuple[int, ...],
    chunks: Union[bool, int, Tuple[int, ...]] = True,
    dtype=None,
    compressor="default",
    fill_value=None,
    order: str = "C",
    overwrite: bool = False,
    path: Optional[Path] = None,
    chunk_store: Optional[StoreLike] = None,
    filters=None,
    object_codec=None,
    dimension_separator=None,
):
def _init_array_metadata(
    store: StoreLike,
    shape,
    chunks=None,
    dtype=None,
    compressor="default",
    fill_value=None,
    order="C",
    overwrite=False,
    path: Optional[str] = None,
    chunk_store: Optional[StoreLike] = None,
    filters=None,
    object_codec=None,
    dimension_separator=None,
):
def init_group(
    store: StoreLike,
    overwrite: bool = False,
    path: Path = None,
    chunk_store: StoreLike = None,
):
def _init_group_metadata(
    store: StoreLike,
    overwrite: Optional[bool] = False,
    path: Optional[str] = None,
    chunk_store: StoreLike = None,
):
def _dict_store_keys(d: Dict, prefix="", cls=dict):
    def __init__(self, mutablemapping):
    def __getitem__(self, key):
    def __setitem__(self, key, value):
    def __delitem__(self, key):
    def get(self, key, default=None):
    def values(self):
    def __iter__(self):
    def __len__(self):
    def __repr__(self):
    def __eq__(self, other):
    def __init__(self, root=None, cls=dict, dimension_separator=None):
    def __getstate__(self):
    def __setstate__(self, state):
    def _get_parent(self, item: str):
    def _require_parent(self, item):
    def __getitem__(self, item: str):
    def __setitem__(self, item: str, value):
    def __delitem__(self, item: str):
    def __contains__(self, item: str):  # type: ignore[override]
    def __eq__(self, other):
    def keys(self):
    def __iter__(self):
    def __len__(self) -> int:
    def listdir(self, path: Path = None) -> List[str]:
    def rename(self, src_path: Path, dst_path: Path):
    def rmdir(self, path: Path = None):
    def getsize(self, path: Path = None):
    def clear(self):
    def __init__(self, *args, **kwargs):
    def __init__(self, path, normalize_keys=False, dimension_separator=None):
    def _normalize_key(self, key):
    def _fromfile(fn):
    def _tofile(a, fn):
    def __getitem__(self, key):
    def __setitem__(self, key, value):
    def __delitem__(self, key):
    def __contains__(self, key):
    def __eq__(self, other):
    def keys(self):
    def _keys_fast(path, walker=os.walk):
    def __iter__(self):
    def __len__(self):
    def dir_path(self, path=None):
    def listdir(self, path=None):
    def _flat_listdir(self, path=None):
    def _nested_listdir(self, path=None):
    def rename(self, src_path, dst_path):
    def rmdir(self, path=None):
    def getsize(self, path=None):
    def clear(self):
def atexit_rmtree(path,
                  isdir=os.path.isdir,
                  rmtree=shutil.rmtree):  # pragma: no cover
def atexit_rmglob(path,
                  glob=glob.glob,
                  isdir=os.path.isdir,
                  isfile=os.path.isfile,
                  remove=os.remove,
                  rmtree=shutil.rmtree):  # pragma: no cover
    def __init__(self, url, normalize_keys=False, key_separator=None,
                 mode='w',
                 exceptions=(KeyError, PermissionError, IOError),
                 dimension_separator=None,
                 **storage_options):
    def _normalize_key(self, key):
    def getitems(self, keys, **kwargs):
    def __getitem__(self, key):
    def setitems(self, values):
    def __setitem__(self, key, value):
    def __delitem__(self, key):
    def delitems(self, keys):
    def __contains__(self, key):
    def __eq__(self, other):
    def keys(self):
    def __iter__(self):
    def __len__(self):
    def dir_path(self, path=None):
    def listdir(self, path=None):
    def rmdir(self, path=None):
    def getsize(self, path=None):
    def clear(self):
    def __init__(self, suffix='', prefix='zarr', dir=None, normalize_keys=False,
                 dimension_separator=None):
    def __init__(self, path, normalize_keys=False, dimension_separator="/"):
    def __eq__(self, other):
    def __init__(self, path, compression=zipfile.ZIP_STORED, allowZip64=True, mode='a',
                 dimension_separator=None):
    def __getstate__(self):
    def __setstate__(self, state):
    def close(self):
    def flush(self):
    def __enter__(self):
    def __exit__(self, *args):
    def __getitem__(self, key):
    def __setitem__(self, key, value):
    def __delitem__(self, key):
    def __eq__(self, other):
    def keylist(self):
    def keys(self):
    def __iter__(self):
    def __len__(self):
    def __contains__(self, key):
    def listdir(self, path=None):
    def getsize(self, path=None):
    def clear(self):
def migrate_1to2(store):
    def __init__(self, path, flag='c', mode=0o666, open=None, write_lock=True,
                 dimension_separator=None,
                 **open_kwargs):
    def __getstate__(self):
    def __setstate__(self, state):
    def close(self):
    def flush(self):
    def __enter__(self):
    def __exit__(self, *args):
    def __getitem__(self, key):
    def __setitem__(self, key, value):
    def __delitem__(self, key):
    def __eq__(self, other):
    def keys(self):
    def __iter__(self):
    def __len__(self):
    def __contains__(self, key):
    def __init__(self, path, buffers=True, dimension_separator=None, **kwargs):
    def __getstate__(self):
    def __setstate__(self, state):
    def close(self):
    def flush(self):
    def __enter__(self):
    def __exit__(self, *args):
    def __getitem__(self, key):
    def __setitem__(self, key, value):
    def __delitem__(self, key):
    def __contains__(self, key):
    def items(self):
    def keys(self):
    def values(self):
    def __iter__(self):
    def __len__(self):
    def __init__(self, store: StoreLike, max_size: int):
    def __getstate__(self):
    def __setstate__(self, state):
    def __len__(self):
    def __iter__(self):
    def __contains__(self, key):
    def clear(self):
    def keys(self):
    def _keys(self):
    def listdir(self, path: Path = None):
    def getsize(self, path=None) -> int:
    def _pop_value(self):
    def _accommodate_value(self, value_size):
    def _cache_value(self, key: Path, value):
    def invalidate(self):
    def invalidate_values(self):
    def invalidate_keys(self):
    def _invalidate_keys(self):
    def _invalidate_value(self, key):
    def __getitem__(self, key):
    def __setitem__(self, key, value):
    def __delitem__(self, key):
    def __init__(self, path, dimension_separator=None, **kwargs):
    def __getstate__(self):
    def __setstate__(self, state):
    def close(self):
    def __getitem__(self, key):
    def __setitem__(self, key, value):
    def __delitem__(self, key):
    def __contains__(self, key):
    def items(self):
    def keys(self):
    def values(self):
    def __iter__(self):
    def __len__(self):
    def update(self, *args, **kwargs):
    def listdir(self, path=None):
    def getsize(self, path=None):
    def rmdir(self, path=None):
    def clear(self):
    def __init__(self, database='mongodb_zarr', collection='zarr_collection',
                 dimension_separator=None, **kwargs):
    def __getitem__(self, key):
    def __setitem__(self, key, value):
    def __delitem__(self, key):
    def __iter__(self):
    def __len__(self):
    def __getstate__(self):
    def __setstate__(self, state):
    def close(self):
    def clear(self):
    def __init__(self, prefix='zarr', dimension_separator=None, **kwargs):
    def _key(self, key):
    def __getitem__(self, key):
    def __setitem__(self, key, value):
    def __delitem__(self, key):
    def keylist(self):
    def keys(self):
    def __iter__(self):
    def __len__(self):
    def __getstate__(self):
    def __setstate__(self, state):
    def clear(self):
    def __init__(self, store: StoreLike, metadata_key=".zmetadata"):
    def __getitem__(self, key):
    def __contains__(self, item):
    def __iter__(self):
    def __len__(self):
    def __delitem__(self, key):
    def __setitem__(self, key, value):
    def getsize(self, path):
    def listdir(self, path):
class KVStore(Store):
class MemoryStore(Store):
class DictStore(MemoryStore):
class DirectoryStore(Store):
class FSStore(Store):
class TempStore(DirectoryStore):
class NestedDirectoryStore(DirectoryStore):
class ZipStore(Store):
class DBMStore(Store):
class LMDBStore(Store):
class LRUStoreCache(Store):
class SQLiteStore(Store):
class MongoDBStore(Store):
class RedisStore(Store):
class ConsolidatedMetadataStore(Store): class FSStore(Store):
    """Wraps an fsspec.FSMap to give access to arbitrary filesystems

    Requires that ``fsspec`` is installed, as well as any additional
    requirements for the protocol chosen.

    Parameters
    ----------
    url : str
        The destination to map. Should include protocol and path,
        like "s3://bucket/root"
    normalize_keys : bool
    key_separator : str
        public API for accessing dimension_separator. Never `None`
        See dimension_separator for more information.
    mode : str
        "w" for writable, "r" for read-only
    exceptions : list of Exception subclasses
        When accessing data, any of these exceptions will be treated
        as a missing key
    dimension_separator : {'.', '/'}, optional
        Separator placed between the dimensions of a chunk.
    storage_options : passed to the fsspec implementation
    """
    _array_meta_key = array_meta_key
    _group_meta_key = group_meta_key
    _attrs_key = attrs_key

    def __init__(self, url, normalize_keys=False, key_separator=None,
                 mode='w',
                 exceptions=(KeyError, PermissionError, IOError),
                 dimension_separator=None,
                 **storage_options):
        import fsspec
        self.normalize_keys = normalize_keys

        protocol, _ = fsspec.core.split_protocol(url)
        # set auto_mkdir to True for local file system
        if protocol in (None, "file") and not storage_options.get("auto_mkdir"):
            storage_options["auto_mkdir"] = True

        self.map = fsspec.get_mapper(url, **storage_options)
        self.fs = self.map.fs  # for direct operations
        self.path = self.fs._strip_protocol(url)
        self.mode = mode
        self.exceptions = exceptions
        # For backwards compatibility. Guaranteed to be non-None
        if key_separator is not None:
            dimension_separator = key_separator

        self.key_separator = dimension_separator
        if self.key_separator is None:
            self.key_separator = "."

        # Pass attributes to array creation
        self._dimension_separator = dimension_separator
        if self.fs.exists(self.path) and not self.fs.isdir(self.path):
            raise FSPathExistNotDir(url)

    def _normalize_key(self, key):
        key = normalize_storage_path(key).lstrip('/')
        if key:
            *bits, end = key.split('/')

            if end not in (self._array_meta_key, self._group_meta_key, self._attrs_key):
                end = end.replace('.', self.key_separator)
                key = '/'.join(bits + [end])

        return key.lower() if self.normalize_keys else key

    def getitems(self, keys, **kwargs):
        keys_transformed = [self._normalize_key(key) for key in keys]
        results = self.map.getitems(keys_transformed, on_error="omit")
        # The function calling this method may not recognize the transformed keys
        # So we send the values returned by self.map.getitems back into the original key space.
        return {keys[keys_transformed.index(rk)]: rv for rk, rv in results.items()}

    def __getitem__(self, key):
        key = self._normalize_key(key)
        try:
            return self.map[key]
        except self.exceptions as e:
            raise KeyError(key) from e

    def setitems(self, values):
        if self.mode == 'r':
            raise ReadOnlyError()
        values = {self._normalize_key(key): val for key, val in values.items()}
        self.map.setitems(values)

    def __setitem__(self, key, value):
        if self.mode == 'r':
            raise ReadOnlyError()
        key = self._normalize_key(key)
        path = self.dir_path(key)
        try:
            if self.fs.isdir(path):
                self.fs.rm(path, recursive=True)
            self.map[key] = value
            self.fs.invalidate_cache(self.fs._parent(path))
        except self.exceptions as e:
            raise KeyError(key) from e

    def __delitem__(self, key):
        if self.mode == 'r':
            raise ReadOnlyError()
        key = self._normalize_key(key)
        path = self.dir_path(key)
        if self.fs.isdir(path):
            self.fs.rm(path, recursive=True)
        else:
            del self.map[key]

    def delitems(self, keys):
        if self.mode == 'r':
            raise ReadOnlyError()
        # only remove the keys that exist in the store
        nkeys = [self._normalize_key(key) for key in keys if key in self]
        # rm errors if you pass an empty collection
        if len(nkeys) > 0:
            self.map.delitems(nkeys)

    def __contains__(self, key):
        key = self._normalize_key(key)
        return key in self.map

    def __eq__(self, other):
        return (type(self) == type(other) and self.map == other.map
                and self.mode == other.mode)

    def keys(self):
        return iter(self.map)

    def __iter__(self):
        return self.keys()

    def __len__(self):
        return len(list(self.keys()))

    def dir_path(self, path=None):
        store_path = normalize_storage_path(path)
        return self.map._key_to_str(store_path)

    def listdir(self, path=None):
        dir_path = self.dir_path(path)
        try:
            children = sorted(p.rstrip('/').rsplit('/', 1)[-1]
                              for p in self.fs.ls(dir_path, detail=False))
            if self.key_separator != "/":
                return children
            else:
                if self._array_meta_key in children:
                    # special handling of directories containing an array to map nested chunk
                    # keys back to standard chunk keys
                    new_children = []
                    root_path = self.dir_path(path)
                    for entry in children:
                        entry_path = os.path.join(root_path, entry)
                        if _prog_number.match(entry) and self.fs.isdir(entry_path):
                            for file_name in self.fs.find(entry_path):
                                file_path = os.path.join(dir_path, file_name)
                                rel_path = file_path.split(root_path)[1]
                                rel_path = rel_path.lstrip('/')
                                new_children.append(rel_path.replace('/', '.'))
                        else:
                            new_children.append(entry)
                    return sorted(new_children)
                else:
                    return children
        except IOError:
            return []

    def rmdir(self, path=None):
        if self.mode == 'r':
            raise ReadOnlyError()
        store_path = self.dir_path(path)
        if self.fs.isdir(store_path):
            self.fs.rm(store_path, recursive=True)

    def getsize(self, path=None):
        store_path = self.dir_path(path)
        return self.fs.du(store_path, True, True)

    def clear(self):
        if self.mode == 'r':
            raise ReadOnlyError()
        self.map.clear() def rename(store: BaseStore, src_path: Path, dst_path: Path):
    """Rename all items under the given path. If `store` provides a `rename` method,
    this will be called, otherwise will fall back to implementation via the
    `Store` interface."""
    src_path = normalize_storage_path(src_path)
    dst_path = normalize_storage_path(dst_path)
    if hasattr(store, 'rename'):
        # pass through
        store.rename(src_path, dst_path)
    else:
        # slow version, delete one key at a time
        _rename_from_keys(store, src_path, dst_path) def listdir(store: BaseStore, path: Path = None):
    """Obtain a directory listing for the given path. If `store` provides a `listdir`
    method, this will be called, otherwise will fall back to implementation via the
    `MutableMapping` interface."""
    path = normalize_storage_path(path)
    if hasattr(store, 'listdir'):
        # pass through
        return store.listdir(path)  # type: ignore
    else:
        # slow version, iterate through all keys
        warnings.warn(
            f"Store {store} has no `listdir` method. From zarr 2.9 onwards "
            "may want to inherit from `Store`.",
            stacklevel=2,
        )
        return _listdir_from_keys(store, path) class CountingDict(Store):
    def __init__(self):
    def __len__(self):
    def keys(self):
    def __iter__(self):
    def __contains__(self, item):
    def __getitem__(self, item):
    def __setitem__(self, key, value):
    def __delitem__(self, key):
def skip_test_env_var(name):
def abs_container():
    URL = "http://127.0.0.1:10000"
    ACCOUNT_NAME = "devstoreaccount1"
    KEY = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
    CONN_STR = (
        f"DefaultEndpointsProtocol=http;AccountName={ACCOUNT_NAME};"
        f"AccountKey={KEY};BlobEndpoint={URL}/{ACCOUNT_NAME};"
    )
import array
import atexit
import json
import os
import pathlib
import sys
import pickle
import shutil
import tempfile
import numpy as np
import pytest
import zarr
        import zarr
        import zarr
        import zarr
        import zarr
        import zarr
        import zarr
        import zarr
    import shlex
    import subprocess
    import time
        import zarr
        import dbm.dumb as dumbdbm
from contextlib import contextmanager
from pickle import PicklingError
from zipfile import ZipFile
from numpy.testing import assert_array_almost_equal, assert_array_equal
from numcodecs.compat import ensure_bytes
from zarr.codecs import BZ2, AsType, Blosc, Zlib
from zarr.convenience import consolidate_metadata
from zarr.errors import MetadataError
from zarr.hierarchy import group
from zarr.meta import ZARR_FORMAT, decode_array_metadata
from zarr.n5 import N5Store, N5FSStore
from zarr.storage import (ABSStore, ConsolidatedMetadataStore, DBMStore,
                          DictStore, DirectoryStore, KVStore, LMDBStore,
                          LRUStoreCache, MemoryStore, MongoDBStore,
                          NestedDirectoryStore, RedisStore, SQLiteStore,
                          Store, TempStore, ZipStore,
                          array_meta_key, atexit_rmglob, atexit_rmtree,
                          attrs_key, default_compressor, getsize,
                          group_meta_key, init_array, init_group, migrate_1to2)
from zarr.storage import FSStore, rename, listdir
from zarr.tests.util import CountingDict, have_fsspec, skip_test_env_var, abs_container
    from zarr import meta_v1
        store = SQLiteStore(':memory:', **kwargs)
        return store

    def test_pickle(self):

        # setup store
        store = self.create_store()
        store['foo'] = b'bar'
        store['baz'] = b'quux'

        # round-trip through pickle
        with pytest.raises(PicklingError):
            pickle.dumps(store)


@skip_test_env_var("ZARR_TEST_MONGO")
class TestMongoDBStore(StoreTests):

    def create_store(self, **kwargs):
        pytest.importorskip("pymongo")
        store = MongoDBStore(host='127.0.0.1', database='zarr_tests',
                             collection='zarr_tests', **kwargs)
        # start with an empty store
        store.clear()
        return store


@skip_test_env_var("ZARR_TEST_REDIS")
class TestRedisStore(StoreTests):

    def create_store(self, **kwargs):
        # TODO: this is the default host for Redis on Travis,
        # we probably want to generalize this though
        pytest.importorskip("redis")
        store = RedisStore(host='localhost', port=6379, **kwargs)
        # start with an empty store
        store.clear()
        return store


class TestLRUStoreCache(StoreTests):

    def create_store(self, **kwargs):
        # wrapper therefore no dimension_separator argument
        skip_if_nested_chunks(**kwargs)
        return LRUStoreCache(dict(), max_size=2**27)

    def test_cache_values_no_max_size(self):

        # setup store
        store = CountingDict()
        store['foo'] = b'xxx'
        store['bar'] = b'yyy'
        assert 0 == store.counter['__getitem__', 'foo']
        assert 1 == store.counter['__setitem__', 'foo']
        assert 0 == store.counter['__getitem__', 'bar']
        assert 1 == store.counter['__setitem__', 'bar']

        # setup cache
        cache = LRUStoreCache(store, max_size=None)
        assert 0 == cache.hits
        assert 0 == cache.misses

        # test first __getitem__, cache miss
        assert b'xxx' == cache['foo']
        assert 1 == store.counter['__getitem__', 'foo']
        assert 1 == store.counter['__setitem__', 'foo']
        assert 0 == cache.hits
        assert 1 == cache.misses

        # test second __getitem__, cache hit
        assert b'xxx' == cache['foo']
        assert 1 == store.counter['__getitem__', 'foo']
        assert 1 == store.counter['__setitem__', 'foo']
        assert 1 == cache.hits
        assert 1 == cache.misses

        # test __setitem__, __getitem__
        cache['foo'] = b'zzz'
        assert 1 == store.counter['__getitem__', 'foo']
        assert 2 == store.counter['__setitem__', 'foo']
        # should be a cache hit
        assert b'zzz' == cache['foo']
        assert 1 == store.counter['__getitem__', 'foo']
        assert 2 == store.counter['__setitem__', 'foo']
        assert 2 == cache.hits
        assert 1 == cache.misses

        # manually invalidate all cached values
        cache.invalidate_values()
        assert b'zzz' == cache['foo']
        assert 2 == store.counter['__getitem__', 'foo']
        assert 2 == store.counter['__setitem__', 'foo']
        cache.invalidate()
        assert b'zzz' == cache['foo']
        assert 3 == store.counter['__getitem__', 'foo']
        assert 2 == store.counter['__setitem__', 'foo']

        # test __delitem__
        del cache['foo']
        with pytest.raises(KeyError):
            # noinspection PyStatementEffect
            cache['foo']
        with pytest.raises(KeyError):
            # noinspection PyStatementEffect
            store['foo']

        # verify other keys untouched
        assert 0 == store.counter['__getitem__', 'bar']
        assert 1 == store.counter['__setitem__', 'bar']

    def test_cache_values_with_max_size(self):

        # setup store
        store = CountingDict()
        store['foo'] = b'xxx'
        store['bar'] = b'yyy'
        assert 0 == store.counter['__getitem__', 'foo']
        assert 0 == store.counter['__getitem__', 'bar']
        # setup cache - can only hold one item

        cache = LRUStoreCache(store, max_size=5)