def encode(obj):
    """
    Encode numpy arrays and slices
    :param obj: object to serialize
    :return: dictionary with encoded array or slice
    """
    if isinstance(obj, np.ndarray):
        arr = header_data_from_array_1_0(obj)
        arr['arraydata'] = obj.tostring()
        arr['__ndarray__'] = True
        return arr
    elif isinstance(obj, slice):
        return {
            '__slice__': (obj.start, obj.stop, obj.step)
        }
    elif isclass(obj) and issubclass(obj, np.number):
        # make sure numpy type classes such as np.float64 (used, e.g., as dtype
        # arguments) are serialized to strings
        return obj().dtype.name
    elif isinstance(obj, np.dtype):
        return obj.name
    elif isinstance(obj, np.number):
        # convert to Python scalar
        return np.asscalar(obj)
    elif isinstance(obj, File):
        data = {
            RESPONSE_H5FILE: obj.file,
            RESPONSE_NODE_TYPE: NODE_TYPE_FILE,
            RESPONSE_NODE_PATH: obj.path,
        }
        return data
    elif isinstance(obj, Group):
        # TODO include attrs?
        data = {
            RESPONSE_H5FILE: obj.file,
            RESPONSE_NODE_TYPE: NODE_TYPE_GROUP,
            RESPONSE_NODE_PATH: obj.path,
        }
        return data
    elif isinstance(obj, Dataset):
        # TODO include attrs?
        data = {
            RESPONSE_H5FILE: obj.file,
            RESPONSE_NODE_TYPE: NODE_TYPE_DATASET,
            RESPONSE_NODE_PATH: obj.path,
            RESPONSE_NODE_SHAPE: obj.shape,
            RESPONSE_NODE_DTYPE: obj.dtype,
        }
        return data

    return obj def _stderr_supports_color():
def _unicode(value):
def _safe_unicode(s):
    def __init__(self, color=True, fmt=DEFAULT_FORMAT,
                 datefmt=DEFAULT_DATE_FORMAT, colors=DEFAULT_COLORS):
    def format(self, record):
def enable_pretty_logging(options=None, logger=None):
def define_logging_options(options=None):
_TO_UNICODE_TYPES = (unicode_type, type(None))
    DEFAULT_FORMAT = '%(color)s[%(levelname)1.1s %(asctime)s %(module)s:%(lineno)d]%(end_color)s %(message)s'
    DEFAULT_DATE_FORMAT = '%y%m%d %H:%M:%S'
    DEFAULT_COLORS = {
        logging.DEBUG: 4,  # Blue
        logging.INFO: 2,  # Green
        logging.WARNING: 3,  # Yellow
        logging.ERROR: 1,  # Red
    }
class LogFormatter(logging.Formatter): class Error(Exception):
class OptionParser(object):
class _Mockable(object):
class _Option(object):
    def __init__(self):
    def _normalize_name(self, name):
    def __getattr__(self, name):
    def __setattr__(self, name, value):
    def __iter__(self):
    def __contains__(self, name):
    def __getitem__(self, name):
    def __setitem__(self, name, value):
    def items(self):
    def groups(self):
    def group_dict(self, group):
    def as_dict(self):
    def define(self, name, default=None, type=None, help=None, metavar=None,
               multiple=False, group=None, callback=None):
    def parse_command_line(self, args=None, final=True):
    def parse_config_file(self, path, final=True):
    def print_help(self, file=None):
    def _help_callback(self, value):
    def add_parse_callback(self, callback):
    def run_parse_callbacks(self):
    def mockable(self):
    def __init__(self, options):
    def __getattr__(self, name):
    def __setattr__(self, name, value):
    def __delattr__(self, name):
    def __init__(self, name, default=None, type=basestring_type, help=None,
                 metavar=None, multiple=False, file_name=None, group_name=None,
                 callback=None):
    def value(self):
    def parse(self, value):
    def set(self, value):
    def _parse_datetime(self, value):
    def _parse_timedelta(self, value):
    def _parse_bool(self, value):
    def _parse_string(self, value):
def define(name, default=None, type=None, help=None, metavar=None,
           multiple=False, group=None, callback=None):
def parse_command_line(args=None, final=True):
def parse_config_file(path, final=True):
def print_help(file=None):
def add_parse_callback(callback):
    UNSET = object()
    _DATETIME_FORMATS = [
        "%a %b %d %H:%M:%S %Y",
        "%Y-%m-%d %H:%M:%S",
        "%Y-%m-%d %H:%M",
        "%Y-%m-%dT%H:%M",
        "%Y%m%d %H:%M:%S",
        "%Y%m%d %H:%M",
        "%Y-%m-%d",
        "%Y%m%d",
        "%H:%M:%S",
        "%H:%M",
    ]
    _TIMEDELTA_ABBREV_DICT = {
        'h': 'hours',
        'm': 'minutes',
        'min': 'minutes',
        's': 'seconds',
        'sec': 'seconds',
        'ms': 'milliseconds',
        'us': 'microseconds',
        'd': 'days',
        'w': 'weeks',
    }
    _FLOAT_PATTERN = r'[-+]?(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][-+]?\d+)?'
    _TIMEDELTA_PATTERN = re.compile(
        r'\s*(%s)\s*(\w*)\s*' % _FLOAT_PATTERN, re.IGNORECASE) FILE_EXISTS = 300 OK = 100 FILE_NOT_FOUND = 301 GROUP_EXISTS = 400 NODE_NOT_FOUND = 402 DATASET_EXISTS = 401 VALUE_ERROR = 403 TYPE_ERROR = 404 CREATED = 101 UNKNOWN_COMMAND = 200 MISSING_ARGUMENT = 201 MISSING_DATA = 203 INCOMPATIBLE_DATA = 204  # incompatible shape and/or dtype KEY_ERROR = 405 INVALID_ARGUMENT = 202 INTERNAL_SERVER_ERROR = 500 class File(Group):
    """
    Wrapper for h5py.File
    """

    def __init__(self, name, mode="r", *args, **kwargs):
        """
        try to open/create an h5py.File object
        """
        app_log.debug("XXX {},{} ...".format(name, mode))

        # call h5py.File() in case file needs to be created
        if mode in ("w", "w-", "x", "a"):
            self.file = name  # this is crucial for the @writer annotation

            @writer
            def init(self):
                with h5py.File(name=name, mode=mode, *args, **kwargs):
                    pass

            init(self)

        Group.__init__(self, name, '/')

    def __enter__(self):
        """
        simple context manager (so we can use 'with File() as f')
        """
        # TODO can be removed
        return self

    def __exit__(self, type, value, tb):
        pass

    def __repr__(self):
        return "<HDF5 File ({0})>".format(self.file)

    @property
    @reader
    def filesize(self):
        """
        Return size of the file in bytes
        """
        stat = os.stat(self.file)
        return stat.st_size

    @writer
    def rename(self, new):
        """
        Rename hdf5 file

        Args:
            new: new filename

        Raises:
            FileExistsError if file ``new`` already exists
        """
        if os.path.isfile(new):
            raise FileExistsError("file {} exists".format(new))
        os.rename(self.file, new)

    @writer
    def delete(self):
        """
        Remove hdf5 file
        """
        os.remove(self.file) class Dataset(Node):
    """
    Wrapper for h5py.Dataset
    """

    def __init__(self, file, path):
        Node.__init__(self, file, path)

    @reader
    def __getitem__(self, slice):
        """
        implement multidimensional slicing for datasets
        """
        with h5py.File(self.file, 'r') as f:
            return f[self.path][slice]

    @writer
    def __setitem__(self, slice, value):
        """
        Broadcasting for datasets. Example: mydataset[0,:] = np.arange(100)
        """
        with h5py.File(self.file, 'r+') as f:
            f[self.path][slice] = value

    @writer
    def resize(self, size, axis=None):
        with h5py.File(self.file, 'r+') as f:
            f[self.path].resize(size, axis)

    @property
    @reader
    def shape(self):
        with h5py.File(self.file, 'r') as f:
            return f[self.path].shape

    @property
    @reader
    def dtype(self):
        with h5py.File(self.file, 'r') as f:
            return f[self.path].dtype class Group(Node):
    """
    Wrapper for h5py.Group
    """

    def __init__(self, file, path):
        Node.__init__(self, file, path)

    def __repr__(self):
        return "<HDF5 Group (path={0})>".format(self.path)

    @writer
    def create_group(self, name):
        """
        Wrapper around ``h5py.Group.create_group()``
        """
        with h5py.File(self.file, 'r+') as f:
            group = f[self.path]
            created_group = group.create_group(name)
            path = created_group.name

        return Group(self.file, path=path)

    @writer
    def require_group(self, name):
        """
        Wrapper around ``h5py.Group.require_group()``
        """
        with h5py.File(self.file, 'r+') as f:
            group = f[self.path]
            created_group = group.require_group(name)
            path = created_group.name

        return Group(self.file, path=path)

    @writer
    def create_dataset(self, **kwargs):
        """
        Wrapper around ``h5py.Group.create_dataset()``
        """
        overwrite = kwargs.get('overwrite', False)
        name = kwargs['name']
        # remove additional arguments because they are not supported by h5py
        try:
            del kwargs['overwrite']
        except Exception:
            pass
        with h5py.File(self.file, 'r+') as f:
            group = f[self.path]
            if overwrite and name in group:
                del group[name]
            dst = group.create_dataset(**kwargs)
            path = dst.name

        return Dataset(self.file, path=path)

    @writer
    def require_dataset(self, **kwargs):
        """
        Wrapper around ``h5py.Group.require_dataset()``
        """
        with h5py.File(self.file, 'r+') as f:
            group = f[self.path]
            dst = group.require_dataset(**kwargs)
            path = dst.name
        return Dataset(self.file, path=path)

    @reader
    def keys(self):
        with h5py.File(self.file, 'r') as f:
            # w/o list() it does not work with py3 (returns a view on a closed
            # hdf5 file)
            keys = list(f[self.path].keys())
            return keys

    # TODO visit() and visititems() do not yet work because @reader methods
    # are not reentrant! => just wrap code into an inner function!

    # @reader
    # def visit(self, func):
    #     """
    #     Wrapper around h5py.Group.vist()

    #     Args:
    #         func: a unary function
    #     """
    #     with h5py.File(self.file, 'r') as f:
    #         return f[self.path].visit(func)

    # @reader
    # def visititems(self, func):
    #     """
    #     Wrapper around h5py.Group.visititems()

    #     Args:
    #         func: a 2-ary function
    #     """
    #     with h5py.File(self.file, 'r') as f:
    #         grp = f[self.path]
    #         def proxy(name):
    #             obj = self._wrap_class(grp[name])
    #             return func(name, obj)
    #         return self.visit(proxy)

    @reader
    def tree(self):
        """
        Return tree data structure consisting of all groups and datasets.
        A tree node is defined recursively as a tuple:

            [Dataset/Group, [children]]

        Returns: list
        """

        def buildtree(treenode):
            h5py_obj, children = treenode
            assert (children == [])
            if isinstance(h5py_obj, h5py.Dataset):
                # wrap h5py dataset object
                treenode[0] = self._wrap_class(h5py_obj)
                return
            else:
                # wrap h5py group object
                treenode[0] = self._wrap_class(h5py_obj)
                for name, childobj in h5py_obj.items():
                    newnode = [childobj, []]
                    children.append(newnode)
                    buildtree(newnode)

        tree = None
        with h5py.File(self.file, 'r') as f:
            root = f[self.path]
            tree = [root, []]  # [h5py object, children]
            buildtree(tree)

        return tree

    @reader
    def items(self):
        """
        Returns a list of (name, object) pairs for objects directly
        attached to this group. Values for broken soft or external links
        show up as None.
        Note that this differs from h5py, where a list (Py2) or a
        "set-like object" (Py3) is returned.
        """
        result = []
        with h5py.File(self.file, 'r') as f:
            for name, obj in f[self.path].items():
                result.append((name, self._wrap_class(obj)))

        return result

    @reader
    def __contains__(self, key):
        with h5py.File(self.file, 'r') as f:
            group = f[self.path]
            return key in group

    @writer
    def __delitem__(self, key):
        with h5py.File(self.file, 'r+') as f:
            group = f[self.path]
            del group[key]
import os
import msgpack
from hurray.msgpack_ext import encode as encode_msgpack
from hurray.protocol import (CMD_CREATE_DATABASE, CMD_RENAME_DATABASE,
                             CMD_DELETE_DATABASE, CMD_USE_DATABASE,
                             CMD_LIST_DATABASES, CMD_CREATE_GROUP,
                             CMD_REQUIRE_GROUP, CMD_CREATE_DATASET,
                             CMD_REQUIRE_DATASET, CMD_GET_FILESIZE,
                             CMD_GET_NODE, CMD_CONTAINS, CMD_GET_KEYS,
                             CMD_GET_TREE,
                             CMD_SLICE_DATASET, CMD_BROADCAST_DATASET,
                             CMD_ATTRIBUTES_GET, CMD_ATTRIBUTES_SET,
                             CMD_ATTRIBUTES_CONTAINS, CMD_ATTRIBUTES_KEYS,
                             CMD_KW_CMD, CMD_KW_ARGS, CMD_KW_DB,
                             CMD_KW_DB_RENAMETO, CMD_KW_OVERWRITE, CMD_KW_PATH,
                             CMD_KW_DATA, CMD_KW_KEY, CMD_KW_STATUS,
                             CMD_KW_SHAPE, CMD_KW_DTYPE, CMD_KW_REQUIRE_EXACT,
                             CMD_KW_CHUNKS, CMD_KW_FILLVALUE,
                             CMD_KW_COMPRESSION, CMD_KW_COMPRESSION_OPTS,
                             RESPONSE_ATTRS_CONTAINS, RESPONSE_ATTRS_KEYS,
                             RESPONSE_NODE_KEYS, RESPONSE_NODE_TREE)
from hurray.server.log import app_log
from hurray.server.options import define, options
from hurray.status_codes import (FILE_EXISTS, OK, FILE_NOT_FOUND, GROUP_EXISTS,
                                 NODE_NOT_FOUND, DATASET_EXISTS, VALUE_ERROR,
                                 TYPE_ERROR, CREATED, UNKNOWN_COMMAND,
                                 MISSING_ARGUMENT, MISSING_DATA,
                                 INCOMPATIBLE_DATA, KEY_ERROR,
                                 INVALID_ARGUMENT, INTERNAL_SERVER_ERROR)
from .swmr import File, Group, Dataset
                 CMD_REQUIRE_GROUP,
                 CMD_CREATE_DATASET,
                 CMD_REQUIRE_DATASET,
                 CMD_GET_NODE,
                 CMD_CONTAINS,
                 CMD_GET_KEYS,
                 CMD_GET_TREE,
                 CMD_SLICE_DATASET,
                 CMD_BROADCAST_DATASET,
                 CMD_ATTRIBUTES_GET,
                 CMD_ATTRIBUTES_SET,
                 CMD_ATTRIBUTES_CONTAINS,
                 CMD_ATTRIBUTES_KEYS)

define('base', default='~/hurray_data/', group='application',
       help="Location of hdf5 files")


def db_path(database):
    """
    Return the absolute path of the database based on the "base" option
    :param database: Name of hdf5 file
    :return: Absolute path of hdf5 file
    """
    absbase = os.path.abspath(os.path.expanduser(options.base))
    absfilepath = os.path.abspath(os.path.join(absbase, database))
    if not absfilepath.startswith(absbase):
        raise ValueError("File {} is not under base directory {}"
                         .format(absfilepath, absbase))

    return absfilepath


def db_exists(database):
    """
    Check if given database file exists
    :param database:
    :return:
    """
    path = db_path(database)
    app_log.debug('Checking if {} exists'.format(path))
    return os.path.isfile(path)


def response(status, data=None):
    """
    Args:
        status: status code of response
        data: NumPy array or Python object
    """
    resp = {
        CMD_KW_STATUS: status
    }
    if data is not None:
        resp["data"] = data
        # resp.update(data)

    # print("response (PID {}): {}".format(os.getpid(), resp))

    return msgpack.packb(resp, default=encode_msgpack, use_bin_type=True)


def handle_request(msg):
    """
    Process hurray message
    :param msg: Message dictionary with 'cmd' and 'args' keys
    :return: Msgpacked response as bytes
    """
    cmd = msg.get(CMD_KW_CMD, None)
    args = msg.get(CMD_KW_ARGS, {})
    data = msg.get(CMD_KW_DATA, None)

    app_log.debug('Process "%s" (%s)', cmd,
                  ', '.join(['%s=%s' % (k, v) for k, v in args.items()]))

    # return values
    status = OK
    data_response = None

    if cmd in DATABASE_COMMANDS:  # file related commands

        db = args.get(CMD_KW_DB, None)
        if db is not None and len(db) < 1:
            return response(INVALID_ARGUMENT)

        if cmd == CMD_CREATE_DATABASE:
            if db is None:
                return response(MISSING_ARGUMENT)
            overwrite = args[CMD_KW_OVERWRITE]
            if db_exists(db) and not overwrite:
                status = FILE_EXISTS
            else:
                flags = "w" if overwrite else "w-"
                filepath = db_path(db)
                app_log.debug("Creating {} ...".format(filepath))
                # create sub-directories (if any)
                # note that db_path() guarantees that this is safe
                os.makedirs(os.path.split(filepath)[0], exist_ok=True)
                File(filepath, flags)
                status = CREATED
        elif cmd == CMD_RENAME_DATABASE:
            if db is None:
                return response(MISSING_ARGUMENT)
            if not db_exists(db):
                status = FILE_NOT_FOUND
            else:
                f = File(db_path(db), "r")
                filepath_new = db_path(args[CMD_KW_DB_RENAMETO])
                try:
                    f.rename(filepath_new)
                except FileExistsError as e:
                    status = FILE_EXISTS
                else:
                    # we cannot return f because rename() is not "in place"
                    f_renamed = File(filepath_new, "r")
                    data_response = f_renamed
        elif cmd == CMD_DELETE_DATABASE:
            if db is None:
                return response(MISSING_ARGUMENT)
            if not db_exists(db):

                status = FILE_NOT_FOUND