class DatatypeMessage(object):
    """ Representation of a HDF5 Datatype Message. """
    # Contents and layout defined in IV.A.2.d.

    def __init__(self, buf, offset):
        self.buf = buf
        self.offset = offset
        self.dtype = self.determine_dtype()

    def determine_dtype(self):
        """ Return the dtype (often numpy-like) for the datatype message.  """
        datatype_msg = _unpack_struct_from(DATATYPE_MSG, self.buf, self.offset)
        self.offset += DATATYPE_MSG_SIZE
        # last 4 bits
        datatype_class = datatype_msg['class_and_version'] & 0x0F

        if datatype_class == DATATYPE_FIXED_POINT:
            return self._determine_dtype_fixed_point(datatype_msg)
        elif datatype_class == DATATYPE_FLOATING_POINT:
            return self._determine_dtype_floating_point(datatype_msg)
        elif datatype_class == DATATYPE_TIME:
            raise NotImplementedError("Time datatype class not supported.")
        elif datatype_class == DATATYPE_STRING:
            return self._determine_dtype_string(datatype_msg)
        elif datatype_class == DATATYPE_BITFIELD:
            raise NotImplementedError("Bitfield datatype class not supported.")
        elif datatype_class == DATATYPE_OPAQUE:
            raise NotImplementedError("Opaque datatype class not supported.")
        elif datatype_class == DATATYPE_COMPOUND:
            return self._determine_dtype_compound(datatype_msg)
        elif datatype_class == DATATYPE_REFERENCE:
            return ('REFERENCE', datatype_msg['size'])
        elif datatype_class == DATATYPE_ENUMERATED:
            raise NotImplementedError(
                "Enumerated datatype class not supported.")
        elif datatype_class == DATATYPE_ARRAY:
            raise NotImplementedError("Array datatype class not supported.")
        elif datatype_class == DATATYPE_VARIABLE_LENGTH:
            vlen_type = self._determine_dtype_vlen(datatype_msg)
            if vlen_type[0] == 'VLEN_SEQUENCE':
                base_type = self.determine_dtype()
                vlen_type = ('VLEN_SEQUENCE', base_type)
            return vlen_type
        raise InvalidHDF5File('Invalid datatype class %i' % (datatype_class))

    def _determine_dtype_fixed_point(self, datatype_msg):
        """ Return the NumPy dtype for a fixed point class. """
        # fixed-point types are assumed to follow IEEE standard format
        length_in_bytes = datatype_msg['size']
        if length_in_bytes not in [1, 2, 4, 8]:
            raise NotImplementedError("Unsupported datatype size")

        signed = datatype_msg['class_bit_field_0'] & 0x08
        if signed > 0:
            dtype_char = 'i'
        else:
            dtype_char = 'u'

        byte_order = datatype_msg['class_bit_field_0'] & 0x01
        if byte_order == 0:
            byte_order_char = '<'  # little-endian
        else:
            byte_order_char = '>'  # big-endian

        # 4-byte fixed-point property description
        # not read, assumed to be IEEE standard format
        self.offset += 4

        return byte_order_char + dtype_char + str(length_in_bytes)

    def _determine_dtype_floating_point(self, datatype_msg):
        """ Return the NumPy dtype for a floating point class. """
        # Floating point types are assumed to follow IEEE standard formats
        length_in_bytes = datatype_msg['size']
        if length_in_bytes not in [1, 2, 4, 8]:
            raise NotImplementedError("Unsupported datatype size")

        dtype_char = 'f'

        byte_order = datatype_msg['class_bit_field_0'] & 0x01
        if byte_order == 0:
            byte_order_char = '<'  # little-endian
        else:
            byte_order_char = '>'  # big-endian

        # 12-bytes floating-point property description
        # not read, assumed to be IEEE standard format
        self.offset += 12

        return byte_order_char + dtype_char + str(length_in_bytes)

    @staticmethod
    def _determine_dtype_string(datatype_msg):
        """ Return the NumPy dtype for a string class. """
        return 'S' + str(datatype_msg['size'])

    def _determine_dtype_compound(self, datatype_msg):
        """ Return the dtype of a compound class if supported. """
        bit_field_0 = datatype_msg['class_bit_field_0']
        bit_field_1 = datatype_msg['class_bit_field_1']
        n_comp = bit_field_0 + (bit_field_1 << 4)

        # read in the members of the compound datatype
        members = []
        for _ in range(n_comp):
            null_location = self.buf.index(b'\x00', self.offset)
            name_size = _padded_size(null_location - self.offset, 8)
            name = self.buf[self.offset:self.offset+name_size]
            name = name.strip(b'\x00').decode('utf-8')
            self.offset += name_size

            prop_desc = _unpack_struct_from(
                COMPOUND_PROP_DESC_V1, self.buf, self.offset)
            self.offset += COMPOUND_PROP_DESC_V1_SIZE

            comp_dtype = self.determine_dtype()
            members.append((name, prop_desc, comp_dtype))

        # determine if the compound dtype is complex64/complex128
        if len(members) == 2:
            name1, prop1, dtype1 = members[0]
            name2, prop2, dtype2 = members[1]
            names_valid = (name1 == 'r' and name2 == 'i')
            complex_dtype_map = {
                '>f4': '>c8',
                '<f4': '<c8',
                '>f8': '>c16',
                '<f8': '<c16',
            }
            dtypes_valid = (dtype1 == dtype2) and dtype1 in complex_dtype_map
            half = datatype_msg['size'] // 2
            offsets_valid = (prop1['offset'] == 0 and prop2['offset'] == half)
            props_valid = (
                prop1['dimensionality'] == 0 and
                prop1['permutation'] == 0 and
                prop1['dim_size_1'] == 0 and
                prop1['dim_size_2'] == 0 and
                prop1['dim_size_3'] == 0 and
                prop1['dim_size_4'] == 0 and
                prop2['dimensionality'] == 0 and
                prop2['permutation'] == 0 and
                prop2['dim_size_1'] == 0 and
                prop2['dim_size_2'] == 0 and
                prop2['dim_size_3'] == 0 and
                prop2['dim_size_4'] == 0
            )
            if names_valid and dtypes_valid and offsets_valid and props_valid:
                return complex_dtype_map[dtype1]

        raise NotImplementedError("Compond dtype not supported.")

    @staticmethod
    def _determine_dtype_vlen(datatype_msg):
        """ Return the dtype information for a variable length class. """
        vlen_type = datatype_msg['class_bit_field_0'] & 0x01
        if vlen_type != 1:
            return ('VLEN_SEQUENCE', 0, 0)
        padding_type = datatype_msg['class_bit_field_0'] >> 4  # bits 4-7
        character_set = datatype_msg['class_bit_field_1'] & 0x01
        return ('VLEN_STRING', padding_type, character_set) def _padded_size(size, padding_multipe=8):
    """ Return the size of a field padded to be a multiple a give value. """
    return int(ceil(size / padding_multipe) * padding_multipe) def _structure_size(structure):
    """ Return the size of a structure in bytes. """
    fmt = '<' + ''.join(structure.values())
    return struct.calcsize(fmt) def _unpack_struct_from(structure, buf, offset=0):
    """ Unpack a structure into an OrderedDict from a buffer of bytes. """
    fmt = '<' + ''.join(structure.values())
    values = struct.unpack_from(fmt, buf, offset=offset)
    return OrderedDict(zip(structure.keys(), values)) def _unpack_struct_from_file(structure, fh):
    """ Unpack a structure into an OrderedDict from an open file. """
    size = _structure_size(structure)
    buf = fh.read(size)
    return _unpack_struct_from(structure, buf) class InvalidHDF5File(Exception):
    """ Exception raised when an invalid HDF5 file is detected. """
    pass class Reference(object):
    """
    HDF5 Reference.
    """

    def __init__(self, address_of_reference):
        self.address_of_reference = address_of_reference

    def __bool__(self):
        # False for null references (address of 0) True otherwise
        return bool(self.address_of_reference)

    __nonzero__ = __bool__  # Python 2.x requires __nonzero__ for truth value class BTree(object):
    """
    HDF5 version 1 B-Tree.
    """

    def __init__(self, fh, offset):
        """ initalize. """
        self.fh = fh

        # read in the root node
        root_node = self._read_node(offset)
        self.root_node = root_node

        # read in all nodes
        all_nodes = {}
        node_level = root_node['node_level']
        all_nodes[node_level] = [root_node]
        while node_level != 0:
            new_nodes = []
            for parent_node in all_nodes[node_level]:
                for addr in parent_node['addresses']:
                    new_nodes.append(self._read_node(addr))
            new_node_level = new_nodes[0]['node_level']
            all_nodes[new_node_level] = new_nodes
            node_level = new_node_level
        self.all_nodes = all_nodes

    def _read_node(self, offset):
        """ Return a single node in the B-Tree located at a given offset. """
        self.fh.seek(offset)
        node = _unpack_struct_from_file(B_LINK_NODE_V1, self.fh)
        assert node['signature'] == b'TREE'

        keys = []
        addresses = []
        for _ in range(node['entries_used']):
            key = struct.unpack('<Q', self.fh.read(8))[0]
            address = struct.unpack('<Q', self.fh.read(8))[0]
            keys.append(key)
            addresses.append(address)
        # N+1 key
        keys.append(struct.unpack('<Q', self.fh.read(8))[0])
        node['keys'] = keys
        node['addresses'] = addresses
        return node

    def symbol_table_addresses(self):
        """ Return a list of all symbol table address. """
        all_address = []
        for node in self.all_nodes[0]:
            all_address.extend(node['addresses'])
        return all_address class BTreeRawDataChunks(object):
    """
    HDF5 version 1 B-Tree storing raw data chunk nodes (type 1).
    """

    def __init__(self, fh, offset, dims):
        """ initalize. """
        self.fh = fh
        self.dims = dims

        # read in the root node
        root_node = self._read_node(offset)
        self.root_node = root_node

        # read in all other nodes
        all_nodes = {}
        node_level = root_node['node_level']
        all_nodes[node_level] = [root_node]
        while node_level != 0:
            new_nodes = []
            for parent_node in all_nodes[node_level]:
                for addr in parent_node['addresses']:
                    new_nodes.append(self._read_node(addr))
            new_node_level = new_nodes[0]['node_level']
            all_nodes[new_node_level] = new_nodes
            node_level = new_node_level

        self.all_nodes = all_nodes

    def _read_node(self, offset):
        """ Return a single node in the b-tree located at a give offset. """
        self.fh.seek(offset)
        node = _unpack_struct_from_file(B_LINK_NODE_V1, self.fh)
        assert node['signature'] == b'TREE'
        assert node['node_type'] == 1

        keys = []
        addresses = []
        for _ in range(node['entries_used']):
            chunk_size, filter_mask = struct.unpack('<II', self.fh.read(8))
            fmt = '<' + 'Q' * self.dims
            fmt_size = struct.calcsize(fmt)
            chunk_offset = struct.unpack(fmt, self.fh.read(fmt_size))
            chunk_address = struct.unpack('<Q', self.fh.read(8))[0]

            keys.append(OrderedDict((
                ('chunk_size', chunk_size),
                ('filter_mask', filter_mask),
                ('chunk_offset', chunk_offset),
            )))
            addresses.append(chunk_address)
        node['keys'] = keys
        node['addresses'] = addresses
        return node

    def construct_data_from_chunks(
            self, chunk_shape, data_shape, dtype, filter_pipeline):
        """ Build a complete data array from chunks. """
        if isinstance(dtype, tuple):
            true_dtype = tuple(dtype)
            dtype_class = dtype[0]
            if dtype_class == 'REFERENCE':
                size = dtype[1]
                if size != 8:
                    raise NotImplementedError('Unsupported Reference type')
                dtype = '<u8'
            else:
                raise NotImplementedError('datatype not implemented')
        else:
            true_dtype = None

        # create array to store data
        shape = [_padded_size(i, j) for i, j in zip(data_shape, chunk_shape)]
        data = np.zeros(shape, dtype=dtype)

        # loop over chunks reading each into the full data array
        count = np.prod(chunk_shape)
        itemsize = np.dtype(dtype).itemsize
        chunk_buffer_size = count * itemsize
        for node in self.all_nodes[0]:
            for node_key, addr in zip(node['keys'], node['addresses']):
                self.fh.seek(addr)
                if filter_pipeline is None:
                    chunk_buffer = self.fh.read(chunk_buffer_size)
                else:
                    chunk_buffer = self.fh.read(node_key['chunk_size'])
                    filter_mask = node_key['filter_mask']
                    chunk_buffer = self._filter_chunk(
                        chunk_buffer, filter_mask, filter_pipeline, itemsize)

                chunk_data = np.frombuffer(chunk_buffer, dtype=dtype)
                start = node_key['chunk_offset'][:-1]
                region = [slice(i, i+j) for i, j in zip(start, chunk_shape)]
                data[tuple(region)] = chunk_data.reshape(chunk_shape)

        if isinstance(true_dtype, tuple):
            if dtype_class == 'REFERENCE':
                to_reference = np.vectorize(Reference)
                data = to_reference(data)
            else:
                raise NotImplementedError('datatype not implemented')

        non_padded_region = tuple([slice(i) for i in data_shape])
        return data[non_padded_region]

    @staticmethod
    def _filter_chunk(chunk_buffer, filter_mask, filter_pipeline, itemsize):
        """ Apply decompression filters to a chunk of data. """
        num_filters = len(filter_pipeline)
        for i, pipeline_entry in enumerate(filter_pipeline[::-1]):

            # A filter is skipped is the bit corresponding to its index in the
            # pipeline is set in filter_mask
            filter_index = num_filters - i - 1   # 0 to num_filters - 1
            if filter_mask & (1 << filter_index):
                continue

            filter_id = pipeline_entry['filter_id']
            if filter_id == GZIP_DEFLATE_FILTER:
                chunk_buffer = zlib.decompress(chunk_buffer)

            elif filter_id == SHUFFLE_FILTER:
                buffer_size = len(chunk_buffer)
                unshuffled_buffer = bytearray(buffer_size)
                step = buffer_size // itemsize
                for j in range(itemsize):
                    start = j * step
                    end = (j+1) * step
                    unshuffled_buffer[j::itemsize] = chunk_buffer[start:end]
                chunk_buffer = unshuffled_buffer
            elif filter_id == FLETCH32_FILTER:
                _verify_fletcher32(chunk_buffer)
                # strip off 4-byte checksum from end of buffer
                chunk_buffer = chunk_buffer[:-4]
            else:
                raise NotImplementedError(
                    "Filter with id: %i import supported" % (filter_id))
        return chunk_buffer GZIP_DEFLATE_FILTER = 1 SHUFFLE_FILTER = 2 FLETCH32_FILTER = 3 class Heap(object):
    """
    HDF5 local heap.
    """

    def __init__(self, fh, offset):
        """ initalize. """

        fh.seek(offset)
        local_heap = _unpack_struct_from_file(LOCAL_HEAP, fh)
        assert local_heap['signature'] == b'HEAP'
        assert local_heap['version'] == 0
        fh.seek(local_heap['address_of_data_segment'])
        heap_data = fh.read(local_heap['data_segment_size'])
        local_heap['heap_data'] = heap_data
        self._contents = local_heap
        self.data = heap_data

    def get_object_name(self, offset):
        """ Return the name of the object indicated by the given offset. """
        end = self.data.index(b'\x00', offset)
        return self.data[offset:end] class SymbolTable(object):
    """
    HDF5 Symbol Table.
    """

    def __init__(self, fh, offset, root=False):
        """ initialize, root=True for the root group, False otherwise. """

        fh.seek(offset)
        if root:
            # The root symbol table has no Symbol table node header
            # and contains only a single entry
            node = OrderedDict([('symbols', 1)])
        else:
            node = _unpack_struct_from_file(SYMBOL_TABLE_NODE, fh)
            assert node['signature'] == b'SNOD'
        entries = [_unpack_struct_from_file(SYMBOL_TABLE_ENTRY, fh) for i in
                   range(node['symbols'])]
        if root:
            self.group_offset = entries[0]['object_header_address']
        self.entries = entries
        self._contents = node

    def assign_name(self, heap):
        """ Assign link names to all entries in the symbol table. """
        for entry in self.entries:
            offset = entry['link_name_offset']
            link_name = heap.get_object_name(offset).decode('utf-8')
            entry['link_name'] = link_name
        return

    def get_links(self, heap):
        """ Return a dictionary of links (dataset/group) and offsets. """
        links = {}
        for e in self.entries:
            if e['cache_type'] in [0,1]:
                links[e['link_name']] = e['object_header_address']
            elif e['cache_type'] == 2:
                offset = struct.unpack('<4I', e['scratch'])[0]
                links[e['link_name']] = heap.get_object_name(offset).decode('utf-8')
        return links class GlobalHeap(object):
    """
    HDF5 Global Heap collection.
    """

    def __init__(self, fh, offset):

        fh.seek(offset)
        header = _unpack_struct_from_file(GLOBAL_HEAP_HEADER, fh)
        assert header['signature'] == b'GCOL'
        assert header['version'] == 1
        heap_data_size = header['collection_size'] - GLOBAL_HEAP_HEADER_SIZE
        heap_data = fh.read(heap_data_size)
        assert len(heap_data) == heap_data_size  # check for early end of file

        self.heap_data = heap_data
        self._header = header
        self._objects = None

    @property
    def objects(self):
        """ Dictionary of objects in the heap. """
        if self._objects is None:
            self._objects = OrderedDict()
            offset = 0
            while offset < len(self.heap_data):
                info = _unpack_struct_from(
                    GLOBAL_HEAP_OBJECT, self.heap_data, offset)
                if info['object_index'] == 0:
                    break
                offset += GLOBAL_HEAP_OBJECT_SIZE
                fmt = '<' + str(info['object_size']) + 's'
                obj_data = struct.unpack_from(fmt, self.heap_data, offset)[0]
                self._objects[info['object_index']] = obj_data
                offset += _padded_size(info['object_size'])
        return self._objects
from collections import OrderedDict
from .datatype_msg import DatatypeMessage
from .core import _padded_size, _structure_size
from .core import _unpack_struct_from, _unpack_struct_from_file
from .core import InvalidHDF5File
from .core import Reference
from .btree import BTree, BTreeRawDataChunks
from .btree import GZIP_DEFLATE_FILTER, SHUFFLE_FILTER, FLETCH32_FILTER
from .misc_low_level import Heap, SymbolTable, GlobalHeap
import struct
import warnings
import numpy as np
""" HDF5 dataobjects objects.  """

from __future__ import division





class DataObjects(object):
    """
    HDF5 DataObjects.
    """

    def __init__(self, fh, offset):
        """ initalize. """
        fh.seek(offset)
        version_hint = struct.unpack_from('<B', fh.read(1))[0]
        fh.seek(offset)
        if version_hint == 1:
            msgs, msg_data, header = self._parse_v1_objects(fh)
        elif version_hint == ord('O'):   # first character of v2 signature
            msgs, msg_data, header = self._parse_v2_objects(fh)
        else:
            raise InvalidHDF5File('unknown Data Object Header')

        self.fh = fh
        self.msgs = msgs
        self.msg_data = msg_data
        self.offset = offset
        self._global_heaps = {}
        self._header = header

        # cached attributes
        self._filter_pipeline = None
        self._chunk_params_set = False
        self._chunks = None
        self._chunk_dims = None
        self._chunk_address = None

    @staticmethod
    def _parse_v1_objects(fh):
        """ Parse a collection of version 1 Data Objects. """
        header = _unpack_struct_from_file(OBJECT_HEADER_V1, fh)
        assert header['version'] == 1
        msg_data = fh.read(header['object_header_size'])

        offset = 0
        msgs = []
        for _ in range(header['total_header_messages']):
            msg = _unpack_struct_from(HEADER_MSG_INFO_V1, msg_data, offset)
            msg['offset_to_message'] = offset + 8
            if msg['type'] == OBJECT_CONTINUATION_MSG_TYPE:
                fh_off, size = struct.unpack_from('<QQ', msg_data, offset + 8)
                fh.seek(fh_off)
                msg_data += fh.read(size)
            msgs.append(msg)
            offset += 8 + msg['size']
        return msgs, msg_data, header

    def _parse_v2_objects(self, fh):
        """ Parse a collection of version 2 Data Objects. """

        header, creation_order_size = self._parse_v2_header(fh)

        msgs = []
        msg_data = fh.read(header['size_of_chunk_0'])
        offset = 0
        chunk_sizes = [header['size_of_chunk_0']]
        current_chunk = 0
        size_of_processed_chunks = 0

        while offset < (len(msg_data) - 4):
            msg = _unpack_struct_from(HEADER_MSG_INFO_V2, msg_data, offset)
            msg['offset_to_message'] = offset + 4 + creation_order_size

            if msg['type'] == OBJECT_CONTINUATION_MSG_TYPE:
                fh_off, size = struct.unpack_from(
                    '<QQ', msg_data, offset + 4 + creation_order_size)
                fh.seek(fh_off)
                new_msg_data = fh.read(size)
                assert new_msg_data[:4] == b'OCHK'
                chunk_sizes.append(size-4)
                msg_data += new_msg_data[4:]

            msgs.append(msg)
            offset += 4 + msg['size'] + creation_order_size

            chunk_offset = offset - size_of_processed_chunks
            if (chunk_offset + 4) >= chunk_sizes[current_chunk]:
                # move to next chunk
                current_chunk_size = chunk_sizes[current_chunk]
                offset += (current_chunk_size - chunk_offset)
                size_of_processed_chunks += current_chunk_size
                current_chunk += 1

        return msgs, msg_data, header

    @staticmethod
    def _parse_v2_header(fh):
        """ Parse a version 2 data object header. """

        header = _unpack_struct_from_file(OBJECT_HEADER_V2, fh)