"""NetCDF reader.

Pupynere implements a PUre PYthon NEtcdf REader.
"""

__author__ = "<NAME> <<EMAIL>>"


import struct
import mmap

from numpy import ndarray, zeros, array

ABSENT = '\x00' * 8
ZERO = '\x00' * 4
NC_BYTE = '\x00\x00\x00\x01'
NC_CHAR = '\x00\x00\x00\x02'
NC_SHORT = '\x00\x00\x00\x03'
NC_INT = '\x00\x00\x00\x04'
NC_FLOAT = '\x00\x00\x00\x05'
NC_DOUBLE = '\x00\x00\x00\x06'
NC_DIMENSION = '\x00\x00\x00\n'
NC_VARIABLE = '\x00\x00\x00\x0b'
NC_ATTRIBUTE = '\x00\x00\x00\x0c'


class NetCDFFile(object):
    """A NetCDF file parser."""

    def __init__(self, file):
        self._buffer = open(file, 'rb')
        self._parse()

    def read(self, size=-1):
        """Alias for reading the file buffer."""
        return self._buffer.read(size)

    def _parse(self):
        """Initial parsing of the header."""
        # Check magic bytes.
        assert self.read(3) == 'CDF'

        # Read version byte.
        byte = self.read(1)
        self.version_byte = struct.unpack('>b', byte)[0]

        # Read header info.
        self._numrecs()
        self._dim_array()
        self._gatt_array()
        self._var_array()

    def _numrecs(self):
        """Read number of records."""
        self._nrecs = self._unpack_int()

    def _dim_array(self):
        """Read a dict with dimensions names and sizes."""
        assert self.read(4) in [ZERO, NC_DIMENSION]
        count = self._unpack_int()

        self.dimensions = {}
        self._dims = []
        for dim in range(count):
            name = self._read_string()
            length = self._unpack_int()
            if length == 0:
                length = None  # record dimension
            self.dimensions[name] = length
            self._dims.append(name)  # preserve dim order

    def _gatt_array(self):
        """Read global attributes."""
        self.attributes = self._att_array()

        # Update __dict__ for compatibility with S.IO.N
        self.__dict__.update(self.attributes)

    def _att_array(self):
        """Read a dict with attributes."""
        assert self.read(4) in [ZERO, NC_ATTRIBUTE]
        count = self._unpack_int()

        # Read attributes.
        attributes = {}
        for attribute in range(count):
            name = self._read_string()
            nc_type = self._unpack_int()
            n = self._unpack_int()

            # Read value for attributes.
            attributes[name] = self._read_values(n, nc_type)

        return attributes

    def _var_array(self):
        """Read all variables."""
        assert self.read(4) in [ZERO, NC_VARIABLE]

        # Read size of each record, in bytes.
        self._read_recsize()

        # Read variables.
        self.variables = {}
        count = self._unpack_int()
        for variable in range(count):
            name = self._read_string()
            self.variables[name] = self._read_var()

    def _read_recsize(self):
        """Read all variables and compute record bytes."""
        pos = self._buffer.tell()

        recsize = 0
        count = self._unpack_int()
        for variable in range(count):
            name = self._read_string()
            n = self._unpack_int()
            isrec = False
            for i in range(n):
                dimid = self._unpack_int()
                name = self._dims[dimid]
                dim = self.dimensions[name]
                if dim is None and i == 0:
                    isrec = True
            vsize = self._unpack_int()

            if isrec:
                recsize += vsize

        self._recsize = recsize
        self._buffer.seek(pos)

    def _read_var(self):
        dimensions = []
        shape = []
        n = self._unpack_int()
        isrec = False
        for i in range(n):
            dimid = self._unpack_int()
            name = self._dims[dimid]
            dimensions.append(name)
            dim = self.dimensions[name]
            if dim is None and i == 0:
                dim = self._nrecs
                isrec = True
            shape.append(dim)
        dimensions = tuple(dimensions)
        shape = tuple(shape)

        attributes = self._att_array()
        nc_type = self._unpack_int()
        vsize = self._unpack_int()

        # Read offset.
        begin = [self._unpack_int, self._unpack_int64][self.version_byte-1]()

        return NetCDFVariable(
            self._buffer.fileno(), nc_type, vsize, begin, shape, dimensions,
            attributes, isrec, self._recsize)

    def _read_values(self, n, nc_type):
        bytes = [1, 1, 2, 4, 4, 8]
        typecodes = ['b', 'c', 'h', 'i', 'f', 'd']

        count = n * bytes[nc_type-1]
        values = self.read(count)

        typecode = typecodes[nc_type-1]
        if nc_type != 2:  # not char
            values = struct.unpack('>%s' % (typecode * n), values)
            values = array(values, dtype=typecode)
        else:
            # Remove EOL terminator.
            if values.endswith('\x00'):
                values = values[:-1]

        return values

    def _unpack_int(self):
        return struct.unpack('>i', self.read(4))[0]
    _unpack_int32 = _unpack_int

    def _unpack_int64(self):
        return struct.unpack('>q', self.read(8))[0]

    def _read_string(self):
        count = struct.unpack('>i', self.read(4))[0]
        s = self.read(count)
        # Remove EOL terminator.
        if s.endswith('\x00'):
            s = s[:-1]
        return s

    def close(self):
        self._buffer.close()


class NetCDFVariable(object):
    def __init__(self, fileno, nc_type, vsize, begin, shape, dimensions,
                 attributes, isrec=False, recsize=0):
        self._nc_type = nc_type
        self._vsize = vsize
        self._begin = begin
        self.shape = shape
        self.dimensions = dimensions
        self.attributes = attributes  # for ``dap.plugins.netcdf``
        self.__dict__.update(attributes)
        self._is_record = isrec

        # Number of bytes and type.
        self._bytes = [1, 1, 2, 4, 4, 8][self._nc_type-1]
        type_ = ['i', 'S', 'i', 'i', 'f', 'f'][self._nc_type-1]
        dtype = '>%s%d' % (type_, self._bytes)
        bytes = self._begin + self._vsize

        if isrec:
            # Record variables are not stored contiguosly on disk, so we
            # need to create a separate array for each record.
            self.__array_data__ = zeros(shape, dtype)
            bytes += (shape[0] - 1) * recsize
            for n in range(shape[0]):
                offset = self._begin + (n * recsize)
                mm = mmap.mmap(fileno, bytes, access=mmap.ACCESS_READ)
                self.__array_data__[n] = ndarray.__new__(
                    ndarray, shape[1:], dtype=dtype, buffer=mm,
                    offset=offset, order=0)
        else:
            # Create buffer and data.
            mm = mmap.mmap(fileno, bytes, access=mmap.ACCESS_READ)
            self.__array_data__ = ndarray.__new__(
                ndarray, shape, dtype=dtype, buffer=mm,
                offset=self._begin, order=0)

        # N-D array interface
        self.__array_interface__ = {'shape': shape,
                                    'typestr': dtype,
                                    'data': self.__array_data__,
                                    'version': 3,
                                    }

    def __getitem__(self, index):
        return self.__array_data__.__getitem__(index)

    def getValue(self):
        """For scalars."""
        return self.__array_data__.item()

    def typecode(self):
        return ['b', 'c', 'h', 'i', 'f', 'd'][self._nc_type-1]


def _test():
    import doctest
    doctest.testmod()

"""NetCDF reader.

Pupynere implements a PUre PYthon NEtcdf REader.
"""

__author__ = "<NAME> <<EMAIL>>"


import struct
import mmap

from numpy import ndarray, zeros, array

ABSENT = '\x00' * 8
ZERO = '\x00' * 4
NC_BYTE = '\x00\x00\x00\x01'
NC_CHAR = '\x00\x00\x00\x02'
NC_SHORT = '\x00\x00\x00\x03'
NC_INT = '\x00\x00\x00\x04'
NC_FLOAT = '\x00\x00\x00\x05'
NC_DOUBLE = '\x00\x00\x00\x06'
NC_DIMENSION = '\x00\x00\x00\n'
NC_VARIABLE = '\x00\x00\x00\x0b'
NC_ATTRIBUTE = '\x00\x00\x00\x0c'


class NetCDFFile(object):
    """A NetCDF file parser."""

    def __init__(self, file):
        self._buffer = open(file, 'rb')
        self._parse()

    def read(self, size=-1):
        """Alias for reading the file buffer."""
        return self._buffer.read(size)

    def _parse(self):
        """Initial parsing of the header."""
        # Check magic bytes.
        assert self.read(3) == 'CDF'

        # Read version byte.
        byte = self.read(1)
        self.version_byte = struct.unpack('>b', byte)[0]

        # Read header info.
        self._numrecs()
        self._dim_array()
        self._gatt_array()
        self._var_array()

    def _numrecs(self):
        """Read number of records."""
        self._nrecs = self._unpack_int()

    def _dim_array(self):
        """Read a dict with dimensions names and sizes."""
        assert self.read(4) in [ZERO, NC_DIMENSION]
        count = self._unpack_int()

        self.dimensions = {}
        self._dims = []
        for dim in range(count):
            name = self._read_string()
            length = self._unpack_int()
            if length == 0:
                length = None  # record dimension
            self.dimensions[name] = length
            self._dims.append(name)  # preserve dim order

    def _gatt_array(self):
        """Read global attributes."""
        self.attributes = self._att_array()

        # Update __dict__ for compatibility with S.IO.N
        self.__dict__.update(self.attributes)

    def _att_array(self):
        """Read a dict with attributes."""
        assert self.read(4) in [ZERO, NC_ATTRIBUTE]
        count = self._unpack_int()

        # Read attributes.
        attributes = {}
        for attribute in range(count):
            name = self._read_string()
            nc_type = self._unpack_int()
            n = self._unpack_int()

            # Read value for attributes.
            attributes[name] = self._read_values(n, nc_type)

        return attributes

    def _var_array(self):
        """Read all variables."""
        assert self.read(4) in [ZERO, NC_VARIABLE]

        # Read size of each record, in bytes.
        self._read_recsize()

        # Read variables.
        self.variables = {}
        count = self._unpack_int()
        for variable in range(count):
            name = self._read_string()
            self.variables[name] = self._read_var()

    def _read_recsize(self):
        """Read all variables and compute record bytes."""
        pos = self._buffer.tell()

        recsize = 0
        count = self._unpack_int()
        for variable in range(count):
            name = self._read_string()
            n = self._unpack_int()
            isrec = False
            for i in range(n):
                dimid = self._unpack_int()
                name = self._dims[dimid]
                dim = self.dimensions[name]
                if dim is None and i == 0:
                    isrec = True
            vsize = self._unpack_int()

            if isrec:
                recsize += vsize

        self._recsize = recsize
        self._buffer.seek(pos)

    def _read_var(self):
        dimensions = []
        shape = []
        n = self._unpack_int()
        isrec = False
        for i in range(n):
            dimid = self._unpack_int()
            name = self._dims[dimid]
            dimensions.append(name)
            dim = self.dimensions[name]
            if dim is None and i == 0:
                dim = self._nrecs
                isrec = True
            shape.append(dim)
        dimensions = tuple(dimensions)
        shape = tuple(shape)

        attributes = self._att_array()
        nc_type = self._unpack_int()
        vsize = self._unpack_int()

        # Read offset.
        begin = [self._unpack_int, self._unpack_int64][self.version_byte-1]()

        return NetCDFVariable(
            self._buffer.fileno(), nc_type, vsize, begin, shape, dimensions,
            attributes, isrec, self._recsize)

    def _read_values(self, n, nc_type):
        bytes = [1, 1, 2, 4, 4, 8]
        typecodes = ['b', 'c', 'h', 'i', 'f', 'd']

        count = n * bytes[nc_type-1]
        values = self.read(count)

        typecode = typecodes[nc_type-1]
        if nc_type != 2:  # not char
            values = struct.unpack('>%s' % (typecode * n), values)
            values = array(values, dtype=typecode)
        else:
            # Remove EOL terminator.
            if values.endswith('\x00'):
                values = values[:-1]

        return values

    def _unpack_int(self):
        return struct.unpack('>i', self.read(4))[0]
    _unpack_int32 = _unpack_int

    def _unpack_int64(self):
        return struct.unpack('>q', self.read(8))[0]

    def _read_string(self):
        count = struct.unpack('>i', self.read(4))[0]
        s = self.read(count)
        # Remove EOL terminator.
        if s.endswith('\x00'):
            s = s[:-1]
        return s

    def close(self):
        self._buffer.close()


class NetCDFVariable(object):
    def __init__(self, fileno, nc_type, vsize, begin, shape, dimensions,
                 attributes, isrec=False, recsize=0):
        self._nc_type = nc_type
        self._vsize = vsize
        self._begin = begin
        self.shape = shape
        self.dimensions = dimensions
        self.attributes = attributes  # for ``dap.plugins.netcdf``
        self.__dict__.update(attributes)
        self._is_record = isrec

        # Number of bytes and type.
        self._bytes = [1, 1, 2, 4, 4, 8][self._nc_type-1]
        type_ = ['i', 'S', 'i', 'i', 'f', 'f'][self._nc_type-1]
        dtype = '>%s%d' % (type_, self._bytes)
        bytes = self._begin + self._vsize

        if isrec:
            # Record variables are not stored contiguosly on disk, so we
            # need to create a separate array for each record.
            self.__array_data__ = zeros(shape, dtype)
            bytes += (shape[0] - 1) * recsize
            for n in range(shape[0]):
                offset = self._begin + (n * recsize)
                mm = mmap.mmap(fileno, bytes, access=mmap.ACCESS_READ)
                self.__array_data__[n] = ndarray.__new__(
                    ndarray, shape[1:], dtype=dtype, buffer=mm,
                    offset=offset, order=0)
        else:
            # Create buffer and data.
            mm = mmap.mmap(fileno, bytes, access=mmap.ACCESS_READ)
            self.__array_data__ = ndarray.__new__(
                ndarray, shape, dtype=dtype, buffer=mm,
                offset=self._begin, order=0)

        # N-D array interface
        self.__array_interface__ = {'shape': shape,
                                    'typestr': dtype,
                                    'data': self.__array_data__,
                                    'version': 3,
                                    }

    def __getitem__(self, index):
        return self.__array_data__.__getitem__(index)

    def getValue(self):
        """For scalars."""
        return self.__array_data__.item()

    def typecode(self):
        return ['b', 'c', 'h', 'i', 'f', 'd'][self._nc_type-1]


def _test():
    import doctest
    doctest.testmod()

NetCDF reader.  Pupynere implements a PUre PYthon NEtcdf REader. A NetCDF file parser. Alias for reading the file buffer. Initial parsing of the header. # Check magic bytes. # Read version byte. # Read header info. Read number of records. Read a dict with dimensions names and sizes. # record dimension # preserve dim order Read global attributes. # Update __dict__ for compatibility with S.IO.N Read a dict with attributes. # Read attributes. # Read value for attributes. Read all variables. # Read size of each record, in bytes. # Read variables. Read all variables and compute record bytes. # Read offset. # not char # Remove EOL terminator. # Remove EOL terminator. # for ``dap.plugins.netcdf`` # Number of bytes and type. # Record variables are not stored contiguosly on disk, so we # need to create a separate array for each record. # Create buffer and data. # N-D array interface For scalars.