def withfile(method):
    """decorator that adds a named temporary file argument"""
    @wraps(method)
    def wrapper(*args, **kw):
        with test_file() as f:
            args = args + (f,)
            return method(*args, **kw)
    return wrapper @contextmanager
def test_dir():
    """context mgr for tempdir"""
    d = tempfile.mkdtemp()
    try:
        yield d
    finally:
        shutil.rmtree(d) def hdf2pd(group):
    """
    Read a ``pandas.DataFrame`` from an ``h5py.Group``.

    :param group: the ``h5py.Group`` to read from.
    """
    if group.attrs['ptype'] != 'DataFrame':
        raise ValueError('unrecognized HDF format')
    index = group['index']
    index_name = index.attrs.get('name',None)
    col_refs = group['columns']
    col_data = [np.array(group[r]) for r in col_refs]
    col_names = col_refs.attrs.get('names')
    if type(col_names[0]) == np.bytes_:
        col_names = [str(cn,'utf8') for cn in col_names]
    data = { k: v for k, v in zip(col_names, col_data) }
    index = pd.Series(index, name=index_name)
    return pd.DataFrame(data=data, index=index, columns=col_names) class hdfopen(object):
    """
    Context manager that opens an ``h5py.Group`` from an ``h5py.File``
    or other group.

    Parameters:

    * path - path to HDF5 file, or open HDF5 group
    * group - for HDF5 file paths, the path of the group to return (optional)
      for groups, a subgroup to require (optional)
    * replace - whether to replace any existing data

    :Example:

    >>> with hdfopen('myfile.h5','somegroup') as g:
    ...     g.attrs['my_attr'] = 3

    """
    def __init__(self, path, group=None, replace=None):
        if isinstance(path, h5.Group):
            if group is not None:
                self.group = path.require_group(group)
            else:
                self.group = path
            if replace:
                clear_h5_group(self.group)
            self._file = None
        else:
            mode = 'w' if replace else 'r+'
            self._file = h5.File(path, mode)
            if group is not None:
                self.group = self._file.require_group(group)
            else:
                self.group = self._file
    def close(self):
        if self._file is not None:
            self._file.close()
    def __enter__(self, *args, **kw):
        return self.group
    def __exit__(self, *args):
        self.close()
        pass class AdcFile(BaseDictlike):
    """
    Represents an IFCB ``.adc`` file.

    Provides a dict-like interface; keys are target numbers,
    values are target records. Target records are tuples which
    are indexed according to values in the associated schema
    (accessible via the ``schema`` property).

    :Example:

    >>> adc = AdcFile('IFCB4_2013_007_123423.adc')
    >>> adc[23][adc.schema.ROI_WIDTH]
    135

    """
    def __init__(self, adc_path, parse=False):
        """
        :param adc_path: the path of the ``.adc`` file.
        :param parse: whether to parse the file
          (if not, parsing is deferred until data is accessed)
        """
        self.path = adc_path
        self.pid = Pid(adc_path, parse=parse)
        self.schema_version = self.pid.schema_version
        self.schema = SCHEMA[self.schema_version]
        if parse:
            self.csv
    def getsize(self):
        """
        :return int: the size of the file
        """
        return os.path.getsize(self.path)
    @property
    def lid(self):
        """
        The bin's LID
        """
        return self.pid.lid
    @property
    @lru_cache()
    def csv(self):
        """
        The underlying CSV data as a ``pandas.DataFrame``
        """
        return parse_adc_file(self.path)
    def to_dataframe(self):
        """
        Return the ADC data as a ``pandas.DataFrame``. If the
        file has not been parsed yet, this opens the file, parses it,
        and closes it.
        """
        return self.csv
    def to_dict(self):
        """
        Return the ADC data as a dictionary
        """
        return self.csv.to_dict('series')
    def to_hdf(self, hdf_file, group=None, replace=True, **kw):
        """
        Write the ADC file to an HDF file or group.

        :param hdf_file: the root HDF file pathname or
          object (h5py.File or h5py.Group) in which to write
          the ADC data
        :param group: a path below the sub-group
          to use
        :param replace: whether to replace any existing data
          at that location in the HDF file
        """
        from .hdf import adc2hdf
        adc2hdf(self, hdf_file, group, replace=replace, **kw)
    def keys(self):
        """
        Yield the target numbers of this ADC file, in order.
        """
        yield from self.csv.index
    def has_key(self, k):
        return k in self.csv.index
    def __len__(self):
        return len(self.csv)
    def get_target(self, target_number):
        """
        Get the ADC data for a given target, as a tuple.
        """
        d = tuple(self.csv[c][target_number] for c in self.csv.columns)
        return d
    def __getitem__(self, target_number):
        return self.get_target(target_number)
    def __repr__(self):
        return '<ADC file %s>' % self.path
    def __str__(self):
        return self.path class RoiFile(BaseDictlike):
    """
    Wraps and provides access to an IFCB ``.roi`` file.
    Provides context manager support for keeping the
    file open while images are read from it. Also
    provides dict-like support including len, iteration,
    and access to images by target number.

    Requires an associated ``.adc`` file or ``AdcFile`` object.
    """
    def __init__(self, adc, roi_path):
        """
        :param adc: the path of the ``.adc`` file, or an ``AdcFile`` object
        :param roi_path: the path to the ``.roi`` file
        """
        # duck type adc argument
        self.adc = None
        try:
            adc.pid # should work for AdcFile objects
            self.adc = adc
        except AttributeError:
            self.adc = AdcFile(adc)
        self.path = roi_path
        self._inroi = None # start with the file closed
    @property
    @lru_cache()
    def csv(self):
        """adc data with non-ROI targets removed"""
        # remove 0x0 rois from adc data
        csv = self.adc.csv
        s = self.adc.schema
        return csv[csv[s.ROI_WIDTH] != 0]
    @property
    def lid(self):
        """
        The bin's LID
        """
        return self.adc.lid
    def getsize(self):
        """
        :returns int: the size of the file in bytes
        """
        return os.path.getsize(self.path)
    def isopen(self):
        """
        Flag indicating if the file is open
        """
        return self._inroi is not None
    def _open(self):
        if self.isopen():
            raise ValueError('RoiFile already open')
        self._inroi = open(self.path, 'rb')
    def close(self):
        """
        Close the file.

        It is OK to call this even if the file is closed.
        """
        # allow re-closing
        if self.isopen():
            self._inroi.close()
        self._inroi = None
    def __enter__(self):
        self._open()
        return self
    def __exit__(self, *args):
        self.close()
    def shape(self, roi_number):
        roi_number = int(roi_number)
        s = self.adc.schema
        width = self.csv[s.ROI_WIDTH][roi_number]
        height = self.csv[s.ROI_HEIGHT][roi_number]
        return (height, width)
    def get_image(self, roi_number):
        """
        Read an image from the file. Note that the dict-like
        interface can be used to access images by target number.

        :param roi_number: the (1-based) target number for this ROI
        :returns numpy.array: an 8-bit 2d image
        """
        roi_number = int(roi_number)
        s = self.adc.schema
        keys = [s.START_BYTE, s.ROI_WIDTH, s.ROI_HEIGHT]
        try:
            bo, width, height = [self.csv[k][roi_number] for k in keys]
        except KeyError:
            raise KeyError('adc data does not contain a roi #%d' % roi_number)
        if width * height == 0:
            raise KeyError('roi #%d is 0x0' % roi_number)
        if not self.isopen():
            self._open()
            try:
                im = read_image(self._inroi, bo, height, width)
            except:
                raise
            finally:
                self.close()
        else:
            im = read_image(self._inroi, bo, height, width)
        return im
    def __len__(self):
        return len(self.csv)
    @property
    def index(self):
        """
        An array-like object containing the target number
          of each ROI in the file, in order
        """
        return self.csv.index
    def keys(self):
        return self.index
    def has_key(self, k):
        return k in self.keys()
    def __getitem__(self, roi_number):
        return self.get_image(roi_number)
    def to_dict(self):
        """
        Convert item pairs to a dict.

        Warning: holds all image data in memory, so for a typical
        IFCB data file will consume large amounts of memory.

        An alternative is to use RoiFile's dict-like interface, as
        long as you can keep the .roi file open.

        :returns dict: a dict with target numbers as keys and
          images as values.
        """
        return dict(self.items())
    def to_hdf(self, hdf_file, group=None, replace=True):
        """
        Convert the image data to HDF5.

        :param hdf_file: the root HDF
          object (h5py.File or h5py.Group) in which to write
          the image data and index
        :param group: a path below the sub-group
          to use
        :param replace: whether to replace any existing data
          at that location in the HDF file
        """
        from .hdf import roi2hdf
        roi2hdf(self, hdf_file, group, replace=replace)
    def __repr__(self):
        return '<ROI file %s>' % self.path
    def __str__(self):
        return self.path def parse_hdr_file(path):
    """
    Given a path to a header file, return the header properties.

    :param path: a pathname
    :returns dict: the header properties
    :see parse_hdr
    """
    lines = list(fileinput.FileInput(path))
    return parse_hdr(lines) def roi2hdf(roifile, hdf_file, group=None, replace=True):
    """
    Store a ``RoiFile`` in an HDF file or group. ROI
    data is represented as follows in an HDF group:

    * ``{root}.index`` (attribute): target number for each image
    * ``{root}/images`` (dataset): references to images keyed by target number
    * ``{root}/{n}`` (dataset): 2d uint8 image (n = ``str(target_number)``)

    :param roifile: the ``RoiFile`` to store (or similar dictlike)
    :type roifile: RoiFile
    :param hdf_file: the root HDF
      object (h5py.File or h5py.Group) in which to write
      the image data and index
    :param group: a path below the sub-group
      to use
    :param replace: whether to replace any existing data
      at that location in the HDF file
    """
    with hdfopen(hdf_file, group, replace=replace) as root:
        root.attrs['index'] = roifile.keys()
        # create image datasets and map them to roi numbers
        d = { n: root.create_dataset(str(n), data=im) for n, im in roifile.items() }
        # now create sparse array of references keyed by roi number
        n = max(d.keys())+1
        r = [ d[i].ref if i in d else None for i in range(n) ]
        root.create_dataset('images', data=r, dtype=H5_REF_TYPE) def hdr2hdf(hdr_dict, hdf_file, group=None, replace=True):
    """
    Store a header dict in an HDF file or group. Header data
    is represented in HDF as a set of attributes on the group.

    :param hdr_dict: the headers
    :type hdr_dict: dict
    :param hdf_file: the root HDF
      object (h5py.File or h5py.Group) on which to write
      the header attributes
    :param group: a path below the sub-group
      to use
    :param replace: whether to replace any existing data
      at that location in the HDF file
    """
    with hdfopen(hdf_file, group, replace=replace) as root:
        for k, v in hdr_dict.items():
            root.attrs[k] = v def adc2hdf(adcfile, hdf_file, group=None, replace=True):
    """
    Store an ``AdcFile`` in an HDF file or group. ADC
    data is represented as a ``pandas.DataFrame`` with
    a ``schema`` attribute specifying the name of the
    ADC schema.

    :param adcfile: the ``AdcFile`` to store
    :type adcfile: AdcFile
    :param hdf_file: the root HDF
      object (h5py.File or h5py.Group) in which to write
      the ADC data
    :param group: a path below the sub-group
      to use
    :param replace: whether to replace any existing data
      at that location in the HDF file
    """
    with hdfopen(hdf_file, group, replace=replace) as root:
        pd2hdf(root, adcfile.to_dataframe(), compression='gzip')
        root.attrs['schema'] = adcfile.schema._name def fileset2hdf(fileset, hdf_file, group=None, replace=True, archive=False):
    """
    Write a fileset to HDF.

    :see filesetbin2hdf
    """
    with FilesetBin(fileset) as fs_bin:
        filesetbin2hdf(fs_bin, hdf_file, group=group, replace=replace, archive=archive) def hdf2fileset(hdf_path, fileset_path, group=None):
    """
    Unarchive an archived IFCB fileset from an HDF file. Creates
    ``.hdr``, ``.adc``, and ``.roi`` files exactly matching the
    original raw data. This is the inverse operation of
    ``fileset2hdf`` if it was called with ``archive`` set to True.

    :param hdf_path: the path to the HDF file
    :param fileset_path: base path for output files
    :param group: (optional) the path to the HDF group containing
      the archived IFCB data
    """
    with hdfopen(hdf_path, group) as root:
        if not 'archive' in root:
            raise ValueError('no archived IFCB data found')
        hdf2file(root['archive/adc'], fileset_path + '.adc')
        hdf2file(root['archive/hdr'], fileset_path + '.hdr')
        with open(fileset_path + '.roi', 'wb') as outroi:
            schema1 = root['adc'].attrs['schema'] == SCHEMA[1]._name
            if schema1:
                outroi.write("\0")
            imref = root['roi/images']
            for i in root['roi'].attrs['index']:
                image = root[imref[i]]
                outroi.write(np.array(image).ravel())
            if schema1:
                outroi.write("\0") class HdfBin(BaseBin):
    """
    Bin interface to HDF file/group.

    Context manager implementation opens and closes the HDF file.
    This implementation is caching, so if the HDF file changes during
    an instance's lifecycle, those changes may not be reflected in
    subsequent accesses.
    """
    def __init__(self, hdf_file, group=None):
        """
        :param hdf_file: HDF file path or open ``h5py.Group`` containing bin data
        :param group: (optional) path in HDF file/group containing bin data
        """
        # open the file or group
        self._open_params = (hdf_file, group)
        self._hdf = None
        self._open()
    # context manager implementation
    def isopen(self):
        """
        :returns bool: if HDF file is open
        """
        return self._hdf is not None
    def _open(self):
        if self.isopen():
            raise ValueError('HdfBin already open')
        self._hdf = hdfopen(*self._open_params)
        self._group = self._hdf.group
    def close(self):
        """
        Close the HDF file. Will fail if HDF file is already
        closed.
        """
        if not self.isopen():
            raise ValueError('HdfBin is already closed')
        self._hdf.close()
        self._hdf = None
    def __enter__(self):
        return self
    def __exit__(self, *args):
        if self.isopen():
            self.close()
    # Dictlike
    @property
    @lru_cache()
    def adc(self):
        """
        adc(self)
        The bin's ADC data as a ``pandas.DataFrame``
        """
        return hdf2pd(self._group['adc'])
    @property
    @lru_cache()
    def schema(self):
        """
        The bin's schema
        """
        return SCHEMA[self._group['adc'].attrs['schema']]
    @property
    @lru_cache()
    def headers(self):
        """
        The bin's headers
        """
        return dict(self._group['hdr'].attrs)
    @property
    @lru_cache()
    def pid(self):
        """
        The bin's ``Pid``
        """
        return Pid(self._group.attrs['pid'])
    @property
    def images(self):
        """
        The bin's images
        """
        return HdfRoi(self._group['roi']) def filesetbin2hdf(fs_bin, hdf_file, group=None, replace=True, archive=False):
    """
    Write a ``FilesetBin`` to an HDF file.

    A ``FilesetBin`` is represented in HDF as in ``bin2hdf``, with
    the following additional HDF objects:

    * ``{root}/archive`` (group) - optional: archived files
    * ``{root}/archive/adc`` (dataset) - archived ``.adc`` file
    * ``{root}/archive/hdr`` (dataset) - archived ``.hdr`` file

    :param fs_bin: the ``FilesetBin`` to write
    :type fs_bin: FilesetBin
    :param hdf_file: the root HDF file pathname or
      object (h5py.File or h5py.Group) on which to write
      the IFCB data
    :param group: a path below the sub-group
      to use
    :param replace: whether to replace any existing data
      at that location in the HDF file
    :param archive: whether to store copies of the ``.adc`` and ``.hdr``
      files in the HDF file
    """
    with hdfopen(hdf_file, group, replace=replace) as root:
        bin2hdf(fs_bin, root)
        if archive:
            file2hdf(root, 'archive/adc', fs_bin.fileset.adc_path, compression='gzip')
            file2hdf(root, 'archive/hdr', fs_bin.fileset.hdr_path) def bin2hdf(b, hdf_file, group=None, replace=True):
    """
    Write a ``Bin`` to an HDF file.

    A ``Bin`` is represented in HDF relative to some root HDF path as:

    * ``{root}.pid`` (attribute) - full base pathname
    * ``{root}.lid`` (attribute) - bin LID
    * ``{root}.timestamp`` (attribute) - bin timestamp in ISO8601 UTC format
    * ``{root}/hdr`` (group) - see hdr2hdf
    * ``{root}/adc`` (group) - see adc2hdf
    * ``{root}/roi`` (group) - see roi2hdf

    :param fs_bin: the ``FilesetBin`` to write
    :type fs_bin: FilesetBin
    :param hdf_file: the root HDF file pathname or
      object (h5py.File or h5py.Group) on which to write
      the IFCB data
    :param group: a path below the sub-group
      to use
    :param replace: whether to replace any existing data
      at that location in the HDF file
    """
    with hdfopen(hdf_file, group, replace=replace) as root:
        root.attrs['pid'] = str(b.pid)
        root.attrs['lid'] = b.lid
        root.attrs['timestamp'] = b.timestamp.isoformat()
        hdr2hdf(b.headers, root, 'hdr', replace=replace)
        with hdfopen(root, 'adc') as adc:
            pd2hdf(adc, b.adc, compression='gzip')
            adc.attrs['schema'] = b.schema._name
        roi2hdf(b.images, root, 'roi', replace=replace) class FilesetBin(BaseBin):
    """
    Bin interface to Fileset.

    Context manager support opens and closes the ``.roi`` file for image
    access.
    """
    def __init__(self, fileset):
        """
        :param fileset: the ``Fileset`` to represent
        """
        self.fileset = fileset
        self.adc_file = AdcFile(fileset.adc_path)
        self.roi_file = RoiFile(self.adc_file, fileset.roi_path)
    # oo interface to fileset
    @property
    @lru_cache()
    def hdr_attributes(self):
        """
        A ``dict`` representing the headers
        """
        return parse_hdr_file(self.fileset.hdr_path)
    @property
    def timestamp(self):
        """
        The bin's timestamp (as a ``datetime``)
        """
        return self.pid.timestamp
    def to_hdf(self, hdf_file, group=None, replace=True, archive=False):
        """
        Convert the fileset to HDF.

        :param hdf_file: the root HDF file pathname or
          object (``h5py.File`` or ``h5py.Group``) in which to write all raw data
        :param group: a path below the sub-group
          to use
        :param replace: whether to replace any existing data
          at that location in the HDF file
        :param archive: whether to include the full text of the .hdr
          and .roi files
        """
        from .hdf import filesetbin2hdf
        filesetbin2hdf(self, hdf_file, group=group, replace=replace, archive=archive)
    # bin interface
    @property
    def pid(self):
        """
        The bin's PID
        """
        return self.fileset.pid
    @property
    def schema(self):
        """
        The bin's schema
        """
        return self.adc_file.schema
    @property
    def images(self):
        """
        The images
        """
        return self.roi_file
    @property
    def headers(self):
        """
        The header dict
        """
        return self.hdr_attributes
    def header(self, key):
        ci_dict = CaseInsensitiveDict(self.hdr_attributes)
        return ci_dict[key]
    @property
    def adc(self):
        """
        The bin's ADC data as a ``pandas.DataFrame``
        """
        return self.adc_file.csv
    # context manager implementation
    def isopen(self):
        """
        Is the ``.roi`` file open?
        """
        return self.roi_file.isopen()
    def close(self):
        """
        Close the ``.roi`` file, if it is open.
        """
        if self.isopen():
            self.roi_file.close()
    def __enter__(self):
        if not self.isopen():
            self.roi_file._open()
        return self
    def __exit__(self, *args):
        self.close()
    # support for single image reading
    def as_single(self, target):
        """Return a new FilesetBin that only provides access to
        a single target. If called immediately upon construction
        (before accessing any data) this will avoid parsing the
        entire ADC file. Otherwise it will raise ValueError."""
        if self.isopen():
            raise ValueError('as_single must be called before opening FilesetBin')
        return FilesetFragmentBin(self.fileset, target)
    def __repr__(self):
        return '<FilesetBin %s>' % self
    def __str__(self):
        return self.fileset.__str__() def list_test_filesets():
    return [b.fileset for b in list_test_bins()] def list_test_bins():
    return list(_dd()) def assert_bin_equals(in_bin, out_bin):
    # test keys
    assert set(in_bin) == set(out_bin)
    for k in in_bin.keys():
        assert k in in_bin, 'inconsistent dict behavior'
        assert k in out_bin, 'key membership does not match'
    assert 0 not in in_bin, 'zero-based index'
    assert 0 not in out_bin, 'zero-based index'
    # test ADC data
    for k in in_bin.keys():
        for a, b in zip(in_bin[k], out_bin[k]):
            assert np.isclose(a, b, equal_nan=True)
    # pid
    assert in_bin.pid == out_bin.pid, 'pid mismatch'
    assert in_bin.lid == out_bin.lid, 'lid mismatch'
    # timestamp
    assert in_bin.timestamp == out_bin.timestamp, 'timestamp mismatch'
    # schema
    assert in_bin.schema == out_bin.schema, 'schema mismatch'
    # headers
    assert set(in_bin.headers) == set(out_bin.headers), 'header keys mismatch'
    for k in in_bin.headers.keys():
        # headers should all have same names and values
        assert in_bin.headers[k] == out_bin.headers[k], 'header values mismatch'
    # images
    assert set(in_bin.images) == set(out_bin.images), 'image target numbers mismatch'
    for k in in_bin.images:
        assert np.all(in_bin.images[k] == out_bin.images[k]), 'image data mismatch'
import os
import unittest
import numpy as np
import h5py as h5
from pandas.testing import assert_frame_equal
from ifcb.tests.utils import withfile, test_dir
from ifcb.data.h5utils import hdf2pd, hdfopen
from ifcb.data.adc import AdcFile
from ifcb.data.roi import RoiFile
from ifcb.data.hdr import parse_hdr_file
from ifcb.data.hdf import roi2hdf, hdr2hdf, adc2hdf, fileset2hdf, hdf2fileset, HdfBin, filesetbin2hdf, bin2hdf
from ifcb.data.files import FilesetBin
from .fileset_info import list_test_filesets, list_test_bins
from .bins import assert_bin_equals





def test_adc_roundtrip(adc, path, group=None):
    with hdfopen(path, group) as h:
        csv = hdf2pd(h)
        assert h.attrs['schema'] == adc.schema._name
    assert_frame_equal(csv, adc.csv)

def test_hdr_roundtrip(hdr, path, group=None):
    with hdfopen(path, group) as h:
        for k,v in hdr.items():
            assert np.all(h.attrs[k] == hdr[k])

def test_roi_roundtrip(roi, path, group=None):
    with hdfopen(path, group) as h:
        index = h.attrs['index']
        assert np.all(index == roi.keys())
        for roi_number in index:
            assert np.all(roi[roi_number] == h[str(roi_number)])
        for roi_number in index:
            assert np.all(roi[roi_number] == h[h['images'][roi_number]])
    
class TestAdcHdf(unittest.TestCase):
    @withfile
    def test_roundtrip(self, path):
        for fs in list_test_filesets():
            adc = AdcFile(fs.adc_path)
            adc2hdf(adc, path, replace=True)
            test_adc_roundtrip(adc, path)

class TestRoiHdf(unittest.TestCase):
    @withfile
    def test_roundtrip(self, path):
        for fs in list_test_filesets():
            with RoiFile(fs.adc_path, fs.roi_path) as roi:
                roi2hdf(roi, path)
                test_roi_roundtrip(roi, path)

class TestHdrHdf(unittest.TestCase):
    @withfile
    def test_roundtrip(self, path):

        for fs in list_test_filesets():