<reponame>cgohlke/fcsfiles
# fcsfiles.py

# Copyright (c) 2012-2021, <NAME>
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice,
#    this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright notice,
#    this list of conditions and the following disclaimer in the documentation
#    and/or other materials provided with the distribution.
#
# 3. Neither the name of the copyright holder nor the names of its
#    contributors may be used to endorse or promote products derived from
#    this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.

"""Read fluorescence correlation spectroscopy (FCS) data files.

Fcsfiles is a Python library to read Carl Zeiss(r) ConfoCor(r) RAW and ASCII
measurement data files.

:Author:
  `<NAME> <https://www.lfd.uci.edu/~gohlke/>`_

:Organization:
  Laboratory for Fluorescence Dynamics. University of California, Irvine

:License: BSD 3-Clause

:Version: 2021.6.6

Requirements
------------
* `CPython >= 3.7 <https://www.python.org>`_
* `Numpy 1.15.1 <https://www.numpy.org>`_

Revisions
---------
2021.6.6
    Remove support for Python 3.6 (NEP 29).
2020.9.18
    Relax ConfoCor3Raw header requirement.
    Support os.PathLike file names.
2020.1.1
    Remove support for Python 2.7 and 3.5.

Notes
-----
"<NAME>" and "ConfoCor" are registered trademarks of <NAME>, Inc.

The use of this implementation may be subject to patent or license
restrictions.

The API is not stable yet and is expected to change between revisions.

This module does *not* read flow cytometry standard FCS files.

Examples
--------
Read the CountRateArray from a ConfoCor3 ASCII file as a numpy array:

>>> fcs = ConfoCor3Fcs('ConfoCor3.fcs')
>>> fcs['FcsData']['FcsEntry'][0]['FcsDataSet']['CountRateArray'].shape
(60000, 2)

Read data and metadata from a ConfoCor3 RAW file:

>>> fcs = ConfoCor3Raw('ConfoCor3.raw')
>>> fcs.filename()
'f5ee4f36488fca2f89cb6b8626111006_R1_P1_K1_Ch1.raw'
>>> fcs.frequency
20000000
>>> times = fcs.asarray()
>>> times[10858]
1199925494
>>> times, bincounts = fcs.asarray(bins=1000)
>>> times.shape
(1000,)
>>> bincounts[618]
23
>>> fcs.close()

Read data and metadata from a ConfoCor2 RAW file:

>>> fcs = ConfoCor2Raw('ConfoCor2.raw')
>>> fcs.frequency
20000000
>>> ch0, ch1 = fcs.asarray()
>>> ch1[4812432]
999999833
>>> times, ch0, ch1 = fcs.asarray(bins=1000)
>>> times.shape
(1000,)
>>> ch1[428]
10095
>>> fcs.close()

"""

__version__ = '2021.6.6'

__all__ = ('ConfoCor3Fcs', 'ConfoCor3Raw', 'ConfoCor2Raw', 'fcs_bincount')

import os
import struct

import numpy


class ConfoCor3Fcs(dict):
    """Carl Zeiss ConfoCor3 ASCII data file.

    No specification is available. The encoding is 'Windows-1252'.

    """

    HEADER = 'Carl Zeiss ConfoCor3 - measurement data file - version 3.0 ANSI'

    def __init__(self, filename):
        """Read file content and parse into dictionary."""
        dict.__init__(self)
        filename = os.path.abspath(os.fspath(filename))
        self._filepath, self._filename = os.path.split(filename)
        with open(filename) as fh:
            # encoding = 'Windows-1252'
            header = fh.read(63)
            if header != ConfoCor3Fcs.HEADER:
                raise ValueError('not a ConfoCor3 measurement data file')
            fh.readline()
            current = self
            stack = []
            array = None
            key = None
            for line in fh:
                line = line.lstrip()
                if line[0].isdigit():
                    if array:
                        array.append(line)
                    else:
                        array = [line]
                    continue
                if array:
                    shape = tuple(int(s) for s in current[key].split())
                    array = ''.join(array)
                    array = numpy.fromstring(array, dtype='float32', sep=' ')
                    array = array.reshape(*shape)
                    current[key] = array
                    array = None
                elif array == '':
                    # work around https://github.com/numpy/numpy/issues/1714
                    shape = tuple(int(s) for s in current[key].split())
                    current[key] = numpy.zeros(shape)
                    array = None
                if line.startswith('BEGIN'):
                    stack.append(current)
                    _, key, val = line.split()
                    key = key.strip()
                    val = {'_value': int(val.strip())}
                    if key[-1].isdigit():
                        key = key[:-1]
                        if key[-1].isdigit():
                            key = key[:-1]
                        if key in current:
                            current[key].append(val)
                        else:
                            current[key] = [val]
                        current = current[key][-1]
                    else:
                        current[key] = val
                        current = current[key]
                elif line.startswith('END'):
                    current = stack.pop()
                elif '=' in line:
                    key, value = line.split('=', 1)
                    key = key.strip()
                    if key.endswith('ArraySize'):
                        continue
                    if key.endswith('Array'):
                        array = ''
                    value = value.strip()
                    if value:
                        for type_t in (int, float, str):
                            try:
                                value = type_t(value)
                                break
                            except (ValueError, TypeError):
                                pass
                    if key[-1].isdigit():
                        key = key[:-1]
                        if key[-1].isdigit():
                            key = key[:-1]
                        if key in current:
                            current[key].append(value)
                        else:
                            current[key] = [value]
                    else:
                        current[key] = value

    def __str__(self):
        """Return string close to original file format."""
        result = [ConfoCor3Fcs.HEADER]

        def append(key, value, indent='', index=''):
            """Recursively append formatted keys and values to result."""
            if index != '':
                index = str(index + 1)
            if isinstance(value, dict):
                result.append(f'{indent}BEGIN {key}{index} {value["_value"]}')
                for k, v in sorted(value.items(), key=sortkey):
                    append(k, v, indent + '\t')
                result.append(f'{indent}END')
            elif isinstance(value, (list, tuple)):
                for i, val in enumerate(value):
                    append(key, val, indent, i)
            elif isinstance(value, numpy.ndarray):
                size = value.shape[0]
                if size != 1:
                    result.append(f'{indent}{key}Size = {size}')
                result.append(
                    '{}{} = {}'.format(
                        indent, key, ' '.join(str(i) for i in value.shape)
                    )
                )
                for i in range(size):
                    result.append(
                        '{}{}'.format(
                            indent, '\t '.join(f'{v:.8f}' for v in value[i])
                        )
                    )
            elif key != '_value':
                result.append(f'{indent}{key}{index} = {value}')

        def sortkey(item):
            """Sort dictionary items by key string and value type."""
            key, value = item
            key = key.lower()
            if isinstance(value, (list, tuple)):
                return '~' + key
            if isinstance(value, numpy.ndarray):
                return '~~' + key
            if isinstance(value, dict):
                return '~~~' + key
            return key

        for key, val in sorted(self.items(), key=sortkey):
            append(key, val)
        return '\n'.join(result)

    def close(self):
        """Close open file."""

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_value, traceback):
        pass


class ConfoCor3Raw:
    """<NAME> ConfoCor3 raw data file.

    Based on "Confocor3 Raw Data Specification. 2 May 2007. by efg, Stowers
    Institute for Medical Research."

    Attributes
    ----------
    measurement_identifier : str
        Hex form of 4 32-bit little endian integers without leading 0.
        It is the same for all channels of the same repetition.
    measurement_position : int
        Zero based measurement position.
    kinetic_index : int
        Zero based kinetic index.
    repetition_number : int
        Zero based repetition_number.
    frequency : int
        Sampling frequency in Hz.

    """

    HEADER = b'<NAME> ConfoCor3 - raw data file - version 3.000 - '

    def __init__(self, filename):
        """Read file header."""
        filename = os.path.abspath(filename)
        self._filepath, self._filename = os.path.split(filename)
        self._fh = open(filename, 'rb')
        header = self._fh.read(64)
        if not header.startswith(ConfoCor3Raw.HEADER):
            self._fh.close()
            raise ValueError('not a ConfoCor3 raw data file')
        self.file_identifier = header
        self.channel = int(header.strip().rsplit(b' ', 1)[-1]) - 1
        (
            measureid,
            self.measurement_position,
            self.kinetic_index,
            self.repetition_number,
            self.frequency,
            _,
        ) = struct.unpack('<16sIIII32s', self._fh.read(64))
        measureid = struct.unpack('<IIII', measureid)
        measureid = ''.join(hex(int(i))[2:] for i in measureid)
        self.measurement_identifier = measureid.replace('L', '')

    def filename(self):
        """Return normalized file name from file content."""
        return '{}_R{}_P{}_K{}_Ch{}.raw'.format(
            self.measurement_identifier,
            self.repetition_number + 1,
            self.measurement_position + 1,
            self.kinetic_index + 1,
            self.channel + 1,
        )

    def asarray(self, count=-1, skip=0, **kwargs):
        """Read data from file, perform optional binning, and return as array.

        Parameters
        ----------
        count : int (optional)
            Number of data words to process. Default: -1 (all words).
        skip : int (optional)
            Number of data words to skip at beginning of stream. Default: 0.
        kwargs : dict
            Optional argument to the 'fcs_bincount' function, specifying
            size or number of bins.

        Returns
        -------
        times : ndarray
            The lower range of the bins in units of seconds, or, if 'binsize'
            is 0, the times of the events in detector clock units.
        bincounts : ndarray
            The number of events in each bin.

        """
        self._fh.seek(128 + skip * 4)
        times = numpy.fromfile(self._fh, dtype='<u4', count=count)
        times = times.astype('u8')
        times = numpy.cumsum(times, out=times)
        if kwargs:
            result = fcs_bincount((times,), self.frequency, **kwargs)
            return result[0], result[1][0]
        return times

    def __str__(self):
        """Return string with information about ConfoCor3Raw."""
        return '\n '.join(
            (
                self.__class__.__name__,
                os.path.normpath(os.path.normcase(self.filename())),
                f'measurement identifier: {self.measurement_identifier}',
                f'sampling frequency: {self.frequency} Hz',
                f'repetition number {self.repetition_number}',
                f'measurement position: {self.measurement_position}',
                f'kinetic index: {self.kinetic_index}',
                f'channel number: {self.channel}',
            )
        )

    def close(self):
        """Close open file."""
        self._fh.close()

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_value, traceback):
        self.close()


class ConfoCor2Raw:
    """Carl Zeiss ConfoCor2 raw data file.

    The LSM 510 META - ConfoCor 2 system: an integrated imaging and
    spectroscopic platform for single-molecule detection.
    <NAME>, <NAME>, and <NAME>.
    Curr Pharm Biotechnol. 2004 Apr;5(2):135-54.

    ConfoCor 2 - Description of the Raw Data Format. 03.05.01.
    www.zeiss.at/C12567BE00472A5C/EmbedTitelIntern/RawData/$File/RawData.pdf

    Carl Zeiss Advanced Imaging Microscopy claims a patent on the file format
    and data compression scheme.

    Attributes
    ----------
    frequency : int
        Sampling frequency in Hz.

    """

    HEADER = b'ConfoCor 2 - Raw data file 1.0'

    def __init__(self, filename):
        """Read file content and parse into a dictionary."""
        filename = os.path.abspath(filename)
        self._filepath, self._filename = os.path.split(filename)
        self._fh = open(filename, 'rb')
        header = self._fh.read(30)
        if not header.startswith(ConfoCor2Raw.HEADER):
            self._fh.close()
            raise ValueError('not a ConfoCor 2 raw data file')
        self.file_identifier = header
        self.channels = 2
        self.frequency = 20000000

    def asarray(self, count=-1, skip=0, **kwargs):
        """Read data from file, perform optional binning, and return as arrays.

        Parameters
        ----------
        count : int (optional)
            Number of data words to process. Default: -1 (all words).
        skip : int (optional)
            Number of data words to skip at beginning of stream. Default: 0.
        kwargs : dict
            Optional argument to the 'fcs_bincount' function, specifying
            size or number of bins.

        Returns
        -------
        times : ndarray
            If binning, the lower range of the bins in units of seconds.
            Otherwise (default), the times of the events in detector clock
            units for each channel.
        Bincounts : ndarray
            If binning, the number of events in each bin for each channel.

        """
        self._fh.seek(30 + skip * 2)
        data = numpy.fromfile(self._fh, dtype='u1', count=count * 2)
        # accumulate clock time
        times = numpy.empty((len(data) // 2, 4), dtype='u8')
        times[:, 0] = data[::2]
        times[:, 1:] = 1
        times = times.flatten()
        numpy.cumsum(times, out=times)
        # extract events for each channel
        data = numpy.repeat(data[1::2], 8).reshape(-1, 8)
        data &= numpy.array([1, 2, 4, 8, 16, 32, 64, 128], dtype='u1')
        ch0 = numpy.take(times, numpy.where(data[:, 0::2].flatten() != 0)[0])
        ch1 = numpy.take(times, numpy.where(data[:, 1::2].flatten() != 0)[0])
        del data
        del times
        if kwargs:
            result = fcs_bincount((ch0, ch1), self.frequency, **kwargs)
            return result[0], result[1][0], result[1][1]
        return ch0, ch1

    def __str__(self):
        """Return string with information about ConfoCor2Raw."""
        return '\n '.join(
            (
                self.__class__.__name__,
                os.path.normpath(os.path.normcase(self.filename)),
                f'sampling frequency: {self.frequency} Hz',
            )
        )

    def close(self):
        """Close open file."""
        self._fh.close()

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_value, traceback):
        self.close()


def fcs_bincount(data, frequency, binsize=None, bins=None, binspm=None):
    """Count number of events in bins.

    Parameters
    ----------
    data : tuple of ndarray
        For each channel, the times of events in detector clock units.
    frequency : int
        Sampling frequency in Hz.
    binsize : int
        Time interval, as a multiple of the sampling frequency, in which
        to bin events.
    bins : int
        If 'binsize' is None, the number of bins.
    binspm : int
        If 'binsize' and 'bins' are None, the number of bins per minute (60s).

    Returns
    -------
    times : ndarray
        The lower range of the bins in units of seconds.
    bincounts : tuple of ndarray
        For each channel, the number of events in each bin.

    """
    assert type(data) in (tuple, list)
    if binsize is None:
        if bins is None:
            if binspm is None:
                raise ValueError('missing parameter binsize, bins, or binspm')
            binsize = (60 * frequency) // binspm
        else:
            size = int(max((ch[-1] if ch.size else 0) for ch in data))
            binsize = size // bins + 1
    for ch in data:
        ch //= int(binsize)  # ch is ndarray
    size = int(max((ch[-1] if ch.size else 0) for ch in data) + 1)
    times = numpy.linspace(
        0, size * binsize / float(frequency), size, endpoint=False
    )
    # FIXME: work around https://github.com/numpy/numpy/issues/823
    if size < 2 ** 31:
        # use 32 bit signed int if possible
        data = (ch.astype('i4') for ch in data)
    else:
        # else use a signed view; fails on 32 bit
        data = (ch.view('i8') for ch in data)
    bincounts = tuple(numpy.bincount(ch, minlength=size) for ch in data)
    return times, bincounts


if __name__ == '__main__':
    import doctest

    numpy.set_printoptions(suppress=True, precision=5)
    doctest.testmod()

# fcsfiles.py

# Copyright (c) 2012-2021, <NAME>
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice,
#    this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright notice,
#    this list of conditions and the following disclaimer in the documentation
#    and/or other materials provided with the distribution.
#
# 3. Neither the name of the copyright holder nor the names of its
#    contributors may be used to endorse or promote products derived from
#    this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.

"""Read fluorescence correlation spectroscopy (FCS) data files.

Fcsfiles is a Python library to read Carl Zeiss(r) ConfoCor(r) RAW and ASCII
measurement data files.

:Author:
  `<NAME> <https://www.lfd.uci.edu/~gohlke/>`_

:Organization:
  Laboratory for Fluorescence Dynamics. University of California, Irvine

:License: BSD 3-Clause

:Version: 2021.6.6

Requirements
------------
* `CPython >= 3.7 <https://www.python.org>`_
* `Numpy 1.15.1 <https://www.numpy.org>`_

Revisions
---------
2021.6.6
    Remove support for Python 3.6 (NEP 29).
2020.9.18
    Relax ConfoCor3Raw header requirement.
    Support os.PathLike file names.
2020.1.1
    Remove support for Python 2.7 and 3.5.

Notes
-----
"<NAME>" and "ConfoCor" are registered trademarks of <NAME>, Inc.

The use of this implementation may be subject to patent or license
restrictions.

The API is not stable yet and is expected to change between revisions.

This module does *not* read flow cytometry standard FCS files.

Examples
--------
Read the CountRateArray from a ConfoCor3 ASCII file as a numpy array:

>>> fcs = ConfoCor3Fcs('ConfoCor3.fcs')
>>> fcs['FcsData']['FcsEntry'][0]['FcsDataSet']['CountRateArray'].shape
(60000, 2)

Read data and metadata from a ConfoCor3 RAW file:

>>> fcs = ConfoCor3Raw('ConfoCor3.raw')
>>> fcs.filename()
'f5ee4f36488fca2f89cb6b8626111006_R1_P1_K1_Ch1.raw'
>>> fcs.frequency
20000000
>>> times = fcs.asarray()
>>> times[10858]
1199925494
>>> times, bincounts = fcs.asarray(bins=1000)
>>> times.shape
(1000,)
>>> bincounts[618]
23
>>> fcs.close()

Read data and metadata from a ConfoCor2 RAW file:

>>> fcs = ConfoCor2Raw('ConfoCor2.raw')
>>> fcs.frequency
20000000
>>> ch0, ch1 = fcs.asarray()
>>> ch1[4812432]
999999833
>>> times, ch0, ch1 = fcs.asarray(bins=1000)
>>> times.shape
(1000,)
>>> ch1[428]
10095
>>> fcs.close()

"""

__version__ = '2021.6.6'

__all__ = ('ConfoCor3Fcs', 'ConfoCor3Raw', 'ConfoCor2Raw', 'fcs_bincount')

import os
import struct

import numpy


class ConfoCor3Fcs(dict):
    """Carl Zeiss ConfoCor3 ASCII data file.

    No specification is available. The encoding is 'Windows-1252'.

    """

    HEADER = 'Carl Zeiss ConfoCor3 - measurement data file - version 3.0 ANSI'

    def __init__(self, filename):
        """Read file content and parse into dictionary."""
        dict.__init__(self)
        filename = os.path.abspath(os.fspath(filename))
        self._filepath, self._filename = os.path.split(filename)
        with open(filename) as fh:
            # encoding = 'Windows-1252'
            header = fh.read(63)
            if header != ConfoCor3Fcs.HEADER:
                raise ValueError('not a ConfoCor3 measurement data file')
            fh.readline()
            current = self
            stack = []
            array = None
            key = None
            for line in fh:
                line = line.lstrip()
                if line[0].isdigit():
                    if array:
                        array.append(line)
                    else:
                        array = [line]
                    continue
                if array:
                    shape = tuple(int(s) for s in current[key].split())
                    array = ''.join(array)
                    array = numpy.fromstring(array, dtype='float32', sep=' ')
                    array = array.reshape(*shape)
                    current[key] = array
                    array = None
                elif array == '':
                    # work around https://github.com/numpy/numpy/issues/1714
                    shape = tuple(int(s) for s in current[key].split())
                    current[key] = numpy.zeros(shape)
                    array = None
                if line.startswith('BEGIN'):
                    stack.append(current)
                    _, key, val = line.split()
                    key = key.strip()
                    val = {'_value': int(val.strip())}
                    if key[-1].isdigit():
                        key = key[:-1]
                        if key[-1].isdigit():
                            key = key[:-1]
                        if key in current:
                            current[key].append(val)
                        else:
                            current[key] = [val]
                        current = current[key][-1]
                    else:
                        current[key] = val
                        current = current[key]
                elif line.startswith('END'):
                    current = stack.pop()
                elif '=' in line:
                    key, value = line.split('=', 1)
                    key = key.strip()
                    if key.endswith('ArraySize'):
                        continue
                    if key.endswith('Array'):
                        array = ''
                    value = value.strip()
                    if value:
                        for type_t in (int, float, str):
                            try:
                                value = type_t(value)
                                break
                            except (ValueError, TypeError):
                                pass
                    if key[-1].isdigit():
                        key = key[:-1]
                        if key[-1].isdigit():
                            key = key[:-1]
                        if key in current:
                            current[key].append(value)
                        else:
                            current[key] = [value]
                    else:
                        current[key] = value

    def __str__(self):
        """Return string close to original file format."""
        result = [ConfoCor3Fcs.HEADER]

        def append(key, value, indent='', index=''):
            """Recursively append formatted keys and values to result."""
            if index != '':
                index = str(index + 1)
            if isinstance(value, dict):
                result.append(f'{indent}BEGIN {key}{index} {value["_value"]}')
                for k, v in sorted(value.items(), key=sortkey):
                    append(k, v, indent + '\t')
                result.append(f'{indent}END')
            elif isinstance(value, (list, tuple)):
                for i, val in enumerate(value):
                    append(key, val, indent, i)
            elif isinstance(value, numpy.ndarray):
                size = value.shape[0]
                if size != 1:
                    result.append(f'{indent}{key}Size = {size}')
                result.append(
                    '{}{} = {}'.format(
                        indent, key, ' '.join(str(i) for i in value.shape)
                    )
                )
                for i in range(size):
                    result.append(
                        '{}{}'.format(
                            indent, '\t '.join(f'{v:.8f}' for v in value[i])
                        )
                    )
            elif key != '_value':
                result.append(f'{indent}{key}{index} = {value}')

        def sortkey(item):
            """Sort dictionary items by key string and value type."""
            key, value = item
            key = key.lower()
            if isinstance(value, (list, tuple)):
                return '~' + key
            if isinstance(value, numpy.ndarray):
                return '~~' + key
            if isinstance(value, dict):
                return '~~~' + key
            return key

        for key, val in sorted(self.items(), key=sortkey):
            append(key, val)
        return '\n'.join(result)

    def close(self):
        """Close open file."""

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_value, traceback):
        pass


class ConfoCor3Raw:
    """<NAME> ConfoCor3 raw data file.

    Based on "Confocor3 Raw Data Specification. 2 May 2007. by efg, Stowers
    Institute for Medical Research."

    Attributes
    ----------
    measurement_identifier : str
        Hex form of 4 32-bit little endian integers without leading 0.
        It is the same for all channels of the same repetition.
    measurement_position : int
        Zero based measurement position.
    kinetic_index : int
        Zero based kinetic index.
    repetition_number : int
        Zero based repetition_number.
    frequency : int
        Sampling frequency in Hz.

    """

    HEADER = b'<NAME> ConfoCor3 - raw data file - version 3.000 - '

    def __init__(self, filename):
        """Read file header."""
        filename = os.path.abspath(filename)
        self._filepath, self._filename = os.path.split(filename)
        self._fh = open(filename, 'rb')
        header = self._fh.read(64)
        if not header.startswith(ConfoCor3Raw.HEADER):
            self._fh.close()
            raise ValueError('not a ConfoCor3 raw data file')
        self.file_identifier = header
        self.channel = int(header.strip().rsplit(b' ', 1)[-1]) - 1
        (
            measureid,
            self.measurement_position,
            self.kinetic_index,
            self.repetition_number,
            self.frequency,
            _,
        ) = struct.unpack('<16sIIII32s', self._fh.read(64))
        measureid = struct.unpack('<IIII', measureid)
        measureid = ''.join(hex(int(i))[2:] for i in measureid)
        self.measurement_identifier = measureid.replace('L', '')

    def filename(self):
        """Return normalized file name from file content."""
        return '{}_R{}_P{}_K{}_Ch{}.raw'.format(
            self.measurement_identifier,
            self.repetition_number + 1,
            self.measurement_position + 1,
            self.kinetic_index + 1,
            self.channel + 1,
        )

    def asarray(self, count=-1, skip=0, **kwargs):
        """Read data from file, perform optional binning, and return as array.

        Parameters
        ----------
        count : int (optional)
            Number of data words to process. Default: -1 (all words).
        skip : int (optional)
            Number of data words to skip at beginning of stream. Default: 0.
        kwargs : dict
            Optional argument to the 'fcs_bincount' function, specifying
            size or number of bins.

        Returns
        -------
        times : ndarray
            The lower range of the bins in units of seconds, or, if 'binsize'
            is 0, the times of the events in detector clock units.
        bincounts : ndarray
            The number of events in each bin.

        """
        self._fh.seek(128 + skip * 4)
        times = numpy.fromfile(self._fh, dtype='<u4', count=count)
        times = times.astype('u8')
        times = numpy.cumsum(times, out=times)
        if kwargs:
            result = fcs_bincount((times,), self.frequency, **kwargs)
            return result[0], result[1][0]
        return times

    def __str__(self):
        """Return string with information about ConfoCor3Raw."""
        return '\n '.join(
            (
                self.__class__.__name__,
                os.path.normpath(os.path.normcase(self.filename())),
                f'measurement identifier: {self.measurement_identifier}',
                f'sampling frequency: {self.frequency} Hz',
                f'repetition number {self.repetition_number}',
                f'measurement position: {self.measurement_position}',
                f'kinetic index: {self.kinetic_index}',
                f'channel number: {self.channel}',
            )
        )

    def close(self):
        """Close open file."""
        self._fh.close()

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_value, traceback):
        self.close()


class ConfoCor2Raw:
    """Carl Zeiss ConfoCor2 raw data file.

    The LSM 510 META - ConfoCor 2 system: an integrated imaging and
    spectroscopic platform for single-molecule detection.
    <NAME>, <NAME>, and <NAME>.
    Curr Pharm Biotechnol. 2004 Apr;5(2):135-54.

    ConfoCor 2 - Description of the Raw Data Format. 03.05.01.
    www.zeiss.at/C12567BE00472A5C/EmbedTitelIntern/RawData/$File/RawData.pdf

    Carl Zeiss Advanced Imaging Microscopy claims a patent on the file format
    and data compression scheme.

    Attributes
    ----------
    frequency : int
        Sampling frequency in Hz.

    """

    HEADER = b'ConfoCor 2 - Raw data file 1.0'

    def __init__(self, filename):
        """Read file content and parse into a dictionary."""
        filename = os.path.abspath(filename)
        self._filepath, self._filename = os.path.split(filename)
        self._fh = open(filename, 'rb')
        header = self._fh.read(30)
        if not header.startswith(ConfoCor2Raw.HEADER):
            self._fh.close()
            raise ValueError('not a ConfoCor 2 raw data file')
        self.file_identifier = header
        self.channels = 2
        self.frequency = 20000000

    def asarray(self, count=-1, skip=0, **kwargs):
        """Read data from file, perform optional binning, and return as arrays.

        Parameters
        ----------
        count : int (optional)
            Number of data words to process. Default: -1 (all words).
        skip : int (optional)
            Number of data words to skip at beginning of stream. Default: 0.
        kwargs : dict
            Optional argument to the 'fcs_bincount' function, specifying
            size or number of bins.

        Returns
        -------
        times : ndarray
            If binning, the lower range of the bins in units of seconds.
            Otherwise (default), the times of the events in detector clock
            units for each channel.
        Bincounts : ndarray
            If binning, the number of events in each bin for each channel.

        """
        self._fh.seek(30 + skip * 2)
        data = numpy.fromfile(self._fh, dtype='u1', count=count * 2)
        # accumulate clock time
        times = numpy.empty((len(data) // 2, 4), dtype='u8')
        times[:, 0] = data[::2]
        times[:, 1:] = 1
        times = times.flatten()
        numpy.cumsum(times, out=times)
        # extract events for each channel
        data = numpy.repeat(data[1::2], 8).reshape(-1, 8)
        data &= numpy.array([1, 2, 4, 8, 16, 32, 64, 128], dtype='u1')
        ch0 = numpy.take(times, numpy.where(data[:, 0::2].flatten() != 0)[0])
        ch1 = numpy.take(times, numpy.where(data[:, 1::2].flatten() != 0)[0])
        del data
        del times
        if kwargs:
            result = fcs_bincount((ch0, ch1), self.frequency, **kwargs)
            return result[0], result[1][0], result[1][1]
        return ch0, ch1

    def __str__(self):
        """Return string with information about ConfoCor2Raw."""
        return '\n '.join(
            (
                self.__class__.__name__,
                os.path.normpath(os.path.normcase(self.filename)),
                f'sampling frequency: {self.frequency} Hz',
            )
        )

    def close(self):
        """Close open file."""
        self._fh.close()

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_value, traceback):
        self.close()


def fcs_bincount(data, frequency, binsize=None, bins=None, binspm=None):
    """Count number of events in bins.

    Parameters
    ----------
    data : tuple of ndarray
        For each channel, the times of events in detector clock units.
    frequency : int
        Sampling frequency in Hz.
    binsize : int
        Time interval, as a multiple of the sampling frequency, in which
        to bin events.
    bins : int
        If 'binsize' is None, the number of bins.
    binspm : int
        If 'binsize' and 'bins' are None, the number of bins per minute (60s).

    Returns
    -------
    times : ndarray
        The lower range of the bins in units of seconds.
    bincounts : tuple of ndarray
        For each channel, the number of events in each bin.

    """
    assert type(data) in (tuple, list)
    if binsize is None:
        if bins is None:
            if binspm is None:
                raise ValueError('missing parameter binsize, bins, or binspm')
            binsize = (60 * frequency) // binspm
        else:
            size = int(max((ch[-1] if ch.size else 0) for ch in data))
            binsize = size // bins + 1
    for ch in data:
        ch //= int(binsize)  # ch is ndarray
    size = int(max((ch[-1] if ch.size else 0) for ch in data) + 1)
    times = numpy.linspace(
        0, size * binsize / float(frequency), size, endpoint=False
    )
    # FIXME: work around https://github.com/numpy/numpy/issues/823
    if size < 2 ** 31:
        # use 32 bit signed int if possible
        data = (ch.astype('i4') for ch in data)
    else:
        # else use a signed view; fails on 32 bit
        data = (ch.view('i8') for ch in data)
    bincounts = tuple(numpy.bincount(ch, minlength=size) for ch in data)
    return times, bincounts


if __name__ == '__main__':
    import doctest

    numpy.set_printoptions(suppress=True, precision=5)
    doctest.testmod()
# fcsfiles.py # Copyright (c) 2012-2021, <NAME> # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: # # 1. Redistributions of source code must retain the above copyright notice, #    this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright notice, #    this list of conditions and the following disclaimer in the documentation #    and/or other materials provided with the distribution. # # 3. Neither the name of the copyright holder nor the names of its #    contributors may be used to endorse or promote products derived from #    this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. Read fluorescence correlation spectroscopy (FCS) data files.  Fcsfiles is a Python library to read Carl Zeiss(r) ConfoCor(r) RAW and ASCII measurement data files.  :Author:   `<NAME> <https://www.lfd.uci.edu/~gohlke/>`_  :Organization:   Laboratory for Fluorescence Dynamics. University of California, Irvine  :License: BSD 3-Clause  :Version: 2021.6.6  Requirements ------------ * `CPython >= 3.7 <https://www.python.org>`_ * `Numpy 1.15.1 <https://www.numpy.org>`_  Revisions --------- 2021.6.6     Remove support for Python 3.6 (NEP 29). 2020.9.18     Relax ConfoCor3Raw header requirement.     Support os.PathLike file names. 2020.1.1     Remove support for Python 2.7 and 3.5.  Notes ----- "<NAME>" and "ConfoCor" are registered trademarks of <NAME>, Inc.  The use of this implementation may be subject to patent or license restrictions.  The API is not stable yet and is expected to change between revisions.  This module does *not* read flow cytometry standard FCS files.  Examples -------- Read the CountRateArray from a ConfoCor3 ASCII file as a numpy array:  >>> fcs = ConfoCor3Fcs('ConfoCor3.fcs') >>> fcs['FcsData']['FcsEntry'][0]['FcsDataSet']['CountRateArray'].shape (60000, 2)  Read data and metadata from a ConfoCor3 RAW file:  >>> fcs = ConfoCor3Raw('ConfoCor3.raw') >>> fcs.filename() 'f5ee4f36488fca2f89cb6b8626111006_R1_P1_K1_Ch1.raw' >>> fcs.frequency 20000000 >>> times = fcs.asarray() >>> times[10858] 1199925494 >>> times, bincounts = fcs.asarray(bins=1000) >>> times.shape (1000,) >>> bincounts[618] 23 >>> fcs.close()  Read data and metadata from a ConfoCor2 RAW file:  >>> fcs = ConfoCor2Raw('ConfoCor2.raw') >>> fcs.frequency 20000000 >>> ch0, ch1 = fcs.asarray() >>> ch1[4812432] 999999833 >>> times, ch0, ch1 = fcs.asarray(bins=1000) >>> times.shape (1000,) >>> ch1[428] 10095 >>> fcs.close() Carl Zeiss ConfoCor3 ASCII data file.      No specification is available. The encoding is 'Windows-1252'. Read file content and parse into dictionary. # encoding = 'Windows-1252' # work around https://github.com/numpy/numpy/issues/1714 Return string close to original file format. Recursively append formatted keys and values to result. Sort dictionary items by key string and value type. Close open file. <NAME> ConfoCor3 raw data file.      Based on "Confocor3 Raw Data Specification. 2 May 2007. by efg, Stowers     Institute for Medical Research."      Attributes     ----------     measurement_identifier : str         Hex form of 4 32-bit little endian integers without leading 0.         It is the same for all channels of the same repetition.     measurement_position : int         Zero based measurement position.     kinetic_index : int         Zero based kinetic index.     repetition_number : int         Zero based repetition_number.     frequency : int         Sampling frequency in Hz. Read file header. Return normalized file name from file content. Read data from file, perform optional binning, and return as array.          Parameters         ----------         count : int (optional)             Number of data words to process. Default: -1 (all words).         skip : int (optional)             Number of data words to skip at beginning of stream. Default: 0.         kwargs : dict             Optional argument to the 'fcs_bincount' function, specifying             size or number of bins.          Returns         -------         times : ndarray             The lower range of the bins in units of seconds, or, if 'binsize'             is 0, the times of the events in detector clock units.         bincounts : ndarray             The number of events in each bin. Return string with information about ConfoCor3Raw. Close open file. Carl Zeiss ConfoCor2 raw data file.      The LSM 510 META - ConfoCor 2 system: an integrated imaging and     spectroscopic platform for single-molecule detection.     <NAME>, <NAME>, and <NAME>.     Curr Pharm Biotechnol. 2004 Apr;5(2):135-54.      ConfoCor 2 - Description of the Raw Data Format. 03.05.01.     www.zeiss.at/C12567BE00472A5C/EmbedTitelIntern/RawData/$File/RawData.pdf      Carl Zeiss Advanced Imaging Microscopy claims a patent on the file format     and data compression scheme.      Attributes     ----------     frequency : int         Sampling frequency in Hz. Read file content and parse into a dictionary. Read data from file, perform optional binning, and return as arrays.          Parameters         ----------         count : int (optional)             Number of data words to process. Default: -1 (all words).         skip : int (optional)             Number of data words to skip at beginning of stream. Default: 0.         kwargs : dict             Optional argument to the 'fcs_bincount' function, specifying             size or number of bins.          Returns         -------         times : ndarray             If binning, the lower range of the bins in units of seconds.             Otherwise (default), the times of the events in detector clock             units for each channel.         Bincounts : ndarray             If binning, the number of events in each bin for each channel. # accumulate clock time # extract events for each channel Return string with information about ConfoCor2Raw. Close open file. Count number of events in bins.      Parameters     ----------     data : tuple of ndarray         For each channel, the times of events in detector clock units.     frequency : int         Sampling frequency in Hz.     binsize : int         Time interval, as a multiple of the sampling frequency, in which         to bin events.     bins : int         If 'binsize' is None, the number of bins.     binspm : int         If 'binsize' and 'bins' are None, the number of bins per minute (60s).      Returns     -------     times : ndarray         The lower range of the bins in units of seconds.     bincounts : tuple of ndarray         For each channel, the number of events in each bin. # ch is ndarray # FIXME: work around https://github.com/numpy/numpy/issues/823 # use 32 bit signed int if possible # else use a signed view; fails on 32 bit