class TdmsFile(object):
    """ Reads and stores data from a TDMS file.

    There are two main ways to create a new TdmsFile object.
    TdmsFile.read will read all data into memory::

        tdms_file = TdmsFile.read(tdms_file_path)

    or you can use TdmsFile.open to read file metadata but not immediately read all data,
    for cases where a file is too large to easily fit in memory or you don't need to
    read data for all channels::

        with TdmsFile.open(tdms_file_path) as tdms_file:
            # Use tdms_file
            ...

    This class acts like a dictionary, where the keys are names of groups in the TDMS
    files and the values are TdmsGroup objects.
    A TdmsFile can be indexed by group name to access a group within the TDMS file, for example::

        tdms_file = TdmsFile.read(tdms_file_path)
        group = tdms_file[group_name]

    Iterating over a TdmsFile produces the names of groups in this file,
    or you can use the groups method to directly access all groups::

        for group in tdms_file.groups():
            # Use group
            ...
    """

    @staticmethod
    def read(file, raw_timestamps=False, memmap_dir=None):
        """ Creates a new TdmsFile object and reads all data in the file

        :param file: Either the path to the tdms file to read
            as a string or pathlib.Path, or an already opened file.
        :param raw_timestamps: By default TDMS timestamps are read as numpy datetime64
            but this loses some precision.
            Setting this to true will read timestamps as a custom TdmsTimestamp type.
        :param memmap_dir: The directory to store memory mapped data files in,
            or None to read data into memory. The data files are created
            as temporary files and are deleted when the channel data is no
            longer used. tempfile.gettempdir() can be used to get the default
            temporary file directory.
        """
        return TdmsFile(file, raw_timestamps=raw_timestamps, memmap_dir=memmap_dir)

    @staticmethod
    def open(file, raw_timestamps=False, memmap_dir=None):
        """ Creates a new TdmsFile object and reads metadata, leaving the file open
            to allow reading channel data

        :param file: Either the path to the tdms file to read
            as a string or pathlib.Path, or an already opened file.
        :param raw_timestamps: By default TDMS timestamps are read as numpy datetime64
            but this loses some precision.
            Setting this to true will read timestamps as a custom TdmsTimestamp type.
        :param memmap_dir: The directory to store memory mapped data files in,
            or None to read data into memory. The data files are created
            as temporary files and are deleted when the channel data is no
            longer used. tempfile.gettempdir() can be used to get the default
            temporary file directory.
        """
        return TdmsFile(
            file, raw_timestamps=raw_timestamps, memmap_dir=memmap_dir, read_metadata_only=True, keep_open=True)

    @staticmethod
    def read_metadata(file, raw_timestamps=False):
        """ Creates a new TdmsFile object and only reads the metadata

        :param file: Either the path to the tdms file to read
            as a string or pathlib.Path, or an already opened file.
        :param raw_timestamps: By default TDMS timestamps are read as numpy datetime64
            but this loses some precision.
            Setting this to true will read timestamps as a custom TdmsTimestamp type.
        """
        return TdmsFile(file, raw_timestamps=raw_timestamps, read_metadata_only=True)

    def __init__(self, file, raw_timestamps=False, memmap_dir=None, read_metadata_only=False, keep_open=False):
        """Initialise a new TdmsFile object

        :param file: Either the path to the tdms file to read
            as a string or pathlib.Path, or an already opened file.
        :param raw_timestamps: By default TDMS timestamps are read as numpy datetime64
            but this loses some precision.
            Setting this to true will read timestamps as a custom TdmsTimestamp type.
        :param memmap_dir: The directory to store memory mapped data files in,
            or None to read data into memory. The data files are created
            as temporary files and are deleted when the channel data is no
            longer used. tempfile.gettempdir() can be used to get the default
            temporary file directory.
        :param read_metadata_only: If this parameter is enabled then only the
            metadata of the TDMS file will read.
        :param keep_open: Keeps the file open so data can be read if only metadata
            is read initially.
        """

        self._memmap_dir = memmap_dir
        self._raw_timestamps = raw_timestamps
        self._groups = OrderedDict()
        self._properties = OrderedDict()
        self._channel_data = {}
        self._tdms_version = 0
        self.data_read = False

        self._reader = TdmsReader(file)
        try:
            self._read_file(self._reader, read_metadata_only, keep_open)
        finally:
            if not keep_open:
                self._reader.close()

    def groups(self):
        """Returns a list of the groups in this file

        :rtype: List of TdmsGroup.
        """

        return list(self._groups.values())

    @property
    def tdms_version(self):
        """ The TDMS format version of this file
        """
        return self._tdms_version

    @property
    def properties(self):
        """ Return the properties of this file as a dictionary

        These are the properties associated with the root TDMS object.
        """

        return self._properties

    def as_dataframe(self, time_index=False, absolute_time=False, scaled_data=True):
        """
        Converts the TDMS file to a DataFrame. DataFrame columns are named using the TDMS object paths.

        :param time_index: Whether to include a time index for the dataframe.
        :param absolute_time: If time_index is true, whether the time index
            values are absolute times or relative to the start time.
        :param scaled_data: By default the scaled data will be used.
            Set to False to use raw unscaled data.
            For DAQmx data, there will be one column per DAQmx raw scaler and column names will include the scale id.
        :return: The full TDMS file data.
        :rtype: pandas.DataFrame
        """

        return pandas_export.from_tdms_file(self, time_index, absolute_time, scaled_data)

    def as_hdf(self, filepath, mode='w', group='/'):
        """
        Converts the TDMS file into an HDF5 file

        :param filepath: The path of the HDF5 file you want to write to.
        :param mode: The write mode of the HDF5 file. This can be 'w' or 'a'
        :param group: A group in the HDF5 file that will contain the TDMS data.
        """
        return hdf_export.from_tdms_file(self, filepath, mode, group)

    def data_chunks(self):
        """ A generator that streams chunks of data from disk.
        This method may only be used when the TDMS file was opened without reading all data immediately.

        :rtype: Generator that yields :class:`DataChunk` objects
        """
        channel_offsets = defaultdict(int)
        for chunk in self._reader.read_raw_data():
            _convert_data_chunk(chunk, self._raw_timestamps)
            yield DataChunk(self, chunk, channel_offsets)
            for path, data in chunk.channel_data.items():
                channel_offsets[path] += len(data)

    def close(self):
        """ Close the underlying file if it was opened by this TdmsFile

            If this TdmsFile was initialised with an already open file
            then the reference to it is released but the file is not closed.
        """
        if self._reader is not None:
            self._reader.close()
            self._reader = None

    def __len__(self):
        """ Returns the number of groups in this file
        """
        return len(self._groups)

    def __iter__(self):
        """ Returns an iterator over the names of groups in this file
        """
        return iter(self._groups)

    def __getitem__(self, group_name):
        """ Retrieve a TDMS group from the file by name
        """
        try:
            return self._groups[group_name]
        except KeyError:
            raise KeyError("There is no group named '%s' in the TDMS file" % group_name)

    def _ipython_key_completions_(self):
        """ Return possible group names for tab-completion when indexing
        """
        return list(self._groups.keys())

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_value, traceback):
        self.close()

    def _read_file(self, tdms_reader, read_metadata_only, keep_open):
        tdms_reader.read_metadata(require_segment_indexes=keep_open)
        self._tdms_version = tdms_reader.tdms_version

        # Use object metadata to build group and channel objects
        group_properties = OrderedDict()
        group_channels = OrderedDict()
        object_properties = {
            path_string: self._convert_properties(obj.properties)
            for path_string, obj in tdms_reader.object_metadata.items()}
        try:
            self._properties = object_properties['/']
        except KeyError:
            pass

        for (path_string, obj) in tdms_reader.object_metadata.items():
            properties = object_properties[path_string]
            path = ObjectPath.from_string(path_string)
            if path.is_root:
                pass
            elif path.is_group:
                group_properties[path.group] = properties
            else:
                # Object is a channel
                try:
                    channel_group_properties = object_properties[path.group_path()]
                except KeyError:
                    channel_group_properties = OrderedDict()
                channel = TdmsChannel(
                    path, obj.data_type, obj.scaler_data_types, obj.num_values,
                    properties, channel_group_properties, self._properties,
                    tdms_reader, self._raw_timestamps, self._memmap_dir)
                if path.group in group_channels:
                    group_channels[path.group].append(channel)
                else:
                    group_channels[path.group] = [channel]

        # Create group objects containing channels and properties
        for group_name, properties in group_properties.items():
            try:
                channels = group_channels[group_name]
            except KeyError:
                channels = []
            group_path = ObjectPath(group_name)
            self._groups[group_name] = TdmsGroup(group_path, properties, channels)
        for group_name, channels in group_channels.items():
            if group_name not in self._groups:
                # Group with channels but without any corresponding object metadata in the file:
                group_path = ObjectPath(group_name)
                self._groups[group_name] = TdmsGroup(group_path, {}, channels)

        if not read_metadata_only:
            self._read_data(tdms_reader)

    def _read_data(self, tdms_reader):
        with Timer(log, "Allocate space"):
            # Allocate space for data
            for group in self.groups():
                for channel in group.channels():
                    self._channel_data[channel.path] = get_data_receiver(
                        channel, len(channel), self._raw_timestamps, self._memmap_dir)

        with Timer(log, "Read data"):
            # Now actually read all the data
            for chunk in tdms_reader.read_raw_data():
                for (path, data) in chunk.channel_data.items():
                    channel_data = self._channel_data[path]
                    if data.data is not None:
                        channel_data.append_data(data.data)
                    elif data.scaler_data is not None:
                        for scaler_id, scaler_data in data.scaler_data.items():
                            channel_data.append_scaler_data(scaler_id, scaler_data)

            for group in self.groups():
                for channel in group.channels():
                    channel_data = self._channel_data[channel.path]
                    if channel_data is not None:
                        channel._set_raw_data(channel_data)

        self.data_read = True

    def _convert_properties(self, properties):
        def convert_prop(val):
            if isinstance(val, TdmsTimestamp) and not self._raw_timestamps:
                # Convert timestamps to numpy datetime64 if raw timestamps are not requested
                return val.as_datetime64()
            return val
        return OrderedDict((k, convert_prop(v)) for (k, v) in properties.items()) class GeneratedFile(object):
    """Generate a TDMS file for testing"""

    def __init__(self):
        self._content = []

    def add_segment(self, toc, metadata, data, incomplete=False, binary_data=False, version=4713):
        metadata_bytes = _hex_to_bytes(metadata)
        data_bytes = data if binary_data else _hex_to_bytes(data)
        if toc is not None:
            lead_in = b'TDSm'
            toc_mask = long(0)
            for toc_item in toc:
                if toc_item == "kTocMetaData":
                    toc_mask = toc_mask | long(1) << 1
                elif toc_item == "kTocRawData":
                    toc_mask = toc_mask | long(1) << 3
                elif toc_item == "kTocDAQmxRawData":
                    toc_mask = toc_mask | long(1) << 7
                elif toc_item == "kTocInterleavedData":
                    toc_mask = toc_mask | long(1) << 5
                elif toc_item == "kTocBigEndian":
                    toc_mask = toc_mask | long(1) << 6
                elif toc_item == "kTocNewObjList":
                    toc_mask = toc_mask | long(1) << 2
                else:
                    raise ValueError("Unrecognised TOC value: %s" % toc_item)
            lead_in += struct.pack('<i', toc_mask)
            lead_in += struct.pack('<l', version)
            next_segment_offset = len(metadata_bytes) + len(data_bytes)
            raw_data_offset = len(metadata_bytes)
            if incomplete:
                lead_in += _hex_to_bytes('FF' * 8)
            else:
                lead_in += struct.pack('<Q', next_segment_offset)
            lead_in += struct.pack('<Q', raw_data_offset)
        else:
            lead_in = b''
        self._content.append((lead_in, metadata_bytes, data_bytes))

    def get_tempfile(self, **kwargs):
        named_file = tempfile.NamedTemporaryFile(suffix=".tdms", **kwargs)
        file = named_file.file
        file.write(self._get_contents())
        file.seek(0)
        return named_file

    @contextmanager
    def get_tempfile_with_index(self):
        directory = tempfile.mkdtemp()
        tdms_path = os.path.join(directory, 'test_file.tdms')
        tdms_index_path = os.path.join(directory, 'test_file.tdms_index')
        with open(tdms_path, 'wb') as file:
            file.write(self._get_contents())
        with open(tdms_index_path, 'wb') as file:
            file.write(self._get_index_contents())
        try:
            yield tdms_path
        finally:
            os.unlink(tdms_path)
            os.unlink(tdms_index_path)
            os.rmdir(directory)

    def load(self, *args, **kwargs):
        with tempfile.NamedTemporaryFile(suffix=".tdms") as named_file:
            file = named_file.file
            file.write(self._get_contents())
            file.seek(0)
            return tdms.TdmsFile(file, *args, **kwargs)

    def get_bytes_io_file(self):
        file = BytesIO()
        file.write(self._get_contents())
        file.seek(0)
        return file

    def _get_contents(self):
        contents = b''
        for segment in self._content:
            contents += segment[0]
            contents += segment[1]
            contents += segment[2]
        return contents

    def _get_index_contents(self):
        contents = b''
        for segment in self._content:
            lead_in = segment[0]
            if len(lead_in) >= 4:
                lead_in = b'TDSh' + lead_in[4:]
            contents += lead_in
            contents += segment[1]
        return contents def hexlify_value(struct_type, value):
    """Return hex string representation of a value"""
    return binascii.hexlify(struct.pack(struct_type, value)).decode('utf-8') def string_hexlify(input_string):
    """Return hex string representation of string"""
    return binascii.hexlify(input_string.encode('utf-8')).decode('utf-8') def segment_objects_metadata(*args):
    """ Metadata for multiple objects in a segment
    """
    num_objects_hex = hexlify_value("<I", len(args))
    return num_objects_hex + "".join(args) def channel_metadata(channel_name, data_type, num_values, properties=None):
    return (
        # Length of the object path
        hexlify_value('<I', len(channel_name)) +
        # Object path
        string_hexlify(channel_name) +
        # Length of index information
        "14 00 00 00" +
        # Raw data data type
        hexlify_value('<I', data_type) +
        # Dimension
        "01 00 00 00" +
        # Number of raw data values
        hexlify_value('<Q', num_values) +
        hex_properties(properties)
    ) def channel_metadata_with_no_data(channel_name):
    return (
        # Length of the object path
        hexlify_value('<I', len(channel_name)) +
        # Object path
        string_hexlify(channel_name) +
        # Raw data index header meaning no data in this segment
        "FF FF FF FF" +
        # Number of properties (0)
        "00 00 00 00"
    ) def channel_metadata_with_repeated_structure(channel_name):
    return (
        # Length of the object path
        hexlify_value('<I', len(channel_name)) +
        # Object path
        string_hexlify(channel_name) +
        # Raw data index header meaning repeat previous data structure
        "00 00 00 00" +
        # Number of properties (0)
        "00 00 00 00"
    ) TDS_TYPE_INT32 = 3 def timestamp_data_chunk(times):
    return "".join(timestamp_hexlify(time) for time in times)
import os
import numpy as np
import pytest
from nptdms import TdmsFile
from nptdms.test.util import (
    GeneratedFile,
    hexlify_value,
    string_hexlify,
    segment_objects_metadata,
    channel_metadata,
    channel_metadata_with_no_data,
    channel_metadata_with_repeated_structure)
from nptdms.test.scenarios import TDS_TYPE_INT32, timestamp_data_chunk
@pytest.mark.benchmark(group='read-all-channel')
def test_read_contiguous_data_channel(benchmark):
    """ Benchmark reading a single channel's data from a contiguous data file
    """
    with TdmsFile.open(get_contiguous_file().get_bytes_io_file()) as tdms_file:
        channel = tdms_file['group']['channel3']
        channel_data = benchmark(read_channel_data, channel)

        expected_data = np.repeat([3], 10000)
        np.testing.assert_equal(channel_data, expected_data)


@pytest.mark.benchmark(group='read-all-channel')
def test_read_interleaved_data_channel(benchmark):
    """ Benchmark reading a single channel's data from an interleaved data file
    """
    with TdmsFile.open(get_interleaved_file().get_bytes_io_file()) as tdms_file:
        channel = tdms_file['group']['channel3']
        channel_data = benchmark(read_channel_data, channel)

        expected_data = np.repeat([3], 10000)
        np.testing.assert_equal(channel_data, expected_data)


@pytest.mark.benchmark(group='read-all-channel')
def test_stream_contiguous_data_channel(benchmark):
    """ Benchmark streaming channel data from a contiguous data file
    """
    with TdmsFile.open(get_contiguous_file().get_bytes_io_file()) as tdms_file:
        channel = tdms_file['group']['channel3']
        channel_data = benchmark(stream_chunks, channel)

        channel_data = np.concatenate(channel_data)
        expected_data = np.repeat([3], 10000)
        np.testing.assert_equal(channel_data, expected_data)


@pytest.mark.benchmark(group='read-all-channel')
def test_stream_interleaved_data_channel(benchmark):
    """ Benchmark streaming channel data from an interleaved data file
    """
    with TdmsFile.open(get_interleaved_file().get_bytes_io_file()) as tdms_file:
        channel = tdms_file['group']['channel3']
        channel_data = benchmark(stream_chunks, channel)

        channel_data = np.concatenate(channel_data)
        expected_data = np.repeat([3], 10000)
        np.testing.assert_equal(channel_data, expected_data)


@pytest.mark.benchmark(group='slice-channel')
def test_slice_contiguous_data_channel(benchmark):
    """ Benchmark reading a slice of data from a contiguous data file
    """
    with TdmsFile.open(get_contiguous_file().get_bytes_io_file()) as tdms_file:
        channel = tdms_file['group']['channel3']
        channel_data = benchmark(get_slice, channel, 5555, 6555)

        expected_data = np.repeat([3], 1000)
        np.testing.assert_equal(channel_data, expected_data)


@pytest.mark.benchmark(group='slice-channel')
def test_slice_interleaved_data_channel(benchmark):
    """ Benchmark reading a slice of data from an interleaved data file
    """
    with TdmsFile.open(get_interleaved_file().get_bytes_io_file()) as tdms_file:
        channel = tdms_file['group']['channel3']
        channel_data = benchmark(get_slice, channel, 5555, 6555)

        expected_data = np.repeat([3], 1000)
        np.testing.assert_equal(channel_data, expected_data)


@pytest.mark.benchmark(group='read-all-channel')
def test_index_contiguous_data_channel(benchmark):
    """ Benchmark reading a data from a contiguous data file using integer indices
    """
    with TdmsFile.open(get_contiguous_file().get_bytes_io_file()) as tdms_file:
        channel = tdms_file['group']['channel3']
        channel_data = np.zeros(10000, dtype=channel.dtype)
        benchmark(index_values, channel, channel_data)

        expected_data = np.repeat([3], 10000)
        np.testing.assert_equal(channel_data, expected_data)


@pytest.mark.benchmark(group='read-all-channel')
def test_index_interleaved_data_channel(benchmark):
    """ Benchmark reading a data from a interleaved data file using integer indices
    """
    with TdmsFile.open(get_interleaved_file().get_bytes_io_file()) as tdms_file:
        channel = tdms_file['group']['channel3']
        channel_data = np.zeros(10000, dtype=channel.dtype)
        benchmark(index_values, channel, channel_data)

        expected_data = np.repeat([3], 10000)
        np.testing.assert_equal(channel_data, expected_data)


@pytest.mark.benchmark(group='read-scaled-channel')
def test_stream_scaled_data_chunks(benchmark):
    """ Benchmark streaming channel data when the data is scaled
    """
    properties = {
        "NI_Number_Of_Scales":
            (3, "01 00 00 00"),
        "NI_Scale[0]_Scale_Type":
            (0x20, hexlify_value("<I", len("Linear")) + string_hexlify("Linear")),
        "NI_Scale[0]_Linear_Slope":
            (10, hexlify_value("<d", 2.0)),
        "NI_Scale[0]_Linear_Y_Intercept":
            (10, hexlify_value("<d", 10.0))
    }
    test_file = GeneratedFile()
    data_array = np.arange(0, 1000, dtype=np.dtype('int32'))
    data = data_array.tobytes()
    test_file.add_segment(
        ("kTocMetaData", "kTocRawData", "kTocNewObjList"),
        segment_objects_metadata(

            channel_metadata("/'group'/'channel1'", TDS_TYPE_INT32, 100, properties),