class FileNameSequencer:
class SequentialFileBase:
class SequentialFileReader(SequentialFileBase):
class SequentialFileWriter(SequentialFileBase):
    def __init__(self, template, header={}):
        def check_and_convert(x):
    def _process_items(self, file_nr):
    def __getitem__(self, file_nr):
    def __len__(self):
    def __init__(self, files, mode='rb', opener=None):
    def __getattr__(self, attr):
    def _open(self, file_nr):
    def tell(self):
    def memmap(self, dtype=np.uint8, mode=None, offset=None, shape=None,
               order='C'):
    def close(self):
    def __enter__(self):
    def __exit__(self, exc_type, exc_val, exc_tb):
    def __repr__(self):
    def __getattr__(self, attr):
    def file_size(self):
    def size(self):
    def seek(self, offset, whence=0):
    def read(self, count=None):
    def __getstate__(self):
    def __setstate__(self, state):
    def __init__(self, files, mode='w+b', file_size=None, opener=None):
    def write(self, data):
    def memmap(self, dtype=np.uint8, mode=None, offset=None, shape=None,
               order='C'):
def open(files, mode='rb', file_size=None, opener=None): class FileBase:
    """File wrapper, used to add frame methods to a binary data file.

    The underlying file is stored in ``fh_raw`` and all attributes that do not
    exist on the class itself are looked up on it.

    Parameters
    ----------
    fh_raw : filehandle
        Filehandle of the raw binary data file.

    Notes
    -----
    A subclass for reading should define ``read_header``, ``read_frame``, and
    ``get_frame_rate`` methods, and also set an ``info`` property. Often the
    standard one will suffice, i.e., ``info = FileReaderInfo()``.

    A subclass for writing should define a ``write_frame`` method.

    """
    fh_raw = None

    def __init__(self, fh_raw):
        self.fh_raw = fh_raw

    def __getattr__(self, attr):
        """Try to get things on the current open file if it is not on self."""
        if not attr.startswith('_'):
            try:
                return getattr(self.fh_raw, attr)
            except AttributeError:
                pass
        #  __getattribute__ to raise appropriate error.
        return self.__getattribute__(attr)

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        self.close()

    def close(self):
        self.fh_raw.close()

    @contextmanager
    def temporary_offset(self, offset=None, whence=0):
        """Context manager for temporarily seeking to another file position.

        To be used as part of a ``with`` statement::

            with fh_raw.temporary_offset() [as fh_raw]:
                with-block

        On exiting the ``with-block``, the file pointer is moved back to its
        original position.  As a convenience, one can pass on the offset
        to seek to when entering the context manager.  Parameters are as
        for :meth:`io.IOBase.seek`.
        """
        oldpos = self.tell()
        try:
            if offset is not None:
                self.seek(offset, whence)
            yield self
        finally:
            self.seek(oldpos)

    def __repr__(self):
        return "{0}(fh_raw={1})".format(self.__class__.__name__, self.fh_raw)

    def __getstate__(self):
        if self.writable():
            raise TypeError('cannot pickle file opened for writing')

        state = self.__dict__.copy()
        # IOBase instances cannot be pickled, but we can just reopen them
        # when we are unpickled.  Anything else may have internal state that
        # needs preserving (e.g., SequentialFile), so we will assume
        # it takes care of this itself.
        if isinstance(self.fh_raw, io.IOBase):
            fh = state.pop('fh_raw')
            state['fh_info'] = {
                'offset': 'closed' if fh.closed else fh.tell(),
                'filename': fh.name,
                'mode': fh.mode}

        return state

    def __setstate__(self, state):
        fh_info = state.pop('fh_info', None)
        if fh_info is not None:
            fh = io.open(fh_info['filename'], fh_info['mode'])
            if fh_info['offset'] != 'closed':
                fh.seek(fh_info['offset'])
            else:
                fh.close()
            state['fh_raw'] = fh

        self.__dict__.update(state) class StreamReaderBase(StreamBase):
    """Base for all stream readers, providing standard reading methods.

    Parameters
    ----------
    fh_raw : filehandle
        Should be opened in binary mode for reading, and usually wrapped
        in a given format;s ``FileReader``.
    header0 : header instance
        First header of the file, with information like the start time, etc.
    squeeze : bool, optional
        If `True` (default), remove any dimensions of length unity from
        decoded data.
    subset : indexing object or tuple of objects, optional
        Specific components of the complete sample to decode (after possible
        squeezing).
    fill_value : float or complex, optional
        Value to use for invalid or missing data. Default: 0.
    verify : bool, optional
        Whether to do basic checks of frame integrity when reading.  The first
        frameset of the stream is always checked.  Default: `True`.
    **kwargs
        Information that supplements that of the header. In particular, any
        format should define ``bps``, ``complex_data``, ``samples_per_frame``,
        and ``sample_shape`` (pre-squeeze and subset). Furthermore,
        ``sample_rate`` should be provided if the raw file reader cannot
        determine it.

    Notes
    -----
    Accessing frames happens via a number of private methods, which can be
    overridden by subclasses to deal with their peculiarities.

    The process starts in `~baseband.base.base.StreamReaderBase.read` with
    ``_get_frame(offset)``, which determines the ``index`` of the frame a
    given offset falls in, and then either uses ``_read_frame(index)`` to
    read it or returns a possibly cached frame, together with the
    ``sample_offset`` at which ``offset`` falls within the frame. Then, data
    is retrieved as needed until the frame is exhausted, at which point the
    process repeats.

    The ``_read_frame(index)`` method in turn takes a number of steps.
    First, the binary file reader is moved to the correct position using
    ``_seek_frame(index)``, then for the base reader, the actual reading is
    done using ``_fh_raw_read_frame()`` and a check is made that the right
    frame was read.

    Various formats override various steps. For instance, DADA overrides
    ``_fh_raw_read_frame()`` to take care of last frames, which are often
    longer than the fixed lengths recorded in the header. For GUPPI,
    ``_get_frame`` is overridden to take into account overlap between
    frames, which should not be skipped for the last frame.  For GSB,
    ``_seek_frame`` and ``_fh_raw_read_frame`` are both overridden to take
    into account that header and data are read from different files.
    """

    info = StreamReaderInfo()

    def __init__(self, fh_raw, header0, *,
                 squeeze=True, subset=(), fill_value=0., verify=True,
                 **kwargs):
        self._subset = (() if subset is None
                        else subset if isinstance(subset, tuple)
                        else (subset,))
        self._fill_value = float(fill_value)
        self.verify = verify
        super().__init__(fh_raw, header0, squeeze=squeeze, **kwargs)

    @lazyproperty
    def sample_rate(self):
        sample_rate = super().sample_rate
        if sample_rate is None:
            try:
                sample_rate = (self.samples_per_frame
                               * self.fh_raw.get_frame_rate()).to(u.MHz)

            except Exception as exc:
                exc.args += ("the sample rate could not be auto-detected. "
                             "This can happen if the file is too short to "
                             "determine the sample rate, or because it is "
                             "corrupted.  Try passing in an explicit "
                             "`sample_rate`.",)
                raise

        return sample_rate

    @property
    def verify(self):
        """Whether to do consistency checks on frames being read."""
        return self._verify

    @verify.setter
    def verify(self, verify):
        self._verify = bool(verify) if verify != 'fix' else verify

    @property
    def subset(self):
        """Specific components of the complete sample to decode.

        The order of dimensions is the same as for `sample_shape`.  Set by
        the class initializer.
        """
        return self._subset

    def _squeeze_and_subset(self, data):
        """Possibly remove unit dimensions and subset the given data.

        The first dimension (sample number) is never removed.
        """
        if self.squeeze:
            data = data.reshape(data.shape[:1]
                                + tuple(sh for sh in data.shape[1:] if sh > 1))
        if self.subset:
            data = data[(slice(None),) + self.subset]

        return data

    @lazyproperty
    def sample_shape(self):
        """Shape of a complete sample (possibly subset or squeezed)."""
        # First apply base class, which squeezes if needed.
        sample_shape = super().sample_shape
        if not self.subset:
            return sample_shape

        # Now apply subset to a dummy sample that has the sample number as its
        # value (where 13 is to bring bad luck to over-complicated subsets).
        dummy_data = np.arange(13.)
        dummy_sample = np.moveaxis(
            (np.zeros(sample_shape)[..., np.newaxis] + dummy_data), -1, 0)
        try:
            dummy_subset = dummy_sample[(slice(None),) + self.subset]
            # Check whether subset was in range and whether sample numbers were
            # preserved (latter should be, but check anyway).
            assert 0 not in dummy_subset.shape
            assert np.all(np.moveaxis(dummy_subset, 0, -1) == dummy_data)
        except (IndexError, AssertionError) as exc:
            exc.args += ("subset {} cannot be used to properly index "
                         "{}samples with shape {}.".format(
                             self.subset, "squeezed " if self.squeeze else "",
                             sample_shape),)
            raise exc

        # We got the shape.  We only bother trying to associate names with the
        # dimensions when we know for sure what happened in the subsetting.
        subset_shape = dummy_subset.shape[1:]
        if (not hasattr(sample_shape, '_fields') or subset_shape == ()
                or len(self.subset) > len(sample_shape)):
            return subset_shape

        # We can only associate names when indexing each dimension separately
        # gives a consistent result with the complete subset.
        subset_axis = 0
        fields = []
        subset = self.subset + (slice(None),) * (len(sample_shape)
                                                 - len(self.subset))
        try:
            for field, sample_dim, item in zip(sample_shape._fields,
                                               sample_shape, subset):
                subset_dim = np.empty(sample_dim)[item].shape
                assert len(subset_dim) <= 1  # No advanced multi-d indexing.
                if len(subset_dim) == 1:
                    # If this dimension was not removed and matches that
                    # of the real subset, we now have a field label for it.
                    assert subset_dim[0] == subset_shape[subset_axis]
                    fields.append(field)
                    subset_axis += 1
        except Exception:
            # Things did not make sense, probably some advanced indexing;
            # Just don't worry about having a named tuple.
            return subset_shape

        shape_cls = namedtuple('SampleShape', ','.join(fields))
        return shape_cls(*subset_shape)

    @lazyproperty
    def _last_header(self):
        """Header of the last file for this stream."""
        # Base implementation.  Seek forward rather than backward, as last
        # frame can have missing bytes.
        with self.fh_raw.temporary_offset() as fh_raw:
            file_size = fh_raw.seek(0, 2)
            nframes, fframe = divmod(file_size, self.header0.frame_nbytes)
            fh_raw.seek((nframes - 1) * self.header0.frame_nbytes)
            return fh_raw.read_header()

    # Override the following so we can refer to stop_time in the docstring.
    @property
    def start_time(self):
        """Start time of the file.

        See also `time` for the time of the sample pointer's current offset,
        and (if available) `stop_time` for the time at the end of the file.
        """
        return super().start_time

    @property
    def time(self):
        """Time of the sample pointer's current offset in file.

        See also `start_time` for the start time, and (if available)
        `stop_time` for the end time, of the file.
        """
        return super().time

    @lazyproperty
    def stop_time(self):
        """Time at the end of the file, just after the last sample.

        See also `start_time` for the start time of the file, and `time` for
        the time of the sample pointer's current offset.
        """
        return (self._get_time(self._last_header)
                + (self.samples_per_frame / self.sample_rate))

    @lazyproperty
    def _nsample(self):
        """Number of complete samples in the stream data."""
        return int(((self.stop_time - self.start_time)
                    * self.sample_rate).to(u.one).round())

    @property
    def shape(self):
        """Shape of the (squeezed/subset) stream data."""
        return (self._nsample,) + self.sample_shape

    @property
    def size(self):
        """Total number of component samples in the (squeezed/subset) stream
        data.
        """
        prod = 1
        for dim in self.shape:
            prod *= dim
        return prod

    @property
    def ndim(self):
        """Number of dimensions of the (squeezed/subset) stream data."""
        return len(self.shape)

    @property
    def fill_value(self):
        """Value to use for invalid or missing data. Default: 0."""
        return self._fill_value

    @property
    def dtype(self):
        # TODO: arguably, this should be inferred from an actual payload.
        return np.dtype(np.complex64 if self.complex_data else np.float32)

    def readable(self):
        """Whether the file can be read and decoded."""
        return self.fh_raw.readable and self.fh_raw.info.readable

    def seek(self, offset, whence=0):
        """Change the sample pointer position.

        This works like a normal filehandle seek, but the offset is in samples
        (or a relative or absolute time).

        Parameters
        ----------
        offset : int, `~astropy.units.Quantity`, or `~astropy.time.Time`
            Offset to move to.  Can be an (integer) number of samples,
            an offset in time units, or an absolute time.  For the latter
            two, the pointer will be moved to the nearest integer sample.
        whence : {0, 1, 2, 'start', 'current', or 'end'}, optional
            Like regular seek, the offset is taken to be from the start if
            ``whence=0`` (default), from the current position if 1,
            and from the end if 2.  One can alternativey use 'start',
            'current', or 'end' for 0, 1, or 2, respectively.  Ignored if
            ``offset`` is a time.
        """
        try:
            offset = operator.index(offset)
        except Exception:
            try:
                offset = offset - self.start_time
            except Exception:
                pass
            else:
                whence = 0

            offset = int((offset * self.sample_rate).to(u.one).round())

        if whence == 0 or whence == 'start':
            self.offset = offset
        elif whence == 1 or whence == 'current':
            self.offset += offset
        elif whence == 2 or whence == 'end':
            self.offset = self.shape[0] + offset
        else:
            raise ValueError("invalid 'whence'; should be 0 or 'start', 1 or "
                             "'current', or 2 or 'end'.")

        return self.offset

    def read(self, count=None, out=None):
        """Read a number of complete samples.

        Parameters
        ----------
        count : int or None, optional
            Number of complete samples to read. If `None` (default) or
            negative, the number of samples left. Ignored if ``out`` is given.
        out : None or array, optional
            Array to store the samples in. If given, ``count`` will be inferred
            from the first dimension; the remaining dimensions should equal
            `sample_shape`.

        Returns
        -------
        out : `~numpy.ndarray` of float or complex
            The first dimension is sample-time, and the remaining ones are
            as given by `sample_shape`.
        """
        if self.closed:
            raise ValueError("I/O operation on closed stream.")

        samples_left = self.shape[0] - self.offset
        if out is None:
            if count is None or count < 0:
                count = max(0, samples_left)

            out = np.empty((count,) + self.sample_shape, dtype=self.dtype)
        else:
            assert out.shape[1:] == self.sample_shape, (
                "'out' must have trailing shape {}".format(self.sample_shape))
            count = out.shape[0]

        if count > samples_left:
            raise EOFError("cannot read from beyond end of input.")

        offset0 = self.offset
        sample = 0
        while sample < count:
            # For current position, get frame plus offset in that frame.
            frame, sample_offset = self._get_frame(self.offset)
            nsample = min(count - sample, len(frame) - sample_offset)
            data = frame[sample_offset:sample_offset + nsample]
            data = self._squeeze_and_subset(data)
            # Copy to relevant part of output.
            out[sample:sample + nsample] = data
            sample += nsample
            # Explicitly set offset (leaving get_frame free to adjust it).
            self.offset = offset0 + sample

        return out

    def _get_frame(self, offset):
        """Get a frame that includes given offset.

        Finds the index corresponding to the needed frame, assuming frames
        are all the same length.  If not already cached, it retrieves a
        frame by calling ``self._read_frame(index)``.

        Parameters
        ----------
        offset : int
            Offset in the stream for which a frame should be found.

        Returns
        -------
        frame : `~baseband.base.frame.FrameBase`
            Frame holding the sample at ``offset``.
        sample_offset : int
            Offset within the frame corresponding to ``offset``.
        """
        frame_index, sample_offset = divmod(offset, self.samples_per_frame)
        if frame_index != self._frame_index:
            self._frame = self._read_frame(frame_index)
            self._frame.fill_value = self.fill_value
            self._frame_index = frame_index

        return self._frame, sample_offset

    def _read_frame(self, index):
        """Base implementation of reading a frame.

        It cannot handle bad frames, and contains pieces which subclasses
        can override as needed (or override the whole thing).
        """
        self._seek_frame(index)
        frame = self._fh_raw_read_frame()
        # Check whether we actually got the right frame.
        if self.verify and self._get_index(frame) != index:
            raise ValueError('wrong frame number.')

        return frame

    def _seek_frame(self, index):
        """Move the underlying file pointer to the frame of the given index."""
        return self.fh_raw.seek(index * self.header0.frame_nbytes)

    def _fh_raw_read_frame(self):
        """Read a frame at the current position of the underlying file."""
        return self.fh_raw.read_frame(verify=self.verify)

    def __getstate__(self):
        state = self.__dict__.copy()
        # Remove cached frame and associate index, since these
        # are almost certainly not needed and can be regenerated.
        # Sample_shape can also be regenerated, yet not easily pickled,
        # if the namedtuple is defined on the class, so we remove it too.
        for item in ('_frame', '_frame_index', 'sample_shape'):
            state.pop(item, None)

        return state

    def __setstate__(self, state):
        self.__dict__.update(state) class StreamWriterBase(StreamBase):
    """Base for all stream writers, providing a standard write method.

    Parameters
    ----------
    fh_raw : filehandle
        Should be opened in binary mode for writer, and usually wrapped
        in a given format's ``FileWriter``.
    header0 : header instance
        Header for the first frame, holding time information, etc.  Can instead
        give keyword arguments to construct a header (see ``**kwargs``).
    squeeze : bool, optional
        If `True` (default), writer accepts squeezed arrays as input, and adds
        any dimensions of length unity.
    **kwargs
        Information that supplements that of the header. In particular, any
        format should define ``bps``, ``complex_data``, ``samples_per_frame``,
        ``sample_shape`` (on file), and ``sample_rate``.

    Notes
    -----
    Accessing frames happens via a number of private methods, which can be
    overridden by subclasses to deal with their  peculiarities.

    The process starts in `~baseband.base.base.StreamWriterBase.write` with
    ``_get_frame(offset)``, which determines the ``index`` of the frame a
    given offset falls in, and then either uses ``_make_frame(index)`` to
    create it, or returns a possibly cached frame, together with the
    ``sample_offset`` at which ``offset`` falls within the frame. Then, data
    is inserted as needed until the frame is exhausted, at which point the
    frame is written to disk using ``_fh_raw_write_frame`` and the process
    repeats.

    Various formats override various steps. For instance, for DADA and GUPPI,
    ``_make_frame()`` and ``_fh_raw_write_frame()`` are overridden to work
    with memory mapped files. For GSB, ``_fh_raw_write_frame()`` is overridden
    to take into account that header and data are read from different files.
    """

    def _unsqueeze(self, data):
        new_shape = list(data.shape)
        for i, dim in enumerate(self._unsliced_shape):
            if dim == 1:
                new_shape.insert(i + 1, 1)
        return data.reshape(new_shape)

    def write(self, data, valid=True):
        """Write data, buffering by frames as needed.

        Parameters
        ----------
        data : `~numpy.ndarray`
            Piece of data to be written, with sample dimensions as given by
            `sample_shape`. This should be properly scaled to make best use
            of the dynamic range delivered by the encoding.
        valid : bool, optional
            Whether the current data are valid.  Default: `True`.
        """
        assert data.shape[1:] == self.sample_shape, (
            "'data' should have trailing shape {}".format(self.sample_shape))

        if self.squeeze:
            data = self._unsqueeze(data)

        count = data.shape[0]
        offset0 = self.offset
        sample = 0
        while sample < count:
            frame, sample_offset = self._get_frame(self.offset)
            nsample = min(count - sample, len(frame) - sample_offset)
            sample_end = sample_offset + nsample
            frame[sample_offset:sample_end] = data[sample:sample+nsample]
            frame.valid &= valid
            if sample_end == len(frame):
                self._fh_raw_write_frame(frame)

            sample += nsample
            # Explicitly set offset (leaving get_frame free to adjust it).
            self.offset = offset0 + sample

    def _get_frame(self, offset):
        # Nearly identical to StreamReaderBase version, but not
        # quite worth separating it out.
        frame_index, sample_offset = divmod(offset, self.samples_per_frame)
        if frame_index != self._frame_index:
            self._frame = self._make_frame(frame_index)
            self._frame_index = frame_index

        return self._frame, sample_offset

    _get_frame.__doc__ = StreamReaderBase._get_frame.__doc__

    def _make_frame(self, index):
        # Default implementation assumes that an initial _frame was
        # set up and just re-uses it with a new index.
        self._set_index(self._frame, index)
        self._frame.valid = True
        return self._frame

    def _fh_raw_write_frame(self, frame):
        # Default implementation is to assume that the frame knows how to
        # write itself to the underlying file.
        frame.tofile(self.fh_raw)

    def close(self):
        extra = self.offset % self.samples_per_frame
        if extra != 0:
            warnings.warn("closing with partial buffer remaining.  "
                          "Writing padded frame, marked as invalid.")
            self.write(np.zeros((self.samples_per_frame - extra,)
                                + self.sample_shape), valid=False)
            assert self.offset % self.samples_per_frame == 0
        return super().close() class FileOpener:
    """File opener for a baseband format.

    Each instance can be used as a function to open a baseband stream.
    It is probably best used inside a wrapper, so that the documentation
    can reflect the docstring of ``__call__`` rather than of this class.

    Parameters
    ----------
    fmt : str
        Name of the baseband format
    classes : dict
        With the file/stream reader/writer classes keyed by names equal to
        'FileReader', 'FileWriter', 'StreamReader', 'StreamWriter' prefixed
        by ``fmt``.  Typically, one will pass in ``classes=globals()``.
    header_class : `~baseband.base.header.VLBIHeaderBase` subclass
        Used to instantiate a header from keywords as needed.
    """

    FileNameSequencer = sf.FileNameSequencer
    """Sequencer used for templates."""

    non_header_keys = {'squeeze', 'subset', 'fill_value', 'verify',
                       'file_size'}
    """keyword arguments that should never be used to create a header."""

    _name = None

    def __init__(self, fmt, classes, header_class):
        self.fmt = fmt
        self.classes = classes
        self.header_class = header_class

    def normalize_mode(self, mode):
        if mode in self.classes:
            return mode
        if mode[::-1] in self.classes:
            return mode[::-1]
        if mode in {'r', 'w'}:
            return mode + 's'

        raise ValueError(f'invalid mode: {mode} '
                         f'({self.fmt} supports {set(self.classes)}).')

    def _get_type(self, name):
        try:
            pathlib.Path(name)
        except TypeError:
            if hasattr(name, 'read') or hasattr(name, 'write'):
                return 'fh'

            elif isiterable(name):
                return 'sequence'

            raise ValueError("name '{name}' not understood.")

        if isinstance(name, str) and '{' in name and '}' in name:
            return 'template'
        else:
            return 'name'

    def get_type(self, name):
        """Infer the type of file name is pointing to.

        Options are 'fh', 'name', 'sequence', and 'template'.
        """
        if self._name is not name:
            self._type = self._get_type(name)
            self._name = name
        return self._type

    def is_sequence(self, name):
        """Whether name is an (implied) sequence of files."""
        return self.get_type(name) in ('template', 'sequence')

    def is_template(self, name):
        """Whether name is a template for a sequence of files."""
        return self.get_type(name) == 'template'

    def is_name(self, name):
        """Whether name is a name of a file."""
        return self.get_type(name) == 'name'

    def is_fh(self, name):
        """Whether name is a filehandle."""
        return self.get_type(name) == 'fh'

    def get_header0(self, kwargs):
        """Get header0 from kwargs or construct it from kwargs.

        Possible keyword arguments will be popped from kwargs.
        """
        header0 = kwargs.get('header0', None)
        if header0 is None:
            tried = {key: value for key, value in kwargs.items()
                     if key not in self.non_header_keys}
            with warnings.catch_warnings():
                # Ignore possible warnings about extraneous arguments.
                warnings.simplefilter('ignore')
                header0 = self.header_class.fromvalues(**tried)
            # Pop the kwargs that we likely used.  We do this by inspection,
            # but note that this may still let extraneous keywords
            # by eaten up by header classes that just store everything.
            maybe_used = (
                set(inspect.signature(self.header_class.fromvalues).parameters)
                | set(self.header_class._properties)
                | set(header0.keys()))

            maybe_used = {key.lower() for key in maybe_used}
            used = set(key for key in tried if key.lower() in maybe_used)
            for key in used:
                kwargs.pop(key)

        return header0

    def get_fns(self, name, mode, kwargs):
        """Convert a template into a file-name sequencer.

        Any keywords needed to fill the template are popped from kwargs.
        """
        try:
            fns_kwargs = dict(self.get_header0(kwargs))
        except Exception:
            fns_kwargs = {}

        fns_kwargs.update(kwargs)
        fns = self.FileNameSequencer(name, fns_kwargs)
        for key in set(fns.items).intersection(kwargs):
            kwargs.pop(key)
        return fns

    def get_fh(self, name, mode, kwargs={}):
        """Ensure name is a filehandle, opening it if necessary."""
        if mode == 'wb' and self.is_sequence(name):
            raise ValueError(f"{self.fmt} does not support writing to a "
                             f"sequence or template in binary mode.")

        if self.is_fh(name):
            return name

        if self.is_template(name):
            name = self.get_fns(name, mode, kwargs)

        open_kwargs = {'mode': (mode[0].replace('w', 'w+')
                                + mode[1].replace('s', 'b'))}
        if self.is_sequence(name):
            opener = sf.open
            if 'file_size' in kwargs:
                open_kwargs['file_size'] = kwargs.pop('file_size')

        else:
            opener = io.open

        return opener(name, **open_kwargs)

    def __call__(self, name, mode='rs', **kwargs):
        """
        Open baseband file(s) for reading or writing.

        Opened as a binary file, one gets a wrapped filehandle that adds
        methods to read/write a frame.  Opened as a stream, the handle is
        wrapped further, with methods such as reading and writing to the file
        as if it were a stream of samples.

        Parameters
        ----------
        name : str or filehandle, or sequence of str
            File name, filehandle, sequence of file names, or template
            (file name(s) can be `~pathlib.Path` but template has to be `str`).
        mode : {'rb', 'wb', 'rs', or 'ws'}, optional
            Whether to open for reading or writing, and as a regular binary
            file or as a stream. Default: 'rs', for reading a stream.
        **kwargs
            Additional arguments when opening the file as a stream.
        """
        mode = self.normalize_mode(mode)
        if mode == 'ws':
            # Stream writing always needs a header.  Construct from
            # other keywords if necessary.
            kwargs['header0'] = self.get_header0(kwargs)

        fh = self.get_fh(name, mode, kwargs)
        try:
            return self.classes[mode](fh, **kwargs)
        except Exception:
            if fh is not name:
                fh.close()
            raise

    def wrapped(self, module=None, doc=None):
        """Wrap as a function named open, replacing docstring and module."""

        @functools.wraps(self.__call__)
        def open(*args, **kwargs):
            return self(*args, **kwargs)

        if doc:
            open.__doc__ = doc

        # This ensures the function becomes visible to sphinx.
        if module:
            open.__module__ = module

        return open

    @classmethod
    def create(cls, ns, doc=None):
        """Create a standard opener for the given namespace.

        This assumes that the namespace contains file and stream readers
        and writers, as well as a header class, with standard names,
        ``<fmt>FileReader``, ``<fmt>FileWriter``, ``<fmt>StreamReader``,
        ``<fmt>StreamWriter``, and ``<fmt>Header``, where ``fmt`` is the
        name of the format (which is inferred by looking for a
        ``*StreamReader`` entry).

        The opener is instantiated using the format and the above classes,
        and then a wrapping function is created with ``__module__`` set to
        the ``__name__`` of the namespace, and with the documentation of
        its ``__call__`` method extended with ``doc``.

        Parameters
        ----------
        ns : dict
            Namespace to look in.  Generally, pass in ``globals()`` at the
            call site.
        doc : str, optional
            Extra documentation to add to that of the opener's ``__call__``
            method.
        """
        module = ns.get('__name__', None)
        for key in ns:
            if key.endswith('StreamReader'):
                fmt = key.replace('StreamReader', '')
                break
        else:  # noqa
            raise ValueError('namespace does not contain a StreamReader, '
                             'so fmt cannot be guessed.')

        classes = {mode: ns[fmt + cls_type] for (mode, cls_type) in {
            'rb': 'FileReader',
            'wb': 'FileWriter',
            'rs': 'StreamReader',
            'ws': 'StreamWriter'}.items()}
        header_class = ns.get(fmt+'Header')
        opener = cls(fmt, classes, header_class)
        if doc is not None:
            doc = textwrap.dedent(opener.__call__.__doc__) + doc
            if (opener.__call__.__doc__ is FileOpener.__call__.__doc__):
                doc = doc.replace(
                    'Open baseband file(s) for reading or writing.',
                    f'Open {fmt} file(s) for reading or writing.')
        return opener.wrapped(module=module, doc=doc) class FileInfo:
    """File information collector.

    The instance can be used as a function on a file name to get
    information from that file, by opening it and retrieving ``info``.

    Parameters
    ----------
    opener : callable
        The function to use to open files

    Notes
    -----
    The class is perhaps most easily used via the class method
    `~baseband.base.base.FileInfo.create`.
    """

    def __init__(self, opener):
        self.open = opener

    def _get_info(self, name, mode, **kwargs):
        """Open a file in the given mode and retrieve info."""
        try:
            with self.open(name, mode=mode, **kwargs) as fh:
                return fh.info
        except Exception as exc:
            if isinstance(exc, FileNotFoundError):
                raise
            return exc

    def is_ok(self, info):
        """Wether the item returned by _get_info has valid information."""
        return not isinstance(info, Exception) and info

    def get_file_info(self, name, **kwargs):
        """Open a file in binary mode and retrieve info.

        Any keyword arguments that were required to open the file will
        be stored as a ``used_kwargs`` attribute on the returned ``info``.

        Parameters
        ----------
        name : str or filehandle
            Item to be opened for reading in binary mode.
        **kwargs
            Any keyword arguments that might be required to open the
            file successfully (e.g., ``decade`` for Mark 4).

        Returns
        -------
        info : `~baseband.base.file_info.FileReaderInfo`
            Information on the file.  Will evaluate as `False` if the
            file was not in the right format.

        Raises
        ------
        FileNotFoundError
            If the file does not exist.

        Notes
        -----
        Getting information for an existing file should never fail. If an
        `Exception` is raised or returned, it is a bug in the file reader.
        """
        info = self._get_info(name, 'rb')
        # If right format, check if arguments were missing.
        if self.is_ok(info):
            used_kwargs = {key: kwargs[key] for key in info.missing
                           if key in kwargs}
            if used_kwargs:
                info2 = self._get_info(name, mode='rb', **used_kwargs)
                if self.is_ok(info2):
                    info = info2
                else:
                    # An exception due to the arguments must have occurred.
                    info.missing = {k: v for (k, v) in kwargs.items()
                                    if k not in used_kwargs}
                    info.errors[f"kwargs={kwargs}"] = info2

            info.used_kwargs = used_kwargs

        return info

    def get_stream_info(self, name, file_info, **kwargs):
        """Open a file in stream mode and retrieve info.

        Any keyword arguments that were required to open the file will
        be stored as a ``used_kwargs`` attribute on the returned ``info``.

        Parameters
        ----------
        name : str or filehandle
            Item to be opened for reading in stream mode.
        file_info : `~baseband.base.file_info.FileReaderInfo`
            Information gleaned from opening in binary mode.
        **kwargs
            Any keyword arguments that might be required to open the
            file successfully (e.g., ``decade`` for Mark 4).

        Returns
        -------
        info : `~baseband.base.file_info.StreamReaderInfo`
            Information on the file.  Will evaluate as `False` if the
            file was not in the right format. Will return `None` if no
            sample rate information was present, or an `Exception` if
            the opening as a stream failed.
        """
        frame_rate = file_info.frame_rate
        used_kwargs = file_info.used_kwargs.copy()

        # Somewhat inelegant special-cases for sample_rate and verify.
        if frame_rate is None:
            if 'sample_rate' in kwargs:
                used_kwargs['sample_rate'] = kwargs['sample_rate']
            else:
                # frame rate will already be marked as missing in
                # file_info.
                return None

        if 'verify' in kwargs:
            used_kwargs['verify'] = kwargs['verify']

        stream_info = self._get_info(name, mode='rs', **used_kwargs)
        if self.is_ok(stream_info):
            stream_info.used_kwargs = used_kwargs

        return stream_info

    def __call__(self, name, **kwargs):
        """Collect baseband file information.

        First try opening as a binary file and check whether the file is
        of the correct format. If so, and no required information is missing,
        re-open as a stream, and get information like the start time,
        sample rate, etc.

        Parameters
        ----------
        name : str or filehandle, or sequence of str
            File name, filehandle, or sequence of file names.
        **kwargs
            Any other arguments the opener needs to open as a stream.

        Returns
        -------
        info
            :class:`~baseband.base.file_info.FileReaderInfo` or
            :class:`~baseband.base.file_info.StreamReaderInfo`.
            In addition to the normal ``info`` attributes, also stored
            are attributes about what happened to the keyword arguments:
            ``used_kwargs``, ``consistent_kwargs``, ``inconsistent_kwargs``
            and ``irrelevant_kwargs``.
        """
        # NOTE: getting info should never fail or even emit warnings.
        # Hence, warnings or errors should not be suppressed here, but
        # rather in the info implementations.
        file_info = self.get_file_info(name, **kwargs)
        if not file_info or file_info.missing:
            return file_info

        stream_info = self.get_stream_info(name, file_info, **kwargs)
        if not self.is_ok(stream_info):
            if isinstance(stream_info, Exception):
                # Unexpected errors.  Put it in file_info so there is a record.
                file_info.errors['stream'] = str(stream_info)
            return file_info

        self.check_consistency(stream_info, **kwargs)
        return stream_info

    def check_consistency(self, info, **kwargs):
        """Check consistency between info and the given arguments.

        The keyword arguments will be sorted into those that were used
        by the file opener and those that were unused, with the latter
        split in those that had consistent, inconsistent, or irrelevant
        information.  They are stored on the ``info`` instance in
        ``used_kwargs``, ``consistent_kwargs``, ``inconsistent_kwargs``
        and ``irrelevant_kwargs`` attributes, respectively.

        Parameters
        ----------
        info : `~baseband.base.file_info.StreamReaderInfo`
            Information gleaned from a file opened in stream reading mode.
        **kwargs
            Keyword arguments passed to the opener.
        """
        # Store what happened to the kwargs, so one can decide if there are
        # inconsistencies or other problems.
        info.consistent_kwargs = {}
        info.inconsistent_kwargs = {}
        info.irrelevant_kwargs = {}
        for key, value in kwargs.items():
            if key in info.used_kwargs:
                continue

            consistent = self.check_key(key, value, info)

            if consistent is None:
                info.irrelevant_kwargs[key] = value
            elif consistent:
                info.consistent_kwargs[key] = value
            else:
                info.inconsistent_kwargs[key] = value

        return info

    def check_key(self, key, value, info):
        """Check consistency for a given key and value.

        Parameters
        ----------
        key : str
            Name of the key.
        value : object
            Corresponding value.
        info : `~baseband.base.file_info.StreamReaderInfo`
            Information collected by opening a file in stream reader mode.

        Returns
        -------
        consistent : True, False, or None
            Whether the information on ``info`` for ``key`` is consistent
            with ``value``.  `None` if it could not be determined.
        """
        info_value = getattr(info, key, None)
        if info_value is None:
            info_value = getattr(info.file_info, key, None)

        if info_value is not None:
            return info_value == value

        if key == 'nchan':
            sample_shape = info.shape[1:]
            if sample_shape is not None:
                # If we passed nchan, and info doesn't have it, but does have a
                # sample shape, check that consistency with that, either in
                # being equal to `sample_shape.nchan` or equal to the product
                # of all elements (e.g., a VDIF file with 8 threads and 1
                # channel per thread is consistent with nchan=8).
                return (getattr(sample_shape, 'nchan', -1) == value
                        or np.prod(sample_shape) == value)

        elif key in {'ref_time', 'kday', 'decade'}:
            start_time = info.start_time
            if start_time is not None:
                if key == 'ref_time':
                    return abs(value - start_time).jd < 500
                elif key == 'kday':
                    return int(start_time.mjd / 1000.) * 1000 == value
                else:  # decade
                    return int(start_time.isot[:3]) * 10 == value

        return None

    def wrapped(self, module=None, doc=None):
        """Wrap as a function named info, replacing docstring and module."""

        @functools.wraps(self.__call__)
        def info(*args, **kwargs):
            return self(*args, **kwargs)

        if doc:
            info.__doc__ = doc

        # This ensures the function becomes visible to sphinx.
        if module:
            info.__module__ = module

        return info

    @classmethod
    def create(cls, ns):
        """Create an info getter for the given namespace.

        This assumes that the namespace contains an ``open`` function, which
        is used to create an instance of the info class that is wrapped in a
        function with ``__module__`` set to the calling module (inferred
        from the namespace).

        Parameters
        ----------
        ns : dict
            Namespace to look in.  Generally, pass in ``globals()`` at the
            call site.
        """
        module = ns.get('__name__', None)
        for key in ns:
            if key.endswith('StreamReader'):
                fmt = key.replace('StreamReader', '')
                break
        else:  # pragma: no cover
            fmt = None

        opener = ns['open']
        info = cls(opener)
        doc = textwrap.dedent(info.__call__.__doc__)
        if (fmt is not None
                and info.__call__.__doc__ is FileInfo.__call__.__doc__):
            doc = doc.replace(
                'Collect baseband file information.',
                f'Collect {fmt} file information.')
        return info.wrapped(module=module, doc=doc) class FileReaderInfo(InfoBase):
    """Standardized information on file readers.

    The ``info`` descriptor has a number of standard attributes, which are
    determined from arguments passed in opening the file, from the first header
    (``info.header0``) and from possibly scanning the file to determine the
    duration of frames.

    Examples
    --------
    The most common use is simply to print information::

        >>> from baseband.data import SAMPLE_MARK5B
        >>> from baseband import mark5b
        >>> fh = mark5b.open(SAMPLE_MARK5B, 'rb')
        >>> fh.info
        Mark5BFile information:
        format = mark5b
        number_of_frames = 4
        frame_rate = 6400.0 Hz
        bps = 2
        complex_data = False
        readable = False
        <BLANKLINE>
        missing:  nchan: needed to determine sample shape, frame rate, ...
                  kday, ref_time: needed to infer full times.

        >>> fh.close()

        >>> fh = mark5b.open(SAMPLE_MARK5B, 'rb', kday=56000, nchan=8)
        >>> fh.info
        Mark5BFile information:
        format = mark5b
        number_of_frames = 4
        frame_rate = 6400.0 Hz
        sample_rate = 32.0 MHz
        samples_per_frame = 5000
        sample_shape = (8,)
        bps = 2
        complex_data = False
        start_time = 2014-06-13T05:30:01.000000000
        readable = True
        <BLANKLINE>
        checks:  decodable: True
        >>> fh.close()
    """
    attr_names = ('format', 'number_of_frames', 'frame_rate', 'sample_rate',
                  'samples_per_frame', 'sample_shape', 'bps', 'complex_data',
                  'start_time', 'readable',
                  'missing', 'checks', 'errors', 'warnings')
    """Attributes that the container provides."""

    samples_per_frame = info_item(needs='header0', doc=(
        'Number of complete samples in each frame.'))
    sample_shape = info_item(needs='header0', doc=(
        'Shape of each complete sample (e.g., ``(nchan,)``).'))
    bps = info_item(needs='header0', doc=(
        'Number of bits used to encode each elementary sample.'))
    complex_data = info_item(needs='header0', doc=(
        'Whether the data are complex.'))
    start_time = info_item('time', needs='header0', doc=(
        "Time of the first sample."))

    missing = info_item(default={}, copy=True,
                        doc='dict of missing attributes.')
    checks = info_item(default={}, copy=True,
                       doc='dict of checks for readability.')
    errors = info_item(default={}, copy=True,
                       doc='dict of attributes that raised errors.')
    warnings = info_item(default={}, copy=True,
                         doc='dict of attributes that gave warnings.')

    @info_item
    def header0(self):
        """Header of the first frame in the file."""
        # Here, we do not even know whether the file is open or whether we
        # have the right format. We thus use a try/except and filter out all
        # warnings.
        with self._parent.temporary_offset(0) as fh:
            with warnings.catch_warnings():
                warnings.simplefilter('ignore')
                return fh.read_header()

    @info_item(needs='header0')
    def frame0(self):
        """First frame from the file."""
        # Try reading a frame.  This has no business failing if a
        # frame rate could be determined, but try anyway; maybe file is closed.
        with self._parent.temporary_offset(0) as fh:
            return fh.read_frame()

    @info_item(needs='frame0', default=False)
    def decodable(self):
        """Whether decoding the first frame worked."""
        # Getting the first sample can fail if we don't have the right decoder.
        self.frame0[0]
        return True

    @info_item(needs='header0')
    def format(self):
        """The file format."""
        return self._parent.__class__.__name__.split('File')[0].lower()

    @info_item(needs='header0')
    def frame_rate(self):
        """Number of frames per unit time."""
        return self._parent.get_frame_rate()

    @info_item(needs='header0')
    def number_of_frames(self):
        """Total number of frames."""
        with self._parent.temporary_offset() as fh:
            file_size = fh.seek(0, 2)

        number_of_frames = file_size / self.header0.frame_nbytes
        if number_of_frames % 1 == 0:
            return int(number_of_frames)
        else:
            self.warnings['number_of_frames'] = (
                f"file contains non-integer number "
                f"({number_of_frames}) of frames")
            return None

    @info_item(needs='frame0', default=False)
    def readable(self):
        """Whether the file is readable and decodable."""
        self.checks['decodable'] = self.decodable
        return all(bool(v) for v in self.checks.values())

    @info_item(needs=('frame_rate', 'samples_per_frame'))
    def sample_rate(self):
        """Rate of complete samples per unit time."""
        return self.frame_rate * self.samples_per_frame def lcm(a, b):
    """Calculate the least common multiple of a and b."""
    return abs(a * b) // gcd(a, b) class DADAHeader(OrderedDict):
    """DADA baseband file format header.

    Defines a number of routines common to all baseband format headers.

    Parameters
    ----------
    *args : str or iterable
        If a string, parsed as a DADA header from a file, otherwise
        as for the OrderedDict baseclass.
    verify : bool, optional
        Whether to do minimal verification that the header is consistent with
        the DADA standard.  Default: `True`.
    mutable : bool, optional
        Whether to allow the header to be changed after initialisation.
        Default: `True`.
    **kwargs
        Any further header keywords to be set.  If any value is a 2-item tuple,
        the second one will be considered a comment.

    Notes
    -----
    Like `~collections.OrderedDict`, in order to ensure keywords are kept in
    the right order, one should pass on values as a tuple, not as a dict.
    E.g., to copy a header, one should not do ``DADAHeader(**header)``, but
    rather::

        DADAHeader(((key, header[key]) for key in header))

    or, to also keep the comments::

        DADAHeader(((key, (header[key], header.comments[key]))
                   for key in header))
    """

    _properties = ('payload_nbytes', 'frame_nbytes', 'bps', 'complex_data',
                   'sample_shape', 'sample_rate', 'sideband',
                   'tsamp', 'samples_per_frame', 'offset', 'start_time',
                   'time')
    """Properties accessible/usable in initialisation for all headers."""

    _defaults = [('HEADER', 'DADA'),
                 ('HDR_VERSION', '1.0'),
                 ('HDR_SIZE', 4096),
                 ('DADA_VERSION', '1.0'),
                 ('OBS_ID', 'unset'),
                 ('PRIMARY', 'unset'),
                 ('SECONDARY', 'unset'),
                 ('FILE_NAME', 'unset'),
                 ('FILE_NUMBER', 0),
                 ('FILE_SIZE', 0),
                 ('OBS_OFFSET', 0),
                 ('OBS_OVERLAP', 0),
                 ('SOURCE', 'unset'),
                 ('TELESCOPE', 'unset'),
                 ('INSTRUMENT', 'unset'),
                 ('RECEIVER', 'unset'),
                 ('NBIT', 8),
                 ('NDIM', 1),
                 ('NPOL', 1),
                 ('NCHAN', 1),
                 ('RESOLUTION', 1),
                 ('DSB', 1)]

    def __init__(self, *args, verify=True, mutable=True, **kwargs):
        self.mutable = True
        self.comments = {}
        if len(args) == 1 and isinstance(args[0], str):
            args = (self._fromlines(args[0].split('\n')),)

        super().__init__(*args, **kwargs)
        self.mutable = mutable
        if verify and (args or kwargs):
            self.verify()

    def verify(self):
        """Basic check of integrity."""
        assert self['HEADER'] == 'DADA'
        assert all(key in self for key in ('HDR_VERSION',
                                           'HDR_SIZE',
                                           'DADA_VERSION'))

    def copy(self):
        """Create a mutable and independent copy of the header."""
        # Cannot do super().copy(), since this first initializes an empty
        # header, which does not pass verification.
        new = self.__class__(self)
        new.comments = self.comments.copy()
        new.mutable = True
        return new

    def __copy__(self):
        return self.copy()

    @staticmethod
    def _fromlines(lines):
        """Interpret a list of lines as a header, converting its values."""
        args = []
        for line_no, line in enumerate(lines):
            split = line.strip().split('#')
            comment = split[1].strip() if (len(split) > 1
                                           and split[1]) else None
            split = split[0].strip().split() if split else []
            key = split[0] if split and split[0] else '_{0:d}'.format(line_no)
            value = split[1] if (len(split) > 1 and split[1]) else None

            if key in ('FILE_SIZE', 'FILE_NUMBER', 'HDR_SIZE',
                       'OBS_OFFSET', 'OBS_OVERLAP',
                       'NBIT', 'NDIM', 'NPOL', 'NCHAN', 'RESOLUTION', 'DSB'):
                value = int(value)

            elif key in ('FREQ', 'BW', 'TSAMP'):
                value = float(value)

            args.append((key, (value, comment)))

        return args

    def _tolines(self):
        """Write header to a list of strings."""
        lines = []
        for key in self:
            value = self[key]
            comment = self.comments.get(key, None)
            if value is not None:
                if comment is not None:
                    line = '{0} {1} # {2}'.format(key, value, comment)
                else:
                    line = '{0} {1}'.format(key, value)
            else:
                if comment is not None:
                    line = '# {0}'.format(comment)
                else:
                    line = ''
            lines.append(line)
        return lines

    @classmethod
    def fromfile(cls, fh, verify=True):
        """
        Reads in DADA header block from a file.

        The file pointer should be at the start.

        Parameters
        ----------
        fh : filehandle
            To read data from.
        verify: bool, optional
            Whether to do basic checks on whether the header is valid.
            Default: `True`.
        """
        start_pos = fh.tell()
        hdr_size = 4096
        lines = []
        # Headers are of given length and usually have a comment with
        # "end of header", but they can also end abruptly, with the
        # remainder filled with zeros.
        while fh.tell() - start_pos < hdr_size and fh.peek(1)[:1] != b'\x00':
            line = fh.readline().decode('ascii')
            if line == '':  # empty lines are '\n'
                raise EOFError

            if line[0] == '#' and 'end of header' in line:
                break

            if line.startswith('HDR_SIZE'):
                hdr_size = int(line.split()[1])

            lines.append(line)

        if fh.tell() - start_pos > hdr_size:
            warnings.warn("Odd, read {0} bytes while the header size is {1}"
                          .format(fh.tell(), hdr_size))
        else:
            fh.seek(start_pos + hdr_size)

        return cls(cls._fromlines(lines), verify=verify, mutable=False)

    def tofile(self, fh):
        """Write DADA file header to filehandle.

        Parts of the header beyond the ascii lines are filled with 0x00.
        Note that file should in principle be at the start, but we don't check
        for that since that would break SequentialFileWriter.
        """
        start_pos = fh.tell()
        with io.BytesIO() as s:
            for line in self._tolines():
                s.write((line + '\n').encode('ascii'))
            s.write('# end of header\n'.encode('ascii'))
            extra = self.nbytes - s.tell()
            if extra < 0:
                raise ValueError("cannot write header in allocated size of "
                                 "{0}".format(self.nbytes))
            s.seek(0)
            fh.write(s.read())
            if extra:
                fh.write(b'\00' * extra)
            assert fh.tell() - start_pos == self.nbytes

    @classmethod
    def fromkeys(cls, *args, **kwargs):
        """Initialise a header from keyword values.

        Like fromvalues, but without any interpretation of keywords.

        This just calls the class initializer; it is present for compatibility
        with other header classes only.
        """
        # Setting one kwarg ensures we do not return an empty header.
        if not args:
            kwargs.setdefault('HEADER', 'DADA')
        return cls(*args, **kwargs)

    @classmethod
    def fromvalues(cls, **kwargs):
        """Initialise a header from parsed values.

        Here, the parsed values must be given as keyword arguments, i.e., for
        any ``header``, ``cls.fromvalues(**header) == header``.

        However, unlike for the ``fromkeys`` class method, data can also be set
        using arguments named after header methods, such as ``time``.

        Furthermore, some header defaults are set in ``DADAHeader._defaults``.
        """
        self = cls(cls._defaults, verify=False)
        self.update(**kwargs)
        return self

    def update(self, *, verify=True, **kwargs):
        """Update the header with new values.

        Here, any keywords matching properties are processed as well, in the
        order set by the class (in ``_properties``), and after all other
        keywords have been processed.

        Parameters
        ----------
        verify : bool, optional
            If `True` (default), verify integrity after updating.
        **kwargs
            Arguments used to set keywords and properties.
        """
        # Remove kwargs that set properties, in correct order.
        extras = [(key, kwargs.pop(key)) for key in self._properties
                  if key in kwargs]
        # Update the normal keywords.
        super().update(**kwargs)
        # Now set the properties.
        for attr, value in extras:
            setattr(self, attr, value)
        if verify:
            self.verify()

    def __setitem__(self, key, value):
        if not self.mutable:
            raise TypeError("immutable {0} does not support assignment."
                            .format(type(self).__name__))
        if isinstance(value, tuple):
            value, comment = value
            self.comments[key.upper()] = comment

        super().__setitem__(key.upper(), value)

    @property
    def nbytes(self):
        """Size of the header in bytes."""
        return self['HDR_SIZE']

    @property
    def payload_nbytes(self):
        """Size of the payload in bytes."""
        return self['FILE_SIZE']

    @payload_nbytes.setter
    def payload_nbytes(self, payload_nbytes):
        self['FILE_SIZE'] = payload_nbytes

    @property
    def frame_nbytes(self):
        """Size of the frame in bytes."""
        return self.nbytes + self.payload_nbytes

    @frame_nbytes.setter
    def frame_nbytes(self, frame_nbytes):
        self.payload_nbytes = frame_nbytes - self.nbytes

    @property
    def bps(self):
        """Bits per elementary sample."""
        return self['NBIT']

    @bps.setter
    def bps(self, bps):
        self['NBIT'] = bps

    @property
    def complex_data(self):
        """Whether the data are complex."""
        return self['NDIM'] == 2

    @complex_data.setter
    def complex_data(self, complex_data):
        self['NDIM'] = 2 if complex_data else 1

    @property
    def sample_shape(self):
        """Shape of a sample in the payload (npol, nchan)."""
        return self['NPOL'], self['NCHAN']

    @sample_shape.setter
    def sample_shape(self, sample_shape):
        self['NPOL'], self['NCHAN'] = sample_shape

    @property
    def sample_rate(self):
        """Number of complete samples per second.

        Can be set with a negative quantity to set `sideband`.
        """
        return (1. / self['TSAMP']) * u.MHz

    @sample_rate.setter
    def sample_rate(self, sample_rate):
        sample_rate = sample_rate.to_value(u.MHz)
        self['TSAMP'] = 1. / abs(sample_rate)
        bw = sample_rate * self['NCHAN'] / (1 if self.complex_data else 2)
        self['BW'] = (-1 if self.get('BW', bw) < 0 else 1) * bw

    @property
    def sideband(self):
        """True if upper sideband."""
        return self['BW'] > 0

    @sideband.setter
    def sideband(self, sideband):
        self['BW'] = (1 if sideband else -1) * abs(self['BW'])

    @property
    def samples_per_frame(self):
        """Number of complete samples in the frame."""
        return (self.payload_nbytes * 8
                // (self.bps * (2 if self.complex_data else 1))
                // self['NPOL']
                // self['NCHAN'])

    @samples_per_frame.setter
    def samples_per_frame(self, samples_per_frame):
        old_payload_nbytes = self.payload_nbytes
        self.payload_nbytes = int(
            (samples_per_frame * self['NCHAN'] * self['NPOL']
             * self.bps * (2 if self.complex_data else 1) + 7) // 8)
        if self.samples_per_frame != samples_per_frame:
            exc = ValueError("header cannot store {} samples per frame. "
                             "Nearest is {}.".format(samples_per_frame,
                                                     self.samples_per_frame))
            self.payload_nbytes = old_payload_nbytes
            raise exc

    @property
    def offset(self):
        """Offset from start of observation in units of time."""
        return ((self['OBS_OFFSET'] * 8
                 // (self['NBIT'] * self['NDIM']
                     * self['NPOL'] * self['NCHAN']))
                * self['TSAMP'] * u.us)

    @offset.setter
    def offset(self, offset):
        self['OBS_OFFSET'] = (int((offset / (self['TSAMP'] * u.us))
                                  .to(u.one).round())
                              * ((self['NBIT'] * self['NDIM']
                                  * self['NPOL'] * self['NCHAN'] + 7) // 8))

    @property
    def start_time(self):
        """Start time of the observation."""
        mjd_int, frac = self['MJD_START'].split('.')
        mjd_int = int(mjd_int)
        frac = float('.' + frac)
        # replace '-' between date and time with a 'T' and convert to Time
        return Time(mjd_int, frac, scale='utc', format='mjd')

    @start_time.setter
    def start_time(self, start_time):
        start_time = Time(start_time, scale='utc', format='isot', precision=9)
        self['UTC_START'] = (start_time.isot.replace('T', '-')
                             .replace('.000000000', ''))
        mjd_int = int(start_time.mjd)
        mjd_frac = (start_time - Time(mjd_int, format='mjd',
                                      scale=start_time.scale)).jd
        if mjd_frac < 0:
            mjd_int -= 1
            mjd_frac += 1.
        self['MJD_START'] = ('{0:05d}'.format(mjd_int)
                             + ('{0:17.15f}'.format(mjd_frac))[1:])

    @property
    def time(self):
        """Start time of the part of the observation covered by this header."""
        return self.start_time + self.offset

    @time.setter
    def time(self, time):
        """Set header time.

        If ``start_time`` is not already set, this sets it using the time and
        ``offset``.  Otherwise, this sets ``offset`` using the time and
        ``start_time``.

        Parameters
        ----------
        time : `~astropy.time.Time`
            Time for the first sample associated with this header.
        """
        if 'MJD_START' not in self.keys():
            self.start_time = time - self.offset
        else:
            self.offset = time - self.start_time

    def _ipython_key_completions_(self):
        # Enables tab-completion of header keys in IPython.
        return self.keys()

    def __eq__(self, other):
        """Whether headers have the same keys with the same values."""
        # We do a float conversion for MJD_START, since headers often give
        # more digits than can really be stored.
        return (all(self.get(k, None) == other.get(k, None)
                    for k in (set(self.keys()) | set(other.keys()))
                    if not k.startswith('_') and k != 'MJD_START')
                and (float(self.get('MJD_START', 0.))
                     == float(other.get('MJD_START', 0.))))

    def __repr__(self):
        return('{0}("""'.format(self.__class__.__name__)
               + '\n'.join(self._tolines())
               + '""")') class DADAPayload(PayloadBase):
    """Container for decoding and encoding DADA payloads.

    Parameters
    ----------
    words : `~numpy.ndarray`
        Array containg LSB unsigned words (with the right size) that
        encode the payload.
    header : `~baseband.dada.DADAHeader`
        Header that provides information about how the payload is encoded.
        If not given, the following arguments have to be passed in.
    bps : int, optional
        Number of bits per sample part (i.e., per channel and per real or
        imaginary component).  Default: 8.
    sample_shape : tuple, optional
        Shape of the samples; e.g., (nchan,).  Default: ().
    complex_data : bool, optional
        Whether data are complex.  Default: `False`.
    """
    _decoders = {
        8: decode_8bit}
    _encoders = {
        8: encode_8bit}
    _memmap = True
    _sample_shape_maker = namedtuple('SampleShape', 'npol, nchan') class DADAFrame(FrameBase):
    """Representation of a DADA file, consisting of a header and payload.

    Parameters
    ----------
    header : `~baseband.dada.DADAHeader`
        Wrapper around the header lines, providing access to the values.
    payload : `~baseband.dada.DADAPayload`
        Wrapper around the payload, provding mechanisms to decode it.
    valid : bool, optional
        Whether the data are valid.  Default: `True`.
    verify : bool, optional
        Whether to do basic verification of integrity.  Default: `True`.

    Notes
    -----
    DADA files do not support storing whether data are valid or not on disk.
    Hence, this has to be determined independently.  If ``valid=False``, any
    decoded data are set to ``cls.fill_value`` (by default, 0).

    The Frame can also be instantiated using class methods:

      fromfile : read header and map or read payload from a filehandle

      fromdata : encode data as payload

    Of course, one can also do the opposite:

      tofile : method to write header and payload to filehandle

      data : property that yields full decoded payload

    One can decode part of the payload by indexing or slicing the frame.

    A number of properties are defined: `shape`, `dtype` and `size` are
    the shape, type and number of complete samples of the data array, and
    `nbytes` the frame size in bytes.  Furthermore, the frame acts as a
    dictionary, with keys those of the header.  Any attribute that is not
    defined on the frame itself, such as ``.time`` will be looked up on the
    header as well.
    """
    _header_class = DADAHeader
    _payload_class = DADAPayload
import io
import re
import operator
import astropy.units as u
from functools import reduce
from astropy.utils import lazyproperty
from ..helpers import sequentialfile as sf
from ..base.base import (
    FileBase,
    StreamReaderBase, StreamWriterBase,
    FileOpener, FileInfo)
from ..base.file_info import FileReaderInfo
from ..base.utils import lcm
from .header import DADAHeader
from .payload import DADAPayload
from .frame import DADAFrame
            self._file_size = header['FILE_SIZE']

    def _process_items(self, file_nr):
        super()._process_items(file_nr)
        # Pop file_nr, as we need to capitalize it.
        file_nr = self.items.pop('file_nr')
        self.items['FRAME_NR'] = self.items['FILE_NR'] = file_nr
        if self._has_obs_offset:
            self.items['OBS_OFFSET'] = (self._obs_offset0
                                        + file_nr * self._file_size)


class DADAFileReader(FileBase):
    """Simple reader for DADA files.

    Wraps a binary filehandle, providing methods to help interpret the data,
    such as `read_frame` and `get_frame_rate`. By default, frame payloads
    are mapped rather than fully read into physical memory.

    Parameters
    ----------
    fh_raw : filehandle
        Filehandle of the raw binary data file.
    """
    info = FileReaderInfo()

    def read_header(self):
        """Read a single header from the file.

        Returns
        -------
        header : `~baseband.dada.DADAHeader`
        """
        return DADAHeader.fromfile(self.fh_raw)

    def read_frame(self, memmap=True, verify=True):
        """Read the frame header and read or map the corresponding payload.

        Parameters
        ----------
        memmap : bool, optional
            If `True` (default), map the payload using `~numpy.memmap`, so that
            parts are only loaded into memory as needed to access data.
        verify : bool, optional
            Whether to do basic checks of frame integrity.  Default: `True`.

        Returns
        -------
        frame : `~baseband.dada.DADAFrame`
            With ``.header`` and ``.payload`` properties.  The ``.data``
            property returns all data encoded in the frame.  Since this may
            be too large to fit in memory, it may be better to access the
            parts of interest by slicing the frame.
        """
        return DADAFrame.fromfile(self.fh_raw, memmap=memmap, verify=verify)

    def get_frame_rate(self):
        """Determine the number of frames per second.

        The routine uses the sample rate and number of samples per frame
        from the first header in the file.

        Returns
        -------
        frame_rate : `~astropy.units.Quantity`
            Frames per second.
        """
        with self.temporary_offset(0):
            header = self.read_header()
        return (header.sample_rate / header.samples_per_frame).to(u.Hz)


class DADAFileWriter(FileBase):
    """Simple writer/mapper for DADA files.

    Adds `write_frame` and `memmap_frame` methods to the binary file wrapper.
    The latter allows one to encode data in pieces, writing to disk as needed.
    """

    def write_frame(self, data, header=None, **kwargs):
        """Write a single frame (header plus payload).

        Parameters
        ----------
        data : `~numpy.ndarray` or `~baseband.dada.DADAFrame`
            If an array, a ``header`` should be given, which will be used to
            get the information needed to encode the array, and to construct
            the DADA frame.
        header : `~baseband.dada.DADAHeader`
            Can instead give keyword arguments to construct a header.  Ignored
            if ``data`` is a `~baseband.dada.DADAFrame` instance.
        **kwargs
            If ``header`` is not given, these are used to initialize one.
        """
        if not isinstance(data, DADAFrame):
            data = DADAFrame.fromdata(data, header, **kwargs)
        return data.tofile(self.fh_raw)

    def memmap_frame(self, header=None, **kwargs):
        """Get frame by writing the header to disk and mapping its payload.

        The header is written to disk immediately, but the payload is mapped,
        so that it can be filled in pieces, by setting slices of the frame.

        Parameters
        ----------
        header : `~baseband.dada.DADAHeader`
            Written to disk immediately.  Can instead give keyword arguments to
            construct a header.
        **kwargs
            If ``header`` is not given, these are used to initialize one.

        Returns
        -------
        frame: `~baseband.dada.DADAFrame`
            By assigning slices to data, the payload can be encoded piecewise.
        """
        if header is None:
            header = DADAHeader.fromvalues(**kwargs)
        header.tofile(self.fh_raw)

        payload = DADAPayload.fromfile(self.fh_raw, memmap=True, header=header)