class CoordList(list):
    """All the functionality of a standard :class:`list` with added :class:`Coord` context."""

    def __init__(self, *args):
        """ Given many :class:`Coord`s, return a :class:`CoordList` instance. """
        list.__init__(self, *args)

        # Check that all items in the incoming list are coords. Note that this checking
        # does not guarantee that a CoordList instance *always* has just coords in its list as
        # the append & __getitem__ methods have not been overridden.
        if not all([isinstance(coord, Coord) for coord in self]):
            raise ValueError('All items in list_of_coords must be Coord instances.')

    def append(self, other):
        """
        Safely add a new coordinate object to the list, this checks for a unique :attr:`axis` and :attr:`standard_name`.

        :param :class:`Coord` other: Other coord to add
        :raises DuplicateCoordinateError: If the coordinate is not unique in the list
        """
        from cis.exceptions import DuplicateCoordinateError
        if any([other == item for item in self]):
            raise DuplicateCoordinateError()
        super(CoordList, self).append(other)

    def get_coords(self, name_or_coord=None, standard_name=None, long_name=None, attributes=None, axis=None,
                   var_name=None):
        """
        Return a list of coordinates in this :class:`CoordList` fitting the given criteria. This is deliberately very
        similar to :func:`Cube.coords()` to maintain a similar interface and because the functionality is similar. There
        is no distinction between dimension coordinates and auxiliary coordinates here though.

        :param name_or_coord: This should be either: The standard name or long name or default name of the desired
         coordinate; Or, a :class:`Coord` instance whose metadata should be used for the search criteria (note that
         currently only the standard name is compared). If None, does not check for name. Also see, :attr:`Cube.name`.
        :type name: string or None or :class:`Coord`
        :param standard_name: The CF standard name of the desired coordinate. If None, does not check for standard name.
        :type standard_name: string or None
        :param long_name: An unconstrained description of the coordinate. If None, does not check for long_name.
        :type long_name: string or None
        :param attributes: A dictionary of attributes desired on the coordinates. If None, does not check for attributes
        :type attributes: dict or None
        :param axis: The desired coordinate axis, see :func:`iris.util.guess_coord_axis`. If None, does not check for
         axis. Accepts the values 'X', 'Y', 'Z' and 'T' (case-insensitive).
        :type axis: string or None
        :param var_name: The name of the variable which the coordinate was read from. If None, does not check for long_name.
        :type var_name: string or None

        :return: A :class:`CoordList` of coordinates fitting the given criteria
        """
        import six
        from collections.abc import Mapping
        coords = self

        if isinstance(name_or_coord, six.string_types):
            name = name_or_coord
            coord = None
        else:
            name = None
            coord = name_or_coord

        if name is not None:
            coords = [coord_ for coord_ in coords if coord_.name() == name]

        if standard_name is not None:
            coords = [coord_ for coord_ in coords if coord_.standard_name == standard_name]

        if long_name is not None:
            coords = [coord_ for coord_ in coords if coord_.long_name == long_name]

        if axis is not None:
            axis = axis.upper()
            coords = [coord_ for coord_ in coords if coord_.axis == axis]

        if var_name is not None:
            coords = [coord_ for coord_ in coords if coord_.var_name == var_name]

        if attributes is not None:
            if not isinstance(attributes, Mapping):
                raise ValueError(
                    'The attributes keyword was expecting a dictionary type, but got a %s instead.' % type(attributes))
            coords = [coord_ for coord_ in coords if all(
                            k in coord_.attributes and coord_.attributes[k] == v for k, v in attributes.items())]

        if coord is not None:
            coords = [coord_ for coord_ in coords if coord_ == coord]

        return coords

    def get_coord(self, name_or_coord=None, standard_name=None, long_name=None, attributes=None, axis=None,
                  var_name=None):
        """
        Return a single coord fitting the given criteria. This is deliberately very
        similar to :func:`Cube.coord()` method to maintain a similar interface and because the functionality is similar.
        There is no distinction between dimension coordinates and auxilliary coordinates here though.

        :param name_or_coord: This should be either: The standard name or long name or default name of the desired
         coordinate; Or, a :class:`Coord` instance whose metadata should be used for the search criteria (note that
         currently only the standard name is compared). If None, does not check for name. Also see, :attr:`Cube.name`.
        :type name: string or None or :class:`Coord`
        :param standard_name: The CF standard name of the desired coordinate. If None, does not check for standard name.
        :type standard_name: string or None
        :param long_name: An unconstrained description of the coordinate. If None, does not check for long_name.
        :type long_name: string or None
        :param attributes: A dictionary of attributes desired on the coordinates. If None, does not check for attributes
        :type attributes: dict or None
        :param axis: The desired coordinate axis, see :func:`iris.util.guess_coord_axis`. If None, does not check for
         axis. Accepts the values 'X', 'Y', 'Z' and 'T' (case-insensitive).
        :type axis: string or None
        :param var_name: The name of the variable which the coordinate was read from. If None, does not check for long_name.
        :type var_name: string or None

        :raises CoordinateNotFoundError: If the arguments given do not result in precisely
         1 coordinate being matched.
        :return: A single :class:`Coord`.

        """
        from cis.exceptions import CoordinateNotFoundError
        coords = self.get_coords(name_or_coord=name_or_coord, standard_name=standard_name, long_name=long_name,
                                 attributes=attributes, axis=axis, var_name=var_name)
        if len(coords) == 0:  # If we found none by name, try with standard name only
            coords = self.get_coords(standard_name=name_or_coord)

        if len(coords) > 1:
            msg = 'Expected to find exactly 1 coordinate, but found %s. They were: %s.' \
                  % (len(coords), ', '.join(coord.name() for coord in coords))
            raise CoordinateNotFoundError(msg)
        elif len(coords) == 0:
            bad_name = str(name_or_coord) or standard_name or long_name or axis or ''
            msg = 'Expected to find exactly 1 %s coordinate, but found none.' % bad_name
            raise CoordinateNotFoundError(msg)

        return coords[0]

    def get_coordinates_points(self):
        all_coords = self.find_standard_coords()
        flattened_coords = [(c.data_flattened if c is not None else None) for c in all_coords]
        return UngriddedHyperPointView(flattened_coords, None)

    def find_standard_coords(self):
        """Constructs a list of the standard coordinates.
        The standard coordinates are latitude, longitude, altitude, air_pressure and time; they occur in the return
        list in this order.

        :return: :class:`list` of coordinates or None if coordinate not present
        """
        from cis.exceptions import CoordinateNotFoundError

        ret_list = []

        for name in HyperPoint.standard_names:
            try:
                coord = self.get_coord(standard_name=name)
            except CoordinateNotFoundError:
                coord = None
            ret_list.append(coord)

        return ret_list

    def copy(self):
        """
        Create a copy of this CoordList object with new data so that that they can
        be modified without held references being affected. This will call any lazy loading methods in the coordinate
        data

        :return: Copied :class:`CoordList`
        """
        copied = CoordList()
        for coord in self:
            copied.append(coord.copy())
        return copied class InvalidVariableError(CISError):
    pass class FileFormatError(CISError):
    """
    Throw when there is an error determining the type of a file
    """
    error_list = ['Unknown error']

    def __init__(self, error_list, *args, **kwargs):
        super(FileFormatError, self).__init__(args, kwargs)
        self.error_list = error_list class AProduct(object):
    """
    Abstract class for the various possible data products. This just defines the interface which
    the subclasses must implement.
    """

    # If a filename matches two data products' file signatures, the data product with the higher priority will be used
    priority = 10

    # Contains a list of valid spatiotemporal variable names
    valid_dimensions = None

    @abstractmethod
    def create_data_object(self, filenames, variable):
        """
        Create and return an :class:`.CommonData` object for a given variable from one or more files.

        :param list filenames: List of filenames of files to read
        :param str variable: Variable to read from the files
        :return: An :class:`.CommonData` object representing the specified variable

        :raise FileIOError: Unable to read a file
        :raise InvalidVariableError: Variable not present in file
        """

    @abstractmethod
    def create_coords(self, filenames):
        """
        Reads the coordinates from one or more files. Note that this method may have to make certain assumptions about
        the file in order to return a single coordinate set. The user should be warned through the logger if this is the
        case.

        :param list filenames: List of filenames to read coordinates from
        :return: :class:`.CommonData` object
        """

    @abstractmethod
    def get_file_signature(self):
        """
        This method should return a list of regular expressions, which CIS uses to decide which data
        product to use for a given file. If more than one regular expression is provided in the list then the file can
        match `any` of the expressions. The first product with a signature that matches the filename will be used.
        The order in which the products are searched is determined by the priority property, highest value first;
        internal products generally have a priority of 10.

        For example, this would match all files with a name containing the string 'CODE' and with the 'nc' extension.::

            return [r'.*CODE*.nc']

        .. note::

            If the signature has matched the framework will call :meth:`.AProduct.get_file_type_error`, this gives the
            product a chance to open the file and check the contents.

        :return: A list of regex to match the product's file naming convention.
        :rtype: list
        """

    def get_variable_names(self, filenames, data_type=None):
        """
        Get a list of available variable names from the filenames list passed in. This general implementation can be
        overridden in specific products to include/exclude variables which may or may not be relevant.
        The data_type parameter can be used to specify extra information.

        :param list filenames: List of string filenames of files to be read from
        :param str data_type: 'SD' or 'VD' to specify only return SD or VD variables from HDF files. This may take on
         other values in specific product implementations.
        :return: A set of variable names as strings
        :rtype: str
        """
        variables = []
        for filename in filenames:
            file_variables = None
            if filename.endswith(".nc"):
                file_variables = get_netcdf_file_variables(filename)
                remove_variables_with_non_spatiotemporal_dimensions(file_variables, self.valid_dimensions)
            elif filename.endswith(".hdf"):
                if data_type is None:
                    data_type = "SD"
                sd_vars, vd_vars = get_hdf4_file_variables(filename, data_type)
                file_variables = list((sd_vars or {}).keys()) + list((vd_vars or {}).keys())
            variables.extend(file_variables)
        return set(variables)

    def get_file_format(self, filename):
        """
        Returns a file format hierarchy separated by slashes, of the form
        ``TopLevelFormat/SubFormat/SubFormat/Version``.
        E.g. ``NetCDF/GASSP/1.0``, ``ASCII/ASCIIHyperpoint`` or ``HDF4/CloudSat``. This is mainly used within the
        ceda_di indexing tool. If not set it will default to the products name.

        A filename of an example file can be provided to enable the determination of, for example, a dataset version
        number.

        :param str filename: Filename of file to be inspected
        :return: File format, of the form ``[parent/]format/specific instance/version``, or the class name
        :rtype: str
        :raises: FileFormatError if there is an error
        """
        return self.__class__.__name__

    def get_file_type_error(self, filename):
        """
        Check a single file to see if it is of the correct type, and if not return
        a list of errors. If the return is None then there are no errors and
        this is the correct data product to use for this file.

        This method gives a mechanism for a data product to identify itself as the
        correct product when a specific enough file signature cannot be provided. For
        example GASSP is a type of NetCDF file and so filenames end with .nc but
        so do other NetCDF files, so the data product opens the file and looks
        for the GASSP version attribute, and if it doesn't find it returns an
        error.

        :param str filename: The filename for the file
        :return: List of errors, or None
        :rtype: list or None
        """
        return None class UngriddedCoordinates(CommonData):
    """
    Wrapper (adaptor) class for the different types of possible ungridded data.
    """

    def __init__(self, coords):
        """
        Constructor

        :param coords: A list of the associated Coord objects
        """
        from cis.data_io.Coord import CoordList, Coord

        if isinstance(coords, list):
            self._coords = CoordList(coords)
        elif isinstance(coords, CoordList):
            self._coords = coords
        elif isinstance(coords, Coord):
            self._coords = CoordList([coords])
        else:
            raise ValueError("Invalid Coords type")
        self._post_process()

    def _post_process(self):
        """
        Perform a post processing step on lazy loaded Coordinate Data

        :return:
        """
        # Remove any points with missing coordinate values:
        combined_mask = numpy.zeros(self._coords[0].data_flattened.shape, dtype=bool)
        for coord in self._coords:
            combined_mask |= numpy.ma.getmaskarray(coord.data_flattened)
            if coord.data.dtype != 'object':
                combined_mask |= numpy.isnan(coord.data).flatten()
        if combined_mask.any():
            n_points = numpy.count_nonzero(combined_mask)
            logging.warning("Identified {n_points} point(s) which were missing values for some or all coordinates - "
                            "these points have been removed from the data.".format(n_points=n_points))
            for coord in self._coords:
                coord.data = numpy.ma.masked_array(coord.data_flattened, mask=combined_mask).compressed()
                coord.update_shape()
                coord.update_range()

    @property
    def coords_flattened(self):
        all_coords = self.coords().find_standard_coords()
        return [(c.data_flattened if c is not None else None) for c in all_coords]

    @property
    def history(self):
        return "UngriddedCoordinates have no history"

    @property
    def size(self):
        if len(self._coords) > 1:
            return self._coords[0].data.size
        else:
            return 0

    def count(self):
        # There can be no masked coordinate points
        return self.size

    @property
    def x(self):
        return self.coord(axis='X')

    @property
    def y(self):
        return self.coord(axis='Y')

    @property
    def lat(self):
        return self.coord(standard_name='latitude')

    @property
    def lon(self):
        return self.coord(standard_name='longitude')

    @property
    def time(self):
        return self.coord(standard_name='time')

    @property
    def var_name(self):
        return ''

    def hyper_point(self, index):
        """
        :param index: The index in the array to find the point for
        :return: A hyperpoint representing the data at that point
        """
        from cis.data_io.hyperpoint import HyperPoint
        return HyperPoint(self.coord(standard_name='latitude').data.flat[index],
                          self.coord(standard_name='longitude').data.flat[index],
                          self.coord(standard_name='altitude').data.flat[index],
                          self.coord(standard_name='time').data.flat[index],
                          self.coord(standard_name='air_pressure').data.flat[index],
                          None)

    def as_data_frame(self, copy=True, time_index=True, name=None):
        """
        Convert an UngriddedCoordinates object to a Pandas DataFrame.

        :param copy: Create a copy of the data for the new DataFrame? Default is True.
        :return: A Pandas DataFrame representing the data and coordinates. Note that this won't include any metadata.
        """
        return _coords_as_data_frame(self._coords, time_index=time_index)

    def coords(self, name_or_coord=None, standard_name=None, long_name=None, attributes=None, axis=None, var_name=None,
               dim_coords=True):
        """
        :return: A list of coordinates in this UngriddedData object fitting the given criteria
        """
        return self._coords.get_coords(name_or_coord, standard_name, long_name, attributes, axis, var_name)

    def coord(self, name_or_coord=None, standard_name=None, long_name=None, attributes=None, axis=None, var_name=None):
        """
        :raise: CoordinateNotFoundError
        :return: A single coord given the same arguments as :meth:`coords`.

        """
        return self._coords.get_coord(name_or_coord, standard_name, long_name, attributes, axis, var_name)

    def get_coordinates_points(self):
        return UngriddedHyperPointView(self.coords_flattened, None)

    def get_all_points(self):
        """Returns a HyperPointView of the points.

        :return: HyperPointView of all the data points
        """
        return UngriddedHyperPointView(self.coords_flattened, None)

    def get_non_masked_points(self):
        """Returns a HyperPointView for which the default iterator omits masked points.

        :return: HyperPointView of the data points
        """
        return UngriddedHyperPointView(self.coords_flattened, None, non_masked_iteration=True)

    @property
    def is_gridded(self):
        """Returns value indicating whether the data/coordinates are gridded.
        """
        return False

    def set_longitude_range(self, range_start):
        """
        Rotates the longitude coordinate array and changes its values by
        360 as necessary to force the values to be within a 360 range starting
        at the specified value.
        :param range_start: starting value of required longitude range
        """
        from cis.utils import fix_longitude_range
        self.coord(standard_name='longitude').data = fix_longitude_range(self.lon.points, range_start)

    def subset(self, **kwargs):
        raise NotImplementedError("Subset is not available for UngriddedCoordinates objects")

    def collocated_onto(self, sample, how='', kernel=None, **kwargs):
        raise NotImplementedError("UngriddedCoordinates objects cannot be used as sources of data for collocation.")

    def sampled_from(self, data, how='', kernel=None, missing_data_for_missing_sample=False, fill_value=None,
                     var_name='', var_long_name='', var_units='', **kwargs):
        """
        Collocate the CommonData object with another CommonData object using the specified collocator and kernel

        Note - that the default value for missing_data_for_missing_sample is different in this implementation as
        compared to the UngriddedData implementation.

        :param CommonData or CommonDataList data: The data to resample
        :param str how: Collocation method (e.g. lin, nn, bin or box)
        :param str or cis.collocation.col_framework.Kernel kernel:
        :param bool missing_data_for_missing_sample: Should missing values in sample data be ignored for collocation?
        :param float fill_value: Value to use for missing data
        :param str var_name: The output variable name
        :param str var_long_name: The output variable's long name
        :param str var_units: The output variable's units
        :return CommonData: The collocated dataset
        """
        return _ungridded_sampled_from(self, data, how=how, kernel=kernel,
                                       missing_data_for_missing_sample=missing_data_for_missing_sample,
                                       fill_value=fill_value, var_name=var_name, var_long_name=var_long_name,
                                       var_units=var_units, **kwargs)

    def _get_default_plot_type(self, lat_lon=False):
        raise NotImplementedError("UngriddedCoordinates have no default plot type")

    def var_name(self):
        raise NotImplementedError("UngriddedCoordinates have no var name") class UngriddedData(LazyData, CommonData):
    """
        Wrapper (adaptor) class for the different types of possible ungridded data.
    """

    def __init__(self, data, metadata, coords, data_retrieval_callback=None):
        """
        Constructor

        :param data: The data handler (e.g. SDS instance) for the specific data type, or a numpy array of data.
         This can be a list of data handlers, or a single data handler
        :param metadata: Any associated metadata
        :param coords: A list of the associated Coord objects
        :param data_retrieval_callback: A method for retrieving data when needed
        """
        from cis.data_io.Coord import CoordList, Coord

        if isinstance(coords, list):
            self._coords = CoordList(coords)
        elif isinstance(coords, CoordList):
            self._coords = coords
        elif isinstance(coords, Coord):
            self._coords = CoordList([coords])
        else:
            raise ValueError("Invalid Coords type")

        # TODO Find a cleaner workaround for this, for some reason UDUNITS can not parse 'per kilometer per steradian'
        if str(metadata.units) == 'per kilometer per steradian':
            metadata.units = 'kilometer^-1 steradian^-1'

        super(UngriddedData, self).__init__(data, metadata, data_retrieval_callback)

    @property
    def coords_flattened(self):
        all_coords = self.coords().find_standard_coords()
        return [(c.data_flattened if c is not None else None) for c in all_coords]

    def _post_process(self):
        """
        Perform a post processing step on lazy loaded Ungridded Data.

        :return:
        """
        # Load the data if not already loaded
        if self._data is None:
            data = self.data
        else:
            # Remove any points with missing coordinate values:
            combined_mask = numpy.zeros(self._data.shape, dtype=bool).flatten()
            for coord in self._coords:
                combined_mask |= numpy.ma.getmaskarray(coord.data).flatten()
                if coord.data.dtype != 'object':
                    combined_mask |= numpy.isnan(coord.data).flatten()
                coord.update_shape()
                coord.update_range()
            if combined_mask.any():
                n_points = numpy.count_nonzero(combined_mask)
                logging.warning(
                    "Identified {n_points} point(s) which were missing values for some or all coordinates - "
                    "these points have been removed from the data.".format(n_points=n_points))
                for coord in self._coords:
                    coord.data = numpy.ma.masked_array(coord.data.flatten(), mask=combined_mask).compressed()
                    coord.update_shape()
                    coord.update_range()
                if numpy.ma.is_masked(self._data):
                    new_data_mask = numpy.ma.masked_array(self._data.mask.flatten(), mask=combined_mask).compressed()
                    new_data = numpy.ma.masked_array(self._data.data.flatten(), mask=combined_mask).compressed()
                    self._data = numpy.ma.masked_array(new_data, mask=new_data_mask)
                else:
                    self._data = numpy.ma.masked_array(self._data.flatten(), mask=combined_mask).compressed()
            self.update_shape()
            self.update_range()

    def make_new_with_same_coordinates(self, data=None, var_name=None, standard_name=None,
                                       long_name=None, history=None, units=None, flatten=False):
        """
        Create a new, empty UngriddedData object with the same coordinates as this one.

        :param data: Data to use (if None then defaults to all zeros)
        :param var_name: Variable name
        :param standard_name: Variable CF standard name
        :param long_name: Variable long name
        :param history: Data history string
        :param units: Variable units
        :param flatten: Whether to flatten the data and coordinates (for ungridded data only)
        :return: UngriddedData instance
        """
        if data is None:
            data = numpy.zeros(self.shape)
        metadata = Metadata(name=var_name, standard_name=standard_name,
                            long_name=long_name, history='', units=units)
        if flatten:
            from cis.data_io.Coord import Coord
            data = data.flatten()
            new_coords = []
            for coord in self._coords:
                new_coords.append(Coord(coord.data_flattened, coord.metadata, coord.axis))
        else:
            new_coords = self._coords
        ug_data = UngriddedData(data=data, metadata=metadata, coords=new_coords)
        # Copy the history in separately so it gets the timestamp added.
        if history:
            ug_data.add_history(history)
        return ug_data

    def __getitem__(self, keys):
        """
        Return a COPY of the data with the given slice. We copy to emulate the Iris Cube behaviour
        """
        from copy import deepcopy
        # Create a copy of the slice of each of the coords
        new_coords = []
        for c in self.coords():
            new_coords.append(c[keys])
        # The data is just a new LazyData objects with the sliced data. Note this is a slice of the whole (concatenated)
        #  data, and will lead to post-processing before slicing.
        # TODO: We could be cleverer and figure out the right slice across the various data managers to only read the
        #  right data from disk.
        return UngriddedData(data=self.data[keys].copy(), metadata=deepcopy(self.metadata), coords=new_coords)

    def copy(self, data=None):
        """
        Create a copy of this UngriddedData object with new data and coordinates
        so that that they can be modified without held references being affected.
        Will call any lazy loading methods in the data and coordinates

        :param ndarray data: Replace the data of the ungridded data copy with provided data

        :return: Copied UngriddedData object
        """
        from copy import deepcopy
        data = data if data is not None else numpy.ma.copy(self.data)  # This will load the data if lazy load
        coords = self.coords().copy()
        return UngriddedData(data=data, metadata=deepcopy(self.metadata), coords=coords)

    @property
    def size(self):
        return self.data.size

    def count(self):
        return self.data.count() if hasattr(self.data, 'count') else self.data.size

    @property
    def history(self):
        return self.metadata.history

    @property
    def x(self):
        return self.coord(axis='X')

    @property
    def y(self):
        return self.coord(axis='Y')

    @property
    def lat(self):
        return self.coord(standard_name='latitude')

    @property
    def lon(self):
        return self.coord(standard_name='longitude')

    @property
    def time(self):
        return self.coord(standard_name='time')

    def hyper_point(self, index):
        """
        :param index: The index in the array to find the point for
        :return: A hyperpoint representing the data at that point
        """
        from cis.data_io.hyperpoint import HyperPoint
        return HyperPoint(self.coord(standard_name='latitude').data.flat[index],
                          self.coord(standard_name='longitude').data.flat[index],
                          self.coord(standard_name='altitude').data.flat[index],
                          self.coord(standard_name='time').data.flat[index],
                          self.coord(standard_name='air_pressure').data.flat[index],
                          self.data.flat[index])

    def as_data_frame(self, copy=True, time_index=True, name=None):
        """
        Convert an UngriddedData object to a Pandas DataFrame.

        :param copy: Create a copy of the data for the new DataFrame? Default is True.
        :return: A Pandas DataFrame representing the data and coordinates. Note that this won't include any metadata.
        """
        df = _coords_as_data_frame(self.coords(), time_index=time_index)
        try:
            df[name or self.name()] = _to_flat_ndarray(self.data, copy)
        except ValueError:
            logging.warn("Copy created of MaskedArray for {} when creating Pandas DataFrame".format(self.name()))
            df[name or self.name()] = _to_flat_ndarray(self.data, True)

        return df

    def coords(self, name_or_coord=None, standard_name=None, long_name=None, attributes=None, axis=None, var_name=None,
               dim_coords=True):
        """
        :return: A list of coordinates in this UngriddedData object fitting the given criteria
        """
        self._post_process()
        return self._coords.get_coords(name_or_coord, standard_name, long_name, attributes, axis, var_name)

    def coord(self, name_or_coord=None, standard_name=None, long_name=None, attributes=None, axis=None, var_name=None):
        """
        :raise: CoordinateNotFoundError
        :return: A single coord given the same arguments as :meth:`coords`.
        """
        return self.coords().get_coord(name_or_coord, standard_name, long_name, attributes, axis, var_name)

    def get_coordinates_points(self):
        """Returns a HyperPointView of the coordinates of points.

        :return: HyperPointView of the coordinates of points
        """
        return UngriddedHyperPointView(self.coords_flattened, None)

    def get_all_points(self):
        """Returns a HyperPointView of the points.

        :return: HyperPointView of all the data points
        """
        return UngriddedHyperPointView(self.coords_flattened, self.data_flattened)

    def get_non_masked_points(self):
        """Returns a HyperPointView for which the default iterator omits masked points.

        :return: HyperPointView of the data points
        """
        return UngriddedHyperPointView(self.coords_flattened, self.data_flattened, non_masked_iteration=True)

    def find_standard_coords(self):
        """Constructs a list of the standard coordinates.
        The standard coordinates are latitude, longitude, altitude, air_pressure and time; they occur in the return
        list in this order.

        :return: list of coordinates or None if coordinate not present
        """
        return self.coords().find_standard_coords()

    @property
    def is_gridded(self):
        """Returns value indicating whether the data/coordinates are gridded.
        """
        return False

    @classmethod
    def from_points_array(cls, hyperpoints):
        """
        Constuctor for building an UngriddedData object from a list of hyper points

        :param hyperpoints: list of HyperPoints
        """
        from cis.data_io.Coord import Coord, CoordList
        from cis.data_io.hyperpoint import HyperPointList
        from cis.time_util import cis_standard_time_unit

        if not isinstance(hyperpoints, HyperPointList):
            hyperpoints = HyperPointList(hyperpoints)

        values = hyperpoints.vals
        latitude = hyperpoints.latitudes
        longitude = hyperpoints.longitudes
        air_pressure = hyperpoints.air_pressures
        altitude = hyperpoints.altitudes
        time = hyperpoints.times

        coord_list = []
        if latitude is not None:
            coord_list.append(Coord(latitude, Metadata(standard_name='latitude', units='degrees north')))
        if longitude is not None:
            coord_list.append(Coord(longitude, Metadata(standard_name='longitude', units='degrees east')))
        if air_pressure is not None:
            coord_list.append(Coord(air_pressure, Metadata(standard_name='air_pressure', units='Pa')))
        if altitude is not None:
            coord_list.append(Coord(altitude, Metadata(standard_name='altitude', units='meters')))
        if time is not None:
            coord_list.append(Coord(time, Metadata(standard_name='time', units=cis_standard_time_unit)))
        coords = CoordList(coord_list)

        return cls(values, Metadata(), coords)

    def summary(self, shorten=False):
        """
        Unicode summary of the UngriddedData with metadata of itself and its coordinates
        """
        summary = 'Ungridded data: {name} / ({units}) \n'.format(name=self.name(), units=self.units)
        if shorten:
            return summary

        summary += '     Shape = {}\n'.format(self.data.shape) + '\n'
        summary += '     Total number of points = {}\n'.format(self.size)
        summary += '     Number of non-masked points = {}\n'.format(self.count())

        summary += str(self.metadata)

        summary += '     Coordinates: \n'
        for c in self.coords():
            summary += '{pad:{width}}{name}\n'.format(pad=' ', width=7, name=c.name())
            c.update_range()
            summary += c.metadata.summary(offset=10)

        return summary

    def __repr__(self):
        return "<cis 'UngriddedData' of %s>" % self.summary(shorten=True)

    def __str__(self):
        # six has a decorator for this bit, but it doesn't do errors='replace'.
        if six.PY3:
            return self.summary()
        else:
            return self.summary().encode(errors='replace')

    def __unicode__(self):
        return self.summary()

    def set_longitude_range(self, range_start):
        """
        Rotates the longitude coordinate array and changes its values by
        360 as necessary to force the values to be within a 360 range starting
        at the specified value.
        :param range_start: starting value of required longitude range
        """
        from cis.utils import fix_longitude_range
        self.coord(standard_name='longitude').data = fix_longitude_range(self.lon.points, range_start)

    def subset(self, **kwargs):
        """
        Subset the data based on the specified constraints. Note that the limits are inclusive.

        The subset region is defined by passing keyword arguments for each dimension to be subset over, each argument
        must be a slice, or have two entries (a maximum and a minimum). Datetime objects can be used to specify upper
        and lower datetime limits, or a single PartialDateTime object can be used to specify a datetime range.

        The keyword keys are used to find the relevant coordinate, they are looked for in order of name, standard_name,
        axis and var_name.

        For example:
            data.subset(x=[0, 80], y=slice(10, 50))

        or:
            data.aggregate(t=PartialDateTime(2008,9))


        A shape keyword can also be supplied as a WKT string or shapely object to subset in lat/lon by an arbitrary
        shape. In this case the lat/lon bounds are taken as the bounding box of the shape.

        :param kwargs: The constraints for each coordinate dimension
        :return CommonData:
        """
        from cis.subsetting.subset import subset, UngriddedSubsetConstraint
        return subset(self, UngriddedSubsetConstraint, **kwargs)

    def aggregate(self, how=None, **kwargs):
        """
        Aggregate the UngriddedData object based on the specified grids. The grid is defined by passing keyword
        arguments for each dimension, each argument must be a slice, or have three entries (a maximum, a minimum and a
        gridstep). The default aggregation method ('moments') returns the mean, standard deviation and number of points
         as separate GriddedData objects.

        Datetime objects can be used to specify upper and lower datetime limits, or a
        single PartialDateTime object can be used to specify a datetime range. The gridstep can be specified as a
        DateTimeDelta object.

        The keyword keys are used to find the relevant coordinate, they are looked for in order of name, standard_name,
        axis and var_name.

        For example:
            data.aggregate(x=[-180, 180, 360], y=slice(-90, 90, 10))

        or:
            data.aggregate(how='mean', t=[PartialDateTime(2008,9), timedelta(days=1))

        :param str how: The kernel to use in the aggregation (moments, mean, min, etc...). Default is moments
        :param kwargs: The grid specifications for each coordinate dimension
        :return GriddedData:
        """
        agg = _aggregate_ungridded(self, how, **kwargs)
        # Return the single item if there's only one (this depends on the kernel used)
        if len(agg) == 1:
            agg = agg[0]
        return agg

    def sampled_from(self, data, how='', kernel=None, missing_data_for_missing_sample=True, fill_value=None,
                     var_name='', var_long_name='', var_units='', **kwargs):
        """
        Collocate the CommonData object with another CommonData object using the specified collocator and kernel

        :param CommonData or CommonDataList data: The data to resample
        :param str how: Collocation method (e.g. lin, nn, bin or box)
        :param str or cis.collocation.col_framework.Kernel kernel:
        :param bool missing_data_for_missing_sample: Should missing values in sample data be ignored for collocation?
        :param float fill_value: Value to use for missing data
        :param str var_name: The output variable name
        :param str var_long_name: The output variable's long name
        :param str var_units: The output variable's units
        :return CommonData: The collocated dataset
        """
        return _ungridded_sampled_from(self, data, how=how, kernel=kernel,
                                       missing_data_for_missing_sample=missing_data_for_missing_sample,
                                       fill_value=fill_value, var_name=var_name, var_long_name=var_long_name,
                                       var_units=var_units, **kwargs)

    def _get_default_plot_type(self, lat_lon=False):
        if lat_lon:
            return 'scatter2d'
        else:
            return 'line' class Metadata(object):

    @classmethod
    def from_CubeMetadata(cls, cube_meta):
        return cls(name=cube_meta.var_name, standard_name=cube_meta.standard_name, long_name=cube_meta.long_name,
                   units=str(cube_meta.units), misc=cube_meta.attributes)

    def __init__(self, name='', standard_name='', long_name='', shape=None, units='', range=None, factor=None,
                 offset=None, missing_value=None, history='', misc=None):
        self._name = name

        self._standard_name = ''
        if standard_name:
            self.standard_name = standard_name
        elif name:
            self.standard_name = Metadata.guess_standard_name(name)

        self.long_name = long_name
        self.shape = shape

        self.units = units
        self.range = range
        self.factor = factor
        self.offset = offset
        self.missing_value = missing_value
        self.history = history
        if misc is None:
            self.misc = {}
        else:
            self.misc = misc

    def __eq__(self, other):
        result = NotImplemented

        if isinstance(other, Metadata):
            result = self._name == other._name and \
                     self._standard_name == other._standard_name and \
                     self.long_name == other.long_name and \
                     self.units == other.units
        return result

    # Must supply __ne__, Python does not defer to __eq__ for negative equality
    def __ne__(self, other):
        result = self.__eq__(other)
        if result is not NotImplemented:
            result = not result
        return result

    # Must supply __hash__, Python 3 does not enable it if __eq__ is defined
    def __hash__(self):
        return hash(id(self))

    def summary(self, offset=5):
        """
        Creates a unicode summary of the metadata object

        :param offset: The left hand padding to apply to the text
        :return: The summary
        """
        from datetime import datetime
        string = ''
        string += '{pad:{width}}Long name = {lname}\n'.format(pad=' ', width=offset, lname=self.long_name)
        string += '{pad:{width}}Standard name = {sname}\n'.format(pad=' ', width=offset, sname=self.standard_name)
        string += '{pad:{width}}Units = {units}\n'.format(pad=' ', width=offset, units=self.units)
        string += '{pad:{width}}Missing value = {mval}\n'.format(pad=' ', width=offset, mval=self.missing_value)
        # str(tuple) returns repr(obj) on each item in the tuple, if we have a datetime tuple then we want str(obj)
        #  instead. Just make that ourselves here instead (as a str to avoid the extra quotes if we make a 'real' tuple)
        if isinstance(self.range[0], datetime):
            range_tuple = '({}, {})'.format(*self.range)
        else:
            range_tuple = self.range
        string += '{pad:{width}}Range = {range}\n'.format(pad=' ', width=offset, range=range_tuple)
        string += '{pad:{width}}History = {history}\n'.format(pad=' ', width=offset, history=self.history)
        if self.misc:
            string += '{pad:{width}}Misc attributes: \n'.format(pad=' ', width=offset)
            for k, v in self.misc.items():
                string += '{pad:{width}}{att} = {val}\n'.format(pad=' ', width=offset + 2, att=k.title(), val=v)
        return string

    def __str__(self):
        # six has a decorator for this bit, but it doesn't do errors='replace'.
        if six.PY3:
            return self.summary()
        else:
            return self.summary().encode(errors='replace')

    def __unicode__(self):
        return self.summary()

    @property
    def standard_name(self):
        return self._standard_name

    @standard_name.setter
    def standard_name(self, standard_name):
        from iris.std_names import STD_NAMES
        if standard_name is None or standard_name in STD_NAMES:
            # If the standard name is actually changing from one to another then log the fact
            if self.standard_name is not None \
                    and self.standard_name.strip() != "" \
                    and self.standard_name != standard_name:
                logging.debug("Changing standard name for dataset from '{}' to '{}'".format(self.standard_name,
                                                                                            standard_name))
            self._standard_name = standard_name
        else:
            raise ValueError('%r is not a valid standard_name' % standard_name)

    @property
    def units(self):
        return self._units

    @units.setter
    def units(self, units):
        from cf_units import Unit
        if not isinstance(units, Unit):
            try:
                # Try some basic tidying up of unit
                if isinstance(units, six.string_types):
                    if 'since' in units.lower():
                        # Often this time since epoch units are weirdly capitalised, or include extra punctuation
                        units = units.lower().replace("since:", "since").replace(",", "")
                    else:
                        # Replace number with 1 (e.g. #/cm3 == cm-3)
                        units = units.replace('#', '1')
                units = Unit(units)
            except ValueError:
                logging.info("Unable to parse cf-units: {}. Some operations may not be available.".format(units))
        self._units = units

    @staticmethod
    def guess_standard_name(name):
        standard_name = None
        if name.lower().startswith('lat'):
            standard_name = 'latitude'
        elif name.lower().startswith('lon'):
            standard_name = 'longitude'
        elif name.lower().startswith('alt') or name.lower() == 'height':
            standard_name = 'altitude'
        elif name.lower().startswith('pres') or name.lower() == 'air_pressure':
            standard_name = 'air_pressure'
        elif name.lower() == 'time':
            standard_name = 'time'
        return standard_name def add_to_list_if_not_none(item, list):
    """
    Add a value to a list if it is not None

    :param item: the item to add
    :param list: the list to append it to
    :return: nothing
    """
    if item is not None:
        list.append(item) def dimensions_compatible(dimensions, other_dimensions):
    """
    Check to see if two dimensions share the same common coordinates. Note that this will only compare the dimensions
    up to the length of the shortest list.

    :param dimensions: dimension list
    :param other_dimensions: other dimension list
    """

    for dim, other_dim in zip(dimensions, other_dimensions):
        if not dim == other_dim:
            return False
    return True def listify(item):
    """
    If item is not a list, return it as a list

    :param item: Item which may or may not be a list
    :return: List
    """
    from types import GeneratorType
    if isinstance(item, (tuple, GeneratorType)):
        return list(item)
    if not isinstance(item, list):
        return [item]
    return item def get_metadata(var):
    """
    Retrieves all metadata

    :param var: the Variable to read metadata from
    :return: A metadata object
    """
    from cis.data_io.ungridded_data import Metadata
    from cis.utils import set_standard_name_if_valid

    missing_value = find_missing_value(var)

    standard_attributes = ['standard_name', 'long_name', 'history', 'units']

    attrs = {attr: getattr(var, attr, "") for attr in standard_attributes}

    shape = getattr(var, "shape", None)
    if shape is None:
        try:
            shape = (var._recLen[0],)
        except AttributeError:
            shape = ()

    metadata = Metadata(
        var._name,
        long_name=attrs['long_name'],
        units=attrs['units'],
        missing_value=missing_value,
        shape=shape,
        history=attrs['history'],
        misc={k: getattr(var, k) for k in var.ncattrs() if k not in standard_attributes})

    # Only set the standard name if it's CF compliant
    set_standard_name_if_valid(metadata, attrs['standard_name'])

    return metadata def get_netcdf_file_attributes(filename):
    """
    Get all the global attributes from a NetCDF file

    :param filename: The filename of the file to get the variables from
    :return: a dictionary of attributes and their values
    """

    from netCDF4 import Dataset
    try:
        f = Dataset(filename)
    except RuntimeError as e:
        raise IOError(e)

    return f.__dict__ def read_many_files_individually(filenames, usr_variables):
    """
    Read multiple Variables from many NetCDF files manually - i.e. not with MFDataset as this doesn't always work,
    in particular for NetCDF4 files.

    :param filenames: A list of NetCDF filenames to read, or a string with wildcards.
    :param usr_variables: A list of variable (dataset) names to read from the files. The names must appear exactly as
      in in the NetCDF file. Variable names may be fully qualified NetCDF4 Hierarchical group variables in the form
      ``<group1>/<group2....>/<variable_name>``, e.g. ``AVHRR/Ch4CentralWavenumber``.
    :return: A dictionary of lists of variable instances constructed from all of the input files with the fully
      qualified variable name as the key
    """
    from cis.utils import add_element_to_list_in_dict

    usr_variables = listify(usr_variables)

    var_data = {}

    for filename in filenames:

        var_dict = read(filename, usr_variables)
        for var in list(var_dict.keys()):
            add_element_to_list_in_dict(var_data, var, var_dict[var])

    return var_data def get_netcdf_file_variables(filename, exclude_coords=False):
    """
    Get all the variables contained in a NetCDF file. Variables in NetCDF4 Hierarchical groups are returned with their
    fully qualified variable name in the form ``<group1>/<group2....>/<variable_name>``,
    e.g.``AVHRR/Ch4CentralWavenumber``.

    :param filename: The filename of the file to get the variables from
    :param exclude_coords: Exclude coordinate variables if True
    :return: An OrderedDict containing {variable_name: NetCDF Variable instance}
    """
    from netCDF4 import Dataset
    try:
        f = Dataset(filename)
    except RuntimeError as e:
        raise IOError(e)
    # Include NetCDF4 hierarchical groups too:
    variables = _get_all_fully_qualified_variables(f)
    if exclude_coords:
        for var in f.dimensions:
            try:
                del variables[var]
            except KeyError:
                pass
    return variables
import logging
import os
import numpy as np
from cis.data_io.Coord import CoordList
from cis.exceptions import InvalidVariableError, FileFormatError
from cis.data_io.products import AProduct
from cis.data_io.ungridded_data import UngriddedCoordinates, UngriddedData, Metadata
from cis.utils import add_to_list_if_not_none, dimensions_compatible, listify
from cis.data_io.netcdf import get_metadata, get_netcdf_file_attributes, read_many_files_individually, \
    get_netcdf_file_variables
        from cis.exceptions import InvalidVariableError
        from cis.exceptions import InvalidVariableError
        from cis.data_io.Coord import Coord
        from cis.utils import expand_1d_to_2d_array
        from cis.data_io.netcdf import read
        from cis.data_io.Coord import Coord
        from cis.data_io.Coord import Coord
        from six.moves import zip_longest
        from cis.data_io.Coord import Coord




class NCAR_NetCDF_RAF_variable_name_selector(object):
    """
    NCAR-RAF file definition: for the purposes of this data product
        It is a NetCDF file
        It contains attributes and variables
        The attributes must contain the following:
            an attribute GASSP_Version
            or an attribute Conventions with the value "NCAR-RAF/nimbus"

    This should successful differentiate the names of the variables for the various dimensions in NCAR-RAF and GASSP.
    The variables set are:
    station: True if this is for a fixed station data, False otherwise
    station_latitude: the latitude of the station (set only if station is true)
    station_longitude: the longitude of the station (set only if station is true)
    altitude: Altitude of the measurement if it is fixed, none if this is not fixed, defaults to DEFAULT_ALTITUDE
    time_variable_name: variable name for time
    latitude_variable_name: variable name of the latitude, None if fixed
    longitude_variable_name: variable name of the longitude, None if fixed
    altitude_variable_name: variable name of the altitude, None if fixed
    pressure_variable_name: variable name of the pressure, None if not included in data
    time_stamp_info: time stamp info if included, None otherwise

    The choice are in order of preference:
    Aeroplane GASSP (Data with time, lat, lon and altitude vars)
    Ship GASSP (Data with time, lat and lon vars but fixed altitude)
    Station GASSP (Data with time vars but fixed lat, lon and altitude)
    NCAR-RAF: (Data with time, lat, lon and altitude, but names are defined in coordinates attribute)
    """

    # Attribute name for the attribute holding the time coordinate
    TIME_COORDINATE_NAME = 'Time_Coordinate'
    # Attribute name for the attribute holding the latitude coordinate
    LATITUDE_COORDINATE_NAME = 'Latitude_Coordinate'
    # Attribute name for the attribute holding the longitude coordinate
    LONGITUDE_COORDINATE_NAME = 'Longitude_Coordinate'
    # Attribute name for the attribute holding the altitude or vertical coordinate
    ALTITUDE_COORDINATE_NAME = 'Vertical_Coordinate'
    # Variable name for the dataset for corrected pressure
    CORRECTED_PRESSURE_VAR_NAME = 'PSXC'
    # Variable name for the dataset for pressure
    PRESSURE_VAR_NAME = 'PRE'
    # Attribute name for the attribute holding the names of the best coordinates to use for lon, lat, alt and time
    BEST_COORDINATES_NAME = 'Coordinates'
    # Attribute name for the attribute holding the stations latitude
    STATION_LATITUDE_NAME = 'Station_Lat'
    # Attribute name for the attribute holding the stations longitude
    STATION_LONGITUDE_NAME = 'Station_Lon'
    # Attribute name for the attribute holding the stations altitude
    STATION_ALTITUDE_NAME = 'Station_Altitude'
    # Attribute name for the attribute holding the time stamp info (this is used for where time is measured from
    TIME_STAMP_INFO_NAME = 'Time_Stamp_Info'
    # The default altitude to use if it is not set, for instance ship based GASSP does not contain altitude
    DEFAULT_ALTITUDE = 0

    def __init__(self, attributes, variables):
        """
        Initialisation
        :param attributes: dictionary of attributes and their values (or list of dictionarys if multiple files read)
        :param variables: dictionary of variable names and NetCDF Variable objects
        (or list of dictionarys if multiple files read)
        :return: nothing
        """
        self.station = False
        self.station_latitude = None
        self.latitude_variable_name = None
        self.station_longitude = None
        self.longitude_variable_name = None
        self.altitude = None
        self.altitude_variable_name = None
        self.pressure_variable_name = None
        self.time_stamp_info = None
        self.time_dimensions = None


        self._attributes = [{k.lower(): v for k, v in list(attrs.items())} for attrs in listify(attributes)]