# -*- coding: utf-8 -*-

r'''This module contains a base class for modeling computation graphs.'''

import downhill
import gzip
import hashlib
import numpy as np
import pickle
import theano
import time
import warnings

from . import layers
from . import losses
from . import regularizers
from . import trainer
from . import util

class Network(object):
    '''The network class encapsulates a network computation graph.

    Notes
    -----

    Computation graphs are organized into :ref:`layers <layers>`. Each layer
    receives one or more arrays of input data, transforms them, and generates
    one or more arrays of output data.

    Outputs in a computation graph are named according to their layer and output
    type, so the 'pre' output of a layer named 'hid1' would be named 'hid1:pre'.
    The 'out' output is the default output for a layer. By default the last
    layer in a network is named 'out'.

    The parameters in a network graph are optimized by minimizing a :ref:`loss
    function <losses>` with respect to some set of training data. Typically the
    value produced by 'out:out' is compared to some target value, creating an
    error value of some sort. This error value is then propagated back through
    the computation graph to update the parameters in the model.

    Parameters
    ----------
    layers : int, tuple, dict, or :class:`Layer <theanets.layers.base.Layer>`
        A sequence of values specifying the layer configuration for the network.
        For more information, please see :ref:`guide-creating-specifying-layers`.
    loss : str or :class:`Loss <theanets.losses.Loss>`
        The name of a loss function to optimize when training this network
        model.
    weighted : bool, optional
        If True, optimize this model using a "weighted" loss. Weighted losses
        typically require an additional array as input during optimization.
        For more information, see :ref:`losses-weighted`. Defaults to False.
    rng : int or RandomState, optional
        A seed or numpy ``RandomState`` instance for generating randomness in
        the model. Defaults to 13.

    Attributes
    ----------
    layers : list of :class:`Layer <theanets.layers.base.Layer>`
        A list of the layers in this network model.
    losses : list of :class:`Loss <theanets.losses.Loss>`
        A list of losses to be computed when optimizing this network model.
    '''

    DEFAULT_OUTPUT_ACTIVATION = 'linear'
    '''Default activation for the output layer.'''

    INPUT_NDIM = 2
    '''Number of dimensions for holding input data arrays.'''

    OUTPUT_NDIM = 2
    '''Number of dimensions for holding output data arrays.'''

    def __init__(self, layers=(), loss='mse', weighted=False, rng=13):
        self._graphs = {}     # cache of symbolic computation graphs
        self._functions = {}  # cache of callable feedforward functions
        self._rng = rng

        # create layers based on specs provided in the constructor.
        self.layers = []
        for i, layer in enumerate(layers):
            first = i == 0
            last = i == len(layers) - 1
            name = 'in' if first else 'out' if last else 'hid{}'.format(i)
            activation = self.DEFAULT_OUTPUT_ACTIVATION if last else 'relu'
            self.add_layer(layer=layer, name=name, activation=activation)

        # bind layers to this graph after construction. this finalizes layer
        # shapes and does other consistency checks based on the entire graph.
        [l.bind(self) for l in self.layers]

        # create a default loss (usually).
        self.losses = []
        if loss and self.layers:
            self.set_loss(loss,
                          weighted=weighted,
                          target=self.OUTPUT_NDIM,
                          output_name=self.layers[-1].output_name)

    def add_layer(self, layer=None, **kwargs):
        '''Add a :ref:`layer <layers>` to our network graph.

        Parameters
        ----------
        layer : int, tuple, dict, or :class:`Layer <theanets.layers.base.Layer>`
            A value specifying the layer to add. For more information, please
            see :ref:`guide-creating-specifying-layers`.
        '''
        # if the given layer is a Layer instance, just add it and move on.
        if isinstance(layer, layers.Layer):
            self.layers.append(layer)
            return

        form = kwargs.pop('form', 'ff' if self.layers else 'input').lower()

        if isinstance(layer, util.basestring):
            if not layers.Layer.is_registered(layer):
                raise util.ConfigurationError('unknown layer type: {}'.format(layer))
            form = layer
            layer = None

        # if layer is a tuple/list of integers, assume it's a shape.
        if isinstance(layer, (tuple, list)) and all(isinstance(x, int) for x in layer):
            kwargs['shape'] = tuple(layer)
            layer = None

        # if layer is some other tuple/list, assume it's a list of:
        # - the name of a layers.Layer class (str)
        # - the name of an activation function (str)
        # - the number of units in the layer (int)
        if isinstance(layer, (tuple, list)):
            for el in layer:
                if isinstance(el, util.basestring) and layers.Layer.is_registered(el):
                    form = el
                elif isinstance(el, util.basestring):
                    kwargs['activation'] = el
                elif isinstance(el, int):
                    if 'size' in kwargs:
                        raise util.ConfigurationError(
                            'duplicate layer sizes! {}'.format(kwargs))
                    kwargs['size'] = el
            layer = None

        # if layer is a dictionary, try to extract a form for the layer, and
        # override our default keyword arguments with the rest.
        if isinstance(layer, dict):
            for key, value in layer.items():
                if key == 'form':
                    form = value.lower()
                else:
                    kwargs[key] = value
            layer = None

        # if neither shape nor size have been specified yet, check that the
        # "layer" param is an int and use it for "size".
        if 'shape' not in kwargs and 'size' not in kwargs and isinstance(layer, int):
            kwargs['size'] = layer

        # if it hasn't been provided in some other way yet, set input
        # dimensionality based on the model.
        if form == 'input' and 'shape' not in kwargs:
            kwargs.setdefault('ndim', self.INPUT_NDIM)

        # set some default layer parameters.
        if form != 'input':
            kwargs.setdefault('inputs', self.layers[-1].output_name)
            kwargs.setdefault('rng', self._rng)

        if form.lower() == 'tied' and 'partner' not in kwargs:
            # we look backward through our list of layers for a partner.
            # any "tied" layer that we find increases a counter by one,
            # and any "untied" layer decreases the counter by one. our
            # partner is the first layer we find with count zero.
            #
            # this is intended to handle the hopefully common case of a
            # (possibly deep) tied-weights autoencoder.
            tied = 1
            partner = None
            for l in self.layers[::-1]:
                tied += 1 if isinstance(l, layers.Tied) else -1
                if tied == 0:
                    partner = l.name
                    break
            else:
                raise util.ConfigurationError(
                    'cannot find partner for "{}"'.format(kwargs))
            kwargs['partner'] = partner

        layer = layers.Layer.build(form, **kwargs)

        # check that graph inputs have unique names.
        if isinstance(layer, layers.Input):
            if any(layer.name == i.name for i in self.inputs):
                raise util.ConfigurationError(
                    '"{}": duplicate input name!'.format(layer.name))

        self.layers.append(layer)

    def add_loss(self, loss=None, **kwargs):
        '''Add a :ref:`loss function <losses>` to the model.

        Parameters
        ----------
        loss : str, dict, or :class:`theanets.losses.Loss`
            A loss function to add. If this is a Loss instance, it will be added
            immediately. If this is a string, it names a loss function to build
            and add. If it is a dictionary, it should contain a ``'form'`` key
            whose string value names the loss function to add. Other arguments
            will be passed to :func:`theanets.losses.Loss.build`.
        '''
        if isinstance(loss, losses.Loss):
            self.losses.append(loss)
            return

        form = loss or 'mse'
        if 'form' in kwargs:
            form = kwargs.pop('form').lower()

        kw = dict(target=self.INPUT_NDIM, output_name=self.layers[-1].output_name)
        kw.update(kwargs)

        if isinstance(loss, dict):
            loss = dict(loss)
            if 'form' in loss:
                form = loss.pop('form').lower()
            kw.update(loss)

        self.losses.append(losses.Loss.build(form, **kw))

    def set_loss(self, *args, **kwargs):
        '''Clear the current loss functions from the network and add a new one.

        All parameters and keyword arguments are passed to :func:`add_loss`
        after clearing the current losses.
        '''
        self.losses = []
        self.add_loss(*args, **kwargs)

    def itertrain(self, train, valid=None, algo='rmsprop', subalgo='rmsprop',
                  save_every=0, save_progress=None, **kwargs):
        '''Train our network, one batch at a time.

        This method yields a series of ``(train, valid)`` monitor pairs. The
        ``train`` value is a dictionary mapping names to monitor values
        evaluated on the training dataset. The ``valid`` value is also a
        dictionary mapping names to values, but these values are evaluated on
        the validation dataset.

        Because validation might not occur every training iteration, the
        validation monitors might be repeated for multiple training iterations.
        It is probably most helpful to think of the validation monitors as being
        the "most recent" values that have been computed.

        After training completes, the network attribute of this class will
        contain the trained network parameters.

        Parameters
        ----------
        train : :class:`Dataset <downhill.dataset.Dataset>` or list
            A dataset to use when training the network. If this is a
            ``downhill.Dataset`` instance, it will be used directly as the
            training datset. If it is a list of numpy arrays or a list of
            callables, it will be converted to a ``downhill.Dataset`` and then
            used as the training set.
        valid : :class:`Dataset <downhill.dataset.Dataset>` or list, optional
            If this is provided, it will be used as a validation dataset. If not
            provided, the training set will be used for validation. (This is not
            recommended!)
        algo : str, optional
            An optimization algorithm to use for training our network. If not
            provided, :class:`RMSProp <downhill.adaptive.RMSProp>` will be used.
        subalgo : str, optional
            An optimization algorithm to use for a trainer that requires a
            "sub-algorithm," sugh as an unsupervised pretrainer. Defaults to
            :class:`RMSProp <downhill.adaptive.RMSProp>`.
        save_every : int or float, optional
            If this is nonzero and ``save_progress`` is not None, then the model
            being trained will be saved periodically. If this is a float, it is
            treated as a number of minutes to wait between savings. If it is an
            int, it is treated as the number of training epochs to wait between
            savings. Defaults to 0.
        save_progress : str or file handle, optional
            If this is not None, and ``save_progress`` is nonzero, then save the
            model periodically during training. This parameter gives either (a)
            the full path of a file to save the model, or (b) a file-like object
            where the model should be saved. If it is a string and the given
            name contains a "{}" format specifier, it will be filled with the
            integer Unix timestamp at the time the model is saved. Defaults to
            None, which does not save models.

        Yields
        ------
        training : dict
            A dictionary of monitor values computed using the training dataset,
            at the conclusion of training. This dictionary will at least contain
            a 'loss' key that indicates the value of the loss function. Other
            keys may be available depending on the trainer being used.
        validation : dict
            A dictionary of monitor values computed using the validation
            dataset, at the conclusion of training.
        '''
        if 'rng' not in kwargs:
            kwargs['rng'] = self._rng

        def create_dataset(data, **kwargs):
            name = kwargs.get('name', 'dataset')
            s = '{}_batches'.format(name)
            return downhill.Dataset(
                data,
                name=name,
                batch_size=kwargs.get('batch_size', 32),
                iteration_size=kwargs.get('iteration_size', kwargs.get(s)),
                axis=kwargs.get('axis', 0),
                rng=kwargs['rng'])

        # set up datasets ...
        if valid is None:
            valid = train
        if not isinstance(valid, downhill.Dataset):
            valid = create_dataset(valid, name='valid', **kwargs)
        if not isinstance(train, downhill.Dataset):
            train = create_dataset(train, name='train', **kwargs)

        if 'algorithm' in kwargs:
            warnings.warn(
                'please use the "algo" keyword arg instead of "algorithm"',
                DeprecationWarning)
            algo = kwargs.pop('algorithm')
            if isinstance(algo, (list, tuple)):
                algo = algo[0]

        # set up trainer ...
        if isinstance(algo, util.basestring):
            algo = algo.lower()
            if algo == 'sample':
                algo = trainer.SampleTrainer(self)
            elif algo.startswith('layer') or algo.startswith('sup'):
                algo = trainer.SupervisedPretrainer(subalgo, self)
            elif algo.startswith('pre') or algo.startswith('unsup'):
                algo = trainer.UnsupervisedPretrainer(subalgo, self)
            else:
                algo = trainer.DownhillTrainer(algo, self)

        # set up check to save model ...
        def needs_saving(elapsed, iteration):
            if save_progress is None:
                return False
            if isinstance(save_every, float):
                return elapsed > 60 * save_every
            if isinstance(save_every, int):
                return iteration % save_every == 0
            return False

        # train it!
        start = time.time()
        for i, monitors in enumerate(algo.itertrain(train, valid, **kwargs)):
            yield monitors
            now = time.time()
            if i and needs_saving(now - start, i):
                filename_or_handle = save_progress
                if isinstance(filename_or_handle, util.basestring):
                    filename_or_handle = save_progress.format(int(now))
                self.save(filename_or_handle)
                start = now

    def train(self, *args, **kwargs):
        '''Train the network until the trainer converges.

        All arguments are passed to :func:`itertrain`.

        Returns
        -------
        training : dict
            A dictionary of monitor values computed using the training dataset,
            at the conclusion of training. This dictionary will at least contain
            a 'loss' key that indicates the value of the loss function. Other
            keys may be available depending on the trainer being used.
        validation : dict
            A dictionary of monitor values computed using the validation
            dataset, at the conclusion of training.
        '''
        monitors = None
        for monitors in self.itertrain(*args, **kwargs):
            pass
        return monitors

    def _hash(self, regularizers=()):
        '''Construct a string key for representing a computation graph.

        This key will be unique for a given (a) network topology, (b) set of
        losses, and (c) set of regularizers.

        Returns
        -------
        key : str
            A hash representing the computation graph for the current network.
        '''
        def add(s):
            h.update(str(s).encode('utf-8'))
        h = hashlib.md5()
        for l in self.layers:
            add('{}{}{}'.format(l.__class__.__name__, l.name, l.output_shape))
        for l in self.losses:
            add('{}{}'.format(l.__class__.__name__, l.weight))
        for r in regularizers:
            add('{}{}{}'.format(r.__class__.__name__, r.weight, r.pattern))
        return h.hexdigest()

    def build_graph(self, regularizers=()):
        '''Connect the layers in this network to form a computation graph.

        Parameters
        ----------
        regularizers : list of :class:`theanets.regularizers.Regularizer`
            A list of the regularizers to apply while building the computation
            graph.

        Returns
        -------
        outputs : list of Theano variables
            A list of expressions giving the output of each layer in the graph.
        updates : list of update tuples
            A list of updates that should be performed by a Theano function that
            computes something using this graph.
        '''
        key = self._hash(regularizers)
        if key not in self._graphs:
            util.log('building computation graph')
            for loss in self.losses:
                loss.log()
            for reg in regularizers:
                reg.log()
            outputs = {}
            updates = []
            for layer in self.layers:
                out, upd = layer.connect(outputs)
                for reg in regularizers:
                    reg.modify_graph(out)
                outputs.update(out)
                updates.extend(upd)
            self._graphs[key] = outputs, updates
        return self._graphs[key]

    @property
    def inputs(self):
        '''A list of Theano variables for feedforward computations.'''
        return [l.input for l in self.layers if isinstance(l, layers.Input)]

    @property
    def variables(self):
        '''A list of Theano variables for loss computations.'''
        result = self.inputs
        seen = set(i.name for i in result)
        for loss in self.losses:
            for v in loss.variables:
                if v.name not in seen:
                    result.append(v)
                    seen.add(v.name)
        return result

    @property
    def params(self):
        '''A list of the learnable Theano parameters for this network.'''
        return [p for l in self.layers for p in l.params]

    def find(self, which, param):
        '''Get a parameter from a layer in the network.

        Parameters
        ----------
        which : int or str
            The layer that owns the parameter to return.

            If this is an integer, then 0 refers to the input layer, 1 refers
            to the first hidden layer, 2 to the second, and so on.

            If this is a string, the layer with the corresponding name, if any,
            will be used.

        param : int or str
            Name of the parameter to retrieve from the specified layer, or its
            index in the parameter list of the layer.

        Raises
        ------
        KeyError
            If there is no such layer, or if there is no such parameter in the
            specified layer.

        Returns
        -------
        param : Theano shared variable
            A shared parameter variable from the indicated layer.
        '''
        for i, layer in enumerate(self.layers):
            if which == i or which == layer.name:
                return layer.find(param)
        raise KeyError(which)

    def feed_forward(self, x, **kwargs):
        '''Compute a forward pass of all layers from the given input.

        All keyword arguments are passed directly to :func:`build_graph`.

        Parameters
        ----------
        x : ndarray (num-examples, num-variables)
            An array containing data to be fed into the network. Multiple
            examples are arranged as rows in this array, with columns containing
            the variables for each example.

        Returns
        -------
        layers : list of ndarray (num-examples, num-units)
            The activation values of each layer in the the network when given
            input `x`. For each of the hidden layers, an array is returned
            containing one row per input example; the columns of each array
            correspond to units in the respective layer. The "output" of the
            network is the last element of this list.
        '''
        regs = regularizers.from_kwargs(self, **kwargs)
        key = self._hash(regs)
        if key not in self._functions:
            outputs, updates = self.build_graph(regs)
            labels, exprs = list(outputs.keys()), list(outputs.values())
            util.log('compiling feed_forward function')
            self._functions[key] = (labels, theano.function(
                self.inputs, exprs, updates=updates))
        labels, f = self._functions[key]
        return dict(zip(labels, f(x)))

    def predict(self, x, **kwargs):
        '''Compute a forward pass of the inputs, returning the network output.

        All keyword arguments end up being passed to :func:`build_graph`.

        Parameters
        ----------
        x : ndarray (num-examples, num-variables)
            An array containing data to be fed into the network. Multiple
            examples are arranged as rows in this array, with columns containing
            the variables for each example.

        Returns
        -------
        y : ndarray (num-examples, num-variables)
            Returns the values of the network output units when given input `x`.
            Rows in this array correspond to examples, and columns to output
            variables.
        '''
        return self.feed_forward(x, **kwargs)[self.layers[-1].output_name]

    def score(self, x, y, w=None, **kwargs):
        '''Compute R^2 coefficient of determination for a given labeled input.

        Parameters
        ----------
        x : ndarray (num-examples, num-inputs)
            An array containing data to be fed into the network. Multiple
            examples are arranged as rows in this array, with columns containing
            the variables for each example.
        y : ndarray (num-examples, num-outputs)
            An array containing expected target data for the network. Multiple
            examples are arranged as rows in this array, with columns containing
            the variables for each example.

        Returns
        -------
        r2 : float
            The R^2 correlation between the prediction of this netork and its
            target output.
        '''
        u = y - self.predict(x, **kwargs)
        v = y - y.mean()
        if w is None:
            w = np.ones_like(u)
        return 1 - (w * u * u).sum() / (w * v * v).sum()

    def __getstate__(self):
        return (self.layers, self.losses)

    def __setstate__(self, state):
        self.layers, self.losses = state
        self._graphs = {}
        self._functions = {}

    def save(self, filename_or_handle):
        '''Save the state of this network to a pickle file on disk.

        Parameters
        ----------
        filename_or_handle : str or file handle
            Save the state of this network to a pickle file. If this parameter
            is a string, it names the file where the pickle will be saved. If it
            is a file-like object, this object will be used for writing the
            pickle. If the filename ends in ".gz" then the output will
            automatically be gzipped.
        '''
        if isinstance(filename_or_handle, util.basestring):
            opener = gzip.open if filename_or_handle.lower().endswith('.gz') else open
            handle = opener(filename_or_handle, 'wb')
        else:
            handle = filename_or_handle
        pickle.dump(self, handle, -1)
        if isinstance(filename_or_handle, util.basestring):
            handle.close()
        util.log('saved model to {}', filename_or_handle)

    @classmethod
    def load(cls, filename_or_handle):
        '''Load a saved network from disk.

        Parameters
        ----------
        filename_or_handle : str or file handle
            Load the state of this network from a pickle file. If this parameter
            is a string, it names the file where the pickle will be saved. If it
            is a file-like object, this object will be used for reading the
            pickle. If the filename ends in ".gz" then the output will
            automatically be gunzipped.
        '''
        assert not isinstance(cls, Network), \
            'cannot load an instance! say instead: net = Network.load(source)'
        if isinstance(filename_or_handle, util.basestring):
            opener = gzip.open if filename_or_handle.lower().endswith('.gz') else open
            handle = opener(filename_or_handle, 'rb')
        else:
            handle = filename_or_handle
        model = pickle.load(handle)
        if isinstance(filename_or_handle, util.basestring):
            handle.close()
        util.log('loaded model from {}', filename_or_handle)
        return model

    def loss(self, **kwargs):
        '''Return a variable representing the regularized loss for this network.

        The regularized loss includes both the :ref:`loss computation <losses>`
        for the network as well as any :ref:`regularizers <regularizers>` that
        are in place.

        Keyword arguments are passed directly to
        :func:`theanets.regularizers.from_kwargs`.

        Returns
        -------
        loss : Theano expression
            A Theano expression representing the loss of this network.
        '''
        regs = regularizers.from_kwargs(self, **kwargs)
        outputs, _ = self.build_graph(regs)
        return sum(l.weight * l(outputs) for l in self.losses) + \
            sum(r.weight * r.loss(self.layers, outputs) for r in regs)

    def monitors(self, **kwargs):
        '''Return expressions that should be computed to monitor training.

        Returns
        -------
        monitors : list of (name, expression) pairs
            A list of named monitor expressions to compute for this network.
        '''
        regs = regularizers.from_kwargs(self, **kwargs)
        outputs, _ = self.build_graph(regs)
        monitors = [('err', self.losses[0](outputs))]

        def matching(pattern):
            '''Yield all matching outputs or parameters from the graph.'''
            for name, expr in util.outputs_matching(outputs, pattern):
                yield name, expr
            for name, expr in util.params_matching(self.layers, pattern):
                yield name, expr

        def parse_levels(levels):
            '''Yield named monitor callables.'''
            if isinstance(levels, dict):
                levels = levels.items()
            if isinstance(levels, (int, float)):
                levels = [levels]
            for level in levels:
                if isinstance(level, (tuple, list)):
                    label, call = level
                    yield ':{}'.format(label), call
                if isinstance(level, (int, float)):
                    def call(expr):
                        return (expr < level).mean()
                    yield '<{}'.format(level), call

        inputs = kwargs.get('monitors', {})
        if isinstance(inputs, dict):
            inputs = inputs.items()
        for pattern, levels in inputs:
            for name, expr in matching(pattern):
                for key, value in parse_levels(levels):
                    monitors.append(('{}{}'.format(name, key), value(expr)))

        return monitors

    def updates(self, **kwargs):
        '''Return expressions to run as updates during network training.

        Returns
        -------
        updates : list of (parameter, expression) pairs
            A list of named parameter update expressions for this network.
        '''
        regs = regularizers.from_kwargs(self, **kwargs)
        _, updates = self.build_graph(regs)
        return updates


# -*- coding: utf-8 -*-

'''This module contains classes for different types of network layers.'''

from __future__ import division

import numpy as np
import theano
import theano.sparse as SS
import theano.tensor as TT

from .. import activations
from .. import util

__all__ = [
    'Concatenate',
    'Input',
    'Flatten',
    'Layer',
    'Product',
    'Reshape',
]

class Layer(util.Registrar(str('Base'), (), {})):
    '''Base class for network layers.

    In ``theanets``, a layer refers to a logically grouped set of parameters and
    computations. Typically this encompasses a set of weight matrix and bias
    vector parameters, plus the "output" units that produce a signal for further
    layers to consume.

    Subclasses of this class can be created to implement many different forms of
    layer-specific computation. For example, a vanilla :class:`Feedforward`
    layer accepts input from the "preceding" layer in a network, computes an
    affine transformation of that input and applies a pointwise transfer
    function. On the other hand, a :class:`Recurrent` layer computes an affine
    transformation of the current input, and combines that with information
    about the state of the layer at previous time steps.

    Most subclasses will need to provide an implementation of the :func:`setup`
    method, which creates the parameters needed by the layer, and the
    :func:`transform` method, which converts the Theano input expressions coming
    in to the layer into some output expression(s).

    Parameters
    ----------
    size : int, optional
        If provided, give the size of outputs from this layer. Providing both
        ``size`` and ``shape`` will raise an exception -- in general, use
        ``shape`` whenever the architecture uses convolutions, and ``size`` for
        other cases.
    shape : tuple of int, optional
        If provided, define the shape of the values generated by this layer. The
        shape is given for a single data element (not a batch of such elements),
        so for a vanilla feedforward layer, this would be ``(size, )``, while a
        convolutional layer might have shape ``(width, height, channels)``.
    inputs : str or tuple of str, optional
        Name(s) of input(s) to this layer. These names must be
        :func:`resolved <resolve>` to layers by :func:`binding <bind>` the layer
        inside a :class:`network graph <theanets.graph.Network>`. Defaults to an
        empty tuple; in practice this needs to be provided for most layers.
    name : str, optional
        The name of this layer. If not given, layers will be numbered
        sequentially based on the order in which they are created.
    activation : str, optional
        The name of an activation function to use for units in this layer. See
        :func:`build_activation`.
    rng : :class:`numpy.random.RandomState` or int, optional
        A numpy random number generator, or an integer seed for a random number
        generator. If not provided, the random number generator will be created
        with an automatically chosen seed.
    mean, mean_XYZ : float, optional
        Initialize parameters for this layer to have the given mean value. If
        ``mean_XYZ`` is specified, it will apply only to the parameter named
        XYZ. Defaults to 0.
    std, std_XYZ : float, optional
        Initialize parameters for this layer to have the given standard
        deviation. If ``std_XYZ`` is specified, only the parameter named XYZ
        will be so initialized. Defaults to 0.
    sparsity, sparsity_XYZ : float in (0, 1), optional
        If given, create sparse connections in the layer's weight matrix, such
        that this fraction of the weights is set to zero. If ``sparsity_XYZ`` is
        given, it will apply only the parameter with name XYZ. By default, this
        parameter is 0, meaning all weights are nonzero.
    diagonal, diagonal_XYZ : float, optional
        If given, create initial parameter matrices for this layer that are
        initialized to diagonal matrices with this value along the diagonal.
        Defaults to None, which initializes all weights using random values.

    Attributes
    ----------
    name : str
        Name of this layer.
    activate : callable
        The activation function to use on this layer's outputs.
    kwargs : dict
        Additional keyword arguments used when constructing this layer.
    '''

    _count = 0

    def __init__(self, name=None, **kwargs):
        super(Layer, self).__init__()

        Layer._count += 1
        self.name = name or '{}{}'.format(
            self.__class__.__name__.lower(), Layer._count)

        self.kwargs = kwargs
        self.rng = kwargs.get('rng', kwargs.get('nrng'))
        if self.rng is None or isinstance(self.rng, int):
            self.rng = np.random.RandomState(self.rng)

        self._params = []
        self._input_shapes = {}
        self._output_shapes = {}

        inputs = kwargs.get('inputs', ())
        if not isinstance(inputs, (tuple, list)):
            inputs = (inputs, )
        for input in inputs:
            if isinstance(input, Layer):
                self._input_shapes[input.output_name] = input
            else:
                self._input_shapes[input] = None

    @property
    def params(self):
        '''A list of all parameters in this layer.'''
        return self._params + getattr(self.activate, 'params', [])

    @property
    def input_name(self):
        '''Name of layer input (for layers with one input).'''
        if len(self._input_shapes) != 1:
            raise util.ConfigurationError(
                'expected one input for layer "{}", got {}'
                .format(self.name, self._input_shapes))
        return list(self._input_shapes)[0]

    @property
    def input_shape(self):
        '''Shape of layer input (for layers with one input).'''
        return self._input_shapes[self.input_name]

    @property
    def input_size(self):
        '''Size of layer input (for layers with one input).'''
        shape = self.input_shape
        if shape is None:
            raise util.ConfigurationError(
                'undefined input size for layer "{}"'.format(self.name))
        return shape[-1]

    @property
    def output_name(self):
        '''Full name of the default output for this layer.'''
        return self.full_name('out')

    @property
    def output_shape(self):
        '''Shape of default output from this layer.'''
        return self._output_shapes['out']

    @property
    def output_size(self):
        '''Number of "neurons" in this layer's default output.'''
        shape = self.output_shape
        if shape is None:
            raise util.ConfigurationError(
                'undefined output size for layer "{}"'.format(self.name))
        return shape[-1]

    def full_name(self, name):
        '''Return a fully-scoped name for the given layer output.

        Parameters
        ----------
        name : str
            Name of an output for this layer.

        Returns
        -------
        scoped : str
            A fully-scoped name for the given output from this layer.
        '''
        return '{}:{}'.format(self.name, name)

    def connect(self, inputs):
        '''Create Theano variables representing the outputs of this layer.

        Parameters
        ----------
        inputs : dict of Theano expressions
            Symbolic inputs to this layer, given as a dictionary mapping string
            names to Theano expressions. Each string key should be of the form
            "{layer_name}:{output_name}" and refers to a specific output from
            a specific layer in the graph.

        Returns
        -------
        outputs : dict
            A dictionary mapping names to Theano expressions for the outputs
            from this layer.
        updates : sequence of (parameter, expression) tuples
            Updates that should be performed by a Theano function that computes
            something using this layer.
        '''
        outputs, updates = self.transform(inputs)
        # transform the outputs to be a list of ordered pairs if needed.
        if isinstance(outputs, dict):
            outputs = sorted(outputs.items())
        if isinstance(outputs, (TT.TensorVariable, SS.SparseVariable)):
            outputs = [('out', outputs)]
        outs = {self.full_name(name): expr for name, expr in outputs}
        return outs, updates

    def transform(self, inputs):
        '''Transform the inputs for this layer into an output for the layer.

        Parameters
        ----------
        inputs : dict of Theano expressions
            Symbolic inputs to this layer, given as a dictionary mapping string
            names to Theano expressions. See :func:`Layer.connect`.

        Returns
        -------
        output : Theano expression
            The output for this layer is the same as the input.
        updates : list
            An empty updates list.
        '''
        raise NotImplementedError

    def bind(self, graph, reset=True, initialize=True):
        '''Bind this layer into a computation graph.

        This method is a wrapper for performing common initialization tasks. It
        calls :func:`resolve`, :func:`setup`, and :func:`log`.

        Parameters
        ----------
        graph : :class:`Network <theanets.graph.Network>`
            A computation network in which this layer is to be bound.
        reset : bool, optional
            If ``True`` (the default), reset the resolved layers for this layer.
        initialize : bool, optional
            If ``True`` (the default), initialize the parameters for this layer
            by calling :func:`setup`.

        Raises
        ------
        theanets.util.ConfigurationError :
            If an input cannot be resolved.
        '''
        if reset:
            for k in self._input_shapes:
                self._input_shapes[k] = None
            for k in self._output_shapes:
                self._output_shapes[k] = None
        self.resolve_inputs(graph.layers)
        self.resolve_outputs()
        self.activate = activations.build(
            self.kwargs.get('activation', 'relu'), self)
        if initialize:
            self.setup()
        self.log()

    def resolve_inputs(self, layers):
        '''Resolve the names of inputs for this layer into shape tuples.

        Parameters
        ----------
        layers : list of :class:`Layer`
            A list of the layers that are available for resolving inputs.

        Raises
        ------
        theanets.util.ConfigurationError :
            If an input cannot be resolved.
        '''
        resolved = {}
        for name, shape in self._input_shapes.items():
            if shape is None:
                name, shape = self._resolve_shape(name, layers)
            resolved[name] = shape
        self._input_shapes = resolved

    def resolve_outputs(self):
        '''Resolve the names of outputs for this layer into shape tuples.'''
        input_shape = None
        for i, shape in enumerate(self._input_shapes.values()):
            if i == 0:
                input_shape = shape
            if len(input_shape) != len(shape) or any(
                    a is not None and b is not None and a != b
                    for a, b in zip(input_shape[:-1], shape[:-1])):
                raise util.ConfigurationError(
                    'layer "{}" incompatible input shapes {}'
                    .format(self.name, self._input_shapes))
        size = self.kwargs.get('size')
        shape = self.kwargs.get('shape')
        if shape is not None:
            pass
        elif size is not None:
            shape = tuple(input_shape[:-1]) + (size, )
        else:
            raise util.ConfigurationError(
                'layer "{}" does not specify a size'.format(self.name))
        self._output_shapes['out'] = shape

    def setup(self):
        '''Set up the parameters and initial values for this layer.'''
        pass

    def log(self):
        '''Log some information about this layer.'''
        inputs = ', '.join('"{0}" {1}'.format(*ns) for ns in self._input_shapes.items())
        util.log('layer {0.__class__.__name__} "{0.name}" {0.output_shape} {1} from {2}',
                 self, getattr(self.activate, 'name', self.activate), inputs)
        util.log('learnable parameters: {}', self.log_params())

    def log_params(self):
        '''Log information about this layer's parameters.'''
        total = 0
        for p in self.params:
            shape = p.get_value().shape
            util.log('parameter "{}" {}', p.name, shape)
            total += np.prod(shape)
        return total

    def _fmt(self, string):
        '''Helper method to format our name into a string.'''
        if '{' not in string:
            string = '{}.' + string
        return string.format(self.name)

    def _resolve_shape(self, name, layers):
        '''Given a list of layers, find the layer output with the given name.

        Parameters
        ----------
        name : str
            Name of a layer to resolve.
        layers : list of :class:`theanets.layers.base.Layer`
            A list of layers to search in.

        Raises
        ------
        util.ConfigurationError :
            If there is no such layer, or if there are more than one.

        Returns
        -------
        name : str
            The fully-scoped name of the desired output.
        shape : tuple of None and/or int
            The shape of the named output.
        '''
        matches = [l for l in layers if name.split(':')[0] == l.name]
        if len(matches) != 1:
            raise util.ConfigurationError(
                'layer "{}" cannot resolve "{}" using {}'
                .format(self.name, name, [l.name for l in layers]))
        name = name if ':' in name else matches[0].output_name
        return name, matches[0]._output_shapes[name.split(':')[1]]

    def find(self, key):
        '''Get a shared variable for a parameter by name.

        Parameters
        ----------
        key : str or int
            The name of the parameter to look up, or the index of the parameter
            in our parameter list. These are both dependent on the
            implementation of the layer.

        Returns
        -------
        param : shared variable
            A shared variable containing values for the given parameter.

        Raises
        ------
        KeyError
            If a param with the given name does not exist.
        '''
        name = self._fmt(str(key))
        for i, p in enumerate(self._params):
            if key == i or name == p.name:
                return p
        raise KeyError(key)

    def add_weights(self, name, nin, nout, mean=0, std=0, sparsity=0, diagonal=0):
        '''Helper method to create a new weight matrix.

        Parameters
        ----------
        name : str
            Name of the parameter to add.
        nin : int
            Size of "input" for this weight matrix.
        nout : int
            Size of "output" for this weight matrix.
        mean : float, optional
            Mean value for randomly-initialized weights. Defaults to 0.
        std : float, optional
            Standard deviation of initial matrix values. Defaults to
            :math:`1 / sqrt(n_i + n_o)`.
        sparsity : float, optional
            Fraction of weights to be set to zero. Defaults to 0.
        diagonal : float, optional
            Initialize weights to a matrix of zeros with this value along the
            diagonal. Defaults to None, which initializes all weights randomly.
        '''
        glorot = 1 / np.sqrt(nin + nout)
        m = self.kwargs.get(
            'mean_{}'.format(name), self.kwargs.get('mean', mean))
        s = self.kwargs.get(
            'std_{}'.format(name), self.kwargs.get('std', std or glorot))
        p = self.kwargs.get(
            'sparsity_{}'.format(name), self.kwargs.get('sparsity', sparsity))
        d = self.kwargs.get(
            'diagonal_{}'.format(name), self.kwargs.get('diagonal', diagonal))
        self._params.append(theano.shared(
            util.random_matrix(nin, nout, mean=m, std=s, sparsity=p,
                               diagonal=d, rng=self.rng),
            name=self._fmt(name)))

    def add_bias(self, name, size, mean=0, std=1):
        '''Helper method to create a new bias vector.

        Parameters
        ----------
        name : str
            Name of the parameter to add.
        size : int
            Size of the bias vector.
        mean : float, optional
            Mean value for randomly-initialized biases. Defaults to 0.
        std : float, optional
            Standard deviation for randomly-initialized biases. Defaults to 1.
        '''
        mean = self.kwargs.get('mean_{}'.format(name), mean)
        std = self.kwargs.get('std_{}'.format(name), std)
        self._params.append(theano.shared(
            util.random_vector(size, mean, std, rng=self.rng),
            name=self._fmt(name)))

    def to_spec(self):
        '''Create a specification dictionary for this layer.

        Returns
        -------
        spec : dict
            A dictionary specifying the configuration of this layer.
        '''
        spec = dict(**self.kwargs)
        spec.update(
            form=self.__class__.__name__.lower(),
            name=self.name,
            activation=self.kwargs.get('activation', 'relu'),
        )
        return spec

class Input(Layer):
    '''A layer that receives external input data.

    Input layers are responsible for the Theano variables that represent input
    to a network. The name of the layer is passed along to the symbolic Theano
    input variable.

    Input layers essentially add only noise to the input data (if desired), but
    otherwise reproduce their inputs exactly.

    Parameters
    ----------
    ndim : int, optional
        Number of dimensions required to store the input data for this layer.
        If a ``shape`` tuple is given, this defaults to ``1 + len(shape)``;
        otherwise, this defaults to 2, i.e., ``(num-examples, num-variables)``.
    sparse : bool or str, optional
        If this is ``'csr'`` or ``'csc'``, then the inputs to the loss will be
        stored as sparse matrices in the CSR or CSC format (respectively). If
        this is True, sparse input will be enabled in CSR format. By default
        this is False, which means inputs are dense.

    Raises
    ------
    AssertionError :
        If ``sparse`` is enabled and ``ndim`` is not 2.
    '''

    def __init__(self, name='in', ndim=2, sparse=False, **kwargs):
        shape = kwargs.get('shape')
        if shape:
            ndim = 1 + len(shape)
        else:
            kwargs['shape'] = (None, ) * (ndim - 2) + (kwargs.pop('size'), )
        self.input = util.FLOAT_CONTAINERS[ndim](name)
        if sparse is True or \
           isinstance(sparse, util.basestring) and sparse.lower() == 'csr':
            assert ndim == 2, 'Theano only supports sparse arrays with 2 dims'
            self.input = SS.csr_matrix('input')
        if isinstance(sparse, util.basestring) and sparse.lower() == 'csc':
            assert ndim == 2, 'Theano only supports sparse arrays with 2 dims'
            self.input = SS.csc_matrix('input')
        kwargs['activation'] = 'linear'
        super(Input, self).__init__(name=name, **kwargs)

    def resolve_inputs(self, layers):
        pass

    def resolve_outputs(self):
        self._output_shapes['out'] = tuple(self.kwargs['shape'])

    def log(self):
        util.log('layer {0.__class__.__name__} "{0.name}" {0.output_shape}', self)

    def transform(self, inputs):
        return self.input, []

class Product(Layer):
    '''Multiply several inputs together elementwise.

    Notes
    -----

    This layer performs an elementwise multiplication of multiple inputs; all
    inputs must be the same shape.

    *Outputs*

    - ``out`` --- elementwise product of its inputs
    '''

    __extra_registration_keys__ = ['prod']

    def resolve_outputs(self):
        template = None
        for i, shape in enumerate(self._input_shapes.values()):
            if i == 0:
                template = shape
            if shape is None or len(template) != len(shape) or \
               any(t != s for t, s in zip(template, shape)):
                raise util.ConfigurationError(
                    'incompatible inputs for product layer "{}": {}'
                    .format(self.name, self._input_shapes))
        self._output_shapes['out'] = template

    def transform(self, inputs):
        return dict(out=np.prod([inputs[k] for k in self._input_shapes])), []

class Flatten(Layer):
    '''Flatten all but the batch index of the input.

    Notes
    -----

    In ``theanets``, the leading axis of a data array always runs over the
    examples in a mini-batch. Since the number of examples in a mini-batch is
    constant throughout a network graph, this layer always preserves the shape
    of the leading axis of its inputs.

    This layer type flattens all of the non-leading dimensions of its inputs
    into one dimension. If you'd like to perform an arbitrary reshape of the
    input data, use a :class:`Reshape` layer.

    *Outputs*

    - ``out`` --- flattened inputs
    '''

    __extra_registration_keys__ = ['flat']

    def resolve_outputs(self):
        try:
            computed_size = np.prod(self.input_shape)
        except:
            computed_size = None
        specified_size = self.kwargs.get('size')
        if specified_size and specified_size != computed_size:
            raise util.ConfigurationError(
                'flatten layer "{}" size ({},) does not match input {}'
                .format(self.name, computed_size, self.input_shape))
        if computed_size is None:
            raise util.ConfigurationError(
                'cannot compute size of flatten layer "{}" from input {}'
                .format(self.name, self.input_shape))
        self._output_shapes['out'] = (computed_size, )

    def transform(self, inputs):
        x = inputs[self.input_name]
        return dict(out=x.reshape([x.shape[0], -1])), []

class Concatenate(Layer):
    '''Concatenate multiple inputs along the last axis.

    Notes
    -----

    This layer concatenates multiple inputs along their last dimension; all
    inputs must have the same dimensionality and the same shape along all but
    the last dimension. The size of this layer must equal the sum of the sizes
    of the inputs.

    *Outputs*

    - ``out`` --- inputs concatenated along last axis
    '''

    __extra_registration_keys__ = ['concat']

    def resolve_outputs(self):
        template = None
        sizes = []
        for i, shape in enumerate(self._input_shapes.values()):
            if i == 0:
                template = shape
            if shape is None or len(template) != len(shape) or \
               any(t != s for t, s in zip(template[:-1], shape[:-1])):
                raise util.ConfigurationError(
                    'incompatible inputs for concatenate layer "{}": {}'
                    .format(self.name, self._input_shapes))
            sizes.append(shape[-1])
        self._output_shapes['out'] = template[:-1] + (sum(sizes), )

    def transform(self, inputs):
        # using axis=-1 doesn't work with concatenate!
        tensors = [inputs[k] for k in self._input_shapes]
        out = TT.concatenate(tensors, axis=tensors[0].ndim - 1)
        return dict(out=out), []

class Reshape(Layer):
    '''Reshape an input to have different numbers of dimensions.

    Notes
    -----

    In ``theanets``, the leading axis of a data array always runs over the
    examples in a mini-batch. Since the number of examples in a mini-batch is
    constant throughout a network graph, this layer always preserves the shape
    of the leading axis of its inputs.

    If you want to vectorize a data array, you could do that using ``(-1, )`` as
    the shape for this layer. But it's often easier to read if you use the
    :class:`Flatten` layer type to reshape a layer's output into a flat vector.

    *Outputs*

    - ``out`` --- reshaped inputs

    Parameters
    ----------
    shape : sequence of int
        The desired shape of the output "vectors" for this layer. This should
        not include the leading axis of the actual shape of the data arrays
        processed by the graph! For example, to reshape input vectors of length
        a * b into 2D output "images" use ``(a, b)`` as the shape---not
        ``(batch-size, a, b)``.

    Attributes
    ----------
    shape : list of int
        The desired shape of the output "vectors" for this layer.
    '''

    def resolve_outputs(self):
        shape = self.kwargs.get('shape')
        if not isinstance(shape, (tuple, list)):
            raise util.ConfigurationError(
                'reshape layer "{}" has invalid shape {}'.format(self.name, shape))
        self._output_shapes['out'] = tuple(shape)
        try:
            source = np.prod(self.input_shape)
            target = np.prod(shape)
        except TypeError:
            pass
        if not source or not target:
            util.log('reshape layer "{}" has incomplete shape info, '
                     'we will run anyway and hope for the best', self.name)
        elif source != target:
            raise util.ConfigurationError(
                'incompatible shapes for reshape layer "{}": input '
                '{} <> output {}'.format(self.name, self.input_shape, shape))

    def transform(self, inputs):
        x = inputs[self.input_name]
        return dict(out=x.reshape([x.shape[0]] + self.output_shape)), []


# -*- coding: utf8 -*-
from __future__ import division
"""
LogGabor

See http://pythonhosted.org/LogGabor

"""
__author__ = "(c) Laurent Perrinet INT - CNRS"
import numpy as np
from SLIP import Image

class LogGabor(Image):
    """
    Defines a LogGabor framework by defining a ``loggabor`` function which return the envelope of a log-Gabor filter.

    Its envelope is equivalent to a log-normal probability distribution on the frequency axis, and von-mises on the radial axis.

    """
    def __init__(self, pe):
        Image.__init__(self, pe)
        self.init_logging(name='LogGabor')
        self.init()

    ## PYRAMID
    def init(self):
        Image.init(self)

        self.n_levels = int(np.log(np.max((self.pe.N_X, self.pe.N_Y)))/np.log(self.pe.base_levels))
        self.sf_0 = .5 * (1 - 1/self.n_levels) / np.logspace(0, self.n_levels-1, self.n_levels, base=self.pe.base_levels, endpoint=False)
        self.theta = np.linspace(-np.pi/2, np.pi/2, self.pe.n_theta+1)[1:]
        self.oc = (self.pe.N_X * self.pe.N_Y * self.pe.n_theta * self.n_levels) #(1 - self.pe.base_levels**-2)**-1)
        if self.pe.use_cache is True:
            self.cache = {'band':{}, 'orientation':{}}
        # self.envelope = np.zeros((self.pe.N_X, self.pe.N_Y))

    def linear_pyramid(self, image):

        C = np.empty((self.pe.N_X, self.pe.N_Y, self.pe.n_theta, self.n_levels), dtype=np.complex)
        for i_sf_0, sf_0 in enumerate(self.sf_0):
            for i_theta, theta in enumerate(self.theta):
                FT_lg = self.loggabor(0, 0, sf_0=sf_0, B_sf=self.pe.B_sf,
                                    theta=theta, B_theta=self.pe.B_theta)
                C[:, :, i_theta, i_sf_0] = self.FTfilter(image, FT_lg, full=True)
        return C

    def argmax(self, C):
        """
        Returns the ArgMax from C by returning the
        (x_pos, y_pos, theta, scale)  tuple

        >>> C = np.random.randn(10, 10, 5, 4)
        >>> x_pos, y_pos, theta, scale = mp.argmax(C)
        >>> C[x_pos][y_pos][theta][scale] = C.max()

        """
        ind = np.absolute(C).argmax()
        return np.unravel_index(ind, C.shape)

    def golden_pyramid(self, z, mask=False, spiral=True, fig_width=13):
        """
        The Golden Laplacian Pyramid.
        To represent the edges of the image at different levels, we may use a simple recursive approach constructing progressively a set of images of decreasing sizes, from a base to the summit of a pyramid. Using simple down-scaling and up-scaling operators we may approximate well a Laplacian operator. This is represented here by stacking images on a Golden Rectangle, that is where the aspect ratio is the golden section $\phi \eqdef \frac{1+\sqrt{5}}{2}$. We present here the base image on the left and the successive levels of the pyramid in a clockwise fashion (for clarity, we stopped at level $8$). Note that here we also use $\phi^2$ (that is $\phi+1$) as the down-scaling factor so that the resolution of the pyramid images correspond across scales. Note at last that coefficient are very kurtotic: most are near zero, the distribution of coefficients has long tails.

        """
        import matplotlib.pyplot as plt

        opts= {'vmin':0., 'vmax':1., 'interpolation':'nearest', 'origin':'upper'}

        N_X, N_Y = z.shape[0], z.shape[1]
        if spiral:
            phi = (np.sqrt(5)+1.)/2. # golden ratio
            fig = plt.figure(figsize=(fig_width, N_X/N_Y*fig_width/phi), frameon=True)
            xmin, ymin, size = 0, 0, 1.
        else:
            fig = plt.figure(figsize=(fig_width, N_X/N_Y*fig_width*self.n_levels), frameon=True)

        axs = []
        for i_sf_0 in range(len(self.sf_0)):
            if spiral:
                # https://matplotlib.org/api/_as_gen/matplotlib.figure.Figure.html#matplotlib.figure.Figure.add_axes says:
                # Add an axes at position rect [left, bottom, width, height] where all quantities are in fractions of figure width and height.
                ax = fig.add_axes((xmin/phi, ymin, size/phi, size), facecolor='w')
            else:
                ax = fig.add_axes((0, i_sf_0/self.n_levels, 1, 1/self.n_levels), facecolor='w')
            ax.axis(c='r', lw=1)
            plt.setp(ax, xticks=[], yticks=[])
            im_RGB = np.zeros((self.pe.N_X, self.pe.N_Y, 3))
            for i_theta, theta_ in enumerate(self.theta):
                im_abs = np.absolute(z[:, :, i_theta, i_sf_0])
                RGB = np.array([.5*np.sin(2*theta_ + 2*i*np.pi/3)+.5 for i in range(3)])
                im_RGB += im_abs[:,:, np.newaxis] * RGB[np.newaxis, np.newaxis, :]

            im_RGB /= im_RGB.max()
            ax.imshow(1-im_RGB, **opts)
            #ax.grid(b=False, which="both")
            if mask:
                linewidth_mask = 1 #
                from matplotlib.patches import Ellipse
                circ = Ellipse((.5*self.pe.N_Y, .5*self.pe.N_X),
                                self.pe.N_Y-linewidth_mask, self.pe.N_X-linewidth_mask,
                                fill=False, facecolor='none', edgecolor = 'black', alpha = 0.5, ls='dashed', lw=linewidth_mask)
                ax.add_patch(circ)
            if spiral:
                i_orientation = np.mod(i_sf_0, 4)
                if i_orientation==0:
                    xmin += size
                    ymin += size/phi**2
                elif i_orientation==1:
                    xmin += size/phi**2
                    ymin += -size/phi
                elif i_orientation==2:
                    xmin += -size/phi
                elif i_orientation==3:
                    ymin += size
                size /= phi
            axs.append(ax)

        return fig, axs

    ## LOW LEVEL OPERATIONS
    def band(self, sf_0, B_sf, force=False):
        """
        Returns the radial frequency envelope:

        Selects a preferred spatial frequency ``sf_0`` and a bandwidth ``B_sf``.

        """
        if sf_0 == 0.:
            return 1.
        elif self.pe.use_cache and not force:
            tag = str(sf_0) + '_' + str(B_sf)
            try:
                return self.cache['band'][tag]
            except:
                if self.pe.verbose>50: print('doing band cache for tag ', tag)
                self.cache['band'][tag] = self.band(sf_0, B_sf, force=True)
                return self.cache['band'][tag]
        else:
            # see http://en.wikipedia.org/wiki/Log-normal_distribution
            env = 1./self.f*np.exp(-.5*(np.log(self.f/sf_0)**2)/B_sf**2)
        return env

    def orientation(self, theta, B_theta, force=False):
        """
        Returns the orientation envelope:
        We use a von-Mises distribution on the orientation:
        - mean orientation is ``theta`` (in radians),
        - ``B_theta`` is the bandwidth (in radians). It is equal to the standard deviation of the Gaussian
        envelope which approximate the distribution for low bandwidths. The Half-Width at Half Height is
        given by approximately np.sqrt(2*B_theta_**2*np.log(2)).

        # selecting one direction,  theta is the mean direction, B_theta the spread
        # we use a von-mises distribution on the orientation
        # see http://en.wikipedia.org/wiki/Von_Mises_distribution
        """
        if B_theta is np.inf: # for large bandwidth, returns a strictly flat envelope
            enveloppe_orientation = 1.
        elif self.pe.use_cache and not force:
            tag = str(theta) + '_' + str(B_theta)
            try:
                return self.cache['orientation'][tag]
            except:
                if self.pe.verbose>50: print('doing orientation cache for tag ', tag)
                self.cache['orientation'][tag] = self.orientation(theta, B_theta, force=True)
                return self.cache['orientation'][tag]
        else: # non pathological case
            # As shown in:
            #  http://www.csse.uwa.edu.au/~pk/research/matlabfns/PhaseCongruency/Docs/convexpl.html
            # this single bump allows (without the symmetric) to code both symmetric
            # and anti-symmetric parts in one shot.
            cos_angle = np.cos(self.f_theta-theta)
            enveloppe_orientation = np.exp(cos_angle/B_theta**2)
        return enveloppe_orientation

    ## MID LEVEL OPERATIONS
    def loggabor(self, x_pos, y_pos, sf_0, B_sf, theta, B_theta, preprocess=True):
        """
        Returns the envelope of a LogGabor

        Note that the convention for coordinates follows that of matrices:
        the origin is at the top left of the image, and coordinates are first
        the rows (vertical axis, going down) then the columns (horizontal axis,
        going right).

        """

        env = np.multiply(self.band(sf_0, B_sf), self.orientation(theta, B_theta))
        if not(x_pos==0.) and not(y_pos==0.): # bypass translation whenever none is needed
              env = env.astype(np.complex128) * self.trans(x_pos*1., y_pos*1.)
        if preprocess : env *= self.f_mask # retina processing
        # normalizing energy:
        env /= np.sqrt((np.abs(env)**2).mean())
        # in the case a a single bump (see ``orientation``), we should compensate the fact that the distribution gets complex:
        env *= np.sqrt(2.)
        return env

    def loggabor_image(self, x_pos, y_pos, theta, sf_0, phase, B_sf, B_theta):
        """
        Returns the image of a LogGabor

        Note that the convention for coordinates follows that of matrices:
        the origin is at the top left of the image, and coordinates are first
        the rows (vertical axis, going down) then the columns (horizontal axis,
        going right).

        """

        FT_lg = self.loggabor(x_pos, y_pos, sf_0=sf_0, B_sf=B_sf, theta=theta, B_theta=B_theta)
        FT_lg = FT_lg * np.exp(1j * phase)
        return self.invert(FT_lg, full=False)

    def show_loggabor(self, u, v, sf_0, B_sf, theta, B_theta, title='', phase=0.):
        FT_lg = self.loggabor(u, v, sf_0, B_sf, theta, B_theta)
        fig, a1, a2 = self.show_FT(FT_lg * np.exp(-1j*phase))
        return fig, a1, a2

class LogGaborFit(LogGabor):
    """
    Defines a  framework to fit a LogGabor.

    """
    def __init__(self, pe):
        LogGabor.__init__(self, pe)
        self.init_logging(name='LogGaborFit')
        self.init()

    def LogGaborFit(self, patch, do_border=True):
        from lmfit import Parameters, minimize, fit_report
        N_X, N_Y = patch.shape

        #initial guess is the one corresponding to the Maximum Likelihood Estimate over the linear pyramid

        C = self.linear_pyramid(patch) # np.reshape(patch, (N_X, N_Y)))
        idx = self.argmax(C)
        fit_params = Parameters()
        fit_params.add('x_pos', value=idx[0], min=0, max=N_X)
        fit_params.add('y_pos', value=idx[1], min=0, max=N_Y)
        fit_params.add('theta', value=self.theta[idx[2]], min=-np.pi/2, max=np.pi/2)
        fit_params.add('sf_0', value=self.sf_0[idx[3]], min=0.001)
        fit_params.add('B_sf', value=self.pe.B_sf, min=0.001, vary=False)
        fit_params.add('B_theta', value=self.pe.B_theta, min=0.001, vary=False)
        fit_params.add('phase', value=np.angle(C[idx]))

        # step 1
        out = minimize(self.residual, params=fit_params, kws={'data':patch}, nan_policy='omit')
        # step 2
        out.params['B_sf'].set(vary=True)
        out.params['B_theta'].set(vary=True)
        out = minimize(self.residual, params=out.params, kws={'data':patch}, nan_policy='omit', method='Nelder-Mead')

        if do_border: self.set_size((N_X + N_X // 2, N_Y + N_Y // 2))
        patch_fit = self.loggabor_image(**out.params)
        if do_border:
            patch_fit = patch_fit[:N_X, :N_Y]
            self.set_size((N_X, N_Y))

        return patch_fit, out.params

    def residual(self, pars, data):  # =None, eps=None):
        # unpack parameters:
        #  extract .value attribute for each parameter
        parvals = pars.valuesdict()
        x_pos = parvals['x_pos']
        y_pos = parvals['y_pos']
        # theta = (parvals['theta']  % np.pi)  - np.pi/2
        theta = parvals['theta']  # % np.pi
        B_theta = parvals['B_theta']
        sf_0 = np.abs(parvals['sf_0'])
        B_sf = parvals['B_sf']
        # phase = (parvals['phase']  % (2*np.pi)) - np.pi
        phase = parvals['phase']

        model = self.loggabor_image(x_pos, y_pos, theta, sf_0, phase, B_sf, B_theta)

        if np.sqrt(np.sum(model ** 2)) == 0: print('warning: zero norm model with pars=', pars)
        #energy norm
        model /= np.sqrt(np.sum(model ** 2))
        data /= np.sqrt(np.sum(data ** 2))

        return (model - data).ravel()

    def LogGaborFit_dictionary(self, dictx, verbose=False, get_unfitted=False, whoswho=False):

        if whoswho:
            names=[]
            names.append('dictx_fit_param[:,0] = x0')
            names.append('dictx_fit_param[:,1] = y0')
            names.append('dictx_fit_param[:,2] = theta')
            names.append('dictx_fit_param[:,3] = sf_0')
            names.append('dictx_fit_param[:,4] = Phase')
            names.append('dictx_fit_param[:,5] = B_sf')
            names.append('dictx_fit_param[:,6] = B_theta')

        dictx_fit = np.zeros_like(dictx)
        dictx_fit_param = np.zeros((dictx_fit.shape[0], 7))
        idx_unfitted = []

        N_X = int(np.sqrt(dictx.shape[1]))
        N_Y = N_X

        self.set_size((N_X, N_Y))
        self.pe.N_X = N_X
        self.pe.N_Y = N_Y

        for i in range(dictx.shape[0]):

            if verbose:
                print("Fitting patch % 3i /  % 3i" %(i + 1, dictx.shape[0]))

            try:
                N_X, N_Y = int(np.sqrt(dictx.shape[1])), int(np.sqrt(dictx.shape[1]))
                dict_ = np.reshape(dictx[i, :], (N_X, N_Y))
                #print(dictx_fit[i, :].shape)#, dictx_fit_param[i, :].shape, self.LogGaborFit(dict_))
                patch_fit, out_params = self.LogGaborFit(dict_)
                dictx_fit[i, :] = patch_fit.ravel()
                dictx_fit_param[i, :] = np.array([out_params[key].value for key in list(out_params)])
            except ValueError as e:
                print(out_params)
                if verbose:
                    print("Couldn't fit patch number % 3i" % (i+1), "error=", e)
                dictx_fit[i, :] = np.zeros((1, dictx.shape[1]))
                dictx_fit_param[i, :] = np.zeros((1, 7))
                idx_unfitted.append(i)

        if get_unfitted:
            if whoswho:
                return dictx_fit, dictx_fit_param, idx_unfitted, names
            else:
                return dictx_fit, dictx_fit_param, idx_unfitted
        else:
            if whoswho:
                return dictx_fit, dictx_fit_param, names
            else:
                return dictx_fit, dictx_fit_param

def _test():
    import doctest
    doctest.testmod()
#####################################
#
if __name__ == '__main__':
    _test()

    #### Main
    """
    Some examples of use for the class

    """
    lg = LogGabor('default_param.py')
    from SLIP import imread
    image = imread('database/lena512.png')[:,:,0]
    lg.set_size(image)


#!/usr/bin/env python
# -*- coding: utf-8 -*-

import codecs
import re
import struct

VERSION = '1.69'

class TextGrid:
    """Read write and edit Praat's TextGrid files.

    .. note:: All times are in seconds and can have decimals

    :var float xmin: Minimum x value.
    :var float xmax: Maximum x value.
    :var int tier_num: Number of tiers.
    :var list tiers: Internal (unsorted) list of tiers.
    :var str codec: Codec of the input file.
    """
    def __init__(self, file_path=None, xmin=0, xmax=None, codec='utf-8'):
        """Construct either a new TextGrid object or read one from a
        file/stream. When you create an empty TextGrid you must at least
        specify the xmax. When you want to load a TextGrid from file you need
        to specify at least the file_path and optionally the codec. Binary,
        short and normal TextGrids are supported.

        :param str file_path: Path to read from, - for stdin. If ``None`` an
                              empty TextGrid will be created.
        :param int xmin: Xmin value, only needed when not loading from file.
        :param int xmax: Xmax value, needed when not loading from file.
        :param str codec: Text encoding for the input. Note that this will be
            ignored for binary TextGrids.
        :raises Exception: If filepath is not specified but no xmax
        """
        self.tiers = []
        self.codec = codec
        if not file_path:
            if xmax is None:
                raise Exception('No xmax specified')
            self.tier_num = 0
            self.xmin = xmin
            self.xmax = xmax
        else:
            with open(file_path, 'rb') as f:
                self.from_file(f, codec)

    def from_file(self, ifile, codec='ascii'):
        """Read textgrid from stream.

        :param file ifile: Stream to read from.
        :param str codec: Text encoding for the input. Note that this will be
            ignored for binary TextGrids.
        """
        if ifile.read(12) == b'ooBinaryFile':
            def bin2str(ifile):
                textlen = struct.unpack('>h', ifile.read(2))[0]
                # Single byte characters
                if textlen >= 0:
                    return ifile.read(textlen).decode('ascii')
                # Multi byte characters have initial len -1 and then \xff bytes
                elif textlen == -1:
                    textlen = struct.unpack('>h', ifile.read(2))[0]
                    data = ifile.read(textlen*2)
                    # Hack to go from number to unicode in python3 and python2
                    fun = unichr if 'unichr' in __builtins__ else chr
                    charlist = (data[i:i+2] for i in range(0, len(data), 2))
                    return u''.join(
                        fun(struct.unpack('>h', i)[0]) for i in charlist)

            ifile.read(ord(ifile.read(1)))  # skip oo type
            self.xmin = struct.unpack('>d', ifile.read(8))[0]
            self.xmax = struct.unpack('>d', ifile.read(8))[0]
            ifile.read(1)  # skip <exists>
            self.tier_num = struct.unpack('>i', ifile.read(4))[0]
            for i in range(self.tier_num):
                tier_type = ifile.read(ord(ifile.read(1))).decode('ascii')
                name = bin2str(ifile)
                tier = Tier(0, 0, name=name, tier_type=tier_type)
                self.tiers.append(tier)
                tier.xmin = struct.unpack('>d', ifile.read(8))[0]
                tier.xmax = struct.unpack('>d', ifile.read(8))[0]
                nint = struct.unpack('>i', ifile.read(4))[0]
                for i in range(nint):
                    x1 = struct.unpack('>d', ifile.read(8))[0]
                    if tier.tier_type == 'IntervalTier':
                        x2 = struct.unpack('>d', ifile.read(8))[0]
                    text = bin2str(ifile)
                    if tier.tier_type == 'IntervalTier':
                        tier.intervals.append((x1, x2, text))
                    elif tier.tier_type == 'TextTier':
                        tier.intervals.append((x1, text))
                    else:
                        raise Exception('Tiertype does not exist.')
        else:
            def nn(ifile, pat):
                line = next(ifile).decode(codec)
                return pat.search(line).group(1)

            regfloat = re.compile('([\d.]+)\s*$', flags=re.UNICODE)
            regint = re.compile('([\d]+)\s*$', flags=re.UNICODE)
            regstr = re.compile('"(.*)"\s*$', flags=re.UNICODE)
            # Skip the Headers and empty line
            next(ifile), next(ifile), next(ifile)
            self.xmin = float(nn(ifile, regfloat))
            self.xmax = float(nn(ifile, regfloat))
            # Skip <exists>
            line = next(ifile)
            short = line.strip() == b'<exists>'
            self.tier_num = int(nn(ifile, regint))
            not short and next(ifile)
            for i in range(self.tier_num):
                not short and next(ifile)  # skip item[]: and item[\d]:
                tier_type = nn(ifile, regstr)
                name = nn(ifile, regstr)
                tier = Tier(0, 0, name=name, tier_type=tier_type)
                self.tiers.append(tier)
                tier.xmin = float(nn(ifile, regfloat))
                tier.xmax = float(nn(ifile, regfloat))
                for i in range(int(nn(ifile, regint))):
                    not short and next(ifile)  # skip intervals [\d]
                    x1 = float(nn(ifile, regfloat))
                    if tier.tier_type == 'IntervalTier':
                        x2 = float(nn(ifile, regfloat))
                        t = nn(ifile, regstr)
                        tier.intervals.append((x1, x2, t))
                    elif tier.tier_type == 'TextTier':
                        t = nn(ifile, regstr)
                        tier.intervals.append((x1, t))

    def sort_tiers(self, key=lambda x: x.name):
        """Sort the tiers given the key. Example key functions:

        Sort according to the tiername in a list:

        ``lambda x: ['name1', 'name2' ... 'namen'].index(x.name)``.

        Sort according to the number of annotations:

        ``lambda x: len(list(x.get_intervals()))``

        :param func key: A key function. Default sorts alphabetically.
        """
        self.tiers.sort(key=key)

    def add_tier(self, name, tier_type='IntervalTier', number=None):
        """Add an IntervalTier or a TextTier on the specified location.

        :param str name: Name of the tier, duplicate names is allowed.
        :param str tier_type: Type of the tier.
        :param int number: Place to insert the tier, when ``None`` the number
            is generated and the tier will be placed on the bottom.
        :returns: The created tier.
        :raises ValueError: If the number is out of bounds.
        """
        if number is None:
            number = 1 if not self.tiers else len(self.tiers)+1
        elif number < 1 or number > len(self.tiers):
            raise ValueError('Number not in [1..{}]'.format(len(self.tiers)))
        elif tier_type not in Tier.P_TIERS:
            raise ValueError('tier_type has to be in {}'.format(self.P_TIERS))
        self.tiers.insert(number-1,
                          Tier(self.xmin, self.xmax, name, tier_type))
        return self.tiers[number-1]

    def remove_tier(self, name_num):
        """Remove a tier, when multiple tiers exist with that name only the
        first is removed.

        :param name_num: Name or number of the tier to remove.
        :type name_num: int or str
        :raises IndexError: If there is no tier with that number.
        """
        if isinstance(name_num, int):
            del(self.tiers[name_num-1])
        else:
            self.tiers = [i for i in self.tiers if i.name != name_num]

    def get_tier(self, name_num):
        """Gives a tier, when multiple tiers exist with that name only the
        first is returned.

        :param name_num: Name or number of the tier to return.
        :type name_num: int or str
        :returns: The tier.
        :raises IndexError: If the tier doesn't exist.
        """
        return self.tiers[name_num - 1] if isinstance(name_num, int) else\
            [i for i in self.tiers if i.name == name_num][0]

    def change_tier_name(self, name_num, name2):
        """Changes the name of the tier, when multiple tiers exist with that
        name only the first is renamed.

        :param name_num: Name or number of the tier to rename.
        :type name_num: int or str
        :param str name2: New name of the tier.
        :raises IndexError: If the tier doesn't exist.
        """
        self.get_tier(name_num).name = name2

    def get_tiers(self):
        """Give all tiers.

        :yields: All tiers
        """
        for tier in self.tiers:
            yield tier

    def get_tier_name_num(self):
        """Give all tiers with their numbers.

        :yield: Enumerate of the form ``[(num1, tier1),  ... (numn, tiern)]``
        """
        return enumerate((s.name for s in self.tiers), 1)

    def to_file(self, filepath, codec='utf-8', mode='normal'):
        """Write the object to a file.

        :param str filepath: Path of the fil.
        :param str codec: Text encoding.
        :param string mode: Flag to for write mode, possible modes:
            'n'/'normal', 's'/'short' and 'b'/'binary'
        """
        self.tier_num = len(self.tiers)
        if mode in ['binary', 'b']:
            with open(filepath, 'wb') as f:
                def writebstr(s):
                    try:
                        bstr = s.encode('ascii')
                    except UnicodeError:
                        f.write(b'\xff\xff')
                        bstr = b''.join(struct.pack('>h', ord(c)) for c in s)
                    f.write(struct.pack('>h', len(s)))
                    f.write(bstr)

                f.write(b'ooBinaryFile\x08TextGrid')
                f.write(struct.pack('>d', self.xmin))
                f.write(struct.pack('>d', self.xmax))
                f.write(b'\x01')
                f.write(struct.pack('>i', self.tier_num))
                for tier in self.tiers:
                    f.write(chr(len(tier.tier_type)).encode('ascii'))
                    f.write(tier.tier_type.encode('ascii'))
                    writebstr(tier.name)
                    f.write(struct.pack('>d', tier.xmin))
                    f.write(struct.pack('>d', tier.xmax))
                    ints = tier.get_all_intervals()
                    f.write(struct.pack('>i', len(ints)))
                    itier = tier.tier_type == 'IntervalTier'
                    for c in ints:
                        f.write(struct.pack('>d', c[0]))
                        itier and f.write(struct.pack('>d', c[1]))
                        writebstr(c[2 if itier else 1])
        elif mode in ['normal', 'n', 'short', 's']:
            with codecs.open(filepath, 'w', codec) as f:
                short = mode[0] == 's'

                def wrt(indent, prefix, value, ff=''):
                    indent = 0 if short else indent
                    prefix = '' if short else prefix
                    if value is not None or not short:
                        s = u'{{}}{{}}{}\n'.format(ff)
                        f.write(s.format(' '*indent, prefix, value))

                f.write(u'File type = "ooTextFile"\n'
                        u'Object class = "TextGrid"\n\n')
                wrt(0, u'xmin = ', self.xmin, '{:f}')
                wrt(0, u'xmax = ', self.xmax, '{:f}')
                wrt(0, u'tiers? ', u'<exists>', '{}')
                wrt(0, u'size = ', self.tier_num, '{:d}')
                wrt(0, u'item []:', None)
                for tnum, tier in enumerate(self.tiers, 1):
                    wrt(4, u'item [{:d}]:'.format(tnum), None)
                    wrt(8, u'class = ', tier.tier_type, '"{}"')
                    wrt(8, u'name = ', tier.name, '"{}"')
                    wrt(8, u'xmin = ', tier.xmin, '{:f}')
                    wrt(8, u'xmax = ', tier.xmax, '{:f}')
                    if tier.tier_type == 'IntervalTier':
                        ints = tier.get_all_intervals()
                        wrt(8, u'intervals: size = ', len(ints), '{:d}')
                        for i, c in enumerate(ints):
                            wrt(8, 'intervals [{:d}]:'.format(i+1), None)
                            wrt(12, 'xmin = ', c[0], '{:f}')
                            wrt(12, 'xmax = ', c[1], '{:f}')
                            wrt(12, 'text = ', c[2].replace('"', '""'), '"{}"')
                    elif tier.tier_type == 'TextTier':
                        wrt(8, u'points: size = ', len(tier.intervals), '{:d}')
                        for i, c in enumerate(tier.get_intervals()):
                            wrt(8, 'points [{:d}]:'.format(i+1), None)
                            wrt(12, 'number = ', c[0], '{:f}')
                            wrt(12, 'mark = ', c[1].replace('"', '""'), '"{}"')
        else:
            raise Exception('Unknown mode')

    def to_eaf(self, skipempty=True, pointlength=0.1):
        """Convert the object to an pympi.Elan.Eaf object

        :param int pointlength: Length of respective interval from points in
                                seconds
        :param bool skipempty: Skip the empty annotations
        :returns: :class:`pympi.Elan.Eaf` object
        :raises ImportError: If the Eaf module can't be loaded.
        :raises ValueError: If the pointlength is not strictly positive.
        """
        from pympi.Elan import Eaf
        eaf_out = Eaf()
        if pointlength <= 0:
            raise ValueError('Pointlength should be strictly positive')
        for tier in self.get_tiers():
            eaf_out.add_tier(tier.name)
            for ann in tier.get_intervals(True):
                if tier.tier_type == 'TextTier':
                    ann = (ann[0], ann[0]+pointlength, ann[1])
                if ann[2].strip() or not skipempty:
                    eaf_out.add_annotation(tier.name, int(round(ann[0]*1000)),
                                           int(round(ann[1]*1000)), ann[2])
        return eaf_out

class Tier:
    """Class representing a TextGrid tier, either an Interval or TextTier

    :var str name: Name of the tier.
    :var list intervals: List of intervals where each interval is
                         (start, [end,] value).
    :var str tier_type: Type of the tier('IntervalTier' or 'TextTier').
    :var int xmin: Minimum x value.
    :var int xmax: Maximum x value.
    """
    P_TIERS = {'IntervalTier', 'TextTier'}

    def __init__(self, xmin, xmax, name=None, tier_type=None):
        """Creates a tier, if lines is ``None`` a new tier is created.

        :param str name: Name of the tier.
        :param str tier_type: Type of the tier('IntervalTier' or 'TextTier').
        :raises TierTypeException: If the tier type is unknown.
        """
        self.intervals = []
        self.name = name
        self.tier_type = tier_type
        self.xmin, self.xmax = xmin, xmax
        if tier_type not in self.P_TIERS:
            raise Exception('Tiertype does not exist.')

    def add_point(self, point, value, check=True):
        """Add a point to the TextTier

        :param int point: Time of the point.
        :param str value: Text of the point.
        :param bool check: Flag to check for overlap.
        :raises Exception: If overlap or wrong tiertype.
        """
        if self.tier_type != 'TextTier':
            raise Exception('Tiertype must be TextTier.')
        if check and any(i for i in self.intervals if i[0] == point):
                raise Exception('No overlap is allowed')
        self.intervals.append((point, value))

    def add_interval(self, begin, end, value, check=True):
        """Add an interval to the IntervalTier.

        :param float begin: Start time of the interval.
        :param float end: End time of the interval.
        :param str value: Text of the interval.
        :param bool check: Flag to check for overlap.
        :raises Exception: If overlap, begin > end or wrong tiertype.
        """
        if self.tier_type != 'IntervalTier':
            raise Exception('Tiertype must be IntervalTier')
        if check:
            if any(i for i in self.intervals if begin < i[1] and end > i[0]):
                raise Exception('No overlap is allowed')
            if begin > end:
                raise Exception('Begin must be smaller then end')
        self.intervals.append((begin, end, value))

    def remove_interval(self, time):
        """Remove an interval, if no interval is found nothing happens.

        :param int time: Time of the interval.
        :raises TierTypeException: If the tier is not a IntervalTier.
        """
        if self.tier_type != 'IntervalTier':
            raise Exception('Tiertype must be IntervalTier.')
        self.intervals = [i for i in self.intervals
                          if not(i[0] <= time and i[1] >= time)]

    def remove_point(self, time):
        """Remove a point, if no point is found nothing happens.

        :param int time: Time of the point.
        :raises TierTypeException: If the tier is not a TextTier.
        """
        if self.tier_type != 'TextTier':
            raise Exception('Tiertype must be TextTier.')
        self.intervals = [i for i in self.intervals if i[0] != time]

    def get_intervals(self, sort=False):
        """Give all the intervals or points.

        :param bool sort: Flag for yielding the intervals or points sorted.
        :yields: All the intervals
        """
        for i in sorted(self.intervals) if sort else self.intervals:
            yield i

    def clear_intervals(self):
        """Removes all the intervals in the tier"""
        self.intervals = []

    def get_all_intervals(self):
        """Returns the true list of intervals including the empty intervals."""
        ints = sorted(self.get_intervals(True))
        if self.tier_type == 'IntervalTier':
            if not ints:
                ints.append((self.xmin, self.xmax, ''))
            else:
                if ints[0][0] > self.xmin:
                    ints.insert(0, (self.xmin, ints[0][0], ''))
                if ints[-1][1] < self.xmax:
                    ints.append((ints[-1][1], self.xmax, ''))
                p = ints[-1]
                for index, i in reversed(list(enumerate(ints[:-1], 1))):
                    if p[0] - i[1] != 0:
                        ints.insert(index, (i[1], p[0], ''))
                    p = i
        return ints