'''This module contains some glue code encapsulating a "main" process.

The code here wraps the most common tasks involved in creating and, especially,
training a neural network model.
'''

import os

from . import graph
from . import util

class Experiment:
    '''This class encapsulates tasks for training and evaluating a network.

    Parameters
    ----------
    model : :class:`Network <theanets.graph.Network>` or str
        A specification for obtaining a model. If a string is given, it is
        assumed to name a file containing a pickled model; this file will be
        loaded and used. If a network instance is provided, it will be used
        as the model. If a callable (such as a subclass) is provided, it
        will be invoked using the provided keyword arguments to create a
        network instance.
    '''

    def __init__(self, network, *args, **kwargs):
        if isinstance(network, util.basestring) and os.path.isfile(network):
            self.load(network)
        elif isinstance(network, graph.Network):
            self.network = network
        else:
            assert network is not graph.Network, \
                'use a concrete theanets.Network subclass ' \
                'like theanets.{Autoencoder,Regressor,...}'
            self.network = network(*args, **kwargs)

    def train(self, *args, **kwargs):
        '''Train the network until the trainer converges.

        All arguments are passed to :func:`train
        <theanets.graph.Network.itertrain>`.

        Returns
        -------
        training : dict
            A dictionary of monitor values computed using the training dataset,
            at the conclusion of training. This dictionary will at least contain
            a 'loss' key that indicates the value of the loss function. Other
            keys may be available depending on the trainer being used.
        validation : dict
            A dictionary of monitor values computed using the validation
            dataset, at the conclusion of training.
        '''
        return self.network.train(*args, **kwargs)

    def itertrain(self, *args, **kwargs):
        '''Train the network iteratively.

        All arguments are passed to :func:`itertrain
        <theanets.graph.Network.itertrain>`.

        Yields
        ------
        training : dict
            A dictionary of monitor values computed using the training dataset,
            at the conclusion of training. This dictionary will at least contain
            a 'loss' key that indicates the value of the loss function. Other
            keys may be available depending on the trainer being used.
        validation : dict
            A dictionary of monitor values computed using the validation
            dataset, at the conclusion of training.
        '''
        return self.network.itertrain(*args, **kwargs)

    def save(self, path):
        '''Save the current network to a pickle file on disk.

        Parameters
        ----------
        path : str
            Location of the file to save the network.
        '''
        self.network.save(path)

    def load(self, path):
        '''Load a saved network from a pickle file on disk.

        This method sets the ``network`` attribute of the experiment to the
        loaded network model.

        Parameters
        ----------
        filename : str
            Load the keyword arguments and parameters of a network from a pickle
            file at the named path. If this name ends in ".gz" then the input
            will automatically be gunzipped; otherwise the input will be treated
            as a "raw" pickle.

        Returns
        -------
        network : :class:`Network <graph.Network>`
            A newly-constructed network, with topology and parameters loaded
            from the given pickle file.
        '''
        self.network = graph.Network.load(path)
        return self.network


# -*- coding: utf-8 -*-

'''Utility functions and classes.'''

import click
import datetime
import fnmatch
import inspect
import numpy as np
import theano
import theano.tensor as TT

try:
    basestring = basestring
except NameError:
    basestring = str

FLOAT = theano.config.floatX

FLOAT_CONTAINERS = (TT.scalar, TT.vector, TT.matrix, TT.tensor3, TT.tensor4)

INT_CONTAINERS = (TT.iscalar, TT.ivector, TT.imatrix, TT.itensor3, TT.itensor4)

class Error(Exception):
    pass

class ConfigurationError(Error):
    pass

class Registrar(type):
    '''A metaclass that builds a registry of its subclasses.'''

    def __init__(cls, name, bases, dct):
        if not hasattr(cls, '_registry'):
            cls._registry = {}
        else:
            cls._registry[name.lower()] = cls
            for name in getattr(cls, '__extra_registration_keys__', ()):
                cls._registry[name.lower()] = cls
        super(Registrar, cls).__init__(name, bases, dct)

    def build(cls, key, *args, **kwargs):
        return cls._registry[key.lower()](*args, **kwargs)

    def get_class(cls, key):
        return cls._registry[key.lower()]

    def is_registered(cls, key):
        return key.lower() in cls._registry

def random_matrix(rows, cols, mean=0, std=1, sparsity=0, radius=0, diagonal=0, rng=None):
    '''Create a matrix of randomly-initialized weights.

    Parameters
    ----------
    rows : int
        Number of rows of the weight matrix -- equivalently, the number of
        "input" units that the weight matrix connects.
    cols : int
        Number of columns of the weight matrix -- equivalently, the number
        of "output" units that the weight matrix connects.
    mean : float, optional
        Draw initial weight values from a normal with this mean. Defaults to 0.
    std : float, optional
        Draw initial weight values from a normal with this standard deviation.
        Defaults to 1.
    sparsity : float in (0, 1), optional
        If given, ensure that the given fraction of the weight matrix is
        set to zero. Defaults to 0, meaning all weights are nonzero.
    radius : float, optional
        If given, rescale the initial weights to have this spectral radius.
        No scaling is performed by default.
    diagonal : float, optional
        If nonzero, create a matrix containing all zeros except for this value
        along the diagonal. If nonzero, other arguments (except for rows and
        cols) will be ignored.
    rng : :class:`numpy.random.RandomState` or int, optional
        A random number generator, or an integer seed for a random number
        generator. If not provided, the random number generator will be created
        with an automatically chosen seed.

    Returns
    -------
    matrix : numpy array
        An array containing random values. These often represent the weights
        connecting each "input" unit to each "output" unit in a layer.
    '''
    if rng is None or isinstance(rng, int):
        rng = np.random.RandomState(rng)
    arr = mean + std * rng.randn(rows, cols)
    if 1 > sparsity > 0:
        k = min(rows, cols)
        mask = rng.binomial(n=1, p=1 - sparsity, size=(rows, cols)).astype(bool)
        mask[:k, :k] |= np.eye(k).astype(bool)
        arr *= mask
    if radius > 0:
        # rescale weights to have the appropriate spectral radius.
        u, s, vT = np.linalg.svd(arr, full_matrices=False)
        arr = np.dot(np.dot(u, np.diag(radius * s / abs(s[0]))), vT)
    if diagonal != 0:
        # generate a diagonal weight matrix. ignore other options.
        arr = diagonal * np.eye(max(rows, cols))[:rows, :cols]
    return arr.astype(FLOAT)

def random_vector(size, mean=0, std=1, rng=None):
    '''Create a vector of randomly-initialized values.

    Parameters
    ----------
    size : int
        Length of vecctor to create.
    mean : float, optional
        Mean value for initial vector values. Defaults to 0.
    std : float, optional
        Standard deviation for initial vector values. Defaults to 1.
    rng : :class:`numpy.random.RandomState` or int, optional
        A random number generator, or an integer seed for a random number
        generator. If not provided, the random number generator will be created
        with an automatically chosen seed.

    Returns
    -------
    vector : numpy array
        An array containing random values. This often represents the bias for a
        layer of computation units.
    '''
    if rng is None or isinstance(rng, int):
        rng = np.random.RandomState(rng)
    return (mean + std * rng.randn(size)).astype(FLOAT)

def outputs_matching(outputs, patterns):
    '''Get the outputs from a network that match a pattern.

    Parameters
    ----------
    outputs : dict or sequence of (str, theano expression)
        Output expressions to filter for matches. If this is a dictionary, its
        ``items()`` will be processed for matches.
    patterns : sequence of str
        A sequence of glob-style patterns to match against. Any parameter
        matching any pattern in this sequence will be included in the match.

    Yields
    ------
    matches : pair of str, theano expression
        Generates a sequence of (name, expression) pairs. The name is the name
        of the output that matched, and the expression is the symbolic output in
        the network graph.
    '''
    if isinstance(patterns, basestring):
        patterns = (patterns, )
    if isinstance(outputs, dict):
        outputs = outputs.items()
    for name, expr in outputs:
        for pattern in patterns:
            if fnmatch.fnmatch(name, pattern):
                yield name, expr
                break

def params_matching(layers, patterns):
    '''Get the parameters from a network that match a pattern.

    Parameters
    ----------
    layers : list of :class:`theanets.layers.Layer`
        A list of network layers to retrieve parameters from.
    patterns : sequence of str
        A sequence of glob-style patterns to match against. Any parameter
        matching any pattern in this sequence will be included in the match.

    Yields
    ------
    matches : pair of str, theano expression
        Generates a sequence of (name, expression) pairs. The name is the name
        of the parameter that matched, and the expression represents the
        parameter symbolically.
    '''
    if isinstance(patterns, basestring):
        patterns = (patterns, )
    for layer in layers:
        for param in layer.params:
            name = param.name
            for pattern in patterns:
                if fnmatch.fnmatch(name, pattern):
                    yield name, param
                    break

_detailed_callsite = False

def enable_detailed_callsite_logging():
    '''Enable detailed callsite logging.'''
    global _detailed_callsite
    _detailed_callsite = True

def log(msg, *args, **kwargs):
    '''Log a message to the console.

    Parameters
    ----------
    msg : str
        A string to display on the console. This can contain {}-style
        formatting commands; the remaining positional and keyword arguments
        will be used to fill them in.
    '''
    now = datetime.datetime.now()
    module = 'theanets'
    if _detailed_callsite:
        caller = inspect.stack()[1]
        parts = caller.filename.replace('.py', '').split('/')
        module = '{}:{}'.format(
            '.'.join(parts[parts.index('theanets')+1:]), caller.lineno)
    click.echo(' '.join((
        click.style(now.strftime('%Y%m%d'), fg='blue'),
        click.style(now.strftime('%H%M%S'), fg='cyan'),
        click.style(module, fg='green'),
        msg.format(*args, **kwargs),
    )))


# -*- coding: utf-8 -*-

r'''This module contains implementations of common regularizers.

In ``theanets`` regularizers are thought of as additional terms that get
combined with the :class:`Loss <theanets.losses.Loss>` for a model at
optimization time. Regularizer terms in the loss are usually used to "encourage"
a model to behave in a particular way---for example, the pattern and arrangement
of learned features can be changed by including a sparsity (L1-norm) regularizer
on the hidden unit activations, or units can randomly be dropped out (set to
zero) while running the model.
'''

import fnmatch
import theano.tensor as TT

from theano.sandbox.rng_mrg import MRG_RandomStreams as RandomStreams

from . import layers
from . import util

def from_kwargs(graph, **kwargs):
    '''Construct common regularizers from a set of keyword arguments.

    Keyword arguments not listed below will be passed to
    :func:`Regularizer.build` if they specify the name of a registered
    :class:`Regularizer`.

    Parameters
    ----------
    graph : :class:`theanets.graph.Network`
        A network graph to regularize.

    regularizers : dict or tuple/list of :class:`Regularizer`, optional
        If this is a list or a tuple, the contents of the list will be returned
        as the regularizers. This is to permit custom lists of regularizers to
        be passed easily.

        If this is a dict, its contents will be added to the other keyword
        arguments passed in.

    rng : int or theano RandomStreams, optional
        If an integer is provided, it will be used to seed the random number
        generators for the dropout or noise regularizers. If a theano
        RandomStreams object is provided, it will be used directly. Defaults to
        13.

    input_dropout : float, optional
        Apply dropout to input layers in the network graph, with this dropout
        rate. Defaults to 0 (no dropout).

    hidden_dropout : float, optional
        Apply dropout to hidden layers in the network graph, with this dropout
        rate. Defaults to 0 (no dropout).

    output_dropout : float, optional
        Apply dropout to the output layer in the network graph, with this
        dropout rate. Defaults to 0 (no dropout).

    input_noise : float, optional
        Apply noise to input layers in the network graph, with this standard
        deviation. Defaults to 0 (no noise).

    hidden_noise : float, optional
        Apply noise to hidden layers in the network graph, with this standard
        deviation. Defaults to 0 (no noise).

    output_noise : float, optional
        Apply noise to the output layer in the network graph, with this
        standard deviation. Defaults to 0 (no noise).

    Returns
    -------
    regs : list of :class:`Regularizer`
        A list of regularizers to apply to the given network graph.
    '''
    if 'regularizers' in kwargs:
        regs = kwargs['regularizers']
        if isinstance(regs, (tuple, list)):
            return regs
        if isinstance(regs, dict):
            kwargs.update(regs)

    regs = []

    rng = kwargs.get('rng', 13)

    def pattern(ls):
        return tuple(l.output_name for l in ls)

    inputs = pattern([l for l in graph.layers if isinstance(l, layers.Input)])
    hiddens = pattern(graph.layers[1:-1])
    outputs = pattern([graph.layers[-1]])

    # create regularizers for different types of canned dropout.
    spec = {inputs: kwargs.get('input_dropout', 0),
            hiddens: kwargs.get('hidden_dropout', 0),
            outputs: kwargs.get('output_dropout', 0)}
    spec.update(kwargs.get('dropout', {}))
    for pattern, w in spec.items():
        if w:
            regs.append(BernoulliDropout(pattern=pattern, weight=w, rng=rng))

    # create regularizers for different types of canned noise.
    spec = {inputs: kwargs.get('input_noise', 0),
            hiddens: kwargs.get('hidden_noise', 0),
            outputs: kwargs.get('output_noise', 0)}
    spec.update(kwargs.get('noise', {}))
    for pattern, w in spec.items():
        if w:
            regs.append(GaussianNoise(pattern=pattern, weight=w, rng=rng))

    # create regularizers based on other keyword arguments.
    for key, value in kwargs.items():
        if Regularizer.is_registered(key):
            if not isinstance(value, dict):
                value = dict(weight=value)
            regs.append(Regularizer.build(key, **value))

    return regs

class Regularizer(util.Registrar(str('Base'), (), {})):
    r'''A regularizer for a neural network model.

    Subclasses of this base usually either provide an implementation of the
    :func:`modify_graph` method, or an implementation of the :func:`loss` method
    (but (almost?) never both).

    Parameters
    ----------
    pattern : str
        A shell-style glob pattern describing the parameters or outputs that
        this regularizer ought to apply to.
    weight : float
        A scalar weight that indicates the "strength" of this regularizer in the
        overall loss for a model.

    Attributes
    ----------
    pattern : str
        A shell-style glob pattern describing the parameters or outputs that
        this regularizer ought to apply to.
    weight : float
        A scalar weight that indicates the "strength" of this regularizer in the
        overall loss for a model.
    '''

    def __init__(self, pattern='*', weight=0.):
        self.pattern = pattern
        self.weight = weight

    def log(self):
        '''Log some diagnostic info about this regularizer.'''
        util.log('regularizer: {0.weight} * '
                 '{0.__class__.__name__}({0.pattern})', self)

    def modify_graph(self, outputs):
        '''Modify the outputs of a particular layer in the computation graph.

        Parameters
        ----------
        outputs : dict of Theano expressions
            A map from string output names to the corresponding Theano
            expression. This dictionary contains the fully-scoped name of all
            outputs from a single layer in the computation graph.

            This map is mutable, so any changes that the regularizer makes will
            be retained when the caller regains control.

        Notes
        -----

        This method is applied during graph-construction time to change the
        behavior of one or more layer outputs. For example, the
        :class:`BernoulliDropout` class replaces matching outputs with an
        expression containing "masked" outputs, where some elements are randomly
        set to zero each time the expression is evaluated.

        Any regularizer that needs to modify the structure of the computation
        graph should implement this method.
        '''
        pass

    def loss(self, layers, outputs):
        '''Compute a scalar term to add to the loss function for a model.

        Parameters
        ----------
        layers : list of :class:`theanets.layers.Layer`
            A list of the layers in the model being regularized.
        outputs : dict of Theano expressions
            A dictionary mapping string expression names to their corresponding
            Theano expressions in the computation graph. This dictionary
            contains the fully-scoped name of every layer output in the graph.
        '''
        return 0.

class WeightL2(Regularizer):
    r'''Decay the weights in a model using an L2 norm penalty.

    Notes
    -----

    This regularizer implements the :func:`loss` method to add the following
    term to the network's loss function:

    .. math::
        \frac{1}{|\Omega|} \sum_{i \in \Omega} \|W_i\|_F^2

    where :math:`\Omega` is a set of "matching" weight parameters, and
    :math`\|\cdot\|_F` is the Frobenius norm (sum of squared elements).

    This regularizer tends to prevent the weights in a model from getting "too
    large." Large weights are often associated with overfitting in a model, so
    the regularizer tends to help prevent overfitting.

    Examples
    --------

    This regularizer can be specified at training or test time by providing the
    ``weight_l2`` or ``weight_decay`` keyword arguments:

    >>> net = theanets.Regression(...)

    To use this regularizer at training time:

    >>> net.train(..., weight_decay=0.1)

    By default all (2-dimensional) weights in the model are penalized. To
    include only some weights:

    >>> net.train(..., weight_decay=dict(weight=0.1, pattern='hid[23].w'))

    To use this regularizer when running the model forward to generate a
    prediction:

    >>> net.predict(..., weight_decay=0.1)

    The value associated with the keyword argument can be a scalar---in which
    case it provides the weight for the regularizer---or a dictionary, in which
    case it will be passed as keyword arguments directly to the constructor.

    References
    ----------

    .. [Moo95] J. Moody, S. Hanson, A. Krogh, & J. A. Hertz. (1995). "A simple
       weight decay can improve generalization." NIPS 4, 950-957.
    '''

    __extra_registration_keys__ = ['weight_l2', 'weight_decay']

    def loss(self, layers, outputs):
        matches = util.params_matching(layers, self.pattern)
        variables = [var for _, var in matches if var.ndim > 1]
        if not variables:
            return 0
        return sum((v * v).mean() for v in variables) / len(variables)

class WeightL1(Regularizer):
    r'''Decay the weights in a model using an L1 norm penalty.

    Notes
    -----

    This regularizer implements the :func:`loss` method to add the following
    term to the network's loss function:

    .. math::
        \frac{1}{|\Omega|} \sum_{i \in \Omega} \|W_i\|_1

    where :math:`\Omega` is a set of "matching" weight parameters, and the L1
    norm :math`\|\cdot\|_1` is the sum of the absolute values of the elements in
    the matrix.

    This regularizer tends to encourage the weights in a model to be zero.
    Nonzero weights are used only when they are able to reduce the other
    components of the loss (e.g., the squared reconstruction error).

    Examples
    --------

    This regularizer can be specified at training or test time by providing the
    ``weight_l1`` or ``weight_sparsity`` keyword arguments:

    >>> net = theanets.Regression(...)

    To use this regularizer at training time:

    >>> net.train(..., weight_sparsity=0.1)

    By default all (2-dimensional) weights in the model are penalized. To
    include only some weights:

    >>> net.train(..., weight_sparsity=dict(weight=0.1, pattern='hid[23].w'))

    To use this regularizer when running the model forward to generate a
    prediction:

    >>> net.predict(..., weight_sparsity=0.1)

    The value associated with the keyword argument can be a scalar---in which
    case it provides the weight for the regularizer---or a dictionary, in which
    case it will be passed as keyword arguments directly to the constructor.

    References
    ----------

    .. [Qiu11] Q. Qiu, Z. Jiang, & R. Chellappa. (ICCV 2011). "Sparse
       dictionary-based representation and recognition of action attributes."
    '''

    __extra_registration_keys__ = ['weight_l1', 'weight_sparsity']

    def loss(self, layers, outputs):
        matches = util.params_matching(layers, self.pattern)
        variables = [var for _, var in matches if var.ndim > 1]
        if not variables:
            return 0
        return sum(abs(v).mean() for v in variables) / len(variables)

class HiddenL1(Regularizer):
    r'''Penalize the activation of hidden layers under an L1 norm.

    Notes
    -----

    This regularizer implements the :func:`loss` method to add the following
    term to the network's loss function:

    .. math::
        \frac{1}{|\Omega|} \sum_{i \in \Omega} \|Z_i\|_1

    where :math:`\Omega` is a set of "matching" graph output indices, and the L1
    norm :math`\|\cdot\|_1` is the sum of the absolute values of the elements in
    the corresponding array.

    This regularizer tends to encourage the hidden unit activations in a model
    to be zero. Nonzero activations are used only when they are able to reduce
    the other components of the loss (e.g., the squared reconstruction error).

    This regularizer acts indirectly to force a model to cover the space of its
    input dataset using as few features as possible; this pressure often causes
    features to be duplicated with slight variations to "tile" the input space
    in a very different way than a non-regularized model.

    Examples
    --------

    This regularizer can be specified at training or test time by providing the
    ``hidden_l1`` or ``hidden_sparsity`` keyword arguments:

    >>> net = theanets.Regression(...)

    To use this regularizer at training time:

    >>> net.train(..., hidden_sparsity=0.1)

    By default all hidden layer outputs are penalized. To include only some
    graph outputs:

    >>> net.train(..., hidden_sparsity=dict(weight=0.1, pattern='hid3:out'))

    To use this regularizer when running the model forward to generate a
    prediction:

    >>> net.predict(..., hidden_sparsity=0.1)

    The value associated with the keyword argument can be a scalar---in which
    case it provides the weight for the regularizer---or a dictionary, in which
    case it will be passed as keyword arguments directly to the constructor.

    References
    ----------

    .. [Ng11] A. Ng. (2011). "Sparse Autoencoder." Stanford CS294A Lecture Notes
       http://web.stanford.edu/class/cs294a/sae/sparseAutoencoderNotes.pdf
    '''

    __extra_registration_keys__ = ['hidden_l1', 'hidden_sparsity']

    def loss(self, layers, outputs):
        pattern = self.pattern
        if pattern == '*':
            pattern = [l.output_name for l in layers[1:-1]]
        matches = util.outputs_matching(outputs, pattern)
        hiddens = [expr for _, expr in matches]
        if not hiddens:
            return 0
        return sum(abs(h).mean() for h in hiddens) / len(hiddens)

class RecurrentNorm(Regularizer):
    r'''Penalize successive activation norms of recurrent layers.

    Notes
    -----

    This regularizer implements the :func:`loss` method to add the following
    term to a recurrent network's loss function:

    .. math::
        \frac{1}{T|\Omega|} \sum_{i \in \Omega} \sum_{t=1}^T
          \left( \|Z_i^t\|_2^2 - \|Z_i^{t-1}\|_2^2 \right)^2

    where :math:`\Omega` is a set of "matching" graph output indices, and the
    squared L2 norm :math`\|\cdot\|_2^2` is the sum of the squares of the
    elements in the corresponding array.

    This regularizer encourages the norms of the hidden state activations in a
    recurrent layer to remain constant over time.

    Examples
    --------

    This regularizer can be specified at training or test time by providing the
    ``recurrent_norm`` keyword argument:

    >>> net = theanets.Regression(...)

    To use this regularizer at training time:

    >>> net.train(..., recurrent_norm=dict(weight=0.1, pattern='hid3:out'))

    A ``pattern`` must be provided; this pattern will match against all outputs
    in the computation graph, so some care must be taken to ensure that the
    regularizer is applied only to specific layer outputs.

    To use this regularizer when running the model forward to generate a
    prediction:

    >>> net.predict(..., recurrent_norm=0.1)

    The value associated with the keyword argument can be a scalar---in which
    case it provides the weight for the regularizer---or a dictionary, in which
    case it will be passed as keyword arguments directly to the constructor.

    References
    ----------

    .. [Kru15] D. Krueger & R. Memisevic. (ICLR 2016?) "Regularizing RNNs by
       Stabilizing Activations." http://arxiv.org/abs/1511.08400
    '''

    __extra_registration_keys__ = ['recurrent_norm']

    def loss(self, layers, outputs):
        matches = util.outputs_matching(outputs, self.pattern)
        hiddens = [expr for _, expr in matches]
        if not hiddens:
            return 0
        norms = ((e * e).sum(axis=-1) for e in hiddens)
        deltas = ((e[:, :-1] - e[:, 1:]).mean() for e in norms)
        return sum(deltas) / len(hiddens)

class RecurrentState(Regularizer):
    r'''Penalize state changes of recurrent layers.

    Notes
    -----

    This regularizer implements the :func:`loss` method to add the following
    term to a recurrent network's loss function:

    .. math::
        \frac{1}{T|\Omega|} \sum_{i \in \Omega} \sum_{t=1}^T
          \| Z_i^t - Z_i^{t-1} \|_2^2

    where :math:`\Omega` is a set of "matching" graph output indices, and the
    squared L2 norm :math`\|\cdot\|_2^2` is the sum of the squares of the
    elements in the corresponding array.

    This regularizer tends to encourage the hidden state activations in a
    recurrent layer to remain constant over time. Deviations from a constant
    state thus require "evidence" from the loss.

    Examples
    --------

    This regularizer can be specified at training or test time by providing the
    ``recurrent_state`` keyword argument:

    >>> net = theanets.Regression(...)

    To use this regularizer at training time:

    >>> net.train(..., recurrent_state=dict(weight=0.1, pattern='hid3:out'))

    A ``pattern`` must be provided; this pattern will match against all outputs
    in the computation graph, so some care must be taken to ensure that the
    regularizer is applied only to specific layer outputs.

    To use this regularizer when running the model forward to generate a
    prediction:

    >>> net.predict(..., recurrent_state=dict(weight=0.1, pattern='hid3:out'))

    The value associated with the keyword argument can be a scalar---in which
    case it provides the weight for the regularizer---or a dictionary, in which
    case it will be passed as keyword arguments directly to the constructor.

    References
    ----------

    '''

    __extra_registration_keys__ = ['recurrent_state']

    def loss(self, layers, outputs):
        matches = util.outputs_matching(outputs, self.pattern)
        hiddens = [expr for _, expr in matches]
        if not hiddens:
            return 0
        deltas = (e[:, :-1] - e[:, 1:] for e in hiddens)
        return sum((d * d).mean() for d in deltas) / len(hiddens)

class Contractive(Regularizer):
    r'''Penalize the derivative of hidden layers with respect to their inputs.

    Parameters
    ----------
    wrt : str, optional
        A glob-style pattern that specifies the inputs with respect to which the
        derivative should be computed. Defaults to ``'*'``, which matches all
        inputs.

    Notes
    -----

    This regularizer implements the :func:`loss` method to add the following
    term to the network's loss function:

    .. math::
        \frac{1}{|\Omega|} \sum_{i \in \Omega} \|\frac{\partial Z_i}{x}\|_F^2

    where :math:`\Omega` is a set of "matching" graph output indices,
    :math:`Z_i` is the output of network graph :math:`i`, :math:`x` is the input
    to the network graph, and :math`\|\cdot\|_F` is the Frobenius norm (sum of
    the squared elements in the array).

    This regularizer attempts to make the derivative of the hidden representatin
    flat with respect to the input. In theory, this encourages the network to
    learn features that are insensitive to small changes in the input (that is,
    they are mostly perpindicular to the input manifold).

    Like the :class:`HiddenL1` regularizer, this acts indirectly to force a
    model to cover the space of its input dataset using as few features as
    possible; this pressure often causes features to be duplicated with slight
    variations to "tile" the input space in a very different way than a
    non-regularized model.

    Examples
    --------

    This regularizer can be specified at training or test time by providing the
    ``hidden_l1`` or ``hidden_sparsity`` keyword arguments:

    >>> net = theanets.Regression(...)

    To use this regularizer at training time:

    >>> net.train(..., contractive=0.1)

    By default all hidden layer outputs are included. To include only some graph
    outputs:

    >>> net.train(..., contractive=dict(weight=0.1, pattern='hid3:out', wrt='in'))

    To use this regularizer when running the model forward to generate a
    prediction:

    >>> net.predict(..., contractive=0.1)

    The value associated with the keyword argument can be a scalar---in which
    case it provides the weight for the regularizer---or a dictionary, in which
    case it will be passed as keyword arguments directly to the constructor.

    References
    ----------

    .. [Rif11] S. Rifai, P. Vincent, X. Muller, X. Glorot, & Y. Bengio. (ICML
       2011). "Contractive auto-encoders: Explicit invariance during feature
       extraction."

       http://machinelearning.wustl.edu/mlpapers/paper_files/ICML2011Rifai_455.pdf
    '''

    def __init__(self, pattern=None, weight=0., wrt='*'):
        self.wrt = wrt
        super(Contractive, self).__init__(pattern=pattern, weight=weight)

    def log(self):
        '''Log some diagnostic info about this regularizer.'''
        util.log('regularizer: {0.weight} * '
                 '{0.__class__.__name__}({0.pattern} wrt '
                 '{0.wrt})', self)

    def loss(self, layer_list, outputs):
        pattern = self.pattern
        if pattern == '*':
            pattern = [l.output_name for l in layer_list[1:-1]]
        targets = [expr for _, expr in util.outputs_matching(outputs, pattern)]
        if not targets:
            return 0
        wrt = [l.input for l in layer_list
               if isinstance(l, layers.Input) and
               fnmatch.fnmatch(l.input.name, self.wrt)]
        total = sum(TT.sqr(TT.grad(h.mean(), wrt)).mean() for h in targets)
        return total / len(targets)

class GaussianNoise(Regularizer):
    r'''Add isotropic Gaussian noise to one or more graph outputs.

    Parameters
    ----------
    rng : Theano random number generator, optional
        A Theano random number generator to use for creating noise and dropout
        values. If not provided, a new generator will be produced for this
        layer.

    Notes
    -----

    This regularizer implements the :func:`modify_graph` method to "inject"
    noise into the loss function of a network. Suppose we were optimizing a
    linear :class:`regression <theanets.feedforward.Regression>` model with one
    hidden layer under a mean squared error. The loss for an input/output pair
    :math:`(x, y)` would be:

    .. math::
        \mathcal{L} = \| V(Wx + b) + c - y \|_2^2

    where :math:`W (V)` and :math:`b (c)` are the weights and bias parameters of
    the first (and second) layers in the model.

    If we regularized this model with Gaussian noise, the loss for this pair
    would be:

    .. math::
        \mathcal{L} = \| V(W(x+\epsilon) + b) + c - y \|_2^2

    where :math:`\epsilon \sim \mathcal{N}(0, \sigma^2)` is isotropic random
    noise with standard deviation :math:`\sigma`.

    This regularizer encourages the model to develop parameter settings that are
    robust to noisy inputs. There are some parallels to the :class:`Contractive`
    regularizer, in that both models are thought to develop internal
    representations that are orthogonal to the manifold of the input data, so
    that noisy inputs are "pushed back" onto the manifold by the network.

    Examples
    --------

    This regularizer can be specified at training or test time by providing the
    ``noise`` or ``input_noise`` or ``hidden_noise`` keyword arguments:

    >>> net = theanets.Regression(...)

    To apply this regularizer at training time to network inputs:

    >>> net.train(..., input_noise=0.1)

    And to apply the regularizer to hidden states of the network:

    >>> net.train(..., hidden_noise=0.1)

    To target specific network outputs, a pattern can be given manually:

    >>> net.train(..., noise={'hid[23]:out': 0.1, 'in:out': 0.01})

    To use this regularizer when running the model forward to generate a
    prediction:

    >>> net.predict(..., input_noise=0.1)

    The value associated with the ``input_noise`` or ``hidden_noise`` keyword
    arguments should be a scalar giving the standard deviation of the noise to
    apply. The value of the ``noise`` keyword argument should be a dictionary,
    whose keys provide glob-style output name patterns, and the corresponding
    values are the noise level.

    References
    ----------

    .. [Vin10] P. Vincent, H. Larochelle, Y. Bengio, & P.-A. Manzagol. (ICML
       2008). "Extracting and composing robust features with denoising
       autoencoders."

       http://oucsace.cs.ohiou.edu/~razvan/courses/dl6900/papers/vincent08.pdf
    '''

    def __init__(self, pattern='*:out', weight=0., rng=13):
        super(GaussianNoise, self).__init__(pattern=pattern, weight=weight)
        self.rng = RandomStreams(rng) if isinstance(rng, int) else rng

    def log(self):
        '''Log some diagnostic info about this regularizer.'''
        util.log('regularizer: {0.weight} * '
                 '{0.__class__.__name__}({0.pattern})', self)

    def modify_graph(self, outputs):
        for name, expr in list(util.outputs_matching(outputs, self.pattern)):
            outputs[name + '-prenoise'] = expr
            outputs[name] = expr + self.rng.normal(
                size=expr.shape, std=self.weight, dtype=util.FLOAT)

class BernoulliDropout(Regularizer):
    r'''Randomly set activations of a layer output to zero.

    Parameters
    ----------

    rng : Theano random number generator, optional
        A Theano random number generator to use for creating noise and dropout
        values. If not provided, a new generator will be produced for this
        layer.

    Notes
    -----

    This regularizer implements the :func:`modify_graph` method to "inject"
    multiplicative Bernoulli noise into the loss function of a network.

    Suppose we were optimizing a linear :class:`regression
    <theanets.feedforward.Regression>` model with one hidden layer under a mean
    squared error. The loss for an input/output pair :math:`(x, y)` would be:

    .. math::
        \mathcal{L} = \| V(Wx + b) + c - y \|_2^2

    where :math:`W (V)` and :math:`b (c)` are the weights and bias parameters of
    the first (and second) layers in the model.

    If we regularized this model's input with multiplicative Bernoulli "noise,"
    the loss for this pair would be:

    .. math::
        \mathcal{L} = \| V(W(x\cdot\rho) + b) + c - y \|_2^2

    where :math:`\rho \sim \mathcal{B}(p)` is a vector of independent Bernoulli
    samples with probability :math:`p`.

    This regularizer encourages the model to develop parameter settings such
    that internal features are independent. Dropout is widely used as a powerful
    regularizer in many types of neural network models.

    Examples
    --------

    This regularizer can be specified at training or test time by providing the
    ``dropout`` or ``input_dropout`` or ``hidden_dropout`` keyword arguments:

    >>> net = theanets.Regression(...)

    To apply this regularizer at training time to network inputs:

    >>> net.train(..., input_dropout=0.1)

    And to apply the regularizer to hidden states of the network:

    >>> net.train(..., hidden_dropout=0.1)

    To target specific network outputs, a pattern can be given manually:

    >>> net.train(..., dropout={'hid[23]:out': 0.1, 'in:out': 0.01})

    To use this regularizer when running the model forward to generate a
    prediction:

    >>> net.predict(..., input_dropout=0.1)

    The value associated with the ``input_dropout`` or ``hidden_dropout``
    keyword arguments should be a scalar giving the probability of the dropout
    to apply. The value of the ``dropout`` keyword argument should be a
    dictionary, whose keys provide glob-style output name patterns, and the
    corresponding values are the dropout level.

    References
    ----------

    .. [Hin12] G. E. Hinton, N. Srivastava, A. Krizhevsky, I. Sutskever, & R. R.
       Salakhutdinov. (2012). "Improving neural networks by preventing
       co-adaptation of feature detectors." http://arxiv.org/pdf/1207.0580.pdf
    '''

    def __init__(self, pattern='*:out', weight=0., rng=13):
        super(BernoulliDropout, self).__init__(pattern=pattern, weight=weight)
        self.rng = RandomStreams(rng) if isinstance(rng, int) else rng

    def log(self):
        '''Log some diagnostic info about this regularizer.'''
        util.log('regularizer: {0.__class__.__name__}({0.pattern}) ~ '
                 'B({0.weight})', self)

    def modify_graph(self, outputs):
        for name, expr in list(util.outputs_matching(outputs, self.pattern)):
            noise = self.rng.binomial(
                size=expr.shape, n=1, p=1-self.weight, dtype=util.FLOAT)
            outputs[name + '-predrop'] = expr
            outputs[name] = expr * noise / (1-self.weight)


# -*- coding: utf-8 -*-

'''These loss functions are available for neural network models.'''

import numpy as np
import theano.tensor as TT

from . import util

class Loss(util.Registrar(str('Base'), (), {})):
    r'''A loss function base class.

    Parameters
    ----------
    target : int or Theano variable
        If this is an integer, it specifies the number of dimensions required to
        store the target values for computing the loss. If it is a Theano
        variable, this variable will be used directly to access target values.
    weight : float, optional
        The importance of this loss for the model being trained. Defaults to 1.
    weighted : bool, optional
        If True, a floating-point array of weights with the same dimensions as
        ``target`` will be required to compute the "weighted" loss. Defaults
        to False.
    output_name : str, optional
        Name of the network output to tap for computing the loss. Defaults to
        'out:out', the name of the default output of the last layer in a linear
        network.

    Attributes
    ----------
    weight : float
        The importance of this loss for the model being trained.
    output_name : str
        Name of the network output to tap for computing the loss.
    '''

    def __init__(self, target, weight=1., weighted=False, output_name='out'):
        self.weight = weight

        self._target = (util.FLOAT_CONTAINERS[target]('target')
                        if isinstance(target, int) else target)

        self._weights = None
        if weighted:
            self._weights = util.FLOAT_CONTAINERS[self._target.ndim]('weights')

        self.output_name = output_name
        if ':' not in self.output_name:
            self.output_name += ':out'

    @property
    def variables(self):
        '''A list of Theano variables used in this loss.'''
        result = [self._target]
        if self._weights is not None:
            result.append(self._weights)
        return result

    def log(self):
        '''Log some diagnostic info about this loss.'''
        util.log('using loss: {0.weight} * {0.__class__.__name__} '
                 '(output {0.output_name})', self)

    def __call__(self, outputs):
        '''Construct the computation graph for this loss function.

        Parameters
        ----------
        outputs : dict of Theano expressions
            A dictionary mapping network output names to Theano expressions
            representing the outputs of a computation graph.

        Returns
        -------
        loss : Theano expression
            The values of the loss given the network output.
        '''
        raise NotImplementedError

class MeanSquaredError(Loss):
    r'''Mean-squared-error (MSE) loss function.

    Notes
    -----

    The mean squared error (MSE) loss computes the mean of the squared
    difference between the output of a computation graph
    :math:`x = (x_1, \dots, x_d)` and its expected target value
    :math:`t = (t_1, \dots, t_d)`. Mathematically,

    .. math::
       \begin{eqnarray*}
       \mathcal{L}(x, t) &=& \frac{1}{d} \|x - t\|_2^2 \\
                         &=& \frac{1}{d} \sum_{i=1}^d (x_i - t_i)^2
       \end{eqnarray*}

    Whereas some MSE computations return the sum over dimensions, the MSE here
    is computed as an average over the dimensionality of the data.

    For cases where :math:`x` and :math:`t` are matrices, the MSE computes the
    average over corresponding rows in :math:`x` and :math:`t`:

    .. math::
       \mathcal{L}(X, T) = \frac{1}{dm} \sum_{j=1}^m \sum_{i=1}^d (x_{ji} - t_{ji})^2
    '''

    __extra_registration_keys__ = ['MSE']

    def __call__(self, outputs):
        '''Construct the computation graph for this loss function.

        Parameters
        ----------
        outputs : dict of Theano expressions
            A dictionary mapping network output names to Theano expressions
            representing the outputs of a computation graph.

        Returns
        -------
        loss : Theano expression
            The values of the loss given the network output.
        '''
        err = outputs[self.output_name] - self._target
        if self._weights is not None:
            return (self._weights * err * err).sum() / self._weights.sum()
        return (err * err).mean()

class MeanAbsoluteError(Loss):
    r'''Mean-absolute-error (MAE) loss function.

    Notes
    -----

    The mean absolute error (MAE) loss computes the mean difference between the
    output of a computation graph :math:`x = (x_1, \dots, x_d)` and its expected
    target value :math:`t = (t_1, \dots, t_d)`. Mathematically,

    .. math::
       \begin{eqnarray*}
       \mathcal{L}(x, t) &=& \frac{1}{d} \|x - t\|_1 \\
                         &=& \frac{1}{d} \sum_{i=1}^d |x_i - t_i|
       \end{eqnarray*}

    Whereas some MAE computations return the sum over dimensions, the MAE here
    is computed as an average over the dimensionality of the data.

    For cases where :math:`x` and :math:`t` are matrices, the MAE computes the
    average over corresponding rows in :math:`x` and :math:`t`.
    '''

    __extra_registration_keys__ = ['MAE']

    def __call__(self, outputs):
        '''Construct the computation graph for this loss function.

        Parameters
        ----------
        outputs : dict of Theano expressions
            A dictionary mapping network output names to Theano expressions
            representing the outputs of a computation graph.

        Returns
        -------
        loss : Theano expression
            The values of the loss given the network output.
        '''
        err = outputs[self.output_name] - self._target
        if self._weights is not None:
            return abs(self._weights * err).sum() / self._weights.sum()
        return abs(err).mean()

class GaussianLogLikelihood(Loss):
    r'''Gaussian Log Likelihood (GLL) loss function.

    Parameters
    ----------
    mean_name : str
        Name of the network graph output to use for the mean of the Gaussian
        distribution.
    covar_name : str
        Name of the network graph output to use for the diagonal covariance of
        the Gaussian distribution.

    Notes
    -----

    This loss computes the negative log-likelihood of the observed target data
    :math:`y` under a Gaussian distribution, where the neural network computes
    the mean :math:`\mu` and the diagonal of the covariance :math:`\Sigma` as a
    function of its input :math:`x`. The loss is given by:

    .. math::
       \mathcal{L}(x, y) = -\log p(y) = -\log p\left(y|\mu(x),\Sigma(x)\right)

    where

    .. math::
       p(y) = p(y|\mu,\Sigma) = \frac{1}{(2\pi)^{n/2}|\Sigma|^{1/2}}
          \exp\left\{-\frac{1}{2}(y-\mu)^\top\Sigma^{-1}(y-\mu) \right\}

    is the Gaussian density function.

    The log density :math:`\log p(y)` can be parameterized more conveniently
    [Gu08]_ as:

    .. math::
       \log p(y|\nu,\Lambda) = a + \eta^\top y - \frac{1}{2} y^\top \Lambda y

    where :math:`\Lambda = \Sigma^{-1}` is the precision,
    :math:`\eta = \Lambda\mu` is the covariance-skewed mean, and
    :math:`a=-\frac{1}{2}\left(n\log 2\pi-\log|\Lambda|+\eta^\top\Lambda\eta\right)`
    contains all constant terms. (These terms are all computed as a function of
    the input, :math:`x`.)

    This implementation of the Gaussian log-likelihood loss approximates
    :math:`\Sigma` using only its diagonal. This makes the precision easy to
    compute because

    .. math::
       \Sigma^{-1} = \Lambda =
          \mbox{diag}(\frac{1}{\sigma_1}, \dots, \frac{1}{\sigma_n})

    is just the matrix containing the multiplicative inverse of the diagonal
    covariance values. Similarly, the log-determinant of the precision is just
    the sum of the logs of the diagonal terms:

    .. math::
       \log|\Lambda|=\sum_{i=1}^n\log\lambda_i=-\sum_{i=1}^n\log\sigma_i.

    The log-likelihood is computed separately for each input-output pair in a
    batch, and the overall likelihood is the mean of these individual values.

    Weighted targets unfortunately do not work with this loss at the moment.

    References
    ----------

    .. [Gu08] Multivariate Gaussian Distribution.
        https://www.cs.cmu.edu/~epxing/Class/10701-08s/recitation/gaussian.pdf
    '''

    __extra_registration_keys__ = ['GLL']

    def __init__(self, mean_name='mean', covar_name='covar', covar_eps=1e-3, **kwargs):
        self.mean_name = mean_name
        if ':' not in self.mean_name:
            self.mean_name += ':out'
        self.covar_name = covar_name
        if ':' not in self.covar_name:
            self.covar_name += ':out'
        self.covar_eps = covar_eps
        super(GaussianLogLikelihood, self).__init__(**kwargs)

    def log(self):
        '''Log some diagnostic info about this loss.'''
        util.log('using loss: {0.weight} * {0.__class__.__name__} '
                 '(mean {0.mean_name}, covar {0.covar_name})', self)

    def __call__(self, outputs):
        '''Construct the computation graph for this loss function.

        Parameters
        ----------
        outputs : dict of Theano expressions
            A dictionary mapping network output names to Theano expressions
            representing the outputs of a computation graph.

        Returns
        -------
        loss : Theano expression
            The values of the loss given the network output.
        '''
        # this code is going to look weird to people who are used to seeing
        # implementations of the gaussian likelihood function. our mean, covar,
        # and self._target arrays are all of shape (batch-size, dims). each of
        # these arrays codes an independent input/output pair for the loss, but
        # they're stacked together in matrices for computational efficiency.
        #
        # what's worse, the covariance is encoded as a vector of the diagonal
        # elements, again one per input/output pair in the batch.
        #
        # the upshot of this is that many operations written traditionally as
        # three dot products (vector-matrix-vector, e.g., x^T \Lambda x) are
        # here written as three elementwise array products (x * prec * x),
        # followed by a sum across the last dimension. this has the added
        # benefit that it will be way faster than dot products, but it looks
        # strange in the code below.
        mean = outputs[self.mean_name]
        covar = outputs[self.covar_name]
        prec = 1 / (abs(covar) + self.covar_eps)  # prevent nans!
        eta = mean * prec
        logpi = TT.cast(mean.shape[-1] * np.log(2 * np.pi), 'float32')
        logdet = TT.log(prec.sum(axis=-1))
        const = logpi - logdet + (eta * prec * eta).sum(axis=-1)
        squared = (self._target * prec * self._target).sum(axis=-1)
        nll = 0.5 * (const + squared) - (eta * self._target).sum(axis=-1)
        return nll.mean()

class MaximumMeanDiscrepancy(Loss):
    r'''Maximum Mean Discrepancy (MMD) loss function.

    Parameters
    ----------
    kernel : callable or numeric, optional
        A kernel function to call for computing pairwise kernel values. If this
        is a callable, it should take two Theano arrays as arguments and return
        a Theano array. If it is a numeric value, the kernel will be a Gaussian
        with the given value as the bandwidth parameter. Defaults to 1.

    Notes
    -----

    This loss computes the differential between a predicted distribution
    (generated by a network) and an observed distribution (of data within a
    mini-batch). The loss is given by:

    .. math::
       \mathcal{L}(x, y) = \| \sum_{j=1}^N \phi(y_j) - \sum_{i=1}^N \phi(x_i) \|_2^2

    This can be expanded to

    .. math::
       \mathcal{L}(x, y) = \sum_{j=1}^N \sum_{j'=1}^N \phi(y_j)^\top \phi(y_{j'})
          - 2 \sum_{j=1}^N \sum_{i=1}^N \phi(y_j)^\top \phi(x_i)
          + \sum_{i=1}^N \sum_{i'=1}^N \phi(x_i)^\top \phi(x_{i'})

    and then the kernel trick can be applied,

    .. math::
       \mathcal{L}(x, y) = \sum_{j=1}^N \sum_{j'=1}^N k(y_j, y_{j'})
          - 2 \sum_{j=1}^N \sum_{i=1}^N k(y_j, x_i)
          + \sum_{i=1}^N \sum_{i'=1}^N k(x_i, x_{i'})

    By default the loss here uses the Gaussian kernel

    .. math::
       k(x, x') = \exp(-(x-x')^2/\sigma)

    where :math:`\sigma` is a scalar bandwidth parameter. However, other kernels
    can be provided when constructing the loss.

    References
    ----------

    .. [Gre07] A. Gretton, K. M. Borgwardt, M. Rasch, B. Scholkopf, & A. J.
       Smola (NIPS 2007) "A Kernel Method for the Two-Sample-Problem."
       http://papers.nips.cc/paper/3110-a-kernel-method-for-the-two-sample-problem.pdf

    .. [Li15] Y. Li, K. Swersky, & R. Zemel (ICML 2015) "Generative Moment
       Matching Networks." http://jmlr.org/proceedings/papers/v37/li15.pdf
    '''

    __extra_registration_keys__ = ['MMD']

    @staticmethod
    def gaussian(bw):
        def kernel(x, y):
            # this dimshuffle lets us compute squared euclidean distance with a
            # broadcasted subtraction, a square, and a sum.
            r = x.dimshuffle(0, 'x', *tuple(range(1, x.ndim)))
            return TT.exp(TT.sqr(r - y).sum(axis=-1) / -bw)
        return kernel

    def __init__(self, kernel=1, **kwargs):
        super(MaximumMeanDiscrepancy, self).__init__(**kwargs)
        if isinstance(kernel, (int, float)):
            kernel = MaximumMeanDiscrepancy.gaussian(kernel)
        self.kernel = kernel

    def __call__(self, outputs):
        '''Construct the computation graph for this loss function.

        Parameters
        ----------
        outputs : dict of Theano expressions
            A dictionary mapping network output names to Theano expressions
            representing the outputs of a computation graph.

        Returns
        -------
        loss : Theano expression
            The values of the loss given the network output.
        '''
        output = outputs[self.output_name]
        xx = self.kernel(self._target, self._target)
        xy = self.kernel(self._target, output)
        yy = self.kernel(output, output)
        return xx.mean() - 2 * xy.mean() + yy.mean()

class KullbackLeiblerDivergence(Loss):
    r'''The KL divergence loss is computed over probability distributions.

    Notes
    -----

    The KL divergence loss is intended to optimize models that generate
    probability distributions. If the outputs :math:`x_i` of a model represent a
    normalized probability distribution (over the output variables), and the
    targets :math:`t_i` represent a normalized target distribution (over the
    output variables), then the KL divergence is given by:

    .. math::
       \mathcal{L}(x, t) = \frac{1}{d} \sum_{i=1}^d t_i \log \frac{t_i}{x_i}

    Here the KL divergence is computed as a mean value over the output variables
    in the model.
    '''

    __extra_registration_keys__ = ['KL', 'KLD']

    def __call__(self, outputs):
        '''Construct the computation graph for this loss function.

        Parameters
        ----------
        outputs : dict of Theano expressions
            A dictionary mapping network output names to Theano expressions
            representing the outputs of a computation graph.

        Returns
        -------
        loss : Theano expression
            The values of the loss given the network output.
        '''
        output = outputs[self.output_name]
        eps = 1e-8
        t = TT.clip(self._target, eps, 1 - eps)
        kl = t * TT.log(t / TT.clip(output, eps, 1 - eps))
        if self._weights is not None:
            return abs(self._weights * kl).sum() / self._weights.sum()
        return abs(kl).mean()

class CrossEntropy(Loss):
    r'''Cross-entropy (XE) loss function for classifiers.

    Parameters
    ----------
    target : int
        Number of dimensions required to store the target values for computing
        the loss.
    weight : float, optional
        The importance of this loss for the model being trained. Defaults to 1.
    weighted : bool, optional
        If True, a floating-point array of weights with the same dimensions as
        ``out_dim`` will be required to compute the "weighted" loss. Defaults
        to False.
    output_name : str, optional
        Name of the network output to tap for computing the loss. Defaults to
        'out:out', the name of the default output of the last layer in a linear
        network.

    Attributes
    ----------
    weight : float, optional
        The importance of this loss for the model being trained.
    output_name : str
        Name of the network output to tap for computing the loss.

    Notes
    -----

    The cross-entropy between a "true" distribution over discrete classes
    :math:`p(t)` and a "model" distribution over predicted classes :math:`q(x)`
    is the expected number of bits needed to store the model distribution, under
    the expectation of the true distribution. Mathematically, this loss
    computes:

    .. math::
       \mathcal{L}(x, t) = - \sum_{k=1}^K p(t=k) \log q(x=k)

    The loss value is similar to the KL divergence between :math:`p` and
    :math:`q`, but it is specifically aimed at classification models. When using
    this loss, targets are assumed to be integers in the half-open interval
    :math:`[0, k)`; internally, the loss is computed by first taking the log of
    the model distribution and then summing up only the entries in the resulting
    array corresponding to the true class.
    '''

    __extra_registration_keys__ = ['XE']

    def __init__(self, target, weight=1., weighted=False, output_name='out'):
        super(CrossEntropy, self).__init__(
            target, weight=weight, weighted=weighted, output_name=output_name)
        self._target = util.INT_CONTAINERS[target]('target')

    def __call__(self, outputs):
        '''Construct the computation graph for this loss function.

        Parameters
        ----------
        outputs : dict of Theano expressions
            A dictionary mapping network output names to Theano expressions
            representing the outputs of a computation graph.

        Returns
        -------
        loss : Theano expression
            The values of the loss given the network output.
        '''
        output = outputs[self.output_name]
        k = output.shape[-1]
        n = TT.prod(output.shape) // k
        prob = output.reshape((n, k))[TT.arange(n), self._target.reshape((n, ))]
        nlp = -TT.log(TT.clip(prob, 1e-8, 1))
        if self._weights is not None:
            return (self._weights.reshape((n, )) * nlp).sum() / self._weights.sum()
        return nlp.mean()

    def accuracy(self, outputs):
        '''Build a Theano expression for computing the accuracy of graph output.

        Parameters
        ----------
        outputs : dict of Theano expressions
            A dictionary mapping network output names to Theano expressions
            representing the outputs of a computation graph.

        Returns
        -------
        acc : Theano expression
            A Theano expression representing the accuracy of the output compared
            to the target data.
        '''
        output = outputs[self.output_name]
        predict = TT.argmax(output, axis=-1)
        correct = TT.eq(predict, self._target)
        acc = correct.mean()
        if self._weights is not None:
            acc = (self._weights * correct).sum() / self._weights.sum()
        return acc

class Hinge(CrossEntropy):
    r'''Hinge loss function for classifiers.

    Notes
    -----

    The hinge loss as implemented here computes the maximum difference between
    the prediction :math:`q(x=k)` for a class :math:`k` and the prediction
    :math:`q(x=t)` for the correct class :math:`t`:

    .. math::
       \mathcal{L}(x, t) = \max(0, \max_k q(x=k) - q(x=t))

    This loss is zero whenever the prediction for the correct class is the
    largest over classes, and increases linearly when the prediction for an
    incorrect class is the largest.
    '''

    __extra_registration_keys__ = []

    def __call__(self, outputs):
        '''Construct the computation graph for this loss function.

        Parameters
        ----------
        outputs : dict of Theano expressions
            A dictionary mapping network output names to Theano expressions
            representing the outputs of a computation graph.

        Returns
        -------
        loss : Theano expression
            The values of the loss given the network output.
        '''
        output = outputs[self.output_name]
        k = output.shape[-1]
        n = TT.prod(output.shape) // k
        output = output.reshape((n, k))
        true = output[TT.arange(n), self._target.reshape((n, ))]
        err = TT.maximum(0, (output - true[:, None]).max(axis=-1))
        if self._weights is not None:
            return (self._weights.reshape((n, )) * err).sum() / self._weights.sum()
        return err.mean()