Add an activation layer to the model.

        Parameters
        ----------
        name: str
            The name of this layer
        non_linearity: str
            The non_linearity (activation) function of this layer.
            It can be one of the following:

                - 'RELU': Rectified Linear Unit (ReLU) function.
                - 'SIGMOID': sigmoid function.
                - 'TANH': tanh function.
                - 'SCALED_TANH': scaled tanh function, defined as:

                  `f(x) = alpha * tanh(beta * x)`

                  where alpha and beta are constant scalars.

                - 'SOFTPLUS': softplus function.
                - 'SOFTSIGN': softsign function.
                - 'SIGMOID_HARD': hard sigmoid function, defined as:

                  `f(x) = min(max(alpha * x + beta, -1), 1)`

                  where alpha and beta are constant scalars.
                - 'LEAKYRELU': leaky relu function, defined as:

                  `f(x) = (x >= 0) * x + (x < 0) * alpha * x`

                  where alpha is a constant scalar.
                - 'PRELU': Parametric ReLU function, defined as:

                  `f(x) = (x >= 0) * x + (x < 0) * alpha * x`

                  where alpha is a multi-dimensional array of same size as x.
                - 'ELU': Exponential linear unit function, defined as:

                  `f(x) = (x >= 0) * x + (x < 0) * (alpha * exp(x) - 1)`

                  where alpha is a constant scalar.

                - 'PARAMETRICSOFTPLUS': Parametric softplus function, defined as:

                  `f(x) = alpha * log(1 + exp(beta * x))`

                  where alpha and beta are two multi-dimensional arrays of same size as x.
                - 'THRESHOLDEDRELU': Thresholded ReLU function, defined as:

                  `f(x) = (x >= alpha) * x`

                  where alpha is a constant scalar.
                - 'LINEAR': linear function.

                   `f(x) = alpha * x + beta`

        input_name: str
            The input blob name of this layer.
        output_name: str
            The output blob name of this layer.
        params: [float] | [numpy.array]
            Parameters for the activation, depending on non_linearity. Kindly refer to NeuralNetwork.proto for details.

                - When non_linearity is one of ['RELU', 'SIGMOID', 'TANH', 'SCALED_TANH', 'SOFTPLUS', 'SOFTSIGN'], params is ignored.
                - When non_linearity is one of ['SCALED_TANH', 'SIGMOID_HARD', 'LINEAR'], param is a list of 2 floats
                  [alpha, beta].
                - When non_linearity is one of ['LEAKYRELU', 'ELU', 'THRESHOLDEDRELU'], param is a list of 1 float
                  [alpha].
                - When non_linearity is 'PRELU', param is a list of 1 numpy array [alpha]. The shape of
                  alpha is (C,), where C is either the number of input channels or
                  1. When C = 1, same alpha is applied to all channels.
                - When non_linearity is 'PARAMETRICSOFTPLUS', param is a list of 2 numpy arrays [alpha,
                  beta]. The shape of alpha and beta is (C, ), where C is either
                  the number of input channels or 1. When C = 1, same alpha and
                  beta are applied to all channels.

        See Also
        --------
        add_convolution, add_softmax

def add_activation(self, name, non_linearity, input_name, output_name,
        params=None):
        """
        Add an activation layer to the model.

        Parameters
        ----------
        name: str
            The name of this layer
        non_linearity: str
            The non_linearity (activation) function of this layer.
            It can be one of the following:

                - 'RELU': Rectified Linear Unit (ReLU) function.
                - 'SIGMOID': sigmoid function.
                - 'TANH': tanh function.
                - 'SCALED_TANH': scaled tanh function, defined as:

                  `f(x) = alpha * tanh(beta * x)`

                  where alpha and beta are constant scalars.

                - 'SOFTPLUS': softplus function.
                - 'SOFTSIGN': softsign function.
                - 'SIGMOID_HARD': hard sigmoid function, defined as:

                  `f(x) = min(max(alpha * x + beta, -1), 1)`

                  where alpha and beta are constant scalars.
                - 'LEAKYRELU': leaky relu function, defined as:

                  `f(x) = (x >= 0) * x + (x < 0) * alpha * x`

                  where alpha is a constant scalar.
                - 'PRELU': Parametric ReLU function, defined as:

                  `f(x) = (x >= 0) * x + (x < 0) * alpha * x`

                  where alpha is a multi-dimensional array of same size as x.
                - 'ELU': Exponential linear unit function, defined as:

                  `f(x) = (x >= 0) * x + (x < 0) * (alpha * exp(x) - 1)`

                  where alpha is a constant scalar.

                - 'PARAMETRICSOFTPLUS': Parametric softplus function, defined as:

                  `f(x) = alpha * log(1 + exp(beta * x))`

                  where alpha and beta are two multi-dimensional arrays of same size as x.
                - 'THRESHOLDEDRELU': Thresholded ReLU function, defined as:

                  `f(x) = (x >= alpha) * x`

                  where alpha is a constant scalar.
                - 'LINEAR': linear function.

                   `f(x) = alpha * x + beta`

        input_name: str
            The input blob name of this layer.
        output_name: str
            The output blob name of this layer.
        params: [float] | [numpy.array]
            Parameters for the activation, depending on non_linearity. Kindly refer to NeuralNetwork.proto for details.

                - When non_linearity is one of ['RELU', 'SIGMOID', 'TANH', 'SCALED_TANH', 'SOFTPLUS', 'SOFTSIGN'], params is ignored.
                - When non_linearity is one of ['SCALED_TANH', 'SIGMOID_HARD', 'LINEAR'], param is a list of 2 floats
                  [alpha, beta].
                - When non_linearity is one of ['LEAKYRELU', 'ELU', 'THRESHOLDEDRELU'], param is a list of 1 float
                  [alpha].
                - When non_linearity is 'PRELU', param is a list of 1 numpy array [alpha]. The shape of
                  alpha is (C,), where C is either the number of input channels or
                  1. When C = 1, same alpha is applied to all channels.
                - When non_linearity is 'PARAMETRICSOFTPLUS', param is a list of 2 numpy arrays [alpha,
                  beta]. The shape of alpha and beta is (C, ), where C is either
                  the number of input channels or 1. When C = 1, same alpha and
                  beta are applied to all channels.

        See Also
        --------
        add_convolution, add_softmax
        """

        spec = self.spec
        nn_spec = self.nn_spec

        # Add a new layer
        spec_layer = nn_spec.layers.add()
        spec_layer.name = name
        spec_layer.input.append(input_name)
        spec_layer.output.append(output_name)
        spec_layer_params = spec_layer.activation

        # Fill in the parameters
        if non_linearity == 'RELU':
            spec_layer_params.ReLU.MergeFromString(b'')

        elif non_linearity == 'SIGMOID':
            spec_layer_params.sigmoid.MergeFromString(b'')

        elif non_linearity == 'TANH':
            spec_layer_params.tanh.MergeFromString(b'')

        elif non_linearity == 'SCALED_TANH':
            spec_layer_params.scaledTanh.MergeFromString(b'')
            if params is None:
                alpha, beta = (0.0, 0.0)
            else:
                alpha, beta = params[0], params[1]
            spec_layer_params.scaledTanh.alpha = alpha
            spec_layer_params.scaledTanh.beta = beta

        elif non_linearity == 'SOFTPLUS':
            spec_layer_params.softplus.MergeFromString(b'')

        elif non_linearity == 'SOFTSIGN':
            spec_layer_params.softsign.MergeFromString(b'')

        elif non_linearity == 'SIGMOID_HARD':
            if params is None:
                alpha, beta = (0.2, 0.5)
            else:
                alpha, beta = params[0], params[1]
            spec_layer_params.sigmoidHard.alpha = alpha
            spec_layer_params.sigmoidHard.beta = beta

        elif non_linearity == 'LEAKYRELU':
            if params is None:
                alpha = 0.3
            else:
                alpha = params[0]
            spec_layer_params.leakyReLU.alpha = float(alpha)

        elif non_linearity == 'PRELU':
            # PReLU must provide an np array in params[0]
            spec_layer_params.PReLU.alpha.floatValue.extend(map(float, params.flatten()))

        elif non_linearity == 'ELU':
            # ELU must provide an alpha in params[0]
            spec_layer_params.ELU.alpha = float(params)

        elif non_linearity == 'PARAMETRICSOFTPLUS':
            # Parametric softplus must provide two np arrays for alpha and beta
            alphas, betas = (params[0], params[1])
            # Weight alignment: Keras [H,W,C,F], Espresso [
            spec_layer_params.parametricSoftplus.alpha.floatValue.extend(map(float, alphas.flatten()))
            spec_layer_params.parametricSoftplus.beta.floatValue.extend(map(float, betas.flatten()))

        elif non_linearity == 'THRESHOLDEDRELU':
            if params is None:
                theta = 1.0
            else:
                theta = params
            spec_layer_params.thresholdedReLU.alpha = float(theta)

        elif non_linearity == 'LINEAR':
            if params is None:
                alpha, beta = (1.0, 0.0)
            else:
                alpha, beta = params[0], params[1]
            spec_layer_params.linear.alpha = alpha
            spec_layer_params.linear.beta = beta
        else:
            raise TypeError("Unknown activation type %s." %(non_linearity))