<gh_stars>0
#
# Copyright 2016 The BigDL Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

from sklearn.metrics import mean_squared_error
from sklearn.metrics import r2_score
from sklearn.metrics import mean_absolute_error
from sklearn.metrics import mean_squared_log_error
import numpy as np
import pandas as pd
from bigdl.dllib.utils.log4Error import *


EPSILON = 1e-10


def _standardize_input(y_true, y_pred, multioutput):
    """
    This function check the validation of the input
    input should be one of list/tuple/ndarray with same shape and not be None
    input dtype will be cast to np.float64
    input will be changed to corresponding 2-dim ndarray
    """
    if y_true is None or y_pred is None:
        invalidInputError(False,
                          "The input is None.")
    if not isinstance(y_true, (list, tuple, np.ndarray, pd.DataFrame)):
        invalidInputError(False,
                          "Expected array-like input. "
                          "Only list/tuple/ndarray/pd.DataFrame are supported")
    if isinstance(y_true, (list, tuple)):
        y_true = np.array(y_true)
    if isinstance(y_pred, (list, tuple)):
        y_pred = np.array(y_pred)
    if isinstance(y_true, pd.DataFrame):
        y_true = y_true.to_numpy()
    if isinstance(y_pred, pd.DataFrame):
        y_pred = y_pred.to_numpy()

    y_true = y_true.astype(np.float64)
    y_pred = y_pred.astype(np.float64)

    original_shape = y_true.shape[1:]

    if y_true.ndim == 1:
        y_true = y_true.reshape((-1, 1))
    else:
        y_true = y_true.reshape((y_true.shape[0], -1))

    if y_pred.ndim == 1:
        y_pred = y_pred.reshape((-1, 1))
    else:
        y_pred = y_pred.reshape((y_pred.shape[0], -1))

    if y_true.shape[0] != y_pred.shape[0]:
        invalidInputError(False,
                          "y_true and y_pred have different number of samples "
                          "({0}!={1})".format(y_true.shape[0], y_pred.shape[0]))
    if y_true.shape[1] != y_pred.shape[1]:
        invalidInputError(False,
                          "y_true and y_pred have different number of output "
                          "({0}!={1})".format(y_true.shape[1], y_pred.shape[1]))
    allowed_multioutput_str = ('raw_values', 'uniform_average',
                               'variance_weighted')
    if isinstance(multioutput, str):
        if multioutput not in allowed_multioutput_str:
            invalidInputError(False,
                              "Allowed 'multioutput' string values are {}. You provided"
                              " multioutput={!r}".format(allowed_multioutput_str, multioutput))

    return y_true, y_pred, original_shape


def sMAPE(y_true, y_pred, multioutput='raw_values'):
    """
    Calculate Symmetric mean absolute percentage error (sMAPE).

    .. math::

        \\text{sMAPE} = \\frac{100\%}{n} \\sum_{t=1}^n \\frac{|y_t-\\hat{y_t}|}{|y_t|+|\\hat{y_t}|}

    :param y_true: Array-like of shape = (n_samples, \*).
           Ground truth (correct) target values.
    :param y_pred: Array-like of shape = (n_samples, \*).
           Estimated target values.
    :param multioutput: String in ['raw_values', 'uniform_average']

    :return: Float or ndarray of floats.
             A non-negative floating point value (the best value is 0.0), or an
             array of floating point values, one for each individual target.
    """
    y_true, y_pred, original_shape = _standardize_input(y_true, y_pred, multioutput)
    output_errors = np.mean(100 * np.abs(y_true - y_pred) /
                            (np.abs(y_true) + np.abs(y_pred) + EPSILON), axis=0,)
    if multioutput == 'raw_values':
        return output_errors.reshape(original_shape)
    return np.mean(output_errors)


def MPE(y_true, y_pred, multioutput='raw_values'):
    """
    Calculate mean percentage error (MPE).

    .. math::

        \\text{MPE} = \\frac{100\%}{n}\\sum_{t=1}^n \\frac{y_t-\\hat{y_t}}{y_t}

    :param y_true: Array-like of shape = (n_samples, \*).
           Ground truth (correct) target values.
    :param y_pred: Array-like of shape = (n_samples, \*).
           Estimated target values.
    :param multioutput: String in ['raw_values', 'uniform_average']

    :return: Float or ndarray of floats.
             A non-negative floating point value (the best value is 0.0), or an
             array of floating point values, one for each individual target.
    """
    y_true, y_pred, original_shape = _standardize_input(y_true, y_pred, multioutput)
    output_errors = np.mean(100 * (y_true - y_pred) / (y_true + EPSILON), axis=0,)
    if multioutput == 'raw_values':
        return output_errors.reshape(original_shape)
    return np.mean(output_errors)


def MAPE(y_true, y_pred, multioutput='raw_values'):
    """
    Calculate mean absolute percentage error (MAPE).

    .. math::

        \\text{MAPE} = \\frac{100\%}{n}\\sum_{t=1}^n  |\\frac{y_t-\\hat{y_t}}{y_t}|

    :param y_true: Array-like of shape = (n_samples, \*).
           Ground truth (correct) target values.
    :param y_pred: Array-like of shape = (n_samples, \*).
           Estimated target values.
    :param multioutput: String in ['raw_values', 'uniform_average']

    :return: Float or ndarray of floats.
             A non-negative floating point value (the best value is 0.0), or an
             array of floating point values, one for each individual target.
    """
    y_true, y_pred, original_shape = _standardize_input(y_true, y_pred, multioutput)
    output_errors = 100 * np.mean(np.abs((y_true - y_pred) / (y_true + EPSILON)), axis=0,)
    if multioutput == 'raw_values':
        return output_errors.reshape(original_shape)
    return np.mean(output_errors)


def MDAPE(y_true, y_pred, multioutput='raw_values'):
    """
    Calculate Median Absolute Percentage Error (MDAPE).

    .. math::

        \\text{MDAPE} = 100\%\ median(|\\frac{y_1-\\hat{y_1}}{y_1}|,
        \\ldots, |\\frac{y_n-\\hat{y_n}}{y_n}|)

    :param y_true: Array-like of shape = (n_samples, \*).
           Ground truth (correct) target values.
    :param y_pred: Array-like of shape = (n_samples, \*).
           Estimated target values.
    :param multioutput: String in ['raw_values', 'uniform_average']

    :return: Float or ndarray of floats.
             A non-negative floating point value (the best value is 0.0), or an
             array of floating point values, one for each individual target.
    """
    y_true, y_pred, original_shape = _standardize_input(y_true, y_pred, multioutput)
    output_errors = np.median(100 * np.abs((y_true - y_pred) / (y_true + EPSILON)), axis=0,)
    if multioutput == 'raw_values':
        return output_errors.reshape(original_shape)
    return np.mean(output_errors)


def sMDAPE(y_true, y_pred, multioutput='raw_values'):
    """
    Calculate Symmetric Median Absolute Percentage Error (sMDAPE).

    .. math::

        \\text{sMDAPE} = 100\%\ median(\\frac{|y_1-\\hat{y_1}|}{|y_1|+|\\hat{y_1}|},
        \\ldots, \\frac{|y_n-\\hat{y_n}|}{|y_n|+|\\hat{y_n}|})

    :param y_true: Array-like of shape = (n_samples, \*).
        Ground truth (correct) target values.
    :param y_pred: Array-like of shape = (n_samples, \*).
        Estimated target values.
    :param multioutput: String in ['raw_values', 'uniform_average']

    :return: Float or ndarray of floats.
             A non-negative floating point value (the best value is 0.0), or an
             array of floating point values, one for each individual target.
    """
    y_true, y_pred, original_shape = _standardize_input(y_true, y_pred, multioutput)
    output_errors = np.median(100 * np.abs(y_true - y_pred) /
                              (np.abs(y_true) + np.abs(y_pred) + EPSILON), axis=0, )
    if multioutput == 'raw_values':
        return output_errors.reshape(original_shape)
    return np.mean(output_errors)


def ME(y_true, y_pred, multioutput='raw_values'):
    """
    Calculate Mean Error (ME).

    .. math::

        \\text{ME} = \\frac{1}{n}\\sum_{t=1}^n  y_t-\\hat{y_t}

    :param y_true: Array-like of shape = (n_samples, \*).
           Ground truth (correct) target values.
    :param y_pred: Array-like of shape = (n_samples, \*).
           Estimated target values.
    :param multioutput: String in ['raw_values', 'uniform_average']

    :return: Float or ndarray of floats.
             A floating point value (the best value is 0.0), or an
             array of floating point values, one for each individual target.
    """
    y_true, y_pred, original_shape = _standardize_input(y_true, y_pred, multioutput)
    output_errors = np.mean(y_true - y_pred, axis=0,)
    if multioutput == 'raw_values':
        return output_errors.reshape(original_shape)
    return np.mean(output_errors)


def MSPE(y_true, y_pred, multioutput='raw_values'):
    """
    Calculate mean squared percentage error (MSPE).

    .. math::

        \\text{MSPE} = \\frac{100\%}{n}\\sum_{t=1}^n  (\\frac{y_n-\\hat{y_n}}{y_n})^2

    :param y_true: Array-like of shape = (n_samples, \*).
           Ground truth (correct) target values.
    :param y_pred: Array-like of shape = (n_samples, \*).
           Estimated target values.
    :param multioutput: String in ['raw_values', 'uniform_average']

    :return: Float or ndarray of floats.
             A non-negative floating point value (the best value is 0.0), or an
             array of floating point values, one for each individual target.
    """
    y_true, y_pred, original_shape = _standardize_input(y_true, y_pred, multioutput)
    output_errors = 100 * np.mean(np.square((y_true - y_pred) / (y_true + EPSILON)), axis=0, )
    if multioutput == 'raw_values':
        return output_errors.reshape(original_shape)
    return np.mean(output_errors)


def MSLE(y_true, y_pred, multioutput='raw_values'):
    """
    Calculate the mean squared log error(MSLE).

    .. math::

        \\text{MSLE} = \\frac{1}{n}\\sum_{t=1}^n (log_e(1+y_t)-log_e(1+\\hat{y_t}))^2

    :param y_true: Array-like of shape = (n_samples, \*).
           Ground truth (correct) target values.
    :param y_pred: Array-like of shape = (n_samples, \*).
           Estimated target values.
    :param multioutput: String in ['raw_values', 'uniform_average']

    :return: Float or ndarray of floats.
             A non-negative floating point value (the best value is 0.0), or an
             array of floating point values, one for each individual target.
    """
    y_true, y_pred, original_shape = _standardize_input(y_true, y_pred, multioutput)
    result = mean_squared_log_error(y_true, y_pred, multioutput=multioutput)
    if multioutput == 'raw_values':
        return result.reshape(original_shape)
    return result


def R2(y_true, y_pred, multioutput='raw_values'):
    """
    Calculate the r2 score.

    .. math::

        R^2 = 1-\\frac{\\sum_{t=1}^n (y_t-\\hat{y_t})^2}{\\sum_{t=1}^n (y_t-\\bar{y})^2}

    :param y_true: Array-like of shape = (n_samples, \*).
           Ground truth (correct) target values.
    :param y_pred: Array-like of shape = (n_samples, \*).
           Estimated target values.
    :param multioutput: String in ['raw_values', 'uniform_average']

    :return: Float or ndarray of floats.
             A non-negative floating point value (the best value is 1.0), or an
             array of floating point values, one for each individual target.
    """
    y_true, y_pred, original_shape = _standardize_input(y_true, y_pred, multioutput)
    result = r2_score(y_true, y_pred, multioutput=multioutput)
    if multioutput == 'raw_values':
        return result.reshape(original_shape)
    return result


def MAE(y_true, y_pred, multioutput='raw_values'):
    """
    Calculate the mean absolute error (MAE).

    .. math::

        \\text{MAE} = \\frac{1}{n}\\sum_{t=1}^n |y_t-\\hat{y_t}|

    :param y_true: Array-like of shape = (n_samples, \*).
           Ground truth (correct) target values.
    :param y_pred: Array-like of shape = (n_samples, \*).
           Estimated target values.
    :param multioutput: String in ['raw_values', 'uniform_average']

    :return: Float or ndarray of floats.
             A non-negative floating point value (the best value is 0.0), or an
             array of floating point values, one for each individual target.
    """
    y_true, y_pred, original_shape = _standardize_input(y_true, y_pred, multioutput)
    result = mean_absolute_error(y_true, y_pred, multioutput=multioutput)
    if multioutput == 'raw_values':
        return result.reshape(original_shape)
    return result


def RMSE(y_true, y_pred, multioutput='raw_values'):
    """
    Calculate square root of the mean squared error (RMSE).

    .. math::

        \\text{RMSE} = \\sqrt{(\\frac{1}{n}\\sum_{t=1}^n (y_t-\\hat{y_t})^2)}

    :param y_true: Array-like of shape = (n_samples, \*).
           Ground truth (correct) target values.
    :param y_pred: Array-like of shape = (n_samples, \*).
           Estimated target values.
    :param multioutput: String in ['raw_values', 'uniform_average']

    :return: Float or ndarray of floats.
             A non-negative floating point value (the best value is 0.0), or an
             array of floating point values, one for each individual target.
    """
    return np.sqrt(MSE(y_true, y_pred, multioutput=multioutput))


def MSE(y_true, y_pred, multioutput='uniform_average'):
    """
    Calculate the mean squared error (MSE).

    .. math::

        \\text{MSE} = \\frac{1}{n}\\sum_{t=1}^n (y_t-\\hat{y_t})^2

    :param y_true: Array-like of shape = (n_samples, \*).
           Ground truth (correct) target values.
    :param y_pred: Array-like of shape = (n_samples, \*).
           Estimated target values.
    :param multioutput: String in ['raw_values', 'uniform_average']

    :return: Float or ndarray of floats.
             A non-negative floating point value (the best value is 0.0), or an
             array of floating point values, one for each individual target.
    """
    y_true, y_pred, original_shape = _standardize_input(y_true, y_pred, multioutput)
    result = mean_squared_error(y_true, y_pred, multioutput=multioutput)
    if multioutput == 'raw_values':
        return result.reshape(original_shape)
    return result


def Accuracy(y_true, y_pred, multioutput=None):
    """
    Calculate the accuracy score (Accuracy).

    .. math::

        \\text{Accuracy} = \\frac{1}{n}\\sum_{t=1}^n 1(y_t=\\hat{y_t})

    :param y_true: Array-like of shape = (n_samples, \*).
           Ground truth (correct) target values.
    :param y_pred: Array-like of shape = (n_samples, \*).
           Estimated target values.

    :return: Float or ndarray of floats.
             A non-negative floating point value (the best value is 1.0), or an
             array of floating point values, one for each individual target.
    """
    from sklearn.metrics._classification import accuracy_score
    y_true = np.squeeze(y_true)
    y_pred = np.squeeze(y_pred)
    if np.any(y_pred != y_pred.astype(int)):
        # y_pred is probability
        if y_pred.ndim == 1:
            y_pred = np.where(y_pred > 0.5, 1, 0)
        else:
            y_pred = np.argmax(y_pred, axis=1)

    return accuracy_score(y_true, y_pred)


class Evaluator(object):
    """
    Evaluate metrics for y_true and y_pred.
    """

    metrics_func = {
        # Absolute
        'me': ME,
        'mae': MAE,
        'mse': MSE,
        'rmse': RMSE,
        'msle': MSLE,
        'r2': R2,
        # Relative
        'mpe': MPE,
        'mape': MAPE,
        'mspe': MSPE,
        'smape': sMAPE,
        'mdape': MDAPE,
        'smdape': sMDAPE,
        'accuracy': Accuracy,
    }

    max_mode_metrics = ('r2', 'accuracy')

    @staticmethod
    def evaluate(metric, y_true, y_pred, multioutput='raw_values'):
        """
        Evaluate a specific metric for y_true and y_pred.

        :param metric: String in ['me', 'mae', 'mse', 'rmse', 'msle', 'r2'
               , 'mpe', 'mape', 'mspe', 'smape', 'mdape', 'smdape', 'accuracy']
        :param y_true: Array-like of shape = (n_samples, \*).
               Ground truth (correct) target values.
        :param y_pred: Array-like of shape = (n_samples, \*).
               Estimated target values.
        :param multioutput: String in ['raw_values', 'uniform_average']

        :return: Float or ndarray of floats.
                 A floating point value, or an
                 array of floating point values, one for each individual target.
        """
        Evaluator.check_metric(metric)
        result = Evaluator.metrics_func[metric](y_true, y_pred, multioutput=multioutput)
        if result.shape == (1,):
            return result[0]
        else:
            return result

    @staticmethod
    def check_metric(metric):
        if not metric:
            invalidInputError(False,
                              f"Got invalid metric name of {metric}!")
        if metric not in Evaluator.metrics_func.keys():
            invalidInputError(False,
                              "metric " + metric + " is not supported")

    @staticmethod
    def get_metric_mode(metric):
        Evaluator.check_metric(metric)
        if metric in Evaluator.max_mode_metrics:
            return "max"
        else:
            return "min"

#
# Copyright 2016 The BigDL Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

from sklearn.metrics import mean_squared_error
from sklearn.metrics import r2_score
from sklearn.metrics import mean_absolute_error
from sklearn.metrics import mean_squared_log_error
import numpy as np
import pandas as pd
from bigdl.dllib.utils.log4Error import *


EPSILON = 1e-10


def _standardize_input(y_true, y_pred, multioutput):
    """
    This function check the validation of the input
    input should be one of list/tuple/ndarray with same shape and not be None
    input dtype will be cast to np.float64
    input will be changed to corresponding 2-dim ndarray
    """
    if y_true is None or y_pred is None:
        invalidInputError(False,
                          "The input is None.")
    if not isinstance(y_true, (list, tuple, np.ndarray, pd.DataFrame)):
        invalidInputError(False,
                          "Expected array-like input. "
                          "Only list/tuple/ndarray/pd.DataFrame are supported")
    if isinstance(y_true, (list, tuple)):
        y_true = np.array(y_true)
    if isinstance(y_pred, (list, tuple)):
        y_pred = np.array(y_pred)
    if isinstance(y_true, pd.DataFrame):
        y_true = y_true.to_numpy()
    if isinstance(y_pred, pd.DataFrame):
        y_pred = y_pred.to_numpy()

    y_true = y_true.astype(np.float64)
    y_pred = y_pred.astype(np.float64)

    original_shape = y_true.shape[1:]

    if y_true.ndim == 1:
        y_true = y_true.reshape((-1, 1))
    else:
        y_true = y_true.reshape((y_true.shape[0], -1))

    if y_pred.ndim == 1:
        y_pred = y_pred.reshape((-1, 1))
    else:
        y_pred = y_pred.reshape((y_pred.shape[0], -1))

    if y_true.shape[0] != y_pred.shape[0]:
        invalidInputError(False,
                          "y_true and y_pred have different number of samples "
                          "({0}!={1})".format(y_true.shape[0], y_pred.shape[0]))
    if y_true.shape[1] != y_pred.shape[1]:
        invalidInputError(False,
                          "y_true and y_pred have different number of output "
                          "({0}!={1})".format(y_true.shape[1], y_pred.shape[1]))
    allowed_multioutput_str = ('raw_values', 'uniform_average',
                               'variance_weighted')
    if isinstance(multioutput, str):
        if multioutput not in allowed_multioutput_str:
            invalidInputError(False,
                              "Allowed 'multioutput' string values are {}. You provided"
                              " multioutput={!r}".format(allowed_multioutput_str, multioutput))

    return y_true, y_pred, original_shape


def sMAPE(y_true, y_pred, multioutput='raw_values'):
    """
    Calculate Symmetric mean absolute percentage error (sMAPE).

    .. math::

        \\text{sMAPE} = \\frac{100\%}{n} \\sum_{t=1}^n \\frac{|y_t-\\hat{y_t}|}{|y_t|+|\\hat{y_t}|}

    :param y_true: Array-like of shape = (n_samples, \*).
           Ground truth (correct) target values.
    :param y_pred: Array-like of shape = (n_samples, \*).
           Estimated target values.
    :param multioutput: String in ['raw_values', 'uniform_average']

    :return: Float or ndarray of floats.
             A non-negative floating point value (the best value is 0.0), or an
             array of floating point values, one for each individual target.
    """
    y_true, y_pred, original_shape = _standardize_input(y_true, y_pred, multioutput)
    output_errors = np.mean(100 * np.abs(y_true - y_pred) /
                            (np.abs(y_true) + np.abs(y_pred) + EPSILON), axis=0,)
    if multioutput == 'raw_values':
        return output_errors.reshape(original_shape)
    return np.mean(output_errors)


def MPE(y_true, y_pred, multioutput='raw_values'):
    """
    Calculate mean percentage error (MPE).

    .. math::

        \\text{MPE} = \\frac{100\%}{n}\\sum_{t=1}^n \\frac{y_t-\\hat{y_t}}{y_t}

    :param y_true: Array-like of shape = (n_samples, \*).
           Ground truth (correct) target values.
    :param y_pred: Array-like of shape = (n_samples, \*).
           Estimated target values.
    :param multioutput: String in ['raw_values', 'uniform_average']

    :return: Float or ndarray of floats.
             A non-negative floating point value (the best value is 0.0), or an
             array of floating point values, one for each individual target.
    """
    y_true, y_pred, original_shape = _standardize_input(y_true, y_pred, multioutput)
    output_errors = np.mean(100 * (y_true - y_pred) / (y_true + EPSILON), axis=0,)
    if multioutput == 'raw_values':
        return output_errors.reshape(original_shape)
    return np.mean(output_errors)


def MAPE(y_true, y_pred, multioutput='raw_values'):
    """
    Calculate mean absolute percentage error (MAPE).

    .. math::

        \\text{MAPE} = \\frac{100\%}{n}\\sum_{t=1}^n  |\\frac{y_t-\\hat{y_t}}{y_t}|

    :param y_true: Array-like of shape = (n_samples, \*).
           Ground truth (correct) target values.
    :param y_pred: Array-like of shape = (n_samples, \*).
           Estimated target values.
    :param multioutput: String in ['raw_values', 'uniform_average']

    :return: Float or ndarray of floats.
             A non-negative floating point value (the best value is 0.0), or an
             array of floating point values, one for each individual target.
    """
    y_true, y_pred, original_shape = _standardize_input(y_true, y_pred, multioutput)
    output_errors = 100 * np.mean(np.abs((y_true - y_pred) / (y_true + EPSILON)), axis=0,)
    if multioutput == 'raw_values':
        return output_errors.reshape(original_shape)
    return np.mean(output_errors)


def MDAPE(y_true, y_pred, multioutput='raw_values'):
    """
    Calculate Median Absolute Percentage Error (MDAPE).

    .. math::

        \\text{MDAPE} = 100\%\ median(|\\frac{y_1-\\hat{y_1}}{y_1}|,
        \\ldots, |\\frac{y_n-\\hat{y_n}}{y_n}|)

    :param y_true: Array-like of shape = (n_samples, \*).
           Ground truth (correct) target values.
    :param y_pred: Array-like of shape = (n_samples, \*).
           Estimated target values.
    :param multioutput: String in ['raw_values', 'uniform_average']

    :return: Float or ndarray of floats.
             A non-negative floating point value (the best value is 0.0), or an
             array of floating point values, one for each individual target.
    """
    y_true, y_pred, original_shape = _standardize_input(y_true, y_pred, multioutput)
    output_errors = np.median(100 * np.abs((y_true - y_pred) / (y_true + EPSILON)), axis=0,)
    if multioutput == 'raw_values':
        return output_errors.reshape(original_shape)
    return np.mean(output_errors)


def sMDAPE(y_true, y_pred, multioutput='raw_values'):
    """
    Calculate Symmetric Median Absolute Percentage Error (sMDAPE).

    .. math::

        \\text{sMDAPE} = 100\%\ median(\\frac{|y_1-\\hat{y_1}|}{|y_1|+|\\hat{y_1}|},
        \\ldots, \\frac{|y_n-\\hat{y_n}|}{|y_n|+|\\hat{y_n}|})

    :param y_true: Array-like of shape = (n_samples, \*).
        Ground truth (correct) target values.
    :param y_pred: Array-like of shape = (n_samples, \*).
        Estimated target values.
    :param multioutput: String in ['raw_values', 'uniform_average']

    :return: Float or ndarray of floats.
             A non-negative floating point value (the best value is 0.0), or an
             array of floating point values, one for each individual target.
    """
    y_true, y_pred, original_shape = _standardize_input(y_true, y_pred, multioutput)
    output_errors = np.median(100 * np.abs(y_true - y_pred) /
                              (np.abs(y_true) + np.abs(y_pred) + EPSILON), axis=0, )
    if multioutput == 'raw_values':
        return output_errors.reshape(original_shape)
    return np.mean(output_errors)


def ME(y_true, y_pred, multioutput='raw_values'):
    """
    Calculate Mean Error (ME).

    .. math::

        \\text{ME} = \\frac{1}{n}\\sum_{t=1}^n  y_t-\\hat{y_t}

    :param y_true: Array-like of shape = (n_samples, \*).
           Ground truth (correct) target values.
    :param y_pred: Array-like of shape = (n_samples, \*).
           Estimated target values.
    :param multioutput: String in ['raw_values', 'uniform_average']

    :return: Float or ndarray of floats.
             A floating point value (the best value is 0.0), or an
             array of floating point values, one for each individual target.
    """
    y_true, y_pred, original_shape = _standardize_input(y_true, y_pred, multioutput)
    output_errors = np.mean(y_true - y_pred, axis=0,)
    if multioutput == 'raw_values':
        return output_errors.reshape(original_shape)
    return np.mean(output_errors)


def MSPE(y_true, y_pred, multioutput='raw_values'):
    """
    Calculate mean squared percentage error (MSPE).

    .. math::

        \\text{MSPE} = \\frac{100\%}{n}\\sum_{t=1}^n  (\\frac{y_n-\\hat{y_n}}{y_n})^2

    :param y_true: Array-like of shape = (n_samples, \*).
           Ground truth (correct) target values.
    :param y_pred: Array-like of shape = (n_samples, \*).
           Estimated target values.
    :param multioutput: String in ['raw_values', 'uniform_average']

    :return: Float or ndarray of floats.
             A non-negative floating point value (the best value is 0.0), or an
             array of floating point values, one for each individual target.
    """
    y_true, y_pred, original_shape = _standardize_input(y_true, y_pred, multioutput)
    output_errors = 100 * np.mean(np.square((y_true - y_pred) / (y_true + EPSILON)), axis=0, )
    if multioutput == 'raw_values':
        return output_errors.reshape(original_shape)
    return np.mean(output_errors)


def MSLE(y_true, y_pred, multioutput='raw_values'):
    """
    Calculate the mean squared log error(MSLE).

    .. math::

        \\text{MSLE} = \\frac{1}{n}\\sum_{t=1}^n (log_e(1+y_t)-log_e(1+\\hat{y_t}))^2

    :param y_true: Array-like of shape = (n_samples, \*).
           Ground truth (correct) target values.
    :param y_pred: Array-like of shape = (n_samples, \*).
           Estimated target values.
    :param multioutput: String in ['raw_values', 'uniform_average']

    :return: Float or ndarray of floats.
             A non-negative floating point value (the best value is 0.0), or an
             array of floating point values, one for each individual target.
    """
    y_true, y_pred, original_shape = _standardize_input(y_true, y_pred, multioutput)
    result = mean_squared_log_error(y_true, y_pred, multioutput=multioutput)
    if multioutput == 'raw_values':
        return result.reshape(original_shape)
    return result


def R2(y_true, y_pred, multioutput='raw_values'):
    """
    Calculate the r2 score.

    .. math::

        R^2 = 1-\\frac{\\sum_{t=1}^n (y_t-\\hat{y_t})^2}{\\sum_{t=1}^n (y_t-\\bar{y})^2}

    :param y_true: Array-like of shape = (n_samples, \*).
           Ground truth (correct) target values.
    :param y_pred: Array-like of shape = (n_samples, \*).
           Estimated target values.
    :param multioutput: String in ['raw_values', 'uniform_average']

    :return: Float or ndarray of floats.
             A non-negative floating point value (the best value is 1.0), or an
             array of floating point values, one for each individual target.
    """
    y_true, y_pred, original_shape = _standardize_input(y_true, y_pred, multioutput)
    result = r2_score(y_true, y_pred, multioutput=multioutput)
    if multioutput == 'raw_values':
        return result.reshape(original_shape)
    return result


def MAE(y_true, y_pred, multioutput='raw_values'):
    """
    Calculate the mean absolute error (MAE).

    .. math::

        \\text{MAE} = \\frac{1}{n}\\sum_{t=1}^n |y_t-\\hat{y_t}|

    :param y_true: Array-like of shape = (n_samples, \*).
           Ground truth (correct) target values.
    :param y_pred: Array-like of shape = (n_samples, \*).
           Estimated target values.
    :param multioutput: String in ['raw_values', 'uniform_average']

    :return: Float or ndarray of floats.
             A non-negative floating point value (the best value is 0.0), or an
             array of floating point values, one for each individual target.
    """
    y_true, y_pred, original_shape = _standardize_input(y_true, y_pred, multioutput)
    result = mean_absolute_error(y_true, y_pred, multioutput=multioutput)
    if multioutput == 'raw_values':
        return result.reshape(original_shape)
    return result


def RMSE(y_true, y_pred, multioutput='raw_values'):
    """
    Calculate square root of the mean squared error (RMSE).

    .. math::

        \\text{RMSE} = \\sqrt{(\\frac{1}{n}\\sum_{t=1}^n (y_t-\\hat{y_t})^2)}

    :param y_true: Array-like of shape = (n_samples, \*).
           Ground truth (correct) target values.
    :param y_pred: Array-like of shape = (n_samples, \*).
           Estimated target values.
    :param multioutput: String in ['raw_values', 'uniform_average']

    :return: Float or ndarray of floats.
             A non-negative floating point value (the best value is 0.0), or an
             array of floating point values, one for each individual target.
    """
    return np.sqrt(MSE(y_true, y_pred, multioutput=multioutput))


def MSE(y_true, y_pred, multioutput='uniform_average'):
    """
    Calculate the mean squared error (MSE).

    .. math::

        \\text{MSE} = \\frac{1}{n}\\sum_{t=1}^n (y_t-\\hat{y_t})^2

    :param y_true: Array-like of shape = (n_samples, \*).
           Ground truth (correct) target values.
    :param y_pred: Array-like of shape = (n_samples, \*).
           Estimated target values.
    :param multioutput: String in ['raw_values', 'uniform_average']

    :return: Float or ndarray of floats.
             A non-negative floating point value (the best value is 0.0), or an
             array of floating point values, one for each individual target.
    """
    y_true, y_pred, original_shape = _standardize_input(y_true, y_pred, multioutput)
    result = mean_squared_error(y_true, y_pred, multioutput=multioutput)
    if multioutput == 'raw_values':
        return result.reshape(original_shape)
    return result


def Accuracy(y_true, y_pred, multioutput=None):
    """
    Calculate the accuracy score (Accuracy).

    .. math::

        \\text{Accuracy} = \\frac{1}{n}\\sum_{t=1}^n 1(y_t=\\hat{y_t})

    :param y_true: Array-like of shape = (n_samples, \*).
           Ground truth (correct) target values.
    :param y_pred: Array-like of shape = (n_samples, \*).
           Estimated target values.

    :return: Float or ndarray of floats.
             A non-negative floating point value (the best value is 1.0), or an
             array of floating point values, one for each individual target.
    """
    from sklearn.metrics._classification import accuracy_score
    y_true = np.squeeze(y_true)
    y_pred = np.squeeze(y_pred)
    if np.any(y_pred != y_pred.astype(int)):
        # y_pred is probability
        if y_pred.ndim == 1:
            y_pred = np.where(y_pred > 0.5, 1, 0)
        else:
            y_pred = np.argmax(y_pred, axis=1)

    return accuracy_score(y_true, y_pred)


class Evaluator(object):
    """
    Evaluate metrics for y_true and y_pred.
    """

    metrics_func = {
        # Absolute
        'me': ME,
        'mae': MAE,
        'mse': MSE,
        'rmse': RMSE,
        'msle': MSLE,
        'r2': R2,
        # Relative
        'mpe': MPE,
        'mape': MAPE,
        'mspe': MSPE,
        'smape': sMAPE,
        'mdape': MDAPE,
        'smdape': sMDAPE,
        'accuracy': Accuracy,
    }

    max_mode_metrics = ('r2', 'accuracy')

    @staticmethod
    def evaluate(metric, y_true, y_pred, multioutput='raw_values'):
        """
        Evaluate a specific metric for y_true and y_pred.

        :param metric: String in ['me', 'mae', 'mse', 'rmse', 'msle', 'r2'
               , 'mpe', 'mape', 'mspe', 'smape', 'mdape', 'smdape', 'accuracy']
        :param y_true: Array-like of shape = (n_samples, \*).
               Ground truth (correct) target values.
        :param y_pred: Array-like of shape = (n_samples, \*).
               Estimated target values.
        :param multioutput: String in ['raw_values', 'uniform_average']

        :return: Float or ndarray of floats.
                 A floating point value, or an
                 array of floating point values, one for each individual target.
        """
        Evaluator.check_metric(metric)
        result = Evaluator.metrics_func[metric](y_true, y_pred, multioutput=multioutput)
        if result.shape == (1,):
            return result[0]
        else:
            return result

    @staticmethod
    def check_metric(metric):
        if not metric:
            invalidInputError(False,
                              f"Got invalid metric name of {metric}!")
        if metric not in Evaluator.metrics_func.keys():
            invalidInputError(False,
                              "metric " + metric + " is not supported")

    @staticmethod
    def get_metric_mode(metric):
        Evaluator.check_metric(metric)
        if metric in Evaluator.max_mode_metrics:
            return "max"
        else:
            return "min"
# # Copyright 2016 The BigDL Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # #     http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # This function check the validation of the input     input should be one of list/tuple/ndarray with same shape and not be None     input dtype will be cast to np.float64     input will be changed to corresponding 2-dim ndarray Calculate Symmetric mean absolute percentage error (sMAPE).      .. math::          \\text{sMAPE} = \\frac{100\%}{n} \\sum_{t=1}^n \\frac{|y_t-\\hat{y_t}|}{|y_t|+|\\hat{y_t}|}      :param y_true: Array-like of shape = (n_samples, \*).            Ground truth (correct) target values.     :param y_pred: Array-like of shape = (n_samples, \*).            Estimated target values.     :param multioutput: String in ['raw_values', 'uniform_average']      :return: Float or ndarray of floats.              A non-negative floating point value (the best value is 0.0), or an              array of floating point values, one for each individual target. Calculate mean percentage error (MPE).      .. math::          \\text{MPE} = \\frac{100\%}{n}\\sum_{t=1}^n \\frac{y_t-\\hat{y_t}}{y_t}      :param y_true: Array-like of shape = (n_samples, \*).            Ground truth (correct) target values.     :param y_pred: Array-like of shape = (n_samples, \*).            Estimated target values.     :param multioutput: String in ['raw_values', 'uniform_average']      :return: Float or ndarray of floats.              A non-negative floating point value (the best value is 0.0), or an              array of floating point values, one for each individual target. Calculate mean absolute percentage error (MAPE).      .. math::          \\text{MAPE} = \\frac{100\%}{n}\\sum_{t=1}^n  |\\frac{y_t-\\hat{y_t}}{y_t}|      :param y_true: Array-like of shape = (n_samples, \*).            Ground truth (correct) target values.     :param y_pred: Array-like of shape = (n_samples, \*).            Estimated target values.     :param multioutput: String in ['raw_values', 'uniform_average']      :return: Float or ndarray of floats.              A non-negative floating point value (the best value is 0.0), or an              array of floating point values, one for each individual target. Calculate Median Absolute Percentage Error (MDAPE).      .. math::          \\text{MDAPE} = 100\%\ median(|\\frac{y_1-\\hat{y_1}}{y_1}|,         \\ldots, |\\frac{y_n-\\hat{y_n}}{y_n}|)      :param y_true: Array-like of shape = (n_samples, \*).            Ground truth (correct) target values.     :param y_pred: Array-like of shape = (n_samples, \*).            Estimated target values.     :param multioutput: String in ['raw_values', 'uniform_average']      :return: Float or ndarray of floats.              A non-negative floating point value (the best value is 0.0), or an              array of floating point values, one for each individual target. Calculate Symmetric Median Absolute Percentage Error (sMDAPE).      .. math::          \\text{sMDAPE} = 100\%\ median(\\frac{|y_1-\\hat{y_1}|}{|y_1|+|\\hat{y_1}|},         \\ldots, \\frac{|y_n-\\hat{y_n}|}{|y_n|+|\\hat{y_n}|})      :param y_true: Array-like of shape = (n_samples, \*).         Ground truth (correct) target values.     :param y_pred: Array-like of shape = (n_samples, \*).         Estimated target values.     :param multioutput: String in ['raw_values', 'uniform_average']      :return: Float or ndarray of floats.              A non-negative floating point value (the best value is 0.0), or an              array of floating point values, one for each individual target. Calculate Mean Error (ME).      .. math::          \\text{ME} = \\frac{1}{n}\\sum_{t=1}^n  y_t-\\hat{y_t}      :param y_true: Array-like of shape = (n_samples, \*).            Ground truth (correct) target values.     :param y_pred: Array-like of shape = (n_samples, \*).            Estimated target values.     :param multioutput: String in ['raw_values', 'uniform_average']      :return: Float or ndarray of floats.              A floating point value (the best value is 0.0), or an              array of floating point values, one for each individual target. Calculate mean squared percentage error (MSPE).      .. math::          \\text{MSPE} = \\frac{100\%}{n}\\sum_{t=1}^n  (\\frac{y_n-\\hat{y_n}}{y_n})^2      :param y_true: Array-like of shape = (n_samples, \*).            Ground truth (correct) target values.     :param y_pred: Array-like of shape = (n_samples, \*).            Estimated target values.     :param multioutput: String in ['raw_values', 'uniform_average']      :return: Float or ndarray of floats.              A non-negative floating point value (the best value is 0.0), or an              array of floating point values, one for each individual target. Calculate the mean squared log error(MSLE).      .. math::          \\text{MSLE} = \\frac{1}{n}\\sum_{t=1}^n (log_e(1+y_t)-log_e(1+\\hat{y_t}))^2      :param y_true: Array-like of shape = (n_samples, \*).            Ground truth (correct) target values.     :param y_pred: Array-like of shape = (n_samples, \*).            Estimated target values.     :param multioutput: String in ['raw_values', 'uniform_average']      :return: Float or ndarray of floats.              A non-negative floating point value (the best value is 0.0), or an              array of floating point values, one for each individual target. Calculate the r2 score.      .. math::          R^2 = 1-\\frac{\\sum_{t=1}^n (y_t-\\hat{y_t})^2}{\\sum_{t=1}^n (y_t-\\bar{y})^2}      :param y_true: Array-like of shape = (n_samples, \*).            Ground truth (correct) target values.     :param y_pred: Array-like of shape = (n_samples, \*).            Estimated target values.     :param multioutput: String in ['raw_values', 'uniform_average']      :return: Float or ndarray of floats.              A non-negative floating point value (the best value is 1.0), or an              array of floating point values, one for each individual target. Calculate the mean absolute error (MAE).      .. math::          \\text{MAE} = \\frac{1}{n}\\sum_{t=1}^n |y_t-\\hat{y_t}|      :param y_true: Array-like of shape = (n_samples, \*).            Ground truth (correct) target values.     :param y_pred: Array-like of shape = (n_samples, \*).            Estimated target values.     :param multioutput: String in ['raw_values', 'uniform_average']      :return: Float or ndarray of floats.              A non-negative floating point value (the best value is 0.0), or an              array of floating point values, one for each individual target. Calculate square root of the mean squared error (RMSE).      .. math::          \\text{RMSE} = \\sqrt{(\\frac{1}{n}\\sum_{t=1}^n (y_t-\\hat{y_t})^2)}      :param y_true: Array-like of shape = (n_samples, \*).            Ground truth (correct) target values.     :param y_pred: Array-like of shape = (n_samples, \*).            Estimated target values.     :param multioutput: String in ['raw_values', 'uniform_average']      :return: Float or ndarray of floats.              A non-negative floating point value (the best value is 0.0), or an              array of floating point values, one for each individual target. Calculate the mean squared error (MSE).      .. math::          \\text{MSE} = \\frac{1}{n}\\sum_{t=1}^n (y_t-\\hat{y_t})^2      :param y_true: Array-like of shape = (n_samples, \*).            Ground truth (correct) target values.     :param y_pred: Array-like of shape = (n_samples, \*).            Estimated target values.     :param multioutput: String in ['raw_values', 'uniform_average']      :return: Float or ndarray of floats.              A non-negative floating point value (the best value is 0.0), or an              array of floating point values, one for each individual target. Calculate the accuracy score (Accuracy).      .. math::          \\text{Accuracy} = \\frac{1}{n}\\sum_{t=1}^n 1(y_t=\\hat{y_t})      :param y_true: Array-like of shape = (n_samples, \*).            Ground truth (correct) target values.     :param y_pred: Array-like of shape = (n_samples, \*).            Estimated target values.      :return: Float or ndarray of floats.              A non-negative floating point value (the best value is 1.0), or an              array of floating point values, one for each individual target. # y_pred is probability Evaluate metrics for y_true and y_pred. # Absolute # Relative Evaluate a specific metric for y_true and y_pred.          :param metric: String in ['me', 'mae', 'mse', 'rmse', 'msle', 'r2'                , 'mpe', 'mape', 'mspe', 'smape', 'mdape', 'smdape', 'accuracy']         :param y_true: Array-like of shape = (n_samples, \*).                Ground truth (correct) target values.         :param y_pred: Array-like of shape = (n_samples, \*).                Estimated target values.         :param multioutput: String in ['raw_values', 'uniform_average']          :return: Float or ndarray of floats.                  A floating point value, or an                  array of floating point values, one for each individual target.