def get_dataset(dataset_id, include_results=False):
    """
    get the descriptive data of a dataset

    :param dataset_id: id of the dataset
    :param include_results: if need to extract results also
    :return: dataset object
    """
    dt = make_dataset(get_key_store('dataset:%s' % dataset_id))
    dt.status = get_key_store('dataset:%s:status' % dataset_id)

    # add counters and results
    dt.grapher = get_key_store('dataset:%s:grapher' % dataset_id)
    dt.round_counter = get_counter_store('dataset:%s:round_counter' % dataset_id)

    if include_results:
        dt.results = get_key_store('dataset:%s:results' % dataset_id)

    return dt def get_dataset_status(dataset_id):
    """
    get the status of a dataset
    """
    return get_key_store('dataset:%s:status' % dataset_id) def graph_histogram_regression(dataset, round_id, y, part='eval'):
    """
    generate the histogram of predictions

    :param dataset: dataset object
    :param round_id: id of the round (model)
    :param y: prediction values
    :param part: set (eval / train set)
    :return: None
    """
    try:
        for dark, theme in [(True, 'dark_background'), (False, 'seaborn-whitegrid')]:
            with plt.style.context(theme, after_reset=True):
                plt.figure(figsize=(6, 6))
                sns.distplot(y)
                plt.title('histogram of predictions (%s set)' % part)
                plt.xlabel('values')
                plt.ylabel('frequencies')
                __save_fig(dataset.dataset_id, 'hist_%s_%s' % (part, round_id), dark)
    except:
        log.error('error in graph_histogram_regression with dataset_id %s' % dataset.dataset_id) def graph_histogram_classification(dataset, round_id, y, part='eval'):
    """
    generate the histogram of predictions

    :param dataset: dataset object
    :param round_id: id of the round (model)
    :param y: prediction values
    :param part: set (eval / train set)
    :return: None
    """
    try:
        for dark, theme in [(True, 'dark_background'), (False, 'seaborn-whitegrid')]:
            with plt.style.context(theme, after_reset=True):
                plt.figure(figsize=(6, 6))
                for i, name in enumerate(dataset.y_class_names):
                    sns.distplot(y[:, i], hist=False, label=name)
                plt.title('histogram of probabilities (%s set)' % part)
                plt.xlabel('values')
                plt.ylabel('frequencies')
                plt.legend()
                __save_fig(dataset.dataset_id, 'hist_%s_%s' % (part, round_id), dark)
    except:
        log.error('error in graph_histogram_classification with dataset_id %s' % dataset.dataset_id) def graph_predict_regression(dataset, round_id, y, y_pred, part='eval'):
    """
    generate a graph prediction versus actuals

    :param dataset: dataset object
    :param round_id: id of the round
    :param y: actual values
    :param y_pred: predicted values
    :param part: part of the dataset
    :return: None
    """
    try:
        for dark, theme in [(True, 'dark_background'), (False, 'seaborn-whitegrid')]:
            with plt.style.context(theme, after_reset=True):
                plt.figure(figsize=(6, 6))
                # plot a graph prediction versus actuals
                df = pd.DataFrame([y, y_pred]).transpose()
                df.columns = ['actuals', 'predict']
                g = sns.jointplot(x='actuals', y='predict', data=df, kind="kde", size=6)
                g.plot_joint(plt.scatter, s=5, alpha=0.8)
                g.ax_joint.collections[0].set_alpha(0)
                mn, mx = __standard_range(y, 1, 99)
                plt.plot((mn, mx), (mn, mx), color='r', lw=0.7)
                plt.xlim(mn, mx)
                plt.ylim(mn, mx)
                plt.title('%s' % part)
                __save_fig(dataset.dataset_id, 'predict_%s_%s' % (part, round_id), dark)
    except:
        log.error('error in graph_predict_regression with dataset_id %s' % dataset.dataset_id) def graph_predict_classification(dataset, round_id, y, y_pred, part='eval'):
    """
    generate a confusion matrix

    :param dataset: dataset object
    :param round_id: id of the round
    :param y: actual values
    :param y_pred: predicted values
    :param part: part of the dataset
    :return: None
    """
    try:
        for dark, theme in [(True, 'dark_background'), (False, 'seaborn-whitegrid')]:
            with plt.style.context(theme, after_reset=True):
                for normalize in [False, True]:
                    plt.figure(figsize=(8, 6))

                    # convert proba to classes
                    y_pred_class = np.argmax(y_pred, axis=1)

                    # plot a confusion matrix
                    cnf_matrix = confusion_matrix(y, y_pred_class)
                    np.set_printoptions(precision=2)
                    plt.imshow(cnf_matrix, interpolation='nearest', cmap=plt.cm.Blues)
                    plt.colorbar()

                    tick_marks = np.arange(dataset.y_n_classes)
                    plt.xticks(tick_marks, dataset.y_class_names, rotation=45)
                    plt.yticks(tick_marks, dataset.y_class_names)

                    fmt = '.2f' if normalize else 'd'
                    thresh = cnf_matrix.max() / 2.
                    for i, j in itertools.product(range(cnf_matrix.shape[0]), range(cnf_matrix.shape[1])):
                        plt.text(j, i, format(cnf_matrix[i, j], fmt),
                                 horizontalalignment="center",
                                 color="white" if cnf_matrix[i, j] > thresh else "black")

                    plt.tight_layout()
                    plt.ylabel('True label')
                    plt.xlabel('Predicted label')
                    plt.title('confusion matrix (%s set)' % part)
                    name = "predict_norm" if normalize else "predict"
                    __save_fig(dataset.dataset_id, '%s_%s_%s' % (name, part, round_id), dark)

                    # save confusion matrix
                    __save_cnf_matrix(dataset.dataset_id, round_id, part, dataset.y_class_names, cnf_matrix)

    except:
        log.error('error in graph_predict_classification with dataset_id %s' % dataset.dataset_id) def get_eval_sets(dataset_id):
    """
    get the complete eval set

    :param dataset_id: id of the dataset
    :return: eval set (XySet object)
    """
    return pickle.load(open(get_dataset_folder(dataset_id) + '/data/eval_set.pkl', 'rb')) def evaluate_metric(y_act, y_pred, metric_name, n_classes):
    # evaluate score with the metric of the dataset
    metric = metric_map[metric_name]
    try:
        if metric.need_class:
            # convert proba to classes
            y_pred_metric = np.argmax(y_pred, axis=1)
        else:
            if metric.binary:
                y_pred_metric = y_pred[:, 1]
            else:
                y_pred_metric = y_pred

        # use sign before metrics to always compare best is min in comparisons
        # but requires to display abs value for display
        if metric.best_is_min:
            if metric.name == 'log_loss':
                return metric.function(y_act, y_pred_metric, labels=list(range(n_classes)))
            else:
                return metric.function(y_act, y_pred_metric)
        else:
            if metric.average and n_classes > 2:
                # need average if multi-class
                return -metric.function(y_act, y_pred_metric, average='micro')
            else:
                return -metric.function(y_act, y_pred_metric)
    except Exception as e:
        log.error('error in evaluating metric %s: %s' % (metric_name, e))
        return METRIC_NULL def apply_specific_metrics(dataset_id):
    """
    executes the specific_metrics definition

    :param dataset_id: dataset id
    :return:
    """
    global specific_metrics
    name, best_is_min, content = get_specific_metrics(dataset_id)
    exec_specific_metrics(content) def return_specific_metrics(y_act, y_pred):
    """
    return the value of the metrics

    :param dataset_id: dataset id
    :return:
    """
    global specific_metrics
    return specific_metrics(y_act, y_pred) PP_CATEGORIES = ['missing', 'categorical', 'float', 'date', 'text', 'scaling', 'feature']
class PpSolution(object):
    def __init__(self, ref, name, process, default_params, space_params, pp_type, selectable=True, limit_size=1e32,
                 problem_type='*'): def get_pred_eval_test(dataset_id, round_id):
    """
    prediction on eval set & test & submit set

    :param dataset_id: id of the dataset
    :param round_id: id of the round
    :return: list of predictions for eval set, test and submit set
    """
    return pickle.load(open(get_dataset_folder(dataset_id) + '/predict/%s.pkl' % round_id, 'rb')) class XySet(object):
    def __init__(self, X, y, X_train, y_train, X_test, y_test, X_submit,
                 id_submit, cv_folds, y_eval_list, y_eval, i_eval, i_eval0):
        """
        this class stores the various data required for analysis and hyepr-optimisation

        :param X: X features for the complete dataset (excluding bechnmark/test and submit)
        :param y: y for this complete set
        :param X_train: X features for the train set after holdout
        :param y_train: y for this set
        :param X_test: X features for the holdout set or test set in benchmark
        :param y_test: y for this set
        :param X_submit: X features for the submit set (competition mode only)
        :param id_submit: y for this set
        :param cv_folds: cross validation folds
        :param y_eval_list: list of folds, with indexes for each fold
        :param y_eval: y for the eval set
        :param i_eval: indexes for the eval set
        :param i_eval: indexes for the 1st round of the eval set
        """
        self.X = X
        self.y = y
        self.X_train = X_train
        self.y_train = y_train
        self.X_test = X_test
        self.y_test = y_test
        self.X_submit = X_submit
        self.id_submit = id_submit
        self.cv_folds = cv_folds
        self.y_eval_list = y_eval_list
        self.y_eval = y_eval
        self.i_eval = i_eval
        self.i_eval0 = i_eval0
import eli5
import threading
import _thread
import sys
import os
from copy import deepcopy
from .config import *
from .dataset import get_dataset, get_dataset_status
from .graphs import graph_histogram_regression, graph_histogram_classification, graph_predict_regression, \
    graph_predict_classification
from .prepare import get_eval_sets
from .solutions import *
from .monitor import *
from .metrics import evaluate_metric
from .specific import apply_specific_metrics, return_specific_metrics
from .solutions_pp import pp_solutions_map
from .results import get_pred_eval_test
from .xyset import XySet
from sklearn.pipeline import make_pipeline
        model.fit(X1, y1)
        y_pred = __predict(solution, model, X2)

        if threshold != 0:
            # test outlier:
            score = __evaluate_metric(dataset, y2, y_pred)
            if score > threshold:
                log.info('%dth round found outlier: %.5f with threshold %.5f' % (i, score, threshold))
                return True, 0, 0, 0, ds

        y_pred_eval.append(y_pred)

        # we also predict on test & submit set (to be averaged later)
        y_pred_test.append(__predict(solution, model, ds.X_test))

        if not cv:
            # we stop at the first fold
            y_pred_test = y_pred_test[0]
            if dataset.mode == 'competition':
                y_pred_submit = __predict(solution, model, ds.X_submit)
            # update y_train on fold, in order to compute metrics and graphs
            ds.y_train = y2
            return False, y_pred_eval, y_pred_test, y_pred_submit, ds

    if dataset.mode == 'standard':
        # train on complete train set
        model.fit(ds.X_train, ds.y_train)
        y_pred_test = __predict(solution, model, ds.X_test)
    else:
        # train on complete X y set
        model.fit(ds.X, ds.y)
        if dataset.mode == 'competition':
            y_pred_submit = __predict(solution, model, ds.X_submit)
            # test = mean of y_pred_test on multiple folds
            y_pred_test = np.mean(y_pred_test, axis=0)
        else:
            y_pred_test = __predict(solution, model, ds.X_test)

    return False, y_pred_eval, y_pred_test, y_pred_submit, ds


def __create_stacking(dataset, pool, ds):
    # create X by stacking predictions
    for j, (u, m, p_eval, p_test, p_submit) in enumerate(
            zip(pool.pool_model_round_ids, pool.pool_model_names, pool.pool_eval_preds,
                pool.pool_test_preds, pool.pool_submit_preds)):
        # check if array has 2 dimensions
        shape = len(np.shape(p_eval))
        if shape == 1:
            p_eval = np.reshape(p_eval, (len(p_eval), 1))
            p_test = np.reshape(p_test, (len(p_test), 1))
            if dataset.mode == 'competition':
                p_submit = np.reshape(p_submit, (len(p_submit), 1))
        if j == 0:
            ds.X_train = p_eval
            ds.X_test = p_test
            if dataset.mode == 'competition':
                ds.X_submit = p_submit
        else:
            # stack vertically the predictions
            ds.X_train = np.concatenate((ds.X_train, p_eval), axis=1)
            ds.X_test = np.concatenate((ds.X_test, p_test), axis=1)
            if dataset.mode == 'competition':
                ds.X_submit = np.concatenate((ds.X_submit, p_submit), axis=1)

    # then convert to dataframes
    ds.X_train, ds.X_test, ds.X_submit = pd.DataFrame(ds.X_train), pd.DataFrame(ds.X_test), pd.DataFrame(ds.X_submit)

    # update feature names
    feature_names = __get_pool_features(dataset, pool)
    ds.X_train.columns = feature_names
    if len(ds.X_test) > 0:
        ds.X_test.columns = feature_names
    if len(ds.X_submit) > 0:
        ds.X_submit.columns = feature_names

    # X and y for fit
    ds.X = pd.concat([ds.X_train, ds.X_test])
    ds.y = np.concatenate((ds.y_train, ds.y_test))

    return ds


def __create_eval_submit(dataset, ds, round_id, y_pred_eval_list, y_pred_submit, cv):
    # create eval and submit results from lists

    if cv:
        # y_pred_eval as concat of folds
        y_pred_eval = np.concatenate(y_pred_eval_list)
        # reindex eval to be aligned with y
        y_pred_eval[ds.i_eval] = y_pred_eval.copy()
    else:
        y_pred_eval = y_pred_eval_list[0]

    # generate submit file
    if dataset.filename_submit != '':
        ls = len(ds.id_submit)
        # if dataset.problem_type == 'regression':
        if np.shape(y_pred_submit)[1] == 1:
            submit = np.concatenate((np.reshape(ds.id_submit, (ls, 1)), np.reshape(y_pred_submit, (ls, 1))), axis=1)
        else:
            submit = np.concatenate((np.reshape(ds.id_submit, (ls, 1)), np.reshape(y_pred_submit[:, 1], (ls, 1))),
                                    axis=1)
        # create submission file

        df_submit = pd.DataFrame(submit)
        df_submit.columns = [dataset.col_submit, dataset.y_col]
        # allocate id column to avoid type conversion (to float)
        df_submit[dataset.col_submit] = np.reshape(ds.id_submit, (ls, 1))
        df_submit.to_csv(get_dataset_folder(dataset.dataset_id) + '/submit/submit_%s.csv' % round_id, index=False)

    return y_pred_eval


def __create_graphs(dataset, round_id, ds, y_pred_eval, y_pred_test):
    # generate graphs
    if dataset.problem_type == 'regression':
        graph_predict_regression(dataset, round_id, ds.y_train, y_pred_eval, 'eval')
        graph_predict_regression(dataset, round_id, ds.y_test, y_pred_test, 'test')
        graph_histogram_regression(dataset, round_id, y_pred_eval, 'eval')

        graph_histogram_regression(dataset, round_id, y_pred_test, 'test')