"""
Contains the derived saveable search from GridSearchCV
"""
import numpy as np
import pickle
import os.path
import six
import time
import numbers
import warnings
from collections import defaultdict
from functools import partial
from joblib import Parallel, delayed, logger
from itertools import product
from scipy.stats import rankdata

from sklearn.model_selection import GridSearchCV
from sklearn.model_selection._split import check_cv
from sklearn.base import is_classifier, clone
from sklearn.metrics.scorer import _check_multimetric_scoring
from sklearn.utils.validation import indexable, _num_samples
from sklearn.model_selection._validation import _aggregate_score_dicts, _score
from sklearn.utils.deprecation import DeprecationDict
from sklearn.utils.metaestimators import _safe_split
from sklearn.exceptions import FitFailedWarning
from sklearn.model_selection import StratifiedKFold
from sklearn.utils.fixes import MaskedArray

from ..model.basicmodel import BasicModel



def _fit_and_score_save(estimator, X, y, scorer, train, test, verbose,
                   parameters, fold, save, fit_params, return_train_score=False,
                   return_parameters=False, return_n_test_samples=False,
                   return_times=False, error_score='raise'):
    """Fit estimator and compute scores for a given dataset split.
    Parameters
    ----------
    estimator : estimator object implementing 'fit'
        The object to use to fit the data.
    X : array-like of shape at least 2D
        The data to fit.
    y : array-like, optional, default: None
        The target variable to try to predict in the case of
        supervised learning.
    scorer : A single callable or dict mapping scorer name to the callable
        If it is a single callable, the return value for ``train_scores`` and
        ``test_scores`` is a single float.
        For a dict, it should be one mapping the scorer name to the scorer
        callable object / function.
        The callable object / fn should have signature
        ``scorer(estimator, X, y)``.
    train : array-like, shape (n_train_samples,)
        Indices of training samples.
    test : array-like, shape (n_test_samples,)
        Indices of test samples.
    verbose : integer
        The verbosity level.
    error_score : 'raise' (default) or numeric
        Value to assign to the score if an error occurs in estimator fitting.
        If set to 'raise', the error is raised. If a numeric value is given,
        FitFailedWarning is raised. This parameter does not affect the refit
        step, which will always raise the error.
    parameters : dict or None
        Parameters to be set on the estimator.
    fit_params : dict or None
        Parameters that will be passed to ``estimator.fit``.
    return_train_score : boolean, optional, default: False
        Compute and return score on training set.
    return_parameters : boolean, optional, default: False
        Return parameters that has been used for the estimator.
    return_n_test_samples : boolean, optional, default: False
        Whether to return the ``n_test_samples``
    return_times : boolean, optional, default: False
        Whether to return the fit/score times.
    Returns
    -------
    train_scores : dict of scorer name -> float, optional
        Score on training set (for all the scorers),
        returned only if `return_train_score` is `True`.
    test_scores : dict of scorer name -> float, optional
        Score on testing set (for all the scorers).
    n_test_samples : int
        Number of test samples.
    fit_time : float
        Time spent for fitting in seconds.
    score_time : float
        Time spent for scoring in seconds.
    parameters : dict or None, optional
        The parameters that have been evaluated.
    """
    path = None

    if verbose > 1:
        if parameters is None:
            msg = ''
        else:
            msg = '%s' % (', '.join('%s=%s' % (k, v)
                          for k, v in parameters.items()))
        print("[CV] %s %s" % (msg, (64 - len(msg)) * '.'))

    if save is not None:
        path = "{}/{}.ret".format(save,fold)
        if os.path.isfile(path):
            if verbose > 1:
                print("[INFO] Already calculated Fold: %d. Load result from %s."
                      %(fold,path))
            with open(path, 'rb') as fp:
                ret = pickle.load(fp)
            print(ret)
            return ret

    # Adjust length of sample weights
    fit_params = fit_params if fit_params is not None else {}
    fit_params = dict([(k, _index_param_value(X, v, train))
                      for k, v in fit_params.items()])

    test_scores = {}
    train_scores = {}
    if parameters is not None:
        estimator.set_params(**parameters)

    start_time = time.time()

    X_train, y_train = _safe_split(estimator, X, y, train)
    X_test, y_test = _safe_split(estimator, X, y, test, train)

    is_multimetric = not callable(scorer)
    n_scorers = len(scorer.keys()) if is_multimetric else 1

    try:
        if y_train is None:
            estimator.fit(X_train, **fit_params)
        else:
            estimator.fit(X_train, y_train, **fit_params)

    except Exception as e:
        # Note fit time as time until error
        fit_time = time.time() - start_time
        score_time = 0.0
        if error_score == 'raise':
            raise
        elif isinstance(error_score, numbers.Number):
            if is_multimetric:
                test_scores = dict(zip(scorer.keys(),
                                   [error_score, ] * n_scorers))
                if return_train_score:
                    train_scores = dict(zip(scorer.keys(),
                                        [error_score, ] * n_scorers))
            else:
                test_scores = error_score
                if return_train_score:
                    train_scores = error_score
            warnings.warn("Classifier fit failed. The score on this train-test"
                          " partition for these parameters will be set to %f. "
                          "Details: \n%r" % (error_score, e), FitFailedWarning)
        else:
            raise ValueError("error_score must be the string 'raise' or a"
                             " numeric value. (Hint: if using 'raise', please"
                             " make sure that it has been spelled correctly.)")

    else:
        fit_time = time.time() - start_time
        # _score will return dict if is_multimetric is True
        test_scores = _score(estimator, X_test, y_test, scorer, is_multimetric)
        score_time = time.time() - start_time - fit_time
        if return_train_score:
            train_scores = _score(estimator, X_train, y_train, scorer,
                                  is_multimetric)

    end_msg = ""
    if verbose > 2:
        if is_multimetric:
            for scorer_name, score in test_scores.items():
                msg += "; %s=%s" % (scorer_name, score)
        else:
            msg += "; score=%s" % test_scores
    if verbose > 1:
        total_time = score_time + fit_time
        end_msg = "%s; total=%s" % (msg, logger.short_format_time(total_time))
        print("[CV] %s %s" % ((64 - len(end_msg)) * '.', end_msg))

    ret = [train_scores, test_scores] if return_train_score else [test_scores]

    if return_n_test_samples:
        ret.append(_num_samples(X_test))
    if return_times:
        ret.extend([fit_time, score_time])
    if return_parameters:
        ret.append(parameters)

    if path is not None:
        with open(path, "wb") as fp:
            pickle.dump(ret, fp)
        if verbose > 1:
            with open("%s/results.csv" % (save), "a") as fp:
                fp.write("%s \n" % (end_msg))
    return ret







class SaveGridSearchCV(GridSearchCV):
    """
    TODO
    """
    def __init__(self, estimator, param_grid, save=None, scoring=None, 
                 fit_params=None, n_jobs=1, iid=True, refit=True, cv=None, 
                 verbose=0, pre_dispatch='2*n_jobs', error_score='raise',
                 random_state=None, return_train_score="warn"):
        _cv = self._process_cv(cv, random_state)
        super(SaveGridSearchCV, self).__init__(
            estimator=estimator, param_grid=param_grid, scoring=scoring, 
            fit_params=fit_params, n_jobs=n_jobs, iid=iid, refit=refit, cv=_cv, 
            verbose=verbose, pre_dispatch=pre_dispatch, error_score=error_score,
            return_train_score=return_train_score)
        self.save = self._check_save(save, estimator)

    def _process_cv(self, cv, random_state):
        if cv is None:
            cv = 3
        if isinstance(cv, int):
            return StratifiedKFold(n_splits=cv, random_state=random_state)           
        else:
            return cv


    def _check_save(self, save, estimator):
        if save is None:
            return None
        else:
            if isinstance(estimator, BasicModel):
                estimator_name = estimator.get_name()
            else:
                estimator_name = estimator.__class__.__name__
            path = "./{}_{}" .format(save,estimator_name)
            try: 
                os.makedirs(path)
            except OSError:
                if not os.path.isdir(path):
                    raise
            return path



    def fit(self, X, y=None, groups=None, **fit_params):
        """Run fit with all sets of parameters.
        Parameters
        ----------
        X : array-like, shape = [n_samples, n_features]
            Training vector, where n_samples is the number of samples and
            n_features is the number of features.
        y : array-like, shape = [n_samples] or [n_samples, n_output], optional
            Target relative to X for classification or regression;
            None for unsupervised learning.
        groups : array-like, with shape (n_samples,), optional
            Group labels for the samples used while splitting the dataset into
            train/test set.
        **fit_params : dict of string -> object
            Parameters passed to the ``fit`` method of the estimator
        """
        if self.fit_params is not None:
            warnings.warn('"fit_params" as a constructor argument was '
                          'deprecated in version 0.19 and will be removed '
                          'in version 0.21. Pass fit parameters to the '
                          '"fit" method instead.', DeprecationWarning)
            if fit_params:
                warnings.warn('Ignoring fit_params passed as a constructor '
                              'argument in favor of keyword arguments to '
                              'the "fit" method.', RuntimeWarning)
            else:
                fit_params = self.fit_params
        estimator = self.estimator

        #NEW!!!
        if isinstance(self.cv, StratifiedKFold):
            self.cv = self.cv.split(X,y)

        cv = check_cv(self.cv, y, classifier=is_classifier(estimator))

        scorers, self.multimetric_ = _check_multimetric_scoring(
            self.estimator, scoring=self.scoring)

        if self.multimetric_:
            if self.refit is not False and (
                    not isinstance(self.refit, six.string_types) or
                    # This will work for both dict / list (tuple)
                    self.refit not in scorers):
                raise ValueError("For multi-metric scoring, the parameter "
                                 "refit must be set to a scorer key "
                                 "to refit an estimator with the best "
                                 "parameter setting on the whole data and "
                                 "make the best_* attributes "
                                 "available for that metric. If this is not "
                                 "needed, refit should be set to False "
                                 "explicitly. %r was passed." % self.refit)
            else:
                refit_metric = self.refit
        else:
            refit_metric = 'score'

        X, y, groups = indexable(X, y, groups)
        n_splits = cv.get_n_splits(X, y, groups)
        # Regenerate parameter iterable for each fit
        candidate_params = list(self._get_param_iterator())
        n_candidates = len(candidate_params)
        if self.verbose > 0:
            print("Fitting {0} folds for each of {1} candidates, totalling"
                  " {2} fits".format(n_splits, n_candidates,
                                     n_candidates * n_splits))

        base_estimator = clone(self.estimator)
        pre_dispatch = self.pre_dispatch

        experiments = list(product(candidate_params,cv.split(X, y, groups)))


        print("START: %s" % (self.estimator.__class__.__name__))

        out = Parallel(
            n_jobs=self.n_jobs, verbose=self.verbose,
            pre_dispatch=pre_dispatch
        )(delayed(_fit_and_score_save)(clone(base_estimator), X, y, scorers, 
                                  experiments[fold][1][0], 
                                  experiments[fold][1][1], self.verbose, 
                                  experiments[fold][0], fold, self.save,
                                  fit_params=fit_params,
                                  return_train_score=self.return_train_score,
                                  return_n_test_samples=True,
                                  return_times=True, return_parameters=False,
                                  error_score=self.error_score)
          for fold in range(len(experiments)))

        print(out)
        print("Len: %d, Len_1: %d" % (len(out), len(out[0])))

        # if one choose to see train score, "out" will contain train score info
        if self.return_train_score:
            (train_score_dicts, test_score_dicts, test_sample_counts, fit_time,
             score_time) = zip(*out)
        else:
            (test_score_dicts, test_sample_counts, fit_time,
             score_time) = zip(*out)

        # test_score_dicts and train_score dicts are lists of dictionaries and
        # we make them into dict of lists
        test_scores = _aggregate_score_dicts(test_score_dicts)
        if self.return_train_score:
            train_scores = _aggregate_score_dicts(train_score_dicts)

        # TODO: replace by a dict in 0.21
        results = (DeprecationDict() if self.return_train_score == 'warn'
                   else {})

        def _store(key_name, array, weights=None, splits=False, rank=False):
            """A small helper to store the scores/times to the cv_results_"""
            # When iterated first by splits, then by parameters
            # We want `array` to have `n_candidates` rows and `n_splits` cols.
            array = np.array(array, dtype=np.float64).reshape(n_candidates,
                                                              n_splits)
            if splits:
                for split_i in range(n_splits):
                    # Uses closure to alter the results
                    results["split%d_%s"
                            % (split_i, key_name)] = array[:, split_i]

            array_means = np.average(array, axis=1, weights=weights)
            results['mean_%s' % key_name] = array_means
            # Weighted std is not directly available in numpy
            array_stds = np.sqrt(np.average((array -
                                             array_means[:, np.newaxis]) ** 2,
                                            axis=1, weights=weights))
            results['std_%s' % key_name] = array_stds

            if rank:
                results["rank_%s" % key_name] = np.asarray(
                    rankdata(-array_means, method='min'), dtype=np.int32)

        _store('fit_time', fit_time)
        _store('score_time', score_time)
        # Use one MaskedArray and mask all the places where the param is not
        # applicable for that candidate. Use defaultdict as each candidate may
        # not contain all the params
        param_results = defaultdict(partial(MaskedArray,
                                            np.empty(n_candidates,),
                                            mask=True,
                                            dtype=object))
        for cand_i, params in enumerate(candidate_params):
            for name, value in params.items():
                # An all masked empty array gets created for the key
                # `"param_%s" % name` at the first occurence of `name`.
                # Setting the value at an index also unmasks that index
                param_results["param_%s" % name][cand_i] = value

        results.update(param_results)
        # Store a list of param dicts at the key 'params'
        results['params'] = candidate_params

        # NOTE test_sample counts (weights) remain the same for all candidates
        test_sample_counts = np.array(test_sample_counts[:n_splits],
                                      dtype=np.int)
        for scorer_name in scorers.keys():
            # Computed the (weighted) mean and std for test scores alone
            _store('test_%s' % scorer_name, test_scores[scorer_name],
                   splits=True, rank=True,
                   weights=test_sample_counts if self.iid else None)
            if self.return_train_score:
                prev_keys = set(results.keys())
                _store('train_%s' % scorer_name, train_scores[scorer_name],
                       splits=True)

                if self.return_train_score == 'warn':
                    for key in set(results.keys()) - prev_keys:
                        message = (
                            'You are accessing a training score ({!r}), '
                            'which will not be available by default '
                            'any more in 0.21. If you need training scores, '
                            'please set return_train_score=True').format(key)
                        # warn on key access
                        results.add_warning(key, message, FutureWarning)

        # For multi-metric evaluation, store the best_index_, best_params_ and
        # best_score_ iff refit is one of the scorer names
        # In single metric evaluation, refit_metric is "score"
        if self.refit or not self.multimetric_:
            self.best_index_ = results["rank_test_%s" % refit_metric].argmin()
            self.best_params_ = candidate_params[self.best_index_]
            self.best_score_ = results["mean_test_%s" % refit_metric][
                self.best_index_]

        if self.refit:
            self.best_estimator_ = clone(base_estimator).set_params(
                **self.best_params_)
            if y is not None:
                self.best_estimator_.fit(X, y, **fit_params)
            else:
                self.best_estimator_.fit(X, **fit_params)

        # Store the only scorer not as a dict for single metric evaluation
        self.scorer_ = scorers if self.multimetric_ else scorers['score']

        self.cv_results_ = results
        self.n_splits_ = n_splits

        return self


"""
Contains the derived saveable search from GridSearchCV
"""
import numpy as np
import pickle
import os.path
import six
import time
import numbers
import warnings
from collections import defaultdict
from functools import partial
from joblib import Parallel, delayed, logger
from itertools import product
from scipy.stats import rankdata

from sklearn.model_selection import GridSearchCV
from sklearn.model_selection._split import check_cv
from sklearn.base import is_classifier, clone
from sklearn.metrics.scorer import _check_multimetric_scoring
from sklearn.utils.validation import indexable, _num_samples
from sklearn.model_selection._validation import _aggregate_score_dicts, _score
from sklearn.utils.deprecation import DeprecationDict
from sklearn.utils.metaestimators import _safe_split
from sklearn.exceptions import FitFailedWarning
from sklearn.model_selection import StratifiedKFold
from sklearn.utils.fixes import MaskedArray

from ..model.basicmodel import BasicModel



def _fit_and_score_save(estimator, X, y, scorer, train, test, verbose,
                   parameters, fold, save, fit_params, return_train_score=False,
                   return_parameters=False, return_n_test_samples=False,
                   return_times=False, error_score='raise'):
    """Fit estimator and compute scores for a given dataset split.
    Parameters
    ----------
    estimator : estimator object implementing 'fit'
        The object to use to fit the data.
    X : array-like of shape at least 2D
        The data to fit.
    y : array-like, optional, default: None
        The target variable to try to predict in the case of
        supervised learning.
    scorer : A single callable or dict mapping scorer name to the callable
        If it is a single callable, the return value for ``train_scores`` and
        ``test_scores`` is a single float.
        For a dict, it should be one mapping the scorer name to the scorer
        callable object / function.
        The callable object / fn should have signature
        ``scorer(estimator, X, y)``.
    train : array-like, shape (n_train_samples,)
        Indices of training samples.
    test : array-like, shape (n_test_samples,)
        Indices of test samples.
    verbose : integer
        The verbosity level.
    error_score : 'raise' (default) or numeric
        Value to assign to the score if an error occurs in estimator fitting.
        If set to 'raise', the error is raised. If a numeric value is given,
        FitFailedWarning is raised. This parameter does not affect the refit
        step, which will always raise the error.
    parameters : dict or None
        Parameters to be set on the estimator.
    fit_params : dict or None
        Parameters that will be passed to ``estimator.fit``.
    return_train_score : boolean, optional, default: False
        Compute and return score on training set.
    return_parameters : boolean, optional, default: False
        Return parameters that has been used for the estimator.
    return_n_test_samples : boolean, optional, default: False
        Whether to return the ``n_test_samples``
    return_times : boolean, optional, default: False
        Whether to return the fit/score times.
    Returns
    -------
    train_scores : dict of scorer name -> float, optional
        Score on training set (for all the scorers),
        returned only if `return_train_score` is `True`.
    test_scores : dict of scorer name -> float, optional
        Score on testing set (for all the scorers).
    n_test_samples : int
        Number of test samples.
    fit_time : float
        Time spent for fitting in seconds.
    score_time : float
        Time spent for scoring in seconds.
    parameters : dict or None, optional
        The parameters that have been evaluated.
    """
    path = None

    if verbose > 1:
        if parameters is None:
            msg = ''
        else:
            msg = '%s' % (', '.join('%s=%s' % (k, v)
                          for k, v in parameters.items()))
        print("[CV] %s %s" % (msg, (64 - len(msg)) * '.'))

    if save is not None:
        path = "{}/{}.ret".format(save,fold)
        if os.path.isfile(path):
            if verbose > 1:
                print("[INFO] Already calculated Fold: %d. Load result from %s."
                      %(fold,path))
            with open(path, 'rb') as fp:
                ret = pickle.load(fp)
            print(ret)
            return ret

    # Adjust length of sample weights
    fit_params = fit_params if fit_params is not None else {}
    fit_params = dict([(k, _index_param_value(X, v, train))
                      for k, v in fit_params.items()])

    test_scores = {}
    train_scores = {}
    if parameters is not None:
        estimator.set_params(**parameters)

    start_time = time.time()

    X_train, y_train = _safe_split(estimator, X, y, train)
    X_test, y_test = _safe_split(estimator, X, y, test, train)

    is_multimetric = not callable(scorer)
    n_scorers = len(scorer.keys()) if is_multimetric else 1

    try:
        if y_train is None:
            estimator.fit(X_train, **fit_params)
        else:
            estimator.fit(X_train, y_train, **fit_params)

    except Exception as e:
        # Note fit time as time until error
        fit_time = time.time() - start_time
        score_time = 0.0
        if error_score == 'raise':
            raise
        elif isinstance(error_score, numbers.Number):
            if is_multimetric:
                test_scores = dict(zip(scorer.keys(),
                                   [error_score, ] * n_scorers))
                if return_train_score:
                    train_scores = dict(zip(scorer.keys(),
                                        [error_score, ] * n_scorers))
            else:
                test_scores = error_score
                if return_train_score:
                    train_scores = error_score
            warnings.warn("Classifier fit failed. The score on this train-test"
                          " partition for these parameters will be set to %f. "
                          "Details: \n%r" % (error_score, e), FitFailedWarning)
        else:
            raise ValueError("error_score must be the string 'raise' or a"
                             " numeric value. (Hint: if using 'raise', please"
                             " make sure that it has been spelled correctly.)")

    else:
        fit_time = time.time() - start_time
        # _score will return dict if is_multimetric is True
        test_scores = _score(estimator, X_test, y_test, scorer, is_multimetric)
        score_time = time.time() - start_time - fit_time
        if return_train_score:
            train_scores = _score(estimator, X_train, y_train, scorer,
                                  is_multimetric)

    end_msg = ""
    if verbose > 2:
        if is_multimetric:
            for scorer_name, score in test_scores.items():
                msg += "; %s=%s" % (scorer_name, score)
        else:
            msg += "; score=%s" % test_scores
    if verbose > 1:
        total_time = score_time + fit_time
        end_msg = "%s; total=%s" % (msg, logger.short_format_time(total_time))
        print("[CV] %s %s" % ((64 - len(end_msg)) * '.', end_msg))

    ret = [train_scores, test_scores] if return_train_score else [test_scores]

    if return_n_test_samples:
        ret.append(_num_samples(X_test))
    if return_times:
        ret.extend([fit_time, score_time])
    if return_parameters:
        ret.append(parameters)

    if path is not None:
        with open(path, "wb") as fp:
            pickle.dump(ret, fp)
        if verbose > 1:
            with open("%s/results.csv" % (save), "a") as fp:
                fp.write("%s \n" % (end_msg))
    return ret







class SaveGridSearchCV(GridSearchCV):
    """
    TODO
    """
    def __init__(self, estimator, param_grid, save=None, scoring=None, 
                 fit_params=None, n_jobs=1, iid=True, refit=True, cv=None, 
                 verbose=0, pre_dispatch='2*n_jobs', error_score='raise',
                 random_state=None, return_train_score="warn"):
        _cv = self._process_cv(cv, random_state)
        super(SaveGridSearchCV, self).__init__(
            estimator=estimator, param_grid=param_grid, scoring=scoring, 
            fit_params=fit_params, n_jobs=n_jobs, iid=iid, refit=refit, cv=_cv, 
            verbose=verbose, pre_dispatch=pre_dispatch, error_score=error_score,
            return_train_score=return_train_score)
        self.save = self._check_save(save, estimator)

    def _process_cv(self, cv, random_state):
        if cv is None:
            cv = 3
        if isinstance(cv, int):
            return StratifiedKFold(n_splits=cv, random_state=random_state)           
        else:
            return cv


    def _check_save(self, save, estimator):
        if save is None:
            return None
        else:
            if isinstance(estimator, BasicModel):
                estimator_name = estimator.get_name()
            else:
                estimator_name = estimator.__class__.__name__
            path = "./{}_{}" .format(save,estimator_name)
            try: 
                os.makedirs(path)
            except OSError:
                if not os.path.isdir(path):
                    raise
            return path



    def fit(self, X, y=None, groups=None, **fit_params):
        """Run fit with all sets of parameters.
        Parameters
        ----------
        X : array-like, shape = [n_samples, n_features]
            Training vector, where n_samples is the number of samples and
            n_features is the number of features.
        y : array-like, shape = [n_samples] or [n_samples, n_output], optional
            Target relative to X for classification or regression;
            None for unsupervised learning.
        groups : array-like, with shape (n_samples,), optional
            Group labels for the samples used while splitting the dataset into
            train/test set.
        **fit_params : dict of string -> object
            Parameters passed to the ``fit`` method of the estimator
        """
        if self.fit_params is not None:
            warnings.warn('"fit_params" as a constructor argument was '
                          'deprecated in version 0.19 and will be removed '
                          'in version 0.21. Pass fit parameters to the '
                          '"fit" method instead.', DeprecationWarning)
            if fit_params:
                warnings.warn('Ignoring fit_params passed as a constructor '
                              'argument in favor of keyword arguments to '
                              'the "fit" method.', RuntimeWarning)
            else:
                fit_params = self.fit_params
        estimator = self.estimator

        #NEW!!!
        if isinstance(self.cv, StratifiedKFold):
            self.cv = self.cv.split(X,y)

        cv = check_cv(self.cv, y, classifier=is_classifier(estimator))

        scorers, self.multimetric_ = _check_multimetric_scoring(
            self.estimator, scoring=self.scoring)

        if self.multimetric_:
            if self.refit is not False and (
                    not isinstance(self.refit, six.string_types) or
                    # This will work for both dict / list (tuple)
                    self.refit not in scorers):
                raise ValueError("For multi-metric scoring, the parameter "
                                 "refit must be set to a scorer key "
                                 "to refit an estimator with the best "
                                 "parameter setting on the whole data and "
                                 "make the best_* attributes "
                                 "available for that metric. If this is not "
                                 "needed, refit should be set to False "
                                 "explicitly. %r was passed." % self.refit)
            else:
                refit_metric = self.refit
        else:
            refit_metric = 'score'

        X, y, groups = indexable(X, y, groups)
        n_splits = cv.get_n_splits(X, y, groups)
        # Regenerate parameter iterable for each fit
        candidate_params = list(self._get_param_iterator())
        n_candidates = len(candidate_params)
        if self.verbose > 0:
            print("Fitting {0} folds for each of {1} candidates, totalling"
                  " {2} fits".format(n_splits, n_candidates,
                                     n_candidates * n_splits))

        base_estimator = clone(self.estimator)
        pre_dispatch = self.pre_dispatch

        experiments = list(product(candidate_params,cv.split(X, y, groups)))


        print("START: %s" % (self.estimator.__class__.__name__))

        out = Parallel(
            n_jobs=self.n_jobs, verbose=self.verbose,
            pre_dispatch=pre_dispatch
        )(delayed(_fit_and_score_save)(clone(base_estimator), X, y, scorers, 
                                  experiments[fold][1][0], 
                                  experiments[fold][1][1], self.verbose, 
                                  experiments[fold][0], fold, self.save,
                                  fit_params=fit_params,
                                  return_train_score=self.return_train_score,
                                  return_n_test_samples=True,
                                  return_times=True, return_parameters=False,
                                  error_score=self.error_score)
          for fold in range(len(experiments)))

        print(out)
        print("Len: %d, Len_1: %d" % (len(out), len(out[0])))

        # if one choose to see train score, "out" will contain train score info
        if self.return_train_score:
            (train_score_dicts, test_score_dicts, test_sample_counts, fit_time,
             score_time) = zip(*out)
        else:
            (test_score_dicts, test_sample_counts, fit_time,
             score_time) = zip(*out)

        # test_score_dicts and train_score dicts are lists of dictionaries and
        # we make them into dict of lists
        test_scores = _aggregate_score_dicts(test_score_dicts)
        if self.return_train_score:
            train_scores = _aggregate_score_dicts(train_score_dicts)

        # TODO: replace by a dict in 0.21
        results = (DeprecationDict() if self.return_train_score == 'warn'
                   else {})

        def _store(key_name, array, weights=None, splits=False, rank=False):
            """A small helper to store the scores/times to the cv_results_"""
            # When iterated first by splits, then by parameters
            # We want `array` to have `n_candidates` rows and `n_splits` cols.
            array = np.array(array, dtype=np.float64).reshape(n_candidates,
                                                              n_splits)
            if splits:
                for split_i in range(n_splits):
                    # Uses closure to alter the results
                    results["split%d_%s"
                            % (split_i, key_name)] = array[:, split_i]

            array_means = np.average(array, axis=1, weights=weights)
            results['mean_%s' % key_name] = array_means
            # Weighted std is not directly available in numpy
            array_stds = np.sqrt(np.average((array -
                                             array_means[:, np.newaxis]) ** 2,
                                            axis=1, weights=weights))
            results['std_%s' % key_name] = array_stds

            if rank:
                results["rank_%s" % key_name] = np.asarray(
                    rankdata(-array_means, method='min'), dtype=np.int32)

        _store('fit_time', fit_time)
        _store('score_time', score_time)
        # Use one MaskedArray and mask all the places where the param is not
        # applicable for that candidate. Use defaultdict as each candidate may
        # not contain all the params
        param_results = defaultdict(partial(MaskedArray,
                                            np.empty(n_candidates,),
                                            mask=True,
                                            dtype=object))
        for cand_i, params in enumerate(candidate_params):
            for name, value in params.items():
                # An all masked empty array gets created for the key
                # `"param_%s" % name` at the first occurence of `name`.
                # Setting the value at an index also unmasks that index
                param_results["param_%s" % name][cand_i] = value

        results.update(param_results)
        # Store a list of param dicts at the key 'params'
        results['params'] = candidate_params

        # NOTE test_sample counts (weights) remain the same for all candidates
        test_sample_counts = np.array(test_sample_counts[:n_splits],
                                      dtype=np.int)
        for scorer_name in scorers.keys():
            # Computed the (weighted) mean and std for test scores alone
            _store('test_%s' % scorer_name, test_scores[scorer_name],
                   splits=True, rank=True,
                   weights=test_sample_counts if self.iid else None)
            if self.return_train_score:
                prev_keys = set(results.keys())
                _store('train_%s' % scorer_name, train_scores[scorer_name],
                       splits=True)

                if self.return_train_score == 'warn':
                    for key in set(results.keys()) - prev_keys:
                        message = (
                            'You are accessing a training score ({!r}), '
                            'which will not be available by default '
                            'any more in 0.21. If you need training scores, '
                            'please set return_train_score=True').format(key)
                        # warn on key access
                        results.add_warning(key, message, FutureWarning)

        # For multi-metric evaluation, store the best_index_, best_params_ and
        # best_score_ iff refit is one of the scorer names
        # In single metric evaluation, refit_metric is "score"
        if self.refit or not self.multimetric_:
            self.best_index_ = results["rank_test_%s" % refit_metric].argmin()
            self.best_params_ = candidate_params[self.best_index_]
            self.best_score_ = results["mean_test_%s" % refit_metric][
                self.best_index_]

        if self.refit:
            self.best_estimator_ = clone(base_estimator).set_params(
                **self.best_params_)
            if y is not None:
                self.best_estimator_.fit(X, y, **fit_params)
            else:
                self.best_estimator_.fit(X, **fit_params)

        # Store the only scorer not as a dict for single metric evaluation
        self.scorer_ = scorers if self.multimetric_ else scorers['score']

        self.cv_results_ = results
        self.n_splits_ = n_splits

        return self


Contains the derived saveable search from GridSearchCV Fit estimator and compute scores for a given dataset split.     Parameters     ----------     estimator : estimator object implementing 'fit'         The object to use to fit the data.     X : array-like of shape at least 2D         The data to fit.     y : array-like, optional, default: None         The target variable to try to predict in the case of         supervised learning.     scorer : A single callable or dict mapping scorer name to the callable         If it is a single callable, the return value for ``train_scores`` and         ``test_scores`` is a single float.         For a dict, it should be one mapping the scorer name to the scorer         callable object / function.         The callable object / fn should have signature         ``scorer(estimator, X, y)``.     train : array-like, shape (n_train_samples,)         Indices of training samples.     test : array-like, shape (n_test_samples,)         Indices of test samples.     verbose : integer         The verbosity level.     error_score : 'raise' (default) or numeric         Value to assign to the score if an error occurs in estimator fitting.         If set to 'raise', the error is raised. If a numeric value is given,         FitFailedWarning is raised. This parameter does not affect the refit         step, which will always raise the error.     parameters : dict or None         Parameters to be set on the estimator.     fit_params : dict or None         Parameters that will be passed to ``estimator.fit``.     return_train_score : boolean, optional, default: False         Compute and return score on training set.     return_parameters : boolean, optional, default: False         Return parameters that has been used for the estimator.     return_n_test_samples : boolean, optional, default: False         Whether to return the ``n_test_samples``     return_times : boolean, optional, default: False         Whether to return the fit/score times.     Returns     -------     train_scores : dict of scorer name -> float, optional         Score on training set (for all the scorers),         returned only if `return_train_score` is `True`.     test_scores : dict of scorer name -> float, optional         Score on testing set (for all the scorers).     n_test_samples : int         Number of test samples.     fit_time : float         Time spent for fitting in seconds.     score_time : float         Time spent for scoring in seconds.     parameters : dict or None, optional         The parameters that have been evaluated. # Adjust length of sample weights # Note fit time as time until error # _score will return dict if is_multimetric is True TODO Run fit with all sets of parameters.         Parameters         ----------         X : array-like, shape = [n_samples, n_features]             Training vector, where n_samples is the number of samples and             n_features is the number of features.         y : array-like, shape = [n_samples] or [n_samples, n_output], optional             Target relative to X for classification or regression;             None for unsupervised learning.         groups : array-like, with shape (n_samples,), optional             Group labels for the samples used while splitting the dataset into             train/test set.         **fit_params : dict of string -> object             Parameters passed to the ``fit`` method of the estimator #NEW!!! # This will work for both dict / list (tuple) # Regenerate parameter iterable for each fit # if one choose to see train score, "out" will contain train score info # test_score_dicts and train_score dicts are lists of dictionaries and # we make them into dict of lists # TODO: replace by a dict in 0.21 A small helper to store the scores/times to the cv_results_ # When iterated first by splits, then by parameters # We want `array` to have `n_candidates` rows and `n_splits` cols. # Uses closure to alter the results # Weighted std is not directly available in numpy # Use one MaskedArray and mask all the places where the param is not # applicable for that candidate. Use defaultdict as each candidate may # not contain all the params # An all masked empty array gets created for the key # `"param_%s" % name` at the first occurence of `name`. # Setting the value at an index also unmasks that index # Store a list of param dicts at the key 'params' # NOTE test_sample counts (weights) remain the same for all candidates # Computed the (weighted) mean and std for test scores alone # warn on key access # For multi-metric evaluation, store the best_index_, best_params_ and # best_score_ iff refit is one of the scorer names # In single metric evaluation, refit_metric is "score" # Store the only scorer not as a dict for single metric evaluation