def parse_key(key):
    """Helper to format keys"""
    key = key.split('.')
    case_est, draw = '.'.join(key[:-1]), key[-1]
    return case_est, draw def set_job(estimators, preprocessing):
    """Set job to run"""
    if estimators is None:
        if preprocessing is None:
            raise ValueError("Need to specify at least one of"
                             "[estimators, preprocessing]")
        else:
            job = 'preprocess'
    elif preprocessing is None:
        job = 'evaluate'
    else:
        job = 'preprocess-evaluate'
    return job def cat(pr_name, est_name, union='.'):
    """Concat preprocess and estimator name if applicable."""
    if not pr_name:
        return est_name
    return union.join([pr_name, est_name]) def check_scorer(scorer):
    """Check that the scorer instance passed behaves as expected."""
    if not type(scorer).__name__ in ['_PredictScorer', '_ProbaScorer']:

        raise ValueError(
            "The passes scorer does not seem to be a valid scorer. Expected "
            "type '_PredictScorer', got '%s'. Use the "
            "mlens.metrics.make_scorer function to construct a valid scorer." %
            type(scorer).__name__) def make_learners(generator, indexer, scorer, error_score, **kwargs):
    """Set up generators for the job to be performed"""
    learners = [
        EvalLearner(
            estimator=clone(est).set_params(**params),
            preprocess=p_name, indexer=indexer,
            name='%s.%s' % (l_name, i) if i is not None else l_name,
            attr='predict', scorer=scorer, error_score=error_score, **kwargs)
        for p_name, l_name, est, i, params in generator]
    return learners def make_tansformers(generator, indexer, **kwargs):
    """Set up generators for the job to be performed"""
    transformers = [
        EvalTransformer(estimator=Pipeline(pipeline, return_y=True),
                        name=preprocess_name, indexer=indexer, **kwargs)
        for preprocess_name, pipeline in generator]
    return transformers def check_instances(instances):
    """Helper to ensure all instances are named.

    Check if ``instances`` is formatted as expected, and if not convert
    formatting or throw traceback error if impossible to anticipate formatting.

    Parameters
    ----------
    instances : iterable
        instance iterable to test.

    Returns
    -------
    formatted : list or dict
        formatted ``instances`` object. Will be formatted as a dict if
        preprocessing cases are detected, otherwise as a list. The dict will
        contain lists identical to those in the single preprocessing case.
        Each list is of the form ``[('name', instance)]`` and no names overlap.

    Raises
    ------
    ValueError:
        Raises error if formatting fails, which is most likely due to wrong
        ordering of tuple entries, or wrong argument in the wrong position.
    """
    is_iterable = isinstance(instances, (list, tuple, dict))
    if not is_iterable:
        instances = [instances]
    if not instances or None in instances:
        return None

    if _assert_format(instances):
        out = instances
    elif isinstance(instances, list):
        out = _format_instances(instances)
    else:
        # We need to check the instance list of each case
        out = {}
        for case, case_list in instances.items():
            out['-'.join(case.lower().split())] = \
                _format_instances(case_list)
    return out class FoldIndex(BaseIndex):

    """Indexer that generates the full size of ``X``.

    K-Fold iterator that generates fold index tuples.

    FoldIndex creates a generator that returns a tuple of stop and start
    positions to be used for numpy array slicing [stop:start]. Note that
    slicing works well for the test set, but for the training set it is
    recommended to concatenate the index for training data that comes before
    the current test set with the index for the training data that comes after.
    This can easily be achieved with::

        for train_tup, test_tup in self.generate():
            train_slice = numpy.hstack([numpy.arange(t0, t1) for t0, t1 in
                                      train_tup])

            xtrain, xtest = X[train_slice], X[test_tup[0]:test_tup[1]]

    Warnings
    --------
    Simple clicing (i.e. ``X[start:stop]`` generally does not work for the
    train set, which often requires concatenating the train index range
    below the current test set, and the train index range above the current
    test set. To build get a training index, use ::

        ``hstack([np.arange(t0, t1) for t0, t1 in train_index_tuples])``.

    See Also
    --------
    :class:`BlendIndex`, :class:`SubsetIndex`

    Parameters
    ----------
    folds : int (default = 2)
        Number of splits to create in each partition. ``folds`` can
        not be 1 if ``n_partition > 1``. Note that if ``folds = 1``,
        both the train and test set will index the full data.

    X : array-like of shape [n_samples,] , optional
        the training set to partition. The training label array is also,
        accepted, as only the first dimension is used. If ``X`` is not
        passed at instantiation, the ``fit`` method must be called before
        ``generate``, or ``X`` must be passed as an argument of
        ``generate``.

    raise_on_exception : bool (default = True)
        whether to warn on suspicious slices or raise an error.

    Examples
    --------

    Creating arrays of folds and checking overlap

    >>> import numpy as np
    >>> from mlens.index import FoldIndex
    >>> X = np.arange(10)
    >>> print("Data set: %r" % X)
    >>> print()
    >>>
    >>> idx = FoldIndex(4, X)
    >>>
    >>> for train, test in idx.generate(as_array=True):
    ...     print('TRAIN IDX: %32r | TEST IDX: %16r' % (train, test))
    >>>
    >>> print()
    >>>
    >>> for train, test in idx.generate(as_array=True):
    ...     print('TRAIN SET: %32r | TEST SET: %16r' % (X[train], X[test]))
    >>>
    >>> for train_idx, test_idx in idx.generate(as_array=True):
    ...     assert not any([i in X[test_idx] for i in X[train_idx]])
    >>>
    >>> print()
    >>>
    >>> print("No overlap between train set and test set.")
    Data set: array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])
    TRAIN IDX:     array([3, 4, 5, 6, 7, 8, 9]) | TEST IDX: array([0, 1, 2])
    TRAIN IDX:     array([0, 1, 2, 6, 7, 8, 9]) | TEST IDX: array([3, 4, 5])
    TRAIN IDX:  array([0, 1, 2, 3, 4, 5, 8, 9]) | TEST IDX:    array([6, 7])
    TRAIN IDX:  array([0, 1, 2, 3, 4, 5, 6, 7]) | TEST IDX:    array([8, 9])
    TRAIN SET:     array([3, 4, 5, 6, 7, 8, 9]) | TEST SET: array([0, 1, 2])
    TRAIN SET:     array([0, 1, 2, 6, 7, 8, 9]) | TEST SET: array([3, 4, 5])
    TRAIN SET:  array([0, 1, 2, 3, 4, 5, 8, 9]) | TEST SET:    array([6, 7])
    TRAIN SET:  array([0, 1, 2, 3, 4, 5, 6, 7]) | TEST SET:    array([8, 9])
    No overlap between train set and test set.

    Passing ``folds = 1`` without raising exception:

    >>> import numpy as np
    >>> from mlens.index import FoldIndex
    >>> X = np.arange(3)
    >>> print("Data set: %r" % X)
    >>> print()
    >>>
    >>> idx = FoldIndex(1, X, raise_on_exception=False)
    >>>
    >>> for train, test in idx.generate(as_array=True):
    ...     print('TRAIN IDX: %10r | TEST IDX: %10r' % (train, test))
    /../mlens/base/indexer.py:167: UserWarning: 'folds' is 1, will return
    full index as both training set and test set.
    warnings.warn("'folds' is 1, will return full index as "
    Data set: array([0, 1, 2])
    TRAIN IDX: array([0, 1, 2]) | TEST IDX: array([0, 1, 2])
    """

    def __init__(self, folds=2, X=None, raise_on_exception=True):
        super(FoldIndex, self).__init__()
        self.folds = folds
        self.raise_on_exception = raise_on_exception

        if X is not None:
            self.fit(X)

    def fit(self, X, y=None, job=None):
        """Method for storing array data.

        Parameters
        ----------
        X : array-like of shape [n_samples, optional]
            array to _collect dimension data from.
        y : None
            for compatibility
        job : None
            for compatibility

        Returns
        -------
        instance :
            indexer with stores sample size data.
        """
        n = X.shape[0]
        check_full_index(n, self.folds, self.raise_on_exception)

        self.n_test_samples = self.n_samples = n
        self.__fitted__ = True
        return self

    def _gen_indices(self):
        """Generate K-Fold iterator."""
        return super(FoldIndex, self)._gen_indices() class ParallelEvaluation(BaseProcessor):

    """Parallel cross-validation engine.

    Minimal parallel processing engine. Similar to :class:`ParallelProcessing`,
    but offers less features, only fits the *callers* indexer, and excepts
    no task output.
    """

    def __init__(self, *args, **kwargs):
        super(ParallelEvaluation, self).__init__(*args, **kwargs)

    def process(self, caller, case, X, y, path=None, **kwargs):
        """Process caller.

        Parameters
        ----------
        caller: iterable
            Iterable for evaluation job.s Expected caller is a
            :class:`Evaluator` instance.

        case: str
            evaluation case to run on the evaluator. One of
            ``'preprocess'`` and ``'evaluate'``.

        X: array-like of shape [n_samples, n_features]
            Input data

        y: array-like of shape [n_samples,], optional.
            targets. Required for fit, should not be passed to predict or
            transform jobs.

        path: str or dict, optional
            Custom estimation cache. Pass a string to force use of persistent
            cache on disk. Pass a ``dict`` for in-memory cache (requires
            ``backend != 'multiprocessing'``.
        """
        self._initialize(
            job='fit', X=X, y=y, path=path, split=False, stack=False)
        check_initialized(self)

        # Use context manager to ensure same parallel job during entire process
        tf = self.job.dir if not isinstance(self.job.dir, list) else None
        with Parallel(n_jobs=self.n_jobs, temp_folder=tf, max_nbytes=None,
                      mmap_mode='w+', verbose=self.verbose,
                      backend=self.backend) as parallel:

            caller.indexer.fit(self.job.predict_in, self.job.targets, self.job.job)
            caller(parallel, self.job.args(**kwargs), case) class BaseBackend(object):

    """Base class for parallel backend

    Implements default backend settings.
    """

    def __init__(self, backend=None, n_jobs=-1, dtype=None,
                 raise_on_exception=True):
        self.n_jobs = n_jobs
        self.dtype = dtype if dtype is not None else config.get_dtype()
        self.backend = backend if backend is not None else config.get_backend()
        self.raise_on_exception = raise_on_exception

    @abstractmethod
    def __iter__(self):
        yield class IndexMixin(object):

    """Indexer mixin

    Mixin for handling indexers.

    .. note::
       To use this mixin the instance inheriting it must set the
        ``indexer`` or ``indexers`` attribute in ``__init__`` (not both).
    """

    @property
    def __indexer__(self):
        """Flag for existence of indexer"""
        return hasattr(self, 'indexer') or hasattr(self, 'indexers')

    def _check_indexer(self, indexer):
        """Check consistent indexer classes"""
        cls = indexer.__class__.__name__.lower()
        if 'index' not in cls:
            ValueError("Passed indexer does not appear to be valid indexer")

        lcls = [idx.__class__.__name__.lower() for idx in self._get_indexers()]
        if lcls:
            if 'blendindex' in lcls and cls != 'blendindex':
                raise ValueError(
                    "Instance has blendindex, but was passed full type")
            elif 'blendindex' not in lcls and cls == 'blendindex':
                raise ValueError(
                    "Instance has full type index, but was passed blendindex")

    def _get_indexers(self):
        """Return list of indexers"""
        if not self.__indexer__:
            raise AttributeError("No indexer or indexers attribute available")
        indexers = [getattr(self, 'indexer', None)]
        if None in indexers:
            indexers = getattr(self, 'indexers', [None])
        return indexers

    def _setup_0_index(self, X, y, job):
        indexers = self._get_indexers()
        for indexer in indexers:
            indexer.fit(X, y, job) def assemble_data(data_list):
    """Build a data dictionary out of a list of entries and data dicts

    Given a list named tuples of dictionaries, :func:`assemble_data`
    returns a nested ordered dictionary with data keys as outer keys and
    tuple names as inner keys. The returned dictionary can be printed in
    tabular format by :func:`assemble_table`.

    .. seealso::
        :class:`Data`, :func:`assemble_table`

    Examples
    --------
    >>> from mlens.metrics import assemble_data, assemble_table
    >>> d = [('row-idx-1.row-idx-2.a.b', {'column-1': 0.1, 'column-2': 0.1})]
    >>> print(assemble_table(assemble_data(d)))
                            column-2-m  column-2-s  column-1-m  column-1-s
    row-idx-1  row-idx-2          0.10        0.00        0.10        0.00
    """
    data = _dict()
    tmp = _dict()

    partitions = _get_partitions(data_list)

    # Collect scores per preprocessing case and estimator(s)
    for name, data_dict in data_list:
        if not data_dict:
            continue

        prefix, name = _split(name, '/', a_s='/')

        # Names are either est.i.j or case.est.i.j
        splitted = name.split('.')
        if partitions:
            name = tuple(splitted[:-1])

            if len(name) == 3:
                name = '%s.%s--%s' % name
            else:
                name = '%s--%s' % name
        else:
            name = '.'.join(splitted[:-2])

        name = '%s%s' % (prefix, name)

        if name not in tmp:
            # Set up data struct for name
            tmp[name] = _dict()
            for k in data_dict.keys():
                tmp[name][k] = list()
                if '%s-m' % k not in data:
                    data['%s-m' % k] = _dict()
                    data['%s-s' % k] = _dict()
                data['%s-m' % k][name] = list()
                data['%s-s' % k][name] = list()

        # collect all data dicts belonging to name
        for k, v in data_dict.items():
            tmp[name][k].append(v)

    # Aggregate to get mean and std
    for name, data_dict in tmp.items():
        for k, v in data_dict.items():
            if not v:
                continue
            try:
                # Purge None values from the main est due to no predict times
                v = [i for i in v if i is not None]
                if v:
                    data['%s-m' % k][name] = np.mean(v)
                    data['%s-s' % k][name] = np.std(v)
            except Exception as exc:
                warnings.warn(
                    "Aggregating data for %s failed. Raw data:\n%r\n"
                    "Details: %r" % (k, v, exc), MetricWarning)

    # Check if there are empty columns
    discard = list()
    for key, data_dict in data.items():
        empty = True
        for val in data_dict.values():
            if val or val == 0:
                empty = False
        if empty:
            discard.append(key)
    for key in discard:
        data.pop(key)
    return data class Data(_dict):

    """Wrapper class around dict to get pretty prints

    :class:`Data` is an ordered dictionary that implements a dedicated
    pretty print method for a nested dictionary. Printing a :class:`Data`
    dictionary provides a human-readable table. The input dictionary is
    expected to have two levels: the first level gives the columns and the
    second level the rows. Rows names are parsed as
    ``[OUTER]/[MIDDLE].[INNER]--[IDX]``, where IDX has to be an integer. All
    entries are optional.

    .. seealso::
        :func:`assemble_data`, :func:`assemble_table`

    Warning
    -------
    :class:`Data` is an internal class that expects a particular functions.
    This class cannot be used as a general drop-in replacement for the standard
    ``dict`` class.

    Examples
    --------
    >>> from mlens.metrics import Data
    >>> d = [('row-idx-1.row-idx-2.0.0', {'column-1': 0.1, 'column-2': 0.1})]
    >>> data = Data(d)
    >>> print(data)
                            column-a  column-b
    row-idx-1  row-idx-2        0.10      0.20
    """

    def __init__(self, data=None, padding=2, decimals=2):
        if isinstance(data, list):
            data = assemble_data(data)
        super(Data, self).__init__(data)
        self.__padding__ = padding
        self.__decimals__ = decimals

    def __repr__(self):
        return assemble_table(self, self.__padding__, self.__decimals__) def _flatten(instances):
    """Flatten iterator"""
    # Flattened version
    if isinstance(instances, list):
        flattened = [(None, name, est) for name, est in sorted(instances)]
    else:
        # Compress dictionary and sort on case_est keys
        vps = [('%s__%s' % (case, est_name), est)
               for case, instance_list in instances.items()
               for est_name, est in instance_list]
        flattened = [(name.split('__')[0], name.split('__')[1], est)
                     for name, est in sorted(vps)]
    return flattened def _check_instances(instances, namespace=None):
    """Helper to ensure all instances are named.

    Check if ``instances`` is formatted as expected, and if not convert
    formatting or throw traceback error if impossible to anticipate formatting.

    Parameters
    ----------
    instances : iterable
        instance iterable to test.

    namespace: list, optional
        list of assigned instance names.

    Returns
    -------
    formatted : list or dict
        formatted ``instances`` object. Will be formatted as a dict if
        preprocessing cases are detected, otherwise as a list. The dict will
        contain lists identical to those in the single preprocessing case.
        Each list is of the form ``[('name', instance)]`` and no names overlap.

    Raises
    ------
    LayerSpecificationError :
        Raises error if formatting fails, which is most likely due to wrong
        ordering of tuple entries, or wrong argument in the wrong position.

    See Also
    --------
        :class:`mlens.ensemble.base.Layer`
    """
    if not namespace:
        namespace = list()

    is_iterable = isinstance(instances, (list, tuple, dict))
    if not is_iterable:
        instances = [instances]
    if not instances or None in instances:
        return None

    if _assert_format(instances, namespace):
        out = instances
    else:
        out = _format_instances(
            instances, namespace=namespace)
    return out def print_time(t0, message='', **kwargs):
    """Utility function for printing time"""
    if len(message) > 0:
        message += ' | '

    m, s = divmod(time() - t0, 60)
    h, m = divmod(m, 60)

    safe_print(message + '%02d:%02d:%02d' % (h, m, s), **kwargs) def safe_print(*objects, **kwargs):
    """Safe print function for backwards compatibility."""
    # Get stream
    file = kwargs.pop('file', sys.stdout)
    if isinstance(file, str):
        file = getattr(sys, file)

    # Get flush
    flush = kwargs.pop('flush', False)

    # Print
    print(*objects, file=file, **kwargs)

    # Need to flush outside print function for python2 compatibility
    if flush:
        file.flush() def assert_correct_format(estimators, preprocessing):
    """Initial check to assert layer can be constructed."""
    if (preprocessing is None) or (not isinstance(preprocessing, dict)):
        if isinstance(estimators, dict):
            # Either no or uniform preprocessing, estimators should be list
            msg = ("Preprocessing is either 'None' or 'list': 'estimators' "
                   "must be of type 'list'. Got %s.")
            raise LayerSpecificationError(msg % type(estimators))
    else:
        # Check that both estimators and preprocessing are dicts
        if not isinstance(preprocessing, dict):
            msg = ("Unexpected format of 'preprocessing'. Needs to be "
                   " of type 'None' or 'list' or 'dict'. Got %s .")
            raise LayerSpecificationError(msg % type(preprocessing))

        if not isinstance(estimators, dict):
            msg = ("Unexpected format of 'estimators'. Needs to be "
                   "'dict' when preprocessing is 'dict'. Got %s.")
            raise LayerSpecificationError(msg % type(estimators))

        # Check that keys overlap
        prep_check = [key in preprocessing for key in estimators]
        est_check = [key in estimators for key in preprocessing]

        if not all(est_check):
            msg = ("Too few estimator cases to match preprocessing cases:\n"
                   "estimator cases:     %r\npreprocessing cases: %r")
            raise LayerSpecificationError(msg % (list(estimators),
                                                 list(preprocessing)))
        if not all(prep_check):
            msg = ("Too few preprocessing cases to match estimators cases:\n"
                   "preprocessing cases: %r\nestimator cases:     %r")
            raise LayerSpecificationError(msg % (list(preprocessing),
                                                 list(estimators))) def delayed(function, check_pickle=True):
    """Decorator used to capture the arguments of a function.

    Pass `check_pickle=False` when:

    - performing a possibly repeated check is too costly and has been done
      already once outside of the call to delayed.

    - when used in conjunction `Parallel(backend='threading')`.

    """
    # Try to pickle the input function, to catch the problems early when
    # using with multiprocessing:
    if check_pickle:
        pickle.dumps(function)

    def delayed_function(*args, **kwargs):
        return function, args, kwargs
    try:
        delayed_function = functools.wraps(function)(delayed_function)
    except AttributeError:
        " functools.wraps fails on some callable objects "
    return delayed_function def clone(estimator, safe=True):
    """Constructs a new estimator with the same parameters.
    Clone does a deep copy of the model in an estimator
    without actually copying attached data. It yields a new estimator
    with the same parameters that has not been fit on any data.
    Parameters
    ----------
    estimator : estimator object, or list, tuple or set of objects
        The estimator or group of estimators to be cloned
    safe : boolean, optional
        If safe is false, clone will fall back to a deep copy on objects
        that are not estimators.
    """
    estimator_type = type(estimator)
    # XXX: not handling dictionaries
    if estimator_type in (list, tuple, set, frozenset):
        return estimator_type([clone(e, safe=safe) for e in estimator])
    elif not hasattr(estimator, 'get_params'):
        if not safe:
            return copy.deepcopy(estimator)
        else:
            raise TypeError("Cannot clone object '%s' (type %s): "
                            "it does not seem to be a scikit-learn estimator "
                            "as it does not implement a 'get_params' methods."
                            % (repr(estimator), type(estimator)))
    klass = estimator.__class__
    new_object_params = estimator.get_params(deep=False)
    for name, param in six.iteritems(new_object_params):
        new_object_params[name] = clone(param, safe=False)
    new_object = klass(**new_object_params)
    params_set = new_object.get_params(deep=False)

    # quick sanity check of the parameters of the clone
    for name in new_object_params:
        param1 = new_object_params[name]
        param2 = params_set[name]
        if param1 is param2:
            # this should always happen
            continue
        if isinstance(param1, np.ndarray):
            # For most ndarrays, we do not test for complete equality
            if not isinstance(param2, type(param1)):
                equality_test = False
            elif (param1.ndim > 0
                    and param1.shape[0] > 0
                    and isinstance(param2, np.ndarray)
                    and param2.ndim > 0
                    and param2.shape[0] > 0):
                equality_test = (
                    param1.shape == param2.shape
                    and param1.dtype == param2.dtype
                    and (_first_and_last_element(param1) ==
                         _first_and_last_element(param2))
                )
            else:
                equality_test = np.all(param1 == param2)
        elif sparse.issparse(param1):
            # For sparse matrices equality doesn't work
            if not sparse.issparse(param2):
                equality_test = False
            elif param1.size == 0 or param2.size == 0:
                equality_test = (
                    param1.__class__ == param2.__class__
                    and param1.size == 0
                    and param2.size == 0
                )
            else:
                equality_test = (
                    param1.__class__ == param2.__class__
                    and (_first_and_last_element(param1) ==
                         _first_and_last_element(param2))
                    and param1.nnz == param2.nnz
                    and param1.shape == param2.shape
                )
        else:
            # fall back on standard equality
            equality_test = param1 == param2
        if equality_test:
            warnings.warn("Estimator %s modifies parameters in __init__."
                          " This behavior is deprecated as of 0.18 and "
                          "support for this behavior will be removed in 0.20."
                          % type(estimator).__name__, DeprecationWarning)
        else:
            raise RuntimeError('Cannot clone object %s, as the constructor '
                               'does not seem to set parameter %s' %
                               (estimator, name))

    return new_object
import warnings
import numpy as np
from ._base_functions import (parse_key, set_job, cat, check_scorer,
                              make_learners, make_tansformers, check_instances)
from ..index import FoldIndex
from ..parallel import ParallelEvaluation
from ..parallel.base import BaseBackend, IndexMixin
from ..metrics import Data, assemble_data
from ..utils.formatting import _flatten, _check_instances
from ..utils import print_time, safe_print, assert_correct_format
from ..externals.joblib import delayed
from ..externals.sklearn.base import clone
    from time import perf_counter as time
    from time import time
    from collections import OrderedDict as _dict

    def __init__(
            self, scorer, cv=2, shuffle=True, random_state=None,
            error_score=None, metrics=None, verbose=False, **kwargs):
        super(Evaluator, self).__init__(**kwargs)

        check_scorer(scorer)
        self.scorer = scorer
        self.scores_ = None

        # TODO: Need to make this accept more than just FoldIndex
        self.cv = cv
        self.indexer = FoldIndex(cv)

        self.shuffle = shuffle
        self.error_score = error_score
        self.metrics = [np.mean, np.std] if metrics is None else metrics
        self.random_state = random_state
        self.verbose = verbose
        self._preprocessing = None
        self._transformers = None
        self._estimators = None
        self._learners = None
        self.n_iter = None
        self.params = None
        self.results = None

    def fit(self, X, y, estimators=None, param_dicts=None,
            n_iter=2, preprocessing=None):
        """Fit

        Fit preprocessing if applicable and evaluate estimators if applicable.
        The method automatically determines whether to only run preprocessing,
        only evaluation (possibly on previously fitted preprocessing), or both.
        Calling ``fit`` will overwrite previously stored data where applicable.

        Parameters
        ----------
        X : array-like, shape=[n_samples, n_features]
            input data to preprocess and create folds from.

        y : array-like, shape=[n_samples, ]
            training labels.

        estimators : list or dict, optional
            set of estimators to use. If no preprocessing is desired or if
            only on preprocessing pipeline should apply to all, pass a list of
            estimators. The list can contain elements of named tuples
            (i.e. ``('my_name', my_est)``).

            If different estimators should be mapped to preprocessing cases,
            a dictionary that maps estimators to each case should
            be passed: ``{'case_a': list_of_est, ...}``.

        param_dicts : dict, optional
            parameter distribution mapping for estimators. Current
            implementation only supports randomized grid search. Passed
            distribution object must have an ``rvs`` method.
            See :mod:`scipy.stats` for details.

            There is quite some flexibility in specifying ``param_dicts``. If
            there is no preprocessing, or if all estimators are fitted on all
            preprocessing cases, the ``param_dict`` should have keys matching
            the names of the estimators. ::

                estimators = [('name', est), est]

                param_dicts = {'name': {'param-1': some_distribution},
                               'est': {'param-1': some_distribution}
                              }

            It is possible to specify different distributions for some or all
            preprocessing cases::

                preprocessing = {'case-1': transformer_list,
                                 'case-2': transformer_list}

                estimators = [('name', est), est]

                param_dicts = {'name':
                                   {'param-1': some_distribution},
                               ('case-1', 'est'):
                                   {'param-1': some_distribution}
                               ('case-2', 'est'):
                                   {'param-1': some_distribution,
                                    'param-2': some_distribution}
                              }

            If estimators are mapped on a per-preprocessing case basis as a
            dictionary, ``param_dict`` must have key entries of the form
            ``(case_name, est_name)``.

        n_iter : int
            number of parameter draws to evaluate.

        preprocessing : dict, optional
            preprocessing cases to consider. Pass a dictionary mapping a
            case name to a preprocessing pipeline. ::

                preprocessing = {'case_name': transformer_list,}

        Returns
        -------
        self : instance
            class instance with stored estimator evaluation results in
            the ``results`` attribute.
        """
        job = set_job(estimators, preprocessing)
        self._initialize(job, estimators, preprocessing, param_dicts, n_iter)
        self._fit(X, y, job)
        self._get_results()
        return self

    def _initialize(self, job, estimators, preprocessing, param_dicts, n_iter):
        """Set up generators for the job to be performed"""
        if preprocessing and isinstance(preprocessing, list):
            preprocessing = {'pr': preprocessing}

        if 'preprocess' in job:
            self._preprocessing = check_instances(preprocessing)

            self._transformers = make_tansformers(