class Probabilities(Estimator):
    """
    Wraps a scikit-learn-like estimator to return probabilities (if
    it supports it)
    """
    def __init__(self, estimator, binary=False):
        """
        binary: If True, predict returns only the probability
            for the positive class. If False, returns probabilities for
            all classes.
        """
        self.binary = binary
        super(Probabilities, self).__init__(estimator)

    def __str__(self):
        return u"Probabilites for %s" % self.base_estimator_

    def predict(self, x):
        probs = self.base_estimator_.predict_proba(x)
        if probs.shape[1] == 2 or self.binary:
            return probs[:,1]
        return probs F = Feature class Map(Feature):
    """
    Applies given function to feature. Feature *cannot*
    be anonymous (ie lambda). Must be defined in top level
    (and thus picklable).
    """
    def __init__(self, feature, function, name=None):
        super(Map, self).__init__(feature)
        self.function = function
        if name is None:
            name = function.__name__
        self.name = name
        self._name = name

    def _apply(self, data, fitted_feature):
        return data.applymap(self.function) class Predictions(TrainedFeature):
    # TODO: update for new context

    def __init__(self, model_def, name=None, external_data=None,
            cv_folds=None):
        """
        If cv-folds is specified, will use k-fold cross-validation to
        provide robust predictions.
        (The predictions returned are those predicted on hold-out sets only.)
        Will not provide overly-optimistic fit like Predictions will, but can
        increase runtime significantly (nested cross-validation).
        Can be int or iteratable of (train, test) indices
        """
        self.cv_folds = cv_folds
        self.model_def = model_def
        self.external_data = external_data
        super(Predictions, self).__init__()
        #TODO
        # if self.external_context is not none:
        #     # dont need to retrain if using external dataset to train
        #     self.trained = false
        if not name:
            name = 'predictions'
        self._name = '%s[%s,%d features]'%(name,
                model_def.estimator.__class__.__name__, len(model_def.features))

    def _train(self, train_data):
        x, y, fitted_model = fit_model(self.model_def, train_data)
        return fitted_model

    def _apply(self, data, fitted_feature):
        fitted_model = fitted_feature.trained_data

        #TODO
        # if self.cv_folds:
        #     if isinstance(self.cv_folds, int):
        #         folds = make_folds(context.train_index, self.cv_folds)
        #     else:
        #         folds = self.cv_folds
        #     preds = []
        #     for train, test in folds:
        #         ctx = context.copy()
        #         ctx.train_index = train
        #         preds.append(self._predict(ctx, test, fit_model=True))
        #     # if there is held-out data, use all of train to predict
        #     # (these predictions use more data, so will be "better",
        #     # not sure if that is problematic...)
        #     remaining = context.data.index - context.train_index
        #     if len(remaining):
        #         preds.append(self._predict(context, remaining))
        #     preds = concat(preds, axis=0)
        # else:
        preds = self._predict(fitted_model, data)
        preds = DataFrame(preds, index=data.index)
        return preds

    def _predict(self, fitted_model, predict_data):
        x_test, y_true = generate_test(self.model_def, predict_data, fitted_model)
        y_preds = self.model_def.estimator.predict(x_test)
        return y_preds

    def make_cross_validated_models(self, data, fitted_feature):
        pass class ModelDefinition(object):
    """
    Defines a specific data analysis model,
    including features, estimator and target metric.
    Can be stored (pickled) and retrieved.
    """
    DEFAULT_PREDICTIONS_NAME = '__predictions'
    params = ['target', 'features', 'estimator', 'column_subset',
              'evaluation_transformation', 'predictions_name', 'evaluation_target',
              'fill_missing', 'discard_incomplete', 'categorical_indicators',]

    def __init__(self,
                 target=None,
                 features=None,
                 estimator=None,
                 evaluation_transformation=None,
                 predictions_name=None,
                 evaluation_target=None,
                 column_subset=None,
                 filters=None,
                 fill_missing=None,
                 discard_incomplete=False,
                 categorical_indicators=False):
        """
        Parameters:
        ___________

        target: `Feature` or string, default None
            `Feature` or basestring specifying the target ("y") variable of
            the analysis.

        features: `Feature`, default None
            An iterable of `Features <Feature>` to be used by the estimator
            in the analysis.

        estimator: estimator (compatible with sklearn estimators), default None
            An estimator instance compatible with sklearn estimator
            conventions: Has fit(x, y) and predict(y) methods. If the object is
            not a ramp Estimator, it will be wrapped to add sensible
            prediction methods.

        predictions_name: string, default None
            A unique string used as a column identifier for model predictions.
            Must be unique among all feature names. Defaults to
            '__predictions'

        evaluation_transformation: `Feature`, default None
            A `Feature` transformation of the special `predictions_name`
            column used to post-process predictions prior to metric scoring.

        evaluation_target: `Feature`, default None
            `Feature`. Used if `target` represents a transformation that is
            NOT the actual target "y" values. Used in conjuction with
            `evaluation_transformation` to allow model training, predictions and scoring to
            operate on different values.
        """
        self.set_attrs(target,
                       features,
                       estimator,
                       evaluation_transformation,
                       predictions_name,
                       evaluation_target,
                       column_subset,
                       filters,
                       fill_missing,
                       discard_incomplete,
                       categorical_indicators)


    def set_attrs(self,
                  target=None,
                  features=None,
                  estimator=None,
                  evaluation_transformation=None,
                  predictions_name=None,
                  evaluation_target=None,
                  column_subset=None,
                  filters=None,
                  fill_missing=None,
                  discard_incomplete=False,
                  categorical_indicators=False):

        if isinstance(target, BaseFeature) or target is None:
            self.target = target
        else:
            self.target = Feature(target)

        # Alternative predictions
        if isinstance(evaluation_transformation, BaseFeature) or evaluation_transformation is None:
            self.evaluation_transformation = evaluation_transformation
        else:
            self.evaluation_transformation = Feature(evaluation_transformation)

        self.predictions_name = predictions_name
        if self.predictions_name is None:
            self.predictions_name = self.DEFAULT_PREDICTIONS_NAME

        if evaluation_target is not None:
            evaluation_target = (evaluation_target if isinstance(evaluation_target, BaseFeature)
                       else Feature(evaluation_target))
        self.evaluation_target = evaluation_target

        if (self.evaluation_target is not None) ^ (self.evaluation_transformation is not None):
            raise ValueError("You must specify both or neither of\
                             evaluation_target and evaluation_transformation")

        # Transformations
        self.filters = filters if filters else []
        if discard_incomplete:
            self.filters.append(filter_incomplete)

        self.fill_missing = fill_missing
        self.categorical_indicators = categorical_indicators
        self.discard_incomplete = discard_incomplete

        # Features
        if features:
            self.features = ([f if isinstance(f, BaseFeature) else Feature(f)
                              for f in features])

            if categorical_indicators:
                self.features = pre_transform_features(self.features,
                                                       AsFactorIndicators,
                                                       only_if_categorical=True)
            if fill_missing is not None:
                self.features = pre_transform_features(self.features,
                                                       FillMissing,
                                                       fill_value=fill_missing)
        else:
            self.features = None

        # Wrap estimator to return probabilities in the case of a classifier
        self.estimator = wrap_sklearn_like_estimator(estimator)

        self.column_subset = column_subset

    def __getstate__(self):
        # shallow copy dict and keep references
        dct = self.__dict__.copy()
        return dct

    def __repr__(self):
        return stable_repr(self)

    def __str__(self):
        if self.features is not None:
            feature_count = len(self.features)
        else:
            feature_count = 0
        return 'estimator: %s\nfeatures: %d [%s ...]\ntarget: %s' % (
            self.estimator,
            feature_count,
            ' '.join([str(f) for f in self.features])[:50],
            self.target
        )

    def describe(self):
        if self.features is not None:
            feature_count = len(self.features)
        else:
            feature_count = 0
        return {
            'estimator': self.estimator,
            'features': "%d [%s ...]" % (feature_count,
                                         ' '.join([str(f) for f in self.features])[:50]),
            'target': self.target,
        }

    @property
    def summary(self):
        """
        Summary of model definition for labeling. Intended to be somewhat
        readable but unique to a given model definition.
        """
        if self.features is not None:
            feature_count = len(self.features)
        else:
            feature_count = 0
        feature_hash = 'feathash:' + str(hash(tuple(self.features)))
        return (str(self.estimator), feature_count, feature_hash, self.target)

    def update(self, dct):
        """Update the configuration with new parameters. Must use same
        kwargs as __init__"""
        d = self.__dict__.copy()
        d.update(dct)
        self.set_attrs(**d) class FittedModel(Storable):
class PackagedModel(Storable):
    def __init__(self,
                 model_def,
                 fitted_features,
                 fitted_target,
                 fitted_estimator):
    def __init__(self, fitted_model, data, data_description=None,
                 built_at=None, result=None, reports=None):
def generate_train(model_def, data, prep_index=None, train_index=None):
def generate_test(model_def, predict_data, fitted_model, compute_actuals=True):
def fit_model(model_def, data, prep_index=None, train_index=None, **fit_args):
def build_fitted_model(*args, **kwargs):
def predict(fitted_model, x_data):
def predict_with_model(fitted_model, data, compute_actuals=False):
def fit_and_predict(model_def, data, prep_index=None, train_index=None):
def cross_validate(model_def, data, folds, repeat=1):
def build_and_package_model(model_def, data, data_description=None, evaluate=False,
                            reporters=None, prep_index=None, train_index=None): def fit_model(model_def, data, prep_index=None, train_index=None, **fit_args):
    #TODO: There should be an option for computing y_train_predictions
    x_train, y_train, fitted_features, fitted_target = generate_train(model_def,
                                                                      data,
                                                                      prep_index,
                                                                      train_index)

    # fit estimator
    if fit_args:
        model_def.estimator.fit(x_train, y_train, **fit_args)
    else:
        model_def.estimator.fit(x_train, y_train)

    fitted_estimator = model_def.estimator

    fitted_model = FittedModel(model_def, fitted_features, fitted_target, fitted_estimator)
    return x_train, y_train, fitted_model def cross_validate(model_def, data, folds, repeat=1):
    """
    """
    results = []

    if isinstance(folds, int):
        folds = make_default_folds(num_folds=folds, data=data)

    for i in range(repeat):
        for fold in folds:
            if len(fold) == 2:
                train_index, test_index = fold
                prep_index = None
            elif len(fold) == 3:
                train_index, test_index, prep_index = fold
            else:
                raise ValueError("Fold is not of right dimension (%d, not 2 or 3)"%len(fold))
            assert len(train_index & test_index) == 0, "train and test overlap!!! %s, %s" % (train_index, test_index)
            x_train, y_train, fitted_model = fit_model(model_def, data, prep_index, train_index)
            test_data = data.loc[test_index]
            x_test, y_test = generate_test(model_def, test_data, fitted_model)
            assert len(x_train.index & x_test.index) == 0, "train and test overlap!!! %s" % (x_train.index & x_test.index)
            y_preds = predict(fitted_model, x_test)
            if model_def.evaluation_target is not None:
                y_test, ff = build_target_safe(model_def.evaluation_target, test_data)
            result = Result(x_train, x_test, y_train, y_test, y_preds, model_def, fitted_model, data)
            results.append(result)

            # for reporter in reporters:
            #     reporter.update(result)
    return results def build_and_package_model(model_def, data, data_description=None, evaluate=False,
                            reporters=None, prep_index=None, train_index=None):
    x_train, y_train, fitted_model = fit_model(model_def, data, prep_index, train_index)
    y_preds = predict(fitted_model, x_train)
    result = None
    if evaluate:
        # only evaluate on train (this seems reasonable)
        result = Result(x_train, x_train, y_train, y_train, y_preds, model_def, fitted_model, data)
        #TODO
        # reports = evaluate(result, reporters)

    # TODO
    reports = []

    packaged_model = PackagedModel(fitted_model,
                                   data,
                                   data_description,
                                   result,
                                   reports)
    return packaged_model def generate_train(model_def, data, prep_index=None, train_index=None):
    # create training set
    data, prep_index, train_index = filter_data_and_indexes(model_def, data, prep_index, train_index)
    x_train, fitted_features = build_featureset_safe(model_def.features, data, prep_index, train_index)
    y_train, fitted_target = build_target_safe(model_def.target, data, prep_index, train_index)
    x_train = x_train.reindex(train_index)
    y_train = y_train.reindex(train_index)
    return x_train, y_train, fitted_features, fitted_target def generate_test(model_def, predict_data, fitted_model, compute_actuals=True):
    # create test set
    predict_data = filter_data(model_def, predict_data)
    x_test = apply_featureset_safe(model_def.features, predict_data, fitted_model.fitted_features)
    if compute_actuals:
        y_test = apply_target_safe(model_def.target, predict_data, fitted_model.fitted_target)
    else:
        y_test = None
    return x_test, y_test def make_data(n):
    data = pd.DataFrame(
               np.random.randn(n, 3),
               columns=['a','b','c'],
               index=range(10, n+10))
    data['const'] = np.zeros(n)
    data['ints'] = range(n)
    data['y'] = data['a'] ** 2
    return data
import os
import sys
import unittest
import numpy as np
import pandas as pd
from pandas import DataFrame, Series, Index
from pandas.util.testing import assert_almost_equal
from ramp.estimators.base import Probabilities
from ramp.features.base import F, Map
from ramp.features.trained import Predictions
from ramp.model_definition import ModelDefinition
from ramp import modeling
from ramp.modeling import (fit_model,
                           cross_validate,
                           build_and_package_model,
                           generate_train,
                           generate_test)
from ramp.tests.test_features import make_data


class DummyCVEstimator(object):
    def __init__(self):
        self.fitx = []
        self.fity = []
        self.predictx = []

    def fit(self, x, y):
        self.fitx.append(x)
        self.fity.append(y)

    def predict(self, x):
        self.predictx.append(x)
        p = np.zeros(len(x))
        return p


class DummyProbEstimator(object):
    def __init__(self, n_clses):
        self.n_clses = n_clses

    def fit(self, x, y):
        pass

    def predict_proba(self, x):
        return np.zeros((len(x), self.n_clses))


class TestBasicModeling(unittest.TestCase):
    def setUp(self):
        self.data = make_data(10)

    def make_model_def_basic(self):
        features = [F(10), F('a')]
        target = F('b')
        estimator = DummyEstimator()

        model_def = ModelDefinition(features=features,
                                    estimator=estimator,
                                    target=target)
        return model_def

    def test_generate_train(self):
        model_def = self.make_model_def_basic()
        train_index = self.data.index[:5]
        x_train, y_train, ff, x = generate_train(model_def, self.data,
                                          train_index=train_index)
        assert_almost_equal(x_train.index, train_index)

    def test_generate_test(self):
        model_def = self.make_model_def_basic()
        x, y, fitted_model = fit_model(model_def, self.data)
        test_index = self.data.index[:5]
        x_test, y_test = generate_test(model_def, self.data.loc[test_index],
                                       fitted_model)
        assert_almost_equal(x_test.index, test_index)

    def test_fit_model(self):
        model_def = self.make_model_def_basic()
        x, y, fitted_model = fit_model(model_def, self.data)
        fe = fitted_model.fitted_estimator
        self.assertEqual(fe.fitx.shape, x.shape)
        self.assertEqual(fe.fity.shape, y.shape)

    def test_predict(self):
        model_def = self.make_model_def_basic()
        x, y, fitted_model = fit_model(model_def, self.data)
        x, y_true = modeling.generate_test(model_def, self.data[:3], fitted_model)
        y_preds = fitted_model.fitted_estimator.predict(x)
        self.assertEqual(len(x), 3)
        self.assertEqual(len(y_true), 3)
        self.assertEqual(len(y_preds), 3)
        y_preds2 = modeling.predict_with_model(fitted_model, self.data[:3])
        assert_almost_equal(y_preds, y_preds2.values)

    def test_cross_validate(self):
        model_def = self.make_model_def_basic()
        results = cross_validate(model_def, self.data, folds=3)
        self.assertEqual(len(results), 3)

    def test_build_and_package_model(self):
        model_def = self.make_model_def_basic()
        desc =  "State-of-the-Art Model"
        pkg = build_and_package_model(model_def, self.data, desc,
                                      train_index=self.data.index[:3])
        self.assertEqual(pkg.data_description, desc)
        self.assertTrue(pkg.fitted_model)

        # and evaluate
        pkg = build_and_package_model(model_def, self.data, desc, evaluate=True,
                                      train_index=self.data.index[:3])
        self.assertEqual(pkg.data_description, desc)
        self.assertTrue(pkg.fitted_model)

    def test_cross_validate_with_alternative_predictions(self):
        features = [F(10), F('a')]
        target = Map('b', np.log)
        estimator = DummyEstimator()

        model_def = ModelDefinition(features=features,
                                    estimator=estimator,
                                    target=target,
                                    evaluation_transformation=Map('__predictions', np.exp),
                                    evaluation_target=F('b'))
        results = cross_validate(model_def, self.data, folds=3)
        self.assertEqual(len(results), 3)
        # assert we've transformed predictions correctly
        yt = results[0].y_test
        assert_almost_equal(yt.values, self.data['b'].reindex(yt.index).values)
        assert_almost_equal(estimator.fity,
                            np.log(self.data['b'].reindex(results[-1].y_train.index)).values)


class TestNestedModeling(unittest.TestCase):
    def setUp(self):
        self.data = make_data(10)

    def test_predictions_nest(self):
        inner_estimator = DummyEstimator()

        inner_model = ModelDefinition(features=[F('a')],