<reponame>rolando-archive/theusual-kaggle-seeclickfix-ensemble
"""
Classes and functions for working with base models and ensembles.
"""
__author__ = 'bgregory'
__email__ = '<EMAIL>'
__date__ = '11-23-2013'

#Internal modules
import utils
#Start logger to record all info, warnings, and errors to Logs/logfile.log
log = utils.start_logging(__name__)
import ml_metrics
import data_io
import features
import train

#External modules
import numpy as np
import pandas as pd
from sklearn import (metrics, cross_validation, linear_model, ensemble, tree, preprocessing, svm, neighbors, gaussian_process, naive_bayes, neural_network, pipeline, lda)


########################################################################################
class Model(object):
    """Base class for all models:  stand-alone independent models, base models,
    and ensemble models.

    Parameters
    ----------
    model_name: string, required
        Descriptive name of model for use in logging and output

    estimator_class: string, required
        SKLearn estimator class for model fitting and training

    features: dictionary, required
        Features of the model.  Key is feature name, value is a dictionary with
        'train' and 'test' arrays.
            Ex.- model.features['foo_feature']['train'] will return an
            array with the values in training set for foo_feature

    target: string, optional (default = global_target from settings file)
        Target variable (column name) for this model

    segment: string, optional (default = none)
        Segment of data for this model to use

    estimator_params: dictionary, optional (default=none, which passes to SKLearn defaults for that estimator)
        Parameters of the estimator class

    postprocess_scalar: float, optional (default=0)
        Scalar to apply to all predictions after model predicting, useful for calibrating predictions

    Attributes
    ----------
    """

    def __init__(self, model_name, target, segment, estimator_class, estimator_params, features, postprocess_scalar):
        self.model_name = model_name
        self.target = target
        self.segment = segment
        self.estimator_class = estimator_class
        self.estimator_set(estimator_class, estimator_params)
        self.features_set(features)
        self.postprocess_scalar = round(np.float32(postprocess_scalar), 4)
    def estimator_set(self, estimator_class, estimator_params):
            self.estimator = eval(estimator_class)()
            for param in estimator_params:
                #Convert any boolean parameters from string to bool
                if estimator_params[param] == 'true':
                    estimator_params[param] =  True
                elif estimator_params[param] == 'false':
                    estimator_params[param] =  False
                #Convert any numerical parameters from the required JSON string type
                elif '.' in estimator_params[param]:
                    try:
                        estimator_params[param] = float(estimator_params[param])
                    except:
                        pass
                else:
                    try:
                        estimator_params[param] = int(estimator_params[param])
                    except:
                        pass
                setattr(self.estimator, param, estimator_params[param])
    def features_set(self, features):
        """Initialize dictionary of features where keys are the feature names and values are an empty
        list for storing the training and testing array/matrix"""
        self.features = dict((feature,['','']) for feature in features)
    def features_create(self,dfTrn,dfTest):
        #Vectorize each text, categorical, or boolean feature into a train and test matrix stored in self.features
        features.vectors(dfTrn, dfTest, self.features)
        #Transform or scale any numerical features and create feature vector
        features.numerical(dfTrn, dfTest, self.features)
    def predict(self,dfTrn,dfTest):
        #Create feature vectors
        self.features_create(dfTrn,dfTest)
        #Make predictions
        mtxTrn, mtxTest, mtxTrnTarget, mtxTestTarget = train.combine_features(self, dfTrn, dfTest)
        train.predict(mtxTrn,mtxTrnTarget.ravel(),mtxTest,dfTest,self)
        #Store predictions in dataframe as class attribute
        self.dfPredictions = dfTest.ix[:,['id',self.target]]

########################################################################################
class EnsembleAvg (object):
    """Loads already calculated predictions from individual models in the form of CSV files, then applies
    average weights to each individual model to create an ensemble model.
    If predictions are for a cross-validation, then true target values can be loaded and the ensemble can be scored
    using given weights or using optimally derived weights.

    Attributes:
    df_models = List containing each individual model's predictions
    id = unique ID for each record
    targets = List containing the target (or targets) for the predictions
    df_true = Pandas DataFrame containing the true values for the predictions, only required if performing CV
    """
    def __init__(self, targets, id):
        self.sub_models = []
        self.sub_models_segment = []
        self.targets = targets
        self.id = id
    def load_models_csv(self,filepath, model_no = None):
        """
        Load predictions from an individual sub model into a dataframe stored in the sub_models list, if no model_no is given
        then load data into next available index.  Valid source is a CSV file.
        """
        try:
            if model_no == None:
                model_no = len(self.sub_models)
                self.sub_models.append(data_io.load_flatfile_to_df(filepath, delimiter=''))
            else:
                self.sub_models[model_no]=data_io.load_flatfile_to_df(filepath, delimiter='')
            utils.info('Model loaded into index %s' % str(model_no))
        except IndexError:
            raise Exception('Model number does not exist. Model number given, %s, is out of index range.' % str(model_no))

    def load_true_df(self,df):
        """
        Load true target values (ground truth) into a dataframe attribute from an in-memory dataframe object.
        """
        if type(df) != pd.core.frame.DataFrame:
            raise Exception('Object passed, %s, is not a Dataframe. Object passed is of type %s' % (df, type(df)))
        elif self.id not in df.columns:
            raise Exception('Dataframe passed, %s, does not contain unique ID field: %s' % (df, self.id))
        elif not all(x in df.columns for x in self.targets):
            raise Exception('Dataframe passed, %s, does not contain all target variables: %s' % (df, self.targets))
        else:
            self.df_true = df.copy()
            utils.info('True value for target variables successfully loaded into self.df_true')
    def load_df_true_segment(self,df):
        """
        For segmented data.
        Load true target values (ground truth) into a dataframe attribute from an in-memory dataframe object.
        """
        if type(df) != pd.core.frame.DataFrame:
            raise Exception('Object passed, %s, is not a Dataframe. Object passed is of type %s' % (df, type(df)))
        elif self.id not in df.columns:
            raise Exception('Dataframe passed, %s, does not contain unique ID field: %s' % (df, self.id))
        elif not all(x in df.columns for x in self.targets):
            raise Exception('Dataframe passed, %s, does not contain all target variables: %s' % (df, self.targets))
        else:
            self.df_true_segment = df.copy()
            utils.info('True value for target variables successfully loaded into self.df_true_segment')
    def sort_dataframes(self,sortcolumn):
        """
        Sort all data frame attributes of class by a given column for ease of comparison.
        """
        try:
            for i in range(len(self.sub_models)):
                self.sub_models[i] = self.sub_models[i].sort(sortcolumn)
            if 'df_true' in dir(self):
                self.df_true = self.df_true.sort(sortcolumn)
            if 'df_true_segment' in dir(self):
                self.df_true_segment = self.df_true_segment.sort(sortcolumn)
        except KeyError:
            raise Exception('Sort failed.  Column %s not found in all dataframes.' % (sortcolumn))
    def transform_targets_log(self):
        """
        Apply natural log transformation to all targets (both predictions and true values)
        """
        for target in self.targets:
            if 'df_true' in dir(self):
                self.df_true[target] = np.log(self.df_true[target] + 1)
            if 'df_true_segment' in dir(self):
                self.df_true_segment[target] = np.log(self.df_true_segment[target] + 1)
            for i in range(len(self.sub_models)):
                self.sub_models[i][target] = np.log(self.sub_models[i][target] + 1)
            for i in range(len(self.sub_models_segment)):
                self.sub_models_segment[i][target] = np.log(self.sub_models_segment[i][target] + 1)
    def transform_targets_exp(self):
        """
        Apply exp transformation (inverse of natural log transformation) to all targets (both predictions and true values)
        """
        for target in self.targets:
            if 'df_true' in dir(self):
                self.df_true[target] = np.exp(self.df_true[target])-1
            if 'df_true_segment' in dir(self):
                self.df_true_segment[target] = np.exp(self.df_true_segment[target])-1
            if 'df_ensemble' in dir(self):
                self.df_ensemble[target] = np.exp(self.df_ensemble[target])-1
            if 'df_ensemble_segment' in dir(self):
                self.df_ensemble_segment[target] = np.exp(self.df_ensemble_segment[target])-1
            for i in range(len(self.sub_models)):
                self.sub_models[i][target] = np.exp(self.sub_models[i][target])-1
            for i in range(len(self.sub_models_segment)):
                self.sub_models_segment[i][target] = np.exp(self.sub_models_segment[i][target]) -1
    def score_rmsle(self,df,df_true):
        """
        Calculate CV score of predictions in given dataframe using RMSLE metric.  Score individually for each target and
        total for targets.  Must have df_true loaded prior to running.
        """
        all_true = []
        all_preds = []
        target_scores = []
        #Transform predictions back to normal space for scoring
        self.transform_targets_exp()
        for target in self.targets:
            all_true.append(df_true[target].tolist())
            all_preds.append(df[target].tolist())
            target_score = ml_metrics.rmsle(df_true[target], df[target])
            target_scores.append(target_score)
            utils.info('RMSLE score for %s: %f' % (target,target_score))
        utils.info('Total RMSLE score: %f' % (ml_metrics.rmsle(all_true, all_preds)))
        #Transform predictions to log space again for averaging
        self.transform_targets_log()
    def create_ensemble(self,sub_model_indexes, weights):
        """
        Create ensemble from the given sub models using average weights.
        Sub_model_indexes is a list of indexes to use for the sub_models list.
        Weights is a list of dictionaries with given averages for each target, its ordering must correspond to
        the order of sub_model_indexes.
        Ex. -   >>>  weights = [{'target1':.5,'target2':.5},{'target1':.25,'target2':.75}]
        """
        if len(sub_model_indexes) != len(weights):
            raise Exception('Ensemble failed. Number of sub models, %d, is not equal to number of weights, %d.') \
                            % (len(sub_model_indexes), len(weights))
        else:
            #Create new data frame ensemble
            self.df_ensemble = self.sub_models[0].copy()
            for target in self.targets:
                self.df_ensemble[target] = 0
                for submodel in sub_model_indexes:
                    for idx in self.df_ensemble.index:
                        self.df_ensemble[target][idx] += self.sub_models[submodel][target] * weights[submodel][target]
    def create_ensemble_segment(self,sub_model_indexes, weights):
        """
        Create ensemble for a certain segment, from the given sub models using average weights.
        Sub_model_indexes is a list of indexes to use for the sub_models list.
        Weights is a list of dictionaries with given averages for each target, its ordering must correspond to
        the order of sub_model_indexes.
        Ex. -   >>>  weights = [{'target1':.5,'target2':.5},{'target1':.25,'target2':.75}]
        """
        if len(sub_model_indexes) != len(weights):
            raise Exception('Ensemble failed. Number of sub models, %d, is not equal to number of weights, %d.') \
                            % (len(sub_model_indexes), len(weights))
        else:
            #Create new data frame ensemble
            self.df_ensemble_segment = self.sub_models_segment[0].copy()
            for target in self.targets:
                self.df_ensemble_segment[target] = 0
                for submodel in sub_model_indexes:
                    self.df_ensemble_segment[target] += self.sub_models_segment[submodel][target] * weights[submodel][target]
    def calc_weights(self,sub_model_indexes, step_size):
        """
        Calculate optimal weights to use in averaged ensemble using the given sub-models and given score metric
        """
        for target in self.targets:
            while diff < 0:
                score

###############################################################################################################
"""

    #---Ensemble Averaging----#
    reload(ensembles);ensemble_CV = ensembles.EnsembleAvg(targets=targets,id='id')
    ensemble_CV.load_models_csv(filepath='Submits/BryanModel-Updated-CV.csv')
    ensemble_CV.load_models_csv(filepath='Submits/ridge_38_cv.csv')
    ensemble_CV.load_models_csv(filepath='Submits/weak_geo_cv.csv')
    #Parse segments
    ensemble_CV.sub_models_segment.append\
            (ensemble_CV.sub_models[0][ensemble_CV.sub_models[0]['Segment'] == 'Richmond'].reset_index())
    ensemble_CV.sub_models_segment.append\
            (ensemble_CV.sub_models[1][ensemble_CV.sub_models[1]['Segment'] == 'Richmond'].reset_index())
    ensemble_CV.sub_models_segment.append\
            (ensemble_CV.sub_models[2][ensemble_CV.sub_models[2]['Segment'] == 'Richmond'].reset_index())
    dfSegTestCV = dfTestCV.merge(ensemble_CV.sub_models_segment[0].ix[:,['id']],on='id',how='inner')
    #set targets
    ensemble_CV.targets=['num_views']
    #Transform CV targets back to normal
    for target in ensemble_CV.targets:
        dfSegTestCV[target]=np.exp(dfSegTestCV[target])-1
    #Load groundtruth values for CV
    ensemble_CV.load_df_true_segment(dfSegTestCV)
    #Sort all dataframes by ID for easy comparison
    ensemble_CV.sort_dataframes('id')
    #Transform predictions to log space for averaging
    ensemble_CV.transform_targets_log()
    #Set weights
    #Remote_API: weights = [{'num_views':.16,'num_votes':.3,'num_comments':.9},{'num_views':.84,'num_votes':.7,'num_comments':.1}]
    #Richmond:   weights = [{'num_views':.7,'num_votes':.45,'num_comments':.7},{'num_views':.3,'num_votes':.55,'num_comments':.3},{'num_views':.4'}]
    #Oakland weights = [{'num_views':.2,'num_votes':.1,'num_comments':.7},{'num_views':.8,'num_votes':.9,'num_comments':.3}]
    weights = [{'num_views':.2,'num_votes':.1,'num_comments':.6},{'num_views':.8,'num_votes':.9,'num_comments':.4}]
    #Create ensemble average
    #ensemble_CV.create_ensemble([0,1],weights)
    ensemble_CV.create_ensemble_segment([0,1,2],weights)
    #Score the ensemble
    #ensemble_CV.score_rmsle(ensemble_CV.sub_models_segment[0], df_true=ensemble_CV.df_true_segment)
    ensemble_CV.score_rmsle(ensemble_CV.df_ensemble_segment, df_true=ensemble_CV.df_true_segment)


    #---Use regressor to find ideal weights for ensemble---#
    for target_label in ensemble_CV.targets:
        clf.fit_intercept=False
        train = np.hstack((ensemble_CV.sub_models_segment[0].ix[:,[target_label]].as_matrix(),
                           ensemble_CV.sub_models_segment[1].ix[:,[target_label]].as_matrix(),
                           ensemble_CV.sub_models_segment[2].ix[:,[target_label]].as_matrix()))
        target = ensemble_CV.df_true_segment.ix[:,[target_label]].as_matrix()
        clf.fit(train,target)
        try:
            for i in range(len(ensemble_CV.sub_models_segment)):
                weights[i][target_label]=clf.coef_[i]
        except:
            for i in range(len(ensemble_CV.sub_models_segment)):
                weights[i][target_label]=clf.coef_[0][i]
        utils.info(clf.coef_)

    #-----------Test Ensemble--------#
    reload(ensembles);ensemble_CV = ensembles.EnsembleAvg(targets=['num_views'],id='id')
    ensemble_test.load_models_csv(filepath='Submits/BryanModel-Updated.csv')
    ensemble_test.load_models_csv(filepath='Submits/ridge_38_test.csv')
    ensemble_test.load_models_csv(filepath='Submits/weak_geo_svr_.75.csv')
    #Parse segments
    ensemble_test.sub_models_segment.append\
            (ensemble_test.sub_models[0][ensemble_CV.sub_models[0]['Segment'] == 'Richmond'].reset_index())
    ensemble_test.sub_models_segment.append\
            (ensemble_test.sub_models[1][ensemble_CV.sub_models[1]['Segment'] == 'Richmond'].reset_index())
    ensemble_test.sub_models_segment.append\
            (ensemble_test.sub_models[2][ensemble_CV.sub_models[2]['Segment'] == 'Richmond'].reset_index())
    dfSegTestCV = dfTestCV.merge(ensemble_CV.sub_models_segment[0].ix[:,['id']],on='id',how='inner')

    #Transform CV targets back to normal
    for target in ensemble_CV.targets:
        dfSegTestCV[target]=np.exp(dfSegTestCV[target])-1
    #Load groundtruth values for CV
    ensemble_CV.load_df_true_segment(dfSegTestCV)
    #Sort all dataframes by ID for easy comparison
    ensemble_CV.sort_dataframes('id')
    #Transform predictions to log space for averaging
    ensemble_CV.transform_targets_log()
    #Set weights
    #Remote_API: weights = [{'num_views':.16,'num_votes':.3,'num_comments':.9},{'num_views':.84,'num_votes':.7,'num_comments':.1}]
    #Richmond:   weights = [{'num_views':.7,'num_votes':.45,'num_comments':.7},{'num_views':.3,'num_votes':.55,'num_comments':.3},{'num_views':.4'}]
    #Oakland weights = [{'num_views':.2,'num_votes':.1,'num_comments':.7},{'num_views':.8,'num_votes':.9,'num_comments':.3}]
    weights = [{'num_views':.2,'num_votes':.1,'num_comments':.6},{'num_views':.8,'num_votes':.9,'num_comments':.4}]
    #Create ensemble average
    #ensemble_CV.create_ensemble([0,1],weights)
    ensemble_CV.create_ensemble_segment([0,1,2],weights)
    #Score the ensemble
    #ensemble_CV.score_rmsle(ensemble_CV.sub_models_segment[0], df_true=ensemble_CV.df_true_segment)
    ensemble_CV.score_rmsle(ensemble_CV.df_ensemble_segment, df_true=ensemble_CV.df_true_segment)



    #Clean off outliers
    #Views
    dfTrn = dfTrn[dfTrn.num_views_orig  < 3]
    #dfTest = dfTest[dfTest.num_views_orig < 3]
"""
"""
Classes and functions for working with base models and ensembles.
"""
__author__ = 'bgregory'
__email__ = '<EMAIL>'
__date__ = '11-23-2013'

#Internal modules
import utils
#Start logger to record all info, warnings, and errors to Logs/logfile.log
log = utils.start_logging(__name__)
import ml_metrics
import data_io
import features
import train

#External modules
import numpy as np
import pandas as pd
from sklearn import (metrics, cross_validation, linear_model, ensemble, tree, preprocessing, svm, neighbors, gaussian_process, naive_bayes, neural_network, pipeline, lda)


########################################################################################
class Model(object):
    """Base class for all models:  stand-alone independent models, base models,
    and ensemble models.

    Parameters
    ----------
    model_name: string, required
        Descriptive name of model for use in logging and output

    estimator_class: string, required
        SKLearn estimator class for model fitting and training

    features: dictionary, required
        Features of the model.  Key is feature name, value is a dictionary with
        'train' and 'test' arrays.
            Ex.- model.features['foo_feature']['train'] will return an
            array with the values in training set for foo_feature

    target: string, optional (default = global_target from settings file)
        Target variable (column name) for this model

    segment: string, optional (default = none)
        Segment of data for this model to use

    estimator_params: dictionary, optional (default=none, which passes to SKLearn defaults for that estimator)
        Parameters of the estimator class

    postprocess_scalar: float, optional (default=0)
        Scalar to apply to all predictions after model predicting, useful for calibrating predictions

    Attributes
    ----------
    """

    def __init__(self, model_name, target, segment, estimator_class, estimator_params, features, postprocess_scalar):
        self.model_name = model_name
        self.target = target
        self.segment = segment
        self.estimator_class = estimator_class
        self.estimator_set(estimator_class, estimator_params)
        self.features_set(features)
        self.postprocess_scalar = round(np.float32(postprocess_scalar), 4)
    def estimator_set(self, estimator_class, estimator_params):
            self.estimator = eval(estimator_class)()
            for param in estimator_params:
                #Convert any boolean parameters from string to bool
                if estimator_params[param] == 'true':
                    estimator_params[param] =  True
                elif estimator_params[param] == 'false':
                    estimator_params[param] =  False
                #Convert any numerical parameters from the required JSON string type
                elif '.' in estimator_params[param]:
                    try:
                        estimator_params[param] = float(estimator_params[param])
                    except:
                        pass
                else:
                    try:
                        estimator_params[param] = int(estimator_params[param])
                    except:
                        pass
                setattr(self.estimator, param, estimator_params[param])
    def features_set(self, features):
        """Initialize dictionary of features where keys are the feature names and values are an empty
        list for storing the training and testing array/matrix"""
        self.features = dict((feature,['','']) for feature in features)
    def features_create(self,dfTrn,dfTest):
        #Vectorize each text, categorical, or boolean feature into a train and test matrix stored in self.features
        features.vectors(dfTrn, dfTest, self.features)
        #Transform or scale any numerical features and create feature vector
        features.numerical(dfTrn, dfTest, self.features)
    def predict(self,dfTrn,dfTest):
        #Create feature vectors
        self.features_create(dfTrn,dfTest)
        #Make predictions
        mtxTrn, mtxTest, mtxTrnTarget, mtxTestTarget = train.combine_features(self, dfTrn, dfTest)
        train.predict(mtxTrn,mtxTrnTarget.ravel(),mtxTest,dfTest,self)
        #Store predictions in dataframe as class attribute
        self.dfPredictions = dfTest.ix[:,['id',self.target]]

########################################################################################
class EnsembleAvg (object):
    """Loads already calculated predictions from individual models in the form of CSV files, then applies
    average weights to each individual model to create an ensemble model.
    If predictions are for a cross-validation, then true target values can be loaded and the ensemble can be scored
    using given weights or using optimally derived weights.

    Attributes:
    df_models = List containing each individual model's predictions
    id = unique ID for each record
    targets = List containing the target (or targets) for the predictions
    df_true = Pandas DataFrame containing the true values for the predictions, only required if performing CV
    """
    def __init__(self, targets, id):
        self.sub_models = []
        self.sub_models_segment = []
        self.targets = targets
        self.id = id
    def load_models_csv(self,filepath, model_no = None):
        """
        Load predictions from an individual sub model into a dataframe stored in the sub_models list, if no model_no is given
        then load data into next available index.  Valid source is a CSV file.
        """
        try:
            if model_no == None:
                model_no = len(self.sub_models)
                self.sub_models.append(data_io.load_flatfile_to_df(filepath, delimiter=''))
            else:
                self.sub_models[model_no]=data_io.load_flatfile_to_df(filepath, delimiter='')
            utils.info('Model loaded into index %s' % str(model_no))
        except IndexError:
            raise Exception('Model number does not exist. Model number given, %s, is out of index range.' % str(model_no))

    def load_true_df(self,df):
        """
        Load true target values (ground truth) into a dataframe attribute from an in-memory dataframe object.
        """
        if type(df) != pd.core.frame.DataFrame:
            raise Exception('Object passed, %s, is not a Dataframe. Object passed is of type %s' % (df, type(df)))
        elif self.id not in df.columns:
            raise Exception('Dataframe passed, %s, does not contain unique ID field: %s' % (df, self.id))
        elif not all(x in df.columns for x in self.targets):
            raise Exception('Dataframe passed, %s, does not contain all target variables: %s' % (df, self.targets))
        else:
            self.df_true = df.copy()
            utils.info('True value for target variables successfully loaded into self.df_true')
    def load_df_true_segment(self,df):
        """
        For segmented data.
        Load true target values (ground truth) into a dataframe attribute from an in-memory dataframe object.
        """
        if type(df) != pd.core.frame.DataFrame:
            raise Exception('Object passed, %s, is not a Dataframe. Object passed is of type %s' % (df, type(df)))
        elif self.id not in df.columns:
            raise Exception('Dataframe passed, %s, does not contain unique ID field: %s' % (df, self.id))
        elif not all(x in df.columns for x in self.targets):
            raise Exception('Dataframe passed, %s, does not contain all target variables: %s' % (df, self.targets))
        else:
            self.df_true_segment = df.copy()
            utils.info('True value for target variables successfully loaded into self.df_true_segment')
    def sort_dataframes(self,sortcolumn):
        """
        Sort all data frame attributes of class by a given column for ease of comparison.
        """
        try:
            for i in range(len(self.sub_models)):
                self.sub_models[i] = self.sub_models[i].sort(sortcolumn)
            if 'df_true' in dir(self):
                self.df_true = self.df_true.sort(sortcolumn)
            if 'df_true_segment' in dir(self):
                self.df_true_segment = self.df_true_segment.sort(sortcolumn)
        except KeyError:
            raise Exception('Sort failed.  Column %s not found in all dataframes.' % (sortcolumn))
    def transform_targets_log(self):
        """
        Apply natural log transformation to all targets (both predictions and true values)
        """
        for target in self.targets:
            if 'df_true' in dir(self):
                self.df_true[target] = np.log(self.df_true[target] + 1)
            if 'df_true_segment' in dir(self):
                self.df_true_segment[target] = np.log(self.df_true_segment[target] + 1)
            for i in range(len(self.sub_models)):
                self.sub_models[i][target] = np.log(self.sub_models[i][target] + 1)
            for i in range(len(self.sub_models_segment)):
                self.sub_models_segment[i][target] = np.log(self.sub_models_segment[i][target] + 1)
    def transform_targets_exp(self):
        """
        Apply exp transformation (inverse of natural log transformation) to all targets (both predictions and true values)
        """
        for target in self.targets:
            if 'df_true' in dir(self):
                self.df_true[target] = np.exp(self.df_true[target])-1
            if 'df_true_segment' in dir(self):
                self.df_true_segment[target] = np.exp(self.df_true_segment[target])-1
            if 'df_ensemble' in dir(self):
                self.df_ensemble[target] = np.exp(self.df_ensemble[target])-1
            if 'df_ensemble_segment' in dir(self):
                self.df_ensemble_segment[target] = np.exp(self.df_ensemble_segment[target])-1
            for i in range(len(self.sub_models)):
                self.sub_models[i][target] = np.exp(self.sub_models[i][target])-1
            for i in range(len(self.sub_models_segment)):
                self.sub_models_segment[i][target] = np.exp(self.sub_models_segment[i][target]) -1
    def score_rmsle(self,df,df_true):
        """
        Calculate CV score of predictions in given dataframe using RMSLE metric.  Score individually for each target and
        total for targets.  Must have df_true loaded prior to running.
        """
        all_true = []
        all_preds = []
        target_scores = []
        #Transform predictions back to normal space for scoring
        self.transform_targets_exp()
        for target in self.targets:
            all_true.append(df_true[target].tolist())
            all_preds.append(df[target].tolist())
            target_score = ml_metrics.rmsle(df_true[target], df[target])
            target_scores.append(target_score)
            utils.info('RMSLE score for %s: %f' % (target,target_score))
        utils.info('Total RMSLE score: %f' % (ml_metrics.rmsle(all_true, all_preds)))
        #Transform predictions to log space again for averaging
        self.transform_targets_log()
    def create_ensemble(self,sub_model_indexes, weights):
        """
        Create ensemble from the given sub models using average weights.
        Sub_model_indexes is a list of indexes to use for the sub_models list.
        Weights is a list of dictionaries with given averages for each target, its ordering must correspond to
        the order of sub_model_indexes.
        Ex. -   >>>  weights = [{'target1':.5,'target2':.5},{'target1':.25,'target2':.75}]
        """
        if len(sub_model_indexes) != len(weights):
            raise Exception('Ensemble failed. Number of sub models, %d, is not equal to number of weights, %d.') \
                            % (len(sub_model_indexes), len(weights))
        else:
            #Create new data frame ensemble
            self.df_ensemble = self.sub_models[0].copy()
            for target in self.targets:
                self.df_ensemble[target] = 0
                for submodel in sub_model_indexes:
                    for idx in self.df_ensemble.index:
                        self.df_ensemble[target][idx] += self.sub_models[submodel][target] * weights[submodel][target]
    def create_ensemble_segment(self,sub_model_indexes, weights):
        """
        Create ensemble for a certain segment, from the given sub models using average weights.
        Sub_model_indexes is a list of indexes to use for the sub_models list.
        Weights is a list of dictionaries with given averages for each target, its ordering must correspond to
        the order of sub_model_indexes.
        Ex. -   >>>  weights = [{'target1':.5,'target2':.5},{'target1':.25,'target2':.75}]
        """
        if len(sub_model_indexes) != len(weights):
            raise Exception('Ensemble failed. Number of sub models, %d, is not equal to number of weights, %d.') \
                            % (len(sub_model_indexes), len(weights))
        else:
            #Create new data frame ensemble
            self.df_ensemble_segment = self.sub_models_segment[0].copy()
            for target in self.targets:
                self.df_ensemble_segment[target] = 0
                for submodel in sub_model_indexes:
                    self.df_ensemble_segment[target] += self.sub_models_segment[submodel][target] * weights[submodel][target]
    def calc_weights(self,sub_model_indexes, step_size):
        """
        Calculate optimal weights to use in averaged ensemble using the given sub-models and given score metric
        """
        for target in self.targets:
            while diff < 0:
                score

###############################################################################################################
"""

    #---Ensemble Averaging----#
    reload(ensembles);ensemble_CV = ensembles.EnsembleAvg(targets=targets,id='id')
    ensemble_CV.load_models_csv(filepath='Submits/BryanModel-Updated-CV.csv')
    ensemble_CV.load_models_csv(filepath='Submits/ridge_38_cv.csv')
    ensemble_CV.load_models_csv(filepath='Submits/weak_geo_cv.csv')
    #Parse segments
    ensemble_CV.sub_models_segment.append\
            (ensemble_CV.sub_models[0][ensemble_CV.sub_models[0]['Segment'] == 'Richmond'].reset_index())
    ensemble_CV.sub_models_segment.append\
            (ensemble_CV.sub_models[1][ensemble_CV.sub_models[1]['Segment'] == 'Richmond'].reset_index())
    ensemble_CV.sub_models_segment.append\
            (ensemble_CV.sub_models[2][ensemble_CV.sub_models[2]['Segment'] == 'Richmond'].reset_index())
    dfSegTestCV = dfTestCV.merge(ensemble_CV.sub_models_segment[0].ix[:,['id']],on='id',how='inner')
    #set targets
    ensemble_CV.targets=['num_views']
    #Transform CV targets back to normal
    for target in ensemble_CV.targets:
        dfSegTestCV[target]=np.exp(dfSegTestCV[target])-1
    #Load groundtruth values for CV
    ensemble_CV.load_df_true_segment(dfSegTestCV)
    #Sort all dataframes by ID for easy comparison
    ensemble_CV.sort_dataframes('id')
    #Transform predictions to log space for averaging
    ensemble_CV.transform_targets_log()
    #Set weights
    #Remote_API: weights = [{'num_views':.16,'num_votes':.3,'num_comments':.9},{'num_views':.84,'num_votes':.7,'num_comments':.1}]
    #Richmond:   weights = [{'num_views':.7,'num_votes':.45,'num_comments':.7},{'num_views':.3,'num_votes':.55,'num_comments':.3},{'num_views':.4'}]
    #Oakland weights = [{'num_views':.2,'num_votes':.1,'num_comments':.7},{'num_views':.8,'num_votes':.9,'num_comments':.3}]
    weights = [{'num_views':.2,'num_votes':.1,'num_comments':.6},{'num_views':.8,'num_votes':.9,'num_comments':.4}]
    #Create ensemble average
    #ensemble_CV.create_ensemble([0,1],weights)
    ensemble_CV.create_ensemble_segment([0,1,2],weights)
    #Score the ensemble
    #ensemble_CV.score_rmsle(ensemble_CV.sub_models_segment[0], df_true=ensemble_CV.df_true_segment)
    ensemble_CV.score_rmsle(ensemble_CV.df_ensemble_segment, df_true=ensemble_CV.df_true_segment)


    #---Use regressor to find ideal weights for ensemble---#
    for target_label in ensemble_CV.targets:
        clf.fit_intercept=False
        train = np.hstack((ensemble_CV.sub_models_segment[0].ix[:,[target_label]].as_matrix(),
                           ensemble_CV.sub_models_segment[1].ix[:,[target_label]].as_matrix(),
                           ensemble_CV.sub_models_segment[2].ix[:,[target_label]].as_matrix()))
        target = ensemble_CV.df_true_segment.ix[:,[target_label]].as_matrix()
        clf.fit(train,target)
        try:
            for i in range(len(ensemble_CV.sub_models_segment)):
                weights[i][target_label]=clf.coef_[i]
        except:
            for i in range(len(ensemble_CV.sub_models_segment)):
                weights[i][target_label]=clf.coef_[0][i]
        utils.info(clf.coef_)

    #-----------Test Ensemble--------#
    reload(ensembles);ensemble_CV = ensembles.EnsembleAvg(targets=['num_views'],id='id')
    ensemble_test.load_models_csv(filepath='Submits/BryanModel-Updated.csv')
    ensemble_test.load_models_csv(filepath='Submits/ridge_38_test.csv')
    ensemble_test.load_models_csv(filepath='Submits/weak_geo_svr_.75.csv')
    #Parse segments
    ensemble_test.sub_models_segment.append\
            (ensemble_test.sub_models[0][ensemble_CV.sub_models[0]['Segment'] == 'Richmond'].reset_index())
    ensemble_test.sub_models_segment.append\
            (ensemble_test.sub_models[1][ensemble_CV.sub_models[1]['Segment'] == 'Richmond'].reset_index())
    ensemble_test.sub_models_segment.append\
            (ensemble_test.sub_models[2][ensemble_CV.sub_models[2]['Segment'] == 'Richmond'].reset_index())
    dfSegTestCV = dfTestCV.merge(ensemble_CV.sub_models_segment[0].ix[:,['id']],on='id',how='inner')

    #Transform CV targets back to normal
    for target in ensemble_CV.targets:
        dfSegTestCV[target]=np.exp(dfSegTestCV[target])-1
    #Load groundtruth values for CV
    ensemble_CV.load_df_true_segment(dfSegTestCV)
    #Sort all dataframes by ID for easy comparison
    ensemble_CV.sort_dataframes('id')
    #Transform predictions to log space for averaging
    ensemble_CV.transform_targets_log()
    #Set weights
    #Remote_API: weights = [{'num_views':.16,'num_votes':.3,'num_comments':.9},{'num_views':.84,'num_votes':.7,'num_comments':.1}]
    #Richmond:   weights = [{'num_views':.7,'num_votes':.45,'num_comments':.7},{'num_views':.3,'num_votes':.55,'num_comments':.3},{'num_views':.4'}]
    #Oakland weights = [{'num_views':.2,'num_votes':.1,'num_comments':.7},{'num_views':.8,'num_votes':.9,'num_comments':.3}]
    weights = [{'num_views':.2,'num_votes':.1,'num_comments':.6},{'num_views':.8,'num_votes':.9,'num_comments':.4}]
    #Create ensemble average
    #ensemble_CV.create_ensemble([0,1],weights)
    ensemble_CV.create_ensemble_segment([0,1,2],weights)
    #Score the ensemble
    #ensemble_CV.score_rmsle(ensemble_CV.sub_models_segment[0], df_true=ensemble_CV.df_true_segment)
    ensemble_CV.score_rmsle(ensemble_CV.df_ensemble_segment, df_true=ensemble_CV.df_true_segment)



    #Clean off outliers
    #Views
    dfTrn = dfTrn[dfTrn.num_views_orig  < 3]
    #dfTest = dfTest[dfTest.num_views_orig < 3]
"""
Classes and functions for working with base models and ensembles. #Internal modules #Start logger to record all info, warnings, and errors to Logs/logfile.log #External modules ######################################################################################## Base class for all models:  stand-alone independent models, base models,
     and ensemble models.
 
     Parameters
     ----------
     model_name: string, required
         Descriptive name of model for use in logging and output
 
     estimator_class: string, required
         SKLearn estimator class for model fitting and training
 
     features: dictionary, required
         Features of the model.  Key is feature name, value is a dictionary with
         'train' and 'test' arrays.
             Ex.- model.features['foo_feature']['train'] will return an
             array with the values in training set for foo_feature
 
     target: string, optional (default = global_target from settings file)
         Target variable (column name) for this model
 
     segment: string, optional (default = none)
         Segment of data for this model to use
 
     estimator_params: dictionary, optional (default=none, which passes to SKLearn defaults for that estimator)
         Parameters of the estimator class
 
     postprocess_scalar: float, optional (default=0)
         Scalar to apply to all predictions after model predicting, useful for calibrating predictions
 
     Attributes
     ---------- #Convert any boolean parameters from string to bool #Convert any numerical parameters from the required JSON string type Initialize dictionary of features where keys are the feature names and values are an empty
         list for storing the training and testing array/matrix #Vectorize each text, categorical, or boolean feature into a train and test matrix stored in self.features #Transform or scale any numerical features and create feature vector #Create feature vectors #Make predictions #Store predictions in dataframe as class attribute ######################################################################################## Loads already calculated predictions from individual models in the form of CSV files, then applies
     average weights to each individual model to create an ensemble model.
     If predictions are for a cross-validation, then true target values can be loaded and the ensemble can be scored
     using given weights or using optimally derived weights.
 
     Attributes:
     df_models = List containing each individual model's predictions
     id = unique ID for each record
     targets = List containing the target (or targets) for the predictions
     df_true = Pandas DataFrame containing the true values for the predictions, only required if performing CV Load predictions from an individual sub model into a dataframe stored in the sub_models list, if no model_no is given
         then load data into next available index.  Valid source is a CSV file. Load true target values (ground truth) into a dataframe attribute from an in-memory dataframe object. For segmented data.
         Load true target values (ground truth) into a dataframe attribute from an in-memory dataframe object. Sort all data frame attributes of class by a given column for ease of comparison. Apply natural log transformation to all targets (both predictions and true values) Apply exp transformation (inverse of natural log transformation) to all targets (both predictions and true values) Calculate CV score of predictions in given dataframe using RMSLE metric.  Score individually for each target and
         total for targets.  Must have df_true loaded prior to running. #Transform predictions back to normal space for scoring #Transform predictions to log space again for averaging Create ensemble from the given sub models using average weights.
         Sub_model_indexes is a list of indexes to use for the sub_models list.
         Weights is a list of dictionaries with given averages for each target, its ordering must correspond to
         the order of sub_model_indexes.
         Ex. -   >>>  weights = [{'target1':.5,'target2':.5},{'target1':.25,'target2':.75}] #Create new data frame ensemble Create ensemble for a certain segment, from the given sub models using average weights.
         Sub_model_indexes is a list of indexes to use for the sub_models list.
         Weights is a list of dictionaries with given averages for each target, its ordering must correspond to
         the order of sub_model_indexes.
         Ex. -   >>>  weights = [{'target1':.5,'target2':.5},{'target1':.25,'target2':.75}] #Create new data frame ensemble Calculate optimal weights to use in averaged ensemble using the given sub-models and given score metric ############################################################################################################### #---Ensemble Averaging----#
     reload(ensembles);ensemble_CV = ensembles.EnsembleAvg(targets=targets,id='id')
     ensemble_CV.load_models_csv(filepath='Submits/BryanModel-Updated-CV.csv')
     ensemble_CV.load_models_csv(filepath='Submits/ridge_38_cv.csv')
     ensemble_CV.load_models_csv(filepath='Submits/weak_geo_cv.csv')
     #Parse segments
     ensemble_CV.sub_models_segment.append\
             (ensemble_CV.sub_models[0][ensemble_CV.sub_models[0]['Segment'] == 'Richmond'].reset_index())
     ensemble_CV.sub_models_segment.append\
             (ensemble_CV.sub_models[1][ensemble_CV.sub_models[1]['Segment'] == 'Richmond'].reset_index())
     ensemble_CV.sub_models_segment.append\
             (ensemble_CV.sub_models[2][ensemble_CV.sub_models[2]['Segment'] == 'Richmond'].reset_index())
     dfSegTestCV = dfTestCV.merge(ensemble_CV.sub_models_segment[0].ix[:,['id']],on='id',how='inner')
     #set targets
     ensemble_CV.targets=['num_views']
     #Transform CV targets back to normal
     for target in ensemble_CV.targets:
         dfSegTestCV[target]=np.exp(dfSegTestCV[target])-1
     #Load groundtruth values for CV
     ensemble_CV.load_df_true_segment(dfSegTestCV)
     #Sort all dataframes by ID for easy comparison
     ensemble_CV.sort_dataframes('id')
     #Transform predictions to log space for averaging
     ensemble_CV.transform_targets_log()
     #Set weights
     #Remote_API: weights = [{'num_views':.16,'num_votes':.3,'num_comments':.9},{'num_views':.84,'num_votes':.7,'num_comments':.1}]
     #Richmond:   weights = [{'num_views':.7,'num_votes':.45,'num_comments':.7},{'num_views':.3,'num_votes':.55,'num_comments':.3},{'num_views':.4'}]
     #Oakland weights = [{'num_views':.2,'num_votes':.1,'num_comments':.7},{'num_views':.8,'num_votes':.9,'num_comments':.3}]
     weights = [{'num_views':.2,'num_votes':.1,'num_comments':.6},{'num_views':.8,'num_votes':.9,'num_comments':.4}]
     #Create ensemble average
     #ensemble_CV.create_ensemble([0,1],weights)
     ensemble_CV.create_ensemble_segment([0,1,2],weights)
     #Score the ensemble
     #ensemble_CV.score_rmsle(ensemble_CV.sub_models_segment[0], df_true=ensemble_CV.df_true_segment)
     ensemble_CV.score_rmsle(ensemble_CV.df_ensemble_segment, df_true=ensemble_CV.df_true_segment)
 
 
     #---Use regressor to find ideal weights for ensemble---#
     for target_label in ensemble_CV.targets:
         clf.fit_intercept=False
         train = np.hstack((ensemble_CV.sub_models_segment[0].ix[:,[target_label]].as_matrix(),
                            ensemble_CV.sub_models_segment[1].ix[:,[target_label]].as_matrix(),
                            ensemble_CV.sub_models_segment[2].ix[:,[target_label]].as_matrix()))
         target = ensemble_CV.df_true_segment.ix[:,[target_label]].as_matrix()
         clf.fit(train,target)
         try:
             for i in range(len(ensemble_CV.sub_models_segment)):
                 weights[i][target_label]=clf.coef_[i]
         except:
             for i in range(len(ensemble_CV.sub_models_segment)):
                 weights[i][target_label]=clf.coef_[0][i]
         utils.info(clf.coef_)
 
     #-----------Test Ensemble--------#
     reload(ensembles);ensemble_CV = ensembles.EnsembleAvg(targets=['num_views'],id='id')
     ensemble_test.load_models_csv(filepath='Submits/BryanModel-Updated.csv')
     ensemble_test.load_models_csv(filepath='Submits/ridge_38_test.csv')
     ensemble_test.load_models_csv(filepath='Submits/weak_geo_svr_.75.csv')
     #Parse segments
     ensemble_test.sub_models_segment.append\
             (ensemble_test.sub_models[0][ensemble_CV.sub_models[0]['Segment'] == 'Richmond'].reset_index())
     ensemble_test.sub_models_segment.append\
             (ensemble_test.sub_models[1][ensemble_CV.sub_models[1]['Segment'] == 'Richmond'].reset_index())
     ensemble_test.sub_models_segment.append\
             (ensemble_test.sub_models[2][ensemble_CV.sub_models[2]['Segment'] == 'Richmond'].reset_index())
     dfSegTestCV = dfTestCV.merge(ensemble_CV.sub_models_segment[0].ix[:,['id']],on='id',how='inner')
 
     #Transform CV targets back to normal
     for target in ensemble_CV.targets:
         dfSegTestCV[target]=np.exp(dfSegTestCV[target])-1
     #Load groundtruth values for CV
     ensemble_CV.load_df_true_segment(dfSegTestCV)
     #Sort all dataframes by ID for easy comparison
     ensemble_CV.sort_dataframes('id')
     #Transform predictions to log space for averaging
     ensemble_CV.transform_targets_log()
     #Set weights
     #Remote_API: weights = [{'num_views':.16,'num_votes':.3,'num_comments':.9},{'num_views':.84,'num_votes':.7,'num_comments':.1}]
     #Richmond:   weights = [{'num_views':.7,'num_votes':.45,'num_comments':.7},{'num_views':.3,'num_votes':.55,'num_comments':.3},{'num_views':.4'}]
     #Oakland weights = [{'num_views':.2,'num_votes':.1,'num_comments':.7},{'num_views':.8,'num_votes':.9,'num_comments':.3}]
     weights = [{'num_views':.2,'num_votes':.1,'num_comments':.6},{'num_views':.8,'num_votes':.9,'num_comments':.4}]
     #Create ensemble average
     #ensemble_CV.create_ensemble([0,1],weights)
     ensemble_CV.create_ensemble_segment([0,1,2],weights)
     #Score the ensemble
     #ensemble_CV.score_rmsle(ensemble_CV.sub_models_segment[0], df_true=ensemble_CV.df_true_segment)
     ensemble_CV.score_rmsle(ensemble_CV.df_ensemble_segment, df_true=ensemble_CV.df_true_segment)
 
 
 
     #Clean off outliers
     #Views
     dfTrn = dfTrn[dfTrn.num_views_orig  < 3]
     #dfTest = dfTest[dfTest.num_views_orig < 3]