<s> """<STR_LIT>""" <EOL> from __future__ import division , print_function , absolute_import <EOL> import numbers <EOL> import warnings <EOL> import numpy <EOL> import pandas <EOL> from scipy import sparse <EOL> from scipy . special import expit <EOL> from collections import defaultdict <EOL> from sklearn . utils . validation import check_random_state <EOL> from sklearn . base import BaseEstimator <EOL> from . commonutils import computeSignalKnnIndices , indices_of_values , check_sample_weight , check_uniform_label <EOL> from . metrics_utils import bin_to_group_indices , compute_group_weights , compute_bin_indices <EOL> __author__ = '<STR_LIT>' <EOL> def compute_positions ( y_pred , sample_weight ) : <EOL> """<STR_LIT>""" <EOL> order = numpy . argsort ( y_pred ) <EOL> ordered_weights = sample_weight [ order ] <EOL> ordered_weights /= float ( numpy . sum ( ordered_weights ) ) <EOL> efficiencies = ( numpy . cumsum ( ordered_weights ) - <NUM_LIT:0.5> * ordered_weights ) <EOL> return efficiencies [ numpy . argsort ( order ) ] <EOL> class AbstractLossFunction ( BaseEstimator ) : <EOL> def fit ( self , X , y , sample_weight ) : <EOL> """<STR_LIT>""" <EOL> pass <EOL> def negative_gradient ( self , y_pred ) : <EOL> """<STR_LIT>""" <EOL> raise NotImplementedError ( ) <EOL> def __call__ ( self , y_pred ) : <EOL> """<STR_LIT>""" <EOL> raise NotImplementedError ( ) <EOL> def update_tree ( self , tree , X , y , y_pred , sample_weight , update_mask , residual ) : <EOL> """<STR_LIT>""" <EOL> terminal_regions = tree . apply ( X ) <EOL> masked_terminal_regions = terminal_regions . copy ( ) <EOL> masked_terminal_regions [ ~ update_mask ] = - <NUM_LIT:1> <EOL> for leaf , indices_in_leaf in indices_of_values ( masked_terminal_regions ) : <EOL> if leaf == - <NUM_LIT:1> : <EOL> continue <EOL> tree . value [ leaf , <NUM_LIT:0> , <NUM_LIT:0> ] = self . update_tree_leaf ( <EOL> leaf = leaf , indices_in_leaf = indices_in_leaf , X = X , y = y , y_pred = y_pred , <EOL> sample_weight = sample_weight , update_mask = update_mask , residual = residual ) <EOL> def update_fast_tree ( self , fast_tree , X , y , y_pred , sample_weight , update_mask , residual ) : <EOL> """<STR_LIT>""" <EOL> terminal_regions , _ = fast_tree . apply ( X ) <EOL> terminal_regions [ ~ update_mask ] = - <NUM_LIT:1> <EOL> for leaf , indices_in_leaf in indices_of_values ( terminal_regions ) : <EOL> if leaf == - <NUM_LIT:1> : <EOL> continue <EOL> new_value = self . update_tree_leaf ( <EOL> leaf = leaf , indices_in_leaf = indices_in_leaf , X = X , y = y , y_pred = y_pred , <EOL> sample_weight = sample_weight , update_mask = update_mask , residual = residual ) <EOL> assert len ( fast_tree . nodes_data [ leaf ] ) == <NUM_LIT:1> <EOL> fast_tree . nodes_data [ leaf ] = ( new_value , ) <EOL> def update_tree_leaf ( self , leaf , indices_in_leaf , <EOL> X , y , y_pred , sample_weight , update_mask , residual ) : <EOL> raise NotImplementedError ( '<STR_LIT>' ) <EOL> class AdaLossFunction ( AbstractLossFunction ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , regularization = <NUM_LIT> ) : <EOL> self . regularization = regularization <EOL> def fit ( self , X , y , sample_weight ) : <EOL> self . y = y <EOL> self . sample_weight = sample_weight <EOL> self . y_signed = <NUM_LIT:2> * y - <NUM_LIT:1> <EOL> def __call__ ( self , y_pred ) : <EOL> return numpy . sum ( self . sample_weight * numpy . exp ( - self . y_signed * y_pred ) ) <EOL> def negative_gradient ( self , y_pred ) : <EOL> return self . y_signed * self . sample_weight * numpy . exp ( - self . y_signed * y_pred ) <EOL> def hessian ( self , y_pred ) : <EOL> return self . sample_weight * numpy . exp ( - self . y_signed * y_pred ) <EOL> def update_tree_leaf ( self , leaf , indices_in_leaf , X , y , y_pred , sample_weight , update_mask , residual ) : <EOL> leaf_ans = y [ indices_in_leaf ] <EOL> leaf_exp = sample_weight [ indices_in_leaf ] * numpy . exp ( <EOL> - self . y_signed [ indices_in_leaf ] * y_pred [ indices_in_leaf ] ) <EOL> w1 = numpy . sum ( leaf_exp [ leaf_ans == <NUM_LIT:1> ] ) <EOL> w2 = numpy . sum ( leaf_exp [ leaf_ans == <NUM_LIT:0> ] ) <EOL> w_reg = ( w1 + w2 ) * self . regularization <EOL> return <NUM_LIT:0.5> * numpy . log ( ( w1 + w_reg ) / ( w2 + w_reg ) ) <EOL> class BinomialDevianceLossFunction ( AbstractLossFunction ) : <EOL> def __init__ ( self , regularization = <NUM_LIT:1.> ) : <EOL> self . regularization = regularization <EOL> def fit ( self , X , y , sample_weight ) : <EOL> self . y = y <EOL> self . sample_weight = sample_weight <EOL> self . y_signed = <NUM_LIT:2> * y - <NUM_LIT:1> <EOL> self . adjusted_regularization = numpy . mean ( sample_weight ) * self . regularization <EOL> def __call__ ( self , y_pred ) : <EOL> return numpy . sum ( self . sample_weight * numpy . logaddexp ( <NUM_LIT:0> , - self . y_signed * y_pred ) ) <EOL> def negative_gradient ( self , y_pred ) : <EOL> return self . y_signed * self . sample_weight * expit ( - self . y_signed * y_pred ) <EOL> def hessian ( self , y_pred ) : <EOL> expits = expit ( self . y_signed * y_pred ) <EOL> return self . sample_weight * expits * ( <NUM_LIT:1> - expits ) <EOL> def update_tree_leaf ( self , leaf , indices_in_leaf , X , y , y_pred , sample_weight , update_mask , residual ) : <EOL> leaf_y = y [ indices_in_leaf ] <EOL> y_signed = <NUM_LIT:2> * leaf_y - <NUM_LIT:1> <EOL> leaf_weights = sample_weight [ indices_in_leaf ] <EOL> residual_abs = expit ( numpy . clip ( - y_signed * y_pred [ indices_in_leaf ] , - <NUM_LIT:10> , <NUM_LIT:10> ) ) <EOL> nominator = numpy . sum ( y_signed * residual_abs * leaf_weights ) <EOL> denominator = numpy . sum ( residual_abs * ( <NUM_LIT:1> - residual_abs ) * leaf_weights ) <EOL> return nominator / ( denominator + self . adjusted_regularization ) <EOL> class CompositeLossFunction ( AbstractLossFunction ) : <EOL> """<STR_LIT>""" <EOL> def fit ( self , X , y , sample_weight ) : <EOL> self . y = y <EOL> self . sample_weight = sample_weight <EOL> self . y_signed = <NUM_LIT:2> * y - <NUM_LIT:1> <EOL> self . sig_w = ( y == <NUM_LIT:1> ) * self . sample_weight <EOL> self . bck_w = ( y == <NUM_LIT:0> ) * self . sample_weight <EOL> def __call__ ( self , y_pred ) : <EOL> result = numpy . sum ( self . sig_w * numpy . logaddexp ( <NUM_LIT:0> , - y_pred ) ) <EOL> result += numpy . sum ( self . bck_w * numpy . exp ( <NUM_LIT:0.5> * y_pred ) ) <EOL> return result <EOL> def negative_gradient ( self , y_pred ) : <EOL> result = self . sig_w * expit ( - y_pred ) <EOL> result -= <NUM_LIT:0.5> * self . bck_w * numpy . exp ( <NUM_LIT:0.5> * y_pred ) <EOL> return result <EOL> def hessian ( self , y_pred ) : <EOL> expits = expit ( self . y_signed * y_pred ) <EOL> return self . sig_w * expits * ( <NUM_LIT:1> - expits ) + self . bck_w * <NUM_LIT> * numpy . exp ( <NUM_LIT:0.5> * y_pred ) <EOL> class AbstractMatrixLossFunction ( AbstractLossFunction ) : <EOL> def __init__ ( self , uniform_variables ) : <EOL> """<STR_LIT>""" <EOL> self . uniform_variables = uniform_variables <EOL> self . A = None <EOL> self . A_t = None <EOL> self . w = None <EOL> def fit ( self , X , y , sample_weight ) : <EOL> """<STR_LIT>""" <EOL> assert len ( X ) == len ( y ) , "<STR_LIT>" <EOL> A , w = self . compute_parameters ( X , y ) <EOL> self . A = sparse . csr_matrix ( A ) <EOL> self . A_t = sparse . csr_matrix ( self . A . transpose ( ) ) <EOL> self . A_t_sq = self . A_t . multiply ( self . A_t ) <EOL> self . w = numpy . array ( w ) <EOL> assert A . shape [ <NUM_LIT:0> ] == len ( w ) , "<STR_LIT>" <EOL> assert A . shape [ <NUM_LIT:1> ] == len ( X ) , "<STR_LIT>" <EOL> self . y_signed = <NUM_LIT:2> * y - <NUM_LIT:1> <EOL> return self <EOL> def __call__ ( self , y_pred ) : <EOL> """<STR_LIT>""" <EOL> assert len ( y_pred ) == self . A . shape [ <NUM_LIT:1> ] , "<STR_LIT>" <EOL> exponents = numpy . exp ( - self . A . dot ( self . y_signed * y_pred ) ) <EOL> return numpy . sum ( self . w * exponents ) <EOL> def negative_gradient ( self , y_pred ) : <EOL> """<STR_LIT>""" <EOL> assert len ( y_pred ) == self . A . shape [ <NUM_LIT:1> ] , "<STR_LIT>" <EOL> exponents = numpy . exp ( - self . A . dot ( self . y_signed * y_pred ) ) <EOL> result = self . A_t . dot ( self . w * exponents ) * self . y_signed <EOL> return result <EOL> def hessian ( self , y_pred ) : <EOL> assert len ( y_pred ) == self . A . shape [ <NUM_LIT:1> ] , '<STR_LIT>' <EOL> exponents = numpy . exp ( - self . A . dot ( self . y_signed * y_pred ) ) <EOL> result = self . A_t_sq . dot ( self . w * exponents ) <EOL> return result <EOL> def compute_parameters ( self , trainX , trainY ) : <EOL> """<STR_LIT>""" <EOL> raise NotImplementedError ( ) <EOL> def update_tree ( self , tree , X , y , y_pred , sample_weight , update_mask , residual ) : <EOL> self . update_exponents = self . w * numpy . exp ( - self . A . dot ( self . y_signed * y_pred ) ) <EOL> AbstractLossFunction . update_tree ( self , tree , X , y , y_pred , sample_weight , update_mask , residual ) <EOL> def update_tree_leaf ( self , leaf , indices_in_leaf , X , y , y_pred , sample_weight , update_mask , residual ) : <EOL> terminal_region = numpy . zeros ( len ( X ) , dtype = float ) <EOL> terminal_region [ indices_in_leaf ] += <NUM_LIT:1> <EOL> z = self . A . dot ( terminal_region * self . y_signed ) <EOL> alpha = numpy . sum ( self . update_exponents * z ) / ( numpy . sum ( self . update_exponents * z * z ) + <NUM_LIT> ) <EOL> return alpha <EOL> class SimpleKnnLossFunction ( AbstractMatrixLossFunction ) : <EOL> def __init__ ( self , uniform_variables , knn = <NUM_LIT:10> , uniform_label = <NUM_LIT:1> , distinguish_classes = True , row_norm = <NUM_LIT:1.> ) : <EOL> """<STR_LIT>""" <EOL> self . knn = knn <EOL> self . distinguish_classes = distinguish_classes <EOL> self . row_norm = row_norm <EOL> self . uniform_label = check_uniform_label ( uniform_label ) <EOL> AbstractMatrixLossFunction . __init__ ( self , uniform_variables ) <EOL> def compute_parameters ( self , trainX , trainY ) : <EOL> sample_weight = numpy . ones ( len ( trainX ) ) <EOL> A_parts = [ ] <EOL> w_parts = [ ] <EOL> for label in self . uniform_label : <EOL> label_mask = trainY == label <EOL> n_label = numpy . sum ( label_mask ) <EOL> if self . distinguish_classes : <EOL> mask = label_mask <EOL> else : <EOL> mask = numpy . ones ( len ( trainY ) , dtype = numpy . bool ) <EOL> knn_indices = computeSignalKnnIndices ( self . uniform_variables , trainX , mask , self . knn ) <EOL> knn_indices = knn_indices [ label_mask , : ] <EOL> ind_ptr = numpy . arange ( <NUM_LIT:0> , n_label * self . knn + <NUM_LIT:1> , self . knn ) <EOL> column_indices = knn_indices . flatten ( ) <EOL> data = numpy . ones ( n_label * self . knn , dtype = float ) * self . row_norm / self . knn <EOL> A_part = sparse . csr_matrix ( ( data , column_indices , ind_ptr ) , shape = [ n_label , len ( trainX ) ] ) <EOL> w_part = numpy . mean ( numpy . take ( sample_weight , knn_indices ) , axis = <NUM_LIT:1> ) <EOL> assert A_part . shape [ <NUM_LIT:0> ] == len ( w_part ) <EOL> A_parts . append ( A_part ) <EOL> w_parts . append ( w_part ) <EOL> for label in set ( trainY ) - set ( self . uniform_label ) : <EOL> label_mask = trainY == label <EOL> n_label = numpy . sum ( label_mask ) <EOL> ind_ptr = numpy . arange ( <NUM_LIT:0> , n_label + <NUM_LIT:1> ) <EOL> column_indices = numpy . where ( label_mask ) [ <NUM_LIT:0> ] . flatten ( ) <EOL> data = numpy . ones ( n_label , dtype = float ) * self . row_norm <EOL> A_part = sparse . csr_matrix ( ( data , column_indices , ind_ptr ) , shape = [ n_label , len ( trainX ) ] ) <EOL> w_part = sample_weight [ label_mask ] <EOL> A_parts . append ( A_part ) <EOL> w_parts . append ( w_part ) <EOL> A = sparse . vstack ( A_parts , format = '<STR_LIT>' , dtype = float ) <EOL> w = numpy . concatenate ( w_parts ) <EOL> assert A . shape == ( len ( trainX ) , len ( trainX ) ) <EOL> return A , w <EOL> def exp_margin ( margin ) : <EOL> """<STR_LIT>""" <EOL> return numpy . exp ( numpy . clip ( margin , - <NUM_LIT> , <NUM_LIT:2> ) ) <EOL> class AbstractFlatnessLossFunction ( AbstractLossFunction ) : <EOL> def __init__ ( self , uniform_variables , uniform_label = <NUM_LIT:1> , power = <NUM_LIT> , ada_coefficient = <NUM_LIT:1.> , <EOL> allow_wrong_signs = True , use_median = False , <EOL> keep_debug_info = False ) : <EOL> """<STR_LIT>""" <EOL> self . uniform_variables = uniform_variables <EOL> if isinstance ( uniform_label , numbers . Number ) : <EOL> self . uniform_label = numpy . array ( [ uniform_label ] ) <EOL> else : <EOL> self . uniform_label = numpy . array ( uniform_label ) <EOL> self . power = power <EOL> self . ada_coefficient = ada_coefficient <EOL> self . allow_wrong_signs = allow_wrong_signs <EOL> self . keep_debug_info = keep_debug_info <EOL> self . use_median = use_median <EOL> def fit ( self , X , y , sample_weight = None ) : <EOL> sample_weight = check_sample_weight ( y , sample_weight = sample_weight ) <EOL> assert len ( X ) == len ( y ) , '<STR_LIT>' <EOL> X = pandas . DataFrame ( X ) <EOL> self . group_indices = dict ( ) <EOL> self . group_weights = dict ( ) <EOL> occurences = numpy . zeros ( len ( X ) ) <EOL> for label in self . uniform_label : <EOL> self . group_indices [ label ] = self . compute_groups_indices ( X , y , label = label ) <EOL> self . group_weights [ label ] = compute_group_weights ( self . group_indices [ label ] , sample_weight = sample_weight ) <EOL> for group in self . group_indices [ label ] : <EOL> occurences [ group ] += <NUM_LIT:1> <EOL> out_of_bins = ( occurences == <NUM_LIT:0> ) & numpy . in1d ( y , self . uniform_label ) <EOL> if numpy . mean ( out_of_bins ) > <NUM_LIT> : <EOL> warnings . warn ( "<STR_LIT>" % numpy . sum ( out_of_bins ) , UserWarning ) <EOL> self . y = y <EOL> self . y_signed = <NUM_LIT:2> * y - <NUM_LIT:1> <EOL> self . sample_weight = numpy . copy ( sample_weight ) <EOL> self . divided_weight = sample_weight / numpy . maximum ( occurences , <NUM_LIT:1> ) <EOL> if self . keep_debug_info : <EOL> self . debug_dict = defaultdict ( list ) <EOL> return self <EOL> def compute_groups_indices ( self , X , y , label ) : <EOL> raise NotImplementedError ( ) <EOL> def __call__ ( self , pred ) : <EOL> return <NUM_LIT:0> <EOL> def negative_gradient ( self , y_pred ) : <EOL> y_pred = numpy . ravel ( y_pred ) <EOL> neg_gradient = numpy . zeros ( len ( self . y ) , dtype = numpy . float ) <EOL> for label in self . uniform_label : <EOL> label_mask = self . y == label <EOL> global_positions = numpy . zeros ( len ( y_pred ) , dtype = float ) <EOL> global_positions [ label_mask ] = compute_positions ( y_pred [ label_mask ] , sample_weight = self . sample_weight [ label_mask ] ) <EOL> for indices_in_bin in self . group_indices [ label ] : <EOL> local_pos = compute_positions ( y_pred [ indices_in_bin ] , <EOL> sample_weight = self . sample_weight [ indices_in_bin ] ) <EOL> global_pos = global_positions [ indices_in_bin ] <EOL> bin_gradient = self . power * numpy . sign ( local_pos - global_pos ) * numpy . abs ( local_pos - global_pos ) ** ( self . power - <NUM_LIT:1> ) <EOL> neg_gradient [ indices_in_bin ] += bin_gradient <EOL> neg_gradient *= self . divided_weight <EOL> assert numpy . all ( neg_gradient [ ~ numpy . in1d ( self . y , self . uniform_label ) ] == <NUM_LIT:0> ) <EOL> y_signed = self . y_signed <EOL> if self . keep_debug_info : <EOL> self . debug_dict [ '<STR_LIT>' ] . append ( numpy . copy ( y_pred ) ) <EOL> self . debug_dict [ '<STR_LIT>' ] . append ( numpy . copy ( neg_gradient ) ) <EOL> self . debug_dict [ '<STR_LIT>' ] . append ( y_signed * self . sample_weight * exp_margin ( - y_signed * y_pred ) ) <EOL> neg_gradient += self . ada_coefficient * y_signed * self . sample_weight * exp_margin ( - y_signed * y_pred ) <EOL> if not self . allow_wrong_signs : <EOL> neg_gradient = y_signed * numpy . clip ( y_signed * neg_gradient , <NUM_LIT:0> , <NUM_LIT> ) <EOL> return neg_gradient <EOL> def update_tree_leaf ( self , leaf , indices_in_leaf , <EOL> X , y , y_pred , sample_weight , update_mask , residual ) : <EOL> if self . use_median : <EOL> residual = residual [ indices_in_leaf ] <EOL> return numpy . median ( residual ) <EOL> else : <EOL> return numpy . clip ( y_pred [ indices_in_leaf [ <NUM_LIT:0> ] ] , - <NUM_LIT:10> , <NUM_LIT:10> ) <EOL> class BinFlatnessLossFunction ( AbstractFlatnessLossFunction ) : <EOL> def __init__ ( self , uniform_variables , n_bins = <NUM_LIT:10> , uniform_label = <NUM_LIT:1> , power = <NUM_LIT> , ada_coefficient = <NUM_LIT:1.> , <EOL> allow_wrong_signs = True , use_median = False , keep_debug_info = False ) : <EOL> self . n_bins = n_bins <EOL> AbstractFlatnessLossFunction . __init__ ( self , uniform_variables , <EOL> uniform_label = uniform_label , power = power , ada_coefficient = ada_coefficient , <EOL> allow_wrong_signs = allow_wrong_signs , use_median = use_median , <EOL> keep_debug_info = keep_debug_info ) <EOL> def compute_groups_indices ( self , X , y , label ) : <EOL> """<STR_LIT>""" <EOL> label_mask = y == label <EOL> extended_bin_limits = [ ] <EOL> for var in self . uniform_variables : <EOL> f_min , f_max = numpy . min ( X [ var ] [ label_mask ] ) , numpy . max ( X [ var ] [ label_mask ] ) <EOL> extended_bin_limits . append ( numpy . linspace ( f_min , f_max , <NUM_LIT:2> * self . n_bins + <NUM_LIT:1> ) ) <EOL> groups_indices = list ( ) <EOL> for shift in [ <NUM_LIT:0> , <NUM_LIT:1> ] : <EOL> bin_limits = [ ] <EOL> for axis_limits in extended_bin_limits : <EOL> bin_limits . append ( axis_limits [ <NUM_LIT:1> + shift : - <NUM_LIT:1> : <NUM_LIT:2> ] ) <EOL> bin_indices = compute_bin_indices ( X . ix [ : , self . uniform_variables ] . values , bin_limits = bin_limits ) <EOL> groups_indices += list ( bin_to_group_indices ( bin_indices , mask = label_mask ) ) <EOL> return groups_indices <EOL> class KnnFlatnessLossFunction ( AbstractFlatnessLossFunction ) : <EOL> def __init__ ( self , uniform_variables , n_neighbours = <NUM_LIT:100> , uniform_label = <NUM_LIT:1> , power = <NUM_LIT> , ada_coefficient = <NUM_LIT:1.> , <EOL> max_groups_on_iteration = <NUM_LIT> , allow_wrong_signs = True , use_median = False , keep_debug_info = False , <EOL> random_state = None ) : <EOL> self . n_neighbours = n_neighbours <EOL> self . max_group_on_iteration = max_groups_on_iteration <EOL> self . random_state = random_state <EOL> AbstractFlatnessLossFunction . __init__ ( self , uniform_variables , <EOL> uniform_label = uniform_label , power = power , ada_coefficient = ada_coefficient , <EOL> allow_wrong_signs = allow_wrong_signs , use_median = use_median , <EOL> keep_debug_info = keep_debug_info ) <EOL> def compute_groups_indices ( self , X , y , label ) : <EOL> mask = y == label <EOL> self . random_state = check_random_state ( self . random_state ) <EOL> knn_indices = computeSignalKnnIndices ( self . uniform_variables , X , mask , <EOL> n_neighbors = self . n_neighbours ) [ mask , : ] <EOL> if len ( knn_indices ) > self . max_group_on_iteration : <EOL> selected_group = self . random_state . choice ( len ( knn_indices ) , size = self . max_group_on_iteration ) <EOL> return knn_indices [ selected_group , : ] <EOL> else : <EOL> return knn_indices </s>
