<s> """<STR_LIT>""" <EOL> from __future__ import division <EOL> import itertools <EOL> import numbers <EOL> import numpy as np <EOL> from warnings import warn <EOL> from abc import ABCMeta , abstractmethod <EOL> from . . base import ClassifierMixin , RegressorMixin <EOL> from . . externals . joblib import Parallel , delayed <EOL> from . . externals . six import with_metaclass <EOL> from . . externals . six . moves import zip <EOL> from . . metrics import r2_score , accuracy_score <EOL> from . . tree import DecisionTreeClassifier , DecisionTreeRegressor <EOL> from . . utils import check_random_state , check_X_y , check_array , column_or_1d <EOL> from . . utils . random import sample_without_replacement <EOL> from . . utils . validation import has_fit_parameter , check_is_fitted <EOL> from . . utils . fixes import bincount <EOL> from . . utils . metaestimators import if_delegate_has_method <EOL> from . . utils . multiclass import check_classification_targets <EOL> from . base import BaseEnsemble , _partition_estimators <EOL> __all__ = [ "<STR_LIT>" , <EOL> "<STR_LIT>" ] <EOL> MAX_INT = np . iinfo ( np . int32 ) . max <EOL> def _parallel_build_estimators ( n_estimators , ensemble , X , y , sample_weight , <EOL> max_samples , seeds , total_n_estimators , verbose ) : <EOL> """<STR_LIT>""" <EOL> n_samples , n_features = X . shape <EOL> max_features = ensemble . max_features <EOL> if ( not isinstance ( max_samples , ( numbers . Integral , np . integer ) ) and <EOL> ( <NUM_LIT:0.0> < max_samples <= <NUM_LIT:1.0> ) ) : <EOL> max_samples = int ( max_samples * n_samples ) <EOL> if ( not isinstance ( max_features , ( numbers . Integral , np . integer ) ) and <EOL> ( <NUM_LIT:0.0> < max_features <= <NUM_LIT:1.0> ) ) : <EOL> max_features = int ( max_features * n_features ) <EOL> bootstrap = ensemble . bootstrap <EOL> bootstrap_features = ensemble . bootstrap_features <EOL> support_sample_weight = has_fit_parameter ( ensemble . base_estimator_ , <EOL> "<STR_LIT>" ) <EOL> if not support_sample_weight and sample_weight is not None : <EOL> raise ValueError ( "<STR_LIT>" ) <EOL> estimators = [ ] <EOL> estimators_samples = [ ] <EOL> estimators_features = [ ] <EOL> for i in range ( n_estimators ) : <EOL> if verbose > <NUM_LIT:1> : <EOL> print ( "<STR_LIT>" % <EOL> ( i + <NUM_LIT:1> , n_estimators , total_n_estimators ) ) <EOL> random_state = check_random_state ( seeds [ i ] ) <EOL> seed = random_state . randint ( MAX_INT ) <EOL> estimator = ensemble . _make_estimator ( append = False ) <EOL> try : <EOL> estimator . set_params ( random_state = seed ) <EOL> except ValueError : <EOL> pass <EOL> if bootstrap_features : <EOL> features = random_state . randint ( <NUM_LIT:0> , n_features , max_features ) <EOL> else : <EOL> features = sample_without_replacement ( n_features , <EOL> max_features , <EOL> random_state = random_state ) <EOL> if support_sample_weight : <EOL> if sample_weight is None : <EOL> curr_sample_weight = np . ones ( ( n_samples , ) ) <EOL> else : <EOL> curr_sample_weight = sample_weight . copy ( ) <EOL> if bootstrap : <EOL> indices = random_state . randint ( <NUM_LIT:0> , n_samples , max_samples ) <EOL> sample_counts = bincount ( indices , minlength = n_samples ) <EOL> curr_sample_weight *= sample_counts <EOL> else : <EOL> not_indices = sample_without_replacement ( <EOL> n_samples , <EOL> n_samples - max_samples , <EOL> random_state = random_state ) <EOL> curr_sample_weight [ not_indices ] = <NUM_LIT:0> <EOL> estimator . fit ( X [ : , features ] , y , sample_weight = curr_sample_weight ) <EOL> samples = curr_sample_weight > <NUM_LIT:0.> <EOL> else : <EOL> if bootstrap : <EOL> indices = random_state . randint ( <NUM_LIT:0> , n_samples , max_samples ) <EOL> else : <EOL> indices = sample_without_replacement ( n_samples , <EOL> max_samples , <EOL> random_state = random_state ) <EOL> sample_counts = bincount ( indices , minlength = n_samples ) <EOL> estimator . fit ( ( X [ indices ] ) [ : , features ] , y [ indices ] ) <EOL> samples = sample_counts > <NUM_LIT:0.> <EOL> estimators . append ( estimator ) <EOL> estimators_samples . append ( samples ) <EOL> estimators_features . append ( features ) <EOL> return estimators , estimators_samples , estimators_features <EOL> def _parallel_predict_proba ( estimators , estimators_features , X , n_classes ) : <EOL> """<STR_LIT>""" <EOL> n_samples = X . shape [ <NUM_LIT:0> ] <EOL> proba = np . zeros ( ( n_samples , n_classes ) ) <EOL> for estimator , features in zip ( estimators , estimators_features ) : <EOL> if hasattr ( estimator , "<STR_LIT>" ) : <EOL> proba_estimator = estimator . predict_proba ( X [ : , features ] ) <EOL> if n_classes == len ( estimator . classes_ ) : <EOL> proba += proba_estimator <EOL> else : <EOL> proba [ : , estimator . classes_ ] += proba_estimator [ : , range ( len ( estimator . classes_ ) ) ] <EOL> else : <EOL> predictions = estimator . predict ( X [ : , features ] ) <EOL> for i in range ( n_samples ) : <EOL> proba [ i , predictions [ i ] ] += <NUM_LIT:1> <EOL> return proba <EOL> def _parallel_predict_log_proba ( estimators , estimators_features , X , n_classes ) : <EOL> """<STR_LIT>""" <EOL> n_samples = X . shape [ <NUM_LIT:0> ] <EOL> log_proba = np . empty ( ( n_samples , n_classes ) ) <EOL> log_proba . fill ( - np . inf ) <EOL> all_classes = np . arange ( n_classes , dtype = np . int ) <EOL> for estimator , features in zip ( estimators , estimators_features ) : <EOL> log_proba_estimator = estimator . predict_log_proba ( X [ : , features ] ) <EOL> if n_classes == len ( estimator . classes_ ) : <EOL> log_proba = np . logaddexp ( log_proba , log_proba_estimator ) <EOL> else : <EOL> log_proba [ : , estimator . classes_ ] = np . logaddexp ( <EOL> log_proba [ : , estimator . classes_ ] , <EOL> log_proba_estimator [ : , range ( len ( estimator . classes_ ) ) ] ) <EOL> missing = np . setdiff1d ( all_classes , estimator . classes_ ) <EOL> log_proba [ : , missing ] = np . logaddexp ( log_proba [ : , missing ] , <EOL> - np . inf ) <EOL> return log_proba <EOL> def _parallel_decision_function ( estimators , estimators_features , X ) : <EOL> """<STR_LIT>""" <EOL> return sum ( estimator . decision_function ( X [ : , features ] ) <EOL> for estimator , features in zip ( estimators , <EOL> estimators_features ) ) <EOL> def _parallel_predict_regression ( estimators , estimators_features , X ) : <EOL> """<STR_LIT>""" <EOL> return sum ( estimator . predict ( X [ : , features ] ) <EOL> for estimator , features in zip ( estimators , <EOL> estimators_features ) ) <EOL> class BaseBagging ( with_metaclass ( ABCMeta , BaseEnsemble ) ) : <EOL> """<STR_LIT>""" <EOL> @ abstractmethod <EOL> def __init__ ( self , <EOL> base_estimator = None , <EOL> n_estimators = <NUM_LIT:10> , <EOL> max_samples = <NUM_LIT:1.0> , <EOL> max_features = <NUM_LIT:1.0> , <EOL> bootstrap = True , <EOL> bootstrap_features = False , <EOL> oob_score = False , <EOL> warm_start = False , <EOL> n_jobs = <NUM_LIT:1> , <EOL> random_state = None , <EOL> verbose = <NUM_LIT:0> ) : <EOL> super ( BaseBagging , self ) . __init__ ( <EOL> base_estimator = base_estimator , <EOL> n_estimators = n_estimators ) <EOL> self . max_samples = max_samples <EOL> self . max_features = max_features <EOL> self . bootstrap = bootstrap <EOL> self . bootstrap_features = bootstrap_features <EOL> self . oob_score = oob_score <EOL> self . warm_start = warm_start <EOL> self . n_jobs = n_jobs <EOL> self . random_state = random_state <EOL> self . verbose = verbose <EOL> def fit ( self , X , y , sample_weight = None ) : <EOL> """<STR_LIT>""" <EOL> return self . _fit ( X , y , self . max_samples , sample_weight = sample_weight ) <EOL> def _fit ( self , X , y , max_samples , max_depth = None , sample_weight = None ) : <EOL> """<STR_LIT>""" <EOL> random_state = check_random_state ( self . random_state ) <EOL> X , y = check_X_y ( X , y , [ '<STR_LIT>' , '<STR_LIT>' ] ) <EOL> n_samples , self . n_features_ = X . shape <EOL> y = self . _validate_y ( y ) <EOL> self . _validate_estimator ( ) <EOL> if max_depth is not None : <EOL> self . base_estimator_ . max_depth = max_depth <EOL> if not isinstance ( max_samples , ( numbers . Integral , np . integer ) ) : <EOL> max_samples = int ( max_samples * X . shape [ <NUM_LIT:0> ] ) <EOL> if not ( <NUM_LIT:0> < max_samples <= X . shape [ <NUM_LIT:0> ] ) : <EOL> raise ValueError ( "<STR_LIT>" ) <EOL> if isinstance ( self . max_features , ( numbers . Integral , np . integer ) ) : <EOL> max_features = self . max_features <EOL> else : <EOL> max_features = int ( self . max_features * self . n_features_ ) <EOL> if not ( <NUM_LIT:0> < max_features <= self . n_features_ ) : <EOL> raise ValueError ( "<STR_LIT>" ) <EOL> if not self . bootstrap and self . oob_score : <EOL> raise ValueError ( "<STR_LIT>" <EOL> "<STR_LIT>" ) <EOL> if self . warm_start and self . oob_score : <EOL> raise ValueError ( "<STR_LIT>" <EOL> "<STR_LIT>" ) <EOL> if hasattr ( self , "<STR_LIT>" ) and self . warm_start : <EOL> del self . oob_score_ <EOL> if not self . warm_start or len ( self . estimators_ ) == <NUM_LIT:0> : <EOL> self . estimators_ = [ ] <EOL> self . estimators_samples_ = [ ] <EOL> self . estimators_features_ = [ ] <EOL> n_more_estimators = self . n_estimators - len ( self . estimators_ ) <EOL> if n_more_estimators < <NUM_LIT:0> : <EOL> raise ValueError ( '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> % ( self . n_estimators , len ( self . estimators_ ) ) ) <EOL> elif n_more_estimators == <NUM_LIT:0> : <EOL> warn ( "<STR_LIT>" <EOL> "<STR_LIT>" ) <EOL> return self <EOL> n_jobs , n_estimators , starts = _partition_estimators ( n_more_estimators , <EOL> self . n_jobs ) <EOL> total_n_estimators = sum ( n_estimators ) <EOL> if self . warm_start and len ( self . estimators_ ) > <NUM_LIT:0> : <EOL> random_state . randint ( MAX_INT , size = len ( self . estimators_ ) ) <EOL> seeds = random_state . randint ( MAX_INT , size = n_more_estimators ) <EOL> all_results = Parallel ( n_jobs = n_jobs , verbose = self . verbose ) ( <EOL> delayed ( _parallel_build_estimators ) ( <EOL> n_estimators [ i ] , <EOL> self , <EOL> X , <EOL> y , <EOL> sample_weight , <EOL> max_samples , <EOL> seeds [ starts [ i ] : starts [ i + <NUM_LIT:1> ] ] , <EOL> total_n_estimators , <EOL> verbose = self . verbose ) <EOL> for i in range ( n_jobs ) ) <EOL> self . estimators_ += list ( itertools . chain . from_iterable ( <EOL> t [ <NUM_LIT:0> ] for t in all_results ) ) <EOL> self . estimators_samples_ += list ( itertools . chain . from_iterable ( <EOL> t [ <NUM_LIT:1> ] for t in all_results ) ) <EOL> self . estimators_features_ += list ( itertools . chain . from_iterable ( <EOL> t [ <NUM_LIT:2> ] for t in all_results ) ) <EOL> if self . oob_score : <EOL> self . _set_oob_score ( X , y ) <EOL> return self <EOL> @ abstractmethod <EOL> def _set_oob_score ( self , X , y ) : <EOL> """<STR_LIT>""" <EOL> def _validate_y ( self , y ) : <EOL> return column_or_1d ( y , warn = True ) <EOL> class BaggingClassifier ( BaseBagging , ClassifierMixin ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , <EOL> base_estimator = None , <EOL> n_estimators = <NUM_LIT:10> , <EOL> max_samples = <NUM_LIT:1.0> , <EOL> max_features = <NUM_LIT:1.0> , <EOL> bootstrap = True , <EOL> bootstrap_features = False , <EOL> oob_score = False , <EOL> warm_start = False , <EOL> n_jobs = <NUM_LIT:1> , <EOL> random_state = None , <EOL> verbose = <NUM_LIT:0> ) : <EOL> super ( BaggingClassifier , self ) . __init__ ( <EOL> base_estimator , <EOL> n_estimators = n_estimators , <EOL> max_samples = max_samples , <EOL> max_features = max_features , <EOL> bootstrap = bootstrap , <EOL> bootstrap_features = bootstrap_features , <EOL> oob_score = oob_score , <EOL> warm_start = warm_start , <EOL> n_jobs = n_jobs , <EOL> random_state = random_state , <EOL> verbose = verbose ) <EOL> def _validate_estimator ( self ) : <EOL> """<STR_LIT>""" <EOL> super ( BaggingClassifier , self ) . _validate_estimator ( <EOL> default = DecisionTreeClassifier ( ) ) <EOL> def _set_oob_score ( self , X , y ) : <EOL> n_classes_ = self . n_classes_ <EOL> classes_ = self . classes_ <EOL> n_samples = y . shape [ <NUM_LIT:0> ] <EOL> predictions = np . zeros ( ( n_samples , n_classes_ ) ) <EOL> for estimator , samples , features in zip ( self . estimators_ , <EOL> self . estimators_samples_ , <EOL> self . estimators_features_ ) : <EOL> mask = np . ones ( n_samples , dtype = np . bool ) <EOL> mask [ samples ] = False <EOL> if hasattr ( estimator , "<STR_LIT>" ) : <EOL> predictions [ mask , : ] += estimator . predict_proba ( <EOL> ( X [ mask , : ] ) [ : , features ] ) <EOL> else : <EOL> p = estimator . predict ( ( X [ mask , : ] ) [ : , features ] ) <EOL> j = <NUM_LIT:0> <EOL> for i in range ( n_samples ) : <EOL> if mask [ i ] : <EOL> predictions [ i , p [ j ] ] += <NUM_LIT:1> <EOL> j += <NUM_LIT:1> <EOL> if ( predictions . sum ( axis = <NUM_LIT:1> ) == <NUM_LIT:0> ) . any ( ) : <EOL> warn ( "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" ) <EOL> oob_decision_function = ( predictions / <EOL> predictions . sum ( axis = <NUM_LIT:1> ) [ : , np . newaxis ] ) <EOL> oob_score = accuracy_score ( y , classes_ . take ( np . argmax ( predictions , <EOL> axis = <NUM_LIT:1> ) ) ) <EOL> self . oob_decision_function_ = oob_decision_function <EOL> self . oob_score_ = oob_score <EOL> def _validate_y ( self , y ) : <EOL> y = column_or_1d ( y , warn = True ) <EOL> check_classification_targets ( y ) <EOL> self . classes_ , y = np . unique ( y , return_inverse = True ) <EOL> self . n_classes_ = len ( self . classes_ ) <EOL> return y <EOL> def predict ( self , X ) : <EOL> """<STR_LIT>""" <EOL> predicted_probabilitiy = self . predict_proba ( X ) <EOL> return self . classes_ . take ( ( np . argmax ( predicted_probabilitiy , axis = <NUM_LIT:1> ) ) , <EOL> axis = <NUM_LIT:0> ) <EOL> def predict_proba ( self , X ) : <EOL> """<STR_LIT>""" <EOL> check_is_fitted ( self , "<STR_LIT>" ) <EOL> X = check_array ( X , accept_sparse = [ '<STR_LIT>' , '<STR_LIT>' ] ) <EOL> if self . n_features_ != X . shape [ <NUM_LIT:1> ] : <EOL> raise ValueError ( "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" . format ( self . n_features_ , X . shape [ <NUM_LIT:1> ] ) ) <EOL> n_jobs , n_estimators , starts = _partition_estimators ( self . n_estimators , <EOL> self . n_jobs ) <EOL> all_proba = Parallel ( n_jobs = n_jobs , verbose = self . verbose ) ( <EOL> delayed ( _parallel_predict_proba ) ( <EOL> self . estimators_ [ starts [ i ] : starts [ i + <NUM_LIT:1> ] ] , <EOL> self . estimators_features_ [ starts [ i ] : starts [ i + <NUM_LIT:1> ] ] , <EOL> X , <EOL> self . n_classes_ ) <EOL> for i in range ( n_jobs ) ) <EOL> proba = sum ( all_proba ) / self . n_estimators <EOL> return proba <EOL> def predict_log_proba ( self , X ) : <EOL> """<STR_LIT>""" <EOL> check_is_fitted ( self , "<STR_LIT>" ) <EOL> if hasattr ( self . base_estimator_ , "<STR_LIT>" ) : <EOL> X = check_array ( X , accept_sparse = [ '<STR_LIT>' , '<STR_LIT>' ] ) <EOL> if self . n_features_ != X . shape [ <NUM_LIT:1> ] : <EOL> raise ValueError ( "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" . format ( self . n_features_ , X . shape [ <NUM_LIT:1> ] ) ) <EOL> n_jobs , n_estimators , starts = _partition_estimators ( <EOL> self . n_estimators , self . n_jobs ) <EOL> all_log_proba = Parallel ( n_jobs = n_jobs , verbose = self . verbose ) ( <EOL> delayed ( _parallel_predict_log_proba ) ( <EOL> self . estimators_ [ starts [ i ] : starts [ i + <NUM_LIT:1> ] ] , <EOL> self . estimators_features_ [ starts [ i ] : starts [ i + <NUM_LIT:1> ] ] , <EOL> X , <EOL> self . n_classes_ ) <EOL> for i in range ( n_jobs ) ) <EOL> log_proba = all_log_proba [ <NUM_LIT:0> ] <EOL> for j in range ( <NUM_LIT:1> , len ( all_log_proba ) ) : <EOL> log_proba = np . logaddexp ( log_proba , all_log_proba [ j ] ) <EOL> log_proba -= np . log ( self . n_estimators ) <EOL> return log_proba <EOL> else : <EOL> return np . log ( self . predict_proba ( X ) ) <EOL> @ if_delegate_has_method ( delegate = '<STR_LIT>' ) <EOL> def decision_function ( self , X ) : <EOL> """<STR_LIT>""" <EOL> check_is_fitted ( self , "<STR_LIT>" ) <EOL> X = check_array ( X , accept_sparse = [ '<STR_LIT>' , '<STR_LIT>' ] ) <EOL> if self . n_features_ != X . shape [ <NUM_LIT:1> ] : <EOL> raise ValueError ( "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" . format ( self . n_features_ , X . shape [ <NUM_LIT:1> ] ) ) <EOL> n_jobs , n_estimators , starts = _partition_estimators ( self . n_estimators , <EOL> self . n_jobs ) <EOL> all_decisions = Parallel ( n_jobs = n_jobs , verbose = self . verbose ) ( <EOL> delayed ( _parallel_decision_function ) ( <EOL> self . estimators_ [ starts [ i ] : starts [ i + <NUM_LIT:1> ] ] , <EOL> self . estimators_features_ [ starts [ i ] : starts [ i + <NUM_LIT:1> ] ] , <EOL> X ) <EOL> for i in range ( n_jobs ) ) <EOL> decisions = sum ( all_decisions ) / self . n_estimators <EOL> return decisions <EOL> class BaggingRegressor ( BaseBagging , RegressorMixin ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , <EOL> base_estimator = None , <EOL> n_estimators = <NUM_LIT:10> , <EOL> max_samples = <NUM_LIT:1.0> , <EOL> max_features = <NUM_LIT:1.0> , <EOL> bootstrap = True , <EOL> bootstrap_features = False , <EOL> oob_score = False , <EOL> warm_start = False , <EOL> n_jobs = <NUM_LIT:1> , <EOL> random_state = None , <EOL> verbose = <NUM_LIT:0> ) : <EOL> super ( BaggingRegressor , self ) . __init__ ( <EOL> base_estimator , <EOL> n_estimators = n_estimators , <EOL> max_samples = max_samples , <EOL> max_features = max_features , <EOL> bootstrap = bootstrap , <EOL> bootstrap_features = bootstrap_features , <EOL> oob_score = oob_score , <EOL> warm_start = warm_start , <EOL> n_jobs = n_jobs , <EOL> random_state = random_state , <EOL> verbose = verbose ) <EOL> def predict ( self , X ) : <EOL> """<STR_LIT>""" <EOL> check_is_fitted ( self , "<STR_LIT>" ) <EOL> X = check_array ( X , accept_sparse = [ '<STR_LIT>' , '<STR_LIT>' ] ) <EOL> n_jobs , n_estimators , starts = _partition_estimators ( self . n_estimators , <EOL> self . n_jobs ) <EOL> all_y_hat = Parallel ( n_jobs = n_jobs , verbose = self . verbose ) ( <EOL> delayed ( _parallel_predict_regression ) ( <EOL> self . estimators_ [ starts [ i ] : starts [ i + <NUM_LIT:1> ] ] , <EOL> self . estimators_features_ [ starts [ i ] : starts [ i + <NUM_LIT:1> ] ] , <EOL> X ) <EOL> for i in range ( n_jobs ) ) <EOL> y_hat = sum ( all_y_hat ) / self . n_estimators <EOL> return y_hat <EOL> def _validate_estimator ( self ) : <EOL> """<STR_LIT>""" <EOL> super ( BaggingRegressor , self ) . _validate_estimator ( <EOL> default = DecisionTreeRegressor ( ) ) <EOL> def _set_oob_score ( self , X , y ) : <EOL> n_samples = y . shape [ <NUM_LIT:0> ] <EOL> predictions = np . zeros ( ( n_samples , ) ) <EOL> n_predictions = np . zeros ( ( n_samples , ) ) <EOL> for estimator , samples , features in zip ( self . estimators_ , <EOL> self . estimators_samples_ , <EOL> self . estimators_features_ ) : <EOL> mask = np . ones ( n_samples , dtype = np . bool ) <EOL> mask [ samples ] = False <EOL> predictions [ mask ] += estimator . predict ( ( X [ mask , : ] ) [ : , features ] ) <EOL> n_predictions [ mask ] += <NUM_LIT:1> <EOL> if ( n_predictions == <NUM_LIT:0> ) . any ( ) : <EOL> warn ( "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" ) <EOL> n_predictions [ n_predictions == <NUM_LIT:0> ] = <NUM_LIT:1> <EOL> predictions /= n_predictions <EOL> self . oob_prediction_ = predictions <EOL> self . oob_score_ = r2_score ( y , predictions ) </s>
