<s> """<STR_LIT>""" <EOL> import os <EOL> import sys <EOL> import sqlite3 <EOL> import numpy as np <EOL> from math import sqrt <EOL> from cPickle import loads , dumps <EOL> from collections import Counter <EOL> from sklearn . base import BaseEstimator , ClassifierMixin <EOL> from sklearn . utils import check_random_state <EOL> from sklearn . metrics import f1_score , roc_auc_score <EOL> from sklearn . metrics import mean_squared_error , accuracy_score <EOL> from sklearn . cross_validation import StratifiedKFold <EOL> from sklearn . preprocessing import LabelBinarizer <EOL> def _f1 ( y , y_bin , probs ) : <EOL> """<STR_LIT>""" <EOL> return f1_score ( y , np . argmax ( probs , axis = <NUM_LIT:1> ) ) <EOL> def _auc ( y , y_bin , probs ) : <EOL> """<STR_LIT>""" <EOL> return roc_auc_score ( y , probs [ : , <NUM_LIT:1> ] ) <EOL> def _rmse ( y , y_bin , probs ) : <EOL> """<STR_LIT>""" <EOL> return <NUM_LIT:1.0> - sqrt ( mean_squared_error ( y_bin , probs ) ) <EOL> def _accuracy ( y , y_bin , probs ) : <EOL> """<STR_LIT>""" <EOL> return accuracy_score ( y , np . argmax ( probs , axis = <NUM_LIT:1> ) ) <EOL> def _mxentropy ( y , y_bin , probs ) : <EOL> """<STR_LIT>""" <EOL> eps = <NUM_LIT> <EOL> clipped = np . clip ( probs , eps , <NUM_LIT:1.0> - eps ) <EOL> clipped /= clipped . sum ( axis = <NUM_LIT:1> ) [ : , np . newaxis ] <EOL> return ( y_bin * np . log ( clipped ) ) . sum ( ) / y . shape [ <NUM_LIT:0> ] <EOL> def _bootstraps ( n , rs ) : <EOL> """<STR_LIT>""" <EOL> bs_inds = rs . randint ( n , size = ( n ) ) <EOL> return bs_inds , np . setdiff1d ( range ( n ) , bs_inds ) <EOL> class EnsembleSelectionClassifier ( BaseEstimator , ClassifierMixin ) : <EOL> """<STR_LIT>""" <EOL> _metrics = { <EOL> '<STR_LIT>' : _f1 , <EOL> '<STR_LIT>' : _auc , <EOL> '<STR_LIT>' : _rmse , <EOL> '<STR_LIT>' : _accuracy , <EOL> '<STR_LIT>' : _mxentropy , <EOL> } <EOL> def __init__ ( self , db_file = None , <EOL> models = None , n_best = <NUM_LIT:5> , n_folds = <NUM_LIT:3> , <EOL> n_bags = <NUM_LIT:20> , bag_fraction = <NUM_LIT> , <EOL> prune_fraction = <NUM_LIT> , <EOL> score_metric = '<STR_LIT>' , <EOL> epsilon = <NUM_LIT> , max_models = <NUM_LIT:50> , <EOL> use_epsilon = False , use_bootstrap = False , <EOL> verbose = False , random_state = None ) : <EOL> self . db_file = db_file <EOL> self . models = models <EOL> self . n_best = n_best <EOL> self . n_bags = n_bags <EOL> self . n_folds = n_folds <EOL> self . bag_fraction = bag_fraction <EOL> self . prune_fraction = prune_fraction <EOL> self . score_metric = score_metric <EOL> self . epsilon = epsilon <EOL> self . max_models = max_models <EOL> self . use_epsilon = use_epsilon <EOL> self . use_bootstrap = use_bootstrap <EOL> self . verbose = verbose <EOL> self . random_state = random_state <EOL> self . _check_params ( ) <EOL> self . _folds = None <EOL> self . _n_models = <NUM_LIT:0> <EOL> self . _n_classes = <NUM_LIT:0> <EOL> self . _metric = None <EOL> self . _ensemble = Counter ( ) <EOL> self . _model_scores = [ ] <EOL> self . _scored_models = [ ] <EOL> self . _fitted_models = [ ] <EOL> self . _init_db ( models ) <EOL> def _check_params ( self ) : <EOL> """<STR_LIT>""" <EOL> if ( not self . db_file ) : <EOL> msg = "<STR_LIT>" <EOL> raise ValueError ( msg ) <EOL> if ( self . epsilon < <NUM_LIT:0.0> ) : <EOL> msg = "<STR_LIT>" <EOL> raise ValueError ( msg ) <EOL> metric_names = self . _metrics . keys ( ) <EOL> if ( self . score_metric not in metric_names ) : <EOL> msg = "<STR_LIT>" % metric_names <EOL> raise ValueError ( msg ) <EOL> if ( self . n_best < <NUM_LIT:1> ) : <EOL> msg = "<STR_LIT>" <EOL> raise ValueError ( msg ) <EOL> if ( self . max_models < self . n_best ) : <EOL> msg = "<STR_LIT>" <EOL> raise ValueError ( msg ) <EOL> if ( not self . use_bootstrap ) : <EOL> if ( self . n_folds < <NUM_LIT:2> ) : <EOL> msg = "<STR_LIT>" <EOL> raise ValueError ( msg ) <EOL> else : <EOL> if ( self . n_folds < <NUM_LIT:1> ) : <EOL> msg = "<STR_LIT>" <EOL> raise ValueError ( msg ) <EOL> def _init_db ( self , models ) : <EOL> """<STR_LIT>""" <EOL> _createTablesScript = """<STR_LIT>""" <EOL> if ( models and os . path . exists ( self . db_file ) ) : <EOL> raise ValueError ( "<STR_LIT>" % self . db_file ) <EOL> db_conn = sqlite3 . connect ( self . db_file ) <EOL> with db_conn : <EOL> db_conn . execute ( "<STR_LIT>" ) <EOL> if ( models ) : <EOL> with db_conn : <EOL> db_conn . executescript ( _createTablesScript ) <EOL> insert_stmt = """<STR_LIT>""" <EOL> with db_conn : <EOL> vals = ( ( i , buffer ( dumps ( m ) ) ) for i , m in enumerate ( models ) ) <EOL> db_conn . executemany ( insert_stmt , vals ) <EOL> create_stmt = "<STR_LIT>" <EOL> db_conn . execute ( create_stmt ) <EOL> self . _n_models = len ( models ) <EOL> else : <EOL> curs = db_conn . cursor ( ) <EOL> curs . execute ( "<STR_LIT>" ) <EOL> self . _n_models = curs . fetchone ( ) [ <NUM_LIT:0> ] <EOL> curs . execute ( "<STR_LIT>" ) <EOL> for k , v in curs . fetchall ( ) : <EOL> self . _ensemble [ k ] = v <EOL> curs . execute ( "<STR_LIT>" ) <EOL> r = curs . fetchone ( ) <EOL> probs = loads ( str ( r [ <NUM_LIT:0> ] ) ) <EOL> self . _n_classes = probs . shape [ <NUM_LIT:1> ] <EOL> db_conn . close ( ) <EOL> def fit ( self , X , y ) : <EOL> """<STR_LIT>""" <EOL> self . fit_models ( X , y ) <EOL> self . build_ensemble ( X , y ) <EOL> return self <EOL> def fit_models ( self , X , y ) : <EOL> """<STR_LIT>""" <EOL> if ( self . verbose ) : <EOL> sys . stderr . write ( '<STR_LIT>' ) <EOL> if ( self . use_bootstrap ) : <EOL> n = X . shape [ <NUM_LIT:0> ] <EOL> rs = check_random_state ( self . random_state ) <EOL> self . _folds = [ _bootstraps ( n , rs ) for _ in xrange ( self . n_folds ) ] <EOL> else : <EOL> self . _folds = list ( StratifiedKFold ( y , n_folds = self . n_folds ) ) <EOL> select_stmt = "<STR_LIT>" <EOL> insert_stmt = """<STR_LIT>""" <EOL> db_conn = sqlite3 . connect ( self . db_file ) <EOL> curs = db_conn . cursor ( ) <EOL> for model_idx in xrange ( self . _n_models ) : <EOL> curs . execute ( select_stmt , [ model_idx ] ) <EOL> pickled_model = curs . fetchone ( ) [ <NUM_LIT:0> ] <EOL> model = loads ( str ( pickled_model ) ) <EOL> model_folds = [ ] <EOL> for fold_idx , fold in enumerate ( self . _folds ) : <EOL> train_inds , _ = fold <EOL> model . fit ( X [ train_inds ] , y [ train_inds ] ) <EOL> pickled_model = buffer ( dumps ( model ) ) <EOL> model_folds . append ( ( model_idx , fold_idx , pickled_model ) ) <EOL> with db_conn : <EOL> db_conn . executemany ( insert_stmt , model_folds ) <EOL> if ( self . verbose ) : <EOL> if ( ( model_idx + <NUM_LIT:1> ) % <NUM_LIT:50> == <NUM_LIT:0> ) : <EOL> sys . stderr . write ( '<STR_LIT>' % ( model_idx + <NUM_LIT:1> ) ) <EOL> else : <EOL> sys . stderr . write ( '<STR_LIT:.>' ) <EOL> if ( self . verbose ) : <EOL> sys . stderr . write ( '<STR_LIT:\n>' ) <EOL> with db_conn : <EOL> stmt = """<STR_LIT>""" <EOL> db_conn . execute ( stmt ) <EOL> db_conn . close ( ) <EOL> def _score_models ( self , db_conn , X , y , y_bin ) : <EOL> """<STR_LIT>""" <EOL> self . _metric = self . _metrics [ self . score_metric ] <EOL> if ( self . verbose ) : <EOL> sys . stderr . write ( '<STR_LIT>' ) <EOL> insert_stmt = """<STR_LIT>""" <EOL> select_stmt = """<STR_LIT>""" <EOL> with db_conn : <EOL> stmt = """<STR_LIT>""" <EOL> db_conn . executescript ( stmt ) <EOL> curs = db_conn . cursor ( ) <EOL> for model_idx in xrange ( self . _n_models ) : <EOL> probs = np . zeros ( ( len ( X ) , self . _n_classes ) ) <EOL> for fold_idx , fold in enumerate ( self . _folds ) : <EOL> _ , test_inds = fold <EOL> curs . execute ( select_stmt , [ model_idx , fold_idx ] ) <EOL> res = curs . fetchone ( ) <EOL> model = loads ( str ( res [ <NUM_LIT:0> ] ) ) <EOL> probs [ test_inds ] = model . predict_proba ( X [ test_inds ] ) <EOL> score = self . _metric ( y , y_bin , probs ) <EOL> with db_conn : <EOL> vals = ( model_idx , score , buffer ( dumps ( probs ) ) ) <EOL> db_conn . execute ( insert_stmt , vals ) <EOL> if ( self . verbose ) : <EOL> if ( ( model_idx + <NUM_LIT:1> ) % <NUM_LIT:50> == <NUM_LIT:0> ) : <EOL> sys . stderr . write ( '<STR_LIT>' % ( model_idx + <NUM_LIT:1> ) ) <EOL> else : <EOL> sys . stderr . write ( '<STR_LIT:.>' ) <EOL> if ( self . verbose ) : <EOL> sys . stderr . write ( '<STR_LIT:\n>' ) <EOL> with db_conn : <EOL> stmt = """<STR_LIT>""" <EOL> db_conn . execute ( stmt ) <EOL> def _get_ensemble_score ( self , db_conn , ensemble , y , y_bin ) : <EOL> """<STR_LIT>""" <EOL> n_models = float ( sum ( ensemble . values ( ) ) ) <EOL> ensemble_probs = np . zeros ( ( len ( y ) , self . _n_classes ) ) <EOL> curs = db_conn . cursor ( ) <EOL> select_stmt = """<STR_LIT>""" <EOL> in_str = str ( tuple ( ensemble ) ) . replace ( '<STR_LIT>' , '<STR_LIT:)>' ) <EOL> curs . execute ( select_stmt % in_str ) <EOL> for row in curs . fetchall ( ) : <EOL> model_idx , probs = row <EOL> probs = loads ( str ( probs ) ) <EOL> weight = ensemble [ model_idx ] <EOL> ensemble_probs += probs * weight <EOL> ensemble_probs /= n_models <EOL> score = self . _metric ( y , y_bin , ensemble_probs ) <EOL> return score , ensemble_probs <EOL> def _score_with_model ( self , db_conn , y , y_bin , probs , n_models , model_idx ) : <EOL> """<STR_LIT>""" <EOL> curs = db_conn . cursor ( ) <EOL> select_stmt = """<STR_LIT>""" <EOL> curs . execute ( select_stmt % model_idx ) <EOL> row = curs . fetchone ( ) <EOL> n_models = float ( n_models ) <EOL> new_probs = loads ( str ( row [ <NUM_LIT:0> ] ) ) <EOL> new_probs = ( probs * n_models + new_probs ) / ( n_models + <NUM_LIT:1.0> ) <EOL> score = self . _metric ( y , y_bin , new_probs ) <EOL> return score , new_probs <EOL> def _ensemble_from_candidates ( self , db_conn , candidates , y , y_bin ) : <EOL> """<STR_LIT>""" <EOL> ensemble = Counter ( candidates [ : self . n_best ] ) <EOL> ens_score , ens_probs = self . _get_ensemble_score ( db_conn , <EOL> ensemble , <EOL> y , y_bin ) <EOL> ens_count = sum ( ensemble . values ( ) ) <EOL> if ( self . verbose ) : <EOL> sys . stderr . write ( '<STR_LIT>' % ( ens_count , ens_score ) ) <EOL> cand_ensembles = [ ] <EOL> while ( ens_count < self . max_models ) : <EOL> new_scores = [ ] <EOL> for new_model_idx in candidates : <EOL> score , _ = self . _score_with_model ( db_conn , y , y_bin , <EOL> ens_probs , ens_count , <EOL> new_model_idx ) <EOL> new_scores . append ( { '<STR_LIT>' : score , <EOL> '<STR_LIT>' : new_model_idx } ) <EOL> new_scores . sort ( key = lambda x : x [ '<STR_LIT>' ] , reverse = True ) <EOL> last_ens_score = ens_score <EOL> ens_score = new_scores [ <NUM_LIT:0> ] [ '<STR_LIT>' ] <EOL> if ( self . use_epsilon ) : <EOL> score_diff = ens_score - last_ens_score <EOL> if ( score_diff < self . epsilon ) : <EOL> break <EOL> new_model_idx = new_scores [ <NUM_LIT:0> ] [ '<STR_LIT>' ] <EOL> ensemble . update ( { new_model_idx : <NUM_LIT:1> } ) <EOL> _ , ens_probs = self . _score_with_model ( db_conn , y , y_bin , <EOL> ens_probs , ens_count , <EOL> new_model_idx ) <EOL> if ( not self . use_epsilon ) : <EOL> ens_copy = Counter ( ensemble ) <EOL> cand = { '<STR_LIT>' : ens_copy , '<STR_LIT>' : ens_score } <EOL> cand_ensembles . append ( cand ) <EOL> ens_count = sum ( ensemble . values ( ) ) <EOL> if ( self . verbose ) : <EOL> if ( ( ens_count - self . n_best ) % <NUM_LIT:8> == <NUM_LIT:0> ) : <EOL> sys . stderr . write ( "<STR_LIT>" ) <EOL> msg = '<STR_LIT>' % ( ens_count , ens_score ) <EOL> sys . stderr . write ( msg ) <EOL> if ( self . verbose ) : <EOL> sys . stderr . write ( '<STR_LIT:\n>' ) <EOL> if ( not self . use_epsilon and ens_count == self . max_models ) : <EOL> cand_ensembles . sort ( key = lambda x : x [ '<STR_LIT>' ] , reverse = True ) <EOL> ensemble = cand_ensembles [ <NUM_LIT:0> ] [ '<STR_LIT>' ] <EOL> return ensemble <EOL> def _get_best_model ( self , curs ) : <EOL> """<STR_LIT>""" <EOL> select_stmt = """<STR_LIT>""" <EOL> curs . execute ( select_stmt ) <EOL> row = curs . fetchone ( ) <EOL> return row [ <NUM_LIT:0> ] , loads ( str ( row [ <NUM_LIT:1> ] ) ) <EOL> def best_model ( self ) : <EOL> """<STR_LIT>""" <EOL> db_conn = sqlite3 . connect ( self . db_file ) <EOL> _ , model = self . _get_best_model ( db_conn . cursor ( ) ) <EOL> db_conn . close ( ) <EOL> return model <EOL> def _print_best_results ( self , curs , best_model_score ) : <EOL> """<STR_LIT>""" <EOL> sys . stderr . write ( '<STR_LIT>' % best_model_score ) <EOL> _ , best_model = self . _get_best_model ( curs ) <EOL> sys . stderr . write ( '<STR_LIT>' % repr ( best_model ) ) <EOL> def build_ensemble ( self , X , y , rescore = True ) : <EOL> """<STR_LIT>""" <EOL> self . _n_classes = len ( np . unique ( y ) ) <EOL> db_conn = sqlite3 . connect ( self . db_file ) <EOL> curs = db_conn . cursor ( ) <EOL> if ( self . _n_classes > <NUM_LIT:2> ) : <EOL> y_bin = LabelBinarizer ( ) . fit_transform ( y ) <EOL> else : <EOL> y_bin = np . column_stack ( ( <NUM_LIT:1> - y , y ) ) <EOL> if ( rescore ) : <EOL> self . _score_models ( db_conn , X , y , y_bin ) <EOL> n_models = int ( self . _n_models * ( <NUM_LIT:1.0> - self . prune_fraction ) ) <EOL> bag_size = int ( self . bag_fraction * n_models ) <EOL> if ( self . verbose ) : <EOL> sys . stderr . write ( '<STR_LIT>' % n_models ) <EOL> sys . stderr . write ( '<STR_LIT>' % bag_size ) <EOL> select_stmt = """<STR_LIT>""" <EOL> curs . execute ( select_stmt % n_models ) <EOL> ranked_model_scores = [ ( r [ <NUM_LIT:0> ] , r [ <NUM_LIT:1> ] ) for r in curs . fetchall ( ) ] <EOL> best_model_score = ranked_model_scores [ <NUM_LIT:0> ] [ <NUM_LIT:1> ] <EOL> if ( self . verbose ) : <EOL> self . _print_best_results ( curs , best_model_score ) <EOL> sys . stderr . write ( "<STR_LIT>" ) <EOL> ensembles = [ ] <EOL> rs = check_random_state ( self . random_state ) <EOL> for i in xrange ( self . n_bags ) : <EOL> cand_indices = rs . permutation ( n_models ) [ : bag_size ] <EOL> candidates = [ ranked_model_scores [ ci ] [ <NUM_LIT:0> ] for ci in cand_indices ] <EOL> if ( self . verbose ) : <EOL> sys . stderr . write ( '<STR_LIT>' % ( i + <NUM_LIT:1> ) ) <EOL> ensemble = self . _ensemble_from_candidates ( db_conn , <EOL> candidates , <EOL> y , y_bin ) <EOL> ensembles . append ( ensemble ) <EOL> for e in ensembles : <EOL> self . _ensemble += e <EOL> insert_stmt = "<STR_LIT>" <EOL> with db_conn : <EOL> val_gen = ( ( mi , w ) for mi , w in self . _ensemble . most_common ( ) ) <EOL> db_conn . executemany ( insert_stmt , val_gen ) <EOL> if ( self . verbose ) : <EOL> score , _ = self . _get_ensemble_score ( db_conn , <EOL> self . _ensemble , <EOL> y , y_bin ) <EOL> fmt = "<STR_LIT>" <EOL> sys . stderr . write ( fmt % ( sum ( self . _ensemble . values ( ) ) , score ) ) <EOL> db_conn . close ( ) <EOL> def _model_predict_proba ( self , X , model_idx = <NUM_LIT:0> ) : <EOL> """<STR_LIT>""" <EOL> db_conn = sqlite3 . connect ( self . db_file ) <EOL> curs = db_conn . cursor ( ) <EOL> select_stmt = """<STR_LIT>""" <EOL> probs = np . zeros ( ( len ( X ) , self . _n_classes ) ) <EOL> for fold_idx in xrange ( self . n_folds ) : <EOL> curs . execute ( select_stmt , [ model_idx , fold_idx ] ) <EOL> res = curs . fetchone ( ) <EOL> model = loads ( str ( res [ <NUM_LIT:0> ] ) ) <EOL> probs += model . predict_proba ( X ) / float ( self . n_folds ) <EOL> db_conn . close ( ) <EOL> return probs <EOL> def best_model_predict_proba ( self , X ) : <EOL> """<STR_LIT>""" <EOL> db_conn = sqlite3 . connect ( self . db_file ) <EOL> best_model_idx , _ = self . _get_best_model ( db_conn . cursor ( ) ) <EOL> db_conn . close ( ) <EOL> return self . _model_predict_proba ( X , best_model_idx ) <EOL> def best_model_predict ( self , X ) : <EOL> """<STR_LIT>""" <EOL> return np . argmax ( self . best_model_predict_proba ( X ) , axis = <NUM_LIT:1> ) <EOL> def predict_proba ( self , X ) : <EOL> """<STR_LIT>""" <EOL> n_models = float ( sum ( self . _ensemble . values ( ) ) ) <EOL> probs = np . zeros ( ( len ( X ) , self . _n_classes ) ) <EOL> for model_idx , weight in self . _ensemble . items ( ) : <EOL> probs += self . _model_predict_proba ( X , model_idx ) * weight / n_models <EOL> return probs <EOL> def predict ( self , X ) : <EOL> """<STR_LIT>""" <EOL> return np . argmax ( self . predict_proba ( X ) , axis = <NUM_LIT:1> ) </s>
