<s> import numpy as np <EOL> import sklearn . linear_model <EOL> import sklearn . ensemble as en <EOL> from sklearn . grid_search import GridSearchCV <EOL> import sklearn <EOL> from sklearn . linear_model import LinearRegression <EOL> import scipy as sp <EOL> from regression import linreg_on_fold <EOL> import sklearn <EOL> import sklearn . tree as tree <EOL> from sklearn import svm <EOL> from sklearn . grid_search import GridSearchCV <EOL> from sklearn . cross_validation import cross_val_score <EOL> def spearman_scoring ( clf , X , y ) : <EOL> y_pred = clf . predict ( X ) . flatten ( ) <EOL> return sp . stats . spearmanr ( y_pred , y . flatten ( ) ) [ <NUM_LIT:0> ] <EOL> def adaboost_on_fold ( feature_sets , train , test , y , y_all , X , dim , dimsum , learn_options , classification = False ) : <EOL> '''<STR_LIT>''' <EOL> if learn_options [ '<STR_LIT>' ] == '<STR_LIT>' : <EOL> if not learn_options [ '<STR_LIT>' ] : <EOL> if not classification : <EOL> clf = en . GradientBoostingRegressor ( loss = learn_options [ '<STR_LIT>' ] , learning_rate = learn_options [ '<STR_LIT>' ] , <EOL> n_estimators = learn_options [ '<STR_LIT>' ] , <EOL> alpha = learn_options [ '<STR_LIT>' ] , <EOL> subsample = <NUM_LIT:1.0> , min_samples_split = <NUM_LIT:2> , min_samples_leaf = <NUM_LIT:1> , max_depth = learn_options [ '<STR_LIT>' ] , <EOL> init = None , random_state = None , max_features = None , <EOL> verbose = <NUM_LIT:0> , max_leaf_nodes = None , warm_start = False ) <EOL> else : <EOL> clf = en . GradientBoostingClassifier ( learning_rate = learn_options [ '<STR_LIT>' ] , <EOL> n_estimators = learn_options [ '<STR_LIT>' ] , <EOL> subsample = <NUM_LIT:1.0> , min_samples_split = <NUM_LIT:2> , min_samples_leaf = <NUM_LIT:1> , max_depth = learn_options [ '<STR_LIT>' ] , <EOL> init = None , random_state = None , max_features = None , <EOL> verbose = <NUM_LIT:0> , max_leaf_nodes = None , warm_start = False ) <EOL> clf . fit ( X [ train ] , y [ train ] . flatten ( ) ) <EOL> y_pred = clf . predict ( X [ test ] ) [ : , None ] <EOL> else : <EOL> if learn_options [ "<STR_LIT>" ] == "<STR_LIT>" : <EOL> print <EOL> from hyperopt import hp , fmin , tpe , rand <EOL> def adaboost_scoring_bo ( params ) : <EOL> cv = sklearn . cross_validation . KFold ( y_all [ '<STR_LIT>' ] . values [ train ] . shape [ <NUM_LIT:0> ] , n_folds = <NUM_LIT:20> , shuffle = True ) <EOL> est = en . GradientBoostingRegressor ( n_estimators = <NUM_LIT:1000> , learning_rate = params [ '<STR_LIT>' ] , max_depth = params [ '<STR_LIT>' ] , <EOL> min_samples_leaf = params [ '<STR_LIT>' ] , max_features = params [ '<STR_LIT>' ] ) <EOL> scorer = cross_val_score ( est , X [ train ] , y [ train ] . flatten ( ) , cv = cv , n_jobs = <NUM_LIT:20> ) <EOL> return np . median ( scorer ) <EOL> space = { <EOL> '<STR_LIT>' : hp . uniform ( '<STR_LIT>' , <NUM_LIT> , <NUM_LIT:0.1> ) , <EOL> '<STR_LIT>' : hp . quniform ( '<STR_LIT>' , <NUM_LIT:1> , <NUM_LIT:8> , <NUM_LIT:1> ) , <EOL> '<STR_LIT>' : hp . quniform ( '<STR_LIT>' , <NUM_LIT:3> , <NUM_LIT:20> , <NUM_LIT:1> ) , <EOL> '<STR_LIT>' : hp . uniform ( '<STR_LIT>' , <NUM_LIT> , <NUM_LIT:1.0> ) } <EOL> best = fmin ( adaboost_scoring_bo , space , algo = tpe . suggest , max_evals = <NUM_LIT:50> , verbose = <NUM_LIT:1> ) <EOL> print best <EOL> clf = en . GradientBoostingRegressor ( n_estimators = learn_options [ '<STR_LIT>' ] , <EOL> learning_rate = best [ '<STR_LIT>' ] , <EOL> max_depth = best [ '<STR_LIT>' ] , <EOL> min_samples_leaf = best [ '<STR_LIT>' ] , <EOL> max_features = best [ '<STR_LIT>' ] ) <EOL> clf . fit ( X [ train ] , y [ train ] . flatten ( ) ) <EOL> elif learn_options [ "<STR_LIT>" ] == "<STR_LIT>" : <EOL> assert not classification , "<STR_LIT>" <EOL> n_jobs = <NUM_LIT:20> <EOL> print "<STR_LIT>" <EOL> from sklearn . grid_search import GridSearchCV <EOL> param_grid = { '<STR_LIT>' : [ <NUM_LIT:0.1> , <NUM_LIT> ] , <EOL> '<STR_LIT>' : [ <NUM_LIT:4> , <NUM_LIT:7> ] , <EOL> '<STR_LIT>' : [ <NUM_LIT:5> , <NUM_LIT:15> ] , <EOL> '<STR_LIT>' : [ <NUM_LIT:1.0> , <NUM_LIT:0.1> ] } <EOL> label_encoder = sklearn . preprocessing . LabelEncoder ( ) <EOL> label_encoder . fit ( y_all [ '<STR_LIT>' ] . values [ train ] ) <EOL> gene_classes = label_encoder . transform ( y_all [ '<STR_LIT>' ] . values [ train ] ) <EOL> n_folds = len ( np . unique ( gene_classes ) ) <EOL> cv = sklearn . cross_validation . StratifiedKFold ( gene_classes , n_folds = n_folds , shuffle = True ) <EOL> est = en . GradientBoostingRegressor ( loss = learn_options [ '<STR_LIT>' ] , n_estimators = learn_options [ '<STR_LIT>' ] ) <EOL> clf = GridSearchCV ( est , param_grid , n_jobs = n_jobs , verbose = <NUM_LIT:1> , cv = cv , scoring = spearman_scoring , iid = False ) . fit ( X [ train ] , y [ train ] . flatten ( ) ) <EOL> print clf . best_params_ <EOL> else : <EOL> raise Exception ( "<STR_LIT>" ) <EOL> y_pred = clf . predict ( X [ test ] ) [ : , None ] <EOL> else : <EOL> raise NotImplementedError <EOL> return y_pred , clf <EOL> def LASSOs_ensemble_on_fold ( feature_sets , train , test , y , y_all , X , dim , dimsum , learn_options ) : <EOL> train_indices = np . where ( train ) [ <NUM_LIT:0> ] <EOL> sel = len ( train_indices ) * <NUM_LIT> <EOL> permuted_ind = np . random . permutation ( train_indices ) <EOL> valid_indices = permuted_ind [ : sel ] <EOL> train_indices = permuted_ind [ sel : ] <EOL> train_sub = np . zeros_like ( train , dtype = bool ) <EOL> valid_sub = np . zeros_like ( train , dtype = bool ) <EOL> train_sub [ train_indices ] = True <EOL> valid_sub [ valid_indices ] = True <EOL> validations = np . zeros ( ( len ( valid_indices ) , len ( feature_sets . keys ( ) ) ) ) <EOL> predictions = np . zeros ( ( test . sum ( ) , len ( feature_sets . keys ( ) ) ) ) <EOL> for i , feature_name in enumerate ( feature_sets . keys ( ) ) : <EOL> X_feature = feature_sets [ feature_name ] . values <EOL> y_pred , m = linreg_on_fold ( feature_sets , train_sub , valid_sub , y , y_all , X_feature , dim , dimsum , learn_options ) <EOL> predictions [ : , i ] = m . predict ( X_feature [ test ] ) . flatten ( ) <EOL> validations [ : , i ] = y_pred . flatten ( ) <EOL> clf = LinearRegression ( ) <EOL> clf . fit ( validations , y [ valid_sub ] ) <EOL> y_pred = clf . predict ( predictions ) <EOL> return y_pred , None <EOL> def randomforest_on_fold ( feature_sets , train , test , y , y_all , X , dim , dimsum , learn_options ) : <EOL> '''<STR_LIT>''' <EOL> clf = en . RandomForestRegressor ( oob_score = True ) <EOL> clf . fit ( X [ train ] , y [ train ] [ : , <NUM_LIT:0> ] ) <EOL> y_pred = clf . predict ( X [ test ] ) [ : , None ] <EOL> return y_pred , clf <EOL> def decisiontree_on_fold ( feature_sets , train , test , y , y_all , X , dim , dimsum , learn_options ) : <EOL> '''<STR_LIT>''' <EOL> clf = tree . DecisionTreeRegressor ( ) <EOL> clf . fit ( X [ train ] , y [ train ] [ : , <NUM_LIT:0> ] ) <EOL> y_pred = clf . predict ( X [ test ] ) [ : , None ] <EOL> return y_pred , clf <EOL> def linear_stacking ( y_train , X_train , X_test ) : <EOL> clf = sklearn . linear_model . LinearRegression ( ) <EOL> clf . fit ( X_train , y_train ) <EOL> y_pred = clf . predict ( X_test ) <EOL> return y_pred . flatten ( ) <EOL> def pairwise_majority_voting ( y ) : <EOL> N = y . shape [ <NUM_LIT:0> ] <EOL> y_pred = np . zeros ( ( N , N ) ) <EOL> for i in range ( N ) : <EOL> for j in range ( N ) : <EOL> if i == j : <EOL> continue <EOL> y_pred [ i , j ] = ( y [ i ] > y [ j ] ) . sum ( ) > y . shape [ <NUM_LIT:1> ] / <NUM_LIT:2> <EOL> return y_pred . sum ( <NUM_LIT:1> ) / y_pred . sum ( <NUM_LIT:1> ) . max ( ) <EOL> def median ( y ) : <EOL> return np . median ( y , axis = <NUM_LIT:1> ) <EOL> def GBR_stacking ( y_train , X_train , X_test ) : <EOL> param_grid = { '<STR_LIT>' : [ <NUM_LIT:0.1> , <NUM_LIT> , <NUM_LIT> ] , <EOL> '<STR_LIT>' : [ <NUM_LIT:2> , <NUM_LIT:3> , <NUM_LIT:4> , <NUM_LIT:5> ] , <EOL> '<STR_LIT>' : [ <NUM_LIT:1> , <NUM_LIT:2> , <NUM_LIT:3> ] , <EOL> '<STR_LIT>' : [ <NUM_LIT:1.0> , <NUM_LIT:0.5> , <NUM_LIT> , <NUM_LIT:0.1> ] } <EOL> est = en . GradientBoostingRegressor ( loss = '<STR_LIT>' , n_estimators = <NUM_LIT:100> ) <EOL> clf = GridSearchCV ( est , param_grid , n_jobs = <NUM_LIT:3> , verbose = <NUM_LIT:1> , cv = <NUM_LIT:20> , scoring = spearman_scoring ) . fit ( X_train , y_train . flatten ( ) ) <EOL> return clf . predict ( X_test ) <EOL> def GP_stacking ( y_train , X_train , X_test ) : <EOL> import GPy <EOL> m = GPy . models . SparseGPRegression ( X_train , y_train , num_inducing = <NUM_LIT:20> , kernel = GPy . kern . RBF ( X_train . shape [ <NUM_LIT:1> ] ) ) <EOL> m . optimize ( '<STR_LIT>' , messages = <NUM_LIT:0> ) <EOL> y_pred = m . predict ( X_test ) [ <NUM_LIT:0> ] <EOL> return y_pred . flatten ( ) <EOL> def SVM_stacking ( y_train , X_train , X_test ) : <EOL> parameters = { '<STR_LIT>' : ( '<STR_LIT>' , '<STR_LIT>' ) , '<STR_LIT:C>' : np . linspace ( <NUM_LIT:1> , <NUM_LIT:10> , <NUM_LIT:10> ) , '<STR_LIT>' : np . linspace ( <NUM_LIT> , <NUM_LIT:1.> , <NUM_LIT:10> ) } <EOL> svr = svm . SVR ( ) <EOL> clf = GridSearchCV ( svr , parameters , n_jobs = <NUM_LIT:3> , verbose = <NUM_LIT:1> , cv = <NUM_LIT:10> , scoring = spearman_scoring ) <EOL> clf . fit ( X_train , y_train . flatten ( ) ) <EOL> return clf . predict ( X_test ) </s>
