<s> """<STR_LIT>""" <EOL> import warnings <EOL> from operator import itemgetter <EOL> import numpy as np <EOL> from scipy . linalg import cholesky , cho_solve , solve <EOL> from scipy . optimize import fmin_l_bfgs_b <EOL> from scipy . special import erf <EOL> from sklearn . base import BaseEstimator , ClassifierMixin , clone <EOL> from sklearn . gaussian_process . kernels import RBF , CompoundKernel , ConstantKernel as C <EOL> from sklearn . utils . validation import check_X_y , check_is_fitted , check_array <EOL> from sklearn . utils import check_random_state <EOL> from sklearn . preprocessing import LabelEncoder <EOL> from sklearn . multiclass import OneVsRestClassifier , OneVsOneClassifier <EOL> LAMBDAS = np . array ( [ <NUM_LIT> , <NUM_LIT> , <NUM_LIT> , <NUM_LIT> , <NUM_LIT> ] ) [ : , np . newaxis ] <EOL> COEFS = np . array ( [ - <NUM_LIT> , <NUM_LIT> , <NUM_LIT> , <EOL> <NUM_LIT> , - <NUM_LIT> ] ) [ : , np . newaxis ] <EOL> class _BinaryGaussianProcessClassifierLaplace ( BaseEstimator ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , kernel = None , optimizer = "<STR_LIT>" , <EOL> n_restarts_optimizer = <NUM_LIT:0> , max_iter_predict = <NUM_LIT:100> , <EOL> warm_start = False , copy_X_train = True , random_state = None ) : <EOL> self . kernel = kernel <EOL> self . optimizer = optimizer <EOL> self . n_restarts_optimizer = n_restarts_optimizer <EOL> self . max_iter_predict = max_iter_predict <EOL> self . warm_start = warm_start <EOL> self . copy_X_train = copy_X_train <EOL> self . random_state = random_state <EOL> def fit ( self , X , y ) : <EOL> """<STR_LIT>""" <EOL> if self . kernel is None : <EOL> self . kernel_ = C ( <NUM_LIT:1.0> , constant_value_bounds = "<STR_LIT>" ) * RBF ( <NUM_LIT:1.0> , length_scale_bounds = "<STR_LIT>" ) <EOL> else : <EOL> self . kernel_ = clone ( self . kernel ) <EOL> self . rng = check_random_state ( self . random_state ) <EOL> self . X_train_ = np . copy ( X ) if self . copy_X_train else X <EOL> label_encoder = LabelEncoder ( ) <EOL> self . y_train_ = label_encoder . fit_transform ( y ) <EOL> self . classes_ = label_encoder . classes_ <EOL> if self . classes_ . size > <NUM_LIT:2> : <EOL> raise ValueError ( "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> % ( self . __class__ . __name__ , self . classes_ ) ) <EOL> elif self . classes_ . size == <NUM_LIT:1> : <EOL> raise ValueError ( "<STR_LIT>" . format ( <EOL> self . __class__ . __name__ ) ) <EOL> if self . optimizer is not None and self . kernel_ . n_dims > <NUM_LIT:0> : <EOL> def obj_func ( theta , eval_gradient = True ) : <EOL> if eval_gradient : <EOL> lml , grad = self . log_marginal_likelihood ( <EOL> theta , eval_gradient = True ) <EOL> return - lml , - grad <EOL> else : <EOL> return - self . log_marginal_likelihood ( theta ) <EOL> optima = [ self . _constrained_optimization ( obj_func , <EOL> self . kernel_ . theta , <EOL> self . kernel_ . bounds ) ] <EOL> if self . n_restarts_optimizer > <NUM_LIT:0> : <EOL> if not np . isfinite ( self . kernel_ . bounds ) . all ( ) : <EOL> raise ValueError ( <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" ) <EOL> bounds = self . kernel_ . bounds <EOL> for iteration in range ( self . n_restarts_optimizer ) : <EOL> theta_initial = np . exp ( self . rng . uniform ( bounds [ : , <NUM_LIT:0> ] , <EOL> bounds [ : , <NUM_LIT:1> ] ) ) <EOL> optima . append ( <EOL> self . _constrained_optimization ( obj_func , theta_initial , <EOL> bounds ) ) <EOL> lml_values = list ( map ( itemgetter ( <NUM_LIT:1> ) , optima ) ) <EOL> self . kernel_ . theta = optima [ np . argmin ( lml_values ) ] [ <NUM_LIT:0> ] <EOL> self . log_marginal_likelihood_value_ = - np . min ( lml_values ) <EOL> else : <EOL> self . log_marginal_likelihood_value_ = self . log_marginal_likelihood ( self . kernel_ . theta ) <EOL> K = self . kernel_ ( self . X_train_ ) <EOL> _ , ( self . pi_ , self . W_sr_ , self . L_ , _ , _ ) = self . _posterior_mode ( K , return_temporaries = True ) <EOL> return self <EOL> def predict ( self , X ) : <EOL> """<STR_LIT>""" <EOL> check_is_fitted ( self , [ "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" ] ) <EOL> K_star = self . kernel_ ( self . X_train_ , X ) <EOL> f_star = K_star . T . dot ( self . y_train_ - self . pi_ ) <EOL> return np . where ( f_star > <NUM_LIT:0> , self . classes_ [ <NUM_LIT:1> ] , self . classes_ [ <NUM_LIT:0> ] ) <EOL> def predict_proba ( self , X ) : <EOL> """<STR_LIT>""" <EOL> check_is_fitted ( self , [ "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" ] ) <EOL> K_star = self . kernel_ ( self . X_train_ , X ) <EOL> f_star = K_star . T . dot ( self . y_train_ - self . pi_ ) <EOL> v = solve ( self . L_ , self . W_sr_ [ : , np . newaxis ] * K_star ) <EOL> var_f_star = self . kernel_ . diag ( X ) - np . einsum ( "<STR_LIT>" , v , v ) <EOL> alpha = <NUM_LIT:1> / ( <NUM_LIT:2> * var_f_star ) <EOL> gamma = LAMBDAS * f_star <EOL> integrals = np . sqrt ( np . pi / alpha ) * erf ( gamma * np . sqrt ( alpha / ( alpha + LAMBDAS ** <NUM_LIT:2> ) ) ) / ( <NUM_LIT:2> * np . sqrt ( var_f_star * <NUM_LIT:2> * np . pi ) ) <EOL> pi_star = ( COEFS * integrals ) . sum ( axis = <NUM_LIT:0> ) + <NUM_LIT> * COEFS . sum ( ) <EOL> return np . vstack ( ( <NUM_LIT:1> - pi_star , pi_star ) ) . T <EOL> def log_marginal_likelihood ( self , theta = None , eval_gradient = False ) : <EOL> """<STR_LIT>""" <EOL> if theta is None : <EOL> if eval_gradient : <EOL> raise ValueError ( <EOL> "<STR_LIT>" ) <EOL> return self . log_marginal_likelihood_value_ <EOL> kernel = self . kernel_ . clone_with_theta ( theta ) <EOL> if eval_gradient : <EOL> K , K_gradient = kernel ( self . X_train_ , eval_gradient = True ) <EOL> else : <EOL> K = kernel ( self . X_train_ ) <EOL> Z , ( pi , W_sr , L , b , a ) = self . _posterior_mode ( K , return_temporaries = True ) <EOL> if not eval_gradient : <EOL> return Z <EOL> d_Z = np . empty ( theta . shape [ <NUM_LIT:0> ] ) <EOL> R = W_sr [ : , np . newaxis ] * cho_solve ( ( L , True ) , np . diag ( W_sr ) ) <EOL> C = solve ( L , W_sr [ : , np . newaxis ] * K ) <EOL> s_2 = - <NUM_LIT:0.5> * ( np . diag ( K ) - np . einsum ( '<STR_LIT>' , C , C ) ) * ( pi * ( <NUM_LIT:1> - pi ) * ( <NUM_LIT:1> - <NUM_LIT:2> * pi ) ) <EOL> for j in range ( d_Z . shape [ <NUM_LIT:0> ] ) : <EOL> C = K_gradient [ : , : , j ] <EOL> s_1 = <NUM_LIT> * a . T . dot ( C ) . dot ( a ) - <NUM_LIT> * R . T . ravel ( ) . dot ( C . ravel ( ) ) <EOL> b = C . dot ( self . y_train_ - pi ) <EOL> s_3 = b - K . dot ( R . dot ( b ) ) <EOL> d_Z [ j ] = s_1 + s_2 . T . dot ( s_3 ) <EOL> return Z , d_Z <EOL> def _posterior_mode ( self , K , return_temporaries = False ) : <EOL> """<STR_LIT>""" <EOL> if self . warm_start and hasattr ( self , "<STR_LIT>" ) and self . f_cached . shape == self . y_train_ . shape : <EOL> f = self . f_cached <EOL> else : <EOL> f = np . zeros_like ( self . y_train_ , dtype = np . float64 ) <EOL> log_marginal_likelihood = - np . inf <EOL> for _ in range ( self . max_iter_predict ) : <EOL> pi = <NUM_LIT:1> / ( <NUM_LIT:1> + np . exp ( - f ) ) <EOL> W = pi * ( <NUM_LIT:1> - pi ) <EOL> W_sr = np . sqrt ( W ) <EOL> W_sr_K = W_sr [ : , np . newaxis ] * K <EOL> B = np . eye ( W . shape [ <NUM_LIT:0> ] ) + W_sr_K * W_sr <EOL> L = cholesky ( B , lower = True ) <EOL> b = W * f + ( self . y_train_ - pi ) <EOL> a = b - W_sr * cho_solve ( ( L , True ) , W_sr_K . dot ( b ) ) <EOL> f = K . dot ( a ) <EOL> lml = - <NUM_LIT:0.5> * a . T . dot ( f ) - np . log ( <NUM_LIT:1> + np . exp ( - ( self . y_train_ * <NUM_LIT:2> - <NUM_LIT:1> ) * f ) ) . sum ( ) - np . log ( np . diag ( L ) ) . sum ( ) <EOL> if lml - log_marginal_likelihood < <NUM_LIT> : <EOL> break <EOL> log_marginal_likelihood = lml <EOL> self . f_cached = f <EOL> if return_temporaries : <EOL> return log_marginal_likelihood , ( pi , W_sr , L , b , a ) <EOL> else : <EOL> return log_marginal_likelihood <EOL> def _constrained_optimization ( self , obj_func , initial_theta , bounds ) : <EOL> if self . optimizer == "<STR_LIT>" : <EOL> theta_opt , func_min , convergence_dict = fmin_l_bfgs_b ( obj_func , initial_theta , bounds = bounds ) <EOL> if convergence_dict [ "<STR_LIT>" ] != <NUM_LIT:0> : <EOL> warnings . warn ( "<STR_LIT>" <EOL> "<STR_LIT>" % convergence_dict ) <EOL> elif callable ( self . optimizer ) : <EOL> theta_opt , func_min = self . optimizer ( obj_func , initial_theta , bounds = bounds ) <EOL> else : <EOL> raise ValueError ( "<STR_LIT>" % self . optimizer ) <EOL> return theta_opt , func_min <EOL> class GaussianProcessClassifier ( BaseEstimator , ClassifierMixin ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , kernel = None , optimizer = "<STR_LIT>" , <EOL> n_restarts_optimizer = <NUM_LIT:0> , max_iter_predict = <NUM_LIT:100> , <EOL> warm_start = False , copy_X_train = True , random_state = None , <EOL> multi_class = "<STR_LIT>" , n_jobs = <NUM_LIT:1> ) : <EOL> self . kernel = kernel <EOL> self . optimizer = optimizer <EOL> self . n_restarts_optimizer = n_restarts_optimizer <EOL> self . max_iter_predict = max_iter_predict <EOL> self . warm_start = warm_start <EOL> self . copy_X_train = copy_X_train <EOL> self . random_state = random_state <EOL> self . multi_class = multi_class <EOL> self . n_jobs = n_jobs <EOL> def fit ( self , X , y ) : <EOL> """<STR_LIT>""" <EOL> X , y = check_X_y ( X , y , multi_output = False ) <EOL> self . base_estimator_ = _BinaryGaussianProcessClassifierLaplace ( <EOL> self . kernel , self . optimizer , self . n_restarts_optimizer , <EOL> self . max_iter_predict , self . warm_start , self . copy_X_train , <EOL> self . random_state ) <EOL> self . classes_ = np . unique ( y ) <EOL> self . n_classes_ = self . classes_ . size <EOL> if self . n_classes_ == <NUM_LIT:1> : <EOL> raise ValueError ( "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> % self . classes_ [ <NUM_LIT:0> ] ) <EOL> if self . n_classes_ > <NUM_LIT:2> : <EOL> if self . multi_class == "<STR_LIT>" : <EOL> self . base_estimator_ = OneVsRestClassifier ( self . base_estimator_ , <EOL> n_jobs = self . n_jobs ) <EOL> elif self . multi_class == "<STR_LIT>" : <EOL> self . base_estimator_ = OneVsOneClassifier ( self . base_estimator_ , <EOL> n_jobs = self . n_jobs ) <EOL> else : <EOL> raise ValueError ( "<STR_LIT>" <EOL> % self . multi_class ) <EOL> self . base_estimator_ . fit ( X , y ) <EOL> if self . n_classes_ > <NUM_LIT:2> : <EOL> self . log_marginal_likelihood_value_ = np . mean ( <EOL> [ estimator . log_marginal_likelihood ( ) <EOL> for estimator in self . base_estimator_ . estimators_ ] ) <EOL> else : <EOL> self . log_marginal_likelihood_value_ = self . base_estimator_ . log_marginal_likelihood ( ) <EOL> return self <EOL> def predict ( self , X ) : <EOL> """<STR_LIT>""" <EOL> check_is_fitted ( self , [ "<STR_LIT>" , "<STR_LIT>" ] ) <EOL> X = check_array ( X ) <EOL> return self . base_estimator_ . predict ( X ) <EOL> def predict_proba ( self , X ) : <EOL> """<STR_LIT>""" <EOL> check_is_fitted ( self , [ "<STR_LIT>" , "<STR_LIT>" ] ) <EOL> if self . n_classes_ > <NUM_LIT:2> and self . multi_class == "<STR_LIT>" : <EOL> raise ValueError ( "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" ) <EOL> X = check_array ( X ) <EOL> return self . base_estimator_ . predict_proba ( X ) <EOL> @ property <EOL> def kernel_ ( self ) : <EOL> if self . n_classes_ == <NUM_LIT:2> : <EOL> return self . base_estimator_ . kernel_ <EOL> else : <EOL> return CompoundKernel ( <EOL> [ estimator . kernel_ <EOL> for estimator in self . base_estimator_ . estimators_ ] ) <EOL> def log_marginal_likelihood ( self , theta = None , eval_gradient = False ) : <EOL> """<STR_LIT>""" <EOL> check_is_fitted ( self , [ "<STR_LIT>" , "<STR_LIT>" ] ) <EOL> if theta is None : <EOL> if eval_gradient : <EOL> raise ValueError ( <EOL> "<STR_LIT>" ) <EOL> return self . log_marginal_likelihood_value_ <EOL> theta = np . asarray ( theta ) <EOL> if self . n_classes_ == <NUM_LIT:2> : <EOL> return self . base_estimator_ . log_marginal_likelihood ( <EOL> theta , eval_gradient ) <EOL> else : <EOL> if eval_gradient : <EOL> raise NotImplementedError ( <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" ) <EOL> estimators = self . base_estimator_ . estimators_ <EOL> n_dims = estimators [ <NUM_LIT:0> ] . kernel_ . n_dims <EOL> if theta . shape [ <NUM_LIT:0> ] == n_dims : <EOL> return np . mean ( <EOL> [ estimator . log_marginal_likelihood ( theta ) <EOL> for i , estimator in enumerate ( estimators ) ] ) <EOL> elif theta . shape [ <NUM_LIT:0> ] == n_dims * self . classes_ . shape [ <NUM_LIT:0> ] : <EOL> return np . mean ( <EOL> [ estimator . log_marginal_likelihood ( <EOL> theta [ n_dims * i : n_dims * ( i + <NUM_LIT:1> ) ] ) <EOL> for i , estimator in enumerate ( estimators ) ] ) <EOL> else : <EOL> raise ValueError ( "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> % ( n_dims , n_dims * self . classes_ . shape [ <NUM_LIT:0> ] , <EOL> theta . shape [ <NUM_LIT:0> ] ) ) </s>
