<s> import numpy as np <EOL> from scipy import sparse <EOL> from sklearn . utils . testing import ( assert_array_almost_equal , assert_equal , <EOL> assert_greater , assert_almost_equal , <EOL> assert_greater_equal , <EOL> assert_array_equal , <EOL> assert_raises , <EOL> ignore_warnings , <EOL> assert_warns_message ) <EOL> from sklearn . datasets import make_classification , make_blobs <EOL> from sklearn . naive_bayes import MultinomialNB <EOL> from sklearn . ensemble import RandomForestClassifier , RandomForestRegressor <EOL> from sklearn . svm import LinearSVC <EOL> from sklearn . linear_model import Ridge <EOL> from sklearn . pipeline import Pipeline <EOL> from sklearn . preprocessing import Imputer <EOL> from sklearn . metrics import brier_score_loss , log_loss <EOL> from sklearn . calibration import CalibratedClassifierCV <EOL> from sklearn . calibration import _sigmoid_calibration , _SigmoidCalibration <EOL> from sklearn . calibration import calibration_curve <EOL> @ ignore_warnings <EOL> def test_calibration ( ) : <EOL> """<STR_LIT>""" <EOL> n_samples = <NUM_LIT:100> <EOL> X , y = make_classification ( n_samples = <NUM_LIT:2> * n_samples , n_features = <NUM_LIT:6> , <EOL> random_state = <NUM_LIT> ) <EOL> sample_weight = np . random . RandomState ( seed = <NUM_LIT> ) . uniform ( size = y . size ) <EOL> X -= X . min ( ) <EOL> X_train , y_train , sw_train = X [ : n_samples ] , y [ : n_samples ] , sample_weight [ : n_samples ] <EOL> X_test , y_test = X [ n_samples : ] , y [ n_samples : ] <EOL> clf = MultinomialNB ( ) . fit ( X_train , y_train , sample_weight = sw_train ) <EOL> prob_pos_clf = clf . predict_proba ( X_test ) [ : , <NUM_LIT:1> ] <EOL> pc_clf = CalibratedClassifierCV ( clf , cv = y . size + <NUM_LIT:1> ) <EOL> assert_raises ( ValueError , pc_clf . fit , X , y ) <EOL> for this_X_train , this_X_test in [ ( X_train , X_test ) , <EOL> ( sparse . csr_matrix ( X_train ) , <EOL> sparse . csr_matrix ( X_test ) ) ] : <EOL> for method in [ '<STR_LIT>' , '<STR_LIT>' ] : <EOL> pc_clf = CalibratedClassifierCV ( clf , method = method , cv = <NUM_LIT:2> ) <EOL> pc_clf . fit ( this_X_train , y_train , sample_weight = sw_train ) <EOL> prob_pos_pc_clf = pc_clf . predict_proba ( this_X_test ) [ : , <NUM_LIT:1> ] <EOL> assert_greater ( brier_score_loss ( y_test , prob_pos_clf ) , <EOL> brier_score_loss ( y_test , prob_pos_pc_clf ) ) <EOL> pc_clf . fit ( this_X_train , y_train + <NUM_LIT:1> , sample_weight = sw_train ) <EOL> prob_pos_pc_clf_relabeled = pc_clf . predict_proba ( this_X_test ) [ : , <NUM_LIT:1> ] <EOL> assert_array_almost_equal ( prob_pos_pc_clf , <EOL> prob_pos_pc_clf_relabeled ) <EOL> pc_clf . fit ( this_X_train , <NUM_LIT:2> * y_train - <NUM_LIT:1> , sample_weight = sw_train ) <EOL> prob_pos_pc_clf_relabeled = pc_clf . predict_proba ( this_X_test ) [ : , <NUM_LIT:1> ] <EOL> assert_array_almost_equal ( prob_pos_pc_clf , <EOL> prob_pos_pc_clf_relabeled ) <EOL> pc_clf . fit ( this_X_train , ( y_train + <NUM_LIT:1> ) % <NUM_LIT:2> , <EOL> sample_weight = sw_train ) <EOL> prob_pos_pc_clf_relabeled = pc_clf . predict_proba ( this_X_test ) [ : , <NUM_LIT:1> ] <EOL> if method == "<STR_LIT>" : <EOL> assert_array_almost_equal ( prob_pos_pc_clf , <EOL> <NUM_LIT:1> - prob_pos_pc_clf_relabeled ) <EOL> else : <EOL> assert_greater ( brier_score_loss ( y_test , prob_pos_clf ) , <EOL> brier_score_loss ( ( y_test + <NUM_LIT:1> ) % <NUM_LIT:2> , <EOL> prob_pos_pc_clf_relabeled ) ) <EOL> clf_base_regressor = CalibratedClassifierCV ( Ridge ( ) ) <EOL> clf_base_regressor . fit ( X_train , y_train ) <EOL> clf_base_regressor . predict ( X_test ) <EOL> clf_invalid_method = CalibratedClassifierCV ( clf , method = "<STR_LIT:foo>" ) <EOL> assert_raises ( ValueError , clf_invalid_method . fit , X_train , y_train ) <EOL> clf_base_regressor = CalibratedClassifierCV ( RandomForestRegressor ( ) , method = "<STR_LIT>" ) <EOL> assert_raises ( RuntimeError , clf_base_regressor . fit , X_train , y_train ) <EOL> def test_sample_weight_warning ( ) : <EOL> n_samples = <NUM_LIT:100> <EOL> X , y = make_classification ( n_samples = <NUM_LIT:2> * n_samples , n_features = <NUM_LIT:6> , <EOL> random_state = <NUM_LIT> ) <EOL> sample_weight = np . random . RandomState ( seed = <NUM_LIT> ) . uniform ( size = len ( y ) ) <EOL> X_train , y_train , sw_train = X [ : n_samples ] , y [ : n_samples ] , sample_weight [ : n_samples ] <EOL> X_test = X [ n_samples : ] <EOL> for method in [ '<STR_LIT>' , '<STR_LIT>' ] : <EOL> base_estimator = LinearSVC ( random_state = <NUM_LIT> ) <EOL> calibrated_clf = CalibratedClassifierCV ( base_estimator , method = method ) <EOL> msg = "<STR_LIT>" <EOL> assert_warns_message ( <EOL> UserWarning , msg , <EOL> calibrated_clf . fit , X_train , y_train , sample_weight = sw_train ) <EOL> probs_with_sw = calibrated_clf . predict_proba ( X_test ) <EOL> calibrated_clf . fit ( X_train , y_train ) <EOL> probs_without_sw = calibrated_clf . predict_proba ( X_test ) <EOL> diff = np . linalg . norm ( probs_with_sw - probs_without_sw ) <EOL> assert_greater ( diff , <NUM_LIT:0.1> ) <EOL> def test_calibration_multiclass ( ) : <EOL> """<STR_LIT>""" <EOL> clf = LinearSVC ( ) <EOL> X , y_idx = make_blobs ( n_samples = <NUM_LIT:100> , n_features = <NUM_LIT:2> , random_state = <NUM_LIT> , <EOL> centers = <NUM_LIT:3> , cluster_std = <NUM_LIT> ) <EOL> target_names = np . array ( [ '<STR_LIT:a>' , '<STR_LIT:b>' , '<STR_LIT:c>' ] ) <EOL> y = target_names [ y_idx ] <EOL> X_train , y_train = X [ : : <NUM_LIT:2> ] , y [ : : <NUM_LIT:2> ] <EOL> X_test , y_test = X [ <NUM_LIT:1> : : <NUM_LIT:2> ] , y [ <NUM_LIT:1> : : <NUM_LIT:2> ] <EOL> clf . fit ( X_train , y_train ) <EOL> for method in [ '<STR_LIT>' , '<STR_LIT>' ] : <EOL> cal_clf = CalibratedClassifierCV ( clf , method = method , cv = <NUM_LIT:2> ) <EOL> cal_clf . fit ( X_train , y_train ) <EOL> probas = cal_clf . predict_proba ( X_test ) <EOL> assert_array_almost_equal ( np . sum ( probas , axis = <NUM_LIT:1> ) , np . ones ( len ( X_test ) ) ) <EOL> def softmax ( y_pred ) : <EOL> e = np . exp ( - y_pred ) <EOL> return e / e . sum ( axis = <NUM_LIT:1> ) . reshape ( - <NUM_LIT:1> , <NUM_LIT:1> ) <EOL> uncalibrated_log_loss = log_loss ( y_test , softmax ( clf . decision_function ( X_test ) ) ) <EOL> calibrated_log_loss = log_loss ( y_test , probas ) <EOL> assert_greater_equal ( uncalibrated_log_loss , calibrated_log_loss ) <EOL> X , y = make_blobs ( n_samples = <NUM_LIT:100> , n_features = <NUM_LIT:2> , random_state = <NUM_LIT> , <EOL> cluster_std = <NUM_LIT> ) <EOL> X_train , y_train = X [ : : <NUM_LIT:2> ] , y [ : : <NUM_LIT:2> ] <EOL> X_test , y_test = X [ <NUM_LIT:1> : : <NUM_LIT:2> ] , y [ <NUM_LIT:1> : : <NUM_LIT:2> ] <EOL> clf = RandomForestClassifier ( n_estimators = <NUM_LIT:10> , random_state = <NUM_LIT> ) <EOL> clf . fit ( X_train , y_train ) <EOL> clf_probs = clf . predict_proba ( X_test ) <EOL> loss = log_loss ( y_test , clf_probs ) <EOL> for method in [ '<STR_LIT>' , '<STR_LIT>' ] : <EOL> cal_clf = CalibratedClassifierCV ( clf , method = method , cv = <NUM_LIT:3> ) <EOL> cal_clf . fit ( X_train , y_train ) <EOL> cal_clf_probs = cal_clf . predict_proba ( X_test ) <EOL> cal_loss = log_loss ( y_test , cal_clf_probs ) <EOL> assert_greater ( loss , cal_loss ) <EOL> def test_calibration_prefit ( ) : <EOL> """<STR_LIT>""" <EOL> n_samples = <NUM_LIT:50> <EOL> X , y = make_classification ( n_samples = <NUM_LIT:3> * n_samples , n_features = <NUM_LIT:6> , <EOL> random_state = <NUM_LIT> ) <EOL> sample_weight = np . random . RandomState ( seed = <NUM_LIT> ) . uniform ( size = y . size ) <EOL> X -= X . min ( ) <EOL> X_train , y_train , sw_train = X [ : n_samples ] , y [ : n_samples ] , sample_weight [ : n_samples ] <EOL> X_calib , y_calib , sw_calib = X [ n_samples : <NUM_LIT:2> * n_samples ] , y [ n_samples : <NUM_LIT:2> * n_samples ] , sample_weight [ n_samples : <NUM_LIT:2> * n_samples ] <EOL> X_test , y_test = X [ <NUM_LIT:2> * n_samples : ] , y [ <NUM_LIT:2> * n_samples : ] <EOL> clf = MultinomialNB ( ) <EOL> clf . fit ( X_train , y_train , sw_train ) <EOL> prob_pos_clf = clf . predict_proba ( X_test ) [ : , <NUM_LIT:1> ] <EOL> for this_X_calib , this_X_test in [ ( X_calib , X_test ) , <EOL> ( sparse . csr_matrix ( X_calib ) , <EOL> sparse . csr_matrix ( X_test ) ) ] : <EOL> for method in [ '<STR_LIT>' , '<STR_LIT>' ] : <EOL> pc_clf = CalibratedClassifierCV ( clf , method = method , cv = "<STR_LIT>" ) <EOL> for sw in [ sw_calib , None ] : <EOL> pc_clf . fit ( this_X_calib , y_calib , sample_weight = sw ) <EOL> y_prob = pc_clf . predict_proba ( this_X_test ) <EOL> y_pred = pc_clf . predict ( this_X_test ) <EOL> prob_pos_pc_clf = y_prob [ : , <NUM_LIT:1> ] <EOL> assert_array_equal ( y_pred , <EOL> np . array ( [ <NUM_LIT:0> , <NUM_LIT:1> ] ) [ np . argmax ( y_prob , axis = <NUM_LIT:1> ) ] ) <EOL> assert_greater ( brier_score_loss ( y_test , prob_pos_clf ) , <EOL> brier_score_loss ( y_test , prob_pos_pc_clf ) ) <EOL> def test_sigmoid_calibration ( ) : <EOL> """<STR_LIT>""" <EOL> exF = np . array ( [ <NUM_LIT:5> , - <NUM_LIT:4> , <NUM_LIT:1.0> ] ) <EOL> exY = np . array ( [ <NUM_LIT:1> , - <NUM_LIT:1> , - <NUM_LIT:1> ] ) <EOL> AB_lin_libsvm = np . array ( [ - <NUM_LIT> , <NUM_LIT> ] ) <EOL> assert_array_almost_equal ( AB_lin_libsvm , <EOL> _sigmoid_calibration ( exF , exY ) , <NUM_LIT:3> ) <EOL> lin_prob = <NUM_LIT:1.> / ( <NUM_LIT:1.> + np . exp ( AB_lin_libsvm [ <NUM_LIT:0> ] * exF + AB_lin_libsvm [ <NUM_LIT:1> ] ) ) <EOL> sk_prob = _SigmoidCalibration ( ) . fit ( exF , exY ) . predict ( exF ) <EOL> assert_array_almost_equal ( lin_prob , sk_prob , <NUM_LIT:6> ) <EOL> assert_raises ( ValueError , _SigmoidCalibration ( ) . fit , <EOL> np . vstack ( ( exF , exF ) ) , exY ) <EOL> def test_calibration_curve ( ) : <EOL> """<STR_LIT>""" <EOL> y_true = np . array ( [ <NUM_LIT:0> , <NUM_LIT:0> , <NUM_LIT:0> , <NUM_LIT:1> , <NUM_LIT:1> , <NUM_LIT:1> ] ) <EOL> y_pred = np . array ( [ <NUM_LIT:0.> , <NUM_LIT:0.1> , <NUM_LIT> , <NUM_LIT> , <NUM_LIT> , <NUM_LIT:1.> ] ) <EOL> prob_true , prob_pred = calibration_curve ( y_true , y_pred , n_bins = <NUM_LIT:2> ) <EOL> prob_true_unnormalized , prob_pred_unnormalized = calibration_curve ( y_true , y_pred * <NUM_LIT:2> , n_bins = <NUM_LIT:2> , normalize = True ) <EOL> assert_equal ( len ( prob_true ) , len ( prob_pred ) ) <EOL> assert_equal ( len ( prob_true ) , <NUM_LIT:2> ) <EOL> assert_almost_equal ( prob_true , [ <NUM_LIT:0> , <NUM_LIT:1> ] ) <EOL> assert_almost_equal ( prob_pred , [ <NUM_LIT:0.1> , <NUM_LIT> ] ) <EOL> assert_almost_equal ( prob_true , prob_true_unnormalized ) <EOL> assert_almost_equal ( prob_pred , prob_pred_unnormalized ) <EOL> assert_raises ( ValueError , calibration_curve , [ <NUM_LIT> ] , [ - <NUM_LIT:0.1> ] , <EOL> normalize = False ) <EOL> def test_calibration_nan_imputer ( ) : <EOL> """<STR_LIT>""" <EOL> X , y = make_classification ( n_samples = <NUM_LIT:10> , n_features = <NUM_LIT:2> , <EOL> n_informative = <NUM_LIT:2> , n_redundant = <NUM_LIT:0> , <EOL> random_state = <NUM_LIT> ) <EOL> X [ <NUM_LIT:0> , <NUM_LIT:0> ] = np . nan <EOL> clf = Pipeline ( <EOL> [ ( '<STR_LIT>' , Imputer ( ) ) , <EOL> ( '<STR_LIT>' , RandomForestClassifier ( n_estimators = <NUM_LIT:1> ) ) ] ) <EOL> clf_c = CalibratedClassifierCV ( clf , cv = <NUM_LIT:2> , method = '<STR_LIT>' ) <EOL> clf_c . fit ( X , y ) <EOL> clf_c . predict ( X ) </s>
