<s> from __future__ import division , print_function , absolute_import <EOL> import numpy <EOL> import pandas <EOL> from numpy . random . mtrand import RandomState <EOL> from scipy . stats import ks_2samp <EOL> from hep_ml . commonutils import generate_sample <EOL> from hep_ml . metrics_utils import compute_sde_on_bins , compute_sde_on_groups , compute_theil_on_bins , compute_theil_on_groups , prepare_distibution , _ks_2samp_fast , ks_2samp_weighted , bin_based_ks , groups_based_ks , cvm_2samp , _cvm_2samp_fast , bin_based_cvm , group_based_cvm <EOL> from hep_ml . metrics import sde , theil_flatness , cvm_flatness , KnnBasedSDE , KnnBasedTheil , KnnBasedCvM , BinBasedSDE , BinBasedTheil , BinBasedCvM <EOL> from hep_ml . metrics_utils import bin_to_group_indices , compute_bin_indices <EOL> __author__ = '<STR_LIT>' <EOL> random = RandomState ( ) <EOL> def generate_binned_dataset ( n_samples , n_bins ) : <EOL> """<STR_LIT>""" <EOL> random = RandomState ( ) <EOL> y = random . uniform ( size = n_samples ) > <NUM_LIT:0.5> <EOL> pred = random . uniform ( size = ( n_samples , <NUM_LIT:2> ) ) <EOL> weights = random . exponential ( size = ( n_samples , ) ) <EOL> bins = random . randint ( <NUM_LIT:0> , n_bins , n_samples ) <EOL> groups = bin_to_group_indices ( bin_indices = bins , mask = ( y == <NUM_LIT:1> ) ) <EOL> return y , pred , weights , bins , groups <EOL> def test_bin_to_group_indices ( size = <NUM_LIT:100> , bins = <NUM_LIT:10> ) : <EOL> bin_indices = RandomState ( ) . randint ( <NUM_LIT:0> , bins , size = size ) <EOL> mask = RandomState ( ) . randint ( <NUM_LIT:0> , <NUM_LIT:2> , size = size ) > <NUM_LIT:0.5> <EOL> group_indices = bin_to_group_indices ( bin_indices , mask = mask ) <EOL> assert numpy . sum ( [ len ( group ) for group in group_indices ] ) == numpy . sum ( mask ) <EOL> a = numpy . sort ( numpy . concatenate ( group_indices ) ) <EOL> b = numpy . where ( mask > <NUM_LIT:0.5> ) [ <NUM_LIT:0> ] <EOL> assert numpy . all ( a == b ) , '<STR_LIT>' <EOL> def test_bins ( size = <NUM_LIT> , n_bins = <NUM_LIT:10> ) : <EOL> columns = [ '<STR_LIT>' , '<STR_LIT>' ] <EOL> df = pandas . DataFrame ( random . uniform ( size = ( size , <NUM_LIT:2> ) ) , columns = columns ) <EOL> x_limits = numpy . linspace ( <NUM_LIT:0> , <NUM_LIT:1> , n_bins + <NUM_LIT:1> ) [ <NUM_LIT:1> : - <NUM_LIT:1> ] <EOL> bins = compute_bin_indices ( df [ columns ] . values , bin_limits = [ x_limits , x_limits ] ) <EOL> assert numpy . all ( <NUM_LIT:0> <= bins ) and numpy . all ( bins < n_bins * n_bins ) , "<STR_LIT>" <EOL> def test_compare_sde_computations ( n_samples = <NUM_LIT:1000> , n_bins = <NUM_LIT:10> ) : <EOL> y , pred , weights , bins , groups = generate_binned_dataset ( n_samples = n_samples , n_bins = n_bins ) <EOL> target_efficiencies = RandomState ( ) . uniform ( size = <NUM_LIT:3> ) <EOL> a = compute_sde_on_bins ( pred [ : , <NUM_LIT:1> ] , mask = ( y == <NUM_LIT:1> ) , bin_indices = bins , <EOL> target_efficiencies = target_efficiencies , sample_weight = weights ) <EOL> b = compute_sde_on_groups ( pred [ : , <NUM_LIT:1> ] , mask = ( y == <NUM_LIT:1> ) , groups_indices = groups , <EOL> target_efficiencies = target_efficiencies , sample_weight = weights ) <EOL> assert numpy . allclose ( a , b ) <EOL> def test_theil ( n_samples = <NUM_LIT:1000> , n_bins = <NUM_LIT:10> ) : <EOL> y , pred , weights , bins , groups = generate_binned_dataset ( n_samples = n_samples , n_bins = n_bins ) <EOL> a = compute_theil_on_bins ( pred [ : , <NUM_LIT:1> ] , y == <NUM_LIT:1> , bins , [ <NUM_LIT:0.5> , <NUM_LIT> ] , sample_weight = weights ) <EOL> b = compute_theil_on_groups ( pred [ : , <NUM_LIT:1> ] , y == <NUM_LIT:1> , groups , [ <NUM_LIT:0.5> , <NUM_LIT> ] , sample_weight = weights ) <EOL> assert numpy . allclose ( a , b ) <EOL> def test_ks2samp_fast ( size = <NUM_LIT:1000> ) : <EOL> y1 = RandomState ( ) . uniform ( size = size ) <EOL> y2 = y1 [ RandomState ( ) . uniform ( size = size ) > <NUM_LIT:0.5> ] <EOL> a = ks_2samp ( y1 , y2 ) [ <NUM_LIT:0> ] <EOL> prep_data , prep_weights , prep_F = prepare_distibution ( y1 , numpy . ones ( len ( y1 ) ) ) <EOL> b = _ks_2samp_fast ( prep_data , y2 , prep_weights , numpy . ones ( len ( y2 ) ) , F1 = prep_F ) <EOL> c = _ks_2samp_fast ( prep_data , y2 , prep_weights , numpy . ones ( len ( y2 ) ) , F1 = prep_F ) <EOL> d = ks_2samp_weighted ( y1 , y2 , numpy . ones ( len ( y1 ) ) / <NUM_LIT:3> , numpy . ones ( len ( y2 ) ) / <NUM_LIT:4> ) <EOL> assert numpy . allclose ( a , b , rtol = <NUM_LIT> , atol = <NUM_LIT> ) <EOL> assert numpy . allclose ( b , c ) <EOL> assert numpy . allclose ( b , d ) <EOL> print ( '<STR_LIT>' ) <EOL> def test_ks ( n_samples = <NUM_LIT:1000> , n_bins = <NUM_LIT:10> ) : <EOL> y , pred , weights , bins , groups = generate_binned_dataset ( n_samples = n_samples , n_bins = n_bins ) <EOL> mask = y == <NUM_LIT:1> <EOL> a = bin_based_ks ( pred [ : , <NUM_LIT:1> ] , mask = mask , sample_weight = weights , bin_indices = bins ) <EOL> b = groups_based_ks ( pred [ : , <NUM_LIT:1> ] , mask = mask , sample_weight = weights , groups_indices = groups ) <EOL> assert numpy . allclose ( a , b ) <EOL> def test_fast_cvm ( n_samples = <NUM_LIT:1000> ) : <EOL> random = RandomState ( ) <EOL> data1 = random . uniform ( size = n_samples ) <EOL> weights1 = random . uniform ( size = n_samples ) <EOL> mask = random . uniform ( size = n_samples ) > <NUM_LIT:0.5> <EOL> data2 = data1 [ mask ] <EOL> weights2 = weights1 [ mask ] <EOL> a = cvm_2samp ( data1 , data2 , weights1 , weights2 ) <EOL> prepared_data1 , prepared_weights1 , F1 = prepare_distibution ( data1 , weights1 ) <EOL> b = _cvm_2samp_fast ( prepared_data1 , data2 , prepared_weights1 , weights2 , F1 = F1 ) <EOL> assert numpy . allclose ( a , b ) <EOL> def test_cvm ( size = <NUM_LIT:1000> ) : <EOL> y_pred = random . uniform ( size = size ) <EOL> y = random . uniform ( size = size ) > <NUM_LIT:0.5> <EOL> sample_weight = random . exponential ( size = size ) <EOL> bin_indices = random . randint ( <NUM_LIT:0> , <NUM_LIT:10> , size = size ) <EOL> mask = y == <NUM_LIT:1> <EOL> groups_indices = bin_to_group_indices ( bin_indices = bin_indices , mask = mask ) <EOL> cvm1 = bin_based_cvm ( y_pred [ mask ] , sample_weight = sample_weight [ mask ] , bin_indices = bin_indices [ mask ] ) <EOL> cvm2 = group_based_cvm ( y_pred , mask = mask , sample_weight = sample_weight , groups_indices = groups_indices ) <EOL> assert numpy . allclose ( cvm1 , cvm2 ) <EOL> def test_cvm_sde_limit ( size = <NUM_LIT> ) : <EOL> """<STR_LIT>""" <EOL> effs = numpy . linspace ( <NUM_LIT:0> , <NUM_LIT:1> , <NUM_LIT> ) <EOL> y_pred = random . uniform ( size = size ) <EOL> y = random . uniform ( size = size ) > <NUM_LIT:0.5> <EOL> sample_weight = random . exponential ( size = size ) <EOL> bin_indices = random . randint ( <NUM_LIT:0> , <NUM_LIT:10> , size = size ) <EOL> y_pred += bin_indices * random . uniform ( ) <EOL> mask = y == <NUM_LIT:1> <EOL> val1 = bin_based_cvm ( y_pred [ mask ] , sample_weight = sample_weight [ mask ] , bin_indices = bin_indices [ mask ] ) <EOL> val2 = compute_sde_on_bins ( y_pred , mask = mask , bin_indices = bin_indices , target_efficiencies = effs , <EOL> sample_weight = sample_weight ) <EOL> assert numpy . allclose ( val1 , val2 ** <NUM_LIT:2> , atol = <NUM_LIT> , rtol = <NUM_LIT> ) <EOL> def test_new_metrics ( n_samples = <NUM_LIT> , knn = <NUM_LIT:50> ) : <EOL> X , y = generate_sample ( n_samples = n_samples , n_features = <NUM_LIT:10> ) <EOL> sample_weight = numpy . random . exponential ( size = n_samples ) ** <NUM_LIT:0.> <EOL> predictions = numpy . random . random ( size = [ n_samples , <NUM_LIT:2> ] ) <EOL> predictions /= predictions . sum ( axis = <NUM_LIT:1> , keepdims = True ) <EOL> predictions *= <NUM_LIT> <EOL> features = X . columns [ : <NUM_LIT:1> ] <EOL> sde_val1 = sde ( y , predictions , X , uniform_variables = features , sample_weight = sample_weight , label = <NUM_LIT:0> , knn = knn ) <EOL> sde2 = KnnBasedSDE ( n_neighbours = knn , uniform_features = features , uniform_label = <NUM_LIT:0> , ) <EOL> sde2 . fit ( X , y , sample_weight = sample_weight ) <EOL> sde_val2 = sde2 ( y , predictions , sample_weight = sample_weight ) <EOL> assert sde_val1 == sde_val2 , '<STR_LIT>' <EOL> features = X . columns [ : <NUM_LIT:1> ] <EOL> cvm_val1 = cvm_flatness ( y , predictions , X , uniform_variables = features , sample_weight = sample_weight , label = <NUM_LIT:0> , <EOL> knn = knn ) <EOL> cvm2 = KnnBasedCvM ( n_neighbours = knn , uniform_features = features , uniform_label = <NUM_LIT:0> , ) <EOL> cvm2 . fit ( X , y , sample_weight = sample_weight ) <EOL> cvm_val2 = cvm2 ( y , predictions , sample_weight = sample_weight ) <EOL> assert cvm_val1 == cvm_val2 , '<STR_LIT>' <EOL> def test_metrics_clear ( n_samples = <NUM_LIT> , knn = <NUM_LIT:50> , uniform_class = <NUM_LIT:0> ) : <EOL> """<STR_LIT>""" <EOL> X , y = generate_sample ( n_samples = n_samples , n_features = <NUM_LIT:10> ) <EOL> sample_weight = numpy . random . exponential ( size = n_samples ) <EOL> predictions = numpy . random . random ( size = [ n_samples , <NUM_LIT:2> ] ) <EOL> predictions /= predictions . sum ( axis = <NUM_LIT:1> , keepdims = True ) <EOL> features = X . columns [ : <NUM_LIT:1> ] <EOL> mask = ( y == uniform_class ) <EOL> X_clear = X . ix [ mask , : ] <EOL> y_clear = y [ mask ] <EOL> sample_weight_clear = sample_weight [ mask ] <EOL> predictions_clear = predictions [ mask ] <EOL> for function in [ sde , theil_flatness , cvm_flatness ] : <EOL> flatness_val = function ( y , predictions , X , uniform_variables = features , sample_weight = sample_weight , label = <NUM_LIT:0> , <EOL> knn = knn ) <EOL> flatness_val_clear = function ( y_clear , predictions_clear , X_clear , uniform_variables = features , <EOL> sample_weight = sample_weight_clear , label = <NUM_LIT:0> , knn = knn ) <EOL> assert flatness_val == flatness_val_clear , '<STR_LIT>' <EOL> for class_ in [ KnnBasedSDE , KnnBasedTheil , KnnBasedCvM ] : <EOL> metric1 = class_ ( n_neighbours = knn , uniform_features = features , uniform_label = <NUM_LIT:0> , ) <EOL> metric1 . fit ( X , y , sample_weight = sample_weight ) <EOL> flatness_val1 = metric1 ( y , predictions , sample_weight ) <EOL> metric2 = class_ ( n_neighbours = knn , uniform_features = features , uniform_label = <NUM_LIT:0> , ) <EOL> metric2 . fit ( X_clear , y_clear , sample_weight = sample_weight_clear ) <EOL> flatness_val2 = metric2 ( y_clear , predictions_clear , sample_weight_clear ) <EOL> assert flatness_val1 == flatness_val2 , '<STR_LIT>' <EOL> def test_workability ( n_samples = <NUM_LIT> , knn = <NUM_LIT:50> , uniform_label = <NUM_LIT:0> , n_bins = <NUM_LIT:10> ) : <EOL> """<STR_LIT>""" <EOL> X , y = generate_sample ( n_samples = n_samples , n_features = <NUM_LIT:10> ) <EOL> sample_weight = numpy . random . exponential ( size = n_samples ) <EOL> predictions = numpy . random . random ( size = [ n_samples , <NUM_LIT:2> ] ) <EOL> predictions /= predictions . sum ( axis = <NUM_LIT:1> , keepdims = True ) <EOL> features = X . columns [ : <NUM_LIT:1> ] <EOL> for class_ in [ KnnBasedSDE , KnnBasedTheil , KnnBasedCvM ] : <EOL> metric = class_ ( n_neighbours = knn , uniform_features = features , uniform_label = uniform_label , ) <EOL> metric . fit ( X , y , sample_weight = sample_weight ) <EOL> flatness_val_ = metric ( y , predictions , sample_weight ) <EOL> for class_ in [ BinBasedSDE , BinBasedTheil , BinBasedCvM ] : <EOL> metric = class_ ( n_bins = n_bins , uniform_features = features , uniform_label = uniform_label , ) <EOL> metric . fit ( X , y , sample_weight = sample_weight ) <EOL> flatness_val_ = metric ( y , predictions , sample_weight ) </s>
