<s> __author__ = '<STR_LIT>' <EOL> from . . utils . UMath import * <EOL> from . . View import * <EOL> import collections <EOL> class RelevanceAssessment : <EOL> def __init__ ( self , labels ) : <EOL> self . labels = labels <EOL> self . view = View ( False , True ) <EOL> self . positives = [ ] <EOL> self . negatives = [ ] <EOL> self . reliabilities = [ ] <EOL> self . errors = [ ] <EOL> self . all_f1_score = [ ] <EOL> self . all_precision = [ ] <EOL> self . all_recall = [ ] <EOL> self . all_reliability = [ ] <EOL> self . confusion_matrix = self . build_confusion_matrix ( labels ) <EOL> def update_training ( self , error ) : <EOL> self . errors . append ( error ) <EOL> def update_evaluation ( self , predicted_label , expected_label , predictions ) : <EOL> self . confusion_matrix [ expected_label ] [ predicted_label ] += <NUM_LIT:1> <EOL> if predicted_label == expected_label : <EOL> self . positives . append ( expected_label ) <EOL> else : <EOL> self . negatives . append ( expected_label ) <EOL> self . reliabilities . append ( UMath . reliability ( [ x / sum ( predictions ) for x in predictions ] ) ) <EOL> print "<STR_LIT>" . format ( predicted_label , expected_label , "<STR_LIT:OK>" if predicted_label == expected_label else "<STR_LIT>" ) <EOL> def compute ( self , collection_size ) : <EOL> true_positives = float ( len ( self . positives ) ) <EOL> false_negatives = self . get_false_positives ( self . negatives , self . positives ) <EOL> precision = true_positives / collection_size <EOL> recall = true_positives / UMath . get_denominator ( true_positives + false_negatives ) <EOL> f1_score = <NUM_LIT:2> * ( precision * recall ) / UMath . get_denominator ( precision + recall ) <EOL> reliability = np . mean ( self . reliabilities ) <EOL> print "<STR_LIT>" . format ( f1_score , precision , recall , reliability ) <EOL> self . all_f1_score . append ( f1_score ) <EOL> self . all_precision . append ( precision ) <EOL> self . all_recall . append ( recall ) <EOL> self . all_reliability . append ( reliability ) <EOL> self . positives = [ ] <EOL> self . negatives = [ ] <EOL> self . reliabilities = [ ] <EOL> def get_statistics ( self , data ) : <EOL> amin = np . amin ( data ) <EOL> mean = np . mean ( data ) <EOL> amax = np . amax ( data ) <EOL> stddev = np . std ( data ) <EOL> return "<STR_LIT>" . format ( amin , mean , amax , stddev ) <EOL> def get_false_positives ( self , negatives , true_positives ) : <EOL> unique_false_negatives = set ( true_positives ) & set ( negatives ) <EOL> false_negatives = <NUM_LIT:0> <EOL> for x in unique_false_negatives : <EOL> for y in negatives : <EOL> if x == y : <EOL> false_negatives += <NUM_LIT:1> <EOL> return float ( false_negatives ) <EOL> def get_confusion_matrix ( self ) : <EOL> matrix = self . convert_to_matrix ( self . confusion_matrix ) <EOL> return UMath . normalize_array ( matrix ) <EOL> def build_confusion_matrix ( self , label_set ) : <EOL> expected_labels = collections . OrderedDict ( ) <EOL> for expected_label in label_set : <EOL> expected_labels [ expected_label ] = collections . OrderedDict ( ) <EOL> for predicted_label in label_set : <EOL> expected_labels [ expected_label ] [ predicted_label ] = <NUM_LIT:0.0> <EOL> return expected_labels <EOL> def convert_to_matrix ( self , dictionary ) : <EOL> length = len ( dictionary ) <EOL> confusion_matrix = np . zeros ( ( length , length ) ) <EOL> i = <NUM_LIT:0> <EOL> for row in dictionary : <EOL> j = <NUM_LIT:0> <EOL> for column in dictionary : <EOL> confusion_matrix [ i ] [ j ] = dictionary [ row ] [ column ] <EOL> j += <NUM_LIT:1> <EOL> i += <NUM_LIT:1> <EOL> return confusion_matrix <EOL> def output_compared_plot ( self , path ) : <EOL> length = len ( self . all_f1_score ) <EOL> elength = len ( self . errors ) <EOL> if elength > length : <EOL> ratio = elength / length <EOL> self . all_f1_score = UMath . interpolate ( self . all_f1_score , length * ratio ) <EOL> self . all_precision = UMath . interpolate ( self . all_precision , length * ratio ) <EOL> self . all_recall = UMath . interpolate ( self . all_recall , length * ratio ) <EOL> self . all_reliability = UMath . interpolate ( self . all_reliability , length * ratio ) <EOL> data = [ self . errors , self . all_f1_score , self . all_precision , self . all_recall , self . all_reliability ] <EOL> colors = [ '<STR_LIT:m>' , '<STR_LIT:c>' , '<STR_LIT:g>' , '<STR_LIT:b>' , '<STR_LIT:r>' ] <EOL> labels = [ "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" ] <EOL> self . view . plot_data ( "<STR_LIT>" , data , "<STR_LIT>" , "<STR_LIT>" , colors , labels ) <EOL> self . view . show ( ) <EOL> self . view . save ( path ) <EOL> def output_statistics ( self , path , run_name ) : <EOL> output_file = open ( path , '<STR_LIT:a>' ) <EOL> output_file . write ( "<STR_LIT>" . format ( run_name ) ) <EOL> output_file . write ( "<STR_LIT>" . format ( self . get_statistics ( self . all_f1_score ) ) ) <EOL> output_file . write ( "<STR_LIT>" . format ( self . get_statistics ( self . all_precision ) ) ) <EOL> output_file . write ( "<STR_LIT>" . format ( self . get_statistics ( self . all_recall ) ) ) <EOL> output_file . write ( "<STR_LIT>" . format ( self . get_statistics ( self . all_reliability ) ) ) <EOL> if len ( self . errors ) > <NUM_LIT:0> : <EOL> output_file . write ( "<STR_LIT>" . format ( self . get_statistics ( self . errors ) ) ) <EOL> output_file . write ( "<STR_LIT:\n>" ) <EOL> output_file . close ( ) <EOL> def output_mean_square_mean_error ( self , path , k = <NUM_LIT:1> ) : <EOL> if k > <NUM_LIT:1> : <EOL> step = len ( self . errors ) / k <EOL> errors = np . zeros ( step ) <EOL> for i in xrange ( <NUM_LIT:0> , len ( self . errors ) , step ) : <EOL> error_run = [ ] <EOL> for j in range ( i , i + step ) : <EOL> error_run . append ( self . errors [ j ] ) <EOL> errors = [ sum ( x ) for x in zip ( errors , error_run ) ] <EOL> errors = [ x / k for x in errors ] <EOL> else : <EOL> errors = self . errors <EOL> self . view . plot_data ( "<STR_LIT>" , [ errors ] , "<STR_LIT>" , "<STR_LIT>" ) <EOL> self . view . show ( ) <EOL> self . view . save ( path ) <EOL> def output_confusion_matrix ( self , path ) : <EOL> matrix = self . get_confusion_matrix ( ) <EOL> self . view . plot_confusion_matrix ( matrix , self . labels ) <EOL> self . view . show ( ) <EOL> self . view . save ( path ) </s>
