<s> import numpy as np <EOL> def confusion_matrix ( rater_a , rater_b , min_rating = None , max_rating = None ) : <EOL> """<STR_LIT>""" <EOL> assert ( len ( rater_a ) == len ( rater_b ) ) <EOL> if min_rating is None : <EOL> min_rating = min ( rater_a + rater_b ) <EOL> if max_rating is None : <EOL> max_rating = max ( rater_a + rater_b ) <EOL> num_ratings = int ( max_rating - min_rating + <NUM_LIT:1> ) <EOL> conf_mat = [ [ <NUM_LIT:0> for i in range ( num_ratings ) ] <EOL> for j in range ( num_ratings ) ] <EOL> for a , b in zip ( rater_a , rater_b ) : <EOL> conf_mat [ a - min_rating ] [ b - min_rating ] += <NUM_LIT:1> <EOL> return conf_mat <EOL> def histogram ( ratings , min_rating = None , max_rating = None ) : <EOL> """<STR_LIT>""" <EOL> if min_rating is None : <EOL> min_rating = min ( ratings ) <EOL> if max_rating is None : <EOL> max_rating = max ( ratings ) <EOL> num_ratings = int ( max_rating - min_rating + <NUM_LIT:1> ) <EOL> hist_ratings = [ <NUM_LIT:0> for x in range ( num_ratings ) ] <EOL> for r in ratings : <EOL> hist_ratings [ r - min_rating ] += <NUM_LIT:1> <EOL> return hist_ratings <EOL> def quadratic_weighted_kappa ( rater_a , rater_b , min_rating = None , max_rating = None ) : <EOL> """<STR_LIT>""" <EOL> rater_a = np . array ( rater_a , dtype = int ) <EOL> rater_b = np . array ( rater_b , dtype = int ) <EOL> assert ( len ( rater_a ) == len ( rater_b ) ) <EOL> if min_rating is None : <EOL> min_rating = min ( min ( rater_a ) , min ( rater_b ) ) <EOL> if max_rating is None : <EOL> max_rating = max ( max ( rater_a ) , max ( rater_b ) ) <EOL> conf_mat = confusion_matrix ( rater_a , rater_b , <EOL> min_rating , max_rating ) <EOL> num_ratings = len ( conf_mat ) <EOL> num_scored_items = float ( len ( rater_a ) ) <EOL> hist_rater_a = histogram ( rater_a , min_rating , max_rating ) <EOL> hist_rater_b = histogram ( rater_b , min_rating , max_rating ) <EOL> numerator = <NUM_LIT:0.0> <EOL> denominator = <NUM_LIT:0.0> <EOL> for i in range ( num_ratings ) : <EOL> for j in range ( num_ratings ) : <EOL> expected_count = ( hist_rater_a [ i ] * hist_rater_b [ j ] <EOL> / num_scored_items ) <EOL> d = pow ( i - j , <NUM_LIT> ) / pow ( num_ratings - <NUM_LIT:1> , <NUM_LIT> ) <EOL> numerator += d * conf_mat [ i ] [ j ] / num_scored_items <EOL> denominator += d * expected_count / num_scored_items <EOL> return <NUM_LIT:1.0> - numerator / denominator <EOL> def linear_weighted_kappa ( rater_a , rater_b , min_rating = None , max_rating = None ) : <EOL> """<STR_LIT>""" <EOL> assert ( len ( rater_a ) == len ( rater_b ) ) <EOL> if min_rating is None : <EOL> min_rating = min ( rater_a + rater_b ) <EOL> if max_rating is None : <EOL> max_rating = max ( rater_a + rater_b ) <EOL> conf_mat = confusion_matrix ( rater_a , rater_b , <EOL> min_rating , max_rating ) <EOL> num_ratings = len ( conf_mat ) <EOL> num_scored_items = float ( len ( rater_a ) ) <EOL> hist_rater_a = histogram ( rater_a , min_rating , max_rating ) <EOL> hist_rater_b = histogram ( rater_b , min_rating , max_rating ) <EOL> numerator = <NUM_LIT:0.0> <EOL> denominator = <NUM_LIT:0.0> <EOL> for i in range ( num_ratings ) : <EOL> for j in range ( num_ratings ) : <EOL> expected_count = ( hist_rater_a [ i ] * hist_rater_b [ j ] <EOL> / num_scored_items ) <EOL> d = abs ( i - j ) / float ( num_ratings - <NUM_LIT:1> ) <EOL> numerator += d * conf_mat [ i ] [ j ] / num_scored_items <EOL> denominator += d * expected_count / num_scored_items <EOL> return <NUM_LIT:1.0> - numerator / denominator <EOL> def kappa ( rater_a , rater_b , min_rating = None , max_rating = None ) : <EOL> """<STR_LIT>""" <EOL> assert ( len ( rater_a ) == len ( rater_b ) ) <EOL> if min_rating is None : <EOL> min_rating = min ( rater_a + rater_b ) <EOL> if max_rating is None : <EOL> max_rating = max ( rater_a + rater_b ) <EOL> conf_mat = confusion_matrix ( rater_a , rater_b , <EOL> min_rating , max_rating ) <EOL> num_ratings = len ( conf_mat ) <EOL> num_scored_items = float ( len ( rater_a ) ) <EOL> hist_rater_a = histogram ( rater_a , min_rating , max_rating ) <EOL> hist_rater_b = histogram ( rater_b , min_rating , max_rating ) <EOL> numerator = <NUM_LIT:0.0> <EOL> denominator = <NUM_LIT:0.0> <EOL> for i in range ( num_ratings ) : <EOL> for j in range ( num_ratings ) : <EOL> expected_count = ( hist_rater_a [ i ] * hist_rater_b [ j ] <EOL> / num_scored_items ) <EOL> if i == j : <EOL> d = <NUM_LIT:0.0> <EOL> else : <EOL> d = <NUM_LIT:1.0> <EOL> numerator += d * conf_mat [ i ] [ j ] / num_scored_items <EOL> denominator += d * expected_count / num_scored_items <EOL> return <NUM_LIT:1.0> - numerator / denominator <EOL> def mean_quadratic_weighted_kappa ( kappas , weights = None ) : <EOL> """<STR_LIT>""" <EOL> kappas = np . array ( kappas , dtype = float ) <EOL> if weights is None : <EOL> weights = np . ones ( np . shape ( kappas ) ) <EOL> else : <EOL> weights = weights / np . mean ( weights ) <EOL> kappas = np . array ( [ min ( x , <NUM_LIT> ) for x in kappas ] ) <EOL> kappas = np . array ( [ max ( x , - <NUM_LIT> ) for x in kappas ] ) <EOL> z = <NUM_LIT:0.5> * np . log ( ( <NUM_LIT:1> + kappas ) / ( <NUM_LIT:1> - kappas ) ) * weights <EOL> z = np . mean ( z ) <EOL> return ( np . exp ( <NUM_LIT:2> * z ) - <NUM_LIT:1> ) / ( np . exp ( <NUM_LIT:2> * z ) + <NUM_LIT:1> ) <EOL> def weighted_mean_quadratic_weighted_kappa ( solution , submission ) : <EOL> predicted_score = submission [ submission . columns [ - <NUM_LIT:1> ] ] . copy ( ) <EOL> predicted_score . name = "<STR_LIT>" <EOL> if predicted_score . index [ <NUM_LIT:0> ] == <NUM_LIT:0> : <EOL> predicted_score = predicted_score [ : len ( solution ) ] <EOL> predicted_score . index = solution . index <EOL> combined = solution . join ( predicted_score , how = "<STR_LIT:left>" ) <EOL> groups = combined . groupby ( by = "<STR_LIT>" ) <EOL> kappas = [ quadratic_weighted_kappa ( group [ <NUM_LIT:1> ] [ "<STR_LIT>" ] , group [ <NUM_LIT:1> ] [ "<STR_LIT>" ] ) for group in groups ] <EOL> weights = [ group [ <NUM_LIT:1> ] [ "<STR_LIT>" ] . irow ( <NUM_LIT:0> ) for group in groups ] <EOL> return mean_quadratic_weighted_kappa ( kappas , weights = weights ) </s>
