<s> """<STR_LIT>""" <EOL> from itertools import combinations <EOL> import argparse <EOL> PARSER = argparse . ArgumentParser ( description = "<STR_LIT>" "<STR_LIT>" ) <EOL> PARSER . add_argument ( "<STR_LIT>" , type = file , metavar = "<STR_LIT>" , <EOL> help = "<STR_LIT>" ) <EOL> PARSER . add_argument ( "<STR_LIT>" , type = file , metavar = "<STR_LIT>" , <EOL> help = "<STR_LIT>" ) <EOL> def compare_data_point_to_dict ( _data_point , target_dict ) : <EOL> """<STR_LIT>""" <EOL> _language_pair , _sentence_no , _ranking_decision = _data_point <EOL> if not target_dict . has_key ( _language_pair ) or not target_dict [ language_pair ] . has_key ( _sentence_no ) : <EOL> return False <EOL> inverted_decision = '<STR_LIT>' . format ( * _ranking_decision . split ( '<STR_LIT:>>' ) ) <EOL> _store = target_dict [ _language_pair ] [ _sentence_no ] <EOL> _score = _store . get ( '<STR_LIT>' , <NUM_LIT:0> ) <EOL> _inverted = _store . get ( '<STR_LIT>' , <NUM_LIT:0> ) <EOL> return _score >= _inverted <EOL> def add_data_point_to_dict ( _data_point , target_dict ) : <EOL> """<STR_LIT>""" <EOL> _language_pair , _sentence_no , _ranking_decision = _data_point <EOL> if not target_dict . has_key ( _language_pair ) : <EOL> target_dict [ _language_pair ] = { } <EOL> if not target_dict [ _language_pair ] . has_key ( _sentence_no ) : <EOL> target_dict [ _language_pair ] [ _sentence_no ] = { } <EOL> _store = target_dict [ _language_pair ] [ _sentence_no ] <EOL> if not _ranking_decision in _store . keys ( ) : <EOL> _store [ _ranking_decision ] = <NUM_LIT:1> <EOL> else : <EOL> _store [ _ranking_decision ] = _store [ _ranking_decision ] + <NUM_LIT:1> <EOL> if __name__ == "<STR_LIT:__main__>" : <EOL> args = PARSER . parse_args ( ) <EOL> LANGUAGE_MAP = { '<STR_LIT>' : '<STR_LIT>' , '<STR_LIT>' : '<STR_LIT>' , '<STR_LIT>' : '<STR_LIT>' , '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , '<STR_LIT>' : '<STR_LIT>' , '<STR_LIT>' : '<STR_LIT>' , '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , '<STR_LIT>' : '<STR_LIT>' , '<STR_LIT>' : '<STR_LIT>' , '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , '<STR_LIT>' : '<STR_LIT>' , '<STR_LIT>' : '<STR_LIT>' } <EOL> MTURK_DATA = { } <EOL> APPRAISE_DATA = { } <EOL> line_no = <NUM_LIT:0> <EOL> results = [ ] <EOL> for line in args . mturk_file : <EOL> line_no = line_no + <NUM_LIT:1> <EOL> if line_no == <NUM_LIT:1> : <EOL> continue <EOL> line_data = line . strip ( ) . split ( '<STR_LIT:U+002C>' ) <EOL> src_lang = LANGUAGE_MAP [ line_data [ <NUM_LIT:0> ] . lower ( ) ] <EOL> trg_lang = LANGUAGE_MAP [ line_data [ <NUM_LIT:1> ] . lower ( ) ] <EOL> language_pair = '<STR_LIT>' . format ( src_lang , trg_lang ) <EOL> sentence_no = int ( line_data [ <NUM_LIT:2> ] ) <EOL> judge_id = line_data [ <NUM_LIT:5> ] <EOL> sysIds = line_data [ <NUM_LIT:7> : <NUM_LIT> : <NUM_LIT:2> ] <EOL> sysRanks = line_data [ - <NUM_LIT:5> : ] <EOL> if not judge_id in MTURK_DATA . keys ( ) : <EOL> MTURK_DATA [ judge_id ] = { } <EOL> decisions = [ ] <EOL> for a , b in combinations ( range ( <NUM_LIT:5> ) , <NUM_LIT:2> ) : <EOL> if sysRanks [ a ] > sysRanks [ b ] : <EOL> decisions . append ( '<STR_LIT>' . format ( sysIds [ a ] , sysIds [ b ] ) ) <EOL> elif sysRanks [ a ] < sysRanks [ b ] : <EOL> decisions . append ( '<STR_LIT>' . format ( sysIds [ b ] , sysIds [ a ] ) ) <EOL> for ranking_decision in decisions : <EOL> data_point = ( language_pair , sentence_no , ranking_decision ) <EOL> add_data_point_to_dict ( data_point , MTURK_DATA [ judge_id ] ) <EOL> line_no = <NUM_LIT:0> <EOL> for line in args . appraise_file : <EOL> line_no = line_no + <NUM_LIT:1> <EOL> if line_no == <NUM_LIT:1> : <EOL> continue <EOL> line_data = line . strip ( ) . split ( '<STR_LIT:U+002C>' ) <EOL> src_lang = LANGUAGE_MAP [ line_data [ <NUM_LIT:0> ] . lower ( ) ] <EOL> trg_lang = LANGUAGE_MAP [ line_data [ <NUM_LIT:1> ] . lower ( ) ] <EOL> language_pair = '<STR_LIT>' . format ( src_lang , trg_lang ) <EOL> sentence_no = int ( line_data [ <NUM_LIT:2> ] ) <EOL> sysIds = line_data [ <NUM_LIT:7> : <NUM_LIT> : <NUM_LIT:2> ] <EOL> sysRanks = line_data [ - <NUM_LIT:5> : ] <EOL> decisions = [ ] <EOL> for a , b in combinations ( range ( <NUM_LIT:5> ) , <NUM_LIT:2> ) : <EOL> if sysRanks [ a ] > sysRanks [ b ] : <EOL> decisions . append ( '<STR_LIT>' . format ( sysIds [ a ] , sysIds [ b ] ) ) <EOL> elif sysRanks [ a ] < sysRanks [ b ] : <EOL> decisions . append ( '<STR_LIT>' . format ( sysIds [ b ] , sysIds [ a ] ) ) <EOL> for ranking_decision in decisions : <EOL> data_point = ( language_pair , sentence_no , ranking_decision ) <EOL> add_data_point_to_dict ( data_point , APPRAISE_DATA ) <EOL> user_agreement = [ ] <EOL> for user_id in MTURK_DATA . keys ( ) : <EOL> rankings = <NUM_LIT:0> <EOL> overlap = <NUM_LIT:0> <EOL> for language_pair , sentence_nos in MTURK_DATA [ user_id ] . items ( ) : <EOL> for sentence_no , ranking_decisions in sentence_nos . items ( ) : <EOL> for decision , _ in ranking_decisions . items ( ) : <EOL> rankings = rankings + <NUM_LIT:1> <EOL> data_point = ( language_pair , sentence_no , decision ) <EOL> if compare_data_point_to_dict ( data_point , APPRAISE_DATA ) : <EOL> overlap = overlap + <NUM_LIT:1> <EOL> _data = ( overlap / float ( rankings or <NUM_LIT:1> ) , overlap , rankings , user_id ) <EOL> user_agreement . append ( _data ) <EOL> print <EOL> user_agreement . sort ( ) <EOL> user_agreement . reverse ( ) <EOL> print '<STR_LIT>' <EOL> for user_data in user_agreement : <EOL> print '<STR_LIT>' . format ( user_data [ <NUM_LIT:3> ] , <EOL> user_data [ <NUM_LIT:1> ] , user_data [ <NUM_LIT:2> ] , user_data [ <NUM_LIT:0> ] ) <EOL> print u'<STR_LIT:\n>' . join ( results ) </s>
