<s> import operator <EOL> import os . path <EOL> import sys <EOL> import time <EOL> import settings <EOL> from parse_CP import Parse <EOL> from phraseTable import PhraseTable <EOL> from refPhrases import RefPhrases <EOL> refFiles = [ ] <EOL> def readNParse ( sent_count ) : <EOL> '''<STR_LIT>''' <EOL> sent_indx = <NUM_LIT:0> <EOL> coverage_cnt = <NUM_LIT:0> <EOL> relaxed_decoding = False <EOL> refsLst = [ ] <EOL> tot_time = <NUM_LIT:0.0> <EOL> inF = open ( settings . opts . inFile , '<STR_LIT:r>' ) <EOL> sys . stderr . write ( "<STR_LIT>" % ( settings . opts . inFile ) ) <EOL> for sent in inF : <EOL> sent = sent . strip ( ) <EOL> if settings . opts . force_decode : refsLst = getReferences ( sent_indx ) <EOL> if settings . opts . skip_sents is not None and sent_indx < settings . opts . skip_sents : <EOL> sent_indx += <NUM_LIT:1> <EOL> sent_count += <NUM_LIT:1> <EOL> continue <EOL> parse_begin = time . time ( ) <EOL> parse_obj = Parse ( sent_count , sent , relaxed_decoding , refsLst ) <EOL> sys . stderr . write ( "<STR_LIT>" % ( sent_count , sent ) ) <EOL> if ( parse_obj . parse ( ) == <NUM_LIT> ) : <EOL> parse_obj = '<STR_LIT>' <EOL> relaxed_decoding = True <EOL> parse_obj = Parse ( sent_count , sent , relaxed_decoding , refsLst ) <EOL> dec_status = parse_obj . parse ( ) <EOL> relaxed_decoding = False <EOL> parse_time = time . time ( ) - parse_begin <EOL> tot_time += parse_time <EOL> sys . stderr . write ( "<STR_LIT>" % ( parse_time ) ) <EOL> sent_count += <NUM_LIT:1> <EOL> sent_indx += <NUM_LIT:1> <EOL> if settings . opts . force_decode : coverage_cnt += dec_status <EOL> parse_obj = '<STR_LIT>' <EOL> """<STR_LIT>""" <EOL> inF . close ( ) <EOL> sys . stderr . write ( "<STR_LIT>" % ( tot_time ) ) <EOL> if settings . opts . force_decode : <EOL> sys . stderr . write ( "<STR_LIT>" % ( coverage_cnt ) ) <EOL> sys . stderr . write ( "<STR_LIT>" % ( float ( coverage_cnt ) * <NUM_LIT:100> / float ( sent_indx ) ) ) <EOL> def getReferences ( sent_indx ) : <EOL> '''<STR_LIT>''' <EOL> global refFiles <EOL> refsLst = [ ] <EOL> refHLst = [ open ( file , '<STR_LIT:r>' ) for file in refFiles ] <EOL> for refH in refHLst : <EOL> indx = <NUM_LIT:0> <EOL> for sent in refH : <EOL> if indx == sent_indx : <EOL> sent = sent . strip ( ) <EOL> refsLst . append ( sent ) <EOL> break <EOL> indx += <NUM_LIT:1> <EOL> refH . close ( ) <EOL> del refHLst [ : ] <EOL> return refsLst <EOL> def getRefFiles ( ) : <EOL> '''<STR_LIT>''' <EOL> global refFiles <EOL> if os . path . exists ( settings . opts . refFile ) : refFiles . append ( settings . opts . refFile ) <EOL> else : <EOL> i = <NUM_LIT:0> <EOL> ref_i = settings . opts . refFile + str ( i ) <EOL> while os . path . exists ( ref_i ) : <EOL> refFiles . append ( ref_i ) <EOL> i += <NUM_LIT:1> <EOL> ref_i = settings . opts . refFile + str ( i ) <EOL> if not refFiles : <EOL> sys . stderr . write ( "<STR_LIT>" ) <EOL> sys . stderr . write ( "<STR_LIT>" ) <EOL> sys . exit ( <NUM_LIT:1> ) <EOL> def consolidateRules ( cntsFile ) : <EOL> sys . stderr . write ( "<STR_LIT>" % ( cntsFile ) ) <EOL> rulesUsedDict = { } <EOL> rC = open ( cntsFile , '<STR_LIT:r>' ) <EOL> try : <EOL> for line in rC : <EOL> ( src , tgt , cnts ) = line . split ( "<STR_LIT>" ) <EOL> rule = src + "<STR_LIT>" + tgt <EOL> if ( rulesUsedDict . has_key ( rule ) ) : rulesUsedDict [ rule ] += int ( cnts ) <EOL> else : rulesUsedDict [ rule ] = int ( cnts ) <EOL> finally : <EOL> rC . close ( ) <EOL> tot_used_rules = len ( rulesUsedDict . keys ( ) ) <EOL> tot_PT_rules = PhraseTable . getTotalRules ( ) <EOL> sys . stderr . write ( "<STR_LIT>" % ( tot_PT_rules ) ) <EOL> sys . stderr . write ( "<STR_LIT>" % ( tot_used_rules ) ) <EOL> sys . stderr . write ( "<STR_LIT>" % ( ( float ( tot_used_rules ) * <NUM_LIT> ) / float ( tot_PT_rules ) ) ) <EOL> wC = open ( cntsFile , '<STR_LIT:w>' ) <EOL> for rule , r_cnt in sorted ( rulesUsedDict . iteritems ( ) , key = operator . itemgetter ( <NUM_LIT:1> ) ) : <EOL> wC . write ( "<STR_LIT>" % ( rule , r_cnt ) ) <EOL> wC . close ( ) <EOL> def main ( ) : <EOL> global refFiles <EOL> sent_count = settings . opts . sentindex * settings . opts . sent_per_file <EOL> if settings . opts . force_decode : <EOL> getRefFiles ( ) <EOL> RefPhrases ( sent_count , refFiles ) <EOL> PhraseTable ( ) <EOL> readNParse ( sent_count ) <EOL> if __name__ == "<STR_LIT:__main__>" : <EOL> settings . args ( ) <EOL> main ( ) </s>
