<s> import sys <EOL> max_phr_len = <NUM_LIT:10> <EOL> tot_src_terms = <NUM_LIT:7> <EOL> X1_only = False <EOL> weight_rules = False <EOL> tight_phrases_only = True <EOL> rule_indx = <NUM_LIT:1> <EOL> tot_rules_derived = <NUM_LIT:0> <EOL> srcWrds = [ ] <EOL> tgtWrds = [ ] <EOL> ruleDict = { } <EOL> alignDoD = { } <EOL> revAlignDoD = { } <EOL> sentInitDoD = { } <EOL> ppairRulesSet = set ( [ ] ) <EOL> tgtCntDict = { } <EOL> ruleIndxCntDict = { } <EOL> fAlignDoD = { } <EOL> rAlignDoD = { } <EOL> def readPhraseSpan ( spanFile , outFile , tgtFile ) : <EOL> '<STR_LIT>' <EOL> global tight_phrases_only <EOL> global ruleDict , ruleIndxCntDict , tgtCntDict <EOL> global srcWrds , tgtWrds <EOL> sent_count = <NUM_LIT:0> <EOL> phrLst = [ ] <EOL> print "<STR_LIT>" , spanFile <EOL> inF = open ( spanFile , '<STR_LIT:r>' ) <EOL> for line in inF : <EOL> line = line . strip ( ) <EOL> if line . startswith ( '<STR_LIT>' ) : <EOL> src = line . replace ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> srcWrds = src . split ( ) <EOL> elif line . startswith ( '<STR_LIT>' ) : <EOL> tgt = line . replace ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> tgtWrds = tgt . split ( ) <EOL> elif line . startswith ( '<STR_LIT>' ) : <EOL> align = line . replace ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> for align_pos in align . split ( ) : <EOL> m = align_pos . split ( '<STR_LIT:->' ) <EOL> try : <EOL> alignDoD [ m [ <NUM_LIT:0> ] ] [ m [ <NUM_LIT:1> ] ] = <NUM_LIT:1> <EOL> except KeyError : <EOL> alignDoD [ m [ <NUM_LIT:0> ] ] = { } <EOL> alignDoD [ m [ <NUM_LIT:0> ] ] [ m [ <NUM_LIT:1> ] ] = <NUM_LIT:1> <EOL> try : <EOL> revAlignDoD [ m [ <NUM_LIT:1> ] ] [ m [ <NUM_LIT:0> ] ] = <NUM_LIT:1> <EOL> except KeyError : <EOL> revAlignDoD [ m [ <NUM_LIT:1> ] ] = { } <EOL> revAlignDoD [ m [ <NUM_LIT:1> ] ] [ m [ <NUM_LIT:0> ] ] = <NUM_LIT:1> <EOL> elif line . startswith ( '<STR_LIT>' ) : continue <EOL> elif line . startswith ( '<STR_LIT>' ) : <EOL> xtractRules ( ) <EOL> for rule in ruleDict . keys ( ) : compFeatureCounts ( rule ) <EOL> alignDoD . clear ( ) <EOL> revAlignDoD . clear ( ) <EOL> ruleDict . clear ( ) <EOL> sentInitDoD . clear ( ) <EOL> sent_count += <NUM_LIT:1> <EOL> if sent_count % <NUM_LIT:1000> == <NUM_LIT:0> : <EOL> print "<STR_LIT>" % sent_count <EOL> else : <EOL> unaligned_edge = False <EOL> phrLst = [ int ( x ) for x in line . split ( ) ] <EOL> if not alignDoD . has_key ( str ( phrLst [ <NUM_LIT:0> ] ) ) or not revAlignDoD . has_key ( str ( phrLst [ <NUM_LIT:2> ] ) ) or not alignDoD . has_key ( str ( phrLst [ <NUM_LIT:1> ] ) ) or not revAlignDoD . has_key ( str ( phrLst [ <NUM_LIT:3> ] ) ) : <EOL> if tight_phrases_only : continue <EOL> else : unaligned_edge = True <EOL> sphr_len = phrLst [ <NUM_LIT:1> ] - phrLst [ <NUM_LIT:0> ] + <NUM_LIT:1> <EOL> init_phr_pair = ( '<STR_LIT:U+0020>' . join ( [ str ( x ) for x in xrange ( phrLst [ <NUM_LIT:0> ] , phrLst [ <NUM_LIT:1> ] + <NUM_LIT:1> ) ] ) , '<STR_LIT:U+0020>' . join ( [ str ( x ) for x in xrange ( phrLst [ <NUM_LIT:2> ] , phrLst [ <NUM_LIT:3> ] + <NUM_LIT:1> ) ] ) ) <EOL> if unaligned_edge : <EOL> ruleDict [ init_phr_pair ] = <NUM_LIT:1.0> <EOL> continue <EOL> if not sentInitDoD . has_key ( sphr_len ) : <EOL> sentInitDoD [ sphr_len ] = { } <EOL> sentInitDoD [ sphr_len ] [ init_phr_pair ] = <NUM_LIT:1> <EOL> inF . close ( ) <EOL> with open ( outFile , '<STR_LIT:w>' ) as oF : <EOL> for rule in sorted ( ruleIndxCntDict . iterkeys ( ) ) : <EOL> r_indx , rule_count = ruleIndxCntDict [ rule ] <EOL> f_alignments = '<STR_LIT>' . join ( fAlignDoD [ r_indx ] . keys ( ) ) <EOL> r_alignments = '<STR_LIT>' . join ( rAlignDoD [ r_indx ] . keys ( ) ) <EOL> oF . write ( "<STR_LIT>" % ( rule , rule_count , r_alignments , f_alignments ) ) <EOL> with open ( tgtFile , '<STR_LIT:w>' ) as tF : <EOL> for tgt in sorted ( tgtCntDict . iterkeys ( ) ) : <EOL> tF . write ( "<STR_LIT>" % ( tgt , tgtCntDict [ tgt ] ) ) <EOL> return None <EOL> def xtractRules ( ) : <EOL> '''<STR_LIT>''' <EOL> for sphr_len in xrange ( max_phr_len , <NUM_LIT:1> , - <NUM_LIT:1> ) : <EOL> if sentInitDoD . has_key ( sphr_len ) : <EOL> check4Subphrase ( sphr_len ) <EOL> sphr_len = <NUM_LIT:1> <EOL> if sentInitDoD . has_key ( sphr_len ) : <EOL> for init_phr_pair in sentInitDoD [ sphr_len ] : <EOL> if ruleDict . has_key ( init_phr_pair ) : ruleDict [ init_phr_pair ] += <NUM_LIT:1.0> <EOL> else : ruleDict [ init_phr_pair ] = <NUM_LIT:1.0> <EOL> def check4Subphrase ( sphr_len ) : <EOL> global tot_src_terms , X1_only , tot_rules_derived <EOL> global ppairRulesSet <EOL> ppairRulesDict = { } <EOL> for phr_pair in sentInitDoD [ sphr_len ] . keys ( ) : <EOL> rule_cnt = <NUM_LIT:0.0> <EOL> tot_rules_derived = <NUM_LIT:0> <EOL> ppairRulesSet . clear ( ) <EOL> ppairRulesDict . clear ( ) <EOL> if sphr_len <= tot_src_terms : <EOL> tot_rules_derived += <NUM_LIT:1> <EOL> if ppairRulesDict . has_key ( phr_pair ) : ppairRulesDict [ phr_pair ] += <NUM_LIT:1> <EOL> else : ppairRulesDict [ phr_pair ] = <NUM_LIT:1> <EOL> iterateInitPhrPairs ( sphr_len , phr_pair ) <EOL> while ppairRulesSet : <EOL> sub_phr_pair = ppairRulesSet . pop ( ) <EOL> sub_s_span , sub_t_span = sub_phr_pair <EOL> sub_phr_slen = len ( sub_s_span . split ( ) ) <EOL> if sub_phr_slen <= tot_src_terms : <EOL> if ppairRulesDict . has_key ( sub_phr_pair ) : ppairRulesDict [ sub_phr_pair ] += <NUM_LIT:1> <EOL> else : ppairRulesDict [ sub_phr_pair ] = <NUM_LIT:1> <EOL> if ( X1_only and sub_s_span . find ( '<STR_LIT>' ) != - <NUM_LIT:1> ) or ( sub_s_span . find ( '<STR_LIT>' ) != - <NUM_LIT:1> ) : <EOL> pass <EOL> elif sub_s_span . find ( '<STR_LIT>' ) != - <NUM_LIT:1> and not isRuleDecomposable ( sub_s_span ) : <EOL> pass <EOL> else : <EOL> iterateInitPhrPairs ( sub_phr_slen , sub_phr_pair ) <EOL> if tot_rules_derived > <NUM_LIT:0> : rule_cnt = <NUM_LIT:1.0> / tot_rules_derived <EOL> for rule in ppairRulesDict . keys ( ) : <EOL> weighted_cnt = rule_cnt <EOL> if weight_rules : weighted_cnt *= ppairRulesDict [ rule ] <EOL> if ruleDict . has_key ( rule ) : ruleDict [ rule ] += weighted_cnt <EOL> else : ruleDict [ rule ] = weighted_cnt <EOL> def isRuleDecomposable ( s_span ) : <EOL> srcRuleTerms = s_span . split ( ) <EOL> try : <EOL> term_indx = srcRuleTerms . index ( '<STR_LIT>' ) <EOL> if term_indx >= <NUM_LIT:2> or ( len ( srcRuleTerms ) - <NUM_LIT:1> - term_indx ) >= <NUM_LIT:2> : <EOL> return True <EOL> else : return False <EOL> except ValueError : <EOL> if srcRuleTerms : return True <EOL> else : return False <EOL> def iterateInitPhrPairs ( sphr_len , ( s_span , t_span ) ) : <EOL> global tot_rules_derived <EOL> global ppairRulesSet <EOL> for sub_phr_slen in xrange ( sphr_len - <NUM_LIT:1> , <NUM_LIT:0> , - <NUM_LIT:1> ) : <EOL> if not sentInitDoD . has_key ( sub_phr_slen ) : continue <EOL> for sub_phr_pair in sentInitDoD [ sub_phr_slen ] . keys ( ) : <EOL> subSpanTuple = sentInitDoD [ sub_phr_slen ] [ sub_phr_pair ] <EOL> src_side_compatible = checkRuleCompatibility ( s_span , sub_phr_pair [ <NUM_LIT:0> ] ) <EOL> tgt_side_compatible = checkRuleCompatibility ( t_span , sub_phr_pair [ <NUM_LIT:1> ] ) <EOL> if src_side_compatible and tgt_side_compatible : <EOL> sub_s_span , sub_t_span = sub_phr_pair <EOL> rule = checkConstraints ( s_span , t_span , sub_s_span , sub_t_span ) <EOL> if rule is not None : <EOL> ppairRulesSet . add ( rule ) <EOL> tot_rules_derived += <NUM_LIT:1> <EOL> def checkRuleCompatibility ( rule , sub_rule ) : <EOL> '<STR_LIT>' <EOL> if rule . startswith ( sub_rule + '<STR_LIT:U+0020>' ) or rule . endswith ( '<STR_LIT:U+0020>' + sub_rule ) or rule . find ( '<STR_LIT:U+0020>' + sub_rule + '<STR_LIT:U+0020>' ) != - <NUM_LIT:1> : <EOL> return True <EOL> return False <EOL> def checkConstraints ( src , tgt , sub_src , sub_tgt ) : <EOL> '''<STR_LIT>''' <EOL> X1_indx = src . find ( '<STR_LIT>' ) <EOL> if X1_indx != - <NUM_LIT:1> : <EOL> if ( len ( src . split ( ) ) - len ( sub_src . split ( ) ) + <NUM_LIT:1> ) > tot_src_terms : <EOL> return None <EOL> adj_X1_right = sub_src + '<STR_LIT>' <EOL> adj_X1_left = '<STR_LIT>' + sub_src <EOL> if src . startswith ( adj_X1_right ) or src . find ( '<STR_LIT:U+0020>' + adj_X1_right ) != - <NUM_LIT:1> or src . endswith ( adj_X1_left ) or src . find ( adj_X1_left + '<STR_LIT:U+0020>' ) != - <NUM_LIT:1> : <EOL> return None <EOL> if src . startswith ( sub_src + '<STR_LIT:U+0020>' ) or src . find ( '<STR_LIT:U+0020>' + sub_src + '<STR_LIT:U+0020>' , <NUM_LIT:0> , X1_indx ) != - <NUM_LIT:1> : <EOL> src = src . replace ( '<STR_LIT>' , '<STR_LIT>' , <NUM_LIT:1> ) <EOL> tgt = tgt . replace ( '<STR_LIT>' , '<STR_LIT>' , <NUM_LIT:1> ) <EOL> rep_str = '<STR_LIT>' <EOL> else : rep_str = '<STR_LIT>' <EOL> else : <EOL> rep_str = '<STR_LIT>' <EOL> s_side = replaceItem ( src , sub_src , rep_str ) <EOL> t_side = replaceItem ( tgt , sub_tgt , rep_str ) <EOL> tgtTermLst = t_side . split ( ) <EOL> aligned_terminal = False <EOL> for pos in s_side . split ( ) : <EOL> if not alignDoD . has_key ( pos ) : continue <EOL> for tgt_align_pos in alignDoD [ pos ] . keys ( ) : <EOL> try : <EOL> tgtTermLst . index ( tgt_align_pos ) <EOL> aligned_terminal = True <EOL> break <EOL> except ValueError : <EOL> pass <EOL> if aligned_terminal : <EOL> return ( s_side , t_side ) <EOL> return None <EOL> def replaceItem ( full_str , p_match , rep_str ) : <EOL> if len ( p_match . split ( ) ) > <NUM_LIT:1> : <EOL> return full_str . replace ( p_match , rep_str , <NUM_LIT:1> ) <EOL> else : <EOL> tmpLst = [ ] <EOL> for tok in full_str . split ( ) : <EOL> if tok == p_match : <EOL> tmpLst . append ( rep_str ) <EOL> else : <EOL> tmpLst . append ( tok ) <EOL> return '<STR_LIT:U+0020>' . join ( tmpLst ) <EOL> def compFeatureCounts ( rule ) : <EOL> '<STR_LIT>' <EOL> global srcWrds , tgtWrds <EOL> global fAlignDoD , rAlignDoD <EOL> srcLexLst = [ ] <EOL> tgtLexLst = [ ] <EOL> alignLst = [ ] <EOL> sPosLst = rule [ <NUM_LIT:0> ] . split ( ) <EOL> tPosLst = rule [ <NUM_LIT:1> ] . split ( ) <EOL> item_indx = <NUM_LIT:0> <EOL> for s_tok in sPosLst : <EOL> if s_tok . startswith ( '<STR_LIT>' ) : <EOL> srcLexLst . append ( s_tok ) <EOL> else : <EOL> srcLexLst . append ( srcWrds [ int ( s_tok ) ] ) <EOL> alignment = getFwrdAlignment ( item_indx , s_tok , tPosLst ) <EOL> alignLst . append ( alignment ) <EOL> item_indx += <NUM_LIT:1> <EOL> fAlignment = '<STR_LIT:U+0020>' . join ( alignLst ) <EOL> del alignLst [ : ] <EOL> item_indx = <NUM_LIT:0> <EOL> for t_tok in tPosLst : <EOL> if t_tok . startswith ( '<STR_LIT>' ) : <EOL> tgtLexLst . append ( t_tok ) <EOL> else : <EOL> tgtLexLst . append ( tgtWrds [ int ( t_tok ) ] ) <EOL> alignment = getRvrsAlignment ( item_indx , t_tok , sPosLst ) <EOL> alignLst . append ( alignment ) <EOL> item_indx += <NUM_LIT:1> <EOL> rAlignment = '<STR_LIT:U+0020>' . join ( alignLst ) <EOL> curr_rindx = updateRuleCount ( '<STR_LIT:U+0020>' . join ( srcLexLst ) , '<STR_LIT:U+0020>' . join ( tgtLexLst ) , rule ) <EOL> f_align_indx = getAlignIndex ( fAlignment ) <EOL> r_align_indx = getAlignIndex ( rAlignment ) <EOL> if not fAlignDoD . has_key ( curr_rindx ) : <EOL> fAlignDoD [ curr_rindx ] = { } <EOL> rAlignDoD [ curr_rindx ] = { } <EOL> if not fAlignDoD [ curr_rindx ] . has_key ( f_align_indx ) : <EOL> fAlignDoD [ curr_rindx ] [ f_align_indx ] = <NUM_LIT:1> <EOL> if not rAlignDoD [ curr_rindx ] . has_key ( r_align_indx ) : <EOL> rAlignDoD [ curr_rindx ] [ r_align_indx ] = <NUM_LIT:1> <EOL> def updateRuleCount ( mc_src , mc_tgt , rule ) : <EOL> '''<STR_LIT>''' <EOL> global rule_indx , ruleDict , ruleIndxCntDict , tgtCntDict <EOL> if not tgtCntDict . has_key ( mc_tgt ) : <EOL> tgtCntDict [ mc_tgt ] = <NUM_LIT:0> <EOL> tgtCntDict [ mc_tgt ] += ruleDict [ rule ] <EOL> mc_key = mc_src + '<STR_LIT>' + mc_tgt <EOL> if ruleIndxCntDict . has_key ( mc_key ) : <EOL> curr_rindx , curr_cnt = ruleIndxCntDict [ mc_key ] <EOL> ruleIndxCntDict [ mc_key ] = ( curr_rindx , curr_cnt + ruleDict [ rule ] ) <EOL> else : <EOL> ruleIndxCntDict [ mc_key ] = ( rule_indx , ruleDict [ rule ] ) <EOL> curr_rindx = rule_indx <EOL> rule_indx += <NUM_LIT:1> <EOL> return curr_rindx <EOL> def getAlignIndex ( align_str ) : <EOL> tmpLst = align_str . split ( '<STR_LIT:U+0020>' ) <EOL> tmpLst . sort ( ) <EOL> aindx = '<STR_LIT>' . join ( tmpLst ) <EOL> return aindx . replace ( '<STR_LIT:->' , '<STR_LIT>' ) <EOL> def getFwrdAlignment ( item_indx , s_pos , tPosLst ) : <EOL> '<STR_LIT>' <EOL> alignLst = [ ] <EOL> if alignDoD . has_key ( s_pos ) : <EOL> alignKeyLst = alignDoD [ s_pos ] . keys ( ) <EOL> alignKeyLst . sort ( ) <EOL> for t_pos in alignKeyLst : <EOL> try : <EOL> alignment = str ( item_indx ) + '<STR_LIT:->' + str ( tPosLst . index ( t_pos ) ) <EOL> alignLst . append ( alignment ) <EOL> except ValueError : <EOL> pass <EOL> else : <EOL> alignLst . append ( str ( item_indx ) + '<STR_LIT>' ) <EOL> return '<STR_LIT:U+0020>' . join ( alignLst ) <EOL> def getRvrsAlignment ( item_indx , t_pos , sPosLst ) : <EOL> '<STR_LIT>' <EOL> alignLst = [ ] <EOL> if revAlignDoD . has_key ( t_pos ) : <EOL> alignKeyLst = revAlignDoD [ t_pos ] . keys ( ) <EOL> alignKeyLst . sort ( ) <EOL> for s_pos in alignKeyLst : <EOL> try : <EOL> alignment = str ( sPosLst . index ( s_pos ) ) + '<STR_LIT:->' + str ( item_indx ) <EOL> alignLst . append ( alignment ) <EOL> except ValueError : <EOL> pass <EOL> else : <EOL> alignLst . append ( '<STR_LIT>' + str ( item_indx ) ) <EOL> return '<STR_LIT:U+0020>' . join ( alignLst ) <EOL> def main ( ) : <EOL> global tight_phrases_only <EOL> global tot_src_terms <EOL> global X1_only <EOL> if len ( sys . argv ) < <NUM_LIT:4> and len ( sys . argv ) > <NUM_LIT:6> : <EOL> print '<STR_LIT>' % ( sys . argv [ <NUM_LIT:0> ] ) <EOL> print '<STR_LIT>' <EOL> sys . exit ( ) <EOL> file_indx = sys . argv [ <NUM_LIT:1> ] <EOL> dDir = sys . argv [ <NUM_LIT:2> ] <EOL> oDir = sys . argv [ <NUM_LIT:3> ] <EOL> if len ( sys . argv ) > <NUM_LIT:4> : <EOL> tot_src_terms = int ( sys . argv [ <NUM_LIT:4> ] ) <EOL> if len ( sys . argv ) == <NUM_LIT:6> and sys . argv [ <NUM_LIT:5> ] != '<STR_LIT:True>' and sys . argv [ <NUM_LIT:5> ] != '<STR_LIT:False>' : <EOL> print "<STR_LIT>" <EOL> sys . exit ( <NUM_LIT:1> ) <EOL> if len ( sys . argv ) == <NUM_LIT:6> and sys . argv [ <NUM_LIT:5> ] == '<STR_LIT:True>' : <EOL> X1_only = True <EOL> if len ( sys . argv ) == <NUM_LIT:7> and sys . argv [ <NUM_LIT:6> ] == '<STR_LIT:False>' : <EOL> tight_phrases_only = False <EOL> print "<STR_LIT>" , tot_src_terms <EOL> print "<STR_LIT>" , tight_phrases_only <EOL> if not dDir . endswith ( "<STR_LIT:/>" ) : dDir += "<STR_LIT:/>" <EOL> if not oDir . endswith ( "<STR_LIT:/>" ) : oDir += "<STR_LIT:/>" <EOL> spanFile = dDir + file_indx + '<STR_LIT>' <EOL> outFile = oDir + file_indx + '<STR_LIT>' <EOL> tgtFile = oDir + '<STR_LIT>' + file_indx + '<STR_LIT>' <EOL> readPhraseSpan ( spanFile , outFile , tgtFile ) <EOL> if __name__ == '<STR_LIT:__main__>' : <EOL> main ( ) </s>
