<s> import sys , time <EOL> from lmKENLM import KENLangModel <EOL> from lmSRILM import SRILangModel <EOL> import settings <EOL> class LanguageModelManager ( object ) : <EOL> debug = False <EOL> use_srilm = False <EOL> no_dscnt_UNKlm = False <EOL> lmLst = [ ] <EOL> lmWgts = [ ] <EOL> __slots__ = ( ) <EOL> @ classmethod <EOL> def initLMs ( cls , tot_lm_feats , lmTupLst , use_srilm = False ) : <EOL> assert ( tot_lm_feats == len ( lmTupLst ) ) , "<STR_LIT>" <EOL> cls . debug = settings . opts . debug <EOL> cls . use_srilm = use_srilm <EOL> cls . no_dscnt_UNKlm = settings . opts . no_dscnt_UNKlm <EOL> cls . lmLst = [ ] <EOL> cls . lmWgts = [ ] <EOL> for lm_indx in xrange ( tot_lm_feats ) : <EOL> ( lm_order , lm_file ) = lmTupLst [ lm_indx ] <EOL> sys . stderr . write ( "<STR_LIT>" % ( lm_file ) ) <EOL> t_beg = time . time ( ) <EOL> if cls . use_srilm : lm = SRILangModel ( lm_order , lm_file , settings . opts . elider ) <EOL> else : lm = KENLangModel ( lm_order , lm_file , settings . opts . elider ) <EOL> cls . lmLst . append ( lm ) <EOL> t_end = time . time ( ) <EOL> sys . stderr . write ( "<STR_LIT>" % ( t_end - t_beg ) ) <EOL> @ classmethod <EOL> def closeLM ( cls ) : <EOL> '''<STR_LIT>''' <EOL> for lm_obj in cls . lmLst : <EOL> lm_obj = '<STR_LIT>' <EOL> @ classmethod <EOL> def setLMInfo ( cls , lmWgtsLst ) : <EOL> assert ( len ( cls . lmLst ) == len ( lmWgtsLst ) ) , "<STR_LIT>" <EOL> cls . lmWgts = lmWgtsLst [ : ] <EOL> @ classmethod <EOL> def scoreLMFeat ( cls , tgt_phr ) : <EOL> '''<STR_LIT>''' <EOL> lmHeu = [ <NUM_LIT:0.0> for x in cls . lmLst ] <EOL> for tgt_term in tgt_phr . split ( ) : <EOL> if tgt_term == '<STR_LIT>' or tgt_term == '<STR_LIT>' : continue <EOL> lm_indx = <NUM_LIT:0> <EOL> for lm_obj in cls . lmLst : <EOL> lmHeu [ lm_indx ] += lm_obj . queryLM ( tgt_term , <NUM_LIT:1> ) <EOL> lm_indx += <NUM_LIT:1> <EOL> lm_indx = <NUM_LIT:0> <EOL> lm_score = <NUM_LIT:0.0> <EOL> for lm_H in lmHeu : <EOL> lm_score += ( cls . lmWgts [ lm_indx ] * lm_H ) <EOL> lm_indx += <NUM_LIT:1> <EOL> return lm_score <EOL> @ classmethod <EOL> def getLMScore ( cls , lmFeatVec ) : <EOL> lm_score = <NUM_LIT:0.0> <EOL> lm_indx = <NUM_LIT:0> <EOL> for lm_val in lmFeatVec : <EOL> lm_score += ( cls . lmWgts [ lm_indx ] * lm_val ) <EOL> lm_indx += <NUM_LIT:1> <EOL> return lm_score <EOL> @ classmethod <EOL> def copyLMScores ( cls , featVecSrc , featVecTgt ) : <EOL> lm_score = <NUM_LIT:0.0> <EOL> for lm_indx in xrange ( len ( cls . lmWgts ) ) : <EOL> featVecTgt [ lm_indx ] = featVecSrc [ lm_indx ] <EOL> return lm_score <EOL> @ classmethod <EOL> def adjustUNKLMScore ( cls , cand_hyp , featVec ) : <EOL> lm_indx = <NUM_LIT:0> <EOL> lmFeats = [ ] <EOL> for lm_obj in cls . lmLst : <EOL> if cls . no_dscnt_UNKlm : lmFeats . append ( str ( "<STR_LIT>" % featVec [ lm_indx ] ) ) <EOL> else : lmFeats . append ( str ( "<STR_LIT>" % ( featVec [ lm_indx ] - lm_obj . calcUNKLMScore ( cand_hyp ) ) ) ) <EOL> lm_indx += <NUM_LIT:1> <EOL> return '<STR_LIT:U+0020>' . join ( lmFeats ) <EOL> @ classmethod <EOL> def helperLM ( cls , consItems , is_last_cell , lmFeatVec ) : <EOL> '''<STR_LIT>''' <EOL> lm_indx = <NUM_LIT:0> <EOL> tot_lm_heu = <NUM_LIT:0.0> <EOL> tot_lm_score = <NUM_LIT:0.0> <EOL> for lm_obj in cls . lmLst : <EOL> cons_item = consItems [ lm_indx ] <EOL> lm_lprob = <NUM_LIT:0.0> <EOL> if cons_item . e_len < lm_obj . lm_order : <EOL> if cls . use_srilm : lm_H = lm_obj . getLMHeuCost ( cons_item . eTgtLst , cons_item . e_len ) <EOL> else : lm_H = lm_obj . getLMHeuCost ( cons_item ) <EOL> else : <EOL> if cls . use_srilm : <EOL> ( lm_lprob , eTgtLst ) = lm_obj . scorePhrnElide ( cons_item . eTgtLst , cons_item . e_len , cons_item . mgramSpans ) <EOL> lm_H = lm_obj . getLMHeuCost ( cons_item . eTgtLst , cons_item . e_len ) <EOL> else : <EOL> lm_lprob = lm_obj . scoremGrams ( cons_item . phrStateTupLst ) <EOL> lm_H = lm_obj . getLMHeuCost ( cons_item ) <EOL> if ( is_last_cell ) : <EOL> if cons_item . new_elided_tgt : lm_lprob += lm_H <EOL> lm_H = <NUM_LIT:0.0> <EOL> lmFeatVec [ lm_indx ] += lm_lprob <EOL> tot_lm_heu += ( cls . lmWgts [ lm_indx ] * lm_H ) <EOL> if cons_item . new_elided_tgt : <EOL> tot_lm_score += ( cls . lmWgts [ lm_indx ] * lm_lprob ) <EOL> lm_indx += <NUM_LIT:1> <EOL> return ( tot_lm_score , tot_lm_heu ) <EOL> @ classmethod <EOL> def helperConsItem ( cls , is_last_cell , cell_type , cell_span , goalTgt , anteHyps , anteItemStates ) : <EOL> anteTgts = [ ] <EOL> for tgt in anteHyps : <EOL> anteTgts . append ( tgt . split ( ) ) <EOL> tgt = '<STR_LIT>' <EOL> lm_indx = <NUM_LIT:0> <EOL> consItemStates = [ ] <EOL> for lm_obj in cls . lmLst : <EOL> anteItems = [ x [ lm_indx ] for x in anteItemStates ] <EOL> cons_item = ConsequentItem ( goalTgt ) <EOL> cons_item . setStateNMerge ( is_last_cell , cell_type , cell_span , anteTgts , anteItems ) <EOL> tgt = cons_item . mergeAntecedents ( anteHyps , anteItems , lm_obj . lm_order , lm_obj ) <EOL> if LanguageModelManager . debug : <EOL> if lm_indx > <NUM_LIT:0> : <EOL> assert ( prev_tgt == tgt ) , "<STR_LIT>" % ( prev_tgt , tgt ) <EOL> prev_tgt = tgt <EOL> consItemStates . append ( cons_item ) <EOL> lm_indx += <NUM_LIT:1> <EOL> return tgt , consItemStates <EOL> class ConsequentItem ( object ) : <EOL> """<STR_LIT>""" <EOL> __slots__ = "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" <EOL> def __init__ ( self , goalTgt ) : <EOL> self . e_len = <NUM_LIT:0> <EOL> self . r_lm_state = None <EOL> self . eTgtLst = goalTgt [ : ] <EOL> self . phrStateTupLst = [ ] <EOL> self . new_elided_tgt = True <EOL> def verify ( self , lm_obj , anteItems ) : <EOL> '''<STR_LIT>''' <EOL> if self . r_lm_state is not None : lm_obj . printState ( self . r_lm_state ) <EOL> else : print "<STR_LIT>" <EOL> if len ( anteItems ) >= <NUM_LIT:1> : <EOL> if anteItems [ <NUM_LIT:0> ] . r_lm_state is not None : lm_obj . printState ( anteItems [ <NUM_LIT:0> ] . r_lm_state ) <EOL> else : print "<STR_LIT>" <EOL> if len ( anteItems ) == <NUM_LIT:2> : <EOL> if anteItems [ <NUM_LIT:1> ] . r_lm_state is not None : lm_obj . printState ( anteItems [ <NUM_LIT:1> ] . r_lm_state ) <EOL> else : print "<STR_LIT>" <EOL> def setStateNMerge ( self , is_last_cell , cell_type , cell_span , anteTgts , anteItems ) : <EOL> '''<STR_LIT>''' <EOL> beg_state = <NUM_LIT:0> <EOL> end_state = <NUM_LIT:0> <EOL> if self . eTgtLst [ <NUM_LIT:0> ] == '<STR_LIT>' or self . eTgtLst [ <NUM_LIT:0> ] == '<STR_LIT>' : beg_state = <NUM_LIT:1> <EOL> elif self . eTgtLst [ <NUM_LIT:0> ] == '<STR_LIT>' : beg_state = <NUM_LIT:2> <EOL> if self . eTgtLst [ - <NUM_LIT:1> ] == '<STR_LIT>' : end_state = <NUM_LIT:1> <EOL> elif self . eTgtLst [ - <NUM_LIT:1> ] == '<STR_LIT>' : end_state = <NUM_LIT:2> <EOL> edgeTup = ( beg_state , end_state ) <EOL> if ( is_last_cell or ( cell_span [ <NUM_LIT:0> ] == <NUM_LIT:0> and cell_type == '<STR_LIT:S>' ) ) : <EOL> self . addLeftSMarker ( edgeTup , anteTgts ) <EOL> if ( is_last_cell ) : <EOL> edgeTup = self . addRightSMarker ( edgeTup , anteTgts ) <EOL> self . setLMState ( edgeTup , anteItems ) <EOL> def addLeftSMarker ( self , edgeTup , anteTgts ) : <EOL> '''<STR_LIT>''' <EOL> if ( edgeTup [ <NUM_LIT:0> ] == <NUM_LIT:0> and not self . eTgtLst [ <NUM_LIT:0> ] == '<STR_LIT>' ) or ( edgeTup [ <NUM_LIT:0> ] == <NUM_LIT:1> and not anteTgts [ <NUM_LIT:0> ] [ <NUM_LIT:0> ] == '<STR_LIT>' ) or ( edgeTup [ <NUM_LIT:0> ] == <NUM_LIT:2> and not anteTgts [ <NUM_LIT:1> ] [ <NUM_LIT:0> ] == '<STR_LIT>' ) : <EOL> self . eTgtLst . insert ( <NUM_LIT:0> , '<STR_LIT>' ) <EOL> def addRightSMarker ( self , edgeTup , anteTgts ) : <EOL> '''<STR_LIT>''' <EOL> if ( edgeTup [ <NUM_LIT:1> ] == <NUM_LIT:0> and not self . eTgtLst [ - <NUM_LIT:1> ] == '<STR_LIT>' ) or ( edgeTup [ <NUM_LIT:1> ] == <NUM_LIT:1> and not anteTgts [ <NUM_LIT:0> ] [ - <NUM_LIT:1> ] == '<STR_LIT>' ) or ( edgeTup [ <NUM_LIT:1> ] == <NUM_LIT:2> and not anteTgts [ <NUM_LIT:1> ] [ - <NUM_LIT:1> ] == '<STR_LIT>' ) : <EOL> self . eTgtLst . append ( '<STR_LIT>' ) <EOL> self . r_lm_state = None <EOL> if edgeTup [ <NUM_LIT:1> ] != <NUM_LIT:0> : edgeTup = ( edgeTup [ <NUM_LIT:0> ] , <NUM_LIT:0> ) <EOL> return edgeTup <EOL> def setLMState ( self , edgeTup , anteItems ) : <EOL> '''<STR_LIT>''' <EOL> if self . r_lm_state is not None : pass <EOL> elif edgeTup [ <NUM_LIT:1> ] == <NUM_LIT:1> : <EOL> self . r_lm_state = anteItems [ <NUM_LIT:0> ] . r_lm_state <EOL> elif edgeTup [ <NUM_LIT:1> ] == <NUM_LIT:2> : <EOL> self . r_lm_state = anteItems [ <NUM_LIT:1> ] . r_lm_state <EOL> def mergeAntecedents ( self , anteHyps , anteItems , lm_order , lm_obj ) : <EOL> mgram_beg = <NUM_LIT:0> <EOL> e_new_len = <NUM_LIT:0> <EOL> eTgtLstNew = [ ] <EOL> tgtItems = [ ] <EOL> phrStateTupLst = [ ] <EOL> curr_state = None <EOL> old_e_tgt = '<STR_LIT>' <EOL> for term in self . eTgtLst : <EOL> if term == "<STR_LIT>" or term == "<STR_LIT>" : <EOL> tgtItems . append ( anteHyps [ <NUM_LIT:0> ] ) <EOL> tempLst = anteItems [ <NUM_LIT:0> ] . eTgtLst <EOL> old_e_tgt = '<STR_LIT:U+0020>' . join ( tempLst ) <EOL> next_state = anteItems [ <NUM_LIT:0> ] . r_lm_state <EOL> elif term == "<STR_LIT>" : <EOL> tgtItems . append ( anteHyps [ <NUM_LIT:1> ] ) <EOL> tempLst = anteItems [ <NUM_LIT:1> ] . eTgtLst <EOL> next_state = anteItems [ <NUM_LIT:1> ] . r_lm_state <EOL> else : <EOL> tgtItems . append ( term ) <EOL> eTgtLstNew . append ( term ) <EOL> e_new_len += <NUM_LIT:1> <EOL> continue <EOL> eTgtLstNew . extend ( tempLst ) <EOL> mgram_beg , curr_state = self . handleEdges ( lm_order , mgram_beg , e_new_len , curr_state , next_state , tempLst , eTgtLstNew ) <EOL> e_new_len += len ( tempLst ) <EOL> self . elideNewTgt ( old_e_tgt , lm_obj , lm_order , mgram_beg , e_new_len , curr_state , eTgtLstNew ) <EOL> return '<STR_LIT:U+0020>' . join ( tgtItems ) <EOL> def elideNewTgt ( self , old_e_tgt , lm_obj , lm_order , mgram_beg , e_new_len , curr_state , eTgtLstNew ) : <EOL> if mgram_beg != e_new_len and ( curr_state is not None or e_new_len - mgram_beg >= lm_order ) : <EOL> self . phrStateTupLst . append ( ( '<STR_LIT:U+0020>' . join ( eTgtLstNew [ mgram_beg : e_new_len ] ) , curr_state ) ) <EOL> if e_new_len < lm_order : <EOL> self . eTgtLst = eTgtLstNew <EOL> else : <EOL> if self . r_lm_state is None and not settings . opts . no_lm_state : <EOL> self . r_lm_state = lm_obj . getLMState ( '<STR_LIT:U+0020>' . join ( eTgtLstNew [ - lm_order : ] ) ) <EOL> self . eTgtLst = eTgtLstNew [ <NUM_LIT:0> : lm_order - <NUM_LIT:1> ] + [ lm_obj . elider ] + eTgtLstNew [ <NUM_LIT:1> - lm_order : ] <EOL> self . e_len = len ( self . eTgtLst ) <EOL> if old_e_tgt == '<STR_LIT:U+0020>' . join ( self . eTgtLst ) : <EOL> self . new_elided_tgt = False <EOL> def handleEdges ( self , lm_order , mgram_beg , curr_e_len , curr_state , next_state , tempLst , eTgtLstNew ) : <EOL> for ante_term in tempLst : <EOL> if ante_term != settings . opts . elider : <EOL> curr_e_len += <NUM_LIT:1> <EOL> continue <EOL> if ( curr_e_len - mgram_beg >= lm_order or curr_state is not None ) and mgram_beg != curr_e_len : <EOL> self . phrStateTupLst . append ( ( '<STR_LIT:U+0020>' . join ( eTgtLstNew [ mgram_beg : curr_e_len ] ) , curr_state ) ) <EOL> curr_state = next_state <EOL> if settings . opts . no_lm_state : mgram_beg = curr_e_len + <NUM_LIT:1> <EOL> else : mgram_beg = curr_e_len + lm_order <EOL> break <EOL> return mgram_beg , curr_state </s>
