<s> import os <EOL> import time <EOL> import datetime <EOL> from sulci . thesaurus import Trigger , TriggerToDescriptor , Descriptor <EOL> from sulci . textmining import SemanticalTagger , GlobalPMI <EOL> from sulci . rules_templates import LemmatizerTemplateGenerator , RuleTemplate , ContextualTemplateGenerator , LexicalTemplateGenerator <EOL> from sulci . log import sulci_logger <EOL> from sulci import config <EOL> class ZMQTrainer ( object ) : <EOL> """<STR_LIT>""" <EOL> def setup_socket_master ( self ) : <EOL> """<STR_LIT>""" <EOL> import zmq <EOL> context = zmq . Context ( ) <EOL> self . reqsocket = context . socket ( zmq . XREQ ) <EOL> self . reqsocket . bind ( "<STR_LIT>" ) <EOL> self . pubsocket = context . socket ( zmq . PUB ) <EOL> self . pubsocket . bind ( "<STR_LIT>" ) <EOL> def setup_socket_slave ( self ) : <EOL> """<STR_LIT>""" <EOL> import zmq <EOL> context = zmq . Context ( ) <EOL> self . repsocket = context . socket ( zmq . XREP ) <EOL> self . repsocket . connect ( "<STR_LIT>" ) <EOL> self . subsocket = context . socket ( zmq . SUB ) <EOL> self . subsocket . connect ( "<STR_LIT>" ) <EOL> self . subsocket . setsockopt ( zmq . SUBSCRIBE , "<STR_LIT>" ) <EOL> self . subpoller = zmq . Poller ( ) <EOL> self . subpoller . register ( self . subsocket , zmq . POLLIN ) <EOL> class ContentBaseTrainer ( ZMQTrainer ) : <EOL> """<STR_LIT>""" <EOL> def do ( self , * args , ** kwargs ) : <EOL> try : <EOL> if self . mode == "<STR_LIT>" : <EOL> self . slave ( ) <EOL> else : <EOL> self . master ( ** kwargs ) <EOL> except KeyboardInterrupt : <EOL> self . stop ( ) <EOL> def master ( self , ** kwargs ) : <EOL> t_init = time . time ( ) <EOL> if self . mode == "<STR_LIT>" : <EOL> self . setup_socket_master ( ) <EOL> print "<STR_LIT>" <EOL> start = kwargs . get ( "<STR_LIT:start>" , None ) <EOL> qs = config . content_model_pks_for_trainer ( start = start ) <EOL> step = <NUM_LIT:1000> <EOL> forloop_remaining = total = len ( qs ) <EOL> for loop in range ( len ( qs ) / step + <NUM_LIT:1> ) : <EOL> _from = loop * step <EOL> _to = _from + step <EOL> current_qs = qs [ _from : _to ] <EOL> print "<STR_LIT>" % ( _from , _to ) <EOL> for pk in current_qs : <EOL> if self . mode == "<STR_LIT>" : <EOL> self . reqsocket . send_multipart ( [ str ( pk ) ] ) <EOL> else : <EOL> self . train ( pk ) <EOL> if self . mode == "<STR_LIT>" : <EOL> for idx in current_qs : <EOL> status = self . reqsocket . recv ( ) <EOL> forloop_remaining -= <NUM_LIT:1> <EOL> t_now = time . time ( ) <EOL> t_diff = t_now - t_init <EOL> average = t_diff / ( total - forloop_remaining ) <EOL> time_remaining = average * forloop_remaining <EOL> ETA = datetime . datetime . now ( ) + datetime . timedelta ( seconds = time_remaining ) <EOL> print "<STR_LIT>" % ( status , forloop_remaining , round ( average , <NUM_LIT:2> ) , ETA . strftime ( "<STR_LIT>" ) ) <EOL> self . stop ( ) <EOL> def stop ( self ) : <EOL> if self . mode == "<STR_LIT>" : <EOL> print "<STR_LIT>" <EOL> self . pubsocket . send ( "<STR_LIT>" ) <EOL> def slave ( self ) : <EOL> self . setup_socket_slave ( ) <EOL> print "<STR_LIT>" % os . getpid ( ) <EOL> while True : <EOL> if self . subpoller . poll ( <NUM_LIT:0> ) : <EOL> action = self . subsocket . recv ( ) [ <NUM_LIT:1> : ] <EOL> if action == "<STR_LIT>" : <EOL> print "<STR_LIT>" % os . getpid ( ) <EOL> return <EOL> idx , pk = self . repsocket . recv_multipart ( ) <EOL> print "<STR_LIT>" % ( os . getpid ( ) , pk ) <EOL> t_before = time . time ( ) <EOL> self . train ( pk ) <EOL> tot_time = time . time ( ) - t_before <EOL> self . repsocket . send_multipart ( <EOL> [ idx , "<STR_LIT>" % ( pk , round ( tot_time , <NUM_LIT:2> ) ) ] <EOL> ) <EOL> class SemanticalTrainer ( ContentBaseTrainer ) : <EOL> """<STR_LIT>""" <EOL> PENDING_EXT = "<STR_LIT>" <EOL> VALID_EXT = "<STR_LIT>" <EOL> def __init__ ( self , thesaurus , pos_tagger , mode = "<STR_LIT>" ) : <EOL> self . thesaurus = thesaurus <EOL> self . pos_tagger = pos_tagger <EOL> self . mode = mode <EOL> def train ( self , inst ) : <EOL> """<STR_LIT>""" <EOL> if isinstance ( inst , ( int , str ) ) : <EOL> inst = config . content_model_getter ( inst ) <EOL> text = getattr ( inst , config . SULCI_CONTENT_PROPERTY ) <EOL> descriptors = config . descriptors_getter ( inst ) <EOL> if not descriptors or not text : <EOL> sulci_logger . info ( u"<STR_LIT>" ) <EOL> return <EOL> validated_descriptors = set ( ) <EOL> for d in descriptors : <EOL> if not d : <EOL> continue <EOL> dsc , created = Descriptor . get_or_connect ( name = d ) <EOL> dsc . count . hincrby ( <NUM_LIT:1> ) <EOL> validated_descriptors . add ( dsc ) <EOL> if created : <EOL> sulci_logger . info ( u"<STR_LIT>" % unicode ( dsc ) , "<STR_LIT>" ) <EOL> try : <EOL> S = SemanticalTagger ( <EOL> text , <EOL> thesaurus = self . thesaurus , <EOL> pos_tagger = self . pos_tagger , <EOL> lexicon = self . pos_tagger . lexicon <EOL> ) <EOL> S . deduplicate_keyentities ( ) <EOL> except ValueError : <EOL> return <EOL> current_triggers = set ( ) <EOL> for ke in S . keyentities : <EOL> t , created = Trigger . get_or_connect ( original = unicode ( ke ) ) <EOL> current_triggers . add ( t ) <EOL> t . count . hincrby ( <NUM_LIT:1> ) <EOL> for d in validated_descriptors : <EOL> for t in current_triggers : <EOL> t . connect ( d , <NUM_LIT:1> ) <EOL> def clean_connections ( self ) : <EOL> """<STR_LIT>""" <EOL> TriggerToDescriptor . clean_all_connections ( ) <EOL> class GlobalPMITrainer ( ContentBaseTrainer ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , thesaurus , pos_tagger , mode = "<STR_LIT>" ) : <EOL> self . thesaurus = thesaurus <EOL> self . pos_tagger = pos_tagger <EOL> self . mode = mode <EOL> self . global_pmi = GlobalPMI . get ( corpus = config . DEFAULT_DATABASE ) <EOL> def train ( self , inst ) : <EOL> if isinstance ( inst , ( int , str ) ) : <EOL> inst = config . content_model_getter ( inst ) <EOL> text = getattr ( inst , config . SULCI_CONTENT_PROPERTY ) <EOL> try : <EOL> S = SemanticalTagger ( <EOL> text , <EOL> thesaurus = self . thesaurus , <EOL> pos_tagger = self . pos_tagger , <EOL> lexicon = self . pos_tagger . lexicon <EOL> ) <EOL> S . deduplicate_keyentities ( ) <EOL> except ValueError : <EOL> return <EOL> ngrams = S . ngrams ( min_length = <NUM_LIT:1> , max_length = <NUM_LIT:5> ) <EOL> for key , values in ngrams . iteritems ( ) : <EOL> self . global_pmi . add_ngram ( values [ '<STR_LIT>' ] , amount = values [ '<STR_LIT:count>' ] ) <EOL> class RuleTrainer ( ZMQTrainer ) : <EOL> """<STR_LIT>""" <EOL> def do ( self ) : <EOL> if self . mode == "<STR_LIT>" : <EOL> self . slave ( ) <EOL> elif self . mode == "<STR_LIT>" : <EOL> self . setup_socket_master ( ) <EOL> self . train ( ) <EOL> self . pubsocket . send ( "<STR_LIT>" ) <EOL> else : <EOL> self . train ( ) <EOL> def slave ( self ) : <EOL> self . setup_socket_slave ( ) <EOL> while True : <EOL> if self . subpoller . poll ( <NUM_LIT:0> ) : <EOL> rule = self . subsocket . recv ( ) [ <NUM_LIT:1> : ] <EOL> if rule == "<STR_LIT>" : <EOL> return <EOL> rule = rule . decode ( "<STR_LIT:utf-8>" ) <EOL> template = self . get_template_instance ( rule ) <EOL> sulci_logger . info ( u"<STR_LIT>" % rule , "<STR_LIT>" ) <EOL> template . apply_rule ( self . tokens , rule ) <EOL> if self . reppoller . poll ( <NUM_LIT:0> ) : <EOL> idx , action , rule = self . repsocket . recv_multipart ( ) <EOL> _ , good , bad = self . test_rule ( rule . decode ( "<STR_LIT:utf-8>" ) ) <EOL> self . repsocket . send_multipart ( [ idx , rule , str ( good ) , str ( bad ) ] ) <EOL> def test_rules ( self , rules_candidates ) : <EOL> pondered_rules = [ ] <EOL> if self . mode == "<STR_LIT>" : <EOL> for rule in rules_candidates : <EOL> self . reqsocket . send_multipart ( [ "<STR_LIT>" , rule . encode ( "<STR_LIT:utf-8>" ) ] ) <EOL> for rule in rules_candidates : <EOL> resp = self . reqsocket . recv_multipart ( ) <EOL> r , good , bad = resp <EOL> pondered_rules . append ( ( r . decode ( "<STR_LIT:utf-8>" ) , int ( good ) , int ( bad ) ) ) <EOL> sulci_logger . info ( u"<STR_LIT>" % r . decode ( "<STR_LIT:utf-8>" ) , "<STR_LIT>" ) <EOL> sulci_logger . info ( u"<STR_LIT>" ) <EOL> else : <EOL> for rule in rules_candidates : <EOL> pondered_rules . append ( self . test_rule ( rule ) ) <EOL> return pondered_rules <EOL> def pretrain ( self ) : <EOL> """<STR_LIT>""" <EOL> pass <EOL> def log_error ( self , token ) : <EOL> pass <EOL> def get_template_instance ( self , tpl ) : <EOL> pass <EOL> def select_one_rule ( self , rules ) : <EOL> pass <EOL> def train ( self ) : <EOL> """<STR_LIT>""" <EOL> sulci_logger . info ( "<STR_LIT>" , "<STR_LIT>" , True ) <EOL> final_rules = [ ] <EOL> errors = self . get_errors ( ) <EOL> while errors : <EOL> run_applied_rule = False <EOL> sulci_logger . info ( "<STR_LIT>" % len ( errors ) , "<STR_LIT>" , True ) <EOL> for token_with_error in errors [ : ] : <EOL> rules_candidates = [ ] <EOL> self . log_error ( token_with_error ) <EOL> for tpl , _ in self . template_generator . register . items ( ) : <EOL> template = self . get_template_instance ( tpl ) <EOL> rules_candidates += template . make_rules ( token_with_error ) <EOL> pondered_rules = self . test_rules ( rules_candidates ) <EOL> rule_candidate , score = self . select_one_rule ( pondered_rules ) <EOL> if rule_candidate and not rule_candidate in final_rules : <EOL> template = self . get_template_instance ( rule_candidate ) <EOL> final_rules . append ( ( rule_candidate , score ) ) <EOL> sulci_logger . info ( u"<STR_LIT>" % ( rule_candidate , score ) , "<STR_LIT>" ) <EOL> template . apply_rule ( self . tokens , rule_candidate ) <EOL> if self . mode == "<STR_LIT>" : <EOL> self . pubsocket . send ( "<STR_LIT>" % rule_candidate . encode ( "<STR_LIT:utf-8>" ) ) <EOL> run_applied_rule = True <EOL> errors = self . get_errors ( ) <EOL> break <EOL> else : <EOL> token_with_error . sample . set_trained_position ( token_with_error . position ) <EOL> if run_applied_rule : <EOL> continue <EOL> errors = None <EOL> self . display_errors ( ) <EOL> self . template_generator . export ( final_rules ) <EOL> def test_rule ( self , rule ) : <EOL> template = self . get_template_instance ( rule ) <EOL> bad = <NUM_LIT:0> <EOL> good = <NUM_LIT:0> <EOL> for ttk in self . tokens : <EOL> test = template . test_rule ( ttk , rule ) <EOL> if test == <NUM_LIT:1> : <EOL> good += <NUM_LIT:1> <EOL> elif test == - <NUM_LIT:1> : <EOL> bad += <NUM_LIT:1> <EOL> sulci_logger . info ( u"<STR_LIT>" % ( rule , good , bad ) , "<STR_LIT>" ) <EOL> return rule , good , bad <EOL> def display_errors ( self ) : <EOL> """<STR_LIT>""" <EOL> remaining_errors = self . get_errors ( ) <EOL> errors_count = len ( remaining_errors ) <EOL> total_words = len ( self . tokens ) <EOL> sulci_logger . info ( u"<STR_LIT>" % <EOL> ( errors_count , <NUM_LIT> * errors_count / total_words , total_words ) , "<STR_LIT>" ) <EOL> for r_error in remaining_errors : <EOL> self . log_error ( r_error ) <EOL> def get_errors ( self ) : <EOL> """<STR_LIT>""" <EOL> final = [ ] <EOL> for sample in self . samples : <EOL> final += sample . get_errors ( self . attr_name ) <EOL> return final <EOL> class LemmatizerTrainer ( RuleTrainer ) : <EOL> """<STR_LIT>""" <EOL> template_generator = LemmatizerTemplateGenerator <EOL> attr_name = "<STR_LIT>" <EOL> def __init__ ( self , lemmatizer , mode = "<STR_LIT>" ) : <EOL> self . lemmatizer = lemmatizer <EOL> self . mode = mode <EOL> self . tokens = self . lemmatizer . tokens <EOL> self . samples = self . lemmatizer . samples <EOL> self . pretrain ( ) <EOL> def pretrain ( self ) : <EOL> """<STR_LIT>""" <EOL> for token in self . tokens : <EOL> token . tag = token . verified_tag <EOL> def log_error ( self , token ) : <EOL> sulci_logger . info ( u"<STR_LIT>" % ( unicode ( token ) , token . lemme , token . verified_lemme ) , "<STR_LIT>" ) <EOL> def get_template_instance ( self , tpl ) : <EOL> template , _ = self . template_generator . get_instance ( tpl ) <EOL> return template <EOL> def select_one_rule ( self , rules ) : <EOL> """<STR_LIT>""" <EOL> return RuleTemplate . select_one ( rules , len ( self . tokens ) ) <EOL> class POSTrainer ( RuleTrainer ) : <EOL> """<STR_LIT>""" <EOL> attr_name = "<STR_LIT>" <EOL> def __init__ ( self , tagger , corpus , mode = "<STR_LIT>" ) : <EOL> self . tagger = tagger <EOL> self . corpus = corpus <EOL> self . pretrain ( ) <EOL> self . tokens = self . corpus . tokens <EOL> self . samples = self . corpus . samples <EOL> self . mode = mode <EOL> def log_error ( self , token ) : <EOL> sulci_logger . info ( u"<STR_LIT>" % ( unicode ( token ) , token . tag , token . verified_tag ) , "<STR_LIT>" ) <EOL> def get_template_instance ( self , tpl ) : <EOL> template , _ = self . template_generator . get_instance ( tpl , lexicon = self . tagger . lexicon ) <EOL> return template <EOL> def select_one_rule ( self , rules ) : <EOL> return RuleTemplate . select_one ( rules , len ( self . corpus ) , <NUM_LIT:3> ) <EOL> def pretrain ( self ) : <EOL> self . tagger . tag_all ( self . corpus . tokens ) <EOL> class LexicalTrainer ( POSTrainer ) : <EOL> template_generator = LexicalTemplateGenerator <EOL> def pretrain ( self ) : <EOL> """<STR_LIT>""" <EOL> self . tagger . tag_all ( self . corpus . tokens , lexical = False , contextual = False ) <EOL> def get_errors ( self ) : <EOL> """<STR_LIT>""" <EOL> final = [ ] <EOL> for sample in self . corpus . samples : <EOL> final += [ t for t in sample . get_errors ( ) if not t in self . tagger . lexicon ] <EOL> return final <EOL> class ContextualTrainer ( POSTrainer ) : <EOL> template_generator = ContextualTemplateGenerator <EOL> def pretrain ( self ) : <EOL> """<STR_LIT>""" <EOL> self . tagger . tag_all ( self . corpus . tokens , contextual = False ) </s>
