<s> """<STR_LIT>""" <EOL> import base <EOL> import constants as cons <EOL> import crane <EOL> from relation import MatchingRelation <EOL> from monk . utils . utils import binary2decimal , translate , monitor_accuracy <EOL> from monk . math . cmath import sign0 <EOL> import logging <EOL> import nltk <EOL> from nltk . stem import PorterStemmer <EOL> from nltk . corpus import stopwords <EOL> logger = logging . getLogger ( "<STR_LIT>" ) <EOL> metricLog = logging . getLogger ( "<STR_LIT>" ) <EOL> stopwords_english = set ( stopwords . words ( '<STR_LIT>' ) ) <EOL> symbols = { '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT:[>' , '<STR_LIT:]>' , '<STR_LIT:{>' , '<STR_LIT:}>' , '<STR_LIT:(>' , '<STR_LIT:)>' , '<STR_LIT:.>' , '<STR_LIT:$>' , '<STR_LIT:#>' } <EOL> class Turtle ( base . MONKObject ) : <EOL> FPANDAS = '<STR_LIT>' <EOL> FTIGRESS = '<STR_LIT>' <EOL> FMAPPING = '<STR_LIT>' <EOL> FDESCRIPTION = '<STR_LIT:description>' <EOL> FPENALTY = '<STR_LIT>' <EOL> FEPS = '<STR_LIT>' <EOL> FMAXPATHLENGTH = '<STR_LIT>' <EOL> FMAXINFERENCESTEPS = '<STR_LIT>' <EOL> FPARTIALBARRIER = '<STR_LIT>' <EOL> FMERGECLOCK = '<STR_LIT>' <EOL> FTRAINCLOCK = '<STR_LIT>' <EOL> FENTITYCOLLECTIONNAME = '<STR_LIT>' <EOL> FREQUIRES = '<STR_LIT>' <EOL> FREQUIRES_UIDS = '<STR_LIT>' <EOL> FREQUIRES_TURTLES = '<STR_LIT>' <EOL> FFOLLOWERS = '<STR_LIT>' <EOL> FLEADER = '<STR_LIT>' <EOL> store = crane . turtleStore <EOL> def __default__ ( self ) : <EOL> super ( Turtle , self ) . __default__ ( ) <EOL> self . pandas = [ ] <EOL> self . tigress = None <EOL> self . mapping = { } <EOL> self . description = cons . DEFAULT_EMPTY <EOL> self . pPenalty = <NUM_LIT:1.0> <EOL> self . pEPS = <NUM_LIT> <EOL> self . pMaxPathLength = <NUM_LIT:1> <EOL> self . pMaxInferenceSteps = <NUM_LIT:1000> <EOL> self . entityCollectionName = cons . DEFAULT_EMPTY <EOL> self . requires = dict ( ) <EOL> self . followers = [ ] <EOL> self . leader = None <EOL> self . mergeQueue = set ( ) <EOL> self . pPartialBarrier = <NUM_LIT:50> <EOL> self . pMergeClock = <NUM_LIT:0> <EOL> self . pTrainClock = <NUM_LIT:0> <EOL> def __restore__ ( self ) : <EOL> super ( Turtle , self ) . __restore__ ( ) <EOL> try : <EOL> [ panda . setdefault ( self . CREATOR , self . creator ) for panda in self . pandas ] <EOL> [ panda . setdefault ( self . NAME , self . name ) for panda in self . pandas ] <EOL> self . tigress . setdefault ( self . CREATOR , self . creator ) <EOL> self . tigress . setdefault ( self . NAME , self . name ) <EOL> except : <EOL> pass <EOL> self . pandas = crane . pandaStore . load_or_create_all ( self . pandas ) <EOL> self . tigress = crane . tigressStore . load_or_create ( self . tigress ) <EOL> self . pandaUids = set ( ( p . uid for p in self . pandas ) ) <EOL> self . invertedMapping = { tuple ( v ) : k for k , v in self . mapping . iteritems ( ) } <EOL> self . followers = set ( self . followers ) <EOL> if self . FREQUIRES_UIDS in self . requires : <EOL> uids = self . requires [ self . FREQUIRES_UIDS ] <EOL> if isinstance ( uids , basestring ) : <EOL> uids = eval ( uids ) <EOL> [ panda . add_features ( uids ) for panda in self . pandas ] <EOL> elif self . FREQUIRES_TURTLES in self . requires : <EOL> turtles = self . store . load_or_create_all ( <EOL> [ { '<STR_LIT:name>' : t , '<STR_LIT>' : self . creator } <EOL> for t in self . requires [ self . FREQUIRES_TURTLES ] ] ) <EOL> [ panda . add_features ( turtle . get_panda_uids ( ) ) for turtle in turtles for panda in self . pandas ] <EOL> elif self . requires : <EOL> logger . error ( '<STR_LIT>' . format ( <EOL> self . FREQUIRES_UIDS , <EOL> self . FREQUIRES_TURTLES , <EOL> self . requires ) ) <EOL> def generic ( self ) : <EOL> result = super ( Turtle , self ) . generic ( ) <EOL> if self . tigress : <EOL> result [ self . FTIGRESS ] = self . tigress . signature ( ) <EOL> result [ self . FPANDAS ] = [ panda . signature ( ) for panda in self . pandas ] <EOL> result [ self . FFOLLOWERS ] = list ( self . followers ) <EOL> del result [ '<STR_LIT>' ] <EOL> del result [ '<STR_LIT>' ] <EOL> del result [ '<STR_LIT>' ] <EOL> return result <EOL> def clone ( self , userName ) : <EOL> obj = super ( Turtle , self ) . clone ( userName ) <EOL> obj . pandaUids = set ( self . pandaUids ) <EOL> obj . tigress = self . tigress . clone ( userName ) <EOL> obj . pandas = [ p . clone ( userName ) for p in self . pandas ] <EOL> obj . requires = dict ( self . requires ) <EOL> return obj <EOL> def save ( self ) : <EOL> super ( Turtle , self ) . save ( ) <EOL> if self . tigress : <EOL> self . tigress . save ( ) <EOL> [ pa . save ( ) for pa in self . pandas ] <EOL> def delete ( self , deep = False ) : <EOL> result = super ( Turtle , self ) . delete ( ) <EOL> if self . tigress : <EOL> result = result and self . tigress . delete ( ) <EOL> if deep : <EOL> result = result and [ pa . delete ( ) for pa in self . pandas ] <EOL> return result <EOL> def add_follower ( self , follower ) : <EOL> if follower not in self . followers : <EOL> self . followers . add ( follower ) <EOL> self . store . push_one_in_fields ( self , { '<STR_LIT>' : follower } ) <EOL> [ pa . increment ( ) for pa in self . pandas ] <EOL> return True <EOL> else : <EOL> logger . info ( '<STR_LIT>' . format ( follower , self . creator ) ) <EOL> return False <EOL> def add_leader ( self , leader ) : <EOL> if self . leader != leader : <EOL> self . leader = leader <EOL> self . store . update_one_in_fields ( self , { '<STR_LIT>' : leader } ) <EOL> return True <EOL> else : <EOL> logger . info ( '<STR_LIT>' . format ( leader , self . creator ) ) <EOL> return False <EOL> def remove_leader ( self , leader ) : <EOL> if self . leader and leader == self . leader : <EOL> self . leader = None <EOL> self . store . update_one_in_fields ( self , { '<STR_LIT>' : None } ) <EOL> return True <EOL> else : <EOL> logger . info ( '<STR_LIT>' . format ( leader , self . name , self . creator ) ) <EOL> return False <EOL> def remove_follower ( self , follower ) : <EOL> if follower in self . followers : <EOL> self . followers . remove ( follower ) <EOL> self . store . pull_one_in_fields ( self , { '<STR_LIT>' : follower } ) <EOL> [ pa . decrease ( ) for pa in self . pandas ] <EOL> return True <EOL> else : <EOL> logger . info ( '<STR_LIT>' . format ( follower , self . name , self . crcreator ) ) <EOL> return False <EOL> def require_panda ( self , panda ) : <EOL> if self . has_panda ( panda ) : <EOL> logger . error ( '<STR_LIT>' . format ( panda . _id ) ) <EOL> return <EOL> [ pa . add_features ( panda . uid ) for pa in self . pandas ] <EOL> def require ( self , turtleName ) : <EOL> if self . name == turtleName : <EOL> logger . error ( '<STR_LIT>' . format ( turtleName ) ) <EOL> return <EOL> turtle = self . store . load_or_create ( { '<STR_LIT:name>' : turtleName , '<STR_LIT>' : self . creator } ) <EOL> [ panda . add_features ( turtle . get_panda_uids ( ) ) for panda in self . pandas ] <EOL> def get_panda_uids ( self ) : <EOL> return [ pa . uid for pa in self . pandas ] <EOL> def has_panda ( self , panda ) : <EOL> return panda . uid in self . pandaUids <EOL> def add_panda ( self , panda ) : <EOL> if not self . has_panda ( panda ) : <EOL> self . pandas . append ( panda ) <EOL> self . pandaUids . add ( panda . uid ) <EOL> crane . pandaStore . push_one_in_fields ( self , { '<STR_LIT>' : panda . _id } ) <EOL> return True <EOL> else : <EOL> logger . info ( '<STR_LIT>' . format ( panda . name , self . name ) ) <EOL> return False <EOL> def delete_panda ( self , panda ) : <EOL> if self . has_panda ( panda ) : <EOL> self . pandas . remove ( panda ) <EOL> self . pandaUids . remove ( panda . uid ) <EOL> crane . pandaStore . pull_one_in_fields ( self , { '<STR_LIT>' : panda . _id } ) <EOL> return True <EOL> else : <EOL> logger . info ( '<STR_LIT>' . format ( panda . name , self . name ) ) <EOL> return False <EOL> def predict ( self , entity , fields = None ) : <EOL> scores = [ panda . predict ( entity ) for panda in self . pandas ] <EOL> predicted = self . invertedMapping [ tuple ( map ( sign0 , scores ) ) ] <EOL> self . tigress . measure ( entity , predicted ) <EOL> return predicted <EOL> def test_data ( self , entity ) : <EOL> test = [ panda . predict ( entity ) for panda in self . pandas ] <EOL> logger . info ( '<STR_LIT>' . format ( self . creator , test [ <NUM_LIT:0> ] ) ) <EOL> return test <EOL> def add_data ( self , entity ) : <EOL> return self . tigress . supervise ( self , entity ) <EOL> def active_train ( self ) : <EOL> try : <EOL> self . tigress . supervise ( self ) <EOL> except Exception as e : <EOL> logger . info ( e . message ) <EOL> logger . info ( "<STR_LIT>" . format ( self . name ) ) <EOL> def train ( self ) : <EOL> [ panda . train ( self . leader ) for panda in self . pandas ] <EOL> self . pTrainClock += <NUM_LIT:1> <EOL> self . update_fields ( { self . FTRAINCLOCK : self . pTrainClock } ) <EOL> logger . debug ( '<STR_LIT>' . format ( self . pTrainClock ) ) <EOL> def checkout ( self ) : <EOL> [ panda . checkout ( self . leader ) for panda in self . pandas ] <EOL> def commit ( self ) : <EOL> [ panda . commit ( ) for panda in self . pandas ] <EOL> def merge ( self , follower ) : <EOL> if follower not in self . followers and follower != self . creator : <EOL> logger . error ( '<STR_LIT>' . format ( follower , self . creator , self . name ) ) <EOL> return False <EOL> self . mergeQueue . add ( follower ) <EOL> if len ( self . mergeQueue ) >= min ( len ( self . followers ) , self . pPartialBarrier ) : <EOL> for follower in self . mergeQueue : <EOL> [ panda . merge ( follower ) for panda in self . pandas ] <EOL> self . mergeQueue . clear ( ) <EOL> [ panda . update_fields ( { panda . FCONSENSUS : panda . z . generic ( ) } ) for panda in self . pandas ] <EOL> self . pMergeClock += <NUM_LIT:1> <EOL> self . update_fields ( { self . FMERGECLOCK : self . pMergeClock } ) <EOL> logger . debug ( '<STR_LIT>' . format ( self . pMergeClock ) ) <EOL> return True <EOL> return False <EOL> def reset ( self ) : <EOL> [ panda . reset ( ) for panda in self . pandas ] <EOL> self . pTrainClock = <NUM_LIT:0> <EOL> self . pMergeClock = <NUM_LIT:0> <EOL> self . update_fields ( { self . FTRAINCLOCK : self . pTrainClock , self . FMERGECLOCK : self . pMergeClock } ) <EOL> def reset_data ( self ) : <EOL> [ panda . reset_data ( ) for panda in self . pandas ] <EOL> def set_mantis_parameter ( self , para , value ) : <EOL> [ panda . set_mantis_parameter ( para , value ) for panda in self . pandas ] <EOL> class SingleTurtle ( Turtle ) : <EOL> def predict ( self , entity , fields = None ) : <EOL> panda = self . pandas [ <NUM_LIT:0> ] <EOL> score = panda . predict ( entity ) <EOL> if sign0 ( score ) > <NUM_LIT:0> : <EOL> self . tigress . measure ( entity , panda . name ) <EOL> monitor_accuracy ( panda . name , self . creator , score , '<STR_LIT:True>' ) <EOL> return panda . name <EOL> else : <EOL> self . tigress . measure ( entity , cons . DEFAULT_NONE ) <EOL> monitor_accuracy ( panda . name , self . creator , score , '<STR_LIT:False>' ) <EOL> return cons . DEFAULT_NONE <EOL> class MultiLabelTurtle ( Turtle ) : <EOL> def predict ( self , entity , fields = None ) : <EOL> predicted = [ panda . name for panda in self . pandas if panda . predict ( entity ) > <NUM_LIT:0> ] <EOL> self . tigress . measure ( entity , predicted ) <EOL> return predicted <EOL> class RankingTurtle ( Turtle ) : <EOL> FTARGET_CONNECTION_STRING = '<STR_LIT>' <EOL> FTARGET_DATABASE_NAME = '<STR_LIT>' <EOL> FTARGET_COLLECTION_NAME = '<STR_LIT>' <EOL> FTARGET_NUM_LEVELS = '<STR_LIT>' <EOL> FBEAM_SIZE = '<STR_LIT>' <EOL> FWINDOW_SIZE = '<STR_LIT>' <EOL> FQUERY = '<STR_LIT>' <EOL> FTARGET_STORE = '<STR_LIT>' <EOL> RAW_TARGETID = '<STR_LIT>' <EOL> RAW_RELEVANCE = '<STR_LIT>' <EOL> def __default__ ( self ) : <EOL> super ( RankingTurtle , self ) . __default__ ( ) <EOL> self . targetConnectionString = cons . DEFAULT_EMPTY <EOL> self . targetDatabaseName = cons . DEFAULT_EMPTY <EOL> self . targetCollectionName = cons . DEFAULT_EMPTY <EOL> self . targetStore = crane . Crane ( ) <EOL> self . numLevels = <NUM_LIT:1> <EOL> self . beamSize = <NUM_LIT:10> <EOL> self . windowSize = <NUM_LIT:2> * self . beamSize <EOL> self . queryFunc = cons . DEFAULT_FUNC <EOL> def __restore__ ( self ) : <EOL> super ( RankingTurtle , self ) . __restore__ ( ) <EOL> self . targetStore = crane . Crane ( self . targetConnectionString , <EOL> self . targetDatabaseName , <EOL> self . targetCollectionName ) <EOL> def generic ( self ) : <EOL> result = super ( RankingTurtle , self ) . generic ( ) <EOL> del result [ self . FTARGET_STORE ] <EOL> return result <EOL> def clone ( self , userName ) : <EOL> obj = super ( RankingTurtle , self ) . clone ( userName ) <EOL> return obj <EOL> def predict ( self , entity , fields = None ) : <EOL> query = eval ( self . queryFunc ) ( entity ) <EOL> targetIds = self . targetStore . load_all_in_ids ( query , <NUM_LIT:0> , self . windowSize ) <EOL> targets = self . targetStore . load_all_by_ids ( targetIds ) <EOL> relation = MatchingRelation ( ) <EOL> relation . set_argument ( <NUM_LIT:0> , entity ) <EOL> results = [ ] <EOL> for target in targets : <EOL> relation . set_argument ( <NUM_LIT:1> , target ) <EOL> relation . compute ( ) <EOL> rank = self . invertedMapping [ [ sign0 ( panda . predict ( relation ) ) for panda in self . pandas ] ] <EOL> results . append ( ( rank , target ) ) <EOL> results . sort ( reverse = True ) <EOL> return results [ : self . beamSize ] <EOL> def add_data ( self , entity ) : <EOL> targetId = entity . get_raw ( self . RAW_TARGETID , None ) <EOL> relevance = entity . get_raw ( self . RAW_RELEVANCE , <NUM_LIT:0> ) <EOL> if targetId : <EOL> target = self . targetStore . load_one_by_id ( targetId ) <EOL> relation = MatchingRelation ( ) <EOL> relation . set_argument ( <NUM_LIT:0> , entity ) <EOL> relation . set_argument ( <NUM_LIT:1> , target ) <EOL> relation . compute ( ) <EOL> relation . set_raw ( self . RAW_RELEVANCE , relevance ) <EOL> self . tigress . supervise ( self , relation ) <EOL> class SPNTurtle ( Turtle ) : <EOL> pass <EOL> class DictionaryTurtle ( Turtle ) : <EOL> def __default__ ( self ) : <EOL> super ( DictionaryTurtle , self ) . __default__ ( ) <EOL> self . dictionary = dict ( ) <EOL> def __restore__ ( self ) : <EOL> super ( DictionaryTurtle , self ) . __restore__ ( ) <EOL> self . dictionary = { p . name : p . uid for p in self . pandas } <EOL> def generic ( self ) : <EOL> result = super ( DictionaryTurtle , self ) . generic ( ) <EOL> try : <EOL> del result [ '<STR_LIT>' ] <EOL> del result [ '<STR_LIT>' ] <EOL> except : <EOL> pass <EOL> return result <EOL> def clone ( self , userName ) : <EOL> obj = super ( DictionaryTurtle , self ) . clone ( userName ) <EOL> obj . dictionary = dict ( self . dictionary ) <EOL> return obj <EOL> def _process ( self , field ) : <EOL> return { } <EOL> def _get_or_new_panda ( self , name ) : <EOL> if name not in self . dictionary : <EOL> panda = { self . MONK_TYPE : '<STR_LIT>' , <EOL> self . NAME : name , <EOL> self . CREATOR : self . creator } <EOL> panda = crane . pandaStore . load_or_create ( panda , tosave = True ) <EOL> self . add_panda ( panda ) <EOL> self . dictionary [ name ] = panda . uid <EOL> uid = panda . uid <EOL> else : <EOL> uid = self . dictionary [ name ] <EOL> return ( uid , <NUM_LIT:1.0> ) <EOL> def is_stop ( self , w ) : <EOL> if w in stopwords_english : <EOL> return True <EOL> else : <EOL> return False <EOL> def is_symbol ( self , w ) : <EOL> if w in symbols : <EOL> return True <EOL> elif w . find ( '<STR_LIT:.>' ) >= <NUM_LIT:0> : <EOL> return True <EOL> else : <EOL> return False <EOL> def is_single ( self , w ) : <EOL> if len ( w ) <= <NUM_LIT:1> : <EOL> return True <EOL> else : <EOL> return False <EOL> def predict ( self , entity , fields ) : <EOL> total = <NUM_LIT:0> <EOL> for field in fields : <EOL> value = translate ( getattr ( entity , field , '<STR_LIT>' ) ) <EOL> if not value : <EOL> continue <EOL> allTokens = self . _process ( value ) <EOL> entity . _raws . update ( allTokens ) <EOL> entity . _features . update ( [ self . _get_or_new_panda ( t ) for t in allTokens ] ) <EOL> total += len ( allTokens ) <EOL> return total <EOL> def merge ( self ) : <EOL> pass <EOL> class UniGramTurtle ( DictionaryTurtle ) : <EOL> def _process ( self , field ) : <EOL> allTokens = { } <EOL> sents = nltk . tokenize . sent_tokenize ( field ) <EOL> for sent in sents : <EOL> tokens = nltk . tokenize . word_tokenize ( sent . lower ( ) ) <EOL> allTokens . update ( ( ( t , <NUM_LIT:1> ) for t in tokens if not self . is_stop ( t ) and <EOL> not self . is_symbol ( t ) and <EOL> not self . is_single ( t ) ) ) <EOL> return allTokens <EOL> class POSTurtle ( DictionaryTurtle ) : <EOL> def _process ( self , field ) : <EOL> allTokens = { } <EOL> sents = nltk . tokenize . sent_tokenize ( field ) <EOL> for sent in sents : <EOL> tokens = nltk . tokenize . word_tokenize ( sent . lower ( ) ) <EOL> tagged = nltk . pos_tag ( [ t for t in tokens if not self . is_stop ( t ) and <EOL> not self . is_symbol ( t ) and <EOL> not self . is_single ( t ) ] ) <EOL> allTokens . update ( ( ( '<STR_LIT:_>' . join ( t ) , <NUM_LIT:1> ) for t in tagged ) ) <EOL> return allTokens <EOL> class StemTurtle ( DictionaryTurtle ) : <EOL> def _process ( self , field ) : <EOL> allTokens = { } <EOL> sents = nltk . tokenize . sent_tokenize ( field ) <EOL> logger . debug ( sents ) <EOL> port = PorterStemmer ( ) <EOL> for sent in sents : <EOL> tokens = nltk . tokenize . word_tokenize ( sent . lower ( ) ) <EOL> stems = [ port . stem ( t ) for t in tokens if not self . is_stop ( t ) and not self . is_symbol ( t ) ] <EOL> allTokens . update ( ( ( t , <NUM_LIT:1> ) for t in stems ) ) <EOL> logger . debug ( '<STR_LIT:U+0020>' . join ( allTokens ) ) <EOL> return allTokens <EOL> base . register ( Turtle ) <EOL> base . register ( SingleTurtle ) <EOL> base . register ( MultiLabelTurtle ) <EOL> base . register ( RankingTurtle ) <EOL> base . register ( DictionaryTurtle ) <EOL> base . register ( SPNTurtle ) <EOL> base . register ( UniGramTurtle ) <EOL> base . register ( POSTurtle ) <EOL> base . register ( StemTurtle ) </s>
