<s> import re <EOL> import logging <EOL> from collections import defaultdict <EOL> from aleph . core import db <EOL> from aleph . util import normalize_strong <EOL> from aleph . model import Reference , Entity , Collection <EOL> from aleph . analyze . analyzer import Analyzer <EOL> log = logging . getLogger ( __name__ ) <EOL> BATCH_SIZE = <NUM_LIT> <EOL> class EntityCache ( object ) : <EOL> def __init__ ( self ) : <EOL> self . collections = { } <EOL> def compile_collection ( self , collection_id ) : <EOL> matchers = defaultdict ( set ) <EOL> q = db . session . query ( Entity ) <EOL> q = q . filter ( Entity . collection_id == collection_id ) <EOL> q = q . filter ( Entity . deleted_at == None ) <EOL> for entity in q : <EOL> for term in entity . terms : <EOL> matchers [ normalize_strong ( term ) ] . add ( entity . id ) <EOL> body = '<STR_LIT:|>' . join ( matchers . keys ( ) ) <EOL> rex = re . compile ( '<STR_LIT>' % body ) <EOL> return rex , matchers <EOL> def matchers ( self ) : <EOL> timestamps = Collection . timestamps ( ) <EOL> for ts in self . collections . keys ( ) : <EOL> if ts not in timestamps : <EOL> self . collections . pop ( ts , None ) <EOL> for ts in timestamps : <EOL> if ts not in self . collections : <EOL> log . info ( '<STR_LIT>' , ts ) <EOL> self . collections [ ts ] = self . compile_collection ( ts [ <NUM_LIT:0> ] ) <EOL> return self . collections . values ( ) <EOL> class EntityAnalyzer ( Analyzer ) : <EOL> cache = EntityCache ( ) <EOL> @ property <EOL> def matchers ( self ) : <EOL> if not hasattr ( self , '<STR_LIT>' ) : <EOL> self . _matchers = self . cache . matchers ( ) <EOL> return self . _matchers <EOL> def analyze ( self , document , meta ) : <EOL> entities = defaultdict ( int ) <EOL> for text , rec in document . text_parts ( ) : <EOL> text = normalize_strong ( text ) <EOL> if text is None or not len ( text ) : <EOL> continue <EOL> for rex , matches in self . matchers : <EOL> for match in rex . finditer ( text ) : <EOL> match = match . group ( <NUM_LIT:2> ) <EOL> for entity_id in matches . get ( match , [ ] ) : <EOL> entities [ entity_id ] += <NUM_LIT:1> <EOL> if len ( entities ) : <EOL> log . info ( "<STR_LIT>" , document , len ( entities ) ) <EOL> Reference . delete_document ( document . id ) <EOL> for entity_id , weight in entities . items ( ) : <EOL> ref = Reference ( ) <EOL> ref . document_id = document . id <EOL> ref . entity_id = entity_id <EOL> ref . weight = weight <EOL> db . session . add ( ref ) <EOL> self . save ( document , meta ) </s>
