<s> import codecs <EOL> from iepy . preprocess . ner . base import BaseNERRunner <EOL> class LiteralNER ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , labels , src_filenames ) : <EOL> """<STR_LIT>""" <EOL> assert len ( labels ) == len ( src_filenames ) <EOL> self . labels = labels <EOL> self . src_filenames = src_filenames <EOL> names = set ( ) <EOL> names_map = { } <EOL> for label , filename in zip ( labels , src_filenames ) : <EOL> f = codecs . open ( filename , encoding = "<STR_LIT:utf8>" ) <EOL> namelist = f . read ( ) . strip ( ) . split ( '<STR_LIT:\n>' ) <EOL> names . update ( namelist ) <EOL> for name in namelist : <EOL> names_map [ name ] = label <EOL> self . names = frozenset ( names ) <EOL> self . names_map = names_map <EOL> prefixes = set ( ) <EOL> for name in self . names : <EOL> sname = name . split ( ) <EOL> prefixes . update ( [ '<STR_LIT:U+0020>' . join ( sname [ : i ] ) for i in range ( <NUM_LIT:1> , len ( sname ) + <NUM_LIT:1> ) ] ) <EOL> self . prefixes = frozenset ( prefixes ) <EOL> def tag ( self , sent ) : <EOL> """<STR_LIT>""" <EOL> entities = self . entities ( sent ) <EOL> entities . append ( ( ( len ( sent ) , len ( sent ) ) , '<STR_LIT:X>' ) ) <EOL> next_entity = entities . pop ( <NUM_LIT:0> ) <EOL> result = [ ] <EOL> for i , t in enumerate ( sent ) : <EOL> if i >= next_entity [ <NUM_LIT:0> ] [ <NUM_LIT:1> ] : <EOL> next_entity = entities . pop ( <NUM_LIT:0> ) <EOL> if i < next_entity [ <NUM_LIT:0> ] [ <NUM_LIT:0> ] : <EOL> result . append ( ( t , '<STR_LIT:O>' ) ) <EOL> elif i < next_entity [ <NUM_LIT:0> ] [ <NUM_LIT:1> ] : <EOL> result . append ( ( t , next_entity [ <NUM_LIT:1> ] ) ) <EOL> return result <EOL> def entities ( self , sent ) : <EOL> """<STR_LIT>""" <EOL> result = [ ] <EOL> i = <NUM_LIT:0> <EOL> while i < len ( sent ) : <EOL> j = i + <NUM_LIT:1> <EOL> prev_segment = segment = '<STR_LIT:U+0020>' . join ( sent [ i : j ] ) <EOL> while segment in self . prefixes and j <= len ( sent ) : <EOL> j += <NUM_LIT:1> <EOL> prev_segment = segment <EOL> segment = '<STR_LIT:U+0020>' . join ( sent [ i : j ] ) <EOL> if prev_segment in self . names : <EOL> label = self . names_map [ prev_segment ] <EOL> result . append ( ( ( i , j - <NUM_LIT:1> ) , label ) ) <EOL> i = j - <NUM_LIT:1> <EOL> else : <EOL> i += <NUM_LIT:1> <EOL> return result <EOL> class LiteralNERRunner ( BaseNERRunner ) : <EOL> def __init__ ( self , labels , src_filenames , override = False ) : <EOL> super ( LiteralNERRunner , self ) . __init__ ( override = override ) <EOL> self . lit_tagger = LiteralNER ( labels , src_filenames ) <EOL> def run_ner ( self , doc ) : <EOL> entities = [ ] <EOL> sent_offset = <NUM_LIT:0> <EOL> for sent in doc . get_sentences ( ) : <EOL> sent_entities = self . lit_tagger . entities ( sent ) <EOL> for ( ( i , j ) , label ) in sent_entities : <EOL> name = '<STR_LIT:U+0020>' . join ( sent [ i : j ] ) <EOL> kind = label . lower ( ) <EOL> entities . append ( <EOL> self . build_occurrence ( <EOL> key = name , <EOL> kind_name = kind , <EOL> alias = name , <EOL> offset = sent_offset + i , <EOL> offset_end = sent_offset + j ) <EOL> ) <EOL> sent_offset += len ( sent ) <EOL> return entities <EOL> def to_lower_normalizer ( name ) : <EOL> """<STR_LIT>""" <EOL> words = name . split ( ) <EOL> result = [ ] <EOL> for w in words : <EOL> if not w . isupper ( ) : <EOL> w = w . lower ( ) <EOL> result . append ( w ) <EOL> return '<STR_LIT:U+0020>' . join ( result ) </s>
