<s> """<STR_LIT>""" <EOL> import os . path <EOL> from nltk2 . parse . graf . PyGraph import * <EOL> from nltk2 . parse . graf . PyGraphParser import * <EOL> from util import * <EOL> from api import * <EOL> class MascCorpusReader ( CorpusReader ) : <EOL> """<STR_LIT>""" <EOL> CorpusView = StreamBackedCorpusView <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , root , fileids , encoding ) : <EOL> """<STR_LIT>""" <EOL> self . _cur_file = "<STR_LIT>" <EOL> self . _cur_sents_file = "<STR_LIT>" <EOL> self . _cur_paras_file = "<STR_LIT>" <EOL> self . _cur_offsets = [ ] <EOL> self . _cur_sents_offsets = [ ] <EOL> self . _cur_paras_offsets = [ ] <EOL> self . _char_to_byte = { } <EOL> self . _byte_to_char = { } <EOL> self . _file_end = <NUM_LIT:0> <EOL> CorpusReader . __init__ ( self , root , fileids , encoding ) <EOL> def raw ( self , fileids = None ) : <EOL> """<STR_LIT>""" <EOL> if fileids is None : fileids = self . _fileids <EOL> elif isinstance ( fileids , basestring ) : fileids = [ fileids ] <EOL> return concat ( [ self . open ( f ) . read ( ) for f in fileids ] ) <EOL> def words ( self , fileids = None ) : <EOL> """<STR_LIT>""" <EOL> return concat ( [ self . CorpusView ( fileid , self . _read_word_block , <EOL> encoding = '<STR_LIT:utf-8>' ) <EOL> for fileid in self . abspaths ( fileids ) ] ) <EOL> def sents ( self , fileids = None ) : <EOL> """<STR_LIT>""" <EOL> return concat ( [ self . CorpusView ( fileid , self . _read_sent_block , <EOL> encoding = '<STR_LIT:utf-8>' ) <EOL> for fileid in self . abspaths ( fileids ) ] ) <EOL> def paras ( self , fileids = None ) : <EOL> """<STR_LIT>""" <EOL> return concat ( [ self . CorpusView ( fileid , self . _read_para_block , <EOL> encoding = '<STR_LIT:utf-8>' ) <EOL> for fileid in self . abspaths ( fileids ) ] ) <EOL> def nouns ( self , fileids = None ) : <EOL> """<STR_LIT>""" <EOL> return concat ( [ self . CorpusView ( fileid , self . _read_noun_block , <EOL> encoding = '<STR_LIT:utf-8>' ) <EOL> for fileid in self . abspaths ( fileids ) ] ) <EOL> def verbs ( self , fileids = None ) : <EOL> """<STR_LIT>""" <EOL> return concat ( [ self . CorpusView ( fileid , self . _read_verb_block , <EOL> encoding = '<STR_LIT:utf-8>' ) <EOL> for fileid in self . abspaths ( fileids ) ] ) <EOL> def persons ( self , fileids = None ) : <EOL> """<STR_LIT>""" <EOL> return concat ( [ self . CorpusView ( fileid , self . _read_person_block , <EOL> encoding = '<STR_LIT:utf-8>' ) <EOL> for fileid in self . abspaths ( fileids ) ] ) <EOL> def _get_basename ( self , file ) : <EOL> """<STR_LIT>""" <EOL> return file [ <NUM_LIT:0> : len ( file ) - <NUM_LIT:4> ] <EOL> def _get_disc ( self , stream ) : <EOL> """<STR_LIT>""" <EOL> self . _char_to_byte = { } <EOL> self . _byte_to_char = { } <EOL> stream . read ( ) <EOL> file_end = stream . tell ( ) <EOL> self . _file_end = file_end <EOL> stream . seek ( <NUM_LIT:0> ) <EOL> for i in range ( file_end + <NUM_LIT:1> ) : <EOL> if i != stream . tell ( ) : <EOL> self . _char_to_byte [ i ] = stream . tell ( ) <EOL> self . _byte_to_char [ stream . tell ( ) ] = i <EOL> stream . read ( <NUM_LIT:1> ) <EOL> stream . seek ( <NUM_LIT:0> ) <EOL> def _get_subset ( self , offsets , offsets_start , offsets_end ) : <EOL> """<STR_LIT>""" <EOL> subset = [ ] <EOL> for i in offsets : <EOL> if ( i [ <NUM_LIT:0> ] >= offsets_start and i [ <NUM_LIT:1> ] <= offsets_end and <EOL> i [ <NUM_LIT:0> ] != i [ <NUM_LIT:1> ] ) : <EOL> subset . append ( i ) <EOL> elif ( i [ <NUM_LIT:0> ] >= offsets_start and i [ <NUM_LIT:1> ] > offsets_end and <EOL> i [ <NUM_LIT:0> ] != i [ <NUM_LIT:1> ] and i [ <NUM_LIT:0> ] <= offsets_end ) : <EOL> subset . append ( i ) <EOL> return subset <EOL> def _get_read_size ( self , subset , char_to_byte , slimit , offset_end ) : <EOL> """<STR_LIT>""" <EOL> if len ( subset ) != <NUM_LIT:0> : <EOL> last1 = subset [ len ( subset ) - <NUM_LIT:1> ] <EOL> last = last1 [ <NUM_LIT:1> ] <EOL> last = char_to_byte . get ( last , last ) <EOL> read_size = last - slimit <EOL> else : <EOL> elimit = char_to_byte . get ( offset_end , offset_end ) <EOL> read_size = elimit - slimit <EOL> return read_size <EOL> def _get_block ( self , subset , text , offsets_start ) : <EOL> """<STR_LIT>""" <EOL> block = [ ] <EOL> for s in subset : <EOL> start = s [ <NUM_LIT:0> ] - offsets_start <EOL> end = s [ <NUM_LIT:1> ] - offsets_start <EOL> chunk = text [ start : end ] . encode ( '<STR_LIT:utf-8>' ) <EOL> chunk = self . _remove_ws ( chunk ) <EOL> block . append ( chunk ) <EOL> return block <EOL> def _read_block ( self , stream , file_ending , label ) : <EOL> """<STR_LIT>""" <EOL> file = self . _get_basename ( stream . name ) + file_ending <EOL> if file != self . _cur_file : <EOL> self . _cur_file = file <EOL> offsets = self . _get_annotation ( file , label ) <EOL> self . _cur_offsets = offsets <EOL> self . _get_disc ( stream ) <EOL> char_to_byte = self . _char_to_byte <EOL> byte_to_char = self . _byte_to_char <EOL> else : <EOL> offsets = self . _cur_offsets <EOL> char_to_byte = self . _char_to_byte <EOL> byte_to_char = self . _byte_to_char <EOL> slimit = stream . tell ( ) <EOL> offset_slimit = byte_to_char . get ( slimit , slimit ) <EOL> offset_elimit = offset_slimit + <NUM_LIT> <EOL> subset = self . _get_subset ( offsets , offset_slimit , offset_elimit ) <EOL> read_size = ( self . _get_read_size ( subset , char_to_byte , slimit , <EOL> offset_elimit ) ) <EOL> text = stream . read ( read_size ) <EOL> block = self . _get_block ( subset , text , offset_slimit ) <EOL> return block <EOL> def _read_person_block ( self , stream ) : <EOL> """<STR_LIT>""" <EOL> return self . _read_block ( stream , '<STR_LIT>' , '<STR_LIT>' ) <EOL> def _read_verb_block ( self , stream ) : <EOL> """<STR_LIT>""" <EOL> return self . _read_block ( stream , '<STR_LIT>' , '<STR_LIT>' ) <EOL> def _read_noun_block ( self , stream ) : <EOL> """<STR_LIT>""" <EOL> return self . _read_block ( stream , '<STR_LIT>' , '<STR_LIT>' ) <EOL> def _read_word_block ( self , stream ) : <EOL> """<STR_LIT>""" <EOL> return self . _read_block ( stream , '<STR_LIT>' , '<STR_LIT>' ) <EOL> def _read_sent_block ( self , stream ) : <EOL> """<STR_LIT>""" <EOL> file = self . _get_basename ( stream . name ) + '<STR_LIT>' <EOL> words_file = self . _get_basename ( stream . name ) + '<STR_LIT>' <EOL> if not file == self . _cur_sents_file : <EOL> self . _cur_sents_file = file <EOL> self . _cur_words_file = words_file <EOL> offsets = self . _get_annotation ( file , '<STR_LIT:s>' ) <EOL> words_offsets = self . _get_annotation ( words_file , '<STR_LIT>' ) <EOL> self . _cur_sents_offsets = offsets <EOL> self . _cur_words_offsets = words_offsets <EOL> self . _get_disc ( stream ) <EOL> char_to_byte = self . _char_to_byte <EOL> byte_to_char = self . _byte_to_char <EOL> else : <EOL> offsets = self . _cur_sents_offsets <EOL> words_offsets = self . _cur_words_offsets <EOL> char_to_byte = self . _char_to_byte <EOL> byte_to_char = self . _byte_to_char <EOL> slimit = stream . tell ( ) <EOL> offset_slimit = byte_to_char . get ( slimit , slimit ) <EOL> offset_elimit = offset_slimit + <NUM_LIT> <EOL> subset = self . _get_subset ( offsets , offset_slimit , offset_elimit ) <EOL> read_size = self . _get_read_size ( subset , char_to_byte , <EOL> slimit , offset_elimit ) <EOL> text = stream . read ( read_size ) <EOL> block = [ ] <EOL> for s in subset : <EOL> sent = [ ] <EOL> for w in words_offsets : <EOL> if w [ <NUM_LIT:0> ] >= s [ <NUM_LIT:0> ] and w [ <NUM_LIT:1> ] <= s [ <NUM_LIT:1> ] and w [ <NUM_LIT:0> ] != w [ <NUM_LIT:1> ] : <EOL> start = w [ <NUM_LIT:0> ] - offset_slimit <EOL> end = w [ <NUM_LIT:1> ] - offset_slimit <EOL> chunk = text [ start : end ] . encode ( '<STR_LIT:utf-8>' ) <EOL> chunk = self . _remove_ws ( chunk ) <EOL> sent . append ( chunk ) <EOL> block . append ( sent ) <EOL> return block <EOL> def _read_para_block ( self , stream ) : <EOL> """<STR_LIT>""" <EOL> file = self . _get_basename ( stream . name ) + '<STR_LIT>' <EOL> sents_file = self . _get_basename ( stream . name ) + '<STR_LIT>' <EOL> words_file = self . _get_basename ( stream . name ) + '<STR_LIT>' <EOL> if not file == self . _cur_paras_file : <EOL> self . _cur_paras_file = file <EOL> self . _cur_sents_file = sents_file <EOL> self . _cur_words_file = words_file <EOL> offsets = self . _get_annotation ( file , '<STR_LIT:p>' ) <EOL> sents_offsets = self . _get_annotation ( sents_file , '<STR_LIT:s>' ) <EOL> words_offsets = self . _get_annotation ( words_file , '<STR_LIT>' ) <EOL> self . _cur_paras_offsets = offsets <EOL> self . _cur_sents_offsets = sents_offsets <EOL> self . _cur_words_offsets = words_offsets <EOL> self . _get_disc ( stream ) <EOL> char_to_byte = self . _char_to_byte <EOL> byte_to_char = self . _byte_to_char <EOL> else : <EOL> offsets = self . _cur_paras_offsets <EOL> sents_offsets = self . _cur_sents_offsets <EOL> words_offsets = self . _cur_words_offsets <EOL> char_to_byte = self . _char_to_byte <EOL> byte_to_char = self . _byte_to_char <EOL> slimit = stream . tell ( ) <EOL> offset_slimit = byte_to_char . get ( slimit , slimit ) <EOL> offset_elimit = offset_slimit + <NUM_LIT> <EOL> subset = [ ] <EOL> for i in offsets : <EOL> if ( i [ <NUM_LIT:0> ] >= offset_slimit and i [ <NUM_LIT:1> ] <= offset_elimit <EOL> and i [ <NUM_LIT:0> ] != i [ <NUM_LIT:1> ] ) : <EOL> subset . append ( i ) <EOL> if ( i [ <NUM_LIT:0> ] >= offset_slimit and i [ <NUM_LIT:1> ] > offset_elimit <EOL> and i [ <NUM_LIT:0> ] != i [ <NUM_LIT:1> ] ) : <EOL> subset . append ( i ) <EOL> break <EOL> if len ( subset ) != <NUM_LIT:0> : <EOL> last1 = subset [ len ( subset ) - <NUM_LIT:1> ] <EOL> last = last1 [ <NUM_LIT:1> ] <EOL> last = char_to_byte . get ( last , last ) <EOL> read_size = last - slimit <EOL> text = stream . read ( read_size ) <EOL> else : <EOL> if offset_elimit < self . _file_end : <EOL> elimit = char_to_byte . get ( offset_elimit , offset_elimit ) <EOL> read_size = elimit - slimit <EOL> text = stream . read ( read_size ) <EOL> else : <EOL> stream . read ( ) <EOL> block = [ ] <EOL> for p in subset : <EOL> para = [ ] <EOL> for s in sents_offsets : <EOL> if s [ <NUM_LIT:0> ] >= p [ <NUM_LIT:0> ] and s [ <NUM_LIT:1> ] <= p [ <NUM_LIT:1> ] and s [ <NUM_LIT:0> ] != s [ <NUM_LIT:1> ] : <EOL> sent = [ ] <EOL> for w in words_offsets : <EOL> if w [ <NUM_LIT:0> ] >= s [ <NUM_LIT:0> ] and w [ <NUM_LIT:1> ] <= s [ <NUM_LIT:1> ] and w [ <NUM_LIT:0> ] != w [ <NUM_LIT:1> ] : <EOL> start = w [ <NUM_LIT:0> ] - offset_slimit <EOL> end = w [ <NUM_LIT:1> ] - offset_slimit <EOL> chunk = text [ start : end ] . encode ( '<STR_LIT:utf-8>' ) <EOL> chunk = self . _remove_ws ( chunk ) <EOL> sent . append ( chunk ) <EOL> para . append ( sent ) <EOL> if len ( para ) != <NUM_LIT:0> : <EOL> block . append ( para ) <EOL> return block <EOL> def _get_annotation ( self , annfile , label ) : <EOL> """<STR_LIT>""" <EOL> parser = PyGraphParser ( ) <EOL> g = parser . parse ( annfile ) <EOL> node_list = g . nodes ( ) <EOL> offsets = [ ] <EOL> for node in node_list : <EOL> pair = self . _add_annotations ( node , label ) <EOL> offsets . extend ( pair ) <EOL> offsets . sort ( ) <EOL> return offsets <EOL> def _add_annotations ( self , node , label ) : <EOL> """<STR_LIT>""" <EOL> node_offsets = [ ] <EOL> for a in node . _annotations : <EOL> if a . _label == label : <EOL> pair = self . _get_offsets ( node ) <EOL> if pair is not None : <EOL> pair . sort ( ) <EOL> node_offsets . append ( pair ) <EOL> return node_offsets <EOL> def _get_offsets ( self , node ) : <EOL> """<STR_LIT>""" <EOL> if len ( node . _links ) == <NUM_LIT:0> and node . _outEdgeList != [ ] : <EOL> offsets = [ ] <EOL> edge_list = node . _outEdgeList <EOL> edge_list . reverse ( ) <EOL> for edge in edge_list : <EOL> temp_offsets = self . _get_offsets ( edge . _toNode ) <EOL> if temp_offsets is not None : <EOL> offsets . extend ( self . _get_offsets ( edge . _toNode ) ) <EOL> if len ( offsets ) == <NUM_LIT:0> : <EOL> return None <EOL> offsets . sort ( ) <EOL> start = offsets [ <NUM_LIT:0> ] <EOL> end = offsets [ len ( offsets ) - <NUM_LIT:1> ] <EOL> return [ start , end ] <EOL> elif len ( node . _links ) != <NUM_LIT:0> : <EOL> offsets = [ ] <EOL> for link in node . _links : <EOL> for region in link . _regions : <EOL> for anchor in region . _anchors : <EOL> offsets . append ( int ( anchor . _offset ) ) <EOL> offsets . sort ( ) <EOL> start = offsets [ <NUM_LIT:0> ] <EOL> end = offsets [ len ( offsets ) - <NUM_LIT:1> ] <EOL> return [ start , end ] <EOL> else : <EOL> return None <EOL> def _remove_ws ( self , chunk ) : <EOL> """<STR_LIT>""" <EOL> chunk = chunk . replace ( "<STR_LIT:\n>" , "<STR_LIT>" ) <EOL> words = chunk . split ( ) <EOL> new_str = "<STR_LIT>" <EOL> for i in words : <EOL> if i == words [ len ( words ) - <NUM_LIT:1> ] : <EOL> new_str += i <EOL> else : <EOL> new_str = new_str + i + "<STR_LIT:U+0020>" <EOL> return new_str </s>
