<s> import functools <EOL> import os <EOL> import tempfile <EOL> from nltk import compat <EOL> from nltk . corpus . reader . util import concat <EOL> from nltk . corpus . reader . xmldocs import XMLCorpusReader , XMLCorpusView <EOL> import re <EOL> def _parse_args ( fun ) : <EOL> """<STR_LIT>""" <EOL> @ functools . wraps ( fun ) <EOL> def decorator ( self , fileids = None , ** kwargs ) : <EOL> if not fileids : <EOL> fileids = self . _paths <EOL> return fun ( self , fileids , ** kwargs ) <EOL> return decorator <EOL> class NKJPCorpusReader ( XMLCorpusReader ) : <EOL> WORDS_MODE = <NUM_LIT:0> <EOL> SENTS_MODE = <NUM_LIT:1> <EOL> HEADER_MODE = <NUM_LIT:2> <EOL> RAW_MODE = <NUM_LIT:3> <EOL> def __init__ ( self , root , fileids = '<STR_LIT>' ) : <EOL> """<STR_LIT>""" <EOL> if isinstance ( fileids , compat . string_types ) : <EOL> XMLCorpusReader . __init__ ( self , root , fileids + '<STR_LIT>' ) <EOL> else : <EOL> XMLCorpusReader . __init__ ( self , root , [ fileid + '<STR_LIT>' for fileid in fileids ] ) <EOL> self . _paths = self . get_paths ( ) <EOL> def get_paths ( self ) : <EOL> return [ os . path . join ( str ( self . _root ) , f . split ( "<STR_LIT>" ) [ <NUM_LIT:0> ] ) for f in self . _fileids ] <EOL> def fileids ( self ) : <EOL> """<STR_LIT>""" <EOL> return [ f . split ( "<STR_LIT>" ) [ <NUM_LIT:0> ] for f in self . _fileids ] <EOL> def _view ( self , filename , tags = None , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> mode = kwargs . pop ( '<STR_LIT>' , NKJPCorpusReader . WORDS_MODE ) <EOL> if mode is NKJPCorpusReader . WORDS_MODE : <EOL> return NKJPCorpus_Morph_View ( filename , tags = tags ) <EOL> elif mode is NKJPCorpusReader . SENTS_MODE : <EOL> return NKJPCorpus_Segmentation_View ( filename , tags = tags ) <EOL> elif mode is NKJPCorpusReader . HEADER_MODE : <EOL> return NKJPCorpus_Header_View ( filename , tags = tags ) <EOL> elif mode is NKJPCorpusReader . RAW_MODE : <EOL> return NKJPCorpus_Text_View ( filename , tags = tags , mode = NKJPCorpus_Text_View . RAW_MODE ) <EOL> else : <EOL> raise NameError ( '<STR_LIT>' ) <EOL> def add_root ( self , fileid ) : <EOL> """<STR_LIT>""" <EOL> if self . root in fileid : <EOL> return fileid <EOL> return self . root + fileid <EOL> @ _parse_args <EOL> def header ( self , fileids = None , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> return concat ( [ self . _view ( self . add_root ( fileid ) , <EOL> mode = NKJPCorpusReader . HEADER_MODE , ** kwargs ) . handle_query ( ) <EOL> for fileid in fileids ] ) <EOL> @ _parse_args <EOL> def sents ( self , fileids = None , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> return concat ( [ self . _view ( self . add_root ( fileid ) , <EOL> mode = NKJPCorpusReader . SENTS_MODE , ** kwargs ) . handle_query ( ) <EOL> for fileid in fileids ] ) <EOL> @ _parse_args <EOL> def words ( self , fileids = None , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> return concat ( [ self . _view ( self . add_root ( fileid ) , <EOL> mode = NKJPCorpusReader . WORDS_MODE , ** kwargs ) . handle_query ( ) <EOL> for fileid in fileids ] ) <EOL> @ _parse_args <EOL> def tagged_words ( self , fileids = None , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> tags = kwargs . pop ( '<STR_LIT>' , [ ] ) <EOL> return concat ( [ self . _view ( self . add_root ( fileid ) , <EOL> mode = NKJPCorpusReader . WORDS_MODE , tags = tags , ** kwargs ) . handle_query ( ) <EOL> for fileid in fileids ] ) <EOL> @ _parse_args <EOL> def raw ( self , fileids = None , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> return concat ( [ self . _view ( self . add_root ( fileid ) , <EOL> mode = NKJPCorpusReader . RAW_MODE , ** kwargs ) . handle_query ( ) <EOL> for fileid in fileids ] ) <EOL> class NKJPCorpus_Header_View ( XMLCorpusView ) : <EOL> def __init__ ( self , filename , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> self . tagspec = "<STR_LIT>" <EOL> XMLCorpusView . __init__ ( self , filename + '<STR_LIT>' , self . tagspec ) <EOL> def handle_query ( self ) : <EOL> self . _open ( ) <EOL> header = [ ] <EOL> while True : <EOL> segm = XMLCorpusView . read_block ( self , self . _stream ) <EOL> if len ( segm ) == <NUM_LIT:0> : <EOL> break <EOL> header . extend ( segm ) <EOL> self . close ( ) <EOL> return header <EOL> def handle_elt ( self , elt , context ) : <EOL> titles = elt . findall ( '<STR_LIT>' ) <EOL> title = [ ] <EOL> if titles : <EOL> title = '<STR_LIT:\n>' . join ( title . text . strip ( ) for title in titles ) <EOL> authors = elt . findall ( '<STR_LIT>' ) <EOL> author = [ ] <EOL> if authors : <EOL> author = '<STR_LIT:\n>' . join ( author . text . strip ( ) for author in authors ) <EOL> dates = elt . findall ( '<STR_LIT>' ) <EOL> date = [ ] <EOL> if dates : <EOL> date = '<STR_LIT:\n>' . join ( date . text . strip ( ) for date in dates ) <EOL> publishers = elt . findall ( '<STR_LIT>' ) <EOL> publisher = [ ] <EOL> if publishers : <EOL> publisher = '<STR_LIT:\n>' . join ( publisher . text . strip ( ) for publisher in publishers ) <EOL> idnos = elt . findall ( '<STR_LIT>' ) <EOL> idno = [ ] <EOL> if idnos : <EOL> idno = '<STR_LIT:\n>' . join ( idno . text . strip ( ) for idno in idnos ) <EOL> notes = elt . findall ( '<STR_LIT>' ) <EOL> note = [ ] <EOL> if notes : <EOL> note = '<STR_LIT:\n>' . join ( note . text . strip ( ) for note in notes ) <EOL> return { '<STR_LIT:title>' : title , '<STR_LIT>' : author , '<STR_LIT:date>' : date , '<STR_LIT>' : publisher , <EOL> '<STR_LIT>' : idno , '<STR_LIT>' : note } <EOL> class XML_Tool ( ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , root , filename ) : <EOL> self . read_file = os . path . join ( root , filename ) <EOL> self . write_file = tempfile . NamedTemporaryFile ( delete = False ) <EOL> def build_preprocessed_file ( self ) : <EOL> try : <EOL> fr = open ( self . read_file , '<STR_LIT:r>' ) <EOL> fw = self . write_file <EOL> line = '<STR_LIT:U+0020>' <EOL> while len ( line ) : <EOL> line = fr . readline ( ) <EOL> x = re . split ( r'<STR_LIT>' , line ) <EOL> ret = '<STR_LIT:U+0020>' . join ( x ) <EOL> x = re . split ( '<STR_LIT>' , ret ) <EOL> ret = '<STR_LIT:U+0020>' . join ( x ) <EOL> x = re . split ( '<STR_LIT>' , ret ) <EOL> ret = '<STR_LIT:U+0020>' . join ( x ) <EOL> x = re . split ( '<STR_LIT>' , ret ) <EOL> ret = '<STR_LIT:U+0020>' . join ( x ) <EOL> x = re . split ( '<STR_LIT>' , ret ) <EOL> ret = '<STR_LIT:U+0020>' . join ( x ) <EOL> fw . write ( ret ) <EOL> fr . close ( ) <EOL> fw . close ( ) <EOL> return self . write_file . name <EOL> except Exception : <EOL> self . remove_preprocessed_file ( ) <EOL> raise Exception <EOL> def remove_preprocessed_file ( self ) : <EOL> os . remove ( self . write_file . name ) <EOL> pass <EOL> class NKJPCorpus_Segmentation_View ( XMLCorpusView ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , filename , ** kwargs ) : <EOL> self . tagspec = '<STR_LIT>' <EOL> self . text_view = NKJPCorpus_Text_View ( filename , mode = NKJPCorpus_Text_View . SENTS_MODE ) <EOL> self . text_view . handle_query ( ) <EOL> self . xml_tool = XML_Tool ( filename , '<STR_LIT>' ) <EOL> XMLCorpusView . __init__ ( self , self . xml_tool . build_preprocessed_file ( ) , self . tagspec ) <EOL> def get_segm_id ( self , example_word ) : <EOL> return example_word . split ( '<STR_LIT:(>' ) [ <NUM_LIT:1> ] . split ( '<STR_LIT:U+002C>' ) [ <NUM_LIT:0> ] <EOL> def get_sent_beg ( self , beg_word ) : <EOL> return int ( beg_word . split ( '<STR_LIT:U+002C>' ) [ <NUM_LIT:1> ] ) <EOL> def get_sent_end ( self , end_word ) : <EOL> splitted = end_word . split ( '<STR_LIT:)>' ) [ <NUM_LIT:0> ] . split ( '<STR_LIT:U+002C>' ) <EOL> return int ( splitted [ <NUM_LIT:1> ] ) + int ( splitted [ <NUM_LIT:2> ] ) <EOL> def get_sentences ( self , sent_segm ) : <EOL> id = self . get_segm_id ( sent_segm [ <NUM_LIT:0> ] ) <EOL> segm = self . text_view . segm_dict [ id ] <EOL> beg = self . get_sent_beg ( sent_segm [ <NUM_LIT:0> ] ) <EOL> end = self . get_sent_end ( sent_segm [ len ( sent_segm ) - <NUM_LIT:1> ] ) <EOL> return segm [ beg : end ] <EOL> def remove_choice ( self , segm ) : <EOL> ret = [ ] <EOL> prev_txt_end = - <NUM_LIT:1> <EOL> prev_txt_nr = - <NUM_LIT:1> <EOL> for word in segm : <EOL> txt_nr = self . get_segm_id ( word ) <EOL> if self . get_sent_beg ( word ) > prev_txt_end - <NUM_LIT:1> or prev_txt_nr != txt_nr : <EOL> ret . append ( word ) <EOL> prev_txt_end = self . get_sent_end ( word ) <EOL> prev_txt_nr = txt_nr <EOL> return ret <EOL> def handle_query ( self ) : <EOL> try : <EOL> self . _open ( ) <EOL> sentences = [ ] <EOL> while True : <EOL> sent_segm = XMLCorpusView . read_block ( self , self . _stream ) <EOL> if len ( sent_segm ) == <NUM_LIT:0> : <EOL> break <EOL> for segm in sent_segm : <EOL> segm = self . remove_choice ( segm ) <EOL> sentences . append ( self . get_sentences ( segm ) ) <EOL> self . close ( ) <EOL> self . xml_tool . remove_preprocessed_file ( ) <EOL> return sentences <EOL> except Exception : <EOL> self . xml_tool . remove_preprocessed_file ( ) <EOL> raise Exception <EOL> def handle_elt ( self , elt , context ) : <EOL> ret = [ ] <EOL> for seg in elt : <EOL> ret . append ( seg . get ( '<STR_LIT>' ) ) <EOL> return ret <EOL> class NKJPCorpus_Text_View ( XMLCorpusView ) : <EOL> """<STR_LIT>""" <EOL> SENTS_MODE = <NUM_LIT:0> <EOL> RAW_MODE = <NUM_LIT:1> <EOL> def __init__ ( self , filename , ** kwargs ) : <EOL> self . mode = kwargs . pop ( '<STR_LIT>' , <NUM_LIT:0> ) <EOL> self . tagspec = '<STR_LIT>' <EOL> self . segm_dict = dict ( ) <EOL> self . xml_tool = XML_Tool ( filename , '<STR_LIT>' ) <EOL> XMLCorpusView . __init__ ( self , self . xml_tool . build_preprocessed_file ( ) , self . tagspec ) <EOL> def handle_query ( self ) : <EOL> try : <EOL> self . _open ( ) <EOL> x = self . read_block ( self . _stream ) <EOL> self . close ( ) <EOL> self . xml_tool . remove_preprocessed_file ( ) <EOL> return x <EOL> except Exception : <EOL> self . xml_tool . remove_preprocessed_file ( ) <EOL> raise Exception <EOL> def read_block ( self , stream , tagspec = None , elt_handler = None ) : <EOL> """<STR_LIT>""" <EOL> txt = [ ] <EOL> while True : <EOL> segm = XMLCorpusView . read_block ( self , stream ) <EOL> if len ( segm ) == <NUM_LIT:0> : <EOL> break <EOL> for part in segm : <EOL> txt . append ( part ) <EOL> return [ '<STR_LIT:U+0020>' . join ( [ segm for segm in txt ] ) ] <EOL> def get_segm_id ( self , elt ) : <EOL> for attr in elt . attrib : <EOL> if attr . endswith ( '<STR_LIT:id>' ) : <EOL> return elt . get ( attr ) <EOL> def handle_elt ( self , elt , context ) : <EOL> if self . mode is NKJPCorpus_Text_View . SENTS_MODE : <EOL> self . segm_dict [ self . get_segm_id ( elt ) ] = elt . text <EOL> return elt . text <EOL> class NKJPCorpus_Morph_View ( XMLCorpusView ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , filename , ** kwargs ) : <EOL> self . tags = kwargs . pop ( '<STR_LIT>' , None ) <EOL> self . tagspec = '<STR_LIT>' <EOL> self . xml_tool = XML_Tool ( filename , '<STR_LIT>' ) <EOL> XMLCorpusView . __init__ ( self , self . xml_tool . build_preprocessed_file ( ) , self . tagspec ) <EOL> def handle_query ( self ) : <EOL> try : <EOL> self . _open ( ) <EOL> words = [ ] <EOL> while True : <EOL> segm = XMLCorpusView . read_block ( self , self . _stream ) <EOL> if len ( segm ) == <NUM_LIT:0> : <EOL> break <EOL> for part in segm : <EOL> if part is not None : <EOL> words . append ( part ) <EOL> self . close ( ) <EOL> self . xml_tool . remove_preprocessed_file ( ) <EOL> return words <EOL> except Exception : <EOL> self . xml_tool . remove_preprocessed_file ( ) <EOL> raise Exception <EOL> def handle_elt ( self , elt , context ) : <EOL> word = '<STR_LIT>' <EOL> flag = False <EOL> is_not_interp = True <EOL> if self . tags is None : <EOL> flag = True <EOL> for child in elt : <EOL> if '<STR_LIT:name>' in child . keys ( ) and child . attrib [ '<STR_LIT:name>' ] == '<STR_LIT>' : <EOL> for symbol in child : <EOL> if symbol . tag == '<STR_LIT:string>' : <EOL> word = symbol . text <EOL> elif '<STR_LIT:name>' in child . keys ( ) and child . attrib [ '<STR_LIT:name>' ] == '<STR_LIT>' : <EOL> for symbol in child : <EOL> if '<STR_LIT:type>' in symbol . keys ( ) and symbol . attrib [ '<STR_LIT:type>' ] == '<STR_LIT>' : <EOL> for symbol2 in symbol : <EOL> if '<STR_LIT:name>' in symbol2 . keys ( ) and symbol2 . attrib [ '<STR_LIT:name>' ] == '<STR_LIT>' : <EOL> for symbol3 in symbol2 : <EOL> if '<STR_LIT:value>' in symbol3 . keys ( ) and self . tags is not None and symbol3 . attrib [ '<STR_LIT:value>' ] in self . tags : <EOL> flag = True <EOL> elif '<STR_LIT:value>' in symbol3 . keys ( ) and symbol3 . attrib [ '<STR_LIT:value>' ] == '<STR_LIT>' : <EOL> is_not_interp = False <EOL> if flag and is_not_interp : <EOL> return word </s>
