<s> import os <EOL> import bisect <EOL> import re <EOL> import tempfile <EOL> from functools import reduce <EOL> try : <EOL> import cPickle as pickle <EOL> except ImportError : <EOL> import pickle <EOL> try : from xml . etree import cElementTree as ElementTree <EOL> except ImportError : from xml . etree import ElementTree <EOL> from nltk . compat import string_types , text_type <EOL> from nltk . tokenize import wordpunct_tokenize <EOL> from nltk . internals import slice_bounds <EOL> from nltk . data import PathPointer , FileSystemPathPointer , ZipFilePathPointer <EOL> from nltk . data import SeekableUnicodeStreamReader <EOL> from nltk . util import AbstractLazySequence , LazySubsequence , LazyConcatenation , py25 <EOL> class StreamBackedCorpusView ( AbstractLazySequence ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , fileid , block_reader = None , startpos = <NUM_LIT:0> , <EOL> encoding = '<STR_LIT:utf8>' ) : <EOL> """<STR_LIT>""" <EOL> if block_reader : <EOL> self . read_block = block_reader <EOL> self . _toknum = [ <NUM_LIT:0> ] <EOL> self . _filepos = [ startpos ] <EOL> self . _encoding = encoding <EOL> self . _len = None <EOL> self . _fileid = fileid <EOL> self . _stream = None <EOL> self . _current_toknum = None <EOL> """<STR_LIT>""" <EOL> self . _current_blocknum = None <EOL> """<STR_LIT>""" <EOL> try : <EOL> if isinstance ( self . _fileid , PathPointer ) : <EOL> self . _eofpos = self . _fileid . file_size ( ) <EOL> else : <EOL> self . _eofpos = os . stat ( self . _fileid ) . st_size <EOL> except Exception as exc : <EOL> raise ValueError ( '<STR_LIT>' % <EOL> ( fileid , exc ) ) <EOL> self . _cache = ( - <NUM_LIT:1> , - <NUM_LIT:1> , None ) <EOL> fileid = property ( lambda self : self . _fileid , doc = """<STR_LIT>""" ) <EOL> def read_block ( self , stream ) : <EOL> """<STR_LIT>""" <EOL> raise NotImplementedError ( '<STR_LIT>' ) <EOL> def _open ( self ) : <EOL> """<STR_LIT>""" <EOL> if isinstance ( self . _fileid , PathPointer ) : <EOL> self . _stream = self . _fileid . open ( self . _encoding ) <EOL> elif self . _encoding : <EOL> self . _stream = SeekableUnicodeStreamReader ( <EOL> open ( self . _fileid , '<STR_LIT:rb>' ) , self . _encoding ) <EOL> else : <EOL> self . _stream = open ( self . _fileid , '<STR_LIT:rb>' ) <EOL> def close ( self ) : <EOL> """<STR_LIT>""" <EOL> if self . _stream is not None : <EOL> self . _stream . close ( ) <EOL> self . _stream = None <EOL> def __len__ ( self ) : <EOL> if self . _len is None : <EOL> for tok in self . iterate_from ( self . _toknum [ - <NUM_LIT:1> ] ) : pass <EOL> return self . _len <EOL> def __getitem__ ( self , i ) : <EOL> if isinstance ( i , slice ) : <EOL> start , stop = slice_bounds ( self , i ) <EOL> offset = self . _cache [ <NUM_LIT:0> ] <EOL> if offset <= start and stop <= self . _cache [ <NUM_LIT:1> ] : <EOL> return self . _cache [ <NUM_LIT:2> ] [ start - offset : stop - offset ] <EOL> return LazySubsequence ( self , start , stop ) <EOL> else : <EOL> if i < <NUM_LIT:0> : i += len ( self ) <EOL> if i < <NUM_LIT:0> : raise IndexError ( '<STR_LIT>' ) <EOL> offset = self . _cache [ <NUM_LIT:0> ] <EOL> if offset <= i < self . _cache [ <NUM_LIT:1> ] : <EOL> return self . _cache [ <NUM_LIT:2> ] [ i - offset ] <EOL> try : <EOL> return next ( self . iterate_from ( i ) ) <EOL> except StopIteration : <EOL> raise IndexError ( '<STR_LIT>' ) <EOL> def iterate_from ( self , start_tok ) : <EOL> if self . _cache [ <NUM_LIT:0> ] <= start_tok < self . _cache [ <NUM_LIT:1> ] : <EOL> for tok in self . _cache [ <NUM_LIT:2> ] [ start_tok - self . _cache [ <NUM_LIT:0> ] : ] : <EOL> yield tok <EOL> start_tok += <NUM_LIT:1> <EOL> if start_tok < self . _toknum [ - <NUM_LIT:1> ] : <EOL> block_index = bisect . bisect_right ( self . _toknum , start_tok ) - <NUM_LIT:1> <EOL> toknum = self . _toknum [ block_index ] <EOL> filepos = self . _filepos [ block_index ] <EOL> else : <EOL> block_index = len ( self . _toknum ) - <NUM_LIT:1> <EOL> toknum = self . _toknum [ - <NUM_LIT:1> ] <EOL> filepos = self . _filepos [ - <NUM_LIT:1> ] <EOL> if self . _stream is None : <EOL> self . _open ( ) <EOL> while filepos < self . _eofpos : <EOL> self . _stream . seek ( filepos ) <EOL> self . _current_toknum = toknum <EOL> self . _current_blocknum = block_index <EOL> tokens = self . read_block ( self . _stream ) <EOL> assert isinstance ( tokens , ( tuple , list , AbstractLazySequence ) ) , ( <EOL> '<STR_LIT>' % <EOL> self . read_block . __name__ ) <EOL> num_toks = len ( tokens ) <EOL> new_filepos = self . _stream . tell ( ) <EOL> assert new_filepos > filepos , ( <EOL> '<STR_LIT>' % <EOL> ( self . read_block . __name__ , filepos ) ) <EOL> self . _cache = ( toknum , toknum + num_toks , list ( tokens ) ) <EOL> assert toknum <= self . _toknum [ - <NUM_LIT:1> ] <EOL> if num_toks > <NUM_LIT:0> : <EOL> block_index += <NUM_LIT:1> <EOL> if toknum == self . _toknum [ - <NUM_LIT:1> ] : <EOL> assert new_filepos > self . _filepos [ - <NUM_LIT:1> ] <EOL> self . _filepos . append ( new_filepos ) <EOL> self . _toknum . append ( toknum + num_toks ) <EOL> else : <EOL> assert new_filepos == self . _filepos [ block_index ] , ( <EOL> '<STR_LIT>' ) <EOL> assert toknum + num_toks == self . _toknum [ block_index ] , ( <EOL> '<STR_LIT>' ) <EOL> if new_filepos == self . _eofpos : <EOL> self . _len = toknum + num_toks <EOL> for tok in tokens [ max ( <NUM_LIT:0> , start_tok - toknum ) : ] : <EOL> yield tok <EOL> assert new_filepos <= self . _eofpos <EOL> if new_filepos == self . _eofpos : <EOL> break <EOL> toknum += num_toks <EOL> filepos = new_filepos <EOL> assert self . _len is not None <EOL> self . close ( ) <EOL> def __add__ ( self , other ) : <EOL> return concat ( [ self , other ] ) <EOL> def __radd__ ( self , other ) : <EOL> return concat ( [ other , self ] ) <EOL> def __mul__ ( self , count ) : <EOL> return concat ( [ self ] * count ) <EOL> def __rmul__ ( self , count ) : <EOL> return concat ( [ self ] * count ) <EOL> class ConcatenatedCorpusView ( AbstractLazySequence ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , corpus_views ) : <EOL> self . _pieces = corpus_views <EOL> """<STR_LIT>""" <EOL> self . _offsets = [ <NUM_LIT:0> ] <EOL> """<STR_LIT>""" <EOL> self . _open_piece = None <EOL> """<STR_LIT>""" <EOL> def __len__ ( self ) : <EOL> if len ( self . _offsets ) <= len ( self . _pieces ) : <EOL> for tok in self . iterate_from ( self . _offsets [ - <NUM_LIT:1> ] ) : pass <EOL> return self . _offsets [ - <NUM_LIT:1> ] <EOL> def close ( self ) : <EOL> for piece in self . _pieces : <EOL> piece . close ( ) <EOL> def iterate_from ( self , start_tok ) : <EOL> piecenum = bisect . bisect_right ( self . _offsets , start_tok ) - <NUM_LIT:1> <EOL> while piecenum < len ( self . _pieces ) : <EOL> offset = self . _offsets [ piecenum ] <EOL> piece = self . _pieces [ piecenum ] <EOL> if self . _open_piece is not piece : <EOL> if self . _open_piece is not None : <EOL> self . _open_piece . close ( ) <EOL> self . _open_piece = piece <EOL> for tok in piece . iterate_from ( max ( <NUM_LIT:0> , start_tok - offset ) ) : <EOL> yield tok <EOL> if piecenum + <NUM_LIT:1> == len ( self . _offsets ) : <EOL> self . _offsets . append ( self . _offsets [ - <NUM_LIT:1> ] + len ( piece ) ) <EOL> piecenum += <NUM_LIT:1> <EOL> def concat ( docs ) : <EOL> """<STR_LIT>""" <EOL> if len ( docs ) == <NUM_LIT:1> : <EOL> return docs [ <NUM_LIT:0> ] <EOL> if len ( docs ) == <NUM_LIT:0> : <EOL> raise ValueError ( '<STR_LIT>' ) <EOL> types = set ( d . __class__ for d in docs ) <EOL> if all ( isinstance ( doc , string_types ) for doc in docs ) : <EOL> return '<STR_LIT>' . join ( docs ) <EOL> for typ in types : <EOL> if not issubclass ( typ , ( StreamBackedCorpusView , <EOL> ConcatenatedCorpusView ) ) : <EOL> break <EOL> else : <EOL> return ConcatenatedCorpusView ( docs ) <EOL> for typ in types : <EOL> if not issubclass ( typ , AbstractLazySequence ) : <EOL> break <EOL> else : <EOL> return LazyConcatenation ( docs ) <EOL> if len ( types ) == <NUM_LIT:1> : <EOL> typ = list ( types ) [ <NUM_LIT:0> ] <EOL> if issubclass ( typ , list ) : <EOL> return reduce ( ( lambda a , b : a + b ) , docs , [ ] ) <EOL> if issubclass ( typ , tuple ) : <EOL> return reduce ( ( lambda a , b : a + b ) , docs , ( ) ) <EOL> if ElementTree . iselement ( typ ) : <EOL> xmltree = ElementTree . Element ( '<STR_LIT>' ) <EOL> for doc in docs : xmltree . append ( doc ) <EOL> return xmltree <EOL> raise ValueError ( "<STR_LIT>" % types ) <EOL> class PickleCorpusView ( StreamBackedCorpusView ) : <EOL> """<STR_LIT>""" <EOL> BLOCK_SIZE = <NUM_LIT:100> <EOL> PROTOCOL = - <NUM_LIT:1> <EOL> def __init__ ( self , fileid , delete_on_gc = False ) : <EOL> """<STR_LIT>""" <EOL> self . _delete_on_gc = delete_on_gc <EOL> StreamBackedCorpusView . __init__ ( self , fileid ) <EOL> def read_block ( self , stream ) : <EOL> result = [ ] <EOL> for i in range ( self . BLOCK_SIZE ) : <EOL> try : result . append ( pickle . load ( stream ) ) <EOL> except EOFError : break <EOL> return result <EOL> def __del__ ( self ) : <EOL> """<STR_LIT>""" <EOL> if getattr ( self , '<STR_LIT>' ) : <EOL> if os . path . exists ( self . _fileid ) : <EOL> try : os . remove ( self . _fileid ) <EOL> except ( OSError , IOError ) : pass <EOL> self . __dict__ . clear ( ) <EOL> @ classmethod <EOL> def write ( cls , sequence , output_file ) : <EOL> if isinstance ( output_file , string_types ) : <EOL> output_file = open ( output_file , '<STR_LIT:wb>' ) <EOL> for item in sequence : <EOL> pickle . dump ( item , output_file , cls . PROTOCOL ) <EOL> @ classmethod <EOL> def cache_to_tempfile ( cls , sequence , delete_on_gc = True ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> fd , output_file_name = tempfile . mkstemp ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> output_file = os . fdopen ( fd , '<STR_LIT:wb>' ) <EOL> cls . write ( sequence , output_file ) <EOL> output_file . close ( ) <EOL> return PickleCorpusView ( output_file_name , delete_on_gc ) <EOL> except ( OSError , IOError ) as e : <EOL> raise ValueError ( '<STR_LIT>' % e ) <EOL> def read_whitespace_block ( stream ) : <EOL> toks = [ ] <EOL> for i in range ( <NUM_LIT:20> ) : <EOL> toks . extend ( stream . readline ( ) . split ( ) ) <EOL> return toks <EOL> def read_wordpunct_block ( stream ) : <EOL> toks = [ ] <EOL> for i in range ( <NUM_LIT:20> ) : <EOL> toks . extend ( wordpunct_tokenize ( stream . readline ( ) ) ) <EOL> return toks <EOL> def read_line_block ( stream ) : <EOL> toks = [ ] <EOL> for i in range ( <NUM_LIT:20> ) : <EOL> line = stream . readline ( ) <EOL> if not line : return toks <EOL> toks . append ( line . rstrip ( '<STR_LIT:\n>' ) ) <EOL> return toks <EOL> def read_blankline_block ( stream ) : <EOL> s = '<STR_LIT>' <EOL> while True : <EOL> line = stream . readline ( ) <EOL> if not line : <EOL> if s : return [ s ] <EOL> else : return [ ] <EOL> elif line and not line . strip ( ) : <EOL> if s : return [ s ] <EOL> else : <EOL> s += line <EOL> def read_alignedsent_block ( stream ) : <EOL> s = '<STR_LIT>' <EOL> while True : <EOL> line = stream . readline ( ) <EOL> if line [ <NUM_LIT:0> ] == '<STR_LIT:=>' or line [ <NUM_LIT:0> ] == '<STR_LIT:\n>' or line [ : <NUM_LIT:2> ] == '<STR_LIT:\r\n>' : <EOL> continue <EOL> if not line : <EOL> if s : return [ s ] <EOL> else : return [ ] <EOL> else : <EOL> s += line <EOL> if re . match ( '<STR_LIT>' , line ) is not None : <EOL> return [ s ] <EOL> def read_regexp_block ( stream , start_re , end_re = None ) : <EOL> """<STR_LIT>""" <EOL> while True : <EOL> line = stream . readline ( ) <EOL> if not line : return [ ] <EOL> if re . match ( start_re , line ) : break <EOL> lines = [ line ] <EOL> while True : <EOL> oldpos = stream . tell ( ) <EOL> line = stream . readline ( ) <EOL> if not line : <EOL> return [ '<STR_LIT>' . join ( lines ) ] <EOL> if end_re is not None and re . match ( end_re , line ) : <EOL> return [ '<STR_LIT>' . join ( lines ) ] <EOL> if end_re is None and re . match ( start_re , line ) : <EOL> stream . seek ( oldpos ) <EOL> return [ '<STR_LIT>' . join ( lines ) ] <EOL> lines . append ( line ) <EOL> def read_sexpr_block ( stream , block_size = <NUM_LIT> , comment_char = None ) : <EOL> """<STR_LIT>""" <EOL> start = stream . tell ( ) <EOL> block = stream . read ( block_size ) <EOL> encoding = getattr ( stream , '<STR_LIT>' , None ) <EOL> assert encoding is not None or isinstance ( block , text_type ) <EOL> if encoding not in ( None , '<STR_LIT:utf-8>' ) : <EOL> import warnings <EOL> warnings . warn ( '<STR_LIT>' <EOL> '<STR_LIT>' % encoding ) <EOL> if comment_char : <EOL> COMMENT = re . compile ( '<STR_LIT>' % re . escape ( comment_char ) ) <EOL> while True : <EOL> try : <EOL> if comment_char : <EOL> block += stream . readline ( ) <EOL> block = re . sub ( COMMENT , _sub_space , block ) <EOL> tokens , offset = _parse_sexpr_block ( block ) <EOL> offset = re . compile ( r'<STR_LIT>' ) . search ( block , offset ) . end ( ) <EOL> if encoding is None : <EOL> stream . seek ( start + offset ) <EOL> else : <EOL> stream . seek ( start + len ( block [ : offset ] . encode ( encoding ) ) ) <EOL> return tokens <EOL> except ValueError as e : <EOL> if e . args [ <NUM_LIT:0> ] == '<STR_LIT>' : <EOL> next_block = stream . read ( block_size ) <EOL> if next_block : <EOL> block += next_block <EOL> continue <EOL> else : <EOL> return [ block . strip ( ) ] <EOL> else : raise <EOL> def _sub_space ( m ) : <EOL> """<STR_LIT>""" <EOL> return '<STR_LIT:U+0020>' * ( m . end ( ) - m . start ( ) ) <EOL> def _parse_sexpr_block ( block ) : <EOL> tokens = [ ] <EOL> start = end = <NUM_LIT:0> <EOL> while end < len ( block ) : <EOL> m = re . compile ( r'<STR_LIT>' ) . search ( block , end ) <EOL> if not m : <EOL> return tokens , end <EOL> start = m . start ( ) <EOL> if m . group ( ) != '<STR_LIT:(>' : <EOL> m2 = re . compile ( r'<STR_LIT>' ) . search ( block , start ) <EOL> if m2 : <EOL> end = m2 . start ( ) <EOL> else : <EOL> if tokens : return tokens , end <EOL> raise ValueError ( '<STR_LIT>' ) <EOL> else : <EOL> nesting = <NUM_LIT:0> <EOL> for m in re . compile ( r'<STR_LIT>' ) . finditer ( block , start ) : <EOL> if m . group ( ) == '<STR_LIT:(>' : nesting += <NUM_LIT:1> <EOL> else : nesting -= <NUM_LIT:1> <EOL> if nesting == <NUM_LIT:0> : <EOL> end = m . end ( ) <EOL> break <EOL> else : <EOL> if tokens : return tokens , end <EOL> raise ValueError ( '<STR_LIT>' ) <EOL> tokens . append ( block [ start : end ] ) <EOL> return tokens , end <EOL> def find_corpus_fileids ( root , regexp ) : <EOL> if not isinstance ( root , PathPointer ) : <EOL> raise TypeError ( '<STR_LIT>' ) <EOL> regexp += '<STR_LIT:$>' <EOL> if isinstance ( root , ZipFilePathPointer ) : <EOL> fileids = [ name [ len ( root . entry ) : ] for name in root . zipfile . namelist ( ) <EOL> if not name . endswith ( '<STR_LIT:/>' ) ] <EOL> items = [ name for name in fileids if re . match ( regexp , name ) ] <EOL> return sorted ( items ) <EOL> elif isinstance ( root , FileSystemPathPointer ) : <EOL> items = [ ] <EOL> kwargs = { } <EOL> if not py25 ( ) : <EOL> kwargs = { '<STR_LIT>' : True } <EOL> for dirname , subdirs , fileids in os . walk ( root . path , ** kwargs ) : <EOL> prefix = '<STR_LIT>' . join ( '<STR_LIT>' % p for p in _path_from ( root . path , dirname ) ) <EOL> items += [ prefix + fileid for fileid in fileids <EOL> if re . match ( regexp , prefix + fileid ) ] <EOL> if '<STR_LIT>' in subdirs : subdirs . remove ( '<STR_LIT>' ) <EOL> return sorted ( items ) <EOL> else : <EOL> raise AssertionError ( "<STR_LIT>" % root ) <EOL> def _path_from ( parent , child ) : <EOL> if os . path . split ( parent ) [ <NUM_LIT:1> ] == '<STR_LIT>' : <EOL> parent = os . path . split ( parent ) [ <NUM_LIT:0> ] <EOL> path = [ ] <EOL> while parent != child : <EOL> child , dirname = os . path . split ( child ) <EOL> path . insert ( <NUM_LIT:0> , dirname ) <EOL> assert os . path . split ( child ) [ <NUM_LIT:0> ] != child <EOL> return path <EOL> def tagged_treebank_para_block_reader ( stream ) : <EOL> para = '<STR_LIT>' <EOL> while True : <EOL> line = stream . readline ( ) <EOL> if re . match ( '<STR_LIT>' , line ) : <EOL> if para . strip ( ) : return [ para ] <EOL> elif line == '<STR_LIT>' : <EOL> if para . strip ( ) : return [ para ] <EOL> else : return [ ] <EOL> else : <EOL> para += line </s>
