<s> """<STR_LIT>""" <EOL> from nltk import compat <EOL> from nltk . tag import str2tuple , map_tag <EOL> from nltk . corpus . reader . util import * <EOL> from nltk . corpus . reader . api import * <EOL> class IndianCorpusReader ( CorpusReader ) : <EOL> """<STR_LIT>""" <EOL> def words ( self , fileids = None ) : <EOL> return concat ( [ IndianCorpusView ( fileid , enc , <EOL> False , False ) <EOL> for ( fileid , enc ) in self . abspaths ( fileids , True ) ] ) <EOL> def tagged_words ( self , fileids = None , tagset = None ) : <EOL> if tagset and tagset != self . _tagset : <EOL> tag_mapping_function = lambda t : map_tag ( self . _tagset , tagset , t ) <EOL> else : <EOL> tag_mapping_function = None <EOL> return concat ( [ IndianCorpusView ( fileid , enc , <EOL> True , False , tag_mapping_function ) <EOL> for ( fileid , enc ) in self . abspaths ( fileids , True ) ] ) <EOL> def sents ( self , fileids = None ) : <EOL> return concat ( [ IndianCorpusView ( fileid , enc , <EOL> False , True ) <EOL> for ( fileid , enc ) in self . abspaths ( fileids , True ) ] ) <EOL> def tagged_sents ( self , fileids = None , tagset = None ) : <EOL> if tagset and tagset != self . _tagset : <EOL> tag_mapping_function = lambda t : map_tag ( self . _tagset , tagset , t ) <EOL> else : <EOL> tag_mapping_function = None <EOL> return concat ( [ IndianCorpusView ( fileid , enc , <EOL> True , True , tag_mapping_function ) <EOL> for ( fileid , enc ) in self . abspaths ( fileids , True ) ] ) <EOL> def raw ( self , fileids = None ) : <EOL> if fileids is None : fileids = self . _fileids <EOL> elif isinstance ( fileids , compat . string_types ) : fileids = [ fileids ] <EOL> return concat ( [ self . open ( f ) . read ( ) for f in fileids ] ) <EOL> class IndianCorpusView ( StreamBackedCorpusView ) : <EOL> def __init__ ( self , corpus_file , encoding , tagged , <EOL> group_by_sent , tag_mapping_function = None ) : <EOL> self . _tagged = tagged <EOL> self . _group_by_sent = group_by_sent <EOL> self . _tag_mapping_function = tag_mapping_function <EOL> StreamBackedCorpusView . __init__ ( self , corpus_file , encoding = encoding ) <EOL> def read_block ( self , stream ) : <EOL> line = stream . readline ( ) <EOL> if line . startswith ( '<STR_LIT:<>' ) : <EOL> return [ ] <EOL> sent = [ str2tuple ( word , sep = '<STR_LIT:_>' ) for word in line . split ( ) ] <EOL> if self . _tag_mapping_function : <EOL> sent = [ ( w , self . _tag_mapping_function ( t ) ) for ( w , t ) in sent ] <EOL> if not self . _tagged : sent = [ w for ( w , t ) in sent ] <EOL> if self . _group_by_sent : <EOL> return [ sent ] <EOL> else : <EOL> return sent </s>
