<s> r"""<STR_LIT>""" <EOL> from __future__ import unicode_literals <EOL> from nltk . tokenize . api import TokenizerI , StringTokenizer <EOL> from nltk . tokenize . util import string_span_tokenize , regexp_span_tokenize <EOL> class SpaceTokenizer ( StringTokenizer ) : <EOL> r"""<STR_LIT>""" <EOL> _string = '<STR_LIT:U+0020>' <EOL> class TabTokenizer ( StringTokenizer ) : <EOL> r"""<STR_LIT>""" <EOL> _string = '<STR_LIT:\t>' <EOL> class CharTokenizer ( StringTokenizer ) : <EOL> """<STR_LIT>""" <EOL> def tokenize ( self , s ) : <EOL> return list ( s ) <EOL> def span_tokenize ( self , s ) : <EOL> for i , j in enumerate ( range ( <NUM_LIT:1> , len ( s ) + <NUM_LIT:1> ) ) : <EOL> yield i , j <EOL> class LineTokenizer ( TokenizerI ) : <EOL> r"""<STR_LIT>""" <EOL> def __init__ ( self , blanklines = '<STR_LIT>' ) : <EOL> valid_blanklines = ( '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ) <EOL> if blanklines not in valid_blanklines : <EOL> raise ValueError ( '<STR_LIT>' % <EOL> '<STR_LIT:U+0020>' . join ( valid_blanklines ) ) <EOL> self . _blanklines = blanklines <EOL> def tokenize ( self , s ) : <EOL> lines = s . splitlines ( ) <EOL> if self . _blanklines == '<STR_LIT>' : <EOL> lines = [ l for l in lines if l . rstrip ( ) ] <EOL> elif self . _blanklines == '<STR_LIT>' : <EOL> if lines and not lines [ - <NUM_LIT:1> ] . strip ( ) : <EOL> lines . pop ( ) <EOL> return lines <EOL> def span_tokenize ( self , s ) : <EOL> if self . _blanklines == '<STR_LIT>' : <EOL> for span in string_span_tokenize ( s , r'<STR_LIT:\n>' ) : <EOL> yield span <EOL> else : <EOL> for span in regexp_span_tokenize ( s , r'<STR_LIT>' ) : <EOL> yield span <EOL> def line_tokenize ( text , blanklines = '<STR_LIT>' ) : <EOL> return LineTokenizer ( blanklines ) . tokenize ( text ) </s>
