<s> import re <EOL> import string <EOL> PUNCTUATION_REGEX = re . compile ( '<STR_LIT>' . format ( re . escape ( string . punctuation ) ) ) <EOL> def strip_punc ( s , all = False ) : <EOL> """<STR_LIT>""" <EOL> if all : <EOL> return PUNCTUATION_REGEX . sub ( '<STR_LIT>' , s . strip ( ) ) <EOL> else : <EOL> return s . strip ( ) . strip ( string . punctuation ) <EOL> def lowerstrip ( s , all = False ) : <EOL> """<STR_LIT>""" <EOL> return strip_punc ( s . lower ( ) . strip ( ) , all = all ) <EOL> def tree2str ( tree , concat = '<STR_LIT:U+0020>' ) : <EOL> """<STR_LIT>""" <EOL> return concat . join ( [ word for ( word , tag ) in tree ] ) <EOL> def filter_insignificant ( chunk , tag_suffixes = ( '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ) ) : <EOL> """<STR_LIT>""" <EOL> good = [ ] <EOL> for word , tag in chunk : <EOL> ok = True <EOL> for suffix in tag_suffixes : <EOL> if tag . endswith ( suffix ) : <EOL> ok = False <EOL> break <EOL> if ok : <EOL> good . append ( ( word , tag ) ) <EOL> return good <EOL> def is_filelike ( obj ) : <EOL> """<STR_LIT>""" <EOL> return hasattr ( obj , '<STR_LIT>' ) </s>
