<s> import itertools <EOL> import os <EOL> import random <EOL> import re <EOL> import sys <EOL> from nltk . corpus import stopwords as nltk_stopwords <EOL> class ImmutableMixin ( object ) : <EOL> _inited = False <EOL> def __init__ ( self ) : <EOL> self . _inited = True <EOL> def __setattr__ ( self , key , value ) : <EOL> if self . _inited : <EOL> raise exceptions . ImmutableException ( ) <EOL> super ( ImmutableMixin , self ) . __setattr__ ( key , value ) <EOL> class EnumMixin ( object ) : <EOL> def __iter__ ( self ) : <EOL> for k , v in itertools . imap ( lambda x : ( x , getattr ( self , x ) ) , dir ( self ) ) : <EOL> if not k . startswith ( '<STR_LIT:_>' ) : <EOL> yield v <EOL> def prepare_text ( text ) : <EOL> text = unicode ( text . decode ( '<STR_LIT:utf-8>' , errors = '<STR_LIT:replace>' ) ) <EOL> text = text . upper ( ) <EOL> return text <EOL> def tokenize ( text ) : <EOL> return re . findall ( re . compile ( "<STR_LIT>" , re . U ) , text ) <EOL> def tokenize_and_filter ( text , min_word_length = <NUM_LIT:3> , stopwords = None ) : <EOL> tokens = tokenize ( text ) <EOL> stopwords = stopwords or set ( word . upper ( ) for word in nltk_stopwords . words ( "<STR_LIT>" ) ) <EOL> return [ token for token in tokens <EOL> if len ( token ) >= min_word_length and token not in stopwords ] <EOL> def text_to_strings_collection ( text , words = <NUM_LIT:3> ) : <EOL> """<STR_LIT>""" <EOL> text = prepare_text ( text ) <EOL> strings_collection = tokenize ( text ) <EOL> strings_collection = filter ( lambda s : len ( s ) > <NUM_LIT:2> and not s . isdigit ( ) , strings_collection ) <EOL> i = <NUM_LIT:0> <EOL> strings_collection_grouped = [ ] <EOL> while i < len ( strings_collection ) : <EOL> group = '<STR_LIT>' <EOL> for j in xrange ( words ) : <EOL> if i + j < len ( strings_collection ) : <EOL> group += strings_collection [ i + j ] <EOL> strings_collection_grouped . append ( group ) <EOL> i += words <EOL> if not strings_collection_grouped : <EOL> strings_collection_grouped = [ "<STR_LIT:U+0020>" ] <EOL> return strings_collection_grouped <EOL> def text_collection_to_string_collection ( text_collection , words = <NUM_LIT:3> ) : <EOL> return flatten ( [ text_to_strings_collection ( text ) for text in text_collection ] ) <EOL> def random_string ( length ) : <EOL> string = "<STR_LIT>" . join ( [ unichr ( ord ( "<STR_LIT:A>" ) + random . randint ( <NUM_LIT:0> , <NUM_LIT> ) ) for _ in xrange ( length - <NUM_LIT:2> ) ] ) <EOL> return string <EOL> def flatten ( lst ) : <EOL> return list ( itertools . chain . from_iterable ( lst ) ) <EOL> def output_is_redirected ( ) : <EOL> return os . fstat ( <NUM_LIT:0> ) != os . fstat ( <NUM_LIT:1> ) <EOL> def itersubclasses ( cls , _seen = None ) : <EOL> """<STR_LIT>""" <EOL> if not isinstance ( cls , type ) : <EOL> raise TypeError ( _ ( '<STR_LIT>' <EOL> '<STR_LIT>' ) % cls ) <EOL> _seen = _seen or set ( ) <EOL> try : <EOL> subs = cls . __subclasses__ ( ) <EOL> except TypeError : <EOL> subs = cls . __subclasses__ ( cls ) <EOL> for sub in subs : <EOL> if sub not in _seen : <EOL> _seen . add ( sub ) <EOL> yield sub <EOL> for sub in itersubclasses ( sub , _seen ) : <EOL> yield sub <EOL> def import_modules_from_package ( package ) : <EOL> """<STR_LIT>""" <EOL> path = [ os . path . dirname ( __file__ ) , '<STR_LIT:..>' ] + package . split ( '<STR_LIT:.>' ) <EOL> path = os . path . join ( * path ) <EOL> for root , dirs , files in os . walk ( path ) : <EOL> for filename in files : <EOL> if filename . startswith ( '<STR_LIT>' ) or not filename . endswith ( '<STR_LIT>' ) : <EOL> continue <EOL> new_package = "<STR_LIT:.>" . join ( root . split ( os . sep ) ) . split ( "<STR_LIT>" ) [ <NUM_LIT:1> ] <EOL> module_name = '<STR_LIT>' % ( new_package , filename [ : - <NUM_LIT:3> ] ) <EOL> if module_name not in sys . modules : <EOL> __import__ ( module_name ) </s>
