<s> import unittest , os <EOL> from marmot . preprocessing . words_from_file import get_tokens <EOL> class WordsFromFileTests ( unittest . TestCase ) : <EOL> def setUp ( self ) : <EOL> self . interesting_tokens = set ( [ '<STR_LIT>' , '<STR_LIT>' ] ) <EOL> module_path = os . path . dirname ( __file__ ) <EOL> self . corpus_file = os . path . join ( module_path , '<STR_LIT>' ) <EOL> def test_words_from_file ( self ) : <EOL> token_generator = get_tokens ( self . corpus_file ) <EOL> token_set = set ( token_generator ) <EOL> self . assertTrue ( len ( token_set ) > <NUM_LIT:0> ) <EOL> for word in token_set : <EOL> self . assertTrue ( type ( word ) == unicode ) <EOL> if __name__ == '<STR_LIT:__main__>' : <EOL> unittest . main ( ) </s>
