<s> import unittest <EOL> from nose . plugins . attrib import attr <EOL> from nose . tools import * <EOL> from textblob . tokenizers import WordTokenizer , SentenceTokenizer , word_tokenize , sent_tokenize <EOL> from textblob . compat import PY2 <EOL> def is_generator ( obj ) : <EOL> if PY2 : <EOL> return hasattr ( obj , '<STR_LIT>' ) <EOL> else : <EOL> return hasattr ( obj , '<STR_LIT>' ) <EOL> class TestWordTokenizer ( unittest . TestCase ) : <EOL> def setUp ( self ) : <EOL> self . tokenizer = WordTokenizer ( ) <EOL> self . text = "<STR_LIT>" <EOL> def tearDown ( self ) : <EOL> pass <EOL> def test_tokenize ( self ) : <EOL> assert_equal ( self . tokenizer . tokenize ( self . text ) , <EOL> [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT:a>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT:.>' ] ) <EOL> def test_exclude_punc ( self ) : <EOL> assert_equal ( self . tokenizer . tokenize ( self . text , include_punc = False ) , <EOL> [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT:a>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' ] ) <EOL> def test_itokenize ( self ) : <EOL> gen = self . tokenizer . itokenize ( self . text ) <EOL> assert_equal ( next ( gen ) , "<STR_LIT>" ) <EOL> assert_equal ( next ( gen ) , "<STR_LIT>" ) <EOL> def test_word_tokenize ( self ) : <EOL> tokens = word_tokenize ( self . text ) <EOL> assert_true ( is_generator ( tokens ) ) <EOL> assert_equal ( list ( tokens ) , self . tokenizer . tokenize ( self . text ) ) <EOL> class TestSentenceTokenizer ( unittest . TestCase ) : <EOL> def setUp ( self ) : <EOL> self . tokenizer = SentenceTokenizer ( ) <EOL> self . text = "<STR_LIT>" <EOL> def test_tokenize ( self ) : <EOL> assert_equal ( self . tokenizer . tokenize ( self . text ) , <EOL> [ "<STR_LIT>" , "<STR_LIT>" ] ) <EOL> @ attr ( "<STR_LIT>" ) <EOL> def test_tokenize_with_multiple_punctuation ( self ) : <EOL> text = "<STR_LIT>" <EOL> assert_equal ( self . tokenizer . tokenize ( text ) , <EOL> [ "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" ] ) <EOL> text2 = '<STR_LIT>' <EOL> tokens = self . tokenizer . tokenize ( text2 ) <EOL> assert_equal ( len ( tokens ) , <NUM_LIT:2> ) <EOL> assert_equal ( tokens , <EOL> [ "<STR_LIT>" , "<STR_LIT>" ] ) <EOL> def test_itokenize ( self ) : <EOL> gen = self . tokenizer . itokenize ( self . text ) <EOL> assert_equal ( next ( gen ) , "<STR_LIT>" ) <EOL> assert_equal ( next ( gen ) , "<STR_LIT>" ) <EOL> def test_sent_tokenize ( self ) : <EOL> tokens = sent_tokenize ( self . text ) <EOL> assert_true ( is_generator ( tokens ) ) <EOL> assert_equal ( list ( tokens ) , self . tokenizer . tokenize ( self . text ) ) <EOL> if __name__ == '<STR_LIT:__main__>' : <EOL> unittest . main ( ) </s>
