<s> from click . testing import CliRunner <EOL> from textkit . tokenize . words import text2words <EOL> from textkit . tokenize . bigrams import words2bigrams <EOL> from textkit . tokenize . punc import text2punc <EOL> from textkit . tokenize . sentences import text2sentences <EOL> from textkit . tokenize . ngrams import words2ngrams <EOL> from tests . utils import create_single_output , create_multifile_output , compare_results <EOL> def test_text2words ( ) : <EOL> runner = CliRunner ( ) <EOL> with runner . isolated_filesystem ( ) : <EOL> filename = '<STR_LIT>' <EOL> sentence = '<STR_LIT>' <EOL> expected_tokens = [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT:!>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT:to>' , '<STR_LIT>' ] <EOL> create_single_output ( filename , sentence ) <EOL> result = runner . invoke ( text2words , [ filename ] ) <EOL> tokens = result . output . split ( '<STR_LIT:\n>' ) <EOL> assert result . exit_code == <NUM_LIT:0> <EOL> compare_results ( tokens , expected_tokens ) <EOL> def test_text2words_multifile ( ) : <EOL> runner = CliRunner ( ) <EOL> with runner . isolated_filesystem ( ) : <EOL> filenames = [ '<STR_LIT>' , '<STR_LIT>' ] <EOL> sentences = ( '<STR_LIT>' , <EOL> '<STR_LIT>' ) <EOL> expected_tokens = [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT:!>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT:to>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT:!>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] <EOL> create_multifile_output ( filenames , sentences ) <EOL> result = runner . invoke ( text2words , filenames ) <EOL> tokens = result . output . split ( '<STR_LIT:\n>' ) <EOL> assert result . exit_code == <NUM_LIT:0> <EOL> compare_results ( tokens , expected_tokens ) <EOL> def test_words2bigrams ( ) : <EOL> runner = CliRunner ( ) <EOL> with runner . isolated_filesystem ( ) : <EOL> filename = '<STR_LIT>' <EOL> sentence = '<STR_LIT>' <EOL> expected_tokens = [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] <EOL> create_single_output ( filename , sentence ) <EOL> result = runner . invoke ( words2bigrams , [ filename ] ) <EOL> tokens = result . output . split ( '<STR_LIT:\n>' ) <EOL> assert result . exit_code == <NUM_LIT:0> <EOL> compare_results ( tokens , expected_tokens ) <EOL> def test_sentences ( ) : <EOL> runner = CliRunner ( ) <EOL> with runner . isolated_filesystem ( ) : <EOL> filename = '<STR_LIT>' <EOL> sentence = '<STR_LIT>' <EOL> expected_tokens = [ '<STR_LIT>' , '<STR_LIT>' ] <EOL> create_single_output ( filename , sentence ) <EOL> result = runner . invoke ( text2sentences , [ filename ] ) <EOL> tokens = result . output . split ( '<STR_LIT:\n>' ) <EOL> assert result . exit_code == <NUM_LIT:0> <EOL> compare_results ( tokens , expected_tokens ) <EOL> def test_punc ( ) : <EOL> runner = CliRunner ( ) <EOL> with runner . isolated_filesystem ( ) : <EOL> filename = '<STR_LIT>' <EOL> sentence = '<STR_LIT>' <EOL> expected_tokens = [ '<STR_LIT:!>' , '<STR_LIT:U+002C>' , '<STR_LIT:.>' ] <EOL> create_single_output ( filename , sentence ) <EOL> result = runner . invoke ( text2punc , [ filename ] ) <EOL> tokens = result . output . split ( '<STR_LIT:\n>' ) <EOL> assert result . exit_code == <NUM_LIT:0> <EOL> compare_results ( tokens , expected_tokens ) <EOL> def test_punc_multifile ( ) : <EOL> runner = CliRunner ( ) <EOL> with runner . isolated_filesystem ( ) : <EOL> filenames = [ '<STR_LIT>' , '<STR_LIT>' ] <EOL> sentences = [ '<STR_LIT>' , <EOL> '<STR_LIT>' ] <EOL> expected_tokens = [ '<STR_LIT:!>' , '<STR_LIT:U+002C>' , '<STR_LIT:.>' , '<STR_LIT:!>' , '<STR_LIT>' , "<STR_LIT:'>" , '<STR_LIT:?>' ] <EOL> create_multifile_output ( filenames , sentences ) <EOL> result = runner . invoke ( text2punc , filenames ) <EOL> tokens = result . output . split ( '<STR_LIT:\n>' ) <EOL> assert result . exit_code == <NUM_LIT:0> <EOL> compare_results ( tokens , expected_tokens ) <EOL> def test_words2ngrams ( ) : <EOL> runner = CliRunner ( ) <EOL> with runner . isolated_filesystem ( ) : <EOL> filename = '<STR_LIT>' <EOL> sentence = '<STR_LIT>' <EOL> expected_tokens = [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] <EOL> create_single_output ( filename , sentence ) <EOL> result = runner . invoke ( words2ngrams , [ '<STR_LIT>' , <NUM_LIT:3> , filename ] ) <EOL> tokens = result . output . split ( '<STR_LIT:\n>' ) <EOL> assert result . exit_code == <NUM_LIT:0> <EOL> compare_results ( tokens , expected_tokens ) </s>
