#!/usr/bin/env python3

"""
Created on 6/26/18

@author: <NAME>

Last modified: 7/2/18
"""

import os
import warnings
import pickle as pkl
import unittest
from text_pipeline import TokenFilter
from text_pipeline import Tokenizer 
from text_pipeline import Stemmer
from text_pipeline import Pipeline

ROOT_PATH = os.path.dirname(os.path.realpath(__file__))

class TestPipeline(unittest.TestCase):
   
    def setUp(self):
        warnings.simplefilter('ignore')
        filename = 'test_docs.pkl'
        with open(ROOT_PATH + '/' + filename, 'rb') as f:
            self.test_docs = pkl.load(f)

    def test_spacy_stops(self):
        warnings.simplefilter('ignore')
        expected_output = [[':', 'look', 'like', '-', 'sentence']]
        docs = []
        docs.append(self.test_docs['stopwords_01'])
        params = {
                'keep_alpha': False,
                'keep_alpha_nums': False,
                'remove_stops': True,
                'remove_oov': False,
                'remove_nums': False,
                'remove_url': False,
                'remove_email': True,
                'remove_punct': False
                }
        
        t = Tokenizer('spacy')
        f = TokenFilter('spacy', **params)
        p = Pipeline(t, f)
        test_output = p.apply(docs)
        self.assertEqual(test_output, expected_output, 
                'Spacy did not remove stop words')

    def test_nltk_stops(self):
        warnings.simplefilter('ignore')
        expected_output = [['except', 'could', ':', 'look', 'like-this', 'sentence']]
        docs = []
        docs.append(self.test_docs['stopwords_01'])
        params = {
                'keep_alpha': False,
                'keep_alpha_nums': False,
                'remove_stops': True,
                'remove_oov': False,
                'remove_nums': False,
                'remove_url': False,
                'remove_email': True,
                'remove_punct': False
                }
        
        t = Tokenizer('nltk')
        f = TokenFilter('nltk', **params)
        p = Pipeline(t, f)
        test_output = p.apply(docs)
        self.assertEqual(test_output, expected_output, 
                'NLTK did not remove stop words')
    
    def test_nltk_stops_02(self):
        warnings.simplefilter('ignore')
        expected_output = [['except', 'could', ':', 'look', 'like', '-', 'sentence']]
        docs = []
        docs.append(self.test_docs['stopwords_01'])
        params = {
                'keep_alpha': False,
                'keep_alpha_nums': False,
                'remove_stops': True,
                'remove_oov': False,
                'remove_nums': False,
                'remove_url': False,
                'remove_email': True,
                'remove_punct': False
                }
        
        t = Tokenizer('spacy')
        f = TokenFilter('nltk', **params)
        p = Pipeline(t, f)
        test_output = p.apply(docs)
        self.assertEqual(test_output, expected_output, 
                'NLTK did not remove stop words')

    def test_spacy_oov(self):
        warnings.simplefilter('ignore')
        expected_output = [['this', 'sentence', 'has', 'only', 'english']]
        docs = []
        docs.append(self.test_docs['oov_01'])
        params = {
                'keep_alpha': False,
                'keep_alpha_nums': False,
                'remove_stops': False,
                'remove_oov': True,
                'remove_nums': False,
                'remove_url': False,
                'remove_email': True,
                'remove_punct': False
                }
        
        t = Tokenizer('spacy')
        f = TokenFilter('spacy', **params)
        p = Pipeline(t, f)
        test_output = p.apply(docs)
        self.assertEqual(test_output, expected_output, 
                'Spacy did not remove non English words')

    def test_nltk_oov(self):
        warnings.simplefilter('ignore')
        expected_output = [['this', 'sentence', 'only']]
        docs = []
        docs.append(self.test_docs['oov_01'])
        params = {
                'keep_alpha': False,
                'keep_alpha_nums': False,
                'remove_stops': False,
                'remove_oov': True,
                'remove_nums': False,
                'remove_url': False,
                'remove_email': True,
                'remove_punct': False
                }
        
        t = Tokenizer('nltk')
        f = TokenFilter('nltk', **params)
        p = Pipeline(t, f)
        test_output = p.apply(docs)
        self.assertEqual(test_output, expected_output, 
                'NLTK did not remove non English words')

    def test_spacy_numbers(self):
        warnings.simplefilter('ignore')
        expected_output = [['this', 'sentence', '$', 'has', 'no', 'numbers']]
        docs = []
        docs.append(self.test_docs['numbers_01'])
        params = {
                'keep_alpha': False,
                'keep_alpha_nums': False,
                'remove_stops': False,
                'remove_oov': False,
                'remove_nums': True,
                'remove_url': False,
                'remove_email': True,
                'remove_punct': False
                }
        
        t = Tokenizer('spacy')
        f = TokenFilter('spacy', **params)
        p = Pipeline(t, f)
        test_output = p.apply(docs)
        self.assertEqual(test_output, expected_output, 
                'Spacy did not remove numbers')

    def test_spacy_punct(self):
        warnings.simplefilter('ignore')
        expected_output = [['this', 'sentence', 'has', 'no', 'punctuation']]
        docs = []
        docs.append(self.test_docs['punct_01'])
        params = {
                'keep_alpha': False,
                'keep_alpha_nums': False,
                'remove_stops': False,
                'remove_oov': False,
                'remove_nums': False,
                'remove_url': False,
                'remove_email': True,
                'remove_punct': True
                }
        
        t = Tokenizer('spacy')
        f = TokenFilter('spacy', **params)
        p = Pipeline(t, f)
        test_output = p.apply(docs)
        self.assertEqual(test_output, expected_output, 
                'Spacy did not remove punctuation')

    def test_spacy_emails(self):
        warnings.simplefilter('ignore')
        expected_output = [['this', 'sentence', 'has', 'no', 'emails']]
        docs = []
        docs.append(self.test_docs['emails_01'])
        params = {
                'keep_alpha': False,
                'keep_alpha_nums': False,
                'remove_stops': False,
                'remove_oov': False,
                'remove_nums': False,
                'remove_url': False,
                'remove_email': True,
                'remove_punct': False
                }
        
        t = Tokenizer('spacy')
        f = TokenFilter('spacy', **params)
        p = Pipeline(t, f)
        test_output = p.apply(docs)
        self.assertEqual(test_output, expected_output, 
                'Spacy did not remove emails')

    def test_spacy_url(self):
        warnings.simplefilter('ignore')
        expected_output = [['this', 'sentence', 'has', 'no', 'urls']]
        docs = []
        docs.append(self.test_docs['url_01'])
        params = {
                'keep_alpha': False,
                'keep_alpha_nums': False,
                'remove_stops': False,
                'remove_oov': False,
                'remove_nums': False,
                'remove_url': True,
                'remove_email': False,
                'remove_punct': False
                }
        
        t = Tokenizer('spacy')
        f = TokenFilter('spacy', **params)
        p = Pipeline(t, f)
        test_output = p.apply(docs)
        self.assertEqual(test_output, expected_output, 
                'Spacy did not remove urls')

    def test_spacy_lemma(self):
        warnings.simplefilter('ignore')
        expected_output = [['run', 'not', 'not', 'cry', 'go', 'go', 'test']]
        docs = []
        docs.append(self.test_docs['stems_01'])
        params = {
                'keep_alpha': False,
                'keep_alpha_nums': False,
                'remove_stops': True,
                'remove_oov': False,
                'remove_nums': False,
                'remove_url': False,
                'remove_email': False,
                'remove_punct': False
                }
        
        t = Tokenizer('spacy')
        f = TokenFilter('spacy', **params)
        s = Stemmer('spacy')
        p = Pipeline(t, f, s)
        test_output = p.apply(docs)
        self.assertEqual(test_output, expected_output, 
                'Spacy did not lemmatize') 

    def test_nltk_snow_stemmer(self):
        warnings.simplefilter('ignore')
        expected_output = [['run', "n't", "n't", 'cri', 'go', 'gone', 'test']]
        docs = []
        docs.append(self.test_docs['stems_01'])
        params = {
                'keep_alpha': False,
                'keep_alpha_nums': False,
                'remove_stops': True,
                'remove_oov': False,
                'remove_nums': False,
                'remove_url': False,
                'remove_email': False,
                'remove_punct': False
                }
        
        t = Tokenizer('spacy')
        f = TokenFilter('spacy', **params)
        s = Stemmer('nltk', stemmer='snowball')
        p = Pipeline(t, f, s)
        test_output = p.apply(docs)
        self.assertEqual(test_output, expected_output, 
                'NLTK Snowball did not stem')

    def test_nltk_porter_stemmer(self):
        warnings.simplefilter('ignore')
        expected_output = [['run', "n't", "n't", 'cri', 'go', 'gone', 'test']]
        docs = []
        docs.append(self.test_docs['stems_01'])
        params = {
                'keep_alpha': False,
                'keep_alpha_nums': False,
                'remove_stops': True,
                'remove_oov': False,
                'remove_nums': False,
                'remove_url': False,
                'remove_email': False,
                'remove_punct': False
                }
        
        t = Tokenizer('spacy')
        f = TokenFilter('spacy', **params)
        s = Stemmer('nltk', stemmer='porter')
        p = Pipeline(t, f, s)
        test_output = p.apply(docs)
        self.assertEqual(test_output, expected_output, 
                'NLTK Porter did not stem')

    def test_nltk_lemmatizer(self):
        warnings.simplefilter('ignore')
        expected_output = [['running', "n't", "n't", 'cry', 'going', 'gone', 'tested']]
        docs = []
        docs.append(self.test_docs['stems_01'])
        params = {
                'keep_alpha': False,
                'keep_alpha_nums': False,
                'remove_stops': True,
                'remove_oov': False,
                'remove_nums': False,
                'remove_url': False,
                'remove_email': False,
                'remove_punct': False
                }
        
        t = Tokenizer('spacy')
        f = TokenFilter('spacy', **params)
        s = Stemmer('nltk', lemmatizer='wordnet')
        p = Pipeline(t, f, s)
        test_output = p.apply(docs)
        self.assertEqual(test_output, expected_output, 
                'NLTK lemmatiZZZZzer did not lemmatize')

    def test_remove_non_alphas_01(self):
        warnings.simplefilter('ignore')
        expected_output = [
                ['look', 'like', 'sentence'],
                ['sentence', 'numbers'],
                ['sentence', 'punctuation'],
                ['sentence', 'emails']
                ]
        docs = [
                self.test_docs['stopwords_01'],
                self.test_docs['numbers_01'],
                self.test_docs['punct_01'],
                self.test_docs['emails_01'],
                ]
        params = {
                'keep_alpha': True,
                'keep_alpha_nums': False,
                'remove_stops': True,
                'remove_oov': False,
                'remove_nums': False,
                'remove_url': False,
                'remove_email': False,
                'remove_punct': False
                }
        
        t = Tokenizer('spacy')
        f = TokenFilter('spacy', **params)
        p = Pipeline(t, f)
        test_output = p.apply(docs)
        self.assertEqual(test_output, expected_output) 
    
    def test_remove_non_alphas_02(self):
        warnings.simplefilter('ignore')
        expected_output = [
                ['look', 'sentence'],
                ['sentence', 'numbers'],
                ['sentence', 'punctuation'],
                ['sentence', 'john', 'paul', 'emails']
                ]
        docs = [
                self.test_docs['stopwords_01'],
                self.test_docs['numbers_01'],
                self.test_docs['punct_01'],
                self.test_docs['emails_01'],
                ]
        params = {
                'keep_alpha': True,
                'keep_alpha_nums': False,
                'remove_stops': True,
                'remove_oov': False,
                'remove_nums': False,
                'remove_url': False,
                'remove_email': False,
                'remove_punct': False
                }
        
        t = Tokenizer('nltk')
        f = TokenFilter('spacy', **params)
        p = Pipeline(t, f)
        test_output = p.apply(docs)
        self.assertEqual(test_output, expected_output) 

    def test_remove_non_alphas_03(self):
        warnings.simplefilter('ignore')
        expected_output = [
                ['look', 'like', 'sentence'],
                ['sentence', '59', '1', 'numbers'],
                ['sentence', 'punctuation'],
                ['sentence', 'emails']
                ]
        docs = [
                self.test_docs['stopwords_01'],
                self.test_docs['numbers_01'],
                self.test_docs['punct_01'],
                self.test_docs['emails_01'],
                ]
        params = {
                'keep_alpha': False,
                'keep_alpha_nums': True,
                'remove_stops': True,
                'remove_oov': False,
                'remove_nums': False,
                'remove_url': False,
                'remove_email': False,
                'remove_punct': False
                }
        
        t = Tokenizer('spacy')
        f = TokenFilter('spacy', **params)
        p = Pipeline(t, f)
        test_output = p.apply(docs)
        self.assertEqual(test_output, expected_output) 
    
    def test_remove_non_alphas_04(self):
        warnings.simplefilter('ignore')
        expected_output = [
                ['look', 'sentence'],
                ['sentence', '59', '1', 'numbers'],
                ['sentence', 'punctuation'],
                ['sentence', 'john', 'paul', 'emails']
                ]
        docs = [
                self.test_docs['stopwords_01'],
                self.test_docs['numbers_01'],
                self.test_docs['punct_01'],
                self.test_docs['emails_01'],
                ]
        params = {
                'keep_alpha': False,
                'keep_alpha_nums': True,
                'remove_stops': True,
                'remove_oov': False,
                'remove_nums': False,
                'remove_url': False,
                'remove_email': False,
                'remove_punct': False
                }
        
        t = Tokenizer('nltk')
        f = TokenFilter('spacy', **params)
        p = Pipeline(t, f)
        test_output = p.apply(docs)
        self.assertEqual(test_output, expected_output) 
   
    def test_freq_filter(self):
        pass



if __name__=="__main__":
    unittest.main()

#!/usr/bin/env python3

"""
Created on 6/26/18

@author: <NAME>

Last modified: 7/2/18
"""

import os
import warnings
import pickle as pkl
import unittest
from text_pipeline import TokenFilter
from text_pipeline import Tokenizer 
from text_pipeline import Stemmer
from text_pipeline import Pipeline

ROOT_PATH = os.path.dirname(os.path.realpath(__file__))

class TestPipeline(unittest.TestCase):
   
    def setUp(self):
        warnings.simplefilter('ignore')
        filename = 'test_docs.pkl'
        with open(ROOT_PATH + '/' + filename, 'rb') as f:
            self.test_docs = pkl.load(f)

    def test_spacy_stops(self):
        warnings.simplefilter('ignore')
        expected_output = [[':', 'look', 'like', '-', 'sentence']]
        docs = []
        docs.append(self.test_docs['stopwords_01'])
        params = {
                'keep_alpha': False,
                'keep_alpha_nums': False,
                'remove_stops': True,
                'remove_oov': False,
                'remove_nums': False,
                'remove_url': False,
                'remove_email': True,
                'remove_punct': False
                }
        
        t = Tokenizer('spacy')
        f = TokenFilter('spacy', **params)
        p = Pipeline(t, f)
        test_output = p.apply(docs)
        self.assertEqual(test_output, expected_output, 
                'Spacy did not remove stop words')

    def test_nltk_stops(self):
        warnings.simplefilter('ignore')
        expected_output = [['except', 'could', ':', 'look', 'like-this', 'sentence']]
        docs = []
        docs.append(self.test_docs['stopwords_01'])
        params = {
                'keep_alpha': False,
                'keep_alpha_nums': False,
                'remove_stops': True,
                'remove_oov': False,
                'remove_nums': False,
                'remove_url': False,
                'remove_email': True,
                'remove_punct': False
                }
        
        t = Tokenizer('nltk')
        f = TokenFilter('nltk', **params)
        p = Pipeline(t, f)
        test_output = p.apply(docs)
        self.assertEqual(test_output, expected_output, 
                'NLTK did not remove stop words')
    
    def test_nltk_stops_02(self):
        warnings.simplefilter('ignore')
        expected_output = [['except', 'could', ':', 'look', 'like', '-', 'sentence']]
        docs = []
        docs.append(self.test_docs['stopwords_01'])
        params = {
                'keep_alpha': False,
                'keep_alpha_nums': False,
                'remove_stops': True,
                'remove_oov': False,
                'remove_nums': False,
                'remove_url': False,
                'remove_email': True,
                'remove_punct': False
                }
        
        t = Tokenizer('spacy')
        f = TokenFilter('nltk', **params)
        p = Pipeline(t, f)
        test_output = p.apply(docs)
        self.assertEqual(test_output, expected_output, 
                'NLTK did not remove stop words')

    def test_spacy_oov(self):
        warnings.simplefilter('ignore')
        expected_output = [['this', 'sentence', 'has', 'only', 'english']]
        docs = []
        docs.append(self.test_docs['oov_01'])
        params = {
                'keep_alpha': False,
                'keep_alpha_nums': False,
                'remove_stops': False,
                'remove_oov': True,
                'remove_nums': False,
                'remove_url': False,
                'remove_email': True,
                'remove_punct': False
                }
        
        t = Tokenizer('spacy')
        f = TokenFilter('spacy', **params)
        p = Pipeline(t, f)
        test_output = p.apply(docs)
        self.assertEqual(test_output, expected_output, 
                'Spacy did not remove non English words')

    def test_nltk_oov(self):
        warnings.simplefilter('ignore')
        expected_output = [['this', 'sentence', 'only']]
        docs = []
        docs.append(self.test_docs['oov_01'])
        params = {
                'keep_alpha': False,
                'keep_alpha_nums': False,
                'remove_stops': False,
                'remove_oov': True,
                'remove_nums': False,
                'remove_url': False,
                'remove_email': True,
                'remove_punct': False
                }
        
        t = Tokenizer('nltk')
        f = TokenFilter('nltk', **params)
        p = Pipeline(t, f)
        test_output = p.apply(docs)
        self.assertEqual(test_output, expected_output, 
                'NLTK did not remove non English words')

    def test_spacy_numbers(self):
        warnings.simplefilter('ignore')
        expected_output = [['this', 'sentence', '$', 'has', 'no', 'numbers']]
        docs = []
        docs.append(self.test_docs['numbers_01'])
        params = {
                'keep_alpha': False,
                'keep_alpha_nums': False,
                'remove_stops': False,
                'remove_oov': False,
                'remove_nums': True,
                'remove_url': False,
                'remove_email': True,
                'remove_punct': False
                }
        
        t = Tokenizer('spacy')
        f = TokenFilter('spacy', **params)
        p = Pipeline(t, f)
        test_output = p.apply(docs)
        self.assertEqual(test_output, expected_output, 
                'Spacy did not remove numbers')

    def test_spacy_punct(self):
        warnings.simplefilter('ignore')
        expected_output = [['this', 'sentence', 'has', 'no', 'punctuation']]
        docs = []
        docs.append(self.test_docs['punct_01'])
        params = {
                'keep_alpha': False,
                'keep_alpha_nums': False,
                'remove_stops': False,
                'remove_oov': False,
                'remove_nums': False,
                'remove_url': False,
                'remove_email': True,
                'remove_punct': True
                }
        
        t = Tokenizer('spacy')
        f = TokenFilter('spacy', **params)
        p = Pipeline(t, f)
        test_output = p.apply(docs)
        self.assertEqual(test_output, expected_output, 
                'Spacy did not remove punctuation')

    def test_spacy_emails(self):
        warnings.simplefilter('ignore')
        expected_output = [['this', 'sentence', 'has', 'no', 'emails']]
        docs = []
        docs.append(self.test_docs['emails_01'])
        params = {
                'keep_alpha': False,
                'keep_alpha_nums': False,
                'remove_stops': False,
                'remove_oov': False,
                'remove_nums': False,
                'remove_url': False,
                'remove_email': True,
                'remove_punct': False
                }
        
        t = Tokenizer('spacy')
        f = TokenFilter('spacy', **params)
        p = Pipeline(t, f)
        test_output = p.apply(docs)
        self.assertEqual(test_output, expected_output, 
                'Spacy did not remove emails')

    def test_spacy_url(self):
        warnings.simplefilter('ignore')
        expected_output = [['this', 'sentence', 'has', 'no', 'urls']]
        docs = []
        docs.append(self.test_docs['url_01'])
        params = {
                'keep_alpha': False,
                'keep_alpha_nums': False,
                'remove_stops': False,
                'remove_oov': False,
                'remove_nums': False,
                'remove_url': True,
                'remove_email': False,
                'remove_punct': False
                }
        
        t = Tokenizer('spacy')
        f = TokenFilter('spacy', **params)
        p = Pipeline(t, f)
        test_output = p.apply(docs)
        self.assertEqual(test_output, expected_output, 
                'Spacy did not remove urls')

    def test_spacy_lemma(self):
        warnings.simplefilter('ignore')
        expected_output = [['run', 'not', 'not', 'cry', 'go', 'go', 'test']]
        docs = []
        docs.append(self.test_docs['stems_01'])
        params = {
                'keep_alpha': False,
                'keep_alpha_nums': False,
                'remove_stops': True,
                'remove_oov': False,
                'remove_nums': False,
                'remove_url': False,
                'remove_email': False,
                'remove_punct': False
                }
        
        t = Tokenizer('spacy')
        f = TokenFilter('spacy', **params)
        s = Stemmer('spacy')
        p = Pipeline(t, f, s)
        test_output = p.apply(docs)
        self.assertEqual(test_output, expected_output, 
                'Spacy did not lemmatize') 

    def test_nltk_snow_stemmer(self):
        warnings.simplefilter('ignore')
        expected_output = [['run', "n't", "n't", 'cri', 'go', 'gone', 'test']]
        docs = []
        docs.append(self.test_docs['stems_01'])
        params = {
                'keep_alpha': False,
                'keep_alpha_nums': False,
                'remove_stops': True,
                'remove_oov': False,
                'remove_nums': False,
                'remove_url': False,
                'remove_email': False,
                'remove_punct': False
                }
        
        t = Tokenizer('spacy')
        f = TokenFilter('spacy', **params)
        s = Stemmer('nltk', stemmer='snowball')
        p = Pipeline(t, f, s)
        test_output = p.apply(docs)
        self.assertEqual(test_output, expected_output, 
                'NLTK Snowball did not stem')

    def test_nltk_porter_stemmer(self):
        warnings.simplefilter('ignore')
        expected_output = [['run', "n't", "n't", 'cri', 'go', 'gone', 'test']]
        docs = []
        docs.append(self.test_docs['stems_01'])
        params = {
                'keep_alpha': False,
                'keep_alpha_nums': False,
                'remove_stops': True,
                'remove_oov': False,
                'remove_nums': False,
                'remove_url': False,
                'remove_email': False,
                'remove_punct': False
                }
        
        t = Tokenizer('spacy')
        f = TokenFilter('spacy', **params)
        s = Stemmer('nltk', stemmer='porter')
        p = Pipeline(t, f, s)
        test_output = p.apply(docs)
        self.assertEqual(test_output, expected_output, 
                'NLTK Porter did not stem')

    def test_nltk_lemmatizer(self):
        warnings.simplefilter('ignore')
        expected_output = [['running', "n't", "n't", 'cry', 'going', 'gone', 'tested']]
        docs = []
        docs.append(self.test_docs['stems_01'])
        params = {
                'keep_alpha': False,
                'keep_alpha_nums': False,
                'remove_stops': True,
                'remove_oov': False,
                'remove_nums': False,
                'remove_url': False,
                'remove_email': False,
                'remove_punct': False
                }
        
        t = Tokenizer('spacy')
        f = TokenFilter('spacy', **params)
        s = Stemmer('nltk', lemmatizer='wordnet')
        p = Pipeline(t, f, s)
        test_output = p.apply(docs)
        self.assertEqual(test_output, expected_output, 
                'NLTK lemmatiZZZZzer did not lemmatize')

    def test_remove_non_alphas_01(self):
        warnings.simplefilter('ignore')
        expected_output = [
                ['look', 'like', 'sentence'],
                ['sentence', 'numbers'],
                ['sentence', 'punctuation'],
                ['sentence', 'emails']
                ]
        docs = [
                self.test_docs['stopwords_01'],
                self.test_docs['numbers_01'],
                self.test_docs['punct_01'],
                self.test_docs['emails_01'],
                ]
        params = {
                'keep_alpha': True,
                'keep_alpha_nums': False,
                'remove_stops': True,
                'remove_oov': False,
                'remove_nums': False,
                'remove_url': False,
                'remove_email': False,
                'remove_punct': False
                }
        
        t = Tokenizer('spacy')
        f = TokenFilter('spacy', **params)
        p = Pipeline(t, f)
        test_output = p.apply(docs)
        self.assertEqual(test_output, expected_output) 
    
    def test_remove_non_alphas_02(self):
        warnings.simplefilter('ignore')
        expected_output = [
                ['look', 'sentence'],
                ['sentence', 'numbers'],
                ['sentence', 'punctuation'],
                ['sentence', 'john', 'paul', 'emails']
                ]
        docs = [
                self.test_docs['stopwords_01'],
                self.test_docs['numbers_01'],
                self.test_docs['punct_01'],
                self.test_docs['emails_01'],
                ]
        params = {
                'keep_alpha': True,
                'keep_alpha_nums': False,
                'remove_stops': True,
                'remove_oov': False,
                'remove_nums': False,
                'remove_url': False,
                'remove_email': False,
                'remove_punct': False
                }
        
        t = Tokenizer('nltk')
        f = TokenFilter('spacy', **params)
        p = Pipeline(t, f)
        test_output = p.apply(docs)
        self.assertEqual(test_output, expected_output) 

    def test_remove_non_alphas_03(self):
        warnings.simplefilter('ignore')
        expected_output = [
                ['look', 'like', 'sentence'],
                ['sentence', '59', '1', 'numbers'],
                ['sentence', 'punctuation'],
                ['sentence', 'emails']
                ]
        docs = [
                self.test_docs['stopwords_01'],
                self.test_docs['numbers_01'],
                self.test_docs['punct_01'],
                self.test_docs['emails_01'],
                ]
        params = {
                'keep_alpha': False,
                'keep_alpha_nums': True,
                'remove_stops': True,
                'remove_oov': False,
                'remove_nums': False,
                'remove_url': False,
                'remove_email': False,
                'remove_punct': False
                }
        
        t = Tokenizer('spacy')
        f = TokenFilter('spacy', **params)
        p = Pipeline(t, f)
        test_output = p.apply(docs)
        self.assertEqual(test_output, expected_output) 
    
    def test_remove_non_alphas_04(self):
        warnings.simplefilter('ignore')
        expected_output = [
                ['look', 'sentence'],
                ['sentence', '59', '1', 'numbers'],
                ['sentence', 'punctuation'],
                ['sentence', 'john', 'paul', 'emails']
                ]
        docs = [
                self.test_docs['stopwords_01'],
                self.test_docs['numbers_01'],
                self.test_docs['punct_01'],
                self.test_docs['emails_01'],
                ]
        params = {
                'keep_alpha': False,
                'keep_alpha_nums': True,
                'remove_stops': True,
                'remove_oov': False,
                'remove_nums': False,
                'remove_url': False,
                'remove_email': False,
                'remove_punct': False
                }
        
        t = Tokenizer('nltk')
        f = TokenFilter('spacy', **params)
        p = Pipeline(t, f)
        test_output = p.apply(docs)
        self.assertEqual(test_output, expected_output) 
   
    def test_freq_filter(self):
        pass



if __name__=="__main__":
    unittest.main()

#!/usr/bin/env python3 Created on 6/26/18  @author: <NAME>  Last modified: 7/2/18