def __init__(self, min_df=1, max_df=0.9, tokenizer=LemmaTokenizer, hash=False):
        """
        `min_df` is set to filter out extremely rare words,
        since we don't want those to dominate the distance metric.

        `max_df` is set to filter out extremely common words,
        since they don't convey much information.
        """

        # Wrap the specified tokenizer
        t = Tokenizer(tokenizer())

        if hash:
            vectr = HashingVectorizer(input='content', stop_words='english', lowercase=True, tokenizer=t)
        else:
            vectr = CountVectorizer(input='content', stop_words='english', lowercase=True, tokenizer=t, min_df=min_df, max_df=max_df)

        args = [
            ('vectorizer', vectr),
            ('tfidf', TfidfTransformer(norm=None, use_idf=True, smooth_idf=True)),
            ('normalizer', Normalizer(copy=False))
        ]

        self.pipeline = Pipeline(args)
        self.trained = False
def __init__ ( self , min_df = 1 , max_df = 0.9 , tokenizer = LemmaTokenizer , hash = False ) : t = Tokenizer ( tokenizer ( ) ) if hash : vectr = HashingVectorizer ( input = 'content' , stop_words = 'english' , lowercase = True , tokenizer = t ) else : vectr = CountVectorizer ( input = 'content' , stop_words = 'english' , lowercase = True , tokenizer = t , min_df = min_df , max_df = max_df ) args = [ ( 'vectorizer' , vectr ) , ( 'tfidf' , TfidfTransformer ( norm = None , use_idf = True , smooth_idf = True ) ) , ( 'normalizer' , Normalizer ( copy = False ) ) ] self . pipeline = Pipeline ( args ) self . trained = False
`min_df` is set to filter out extremely rare words,
        since we don't want those to dominate the distance metric.

        `max_df` is set to filter out extremely common words,
        since they don't convey much information.