def _get_words_from_dataset(dataset):
    # Words may be either a string or a list of tokens. Return an iterator
    # of tokens accordingly
    def tokenize(words):
        if isinstance(words, basestring):
            return word_tokenize(words, include_punc=False)
        else:
            return words
    all_words = chain.from_iterable(tokenize(words) for words, _ in dataset)
    return set(all_words)
Return a set of all words in a dataset.
dataset/ETHPy150Open sloria/TextBlob/textblob/classifiers.py/_get_words_from_dataset