def _get_words_from_dataset(dataset):
 def tokenize(words):
  if isinstance(words, basestring):
   return word_tokenize(words, include_punc=False)
  else:
   return words
 all_words = chain.from_iterable((tokenize(words) for (words, _) in dataset))
 return set(all_words)

def _get_words_from_dataset ( dataset ) : def tokenize ( words ) : if isinstance ( words , basestring ) : return word_tokenize ( words , include_punc = False ) else : return words all_words = chain . from_iterable ( ( tokenize ( words ) for ( words , _ ) in dataset ) ) return set ( all_words )
return a set of all words in a dataset .