def tokenize_fielded(self, tw):
        """ Tokenize tweet and prepend field id before each token.
        >>> t = Tokenizer(fields=['text', 'user.name', 'user.location'])
        >>> list(t.tokenize_fielded(Tweet(json.loads(data.test_tweet))))
        ['text___if', 'text___only', "text___bradley's", 'text___arm', 'text___was', 'text___longer', 'text___.', 'text___best', 'text___photo', 'text___ever', 'text___.', 'text___HASHTAG_oscars', 'text___THIS_IS_A_URL', 'user.name___ellen', 'user.name___degeneres']
        """
        toks = self.tokenize(tw)
        for field, tokens in sorted(toks.items()):
            for tok in tokens:
                yield '%s___%s' % (field, tok)
def tokenize_fielded ( self , tw ) : toks = self . tokenize ( tw ) for field , tokens in sorted ( toks . items ( ) ) : for tok in tokens : yield '%s___%s' % ( field , tok )
Tokenize tweet and prepend field id before each token.
        >>> t = Tokenizer(fields=['text', 'user.name', 'user.location'])
        >>> list(t.tokenize_fielded(Tweet(json.loads(data.test_tweet))))
        ['text___if', 'text___only', "text___bradley's", 'text___arm', 'text___was', 'text___longer', 'text___.', 'text___best', 'text___photo', 'text___ever', 'text___.', 'text___HASHTAG_oscars', 'text___THIS_IS_A_URL', 'user.name___ellen', 'user.name___degeneres']