def tokenize(self, value):
        """
        Split the incoming value into tokens and process each token,
        optionally stemming or running metaphone.

        :returns: A ``dict`` mapping token to score. The score is
            based on the relative frequency of the word in the
            document.
        """
        words = self.split_phrase(decode(value).lower())
        if self._stopwords:
            words = [w for w in words if w not in self._stopwords]
        if self._min_word_length:
            words = [w for w in words if len(w) >= self._min_word_length]

        fraction = 1. / (len(words) + 1)  # Prevent division by zero.

        # Apply optional transformations.
        if self._use_stemmer:
            words = self.stem(words)
        if self._use_metaphone:
            words = self.metaphone(words)

        scores = {}
        for word in words:
            scores.setdefault(word, 0)
            scores[word] += fraction
        return scores
def tokenize ( self , value ) : words = self . split_phrase ( decode ( value ) . lower ( ) ) if self . _stopwords : words = [ w for w in words if w not in self . _stopwords ] if self . _min_word_length : words = [ w for w in words if len ( w ) >= self . _min_word_length ] fraction = 1. / ( len ( words ) + 1 ) if self . _use_stemmer : words = self . stem ( words ) if self . _use_metaphone : words = self . metaphone ( words ) scores = { } for word in words : scores . setdefault ( word , 0 ) scores [ word ] += fraction return scores
Split the incoming value into tokens and process each token,
        optionally stemming or running metaphone.

        :returns: A ``dict`` mapping token to score. The score is
            based on the relative frequency of the word in the
            document.