def _normalize(self, tokens):
        """Normalization transform to apply to both dictionary words and input tokens."""
        if self.case_sensitive:
            return ' '.join(self.lexicon[t].normalized for t in tokens)
        else:
            return ' '.join(self.lexicon[t].lower for t in tokens)
def _normalize ( self , tokens ) : if self . case_sensitive : return ' ' . join ( self . lexicon [ t ] . normalized for t in tokens ) else : return ' ' . join ( self . lexicon [ t ] . lower for t in tokens )
Normalization transform to apply to both dictionary words and input tokens.