<s> from . ignore import STOP_WORDS , PUNCTUATION_CHARS <EOL> from . metaphone import dm as double_metaphone <EOL> from . porter import PorterStemmer <EOL> class stopwords : <EOL> def __init__ ( self , stp = None ) : <EOL> self . stp = stp if stp is not None else STOP_WORDS <EOL> def __call__ ( self , words ) : <EOL> stp = self . stp <EOL> for word in words : <EOL> if word not in stp : <EOL> yield word <EOL> def metaphone_processor ( words ) : <EOL> '''<STR_LIT>''' <EOL> for word in words : <EOL> for w in double_metaphone ( word ) : <EOL> if w : <EOL> w = w . strip ( ) <EOL> if w : <EOL> yield w <EOL> def tolerant_metaphone_processor ( words ) : <EOL> '''<STR_LIT>''' <EOL> for word in words : <EOL> r = <NUM_LIT:0> <EOL> for w in double_metaphone ( word ) : <EOL> if w : <EOL> w = w . strip ( ) <EOL> if w : <EOL> r += <NUM_LIT:1> <EOL> yield w <EOL> if not r : <EOL> yield word <EOL> def stemming_processor ( words ) : <EOL> '''<STR_LIT>''' <EOL> stem = PorterStemmer ( ) . stem <EOL> for word in words : <EOL> word = stem ( word , <NUM_LIT:0> , len ( word ) - <NUM_LIT:1> ) <EOL> yield word </s>
