<s> """<STR_LIT>""" <EOL> from __future__ import unicode_literals <EOL> import re <EOL> from nltk . stem . api import StemmerI <EOL> from nltk . compat import python_2_unicode_compatible <EOL> @ python_2_unicode_compatible <EOL> class LancasterStemmer ( StemmerI ) : <EOL> """<STR_LIT>""" <EOL> rule_tuple = ( <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" <EOL> ) <EOL> def __init__ ( self ) : <EOL> """<STR_LIT>""" <EOL> self . rule_dictionary = { } <EOL> def parseRules ( self , rule_tuple ) : <EOL> """<STR_LIT>""" <EOL> valid_rule = re . compile ( "<STR_LIT>" ) <EOL> self . rule_dictionary = { } <EOL> for rule in rule_tuple : <EOL> if not valid_rule . match ( rule ) : <EOL> raise ValueError ( "<STR_LIT>" % rule ) <EOL> first_letter = rule [ <NUM_LIT:0> : <NUM_LIT:1> ] <EOL> if first_letter in self . rule_dictionary : <EOL> self . rule_dictionary [ first_letter ] . append ( rule ) <EOL> else : <EOL> self . rule_dictionary [ first_letter ] = [ rule ] <EOL> def stem ( self , word ) : <EOL> """<STR_LIT>""" <EOL> word = word . lower ( ) <EOL> intact_word = word <EOL> if len ( self . rule_dictionary ) == <NUM_LIT:0> : <EOL> self . parseRules ( LancasterStemmer . rule_tuple ) <EOL> return self . __doStemming ( word , intact_word ) <EOL> def __doStemming ( self , word , intact_word ) : <EOL> """<STR_LIT>""" <EOL> valid_rule = re . compile ( "<STR_LIT>" ) <EOL> proceed = True <EOL> while proceed : <EOL> last_letter_position = self . __getLastLetter ( word ) <EOL> if last_letter_position < <NUM_LIT:0> or word [ last_letter_position ] not in self . rule_dictionary : <EOL> proceed = False <EOL> else : <EOL> rule_was_applied = False <EOL> for rule in self . rule_dictionary [ word [ last_letter_position ] ] : <EOL> rule_match = valid_rule . match ( rule ) <EOL> if rule_match : <EOL> ( ending_string , <EOL> intact_flag , <EOL> remove_total , <EOL> append_string , <EOL> cont_flag ) = rule_match . groups ( ) <EOL> remove_total = int ( remove_total ) <EOL> if word . endswith ( ending_string [ : : - <NUM_LIT:1> ] ) : <EOL> if intact_flag : <EOL> if ( word == intact_word and <EOL> self . __isAcceptable ( word , remove_total ) ) : <EOL> word = self . __applyRule ( word , <EOL> remove_total , <EOL> append_string ) <EOL> rule_was_applied = True <EOL> if cont_flag == '<STR_LIT:.>' : <EOL> proceed = False <EOL> break <EOL> elif self . __isAcceptable ( word , remove_total ) : <EOL> word = self . __applyRule ( word , <EOL> remove_total , <EOL> append_string ) <EOL> rule_was_applied = True <EOL> if cont_flag == '<STR_LIT:.>' : <EOL> proceed = False <EOL> break <EOL> if rule_was_applied == False : <EOL> proceed = False <EOL> return word <EOL> def __getLastLetter ( self , word ) : <EOL> """<STR_LIT>""" <EOL> last_letter = - <NUM_LIT:1> <EOL> for position in range ( len ( word ) ) : <EOL> if word [ position ] . isalpha ( ) : <EOL> last_letter = position <EOL> else : <EOL> break <EOL> return last_letter <EOL> def __isAcceptable ( self , word , remove_total ) : <EOL> """<STR_LIT>""" <EOL> word_is_acceptable = False <EOL> if word [ <NUM_LIT:0> ] in "<STR_LIT>" : <EOL> if ( len ( word ) - remove_total >= <NUM_LIT:2> ) : <EOL> word_is_acceptable = True <EOL> elif ( len ( word ) - remove_total >= <NUM_LIT:3> ) : <EOL> if word [ <NUM_LIT:1> ] in "<STR_LIT>" : <EOL> word_is_acceptable = True <EOL> elif word [ <NUM_LIT:2> ] in "<STR_LIT>" : <EOL> word_is_acceptable = True <EOL> return word_is_acceptable <EOL> def __applyRule ( self , word , remove_total , append_string ) : <EOL> """<STR_LIT>""" <EOL> new_word_length = len ( word ) - remove_total <EOL> word = word [ <NUM_LIT:0> : new_word_length ] <EOL> if append_string : <EOL> word += append_string <EOL> return word <EOL> def __repr__ ( self ) : <EOL> return '<STR_LIT>' </s>
