<s> """<STR_LIT>""" <EOL> __title__ = '<STR_LIT>' <EOL> __author__ = '<STR_LIT>' <EOL> __license__ = '<STR_LIT>' <EOL> __copyright__ = '<STR_LIT>' <EOL> import os <EOL> import re <EOL> import string <EOL> from . utils import FileHelper <EOL> TABSSPACE = re . compile ( r'<STR_LIT>' ) <EOL> def innerTrim ( value ) : <EOL> if isinstance ( value , str ) : <EOL> value = re . sub ( TABSSPACE , '<STR_LIT:U+0020>' , value ) <EOL> value = '<STR_LIT>' . join ( value . splitlines ( ) ) <EOL> return value . strip ( ) <EOL> return '<STR_LIT>' <EOL> class WordStats ( object ) : <EOL> def __init__ ( self ) : <EOL> self . stop_word_count = <NUM_LIT:0> <EOL> self . word_count = <NUM_LIT:0> <EOL> self . stop_words = [ ] <EOL> def get_stop_words ( self ) : <EOL> return self . stop_words <EOL> def set_stop_words ( self , words ) : <EOL> self . stop_words = words <EOL> def get_stopword_count ( self ) : <EOL> return self . stop_word_count <EOL> def set_stopword_count ( self , wordcount ) : <EOL> self . stop_word_count = wordcount <EOL> def get_word_count ( self ) : <EOL> return self . word_count <EOL> def set_word_count ( self , cnt ) : <EOL> self . word_count = cnt <EOL> class StopWords ( object ) : <EOL> PUNCTUATION = re . compile ( <EOL> "<STR_LIT>" ) <EOL> TRANS_TABLE = str . maketrans ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> _cached_stop_words = { } <EOL> def __init__ ( self , language = '<STR_LIT>' ) : <EOL> if language not in self . _cached_stop_words : <EOL> path = os . path . join ( '<STR_LIT:text>' , '<STR_LIT>' % language ) <EOL> self . _cached_stop_words [ language ] = set ( FileHelper . loadResourceFile ( path ) . splitlines ( ) ) <EOL> self . STOP_WORDS = self . _cached_stop_words [ language ] <EOL> def remove_punctuation ( self , content ) : <EOL> content_is_unicode = isinstance ( content , str ) <EOL> if content_is_unicode : <EOL> content = content . encode ( '<STR_LIT:utf-8>' ) <EOL> trans_table = { ord ( c ) : None for c in string . punctuation } <EOL> stripped_input = content . decode ( '<STR_LIT:utf-8>' ) . translate ( trans_table ) <EOL> return stripped_input <EOL> def candidate_words ( self , stripped_input ) : <EOL> return stripped_input . split ( '<STR_LIT:U+0020>' ) <EOL> def get_stopword_count ( self , content ) : <EOL> if not content : <EOL> return WordStats ( ) <EOL> ws = WordStats ( ) <EOL> stripped_input = self . remove_punctuation ( content ) <EOL> candidate_words = self . candidate_words ( stripped_input ) <EOL> overlapping_stopwords = [ ] <EOL> c = <NUM_LIT:0> <EOL> for w in candidate_words : <EOL> c += <NUM_LIT:1> <EOL> if w . lower ( ) in self . STOP_WORDS : <EOL> overlapping_stopwords . append ( w . lower ( ) ) <EOL> ws . set_word_count ( c ) <EOL> ws . set_stopword_count ( len ( overlapping_stopwords ) ) <EOL> ws . set_stop_words ( overlapping_stopwords ) <EOL> return ws <EOL> class StopWordsChinese ( StopWords ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , language = '<STR_LIT>' ) : <EOL> super ( StopWordsChinese , self ) . __init__ ( language = '<STR_LIT>' ) <EOL> def candidate_words ( self , stripped_input ) : <EOL> import jieba <EOL> return jieba . cut ( stripped_input , cut_all = True ) <EOL> class StopWordsArabic ( StopWords ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , language = '<STR_LIT>' ) : <EOL> super ( StopWordsArabic , self ) . __init__ ( language = '<STR_LIT>' ) <EOL> def remove_punctuation ( self , content ) : <EOL> return content <EOL> def candidate_words ( self , stripped_input ) : <EOL> import nltk <EOL> s = nltk . stem . isri . ISRIStemmer ( ) <EOL> words = [ ] <EOL> for word in nltk . tokenize . wordpunct_tokenize ( stripped_input ) : <EOL> words . append ( s . stem ( word ) ) <EOL> return words <EOL> class StopWordsKorean ( StopWords ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , language = '<STR_LIT>' ) : <EOL> super ( StopWordsKorean , self ) . __init__ ( language = '<STR_LIT>' ) <EOL> def get_stopword_count ( self , content ) : <EOL> if not content : <EOL> return WordStats ( ) <EOL> ws = WordStats ( ) <EOL> stripped_input = self . remove_punctuation ( content ) <EOL> candidate_words = self . candidate_words ( stripped_input ) <EOL> overlapping_stopwords = [ ] <EOL> c = <NUM_LIT:0> <EOL> for w in candidate_words : <EOL> c += <NUM_LIT:1> <EOL> for stop_word in self . STOP_WORDS : <EOL> overlapping_stopwords . append ( stop_word ) <EOL> ws . set_word_count ( c ) <EOL> ws . set_stopword_count ( len ( overlapping_stopwords ) ) <EOL> ws . set_stop_words ( overlapping_stopwords ) <EOL> return ws </s>
