<s> '''<STR_LIT>''' <EOL> import os , math , re , sys , fnmatch , string <EOL> reload ( sys ) <EOL> def make_lex_dict ( f ) : <EOL> return dict ( map ( lambda ( w , m ) : ( w , float ( m ) ) , [ wmsr . strip ( ) . split ( '<STR_LIT:\t>' ) [ <NUM_LIT:0> : <NUM_LIT:2> ] for wmsr in open ( f ) ] ) ) <EOL> f = '<STR_LIT>' <EOL> try : <EOL> WORD_VALENCE_DICT = make_lex_dict ( f ) <EOL> except : <EOL> f = os . path . join ( os . path . dirname ( __file__ ) , '<STR_LIT>' ) <EOL> WORD_VALENCE_DICT = make_lex_dict ( f ) <EOL> B_INCR = <NUM_LIT> <EOL> B_DECR = - <NUM_LIT> <EOL> c_INCR = <NUM_LIT> <EOL> REGEX_REMOVE_PUNCTUATION = re . compile ( '<STR_LIT>' % re . escape ( string . punctuation ) ) <EOL> PUNC_LIST = [ "<STR_LIT:.>" , "<STR_LIT:!>" , "<STR_LIT:?>" , "<STR_LIT:U+002C>" , "<STR_LIT:;>" , "<STR_LIT::>" , "<STR_LIT:->" , "<STR_LIT:'>" , "<STR_LIT>" , <EOL> "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" ] <EOL> NEGATE = [ "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , <EOL> "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , <EOL> "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , <EOL> "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , <EOL> "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT:none>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , <EOL> "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , <EOL> "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , <EOL> "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" ] <EOL> BOOSTER_DICT = { "<STR_LIT>" : B_INCR , "<STR_LIT>" : B_INCR , "<STR_LIT>" : B_INCR , "<STR_LIT>" : B_INCR , "<STR_LIT>" : B_INCR , <EOL> "<STR_LIT>" : B_INCR , "<STR_LIT>" : B_INCR , "<STR_LIT>" : B_INCR , "<STR_LIT>" : B_INCR , <EOL> "<STR_LIT>" : B_INCR , "<STR_LIT>" : B_INCR , "<STR_LIT>" : B_INCR , "<STR_LIT>" : B_INCR , <EOL> "<STR_LIT>" : B_INCR , "<STR_LIT>" : B_INCR , "<STR_LIT>" : B_INCR , <EOL> "<STR_LIT>" : B_INCR , "<STR_LIT>" : B_INCR , "<STR_LIT>" : B_INCR , "<STR_LIT>" : B_INCR , "<STR_LIT>" : B_INCR , "<STR_LIT>" : B_INCR , <EOL> "<STR_LIT>" : B_INCR , "<STR_LIT>" : B_INCR , "<STR_LIT>" : B_INCR , "<STR_LIT>" : B_INCR , "<STR_LIT>" : B_INCR , <EOL> "<STR_LIT>" : B_INCR , "<STR_LIT>" : B_INCR , "<STR_LIT>" : B_INCR , "<STR_LIT>" : B_INCR , "<STR_LIT>" : B_INCR , <EOL> "<STR_LIT>" : B_INCR , "<STR_LIT>" : B_INCR , "<STR_LIT>" : B_INCR , "<STR_LIT>" : B_INCR , <EOL> "<STR_LIT>" : B_INCR , "<STR_LIT>" : B_INCR , <EOL> "<STR_LIT>" : B_INCR , "<STR_LIT>" : B_INCR , "<STR_LIT>" : B_INCR , <EOL> "<STR_LIT>" : B_INCR , "<STR_LIT>" : B_INCR , "<STR_LIT>" : B_INCR , "<STR_LIT>" : B_INCR , <EOL> "<STR_LIT>" : B_INCR , <EOL> "<STR_LIT>" : B_DECR , "<STR_LIT>" : B_DECR , "<STR_LIT>" : B_DECR , "<STR_LIT>" : B_DECR , <EOL> "<STR_LIT>" : B_DECR , "<STR_LIT>" : B_DECR , "<STR_LIT>" : B_DECR , "<STR_LIT>" : B_DECR , <EOL> "<STR_LIT>" : B_DECR , "<STR_LIT>" : B_DECR , "<STR_LIT>" : B_DECR , "<STR_LIT>" : B_DECR , "<STR_LIT>" : B_DECR , <EOL> "<STR_LIT>" : B_DECR , "<STR_LIT>" : B_DECR , "<STR_LIT>" : B_DECR , <EOL> "<STR_LIT>" : B_DECR , "<STR_LIT>" : B_DECR , "<STR_LIT>" : B_DECR , "<STR_LIT>" : B_DECR } <EOL> SPECIAL_CASE_IDIOMS = { "<STR_LIT>" : <NUM_LIT:3> , "<STR_LIT>" : <NUM_LIT:3> , "<STR_LIT>" : <NUM_LIT> , "<STR_LIT>" : - <NUM_LIT:2> , <EOL> "<STR_LIT>" : <NUM_LIT:2> , "<STR_LIT>" : - <NUM_LIT> , "<STR_LIT>" : - <NUM_LIT:2> } <EOL> def negated ( list , nWords = [ ] , includeNT = True ) : <EOL> nWords . extend ( NEGATE ) <EOL> for word in nWords : <EOL> if word in list : <EOL> return True <EOL> if includeNT : <EOL> for word in list : <EOL> if "<STR_LIT>" in word : <EOL> return True <EOL> if "<STR_LIT>" in list : <EOL> i = list . index ( "<STR_LIT>" ) <EOL> if i > <NUM_LIT:0> and list [ i - <NUM_LIT:1> ] != "<STR_LIT>" : <EOL> return True <EOL> return False <EOL> def normalize ( score , alpha = <NUM_LIT:15> ) : <EOL> normScore = score / math . sqrt ( ( ( score * score ) + alpha ) ) <EOL> return normScore <EOL> def wildCardMatch ( patternWithWildcard , listOfStringsToMatchAgainst ) : <EOL> listOfMatches = fnmatch . filter ( listOfStringsToMatchAgainst , patternWithWildcard ) <EOL> return listOfMatches <EOL> def isALLCAP_differential ( wordList ) : <EOL> countALLCAPS = <NUM_LIT:0> <EOL> for w in wordList : <EOL> if w . isupper ( ) : <EOL> countALLCAPS += <NUM_LIT:1> <EOL> cap_differential = len ( wordList ) - countALLCAPS <EOL> if cap_differential > <NUM_LIT:0> and cap_differential < len ( wordList ) : <EOL> isDiff = True <EOL> else : isDiff = False <EOL> return isDiff <EOL> def scalar_inc_dec ( word , valence , isCap_diff ) : <EOL> scalar = <NUM_LIT:0.0> <EOL> word_lower = word . lower ( ) <EOL> if word_lower in BOOSTER_DICT : <EOL> scalar = BOOSTER_DICT [ word_lower ] <EOL> if valence < <NUM_LIT:0> : scalar *= - <NUM_LIT:1> <EOL> if word . isupper ( ) and isCap_diff : <EOL> if valence > <NUM_LIT:0> : scalar += c_INCR <EOL> else : scalar -= c_INCR <EOL> return scalar <EOL> def sentiment ( text ) : <EOL> """<STR_LIT>""" <EOL> if not isinstance ( text , unicode ) and not isinstance ( text , str ) : <EOL> text = str ( text ) <EOL> wordsAndEmoticons = text . split ( ) <EOL> text_mod = REGEX_REMOVE_PUNCTUATION . sub ( '<STR_LIT>' , text ) <EOL> wordsOnly = text_mod . split ( ) <EOL> for word in wordsOnly : <EOL> if len ( word ) <= <NUM_LIT:1> : <EOL> wordsOnly . remove ( word ) <EOL> for word in wordsOnly : <EOL> for p in PUNC_LIST : <EOL> pword = p + word <EOL> x1 = wordsAndEmoticons . count ( pword ) <EOL> while x1 > <NUM_LIT:0> : <EOL> i = wordsAndEmoticons . index ( pword ) <EOL> wordsAndEmoticons . remove ( pword ) <EOL> wordsAndEmoticons . insert ( i , word ) <EOL> x1 = wordsAndEmoticons . count ( pword ) <EOL> wordp = word + p <EOL> x2 = wordsAndEmoticons . count ( wordp ) <EOL> while x2 > <NUM_LIT:0> : <EOL> i = wordsAndEmoticons . index ( wordp ) <EOL> wordsAndEmoticons . remove ( wordp ) <EOL> wordsAndEmoticons . insert ( i , word ) <EOL> x2 = wordsAndEmoticons . count ( wordp ) <EOL> for word in wordsAndEmoticons : <EOL> if len ( word ) <= <NUM_LIT:1> : <EOL> wordsAndEmoticons . remove ( word ) <EOL> isCap_diff = isALLCAP_differential ( wordsAndEmoticons ) <EOL> sentiments = [ ] <EOL> for item in wordsAndEmoticons : <EOL> v = <NUM_LIT:0> <EOL> i = wordsAndEmoticons . index ( item ) <EOL> if ( i < len ( wordsAndEmoticons ) - <NUM_LIT:1> and item . lower ( ) == "<STR_LIT>" and wordsAndEmoticons [ i + <NUM_LIT:1> ] . lower ( ) == "<STR_LIT>" ) or item . lower ( ) in BOOSTER_DICT : <EOL> sentiments . append ( v ) <EOL> continue <EOL> item_lowercase = item . lower ( ) <EOL> if item_lowercase in WORD_VALENCE_DICT : <EOL> v = float ( WORD_VALENCE_DICT [ item_lowercase ] ) <EOL> if item . isupper ( ) and isCap_diff : <EOL> if v > <NUM_LIT:0> : v += c_INCR <EOL> else : v -= c_INCR <EOL> n_scalar = - <NUM_LIT> <EOL> if i > <NUM_LIT:0> and wordsAndEmoticons [ i - <NUM_LIT:1> ] . lower ( ) not in WORD_VALENCE_DICT : <EOL> s1 = scalar_inc_dec ( wordsAndEmoticons [ i - <NUM_LIT:1> ] , v , isCap_diff ) <EOL> v = v + s1 <EOL> if negated ( [ wordsAndEmoticons [ i - <NUM_LIT:1> ] ] ) : v = v * n_scalar <EOL> if i > <NUM_LIT:1> and wordsAndEmoticons [ i - <NUM_LIT:2> ] . lower ( ) not in WORD_VALENCE_DICT : <EOL> s2 = scalar_inc_dec ( wordsAndEmoticons [ i - <NUM_LIT:2> ] , v , isCap_diff ) <EOL> if s2 != <NUM_LIT:0> : s2 = s2 * <NUM_LIT> <EOL> v = v + s2 <EOL> if wordsAndEmoticons [ i - <NUM_LIT:2> ] == "<STR_LIT>" and ( wordsAndEmoticons [ i - <NUM_LIT:1> ] == "<STR_LIT>" or wordsAndEmoticons [ i - <NUM_LIT:1> ] == "<STR_LIT>" ) : <EOL> v = v * <NUM_LIT> <EOL> elif negated ( [ wordsAndEmoticons [ i - <NUM_LIT:2> ] ] ) : v = v * n_scalar <EOL> if i > <NUM_LIT:2> and wordsAndEmoticons [ i - <NUM_LIT:3> ] . lower ( ) not in WORD_VALENCE_DICT : <EOL> s3 = scalar_inc_dec ( wordsAndEmoticons [ i - <NUM_LIT:3> ] , v , isCap_diff ) <EOL> if s3 != <NUM_LIT:0> : s3 = s3 * <NUM_LIT> <EOL> v = v + s3 <EOL> if wordsAndEmoticons [ i - <NUM_LIT:3> ] == "<STR_LIT>" and ( wordsAndEmoticons [ i - <NUM_LIT:2> ] == "<STR_LIT>" or wordsAndEmoticons [ i - <NUM_LIT:2> ] == "<STR_LIT>" ) or ( wordsAndEmoticons [ i - <NUM_LIT:1> ] == "<STR_LIT>" or wordsAndEmoticons [ i - <NUM_LIT:1> ] == "<STR_LIT>" ) : <EOL> v = v * <NUM_LIT> <EOL> elif negated ( [ wordsAndEmoticons [ i - <NUM_LIT:3> ] ] ) : v = v * n_scalar <EOL> onezero = u"<STR_LIT>" . format ( wordsAndEmoticons [ i - <NUM_LIT:1> ] , wordsAndEmoticons [ i ] ) <EOL> twoonezero = u"<STR_LIT>" . format ( wordsAndEmoticons [ i - <NUM_LIT:2> ] , wordsAndEmoticons [ i - <NUM_LIT:1> ] , wordsAndEmoticons [ i ] ) <EOL> twoone = u"<STR_LIT>" . format ( wordsAndEmoticons [ i - <NUM_LIT:2> ] , wordsAndEmoticons [ i - <NUM_LIT:1> ] ) <EOL> threetwoone = u"<STR_LIT>" . format ( wordsAndEmoticons [ i - <NUM_LIT:3> ] , wordsAndEmoticons [ i - <NUM_LIT:2> ] , wordsAndEmoticons [ i - <NUM_LIT:1> ] ) <EOL> threetwo = u"<STR_LIT>" . format ( wordsAndEmoticons [ i - <NUM_LIT:3> ] , wordsAndEmoticons [ i - <NUM_LIT:2> ] ) <EOL> if onezero in SPECIAL_CASE_IDIOMS : <EOL> v = SPECIAL_CASE_IDIOMS [ onezero ] <EOL> elif twoonezero in SPECIAL_CASE_IDIOMS : <EOL> v = SPECIAL_CASE_IDIOMS [ twoonezero ] <EOL> elif twoone in SPECIAL_CASE_IDIOMS : <EOL> v = SPECIAL_CASE_IDIOMS [ twoone ] <EOL> elif threetwoone in SPECIAL_CASE_IDIOMS : <EOL> v = SPECIAL_CASE_IDIOMS [ threetwoone ] <EOL> elif threetwo in SPECIAL_CASE_IDIOMS : <EOL> v = SPECIAL_CASE_IDIOMS [ threetwo ] <EOL> if len ( wordsAndEmoticons ) - <NUM_LIT:1> > i : <EOL> zeroone = u"<STR_LIT>" . format ( wordsAndEmoticons [ i ] , wordsAndEmoticons [ i + <NUM_LIT:1> ] ) <EOL> if zeroone in SPECIAL_CASE_IDIOMS : <EOL> v = SPECIAL_CASE_IDIOMS [ zeroone ] <EOL> if len ( wordsAndEmoticons ) - <NUM_LIT:1> > i + <NUM_LIT:1> : <EOL> zeroonetwo = u"<STR_LIT>" . format ( wordsAndEmoticons [ i ] , wordsAndEmoticons [ i + <NUM_LIT:1> ] , wordsAndEmoticons [ i + <NUM_LIT:2> ] ) <EOL> if zeroonetwo in SPECIAL_CASE_IDIOMS : <EOL> v = SPECIAL_CASE_IDIOMS [ zeroonetwo ] <EOL> if threetwo in BOOSTER_DICT or twoone in BOOSTER_DICT : <EOL> v = v + B_DECR <EOL> if i > <NUM_LIT:1> and wordsAndEmoticons [ i - <NUM_LIT:1> ] . lower ( ) not in WORD_VALENCE_DICT and wordsAndEmoticons [ i - <NUM_LIT:1> ] . lower ( ) == "<STR_LIT>" : <EOL> if ( wordsAndEmoticons [ i - <NUM_LIT:2> ] . lower ( ) != "<STR_LIT>" and wordsAndEmoticons [ i - <NUM_LIT:2> ] . lower ( ) != "<STR_LIT>" ) : <EOL> v = v * n_scalar <EOL> elif i > <NUM_LIT:0> and wordsAndEmoticons [ i - <NUM_LIT:1> ] . lower ( ) not in WORD_VALENCE_DICT and wordsAndEmoticons [ i - <NUM_LIT:1> ] . lower ( ) == "<STR_LIT>" : <EOL> v = v * n_scalar <EOL> sentiments . append ( v ) <EOL> if '<STR_LIT>' in wordsAndEmoticons or '<STR_LIT>' in wordsAndEmoticons : <EOL> try : bi = wordsAndEmoticons . index ( '<STR_LIT>' ) <EOL> except : bi = wordsAndEmoticons . index ( '<STR_LIT>' ) <EOL> for s in sentiments : <EOL> si = sentiments . index ( s ) <EOL> if si < bi : <EOL> sentiments . pop ( si ) <EOL> sentiments . insert ( si , s * <NUM_LIT:0.5> ) <EOL> elif si > bi : <EOL> sentiments . pop ( si ) <EOL> sentiments . insert ( si , s * <NUM_LIT> ) <EOL> if sentiments : <EOL> sum_s = float ( sum ( sentiments ) ) <EOL> ep_count = text . count ( "<STR_LIT:!>" ) <EOL> if ep_count > <NUM_LIT:4> : ep_count = <NUM_LIT:4> <EOL> ep_amplifier = ep_count * <NUM_LIT> <EOL> if sum_s > <NUM_LIT:0> : sum_s += ep_amplifier <EOL> elif sum_s < <NUM_LIT:0> : sum_s -= ep_amplifier <EOL> qm_count = text . count ( "<STR_LIT:?>" ) <EOL> qm_amplifier = <NUM_LIT:0> <EOL> if qm_count > <NUM_LIT:1> : <EOL> if qm_count <= <NUM_LIT:3> : qm_amplifier = qm_count * <NUM_LIT> <EOL> else : qm_amplifier = <NUM_LIT> <EOL> if sum_s > <NUM_LIT:0> : sum_s += qm_amplifier <EOL> elif sum_s < <NUM_LIT:0> : sum_s -= qm_amplifier <EOL> compound = normalize ( sum_s ) <EOL> pos_sum = <NUM_LIT:0.0> <EOL> neg_sum = <NUM_LIT:0.0> <EOL> neu_count = <NUM_LIT:0> <EOL> for sentiment_score in sentiments : <EOL> if sentiment_score > <NUM_LIT:0> : <EOL> pos_sum += ( float ( sentiment_score ) + <NUM_LIT:1> ) <EOL> if sentiment_score < <NUM_LIT:0> : <EOL> neg_sum += ( float ( sentiment_score ) - <NUM_LIT:1> ) <EOL> if sentiment_score == <NUM_LIT:0> : <EOL> neu_count += <NUM_LIT:1> <EOL> if pos_sum > math . fabs ( neg_sum ) : pos_sum += ( ep_amplifier + qm_amplifier ) <EOL> elif pos_sum < math . fabs ( neg_sum ) : neg_sum -= ( ep_amplifier + qm_amplifier ) <EOL> total = pos_sum + math . fabs ( neg_sum ) + neu_count <EOL> pos = math . fabs ( pos_sum / total ) <EOL> neg = math . fabs ( neg_sum / total ) <EOL> neu = math . fabs ( neu_count / total ) <EOL> else : <EOL> compound = <NUM_LIT:0.0> ; pos = <NUM_LIT:0.0> ; neg = <NUM_LIT:0.0> ; neu = <NUM_LIT:0.0> <EOL> s = { "<STR_LIT>" : round ( neg , <NUM_LIT:3> ) , <EOL> "<STR_LIT>" : round ( neu , <NUM_LIT:3> ) , <EOL> "<STR_LIT>" : round ( pos , <NUM_LIT:3> ) , <EOL> "<STR_LIT>" : round ( compound , <NUM_LIT:4> ) } <EOL> return s <EOL> if __name__ == '<STR_LIT:__main__>' : <EOL> sentences = [ <EOL> u"<STR_LIT>" , <EOL> u"<STR_LIT>" , <EOL> u"<STR_LIT>" , <EOL> u"<STR_LIT>" , <EOL> u"<STR_LIT>" , <EOL> u"<STR_LIT>" , <EOL> u"<STR_LIT>" , <EOL> u"<STR_LIT>" , <EOL> u"<STR_LIT>" , <EOL> u"<STR_LIT>" , <EOL> u"<STR_LIT>" , <EOL> u"<STR_LIT>" , <EOL> u"<STR_LIT>" , <EOL> u"<STR_LIT>" , <EOL> u"<STR_LIT>" , <EOL> u"<STR_LIT>" , <EOL> u"<STR_LIT>" <EOL> ] <EOL> paragraph = "<STR_LIT>" <EOL> from nltk import tokenize <EOL> lines_list = tokenize . sent_tokenize ( paragraph ) <EOL> sentences . extend ( lines_list ) <EOL> tricky_sentences = [ <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" <EOL> ] <EOL> sentences . extend ( tricky_sentences ) <EOL> for sentence in sentences : <EOL> print sentence <EOL> ss = sentiment ( sentence ) <EOL> print "<STR_LIT:\t>" + str ( ss ) <EOL> print "<STR_LIT>" </s>
