<s> from __future__ import division <EOL> from math import log , exp <EOL> from operator import mul <EOL> from collections import Counter <EOL> import os <EOL> import pylab <EOL> import cPickle <EOL> class MyDict ( dict ) : <EOL> def __getitem__ ( self , key ) : <EOL> if key in self : <EOL> return self . get ( key ) <EOL> return <NUM_LIT:0> <EOL> pos = MyDict ( ) <EOL> neg = MyDict ( ) <EOL> features = set ( ) <EOL> totals = [ <NUM_LIT:0> , <NUM_LIT:0> ] <EOL> delchars = '<STR_LIT>' . join ( c for c in map ( chr , range ( <NUM_LIT> ) ) if not c . isalnum ( ) ) <EOL> CDATA_FILE = "<STR_LIT>" <EOL> FDATA_FILE = "<STR_LIT>" <EOL> def negate_sequence ( text ) : <EOL> """<STR_LIT>""" <EOL> negation = False <EOL> delims = "<STR_LIT>" <EOL> result = [ ] <EOL> words = text . split ( ) <EOL> prev = None <EOL> pprev = None <EOL> for word in words : <EOL> stripped = word . strip ( delims ) . lower ( ) <EOL> negated = "<STR_LIT>" + stripped if negation else stripped <EOL> result . append ( negated ) <EOL> if prev : <EOL> bigram = prev + "<STR_LIT:U+0020>" + negated <EOL> result . append ( bigram ) <EOL> if pprev : <EOL> trigram = pprev + "<STR_LIT:U+0020>" + bigram <EOL> result . append ( trigram ) <EOL> pprev = prev <EOL> prev = negated <EOL> if any ( neg in word for neg in [ "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" ] ) : <EOL> negation = not negation <EOL> if any ( c in word for c in delims ) : <EOL> negation = False <EOL> return result <EOL> def train ( ) : <EOL> global pos , neg , totals <EOL> retrain = False <EOL> if not retrain and os . path . isfile ( CDATA_FILE ) : <EOL> pos , neg , totals = cPickle . load ( open ( CDATA_FILE ) ) <EOL> return <EOL> limit = <NUM_LIT> <EOL> for file in os . listdir ( "<STR_LIT>" ) [ : limit ] : <EOL> for word in set ( negate_sequence ( open ( "<STR_LIT>" + file ) . read ( ) ) ) : <EOL> pos [ word ] += <NUM_LIT:1> <EOL> neg [ '<STR_LIT>' + word ] += <NUM_LIT:1> <EOL> for file in os . listdir ( "<STR_LIT>" ) [ : limit ] : <EOL> for word in set ( negate_sequence ( open ( "<STR_LIT>" + file ) . read ( ) ) ) : <EOL> neg [ word ] += <NUM_LIT:1> <EOL> pos [ '<STR_LIT>' + word ] += <NUM_LIT:1> <EOL> prune_features ( ) <EOL> totals [ <NUM_LIT:0> ] = sum ( pos . values ( ) ) <EOL> totals [ <NUM_LIT:1> ] = sum ( neg . values ( ) ) <EOL> countdata = ( pos , neg , totals ) <EOL> cPickle . dump ( countdata , open ( CDATA_FILE , '<STR_LIT:w>' ) ) <EOL> def classify ( text ) : <EOL> words = set ( word for word in negate_sequence ( text ) if word in features ) <EOL> if ( len ( words ) == <NUM_LIT:0> ) : return True <EOL> pos_prob = sum ( log ( ( pos [ word ] + <NUM_LIT:1> ) / ( <NUM_LIT:2> * totals [ <NUM_LIT:0> ] ) ) for word in words ) <EOL> neg_prob = sum ( log ( ( neg [ word ] + <NUM_LIT:1> ) / ( <NUM_LIT:2> * totals [ <NUM_LIT:1> ] ) ) for word in words ) <EOL> return pos_prob > neg_prob <EOL> def classify2 ( text ) : <EOL> """<STR_LIT>""" <EOL> words = set ( word for word in negate_sequence ( text ) if word in pos or word in neg ) <EOL> if ( len ( words ) == <NUM_LIT:0> ) : return True <EOL> pos_prob = sum ( log ( ( pos [ word ] + <NUM_LIT:1> ) / ( <NUM_LIT:2> * totals [ <NUM_LIT:0> ] ) ) for word in words ) <EOL> neg_prob = sum ( log ( ( neg [ word ] + <NUM_LIT:1> ) / ( <NUM_LIT:2> * totals [ <NUM_LIT:1> ] ) ) for word in words ) <EOL> return pos_prob > neg_prob <EOL> def classify_demo ( text ) : <EOL> words = set ( word for word in negate_sequence ( text ) if word in pos or word in neg ) <EOL> if ( len ( words ) == <NUM_LIT:0> ) : <EOL> print "<STR_LIT>" <EOL> return True <EOL> pprob , nprob = <NUM_LIT:0> , <NUM_LIT:0> <EOL> for word in words : <EOL> pp = log ( ( pos [ word ] + <NUM_LIT:1> ) / ( <NUM_LIT:2> * totals [ <NUM_LIT:0> ] ) ) <EOL> np = log ( ( neg [ word ] + <NUM_LIT:1> ) / ( <NUM_LIT:2> * totals [ <NUM_LIT:1> ] ) ) <EOL> print "<STR_LIT>" % ( word , exp ( pp ) , exp ( np ) ) <EOL> pprob += pp <EOL> nprob += np <EOL> print ( "<STR_LIT>" if pprob > nprob else "<STR_LIT>" ) , "<STR_LIT>" % abs ( pprob - nprob ) <EOL> def MI ( word ) : <EOL> """<STR_LIT>""" <EOL> T = totals [ <NUM_LIT:0> ] + totals [ <NUM_LIT:1> ] <EOL> W = pos [ word ] + neg [ word ] <EOL> I = <NUM_LIT:0> <EOL> if W == <NUM_LIT:0> : <EOL> return <NUM_LIT:0> <EOL> if neg [ word ] > <NUM_LIT:0> : <EOL> I += ( totals [ <NUM_LIT:1> ] - neg [ word ] ) / T * log ( ( totals [ <NUM_LIT:1> ] - neg [ word ] ) * T / ( T - W ) / totals [ <NUM_LIT:1> ] ) <EOL> I += neg [ word ] / T * log ( neg [ word ] * T / W / totals [ <NUM_LIT:1> ] ) <EOL> if pos [ word ] > <NUM_LIT:0> : <EOL> I += ( totals [ <NUM_LIT:0> ] - pos [ word ] ) / T * log ( ( totals [ <NUM_LIT:0> ] - pos [ word ] ) * T / ( T - W ) / totals [ <NUM_LIT:0> ] ) <EOL> I += pos [ word ] / T * log ( pos [ word ] * T / W / totals [ <NUM_LIT:0> ] ) <EOL> return I <EOL> def get_relevant_features ( ) : <EOL> pos_dump = MyDict ( { k : pos [ k ] for k in pos if k in features } ) <EOL> neg_dump = MyDict ( { k : neg [ k ] for k in neg if k in features } ) <EOL> totals_dump = [ sum ( pos_dump . values ( ) ) , sum ( neg_dump . values ( ) ) ] <EOL> return ( pos_dump , neg_dump , totals_dump ) <EOL> def prune_features ( ) : <EOL> """<STR_LIT>""" <EOL> global pos , neg <EOL> for k in pos . keys ( ) : <EOL> if pos [ k ] <= <NUM_LIT:1> and neg [ k ] <= <NUM_LIT:1> : <EOL> del pos [ k ] <EOL> for k in neg . keys ( ) : <EOL> if neg [ k ] <= <NUM_LIT:1> and pos [ k ] <= <NUM_LIT:1> : <EOL> del neg [ k ] <EOL> def feature_selection_trials ( ) : <EOL> """<STR_LIT>""" <EOL> global pos , neg , totals , features <EOL> retrain = True <EOL> if not retrain and os . path . isfile ( FDATA_FILE ) : <EOL> pos , neg , totals = cPickle . load ( open ( FDATA_FILE ) ) <EOL> return <EOL> words = list ( set ( pos . keys ( ) + neg . keys ( ) ) ) <EOL> print "<STR_LIT>" , len ( words ) <EOL> words . sort ( key = lambda w : - MI ( w ) ) <EOL> num_features , accuracy = [ ] , [ ] <EOL> bestk = <NUM_LIT:0> <EOL> limit = <NUM_LIT> <EOL> path = "<STR_LIT>" <EOL> step = <NUM_LIT> <EOL> start = <NUM_LIT> <EOL> best_accuracy = <NUM_LIT:0.0> <EOL> for w in words [ : start ] : <EOL> features . add ( w ) <EOL> for k in xrange ( start , <NUM_LIT> , step ) : <EOL> for w in words [ k : k + step ] : <EOL> features . add ( w ) <EOL> correct = <NUM_LIT:0> <EOL> size = <NUM_LIT:0> <EOL> for file in os . listdir ( path + "<STR_LIT>" ) [ : limit ] : <EOL> correct += classify ( open ( path + "<STR_LIT>" + file ) . read ( ) ) == True <EOL> size += <NUM_LIT:1> <EOL> for file in os . listdir ( path + "<STR_LIT>" ) [ : limit ] : <EOL> correct += classify ( open ( path + "<STR_LIT>" + file ) . read ( ) ) == False <EOL> size += <NUM_LIT:1> <EOL> num_features . append ( k + step ) <EOL> accuracy . append ( correct / size ) <EOL> if ( correct / size ) > best_accuracy : <EOL> bestk = k <EOL> print k + step , correct / size <EOL> features = set ( words [ : bestk ] ) <EOL> cPickle . dump ( get_relevant_features ( ) , open ( FDATA_FILE , '<STR_LIT:w>' ) ) <EOL> pylab . plot ( num_features , accuracy ) <EOL> pylab . show ( ) <EOL> def test_pang_lee ( ) : <EOL> """<STR_LIT>""" <EOL> total , correct = <NUM_LIT:0> , <NUM_LIT:0> <EOL> for fname in os . listdir ( "<STR_LIT>" ) : <EOL> correct += int ( classify2 ( open ( "<STR_LIT>" + fname ) . read ( ) ) == True ) <EOL> total += <NUM_LIT:1> <EOL> for fname in os . listdir ( "<STR_LIT>" ) : <EOL> correct += int ( classify2 ( open ( "<STR_LIT>" + fname ) . read ( ) ) == False ) <EOL> total += <NUM_LIT:1> <EOL> print "<STR_LIT>" % ( correct / total ) <EOL> if __name__ == '<STR_LIT:__main__>' : <EOL> train ( ) <EOL> feature_selection_trials ( ) </s>
