<s> import sys <EOL> import utils <EOL> import os <EOL> import datetime <EOL> import tweepy1 as t <EOL> from tweepy1 . parsers import ModelParser <EOL> from tweepy1 import StreamListener <EOL> from tweepy1 import Stream <EOL> from optparse import OptionParser <EOL> import config as c <EOL> import nltk <EOL> from nltk . corpus import stopwords <EOL> def update_hashtags_stats ( hashtags_fd , json_tweet ) : <EOL> tweet = utils . extract_tweet_from_json ( json_tweet ) <EOL> tweet_terms = [ ] <EOL> if tweet is None or '<STR_LIT:#>' not in tweet : <EOL> return False <EOL> tokenizer = nltk . RegexpTokenizer ( '<STR_LIT>' ) <EOL> doc = tokenizer . tokenize ( tweet ) <EOL> for w_raw in doc : <EOL> if '<STR_LIT:#>' not in w_raw : <EOL> continue <EOL> w = ( w_raw . strip ( '<STR_LIT>' ) ) . lower ( ) <EOL> tweet_terms . append ( w ) <EOL> hashtags_fd . inc ( w ) <EOL> return True <EOL> def update_terms_stats ( terms_fd , json_tweet , lex ) : <EOL> tweet = utils . extract_tweet_from_json ( json_tweet ) <EOL> tweet_terms = [ ] <EOL> if tweet is None : <EOL> return False <EOL> tokenizer = nltk . RegexpTokenizer ( '<STR_LIT>' ) <EOL> doc = tokenizer . tokenize ( tweet ) <EOL> for w_raw in doc : <EOL> w = w_raw . strip ( '<STR_LIT>' ) <EOL> if not ( w . strip ( '<STR_LIT:#>' ) ) . isalpha ( ) : <EOL> w_aux = '<STR_LIT>' <EOL> for s in w : <EOL> if ord ( s ) < <NUM_LIT> : <EOL> w_aux += s <EOL> else : <EOL> break <EOL> w = w_aux <EOL> w = w . lower ( ) <EOL> if ( w not in stopwords . words ( '<STR_LIT>' ) and w not in set ( [ '<STR_LIT>' , '<STR_LIT:http>' , '<STR_LIT>' ] ) ) and len ( w ) in range ( <NUM_LIT:3> , <NUM_LIT:16> ) : <EOL> if w in lex : <EOL> continue <EOL> tweet_terms . append ( w ) <EOL> terms_fd . inc ( w ) <EOL> bigrams = nltk . bigrams ( tweet_terms ) <EOL> for b in bigrams : <EOL> if b [ <NUM_LIT:1> ] + "<STR_LIT:U+0020>" + b [ <NUM_LIT:0> ] in lex or b [ <NUM_LIT:0> ] + "<STR_LIT:U+0020>" + b [ <NUM_LIT:1> ] in lex : <EOL> continue <EOL> if b [ <NUM_LIT:1> ] + "<STR_LIT:U+0020>" + b [ <NUM_LIT:0> ] in terms_fd : <EOL> terms_fd . inc ( b [ <NUM_LIT:1> ] + "<STR_LIT:U+0020>" + b [ <NUM_LIT:0> ] ) <EOL> else : <EOL> terms_fd . inc ( b [ <NUM_LIT:0> ] + "<STR_LIT:U+0020>" + b [ <NUM_LIT:1> ] ) <EOL> return True <EOL> class AdaptiveListener ( StreamListener ) : <EOL> output = None <EOL> adaptive = False <EOL> start_time = None <EOL> end_time = None <EOL> terms = None <EOL> terms_fd = None <EOL> terms_no = <NUM_LIT:0> <EOL> lex_set = None <EOL> use_hashtags = True <EOL> def on_data ( self , data ) : <EOL> if self . output is None : <EOL> print data <EOL> return True <EOL> print >> self . output , data . strip ( ) <EOL> if self . adaptive is False : <EOL> return True <EOL> if self . adaptive : <EOL> if datetime . datetime . now ( ) > self . end_time : <EOL> self . terms = self . terms_fd . keys ( ) [ : self . terms_no ] <EOL> print "<STR_LIT>" <EOL> print self . terms <EOL> return False <EOL> if self . use_hashtags : <EOL> update_hashtags_stats ( self . terms_fd , data ) <EOL> else : <EOL> update_terms_stats ( self . terms_fd , data , self . lex_set ) <EOL> return True <EOL> def on_error ( self , status ) : <EOL> if status == <NUM_LIT> : <EOL> print status , "<STR_LIT>" <EOL> elif status == <NUM_LIT> : <EOL> print status , "<STR_LIT>" <EOL> else : <EOL> print status <EOL> def set_output ( self , output_json ) : <EOL> self . output = output_json <EOL> def set_adaptive ( self , lex , learning_time = <NUM_LIT:3> , use_hashtags = True , new_terms_no = <NUM_LIT:10> ) : <EOL> self . adaptive = True <EOL> self . lex_set = set ( lex ) <EOL> self . terms_no = new_terms_no <EOL> self . use_hashtags = use_hashtags <EOL> self . start_time = datetime . datetime . now ( ) <EOL> self . end_time = self . start_time + datetime . timedelta ( hours = learning_time ) <EOL> print "<STR_LIT>" % ( self . start_time , self . end_time ) <EOL> self . terms_fd = nltk . FreqDist ( ) <EOL> self . terms = [ ] <EOL> if __name__ == "<STR_LIT:__main__>" : <EOL> parser = OptionParser ( ) <EOL> parser . add_option ( "<STR_LIT>" , "<STR_LIT>" , dest = "<STR_LIT>" , <EOL> help = "<STR_LIT>" , <EOL> metavar = "<STR_LIT>" , default = "<STR_LIT>" ) <EOL> parser . add_option ( "<STR_LIT>" , "<STR_LIT>" , dest = "<STR_LIT>" , <EOL> help = "<STR_LIT>" , <EOL> metavar = "<STR_LIT>" , default = "<STR_LIT>" ) <EOL> parser . add_option ( "<STR_LIT>" , "<STR_LIT>" , dest = "<STR_LIT:filename>" , <EOL> help = "<STR_LIT>" , <EOL> metavar = "<STR_LIT>" , default = "<STR_LIT>" ) <EOL> parser . add_option ( "<STR_LIT>" , "<STR_LIT>" , dest = "<STR_LIT>" , <EOL> help = "<STR_LIT>" , <EOL> metavar = "<STR_LIT>" , type = int , default = <NUM_LIT:1> ) <EOL> parser . add_option ( "<STR_LIT>" , "<STR_LIT>" , dest = "<STR_LIT>" , <EOL> help = "<STR_LIT>" , <EOL> action = "<STR_LIT:store_true>" , <EOL> default = False ) <EOL> parser . add_option ( "<STR_LIT>" , "<STR_LIT>" , dest = "<STR_LIT>" , <EOL> help = "<STR_LIT>" , <EOL> default = <NUM_LIT:1> ) <EOL> parser . add_option ( "<STR_LIT>" , "<STR_LIT>" , dest = "<STR_LIT>" , <EOL> help = "<STR_LIT>" , <EOL> type = int , default = <NUM_LIT:5> ) <EOL> parser . add_option ( "<STR_LIT>" , "<STR_LIT>" , dest = "<STR_LIT>" , <EOL> help = "<STR_LIT>" , <EOL> metavar = "<STR_LIT>" , default = "<STR_LIT>" ) <EOL> ( options , args ) = parser . parse_args ( ) <EOL> if len ( options . proxy ) != <NUM_LIT:0> : <EOL> os . environ [ '<STR_LIT>' ] = options . proxy <EOL> os . environ [ '<STR_LIT>' ] = options . proxy <EOL> auth = t . OAuthHandler ( c . CONSUMER_KEY , c . CONSUMER_SECRET ) <EOL> auth . set_access_token ( c . ACCESS_KEY , c . ACCESS_SECRET ) <EOL> api = t . API ( auth_handler = auth , parser = ModelParser ( ) ) <EOL> pl = AdaptiveListener ( ) <EOL> pl . set_output ( open ( options . filename , "<STR_LIT:w>" ) ) <EOL> if len ( options . lexicon ) == <NUM_LIT:0> : <EOL> sys . exit ( "<STR_LIT>" + <EOL> "<STR_LIT>" + <EOL> "<STR_LIT>" ) <EOL> try : <EOL> to_track = utils . get_query_terms ( open ( options . lexicon , "<STR_LIT:r>" ) ) <EOL> except Exception as e : <EOL> print "<STR_LIT>" <EOL> print e <EOL> exit ( <NUM_LIT:0> ) <EOL> if len ( options . additional ) > <NUM_LIT:0> : <EOL> additional_terms = utils . get_query_terms ( open ( options . additional , "<STR_LIT:r>" ) ) <EOL> to_track [ <NUM_LIT:0> : <NUM_LIT:0> ] = additional_terms <EOL> if options . adaptive : <EOL> pl . set_adaptive ( to_track , options . prf_time , ( options . hashtags == <NUM_LIT:1> ) , options . new_terms_no ) <EOL> stream = Stream ( auth , pl ) <EOL> stream . filter ( track = to_track [ <NUM_LIT:0> : <NUM_LIT> ] ) <EOL> print "<STR_LIT>" <EOL> if pl . adaptive : <EOL> assert pl . terms <EOL> to_track [ <NUM_LIT:0> : <NUM_LIT:0> ] = pl . terms <EOL> pl . adaptive = False <EOL> stream . filter ( track = to_track [ <NUM_LIT:0> : <NUM_LIT> ] ) </s>
