<s> import csv <EOL> import os <EOL> import sys <EOL> import psycopg2 <EOL> import psycopg2 . extensions <EOL> import redis <EOL> import settings <EOL> import tempfile <EOL> import time <EOL> from tweepy . api import API <EOL> from tweepy . utils import convert_to_utf8_str <EOL> from tweepy . utils import import_simplejson <EOL> from tweepy . models import Status <EOL> api = API ( ) <EOL> json = import_simplejson ( ) <EOL> def gen_tuple ( jsontweet ) : <EOL> tweet = Status . parse ( api , json . loads ( jsontweet ) ) <EOL> retweeted = ( getattr ( tweet , '<STR_LIT>' , None ) != None ) <EOL> return ( tweet . author . id , tweet . created_at , convert_to_utf8_str ( tweet . text ) , retweeted ) <EOL> def bulk_load ( listkey , tweets , db ) : <EOL> print "<STR_LIT>" % ( len ( tweets ) , listkey ) <EOL> insert_cmd = "<STR_LIT>" <EOL> cur = db . cursor ( ) <EOL> cur . executemany ( insert_cmd , ( gen_tuple ( jsontweet ) for jsontweet in tweets ) ) <EOL> db . commit ( ) <EOL> def poll_data ( r , db ) : <EOL> listkey = r . rpoplpush ( settings . TO_INDEX , settings . CONSUMED_INDICES ) <EOL> if listkey != None : <EOL> tweets = r . lrange ( listkey , <NUM_LIT:0> , - <NUM_LIT:1> ) <EOL> bulk_load ( listkey , tweets , db ) <EOL> r . lrem ( settings . CONSUMED_INDICES , listkey , <NUM_LIT:0> ) <EOL> r . delete ( listkey ) <EOL> def main ( ) : <EOL> r = redis . Redis ( host = settings . REDIS_HOSTNAME , port = settings . REDIS_PORT , db = settings . REDIS_DB ) <EOL> conn_str = "<STR_LIT>" % ( settings . DATABASE_NAME , settings . DATABASE_USER , settings . DATABASE_HOST , settings . DATABASE_PASSWORD ) <EOL> db = psycopg2 . connect ( conn_str ) ; <EOL> db . set_isolation_level ( psycopg2 . extensions . ISOLATION_LEVEL_SERIALIZABLE ) <EOL> while True : <EOL> poll_data ( r , db ) <EOL> time . sleep ( settings . POLL_TIME ) <EOL> if __name__ == '<STR_LIT:__main__>' : <EOL> try : <EOL> main ( ) <EOL> except KeyboardInterrupt : <EOL> print '<STR_LIT>' </s>
