<s> """<STR_LIT>""" <EOL> import logging <EOL> import time <EOL> import urlparse <EOL> from birdy . twitter import AppClient <EOL> from birdy . twitter import TwitterRateLimitError , TwitterClientError <EOL> from delorean import parse , epoch <EOL> DEFAULT_MAX_PAGES = <NUM_LIT:10> <EOL> class TwitterPager ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , twitter_consumer_key , twitter_consumer_secret , <EOL> twitter_app_client_access_token , <EOL> default_max_pages = DEFAULT_MAX_PAGES ) : <EOL> self . _client = None <EOL> self . consumer_key = twitter_consumer_key <EOL> self . consumer_secret = twitter_consumer_secret <EOL> self . app_client_access_token = twitter_app_client_access_token <EOL> self . default_max_pages = default_max_pages <EOL> self . last_twitter_search = None <EOL> self . rate_limit_remaining = <NUM_LIT:1> <EOL> self . rate_limit_limit = None <EOL> self . rate_limit_reset = None <EOL> self . twitter_date = None <EOL> self . first_request = True <EOL> @ property <EOL> def client ( self ) : <EOL> if self . _client is None : <EOL> logging . debug ( '<STR_LIT>' ) <EOL> self . _client = AppClient ( <EOL> self . consumer_key , <EOL> self . consumer_secret , <EOL> access_token = self . app_client_access_token <EOL> ) <EOL> return self . _client <EOL> def reset_client ( self ) : <EOL> """<STR_LIT>""" <EOL> self . _client = None <EOL> def extract_rate_limit ( self , response ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> data = response . data [ '<STR_LIT>' ] [ '<STR_LIT>' ] [ '<STR_LIT>' ] <EOL> self . rate_limit_remaining = data [ '<STR_LIT>' ] <EOL> self . rate_limit_limit = data [ '<STR_LIT>' ] <EOL> self . rate_limit_reset = epoch ( data [ '<STR_LIT>' ] ) . datetime <EOL> except ( KeyError , TypeError ) : <EOL> self . rate_limit_remaining = int ( response . headers [ '<STR_LIT>' ] ) <EOL> self . rate_limit_limit = int ( response . headers [ '<STR_LIT>' ] ) <EOL> self . rate_limit_reset = epoch ( int ( response . headers [ '<STR_LIT>' ] ) ) . datetime <EOL> self . twitter_date = parse ( response . headers [ '<STR_LIT:date>' ] ) . datetime <EOL> logging . debug ( <EOL> '<STR_LIT>' '<STR_LIT>' % ( self . rate_limit_limit , <EOL> self . rate_limit_remaining , self . rate_limit_reset , self . twitter_date ) ) <EOL> def fetch_rate_limit ( self ) : <EOL> """<STR_LIT>""" <EOL> response = self . client . api . application . rate_limit_status . get ( <EOL> resources = '<STR_LIT>' ) <EOL> self . extract_rate_limit ( response ) <EOL> return { <EOL> '<STR_LIT>' : self . rate_limit_limit , <EOL> '<STR_LIT>' : self . rate_limit_remaining , <EOL> '<STR_LIT>' : self . rate_limit_reset <EOL> } <EOL> def wait_for_reset ( self ) : <EOL> """<STR_LIT>""" <EOL> t = ( self . rate_limit_reset - self . twitter_date ) . seconds + <NUM_LIT:1> <EOL> logging . info ( '<STR_LIT>' % t ) <EOL> time . sleep ( t ) <EOL> def search ( self , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> if self . first_request : <EOL> self . fetch_rate_limit ( ) <EOL> self . first_request = False <EOL> if self . rate_limit_remaining <= <NUM_LIT:0> : <EOL> logging . info ( '<STR_LIT>' ) <EOL> self . wait_for_reset ( ) <EOL> try : <EOL> response = self . client . api . statuses . user_timeline . get ( ** kwargs ) <EOL> logging . debug ( '<STR_LIT>' % str ( response ) ) <EOL> self . extract_rate_limit ( response ) <EOL> return response <EOL> except TwitterRateLimitError , e : <EOL> logging . warning ( '<STR_LIT>' ) <EOL> self . fetch_rate_limit ( ) <EOL> self . wait_for_reset ( ) <EOL> return self . search ( ** kwargs ) <EOL> except TwitterClientError , e : <EOL> if str ( e ) . startswith ( '<STR_LIT>' ) : <EOL> logging . debug ( '<STR_LIT>' ) <EOL> self . reset_client ( ) <EOL> return self . search ( ** kwargs ) <EOL> else : <EOL> raise e <EOL> def paginated_search ( self , page = <NUM_LIT:1> , page_handler = None , <EOL> max_pages = None , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> if max_pages is None : <EOL> max_pages = self . default_max_pages <EOL> response = self . search ( ** kwargs ) <EOL> if page_handler : <EOL> page_handler ( response ) <EOL> if page < max_pages : <EOL> try : <EOL> kwargs . update ( { k : v for k , v in urlparse . parse_qsl ( <EOL> response . data . search_metadata . next_results [ <NUM_LIT:1> : ] ) } ) <EOL> if int ( kwargs [ '<STR_LIT>' ] ) > int ( kwargs . get ( '<STR_LIT>' , <NUM_LIT:0> ) ) : <EOL> logging . debug ( '<STR_LIT>' % str ( kwargs ) ) <EOL> self . paginated_search ( page = page + <NUM_LIT:1> , <EOL> page_handler = page_handler , <EOL> max_pages = max_pages , ** kwargs ) <EOL> except AttributeError : <EOL> try : <EOL> kwargs [ '<STR_LIT>' ] = str ( response . data [ - <NUM_LIT:1> ] . id ) <EOL> logging . debug ( '<STR_LIT>' % str ( kwargs ) ) <EOL> self . paginated_search ( page = page + <NUM_LIT:1> , <EOL> page_handler = page_handler , <EOL> max_pages = max_pages , ** kwargs ) <EOL> except IndexError : <EOL> logging . debug ( '<STR_LIT>' ) <EOL> else : <EOL> logging . debug ( '<STR_LIT>' ) </s>
