<s> import json <EOL> from optparse import make_option <EOL> import sys <EOL> from socket import error as socket_error <EOL> import codecs <EOL> import unicodedata <EOL> import requests <EOL> from django . core . management . base import BaseCommand , CommandError <EOL> from ui . models import TwitterUser , TwitterUserItem , TwitterUserItemUrl <EOL> from ui . utils import make_date_aware <EOL> from queryset_iterator import queryset_iterator <EOL> class Command ( BaseCommand ) : <EOL> help = '<STR_LIT>' <EOL> option_list = BaseCommand . option_list + ( <EOL> make_option ( '<STR_LIT>' , action = '<STR_LIT:store>' , default = None , <EOL> type = '<STR_LIT:string>' , dest = '<STR_LIT>' , <EOL> help = '<STR_LIT>' ) , <EOL> make_option ( '<STR_LIT>' , action = '<STR_LIT:store>' , default = None , <EOL> type = '<STR_LIT:string>' , dest = '<STR_LIT>' , <EOL> help = '<STR_LIT>' ) , <EOL> make_option ( '<STR_LIT>' , action = '<STR_LIT:store>' , default = None , <EOL> type = '<STR_LIT:string>' , dest = '<STR_LIT>' , <EOL> help = '<STR_LIT>' ) , <EOL> make_option ( '<STR_LIT>' , action = '<STR_LIT:store>' , default = <NUM_LIT:0> , <EOL> type = '<STR_LIT:int>' , dest = '<STR_LIT>' , <EOL> help = '<STR_LIT>' ) , <EOL> make_option ( '<STR_LIT>' , action = '<STR_LIT:store_true>' , default = False , <EOL> help = '<STR_LIT>' ) , <EOL> ) <EOL> def handle ( self , * args , ** options ) : <EOL> twitter_user = None <EOL> start_dt = None <EOL> end_dt = None <EOL> if options [ '<STR_LIT>' ] : <EOL> try : <EOL> twitter_user = TwitterUser . objects . get ( <EOL> name = options [ '<STR_LIT>' ] ) <EOL> except TwitterUser . DoesNotExist : <EOL> raise CommandError ( '<STR_LIT>' % <EOL> options [ '<STR_LIT>' ] ) <EOL> if options [ '<STR_LIT>' ] : <EOL> start_dt = make_date_aware ( options [ '<STR_LIT>' ] ) <EOL> if not start_dt : <EOL> raise CommandError ( '<STR_LIT>' ) <EOL> else : <EOL> start_dt = None <EOL> if options [ '<STR_LIT>' ] : <EOL> end_dt = make_date_aware ( options [ '<STR_LIT>' ] ) <EOL> if not end_dt : <EOL> raise CommandError ( '<STR_LIT>' ) <EOL> else : <EOL> end_dt = None <EOL> if start_dt and end_dt : <EOL> if end_dt < start_dt : <EOL> raise CommandError ( '<STR_LIT>' ) <EOL> if twitter_user : <EOL> qs = twitter_user . items . all ( ) <EOL> else : <EOL> qs = TwitterUserItem . objects . all ( ) <EOL> if not options [ '<STR_LIT>' ] : <EOL> qs = qs . filter ( urls__isnull = True ) <EOL> if start_dt : <EOL> qs = qs . filter ( date_published__gte = start_dt ) <EOL> if end_dt : <EOL> qs = qs . filter ( date_published__lte = end_dt ) <EOL> qs = queryset_iterator ( qs ) <EOL> count = <NUM_LIT:0> <EOL> for tui in qs : <EOL> urls = [ ] <EOL> urls . extend ( tui . tweet [ '<STR_LIT>' ] [ '<STR_LIT>' ] ) <EOL> if '<STR_LIT>' in tui . tweet [ '<STR_LIT>' ] . keys ( ) : <EOL> urls . extend ( tui . tweet [ '<STR_LIT>' ] [ '<STR_LIT>' ] ) <EOL> if not urls : <EOL> for u in tui . links : <EOL> if ( '<STR_LIT>' in unicodedata . normalize ( '<STR_LIT>' , u ) . encode ( '<STR_LIT:ascii>' , '<STR_LIT:ignore>' ) <EOL> and tui . tweet [ '<STR_LIT>' ] > <NUM_LIT:0> ) : <EOL> continue <EOL> urls . append ( { '<STR_LIT:url>' : u , '<STR_LIT>' : u } ) <EOL> for url in urls : <EOL> try : <EOL> r = requests . head ( url [ '<STR_LIT>' ] , <EOL> allow_redirects = True , <EOL> timeout = <NUM_LIT:10> ) <EOL> if r . status_code == <NUM_LIT> : <EOL> r = requests . get ( url [ '<STR_LIT>' ] , <EOL> allow_redirects = True , <EOL> stream = True , timeout = <NUM_LIT:10> ) <EOL> r . close ( ) <EOL> req_history_headers = [ ] <EOL> for req in r . history : <EOL> req_headers = self . decode_headers ( req . headers , req . encoding ) <EOL> req_history_headers . append ( ( <EOL> req . status_code , <EOL> req . url , <EOL> req_headers ) ) <EOL> final_req_headers = self . decode_headers ( r . headers , r . encoding ) <EOL> tuiu = TwitterUserItemUrl ( <EOL> item = tui , <EOL> start_url = url [ '<STR_LIT:url>' ] , <EOL> expanded_url = url [ '<STR_LIT>' ] , <EOL> history = json . dumps ( req_history_headers ) , <EOL> final_url = r . url , <EOL> final_status = r . status_code , <EOL> final_headers = json . dumps ( final_req_headers ) , <EOL> duration_seconds = r . elapsed . total_seconds ( ) ) <EOL> tuiu . save ( ) <EOL> except ( requests . RequestException ) as e : <EOL> print ( "<STR_LIT>" % ( url [ '<STR_LIT>' ] . encode ( '<STR_LIT:utf-8>' ) , e ) ) <EOL> except ( requests . packages . urllib3 . exceptions . HTTPError ) as e : <EOL> print ( "<STR_LIT>" % ( url [ '<STR_LIT>' ] . encode ( '<STR_LIT:utf-8>' ) , e ) ) <EOL> except ( socket_error ) as e : <EOL> print ( "<STR_LIT>" % ( url [ '<STR_LIT>' ] . encode ( '<STR_LIT:utf-8>' ) , e ) ) <EOL> tuiu = TwitterUserItemUrl ( <EOL> item = tui , <EOL> start_url = url [ '<STR_LIT:url>' ] , <EOL> expanded_url = url [ '<STR_LIT>' ] , <EOL> final_url = url [ '<STR_LIT:url>' ] , <EOL> final_status = <NUM_LIT> ) <EOL> tuiu . save ( ) <EOL> if urls : <EOL> count += <NUM_LIT:1> <EOL> if options [ '<STR_LIT>' ] : <EOL> if count >= options [ '<STR_LIT>' ] : <EOL> sys . exit ( ) <EOL> def decode_headers ( self , headers , encoding ) : <EOL> try : <EOL> header_codec = codecs . lookup ( encoding <EOL> or '<STR_LIT:utf-8>' ) <EOL> except LookupError : <EOL> header_codec = codecs . lookup ( '<STR_LIT:utf-8>' ) <EOL> final_headers = { } <EOL> for k , v in headers . items ( ) : <EOL> final_headers . update ( { <EOL> header_codec . decode ( k ) , <EOL> header_codec . decode ( v ) } ) <EOL> return final_headers </s>
