<s> import sys <EOL> import re <EOL> import json <EOL> import hashlib <EOL> from . . console_write import console_write <EOL> class CachingDownloader ( object ) : <EOL> """<STR_LIT>""" <EOL> def add_conditional_headers ( self , url , headers ) : <EOL> """<STR_LIT>""" <EOL> if not self . settings . get ( '<STR_LIT>' ) : <EOL> return headers <EOL> info_key = self . generate_key ( url , '<STR_LIT>' ) <EOL> info_json = self . settings [ '<STR_LIT>' ] . get ( info_key ) <EOL> if not info_json : <EOL> return headers <EOL> key = self . generate_key ( url ) <EOL> if not self . settings [ '<STR_LIT>' ] . has ( key ) : <EOL> return headers <EOL> try : <EOL> info = json . loads ( info_json . decode ( '<STR_LIT:utf-8>' ) ) <EOL> except ValueError : <EOL> return headers <EOL> etag = info . get ( '<STR_LIT>' ) <EOL> if etag : <EOL> headers [ '<STR_LIT>' ] = etag <EOL> last_modified = info . get ( '<STR_LIT>' ) <EOL> if last_modified : <EOL> headers [ '<STR_LIT>' ] = last_modified <EOL> return headers <EOL> def cache_result ( self , method , url , status , headers , content ) : <EOL> """<STR_LIT>""" <EOL> debug = self . settings . get ( '<STR_LIT>' , False ) <EOL> cache = self . settings . get ( '<STR_LIT>' ) <EOL> if not cache : <EOL> if debug : <EOL> console_write ( u"<STR_LIT>" , True ) <EOL> return content <EOL> if method . lower ( ) != '<STR_LIT>' : <EOL> if debug : <EOL> console_write ( u"<STR_LIT>" , True ) <EOL> return content <EOL> status = int ( status ) <EOL> if status not in [ <NUM_LIT:200> , <NUM_LIT> ] : <EOL> if debug : <EOL> console_write ( u"<STR_LIT>" , True ) <EOL> return content <EOL> key = self . generate_key ( url ) <EOL> if status == <NUM_LIT> : <EOL> cached_content = cache . get ( key ) <EOL> if cached_content : <EOL> if debug : <EOL> console_write ( u"<STR_LIT>" % ( url , cache . path ( key ) ) , True ) <EOL> return cached_content <EOL> return content <EOL> cache_control = headers . get ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> if cache_control : <EOL> fields = re . split ( '<STR_LIT>' , cache_control ) <EOL> for field in fields : <EOL> if field == '<STR_LIT>' : <EOL> return content <EOL> if headers . get ( '<STR_LIT>' ) in [ '<STR_LIT>' , '<STR_LIT>' ] : <EOL> if debug : <EOL> console_write ( u"<STR_LIT>" , True ) <EOL> return content <EOL> etag = headers . get ( '<STR_LIT>' ) <EOL> last_modified = headers . get ( '<STR_LIT>' ) <EOL> if not etag and not last_modified : <EOL> return content <EOL> struct = { '<STR_LIT>' : etag , '<STR_LIT>' : last_modified } <EOL> struct_json = json . dumps ( struct , indent = <NUM_LIT:4> ) <EOL> info_key = self . generate_key ( url , '<STR_LIT>' ) <EOL> if debug : <EOL> console_write ( u"<STR_LIT>" % ( url , cache . path ( key ) ) , True ) <EOL> cache . set ( info_key , struct_json . encode ( '<STR_LIT:utf-8>' ) ) <EOL> cache . set ( key , content ) <EOL> return content <EOL> def generate_key ( self , url , suffix = '<STR_LIT>' ) : <EOL> """<STR_LIT>""" <EOL> if sys . version_info >= ( <NUM_LIT:3> , ) or isinstance ( url , unicode ) : <EOL> url = url . encode ( '<STR_LIT:utf-8>' ) <EOL> key = hashlib . md5 ( url ) . hexdigest ( ) <EOL> return key + suffix <EOL> def retrieve_cached ( self , url ) : <EOL> """<STR_LIT>""" <EOL> key = self . generate_key ( url ) <EOL> cache = self . settings [ '<STR_LIT>' ] <EOL> if not cache . has ( key ) : <EOL> return False <EOL> if self . settings . get ( '<STR_LIT>' ) : <EOL> console_write ( u"<STR_LIT>" % ( url , cache . path ( key ) ) , True ) <EOL> return cache . get ( key ) </s>
