<s> from __future__ import print_function <EOL> import os <EOL> import gzip <EOL> from six . moves import cPickle as pickle <EOL> from importlib import import_module <EOL> from time import time <EOL> from weakref import WeakKeyDictionary <EOL> from email . utils import mktime_tz , parsedate_tz <EOL> from w3lib . http import headers_raw_to_dict , headers_dict_to_raw <EOL> from scrapy . http import Headers , Response <EOL> from scrapy . responsetypes import responsetypes <EOL> from scrapy . utils . request import request_fingerprint <EOL> from scrapy . utils . project import data_path <EOL> from scrapy . utils . httpobj import urlparse_cached <EOL> from scrapy . utils . python import to_bytes , to_unicode <EOL> class DummyPolicy ( object ) : <EOL> def __init__ ( self , settings ) : <EOL> self . ignore_schemes = settings . getlist ( '<STR_LIT>' ) <EOL> self . ignore_http_codes = [ int ( x ) for x in settings . getlist ( '<STR_LIT>' ) ] <EOL> def should_cache_request ( self , request ) : <EOL> return urlparse_cached ( request ) . scheme not in self . ignore_schemes <EOL> def should_cache_response ( self , response , request ) : <EOL> return response . status not in self . ignore_http_codes <EOL> def is_cached_response_fresh ( self , response , request ) : <EOL> return True <EOL> def is_cached_response_valid ( self , cachedresponse , response , request ) : <EOL> return True <EOL> class RFC2616Policy ( object ) : <EOL> MAXAGE = <NUM_LIT> * <NUM_LIT> * <NUM_LIT> <EOL> def __init__ ( self , settings ) : <EOL> self . always_store = settings . getbool ( '<STR_LIT>' ) <EOL> self . ignore_schemes = settings . getlist ( '<STR_LIT>' ) <EOL> self . ignore_response_cache_controls = [ to_bytes ( cc ) for cc in <EOL> settings . getlist ( '<STR_LIT>' ) ] <EOL> self . _cc_parsed = WeakKeyDictionary ( ) <EOL> def _parse_cachecontrol ( self , r ) : <EOL> if r not in self . _cc_parsed : <EOL> cch = r . headers . get ( b'<STR_LIT>' , b'<STR_LIT>' ) <EOL> parsed = parse_cachecontrol ( cch ) <EOL> if isinstance ( r , Response ) : <EOL> for key in self . ignore_response_cache_controls : <EOL> parsed . pop ( key , None ) <EOL> self . _cc_parsed [ r ] = parsed <EOL> return self . _cc_parsed [ r ] <EOL> def should_cache_request ( self , request ) : <EOL> if urlparse_cached ( request ) . scheme in self . ignore_schemes : <EOL> return False <EOL> cc = self . _parse_cachecontrol ( request ) <EOL> if b'<STR_LIT>' in cc : <EOL> return False <EOL> return True <EOL> def should_cache_response ( self , response , request ) : <EOL> cc = self . _parse_cachecontrol ( response ) <EOL> if b'<STR_LIT>' in cc : <EOL> return False <EOL> elif response . status == <NUM_LIT> : <EOL> return False <EOL> elif self . always_store : <EOL> return True <EOL> elif b'<STR_LIT>' in cc or b'<STR_LIT>' in response . headers : <EOL> return True <EOL> elif response . status in ( <NUM_LIT> , <NUM_LIT> , <NUM_LIT> ) : <EOL> return True <EOL> elif response . status in ( <NUM_LIT:200> , <NUM_LIT> , <NUM_LIT> ) : <EOL> return b'<STR_LIT>' in response . headers or b'<STR_LIT>' in response . headers <EOL> else : <EOL> return False <EOL> def is_cached_response_fresh ( self , cachedresponse , request ) : <EOL> cc = self . _parse_cachecontrol ( cachedresponse ) <EOL> ccreq = self . _parse_cachecontrol ( request ) <EOL> if b'<STR_LIT>' in cc or b'<STR_LIT>' in ccreq : <EOL> return False <EOL> now = time ( ) <EOL> freshnesslifetime = self . _compute_freshness_lifetime ( cachedresponse , request , now ) <EOL> currentage = self . _compute_current_age ( cachedresponse , request , now ) <EOL> reqmaxage = self . _get_max_age ( ccreq ) <EOL> if reqmaxage is not None : <EOL> freshnesslifetime = min ( freshnesslifetime , reqmaxage ) <EOL> if currentage < freshnesslifetime : <EOL> return True <EOL> if b'<STR_LIT>' in ccreq and b'<STR_LIT>' not in cc : <EOL> staleage = ccreq [ b'<STR_LIT>' ] <EOL> if staleage is None : <EOL> return True <EOL> try : <EOL> if currentage < freshnesslifetime + max ( <NUM_LIT:0> , int ( staleage ) ) : <EOL> return True <EOL> except ValueError : <EOL> pass <EOL> self . _set_conditional_validators ( request , cachedresponse ) <EOL> return False <EOL> def is_cached_response_valid ( self , cachedresponse , response , request ) : <EOL> if response . status >= <NUM_LIT> : <EOL> cc = self . _parse_cachecontrol ( cachedresponse ) <EOL> if b'<STR_LIT>' not in cc : <EOL> return True <EOL> return response . status == <NUM_LIT> <EOL> def _set_conditional_validators ( self , request , cachedresponse ) : <EOL> if b'<STR_LIT>' in cachedresponse . headers : <EOL> request . headers [ b'<STR_LIT>' ] = cachedresponse . headers [ b'<STR_LIT>' ] <EOL> if b'<STR_LIT>' in cachedresponse . headers : <EOL> request . headers [ b'<STR_LIT>' ] = cachedresponse . headers [ b'<STR_LIT>' ] <EOL> def _get_max_age ( self , cc ) : <EOL> try : <EOL> return max ( <NUM_LIT:0> , int ( cc [ b'<STR_LIT>' ] ) ) <EOL> except ( KeyError , ValueError ) : <EOL> return None <EOL> def _compute_freshness_lifetime ( self , response , request , now ) : <EOL> cc = self . _parse_cachecontrol ( response ) <EOL> maxage = self . _get_max_age ( cc ) <EOL> if maxage is not None : <EOL> return maxage <EOL> date = rfc1123_to_epoch ( response . headers . get ( b'<STR_LIT>' ) ) or now <EOL> if b'<STR_LIT>' in response . headers : <EOL> expires = rfc1123_to_epoch ( response . headers [ b'<STR_LIT>' ] ) <EOL> return max ( <NUM_LIT:0> , expires - date ) if expires else <NUM_LIT:0> <EOL> lastmodified = rfc1123_to_epoch ( response . headers . get ( b'<STR_LIT>' ) ) <EOL> if lastmodified and lastmodified <= date : <EOL> return ( date - lastmodified ) / <NUM_LIT:10> <EOL> if response . status in ( <NUM_LIT> , <NUM_LIT> , <NUM_LIT> ) : <EOL> return self . MAXAGE <EOL> return <NUM_LIT:0> <EOL> def _compute_current_age ( self , response , request , now ) : <EOL> currentage = <NUM_LIT:0> <EOL> date = rfc1123_to_epoch ( response . headers . get ( b'<STR_LIT>' ) ) or now <EOL> if now > date : <EOL> currentage = now - date <EOL> if b'<STR_LIT>' in response . headers : <EOL> try : <EOL> age = int ( response . headers [ b'<STR_LIT>' ] ) <EOL> currentage = max ( currentage , age ) <EOL> except ValueError : <EOL> pass <EOL> return currentage <EOL> class DbmCacheStorage ( object ) : <EOL> def __init__ ( self , settings ) : <EOL> self . cachedir = data_path ( settings [ '<STR_LIT>' ] , createdir = True ) <EOL> self . expiration_secs = settings . getint ( '<STR_LIT>' ) <EOL> self . dbmodule = import_module ( settings [ '<STR_LIT>' ] ) <EOL> self . db = None <EOL> def open_spider ( self , spider ) : <EOL> dbpath = os . path . join ( self . cachedir , '<STR_LIT>' % spider . name ) <EOL> self . db = self . dbmodule . open ( dbpath , '<STR_LIT:c>' ) <EOL> def close_spider ( self , spider ) : <EOL> self . db . close ( ) <EOL> def retrieve_response ( self , spider , request ) : <EOL> data = self . _read_data ( spider , request ) <EOL> if data is None : <EOL> return <EOL> url = data [ '<STR_LIT:url>' ] <EOL> status = data [ '<STR_LIT:status>' ] <EOL> headers = Headers ( data [ '<STR_LIT>' ] ) <EOL> body = data [ '<STR_LIT:body>' ] <EOL> respcls = responsetypes . from_args ( headers = headers , url = url ) <EOL> response = respcls ( url = url , headers = headers , status = status , body = body ) <EOL> return response <EOL> def store_response ( self , spider , request , response ) : <EOL> key = self . _request_key ( request ) <EOL> data = { <EOL> '<STR_LIT:status>' : response . status , <EOL> '<STR_LIT:url>' : response . url , <EOL> '<STR_LIT>' : dict ( response . headers ) , <EOL> '<STR_LIT:body>' : response . body , <EOL> } <EOL> self . db [ '<STR_LIT>' % key ] = pickle . dumps ( data , protocol = <NUM_LIT:2> ) <EOL> self . db [ '<STR_LIT>' % key ] = str ( time ( ) ) <EOL> def _read_data ( self , spider , request ) : <EOL> key = self . _request_key ( request ) <EOL> db = self . db <EOL> tkey = '<STR_LIT>' % key <EOL> if tkey not in db : <EOL> return <EOL> ts = db [ tkey ] <EOL> if <NUM_LIT:0> < self . expiration_secs < time ( ) - float ( ts ) : <EOL> return <EOL> return pickle . loads ( db [ '<STR_LIT>' % key ] ) <EOL> def _request_key ( self , request ) : <EOL> return request_fingerprint ( request ) <EOL> class FilesystemCacheStorage ( object ) : <EOL> def __init__ ( self , settings ) : <EOL> self . cachedir = data_path ( settings [ '<STR_LIT>' ] ) <EOL> self . expiration_secs = settings . getint ( '<STR_LIT>' ) <EOL> self . use_gzip = settings . getbool ( '<STR_LIT>' ) <EOL> self . _open = gzip . open if self . use_gzip else open <EOL> def open_spider ( self , spider ) : <EOL> pass <EOL> def close_spider ( self , spider ) : <EOL> pass <EOL> def retrieve_response ( self , spider , request ) : <EOL> """<STR_LIT>""" <EOL> metadata = self . _read_meta ( spider , request ) <EOL> if metadata is None : <EOL> return <EOL> rpath = self . _get_request_path ( spider , request ) <EOL> with self . _open ( os . path . join ( rpath , '<STR_LIT>' ) , '<STR_LIT:rb>' ) as f : <EOL> body = f . read ( ) <EOL> with self . _open ( os . path . join ( rpath , '<STR_LIT>' ) , '<STR_LIT:rb>' ) as f : <EOL> rawheaders = f . read ( ) <EOL> url = metadata . get ( '<STR_LIT>' ) <EOL> status = metadata [ '<STR_LIT:status>' ] <EOL> headers = Headers ( headers_raw_to_dict ( rawheaders ) ) <EOL> respcls = responsetypes . from_args ( headers = headers , url = url ) <EOL> response = respcls ( url = url , headers = headers , status = status , body = body ) <EOL> return response <EOL> def store_response ( self , spider , request , response ) : <EOL> """<STR_LIT>""" <EOL> rpath = self . _get_request_path ( spider , request ) <EOL> if not os . path . exists ( rpath ) : <EOL> os . makedirs ( rpath ) <EOL> metadata = { <EOL> '<STR_LIT:url>' : request . url , <EOL> '<STR_LIT>' : request . method , <EOL> '<STR_LIT:status>' : response . status , <EOL> '<STR_LIT>' : response . url , <EOL> '<STR_LIT>' : time ( ) , <EOL> } <EOL> with self . _open ( os . path . join ( rpath , '<STR_LIT>' ) , '<STR_LIT:wb>' ) as f : <EOL> f . write ( to_bytes ( repr ( metadata ) ) ) <EOL> with self . _open ( os . path . join ( rpath , '<STR_LIT>' ) , '<STR_LIT:wb>' ) as f : <EOL> pickle . dump ( metadata , f , protocol = <NUM_LIT:2> ) <EOL> with self . _open ( os . path . join ( rpath , '<STR_LIT>' ) , '<STR_LIT:wb>' ) as f : <EOL> f . write ( headers_dict_to_raw ( response . headers ) ) <EOL> with self . _open ( os . path . join ( rpath , '<STR_LIT>' ) , '<STR_LIT:wb>' ) as f : <EOL> f . write ( response . body ) <EOL> with self . _open ( os . path . join ( rpath , '<STR_LIT>' ) , '<STR_LIT:wb>' ) as f : <EOL> f . write ( headers_dict_to_raw ( request . headers ) ) <EOL> with self . _open ( os . path . join ( rpath , '<STR_LIT>' ) , '<STR_LIT:wb>' ) as f : <EOL> f . write ( request . body ) <EOL> def _get_request_path ( self , spider , request ) : <EOL> key = request_fingerprint ( request ) <EOL> return os . path . join ( self . cachedir , spider . name , key [ <NUM_LIT:0> : <NUM_LIT:2> ] , key ) <EOL> def _read_meta ( self , spider , request ) : <EOL> rpath = self . _get_request_path ( spider , request ) <EOL> metapath = os . path . join ( rpath , '<STR_LIT>' ) <EOL> if not os . path . exists ( metapath ) : <EOL> return <EOL> mtime = os . stat ( metapath ) . st_mtime <EOL> if <NUM_LIT:0> < self . expiration_secs < time ( ) - mtime : <EOL> return <EOL> with self . _open ( metapath , '<STR_LIT:rb>' ) as f : <EOL> return pickle . load ( f ) <EOL> class LeveldbCacheStorage ( object ) : <EOL> def __init__ ( self , settings ) : <EOL> import leveldb <EOL> self . _leveldb = leveldb <EOL> self . cachedir = data_path ( settings [ '<STR_LIT>' ] , createdir = True ) <EOL> self . expiration_secs = settings . getint ( '<STR_LIT>' ) <EOL> self . db = None <EOL> def open_spider ( self , spider ) : <EOL> dbpath = os . path . join ( self . cachedir , '<STR_LIT>' % spider . name ) <EOL> self . db = self . _leveldb . LevelDB ( dbpath ) <EOL> def close_spider ( self , spider ) : <EOL> self . db . CompactRange ( ) <EOL> del self . db <EOL> def retrieve_response ( self , spider , request ) : <EOL> data = self . _read_data ( spider , request ) <EOL> if data is None : <EOL> return <EOL> url = data [ '<STR_LIT:url>' ] <EOL> status = data [ '<STR_LIT:status>' ] <EOL> headers = Headers ( data [ '<STR_LIT>' ] ) <EOL> body = data [ '<STR_LIT:body>' ] <EOL> respcls = responsetypes . from_args ( headers = headers , url = url ) <EOL> response = respcls ( url = url , headers = headers , status = status , body = body ) <EOL> return response <EOL> def store_response ( self , spider , request , response ) : <EOL> key = self . _request_key ( request ) <EOL> data = { <EOL> '<STR_LIT:status>' : response . status , <EOL> '<STR_LIT:url>' : response . url , <EOL> '<STR_LIT>' : dict ( response . headers ) , <EOL> '<STR_LIT:body>' : response . body , <EOL> } <EOL> batch = self . _leveldb . WriteBatch ( ) <EOL> batch . Put ( key + b'<STR_LIT>' , pickle . dumps ( data , protocol = <NUM_LIT:2> ) ) <EOL> batch . Put ( key + b'<STR_LIT>' , to_bytes ( str ( time ( ) ) ) ) <EOL> self . db . Write ( batch ) <EOL> def _read_data ( self , spider , request ) : <EOL> key = self . _request_key ( request ) <EOL> try : <EOL> ts = self . db . Get ( key + b'<STR_LIT>' ) <EOL> except KeyError : <EOL> return <EOL> if <NUM_LIT:0> < self . expiration_secs < time ( ) - float ( ts ) : <EOL> return <EOL> try : <EOL> data = self . db . Get ( key + b'<STR_LIT>' ) <EOL> except KeyError : <EOL> return <EOL> else : <EOL> return pickle . loads ( data ) <EOL> def _request_key ( self , request ) : <EOL> return to_bytes ( request_fingerprint ( request ) ) <EOL> def parse_cachecontrol ( header ) : <EOL> """<STR_LIT>""" <EOL> directives = { } <EOL> for directive in header . split ( b'<STR_LIT:U+002C>' ) : <EOL> key , sep , val = directive . strip ( ) . partition ( b'<STR_LIT:=>' ) <EOL> if key : <EOL> directives [ key . lower ( ) ] = val if sep else None <EOL> return directives <EOL> def rfc1123_to_epoch ( date_str ) : <EOL> try : <EOL> date_str = to_unicode ( date_str , encoding = '<STR_LIT:ascii>' ) <EOL> return mktime_tz ( parsedate_tz ( date_str ) ) <EOL> except Exception : <EOL> return None </s>
