<s> """<STR_LIT>""" <EOL> import cPickle <EOL> import time <EOL> from datetime import date <EOL> from retools import global_connection <EOL> from retools . exc import CacheConfigurationError <EOL> from retools . lock import Lock <EOL> from retools . lock import LockTimeout <EOL> from retools . util import func_namespace <EOL> from retools . util import has_self_arg <EOL> from functools import wraps <EOL> class _NoneMarker ( object ) : <EOL> pass <EOL> NoneMarker = _NoneMarker ( ) <EOL> class CacheKey ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , region , namespace , key , today = None ) : <EOL> """<STR_LIT>""" <EOL> if not today : <EOL> today = str ( date . today ( ) ) <EOL> self . lock_key = '<STR_LIT>' % ( region , namespace , key ) <EOL> self . redis_key = '<STR_LIT>' % ( region , namespace , key ) <EOL> self . redis_hit_key = '<STR_LIT>' % ( <EOL> today , region , namespace , key ) <EOL> self . redis_miss_key = '<STR_LIT>' % ( <EOL> today , region , namespace , key ) <EOL> self . redis_keyset = '<STR_LIT>' % ( region , namespace ) <EOL> class CacheRegion ( object ) : <EOL> """<STR_LIT>""" <EOL> regions = { } <EOL> enabled = True <EOL> statistics = True <EOL> @ classmethod <EOL> def add_region ( cls , name , expires , redis_expiration = <NUM_LIT> * <NUM_LIT> * <NUM_LIT> * <NUM_LIT:7> ) : <EOL> """<STR_LIT>""" <EOL> cls . regions [ name ] = dict ( expires = expires , <EOL> redis_expiration = redis_expiration ) <EOL> @ classmethod <EOL> def _add_tracking ( cls , pipeline , region , namespace , key ) : <EOL> """<STR_LIT>""" <EOL> pipeline . sadd ( '<STR_LIT>' , region ) <EOL> pipeline . sadd ( '<STR_LIT>' % region , namespace ) <EOL> pipeline . sadd ( '<STR_LIT>' % ( region , namespace ) , key ) <EOL> @ classmethod <EOL> def invalidate ( cls , region ) : <EOL> """<STR_LIT>""" <EOL> redis = global_connection . redis <EOL> namespaces = redis . smembers ( '<STR_LIT>' % region ) <EOL> if not namespaces : <EOL> return None <EOL> longest_expire = max ( <EOL> [ x [ '<STR_LIT>' ] for x in CacheRegion . regions . values ( ) ] ) <EOL> new_created = time . time ( ) - longest_expire - <NUM_LIT> <EOL> for ns in namespaces : <EOL> cache_keyset_key = '<STR_LIT>' % ( region , ns ) <EOL> keys = set ( [ '<STR_LIT>' ] ) | redis . smembers ( cache_keyset_key ) <EOL> for key in keys : <EOL> cache_key = '<STR_LIT>' % ( region , ns , key ) <EOL> if not redis . exists ( cache_key ) : <EOL> redis . srem ( cache_keyset_key , key ) <EOL> else : <EOL> redis . hset ( cache_key , '<STR_LIT>' , new_created ) <EOL> @ classmethod <EOL> def load ( cls , region , namespace , key , regenerate = True , callable = None , <EOL> statistics = None ) : <EOL> """<STR_LIT>""" <EOL> if statistics is None : <EOL> statistics = cls . statistics <EOL> redis = global_connection . redis <EOL> now = time . time ( ) <EOL> region_settings = cls . regions [ region ] <EOL> expires = region_settings [ '<STR_LIT>' ] <EOL> redis_expiration = region_settings [ '<STR_LIT>' ] <EOL> keys = CacheKey ( region = region , namespace = namespace , key = key ) <EOL> if statistics : <EOL> p = redis . pipeline ( transaction = True ) <EOL> p . hgetall ( keys . redis_key ) <EOL> p . get ( keys . redis_hit_key ) <EOL> p . incr ( keys . redis_hit_key ) <EOL> results = p . execute ( ) <EOL> result , existing_hits = results [ <NUM_LIT:0> ] , results [ <NUM_LIT:1> ] <EOL> if existing_hits is None : <EOL> existing_hits = <NUM_LIT:0> <EOL> else : <EOL> existing_hits = int ( existing_hits ) <EOL> else : <EOL> result = redis . hgetall ( keys . redis_key ) <EOL> expired = True <EOL> if result and now - float ( result [ '<STR_LIT>' ] ) < expires : <EOL> expired = False <EOL> if ( result and not regenerate ) or not expired : <EOL> return cPickle . loads ( result [ '<STR_LIT:value>' ] ) <EOL> if not result and not regenerate : <EOL> return NoneMarker <EOL> if result and '<STR_LIT:value>' in result : <EOL> timeout = <NUM_LIT:0> <EOL> else : <EOL> timeout = <NUM_LIT> * <NUM_LIT> <EOL> try : <EOL> with Lock ( keys . lock_key , expires = expires , timeout = timeout ) : <EOL> result = redis . hgetall ( keys . redis_key ) <EOL> now = time . time ( ) <EOL> if result and '<STR_LIT:value>' in result and now - float ( result [ '<STR_LIT>' ] ) < expires : <EOL> return cPickle . loads ( result [ '<STR_LIT:value>' ] ) <EOL> value = callable ( ) <EOL> p = redis . pipeline ( transaction = True ) <EOL> p . hmset ( keys . redis_key , { '<STR_LIT>' : now , <EOL> '<STR_LIT:value>' : cPickle . dumps ( value ) } ) <EOL> p . expire ( keys . redis_key , redis_expiration ) <EOL> cls . _add_tracking ( p , region , namespace , key ) <EOL> if statistics : <EOL> p . getset ( keys . redis_hit_key , <NUM_LIT:0> ) <EOL> new_hits = int ( p . execute ( ) [ <NUM_LIT:0> ] ) <EOL> else : <EOL> p . execute ( ) <EOL> except LockTimeout : <EOL> if result : <EOL> return cPickle . loads ( result [ '<STR_LIT:value>' ] ) <EOL> else : <EOL> return NoneMarker <EOL> if not statistics : <EOL> return value <EOL> misses = new_hits - existing_hits <EOL> if misses : <EOL> p = redis . pipeline ( transaction = True ) <EOL> p . incr ( keys . redis_hit_key , amount = existing_hits ) <EOL> p . incr ( keys . redis_miss_key , amount = misses ) <EOL> p . execute ( ) <EOL> else : <EOL> redis . incr ( keys . redis_hit_key , amount = existing_hits ) <EOL> return value <EOL> def invalidate_region ( region ) : <EOL> """<STR_LIT>""" <EOL> CacheRegion . invalidate ( region ) <EOL> def invalidate_callable ( callable , * args ) : <EOL> """<STR_LIT>""" <EOL> redis = global_connection . redis <EOL> region = callable . _region <EOL> namespace = callable . _namespace <EOL> new_created = time . time ( ) - CacheRegion . regions [ region ] [ '<STR_LIT>' ] - <NUM_LIT> <EOL> if args : <EOL> try : <EOL> cache_key = "<STR_LIT:U+0020>" . join ( map ( str , args ) ) <EOL> except UnicodeEncodeError : <EOL> cache_key = "<STR_LIT:U+0020>" . join ( map ( unicode , args ) ) <EOL> redis . hset ( '<STR_LIT>' % ( region , namespace , cache_key ) , <EOL> '<STR_LIT>' , new_created ) <EOL> else : <EOL> cache_keyset_key = '<STR_LIT>' % ( region , namespace ) <EOL> keys = set ( [ '<STR_LIT>' ] ) | redis . smembers ( cache_keyset_key ) <EOL> p = redis . pipeline ( transaction = True ) <EOL> for key in keys : <EOL> p . hset ( '<STR_LIT>' % ( region , namespace , key ) , '<STR_LIT>' , <EOL> new_created ) <EOL> p . execute ( ) <EOL> return None <EOL> invalidate_function = invalidate_callable <EOL> def cache_region ( region , * deco_args , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> def decorate ( func ) : <EOL> namespace = func_namespace ( func , deco_args ) <EOL> skip_self = has_self_arg ( func ) <EOL> regenerate = kwargs . get ( '<STR_LIT>' , True ) <EOL> @ wraps ( func ) <EOL> def cached ( * args ) : <EOL> if region not in CacheRegion . regions : <EOL> raise CacheConfigurationError ( <EOL> '<STR_LIT>' % region ) <EOL> if not CacheRegion . enabled : <EOL> return func ( * args ) <EOL> if skip_self : <EOL> try : <EOL> cache_key = "<STR_LIT:U+0020>" . join ( map ( str , args [ <NUM_LIT:1> : ] ) ) <EOL> except UnicodeEncodeError : <EOL> cache_key = "<STR_LIT:U+0020>" . join ( map ( unicode , args [ <NUM_LIT:1> : ] ) ) <EOL> else : <EOL> try : <EOL> cache_key = "<STR_LIT:U+0020>" . join ( map ( str , args ) ) <EOL> except UnicodeEncodeError : <EOL> cache_key = "<STR_LIT:U+0020>" . join ( map ( unicode , args ) ) <EOL> def go ( ) : <EOL> return func ( * args ) <EOL> return CacheRegion . load ( region , namespace , cache_key , <EOL> regenerate = regenerate , callable = go ) <EOL> cached . _region = region <EOL> cached . _namespace = namespace <EOL> return cached <EOL> return decorate </s>
