<s> import time <EOL> import pappyproxy <EOL> from . sortedcollection import SortedCollection <EOL> from twisted . internet import defer <EOL> class RequestCache ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , cache_size = <NUM_LIT:100> , cust_dbpool = None ) : <EOL> self . _cache_size = cache_size <EOL> if cache_size >= <NUM_LIT:100> : <EOL> RequestCache . _preload_limit = int ( cache_size * <NUM_LIT> ) <EOL> self . _cached_reqs = { } <EOL> self . _last_used = { } <EOL> self . _min_time = None <EOL> self . hits = <NUM_LIT:0> <EOL> self . misses = <NUM_LIT:0> <EOL> self . dbpool = cust_dbpool <EOL> self . _next_in_mem_id = <NUM_LIT:1> <EOL> self . _preload_limit = <NUM_LIT:10> <EOL> self . all_ids = set ( ) <EOL> self . unmangled_ids = set ( ) <EOL> self . ordered_ids = SortedCollection ( key = lambda x : - self . req_times [ x ] ) <EOL> self . inmem_reqs = set ( ) <EOL> self . req_times = { } <EOL> @ property <EOL> def hit_ratio ( self ) : <EOL> if self . hits == <NUM_LIT:0> and self . misses == <NUM_LIT:0> : <EOL> return <NUM_LIT:0> <EOL> return float ( self . hits ) / float ( self . hits + self . misses ) <EOL> def get_memid ( self ) : <EOL> i = '<STR_LIT>' % self . _next_in_mem_id <EOL> self . _next_in_mem_id += <NUM_LIT:1> <EOL> return i <EOL> @ defer . inlineCallbacks <EOL> def load_ids ( self ) : <EOL> if not self . dbpool : <EOL> self . dbpool = pappyproxy . http . dbpool <EOL> rows = yield self . dbpool . runQuery ( <EOL> """<STR_LIT>""" <EOL> ) <EOL> for row in rows : <EOL> if row [ <NUM_LIT:1> ] : <EOL> self . req_times [ str ( row [ <NUM_LIT:0> ] ) ] = row [ <NUM_LIT:1> ] <EOL> else : <EOL> self . req_times [ str ( row [ <NUM_LIT:0> ] ) ] = <NUM_LIT:0> <EOL> if str ( row [ <NUM_LIT:0> ] ) not in self . all_ids : <EOL> self . ordered_ids . insert ( str ( row [ <NUM_LIT:0> ] ) ) <EOL> self . all_ids . add ( str ( row [ <NUM_LIT:0> ] ) ) <EOL> rows = yield self . dbpool . runQuery ( <EOL> """<STR_LIT>""" <EOL> ) <EOL> for row in rows : <EOL> self . unmangled_ids . add ( str ( row [ <NUM_LIT:0> ] ) ) <EOL> def resize ( self , size ) : <EOL> if size >= self . _cache_size or size == - <NUM_LIT:1> : <EOL> self . _cache_size = size <EOL> else : <EOL> while len ( self . _cached_reqs ) > size : <EOL> self . _evict_single ( ) <EOL> self . _cache_size = size <EOL> @ defer . inlineCallbacks <EOL> def get ( self , reqid ) : <EOL> """<STR_LIT>""" <EOL> if self . check ( reqid ) : <EOL> self . _update_last_used ( reqid ) <EOL> self . hits += <NUM_LIT:1> <EOL> req = self . _cached_reqs [ reqid ] <EOL> defer . returnValue ( req ) <EOL> else : <EOL> self . misses += <NUM_LIT:1> <EOL> newreq = yield pappyproxy . http . Request . load_request ( reqid , use_cache = False ) <EOL> self . add ( newreq ) <EOL> defer . returnValue ( newreq ) <EOL> def check ( self , reqid ) : <EOL> """<STR_LIT>""" <EOL> return reqid in self . _cached_reqs <EOL> def add ( self , req ) : <EOL> """<STR_LIT>""" <EOL> if not req . reqid : <EOL> req . reqid = self . get_memid ( ) <EOL> if req . reqid [ <NUM_LIT:0> ] == '<STR_LIT:m>' : <EOL> self . inmem_reqs . add ( req ) <EOL> if req . is_unmangled_version : <EOL> self . unmangled_ids . add ( req . reqid ) <EOL> if req . unmangled : <EOL> self . unmangled_ids . add ( req . unmangled . reqid ) <EOL> self . _cached_reqs [ req . reqid ] = req <EOL> self . _update_last_used ( req . reqid ) <EOL> self . req_times [ req . reqid ] = req . sort_time <EOL> if req . reqid not in self . all_ids : <EOL> self . ordered_ids . insert ( req . reqid ) <EOL> self . all_ids . add ( req . reqid ) <EOL> if len ( self . _cached_reqs ) > self . _cache_size and self . _cache_size != - <NUM_LIT:1> : <EOL> self . _evict_single ( ) <EOL> def evict ( self , reqid ) : <EOL> """<STR_LIT>""" <EOL> if reqid in self . _cached_reqs : <EOL> del self . _cached_reqs [ reqid ] <EOL> del self . _last_used [ reqid ] <EOL> self . _update_min ( reqid ) <EOL> @ defer . inlineCallbacks <EOL> def load ( self , first , num ) : <EOL> """<STR_LIT>""" <EOL> reqs = yield pappyproxy . http . Request . load_requests_by_time ( first , num , cust_dbpool = self . dbpool , cust_cache = self ) <EOL> for r in reqs : <EOL> self . add ( r ) <EOL> self . misses += len ( reqs ) / <NUM_LIT> <EOL> def req_it ( self , num = - <NUM_LIT:1> , ids = None , include_unmangled = False ) : <EOL> """<STR_LIT>""" <EOL> count = <NUM_LIT:0> <EOL> @ defer . inlineCallbacks <EOL> def def_wrapper ( reqid , load = False , num = <NUM_LIT:1> ) : <EOL> if not self . check ( reqid ) and load : <EOL> yield self . load ( reqid , num ) <EOL> req = yield self . get ( reqid ) <EOL> defer . returnValue ( req ) <EOL> over = list ( self . ordered_ids ) <EOL> for reqid in over : <EOL> if ids is not None and reqid not in ids : <EOL> continue <EOL> if not include_unmangled and reqid in self . unmangled_ids : <EOL> continue <EOL> do_load = True <EOL> if reqid in self . all_ids : <EOL> if count % self . _preload_limit == <NUM_LIT:0> : <EOL> do_load = True <EOL> if do_load and not self . check ( reqid ) : <EOL> do_load = False <EOL> if ( num - count ) < self . _preload_limit and num != - <NUM_LIT:1> : <EOL> loadnum = num - count <EOL> else : <EOL> loadnum = self . _preload_limit <EOL> yield def_wrapper ( reqid , load = True , num = loadnum ) <EOL> else : <EOL> yield def_wrapper ( reqid ) <EOL> count += <NUM_LIT:1> <EOL> if count >= num and num != - <NUM_LIT:1> : <EOL> break <EOL> @ defer . inlineCallbacks <EOL> def load_by_tag ( self , tag ) : <EOL> reqs = yield pappyproxy . http . Request . load_requests_by_tag ( tag , cust_cache = self , cust_dbpool = self . dbpool ) <EOL> for req in reqs : <EOL> self . add ( req ) <EOL> defer . returnValue ( reqs ) <EOL> def _evict_single ( self ) : <EOL> """<STR_LIT>""" <EOL> victim_id = self . _min_time [ <NUM_LIT:0> ] <EOL> self . evict ( victim_id ) <EOL> def _update_min ( self , updated_reqid = None ) : <EOL> new_min = None <EOL> if updated_reqid is None or self . _min_time is None or self . _min_time [ <NUM_LIT:0> ] == updated_reqid : <EOL> for k , v in self . _last_used . iteritems ( ) : <EOL> if new_min is None or v < new_min [ <NUM_LIT:1> ] : <EOL> new_min = ( k , v ) <EOL> self . _min_time = new_min <EOL> def _update_last_used ( self , reqid ) : <EOL> t = time . time ( ) <EOL> self . _last_used [ reqid ] = t <EOL> self . _update_min ( reqid ) <EOL> class RequestCacheIterator ( object ) : <EOL> """<STR_LIT>""" <EOL> pass </s>
