<s> import time <EOL> from collections import deque <EOL> from weakref import WeakValueDictionary <EOL> from redis import ConnectionPool <EOL> from redis . client import StrictRedis <EOL> from lampost . context . resource import m_requires <EOL> from lampost . datastore . classes import get_dbo_class , get_mixed_type <EOL> from lampost . datastore . exceptions import ObjectExistsError , NonUniqueError <EOL> m_requires ( __name__ , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ) <EOL> class RedisStore ( ) : <EOL> def __init__ ( self , db_host , db_port , db_num , db_pw ) : <EOL> self . pool = ConnectionPool ( max_connections = <NUM_LIT:2> , db = db_num , host = db_host , port = db_port , password = db_pw , <EOL> decode_responses = True ) <EOL> self . redis = StrictRedis ( connection_pool = self . pool ) <EOL> self . redis . ping ( ) <EOL> self . _object_map = WeakValueDictionary ( ) <EOL> def create_object ( self , dbo_class , dbo_dict , update_timestamp = True ) : <EOL> dbo_class = get_dbo_class ( getattr ( dbo_class , '<STR_LIT>' , dbo_class ) ) <EOL> if not dbo_class : <EOL> return <EOL> try : <EOL> dbo_id = dbo_dict [ '<STR_LIT>' ] <EOL> except KeyError : <EOL> dbo_id , dbo_dict = dbo_dict , { } <EOL> if dbo_id is None or dbo_id == '<STR_LIT>' : <EOL> warn ( "<STR_LIT>" ) <EOL> return <EOL> dbo_id = str ( dbo_id ) . lower ( ) <EOL> if self . object_exists ( dbo_class . dbo_key_type , dbo_id ) : <EOL> raise ObjectExistsError ( dbo_id ) <EOL> dbo = dbo_class ( ) <EOL> dbo . dbo_id = dbo_id <EOL> dbo . hydrate ( dbo_dict ) <EOL> dbo . db_created ( ) <EOL> if dbo . dbo_set_key : <EOL> self . redis . sadd ( dbo . dbo_set_key , dbo . dbo_id ) <EOL> self . save_object ( dbo , update_timestamp ) <EOL> return dbo <EOL> def save_object ( self , dbo , update_timestamp = False , autosave = False ) : <EOL> if update_timestamp : <EOL> dbo . dbo_ts = int ( time . time ( ) ) <EOL> if dbo . dbo_indexes : <EOL> self . _update_indexes ( dbo ) <EOL> self . _clear_old_refs ( dbo ) <EOL> save_root , new_refs = dbo . to_db_value ( ) <EOL> self . redis . set ( dbo . dbo_key , json_encode ( save_root ) ) <EOL> if new_refs : <EOL> self . _set_new_refs ( dbo , new_refs ) <EOL> debug ( "<STR_LIT>" , dbo . dbo_key , "<STR_LIT>" if autosave else "<STR_LIT>" ) <EOL> self . _object_map [ dbo . dbo_key ] = dbo <EOL> return dbo <EOL> def save_raw ( self , key , raw ) : <EOL> self . redis . set ( key , json_encode ( raw ) ) <EOL> def load_raw ( self , key , default = None ) : <EOL> json = self . redis . get ( key ) <EOL> if json : <EOL> return json_decode ( json ) <EOL> return default <EOL> def load_cached ( self , dbo_key ) : <EOL> return self . _object_map . get ( dbo_key ) <EOL> def load_object ( self , dbo_key , key_type = None , silent = False ) : <EOL> if key_type : <EOL> try : <EOL> key_type = key_type . dbo_key_type <EOL> except AttributeError : <EOL> pass <EOL> try : <EOL> dbo_key , dbo_id = '<STR_LIT::>' . join ( [ key_type , dbo_key ] ) , dbo_key <EOL> except TypeError : <EOL> if not silent : <EOL> exception ( "<STR_LIT>" , stack_info = True ) <EOL> return <EOL> else : <EOL> key_type , _ , dbo_id = dbo_key . partition ( '<STR_LIT::>' ) <EOL> cached_dbo = self . _object_map . get ( dbo_key ) <EOL> if cached_dbo : <EOL> return cached_dbo <EOL> json_str = self . redis . get ( dbo_key ) <EOL> if not json_str : <EOL> if not silent : <EOL> warn ( "<STR_LIT>" , dbo_key ) <EOL> return <EOL> return self . load_from_json ( json_str , key_type , dbo_id ) <EOL> def load_from_json ( self , json_str , key_type , dbo_id ) : <EOL> dbo_dict = json_decode ( json_str ) <EOL> dbo = get_mixed_type ( key_type , dbo_dict . get ( '<STR_LIT>' ) ) ( ) <EOL> dbo . dbo_id = dbo_id <EOL> self . _object_map [ dbo . dbo_key ] = dbo <EOL> dbo . hydrate ( dbo_dict ) <EOL> return dbo <EOL> def object_exists ( self , obj_type , obj_id ) : <EOL> return self . redis . exists ( '<STR_LIT>' . format ( obj_type , obj_id ) ) <EOL> def load_object_set ( self , dbo_class , set_key = None ) : <EOL> key_type = dbo_class . dbo_key_type <EOL> if not set_key : <EOL> set_key = dbo_class . dbo_set_key <EOL> results = set ( ) <EOL> keys = deque ( ) <EOL> pipeline = self . redis . pipeline ( ) <EOL> for key in self . fetch_set_keys ( set_key ) : <EOL> dbo_key = '<STR_LIT::>' . join ( [ key_type , key ] ) <EOL> try : <EOL> results . add ( self . _object_map [ dbo_key ] ) <EOL> except KeyError : <EOL> keys . append ( key ) <EOL> pipeline . get ( dbo_key ) <EOL> for dbo_id , json_str in zip ( keys , pipeline . execute ( ) ) : <EOL> if json_str : <EOL> obj = self . load_from_json ( json_str , key_type , dbo_id ) <EOL> if obj : <EOL> results . add ( obj ) <EOL> continue <EOL> warn ( "<STR_LIT>" , set_key ) <EOL> self . delete_set_key ( set_key , dbo_id ) <EOL> return results <EOL> def delete_object_set ( self , dbo_class , set_key = None ) : <EOL> if not set_key : <EOL> set_key = dbo_class . dbo_set_key <EOL> for dbo in self . load_object_set ( dbo_class , set_key ) : <EOL> self . delete_object ( dbo ) <EOL> self . delete_key ( set_key ) <EOL> def update_object ( self , dbo , dbo_dict ) : <EOL> dbo . hydrate ( dbo_dict ) <EOL> return self . save_object ( dbo , True ) <EOL> def delete_object ( self , dbo ) : <EOL> key = dbo . dbo_key <EOL> dbo . db_deleted ( ) <EOL> self . delete_key ( key ) <EOL> self . _clear_old_refs ( dbo ) <EOL> if dbo . dbo_set_key : <EOL> self . redis . srem ( dbo . dbo_set_key , dbo . dbo_id ) <EOL> for children_type in dbo . dbo_children_types : <EOL> self . delete_object_set ( get_dbo_class ( children_type ) , <EOL> "<STR_LIT>" . format ( dbo . dbo_key_type , children_type , dbo . dbo_id ) ) <EOL> for ix_name in dbo . dbo_indexes : <EOL> ix_value = getattr ( dbo , ix_name , None ) <EOL> if ix_value is not None and ix_value != '<STR_LIT>' : <EOL> self . delete_index ( '<STR_LIT>' . format ( dbo . dbo_key_type , ix_name ) , ix_value ) <EOL> debug ( "<STR_LIT>" , key ) <EOL> self . evict_object ( dbo ) <EOL> def reload_object ( self , dbo_key ) : <EOL> dbo = self . _object_map . get ( dbo_key ) <EOL> if dbo : <EOL> json_str = self . redis . get ( dbo_key ) <EOL> if not json_str : <EOL> warn ( "<STR_LIT>" , dbo_key ) <EOL> return None <EOL> return self . update_object ( dbo , json_decode ( json_str ) ) <EOL> return self . load_object ( dbo_key ) <EOL> def evict_object ( self , dbo ) : <EOL> self . _object_map . pop ( dbo . dbo_key , None ) <EOL> def fetch_set_keys ( self , set_key ) : <EOL> return self . redis . smembers ( set_key ) <EOL> def add_set_key ( self , set_key , * values ) : <EOL> self . redis . sadd ( set_key , * values ) <EOL> def delete_set_key ( self , set_key , value ) : <EOL> self . redis . srem ( set_key , value ) <EOL> def set_key_exists ( self , set_key , value ) : <EOL> return self . redis . sismember ( set_key , value ) <EOL> def db_counter ( self , counter_id , inc = <NUM_LIT:1> ) : <EOL> return self . redis . incr ( "<STR_LIT>" . format ( counter_id ) , inc ) <EOL> def delete_key ( self , key ) : <EOL> self . redis . delete ( key ) <EOL> def set_index ( self , index_name , key , value ) : <EOL> return self . redis . hset ( index_name , key , value ) <EOL> def get_index ( self , index_name , key ) : <EOL> return self . redis . hget ( index_name , key ) <EOL> def get_full_index ( self , index_name ) : <EOL> return self . redis . hgetall ( index_name ) <EOL> def delete_index ( self , index_name , key ) : <EOL> return self . redis . hdel ( index_name , key ) <EOL> def get_all_hash ( self , index_name ) : <EOL> return { key : json_decode ( value ) for key , value in self . redis . hgetall ( index_name ) . items ( ) } <EOL> def get_hash_keys ( self , hash_id ) : <EOL> return self . redis . hkeys ( hash_id ) <EOL> def set_db_hash ( self , hash_id , hash_key , value ) : <EOL> return self . redis . hset ( hash_id , hash_key , json_encode ( value ) ) <EOL> def get_db_hash ( self , hash_id , hash_key ) : <EOL> return json_decode ( self . redis . hget ( hash_id , hash_key ) ) <EOL> def remove_db_hash ( self , hash_id , hash_key ) : <EOL> self . redis . hdel ( hash_id , hash_key ) <EOL> def get_all_db_hash ( self , hash_id ) : <EOL> return [ json_decode ( value ) for value in self . redis . hgetall ( hash_id ) . values ( ) ] <EOL> def get_db_list ( self , list_id , start = <NUM_LIT:0> , end = - <NUM_LIT:1> ) : <EOL> return [ json_decode ( value ) for value in self . redis . lrange ( list_id , start , end ) ] <EOL> def add_db_list ( self , list_id , value ) : <EOL> self . redis . lpush ( list_id , json_encode ( value ) ) <EOL> def trim_db_list ( self , list_id , start , end ) : <EOL> return self . redis . ltrim ( list_id , start , end ) <EOL> def _update_indexes ( self , dbo ) : <EOL> try : <EOL> old_dbo = json_decode ( self . redis . get ( dbo . dbo_key ) ) <EOL> except TypeError : <EOL> old_dbo = None <EOL> for ix_name in dbo . dbo_indexes : <EOL> new_val = getattr ( dbo , ix_name , None ) <EOL> old_val = old_dbo . get ( ix_name ) if old_dbo else None <EOL> if old_val == new_val : <EOL> continue <EOL> ix_key = '<STR_LIT>' . format ( dbo . dbo_key_type , ix_name ) <EOL> if old_val is not None : <EOL> self . delete_index ( ix_key , old_val ) <EOL> if new_val is not None and new_val != '<STR_LIT>' : <EOL> if self . get_index ( ix_key , new_val ) : <EOL> raise NonUniqueError ( ix_key , new_val ) <EOL> self . set_index ( ix_key , new_val , dbo . dbo_id ) <EOL> def _clear_old_refs ( self , dbo ) : <EOL> dbo_key = dbo . dbo_key <EOL> ref_key = '<STR_LIT>' . format ( dbo_key ) <EOL> for ref_id in self . fetch_set_keys ( ref_key ) : <EOL> holder_key = '<STR_LIT>' . format ( ref_id ) <EOL> self . delete_set_key ( holder_key , dbo_key ) <EOL> self . delete_key ( ref_key ) <EOL> def _set_new_refs ( self , dbo , new_refs ) : <EOL> dbo_key = dbo . dbo_key <EOL> self . add_set_key ( "<STR_LIT>" . format ( dbo_key ) , * new_refs ) <EOL> for ref_id in new_refs : <EOL> holder_key = '<STR_LIT>' . format ( ref_id ) <EOL> self . add_set_key ( holder_key , dbo_key ) </s>
