<s> import os , sys <EOL> import base64 <EOL> import time <EOL> from dbinterface import * <EOL> from dbconstants import * <EOL> import appscale_logger <EOL> import threading <EOL> import memcache <EOL> import memcache_mutex <EOL> import weakref <EOL> import cPickle as pickle <EOL> MAX_KEYS = <NUM_LIT> <EOL> KEY_REPOSITORY = "<STR_LIT>" <EOL> TABLE_PREFIX = "<STR_LIT>" <EOL> CREATE_LOCK_PREFIX = "<STR_LIT>" <EOL> GLOBAL_TABLES_KEY = "<STR_LIT>" <EOL> SCHEMA_COLUMN = "<STR_LIT>" <EOL> DEFAULT_VALUE = "<STR_LIT:value>" <EOL> ERROR_DEFAULT = "<STR_LIT>" <EOL> USE_MEMCACHED = True <EOL> DEFAULT_KEYSIZE = <NUM_LIT:16> <EOL> class DHashDatastore ( AppDBInterface ) : <EOL> def __init__ ( self , logger = appscale_logger . getLogger ( "<STR_LIT>" ) ) : <EOL> self . logger = logger <EOL> if USE_MEMCACHED : <EOL> self . memcache_locations = self . __getMemcachedLocations ( ) <EOL> self . memcache = memcache . Client ( self . memcache_locations , debug = <NUM_LIT:0> ) <EOL> else : <EOL> self . lockhash = weakref . WeakValueDictionary ( ) <EOL> self . hashkey_size = DEFAULT_KEYSIZE <EOL> def get ( self , key ) : <EOL> raise NotImplementedError ( "<STR_LIT>" % self . __class__ ) <EOL> def put ( self , key , value ) : <EOL> raise NotImplementedError ( "<STR_LIT>" % self . __class__ ) <EOL> def remove ( self , key ) : <EOL> raise NotImplementedError ( "<STR_LIT>" % self . __class__ ) <EOL> def get_all ( self , keys ) : <EOL> values = [ self . get ( key ) for key in keys ] <EOL> return values <EOL> def lock ( self , table ) : <EOL> lock = None <EOL> if USE_MEMCACHED : <EOL> lock = memcache_mutex . MemcacheMutex ( table , self . memcache ) <EOL> lock . acquire ( ) <EOL> return lock <EOL> try : <EOL> lock = self . lockhash [ table ] <EOL> lock . acquire ( ) <EOL> except KeyError : <EOL> lock = threading . Condition ( ) <EOL> self . lockhash [ table ] = lock <EOL> lock . acquire ( ) <EOL> return lock <EOL> def unlock ( self , table , lock ) : <EOL> lock . release ( ) <EOL> def __getMemcachedLocations ( self ) : <EOL> """<STR_LIT>""" <EOL> locations = [ ] <EOL> try : <EOL> f = open ( APPSCALE_HOME + "<STR_LIT>" , "<STR_LIT:r>" ) <EOL> for line in f . readlines ( ) : <EOL> server = line . strip ( ) <EOL> if len ( server ) > <NUM_LIT:0> : <EOL> locations . append ( server + "<STR_LIT>" ) <EOL> f . close ( ) <EOL> except IOError as e : <EOL> self . logger . warn ( "<STR_LIT>" % str ( e ) ) <EOL> locations . append ( "<STR_LIT>" ) <EOL> return locations <EOL> def get_entity ( self , table_name , row_key , column_names ) : <EOL> self . logger . debug ( "<STR_LIT>" % ( table_name , row_key , column_names ) ) <EOL> elist = [ ERROR_DEFAULT ] <EOL> if None in [ table_name , row_key , column_names ] : <EOL> self . logger . debug ( "<STR_LIT>" ) <EOL> return [ ERROR_DEFAULT + "<STR_LIT>" ] <EOL> internal_key = self . __make_internal_key ( table_name , row_key ) <EOL> value = self . get ( internal_key ) <EOL> try : <EOL> if not value : <EOL> elist [ <NUM_LIT:0> ] += "<STR_LIT>" <EOL> else : <EOL> row = pickle . loads ( value ) <EOL> for column in column_names : <EOL> if column == "<STR_LIT>" : <EOL> elist . append ( base64 . b64decode ( row [ column ] ) ) <EOL> else : <EOL> elist . append ( row [ column ] ) <EOL> except Exception as e : <EOL> self . logger . error ( "<STR_LIT>" % e ) <EOL> elist [ <NUM_LIT:0> ] += "<STR_LIT>" <EOL> return elist <EOL> def put_entity ( self , table_name , row_key , column_names , cell_values ) : <EOL> self . logger . debug ( "<STR_LIT>" % ( table_name , row_key ) ) <EOL> elist = [ ERROR_DEFAULT ] <EOL> if None in [ table_name , row_key , column_names , cell_values ] : <EOL> self . logger . debug ( "<STR_LIT>" ) <EOL> return [ ERROR_DEFAULT + "<STR_LIT>" ] <EOL> internal_key = self . __make_internal_key ( table_name , row_key ) <EOL> if len ( column_names ) != len ( cell_values ) : <EOL> self . logger . debug ( "<STR_LIT>" ) <EOL> elist [ <NUM_LIT:0> ] += "<STR_LIT>" <EOL> return elist <EOL> if not self . table_exist ( table_name ) : <EOL> lock = self . lock ( CREATE_LOCK_PREFIX + table_name ) <EOL> if not self . table_exist ( table_name ) : <EOL> self . create_table ( table_name , column_names ) <EOL> self . unlock ( CREATE_LOCK_PREFIX + table_name , lock ) <EOL> dict_row = { } <EOL> try : <EOL> value = self . get ( internal_key ) <EOL> if value : <EOL> dict_row = pickle . loads ( value ) <EOL> for i in range ( len ( column_names ) ) : <EOL> column_name , value = column_names [ i ] , cell_values [ i ] <EOL> if column_name == "<STR_LIT>" : <EOL> value = base64 . b64encode ( value ) <EOL> dict_row [ column_name ] = value <EOL> self . put ( internal_key , pickle . dumps ( dict_row ) ) <EOL> self . add_key ( row_key , table_name ) <EOL> except Exception , ex : <EOL> self . logger . debug ( "<STR_LIT>" % ex ) <EOL> elist . append ( "<STR_LIT:0>" ) <EOL> return elist <EOL> def get_table ( self , table_name , column_names ) : <EOL> self . logger . debug ( "<STR_LIT>" % ( table_name , column_names ) ) <EOL> elist = [ ERROR_DEFAULT ] <EOL> if None in [ table_name , column_names ] : <EOL> self . logger . debug ( "<STR_LIT>" ) <EOL> return [ ERROR_DEFAULT + "<STR_LIT>" ] <EOL> keys_in_table = self . get_keys ( table_name ) <EOL> if SCHEMA_COLUMN in keys_in_table : <EOL> keys_in_table . remove ( SCHEMA_COLUMN ) <EOL> values = self . get_entities ( table_name , keys_in_table , column_names ) <EOL> if values [ <NUM_LIT:0> ] == ERROR_DEFAULT : <EOL> elist . extend ( values [ <NUM_LIT:1> : ] ) <EOL> else : <EOL> elist [ <NUM_LIT:0> ] = values [ <NUM_LIT:0> ] <EOL> self . logger . debug ( "<STR_LIT>" % values ) <EOL> elist . extend ( values ) <EOL> return elist <EOL> def get_entities ( self , table_name , row_keys , column_names ) : <EOL> elist = [ ERROR_DEFAULT ] <EOL> if None in [ table_name , row_keys , column_names ] : <EOL> self . logger . debug ( "<STR_LIT>" ) <EOL> elist [ <NUM_LIT:0> ] += "<STR_LIT>" <EOL> return elist <EOL> internal_keys = [ self . __make_internal_key ( table_name , key ) for key in row_keys ] <EOL> values = self . get_all ( internal_keys ) <EOL> try : <EOL> for value in values : <EOL> if value is None : <EOL> elist [ <NUM_LIT:0> ] += "<STR_LIT>" <EOL> else : <EOL> row = pickle . loads ( value ) <EOL> for column in column_names : <EOL> if column == "<STR_LIT>" : <EOL> elist . append ( base64 . b64decode ( row [ column ] ) ) <EOL> else : <EOL> elist . append ( row [ column ] ) <EOL> except TypeError as type : <EOL> self . logger . debug ( "<STR_LIT>" % str ( type ) ) <EOL> elist [ <NUM_LIT:0> ] += "<STR_LIT>" <EOL> except Exception as e : <EOL> self . logger . debug ( "<STR_LIT>" % str ( e ) ) <EOL> elist [ <NUM_LIT:0> ] += "<STR_LIT>" <EOL> return elist <EOL> def delete_row ( self , table_name , row_id ) : <EOL> self . logger . debug ( "<STR_LIT>" % ( table_name , row_id ) ) <EOL> elist = [ ERROR_DEFAULT ] <EOL> if None in [ table_name , row_id ] : <EOL> self . logger . debug ( "<STR_LIT>" ) <EOL> return [ ERROR_DEFAULT + "<STR_LIT>" ] <EOL> internal_key = self . __make_internal_key ( table_name , row_id ) <EOL> if self . remove_key ( row_id , table_name ) : <EOL> self . remove ( internal_key ) <EOL> return elist <EOL> elist . append ( "<STR_LIT:0>" ) <EOL> return elist <EOL> def get_schema ( self , table_name ) : <EOL> self . logger . debug ( "<STR_LIT>" % table_name ) <EOL> elist = [ ERROR_DEFAULT ] <EOL> try : <EOL> key = self . __make_internal_key ( table_name , "<STR_LIT>" ) <EOL> value = self . get ( key ) <EOL> if value : <EOL> elist . extend ( pickle . loads ( value ) ) <EOL> else : <EOL> elist [ <NUM_LIT:0> ] += "<STR_LIT>" <EOL> except Exception , ex : <EOL> self . logger . debug ( "<STR_LIT>" % ex ) <EOL> return [ ERROR_DEFAULT + "<STR_LIT>" ] <EOL> return elist <EOL> def delete_table ( self , table_name ) : <EOL> self . logger . debug ( "<STR_LIT>" % table_name ) <EOL> elist = [ ERROR_DEFAULT ] <EOL> lock = self . lock ( CREATE_LOCK_PREFIX + table_name ) <EOL> if not self . table_exist ( table_name ) : <EOL> elist [ <NUM_LIT:0> ] += "<STR_LIT>" <EOL> self . unlock ( CREATE_LOCK_PREFIX + table_name , lock ) <EOL> return elist <EOL> keys_in_table = self . get_keys ( table_name ) <EOL> for key in keys_in_table : <EOL> self . delete_row ( table_name , key ) <EOL> self . remove ( self . __make_internal_key ( table_name , "<STR_LIT>" ) ) <EOL> self . remove_key ( table_name ) <EOL> self . unlock ( CREATE_LOCK_PREFIX + table_name , lock ) <EOL> return elist <EOL> def get_row_count ( self , table_name ) : <EOL> self . logger . debug ( "<STR_LIT>" % ( table_name ) ) <EOL> elist = [ ERROR_DEFAULT ] <EOL> key_list = self . get_keys ( table_name ) <EOL> elist . append ( len ( key_list ) - <NUM_LIT:1> ) <EOL> return elist <EOL> def get_row ( self , table_name , key ) : <EOL> self . logger . debug ( "<STR_LIT>" % ( table_name , key ) ) <EOL> elist = [ ERROR_DEFAULT ] <EOL> try : <EOL> schema = self . get_schema ( table_name ) [ <NUM_LIT:1> : ] <EOL> internal_key = self . __make_internal_key ( table_name , key ) <EOL> value = self . get ( internal_key ) <EOL> if not value : <EOL> elist [ <NUM_LIT:0> ] += "<STR_LIT>" <EOL> else : <EOL> row = pickle . loads ( value ) <EOL> for column in schema : <EOL> elist . append ( row [ column ] ) <EOL> except Exception , ex : <EOL> self . logger . debug ( "<STR_LIT>" % ex ) <EOL> return [ ERROR_DEFAULT + "<STR_LIT>" ] <EOL> return elist <EOL> def get_keys ( self , tablename = GLOBAL_TABLES_KEY ) : <EOL> ret = [ ] <EOL> for i in range ( <NUM_LIT:0> , self . hashkey_size ) : <EOL> key_list = self . get_keys_list ( tablename + str ( i ) ) <EOL> if key_list : <EOL> ret . extend ( key_list ) <EOL> return ret <EOL> def get_keys_list ( self , tablename = GLOBAL_TABLES_KEY ) : <EOL> key_name = KEY_REPOSITORY + tablename <EOL> value = self . get ( key_name ) <EOL> if value : <EOL> try : <EOL> key_list = pickle . loads ( value ) <EOL> if isinstance ( key_list , list ) : <EOL> return key_list <EOL> except TypeError , detail : <EOL> self . logger . error ( "<STR_LIT>" % detail ) <EOL> return [ ] <EOL> def put_keys_list ( self , key_list , tablename = GLOBAL_TABLES_KEY ) : <EOL> try : <EOL> key_name = KEY_REPOSITORY + tablename <EOL> self . put ( key_name , pickle . dumps ( key_list ) ) <EOL> except : <EOL> self . logger . error ( "<STR_LIT>" ) <EOL> return False <EOL> return True <EOL> def add_key ( self , key , orgtablename = GLOBAL_TABLES_KEY ) : <EOL> tablename = orgtablename + str ( hash ( key ) % self . hashkey_size ) <EOL> lock = self . lock ( tablename ) <EOL> if not lock : <EOL> return False <EOL> key_list = self . get_keys_list ( tablename ) <EOL> if key in key_list : <EOL> ret = False <EOL> else : <EOL> key_list . append ( key ) <EOL> ret = self . put_keys_list ( key_list , tablename ) <EOL> self . unlock ( tablename , lock ) <EOL> return ret <EOL> def remove_key ( self , key , orgtablename = GLOBAL_TABLES_KEY ) : <EOL> tablename = orgtablename + str ( hash ( key ) % self . hashkey_size ) <EOL> lock = self . lock ( tablename ) <EOL> if not lock : <EOL> return False <EOL> key_list = self . get_keys_list ( tablename ) <EOL> if key in key_list : <EOL> key_list . remove ( key ) <EOL> ret = self . put_keys_list ( key_list , tablename ) <EOL> else : <EOL> ret = False <EOL> self . unlock ( tablename , lock ) <EOL> return ret <EOL> def __make_internal_key ( self , table_name , key ) : <EOL> if None in [ table_name , key ] : <EOL> self . logger . debug ( "<STR_LIT>" ) <EOL> return TABLE_PREFIX + "<STR_LIT>" <EOL> return TABLE_PREFIX + table_name + "<STR_LIT:_>" + key <EOL> def create_table ( self , table_name , schema ) : <EOL> elist = [ ERROR_DEFAULT ] <EOL> key = self . __make_internal_key ( table_name , "<STR_LIT>" ) <EOL> self . put ( key , pickle . dumps ( schema ) ) <EOL> self . add_key ( table_name ) <EOL> self . add_key ( "<STR_LIT>" , table_name ) <EOL> return elist <EOL> def table_exist ( self , table_name ) : <EOL> key = self . __make_internal_key ( table_name , "<STR_LIT>" ) <EOL> if self . get ( key ) : <EOL> return True <EOL> else : <EOL> return False </s>
