<s> """<STR_LIT>""" <EOL> import datetime <EOL> import logging <EOL> import md5 <EOL> import os <EOL> import struct <EOL> import sys <EOL> import tempfile <EOL> import threading <EOL> import urllib <EOL> import warnings <EOL> try : <EOL> from urlparse import parse_qsl <EOL> except ImportError : <EOL> from cgi import parse_qsl <EOL> import cPickle as pickle <EOL> from google . appengine . api import api_base_pb <EOL> from google . appengine . api import apiproxy_stub <EOL> from google . appengine . api import apiproxy_stub_map <EOL> from google . appengine . api import datastore <EOL> from google . appengine . api import datastore_errors <EOL> from google . appengine . api import datastore_types <EOL> from google . appengine . api import users <EOL> from google . appengine . datastore import datastore_pb <EOL> from google . appengine . datastore import datastore_index <EOL> from google . appengine . runtime import apiproxy_errors <EOL> from google . net . proto import ProtocolBuffer <EOL> from google . appengine . datastore import entity_pb <EOL> warnings . filterwarnings ( '<STR_LIT:ignore>' , '<STR_LIT>' ) <EOL> entity_pb . Reference . __hash__ = lambda self : hash ( self . Encode ( ) ) <EOL> datastore_pb . Query . __hash__ = lambda self : hash ( self . Encode ( ) ) <EOL> _MAXIMUM_RESULTS = <NUM_LIT:1000> <EOL> _MAX_QUERY_OFFSET = <NUM_LIT:1000> <EOL> _MAX_QUERY_COMPONENTS = <NUM_LIT:100> <EOL> _BATCH_SIZE = <NUM_LIT:20> <EOL> _MAX_ACTIONS_PER_TXN = <NUM_LIT:5> <EOL> class _StoredEntity ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , entity ) : <EOL> """<STR_LIT>""" <EOL> self . protobuf = entity <EOL> self . encoded_protobuf = entity . Encode ( ) <EOL> self . native = datastore . Entity . _FromPb ( entity ) <EOL> class _Cursor ( object ) : <EOL> """<STR_LIT>""" <EOL> _next_cursor = <NUM_LIT:1> <EOL> _next_cursor_lock = threading . Lock ( ) <EOL> def __init__ ( self , results , keys_only , limit = None ) : <EOL> """<STR_LIT>""" <EOL> self . __results = results <EOL> self . count = len ( results ) <EOL> self . keys_only = keys_only <EOL> self . _offset = <NUM_LIT:0> <EOL> self . _limit = limit <EOL> self . _next_cursor_lock . acquire ( ) <EOL> try : <EOL> self . cursor = _Cursor . _next_cursor <EOL> _Cursor . _next_cursor += <NUM_LIT:1> <EOL> finally : <EOL> self . _next_cursor_lock . release ( ) <EOL> def PopulateQueryResult ( self , result , count , offset = None , compiled = False ) : <EOL> """<STR_LIT>""" <EOL> _offset = offset <EOL> if offset is None : <EOL> _offset = self . _offset <EOL> if count > _MAXIMUM_RESULTS : <EOL> count = _MAXIMUM_RESULTS <EOL> result . mutable_cursor ( ) . set_cursor ( self . cursor ) <EOL> result . set_keys_only ( self . keys_only ) <EOL> self . count = len ( self . __results [ _offset : _offset + count ] ) <EOL> results_pbs = [ r . _ToPb ( ) for r in self . __results [ _offset : _offset + count ] ] <EOL> result . result_list ( ) . extend ( results_pbs ) <EOL> if offset is None : <EOL> self . _offset += self . count <EOL> if self . _limit : <EOL> limit = max ( <NUM_LIT:0> , self . _limit - self . _offset ) <EOL> result . set_more_results ( limit > <NUM_LIT:0> ) <EOL> else : <EOL> limit = None <EOL> result . set_more_results ( len ( self . __results [ _offset + self . count : ] ) > <NUM_LIT:0> ) <EOL> if compiled : <EOL> compiled_query = _FakeCompiledQuery ( cursor = self . cursor , <EOL> offset = _offset + self . count , <EOL> keys_only = self . keys_only , <EOL> limit = limit ) <EOL> result . mutable_compiled_query ( ) . CopyFrom ( compiled_query . _ToPb ( ) ) <EOL> class _FakeCompiledQuery ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , cursor , offset , keys_only = False , limit = None ) : <EOL> """<STR_LIT>""" <EOL> self . cursor = cursor <EOL> self . limit = limit <EOL> self . offset = offset <EOL> self . keys_only = keys_only <EOL> def _ToPb ( self ) : <EOL> compiled_pb = datastore_pb . CompiledQuery ( ) <EOL> compiled_pb . set_keys_only ( self . keys_only ) <EOL> if self . limit : <EOL> compiled_pb . set_limit ( self . limit ) <EOL> primaryscan = compiled_pb . mutable_primaryscan ( ) <EOL> primaryscan . set_start_key ( urllib . urlencode ( { '<STR_LIT>' : self . cursor , <EOL> '<STR_LIT>' : self . offset } ) ) <EOL> return compiled_pb <EOL> class DatastoreFileStub ( apiproxy_stub . APIProxyStub ) : <EOL> """<STR_LIT>""" <EOL> _PROPERTY_TYPE_TAGS = { <EOL> datastore_types . Blob : entity_pb . PropertyValue . kstringValue , <EOL> bool : entity_pb . PropertyValue . kbooleanValue , <EOL> datastore_types . Category : entity_pb . PropertyValue . kstringValue , <EOL> datetime . datetime : entity_pb . PropertyValue . kint64Value , <EOL> datastore_types . Email : entity_pb . PropertyValue . kstringValue , <EOL> float : entity_pb . PropertyValue . kdoubleValue , <EOL> datastore_types . GeoPt : entity_pb . PropertyValue . kPointValueGroup , <EOL> datastore_types . IM : entity_pb . PropertyValue . kstringValue , <EOL> int : entity_pb . PropertyValue . kint64Value , <EOL> datastore_types . Key : entity_pb . PropertyValue . kReferenceValueGroup , <EOL> datastore_types . Link : entity_pb . PropertyValue . kstringValue , <EOL> long : entity_pb . PropertyValue . kint64Value , <EOL> datastore_types . PhoneNumber : entity_pb . PropertyValue . kstringValue , <EOL> datastore_types . PostalAddress : entity_pb . PropertyValue . kstringValue , <EOL> datastore_types . Rating : entity_pb . PropertyValue . kint64Value , <EOL> str : entity_pb . PropertyValue . kstringValue , <EOL> datastore_types . Text : entity_pb . PropertyValue . kstringValue , <EOL> type ( None ) : <NUM_LIT:0> , <EOL> unicode : entity_pb . PropertyValue . kstringValue , <EOL> users . User : entity_pb . PropertyValue . kUserValueGroup , <EOL> } <EOL> WRITE_ONLY = entity_pb . CompositeIndex . WRITE_ONLY <EOL> READ_WRITE = entity_pb . CompositeIndex . READ_WRITE <EOL> DELETED = entity_pb . CompositeIndex . DELETED <EOL> ERROR = entity_pb . CompositeIndex . ERROR <EOL> _INDEX_STATE_TRANSITIONS = { <EOL> WRITE_ONLY : frozenset ( ( READ_WRITE , DELETED , ERROR ) ) , <EOL> READ_WRITE : frozenset ( ( DELETED , ) ) , <EOL> ERROR : frozenset ( ( DELETED , ) ) , <EOL> DELETED : frozenset ( ( ERROR , ) ) , <EOL> } <EOL> def __init__ ( self , <EOL> app_id , <EOL> datastore_file , <EOL> history_file = None , <EOL> require_indexes = False , <EOL> service_name = '<STR_LIT>' , <EOL> trusted = False ) : <EOL> """<STR_LIT>""" <EOL> super ( DatastoreFileStub , self ) . __init__ ( service_name ) <EOL> assert isinstance ( app_id , basestring ) and app_id != '<STR_LIT>' <EOL> self . __app_id = app_id <EOL> self . __datastore_file = datastore_file <EOL> self . SetTrusted ( trusted ) <EOL> self . __entities = { } <EOL> self . __schema_cache = { } <EOL> self . __tx_snapshot = { } <EOL> self . __tx_actions = [ ] <EOL> self . __queries = { } <EOL> self . __transactions = { } <EOL> self . __indexes = { } <EOL> self . __require_indexes = require_indexes <EOL> self . __query_history = { } <EOL> self . __next_id = <NUM_LIT:1> <EOL> self . __next_tx_handle = <NUM_LIT:1> <EOL> self . __next_index_id = <NUM_LIT:1> <EOL> self . __id_lock = threading . Lock ( ) <EOL> self . __tx_handle_lock = threading . Lock ( ) <EOL> self . __index_id_lock = threading . Lock ( ) <EOL> self . __tx_lock = threading . Lock ( ) <EOL> self . __entities_lock = threading . Lock ( ) <EOL> self . __file_lock = threading . Lock ( ) <EOL> self . __indexes_lock = threading . Lock ( ) <EOL> self . Read ( ) <EOL> def Clear ( self ) : <EOL> """<STR_LIT>""" <EOL> self . __entities = { } <EOL> self . __queries = { } <EOL> self . __transactions = { } <EOL> self . __query_history = { } <EOL> self . __schema_cache = { } <EOL> def SetTrusted ( self , trusted ) : <EOL> """<STR_LIT>""" <EOL> self . __trusted = trusted <EOL> def __ValidateAppId ( self , app_id ) : <EOL> """<STR_LIT>""" <EOL> if not self . __trusted and app_id != self . __app_id : <EOL> raise datastore_errors . BadRequestError ( <EOL> '<STR_LIT>' % ( self . __app_id , app_id ) ) <EOL> def __ValidateKey ( self , key ) : <EOL> """<STR_LIT>""" <EOL> assert isinstance ( key , entity_pb . Reference ) <EOL> self . __ValidateAppId ( key . app ( ) ) <EOL> for elem in key . path ( ) . element_list ( ) : <EOL> if elem . has_id ( ) == elem . has_name ( ) : <EOL> raise datastore_errors . BadRequestError ( <EOL> '<STR_LIT>' % key ) <EOL> def _AppIdNamespaceKindForKey ( self , key ) : <EOL> """<STR_LIT>""" <EOL> last_path = key . path ( ) . element_list ( ) [ - <NUM_LIT:1> ] <EOL> return key . app ( ) , last_path . type ( ) <EOL> def _StoreEntity ( self , entity ) : <EOL> """<STR_LIT>""" <EOL> key = entity . key ( ) <EOL> app_kind = self . _AppIdNamespaceKindForKey ( key ) <EOL> if app_kind not in self . __entities : <EOL> self . __entities [ app_kind ] = { } <EOL> self . __entities [ app_kind ] [ key ] = _StoredEntity ( entity ) <EOL> if app_kind in self . __schema_cache : <EOL> del self . __schema_cache [ app_kind ] <EOL> READ_PB_EXCEPTIONS = ( ProtocolBuffer . ProtocolBufferDecodeError , LookupError , <EOL> TypeError , ValueError ) <EOL> READ_ERROR_MSG = ( '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> READ_PY250_MSG = ( '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> def Read ( self ) : <EOL> """<STR_LIT>""" <EOL> if self . __datastore_file and self . __datastore_file != '<STR_LIT>' : <EOL> for encoded_entity in self . __ReadPickled ( self . __datastore_file ) : <EOL> try : <EOL> entity = entity_pb . EntityProto ( encoded_entity ) <EOL> except self . READ_PB_EXCEPTIONS , e : <EOL> raise datastore_errors . InternalError ( self . READ_ERROR_MSG % <EOL> ( self . __datastore_file , e ) ) <EOL> except struct . error , e : <EOL> if ( sys . version_info [ <NUM_LIT:0> : <NUM_LIT:3> ] == ( <NUM_LIT:2> , <NUM_LIT:5> , <NUM_LIT:0> ) <EOL> and e . message . startswith ( '<STR_LIT>' ) ) : <EOL> raise datastore_errors . InternalError ( self . READ_PY250_MSG + <EOL> self . READ_ERROR_MSG % <EOL> ( self . __datastore_file , e ) ) <EOL> else : <EOL> raise <EOL> self . _StoreEntity ( entity ) <EOL> last_path = entity . key ( ) . path ( ) . element_list ( ) [ - <NUM_LIT:1> ] <EOL> if last_path . has_id ( ) and last_path . id ( ) >= self . __next_id : <EOL> self . __next_id = last_path . id ( ) + <NUM_LIT:1> <EOL> def Write ( self ) : <EOL> """<STR_LIT>""" <EOL> self . __WriteDatastore ( ) <EOL> def __WriteDatastore ( self ) : <EOL> """<STR_LIT>""" <EOL> if self . __datastore_file and self . __datastore_file != '<STR_LIT>' : <EOL> encoded = [ ] <EOL> for kind_dict in self . __entities . values ( ) : <EOL> for entity in kind_dict . values ( ) : <EOL> encoded . append ( entity . encoded_protobuf ) <EOL> self . __WritePickled ( encoded , self . __datastore_file ) <EOL> def __ReadPickled ( self , filename ) : <EOL> """<STR_LIT>""" <EOL> self . __file_lock . acquire ( ) <EOL> try : <EOL> try : <EOL> if filename and filename != '<STR_LIT>' and os . path . isfile ( filename ) : <EOL> return pickle . load ( open ( filename , '<STR_LIT:rb>' ) ) <EOL> else : <EOL> logging . warning ( '<STR_LIT>' , filename ) <EOL> except ( AttributeError , LookupError , ImportError , NameError , TypeError , <EOL> ValueError , struct . error , pickle . PickleError ) , e : <EOL> raise datastore_errors . InternalError ( <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' % ( filename , e ) ) <EOL> finally : <EOL> self . __file_lock . release ( ) <EOL> return [ ] <EOL> def __WritePickled ( self , obj , filename , openfile = file ) : <EOL> """<STR_LIT>""" <EOL> if not filename or filename == '<STR_LIT>' or not obj : <EOL> return <EOL> tmpfile = openfile ( os . tempnam ( os . path . dirname ( filename ) ) , '<STR_LIT:wb>' ) <EOL> pickler = pickle . Pickler ( tmpfile , protocol = <NUM_LIT:1> ) <EOL> pickler . fast = True <EOL> pickler . dump ( obj ) <EOL> tmpfile . close ( ) <EOL> self . __file_lock . acquire ( ) <EOL> try : <EOL> try : <EOL> os . rename ( tmpfile . name , filename ) <EOL> except OSError : <EOL> try : <EOL> os . remove ( filename ) <EOL> except : <EOL> pass <EOL> os . rename ( tmpfile . name , filename ) <EOL> finally : <EOL> self . __file_lock . release ( ) <EOL> def MakeSyncCall ( self , service , call , request , response ) : <EOL> """<STR_LIT>""" <EOL> self . assertPbIsInitialized ( request ) <EOL> super ( DatastoreFileStub , self ) . MakeSyncCall ( service , <EOL> call , <EOL> request , <EOL> response ) <EOL> self . assertPbIsInitialized ( response ) <EOL> def assertPbIsInitialized ( self , pb ) : <EOL> """<STR_LIT>""" <EOL> explanation = [ ] <EOL> assert pb . IsInitialized ( explanation ) , explanation <EOL> pb . Encode ( ) <EOL> def QueryHistory ( self ) : <EOL> """<STR_LIT>""" <EOL> return dict ( ( pb , times ) for pb , times in self . __query_history . items ( ) <EOL> if pb . app ( ) == self . __app_id ) <EOL> def _Dynamic_Put ( self , put_request , put_response ) : <EOL> clones = [ ] <EOL> for entity in put_request . entity_list ( ) : <EOL> self . __ValidateKey ( entity . key ( ) ) <EOL> clone = entity_pb . EntityProto ( ) <EOL> clone . CopyFrom ( entity ) <EOL> for property in clone . property_list ( ) : <EOL> if property . value ( ) . has_uservalue ( ) : <EOL> uid = md5 . new ( property . value ( ) . uservalue ( ) . email ( ) . lower ( ) ) . digest ( ) <EOL> uid = '<STR_LIT:1>' + '<STR_LIT>' . join ( [ '<STR_LIT>' % ord ( x ) for x in uid ] ) [ : <NUM_LIT:20> ] <EOL> property . mutable_value ( ) . mutable_uservalue ( ) . set_obfuscated_gaiaid ( <EOL> uid ) <EOL> clones . append ( clone ) <EOL> assert clone . has_key ( ) <EOL> assert clone . key ( ) . path ( ) . element_size ( ) > <NUM_LIT:0> <EOL> last_path = clone . key ( ) . path ( ) . element_list ( ) [ - <NUM_LIT:1> ] <EOL> if last_path . id ( ) == <NUM_LIT:0> and not last_path . has_name ( ) : <EOL> self . __id_lock . acquire ( ) <EOL> last_path . set_id ( self . __next_id ) <EOL> self . __next_id += <NUM_LIT:1> <EOL> self . __id_lock . release ( ) <EOL> assert clone . entity_group ( ) . element_size ( ) == <NUM_LIT:0> <EOL> group = clone . mutable_entity_group ( ) <EOL> root = clone . key ( ) . path ( ) . element ( <NUM_LIT:0> ) <EOL> group . add_element ( ) . CopyFrom ( root ) <EOL> else : <EOL> assert ( clone . has_entity_group ( ) and <EOL> clone . entity_group ( ) . element_size ( ) > <NUM_LIT:0> ) <EOL> self . __entities_lock . acquire ( ) <EOL> try : <EOL> for clone in clones : <EOL> self . _StoreEntity ( clone ) <EOL> finally : <EOL> self . __entities_lock . release ( ) <EOL> if not put_request . has_transaction ( ) : <EOL> self . __WriteDatastore ( ) <EOL> put_response . key_list ( ) . extend ( [ c . key ( ) for c in clones ] ) <EOL> def _Dynamic_Get ( self , get_request , get_response ) : <EOL> if get_request . has_transaction ( ) : <EOL> entities = self . __tx_snapshot <EOL> else : <EOL> entities = self . __entities <EOL> for key in get_request . key_list ( ) : <EOL> self . __ValidateAppId ( key . app ( ) ) <EOL> app_kind = self . _AppIdNamespaceKindForKey ( key ) <EOL> group = get_response . add_entity ( ) <EOL> try : <EOL> entity = entities [ app_kind ] [ key ] . protobuf <EOL> except KeyError : <EOL> entity = None <EOL> if entity : <EOL> group . mutable_entity ( ) . CopyFrom ( entity ) <EOL> def _Dynamic_Delete ( self , delete_request , delete_response ) : <EOL> self . __entities_lock . acquire ( ) <EOL> try : <EOL> for key in delete_request . key_list ( ) : <EOL> self . __ValidateAppId ( key . app ( ) ) <EOL> app_kind = self . _AppIdNamespaceKindForKey ( key ) <EOL> try : <EOL> del self . __entities [ app_kind ] [ key ] <EOL> if not self . __entities [ app_kind ] : <EOL> del self . __entities [ app_kind ] <EOL> del self . __schema_cache [ app_kind ] <EOL> except KeyError : <EOL> pass <EOL> if not delete_request . has_transaction ( ) : <EOL> self . __WriteDatastore ( ) <EOL> finally : <EOL> self . __entities_lock . release ( ) <EOL> def _Dynamic_RunQuery ( self , query , query_result , count = None ) : <EOL> if not self . __tx_lock . acquire ( False ) : <EOL> if not query . has_ancestor ( ) : <EOL> raise apiproxy_errors . ApplicationError ( <EOL> datastore_pb . Error . BAD_REQUEST , <EOL> '<STR_LIT>' ) <EOL> entities = self . __tx_snapshot <EOL> else : <EOL> entities = self . __entities <EOL> self . __tx_lock . release ( ) <EOL> app_id_namespace = datastore_types . parse_app_id_namespace ( query . app ( ) ) <EOL> app_id = app_id_namespace . app_id ( ) <EOL> self . __ValidateAppId ( app_id ) <EOL> if query . has_offset ( ) and query . offset ( ) > _MAX_QUERY_OFFSET : <EOL> raise apiproxy_errors . ApplicationError ( <EOL> datastore_pb . Error . BAD_REQUEST , '<STR_LIT>' ) <EOL> num_components = len ( query . filter_list ( ) ) + len ( query . order_list ( ) ) <EOL> if query . has_ancestor ( ) : <EOL> num_components += <NUM_LIT:1> <EOL> if num_components > _MAX_QUERY_COMPONENTS : <EOL> raise apiproxy_errors . ApplicationError ( <EOL> datastore_pb . Error . BAD_REQUEST , <EOL> ( '<STR_LIT>' <EOL> '<STR_LIT>' % _MAX_QUERY_COMPONENTS ) ) <EOL> ( filters , orders ) = datastore_index . Normalize ( query . filter_list ( ) , <EOL> query . order_list ( ) ) <EOL> if self . __require_indexes : <EOL> required , kind , ancestor , props , num_eq_filters = datastore_index . CompositeIndexForQuery ( query ) <EOL> if required : <EOL> required_key = kind , ancestor , props <EOL> indexes = self . __indexes . get ( app_id ) <EOL> if not indexes : <EOL> raise apiproxy_errors . ApplicationError ( <EOL> datastore_pb . Error . NEED_INDEX , <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" ) <EOL> eq_filters_set = set ( props [ : num_eq_filters ] ) <EOL> remaining_filters = props [ num_eq_filters : ] <EOL> for index in indexes : <EOL> definition = datastore_index . ProtoToIndexDefinition ( index ) <EOL> index_key = datastore_index . IndexToKey ( definition ) <EOL> if required_key == index_key : <EOL> break <EOL> if num_eq_filters > <NUM_LIT:1> and ( kind , ancestor ) == index_key [ : <NUM_LIT:2> ] : <EOL> this_props = index_key [ <NUM_LIT:2> ] <EOL> this_eq_filters_set = set ( this_props [ : num_eq_filters ] ) <EOL> this_remaining_filters = this_props [ num_eq_filters : ] <EOL> if ( eq_filters_set == this_eq_filters_set and <EOL> remaining_filters == this_remaining_filters ) : <EOL> break <EOL> else : <EOL> raise apiproxy_errors . ApplicationError ( <EOL> datastore_pb . Error . NEED_INDEX , <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" ) <EOL> try : <EOL> query . set_app ( app_id_namespace . to_encoded ( ) ) <EOL> if query . has_kind ( ) : <EOL> results = entities [ app_id_namespace . to_encoded ( ) , query . kind ( ) ] . values ( ) <EOL> results = [ entity . native for entity in results ] <EOL> else : <EOL> results = [ ] <EOL> for key in entities : <EOL> if key [ <NUM_LIT:0> ] == app_id_namespace . to_encoded ( ) : <EOL> results += [ entity . native for entity in entities [ key ] . values ( ) ] <EOL> except KeyError : <EOL> results = [ ] <EOL> if query . has_ancestor ( ) : <EOL> ancestor_path = query . ancestor ( ) . path ( ) . element_list ( ) <EOL> def is_descendant ( entity ) : <EOL> path = entity . key ( ) . _Key__reference . path ( ) . element_list ( ) <EOL> return path [ : len ( ancestor_path ) ] == ancestor_path <EOL> results = filter ( is_descendant , results ) <EOL> operators = { datastore_pb . Query_Filter . LESS_THAN : '<STR_LIT:<>' , <EOL> datastore_pb . Query_Filter . LESS_THAN_OR_EQUAL : '<STR_LIT>' , <EOL> datastore_pb . Query_Filter . GREATER_THAN : '<STR_LIT:>>' , <EOL> datastore_pb . Query_Filter . GREATER_THAN_OR_EQUAL : '<STR_LIT>' , <EOL> datastore_pb . Query_Filter . EQUAL : '<STR_LIT>' , <EOL> } <EOL> def has_prop_indexed ( entity , prop ) : <EOL> """<STR_LIT>""" <EOL> if prop in datastore_types . _SPECIAL_PROPERTIES : <EOL> return True <EOL> elif prop in entity . unindexed_properties ( ) : <EOL> return False <EOL> values = entity . get ( prop , [ ] ) <EOL> if not isinstance ( values , ( tuple , list ) ) : <EOL> values = [ values ] <EOL> for value in values : <EOL> if type ( value ) not in datastore_types . _RAW_PROPERTY_TYPES : <EOL> return True <EOL> return False <EOL> for filt in filters : <EOL> assert filt . op ( ) != datastore_pb . Query_Filter . IN <EOL> prop = filt . property ( <NUM_LIT:0> ) . name ( ) . decode ( '<STR_LIT:utf-8>' ) <EOL> op = operators [ filt . op ( ) ] <EOL> filter_val_list = [ datastore_types . FromPropertyPb ( filter_prop ) <EOL> for filter_prop in filt . property_list ( ) ] <EOL> def passes_filter ( entity ) : <EOL> """<STR_LIT>""" <EOL> if not has_prop_indexed ( entity , prop ) : <EOL> return False <EOL> try : <EOL> entity_vals = datastore . _GetPropertyValue ( entity , prop ) <EOL> except KeyError : <EOL> entity_vals = [ ] <EOL> if not isinstance ( entity_vals , list ) : <EOL> entity_vals = [ entity_vals ] <EOL> for fixed_entity_val in entity_vals : <EOL> for filter_val in filter_val_list : <EOL> fixed_entity_type = self . _PROPERTY_TYPE_TAGS . get ( <EOL> fixed_entity_val . __class__ ) <EOL> filter_type = self . _PROPERTY_TYPE_TAGS . get ( filter_val . __class__ ) <EOL> if fixed_entity_type == filter_type : <EOL> comp = u'<STR_LIT>' % ( fixed_entity_val , op , filter_val ) <EOL> elif op != '<STR_LIT>' : <EOL> comp = '<STR_LIT>' % ( fixed_entity_type , op , filter_type ) <EOL> else : <EOL> continue <EOL> logging . log ( logging . DEBUG - <NUM_LIT:1> , <EOL> '<STR_LIT>' , comp ) <EOL> try : <EOL> ret = eval ( comp ) <EOL> if ret and ret != NotImplementedError : <EOL> return True <EOL> except TypeError : <EOL> pass <EOL> return False <EOL> results = filter ( passes_filter , results ) <EOL> for order in orders : <EOL> prop = order . property ( ) . decode ( '<STR_LIT:utf-8>' ) <EOL> results = [ entity for entity in results if has_prop_indexed ( entity , prop ) ] <EOL> def order_compare_entities ( a , b ) : <EOL> """<STR_LIT>""" <EOL> cmped = <NUM_LIT:0> <EOL> for o in orders : <EOL> prop = o . property ( ) . decode ( '<STR_LIT:utf-8>' ) <EOL> reverse = ( o . direction ( ) is datastore_pb . Query_Order . DESCENDING ) <EOL> a_val = datastore . _GetPropertyValue ( a , prop ) <EOL> if isinstance ( a_val , list ) : <EOL> a_val = sorted ( a_val , order_compare_properties , reverse = reverse ) [ <NUM_LIT:0> ] <EOL> b_val = datastore . _GetPropertyValue ( b , prop ) <EOL> if isinstance ( b_val , list ) : <EOL> b_val = sorted ( b_val , order_compare_properties , reverse = reverse ) [ <NUM_LIT:0> ] <EOL> cmped = order_compare_properties ( a_val , b_val ) <EOL> if o . direction ( ) is datastore_pb . Query_Order . DESCENDING : <EOL> cmped = - cmped <EOL> if cmped != <NUM_LIT:0> : <EOL> return cmped <EOL> if cmped == <NUM_LIT:0> : <EOL> return cmp ( a . key ( ) , b . key ( ) ) <EOL> def order_compare_properties ( x , y ) : <EOL> """<STR_LIT>""" <EOL> if isinstance ( x , datetime . datetime ) : <EOL> x = datastore_types . DatetimeToTimestamp ( x ) <EOL> if isinstance ( y , datetime . datetime ) : <EOL> y = datastore_types . DatetimeToTimestamp ( y ) <EOL> x_type = self . _PROPERTY_TYPE_TAGS . get ( x . __class__ ) <EOL> y_type = self . _PROPERTY_TYPE_TAGS . get ( y . __class__ ) <EOL> if x_type == y_type : <EOL> try : <EOL> return cmp ( x , y ) <EOL> except TypeError : <EOL> return <NUM_LIT:0> <EOL> else : <EOL> return cmp ( x_type , y_type ) <EOL> results . sort ( order_compare_entities ) <EOL> offset = <NUM_LIT:0> <EOL> limit = None <EOL> if query . has_offset ( ) : <EOL> offset = query . offset ( ) <EOL> if query . has_limit ( ) : <EOL> limit = query . limit ( ) <EOL> if limit > _MAXIMUM_RESULTS : <EOL> limit = _MAXIMUM_RESULTS <EOL> results = results [ offset : ] <EOL> clone = datastore_pb . Query ( ) <EOL> clone . CopyFrom ( query ) <EOL> clone . clear_hint ( ) <EOL> if clone in self . __query_history : <EOL> self . __query_history [ clone ] += <NUM_LIT:1> <EOL> else : <EOL> self . __query_history [ clone ] = <NUM_LIT:1> <EOL> if query . has_start_key ( ) and query . start_key ( ) : <EOL> start_key = dict ( parse_qsl ( query . start_key ( ) ) ) <EOL> cursor_handle = int ( start_key . get ( '<STR_LIT>' ) ) <EOL> cursor_offset = start_key . get ( '<STR_LIT>' , None ) <EOL> if cursor_offset is not None : <EOL> cursor_offset = int ( cursor_offset ) <EOL> cursor_offset += offset <EOL> compiled = True <EOL> try : <EOL> cursor = self . __queries [ cursor_handle ] <EOL> except KeyError : <EOL> raise apiproxy_errors . ApplicationError ( <EOL> datastore_pb . Error . BAD_REQUEST , <EOL> '<STR_LIT>' % cursor_handle ) <EOL> else : <EOL> cursor = _Cursor ( results , query . keys_only ( ) , limit = limit ) <EOL> self . __queries [ cursor . cursor ] = cursor <EOL> cursor_offset = None <EOL> compiled = query . compile ( ) <EOL> if count is None : <EOL> if query . has_count ( ) : <EOL> count = query . count ( ) <EOL> elif query . has_limit ( ) : <EOL> count = query . limit ( ) <EOL> else : <EOL> count = _BATCH_SIZE <EOL> cursor . PopulateQueryResult ( <EOL> query_result , count , cursor_offset , compiled = compiled ) <EOL> def _Dynamic_Next ( self , next_request , query_result ) : <EOL> cursor_handle = next_request . cursor ( ) . cursor ( ) <EOL> try : <EOL> cursor = self . __queries [ cursor_handle ] <EOL> except KeyError : <EOL> raise apiproxy_errors . ApplicationError ( <EOL> datastore_pb . Error . BAD_REQUEST , '<STR_LIT>' % cursor_handle ) <EOL> count = _BATCH_SIZE <EOL> if next_request . has_count ( ) : <EOL> count = next_request . count ( ) <EOL> cursor . PopulateQueryResult ( query_result , count ) <EOL> def _Dynamic_Count ( self , query , integer64proto ) : <EOL> self . __ValidateAppId ( query . app ( ) ) <EOL> query_result = datastore_pb . QueryResult ( ) <EOL> count = _MAXIMUM_RESULTS <EOL> if query . has_limit ( ) : <EOL> count = query . limit ( ) <EOL> self . _Dynamic_RunQuery ( query , query_result , count = count ) <EOL> cursor = query_result . cursor ( ) . cursor ( ) <EOL> integer64proto . set_value ( self . __queries [ cursor ] . count ) <EOL> del self . __queries [ cursor ] <EOL> def _Dynamic_BeginTransaction ( self , request , transaction ) : <EOL> self . __tx_handle_lock . acquire ( ) <EOL> handle = self . __next_tx_handle <EOL> self . __next_tx_handle += <NUM_LIT:1> <EOL> self . __tx_handle_lock . release ( ) <EOL> self . __transactions [ handle ] = None <EOL> transaction . set_handle ( handle ) <EOL> self . __tx_lock . acquire ( ) <EOL> snapshot = [ ( app_kind , dict ( entities ) ) <EOL> for app_kind , entities in self . __entities . items ( ) ] <EOL> self . __tx_snapshot = dict ( snapshot ) <EOL> self . __tx_actions = [ ] <EOL> def _Dynamic_AddAction ( self , request , void ) : <EOL> if not self . __transactions . has_key ( request . transaction ( ) . handle ( ) ) : <EOL> raise apiproxy_errors . ApplicationError ( <EOL> datastore_pb . Error . BAD_REQUEST , <EOL> '<STR_LIT>' % request . transaction ( ) . handle ( ) ) <EOL> if len ( self . __tx_actions ) >= _MAX_ACTIONS_PER_TXN : <EOL> raise apiproxy_errors . ApplicationError ( <EOL> datastore_pb . Error . BAD_REQUEST , <EOL> '<STR_LIT>' % _MAX_ACTIONS_PER_TXN ) <EOL> request . clear_transaction ( ) <EOL> self . __tx_actions . append ( request ) <EOL> def _Dynamic_Commit ( self , transaction , transaction_response ) : <EOL> if not self . __transactions . has_key ( transaction . handle ( ) ) : <EOL> raise apiproxy_errors . ApplicationError ( <EOL> datastore_pb . Error . BAD_REQUEST , <EOL> '<STR_LIT>' % transaction . handle ( ) ) <EOL> self . __tx_snapshot = { } <EOL> try : <EOL> self . __WriteDatastore ( ) <EOL> for action in self . __tx_actions : <EOL> try : <EOL> apiproxy_stub_map . MakeSyncCall ( <EOL> '<STR_LIT>' , '<STR_LIT>' , action , api_base_pb . VoidProto ( ) ) <EOL> except apiproxy_errors . ApplicationError , e : <EOL> logging . warning ( '<STR_LIT>' , <EOL> action , e ) <EOL> pass <EOL> finally : <EOL> self . __tx_actions = [ ] <EOL> self . __tx_lock . release ( ) <EOL> def _Dynamic_Rollback ( self , transaction , transaction_response ) : <EOL> if not self . __transactions . has_key ( transaction . handle ( ) ) : <EOL> raise apiproxy_errors . ApplicationError ( <EOL> datastore_pb . Error . BAD_REQUEST , <EOL> '<STR_LIT>' % transaction . handle ( ) ) <EOL> self . __entities = self . __tx_snapshot <EOL> self . __tx_snapshot = { } <EOL> self . __tx_actions = [ ] <EOL> self . __tx_lock . release ( ) <EOL> def _Dynamic_GetSchema ( self , req , schema ) : <EOL> app_str = req . app ( ) <EOL> self . __ValidateAppId ( app_str ) <EOL> kinds = [ ] <EOL> for app , kind in self . __entities : <EOL> if ( app != app_str or <EOL> ( req . has_start_kind ( ) and kind < req . start_kind ( ) ) or <EOL> ( req . has_end_kind ( ) and kind > req . end_kind ( ) ) ) : <EOL> continue <EOL> app_kind = ( app , kind ) <EOL> if app_kind in self . __schema_cache : <EOL> kinds . append ( self . __schema_cache [ app_kind ] ) <EOL> continue <EOL> kind_pb = entity_pb . EntityProto ( ) <EOL> kind_pb . mutable_key ( ) . set_app ( '<STR_LIT>' ) <EOL> kind_pb . mutable_key ( ) . mutable_path ( ) . add_element ( ) . set_type ( kind ) <EOL> kind_pb . mutable_entity_group ( ) <EOL> props = { } <EOL> for entity in self . __entities [ app_kind ] . values ( ) : <EOL> for prop in entity . protobuf . property_list ( ) : <EOL> if prop . name ( ) not in props : <EOL> props [ prop . name ( ) ] = entity_pb . PropertyValue ( ) <EOL> props [ prop . name ( ) ] . MergeFrom ( prop . value ( ) ) <EOL> for value_pb in props . values ( ) : <EOL> if value_pb . has_int64value ( ) : <EOL> value_pb . set_int64value ( <NUM_LIT:0> ) <EOL> if value_pb . has_booleanvalue ( ) : <EOL> value_pb . set_booleanvalue ( False ) <EOL> if value_pb . has_stringvalue ( ) : <EOL> value_pb . set_stringvalue ( '<STR_LIT:none>' ) <EOL> if value_pb . has_doublevalue ( ) : <EOL> value_pb . set_doublevalue ( <NUM_LIT:0.0> ) <EOL> if value_pb . has_pointvalue ( ) : <EOL> value_pb . mutable_pointvalue ( ) . set_x ( <NUM_LIT:0.0> ) <EOL> value_pb . mutable_pointvalue ( ) . set_y ( <NUM_LIT:0.0> ) <EOL> if value_pb . has_uservalue ( ) : <EOL> value_pb . mutable_uservalue ( ) . set_gaiaid ( <NUM_LIT:0> ) <EOL> value_pb . mutable_uservalue ( ) . set_email ( '<STR_LIT:none>' ) <EOL> value_pb . mutable_uservalue ( ) . set_auth_domain ( '<STR_LIT:none>' ) <EOL> value_pb . mutable_uservalue ( ) . clear_nickname ( ) <EOL> value_pb . mutable_uservalue ( ) . clear_obfuscated_gaiaid ( ) <EOL> if value_pb . has_referencevalue ( ) : <EOL> value_pb . clear_referencevalue ( ) <EOL> value_pb . mutable_referencevalue ( ) . set_app ( '<STR_LIT:none>' ) <EOL> pathelem = value_pb . mutable_referencevalue ( ) . add_pathelement ( ) <EOL> pathelem . set_type ( '<STR_LIT:none>' ) <EOL> pathelem . set_name ( '<STR_LIT:none>' ) <EOL> for name , value_pb in props . items ( ) : <EOL> prop_pb = kind_pb . add_property ( ) <EOL> prop_pb . set_name ( name ) <EOL> prop_pb . set_multiple ( False ) <EOL> prop_pb . mutable_value ( ) . CopyFrom ( value_pb ) <EOL> kinds . append ( kind_pb ) <EOL> self . __schema_cache [ app_kind ] = kind_pb <EOL> for kind_pb in kinds : <EOL> kind = schema . add_kind ( ) <EOL> kind . CopyFrom ( kind_pb ) <EOL> if not req . properties ( ) : <EOL> kind . clear_property ( ) <EOL> schema . set_more_results ( False ) <EOL> def _Dynamic_AllocateIds ( self , allocate_ids_request , allocate_ids_response ) : <EOL> model_key = allocate_ids_request . model_key ( ) <EOL> size = allocate_ids_request . size ( ) <EOL> self . __ValidateAppId ( model_key . app ( ) ) <EOL> try : <EOL> self . __id_lock . acquire ( ) <EOL> start = self . __next_id <EOL> self . __next_id += size <EOL> end = self . __next_id - <NUM_LIT:1> <EOL> finally : <EOL> self . __id_lock . release ( ) <EOL> allocate_ids_response . set_start ( start ) <EOL> allocate_ids_response . set_end ( end ) <EOL> def _Dynamic_CreateIndex ( self , index , id_response ) : <EOL> self . __ValidateAppId ( index . app_id ( ) ) <EOL> if index . id ( ) != <NUM_LIT:0> : <EOL> raise apiproxy_errors . ApplicationError ( datastore_pb . Error . BAD_REQUEST , <EOL> '<STR_LIT>' ) <EOL> elif self . __FindIndex ( index ) : <EOL> raise apiproxy_errors . ApplicationError ( datastore_pb . Error . BAD_REQUEST , <EOL> '<STR_LIT>' ) <EOL> self . __index_id_lock . acquire ( ) <EOL> index . set_id ( self . __next_index_id ) <EOL> id_response . set_value ( self . __next_index_id ) <EOL> self . __next_index_id += <NUM_LIT:1> <EOL> self . __index_id_lock . release ( ) <EOL> clone = entity_pb . CompositeIndex ( ) <EOL> clone . CopyFrom ( index ) <EOL> app = index . app_id ( ) <EOL> clone . set_app_id ( app ) <EOL> self . __indexes_lock . acquire ( ) <EOL> try : <EOL> if app not in self . __indexes : <EOL> self . __indexes [ app ] = [ ] <EOL> self . __indexes [ app ] . append ( clone ) <EOL> finally : <EOL> self . __indexes_lock . release ( ) <EOL> def _Dynamic_GetIndices ( self , app_str , composite_indices ) : <EOL> self . __ValidateAppId ( app_str . value ( ) ) <EOL> composite_indices . index_list ( ) . extend ( <EOL> self . __indexes . get ( app_str . value ( ) , [ ] ) ) <EOL> def _Dynamic_UpdateIndex ( self , index , void ) : <EOL> self . __ValidateAppId ( index . app_id ( ) ) <EOL> stored_index = self . __FindIndex ( index ) <EOL> if not stored_index : <EOL> raise apiproxy_errors . ApplicationError ( datastore_pb . Error . BAD_REQUEST , <EOL> "<STR_LIT>" ) <EOL> elif ( index . state ( ) != stored_index . state ( ) and <EOL> index . state ( ) not in self . _INDEX_STATE_TRANSITIONS [ stored_index . state ( ) ] ) : <EOL> raise apiproxy_errors . ApplicationError ( <EOL> datastore_pb . Error . BAD_REQUEST , <EOL> "<STR_LIT>" % <EOL> ( entity_pb . CompositeIndex . State_Name ( stored_index . state ( ) ) , <EOL> ( entity_pb . CompositeIndex . State_Name ( index . state ( ) ) ) ) ) <EOL> self . __indexes_lock . acquire ( ) <EOL> try : <EOL> stored_index . set_state ( index . state ( ) ) <EOL> finally : <EOL> self . __indexes_lock . release ( ) <EOL> def _Dynamic_DeleteIndex ( self , index , void ) : <EOL> self . __ValidateAppId ( index . app_id ( ) ) <EOL> stored_index = self . __FindIndex ( index ) <EOL> if not stored_index : <EOL> raise apiproxy_errors . ApplicationError ( datastore_pb . Error . BAD_REQUEST , <EOL> "<STR_LIT>" ) <EOL> app = index . app_id ( ) <EOL> self . __indexes_lock . acquire ( ) <EOL> try : <EOL> self . __indexes [ app ] . remove ( stored_index ) <EOL> finally : <EOL> self . __indexes_lock . release ( ) <EOL> def __FindIndex ( self , index ) : <EOL> """<STR_LIT>""" <EOL> app = index . app_id ( ) <EOL> self . __ValidateAppId ( app ) <EOL> if app in self . __indexes : <EOL> for stored_index in self . __indexes [ app ] : <EOL> if index . definition ( ) == stored_index . definition ( ) : <EOL> return stored_index <EOL> return None </s>
