<s> """<STR_LIT>""" <EOL> from __future__ import with_statement <EOL> try : <EOL> import hashlib <EOL> _MD5_FUNC = hashlib . md5 <EOL> except ImportError : <EOL> import md5 <EOL> _MD5_FUNC = md5 . new <EOL> import atexit <EOL> import collections <EOL> import itertools <EOL> import logging <EOL> import os <EOL> import random <EOL> import struct <EOL> import threading <EOL> import time <EOL> import weakref <EOL> from google . appengine . api import api_base_pb <EOL> from google . appengine . api import apiproxy_stub_map <EOL> from google . appengine . api import datastore_admin <EOL> from google . appengine . api import datastore_errors <EOL> from google . appengine . api import datastore_types <EOL> from google . appengine . api import yaml_errors <EOL> from google . appengine . api . taskqueue import taskqueue_service_pb <EOL> from google . appengine . datastore import datastore_index <EOL> from google . appengine . datastore import datastore_pb <EOL> from google . appengine . datastore import datastore_pbs <EOL> from google . appengine . datastore import datastore_query <EOL> from google . appengine . datastore import datastore_stub_index <EOL> from google . appengine . datastore import datastore_v4_pb <EOL> from google . appengine . runtime import apiproxy_errors <EOL> from google . net . proto import ProtocolBuffer <EOL> from google . appengine . datastore import entity_pb <EOL> if datastore_pbs . _CLOUD_DATASTORE_ENABLED : <EOL> from google . appengine . datastore . datastore_pbs import googledatastore <EOL> _MAXIMUM_RESULTS = <NUM_LIT> <EOL> _MAXIMUM_QUERY_RESULT_BYTES = <NUM_LIT> <EOL> _MAX_QUERY_OFFSET = <NUM_LIT:1000> <EOL> _PROPERTY_TYPE_NAMES = { <EOL> <NUM_LIT:0> : '<STR_LIT>' , <EOL> entity_pb . PropertyValue . kint64Value : '<STR_LIT>' , <EOL> entity_pb . PropertyValue . kbooleanValue : '<STR_LIT>' , <EOL> entity_pb . PropertyValue . kstringValue : '<STR_LIT>' , <EOL> entity_pb . PropertyValue . kdoubleValue : '<STR_LIT>' , <EOL> entity_pb . PropertyValue . kPointValueGroup : '<STR_LIT>' , <EOL> entity_pb . PropertyValue . kUserValueGroup : '<STR_LIT>' , <EOL> entity_pb . PropertyValue . kReferenceValueGroup : '<STR_LIT>' <EOL> } <EOL> _SCATTER_PROPORTION = <NUM_LIT> <EOL> _MAX_EG_PER_TXN = <NUM_LIT> <EOL> _BLOB_MEANINGS = frozenset ( ( entity_pb . Property . BLOB , <EOL> entity_pb . Property . ENTITY_PROTO , <EOL> entity_pb . Property . TEXT ) ) <EOL> _RETRIES = <NUM_LIT:3> <EOL> _INITIAL_RETRY_DELAY_MS = <NUM_LIT:100> <EOL> _RETRY_DELAY_MULTIPLIER = <NUM_LIT:2> <EOL> _MAX_RETRY_DELAY_MS = <NUM_LIT> <EOL> MINIMUM_VERSION = <NUM_LIT:1> <EOL> SEQUENTIAL = '<STR_LIT>' <EOL> SCATTERED = '<STR_LIT>' <EOL> _MAX_SEQUENTIAL_BIT = <NUM_LIT> <EOL> _MAX_SEQUENTIAL_COUNTER = ( <NUM_LIT:1> << _MAX_SEQUENTIAL_BIT ) - <NUM_LIT:1> <EOL> _MAX_SEQUENTIAL_ID = _MAX_SEQUENTIAL_COUNTER <EOL> _MAX_SCATTERED_COUNTER = ( <NUM_LIT:1> << ( _MAX_SEQUENTIAL_BIT - <NUM_LIT:1> ) ) - <NUM_LIT:1> <EOL> _MAX_SCATTERED_ID = _MAX_SEQUENTIAL_ID + <NUM_LIT:1> + _MAX_SCATTERED_COUNTER <EOL> _SCATTER_SHIFT = <NUM_LIT:64> - _MAX_SEQUENTIAL_BIT + <NUM_LIT:1> <EOL> def _GetScatterProperty ( entity_proto ) : <EOL> """<STR_LIT>""" <EOL> hash_obj = _MD5_FUNC ( ) <EOL> for element in entity_proto . key ( ) . path ( ) . element_list ( ) : <EOL> if element . has_name ( ) : <EOL> hash_obj . update ( element . name ( ) ) <EOL> elif element . has_id ( ) : <EOL> hash_obj . update ( str ( element . id ( ) ) ) <EOL> hash_bytes = hash_obj . digest ( ) [ <NUM_LIT:0> : <NUM_LIT:2> ] <EOL> ( hash_int , ) = struct . unpack ( '<STR_LIT:H>' , hash_bytes ) <EOL> if hash_int >= _SCATTER_PROPORTION : <EOL> return None <EOL> scatter_property = entity_pb . Property ( ) <EOL> scatter_property . set_name ( datastore_types . SCATTER_SPECIAL_PROPERTY ) <EOL> scatter_property . set_meaning ( entity_pb . Property . BYTESTRING ) <EOL> scatter_property . set_multiple ( False ) <EOL> property_value = scatter_property . mutable_value ( ) <EOL> property_value . set_stringvalue ( hash_bytes ) <EOL> return scatter_property <EOL> _SPECIAL_PROPERTY_MAP = { <EOL> datastore_types . SCATTER_SPECIAL_PROPERTY : ( False , True , _GetScatterProperty ) <EOL> } <EOL> def GetInvisibleSpecialPropertyNames ( ) : <EOL> """<STR_LIT>""" <EOL> invisible_names = [ ] <EOL> for name , value in _SPECIAL_PROPERTY_MAP . items ( ) : <EOL> is_visible , _ , _ = value <EOL> if not is_visible : <EOL> invisible_names . append ( name ) <EOL> return invisible_names <EOL> def _PrepareSpecialProperties ( entity_proto , is_load ) : <EOL> """<STR_LIT>""" <EOL> for i in xrange ( entity_proto . property_size ( ) - <NUM_LIT:1> , - <NUM_LIT:1> , - <NUM_LIT:1> ) : <EOL> if _SPECIAL_PROPERTY_MAP . has_key ( entity_proto . property ( i ) . name ( ) ) : <EOL> del entity_proto . property_list ( ) [ i ] <EOL> for is_visible , is_stored , property_func in _SPECIAL_PROPERTY_MAP . values ( ) : <EOL> if is_load : <EOL> should_process = is_visible <EOL> else : <EOL> should_process = is_stored <EOL> if should_process : <EOL> special_property = property_func ( entity_proto ) <EOL> if special_property : <EOL> entity_proto . property_list ( ) . append ( special_property ) <EOL> _METADATA_PROPERTY_NAME = '<STR_LIT>' <EOL> def _FromStorageEntity ( entity ) : <EOL> """<STR_LIT>""" <EOL> clone = entity_pb . EntityProto ( ) <EOL> clone . CopyFrom ( entity ) <EOL> metadata = entity_pb . EntityMetadata ( ) <EOL> for i in xrange ( clone . property_size ( ) - <NUM_LIT:1> , - <NUM_LIT:1> , - <NUM_LIT:1> ) : <EOL> property = clone . property ( i ) <EOL> if _METADATA_PROPERTY_NAME == property . name ( ) : <EOL> del clone . property_list ( ) [ i ] <EOL> metadata = entity_pb . EntityMetadata ( property . value ( ) . stringvalue ( ) ) <EOL> return EntityRecord ( clone , metadata ) <EOL> def _ToStorageEntity ( record ) : <EOL> """<STR_LIT>""" <EOL> if record : <EOL> clone = entity_pb . EntityProto ( ) <EOL> clone . CopyFrom ( record . entity ) <EOL> serialized_metadata = record . metadata . SerializeToString ( ) <EOL> metadata_property = clone . add_property ( ) <EOL> metadata_property . set_name ( _METADATA_PROPERTY_NAME ) <EOL> metadata_property . set_meaning ( entity_pb . Property . BLOB ) <EOL> metadata_property . set_multiple ( False ) <EOL> metadata_property . mutable_value ( ) . set_stringvalue ( serialized_metadata ) <EOL> return clone <EOL> def _GetGroupByKey ( entity , property_names ) : <EOL> """<STR_LIT>""" <EOL> return frozenset ( ( prop . name ( ) , prop . value ( ) . SerializeToString ( ) ) <EOL> for prop in entity . property_list ( ) <EOL> if prop . name ( ) in property_names ) <EOL> def PrepareSpecialPropertiesForStore ( entity_proto ) : <EOL> """<STR_LIT>""" <EOL> _PrepareSpecialProperties ( entity_proto , False ) <EOL> def LoadEntity ( entity , keys_only = False , property_names = None ) : <EOL> """<STR_LIT>""" <EOL> if entity : <EOL> clone = entity_pb . EntityProto ( ) <EOL> if property_names : <EOL> clone . mutable_key ( ) . CopyFrom ( entity . key ( ) ) <EOL> clone . mutable_entity_group ( ) <EOL> seen = set ( ) <EOL> for prop in entity . property_list ( ) : <EOL> if prop . name ( ) in property_names : <EOL> Check ( prop . name ( ) not in seen , <EOL> '<STR_LIT>' , <EOL> datastore_pb . Error . INTERNAL_ERROR ) <EOL> seen . add ( prop . name ( ) ) <EOL> new_prop = clone . add_property ( ) <EOL> new_prop . set_name ( prop . name ( ) ) <EOL> new_prop . set_meaning ( entity_pb . Property . INDEX_VALUE ) <EOL> new_prop . mutable_value ( ) . CopyFrom ( prop . value ( ) ) <EOL> new_prop . set_multiple ( False ) <EOL> elif keys_only : <EOL> clone . mutable_key ( ) . CopyFrom ( entity . key ( ) ) <EOL> clone . mutable_entity_group ( ) <EOL> else : <EOL> clone . CopyFrom ( entity ) <EOL> PrepareSpecialPropertiesForLoad ( clone ) <EOL> return clone <EOL> def LoadRecord ( record , keys_only = False , property_names = None ) : <EOL> """<STR_LIT>""" <EOL> if record : <EOL> metadata = record . metadata <EOL> if keys_only or property_names : <EOL> metadata = entity_pb . EntityMetadata ( ) <EOL> return EntityRecord ( LoadEntity ( record . entity , keys_only , property_names ) , <EOL> metadata ) <EOL> def StoreRecord ( record ) : <EOL> """<STR_LIT>""" <EOL> clone = entity_pb . EntityProto ( ) <EOL> clone . CopyFrom ( record . entity ) <EOL> PrepareSpecialPropertiesForStore ( clone ) <EOL> return EntityRecord ( clone , record . metadata ) <EOL> def PrepareSpecialPropertiesForLoad ( entity_proto ) : <EOL> """<STR_LIT>""" <EOL> _PrepareSpecialProperties ( entity_proto , True ) <EOL> def Check ( test , msg = '<STR_LIT>' , error_code = datastore_pb . Error . BAD_REQUEST ) : <EOL> """<STR_LIT>""" <EOL> if not test : <EOL> raise apiproxy_errors . ApplicationError ( error_code , msg ) <EOL> def CheckValidUTF8 ( string , desc ) : <EOL> """<STR_LIT>""" <EOL> if isinstance ( string , unicode ) : <EOL> return True <EOL> try : <EOL> string . decode ( '<STR_LIT:utf-8>' ) <EOL> except UnicodeDecodeError : <EOL> Check ( False , '<STR_LIT>' % desc ) <EOL> def CheckAppId ( request_trusted , request_app_id , app_id ) : <EOL> """<STR_LIT>""" <EOL> assert app_id <EOL> CheckValidUTF8 ( app_id , '<STR_LIT>' ) <EOL> Check ( request_trusted or app_id == request_app_id , <EOL> '<STR_LIT>' % ( request_app_id , app_id ) ) <EOL> def CheckReference ( request_trusted , <EOL> request_app_id , <EOL> key , <EOL> require_id_or_name = True ) : <EOL> """<STR_LIT>""" <EOL> assert isinstance ( key , entity_pb . Reference ) <EOL> CheckAppId ( request_trusted , request_app_id , key . app ( ) ) <EOL> Check ( key . path ( ) . element_size ( ) > <NUM_LIT:0> , '<STR_LIT>' ) <EOL> if require_id_or_name and not datastore_pbs . is_complete_v3_key ( key ) : <EOL> raise datastore_errors . BadRequestError ( '<STR_LIT>' ) <EOL> for elem in key . path ( ) . element_list ( ) : <EOL> Check ( not elem . has_id ( ) or not elem . has_name ( ) , <EOL> '<STR_LIT>' % key ) <EOL> CheckValidUTF8 ( elem . type ( ) , '<STR_LIT>' ) <EOL> if elem . has_name ( ) : <EOL> CheckValidUTF8 ( elem . name ( ) , '<STR_LIT>' ) <EOL> def CheckEntity ( request_trusted , request_app_id , entity ) : <EOL> """<STR_LIT>""" <EOL> CheckReference ( request_trusted , request_app_id , entity . key ( ) , False ) <EOL> for prop in entity . property_list ( ) : <EOL> CheckProperty ( request_trusted , request_app_id , prop ) <EOL> for prop in entity . raw_property_list ( ) : <EOL> CheckProperty ( request_trusted , request_app_id , prop , indexed = False ) <EOL> def CheckProperty ( request_trusted , request_app_id , prop , indexed = True ) : <EOL> """<STR_LIT>""" <EOL> name = prop . name ( ) <EOL> value = prop . value ( ) <EOL> meaning = prop . meaning ( ) <EOL> CheckValidUTF8 ( name , '<STR_LIT>' ) <EOL> Check ( request_trusted or <EOL> not datastore_types . RESERVED_PROPERTY_NAME . match ( name ) , <EOL> '<STR_LIT>' % name ) <EOL> Check ( prop . meaning ( ) != entity_pb . Property . INDEX_VALUE , <EOL> '<STR_LIT>' ) <EOL> is_blob = meaning in _BLOB_MEANINGS <EOL> if indexed : <EOL> Check ( not is_blob , <EOL> '<STR_LIT>' + name + <EOL> '<STR_LIT>' ) <EOL> max_length = datastore_types . _MAX_STRING_LENGTH <EOL> else : <EOL> if is_blob : <EOL> Check ( value . has_stringvalue ( ) , <EOL> '<STR_LIT>' + name + <EOL> '<STR_LIT>' ) <EOL> max_length = datastore_types . _MAX_RAW_PROPERTY_BYTES <EOL> if meaning == entity_pb . Property . ATOM_LINK : <EOL> max_length = datastore_types . _MAX_LINK_PROPERTY_LENGTH <EOL> CheckPropertyValue ( name , value , max_length , meaning ) <EOL> def CheckPropertyValue ( name , value , max_length , meaning ) : <EOL> """<STR_LIT>""" <EOL> num_values = ( value . has_int64value ( ) + <EOL> value . has_stringvalue ( ) + <EOL> value . has_booleanvalue ( ) + <EOL> value . has_doublevalue ( ) + <EOL> value . has_pointvalue ( ) + <EOL> value . has_uservalue ( ) + <EOL> value . has_referencevalue ( ) ) <EOL> Check ( num_values <= <NUM_LIT:1> , '<STR_LIT>' + name + <EOL> '<STR_LIT>' ) <EOL> if value . has_stringvalue ( ) : <EOL> s = value . stringvalue ( ) <EOL> if isinstance ( s , unicode ) : <EOL> s = s . encode ( '<STR_LIT:utf-8>' ) <EOL> Check ( len ( s ) <= max_length , <EOL> '<STR_LIT>' % ( name , max_length ) ) <EOL> if ( meaning not in _BLOB_MEANINGS and <EOL> meaning != entity_pb . Property . BYTESTRING ) : <EOL> CheckValidUTF8 ( value . stringvalue ( ) , '<STR_LIT>' % name ) <EOL> def CheckTransaction ( request_trusted , request_app_id , transaction ) : <EOL> """<STR_LIT>""" <EOL> assert isinstance ( transaction , datastore_pb . Transaction ) <EOL> CheckAppId ( request_trusted , request_app_id , transaction . app ( ) ) <EOL> def CheckQuery ( query , filters , orders , max_query_components ) : <EOL> """<STR_LIT>""" <EOL> Check ( query . property_name_size ( ) == <NUM_LIT:0> or not query . keys_only ( ) , <EOL> '<STR_LIT>' ) <EOL> projected_properties = set ( query . property_name_list ( ) ) <EOL> for prop_name in query . property_name_list ( ) : <EOL> Check ( not datastore_types . RESERVED_PROPERTY_NAME . match ( prop_name ) , <EOL> '<STR_LIT>' + prop_name ) <EOL> Check ( len ( projected_properties ) == len ( query . property_name_list ( ) ) , <EOL> '<STR_LIT>' ) <EOL> key_prop_name = datastore_types . KEY_SPECIAL_PROPERTY <EOL> unapplied_log_timestamp_us_name = ( <EOL> datastore_types . _UNAPPLIED_LOG_TIMESTAMP_SPECIAL_PROPERTY ) <EOL> if query . has_transaction ( ) : <EOL> Check ( query . has_ancestor ( ) , <EOL> '<STR_LIT>' ) <EOL> num_components = len ( filters ) + len ( orders ) <EOL> if query . has_ancestor ( ) : <EOL> num_components += <NUM_LIT:1> <EOL> Check ( num_components <= max_query_components , <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' % max_query_components ) <EOL> if query . has_ancestor ( ) : <EOL> ancestor = query . ancestor ( ) <EOL> Check ( query . app ( ) == ancestor . app ( ) , <EOL> '<STR_LIT>' % <EOL> ( query . app ( ) , ancestor . app ( ) ) ) <EOL> Check ( query . name_space ( ) == ancestor . name_space ( ) , <EOL> '<STR_LIT>' % <EOL> ( query . name_space ( ) , ancestor . name_space ( ) ) ) <EOL> if query . group_by_property_name_size ( ) : <EOL> group_by_set = set ( query . group_by_property_name_list ( ) ) <EOL> for order in orders : <EOL> if not group_by_set : <EOL> break <EOL> Check ( order . property ( ) in group_by_set , <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> group_by_set . remove ( order . property ( ) ) <EOL> ineq_prop_name = None <EOL> for filter in filters : <EOL> Check ( filter . property_size ( ) == <NUM_LIT:1> , <EOL> '<STR_LIT>' % filter . property_size ( ) ) <EOL> prop = filter . property ( <NUM_LIT:0> ) <EOL> prop_name = prop . name ( ) . decode ( '<STR_LIT:utf-8>' ) <EOL> if prop_name == key_prop_name : <EOL> Check ( prop . value ( ) . has_referencevalue ( ) , <EOL> '<STR_LIT>' % key_prop_name ) <EOL> ref_val = prop . value ( ) . referencevalue ( ) <EOL> Check ( ref_val . app ( ) == query . app ( ) , <EOL> '<STR_LIT>' % <EOL> ( key_prop_name , ref_val . app ( ) , query . app ( ) ) ) <EOL> Check ( ref_val . name_space ( ) == query . name_space ( ) , <EOL> '<STR_LIT>' % <EOL> ( key_prop_name , ref_val . name_space ( ) , query . name_space ( ) ) ) <EOL> if filter . op ( ) in datastore_index . EQUALITY_OPERATORS : <EOL> Check ( prop_name not in projected_properties , <EOL> '<STR_LIT>' ) <EOL> if ( filter . op ( ) in datastore_index . INEQUALITY_OPERATORS and <EOL> prop_name != unapplied_log_timestamp_us_name ) : <EOL> if ineq_prop_name is None : <EOL> ineq_prop_name = prop_name <EOL> else : <EOL> Check ( ineq_prop_name == prop_name , <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' % ( ineq_prop_name , prop_name ) ) <EOL> if ( ineq_prop_name is not None <EOL> and query . group_by_property_name_size ( ) > <NUM_LIT:0> <EOL> and not orders ) : <EOL> Check ( ineq_prop_name in group_by_set , <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> % ( ineq_prop_name ) ) <EOL> if ineq_prop_name is not None and orders : <EOL> first_order_prop = orders [ <NUM_LIT:0> ] . property ( ) . decode ( '<STR_LIT:utf-8>' ) <EOL> Check ( first_order_prop == ineq_prop_name , <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' % ( first_order_prop , ineq_prop_name ) ) <EOL> if not query . has_kind ( ) : <EOL> for filter in filters : <EOL> prop_name = filter . property ( <NUM_LIT:0> ) . name ( ) . decode ( '<STR_LIT:utf-8>' ) <EOL> Check ( prop_name == key_prop_name or <EOL> prop_name == unapplied_log_timestamp_us_name , <EOL> '<STR_LIT>' ) <EOL> for order in orders : <EOL> prop_name = order . property ( ) . decode ( '<STR_LIT:utf-8>' ) <EOL> Check ( prop_name == key_prop_name and <EOL> order . direction ( ) is datastore_pb . Query_Order . ASCENDING , <EOL> '<STR_LIT>' ) <EOL> class ValueRange ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self ) : <EOL> """<STR_LIT>""" <EOL> self . __start = self . __end = None <EOL> self . __start_inclusive = self . __end_inclusive = False <EOL> def Update ( self , rel_op , limit ) : <EOL> """<STR_LIT>""" <EOL> if rel_op == datastore_pb . Query_Filter . LESS_THAN : <EOL> if self . __end is None or limit <= self . __end : <EOL> self . __end = limit <EOL> self . __end_inclusive = False <EOL> elif ( rel_op == datastore_pb . Query_Filter . LESS_THAN_OR_EQUAL or <EOL> rel_op == datastore_pb . Query_Filter . EQUAL ) : <EOL> if self . __end is None or limit < self . __end : <EOL> self . __end = limit <EOL> self . __end_inclusive = True <EOL> if rel_op == datastore_pb . Query_Filter . GREATER_THAN : <EOL> if self . __start is None or limit >= self . __start : <EOL> self . __start = limit <EOL> self . __start_inclusive = False <EOL> elif ( rel_op == datastore_pb . Query_Filter . GREATER_THAN_OR_EQUAL or <EOL> rel_op == datastore_pb . Query_Filter . EQUAL ) : <EOL> if self . __start is None or limit > self . __start : <EOL> self . __start = limit <EOL> self . __start_inclusive = True <EOL> def Contains ( self , value ) : <EOL> """<STR_LIT>""" <EOL> if self . __start is not None : <EOL> if self . __start_inclusive and value < self . __start : return False <EOL> if not self . __start_inclusive and value <= self . __start : return False <EOL> if self . __end is not None : <EOL> if self . __end_inclusive and value > self . __end : return False <EOL> if not self . __end_inclusive and value >= self . __end : return False <EOL> return True <EOL> def Remap ( self , mapper ) : <EOL> """<STR_LIT>""" <EOL> self . __start = self . __start and mapper ( self . __start ) <EOL> self . __end = self . __end and mapper ( self . __end ) <EOL> def MapExtremes ( self , mapper ) : <EOL> """<STR_LIT>""" <EOL> return ( <EOL> self . __start and mapper ( self . __start , self . __start_inclusive , False ) , <EOL> self . __end and mapper ( self . __end , self . __end_inclusive , True ) ) <EOL> def ParseKeyFilteredQuery ( filters , orders ) : <EOL> """<STR_LIT>""" <EOL> remaining_filters = [ ] <EOL> key_range = ValueRange ( ) <EOL> key_prop = datastore_types . KEY_SPECIAL_PROPERTY <EOL> for f in filters : <EOL> op = f . op ( ) <EOL> if not ( f . property_size ( ) == <NUM_LIT:1> and <EOL> f . property ( <NUM_LIT:0> ) . name ( ) == key_prop and <EOL> not ( op == datastore_pb . Query_Filter . IN or <EOL> op == datastore_pb . Query_Filter . EXISTS ) ) : <EOL> remaining_filters . append ( f ) <EOL> continue <EOL> val = f . property ( <NUM_LIT:0> ) . value ( ) <EOL> Check ( val . has_referencevalue ( ) , '<STR_LIT>' ) <EOL> limit = datastore_types . FromReferenceProperty ( val ) <EOL> key_range . Update ( op , limit ) <EOL> remaining_orders = [ ] <EOL> for o in orders : <EOL> if not ( o . direction ( ) == datastore_pb . Query_Order . ASCENDING and <EOL> o . property ( ) == datastore_types . KEY_SPECIAL_PROPERTY ) : <EOL> remaining_orders . append ( o ) <EOL> else : <EOL> break <EOL> Check ( not remaining_filters , <EOL> '<STR_LIT>' + key_prop + '<STR_LIT>' ) <EOL> Check ( not remaining_orders , <EOL> '<STR_LIT>' + key_prop + '<STR_LIT>' ) <EOL> return key_range <EOL> def ParseKindQuery ( query , filters , orders ) : <EOL> """<STR_LIT>""" <EOL> Check ( not query . has_ancestor ( ) , '<STR_LIT>' ) <EOL> key_range = ParseKeyFilteredQuery ( filters , orders ) <EOL> key_range . Remap ( _KindKeyToString ) <EOL> return key_range <EOL> def _KindKeyToString ( key ) : <EOL> """<STR_LIT>""" <EOL> key_path = key . to_path ( ) <EOL> if ( len ( key_path ) == <NUM_LIT:2> and key_path [ <NUM_LIT:0> ] == '<STR_LIT>' and <EOL> isinstance ( key_path [ <NUM_LIT:1> ] , basestring ) ) : <EOL> return key_path [ <NUM_LIT:1> ] <EOL> Check ( False , '<STR_LIT>' ) <EOL> def ParseNamespaceQuery ( query , filters , orders ) : <EOL> """<STR_LIT>""" <EOL> Check ( not query . has_ancestor ( ) , <EOL> '<STR_LIT>' ) <EOL> key_range = ParseKeyFilteredQuery ( filters , orders ) <EOL> key_range . Remap ( _NamespaceKeyToString ) <EOL> return key_range <EOL> def _NamespaceKeyToString ( key ) : <EOL> """<STR_LIT>""" <EOL> key_path = key . to_path ( ) <EOL> if len ( key_path ) == <NUM_LIT:2> and key_path [ <NUM_LIT:0> ] == '<STR_LIT>' : <EOL> if key_path [ <NUM_LIT:1> ] == datastore_types . _EMPTY_NAMESPACE_ID : <EOL> return '<STR_LIT>' <EOL> if isinstance ( key_path [ <NUM_LIT:1> ] , basestring ) : <EOL> return key_path [ <NUM_LIT:1> ] <EOL> Check ( False , '<STR_LIT>' ) <EOL> def ParsePropertyQuery ( query , filters , orders ) : <EOL> """<STR_LIT>""" <EOL> Check ( not query . has_transaction ( ) , <EOL> '<STR_LIT>' ) <EOL> key_range = ParseKeyFilteredQuery ( filters , orders ) <EOL> key_range . Remap ( lambda x : _PropertyKeyToString ( x , '<STR_LIT>' ) ) <EOL> if query . has_ancestor ( ) : <EOL> ancestor = datastore_types . Key . _FromPb ( query . ancestor ( ) ) <EOL> ancestor_kind , ancestor_property = _PropertyKeyToString ( ancestor , None ) <EOL> if ancestor_property is not None : <EOL> key_range . Update ( datastore_pb . Query_Filter . EQUAL , <EOL> ( ancestor_kind , ancestor_property ) ) <EOL> else : <EOL> key_range . Update ( datastore_pb . Query_Filter . GREATER_THAN_OR_EQUAL , <EOL> ( ancestor_kind , '<STR_LIT>' ) ) <EOL> key_range . Update ( datastore_pb . Query_Filter . LESS_THAN_OR_EQUAL , <EOL> ( ancestor_kind + '<STR_LIT>' , '<STR_LIT>' ) ) <EOL> query . clear_ancestor ( ) <EOL> return key_range <EOL> def _PropertyKeyToString ( key , default_property ) : <EOL> """<STR_LIT>""" <EOL> key_path = key . to_path ( ) <EOL> if ( len ( key_path ) == <NUM_LIT:2> and <EOL> key_path [ <NUM_LIT:0> ] == '<STR_LIT>' and isinstance ( key_path [ <NUM_LIT:1> ] , basestring ) ) : <EOL> return ( key_path [ <NUM_LIT:1> ] , default_property ) <EOL> if ( len ( key_path ) == <NUM_LIT:4> and <EOL> key_path [ <NUM_LIT:0> ] == '<STR_LIT>' and isinstance ( key_path [ <NUM_LIT:1> ] , basestring ) and <EOL> key_path [ <NUM_LIT:2> ] == '<STR_LIT>' and isinstance ( key_path [ <NUM_LIT:3> ] , basestring ) ) : <EOL> return ( key_path [ <NUM_LIT:1> ] , key_path [ <NUM_LIT:3> ] ) <EOL> Check ( False , '<STR_LIT>' ) <EOL> def SynthesizeUserId ( email ) : <EOL> """<STR_LIT>""" <EOL> user_id_digest = _MD5_FUNC ( email . lower ( ) ) . digest ( ) <EOL> user_id = '<STR_LIT:1>' + '<STR_LIT>' . join ( [ '<STR_LIT>' % ord ( x ) for x in user_id_digest ] ) [ : <NUM_LIT:20> ] <EOL> return user_id <EOL> def FillUsersInQuery ( filters ) : <EOL> """<STR_LIT>""" <EOL> for filter in filters : <EOL> for property in filter . property_list ( ) : <EOL> FillUser ( property ) <EOL> def FillUser ( property ) : <EOL> """<STR_LIT>""" <EOL> if property . value ( ) . has_uservalue ( ) : <EOL> uid = SynthesizeUserId ( property . value ( ) . uservalue ( ) . email ( ) ) <EOL> if uid : <EOL> property . mutable_value ( ) . mutable_uservalue ( ) . set_obfuscated_gaiaid ( uid ) <EOL> class BaseCursor ( object ) : <EOL> """<STR_LIT>""" <EOL> _next_cursor = <NUM_LIT:1> <EOL> _next_cursor_lock = threading . Lock ( ) <EOL> def __init__ ( self , query , dsquery , orders , index_list ) : <EOL> """<STR_LIT>""" <EOL> self . keys_only = query . keys_only ( ) <EOL> self . property_names = set ( query . property_name_list ( ) ) <EOL> self . group_by = set ( query . group_by_property_name_list ( ) ) <EOL> self . app = query . app ( ) <EOL> self . cursor = self . _AcquireCursorID ( ) <EOL> if query . has_count ( ) : <EOL> count = query . count ( ) <EOL> elif query . has_limit ( ) : <EOL> count = query . limit ( ) <EOL> else : <EOL> count = BaseDatastore . _BATCH_SIZE <EOL> self . __use_persisted_offset = query . persist_offset ( ) <EOL> self . __persisted_offset = query . offset ( ) <EOL> self . __persisted_count = count <EOL> self . __order_compare_entities = dsquery . _order . cmp_for_filter ( <EOL> dsquery . _filter_predicate ) <EOL> if self . group_by : <EOL> self . __cursor_properties = self . group_by <EOL> else : <EOL> self . __cursor_properties = set ( order . property ( ) for order in orders ) <EOL> self . __cursor_properties . add ( '<STR_LIT>' ) <EOL> self . __cursor_properties = frozenset ( self . __cursor_properties ) <EOL> self . __first_sort_order = orders [ <NUM_LIT:0> ] . direction ( ) <EOL> self . __index_list = index_list <EOL> def _PopulateResultMetadata ( self , query_result , compile , <EOL> first_result , last_result ) : <EOL> query_result . set_keys_only ( self . keys_only ) <EOL> if query_result . more_results ( ) : <EOL> cursor = query_result . mutable_cursor ( ) <EOL> cursor . set_app ( self . app ) <EOL> cursor . set_cursor ( self . cursor ) <EOL> if compile : <EOL> self . _EncodeCompiledCursor ( last_result , <EOL> query_result . mutable_compiled_cursor ( ) ) <EOL> if first_result : <EOL> query_result . index_list ( ) . extend ( self . __index_list ) <EOL> @ classmethod <EOL> def _AcquireCursorID ( cls ) : <EOL> """<STR_LIT>""" <EOL> cls . _next_cursor_lock . acquire ( ) <EOL> try : <EOL> cursor_id = cls . _next_cursor <EOL> cls . _next_cursor += <NUM_LIT:1> <EOL> finally : <EOL> cls . _next_cursor_lock . release ( ) <EOL> return cursor_id <EOL> def _IsBeforeCursor ( self , record , cursor ) : <EOL> """<STR_LIT>""" <EOL> comparison_entity = entity_pb . EntityProto ( ) <EOL> for prop in record . entity . property_list ( ) : <EOL> if prop . name ( ) in self . __cursor_properties : <EOL> comparison_entity . add_property ( ) . MergeFrom ( prop ) <EOL> if cursor [ <NUM_LIT:0> ] . has_key ( ) : <EOL> comparison_entity . mutable_key ( ) . MergeFrom ( record . entity . key ( ) ) <EOL> x = self . __order_compare_entities ( comparison_entity , cursor [ <NUM_LIT:0> ] ) <EOL> if cursor [ <NUM_LIT:1> ] : <EOL> return x < <NUM_LIT:0> <EOL> else : <EOL> return x <= <NUM_LIT:0> <EOL> def _DecodeCompiledCursor ( self , compiled_cursor ) : <EOL> """<STR_LIT>""" <EOL> assert compiled_cursor . has_postfix_position ( ) <EOL> position = compiled_cursor . postfix_position ( ) <EOL> remaining_properties = set ( self . __cursor_properties ) <EOL> cursor_entity = entity_pb . EntityProto ( ) <EOL> if position . has_key ( ) : <EOL> cursor_entity . mutable_key ( ) . CopyFrom ( position . key ( ) ) <EOL> try : <EOL> remaining_properties . remove ( '<STR_LIT>' ) <EOL> except KeyError : <EOL> Check ( False , '<STR_LIT>' ) <EOL> for index_value in position . index_value_list ( ) : <EOL> prop = cursor_entity . add_property ( ) <EOL> prop . set_name ( index_value . property_name ( ) ) <EOL> prop . mutable_value ( ) . CopyFrom ( index_value . value ( ) ) <EOL> try : <EOL> remaining_properties . remove ( index_value . property_name ( ) ) <EOL> except KeyError : <EOL> Check ( False , '<STR_LIT>' % <EOL> index_value . property_name ( ) ) <EOL> Check ( not remaining_properties , <EOL> '<STR_LIT>' % <EOL> remaining_properties ) <EOL> return ( cursor_entity , position . before ( ) ) <EOL> def _EncodeCompiledCursor ( self , last_result , compiled_cursor ) : <EOL> """<STR_LIT>""" <EOL> if last_result is not None : <EOL> position = compiled_cursor . mutable_postfix_position ( ) <EOL> if '<STR_LIT>' in self . __cursor_properties : <EOL> position . mutable_key ( ) . MergeFrom ( last_result . key ( ) ) <EOL> for prop in last_result . property_list ( ) : <EOL> if prop . name ( ) in self . __cursor_properties : <EOL> index_value = position . add_index_value ( ) <EOL> index_value . set_property_name ( prop . name ( ) ) <EOL> index_value . mutable_value ( ) . CopyFrom ( prop . value ( ) ) <EOL> position . set_before ( False ) <EOL> _SetBeforeAscending ( position , self . __first_sort_order ) <EOL> def PopulateQueryResult ( self , result , count , deprecated_offset , <EOL> compile = False , first_result = False ) : <EOL> """<STR_LIT>""" <EOL> if count is None : <EOL> count = self . __persisted_count <EOL> if ( deprecated_offset is not None <EOL> and self . __persisted_offset != deprecated_offset ) : <EOL> raise datastore_errors . BadArgumentError ( <EOL> '<STR_LIT>' <EOL> % ( deprecated_offset , self . __persisted_offset ) ) <EOL> self . _PopulateQueryResult ( result , count , self . __persisted_offset , <EOL> compile , first_result ) <EOL> self . __persisted_offset -= result . skipped_results ( ) <EOL> def _PopulateQueryResult ( self , result , count , offset , <EOL> compile , first_result ) : <EOL> raise NotImplementedError <EOL> class ListCursor ( BaseCursor ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , query , dsquery , orders , index_list , results ) : <EOL> """<STR_LIT>""" <EOL> super ( ListCursor , self ) . __init__ ( query , dsquery , orders , index_list ) <EOL> if self . group_by : <EOL> distincts = set ( ) <EOL> new_results = [ ] <EOL> for result in results : <EOL> key_value = _GetGroupByKey ( result . entity , self . group_by ) <EOL> if key_value not in distincts : <EOL> distincts . add ( key_value ) <EOL> new_results . append ( result ) <EOL> results = new_results <EOL> if ( query . has_compiled_cursor ( ) <EOL> and query . compiled_cursor ( ) . has_postfix_position ( ) ) : <EOL> start_cursor = self . _DecodeCompiledCursor ( query . compiled_cursor ( ) ) <EOL> self . __last_result = start_cursor [ <NUM_LIT:0> ] <EOL> start_cursor_position = self . _GetCursorOffset ( results , start_cursor ) <EOL> else : <EOL> self . __last_result = None <EOL> start_cursor_position = <NUM_LIT:0> <EOL> if query . has_end_compiled_cursor ( ) : <EOL> if query . end_compiled_cursor ( ) . has_postfix_position ( ) : <EOL> end_cursor = self . _DecodeCompiledCursor ( query . end_compiled_cursor ( ) ) <EOL> end_cursor_position = self . _GetCursorOffset ( results , end_cursor ) <EOL> else : <EOL> end_cursor_position = <NUM_LIT:0> <EOL> else : <EOL> end_cursor_position = len ( results ) <EOL> results = results [ start_cursor_position : end_cursor_position ] <EOL> if query . has_limit ( ) : <EOL> limit = query . limit ( ) <EOL> if query . offset ( ) : <EOL> limit += query . offset ( ) <EOL> if limit >= <NUM_LIT:0> and limit < len ( results ) : <EOL> results = results [ : limit ] <EOL> self . __results = results <EOL> self . __offset = <NUM_LIT:0> <EOL> self . __count = len ( self . __results ) <EOL> def _GetCursorOffset ( self , results , cursor ) : <EOL> """<STR_LIT>""" <EOL> lo = <NUM_LIT:0> <EOL> hi = len ( results ) <EOL> while lo < hi : <EOL> mid = ( lo + hi ) // <NUM_LIT:2> <EOL> if self . _IsBeforeCursor ( results [ mid ] , cursor ) : <EOL> lo = mid + <NUM_LIT:1> <EOL> else : <EOL> hi = mid <EOL> return lo <EOL> def _PopulateQueryResult ( self , result , count , offset , compile , first_result ) : <EOL> Check ( offset >= <NUM_LIT:0> , '<STR_LIT>' ) <EOL> offset = min ( offset , self . __count - self . __offset ) <EOL> limited_offset = min ( offset , _MAX_QUERY_OFFSET ) <EOL> if limited_offset : <EOL> self . __offset += limited_offset <EOL> result . set_skipped_results ( limited_offset ) <EOL> if compile and result . skipped_results ( ) > <NUM_LIT:0> : <EOL> self . _EncodeCompiledCursor ( <EOL> self . __results [ self . __offset - <NUM_LIT:1> ] . entity , <EOL> result . mutable_skipped_results_compiled_cursor ( ) ) <EOL> if offset == limited_offset and count : <EOL> if count > _MAXIMUM_RESULTS : <EOL> count = _MAXIMUM_RESULTS <EOL> results = self . __results [ self . __offset : self . __offset + count ] <EOL> count = len ( results ) <EOL> self . __offset += count <EOL> records = [ LoadRecord ( record , self . keys_only , self . property_names ) <EOL> for record in results ] <EOL> entities = [ record . entity for record in records ] <EOL> versions = [ record . metadata . updated_version ( ) for record in records <EOL> if record . metadata . has_updated_version ( ) ] <EOL> result . result_list ( ) . extend ( entities ) <EOL> if len ( versions ) == len ( entities ) : <EOL> result . version_list ( ) . extend ( versions ) <EOL> if compile : <EOL> for record in results : <EOL> self . _EncodeCompiledCursor ( record . entity , <EOL> result . add_result_compiled_cursor ( ) ) <EOL> if self . __offset : <EOL> self . __last_result = self . __results [ self . __offset - <NUM_LIT:1> ] . entity <EOL> result . set_more_results ( self . __offset < self . __count ) <EOL> self . _PopulateResultMetadata ( result , compile , <EOL> first_result , self . __last_result ) <EOL> def _SynchronizeTxn ( function ) : <EOL> """<STR_LIT>""" <EOL> def sync ( txn , * args , ** kwargs ) : <EOL> txn . _lock . acquire ( ) <EOL> try : <EOL> Check ( txn . _state is LiveTxn . ACTIVE , '<STR_LIT>' ) <EOL> return function ( txn , * args , ** kwargs ) <EOL> finally : <EOL> txn . _lock . release ( ) <EOL> return sync <EOL> def _GetEntityGroup ( ref ) : <EOL> """<STR_LIT>""" <EOL> entity_group = entity_pb . Reference ( ) <EOL> entity_group . CopyFrom ( ref ) <EOL> assert ( entity_group . path ( ) . element_list ( ) [ <NUM_LIT:0> ] . has_id ( ) or <EOL> entity_group . path ( ) . element_list ( ) [ <NUM_LIT:0> ] . has_name ( ) ) <EOL> del entity_group . path ( ) . element_list ( ) [ <NUM_LIT:1> : ] <EOL> return entity_group <EOL> def _GetKeyKind ( key ) : <EOL> """<STR_LIT>""" <EOL> return key . path ( ) . element_list ( ) [ - <NUM_LIT:1> ] . type ( ) <EOL> def _FilterIndexesByKind ( key , indexes ) : <EOL> """<STR_LIT>""" <EOL> return filter ( ( lambda index : <EOL> index . definition ( ) . entity_type ( ) == _GetKeyKind ( key ) ) , indexes ) <EOL> class LiveTxn ( object ) : <EOL> """<STR_LIT>""" <EOL> ACTIVE = <NUM_LIT:1> <EOL> COMMITTED = <NUM_LIT:2> <EOL> ROLLEDBACK = <NUM_LIT:3> <EOL> FAILED = <NUM_LIT:4> <EOL> _state = ACTIVE <EOL> _commit_time_s = None <EOL> def __init__ ( self , txn_manager , app , allow_multiple_eg ) : <EOL> assert isinstance ( txn_manager , BaseTransactionManager ) <EOL> assert isinstance ( app , basestring ) <EOL> self . _txn_manager = txn_manager <EOL> self . _app = app <EOL> self . _allow_multiple_eg = allow_multiple_eg <EOL> self . _entity_groups = { } <EOL> self . _lock = threading . RLock ( ) <EOL> self . _apply_lock = threading . Lock ( ) <EOL> self . _actions = [ ] <EOL> self . _cost = datastore_pb . Cost ( ) <EOL> self . _mutation_versions = { } <EOL> self . _mutated_references = [ ] <EOL> self . _kind_to_indexes = collections . defaultdict ( list ) <EOL> def _GetTracker ( self , reference ) : <EOL> """<STR_LIT>""" <EOL> entity_group = _GetEntityGroup ( reference ) <EOL> key = datastore_types . ReferenceToKeyValue ( entity_group ) <EOL> tracker = self . _entity_groups . get ( key , None ) <EOL> if tracker is None : <EOL> Check ( self . _app == reference . app ( ) , <EOL> '<STR_LIT>' % <EOL> ( self . _app , reference . app ( ) ) ) <EOL> if self . _allow_multiple_eg : <EOL> Check ( len ( self . _entity_groups ) < _MAX_EG_PER_TXN , <EOL> '<STR_LIT>' ) <EOL> else : <EOL> Check ( len ( self . _entity_groups ) < <NUM_LIT:1> , <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> tracker = EntityGroupTracker ( entity_group ) <EOL> self . _entity_groups [ key ] = tracker <EOL> return tracker <EOL> def _GetAllTrackers ( self ) : <EOL> """<STR_LIT>""" <EOL> if not self . _entity_groups : <EOL> self . _GetTracker ( datastore_types . Key . from_path ( <EOL> '<STR_LIT>' , <NUM_LIT:1> , _app = self . _app ) . _ToPb ( ) ) <EOL> return self . _entity_groups . values ( ) <EOL> def _GrabSnapshot ( self , reference ) : <EOL> """<STR_LIT>""" <EOL> tracker = self . _GetTracker ( reference ) <EOL> check_contention = tracker . _snapshot is None <EOL> snapshot = tracker . _GrabSnapshot ( self . _txn_manager ) <EOL> if check_contention : <EOL> candidates = [ other for other in self . _entity_groups . values ( ) <EOL> if other . _snapshot is not None and other != tracker ] <EOL> meta_data_list = [ other . _meta_data for other in candidates ] <EOL> self . _txn_manager . _AcquireWriteLocks ( meta_data_list ) <EOL> try : <EOL> for other in candidates : <EOL> if other . _meta_data . _log_pos != other . _read_pos : <EOL> self . _state = self . FAILED <EOL> raise apiproxy_errors . ApplicationError ( <EOL> datastore_pb . Error . CONCURRENT_TRANSACTION , <EOL> '<STR_LIT>' ) <EOL> finally : <EOL> self . _txn_manager . _ReleaseWriteLocks ( meta_data_list ) <EOL> return snapshot <EOL> @ _SynchronizeTxn <EOL> def Get ( self , reference ) : <EOL> """<STR_LIT>""" <EOL> snapshot = self . _GrabSnapshot ( reference ) <EOL> record = snapshot . get ( datastore_types . ReferenceToKeyValue ( reference ) ) <EOL> return LoadRecord ( record ) <EOL> @ _SynchronizeTxn <EOL> def GetQueryCursor ( self , query , filters , orders , index_list ) : <EOL> """<STR_LIT>""" <EOL> Check ( query . has_ancestor ( ) , <EOL> '<STR_LIT>' ) <EOL> snapshot = self . _GrabSnapshot ( query . ancestor ( ) ) <EOL> return _ExecuteQuery ( snapshot . values ( ) , query , filters , orders , index_list ) <EOL> @ _SynchronizeTxn <EOL> def Put ( self , entity , insert , indexes ) : <EOL> """<STR_LIT>""" <EOL> tracker = self . _GetTracker ( entity . key ( ) ) <EOL> key = datastore_types . ReferenceToKeyValue ( entity . key ( ) ) <EOL> tracker . _delete . pop ( key , None ) <EOL> tracker . _put [ key ] = ( entity , insert ) <EOL> self . _kind_to_indexes [ _GetKeyKind ( entity . key ( ) ) ] = indexes <EOL> @ _SynchronizeTxn <EOL> def Delete ( self , reference , indexes ) : <EOL> """<STR_LIT>""" <EOL> tracker = self . _GetTracker ( reference ) <EOL> key = datastore_types . ReferenceToKeyValue ( reference ) <EOL> tracker . _put . pop ( key , None ) <EOL> tracker . _delete [ key ] = reference <EOL> self . _kind_to_indexes [ _GetKeyKind ( reference ) ] = indexes <EOL> @ _SynchronizeTxn <EOL> def AddActions ( self , actions , max_actions = None ) : <EOL> """<STR_LIT>""" <EOL> Check ( not max_actions or len ( self . _actions ) + len ( actions ) <= max_actions , <EOL> '<STR_LIT>' % max_actions ) <EOL> self . _actions . extend ( actions ) <EOL> def Rollback ( self ) : <EOL> """<STR_LIT>""" <EOL> self . _lock . acquire ( ) <EOL> try : <EOL> Check ( self . _state is self . ACTIVE or self . _state is self . FAILED , <EOL> '<STR_LIT>' ) <EOL> self . _state = self . ROLLEDBACK <EOL> finally : <EOL> self . _txn_manager . _RemoveTxn ( self ) <EOL> self . _lock . release ( ) <EOL> @ _SynchronizeTxn <EOL> def Commit ( self ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> trackers = self . _GetAllTrackers ( ) <EOL> empty = True <EOL> for tracker in trackers : <EOL> snapshot = tracker . _GrabSnapshot ( self . _txn_manager ) <EOL> empty = empty and not tracker . _put and not tracker . _delete <EOL> for entity , insert in tracker . _put . itervalues ( ) : <EOL> Check ( not insert or self . Get ( entity . key ( ) ) is None , <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> old_entity = None <EOL> old_version = None <EOL> key = datastore_types . ReferenceToKeyValue ( entity . key ( ) ) <EOL> self . _mutated_references . append ( entity . key ( ) ) <EOL> if key in snapshot : <EOL> old_entity = snapshot [ key ] . entity <EOL> old_version = snapshot [ key ] . metadata . updated_version ( ) <EOL> self . _AddWriteOps ( old_entity , entity ) <EOL> if _IsNoOpWrite ( old_entity , entity ) : <EOL> self . _mutation_versions [ key ] = long ( old_version ) <EOL> for reference in tracker . _delete . itervalues ( ) : <EOL> old_entity = None <EOL> key = datastore_types . ReferenceToKeyValue ( reference ) <EOL> self . _mutated_references . append ( reference ) <EOL> if key in snapshot : <EOL> old_entity = snapshot [ key ] . entity <EOL> self . _AddWriteOps ( None , old_entity ) <EOL> if _IsNoOpWrite ( old_entity , None ) : <EOL> self . _mutation_versions [ key ] = long ( tracker . _read_timestamp ) <EOL> if empty and not self . _actions : <EOL> self . Rollback ( ) <EOL> return datastore_pb . Cost ( ) <EOL> meta_data_list = [ tracker . _meta_data for tracker in trackers ] <EOL> self . _txn_manager . _AcquireWriteLocks ( meta_data_list ) <EOL> except : <EOL> self . Rollback ( ) <EOL> raise <EOL> try : <EOL> for tracker in trackers : <EOL> Check ( tracker . _meta_data . _log_pos == tracker . _read_pos , <EOL> '<STR_LIT>' , <EOL> datastore_pb . Error . CONCURRENT_TRANSACTION ) <EOL> for tracker in trackers : <EOL> tracker . _meta_data . Log ( self ) <EOL> self . _state = self . COMMITTED <EOL> self . _commit_time_s = time . time ( ) <EOL> write_timestamp = self . _txn_manager . _IncrementAndGetCommitTimestamp ( ) <EOL> for reference in self . _mutated_references : <EOL> key = datastore_types . ReferenceToKeyValue ( reference ) <EOL> if key not in self . _mutation_versions : <EOL> self . _mutation_versions [ key ] = long ( write_timestamp ) <EOL> except : <EOL> self . Rollback ( ) <EOL> raise <EOL> else : <EOL> for action in self . _actions : <EOL> try : <EOL> apiproxy_stub_map . MakeSyncCall ( <EOL> '<STR_LIT>' , '<STR_LIT>' , action , api_base_pb . VoidProto ( ) ) <EOL> except apiproxy_errors . ApplicationError , e : <EOL> logging . warning ( '<STR_LIT>' , <EOL> action , e ) <EOL> self . _actions = [ ] <EOL> finally : <EOL> self . _txn_manager . _RemoveTxn ( self ) <EOL> self . _txn_manager . _ReleaseWriteLocks ( meta_data_list ) <EOL> self . _txn_manager . _consistency_policy . _OnCommit ( self ) <EOL> return self . _cost <EOL> def GetMutationVersion ( self , reference ) : <EOL> """<STR_LIT>""" <EOL> assert self . _state == self . COMMITTED <EOL> key = datastore_types . ReferenceToKeyValue ( reference ) <EOL> return self . _mutation_versions [ key ] <EOL> def _AddWriteOps ( self , old_entity , new_entity ) : <EOL> """<STR_LIT>""" <EOL> composite_indexes = self . _kind_to_indexes [ _GetKeyKind ( new_entity . key ( ) ) ] <EOL> entity_writes , index_writes = _CalculateWriteOps ( <EOL> composite_indexes , old_entity , new_entity ) <EOL> _UpdateCost ( self . _cost , entity_writes , index_writes ) <EOL> def _Apply ( self , meta_data ) : <EOL> """<STR_LIT>""" <EOL> self . _apply_lock . acquire ( ) <EOL> try : <EOL> assert self . _state == self . COMMITTED <EOL> for tracker in self . _entity_groups . values ( ) : <EOL> if tracker . _meta_data is meta_data : <EOL> break <EOL> else : <EOL> assert False <EOL> assert tracker . _read_pos != tracker . APPLIED <EOL> for entity , insert in tracker . _put . itervalues ( ) : <EOL> key = datastore_types . ReferenceToKeyValue ( entity . key ( ) ) <EOL> if key in tracker . _snapshot : <EOL> metadata = tracker . _snapshot [ key ] . metadata <EOL> else : <EOL> metadata = entity_pb . EntityMetadata ( ) <EOL> metadata . set_updated_version ( self . GetMutationVersion ( entity . key ( ) ) ) <EOL> record = EntityRecord ( entity , metadata ) <EOL> self . _txn_manager . _Put ( record , insert ) <EOL> for key in tracker . _delete . itervalues ( ) : <EOL> self . _txn_manager . _Delete ( key ) <EOL> tracker . _read_pos = EntityGroupTracker . APPLIED <EOL> tracker . _meta_data . Unlog ( self ) <EOL> finally : <EOL> self . _apply_lock . release ( ) <EOL> class EntityRecord ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , entity , metadata = None ) : <EOL> self . entity = entity <EOL> self . metadata = metadata or entity_pb . EntityMetadata ( ) <EOL> class EntityGroupTracker ( object ) : <EOL> """<STR_LIT>""" <EOL> APPLIED = - <NUM_LIT:2> <EOL> _read_pos = None <EOL> _read_timestamp = None <EOL> _snapshot = None <EOL> _meta_data = None <EOL> def __init__ ( self , entity_group ) : <EOL> self . _entity_group = entity_group <EOL> self . _put = { } <EOL> self . _delete = { } <EOL> def _GrabSnapshot ( self , txn_manager ) : <EOL> """<STR_LIT>""" <EOL> if self . _snapshot is None : <EOL> self . _meta_data , self . _read_pos , self . _read_timestamp , self . _snapshot = ( <EOL> txn_manager . _GrabSnapshot ( self . _entity_group ) ) <EOL> return self . _snapshot <EOL> class EntityGroupMetaData ( object ) : <EOL> """<STR_LIT>""" <EOL> _log_pos = - <NUM_LIT:1> <EOL> _read_timestamp = None <EOL> _snapshot = None <EOL> def __init__ ( self , entity_group ) : <EOL> self . _entity_group = entity_group <EOL> self . _write_lock = threading . Lock ( ) <EOL> self . _apply_queue = [ ] <EOL> def CatchUp ( self ) : <EOL> """<STR_LIT>""" <EOL> assert self . _write_lock . acquire ( False ) is False <EOL> while self . _apply_queue : <EOL> self . _apply_queue [ <NUM_LIT:0> ] . _Apply ( self ) <EOL> def Log ( self , txn ) : <EOL> """<STR_LIT>""" <EOL> assert self . _write_lock . acquire ( False ) is False <EOL> self . _apply_queue . append ( txn ) <EOL> self . _log_pos += <NUM_LIT:1> <EOL> self . _snapshot = None <EOL> def Unlog ( self , txn ) : <EOL> """<STR_LIT>""" <EOL> assert self . _write_lock . acquire ( False ) is False <EOL> Check ( self . _apply_queue and self . _apply_queue [ <NUM_LIT:0> ] is txn , <EOL> '<STR_LIT>' , <EOL> datastore_pb . Error . INTERNAL_ERROR ) <EOL> self . _apply_queue . pop ( <NUM_LIT:0> ) <EOL> class BaseConsistencyPolicy ( object ) : <EOL> """<STR_LIT>""" <EOL> def _OnCommit ( self , txn ) : <EOL> """<STR_LIT>""" <EOL> raise NotImplementedError <EOL> def _OnGroom ( self , meta_data_list ) : <EOL> """<STR_LIT>""" <EOL> raise NotImplementedError <EOL> class MasterSlaveConsistencyPolicy ( BaseConsistencyPolicy ) : <EOL> """<STR_LIT>""" <EOL> def _OnCommit ( self , txn ) : <EOL> for tracker in txn . _GetAllTrackers ( ) : <EOL> tracker . _meta_data . _write_lock . acquire ( ) <EOL> try : <EOL> tracker . _meta_data . CatchUp ( ) <EOL> finally : <EOL> tracker . _meta_data . _write_lock . release ( ) <EOL> txn . _txn_manager . Write ( ) <EOL> def _OnGroom ( self , meta_data_list ) : <EOL> pass <EOL> class BaseHighReplicationConsistencyPolicy ( BaseConsistencyPolicy ) : <EOL> """<STR_LIT>""" <EOL> def _OnCommit ( self , txn ) : <EOL> pass <EOL> def _OnGroom ( self , meta_data_list ) : <EOL> for meta_data in meta_data_list : <EOL> if not meta_data . _apply_queue : <EOL> continue <EOL> meta_data . _write_lock . acquire ( ) <EOL> try : <EOL> while meta_data . _apply_queue : <EOL> txn = meta_data . _apply_queue [ <NUM_LIT:0> ] <EOL> if self . _ShouldApply ( txn , meta_data ) : <EOL> txn . _Apply ( meta_data ) <EOL> else : <EOL> break <EOL> finally : <EOL> meta_data . _write_lock . release ( ) <EOL> def _ShouldApply ( self , txn , meta_data ) : <EOL> """<STR_LIT>""" <EOL> raise NotImplementedError <EOL> class TimeBasedHRConsistencyPolicy ( BaseHighReplicationConsistencyPolicy ) : <EOL> """<STR_LIT>""" <EOL> _classification_map = [ ( <NUM_LIT> , <NUM_LIT:100> ) , <EOL> ( <NUM_LIT> , <NUM_LIT> ) , <EOL> ( <NUM_LIT> , <NUM_LIT> ) , <EOL> ( <NUM_LIT:1> , <NUM_LIT> ) <EOL> ] <EOL> def SetClassificationMap ( self , classification_map ) : <EOL> """<STR_LIT>""" <EOL> for prob , delay in classification_map : <EOL> if prob < <NUM_LIT:0> or prob > <NUM_LIT:1> or delay <= <NUM_LIT:0> : <EOL> raise TypeError ( <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' % ( classification_map , ) ) <EOL> self . _classification_map = sorted ( classification_map ) <EOL> def _ShouldApplyImpl ( self , elapsed_ms , classification ) : <EOL> for rate , ms in self . _classification_map : <EOL> if classification <= rate : <EOL> break <EOL> return elapsed_ms >= ms <EOL> def _Classify ( self , txn , meta_data ) : <EOL> return random . Random ( id ( txn ) ^ id ( meta_data ) ) . random ( ) <EOL> def _ShouldApply ( self , txn , meta_data ) : <EOL> elapsed_ms = ( time . time ( ) - txn . _commit_time_s ) * <NUM_LIT:1000> <EOL> classification = self . _Classify ( txn , meta_data ) <EOL> return self . _ShouldApplyImpl ( elapsed_ms , classification ) <EOL> class PseudoRandomHRConsistencyPolicy ( BaseHighReplicationConsistencyPolicy ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , probability = <NUM_LIT> , seed = <NUM_LIT:0> ) : <EOL> """<STR_LIT>""" <EOL> self . SetProbability ( probability ) <EOL> self . SetSeed ( seed ) <EOL> def SetProbability ( self , probability ) : <EOL> """<STR_LIT>""" <EOL> if probability < <NUM_LIT:0> or probability > <NUM_LIT:1> : <EOL> raise TypeError ( '<STR_LIT>' % <EOL> probability ) <EOL> self . _probability = probability <EOL> def SetSeed ( self , seed ) : <EOL> """<STR_LIT>""" <EOL> self . _random = random . Random ( seed ) <EOL> def _ShouldApply ( self , txn , meta_data ) : <EOL> return self . _random . random ( ) < self . _probability <EOL> class BaseTransactionManager ( object ) : <EOL> """<STR_LIT>""" <EOL> _commit_timestamp = MINIMUM_VERSION <EOL> def __init__ ( self , consistency_policy = None ) : <EOL> super ( BaseTransactionManager , self ) . __init__ ( ) <EOL> self . _consistency_policy = ( consistency_policy or <EOL> MasterSlaveConsistencyPolicy ( ) ) <EOL> self . _meta_data_lock = threading . Lock ( ) <EOL> self . _commit_timestamp_lock = threading . Lock ( ) <EOL> BaseTransactionManager . Clear ( self ) <EOL> def SetConsistencyPolicy ( self , policy ) : <EOL> """<STR_LIT>""" <EOL> if not isinstance ( policy , BaseConsistencyPolicy ) : <EOL> raise TypeError ( '<STR_LIT>' <EOL> '<STR_LIT>' % <EOL> ( policy , ) ) <EOL> self . Flush ( ) <EOL> self . _consistency_policy = policy <EOL> def Clear ( self ) : <EOL> """<STR_LIT>""" <EOL> self . _meta_data = { } <EOL> self . _txn_map = { } <EOL> def BeginTransaction ( self , app , allow_multiple_eg ) : <EOL> """<STR_LIT>""" <EOL> Check ( not ( allow_multiple_eg and isinstance ( <EOL> self . _consistency_policy , MasterSlaveConsistencyPolicy ) ) , <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> txn = self . _BeginTransaction ( app , allow_multiple_eg ) <EOL> self . _txn_map [ id ( txn ) ] = txn <EOL> transaction = datastore_pb . Transaction ( ) <EOL> transaction . set_app ( app ) <EOL> transaction . set_handle ( id ( txn ) ) <EOL> return transaction <EOL> def GetTxn ( self , transaction , request_trusted , request_app ) : <EOL> """<STR_LIT>""" <EOL> request_app = datastore_types . ResolveAppId ( request_app ) <EOL> CheckTransaction ( request_trusted , request_app , transaction ) <EOL> txn = self . _txn_map . get ( transaction . handle ( ) ) <EOL> Check ( txn and txn . _app == transaction . app ( ) , <EOL> '<STR_LIT>' % str ( transaction ) . replace ( '<STR_LIT:\n>' , '<STR_LIT:U+002CU+0020>' ) ) <EOL> return txn <EOL> def Groom ( self ) : <EOL> """<STR_LIT>""" <EOL> self . _meta_data_lock . acquire ( ) <EOL> try : <EOL> self . _consistency_policy . _OnGroom ( self . _meta_data . itervalues ( ) ) <EOL> finally : <EOL> self . _meta_data_lock . release ( ) <EOL> def Flush ( self ) : <EOL> """<STR_LIT>""" <EOL> self . _meta_data_lock . acquire ( ) <EOL> try : <EOL> for meta_data in self . _meta_data . itervalues ( ) : <EOL> if not meta_data . _apply_queue : <EOL> continue <EOL> meta_data . _write_lock . acquire ( ) <EOL> try : <EOL> meta_data . CatchUp ( ) <EOL> finally : <EOL> meta_data . _write_lock . release ( ) <EOL> finally : <EOL> self . _meta_data_lock . release ( ) <EOL> def _IncrementAndGetCommitTimestamp ( self ) : <EOL> with self . _commit_timestamp_lock : <EOL> now_ms = int ( time . time ( ) * <NUM_LIT:1000> ) <EOL> self . _commit_timestamp = max ( self . _commit_timestamp + <NUM_LIT:1> , now_ms ) <EOL> return self . _commit_timestamp <EOL> def _GetReadTimestamp ( self ) : <EOL> return self . _commit_timestamp <EOL> def _GetMetaData ( self , entity_group ) : <EOL> """<STR_LIT>""" <EOL> self . _meta_data_lock . acquire ( ) <EOL> try : <EOL> key = datastore_types . ReferenceToKeyValue ( entity_group ) <EOL> meta_data = self . _meta_data . get ( key , None ) <EOL> if not meta_data : <EOL> meta_data = EntityGroupMetaData ( entity_group ) <EOL> self . _meta_data [ key ] = meta_data <EOL> return meta_data <EOL> finally : <EOL> self . _meta_data_lock . release ( ) <EOL> def _BeginTransaction ( self , app , allow_multiple_eg ) : <EOL> """<STR_LIT>""" <EOL> return LiveTxn ( self , app , allow_multiple_eg ) <EOL> def _GrabSnapshot ( self , entity_group ) : <EOL> """<STR_LIT>""" <EOL> meta_data = self . _GetMetaData ( entity_group ) <EOL> meta_data . _write_lock . acquire ( ) <EOL> try : <EOL> if not meta_data . _snapshot : <EOL> meta_data . CatchUp ( ) <EOL> meta_data . _snapshot = self . _GetEntitiesInEntityGroup ( entity_group ) <EOL> meta_data . _read_timestamp = self . _GetReadTimestamp ( ) <EOL> return ( meta_data , meta_data . _log_pos , meta_data . _read_timestamp , <EOL> meta_data . _snapshot ) <EOL> finally : <EOL> meta_data . _write_lock . release ( ) <EOL> def _AcquireWriteLocks ( self , meta_data_list ) : <EOL> """<STR_LIT>""" <EOL> for meta_data in sorted ( meta_data_list ) : <EOL> meta_data . _write_lock . acquire ( ) <EOL> def _ReleaseWriteLocks ( self , meta_data_list ) : <EOL> """<STR_LIT>""" <EOL> for meta_data in sorted ( meta_data_list ) : <EOL> meta_data . _write_lock . release ( ) <EOL> def _RemoveTxn ( self , txn ) : <EOL> """<STR_LIT>""" <EOL> self . _txn_map . pop ( id ( txn ) , None ) <EOL> def _Put ( self , record , insert ) : <EOL> """<STR_LIT>""" <EOL> raise NotImplementedError <EOL> def _Delete ( self , reference ) : <EOL> """<STR_LIT>""" <EOL> raise NotImplementedError <EOL> def _GetEntitiesInEntityGroup ( self , entity_group ) : <EOL> """<STR_LIT>""" <EOL> raise NotImplementedError <EOL> class BaseIndexManager ( object ) : <EOL> """<STR_LIT>""" <EOL> WRITE_ONLY = entity_pb . CompositeIndex . WRITE_ONLY <EOL> READ_WRITE = entity_pb . CompositeIndex . READ_WRITE <EOL> DELETED = entity_pb . CompositeIndex . DELETED <EOL> ERROR = entity_pb . CompositeIndex . ERROR <EOL> _INDEX_STATE_TRANSITIONS = { <EOL> WRITE_ONLY : frozenset ( ( READ_WRITE , DELETED , ERROR ) ) , <EOL> READ_WRITE : frozenset ( ( DELETED , ) ) , <EOL> ERROR : frozenset ( ( DELETED , ) ) , <EOL> DELETED : frozenset ( ( ERROR , ) ) , <EOL> } <EOL> def __init__ ( self ) : <EOL> self . __indexes = collections . defaultdict ( list ) <EOL> self . __indexes_lock = threading . Lock ( ) <EOL> self . __next_index_id = <NUM_LIT:1> <EOL> self . __index_id_lock = threading . Lock ( ) <EOL> def __FindIndex ( self , index ) : <EOL> """<STR_LIT>""" <EOL> app = index . app_id ( ) <EOL> if app in self . __indexes : <EOL> for stored_index in self . __indexes [ app ] : <EOL> if index . definition ( ) == stored_index . definition ( ) : <EOL> return stored_index <EOL> return None <EOL> def CreateIndex ( self , index , trusted = False , calling_app = None ) : <EOL> calling_app = datastore_types . ResolveAppId ( calling_app ) <EOL> CheckAppId ( trusted , calling_app , index . app_id ( ) ) <EOL> Check ( index . id ( ) == <NUM_LIT:0> , '<STR_LIT>' ) <EOL> Check ( not self . __FindIndex ( index ) , '<STR_LIT>' ) <EOL> self . __index_id_lock . acquire ( ) <EOL> index . set_id ( self . __next_index_id ) <EOL> self . __next_index_id += <NUM_LIT:1> <EOL> self . __index_id_lock . release ( ) <EOL> clone = entity_pb . CompositeIndex ( ) <EOL> clone . CopyFrom ( index ) <EOL> app = index . app_id ( ) <EOL> clone . set_app_id ( app ) <EOL> self . __indexes_lock . acquire ( ) <EOL> try : <EOL> self . __indexes [ app ] . append ( clone ) <EOL> finally : <EOL> self . __indexes_lock . release ( ) <EOL> self . _OnIndexChange ( index . app_id ( ) ) <EOL> return index . id ( ) <EOL> def GetIndexes ( self , app , trusted = False , calling_app = None ) : <EOL> """<STR_LIT>""" <EOL> calling_app = datastore_types . ResolveAppId ( calling_app ) <EOL> CheckAppId ( trusted , calling_app , app ) <EOL> return self . __indexes [ app ] <EOL> def UpdateIndex ( self , index , trusted = False , calling_app = None ) : <EOL> CheckAppId ( trusted , calling_app , index . app_id ( ) ) <EOL> stored_index = self . __FindIndex ( index ) <EOL> Check ( stored_index , '<STR_LIT>' ) <EOL> Check ( index . state ( ) == stored_index . state ( ) or <EOL> index . state ( ) in self . _INDEX_STATE_TRANSITIONS [ stored_index . state ( ) ] , <EOL> '<STR_LIT>' % <EOL> ( entity_pb . CompositeIndex . State_Name ( stored_index . state ( ) ) , <EOL> ( entity_pb . CompositeIndex . State_Name ( index . state ( ) ) ) ) ) <EOL> self . __indexes_lock . acquire ( ) <EOL> try : <EOL> stored_index . set_state ( index . state ( ) ) <EOL> finally : <EOL> self . __indexes_lock . release ( ) <EOL> self . _OnIndexChange ( index . app_id ( ) ) <EOL> def DeleteIndex ( self , index , trusted = False , calling_app = None ) : <EOL> CheckAppId ( trusted , calling_app , index . app_id ( ) ) <EOL> stored_index = self . __FindIndex ( index ) <EOL> Check ( stored_index , '<STR_LIT>' ) <EOL> app = index . app_id ( ) <EOL> self . __indexes_lock . acquire ( ) <EOL> try : <EOL> self . __indexes [ app ] . remove ( stored_index ) <EOL> finally : <EOL> self . __indexes_lock . release ( ) <EOL> self . _OnIndexChange ( index . app_id ( ) ) <EOL> def _SideLoadIndex ( self , index ) : <EOL> self . __indexes [ index . app ( ) ] . append ( index ) <EOL> def _OnIndexChange ( self , app_id ) : <EOL> pass <EOL> class BaseDatastore ( BaseTransactionManager , BaseIndexManager ) : <EOL> """<STR_LIT>""" <EOL> _MAX_QUERY_COMPONENTS = <NUM_LIT:100> <EOL> _BATCH_SIZE = <NUM_LIT:20> <EOL> _MAX_ACTIONS_PER_TXN = <NUM_LIT:5> <EOL> def __init__ ( self , require_indexes = False , consistency_policy = None , <EOL> use_atexit = True , auto_id_policy = SEQUENTIAL ) : <EOL> BaseTransactionManager . __init__ ( self , consistency_policy = consistency_policy ) <EOL> BaseIndexManager . __init__ ( self ) <EOL> self . _require_indexes = require_indexes <EOL> self . _pseudo_kinds = { } <EOL> self . SetAutoIdPolicy ( auto_id_policy ) <EOL> if use_atexit : <EOL> atexit . register ( self . Write ) <EOL> def Clear ( self ) : <EOL> """<STR_LIT>""" <EOL> BaseTransactionManager . Clear ( self ) <EOL> def _RegisterPseudoKind ( self , kind ) : <EOL> """<STR_LIT>""" <EOL> self . _pseudo_kinds [ kind . name ] = kind <EOL> kind . _stub = weakref . proxy ( self ) <EOL> def GetQueryCursor ( self , raw_query , trusted = False , calling_app = None ) : <EOL> """<STR_LIT>""" <EOL> calling_app = datastore_types . ResolveAppId ( calling_app ) <EOL> CheckAppId ( trusted , calling_app , raw_query . app ( ) ) <EOL> filters , orders = datastore_index . Normalize ( raw_query . filter_list ( ) , <EOL> raw_query . order_list ( ) , <EOL> raw_query . property_name_list ( ) ) <EOL> CheckQuery ( raw_query , filters , orders , self . _MAX_QUERY_COMPONENTS ) <EOL> FillUsersInQuery ( filters ) <EOL> if self . _require_indexes : <EOL> self . _CheckHasIndex ( raw_query , trusted , calling_app ) <EOL> index_list = self . __IndexListForQuery ( raw_query ) <EOL> if raw_query . has_transaction ( ) : <EOL> Check ( raw_query . kind ( ) not in self . _pseudo_kinds , <EOL> '<STR_LIT>' % raw_query . kind ( ) ) <EOL> txn = self . GetTxn ( raw_query . transaction ( ) , trusted , calling_app ) <EOL> return txn . GetQueryCursor ( raw_query , filters , orders , index_list ) <EOL> if raw_query . has_ancestor ( ) and raw_query . kind ( ) not in self . _pseudo_kinds : <EOL> txn = self . _BeginTransaction ( raw_query . app ( ) , False ) <EOL> return txn . GetQueryCursor ( raw_query , filters , orders , index_list ) <EOL> self . Groom ( ) <EOL> return self . _GetQueryCursor ( raw_query , filters , orders , index_list ) <EOL> def __IndexListForQuery ( self , query ) : <EOL> """<STR_LIT>""" <EOL> required , kind , ancestor , props = ( <EOL> datastore_index . CompositeIndexForQuery ( query ) ) <EOL> if not required : <EOL> return [ ] <EOL> composite_index_pb = entity_pb . CompositeIndex ( ) <EOL> composite_index_pb . set_app_id ( query . app ( ) ) <EOL> composite_index_pb . set_id ( <NUM_LIT:0> ) <EOL> composite_index_pb . set_state ( entity_pb . CompositeIndex . READ_WRITE ) <EOL> index_pb = composite_index_pb . mutable_definition ( ) <EOL> index_pb . set_entity_type ( kind ) <EOL> index_pb . set_ancestor ( bool ( ancestor ) ) <EOL> for prop in datastore_index . GetRecommendedIndexProperties ( props ) : <EOL> prop_pb = entity_pb . Index_Property ( ) <EOL> prop . CopyToIndexPb ( prop_pb ) <EOL> index_pb . property_list ( ) . append ( prop_pb ) <EOL> return [ composite_index_pb ] <EOL> def Get ( self , raw_keys , transaction = None , eventual_consistency = False , <EOL> trusted = False , calling_app = None ) : <EOL> """<STR_LIT>""" <EOL> if not raw_keys : <EOL> return [ ] <EOL> calling_app = datastore_types . ResolveAppId ( calling_app ) <EOL> records = [ ] <EOL> if not transaction and eventual_consistency : <EOL> self . Groom ( ) <EOL> for key in raw_keys : <EOL> CheckReference ( calling_app , trusted , key ) <EOL> records . append ( self . _GetWithPseudoKinds ( None , key , <EOL> eventual_consistency ) ) <EOL> else : <EOL> grouped_keys = collections . defaultdict ( list ) <EOL> for i , key in enumerate ( raw_keys ) : <EOL> CheckReference ( trusted , calling_app , key ) <EOL> entity_group = _GetEntityGroup ( key ) <EOL> entity_group_key = datastore_types . ReferenceToKeyValue ( entity_group ) <EOL> grouped_keys [ entity_group_key ] . append ( ( key , i ) ) <EOL> if transaction : <EOL> txn = self . GetTxn ( transaction , trusted , calling_app ) <EOL> records = [ self . _GetWithPseudoKinds ( txn , key , eventual_consistency ) <EOL> for key in raw_keys ] <EOL> else : <EOL> records = [ None ] * len ( raw_keys ) <EOL> def op ( txn , v ) : <EOL> key , i = v <EOL> records [ i ] = self . _GetWithPseudoKinds ( txn , key , eventual_consistency ) <EOL> for keys in grouped_keys . itervalues ( ) : <EOL> self . _RunInTxn ( keys , keys [ <NUM_LIT:0> ] [ <NUM_LIT:0> ] . app ( ) , op ) <EOL> return records <EOL> def _GetWithPseudoKinds ( self , txn , key , eventual_consistency = False ) : <EOL> """<STR_LIT>""" <EOL> pseudo_kind = self . _pseudo_kinds . get ( _GetKeyKind ( key ) , None ) <EOL> if pseudo_kind : <EOL> return EntityRecord ( pseudo_kind . Get ( txn , key ) ) <EOL> else : <EOL> if txn : <EOL> record = txn . Get ( key ) <EOL> else : <EOL> record = self . _Get ( key ) <EOL> if not record : <EOL> metadata = entity_pb . EntityMetadata ( ) <EOL> if not eventual_consistency : <EOL> metadata . set_updated_version ( self . _GetReadTimestamp ( ) ) <EOL> record = EntityRecord ( None , metadata ) <EOL> return record <EOL> def Put ( self , raw_entities , cost , transaction = None , <EOL> trusted = False , calling_app = None ) : <EOL> """<STR_LIT>""" <EOL> if not raw_entities : <EOL> return [ ] <EOL> calling_app = datastore_types . ResolveAppId ( calling_app ) <EOL> result = [ None ] * len ( raw_entities ) <EOL> grouped_entities = collections . defaultdict ( list ) <EOL> for i , raw_entity in enumerate ( raw_entities ) : <EOL> CheckEntity ( trusted , calling_app , raw_entity ) <EOL> entity = entity_pb . EntityProto ( ) <EOL> entity . CopyFrom ( raw_entity ) <EOL> for prop in itertools . chain ( entity . property_list ( ) , <EOL> entity . raw_property_list ( ) ) : <EOL> FillUser ( prop ) <EOL> last_element = entity . key ( ) . path ( ) . element_list ( ) [ - <NUM_LIT:1> ] <EOL> if not ( last_element . id ( ) or last_element . has_name ( ) ) : <EOL> insert = True <EOL> if self . _auto_id_policy == SEQUENTIAL : <EOL> last_element . set_id ( self . _AllocateSequentialIds ( entity . key ( ) ) [ <NUM_LIT:0> ] ) <EOL> else : <EOL> full_key = self . _AllocateIds ( [ entity . key ( ) ] ) [ <NUM_LIT:0> ] <EOL> last_element . set_id ( full_key . path ( ) . element_list ( ) [ - <NUM_LIT:1> ] . id ( ) ) <EOL> else : <EOL> insert = False <EOL> entity_group = _GetEntityGroup ( entity . key ( ) ) <EOL> entity . mutable_entity_group ( ) . CopyFrom ( entity_group . path ( ) ) <EOL> entity_group_key = datastore_types . ReferenceToKeyValue ( entity_group ) <EOL> grouped_entities [ entity_group_key ] . append ( ( entity , insert , i ) ) <EOL> key = entity_pb . Reference ( ) <EOL> key . CopyFrom ( entity . key ( ) ) <EOL> result [ i ] = key <EOL> mutation_versions = [ None ] * len ( raw_entities ) <EOL> if transaction : <EOL> txn = self . GetTxn ( transaction , trusted , calling_app ) <EOL> for group in grouped_entities . values ( ) : <EOL> for entity , insert , _ in group : <EOL> indexes = _FilterIndexesByKind ( entity . key ( ) , self . GetIndexes ( <EOL> entity . key ( ) . app ( ) , trusted , calling_app ) ) <EOL> txn . Put ( entity , insert , indexes ) <EOL> else : <EOL> for entities in grouped_entities . itervalues ( ) : <EOL> txn = self . _RunInTxn ( <EOL> entities , entities [ <NUM_LIT:0> ] [ <NUM_LIT:0> ] . key ( ) . app ( ) , <EOL> lambda txn , v : txn . Put ( v [ <NUM_LIT:0> ] , v [ <NUM_LIT:1> ] , _FilterIndexesByKind ( <EOL> v [ <NUM_LIT:0> ] . key ( ) , <EOL> self . GetIndexes ( v [ <NUM_LIT:0> ] . key ( ) . app ( ) , trusted , calling_app ) ) ) ) <EOL> for entity , _ , index in entities : <EOL> mutation_versions [ index ] = txn . GetMutationVersion ( entity . key ( ) ) <EOL> _UpdateCost ( cost , txn . _cost . entity_writes ( ) , txn . _cost . index_writes ( ) ) <EOL> return zip ( result , mutation_versions ) <EOL> def Delete ( self , raw_keys , cost , transaction = None , <EOL> trusted = False , calling_app = None ) : <EOL> """<STR_LIT>""" <EOL> if not raw_keys : <EOL> return [ ] <EOL> calling_app = datastore_types . ResolveAppId ( calling_app ) <EOL> grouped_keys = collections . defaultdict ( list ) <EOL> for i , key in enumerate ( raw_keys ) : <EOL> CheckReference ( trusted , calling_app , key ) <EOL> entity_group = _GetEntityGroup ( key ) <EOL> entity_group_key = datastore_types . ReferenceToKeyValue ( entity_group ) <EOL> grouped_keys [ entity_group_key ] . append ( ( key , i ) ) <EOL> mutation_versions = [ None ] * len ( raw_keys ) <EOL> if transaction : <EOL> txn = self . GetTxn ( transaction , trusted , calling_app ) <EOL> for key in raw_keys : <EOL> indexes = _FilterIndexesByKind ( key , self . GetIndexes ( <EOL> key . app ( ) , trusted , calling_app ) ) <EOL> txn . Delete ( key , indexes ) <EOL> else : <EOL> for keys in grouped_keys . itervalues ( ) : <EOL> txn = self . _RunInTxn ( <EOL> keys , keys [ <NUM_LIT:0> ] [ <NUM_LIT:0> ] . app ( ) , <EOL> lambda txn , key : txn . Delete ( key [ <NUM_LIT:0> ] , _FilterIndexesByKind ( <EOL> key [ <NUM_LIT:0> ] , self . GetIndexes ( key [ <NUM_LIT:0> ] . app ( ) , trusted , calling_app ) ) ) ) <EOL> for key , index in keys : <EOL> mutation_versions [ index ] = txn . GetMutationVersion ( key ) <EOL> _UpdateCost ( cost , txn . _cost . entity_writes ( ) , txn . _cost . index_writes ( ) ) <EOL> return mutation_versions <EOL> def Touch ( self , raw_keys , trusted = False , calling_app = None ) : <EOL> """<STR_LIT>""" <EOL> calling_app = datastore_types . ResolveAppId ( calling_app ) <EOL> grouped_keys = collections . defaultdict ( list ) <EOL> for key in raw_keys : <EOL> CheckReference ( trusted , calling_app , key ) <EOL> entity_group = _GetEntityGroup ( key ) <EOL> entity_group_key = datastore_types . ReferenceToKeyValue ( entity_group ) <EOL> grouped_keys [ entity_group_key ] . append ( key ) <EOL> for keys in grouped_keys . itervalues ( ) : <EOL> self . _RunInTxn ( keys , keys [ <NUM_LIT:0> ] . app ( ) , lambda txn , key : None ) <EOL> def _RunInTxn ( self , values , app , op ) : <EOL> """<STR_LIT>""" <EOL> retries = <NUM_LIT:0> <EOL> backoff = _INITIAL_RETRY_DELAY_MS / <NUM_LIT> <EOL> while True : <EOL> try : <EOL> txn = self . _BeginTransaction ( app , False ) <EOL> for value in values : <EOL> op ( txn , value ) <EOL> txn . Commit ( ) <EOL> return txn <EOL> except apiproxy_errors . ApplicationError , e : <EOL> if e . application_error == datastore_pb . Error . CONCURRENT_TRANSACTION : <EOL> retries += <NUM_LIT:1> <EOL> if retries <= _RETRIES : <EOL> time . sleep ( backoff ) <EOL> backoff *= _RETRY_DELAY_MULTIPLIER <EOL> if backoff * <NUM_LIT> > _MAX_RETRY_DELAY_MS : <EOL> backoff = _MAX_RETRY_DELAY_MS / <NUM_LIT> <EOL> continue <EOL> raise <EOL> def _CheckHasIndex ( self , query , trusted = False , calling_app = None ) : <EOL> """<STR_LIT>""" <EOL> if query . kind ( ) in self . _pseudo_kinds : <EOL> return <EOL> minimal_index = datastore_index . MinimalCompositeIndexForQuery ( query , <EOL> ( datastore_index . ProtoToIndexDefinition ( index ) <EOL> for index in self . GetIndexes ( query . app ( ) , trusted , calling_app ) <EOL> if index . state ( ) == entity_pb . CompositeIndex . READ_WRITE ) ) <EOL> if minimal_index is not None : <EOL> msg = ( '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> is_most_efficient , kind , ancestor , properties = minimal_index <EOL> if not is_most_efficient : <EOL> yaml = datastore_index . IndexYamlForQuery ( <EOL> kind , ancestor , <EOL> datastore_index . GetRecommendedIndexProperties ( properties ) ) <EOL> msg += '<STR_LIT>' + yaml <EOL> raise apiproxy_errors . ApplicationError ( datastore_pb . Error . NEED_INDEX , msg ) <EOL> def SetAutoIdPolicy ( self , auto_id_policy ) : <EOL> """<STR_LIT>""" <EOL> valid_policies = ( SEQUENTIAL , SCATTERED ) <EOL> if auto_id_policy not in valid_policies : <EOL> raise TypeError ( '<STR_LIT>' , <EOL> valid_policies , auto_id_policy ) <EOL> self . _auto_id_policy = auto_id_policy <EOL> def Write ( self ) : <EOL> """<STR_LIT>""" <EOL> self . Flush ( ) <EOL> def Close ( self ) : <EOL> """<STR_LIT>""" <EOL> self . Write ( ) <EOL> def _GetQueryCursor ( self , query , filters , orders , index_list ) : <EOL> """<STR_LIT>""" <EOL> raise NotImplementedError <EOL> def _Get ( self , reference ) : <EOL> """<STR_LIT>""" <EOL> raise NotImplementedError <EOL> def _AllocateSequentialIds ( self , reference , size = <NUM_LIT:1> , max_id = None ) : <EOL> """<STR_LIT>""" <EOL> raise NotImplementedError <EOL> def _AllocateIds ( self , references ) : <EOL> """<STR_LIT>""" <EOL> raise NotImplementedError <EOL> def _NeedsIndexes ( func ) : <EOL> """<STR_LIT>""" <EOL> def UpdateIndexesWrapper ( self , * args , ** kwargs ) : <EOL> self . _SetupIndexes ( ) <EOL> try : <EOL> return func ( self , * args , ** kwargs ) <EOL> finally : <EOL> self . _UpdateIndexes ( ) <EOL> return UpdateIndexesWrapper <EOL> class EntityGroupPseudoKind ( object ) : <EOL> """<STR_LIT>""" <EOL> name = '<STR_LIT>' <EOL> base_version = int ( time . time ( ) * <NUM_LIT> ) <EOL> def Get ( self , txn , key ) : <EOL> """<STR_LIT>""" <EOL> if not txn : <EOL> txn = self . _stub . _BeginTransaction ( key . app ( ) , False ) <EOL> try : <EOL> return self . Get ( txn , key ) <EOL> finally : <EOL> txn . Rollback ( ) <EOL> if isinstance ( txn . _txn_manager . _consistency_policy , <EOL> MasterSlaveConsistencyPolicy ) : <EOL> return None <EOL> path = key . path ( ) <EOL> if path . element_size ( ) != <NUM_LIT:2> or path . element_list ( ) [ - <NUM_LIT:1> ] . id ( ) != <NUM_LIT:1> : <EOL> return None <EOL> tracker = txn . _GetTracker ( key ) <EOL> tracker . _GrabSnapshot ( txn . _txn_manager ) <EOL> eg = entity_pb . EntityProto ( ) <EOL> eg . mutable_key ( ) . CopyFrom ( key ) <EOL> eg . mutable_entity_group ( ) . CopyFrom ( _GetEntityGroup ( key ) . path ( ) ) <EOL> version = entity_pb . Property ( ) <EOL> version . set_name ( '<STR_LIT>' ) <EOL> version . set_multiple ( False ) <EOL> version . mutable_value ( ) . set_int64value ( <EOL> tracker . _read_pos + self . base_version ) <EOL> eg . property_list ( ) . append ( version ) <EOL> return eg <EOL> def Query ( self , query , filters , orders ) : <EOL> """<STR_LIT>""" <EOL> raise apiproxy_errors . ApplicationError ( <EOL> datastore_pb . Error . BAD_REQUEST , '<STR_LIT>' + self . name ) <EOL> class _CachedIndexDefinitions ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , file_names , last_modifieds , index_protos ) : <EOL> assert len ( file_names ) <= <NUM_LIT:1> <EOL> self . file_names = file_names <EOL> self . last_modifieds = last_modifieds <EOL> self . index_protos = index_protos <EOL> class DatastoreStub ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , <EOL> datastore , <EOL> app_id = None , <EOL> trusted = None , <EOL> root_path = None ) : <EOL> super ( DatastoreStub , self ) . __init__ ( ) <EOL> self . _datastore = datastore <EOL> self . _app_id = datastore_types . ResolveAppId ( app_id ) <EOL> self . _trusted = trusted <EOL> self . _root_path = root_path <EOL> self . _xml_configuration = self . _XmlConfiguration ( ) <EOL> self . __query_history = { } <EOL> self . __query_ci_history = set ( ) <EOL> self . _cached_index_definitions = _CachedIndexDefinitions ( [ ] , [ ] , None ) <EOL> if self . _require_indexes or root_path is None : <EOL> self . _index_config_updater = None <EOL> else : <EOL> updater_class = ( <EOL> datastore_stub_index . DatastoreIndexesAutoXmlUpdater <EOL> if self . _xml_configuration else datastore_stub_index . IndexYamlUpdater ) <EOL> self . _index_config_updater = updater_class ( root_path ) <EOL> DatastoreStub . Clear ( self ) <EOL> def _XmlConfiguration ( self ) : <EOL> """<STR_LIT>""" <EOL> if not self . _root_path : <EOL> return False <EOL> index_yaml = os . path . join ( self . _root_path , '<STR_LIT>' ) <EOL> web_inf = os . path . join ( self . _root_path , '<STR_LIT>' ) <EOL> datastore_indexes_xml = os . path . join ( web_inf , '<STR_LIT>' ) <EOL> datastore_indexes_auto_xml = os . path . join ( <EOL> web_inf , '<STR_LIT>' , '<STR_LIT>' ) <EOL> existing = [ <EOL> f for f in [ <EOL> index_yaml , datastore_indexes_xml , datastore_indexes_auto_xml ] <EOL> if os . path . isfile ( f ) ] <EOL> if existing == [ index_yaml ] : <EOL> return False <EOL> elif index_yaml in existing : <EOL> raise yaml_errors . AmbiguousConfigurationFiles ( <EOL> '<STR_LIT>' % existing ) <EOL> else : <EOL> return os . path . isdir ( web_inf ) <EOL> def Clear ( self ) : <EOL> """<STR_LIT>""" <EOL> self . _query_cursors = { } <EOL> self . __query_history = { } <EOL> self . __query_ci_history = set ( ) <EOL> def QueryHistory ( self ) : <EOL> """<STR_LIT>""" <EOL> return dict ( ( pb , times ) for pb , times in self . __query_history . items ( ) <EOL> if pb . app ( ) == self . _app_id ) <EOL> def _QueryCompositeIndexHistoryLength ( self ) : <EOL> """<STR_LIT>""" <EOL> return len ( self . __query_ci_history ) <EOL> def SetTrusted ( self , trusted ) : <EOL> """<STR_LIT>""" <EOL> self . _trusted = trusted <EOL> def _Dynamic_Get ( self , req , res ) : <EOL> transaction = req . has_transaction ( ) and req . transaction ( ) or None <EOL> if req . allow_deferred ( ) and req . key_size ( ) > _MAXIMUM_RESULTS : <EOL> keys_to_get = req . key_list ( ) [ - _MAXIMUM_RESULTS : ] <EOL> deferred_keys = req . key_list ( ) [ : - _MAXIMUM_RESULTS ] <EOL> res . deferred_list ( ) . extend ( deferred_keys ) <EOL> else : <EOL> keys_to_get = req . key_list ( ) <EOL> res . set_in_order ( not req . allow_deferred ( ) ) <EOL> total_response_bytes = <NUM_LIT:0> <EOL> for index , record in enumerate ( self . _datastore . Get ( keys_to_get , <EOL> transaction , <EOL> req . has_failover_ms ( ) , <EOL> self . _trusted , <EOL> self . _app_id ) ) : <EOL> entity = record . entity <EOL> entity_size = entity and entity . ByteSize ( ) or <NUM_LIT:0> <EOL> if ( req . allow_deferred ( ) <EOL> and index > <NUM_LIT:0> <EOL> and total_response_bytes + entity_size > _MAXIMUM_QUERY_RESULT_BYTES ) : <EOL> res . deferred_list ( ) . extend ( keys_to_get [ index : ] ) <EOL> break <EOL> else : <EOL> entity_result = res . add_entity ( ) <EOL> if record . metadata . has_updated_version ( ) : <EOL> entity_result . set_version ( record . metadata . updated_version ( ) ) <EOL> if entity : <EOL> entity_result . mutable_entity ( ) . CopyFrom ( entity ) <EOL> total_response_bytes += entity_size <EOL> else : <EOL> entity_result . mutable_key ( ) . CopyFrom ( keys_to_get [ index ] ) <EOL> def _Dynamic_Put ( self , req , res ) : <EOL> transaction = req . has_transaction ( ) and req . transaction ( ) or None <EOL> results = self . _datastore . Put ( req . entity_list ( ) , res . mutable_cost ( ) , <EOL> transaction , self . _trusted , self . _app_id ) <EOL> res . key_list ( ) . extend ( result [ <NUM_LIT:0> ] for result in results ) <EOL> if not transaction : <EOL> res . version_list ( ) . extend ( result [ <NUM_LIT:1> ] for result in results ) <EOL> def _Dynamic_Delete ( self , req , res ) : <EOL> transaction = req . has_transaction ( ) and req . transaction ( ) or None <EOL> versions = self . _datastore . Delete ( req . key_list ( ) , res . mutable_cost ( ) , <EOL> transaction , self . _trusted , self . _app_id ) <EOL> if not transaction : <EOL> res . version_list ( ) . extend ( versions ) <EOL> def _Dynamic_Touch ( self , req , _ ) : <EOL> self . _datastore . Touch ( req . key_list ( ) , self . _trusted , self . _app_id ) <EOL> @ _NeedsIndexes <EOL> def _Dynamic_RunQuery ( self , query , query_result ) : <EOL> cursor = self . _datastore . GetQueryCursor ( query , self . _trusted , self . _app_id ) <EOL> count = query . count ( ) if query . has_count ( ) else None <EOL> cursor . PopulateQueryResult ( query_result , count , query . offset ( ) , <EOL> query . compile ( ) , first_result = True ) <EOL> if query_result . has_cursor ( ) : <EOL> self . _query_cursors [ query_result . cursor ( ) . cursor ( ) ] = cursor <EOL> if query . compile ( ) : <EOL> compiled_query = query_result . mutable_compiled_query ( ) <EOL> compiled_query . set_keys_only ( query . keys_only ( ) ) <EOL> compiled_query . mutable_primaryscan ( ) . set_index_name ( query . Encode ( ) ) <EOL> self . __UpdateQueryHistory ( query ) <EOL> def __UpdateQueryHistory ( self , query ) : <EOL> clone = datastore_pb . Query ( ) <EOL> clone . CopyFrom ( query ) <EOL> clone . clear_hint ( ) <EOL> clone . clear_limit ( ) <EOL> clone . clear_offset ( ) <EOL> clone . clear_count ( ) <EOL> if clone in self . __query_history : <EOL> self . __query_history [ clone ] += <NUM_LIT:1> <EOL> else : <EOL> self . __query_history [ clone ] = <NUM_LIT:1> <EOL> if clone . app ( ) == self . _app_id : <EOL> self . __query_ci_history . add ( <EOL> datastore_index . CompositeIndexForQuery ( clone ) ) <EOL> def _Dynamic_Next ( self , next_request , query_result ) : <EOL> app = next_request . cursor ( ) . app ( ) <EOL> CheckAppId ( self . _trusted , self . _app_id , app ) <EOL> cursor = self . _query_cursors . get ( next_request . cursor ( ) . cursor ( ) ) <EOL> Check ( cursor and cursor . app == app , <EOL> '<STR_LIT>' % next_request . cursor ( ) . cursor ( ) ) <EOL> count = next_request . count ( ) if next_request . has_count ( ) else None <EOL> offset = next_request . offset ( ) if next_request . has_offset ( ) else None <EOL> cursor . PopulateQueryResult ( query_result , <EOL> count , <EOL> offset , <EOL> next_request . compile ( ) , <EOL> first_result = False ) <EOL> if not query_result . has_cursor ( ) : <EOL> del self . _query_cursors [ next_request . cursor ( ) . cursor ( ) ] <EOL> def _Dynamic_AddActions ( self , request , _ ) : <EOL> """<STR_LIT>""" <EOL> if not request . add_request_list ( ) : <EOL> return <EOL> first_add_request = request . add_request_list ( ) [ <NUM_LIT:0> ] <EOL> datastore_transaction = None <EOL> if first_add_request . has_datastore_transaction ( ) : <EOL> datastore_transaction = first_add_request . datastore_transaction ( ) <EOL> transaction = datastore_pb . Transaction ( ) <EOL> get_service_converter ( ) . v1_to_v3_txn ( datastore_transaction , transaction ) <EOL> else : <EOL> transaction = first_add_request . transaction ( ) <EOL> txn = self . _datastore . GetTxn ( transaction , self . _trusted , self . _app_id ) <EOL> new_actions = [ ] <EOL> for add_request in request . add_request_list ( ) : <EOL> Check ( datastore_transaction is not None <EOL> and add_request . datastore_transaction ( ) == datastore_transaction <EOL> or add_request . transaction ( ) == transaction , <EOL> '<STR_LIT>' ) <EOL> clone = taskqueue_service_pb . TaskQueueAddRequest ( ) <EOL> clone . CopyFrom ( add_request ) <EOL> clone . clear_transaction ( ) <EOL> clone . clear_datastore_transaction ( ) <EOL> new_actions . append ( clone ) <EOL> txn . AddActions ( new_actions , self . _MAX_ACTIONS_PER_TXN ) <EOL> def _Dynamic_BeginTransaction ( self , req , transaction ) : <EOL> CheckAppId ( self . _trusted , self . _app_id , req . app ( ) ) <EOL> transaction . CopyFrom ( self . _datastore . BeginTransaction ( <EOL> req . app ( ) , req . allow_multiple_eg ( ) ) ) <EOL> def _Dynamic_Commit ( self , transaction , res ) : <EOL> CheckAppId ( self . _trusted , self . _app_id , transaction . app ( ) ) <EOL> txn = self . _datastore . GetTxn ( transaction , self . _trusted , self . _app_id ) <EOL> res . mutable_cost ( ) . CopyFrom ( txn . Commit ( ) ) <EOL> for reference in txn . _mutated_references : <EOL> commit_version = res . add_version ( ) <EOL> commit_version . mutable_root_entity_key ( ) . CopyFrom ( reference ) <EOL> commit_version . set_version ( txn . GetMutationVersion ( reference ) ) <EOL> def _Dynamic_Rollback ( self , transaction , _ ) : <EOL> CheckAppId ( self . _trusted , self . _app_id , transaction . app ( ) ) <EOL> txn = self . _datastore . GetTxn ( transaction , self . _trusted , self . _app_id ) <EOL> txn . Rollback ( ) <EOL> def _Dynamic_CreateIndex ( self , index , id_response ) : <EOL> id_response . set_value ( self . _datastore . CreateIndex ( index , <EOL> self . _trusted , <EOL> self . _app_id ) ) <EOL> @ _NeedsIndexes <EOL> def _Dynamic_GetIndices ( self , get_indicies_request , composite_indices ) : <EOL> composite_indices . index_list ( ) . extend ( self . _datastore . GetIndexes ( <EOL> get_indicies_request . app_id ( ) , self . _trusted , self . _app_id ) ) <EOL> def _Dynamic_UpdateIndex ( self , index , _ ) : <EOL> self . _datastore . UpdateIndex ( index , self . _trusted , self . _app_id ) <EOL> def _Dynamic_DeleteIndex ( self , index , _ ) : <EOL> self . _datastore . DeleteIndex ( index , self . _trusted , self . _app_id ) <EOL> def _Dynamic_AllocateIds ( self , allocate_ids_request , allocate_ids_response ) : <EOL> Check ( not allocate_ids_request . has_model_key ( ) <EOL> or not allocate_ids_request . reserve_list ( ) , <EOL> '<STR_LIT>' ) <EOL> if allocate_ids_request . reserve_list ( ) : <EOL> Check ( not allocate_ids_request . has_size ( ) , <EOL> '<STR_LIT>' ) <EOL> Check ( not allocate_ids_request . has_max ( ) , <EOL> '<STR_LIT>' ) <EOL> if allocate_ids_request . has_model_key ( ) : <EOL> CheckAppId ( allocate_ids_request . model_key ( ) . app ( ) , <EOL> self . _trusted , self . _app_id ) <EOL> reference = allocate_ids_request . model_key ( ) <EOL> ( start , end ) = self . _datastore . _AllocateSequentialIds ( <EOL> reference , allocate_ids_request . size ( ) , allocate_ids_request . max ( ) ) <EOL> allocate_ids_response . set_start ( start ) <EOL> allocate_ids_response . set_end ( end ) <EOL> else : <EOL> for reference in allocate_ids_request . reserve_list ( ) : <EOL> CheckAppId ( reference . app ( ) , self . _trusted , self . _app_id ) <EOL> self . _datastore . _AllocateIds ( allocate_ids_request . reserve_list ( ) ) <EOL> allocate_ids_response . set_start ( <NUM_LIT:0> ) <EOL> allocate_ids_response . set_end ( <NUM_LIT:0> ) <EOL> def _SetupIndexes ( self , _open = open ) : <EOL> """<STR_LIT>""" <EOL> if not self . _root_path : <EOL> return <EOL> file_names = [ os . path . join ( self . _root_path , '<STR_LIT>' ) ] <EOL> file_mtimes = [ os . path . getmtime ( f ) for f in file_names if os . path . exists ( f ) ] <EOL> if ( self . _cached_index_definitions . file_names == file_names and <EOL> all ( os . path . exists ( f ) for f in file_names ) and <EOL> self . _cached_index_definitions . last_modifieds == file_mtimes ) : <EOL> requested_indexes = self . _cached_index_definitions . index_protos <EOL> else : <EOL> file_mtimes = [ ] <EOL> index_texts = [ ] <EOL> for file_name in file_names : <EOL> try : <EOL> file_mtimes . append ( os . path . getmtime ( file_name ) ) <EOL> with _open ( file_name , '<STR_LIT:r>' ) as fh : <EOL> index_texts . append ( fh . read ( ) ) <EOL> except ( OSError , IOError ) : <EOL> pass <EOL> requested_indexes = [ ] <EOL> if len ( index_texts ) == len ( file_names ) : <EOL> all_ok = True <EOL> for index_text in index_texts : <EOL> index_defs = datastore_index . ParseIndexDefinitions ( index_text ) <EOL> if index_defs is None or index_defs . indexes is None : <EOL> all_ok = False <EOL> else : <EOL> requested_indexes . extend ( <EOL> datastore_index . IndexDefinitionsToProtos ( <EOL> self . _app_id , index_defs . indexes ) ) <EOL> if all_ok : <EOL> self . _cached_index_definitions = _CachedIndexDefinitions ( <EOL> file_names , file_mtimes , requested_indexes ) <EOL> existing_indexes = self . _datastore . GetIndexes ( <EOL> self . _app_id , self . _trusted , self . _app_id ) <EOL> requested = dict ( ( x . definition ( ) . Encode ( ) , x ) for x in requested_indexes ) <EOL> existing = dict ( ( x . definition ( ) . Encode ( ) , x ) for x in existing_indexes ) <EOL> created = <NUM_LIT:0> <EOL> for key , index in requested . iteritems ( ) : <EOL> if key not in existing : <EOL> new_index = entity_pb . CompositeIndex ( ) <EOL> new_index . CopyFrom ( index ) <EOL> new_index . set_id ( datastore_admin . CreateIndex ( new_index ) ) <EOL> new_index . set_state ( entity_pb . CompositeIndex . READ_WRITE ) <EOL> datastore_admin . UpdateIndex ( new_index ) <EOL> created += <NUM_LIT:1> <EOL> deleted = <NUM_LIT:0> <EOL> for key , index in existing . iteritems ( ) : <EOL> if key not in requested : <EOL> datastore_admin . DeleteIndex ( index ) <EOL> deleted += <NUM_LIT:1> <EOL> if created or deleted : <EOL> logging . debug ( '<STR_LIT>' , <EOL> created , deleted , len ( requested ) ) <EOL> def _UpdateIndexes ( self ) : <EOL> if self . _index_config_updater is not None : <EOL> self . _index_config_updater . UpdateIndexConfig ( ) <EOL> class StubQueryConverter ( datastore_pbs . _QueryConverter ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , entity_converter ) : <EOL> super ( StubQueryConverter , self ) . __init__ ( entity_converter ) <EOL> def v4_to_v3_compiled_cursor ( self , v4_cursor , v3_compiled_cursor ) : <EOL> """<STR_LIT>""" <EOL> v3_compiled_cursor . Clear ( ) <EOL> try : <EOL> v3_compiled_cursor . ParseFromString ( v4_cursor ) <EOL> except ProtocolBuffer . ProtocolBufferDecodeError : <EOL> raise datastore_pbs . InvalidConversionError ( '<STR_LIT>' ) <EOL> def v3_to_v4_compiled_cursor ( self , v3_compiled_cursor ) : <EOL> """<STR_LIT>""" <EOL> return v3_compiled_cursor . SerializeToString ( ) <EOL> def v4_to_v3_query ( self , v4_partition_id , v4_query , v3_query ) : <EOL> """<STR_LIT>""" <EOL> v3_query . Clear ( ) <EOL> if v4_partition_id . dataset_id ( ) : <EOL> v3_query . set_app ( v4_partition_id . dataset_id ( ) ) <EOL> if v4_partition_id . has_namespace ( ) : <EOL> v3_query . set_name_space ( v4_partition_id . namespace ( ) ) <EOL> v3_query . set_persist_offset ( True ) <EOL> v3_query . set_require_perfect_plan ( True ) <EOL> v3_query . set_compile ( True ) <EOL> if v4_query . has_limit ( ) : <EOL> v3_query . set_limit ( v4_query . limit ( ) ) <EOL> if v4_query . offset ( ) : <EOL> v3_query . set_offset ( v4_query . offset ( ) ) <EOL> if v4_query . has_start_cursor ( ) : <EOL> self . v4_to_v3_compiled_cursor ( v4_query . start_cursor ( ) , <EOL> v3_query . mutable_compiled_cursor ( ) ) <EOL> if v4_query . has_end_cursor ( ) : <EOL> self . v4_to_v3_compiled_cursor ( v4_query . end_cursor ( ) , <EOL> v3_query . mutable_end_compiled_cursor ( ) ) <EOL> if v4_query . kind_list ( ) : <EOL> datastore_pbs . check_conversion ( len ( v4_query . kind_list ( ) ) == <NUM_LIT:1> , <EOL> '<STR_LIT>' ) <EOL> v3_query . set_kind ( v4_query . kind ( <NUM_LIT:0> ) . name ( ) ) <EOL> has_key_projection = False <EOL> for prop in v4_query . projection_list ( ) : <EOL> if prop . property ( ) . name ( ) == datastore_pbs . PROPERTY_NAME_KEY : <EOL> has_key_projection = True <EOL> else : <EOL> v3_query . add_property_name ( prop . property ( ) . name ( ) ) <EOL> if has_key_projection and not v3_query . property_name_list ( ) : <EOL> v3_query . set_keys_only ( True ) <EOL> for prop in v4_query . group_by_list ( ) : <EOL> v3_query . add_group_by_property_name ( prop . name ( ) ) <EOL> self . __populate_v3_filters_from_v4 ( v4_query . filter ( ) , v3_query ) <EOL> for v4_order in v4_query . order_list ( ) : <EOL> v3_order = v3_query . add_order ( ) <EOL> v3_order . set_property ( v4_order . property ( ) . name ( ) ) <EOL> if v4_order . has_direction ( ) : <EOL> v3_order . set_direction ( v4_order . direction ( ) ) <EOL> def v3_to_v4_query ( self , v3_query , v4_query ) : <EOL> """<STR_LIT>""" <EOL> v4_query . Clear ( ) <EOL> datastore_pbs . check_conversion ( not v3_query . has_distinct ( ) , <EOL> '<STR_LIT>' ) <EOL> datastore_pbs . check_conversion ( v3_query . require_perfect_plan ( ) , <EOL> '<STR_LIT>' ) <EOL> if v3_query . has_limit ( ) : <EOL> v4_query . set_limit ( v3_query . limit ( ) ) <EOL> if v3_query . offset ( ) : <EOL> v4_query . set_offset ( v3_query . offset ( ) ) <EOL> if v3_query . has_compiled_cursor ( ) : <EOL> v4_query . set_start_cursor ( <EOL> self . v3_to_v4_compiled_cursor ( v3_query . compiled_cursor ( ) ) ) <EOL> if v3_query . has_end_compiled_cursor ( ) : <EOL> v4_query . set_end_cursor ( <EOL> self . v3_to_v4_compiled_cursor ( v3_query . end_compiled_cursor ( ) ) ) <EOL> if v3_query . has_kind ( ) : <EOL> v4_query . add_kind ( ) . set_name ( v3_query . kind ( ) ) <EOL> for name in v3_query . property_name_list ( ) : <EOL> v4_query . add_projection ( ) . mutable_property ( ) . set_name ( name ) <EOL> if v3_query . keys_only ( ) : <EOL> v4_query . add_projection ( ) . mutable_property ( ) . set_name ( <EOL> datastore_pbs . PROPERTY_NAME_KEY ) <EOL> for name in v3_query . group_by_property_name_list ( ) : <EOL> v4_query . add_group_by ( ) . set_name ( name ) <EOL> num_v4_filters = len ( v3_query . filter_list ( ) ) <EOL> if v3_query . has_ancestor ( ) : <EOL> num_v4_filters += <NUM_LIT:1> <EOL> if num_v4_filters == <NUM_LIT:1> : <EOL> get_property_filter = self . __get_property_filter_from_v4 <EOL> elif num_v4_filters >= <NUM_LIT:1> : <EOL> v4_query . mutable_filter ( ) . mutable_composite_filter ( ) . set_operator ( <EOL> datastore_v4_pb . CompositeFilter . AND ) <EOL> get_property_filter = self . __add_property_filter_from_v4 <EOL> if v3_query . has_ancestor ( ) : <EOL> self . _v3_query_to_v4_ancestor_filter ( v3_query , <EOL> get_property_filter ( v4_query ) ) <EOL> for v3_filter in v3_query . filter_list ( ) : <EOL> self . _v3_filter_to_v4_property_filter ( v3_filter , <EOL> get_property_filter ( v4_query ) ) <EOL> for v3_order in v3_query . order_list ( ) : <EOL> self . v3_order_to_v4_order ( v3_order , v4_query . add_order ( ) ) <EOL> def __get_property_filter_from_v4 ( self , v4_query ) : <EOL> """<STR_LIT>""" <EOL> return v4_query . mutable_filter ( ) . mutable_property_filter ( ) <EOL> def __add_property_filter_from_v4 ( self , v4_query ) : <EOL> """<STR_LIT>""" <EOL> v4_comp_filter = v4_query . mutable_filter ( ) . mutable_composite_filter ( ) <EOL> return v4_comp_filter . add_filter ( ) . mutable_property_filter ( ) <EOL> def __populate_v3_filters_from_v4 ( self , v4_filter , v3_query ) : <EOL> """<STR_LIT>""" <EOL> if v4_filter . has_property_filter ( ) : <EOL> v4_property_filter = v4_filter . property_filter ( ) <EOL> if ( v4_property_filter . operator ( ) <EOL> == datastore_v4_pb . PropertyFilter . HAS_ANCESTOR ) : <EOL> datastore_pbs . check_conversion ( <EOL> v4_property_filter . value ( ) . has_key_value ( ) , <EOL> '<STR_LIT>' ) <EOL> datastore_pbs . check_conversion ( ( v4_property_filter . property ( ) . name ( ) <EOL> == datastore_pbs . PROPERTY_NAME_KEY ) , <EOL> '<STR_LIT>' ) <EOL> datastore_pbs . check_conversion ( not v3_query . has_ancestor ( ) , <EOL> '<STR_LIT>' ) <EOL> self . _entity_converter . v4_to_v3_reference ( <EOL> v4_property_filter . value ( ) . key_value ( ) , <EOL> v3_query . mutable_ancestor ( ) ) <EOL> else : <EOL> v3_filter = v3_query . add_filter ( ) <EOL> property_name = v4_property_filter . property ( ) . name ( ) <EOL> v3_filter . set_op ( v4_property_filter . operator ( ) ) <EOL> datastore_pbs . check_conversion ( <EOL> not v4_property_filter . value ( ) . list_value_list ( ) , <EOL> ( '<STR_LIT>' <EOL> '<STR_LIT>' % ( '<STR_LIT>' , property_name ) ) ) <EOL> self . _entity_converter . v4_to_v3_property ( property_name , <EOL> False , <EOL> False , <EOL> v4_property_filter . value ( ) , <EOL> v3_filter . add_property ( ) ) <EOL> elif v4_filter . has_composite_filter ( ) : <EOL> datastore_pbs . check_conversion ( ( v4_filter . composite_filter ( ) . operator ( ) <EOL> == datastore_v4_pb . CompositeFilter . AND ) , <EOL> '<STR_LIT>' ) <EOL> for v4_sub_filter in v4_filter . composite_filter ( ) . filter_list ( ) : <EOL> self . __populate_v3_filters_from_v4 ( v4_sub_filter , v3_query ) <EOL> def v1_to_v3_compiled_cursor ( self , v1_cursor , v3_compiled_cursor ) : <EOL> """<STR_LIT>""" <EOL> v3_compiled_cursor . Clear ( ) <EOL> try : <EOL> v3_compiled_cursor . ParseFromString ( v1_cursor ) <EOL> except ProtocolBuffer . ProtocolBufferDecodeError : <EOL> raise datastore_pbs . InvalidConversionError ( '<STR_LIT>' ) <EOL> def v3_to_v1_compiled_cursor ( self , v3_compiled_cursor ) : <EOL> """<STR_LIT>""" <EOL> return v3_compiled_cursor . SerializeToString ( ) <EOL> def v1_to_v3_query ( self , v1_partition_id , v1_query , v3_query ) : <EOL> """<STR_LIT>""" <EOL> v3_query . Clear ( ) <EOL> if v1_partition_id . project_id : <EOL> v3_query . set_app ( v1_partition_id . project_id ) <EOL> if v1_partition_id . namespace_id : <EOL> v3_query . set_name_space ( v1_partition_id . namespace_id ) <EOL> v3_query . set_persist_offset ( True ) <EOL> v3_query . set_require_perfect_plan ( True ) <EOL> v3_query . set_compile ( True ) <EOL> if v1_query . HasField ( '<STR_LIT>' ) : <EOL> v3_query . set_limit ( v1_query . limit . value ) <EOL> if v1_query . offset : <EOL> v3_query . set_offset ( v1_query . offset ) <EOL> if v1_query . start_cursor : <EOL> self . v1_to_v3_compiled_cursor ( v1_query . start_cursor , <EOL> v3_query . mutable_compiled_cursor ( ) ) <EOL> if v1_query . end_cursor : <EOL> self . v1_to_v3_compiled_cursor ( v1_query . end_cursor , <EOL> v3_query . mutable_end_compiled_cursor ( ) ) <EOL> if v1_query . kind : <EOL> datastore_pbs . check_conversion ( len ( v1_query . kind ) == <NUM_LIT:1> , <EOL> '<STR_LIT>' ) <EOL> v3_query . set_kind ( v1_query . kind [ <NUM_LIT:0> ] . name . encode ( '<STR_LIT>' ) ) <EOL> has_key_projection = False <EOL> for prop in v1_query . projection : <EOL> if prop . property . name == datastore_pbs . PROPERTY_NAME_KEY : <EOL> has_key_projection = True <EOL> else : <EOL> v3_query . add_property_name ( prop . property . name ) <EOL> if has_key_projection and not v3_query . property_name_size ( ) : <EOL> v3_query . set_keys_only ( True ) <EOL> for prop in v1_query . distinct_on : <EOL> v3_query . add_group_by_property_name ( prop . name ) <EOL> self . __populate_v3_filters_from_v1 ( v1_query . filter , v3_query ) <EOL> for v1_order in v1_query . order : <EOL> v3_order = v3_query . add_order ( ) <EOL> v3_order . set_property ( v1_order . property . name ) <EOL> if v1_order . direction : <EOL> v3_order . set_direction ( v1_order . direction ) <EOL> def v3_to_v1_query ( self , v3_query , v1_query ) : <EOL> """<STR_LIT>""" <EOL> v1_query . Clear ( ) <EOL> datastore_pbs . check_conversion ( not v3_query . has_distinct ( ) , <EOL> '<STR_LIT>' ) <EOL> datastore_pbs . check_conversion ( v3_query . require_perfect_plan ( ) , <EOL> '<STR_LIT>' ) <EOL> if v3_query . has_limit ( ) : <EOL> v1_query . limit . value = v3_query . limit ( ) <EOL> if v3_query . offset ( ) : <EOL> v1_query . offset = v3_query . offset ( ) <EOL> if v3_query . has_compiled_cursor ( ) : <EOL> v1_query . start_cursor = ( <EOL> self . v3_to_v1_compiled_cursor ( v3_query . compiled_cursor ( ) ) ) <EOL> if v3_query . has_end_compiled_cursor ( ) : <EOL> v1_query . end_cursor = ( <EOL> self . v3_to_v1_compiled_cursor ( v3_query . end_compiled_cursor ( ) ) ) <EOL> if v3_query . has_kind ( ) : <EOL> v1_query . kind . add ( ) . name = v3_query . kind ( ) <EOL> for name in v3_query . property_name_list ( ) : <EOL> v1_query . projection . add ( ) . property . name = name <EOL> if v3_query . keys_only ( ) : <EOL> v1_query . projection . add ( ) . property . name = ( <EOL> datastore_pbs . PROPERTY_NAME_KEY ) <EOL> for name in v3_query . group_by_property_name_list ( ) : <EOL> v1_query . distinct_on . add ( ) . name = name <EOL> num_v1_filters = len ( v3_query . filter_list ( ) ) <EOL> if v3_query . has_ancestor ( ) : <EOL> num_v1_filters += <NUM_LIT:1> <EOL> if num_v1_filters == <NUM_LIT:1> : <EOL> get_property_filter = self . __get_property_filter_from_v1 <EOL> elif num_v1_filters >= <NUM_LIT:1> : <EOL> v1_query . filter . composite_filter . operator = ( <EOL> googledatastore . CompositeFilter . AND ) <EOL> get_property_filter = self . __add_property_filter_from_V1 <EOL> if v3_query . has_ancestor ( ) : <EOL> self . _v3_query_to_v1_ancestor_filter ( v3_query , <EOL> get_property_filter ( v1_query ) ) <EOL> for v3_filter in v3_query . filter_list ( ) : <EOL> self . _v3_filter_to_v1_property_filter ( v3_filter , <EOL> get_property_filter ( v1_query ) ) <EOL> for v3_order in v3_query . order_list ( ) : <EOL> self . v3_order_to_v1_order ( v3_order , v1_query . order . add ( ) ) <EOL> def __get_property_filter_from_v1 ( self , v1_query ) : <EOL> """<STR_LIT>""" <EOL> return v1_query . filter . add ( ) . property_filter <EOL> def __add_property_filter_from_v1 ( self , v1_query ) : <EOL> """<STR_LIT>""" <EOL> v1_comp_filter = v1_query . filter . composite_filter <EOL> return v1_comp_filter . filter . add ( ) . property_filter <EOL> def __populate_v3_filters_from_v1 ( self , v1_filter , v3_query ) : <EOL> """<STR_LIT>""" <EOL> filter_type = v1_filter . WhichOneof ( '<STR_LIT>' ) <EOL> if filter_type == '<STR_LIT>' : <EOL> v1_property_filter = v1_filter . property_filter <EOL> v1_property_name = v1_property_filter . property . name <EOL> if ( v1_property_filter . op <EOL> == googledatastore . PropertyFilter . HAS_ANCESTOR ) : <EOL> datastore_pbs . check_conversion ( <EOL> v1_property_filter . value . HasField ( '<STR_LIT>' ) , <EOL> '<STR_LIT>' ) <EOL> datastore_pbs . check_conversion ( ( v1_property_name <EOL> == datastore_pbs . PROPERTY_NAME_KEY ) , <EOL> '<STR_LIT>' ) <EOL> datastore_pbs . check_conversion ( not v3_query . has_ancestor ( ) , <EOL> '<STR_LIT>' ) <EOL> self . _entity_converter . v1_to_v3_reference ( <EOL> v1_property_filter . value . key_value , <EOL> v3_query . mutable_ancestor ( ) ) <EOL> else : <EOL> v3_filter = v3_query . add_filter ( ) <EOL> property_name = v1_property_name <EOL> v3_filter . set_op ( v1_property_filter . op ) <EOL> datastore_pbs . check_conversion ( <EOL> not v1_property_filter . value . HasField ( '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' <EOL> '<STR_LIT>' % ( '<STR_LIT>' , property_name ) ) ) <EOL> self . _entity_converter . v1_to_v3_property ( property_name , <EOL> False , <EOL> False , <EOL> v1_property_filter . value , <EOL> v3_filter . add_property ( ) ) <EOL> elif filter_type == '<STR_LIT>' : <EOL> datastore_pbs . check_conversion ( ( v1_filter . composite_filter . op <EOL> == googledatastore . CompositeFilter . AND ) , <EOL> '<STR_LIT>' ) <EOL> for v1_sub_filter in v1_filter . composite_filter . filters : <EOL> self . __populate_v3_filters_from_v1 ( v1_sub_filter , v3_query ) <EOL> def get_query_converter ( id_resolver = None ) : <EOL> """<STR_LIT>""" <EOL> return StubQueryConverter ( datastore_pbs . get_entity_converter ( id_resolver ) ) <EOL> class StubServiceConverter ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , entity_converter , query_converter ) : <EOL> self . _entity_converter = entity_converter <EOL> self . _query_converter = query_converter <EOL> def v1_to_v3_cursor ( self , v1_query_handle , v3_cursor ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> v3_cursor . ParseFromString ( v1_query_handle ) <EOL> except ProtocolBuffer . ProtocolBufferDecodeError : <EOL> raise datastore_pbs . InvalidConversionError ( '<STR_LIT>' ) <EOL> return v3_cursor <EOL> def _v3_to_v1_query_handle ( self , v3_cursor ) : <EOL> """<STR_LIT>""" <EOL> return v3_cursor . SerializeToString ( ) <EOL> def v1_to_v3_txn ( self , v1_txn , v3_txn ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> v3_txn . ParseFromString ( v1_txn ) <EOL> except ProtocolBuffer . ProtocolBufferDecodeError : <EOL> raise datastore_pbs . InvalidConversionError ( '<STR_LIT>' ) <EOL> return v3_txn <EOL> def _v3_to_v1_txn ( self , v3_txn ) : <EOL> """<STR_LIT>""" <EOL> return v3_txn . SerializeToString ( ) <EOL> def v1_to_v3_begin_transaction_req ( self , app_id , v1_req ) : <EOL> """<STR_LIT>""" <EOL> v3_req = datastore_pb . BeginTransactionRequest ( ) <EOL> v3_req . set_app ( app_id ) <EOL> v3_req . set_allow_multiple_eg ( True ) <EOL> return v3_req <EOL> def v3_to_v1_begin_transaction_resp ( self , v3_resp ) : <EOL> """<STR_LIT>""" <EOL> v1_resp = googledatastore . BeginTransactionResponse ( ) <EOL> v1_resp . transaction = self . _v3_to_v1_txn ( v3_resp ) <EOL> return v1_resp <EOL> def v1_rollback_req_to_v3_txn ( self , v1_req ) : <EOL> """<STR_LIT>""" <EOL> v3_txn = datastore_pb . Transaction ( ) <EOL> self . v1_to_v3_txn ( v1_req . transaction , v3_txn ) <EOL> return v3_txn <EOL> def v1_commit_req_to_v3_txn ( self , v1_req ) : <EOL> """<STR_LIT>""" <EOL> v3_txn = datastore_pb . Transaction ( ) <EOL> self . v1_to_v3_txn ( v1_req . transaction , v3_txn ) <EOL> return v3_txn <EOL> def v1_run_query_req_to_v3_query ( self , v1_req , new_txn = None ) : <EOL> """<STR_LIT>""" <EOL> consistency_type = v1_req . read_options . WhichOneof ( '<STR_LIT>' ) <EOL> datastore_pbs . check_conversion ( not v1_req . HasField ( '<STR_LIT>' ) , <EOL> '<STR_LIT>' ) <EOL> if ( new_txn is None ) == ( consistency_type == '<STR_LIT>' ) : <EOL> raise datastore_errors . InternalError ( '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> v3_query = datastore_pb . Query ( ) <EOL> self . _query_converter . v1_to_v3_query ( v1_req . partition_id , v1_req . query , <EOL> v3_query ) <EOL> if consistency_type == '<STR_LIT>' : <EOL> self . v1_to_v3_txn ( v1_req . read_options . transaction , <EOL> v3_query . mutable_transaction ( ) ) <EOL> elif consistency_type == '<STR_LIT>' : <EOL> self . v1_to_v3_txn ( new_txn , v3_query . mutable_transaction ( ) ) <EOL> elif consistency_type == '<STR_LIT>' : <EOL> read_consistency = v1_req . read_options . read_consistency <EOL> if read_consistency == googledatastore . ReadOptions . EVENTUAL : <EOL> v3_query . set_strong ( False ) <EOL> v3_query . set_failover_ms ( - <NUM_LIT:1> ) <EOL> elif read_consistency == googledatastore . ReadOptions . STRONG : <EOL> v3_query . set_strong ( True ) <EOL> elif ( read_consistency != <EOL> googledatastore . ReadOptions . READ_CONSISTENCY_UNSPECIFIED ) : <EOL> raise datastore_errors . InternalError ( '<STR_LIT>' <EOL> % read_consistency ) <EOL> elif consistency_type is not None : <EOL> raise datastore_errors . InternalError ( '<STR_LIT>' <EOL> % consistency_type ) <EOL> return v3_query <EOL> def v3_to_v1_run_query_req ( self , v3_req ) : <EOL> """<STR_LIT>""" <EOL> v1_req = googledatastore . RunQueryRequest ( ) <EOL> v1_partition_id = v1_req . partition_id <EOL> v1_partition_id . project_id = v3_req . app ( ) <EOL> if v3_req . name_space ( ) : <EOL> v1_partition_id . namespace = v3_req . name_space ( ) <EOL> if v3_req . has_transaction ( ) : <EOL> v1_req . read_options . transaction = self . _v3_to_v1_txn ( v3_req . transaction ( ) ) <EOL> elif v3_req . strong ( ) : <EOL> v1_req . read_options . read_consistency = ( <EOL> googledatastore . ReadOptions . STRONG ) <EOL> elif v3_req . has_strong ( ) : <EOL> v1_req . read_options . read_consistency = ( <EOL> googledatastore . ReadOptions . EVENTUAL ) <EOL> self . _query_converter . v3_to_v1_query ( v3_req , v1_req . mutable_query ( ) ) <EOL> return v1_req <EOL> def v1_run_query_resp_to_v3_query_result ( self , v1_resp ) : <EOL> """<STR_LIT>""" <EOL> v3_resp = self . v1_to_v3_query_result ( v1_resp . batch ) <EOL> return v3_resp <EOL> def v3_to_v1_run_query_resp ( self , v3_resp , new_txn = None ) : <EOL> """<STR_LIT>""" <EOL> v1_resp = googledatastore . RunQueryResponse ( ) <EOL> self . v3_to_v1_query_result_batch ( v3_resp , v1_resp . batch ) <EOL> if new_txn : <EOL> v1_resp . transaction = new_txn <EOL> return v1_resp <EOL> def v1_to_v3_get_req ( self , v1_req , new_txn = None ) : <EOL> """<STR_LIT>""" <EOL> consistency_type = v1_req . read_options . WhichOneof ( '<STR_LIT>' ) <EOL> if ( new_txn is None ) == ( consistency_type == '<STR_LIT>' ) : <EOL> raise datastore_errors . InternalError ( '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> v3_req = datastore_pb . GetRequest ( ) <EOL> v3_req . set_allow_deferred ( True ) <EOL> if consistency_type == '<STR_LIT>' : <EOL> self . v1_to_v3_txn ( v1_req . read_options . transaction , <EOL> v3_req . mutable_transaction ( ) ) <EOL> elif consistency_type == '<STR_LIT>' : <EOL> self . v1_to_v3_txn ( new_txn , v3_req . mutable_transaction ( ) ) <EOL> elif consistency_type == '<STR_LIT>' : <EOL> read_consistency = v1_req . read_options . read_consistency <EOL> if read_consistency == googledatastore . ReadOptions . EVENTUAL : <EOL> v3_req . set_strong ( False ) <EOL> v3_req . set_failover_ms ( - <NUM_LIT:1> ) <EOL> elif read_consistency == googledatastore . ReadOptions . STRONG : <EOL> v3_req . set_strong ( True ) <EOL> elif ( read_consistency != <EOL> googledatastore . ReadOptions . READ_CONSISTENCY_UNSPECIFIED ) : <EOL> raise datastore_errors . InternalError ( '<STR_LIT>' <EOL> % read_consistency ) <EOL> elif consistency_type is not None : <EOL> raise datastore_errors . InternalError ( '<STR_LIT>' <EOL> % consistency_type ) <EOL> for v1_key in v1_req . keys : <EOL> self . _entity_converter . v1_to_v3_reference ( v1_key , v3_req . add_key ( ) ) <EOL> return v3_req <EOL> def v3_to_v1_lookup_resp ( self , v3_resp , new_txn = None ) : <EOL> """<STR_LIT>""" <EOL> v1_resp = googledatastore . LookupResponse ( ) <EOL> if new_txn : <EOL> v1_resp . transaction = new_txn <EOL> for v3_ref in v3_resp . deferred_list ( ) : <EOL> self . _entity_converter . v3_to_v1_key ( v3_ref , v1_resp . deferred . add ( ) ) <EOL> for v3_entity in v3_resp . entity_list ( ) : <EOL> if v3_entity . has_entity ( ) : <EOL> v1_entity_result = v1_resp . found . add ( ) <EOL> self . _entity_converter . v3_to_v1_entity ( <EOL> v3_entity . entity ( ) , v1_entity_result . entity ) <EOL> if v3_entity . has_key ( ) : <EOL> v1_entity_result = v1_resp . missing . add ( ) <EOL> self . _entity_converter . v3_to_v1_key ( <EOL> v3_entity . key ( ) , v1_entity_result . entity . key ) <EOL> if v3_entity . has_version ( ) : <EOL> v1_entity_result . version = v3_entity . version ( ) <EOL> return v1_resp <EOL> def v1_to_v3_query_result ( self , v1_batch ) : <EOL> """<STR_LIT>""" <EOL> v3_result = datastore_pb . QueryResult ( ) <EOL> v3_result . set_more_results ( <EOL> ( v1_batch . more_results <EOL> == googledatastore . QueryResultBatch . NOT_FINISHED ) ) <EOL> if v1_batch . end_cursor : <EOL> self . _query_converter . v1_to_v3_compiled_cursor ( <EOL> v1_batch . end_cursor , v3_result . mutable_compiled_cursor ( ) ) <EOL> if v1_batch . skipped_cursor : <EOL> self . _query_converter . v1_to_v3_compiled_cursor ( <EOL> v1_batch . skipped_cursor , <EOL> v3_result . mutable_skipped_results_compiled_cursor ( ) ) <EOL> if v1_batch . entity_result_type ( ) == googledatastore . EntityResult . PROJECTION : <EOL> v3_result . set_index_only ( True ) <EOL> elif v1_batch . entity_result_type ( ) == googledatastore . EntityResult . KEY_ONLY : <EOL> v3_result . set_keys_only ( True ) <EOL> if v1_batch . skipped_results : <EOL> v3_result . set_skipped_results ( v1_batch . skipped_results ) <EOL> for v1_entity_result in v1_batch . entity_result : <EOL> v3_entity = v3_result . add_result ( ) <EOL> self . _entity_converter . v1_to_v3_entity ( v1_entity_result . entity , v3_entity ) <EOL> if v1_entity_result . cursor : <EOL> cursor = v3_result . add_result_compiled_cursor ( ) <EOL> self . _query_converter . v1_to_v3_compiled_cursor ( v1_entity_result . cursor , <EOL> cursor ) <EOL> if v1_batch . entity_result_type != googledatastore . EntityResult . FULL : <EOL> v3_entity . clear_entity_group ( ) <EOL> return v3_result <EOL> def v3_to_v1_query_result_batch ( self , v3_result , v1_batch ) : <EOL> """<STR_LIT>""" <EOL> v1_batch . Clear ( ) <EOL> if v3_result . more_results ( ) : <EOL> v1_batch . more_results = googledatastore . QueryResultBatch . NOT_FINISHED <EOL> else : <EOL> v1_batch . more_results = ( <EOL> googledatastore . QueryResultBatch . MORE_RESULTS_AFTER_LIMIT ) <EOL> if v3_result . has_compiled_cursor ( ) : <EOL> v1_batch . end_cursor = ( <EOL> self . _query_converter . v3_to_v1_compiled_cursor ( <EOL> v3_result . compiled_cursor ( ) ) ) <EOL> if v3_result . has_skipped_results_compiled_cursor ( ) : <EOL> v1_batch . skipped_cursor = ( <EOL> self . _query_converter . v3_to_v1_compiled_cursor ( <EOL> v3_result . skipped_results_compiled_cursor ( ) ) ) <EOL> if v3_result . keys_only ( ) : <EOL> v1_batch . entity_result_type = googledatastore . EntityResult . KEY_ONLY <EOL> elif v3_result . index_only ( ) : <EOL> v1_batch . entity_result_type = googledatastore . EntityResult . PROJECTION <EOL> else : <EOL> v1_batch . entity_result_type = googledatastore . EntityResult . FULL <EOL> if v3_result . has_skipped_results ( ) : <EOL> v1_batch . skipped_results = v3_result . skipped_results ( ) <EOL> for v3_entity , v3_version , v3_cursor in itertools . izip_longest ( <EOL> v3_result . result_list ( ) , <EOL> v3_result . version_list ( ) , <EOL> v3_result . result_compiled_cursor_list ( ) ) : <EOL> v1_entity_result = v1_batch . entity_results . add ( ) <EOL> self . _entity_converter . v3_to_v1_entity ( v3_entity , <EOL> v1_entity_result . entity ) <EOL> if v3_version is not None : <EOL> v1_entity_result . version = v3_version <EOL> if v3_cursor is not None : <EOL> v1_entity_result . cursor = ( <EOL> self . _query_converter . v3_to_v1_compiled_cursor ( v3_cursor ) ) <EOL> def v4_to_v3_cursor ( self , v4_query_handle , v3_cursor ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> v3_cursor . ParseFromString ( v4_query_handle ) <EOL> except ProtocolBuffer . ProtocolBufferDecodeError : <EOL> raise datastore_pbs . InvalidConversionError ( '<STR_LIT>' ) <EOL> return v3_cursor <EOL> def _v3_to_v4_query_handle ( self , v3_cursor ) : <EOL> """<STR_LIT>""" <EOL> return v3_cursor . SerializeToString ( ) <EOL> def v4_to_v3_txn ( self , v4_txn , v3_txn ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> v3_txn . ParseFromString ( v4_txn ) <EOL> except ProtocolBuffer . ProtocolBufferDecodeError : <EOL> raise datastore_pbs . InvalidConversionError ( '<STR_LIT>' ) <EOL> return v3_txn <EOL> def _v3_to_v4_txn ( self , v3_txn ) : <EOL> """<STR_LIT>""" <EOL> return v3_txn . SerializeToString ( ) <EOL> def v4_to_v3_begin_transaction_req ( self , app_id , v4_req ) : <EOL> """<STR_LIT>""" <EOL> v3_req = datastore_pb . BeginTransactionRequest ( ) <EOL> v3_req . set_app ( app_id ) <EOL> v3_req . set_allow_multiple_eg ( v4_req . cross_group ( ) ) <EOL> return v3_req <EOL> def v3_to_v4_begin_transaction_resp ( self , v3_resp ) : <EOL> """<STR_LIT>""" <EOL> v4_resp = datastore_v4_pb . BeginTransactionResponse ( ) <EOL> v4_resp . set_transaction ( self . _v3_to_v4_txn ( v3_resp ) ) <EOL> return v4_resp <EOL> def v4_rollback_req_to_v3_txn ( self , v4_req ) : <EOL> """<STR_LIT>""" <EOL> v3_txn = datastore_pb . Transaction ( ) <EOL> self . v4_to_v3_txn ( v4_req . transaction ( ) , v3_txn ) <EOL> return v3_txn <EOL> def v4_commit_req_to_v3_txn ( self , v4_req ) : <EOL> """<STR_LIT>""" <EOL> v3_txn = datastore_pb . Transaction ( ) <EOL> self . v4_to_v3_txn ( v4_req . transaction ( ) , v3_txn ) <EOL> return v3_txn <EOL> def v4_run_query_req_to_v3_query ( self , v4_req ) : <EOL> """<STR_LIT>""" <EOL> datastore_pbs . check_conversion ( not v4_req . has_gql_query ( ) , <EOL> '<STR_LIT>' ) <EOL> v3_query = datastore_pb . Query ( ) <EOL> self . _query_converter . v4_to_v3_query ( v4_req . partition_id ( ) , v4_req . query ( ) , <EOL> v3_query ) <EOL> if v4_req . has_suggested_batch_size ( ) : <EOL> v3_query . set_count ( v4_req . suggested_batch_size ( ) ) <EOL> read_options = v4_req . read_options ( ) <EOL> if read_options . has_transaction ( ) : <EOL> self . v4_to_v3_txn ( read_options . transaction ( ) , <EOL> v3_query . mutable_transaction ( ) ) <EOL> elif ( read_options . read_consistency ( ) <EOL> == datastore_v4_pb . ReadOptions . EVENTUAL ) : <EOL> v3_query . set_strong ( False ) <EOL> v3_query . set_failover_ms ( - <NUM_LIT:1> ) <EOL> elif read_options . read_consistency ( ) == datastore_v4_pb . ReadOptions . STRONG : <EOL> v3_query . set_strong ( True ) <EOL> if v4_req . has_min_safe_time_seconds ( ) : <EOL> v3_query . set_min_safe_time_seconds ( v4_req . min_safe_time_seconds ( ) ) <EOL> return v3_query <EOL> def v3_to_v4_run_query_req ( self , v3_req ) : <EOL> """<STR_LIT>""" <EOL> v4_req = datastore_v4_pb . RunQueryRequest ( ) <EOL> v4_partition_id = v4_req . mutable_partition_id ( ) <EOL> v4_partition_id . set_dataset_id ( v3_req . app ( ) ) <EOL> if v3_req . name_space ( ) : <EOL> v4_partition_id . set_namespace ( v3_req . name_space ( ) ) <EOL> if v3_req . has_count ( ) : <EOL> v4_req . set_suggested_batch_size ( v3_req . count ( ) ) <EOL> if v3_req . has_transaction ( ) : <EOL> v4_req . mutable_read_options ( ) . set_transaction ( <EOL> self . _v3_to_v4_txn ( v3_req . transaction ( ) ) ) <EOL> elif v3_req . strong ( ) : <EOL> v4_req . mutable_read_options ( ) . set_read_consistency ( <EOL> datastore_v4_pb . ReadOptions . STRONG ) <EOL> elif v3_req . has_strong ( ) : <EOL> v4_req . mutable_read_options ( ) . set_read_consistency ( <EOL> datastore_v4_pb . ReadOptions . EVENTUAL ) <EOL> if v3_req . has_min_safe_time_seconds ( ) : <EOL> v4_req . set_min_safe_time_seconds ( v3_req . min_safe_time_seconds ( ) ) <EOL> self . _query_converter . v3_to_v4_query ( v3_req , v4_req . mutable_query ( ) ) <EOL> return v4_req <EOL> def v4_run_query_resp_to_v3_query_result ( self , v4_resp ) : <EOL> """<STR_LIT>""" <EOL> v3_resp = self . v4_to_v3_query_result ( v4_resp . batch ( ) ) <EOL> if v4_resp . has_query_handle ( ) : <EOL> self . v4_to_v3_cursor ( v4_resp . query_handle ( ) , v3_resp . mutable_cursor ( ) ) <EOL> return v3_resp <EOL> def v3_to_v4_run_query_resp ( self , v3_resp ) : <EOL> """<STR_LIT>""" <EOL> v4_resp = datastore_v4_pb . RunQueryResponse ( ) <EOL> self . v3_to_v4_query_result_batch ( v3_resp , v4_resp . mutable_batch ( ) ) <EOL> if v3_resp . has_cursor ( ) : <EOL> v4_resp . set_query_handle ( <EOL> self . _query_converter . v3_to_v4_compiled_cursor ( v3_resp . cursor ( ) ) ) <EOL> return v4_resp <EOL> def v4_to_v3_next_req ( self , v4_req ) : <EOL> """<STR_LIT>""" <EOL> v3_req = datastore_pb . NextRequest ( ) <EOL> v3_req . set_compile ( True ) <EOL> self . v4_to_v3_cursor ( v4_req . query_handle ( ) , v3_req . mutable_cursor ( ) ) <EOL> return v3_req <EOL> def v3_to_v4_continue_query_resp ( self , v3_resp ) : <EOL> """<STR_LIT>""" <EOL> v4_resp = datastore_v4_pb . ContinueQueryResponse ( ) <EOL> self . v3_to_v4_query_result_batch ( v3_resp , v4_resp . mutable_batch ( ) ) <EOL> return v4_resp <EOL> def v4_to_v3_get_req ( self , v4_req ) : <EOL> """<STR_LIT>""" <EOL> v3_req = datastore_pb . GetRequest ( ) <EOL> v3_req . set_allow_deferred ( True ) <EOL> if v4_req . read_options ( ) . has_transaction ( ) : <EOL> self . v4_to_v3_txn ( v4_req . read_options ( ) . transaction ( ) , <EOL> v3_req . mutable_transaction ( ) ) <EOL> elif ( v4_req . read_options ( ) . read_consistency ( ) <EOL> == datastore_v4_pb . ReadOptions . EVENTUAL ) : <EOL> v3_req . set_strong ( False ) <EOL> v3_req . set_failover_ms ( - <NUM_LIT:1> ) <EOL> elif ( v4_req . read_options ( ) . read_consistency ( ) <EOL> == datastore_v4_pb . ReadOptions . STRONG ) : <EOL> v3_req . set_strong ( True ) <EOL> for v4_key in v4_req . key_list ( ) : <EOL> self . _entity_converter . v4_to_v3_reference ( v4_key , v3_req . add_key ( ) ) <EOL> return v3_req <EOL> def v3_to_v4_lookup_resp ( self , v3_resp ) : <EOL> """<STR_LIT>""" <EOL> v4_resp = datastore_v4_pb . LookupResponse ( ) <EOL> for v3_ref in v3_resp . deferred_list ( ) : <EOL> self . _entity_converter . v3_to_v4_key ( v3_ref , v4_resp . add_deferred ( ) ) <EOL> for v3_entity in v3_resp . entity_list ( ) : <EOL> if v3_entity . has_entity ( ) : <EOL> self . _entity_converter . v3_to_v4_entity ( <EOL> v3_entity . entity ( ) , <EOL> v4_resp . add_found ( ) . mutable_entity ( ) ) <EOL> if v3_entity . has_key ( ) : <EOL> self . _entity_converter . v3_to_v4_key ( <EOL> v3_entity . key ( ) , <EOL> v4_resp . add_missing ( ) . mutable_entity ( ) . mutable_key ( ) ) <EOL> return v4_resp <EOL> def v4_to_v3_query_result ( self , v4_batch ) : <EOL> """<STR_LIT>""" <EOL> v3_result = datastore_pb . QueryResult ( ) <EOL> v3_result . set_more_results ( <EOL> ( v4_batch . more_results ( ) <EOL> == datastore_v4_pb . QueryResultBatch . NOT_FINISHED ) ) <EOL> if v4_batch . has_end_cursor ( ) : <EOL> self . _query_converter . v4_to_v3_compiled_cursor ( <EOL> v4_batch . end_cursor ( ) , v3_result . mutable_compiled_cursor ( ) ) <EOL> if v4_batch . has_skipped_cursor ( ) : <EOL> self . _query_converter . v4_to_v3_compiled_cursor ( <EOL> v4_batch . skipped_cursor ( ) , <EOL> v3_result . mutable_skipped_results_compiled_cursor ( ) ) <EOL> if v4_batch . entity_result_type ( ) == datastore_v4_pb . EntityResult . PROJECTION : <EOL> v3_result . set_index_only ( True ) <EOL> elif v4_batch . entity_result_type ( ) == datastore_v4_pb . EntityResult . KEY_ONLY : <EOL> v3_result . set_keys_only ( True ) <EOL> if v4_batch . has_skipped_results ( ) : <EOL> v3_result . set_skipped_results ( v4_batch . skipped_results ( ) ) <EOL> for v4_entity in v4_batch . entity_result_list ( ) : <EOL> v3_entity = v3_result . add_result ( ) <EOL> self . _entity_converter . v4_to_v3_entity ( v4_entity . entity ( ) , v3_entity ) <EOL> if v4_entity . has_cursor ( ) : <EOL> cursor = v3_result . add_result_compiled_cursor ( ) <EOL> self . _query_converter . v4_to_v3_compiled_cursor ( v4_entity . cursor ( ) , <EOL> cursor ) <EOL> if v4_batch . entity_result_type ( ) != datastore_v4_pb . EntityResult . FULL : <EOL> v3_entity . clear_entity_group ( ) <EOL> return v3_result <EOL> def v3_to_v4_query_result_batch ( self , v3_result , v4_batch ) : <EOL> """<STR_LIT>""" <EOL> v4_batch . Clear ( ) <EOL> if v3_result . more_results ( ) : <EOL> v4_batch . set_more_results ( datastore_v4_pb . QueryResultBatch . NOT_FINISHED ) <EOL> else : <EOL> v4_batch . set_more_results ( <EOL> datastore_v4_pb . QueryResultBatch . MORE_RESULTS_AFTER_LIMIT ) <EOL> if v3_result . has_compiled_cursor ( ) : <EOL> v4_batch . set_end_cursor ( <EOL> self . _query_converter . v3_to_v4_compiled_cursor ( <EOL> v3_result . compiled_cursor ( ) ) ) <EOL> if v3_result . has_skipped_results_compiled_cursor ( ) : <EOL> v4_batch . set_skipped_cursor ( <EOL> self . _query_converter . v3_to_v4_compiled_cursor ( <EOL> v3_result . skipped_results_compiled_cursor ( ) ) ) <EOL> if v3_result . keys_only ( ) : <EOL> v4_batch . set_entity_result_type ( datastore_v4_pb . EntityResult . KEY_ONLY ) <EOL> elif v3_result . index_only ( ) : <EOL> v4_batch . set_entity_result_type ( datastore_v4_pb . EntityResult . PROJECTION ) <EOL> else : <EOL> v4_batch . set_entity_result_type ( datastore_v4_pb . EntityResult . FULL ) <EOL> if v3_result . has_skipped_results ( ) : <EOL> v4_batch . set_skipped_results ( v3_result . skipped_results ( ) ) <EOL> for v3_entity , v3_cursor in itertools . izip_longest ( <EOL> v3_result . result_list ( ) , <EOL> v3_result . result_compiled_cursor_list ( ) ) : <EOL> v4_entity_result = datastore_v4_pb . EntityResult ( ) <EOL> self . _entity_converter . v3_to_v4_entity ( v3_entity , <EOL> v4_entity_result . mutable_entity ( ) ) <EOL> if v3_cursor is not None : <EOL> v4_entity_result . set_cursor ( <EOL> self . _query_converter . v3_to_v4_compiled_cursor ( v3_cursor ) ) <EOL> v4_batch . entity_result_list ( ) . append ( v4_entity_result ) <EOL> def get_service_converter ( id_resolver = None ) : <EOL> """<STR_LIT>""" <EOL> query_converter = get_query_converter ( id_resolver ) <EOL> return StubServiceConverter ( query_converter . get_entity_converter ( ) , <EOL> query_converter ) <EOL> def ReverseBitsInt64 ( v ) : <EOL> """<STR_LIT>""" <EOL> v = ( ( v >> <NUM_LIT:1> ) & <NUM_LIT> ) | ( ( v & <NUM_LIT> ) << <NUM_LIT:1> ) <EOL> v = ( ( v >> <NUM_LIT:2> ) & <NUM_LIT> ) | ( ( v & <NUM_LIT> ) << <NUM_LIT:2> ) <EOL> v = ( ( v >> <NUM_LIT:4> ) & <NUM_LIT> ) | ( ( v & <NUM_LIT> ) << <NUM_LIT:4> ) <EOL> v = ( ( v >> <NUM_LIT:8> ) & <NUM_LIT> ) | ( ( v & <NUM_LIT> ) << <NUM_LIT:8> ) <EOL> v = ( ( v >> <NUM_LIT:16> ) & <NUM_LIT> ) | ( ( v & <NUM_LIT> ) << <NUM_LIT:16> ) <EOL> v = int ( ( v >> <NUM_LIT:32> ) | ( v << <NUM_LIT:32> ) & <NUM_LIT> ) <EOL> return v <EOL> def ToScatteredId ( v ) : <EOL> """<STR_LIT>""" <EOL> if v >= _MAX_SCATTERED_COUNTER : <EOL> raise datastore_errors . BadArgumentError ( '<STR_LIT>' % v ) <EOL> return _MAX_SEQUENTIAL_ID + <NUM_LIT:1> + long ( ReverseBitsInt64 ( v << _SCATTER_SHIFT ) ) <EOL> def IdToCounter ( k ) : <EOL> """<STR_LIT>""" <EOL> if k > _MAX_SCATTERED_ID : <EOL> return <NUM_LIT:0> , SCATTERED <EOL> elif k > _MAX_SEQUENTIAL_ID and k <= _MAX_SCATTERED_ID : <EOL> return long ( ReverseBitsInt64 ( k ) >> _SCATTER_SHIFT ) , SCATTERED <EOL> elif k > <NUM_LIT:0> : <EOL> return long ( k ) , SEQUENTIAL <EOL> else : <EOL> raise datastore_errors . BadArgumentError ( '<STR_LIT>' % k ) <EOL> def CompareEntityPbByKey ( a , b ) : <EOL> """<STR_LIT>""" <EOL> return cmp ( datastore_types . Key . _FromPb ( a . key ( ) ) , <EOL> datastore_types . Key . _FromPb ( b . key ( ) ) ) <EOL> def NormalizeCursors ( query , first_sort_direction ) : <EOL> """<STR_LIT>""" <EOL> _NormalizeCursor ( query . compiled_cursor ( ) , first_sort_direction ) <EOL> _NormalizeCursor ( query . end_compiled_cursor ( ) , first_sort_direction ) <EOL> def _GuessOrders ( filters , orders ) : <EOL> """<STR_LIT>""" <EOL> orders = orders [ : ] <EOL> if not orders : <EOL> for filter_pb in filters : <EOL> if filter_pb . op ( ) in datastore_index . INEQUALITY_OPERATORS : <EOL> order = datastore_pb . Query_Order ( ) <EOL> order . set_property ( filter_pb . property ( <NUM_LIT:0> ) . name ( ) ) <EOL> orders . append ( order ) <EOL> break <EOL> exists_props = ( filter_pb . property ( <NUM_LIT:0> ) . name ( ) for filter_pb in filters <EOL> if filter_pb . op ( ) == datastore_pb . Query_Filter . EXISTS ) <EOL> for prop in sorted ( exists_props ) : <EOL> order = datastore_pb . Query_Order ( ) <EOL> order . set_property ( prop ) <EOL> orders . append ( order ) <EOL> if not orders or orders [ - <NUM_LIT:1> ] . property ( ) != '<STR_LIT>' : <EOL> order = datastore_pb . Query_Order ( ) <EOL> order . set_property ( '<STR_LIT>' ) <EOL> orders . append ( order ) <EOL> return orders <EOL> def _MakeQuery ( query_pb , filters , orders ) : <EOL> """<STR_LIT>""" <EOL> clone_pb = datastore_pb . Query ( ) <EOL> clone_pb . CopyFrom ( query_pb ) <EOL> clone_pb . clear_filter ( ) <EOL> clone_pb . clear_order ( ) <EOL> clone_pb . filter_list ( ) . extend ( filters ) <EOL> clone_pb . order_list ( ) . extend ( orders ) <EOL> return datastore_query . Query . _from_pb ( clone_pb ) <EOL> def _CreateIndexEntities ( entity , postfix_props ) : <EOL> """<STR_LIT>""" <EOL> to_split = { } <EOL> split_required = False <EOL> base_props = [ ] <EOL> for prop in entity . property_list ( ) : <EOL> if prop . name ( ) in postfix_props : <EOL> values = to_split . get ( prop . name ( ) ) <EOL> if values is None : <EOL> values = [ ] <EOL> to_split [ prop . name ( ) ] = values <EOL> else : <EOL> split_required = True <EOL> if prop . value ( ) not in values : <EOL> values . append ( prop . value ( ) ) <EOL> else : <EOL> base_props . append ( prop ) <EOL> if not split_required : <EOL> return [ entity ] <EOL> clone = entity_pb . EntityProto ( ) <EOL> clone . CopyFrom ( entity ) <EOL> clone . clear_property ( ) <EOL> clone . property_list ( ) . extend ( base_props ) <EOL> results = [ clone ] <EOL> for name , splits in to_split . iteritems ( ) : <EOL> if len ( splits ) == <NUM_LIT:1> : <EOL> for result in results : <EOL> prop = result . add_property ( ) <EOL> prop . set_name ( name ) <EOL> prop . set_multiple ( False ) <EOL> prop . set_meaning ( entity_pb . Property . INDEX_VALUE ) <EOL> prop . mutable_value ( ) . CopyFrom ( splits [ <NUM_LIT:0> ] ) <EOL> continue <EOL> new_results = [ ] <EOL> for result in results : <EOL> for split in splits : <EOL> clone = entity_pb . EntityProto ( ) <EOL> clone . CopyFrom ( result ) <EOL> prop = clone . add_property ( ) <EOL> prop . set_name ( name ) <EOL> prop . set_multiple ( False ) <EOL> prop . set_meaning ( entity_pb . Property . INDEX_VALUE ) <EOL> prop . mutable_value ( ) . CopyFrom ( split ) <EOL> new_results . append ( clone ) <EOL> results = new_results <EOL> return results <EOL> def _CreateIndexOnlyQueryResults ( records , postfix_props ) : <EOL> """<STR_LIT>""" <EOL> new_records = [ ] <EOL> for record in records : <EOL> index_entities = _CreateIndexEntities ( record . entity , postfix_props ) <EOL> new_records . extend ( [ EntityRecord ( e , record . metadata ) <EOL> for e in index_entities ] ) <EOL> return new_records <EOL> def _ExecuteQuery ( results , query , filters , orders , index_list ) : <EOL> """<STR_LIT>""" <EOL> orders = _GuessOrders ( filters , orders ) <EOL> NormalizeCursors ( query , orders [ <NUM_LIT:0> ] . direction ( ) ) <EOL> dsquery = _MakeQuery ( query , filters , orders ) <EOL> if query . property_name_size ( ) : <EOL> order_properties = set ( order . property ( ) for order in orders ) <EOL> results = _CreateIndexOnlyQueryResults ( results , order_properties ) <EOL> filtered_results = datastore_query . apply_query ( dsquery , results , <EOL> lambda r : r . entity ) <EOL> return ListCursor ( query , dsquery , orders , index_list , filtered_results ) <EOL> def _UpdateCost ( cost , entity_writes , index_writes ) : <EOL> """<STR_LIT>""" <EOL> cost . set_entity_writes ( cost . entity_writes ( ) + entity_writes ) <EOL> cost . set_index_writes ( cost . index_writes ( ) + index_writes ) <EOL> def _PropertyListToMultimap ( property_list ) : <EOL> """<STR_LIT>""" <EOL> result = collections . defaultdict ( list ) <EOL> for prop in property_list : <EOL> result [ prop . name ( ) ] . append ( prop ) <EOL> return result <EOL> def _IsNoOpWrite ( old_entity , new_entity ) : <EOL> """<STR_LIT>""" <EOL> if old_entity is not None and new_entity is not None : <EOL> old_raw_properties = _PropertyListToMultimap ( old_entity . raw_property_list ( ) ) <EOL> new_raw_properties = _PropertyListToMultimap ( new_entity . raw_property_list ( ) ) <EOL> old_properties = _PropertyListToMultimap ( old_entity . property_list ( ) ) <EOL> new_properties = _PropertyListToMultimap ( new_entity . property_list ( ) ) <EOL> return ( old_properties == new_properties <EOL> and old_raw_properties == new_raw_properties ) <EOL> else : <EOL> return old_entity == new_entity <EOL> def _CalculateWriteOps ( composite_indexes , old_entity , new_entity ) : <EOL> """<STR_LIT>""" <EOL> if _IsNoOpWrite ( old_entity , new_entity ) : <EOL> return <NUM_LIT:0> , <NUM_LIT:0> <EOL> index_writes = _ChangedIndexRows ( composite_indexes , old_entity , new_entity ) <EOL> if old_entity is None : <EOL> index_writes += <NUM_LIT:1> <EOL> return <NUM_LIT:1> , index_writes <EOL> def _ChangedIndexRows ( composite_indexes , old_entity , new_entity ) : <EOL> """<STR_LIT>""" <EOL> unique_old_properties = collections . defaultdict ( set ) <EOL> unique_new_properties = collections . defaultdict ( set ) <EOL> if old_entity is not None : <EOL> for old_prop in old_entity . property_list ( ) : <EOL> _PopulateUniquePropertiesSet ( old_prop , unique_old_properties ) <EOL> unchanged = collections . defaultdict ( int ) <EOL> for new_prop in new_entity . property_list ( ) : <EOL> new_prop_as_str = _PopulateUniquePropertiesSet ( <EOL> new_prop , unique_new_properties ) <EOL> if new_prop_as_str in unique_old_properties [ new_prop . name ( ) ] : <EOL> unchanged [ new_prop . name ( ) ] += <NUM_LIT:1> <EOL> all_property_names = set ( unique_old_properties . iterkeys ( ) ) <EOL> all_property_names . update ( unique_old_properties . iterkeys ( ) ) <EOL> all_property_names . update ( unchanged . iterkeys ( ) ) <EOL> all_indexes = _GetEntityByPropertyIndexes ( all_property_names ) <EOL> all_indexes . extend ( [ comp . definition ( ) for comp in composite_indexes ] ) <EOL> path_size = new_entity . key ( ) . path ( ) . element_size ( ) <EOL> writes = <NUM_LIT:0> <EOL> for index in all_indexes : <EOL> ancestor_multiplier = <NUM_LIT:1> <EOL> if index . ancestor ( ) and index . property_size ( ) > <NUM_LIT:1> : <EOL> ancestor_multiplier = path_size <EOL> writes += ( _CalculateWritesForCompositeIndex ( <EOL> index , unique_old_properties , unique_new_properties , unchanged ) * <EOL> ancestor_multiplier ) <EOL> return writes <EOL> def _PopulateUniquePropertiesSet ( prop , unique_properties ) : <EOL> """<STR_LIT>""" <EOL> if prop . multiple ( ) : <EOL> prop = _CopyAndSetMultipleToFalse ( prop ) <EOL> prop_as_str = prop . SerializePartialToString ( ) <EOL> unique_properties [ prop . name ( ) ] . add ( prop_as_str ) <EOL> return prop_as_str <EOL> def _CalculateWritesForCompositeIndex ( index , unique_old_properties , <EOL> unique_new_properties , <EOL> common_properties ) : <EOL> """<STR_LIT>""" <EOL> old_count = <NUM_LIT:1> <EOL> new_count = <NUM_LIT:1> <EOL> common_count = <NUM_LIT:1> <EOL> for prop in index . property_list ( ) : <EOL> old_count *= len ( unique_old_properties [ prop . name ( ) ] ) <EOL> new_count *= len ( unique_new_properties [ prop . name ( ) ] ) <EOL> common_count *= common_properties [ prop . name ( ) ] <EOL> return ( old_count - common_count ) + ( new_count - common_count ) <EOL> def _GetEntityByPropertyIndexes ( all_property_names ) : <EOL> indexes = [ ] <EOL> for prop_name in all_property_names : <EOL> indexes . append ( <EOL> _SinglePropertyIndex ( prop_name , entity_pb . Index_Property . ASCENDING ) ) <EOL> indexes . append ( <EOL> _SinglePropertyIndex ( prop_name , entity_pb . Index_Property . DESCENDING ) ) <EOL> return indexes <EOL> def _SinglePropertyIndex ( prop_name , direction ) : <EOL> """<STR_LIT>""" <EOL> index = entity_pb . Index ( ) <EOL> prop = index . add_property ( ) <EOL> prop . set_name ( prop_name ) <EOL> prop . set_direction ( direction ) <EOL> return index <EOL> def _CopyAndSetMultipleToFalse ( prop ) : <EOL> """<STR_LIT>""" <EOL> prop_copy = entity_pb . Property ( ) <EOL> prop_copy . MergeFrom ( prop ) <EOL> prop_copy . set_multiple ( False ) <EOL> return prop_copy <EOL> def _NormalizeCursor ( cursor , first_sort_direction ) : <EOL> """<STR_LIT>""" <EOL> Check ( ( cursor . has_position ( ) <EOL> + cursor . has_postfix_position ( ) <EOL> + cursor . has_absolute_position ( ) ) <= <NUM_LIT:1> , <EOL> ( '<STR_LIT>' <EOL> '<STR_LIT>' ) ) <EOL> if cursor . has_position ( ) : <EOL> pos = cursor . position ( ) <EOL> if pos . has_start_key ( ) : <EOL> index_pos = cursor . mutable_absolute_position ( ) <EOL> index_pos . set_key ( pos . start_key ( ) ) <EOL> if pos . has_start_inclusive ( ) : <EOL> index_pos . set_before ( pos . start_inclusive ( ) ) <EOL> if pos . has_before_ascending ( ) : <EOL> index_pos . set_before_ascending ( pos . before_ascending ( ) ) <EOL> elif pos . has_key ( ) or len ( pos . indexvalue_list ( ) ) > <NUM_LIT:0> : <EOL> postfix_pos = cursor . mutable_postfix_position ( ) <EOL> for value in pos . indexvalue_list ( ) : <EOL> index_value = postfix_pos . add_index_value ( ) <EOL> index_value . set_property_name ( value . property ( ) ) <EOL> index_value . mutable_value ( ) . MergeFrom ( value . value ( ) ) <EOL> if pos . has_key ( ) : <EOL> postfix_pos . mutable_key ( ) . MergeFrom ( pos . key ( ) ) <EOL> if pos . has_start_inclusive ( ) : <EOL> postfix_pos . set_before ( pos . start_inclusive ( ) ) <EOL> if pos . has_before_ascending ( ) : <EOL> postfix_pos . set_before_ascending ( pos . before_ascending ( ) ) <EOL> cursor . clear_position ( ) <EOL> pos = None <EOL> if cursor . has_absolute_position ( ) : <EOL> pos = cursor . absolute_position ( ) <EOL> elif cursor . has_postfix_position ( ) : <EOL> pos = cursor . postfix_position ( ) <EOL> if pos : <EOL> if pos . has_before_ascending ( ) : <EOL> _SetBefore ( pos , first_sort_direction ) <EOL> else : <EOL> _SetBeforeAscending ( pos , first_sort_direction ) <EOL> def _SetBefore ( position , first_direction ) : <EOL> """<STR_LIT>""" <EOL> position . set_before ( <EOL> position . before_ascending ( ) <EOL> != ( first_direction == datastore_pb . Query_Order . DESCENDING ) ) <EOL> def _SetBeforeAscending ( position , first_direction ) : <EOL> """<STR_LIT>""" <EOL> position . set_before_ascending ( <EOL> position . before ( ) <EOL> != ( first_direction == datastore_pb . Query_Order . DESCENDING ) ) </s>
