<s> """<STR_LIT>""" <EOL> __all__ = [ "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , <EOL> "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , <EOL> "<STR_LIT>" ] <EOL> from google . appengine . api import datastore <EOL> from google . appengine . ext import db <EOL> MAX_POOL_SIZE = <NUM_LIT> * <NUM_LIT:1000> <EOL> MAX_ENTITY_COUNT = <NUM_LIT> <EOL> DATASTORE_DEADLINE = <NUM_LIT:15> <EOL> COUNTER_MAPPER_CALLS = "<STR_LIT>" <EOL> def _normalize_entity ( value ) : <EOL> """<STR_LIT>""" <EOL> if getattr ( value , "<STR_LIT>" , None ) : <EOL> return value . _populate_internal_entity ( ) <EOL> return value <EOL> def _normalize_key ( value ) : <EOL> """<STR_LIT>""" <EOL> if getattr ( value , "<STR_LIT:key>" , None ) : <EOL> return value . key ( ) <EOL> elif isinstance ( value , basestring ) : <EOL> return datastore . Key ( value ) <EOL> else : <EOL> return value <EOL> class ItemList ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self ) : <EOL> """<STR_LIT>""" <EOL> self . items = [ ] <EOL> self . length = <NUM_LIT:0> <EOL> self . size = <NUM_LIT:0> <EOL> def append ( self , item , item_size ) : <EOL> """<STR_LIT>""" <EOL> self . items . append ( item ) <EOL> self . length += <NUM_LIT:1> <EOL> self . size += item_size <EOL> def clear ( self ) : <EOL> """<STR_LIT>""" <EOL> self . items = [ ] <EOL> self . length = <NUM_LIT:0> <EOL> self . size = <NUM_LIT:0> <EOL> @ property <EOL> def entities ( self ) : <EOL> """<STR_LIT>""" <EOL> return self . items <EOL> EntityList = ItemList <EOL> class MutationPool ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , <EOL> max_pool_size = MAX_POOL_SIZE , <EOL> max_entity_count = MAX_ENTITY_COUNT ) : <EOL> """<STR_LIT>""" <EOL> self . max_pool_size = max_pool_size <EOL> self . max_entity_count = max_entity_count <EOL> self . puts = ItemList ( ) <EOL> self . deletes = ItemList ( ) <EOL> def put ( self , entity ) : <EOL> """<STR_LIT>""" <EOL> actual_entity = _normalize_entity ( entity ) <EOL> entity_size = len ( actual_entity . _ToPb ( ) . Encode ( ) ) <EOL> if ( self . puts . length >= self . max_entity_count or <EOL> ( self . puts . size + entity_size ) > self . max_pool_size ) : <EOL> self . __flush_puts ( ) <EOL> self . puts . append ( actual_entity , entity_size ) <EOL> def delete ( self , entity ) : <EOL> """<STR_LIT>""" <EOL> key = _normalize_key ( entity ) <EOL> key_size = len ( key . _ToPb ( ) . Encode ( ) ) <EOL> if ( self . deletes . length >= self . max_entity_count or <EOL> ( self . deletes . size + key_size ) > self . max_pool_size ) : <EOL> self . __flush_deletes ( ) <EOL> self . deletes . append ( key , key_size ) <EOL> def flush ( self ) : <EOL> """<STR_LIT>""" <EOL> self . __flush_puts ( ) <EOL> self . __flush_deletes ( ) <EOL> def __flush_puts ( self ) : <EOL> """<STR_LIT>""" <EOL> if self . puts . length : <EOL> datastore . Put ( self . puts . items , rpc = self . __create_rpc ( ) ) <EOL> self . puts . clear ( ) <EOL> def __flush_deletes ( self ) : <EOL> """<STR_LIT>""" <EOL> if self . deletes . length : <EOL> datastore . Delete ( self . deletes . items , rpc = self . __create_rpc ( ) ) <EOL> self . deletes . clear ( ) <EOL> def __create_rpc ( self ) : <EOL> """<STR_LIT>""" <EOL> return datastore . CreateRPC ( deadline = DATASTORE_DEADLINE ) <EOL> class Counters ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , shard_state ) : <EOL> """<STR_LIT>""" <EOL> self . _shard_state = shard_state <EOL> def increment ( self , counter_name , delta = <NUM_LIT:1> ) : <EOL> """<STR_LIT>""" <EOL> self . _shard_state . counters_map . increment ( counter_name , delta ) <EOL> def flush ( self ) : <EOL> """<STR_LIT>""" <EOL> pass <EOL> class Context ( object ) : <EOL> """<STR_LIT>""" <EOL> _context_instance = None <EOL> def __init__ ( self , mapreduce_spec , shard_state , task_retry_count = <NUM_LIT:0> ) : <EOL> """<STR_LIT>""" <EOL> self . mapreduce_spec = mapreduce_spec <EOL> self . shard_state = shard_state <EOL> self . task_retry_count = task_retry_count <EOL> if self . mapreduce_spec : <EOL> self . mapreduce_id = self . mapreduce_spec . mapreduce_id <EOL> else : <EOL> self . mapreduce_id = None <EOL> if self . shard_state : <EOL> self . shard_id = self . shard_state . get_shard_id ( ) <EOL> else : <EOL> self . shard_id = None <EOL> self . mutation_pool = MutationPool ( <EOL> max_pool_size = ( MAX_POOL_SIZE / ( <NUM_LIT:2> ** self . task_retry_count ) ) , <EOL> max_entity_count = ( MAX_ENTITY_COUNT / ( <NUM_LIT:2> ** self . task_retry_count ) ) ) <EOL> self . counters = Counters ( shard_state ) <EOL> self . _pools = { } <EOL> self . register_pool ( "<STR_LIT>" , self . mutation_pool ) <EOL> self . register_pool ( "<STR_LIT>" , self . counters ) <EOL> def flush ( self ) : <EOL> """<STR_LIT>""" <EOL> for pool in self . _pools . values ( ) : <EOL> pool . flush ( ) <EOL> def register_pool ( self , key , pool ) : <EOL> """<STR_LIT>""" <EOL> self . _pools [ key ] = pool <EOL> def get_pool ( self , key ) : <EOL> """<STR_LIT>""" <EOL> return self . _pools . get ( key , None ) <EOL> @ classmethod <EOL> def _set ( cls , context ) : <EOL> """<STR_LIT>""" <EOL> cls . _context_instance = context <EOL> def get ( ) : <EOL> """<STR_LIT>""" <EOL> return Context . _context_instance </s>
