<s> """<STR_LIT>""" <EOL> import os <EOL> import logging <EOL> from google . appengine . api import apiproxy_stub_map <EOL> from google . appengine . api import datastore_admin <EOL> from google . appengine . api import yaml_errors <EOL> from google . appengine . datastore import datastore_index <EOL> import yaml <EOL> AUTO_MARKER = '<STR_LIT>' <EOL> AUTO_COMMENT = '''<STR_LIT>''' <EOL> def GenerateIndexFromHistory ( query_history , <EOL> all_indexes = None , manual_indexes = None ) : <EOL> """<STR_LIT>""" <EOL> all_keys = datastore_index . IndexDefinitionsToKeys ( all_indexes ) <EOL> manual_keys = datastore_index . IndexDefinitionsToKeys ( manual_indexes ) <EOL> indexes = dict ( ( key , <NUM_LIT:0> ) for key in all_keys - manual_keys ) <EOL> for query , count in query_history . iteritems ( ) : <EOL> key = datastore_index . CompositeIndexForQuery ( query ) <EOL> if key is not None : <EOL> key = key [ : <NUM_LIT:3> ] <EOL> if key not in manual_keys : <EOL> if key in indexes : <EOL> indexes [ key ] += count <EOL> else : <EOL> indexes [ key ] = count <EOL> res = [ ] <EOL> for ( kind , ancestor , props ) , count in sorted ( indexes . iteritems ( ) ) : <EOL> res . append ( '<STR_LIT>' ) <EOL> if count == <NUM_LIT:0> : <EOL> message = '<STR_LIT>' <EOL> elif count == <NUM_LIT:1> : <EOL> message = '<STR_LIT>' <EOL> else : <EOL> message = '<STR_LIT>' % count <EOL> res . append ( message ) <EOL> res . append ( '<STR_LIT>' % kind ) <EOL> if ancestor : <EOL> res . append ( '<STR_LIT>' ) <EOL> if props : <EOL> res . append ( '<STR_LIT>' ) <EOL> for name , direction in props : <EOL> res . append ( '<STR_LIT>' % name ) <EOL> if direction == datastore_index . DESCENDING : <EOL> res . append ( '<STR_LIT>' ) <EOL> res . append ( '<STR_LIT>' ) <EOL> return '<STR_LIT:\n>' . join ( res ) <EOL> class IndexYamlUpdater ( object ) : <EOL> """<STR_LIT>""" <EOL> index_yaml_is_manual = False <EOL> index_yaml_mtime = <NUM_LIT:0> <EOL> last_history_size = <NUM_LIT:0> <EOL> def __init__ ( self , root_path ) : <EOL> """<STR_LIT>""" <EOL> self . root_path = root_path <EOL> def UpdateIndexYaml ( self , openfile = open ) : <EOL> """<STR_LIT>""" <EOL> index_yaml_file = os . path . join ( self . root_path , '<STR_LIT>' ) <EOL> try : <EOL> index_yaml_mtime = os . path . getmtime ( index_yaml_file ) <EOL> except os . error : <EOL> index_yaml_mtime = None <EOL> index_yaml_changed = ( index_yaml_mtime != self . index_yaml_mtime ) <EOL> self . index_yaml_mtime = index_yaml_mtime <EOL> datastore_stub = apiproxy_stub_map . apiproxy . GetStub ( '<STR_LIT>' ) <EOL> query_history = datastore_stub . QueryHistory ( ) <EOL> history_changed = ( len ( query_history ) != self . last_history_size ) <EOL> self . last_history_size = len ( query_history ) <EOL> if not ( index_yaml_changed or history_changed ) : <EOL> logging . debug ( '<STR_LIT>' ) <EOL> return <EOL> if self . index_yaml_is_manual and not index_yaml_changed : <EOL> logging . debug ( '<STR_LIT>' ) <EOL> return <EOL> if index_yaml_mtime is None : <EOL> index_yaml_data = None <EOL> else : <EOL> try : <EOL> fh = open ( index_yaml_file , '<STR_LIT:r>' ) <EOL> except IOError : <EOL> index_yaml_data = None <EOL> else : <EOL> try : <EOL> index_yaml_data = fh . read ( ) <EOL> finally : <EOL> fh . close ( ) <EOL> self . index_yaml_is_manual = ( index_yaml_data is not None and <EOL> AUTO_MARKER not in index_yaml_data ) <EOL> if self . index_yaml_is_manual : <EOL> logging . info ( '<STR_LIT>' ) <EOL> return <EOL> if index_yaml_data is None : <EOL> all_indexes = None <EOL> else : <EOL> try : <EOL> all_indexes = datastore_index . ParseIndexDefinitions ( index_yaml_data ) <EOL> except yaml_errors . EventListenerError , e : <EOL> logging . error ( '<STR_LIT>' , index_yaml_file , e ) <EOL> return <EOL> except Exception , err : <EOL> logging . error ( '<STR_LIT>' , index_yaml_file , <EOL> err . __class__ . __module__ , err . __class__ . __name__ , err ) <EOL> return <EOL> if index_yaml_data is None : <EOL> manual_part , automatic_part = '<STR_LIT>' , '<STR_LIT>' <EOL> manual_indexes = None <EOL> else : <EOL> manual_part , automatic_part = index_yaml_data . split ( AUTO_MARKER , <NUM_LIT:1> ) <EOL> try : <EOL> manual_indexes = datastore_index . ParseIndexDefinitions ( manual_part ) <EOL> except Exception , err : <EOL> logging . error ( '<STR_LIT>' , <EOL> index_yaml_file , err ) <EOL> return <EOL> automatic_part = GenerateIndexFromHistory ( query_history , <EOL> all_indexes , manual_indexes ) <EOL> try : <EOL> fh = openfile ( index_yaml_file , '<STR_LIT:w>' ) <EOL> except IOError , err : <EOL> logging . error ( '<STR_LIT>' , err ) <EOL> return <EOL> try : <EOL> logging . info ( '<STR_LIT>' , index_yaml_file ) <EOL> fh . write ( manual_part ) <EOL> fh . write ( AUTO_MARKER ) <EOL> fh . write ( AUTO_COMMENT ) <EOL> fh . write ( automatic_part ) <EOL> finally : <EOL> fh . close ( ) <EOL> try : <EOL> self . index_yaml_mtime = os . path . getmtime ( index_yaml_file ) <EOL> except os . error , err : <EOL> logging . error ( '<STR_LIT>' , err ) <EOL> self . index_yaml_mtime = None <EOL> def SetupIndexes ( app_id , root_path ) : <EOL> """<STR_LIT>""" <EOL> index_yaml_file = os . path . join ( root_path , '<STR_LIT>' ) <EOL> try : <EOL> fh = open ( index_yaml_file , '<STR_LIT:r>' ) <EOL> except IOError : <EOL> index_yaml_data = None <EOL> else : <EOL> try : <EOL> index_yaml_data = fh . read ( ) <EOL> finally : <EOL> fh . close ( ) <EOL> indexes = [ ] <EOL> if index_yaml_data is not None : <EOL> index_defs = datastore_index . ParseIndexDefinitions ( index_yaml_data ) <EOL> if index_defs is not None : <EOL> indexes = index_defs . indexes <EOL> if indexes is None : <EOL> indexes = [ ] <EOL> requested_indexes = datastore_admin . IndexDefinitionsToProtos ( app_id , indexes ) <EOL> existing_indexes = datastore_admin . GetIndices ( app_id ) <EOL> requested = dict ( ( x . definition ( ) . Encode ( ) , x ) for x in requested_indexes ) <EOL> existing = dict ( ( x . definition ( ) . Encode ( ) , x ) for x in existing_indexes ) <EOL> created = <NUM_LIT:0> <EOL> for key , index in requested . iteritems ( ) : <EOL> if key not in existing : <EOL> datastore_admin . CreateIndex ( index ) <EOL> created += <NUM_LIT:1> <EOL> deleted = <NUM_LIT:0> <EOL> for key , index in existing . iteritems ( ) : <EOL> if key not in requested : <EOL> datastore_admin . DeleteIndex ( index ) <EOL> deleted += <NUM_LIT:1> <EOL> if created or deleted : <EOL> logging . info ( "<STR_LIT>" , <EOL> created , deleted , len ( requested ) ) </s>
