def validate_and_update_entities(db_access, zookeeper, log_postfix, total_entities):
 last_key = ''
 entities_checked = 0
 last_logged = time.time()
 while True:
  logging.debug('Fetching {} entities'.format(BATCH_SIZE))
  entities = get_entity_batch(last_key, db_access, BATCH_SIZE)
  if (not entities):
   break
  for entity in entities:
   process_entity(entity, db_access, zookeeper)
  last_key = entities[(-1)].keys()[0]
  entities_checked += len(entities)
  if (time.time() > (last_logged + LOG_PROGRESS_FREQUENCY)):
   progress = str(entities_checked)
   if (total_entities is not None):
    progress += '/{}'.format(total_entities)
   message = 'Processed {} entities'.format(progress)
   logging.info(message)
   write_to_json_file({'status': 'inProgress', 'message': message}, log_postfix)
   last_logged = time.time()

def validate_and_update_entities ( db_access , zookeeper , log_postfix , total_entities ) : last_key = '' entities_checked = 0 last_logged = time . time ( ) while True : logging . debug ( 'Fetching {} entities' . format ( BATCH_SIZE ) ) entities = get_entity_batch ( last_key , db_access , BATCH_SIZE ) if ( not entities ) : break for entity in entities : process_entity ( entity , db_access , zookeeper ) last_key = entities [ ( - 1 ) ] . keys ( ) [ 0 ] entities_checked += len ( entities ) if ( time . time ( ) > ( last_logged + LOG_PROGRESS_FREQUENCY ) ) : progress = str ( entities_checked ) if ( total_entities is not None ) : progress += '/{}' . format ( total_entities ) message = 'Processed {} entities' . format ( progress ) logging . info ( message ) write_to_json_file ( { 'status' : 'inProgress' , 'message' : message } , log_postfix ) last_logged = time . time ( )
validates entities in batches of batch_size .