Iterate with exponential backoff on failures.

  Useful to wrap results of datastore Query.fetch to avoid 429 error.

  Args:
    base_iter: basic iterator of generator object
    max_num_tries: maximum number of tries for each request
    max_backoff: maximum backoff, in seconds
    start_backoff: initial value of backoff
    backoff_multiplier: backoff multiplier
    frac_random_backoff: fraction of the value of random part of the backoff

  Yields:
    values of yielded by base iterator

def iterate_with_exp_backoff(base_iter,
                             max_num_tries=6,
                             max_backoff=300.0,
                             start_backoff=4.0,
                             backoff_multiplier=2.0,
                             frac_random_backoff=0.25):
  """Iterate with exponential backoff on failures.

  Useful to wrap results of datastore Query.fetch to avoid 429 error.

  Args:
    base_iter: basic iterator of generator object
    max_num_tries: maximum number of tries for each request
    max_backoff: maximum backoff, in seconds
    start_backoff: initial value of backoff
    backoff_multiplier: backoff multiplier
    frac_random_backoff: fraction of the value of random part of the backoff

  Yields:
    values of yielded by base iterator
  """
  try_number = 0
  if hasattr(base_iter, '__iter__'):
    base_iter = iter(base_iter)
  while True:
    try:
      yield next(base_iter)
      try_number = 0
    except StopIteration:
      break
    except TooManyRequests as e:
      logging.warning('TooManyRequests error: %s', tb.format_exc())
      if try_number >= max_num_tries:
        logging.error('Number of tries exceeded, too many requests: %s', e)
        raise
      # compute sleep time for truncated exponential backoff
      sleep_time = start_backoff * math.pow(backoff_multiplier, try_number)
      sleep_time *= (1.0 + frac_random_backoff * random.random())
      sleep_time = min(sleep_time, max_backoff)
      logging.warning('Too many requests error, '
                      'retrying with exponential backoff %.3f', sleep_time)
      time.sleep(sleep_time)
      try_number += 1