def robust_getitem(array, key, catch=Exception, max_retries=6,
                   initial_delay=500):
    assert max_retries >= 0
    for n in range(max_retries + 1):
        try:
            return array[key]
        except catch:
            if n == max_retries:
                raise
            base_delay = initial_delay * 2 ** n
            next_delay = base_delay + np.random.randint(base_delay)
            msg = ('getitem failed, waiting %s ms before trying again '
                   '(%s tries remaining). Full traceback: %s' %
                   (next_delay, max_retries - n, traceback.format_exc()))
            logger.debug(msg)
            time.sleep(1e-3 * next_delay)
Robustly index an array, using retry logic with exponential backoff if any
dataset/ETHPy150Open pydata/xarray/xarray/backends/common.py/robust_getitem