def teb_retry(exc=RequestException,
              when=dict(response__status_code=429),
              delay='response__headers__Retry-After',
              max_collisions=MAX_COLLISIONS,
              default_retry=DEFAULT_RETRY):
    """Decorator catching rate limits exceed events during a crawl task.
    It retries the task later on, following a truncated exponential backoff.
    """
    def wrap(f):
        @functools.wraps(f)
        def wrapped_f(*args, **kwargs):
            attempt = kwargs.pop('teb_retry_attempt', 0)
            try:
                return f(*args, **kwargs)
            except exc as e:
                if kwargsql.and_(e, **when):
                    try:
                        retry_after = kwargsql.get(e, delay)
                    except:
                        retry_after = default_retry
                    else:
                        if retry_after is not None:
                            retry_after = int(retry_after)
                        else:
                            retry_after = default_retry
                    countdown = retry_after + truncated_exponential_backoff(
                        retry_after, attempt % max_collisions)
                    raise Retry(kwargs=dict(attempt=attempt + 1),
                                countdown=countdown)
                else:
                    raise e, None, sys.exc_info()[2] # flake8: noqa.
        return wrapped_f
    return wrap
def teb_retry ( exc = RequestException , when = dict ( response__status_code = 429 ) , delay = 'response__headers__Retry-After' , max_collisions = MAX_COLLISIONS , default_retry = DEFAULT_RETRY ) : def wrap ( f ) : @ functools . wraps ( f ) def wrapped_f ( * args , ** kwargs ) : attempt = kwargs . pop ( 'teb_retry_attempt' , 0 ) try : return f ( * args , ** kwargs ) except exc as e : if kwargsql . and_ ( e , ** when ) : try : retry_after = kwargsql . get ( e , delay ) except : retry_after = default_retry else : if retry_after is not None : retry_after = int ( retry_after ) else : retry_after = default_retry countdown = retry_after + truncated_exponential_backoff ( retry_after , attempt % max_collisions ) raise Retry ( kwargs = dict ( attempt = attempt + 1 ) , countdown = countdown ) else : raise e , None , sys . exc_info ( ) [ 2 ] return wrapped_f return wrap
Decorator catching rate limits exceed events during a crawl task.
    It retries the task later on, following a truncated exponential backoff.