sportsref/utils.py
def get_html(url):
    """Gets the HTML for the given URL using a GET request.

    :url: the absolute URL of the desired page.
    :returns: a string of HTML.
    """
    global last_request_time
    with throttle_process_lock:
        with throttle_thread_lock:
            # sleep until THROTTLE_DELAY secs have passed since last request
            wait_left = THROTTLE_DELAY - (time.time() - last_request_time.value)
            if wait_left > 0:
                time.sleep(wait_left)

            # make request
            response = requests.get(url)

            # update last request time for throttling
            last_request_time.value = time.time()

    # raise ValueError on 4xx status code, get rid of comments, and return
    if 400 <= response.status_code < 500:
        raise ValueError(
            'Status Code {} received fetching URL "{}"'
            .format(response.status_code, url)
        )
    html = response.text
    html = html.replace('<!--', '').replace('-->', '')

    return html
def get_html(url):
    global last_request_time
    with throttle_process_lock:
        with throttle_thread_lock:
            # sleep until THROTTLE_DELAY secs have passed since last request
            wait_left = THROTTLE_DELAY - (time.time() - last_request_time.value)
            if wait_left > 0:
                time.sleep(wait_left)

            # make request
            response = requests.get(url)

            # update last request time for throttling
            last_request_time.value = time.time()

    # raise ValueError on 4xx status code, get rid of comments, and return
    if 400 <= response.status_code < 500:
        raise ValueError(
            'Status Code {} received fetching URL "{}"'
            .format(response.status_code, url)
        )
    html = response.text
    html = html.replace('<!--', '').replace('-->', '')

    return html
def get_html ( url ) : global last_request_time with throttle_process_lock : with throttle_thread_lock : # sleep until THROTTLE_DELAY secs have passed since last request wait_left = THROTTLE_DELAY - ( time . time ( ) - last_request_time . value ) if wait_left > 0 : time . sleep ( wait_left ) # make request response = requests . get ( url ) # update last request time for throttling last_request_time . value = time . time ( ) # raise ValueError on 4xx status code, get rid of comments, and return if 400 <= response . status_code < 500 : raise ValueError ( 'Status Code {} received fetching URL "{}"' . format ( response . status_code , url ) ) html = response . text html = html . replace ( '<!--' , '' ) . replace ( '-->' , '' ) return html
Gets the HTML for the given URL using a GET request.

    :url: the absolute URL of the desired page.
    :returns: a string of HTML.
Gets the HTML for the given URL using a GET request .