sregistry/main/workers/aws.py
def stream(url, headers, stream_to=None, retry=True):
    '''stream is a get that will stream to file_name. Since this is a worker
       task, it differs from the client provided version in that it requires
       headers.
    '''
    bot.debug("GET %s" % url)

    if DISABLE_SSL_CHECK is True:
        bot.warning('Verify of certificates disabled! ::TESTING USE ONLY::')

    # Ensure headers are present, update if not
    response = requests.get(url,  
                            headers=headers,
                            verify=not DISABLE_SSL_CHECK,
                            stream=True)

    # If we get permissions error, one more try with updated token
    if response.status_code in [401, 403]:
        headers = update_token(headers)
        return stream(url, headers, stream_to, retry=False)

    # Successful Response
    elif response.status_code == 200:

        # Keep user updated with Progress Bar
        content_size = None
        if 'Content-Length' in response.headers:
            progress = 0
            content_size = int(response.headers['Content-Length'])
            bot.show_progress(progress,content_size,length=35)

        chunk_size = 1 << 20
        with open(stream_to,'wb') as filey:
            for chunk in response.iter_content(chunk_size=chunk_size):
                filey.write(chunk)
                if content_size is not None:
                    progress+=chunk_size
                    bot.show_progress(iteration=progress,
                                      total=content_size,
                                      length=35,
                                      carriage_return=False)

        # Newline to finish download
        sys.stdout.write('\n')

        return stream_to 

    bot.error("Problem with stream, response %s" %(response.status_code))
    sys.exit(1)
def stream(url, headers, stream_to=None, retry=True):
    '''stream is a get that will stream to file_name. Since this is a worker
       task, it differs from the client provided version in that it requires
       headers.
    '''
    bot.debug("GET %s" % url)

    if DISABLE_SSL_CHECK is True:
        bot.warning('Verify of certificates disabled! ::TESTING USE ONLY::')

    # Ensure headers are present, update if not
    response = requests.get(url,  
                            headers=headers,
                            verify=not DISABLE_SSL_CHECK,
                            stream=True)

    # If we get permissions error, one more try with updated token
    if response.status_code in [401, 403]:
        headers = update_token(headers)
        return stream(url, headers, stream_to, retry=False)

    # Successful Response
    elif response.status_code == 200:

        # Keep user updated with Progress Bar
        content_size = None
        if 'Content-Length' in response.headers:
            progress = 0
            content_size = int(response.headers['Content-Length'])
            bot.show_progress(progress,content_size,length=35)

        chunk_size = 1 << 20
        with open(stream_to,'wb') as filey:
            for chunk in response.iter_content(chunk_size=chunk_size):
                filey.write(chunk)
                if content_size is not None:
                    progress+=chunk_size
                    bot.show_progress(iteration=progress,
                                      total=content_size,
                                      length=35,
                                      carriage_return=False)

        # Newline to finish download
        sys.stdout.write('\n')

        return stream_to 

    bot.error("Problem with stream, response %s" %(response.status_code))
    sys.exit(1)
def stream ( url , headers , stream_to = None , retry = True ) : bot . debug ( "GET %s" % url ) if DISABLE_SSL_CHECK is True : bot . warning ( 'Verify of certificates disabled! ::TESTING USE ONLY::' ) # Ensure headers are present, update if not response = requests . get ( url , headers = headers , verify = not DISABLE_SSL_CHECK , stream = True ) # If we get permissions error, one more try with updated token if response . status_code in [ 401 , 403 ] : headers = update_token ( headers ) return stream ( url , headers , stream_to , retry = False ) # Successful Response elif response . status_code == 200 : # Keep user updated with Progress Bar content_size = None if 'Content-Length' in response . headers : progress = 0 content_size = int ( response . headers [ 'Content-Length' ] ) bot . show_progress ( progress , content_size , length = 35 ) chunk_size = 1 << 20 with open ( stream_to , 'wb' ) as filey : for chunk in response . iter_content ( chunk_size = chunk_size ) : filey . write ( chunk ) if content_size is not None : progress += chunk_size bot . show_progress ( iteration = progress , total = content_size , length = 35 , carriage_return = False ) # Newline to finish download sys . stdout . write ( '\n' ) return stream_to bot . error ( "Problem with stream, response %s" % ( response . status_code ) ) sys . exit ( 1 )
stream is a get that will stream to file_name. Since this is a worker
       task, it differs from the client provided version in that it requires
       headers.
stream is a get that will stream to file_name . Since this is a worker task it differs from the client provided version in that it requires headers .