Download a url (if it corresponds to a remote file) and return a
    local url to it. If format is specified, a check will be performed
    in order to make sure that the format of the downloaded file is equal
    to the expected format.

    @param url: the URL to download
    @type url: string

    @param download_to_file: the path to download the file to
    @type download_to_file: string

    @param content_type: the content_type of the file (optional)
    @type content_type: string

    @param retry_count: max number of retries for downloading the file
    @type retry_count: int

    @param timeout: time to sleep in between attemps
    @type timeout: int

    @return: the path to the download local file
    @rtype: string
    @raise StandardError: if the download failed

def download_external_url(url, download_to_file, content_type=None,
                          retry_count=10, timeout=10.0, verbose=False):
    """
    Download a url (if it corresponds to a remote file) and return a
    local url to it. If format is specified, a check will be performed
    in order to make sure that the format of the downloaded file is equal
    to the expected format.

    @param url: the URL to download
    @type url: string

    @param download_to_file: the path to download the file to
    @type download_to_file: string

    @param content_type: the content_type of the file (optional)
    @type content_type: string

    @param retry_count: max number of retries for downloading the file
    @type retry_count: int

    @param timeout: time to sleep in between attemps
    @type timeout: int

    @return: the path to the download local file
    @rtype: string
    @raise StandardError: if the download failed
    """
    error_str = ""
    error_code = None
    retry_attempt = 0

    while retry_attempt < retry_count:
        try:
            # Attempt to download the external file
            request = open_url(url)
            if request.code == 200 and "Refresh" in request.headers:
                # PDF is being generated, they ask us to wait for
                # n seconds.
                # New arxiv responses, we are not sure if the old ones are
                # deactivated
                try:
                    retry_after = int(request.headers["Refresh"])
                    # We make sure that we do not retry too often even if
                    # they tell us to retry after 1s
                    retry_after = max(retry_after, timeout)
                except ValueError:
                    retry_after = timeout
                if verbose:
                    msg = "retrying after %ss" % (retry_after,)
                    print >> sys.stderr, msg
                time.sleep(retry_after)
                retry_attempt += 1
                continue
        except urllib2.HTTPError as e:
            error_code = e.code
            error_str = str(e)
            retry_after = timeout
            # This handling is the same as OAI queries.
            # We are getting 503 errors when PDFs are being generated
            if e.code == 503 and "Retry-After" in e.headers:
                # PDF is being generated, they ask us to wait for n seconds
                try:
                    retry_after = int(e.headers["Retry-After"])
                    # We make sure that we do not retry too often even if
                    # they tell us to retry after 1s
                    retry_after = max(retry_after, timeout)
                except ValueError:
                    pass
            if verbose:
                msg = "retrying after %ss" % (retry_after,)
                print >> sys.stderr, msg
            time.sleep(retry_after)
            retry_attempt += 1
        except (urllib2.URLError,
                socket.timeout,
                socket.gaierror,
                socket.error) as e:
            if verbose:
                error_str = str(e)
                msg = "socket error, retrying after %ss" % (timeout,)
                print >> sys.stderr, msg
            time.sleep(timeout)
            retry_attempt += 1
        else:
            # When we get here, it means that the download was a success.
            try:
                finalize_download(url, download_to_file, content_type, request)
            finally:
                request.close()
            return download_to_file

    # All the attempts were used, but no successfull download - so raise error
    msg = 'URL could not be opened: %s' % (error_str,)
    raise InvenioFileDownloadError(msg, code=error_code)