r""" downloads a url to a filename.

    Args:
        url (str): url to download
        filename (str): path to download to. Defaults to basename of url
        spoof (bool): if True pretends to by Firefox
        iri_fallback : falls back to requests get call if there is a UnicodeError

    References:
        http://blog.moleculea.com/2012/10/04/urlretrieve-progres-indicator/
        http://stackoverflow.com/questions/15644964/python-progress-bar-and-downloads
        http://stackoverflow.com/questions/16694907/how-to-download-large-file-in-python-with-requests-py

    TODO:
        Delete any partially downloaded files

    Example:
        >>> # DISABLE_DOCTEST
        >>> from utool.util_grabdata import *  # NOQA
        >>> url = 'http://www.jrsoftware.org/download.php/ispack.exe'
        >>> fpath = download_url(url)
        >>> print(fpath)
        ispack.exe

def download_url(url, filename=None, spoof=False, iri_fallback=True,
                 verbose=True, new=False, chunk_size=None):
    r""" downloads a url to a filename.

    Args:
        url (str): url to download
        filename (str): path to download to. Defaults to basename of url
        spoof (bool): if True pretends to by Firefox
        iri_fallback : falls back to requests get call if there is a UnicodeError

    References:
        http://blog.moleculea.com/2012/10/04/urlretrieve-progres-indicator/
        http://stackoverflow.com/questions/15644964/python-progress-bar-and-downloads
        http://stackoverflow.com/questions/16694907/how-to-download-large-file-in-python-with-requests-py

    TODO:
        Delete any partially downloaded files

    Example:
        >>> # DISABLE_DOCTEST
        >>> from utool.util_grabdata import *  # NOQA
        >>> url = 'http://www.jrsoftware.org/download.php/ispack.exe'
        >>> fpath = download_url(url)
        >>> print(fpath)
        ispack.exe
    """
    def reporthook_(num_blocks, block_nBytes, total_nBytes, start_time=0):
        total_seconds = time.time() - start_time + 1E-9
        num_kb_down   = int(num_blocks * block_nBytes) / 1024
        num_mb_down   = num_kb_down / 1024
        percent_down  = int(num_blocks * block_nBytes * 100 / total_nBytes)
        kb_per_second = int(num_kb_down / (total_seconds))
        fmt_msg = '\r...%d%%, %d MB, %d KB/s, %d seconds passed'
        msg = fmt_msg % (percent_down, num_mb_down, kb_per_second, total_seconds)
        sys.stdout.write(msg)
        sys.stdout.flush()
    if verbose:
        reporthook = functools.partial(reporthook_, start_time=time.time())
    else:
        reporthook = None
    if filename is None:
        filename = basename(url)
    if verbose:
        print('[utool] Downloading url=%r to filename=%r' % (url, filename))
    if new:
        import requests
        #from contextlib import closing
        con = requests.get(url, stream=True, timeout=TIMEOUT)
        try:
            #import math
            content_length = con.headers.get('content-length', None)
            if content_length is None:
                # No progress available
                with open(filename, 'wb') as file_:
                    file_.write(con.content)
            else:
                if chunk_size is None:
                    chunk_size = 2 ** 20  # one megabyte at a time
                content_length = int(content_length)
                #length = int(math.ceil(content_length / chunk_size))
                with open(filename, 'wb') as file_:
                    chunk_iter = con.iter_content(chunk_size=chunk_size)
                    #chunk_iter = ut.ProgIter(chunk_iter, length=length, lbl='downloading', freq=1)
                    for count, chunk in enumerate(chunk_iter):
                        if chunk:
                            if reporthook:
                                reporthook(count, chunk_size, content_length)
                            file_.write(chunk)
        finally:
            con.close()
        return filename
    # Weird that we seem to need this here for tests
    import urllib  # NOQA
    try:
        if spoof:
            # Different agents that can be used for spoofing
            user_agents = [
                'Mozilla/5.0 (Windows; U; Windows NT 5.1; it; rv:1.8.1.11) Gecko/20071127 Firefox/2.0.0.11',  # NOQA
                'Opera/9.25 (Windows NT 5.1; U; en)',
                'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)',  # NOQA
                'Mozilla/5.0 (compatible; Konqueror/3.5; Linux) KHTML/3.5.5 (like Gecko) (Kubuntu)',
                'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.0.12) Gecko/20070731 Ubuntu/dapper-security Firefox/1.5.0.12',  # NOQA
                'Lynx/2.8.5rel.1 libwww-FM/2.14 SSL-MM/1.4.1 GNUTLS/1.2.9'
            ]
            class SpoofingOpener(urllib.FancyURLopener, object):
                version = user_agents[0]
            spoofing_opener = SpoofingOpener()
            spoofing_opener.retrieve(url, filename=filename, reporthook=reporthook)
        else:
            # no spoofing
            if six.PY2:
                urllib.urlretrieve(url, filename=filename, reporthook=reporthook)
            elif six.PY3:
                import urllib.request
                urllib.request.urlretrieve(url, filename=filename, reporthook=reporthook)
            else:
                assert False, 'unknown python'
    except UnicodeError as ex:
        import requests
        # iri error
        print('Detected iri error: %r' % (ex,))
        print('Falling back to requests.get (no progress is shown)')
        resp = requests.get(url, timeout=TIMEOUT)
        with open(filename, 'wb') as file_:
            file_.write(resp.content)
    if verbose:
        print('')
        print('[utool] Finished downloading filename=%r' % (filename,))
    return filename