Uses stream=True and a reasonable chunk size to be able to download large (GB) files over https

    Downloading this small file takes 1.5 sec. All subsequent "downloads" takes .6 sec to verify path and size.
    >>> import time
    >>> meta = BIG_URLS['ubuntu_dialog_test']
    >>> pathend = os.path.join(*('nlpia/src/nlpia/bigdata/ubuntu_dialog_test.csv.gz'.split('/')))
    >>> download_file(url=meta[0], verbose=False).endswith(pathend)
    True
    >>> t0 = time.time()
    >>> localpath = download_file(url=BIG_URLS['ubuntu_dialog_test'][0], verbose=False)
    >>> t1 = time.time()
    >>> localpath is None or ((0.015 < (t1 - t0) < 5.0) and localpath.endswith(pathend))
    True
    >>> t0 = time.time()
    >>> download_file(url=meta[0], size=meta[1], verbose=False).endswith(pathend)
    True
    >>> time.time() - t0 < 0.02
    True

def download_file(url, data_path=BIGDATA_PATH, filename=None, size=None, chunk_size=4096, normalize_filename=False, verbose=True):
    """Uses stream=True and a reasonable chunk size to be able to download large (GB) files over https

    Downloading this small file takes 1.5 sec. All subsequent "downloads" takes .6 sec to verify path and size.
    >>> import time
    >>> meta = BIG_URLS['ubuntu_dialog_test']
    >>> pathend = os.path.join(*('nlpia/src/nlpia/bigdata/ubuntu_dialog_test.csv.gz'.split('/')))
    >>> download_file(url=meta[0], verbose=False).endswith(pathend)
    True
    >>> t0 = time.time()
    >>> localpath = download_file(url=BIG_URLS['ubuntu_dialog_test'][0], verbose=False)
    >>> t1 = time.time()
    >>> localpath is None or ((0.015 < (t1 - t0) < 5.0) and localpath.endswith(pathend))
    True
    >>> t0 = time.time()
    >>> download_file(url=meta[0], size=meta[1], verbose=False).endswith(pathend)
    True
    >>> time.time() - t0 < 0.02
    True
    """
    if isinstance(url, (list, tuple)):
        return [
            download_file(
                s, data_path=data_path, filename=filename, size=size, chunk_size=chunk_size, verbose=verbose)
            for s in url]
    if url.endswith('dl=0'):
        url = url[:-1] + '1'  # noninteractive Dropbox download
    remote_size = size

    # figure out what filename to expect after download and how big it should be
    if filename is None:
        filename = dropbox_basename(url)
    filepath = os.path.join(data_path, filename)
    if normalize_filename:
        filepath = normalize_filepath(filepath)
    logger.info('expanded+normalized file path: {}'.format(filepath))
    tqdm_prog = tqdm if verbose else no_tqdm
    logger.info('requesting URL: {}'.format(url))

    logger.info('remote_size: {}'.format(remote_size))
    stat = path_status(filepath)
    local_size = stat.get('size', None)
    logger.info('local_size: {}'.format(local_size))

    r = None
    if not remote_size or not stat['type'] == 'file' or not local_size >= remote_size or not stat['size'] > MIN_DATA_FILE_SIZE:
        try:
            r = requests.get(url, stream=True, allow_redirects=True, timeout=5)
            remote_size = r.headers.get('Content-Length', -1)
        except ConnectionError:
            logger.error('ConnectionError for url: {} => request {}'.format(url, r))
            remote_size = -1 if remote_size is None else remote_size
        except (InvalidURL, InvalidSchema, InvalidHeader, MissingSchema) as e:
            logger.warn(e)
            logger.warn('HTTP Error for url: {}\n request: {}\n traceback: {}'.format(url, r, format_exc()))
            logger.warn('This can happen for Google Word Vector download links to Dropbox or Google Docs.')
    try:
        remote_size = int(remote_size)
    except ValueError:
        remote_size = -1

    # remote_size has changed so need to check it again
    # TODO: check md5 or get the right size of remote file
    if stat['type'] == 'file' and local_size >= remote_size and stat['size'] > MIN_DATA_FILE_SIZE:
        r = r.close() if r else r
        logger.info('retained: {}'.format(filepath))
        return filepath

    filedir = os.path.dirname(filepath)
    created_dir = mkdir_p(filedir)
    logger.info('data path created: {}'.format(created_dir))
    assert os.path.isdir(filedir)
    assert created_dir.endswith(filedir)
    bytes_downloaded = 0
    if r:
        logger.info('downloading to: {}'.format(filepath))
        with open(filepath, 'wb') as f:
            for chunk in tqdm_prog(r.iter_content(chunk_size=chunk_size), total=ceil(remote_size / float(chunk_size))):
                bytes_downloaded += len(chunk)
                if chunk:  # filter out keep-alive chunks
                    f.write(chunk)
        r.close()
    else:
        logger.error('Unable to request URL: {} using request object {}'.format(url, r))
        return None

    logger.debug('nlpia.loaders.download_file: bytes={}'.format(bytes_downloaded))
    stat = path_status(filepath)
    logger.info("local file stat {}".format(stat))
    logger.debug("filepath={}: local_size={}, remote_size={}, downloaded_bytes={}".format(
        filepath, size, remote_size, bytes_downloaded))
    return filepath