deeppavlov/core/data/utils.py
def download_decompress(url: str, download_path: [Path, str], extract_paths=None):
    """Download and extract .tar.gz or .gz file to one or several target locations.
    The archive is deleted if extraction was successful.

    Args:
        url: URL for file downloading
        download_path: path to the directory where downloaded file will be stored
        until the end of extraction
        extract_paths: path or list of paths where contents of archive will be extracted
    """
    file_name = Path(urlparse(url).path).name
    download_path = Path(download_path)

    if extract_paths is None:
        extract_paths = [download_path]
    elif isinstance(extract_paths, list):
        extract_paths = [Path(path) for path in extract_paths]
    else:
        extract_paths = [Path(extract_paths)]

    cache_dir = os.getenv('DP_CACHE_DIR')
    extracted = False
    if cache_dir:
        cache_dir = Path(cache_dir)
        url_hash = md5(url.encode('utf8')).hexdigest()[:15]
        arch_file_path = cache_dir / url_hash
        extracted_path = cache_dir / (url_hash + '_extracted')
        extracted = extracted_path.exists()
        if not extracted and not arch_file_path.exists():
            simple_download(url, arch_file_path)
    else:
        arch_file_path = download_path / file_name
        simple_download(url, arch_file_path)
        extracted_path = extract_paths.pop()

    if not extracted:
        log.info('Extracting {} archive into {}'.format(arch_file_path, extracted_path))
        extracted_path.mkdir(parents=True, exist_ok=True)

        if file_name.endswith('.tar.gz'):
            untar(arch_file_path, extracted_path)
        elif file_name.endswith('.gz'):
            ungzip(arch_file_path, extracted_path / Path(file_name).with_suffix('').name)
        elif file_name.endswith('.zip'):
            with zipfile.ZipFile(arch_file_path, 'r') as zip_ref:
                zip_ref.extractall(extracted_path)
        else:
            raise RuntimeError(f'Trying to extract an unknown type of archive {file_name}')

        if not cache_dir:
            arch_file_path.unlink()

    for extract_path in extract_paths:
        for src in extracted_path.iterdir():
            dest = extract_path / src.name
            if src.is_dir():
                copytree(src, dest)
            else:
                extract_path.mkdir(parents=True, exist_ok=True)
                shutil.copy(str(src), str(dest))
def download_decompress(url: str, download_path: [Path, str], extract_paths=None):
    file_name = Path(urlparse(url).path).name
    download_path = Path(download_path)

    if extract_paths is None:
        extract_paths = [download_path]
    elif isinstance(extract_paths, list):
        extract_paths = [Path(path) for path in extract_paths]
    else:
        extract_paths = [Path(extract_paths)]

    cache_dir = os.getenv('DP_CACHE_DIR')
    extracted = False
    if cache_dir:
        cache_dir = Path(cache_dir)
        url_hash = md5(url.encode('utf8')).hexdigest()[:15]
        arch_file_path = cache_dir / url_hash
        extracted_path = cache_dir / (url_hash + '_extracted')
        extracted = extracted_path.exists()
        if not extracted and not arch_file_path.exists():
            simple_download(url, arch_file_path)
    else:
        arch_file_path = download_path / file_name
        simple_download(url, arch_file_path)
        extracted_path = extract_paths.pop()

    if not extracted:
        log.info('Extracting {} archive into {}'.format(arch_file_path, extracted_path))
        extracted_path.mkdir(parents=True, exist_ok=True)

        if file_name.endswith('.tar.gz'):
            untar(arch_file_path, extracted_path)
        elif file_name.endswith('.gz'):
            ungzip(arch_file_path, extracted_path / Path(file_name).with_suffix('').name)
        elif file_name.endswith('.zip'):
            with zipfile.ZipFile(arch_file_path, 'r') as zip_ref:
                zip_ref.extractall(extracted_path)
        else:
            raise RuntimeError(f'Trying to extract an unknown type of archive {file_name}')

        if not cache_dir:
            arch_file_path.unlink()

    for extract_path in extract_paths:
        for src in extracted_path.iterdir():
            dest = extract_path / src.name
            if src.is_dir():
                copytree(src, dest)
            else:
                extract_path.mkdir(parents=True, exist_ok=True)
                shutil.copy(str(src), str(dest))
def download_decompress ( url : str , download_path : [ Path , str ] , extract_paths = None ) : file_name = Path ( urlparse ( url ) . path ) . name download_path = Path ( download_path ) if extract_paths is None : extract_paths = [ download_path ] elif isinstance ( extract_paths , list ) : extract_paths = [ Path ( path ) for path in extract_paths ] else : extract_paths = [ Path ( extract_paths ) ] cache_dir = os . getenv ( 'DP_CACHE_DIR' ) extracted = False if cache_dir : cache_dir = Path ( cache_dir ) url_hash = md5 ( url . encode ( 'utf8' ) ) . hexdigest ( ) [ : 15 ] arch_file_path = cache_dir / url_hash extracted_path = cache_dir / ( url_hash + '_extracted' ) extracted = extracted_path . exists ( ) if not extracted and not arch_file_path . exists ( ) : simple_download ( url , arch_file_path ) else : arch_file_path = download_path / file_name simple_download ( url , arch_file_path ) extracted_path = extract_paths . pop ( ) if not extracted : log . info ( 'Extracting {} archive into {}' . format ( arch_file_path , extracted_path ) ) extracted_path . mkdir ( parents = True , exist_ok = True ) if file_name . endswith ( '.tar.gz' ) : untar ( arch_file_path , extracted_path ) elif file_name . endswith ( '.gz' ) : ungzip ( arch_file_path , extracted_path / Path ( file_name ) . with_suffix ( '' ) . name ) elif file_name . endswith ( '.zip' ) : with zipfile . ZipFile ( arch_file_path , 'r' ) as zip_ref : zip_ref . extractall ( extracted_path ) else : raise RuntimeError ( f'Trying to extract an unknown type of archive {file_name}' ) if not cache_dir : arch_file_path . unlink ( ) for extract_path in extract_paths : for src in extracted_path . iterdir ( ) : dest = extract_path / src . name if src . is_dir ( ) : copytree ( src , dest ) else : extract_path . mkdir ( parents = True , exist_ok = True ) shutil . copy ( str ( src ) , str ( dest ) )
Download and extract .tar.gz or .gz file to one or several target locations.
    The archive is deleted if extraction was successful.

    Args:
        url: URL for file downloading
        download_path: path to the directory where downloaded file will be stored
        until the end of extraction
        extract_paths: path or list of paths where contents of archive will be extracted
Download and extract . tar . gz or . gz file to one or several target locations . The archive is deleted if extraction was successful .