<s> """<STR_LIT>""" <EOL> from __future__ import print_function <EOL> import os <EOL> import sys <EOL> import zipfile <EOL> import shutil <EOL> from rplibs . six . moves import urllib <EOL> from rplibs . six import BytesIO , binary_type <EOL> def download_file ( url , chunk_size = <NUM_LIT:100> * <NUM_LIT> ) : <EOL> """<STR_LIT>""" <EOL> print ( "<STR_LIT>" , url ) <EOL> file_content = None <EOL> progressbar = None <EOL> if sys . version_info . major <= <NUM_LIT:2> : <EOL> from rplibs . progressbar import FileTransferSpeed , ETA , ProgressBar , Percentage <EOL> from rplibs . progressbar import Bar <EOL> widgets = [ '<STR_LIT>' , FileTransferSpeed ( ) , '<STR_LIT:U+0020>' , Bar ( ) , Percentage ( ) , '<STR_LIT:U+0020>' , ETA ( ) ] <EOL> file_content = [ ] <EOL> bytes_read = <NUM_LIT:0> <EOL> try : <EOL> usock = urllib . request . urlopen ( url ) <EOL> file_size = int ( usock . headers . get ( "<STR_LIT>" , <NUM_LIT> ) ) <EOL> print ( "<STR_LIT>" , round ( file_size / ( <NUM_LIT> ** <NUM_LIT:2> ) , <NUM_LIT:2> ) , "<STR_LIT>" ) <EOL> progressbar = ProgressBar ( widgets = widgets , maxval = file_size ) . start ( ) <EOL> while True : <EOL> data = usock . read ( chunk_size ) <EOL> file_content . append ( data ) <EOL> bytes_read += len ( data ) <EOL> progressbar . update ( bytes_read ) <EOL> if not data : <EOL> break <EOL> usock . close ( ) <EOL> except Exception : <EOL> print ( "<STR_LIT>" , url , "<STR_LIT:!>" , file = sys . stderr ) <EOL> raise <EOL> else : <EOL> print ( "<STR_LIT>" ) <EOL> try : <EOL> usock = urllib . request . urlopen ( url ) <EOL> file_content = [ ] <EOL> while True : <EOL> data = usock . read ( chunk_size ) <EOL> file_content . append ( data ) <EOL> if not data : <EOL> break <EOL> usock . close ( ) <EOL> except Exception : <EOL> print ( "<STR_LIT>" , url , "<STR_LIT:!>" , file = sys . stderr ) <EOL> raise <EOL> if progressbar : <EOL> progressbar . finish ( ) <EOL> return binary_type ( ) . join ( file_content ) <EOL> def download_submodule ( author , module_name , dest_path , ignore_list ) : <EOL> """<STR_LIT>""" <EOL> if not os . path . isdir ( dest_path ) : <EOL> os . makedirs ( dest_path ) <EOL> source_url = "<STR_LIT>" + author + "<STR_LIT:/>" + module_name + "<STR_LIT>" <EOL> prefix = module_name + "<STR_LIT>" <EOL> zip_ptr = BytesIO ( download_file ( source_url ) ) <EOL> print ( "<STR_LIT>" ) <EOL> try : <EOL> zip_handle = zipfile . ZipFile ( zip_ptr ) <EOL> except zipfile . BadZipfile : <EOL> print ( "<STR_LIT>" , file = sys . stderr ) <EOL> sys . exit ( <NUM_LIT:3> ) <EOL> if zip_handle . testzip ( ) is not None : <EOL> print ( "<STR_LIT>" , file = sys . stderr ) <EOL> sys . exit ( <NUM_LIT:1> ) <EOL> num_files , num_dirs = <NUM_LIT:0> , <NUM_LIT:0> <EOL> for fname in zip_handle . namelist ( ) : <EOL> rel_name = fname . replace ( prefix , "<STR_LIT>" ) . replace ( "<STR_LIT:\\>" , "<STR_LIT:/>" ) . lstrip ( "<STR_LIT:/>" ) <EOL> if not rel_name : <EOL> continue <EOL> is_file = not rel_name . endswith ( "<STR_LIT:/>" ) <EOL> rel_name = dest_path . rstrip ( "<STR_LIT>" ) + "<STR_LIT:/>" + rel_name <EOL> if is_file : <EOL> for ignore in ignore_list : <EOL> if ignore in rel_name : <EOL> break <EOL> else : <EOL> with zip_handle . open ( fname , "<STR_LIT:r>" ) as source , open ( rel_name , "<STR_LIT:wb>" ) as dest : <EOL> shutil . copyfileobj ( source , dest ) <EOL> num_files += <NUM_LIT:1> <EOL> else : <EOL> if not os . path . isdir ( rel_name ) : <EOL> os . makedirs ( rel_name ) <EOL> num_dirs += <NUM_LIT:1> <EOL> print ( "<STR_LIT>" , num_files , "<STR_LIT>" , num_dirs , "<STR_LIT>" ) </s>
