<s> """<STR_LIT>""" <EOL> import multiprocessing <EOL> import os <EOL> import urllib2 <EOL> import constituency_areas <EOL> from constituency_areas import ALL_AREA_CODES , ALL_FILES <EOL> from log import logger <EOL> import config <EOL> base_path = os . path . abspath ( config . DIR_DATA_DOWNLOAD ) <EOL> def download_file ( remote_local_pair ) : <EOL> url = remote_local_pair [ <NUM_LIT:0> ] <EOL> local_file_path = remote_local_pair [ <NUM_LIT:1> ] <EOL> logger . info ( "<STR_LIT>" . format ( url , local_file_path ) ) <EOL> with open ( local_file_path , '<STR_LIT:wb>' ) as f : <EOL> remote = urllib2 . urlopen ( url ) <EOL> f . write ( remote . read ( ) ) <EOL> def main ( ) : <EOL> base_remote_path = "<STR_LIT>" <EOL> if not os . path . exists ( base_path ) : <EOL> logger . info ( "<STR_LIT>" . format ( base_path ) ) <EOL> os . mkdir ( base_path ) <EOL> if not os . path . isdir ( base_path ) : <EOL> raise RuntimeError ( "<STR_LIT>" . format ( base_path ) ) <EOL> files_to_download = [ ] <EOL> skipped = [ ] <EOL> for code in ALL_AREA_CODES : <EOL> filename = code + "<STR_LIT>" <EOL> url = base_remote_path + filename <EOL> local_file_path = os . path . join ( base_path , filename ) <EOL> if os . path . exists ( local_file_path ) : <EOL> skipped . append ( local_file_path ) <EOL> else : <EOL> files_to_download . append ( ( url , local_file_path ) ) <EOL> logger . info ( '<STR_LIT>' . format ( len ( files_to_download ) , len ( skipped ) ) ) <EOL> if len ( files_to_download ) > <NUM_LIT:0> : <EOL> pool = multiprocessing . Pool ( ) <EOL> pool . map ( download_file , files_to_download ) <EOL> files_in_folder = set ( os . listdir ( base_path ) ) <EOL> assert ALL_FILES . issubset ( files_in_folder ) , "<STR_LIT>" . format ( ALL_FILES . difference ( files_in_folder ) ) <EOL> logger . info ( "<STR_LIT>" . format ( len ( ALL_FILES ) ) ) <EOL> if __name__ == "<STR_LIT:__main__>" : <EOL> main ( ) </s>
