<s> '''<STR_LIT>''' <EOL> import boto3 <EOL> from random import shuffle <EOL> import sys , os , subprocess , time <EOL> GEO_DATA_DIR = os . environ . get ( "<STR_LIT>" ) <EOL> NAIP_DATA_DIR = os . path . join ( GEO_DATA_DIR , "<STR_LIT>" ) <EOL> class NAIPDownloader : <EOL> def __init__ ( self , number_of_naips , <EOL> should_randomize , <EOL> state , year , resolution , spectrum , grid , <EOL> hardcoded_naip_list = None ) : <EOL> '''<STR_LIT>''' <EOL> self . number_of_naips = number_of_naips <EOL> self . should_randomize = should_randomize <EOL> self . hardcoded_naip_list = hardcoded_naip_list <EOL> self . state = state <EOL> self . year = year <EOL> self . resolution = resolution <EOL> self . spectrum = spectrum <EOL> self . grid = grid <EOL> self . bucket_url = '<STR_LIT>' <EOL> self . url_base = '<STR_LIT>' . format ( self . bucket_url , self . state , self . year , self . resolution , self . spectrum , self . grid ) <EOL> self . make_directory ( NAIP_DATA_DIR , full_path = True ) <EOL> def make_directory ( self , new_dir , full_path = False ) : <EOL> '''<STR_LIT>''' <EOL> if full_path : <EOL> path = '<STR_LIT>' <EOL> for token in new_dir . split ( '<STR_LIT:/>' ) : <EOL> path += token + '<STR_LIT:/>' <EOL> try : <EOL> os . mkdir ( path ) ; <EOL> except : <EOL> pass <EOL> return path <EOL> try : <EOL> os . mkdir ( new_dir ) ; <EOL> except : <EOL> pass <EOL> return new_dir <EOL> def download_naips ( self ) : <EOL> '''<STR_LIT>''' <EOL> self . configure_s3cmd ( ) <EOL> naip_filenames = self . hardcoded_naip_list <EOL> if not naip_filenames : <EOL> naip_filenames = self . list_naips ( ) <EOL> if self . should_randomize : <EOL> shuffle ( naip_filenames ) <EOL> naip_local_paths = self . download_from_s3 ( naip_filenames ) <EOL> return naip_local_paths <EOL> def configure_s3cmd ( self ) : <EOL> '''<STR_LIT>''' <EOL> file_path = '<STR_LIT:/>' + os . environ . get ( "<STR_LIT>" ) + '<STR_LIT>' <EOL> f = open ( file_path , '<STR_LIT:r>' ) <EOL> filedata = f . read ( ) <EOL> f . close ( ) <EOL> newdata = filedata . replace ( "<STR_LIT>" , os . environ . get ( "<STR_LIT>" ) ) <EOL> newdata = newdata . replace ( "<STR_LIT>" , os . environ . get ( "<STR_LIT>" ) ) <EOL> f = open ( file_path , '<STR_LIT:w>' ) <EOL> f . write ( newdata ) <EOL> f . close ( ) <EOL> def list_naips ( self ) : <EOL> '''<STR_LIT>''' <EOL> bash_command = "<STR_LIT>" . format ( self . url_base ) <EOL> process = subprocess . Popen ( bash_command . split ( ) , stdout = subprocess . PIPE ) <EOL> output = process . communicate ( ) [ <NUM_LIT:0> ] <EOL> naip_filenames = [ ] <EOL> for line in output . split ( '<STR_LIT:\n>' ) : <EOL> parts = line . split ( self . url_base ) <EOL> if len ( parts ) == <NUM_LIT:2> : <EOL> naip_filenames . append ( parts [ <NUM_LIT:1> ] ) <EOL> else : <EOL> pass <EOL> return naip_filenames <EOL> def download_from_s3 ( self , naip_filenames ) : <EOL> '''<STR_LIT>''' <EOL> s3_client = boto3 . client ( '<STR_LIT>' ) <EOL> naip_local_paths = [ ] <EOL> max_range = self . number_of_naips <EOL> if max_range == - <NUM_LIT:1> : <EOL> max_range = len ( naip_filenames ) <EOL> t0 = time . time ( ) <EOL> has_printed = False <EOL> for filename in naip_filenames [ <NUM_LIT:0> : max_range ] : <EOL> full_path = os . path . join ( NAIP_DATA_DIR , filename ) <EOL> if os . path . exists ( full_path ) : <EOL> print ( "<STR_LIT>" . format ( full_path ) ) <EOL> else : <EOL> if not has_printed : <EOL> print ( "<STR_LIT>" . format ( max_range ) ) <EOL> has_printed = True <EOL> url_without_prefix = self . url_base . split ( self . bucket_url ) [ <NUM_LIT:1> ] <EOL> s3_url = '<STR_LIT>' . format ( url_without_prefix , filename ) <EOL> s3_client . download_file ( '<STR_LIT>' , s3_url , full_path , { '<STR_LIT>' : '<STR_LIT>' } ) <EOL> naip_local_paths . append ( full_path ) <EOL> if time . time ( ) - t0 > <NUM_LIT> : <EOL> print ( "<STR_LIT>" . format ( time . time ( ) - t0 ) ) <EOL> return naip_local_paths <EOL> if __name__ == '<STR_LIT:__main__>' : <EOL> parameters_message = "<STR_LIT>" <EOL> if len ( sys . argv ) == <NUM_LIT:1> : <EOL> print ( parameters_message ) <EOL> elif sys . argv [ <NUM_LIT:1> ] == '<STR_LIT>' : <EOL> naiper = NAIPDownloader ( ) <EOL> naiper . download_naips ( ) <EOL> else : <EOL> print ( parameters_message ) </s>
