<s> '''<STR_LIT>''' <EOL> import os <EOL> import sys <EOL> import argparse <EOL> import logging <EOL> import re <EOL> import psycopg2 <EOL> from osgeo import gdal , osr , gdalconst <EOL> from copy import copy <EOL> from datetime import datetime , time , timedelta <EOL> from scipy import ndimage <EOL> import numpy <EOL> import numpy . ma as ma <EOL> import shutil <EOL> from time import sleep <EOL> from EOtools . execute import execute <EOL> from EOtools . utils import log_multiline <EOL> from agdc import DataCube <EOL> from agdc . band_lookup import BandLookup <EOL> PQA_CONTIGUITY = <NUM_LIT> <EOL> DEFAULT_BAND_LOOKUP_SCHEME = '<STR_LIT>' <EOL> console_handler = logging . StreamHandler ( sys . stdout ) <EOL> console_handler . setLevel ( logging . INFO ) <EOL> console_formatter = logging . Formatter ( '<STR_LIT>' ) <EOL> console_handler . setFormatter ( console_formatter ) <EOL> logger = logging . getLogger ( __name__ ) <EOL> if not logger . level : <EOL> logger . setLevel ( logging . DEBUG ) <EOL> logger . addHandler ( console_handler ) <EOL> class Stacker ( DataCube ) : <EOL> def parse_args ( self ) : <EOL> """<STR_LIT>""" <EOL> logger . debug ( '<STR_LIT>' ) <EOL> _arg_parser = argparse . ArgumentParser ( '<STR_LIT>' ) <EOL> _arg_parser . add_argument ( '<STR_LIT>' , '<STR_LIT>' , dest = '<STR_LIT>' , <EOL> default = os . path . join ( self . agdc_root , '<STR_LIT>' ) , <EOL> help = '<STR_LIT>' ) <EOL> _arg_parser . add_argument ( '<STR_LIT>' , '<STR_LIT>' , dest = '<STR_LIT>' , <EOL> default = False , action = '<STR_LIT>' , const = True , <EOL> help = '<STR_LIT>' ) <EOL> _arg_parser . add_argument ( '<STR_LIT>' , '<STR_LIT>' , dest = '<STR_LIT>' , <EOL> required = False , default = None , <EOL> help = '<STR_LIT>' ) <EOL> _arg_parser . add_argument ( '<STR_LIT>' , '<STR_LIT>' , dest = '<STR_LIT>' , <EOL> required = False , default = None , <EOL> help = '<STR_LIT>' ) <EOL> _arg_parser . add_argument ( '<STR_LIT>' , '<STR_LIT>' , dest = '<STR_LIT>' , <EOL> required = False , default = <NUM_LIT:1> , <EOL> help = '<STR_LIT>' ) <EOL> _arg_parser . add_argument ( '<STR_LIT>' , '<STR_LIT>' , dest = '<STR_LIT>' , <EOL> required = False , default = None , <EOL> help = '<STR_LIT>' ) <EOL> _arg_parser . add_argument ( '<STR_LIT>' , '<STR_LIT>' , dest = '<STR_LIT>' , <EOL> required = False , default = None , <EOL> help = '<STR_LIT>' ) <EOL> _arg_parser . add_argument ( '<STR_LIT>' , '<STR_LIT>' , dest = '<STR_LIT>' , <EOL> required = False , default = None , <EOL> help = '<STR_LIT>' ) <EOL> _arg_parser . add_argument ( '<STR_LIT>' , '<STR_LIT>' , dest = '<STR_LIT>' , <EOL> required = False , default = None , <EOL> help = '<STR_LIT>' ) <EOL> _arg_parser . add_argument ( '<STR_LIT>' , '<STR_LIT>' , dest = '<STR_LIT>' , <EOL> required = False , default = <NUM_LIT:1> , <EOL> help = '<STR_LIT>' ) <EOL> _arg_parser . add_argument ( '<STR_LIT>' , '<STR_LIT>' , dest = '<STR_LIT:path>' , <EOL> required = False , default = None , <EOL> help = '<STR_LIT>' ) <EOL> _arg_parser . add_argument ( '<STR_LIT>' , '<STR_LIT>' , dest = '<STR_LIT>' , <EOL> required = False , default = None , <EOL> help = '<STR_LIT>' ) <EOL> _arg_parser . add_argument ( '<STR_LIT>' , dest = '<STR_LIT>' , <EOL> default = False , action = '<STR_LIT>' , const = True , <EOL> help = '<STR_LIT>' ) <EOL> _arg_parser . add_argument ( '<STR_LIT>' , '<STR_LIT>' , dest = '<STR_LIT>' , <EOL> required = False , default = None , <EOL> help = '<STR_LIT>' ) <EOL> _arg_parser . add_argument ( '<STR_LIT>' , '<STR_LIT>' , dest = '<STR_LIT>' , <EOL> required = False , default = None , <EOL> help = '<STR_LIT>' ) <EOL> _arg_parser . add_argument ( '<STR_LIT>' , '<STR_LIT>' , dest = '<STR_LIT>' , <EOL> required = False , default = DEFAULT_BAND_LOOKUP_SCHEME , <EOL> help = '<STR_LIT>' % DEFAULT_BAND_LOOKUP_SCHEME ) <EOL> args , unknown_args = _arg_parser . parse_known_args ( ) <EOL> return args <EOL> def __init__ ( self , source_datacube = None , default_tile_type_id = <NUM_LIT:1> ) : <EOL> """<STR_LIT>""" <EOL> if source_datacube : <EOL> self . __dict__ = copy ( source_datacube . __dict__ ) <EOL> args = self . parse_args ( ) <EOL> for attribute_name in args . __dict__ . keys ( ) : <EOL> attribute_value = args . __dict__ [ attribute_name ] <EOL> self . __setattr__ ( attribute_name , attribute_value ) <EOL> else : <EOL> DataCube . __init__ ( self ) <EOL> if self . debug : <EOL> console_handler . setLevel ( logging . DEBUG ) <EOL> try : <EOL> self . default_tile_type_id = int ( self . default_tile_type_id ) <EOL> except : <EOL> self . default_tile_type_id = default_tile_type_id <EOL> try : <EOL> self . start_date = datetime . strptime ( self . start_date , '<STR_LIT>' ) . date ( ) <EOL> except : <EOL> try : <EOL> self . start_date = datetime . strptime ( self . start_date , '<STR_LIT>' ) . date ( ) <EOL> except : <EOL> self . start_date = None <EOL> try : <EOL> self . end_date = datetime . strptime ( self . end_date , '<STR_LIT>' ) . date ( ) <EOL> except : <EOL> try : <EOL> self . end_date = datetime . strptime ( self . end_date , '<STR_LIT>' ) . date ( ) <EOL> except : <EOL> self . end_date = None <EOL> try : <EOL> self . x_index = int ( self . x_index ) <EOL> except : <EOL> self . x_index = None <EOL> try : <EOL> self . y_index = int ( self . y_index ) <EOL> except : <EOL> self . y_index = None <EOL> try : <EOL> self . path = int ( self . path ) <EOL> except : <EOL> self . path = None <EOL> try : <EOL> self . row = int ( self . row ) <EOL> except : <EOL> self . row = None <EOL> try : <EOL> self . min_path = int ( self . min_path ) <EOL> except : <EOL> self . min_path = None <EOL> try : <EOL> self . max_path = int ( self . max_path ) <EOL> except : <EOL> self . max_path = None <EOL> try : <EOL> self . min_row = int ( self . min_row ) <EOL> except : <EOL> self . min_row = None <EOL> try : <EOL> self . max_row = int ( self . max_row ) <EOL> except : <EOL> self . max_row = None <EOL> band_lookup = BandLookup ( self ) <EOL> self . band_lookup_dict = band_lookup . band_lookup_dict [ self . band_lookup_scheme ] <EOL> def stack_files ( self , timeslice_info_list , stack_dataset_path , band1_vrt_path = None , overwrite = False ) : <EOL> if os . path . exists ( stack_dataset_path ) and not overwrite : <EOL> logger . debug ( '<STR_LIT>' , stack_dataset_path ) <EOL> return <EOL> band_no = timeslice_info_list [ <NUM_LIT:0> ] [ '<STR_LIT>' ] <EOL> build_vrt = True <EOL> if band_no == <NUM_LIT:1> : <EOL> intermediate_path = stack_dataset_path <EOL> elif band1_vrt_path : <EOL> intermediate_path = band1_vrt_path <EOL> build_vrt = False <EOL> else : <EOL> intermediate_path = re . sub ( '<STR_LIT>' , '<STR_LIT>' , stack_dataset_path ) <EOL> file_list_path = re . sub ( '<STR_LIT>' , '<STR_LIT>' , stack_dataset_path ) <EOL> if build_vrt : <EOL> logger . info ( '<STR_LIT>' , len ( timeslice_info_list ) , stack_dataset_path ) <EOL> list_file = open ( file_list_path , '<STR_LIT:w>' ) <EOL> list_file . write ( '<STR_LIT:\n>' . join ( [ timeslice_info [ '<STR_LIT>' ] for timeslice_info in timeslice_info_list ] ) ) <EOL> list_file . close ( ) <EOL> del list_file <EOL> command_string = '<STR_LIT>' <EOL> if not self . debug : <EOL> command_string += '<STR_LIT>' <EOL> command_string += '<STR_LIT>' % ( <EOL> file_list_path , <EOL> intermediate_path <EOL> ) <EOL> if not self . debug : <EOL> command_string += '<STR_LIT>' % file_list_path <EOL> else : <EOL> command_string = '<STR_LIT>' <EOL> if band_no > <NUM_LIT:1> : <EOL> if command_string : <EOL> command_string += '<STR_LIT:\n>' <EOL> command_string += '<STR_LIT>' % ( intermediate_path , band_no , stack_dataset_path ) <EOL> if build_vrt : <EOL> if not self . debug : <EOL> command_string += '<STR_LIT>' % intermediate_path <EOL> else : <EOL> logger . info ( '<STR_LIT>' , len ( timeslice_info_list ) , stack_dataset_path , intermediate_path ) <EOL> logger . debug ( '<STR_LIT>' , command_string ) <EOL> result = execute ( command_string = command_string ) <EOL> if result [ '<STR_LIT>' ] : <EOL> log_multiline ( logger . info , result [ '<STR_LIT>' ] , '<STR_LIT>' + command_string , '<STR_LIT:\t>' ) <EOL> if result [ '<STR_LIT>' ] : <EOL> log_multiline ( logger . debug , result [ '<STR_LIT>' ] , '<STR_LIT>' + command_string , '<STR_LIT:\t>' ) <EOL> if result [ '<STR_LIT>' ] : <EOL> raise Exception ( '<STR_LIT>' , command_string ) <EOL> temporal_stack_dataset = gdal . Open ( stack_dataset_path ) <EOL> assert temporal_stack_dataset , '<STR_LIT>' % stack_dataset_path <EOL> for band_index in range ( len ( timeslice_info_list ) ) : <EOL> band = temporal_stack_dataset . GetRasterBand ( band_index + <NUM_LIT:1> ) <EOL> metadata_dict = dict ( timeslice_info_list [ band_index ] ) <EOL> for key in metadata_dict . keys ( ) : <EOL> metadata_dict [ key ] = str ( metadata_dict [ key ] ) <EOL> band . SetMetadata ( metadata_dict ) <EOL> log_multiline ( logger . debug , band . GetMetadata ( ) , '<STR_LIT>' , '<STR_LIT:\t>' ) <EOL> nodata_value = timeslice_info_list [ band_index ] [ '<STR_LIT>' ] <EOL> if nodata_value is not None : <EOL> logger . debug ( '<STR_LIT>' , nodata_value ) <EOL> band . SetNoDataValue ( nodata_value ) <EOL> temporal_stack_dataset . FlushCache ( ) <EOL> def stack_tile ( self , x_index , y_index , stack_output_dir = None , <EOL> start_datetime = None , end_datetime = None , <EOL> satellite = None , sensor = None , <EOL> tile_type_id = None , <EOL> path = None , <EOL> row = None , <EOL> create_band_stacks = True , <EOL> disregard_incomplete_data = False ) : <EOL> """<STR_LIT>""" <EOL> assert stack_output_dir or not create_band_stacks , '<STR_LIT>' <EOL> tile_type_id = tile_type_id or self . default_tile_type_id <EOL> db_cursor2 = self . db_connection . cursor ( ) <EOL> tile_type_ids_tuple = ( tile_type_id , ) if tile_type_id is not None else None <EOL> tile_indices_tuple = ( ( x_index , y_index ) , ) if x_index is not None and y_index is not None else None <EOL> satellites_tuple = ( satellite , ) if satellite is not None else None <EOL> sensors_tuple = ( sensor , ) if sensor is not None else None <EOL> paths_tuple = ( path , ) if path is not None else None <EOL> rows_tuple = ( row , ) if row is not None else None <EOL> params = { '<STR_LIT>' : tile_type_ids_tuple , <EOL> '<STR_LIT>' : tile_indices_tuple , <EOL> '<STR_LIT>' : satellites_tuple , <EOL> '<STR_LIT>' : sensors_tuple , <EOL> '<STR_LIT>' : paths_tuple , <EOL> '<STR_LIT>' : rows_tuple , <EOL> '<STR_LIT>' : start_datetime , <EOL> '<STR_LIT>' : end_datetime <EOL> } <EOL> log_multiline ( logger . debug , params , '<STR_LIT>' , '<STR_LIT:\t>' ) <EOL> sql = """<STR_LIT>""" <EOL> if params [ '<STR_LIT>' ] : <EOL> sql += """<STR_LIT>""" <EOL> if params [ '<STR_LIT>' ] : <EOL> sql += """<STR_LIT>""" <EOL> if params [ '<STR_LIT>' ] : <EOL> sql += """<STR_LIT>""" <EOL> if params [ '<STR_LIT>' ] : <EOL> sql += """<STR_LIT>""" <EOL> if params [ '<STR_LIT>' ] : <EOL> sql += """<STR_LIT>""" <EOL> if params [ '<STR_LIT>' ] : <EOL> sql += """<STR_LIT>""" <EOL> sql += """<STR_LIT>""" <EOL> log_multiline ( logger . debug , db_cursor2 . mogrify ( sql , params ) , '<STR_LIT>' , '<STR_LIT:\t>' ) <EOL> db_cursor2 . execute ( sql , params ) <EOL> stack_info_dict = { } <EOL> for record in db_cursor2 : <EOL> assert record , '<STR_LIT>' <EOL> tile_info = { '<STR_LIT>' : record [ <NUM_LIT:0> ] , <EOL> '<STR_LIT>' : record [ <NUM_LIT:1> ] , <EOL> '<STR_LIT>' : record [ <NUM_LIT:2> ] , <EOL> '<STR_LIT>' : record [ <NUM_LIT:3> ] , <EOL> '<STR_LIT>' : record [ <NUM_LIT:4> ] , <EOL> '<STR_LIT>' : record [ <NUM_LIT:5> ] , <EOL> '<STR_LIT>' : record [ <NUM_LIT:6> ] , <EOL> '<STR_LIT>' : record [ <NUM_LIT:7> ] , <EOL> '<STR_LIT:path>' : record [ <NUM_LIT:8> ] , <EOL> '<STR_LIT>' : record [ <NUM_LIT:9> ] , <EOL> '<STR_LIT>' : record [ <NUM_LIT:10> ] , <EOL> '<STR_LIT>' : record [ <NUM_LIT:11> ] , <EOL> '<STR_LIT>' : record [ <NUM_LIT:12> ] , <EOL> '<STR_LIT>' : record [ <NUM_LIT> ] , <EOL> '<STR_LIT>' : record [ <NUM_LIT> ] <EOL> } <EOL> assert os . path . exists ( tile_info [ '<STR_LIT>' ] ) , '<STR_LIT>' % tile_info [ '<STR_LIT>' ] <EOL> timeslice_dict = stack_info_dict . get ( tile_info [ '<STR_LIT>' ] ) or { } <EOL> if not timeslice_dict : <EOL> stack_info_dict [ tile_info [ '<STR_LIT>' ] ] = timeslice_dict <EOL> level_dict = timeslice_dict . get ( tile_info [ '<STR_LIT>' ] ) or { } <EOL> if not level_dict : <EOL> level_dict = tile_info <EOL> timeslice_dict [ tile_info [ '<STR_LIT>' ] ] = level_dict <EOL> log_multiline ( logger . debug , stack_info_dict , '<STR_LIT>' , '<STR_LIT:\t>' ) <EOL> logger . debug ( '<STR_LIT>' , len ( stack_info_dict ) ) <EOL> if disregard_incomplete_data : <EOL> stack_info_dict = { start_datetime : stack_info_dict [ start_datetime ] <EOL> for start_datetime in stack_info_dict . keys ( ) <EOL> if { '<STR_LIT>' , '<STR_LIT>' } & set ( stack_info_dict [ start_datetime ] . keys ( ) ) <EOL> and { '<STR_LIT>' , '<STR_LIT>' } <= set ( stack_info_dict [ start_datetime ] . keys ( ) ) <EOL> } <EOL> logger . debug ( '<STR_LIT>' , len ( stack_info_dict ) ) <EOL> if ( stack_output_dir ) : <EOL> self . create_directory ( stack_output_dir ) <EOL> if create_band_stacks : <EOL> band_stack_dict = { } <EOL> for start_datetime in sorted ( stack_info_dict . keys ( ) ) : <EOL> logger . debug ( '<STR_LIT>' , start_datetime ) <EOL> timeslice_dict = stack_info_dict [ start_datetime ] <EOL> log_multiline ( logger . debug , timeslice_dict , '<STR_LIT>' , '<STR_LIT:\t>' ) <EOL> tile_info = timeslice_dict . values ( ) [ <NUM_LIT:0> ] <EOL> log_multiline ( logger . debug , self . band_lookup_dict , '<STR_LIT>' , '<STR_LIT:\t>' ) <EOL> band_lookup_dict = ( self . band_lookup_dict [ tile_info [ '<STR_LIT>' ] ] <EOL> [ tile_info [ '<STR_LIT>' ] ] <EOL> [ tile_info [ '<STR_LIT>' ] ] <EOL> ) <EOL> log_multiline ( logger . debug , band_lookup_dict , '<STR_LIT>' , '<STR_LIT:\t>' ) <EOL> derived_band_dict = { key [ <NUM_LIT:1> ] : self . bands [ tile_info [ '<STR_LIT>' ] ] [ key ] for key in self . bands [ tile_info [ '<STR_LIT>' ] ] . keys ( ) if key [ <NUM_LIT:0> ] == '<STR_LIT>' } <EOL> log_multiline ( logger . debug , derived_band_dict , '<STR_LIT>' , '<STR_LIT:\t>' ) <EOL> derived_band_dict = { level_name : { value [ '<STR_LIT>' ] : value for value in derived_band_dict [ level_name ] . values ( ) } <EOL> for level_name in derived_band_dict . keys ( ) } <EOL> log_multiline ( logger . debug , derived_band_dict , '<STR_LIT>' , '<STR_LIT:\t>' ) <EOL> band_lookup_dict . update ( derived_band_dict ) <EOL> log_multiline ( logger . debug , band_lookup_dict , '<STR_LIT>' , '<STR_LIT:\t>' ) <EOL> for level_name in sorted ( timeslice_dict . keys ( ) ) : <EOL> logger . debug ( '<STR_LIT>' , level_name ) <EOL> level_band_dict = band_lookup_dict . get ( level_name ) <EOL> log_multiline ( logger . debug , level_band_dict , '<STR_LIT>' , '<STR_LIT:\t>' ) <EOL> if not level_band_dict : <EOL> continue <EOL> tile_info_dict = timeslice_dict [ level_name ] <EOL> for band_tag in level_band_dict : <EOL> band_tile_info = { start_datetime : dict ( tile_info_dict ) } <EOL> band_tile_info [ start_datetime ] . update ( level_band_dict [ band_tag ] ) <EOL> band_tile_dict = band_stack_dict . get ( ( tile_info_dict [ '<STR_LIT>' ] , <EOL> tile_info_dict [ '<STR_LIT>' ] , <EOL> tile_info_dict [ '<STR_LIT>' ] , <EOL> level_name , <EOL> band_tag ) ) <EOL> if not band_tile_dict : <EOL> band_stack_dict [ ( tile_info_dict [ '<STR_LIT>' ] , <EOL> tile_info_dict [ '<STR_LIT>' ] , <EOL> tile_info_dict [ '<STR_LIT>' ] , <EOL> level_name , <EOL> band_tag ) <EOL> ] = band_tile_info <EOL> else : <EOL> band_tile_dict . update ( band_tile_info ) <EOL> log_multiline ( logger . debug , band_stack_dict , '<STR_LIT>' , '<STR_LIT:\t>' ) <EOL> for tile_type_id , x_index , y_index , level_name , band_tag in band_stack_dict . keys ( ) : <EOL> file_stack_dict = band_stack_dict [ ( tile_type_id , x_index , y_index , level_name , band_tag ) ] <EOL> stack_filename = os . path . join ( stack_output_dir , <EOL> '<STR_LIT:_>' . join ( ( level_name , <EOL> re . sub ( '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' % ( x_index , y_index ) ) , <EOL> band_tag ) ) + '<STR_LIT>' ) <EOL> logger . debug ( '<STR_LIT>' , stack_filename ) <EOL> template_dataset = gdal . Open ( file_stack_dict . values ( ) [ <NUM_LIT:0> ] [ '<STR_LIT>' ] ) <EOL> raster_size = { '<STR_LIT:x>' : template_dataset . RasterXSize , '<STR_LIT:y>' : template_dataset . RasterYSize } <EOL> block_size = dict ( zip ( [ '<STR_LIT:x>' , '<STR_LIT:y>' ] , template_dataset . GetRasterBand ( <NUM_LIT:1> ) . GetBlockSize ( ) ) ) <EOL> gdal_driver = gdal . GetDriverByName ( "<STR_LIT>" ) <EOL> gdal_dtype = template_dataset . GetRasterBand ( <NUM_LIT:1> ) . DataType <EOL> dtype_name = gdal . GetDataTypeName ( gdal_dtype ) <EOL> vrt_dataset = gdal_driver . Create ( stack_filename , <EOL> raster_size [ '<STR_LIT:x>' ] , <EOL> raster_size [ '<STR_LIT:y>' ] , <EOL> <NUM_LIT:0> ) <EOL> vrt_dataset . SetGeoTransform ( template_dataset . GetGeoTransform ( ) ) <EOL> vrt_dataset . SetProjection ( template_dataset . GetProjection ( ) ) <EOL> del template_dataset <EOL> for start_datetime in sorted ( file_stack_dict . keys ( ) ) : <EOL> tile_info = file_stack_dict [ start_datetime ] <EOL> vrt_dataset . AddBand ( gdal_dtype ) <EOL> output_band = vrt_dataset . GetRasterBand ( vrt_dataset . RasterCount ) <EOL> complex_source = '<STR_LIT>' + '<STR_LIT>' % tile_info [ '<STR_LIT>' ] + '<STR_LIT>' % tile_info [ '<STR_LIT>' ] + '<STR_LIT>' % ( raster_size [ '<STR_LIT:x>' ] , raster_size [ '<STR_LIT:y>' ] , <EOL> dtype_name , block_size [ '<STR_LIT:x>' ] , <EOL> block_size [ '<STR_LIT:y>' ] ) + '<STR_LIT>' % ( <NUM_LIT:0> , <NUM_LIT:0> , raster_size [ '<STR_LIT:x>' ] , raster_size [ '<STR_LIT:y>' ] ) + '<STR_LIT>' % ( <NUM_LIT:0> , <NUM_LIT:0> , raster_size [ '<STR_LIT:x>' ] , raster_size [ '<STR_LIT:y>' ] ) + ( '<STR_LIT>' % tile_info [ '<STR_LIT>' ] if tile_info [ '<STR_LIT>' ] is not None else "<STR_LIT>" ) + '<STR_LIT>' <EOL> log_multiline ( logger . debug , complex_source , '<STR_LIT>' , '<STR_LIT:\t>' ) <EOL> output_band . SetMetadataItem ( "<STR_LIT>" , complex_source , "<STR_LIT>" ) <EOL> output_band . SetMetadata ( { key : str ( tile_info [ key ] ) for key in tile_info . keys ( ) } ) <EOL> if tile_info [ '<STR_LIT>' ] is not None : <EOL> output_band . SetNoDataValue ( tile_info [ '<STR_LIT>' ] ) <EOL> return stack_info_dict <EOL> def get_pqa_mask ( self , pqa_dataset_path , good_pixel_masks = [ <NUM_LIT> , <NUM_LIT> , <NUM_LIT> ] , dilation = <NUM_LIT:3> ) : <EOL> pqa_gdal_dataset = gdal . Open ( pqa_dataset_path ) <EOL> assert pqa_gdal_dataset , '<STR_LIT>' % pqa_dataset_path <EOL> pqa_array = pqa_gdal_dataset . GetRasterBand ( <NUM_LIT:1> ) . ReadAsArray ( ) <EOL> del pqa_gdal_dataset <EOL> log_multiline ( logger . debug , pqa_array , '<STR_LIT>' , '<STR_LIT:\t>' ) <EOL> pqa_array = pqa_array | <NUM_LIT:64> <EOL> s = [ [ <NUM_LIT:1> , <NUM_LIT:1> , <NUM_LIT:1> ] , [ <NUM_LIT:1> , <NUM_LIT:1> , <NUM_LIT:1> ] , [ <NUM_LIT:1> , <NUM_LIT:1> , <NUM_LIT:1> ] ] <EOL> acca = ( pqa_array & <NUM_LIT> ) >> <NUM_LIT:10> <EOL> erode = ndimage . binary_erosion ( acca , s , iterations = dilation , border_value = <NUM_LIT:1> ) <EOL> dif = erode - acca <EOL> dif [ dif < <NUM_LIT:0> ] = <NUM_LIT:1> <EOL> pqa_array += ( dif << <NUM_LIT:10> ) <EOL> del acca <EOL> fmask = ( pqa_array & <NUM_LIT> ) >> <NUM_LIT:11> <EOL> erode = ndimage . binary_erosion ( fmask , s , iterations = dilation , border_value = <NUM_LIT:1> ) <EOL> dif = erode - fmask <EOL> dif [ dif < <NUM_LIT:0> ] = <NUM_LIT:1> <EOL> pqa_array += ( dif << <NUM_LIT:11> ) <EOL> del fmask <EOL> acca_shad = ( pqa_array & <NUM_LIT> ) >> <NUM_LIT:12> <EOL> erode = ndimage . binary_erosion ( acca_shad , s , iterations = dilation , border_value = <NUM_LIT:1> ) <EOL> dif = erode - acca_shad <EOL> dif [ dif < <NUM_LIT:0> ] = <NUM_LIT:1> <EOL> pqa_array += ( dif << <NUM_LIT:12> ) <EOL> del acca_shad <EOL> fmask_shad = ( pqa_array & <NUM_LIT> ) >> <NUM_LIT> <EOL> erode = ndimage . binary_erosion ( fmask_shad , s , iterations = dilation , border_value = <NUM_LIT:1> ) <EOL> dif = erode - fmask_shad <EOL> dif [ dif < <NUM_LIT:0> ] = <NUM_LIT:1> <EOL> pqa_array += ( dif << <NUM_LIT> ) <EOL> pqa_mask = numpy . zeros ( pqa_array . shape , dtype = numpy . bool ) <EOL> for good_pixel_mask in good_pixel_masks : <EOL> pqa_mask [ pqa_array == good_pixel_mask ] = True <EOL> return pqa_mask <EOL> def apply_pqa_mask ( self , data_array , pqa_mask , no_data_value ) : <EOL> assert len ( data_array . shape ) == <NUM_LIT:2> , '<STR_LIT>' <EOL> assert data_array . shape == pqa_mask . shape , '<STR_LIT>' <EOL> data_array [ ~ pqa_mask ] = no_data_value <EOL> def get_static_info ( self , level_name = None , x_index = None , y_index = None , tile_type_id = None ) : <EOL> """<STR_LIT>""" <EOL> x_index = x_index or self . x_index <EOL> y_index = y_index or self . y_index <EOL> tile_type_id = tile_type_id or self . default_tile_type_id <EOL> db_cursor2 = self . db_connection . cursor ( ) <EOL> sql = """<STR_LIT>""" <EOL> params = { '<STR_LIT>' : level_name , <EOL> '<STR_LIT>' : x_index , <EOL> '<STR_LIT>' : y_index , <EOL> '<STR_LIT>' : tile_type_id , <EOL> } <EOL> log_multiline ( logger . debug , db_cursor2 . mogrify ( sql , params ) , '<STR_LIT>' , '<STR_LIT:\t>' ) <EOL> db_cursor2 . execute ( sql , params ) <EOL> static_info_dict = { } <EOL> last_level_name = '<STR_LIT>' <EOL> for record in db_cursor2 : <EOL> static_data = { '<STR_LIT>' : record [ <NUM_LIT:0> ] , <EOL> '<STR_LIT>' : record [ <NUM_LIT:1> ] , <EOL> '<STR_LIT>' : record [ <NUM_LIT:2> ] <EOL> } <EOL> assert static_data [ '<STR_LIT>' ] != last_level_name , '<STR_LIT>' % static_data [ '<STR_LIT>' ] <EOL> band_info = self . bands [ tile_type_id ] . get ( ( '<STR_LIT>' , static_data [ '<STR_LIT>' ] ) ) <EOL> static_info_dict [ static_data [ '<STR_LIT>' ] ] = { <EOL> '<STR_LIT>' : static_data [ '<STR_LIT>' ] , <EOL> '<STR_LIT>' : band_info . values ( ) [ <NUM_LIT:0> ] [ '<STR_LIT>' ] , <EOL> '<STR_LIT>' : static_data [ '<STR_LIT>' ] , <EOL> '<STR_LIT>' : x_index , <EOL> '<STR_LIT>' : y_index <EOL> } <EOL> return static_info_dict <EOL> def stack_derived ( self , x_index , y_index , stack_output_dir , <EOL> start_datetime = None , end_datetime = None , <EOL> satellite = None , sensor = None , <EOL> tile_type_id = None , <EOL> create_stacks = True ) : <EOL> tile_type_id = tile_type_id or self . default_tile_type_id <EOL> tile_type_info = self . tile_type_dict [ tile_type_id ] <EOL> stack_output_info = { '<STR_LIT>' : x_index , <EOL> '<STR_LIT>' : y_index , <EOL> '<STR_LIT>' : stack_output_dir , <EOL> '<STR_LIT>' : start_datetime , <EOL> '<STR_LIT>' : end_datetime , <EOL> '<STR_LIT>' : satellite , <EOL> '<STR_LIT>' : sensor } <EOL> stack_info_dict = self . stack_tile ( x_index = x_index , <EOL> y_index = y_index , <EOL> stack_output_dir = stack_output_dir , <EOL> start_datetime = start_datetime , <EOL> end_datetime = end_datetime , <EOL> satellite = satellite , <EOL> sensor = sensor , <EOL> tile_type_id = None , <EOL> create_band_stacks = False , <EOL> disregard_incomplete_data = False ) <EOL> logger . debug ( '<STR_LIT>' , <EOL> x_index , y_index , <EOL> stack_output_dir , <EOL> start_datetime , <EOL> end_datetime , <EOL> satellite , <EOL> sensor , <EOL> None , <EOL> False , <EOL> False ) <EOL> log_multiline ( logger . debug , stack_info_dict , '<STR_LIT>' , '<STR_LIT:\t>' ) <EOL> static_info_dict = self . get_static_info ( level_name = None , x_index = x_index , y_index = y_index ) <EOL> log_multiline ( logger . debug , static_info_dict , '<STR_LIT>' , '<STR_LIT:\t>' ) <EOL> start_datetimes = sorted ( stack_info_dict . keys ( ) ) <EOL> derived_stack_dict = { } <EOL> for start_datetime in start_datetimes : <EOL> input_dataset_dict = dict ( stack_info_dict [ start_datetime ] ) <EOL> input_dataset_dict . update ( static_info_dict ) <EOL> output_dataset_info = self . derive_datasets ( input_dataset_dict , stack_output_info , tile_type_info ) <EOL> if output_dataset_info is not None : <EOL> for output_stack_path in output_dataset_info : <EOL> stack_list = derived_stack_dict . get ( output_stack_path , [ ] ) <EOL> if not stack_list : <EOL> derived_stack_dict [ output_stack_path ] = stack_list <EOL> stack_list . append ( output_dataset_info [ output_stack_path ] ) <EOL> log_multiline ( logger . debug , derived_stack_dict , '<STR_LIT>' , '<STR_LIT:\t>' ) <EOL> if create_stacks : <EOL> for output_stack_path in sorted ( derived_stack_dict . keys ( ) ) : <EOL> if os . path . exists ( output_stack_path ) and not self . refresh : <EOL> logger . info ( '<STR_LIT>' , output_stack_path ) <EOL> continue <EOL> if ( self . lock_object ( output_stack_path ) ) : <EOL> logger . debug ( '<STR_LIT>' , output_stack_path ) <EOL> self . stack_files ( timeslice_info_list = derived_stack_dict [ output_stack_path ] , <EOL> stack_dataset_path = output_stack_path , <EOL> band1_vrt_path = None , overwrite = True ) <EOL> self . unlock_object ( output_stack_path ) <EOL> logger . info ( '<STR_LIT>' , output_stack_path ) <EOL> return derived_stack_dict <EOL> def derive_datasets ( self , input_dataset_dict , stack_output_info , tile_type_info ) : <EOL> """<STR_LIT>""" <EOL> assert type ( input_dataset_dict ) == dict , '<STR_LIT>' <EOL> log_multiline ( logger . debug , input_dataset_dict , '<STR_LIT>' , '<STR_LIT:\t>' ) <EOL> output_dataset_dict = { } <EOL> for input_level in [ '<STR_LIT>' , '<STR_LIT>' ] : <EOL> input_dataset_info = input_dataset_dict [ input_level ] <EOL> input_path = input_dataset_info [ '<STR_LIT>' ] <EOL> band_dict = self . bands [ tile_type_info [ '<STR_LIT>' ] ] [ ( input_dataset_info [ '<STR_LIT>' ] , input_dataset_info [ '<STR_LIT>' ] ) ] <EOL> band_info_list = [ band_dict [ tile_layer ] for tile_layer in sorted ( band_dict . keys ( ) ) if band_dict [ tile_layer ] [ '<STR_LIT>' ] == input_level ] <EOL> pqa_mask = self . get_pqa_mask ( input_dataset_dict [ '<STR_LIT>' ] [ '<STR_LIT>' ] ) <EOL> input_dataset = gdal . Open ( input_path ) <EOL> assert input_dataset , '<STR_LIT>' % input_dataset <EOL> no_data_value = input_dataset_info [ '<STR_LIT>' ] <EOL> for band_index in range ( input_dataset . RasterCount ) : <EOL> output_stack_path = os . path . join ( self . output_dir , '<STR_LIT>' % ( input_level , <EOL> band_info_list [ band_index ] [ '<STR_LIT>' ] <EOL> ) <EOL> ) <EOL> output_tile_path = os . path . join ( self . output_dir , re . sub ( '<STR_LIT>' , <EOL> '<STR_LIT>' % ( band_info_list [ band_index ] [ '<STR_LIT>' ] , <EOL> tile_type_info [ '<STR_LIT>' ] ) , <EOL> os . path . basename ( input_path ) <EOL> ) <EOL> ) <EOL> output_dataset_info = dict ( input_dataset_info ) <EOL> output_dataset_info [ '<STR_LIT>' ] = output_tile_path <EOL> output_dataset_info [ '<STR_LIT>' ] = '<STR_LIT>' % band_info_list [ band_index ] [ '<STR_LIT>' ] <EOL> output_dataset_info [ '<STR_LIT>' ] = '<STR_LIT>' % band_info_list [ band_index ] [ '<STR_LIT>' ] <EOL> output_dataset_info [ '<STR_LIT>' ] = <NUM_LIT:1> <EOL> if self . refresh or not os . path . exists ( output_tile_path ) or not gdal . Open ( output_tile_path ) : <EOL> if self . lock_object ( output_tile_path ) : <EOL> input_band = input_dataset . GetRasterBand ( band_index + <NUM_LIT:1> ) <EOL> gdal_driver = gdal . GetDriverByName ( tile_type_info [ '<STR_LIT>' ] ) <EOL> output_dataset = gdal_driver . Create ( output_tile_path , <EOL> input_dataset . RasterXSize , input_dataset . RasterYSize , <EOL> <NUM_LIT:1> , input_band . DataType , <EOL> tile_type_info [ '<STR_LIT>' ] . split ( '<STR_LIT:U+002C>' ) ) <EOL> assert output_dataset , '<STR_LIT>' % output_dataset <EOL> output_dataset . SetGeoTransform ( input_dataset . GetGeoTransform ( ) ) <EOL> output_dataset . SetProjection ( input_dataset . GetProjection ( ) ) <EOL> output_band = output_dataset . GetRasterBand ( <NUM_LIT:1> ) <EOL> data_array = input_band . ReadAsArray ( ) <EOL> self . apply_pqa_mask ( data_array , pqa_mask , no_data_value ) <EOL> output_band . WriteArray ( data_array ) <EOL> output_band . SetNoDataValue ( no_data_value ) <EOL> output_band . FlushCache ( ) <EOL> output_dataset_metadata = input_dataset . GetMetadata ( ) <EOL> output_dataset_metadata . update ( input_band . GetMetadata ( ) ) <EOL> if output_dataset_metadata : <EOL> output_dataset . SetMetadata ( output_dataset_metadata ) <EOL> log_multiline ( logger . debug , output_dataset_metadata , '<STR_LIT>' , '<STR_LIT:\t>' ) <EOL> output_dataset . FlushCache ( ) <EOL> self . unlock_object ( output_tile_path ) <EOL> logger . info ( '<STR_LIT>' , output_tile_path ) <EOL> else : <EOL> logger . info ( '<STR_LIT>' , output_tile_path ) <EOL> sleep ( <NUM_LIT:5> ) <EOL> else : <EOL> logger . info ( '<STR_LIT>' , output_tile_path ) <EOL> output_dataset_dict [ output_stack_path ] = output_dataset_info <EOL> log_multiline ( logger . debug , output_dataset_dict , '<STR_LIT>' , '<STR_LIT:\t>' ) <EOL> return output_dataset_dict <EOL> if __name__ == '<STR_LIT:__main__>' : <EOL> def date2datetime ( input_date , time_offset = time . min ) : <EOL> if not input_date : <EOL> return None <EOL> return datetime . combine ( input_date , time_offset ) <EOL> stacker = Stacker ( ) <EOL> assert stacker . x_index , '<STR_LIT>' <EOL> assert stacker . y_index , '<STR_LIT>' <EOL> assert stacker . output_dir , '<STR_LIT>' <EOL> assert os . path . isdir ( stacker . output_dir ) , '<STR_LIT>' <EOL> stacker . output_dir = os . path . abspath ( stacker . output_dir ) <EOL> log_multiline ( logger . debug , stacker . __dict__ , '<STR_LIT>' , '<STR_LIT:\t>' ) <EOL> stack_info_dict = stacker . stack_tile ( x_index = stacker . x_index , <EOL> y_index = stacker . y_index , <EOL> stack_output_dir = stacker . output_dir , <EOL> start_datetime = date2datetime ( stacker . start_date , time . min ) , <EOL> end_datetime = date2datetime ( stacker . end_date , time . max ) , <EOL> satellite = stacker . satellite , <EOL> sensor = stacker . sensor , <EOL> path = stacker . path , <EOL> row = stacker . row , <EOL> tile_type_id = None , <EOL> create_band_stacks = True , <EOL> disregard_incomplete_data = False ) <EOL> log_multiline ( logger . debug , stack_info_dict , '<STR_LIT>' , '<STR_LIT:\t>' ) <EOL> logger . info ( '<STR_LIT>' , len ( stack_info_dict ) , stacker . output_dir ) </s>
