<reponame>tonybutzer/enduser
#!/usr/bin/env python

import sys
import os
import argparse

sys.path.append('./enduserLib')


from enduserLib.xr_netcdf import xr_build_cube_concat_ds
from enduserLib.tony_stuff import tony_mkdir
from enduserLib.s3_func import s3_push_delete_local

from zipfile import ZipFile

def zip_me(myfilepath):
    #myfilepath = './ncdir/etasw__monthly_2003.nc'
    myzippath = myfilepath.replace('.nc', '.zip')

    with ZipFile(myzippath,'w') as zip:
            # writing each file one by one
            zip.write(myfilepath)
    os.remove(myfilepath)

def write_bummer(output_name, year):
    
    dirn = './bummers'
    tony_mkdir(dirn)
    filename = dirn + '/' + output_name + str(year) + '.txt'
    f = open(filename, "a")
    f.write(f'{output_name} and {year} - missing data?')
    f.close()
    

tony_mkdir('./ncdir')


def create_s3_list_of_days(main_prefix, year, input_name='PPT_'):
    the_list = []
    for i in range(1,366):
        month = f'{i:03d}'
        file_object = main_prefix + str(year) +os.sep+ input_name + str(year) + month + '.tif'
        the_list.append(file_object)
    return the_list


def create_s3_list_of_months(main_prefix, year, output_name='etasw_'):
    the_list = []
    for i in range(1,13):
        month = f'{i:02d}'
        file_object = main_prefix  + str(year) + '/' + output_name + str(year) + month + '.tif'
        the_list.append(file_object)
    return the_list


def mk_full_path(localfile, path):
    true_file = os.path.basename(localfile)
    full_path = path + '/' + true_file
    return full_path

def netcdf_creator(output_name, year):
    #working_bucket = 'dev-et-data'
    main_prefix = 's3://ws-enduser/DelawareRiverBasin/'
    bucket = 'ws-enduser'
    my_path = 'wzell'
    #output_name = 'etasw_'
    #output_name = 'dd_'
    #output_name = 'srf_'


    my_tifs = create_s3_list_of_months(main_prefix, year, output_name)
    ds = xr_build_cube_concat_ds(my_tifs, output_name)
    nc_file_name = './ncdir' +'/' + output_name + '_monthly_'+ str(year) + '.nc'
    ds.to_netcdf(nc_file_name, engine='h5netcdf')
    full_path = mk_full_path(nc_file_name, my_path)
    s3_push_delete_local(nc_file_name, bucket, full_path)
    # zip_me(nc_file_name)
    
    my_tifs = create_s3_list_of_days(main_prefix, year, output_name)
    # print(my_tifs)
    try:
        ds = xr_build_cube_concat_ds(my_tifs, output_name)
        nc_file_name = './ncdir' +'/' + output_name + '_daily_'+ str(year) + '.nc'
        ds.to_netcdf(nc_file_name, engine='h5netcdf')
        full_path = mk_full_path(nc_file_name, my_path)
        s3_push_delete_local(nc_file_name, bucket, full_path)
        # zip_me(nc_file_name) # zip not useful for COG
    except:
        print(f'BUMMER - on - {main_prefix}, {year}, {output_name}')
        write_bummer(output_name, year)
        dirn = './bummers'
        filename = dirn + '/' + output_name + str(year) + '.txt'
        full_path = mk_full_path(filename, f'{my_path}/bummers')
        s3_push_delete_local(filename, bucket, full_path)


def create_local_list_of_months(main_prefix, year, output_name='etasw_'):
    the_list = []
    for i in range(1,13):
        month = f'{i:02d}'
        #file_object = main_prefix  + str(year) + '/' + output_name + str(year) + month + '.tif'
        file_object = main_prefix + output_name + str(year) + month + '.tif'
        the_list.append(file_object)
    return the_list



def get_parser():
    parser = argparse.ArgumentParser(description='Run the eto code')
    parser.add_argument('products', metavar='products', type=str, nargs='*',
            help='the products (netet, etasw ...)  to process - example: etasw netet')
    parser.add_argument('-y', '--year', help='specify year or Annual or all example: -y 1999 ', default='1999', type=str)
    parser.add_argument('-i', '--in', help='input prefix_path = out/DelawareRiverBasin/Run09_13_2020/'  , default='out/wip', type=str)
    parser.add_argument('-o', '--out', help='out_prefix_path = enduser/DelawareRiverBasin/Run09_13_2020/ward_sandford_customer/' , default='enduser/wip', type=str)
    return parser


def command_line_runner():
    parser = get_parser()
    args = vars(parser.parse_args())
    

    products = args['products']
    print(products)

    if args['in']:
        print("in", args['in'])
    if args['out']:
        print("out", args['out'])

    if args['year']:
        print("year", args['year'])
        year = args['year']


    #outs = ['etasw_', 'dd_', 'srf_']
    outs = ['etasw_']


    for out in outs:
        netcdf_creator(out, year)


if __name__ == '__main__':

    try:
        command_line_runner()
    finally:
        print('create_netcdf I DIED BADLY hope the log helps', flush=True)

#!/usr/bin/env python

import sys
import os
import argparse

sys.path.append('./enduserLib')


from enduserLib.xr_netcdf import xr_build_cube_concat_ds
from enduserLib.tony_stuff import tony_mkdir
from enduserLib.s3_func import s3_push_delete_local

from zipfile import ZipFile

def zip_me(myfilepath):
    #myfilepath = './ncdir/etasw__monthly_2003.nc'
    myzippath = myfilepath.replace('.nc', '.zip')

    with ZipFile(myzippath,'w') as zip:
            # writing each file one by one
            zip.write(myfilepath)
    os.remove(myfilepath)

def write_bummer(output_name, year):
    
    dirn = './bummers'
    tony_mkdir(dirn)
    filename = dirn + '/' + output_name + str(year) + '.txt'
    f = open(filename, "a")
    f.write(f'{output_name} and {year} - missing data?')
    f.close()
    

tony_mkdir('./ncdir')


def create_s3_list_of_days(main_prefix, year, input_name='PPT_'):
    the_list = []
    for i in range(1,366):
        month = f'{i:03d}'
        file_object = main_prefix + str(year) +os.sep+ input_name + str(year) + month + '.tif'
        the_list.append(file_object)
    return the_list


def create_s3_list_of_months(main_prefix, year, output_name='etasw_'):
    the_list = []
    for i in range(1,13):
        month = f'{i:02d}'
        file_object = main_prefix  + str(year) + '/' + output_name + str(year) + month + '.tif'
        the_list.append(file_object)
    return the_list


def mk_full_path(localfile, path):
    true_file = os.path.basename(localfile)
    full_path = path + '/' + true_file
    return full_path

def netcdf_creator(output_name, year):
    #working_bucket = 'dev-et-data'
    main_prefix = 's3://ws-enduser/DelawareRiverBasin/'
    bucket = 'ws-enduser'
    my_path = 'wzell'
    #output_name = 'etasw_'
    #output_name = 'dd_'
    #output_name = 'srf_'


    my_tifs = create_s3_list_of_months(main_prefix, year, output_name)
    ds = xr_build_cube_concat_ds(my_tifs, output_name)
    nc_file_name = './ncdir' +'/' + output_name + '_monthly_'+ str(year) + '.nc'
    ds.to_netcdf(nc_file_name, engine='h5netcdf')
    full_path = mk_full_path(nc_file_name, my_path)
    s3_push_delete_local(nc_file_name, bucket, full_path)
    # zip_me(nc_file_name)
    
    my_tifs = create_s3_list_of_days(main_prefix, year, output_name)
    # print(my_tifs)
    try:
        ds = xr_build_cube_concat_ds(my_tifs, output_name)
        nc_file_name = './ncdir' +'/' + output_name + '_daily_'+ str(year) + '.nc'
        ds.to_netcdf(nc_file_name, engine='h5netcdf')
        full_path = mk_full_path(nc_file_name, my_path)
        s3_push_delete_local(nc_file_name, bucket, full_path)
        # zip_me(nc_file_name) # zip not useful for COG
    except:
        print(f'BUMMER - on - {main_prefix}, {year}, {output_name}')
        write_bummer(output_name, year)
        dirn = './bummers'
        filename = dirn + '/' + output_name + str(year) + '.txt'
        full_path = mk_full_path(filename, f'{my_path}/bummers')
        s3_push_delete_local(filename, bucket, full_path)


def create_local_list_of_months(main_prefix, year, output_name='etasw_'):
    the_list = []
    for i in range(1,13):
        month = f'{i:02d}'
        #file_object = main_prefix  + str(year) + '/' + output_name + str(year) + month + '.tif'
        file_object = main_prefix + output_name + str(year) + month + '.tif'
        the_list.append(file_object)
    return the_list



def get_parser():
    parser = argparse.ArgumentParser(description='Run the eto code')
    parser.add_argument('products', metavar='products', type=str, nargs='*',
            help='the products (netet, etasw ...)  to process - example: etasw netet')
    parser.add_argument('-y', '--year', help='specify year or Annual or all example: -y 1999 ', default='1999', type=str)
    parser.add_argument('-i', '--in', help='input prefix_path = out/DelawareRiverBasin/Run09_13_2020/'  , default='out/wip', type=str)
    parser.add_argument('-o', '--out', help='out_prefix_path = enduser/DelawareRiverBasin/Run09_13_2020/ward_sandford_customer/' , default='enduser/wip', type=str)
    return parser


def command_line_runner():
    parser = get_parser()
    args = vars(parser.parse_args())
    

    products = args['products']
    print(products)

    if args['in']:
        print("in", args['in'])
    if args['out']:
        print("out", args['out'])

    if args['year']:
        print("year", args['year'])
        year = args['year']


    #outs = ['etasw_', 'dd_', 'srf_']
    outs = ['etasw_']


    for out in outs:
        netcdf_creator(out, year)


if __name__ == '__main__':

    try:
        command_line_runner()
    finally:
        print('create_netcdf I DIED BADLY hope the log helps', flush=True)
#!/usr/bin/env python #myfilepath = './ncdir/etasw__monthly_2003.nc' # writing each file one by one #working_bucket = 'dev-et-data' #output_name = 'etasw_' #output_name = 'dd_' #output_name = 'srf_' # zip_me(nc_file_name) # print(my_tifs) # zip_me(nc_file_name) # zip not useful for COG #file_object = main_prefix  + str(year) + '/' + output_name + str(year) + month + '.tif' #outs = ['etasw_', 'dd_', 'srf_']