def run_lsm_rapid_process(rapid_executable_location,
                          lsm_data_location,
                          rapid_io_files_location=None,
                          rapid_input_location=None,
                          rapid_output_location=None,
                          simulation_start_datetime=None,
                          simulation_end_datetime=datetime.utcnow(),
                          file_datetime_pattern=None,
                          file_datetime_re_pattern=None,
                          initial_flows_file=None,
                          ensemble_list=(None,),
                          generate_rapid_namelist_file=True,
                          run_rapid_simulation=True,
                          generate_return_periods_file=False,
                          return_period_method='weibul',
                          generate_seasonal_averages_file=False,
                          generate_seasonal_initialization_file=False,
                          generate_initialization_file=False,
                          use_all_processors=True,
                          num_processors=1,
                          mpiexec_command="mpiexec",
                          cygwin_bin_location="",
                          modeling_institution="US Army Engineer Research "
                                               "and Development Center",
                          convert_one_hour_to_three=False,
                          expected_time_step=None):
    # pylint: disable=anomalous-backslash-in-string
    """
    This is the main process to generate inflow for RAPID and to run RAPID.

    Parameters
    ----------
    rapid_executable_location: str
        Path to the RAPID executable.
    lsm_data_location: str
        Path to the directory containing the Land Surface Model output files.
    rapid_io_files_location: str, optional
        Path to the directory containing the input and output folders for
        RAPID. This is for running multiple watersheds.
    rapid_input_location: str, optional
        Path to directory with RAPID simulation input data.
        Required if `rapid_io_files_location` is not set.
    rapid_output_location: str, optional
        Path to directory to put output. Required if
        `rapid_io_files_location` is not set.
    simulation_start_datetime: datetime, optional
        Datetime object with date bound of earliest simulation start.
    simulation_end_datetime: datetime, optional
        Datetime object with date bound of latest simulation end.
        Defaults to :obj:`datetime.utcnow`.
    file_datetime_pattern: str, optional
        Datetime pattern for files (Ex. '%Y%m%d%H'). If set,
        `file_datetime_re_pattern` is required.
        Various defaults used by each model.
    file_datetime_re_pattern: raw str, optional
        Regex pattern to extract datetime (Ex. r'\d{10}').
        If set, `file_datetime_pattern` is required.
        Various defaults used by each model.
    initial_flows_file: str, optional
        If given, this is the path to a file with initial flows
        for the simulaion.
    ensemble_list: list, optional
        This is the expexted ensemble name appended to the end of the
        file name.
    generate_rapid_namelist_file: bool, optional
        If True, this will create a RAPID namelist file for the run in
        your RAPID input directory. Default is True.
    run_rapid_simulation: bool, optional
        If True, the RAPID simulation will run after generating the
            inflow file. Default is True.
    generate_return_periods_file: bool, optional
        If True, the return period file will be generated in the output.
        Default is False.
    return_period_method: str, optional
        If True, the return period file will be generated in the output.
        Default is False.
    generate_seasonal_averages_file: bool, optional
        If True, the season average file will be generated. Default is False.
    generate_seasonal_initialization_file: bool, optional
        If True, an intialization based on the seasonal average for the
        current day of the year will be created. Default is False.
    generate_initialization_file: bool, optional
        If True, an initialization file from the last time step of the
        simulation willl be created. Default is False.
    use_all_processors: bool, optional
        If True, it will use all available processors to perform this
        operation. Default is True.
    num_processors: int, optional
        If use_all_processors is False, this argument will determine the
        number of processors to use. Default is 1.
    mpiexec_command: str, optional
        This is the command to execute RAPID. Default is "mpiexec".
    cygwin_bin_location: str, optional
        If using Windows, this is the path to the Cygwin bin location.
        Default is "".
    modeling_institution: str, optional
        This is the institution performing the modeling and is in the
        output files.
        Default is "US Army Engineer Research and Development Center".
    convert_one_hour_to_three: bool, optional
        If the time step is expected to be 1-hr it will convert to 3.
        Set to False if the LIS, NLDAS, or Joules grid time step is
        greater than 1-hr.
    expected_time_step: int, optional
        The time step in seconds of your LSM input data if only one file
        is given. Required if only one file is present.


    Returns
    -------
    list:
        A list of output file information.


    Example of regular run:

    .. code:: python

        from datetime import datetime
        from RAPIDpy.inflow import run_lsm_rapid_process

        run_lsm_rapid_process(
            rapid_executable_location='/home/alan/rapid/src/rapid',
            rapid_io_files_location='/home/alan/rapid-io',
            lsm_data_location='/home/alan/era_data',
        )

    Example of single input/output run:

    .. code:: python

        from datetime import datetime
        from RAPIDpy.inflow import run_lsm_rapid_process

        run_lsm_rapid_process(
            rapid_executable_location='/home/alan/rapid/src/rapid',
            rapid_input_location='/home/alan/rapid-io/input/provo_watershed',
            rapid_output_location='/home/alan/rapid-io/output/provo_watershed',
            lsm_data_location='/home/alan/era_data',
        )


    Example of run with FLDAS and datetime filter:

    .. note:: http://disc.sci.gsfc.nasa.gov/uui/datasets?keywords=FLDAS

    .. code:: python

        from datetime import datetime
        from RAPIDpy.inflow import run_lsm_rapid_process

        run_lsm_rapid_process(
            rapid_executable_location='/home/alan/rapid/src/rapid',
            rapid_io_files_location='/home/alan/rapid-io',
            lsm_data_location='/home/alan/lsm_data',
            simulation_start_datetime=datetime(1980, 1, 1),
            file_datetime_re_pattern = r'\d{8}',
            file_datetime_pattern = "%Y%m%d",
        )


    Example of run with CMIP5:

    .. note:: http://gdo-dcp.ucllnl.org/downscaled_cmip_projections/techmemo/BCSD5HydrologyMemo.pdf


    .. code:: python

        from datetime import datetime
        from RAPIDpy.inflow import run_lsm_rapid_process

        run_lsm_rapid_process(
            rapid_executable_location='/home/jimwlewis/rapid/src/rapid',
            rapid_io_files_location='/data/rapid-io4',
            lsm_data_location='/data/rapid-io4/input/cmip5-jun01',
            simulation_start_datetime=datetime(2001, 1, 1),
            simulation_end_datetime=datetime(2002, 12, 31),
            file_datetime_pattern="%Y",
            file_datetime_re_pattern=r'\d{4}',
        )
    """  # noqa
    time_begin_all = datetime.utcnow()

    # use all processors makes precedent over num_processors arg
    if use_all_processors is True:
        num_cpus = multiprocessing.cpu_count()
    elif num_processors > multiprocessing.cpu_count():
        print("WARNING: Num processors requested exceeded max. Set to max ...")
        num_cpus = multiprocessing.cpu_count()
    else:
        num_cpus = num_processors

    # get list of correctly formatted rapid input directories in
    # rapid directory
    rapid_directories = []
    if rapid_io_files_location is not None:
        main_rapid_input_directory = os.path.join(rapid_io_files_location,
                                                  'input')
        for watershed_directory in \
                get_valid_directory_list(main_rapid_input_directory):
            watershed_input_path = os.path.join(main_rapid_input_directory,
                                                watershed_directory)
            watershed_output_path = os.path.join(rapid_io_files_location,
                                                 'output',
                                                 watershed_directory)
            rapid_directories.append(
                (watershed_input_path, watershed_output_path))
    elif None not in (rapid_input_location, rapid_output_location):
        rapid_directories = [(rapid_input_location, rapid_output_location)]
    else:
        raise ValueError("Need 'rapid_io_files_location' or "
                         "'rapid_input_location' and 'rapid_output_location'"
                         " set to continue.")

    all_output_file_information = []
    for ensemble in ensemble_list:
        output_file_information = {
            'ensemble': ensemble,
        }
        ensemble_file_ending = ".nc"
        ensemble_file_ending4 = ".nc4"
        if ensemble is not None:
            ensemble_file_ending = "_{0}.nc".format(ensemble)
            ensemble_file_ending4 = "_{0}.nc4".format(ensemble)

        # get list of files
        lsm_file_list = []
        for walkdir_info in os.walk(lsm_data_location,
                                    followlinks=True):
            for lsm_file in walkdir_info[2]:
                if lsm_file.endswith(ensemble_file_ending) or \
                        lsm_file.endswith(ensemble_file_ending4):
                    lsm_file_list.append(
                        os.path.join(walkdir_info[0], lsm_file))
        lsm_file_list = sorted(lsm_file_list)

        # IDENTIFY THE GRID
        lsm_file_data = identify_lsm_grid(lsm_file_list[0])

        # load in the datetime pattern
        if file_datetime_pattern is None or file_datetime_re_pattern is None:
            file_datetime_re_pattern = \
                DEFAULT_LSM_INPUTS[lsm_file_data['grid_type']][
                    'file_datetime_re_pattern']
            file_datetime_pattern = \
                DEFAULT_LSM_INPUTS[lsm_file_data['grid_type']][
                    'file_datetime_pattern']
        file_re_match = re.compile(file_datetime_re_pattern)

        # get subset based on time bounds
        if simulation_start_datetime is not None:
            print("Filtering files by datetime ...")
            lsm_file_list_subset = []
            for lsm_file in lsm_file_list:
                match = file_re_match.search(lsm_file)
                file_date = datetime.strptime(match.group(0),
                                              file_datetime_pattern)
                if file_date > simulation_end_datetime:
                    break
                if file_date >= simulation_start_datetime:
                    lsm_file_list_subset.append(lsm_file)

            lsm_file_list = sorted(lsm_file_list_subset)

        print("Running from {0} to {1}".format(lsm_file_list[0],
                                               lsm_file_list[-1]))

        # get number of time steps in file
        actual_simulation_start_datetime, actual_simulation_end_datetime, \
            time_step, total_num_time_steps = \
            determine_start_end_timestep(
                lsm_file_list,
                file_re_match=file_re_match,
                file_datetime_pattern=file_datetime_pattern,
                expected_time_step=expected_time_step,
                lsm_grid_info=lsm_file_data)

        # VALIDATING INPUT IF DIVIDING BY 3
        if (lsm_file_data['grid_type'] in ('nldas', 'lis', 'joules')) \
                and convert_one_hour_to_three:
            num_extra_files = total_num_time_steps % 3
            if num_extra_files != 0:
                print("WARNING: Number of files needs to be divisible by 3. "
                      "Remainder is {0}".format(num_extra_files))
                print("This means your simulation will be truncated")
            total_num_time_steps /= 3
            time_step *= 3

        # compile the file ending
        out_file_ending = "{0}_{1}_{2}hr_{3:%Y%m%d}to{4:%Y%m%d}{5}"\
            .format(lsm_file_data['model_name'],
                    lsm_file_data['grid_type'],
                    int(time_step/3600),
                    actual_simulation_start_datetime,
                    actual_simulation_end_datetime,
                    ensemble_file_ending)

        # run LSM processes
        for master_watershed_input_directory, \
                master_watershed_output_directory in rapid_directories:
            print("Running from: {0}".format(master_watershed_input_directory))
            try:
                os.makedirs(master_watershed_output_directory)
            except OSError:
                pass

            # create inflow to dump data into
            master_rapid_runoff_file = \
                os.path.join(master_watershed_output_directory,
                             'm3_riv_bas_{0}'.format(out_file_ending))

            weight_table_file = \
                case_insensitive_file_search(master_watershed_input_directory,
                                             lsm_file_data['weight_file_name'])

            try:
                in_rivid_lat_lon_z_file = \
                    case_insensitive_file_search(
                        master_watershed_input_directory,
                        r'comid_lat_lon_z\.csv')
            except IndexError:
                in_rivid_lat_lon_z_file = ""
                print("WARNING: comid_lat_lon_z file not found."
                      " The lat/lon will not be added ...")

            print("Writing inflow file to: {0}"
                  .format(master_rapid_runoff_file))
            lsm_file_data['rapid_inflow_tool'].generateOutputInflowFile(
                out_nc=master_rapid_runoff_file,
                start_datetime_utc=actual_simulation_start_datetime,
                number_of_timesteps=total_num_time_steps,
                simulation_time_step_seconds=time_step,
                in_rapid_connect_file=case_insensitive_file_search(
                    master_watershed_input_directory,
                    r'rapid_connect\.csv'),
                in_rivid_lat_lon_z_file=in_rivid_lat_lon_z_file,
                land_surface_model_description=lsm_file_data['description'],
                modeling_institution=modeling_institution
            )

            job_combinations = []
            if (lsm_file_data['grid_type'] in ('nldas', 'lis', 'joules')) \
                    and convert_one_hour_to_three:
                print("Grouping {0} in threes"
                      .format(lsm_file_data['grid_type']))
                lsm_file_list = [lsm_file_list[nldas_index:nldas_index+3]
                                 for nldas_index in
                                 range(0, len(lsm_file_list), 3)
                                 if len(lsm_file_list[
                                        nldas_index:nldas_index+3]) == 3]

            if len(lsm_file_list) < num_cpus:
                num_cpus = len(lsm_file_list)
            # pylint: disable=no-member
            mp_lock = multiprocessing.Manager().Lock()
            partition_list, partition_index_list = \
                partition(lsm_file_list, num_cpus)

            for loop_index, cpu_grouped_file_list in enumerate(partition_list):
                if cpu_grouped_file_list and partition_index_list[loop_index]:
                    job_combinations.append((
                        cpu_grouped_file_list,
                        partition_index_list[loop_index],
                        weight_table_file,
                        lsm_file_data['grid_type'],
                        master_rapid_runoff_file,
                        lsm_file_data['rapid_inflow_tool'],
                        mp_lock))
#                   # COMMENTED CODE IS FOR DEBUGGING
#                   generate_inflows_from_runoff((
#                       cpu_grouped_file_list,
#                       partition_index_list[loop_index],
#                       lsm_file_data['weight_table_file'],
#                       lsm_file_data['grid_type'],
#                       master_rapid_runoff_file,
#                       lsm_file_data['rapid_inflow_tool'],
#                       mp_lock))
            pool = multiprocessing.Pool(num_cpus)
            pool.map(generate_inflows_from_runoff,
                     job_combinations)
            pool.close()
            pool.join()

            # set up RAPID manager
            rapid_manager = RAPID(
                rapid_executable_location=rapid_executable_location,
                cygwin_bin_location=cygwin_bin_location,
                num_processors=num_cpus,
                mpiexec_command=mpiexec_command,
                ZS_TauR=time_step,
                ZS_dtR=15 * 60,
                ZS_TauM=total_num_time_steps * time_step,
                ZS_dtM=time_step)

            if initial_flows_file and os.path.exists(initial_flows_file):
                rapid_manager.update_parameters(
                    Qinit_file=initial_flows_file,
                    BS_opt_Qinit=True
                )

            # run RAPID for the watershed
            lsm_rapid_output_file = \
                os.path.join(master_watershed_output_directory,
                             'Qout_{0}'.format(out_file_ending))
            rapid_manager.update_parameters(
                rapid_connect_file=case_insensitive_file_search(
                    master_watershed_input_directory,
                    r'rapid_connect\.csv'),
                Vlat_file=master_rapid_runoff_file,
                riv_bas_id_file=case_insensitive_file_search(
                    master_watershed_input_directory,
                    r'riv_bas_id\.csv'),
                k_file=case_insensitive_file_search(
                    master_watershed_input_directory,
                    r'k\.csv'),
                x_file=case_insensitive_file_search(
                    master_watershed_input_directory,
                    r'x\.csv'),
                Qout_file=lsm_rapid_output_file
            )

            rapid_manager.update_reach_number_data()

            output_file_information[
                os.path.basename(master_watershed_input_directory)] = {
                    'm3_riv': master_rapid_runoff_file,
                    'qout': lsm_rapid_output_file
                }

            if generate_rapid_namelist_file:
                rapid_manager.generate_namelist_file(
                    os.path.join(master_watershed_input_directory,
                                 "rapid_namelist_{}"
                                 .format(out_file_ending[:-3])))
            if run_rapid_simulation:
                rapid_manager.run()

                rapid_manager.make_output_cf_compliant(
                    simulation_start_datetime=actual_simulation_start_datetime,
                    comid_lat_lon_z_file=in_rivid_lat_lon_z_file,
                    project_name="{0} Based Historical flows by {1}"
                                 .format(lsm_file_data['description'],
                                         modeling_institution)
                )

                # generate return periods
                if generate_return_periods_file and \
                        os.path.exists(lsm_rapid_output_file) and \
                        lsm_rapid_output_file:
                    return_periods_file = os.path.join(
                        master_watershed_output_directory,
                        'return_periods_{0}'.format(out_file_ending))
                    # assume storm has 3 day length
                    storm_length_days = 3
                    generate_return_periods(
                        qout_file=lsm_rapid_output_file,
                        return_period_file=return_periods_file,
                        num_cpus=num_cpus,
                        storm_duration_days=storm_length_days,
                        method=return_period_method)

                # generate seasonal averages file
                if generate_seasonal_averages_file and \
                        os.path.exists(lsm_rapid_output_file) and \
                        lsm_rapid_output_file:
                    seasonal_averages_file = os.path.join(
                        master_watershed_output_directory,
                        'seasonal_averages_{0}'.format(out_file_ending))
                    generate_seasonal_averages(lsm_rapid_output_file,
                                               seasonal_averages_file,
                                               num_cpus)

                # generate seasonal initialization file
                if generate_seasonal_initialization_file and \
                        os.path.exists(lsm_rapid_output_file) and \
                        lsm_rapid_output_file:
                    seasonal_qinit_file = os.path.join(
                        master_watershed_input_directory,
                        'seasonal_qinit_{0}.csv'.format(out_file_ending[:-3]))
                    rapid_manager.generate_seasonal_intitialization(
                        seasonal_qinit_file)

                # generate initialization file
                if generate_initialization_file and \
                        os.path.exists(lsm_rapid_output_file) and \
                        lsm_rapid_output_file:
                    qinit_file = os.path.join(
                        master_watershed_input_directory,
                        'qinit_{0}.csv'.format(out_file_ending[:-3]))
                    rapid_manager.generate_qinit_from_past_qout(qinit_file)

        all_output_file_information.append(output_file_information)

    # print info to user
    time_end = datetime.utcnow()
    print("Time Begin All: {0}".format(time_begin_all))
    print("Time Finish All: {0}".format(time_end))
    print("TOTAL TIME: {0}".format(time_end-time_begin_all))

    return all_output_file_information
def run_lsm_rapid_process ( rapid_executable_location , lsm_data_location , rapid_io_files_location = None , rapid_input_location = None , rapid_output_location = None , simulation_start_datetime = None , simulation_end_datetime = datetime . utcnow ( ) , file_datetime_pattern = None , file_datetime_re_pattern = None , initial_flows_file = None , ensemble_list = ( None , ) , generate_rapid_namelist_file = True , run_rapid_simulation = True , generate_return_periods_file = False , return_period_method = 'weibul' , generate_seasonal_averages_file = False , generate_seasonal_initialization_file = False , generate_initialization_file = False , use_all_processors = True , num_processors = 1 , mpiexec_command = "mpiexec" , cygwin_bin_location = "" , modeling_institution = "US Army Engineer Research " "and Development Center" , convert_one_hour_to_three = False , expected_time_step = None ) : time_begin_all = datetime . utcnow ( ) if use_all_processors is True : num_cpus = multiprocessing . cpu_count ( ) elif num_processors > multiprocessing . cpu_count ( ) : print ( "WARNING: Num processors requested exceeded max. Set to max ..." ) num_cpus = multiprocessing . cpu_count ( ) else : num_cpus = num_processors rapid_directories = [ ] if rapid_io_files_location is not None : main_rapid_input_directory = os . path . join ( rapid_io_files_location , 'input' ) for watershed_directory in get_valid_directory_list ( main_rapid_input_directory ) : watershed_input_path = os . path . join ( main_rapid_input_directory , watershed_directory ) watershed_output_path = os . path . join ( rapid_io_files_location , 'output' , watershed_directory ) rapid_directories . append ( ( watershed_input_path , watershed_output_path ) ) elif None not in ( rapid_input_location , rapid_output_location ) : rapid_directories = [ ( rapid_input_location , rapid_output_location ) ] else : raise ValueError ( "Need 'rapid_io_files_location' or " "'rapid_input_location' and 'rapid_output_location'" " set to continue." ) all_output_file_information = [ ] for ensemble in ensemble_list : output_file_information = { 'ensemble' : ensemble , } ensemble_file_ending = ".nc" ensemble_file_ending4 = ".nc4" if ensemble is not None : ensemble_file_ending = "_{0}.nc" . format ( ensemble ) ensemble_file_ending4 = "_{0}.nc4" . format ( ensemble ) lsm_file_list = [ ] for walkdir_info in os . walk ( lsm_data_location , followlinks = True ) : for lsm_file in walkdir_info [ 2 ] : if lsm_file . endswith ( ensemble_file_ending ) or lsm_file . endswith ( ensemble_file_ending4 ) : lsm_file_list . append ( os . path . join ( walkdir_info [ 0 ] , lsm_file ) ) lsm_file_list = sorted ( lsm_file_list ) lsm_file_data = identify_lsm_grid ( lsm_file_list [ 0 ] ) if file_datetime_pattern is None or file_datetime_re_pattern is None : file_datetime_re_pattern = DEFAULT_LSM_INPUTS [ lsm_file_data [ 'grid_type' ] ] [ 'file_datetime_re_pattern' ] file_datetime_pattern = DEFAULT_LSM_INPUTS [ lsm_file_data [ 'grid_type' ] ] [ 'file_datetime_pattern' ] file_re_match = re . compile ( file_datetime_re_pattern ) if simulation_start_datetime is not None : print ( "Filtering files by datetime ..." ) lsm_file_list_subset = [ ] for lsm_file in lsm_file_list : match = file_re_match . search ( lsm_file ) file_date = datetime . strptime ( match . group ( 0 ) , file_datetime_pattern ) if file_date > simulation_end_datetime : break if file_date >= simulation_start_datetime : lsm_file_list_subset . append ( lsm_file ) lsm_file_list = sorted ( lsm_file_list_subset ) print ( "Running from {0} to {1}" . format ( lsm_file_list [ 0 ] , lsm_file_list [ - 1 ] ) ) actual_simulation_start_datetime , actual_simulation_end_datetime , time_step , total_num_time_steps = determine_start_end_timestep ( lsm_file_list , file_re_match = file_re_match , file_datetime_pattern = file_datetime_pattern , expected_time_step = expected_time_step , lsm_grid_info = lsm_file_data ) if ( lsm_file_data [ 'grid_type' ] in ( 'nldas' , 'lis' , 'joules' ) ) and convert_one_hour_to_three : num_extra_files = total_num_time_steps % 3 if num_extra_files != 0 : print ( "WARNING: Number of files needs to be divisible by 3. " "Remainder is {0}" . format ( num_extra_files ) ) print ( "This means your simulation will be truncated" ) total_num_time_steps /= 3 time_step *= 3 out_file_ending = "{0}_{1}_{2}hr_{3:%Y%m%d}to{4:%Y%m%d}{5}" . format ( lsm_file_data [ 'model_name' ] , lsm_file_data [ 'grid_type' ] , int ( time_step / 3600 ) , actual_simulation_start_datetime , actual_simulation_end_datetime , ensemble_file_ending ) for master_watershed_input_directory , master_watershed_output_directory in rapid_directories : print ( "Running from: {0}" . format ( master_watershed_input_directory ) ) try : os . makedirs ( master_watershed_output_directory ) except OSError : pass master_rapid_runoff_file = os . path . join ( master_watershed_output_directory , 'm3_riv_bas_{0}' . format ( out_file_ending ) ) weight_table_file = case_insensitive_file_search ( master_watershed_input_directory , lsm_file_data [ 'weight_file_name' ] ) try : in_rivid_lat_lon_z_file = case_insensitive_file_search ( master_watershed_input_directory , r'comid_lat_lon_z\.csv' ) except IndexError : in_rivid_lat_lon_z_file = "" print ( "WARNING: comid_lat_lon_z file not found." " The lat/lon will not be added ..." ) print ( "Writing inflow file to: {0}" . format ( master_rapid_runoff_file ) ) lsm_file_data [ 'rapid_inflow_tool' ] . generateOutputInflowFile ( out_nc = master_rapid_runoff_file , start_datetime_utc = actual_simulation_start_datetime , number_of_timesteps = total_num_time_steps , simulation_time_step_seconds = time_step , in_rapid_connect_file = case_insensitive_file_search ( master_watershed_input_directory , r'rapid_connect\.csv' ) , in_rivid_lat_lon_z_file = in_rivid_lat_lon_z_file , land_surface_model_description = lsm_file_data [ 'description' ] , modeling_institution = modeling_institution ) job_combinations = [ ] if ( lsm_file_data [ 'grid_type' ] in ( 'nldas' , 'lis' , 'joules' ) ) and convert_one_hour_to_three : print ( "Grouping {0} in threes" . format ( lsm_file_data [ 'grid_type' ] ) ) lsm_file_list = [ lsm_file_list [ nldas_index : nldas_index + 3 ] for nldas_index in range ( 0 , len ( lsm_file_list ) , 3 ) if len ( lsm_file_list [ nldas_index : nldas_index + 3 ] ) == 3 ] if len ( lsm_file_list ) < num_cpus : num_cpus = len ( lsm_file_list ) mp_lock = multiprocessing . Manager ( ) . Lock ( ) partition_list , partition_index_list = partition ( lsm_file_list , num_cpus ) for loop_index , cpu_grouped_file_list in enumerate ( partition_list ) : if cpu_grouped_file_list and partition_index_list [ loop_index ] : job_combinations . append ( ( cpu_grouped_file_list , partition_index_list [ loop_index ] , weight_table_file , lsm_file_data [ 'grid_type' ] , master_rapid_runoff_file , lsm_file_data [ 'rapid_inflow_tool' ] , mp_lock ) ) pool = multiprocessing . Pool ( num_cpus ) pool . map ( generate_inflows_from_runoff , job_combinations ) pool . close ( ) pool . join ( ) rapid_manager = RAPID ( rapid_executable_location = rapid_executable_location , cygwin_bin_location = cygwin_bin_location , num_processors = num_cpus , mpiexec_command = mpiexec_command , ZS_TauR = time_step , ZS_dtR = 15 * 60 , ZS_TauM = total_num_time_steps * time_step , ZS_dtM = time_step ) if initial_flows_file and os . path . exists ( initial_flows_file ) : rapid_manager . update_parameters ( Qinit_file = initial_flows_file , BS_opt_Qinit = True ) lsm_rapid_output_file = os . path . join ( master_watershed_output_directory , 'Qout_{0}' . format ( out_file_ending ) ) rapid_manager . update_parameters ( rapid_connect_file = case_insensitive_file_search ( master_watershed_input_directory , r'rapid_connect\.csv' ) , Vlat_file = master_rapid_runoff_file , riv_bas_id_file = case_insensitive_file_search ( master_watershed_input_directory , r'riv_bas_id\.csv' ) , k_file = case_insensitive_file_search ( master_watershed_input_directory , r'k\.csv' ) , x_file = case_insensitive_file_search ( master_watershed_input_directory , r'x\.csv' ) , Qout_file = lsm_rapid_output_file ) rapid_manager . update_reach_number_data ( ) output_file_information [ os . path . basename ( master_watershed_input_directory ) ] = { 'm3_riv' : master_rapid_runoff_file , 'qout' : lsm_rapid_output_file } if generate_rapid_namelist_file : rapid_manager . generate_namelist_file ( os . path . join ( master_watershed_input_directory , "rapid_namelist_{}" . format ( out_file_ending [ : - 3 ] ) ) ) if run_rapid_simulation : rapid_manager . run ( ) rapid_manager . make_output_cf_compliant ( simulation_start_datetime = actual_simulation_start_datetime , comid_lat_lon_z_file = in_rivid_lat_lon_z_file , project_name = "{0} Based Historical flows by {1}" . format ( lsm_file_data [ 'description' ] , modeling_institution ) ) if generate_return_periods_file and os . path . exists ( lsm_rapid_output_file ) and lsm_rapid_output_file : return_periods_file = os . path . join ( master_watershed_output_directory , 'return_periods_{0}' . format ( out_file_ending ) ) storm_length_days = 3 generate_return_periods ( qout_file = lsm_rapid_output_file , return_period_file = return_periods_file , num_cpus = num_cpus , storm_duration_days = storm_length_days , method = return_period_method ) if generate_seasonal_averages_file and os . path . exists ( lsm_rapid_output_file ) and lsm_rapid_output_file : seasonal_averages_file = os . path . join ( master_watershed_output_directory , 'seasonal_averages_{0}' . format ( out_file_ending ) ) generate_seasonal_averages ( lsm_rapid_output_file , seasonal_averages_file , num_cpus ) if generate_seasonal_initialization_file and os . path . exists ( lsm_rapid_output_file ) and lsm_rapid_output_file : seasonal_qinit_file = os . path . join ( master_watershed_input_directory , 'seasonal_qinit_{0}.csv' . format ( out_file_ending [ : - 3 ] ) ) rapid_manager . generate_seasonal_intitialization ( seasonal_qinit_file ) if generate_initialization_file and os . path . exists ( lsm_rapid_output_file ) and lsm_rapid_output_file : qinit_file = os . path . join ( master_watershed_input_directory , 'qinit_{0}.csv' . format ( out_file_ending [ : - 3 ] ) ) rapid_manager . generate_qinit_from_past_qout ( qinit_file ) all_output_file_information . append ( output_file_information ) time_end = datetime . utcnow ( ) print ( "Time Begin All: {0}" . format ( time_begin_all ) ) print ( "Time Finish All: {0}" . format ( time_end ) ) print ( "TOTAL TIME: {0}" . format ( time_end - time_begin_all ) ) return all_output_file_information
This is the main process to generate inflow for RAPID and to run RAPID.

    Parameters
    ----------
    rapid_executable_location: str
        Path to the RAPID executable.
    lsm_data_location: str
        Path to the directory containing the Land Surface Model output files.
    rapid_io_files_location: str, optional
        Path to the directory containing the input and output folders for
        RAPID. This is for running multiple watersheds.
    rapid_input_location: str, optional
        Path to directory with RAPID simulation input data.
        Required if `rapid_io_files_location` is not set.
    rapid_output_location: str, optional
        Path to directory to put output. Required if
        `rapid_io_files_location` is not set.
    simulation_start_datetime: datetime, optional
        Datetime object with date bound of earliest simulation start.
    simulation_end_datetime: datetime, optional
        Datetime object with date bound of latest simulation end.
        Defaults to :obj:`datetime.utcnow`.
    file_datetime_pattern: str, optional
        Datetime pattern for files (Ex. '%Y%m%d%H'). If set,
        `file_datetime_re_pattern` is required.
        Various defaults used by each model.
    file_datetime_re_pattern: raw str, optional
        Regex pattern to extract datetime (Ex. r'\d{10}').
        If set, `file_datetime_pattern` is required.
        Various defaults used by each model.
    initial_flows_file: str, optional
        If given, this is the path to a file with initial flows
        for the simulaion.
    ensemble_list: list, optional
        This is the expexted ensemble name appended to the end of the
        file name.
    generate_rapid_namelist_file: bool, optional
        If True, this will create a RAPID namelist file for the run in
        your RAPID input directory. Default is True.
    run_rapid_simulation: bool, optional
        If True, the RAPID simulation will run after generating the
            inflow file. Default is True.
    generate_return_periods_file: bool, optional
        If True, the return period file will be generated in the output.
        Default is False.
    return_period_method: str, optional
        If True, the return period file will be generated in the output.
        Default is False.
    generate_seasonal_averages_file: bool, optional
        If True, the season average file will be generated. Default is False.
    generate_seasonal_initialization_file: bool, optional
        If True, an intialization based on the seasonal average for the
        current day of the year will be created. Default is False.
    generate_initialization_file: bool, optional
        If True, an initialization file from the last time step of the
        simulation willl be created. Default is False.
    use_all_processors: bool, optional
        If True, it will use all available processors to perform this
        operation. Default is True.
    num_processors: int, optional
        If use_all_processors is False, this argument will determine the
        number of processors to use. Default is 1.
    mpiexec_command: str, optional
        This is the command to execute RAPID. Default is "mpiexec".
    cygwin_bin_location: str, optional
        If using Windows, this is the path to the Cygwin bin location.
        Default is "".
    modeling_institution: str, optional
        This is the institution performing the modeling and is in the
        output files.
        Default is "US Army Engineer Research and Development Center".
    convert_one_hour_to_three: bool, optional
        If the time step is expected to be 1-hr it will convert to 3.
        Set to False if the LIS, NLDAS, or Joules grid time step is
        greater than 1-hr.
    expected_time_step: int, optional
        The time step in seconds of your LSM input data if only one file
        is given. Required if only one file is present.


    Returns
    -------
    list:
        A list of output file information.


    Example of regular run:

    .. code:: python

        from datetime import datetime
        from RAPIDpy.inflow import run_lsm_rapid_process

        run_lsm_rapid_process(
            rapid_executable_location='/home/alan/rapid/src/rapid',
            rapid_io_files_location='/home/alan/rapid-io',
            lsm_data_location='/home/alan/era_data',
        )

    Example of single input/output run:

    .. code:: python

        from datetime import datetime
        from RAPIDpy.inflow import run_lsm_rapid_process

        run_lsm_rapid_process(
            rapid_executable_location='/home/alan/rapid/src/rapid',
            rapid_input_location='/home/alan/rapid-io/input/provo_watershed',
            rapid_output_location='/home/alan/rapid-io/output/provo_watershed',
            lsm_data_location='/home/alan/era_data',
        )


    Example of run with FLDAS and datetime filter:

    .. note:: http://disc.sci.gsfc.nasa.gov/uui/datasets?keywords=FLDAS

    .. code:: python

        from datetime import datetime
        from RAPIDpy.inflow import run_lsm_rapid_process

        run_lsm_rapid_process(
            rapid_executable_location='/home/alan/rapid/src/rapid',
            rapid_io_files_location='/home/alan/rapid-io',
            lsm_data_location='/home/alan/lsm_data',
            simulation_start_datetime=datetime(1980, 1, 1),
            file_datetime_re_pattern = r'\d{8}',
            file_datetime_pattern = "%Y%m%d",
        )


    Example of run with CMIP5:

    .. note:: http://gdo-dcp.ucllnl.org/downscaled_cmip_projections/techmemo/BCSD5HydrologyMemo.pdf


    .. code:: python

        from datetime import datetime
        from RAPIDpy.inflow import run_lsm_rapid_process

        run_lsm_rapid_process(
            rapid_executable_location='/home/jimwlewis/rapid/src/rapid',
            rapid_io_files_location='/data/rapid-io4',
            lsm_data_location='/data/rapid-io4/input/cmip5-jun01',
            simulation_start_datetime=datetime(2001, 1, 1),
            simulation_end_datetime=datetime(2002, 12, 31),
            file_datetime_pattern="%Y",
            file_datetime_re_pattern=r'\d{4}',
        )