<filename>stitches/fx_stitch.py

# Import packages
import numpy as np
import pandas as pd
import pkg_resources
import xarray as xr
import os as os
import stitches.fx_util as util
import stitches.fx_data as data
import stitches.fx_pangeo as pangeo


def find_zfiles(rp):
    """ Determine which cmip files must be downlaoded from pangeo.
        :param rp:             data frame of the recepies
        :return:               numpy.ndarray array of the gs:// files to pull from pangeo
    """

    # Figure out which columns contain the string file
    flat_list = rp.filter(regex='file', axis=1).values.flatten()
    unique_list = np.unique(flat_list)
    return unique_list


def find_var_cols(x):
    """ Determine which variables that are going to be downloaded.
        :param x:             pandas data frame of the stitches recipe
        :return:              a list of the variables that are going to be written out to the netcdf files.
    """

    # Parse out the variable name so that we can use it
    # to label the final output.
    set = x.filter(regex='file').columns.tolist()

    out = []
    for text in set:
        new = text.replace("_file", "")
        out.append(new)
    return out


def get_netcdf_values(i, dl, rp, fl, name):

    """Extract the archive values from the list of downloaded cmip data
        :param i:              int index of the row of the recipe data frame
        :param dl:             list of xarray cmip files
        :param rp:             data frame of the recipe
        :param fl:             list of the cmip files
        :param name:           name of the variable file that is going to be processed.
        :return:               a slice of xarray (not sure confident on the technical term)
    """

    file = rp[name][i]
    start_yr = rp["archive_start_yr"][i]
    end_yr = rp["archive_end_yr"][i]

    # Figure out which index level we are on and then get the
    # xarray from the list.
    index = int(np.where(fl == file)[0])
    extracted = dl[index].sortby('time')
    v = name.replace("_file", "")

    # Have to have special time handeler, consider functionalizinng this.
    times = extracted.indexes['time']

    if type(times) in [xr.coding.cftimeindex.CFTimeIndex, pd.core.indexes.datetimes.DatetimeIndex]:
        yrs = extracted.indexes['time'].year # pull out the year information from the time index
        flags = list(map(lambda x: x in range(start_yr, end_yr+1), yrs))
        to_keep = times[flags]
    else:
        raise TypeError(f"unsupported time type")
    dat = extracted.sel(time=to_keep)[v].values.copy()

    if ((times.freq == 'D') | (times.freq == 'day')):
        expected_times = pd.date_range(start=str(start_yr) + "-01-01", end=str(end_yr) + "-12-31", freq='D')
        if times.calendar == 'noleap':
            expected_len = len(expected_times[~((expected_times.month == 2) & (expected_times.day == 29))])
    else:
        expected_len = len(pd.date_range(start=str(start_yr) + "-01-01", end=str(end_yr) + "-12-31", freq='M'))


    assert (len(dat) == expected_len), "Not enough data in " + file + "for period " + str(start_yr) + "-" + str(end_yr)

    return dat


def get_var_info(rp, dl, fl, name):
    """Extract the cmip variable attribute information.
               :param rp:             data frame of the recipes
               :param dl:             list of the data files
               :param fl:             list of the data file names
               :param name:           string of the column containing the variable file name from rp
               :return:               pandas dataframe of the variable meta data
               TODO add a check to make sure that there is only one stitching id being passed into
               the function.
           """
    util.check_columns(rp, {name})
    file = rp[name][0]
    index = int(np.where(fl == file)[0])
    extracted = dl[index]

    attrs = data.get_ds_meta(extracted)
    attrs["calendar"] = extracted.indexes['time'].calendar

    return attrs


def get_atts(rp, dl, fl, name):
    """Extract the cmip variable attribute information.
           :param rp:             data frame of the recepies
           :param dl:             list of the data files
           :param fl:             list of the data file names
           :param name:           string of the column containing the variable files to process
           :return:               dict object containing the cmip variable information
           TODO add a check to make sure that there is only one stitching id being passed into
           the function.
       """
    file = rp[name][0]
    index = int(np.where(fl == file)[0])
    extracted = dl[index]
    v=name.replace("_file", "")

    out=extracted[v].attrs.copy()

    return out


def internal_stitch(rp, dl, fl):
    """Stitch a single recpie into netcdf outputs
        :param dl:             list of xarray cmip files
        :param rp:             data frame of the recipe
        :param fl:             list of the cmip files
        :return:               a list of the data arrays for the stitched products of the different variables.
    """

    rp.reset_index(drop=True, inplace=True)
    variables = find_var_cols(rp)
    out = []

    # For each of the of the variables stitch the
    # data together.
    for v in variables:

        # Get the information about the variable that is going to be stitched together.
        col = v + '_file'
        var_info = get_var_info(rp, dl, fl, col)

        # For each of time slices extract the data & concatenate together.
        gridded_data = get_netcdf_values(i=0, dl=dl, rp=rp, fl=fl, name=col)

        # Now add the other time slices.
        for i in range(1, len(rp)):
            new_vals = get_netcdf_values(i=i, dl=dl, rp=rp, fl=fl, name=col)
            gridded_data = np.concatenate((gridded_data, new_vals), axis=0)

        # Note that the pd.date_range call need the date/month defined otherwise it will
        # truncate the year from start of first year to start of end year which is not
        # what we want. We want the full final year to be included in the times series.
        start = str(min(rp["target_start_yr"]))
        end = str(max(rp["target_end_yr"]))

        if var_info["frequency"][0].lower() == "mon":
            freq = "M"
        elif var_info["frequency"][0].lower() == "day":
            freq = "D"
        else:
            raise TypeError(f"unsupported frequency")

        times = pd.date_range(start=start + "-01-01", end=end + "-12-31", freq=freq)

        # Again, some ESMs stop in 2099 instead of 2100 - so wejust drop the
        # last year of gridded_data when that is the case.
        #TODO this will need something extra/different for daily data; maybe just
        # a simple len(times)==len(gridded_data)-12 : len(times) == len(gridded_data)-(nDaysInYear)
        # with correct parentheses would do it
        if ((max(rp["target_end_yr"]) == 2099) & (len(times) == (len(gridded_data) - 12))):
            gridded_data = gridded_data[0:len(times), 0:, 0:].copy()

        if ((var_info["calendar"][0].lower() == "noleap") & (freq == "D")):
            times = times[~((times.month == 2) & (times.day == 29))]

        assert (len(gridded_data) == len(times)), "Problem with the length of time"

        # Extract the lat and lon information that will be used to structure the
        # empty netcdf file. Make sure to copy all of the information including
        # the attributes!
        lat = dl[0].lat.copy()
        lon = dl[0].lon.copy()

        r = rp.reset_index(drop=True).to_string()
        rslt = xr.Dataset({v: xr.DataArray(
            gridded_data,
            coords=[times, lat, lon],
            dims=["time", "lat", 'lon'],
            attrs={'units': var_info['units'][0],
                   'variable': var_info['variable'][0],
                   'experiment': var_info['experiment'][0],
                   'ensemble': var_info['ensemble'][0],
                   'model': var_info['model'][0],
                   'stitching_id': rp['stitching_id'].unique()[0],
                   'recipe': r})
        })

        out.append(rslt)

    out_dict = dict(zip(variables, out))

    return out_dict


def gridded_stitching(out_dir, rp):
    """Stitch
        :param out_dir:        string directory location where to write the netcdf files out to
        :param rp:             data frame of the recipe
        :return:               a list of the netcdf files paths
    """

    flag = os.path.isdir(out_dir)
    if not flag:
        raise TypeError(f'The output directory does not exist.')

    # Check inputs.
    util.check_columns(rp, {'target_start_yr', 'target_end_yr', 'archive_experiment',
                            'archive_variable', 'archive_model', 'archive_ensemble', 'stitching_id',
                            'archive_start_yr', 'archive_end_yr'})

    # Determine which variables will be downloaded.
    variables = find_var_cols(rp)
    if not (len(variables) >= 1):
        raise TypeError(f'No variables were found to be processed.')

    # Determine which files need to be downloaded from pangeo.
    file_list = find_zfiles(rp)

    # Make sure that all of the files are available to download from pangeo.
    # Note that this might be excessively cautious but this is an issue we have run into in
    # the past.
    avail = pangeo.fetch_pangeo_table()
    flag = all(item in list(avail['zstore']) for item in list(file_list))
    if not flag:
        raise TypeError(f'Trying to request a zstore file that does not exist')

    # Download all of the data from pangeo.
    data_list = list(map(pangeo.fetch_nc, file_list))

    # For each of the stitching recipes go through and stitch a recipe.
    for single_id in rp['stitching_id'].unique():
        # initialize f to be empty just to be safe now that we've added a
        # try...except approach. It's technically possible the first id
        # tried will fail and the function will try to return a non-existent f.
        f = []

        try:
            print((
                    'Stitching gridded netcdf for: ' + rp.archive_model.unique() + " " + rp.archive_variable.unique() + " " + single_id))

            # Do the stitching!
            # ** this can be a slow step and prone to errors
            single_rp = rp.loc[rp['stitching_id'] == single_id].copy()
            rslt = internal_stitch(rp=single_rp, dl=data_list, fl=file_list)

            # Print the files out at netcdf files
            f = []
            for i in rslt.keys():
                ds = rslt[i]
                ds = ds.sortby('time').copy()
                fname = (out_dir + '/' + "stitched_" + ds[i].attrs['model'] + '_' +
                         ds[i].attrs['variable'] + '_' + single_id + '.nc')
                ds.to_netcdf(fname)
                f.append(fname)
            # end For loop over rslt keys
        #end try

        except:
            print(('Stitching gridded netcdf for: ' + rp.archive_model.unique() + " " + rp.archive_variable.unique() + " " + single_id +' failed. Skipping. Error thrown within gridded_stitching fxn.'))
        # end except
     # end for loop over single_id

    return f
# end gridded stitching function


def gmat_internal_stitch(row, data):
    """ Select data from a tas archive based on a single row in a recipe data frame, this
            function is used to iterate over an entire recipe to do the stitching.

            :param row:        pandas.core.series.Series a row entry of a fully formatted recpie
            :param data:       pandas.core.frame.DataFrame containing the tas values to be stiched togeher
            :return:           pandas.core.frame.DataFrame of tas values
    """
    years = list(range(int(row["target_start_yr"]), int(row["target_end_yr"]) + 1))
    select_years = list(range(int(row["archive_start_yr"]), int(row["archive_end_yr"]) + 1))

    selected_data = data.loc[(data["experiment"] == row["archive_experiment"]) &
                             (data["year"].isin(select_years)) &
                             (data["ensemble"] == row["archive_ensemble"])]

    # some models stop at 2099 instead of 2100 - so there is a mismatch
    # between len(years) and selected data but not a fatal one.
    # Write a very specific if statement to catch this & just chop the extra year
    # off the end of selected_data.
    if ((len(years) == (util.nrow(selected_data) - 1)) & (max(years) == 2099) ):
        selected_data = selected_data.iloc[0:len(years), ].copy()

    if len(years) != util.nrow(selected_data):
        raise TypeError(f"Trouble with selecting the tas data.")

    new_vals = selected_data['value']
    d = {'year': years,
         'value': new_vals}
    df = pd.DataFrame(data=d)
    df['variable'] = 'tas'

    return df


# TODO ACS we do have a bit of a behavior change here so that this function so that the
# TODO rp read in here is the same as the rp read in to the gridded_stitching function.
def gmat_stitching(rp):
    """ Based on a recipe data frame stitch together a time series of global tas data.

        :param rp:        pandas.core.frame.DataFrame a fully formatted recipe data frame.
        :return:          pandas.core.frame.DataFrame of stitched together tas data.
    """

    # Check inputs.
    util.check_columns(rp, {'target_start_yr', 'target_end_yr', 'archive_experiment',
                            'archive_variable', 'archive_model', 'archive_ensemble', 'stitching_id',
                            'archive_start_yr', 'archive_end_yr', 'tas_file'})

    # One the assumptions of this function is that it only works with tas, so
    # we can safely add tas as the variable column.
    rp['variable'] = 'tas'
    out = []
    for name, match in rp.groupby(['stitching_id']):

        # Reset the index in the match data frame so that we can use a for loop
        # to iterate through match data frame an apply the gmat_internal_stitch.
        match = match.reset_index(drop=True)

        # Find the tas data to be stitched together.
        dir_path = pkg_resources.resource_filename('stitches', 'data/tas-data')
        all_files = util.list_files(dir_path)

        # Load the tas data for a particular model.
        model = match['archive_model'].unique()[0]
        csv_to_load = [file for file in all_files if (model in file)][0]
        data = pd.read_csv(csv_to_load)

        # Format the data so that if we have historical years in the future scenarios
        # then that experiment is relabeled as "historical".
        fut_exps = ['ssp245', 'ssp126', 'ssp585', 'ssp119', 'ssp370', 'ssp434', 'ssp534-over', 'ssp460']
        nonssp_data = data.loc[~data["experiment"].isin(fut_exps)]
        fut_data = data.loc[(data["experiment"].isin(fut_exps)) &
                            (data["year"] > 2014)]
        hist_data = data.loc[(data["experiment"].isin(fut_exps)) &
                             (data["year"] <= 2014)]
        hist_data["experiment"] = "historical"
        tas_data = pd.concat([nonssp_data, fut_data, hist_data])[['variable', 'experiment', 'ensemble', 'model', 'year',
                                                                  'value']].drop_duplicates().reset_index(drop=True)
        # Stitch the data together based on the matched recpies.
        dat = []
        for i in match.index:
            row = match.iloc[i, :]
            dat.append(gmat_internal_stitch(row, tas_data))
        dat = pd.concat(dat)

        # Add the stitiching id column to the data frame.
        dat['stitching_id'] = name

        # Add the data to the out list
        out.append(dat)

    # Format the list of data frames into a single data frame.
    final_output = pd.concat(out)
    final_output = final_output.reset_index(drop=True).copy()
    final_output = final_output.sort_values(['stitching_id', 'year']).copy()
    final_output = final_output.reset_index(drop=True).copy()
    return final_output




































<filename>stitches/fx_stitch.py

# Import packages
import numpy as np
import pandas as pd
import pkg_resources
import xarray as xr
import os as os
import stitches.fx_util as util
import stitches.fx_data as data
import stitches.fx_pangeo as pangeo


def find_zfiles(rp):
    """ Determine which cmip files must be downlaoded from pangeo.
        :param rp:             data frame of the recepies
        :return:               numpy.ndarray array of the gs:// files to pull from pangeo
    """

    # Figure out which columns contain the string file
    flat_list = rp.filter(regex='file', axis=1).values.flatten()
    unique_list = np.unique(flat_list)
    return unique_list


def find_var_cols(x):
    """ Determine which variables that are going to be downloaded.
        :param x:             pandas data frame of the stitches recipe
        :return:              a list of the variables that are going to be written out to the netcdf files.
    """

    # Parse out the variable name so that we can use it
    # to label the final output.
    set = x.filter(regex='file').columns.tolist()

    out = []
    for text in set:
        new = text.replace("_file", "")
        out.append(new)
    return out


def get_netcdf_values(i, dl, rp, fl, name):

    """Extract the archive values from the list of downloaded cmip data
        :param i:              int index of the row of the recipe data frame
        :param dl:             list of xarray cmip files
        :param rp:             data frame of the recipe
        :param fl:             list of the cmip files
        :param name:           name of the variable file that is going to be processed.
        :return:               a slice of xarray (not sure confident on the technical term)
    """

    file = rp[name][i]
    start_yr = rp["archive_start_yr"][i]
    end_yr = rp["archive_end_yr"][i]

    # Figure out which index level we are on and then get the
    # xarray from the list.
    index = int(np.where(fl == file)[0])
    extracted = dl[index].sortby('time')
    v = name.replace("_file", "")

    # Have to have special time handeler, consider functionalizinng this.
    times = extracted.indexes['time']

    if type(times) in [xr.coding.cftimeindex.CFTimeIndex, pd.core.indexes.datetimes.DatetimeIndex]:
        yrs = extracted.indexes['time'].year # pull out the year information from the time index
        flags = list(map(lambda x: x in range(start_yr, end_yr+1), yrs))
        to_keep = times[flags]
    else:
        raise TypeError(f"unsupported time type")
    dat = extracted.sel(time=to_keep)[v].values.copy()

    if ((times.freq == 'D') | (times.freq == 'day')):
        expected_times = pd.date_range(start=str(start_yr) + "-01-01", end=str(end_yr) + "-12-31", freq='D')
        if times.calendar == 'noleap':
            expected_len = len(expected_times[~((expected_times.month == 2) & (expected_times.day == 29))])
    else:
        expected_len = len(pd.date_range(start=str(start_yr) + "-01-01", end=str(end_yr) + "-12-31", freq='M'))


    assert (len(dat) == expected_len), "Not enough data in " + file + "for period " + str(start_yr) + "-" + str(end_yr)

    return dat


def get_var_info(rp, dl, fl, name):
    """Extract the cmip variable attribute information.
               :param rp:             data frame of the recipes
               :param dl:             list of the data files
               :param fl:             list of the data file names
               :param name:           string of the column containing the variable file name from rp
               :return:               pandas dataframe of the variable meta data
               TODO add a check to make sure that there is only one stitching id being passed into
               the function.
           """
    util.check_columns(rp, {name})
    file = rp[name][0]
    index = int(np.where(fl == file)[0])
    extracted = dl[index]

    attrs = data.get_ds_meta(extracted)
    attrs["calendar"] = extracted.indexes['time'].calendar

    return attrs


def get_atts(rp, dl, fl, name):
    """Extract the cmip variable attribute information.
           :param rp:             data frame of the recepies
           :param dl:             list of the data files
           :param fl:             list of the data file names
           :param name:           string of the column containing the variable files to process
           :return:               dict object containing the cmip variable information
           TODO add a check to make sure that there is only one stitching id being passed into
           the function.
       """
    file = rp[name][0]
    index = int(np.where(fl == file)[0])
    extracted = dl[index]
    v=name.replace("_file", "")

    out=extracted[v].attrs.copy()

    return out


def internal_stitch(rp, dl, fl):
    """Stitch a single recpie into netcdf outputs
        :param dl:             list of xarray cmip files
        :param rp:             data frame of the recipe
        :param fl:             list of the cmip files
        :return:               a list of the data arrays for the stitched products of the different variables.
    """

    rp.reset_index(drop=True, inplace=True)
    variables = find_var_cols(rp)
    out = []

    # For each of the of the variables stitch the
    # data together.
    for v in variables:

        # Get the information about the variable that is going to be stitched together.
        col = v + '_file'
        var_info = get_var_info(rp, dl, fl, col)

        # For each of time slices extract the data & concatenate together.
        gridded_data = get_netcdf_values(i=0, dl=dl, rp=rp, fl=fl, name=col)

        # Now add the other time slices.
        for i in range(1, len(rp)):
            new_vals = get_netcdf_values(i=i, dl=dl, rp=rp, fl=fl, name=col)
            gridded_data = np.concatenate((gridded_data, new_vals), axis=0)

        # Note that the pd.date_range call need the date/month defined otherwise it will
        # truncate the year from start of first year to start of end year which is not
        # what we want. We want the full final year to be included in the times series.
        start = str(min(rp["target_start_yr"]))
        end = str(max(rp["target_end_yr"]))

        if var_info["frequency"][0].lower() == "mon":
            freq = "M"
        elif var_info["frequency"][0].lower() == "day":
            freq = "D"
        else:
            raise TypeError(f"unsupported frequency")

        times = pd.date_range(start=start + "-01-01", end=end + "-12-31", freq=freq)

        # Again, some ESMs stop in 2099 instead of 2100 - so wejust drop the
        # last year of gridded_data when that is the case.
        #TODO this will need something extra/different for daily data; maybe just
        # a simple len(times)==len(gridded_data)-12 : len(times) == len(gridded_data)-(nDaysInYear)
        # with correct parentheses would do it
        if ((max(rp["target_end_yr"]) == 2099) & (len(times) == (len(gridded_data) - 12))):
            gridded_data = gridded_data[0:len(times), 0:, 0:].copy()

        if ((var_info["calendar"][0].lower() == "noleap") & (freq == "D")):
            times = times[~((times.month == 2) & (times.day == 29))]

        assert (len(gridded_data) == len(times)), "Problem with the length of time"

        # Extract the lat and lon information that will be used to structure the
        # empty netcdf file. Make sure to copy all of the information including
        # the attributes!
        lat = dl[0].lat.copy()
        lon = dl[0].lon.copy()

        r = rp.reset_index(drop=True).to_string()
        rslt = xr.Dataset({v: xr.DataArray(
            gridded_data,
            coords=[times, lat, lon],
            dims=["time", "lat", 'lon'],
            attrs={'units': var_info['units'][0],
                   'variable': var_info['variable'][0],
                   'experiment': var_info['experiment'][0],
                   'ensemble': var_info['ensemble'][0],
                   'model': var_info['model'][0],
                   'stitching_id': rp['stitching_id'].unique()[0],
                   'recipe': r})
        })

        out.append(rslt)

    out_dict = dict(zip(variables, out))

    return out_dict


def gridded_stitching(out_dir, rp):
    """Stitch
        :param out_dir:        string directory location where to write the netcdf files out to
        :param rp:             data frame of the recipe
        :return:               a list of the netcdf files paths
    """

    flag = os.path.isdir(out_dir)
    if not flag:
        raise TypeError(f'The output directory does not exist.')

    # Check inputs.
    util.check_columns(rp, {'target_start_yr', 'target_end_yr', 'archive_experiment',
                            'archive_variable', 'archive_model', 'archive_ensemble', 'stitching_id',
                            'archive_start_yr', 'archive_end_yr'})

    # Determine which variables will be downloaded.
    variables = find_var_cols(rp)
    if not (len(variables) >= 1):
        raise TypeError(f'No variables were found to be processed.')

    # Determine which files need to be downloaded from pangeo.
    file_list = find_zfiles(rp)

    # Make sure that all of the files are available to download from pangeo.
    # Note that this might be excessively cautious but this is an issue we have run into in
    # the past.
    avail = pangeo.fetch_pangeo_table()
    flag = all(item in list(avail['zstore']) for item in list(file_list))
    if not flag:
        raise TypeError(f'Trying to request a zstore file that does not exist')

    # Download all of the data from pangeo.
    data_list = list(map(pangeo.fetch_nc, file_list))

    # For each of the stitching recipes go through and stitch a recipe.
    for single_id in rp['stitching_id'].unique():
        # initialize f to be empty just to be safe now that we've added a
        # try...except approach. It's technically possible the first id
        # tried will fail and the function will try to return a non-existent f.
        f = []

        try:
            print((
                    'Stitching gridded netcdf for: ' + rp.archive_model.unique() + " " + rp.archive_variable.unique() + " " + single_id))

            # Do the stitching!
            # ** this can be a slow step and prone to errors
            single_rp = rp.loc[rp['stitching_id'] == single_id].copy()
            rslt = internal_stitch(rp=single_rp, dl=data_list, fl=file_list)

            # Print the files out at netcdf files
            f = []
            for i in rslt.keys():
                ds = rslt[i]
                ds = ds.sortby('time').copy()
                fname = (out_dir + '/' + "stitched_" + ds[i].attrs['model'] + '_' +
                         ds[i].attrs['variable'] + '_' + single_id + '.nc')
                ds.to_netcdf(fname)
                f.append(fname)
            # end For loop over rslt keys
        #end try

        except:
            print(('Stitching gridded netcdf for: ' + rp.archive_model.unique() + " " + rp.archive_variable.unique() + " " + single_id +' failed. Skipping. Error thrown within gridded_stitching fxn.'))
        # end except
     # end for loop over single_id

    return f
# end gridded stitching function


def gmat_internal_stitch(row, data):
    """ Select data from a tas archive based on a single row in a recipe data frame, this
            function is used to iterate over an entire recipe to do the stitching.

            :param row:        pandas.core.series.Series a row entry of a fully formatted recpie
            :param data:       pandas.core.frame.DataFrame containing the tas values to be stiched togeher
            :return:           pandas.core.frame.DataFrame of tas values
    """
    years = list(range(int(row["target_start_yr"]), int(row["target_end_yr"]) + 1))
    select_years = list(range(int(row["archive_start_yr"]), int(row["archive_end_yr"]) + 1))

    selected_data = data.loc[(data["experiment"] == row["archive_experiment"]) &
                             (data["year"].isin(select_years)) &
                             (data["ensemble"] == row["archive_ensemble"])]

    # some models stop at 2099 instead of 2100 - so there is a mismatch
    # between len(years) and selected data but not a fatal one.
    # Write a very specific if statement to catch this & just chop the extra year
    # off the end of selected_data.
    if ((len(years) == (util.nrow(selected_data) - 1)) & (max(years) == 2099) ):
        selected_data = selected_data.iloc[0:len(years), ].copy()

    if len(years) != util.nrow(selected_data):
        raise TypeError(f"Trouble with selecting the tas data.")

    new_vals = selected_data['value']
    d = {'year': years,
         'value': new_vals}
    df = pd.DataFrame(data=d)
    df['variable'] = 'tas'

    return df


# TODO ACS we do have a bit of a behavior change here so that this function so that the
# TODO rp read in here is the same as the rp read in to the gridded_stitching function.
def gmat_stitching(rp):
    """ Based on a recipe data frame stitch together a time series of global tas data.

        :param rp:        pandas.core.frame.DataFrame a fully formatted recipe data frame.
        :return:          pandas.core.frame.DataFrame of stitched together tas data.
    """

    # Check inputs.
    util.check_columns(rp, {'target_start_yr', 'target_end_yr', 'archive_experiment',
                            'archive_variable', 'archive_model', 'archive_ensemble', 'stitching_id',
                            'archive_start_yr', 'archive_end_yr', 'tas_file'})

    # One the assumptions of this function is that it only works with tas, so
    # we can safely add tas as the variable column.
    rp['variable'] = 'tas'
    out = []
    for name, match in rp.groupby(['stitching_id']):

        # Reset the index in the match data frame so that we can use a for loop
        # to iterate through match data frame an apply the gmat_internal_stitch.
        match = match.reset_index(drop=True)

        # Find the tas data to be stitched together.
        dir_path = pkg_resources.resource_filename('stitches', 'data/tas-data')
        all_files = util.list_files(dir_path)

        # Load the tas data for a particular model.
        model = match['archive_model'].unique()[0]
        csv_to_load = [file for file in all_files if (model in file)][0]
        data = pd.read_csv(csv_to_load)

        # Format the data so that if we have historical years in the future scenarios
        # then that experiment is relabeled as "historical".
        fut_exps = ['ssp245', 'ssp126', 'ssp585', 'ssp119', 'ssp370', 'ssp434', 'ssp534-over', 'ssp460']
        nonssp_data = data.loc[~data["experiment"].isin(fut_exps)]
        fut_data = data.loc[(data["experiment"].isin(fut_exps)) &
                            (data["year"] > 2014)]
        hist_data = data.loc[(data["experiment"].isin(fut_exps)) &
                             (data["year"] <= 2014)]
        hist_data["experiment"] = "historical"
        tas_data = pd.concat([nonssp_data, fut_data, hist_data])[['variable', 'experiment', 'ensemble', 'model', 'year',
                                                                  'value']].drop_duplicates().reset_index(drop=True)
        # Stitch the data together based on the matched recpies.
        dat = []
        for i in match.index:
            row = match.iloc[i, :]
            dat.append(gmat_internal_stitch(row, tas_data))
        dat = pd.concat(dat)

        # Add the stitiching id column to the data frame.
        dat['stitching_id'] = name

        # Add the data to the out list
        out.append(dat)

    # Format the list of data frames into a single data frame.
    final_output = pd.concat(out)
    final_output = final_output.reset_index(drop=True).copy()
    final_output = final_output.sort_values(['stitching_id', 'year']).copy()
    final_output = final_output.reset_index(drop=True).copy()
    return final_output




































# Import packages Determine which cmip files must be downlaoded from pangeo.         :param rp:             data frame of the recepies         :return:               numpy.ndarray array of the gs:// files to pull from pangeo # Figure out which columns contain the string file Determine which variables that are going to be downloaded.         :param x:             pandas data frame of the stitches recipe         :return:              a list of the variables that are going to be written out to the netcdf files. # Parse out the variable name so that we can use it # to label the final output. Extract the archive values from the list of downloaded cmip data         :param i:              int index of the row of the recipe data frame         :param dl:             list of xarray cmip files         :param rp:             data frame of the recipe         :param fl:             list of the cmip files         :param name:           name of the variable file that is going to be processed.         :return:               a slice of xarray (not sure confident on the technical term) # Figure out which index level we are on and then get the # xarray from the list. # Have to have special time handeler, consider functionalizinng this. # pull out the year information from the time index Extract the cmip variable attribute information.                :param rp:             data frame of the recipes                :param dl:             list of the data files                :param fl:             list of the data file names                :param name:           string of the column containing the variable file name from rp                :return:               pandas dataframe of the variable meta data                TODO add a check to make sure that there is only one stitching id being passed into                the function. Extract the cmip variable attribute information.            :param rp:             data frame of the recepies            :param dl:             list of the data files            :param fl:             list of the data file names            :param name:           string of the column containing the variable files to process            :return:               dict object containing the cmip variable information            TODO add a check to make sure that there is only one stitching id being passed into            the function. Stitch a single recpie into netcdf outputs         :param dl:             list of xarray cmip files         :param rp:             data frame of the recipe         :param fl:             list of the cmip files         :return:               a list of the data arrays for the stitched products of the different variables. # For each of the of the variables stitch the # data together. # Get the information about the variable that is going to be stitched together. # For each of time slices extract the data & concatenate together. # Now add the other time slices. # Note that the pd.date_range call need the date/month defined otherwise it will # truncate the year from start of first year to start of end year which is not # what we want. We want the full final year to be included in the times series. # Again, some ESMs stop in 2099 instead of 2100 - so wejust drop the # last year of gridded_data when that is the case. #TODO this will need something extra/different for daily data; maybe just # a simple len(times)==len(gridded_data)-12 : len(times) == len(gridded_data)-(nDaysInYear) # with correct parentheses would do it # Extract the lat and lon information that will be used to structure the # empty netcdf file. Make sure to copy all of the information including # the attributes! Stitch         :param out_dir:        string directory location where to write the netcdf files out to         :param rp:             data frame of the recipe         :return:               a list of the netcdf files paths # Check inputs. # Determine which variables will be downloaded. # Determine which files need to be downloaded from pangeo. # Make sure that all of the files are available to download from pangeo. # Note that this might be excessively cautious but this is an issue we have run into in # the past. # Download all of the data from pangeo. # For each of the stitching recipes go through and stitch a recipe. # initialize f to be empty just to be safe now that we've added a # try...except approach. It's technically possible the first id # tried will fail and the function will try to return a non-existent f. # Do the stitching! # ** this can be a slow step and prone to errors # Print the files out at netcdf files # end For loop over rslt keys #end try # end except # end for loop over single_id # end gridded stitching function Select data from a tas archive based on a single row in a recipe data frame, this             function is used to iterate over an entire recipe to do the stitching.              :param row:        pandas.core.series.Series a row entry of a fully formatted recpie             :param data:       pandas.core.frame.DataFrame containing the tas values to be stiched togeher             :return:           pandas.core.frame.DataFrame of tas values # some models stop at 2099 instead of 2100 - so there is a mismatch # between len(years) and selected data but not a fatal one. # Write a very specific if statement to catch this & just chop the extra year # off the end of selected_data. # TODO ACS we do have a bit of a behavior change here so that this function so that the # TODO rp read in here is the same as the rp read in to the gridded_stitching function. Based on a recipe data frame stitch together a time series of global tas data.          :param rp:        pandas.core.frame.DataFrame a fully formatted recipe data frame.         :return:          pandas.core.frame.DataFrame of stitched together tas data. # Check inputs. # One the assumptions of this function is that it only works with tas, so # we can safely add tas as the variable column. # Reset the index in the match data frame so that we can use a for loop # to iterate through match data frame an apply the gmat_internal_stitch. # Find the tas data to be stitched together. # Load the tas data for a particular model. # Format the data so that if we have historical years in the future scenarios # then that experiment is relabeled as "historical". # Stitch the data together based on the matched recpies. # Add the stitiching id column to the data frame. # Add the data to the out list # Format the list of data frames into a single data frame.