def write_truth_imager_match_obj(filename, match_obj, SETTINGS=None, imager_obj_name='pps'):
    """Write *match_obj* to *filename*."""
    datasets = {'diff_sec_1970': match_obj.diff_sec_1970}
    groups = {imager_obj_name: match_obj.imager.all_arrays}
    imager_attrs = {'imager_instrument': match_obj.imager_instrument}
    groups_attrs = {imager_obj_name: imager_attrs}
    for name in ['calipso', 'calipso_aerosol', 'iss', 'modis_lvl2',
                 'amsr', 'synop', 'mora', 'cloudsat', 'extra']:
        if hasattr(match_obj, name):
            groups[name] = getattr(match_obj, name).all_arrays
    write_match_objects(filename, datasets, groups, groups_attrs, SETTINGS=SETTINGS)
    return 1 def total_and_top_layer_optical_depth_5km(calipso, SETTINGS, resolution=5):
    logger.info("Find total optical depth from 5km data")
    optical_depth_in = calipso.feature_optical_depth_532
    o_depth_top_layer = -9.0 + 0*calipso.number_layers_found.ravel()
    total_o_depth = -9.0 + 0*calipso.number_layers_found.ravel()
    if resolution == 5:
        pixels = np.logical_and(
            calipso.number_layers_found.ravel() > 0,
            optical_depth_in[:, 0].ravel() >= 0)
        o_depth_top_layer[pixels] = optical_depth_in[pixels, 0]
        total_o_depth[pixels] = optical_depth_in[pixels, 0]
        for lay in range(1, np.max(calipso.number_layers_found[pixels]), 1):
            pixels = np.logical_and(
                pixels,
                optical_depth_in[:, lay] >= 0)
            total_o_depth[pixels] += optical_depth_in[pixels, lay]
    else:
        print("ERROR this fuction is only for 5km data!")
        print("These features can then added to 1km data set")
    calipso.feature_optical_depth_532_top_layer_5km = o_depth_top_layer
    calipso.total_optical_depth_5km = total_o_depth.astype(np.float32)
    if SETTINGS['CALCULATE_DETECTION_HEIGHT_FROM_5KM_DATA']:
        logger.info("Find detection height using 5km data")
        retv = optical_depth_height_filtering(calipso, 0.0,
                                              limit_ctop=SETTINGS['OPTICAL_LIMIT_CLOUD_TOP'])
        cloud_top5km, _, _, _, _, _, _, detection_height = retv
        calipso.detection_height_5km = detection_height
    else:
        calipso.detection_height_5km = None
    return calipso def reshape_calipso(calipsofiles, res=config.RESOLUTION, ALAY=False):
    # concatenate and reshape calipso files
    start_calipso = get_calipso(calipsofiles[0], res, ALAY=ALAY)
    # Concatenate the data from the different files
    for i in range(len(calipsofiles) - 1):
        new_calipso = get_calipso(calipsofiles[i + 1], res, ALAY=ALAY)
        cal_start_all = start_calipso.profile_time_tai
        cal_new_all = new_calipso.profile_time_tai
        if not cal_start_all[0] < cal_new_all[0]:
            raise ProcessingError("Calipso files are in the wrong order!")
        # Concatenate the feature values
        start_calipso = start_calipso + new_calipso

    return start_calipso def discard_calipso_files_outside_time_range(calipsofiles_list, cloudproducts, values,
                                             SETTINGS, res=config.RESOLUTION, ALAY=False):
    imager_end = cloudproducts.sec1970_end
    imager_start = cloudproducts.sec1970_start
    calipso_within_time_range = []
    for current_file in calipsofiles_list:
        new_calipso = get_calipso(current_file, res, ALAY=ALAY)
        cal_new_all = new_calipso.sec_1970
        if (cal_new_all[0] > imager_end + SETTINGS["sec_timeThr"] or
                cal_new_all[-1] + SETTINGS["sec_timeThr"] < imager_start):
            pass
            # print "skipping file %s outside time_limits"%(current_file)
        else:
            logger.debug("Keeping file %s inside time_limits", os.path.basename(current_file))
            calipso_within_time_range.append(current_file)
    return calipso_within_time_range def match_calipso_imager(values,
                         calipso, calipso_aerosol,
                         cloudproducts, SETTINGS, res=config.RESOLUTION):

    from atrain_match.utils.common import map_imager
    retv = TruthImagerTrackObject(truth='calipso')

    retv.imager_instrument = cloudproducts.instrument.lower()
    retv.calipso = calipso

    cal, cap = map_imager(cloudproducts,
                          calipso.longitude.ravel(),
                          calipso.latitude.ravel(),
                          radius_of_influence=config.RESOLUTION * 0.7 * 1000.0)  # somewhat larger than radius...
    # warn if no matches
    calnan = np.where(cal == config.NODATA, np.nan, cal)
    if (~np.isnan(calnan)).sum() == 0:
        logger.warning("No matches within region.")
        return None
    # check if it is within time limits:
    if len(cloudproducts.time.shape) > 1:
        imager_time_vector = [cloudproducts.time[line, pixel] for line, pixel in zip(cal, cap)]
        imager_lines_sec_1970 = np.where(cal != config.NODATA, imager_time_vector, np.nan)
    else:
        imager_lines_sec_1970 = np.where(cal != config.NODATA, cloudproducts.time[cal], np.nan)
    idx_match = elements_within_range(calipso.sec_1970, imager_lines_sec_1970, SETTINGS["sec_timeThr"])
    if idx_match.sum() == 0:
        logger.warning("No matches in region within time threshold %d s.", SETTINGS["sec_timeThr"])
        return None
    retv.calipso = retv.calipso.extract_elements(idx=idx_match)

    # Calipso line, pixel inside IMAGER swath:
    retv.calipso.imager_linnum = np.repeat(cal, idx_match).astype('i')
    retv.calipso.imager_pixnum = np.repeat(cap, idx_match).astype('i')

    # Imager time
    retv.imager.sec_1970 = imager_lines_sec_1970[idx_match.ravel()]
    retv.diff_sec_1970 = retv.calipso.sec_1970 - retv.imager.sec_1970
    do_some_logging(retv, calipso)
    logger.debug("Generate the latitude, cloudtype tracks!")
    from atrain_match.libs.extract_imager_along_track import imager_track_from_matched
    retv = imager_track_from_matched(retv, SETTINGS,
                                     cloudproducts)
    if calipso_aerosol is not None:
        retv.calipso_aerosol = calipso_aerosol.extract_elements(idx=idx_match)
    max_cloud_top_calipso = np.maximum.reduce(retv.calipso.layer_top_altitude.ravel())
    logger.debug("max_cloud_top_calipso: %2.1f", max_cloud_top_calipso)
    return retv def find_break_points(calipso, cloudproducts, SETTINGS):
    """
    Find the start and end point where calipso and imager matches is within
    time limits.
    """
    imager_end = cloudproducts.sec1970_end
    imager_start = cloudproducts.sec1970_start
    # Finds Break point
    start_break = np.argmin((np.abs((calipso.sec_1970)
                                    - (imager_start - SETTINGS["sec_timeThr"]))))
    end_break = np.argmin((np.abs((calipso.sec_1970)
                                  - (imager_end + SETTINGS["sec_timeThr"])))) + 2    # Plus two, just to be certain
    if start_break != 0:
        start_break = start_break - 1  # Minus one to get one extra, just to be certain
    return start_break, end_break def add_1km_to_5km(calipso1km, calipso5km):
    # First check if length of 5 km and 1 km arrays correspond
    # (i.e. 1 km array = 5 times longer array)
    # Here we check the middle time (index 1) out of the three time values
    # given (start, mid, end) for 5 km data
    length_ok = (calipso5km.profile_utc_time[:, 1] == calipso1km.profile_utc_time[2::5]).sum(
    ) == calipso5km.profile_utc_time.shape[0]
    if not length_ok:
        print("length mismatch")
        raise(ProcessingError)

    # First making a preliminary check of the differences in fraction of cloudy
    # calipso columns in 1 km and 5 km data.
    cfc_5km = 0
    cfc_1km = 0
    len_5km = calipso5km.profile_utc_time.shape[0]
    len_1km = calipso5km.profile_utc_time.shape[0]*5
    for i in range(len_5km):
        if calipso5km.number_layers_found[i] > 0:
            cfc_5km = cfc_5km + 1
    for i in range(len_1km):
        if calipso1km.number_layers_found[i] > 0:
            cfc_1km = cfc_1km + 1

    output = (
        "*****CHECKING CLOUD FREQUENCY DIFFERENCES IN 1KM AND 5KM "
        "DATASETS:"
        "\n"
        "\n Number of 5 km FOVS: {:d}".format(len_5km) +
        "\n Number of cloudy 5 km FOVS: {3.3f}".format(cfc_5km) +
        "\n Cloudy fraction 5 km: {3.3f}".format(float(cfc_5km)/float(len_5km)) +
        "\n Number of 1 km FOVS: {:d}".format(len_1km) +
        "\n Number of cloudy 1 km FOVS:   {3.3f}".format(cfc_1km) +
        "\n Cloudy fraction 1 km:  {3.3f}".format(float(cfc_1km)/float(len_1km)))
    # print(output)
    # Now calculate the cloud fraction in 5 km data from 1 km data
    # (discretized to 0.0, 0.2, 0.4, 0.6, 0.8 and 1.0).
    # In addition, if there are cloud layers in 5 km data but nothing in 1 km
    # data, set cloud fraction to 1.0.
    # This latter case represents when very thin cloud layers are being
    # detected over longer distances
    # Finally, if there are cloud layers in 1 km data but not in 5 km data,
    # add a layer to 5 km data and set corresponding
    # COT to 1.0. Cloud base and cloud tp for this layer is calculated as
    # averages from original levels (max height for
    # top and min height for base if there are more than one layer).
    # This is a pragmatic solution to take care of a
    # weakness or bug in the CALIPSO retrieval of clouds below 4 km
    calipso5km.number_layers_found_1km = 1.0*calipso5km.number_layers_found.copy()
    for i in range(calipso5km.profile_utc_time.shape[0]):
        cfc = 0.0
        for j in range(5):
            if calipso1km.number_layers_found[i*5+j] > 0:
                cfc = cfc + 0.2000
        calipso5km.number_layers_found_1km[i] = cfc
        if cfc == 1.0:
            cfc = 0.99  # Just to be able to track the case
            # when no cloud layers existed in 5 km data
            # but all 1 km FOVs were cloudy /KG 20170213
        if (calipso5km.number_layers_found[i] > 0):
            calipso5km.cloud_fraction[i] = 1.0
        if ((cfc > 0.1) and (calipso5km.number_layers_found[i] == 0)):
            # Add missing layer due to CALIPSO processing bug
            cloudtop_sum = 0.0
            cloudbase_sum = 0.0
            cloud_layers = 0
            feature_array_list = []
            for j in range(5):
                if calipso1km.number_layers_found[i*5+j] != 0:
                    for k in range(calipso1km.number_layers_found[i*5+j]):
                        cloudtop_sum = (cloudtop_sum +
                                        calipso1km.layer_top_altitude[i, k])
                        cloudbase_sum = (cloudbase_sum +
                                         calipso1km.layer_base_altitude[i, k])
                        cloud_layers = cloud_layers + 1
                        feature_array_list.append(
                            calipso1km.feature_classification_flags[i, k])
            calipso5km.number_layers_found[i] = 1
            calipso5km.layer_top_altitude[i, 0] = cloudtop_sum/cloud_layers
            calipso5km.layer_base_altitude[i, 0] = cloudbase_sum/cloud_layers
            calipso5km.feature_optical_depth_532[i, 0] = 1.0  # Just put it safely
            # away from the thinnest cloud layers - the best we can do!
            # calipso5km.feature_classification_flags[i, 0] = 22218
            # if assuming like below:
            # cloud, low quality, water phase, low quality, low broken cumulus
            # , confident, 1 km horizontal averaging)
            feature_array = np.asarray(feature_array_list)
            calipso5km.feature_classification_flags[i, 0] = np.median(
                feature_array[:])  # However, let's take the median value
            calipso5km.cloud_fraction[i] = cfc
    return calipso5km def add_singleshot_to5km(calipso5km, SETTINGS):  # Valid only for CALIPSO-CALIOP version 4.10
    logger.info("Making use of new Single Shot cloud cleared information")
    cfc_5km = 0
    cfc_Single_Shot = 0
    len_5km = calipso5km.profile_utc_time.shape[0]
    len_single_shot = calipso5km.number_cloudy_single_shots.shape[0]
    cloudy_5km = np.greater(calipso5km.number_layers_found, 0)
    cloudy_singleshot = np.greater_equal(calipso5km.number_cloudy_single_shots*1.0,
                                         15.0*SETTINGS['CALIPSO_CLOUDY_MIN_CFC'])
    if len_5km == len_single_shot:
        cfc_5km = np.sum(cloudy_5km)
        cfc_Single_Shot = np.sum(cloudy_singleshot)
        cfc_combined = np.sum(np.logical_or(cloudy_5km.ravel(),
                                            cloudy_singleshot.ravel()))
    else:
        raise InputError("Different array length: 5km and Single Shot data!")
    logger.info(
        "*CHECKING CLOUD FREQUENCY DIFFERENCES SINGLE SHOT AND 5KM DATASETS:\n"
        " \t \t Number of 5 km FOVS:                 %d\n"
        " \t \t Number of cloudy 5 km FOVS:          %d\n"
        " \t \t Cloudy fraction 5 km:             %3.2f\n"
        " \t \t Number of Single shot FOVS:          %d\n"
        " \t \t Number of cloudy Single shot FOVS:   %d\n"
        " \t \t Cloudy fraction Single Shots:     %3.2f\n"
        " \t \t Number of Combined FOVS:             %d\n"
        " \t \t Number of cloudy combined FOVS:      %d\n"
        " \t \t Cloudy fraction combined:         %3.2f\n",
        len_5km,
        cfc_5km,
        float(cfc_5km)/float(len_5km),
        len_single_shot,
        cfc_Single_Shot,
        float(cfc_Single_Shot)/float(len_single_shot),
        len_single_shot,
        cfc_combined,
        float(cfc_combined)/float(len_single_shot))
    # Now calculate the cloud fraction in 5 km data from single shot data
    # (fraction of 15 FOVs declared cloudy).
    # In addition, if there are cloud layers in 5 km data but nothing in
    # single shot data, set cloud fraction to 1.0.
    # This latter case represents when very thin cloud layers are being
    # detected over longer distances.
    # Finally, if there are cloud layers in single shot data but not in 5 km
    # data, add a layer to 5 km data and set corresponding
    # COT to 1.0. Cloud base and cloud tp for this layer is calculated as
    # averages from original levels (max height for
    # top and min height for base if there are more than one layer).
    # This is a pragmatic solution to take care of a
    # weakness or bug in the (CALIPSO-Version3?) retrieval of clouds below 4 km
    cfc = calipso5km.number_cloudy_single_shots.ravel()/15.0
    layers_found_5km = calipso5km.number_layers_found.ravel()
    cfc[np.equal(cfc, 1.0)] = 0.99
    cfc[np.greater(layers_found_5km, 0)] = 1.0
    update_these = np.logical_and(np.greater(cfc, 0.001),
                                  np.equal(layers_found_5km, 0))
    calipso5km.number_layers_found[update_these] = 1

    calipso5km.layer_top_altitude[update_these, 0] = (
        calipso5km.average_cloud_top_single_shots[update_these].ravel())
    # averaged from single shot data
    calipso5km.layer_base_altitude[update_these, 0] = (
        calipso5km.average_cloud_base_single_shots[update_these].ravel())
    # averaged from single shot data
    calipso5km.feature_optical_depth_532[update_these, 0] = 1.0  # not very thin
    calipso5km.cloud_fraction = cfc
    return calipso5km def add_5km_variables_to_1km(calipso1km, calipso5km, CALIPSO_version):
    logger.debug("Repeat 5km calipso data to fit 1km resoluiton")
    for variable in ["cfc_single_shots_1km_from_5km_file"]:
        if hasattr(calipso5km, variable):
            cfc_single_shot_1km_from_5km_file = getattr(calipso5km, variable)
            setattr(calipso1km, variable, cfc_single_shot_1km_from_5km_file)
    for variable_5km in [
            "column_optical_depth_tropospheric_aerosols_1064",
            "column_optical_depth_tropospheric_aerosols_532",
            "column_optical_depth_tropospheric_aerosols_uncertainty_1064",
            "column_optical_depth_tropospheric_aerosols_uncertainty_532",
            "column_optical_depth_cloud_532",
            "column_optical_depth_cloud_uncertainty_532",
            "average_cloud_base_single_shots",
            "average_cloud_top_pressure_single_shots",
            "average_cloud_top_single_shots",
            # "feature_optical_depth_532",
            # "feature_optical_depth_uncertainty_532"
    ]:
        if CALIPSO_version == 3:
            if not hasattr(calipso5km, variable_5km) or getattr(calipso5km, variable_5km) is None:
                # in version 3 names were different
                variable_5km = variable_5km.replace('tropospheric_', '')

        data = getattr(calipso5km, variable_5km)
        new_data = np.repeat(data, 5, axis=0)
        setattr(calipso1km, variable_5km + "_5km", new_data)
    for variable_5km in ["feature_optical_depth_532_top_layer_5km",
                         "detection_height_5km",
                         "total_optical_depth_5km"]:
        data = getattr(calipso5km, variable_5km)
        if data is None:
            setattr(calipso1km, variable_5km, None)
        else:
            new_data = np.repeat(data, 5, axis=0)
            setattr(calipso1km, variable_5km, new_data)
    for variable_5km in ["layer_top_altitude",
                         "layer_top_pressure",
                         "feature_optical_depth_532"]:
        data = getattr(calipso5km, variable_5km)
        new_data = np.repeat(data, 5, axis=0)
        new_data = new_data[:, 0:3]
        setattr(calipso1km, variable_5km + "_5km", new_data)

    for variable_5km in ["number_layers_found"]:
        data = getattr(calipso5km, variable_5km)
        new_data = np.repeat(data, 5, axis=0)
        setattr(calipso1km, variable_5km + "_5km", new_data)
    # for variable_5km  in ["layer_top_altitude", "layer_base_altitude"]:
    #    data = getattr(calipso5km, variable_5km)# [:, 0]*1000
    #    data[data<0] =-9
    #    new_data = np.repeat(data, 5, axis=0)
    #    setattr(calipso1km, variable_5km + "_5km", new_data)
    # cfc_5km = np.repeat(calipso5km.cloud_fraction, 5, axis=0)
    # number_layers_found_5km = np.repeat(calipso5km.number_layers_found,
    # 5, axis=0).ravel()
    isCloudOnlyIn5km = np.logical_and(calipso1km.cloud_fraction < 0.1,
                                      calipso1km.total_optical_depth_5km > 0)
    # Give clouds in only 5km  cloud fraction equal to 0.15
    # these clouds are either:
    # 1) thin or
    # 2) other pixels of the 5 1km pixles are cloudy
    # In case 1 we might want to have them as clouds in case 2 we do not.
    # For now just set all of them to 0.15.
    calipso1km.cloud_fraction[
        isCloudOnlyIn5km] = 0.15
    if CALIPSO_version == 4:
        isCloudOnlyIn300m = np.logical_and(calipso1km.cloud_fraction <= 0,
                                           cfc_single_shot_1km_from_5km_file > 0)
        calipso1km.cloud_fraction[
            isCloudOnlyIn300m] = cfc_single_shot_1km_from_5km_file[isCloudOnlyIn300m]
    return calipso1km def adjust_5km_to_1km_resolution(calipso5km):
    logger.debug("Repeat 5km calipso data to fit 1km resoluiton")
    calipso = CalipsoObject()
    for arname, value in calipso5km.all_arrays.items():
        if value is not None:
            new_values = np.repeat(value, 5, axis=0)
            calipso.all_arrays[arname] = new_values
    return calipso def reshape_iss(issfiles, imager, SETTINGS):
    # imager_end = imager.sec1970_end
    # imager_start = imager.sec1970_start
    iss = get_iss(issfiles[0])
    for i in range(len(issfiles) - 1):
        newIss = get_iss(issfiles[i + 1])
        iss_start_all = iss.sec_1970.ravel()
        iss_new_all = newIss.sec_1970.ravel()
        if not iss_start_all[0] < iss_new_all[0]:
            raise ProcessingError("Iss files are in the wrong order")
        # Concatenate the feature values
        iss = iss + newIss

    # Finds Break point
    start_break, end_break = find_break_points(iss, imager, SETTINGS)
    iss = iss.extract_elements(starti=start_break,
                               endi=end_break)
    return iss def match_iss_imager(iss, cloudproducts, SETTINGS):
    retv = TruthImagerTrackObject(truth='iss')
    retv.imager_instrument = cloudproducts.instrument.lower()
    retv.iss = iss

    from atrain_match.utils.common import map_imager
    cal, cap = map_imager(cloudproducts, iss.longitude.ravel(),
                          iss.latitude.ravel(),
                          radius_of_influence=config.RESOLUTION * 0.7 * 1000.0)  # larger than radius...
    calnan = np.where(cal == config.NODATA, np.nan, cal)

    if (~np.isnan(calnan)).sum() == 0:
        logger.warning("No matches within region.")
        return None
    # check if it is within time limits:
    if len(cloudproducts.time.shape) > 1:
        imager_time_vector = [cloudproducts.time[line, pixel] for line, pixel
                              in zip(cal, cap)]
        imager_lines_sec_1970 = np.where(cal != config.NODATA, imager_time_vector, np.nan)
    else:
        imager_lines_sec_1970 = np.where(cal != config.NODATA, cloudproducts.time[cal], np.nan)
    # Find all matching Iss pixels within +/- sec_timeThr from the IMAGER data
    idx_match = elements_within_range(iss.sec_1970, imager_lines_sec_1970, SETTINGS["sec_timeThr"])

    if idx_match.sum() == 0:
        logger.warning("No matches in region within time threshold %d s.", SETTINGS["sec_timeThr"])
        return None

    retv.iss = retv.iss.extract_elements(idx=idx_match)

    # Calipso line, pixel inside IMAGER swath:
    retv.iss.imager_linnum = np.repeat(cal, idx_match).astype('i')
    retv.iss.imager_pixnum = np.repeat(cap, idx_match).astype('i')
    # Imager time
    retv.imager.sec_1970 = np.repeat(imager_lines_sec_1970, idx_match)
    retv.diff_sec_1970 = retv.iss.sec_1970 - retv.imager.sec_1970

    do_some_logging(retv, iss)
    logger.info("Generate the latitude, cloudtype tracks!")
    retv = imager_track_from_matched(retv, SETTINGS, cloudproducts)
    return retv def reshape_synop(synopfiles, imager, SETTINGS):
    start_t = datetime.utcfromtimestamp(imager.sec1970_start)
    end_t = datetime.utcfromtimestamp(imager.sec1970_end)
    # datetime.datetime.fromtimestamp(
    from atrain_match.truths.read_synop_dwd import get_synop_data
    items = [get_synop_data(filename) for filename in synopfiles]
    panda_synops = pd.concat(items, ignore_index=True)
    # import pdb
    # pdb.set_trace()
    dt_ = timedelta(seconds=SETTINGS["sec_timeThr_synop"])
    newsynops = panda_synops[panda_synops['date'] < end_t + dt_]
    panda_synops = newsynops[newsynops['date'] > start_t - dt_]
    retv = SynopObject()
    retv.longitude = np.array(panda_synops['lon'])
    retv.latitude = np.array(panda_synops['lat'])
    retv.cloud_fraction = np.array(panda_synops['total_cloud_cover'])/8.0


#    clmask_obs = np.logical_and(
#        matchups['obs'] > 2. / 8., matchups['obs'] < 6. / 8.)
#    clmask_sat = np.logical_and(
#        matchups['sat'] > 5. / 16., matchups['sat'] < 10. / 16.)

    retv.sec_1970 = np.array([calendar.timegm(tobj.timetuple()) for tobj in panda_synops['date']])
    return retv def match_synop_imager(synop, cloudproducts, SETTINGS):
    retv = TruthImagerTrackObject(truth='synop')
    retv.imager_instrument = cloudproducts.instrument.lower()
    retv.synop = synop
    from atrain_match.utils.common import map_imager_distances
    n_neighbours = 250
    if config.RESOLUTION == 5:
        n_neighbours = 16
    mapper_and_dist = map_imager_distances(cloudproducts,
                                           synop.longitude.ravel(),
                                           synop.latitude.ravel(),
                                           radius_of_influence=SETTINGS["SYNOP_RADIUS"],
                                           n_neighbours=n_neighbours)
    # pdb.set_trace()
    cal, cap = mapper_and_dist["mapper"]
    distances = mapper_and_dist["distances"]
    cal_1 = cal[:, 0]
    cap_1 = cap[:, 0]

    calnan = np.where(cal_1 == config.NODATA, np.nan, cal_1)
    if (~np.isnan(calnan)).sum() == 0:
        logger.warning("No matches within region.")
        return None
    # check if it is within time limits:
    if len(cloudproducts.time.shape) > 1:
        imager_time_vector = [cloudproducts.time[line, pixel] for line, pixel in zip(cal_1, cap_1)]
        imager_lines_sec_1970 = np.where(cal_1 != config.NODATA, imager_time_vector, np.nan)
    else:
        imager_lines_sec_1970 = np.where(cal_1 != config.NODATA, cloudproducts.time[cal_1], np.nan)
    idx_match = elements_within_range(synop.sec_1970, imager_lines_sec_1970, SETTINGS["sec_timeThr_synop"])
    if idx_match.sum() == 0:
        logger.warning("No  matches in region within time threshold %d s.", SETTINGS["sec_timeThr_synop"])
        return None
    retv.synop = retv.synop.extract_elements(idx=idx_match)

    # Synop line, pixel inside IMAGER swath (one nearest neighbour):
    retv.synop.imager_linnum = np.repeat(cal_1, idx_match).astype('i')
    retv.synop.imager_pixnum = np.repeat(cap_1, idx_match).astype('i')
    # Synop line, pixel inside IMAGER swath (several neighbours):
    retv.synop.imager_linnum_nneigh = np.repeat(cal, idx_match, axis=0)
    retv.synop.imager_pixnum_nneigh = np.repeat(cap, idx_match, axis=0)
    retv.synop.imager_synop_dist = np.repeat(distances, idx_match, axis=0)

    # Imager time
    retv.imager.sec_1970 = np.repeat(imager_lines_sec_1970, idx_match)
    retv.diff_sec_1970 = retv.synop.sec_1970 - retv.imager.sec_1970

    do_some_logging(retv, synop)
    logger.debug("Extract imager along track!")

    retv = imager_track_from_matched(retv, SETTINGS,
                                     cloudproducts,
                                     extract_radiances=False,
                                     extract_ctth=False,
                                     extract_cpp=False,
                                     extract_aux_segments=False,
                                     aux_params=['fractionofland', 'landuse'],
                                     find_mean_data_for_x_neighbours=True)
    return retv def reshape_mora(morafiles, imager, SETTINGS):
    start_t = datetime.utcfromtimestamp(imager.sec1970_start)
    end_t = datetime.utcfromtimestamp(imager.sec1970_end)
    # datetime.datetime.fromtimestamp(
    items = [get_mora_data(filename) for filename in morafiles]
    panda_moras = pd.concat(items, ignore_index=True)
    # import pdb
    # pdb.set_trace()
    dt_ = timedelta(seconds=SETTINGS["sec_timeThr_synop"])
    newmoras = panda_moras[panda_moras['date'] < end_t + dt_]
    panda_moras = newmoras[newmoras['date'] > start_t - dt_]
    retv = MoraObject()
    retv.longitude = np.array(panda_moras['lon'])
    retv.latitude = np.array(panda_moras['lat'])
    retv.cloud_base_height = np.array(panda_moras['cloud_base_height'])
    retv.sec_1970 = np.array([calendar.timegm(tobj.timetuple()) for tobj in panda_moras['date']])
    return retv def match_mora_imager(mora, cloudproducts, SETTINGS):
    retv = TruthImagerTrackObject('mora')
    retv.imager_instrument = cloudproducts.instrument.lower()
    retv.mora = mora

    from atrain_match.utils.common import map_imager

    """
    import matplotlib.pyplot as plt
    fig = plt.figure()
    ax = fig.add_subplot(111)
    from atrain_match.plotting.histogram_plotting import distribution_map
    plt.plot(cloudproducts.longitude.ravel(),
              cloudproducts.latitude.ravel(), 'r.')
    plt.plot(mora.longitude.ravel(), mora.latitude.ravel(), 'b.')
    plt.show()
    """
    cal, cap = map_imager(cloudproducts,
                          mora.longitude.ravel(),
                          mora.latitude.ravel(),
                          radius_of_influence=config.RESOLUTION*0.7*1000.0)
    calnan = np.where(cal == config.NODATA, np.nan, cal)
    if (~np.isnan(calnan)).sum() == 0:
        logger.warning("No matches within region.")
        return None
    # check if it is within time limits:
    if len(cloudproducts.time.shape) > 1:
        imager_time_vector = [cloudproducts.time[line, pixel] for line, pixel in zip(cal, cap)]
        imager_lines_sec_1970 = np.where(cal != config.NODATA, imager_time_vector, np.nan)
    else:
        imager_lines_sec_1970 = np.where(cal != config.NODATA, cloudproducts.time[cal], np.nan)
    idx_match = elements_within_range(mora.sec_1970, imager_lines_sec_1970, SETTINGS["sec_timeThr_synop"])
    if idx_match.sum() == 0:
        logger.warning("No matches in region within time threshold %d s.", SETTINGS["sec_timeThr_synop"])
        return None
    retv.mora = retv.mora.extract_elements(idx=idx_match)
    # Mora line, pixel inside IMAGER swath (one nearest neighbour):
    retv.mora.imager_linnum = np.repeat(cal, idx_match).astype('i')
    retv.mora.imager_pixnum = np.repeat(cap, idx_match).astype('i')
    # Imager time
    retv.imager.sec_1970 = np.repeat(imager_lines_sec_1970, idx_match)
    retv.diff_sec_1970 = retv.mora.sec_1970 - retv.imager.sec_1970

    do_some_logging(retv, mora)
    logger.debug("Extract imager along track!")

    retv = imager_track_from_matched(retv,
                                     SETTINGS,
                                     cloudproducts,
                                     extract_nwp_segments=False)
    return retv def reshape_amsr(amsrfiles, imager, SETTINGS):
    amsr = get_amsr(amsrfiles[0])
    for i in range(len(amsrfiles) - 1):
        new_amsr = get_amsr(amsrfiles[i + 1])
        amsr_start_all = amsr.sec_1970.ravel()
        amsr_new_all = new_amsr.sec_1970.ravel()
        if not amsr_start_all[0] < amsr_new_all[0]:
            raise ProcessingError("AMSR files are in the wrong order")
        # Concatenate the feature values
        amsr = amsr + new_amsr
    start_break, end_break = find_break_points(amsr, imager, SETTINGS)
    amsr = amsr.extract_elements(starti=start_break,
                                 endi=end_break)
    return amsr def match_amsr_imager(amsr, cloudproducts, SETTINGS):
    retv = TruthImagerTrackObject(truth='amsr')
    retv.imager_instrument = cloudproducts.instrument.lower()
    retv.amsr = amsr
    if (getattr(cloudproducts.cpp, "cpp_lwp") < 0).all():
        logger.warning("Not matching AMSR-E with scene with no lwp.")
        return None
        # return MatchupError("No imager Lwp.") # if only LWP matching?

    from atrain_match.utils.common import map_imager_distances
    n_neighbours = 8
    if config.RESOLUTION == 5:
        n_neighbours = 5
    mapper_and_dist = map_imager_distances(cloudproducts,
                                           amsr.longitude.ravel(),
                                           amsr.latitude.ravel(),
                                           radius_of_influence=AMSR_RADIUS,
                                           n_neighbours=n_neighbours)
    cal, cap = mapper_and_dist["mapper"]
    distances = mapper_and_dist["distances"]
    cal_1 = cal[:, 0]
    cap_1 = cap[:, 0]

    calnan = np.where(cal_1 == config.NODATA, np.nan, cal_1)
    if (~np.isnan(calnan)).sum() == 0:
        logger.warning("No matches within region.")
        return None
    # check if it is within time limits:
    if len(cloudproducts.time.shape) > 1:
        imager_time_vector = [cloudproducts.time[line, pixel] for line, pixel in zip(cal_1, cap_1)]
        imager_lines_sec_1970 = np.where(cal_1 != config.NODATA, imager_time_vector, np.nan)
    else:
        imager_lines_sec_1970 = np.where(cal_1 != config.NODATA, cloudproducts.time[cal_1], np.nan)
    # Find all matching Amsr pixels within +/- sec_timeThr from the IMAGER data
    imager_sunz_vector = np.array([cloudproducts.imager_angles.sunz.data[line, pixel]
                                   for line, pixel in zip(cal_1, cap_1)])
    idx_match = np.logical_and(
        elements_within_range(amsr.sec_1970, imager_lines_sec_1970, SETTINGS["sec_timeThr"]),
        imager_sunz_vector <= 84)  # something larger than 84 (max for lwp)

    if idx_match.sum() == 0:
        logger.warning("No light (sunz<84)  matches in region within time threshold %d s.", SETTINGS["sec_timeThr"])
        return None
    retv.amsr = retv.amsr.extract_elements(idx=idx_match)

    # Amsr line, pixel inside IMAGER swath (one neighbour):
    retv.amsr.imager_linnum = np.repeat(cal_1, idx_match).astype('i')
    retv.amsr.imager_pixnum = np.repeat(cap_1, idx_match).astype('i')
    retv.amsr.imager_linnum_nneigh = np.repeat(cal, idx_match, axis=0)
    retv.amsr.imager_pixnum_nneigh = np.repeat(cap, idx_match, axis=0)
    retv.amsr.imager_amsr_dist = np.repeat(distances, idx_match, axis=0)

    # Imager time
    retv.imager.sec_1970 = np.repeat(imager_lines_sec_1970, idx_match)
    retv.diff_sec_1970 = retv.amsr.sec_1970 - retv.imager.sec_1970

    do_some_logging(retv, amsr)
    logger.debug("Extract imager lwp along track!")

    retv = imager_track_from_matched(retv, SETTINGS,
                                     cloudproducts,
                                     extract_radiances=False,
                                     extract_aux_segments=False,
                                     extract_ctth=False,
                                     extract_ctype=False,
                                     aux_params=['fractionofland', 'landuse'],
                                     extract_some_data_for_x_neighbours=True)
    return retv def reshapeCloudsat(cloudsatfiles, imager, SETTINGS):
    # imager_end = imager.sec1970_end
    # imager_start = imager.sec1970_start
    clsat = get_cloudsat(cloudsatfiles[0])
    for i in range(len(cloudsatfiles) - 1):
        newCloudsat = get_cloudsat(cloudsatfiles[i + 1])
        clsat_start_all = clsat.sec_1970.ravel()
        clsat_new_all = newCloudsat.sec_1970.ravel()
        if not clsat_start_all[0] < clsat_new_all[0]:
            raise ProcessingError("CloudSat files are in the wrong order!")
        # clsat_break = np.argmin(np.abs(clsat_start_all - clsat_new_all[0]))+1
        # Concatenate the feature values
        clsat = clsat + newCloudsat
        # print("taistart", clsat.TAI_start)

    # Finds Break point
    start_break, end_break = find_break_points(clsat, imager, SETTINGS)
    clsat = clsat.extract_elements(starti=start_break,
                                   endi=end_break)
    return clsat def match_cloudsat_imager(cloudsat, cloudproducts, SETTINGS):
    retv = TruthImagerTrackObject(truth='cloudsat')
    retv.imager_instrument = cloudproducts.instrument.lower()
    retv.cloudsat = cloudsat
    # Nina 20150313 Swithcing to mapping without area as in cpp. Following suggestion from Jakob
    from atrain_match.utils.common import map_imager
    cal, cap = map_imager(cloudproducts,
                          cloudsat.longitude.ravel(),
                          cloudsat.latitude.ravel(),
                          radius_of_influence=config.RESOLUTION * 0.7 * 1000.0)  # somewhat larger than radius...
    calnan = np.where(cal == config.NODATA, np.nan, cal)
    if (~np.isnan(calnan)).sum() == 0:
        logger.warning("No matches within region.")
        return None
    # check if it is within time limits:
    if len(cloudproducts.time.shape) > 1:
        imager_time_vector = [cloudproducts.time[line, pixel] for line, pixel in zip(cal, cap)]
        imager_lines_sec_1970 = np.where(cal != config.NODATA, imager_time_vector, np.nan)
    else:
        imager_lines_sec_1970 = np.where(cal != config.NODATA, cloudproducts.time[cal], np.nan)
    # Find all matching Cloudsat pixels within +/- sec_timeThr from the IMAGER data
    idx_match = elements_within_range(cloudsat.sec_1970, imager_lines_sec_1970, SETTINGS["sec_timeThr"])

    if idx_match.sum() == 0:
        logger.warning("No matches in region within time threshold %d s.", SETTINGS["sec_timeThr"])
        return None
    retv.cloudsat = retv.cloudsat.extract_elements(idx=idx_match)

    # Cloudsat line, pixel inside IMAGER swath:
    retv.cloudsat.imager_linnum = np.repeat(cal, idx_match).astype('i')
    retv.cloudsat.imager_pixnum = np.repeat(cap, idx_match).astype('i')

    # Imager time
    retv.imager.sec_1970 = np.repeat(imager_lines_sec_1970, idx_match)
    retv.diff_sec_1970 = retv.cloudsat.sec_1970 - retv.imager.sec_1970
    do_some_logging(retv, cloudsat)
    logger.debug("Generate the latitude, cloudtype tracks!")
    retv = imager_track_from_matched(retv, SETTINGS,
                                     cloudproducts)
    return retv def merge_cloudsat(cloudsat, cloudsatlwp):
    # map cloudsat_lwp to cloudsat
    from atrain_match.utils.match import match_lonlat
    source = (cloudsatlwp.longitude.astype(np.float64).reshape(-1, 1),
              cloudsatlwp.latitude.astype(np.float64).reshape(-1, 1))
    target = (cloudsat.longitude.astype(np.float64).reshape(-1, 1),
              cloudsat.latitude.astype(np.float64).reshape(-1, 1))
    mapper, dummy = match_lonlat(source, target, radius_of_influence=10, n_neighbours=1)
    cloudsat_lwp_index = mapper.rows.filled(config.NODATA).ravel()
    # Transfer CloudSat LWP to ordinary cloudsat obj
    index = cloudsat_lwp_index.copy()
    index[index < 0] = 0
    for key in cloudsatlwp.all_arrays.keys():
        if cloudsatlwp.all_arrays[key] is None:
            continue
        if key not in cloudsat.all_arrays.keys() or cloudsat.all_arrays[key] is None:
            # cloudsat.all_arrays[key] = cloudsatlwp.all_arrays[key]
            if cloudsatlwp.all_arrays[key].ndim == 1:
                cloudsat.all_arrays[key] = np.where(
                    cloudsat_lwp_index >= 0,
                    cloudsatlwp.all_arrays[key][index], -9)
            else:
                cloudsat.all_arrays[key] = np.zeros(cloudsat.all_arrays["Height"].shape)
                # data
                cloudsat.all_arrays[key][:, :] = cloudsatlwp.all_arrays[key][index, :]
                # nodata where data are missing
                cloudsat.all_arrays[key][cloudsat_lwp_index < 0, :] = -9
        if key.lower() in ["liq_water_path", "precip_liq_water_path", "cloud_liq_water_path",
                           "ice_water_path"]:
            # kg/m2 to g/m2 R05
            density = 1000
            cloudsat.all_arrays[key.lower() + "_gm2"] = np.where(
                cloudsat.all_arrays[key] < 0, -9, density * cloudsat.all_arrays[key])
        if key in ["RVOD_liq_water_path, RVOD_ice_water_path"]:
            # already in g/m2 R04 RVOD
            density = 1.0
            cloudsat.all_arrays[key.replace("RVOD_", "") + "_gm2"] = np.where(
                cloudsat.all_arrays[key] < 0, -9, density * cloudsat.all_arrays[key])
        if key in ["RO_liq_water_path", "RO_ice_water_path"]:
            # already in g/m2 RO-05 (not in RVOD-05)
            density = 1.0
            cloudsat.all_arrays[key.replace("RO_", "") + "_gm2"] = np.where(
                cloudsat.all_arrays[key] < 0, -9, density * cloudsat.all_arrays[key])
    return cloudsat class MatchupError(Exception):
    """This exception is used when a problem matching IMAGER data with Cloudsat / CALIPSO data has occured."""

    pass class ProcessingError(Exception):
    """This exception is used when the processing fails."""

    pass INSTRUMENT = {'npp': 'viirs',
              'noaa18': 'avhrr',
              'meteosat8': 'seviri',
              'meteosat9': 'seviri',
              'meteosat10': 'seviri',
              'meteosat11': 'seviri',
              'fy3d': 'mersi2',
              'noaa20': 'viirs',
              'sga1': 'metimage',
              'epsga1': 'metimage',
              'metopsga1': 'metimage',
              'eos1': 'modis',
              'eos2': 'modis'}
from atrain_match.matchobject_io import write_truth_imager_match_obj
from atrain_match.truths.calipso import (
    total_and_top_layer_optical_depth_5km,
    reshape_calipso,
    discard_calipso_files_outside_time_range,
    match_calipso_imager,
    find_break_points,
    add_1km_to_5km,
    add_singleshot_to5km,
    add_5km_variables_to_1km,
    adjust_5km_to_1km_resolution)
from atrain_match.truths.iss import reshape_iss, match_iss_imager
from atrain_match.truths.synop import (reshape_synop, match_synop_imager)
from atrain_match.truths.mora import (reshape_mora, match_mora_imager)
from atrain_match.truths.amsr import (reshape_amsr, match_amsr_imager)
from atrain_match.truths.cloudsat import (reshapeCloudsat,
                                          match_cloudsat_imager,
                                          merge_cloudsat)
from atrain_match.utils.common import MatchupError, ProcessingError
from atrain_match.config import INSTRUMENT
from datetime import timedelta, datetime
    from datetime import timezone as my_tz
from glob import glob
    from atrain_match.cloudproducts.read_pps import get_satid_datetime_orbit_from_fname_pps
    from atrain_match.cloudproducts.read_cci import get_satid_datetime_orbit_from_fname_cci
    from atrain_match.cloudproducts.read_oca import get_satid_datetime_orbit_from_fname_oca
    from atrain_match.cloudproducts.read_maia import get_satid_datetime_orbit_from_fname_maia
    from atrain_match.cloudproducts.read_patmosx import get_satid_datetime_orbit_from_fname_patmosx
    from atrain_match.cloudproducts.read_cci import cci_read_all
    from atrain_match.cloudproducts.read_oca import oca_read_all
    from atrain_match.cloudproducts.read_maia import maia_read_all
    from atrain_match.cloudproducts.read_patmosx import patmosx_read_all
    from atrain_match.cloudproducts.read_pps import pps_read_all
        from atrain_match.utils.match import match_lonlat
            from atrain_match.utils.pps_prototyping_util import add_cnn_features_full
            from atrain_match.cloudproducts.read_modis_products import add_modis_06
            from atrain_match.libs.extract_imager_along_track import _interpolate_height_and_temperature_from_pressure
import atrain_match.config as config
import os
import numpy as np
import logging
    import pytz as my_tz
    import glob
            import pps_nwp
    if (file_name_dict['cma'] is None and
        file_name_dict['cloudtype'] is None and
            file_name_dict['cmaprob'] is None):
        raise MatchupError("No cma, cloudtype or cmaprob file "
                           "found atrain_match.cfg")
    if (SETTINGS['USE_CT_FOR_CFC_STATISTICS'] and
            file_name_dict['cloudtype'] is None):
        logger.error(
            "\n\tError: USE_CT_FOR_CFC_STATISTICS=True, but ..."
            "\n\t... no cloudtype file in atrain_match.cfg")
        raise MatchupError("Configure problems, see messages above.")
    if (SETTINGS['USE_CMA_FOR_CFC_STATISTICS'] and
            file_name_dict['cma'] is None):
        logger.error(
            "\n\tError: USE_CMA_FOR_CFC_STATISTICS=True, but..."
            "\n\t... no cma file in atrain_match.cfg")
        raise MatchupError("Configure problems, see messages above.")
    if (SETTINGS['USE_CMAPROB_FOR_CFC_STATISTICS'] and
            file_name_dict['cmaprob'] is None):
        logger.error(
            "\n\tError: USE_CMAPROB_FOR_CFC_STATISTICS=True, but..."
            "\n\t... no cmaprob file in atrain_match.cfg")
        raise MatchupError("Configure problems, see messages above.")


def find_pps_files_from_imager_file(imager_file, AM_PATHS, SETTINGS, as_oldstyle=False):
    """Find all addidional files needed to process matchup for PPS."""
    logger.debug("IMAGER: %s", imager_file)
    values = get_satid_datetime_orbit_from_fname(imager_file,
                                                 SETTINGS,
                                                 cross=None,
                                                 as_oldstyle=as_oldstyle)
    file_name_dict = {}
    file_name_dict['cma'] = get_one_pps_file(imager_file, AM_PATHS, values,
                                             'cma_file', 'cma_dir',
                                             FailIfRequestedAndMissing=True)
    file_name_dict['cloudtype'] = get_one_pps_file(imager_file, AM_PATHS, values,
                                                   'cloudtype_file', 'cloudtype_dir',
                                                   FailIfRequestedAndMissing=True)
    file_name_dict['cmaprob'] = get_one_pps_file(imager_file, AM_PATHS, values, 'cmaprob_file', 'cmaprob_dir',
                                                 FailIfRequestedAndMissing=True)
    # Check cfc configuration
    check_cfc_configuration(file_name_dict, SETTINGS)
    # For CTTH can have several files:
    ctth_files = {}
    file_name_dict['ctth'] = ctth_files  # If no ctth matching requested
    if 'ctth_file' in AM_PATHS.keys():
        for ctth_type in SETTINGS['CTTH_TYPES']:
            values['ctth_type'] = ctth_type
            ctth_files[ctth_type] = get_one_pps_file(imager_file, AM_PATHS, values,
                                                     'ctth_file', 'ctth_dir',
                                                     FailIfRequestedAndMissing=True)
        file_name_dict.update({'ctth': ctth_files})

    file_name_dict['cpp'] = get_one_pps_file(imager_file, AM_PATHS, values, 'cpp_file', 'cpp_dir',
                                             FailIfRequestedAndMissing=True)
    file_name_dict['sunsatangles'] = get_one_pps_file(imager_file, AM_PATHS, values, 'sunsatangles_file',
                                                      'sunsatangles_dir',
                                                      FailIfRequestedAndMissing=True)
    file_name_dict['physiography'] = get_one_pps_file(imager_file, AM_PATHS, values, 'physiography_file',
                                                      'physiography_dir',
                                                      FailIfRequestedAndMissing=True)
    file_name_dict['r37'] = get_one_pps_file(imager_file, AM_PATHS, values, 'r37_file', 'r37_dir',
                                             FailIfRequestedAndMissing=True)

    file_name_dict['nwp_tsur'] = get_one_pps_file(imager_file, AM_PATHS, values,
                                                  'nwp_tsur_file', 'nwp_nwp_dir',
                                                  FailIfRequestedAndMissing=True)
    file_name_dict['emis'] = get_one_pps_file(imager_file, AM_PATHS, values,
                                              'emis_file', 'emis_dir',
                                              FailIfRequestedAndMissing=True)

    file_name_dict['seaice'] = get_one_pps_file(imager_file, AM_PATHS, values,
                                                'seaice_file', 'seaice_dir',
                                                FailIfRequestedAndMissing=True)
    # Textures and Thresholds:
    file_name_dict['text_t11'] = get_one_pps_file(imager_file, AM_PATHS, values,
                                                  'text_t11_file', 'text_dir',
                                                  FailIfRequestedAndMissing=True)
    file_name_dict['thr_t11ts'] = get_one_pps_file(imager_file, AM_PATHS, values,
                                                   'thr_t11ts_file', 'thr_dir',
                                                   FailIfRequestedAndMissing=True)
    # More NWP, textures and thresholds for v2014:
    for nwp_file in ['nwp_t500', 'nwp_t700',
                     'nwp_t850', 'nwp_t950', 'nwp_ciwv', 'nwp_ttro']:
        file_name_dict[nwp_file] = get_one_pps_file(imager_file, AM_PATHS, values,
                                                    nwp_file+'_file', 'nwp_nwp_dir',
                                                    OnlyPrintInDebugMode=True)
    for text_file in ['text_r06', 'text_t37t12', 'text_t37']:
        file_name_dict[text_file] = get_one_pps_file(imager_file, AM_PATHS, values,
                                                     text_file+'_file', 'text_dir',
                                                     OnlyPrintInDebugMode=True)

    for thr_file in ['thr_t11ts_inv', 'thr_t11t37_inv',
                     'thr_t37t12_inv', 'thr_t11t12_inv',
                     'thr_t11ts', 'thr_t11t37', 'thr_t37t12', 'thr_t11t12',
                     'thr_r09', 'thr_r06', 'thr_t85t11_inv', 'thr_t85t11']:
        file_name_dict[thr_file] = get_one_pps_file(imager_file, AM_PATHS, values,
                                                    thr_file+'_file', 'thr_dir',
                                                    OnlyPrintInDebugMode=True)
    file_name_dict['nwp_segments'] = get_one_pps_file(imager_file, AM_PATHS,
                                                      values,
                                                      'segment_file', 'segment_dir')
    file_name_dict['nnextra'] = get_one_pps_file(imager_file, AM_PATHS,
                                                 values,
                                                 'nnextra_file', 'nnextra_dir')   
    # -------------------------------------#
    ppsfiles = ppsFiles(file_name_dict)
    return ppsfiles


def get_cloudsat_matchups(cloudsat_files, cloudsat_files_lwp, cloudproducts, SETTINGS):
    """Read Cloudsat data and match with the given PPS data."""
    cloudsat_lwp = None
    cloudsat = None
    if cloudsat_files is not None:
        logger.debug("Reading cloudsat for type GEOPROF.")
        cloudsat = reshapeCloudsat(cloudsat_files, cloudproducts, SETTINGS)
    if cloudsat_files_lwp is not None:
        logger.debug("Reading cloudsat for type CWC-RVOD.")

        cloudsat_lwp = reshapeCloudsat(cloudsat_files_lwp, cloudproducts, SETTINGS)