
def ml_optimisation_selection(c, density_percentile, weights_percentile):
    weight = df_all.WEIGHT_GRIZ
    density = df_all.GAL_DENS
    limit_density = density > np.percentile(density, density_percentile)
    limit_weights = weight > np.percentile(weight, weights_percentile)
    return np.where(limit_density & limit_weights)[0]


def ml_optimisation_pipeline(candidate_list):
    try:
        name = 'cfht_all_matched_galaxies.pkl'
        data = os.path.join(glo.dir_interim, name)
        with open(data, "rb") as data_in:
            recs = pickle.load(data_in)

    # TODO: fix this, what if i=0 retuns None?...then you're fucked
    except FileNotFoundError as e:
        j = 0
        for i, c in enumerate(candidate_list):
            new_recs = cfht_match(c)
            if new_recs is not None:
                name = '{0}_cfht_matched.csv'.format(c)
                out = os.path.join(c.entry, 'processed', name)
                # data_out = open(out_dir, "wb")
                # new_recs.tofile(data_out, sep=",")
                # data_out.close()
                new_recs.to_csv(path_or_buf=out, sep=",", index=False)
                if j == 0:
                    recs = new_recs
                elif new_recs:
                    # recs = np.hstack((recs, new_recs))
                    recs = pd.concat([recs, new_recs])
                    j = j + 1

        # To improve the speed of plotting without needed to rerun.
        out_dir = os.path.join(glo.dir_interim, 'cfht_all_matched_galaxies.pkl')
        data_out = open(out_dir, "wb")
        pickle.dump(recs, data_out)
        data_out.close()

    # param_grid = {  #     selection__density_percentile : [0.7, 0.8, 0.9],
    #     selection__weights_percentile : [0.6, 0.7, 0.8]  # }  #  # # Ra,
    #  Dec, weight and density goes in, match (array-like of bool) comes out.
    #  pipeline = make_pipeline(  #                     # pass each candidate
    #  name to an estimator for  #                     # selection.  #  #  []
    #  [] [] [] []  #                     # [bools] [bools] [bools] [  #  #
    #  bools]  #                     # horizontal union of estimator [bools])
    #     ml_optimisation_selection(density_percentile=density_percentile,
    #  weights_percentile=weights_percentile)  # )  #  #  #  # grid =  #  #
    #  GridSearchCV(estimator=pipeline, param_grid=param_grid)  #  #  #  #  #
    #  n_true_members = len(idx_sky)  # truth = np.ones(n_true_members,
    #  dtype=bool)  # grid.fit(candidate_list, truth)  # # grid.fit(  #  #  #
    #  dataset.data, dataset.target)




def ml_optimisation_selection(c, density_percentile, weights_percentile):
    weight = df_all.WEIGHT_GRIZ
    density = df_all.GAL_DENS
    limit_density = density > np.percentile(density, density_percentile)
    limit_weights = weight > np.percentile(weight, weights_percentile)
    return np.where(limit_density & limit_weights)[0]


def ml_optimisation_pipeline(candidate_list):
    try:
        name = 'cfht_all_matched_galaxies.pkl'
        data = os.path.join(glo.dir_interim, name)
        with open(data, "rb") as data_in:
            recs = pickle.load(data_in)

    # TODO: fix this, what if i=0 retuns None?...then you're fucked
    except FileNotFoundError as e:
        j = 0
        for i, c in enumerate(candidate_list):
            new_recs = cfht_match(c)
            if new_recs is not None:
                name = '{0}_cfht_matched.csv'.format(c)
                out = os.path.join(c.entry, 'processed', name)
                # data_out = open(out_dir, "wb")
                # new_recs.tofile(data_out, sep=",")
                # data_out.close()
                new_recs.to_csv(path_or_buf=out, sep=",", index=False)
                if j == 0:
                    recs = new_recs
                elif new_recs:
                    # recs = np.hstack((recs, new_recs))
                    recs = pd.concat([recs, new_recs])
                    j = j + 1

        # To improve the speed of plotting without needed to rerun.
        out_dir = os.path.join(glo.dir_interim, 'cfht_all_matched_galaxies.pkl')
        data_out = open(out_dir, "wb")
        pickle.dump(recs, data_out)
        data_out.close()

    # param_grid = {  #     selection__density_percentile : [0.7, 0.8, 0.9],
    #     selection__weights_percentile : [0.6, 0.7, 0.8]  # }  #  # # Ra,
    #  Dec, weight and density goes in, match (array-like of bool) comes out.
    #  pipeline = make_pipeline(  #                     # pass each candidate
    #  name to an estimator for  #                     # selection.  #  #  []
    #  [] [] [] []  #                     # [bools] [bools] [bools] [  #  #
    #  bools]  #                     # horizontal union of estimator [bools])
    #     ml_optimisation_selection(density_percentile=density_percentile,
    #  weights_percentile=weights_percentile)  # )  #  #  #  # grid =  #  #
    #  GridSearchCV(estimator=pipeline, param_grid=param_grid)  #  #  #  #  #
    #  n_true_members = len(idx_sky)  # truth = np.ones(n_true_members,
    #  dtype=bool)  # grid.fit(candidate_list, truth)  # # grid.fit(  #  #  #
    #  dataset.data, dataset.target)



# TODO: fix this, what if i=0 retuns None?...then you're fucked # data_out = open(out_dir, "wb") # new_recs.tofile(data_out, sep=",") # data_out.close() # recs = np.hstack((recs, new_recs)) # To improve the speed of plotting without needed to rerun. # param_grid = {  #     selection__density_percentile : [0.7, 0.8, 0.9], #     selection__weights_percentile : [0.6, 0.7, 0.8]  # }  #  # # Ra, #  Dec, weight and density goes in, match (array-like of bool) comes out. #  pipeline = make_pipeline(  #                     # pass each candidate #  name to an estimator for  #                     # selection.  #  #  [] #  [] [] [] []  #                     # [bools] [bools] [bools] [  #  # #  bools]  #                     # horizontal union of estimator [bools]) #     ml_optimisation_selection(density_percentile=density_percentile, #  weights_percentile=weights_percentile)  # )  #  #  #  # grid =  #  # #  GridSearchCV(estimator=pipeline, param_grid=param_grid)  #  #  #  #  # #  n_true_members = len(idx_sky)  # truth = np.ones(n_true_members, #  dtype=bool)  # grid.fit(candidate_list, truth)  # # grid.fit(  #  #  # #  dataset.data, dataset.target)