<gh_stars>0
from os import path
import os
import logging

import pandas as pd
import click
import numpy as np
from tqdm import tqdm
from joblib import Parallel, delayed
from dataprepro.csv2hdf5 import glob_and_check

logger = logging.getLogger(__name__)

DATAM1SSTARCOLS = ["nanotime_1", "time2_1", "id_1"]
DATAM2SSTARCOLS = ["nanotime_2", "time2_2", "id_2"]
MCM1SSTARCOLS = ["mcE_1", "id_1", "zd_1", "az_1"]
MCM2SSTARCOLS = ["mcE_2", "id_2", "zd_2", "az_2"]
CALIBCOLS = ["nanotime", "time2", "id"]
DATACALIBCOLS = ["nanotime", "time2", "id"]
MERGECOLSIDX = [0, 1, 2]
DATAMERGECOLSIDX = [0, 1, 2]
MCMERGECOLSIDX = [0, 1, -2, -1]
FILTERPREFIX = "eventfiltered_"
FILTERDIR = "filtered"
PROCESSEDDIR = "processed/"
BUGDIR = "weirdbug/"
MISSINGDIR = "missing/"


def within_pm1(len1, len2):
    if (len1 == len2) or (len1+1 == len2) or (len1-1 == len2):
        return True
    else:
        return False

def asssert_anyx_in_y(x, y):
    assert isinstance(x, str)
    if isinstance(y, (list, np.array)):
        assert np.any([x in f for f in y]), f"No {x} in {y[0:2]}..., aborting."
    elif isinstance(y, str):
        assert x in y, f"No {x} in {y}..., aborting."
    else:
        raise NotImplementedError(f"checking if {x}, type {type(x)} is in {y}, type {type(y)} is not implemented")

def split_into_telescopes(fnames):
    """Takes a list of fnames and returns two lists,
       each containing only fnames from M1 and M2"""
    if isinstance(fnames, str):
        fnames = glob_and_check(fnames)
    fnames1 = [fname for fname in fnames if "_M1_" in fname]
    fnames2 = [fname for fname in fnames if "_M2_" in fname]
    return fnames1, fnames2

def get_run(fname, full=False):
    """Returns the run number from a str fname. If full is set it returns the
       subrun part of the str as well. (_Y_ and _I_ only)"""
    basename = path.basename(fname)
    run = ""
    if "GA_" in basename:
        if "_S_" in basename:
            # example: GA_za05to35_8_1743990to1745989_S_wr_2.root
            if "_S_" in basename:
                run = basename.split("_")[3]
            # GA_M1_za35to50_8_1740993_Y_wr.root
            else:
                run = basename.split("_")[4]
    elif ("_Y_" in basename) or ("_I_" in basename):
        # 20161008_M1_05057440.005_Y_CrabNebula-W0.40+215.root or
        # eventfiltered_20161008_05057440.005_Y_CrabNebula-W0.40+215.root
        runid = basename.split("_")[2]
        if full is False:
            run = runid.split(".")[0]
        else:
            run = runid
    elif ("_S_" in basename) or ("_Q_" in basename):
        run = basename.split("_")[1]
    else:
        raise NotImplementedError(f"Wrong filename: {fname}, runname retrieval is undefined")
    return run


class EventRemover():
    """Removes all events from calibfeatures that arent in starmergecols
       (that is post image cleaning), resulting in new calibrated files with only the events remaining
       after after the image cleaning. These can then be preprocessed and classified."""

    def __init__(self, tofiltercalibglob, starglob, superstarmcolglob, njobs=2, invert=False):
        self.validfullruns = list(set(self._get_valid_fullruns(starglob)))
        self.superstarmcolfnames = self._get_fnames(superstarmcolglob,
                                                    filetype="_S_")
        self.tofiltercalibfnames = self._get_fnames(tofiltercalibglob,
                                                    fullrunfilter=True)
        self.assert_startup_fnames()
        self.invert = False #hardcoded for now not implemented yet
        self.outfilenames = []
        self.setup_dirs(tofiltercalibglob)
        self.njobs = njobs
        if "GA_" in self.tofiltercalibfnames[0]:
            self.mc = True
        else:
            self.mc = False
        self.set_mcols(self.mc)

    def setup_dirs(self, tofiltercalibglob):
        olddir = path.dirname(tofiltercalibglob)
        self.processeddir = path.join(olddir, PROCESSEDDIR)
        os.makedirs(self.processeddir, exist_ok=True)
        self.bugdir = path.join(olddir, BUGDIR)
        os.makedirs(path.join(olddir, BUGDIR), exist_ok=True)
        os.makedirs(path.join(olddir, MISSINGDIR), exist_ok=True)
        os.makedirs(path.join(olddir, FILTERDIR), exist_ok=True)

    def set_mcols(self, mc):
        if mc is False:
            self.m1sscols = DATAM1SSTARCOLS
            self.m2sscols = DATAM2SSTARCOLS
            self.mcolidx = DATAMERGECOLSIDX
        else:
            self.m1sscols = MCM1SSTARCOLS
            self.m2sscols = MCM2SSTARCOLS
            self.mcolidx = MCMERGECOLSIDX

    def assert_startup_fnames(self):
        assert np.any(["_M1_" in f for f in self.tofiltercalibfnames]), \
            "No M1 files in tofiltercalibfnames, aborting."
        assert np.any(["_M2_" in f for f in self.tofiltercalibfnames]), \
            "No M2 files in tofiltercalibfnames, aborting."
        assert not np.any([".root" in f for f in self.tofiltercalibfnames]), \
            "Change the tofiltercalibglob, these are .root but are supposed to be csv files."

    def single_remove_events(self, ssmcoldf, ssmcolfname, tofiltercalibfnames, sscols):
        currentrun = get_run(ssmcolfname)
        tofiltercalibdf, oldname = self._merge_subruns(tofiltercalibfnames,
                                                       currentrun, getoldname=True)
        if len(tofiltercalibdf1) == 0:
            logger.warning(f"calibfiles already processed, skipping {path.basename(ssmcolfname)} current run: {currentrun} \n")
            return False
        filtermask = self.get_filtermask(tofiltercalibdf.iloc[:, self.mcolidx], ssmcoldf[sscols])
        if self.invert is False:
            filtermask, tofiltercalibdf = self.fix_idiosyncracies(filtermask, tofiltercalibdf, ssmcoldf[sscols])
        newfname = self._get_newfilename(oldname=oldname)
        nameiter.set_description(f"Saving {path.basename(newfname)}")
        # if not assert_single_fmask_and_df(filtermask, tofiltercalibdf, ssmcoldf):
            # return False
        tofiltercalibdf.iloc[filtermask].to_csv(newfname, header=False, index=False)
        del tofiltercalibdf
        return newfname

    def _move_processed_files(self, oldnames, outdir=None):
        if isinstance(oldnames, str):
            oldnames = [oldnames]
        if outdir is None:
            outdir = self.processeddir
        for oldname in oldnames:
            oldbasename = path.basename(oldname)
            newname = path.join(outdir, oldbasename)
            os.rename(oldname, newname)

    def _get_fnames(self, glob, filetype=None, fullrunfilter=False):
        filetypes = ["_Y_", "_I_", "_S_"]
        if isinstance(glob, str):
            fnames = glob_and_check(glob)
        elif isinstance(glob, list):
            fnames = glob
        else:
            raise NotImplementedError(f"Argument not implemented. type(glob): {type(glob)}")
        if filetype is None:
            for ftype in filetypes:
                if ftype in fnames[0]:
                    filetype = ftype
        if filetype not in filetypes:
            raise NotImplementedError(f"Wrong filetpye: {filetype}, glob: {glob} filename retrieval is undefined")
        allfit = np.all([filetype in f for f in fnames])
        assert allfit, \
               f"Not all files are from the same processing step, aborting. Filetype: {filetype}, glob: {glob}"

        if fullrunfilter:
            fnamesnew = [f for f in fnames
                      if get_run(f, full=True) in self.validfullruns]
            assert fnamesnew, ("fnames after fullrunfilter empty, aborting."
                   f"{path.dirname(fnames[0])} oldfnames: {[path.basename(f) for f in fnames]}")
            fnames = fnamesnew

        return sorted(fnames)

    def _get_valid_fullruns(self, starglob):
        """Takes the star filenames corresponding to the superstar files used for filtering and
           creates a list of valid runs+subruns that are to be used at the calibrated level"""
        starfnames = self._get_fnames(starglob, filetype="_I_")
        validfullruns = set([get_run(f, full=True) for f in starfnames])
        return sorted(list(validfullruns))


    def get_relevant_fullruns(self, run):
        return [f for f in self.validfullruns if run in f]


    def assert_filtermask_and_dfs(self, fmask1, fmask2, calibdf1, calibdf2, ssmcoldf):
        status = True
        # more detail inside of get_filtermask
        if (fmask1 is None) or (fmask2 is None):
            status = False
        elif (len(fmask1) == 0) or (len(fmask2) == 0):
            status = False
        elif np.sum(fmask1) != np.sum(fmask2):
            logger.warning("Number of filtered events are not the same for each telescope, skipping."
             f"#M1: {np.sum(fmask1)}, #M2: {np.sum(fmask2)}")
            status = False
        else:
            status = True
        if status is True:
            try:
                assert within_pm1(len(calibdf1.iloc[fmask1]), len(ssmcoldf)),\
                    ("len of M1 is not within +-1 of ssmcoldf, aborting.\n"
                     f"len calibdf: {len(calibdf1.iloc[fmask1])}, len ssmcoldf: {len(ssmcoldf)}")
                assert within_pm1(len(calibdf2.iloc[fmask2]), len(ssmcoldf)),\
                    ("len of M2 is not within +-1 of ssmcoldf, aborting.\n"
                     f"len calibdf: {len(calibdf2.iloc[fmask2])}, len ssmcoldf: {len(ssmcoldf)}")
            except IndexError:
                logger.critical("\n"
                  f"len ssmcoldf: {len(ssmcoldf)}\n"
                  f"true sum filtermask1: {np.sum(fmask1)}\n"
                  f"len fmask1: {len(fmask1)}\n"
                  f"len calibdf1: {len(calibdf1)}\n"
                  f"true sum filtermask2: {np.sum(fmask2)}\n"
                  f"len fmask2: {len(fmask2)}\n"
                  f"len calibdf2: {len(calibdf2)}\n")
                raise
        return status

    def fix_idiosyncracies(self, filtermask, tofiltercalibdf, ssmcoldf):
        calibmcols = tofiltercalibdf.iloc[:, self.mcolidx].values
        if np.sum(filtermask) == 0:
            logger.warning("filtermask is all false, skipping")
            return None, None
        # handle duplicates at the end of superstar file
        if len(calibmcols[filtermask]) == (len(ssmcoldf)-1):
            if np.array_equal(ssmcoldf.values[-1], ssmcoldf.values[-2]):
                calibmcols = np.vstack([calibmcols, ssmcoldf.values[-1].reshape(1, 3)])
                lastcalibrow = tofiltercalibdf.iloc[filtermask].iloc[-1]
                filtermask = np.append(filtermask, True)
                tofiltercalibdf = tofiltercalibdf.append(lastcalibrow)
                if np.array_equal(lastcalibrow.values, ssmcoldf.values[-1]):
                    return filtermask, tofiltercalibdf
                logger.warning("idk what is happening")
        # when did this happen? when the last calibrows pre or post filtermask were the same?
        # probably post filtermask, or else the condition for equality after filtermask wouldnt have worked
        elif len(calibmcols[filtermask]) == (len(ssmcoldf)+1):
            lastcalibrowequal = np.array_equal(calibmcols[-1], calibmcols[-2])
            lastssrowequal = np.array_equal(ssmcoldf.iloc[-1], ssmcoldf.iloc[-2])
            lastssandcalibrowequal = np.array_equal(calibmcols[-1], ssmcoldf.iloc[-1])
            if lastcalibrowequal and lastssandcalibrowequal and not lastssrowequal:
                filtermask[-1] = False
        if not np.array_equal(calibmcols[filtermask][:len(ssmcoldf)], ssmcoldf.values):
            logger.warning("calib and superstar still not equal, idk what is going on, skipping.")
            logger.warning(f"len fmask: {len(filtermask)}\n"
                  f"true sum filtermask: {np.sum(filtermask)}\n"
                  f"len calibmcols: {len(calibmcols)}\n"
                  f"len ssmcols: {len(ssmcoldf)}\n")
            # raise ArithmeticError("raising error for now, start bugfixing")
            return None, None
        else:
            return filtermask, tofiltercalibdf


    def assert_single_fmask_and_df(self, filtermask, tofiltercalibdf, ssmcoldf):
        pass

    def remove_events(self):
        """Filters all tofiltercalib files with the starfiles provided and throws out
           not matching subruns, merges the remaining subruns to runlevel.
           Same for the mergecolcalibfiles. Then compares each run of mcolcalibfiles
           to the superstarmergecolfile of the same run and throws out all events in
           tofiltercalib based on the filtermask. Finally writes to disk."""
        tofiltercalibfnames1, tofiltercalibfnames2 = split_into_telescopes(self.tofiltercalibfnames)

        nameiter = tqdm(self.superstarmcolfnames)
        for ssmcolfname in nameiter:
            nameiter.set_description(f"Processing {path.basename(ssmcolfname)}")
            currentrun = get_run(ssmcolfname)
            ssmcoldf = pd.read_csv(ssmcolfname, index_col=False)
            # newfname1 = self.single_remove_events(ssmcoldf, ssmcolfname, tofiltercalibfnames1, self.m1sscols)
            # newfname2 = self.single_remove_events(ssmcoldf, ssmcolfname, tofiltercalibfnames2, self.m2sscols)
                # if self.assert_filtermask_and_dfs(filtermask1, filtermask2, tofiltercalibdf1, tofiltercalibdf2, ssmcoldf) is False:
                    # self._move_processed_files(self._handle_fnames_for_merging(tofiltercalibfnames1, currentrun), self.bugdir)
                    # self._move_processed_files(self._handle_fnames_for_merging(tofiltercalibfnames2, currentrun), self.bugdir)
                    # os.remove(newfname1)
                    # os.remove(newfname2)
                    # continue
            tofiltercalibdf1, oldname1 = self._merge_subruns(tofiltercalibfnames1,
                                                             currentrun, getoldname=True)
            tofiltercalibdf2, oldname2 = self._merge_subruns(tofiltercalibfnames2,
                                                             currentrun, getoldname=True)
            if len(tofiltercalibdf1) == 0:
                logger.warning(f"calibfiles already processed, skipping {path.basename(ssmcolfname)} current run: {currentrun} \n")
                continue

            filtermask1 = self.get_filtermask(tofiltercalibdf1.iloc[:, self.mcolidx], ssmcoldf[self.m1sscols])
            filtermask2 = self.get_filtermask(tofiltercalibdf2.iloc[:, self.mcolidx], ssmcoldf[self.m2sscols])
            if self.invert is False:
                filtermask1, tofiltercalibdf1 = self.fix_idiosyncracies(filtermask1, tofiltercalibdf1, ssmcoldf[self.m1sscols])
                filtermask2, tofiltercalibdf2 = self.fix_idiosyncracies(filtermask2, tofiltercalibdf2, ssmcoldf[self.m2sscols])

                if self.assert_filtermask_and_dfs(filtermask1, filtermask2, tofiltercalibdf1, tofiltercalibdf2, ssmcoldf) is False:
                    self._move_processed_files(self._handle_fnames_for_merging(tofiltercalibfnames1, currentrun), self.bugdir)
                    self._move_processed_files(self._handle_fnames_for_merging(tofiltercalibfnames2, currentrun), self.bugdir)
                    continue

            newfname1 = self._get_newfilename(oldname=oldname1)
            newfname2 = self._get_newfilename(oldname=oldname2)

            nameiter.set_description(f"Saving {path.basename(newfname1)}")
            tofiltercalibdf1.iloc[filtermask1].to_csv(newfname1, header=False, index=False)
            nameiter.set_description(f"Saving {path.basename(newfname2)}")
            tofiltercalibdf2.iloc[filtermask2].to_csv(newfname2, header=False, index=False)
            self.outfilenames.extend([newfname1, newfname2])
            # self._move_processed_files(self._handle_fnames_for_merging(tofiltercalibfnames1, currentrun))
            # self._move_processed_files(self._handle_fnames_for_merging(tofiltercalibfnames2, currentrun))


    def parallel_remove_events(self):
        tofiltercalibfnames1, tofiltercalibfnames2 = split_into_telescopes(self.tofiltercalibfnames)
        # this doesnt work, the called function needs to be a regular function that takes the class as an argument and then does the computation on the passed class
        newfnames = Parallel(n_jobs=self.njobs)\
            (delayed(self.single_remove_events)\
             (ssmcolfname, tofiltercalibfnames1, tofiltercalibfnames2)
             for ssmcolfname in self.superstarmcolfnames)
        self.outfilenames.extend(newfnames)

    def single_remove_events(self):
        pass

    def get_filtermask(self, calibmcoldf, ssmcoldf):
        """Takes the mergecols extracted from the superstar file and compares the ones from the
        calib files to it. When calib event doesnt match superstar event, the filtermask is set
        to 0. Once ss=calib for the current row the next ss event is looked at
        till the end of the file. In the end all events not in the ss file are
        removed from the calib file. Calibdf has to contain the same run as the ssmcoldf.
        Maybe do a filename assertion."""
        if isinstance(calibmcoldf, pd.DataFrame):
            calibmcoldf = calibmcoldf.values.astype(float)
        else:
            calibmcoldf = calibmcoldf.astype(float)
        if isinstance(ssmcoldf, pd.DataFrame):
            ssmcoldf = ssmcoldf.values.astype(float)
        else:
            ssmcoldf = ssmcoldf.astype(float)

        sstarindex = 0
        filtermask = np.ones(len(calibmcoldf), dtype=bool)
        ssmcollen = len(ssmcoldf)
        missingvalues = len(calibmcoldf)-ssmcollen
        assert missingvalues >= 0, f"Number of missing values is supposed to be positive no.: {missingvalues}"
        for calibindex, calibrow in enumerate(calibmcoldf):
            if sstarindex == ssmcollen:
                filtermask[calibindex:] = False
                break
            if not np.array_equal(ssmcoldf[sstarindex], calibrow):
                if self.invert is True:
                    filtermask[calibindex] = True
                else:
                    filtermask[calibindex] = False
            else:
                sstarindex += 1
        # set as assertion once bug is fixed
        if self.invert is False:
            if not np.array_equal(calibmcoldf[filtermask], ssmcoldf):#, \
                logger.info("filtermask doesnt make equal, trying to fix")
                # raise ArithmeticError("Start debugging")
            # needs a fix for when there is a duplicate in the middle/end of the ssmcoldf (results in the filtered calibdf being too short)
            # sometimes the ss file has a duplicate event at the end which is not duplicated in the calibrated file ?!?!?
            # assert np.sum(filtermask) != 0, "something went wrong len of filtermask is 0 start debugging"

        return filtermask


    def get_fnames_for_samerun(self, fnames):
        samerun = np.all([get_run(f) in get_run(fnames[0])
                          for f in fnames])
        assert samerun, "files are not all from the same run, aborting"
        return samerun


    def _handle_fnames_for_merging(self, fnames, run=None):
        assert np.all(["_Y_" in f for f in fnames]) or np.all(["_I_" in f for f in fnames]),\
            "not all files are calibfiles, aborting"
        allM1 = np.all(["_M1_" in f for f in fnames])
        allM2 = np.all(["_M2_" in f for f in fnames])
        assert (allM1 or allM2), "files are not all from the same telescope, aborting"
        assert fnames, "List of fnames is empty aborting"
        if run is None:
            samerun = self.get_fnames_for_samerun(fnames)
        else:
            olddir = path.dirname(fnames[0])
            newfnames = [path.join(self.processeddir, path.basename(f)) for f in fnames]
            newfnames = [f for f in newfnames if get_run(f) == run]
            fnames = [f for f in fnames if run in path.basename(f)]
            if not fnames:
                if not newfnames == 0:
                    f"No star or calibfiles for run {run}"
                return []

        assert len(fnames) != 0, "len fnames is zero that shouldnt happen, aborting"
        if len(fnames) < len(self.get_relevant_fullruns(run)):
            logger.warning("missing files for current run, skipping")
            return []
        fnames =  sorted(list(set(fnames)))
        return fnames


    def _merge_subruns(self, fnames, run=None, getoldname=False, mergecolsonly=False):
        """Takes multiple fnames (subruns) from the same run and merges them in a DF. Returns the DF"""
        fnames = self._handle_fnames_for_merging(fnames, run)
        if not fnames:
            if getoldname is True:
                return [], ""
            else:
                return []

        if "mergecols" in fnames[0]:
            if len(fnames) == 1:
                rundf = pd.read_csv(fnames[0], index_col=False)
            else:
                rundf = pd.DataFrame(np.vstack([pd.read_csv(f, index_col=False).values for f in fnames]))
        else:
            # depends on the fact the merge cols extract in globRoot2csv.C sets headers, if that is changed set header=None here too
            if len(fnames) == 1:
                rundf = pd.read_csv(fnames[0], header=None, index_col=False)
            else:
                rundf = pd.DataFrame(np.vstack([pd.read_csv(f, header=None, index_col=False).values for f in fnames]))

        colnames = pd.read_csv(fnames[0]).columns
        rundf.columns = colnames
        if getoldname is True:
            oldname = self.get_oldname(fnames[0])
            return rundf, oldname
        else:
            return rundf


    @staticmethod
    def get_oldname(fname):
        """Takes fname and removes the subrun from it, leaving only the name with run identifier."""
        assert isinstance(fname, str), f"fname is not a string, aborting. fname: {fname}"
        dirname = path.dirname(fname)
        oldbasename = path.basename(fname)
        if "GA_" in oldbasename:
            return fname
        elif ("_Y_" in oldbasename) or ("_I_" in oldbasename):
            split = oldbasename.split("_")
            split[2] = split[2].split(".")[0]
            newbasename = "_".join(split)
            return path.join(dirname, newbasename)
        elif "_S_" in fname:
            return fname
        else:
            raise NotImplementedError(f"filetype not supported. fname: {fname}")


    @staticmethod
    def _get_newfilename(oldname):
        """Takes the old path and returns it with the filename prepended with `eventfiltered_`"""
            # example filename: 20160929_M1_05057144.015_I_CrabNebula-W0.40+215_mergecols.csv
            # removes subrun (bc files are merged to runlevel) and prepends eventfiltered
        dirname = path.dirname(oldname)
        newdirname = path.join(dirname, FILTERDIR)
        oldbasename = path.basename(oldname)
        if "GA_" in oldbasename:
            if ("_Y_" in oldbasename) or ("_I_" in oldbasename):
                newbasename = FILTERPREFIX + oldbasename
        elif ("_Y_" in oldbasename) or ("_I_" in oldbasename):
            splitname = oldbasename.split("_")
            splitname[2] = splitname[2].split(".")[0]
            newbasename = FILTERPREFIX + "_".join(splitname)
        newfilename = path.join(newdirname, newbasename)
        return newfilename


    def __call__(self):
        self.remove_events()


    def invert_selection(self):
        """Selects and writes a new csv file with all events that do not conform to the filtermask
        (which means they did get filtered out from calibrated->star or whatever step)"""
        pass


def process_events(**args):
    remover = EventRemover(**args)
    remover.remove_events()


@click.command()
@click.option('--tofiltercalibglob', "-fcg", default="./csv/*_Y_*.csv", type=click.Path(),
              help=('Dir from which to read the calibrated (subrun level) csv files'
                    'which are going to get filtered.'))
@click.option('--starglob', "-sg", default="./star/*_M1_*I_*.root", type=click.Path(),
              help=('Glob to the star files generated after image cleaning can be root or whatever.'
                    'Only the subrun names are important for filtering.'))
@click.option('--superstarmcolglob', "-ssg", default="./superstar/mergecols/*_S_*.csv", type=click.Path(),
              help=('CSV Files from which to read the features (mergecols) for filtering events.'
                    'Generated after merging (superstar) and extracting from root.'))
@click.option('--njobs', "-n", default=1, type=int,
              help='Number of jobs for processing.')
@click.option('--invert', "-i", default=False, type=bool,
              help='Whether to do a inverse filter (only leave events that get thrown out during star cleaning).')
def main(**args):
    try:
        if args["njobs"] == 1:
            process_events(**args)
        else:
            raise NotImplementedError("Multiprocessing needs to be implemented still.")
            # func that splits the globstirngs into new globstrings (should be in process_star.py or somewhere else)
            # newcalibglobs = split_glob(tofiltercalibglob)
            # newstarglobs = split_glob(starglob)
            # newsuperstarglobs = split_glob(superstarmcolglob)
            # globzip = zip(newcalibglobs, newstarglobs, newsuperstarglobs)
            # Parallel(n_jobs=njobs)(delayed(process_events)(ncalibglob, nstarglob, nsuperstarglob, njobs, invert)
            #                        for ncalibglob, nstarglob, nsuperstarglob in globzip)
    except:
        import pdb, traceback, sys
        type, value, tb = sys.exc_info()
        traceback.print_exc()
        pdb.post_mortem(tb)


if __name__ == "__main__":
    main()

from os import path
import os
import logging

import pandas as pd
import click
import numpy as np
from tqdm import tqdm
from joblib import Parallel, delayed
from dataprepro.csv2hdf5 import glob_and_check

logger = logging.getLogger(__name__)

DATAM1SSTARCOLS = ["nanotime_1", "time2_1", "id_1"]
DATAM2SSTARCOLS = ["nanotime_2", "time2_2", "id_2"]
MCM1SSTARCOLS = ["mcE_1", "id_1", "zd_1", "az_1"]
MCM2SSTARCOLS = ["mcE_2", "id_2", "zd_2", "az_2"]
CALIBCOLS = ["nanotime", "time2", "id"]
DATACALIBCOLS = ["nanotime", "time2", "id"]
MERGECOLSIDX = [0, 1, 2]
DATAMERGECOLSIDX = [0, 1, 2]
MCMERGECOLSIDX = [0, 1, -2, -1]
FILTERPREFIX = "eventfiltered_"
FILTERDIR = "filtered"
PROCESSEDDIR = "processed/"
BUGDIR = "weirdbug/"
MISSINGDIR = "missing/"


def within_pm1(len1, len2):
    if (len1 == len2) or (len1+1 == len2) or (len1-1 == len2):
        return True
    else:
        return False

def asssert_anyx_in_y(x, y):
    assert isinstance(x, str)
    if isinstance(y, (list, np.array)):
        assert np.any([x in f for f in y]), f"No {x} in {y[0:2]}..., aborting."
    elif isinstance(y, str):
        assert x in y, f"No {x} in {y}..., aborting."
    else:
        raise NotImplementedError(f"checking if {x}, type {type(x)} is in {y}, type {type(y)} is not implemented")

def split_into_telescopes(fnames):
    """Takes a list of fnames and returns two lists,
       each containing only fnames from M1 and M2"""
    if isinstance(fnames, str):
        fnames = glob_and_check(fnames)
    fnames1 = [fname for fname in fnames if "_M1_" in fname]
    fnames2 = [fname for fname in fnames if "_M2_" in fname]
    return fnames1, fnames2

def get_run(fname, full=False):
    """Returns the run number from a str fname. If full is set it returns the
       subrun part of the str as well. (_Y_ and _I_ only)"""
    basename = path.basename(fname)
    run = ""
    if "GA_" in basename:
        if "_S_" in basename:
            # example: GA_za05to35_8_1743990to1745989_S_wr_2.root
            if "_S_" in basename:
                run = basename.split("_")[3]
            # GA_M1_za35to50_8_1740993_Y_wr.root
            else:
                run = basename.split("_")[4]
    elif ("_Y_" in basename) or ("_I_" in basename):
        # 20161008_M1_05057440.005_Y_CrabNebula-W0.40+215.root or
        # eventfiltered_20161008_05057440.005_Y_CrabNebula-W0.40+215.root
        runid = basename.split("_")[2]
        if full is False:
            run = runid.split(".")[0]
        else:
            run = runid
    elif ("_S_" in basename) or ("_Q_" in basename):
        run = basename.split("_")[1]
    else:
        raise NotImplementedError(f"Wrong filename: {fname}, runname retrieval is undefined")
    return run


class EventRemover():
    """Removes all events from calibfeatures that arent in starmergecols
       (that is post image cleaning), resulting in new calibrated files with only the events remaining
       after after the image cleaning. These can then be preprocessed and classified."""

    def __init__(self, tofiltercalibglob, starglob, superstarmcolglob, njobs=2, invert=False):
        self.validfullruns = list(set(self._get_valid_fullruns(starglob)))
        self.superstarmcolfnames = self._get_fnames(superstarmcolglob,
                                                    filetype="_S_")
        self.tofiltercalibfnames = self._get_fnames(tofiltercalibglob,
                                                    fullrunfilter=True)
        self.assert_startup_fnames()
        self.invert = False #hardcoded for now not implemented yet
        self.outfilenames = []
        self.setup_dirs(tofiltercalibglob)
        self.njobs = njobs
        if "GA_" in self.tofiltercalibfnames[0]:
            self.mc = True
        else:
            self.mc = False
        self.set_mcols(self.mc)

    def setup_dirs(self, tofiltercalibglob):
        olddir = path.dirname(tofiltercalibglob)
        self.processeddir = path.join(olddir, PROCESSEDDIR)
        os.makedirs(self.processeddir, exist_ok=True)
        self.bugdir = path.join(olddir, BUGDIR)
        os.makedirs(path.join(olddir, BUGDIR), exist_ok=True)
        os.makedirs(path.join(olddir, MISSINGDIR), exist_ok=True)
        os.makedirs(path.join(olddir, FILTERDIR), exist_ok=True)

    def set_mcols(self, mc):
        if mc is False:
            self.m1sscols = DATAM1SSTARCOLS
            self.m2sscols = DATAM2SSTARCOLS
            self.mcolidx = DATAMERGECOLSIDX
        else:
            self.m1sscols = MCM1SSTARCOLS
            self.m2sscols = MCM2SSTARCOLS
            self.mcolidx = MCMERGECOLSIDX

    def assert_startup_fnames(self):
        assert np.any(["_M1_" in f for f in self.tofiltercalibfnames]), \
            "No M1 files in tofiltercalibfnames, aborting."
        assert np.any(["_M2_" in f for f in self.tofiltercalibfnames]), \
            "No M2 files in tofiltercalibfnames, aborting."
        assert not np.any([".root" in f for f in self.tofiltercalibfnames]), \
            "Change the tofiltercalibglob, these are .root but are supposed to be csv files."

    def single_remove_events(self, ssmcoldf, ssmcolfname, tofiltercalibfnames, sscols):
        currentrun = get_run(ssmcolfname)
        tofiltercalibdf, oldname = self._merge_subruns(tofiltercalibfnames,
                                                       currentrun, getoldname=True)
        if len(tofiltercalibdf1) == 0:
            logger.warning(f"calibfiles already processed, skipping {path.basename(ssmcolfname)} current run: {currentrun} \n")
            return False
        filtermask = self.get_filtermask(tofiltercalibdf.iloc[:, self.mcolidx], ssmcoldf[sscols])
        if self.invert is False:
            filtermask, tofiltercalibdf = self.fix_idiosyncracies(filtermask, tofiltercalibdf, ssmcoldf[sscols])
        newfname = self._get_newfilename(oldname=oldname)
        nameiter.set_description(f"Saving {path.basename(newfname)}")
        # if not assert_single_fmask_and_df(filtermask, tofiltercalibdf, ssmcoldf):
            # return False
        tofiltercalibdf.iloc[filtermask].to_csv(newfname, header=False, index=False)
        del tofiltercalibdf
        return newfname

    def _move_processed_files(self, oldnames, outdir=None):
        if isinstance(oldnames, str):
            oldnames = [oldnames]
        if outdir is None:
            outdir = self.processeddir
        for oldname in oldnames:
            oldbasename = path.basename(oldname)
            newname = path.join(outdir, oldbasename)
            os.rename(oldname, newname)

    def _get_fnames(self, glob, filetype=None, fullrunfilter=False):
        filetypes = ["_Y_", "_I_", "_S_"]
        if isinstance(glob, str):
            fnames = glob_and_check(glob)
        elif isinstance(glob, list):
            fnames = glob
        else:
            raise NotImplementedError(f"Argument not implemented. type(glob): {type(glob)}")
        if filetype is None:
            for ftype in filetypes:
                if ftype in fnames[0]:
                    filetype = ftype
        if filetype not in filetypes:
            raise NotImplementedError(f"Wrong filetpye: {filetype}, glob: {glob} filename retrieval is undefined")
        allfit = np.all([filetype in f for f in fnames])
        assert allfit, \
               f"Not all files are from the same processing step, aborting. Filetype: {filetype}, glob: {glob}"

        if fullrunfilter:
            fnamesnew = [f for f in fnames
                      if get_run(f, full=True) in self.validfullruns]
            assert fnamesnew, ("fnames after fullrunfilter empty, aborting."
                   f"{path.dirname(fnames[0])} oldfnames: {[path.basename(f) for f in fnames]}")
            fnames = fnamesnew

        return sorted(fnames)

    def _get_valid_fullruns(self, starglob):
        """Takes the star filenames corresponding to the superstar files used for filtering and
           creates a list of valid runs+subruns that are to be used at the calibrated level"""
        starfnames = self._get_fnames(starglob, filetype="_I_")
        validfullruns = set([get_run(f, full=True) for f in starfnames])
        return sorted(list(validfullruns))


    def get_relevant_fullruns(self, run):
        return [f for f in self.validfullruns if run in f]


    def assert_filtermask_and_dfs(self, fmask1, fmask2, calibdf1, calibdf2, ssmcoldf):
        status = True
        # more detail inside of get_filtermask
        if (fmask1 is None) or (fmask2 is None):
            status = False
        elif (len(fmask1) == 0) or (len(fmask2) == 0):
            status = False
        elif np.sum(fmask1) != np.sum(fmask2):
            logger.warning("Number of filtered events are not the same for each telescope, skipping."
             f"#M1: {np.sum(fmask1)}, #M2: {np.sum(fmask2)}")
            status = False
        else:
            status = True
        if status is True:
            try:
                assert within_pm1(len(calibdf1.iloc[fmask1]), len(ssmcoldf)),\
                    ("len of M1 is not within +-1 of ssmcoldf, aborting.\n"
                     f"len calibdf: {len(calibdf1.iloc[fmask1])}, len ssmcoldf: {len(ssmcoldf)}")
                assert within_pm1(len(calibdf2.iloc[fmask2]), len(ssmcoldf)),\
                    ("len of M2 is not within +-1 of ssmcoldf, aborting.\n"
                     f"len calibdf: {len(calibdf2.iloc[fmask2])}, len ssmcoldf: {len(ssmcoldf)}")
            except IndexError:
                logger.critical("\n"
                  f"len ssmcoldf: {len(ssmcoldf)}\n"
                  f"true sum filtermask1: {np.sum(fmask1)}\n"
                  f"len fmask1: {len(fmask1)}\n"
                  f"len calibdf1: {len(calibdf1)}\n"
                  f"true sum filtermask2: {np.sum(fmask2)}\n"
                  f"len fmask2: {len(fmask2)}\n"
                  f"len calibdf2: {len(calibdf2)}\n")
                raise
        return status

    def fix_idiosyncracies(self, filtermask, tofiltercalibdf, ssmcoldf):
        calibmcols = tofiltercalibdf.iloc[:, self.mcolidx].values
        if np.sum(filtermask) == 0:
            logger.warning("filtermask is all false, skipping")
            return None, None
        # handle duplicates at the end of superstar file
        if len(calibmcols[filtermask]) == (len(ssmcoldf)-1):
            if np.array_equal(ssmcoldf.values[-1], ssmcoldf.values[-2]):
                calibmcols = np.vstack([calibmcols, ssmcoldf.values[-1].reshape(1, 3)])
                lastcalibrow = tofiltercalibdf.iloc[filtermask].iloc[-1]
                filtermask = np.append(filtermask, True)
                tofiltercalibdf = tofiltercalibdf.append(lastcalibrow)
                if np.array_equal(lastcalibrow.values, ssmcoldf.values[-1]):
                    return filtermask, tofiltercalibdf
                logger.warning("idk what is happening")
        # when did this happen? when the last calibrows pre or post filtermask were the same?
        # probably post filtermask, or else the condition for equality after filtermask wouldnt have worked
        elif len(calibmcols[filtermask]) == (len(ssmcoldf)+1):
            lastcalibrowequal = np.array_equal(calibmcols[-1], calibmcols[-2])
            lastssrowequal = np.array_equal(ssmcoldf.iloc[-1], ssmcoldf.iloc[-2])
            lastssandcalibrowequal = np.array_equal(calibmcols[-1], ssmcoldf.iloc[-1])
            if lastcalibrowequal and lastssandcalibrowequal and not lastssrowequal:
                filtermask[-1] = False
        if not np.array_equal(calibmcols[filtermask][:len(ssmcoldf)], ssmcoldf.values):
            logger.warning("calib and superstar still not equal, idk what is going on, skipping.")
            logger.warning(f"len fmask: {len(filtermask)}\n"
                  f"true sum filtermask: {np.sum(filtermask)}\n"
                  f"len calibmcols: {len(calibmcols)}\n"
                  f"len ssmcols: {len(ssmcoldf)}\n")
            # raise ArithmeticError("raising error for now, start bugfixing")
            return None, None
        else:
            return filtermask, tofiltercalibdf


    def assert_single_fmask_and_df(self, filtermask, tofiltercalibdf, ssmcoldf):
        pass

    def remove_events(self):
        """Filters all tofiltercalib files with the starfiles provided and throws out
           not matching subruns, merges the remaining subruns to runlevel.
           Same for the mergecolcalibfiles. Then compares each run of mcolcalibfiles
           to the superstarmergecolfile of the same run and throws out all events in
           tofiltercalib based on the filtermask. Finally writes to disk."""
        tofiltercalibfnames1, tofiltercalibfnames2 = split_into_telescopes(self.tofiltercalibfnames)

        nameiter = tqdm(self.superstarmcolfnames)
        for ssmcolfname in nameiter:
            nameiter.set_description(f"Processing {path.basename(ssmcolfname)}")
            currentrun = get_run(ssmcolfname)
            ssmcoldf = pd.read_csv(ssmcolfname, index_col=False)
            # newfname1 = self.single_remove_events(ssmcoldf, ssmcolfname, tofiltercalibfnames1, self.m1sscols)
            # newfname2 = self.single_remove_events(ssmcoldf, ssmcolfname, tofiltercalibfnames2, self.m2sscols)
                # if self.assert_filtermask_and_dfs(filtermask1, filtermask2, tofiltercalibdf1, tofiltercalibdf2, ssmcoldf) is False:
                    # self._move_processed_files(self._handle_fnames_for_merging(tofiltercalibfnames1, currentrun), self.bugdir)
                    # self._move_processed_files(self._handle_fnames_for_merging(tofiltercalibfnames2, currentrun), self.bugdir)
                    # os.remove(newfname1)
                    # os.remove(newfname2)
                    # continue
            tofiltercalibdf1, oldname1 = self._merge_subruns(tofiltercalibfnames1,
                                                             currentrun, getoldname=True)
            tofiltercalibdf2, oldname2 = self._merge_subruns(tofiltercalibfnames2,
                                                             currentrun, getoldname=True)
            if len(tofiltercalibdf1) == 0:
                logger.warning(f"calibfiles already processed, skipping {path.basename(ssmcolfname)} current run: {currentrun} \n")
                continue

            filtermask1 = self.get_filtermask(tofiltercalibdf1.iloc[:, self.mcolidx], ssmcoldf[self.m1sscols])
            filtermask2 = self.get_filtermask(tofiltercalibdf2.iloc[:, self.mcolidx], ssmcoldf[self.m2sscols])
            if self.invert is False:
                filtermask1, tofiltercalibdf1 = self.fix_idiosyncracies(filtermask1, tofiltercalibdf1, ssmcoldf[self.m1sscols])
                filtermask2, tofiltercalibdf2 = self.fix_idiosyncracies(filtermask2, tofiltercalibdf2, ssmcoldf[self.m2sscols])

                if self.assert_filtermask_and_dfs(filtermask1, filtermask2, tofiltercalibdf1, tofiltercalibdf2, ssmcoldf) is False:
                    self._move_processed_files(self._handle_fnames_for_merging(tofiltercalibfnames1, currentrun), self.bugdir)
                    self._move_processed_files(self._handle_fnames_for_merging(tofiltercalibfnames2, currentrun), self.bugdir)
                    continue

            newfname1 = self._get_newfilename(oldname=oldname1)
            newfname2 = self._get_newfilename(oldname=oldname2)

            nameiter.set_description(f"Saving {path.basename(newfname1)}")
            tofiltercalibdf1.iloc[filtermask1].to_csv(newfname1, header=False, index=False)
            nameiter.set_description(f"Saving {path.basename(newfname2)}")
            tofiltercalibdf2.iloc[filtermask2].to_csv(newfname2, header=False, index=False)
            self.outfilenames.extend([newfname1, newfname2])
            # self._move_processed_files(self._handle_fnames_for_merging(tofiltercalibfnames1, currentrun))
            # self._move_processed_files(self._handle_fnames_for_merging(tofiltercalibfnames2, currentrun))


    def parallel_remove_events(self):
        tofiltercalibfnames1, tofiltercalibfnames2 = split_into_telescopes(self.tofiltercalibfnames)
        # this doesnt work, the called function needs to be a regular function that takes the class as an argument and then does the computation on the passed class
        newfnames = Parallel(n_jobs=self.njobs)\
            (delayed(self.single_remove_events)\
             (ssmcolfname, tofiltercalibfnames1, tofiltercalibfnames2)
             for ssmcolfname in self.superstarmcolfnames)
        self.outfilenames.extend(newfnames)

    def single_remove_events(self):
        pass

    def get_filtermask(self, calibmcoldf, ssmcoldf):
        """Takes the mergecols extracted from the superstar file and compares the ones from the
        calib files to it. When calib event doesnt match superstar event, the filtermask is set
        to 0. Once ss=calib for the current row the next ss event is looked at
        till the end of the file. In the end all events not in the ss file are
        removed from the calib file. Calibdf has to contain the same run as the ssmcoldf.
        Maybe do a filename assertion."""
        if isinstance(calibmcoldf, pd.DataFrame):
            calibmcoldf = calibmcoldf.values.astype(float)
        else:
            calibmcoldf = calibmcoldf.astype(float)
        if isinstance(ssmcoldf, pd.DataFrame):
            ssmcoldf = ssmcoldf.values.astype(float)
        else:
            ssmcoldf = ssmcoldf.astype(float)

        sstarindex = 0
        filtermask = np.ones(len(calibmcoldf), dtype=bool)
        ssmcollen = len(ssmcoldf)
        missingvalues = len(calibmcoldf)-ssmcollen
        assert missingvalues >= 0, f"Number of missing values is supposed to be positive no.: {missingvalues}"
        for calibindex, calibrow in enumerate(calibmcoldf):
            if sstarindex == ssmcollen:
                filtermask[calibindex:] = False
                break
            if not np.array_equal(ssmcoldf[sstarindex], calibrow):
                if self.invert is True:
                    filtermask[calibindex] = True
                else:
                    filtermask[calibindex] = False
            else:
                sstarindex += 1
        # set as assertion once bug is fixed
        if self.invert is False:
            if not np.array_equal(calibmcoldf[filtermask], ssmcoldf):#, \
                logger.info("filtermask doesnt make equal, trying to fix")
                # raise ArithmeticError("Start debugging")
            # needs a fix for when there is a duplicate in the middle/end of the ssmcoldf (results in the filtered calibdf being too short)
            # sometimes the ss file has a duplicate event at the end which is not duplicated in the calibrated file ?!?!?
            # assert np.sum(filtermask) != 0, "something went wrong len of filtermask is 0 start debugging"

        return filtermask


    def get_fnames_for_samerun(self, fnames):
        samerun = np.all([get_run(f) in get_run(fnames[0])
                          for f in fnames])
        assert samerun, "files are not all from the same run, aborting"
        return samerun


    def _handle_fnames_for_merging(self, fnames, run=None):
        assert np.all(["_Y_" in f for f in fnames]) or np.all(["_I_" in f for f in fnames]),\
            "not all files are calibfiles, aborting"
        allM1 = np.all(["_M1_" in f for f in fnames])
        allM2 = np.all(["_M2_" in f for f in fnames])
        assert (allM1 or allM2), "files are not all from the same telescope, aborting"
        assert fnames, "List of fnames is empty aborting"
        if run is None:
            samerun = self.get_fnames_for_samerun(fnames)
        else:
            olddir = path.dirname(fnames[0])
            newfnames = [path.join(self.processeddir, path.basename(f)) for f in fnames]
            newfnames = [f for f in newfnames if get_run(f) == run]
            fnames = [f for f in fnames if run in path.basename(f)]
            if not fnames:
                if not newfnames == 0:
                    f"No star or calibfiles for run {run}"
                return []

        assert len(fnames) != 0, "len fnames is zero that shouldnt happen, aborting"
        if len(fnames) < len(self.get_relevant_fullruns(run)):
            logger.warning("missing files for current run, skipping")
            return []
        fnames =  sorted(list(set(fnames)))
        return fnames


    def _merge_subruns(self, fnames, run=None, getoldname=False, mergecolsonly=False):
        """Takes multiple fnames (subruns) from the same run and merges them in a DF. Returns the DF"""
        fnames = self._handle_fnames_for_merging(fnames, run)
        if not fnames:
            if getoldname is True:
                return [], ""
            else:
                return []

        if "mergecols" in fnames[0]:
            if len(fnames) == 1:
                rundf = pd.read_csv(fnames[0], index_col=False)
            else:
                rundf = pd.DataFrame(np.vstack([pd.read_csv(f, index_col=False).values for f in fnames]))
        else:
            # depends on the fact the merge cols extract in globRoot2csv.C sets headers, if that is changed set header=None here too
            if len(fnames) == 1:
                rundf = pd.read_csv(fnames[0], header=None, index_col=False)
            else:
                rundf = pd.DataFrame(np.vstack([pd.read_csv(f, header=None, index_col=False).values for f in fnames]))

        colnames = pd.read_csv(fnames[0]).columns
        rundf.columns = colnames
        if getoldname is True:
            oldname = self.get_oldname(fnames[0])
            return rundf, oldname
        else:
            return rundf


    @staticmethod
    def get_oldname(fname):
        """Takes fname and removes the subrun from it, leaving only the name with run identifier."""
        assert isinstance(fname, str), f"fname is not a string, aborting. fname: {fname}"
        dirname = path.dirname(fname)
        oldbasename = path.basename(fname)
        if "GA_" in oldbasename:
            return fname
        elif ("_Y_" in oldbasename) or ("_I_" in oldbasename):
            split = oldbasename.split("_")
            split[2] = split[2].split(".")[0]
            newbasename = "_".join(split)
            return path.join(dirname, newbasename)
        elif "_S_" in fname:
            return fname
        else:
            raise NotImplementedError(f"filetype not supported. fname: {fname}")


    @staticmethod
    def _get_newfilename(oldname):
        """Takes the old path and returns it with the filename prepended with `eventfiltered_`"""
            # example filename: 20160929_M1_05057144.015_I_CrabNebula-W0.40+215_mergecols.csv
            # removes subrun (bc files are merged to runlevel) and prepends eventfiltered
        dirname = path.dirname(oldname)
        newdirname = path.join(dirname, FILTERDIR)
        oldbasename = path.basename(oldname)
        if "GA_" in oldbasename:
            if ("_Y_" in oldbasename) or ("_I_" in oldbasename):
                newbasename = FILTERPREFIX + oldbasename
        elif ("_Y_" in oldbasename) or ("_I_" in oldbasename):
            splitname = oldbasename.split("_")
            splitname[2] = splitname[2].split(".")[0]
            newbasename = FILTERPREFIX + "_".join(splitname)
        newfilename = path.join(newdirname, newbasename)
        return newfilename


    def __call__(self):
        self.remove_events()


    def invert_selection(self):
        """Selects and writes a new csv file with all events that do not conform to the filtermask
        (which means they did get filtered out from calibrated->star or whatever step)"""
        pass


def process_events(**args):
    remover = EventRemover(**args)
    remover.remove_events()


@click.command()
@click.option('--tofiltercalibglob', "-fcg", default="./csv/*_Y_*.csv", type=click.Path(),
              help=('Dir from which to read the calibrated (subrun level) csv files'
                    'which are going to get filtered.'))
@click.option('--starglob', "-sg", default="./star/*_M1_*I_*.root", type=click.Path(),
              help=('Glob to the star files generated after image cleaning can be root or whatever.'
                    'Only the subrun names are important for filtering.'))
@click.option('--superstarmcolglob', "-ssg", default="./superstar/mergecols/*_S_*.csv", type=click.Path(),
              help=('CSV Files from which to read the features (mergecols) for filtering events.'
                    'Generated after merging (superstar) and extracting from root.'))
@click.option('--njobs', "-n", default=1, type=int,
              help='Number of jobs for processing.')
@click.option('--invert', "-i", default=False, type=bool,
              help='Whether to do a inverse filter (only leave events that get thrown out during star cleaning).')
def main(**args):
    try:
        if args["njobs"] == 1:
            process_events(**args)
        else:
            raise NotImplementedError("Multiprocessing needs to be implemented still.")
            # func that splits the globstirngs into new globstrings (should be in process_star.py or somewhere else)
            # newcalibglobs = split_glob(tofiltercalibglob)
            # newstarglobs = split_glob(starglob)
            # newsuperstarglobs = split_glob(superstarmcolglob)
            # globzip = zip(newcalibglobs, newstarglobs, newsuperstarglobs)
            # Parallel(n_jobs=njobs)(delayed(process_events)(ncalibglob, nstarglob, nsuperstarglob, njobs, invert)
            #                        for ncalibglob, nstarglob, nsuperstarglob in globzip)
    except:
        import pdb, traceback, sys
        type, value, tb = sys.exc_info()
        traceback.print_exc()
        pdb.post_mortem(tb)


if __name__ == "__main__":
    main()
Takes a list of fnames and returns two lists,        each containing only fnames from M1 and M2 Returns the run number from a str fname. If full is set it returns the        subrun part of the str as well. (_Y_ and _I_ only) # example: GA_za05to35_8_1743990to1745989_S_wr_2.root # GA_M1_za35to50_8_1740993_Y_wr.root # 20161008_M1_05057440.005_Y_CrabNebula-W0.40+215.root or # eventfiltered_20161008_05057440.005_Y_CrabNebula-W0.40+215.root Removes all events from calibfeatures that arent in starmergecols        (that is post image cleaning), resulting in new calibrated files with only the events remaining        after after the image cleaning. These can then be preprocessed and classified. #hardcoded for now not implemented yet # if not assert_single_fmask_and_df(filtermask, tofiltercalibdf, ssmcoldf): # return False Takes the star filenames corresponding to the superstar files used for filtering and            creates a list of valid runs+subruns that are to be used at the calibrated level # more detail inside of get_filtermask #M2: {np.sum(fmask2)}") # handle duplicates at the end of superstar file # when did this happen? when the last calibrows pre or post filtermask were the same? # probably post filtermask, or else the condition for equality after filtermask wouldnt have worked # raise ArithmeticError("raising error for now, start bugfixing") Filters all tofiltercalib files with the starfiles provided and throws out            not matching subruns, merges the remaining subruns to runlevel.            Same for the mergecolcalibfiles. Then compares each run of mcolcalibfiles            to the superstarmergecolfile of the same run and throws out all events in            tofiltercalib based on the filtermask. Finally writes to disk. # newfname1 = self.single_remove_events(ssmcoldf, ssmcolfname, tofiltercalibfnames1, self.m1sscols) # newfname2 = self.single_remove_events(ssmcoldf, ssmcolfname, tofiltercalibfnames2, self.m2sscols) # if self.assert_filtermask_and_dfs(filtermask1, filtermask2, tofiltercalibdf1, tofiltercalibdf2, ssmcoldf) is False: # self._move_processed_files(self._handle_fnames_for_merging(tofiltercalibfnames1, currentrun), self.bugdir) # self._move_processed_files(self._handle_fnames_for_merging(tofiltercalibfnames2, currentrun), self.bugdir) # os.remove(newfname1) # os.remove(newfname2) # continue # self._move_processed_files(self._handle_fnames_for_merging(tofiltercalibfnames1, currentrun)) # self._move_processed_files(self._handle_fnames_for_merging(tofiltercalibfnames2, currentrun)) # this doesnt work, the called function needs to be a regular function that takes the class as an argument and then does the computation on the passed class Takes the mergecols extracted from the superstar file and compares the ones from the         calib files to it. When calib event doesnt match superstar event, the filtermask is set         to 0. Once ss=calib for the current row the next ss event is looked at         till the end of the file. In the end all events not in the ss file are         removed from the calib file. Calibdf has to contain the same run as the ssmcoldf.         Maybe do a filename assertion. # set as assertion once bug is fixed #, \ # raise ArithmeticError("Start debugging") # needs a fix for when there is a duplicate in the middle/end of the ssmcoldf (results in the filtered calibdf being too short) # sometimes the ss file has a duplicate event at the end which is not duplicated in the calibrated file ?!?!? # assert np.sum(filtermask) != 0, "something went wrong len of filtermask is 0 start debugging" Takes multiple fnames (subruns) from the same run and merges them in a DF. Returns the DF # depends on the fact the merge cols extract in globRoot2csv.C sets headers, if that is changed set header=None here too Takes fname and removes the subrun from it, leaving only the name with run identifier. Takes the old path and returns it with the filename prepended with `eventfiltered_` # example filename: 20160929_M1_05057144.015_I_CrabNebula-W0.40+215_mergecols.csv # removes subrun (bc files are merged to runlevel) and prepends eventfiltered Selects and writes a new csv file with all events that do not conform to the filtermask         (which means they did get filtered out from calibrated->star or whatever step) # func that splits the globstirngs into new globstrings (should be in process_star.py or somewhere else) # newcalibglobs = split_glob(tofiltercalibglob) # newstarglobs = split_glob(starglob) # newsuperstarglobs = split_glob(superstarmcolglob) # globzip = zip(newcalibglobs, newstarglobs, newsuperstarglobs) # Parallel(n_jobs=njobs)(delayed(process_events)(ncalibglob, nstarglob, nsuperstarglob, njobs, invert) #                        for ncalibglob, nstarglob, nsuperstarglob in globzip)