class PLibParser(object):
    def __init__(self):
        self.parser = None
        self.config = None

    def get_parser(self):
        parser = argparse.ArgumentParser(description='Process Pupil-Lib data into trials through the use of \n'
                                                     'markers obtained from either LSL or the native marker \n'
                                                     'producer.')
        parser.add_argument('--datasets', '-D', type=str, nargs='*',
                            help='Datasets to process. Either this flag or the --proc-datasets must be used.')
        parser.add_argument('--proc-datasets', '-P', type=str, nargs='*',
                            help='Previously processed datasets to be used. Either this flag or the --datasets \n'
                                 'must be used.')
        parser.add_argument('--max-workers', '-MW', type=int, action='store', nargs=1,
                            help='Max number of workers that can be used.')
        parser.add_argument('--store', '-s', type=str, action='store', nargs=1,
                            help='Path to store data in. Metadata is stored in a pickled json file called \n'
                                 'data_state.json and alongside it is a folder that contains the raw data \n'
                                 'and broken down data for each of the epochs.')
        parser.add_argument('--dataset-pre-processing', type=yaml.load, nargs='*',
                            help='List of functions that should be run before epoching the dataset. These' +
                                 'are defined in the file dataset_processor.py.')
        parser.add_argument('--eye-pre-processing', type=yaml.load, nargs='*',
                            help='List of functions that should be run before epoching an eye. These' +
                                 'are defined in the file eye_processor.py.')
        parser.add_argument('--trigger-pre-processing', type=yaml.load, nargs='*',
                            help='List of functions that should be run before epoching a trigger. These' +
                                 'are defined in the file eye_processor.py.')
        parser.add_argument('--trial-pre-processing', type=yaml.load, nargs='*',
                            help='List of functions that should be run on the data chunk before a trial ' +
                                 'worker segments the trial from the data streams. '
                                 'are defined in the file eye_processor.py.')
        parser.add_argument('--dataset-post-processing', type=yaml.load, nargs='*',
                            help='List of functions that should be run after epoching the dataset. These' +
                                 'are defined in the file dataset_processor.py.')
        parser.add_argument('--eye-post-processing', type=yaml.load, nargs='*',
                            help='List of functions that should be run after epoching an eye. These' +
                                 'are defined in the file eye_processor.py.')
        parser.add_argument('--trigger-post-processing', type=yaml.load, nargs='*',
                            help='List of functions that should be run after epoching a trigger. These' +
                                 'are defined in the file eye_processor.py.')
        parser.add_argument('--trial-post-processing', type=yaml.load, nargs='*',
                            help='List of functions that should be run on the data chunk after a trial ' +
                                 'worker segments the trial from the data streams. '
                                 'are defined in the file eye_processor.py.')
        parser.add_argument('--trial-range', '-tr', type=int, action='append', nargs=2,
                            help='The trial range to extract for the given trigger names. The first number given\n'
                                 'is assumed to be the number of seconds before the before the triggers and the \n'
                                 'second number is the number of seconds after the triggers.')
        parser.add_argument('--baseline', '-b', type=float, action='store', nargs=2,
                            help='The time bounds that should be considered as a baseline \n'
                                 'for the trial. If it is not specified, pupil-lib will not produce a dataset \n'
                                 'that has baseline means removed and neither will it produce a dataset that has\n'
                                 'a percent change representation of the data.')
        parser.add_argument('--triggers', '-t', type=str, nargs='+',
                            help='The names of the triggers to process. Use --trigger-trials to give a list of \n'
                                 'triggers and a time span for each of them.')
        parser.add_argument('--logger', '-lg', type=str, nargs=1,
                            help='By default, a log is saved in `core/logs/log` and this flag can change where the \n'
                                 'log will go. Currently, there are two options: \n'
                                 '          1 - default\n'
                                 '          2 - stdout\n'
                                 'Add a new handler to resources logger_config.json to have more options. It will \n'
                                 'become available through this flag.')
        parser.add_argument('--test', action='store_true',
                            help='This specifies if the output should be tested. Consider this as `debug` mode')
        parser.add_argument('--testingdepth', type=str, nargs=1,
                            help='By default, a low amount of depth will be output during testing. If more is needed'
                                 'set it to `deep`. For now, it`s all or nothing.')
        parser.add_argument('--run-config', '-rc', type=str, nargs=1,
                            help='A run configuration (YAML, .yml) that lets you specify specific processing (pre, '
                            'and post) functions at each level of processing. See the docs for information and '
                            ' examples of how this configuration file should be specified. Using this argument '
                            ' overrides all the other options.')
        parser.add_argument('--data-names', type=str, nargs='*',
                            help='Must be given if an xdf dataset is used. This specifies which data fields you are '
                                 'interested in. For example giving `eye0`, and `eye1` will give you the diameters of '
                                 'the eyes. There will be more...')
        parser.add_argument('--only-markers-in-streams', action='store_true', default=False,
                            help='This specifies if only the marker positions in the streams should be returned.')
        parser.add_argument('--save-mat', action='store_true', default=False,
                            help='If this flag is supplied, `.mat` files will be saved instead of `.csv` when running '
                                 'from CLI.')
        parser.add_argument('--prefix', type=str, default='',
                            help='If this flag is supplied, all files saved when running from CLI will be prefixed '
                                 'with this string.')
        self.parser = parser
        return parser

    '''
        Use this function to parse the arguments given through the command line interface
        and produce the configuration that will be used for decision making in the
        epoching process.
    '''
    def build_config_from_cli(self, args):
        self.config = {}
        if 'parsed_yaml' not in self.config:
            self.config['parsed_yaml'] = {}

        # Make sure there is some data to be ingested.
        if args.datasets is not None:
            self.config['proc_datasets'] = []
            for dataset_path in args.datasets:
                self.config['datasets'].append(os.path.abspath(dataset_path))
            self.config['processing_old'] = False
        elif args.proc_datasets is not None:
            self.config['proc_datasets'] = []
            for dataset_path in args.datasets:
                self.config['proc_datasets'].append(os.path.abspath(dataset_path))
            self.config['processing_old'] = True
        else:
            raise Exception('ERROR: Missing datasets. See `--help` for help with `--datasets` and `--proc-datasets`')

        # Get all the other arguments, let everything else take care
        # of errors in what it needs.
        self.config['max_workers'] = args.max_workers[0] if args.max_workers is not None else None
        self.config['store'] = args.store[0] if args.store is not None else None
        self.config['trial_range'] = args.trial_range[0] if args.trial_range is not None else None
        self.config['baseline'] = args.baseline[0] if args.baseline is not None else None
        self.config['logger'] = args.logger[0] if args.logger is not None else None
        self.config['triggers'] = args.triggers if args.triggers is not None else None
        self.config['logger'] = args.logger[0] if args.logger is not None else 'default'
        self.config['testing'] = args.test if args.test is not None else False
        self.config['testing_depth'] = args.testingdepth[0] if args.testingdepth is not None else 'low'
        self.config['only_markers_in_streams'] = args.only_markers_in_streams

        # If this is set to anything else, there must be a loader for the combination of fields.
        # Request these or build them yourself in xdfloader_processor.py.
        self.config['dataname_list'] = args.data_names if args.data_names is not None else DEFAULT_DATA_STREAMS

        # Always set defaults.

        # TODO: Use custom type to split pre and post processing functions
        # TODO: in the arguments.
        self.config['dataset_pre_processing'] = DEFAULT_PROCESSING if args.dataset_pre_processing is None \
            else args.dataset_pre_processing
        self.config['eye_pre_processing'] = DEFAULT_PROCESSING if args.eye_pre_processing is None \
            else args.eye_pre_processing
        self.config['trigger_pre_processing'] = DEFAULT_PROCESSING if args.trigger_pre_processing is None \
            else args.trigger_pre_processing
        self.config['trial_pre_processing'] = DEFAULT_PROCESSING if args.trial_pre_processing is None \
            else args.trial_pre_processing

        self.config['dataset_post_processing'] = DEFAULT_PROCESSING if args.dataset_post_processing is None \
            else args.dataset_post_processing
        self.config['eye_post_processing'] = DEFAULT_PROCESSING if args.eye_post_processing is None \
            else args.eye_post_processing
        self.config['trigger_post_processing'] = DEFAULT_PROCESSING if args.trigger_post_processing is None \
            else args.trigger_post_processing
        self.config['trial_post_processing'] = DEFAULT_PROCESSING if args.trial_post_processing is None \
            else args.trial_post_processing

        return self.config

    def build_config_from_yaml(self, args, yaml_path=None):
        print('Building config from yml file...')
        self.config = {}
        self.config['parsed_yaml'] = {}

        if yaml_path:
            data_to_load = yaml_path
        else:
            data_to_load = args.run_config[0]

        # Load the yaml config and parse it.
        with open(data_to_load, 'r') as stream:
            data_loaded = yaml.safe_load(stream)

        # There is nothing that verifies the YAML structure because
        # it is highly variable. So, we do our best here.

        yaml_config = data_loaded['config'] if 'config' in data_loaded else None

        def check_and_get(data_dict, field_name, default):
            val_to_return = default
            if data_dict:
                if field_name in data_dict:
                    val_to_return = data_dict[field_name]
            return val_to_return

        # If the yaml has group-wide configurations set
        # them through the 'config' field.
        self.config['max_workers'] = check_and_get(yaml_config, 'workers', None)
        self.config['logger'] = check_and_get(yaml_config, 'logger', 'default')
        self.config['store'] = check_and_get(yaml_config, 'output_dir', os.getcwd())
        self.config['trial_range'] = [check_and_get(yaml_config, 'baseline_time', None),
                                      check_and_get(yaml_config, 'trial_time', None)]
        self.config['baseline_time'] = check_and_get(yaml_config, 'baseline_time', None)
        self.config['baseline'] = check_and_get(yaml_config, 'baseline', None)
        self.config['trial_time'] = check_and_get(yaml_config, 'trial_time', None)
        self.config['triggers'] = check_and_get(yaml_config, 'triggers', None)
        # Testing is still specified from the command line.
        self.config['testing'] = check_and_get(yaml_config, 'testing', None)
        self.config['testing_depth'] = check_and_get(yaml_config, 'testing_depth', 'low')
        self.config['only_markers_in_streams'] = check_and_get(yaml_config, 'only_markers_in_streams', False)

        # Always set default processing functions unless given an empty list.
        # Pre-processing
        self.config['dataset_pre_processing'] = check_and_get(yaml_config, 'dataset_pre_processing',
                                                              DEFAULT_PROCESSING)

        self.config['eye_pre_processing'] = check_and_get(yaml_config, 'eye_pre_processing', DEFAULT_PROCESSING)

        self.config['trigger_pre_processing'] = check_and_get(yaml_config, 'trigger_pre_processing',
                                                              DEFAULT_PROCESSING)

        self.config['trial_pre_processing'] = check_and_get(yaml_config, 'trial_pre_processing', DEFAULT_PROCESSING)

        # Post-processing
        self.config['dataset_post_processing'] = check_and_get(yaml_config, 'dataset_post_processing',
                                                               DEFAULT_PROCESSING)

        self.config['eye_post_processing'] = check_and_get(yaml_config, 'eye_post_processing', DEFAULT_PROCESSING)

        self.config['trigger_post_processing'] = check_and_get(yaml_config, 'trigger_post_processing',
                                                               DEFAULT_PROCESSING)

        self.config['trial_post_processing'] = check_and_get(yaml_config, 'trial_post_processing',
                                                             DEFAULT_PROCESSING)

        self.config['dataname_list'] = check_and_get(yaml_config, 'data_names', DEFAULT_DATA_STREAMS)

        # If the trial range and triggers, were not declared in the
        # config section, then they must either be declared in the
        # dataset, or eye sections. For each dataset, keep a list of triggers
        # and union them as we find new references so that we don't discard
        # any of them. If either dataset has None as it's set of triggers
        # an error will be raised.
        all_info = {}
        default_triggers = self.config['triggers'] if 'triggers' in self.config else None

        # A couple helper functions.
        def get_latest_default(old_default, dict_with_new, field):
            if field in dict_with_new:
                return dict_with_new[field]
            return old_default

        def union(a, b):
            if a:
                return list(set().union(a, b))
            return b

        def replace_keys(given_config, defaults_config):
            new_config = {}

            for key in defaults_config:
                new_config[key] = copy.deepcopy(get_latest_default(defaults_config[key], given_config, key))
            '''
            for key in given_config:
                if key not in defaults_config:
                    new_config[key] = given_config[key]
            '''

            return new_config

        dataset_names = []
        data_name_per_dataset = {}
        for dataset_name in data_loaded:
            if dataset_name != 'config':

                # Default is what is given in 'config' field.
                dataset_default_triggers = default_triggers

                # Process a dataset configuration.
                dataset_config = data_loaded[dataset_name]

                # Use path as the dict entries. Must always be given.
                if 'dataset_path' in dataset_config:
                    dataset_path = os.path.abspath(
                        dataset_config['dataset_path']
                    ).replace('\\', os.path.sep).replace('/', os.path.sep)
                else:
                    raise Exception("Error: Path to data folder must be given.")

                dataset_config_name = dataset_name + '|' + dataset_path
                data_name_per_dataset[dataset_config_name] = []
                dataset_names.append(dataset_config_name)

                # If we find a trigger field in the datatset config,
                # set it as the new default for the entire dataset.
                # Otherwise, leave it to what the config set it as.
                # Do this for baseline_time and trial_time as well.
                dataset_info = replace_keys(dataset_config, self.config)
                dataset_info['trial_range'] = [dataset_info['baseline_time'], dataset_info['trial_time']]

                # If no data names are specified, check for an additional list or
                # assume the eye diameters by default.
                if 'data_names' in dataset_config:
                    data_name_per_dataset[dataset_config_name] = dataset_config['data_names']
                elif 'datasets' in dataset_config:
                    data_name_per_dataset[dataset_config_name] = list(dataset_config['datasets'].keys())
                else:
                    print("Using default set of data streams: %s" % dataset_info['dataname_list'])
                    data_name_per_dataset[dataset_config_name] = dataset_info['dataname_list']

                # Now, go through the data names specified, if there
                # are none, eye0 and eye1 will be used as default.
                if 'datasets' in dataset_config:
                    for data_name in dataset_config['datasets']:
                        eye_default_triggers = dataset_info['triggers']

                        eye_config = dataset_config['datasets'][data_name]
                        data_name_per_dataset[dataset_config_name] = union([data_name],
                                                                           data_name_per_dataset[dataset_config_name])
                        eye_config_name = dataset_config_name + ":" + data_name

                        # Get the newest settings for this eye
                        eye_info = replace_keys(eye_config, dataset_info)

                        # Sets the triggers to a new list.
                        eye_default_triggers = get_latest_default(eye_default_triggers, eye_config,
                                                                  'triggers-list')
                        eye_info['triggers'] = eye_default_triggers
                        eye_info['trial_range'] = [eye_info['baseline_time'], eye_info['trial_time']]

                        # More specific functions for some trigger (may or may not be listed in
                        # 'triggers_list').
                        if 'triggers' in eye_config:
                            for trigger_name in eye_config['triggers']:
                                # For each trigger name, union it with the list of already running
                                # triggers specified in eye_info['triggers'] - in case of differences.
                                eye_info['triggers'] = union(eye_info['triggers'], [trigger_name])

                                trigger_config = eye_config['triggers'][trigger_name]
                                trigger_config_name = eye_config_name + ":trigger" + trigger_name

                                # Get the newest settings for this trigger, if any.
                                trigger_info = replace_keys(trigger_config, eye_info)

                                trigger_info['only_markers_in_streams'] = self.config['only_markers_in_streams']
                                trigger_info['trial_range'] = [trigger_info['baseline_time'], trigger_info['trial_time']]

                                if 'trials' in trigger_config:
                                    for trial_num in trigger_config['trials']:
                                        trial_config = trigger_config['trials'][trial_num]
                                        trial_config_name = trigger_config_name + ":trial" + str(trial_num)

                                        # Get the newest settings for this trial, if any.
                                        trial_info = replace_keys(trial_config, trigger_info)
                                        trial_info['trial_range'] = [trial_info['baseline_time'], trial_info['trial_time']]

                                        if trial_info['trial_time'] is None:
                                            raise Exception("Error: No trial time was given for " +
                                                            trial_config_name + " at: " + dataset_path)
                                        elif trial_info['baseline_time'] is None:
                                            raise Exception("Error: No baseline time was given for " +
                                                            trial_config_name + " at: " + dataset_path)

                                        all_info[trial_config_name] = copy.deepcopy(trial_info)

                                all_info[trigger_config_name] = copy.deepcopy(trigger_info)

                        # If the eye triggers are empty, then it means triggers were missing either
                        # in this field or above in the dataset, or config fields.
                        if not eye_info['triggers']:
                            raise Exception(
                                "Error: No triggers were given for " + dataset_name + " at: " + dataset_path)

                        all_info[eye_config_name] = copy.deepcopy(eye_info)
                if len(data_name_per_dataset[dataset_config_name]) == 0:
                    data_name_per_dataset[dataset_config_name] = ['eye0', 'eye1']

                all_info[dataset_config_name] = copy.deepcopy(dataset_info)
        # If no datasets were found.
        if not all_info:
            raise Exception("Error: No datasets given.")

        self.config['datasets'] = dataset_names
        self.config['data_name_per_dataset'] = data_name_per_dataset
        self.config['parsed_yaml'] = all_info

        if yaml_path:
            self.config = self.parse_processing_defaults(self.config)

    def parse_processing_defaults(self, data_dict):
        def choose_defaults(level, position):
            return {
                'dataset': {
                    'pre_processing': DatasetDefaults.pre_defaults(),
                    'post_processing': DatasetDefaults.post_defaults(),
                },
                'eye': {
                    'pre_processing': EyeDefaults.pre_defaults(),
                    'post_processing': EyeDefaults.post_defaults(),
                },
                'trigger': {
                    'pre_processing': TriggerDefaults.pre_defaults(),
                    'post_processing': TriggerDefaults.post_defaults(),
                },
                'trial': {
                    'pre_processing': TrialDefaults.pre_defaults(),
                    'post_processing': TrialDefaults.post_defaults(),
                },
            }[level][position]

        def replace_default_in_list(proc_list, new_defaults):
            new_proc_list = []
            for proc in proc_list:
                if proc['name'] == 'default':
                    new_proc_list += new_defaults
                else:
                    new_proc_list.append(proc)
            return new_proc_list

        new_dict = data_dict
        for entry in data_dict:
            if '_pre_processing' in entry or '_post_processing' in entry:
                for functs in data_dict[entry]:
                    if 'default' in functs['name']:
                        defaults = choose_defaults(entry.split('_')[0], '_'.join(entry.split('_')[1:]))
                        new_dict[entry] = replace_default_in_list(data_dict[entry], defaults)
                        break
            elif isinstance(data_dict[entry], dict):
                new_dict[entry] = self.parse_processing_defaults(data_dict[entry])

        return new_dict

    def build_config(self, args):
        # Determine if we build from parser options
        # or from yaml options.
        if args.run_config is not None:
            self.build_config_from_yaml(args)
        else:
            self.build_config_from_cli(args)

        # Store some optional CLI arguments here,
        # these are only used when pupillib is used
        # from the command line.
        self.config['save_mat'] = False
        self.config['prefix'] = ''
        if args.save_mat:
            self.config['save_mat'] = True
        if args.prefix:
            self.config['prefix'] = args.prefix

        self.config = self.parse_processing_defaults(self.config)

    '''
        Run this function determine the maximum worker count. Here, we assume that each
        trigger worker must process all trials for the sake of simplicity. It is changed
        later in the loader to a more specific value.
    '''
    def get_worker_count(self):
        # In total, there are 2*n*x processes that can spawn, where n
        # is the number of datasets and x is the number of trials across
        # all triggers. This can be changed with '--max-workers'.

        # Get max dataset workers and if we are processing old or new datasets.
        # It's one or the other and not both.
        if self.config['datasets'] is not None:
            num_datasets = len(self.config['datasets'])
        else:
            num_datasets = len(self.config['proc_datasets'])

        # Set a flag for if we are processing old or new datasets.

        num_datasets_per_worker = num_datasets
        num_dataset_workers = 1

        # Get eye workers.
        num_eyes = num_datasets * 2
        num_eyes_per_worker = num_eyes
        num_eye_workers = 0

        # Get max trigger workers.
        total_triggers = len(self.config['triggers'])*num_eyes
        num_triggers = len(self.config['triggers'])
        num_triggers_per_worker = total_triggers
        num_trigger_workers = 0

        max_workers = self.config['max_workers'] if self.config['max_workers'] is not None else -1

        self.config['maximize'] = False
        total_possible_workers = num_datasets + (num_datasets * 2) + (num_datasets * num_triggers * 2)
        if max_workers == -1 or max_workers >= total_possible_workers:
            # Set the maximums
            num_dataset_workers = num_datasets
            num_datasets_per_worker = 1

            num_eye_workers = num_eyes
            num_eyes_per_worker = 1

            num_trigger_workers = total_triggers
            num_triggers_per_worker = 1

            self.config['leftover_workers'] = 0
            self.config['total_workers'] = total_possible_workers

            self.config['num_datasets'] = num_datasets
            self.config['dataset_workers'] = num_dataset_workers
            self.config['datasets_per_worker'] = num_datasets_per_worker

            self.config['num_eyes'] = num_eyes
            self.config['eye_workers'] = num_eye_workers
            self.config['eyes_per_worker'] = num_eyes_per_worker

            self.config['total_triggers'] = total_triggers
            self.config['num_triggers'] = num_triggers
            self.config['trigger_workers'] = num_trigger_workers
            self.config['triggers_per_worker'] = num_triggers_per_worker

            # May or may not be used.
            self.config['num_trials'] = sys.maxsize
            self.config['trial_workers'] = sys.maxsize
            self.config['trials_per_worker'] = 1

            self.config['maximize'] = True
            return self.config
        elif self.config['max_workers'] == 1 or self.config['max_workers'] == 0:
            if self.config['max_workers'] == 0:
                self.config['max_workers'] = 1
            # If 'no_parallel' is set, other fields will never be
            # checked.
            self.config['no_parallel'] = True
            self.config['worker_count_complete'] = True
            self.config['num_datasets'] = num_datasets
            self.config['num_eyes'] = num_eyes
            self.config['num_triggers'] = num_triggers
            self.config['total_triggers'] = total_triggers
            return self.config

        # Otherwise, we have to do some splitting.
        # To make sure we don't make odd splitting decisions
        # and to keep the splits "even" we will generously underestimate
        # at certain times.
        if self.config['max_workers'] < num_datasets:
            # Leave everything else as zero and spread the work across dataset workers.
            num_datasets_per_worker = math.ceil(num_datasets/self.config['max_workers'])
            num_dataset_workers = self.config['max_workers']
        elif num_datasets < self.config['max_workers'] < num_eyes + num_datasets:
            # Give each dataset a worker.
            num_dataset_workers = num_datasets
            num_datasets_per_worker = 1

            # Split the remaining across the eye workers.
            curr_max_workers = self.config['max_workers'] - num_datasets
            if curr_max_workers > num_datasets:
                # We have enough to make a 'good enough' split.
                # i.e. we won't reduce the number of processes
                num_eyes_per_worker = math.ceil(num_eyes/curr_max_workers)
                num_eye_workers = curr_max_workers
        elif num_eyes+num_datasets < self.config['max_workers'] \
                < num_eyes + num_datasets + (num_datasets*num_triggers*2):
            # Give each dataset, and eye a worker.
            num_dataset_workers = num_datasets
            num_datasets_per_worker = 1

            num_eye_workers = num_eyes
            num_eyes_per_worker = 1

            # Split the remaining across the trigger workers.
            curr_max_workers = self.config['max_workers'] - num_datasets - num_eyes
            if curr_max_workers > num_eyes:
                # We have enough to make a 'good enough' split.
                # i.e. we won't reduce the number of processes
                num_triggers_per_worker = math.ceil(total_triggers/curr_max_workers)
                num_trigger_workers = curr_max_workers
        else:
            # We have enough to split the work across all of them and maybe
            # some extra for processing the trials.

            # Give each dataset, eye, and trigger a worker.
            num_dataset_workers = num_datasets
            num_datasets_per_worker = 1

            num_eye_workers = num_eyes
            num_eyes_per_worker = 1

            num_trigger_workers = total_triggers
            num_triggers_per_worker = 1
            self.config['leftover_workers'] = self.config['max_workers'] - \
                num_eyes + num_datasets + (num_datasets * num_triggers * 2)

        self.config['ideal_num_workers'] = total_possible_workers
        self.config['total_workers'] = num_dataset_workers + num_eye_workers + num_trigger_workers

        self.config['num_datasets'] = num_datasets
        self.config['dataset_workers'] = num_dataset_workers
        self.config['datasets_per_worker'] = num_datasets_per_worker

        self.config['num_eyes'] = num_eyes
        self.config['eye_workers'] = num_eye_workers
        self.config['eyes_per_worker'] = num_eyes_per_worker

        self.config['total_triggers'] = total_triggers
        self.config['num_triggers'] = num_triggers
        self.config['trigger_workers'] = num_trigger_workers
        self.config['triggers_per_worker'] = num_triggers_per_worker

        # May or may not be used.
        self.config['num_trials'] = sys.maxsize
        self.config['trial_workers'] = sys.maxsize
        self.config['trials_per_worker'] = 1
        if 'parsed_yaml' not in self.config:
            self.config['parsed_yaml'] = {}
        return self.config class MultiProcessingLog:
    quiet = False

    class _MultiProcessingLog:

        def __init__(self, logger_type='default', include_threadprocs=False):
            def setup_logging(default_path='logging.json', default_level=logging.INFO, env_key='LOG_CFG'):
                """
                    Setup logging configuration
                """
                path = default_path
                value = os.getenv(env_key, None)
                if value:
                    path = value
                if os.path.exists(path):
                    with open(path, 'rt') as f:
                        config = json.load(f)
                        config['loggers']['']['handlers'] = [str(logger_type)]
                        config['handlers'][logger_type]['level'] = os.environ.get("LOGLEVEL", "NOTSET")
                    logging.config.dictConfig(config)
                else:
                    logging.basicConfig(level=default_level)

            setup_logging(default_path='resources/logger_config.json')
            self.include_threadprocs = include_threadprocs
            self.logger = logging.getLogger('pupillib')
            self.queue = multiprocessing.Queue(-1)
            self._closing = False

            class Redirector(object):
                def __init__(self, mplogger):
                    self.mplogger = mplogger
                def write(self, msg):
                    if MultiProcessingLog.quiet:
                        return
                    if not msg.replace(' ', '').replace('\n', ''):
                        return
                    caller = inspect.getouterframes(inspect.currentframe())[1]
                    self.mplogger.info(msg, caller=caller)
                def flush(self):
                    pass

            self.redirector = Redirector(self)
            sys.stdout = self.redirector

            self.t = threading.Thread(target=self.receive)
            self.t.daemon = True
            self.t.start()

        def enable_redirect(self):
            sys.stdout = self.redirector

        def disable_redirect(self):
            sys.stdout = sys.__stdout__

        def receive(self):
            while (not self._closing) or (not self.queue.empty()):
                if not self.queue.empty():
                    record = self.queue.get()
                    self.logger.log(LOGLEVELS[record.level], record.msg)
                else:
                    continue

        def send(self, level, msg, processid=None, thid=None, caller=None):
            if not processid:
                processid = os.getpid()
            if not thid:
                thid = threading.get_ident()

            if MultiProcessingLog.quiet:
                # Logger was requested to be quiet.
                return

            try:
                if not caller:
                    caller = inspect.getouterframes(inspect.currentframe())[1]

                repo_path = caller[1]
                repo_path_split = repo_path.split('pupillib')
                if len(repo_path_split) > 1:
                    repo_path = 'pupillib' + repo_path_split[-1]

                out_str = ' - ' + repo_path + ":" + caller[3] + ":" + str(caller[2])
                if self.include_threadprocs:
                    out_str += ' ' + str(processid) + ' ' + str(thid)
                out_str += ' - ' + msg

                self.queue.put_nowait(SimpleRecord(level, out_str))
            except Exception as e:
                self.queue.put_nowait(SimpleRecord('ERROR', 'During message:  ' + msg +
                                                   ' from pid-' + str(processid) + ',thid-' + str(thid) +
                                                   '. \nAn exception occurred: \n' + traceback.format_exc()))

        def close(self):
            self._closing = True
            if self.t.is_alive():
                self.t.join()

        def info(self, msg, processid=None, thid=None, caller=None):
            self.send("INFO", msg, processid=processid, thid=thid, caller=caller)

        def debug(self, msg, processid=None, thid=None):
            self.send("DEBUG", msg, processid=processid, thid=thid)

        def warning(self, msg, processid=None, thid=None):
            self.send("WARNING", msg, processid=processid, thid=thid)

        def error(self, msg, processid=None, thid=None):
            self.send("ERROR", msg, processid=processid, thid=thid)

        def critical(self, msg, processid=None, thid=None):
            self.send("CRITICAL", msg, processid=processid, thid=thid)


    instance = None

    @staticmethod
    def set_logger_type(logger_type):
        if MultiProcessingLog.instance is None:
            MultiProcessingLog.instance = MultiProcessingLog._MultiProcessingLog(logger_type)
        else:
            MultiProcessingLog.instance.send('WARNING', 'Cannot open a logger of a second type after it has already '
                                                        'been opened.')
        return MultiProcessingLog.instance

    @staticmethod
    def get_logger():
        if MultiProcessingLog.instance is None:
            MultiProcessingLog.instance = MultiProcessingLog._MultiProcessingLog()
        return MultiProcessingLog.instance

    @staticmethod
    def setQuiet(quiet):
        MultiProcessingLog.quiet = quiet class PLibDatasetWorker(Thread):
    def __init__(self, config, dataset=None):
        Thread.__init__(self)
        self.config = config    # Metadata about how to process the given datasets.
        self.dataset = dataset
        self.logger = MultiProcessingLog.get_logger()

        self.initial_data = {
            'config': config,    # Metadata about how to process the given datasets.
            'dataset': dataset,
        }

        self.proc_eye_data = {}
        self.proc_generic_data = {}
        self.proc_dataset_data = {}

    def run_eye_workers(self):
        # If this dataset is in the yaml config, specify it's
        # configuration by replacing the current one with a new one.
        self.config = utilities.parse_yaml_for_config(self.config, self.getName())

        # Run the pre processors.
        # Run the pre processors.
        dataset_processor = DatasetProcessor()
        self.initial_data['dataset'] = self.dataset
        rejected_streams = dataset_processor.reject_streams(self.initial_data)
        self.initial_data['dataset'] = {
            name: data
                for name, data in self.initial_data['dataset'].items()
                if name not in rejected_streams
        }
        if len(rejected_streams) > 0:
            self.logger.send(
                'INFO', 'Rejecting the following streams: ' + str(rejected_streams), os.getpid(), threading.get_ident()
            )
            if 'all' in rejected_streams:
                self.logger.send(
                    "INFO", "No data was given, or all streams are broken.", os.getpid(), threading.get_ident()
                )
                return None

        if self.config['dataset_pre_processing']:
            for config in self.config['dataset_pre_processing']:
                if config['name'] in dataset_processor.pre_processing.all:
                    self.initial_data = dataset_processor.pre_processing.all[config['name']](self.initial_data, config)
        self.dataset = self.initial_data['dataset']

        eye_worker0 = PLibEyeWorker(self.config, self.dataset['eye0'], self.dataset['markers'])
        eye_worker1 = PLibEyeWorker(self.config, self.dataset['eye1'], self.dataset['markers'])
        eye_workers = [eye_worker0, eye_worker1]
        if self.config['max_workers'] >= self.config['num_datasets'] + self.config['num_eyes']:
            eye_count = 0
            for i in eye_workers:
                i.setName(self.getName() + ':eye' + str(eye_count))
                eye_count += 1
                i.start()
            for i in eye_workers:
                i.join()
        else:
            if 'eye0' not in rejected_streams:
                eye_worker0.setName(self.getName() + ':eye0')
                eye_worker0.run()
            if 'eye1' not in rejected_streams:
                eye_worker1.setName(self.getName() + ':eye1')
                eye_worker1.run()

        eye_count = 0
        self.proc_dataset_data = {
            'config': {
                'name': self.getName(),
                'srate': eye_worker0.proc_data['config']['srate']
            },
            'data': {
                'eye0': eye_worker0.proc_data,
                'eye1': eye_worker1.proc_data
            }
        }

        # Run the post processors.
        if self.config['dataset_post_processing']:
            if not dataset_processor:
                dataset_processor = DatasetProcessor()

            for config in self.config['dataset_post_processing']:
                if config['name'] in dataset_processor.post_processing.all:
                    dataset_processor.post_processing.all[config['name']](self.proc_dataset_data, config)

    def run_generic_workers(self):
        # If this dataset is in the yaml config, specify it's
        # configuration by replacing the current one with a new one.
        testing = self.config['testing']
        self.config = utilities.parse_yaml_for_config(self.config, self.getName())

        # Run the pre processors.
        dataset_processor = DatasetProcessor()
        self.initial_data['dataset'] = self.dataset
        rejected_streams = dataset_processor.reject_streams(self.initial_data)
        self.initial_data['dataset'] = {
            name: data
            for name, data in self.initial_data['dataset'].items()
            if name not in rejected_streams
        }
        if len(rejected_streams) > 0:
            self.logger.send(
                'INFO', 'Rejecting the following streams: ' + str(rejected_streams), os.getpid(), threading.get_ident()
            )
            if 'all' in rejected_streams:
                self.logger.send(
                    "INFO", "No data was given, or all streams are broken.", os.getpid(), threading.get_ident()
                )
                return None
        self.dataset = self.initial_data['dataset']

        if self.config['dataset_pre_processing']:
            for config in self.config['dataset_pre_processing']:
                if config['name'] in dataset_processor.pre_processing.all:
                    self.initial_data = dataset_processor.pre_processing.all[config['name']](self.initial_data, config)
            self.dataset = self.initial_data['dataset']

        generic_workers = {}
        base_generic_worker = GenericEyeLevelWorker(self.config)
        proc_data_for_data_name = {}
        parallel = False

        srate = 0
        datanames = self.dataset['dataname_list']
        for data_name in datanames:
            if data_name in rejected_streams:
                continue
            data_for_data_name = self.dataset[data_name]

            if self.config['max_workers'] >= self.config['num_datasets'] + len(datanames):
                generic_worker = GenericEyeLevelWorker(self.config, data_for_data_name, self.dataset['markers'])
                generic_worker.setName(self.getName() + ":" + data_name)
                generic_workers[data_name] = generic_worker
                generic_worker.start()
                parallel = True
            else:
                base_generic_worker.setName(self.getName() + ":" + data_name)
                base_generic_worker.set_data(data_for_data_name, self.dataset['markers'])
                base_generic_worker.run()
                proc_data_for_data_name[data_name] = base_generic_worker.proc_data
                base_generic_worker.reset_initial_data()

        if parallel:
            for data_name in generic_workers:
                generic_workers[data_name].join()
            for data_name in datanames:
                proc_data_for_data_name[data_name] = generic_workers[data_name].proc_data

        self.proc_dataset_data = {
            'config': {
                'name': self.getName(),
                'srate': base_generic_worker.proc_data['config']['srate']
            }
        }

        self.proc_generic_data = proc_data_for_data_name
        self.proc_dataset_data['data'] = proc_data_for_data_name

        # Run the post processors.
        if self.config['dataset_post_processing']:
            if not dataset_processor:
                dataset_processor = DatasetProcessor()

            for config in self.config['dataset_post_processing']:
                if config['name'] in dataset_processor.post_processing.all:
                    dataset_processor.post_processing.all[config['name']](self.proc_dataset_data, config)

    def run(self):
        if self.config['testing']:
            self.logger.send('INFO', 'Dataset worker is here.', os.getpid(), threading.get_ident())
        if self.dataset is None:
            self.logger.send('ERROR', 'Dataset worker is missing a dataset', os.getpid(), threading.get_ident())
            return

        # TODO: Completely disable and remove eye_workers if possible
        if not self.dataset['custom_data']:
            self.run_eye_workers()
        else:
            self.logger.send('INFO', 'Processing custom data.')
            self.run_generic_workers()
        self.logger.send('INFO', 'Done all data streams for ' + self.getName()) class XdfLoaderProcessor():
    def __init__(self):
        transform = makeregistrar()

        @transform
        def tester(trigger_data, config):
            print('helloooooo')

        @transform
        def get_primitives_column(primitive_entry, config):
            # Given an entry in an xdf data file, like for 'Gaze Primitive Data',
            # get a specified column entry from it.
            col_num = config['num']
            return np.asarray(primitive_entry['time_series'][:, col_num])

        @transform
        def get_pyrep_column(pyrep_entry, config):
            val2get = config['name']
            if 'num' in config:
                num2get = config['num']
                vals = [eval(el[0])[val2get][num2get] for el in pyrep_entry['time_series']]
            else:
                vals = [eval(el[0])[val2get] for el in pyrep_entry['time_series']]

            return np.asarray(vals)

        @transform
        def get_xdf_timestamps(entry, config):
            return np.asarray(list(entry['time_stamps']))

        @transform
        def srate(no_data, config):
            data = config['data']
            ts = config['timestamps']
            return np.size(data, 0) / (np.max(ts) - np.min(ts))

        @transform
        def get_marker_times(marker_entry, config):
            tmp_times = np.asarray(marker_entry['time_stamps'])
            print("Marker times found: " + str(tmp_times))
            return tmp_times

        @transform
        def get_marker_eventnames(marker_entry, config):
            event_names = []
            for event_name in marker_entry['time_series']:
                # Remove the unecessary dimension.
                event_names.append(event_name[0])
            print("Marker events found: " + str(event_names))
            return np.asarray(event_names)

        @transform
        def test_results(data_for_data_name, data_name):
            # Check data fields to make sure they were created and loaded.
            # Skip and warn if we are not testing, otherwise raise and exception.
            testing = ConfigStore.get_instance().frozen_config['testing']
            logger = MultiProcessingLog.get_logger()
            test_pass = True

            if 'data' in data_for_data_name:
                if not data_for_data_name['data'].any():
                    if testing:
                        raise Exception("No data was loaded for the data name entry " + data_name)
                    else:
                        logger.send('WARNING',' Data entry name ' + data_name + ' has no '
                                              'data loaded. Skipping it.',
                                              os.getpid(), threading.get_ident())
                        test_pass = False
            else:
                if testing:
                    raise Exception("No data field was created and loaded for data name entry " + data_name)
                else:
                    logger.send('WARNING', ' Data entry name ' + data_name + ' has no '
                                           'data field created and loaded. Skipping it.',
                                           os.getpid(), threading.get_ident())
                    test_pass = False

            # Check timestamp fields to make sure they were created and loaded.
            # Skip and warn if we are not testing, otherwise raise and exception.
            if 'timestamps' in data_for_data_name:
                if not data_for_data_name['timestamps'].any():
                    if testing:
                        raise Exception("No timestamps were loaded for the data name entry " + data_name)
                    else:
                        logger.send('WARNING', ' Data entry name ' + data_name + ' has no '
                                               'timestamps loaded. Skipping it.',
                                               os.getpid(), threading.get_ident())
                        test_pass = False
            else:
                if testing:
                    raise Exception("No timestamps were created and loaded for data name entry " + data_name)
                else:
                    logger.send('WARNING', ' Data entry name ' + data_name + ' has no '
                                           'timestamps field created and loaded. Skipping it.',
                                           os.getpid(),
                                           threading.get_ident())
                    test_pass = False

            # Check sampling rate fields to make sure they were created and loaded.
            # Skip and warn if we are not testing, otherwise raise and exception.
            if 'srate' in data_for_data_name:
                if not data_for_data_name['srate'].any() or \
                   (isinstance(data_for_data_name['srate'], int) and data_for_data_name['srate'] == 0):
                    if testing:
                        raise Exception("No srate was loaded for the data name entry " + data_name)
                    else:
                        logger.send('WARNING', ' Data entry name ' + data_name + ' has no '
                                               'srate loaded. Skipping it.',
                                               os.getpid(), threading.get_ident())
                        test_pass = False
            else:
                if testing:
                    raise Exception("No srate was created and loaded for data name entry " + data_name)
                else:
                    logger.send('WARNING', ' Data entry name ' + data_name + ' has no '
                                           'srate field created and loaded. Skipping it.',
                                           os.getpid(),
                                           threading.get_ident())
                    test_pass = False
            return test_pass

        self.transform = transform

    def data_name_to_function(self, data_type):
        return {
            # Each of these must always return data, timestamps, and the sampling rate,
            # in that order.
            'eye0': [
                {   # Get data
                    'fn_name': 'get_primitives_column',
                    'field': 'data',
                    'config': {
                        'num': 0
                    }
                },
                {   # Get timestamps
                    'fn_name': 'get_xdf_timestamps',
                    'field': 'timestamps',
                    'config': {
                        'num': 2
                    }
                },
                {   # Get sampling rate
                    'fn_name': 'srate',
                    'field': 'srate',
                    'config': {
                        'data': None,
                        'timestamps': None
                    }
                },
            ],

            'eye1': [
                {  # Get data
                    'fn_name': 'get_primitives_column',
                    'field': 'data',
                    'config': {
                        'num': 0
                    }
                },
                {  # Get timestamps
                    'fn_name': 'get_xdf_timestamps',
                    'field': 'timestamps',
                    'config': {
                        'num': 2
                    }
                },
                {  # Get sampling rate
                    'fn_name': 'srate',
                    'field': 'srate',
                    'config': {
                        'data': None,
                        'timestamps': None
                    }
                },
            ],

            'eye0-pyrep': [
                {  # Get data
                    'fn_name': 'get_pyrep_column',
                    'field': 'data',
                    'config': {
                        'name': 'diameter_3d'
                    }
                },
                {  # Get timestamps
                    'fn_name': 'get_xdf_timestamps',
                    'field': 'timestamps',
                    'config': {
                        'num': 2
                    }
                },
                {  # Get sampling rate
                    'fn_name': 'srate',
                    'field': 'srate',
                    'config': {
                        'data': None,
                        'timestamps': None
                    }
                },
            ],

            'eye1-pyrep': [
                {  # Get data
                    'fn_name': 'get_pyrep_column',
                    'field': 'data',
                    'config': {
                        'name': 'diameter_3d'
                    }
                },
                {  # Get timestamps
                    'fn_name': 'get_xdf_timestamps',
                    'field': 'timestamps',
                    'config': {
                        'num': 2
                    }
                },
                {  # Get sampling rate
                    'fn_name': 'srate',
                    'field': 'srate',
                    'config': {
                        'data': None,
                        'timestamps': None
                    }
                },
            ],

            'gaze_x': [
                {  # Get data
                    'fn_name': 'get_primitives_column',
                    'field': 'data',
                    'config': {
                        'num': 3
                    }
                },
                {  # Get timestamps
                    'fn_name': 'get_xdf_timestamps',
                    'field': 'timestamps',
                    'config': {
                        'num': 2
                    }
                },
                {  # Get sampling rate
                    'fn_name': 'srate',
                    'field': 'srate',
                    'config': {
                        'data': None,
                        'timestamps': None
                    }
                },
            ],

            'gaze_y': [
                {  # Get data
                    'fn_name': 'get_primitives_column',
                    'field': 'data',
                    'config': {
                        'num': 4
                    }
                },
                {  # Get timestamps
                    'fn_name': 'get_xdf_timestamps',
                    'field': 'timestamps',
                    'config': {
                        'num': 2
                    }
                },
                {  # Get sampling rate
                    'fn_name': 'srate',
                    'field': 'srate',
                    'config': {
                        'data': None,
                        'timestamps': None
                    }
                },
            ],

            'gaze_x-pyrep': [
                {  # Get data
                    'fn_name': 'get_pyrep_column',
                    'field': 'data',
                    'config': {
                        'name': 'norm_pos',
                        'num': 0
                    }
                },
                {  # Get timestamps
                    'fn_name': 'get_xdf_timestamps',
                    'field': 'timestamps',
                    'config': {
                        'num': 2
                    }
                },
                {  # Get sampling rate
                    'fn_name': 'srate',
                    'field': 'srate',
                    'config': {
                        'data': None,
                        'timestamps': None
                    }
                },
            ],

            'gaze_y-pyrep': [
                {  # Get data
                    'fn_name': 'get_pyrep_column',
                    'field': 'data',
                    'config': {
                        'name': 'norm_pos',
                        'num': 1
                    }
                },
                {  # Get timestamps
                    'fn_name': 'get_xdf_timestamps',
                    'field': 'timestamps',
                    'config': {
                        'num': 2
                    }
                },
                {  # Get sampling rate
                    'fn_name': 'srate',
                    'field': 'srate',
                    'config': {
                        'data': None,
                        'timestamps': None
                    }
                },
            ]

        }[data_type] def load_xdf(filename,
             on_chunk=None,
             verbose=True,
             synchronize_clocks=True,
             handle_clock_resets=True,
             dejitter_timestamps=True,
             jitter_break_threshold_seconds=1,
             jitter_break_threshold_samples=500,
             clock_reset_threshold_seconds=5,
             clock_reset_threshold_stds=5,
             clock_reset_threshold_offset_seconds=1,
             clock_reset_threshold_offset_stds=10,
             winsor_threshold=0.0001):
    """Import an XDF file.

    This is an importer for multi-stream XDF (Extensible Data Format) 
    recordings. All information covered by the XDF 1.0 specification is 
    imported, plus any additional meta-data associated with streams or with 
    the container file itself. 

    See http://code.google.com/p/xdf/ for more information on XDF.

    The function supports several further features, such as robust time
    synchronization, support for breaks in the data, as well as some other
    defects.

    Args: 
        filename : name of the file to import (*.xdf or *.xdfz)
        
        verbose : Whether to print verbose diagnostics. (default: false)

        synchronize_clocks : Whether to enable clock synchronization based on
          ClockOffset chunks. (default: true)

        dejitter_timestamps : Whether to perform jitter removal for regularly
          sampled streams. (default: true)

        on_chunk : Function that is called for each chunk of data as it is
           being retrieved from the file; the function is allowed to modify
           the data (for example, sub-sample it). The four input arguments
           are 1) the matrix of [#channels x #samples] values (either numeric
           or 2d cell array of strings), 2) the vector of unprocessed local
           time stamps ( one per sample), 3) the info struct for the stream (
           same as the .info field in the final output, buth without the
           .effective_srate sub-field), and 4) the scalar stream number (
           1-based integers). The three return values are 1) the (optionally
           modified) data, 2) the (optionally modified) time stamps, and 3)
           the (optionally modified) header (default: []).

        Parameters for advanced failure recovery in clock synchronization:

        handle_clock_resets : Whether the importer should check for potential
          resets of the clock of a stream (e.g. computer restart during
          recording, or hot-swap). Only useful if the recording system
          supports recording under such circumstances. (default: true)

        clock_reset_threshold_stds : A clock reset must be accompanied by a
          ClockOffset chunk being delayed by at least this many standard
          deviations from the distribution. (default: 5)

        clock_reset_threshold_seconds : A clock reset must be accompanied by a
          ClockOffset chunk being delayed by at least this many seconds. (
          default: 5)

        clock_reset_threshold_offset_stds : A clock reset must be accompanied
          by a ClockOffset difference that lies at least this many standard
          deviations from the distribution. (default: 10)

        clock_reset_threshold_offset_seconds : A clock reset must be
          accompanied by a ClockOffset difference that is at least this many
          seconds away from the median. (default: 1)

        clock_reset_max_jitter : Maximum tolerable jitter (in seconds of error)
          for clock reset handling. (default: 5)

        Parameters for jitter removal in the presence of data breaks:

        jitter_break_threshold_seconds : An interruption in a regularly-sampled
          stream of at least this many seconds will be considered as a
          potential break (if also the jitter_break_threshold_samples is
          crossed) and multiple segments will be returned. (default: 1)

        jitter_break_threshold_samples : An interruption in a regularly-sampled
          stream of at least this many samples will be considered as a
          potential break (if also the jitter_break_threshold_samples is
          crossed) and multiple segments will be returned. (default: 500)

    Returns:
        streams : list of dicts, one for each stream; the dicts
                  have the following content:
                 ['time_series'] entry: contains the stream's time series
                   [#Channels x #Samples] this matrix is of the type declared in
                   ['info']['channel_format']
                 ['time_stamps'] entry: contains the time stamps for each sample
                   (synced across streams)

                 ['info'] field: contains the meta-data of the stream
                   (all values are strings)
                   ['name']: name of the stream
                   ['type']: content-type of the stream ('EEG','Events', ...)
                   ['channel_format']: value format ('int8', 'int16', 'int32',
                     'int64', 'float32', 'double64', 'string')
                   ['nominal_srate']: nominal sampling rate of the stream
                     (as declared by the device); zero for streams with
                     irregular sampling rate
                   ['effective_srate']: effective (measured) sampling rate of
                     the stream, if regular (otherwise omitted)
                   ['desc']: dict with any domain-specific meta-data declared
                     for the stream; see www.xdf.org for the declared
                     specifications

        fileheader : dict with file header contents in the "info" field

    Examples:
        load the streams contained in a given XDF file
        >>> streams = load_xdf('C:\Recordings\myrecording.xdf')

    License:
        This file is covered by the BSD license.

        Copyright (c) 2015, Alejandro Ojeda and Christian Kothe
        All rights reserved.

        Redistribution and use in source and binary forms, with or without
        modification, are permitted provided that the following conditions are
        met:

            * Redistributions of source code must retain the above copyright
              notice, this list of conditions and the following disclaimer.
            * Redistributions in binary form must reproduce the above copyright
              notice, this list of conditions and the following disclaimer in
              the documentation and/or other materials provided with the
              distribution

        THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
        "AS IS AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
        LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
        A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
        OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
        SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
        LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
        DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
        THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (
        INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
        OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

    """

    class StreamData:
        """Temporary per-stream data."""
        def __init__(self, xml):
            """Init a new StreamData object from a stream header."""
            fmt2char = {'int8': 'b', 'int16': 'h', 'int32': 'i', 'int64': 'q',
                        'float32': 'f', 'double64': 'd'}
            fmt2nbytes = {'int8': 1, 'int16': 2, 'int32': 4, 'int64': 8,
                          'float32': 4, 'double64': 8}
            # number of channels
            self.nchns = int(xml['info']['channel_count'][0])
            # nominal sampling rate in Hz
            self.srate = int(xml['info']['nominal_srate'][0])
            # format string (int8, int16, int32, float32, double64, string)
            self.fmt = xml['info']['channel_format'][0]
            # list of time-stamp chunks (each an ndarray, in seconds)
            self.time_stamps = []
            # list of time-series chunks (each an ndarray or list of lists)
            self.time_series = []
            # list of clock offset measurement times (in seconds)
            self.clock_times = []
            # list of clock offset measurement values (in seconds)
            self.clock_values = []
            # last observed time stamp, for delta decompression
            self.last_timestamp = 0.0
            # nominal sampling interval, in seconds, for delta decompression
            self.tdiff = 1.0/self.srate if self.srate > 0 else 0.0
            # pre-calc some parsing parameters for efficiency
            if self.fmt != 'string':
                # number of bytes to read from stream to handle one sample
                self.samplebytes = self.nchns * fmt2nbytes[self.fmt]
                # format string to pass to struct.unpack() to handle one sample
                self.structfmt = '<%s%s' % (self.nchns, fmt2char[self.fmt])

    if verbose:
        print('Importing XDF file %s...' % filename)
    if not os.path.exists(filename):
        raise Exception('file %s does not exist.' % filename)

    # dict of returned streams, in order of apparance, indexed by stream id
    streams = OrderedDict()
    # dict of per-stream temporary data (StreamData), indexed by stream id
    temp = {}
    # XML content of the file header chunk
    fileheader = None
    # number of bytes in the file for fault tolerance
    filesize = os.path.getsize(filename)

    # read file contents ([SomeText] below refers to items in the XDF Spec)
    with open(filename, 'rb') as f:

        # read [MagicCode]
        if f.read(4) != b'XDF:':
            raise Exception('not a valid XDF file: %s' % filename)
        
        # for each chunk...
        while True:

            # noinspection PyBroadException
            try:
                # read [NumLengthBytes], [Length]
                chunklen = _read_varlen_int(f)
            except:
                if f.tell() < filesize-1024:
                    print('  got zero-length chunk, scanning forward to next '
                          'boundary chunk.')
                    _scan_forward(f)
                    continue
                else:
                    if verbose:
                        print('  reached end of file.')
                    break

            # read [Tag]
            tag = struct.unpack('<H', f.read(2))[0]
            if verbose:
                print('  read tag: %i at %d bytes, length=%d'
                      % (tag, f.tell(), chunklen))

            # read the chunk's [Content]...
            if tag == 1:
                # read [FileHeader] chunk
                xml_string = f.read(chunklen-2)
                fileheader = _xml2dict(ET.fromstring(xml_string))
            elif tag == 2:
                # read [StreamHeader] chunk...
                # read [StreamId]
                s = struct.unpack('<I', f.read(4))[0]
                # read [Content]
                xml_string = f.read(chunklen-6)
                hdr = _xml2dict(ET.fromstring(xml_string))
                streams[s] = hdr
                if verbose:
                    print('  found stream ' + hdr['info']['name'][0])
                # initialize per-stream temp data
                temp[s] = StreamData(hdr)
            elif tag == 3:
                # read [Samples] chunk...
                try:
                    # read [StreamId]
                    s = struct.unpack('<I', f.read(4))[0]
                    # read [NumSampleBytes], [NumSamples]
                    nsamples = _read_varlen_int(f)
                    # allocate space
                    stamps = np.zeros((nsamples,))
                    if temp[s].fmt == 'string':
                        # read a sample comprised of strings
                        values = [[None]*temp[s].nchns for _ in range(nsamples)]
                        # for each sample...
                        for k in range(nsamples):
                            # read or deduce time stamp
                            if struct.unpack('B', f.read(1))[0]:
                                stamps[k] = struct.unpack('<d', f.read(8))[0]
                            else:
                                stamps[k] = (temp[s].last_timestamp +
                                             temp[s].tdiff)
                            temp[s].last_timestamp = stamps[k]
                            # read the values
                            for ch in range(temp[s].nchns):
                                raw = f.read(_read_varlen_int(f))
                                values[k][ch] = raw.decode(errors='replace')
                    else:
                        # read a sample comprised of numeric values
                        values = np.zeros((nsamples, temp[s].nchns))
                        # for each sample...
                        for k in range(nsamples):
                            # read or deduce time stamp
                            if struct.unpack('B', f.read(1))[0]:
                                stamps[k] = struct.unpack('<d', f.read(8))[0]
                            else:
                                stamps[k] = (temp[s].last_timestamp +
                                             temp[s].tdiff)
                            temp[s].last_timestamp = stamps[k]
                            # read the values
                            raw = f.read(temp[s].samplebytes)
                            values[k, :] = struct.unpack(temp[s].structfmt, raw)
                    if verbose:
                        print('  reading [%s,%s]' % (temp[s].nchns, nsamples))
                    # optionally send through the on_chunk function
                    if on_chunk is not None:
                        values, stamps, streams[s] = on_chunk(values, stamps,
                                                              streams[s], s)
                    # append to the time series...
                    temp[s].time_series.append(values)
                    temp[s].time_stamps.append(stamps)
                except Exception as e:
                    # an error occurred (perhaps a chopped-off file): emit a
                    # warning and scan forward to the next recognized chunk
                    print('  got error (%s), scanning forward to next '
                          'boundary chunk.', e)
                    _scan_forward(f)
            elif tag == 6:
                # read [StreamFooter] chunk
                s = struct.unpack('<I', f.read(4))[0]
                xml_string = f.read(chunklen-6)
                streams[s]['footer'] = _xml2dict(ET.fromstring(xml_string))
            elif tag == 4:
                # read [ClockOffset] chunk
                s = struct.unpack('<I', f.read(4))[0]
                temp[s].clock_times.append(struct.unpack('<d', f.read(8))[0])
                temp[s].clock_values.append(struct.unpack('<d', f.read(8))[0])
            else:
                # skip other chunk types (Boundary, ...)
                f.read(chunklen-2)
    
    # Concatenate the signal across chunks
    for stream in temp.values():
        if stream.time_stamps:
            # stream with non-empty list of chunks
            stream.time_stamps = np.concatenate(stream.time_stamps)
            if stream.fmt == 'string':
                stream.time_series = list(itertools.chain(*stream.time_series))
            else:
                stream.time_series = np.concatenate(stream.time_series)
        else:
            # stream without any chunks
            stream.time_stamps = np.zeros((0,))
            if stream.fmt == 'string':
                stream.time_series = []
            else:
                stream.time_series = np.zeros((stream.nchns, 0))

    # perform (fault-tolerant) clock synchronization if requested
    if synchronize_clocks:
        if verbose:
            print('  performing clock synchronization...')
        temp = _clock_sync(temp, handle_clock_resets,
                           clock_reset_threshold_stds,
                           clock_reset_threshold_seconds,
                           clock_reset_threshold_offset_stds,
                           clock_reset_threshold_offset_seconds,
                           winsor_threshold)
    
    # perform jitter removal if requested
    if dejitter_timestamps:
        if verbose:
            print('  performing jitter removal...')
        temp = _jitter_removal(temp, jitter_break_threshold_seconds,
                               jitter_break_threshold_samples,)
    else:
        for stream in temp.values():
            duration = stream.time_stamps[-1] - stream.time_stamps[0]
            stream.effective_srate = len(stream.time_stamps)/duration

    for k in streams.keys():
        stream = streams[k]
        tmp = temp[k]
        stream['info']['effective_srate'] = tmp.effective_srate
        stream['time_series'] = tmp.time_series
        stream['time_stamps'] = tmp.time_stamps

    streams = [s for s in streams.values()]
    return streams, fileheader class ConfigStore:

    class innerConfigStore:
        def __init__(self, config):
            self.frozen_config = frozendict(config)

    instance = None

    @staticmethod
    def get_instance(a_dict=None):
        if a_dict is None:
            a_dict = dict()
        if not ConfigStore.instance:
            ConfigStore.instance = ConfigStore.innerConfigStore(a_dict)
        else:
            return ConfigStore.instance

    @staticmethod
    def set_instance(a_dict=None):
        if a_dict is None:
            a_dict = dict()
        if not ConfigStore.instance:
            ConfigStore.instance = ConfigStore.innerConfigStore(a_dict) def eye_pyrep_to_prim_default(pyrep_stream, all_data_field, all_data):
    return {
        'data': [eval(el[0])['diameter'] for el in pyrep_stream['time_series']],
        'timestamps': all_data[all_data_field]['timestamps'],
        'srate': all_data[all_data_field]['srate']
    } def gaze_pyrep_to_prim_default(pyrep_stream, all_data_field, all_data):
    return {
        'data': all_data[all_data_field]['data'],
        'timestamps': all_data[all_data_field]['timestamps'],
        'srate': all_data[all_data_field]['srate']
    } def gaze_prim_to_pyrep_default(pyrep_stream, all_data_field, all_data):
    return gaze_pyrep_to_prim_default(pyrep_stream, all_data_field, all_data) class PupilDatasets(CommonPupilData):
    def __init__(self, config, all_data):
        self.config = config

        try:
            srate = all_data['datasets'][list(all_data['datasets'].keys())[0]]['config']['srate']
            self.srate = round(srate, 0)
            logger.send(
                "WARNING",
                "Using {} as sampling rate - ".format(self.srate) +\
                "be careful with this value if it varies across datastreams."
            )
        except:
            logger.send("ERROR","Could not obtain sampling rate.")

        self.datasets = {}
        CommonPupilData.__init__(self, all_data, 'datasets')

    @property
    def data(self):
        return self.datasets

    @CommonPupilData.data_type.setter
    def data_type(self, data_type):
        self._data_type = data_type

        for _, ds in self.datasets.items():
            if ds:
                ds.data_type = self._data_type

    @CommonPupilData.time_or_data.setter
    def time_or_data(self, val):
        self._time_or_data = val

        for _, ds in self.datasets.items():
            if ds:
                ds.time_or_data = self._time_or_data

    def save_csv(self, output_dir, name=''):
        for _, dataset in self.datasets.items():
            if dataset:
                dataset.save_csv(output_dir, name=name)

    def save_trigger_csv(self, output_dir, name=''):
        for _, dataset in self.datasets.items():
            if dataset:
                dataset.save_trigger_csv(output_dir, name=name)

    def save_rawstream_csv(self, output_dir, name=''):
        for _, dataset in self.datasets.items():
            if dataset:
                dataset.save_rawstream_csv(output_dir, name=name)

    def save_mat(self, output_dir, name=''):
        for _, dataset in self.datasets.items():
            if dataset:
                dataset.save_mat(output_dir, name=name)

    def save_trigger_mat(self, output_dir, name=''):
        for _, dataset in self.datasets.items():
            if dataset:
                dataset.save_trigger_mat(output_dir, name=name)

    def save_rawstream_mat(self, output_dir, name=''):
        for _, dataset in self.datasets.items():
            if dataset:
                dataset.save_rawstream_mat(output_dir, name=name)

    def get_csv(self):
        csv_files = {}
        for dname, dataset in self.datasets.items():
            if dataset:
                csv_files[dname] = dataset.get_csv()
        return csv_files

    def get_matrix(self):
        dataset_mat = {}
        for dname, dataset in self.datasets.items():
            if dataset:
                dataset_mat[dname] = dataset.get_matrix()
        return dataset_mat

    # If source_dset_name is defined, then we will take
    # a dataset from within this object and merge it into
    # target_dset_name. If dsets_object is defined then we
    # will take all streams, from all datasets, from that
    # object and merge them into target_dset_name.
    def merge(self, target_dset_name, source_dset_name=None, dsets_object=None, keep_raw=False):
        if target_dset_name not in self.datasets:
            raise Exception('Error merging: target_dset_name=' + target_dset_name + ' does not exist in this datastore object.')
        if not self.datasets[target_dset_name]:
            raise Exception(
                'Error merging: target_dset_name=' + target_dset_name + ' is None in this datastore object.')
        merge_into = self.datasets[target_dset_name]

        if source_dset_name:
            merge_src = self.datasets[source_dset_name]
            merge_into.merge(merge_src, keep_raw=keep_raw)
        elif dsets_object:
            for dsetname, dset in dsets_object.datasets.items():
                if not dset: continue;
                if dset.reject: continue;
                merge_into.merge(dset, keep_raw=keep_raw)
        else:
            new_dsets = {}
            for dset in self.datasets:
                if not self.datasets[dset]: continue
                if dset != target_dset_name:
                    merge_into.merge(self.datasets[dset], keep_raw=keep_raw)
                    self.datasets[dset] = None

        keys_to_del = []
        for key, dset in self.datasets.items():
            if not dset: keys_to_del.append(key)

        for key in keys_to_del:
            del self.datasets[key]

        self.datasets[target_dset_name] = merge_into

    # Suggested usage: Use entire dataset in loading and
    # don't load per trial by yourself. Otherwise, you will
    # need to know the data structure of the data obtained
    # and it's highly likely that it could change - breaking
    # your code.
    def load(self, all_data=None):
        if all_data is not None:
            self.all_data = all_data

        data = self.all_data['datasets']
        for dataset_name, dataset in data.items():
            pd = PupilDataset(dataset, dataset_name)
            pd.load()
            self.datasets[dataset_name] = pd
        self.destroy_all_data()

    def reject_datasets(self, dataset_names):
        for dataset in dataset_names:
            if dataset in self.datasets:
                self.datasets[dataset].reject = True

    def reject_trials(self, trials, trigger, datastream_names=[]):
        for _, dset in self.datasets.items():
            if not dset: continue
            dset.reject_trials(trials, trigger, datastream_names=datastream_names)

    def process_trials(self, func, **kwargs):
        for _, dset in self.datasets.items():
            if not dset: continue
            dset.process_trials(func, **kwargs)
import os
import threading
import numpy as np
import time
import datetime
import json
import pickle
from pupillib.core.plib_parser import PLibParser
from pupillib.core.utilities.MPLogger import MultiProcessingLog
from pupillib.core.utilities.utilities import *
from pupillib.core.workers.dataset_worker import PLibDatasetWorker
from pupillib.core.workers.processors.xdfloader_processor import XdfLoaderProcessor
from pupillib.dependencies.xdf.Python.xdf import load_xdf
from pupillib.core.utilities.config_store import ConfigStore
from pupillib.core.utilities.default_dataset_processors import (
    eye_pyrep_to_prim_default,
    gaze_pyrep_to_prim_default,
    gaze_prim_to_pyrep_default
)
from pupillib.core.data_container import PupilDatasets
from threading import Thread
        'gaze_x-pyrep': gaze_prim_to_pyrep_default,
        'gaze_y-pyrep': gaze_prim_to_pyrep_default
    }

    for n in name_list:
        if check_matchers(n, data_entries):
            func = default_proc_functions[n]
            default = matchers[n] # This is the field that we should take data from
            new_data = func(data_entries[default], default, all_data)
            all_data[n] = new_data

    dataset['custom_data'] = custom_data

    new_dict = all_data
    for entry in dataset:
        if entry not in all_data:
            new_dict[entry] = dataset[entry]

    if logger:
        logger.enable_redirect()

    return new_dict


class PupilLibLoader(Thread):
    def __init__(self, config, num=0):
        Thread.__init__(self)
        self.config = config
        self.dataset_path_and_name = self.config['datasets'][num]
        self.datasets = []
        self.dataset = {}
        self.index = num

    # Returns the sampling rate of the given data.
    #   INPUT:
    #       pupil_data - The raw pupil data that was recorded.
    #       pupil_ts   - The raw timestamps from the recorded data.
    #   RETURN:
    #       srate      - The sampling rate.
    @staticmethod
    def pupil_srate(pupil_data, pupil_ts):
        return np.size(pupil_data, 0) / (np.max(pupil_ts) - np.min(pupil_ts))

    def load(self):
        self.dataset = {
            'dir': self.dataset_path_and_name,
            'custom_data': False,
            'merged': 0
        }

        # current_dir = os.getcwd()
        # if not os.path.exists(dataset['dir']):
        #    logger.send('ERROR', """Can't find directory: """ + dataset['dir'])
        #    return None

        # Making artifact directory
        # Set the directory name to the dataset name suffixed with
        # a timestamp.
        # epoch_time = str(int(time.time()))
        # artifacts_dir_name = os.path.normpath(dataset['dir']).split(os.sep)[-1] + "_" + epoch_time
        # print(artifacts_dir_name)
        # print(os.getcwd())

        if '.xdf' in self.dataset['dir']:
            self.dataset['dataname_list'] = self.config['data_name_per_dataset'][self.dataset_path_and_name] if \
                'data_name_per_dataset' in self.config else self.config['dataname_list']
            self.dataset = xdf_pupil_load(self.dataset, self.dataset['dir'], data_num=self.index)
        else:
            self.dataset = pupil_old_load(self.dataset, self.dataset['dir'], data_num=self.index)
        return self.dataset

    def run(self):
        self.load()


class PupilLibRunner(object):
    def __init__(self, config=None, quiet=False):
        self.config = config

        MultiProcessingLog.setQuiet(quiet)

        if not self.config:
            print("Config not provided, ensure that get_build_config is called "
                  "before running.")
            self.loader = None
        else:
            ConfigStore.set_instance(config)
            self.loader = PupilLibLoader(config)
            self.logger = MultiProcessingLog.set_logger_type(self.config['logger'])

        self.loaded_datasets = []
        self.proc_datasets = {}
        self.proc_data = {}
        self.data_store = None

    @property
    def data(self):
        return self.data_store

    @data.setter
    def data(self, val):
        if isinstance(val, PupilDatasets):
            self.data_store = val
            return
        output = type(val)
        try:
            if output in (object,):
                output = output.__class__.__name__
        except:
            pass
        self.logger.warning(
            "PupilLibRunner data value must be a PupilDatasets object, got %s instead." %
            output
        )

    def set_config(self, config):
        if config:
            self.config = config
            ConfigStore.set_instance(config)
            self.loader = PupilLibLoader(config)

            self.logger = MultiProcessingLog.set_logger_type(self.config['logger'])