def get_col_row_array_from_data_record_array(array):
    col, row, _ = get_col_row_tot_array_from_data_record_array(array)
    return col, row
 def convert_data_array(array, filter_func=None, converter_func=None):  # TODO: add copy parameter, otherwise in-place
    '''Filter and convert raw data numpy array (numpy.ndarray).

    Parameters
    ----------
    array : numpy.array
        Raw data array.
    filter_func : function
        Function that takes array and returns true or false for each item in array.
    converter_func : function
        Function that takes array and returns an array or tuple of arrays.

    Returns
    -------
    data_array : numpy.array
        Data numpy array of specified dimension (converter_func) and content (filter_func)
    '''
#     if filter_func != None:
#         if not hasattr(filter_func, '__call__'):
#             raise ValueError('Filter is not callable')
    if filter_func:
        array = array[filter_func(array)]
#     if converter_func != None:
#         if not hasattr(converter_func, '__call__'):
#             raise ValueError('Converter is not callable')
    if converter_func:
        array = converter_func(array)
    return array
 def is_data_record(value):
    return np.logical_and(np.logical_and(np.less_equal(np.bitwise_and(value, 0x00FE0000), 0x00A00000), np.less_equal(np.bitwise_and(value, 0x0001FF00), 0x00015000)), np.logical_and(np.not_equal(np.bitwise_and(value, 0x00FE0000), 0x00000000), np.not_equal(np.bitwise_and(value, 0x0001FF00), 0x00000000)))
 class AnalyzeRawData(object):

    """A class to analyze FE-I4 raw data"""

    def __init__(self, raw_data_file=None, analyzed_data_file=None, create_pdf=True, scan_parameter_name=None):
        '''Initialize the AnalyzeRawData object:
            - The c++ objects (Interpreter, Histogrammer, Clusterizer) are constructed
            - Create one scan parameter table from all provided raw data files
            - Create PdfPages object if needed

        Parameters
        ----------
        raw_data_file : string or list, tuple, set of strings
            Filename or a list of filenames of the raw data file(s) and analyzed_data_file will be overwritten.
            If None and if analyzed_data_file is not None, any existing analyzed_data_file will be opened, otherwise created.
            Filename extension (.h5) does not need to be provided.
        analyzed_data_file : string
            The file name of the output analyzed data file.
            If None and if raw_data_file is not None, the filename will be generated from the raw_data_file.
            Filename extension (.h5) does not need to be provided.
        create_pdf : boolean
            If True, plots will be written into a PDF file. Will be set to False, if raw_data_file is None.
        scan_parameter_name : string or iterable
            The name/names of scan parameter(s) to be used during analysis. If None, the scan parameter
            table is used to extract the scan parameters. Otherwise no scan parameter is set.
        '''
        self.interpreter = PyDataInterpreter()
        self.histogram = PyDataHistograming()

        raw_data_files = []
        if isinstance(raw_data_file, basestring):
            # normalize path
            raw_data_file = os.path.abspath(raw_data_file)
            f_list = analysis_utils.get_data_file_names_from_scan_base(raw_data_file, sort_by_time=True, meta_data_v2=self.interpreter.meta_table_v2)
            if f_list:
                raw_data_files = f_list
            else:
                if os.path.splitext(raw_data_file)[1].lower() != ".h5":
                    raw_data_files.append(os.path.splitext(raw_data_file)[0] + ".h5")
                else:
                    raw_data_files.append(raw_data_file)
        elif isinstance(raw_data_file, (list, tuple, set)):  # iterable of raw data files
            for one_raw_data_file in raw_data_file:
                # normalize path
                one_raw_data_file = os.path.abspath(one_raw_data_file)
                if os.path.splitext(one_raw_data_file)[1].lower() != ".h5":
                    raw_data_files.append(os.path.splitext(one_raw_data_file)[0] + ".h5")
                else:
                    raw_data_files.append(one_raw_data_file)
        else:
            raw_data_files = None

        if analyzed_data_file is not None:
            # normalize path
            analyzed_data_file = os.path.abspath(analyzed_data_file)
            if os.path.splitext(analyzed_data_file)[1].lower() != ".h5":
                self._analyzed_data_file = os.path.splitext(analyzed_data_file)[0] + ".h5"
            else:  # iterable of raw data files
                self._analyzed_data_file = analyzed_data_file
        else:
            if raw_data_file is not None:
                if isinstance(raw_data_file, basestring):
                    self._analyzed_data_file = os.path.splitext(raw_data_file)[0] + '_interpreted.h5'
                else:  # iterable of raw data files
                    commonprefix = os.path.commonprefix(raw_data_files)
                    if commonprefix:
                        # use common string for output filename
                        one_raw_data_file = os.path.abspath(commonprefix)
                    else:
                        # take 1st filename for output filename
                        one_raw_data_file = os.path.abspath(raw_data_files[0])
                    self._analyzed_data_file = os.path.splitext(one_raw_data_file)[0] + '_interpreted.h5'
            else:
                self._analyzed_data_file = None

        # create a scan parameter table from all raw data files
        if raw_data_files is not None:
            self.files_dict = analysis_utils.get_parameter_from_files(raw_data_files, parameters=scan_parameter_name)
            if not analysis_utils.check_parameter_similarity(self.files_dict):
                raise analysis_utils.NotSupportedError('Different scan parameters in multiple files are not supported.')
            self.scan_parameters = analysis_utils.create_parameter_table(self.files_dict)
            scan_parameter_names = analysis_utils.get_scan_parameter_names(self.scan_parameters)
            logging.info('Scan parameter(s) from raw data file(s): %s', (', ').join(scan_parameter_names) if scan_parameter_names else 'None',)
        else:
            self.files_dict = None
            self.scan_parameters = None

        self.out_file_h5 = None
        self.set_standard_settings()
        if self._analyzed_data_file is not None:
            if raw_data_file is None:
                # assume that output file already exists containing analyzed raw data
                self.out_file_h5 = tb.open_file(self._analyzed_data_file, mode="a", title="Interpreted FE-I4 raw data")
            else:
                # raw data files are given, overwrite any existing file
                self.out_file_h5 = tb.open_file(self._analyzed_data_file, mode="w", title="Interpreted FE-I4 raw data")

        if raw_data_file is not None and create_pdf:
            if isinstance(raw_data_file, basestring):
                output_pdf_filename = os.path.splitext(raw_data_file)[0] + ".pdf"
            else:  # iterable of raw data files
                one_raw_data_file = os.path.abspath(raw_data_files[0])
                output_pdf_filename = os.path.splitext(one_raw_data_file)[0] + ".pdf"
            logging.info('Opening output PDF file: %s', output_pdf_filename)
            self.output_pdf = PdfPages(output_pdf_filename)
        else:
            self.output_pdf = None
        self._scan_parameter_name = scan_parameter_name
        self._settings_from_file_set = False  # the scan settings are in a list of files only in the first one, thus set this flag to suppress warning for other files

    def __enter__(self):
        return self

    def __exit__(self, *exc_info):
        self.close()

    def _setup_clusterizer(self):
        # Define all field names and data types
        hit_fields = {'event_number': 'event_number',
                      'column': 'column',
                      'row': 'row',
                      'relative_BCID': 'frame',
                      'tot': 'charge',
                      'LVL1ID': 'LVL1ID',
                      'trigger_number': 'trigger_number',
                      'trigger_time_stamp': 'trigger_time_stamp',
                      'BCID': 'BCID',
                      'TDC': 'TDC',
                      'TDC_time_stamp': 'TDC_time_stamp',
                      'trigger_status': 'trigger_status',
                      'service_record': 'service_record',
                      'event_status': 'event_status'
                      }

        hit_dtype = np.dtype([('event_number', '<i8'),
                              ('trigger_number', '<u4'),
                              ('trigger_time_stamp', '<u4'),
                              ('relative_BCID', '<u1'),
                              ('LVL1ID', '<u2'),
                              ('column', '<u1'),
                              ('row', '<u2'),
                              ('tot', '<u1'),
                              ('BCID', '<u2'),
                              ('TDC', '<u2'),
                              ('TDC_time_stamp', '<u1'),
                              ('trigger_status', '<u1'),
                              ('service_record', '<u4'),
                              ('event_status', '<u2')])

        cluster_fields = {'event_number': 'event_number',
                          'column': 'column',
                          'row': 'row',
                          'size': 'n_hits',
                          'ID': 'ID',
                          'tot': 'charge',
                          'seed_column': 'seed_column',
                          'seed_row': 'seed_row',
                          'mean_column': 'mean_column',
                          'mean_row': 'mean_row'}

        cluster_dtype = np.dtype([('event_number', '<i8'),
                                  ('ID', '<u2'),
                                  ('size', '<u2'),
                                  ('tot', '<u2'),
                                  ('seed_column', '<u1'),
                                  ('seed_row', '<u2'),
                                  ('mean_column', '<f4'),
                                  ('mean_row', '<f4'),
                                  ('event_status', '<u2')])

        # Initialize clusterizer with custom hit/cluster fields
        self.clusterizer = HitClusterizer(
            hit_fields=hit_fields,
            hit_dtype=hit_dtype,
            cluster_fields=cluster_fields,
            cluster_dtype=cluster_dtype,
            min_hit_charge=0,
            max_hit_charge=13,
            column_cluster_distance=2,
            row_cluster_distance=3,
            frame_cluster_distance=2,
            ignore_same_hits=True)

        # Set the cluster event status from the hit event status
        def end_of_cluster_function(hits, clusters, cluster_size, cluster_hit_indices, cluster_index, cluster_id, charge_correction, noisy_pixels, disabled_pixels, seed_hit_index):
            clusters[cluster_index].event_status = hits[seed_hit_index].event_status

        # Set the new function to the clusterizer
        self.clusterizer.set_end_of_cluster_function(end_of_cluster_function)

    def close(self):
        del self.interpreter
        del self.histogram
        del self.clusterizer
        self._close_h5()
        self._close_pdf()

    def _close_h5(self):
        if self.is_open(self.out_file_h5):
            self.out_file_h5.close()
            self.out_file_h5 = None

    def _close_pdf(self):
        if self.output_pdf is not None:
            logging.info('Closing output PDF file: %s', str(self.output_pdf._file.fh.name))
            self.output_pdf.close()
            self.output_pdf = None

    def set_standard_settings(self):
        '''Set all settings to their standard values.
        '''
        if self.is_open(self.out_file_h5):
            self.out_file_h5.close()
        self.out_file_h5 = None
        self._setup_clusterizer()
        self.chunk_size = 3000000
        self.n_injections = None
        self.trig_count = 0  # 0 trig_count = 16 BCID per trigger
        self.max_tot_value = 13
        self.vcal_c0, self.vcal_c1 = None, None
        self.c_low, self.c_mid, self.c_high = None, None, None
        self.c_low_mask, self.c_high_mask = None, None
        self._filter_table = tb.Filters(complib='blosc', complevel=5, fletcher32=False)
        warnings.simplefilter("ignore", OptimizeWarning)
        self.meta_event_index = None
        self.fei4b = False
        self.create_hit_table = False
        self.create_empty_event_hits = False
        self.create_meta_event_index = True
        self.create_tot_hist = True
        self.create_mean_tot_hist = False
        self.create_tot_pixel_hist = True
        self.create_rel_bcid_hist = True
        self.correct_corrupted_data = False
        self.create_error_hist = True
        self.create_service_record_hist = True
        self.create_occupancy_hist = True
        self.create_meta_word_index = False
        self.create_source_scan_hist = False
        self.create_tdc_hist = False
        self.create_tdc_counter_hist = False
        self.create_tdc_pixel_hist = False
        self.create_trigger_error_hist = False
        self.create_threshold_hists = False
        self.create_threshold_mask = True  # Threshold/noise histogram mask: masking all pixels out of bounds
        self.create_fitted_threshold_mask = True  # Fitted threshold/noise histogram mask: masking all pixels out of bounds
        self.create_fitted_threshold_hists = False
        self.create_cluster_hit_table = False
        self.create_cluster_table = False
        self.create_cluster_size_hist = False
        self.create_cluster_tot_hist = False
        self.align_at_trigger = False  # use the trigger word to align the events
        self.align_at_tdc = False  # use the trigger word to align the events
        self.trigger_data_format = 0  # 0: 31bit trigger number, 1: 31bit trigger time stamp, 2: 15bit trigger time stamp + 16bit trigger number
        self.use_tdc_trigger_time_stamp = False  # the tdc time stamp is the difference between trigger and tdc rising edge
        self.max_tdc_delay = 255
        self.max_trigger_number = 2 ** 16 - 1
        self.set_stop_mode = False  # The FE is read out with stop mode, therefore the BCID plot is different

    def reset(self):
        '''Reset the c++ libraries for new analysis.
        '''
        self.interpreter.reset()
        self.histogram.reset()

    @property
    def chunk_size(self):
        return self._chunk_size

    @chunk_size.setter
    def chunk_size(self, value):
        self.interpreter.set_hit_array_size(2 * value)  # worst case: one raw data word becoming 2 hit words
        self._chunk_size = value

    @property
    def create_hit_table(self):
        return self._create_hit_table

    @create_hit_table.setter
    def create_hit_table(self, value):
        self._create_hit_table = value

    @property
    def create_empty_event_hits(self):
        return self._create_empty_event_hits

    @create_empty_event_hits.setter
    def create_empty_event_hits(self, value):
        self._create_empty_event_hits = value
        self.interpreter.create_empty_event_hits(value)

    @property
    def create_occupancy_hist(self):
        return self._create_occupancy_hist

    @create_occupancy_hist.setter
    def create_occupancy_hist(self, value):
        self._create_occupancy_hist = value
        self.histogram.create_occupancy_hist(value)

    @property
    def create_mean_tot_hist(self):
        return self._create_mean_tot_hist

    @create_mean_tot_hist.setter
    def create_mean_tot_hist(self, value):
        self._create_mean_tot_hist = value
        self.histogram.create_mean_tot_hist(value)

    @property
    def create_source_scan_hist(self):
        return self._create_source_scan_hist

    @create_source_scan_hist.setter
    def create_source_scan_hist(self, value):
        self._create_source_scan_hist = value

    @property
    def create_tot_hist(self):
        return self.create_tot_hist

    @create_tot_hist.setter
    def create_tot_hist(self, value):
        self._create_tot_hist = value
        self.histogram.create_tot_hist(value)

    @property
    def create_tdc_hist(self):
        return self._create_tdc_hist

    @create_tdc_hist.setter
    def create_tdc_hist(self, value):
        self._create_tdc_hist = value
        self.histogram.create_tdc_hist(value)

    @property
    def create_tdc_pixel_hist(self):
        return self._create_tdc_pixel_hist

    @create_tdc_pixel_hist.setter
    def create_tdc_pixel_hist(self, value):
        self._create_tdc_pixel_hist = value
        self.histogram.create_tdc_pixel_hist(value)

    @property
    def create_tot_pixel_hist(self):
        return self._create_tot_pixel_hist

    @create_tot_pixel_hist.setter
    def create_tot_pixel_hist(self, value):
        self._create_tot_pixel_hist = value
        self.histogram.create_tot_pixel_hist(value)

    @property
    def create_rel_bcid_hist(self):
        return self._create_rel_bcid_hist

    @create_rel_bcid_hist.setter
    def create_rel_bcid_hist(self, value):
        self._create_rel_bcid_hist = value
        self.histogram.create_rel_bcid_hist(value)

    @property
    def create_threshold_hists(self):
        return self._create_threshold_hists

    @create_threshold_hists.setter
    def create_threshold_hists(self, value):
        self._create_threshold_hists = value

    @property
    def create_threshold_mask(self):
        return self._create_threshold_mask

    @create_threshold_mask.setter
    def create_threshold_mask(self, value):
        self._create_threshold_mask = value

    @property
    def create_fitted_threshold_mask(self):
        return self._create_fitted_threshold_mask

    @create_fitted_threshold_mask.setter
    def create_fitted_threshold_mask(self, value):
        self._create_fitted_threshold_mask = value

    @property
    def create_fitted_threshold_hists(self):
        return self._create_fitted_threshold_hists

    @create_fitted_threshold_hists.setter
    def create_fitted_threshold_hists(self, value):
        self._create_fitted_threshold_hists = value

    @property
    def correct_corrupted_data(self):
        return self._correct_corrupted_data

    @correct_corrupted_data.setter
    def correct_corrupted_data(self, value):
        self._correct_corrupted_data = value

    @property
    def create_error_hist(self):
        return self._create_error_hist

    @create_error_hist.setter
    def create_error_hist(self, value):
        self._create_error_hist = value

    @property
    def create_trigger_error_hist(self):
        return self._create_trigger_error_hist

    @create_trigger_error_hist.setter
    def create_trigger_error_hist(self, value):
        self._create_trigger_error_hist = value

    @property
    def create_service_record_hist(self):
        return self._create_service_record_hist

    @create_service_record_hist.setter
    def create_service_record_hist(self, value):
        self._create_service_record_hist = value

    @property
    def create_tdc_counter_hist(self):
        return self._create_tdc_counter_hist

    @create_tdc_counter_hist.setter
    def create_tdc_counter_hist(self, value):
        self._create_tdc_counter_hist = value

    @property
    def create_meta_event_index(self):
        return self._create_meta_event_index

    @create_meta_event_index.setter
    def create_meta_event_index(self, value):
        self._create_meta_event_index = value

    @property
    def create_meta_word_index(self):
        return self._create_meta_word_index

    @create_meta_word_index.setter
    def create_meta_word_index(self, value):
        self._create_meta_word_index = value
        self.interpreter.create_meta_data_word_index(value)

    @property
    def fei4b(self):
        return self._fei4b

    @fei4b.setter
    def fei4b(self, value):
        self._fei4b = value
        self.interpreter.set_FEI4B(value)

    @property
    def n_injections(self):
        """Get the numbers of injections per pixel."""
        return self._n_injection

    @n_injections.setter
    def n_injections(self, value):
        """Set the numbers of injections per pixel."""
        self._n_injection = value

    @property
    def trig_count(self):
        """Get the numbers of BCIDs (usually 16) of one event."""
        return self._trig_count

    @trig_count.setter
    def trig_count(self, value):
        """Set the numbers of BCIDs (usually 16) of one event."""
        self._trig_count = 16 if value == 0 else value
        self.interpreter.set_trig_count(self._trig_count)

    @property
    def max_tot_value(self):
        """Get maximum ToT value that is considered to be a hit"""
        return self._max_tot_value

    @max_tot_value.setter
    def max_tot_value(self, value):
        """Set maximum ToT value that is considered to be a hit"""
        self._max_tot_value = value
        self.interpreter.set_max_tot(self._max_tot_value)
        self.histogram.set_max_tot(self._max_tot_value)
        self.clusterizer.set_max_hit_charge(self._max_tot_value)

    @property
    def create_cluster_hit_table(self):
        return self._create_cluster_hit_table

    @create_cluster_hit_table.setter
    def create_cluster_hit_table(self, value):
        self._create_cluster_hit_table = value

    @property
    def create_cluster_table(self):
        return self._create_cluster_table

    @create_cluster_table.setter
    def create_cluster_table(self, value):
        self._create_cluster_table = value

    @property
    def create_cluster_size_hist(self):
        return self._create_cluster_size_hist

    @create_cluster_size_hist.setter
    def create_cluster_size_hist(self, value):
        self._create_cluster_size_hist = value

    @property
    def create_cluster_tot_hist(self):
        return self._create_cluster_tot_hist

    @create_cluster_tot_hist.setter
    def create_cluster_tot_hist(self, value):
        self._create_cluster_tot_hist = value

    @property
    def align_at_trigger(self):
        return self._align_at_trigger

    @align_at_trigger.setter
    def align_at_trigger(self, value):
        self._align_at_trigger = value
        self.interpreter.align_at_trigger(value)

    @property
    def align_at_tdc(self):
        return self._align_at_tdc

    @align_at_tdc.setter
    def align_at_tdc(self, value):
        self._align_at_tdc = value
        self.interpreter.align_at_tdc(value)

    @property
    def trigger_data_format(self):
        return self._trigger_data_format

    @trigger_data_format.setter
    def trigger_data_format(self, value):
        self._trigger_data_format = value
        self.interpreter.set_trigger_data_format(value)

    @property
    def use_tdc_trigger_time_stamp(self):
        return self._use_tdc_trigger_time_stamp

    @use_tdc_trigger_time_stamp.setter
    def use_tdc_trigger_time_stamp(self, value):
        self._use_tdc_trigger_time_stamp = value
        self.interpreter.use_tdc_trigger_time_stamp(value)

    @property
    def max_tdc_delay(self):
        return self._max_tdc_delay

    @max_tdc_delay.setter
    def max_tdc_delay(self, value):
        self._max_tdc_delay = value
        self.interpreter.set_max_tdc_delay(value)

    @property
    def max_trigger_number(self):
        return self._max_trigger_number

    @max_trigger_number.setter
    def max_trigger_number(self, value):
        self._max_trigger_number = value
        self.interpreter.set_max_trigger_number(value)

    @property
    def set_stop_mode(self):
        return self._set_stop_mode

    @set_stop_mode.setter
    def set_stop_mode(self, value):
        self._set_stop_mode = value

    def interpret_word_table(self, analyzed_data_file=None, use_settings_from_file=True, fei4b=None):
        '''Interprets the raw data word table of all given raw data files with the c++ library.
        Creates the h5 output file and PDF plots.

        Parameters
        ----------
        analyzed_data_file : string
            The file name of the output analyzed data file. If None, the output analyzed data file
            specified during initialization is taken.
        use_settings_from_file : boolean
            True if the needed parameters should be extracted from the raw data file
        fei4b : boolean
            True if the raw data is from FE-I4B.
        '''

        logging.info('Interpreting raw data file(s): ' + (', ').join(self.files_dict.keys()))

        if self._create_meta_word_index:
            meta_word = np.empty((self._chunk_size,), dtype=dtype_from_descr(data_struct.MetaInfoWordTable))
            self.interpreter.set_meta_data_word_index(meta_word)
        self.interpreter.reset_event_variables()
        self.interpreter.reset_counters()

        self.meta_data = analysis_utils.combine_meta_data(self.files_dict, meta_data_v2=self.interpreter.meta_table_v2)

        if self.meta_data is None or self.meta_data.shape[0] == 0:
            raise analysis_utils.IncompleteInputError('Meta data is empty. Stopping interpretation.')

        self.interpreter.set_meta_data(self.meta_data)  # tell interpreter the word index per readout to be able to calculate the event number per read out
        meta_data_size = self.meta_data.shape[0]
        self.meta_event_index = np.zeros((meta_data_size,), dtype=[('metaEventIndex', np.uint64)])  # this array is filled by the interpreter and holds the event number per read out
        self.interpreter.set_meta_event_data(self.meta_event_index)  # tell the interpreter the data container to write the meta event index to

        if self.scan_parameters is None:
            self.histogram.set_no_scan_parameter()
        else:
            self.scan_parameter_index = analysis_utils.get_scan_parameters_index(self.scan_parameters)  # a array that labels unique scan parameter combinations
            self.histogram.add_scan_parameter(self.scan_parameter_index)  # just add an index for the different scan parameter combinations

        if self._create_cluster_size_hist:  # Cluster size result histogram
            self._cluster_size_hist = np.zeros(shape=(6, ), dtype=np.uint32)

        if self._create_cluster_tot_hist:  # Cluster tot/size result histogram
            self._cluster_tot_hist = np.zeros(shape=(16, 6), dtype=np.uint32)

        close_analyzed_data_file = False
        if analyzed_data_file is not None:  # if an output file name is specified create new file for analyzed data
            if self.is_open(self.out_file_h5) and os.path.abspath(analyzed_data_file) == os.path.abspath(self.out_file_h5.filename):
                out_file_h5 = self.out_file_h5
            else:
                # normalize path
                analyzed_data_file = os.path.abspath(analyzed_data_file)
                if os.path.splitext(analyzed_data_file)[1].lower() != ".h5":
                    analyzed_data_file = os.path.splitext(analyzed_data_file)[0] + ".h5"
                out_file_h5 = tb.open_file(analyzed_data_file, mode="w", title="Interpreted FE-I4 raw data")
                close_analyzed_data_file = True
        elif self.is_open(self.out_file_h5):
                out_file_h5 = self.out_file_h5
        else:
            out_file_h5 = None

        tmp_out_file_h5 = self.out_file_h5
        if not self.is_open(self.out_file_h5) and self.is_open(out_file_h5):
            close_analyzed_data_file = False
            tmp_out_file_h5 = out_file_h5
        self.out_file_h5 = out_file_h5
        if self.is_open(self.out_file_h5):
            self._analyzed_data_file = self.out_file_h5.filename
        else:
            self._analyzed_data_file is None

        if self._analyzed_data_file is not None:
            if self._create_hit_table is True:
                description = data_struct.HitInfoTable().columns.copy()
                hit_table = self.out_file_h5.create_table(self.out_file_h5.root, name='Hits', description=description, title='hit_data', filters=self._filter_table, chunkshape=(self._chunk_size / 100,))
            if self._create_meta_word_index is True:
                meta_word_index_table = self.out_file_h5.create_table(self.out_file_h5.root, name='EventMetaData', description=data_struct.MetaInfoWordTable, title='event_meta_data', filters=self._filter_table, chunkshape=(self._chunk_size / 10,))
            if self._create_cluster_table:
                cluster_table = self.out_file_h5.create_table(self.out_file_h5.root, name='Cluster', description=data_struct.ClusterInfoTable, title='Cluster data', filters=self._filter_table, expectedrows=self._chunk_size)
            if self._create_cluster_hit_table:
                description = data_struct.ClusterHitInfoTable().columns.copy()
                cluster_hit_table = self.out_file_h5.create_table(self.out_file_h5.root, name='ClusterHits', description=description, title='cluster_hit_data', filters=self._filter_table, expectedrows=self._chunk_size)

        logging.info("Interpreting raw data...")
        progress_bar = progressbar.ProgressBar(widgets=['', progressbar.Percentage(), ' ', progressbar.Bar(marker='*', left='|', right='|'), ' ', progressbar.AdaptiveETA()], maxval=analysis_utils.get_total_n_data_words(self.files_dict), term_width=80)
        progress_bar.start()
        total_words = 0

        for file_index, raw_data_file in enumerate(self.files_dict.keys()):  # loop over all raw data files
            self.interpreter.reset_meta_data_counter()
            with tb.open_file(raw_data_file, mode="r") as in_file_h5:
                if use_settings_from_file:
                    self._deduce_settings_from_file(in_file_h5)
                else:
                    self.fei4b = fei4b
                if self.interpreter.meta_table_v2:
                    index_start = in_file_h5.root.meta_data.read(field='index_start')
                    index_stop = in_file_h5.root.meta_data.read(field='index_stop')
                else:
                    index_start = in_file_h5.root.meta_data.read(field='start_index')
                    index_stop = in_file_h5.root.meta_data.read(field='stop_index')
                bad_word_index = set()

                # Check for bad data
                if self._correct_corrupted_data:
                    tw = 2147483648  # trigger word
                    dh = 15269888  # data header
                    is_fe_data_header = logical_and(is_fe_word, is_data_header)
                    found_first_trigger = False
                    readout_slices = np.column_stack((index_start, index_stop))
                    previous_prepend_data_headers = None
                    prepend_data_headers = None
                    last_good_readout_index = None
                    last_index_with_event_data = None
                    for read_out_index, (index_start, index_stop) in enumerate(readout_slices):
                        try:
                            raw_data = in_file_h5.root.raw_data.read(index_start, index_stop)
                        except OverflowError, e:
                            pass
                        except tb.exceptions.HDF5ExtError:
                            break
                        # previous data chunk had bad data, check for good data
                        if (index_start - 1) in bad_word_index:
                            bad_data, current_prepend_data_headers, _ , _ = check_bad_data(raw_data, prepend_data_headers=1, trig_count=None)
                            if bad_data:
                                bad_word_index = bad_word_index.union(range(index_start, index_stop))
                            else:
#                                 logging.info("found good data in %s from index %d to %d (chunk %d, length %d)" % (in_file_h5.filename, index_start, index_stop, read_out_index, (index_stop - index_start)))
                                if last_good_readout_index + 1 == read_out_index - 1:
                                    logging.warning("found bad data in %s from index %d to %d (chunk %d, length %d)" % (in_file_h5.filename, readout_slices[last_good_readout_index][1], readout_slices[read_out_index - 1][1], last_good_readout_index + 1, (readout_slices[read_out_index - 1][1] - readout_slices[last_good_readout_index][1])))
                                else:
                                    logging.warning("found bad data in %s from index %d to %d (chunk %d to %d, length %d)" % (in_file_h5.filename, readout_slices[last_good_readout_index][1], readout_slices[read_out_index - 1][1], last_good_readout_index + 1, read_out_index - 1, (readout_slices[read_out_index - 1][1] - readout_slices[last_good_readout_index][1])))
                                previous_good_raw_data = in_file_h5.root.raw_data.read(readout_slices[last_good_readout_index][0], readout_slices[last_good_readout_index][1] - 1)
                                previous_bad_raw_data = in_file_h5.root.raw_data.read(readout_slices[last_good_readout_index][1] - 1, readout_slices[read_out_index - 1][1])
                                fixed_raw_data, _ = fix_raw_data(previous_bad_raw_data, lsb_byte=None)
                                fixed_raw_data = np.r_[previous_good_raw_data, fixed_raw_data, raw_data]
                                _, prepend_data_headers, n_triggers, n_dh = check_bad_data(fixed_raw_data, prepend_data_headers=previous_prepend_data_headers, trig_count=self.trig_count)
                                last_good_readout_index = read_out_index
                                if n_triggers != 0 or n_dh != 0:
                                    last_index_with_event_data = read_out_index
                                    last_event_data_prepend_data_headers = prepend_data_headers
                                fixed_previous_raw_data = np.r_[previous_good_raw_data, fixed_raw_data]
                                _, previous_prepend_data_headers, _ , _ = check_bad_data(fixed_previous_raw_data, prepend_data_headers=previous_prepend_data_headers, trig_count=self.trig_count)
                        # check for bad data
                        else:
                            # workaround for first data chunk, might have missing trigger in some rare cases (already fixed in firmware)
                            if read_out_index == 0 and (np.any(is_trigger_word(raw_data) >= 1) or np.any(is_fe_data_header(raw_data) >= 1)):
                                bad_data, current_prepend_data_headers, n_triggers , n_dh = check_bad_data(raw_data, prepend_data_headers=1, trig_count=None)
                                # check for full last event in data
                                if current_prepend_data_headers == self.trig_count:
                                    current_prepend_data_headers = None
                            # usually check for bad data happens here
                            else:
                                bad_data, current_prepend_data_headers, n_triggers , n_dh = check_bad_data(raw_data, prepend_data_headers=prepend_data_headers, trig_count=self.trig_count)

                            # do additional check with follow up data chunk and decide whether current chunk is defect or not
                            if bad_data:
                                if read_out_index == 0:
                                    fixed_raw_data_chunk, _ = fix_raw_data(raw_data, lsb_byte=None)
                                    fixed_raw_data_list = [fixed_raw_data_chunk]
                                else:
                                    previous_raw_data = in_file_h5.root.raw_data.read(*readout_slices[read_out_index - 1])
                                    raw_data_with_previous_data_word = np.r_[previous_raw_data[-1], raw_data]
                                    fixed_raw_data_chunk, _ = fix_raw_data(raw_data_with_previous_data_word, lsb_byte=None)
                                    fixed_raw_data = np.r_[previous_raw_data[:-1], fixed_raw_data_chunk]
                                    # last data word of chunk before broken chunk migh be a trigger word or data header which cannot be recovered
                                    fixed_raw_data_with_tw = np.r_[previous_raw_data[:-1], tw, fixed_raw_data_chunk]
                                    fixed_raw_data_with_dh = np.r_[previous_raw_data[:-1], dh, fixed_raw_data_chunk]
                                    fixed_raw_data_list = [fixed_raw_data, fixed_raw_data_with_tw, fixed_raw_data_with_dh]
                                bad_fixed_data, _, _ , _ = check_bad_data(fixed_raw_data_with_dh, prepend_data_headers=previous_prepend_data_headers, trig_count=self.trig_count)
                                bad_fixed_data = map(lambda data: check_bad_data(data, prepend_data_headers=previous_prepend_data_headers, trig_count=self.trig_count)[0], fixed_raw_data_list)
                                if not all(bad_fixed_data): # good fixed data
                                    # last word in chunk before currrent chunk is also bad
                                    if index_start != 0:
                                        bad_word_index.add(index_start - 1)
                                    # adding all word from current chunk
                                    bad_word_index = bad_word_index.union(range(index_start, index_stop))
                                    last_good_readout_index = read_out_index - 1
                                else:
                                    # a previous chunk might be broken and the last data word becomes a trigger word, so do additional checks
                                    if last_index_with_event_data and last_event_data_prepend_data_headers != read_out_index:
                                        before_bad_raw_data = in_file_h5.root.raw_data.read(readout_slices[last_index_with_event_data - 1][0], readout_slices[last_index_with_event_data - 1][1] - 1)
                                        previous_bad_raw_data = in_file_h5.root.raw_data.read(readout_slices[last_index_with_event_data][0] - 1, readout_slices[last_index_with_event_data][1])
                                        fixed_raw_data, _ = fix_raw_data(previous_bad_raw_data, lsb_byte=None)
                                        previous_good_raw_data = in_file_h5.root.raw_data.read(readout_slices[last_index_with_event_data][1], readout_slices[read_out_index - 1][1])
                                        fixed_raw_data = np.r_[before_bad_raw_data, fixed_raw_data, previous_good_raw_data, raw_data]
                                        bad_fixed_previous_data, current_prepend_data_headers, _, _ = check_bad_data(fixed_raw_data, prepend_data_headers=last_event_data_prepend_data_headers, trig_count=self.trig_count)
                                        if not bad_fixed_previous_data:
                                            logging.warning("found bad data in %s from index %d to %d (chunk %d, length %d)" % (in_file_h5.filename, readout_slices[last_index_with_event_data][0], readout_slices[last_index_with_event_data][1], last_index_with_event_data, (readout_slices[last_index_with_event_data][1] - readout_slices[last_index_with_event_data][0])))
                                            bad_word_index = bad_word_index.union(range(readout_slices[last_index_with_event_data][0] - 1, readout_slices[last_index_with_event_data][1]))
                                        else:
                                            logging.warning("found bad data which cannot be corrected in %s from index %d to %d (chunk %d, length %d)" % (in_file_h5.filename, index_start, index_stop, read_out_index, (index_stop - index_start)))
                                    else:
                                        logging.warning("found bad data which cannot be corrected in %s from index %d to %d (chunk %d, length %d)" % (in_file_h5.filename, index_start, index_stop, read_out_index, (index_stop - index_start)))
                            if n_triggers != 0 or n_dh != 0:
                                last_index_with_event_data = read_out_index
                                last_event_data_prepend_data_headers = prepend_data_headers
                            if not bad_data or (bad_data and bad_fixed_data):
                                previous_prepend_data_headers = prepend_data_headers
                                prepend_data_headers = current_prepend_data_headers

                    consecutive_bad_words_list = consecutive(sorted(bad_word_index))

                lsb_byte = None
                # Loop over raw data in chunks
                for word_index in range(0, in_file_h5.root.raw_data.shape[0], self._chunk_size):  # loop over all words in the actual raw data file
                    try:
                        raw_data = in_file_h5.root.raw_data.read(word_index, word_index + self._chunk_size)
                    except OverflowError, e:
                        logging.error('%s: 2^31 xrange() limitation in 32-bit Python', e)
                    except tb.exceptions.HDF5ExtError:
                        logging.warning('Raw data file %s has missing raw data. Continue raw data analysis.', in_file_h5.filename)
                        break
                    total_words += raw_data.shape[0]
                    # fix bad data
                    if self._correct_corrupted_data:
                        # increase word shift for every bad data chunk in raw data chunk
                        word_shift = 0
                        chunk_indices = np.arange(word_index, word_index + self._chunk_size)
                        for consecutive_bad_word_indices in consecutive_bad_words_list:
                            selected_words = np.intersect1d(consecutive_bad_word_indices, chunk_indices, assume_unique=True)
                            if selected_words.shape[0]:
                                fixed_raw_data, lsb_byte = fix_raw_data(raw_data[selected_words - word_index - word_shift], lsb_byte=lsb_byte)
                                raw_data = np.r_[raw_data[:selected_words[0] - word_index - word_shift], fixed_raw_data, raw_data[selected_words[-1] - word_index + 1 - word_shift:]]
                                # check if last word of bad data chunk in current raw data chunk
                                if consecutive_bad_word_indices[-1] in selected_words:
                                    lsb_byte = None
                                    # word shift by removing data word at the beginning of each defect chunk
                                    word_shift += 1
                                # bad data chunk is at the end of current raw data chunk
                                else:
                                    break

                    self.interpreter.interpret_raw_data(raw_data)  # interpret the raw data
                    # store remaining buffered event in the interpreter at the end of the last file
                    if file_index == len(self.files_dict.keys()) - 1 and word_index == range(0, in_file_h5.root.raw_data.shape[0], self._chunk_size)[-1]:  # store hits of the latest event of the last file
                        self.interpreter.store_event()
                    hits = self.interpreter.get_hits()
                    if self.scan_parameters is not None:
                        nEventIndex = self.interpreter.get_n_meta_data_event()
                        self.histogram.add_meta_event_index(self.meta_event_index, nEventIndex)
                    if self.is_histogram_hits():
                        self.histogram_hits(hits)
                    if self.is_cluster_hits():
                        cluster_hits, clusters = self.cluster_hits(hits)
                        if self._create_cluster_hit_table:
                            cluster_hit_table.append(cluster_hits)
                        if self._create_cluster_table:
                            cluster_table.append(clusters)
                        if self._create_cluster_size_hist:
                            if clusters['size'].shape[0] > 0 and np.max(clusters['size']) + 1 > self._cluster_size_hist.shape[0]:
                                self._cluster_size_hist.resize(np.max(clusters['size']) + 1)
                            self._cluster_size_hist += fast_analysis_utils.hist_1d_index(clusters['size'], shape=self._cluster_size_hist.shape)
                        if self._create_cluster_tot_hist:
                            if clusters['tot'].shape[0] > 0 and np.max(clusters['tot']) + 1 > self._cluster_tot_hist.shape[0]:
                                self._cluster_tot_hist.resize((np.max(clusters['tot']) + 1, self._cluster_tot_hist.shape[1]))
                            if clusters['size'].shape[0] > 0 and np.max(clusters['size']) + 1 > self._cluster_tot_hist.shape[1]:
                                self._cluster_tot_hist.resize((self._cluster_tot_hist.shape[0], np.max(clusters['size']) + 1))
                            self._cluster_tot_hist += fast_analysis_utils.hist_2d_index(clusters['tot'], clusters['size'], shape=self._cluster_tot_hist.shape)
                    if self._analyzed_data_file is not None and self._create_hit_table:
                        hit_table.append(hits)
                    if self._analyzed_data_file is not None and self._create_meta_word_index:
                        size = self.interpreter.get_n_meta_data_word()
                        meta_word_index_table.append(meta_word[:size])

                    if total_words <= progress_bar.maxval:  # Otherwise exception is thrown
                        progress_bar.update(total_words)
                    self.out_file_h5.flush()
        progress_bar.finish()
        self._create_additional_data()

        if close_analyzed_data_file:
            self.out_file_h5.close()
            self.out_file_h5 = None
        self.out_file_h5 = out_file_h5
        if self.is_open(self.out_file_h5):
            self._analyzed_data_file = self.out_file_h5.filename
        else:
            self._analyzed_data_file = None

    def _create_additional_data(self):
        logging.info('Creating selected event histograms...')
        if self._analyzed_data_file is not None and self._create_meta_event_index:
            meta_data_size = self.meta_data.shape[0]
            n_event_index = self.interpreter.get_n_meta_data_event()
            if meta_data_size == n_event_index:
                if self.interpreter.meta_table_v2:
                    description = data_struct.MetaInfoEventTableV2().columns.copy()
                else:
                    description = data_struct.MetaInfoEventTable().columns.copy()
                last_pos = len(description)
                if self.scan_parameters is not None:  # add additional column with the scan parameter
                    for index, scan_par_name in enumerate(self.scan_parameters.dtype.names):
                        dtype, _ = self.scan_parameters.dtype.fields[scan_par_name][:2]
                        description[scan_par_name] = Col.from_dtype(dtype, dflt=0, pos=last_pos + index)
                meta_data_out_table = self.out_file_h5.create_table(self.out_file_h5.root, name='meta_data', description=description, title='MetaData', filters=self._filter_table)
                entry = meta_data_out_table.row
                for i in range(0, n_event_index):
                    if self.interpreter.meta_table_v2:
                        entry['event_number'] = self.meta_event_index[i][0]  # event index
                        entry['timestamp_start'] = self.meta_data[i][3]  # timestamp
                        entry['timestamp_stop'] = self.meta_data[i][4]  # timestamp
                        entry['error_code'] = self.meta_data[i][5]  # error code
                    else:
                        entry['event_number'] = self.meta_event_index[i][0]  # event index
                        entry['time_stamp'] = self.meta_data[i][3]  # time stamp
                        entry['error_code'] = self.meta_data[i][4]  # error code
                    if self.scan_parameters is not None:  # scan parameter if available
                        for scan_par_name in self.scan_parameters.dtype.names:
                            entry[scan_par_name] = self.scan_parameters[scan_par_name][i]
                    entry.append()
                    self.out_file_h5.flush()
                if self.scan_parameters is not None:
                    logging.info("Save meta data with scan parameter " + scan_par_name)
            else:
                logging.error('Meta data analysis failed')
        if self._create_service_record_hist:
            self.service_record_hist = self.interpreter.get_service_records_counters()
            if self._analyzed_data_file is not None:
                service_record_hist_table = self.out_file_h5.create_carray(self.out_file_h5.root, name='HistServiceRecord', title='Service Record Histogram', atom=tb.Atom.from_dtype(self.service_record_hist.dtype), shape=self.service_record_hist.shape, filters=self._filter_table)
                service_record_hist_table[:] = self.service_record_hist
        if self._create_tdc_counter_hist:
            self.tdc_counter_hist = self.interpreter.get_tdc_counters()
            if self._analyzed_data_file is not None:
                tdc_counter_hist = self.out_file_h5.create_carray(self.out_file_h5.root, name='HistTdcCounter', title='All Tdc word counter values', atom=tb.Atom.from_dtype(self.tdc_counter_hist.dtype), shape=self.tdc_counter_hist.shape, filters=self._filter_table)
                tdc_counter_hist[:] = self.tdc_counter_hist
        if self._create_error_hist:
            self.error_counter_hist = self.interpreter.get_error_counters()
            if self._analyzed_data_file is not None:
                error_counter_hist_table = self.out_file_h5.create_carray(self.out_file_h5.root, name='HistErrorCounter', title='Error Counter Histogram', atom=tb.Atom.from_dtype(self.error_counter_hist.dtype), shape=self.error_counter_hist.shape, filters=self._filter_table)
                error_counter_hist_table[:] = self.error_counter_hist
        if self._create_trigger_error_hist:
            self.trigger_error_counter_hist = self.interpreter.get_trigger_error_counters()
            if self._analyzed_data_file is not None:
                trigger_error_counter_hist_table = self.out_file_h5.create_carray(self.out_file_h5.root, name='HistTriggerErrorCounter', title='Trigger Error Counter Histogram', atom=tb.Atom.from_dtype(self.trigger_error_counter_hist.dtype), shape=self.trigger_error_counter_hist.shape, filters=self._filter_table)
                trigger_error_counter_hist_table[:] = self.trigger_error_counter_hist

        self._create_additional_hit_data()
        self._create_additional_cluster_data()

    def _create_additional_hit_data(self, safe_to_file=True):
        logging.info('Create selected hit histograms')
        if self._create_tot_hist:
            self.tot_hist = self.histogram.get_tot_hist()
            if self._analyzed_data_file is not None and safe_to_file:
                tot_hist_table = self.out_file_h5.create_carray(self.out_file_h5.root, name='HistTot', title='ToT Histogram', atom=tb.Atom.from_dtype(self.tot_hist.dtype), shape=self.tot_hist.shape, filters=self._filter_table)
                tot_hist_table[:] = self.tot_hist
        if self._create_tot_pixel_hist:
            if self._analyzed_data_file is not None and safe_to_file:
                self.tot_pixel_hist_array = np.swapaxes(self.histogram.get_tot_pixel_hist(), 0, 1)  # swap axis col,row, parameter --> row, col, parameter
                tot_pixel_hist_out = self.out_file_h5.create_carray(self.out_file_h5.root, name='HistTotPixel', title='Tot Pixel Histogram', atom=tb.Atom.from_dtype(self.tot_pixel_hist_array.dtype), shape=self.tot_pixel_hist_array.shape, filters=self._filter_table)
                tot_pixel_hist_out[:] = self.tot_pixel_hist_array
        if self._create_tdc_hist:
            self.tdc_hist = self.histogram.get_tdc_hist()
            if self._analyzed_data_file is not None and safe_to_file:
                tdc_hist_table = self.out_file_h5.create_carray(self.out_file_h5.root, name='HistTdc', title='Tdc Histogram', atom=tb.Atom.from_dtype(self.tdc_hist.dtype), shape=self.tdc_hist.shape, filters=self._filter_table)
                tdc_hist_table[:] = self.tdc_hist
        if self._create_tdc_pixel_hist:
            if self._analyzed_data_file is not None and safe_to_file:
                self.tdc_pixel_hist_array = np.swapaxes(self.histogram.get_tdc_pixel_hist(), 0, 1)  # swap axis col,row, parameter --> row, col, parameter
                tdc_pixel_hist_out = self.out_file_h5.create_carray(self.out_file_h5.root, name='HistTdcPixel', title='Tdc Pixel Histogram', atom=tb.Atom.from_dtype(self.tdc_pixel_hist_array.dtype), shape=self.tdc_pixel_hist_array.shape, filters=self._filter_table)
                tdc_pixel_hist_out[:] = self.tdc_pixel_hist_array
        if self._create_rel_bcid_hist:
            self.rel_bcid_hist = self.histogram.get_rel_bcid_hist()
            if self._analyzed_data_file is not None and safe_to_file:
                if not self.set_stop_mode:
                    rel_bcid_hist_table = self.out_file_h5.create_carray(self.out_file_h5.root, name='HistRelBcid', title='relative BCID Histogram', atom=tb.Atom.from_dtype(self.rel_bcid_hist.dtype), shape=(16, ), filters=self._filter_table)
                    rel_bcid_hist_table[:] = self.rel_bcid_hist[0:16]
                else:
                    rel_bcid_hist_table = self.out_file_h5.create_carray(self.out_file_h5.root, name='HistRelBcid', title='relative BCID Histogram in stop mode read out', atom=tb.Atom.from_dtype(self.rel_bcid_hist.dtype), shape=self.rel_bcid_hist.shape, filters=self._filter_table)
                    rel_bcid_hist_table[:] = self.rel_bcid_hist
        if self._create_occupancy_hist:
            self.occupancy_array = np.swapaxes(self.histogram.get_occupancy(), 0, 1)  # swap axis col,row, parameter --> row, col, parameter
            if self._analyzed_data_file is not None and safe_to_file:
                occupancy_array_table = self.out_file_h5.create_carray(self.out_file_h5.root, name='HistOcc', title='Occupancy Histogram', atom=tb.Atom.from_dtype(self.occupancy_array.dtype), shape=self.occupancy_array.shape, filters=self._filter_table)
                occupancy_array_table[0:336, 0:80, 0:self.histogram.get_n_parameters()] = self.occupancy_array
        if self._create_mean_tot_hist:
            self.mean_tot_array = np.swapaxes(self.histogram.get_mean_tot(), 0, 1)  # swap axis col,row, parameter --> row, col, parameter
            if self._analyzed_data_file is not None and safe_to_file:
                mean_tot_array_table = self.out_file_h5.create_carray(self.out_file_h5.root, name='HistMeanTot', title='Mean ToT Histogram', atom=tb.Atom.from_dtype(self.mean_tot_array.dtype), shape=self.mean_tot_array.shape, filters=self._filter_table)
                mean_tot_array_table[0:336, 0:80, 0:self.histogram.get_n_parameters()] = self.mean_tot_array
        if self._create_threshold_hists:
            _, scan_parameters_idx = np.unique(self.scan_parameters['PlsrDAC'], return_index=True)
            scan_parameters = self.scan_parameters['PlsrDAC'][np.sort(scan_parameters_idx)]
            if scan_parameters[0] >= scan_parameters[-1]:
                raise analysis_utils.AnalysisError('Scan parameter PlsrDAC not increasing')
            threshold, noise = np.zeros(80 * 336, dtype=np.float64), np.zeros(80 * 336, dtype=np.float64)
            # calling fast algorithm function: M. Mertens, PhD thesis, Juelich 2010, note: noise zero if occupancy was zero
            self.histogram.calculate_threshold_scan_arrays(threshold, noise, self._n_injection, np.min(self.scan_parameters['PlsrDAC']), np.max(self.scan_parameters['PlsrDAC']))
            threshold_hist, noise_hist = np.reshape(a=threshold.view(), newshape=(80, 336), order='F'), np.reshape(a=noise.view(), newshape=(80, 336), order='F')
            self.threshold_hist, self.noise_hist = np.swapaxes(threshold_hist, 0, 1), np.swapaxes(noise_hist, 0, 1)
            if self._analyzed_data_file is not None and safe_to_file:
                threshold_hist_table = self.out_file_h5.create_carray(self.out_file_h5.root, name='HistThreshold', title='Threshold Histogram', atom=tb.Atom.from_dtype(self.threshold_hist.dtype), shape=(336, 80), filters=self._filter_table)
                threshold_hist_table[:] = self.threshold_hist
                noise_hist_table = self.out_file_h5.create_carray(self.out_file_h5.root, name='HistNoise', title='Noise Histogram', atom=tb.Atom.from_dtype(self.noise_hist.dtype), shape=(336, 80), filters=self._filter_table)
                noise_hist_table[:] = self.noise_hist
        if self._create_fitted_threshold_hists:
            _, scan_parameters_idx = np.unique(self.scan_parameters['PlsrDAC'], return_index=True)
            scan_parameters = self.scan_parameters['PlsrDAC'][np.sort(scan_parameters_idx)]
            self.scurve_fit_results = self.fit_scurves_multithread(self.out_file_h5, PlsrDAC=scan_parameters)
            if self._analyzed_data_file is not None and safe_to_file:
                fitted_threshold_hist_table = self.out_file_h5.create_carray(self.out_file_h5.root, name='HistThresholdFitted', title='Threshold Fitted Histogram', atom=tb.Atom.from_dtype(self.scurve_fit_results.dtype), shape=(336, 80), filters=self._filter_table)
                fitted_noise_hist_table = self.out_file_h5.create_carray(self.out_file_h5.root, name='HistNoiseFitted', title='Noise Fitted Histogram', atom=tb.Atom.from_dtype(self.scurve_fit_results.dtype), shape=(336, 80), filters=self._filter_table)
                fitted_threshold_hist_table.attrs.dimensions, fitted_noise_hist_table.attrs.dimensions = 'column, row, PlsrDAC', 'column, row, PlsrDAC'
                fitted_threshold_hist_table[:], fitted_noise_hist_table[:] = self.scurve_fit_results[:, :, 0], self.scurve_fit_results[:, :, 1]

                fitted_threshold_hist_table = self.out_file_h5.create_carray(self.out_file_h5.root, name='HistThresholdFittedCalib', title='Threshold Fitted Histogram with PlsrDAC clalibration', atom=tb.Atom.from_dtype(self.scurve_fit_results.dtype), shape=(336, 80), filters=self._filter_table)
                fitted_noise_hist_table = self.out_file_h5.create_carray(self.out_file_h5.root, name='HistNoiseFittedCalib', title='Noise Fitted Histogram with PlsrDAC clalibration', atom=tb.Atom.from_dtype(self.scurve_fit_results.dtype), shape=(336, 80), filters=self._filter_table)
                fitted_threshold_hist_table.attrs.dimensions, fitted_noise_hist_table.attrs.dimensions = 'column, row, electrons', 'column, row, electrons'
                self.threshold_hist_calib, self.noise_hist_calib = self._get_plsr_dac_charge(self.scurve_fit_results[:, :, 0]), self._get_plsr_dac_charge(self.scurve_fit_results[:, :, 1], no_offset=True)
                fitted_threshold_hist_table[:], fitted_noise_hist_table[:] = self.threshold_hist_calib, self.noise_hist_calib

    def _create_additional_cluster_data(self, safe_to_file=True):
        logging.info('Create selected cluster histograms')
        if self._create_cluster_size_hist:
            if self._analyzed_data_file is not None and safe_to_file:
                cluster_size_hist_table = self.out_file_h5.create_carray(self.out_file_h5.root, name='HistClusterSize', title='Cluster Size Histogram', atom=tb.Atom.from_dtype(self._cluster_size_hist.dtype), shape=self._cluster_size_hist.shape, filters=self._filter_table)
                cluster_size_hist_table[:] = self._cluster_size_hist
        if self._create_cluster_tot_hist:
            self._cluster_tot_hist[:, 0] = self._cluster_tot_hist.sum(axis=1)  # First bin is the projection of the others
            if self._analyzed_data_file is not None and safe_to_file:
                cluster_tot_hist_table = self.out_file_h5.create_carray(self.out_file_h5.root, name='HistClusterTot', title='Cluster Tot Histogram', atom=tb.Atom.from_dtype(self._cluster_tot_hist.dtype), shape=self._cluster_tot_hist.shape, filters=self._filter_table)
                cluster_tot_hist_table[:] = self._cluster_tot_hist

    def analyze_hit_table(self, analyzed_data_file=None, analyzed_data_out_file=None):
        '''Analyzes a hit table with the c++ histogrammming/clusterizer.

        Parameters
        ----------
        analyzed_data_file : string
            The filename of the analyzed data file. If None, the analyzed data file
            specified during initialization is taken.
            Filename extension (.h5) does not need to be provided.
        analyzed_data_out_file : string
            The filename of the new analyzed data file. If None, the analyzed data file
            specified during initialization is taken.
            Filename extension (.h5) does not need to be provided.
        '''
        close_analyzed_data_file = False
        if analyzed_data_file is not None:  # if an output file name is specified create new file for analyzed data
            if self.is_open(self.out_file_h5) and os.path.abspath(analyzed_data_file) == os.path.abspath(self.out_file_h5.filename):
                in_file_h5 = self.out_file_h5
            else:
                # normalize path
                analyzed_data_file = os.path.abspath(analyzed_data_file)
                if os.path.splitext(analyzed_data_file)[1].lower() != ".h5":
                    analyzed_data_file = os.path.splitext(analyzed_data_file)[0] + ".h5"
                in_file_h5 = tb.open_file(analyzed_data_file, mode="r+")
                close_analyzed_data_file = True
        elif self.is_open(self.out_file_h5):
                in_file_h5 = self.out_file_h5
        else:
            raise ValueError('Parameter "analyzed_data_file" not specified.')

        # set output file if an output file name is given, otherwise check if an output file is already opened
        close_analyzed_data_out_file = False
        if analyzed_data_out_file is not None:  # if an output file name is specified create new file for analyzed data
            if self.is_open(self.out_file_h5) and os.path.abspath(analyzed_data_out_file) == os.path.abspath(self.out_file_h5.filename):
                out_file_h5 = self.out_file_h5
            elif self.is_open(in_file_h5) and os.path.abspath(analyzed_data_out_file) == os.path.abspath(in_file_h5.filename):
                out_file_h5 = in_file_h5
            else:
                # normalize path
                analyzed_data_out_file = os.path.abspath(analyzed_data_out_file)
                if os.path.splitext(analyzed_data_out_file)[1].lower() != ".h5":
                    analyzed_data_out_file = os.path.splitext(analyzed_data_out_file)[0] + ".h5"
                out_file_h5 = tb.open_file(analyzed_data_out_file, mode="w", title="Analyzed FE-I4 hits")
                close_analyzed_data_out_file = True
        elif self.is_open(self.out_file_h5):
                out_file_h5 = self.out_file_h5
        else:
            raise ValueError('Parameter "analyzed_data_out_file" not specified.')

        tmp_out_file_h5 = self.out_file_h5
        if not self.is_open(self.out_file_h5):
            if os.path.abspath(in_file_h5.filename) == os.path.abspath(out_file_h5.filename):
                close_analyzed_data_file = False
                tmp_out_file_h5 = in_file_h5
        self.out_file_h5 = out_file_h5
        self._analyzed_data_file = self.out_file_h5.filename

        if self._create_cluster_table:
            cluster_table = self.out_file_h5.create_table(self.out_file_h5.root, name='Cluster', description=data_struct.ClusterInfoTable, title='cluster_hit_data', filters=self._filter_table, expectedrows=self._chunk_size)
        if self._create_cluster_hit_table:
            cluster_hit_table = self.out_file_h5.create_table(self.out_file_h5.root, name='ClusterHits', description=data_struct.ClusterHitInfoTable, title='cluster_hit_data', filters=self._filter_table, expectedrows=self._chunk_size)

        if self._create_cluster_size_hist:  # Cluster size result histogram
            self._cluster_size_hist = np.zeros(shape=(6, ), dtype=np.uint32)

        if self._create_cluster_tot_hist:  # Cluster tot/size result histogram
            self._cluster_tot_hist = np.zeros(shape=(16, 6), dtype=np.uint32)

        try:
            meta_data_table = in_file_h5.root.meta_data
            meta_data = meta_data_table[:]
            self.scan_parameters = analysis_utils.get_unique_scan_parameter_combinations(meta_data, scan_parameter_columns_only=True)
            if self.scan_parameters is not None:  # check if there is an additional column after the error code column, if yes this column has scan parameter infos
                meta_event_index = np.ascontiguousarray(analysis_utils.get_unique_scan_parameter_combinations(meta_data)['event_number'].astype(np.uint64))
                self.histogram.add_meta_event_index(meta_event_index, array_length=len(meta_event_index))
                self.scan_parameter_index = analysis_utils.get_scan_parameters_index(self.scan_parameters)  # a array that labels unique scan parameter combinations
                self.histogram.add_scan_parameter(self.scan_parameter_index)  # just add an index for the different scan parameter combinations
                scan_parameter_names = analysis_utils.get_scan_parameter_names(self.scan_parameters)
                logging.info('Adding scan parameter(s) for analysis: %s', (', ').join(scan_parameter_names) if scan_parameter_names else 'None',)
            else:
                logging.info("No scan parameter data provided")
                self.histogram.set_no_scan_parameter()
        except tb.exceptions.NoSuchNodeError:
            logging.info("No meta data provided")
            self.histogram.set_no_scan_parameter()

        table_size = in_file_h5.root.Hits.nrows
        n_hits = 0  # number of hits in actual chunk

        logging.info('Analyzing hits...')
        progress_bar = progressbar.ProgressBar(widgets=['', progressbar.Percentage(), ' ', progressbar.Bar(marker='*', left='|', right='|'), ' ', progressbar.AdaptiveETA()], maxval=table_size, term_width=80)
        progress_bar.start()

        for hits, index in analysis_utils.data_aligned_at_events(in_file_h5.root.Hits, chunk_size=self._chunk_size):
            n_hits += hits.shape[0]

            if self.is_cluster_hits():
                cluster_hits, clusters = self.cluster_hits(hits)

            if self.is_histogram_hits():
                self.histogram_hits(hits)

            if self._analyzed_data_file is not None and self._create_cluster_hit_table:
                cluster_hit_table.append(cluster_hits)
            if self._analyzed_data_file is not None and self._create_cluster_table:
                cluster_table.append(clusters)
                if self._create_cluster_size_hist:
                    if clusters['size'].shape[0] > 0 and np.max(clusters['size']) + 1 > self._cluster_size_hist.shape[0]:
                        self._cluster_size_hist.resize(np.max(clusters['size']) + 1)
                    self._cluster_size_hist += fast_analysis_utils.hist_1d_index(clusters['size'], shape=self._cluster_size_hist.shape)
                if self._create_cluster_tot_hist:
                    if clusters['tot'].shape[0] > 0 and np.max(clusters['tot']) + 1 > self._cluster_tot_hist.shape[0]:
                        self._cluster_tot_hist.resize((np.max(clusters['tot']) + 1, self._cluster_tot_hist.shape[1]))
                    if clusters['size'].shape[0] > 0 and np.max(clusters['size']) + 1 > self._cluster_tot_hist.shape[1]:
                        self._cluster_tot_hist.resize((self._cluster_tot_hist.shape[0], np.max(clusters['size']) + 1))
                    self._cluster_tot_hist += fast_analysis_utils.hist_2d_index(clusters['tot'], clusters['size'], shape=self._cluster_tot_hist.shape)
            self.out_file_h5.flush()
            progress_bar.update(index)
        progress_bar.finish()

        if table_size == 0:
            logging.warning('Found no hits')

        if n_hits != table_size:
            raise analysis_utils.AnalysisError('Tables have different sizes. Not all hits were analyzed.')

        self._create_additional_hit_data()
        self._create_additional_cluster_data()
        if close_analyzed_data_out_file:
            out_file_h5.close()
        if close_analyzed_data_file:
            in_file_h5.close()
        else:
            self.out_file_h5 = tmp_out_file_h5
        if self.is_open(self.out_file_h5):
            self._analyzed_data_file = self.out_file_h5.filename
        else:
            self._analyzed_data_file = None

    def analyze_hits(self, hits, scan_parameter=None):
        n_hits = hits.shape[0]
        logging.debug('Analyze %d hits' % n_hits)

        if scan_parameter is None:  # if nothing specified keep actual setting
            logging.debug('Keep scan parameter settings ')
        elif not scan_parameter:    # set no scan parameter
            logging.debug('No scan parameter used')
            self.histogram.set_no_scan_parameter()
        else:
            logging.info('Setting a scan parameter')
            self.histogram.add_scan_parameter(scan_parameter)

        if self.is_cluster_hits():
            logging.debug('Cluster hits')
            cluster_hits, clusters = self.cluster_hits(hits)
        else:
            cluster_hits = None
            clusters = None

        if self.is_histogram_hits():
            logging.debug('Histogramming hits')
            self.histogram_hits(hits)

        return cluster_hits, clusters

    def cluster_hits(self, hits, start_index=None, stop_index=None):
        return self.clusterizer.cluster_hits(hits[start_index:stop_index])

    def histogram_hits(self, hits, start_index=None, stop_index=None):
        self.histogram.add_hits(hits[start_index:stop_index])

    def histogram_cluster_seed_hits(self, clusters, start_index=None, stop_index=None):
        self.histogram.add_hits(clusters[start_index:stop_index])

    def plot_histograms(self, pdf_filename=None, analyzed_data_file=None, maximum=None, create_hit_hists_only=False):  # plots the histogram from output file if available otherwise from ram
        logging.info('Creating histograms%s', (' (source: %s)' % analyzed_data_file) if analyzed_data_file is not None else (' (source: %s)' % self._analyzed_data_file) if self._analyzed_data_file is not None else '')
        close_analyzed_data_file = False
        if analyzed_data_file is not None:
            if self.is_open(self.out_file_h5) and os.path.abspath(analyzed_data_file) == os.path.abspath(self.out_file_h5.filename):
                out_file_h5 = self.out_file_h5
            else:
                # normalize path
                analyzed_data_file = os.path.abspath(analyzed_data_file)
                if os.path.splitext(analyzed_data_file)[1].lower() != ".h5":
                    analyzed_data_file = os.path.splitext(analyzed_data_file)[0] + ".h5"
                out_file_h5 = tb.open_file(analyzed_data_file, mode="r")
                close_analyzed_data_file = True
        elif self.is_open(self.out_file_h5):
            out_file_h5 = self.out_file_h5
        else:
            logging.info('Parameter "analyzed_data_file" not set. Use histograms from memory.')
        if pdf_filename is not None:
            # normalize path
            pdf_filename = os.path.abspath(pdf_filename)
            if os.path.splitext(pdf_filename)[1].lower() != ".pdf":
                output_pdf_filename = os.path.splitext(pdf_filename)[0] + ".pdf"
            else:
                output_pdf_filename = pdf_filename
        # reuse existing PDF file
        if pdf_filename is not None and os.path.abspath(output_pdf_filename) != self.output_pdf._file.fh.name:
            logging.info('Opening output PDF file: %s', output_pdf_filename)
            output_pdf = PdfPages(output_pdf_filename)
            close_pdf = True
        else:
            output_pdf = self.output_pdf
            close_pdf = False
        if output_pdf is None:
            raise ValueError('Parameter "pdf_filename" not specified.')
        logging.info('Saving histograms to PDF file: %s', str(output_pdf._file.fh.name))
        if self._create_threshold_hists:
            if self._create_threshold_mask:  # mask pixel with bad data for plotting
                if out_file_h5 is not None:
                    self.threshold_mask = analysis_utils.generate_threshold_mask(out_file_h5.root.HistNoise[:])
                else:
                    self.threshold_mask = analysis_utils.generate_threshold_mask(self.noise_hist)
            else:
                self.threshold_mask = np.zeros_like(out_file_h5.root.HistThreshold[:] if out_file_h5 is not None else self.threshold_hist, dtype=np.bool)
            threshold_hist = np.ma.array(out_file_h5.root.HistThreshold[:] if out_file_h5 is not None else self.threshold_hist, mask=self.threshold_mask)
            noise_hist = np.ma.array(out_file_h5.root.HistNoise[:] if out_file_h5 is not None else self.noise_hist, mask=self.threshold_mask)
            mask_cnt = np.ma.count_masked(noise_hist)
            logging.info('Fast algorithm: masking %d pixel(s)', mask_cnt)
            plotting.plot_three_way(hist=threshold_hist, title='Threshold%s' % ((' (masked %i pixel(s))' % mask_cnt) if self._create_threshold_mask else ''), x_axis_title="threshold [PlsrDAC]", filename=output_pdf, bins=100, minimum=0, maximum=maximum)
            plotting.plot_three_way(hist=noise_hist, title='Noise%s' % ((' (masked %i pixel(s))' % mask_cnt) if self._create_threshold_mask else ''), x_axis_title="noise [PlsrDAC]", filename=output_pdf, bins=100, minimum=0, maximum=maximum)
        if self._create_fitted_threshold_hists:
            if self._create_fitted_threshold_mask:
                if out_file_h5 is not None:
                    self.fitted_threshold_mask = analysis_utils.generate_threshold_mask(out_file_h5.root.HistNoiseFitted[:])
                else:
                    self.fitted_threshold_mask = analysis_utils.generate_threshold_mask(self.scurve_fit_results[:, :, 1])
            else:
                self.threshold_mask = np.zeros_like(out_file_h5.root.HistThresholdFitted[:] if out_file_h5 is not None else self.scurve_fit_results, dtype=np.bool8)
            threshold_hist = np.ma.array(out_file_h5.root.HistThresholdFitted[:] if out_file_h5 is not None else self.scurve_fit_results[:, :, 0], mask=self.fitted_threshold_mask)
            noise_hist = np.ma.array(out_file_h5.root.HistNoiseFitted[:] if out_file_h5 is not None else self.scurve_fit_results[:, :, 1], mask=self.fitted_threshold_mask)
            threshold_hist_calib = np.ma.array(out_file_h5.root.HistThresholdFittedCalib[:] if out_file_h5 is not None else self.threshold_hist_calib[:], mask=self.fitted_threshold_mask)
            noise_hist_calib = np.ma.array(out_file_h5.root.HistNoiseFittedCalib[:] if out_file_h5 is not None else self.noise_hist_calib[:], mask=self.fitted_threshold_mask)
            mask_cnt = np.ma.count_masked(noise_hist)
            logging.info('S-curve fit: masking %d pixel(s)', mask_cnt)
            plotting.plot_three_way(hist=threshold_hist, title='Threshold (S-curve fit, masked %i pixel(s))' % mask_cnt, x_axis_title="Threshold [PlsrDAC]", filename=output_pdf, bins=100, minimum=0, maximum=maximum)
            plotting.plot_three_way(hist=noise_hist, title='Noise (S-curve fit, masked %i pixel(s))' % mask_cnt, x_axis_title="Noise [PlsrDAC]", filename=output_pdf, bins=100, minimum=0, maximum=maximum)
            plotting.plot_three_way(hist=threshold_hist_calib, title='Threshold (S-curve fit, masked %i pixel(s))' % mask_cnt, x_axis_title="Threshold [e]", filename=output_pdf, bins=100, minimum=0)
            plotting.plot_three_way(hist=noise_hist_calib, title='Noise (S-curve fit, masked %i pixel(s))' % mask_cnt, x_axis_title="Noise [e]", filename=output_pdf, bins=100, minimum=0)
        if self._create_occupancy_hist:
            if self._create_fitted_threshold_hists:
                _, scan_parameters_idx = np.unique(self.scan_parameters['PlsrDAC'], return_index=True)
                scan_parameters = self.scan_parameters['PlsrDAC'][np.sort(scan_parameters_idx)]
                plotting.plot_scurves(occupancy_hist=out_file_h5.root.HistOcc[:] if out_file_h5 is not None else self.occupancy_array[:], filename=output_pdf, scan_parameters=scan_parameters, scan_parameter_name="PlsrDAC")
            else:
                hist = np.sum(out_file_h5.root.HistOcc[:], axis=2) if out_file_h5 is not None else np.sum(self.occupancy_array[:], axis=2)
                occupancy_array_masked = np.ma.masked_equal(hist, 0)
                if self._create_source_scan_hist:
                    plotting.plot_fancy_occupancy(hist=occupancy_array_masked, filename=output_pdf, z_max='median')
                    plotting.plot_occupancy(hist=occupancy_array_masked, filename=output_pdf, z_max='maximum')
                else:
                    plotting.plot_three_way(hist=occupancy_array_masked, title="Occupancy", x_axis_title="occupancy", filename=output_pdf, maximum=maximum)
                    plotting.plot_occupancy(hist=occupancy_array_masked, filename=output_pdf, z_max='median')
        if self._create_tot_hist:
            plotting.plot_tot(hist=out_file_h5.root.HistTot[:] if out_file_h5 is not None else self.tot_hist, filename=output_pdf)
        if self._create_tot_pixel_hist:
            tot_pixel_hist = out_file_h5.root.HistTotPixel[:] if out_file_h5 is not None else self.tot_pixel_hist_array
            total_hits_masked = np.ma.masked_equal(np.sum(tot_pixel_hist, axis=2), 0)
            mean_pixel_tot = np.average(tot_pixel_hist, axis=2, weights=range(16)) * sum(range(0, 16)) / total_hits_masked
            plotting.plot_three_way(mean_pixel_tot, title='Mean ToT', x_axis_title='mean ToT', filename=output_pdf, minimum=0, maximum=15)
        if self._create_tdc_counter_hist:
            plotting.plot_tdc_counter(hist=out_file_h5.root.HistTdcCounter[:] if out_file_h5 is not None else self.tdc_hist_counter, filename=output_pdf)
        if self._create_tdc_hist:
            plotting.plot_tdc(hist=out_file_h5.root.HistTdc[:] if out_file_h5 is not None else self.tdc_hist, filename=output_pdf)
        if self._create_cluster_size_hist:
            plotting.plot_cluster_size(hist=out_file_h5.root.HistClusterSize[:] if out_file_h5 is not None else self.cluster_size_hist, filename=output_pdf)
        if self._create_cluster_tot_hist:
            plotting.plot_cluster_tot(hist=out_file_h5.root.HistClusterTot[:] if out_file_h5 is not None else self.cluster_tot_hist, filename=output_pdf)
        if self._create_cluster_tot_hist and self._create_cluster_size_hist:
            plotting.plot_cluster_tot_size(hist=out_file_h5.root.HistClusterTot[:] if out_file_h5 is not None else self.cluster_tot_hist, filename=output_pdf)
        if self._create_rel_bcid_hist:
            if self.set_stop_mode:
                plotting.plot_relative_bcid_stop_mode(hist=out_file_h5.root.HistRelBcid[:] if out_file_h5 is not None else self.rel_bcid_hist, filename=output_pdf)
            else:
                plotting.plot_relative_bcid(hist=out_file_h5.root.HistRelBcid[0:16] if out_file_h5 is not None else self.rel_bcid_hist[0:16], filename=output_pdf)
        if self._create_tdc_pixel_hist:
            tdc_pixel_hist = out_file_h5.root.HistTdcPixel[:, :, :1024] if out_file_h5 is not None else self.tdc_pixel_hist_array[:, :, :1024]  # only take first 1024 values, otherwise memory error likely
            total_hits_masked = np.ma.masked_equal(np.sum(tdc_pixel_hist, axis=2), 0)
            mean_pixel_tdc = np.average(tdc_pixel_hist, axis=2, weights=range(1024)) * sum(range(0, 1024)) / total_hits_masked
            plotting.plot_three_way(mean_pixel_tdc, title='Mean TDC', x_axis_title='mean TDC', maximum=2 * np.ma.median(np.ma.masked_invalid(mean_pixel_tdc)), filename=output_pdf)
        if not create_hit_hists_only:
            if analyzed_data_file is None and self._create_error_hist:
                plotting.plot_event_errors(hist=out_file_h5.root.HistErrorCounter[:] if out_file_h5 is not None else self.error_counter_hist, filename=output_pdf)
            if analyzed_data_file is None and self._create_service_record_hist:
                plotting.plot_service_records(hist=out_file_h5.root.HistServiceRecord[:] if out_file_h5 is not None else self.service_record_hist, filename=output_pdf)
            if analyzed_data_file is None and self._create_trigger_error_hist:
                plotting.plot_trigger_errors(hist=out_file_h5.root.HistTriggerErrorCounter[:] if out_file_h5 is not None else self.trigger_error_counter_hist, filename=output_pdf)

        if close_analyzed_data_file:
            out_file_h5.close()
        if close_pdf:
            logging.info('Closing output PDF file: %s', str(output_pdf._file.fh.name))
            output_pdf.close()

    def fit_scurves_multithread(self, hit_table_file=None, PlsrDAC=None):
        logging.info("Start S-curve fit on %d CPU core(s)", mp.cpu_count())
        occupancy_hist = hit_table_file.root.HistOcc[:] if hit_table_file is not None else self.occupancy_array[:]  # take data from RAM if no file is opened
        occupancy_hist_shaped = occupancy_hist.reshape(occupancy_hist.shape[0] * occupancy_hist.shape[1], occupancy_hist.shape[2])
        # reverse data to fit s-curve
        if PlsrDAC[0] > PlsrDAC[-1]:
            occupancy_hist_shaped = np.flip(occupancy_hist_shaped, axis=1)
            PlsrDAC = np.flip(PlsrDAC, axis=0)
        partialfit_scurve = partial(fit_scurve, PlsrDAC=PlsrDAC)  # trick to give a function more than one parameter, needed for pool.map
        pool = mp.Pool()  # create as many workers as physical cores are available
        try:
            result_list = pool.map(partialfit_scurve, occupancy_hist_shaped.tolist())
        except TypeError:
            raise analysis_utils.NotSupportedError('Less than 3 points found for S-curve fit.')
        finally:
            pool.close()
            pool.join()
        result_array = np.array(result_list)
        logging.info("S-curve fit finished")
        return result_array.reshape(occupancy_hist.shape[0], occupancy_hist.shape[1], 2)

    def is_open(self, h5_file):
        if isinstance(h5_file, tb.file.File):
            return True
        return False

    def is_histogram_hits(self):  # returns true if a setting needs to have the hit histogramming active
        if self._create_occupancy_hist or self._create_tot_hist or self._create_rel_bcid_hist or self._create_hit_table or self._create_threshold_hists or self._create_fitted_threshold_hists:
            return True
        return False

    def is_cluster_hits(self):  # returns true if a setting needs to have the clusterizer active
        if self.create_cluster_hit_table or self.create_cluster_table or self.create_cluster_size_hist or self.create_cluster_tot_hist:
            return True
        return False

    def _deduce_settings_from_file(self, opened_raw_data_file):  # TODO: parse better
        '''Tries to get the scan parameters needed for analysis from the raw data file
        '''
        try:  # take infos raw data files (not avalable in old files)
            flavor = opened_raw_data_file.root.configuration.miscellaneous[:][np.where(opened_raw_data_file.root.configuration.miscellaneous[:]['name'] == 'Flavor')]['value'][0]
            self._settings_from_file_set = True
            # adding this for special cases e.g., stop-mode scan
            if "trig_count" in opened_raw_data_file.root.configuration.run_conf[:]['name']:
                trig_count = opened_raw_data_file.root.configuration.run_conf[:][np.where(opened_raw_data_file.root.configuration.run_conf[:]['name'] == 'trig_count')]['value'][0]
            else:
                trig_count = opened_raw_data_file.root.configuration.global_register[:][np.where(opened_raw_data_file.root.configuration.global_register[:]['name'] == 'Trig_Count')]['value'][0]
            vcal_c0 = opened_raw_data_file.root.configuration.calibration_parameters[:][np.where(opened_raw_data_file.root.configuration.calibration_parameters[:]['name'] == 'Vcal_Coeff_0')]['value'][0]
            vcal_c1 = opened_raw_data_file.root.configuration.calibration_parameters[:][np.where(opened_raw_data_file.root.configuration.calibration_parameters[:]['name'] == 'Vcal_Coeff_1')]['value'][0]
            c_low = opened_raw_data_file.root.configuration.calibration_parameters[:][np.where(opened_raw_data_file.root.configuration.calibration_parameters[:]['name'] == 'C_Inj_Low')]['value'][0]
            c_mid = opened_raw_data_file.root.configuration.calibration_parameters[:][np.where(opened_raw_data_file.root.configuration.calibration_parameters[:]['name'] == 'C_Inj_Med')]['value'][0]
            c_high = opened_raw_data_file.root.configuration.calibration_parameters[:][np.where(opened_raw_data_file.root.configuration.calibration_parameters[:]['name'] == 'C_Inj_High')]['value'][0]
            self.c_low_mask = opened_raw_data_file.root.configuration.C_Low[:]
            self.c_high_mask = opened_raw_data_file.root.configuration.C_High[:]
            self.fei4b = False if str(flavor) == 'fei4a' else True
            self.trig_count = int(trig_count)
            self.vcal_c0 = float(vcal_c0)
            self.vcal_c1 = float(vcal_c1)
            self.c_low = float(c_low)
            self.c_mid = float(c_mid)
            self.c_high = float(c_high)
            self.n_injections = int(opened_raw_data_file.root.configuration.run_conf[:][np.where(opened_raw_data_file.root.configuration.run_conf[:]['name'] == 'n_injections')]['value'][0])
        except tb.exceptions.NoSuchNodeError:
            if not self._settings_from_file_set:
                logging.warning('No settings stored in raw data file %s, use standard settings', opened_raw_data_file.filename)
            else:
                logging.info('No settings provided in raw data file %s, use already set settings', opened_raw_data_file.filename)
        except IndexError:  # happens if setting is not available (e.g. repeat_command)
            pass

    def _get_plsr_dac_charge(self, plsr_dac_array, no_offset=False):
        '''Takes the PlsrDAC calibration and the stored C-high/C-low mask to calculate the charge from the PlsrDAC array on a pixel basis
        '''
        charge = np.zeros_like(self.c_low_mask, dtype=np.float16)  # charge in electrons
        if self.vcal_c0 is not None and self.vcal_c1 is not None and self.c_low is not None and self.c_mid is not None and self.c_high is not None:
            voltage = self.vcal_c1 * plsr_dac_array if no_offset else self.vcal_c0 + self.vcal_c1 * plsr_dac_array
            charge[np.logical_and(self.c_low_mask, ~self.c_high_mask)] = voltage[np.logical_and(self.c_low_mask, ~self.c_high_mask)] * self.c_low / 0.16022
            charge[np.logical_and(~self.c_low_mask, self.c_high_mask)] = voltage[np.logical_and(self.c_low_mask, ~self.c_high_mask)] * self.c_mid / 0.16022
            charge[np.logical_and(self.c_low_mask, self.c_high_mask)] = voltage[np.logical_and(self.c_low_mask, ~self.c_high_mask)] * self.c_high / 0.16022
        return charge def make_box_pixel_mask_from_col_row(column, row, default=0, value=1):
    '''Generate box shaped mask from column and row lists. Takes the minimum and maximum value from each list.

    Parameters
    ----------
    column : iterable, int
        List of colums values.
    row : iterable, int
        List of row values.
    default : int
        Value of pixels that are not selected by the mask.
    value : int
        Value of pixels that are selected by the mask.

    Returns
    -------
    numpy.ndarray
    '''
    # FE columns and rows start from 1
    col_array = np.array(column) - 1
    row_array = np.array(row) - 1
    if np.any(col_array >= 80) or np.any(col_array < 0) or np.any(row_array >= 336) or np.any(row_array < 0):
        raise ValueError('Column and/or row out of range')
    shape = (80, 336)
    mask = np.full(shape, default, dtype=np.uint8)
    if column and row:
        mask[col_array.min():col_array.max() + 1, row_array.min():row_array.max() + 1] = value  # advanced indexing
    return mask
 def invert_pixel_mask(mask):
    '''Invert pixel mask (0->1, 1(and greater)->0).

    Parameters
    ----------
    mask : array-like
        Mask.

    Returns
    -------
    inverted_mask : array-like
        Inverted Mask.
    '''
    inverted_mask = np.ones(shape=(80, 336), dtype=np.dtype('>u1'))
    inverted_mask[mask >= 1] = 0
    return inverted_mask
 class Fei4RunBase(RunBase):
    '''Basic FEI4 run meta class.

    Base class for scan- / tune- / analyze-class.

    A fei4 run consist of 3 major steps:
      1. pre_run
        - dut initialization (readout system init)
        - init readout fifo (data taking buffer)
        - load scan parameters from run config
        - init each front-end one by one (configure registers, serial)
      2. do_run
        The following steps are either run for all front-ends
        at once (parallel scan) or one by one (serial scan):
        - scan specific configuration
        - store run attributes
        - run scan
        - restore run attributes (some scans change run conf attributes or add new attributes, this restores to before)
        - restore scan parameters from default run config (they mighte have been changed in scan)
      3. post_run
        - call analysis on raw data files one by one (serial)

    Several handles are provided to encapsulate the underlying hardware
    and scan type to be able to use generic scan definitions:

    - serial scan mode:
      - register: one FE register data
      - register_utils: access to one FE registers
      - output_filename: output file name of a selected module
      - raw_data_file: one output data file

    - parallel scan mode:
      - register: broadcast register or multiple front-end registers
      - register_utils: access all FE registers via broadcast or each front-end registers
        at different channels
      - output_filename: output file name of a selected module
      - raw_data_file: all output data files with channel data filters

    '''
    __metaclass__ = abc.ABCMeta

    def __init__(self, conf):
        # Sets self._conf = conf
        super(Fei4RunBase, self).__init__(conf=conf)
        self.err_queue = Queue()
        self.global_lock = RLock()

        self._module_run_conf = {}  # Store module specific run conf
        self._module_cfgs = {}  # Store module specific configurations
        self._modules = {}  # Module IDs of real modules
        self._tx_module_groups = {}  # Module IDs of TX module groups (virtual modules)
        self._registers = {}  # Store module specific FEI4 registers
        self._register_utils = {}  # Store module specific FEI4 register utils
        self._raw_data_files = {}  # Store module specific raw data files
        self._scan_parameters = {}  # Store module specific scan parameters
        self._module_attr = {}  # Store module specific scan attributes
        self._module_dut = {}  # Store module specific DUT handle
        self.fifo_readout = None  # FIFO readout instances
        self._selected_modules = []
        self._selected_fifos = []  # currently activated FIFOs
        self._readout_fifos = []
        self._converter = []
        self._filter = []
        self._enabled_fe_channels = []  # currently activated receivers
        self._current_module_handle = None  # setting "None" module as default module
        self._scan_threads = []  # list of currently running scan threads
        self._curr_readout_threads = []  # list of currently running threads awaiting start of FIFO readout
        self._readout_lock = Lock()
        self._starting_readout_event = Event()
        self._starting_readout_event.clear()
        self._stopping_readout_event = Event()
        self._stopping_readout_event.clear()
        self._curr_sync_threads = []
        self._sync_lock = Lock()
        self._enter_sync_event = Event()
        self._enter_sync_event.clear()
        self._exit_sync_event = Event()
        self._exit_sync_event.clear()
        self._parse_module_cfgs()
        self._init_default_run_conf()
        # after initialized is set to True, all new attributes are belonging to selected mudule
        # by default the "None" module is selected (current_module_handle is None)
        self._initialized = True

    @property
    def is_initialized(self):
        if "_initialized" in self.__dict__ and self._initialized:
            return True
        else:
            return False

    @property
    def current_module_handle(self):
        if self._current_module_handle is None:
            thread_name = current_thread().name
            module_handles = [module_id for module_id in self._module_cfgs if (module_id is not None and module_id == thread_name)]
            if len(module_handles) > 1:
                raise RuntimeError("Could not determine module handle. Thread name contains multiple module IDs: %s" % ", ".join(module_handles))
            if len(module_handles) == 0:
                return None
#                 raise RuntimeError('Could not determine module handle from thread name "%s"' % thread_name)
            return module_handles[0]
        else:
            return self._current_module_handle

    @property
    def register(self):
        return self._registers[self.current_module_handle]

    @property
    def register_utils(self):
        return self._register_utils[self.current_module_handle]

    @property
    def output_filename(self):
        return self.get_output_filename(module_id=self.current_module_handle)

    @property
    def run_conf(self):
        run_conf = namedtuple('run_conf', field_names=self._module_run_conf[self.current_module_handle].keys())
        return run_conf(*self._module_run_conf[self.current_module_handle])  # prevent changing dict

    @property
    def scan_parameters(self):
        return self._scan_parameters[self.current_module_handle]

    @property
    def dut(self):
        return self._module_dut[self.current_module_handle]

    @property
    def raw_data_file(self):
        return self._raw_data_files[self.current_module_handle]

    def _parse_module_cfgs(self):
        ''' Extracts the configuration of the modules.
        '''
        # Adding here default run config parameters.
        if "dut" not in self._conf or self._conf["dut"] is None:
            raise ValueError('Parameter "dut" not defined.')
        if "dut_configuration" not in self._conf or self._conf["dut_configuration"] is None:
            raise ValueError('Parameter "dut_configuration" not defined.')
        self._conf.setdefault('working_dir', None)  # string, if None, absolute path of configuration.yaml file will be used

        if 'modules' in self._conf and self._conf['modules']:
            for module_id, module_cfg in [(key, value) for key, value in self._conf['modules'].items() if ("activate" not in value or ("activate" in value and value["activate"] is True))]:
                # Check here for missing module config items.
                # Capital letter keys are Basil drivers, other keys are parameters.
                # FIFO, RX, TX, TLU and TDC are generic driver names which are used in the scan implementations.
                # The use of these reserved driver names allows for abstraction.
                # Accessing Basil drivers with real name is still possible.
                if "module_group" in module_id:
                    raise ValueError('The module ID "%s" contains the reserved name "module_group".' % module_id)
                if "flavor" not in module_cfg or module_cfg["flavor"] is None:
                    raise ValueError('No parameter "flavor" defined for module "%s".' % module_id)
                if module_cfg["flavor"] in fe_flavors:
                    for driver_name in _reserved_driver_names:
                        # TDC is not mandatory
                        if driver_name == "TDC":
                            # TDC is allowed to have set None
                            module_cfg.setdefault('TDC', None)
                            continue
                        if driver_name not in module_cfg or module_cfg[driver_name] is None:
                            raise ValueError('No parameter "%s" defined for module "%s".' % (driver_name, module_id))
                    if "rx_channel" not in module_cfg or module_cfg["rx_channel"] is None:
                        raise ValueError('No parameter "rx_channel" defined for module "%s".' % module_id)
                    if "tx_channel" not in module_cfg or module_cfg["tx_channel"] is None:
                        raise ValueError('No parameter "tx_channel" defined for module "%s".' % module_id)
                    if "chip_address" not in module_cfg:
                        raise ValueError('No parameter "chip_address" defined for module "%s".' % module_id)
                    module_cfg.setdefault("tdc_channel", None)
                    module_cfg.setdefault("configuration", None)  # string or number, if None, using the last valid configuration
                    module_cfg.setdefault("send_data", None)  # address string of PUB socket
                    module_cfg.setdefault("activate", True)  # set module active by default
                    # Save config to dict.
                    self._module_cfgs[module_id] = module_cfg
                    self._modules[module_id] = [module_id]
        else:
            raise ValueError("No module configuration specified")

    def _init_default_run_conf(self):
        # set up default run conf parameters
        self._default_run_conf.setdefault('comment', '{}'.format(self.__class__.__name__))
        self._default_run_conf.setdefault('reset_rx_on_error', False)
        # Enabling broadcast commands will significantly improve the speed of scans.
        # Only those scans can be accelerated which commands can be broadcastet and
        # which commands are not individual for each module.
        self._default_run_conf.setdefault('broadcast_commands', False)
        # Enabling threaded scan improves the speed of scans
        # and require multiple TX for sending commands.
        # If only a single TX is available, no speed improvement is gained.
        self._default_run_conf.setdefault('threaded_scan', False)
        # If True, send full configuration to the FE-I4 before the configuration step.
        self._default_run_conf.setdefault('configure_fe', True)
        # If True, perform a FE-I4 reset (ECR and BCR).
        self._default_run_conf.setdefault('reset_fe', True)

    def _init_run_conf(self, run_conf):
        # same implementation as in base class, but ignore "scan_parameters" property
        attribute_names = [key for key in self._default_run_conf.keys() if (key != "scan_parameters" and (key in self.__dict__ or (hasattr(self.__class__, key) and isinstance(getattr(self.__class__, key), property))))]
        if attribute_names:
            raise RuntimeError('Attribute names already in use. Rename the following parameters in run conf: %s' % ', '.join(attribute_names))
        sc = namedtuple('run_configuration', field_names=self._default_run_conf.keys())
        default_run_conf = sc(**self._default_run_conf)
        if run_conf:
            self._run_conf = default_run_conf._replace(**run_conf)._asdict()
        else:
            self._run_conf = default_run_conf._asdict()

        # check for invalid run conf parameters
        if self._run_conf['broadcast_commands'] and not self._default_run_conf['broadcast_commands']:
            raise RuntimeError('Changing "broadcast_commands" parameter from False (default) to True.')
        if self._run_conf['broadcast_commands'] and 'modules' in self._conf and self._conf['modules']:
            tx_groups = groupby_dict({key: value for (key, value) in self._module_cfgs.items() if key in self._modules}, "TX")
            for module_group in tx_groups.values():
                for module_id in module_group:
                    if self.__class__.__name__ in self._conf["modules"][module_id] and self._conf["modules"][module_id][self.__class__.__name__] is not None:
                        for other_module_id in module_group:
                            if self.__class__.__name__ in self._conf["modules"][other_module_id] and self._conf["modules"][other_module_id][self.__class__.__name__] is not None:
                                if any(dict_compare(self._conf["modules"][module_id][self.__class__.__name__], self._conf["modules"][other_module_id][self.__class__.__name__])[:3]):
                                    raise RuntimeError('Module "%s" has specific run configuration that is different from module "%s" for "%s" and "broadcast_commands" parameter is set to True.' % (other_module_id, module_id, self.__class__.__name__))
                            else:
                                raise RuntimeError('Module "%s" has no specific run configuration for "%s" and "broadcast_commands" parameter is set to True.' % (other_module_id, self.__class__.__name__))

    def _set_default_cfg(self):
        ''' Sets the default parameters if they are not specified.
        '''
        # adding special conf for accessing all DUT drivers
        self._module_cfgs[None] = {
            'flavor': None,
            'chip_address': None,
            'FIFO': list(set([self._module_cfgs[module_id]['FIFO'] for module_id in self._modules])),
            'RX': list(set([self._module_cfgs[module_id]['RX'] for module_id in self._modules])),
            'rx_channel': list(set([self._module_cfgs[module_id]['rx_channel'] for module_id in self._modules])),
            'TX': list(set([self._module_cfgs[module_id]['TX'] for module_id in self._modules])),
            'tx_channel': list(set([self._module_cfgs[module_id]['tx_channel'] for module_id in self._modules])),
            'TDC': list(set([self._module_cfgs[module_id]['TDC'] for module_id in self._modules])),
            'tdc_channel': list(set([self._module_cfgs[module_id]['tdc_channel'] for module_id in self._modules])),
            'TLU': list(set([self._module_cfgs[module_id]['TLU'] for module_id in self._modules])),
            'configuration': None,
            'send_data': None}

        tx_groups = groupby_dict({key: value for (key, value) in self._module_cfgs.items() if key in self._modules}, "TX")
        for tx, module_group in tx_groups.items():
            flavors = list(set([module_cfg['flavor'] for module_id, module_cfg in self._module_cfgs.items() if module_id in module_group]))
            if len(flavors) != 1:
                raise ValueError("Parameter 'flavor' must be the same for module group TX=%s." % tx)

            chip_addresses = list(set([module_cfg['chip_address'] for module_id, module_cfg in self._module_cfgs.items() if module_id in module_group]))
            if len(module_group) != len(chip_addresses) or (len(module_group) != 1 and None in chip_addresses):
                raise ValueError("Parameter 'chip_address' must be different for each module in module group TX=%s." % tx)

            # Adding broadcast config for parallel mode.
            self._module_cfgs["module_group_TX=" + tx] = {
                'flavor': flavors[0],
                'chip_address': None,  # broadcast
                'FIFO': list(set([module_cfg['FIFO'] for module_id, module_cfg in self._module_cfgs.items() if module_id in module_group])),
                'RX': list(set([module_cfg['RX'] for module_id, module_cfg in self._module_cfgs.items() if module_id in module_group])),
                'rx_channel': list(set([module_cfg['rx_channel'] for module_id, module_cfg in self._module_cfgs.items() if module_id in module_group])),
                'TX': tx,
                'tx_channel': list(set([module_cfg['tx_channel'] for module_id, module_cfg in self._module_cfgs.items() if module_id in module_group])),
                'TDC': list(set([module_cfg['TDC'] for module_id, module_cfg in self._module_cfgs.items() if module_id in module_group])),
                'tdc_channel': list(set([module_cfg['tdc_channel'] for module_id, module_cfg in self._module_cfgs.items() if module_id in module_group])),
                'TLU': list(set([module_cfg['TLU'] for module_id, module_cfg in self._module_cfgs.items() if module_id in module_group])),
                'configuration': None,
                'send_data': None}
            self._tx_module_groups["module_group_TX=" + tx] = module_group

        # Setting up per module attributes
        self._module_attr = {key: {} for key in self._module_cfgs}
        # Setting up per module run conf
        for module_id in self._module_cfgs:
            sc = namedtuple('run_configuration', field_names=self._default_run_conf.keys())
            run_conf = sc(**self._run_conf)
            if module_id in self._modules and self.__class__.__name__ in self._conf["modules"][module_id] and self._conf["modules"][module_id][self.__class__.__name__] is not None:
                self._module_run_conf[module_id] = run_conf._replace(**self._conf["modules"][module_id][self.__class__.__name__])._asdict()
            else:
                self._module_run_conf[module_id] = run_conf._asdict()
                # update module group with run specific configuration
                if module_id in self._tx_module_groups and self._tx_module_groups[module_id]:
                    selected_module_id = self._tx_module_groups[module_id][0]
                    if self.__class__.__name__ in self._conf["modules"][selected_module_id] and self._conf["modules"][selected_module_id][self.__class__.__name__] is not None:
                        self._module_run_conf[module_id] = run_conf._replace(**self._conf["modules"][selected_module_id][self.__class__.__name__])._asdict()

    def init_dut(self):
        if self.dut.name == 'mio':  # MIO2 with Single Chip Adapter Card (SCAC) or QUAD Module Adapter Card
            if self.dut.get_modules('FEI4AdapterCard') and [adapter_card for adapter_card in self.dut.get_modules('FEI4AdapterCard') if adapter_card.name == 'SINGLE_CHIP_ADAPTER_CARD']:
                try:
                    self.dut['SINGLE_CHIP_ADAPTER_CARD'].set_voltage('VDDA1', 1.5)
                    self.dut['SINGLE_CHIP_ADAPTER_CARD'].set_voltage('VDDA2', 1.5)
                    self.dut['SINGLE_CHIP_ADAPTER_CARD'].set_voltage('VDDD1', 1.2)
                    self.dut['SINGLE_CHIP_ADAPTER_CARD'].set_voltage('VDDD2', 1.2)
                except struct.error:
                    logging.warning('Cannot set voltages. Adapter card not calibrated? Voltages out of range?')
                self.dut['POWER_SCC']['EN_VD1'] = 1
                self.dut['POWER_SCC']['EN_VD2'] = 1  # also EN_VPLL on old SCAC
                self.dut['POWER_SCC']['EN_VA1'] = 1
                self.dut['POWER_SCC']['EN_VA2'] = 1
                self.dut['POWER_SCC'].write()
                self.dut['ENABLE_CHANNEL']['DATA_CH1'] = 0  # RD2Bar on SCAC
                self.dut['ENABLE_CHANNEL']['DATA_CH2'] = 0  # RD1Bar on SCAC
                self.dut['ENABLE_CHANNEL']['DATA_CH3'] = 0  # RABar on SCAC
                self.dut['ENABLE_CHANNEL']['DATA_CH4'] = 1
                self.dut['ENABLE_CHANNEL']['TLU'] = 1
                self.dut['ENABLE_CHANNEL']['TDC'] = 1
                self.dut['ENABLE_CHANNEL'].write()
            elif self.dut.get_modules('FEI4QuadModuleAdapterCard') and [adapter_card for adapter_card in self.dut.get_modules('FEI4QuadModuleAdapterCard') if adapter_card.name == 'QUAD_MODULE_ADAPTER_CARD']:
                # resetting over current status
                self.dut['POWER_QUAD']['EN_DATA_CH1'] = 0
                self.dut['POWER_QUAD']['EN_DATA_CH2'] = 0
                self.dut['POWER_QUAD']['EN_DATA_CH3'] = 0
                self.dut['POWER_QUAD']['EN_DATA_CH4'] = 0
                self.dut['POWER_QUAD'].write()
                try:
                    self.dut['QUAD_MODULE_ADAPTER_CARD'].set_voltage('CH1', 2.1)
                    self.dut['QUAD_MODULE_ADAPTER_CARD'].set_voltage('CH2', 2.1)
                    self.dut['QUAD_MODULE_ADAPTER_CARD'].set_voltage('CH3', 2.1)
                    self.dut['QUAD_MODULE_ADAPTER_CARD'].set_voltage('CH4', 2.1)
                except struct.error:
                    logging.warning('Cannot set voltages. Adapter card not calibrated? Voltages out of range?')
                rx_names = [rx.name for rx in self.dut.get_modules('fei4_rx')]
                active_rx_names = [module_cfg["RX"] for (name, module_cfg) in self._module_cfgs.items() if name in self._modules]
                for rx_name in rx_names:
                    # enabling/disabling Rx
                    if rx_name in active_rx_names:
                        self.dut['ENABLE_CHANNEL'][rx_name] = 1
                        self.dut['POWER_QUAD']['EN_' + rx_name] = 1
                    else:
                        self.dut['ENABLE_CHANNEL'][rx_name] = 0
                        self.dut['POWER_QUAD']['EN_' + rx_name] = 0
                self.dut['ENABLE_CHANNEL']['TLU'] = 1
                self.dut['ENABLE_CHANNEL']['TDC'] = 1
                self.dut['ENABLE_CHANNEL'].write()
                self.dut['POWER_QUAD'].write()
            else:
                logging.warning('Unknown adapter card')
                # do the minimal configuration here
                self.dut['ENABLE_CHANNEL']['DATA_CH1'] = 0  # RD2Bar on SCAC
                self.dut['ENABLE_CHANNEL']['DATA_CH2'] = 0  # RD1Bar on SCAC
                self.dut['ENABLE_CHANNEL']['DATA_CH3'] = 0  # RABar on SCAC
                self.dut['ENABLE_CHANNEL']['DATA_CH4'] = 1
                self.dut['ENABLE_CHANNEL']['TLU'] = 1
                self.dut['ENABLE_CHANNEL']['TDC'] = 1
                self.dut['ENABLE_CHANNEL'].write()
        elif self.dut.name == 'mio_gpac':  # MIO2 with Genaral Purpose Analog Card (GPAC)
            # PWR
            self.dut['V_in'].set_current_limit(0.1, unit='A')  # one for all, max. 1A
            # V_in
            self.dut['V_in'].set_voltage(2.1, unit='V')
            self.dut['V_in'].set_enable(True)
            if self.dut['V_in'].get_over_current():
                self.power_off()
                raise Exception('V_in overcurrent detected')
            # Vdd, also enabling LVDS transceivers
            self.dut['CCPD_Vdd'].set_voltage(1.80, unit='V')
            self.dut['CCPD_Vdd'].set_enable(True)
            if self.dut['CCPD_Vdd'].get_over_current():
                self.power_off()
                raise Exception('Vdd overcurrent detected')
            # Vssa
            self.dut['CCPD_Vssa'].set_voltage(1.50, unit='V')
            self.dut['CCPD_Vssa'].set_enable(True)
            if self.dut['CCPD_Vssa'].get_over_current():
                self.power_off()
                raise Exception('Vssa overcurrent detected')
            # VGate
            self.dut['CCPD_VGate'].set_voltage(2.10, unit='V')
            self.dut['CCPD_VGate'].set_enable(True)
            if self.dut['CCPD_VGate'].get_over_current():
                self.power_off()
                raise Exception('VGate overcurrent detected')
            # enabling readout
            self.dut['ENABLE_CHANNEL']['FE'] = 1
            self.dut['ENABLE_CHANNEL']['TLU'] = 1
            self.dut['ENABLE_CHANNEL']['TDC'] = 1
            self.dut['ENABLE_CHANNEL']['CCPD_TDC'] = 1
            self.dut['ENABLE_CHANNEL'].write()
        elif self.dut.name == 'lx9':  # Avnet LX9
            # enable LVDS RX/TX
            self.dut['I2C'].write(0xe8, [6, 0xf0, 0xff])
            self.dut['I2C'].write(0xe8, [2, 0x01, 0x00])  # select channels here
        elif self.dut.name in ['mmc3_8chip_eth', 'mmc3_8chip_multi_tx_eth', 'mmc3_16chip_multi_tx_eth']:  # Multi Module Card (MMC3)
            for register in self.dut.get_modules('StdRegister'):
                if 'DLY_CONFIG' in register.name:
                    register['CLK_DLY'] = 0
                    register.write()
        elif self.dut.name == 'mmc3_beast_eth':  # MMC3 for BEAST/FANGS experiment at KEK
            for register in self.dut.get_modules('StdRegister'):
                if 'DLY_CONFIG' in register.name:
                    register['CLK_DLY'] = 0
                    register.write()
        else:
            # This helps to enable regulators on Single Chip Adapter Card (SCAC) in a multi-board configuration
            # Note: Keep the original basil names and add a string to it, e.g.:
            #       SINGLE_CHIP_ADAPTER_CARD -> SINGLE_CHIP_ADAPTER_CARD_TELESCOPE_0
            #       POWER_SCC -> POWER_SCC_TELESCOPE_0
            #       ENABLE_CHANNEL -> ENABLE_CHANNEL_TELESCOPE_0
            for adapter_card in self.dut.get_modules('FEI4AdapterCard'):
                if 'SINGLE_CHIP_ADAPTER_CARD' in adapter_card.name:
                    try:
                        adapter_card.set_voltage('VDDA1', 1.5)
                        adapter_card.set_voltage('VDDA2', 1.5)
                        adapter_card.set_voltage('VDDD1', 1.2)
                        adapter_card.set_voltage('VDDD2', 1.2)
                    except struct.error:
                        logging.warning('Cannot set voltages. Adapter card not calibrated? Voltages out of range?')
            for register in self.dut.get_modules('StdRegister'):
                if 'POWER_SCC' in register.name:
                    register['EN_VD1'] = 1
                    register['EN_VD2'] = 1  # also EN_VPLL on old SCAC
                    register['EN_VA1'] = 1
                    register['EN_VA2'] = 1
                    register.write()
            for register in self.dut.get_modules('StdRegister'):
                if 'ENABLE_CHANNEL' in register.name:
                    register['DATA_CH1'] = 0  # RD2Bar on SCAC, CH1 on quad module adapter card
                    register['DATA_CH2'] = 0  # RD1Bar on SCAC, CH2 on quad module adapter card
                    register['DATA_CH3'] = 0  # RABar on SCAC, CH3 on quad module adapter card
                    register['DATA_CH4'] = 1
                    register['TLU'] = 1
                    register['TDC'] = 1
                    register.write()
#        else:
#            logging.warning('Unknown DUT name: %s', self.dut.name)

    def init_modules(self):
        ''' Initialize all modules consecutively'''
        for module_id, module_cfg in self._module_cfgs.items():
            if module_id in self._modules or module_id in self._tx_module_groups:
                if module_id in self._modules:
                    module_id_str = "module " + module_id
                else:
                    module_id_str = module_id.split('=', 1)
                    module_id_str[0] = module_id_str[0].replace("_", " ")
                    module_id_str = "=".join(module_id_str)
                logging.info("Initializing configuration for %s..." % module_id_str)
                # adding scan parameters to dict
                if 'scan_parameters' in self._module_run_conf[module_id] and self._module_run_conf[module_id]['scan_parameters'] is not None:
                    # evaluating string for support of nested lists and other complex data structures
                    if isinstance(self._module_run_conf[module_id]['scan_parameters'], basestring):
                        self._module_run_conf[module_id]['scan_parameters'] = ast.literal_eval(self._module_run_conf[module_id]['scan_parameters'])
                    sp = namedtuple('scan_parameters', field_names=zip(*self._module_run_conf[module_id]['scan_parameters'])[0])
                    self._scan_parameters[module_id] = sp(*zip(*self._module_run_conf[module_id]['scan_parameters'])[1])
                else:
                    sp = namedtuple_with_defaults('scan_parameters', field_names=[])
                    self._scan_parameters[module_id] = sp()
                # init FE config
                if module_id in self._modules:
                    # only real modules can have an existing configuration
                    last_configuration = self.get_configuration(module_id=module_id)
                else:
                    last_configuration = None
                if (('configuration' not in module_cfg or module_cfg['configuration'] is None) and last_configuration is None) or (isinstance(module_cfg['configuration'], (int, long)) and module_cfg['configuration'] <= 0):
                    if 'chip_address' in module_cfg:
                        if module_cfg['chip_address'] is None:
                            chip_address = 0
                            broadcast = True
                        else:
                            chip_address = module_cfg['chip_address']
                            broadcast = False
                    else:
                        raise ValueError('Parameter "chip_address" not specified for module "%s".' % module_id)
                    if 'flavor' in module_cfg and module_cfg['flavor']:
                        module_cfg['configuration'] = FEI4Register(fe_type=module_cfg['flavor'], chip_address=chip_address, broadcast=broadcast)
                    else:
                        raise ValueError('Parameter "flavor" not specified for module "%s".' % module_id)
                # use existing config
                elif not module_cfg['configuration'] and last_configuration:
                    module_cfg['configuration'] = FEI4Register(configuration_file=last_configuration)
                # path string
                elif isinstance(module_cfg['configuration'], basestring):
                    if os.path.isabs(module_cfg['configuration']):  # absolute path
                        module_cfg['configuration'] = FEI4Register(configuration_file=module_cfg['configuration'])
                    else:  # relative path
                        module_cfg['configuration'] = FEI4Register(configuration_file=os.path.join(module_cfg['working_dir'], module_cfg['configuration']))
                # run number
                elif isinstance(module_cfg['configuration'], (int, long)) and module_cfg['configuration'] > 0:
                    module_cfg['configuration'] = FEI4Register(configuration_file=self.get_configuration(module_id=module_id, run_number=module_cfg['configuration']))
                # assume configuration already initialized
                elif not isinstance(module_cfg['configuration'], FEI4Register):
                    raise ValueError('Found no valid value for parameter "configuration" for module "%s".' % module_id)

                # init register utils
                self._registers[module_id] = self._module_cfgs[module_id]['configuration']
                self._register_utils[module_id] = FEI4RegisterUtils(self._module_dut[module_id], self._module_cfgs[module_id]['configuration'])

                if module_id in self._modules:
                    # Create module data path for real modules
                    module_path = self.get_module_path(module_id)
                    if not os.path.exists(module_path):
                        os.makedirs(module_path)

        # Set all modules to conf mode to prevent from receiving BCR and ECR broadcast
        for module_id in self._tx_module_groups:
            with self.access_module(module_id=module_id):
                self.register_utils.set_conf_mode()

        # Initial configuration (reset and configuration) of all modules.
        # This is done by iterating over each module individually
        for module_id in self._modules:
            logging.info("Configuring %s..." % module_id)
            with self.access_module(module_id=module_id):
                if self._run_conf['configure_fe']:
                    self.register_utils.global_reset()
                    self.register_utils.configure_all()
                else:
                    self.register_utils.set_conf_mode()
                if is_fe_ready(self):
                    fe_not_ready = False
                else:
                    fe_not_ready = True
                # BCR and ECR might result in RX errors
                # a reset of the RX and FIFO will happen just before scan()
                if self._run_conf['reset_fe']:
                    self.register_utils.reset_bunch_counter()
                    self.register_utils.reset_event_counter()
                if fe_not_ready:
                    # resetting service records must be done once after power up
                    self.register_utils.reset_service_records()
                    if not is_fe_ready(self):
                        logging.warning('Module "%s" is not sending any data.' % module_id)
                # set all modules to conf mode afterwards to be immune to ECR and BCR
                self.register_utils.set_conf_mode()

    def pre_run(self):
        # clear error queue in case run is executed another time
        self.err_queue.queue.clear()
        # set default configuration, revert any changes that were done by last run
        self._set_default_cfg()

        # init DUT
        if not isinstance(self._conf['dut'], Dut):  # Check if already initialized
            module_path = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
            if isinstance(self._conf['dut'], basestring):
                # dirty fix for Windows pathes
                self._conf['dut'] = os.path.normpath(self._conf['dut'].replace('\\', '/'))
                # abs path
                if os.path.isabs(self._conf['dut']):
                    dut = self._conf['dut']
                # working dir, and directorys upwards
                elif find_file_dir_up(filename=self._conf['dut'], path=self._conf['working_dir']):
                    dut = find_file_dir_up(filename=self._conf['dut'], path=self._conf['working_dir'])
                # path of this file
                elif os.path.isfile(os.path.join(module_path, self._conf['dut'])):
                    dut = os.path.join(module_path, self._conf['dut'])
                else:
                    raise ValueError("Parameter 'dut' is not a valid path: %s" % self._conf['dut'])
                logging.info('Loading DUT configuration from file %s', os.path.abspath(dut))
            else:
                dut = self._conf['dut']
            dut = Dut(dut)

            # only initialize when DUT was not initialized before
            if 'dut_configuration' in self._conf and self._conf['dut_configuration']:
                if isinstance(self._conf['dut_configuration'], basestring):
                    # dirty fix for Windows pathes
                    self._conf['dut_configuration'] = os.path.normpath(self._conf['dut_configuration'].replace('\\', '/'))
                    # abs path
                    if os.path.isabs(self._conf['dut_configuration']):
                        dut_configuration = self._conf['dut_configuration']
                    # working dir, and directorys upwards
                    elif find_file_dir_up(filename=self._conf['dut_configuration'], path=self._conf['working_dir']):
                        dut_configuration = find_file_dir_up(filename=self._conf['dut_configuration'], path=self._conf['working_dir'])
                    # path of this file
                    elif os.path.isfile(os.path.join(module_path, self._conf['dut_configuration'])):
                        dut_configuration = os.path.join(module_path, self._conf['dut_configuration'])
                    else:
                        raise ValueError("Parameter 'dut_configuration' is not a valid path: %s" % self._conf['dut_configuration'])
                    logging.info('Loading DUT initialization parameters from file %s', os.path.abspath(dut_configuration))
                    # convert to dict
                    dut_configuration = RunManager.open_conf(dut_configuration)
                    # change bit file path
                    for drv in dut_configuration.iterkeys():
                        if 'bit_file' in dut_configuration[drv] and dut_configuration[drv]['bit_file']:
                            dut_configuration[drv]['bit_file'] = os.path.normpath(dut_configuration[drv]['bit_file'].replace('\\', '/'))
                            # abs path
                            if os.path.isabs(dut_configuration[drv]['bit_file']):
                                pass
                            # working dir, and directorys upwards
                            elif find_file_dir_up(filename=dut_configuration[drv]['bit_file'], path=self._conf['working_dir']):
                                dut_configuration[drv]['bit_file'] = find_file_dir_up(filename=dut_configuration[drv]['bit_file'], path=self._conf['working_dir'])
                            # path of this file
                            elif os.path.isfile(os.path.join(module_path, dut_configuration[drv]['bit_file'])):
                                dut_configuration[drv]['bit_file'] = os.path.join(module_path, dut_configuration[drv]['bit_file'])
                            else:
                                raise ValueError("Parameter 'bit_file' is not a valid path: %s" % dut_configuration[drv]['bit_file'])
                else:
                    dut_configuration = self._conf['dut_configuration']
            else:
                dut_configuration = None
            # check for existence of reserved driver names
            found_reserved_names = []
            for driver_name in _reserved_driver_names:
                try:
                    dut[driver_name]
                    found_reserved_names.append(driver_name)
                except KeyError:
                    pass
            if found_reserved_names:
                raise RuntimeError("The basil DUT contains reserved driver names: %s" % ", ".join(found_reserved_names))
            logging.info('Initializing basil...')
            dut.init(dut_configuration)
            # assign dut after init in case of exceptions during init
            self._conf['dut'] = dut
            # adding DUT handles
            for module_id, module_cfg in self._module_cfgs.items():
                self._module_dut[module_id] = DutHandle(dut=self._conf['dut'], module_cfg=module_cfg)
            self.fifo_readout = FifoReadout(dut=self._conf['dut'])
            # additional init of the DUT
            self.init_dut()
        else:
            # adding DUT handles
            for module_id, module_cfg in self._module_cfgs.items():
                self._module_dut[module_id] = DutHandle(dut=self._conf['dut'], module_cfg=module_cfg)
            self.fifo_readout = FifoReadout(dut=self._conf['dut'])

        # initialize the modules
        self.init_modules()

    def do_run(self):
        ''' Start runs on all modules sequentially.

        Sets properties to access current module properties.
        '''
        if self.broadcast_commands:  # Broadcast FE commands
            if self.threaded_scan:
                with ExitStack() as restore_config_stack:
                    # Configure each FE individually
                    # Sort module config keys, configure broadcast modules first
                    for module_id in itertools.chain(self._tx_module_groups, self._modules):
                        if self.abort_run.is_set():
                            break
                        with self.access_module(module_id=module_id):
                            if module_id in self._modules:
                                module_id_str = "module " + module_id
                            else:
                                module_id_str = module_id.split('=', 1)
                                module_id_str[0] = module_id_str[0].replace("_", " ")
                                module_id_str = "=".join(module_id_str)
                            logging.info('Scan parameter(s) for %s: %s', module_id_str, ', '.join(['%s=%s' % (key, value) for (key, value) in self.scan_parameters._asdict().items()]) if self.scan_parameters else 'None')
                            # storing register values until scan has finished and then restore configuration
                            restore_config_stack.enter_context(self.register.restored(name=self.run_number))
                            self.configure()
                    for module_id in self._tx_module_groups:
                        if self.abort_run.is_set():
                            break
                        with self.access_module(module_id=module_id):
                            # set all modules to run mode by before entering scan()
                            self.register_utils.set_run_mode()

                    with self.access_module(module_id=None):
                        self.fifo_readout.reset_rx()
                        self.fifo_readout.reset_fifo(self._selected_fifos)
                        self.fifo_readout.print_fei4_rx_status()

                        with self.access_files():
                            self._scan_threads = []
                            for module_id in self._tx_module_groups:
                                if self.abort_run.is_set():
                                    break
                                t = ExcThread(target=self.scan, name=module_id)
                                t.daemon = True  # exiting program even when thread is alive
                                self._scan_threads.append(t)
                            for t in self._scan_threads:
                                t.start()
                            while any([t.is_alive() for t in self._scan_threads]):
#                                 if self.abort_run.is_set():
#                                     break
                                for t in self._scan_threads:
                                    try:
                                        t.join(0.01)
                                    except Exception:
                                        self._scan_threads.remove(t)
                                        self.handle_err(sys.exc_info())
#                             alive_threads = [t.name for t in self._scan_threads if (not t.join(10.0) and t.is_alive())]
#                             if alive_threads:
#                                 raise RuntimeError("Scan thread(s) not finished: %s" % ", ".join(alive_threads))
                            self._scan_threads = []
                for module_id in self._tx_module_groups:
                    if self.abort_run.is_set():
                        break
                    with self.access_module(module_id=module_id):
                        # set modules to conf mode by after finishing scan()
                        self.register_utils.set_conf_mode()
            else:
                for tx_module_id, tx_group in self._tx_module_groups.items():
                    if self.abort_run.is_set():
                        break
                    with ExitStack() as restore_config_stack:
                        for module_id in itertools.chain([tx_module_id], tx_group):
                            if self.abort_run.is_set():
                                break
                            with self.access_module(module_id=module_id):
                                logging.info('Scan parameter(s) for module %s: %s', module_id, ', '.join(['%s=%s' % (key, value) for (key, value) in self.scan_parameters._asdict().items()]) if self.scan_parameters else 'None')
                                # storing register values until scan has finished and then restore configuration
                                restore_config_stack.enter_context(self.register.restored(name=self.run_number))
                                self.configure()
                        with self.access_module(module_id=tx_module_id):
                            # set all modules to run mode by before entering scan()
                            self.register_utils.set_run_mode()

                            self.fifo_readout.reset_rx()
                            self.fifo_readout.reset_fifo(self._selected_fifos)
                            self.fifo_readout.print_fei4_rx_status()

                            # some scans use this event to stop scan loop, clear event here to make another scan possible
                            self.stop_run.clear()
                            with self.access_files():
                                self.scan()

                    with self.access_module(module_id=tx_module_id):
                        # set modules to conf mode by after finishing scan()
                        self.register_utils.set_conf_mode()
        else:  # Scan each FE individually
            if self.threaded_scan:
                self._scan_threads = []
                # loop over grpups of modules with different TX
                for tx_module_ids in zip_nofill(*self._tx_module_groups.values()):
                    if self.abort_run.is_set():
                        break
                    with ExitStack() as restore_config_stack:
                        for module_id in tx_module_ids:
                            if self.abort_run.is_set():
                                break
                            with self.access_module(module_id=module_id):
                                logging.info('Scan parameter(s) for module %s: %s', module_id, ', '.join(['%s=%s' % (key, value) for (key, value) in self.scan_parameters._asdict().items()]) if self.scan_parameters else 'None')
                                # storing register values until scan has finished and then restore configuration
                                restore_config_stack.enter_context(self.register.restored(name=self.run_number))
                                self.configure()
                                # set modules to run mode by before entering scan()
                                self.register_utils.set_run_mode()
                            t = ExcThread(target=self.scan, name=module_id)
                            t.daemon = True  # exiting program even when thread is alive
                            self._scan_threads.append(t)
                        with self.access_module(module_id=tx_module_ids):
                            self.fifo_readout.reset_rx()
                            self.fifo_readout.reset_fifo(self._selected_fifos)
                            self.fifo_readout.print_fei4_rx_status()

                            with self.access_files():
                                # some scans use this event to stop scan loop, clear event here to make another scan possible
                                self.stop_run.clear()
                                for t in self._scan_threads:
                                    t.start()
                                while any([t.is_alive() for t in self._scan_threads]):
#                                     if self.abort_run.is_set():
#                                         break
                                    for t in self._scan_threads:
                                        try:
                                            t.join(0.01)
                                        except Exception:
                                            self._scan_threads.remove(t)
                                            self.handle_err(sys.exc_info())
#                                 alive_threads = [t.name for t in self._scan_threads if (not t.join(10.0) and t.is_alive())]
#                                 if alive_threads:
#                                     raise RuntimeError("Scan thread(s) not finished: %s" % ", ".join(alive_threads))
                                self._scan_threads = []

                    for module_id in tx_module_ids:
                        if self.abort_run.is_set():
                            break
                        with self.access_module(module_id=module_id):
                            # set modules to conf mode by after finishing scan()
                            self.register_utils.set_conf_mode()
            else:
                for module_id in self._modules:
                    if self.abort_run.is_set():
                        break
                    # some scans use this event to stop scan loop, clear event here to make another scan possible
                    self.stop_run.clear()
                    with self.access_module(module_id=module_id):
                        logging.info('Scan parameter(s) for module %s: %s', module_id, ', '.join(['%s=%s' % (key, value) for (key, value) in self.scan_parameters._asdict().items()]) if self.scan_parameters else 'None')
                        with self.register.restored(name=self.run_number):
                            self.configure()
                            # set modules to run mode by before entering scan()
                            self.register_utils.set_run_mode()

                            self.fifo_readout.reset_rx()
                            self.fifo_readout.reset_fifo(self._selected_fifos)
                            self.fifo_readout.print_fei4_rx_status()

                            # some scans use this event to stop scan loop, clear event here to make another scan possible
                            self.stop_run.clear()
                            with self.access_files():
                                self.scan()
                            # set modules to conf mode by after finishing scan()
                            self.register_utils.set_conf_mode()

        if self._modules:
            self.fifo_readout.print_readout_status()

    def post_run(self):
        # analyzing data and store register cfg per front end one by one
        for module_id in [name for name in self._modules if self._module_cfgs[name]["activate"] is True]:
            if self.abort_run.is_set():
                    break
            with self.access_module(module_id=module_id):
                try:
                    self.analyze()
                except Exception:  # analysis errors
                    exc = sys.exc_info()
                    self.err_queue.put(sys.exc_info())
                    logging.warning(exc[1].__class__.__name__ + ": " + str(exc[1]))
                else:  # analyzed data, save config
                    self.register.save_configuration(self.output_filename)
        if not self.err_queue.empty():
            exc = self.err_queue.get()
            # well known errors, do not print traceback
            if isinstance(exc[1], (RxSyncError, EightbTenbError, FifoError, NoDataTimeout, StopTimeout, AnalysisError)):
                raise RunAborted(exc[1])
            # some other error via handle_err(), print traceback
            else:
                raise exc[0], exc[1], exc[2]

    def cleanup_run(self):
        pass

    def close(self):
        '''Releasing hardware resources.
        '''
        try:
            self.dut.close()
        except Exception:
            logging.warning('Closing DUT was not successful')
        else:
            logging.debug('Closed DUT')

    def handle_data(self, data, new_file=False, flush=True):
        '''Handling of the data.

        Parameters
        ----------
        data : list, tuple
            Data tuple of the format (data (np.array), last_time (float), curr_time (float), status (int))
        '''
        for i, module_id in enumerate(self._selected_modules):
            if data[i] is None:
                continue
            self._raw_data_files[module_id].append(data_iterable=data[i], scan_parameters=self._scan_parameters[module_id]._asdict(), new_file=new_file, flush=flush)

    def handle_err(self, exc):
        '''Handling of Exceptions.

        Parameters
        ----------
        exc : list, tuple
            Information of the exception of the format (type, value, traceback).
            Uses the return value of sys.exc_info().
        '''
        if self.reset_rx_on_error and isinstance(exc[1], (RxSyncError, EightbTenbError)):
            self.fifo_readout.print_readout_status()
            self.fifo_readout.reset_rx()
        else:
            # print just the first error massage
            if not self.abort_run.is_set():
                self.abort(msg=exc[1].__class__.__name__ + ": " + str(exc[1]))
            self.err_queue.put(exc)

    def get_module_path(self, module_id):
        if module_id not in self._modules:
            raise ValueError('Module ID "%s" is not a valid name.' % module_id)
        return os.path.join(self.working_dir, module_id)

    def get_configuration(self, module_id, run_number=None):
        ''' Returns the configuration for a given module ID.

        The working directory is searched for a file matching the module_id with the
        given run number. If no run number is defined the last successfull run defines
        the run number.
        '''
        def find_file(run_number):
            module_path = self.get_module_path(module_id)
            for root, _, files in os.walk(module_path):
                for cfgfile in files:
                    cfg_root, cfg_ext = os.path.splitext(cfgfile)
                    if cfg_root.startswith(''.join([str(run_number), '_', module_id])) and cfg_ext.endswith(".cfg"):
                        return os.path.join(root, cfgfile)

        if not run_number:
            run_numbers = sorted(self._get_run_numbers(status='FINISHED').keys(), reverse=True)
            found_fin_run_cfg = True
            if not run_numbers:
                return None
            last_fin_run = run_numbers[0]
            for run_number in run_numbers:
                cfg_file = find_file(run_number)
                if cfg_file:
                    if not found_fin_run_cfg:
                        logging.warning("Module '%s' has no configuration for run %d, use config of run %d", module_id, last_fin_run, run_number)
                    return cfg_file
                else:
                    found_fin_run_cfg = False
        else:
            cfg_file = find_file(run_number)
            if cfg_file:
                return cfg_file
            else:
                raise ValueError('Found no configuration with run number %s' % run_number)

    def set_scan_parameters(self, *args, **kwargs):
        fields = dict(kwargs)
        for index, field in enumerate(self.scan_parameters._fields):
            try:
                value = args[index]
            except IndexError:
                break
            else:
                if field in fields:
                    raise TypeError('Got multiple values for keyword argument %s' % field)
                fields[field] = value
        if self.current_module_handle is None:
            selected_modules = self._modules.keys()
        elif self.current_module_handle in self._modules:
            selected_modules = [self.current_module_handle]
        elif self.current_module_handle in self._tx_module_groups:
            selected_modules = self._tx_module_groups[self.current_module_handle]
        else:
            RuntimeError('Cannot change scan parameters. Module handle "%s" is not valid.' % self.current_module_handle)
        scan_parameters_old = self.scan_parameters._asdict()
        with self.global_lock:
            for module_id in selected_modules:
                self._scan_parameters[module_id] = self.scan_parameters._replace(**fields)
        scan_parameters_new = self.scan_parameters._asdict()
        diff = [name for name in scan_parameters_old.keys() if np.any(scan_parameters_old[name] != scan_parameters_new[name])]
        if diff:
            logging.info('Changing scan parameter(s): %s', ', '.join([('%s=%s' % (name, fields[name])) for name in diff]))

    def __setattr__(self, name, value):
        ''' Always called to retrun the value for an attribute.
        '''
        if self.is_initialized and name not in self.__dict__:
            if name in self._module_run_conf[self.current_module_handle]:
                raise RuntimeError('"%s" (current module handle "%s") cannot use attribute "%s" because it is already used in run conf.' % (self.__class__.__name__, self.current_module_handle, name))
            self._module_attr[self.current_module_handle][name] = value
            # set attribute for all other mudules in module group
            if self.current_module_handle in self._tx_module_groups:
                for module_id in self._tx_module_groups[self.current_module_handle]:
                    self._module_attr[module_id][name] = value
        else:
            super(Fei4RunBase, self).__setattr__(name, value)

    def __getattr__(self, name):
        ''' This is called in a last attempt to receive the value for an attribute that was not found in the usual places.
        '''
        # test for attribute name in module attribute dict first
        if self.is_initialized and name in self._module_attr[self.current_module_handle]:
            return self._module_attr[self.current_module_handle][name]
        else:
            try:
                return self._module_run_conf[self.current_module_handle][name]  # Accessing run conf parameters
            except KeyError:
                try:
                    return super(Fei4RunBase, self).__getattr__(name)
                except AttributeError:
                    if self.is_initialized:
                        raise AttributeError('"%s" (current module handle "%s") has no attribute "%s"' % (self.__class__.__name__, self.current_module_handle, name))
                    else:
                        raise

    @contextmanager
    def access_module(self, module_id):
        self.select_module(module_id=module_id)
        try:
            yield
        finally:
            try:
                self.deselect_module()
            except Exception:
                # in case something fails, call this on last resort
                self._current_module_handle = None
                if isinstance(current_thread(), _MainThread):
                    current_thread().name = "MainThread"

    def select_module(self, module_id):
        ''' Select module and give access to the module.
        '''
        if not isinstance(module_id, basestring) and isinstance(module_id, Iterable) and set(module_id) - set(self._modules):
            raise ValueError('Module IDs invalid:' % ", ".join(set(module_id) - set(self._modules)))
        if isinstance(module_id, basestring) and module_id not in self._module_cfgs:
            raise ValueError('Module ID "%s" is not valid' % module_id)
        if self._current_module_handle is not None:
            raise RuntimeError('Module handle "%s" cannot be set because another module is active' % module_id)

        if module_id is None:
            self._selected_modules = self._modules.keys()
        elif not isinstance(module_id, basestring) and isinstance(module_id, Iterable):
            self._selected_modules = module_id
        elif module_id in self._modules:
            self._selected_modules = [module_id]
        elif module_id in self._tx_module_groups:
            self._selected_modules = self._tx_module_groups[module_id]
        else:
            RuntimeError('Cannot open files. Module handle "%s" is not valid.' % self.current_module_handle)

        # FIFO readout
        self._selected_fifos = list(set([module_cfg['FIFO'] for (name, module_cfg) in self._module_cfgs.items() if name in self._selected_modules]))

        # Module filter functions dict for quick lookup
        self._readout_fifos = []
        self._filter = []
        self._converter = []
        for selected_module_id in self._selected_modules:
            module_cfg = self._module_cfgs[selected_module_id]
            self._readout_fifos.append(module_cfg['FIFO'])
            if 'tdc_channel' not in module_cfg:
                tdc_filter = false
                self._converter.append(None)
            elif module_cfg['tdc_channel'] is None:
                tdc_filter = is_tdc_word
                self._converter.append(convert_tdc_to_channel(channel=module_cfg['tdc_channel']))  # for the raw data analyzer
            else:
                tdc_filter = logical_and(is_tdc_word, is_tdc_from_channel(module_cfg['tdc_channel']))
                self._converter.append(convert_tdc_to_channel(channel=module_cfg['tdc_channel']))  # for the raw data analyzer
            if 'rx_channel' not in module_cfg:
                self._filter.append(logical_or(is_trigger_word, tdc_filter))
            elif module_cfg['rx_channel'] is None:
                self._filter.append(logical_or(is_trigger_word, logical_or(tdc_filter, is_fe_word)))
            else:
                self._filter.append(logical_or(is_trigger_word, logical_or(tdc_filter, logical_and(is_fe_word, is_data_from_channel(module_cfg['rx_channel'])))))

        # select readout channels and report sync status only from actively selected modules
        self._enabled_fe_channels = list(set([config['RX'] for (name, config) in self._module_cfgs.items() if name in self._selected_modules]))

        # enabling specific TX channels
        tx_channels = list(set([1 << config['tx_channel'] for (name, config) in self._module_cfgs.items() if name in self._selected_modules]))
        if tx_channels:
            self.dut['TX']['OUTPUT_ENABLE'] = reduce(lambda x, y: x | y, tx_channels)
        else:
            self.dut['TX']['OUTPUT_ENABLE'] = 0

        if not isinstance(module_id, basestring) and isinstance(module_id, Iterable):
            self._current_module_handle = None
        else:
            self._current_module_handle = module_id

        if module_id is not None and isinstance(module_id, basestring):
            current_thread().name = module_id

    def deselect_module(self):
        ''' Deselect module and cleanup.
        '''
        self._enabled_fe_channels = []  # ignore any RX sync errors
        self._readout_fifos = []
        self._filter = []
        self._converter = []
        self.dut['TX']['OUTPUT_ENABLE'] = 0
        self._current_module_handle = None
        if isinstance(current_thread(), _MainThread):
            current_thread().name = "MainThread"

    @contextmanager
    def access_files(self):
        try:
            self.open_files()
            yield
            self.close_files()
        finally:
            # in case something fails, call this on last resort
            self._raw_data_files.clear()

    def open_files(self):
        for selected_module_id in self._selected_modules:
            self._raw_data_files[selected_module_id] = open_raw_data_file(filename=self.get_output_filename(module_id=selected_module_id),
                                                                          mode='w',
                                                                          title=self.run_id,
                                                                          scan_parameters=self._scan_parameters[selected_module_id]._asdict(),
                                                                          socket_address=self._module_cfgs[selected_module_id]['send_data'])
            # save configuration data to raw data file
            self._registers[selected_module_id].save_configuration(self._raw_data_files[selected_module_id].h5_file)
            save_configuration_dict(self._raw_data_files[selected_module_id].h5_file, 'conf', self._conf)
            save_configuration_dict(self._raw_data_files[selected_module_id].h5_file, 'run_conf', self._module_run_conf[selected_module_id])
            # send configuration data to online monitor
            if self._raw_data_files[selected_module_id].socket:
                send_meta_data(self._raw_data_files[selected_module_id].socket, selected_module_id, name='Filename')
                global_register_config = {}
                for global_reg in sorted(self._registers[selected_module_id].get_global_register_objects(readonly=False), key=itemgetter('name')):
                    global_register_config[global_reg['name']] = global_reg['value']
                send_meta_data(self._raw_data_files[selected_module_id].socket, global_register_config, name='GlobalRegisterConf')
                send_meta_data(self._raw_data_files[selected_module_id].socket, self._run_conf, name='RunConf')

    def close_files(self):
        # close all file objects
        for f in self._raw_data_files.values():
            f.close()
        # delete all file objects
        self._raw_data_files.clear()

    def get_output_filename(self, module_id):
        if module_id not in self._modules:
            raise ValueError('Module ID "%s" is not a valid name.' % module_id)
        module_path = os.path.join(self.working_dir, module_id)
        return os.path.join(module_path, str(self.run_number) + "_" + module_id + "_" + self.run_id)

    def read_data(self, filter_func=None, converter_func=None):
        with self._readout_lock:
            if self.fifo_readout.fill_buffer:
                return self.get_raw_data_from_buffer(filter_func=None, converter_func=None)[self._selected_modules.index(self.current_module_handle)]
            else:
                return self.read_raw_data_from_fifo(filter_func=filter_func, converter_func=converter_func)

    def get_raw_data_from_buffer(self, filter_func=None, converter_func=None):
        return self.fifo_readout.get_raw_data_from_buffer(filter_func=filter_func, converter_func=converter_func)

    def read_raw_data_from_fifo(self, filter_func=None, converter_func=None):
        return self.fifo_readout.read_raw_data_from_fifo(filter_func=filter_func, converter_func=converter_func)

    def data_words_per_second(self):
        if self.current_module_handle is None:
            return sum(self.fifo_readout.data_words_per_second())
        elif self.current_module_handle in self._modules:
            return self.fifo_readout.data_words_per_second()[self._selected_modules.index(self.current_module_handle)]
        elif self.current_module_handle in self._tx_module_groups:
            return sum([dw_per_s for i, dw_per_s in enumerate(self.fifo_readout.data_words_per_second()) if self._selected_modules[i] in self._tx_module_groups[self.current_module_handle]])
        else:
            RuntimeError('Module handle "%s" is not valid.' % self.current_module_handle)

    @contextmanager
    def synchronized(self):
        ''' Synchronize the execution of a section of code between threads.
        '''
        self.enter_sync()
        try:
            yield
            self.exit_sync()
        finally:
            # in case something fails, call this on last resort
            pass

    def enter_sync(self):
        ''' Waiting for all threads to appear, then continue.
        '''
        if self._scan_threads and self.current_module_handle not in [t.name for t in self._scan_threads]:
            raise RuntimeError('Thread name "%s" is not valid.')
        if self._scan_threads and self.current_module_handle in self._curr_sync_threads:
            raise RuntimeError('Thread "%s" is already actively reading FIFO.')
        with self._sync_lock:
            self._curr_sync_threads.append(self.current_module_handle)
        self._enter_sync_event.clear()
        while not self._enter_sync_event.wait(0.01):
            if self.abort_run.is_set():
                break
            with self._sync_lock:
                if len(set(self._curr_sync_threads) & set([t.name for t in self._scan_threads if t.is_alive()])) == len(set([t.name for t in self._scan_threads if t.is_alive()])) or not self._scan_threads:
                    self._enter_sync_event.set()

    def exit_sync(self):
        ''' Waiting for all threads to appear, then continue.
        '''
        if self._scan_threads and self.current_module_handle not in [t.name for t in self._scan_threads]:
            raise RuntimeError('Thread name "%s" is not valid.')
        if self._scan_threads and self.current_module_handle not in self._curr_sync_threads:
            raise RuntimeError('Thread "%s" is not reading FIFO.')
        with self._sync_lock:
            self._curr_sync_threads.remove(self.current_module_handle)
        self._exit_sync_event.clear()
        while not self._exit_sync_event.wait(0.01):
            if self.abort_run.is_set():
                break
            with self._sync_lock:
                if len(set(self._curr_sync_threads) & set([t.name for t in self._scan_threads if t.is_alive()])) == 0 or not self._scan_threads:
                    self._exit_sync_event.set()

    @contextmanager
    def readout(self, *args, **kwargs):
        ''' Running the FIFO readout while executing other statements.

        Starting and stopping of the FIFO readout is synchronized between the threads.
        '''
        timeout = kwargs.pop('timeout', 10.0)
        self.start_readout(*args, **kwargs)
        try:
            yield
        finally:
            try:
                self.stop_readout(timeout=timeout)
            except Exception:
                # in case something fails, call this on last resort
                # if run was aborted, immediately stop readout
                if self.abort_run.is_set():
                    with self._readout_lock:
                        if self.fifo_readout.is_running:
                            self.fifo_readout.stop(timeout=0.0)

    def start_readout(self, *args, **kwargs):
        ''' Starting the FIFO readout.

        Starting of the FIFO readout is executed only once by a random thread.
        Starting of the FIFO readout is synchronized between all threads reading out the FIFO.
        '''
        # Pop parameters for fifo_readout.start
        callback = kwargs.pop('callback', self.handle_data)
        errback = kwargs.pop('errback', self.handle_err)
        reset_rx = kwargs.pop('reset_rx', True)
        reset_fifo = kwargs.pop('reset_fifo', True)
        fill_buffer = kwargs.pop('fill_buffer', False)
        no_data_timeout = kwargs.pop('no_data_timeout', None)
        enabled_fe_channels = kwargs.pop('enabled_fe_channels', self._enabled_fe_channels)
        if args or kwargs:
            self.set_scan_parameters(*args, **kwargs)
        if self._scan_threads and self.current_module_handle not in [t.name for t in self._scan_threads]:
            raise RuntimeError('Thread name "%s" is not valid.' % t.name)
        if self._scan_threads and self.current_module_handle in self._curr_readout_threads:
            raise RuntimeError('Thread "%s" is already actively reading FIFO.')
        with self._readout_lock:
            self._curr_readout_threads.append(self.current_module_handle)
        self._starting_readout_event.clear()
        while not self._starting_readout_event.wait(0.01):
            if self.abort_run.is_set():
                break
            with self._readout_lock:
                if len(set(self._curr_readout_threads) & set([t.name for t in self._scan_threads if t.is_alive()])) == len(set([t.name for t in self._scan_threads if t.is_alive()])) or not self._scan_threads:
                    if not self.fifo_readout.is_running:
                        self.fifo_readout.start(fifos=self._selected_fifos, callback=callback, errback=errback, reset_rx=reset_rx, reset_fifo=reset_fifo, fill_buffer=fill_buffer, no_data_timeout=no_data_timeout, filter_func=self._filter, converter_func=self._converter, fifo_select=self._readout_fifos, enabled_fe_channels=enabled_fe_channels)
                        self._starting_readout_event.set()

    def stop_readout(self, timeout=10.0):
        ''' Stopping the FIFO readout.

        Stopping of the FIFO readout is executed only once by a random thread.
        Stopping of the FIFO readout is synchronized between all threads reading out the FIFO.
        '''
        if self._scan_threads and self.current_module_handle not in [t.name for t in self._scan_threads]:
            raise RuntimeError('Thread name "%s" is not valid.')
        if self._scan_threads and self.current_module_handle not in self._curr_readout_threads:
            raise RuntimeError('Thread "%s" is not reading FIFO.')
        with self._readout_lock:
            self._curr_readout_threads.remove(self.current_module_handle)
        self._stopping_readout_event.clear()
        while not self._stopping_readout_event.wait(0.01):
            with self._readout_lock:
                if len(set(self._curr_readout_threads) & set([t.name for t in self._scan_threads if t.is_alive()])) == 0 or not self._scan_threads or self.abort_run.is_set():
                    if self.fifo_readout.is_running:
                        self.fifo_readout.stop(timeout=timeout)
                    self._stopping_readout_event.set()

    def _cleanup(self):  # called in run base after exception handling
        super(Fei4RunBase, self)._cleanup()
        if 'send_message' in self._conf and self._run_status in self._conf['send_message']['status']:
            subject = '{}{} ({})'.format(self._conf['send_message']['subject_prefix'], self._run_status, gethostname())
            last_status_message = '{} run {} ({}) in {} (total time: {})'.format(self.run_status, self.run_number, self.__class__.__name__, self.working_dir, str(self._total_run_time))
            body = '\n'.join(item for item in [self._last_traceback, last_status_message] if item)
            try:
                send_mail(subject=subject, body=body, smtp_server=self._conf['send_message']['smtp_server'], user=self._conf['send_message']['user'], password=self._conf['send_message']['password'], from_addr=self._conf['send_message']['from_addr'], to_addrs=self._conf['send_message']['to_addrs'])
            except Exception:
                logging.warning("Failed sending pyBAR status report")

    def configure(self):
        '''The module configuration happens here.

        Will be executed before calling the scan method.
        Any changes of the module configuration will be reverted after after finishing the scan method.
        '''
        pass

    def scan(self):
        '''Implementation of the scan.
        '''
        pass

    def analyze(self):
        '''Implementation of the data analysis.

        Will be executed after finishing the scan method.
        '''
        pass class RunManager(object):
    def __init__(self, conf):
        '''Run Manager is taking care of initialization and execution of runs.

        Parameters
        ----------
        conf : str, dict, file
            Configuration for the run(s). Configuration will be passed to all scans.
        '''
        # fixing event handler: http://stackoverflow.com/questions/15457786/ctrl-c-crashes-python-after-importing-scipy-stats
        if os.name == 'nt':
            import thread

            def handler(signum, hook=thread.interrupt_main):
                hook()
                return True

            import win32api
            win32api.SetConsoleCtrlHandler(handler, 1)

        self._conf = None  # configuration dictionary
        self.current_run = None  # current run number
        self._conf_path = None  # absolute path of the configuation file
        self.init(conf)

    @property
    def conf(self):
        '''Configuration (namedtuple)
        '''
        conf = namedtuple('conf', field_names=self._conf.keys())
        return conf(**self._conf)  # prevent changing dict

    def init(self, conf):
        # current working directory
        if isinstance(conf, basestring) and os.path.isfile(os.path.abspath(conf)):
            conf = os.path.abspath(conf)
            self._conf_path = conf
        # search directory upwards form current working directory
        elif isinstance(conf, basestring) and find_file_dir_up(conf):
            conf = find_file_dir_up(conf)
            self._conf_path = conf
        elif isinstance(conf, file):
            self._conf_path = os.path.abspath(conf.name)
        else:
            self._conf_path = None
        self._conf = self.open_conf(conf)
        if 'working_dir' in self._conf and self._conf['working_dir']:
            # dirty fix for Windows pathes
            self._conf['working_dir'] = os.path.normpath(self._conf['working_dir'].replace('\\', '/'))
            if self._conf_path and not os.path.isabs(self._conf['working_dir']):
                # if working_dir is relative path, join path to configuration file and working_dir
                self._conf['working_dir'] = os.path.join(os.path.dirname(self._conf_path), self._conf['working_dir'])
            else:
                # working_dir is absolute path, keep that
                pass
        # use path of configuration file
        elif self._conf_path:
            self._conf['working_dir'] = os.path.dirname(self._conf_path)
        else:
            raise ValueError('Cannot deduce working directory from configuration')
        logging.info('Using working directory %s', self._conf['working_dir'])

    def close(self):
        if self.current_run is not None:
            self.current_run.close()

    def __enter__(self):
        return self

    def __exit__(self, *exc_info):
        self.close()

    @staticmethod
    def open_conf(conf):
        conf_dict = {}
        if not conf:
            pass
        elif isinstance(conf, basestring):  # parse the first YAML document in a stream
            if os.path.isfile(os.path.abspath(conf)):
                logging.info('Loading configuration from file %s', os.path.abspath(conf))
                with open(os.path.abspath(conf), 'r') as f:
                    conf_dict.update(safe_load(f))
            else:  # YAML string
                try:
                    conf_dict.update(safe_load(conf))
                except ValueError:  # invalid path/filename
                    raise IOError("File not found: %s" % os.path.abspath(conf))
                else:
                    logging.info('Loading configuration from file %s', os.path.abspath(conf.name))
        elif isinstance(conf, file):  # parse the first YAML document in a stream
            conf_dict.update(safe_load(conf))
        else:  # conf is already a dict
            conf_dict.update(conf)
        return conf_dict

    def cancel_current_run(self, msg=None):
        '''Control for runs.
        '''
        self.current_run.handle_cancel(msg=msg)

    def run_run(self, run, conf=None, run_conf=None, use_thread=False, catch_exception=True):
        '''Runs a run in another thread. Non-blocking.

        Parameters
        ----------
        run : class, object
            Run class or object.
        run_conf : str, dict, file
            Specific configuration for the run.
        use_thread : bool
            If True, run run in thread and returns blocking function.

        Returns
        -------
        If use_thread is True, returns function, which blocks until thread terminates, and which itself returns run status.
        If use_thread is False, returns run status.
        '''
        if isinstance(conf, basestring) and os.path.isfile(conf):
            logging.info('Updating configuration from file %s', os.path.abspath(conf))
        elif conf is not None:
            logging.info('Updating configuration')
        conf = self.open_conf(conf)
        self._conf.update(conf)

        if isclass(run):
            # instantiate the class
            run = run(conf=self._conf)

        local_run_conf = {}
        # general parameters from conf
        if 'run_conf' in self._conf:
            logging.info('Updating run configuration using run_conf key from configuration')
            local_run_conf.update(self._conf['run_conf'])
        # check for class name, scan specific parameters from conf
        if run.__class__.__name__ in self._conf:
            logging.info('Updating run configuration using %s key from configuration' % (run.__class__.__name__,))
            local_run_conf.update(self._conf[run.__class__.__name__])

        if isinstance(run_conf, basestring) and os.path.isfile(run_conf):
            logging.info('Updating run configuration from file %s', os.path.abspath(run_conf))
        elif run_conf is not None:
            logging.info('Updating run configuration')
        run_conf = self.open_conf(run_conf)
        # check for class name, scan specific parameters from conf
        if run.__class__.__name__ in run_conf:
            run_conf = run_conf[run.__class__.__name__]
        # run_conf parameter has highest priority, updated last
        local_run_conf.update(run_conf)

        if use_thread:
            self.current_run = run

            @thunkify(thread_name='RunThread', daemon=True, default_func=self.current_run.get_run_status)
            def run_run_in_thread():
                return run.run(run_conf=local_run_conf)

            signal.signal(signal.SIGINT, self._signal_handler)
            logging.info('Press Ctrl-C to stop run')

            return run_run_in_thread()
        else:
            self.current_run = run
            status = run.run(run_conf=local_run_conf)
            if not catch_exception and status != run_status.finished:
                raise RuntimeError('Exception occurred. Please read the log.')
            return status

    def run_primlist(self, primlist, skip_remaining=False):
        '''Runs runs from a primlist.

        Parameters
        ----------
        primlist : string
            Filename of primlist.
        skip_remaining : bool
            If True, skip remaining runs, if a run does not exit with status FINISHED.

        Note
        ----
        Primlist is a text file of the following format (comment line by adding '#'):
        <module name (containing class) or class (in either case use dot notation)>; <scan parameter>=<value>; <another scan parameter>=<another value>
        '''
        runlist = self.open_primlist(primlist)
        for index, run in enumerate(runlist):
            logging.info('Progressing with run %i out of %i...', index + 1, len(runlist))
            join = self.run_run(run, use_thread=True)
            status = join()
            if skip_remaining and not status == run_status.finished:
                logging.error('Exited run %i with status %s: Skipping all remaining runs.', run.run_number, status)
                break

    def open_primlist(self, primlist):
        def isrun(item, module):
            return isinstance(item, RunBase.__metaclass__) and item.__module__ == module  # only class from module, not from other imports

        if isinstance(primlist, basestring):
            with open(primlist, 'r') as f:
                f.seek(0)
                run_list = []
                for line in f.readlines():
                    line = line.partition('#')[0].strip()
                    if not line:
                        continue
                    parts = re.split('\s*[;]\s*', line)
                    try:
                        mod = import_module(parts[0])  # points to module
                    except ImportError:
                        mod = import_module(parts[0].rsplit('.', 1)[0])  # points to class
                        islocalrun = partial(isrun, module=parts[0].split('.')[-2])
                        clsmembers = getmembers(mod, islocalrun)
                        run_cls = None
                        for cls in clsmembers:
                            if cls[0] == parts[0].rsplit('.', 1)[1]:
                                run_cls = cls[1]
                                break
                        if not run_cls:
                            raise ValueError('Found no matching class: %s' % parts[0].rsplit('.', 1)[1])
                    else:
                        islocalrun = partial(isrun, module=parts[0])
                        clsmembers = getmembers(mod, islocalrun)
                        if len(clsmembers) > 1:
                            raise ValueError('Found more than one matching class.')
                        elif not len(clsmembers):
                            raise ValueError('Found no matching class.')
                        run_cls = clsmembers[0][1]
                    if run_cls.__class__.__name__ in self._conf:
                        run_conf = self._conf[run_cls.__class__.__name__]
                    else:
                        run_conf = {}
                    for param in parts[1:]:
                        key, value = re.split('\s*[=:]\s*', param, 1)
                        run_conf[key] = literal_eval(value)
                    run_list.append(run_cls(conf=self._conf, run_conf=run_conf))
            return run_list
        else:
            AttributeError('Primlist format not supported.')

    def _signal_handler(self, signum, frame):
        signal.signal(signal.SIGINT, signal.SIG_DFL)  # setting default handler... pressing Ctrl-C a second time will kill application
        self.cancel_current_run(msg='Pressed Ctrl-C') def plot_occupancy(hist, title='Occupancy', z_max=None, filename=None):
    if z_max == 'median':
        z_max = 2 * np.ma.median(hist)
    elif z_max == 'maximum' or z_max is None:
        z_max = np.ma.max(hist)
    if z_max < 1 or hist.all() is np.ma.masked or np.allclose(0, hist):
        z_max = 1.0

    fig = Figure()
    FigureCanvas(fig)
    ax = fig.add_subplot(111)
    extent = [0.5, 80.5, 336.5, 0.5]
    bounds = np.linspace(start=0, stop=z_max, num=255, endpoint=True)
    if z_max == 'median':
        cmap = cm.get_cmap('coolwarm')
    else:
        cmap = cm.get_cmap('cool')
    cmap.set_bad('w', 1.0)
    norm = colors.BoundaryNorm(bounds, cmap.N)

    im = ax.imshow(hist, interpolation='none', aspect='auto', cmap=cmap, norm=norm, extent=extent)  # TODO: use pcolor or pcolormesh
    ax.set_ylim((336.5, 0.5))
    ax.set_xlim((0.5, 80.5))
    ax.set_title(title + r' ($\Sigma$ = {0})'.format((0 if (hist.all() is np.ma.masked or np.allclose(0, hist)) else np.ma.sum(hist))))
    ax.set_xlabel('Column')
    ax.set_ylabel('Row')

    divider = make_axes_locatable(ax)

    cax = divider.append_axes("right", size="5%", pad=0.1)
    cb = fig.colorbar(im, cax=cax, ticks=np.linspace(start=0, stop=z_max, num=9, endpoint=True))
    cb.set_label("#")

    if not filename:
        fig.show()
    elif isinstance(filename, PdfPages):
        filename.savefig(fig)
    else:
        fig.savefig(filename)
 def plot_fancy_occupancy(hist, z_max=None, filename=None):
    if z_max == 'median':
        z_max = 2 * np.ma.median(hist)
    elif z_max == 'maximum' or z_max is None:
        z_max = np.ma.max(hist)
    if z_max < 1 or hist.all() is np.ma.masked or np.allclose(0, hist):
        z_max = 1.0

    fig = Figure()
    FigureCanvas(fig)
    ax = fig.add_subplot(111)
    extent = [0.5, 80.5, 336.5, 0.5]
    bounds = np.linspace(start=0, stop=z_max, num=255, endpoint=True)
    if z_max == 'median':
        cmap = cm.get_cmap('coolwarm')
    else:
        cmap = cm.get_cmap('cool')
    cmap.set_bad('w', 1.0)
    norm = colors.BoundaryNorm(bounds, cmap.N)

    im = ax.imshow(hist, interpolation='none', aspect='auto', cmap=cmap, norm=norm, extent=extent)  # TODO: use pcolor or pcolormesh
    ax.set_ylim((336.5, 0.5))
    ax.set_xlim((0.5, 80.5))
    ax.set_xlabel('Column')
    ax.set_ylabel('Row')

    # create new axes on the right and on the top of the current axes
    # The first argument of the new_vertical(new_horizontal) method is
    # the height (width) of the axes to be created in inches.
    divider = make_axes_locatable(ax)
    axHistx = divider.append_axes("top", 1.2, pad=0.2, sharex=ax)
    axHisty = divider.append_axes("right", 1.2, pad=0.2, sharey=ax)

    cax = divider.append_axes("right", size="5%", pad=0.1)
    cb = fig.colorbar(im, cax=cax, ticks=np.linspace(start=0, stop=z_max, num=9, endpoint=True))
    cb.set_label("#")
    # make some labels invisible
    setp(axHistx.get_xticklabels() + axHisty.get_yticklabels(), visible=False)
    height = np.ma.sum(hist, axis=0)

    axHistx.bar(range(1, 81), height, align='center', linewidth=0)
    axHistx.set_xlim((0.5, 80.5))
    if hist.all() is np.ma.masked or np.allclose(0, hist):
        axHistx.set_ylim((0, 1))
    else:
        x_c_max = np.ceil(np.percentile(height, 99))
        axHistx.set_ylim(0, max(1, x_c_max))
    axHistx.locator_params(axis='y', nbins=3)
    axHistx.ticklabel_format(style='sci', scilimits=(0, 4), axis='y')
    axHistx.set_ylabel('#')
    width = np.ma.sum(hist, axis=1)

    axHisty.barh(range(1, 337), width, align='center', linewidth=0)
    axHisty.set_ylim((336.5, 0.5))
    if hist.all() is np.ma.masked or np.allclose(0, hist):
        axHisty.set_xlim((0, 1))
    else:
        y_c_max = np.ceil(np.percentile(width, 99))
        axHisty.set_xlim(0, max(1, y_c_max))
    axHisty.locator_params(axis='x', nbins=3)
    axHisty.ticklabel_format(style='sci', scilimits=(0, 4), axis='x')
    axHisty.set_xlabel('#')

    if not filename:
        fig.show()
    elif isinstance(filename, PdfPages):
        filename.savefig(fig)
    else:
        fig.savefig(filename)
 def plot_three_way(hist, title, filename=None, x_axis_title=None, minimum=None, maximum=None, bins=101, cmap=None):  # the famous 3 way plot (enhanced)
    if cmap is None:
        if maximum == 'median' or maximum is None:
            cmap = cm.get_cmap('coolwarm')
        else:
            cmap = cm.get_cmap('cool')
    # TODO: set color for bad pixels
    # set nan to special value
    # masked_array = np.ma.array (a, mask=np.isnan(a))
    # cmap = matplotlib.cm.jet
    # cmap.set_bad('w',1.0)
    # ax.imshow(masked_array, interpolation='none', cmap=cmap)
    hist = np.ma.masked_invalid(hist)
    if minimum is None:
        minimum = 0.0
    elif minimum == 'minimum':
        minimum = max(0, np.ma.min(hist))
    if maximum == 'median' or maximum is None:
        maximum = max(1, 2 * np.ma.median(hist))
    elif maximum == 'maximum':
        maximum = max(1, np.ma.max(hist))
    if maximum < 1 or hist.all() is np.ma.masked or np.allclose(0, hist):
        maximum = 1.0

    x_axis_title = '' if x_axis_title is None else x_axis_title
    fig = Figure()
    FigureCanvas(fig)
    fig.patch.set_facecolor('white')
    ax1 = fig.add_subplot(311)
    create_2d_pixel_hist(fig, ax1, hist, title=title, x_axis_title="column", y_axis_title="row", z_min=minimum if minimum else 0, z_max=maximum, cmap=cmap)
    ax2 = fig.add_subplot(312)
    create_1d_hist(ax2, hist, bins=bins, x_axis_title=x_axis_title, y_axis_title="#", x_min=minimum, x_max=maximum)
    ax3 = fig.add_subplot(313)
    create_pixel_scatter_plot(ax3, hist, x_axis_title="channel=row + column*336", y_axis_title=x_axis_title, y_min=minimum, y_max=maximum)
    if not filename:
        fig.show()
    elif isinstance(filename, PdfPages):
        filename.savefig(fig)
    else:
        fig.savefig(filename)
 def hist_quantiles(hist, prob=(0.05, 0.95), return_indices=False, copy=True):
    '''Calculate quantiles from histograms, cuts off hist below and above given quantile. This function will not cut off more than the given values.

    Parameters
    ----------
    hist : array_like, iterable
        Input histogram with dimension at most 1.
    prob : float, list, tuple
        List of quantiles to compute. Upper and lower limit. From 0 to 1. Default is 0.05 and 0.95.
    return_indices : bool, optional
        If true, return the indices of the hist.
    copy : bool, optional
        Whether to copy the input data (True), or to use a reference instead. Default is False.

    Returns
    -------
    masked_hist : masked_array
       Hist with masked elements.
    masked_hist : masked_array, tuple
        Hist with masked elements and indices.
    '''
    # make np array
    hist_t = np.array(hist)
    # calculate cumulative distribution
    cdf = np.cumsum(hist_t)
    # copy, convert and normalize
    if cdf[-1] == 0:
        normcdf = cdf.astype('float')
    else:
        normcdf = cdf.astype('float') / cdf[-1]
    # calculate unique values from cumulative distribution and their indices
    unormcdf, indices = np.unique(normcdf, return_index=True)
    # calculate limits
    try:
        hp = np.where(unormcdf > prob[1])[0][0]
        lp = np.where(unormcdf >= prob[0])[0][0]
    except IndexError:
        hp_index = hist_t.shape[0]
        lp_index = 0
    else:
        hp_index = indices[hp]
        lp_index = indices[lp]
    # copy and create ma
    masked_hist = np.ma.array(hist, copy=copy, mask=True)
    masked_hist.mask[lp_index:hp_index + 1] = False
    if return_indices:
        return masked_hist, (lp_index, hp_index)
    else:
        return masked_hist

import logging
import numpy as np
import progressbar
from time import time
from collections import deque
from matplotlib.figure import Figure
from matplotlib.backends.backend_agg import FigureCanvasAgg as FigureCanvas
from scipy import stats
from pybar.daq.readout_utils import get_col_row_array_from_data_record_array, convert_data_array, is_data_record
from pybar.analysis.analyze_raw_data import AnalyzeRawData
from pybar.fei4.register_utils import make_box_pixel_mask_from_col_row, invert_pixel_mask
from pybar.fei4_run_base import Fei4RunBase
from pybar.run_manager import RunManager
from pybar.analysis.plotting.plotting import plot_occupancy, plot_fancy_occupancy, plot_three_way, hist_quantiles

                                logging.info('Taking data...')
                                self.progressbar = progressbar.ProgressBar(widgets=['', progressbar.Percentage(), ' ', progressbar.Bar(marker='*', left='|', right='|'), ' ', progressbar.Timer()], maxval=total_scan_time, poll=10, term_width=80).start()
                        else:
                            try:
                                self.progressbar.update(time() - start)
                            except ValueError:
                                pass
                # use Numpy for analysis and histogramming
                col_arr, row_arr = convert_data_array(array=self.read_data(), filter_func=is_data_record, converter_func=get_col_row_array_from_data_record_array)
                occ_hist, _, _ = np.histogram2d(col_arr, row_arr, bins=(80, 336), range=[[1, 80], [1, 336]])
                occ_mask = np.zeros(shape=occ_hist.shape, dtype=np.dtype('>u1'))

                # use FEI4 interpreter for analysis and histogramming
#                 from pybar.daq.readout_utils import data_array_from_data_iterable
#                 raw_data = np.ascontiguousarray(data_array_from_data_iterable(self.read_data()), dtype=np.uint32)
#                 interpreter.interpret_raw_data(raw_data)
#                 interpreter.store_event()  # force to create latest event
#                 histogram.add_hits(interpreter.get_hits())
#                 occ_hist = histogram.get_occupancy()[:, :, 0]
#                 # noisy pixels are set to 1
#                 occ_mask = np.zeros(shape=occ_hist.shape, dtype=np.dtype('>u1'))

                occ_mask[occ_hist > abs_occ_limit] = 1

                tdac_reg = self.register.get_pixel_register_value('TDAC')
                decrease_pixel_mask = np.logical_and(occ_mask > 0, tdac_reg > 0)
                disable_pixel_mask = np.logical_and(occ_mask > 0, tdac_reg == 0)
                enable_mask = self.register.get_pixel_register_value('Enable')
                new_enable_mask = np.logical_and(enable_mask, invert_pixel_mask(disable_pixel_mask))
                if np.logical_and(occ_mask > 0, enable_mask == 0).sum():
                    logging.warning('Received data from disabled pixels')
#                     disabled_pixels += disable_pixel_mask.sum()  # can lead to wrong values if the enable reg is corrupted
                disabled_pixels = invert_pixel_mask(new_enable_mask).sum() - preselected_pixels
                logging.info('Found %d noisy pixels', occ_mask.sum())
                logging.info('Increasing threshold of %d pixel(s)', decrease_pixel_mask.sum())
                logging.info('Disabling %d pixel(s), total number of disabled pixel(s): %d', disable_pixel_mask.sum(), disabled_pixels)

                # increasing threshold before writing TDACs to avoid FE becoming noisy
                self.register.set_global_register_value("Vthin_AltFine", self.gdac_range[0])
                commands = []
                commands.extend(self.register.get_commands("ConfMode"))
                commands.extend(self.register.get_commands("WrRegister", name=["Vthin_AltFine"]))
                self.register_utils.send_commands(commands)
                # writing TDAC
                new_tdac_reg = tdac_reg.copy()
                new_tdac_reg[decrease_pixel_mask] -= 1  # smaller TDAC translates to higher threshold
                self.register.set_pixel_register_value('TDAC', new_tdac_reg)
                self.register.set_pixel_register_value('Enable', new_enable_mask)
                commands = []
                commands.extend(self.register.get_commands("WrFrontEnd", same_mask_for_all_dc=False, name='TDAC'))
                commands.extend(self.register.get_commands("WrFrontEnd", same_mask_for_all_dc=False, name='Enable'))
                self.register_utils.send_commands(commands)
                # writing threshold value after writing TDACs
                self.register.set_global_register_value("Vthin_AltFine", reg_val)
                commands = []
                commands.extend(self.register.get_commands("WrRegister", name=["Vthin_AltFine"]))
                commands.extend(self.register.get_commands("RunMode"))
                self.register_utils.send_commands(commands)

                if (not do_refine_tuning and occ_hist.sum() <= total_occ_limit and occ_mask.sum() <= n_expected_pixel_hits) or (tdac_step >= max_tdac_steps):
                    logging.info('Stop tuning TDACs at Vthin_AltFine %d', reg_val)
                    self.threshold.appendleft(self.register.get_global_register_value("Vthin_AltFine"))
                    self.tdac_step.appendleft(tdac_step)
                    self.tdac.appendleft(tdac_reg)
                    self.new_tdac.appendleft(new_tdac_reg)
                    self.enable_mask.appendleft(enable_mask)
                    self.new_enable_mask.appendleft(new_enable_mask)
                    self.occupancy_hist.appendleft(occ_hist.copy())
                    self.occupancy_mask.appendleft(occ_mask.copy())
                    if not do_refine_tuning and np.mean(new_tdac_reg[new_enable_mask]) <= tdac_center + (1 if self.refine_tuning else 0):
                        reached_tdac_center = True
                    if not do_refine_tuning and disabled_pixels > disabled_pixels_limit_cnt:
                        reached_pixels_limit_cnt = True
                        logging.info('Limit of disabled pixels reached: %d (limit %d).' % (disabled_pixels, disabled_pixels_limit_cnt))
                    if not do_refine_tuning and reg_val <= self.gdac_range[1]:
                        reached_gdac_lower_limit = True
                    break
                else:
                    logging.info('Continue tuning TDACs at Vthin_AltFine %d', reg_val)
                # increase scan parameter counter
                tdac_step += 1

            if not self.refine_tuning and (reached_pixels_limit_cnt or reached_tdac_center or reached_gdac_lower_limit):
                pass  # will exit loop
            elif do_refine_tuning:
                logging.info("Finished TDAC refine tuning.")
            elif reached_pixels_limit_cnt or reached_tdac_center or reached_gdac_lower_limit:
                do_refine_tuning = True
                logging.info("Starting TDAC refine tuning...")
                coarse_threshold.append(reg_val - 1)
                abs_occ_limit = stats.poisson.ppf(0.99, mu=self.occupancy_limit * self.n_triggers * self.consecutive_lvl1)
                logging.info('The pixel threshold will be increased when occpancy >%d' % abs_occ_limit)
                max_tdac_steps = max(1, int(np.ceil((1 / self.occupancy_limit) / (self.n_triggers * self.consecutive_lvl1) * (1 / (1 - 0.99)))))
            else:
                coarse_threshold.append(reg_val - 1)

    def analyze(self):
        self.register.set_global_register_value("Vthin_AltFine", self.threshold[0])
        self.register.set_pixel_register_value('TDAC', self.new_tdac[0])
        self.register.set_pixel_register_value('Enable', self.new_enable_mask[0])  # use enable mask from the lowest point to mask bad pixels
        # write configuration to avoid high current states
        commands = []
        commands.extend(self.register.get_commands("ConfMode"))
        commands.extend(self.register.get_commands("WrRegister", name=["Vthin_AltFine"]))
        commands.extend(self.register.get_commands("WrFrontEnd", same_mask_for_all_dc=False, name="TDAC"))
        commands.extend(self.register.get_commands("WrFrontEnd", same_mask_for_all_dc=False, name="Enable"))
        self.register_utils.send_commands(commands)

        with AnalyzeRawData(raw_data_file=self.output_filename, create_pdf=True) as analyze_raw_data:
            analyze_raw_data.create_source_scan_hist = True
            analyze_raw_data.interpreter.set_warning_output(False)
            analyze_raw_data.interpret_word_table()
            analyze_raw_data.interpreter.print_summary()
            analyze_raw_data.plot_histograms()
            last_step = None
            for step in range(self.plot_n_steps, -1, -1):
                if self.threshold[step] is not None:
                    plot_occupancy(self.occupancy_hist[step].T, title='Occupancy at Vthin_AltFine %d Step %d' % (self.threshold[step], self.tdac_step[step]), filename=analyze_raw_data.output_pdf)
                    plot_fancy_occupancy(self.occupancy_hist[step].T, filename=analyze_raw_data.output_pdf)
                    plot_occupancy(self.occupancy_mask[step].T, title='Noisy pixels at Vthin_AltFine %d Step %d' % (self.threshold[step], self.tdac_step[step]), z_max=1, filename=analyze_raw_data.output_pdf)

                    plot_fancy_occupancy(self.occupancy_mask[step].T, filename=analyze_raw_data.output_pdf)
