class RunManager(object):
    def __init__(self, conf):
        '''Run Manager is taking care of initialization and execution of runs.

        Parameters
        ----------
        conf : str, dict, file
            Configuration for the run(s). Configuration will be passed to all scans.
        '''
        # fixing event handler: http://stackoverflow.com/questions/15457786/ctrl-c-crashes-python-after-importing-scipy-stats
        if os.name == 'nt':
            import thread

            def handler(signum, hook=thread.interrupt_main):
                hook()
                return True

            import win32api
            win32api.SetConsoleCtrlHandler(handler, 1)

        self._conf = None  # configuration dictionary
        self.current_run = None  # current run number
        self._conf_path = None  # absolute path of the configuation file
        self.init(conf)

    @property
    def conf(self):
        '''Configuration (namedtuple)
        '''
        conf = namedtuple('conf', field_names=self._conf.keys())
        return conf(**self._conf)  # prevent changing dict

    def init(self, conf):
        # current working directory
        if isinstance(conf, basestring) and os.path.isfile(os.path.abspath(conf)):
            conf = os.path.abspath(conf)
            self._conf_path = conf
        # search directory upwards form current working directory
        elif isinstance(conf, basestring) and find_file_dir_up(conf):
            conf = find_file_dir_up(conf)
            self._conf_path = conf
        elif isinstance(conf, file):
            self._conf_path = os.path.abspath(conf.name)
        else:
            self._conf_path = None
        self._conf = self.open_conf(conf)
        if 'working_dir' in self._conf and self._conf['working_dir']:
            # dirty fix for Windows pathes
            self._conf['working_dir'] = os.path.normpath(self._conf['working_dir'].replace('\\', '/'))
            if self._conf_path and not os.path.isabs(self._conf['working_dir']):
                # if working_dir is relative path, join path to configuration file and working_dir
                self._conf['working_dir'] = os.path.join(os.path.dirname(self._conf_path), self._conf['working_dir'])
            else:
                # working_dir is absolute path, keep that
                pass
        # use path of configuration file
        elif self._conf_path:
            self._conf['working_dir'] = os.path.dirname(self._conf_path)
        else:
            raise ValueError('Cannot deduce working directory from configuration')
        logging.info('Using working directory %s', self._conf['working_dir'])

    def close(self):
        if self.current_run is not None:
            self.current_run.close()

    def __enter__(self):
        return self

    def __exit__(self, *exc_info):
        self.close()

    @staticmethod
    def open_conf(conf):
        conf_dict = {}
        if not conf:
            pass
        elif isinstance(conf, basestring):  # parse the first YAML document in a stream
            if os.path.isfile(os.path.abspath(conf)):
                logging.info('Loading configuration from file %s', os.path.abspath(conf))
                with open(os.path.abspath(conf), 'r') as f:
                    conf_dict.update(safe_load(f))
            else:  # YAML string
                try:
                    conf_dict.update(safe_load(conf))
                except ValueError:  # invalid path/filename
                    raise IOError("File not found: %s" % os.path.abspath(conf))
                else:
                    logging.info('Loading configuration from file %s', os.path.abspath(conf.name))
        elif isinstance(conf, file):  # parse the first YAML document in a stream
            conf_dict.update(safe_load(conf))
        else:  # conf is already a dict
            conf_dict.update(conf)
        return conf_dict

    def cancel_current_run(self, msg=None):
        '''Control for runs.
        '''
        self.current_run.handle_cancel(msg=msg)

    def run_run(self, run, conf=None, run_conf=None, use_thread=False, catch_exception=True):
        '''Runs a run in another thread. Non-blocking.

        Parameters
        ----------
        run : class, object
            Run class or object.
        run_conf : str, dict, file
            Specific configuration for the run.
        use_thread : bool
            If True, run run in thread and returns blocking function.

        Returns
        -------
        If use_thread is True, returns function, which blocks until thread terminates, and which itself returns run status.
        If use_thread is False, returns run status.
        '''
        if isinstance(conf, basestring) and os.path.isfile(conf):
            logging.info('Updating configuration from file %s', os.path.abspath(conf))
        elif conf is not None:
            logging.info('Updating configuration')
        conf = self.open_conf(conf)
        self._conf.update(conf)

        if isclass(run):
            # instantiate the class
            run = run(conf=self._conf)

        local_run_conf = {}
        # general parameters from conf
        if 'run_conf' in self._conf:
            logging.info('Updating run configuration using run_conf key from configuration')
            local_run_conf.update(self._conf['run_conf'])
        # check for class name, scan specific parameters from conf
        if run.__class__.__name__ in self._conf:
            logging.info('Updating run configuration using %s key from configuration' % (run.__class__.__name__,))
            local_run_conf.update(self._conf[run.__class__.__name__])

        if isinstance(run_conf, basestring) and os.path.isfile(run_conf):
            logging.info('Updating run configuration from file %s', os.path.abspath(run_conf))
        elif run_conf is not None:
            logging.info('Updating run configuration')
        run_conf = self.open_conf(run_conf)
        # check for class name, scan specific parameters from conf
        if run.__class__.__name__ in run_conf:
            run_conf = run_conf[run.__class__.__name__]
        # run_conf parameter has highest priority, updated last
        local_run_conf.update(run_conf)

        if use_thread:
            self.current_run = run

            @thunkify(thread_name='RunThread', daemon=True, default_func=self.current_run.get_run_status)
            def run_run_in_thread():
                return run.run(run_conf=local_run_conf)

            signal.signal(signal.SIGINT, self._signal_handler)
            logging.info('Press Ctrl-C to stop run')

            return run_run_in_thread()
        else:
            self.current_run = run
            status = run.run(run_conf=local_run_conf)
            if not catch_exception and status != run_status.finished:
                raise RuntimeError('Exception occurred. Please read the log.')
            return status

    def run_primlist(self, primlist, skip_remaining=False):
        '''Runs runs from a primlist.

        Parameters
        ----------
        primlist : string
            Filename of primlist.
        skip_remaining : bool
            If True, skip remaining runs, if a run does not exit with status FINISHED.

        Note
        ----
        Primlist is a text file of the following format (comment line by adding '#'):
        <module name (containing class) or class (in either case use dot notation)>; <scan parameter>=<value>; <another scan parameter>=<another value>
        '''
        runlist = self.open_primlist(primlist)
        for index, run in enumerate(runlist):
            logging.info('Progressing with run %i out of %i...', index + 1, len(runlist))
            join = self.run_run(run, use_thread=True)
            status = join()
            if skip_remaining and not status == run_status.finished:
                logging.error('Exited run %i with status %s: Skipping all remaining runs.', run.run_number, status)
                break

    def open_primlist(self, primlist):
        def isrun(item, module):
            return isinstance(item, RunBase.__metaclass__) and item.__module__ == module  # only class from module, not from other imports

        if isinstance(primlist, basestring):
            with open(primlist, 'r') as f:
                f.seek(0)
                run_list = []
                for line in f.readlines():
                    line = line.partition('#')[0].strip()
                    if not line:
                        continue
                    parts = re.split('\s*[;]\s*', line)
                    try:
                        mod = import_module(parts[0])  # points to module
                    except ImportError:
                        mod = import_module(parts[0].rsplit('.', 1)[0])  # points to class
                        islocalrun = partial(isrun, module=parts[0].split('.')[-2])
                        clsmembers = getmembers(mod, islocalrun)
                        run_cls = None
                        for cls in clsmembers:
                            if cls[0] == parts[0].rsplit('.', 1)[1]:
                                run_cls = cls[1]
                                break
                        if not run_cls:
                            raise ValueError('Found no matching class: %s' % parts[0].rsplit('.', 1)[1])
                    else:
                        islocalrun = partial(isrun, module=parts[0])
                        clsmembers = getmembers(mod, islocalrun)
                        if len(clsmembers) > 1:
                            raise ValueError('Found more than one matching class.')
                        elif not len(clsmembers):
                            raise ValueError('Found no matching class.')
                        run_cls = clsmembers[0][1]
                    if run_cls.__class__.__name__ in self._conf:
                        run_conf = self._conf[run_cls.__class__.__name__]
                    else:
                        run_conf = {}
                    for param in parts[1:]:
                        key, value = re.split('\s*[=:]\s*', param, 1)
                        run_conf[key] = literal_eval(value)
                    run_list.append(run_cls(conf=self._conf, run_conf=run_conf))
            return run_list
        else:
            AttributeError('Primlist format not supported.')

    def _signal_handler(self, signum, frame):
        signal.signal(signal.SIGINT, signal.SIG_DFL)  # setting default handler... pressing Ctrl-C a second time will kill application
        self.cancel_current_run(msg='Pressed Ctrl-C') class IVScan(Fei4RunBase):
    _default_run_conf = {
        "broadcast_commands": False,
        "threaded_scan": False,
        "voltages": np.arange(-2, -101, -2).tolist(),  # voltage steps of the IV curve
        "max_leakage": 10e-6,  # scan aborts if current is higher
        "max_voltage": -20,  # for safety, scan aborts if voltage is higher
        "minimum_delay": 0.5,  # minimum delay between current measurements in seconds
        "bias_voltage": -10  # if defined ramp bias to bias voltage after scan is finished, has to be less than last scanned voltage
    }

    def configure(self):
        pass

    def scan(self):
        logging.info('Measure IV for V = %s' % self.voltages)
        description = [('voltage', np.float), ('current', np.float)]
        data = self.raw_data_file.h5_file.create_table(self.raw_data_file.h5_file.root, name='IV_data', description=np.zeros((1, ), dtype=description).dtype, title='Data from the IV scan')

        progress_bar = progressbar.ProgressBar(widgets=['', progressbar.Percentage(), ' ', progressbar.Bar(marker='*', left='|', right='|'), ' ', progressbar.AdaptiveETA()], maxval=len(self.voltages), term_width=80)
        progress_bar.start()
        actual_voltage = None
        try:
            for index, voltage in enumerate(self.voltages):
                if self.stop_run.is_set():
                    break
                if voltage > 0:
                    RuntimeError('Voltage has to be negative! Abort to protect device.')
                if self.abort_run.is_set():
                    break
                if abs(voltage) <= abs(self.max_voltage):
                    self.dut['Sourcemeter'].set_voltage(voltage)
                    actual_voltage = voltage
                    time.sleep(self.minimum_delay)
                else:
                    logging.info('Maximum voltage with %f V reached, abort', voltage)
                    break
                current = float(self.dut['Sourcemeter'].get_current().split(',')[1])
                if abs(current) > abs(self.max_leakage):
                    logging.info('Maximum current with %e I reached, abort', current)
                    break
                logging.info('V = %f, I = %e', voltage, current)
                max_repeat = 50
                for i in range(max_repeat):  # repeat current measurement until stable (current does not increase)
                    time.sleep(self.minimum_delay)
                    actual_current = float(self.dut['Sourcemeter'].get_current().split(',')[1])
                    if abs(actual_current) > abs(self.max_leakage):
                        logging.info('Maximum current with %e I reached, abort', actual_current)
                        break
                    if (abs(actual_current) < abs(current)):  # stable criterion
                        break
                    current = actual_current
                if i == max_repeat - 1:  # true if the leakage always increased
                    raise RuntimeError('Leakage current is not stable')
                else:
                    a = np.array([(voltage, current)], dtype=description)
                    data.append(a)
                progress_bar.update(index)
            progress_bar.finish()
            data.flush()
        finally:
            # ramp down
            if self.bias_voltage and self.bias_voltage <= 0 and actual_voltage is not None:
                logging.info('Set bias voltage from %f V to %f V', actual_voltage, self.bias_voltage)
                progress_bar = progressbar.ProgressBar(widgets=['', progressbar.Percentage(), ' ', progressbar.Bar(marker='*', left='|', right='|'), ' ', progressbar.AdaptiveETA()], maxval=len(range(actual_voltage, self.bias_voltage + 1, 2)), term_width=80)
                progress_bar.start()
                for index, voltage in enumerate(range(actual_voltage, self.bias_voltage + 1, 2)):  # ramp until bias
                    time.sleep(self.minimum_delay)
                    self.dut['Sourcemeter'].set_voltage(voltage)
                    progress_bar.update(index)
                progress_bar.finish()

    def analyze(self):
        logging.info('Analyze and plot results')
        with tb.open_file(self.output_filename + '.h5', 'r+') as in_file_h5:
            data = in_file_h5.root.IV_data[:]
            # Plot and fit result
            x, y = data['voltage'], data['current'] * 1e6
            fig = Figure()
            FigureCanvas(fig)
            ax = fig.add_subplot(111)
            ax.plot(x, y, '.-', label='data')
            ax.set_title('IV curve')
            ax.set_ylabel('Current [uA]')
            ax.set_xlabel('Voltage [V]')
            ax.grid(True)
            ax.legend(loc=0)
            fig.savefig(self.output_filename + '.pdf') class RegisterTest(Fei4RunBase):

    '''Testing of FEI4(A/B) global and pixel register and reading of chip S/N.

    Note
    ----
    Number of register errors is some arbitrary number.

    FEI4A specific features:
    Register at address 41 will always fail (EOCHLskipped).
    FEI4A has timing issues when reading pixel registers. The data from pixel registers is corrupted. It is a known bug of the FEI4A. Lowering the digital voltage (VDDD) to 1.0V may improve the result.

    FEI4B specific features:
    Register at address 40 will always fail (ADC output value).
    '''
    _default_run_conf = {
        "broadcast_commands": False,
        "threaded_scan": False,
        "configure_fe": True,
        "reset_fe": True,
        "read_sn": True,
        "test_global": True,
        "test_pixel": True,
        "fail_on_error": False
    }

    def configure(self):
        pass

    def scan(self):
        if self.read_sn:
            read_chip_sn(self)

        if self.test_global:
            global_register_errors = test_global_register(self)
            if global_register_errors and self.fail_on_error:
                raise Exception('Global register test finished with %d errors' % global_register_errors)

        if self.test_pixel:
            pixel_register_errors = self.test_pixel_register()
            if pixel_register_errors and self.fail_on_error:
                raise Exception('Pixel register test finished with %d errors' % pixel_register_errors)

    def analyze(self):
        pass

    def test_pixel_register(self, pix_regs=["EnableDigInj", "Imon", "Enable", "C_High", "C_Low", "TDAC", "FDAC"], dcs=range(40)):
        '''Test Pixel Register
        '''
        logging.info('Running Pixel Register Test for %s', str(pix_regs))
        self.register_utils.configure_pixel()
        commands = []
        commands.extend(self.register.get_commands("ConfMode"))
        self.register_utils.send_commands(commands)
        self.fifo_readout.reset_fifo()

        pixel_register_errors = 0

        plots = PdfPages(self.output_filename + ".pdf")

        for i, result in enumerate(read_pixel_register(self, pix_regs=pix_regs, dcs=dcs)):
            result_array = np.ones_like(result)
            result_array.data[result == self.register.get_pixel_register_value(pix_regs[i])] = 0
            pixel_register_errors += np.count_nonzero(result_array == 1)
            logging.info("Pixel register %s: %d pixel error", pix_regs[i], np.count_nonzero(result_array == 1))
            plotting.plot_three_way(result_array.T, title=str(pix_regs[i]) + " register test with " + str(np.count_nonzero(result_array == 1)) + '/' + str(26880 - np.ma.count_masked(result_array)) + " pixel failing", x_axis_title="0:OK, 1:FAIL", maximum=1, filename=plots)

        plots.close()
        return pixel_register_errors
 class DigitalScan(Fei4RunBase):
    '''Digital scan
    '''
    _default_run_conf = {
        "broadcast_commands": True,
        "threaded_scan": True,
        "mask_steps": 3,  # mask steps
        "n_injections": 100,  # number of injections
        "use_enable_mask": False  # if True, use Enable mask during scan, if False, all pixels will be enabled
    }

    def configure(self):
        commands = []
        commands.extend(self.register.get_commands("ConfMode"))
        self.register.set_global_register_value("PrmpVbp", 0)
        self.register.set_global_register_value("Amp2Vbp", 0)
        self.register.set_global_register_value("DisVbn", 0)
        commands.extend(self.register.get_commands("WrRegister", name=["PrmpVbp", "Amp2Vbp", "DisVbn"]))
        self.register.set_pixel_register_value("C_High", 0)
        commands.extend(self.register.get_commands("WrFrontEnd", same_mask_for_all_dc=True, name="C_High"))
        self.register.set_pixel_register_value("C_Low", 0)
        commands.extend(self.register.get_commands("WrFrontEnd", same_mask_for_all_dc=True, name="C_Low"))
        self.register_utils.send_commands(commands)

    def scan(self):
        with self.readout():
            cal_lvl1_command = self.register.get_commands("CAL")[0] + self.register.get_commands("zeros", length=40)[0] + self.register.get_commands("LV1")[0]
            scan_loop(self, cal_lvl1_command, repeat_command=self.n_injections, use_delay=True, mask_steps=self.mask_steps, enable_mask_steps=None, enable_double_columns=None, same_mask_for_all_dc=True, eol_function=None, digital_injection=True, enable_shift_masks=["Enable", "EnableDigInj"], restore_shift_masks=False, mask=invert_pixel_mask(self.register.get_pixel_register_value('Enable')) if self.use_enable_mask else None)

    def analyze(self):
        with AnalyzeRawData(raw_data_file=self.output_filename, create_pdf=True) as analyze_raw_data:
            analyze_raw_data.interpreter.set_warning_output(True)
            analyze_raw_data.create_tot_hist = False
            analyze_raw_data.interpret_word_table()
            analyze_raw_data.plot_histograms()
            analyze_raw_data.interpreter.print_summary()
 class AnalogScan(Fei4RunBase):
    '''Analog scan
    '''
    _default_run_conf = {
        "broadcast_commands": False,  # keep this False, if this scan is used to verify the tuning
        "threaded_scan": True,
        "mask_steps": 3,  # mask steps, be carefull PlsrDAC injects different charge for different mask steps
        "n_injections": 100,  # number of injections
        "scan_parameters": [('PlsrDAC', 280)],  # the PlsrDAC setting
        "use_enable_mask": False,  # if True, use Enable mask during scan, if False, all pixels will be enabled
        "enable_shift_masks": ["Enable", "C_High", "C_Low"],  # enable masks shifted during scan
        "disable_shift_masks": [],  # disable masks shifted during scan
        "pulser_dac_correction": False,  # PlsrDAC correction for each double column
        "enable_tdc": False,  # if True, enables TDC
        "same_mask_for_all_dc": True,  # if True, all columns have the same mask, if False, mask will be enabled only where injected
        "enable_double_columns": None,  # List of double columns which will be enabled during scan. None will select all double columns
        "enable_mask_steps": None,  # List of mask steps which will be applied. None will select all mask steps.
    }

    def configure(self):
        commands = []
        commands.extend(self.register.get_commands("ConfMode"))
        scan_parameter_value = self.scan_parameters.PlsrDAC[0] if isinstance(self.scan_parameters.PlsrDAC, Iterable) else self.scan_parameters.PlsrDAC
        self.register.set_global_register_value('PlsrDAC', scan_parameter_value)
        commands.extend(self.register.get_commands("WrRegister", name=['PlsrDAC']))
        # C_Low
        if "C_Low".lower() in map(lambda x: x.lower(), self.enable_shift_masks):
            self.register.set_pixel_register_value('C_Low', 1)
            commands.extend(self.register.get_commands("WrFrontEnd", same_mask_for_all_dc=True, name='C_Low'))
        else:
            self.register.set_pixel_register_value('C_Low', 0)
            commands.extend(self.register.get_commands("WrFrontEnd", same_mask_for_all_dc=True, name='C_Low'))
        # C_High
        if "C_High".lower() in map(lambda x: x.lower(), self.enable_shift_masks):
            self.register.set_pixel_register_value('C_High', 1)
            commands.extend(self.register.get_commands("WrFrontEnd", same_mask_for_all_dc=True, name='C_High'))
        else:
            self.register.set_pixel_register_value('C_High', 0)
            commands.extend(self.register.get_commands("WrFrontEnd", same_mask_for_all_dc=True, name='C_High'))
        self.register_utils.send_commands(commands)

    def scan(self):
        with self.readout():
            cal_lvl1_command = self.register.get_commands("CAL")[0] + self.register.get_commands("zeros", length=40)[0] + self.register.get_commands("LV1")[0]

            if self.enable_tdc:
                # activate TDC arming
                self.dut['TDC']['EN_ARMING'] = True
                scan_loop(self, cal_lvl1_command, repeat_command=self.n_injections, use_delay=True, mask_steps=self.mask_steps, enable_mask_steps=self.enable_mask_steps, enable_double_columns=self.enable_double_columns, same_mask_for_all_dc=self.same_mask_for_all_dc, bol_function=self.activate_tdc, eol_function=self.deactivate_tdc, digital_injection=False, enable_shift_masks=self.enable_shift_masks, disable_shift_masks=self.disable_shift_masks, restore_shift_masks=False, mask=invert_pixel_mask(self.register.get_pixel_register_value('Enable')) if self.use_enable_mask else None, double_column_correction=self.pulser_dac_correction)
            else:
                scan_loop(self, cal_lvl1_command, repeat_command=self.n_injections, use_delay=True, mask_steps=self.mask_steps, enable_mask_steps=self.enable_mask_steps, enable_double_columns=self.enable_double_columns, same_mask_for_all_dc=self.same_mask_for_all_dc, digital_injection=False, enable_shift_masks=self.enable_shift_masks, disable_shift_masks=self.disable_shift_masks, restore_shift_masks=False, mask=invert_pixel_mask(self.register.get_pixel_register_value('Enable')) if self.use_enable_mask else None, double_column_correction=self.pulser_dac_correction)

    def analyze(self):
        with AnalyzeRawData(raw_data_file=self.output_filename, create_pdf=True) as analyze_raw_data:
            analyze_raw_data.create_tot_hist = True
            if self.enable_tdc:
                analyze_raw_data.create_tdc_counter_hist = True  # histogram all TDC words
                analyze_raw_data.create_tdc_hist = True  # histogram the hit TDC information
            analyze_raw_data.interpret_word_table()
            analyze_raw_data.plot_histograms()
            analyze_raw_data.interpreter.print_summary()

    def activate_tdc(self):
        self.dut['TDC']['ENABLE'] = True

    def deactivate_tdc(self):
        self.dut['TDC']['ENABLE'] = False
 class Fei4Tuning(GdacTuning, TdacTuning, FeedbackTuning, FdacTuning):
    '''Fully automatic FEI4 Tuning

    This is a meta script implementing GDAC, TDAC, Feedback and FDAC tuning in a single tuning script.
    Values are given in units of PlsrDAC.

    Note:
    C_Low: nominally 1.9fF / measured* 2fF
    C_High: nominally 3.8fF / measured* 4.1fF
    c_Low + C_High: nominally 5.7fF / measured* 6.1fF

    PlsrDAC: ~1.5mV/DAC

    C_Low: 18.7e / 19.7e
    C_High: 35.6e / 38.4e
    C_Low + C_High: 53e / 57e

    *) measurements from IBL wafer probing
    '''
    _default_run_conf = {
        "broadcast_commands": False,  # do not change, keep False, otherwise unexpected bahavior
        "threaded_scan": True,  # if True, parallelize tuning
        # tuning parameters
        "target_threshold": 30,  # target threshold
        "target_charge": 280,  # target charge
        "target_tot": 5,  # target ToT
        "global_iterations": 4,  # the number of iterations to do for the global tuning, 0: only global threshold (GDAC) is tuned, -1: no global tuning
        "local_iterations": 3,  # the number of iterations to do for the local tuning, 0: only local threshold (TDAC) is tuned, -1: no local tuning
        "reset_local_dacs": True,  # if True, reset pixels registers to the middle of the DAC range before the global tuning starts
        "fail_on_warning": True,  # do not continue tuning if a global tuning fails
        # GDAC
        "gdac_tune_bits": range(7, -1, -1),  # GDAC bits to change during tuning
        "start_gdac": 150,  # start value of standard GDAC tuning, not used for fast GDAC tuning
        "step_size": -1,  # step size of the GDAC during scan, not used for fast GDAC tuning
        "gdac_lower_limit": 30,  # set GDAC lower limit to prevent FEI4 from becoming noisy, set to 0 or None to disable
        "n_injections_gdac": 50,  # number of injections per GDAC bit setting
        "max_delta_threshold": 20,  # minimum difference to the target_threshold to abort the tuning, in percent of n_injections_gdac
        "enable_mask_steps_gdac": [0],  # mask steps to do per GDAC setting, 1 step is sufficient and saves time
        # Feedback
        "feedback_tune_bits": range(7, -1, -1),
        "n_injections_feedback": 50,
        "max_delta_tot": 0.1,
        "enable_mask_steps_feedback": [0],  # mask steps to do per PrmpVbpf setting, 1 step is sufficient and saves time
        # TDAC
        "tdac_tune_bits": range(4, -1, -1),
        "n_injections_tdac": 100,
        # FDAC
        "fdac_tune_bits": range(3, -1, -1),
        "n_injections_fdac": 30,
        # general
        "enable_shift_masks": ["Enable", "C_High", "C_Low"],  # enable masks shifted during scan
        "disable_shift_masks": [],  # disable masks shifted during scan
        "pulser_dac_correction": False,  # PlsrDAC correction for each double column
        "scan_parameters": [('GDAC', -1), ('TDAC', -1), ('PrmpVbpf', -1), ('FDAC', -1), ('global_step', 0), ('local_step', 0)],
        # other
        "mask_steps": 3,  # mask steps, be carefull PlsrDAC injects different charge for different mask steps
        "same_mask_for_all_dc": True  # Increases scan speed, should be deactivated for very noisy FE
    }

    def configure(self):
        super(Fei4Tuning, self).configure()

        # overwrite pixel registers and set them to center postion before a global tuning
        if self.reset_local_dacs and self.global_iterations:
            commands = []
            commands.extend(self.register.get_commands("ConfMode"))
            # TDAC
            tdac_center = 2 ** self.register.pixel_registers['TDAC']['bitlength'] / 2
            self.register.set_pixel_register_value('TDAC', tdac_center)
            commands.extend(self.register.get_commands("WrFrontEnd", same_mask_for_all_dc=True, name='TDAC'))
            # FDAC
            fdac_center = 2 ** self.register.pixel_registers['FDAC']['bitlength'] / 2
            self.register.set_pixel_register_value('FDAC', fdac_center)
            commands.extend(self.register.get_commands("WrFrontEnd", same_mask_for_all_dc=True, name='FDAC'))
            commands.extend(self.register.get_commands("RunMode"))
            self.register_utils.send_commands(commands)

        self.close_plots = False

    def scan(self):
        '''Metascript that calls other scripts to tune the FE.

        Parameters
        ----------
        cfg_name : string
            Name of the config to be created. This config holds the tuning results.
        target_threshold : int
            The target threshold value in PlsrDAC.
        target_charge : int
            The target charge in PlsrDAC value to tune to.
        target_tot : float
            The target tot value to tune to.
        global_iterations : int
            Defines how often global threshold (GDAC) / global feedback (PrmpVbpf) current tuning is repeated.
            -1 or None: Global tuning is disabled
            0: Only global threshold tuning
            1: GDAC -> PrmpVbpf -> GDAC
            2: GDAC -> PrmpVbpf -> GDAC -> PrmpVbpf -> GDAC
            ...
        local_iterations : int
            Defines how often local threshold (TDAC) / feedback current (FDAC) tuning is repeated.
            -1 or None: Local tuning is disabled
            0: Only local threshold tuning
            1: TDAC -> FDAC -> TDAC
            2: TDAC -> FDAC -> TDAC -> FDAC -> TDAC
            ...
        '''
        for iteration in range(0, self.global_iterations):  # tune iteratively with decreasing range to save time
            if self.stop_run.is_set():
                break
            logging.info("Global tuning step %d / %d", iteration + 1, self.global_iterations)
            self.set_scan_parameters(global_step=self.scan_parameters.global_step + 1)
            GdacTuning.scan(self)
            commands = []
            commands.extend(self.register.get_commands("ConfMode"))
            commands.extend(self.register.get_commands("WrRegister", name=["Vthin_AltCoarse", "Vthin_AltFine"]))
            commands.extend(self.register.get_commands("RunMode"))
            self.register_utils.send_commands(commands)
            if self.stop_run.is_set():
                break
            self.set_scan_parameters(global_step=self.scan_parameters.global_step + 1)
            FeedbackTuning.scan(self)
            commands = []
            commands.extend(self.register.get_commands("ConfMode"))
            commands.extend(self.register.get_commands("WrRegister", name=["PrmpVbpf"]))
            commands.extend(self.register.get_commands("RunMode"))
            self.register_utils.send_commands(commands)

        if self.global_iterations >= 0 and not self.stop_run.is_set():
            self.set_scan_parameters(global_step=self.scan_parameters.global_step + 1)
            GdacTuning.scan(self)
            commands = []
            commands.extend(self.register.get_commands("ConfMode"))
            commands.extend(self.register.get_commands("WrRegister", name=["Vthin_AltCoarse", "Vthin_AltFine"]))
            commands.extend(self.register.get_commands("RunMode"))
            self.register_utils.send_commands(commands)

            Vthin_AC = self.register.get_global_register_value("Vthin_AltCoarse")
            Vthin_AF = self.register.get_global_register_value("Vthin_AltFine")
            PrmpVbpf = self.register.get_global_register_value("PrmpVbpf")
            logging.info("Results of global threshold tuning: Vthin_AltCoarse / Vthin_AltFine = %d / %d", Vthin_AC, Vthin_AF)
            logging.info("Results of global feedback tuning: PrmpVbpf = %d", PrmpVbpf)

        for iteration in range(0, self.local_iterations):
            if self.stop_run.is_set():
                break
            logging.info("Local tuning step %d / %d", iteration + 1, self.local_iterations)
            self.set_scan_parameters(local_step=self.scan_parameters.local_step + 1)
            TdacTuning.scan(self)
            commands = []
            commands.extend(self.register.get_commands("ConfMode"))
            commands.extend(self.register.get_commands("WrFrontEnd", same_mask_for_all_dc=False, name="TDAC"))
            commands.extend(self.register.get_commands("RunMode"))
            self.register_utils.send_commands(commands)
            if self.stop_run.is_set():
                break
            self.set_scan_parameters(local_step=self.scan_parameters.local_step + 1)
            FdacTuning.scan(self)
            commands = []
            commands.extend(self.register.get_commands("ConfMode"))
            commands.extend(self.register.get_commands("WrFrontEnd", same_mask_for_all_dc=False, name="FDAC"))
            commands.extend(self.register.get_commands("RunMode"))
            self.register_utils.send_commands(commands)

        if self.local_iterations >= 0 and not self.stop_run.is_set():
            self.set_scan_parameters(local_step=self.scan_parameters.local_step + 1)
            TdacTuning.scan(self)
            commands = []
            commands.extend(self.register.get_commands("ConfMode"))
            commands.extend(self.register.get_commands("WrFrontEnd", same_mask_for_all_dc=False, name="TDAC"))
            commands.extend(self.register.get_commands("RunMode"))
            self.register_utils.send_commands(commands)

    def analyze(self):
        if self.global_iterations > 0:
            FeedbackTuning.analyze(self)
        if self.global_iterations >= 0:
            GdacTuning.analyze(self)

        if self.local_iterations > 0:
            FdacTuning.analyze(self)
        if self.local_iterations >= 0:
            TdacTuning.analyze(self)

        # write configuration to avoid high current states
        commands = []
        commands.extend(self.register.get_commands("ConfMode"))
        commands.extend(self.register.get_commands("WrRegister", name=["Vthin_AltCoarse", "Vthin_AltFine", "PrmpVbpf"]))
        commands.extend(self.register.get_commands("WrFrontEnd", same_mask_for_all_dc=False, name="TDAC"))
        commands.extend(self.register.get_commands("WrFrontEnd", same_mask_for_all_dc=False, name="FDAC"))
        self.register_utils.send_commands(commands)

        if self.local_iterations > 0:
            plot_three_way(hist=self.tot_mean_best.transpose(), title="Mean ToT after last FDAC tuning", x_axis_title='Mean ToT', filename=self.plots_filename, maximum=15)
            plot_three_way(hist=self.register.get_pixel_register_value("FDAC").transpose(), title="FDAC distribution after last FDAC tuning", x_axis_title='FDAC', filename=self.plots_filename, maximum=15)
        if self.local_iterations >= 0:
            plot_three_way(hist=self.occupancy_best.transpose(), title="Occupancy after last TDAC tuning", x_axis_title='Occupancy', filename=self.plots_filename, maximum=self.n_injections_tdac)
            plot_three_way(hist=self.register.get_pixel_register_value("TDAC").transpose(), title="TDAC distribution after last TDAC tuning", x_axis_title='TDAC', filename=self.plots_filename, maximum=31)

        self.plots_filename.close()
 class StuckPixelTuning(DigitalScan):
    '''Stuck pixel scan to detect and disable stuck pixels (Hitbus/HitOR always high).
    '''
    _default_run_conf = DigitalScan._default_run_conf.copy()
    _default_run_conf = {
        "broadcast_commands": True,
        "threaded_scan": True,
        "mask_steps": 3,  # mask steps
        "n_injections": 100,  # number of injections
        "use_enable_mask": False,  # if True, use Enable mask during scan, if False, all pixels will be enabled
        "disable_for_mask": ['Enable'],  # list of masks for which noisy pixels will be disabled
        "enable_for_mask": ['Imon'],  # list of masks for which noisy pixels will be enabled
        "overwrite_mask": False  # if True, overwrite existing masks
    }

    def configure(self):
        commands = []
        commands.extend(self.register.get_commands("ConfMode"))
        pixel_reg = "C_High"
        self.register.set_pixel_register_value(pixel_reg, 0)
        commands.extend(self.register.get_commands("WrFrontEnd", same_mask_for_all_dc=True, name=pixel_reg))
        pixel_reg = "C_Low"
        self.register.set_pixel_register_value(pixel_reg, 0)
        commands.extend(self.register.get_commands("WrFrontEnd", same_mask_for_all_dc=True, name=pixel_reg))
        self.register_utils.send_commands(commands)

    def analyze(self):
        with AnalyzeRawData(raw_data_file=self.output_filename, create_pdf=True) as analyze_raw_data:
            analyze_raw_data.create_source_scan_hist = True
            analyze_raw_data.interpreter.set_warning_output(False)
            analyze_raw_data.create_tot_hist = False
            analyze_raw_data.interpret_word_table()
            analyze_raw_data.plot_histograms()
            analyze_raw_data.interpreter.print_summary()

            occ_hist = analyze_raw_data.out_file_h5.root.HistOcc[:, :, 0].T
            occ_mask = np.zeros(shape=occ_hist.shape, dtype=np.dtype('>u1'))
            # noisy pixels are set to 1
            occ_mask[occ_hist < self.n_injections] = 1
            # make inverse
            inv_occ_mask = invert_pixel_mask(occ_mask)
            if self.overwrite_mask:
                for mask in self.disable_for_mask:
                    self.register.set_pixel_register_value(mask, inv_occ_mask)
            else:
                for mask in self.disable_for_mask:
                    enable_mask = np.logical_and(inv_occ_mask, self.register.get_pixel_register_value(mask))
                    self.register.set_pixel_register_value(mask, enable_mask)

            if self.overwrite_mask:
                for mask in self.enable_for_mask:
                    self.register.set_pixel_register_value(mask, occ_mask)
            else:
                for mask in self.enable_for_mask:
                    disable_mask = np.logical_or(occ_mask, self.register.get_pixel_register_value(mask))
                    self.register.set_pixel_register_value(mask, disable_mask)

            plot_occupancy(occ_mask.T, title='Stuck Pixels', z_max=1, filename=analyze_raw_data.output_pdf)
            for mask in self.disable_for_mask:
                mask_name = self.register.pixel_registers[mask]['name']
                plot_occupancy(self.register.get_pixel_register_value(mask).T, title='%s Mask' % mask_name, z_max=1, filename=analyze_raw_data.output_pdf)
            for mask in self.enable_for_mask:
                mask_name = self.register.pixel_registers[mask]['name']
                plot_occupancy(self.register.get_pixel_register_value(mask).T, title='%s Mask' % mask_name, z_max=1, filename=analyze_raw_data.output_pdf)
 class FastThresholdScan(Fei4RunBase):
    '''Fast threshold scan

    Implementation of a fast threshold scan checking for start and end of s-curve.
    '''
    _default_run_conf = {
        "broadcast_commands": True,
        "threaded_scan": True,
        "n_injections": 100,  # number of injections per PlsrDAC step
        "scan_parameters": [('PlsrDAC', (None, 100))],  # the PlsrDAC range
        "mask_steps": 3,  # mask steps, be carefull PlsrDAC injects different charge for different mask steps
        "enable_mask_steps": None,  # list of mask steps to be used, if None use all mask steps
        "step_size": 2,  # step size of the PlsrDAC during scan
        "search_distance": 10,  # step size of the PlsrDAC for testing start condition
        "minimum_data_points": 20,  # PlsrDAC before testing stop condition
        "ignore_columns": (1, 78, 79, 80),  # columns, which will be ignored during scan
        "use_enable_mask": False,  # if True, use Enable mask during scan, if False, all pixels will be enabled
        "enable_shift_masks": ["Enable", "C_High", "C_Low"],  # enable masks shifted during scan
        "disable_shift_masks": [],  # disable masks shifted during scan
        "pulser_dac_correction": False  # PlsrDAC correction for each double column
    }
    scan_parameter_start = 0  # holding last start value (e.g. used in GDAC threshold scan)

    def configure(self):
        commands = []
        commands.extend(self.register.get_commands("ConfMode"))
        # C_Low
        if "C_Low".lower() in map(lambda x: x.lower(), self.enable_shift_masks):
            self.register.set_pixel_register_value('C_Low', 1)
            commands.extend(self.register.get_commands("WrFrontEnd", same_mask_for_all_dc=True, name='C_Low'))
        else:
            self.register.set_pixel_register_value('C_Low', 0)
            commands.extend(self.register.get_commands("WrFrontEnd", same_mask_for_all_dc=True, name='C_Low'))
        # C_High
        if "C_High".lower() in map(lambda x: x.lower(), self.enable_shift_masks):
            self.register.set_pixel_register_value('C_High', 1)
            commands.extend(self.register.get_commands("WrFrontEnd", same_mask_for_all_dc=True, name='C_High'))
        else:
            self.register.set_pixel_register_value('C_High', 0)
            commands.extend(self.register.get_commands("WrFrontEnd", same_mask_for_all_dc=True, name='C_High'))
        commands.extend(self.register.get_commands("RunMode"))
        self.register_utils.send_commands(commands)

    def scan(self):
        self.start_condition_triggered = False  # set to true if the start condition is true once
        self.stop_condition_triggered = False  # set to true if the stop condition is true once

        self.start_at = 0.01  # if more than start_at*activated_pixel see at least one hit the precise scanning is started
        self.stop_at = 0.95  # if more than stop_at*activated_pixel see the maximum numbers of injection, the scan is stopped

        self.record_data = False  # set to true to activate data storage, so far not everything is recorded to ease data analysis

        scan_parameter_range = [0, (2 ** self.register.global_registers['PlsrDAC']['bitlength'] - 1)]
        if self.scan_parameters.PlsrDAC[0]:
            scan_parameter_range[0] = self.scan_parameters.PlsrDAC[0]
        if self.scan_parameters.PlsrDAC[1]:
            scan_parameter_range[1] = self.scan_parameters.PlsrDAC[1]
        logging.info("Scanning %s from %d to %d", 'PlsrDAC', scan_parameter_range[0], scan_parameter_range[1])
        self.scan_parameter_value = scan_parameter_range[0]  # set to start value
        self.search_distance = self.search_distance
        self.data_points = 0  # counter variable to count the data points already recorded, have to be at least minimum_data_ponts

        # calculate DCs to scan from the columns to ignore
        enable_double_columns = range(0, 40)
        if 1 in self.ignore_columns:
            enable_double_columns.remove(0)
        if set((78, 79, 80)).issubset(self.ignore_columns):
            enable_double_columns.remove(39)
        for double_column in range(1, 39):
            if set((double_column * 2, (double_column * 2) + 1)).issubset(self.ignore_columns):
                enable_double_columns.remove(double_column)
        logging.info("Use DCs: %s", str(enable_double_columns))

        self.select_arr_columns = range(0, 80)
        for column in self.ignore_columns:
            self.select_arr_columns.remove(column - 1)

        while self.scan_parameter_value <= scan_parameter_range[1]:  # scan as long as scan parameter is smaller than defined maximum
            if self.stop_run.is_set():
                break

            commands = []
            commands.extend(self.register.get_commands("ConfMode"))
            self.register.set_global_register_value('PlsrDAC', self.scan_parameter_value)
            commands.extend(self.register.get_commands("WrRegister", name=['PlsrDAC']))
            self.register_utils.send_commands(commands)

            with self.readout(PlsrDAC=self.scan_parameter_value, fill_buffer=True, callback=self.handle_data if self.record_data else None):
                cal_lvl1_command = self.register.get_commands("CAL")[0] + self.register.get_commands("zeros", length=40)[0] + self.register.get_commands("LV1")[0]
                scan_loop(self, cal_lvl1_command, repeat_command=self.n_injections, use_delay=True, mask_steps=self.mask_steps, enable_mask_steps=self.enable_mask_steps, enable_double_columns=enable_double_columns, same_mask_for_all_dc=True, eol_function=None, digital_injection=False, enable_shift_masks=self.enable_shift_masks, disable_shift_masks=self.disable_shift_masks, restore_shift_masks=False, mask=invert_pixel_mask(self.register.get_pixel_register_value('Enable')) if self.use_enable_mask else None, double_column_correction=self.pulser_dac_correction)

            if not self.start_condition_triggered or self.data_points > self.minimum_data_points:  # speed up, only create histograms when needed. Python is much too slow here.
                if not self.start_condition_triggered and not self.record_data:
                    logging.info('Testing for start condition: %s %d', 'PlsrDAC', self.scan_parameter_value)
                if not self.stop_condition_triggered and self.record_data:
                    logging.info('Testing for stop condition: %s %d', 'PlsrDAC', self.scan_parameter_value)

                col, row = convert_data_array(array=self.read_data(), filter_func=is_data_record, converter_func=get_col_row_array_from_data_record_array)
                if np.any(np.logical_and(col < 1, col > 80)) or np.any(np.logical_and(row < 1, row > 336)):  # filter bad data records that can happen
                    logging.warning('There are undefined %d data records (e.g. random data)', np.count_nonzero(np.logical_and(col < 1, col > 80)) + np.count_nonzero(np.logical_and(row < 1, row > 336)))
                    col, row = col[np.logical_and(col > 0, col <= 80)], row[np.logical_and(row > 0, row <= 336)]
                occupancy_array = hist_2d_index(col - 1, row - 1, shape=(80, 336))
                self.scan_condition(occupancy_array)

            # start condition is met for the first time
            if self.start_condition_triggered and not self.record_data:
                self.scan_parameter_value = self.scan_parameter_value - self.search_distance + self.step_size
                if self.scan_parameter_value < 0:
                    self.scan_parameter_value = 0
                logging.info('Starting threshold scan at %s %d', 'PlsrDAC', self.scan_parameter_value)
                self.scan_parameter_start = self.scan_parameter_value
                self.record_data = True
                continue

            # saving data
            if self.record_data:
                self.data_points = self.data_points + 1

            # stop condition is met for the first time
            if self.stop_condition_triggered and self.record_data:
                logging.info('Stopping threshold scan at %s %d', 'PlsrDAC', self.scan_parameter_value)
                break

            # increase scan parameter value
            if not self.start_condition_triggered:
                self.scan_parameter_value = self.scan_parameter_value + self.search_distance
            else:
                self.scan_parameter_value = self.scan_parameter_value + self.step_size

        if self.scan_parameter_value >= scan_parameter_range[1]:
            logging.warning("Reached maximum of PlsrDAC range... stopping scan")

    def analyze(self):
        with AnalyzeRawData(raw_data_file=self.output_filename, create_pdf=True) as analyze_raw_data:
            analyze_raw_data.create_tot_hist = False
            analyze_raw_data.create_fitted_threshold_hists = True
            analyze_raw_data.create_threshold_mask = True
            analyze_raw_data.n_injections = self.n_injections
            analyze_raw_data.interpreter.set_warning_output(True)
            analyze_raw_data.interpret_word_table()
            analyze_raw_data.interpreter.print_summary()
            analyze_raw_data.plot_histograms()

    def scan_condition(self, occupancy_array):
        occupancy_array_select = occupancy_array[self.select_arr_columns, :]  # only select not ignored columns
        # stop precise scanning actions
        pixels_with_full_hits = np.ma.array(occupancy_array_select, mask=(occupancy_array_select >= self.n_injections))  # select pixels that see all injections
        pixels_with_full_hits_count = np.ma.count_masked(pixels_with_full_hits)  # count pixels that see all injections
        stop_pixel_cnt = int(np.product(occupancy_array_select.shape) * self.stop_at)
        if pixels_with_full_hits_count >= stop_pixel_cnt and not self.stop_condition_triggered:  # stop precise scanning if this triggers
            logging.info("Triggering stop condition: %d pixel(s) with %d hits or more >= %d pixel(s)", pixels_with_full_hits_count, self.n_injections, stop_pixel_cnt)
            self.stop_condition_triggered = True
        # start precise scanning actions
        pixels_with_hits = np.ma.array(occupancy_array_select, mask=(occupancy_array_select != 0))  # select pixels that see at least one hit
        pixels_with_hits_count = np.ma.count_masked(pixels_with_hits)  # count pixels that see hits
        start_pixel_cnt = int(np.product(occupancy_array_select.shape) * self.start_at)
        if pixels_with_hits_count >= start_pixel_cnt and not self.start_condition_triggered:  # start precise scanning if this is true
            logging.info("Triggering start condition: %d pixel(s) with more than 0 hits >= %d pixel(s)", pixels_with_hits_count, start_pixel_cnt)
            self.start_condition_triggered = True
 class NoiseOccupancyTuning(Fei4RunBase):
    '''Noise occupancy scan detecting and masking noisy pixels.

    Note
    ----
    The total number of triggers which will be sent to the FE are <triggers> * <trig_count> (consecutive LVL1).
    To achieve a broader TDAC distribution it is necessary to decrease TdacVbp.
    '''
    _default_run_conf = {
        "broadcast_commands": True,  # use False to limit data rate
        "threaded_scan": True,
        "occupancy_limit": 1 * 10 ** (-5),  # the lower the number the higher the constraints on noise occupancy; 0 will mask any pixel with occupancy greater than zero
        "occupancy_p_val": 0.99,  # mask pixels with occupancy higher than expected occupancy at given p value
        "n_triggers": 10000000,  # total number of triggers which will be sent to the FE. From 1 to 4294967295 (32-bit unsigned int).
        "trig_count": 1,  # FE-I4 trigger count, number of consecutive BCs, 0 means 16, from 0 to 15
        "trigger_rate_limit": 500,  # artificially limiting the trigger rate, in BCs (25ns)
        "disable_for_mask": ['Enable'],  # list of masks for which noisy pixels will be disabled
        "enable_for_mask": ['Imon'],  # list of masks for which noisy pixels will be disabled
        "col_span": [1, 80],  # defining active column interval, 2-tuple, from 1 to 80
        "row_span": [1, 336],  # defining active row interval, 2-tuple, from 1 to 336
        "overwrite_enable_mask": False,  # if True, use col_span and row_span to define an active region regardless of the Enable pixel register. If False, use col_span and row_span to define active region by also taking Enable pixel register into account.
        "use_enable_mask_for_imon": False,  # if True, apply inverted Enable pixel mask to Imon pixel mask
        "no_data_timeout": 10,  # no data timeout after which the scan will be aborted, in seconds
        "overwrite_mask": False  # if True, overwrite existing masks
    }

    def configure(self):
        if self.trig_count == 0:
            self.consecutive_lvl1 = (2 ** self.register.global_registers['Trig_Count']['bitlength'])
        else:
            self.consecutive_lvl1 = self.trig_count
        self.abs_occ_limit = int(self.occupancy_limit * self.n_triggers * self.consecutive_lvl1)
        self.abs_occ_limit = stats.poisson.ppf(self.occupancy_p_val, mu=self.occupancy_limit * self.n_triggers * self.consecutive_lvl1)
        logging.info('Masking pixels with occupancy >%d (sending %d triggers)', self.abs_occ_limit, self.n_triggers)

        commands = []
        commands.extend(self.register.get_commands("ConfMode"))
        # Enable
        enable_pixel_mask = make_box_pixel_mask_from_col_row(column=self.col_span, row=self.row_span)
        if not self.overwrite_enable_mask:
            enable_pixel_mask = np.logical_and(enable_pixel_mask, self.register.get_pixel_register_value('Enable'))
        self.register.set_pixel_register_value('Enable', enable_pixel_mask)
        commands.extend(self.register.get_commands("WrFrontEnd", same_mask_for_all_dc=False, name='Enable'))
        # Imon
        if self.use_enable_mask_for_imon:
            imon_pixel_mask = invert_pixel_mask(enable_pixel_mask)
        else:
            imon_pixel_mask = make_box_pixel_mask_from_col_row(column=self.col_span, row=self.row_span, default=1, value=0)  # 0 for selected columns, else 1
            imon_pixel_mask = np.logical_or(imon_pixel_mask, self.register.get_pixel_register_value('Imon'))
        self.register.set_pixel_register_value('Imon', imon_pixel_mask)
        commands.extend(self.register.get_commands("WrFrontEnd", same_mask_for_all_dc=False, name='Imon'))
        # C_High
        self.register.set_pixel_register_value('C_High', 0)
        commands.extend(self.register.get_commands("WrFrontEnd", same_mask_for_all_dc=True, name='C_High'))
        # C_Low
        self.register.set_pixel_register_value('C_Low', 0)
        commands.extend(self.register.get_commands("WrFrontEnd", same_mask_for_all_dc=True, name='C_Low'))
        # Registers
#         self.register.set_global_register_value("Trig_Lat", self.trigger_latency)  # set trigger latency
        self.register.set_global_register_value("Trig_Count", self.trig_count)  # set number of consecutive triggers
        commands.extend(self.register.get_commands("WrRegister", name=["Trig_Count"]))
        commands.extend(self.register.get_commands("RunMode"))
        self.register_utils.send_commands(commands)

    def scan(self):
        # preload command
        lvl1_command = self.register.get_commands("LV1")[0] + self.register.get_commands("zeros", length=self.trigger_rate_limit)[0]
        self.total_scan_time = int(lvl1_command.length() * 25 * (10 ** -9) * self.n_triggers)
        logging.info('Estimated scan time: %ds', self.total_scan_time)

        with self.readout(no_data_timeout=self.no_data_timeout):
            got_data = False
            start = time()
            self.register_utils.send_command(lvl1_command, repeat=self.n_triggers, wait_for_finish=False, set_length=True, clear_memory=False)
            while not self.stop_run.wait(1.0):
                if self.register_utils.is_ready:
                    if got_data:
                        self.progressbar.finish()
                    self.stop('Finished sending %d triggers' % self.n_triggers)
                if not got_data:
                    if self.data_words_per_second() > 0:
                        got_data = True
                        logging.info('Taking data...')
                        self.progressbar = progressbar.ProgressBar(widgets=['', progressbar.Percentage(), ' ', progressbar.Bar(marker='*', left='|', right='|'), ' ', progressbar.Timer()], maxval=self.total_scan_time, poll=10, term_width=80).start()
                else:
                    try:
                        self.progressbar.update(time() - start)
                    except ValueError:
                        pass

    def analyze(self):
        with AnalyzeRawData(raw_data_file=self.output_filename, create_pdf=True) as analyze_raw_data:
            analyze_raw_data.interpreter.set_warning_output(False)
            analyze_raw_data.create_source_scan_hist = True
            analyze_raw_data.create_hit_table = False
            analyze_raw_data.interpret_word_table()
            analyze_raw_data.plot_histograms()
            analyze_raw_data.interpreter.print_summary()
            # get occupancy hist
            occ_hist = analyze_raw_data.out_file_h5.root.HistOcc[:, :, 0].T
            self.occ_mask = np.zeros(shape=occ_hist.shape, dtype=np.dtype('>u1'))
            # noisy pixels are set to 1
            self.occ_mask[occ_hist > self.abs_occ_limit] = 1
            # make inverse
            self.inv_occ_mask = invert_pixel_mask(self.occ_mask)
            # generate masked occupancy hist
            masked_occ_hist = occ_hist.copy()
            masked_occ_hist[self.occ_mask == 1] = 0

            if self.overwrite_mask:
                for mask in self.disable_for_mask:
                    self.register.set_pixel_register_value(mask, self.inv_occ_mask)
            else:
                for mask in self.disable_for_mask:
                    enable_mask = self.register.get_pixel_register_value(mask)
                    new_enable_mask = np.logical_and(self.inv_occ_mask, enable_mask)
                    self.register.set_pixel_register_value(mask, new_enable_mask)

            if self.overwrite_mask:
                for mask in self.enable_for_mask:
                    self.register.set_pixel_register_value(mask, self.occ_mask)
            else:
                for mask in self.enable_for_mask:
                    disable_mask = self.register.get_pixel_register_value(mask)
                    new_disable_mask = np.logical_or(self.occ_mask, disable_mask)
                    self.register.set_pixel_register_value(mask, new_disable_mask)
            plot_occupancy(self.occ_mask.T, title='Noisy Pixels', z_max=1, filename=analyze_raw_data.output_pdf)
            plot_fancy_occupancy(self.occ_mask.T, z_max=1, filename=analyze_raw_data.output_pdf)
            for mask in self.disable_for_mask:
                mask_name = self.register.pixel_registers[mask]['name']
                plot_occupancy(self.register.get_pixel_register_value(mask).T, title='%s Mask' % mask_name, z_max=1, filename=analyze_raw_data.output_pdf)
            for mask in self.enable_for_mask:
                mask_name = self.register.pixel_registers[mask]['name']
                plot_occupancy(self.register.get_pixel_register_value(mask).T, title='%s Mask' % mask_name, z_max=1, filename=analyze_raw_data.output_pdf)

            # adding Poisson statistics plots
            fig = Figure()
            FigureCanvas(fig)
            ax = fig.add_subplot(111)
            ax.set_title("Hit statistics")
            hist, bin_edges = np.histogram(occ_hist, bins=np.arange(0, np.max(occ_hist) + 2, 1))
            try:
                _, idx = hist_quantiles(hist, [0.0, 0.9], return_indices=True)
            except IndexError:
                idx = [0, 1]
            bins = np.arange(0, np.maximum(bin_edges[idx[1]], stats.poisson.ppf(0.9999, mu=self.occupancy_limit * self.n_triggers * self.consecutive_lvl1)) + 2, 1)
            _, bin_edges, _ = ax.hist(occ_hist.flatten(), bins=350 if len(bins) > 351 else bins, range=(0.0, bins[-1]), align='left', alpha=0.5, label="Measured occupancy before masking noisy pixels")
            ax.hist(masked_occ_hist.flatten(), bins=bin_edges, align='left', alpha=0.5, label="Measured occupancy after masking noisy pixels")
            # re-binning
            statistic, _, _ = stats.binned_statistic(bins[:-1], stats.poisson.pmf(k=bins[:-1], mu=self.occupancy_limit * self.n_triggers * self.consecutive_lvl1) * self.register.get_pixel_register_value("Enable").sum(), statistic='sum', bins=bin_edges)
            ax.bar(bin_edges[:-1], statistic, alpha=0.5, width=bin_edges[1] - bin_edges[0], color="r", label="Expected occupancy (Poisson statistics)")
            # ax.hist(stats.poisson.rvs(mu=self.occupancy_limit * self.n_triggers * self.consecutive_lvl1, size=self.register.get_pixel_register_value("Enable").sum()), bins=bins, align='left', alpha=0.5, label="Expected occupancy (Poisson statistics)")
            ax.set_xlabel('#Hits')
            ax.set_ylabel('#Pixels')
            ax.legend()
            analyze_raw_data.output_pdf.savefig(fig)
 class PlsrDacCalibration(Fei4RunBase):
    def configure(self):
    def scan(self):
    def analyze(self):
def plot_pulser_dac(x, y, y_err=None, output_pdf=None, title_suffix="", atol_first_dev=1.0 * 1e-04, atol_second_dev=1.0 * 1e-05):
    def slope_fit_fn(x, offset, slope):
    def plateau_fit_fn(x, offset): class HitOrCalibration(Fei4RunBase):
    ''' HitOR calibration scan
    '''
    _default_run_conf = {
        "broadcast_commands": True,
        "threaded_scan": True,
        "n_injections": 200,  # number of injections
        "injection_delay": 5000,  # for really low feedbacks (ToT >> 300 ns) one needs to increase the injection delay
        "scan_parameters": [('column', None),
                            ('row', None),
                            ('PlsrDAC', [40, 50, 60, 80, 130, 180, 230, 280, 340, 440, 540, 640, 740])],  # 0 400 sufficient for most tunings
        "reset_rx_on_error": True,  # reset RX on errors that may occur during scan
        "pixels": (np.dstack(np.where(make_box_pixel_mask_from_col_row([40, 41], [150, 151]) == 1)) + 1).tolist()[0],  # list of (col, row) tupels. From 1 to 80/336.
        "enable_shift_masks": ["Enable", "C_Low", "C_High"],  # enable masks shifted during scan
        "disable_shift_masks": ["Imon"]  # disable masks shifted during scan
    }

    def configure(self):
        commands = []
        commands.extend(self.register.get_commands("ConfMode"))
        self.register.set_global_register_value("Trig_Count", 6)  # decrease trigger count to reduce data
        self.register.set_global_register_value("Trig_Lat", 215)  # adjust delay for smaller bcid window
        self.register.set_global_register_value("ErrorMask", 1536)  # deactivate hit bus service record
        commands.extend(self.register.get_commands("WrRegister", name=["Trig_Lat", "Trig_Count", "ErrorMask"]))
        self.register_utils.send_commands(commands)

    def scan(self):
        def write_double_column(column):
            return (column - 1) / 2

        def inject_double_column(column):
            if column == 80:
                return 39
            else:
                return (column) / 2

        cal_lvl1_command = self.register.get_commands("CAL")[0] + self.register.get_commands("zeros", length=40)[0] + self.register.get_commands("LV1")[0] + self.register.get_commands("zeros", length=self.injection_delay)[0]
        scan_parameter_name = self.scan_parameters._fields[-1]  # scan parameter is in inner loop
        scan_parameter_values = self.scan_parameters[-1][:]  # create deep copy of scan_parameters, they are overwritten in self.readout

        pixels_sorted = sorted(self.pixels)  # , key=lambda tup: tup[0])
        logging.info("Scanning %d pixels" % len(self.pixels))
        # use sorted pixels to prevent overwriting of raw data file when writing a file per column
        for pixel_index, pixel in enumerate(pixels_sorted):
            if self.stop_run.is_set():
                break

            column = pixel[0]
            row = pixel[1]
            logging.info('Scanning pixel: %d / %d (column / row)', column, row)
            if pixel_index:
                dcs = [write_double_column(column)]
                dcs.append(write_double_column(self.pixels[pixel_index - 1][0]))
            else:
                dcs = []
            commands = []
            commands.extend(self.register.get_commands("ConfMode"))
            single_pixel_enable_mask = make_pixel_mask_from_col_row([column], [row])
            map(lambda mask_name: self.register.set_pixel_register_value(mask_name, single_pixel_enable_mask), self.enable_shift_masks)
            commands.extend(self.register.get_commands("WrFrontEnd", same_mask_for_all_dc=False, dcs=dcs, name=self.enable_shift_masks, joint_write=True))
            single_pixel_disable_mask = make_pixel_mask_from_col_row([column], [row], default=1, value=0)
            map(lambda mask_name: self.register.set_pixel_register_value(mask_name, single_pixel_disable_mask), self.disable_shift_masks)
            commands.extend(self.register.get_commands("WrFrontEnd", same_mask_for_all_dc=False, dcs=dcs, name=self.disable_shift_masks, joint_write=True))
            self.register.set_global_register_value("Colpr_Addr", inject_double_column(column))
            commands.append(self.register.get_commands("WrRegister", name=["Colpr_Addr"])[0])
            self.register_utils.send_commands(commands)

            self.dut['TDC']['ENABLE'] = True
            self.dut['TDC']['EN_NO_WRITE_TRIG_ERR'] = False  # Do not trigger TDC words
            for scan_parameter_value in scan_parameter_values:
                if self.stop_run.is_set():
                    break

                commands = []
                commands.extend(self.register.get_commands("ConfMode"))
                self.register.set_global_register_value(scan_parameter_name, scan_parameter_value)
                commands.extend(self.register.get_commands("WrRegister", name=[scan_parameter_name]))
                commands.extend(self.register.get_commands("RunMode"))
                self.register_utils.send_commands(commands)

                self.dut['TDC']['EN_ARMING'] = True
                with self.readout(reset_fifo=True, column=column, row=row, **{scan_parameter_name: scan_parameter_value}):
                    self.register_utils.send_command(command=cal_lvl1_command, repeat=self.n_injections)
                self.dut['TDC']['EN_ARMING'] = False

            self.dut['TDC']['ENABLE'] = False

    def handle_data(self, data, new_file=['column'], flush=True):  # Create new file for each scan parameter change
        super(HitOrCalibration, self).handle_data(data=data, new_file=new_file, flush=flush)

    def analyze(self):
        create_hitor_calibration(self.output_filename, plot_pixel_calibrations=True)
 class ExtTriggerScan(Fei4RunBase):
    '''External trigger scan with FE-I4

    For use with external scintillator (user RX0), TLU (use RJ45), FE-I4 HitOR (USBpix self-trigger).

    Note:
    Set up trigger in DUT configuration file (e.g. dut_configuration_mio.yaml).
    '''
    _default_run_conf = {
        "broadcast_commands": True,
        "threaded_scan": True,
        "trig_count": 0,  # FE-I4 trigger count, number of consecutive BCs, 0 means 16, from 0 to 15
        "trigger_latency": 232,  # FE-I4 trigger latency, in BCs, external scintillator / TLU / HitOR: 232, USBpix self-trigger: 220
        "trigger_delay": 8,  # trigger delay, in BCs
        "trigger_rate_limit": 500,  # artificially limiting the trigger rate, in BCs (25ns)
        "col_span": [1, 80],  # defining active column interval, 2-tuple, from 1 to 80
        "row_span": [1, 336],  # defining active row interval, 2-tuple, from 1 to 336
        "overwrite_enable_mask": False,  # if True, use col_span and row_span to define an active region regardless of the Enable pixel register. If False, use col_span and row_span to define active region by also taking Enable pixel register into account.
        "use_enable_mask_for_imon": True,  # if True, apply inverted Enable pixel mask to Imon pixel mask
        "no_data_timeout": 10,  # no data timeout after which the scan will be aborted, in seconds
        "scan_timeout": 60,  # timeout for scan after which the scan will be stopped, in seconds
        "max_triggers": 10000,  # maximum triggers after which the scan will be stopped, if 0, no maximum triggers are set
        "enable_tdc": False,  # if True, enables TDC
        "reset_rx_on_error": False  # long scans have a high propability for ESD related data transmission errors; recover and continue here
    }

    def configure(self):
        commands = []
        commands.extend(self.register.get_commands("ConfMode"))
        # Enable
        enable_pixel_mask = make_box_pixel_mask_from_col_row(column=self.col_span, row=self.row_span)
        if not self.overwrite_enable_mask:
            enable_pixel_mask = np.logical_and(enable_pixel_mask, self.register.get_pixel_register_value('Enable'))
        self.register.set_pixel_register_value('Enable', enable_pixel_mask)
        commands.extend(self.register.get_commands("WrFrontEnd", same_mask_for_all_dc=False, name='Enable'))
        # Imon
        if self.use_enable_mask_for_imon:
            imon_pixel_mask = invert_pixel_mask(enable_pixel_mask)
        else:
            imon_pixel_mask = make_box_pixel_mask_from_col_row(column=self.col_span, row=self.row_span, default=1, value=0)  # 0 for selected columns, else 1
            imon_pixel_mask = np.logical_or(imon_pixel_mask, self.register.get_pixel_register_value('Imon'))
        self.register.set_pixel_register_value('Imon', imon_pixel_mask)
        commands.extend(self.register.get_commands("WrFrontEnd", same_mask_for_all_dc=False, name='Imon'))
        # C_High
        self.register.set_pixel_register_value('C_High', 0)
        commands.extend(self.register.get_commands("WrFrontEnd", same_mask_for_all_dc=True, name='C_High'))
        # C_Low
        self.register.set_pixel_register_value('C_Low', 0)
        commands.extend(self.register.get_commands("WrFrontEnd", same_mask_for_all_dc=True, name='C_Low'))
        # Registers
        self.register.set_global_register_value("Trig_Lat", self.trigger_latency)  # set trigger latency
        self.register.set_global_register_value("Trig_Count", self.trig_count)  # set number of consecutive triggers
        commands.extend(self.register.get_commands("WrRegister", name=["Trig_Lat", "Trig_Count"]))
        commands.extend(self.register.get_commands("RunMode"))
        self.register_utils.send_commands(commands)

    def scan(self):
        # preload command
        lvl1_command = self.register.get_commands("zeros", length=self.trigger_delay)[0] + self.register.get_commands("LV1")[0] + self.register.get_commands("zeros", length=self.trigger_rate_limit)[0]
        self.register_utils.set_command(lvl1_command)

        with self.readout(no_data_timeout=self.no_data_timeout, **self.scan_parameters._asdict()):
            with self.trigger():
                got_data = False
                start = time()
                while not self.stop_run.wait(1.0):
                    if not got_data:
                        if self.data_words_per_second() > 0:
                            got_data = True
                            logging.info('Taking data...')
                            if self.max_triggers:
                                self.progressbar = progressbar.ProgressBar(widgets=['', progressbar.Percentage(), ' ', progressbar.Bar(marker='*', left='|', right='|'), ' ', progressbar.AdaptiveETA()], maxval=self.max_triggers, poll=10, term_width=80).start()
                            else:
                                self.progressbar = progressbar.ProgressBar(widgets=['', progressbar.Percentage(), ' ', progressbar.Bar(marker='*', left='|', right='|'), ' ', progressbar.Timer()], maxval=self.scan_timeout, poll=10, term_width=80).start()
                    else:
                        triggers = self.dut['TLU']['TRIGGER_COUNTER']
                        try:
                            if self.max_triggers:
                                self.progressbar.update(triggers)
                            else:
                                self.progressbar.update(time() - start)
                        except ValueError:
                            pass
                        if self.max_triggers and triggers >= self.max_triggers:
                            self.progressbar.finish()
                            self.stop(msg='Trigger limit was reached: %i' % self.max_triggers)
        logging.info('Total amount of triggers collected: %d', self.dut['TLU']['TRIGGER_COUNTER'])

    def analyze(self):
        with AnalyzeRawData(raw_data_file=self.output_filename, create_pdf=True) as analyze_raw_data:
            analyze_raw_data.trigger_data_format = self.dut['TLU']['DATA_FORMAT']
            analyze_raw_data.create_source_scan_hist = True
            analyze_raw_data.create_cluster_size_hist = True
            analyze_raw_data.create_cluster_tot_hist = True
            analyze_raw_data.align_at_trigger = True
            if self.enable_tdc:
                analyze_raw_data.create_tdc_counter_hist = True  # histogram all TDC words
                analyze_raw_data.create_tdc_hist = True  # histogram the hit TDC information
                analyze_raw_data.align_at_tdc = False  # align events at the TDC word
            analyze_raw_data.interpreter.set_warning_output(False)
            analyze_raw_data.interpret_word_table()
            analyze_raw_data.interpreter.print_summary()
            analyze_raw_data.plot_histograms()

    @contextmanager
    def trigger(self):
        self.start_trigger()
        try:
            yield
        finally:
            try:
                self.stop_trigger()
            except Exception:
                # in case something fails, call this on last resort
                self.scan_timeout_timer.cancel()
                self.connect_cancel(["abort"])

    def start_trigger(self, *args, **kwargs):
        self.connect_cancel(["stop"])
        self.dut['TDC']['ENABLE'] = self.enable_tdc
        self.dut['TLU']['TRIGGER_COUNTER'] = 0
        if self.max_triggers:
            self.dut['TLU']['MAX_TRIGGERS'] = self.max_triggers
        else:
            self.dut['TLU']['MAX_TRIGGERS'] = 0  # infinity triggers
        self.dut['TX']['EN_EXT_TRIGGER'] = True
        with self.synchronized():
            self.dut['TLU']['TRIGGER_ENABLE'] = True

        def timeout():
            try:
                self.progressbar.finish()
            except AttributeError:
                pass
            self.stop(msg='Scan timeout was reached')

        self.scan_timeout_timer = Timer(self.scan_timeout, timeout)
        if self.scan_timeout:
            self.scan_timeout_timer.start()

    def stop_trigger(self):
        self.scan_timeout_timer.cancel()
        with self.synchronized():
            self.dut['TLU']['TRIGGER_ENABLE'] = False
        self.dut['TX']['EN_EXT_TRIGGER'] = False
        self.dut['TDC']['ENABLE'] = False
        self.connect_cancel(["abort"]) def make_box_pixel_mask_from_col_row(column, row, default=0, value=1):
    '''Generate box shaped mask from column and row lists. Takes the minimum and maximum value from each list.

    Parameters
    ----------
    column : iterable, int
        List of colums values.
    row : iterable, int
        List of row values.
    default : int
        Value of pixels that are not selected by the mask.
    value : int
        Value of pixels that are selected by the mask.

    Returns
    -------
    numpy.ndarray
    '''
    # FE columns and rows start from 1
    col_array = np.array(column) - 1
    row_array = np.array(row) - 1
    if np.any(col_array >= 80) or np.any(col_array < 0) or np.any(row_array >= 336) or np.any(row_array < 0):
        raise ValueError('Column and/or row out of range')
    shape = (80, 336)
    mask = np.full(shape, default, dtype=np.uint8)
    if column and row:
        mask[col_array.min():col_array.max() + 1, row_array.min():row_array.max() + 1] = value  # advanced indexing
    return mask

import os.path
import numpy as np
import tables as tb
import progressbar
import pybar.scans.analyze_source_scan_tdc_data as tdc_analysis
from matplotlib import pyplot as plt
from scipy.optimize import curve_fit, leastsq
from pyLandau import landau
from pybar.run_manager import RunManager  # importing run manager
from pybar.scans.scan_iv import IVScan
from pybar.scans.test_register import RegisterTest
from pybar.scans.scan_digital import DigitalScan
from pybar.scans.scan_analog import AnalogScan
from pybar.scans.tune_fei4 import Fei4Tuning
from pybar.scans.tune_stuck_pixel import StuckPixelTuning
from pybar.scans.scan_threshold_fast import FastThresholdScan
from pybar.scans.tune_noise_occupancy import NoiseOccupancyTuning
from pybar.scans.calibrate_plsr_dac import PlsrDacScan
from pybar.scans.calibrate_hit_or import HitOrCalibration
from pybar.scans.scan_ext_trigger import ExtTriggerScan
from pybar.fei4.register_utils import make_box_pixel_mask_from_col_row



def analyze_tdc(source_scan_filename, calibration_filename, col_span, row_span):
    # Data files
    calibation_file = calibration_filename
    raw_data_file = source_scan_filename
    hit_file = os.path.splitext(raw_data_file)[0] + r'_interpreted.h5'
    # Selection criterions, change this to your needs
    hit_selection = '(column > %d) & (column < %d) & (row > %d) & (row < %d)' % (col_span[0] + 1, col_span[1] - 1, row_span[0] + 5, row_span[1] - 5)  # deselect edge pixels for better cluster size cut
    hit_selection_conditions = ['(n_cluster==1)', '(n_cluster==1) & (cluster_size == 1)', '(n_cluster==1) & (cluster_size == 1) & (relative_BCID > 1) & (relative_BCID < 4) & ((tot > 12) | ((TDC * 1.5625 - tot * 25 < 100) & (tot * 25 - TDC * 1.5625 < 100))) & %s' % hit_selection]
    event_status_select_mask = 0b0000111111011111
    event_status_condition = 0b0000000100000000  # trigger, one in-time tdc word and perfect event structure required
    # Interpret raw data and create hit table
    tdc_analysis.analyze_raw_data(input_files=raw_data_file,
                                  output_file_hits=hit_file,
                                  interpreter_plots=True,
                                  overwrite_output_files=True,
                                  pdf_filename=raw_data_file,
                                  align_at_trigger=True,
                                  use_tdc_trigger_time_stamp=True,
                                  max_tdc_delay=253)
    # Select TDC histograms for different cut criterions, use the charge calibrations
    tdc_analysis.histogram_tdc_hits(hit_file,
                                    hit_selection_conditions,
                                    event_status_select_mask,
                                    event_status_condition,
                                    calibation_file,
                                    max_tdc=1500,
                                    n_bins=350)

    return os.path.splitext(hit_file)[0] + '_tdc_hists.h5'


def plsr_dac_to_charge(source_scan_filename, plsr_dac):
    with tb.open_file(source_scan_filename, 'r') as in_file_h5:
        vcal_c0 = float(in_file_h5.root.configuration.calibration_parameters[:][np.where(in_file_h5.root.configuration.calibration_parameters[:]['name'] == 'Vcal_Coeff_0')]['value'][0])
        vcal_c1 = float(in_file_h5.root.configuration.calibration_parameters[:][np.where(in_file_h5.root.configuration.calibration_parameters[:]['name'] == 'Vcal_Coeff_1')]['value'][0])
        c_high = float(in_file_h5.root.configuration.calibration_parameters[:][np.where(in_file_h5.root.configuration.calibration_parameters[:]['name'] == 'C_Inj_High')]['value'][0])
        voltage = vcal_c0 + vcal_c1 * plsr_dac
        return voltage * c_high / 0.16022


def fit_landau_bootstrap(x, y, p0, n_sigma=1, n_iterations=500, **kwargs):  # fit the landau with bootstrap to give reasonable fit errors
    def errfunc(p, x, y):  # langau errorfunktion to minimize in fit
        return landau.langau(x, *p) - y

    yerr = kwargs.get('yerr', None)
    pfit, _ = curve_fit(landau.langau, x, y, p0=p0)

    residuals = errfunc(pfit, x, y)
    s_res = np.std(residuals)
    ps = []

    progress_bar = progressbar.ProgressBar(widgets=['', progressbar.Percentage(), ' ', progressbar.Bar(marker='*', left='|', right='|'), ' ', progressbar.AdaptiveETA()], maxval=n_iterations, term_width=80)
    progress_bar.start()
    for i in range(n_iterations):
        if yerr is None:
            randomDelta = np.random.normal(0.0, s_res, len(y))
            randomdataY = y + randomDelta
        else:
            randomDelta = np.array([np.random.normal(0.0, derr, 1)[0] for derr in yerr])
            randomdataY = y + randomDelta
        randomfit, _ = leastsq(errfunc, p0, args=(x, randomdataY), full_output=0)
        ps.append(randomfit)
        progress_bar.update(i)
    progress_bar.finish()

    mean_pfit, err_pfit = np.mean(ps, 0), n_sigma * np.std(ps, 0)

    return mean_pfit, err_pfit


def plot_landau(source_scan_filename, tdc_hists, target_threshold, fit_range=(13000, 30000)):
    with tb.open_file(tdc_hists, 'r') as in_file_h5:
        x, count, count_error = in_file_h5.root.HistTdcCalibratedCondition_2[:]['charge'], in_file_h5.root.HistTdcCalibratedCondition_2[:]['count'], in_file_h5.root.HistTdcCalibratedCondition_2[:]['count_error']
        charge = plsr_dac_to_charge(source_scan_filename, x)
        target_threshold_charge = plsr_dac_to_charge(source_scan_filename, target_threshold)
        plt.clf()
        plt.grid()
        x_fit_range = np.logical_and(charge > fit_range[0], charge < fit_range[1])
        coeff, err = fit_landau_bootstrap(charge[x_fit_range], count[x_fit_range], p0=(7000, np.std(charge[x_fit_range]), 150, np.amax(count[x_fit_range])), yerr=count_error[x_fit_range], n_iterations=100)
        plt.bar(charge, count, width=charge[1] - charge[0], color='blue', label='data')
        plt.plot(charge[x_fit_range], landau.langau(charge[x_fit_range], *coeff), 'r-')
        plt.plot([target_threshold_charge, target_threshold_charge], [plt.ylim()[0], plt.ylim()[1]], 'b--', linewidth=2, label='Threshold $%d$ e' % target_threshold_charge)
        plt.plot([coeff[0], coeff[0]], [plt.ylim()[0], plt.ylim()[1]], 'r--', linewidth=2, label='MPV $%d\pm%d$ e' % (coeff[0], err[0]))
        plt.title('Landau, -30 C')
        plt.legend(loc=0)
        plt.show()


if __name__ == "__main__":
    # Settings
    bias_voltage = -80
    max_iv_voltage = -100
    #   Tuning
    cref = 12
    target_threshold = 34
    target_charge = 300
    target_tot = 9

    #   TDC measurements
    plsr_dacs = [target_threshold, 40, 50, 60, 80, 100, 120, 150, 200, 250, 300, 350, 400, 500, 600, 700, 800]  # PlsrDAC range for TDC calibration, should start at threshold
    col_span = [55, 75]  # [50, 78]  # pixel column range to use in TDC scans
    row_span = [125, 225]  # [20, 315]  # pixel row range to use in TDC scans
    tdc_pixel = make_box_pixel_mask_from_col_row(column=[col_span[0], col_span[1]], row=[row_span[0], row_span[1]])  # edge pixel are not used in analysis

    runmngr = RunManager('configuration.yaml')

    # IV scan
    runmngr.run_run(run=IVScan, run_conf={"voltages": np.arange(-1, max_iv_voltage - 1, -1), "max_voltage": max_iv_voltage, "bias_voltage": bias_voltage, "minimum_delay": 0.5})

    # FE check and complete tuning
    runmngr.run_run(run=RegisterTest)
    runmngr.run_run(run=DigitalScan)  # digital scan with std. settings

    if runmngr.current_run.register.flavor == 'fei4a':  # FEI4 A related config changes, Deactivate noisy edge columns if FE-I4A
        runmngr.current_run.register.set_global_register_value("DisableColumnCnfg", 549755813891)  # Disable noisy columns
        runmngr.current_run.register.set_global_register_value("Cref", cref)  # Set correct cref
        runmngr.current_run.register.save_configuration(runmngr.current_run.register.configuration_file)

        runmngr.run_run(run=DigitalScan)  # repeat digital scan with specific settings