def groupby_dict(dictionary, key):
    ''' Group dict of dicts by key.
    '''
    return dict((k, list(g)) for k, g in itertools.groupby(sorted(dictionary.keys(), key=lambda name: dictionary[name][key]), key=lambda name: dictionary[name][key]))
 def dict_compare(d1, d2):
    '''Comparing two dictionaries.

    Note: https://stackoverflow.com/questions/4527942/comparing-two-dictionaries-in-python
    '''
    d1_keys = set(d1.keys())
    d2_keys = set(d2.keys())
    intersect_keys = d1_keys.intersection(d2_keys)
    added = d1_keys - d2_keys
    removed = d2_keys - d1_keys
    modified = {o: (d1[o], d2[o]) for o in intersect_keys if d1[o] != d2[o]}
    same = set(o for o in intersect_keys if d1[o] == d2[o])
    return added, removed, modified, same
 def zip_nofill(*iterables):
    '''Zipping iterables without fillvalue.

    Note: https://stackoverflow.com/questions/38054593/zip-longest-without-fillvalue
    '''
    return (tuple([entry for entry in iterable if entry is not None]) for iterable in itertools.izip_longest(*iterables, fillvalue=None))
 def find_file_dir_up(filename, path=None, n=None):
    '''Finding file in directory upwards.
    '''
    if path is None:
        path = os.getcwd()
    i = 0
    while True:
        current_path = path
        for _ in range(i):
            current_path = os.path.split(current_path)[0]
        if os.path.isfile(os.path.join(current_path, filename)):  # found file and return
            return os.path.join(current_path, filename)
        elif os.path.dirname(current_path) == current_path:  # root of filesystem
            return
        elif n is not None and i == n:
            return
        else:  # file not found
            i += 1
            continue
 class RunManager(object):
    def __init__(self, conf):
        '''Run Manager is taking care of initialization and execution of runs.

        Parameters
        ----------
        conf : str, dict, file
            Configuration for the run(s). Configuration will be passed to all scans.
        '''
        # fixing event handler: http://stackoverflow.com/questions/15457786/ctrl-c-crashes-python-after-importing-scipy-stats
        if os.name == 'nt':
            import thread

            def handler(signum, hook=thread.interrupt_main):
                hook()
                return True

            import win32api
            win32api.SetConsoleCtrlHandler(handler, 1)

        self._conf = None  # configuration dictionary
        self.current_run = None  # current run number
        self._conf_path = None  # absolute path of the configuation file
        self.init(conf)

    @property
    def conf(self):
        '''Configuration (namedtuple)
        '''
        conf = namedtuple('conf', field_names=self._conf.keys())
        return conf(**self._conf)  # prevent changing dict

    def init(self, conf):
        # current working directory
        if isinstance(conf, basestring) and os.path.isfile(os.path.abspath(conf)):
            conf = os.path.abspath(conf)
            self._conf_path = conf
        # search directory upwards form current working directory
        elif isinstance(conf, basestring) and find_file_dir_up(conf):
            conf = find_file_dir_up(conf)
            self._conf_path = conf
        elif isinstance(conf, file):
            self._conf_path = os.path.abspath(conf.name)
        else:
            self._conf_path = None
        self._conf = self.open_conf(conf)
        if 'working_dir' in self._conf and self._conf['working_dir']:
            # dirty fix for Windows pathes
            self._conf['working_dir'] = os.path.normpath(self._conf['working_dir'].replace('\\', '/'))
            if self._conf_path and not os.path.isabs(self._conf['working_dir']):
                # if working_dir is relative path, join path to configuration file and working_dir
                self._conf['working_dir'] = os.path.join(os.path.dirname(self._conf_path), self._conf['working_dir'])
            else:
                # working_dir is absolute path, keep that
                pass
        # use path of configuration file
        elif self._conf_path:
            self._conf['working_dir'] = os.path.dirname(self._conf_path)
        else:
            raise ValueError('Cannot deduce working directory from configuration')
        logging.info('Using working directory %s', self._conf['working_dir'])

    def close(self):
        if self.current_run is not None:
            self.current_run.close()

    def __enter__(self):
        return self

    def __exit__(self, *exc_info):
        self.close()

    @staticmethod
    def open_conf(conf):
        conf_dict = {}
        if not conf:
            pass
        elif isinstance(conf, basestring):  # parse the first YAML document in a stream
            if os.path.isfile(os.path.abspath(conf)):
                logging.info('Loading configuration from file %s', os.path.abspath(conf))
                with open(os.path.abspath(conf), 'r') as f:
                    conf_dict.update(safe_load(f))
            else:  # YAML string
                try:
                    conf_dict.update(safe_load(conf))
                except ValueError:  # invalid path/filename
                    raise IOError("File not found: %s" % os.path.abspath(conf))
                else:
                    logging.info('Loading configuration from file %s', os.path.abspath(conf.name))
        elif isinstance(conf, file):  # parse the first YAML document in a stream
            conf_dict.update(safe_load(conf))
        else:  # conf is already a dict
            conf_dict.update(conf)
        return conf_dict

    def cancel_current_run(self, msg=None):
        '''Control for runs.
        '''
        self.current_run.handle_cancel(msg=msg)

    def run_run(self, run, conf=None, run_conf=None, use_thread=False, catch_exception=True):
        '''Runs a run in another thread. Non-blocking.

        Parameters
        ----------
        run : class, object
            Run class or object.
        run_conf : str, dict, file
            Specific configuration for the run.
        use_thread : bool
            If True, run run in thread and returns blocking function.

        Returns
        -------
        If use_thread is True, returns function, which blocks until thread terminates, and which itself returns run status.
        If use_thread is False, returns run status.
        '''
        if isinstance(conf, basestring) and os.path.isfile(conf):
            logging.info('Updating configuration from file %s', os.path.abspath(conf))
        elif conf is not None:
            logging.info('Updating configuration')
        conf = self.open_conf(conf)
        self._conf.update(conf)

        if isclass(run):
            # instantiate the class
            run = run(conf=self._conf)

        local_run_conf = {}
        # general parameters from conf
        if 'run_conf' in self._conf:
            logging.info('Updating run configuration using run_conf key from configuration')
            local_run_conf.update(self._conf['run_conf'])
        # check for class name, scan specific parameters from conf
        if run.__class__.__name__ in self._conf:
            logging.info('Updating run configuration using %s key from configuration' % (run.__class__.__name__,))
            local_run_conf.update(self._conf[run.__class__.__name__])

        if isinstance(run_conf, basestring) and os.path.isfile(run_conf):
            logging.info('Updating run configuration from file %s', os.path.abspath(run_conf))
        elif run_conf is not None:
            logging.info('Updating run configuration')
        run_conf = self.open_conf(run_conf)
        # check for class name, scan specific parameters from conf
        if run.__class__.__name__ in run_conf:
            run_conf = run_conf[run.__class__.__name__]
        # run_conf parameter has highest priority, updated last
        local_run_conf.update(run_conf)

        if use_thread:
            self.current_run = run

            @thunkify(thread_name='RunThread', daemon=True, default_func=self.current_run.get_run_status)
            def run_run_in_thread():
                return run.run(run_conf=local_run_conf)

            signal.signal(signal.SIGINT, self._signal_handler)
            logging.info('Press Ctrl-C to stop run')

            return run_run_in_thread()
        else:
            self.current_run = run
            status = run.run(run_conf=local_run_conf)
            if not catch_exception and status != run_status.finished:
                raise RuntimeError('Exception occurred. Please read the log.')
            return status

    def run_primlist(self, primlist, skip_remaining=False):
        '''Runs runs from a primlist.

        Parameters
        ----------
        primlist : string
            Filename of primlist.
        skip_remaining : bool
            If True, skip remaining runs, if a run does not exit with status FINISHED.

        Note
        ----
        Primlist is a text file of the following format (comment line by adding '#'):
        <module name (containing class) or class (in either case use dot notation)>; <scan parameter>=<value>; <another scan parameter>=<another value>
        '''
        runlist = self.open_primlist(primlist)
        for index, run in enumerate(runlist):
            logging.info('Progressing with run %i out of %i...', index + 1, len(runlist))
            join = self.run_run(run, use_thread=True)
            status = join()
            if skip_remaining and not status == run_status.finished:
                logging.error('Exited run %i with status %s: Skipping all remaining runs.', run.run_number, status)
                break

    def open_primlist(self, primlist):
        def isrun(item, module):
            return isinstance(item, RunBase.__metaclass__) and item.__module__ == module  # only class from module, not from other imports

        if isinstance(primlist, basestring):
            with open(primlist, 'r') as f:
                f.seek(0)
                run_list = []
                for line in f.readlines():
                    line = line.partition('#')[0].strip()
                    if not line:
                        continue
                    parts = re.split('\s*[;]\s*', line)
                    try:
                        mod = import_module(parts[0])  # points to module
                    except ImportError:
                        mod = import_module(parts[0].rsplit('.', 1)[0])  # points to class
                        islocalrun = partial(isrun, module=parts[0].split('.')[-2])
                        clsmembers = getmembers(mod, islocalrun)
                        run_cls = None
                        for cls in clsmembers:
                            if cls[0] == parts[0].rsplit('.', 1)[1]:
                                run_cls = cls[1]
                                break
                        if not run_cls:
                            raise ValueError('Found no matching class: %s' % parts[0].rsplit('.', 1)[1])
                    else:
                        islocalrun = partial(isrun, module=parts[0])
                        clsmembers = getmembers(mod, islocalrun)
                        if len(clsmembers) > 1:
                            raise ValueError('Found more than one matching class.')
                        elif not len(clsmembers):
                            raise ValueError('Found no matching class.')
                        run_cls = clsmembers[0][1]
                    if run_cls.__class__.__name__ in self._conf:
                        run_conf = self._conf[run_cls.__class__.__name__]
                    else:
                        run_conf = {}
                    for param in parts[1:]:
                        key, value = re.split('\s*[=:]\s*', param, 1)
                        run_conf[key] = literal_eval(value)
                    run_list.append(run_cls(conf=self._conf, run_conf=run_conf))
            return run_list
        else:
            AttributeError('Primlist format not supported.')

    def _signal_handler(self, signum, frame):
        signal.signal(signal.SIGINT, signal.SIG_DFL)  # setting default handler... pressing Ctrl-C a second time will kill application
        self.cancel_current_run(msg='Pressed Ctrl-C') class RunBase(object):
    '''Basic run meta class

    Base class for run class.
    '''
    __metaclass__ = abc.ABCMeta

    def __init__(self, conf):
        '''Initialize object.

        Parameters
        ----------
        conf: dict
            Persistant configuration for all runs.
        run_conf : dict
            Run configuration for single run.
        '''
        self._conf = conf
        self._run_conf = None
        self._run_number = None
        self._run_status = run_status.init
        self.file_lock = Lock()
        self.stop_run = Event()  # abort condition for loops
        self.abort_run = Event()
        self._run_start_time = None
        self._run_stop_time = None
        self._total_run_time = None
        self._last_error_message = None
        self._last_traceback = None
        self._cancel_functions = None
        self.connect_cancel(["abort"])

    def __getattr__(self, name):
        ''' This is called in a last attempt to receive the value for an attribute that was not found in the usual places.
        '''
        try:
            return self._run_conf[name]  # Accessing run conf parameters
        except (KeyError, TypeError):  # If key is not existing or run conf is not a dict
            raise AttributeError("'%s' has no attribute '%s'" % (self.__class__.__name__, name))

    @property
    def run_id(self):
        '''Run name without whitespace
        '''
        s1 = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', self.__class__.__name__)
        return re.sub('([a-z0-9])([A-Z])', r'\1_\2', s1).lower()

    @property
    def conf(self):
        '''Configuration (namedtuple)
        '''
        conf = namedtuple('conf', field_names=self._conf.keys())
        return conf(**self._conf)  # prevent changing dict

    @property
    def run_conf(self):
        '''Run configuration (namedtuple)
        '''
        run_conf = namedtuple('run_conf', field_names=self._run_conf.keys())
        return run_conf(**self._run_conf)  # prevent changing dict

    @abc.abstractproperty
    def _default_run_conf(self):
        '''Defining default run configuration (dictionary)
        '''
        pass

    @property
    def default_run_conf(self):
        '''Default run configuration (namedtuple)
        '''
        default_run_conf = namedtuple('default_run_conf', field_names=self._default_run_conf.keys())
        return default_run_conf(**self._default_run_conf)  # prevent changing dict

    @property
    def working_dir(self):
        return self._conf['working_dir']

    @property
    def run_number(self):
        return self._run_number

    @run_number.setter
    def run_number(self, value):
        raise AttributeError

    @property
    def run_status(self):
        return self._run_status

    @run_status.setter
    def run_status(self, value):
        raise AttributeError

    def get_run_status(self):
        return self._run_status

    def run(self, run_conf, run_number=None, signal_handler=None):
        self._init(run_conf, run_number)
        logging.info('Starting run %d (%s) in %s', self.run_number, self.__class__.__name__, self.working_dir)
        # set up signal handler
        if isinstance(current_thread(), _MainThread):
            logging.info('Press Ctrl-C to stop run')
            if not signal_handler:
                signal_handler = self._signal_handler
            signal.signal(signal.SIGINT, signal_handler)
        try:
            with self._run():
                self.do_run()
        except RunAborted as e:
            self._run_status = run_status.aborted
            self._last_traceback = None
            self._last_error_message = e.__class__.__name__ + ": " + str(e)
        except Exception as e:
            self._run_status = run_status.crashed
            self._last_traceback = traceback.format_exc()
            self._last_error_message = e.__class__.__name__ + ": " + str(e)
        else:
            self._run_status = run_status.finished
            self._last_traceback = None
            self._last_error_message = None
        finally:
            pass
        # revert signal handler to default
        if isinstance(current_thread(), _MainThread):
            signal.signal(signal.SIGINT, signal.SIG_DFL)
        self._cleanup()
        # log message
        if self.run_status == run_status.finished:
            log_status = logging.INFO
        else:
            if self.run_status == run_status.stopped:
                log_status = logging.INFO
            elif self.run_status == run_status.aborted:
                log_status = logging.WARNING
            else:
                log_status = logging.ERROR
            logging.log(log_status, 'Run {} {}{}{}'.format(self.run_number, self.run_status, (': ' + str(self._last_error_message)) if self._last_error_message else '', ('\n' + self._last_traceback) if self._last_traceback else ''))
        if self._last_traceback:
            with open(os.path.join(self.working_dir, "crash" + ".log"), 'a+') as f:
                f.write('-------------------- Run {} ({}) --------------------\n'.format(self.run_number, self.__class__.__name__))
                traceback.print_exc(file=f)
                f.write('\n')
        logging.log(log_status, '{} run {} ({}) in {} (total time: {})'.format(self.run_status, self.run_number, self.__class__.__name__, self.working_dir, str(self._total_run_time)))
        return self.run_status

    def _init(self, run_conf, run_number=None):
        '''Initialization before a new run.
        '''
        self.stop_run.clear()
        self.abort_run.clear()
        self._run_status = run_status.running
        self._write_run_number(run_number)
        self._init_run_conf(run_conf)

    def _init_run_conf(self, run_conf):
        attribute_names = [key for key in self._default_run_conf.keys() if (key in self.__dict__ or (hasattr(self.__class__, key) and isinstance(getattr(self.__class__, key), property)))]
        if attribute_names:
            raise RuntimeError('Attribute names already in use. Rename the following parameters in run conf: %s' % ', '.join(attribute_names))
        sc = namedtuple('run_configuration', field_names=self._default_run_conf.keys())
        default_run_conf = sc(**self._default_run_conf)
        if run_conf:
            self._run_conf = default_run_conf._replace(**run_conf)._asdict()
        else:
            self._run_conf = default_run_conf._asdict()

    @contextmanager
    def _run(self):
        try:
            self.pre_run()
            yield
            self.post_run()
            if self.abort_run.is_set():
                raise RunAborted()
        finally:
            self.cleanup_run()

    @abc.abstractmethod
    def pre_run(self):
        '''Before run.
        '''
        pass

    @abc.abstractmethod
    def do_run(self):
        '''The run.
        '''
        pass

    @abc.abstractmethod
    def post_run(self):
        '''After run.
        '''
        pass

    @abc.abstractmethod
    def cleanup_run(self):
        '''Cleanup after run, will be executed always, even after exception. Avoid throwing exceptions here.
        '''
        pass

    def _cleanup(self):
        '''Cleanup after a new run.
        '''
        self._write_run_status(self.run_status)

    def connect_cancel(self, functions):
        '''Run given functions when a run is cancelled.
        '''
        self._cancel_functions = []
        for func in functions:
            if isinstance(func, basestring) and hasattr(self, func) and callable(getattr(self, func)):
                self._cancel_functions.append(getattr(self, func))
            elif callable(func):
                self._cancel_functions.append(func)
            else:
                raise ValueError("Unknown function %s" % str(func))

    def handle_cancel(self, **kwargs):
        '''Cancelling a run.
        '''
        for func in self._cancel_functions:
            f_args = getargspec(func)[0]
            f_kwargs = {key: kwargs[key] for key in f_args if key in kwargs}
            func(**f_kwargs)

    def stop(self, msg=None):
        '''Stopping a run. Control for loops. Gentle stop/abort.

        This event should provide a more gentle abort. The run should stop ASAP but the run is still considered complete.
        '''
        if not self.stop_run.is_set():
            if msg:
                logging.info('%s%s Stopping run...', msg, ('' if msg[-1] in punctuation else '.'))
            else:
                logging.info('Stopping run...')
        self.stop_run.set()

    def abort(self, msg=None):
        '''Aborting a run. Control for loops. Immediate stop/abort.

        The implementation should stop a run ASAP when this event is set. The run is considered incomplete.
        '''
        if not self.abort_run.is_set():
            if msg:
                logging.error('%s%s Aborting run...', msg, ('' if msg[-1] in punctuation else '.'))
            else:
                logging.error('Aborting run...')
        self.abort_run.set()
        self.stop_run.set()  # set stop_run in case abort_run event is not used

    def close(self):
        '''Close properly and releasing hardware resources.

        This should be called before Python garbage collector takes action.
        '''
        pass

    def _get_run_numbers(self, status=None):
        run_numbers = {}
        with self.file_lock:
            if not os.path.exists(self.working_dir):
                os.makedirs(self.working_dir)
            # In Python 2.x, open on all POSIX systems ultimately just depends on fopen.
            with open(os.path.join(self.working_dir, "run" + ".cfg"), 'a+') as f:
                f.seek(0)
                for line in f.readlines():
                    try:
                        number_parts = re.findall('\d+\s+', line)
                        parts = re.split('\s+', line)
                        if status:
                            if parts[2] in status:
                                run_number = int(number_parts[0])
                            else:
                                continue
                        else:
                            run_number = int(number_parts[0])
                    except IndexError:
                        continue
                    if line[-1] != '\n':
                        line = line + '\n'
                    run_numbers[run_number] = line
        return run_numbers

    def _write_run_number(self, run_number=None):
        self._run_start_time = datetime.datetime.now()
        run_numbers = self._get_run_numbers()
        if run_number:
            self._run_number = run_number
        else:
            if not run_numbers:
                self._run_number = 1
            else:
                self._run_number = max(dict.iterkeys(run_numbers)) + 1
        run_numbers[self.run_number] = str(self.run_number) + ' ' + self.__class__.__name__ + ' ' + 'RUNNING' + ' ' + str(self._run_start_time) + '\n'
        with self.file_lock:
            with open(os.path.join(self.working_dir, "run" + ".cfg"), "w") as f:
                for value in dict.itervalues(run_numbers):
                    f.write(value)

    def _write_run_status(self, status_msg):
        self._run_stop_time = datetime.datetime.now()
        self._total_run_time = self._run_stop_time - self._run_start_time
        run_numbers = self._get_run_numbers()
        if not run_numbers:
            run_numbers[self.run_number] = str(self.run_number) + ' ' + self.__class__.__name__ + ' ' + status_msg + ' ' + str(self._run_stop_time) + ' ' + str(self._total_run_time) + '\n'
        else:
            parts = re.split('\s+', run_numbers[self.run_number])
            parts[2] = status_msg
            run_numbers[self.run_number] = ' '.join(parts[:-1]) + ' ' + str(self._run_stop_time) + ' ' + str(self._total_run_time) + '\n'
        with self.file_lock:
            with open(os.path.join(self.working_dir, "run" + ".cfg"), "w") as f:
                for value in dict.itervalues(run_numbers):
                    f.write(value)

    def _signal_handler(self, signum, frame):
        signal.signal(signal.SIGINT, signal.SIG_DFL)  # setting default handler... pressing Ctrl-C a second time will kill application
        self.handle_cancel(msg='Pressed Ctrl-C') class RunAborted(Exception):
    pass class FEI4Register(object):

    def __init__(self, configuration_file=None, fe_type=None, chip_address=None, broadcast=False):
        '''FE-I4 register class.

        Note:
        Chip ID: This 4-bit field consists of broadcast bit and chip address. The broadcast bit, the most significant one, if set, means that the command is broadcasted to all FE chips receiving the data stream.
        Chip address: The three least significant bits of the chip ID define the chip address and are compared with the geographical address of the chip (selected via wire bonding/jumpers).
        '''
        self.flavor = None
        if fe_type:
            self.init_fe_type(fe_type)

        self.broadcast = None
        self.chip_address = None
        if chip_address is None:
            chip_address = 0

        self.configuration_file = fe_type
        if configuration_file:
            self.load_configuration(configuration_file)
        else:
            self.set_chip_address(chip_address, broadcast)

        self.config_state = OrderedDict()

    def __repr__(self):
        return self.configuration_file

    def set_chip_address(self, chip_address, broadcast):
        self.broadcast = broadcast
        if 7 < chip_address < 0:
            raise ValueError('Chip address out of range: %i' % chip_address)
        self.chip_id_initialized = True
        self.chip_address = chip_address
        self.chip_id = chip_address
        if self.broadcast:
            self.chip_id += 0x8
        self.chip_id_bitarray = bitarray_from_value(value=self.chip_id, size=4, fmt='I')
        logging.info('Setting chip address to %d (broadcast bit %s)', self.chip_address, 'set' if self.broadcast else 'not set')

    def init_fe_type(self, fe_type):
        self.flavor = None
        self.global_registers = {}
        self.pixel_registers = {}
        self.calibration_parameters = OrderedDict()
        self.miscellaneous = OrderedDict()
        self.commands = {}
        fei4_defines = import_module('pybar.fei4.fei4_defines')
        fe_type = getattr(fei4_defines, fe_type)
        if 'flavor' not in fe_type:
            raise ValueError('FEI4 flavor not defined')
        elif fe_type['flavor'] not in flavors:
            raise ValueError('Unknown FEI4 flavor: %s' % fe_type['flavor'])
        else:
            self.flavor = fe_type['flavor']
            logging.info('Initializing FEI4 registers (flavor: %s)', self.flavor)
        for name, reg in fe_type['global_registers'].iteritems():
            address = reg.get('address')
            offset = reg.get('offset', 0)
            bitlength = reg.get('bitlength')
            addresses = range(address, address + (offset + bitlength + 16 - 1) / 16)
            littleendian = reg.get('littleendian', False)
            register_littleendian = reg.get('register_littleendian', False)
            value = reg.get('value', 0)
            if not 0 <= value < 2 ** bitlength:
                raise ValueError("Global register %s: value exceeds limits" % (name,))
            readonly = reg.get('readonly', False)
            description = reg.get('description', '')
            self.global_registers[name] = dict(name=name, address=address, offset=offset, bitlength=bitlength, addresses=addresses, littleendian=littleendian, register_littleendian=register_littleendian, value=value, readonly=readonly, description=description)
        for name, reg in fe_type['pixel_registers'].iteritems():
            pxstrobe = reg.get('pxstrobe')
            bitlength = reg.get('bitlength')
            if bitlength > 8:
                raise Exception('Pixel register %s: up to 8 bits supported' % (name,))  # numpy array dtype is uint8
            littleendian = reg.get('littleendian', False)
            if not 0 <= reg.get('value', 0) < 2 ** bitlength:
                raise ValueError("Global register %s: value exceeds limits" % (name,))
            value = np.full((80, 336), reg.get('value', 0), dtype=np.uint8)
            description = reg.get('description', '')
            self.pixel_registers[name] = dict(name=name, pxstrobe=pxstrobe, bitlength=bitlength, littleendian=littleendian, value=value, description=description)
        for name, command in fe_type['commands'].iteritems():
            bitlength = command.get('bitlength')
            description = command.get('description', '')
            if 'bitstream' in command:
                bitstream = command.get('bitstream')
                self.commands[name] = dict(name=name, bitstream=bitstream, bitlength=bitlength, description=description)
            else:
                self.commands[name] = dict(name=name, bitlength=bitlength, description=description)
        self.calibration_parameters = fe_type['calibration_parameters'].copy()

    def is_chip_flavor(self, chip_flavor):
        if chip_flavor in flavors:
            if chip_flavor == self.flavor:
                return True
            else:
                return False
        else:
            raise ValueError('Unknown FEI4 flavor: %s' % chip_flavor)

    @property
    def chip_flavor(self):
        return self.flavor

    @property
    def fei4a(self):
        return True if self.flavor == 'fei4a' else False

    @property
    def fei4b(self):
        return True if self.flavor == 'fei4b' else False

    def load_configuration(self, configuration_file):
        '''Loading configuration

        Parameters
        ----------
        configuration_file : string
            Path to the configuration file (text or HDF5 file).
        '''
        if os.path.isfile(configuration_file):
            if not isinstance(configuration_file, tb.file.File) and os.path.splitext(configuration_file)[1].strip().lower() != ".h5":
                load_configuration_from_text_file(self, configuration_file)
            else:
                load_configuration_from_hdf5(self, configuration_file)
        else:
            raise ValueError('Cannot find configuration file specified: %s' % configuration_file)

    def save_configuration(self, configuration_file):
        '''Saving configuration

        Parameters
        ----------
        configuration_file : string
            Filename of the configuration file.
        '''
        if not isinstance(configuration_file, tb.file.File) and os.path.splitext(configuration_file)[1].strip().lower() != ".h5":
            return save_configuration_to_text_file(self, configuration_file)
        else:
            return save_configuration_to_hdf5(self, configuration_file)

    '''
    TODO:
    for the following functions use
    filter(function, iterable).

    Make new generic function that uses filter.

    Use next(iterator[, default]).
    '''

    def set_global_register_value(self, name, value):
        if self.global_registers[name]['readonly']:
            raise ValueError('Global register %s: register is read-only' % name)
        value = long(str(value), 0)  # value is decimal string or number or BitVector
        if not 0 <= value < 2 ** self.global_registers[name]['bitlength']:
            raise ValueError('Global register %s: value exceeds limits' % name)
        self.global_registers[name]['value'] = value

    def get_global_register_value(self, name):
        return self.global_registers[name]['value']

    def set_pixel_register_value(self, name, value):
        try:  # value is decimal string or number or array
            self.pixel_registers[name]['value'][:, :] = value
        except ValueError:  # value is path to pixel config
            if self.pixel_registers[name]['bitlength'] == 1:  # pixel mask
                if value[0] == "~" or value[0] == "!":
                    reg_value = parse_pixel_mask_config(os.path.join(os.path.dirname(self.configuration_file), os.path.normpath(value[1:].replace('\\', '/'))))
                    inverted_mask = np.ones(shape=(80, 336), dtype=np.dtype('>u1'))
                    inverted_mask[reg_value >= 1] = 0
                    self.pixel_registers[name]['value'][:, :] = inverted_mask
                else:
                    self.pixel_registers[name]['value'][:, :] = parse_pixel_mask_config(os.path.join(os.path.dirname(self.configuration_file), os.path.normpath(value).replace('\\', '/')))
            else:  # pixel dac
                self.pixel_registers[name]['value'][:, :] = parse_pixel_dac_config(os.path.join(os.path.dirname(self.configuration_file), os.path.normpath(value).replace('\\', '/')))
        if (self.pixel_registers[name]['value'] >= 2 ** self.pixel_registers[name]['bitlength']).any() or (self.pixel_registers[name]['value'] < 0).any():
            raise ValueError("Pixel register %s: value exceeds limits" % name)

    def get_pixel_register_value(self, name):
        return self.pixel_registers[name]['value'].copy()

    def get_commands(self, command_name, **kwargs):
        """get fe_command from command name and keyword arguments

        wrapper for build_commands()
        implements FEI4 specific behavior

        """
        chip_id = kwargs.pop("ChipID", self.chip_id_bitarray)
        commands = []
        if command_name == "zeros":
            bv = bitarray(endian='little')
            if "length" in kwargs:
                bv += bitarray(kwargs["length"], endian='little')  # initialized from int, bits may be random
            elif kwargs:
                raise ValueError("Unknown parameter(s): %s" % ", ".join(kwargs.iterkeys()))
            bv.setall(0)  # all bits to zero
            commands.append(bv)
        elif command_name == "ones":
            bv = bitarray(endian='little')
            if "length" in kwargs:
                bv += bitarray(kwargs["length"], endian='little')  # initialized from int, bits may be random
            elif kwargs:
                raise ValueError("Unknown parameter(s): %s" % ", ".join(kwargs.iterkeys()))
            bv.setall(1)  # all bits to one
            commands.append(bv)
        elif command_name == "WrRegister":
            register_addresses = self.get_global_register_attributes("addresses", **kwargs)
            register_bitsets = self.get_global_register_bitsets(register_addresses)
            commands.extend([self.build_command(command_name, Address=register_address, GlobalData=register_bitset, ChipID=chip_id, **kwargs) for register_address, register_bitset in zip(register_addresses, register_bitsets)])
        elif command_name == "RdRegister":
            register_addresses = self.get_global_register_attributes('addresses', **kwargs)
            commands.extend([self.build_command(command_name, Address=register_address, ChipID=chip_id) for register_address in register_addresses])
        elif command_name == "WrFrontEnd":
            registers = ["S0", "S1", "SR_Clr", "CalEn", "DIGHITIN_SEL", "GateHitOr", "ReadErrorReq", "StopClkPulse", "SR_Clock", "Efuse_Sense", "HITLD_IN", "Colpr_Mode", "Colpr_Addr"]
            if self.fei4a:
                registers.append("ReadSkipped")
            elif self.fei4b:
                registers.append("SR_Read")
            self.create_restore_point()
            dcs = kwargs.pop("dcs", range(40))  # set the double columns to latch
            # in case of empty list
            if not dcs:
                dcs = range(40)
            joint_write = kwargs.pop("joint_write", False)
            same_mask_for_all_dc = kwargs.pop("same_mask_for_all_dc", False)
            register_objects = self.get_pixel_register_objects(do_sort=['pxstrobe'], **kwargs)
            # prepare for writing pixel registers
            if not self.broadcast:
                self.set_global_register_value("Colpr_Mode", 0)  # write only to the addressed double-column
                self.set_global_register_value("Colpr_Addr", 40)  # ivalid address, grounded
                # Broadcasting ConfMode not necessary, writing registers is also possible in RunMode
#                 commands.extend(self.get_commands("ConfMode", ChipID=8))  # set all chips to conf mode to receive commands#
                # set all other chips to invalid addresses, to make broadcasting of WrRegister command possible
                commands.extend(self.get_commands("WrRegister", name=["Colpr_Mode", "Colpr_Addr"], ChipID=8))  # braodcast
            self.set_global_register_value("S0", 0)
            self.set_global_register_value("S1", 0)
            self.set_global_register_value("SR_Clr", 0)
            self.set_global_register_value("CalEn", 0)
            self.set_global_register_value("DIGHITIN_SEL", 0)
            self.set_global_register_value("GateHitOr", 0)
            self.set_global_register_value("ReadErrorReq", 0)
            self.set_global_register_value("StopClkPulse", 0)
            self.set_global_register_value("SR_Clock", 0)
            self.set_global_register_value("Efuse_Sense", 0)
            self.set_global_register_value("HITLD_IN", 0)
            self.set_global_register_value("Colpr_Mode", 3 if same_mask_for_all_dc else 0)  # write only the addressed double-column
            self.set_global_register_value("Colpr_Addr", 0)
            if self.fei4a:
                self.set_global_register_value("ReadSkipped", 0)
            elif self.fei4b:
                self.set_global_register_value("SR_Read", 0)
            commands.extend(self.get_commands("WrRegister", name=registers))

            if joint_write:
                pxstrobes = 0
                first_read = True
                do_latch = False
                for register_object in register_objects:
                    if register_object['bitlength'] != 1:
                        raise ValueError('Pixel register %s: joint write not supported for pixel DACs' % register_object['name'])
                    pxstrobe = register_object['pxstrobe']
                    if not isinstance(pxstrobe, basestring):
                        do_latch = True
                        pxstrobes += 2 ** register_object['pxstrobe']
                    if first_read:
                        pixel_reg_value = register_object['value']
                        first_read = False
                    else:
                        if np.array_equal(pixel_reg_value, register_object['value']):
                            pixel_reg_value = register_object['value']
                        else:
                            raise ValueError('Pixel register %s: joint write not supported, pixel register values must be equal' % register_object['name'])
                if do_latch:
                    self.set_global_register_value("Latch_En", 1)
                else:
                    self.set_global_register_value("Latch_En", 0)
                self.set_global_register_value("Pixel_Strobes", pxstrobes)
                commands.extend(self.get_commands("WrRegister", name=["Pixel_Strobes", "Latch_En"]))
                for dc_no in (dcs[:1] if same_mask_for_all_dc else dcs):
                    self.set_global_register_value("Colpr_Addr", dc_no)
                    commands.extend(self.get_commands("WrRegister", name=["Colpr_Addr"]))
                    register_bitset = self.get_pixel_register_bitset(register_objects[0], 0, dc_no)
                    commands.extend([self.build_command(command_name, PixelData=register_bitset, ChipID=8, **kwargs)])  # broadcast
                    if do_latch:
                        commands.extend(self.get_commands("GlobalPulse", Width=0))
            else:
                for register_object in register_objects:
                    pxstrobe = register_object['pxstrobe']
                    if isinstance(pxstrobe, basestring):
                        do_latch = False
                        self.set_global_register_value("Pixel_Strobes", 0)  # no latch
                        self.set_global_register_value("Latch_En", 0)
                        commands.extend(self.get_commands("WrRegister", name=["Pixel_Strobes", "Latch_En"]))
                    else:
                        do_latch = True
                        self.set_global_register_value("Latch_En", 1)
                        commands.extend(self.get_commands("WrRegister", name=["Latch_En"]))
                    bitlength = register_object['bitlength']
                    for bit_no, pxstrobe_bit_no in (enumerate(range(bitlength)) if (register_object['littleendian'] is False) else enumerate(reversed(range(bitlength)))):
                        if do_latch:
                            self.set_global_register_value("Pixel_Strobes", 2 ** (pxstrobe + bit_no))
                            commands.extend(self.get_commands("WrRegister", name=["Pixel_Strobes"]))
                        for dc_no in (dcs[:1] if same_mask_for_all_dc else dcs):
                            self.set_global_register_value("Colpr_Addr", dc_no)
                            commands.extend(self.get_commands("WrRegister", name=["Colpr_Addr"]))
                            register_bitset = self.get_pixel_register_bitset(register_object, pxstrobe_bit_no, dc_no)
                            commands.extend([self.build_command(command_name, PixelData=register_bitset, ChipID=8, **kwargs)])  # broadcast
                            if do_latch:
                                commands.extend(self.get_commands("GlobalPulse", Width=0))
            self.restore(pixel_register=False)
            commands.extend(self.get_commands("WrRegister", name=registers))
        elif command_name == "RdFrontEnd":
            registers = ["Conf_AddrEnable", "S0", "S1", "SR_Clr", "CalEn", "DIGHITIN_SEL", "GateHitOr", "ReadErrorReq", "StopClkPulse", "SR_Clock", "Efuse_Sense", "HITLD_IN", "Colpr_Mode", "Colpr_Addr", "Pixel_Strobes", "Latch_En"]
            if self.fei4a:
                registers.append("ReadSkipped")
            elif self.fei4b:
                registers.append("SR_Read")
            self.create_restore_point()
            dcs = kwargs.pop("dcs", range(40))  # set the double columns to latch
            # in case of empty list
            if not dcs:
                dcs = range(40)
            register_objects = self.get_pixel_register_objects(**kwargs)
            self.set_global_register_value('Conf_AddrEnable', 1)
            self.set_global_register_value("S0", 0)
            self.set_global_register_value("S1", 0)
            self.set_global_register_value("SR_Clr", 0)
            if self.fei4b:
                self.set_global_register_value("SR_Read", 0)
            self.set_global_register_value("CalEn", 0)
            self.set_global_register_value("DIGHITIN_SEL", 0)
            self.set_global_register_value("GateHitOr", 0)
            if self.fei4a:
                self.set_global_register_value("ReadSkipped", 0)
            self.set_global_register_value("ReadErrorReq", 0)
            self.set_global_register_value("StopClkPulse", 0)
            self.set_global_register_value("SR_Clock", 0)
            self.set_global_register_value("Efuse_Sense", 0)
            self.set_global_register_value("HITLD_IN", 0)
            self.set_global_register_value("Colpr_Mode", 0)  # write only the addressed double-column
            self.set_global_register_value("Colpr_Addr", 0)
            self.set_global_register_value("Latch_En", 0)
            self.set_global_register_value("Pixel_Strobes", 0)
            commands.extend(self.get_commands("WrRegister", name=registers))
            for index, register_object in enumerate(register_objects):  # make sure that EnableDigInj is first read back, because it is not latched
                if register_object['name'] == 'EnableDigInj':
                    register_objects[0], register_objects[index] = register_objects[index], register_objects[0]
                    break
            for register_object in register_objects:
                pxstrobe = register_object['pxstrobe']
                bitlength = register_object['bitlength']
                for pxstrobe_bit_no in range(bitlength):
                    logging.debug('Pixel Register %s Bit %d', register_object['name'], pxstrobe_bit_no)
                    do_latch = True
                    try:
                        self.set_global_register_value("Pixel_Strobes", 2 ** (pxstrobe + pxstrobe_bit_no))
                    except TypeError:  # thrown for not latched digInjection
                        self.set_global_register_value("Pixel_Strobes", 0)  # do not latch
                        do_latch = False
                    commands.extend(self.get_commands("WrRegister", name=["Pixel_Strobes"]))
                    for dc_no in dcs:
                        self.set_global_register_value("Colpr_Addr", dc_no)
                        commands.extend(self.get_commands("WrRegister", name=["Colpr_Addr"]))
                        if do_latch is True:
                            self.set_global_register_value("S0", 1)
                            self.set_global_register_value("S1", 1)
                            self.set_global_register_value("SR_Clock", 1)
                            commands.extend(self.get_commands("WrRegister", name=["S0", "S1", "SR_Clock"]))
                            commands.extend(self.get_commands("GlobalPulse", Width=0))
                        self.set_global_register_value("S0", 0)
                        self.set_global_register_value("S1", 0)
                        self.set_global_register_value("SR_Clock", 0)
                        commands.extend(self.get_commands("WrRegister", name=["S0", "S1", "SR_Clock"]))
                        register_bitset = self.get_pixel_register_bitset(register_object, pxstrobe_bit_no if (register_object['littleendian'] is False) else register_object['bitlength'] - pxstrobe_bit_no - 1, dc_no)
                        if self.fei4b:
                            self.set_global_register_value("SR_Read", 1)
                            commands.extend(self.get_commands("WrRegister", name=["SR_Read"]))
                        commands.extend([self.build_command("WrFrontEnd", PixelData=register_bitset, ChipID=chip_id)])
                        if self.fei4b:
                            self.set_global_register_value("SR_Read", 0)
                            commands.extend(self.get_commands("WrRegister", name=["SR_Read"]))
            self.restore(pixel_register=False)
            commands.extend(self.get_commands("WrRegister", name=registers))
        else:
            commands.append(self.build_command(command_name, ChipID=chip_id, **kwargs))
        return commands

    def build_command(self, command_name, **kwargs):
        """build command from command_name and keyword values

        Returns
        -------
        command_bitvector : list
            List of bitarrays.

        Usage
        -----
        Receives: command name as defined inside xml file, key-value-pairs as defined inside bit stream filed for each command
        """
#         command_name = command_name.lower()
        command_bitvector = bitarray(0, endian='little')
        if command_name not in self.commands:
            raise ValueError('Unknown command %s' % command_name)
        command_object = self.commands[command_name]
        command_parts = re.split(r'\s*[+]\s*', command_object['bitstream'])
        # for index, part in enumerate(command_parts, start = 1): # loop over command parts
        for part in command_parts:  # loop over command parts
            try:
                command_part_object = self.commands[part]
            except KeyError:
                command_part_object = None
            if command_part_object and 'bitstream'in command_part_object:  # command parts of defined content and length, e.g. Slow, ...
                if string_is_binary(command_part_object['bitstream']):
                    command_bitvector += bitarray(command_part_object['bitstream'], endian='little')
                else:
                    command_bitvector += self.build_command(part, **kwargs)
            elif command_part_object:  # Command parts with any content of defined length, e.g. ChipID, Address, ...
                if part in kwargs:
                    value = kwargs[part]
                else:
                    raise ValueError('Value of command part %s not given' % part)
                try:
                    command_bitvector += value
                except TypeError:  # value is no bitarray
                    if string_is_binary(value):
                        value = int(value, 2)
                    try:
                        command_bitvector += bitarray_from_value(value=int(value), size=command_part_object['bitlength'], fmt='I')
                    except Exception:
                        raise TypeError("Type of value not supported")
            elif string_is_binary(part):
                command_bitvector += bitarray(part, endian='little')
            # elif part in kwargs.keys():
            #    command_bitvector += kwargs[command_name]
            else:
                raise ValueError("Cannot process command part %s" % part)
        if command_bitvector.length() != command_object['bitlength']:
            raise ValueError("Command has unexpected length")
        if command_bitvector.length() == 0:
            raise ValueError("Command has length 0")
        return command_bitvector

    def get_global_register_attributes(self, register_attribute, do_sort=True, **kwargs):
        """Calculating register numbers from register names.

        Usage: get_global_register_attributes("attribute_name", name = [regname_1, regname_2, ...], addresses = 2)
        Receives: attribute name to be returned, dictionaries (kwargs) of register attributes and values for making cuts
        Returns: list of attribute values that matches dictionaries of attributes

        """
        # speed up of the most often used keyword name
        try:
            names = iterable(kwargs.pop('name'))
        except KeyError:
            register_attribute_list = []
        else:
            register_attribute_list = [self.global_registers[reg][register_attribute] for reg in names]
        for keyword in kwargs.keys():
            allowed_values = iterable(kwargs[keyword])
            try:
                register_attribute_list.extend(map(itemgetter(register_attribute), filter(lambda global_register: set(iterable(global_register[keyword])).intersection(allowed_values), self.global_registers.itervalues())))
            except AttributeError:
                pass
        if not register_attribute_list and filter(None, kwargs.itervalues()):
            raise ValueError('Global register attribute %s empty' % register_attribute)
        if do_sort:
            return sorted(set(flatten_iterable(register_attribute_list)))
        else:
            return flatten_iterable(register_attribute_list)

    def get_global_register_objects(self, do_sort=None, reverse=False, **kwargs):
        """Generate register objects (list) from register name list

        Usage: get_global_register_objects(name = ["Amp2Vbn", "GateHitOr", "DisableColumnCnfg"], address = [2, 3])
        Receives: keyword lists of register names, addresses,... for making cuts
        Returns: list of register objects

        """
        # speed up of the most often used keyword name
        try:
            names = iterable(kwargs.pop('name'))
        except KeyError:
            register_objects = []
        else:
            register_objects = [self.global_registers[reg] for reg in names]
        for keyword in kwargs.iterkeys():
            allowed_values = iterable(kwargs[keyword])
            register_objects.extend(filter(lambda global_register: set(iterable(global_register[keyword])).intersection(allowed_values), self.global_registers.itervalues()))
        if not register_objects and filter(None, kwargs.itervalues()):
            raise ValueError('Global register objects empty')
        if do_sort:
            return sorted(register_objects, key=itemgetter(*do_sort), reverse=reverse)
        else:
            return register_objects

    def get_global_register_bitsets(self, register_addresses):  # TOTO: add sorting
        """Calculating register bitsets from register addresses.

        Usage: get_global_register_bitsets([regaddress_1, regaddress_2, ...])
        Receives: list of register addresses
        Returns: list of register bitsets

        """
        register_bitsets = []
        for register_address in register_addresses:
            register_objects = self.get_global_register_objects(addresses=register_address)
            register_bitset = bitarray(16, endian='little')  # TODO remove hardcoded register size, see also below
            register_bitset.setall(0)
            register_littleendian = False
            for register_object in register_objects:
                if register_object['register_littleendian']:  # check for register endianness
                    register_littleendian = True
                if (16 * register_object['address'] + register_object['offset'] < 16 * (register_address + 1) and 16 * register_object['address'] + register_object['offset'] + register_object['bitlength'] > 16 * register_address):
                    reg = bitarray_from_value(value=register_object['value'], size=register_object['bitlength'])
                    if register_object['littleendian']:
                        reg.reverse()
# register_bitset[max(0, 16 * (register_object['address'] - register_address) + register_object['offset']):min(16, 16 * (register_object['address'] - register_address) + register_object['offset'] + register_object['bitlength'])] |= reg[max(0, 16 * (register_address - register_object['address']) - register_object['offset']):min(register_object['bitlength'], 16 * (register_address - register_object['address'] + 1) - register_object['offset'])]  # [ bit(n) bit(n-1)... bit(0) ]
                    register_bitset[max(0, 16 - 16 * (register_object['address'] - register_address) - register_object['offset'] - register_object['bitlength']):min(16, 16 - 16 * (register_object['address'] - register_address) - register_object['offset'])] |= reg[max(0, register_object['bitlength'] - 16 - 16 * (register_address - register_object['address']) + register_object['offset']):min(register_object['bitlength'], register_object['bitlength'] + 16 - 16 * (register_address - register_object['address'] + 1) + register_object['offset'])]  # [ bit(0)... bit(n-1) bit(n) ]
                else:
                    raise Exception("wrong register object")
            if register_littleendian:
                register_bitset.reverse()
            register_bitsets.append(register_bitset)
        return register_bitsets

    def get_pixel_register_objects(self, do_sort=None, reverse=False, **kwargs):
        """Generate register objects (list) from register name list

        Usage: get_pixel_register_objects(name = ["TDAC", "FDAC"])
        Receives: keyword lists of register names, addresses,...
        Returns: list of register objects

        """
        # speed up of the most often used keyword name
        try:
            names = iterable(kwargs.pop('name'))
        except KeyError:
            register_objects = []
        else:
            register_objects = [self.pixel_registers[reg] for reg in names]
        for keyword in kwargs.iterkeys():
            allowed_values = iterable(kwargs[keyword])
            register_objects.extend(filter(lambda pixel_register: pixel_register[keyword] in allowed_values, self.pixel_registers.itervalues()))
        if not register_objects and filter(None, kwargs.itervalues()):
            raise ValueError('Pixel register objects empty')
        if do_sort:
            return sorted(register_objects, key=itemgetter(*do_sort), reverse=reverse)
        else:
            return register_objects

    def get_pixel_register_bitset(self, register_object, bit_no, dc_no):
        """Calculating pixel register bitsets from pixel register addresses.

        Usage: get_pixel_register_bitset(object, bit_number, double_column_number)
        Receives: register object, bit number, double column number
        Returns: double column bitset

        """
        if not 0 <= dc_no < 40:
            raise ValueError("Pixel register %s: DC out of range" % register_object['name'])
        if not 0 <= bit_no < register_object['bitlength']:
            raise ValueError("Pixel register %s: bit number out of range" % register_object['name'])
        col0 = register_object['value'][dc_no * 2, :]
        sel0 = (2 ** bit_no == (col0 & 2 ** bit_no))
        bv0 = bitarray(sel0.tolist(), endian='little')
        col1 = register_object['value'][dc_no * 2 + 1, :]
        sel1 = (2 ** bit_no == (col1 & 2 ** bit_no))
        # sel1 = sel1.astype(numpy.uint8) # copy of array
        # sel1 = sel1.view(dtype=np.uint8) # in-place type conversion
        bv1 = bitarray(sel1.tolist(), endian='little')
        bv1.reverse()  # shifted first
        # bv = bv1+bv0
        return bv1 + bv0

    @contextmanager
    def restored(self, name=None):
        name = self.create_restore_point(name=name)
        try:
            yield
        finally:
            self.restore(name=name)

    def create_restore_point(self, name=None):
        '''Creating a configuration restore point.

        Parameters
        ----------
        name : str
            Name of the restore point. If not given, a md5 hash will be generated.
        '''
        if name is None:
            for i in iter(int, 1):
                name = datetime.datetime.utcnow().strftime('%Y%m%d%H%M%S%f') + '_' + str(i)
                try:
                    self.config_state[name]
                except KeyError:
                    break
                else:
                    pass
        if name in self.config_state:
            raise ValueError('Restore point %s already exists' % name)
        self.config_state[name] = (copy.deepcopy(self.global_registers), copy.deepcopy(self.pixel_registers))
        return name

    def restore(self, name=None, keep=False, last=True, global_register=True, pixel_register=True):
        '''Restoring a configuration restore point.

        Parameters
        ----------
        name : str
            Name of the restore point. If not given, a md5 hash will be generated.
        keep : bool
            Keeping restore point for later use.
        last : bool
            If name is not given, the latest restore point will be taken.
        global_register : bool
            Restore global register.
        pixel_register : bool
            Restore pixel register.
        '''
        if name is None:
            if keep:
                name = next(reversed(self.config_state)) if last else next(iter(self.config_state))
                value = self.config_state[name]
            else:
                name, value = self.config_state.popitem(last=last)
        else:
            value = self.config_state[name]
            if not keep:
                value = copy.deepcopy(value)  # make a copy before deleting object
                del self.config_state[name]

        if global_register:
            self.global_registers = copy.deepcopy(value[0])
        if pixel_register:
            self.pixel_registers = copy.deepcopy(value[1])

    def clear_restore_points(self, name=None):
        '''Deleting all/a configuration restore points/point.

        Parameters
        ----------
        name : str
            Name of the restore point to be deleted. If not given, all restore points will be deleted.
        '''
        if name is None:
            self.config_state.clear()
        else:
            del self.config_state[name]

    @property
    def can_restore(self):
        '''Any restore point existing?

        Parameters
        ----------
        none

        Returns
        -------
        True if restore points are existing, else false.
        '''
        if self.config_state:
            return True
        else:
            return False
 class FEI4Register(object):
class BroadcastRegister(FEI4Register):
class NameValue(tb.IsDescription):
    def __init__(self, configuration_file=None, fe_type=None, chip_address=None, broadcast=False):
    def __repr__(self):
    def set_chip_address(self, chip_address, broadcast):
    def init_fe_type(self, fe_type):
    def is_chip_flavor(self, chip_flavor):
    def chip_flavor(self):
    def fei4a(self):
    def fei4b(self):
    def load_configuration(self, configuration_file):
    def save_configuration(self, configuration_file):
    def set_global_register_value(self, name, value):
    def get_global_register_value(self, name):
    def set_pixel_register_value(self, name, value):
    def get_pixel_register_value(self, name):
    def get_commands(self, command_name, **kwargs):
    def build_command(self, command_name, **kwargs):
    def get_global_register_attributes(self, register_attribute, do_sort=True, **kwargs):
    def get_global_register_objects(self, do_sort=None, reverse=False, **kwargs):
    def get_global_register_bitsets(self, register_addresses):  # TOTO: add sorting
    def get_pixel_register_objects(self, do_sort=None, reverse=False, **kwargs):
    def get_pixel_register_bitset(self, register_object, bit_no, dc_no):
    def restored(self, name=None):
    def create_restore_point(self, name=None):
    def restore(self, name=None, keep=False, last=True, global_register=True, pixel_register=True):
    def clear_restore_points(self, name=None):
    def can_restore(self):
    def __init__(self, fe_type=None):
    def get_commands(self, command_name, **kwargs):
def load_configuration_from_text_file(register, configuration_file):
def load_configuration_from_hdf5(register, configuration_file, node=''):
    def load_conf():
def save_configuration_to_text_file(register, configuration_file):
def save_configuration_to_hdf5(register, configuration_file, name=''):
    def save_conf():
def parse_global_config(filename):  # parses the global config text file
def parse_pixel_mask_config(filename):
def write_pixel_mask_config(filename, value):
def parse_pixel_dac_config(filename):
def write_pixel_dac_config(filename, value):
def bitarray_from_value(value, size=None, fmt='Q'):
 class FEI4RegisterUtils(object):

    def __init__(self, dut, register, abort=None):
        self.dut = dut
        self.register = register
        self.command_memory_byte_size = 2048 - 16  # 16 bytes of register data
        self.zero_cmd_length = 1
        self.zero_cmd = self.register.get_commands("zeros", length=self.zero_cmd_length)[0]
        self.zero_cmd_padded = self.zero_cmd.copy()
        self.zero_cmd_padded.fill()
        self.abort = abort

    def add_commands(self, x, y):
        return x + self.zero_cmd + y  # FE needs a zero bits between commands

    def add_byte_padded_commands(self, x, y):
        x_fill = x.copy()
        x_fill.fill()
        y_fill = y.copy()
        y_fill.fill()
        return x_fill + self.zero_cmd_padded + y_fill  # FE needs a zero between commands

    def concatenate_commands(self, commands, byte_padding=False):
        if byte_padding:
            return reduce(self.add_byte_padded_commands, commands)
        else:
            return reduce(self.add_commands, commands)

    def send_commands(self, commands, repeat=1, wait_for_finish=True, concatenate=True, byte_padding=False, clear_memory=False, use_timeout=True):
        if concatenate:
            commands_iter = iter(commands)
            try:
                concatenated_cmd = commands_iter.next()
            except StopIteration:
                logging.warning('No commands to be sent')
            else:
                for command in commands_iter:
                    concatenated_cmd_tmp = self.concatenate_commands((concatenated_cmd, command), byte_padding=byte_padding)
                    if concatenated_cmd_tmp.length() > self.command_memory_byte_size * 8:
                        self.send_command(command=concatenated_cmd, repeat=repeat, wait_for_finish=wait_for_finish, set_length=True, clear_memory=clear_memory, use_timeout=use_timeout)
                        concatenated_cmd = command
                    else:
                        concatenated_cmd = concatenated_cmd_tmp
                # send remaining commands
                self.send_command(command=concatenated_cmd, repeat=repeat, wait_for_finish=wait_for_finish, set_length=True, clear_memory=clear_memory, use_timeout=use_timeout)
        else:
            max_length = 0
            if repeat is not None:
                self.dut['TX']['CMD_REPEAT'] = repeat
            for command in commands:
                max_length = max(command.length(), max_length)
                self.send_command(command=command, repeat=None, wait_for_finish=wait_for_finish, set_length=True, clear_memory=False, use_timeout=use_timeout)
            if clear_memory:
                self.clear_command_memory(length=max_length)

    def send_command(self, command, repeat=1, wait_for_finish=True, set_length=True, clear_memory=False, use_timeout=True):
        if repeat is not None:
            self.dut['TX']['CMD_REPEAT'] = repeat
        # write command into memory
        command_length = self.set_command(command, set_length=set_length)
        # sending command
        self.dut['TX']['START']
        # wait for command to be finished
        if wait_for_finish:
            self.wait_for_command(length=command_length, repeat=repeat, use_timeout=use_timeout)
        # clear command memory
        if clear_memory:
            self.clear_command_memory(length=command_length)

    def clear_command_memory(self, length=None):
        self.set_command(self.register.get_commands("zeros", length=(self.command_memory_byte_size * 8) if length is None else length)[0], set_length=False)

    def set_command(self, command, set_length=True, byte_offset=0):
        command_length = command.length()
        # set command bit length
        if set_length:
            self.dut['TX']['CMD_SIZE'] = command_length
        # set command
        data = bitarray_to_array(command)
        self.dut['TX'].set_data(data=data, addr=byte_offset)
        return command_length

    def wait_for_command(self, length=None, repeat=None, use_timeout=True):
        # for scans using the scan loop, reading length and repeat will decrease processor load by 30 to 50%, but has a marginal influence on scan time
        if length is None:
            length = self.dut['TX']['CMD_SIZE'] - self.dut['TX']['START_SEQUENCE_LENGTH'] - self.dut['TX']['STOP_SEQUENCE_LENGTH']
        if repeat is None:
            repeat = self.dut['TX']['CMD_REPEAT']
        if length and repeat > 1:
            delay = length * 25e-9 * repeat
            if delay <= 0.0:
                delay = None  # no additional delay
        else:
            delay = None
        if use_timeout:
            timeout = 1.0  # minimum timeout threshold
            if delay is not None and delay > 0.0:
                timeout += delay  # adding command delay to timeout
            try:
                msg = "Time out while waiting for sending command becoming ready in %s, module %s. Power cycle or reset readout board!" % (self.dut['TX'].name, self.dut['TX'].__class__.__module__)
                if not self.dut['TX'].wait_for_ready(timeout=timeout, times=None, delay=delay, abort=self.abort) and not self.abort.is_set():
                    raise CmdTimeoutError(msg)
            except RuntimeError:
                raise CmdTimeoutError(msg)
        else:
            if delay is not None and delay > 0.0:
                time.sleep(delay)
            while not self.is_ready:
                pass

    @property
    def is_ready(self):
        return True if self.dut['TX']['READY'] else False

    def set_conf_mode(self):
        commands = []
        commands.extend(self.register.get_commands("ConfMode"))
        self.send_commands(commands)

    def set_run_mode(self):
        commands = []
        commands.extend(self.register.get_commands("RunMode"))
        self.send_commands(commands)

    def global_reset(self):
        '''FEI4 Global Reset

        Special function to do a global reset on FEI4. Sequence of commands has to be like this, otherwise FEI4B will be left in weird state.
        '''
        logging.info('Sending Global Reset')
        commands = []
        commands.extend(self.register.get_commands("ConfMode"))
        commands.extend(self.register.get_commands("GlobalReset"))
        self.send_commands(commands)
        time.sleep(0.1)
        commands = []
        commands.extend(self.register.get_commands("ConfMode"))
#         commands.extend(self.register.get_commands("RunMode"))
        self.send_commands(commands)

    def reset_service_records(self):
        '''Resetting Service Records

        This will reset Service Record counters. This will also bring back alive some FE where the output FIFO is stuck (no data is coming out in run mode).
        This should be only issued after power up and in the case of a stuck FIFO, otherwise the BCID counter starts jumping.
        '''
        logging.info('Resetting Service Records')
        commands = []
        commands.extend(self.register.get_commands("ConfMode"))
        self.register.set_global_register_value('ReadErrorReq', 1)
        commands.extend(self.register.get_commands("WrRegister", name=['ReadErrorReq']))
        commands.extend(self.register.get_commands("GlobalPulse", Width=0))
        self.register.set_global_register_value('ReadErrorReq', 0)
        commands.extend(self.register.get_commands("WrRegister", name=['ReadErrorReq']))
        commands.extend(self.register.get_commands("RunMode"))
        commands.extend(self.register.get_commands("ConfMode"))
        self.send_commands(commands)

    def reset_bunch_counter(self):
        '''Resetting Bunch Counter
        '''
        logging.info('Resetting Bunch Counter')
        commands = []
        commands.extend(self.register.get_commands("RunMode"))
        commands.extend(self.register.get_commands("BCR"))
        self.send_commands(commands)
        time.sleep(0.1)
        commands = []
        commands.extend(self.register.get_commands("ConfMode"))
        self.send_commands(commands)

    def reset_event_counter(self):
        '''Resetting Event Counter
        '''
        logging.info('Resetting Event Counter')
        commands = []
        commands.extend(self.register.get_commands("RunMode"))
        commands.extend(self.register.get_commands("ECR"))  # wait some time after ECR
        self.send_commands(commands)
        time.sleep(0.1)
        commands = []
        commands.extend(self.register.get_commands("ConfMode"))
        self.send_commands(commands)

    def configure_all(self, same_mask_for_all_dc=False):
        self.configure_global()
        self.configure_pixel(same_mask_for_all_dc=same_mask_for_all_dc)

    def configure_global(self):
        logging.info('Sending global configuration to FE')
        commands = []
        commands.extend(self.register.get_commands("ConfMode"))
        commands.extend(self.register.get_commands("WrRegister", readonly=False))
#         commands.extend(self.register.get_commands("RunMode"))
        self.send_commands(commands)

    def configure_pixel(self, same_mask_for_all_dc=False):
        logging.info('Sending pixel configuration to FE')
        commands = []
        commands.extend(self.register.get_commands("ConfMode"))
        commands.extend(self.register.get_commands("WrFrontEnd", same_mask_for_all_dc=same_mask_for_all_dc, name=["TDAC", "FDAC", "Imon", "Enable", "C_High", "C_Low", "EnableDigInj"]))
#         commands.extend(self.register.get_commands("RunMode"))
        self.send_commands(commands)

    def set_gdac(self, value, send_command=True):
        if self.register.fei4b:
            altf = value & 0xff
            altc = (value >> 7)
            altc &= ~0x01
        else:
            altf = value & 0xff
            altc = (value >> 8)
        self.register.set_global_register_value("Vthin_AltCoarse", altc)  # write high word
        self.register.set_global_register_value("Vthin_AltFine", altf)  # write low word
        if send_command:
            commands = []
            commands.extend(self.register.get_commands("ConfMode"))
            commands.extend(self.register.get_commands("WrRegister", name=["Vthin_AltFine", "Vthin_AltCoarse"]))
            commands.extend(self.register.get_commands("RunMode"))
            self.send_commands(commands)
            logging.info("Writing GDAC %d (VthinAltCoarse / VthinAltFine = %d / %d)", value, altc, altf)
        else:
            logging.debug("Setting GDAC to %d (VthinAltCoarse / VthinAltFine = %d / %d)", value, altc, altf)

    def get_gdac(self, altc=None, altf=None):
        if altc is None:
            altc = self.register.get_global_register_value("Vthin_AltCoarse")
        if altf is None:
            altf = self.register.get_global_register_value("Vthin_AltFine")
        if self.register.fei4b:
            value = altf & 0xff
            altc &= ~0x01
            value += (altc << 7)
            return value
        else:
            value = altf & 0xff
            value += (altc << 8)
            return value
 def is_fe_ready(self):
    '''Get FEI4 status of module.

    If FEI4 is not ready, resetting service records is necessary to bring the FEI4 to a defined state.

    Returns
    -------
    value : bool
        True if FEI4 is ready, False if the FEI4 was powered up recently and is not ready.
    '''
    with self.readout(fill_buffer=True, callback=None, errback=None):
        commands = []
        commands.extend(self.register.get_commands("ConfMode"))
        commands.extend(self.register.get_commands("RdRegister", address=[1]))
#         commands.extend(self.register.get_commands("RunMode"))
        self.register_utils.send_commands(commands)
    data = self.read_data()

    if len(data) != 0:
        return True if FEI4Record(data[-1], self.register.chip_flavor) == 'VR' else False
    else:
        return False
 class FifoReadout(object):
    def __init__(self, dut):
        self.dut = dut
        self.is_running_lock = Lock()
        self.data_words_per_second_lock = Lock()
        self.callback = None
        self.errback = None
        self.readout_thread = None
        self.worker_thread = None
        self.writer_threads = None
        self.watchdog_thread = None
        self.fifos = []
        self.fifo_condition = []
        self.fill_buffer = False
        self.filter_func = [None]
        self.converter_func = [None]
        self.fifo_select = [None]
        self.enabled_fe_channels = None
        self.readout_interval = 0.05  # in seconds
        self.write_interval = 1.0  # in seconds
        self.watchdog_interval = 1.0  # in seconds
        self._moving_average_time_period = 10.0  # in seconds
        self._n_empty_reads = 3  # number of empty reads before stopping FIFO readout
        self._fifo_data_deque = None
        self._fifo_conditions = None
        self._data_deque = None  # stores data for writer thread
        self._data_conditions = None
        self._data_buffer = None  # stores data for later readout
        self._data_deque = None
        self._words_per_read = []
        self.stop_readout = Event()
        self.force_stop = None
        self.timestamp = None
        self._is_running = False

    @property
    def is_running(self):
        with self.is_running_lock:
            return self._is_running

    def data_words_per_second(self):
        with self.data_words_per_second_lock:
            result = []
            curr_time = get_float_time()
            for words_per_read in self._words_per_read:
                result.append(sum([item[0] for item in words_per_read if item[1] > (curr_time - self._moving_average_time_period)]) / float(self._moving_average_time_period))
            return result

    def start(self, fifos, callback=None, errback=None, reset_rx=False, reset_fifo=False, fill_buffer=False, no_data_timeout=None, filter_func=None, converter_func=None, fifo_select=None, enabled_fe_channels=None):
        with self.is_running_lock:
            if self._is_running:
                raise RuntimeError('FIFO readout threads already started: use stop()')
            self._is_running = True

            if isinstance(fifos, basestring):
                fifos = [fifos]
            if len(fifos) == 0:
                raise ValueError('"fifos" parameter is empty.')
            if len(set(fifos)) != len(fifos):
                raise ValueError('The following strings are occurring multiple times in "fifos": %s' % set([fifo for fifo in fifos if fifos.count(fifo) > 1]))
            if isinstance(fifo_select, Iterable) and set(fifo_select) - set(fifos):
                raise ValueError("The following FIFOs have filters/converters set but are not read out: %s" % (set(fifo_select) - set(fifos)))
            if isinstance(filter_func, Iterable) or isinstance(converter_func, Iterable) or (isinstance(fifo_select, Iterable) and not isinstance(fifo_select, basestring)):
                if not isinstance(filter_func, Iterable):
                    raise ValueError('"filter_func" is not iterable.')
                if not isinstance(converter_func, Iterable):
                    raise ValueError('"converter_func" is not iterable.')
                if not isinstance(fifo_select, Iterable):
                    raise ValueError('"fifo_select" is not iterable.')
                if len(filter_func) != len(converter_func):
                    raise ValueError('Length of "filter_func" and "converter_func" not equal.')
                if len(filter_func) != len(fifo_select):
                    raise ValueError('Length of "filter_func" and "fifo_select" not equal.')
                if len(converter_func) != len(fifo_select):
                    raise ValueError('Length of "converter_func" and "fifo_select" not equal.')
            else:
                if isinstance(fifo_select, basestring):
                    # convert to iterable
                    filter_func = [filter_func]
                    converter_func = [converter_func]
                    fifo_select = [fifo_select]
                else:
                    # if fifo_select is None:
                    # adding filters and converters for each FIFO
                    filter_func = [filter_func] * len(fifos)
                    converter_func = [converter_func] * len(fifos)
                    fifo_select = fifos
            if not (set(fifos) & set(fifo_select)) == set(fifo_select):
                raise ValueError('"fifo_select" contains non-existing FIFOs: %s' % (set(fifo_select) & set(fifos)))

            if enabled_fe_channels is None:
                self.enabled_fe_channels = [rx.name for rx in self.dut.get_modules('fei4_rx')]
            else:
                self.enabled_fe_channels = enabled_fe_channels
            self.fifos = fifos
            self.callback = callback
            self.errback = errback
            self.fill_buffer = fill_buffer
            self.filter_func = filter_func
            self.converter_func = converter_func
            self.fifo_select = fifo_select

            self._fifo_data_deque = {fifo: deque() for fifo in self.fifos}
            self._fifo_conditions = {fifo: Condition() for fifo in self.fifos}
            self._data_deque = [deque() for _ in self.filter_func]
            self._data_conditions = [Condition() for _ in self.filter_func]
            self._data_buffer = [deque() for _ in self.filter_func]
            self.force_stop = {fifo: Event() for fifo in self.fifos}
            self.timestamp = {fifo: None for fifo in self.fifos}
            len_deque = int(self._moving_average_time_period / self.readout_interval)
            curr_time = get_float_time()
            self._words_per_read = [deque(iterable=[(0, curr_time, curr_time)] * len_deque, maxlen=len_deque) for _ in self.filter_func]
            if reset_rx:
                self.reset_rx(fe_channels=self.enabled_fe_channels)
            for fifo in self.fifos:
                if reset_fifo:
                    self.reset_fifo([fifo])
                self.update_timestamp(fifo)
                fifo_size = self.get_fifo_size(fifo)
                if fifo_size != 0:
                    logging.warning('%s contains data: FIFO_SIZE = %i', fifo, fifo_size)
            self.stop_readout.clear()
            for event in self.force_stop.values():
                event.clear()
            logging.info('Starting FIFO readout...')
            if self.errback:
                self.watchdog_thread = Thread(target=self.watchdog, name='WatchdogThread')
                self.watchdog_thread.daemon = True
                self.watchdog_thread.start()
            self.readout_threads = []
            self.worker_threads = []
            self.writer_threads = []
            for fifo in self.fifos:
                readout_thread = Thread(target=self.readout, name='ReadoutThread %s' % fifo, kwargs={'fifo': fifo, 'no_data_timeout': no_data_timeout})
                worker_thread = Thread(target=self.worker, name='WorkerThread %s' % fifo, kwargs={'fifo': fifo})
                readout_thread.daemon = True
                worker_thread.daemon = True
                self.readout_threads.append(readout_thread)
                self.worker_threads.append(worker_thread)
            for index, _ in enumerate(self.filter_func):
                writer_thread = Thread(target=self.writer, name='WriterThread %d' % index, kwargs={'index': index, 'no_data_timeout': no_data_timeout})
                writer_thread.daemon = True
                self.writer_threads.append(writer_thread)
            for writer_thread in self.writer_threads:
                writer_thread.start()
            for worker_thread in self.worker_threads:
                worker_thread.start()
            for readout_thread in self.readout_threads:
                self.update_timestamp(fifo)
                readout_thread.start()
            # enabling RX channels
            for fifo in self.fifos:
                self.update_timestamp(fifo)
            for fei4_rx_name in self.enabled_fe_channels:
                self.dut[fei4_rx_name].ENABLE_RX = 1

    def stop(self, timeout=10.0):
        with self.is_running_lock:
            if not self._is_running:
                raise RuntimeError('FIFO readout threads not running: use start()')
            self._is_running = False
            self.stop_readout.set()

            def wait_for_thread_timeout(thread, fifo, timeout):
                try:
                    thread.join(timeout=timeout)
                    if thread.is_alive():
                        raise StopTimeout('Stopping %s readout thread timed out after %0.1fs' % (fifo, timeout))
                except StopTimeout, e:
                    self.force_stop[fifo].set()
                    if self.errback:
                        self.errback(sys.exc_info())
                    else:
                        logging.error(e)

            join_threads = []
            for i, fifo in enumerate(self.fifos):
                join_thread = Thread(target=wait_for_thread_timeout, kwargs={'thread': self.readout_threads[i], 'fifo': fifo, 'timeout': timeout})
                join_thread.daemon = True
                join_thread.start()
                join_threads.append(join_thread)
            for join_thread in join_threads:
                if join_thread.is_alive():
                    join_thread.join()
            for readout_thread in self.readout_threads:
                if readout_thread.is_alive():
                    readout_thread.join()
            self.readout_threads = []
            for worker_thread in self.worker_threads:
                worker_thread.join()
            self.worker_threads = []
            for writer_thread in self.writer_threads:
                writer_thread.join()
            self.writer_threads = []
            if self.errback:
                self.watchdog_thread.join()
                self.watchdog_thread = None
            # disabling FEI4 RX channels
            for fei4_rx_name in self.enabled_fe_channels:
                self.dut[fei4_rx_name].ENABLE_RX = 0
            self.callback = None
            self.errback = None
            logging.info('Stopped FIFO readout')

    def print_readout_status(self):
        self.print_fifo_status()
        self.print_fei4_rx_status()

    def print_fifo_status(self):
        fifo_sizes = [self.get_fifo_size(fifo) for fifo in self.fifos]
        fifo_queue_sizes = [len(self._fifo_data_deque[fifo]) for fifo in self.fifos]
        max_len = [max(max(len(repr(fifo_sizes[i])), len(repr(fifo_queue_sizes[i]))), len(fifo)) for i, fifo in enumerate(self.fifos)]
        logging.info('FIFO:            %s', " | ".join([fifo.rjust(max_len[index]) for index, fifo in enumerate(self.fifos)]))
        logging.info('FIFO size:       %s', " | ".join([repr(count).rjust(max_len[index]) for index, count in enumerate(fifo_sizes)]))
        logging.info('FIFO queue size: %s', " | ".join([repr(count).rjust(max_len[index]) for index, count in enumerate(fifo_queue_sizes)]))

    def print_fei4_rx_status(self):
        # FEI4
        enable_status = self.get_rx_enable_status()
        sync_status = self.get_rx_sync_status()
        discard_count = self.get_rx_fifo_discard_count()
        error_count = self.get_rx_8b10b_error_count()
        fei4_rx_names = [rx.name for rx in self.dut.get_modules('fei4_rx')]
        if fei4_rx_names:
            logging.info('FEI4 RX channel:                  %s', " | ".join([name.rjust(3) for name in fei4_rx_names]))
            logging.info('FEI4 RX enabled:                  %s', " | ".join(["YES".rjust(max(3, len(fei4_rx_names[index]))) if status is True else "NO".rjust(max(3, len(fei4_rx_names[index]))) for index, status in enumerate(enable_status)]))
            logging.info('FEI4 RX sync:                     %s', " | ".join(["YES".rjust(max(3, len(fei4_rx_names[index]))) if status is True else "NO".rjust(max(3, len(fei4_rx_names[index]))) for index, status in enumerate(sync_status)]))
            logging.info('FEI4 RX FIFO discard counter:     %s', " | ".join([repr(count).rjust(max(3, len(fei4_rx_names[index]))) for index, count in enumerate(discard_count)]))
            logging.info('FEI4 RX FIFO 8b10b error counter: %s', " | ".join([repr(count).rjust(max(3, len(fei4_rx_names[index]))) for index, count in enumerate(error_count)]))
        if not any(sync_status) or any(discard_count) or any(error_count):
            logging.warning('FEI4 RX errors detected')

    def readout(self, fifo, no_data_timeout=None):
        '''Readout thread continuously reading FIFO.

        Readout thread, which uses read_raw_data_from_fifo() and appends data to self._fifo_data_deque (collection.deque).
        '''
        logging.info('Starting readout thread for %s', fifo)
        time_last_data = time()
        time_wait = 0.0
        empty_reads = 0
        while not self.force_stop[fifo].wait(time_wait if time_wait >= 0.0 else 0.0):
            time_read = time()
            try:
                if no_data_timeout and time_last_data + no_data_timeout < get_float_time():
                    raise NoDataTimeout('Received no data for %0.1f second(s) from %s' % (no_data_timeout, fifo))
                raw_data = self.read_raw_data_from_fifo(fifo)
            except NoDataTimeout:
                no_data_timeout = None  # raise exception only once
                if self.errback:
                    self.errback(sys.exc_info())
                else:
                    raise
            except Exception:
                if self.errback:
                    self.errback(sys.exc_info())
                else:
                    raise
                if self.stop_readout.is_set():  # in case of a exception, break immediately
                    break
            else:
                n_data_words = raw_data.shape[0]
                if n_data_words > 0:
                    time_last_data = time()
                    empty_reads = 0
                    time_start_read, time_stop_read = self.update_timestamp(fifo)
                    status = 0
                    self._fifo_data_deque[fifo].append((raw_data, time_start_read, time_stop_read, status))
                    with self._fifo_conditions[fifo]:
                        self._fifo_conditions[fifo].notify_all()
                elif self.stop_readout.is_set():
                    if empty_reads == self._n_empty_reads:
                        break
                    else:
                        empty_reads += 1
            finally:
                # ensure that the readout interval does not depend on the processing time of the data
                # and stays more or less constant over time
                time_wait = self.readout_interval - (time() - time_read)
        self._fifo_data_deque[fifo].append(None)  # last item, None will stop worker
        with self._fifo_conditions[fifo]:
            self._fifo_conditions[fifo].notify_all()
        logging.info('Stopping readout thread for %s', fifo)

    def worker(self, fifo):
        '''Worker thread continuously filtering and converting data when data becomes available.
        '''
        logging.debug('Starting worker thread for %s', fifo)
        self._fifo_conditions[fifo].acquire()
        while True:
            try:
                data_tuple = self._fifo_data_deque[fifo].popleft()
            except IndexError:
                self._fifo_conditions[fifo].wait(self.readout_interval)  # sleep a little bit, reducing CPU usage
            else:
                if data_tuple is None:  # if None then exit
                    break
                else:
                    for index, (filter_func, converter_func, fifo_select) in enumerate(izip(self.filter_func, self.converter_func, self.fifo_select)):
                        if fifo_select is None or fifo_select == fifo:
                            # filter and do the conversion
                            converted_data_tuple = convert_data_iterable((data_tuple,), filter_func=filter_func, converter_func=converter_func)[0]
                            n_data_words = converted_data_tuple[0].shape[0]
                            with self.data_words_per_second_lock:
                                self._words_per_read[index].append((n_data_words, converted_data_tuple[1], converted_data_tuple[2]))
                            self._data_deque[index].append(converted_data_tuple)
                            with self._data_conditions[index]:
                                self._data_conditions[index].notify_all()
        for index, fifo_select in enumerate(self.fifo_select):
            if fifo_select is None or fifo_select == fifo:
                self._data_deque[index].append(None)
                with self._data_conditions[index]:
                    self._data_conditions[index].notify_all()
        self._fifo_conditions[fifo].release()
        logging.debug('Stopping worker thread for %s', fifo)

    def writer(self, index, no_data_timeout=None):
        '''Writer thread continuously calling callback function for writing data when data becomes available.
        '''
        is_fe_data_header = logical_and(is_fe_word, is_data_header)
        logging.debug('Starting writer thread with index %d', index)
        self._data_conditions[index].acquire()
        time_last_data = time()
        time_write = time()
        converted_data_tuple_list = [None] * len(self.filter_func)  # callback function gets a list of lists of tuples
        while True:
            try:
                if no_data_timeout and time_last_data + no_data_timeout < time():
                    raise NoDataTimeout('Received no data for %0.1f second(s) for writer thread with index %d' % (no_data_timeout, index))
                converted_data_tuple = self._data_deque[index].popleft()
            except NoDataTimeout:  # no data timeout
                no_data_timeout = None  # raise exception only once
                if self.errback:
                    self.errback(sys.exc_info())
                else:
                    raise
            except IndexError:  # no data in queue
                self._data_conditions[index].wait(self.readout_interval)  # sleep a little bit, reducing CPU usage
            else:
                if converted_data_tuple is None:  # if None then write and exit
                    if self.callback and any(converted_data_tuple_list):
                        try:
                            self.callback(converted_data_tuple_list)
                        except Exception:
                            self.errback(sys.exc_info())
                    break
                else:
                    if no_data_timeout and np.any(is_fe_data_header(converted_data_tuple[0])):  # check for FEI4 data words
                        time_last_data = time()
                    if converted_data_tuple_list[index]:
                        converted_data_tuple_list[index].append(converted_data_tuple)
                    else:
                        converted_data_tuple_list[index] = [converted_data_tuple]  # adding iterable
                    if self.fill_buffer:
                        self._data_buffer[index].append(converted_data_tuple)
            # check if calling the callback function is about time
            if self.callback and any(converted_data_tuple_list) and ((self.write_interval and time() - time_write >= self.write_interval) or not self.write_interval):
                try:
                    self.callback(converted_data_tuple_list)  # callback function gets a list of lists of tuples
                except Exception:
                    self.errback(sys.exc_info())
                else:
                    converted_data_tuple_list = [None] * len(self.filter_func)
                    time_write = time()  # update last write timestamp
        self._data_conditions[index].release()
        logging.debug('Stopping writer thread with index %d', index)

    def watchdog(self):
        logging.debug('Starting %s', self.watchdog_thread.name)
        time_wait = 0.0
        while not self.stop_readout.wait(time_wait if time_wait >= 0.0 else 0.0):
            time_read = time()
            try:
                sync_status = self.get_rx_sync_status(channels=self.enabled_fe_channels)
                if not all(sync_status):
                    raise RxSyncError('FEI4 RX sync error in RX: %s' % ', '.join([self.enabled_fe_channels[index] for (index, status) in enumerate(sync_status) if not status]))
                error_count = self.get_rx_8b10b_error_count(channels=self.enabled_fe_channels)
                if any(error_count):
                    raise EightbTenbError('FEI4 RX 8b10b error(s) detected in RX: %s' % ', '.join([self.enabled_fe_channels[index] for (index, status) in enumerate(error_count) if status]))
                discard_count = self.get_rx_fifo_discard_count(channels=self.enabled_fe_channels)
                if any(discard_count):
                    raise FifoError('FEI4 RX FIFO discard error(s) detected in RX: %s' % ', '.join([self.enabled_fe_channels[index] for (index, status) in enumerate(discard_count) if status]))
            except Exception:
                self.errback(sys.exc_info())
            time_wait = self.watchdog_interval - (time() - time_read)
        logging.debug('Stopping %s', self.watchdog_thread.name)

    def get_data_from_buffer(self, filter_func=None, converter_func=None):
        '''Reads local data buffer and returns data and meta data list.

        Returns
        -------
        data : list
            List of data and meta data dicts.
        '''
        if self._is_running:
            raise RuntimeError('Readout thread running')
        if not self.fill_buffer:
            logging.warning('Data buffer is not activated')
        return [convert_data_iterable(data_iterable, filter_func=filter_func, converter_func=converter_func) for data_iterable in self._data_buffer]

    def get_raw_data_from_buffer(self, filter_func=None, converter_func=None):
        '''Reads local data buffer and returns raw data array.

        Returns
        -------
        data : np.array
            An array containing data words from the local data buffer.
        '''
        if self._is_running:
            raise RuntimeError('Readout thread running')
        if not self.fill_buffer:
            logging.warning('Data buffer is not activated')
        return [convert_data_array(data_array_from_data_iterable(data_iterable), filter_func=filter_func, converter_func=converter_func) for data_iterable in self._data_buffer]

    def read_raw_data_from_fifo(self, fifo, filter_func=None, converter_func=None):
        '''Reads FIFO data and returns raw data array.

        Returns
        -------
        data : np.array
            An array containing FIFO data words.
        '''
        return convert_data_array(self.dut[fifo].get_data(), filter_func=filter_func, converter_func=converter_func)

    def update_timestamp(self, fifo):
        curr_time = get_float_time()
        last_time = self.timestamp[fifo]
        if last_time is None:
            last_time = curr_time
        self.timestamp[fifo] = curr_time
        return last_time, curr_time

    def get_fifo_size(self, fifo):
        return self.dut[fifo]['FIFO_SIZE']

    def reset_rx(self, fe_channels=None):
        logging.info('Resetting RX')
        if fe_channels is None:
            fe_channels = [rx.name for rx in self.dut.get_modules('fei4_rx')]
        for fei4_rx_name in fe_channels:
            self.dut[fei4_rx_name].RX_RESET

    def reset_fifo(self, fifos):
        if isinstance(fifos, basestring):
            fifos = [fifos]
        for fifo in fifos:
            fifo_size = self.dut[fifo]['FIFO_SIZE']
            logging.info('Resetting %s: size = %i', fifo, fifo_size)
            self.dut[fifo]['RESET']
            # sleep for a while, if it is a hardware FIFO
            if not isinstance(self.dut[fifo], (sitcp_fifo.sitcp_fifo,)):
                sleep(0.2)
            fifo_size = self.dut[fifo]['FIFO_SIZE']
            if fifo_size != 0:
                logging.warning('%s not empty after reset: size = %i', fifo, fifo_size)

    def get_rx_enable_status(self, channels=None):
        if channels:
            return map(lambda channel: True if (self.dut[channel].ENABLE_RX or channel in self.enabled_fe_channels) else False, channels)
        else:
            return map(lambda channel: True if (channel.ENABLE_RX or channel.name in self.enabled_fe_channels) else False, self.dut.get_modules('fei4_rx'))

    def get_rx_sync_status(self, channels=None):
        if channels is None:
            return map(lambda channel: True if channel.READY else False, self.dut.get_modules('fei4_rx'))
        else:
            return map(lambda channel: True if self.dut[channel].READY else False, channels)

    def get_rx_8b10b_error_count(self, channels=None):
        if channels is None:
            return map(lambda channel: channel.DECODER_ERROR_COUNTER, self.dut.get_modules('fei4_rx'))
        else:
            return map(lambda channel: self.dut[channel].DECODER_ERROR_COUNTER, channels)

    def get_rx_fifo_discard_count(self, channels=None):
        if channels is None:
            return map(lambda channel: channel.LOST_DATA_COUNTER, self.dut.get_modules('fei4_rx'))
        else:
            return map(lambda channel: self.dut[channel].LOST_DATA_COUNTER, channels)
 class RxSyncError(Exception):
    pass
 class EightbTenbError(Exception):
    pass
 class FifoError(Exception):
    pass
 class NoDataTimeout(Exception):
    pass
 class StopTimeout(Exception):
    pass
 def save_configuration_dict(h5_file, configuation_name, configuration, **kwargs):
    '''Stores any configuration dictionary to HDF5 file.

    Parameters
    ----------
    h5_file : string, file
        Filename of the HDF5 configuration file or file object.
    configuation_name : str
        Configuration name. Will be used for table name.
    configuration : dict
        Configuration dictionary.
    '''
    def save_conf():
        try:
            h5_file.remove_node(h5_file.root.configuration, name=configuation_name)
        except tb.NodeError:
            pass
        try:
            configuration_group = h5_file.create_group(h5_file.root, "configuration")
        except tb.NodeError:
            configuration_group = h5_file.root.configuration

        scan_param_table = h5_file.create_table(configuration_group, name=configuation_name, description=NameValue, title=configuation_name)
        row_scan_param = scan_param_table.row
        for key, value in dict.iteritems(configuration):
            row_scan_param['name'] = key
            row_scan_param['value'] = str(value)
            row_scan_param.append()
        scan_param_table.flush()

    if isinstance(h5_file, tb.file.File):
        save_conf()
    else:
        if os.path.splitext(h5_file)[1].strip().lower() != ".h5":
            h5_file = os.path.splitext(h5_file)[0] + ".h5"
        with tb.open_file(h5_file, mode="a", title='', **kwargs) as h5_file:
            save_conf()
 def open_raw_data_file(filename, mode="w", title="", scan_parameters=None, socket_address=None):
    '''Mimics pytables.open_file() and stores the configuration and run configuration

    Returns:
    RawDataFile Object

    Examples:
    with open_raw_data_file(filename = self.scan_data_filename, title=self.scan_id, scan_parameters=[scan_parameter]) as raw_data_file:
        # do something here
        raw_data_file.append(self.readout.data, scan_parameters={scan_parameter:scan_parameter_value})
    '''
    return RawDataFile(filename=filename, mode=mode, title=title, scan_parameters=scan_parameters, socket_address=socket_address) def send_meta_data(socket, conf, name):
    '''Sends the config via ZeroMQ to a specified socket. Is called at the beginning of a run and when the config changes. Conf can be any config dictionary.
    '''
    meta_data = dict(
        name=name,
        conf=conf
    )
    try:
        socket.send_json(meta_data, flags=zmq.NOBLOCK)
    except zmq.Again:
        pass class AnalysisError(Exception):

    """Base class for exceptions in this module.
    """ def logical_or(f1, f2):  # function factory
    '''Logical or from functions.

    Parameters
    ----------
    f1, f2 : function
        Function that takes array and returns true or false for each item in array.

    Returns
    -------
    Function.
    '''
    def f(value):
        return np.logical_or(f1(value), f2(value))
    f.__name__ = "(" + f1.__name__ + "_or_" + f2.__name__ + ")"
    return f
 def logical_and(f1, f2):  # function factory
    '''Logical and from functions.

    Parameters
    ----------
    f1, f2 : function
        Function that takes array and returns true or false for each item in array.

    Returns
    -------
    Function.

    Usage:
    filter_func=logical_and(is_data_record, is_data_from_channel(4))  # new filter function
    filter_func(array) # array that has Data Records from channel 4
    '''
    def f(value):
        return np.logical_and(f1(value), f2(value))
    f.__name__ = "(" + f1.__name__ + "_and_" + f2.__name__ + ")"
    return f
 def is_trigger_word(value):
    return np.equal(np.bitwise_and(value, 0x80000000), 0x80000000)
 def is_fe_word(value):
    return np.equal(np.bitwise_and(value, 0xF0000000), 0)
 def is_data_from_channel(channel=4):  # function factory
    '''Selecting FE data from given channel.

    Parameters
    ----------
    channel : int
        Channel number (4 is default channel on Single Chip Card).

    Returns
    -------
    Function.

    Usage:
    1 Selecting FE data from channel 4 (combine with is_fe_word):
        filter_fe_data_from_channel_4 = logical_and(is_fe_word, is_data_from_channel(4))
        fe_data_from_channel_4 = data_array[filter_fe_data_from_channel_4(data_array)]
    2 Sleceting data from channel 4:
        filter_data_from_channel_4 = is_data_from_channel(4)
        data_from_channel_4 = data_array[filter_data_from_channel_4(fe_data_array)]
    3 Sleceting data from channel 4:
        data_from_channel_4 = is_data_from_channel(4)(fe_raw_data)

    Other usage:
    f_ch4 = functoools.partial(is_data_from_channel, channel=4)
    l_ch4 = lambda x: is_data_from_channel(x, channel=4)
    '''
    if channel >= 0 and channel < 16:
        def f(value):
            return np.equal(np.right_shift(np.bitwise_and(value, 0x0F000000), 24), channel)
        f.__name__ = "is_data_from_channel_" + str(channel)  # or use inspect module: inspect.stack()[0][3]
        return f
    else:
        raise ValueError('Invalid channel number')
 def is_tdc_word(value):
    return np.logical_and(np.equal(np.bitwise_and(value, 0x80000000), 0), np.greater(np.bitwise_and(value, 0x70000000), 0))
 def is_tdc_from_channel(channel=4):  # function factory
    '''Selecting TDC data from given channel.

    Parameters
    ----------
    channel : int
        Channel number (4 is default channel on Single Chip Card).

    Returns
    -------
    Function.

    Usage:
    1 Selecting TDC data from channel 4 (combine with is_tdc_word):
        filter_tdc_data_from_channel_4 = logical_and(is_tdc_word, is_tdc_from_channel(4))
        tdc_data_from_channel_4 = data_array[filter_tdc_data_from_channel_4(data_array)]
    '''
    if channel >= 1 and channel < 8:
        def f(value):
            return np.equal(np.right_shift(np.bitwise_and(value, 0x70000000), 28), channel)
        f.__name__ = "is_tdc_from_channel_" + str(channel)  # or use inspect module: inspect.stack()[0][3]
        return f
    else:
        raise ValueError('Invalid channel number')
 def convert_tdc_to_channel(channel):
    ''' Converts TDC words at a given channel to common TDC header (0x4).
    '''
    def f(value):
        filter_func = logical_and(is_tdc_word, is_tdc_from_channel(channel))
        select = filter_func(value)
        value[select] = np.bitwise_and(value[select], 0x0FFFFFFF)
        value[select] = np.bitwise_or(value[select], 0x40000000)
        f.__name__ = "convert_tdc_to_channel_" + str(channel)
        return value
    return f
 def false(value):
    return np.False_

import logging
import time
import os
import itertools
import string
import struct
import smtplib
import abc
import ast
import inspect
import sys
import numpy as np
import basil
from socket import gethostname
from functools import wraps
from threading import Event, Thread, current_thread, Lock, RLock, _MainThread
from Queue import Queue
from collections import namedtuple, Mapping, Iterable
from contextlib import contextmanager
from operator import itemgetter
from contextlib2 import ExitStack
from basil.dut import Dut
from pybar.utils.utils import groupby_dict, dict_compare, zip_nofill, find_file_dir_up
from pybar.run_manager import RunManager, RunBase, RunAborted
from pybar.fei4.register import FEI4Register
from pybar.fei4.register import flavors as fe_flavors
from pybar.fei4.register_utils import FEI4RegisterUtils, is_fe_ready
from pybar.daq.fifo_readout import FifoReadout, RxSyncError, EightbTenbError, FifoError, NoDataTimeout, StopTimeout
from pybar.daq.readout_utils import save_configuration_dict
from pybar.daq.fei4_raw_data import open_raw_data_file, send_meta_data
from pybar.analysis.analysis_utils import AnalysisError
from pybar.daq.readout_utils import logical_or, logical_and, is_trigger_word, is_fe_word, is_data_from_channel, is_tdc_word, is_tdc_from_channel, convert_tdc_to_channel, false
            with self.access_module(module_id=module_id):
                self.register_utils.set_conf_mode()

        # Initial configuration (reset and configuration) of all modules.
        # This is done by iterating over each module individually
        for module_id in self._modules:
            logging.info("Configuring %s..." % module_id)
            with self.access_module(module_id=module_id):
                if self._run_conf['configure_fe']:
                    self.register_utils.global_reset()
                    self.register_utils.configure_all()
                else:
                    self.register_utils.set_conf_mode()
                if is_fe_ready(self):
                    fe_not_ready = False
                else:
                    fe_not_ready = True
                # BCR and ECR might result in RX errors
                # a reset of the RX and FIFO will happen just before scan()
                if self._run_conf['reset_fe']:
                    self.register_utils.reset_bunch_counter()
                    self.register_utils.reset_event_counter()
                if fe_not_ready:
                    # resetting service records must be done once after power up
                    self.register_utils.reset_service_records()
                    if not is_fe_ready(self):
                        logging.warning('Module "%s" is not sending any data.' % module_id)
                # set all modules to conf mode afterwards to be immune to ECR and BCR
                self.register_utils.set_conf_mode()

    def pre_run(self):
        # clear error queue in case run is executed another time
        self.err_queue.queue.clear()
        # set default configuration, revert any changes that were done by last run
        self._set_default_cfg()

        # init DUT
        if not isinstance(self._conf['dut'], Dut):  # Check if already initialized
            module_path = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
            if isinstance(self._conf['dut'], basestring):
                # dirty fix for Windows pathes
                self._conf['dut'] = os.path.normpath(self._conf['dut'].replace('\\', '/'))
                # abs path
                if os.path.isabs(self._conf['dut']):
                    dut = self._conf['dut']
                # working dir, and directorys upwards
                elif find_file_dir_up(filename=self._conf['dut'], path=self._conf['working_dir']):
                    dut = find_file_dir_up(filename=self._conf['dut'], path=self._conf['working_dir'])
                # path of this file
                elif os.path.isfile(os.path.join(module_path, self._conf['dut'])):
                    dut = os.path.join(module_path, self._conf['dut'])
                else:
                    raise ValueError("Parameter 'dut' is not a valid path: %s" % self._conf['dut'])
                logging.info('Loading DUT configuration from file %s', os.path.abspath(dut))
            else:
                dut = self._conf['dut']
            dut = Dut(dut)

            # only initialize when DUT was not initialized before
            if 'dut_configuration' in self._conf and self._conf['dut_configuration']:
                if isinstance(self._conf['dut_configuration'], basestring):
                    # dirty fix for Windows pathes
                    self._conf['dut_configuration'] = os.path.normpath(self._conf['dut_configuration'].replace('\\', '/'))
                    # abs path
                    if os.path.isabs(self._conf['dut_configuration']):
                        dut_configuration = self._conf['dut_configuration']
                    # working dir, and directorys upwards
                    elif find_file_dir_up(filename=self._conf['dut_configuration'], path=self._conf['working_dir']):
                        dut_configuration = find_file_dir_up(filename=self._conf['dut_configuration'], path=self._conf['working_dir'])
                    # path of this file
                    elif os.path.isfile(os.path.join(module_path, self._conf['dut_configuration'])):
                        dut_configuration = os.path.join(module_path, self._conf['dut_configuration'])
                    else:
                        raise ValueError("Parameter 'dut_configuration' is not a valid path: %s" % self._conf['dut_configuration'])
                    logging.info('Loading DUT initialization parameters from file %s', os.path.abspath(dut_configuration))
                    # convert to dict
                    dut_configuration = RunManager.open_conf(dut_configuration)
                    # change bit file path
                    for drv in dut_configuration.iterkeys():
                        if 'bit_file' in dut_configuration[drv] and dut_configuration[drv]['bit_file']:
                            dut_configuration[drv]['bit_file'] = os.path.normpath(dut_configuration[drv]['bit_file'].replace('\\', '/'))
                            # abs path
                            if os.path.isabs(dut_configuration[drv]['bit_file']):
                                pass
                            # working dir, and directorys upwards
                            elif find_file_dir_up(filename=dut_configuration[drv]['bit_file'], path=self._conf['working_dir']):
                                dut_configuration[drv]['bit_file'] = find_file_dir_up(filename=dut_configuration[drv]['bit_file'], path=self._conf['working_dir'])
                            # path of this file
                            elif os.path.isfile(os.path.join(module_path, dut_configuration[drv]['bit_file'])):
                                dut_configuration[drv]['bit_file'] = os.path.join(module_path, dut_configuration[drv]['bit_file'])
                            else:
                                raise ValueError("Parameter 'bit_file' is not a valid path: %s" % dut_configuration[drv]['bit_file'])
                else:
                    dut_configuration = self._conf['dut_configuration']
            else:
                dut_configuration = None
            # check for existence of reserved driver names
            found_reserved_names = []
            for driver_name in _reserved_driver_names:
                try:
                    dut[driver_name]
                    found_reserved_names.append(driver_name)
                except KeyError:
                    pass
            if found_reserved_names:
                raise RuntimeError("The basil DUT contains reserved driver names: %s" % ", ".join(found_reserved_names))
            logging.info('Initializing basil...')
            dut.init(dut_configuration)
            # assign dut after init in case of exceptions during init
            self._conf['dut'] = dut
            # adding DUT handles
            for module_id, module_cfg in self._module_cfgs.items():
                self._module_dut[module_id] = DutHandle(dut=self._conf['dut'], module_cfg=module_cfg)
            self.fifo_readout = FifoReadout(dut=self._conf['dut'])
            # additional init of the DUT
            self.init_dut()
        else:
            # adding DUT handles
            for module_id, module_cfg in self._module_cfgs.items():
                self._module_dut[module_id] = DutHandle(dut=self._conf['dut'], module_cfg=module_cfg)

            self.fifo_readout = FifoReadout(dut=self._conf['dut'])