class LoggingManager(object):
    """ Manager taking care of all logging related issues.

    :param trajectory: Trajectory container of Mock
    :param log_config: Logging configuration

        Can be a a full name of an `ini` file.
        An already instantiated config parser,
        or a logging dictionary.

    :param log_stdout: If `stdout` should be logged.

    :param report_progress:

        How to report progress.

    """
    def __init__(self, log_config=None, log_stdout=False,
                 report_progress=False):
        self.log_config = log_config
        self._sp_config = None
        self._mp_config = None
        self.log_stdout = log_stdout
        self.report_progress = report_progress
        self._tools = []
        self._null_handler = NullHandler()
        self._format_string = 'PROGRESS: Finished %d/%d runs '
        self._stdout_to_logger = None

        self.env_name = None
        self.traj_name = None
        self.set_name = None
        self.run_name = None
        # Format string for the progressbar

    def extract_replacements(self, trajectory):
        """Extracts the wildcards and file replacements from the `trajectory`"""
        self.env_name = trajectory.v_environment_name
        self.traj_name = trajectory.v_name
        self.set_name =  trajectory.f_wildcard('$set')
        self.run_name = trajectory.f_wildcard('$')

    def __getstate__(self):
        """ConfigParsers are not guaranteed to
        be picklable so we need to remove these."""
        state_dict = self.__dict__.copy()
        if isinstance(state_dict['log_config'], cp.RawConfigParser):
            # Config Parsers are not guaranteed to be picklable
            state_dict['log_config'] = True
        return state_dict

    def show_progress(self, n, total_runs):
        """Displays a progressbar"""
        if self.report_progress:
            percentage, logger_name, log_level = self.report_progress
            if logger_name == 'print':
                logger = 'print'
            else:
                logger = logging.getLogger(logger_name)

            if n == -1:
                # Compute the number of digits and avoid log10(0)
                digits = int(math.log10(total_runs + 0.1)) + 1
                self._format_string = 'PROGRESS: Finished %' + '%d' % digits + 'd/%d runs '

            fmt_string = self._format_string % (n + 1, total_runs) + '%s'
            reprint = log_level == 0
            progressbar(n, total_runs, percentage_step=percentage,
                        logger=logger, log_level=log_level,
                        fmt_string=fmt_string, reprint=reprint)

    def add_null_handler(self):
        """Adds a NullHandler to the root logger.

        This is simply added to avoid warnings that no
        logger has been configured.

        """
        root = logging.getLogger()
        root.addHandler(self._null_handler)

    def remove_null_handler(self):
        """Removes the NullHandler from the root logger."""
        root = logging.getLogger()
        root.removeHandler(self._null_handler)

    @staticmethod
    def tabula_rasa():
        """Removes all loggers and logging handlers. """
        erase_dict = {'disable_existing_loggers': False, 'version': 1}
        dictConfig(erase_dict)

    @staticmethod
    def _check_and_replace_parser_args(parser, section, option, rename_func, make_dirs=True):
        """ Searches for parser settings that define filenames.

        If such settings are found, they are renamed according to the wildcard
        rules. Moreover, it is also tried to create the corresponding folders.

        :param parser:  A config parser
        :param section: A config section
        :param option: The section option
        :param rename_func: A function to rename found files
        :param make_dirs: If the directories of the file should be created.

        """
        args = parser.get(section, option, raw=True)
        strings = get_strings(args)
        replace = False
        for string in strings:
            isfilename = any(x in string for x in FILENAME_INDICATORS)
            if isfilename:
                newstring = rename_func(string)
                if make_dirs:
                    try_make_dirs(newstring)
                # To work with windows path specifications we need this replacement:
                raw_string = string.replace('\\', '\\\\')
                raw_newstring = newstring.replace('\\', '\\\\')
                args = args.replace(raw_string, raw_newstring)
                replace = True
        if replace:
            parser.set(section, option, args)

    @staticmethod
    def _parser_to_string_io(parser):
        """Turns a ConfigParser into a StringIO stream."""
        memory_file = StringIO()
        parser.write(memory_file)
        memory_file.flush()
        memory_file.seek(0)
        return memory_file

    @staticmethod
    def _find_multiproc_options(parser):
        """ Searches for multiprocessing options within a ConfigParser.

        If such options are found, they are copied (without the `'multiproc_'` prefix)
        into a new parser.

        """
        sections = parser.sections()
        if not any(section.startswith('multiproc_') for section in sections):
            return None
        mp_parser = NoInterpolationParser()
        for section in sections:
            if section.startswith('multiproc_'):
                new_section = section.replace('multiproc_', '')
                mp_parser.add_section(new_section)
                options = parser.options(section)
                for option in options:
                    val = parser.get(section, option, raw=True)
                    mp_parser.set(new_section, option, val)
        return mp_parser

    @staticmethod
    def _find_multiproc_dict(dictionary):
        """ Searches for multiprocessing options in a given `dictionary`.

        If found they are copied (without the `'multiproc_'` prefix)
        into a new dictionary

        """
        if not any(key.startswith('multiproc_') for key in dictionary.keys()):
            return None
        mp_dictionary = {}
        for key in dictionary.keys():
            if key.startswith('multiproc_'):
                new_key = key.replace('multiproc_', '')
                mp_dictionary[new_key] = dictionary[key]
        mp_dictionary['version'] = dictionary['version']
        if 'disable_existing_loggers' in dictionary:
            mp_dictionary['disable_existing_loggers'] = dictionary['disable_existing_loggers']
        return mp_dictionary

    def check_log_config(self):
        """ Checks and converts all settings if necessary passed to the Manager.

        Searches for multiprocessing options as well.

        """
        if self.report_progress:
            if self.report_progress is True:
                self.report_progress = (5, 'pypet', logging.INFO)
            elif isinstance(self.report_progress, (int, float)):
                self.report_progress = (self.report_progress, 'pypet', logging.INFO)
            elif isinstance(self.report_progress, str):
                self.report_progress = (5, self.report_progress, logging.INFO)
            elif len(self.report_progress) == 2:
                self.report_progress = (self.report_progress[0], self.report_progress[1],
                                        logging.INFO)

        if self.log_config:
            if self.log_config == pypetconstants.DEFAULT_LOGGING:
                pypet_path = os.path.abspath(os.path.dirname(__file__))
                init_path = os.path.join(pypet_path, 'logging')
                self.log_config = os.path.join(init_path, 'default.ini')

            if isinstance(self.log_config, str):
                if not os.path.isfile(self.log_config):
                    raise ValueError('Could not find the logger init file '
                                     '`%s`.' % self.log_config)
                parser = NoInterpolationParser()
                parser.read(self.log_config)
            elif isinstance(self.log_config, cp.RawConfigParser):
                parser = self.log_config
            else:
                parser = None

            if parser is not None:
                self._sp_config = self._parser_to_string_io(parser)
                self._mp_config = self._find_multiproc_options(parser)
                if self._mp_config is not None:
                    self._mp_config = self._parser_to_string_io(self._mp_config)

            elif isinstance(self.log_config, dict):
                self._sp_config = self.log_config
                self._mp_config = self._find_multiproc_dict(self._sp_config)

        if self.log_stdout:
            if self.log_stdout is True:
                self.log_stdout = ('STDOUT', logging.INFO)
            if isinstance(self.log_stdout, str):
                self.log_stdout = (self.log_stdout, logging.INFO)
            if isinstance(self.log_stdout, int):
                self.log_stdout = ('STDOUT', self.log_stdout)

    def _handle_config_parsing(self, log_config):
        """ Checks for filenames within a config file and translates them.

        Moreover, directories for the files are created as well.

        :param log_config: Config file as a stream (like StringIO)

        """
        parser = NoInterpolationParser()
        parser.readfp(log_config)

        rename_func = lambda string: rename_log_file(string,
                                                     env_name=self.env_name,
                                                     traj_name=self.traj_name,
                                                     set_name=self.set_name,
                                                     run_name=self.run_name)

        sections = parser.sections()
        for section in sections:
            options = parser.options(section)
            for option in options:
                if option == 'args':
                    self._check_and_replace_parser_args(parser, section, option,
                                                        rename_func=rename_func)
        return parser

    def _handle_dict_config(self, log_config):
        """Recursively walks and copies the `log_config` dict and searches for filenames.

        Translates filenames and creates directories if necessary.

        """
        new_dict = dict()
        for key in log_config.keys():
            if key == 'filename':
                filename = log_config[key]
                filename = rename_log_file(filename,
                                           env_name=self.env_name,
                                           traj_name=self.traj_name,
                                           set_name=self.set_name,
                                           run_name=self.run_name)
                new_dict[key] = filename
                try_make_dirs(filename)
            elif isinstance(log_config[key], dict):
                inner_dict = self._handle_dict_config(log_config[key])
                new_dict[key] = inner_dict
            else:
                new_dict[key] = log_config[key]
        return new_dict

    def make_logging_handlers_and_tools(self, multiproc=False):
        """Creates logging handlers and redirects stdout."""

        log_stdout = self.log_stdout
        if sys.stdout is self._stdout_to_logger:
            # If we already redirected stdout we don't neet to redo it again
            log_stdout = False

        if self.log_config:
            if multiproc:
                proc_log_config = self._mp_config
            else:
                proc_log_config = self._sp_config

            if proc_log_config:
                if isinstance(proc_log_config, dict):
                    new_dict = self._handle_dict_config(proc_log_config)
                    dictConfig(new_dict)
                else:
                    parser = self._handle_config_parsing(proc_log_config)
                    memory_file = self._parser_to_string_io(parser)
                    fileConfig(memory_file, disable_existing_loggers=False)

        if log_stdout:
            #  Create a logging mock for stdout
            std_name, std_level = self.log_stdout

            stdout = StdoutToLogger(std_name, log_level=std_level)
            stdout.start()
            self._tools.append(stdout)

    def finalize(self, remove_all_handlers=True):
        """Finalizes the manager, closes and removes all handlers if desired."""
        for tool in self._tools:
            tool.finalize()
        self._tools = []
        self._stdout_to_logger = None
        for config in (self._sp_config, self._mp_config):
            if hasattr(config, 'close'):
                config.close()
        self._sp_config = None
        self._mp_config = None
        if remove_all_handlers:
            self.tabula_rasa() class HasLogger(HasSlots):
    """Abstract super class that automatically adds a logger to a class.

    To add a logger to a sub-class of yours simply call ``myobj._set_logger(name)``.
    If ``name=None`` the logger is chosen as follows:

        ``self._logger = logging.getLogger(self.__class.__.__module__ + '.' + self.__class__.__name__)``

    The logger can be accessed via ``myobj._logger``.

    """

    __slots__ = ('_logger',)

    def __getstate__(self):
        """Called for pickling.

        Removes the logger to allow pickling and returns a copy of `__dict__`.

        """
        state_dict = super(HasLogger, self).__getstate__()
        if '_logger' in state_dict:
            # Pickling does not work with loggers objects,
            # so we just keep the logger's name:
            state_dict['_logger'] = self._logger.name
        return state_dict

    def __setstate__(self, statedict):
        """Called after loading a pickle dump.

        Restores `__dict__` from `statedict` and adds a new logger.

        """
        super(HasLogger, self).__setstate__(statedict)
        if '_logger' in statedict:
            # If we re-instantiate the component the
            # logger attribute only contains a name,
            # so we also need to re-create the logger:
            self._set_logger(statedict['_logger'])

    def _set_logger(self, name=None):
        """Adds a logger with a given `name`.

        If no name is given, name is constructed as
        `type(self).__name__`.

        """
        if name is None:
            cls = self.__class__
            name = '%s.%s' % (cls.__module__, cls.__name__)
        self._logger = logging.getLogger(name) def simple_logging_config(func):
    """Decorator to allow a simple logging configuration.

    This encompasses giving a `log_folder`, `logger_names` as well as `log_levels`.

    """

    @functools.wraps(func)
    def new_func(self, *args, **kwargs):
        if use_simple_logging(kwargs):
            if 'log_config' in kwargs:
                raise ValueError('Please do not specify `log_config` '
                                 'if you want to use the simple '
                                 'way of providing logging configuration '
                                 '(i.e using `log_folder`, `logger_names` and/or `log_levels`).')
            _change_logging_kwargs(kwargs)

        return func(self, *args, **kwargs)

    return new_func class Trajectory(DerivedParameterGroup, ResultGroup, ParameterGroup, ConfigGroup):
    """The trajectory manages results and parameters.

    The trajectory provides all functionality to define how the parameter space
    of your simulation should be explored. During single runs based on a particular
    parameter point, the functionality fo the trajectory is reduced.

    You can add four types of data to the trajectory:

    * Config:

        These are special parameters specifying modalities of how to run your simulations.
        Changing a config parameter should NOT have any influence on the results you
        obtain from your simulations.

        They specify runtime environment parameters like how many CPUs you use for
        multiprocessing etc.

        In fact, if you use the default runtime environment of this project, the environment
        will add some config parameters to your trajectory.

        The method to add more config is :func:`~pypet.naturalnaming.ConfigGroup.f_add_config`

        Config parameters are put into the subtree `traj.config` (with `traj` being your
        trajectory instance).

    * Parameters:

        These are your primary ammunition in numerical simulations. They specify
        how your simulation works. They can only be added before the actual
        running of the simulation exploring the parameter space. They can be
        added via :func:`~pypet.naturalnaming.ParameterGroup.f_add_parameter`
        and be explored using :func:`~pypet.trajectory.Trajectory.f_explore`.
        Or to expand an existing trajectory use :func:`~pypet.trajectory.Trajectory.f_expand`.

        Your parameters should encompass all values that completely define your simulation,
        I recommend also storing random number generator seeds as parameters to
        guarantee that a simulation can be repeated exactly the way it was run the first time.

        Parameters are put into the subtree `traj.parameters`.

    * Derived Parameters:

        They are not much different from parameters except that they can be added anytime.

        Conceptually this encompasses stuff that is intermediately
        computed from the original parameters. For instance, as your original
        parameters you have a random number seed and some other parameters.
        From these you compute a connection matrix for a neural network.
        This connection matrix could be stored as a derived parameter.

        Derived parameters are added via
        :func:`~pypet.naturalnaming.DerivedParameterGroup.f_add_derived_parameter`.

        Derived parameters are put into the subtree `traj.derived_parameters`.
        They are further sorted into  `traj.derived_parameters.runs.run_XXXXXXXX` if they were
        added during a single run. `XXXXXXXX` is replaced by the index of the corresponding run,
        for example `run_00000001`.

    * Results:

        Result are added via the :func:`~pypet.naturalnaming.ResultGroup.f_add_result`.
        They are kept under the subtree `traj.results` and are further sorted into
        `traj.results.runs.run_XXXXXXXX` if they are added during a single run.

    There are several ways to access the parameters and results, to learn about these, fast access,
    and natural naming see :ref:`more-on-access`.


    In case you create a new trajectory you can pass the following arguments:

    :param name:

        Name of the trajectory, if `add_time=True` the current time is added as a string
        to the parameter name.

    :param add_time:

        Boolean whether to add the current time in human readable format to the trajectory name.

    :param comment:

        A useful comment describing the trajectory.

    :param dynamic_imports:

        If you've written a custom parameter that needs to be loaded dynamically during runtime,
        this needs to be specified here as a list of classes or strings naming classes
        and there module paths. For example:
        `dynamic_imports = ['pypet.parameter.PickleParameter',MyCustomParameter]`

        If you only have a single class to import, you do not need the list brackets:
        `dynamic_imports = 'pypet.parameter.PickleParameter'`

    :param wildcard_functions:

        Dictionary of wildcards like `$` and corresponding functions that are called upon
        finding such a wildcard.

    :param storage_service:

        Pass a storage service used by the Trajectory. Alternatively, pass a constructor
        and other ``**kwargs`` are passed onto the constructor.

    :param kwargs:

        Other arguments passed to the storage service constructor

    :raises:

        ValueError: If the name of the trajectory contains invalid characters or
                    not all additional keyword arguments are used.

        TypeError: If the dynamically imported classes are not classes or strings.


    Example usage:

    >>> traj = Trajectory('ExampleTrajectory', dynamic_imports=['Some.custom.class'],\
    comment = 'I am a neat example!', storage_service=HDF5StorageService, \
    filename='experiment.hdf5', file_title='Experiments')

    """

    @kwargs_api_change('dynamically_imported_classes', 'dynamic_imports')
    def __init__(self, name='my_trajectory', add_time=False, comment='',
                 dynamic_imports=None, wildcard_functions=None,
                 storage_service=None, **kwargs):

        copy_traj = kwargs.pop('_copy_traj', False)  # True if trajectory is copied

        # This is true during the actual runs:
        self._is_run = False

        super(Trajectory, self).__init__()

        # Helper variable: During a multiprocessing single run, the trajectory is usually
        # pickled without all the parameter exploration ranges and all run information
        # As a consequence, __len__ would return 1, so we need to store the length in this
        # helper variable to return the correct length during single runs.
        self._length = 1

        if not copy_traj:
            self._set_logger()

            init_time = time.time()
            formatted_time = format_time(init_time)
            self._timestamp = init_time
            self._time = formatted_time

            if add_time:
                self._name = name + '_' + str(formatted_time)
            else:
                self._name = name

        self._version = VERSION
        self._python = '.'.join([str(x) for x in sys.version_info[0:3]])

        self._parameters = {}  # Contains all parameters
        self._derived_parameters = {}  # Contains all derived parameters
        self._results = {}  # Contains all results
        self._explored_parameters = {}  # Contains all explored parameters, even when they are not
        # loaded it still contains all keys!
        self._config = {}  # Contains all config parameters
        self._all_groups = {}  # Contains ALL groups regardless in which subtree they are
        self._run_parent_groups = {}  # Contains all groups which are parents of run groups
        self._new_nodes = OrderedDict()  # Contains all elements added during a
        # particular single run with full names as keys and a (parent, child) pair as value
        self._new_links = OrderedDict()  # Contains all new added links during a single run
        self._other_leaves = {}  # Contains ALL other user added leaves
        self._linked_by = {}  # Contains all nodes that are linked to

        self._changed_default_parameters = {}  # Needed for paremeter presetting

        self._single_run_ids = {}  # A bidrectional dictionary conataining the mapping between
        # a run name and the run index (e.g. `1 <-> 'run_00000001'`), in both directions

        self._run_information = {}  # Nested Dictionary with run names as keys and
        # dictionaries as values. The inner dictionaries contain meta information about the runs
        # like time of creation, whether they have been completed and so on.
        # Check function 'f_get_run_information' for a description

        self._updated_run_information = set() # Set of updated run information which
        # needs to be updated in case the trajectory is stored.

        self._nn_interface = NaturalNamingInterface(root_instance=self)
        self._fast_access = True
        self._shortcuts = True
        self._iter_recursive = False
        self._max_depth = None
        self._auto_load = False
        self._with_links = True

        self._environment_hexsha = None
        self._environment_name = None

        # Index of a trajectory is -1, if the trajectory should behave like a single run
        # and blind out other single run results, this can be changed via 'v_crun'.
        self._idx = -1
        self._crun = None

        self._standard_parameter = Parameter
        self._standard_result = Result
        self._standard_leaf = Result

        self._auto_run_prepend = True
        self._no_clobber = False

        self._run_started = False  # For manually using a trajectory
        self._run_by_environment = False  # To disable manual running of experiment

        self._full_copy = False

        if not copy_traj:
            self._dynamic_imports = ['pypet.parameter.PickleParameter']

            if not dynamic_imports is None:
                self.f_add_to_dynamic_imports(dynamic_imports)

            faulty_names = self._nn_interface._check_names([name])

            if '.' in name:
                faulty_names += ' colons >>.<< are not allowed in trajectory names,'

            if faulty_names:
                raise ValueError('Your Trajectory %s f_contains the following not admissible names: '
                                 '%s please choose other names.'
                                 % (name, faulty_names))

            internal_wildcard_functions = {('$', 'crun'): make_run_name,
                                           ('$set', 'crunset'): make_set_name}
            if wildcard_functions:
                internal_wildcard_functions.update(wildcard_functions)

            self._wildcard_functions = {}
            self._wildcard_cache = {}
            self._wildcard_keys = {}
            self._reversed_wildcards = {} # Only needed for merging

            self.f_add_wildcard_functions(internal_wildcard_functions)

            # Per Definition the length is set to be 1. Even with an empty trajectory
            # in principle you could make a single run
            self._add_run_info(0)
            self._test_run_addition(1)

            self._comment = ''
            self.v_comment = comment

            self._storage_service, unused_kwargs = storage_factory(storage_service=storage_service,
                                                                   trajectory=self, **kwargs)
            if len(unused_kwargs) > 0:
                raise ValueError('The following keyword arguments were not used: `%s`' %
                                 str(unused_kwargs))

    def f_get_wildcards(self):
        """Returns a list of all defined wildcards"""
        return list(self._wildcard_keys.keys())

    def f_is_wildcard(self, wildcard):
        """Checks if a given `wildcard` is really a wildcard."""
        return wildcard in self._wildcard_keys

    def f_add_wildcard_functions(self, func_dict):
        """#TODO"""
        for wildcards, function in func_dict.items():
            if not isinstance(wildcards, tuple):
                wildcards = (wildcards,)
            for wildcard in wildcards:
                if wildcard in self._wildcard_keys:
                    raise ValueError('Your wildcard `%s` is used twice1' % wildcard)
                self._wildcard_keys[wildcard] = wildcards
            self._wildcard_functions[wildcards] = function
            self._logger.debug('Added wildcard function `%s`.' % str(wildcards))

    def f_wildcard(self, wildcard='$', run_idx=None):
        """#TODO"""
        if run_idx is None:
            run_idx = self.v_idx
        wildcards = self._wildcard_keys[wildcard]
        try:
            return self._wildcard_cache[(wildcards, run_idx)]
        except KeyError:
            translation = self._wildcard_functions[wildcards](run_idx)
            self._wildcard_cache[(wildcards, run_idx)] = translation
            return translation

    def __getstate__(self):
        result = super(Trajectory, self).__getstate__()

        # Do not copy run information in case `v_full_copy` is `False`.
        if not self.v_full_copy:
            if self.v_crun is not None:
                idx = self.v_idx
            else:
                idx = 0
            runname = self._single_run_ids[idx]
            result['_run_information'] = {runname: self._run_information[runname]}
            result['_single_run_ids'] = {idx: runname, runname: idx}
            result['_updated_run_information'] = set()

        result['_wildcard_cache'] = {}
        return result

    def __str__(self):
        if self.v_comment:
            commentstring = '`%s`, ' % self.v_comment
        else:
            commentstring = ''

        info_string = '(%slen:%d)'  % (commentstring, len(self))
        children_string = self._get_children_representation()

        if self._is_run:
            runstr = ' (during single run)'
        else:
            runstr = ''

        return '%s%s %s %s: %s' % (self.f_get_class_name(), runstr, self.v_name, info_string,
                                 children_string)

    @property
    def v_version(self):
        """The version of *pypet* that was used to create the trajectory"""
        return self._version

    @property
    def v_python(self):
        """The version of python as a string that was used to create the trajectory"""
        return self._python

    @property
    def v_comment(self):
        """Should be a nice descriptive comment"""
        return self._comment

    @v_comment.setter
    @not_in_run
    def v_comment(self, comment):
        """Sets the comment"""
        comment = str(comment)
        self._comment = comment

    @property
    def v_storage_service(self):
        """The service that can store the trajectory to disk or wherever.

        Default is None or if a filename was provided on construction
        the :class:`~pypet.storageservice.HDF5StorageService`.

        """
        return self._storage_service

    @v_storage_service.setter
    def v_storage_service(self, service):
        """Sets the storage service"""
        self._storage_service = service

    @property
    def v_is_run(self):
        """ ``True`` mak if trajectory is used during a single run initiated by an environment.

        Accordingly, the functionality of the trajectory is reduced.

        """
        return self._is_run

    @property
    def v_idx(self):
        """Index if you want to access the trajectory as during a single run.

        You can turn the trajectory to behave as if during the execution of your runs if you set
        `v_idx` to a particular index. Note that only integer values are appropriate here,
        not names of runs.

        Alternatively instead of directly setting `v_idx` you can call
        :func:`~pypet.trajectory.Trajectory.f_set_crun:`.

        Set to `-1` to make the trajectory turn everything back to default.

        """
        return self._idx

    @v_idx.setter
    @not_in_run
    def v_idx(self, idx):
        """Changes the index to make the trajectory behave as a single run"""
        self.f_set_crun(idx)

    @property
    def v_crun_(self):
        """"
        Similar to ``v_crun`` but returns ``'run_ALL'`` if ``v_crun`` is ``None``.
        """
        return self.v_crun if self.v_crun is not None else self.f_wildcard('$', -1)

    @property
    def v_crun(self):
        """Run name if you want to access the trajectory as a single run.

        You can turn the trajectory to behave as during a single run if you set
        `v_crun` to a particular run name. Note that only string values are appropriate here,
        not indices. Check the `v_idx` property if you want to provide an index.

        Alternatively instead of directly setting `v_crun` you can call
        :func:`~pypet.trajectory.Trajectory.f_set_crun:`.

        Set to `None` to make the trajectory to turn everything back to default.

        """
        return self._crun

    @v_crun.setter
    @not_in_run
    def v_crun(self, run_name):
        """Changes the run name to make the trajectory behave as during a single run"""
        self.f_set_crun(run_name)

    @property
    def v_full_copy(self):
        """Whether trajectory is copied fully during pickling or only the current
        parameter space point.

        Note if the trajectory is copied as a whole, also during a single run
        you can access the full parameter space.

        Changing `v_full_copy` will also change `v_full_copy` of all explored parameters!

        """
        return self._full_copy

    @v_full_copy.setter
    @not_in_run
    def v_full_copy(self, val):
        """ Sets full copy mode of trajectory and (!) ALL explored parameters!"""
        self._full_copy = bool(val)
        for param in self._explored_parameters.values():
            if param is not None:
                param.v_full_copy = bool(val)

    @property
    def v_with_links(self):
        """Whether links should be considered in case using natural naming
        or squared bracket indexing"""
        return self._with_links

    @v_with_links.setter
    def v_with_links(self, val):
        """Sets the usage of links"""
        self._with_links = bool(val)

    def f_set_properties(self, **kwargs):
        """Sets properties like ``v_fast_access``.

        For example: ``traj.f_set_properties(v_fast_access=True, v_auto_load=False)``

        """
        for name in kwargs:
            val = kwargs[name]
            if not name.startswith('v_'):
                name = 'v_' + name
            if not name in self._nn_interface._not_admissible_names:
                raise AttributeError('Cannot set property `%s` does not exist.' % name)
            else:
                setattr(self, name, val)

    @not_in_run
    def f_add_to_dynamic_imports(self, dynamic_imports):
        """Adds classes or paths to classes to the trajectory to create custom parameters.

        :param dynamic_imports:

            If you've written custom parameter that needs to be loaded dynamically during runtime,
            this needs to be specified here as a list of classes or strings naming classes
            and there module paths. For example:
            `dynamic_imports = ['pypet.parameter.PickleParameter',MyCustomParameter]`

            If you only have a single class to import, you do not need the list brackets:
            `dynamic_imports = 'pypet.parameter.PickleParameter'`

        """

        if not isinstance(dynamic_imports, (list, tuple)):
            dynamic_imports = [dynamic_imports]

        for item in dynamic_imports:
            if not (isinstance(item, str) or inspect.isclass(item)):
                raise TypeError('Your dynamic import `%s` is neither a class nor a string.' %
                                str(item))

        self._dynamic_imports.extend(dynamic_imports)

    @not_in_run
    def f_set_crun(self, name_or_idx):
        """Can make the trajectory behave as during a particular single run.

        It allows easier data analysis.

         Has the following effects:

        *
            `v_idx` and `v_crun` are set to the appropriate index and run name

        *
            All explored parameters are set to the corresponding value in the exploration
            ranges, i.e. when you call :func:`~pypet.parameter.Parameter.f_get` (or fast access)
            on them you will get in return the value at the corresponding `v_idx` position
            in the exploration range.

        *
            If you perform a search in the trajectory tree, the trajectory will
            only search the run subtree under *results* and *derived_parameters* with the
            corresponding index.
            For instance, if you use `f_set_crun('run_00000007')` or `f_set_crun(7)`
            and search for `traj.results.z` this will search for `z` only in the subtree
            `traj.results.run_00000007`. Yet, you can still explicitly name other subtrees,
            i.e. `traj.results.run_00000004.z` will still work.

        """
        if (name_or_idx is None or name_or_idx == self.f_wildcard('$', -1) or
                    name_or_idx == -1):
            self.f_restore_default()
        else:
            if isinstance(name_or_idx, str):
                self._idx = self.f_idx_to_run(name_or_idx)
                self._crun = name_or_idx
            else:
                self._crun = self.f_idx_to_run(name_or_idx)
                self._idx = name_or_idx

            self._set_explored_parameters_to_idx(self.v_idx)

    @not_in_run
    def f_iter_runs(self, start=0, stop=None, step=1, yields='name'):
        """Makes the trajectory iterate over all runs.

        :param start: Start index of run

        :param stop: Stop index, leave ``None`` for length of trajectory

        :param step: Stepsize

        :param yields:

            What should be yielded: ``'name'`` of run, ``idx`` of run
            or ``'self'`` to simply return the trajectory container.

            You can also pick ``'copy'`` to get **shallow** copies (ie the tree is copied but
            no leave nodes except explored ones.) of your trajectory,
            might lead to some of overhead.

        Note that after a full iteration, the trajectory is set back to normal.

        Thus, the following code snippet

        ::

            for run_name in traj.f_iter_runs():

                 # Do some stuff here...


        is equivalent to

        ::

            for run_name in traj.f_get_run_names(sort=True):
                traj.f_set_crun(run_name)

                # Do some stuff here...

            traj.f_set_crun(None)


        :return:

            Iterator over runs. The iterator itself will return the run names but modify
            the trajectory in each iteration and set it back do normal in the end.


        """
        if stop is None:
            stop =len(self)
        elif stop > len(self):
            raise ValueError('Stop cannot be larger than the trajectory lenght.')
        yields = yields.lower()
        if yields == 'name':
            yield_func = lambda x: self.f_idx_to_run(x)
        elif yields == 'idx':
            yield_func = lambda x: x
        elif yields == 'self':
            yield_func = lambda x: self
        elif yields == 'copy':
            yield_func = lambda x: self.__copy__()
        else:
            raise ValueError('Please choose yields among: `name`, `idx`, or `self`.')
        for idx in range(start, stop, step):
            self.f_set_crun(idx)
            yield yield_func(idx)

        self.f_set_crun(None)

    @not_in_run
    def f_shrink(self, force=False):
        """ Shrinks the trajectory and removes all exploration ranges from the parameters.
        Only possible if the trajectory has not been stored to disk before or was loaded as new.

        :param force:

            Usually you cannot shrink the trajectory if it has been stored to disk,
            because there's no guarantee that it is actually shrunk if there
            still exist explored parameters on disk. In case you are certain that
            you did not store explored parameters to disk set or you deleted all
            of them from disk set `force=True`.

        :raises: TypeError if the trajectory was stored before.

        """
        if self._stored and not force:
            raise TypeError('Your trajectory is already stored to disk or database, shrinking is '
                            'not allowed.')

        for param in self._explored_parameters.values():
            param.f_unlock()
            try:
                param._shrink()
            except Exception as exc:
                self._logger.error('Could not shrink `%s` because of:`%s`' %
                                   (param.v_full_name, repr(exc)))

        # If we shrink, we do not have any explored parameters left and we can erase all
        # run information, and the length of the trajectory is 1 again.
        self._explored_parameters = {}
        self._run_information = {}
        self._single_run_ids = {}
        self._add_run_info(0)
        self._test_run_addition(1)

    def _preset(self, name, args, kwargs):
        """Generic preset function, marks a parameter or config for presetting."""
        if self.f_contains(name, shortcuts=False):
            raise ValueError('Parameter `%s` is already part of your trajectory, use the normal'
                             'accessing routine to change config.' % name)
        else:
            self._changed_default_parameters[name] = (args, kwargs)

    @not_in_run
    def f_preset_config(self, config_name, *args, **kwargs):
        """Similar to func:`~pypet.trajectory.Trajectory.f_preset_parameter`"""

        if not config_name.startswith('config.'):
            config_name = 'config.' + config_name

        self._preset(config_name, args, kwargs)

    @not_in_run
    def f_preset_parameter(self, param_name, *args, **kwargs):
        """Presets parameter value before a parameter is added.

        Can be called before parameters are added to the Trajectory in order to change the
        values that are stored into the parameter on creation.

        After creation of a parameter, the instance of the parameter is called
        with `param.f_set(*args,**kwargs)` with `*args`, and `**kwargs` provided by the user
        with `f_preset_parameter`.

        Before an experiment is carried out it is checked if all parameters that were
        marked were also preset.

        :param param_name:

            The full name (!) of the parameter that is to be changed after its creation.

        :param args:

            Arguments that will be used for changing the parameter's data

        :param kwargs:

            Keyword arguments that will be used for changing the parameter's data

        Example:

        >>> traj.f_preset_parameter('groupA.param1', data=44)
        >>> traj.f_add_parameter('groupA.param1', data=11)
        >>> traj.parameters.groupA.param1
        44

        """

        if not param_name.startswith('parameters.'):
            param_name = 'parameters.' + param_name

        self._preset(param_name, args, kwargs)

    def _prepare_experiment(self):
        """Called by the environment to make some initial configurations before performing the
        individual runs.

        Checks if all parameters marked for presetting were preset. If not raises a
        DefaultReplacementError.

        Locks all parameters.

        Removal of potential results of previous runs in case the trajectory was expanded to avoid
        mixing up undesired shortcuts in natural naming.

        """
        if len(self._changed_default_parameters):
            raise pex.PresettingError(
                'The following parameters were supposed to replace a '
                'default value, but it was never tried to '
                'add default values with these names: %s' %
                str(self._changed_default_parameters))

        self.f_lock_parameters()
        self.f_lock_derived_parameters()

    @not_in_run
    @kwargs_api_change('backwards_search')
    @kwargs_api_change('where')
    def f_get_from_runs(self, name, include_default_run=True, use_indices=False,
                        fast_access=False, with_links = True,
                        shortcuts=True, max_depth=None, auto_load=False):
        """Searches for all occurrences of `name` in each run.

        Generates an ordered dictionary with the run names or indices as keys and
        found items as values.

        Example:

        >>> traj.f_get_from_runs(self, 'deep.universal_answer', use_indices=True, fast_access=True)
        OrderedDict([(0, 42), (1, 42), (2, 'fortytwo), (3, 43)])


        :param name:

            String description of the item(s) to find.
            Cannot be full names but the part of the names that are below
            a `run_XXXXXXXXX` group.

        :param include_default_run:

            If results found under ``run_ALL`` should be accounted for every run or simply be
            ignored.

        :param use_indices:

            If `True` the keys of the resulting dictionary are the run indices
            (e.g. 0,1,2,3), otherwise the keys are run names (e.g. `run_00000000`,
            `run_000000001`)

        :param fast_access:

            Whether to return parameter or result instances or the values handled by these.

        :param with_links:

            If links should be considered

        :param shortcuts:

            If shortcuts are allowed and the trajectory can *hop* over nodes in the
            path.

        :param max_depth:

            Maximum depth (relative to start node) how search should progress in tree.
            `None` means no depth limit.  Only relevant if `shortcuts` are allowed.

        :param auto_load:

            If data should be loaded from the storage service if it cannot be found in the
            current trajectory tree. Auto-loading will load group and leaf nodes currently
            not in memory and it will load data into empty leaves. Be aware that auto-loading
            does not work with shortcuts.

        :return:

            Ordered dictionary with run names or indices as keys and found items as values.
            Will only include runs where an item was actually found.

        """
        result_dict = OrderedDict()
        old_crun = self.v_crun

        try:

            if len(self._run_parent_groups) > 0:
                for run_name in self.f_iter_runs():
                    # Iterate over all runs
                    value = None
                    already_found = False
                    for run_parent_group in self._run_parent_groups.values():
                        if run_name not in run_parent_group._children:
                            continue

                        try:
                            value = run_parent_group.f_get(run_name + '.' + name,
                                                           fast_access=False,
                                                           with_links=with_links,
                                                           shortcuts=shortcuts,
                                                           max_depth=max_depth,
                                                           auto_load=auto_load)
                            if already_found:
                                raise pex.NotUniqueNodeError('`%s` has been found several times '
                                                             'in one run.' % name)
                            else:
                                already_found = True

                        except (AttributeError, pex.DataNotInStorageError):
                            pass

                    if value is None and include_default_run:
                        for run_parent_group in self._run_parent_groups.values():
                            try:
                                value = run_parent_group.f_get(self.f_wildcard('$', -1) +
                                                               '.' + name,
                                                               fast_access=False,
                                                               with_links=with_links,
                                                               shortcuts=shortcuts,
                                                               max_depth=max_depth,
                                                               auto_load=auto_load)
                                if already_found:
                                    raise pex.NotUniqueNodeError('`%s` has been found several '
                                                                 'times in one run.' % name)
                                else:
                                    already_found = True
                            except (AttributeError, pex.DataNotInStorageError):
                                pass

                    if value is not None:

                        if value.v_is_leaf:
                            value = self._nn_interface._apply_fast_access(value, fast_access)

                        if use_indices:
                            key = self.f_idx_to_run(run_name)
                        else:
                            key = run_name

                        result_dict[key] = value

            return result_dict
        finally:
            self.v_crun = old_crun

    def __len__(self):
        """Length of trajectory, minimum length is 1"""
        return self._length

    @not_in_run
    def f_is_completed(self, name_or_id=None):
        """Whether or not a given run is completed.

        If no run is specified it is checked whether all runs were completed.

        :param name_or_id: Nam or id of a run to check

        :return: True or False

        """
        return self._is_completed(name_or_id)

    def _is_completed(self, name_or_id=None):
        """Private function such that it can still be called by the environment during
        a single run"""

        if name_or_id is None:
            return all(
                (runinfo['completed'] for runinfo in self._run_information.values()))
        else:
            return self.f_get_run_information(name_or_id, copy=False)['completed']

    @not_in_run
    def f_expand(self, build_dict, fail_safe=True):
        """Similar to :func:`~pypet.trajectory.Trajectory.f_explore`, but can be used to enlarge
        already completed trajectories.

        Please ensure before usage, that all explored parameters are loaded!

        :param build_dict:

            Dictionary containing the expansion

        :param fail_safe:

            If old ranges should be **deep-copied** in order to allow to restore
            the original exploration if something fails during expansion.
            Set to `False` if deep-copying your parameter ranges causes errors.

        :raises:

            TypeError: If not all explored parameters are enlarged

            AttributeError: If keys of dictionary cannot be found in the trajectory

            NotUniqueNodeError:

                If dictionary keys do not unambiguously map to single parameters

            ValueError: If not all explored parameter ranges are of the same length

        """
        if len(self._explored_parameters) == 0:
            self._logger.info('Your trajectory has not been explored, yet. '
                              'I will call `f_explore` instead.')
            return self.f_explore(build_dict)

        enlarge_set = set([self.f_get(key).v_full_name
                           for key in build_dict.keys()])

        # Check if all explored parameters will be enlarged, otherwise
        # We cannot enlarge the trajectory
        if not set(self._explored_parameters.keys()) == enlarge_set:
            raise TypeError('You have to enlarge dimensions you have explored before! Currently'
                            ' explored parameters are not the ones you specified in your building'
                            ' dictionary, i.e. %s != %s' %
                            (str(set(self._explored_parameters.keys())),
                             str(set(build_dict.keys()))))
        if any(x is None for x in self._explored_parameters.values()):
            raise TypeError('At least one of your explored parameters is not fully loaded, '
                            'please load it.')

        old_ranges = None
        if fail_safe:
            old_ranges = {}
            for param_name in self._explored_parameters:
                old_ranges[param_name] = self._explored_parameters[param_name].f_get_range()
            try:
                old_ranges = cp.deepcopy(old_ranges)
            except Exception:
                self._logger.error('Cannot deepcopy old parameter ranges, if '
                                     'something fails during `f_expand` I cannot revert the '
                                     'trajectory to old settings.')
                old_ranges = None

        try:
            count = 0
            length = None
            for key, builditerable in build_dict.items():
                act_param = self.f_get(key)

                act_param.f_unlock()
                act_param._expand(builditerable)

                name = act_param.v_full_name

                self._explored_parameters[name] = act_param

                # Compare the length of two consecutive parameters in the `build_dict`
                if count == 0:
                    length = act_param.f_get_range_length()
                elif not length == act_param.f_get_range_length():
                    raise ValueError('The parameters to explore have not the same size!')
                count += 1

            original_length = len(self)
            for irun in range(original_length, length):
                self._add_run_info(irun)
            self._test_run_addition(length)

            # We need to update the explored parameters in case they were stored:
            self._remove_exploration()

        except Exception:
            if old_ranges is not None:
                # Try to restore the original parameter exploration
                for param_name in old_ranges:
                    param_range = old_ranges[param_name]
                    param = self._explored_parameters[param_name]
                    param.f_unlock()
                    try:
                        param._shrink()
                    except Exception as exc:
                        self._logger.error('Could not shrink parameter `%s` '
                                           'because of:`%s`' % (param_name, repr(exc)))
                    param._explore(param_range)
                    param._explored = True
            raise

    def _test_run_addition(self, length):
        """Tests if the user's run naming works and every index produces a new run name"""
        if not len(self) == length:
            raise RuntimeError('Your trajectory has a wrong length. Your naming function '
                               'does not work!')
        dummy = self.f_wildcard('$', -1)
        if dummy in self._run_information:
            raise RuntimeError('Your renaming function does not return an appropriate value for '
                               '`-1`. `%s` is actual an already given run name.' % dummy)

    def _remove_exploration(self):
        """ Called if trajectory is expanded, deletes all explored parameters from disk """
        for param in self._explored_parameters.values():
            if param._stored:
                try:
                    self.f_delete_item(param)
                except Exception:
                    self._logger.exception('Could not delete expanded parameter `%s` '
                                           'from disk.' % param.v_full_name)

    def __copy__(self):
        """Returns a shallow copy"""
        return self.f_copy(copy_leaves=True,
                           with_links=True)

    def f_copy(self, copy_leaves=True,
                     with_links=True):
        """Returns a *shallow* copy of a trajectory.

        :param copy_leaves:

            If leaves should be **shallow** copied or simply referred to by both trees.
            **Shallow** copying is established using the copy module.

            Accepts the setting ``'explored'`` to only copy explored parameters.
            Note that ``v_full_copy`` determines how these will be copied.

        :param with_links: If links should be ignored or followed and copied as well

        :return: A shallow copy

        """
        new_traj = Trajectory(_copy_traj=True)

        new_traj._length = self._length
        new_traj._name = self._name

        new_traj._timestamp = self._timestamp
        new_traj._time = self._time

        new_traj._single_run_ids = self._single_run_ids
        new_traj._run_information = self._run_information
        new_traj._updated_run_information = self._updated_run_information

        new_traj._fast_access = self._fast_access
        new_traj._shortcuts = self._shortcuts
        new_traj._iter_recursive = self._iter_recursive
        new_traj._max_depth = self._max_depth
        new_traj._auto_load = self._auto_load
        new_traj._with_links = self._with_links

        new_traj._environment_hexsha = self._environment_hexsha
        new_traj._environment_name = self._environment_name

        # Index of a trajectory is -1, if the trajectory should behave like a single run
        # and blind out other single run results, this can be changed via 'v_crun'.
        new_traj._idx = self._idx
        new_traj._crun = self._crun

        new_traj._standard_parameter = self._standard_parameter
        new_traj._standard_result = self._standard_result
        new_traj._standard_leaf = self._standard_leaf

        new_traj._auto_run_prepend = self._auto_run_prepend
        new_traj._no_clobber = self._no_clobber

        new_traj._run_started = self._run_started # For manually using a trajectory
        new_traj._run_by_environment = self._run_by_environment  # To disable manual running of experiment

        new_traj._full_copy = self._full_copy

        new_traj._dynamic_imports = self._dynamic_imports

        new_traj._wildcard_functions = self._wildcard_functions
        new_traj._wildcard_keys = self._wildcard_keys
        new_traj._reversed_wildcards = self._reversed_wildcards
        new_traj._wildcard_cache = self._wildcard_cache

        new_traj._comment = self._comment

        new_traj._stored = self._stored

        new_traj._storage_service = self._storage_service

        new_traj._is_run = self._is_run

        new_traj._copy_from(self, copy_leaves=copy_leaves,
                                   with_links=with_links)

        # Copy references to new nodes and leaves
        for my_dict, new_dict in ((self._new_nodes, new_traj._new_nodes),
                                 (self._new_links, new_traj._new_links)):
            for key in my_dict:
                value = my_dict[key]
                parent, child = value
                if parent is self:
                    new_parent = new_traj
                else:
                    new_parent = new_traj.f_get(parent.v_full_name,
                                            shortcuts=False,
                                            with_links=False,
                                            auto_load=False)
                new_child = new_parent._children[key[1]]
                new_dict[key] = (new_parent, new_child)

        return new_traj

    def _copy_from(self, node,
                          copy_leaves=True,
                          overwrite=False,
                          with_links=True):
        """Pass a ``node`` to insert the full tree to the trajectory.

        Considers all links in the given node!
        Ignored nodes already found in the current trajectory.

        :param node: The node to insert

        :param copy_leaves:

            If leaves should be **shallow** copied or simply referred to by both trees.
            **Shallow** copying is established using the copy module.

            Accepts the setting ``'explored'`` to only copy explored parameters.
            Note that ``v_full_copy`` determines how these will be copied.

        :param overwrite:

            If existing elemenst should be overwritten. Requries ``__getstate__`` and
            ``__setstate__`` being implemented in the leaves.

        :param with_links: If links should be ignored or followed and copied as well

        :return: The corresponding (new) node in the tree.

        """

        def _copy_skeleton(node_in, node_out):
            """Copies the skeleton of from `node_out` to `node_in`"""
            new_annotations = node_out.v_annotations
            node_in._annotations = new_annotations
            node_in.v_comment = node_out.v_comment

        def _add_leaf(leaf):
            """Adds a leaf to the trajectory"""
            leaf_full_name = leaf.v_full_name
            try:
                found_leaf = self.f_get(leaf_full_name,
                                        with_links=False,
                                        shortcuts=False,
                                        auto_load=False)
                if overwrite:
                    found_leaf.__setstate__(leaf.__getstate__())
                return found_leaf
            except AttributeError:
                pass
            if copy_leaves is True or (copy_leaves == 'explored' and
                                           leaf.v_is_parameter and leaf.v_explored):
                new_leaf = self.f_add_leaf(cp.copy(leaf))
            else:
                new_leaf = self.f_add_leaf(leaf)
            if new_leaf.v_is_parameter and new_leaf.v_explored:
                self._explored_parameters[new_leaf.v_full_name] = new_leaf
            return new_leaf

        def _add_group(group):
            """Adds a new group to the trajectory"""
            group_full_name = group.v_full_name
            try:
                found_group = self.f_get(group_full_name,
                                             with_links=False,
                                             shortcuts=False,
                                             auto_load=False)
                if overwrite:
                    _copy_skeleton(found_group, group)
                return found_group
            except AttributeError:
                pass
            new_group = self.f_add_group(group_full_name)
            _copy_skeleton(new_group, group)
            return new_group

        is_run = self._is_run
        self._is_run = False  # So that we can copy Config Groups and Config Data
        try:
            if node.v_is_leaf:
                return _add_leaf(node)
            elif node.v_is_group:
                other_root = node.v_root
                if other_root is self:
                    raise RuntimeError('You cannot copy a given tree to itself!')
                result = _add_group(node)
                nodes_iterator = node.f_iter_nodes(recursive=True, with_links=with_links)
                has_links = []
                if node._links:
                    has_links.append(node)
                for child in nodes_iterator:
                    if child.v_is_leaf:
                        _add_leaf(child)
                    else:
                        _add_group(child)
                        if child._links:
                            has_links.append(child)

                if with_links:
                    for current in has_links:
                        mine = self.f_get(current.v_full_name, with_links=False,
                                          shortcuts=False, auto_load=False)
                        my_link_set = set(mine._links.keys())
                        other_link_set = set(current._links.keys())
                        new_links = other_link_set - my_link_set
                        for link in new_links:
                            where_full_name = current._links[link].v_full_name
                            mine.f_add_link(link, where_full_name)

                return result
            else:
                raise RuntimeError('You shall not pass!')
        except Exception:
            self._is_run = is_run

    @not_in_run
    def f_explore(self, build_dict):
        """Prepares the trajectory to explore the parameter space.


        To explore the parameter space you need to provide a dictionary with the names of the
        parameters to explore as keys and iterables specifying the exploration ranges as values.

        All iterables need to have the same length otherwise a ValueError is raised.
        A ValueError is also raised if the names from the dictionary map to groups or results
        and not parameters.

        If your trajectory is already explored but not stored yet and your parameters are
        not locked you can add new explored parameters to the current ones if their
        iterables match the current length of the trajectory.

        Raises an AttributeError if the names from the dictionary are not found at all in
        the trajectory and NotUniqueNodeError if the keys not unambiguously map
        to single parameters.

        Raises a TypeError if the trajectory has been stored already, please use
        :func:`~pypet.trajectory.Trajectory.f_expand` then instead.

        Example usage:

        >>> traj.f_explore({'groupA.param1' : [1,2,3,4,5], 'groupA.param2':['a','b','c','d','e']})

        Could also be called consecutively:

        >>> traj.f_explore({'groupA.param1' : [1,2,3,4,5]})
        >>> traj.f_explore({'groupA.param2':['a','b','c','d','e']})

        NOTE:

        Since parameters are very conservative regarding the data they accept
        (see :ref:`type_conservation`), you sometimes won't be able to use Numpy arrays
        for exploration as iterables.

        For instance, the following code snippet won't work:

        ::

            import numpy a np
            from pypet.trajectory import Trajectory
            traj = Trajectory()
            traj.f_add_parameter('my_float_parameter', 42.4,
                                 comment='My value is a standard python float')

            traj.f_explore( { 'my_float_parameter': np.arange(42.0, 44.876, 0.23) } )


        This will result in a `TypeError` because your exploration iterable
        `np.arange(42.0, 44.876, 0.23)` contains `numpy.float64` values
        whereas you parameter is supposed to use standard python floats.

        Yet, you can use Numpys `tolist()` function to overcome this problem:

        ::

            traj.f_explore( { 'my_float_parameter': np.arange(42.0, 44.876, 0.23).tolist() } )


        Or you could specify your parameter directly as a numpy float:

        ::

            traj.f_add_parameter('my_float_parameter', np.float64(42.4),
                                   comment='My value is a numpy 64 bit float')



        """
        for run_idx in range(len(self)):
            if self.f_is_completed(run_idx):
                raise TypeError('You cannot explore a trajectory which has been explored before, '
                                'please use `f_expand` instead.')

        added_explored_parameters = []
        try:
            length = len(self)
            for key, builditerable in build_dict.items():
                act_param = self.f_get(key)
                if not act_param.v_is_leaf or not act_param.v_is_parameter:
                    raise ValueError('%s is not an appropriate search string for a parameter.' % key)

                act_param.f_unlock()

                act_param._explore(builditerable)
                added_explored_parameters.append(act_param)

                full_name = act_param.v_full_name
                self._explored_parameters[full_name] = act_param

                act_param._explored = True

                # Compare the length of two consecutive parameters in the `build_dict`
                if len(self._explored_parameters) == 1:
                    length = act_param.f_get_range_length()
                elif not length == act_param.f_get_range_length():
                    raise ValueError('The parameters to explore have not the same size!')

            for irun in range(length):
                self._add_run_info(irun)
            self._test_run_addition(length)

        except Exception:
            # Remove the added parameters again
            for param in added_explored_parameters:
                param.f_unlock()
                param._shrink()
                param._explored = False
                full_name = param.v_full_name
                del self._explored_parameters[full_name]
            if len(self._explored_parameters) == 0:
                self.f_shrink(force=True)
            raise

    def _update_run_information(self, run_information_dict):
        """Overwrites the run information of a particular run"""
        idx = run_information_dict['idx']
        name = run_information_dict['name']
        self._run_information[name] = run_information_dict
        self._updated_run_information.add(idx)

    def _add_run_info(self, idx, name='', timestamp=42.0, finish_timestamp=1.337,
                      runtime='forever and ever', time='>>Maybe time`s gone on strike',
                      completed=0, parameter_summary='Not yet my friend!',
                      short_environment_hexsha='N/A'):
        """Adds a new run to the `_run_information` dict."""

        if idx in self._single_run_ids:
            # Delete old entries, they might be replaced by a new name
            old_name = self._single_run_ids[idx]
            del self._single_run_ids[old_name]
            del self._single_run_ids[idx]
            del self._run_information[old_name]

        if name == '':
            name = self.f_wildcard('$', idx)
        # The `_single_run_ids` dict is bidirectional and maps indices to run names and vice versa
        self._single_run_ids[name] = idx
        self._single_run_ids[idx] = name

        info_dict = {'idx': idx,
                     'timestamp': timestamp,
                     'finish_timestamp': finish_timestamp,
                     'runtime': runtime,
                     'time': time,
                     'completed': completed,
                     'name': name,
                     'parameter_summary': parameter_summary,
                     'short_environment_hexsha': short_environment_hexsha}

        self._run_information[name] = info_dict
        self._length = len(self._run_information)

    @not_in_run
    def f_lock_parameters(self):
        """Locks all non-empty parameters"""
        for par in self._parameters.values():
            if not par.f_is_empty():
                par.f_lock()

    @not_in_run
    def f_lock_derived_parameters(self):
        """Locks all non-empty derived parameters"""
        for par in self._derived_parameters.values():
            if not par.f_is_empty():
                par.f_lock()

    def _finalize(self, store_meta_data=True):
        """Final rollback initiated by the environment

        Restores the trajectory as root of the tree, and stores meta data to disk.
        This updates the trajectory's information about single runs, i.e. if they've been
        completed, when they were started, etc.

        """
        self._is_run = False
        self.f_set_crun(None)
        if store_meta_data:
            self.f_store(only_init=True)

    @not_in_run
    def f_load_skeleton(self):
        """Loads the full skeleton from the storage service.

        This needs to be done after a successful exploration in order to update the
        trajectory tree with all results and derived parameters from the individual single runs.
        This will only add empty results and derived parameters (i.e. the skeleton)
        and load annotations.

        """
        self.f_load(self.v_name, as_new=False, load_parameters=pypetconstants.LOAD_SKELETON,
                    load_derived_parameters=pypetconstants.LOAD_SKELETON,
                    load_results=pypetconstants.LOAD_SKELETON,
                    load_other_data=pypetconstants.LOAD_SKELETON,
                    with_run_information=False)

    @not_in_run
    @kwargs_api_change('load_all', 'load_data')
    @kwargs_api_change('dynamically_imported_classes', 'dynamic_imports')
    def f_load(self, name=None, index=None, as_new=False, load_parameters=pypetconstants.LOAD_DATA,
               load_derived_parameters=pypetconstants.LOAD_SKELETON,
               load_results=pypetconstants.LOAD_SKELETON,
               load_other_data=pypetconstants.LOAD_SKELETON,
               recursive=True,
               load_data=None,
               max_depth=None,
               force=False,
               dynamic_imports=None,
               with_run_information=True,
               with_meta_data=True,
               storage_service=None, **kwargs):
        """Loads a trajectory via the storage service.


        If you want to load individual results or parameters manually, you can take
        a look at :func:`~pypet.trajectory.Trajectory.f_load_items`.
        To only load subtrees check out :func:`~pypet.naturalnaming.NNGroupNode.f_load_child`.


        For `f_load` you can pass the following arguments:

        :param name:

            Name of the trajectory to be loaded. If no name or index is specified
            the current name of the trajectory is used.

        :param index:

            If you don't specify a name you can specify an integer index instead.
            The corresponding trajectory in the hdf5 file at the index
            position is loaded (counting starts with 0). Negative indices are also allowed
            counting in reverse order. For instance, `-1` refers to the last trajectory in
            the file, `-2` to the second last, and so on.

        :param as_new:

            Whether you want to rerun the experiments. So the trajectory is loaded only
            with parameters. The current trajectory name is kept in this case, which should be
            different from the trajectory name specified in the input parameter `name`.
            If you load `as_new=True` all parameters are unlocked.
            If you load `as_new=False` the current trajectory is replaced by the one on disk,
            i.e. name, timestamp, formatted time etc. are all taken from disk.

        :param load_parameters: How parameters and config items are loaded

        :param load_derived_parameters: How derived parameters are loaded

        :param load_results: How results are loaded

            You can specify how to load the parameters, derived parameters and results
            as follows:

                * :const:`pypet.pypetconstants.LOAD_NOTHING`: (0)

                    Nothing is loaded.

                * :const:`pypet.pypetconstants.LOAD_SKELETON`: (1)

                    The skeleton is loaded including annotations (See :ref:`more-on-annotations`).
                    This means that only empty
                    *parameter* and *result* objects will
                    be created  and you can manually load the data into them afterwards.
                    Note that :class:`pypet.annotations.Annotations` do not count as data and they
                    will be loaded because they are assumed to be small.

                * :const:`pypet.pypetconstants.LOAD_DATA`: (2)

                    The whole data is loaded. Note in case you have non-empty leaves
                    already in RAM, these are left untouched.

                * :const:`pypet.pypetconstants.OVERWRITE_DATA`: (3)

                    As before, but non-empty nodes are emptied and reloaded.

                Note that in all cases except :const:`pypet.pypetconstants.LOAD_NOTHING`,
                annotations will be reloaded if the corresponding instance
                is created or the annotations of an existing instance were emptied before.

        :param recursive:

            If data should be loaded recursively. If set to `None`, this is equivalent to
            set all data loading to `:const:`pypet.pypetconstants.LOAD_NOTHING`.

        :param load_data:

            As the above, per default set to `None`. If not `None` the setting of `load_data`
            will overwrite the settings of `load_parameters`, `load_derived_parameters`,
            `load_results`, and `load_other_data`. This is more or less or shortcut if all
            types should be loaded the same.

        :param max_depth:

            Maximum depth to load nodes (inclusive).

        :param force:

            *pypet* will refuse to load trajectories that have been created using *pypet* with a
            different version number or a different python version.
            To force the load of a trajectory from a previous version
            simply set ``force = True``. Note that it is not checked if other versions of packages
            differ from previous experiments, i.e. numpy, scipy, etc. But you can check
            for this manually. The versions of other packages can be found under
            ``'config.environment.name_of_environment.versions.package_name'``.

        :param dynamic_imports:

            If you've written a custom parameter that needs to be loaded dynamically
            during runtime, this needs to be specified here as a list of classes or
            strings naming classes and there module paths. For example:
            `dynamic_imports = ['pypet.parameter.PickleParameter',MyCustomParameter]`

            If you only have a single class to import, you do not need the list brackets:
            `dynamic_imports = 'pypet.parameter.PickleParameter'`

            The classes passed here are added for good and will be kept by the trajectory.
            Please add your dynamically imported classes only once.

        :param with_run_information:

            If information about the individual runs should be loaded. If you have many
            runs, like 1,000,000 or more you can spare time by setting
            `with_run_information=False`.
            Note that `f_get_run_information` and `f_idx_to_run` do not work in such a case.
            Moreover, setting `v_idx` does not work either. If you load the trajectory
            without this information, be careful, this is not recommended.

        :param wiht_meta_data:

            If meta data should be loaded.

        :param storage_service:

            Pass a storage service used by the trajectory. Alternatively pass a constructor
            and other ``**kwargs`` are passed onto the constructor. Leave `None` in combination
            with using no other kwargs, if you don't want to change the service
            the trajectory is currently using.

        :param kwargs:

            Other arguments passed to the storage service constructor. Don't pass any
            other kwargs and ``storage_service=None``,
            if you don't want to change the current service.

        """
        # Do some argument validity checks first
        if name is None and index is None:
            name = self.v_name

        if as_new:
            load_parameters = pypetconstants.LOAD_DATA
            load_derived_parameters = pypetconstants.LOAD_NOTHING
            load_results = pypetconstants.LOAD_NOTHING
            load_other_data = pypetconstants.LOAD_NOTHING

        unused_kwargs = set(kwargs.keys())
        if self.v_storage_service is None or storage_service is not None or len(kwargs) > 0:
            self._storage_service, unused_kwargs = storage_factory(storage_service=storage_service,
                                                                   trajectory=self, **kwargs)
        if len(unused_kwargs) > 0:
            raise ValueError('The following keyword arguments were not used: `%s`' %
                             str(unused_kwargs))

        if dynamic_imports is not None:
            self.f_add_to_dynamic_imports(dynamic_imports)

        if load_data is not None:
            load_parameters = load_data
            load_derived_parameters = load_data
            load_results = load_data
            load_other_data = load_data

        self._storage_service.load(pypetconstants.TRAJECTORY, self, trajectory_name=name,
                                   trajectory_index=index,
                                   as_new=as_new, load_parameters=load_parameters,
                                   load_derived_parameters=load_derived_parameters,
                                   load_results=load_results,
                                   load_other_data=load_other_data,
                                   recursive=recursive,
                                   max_depth=max_depth,
                                   with_run_information=with_run_information,
                                   with_meta_data=with_meta_data,
                                   force=force)

        # If a trajectory is newly loaded, all parameters are unlocked.
        if as_new:
            for param in self._parameters.values():
                param.f_unlock()

    def _check_if_both_have_same_parameters(self, other_trajectory,
                                            ignore_data, consecutive_merge):
        """ Checks if two trajectories live in the same space and can be merged. """

        if not isinstance(other_trajectory, Trajectory):
            raise TypeError('Can only merge trajectories, the other trajectory'
                            ' is of type `%s`.' % str(type(other_trajectory)))

        if self._stored and not consecutive_merge:
            self.f_load_skeleton()
        if other_trajectory._stored:
            other_trajectory.f_load_skeleton()

        # Check wildcard set
        other_wildcard_set = set(x[1] for x in other_trajectory._wildcard_functions.keys())
        wildcard_set = set(x[1] for x in self._wildcard_functions.keys())
        diff = wildcard_set.symmetric_difference(other_wildcard_set)
        if diff:
            raise TypeError('The wildcard sets are not matching. `%s` != `%s`' %
                            (str(wildcard_set), str(other_wildcard_set)))

        # Load all parameters of the current and the other trajectory
        if self._stored:
            # To suppress warnings if nothing needs to be loaded
            with self._nn_interface._disable_logging:
                self.f_load_items(self._parameters.keys(), only_empties=True)
        if other_trajectory._stored:
            with self._nn_interface._disable_logging:
                other_trajectory.f_load_items(other_trajectory._parameters.keys(),
                                              only_empties=True)

        self.f_restore_default()
        other_trajectory.f_restore_default()

        allmyparams = self._parameters.copy()
        allotherparams = other_trajectory._parameters.copy()

        # If not ignored, add also the trajectory derived parameters to check for merging
        if 'derived_parameters' in self:
            my_traj_dpars = self._derived_parameters
            if self._stored:
                with self._nn_interface._disable_logging:
                    self.f_load_items(my_traj_dpars.keys(), only_empties=True)
            allmyparams.update(my_traj_dpars)
            other_traj_dpars = other_trajectory._derived_parameters
            if other_trajectory._stored:
                with self._nn_interface._disable_logging:
                    other_trajectory.f_load_items(other_traj_dpars.keys(), only_empties=True)
            allotherparams.update(other_traj_dpars)

        # Check if the trajectories have the same parameters:
        my_keyset = set(allmyparams.keys())
        other_keyset = set(allotherparams.keys())
        diff = my_keyset.symmetric_difference(other_keyset) - ignore_data
        run_dummys = (self.f_wildcard('$', -1), other_trajectory.f_wildcard('$', -1))
        if diff:
            run_difference_can_be_resolved = True
            for full_name in diff:
                split_name = full_name.split('.')
                if not any(x in self._run_information or
                                x in other_trajectory._run_information or
                                    x in run_dummys
                                        for x in split_name):
                    run_difference_can_be_resolved = False
                    break
                elif full_name in allotherparams:
                    del allotherparams[full_name]
            if not run_difference_can_be_resolved:
                raise TypeError('Cannot merge trajectories, '
                                'they do not live in the same space,the '
                                'set of parameters `%s` is only '
                                'found in one trajectory.' % str(diff))

        # Check if corresponding parameters in both trajectories are of the same type
        for key, other_param in allotherparams.items():
            if key in ignore_data:
                continue
            my_param = self.f_get(key)
            split_key = key.split('.')
            if any(x in self._run_information or
                        x in other_trajectory._run_information
                            for x in split_key):
                pass
            else:
                if not my_param._values_of_same_type(my_param.f_get(), other_param.f_get()):
                    raise TypeError('Cannot merge trajectories, values of parameters `%s` are not '
                                    'of the same type. Types are %s (current) and %s (other).' %
                                    (key, str(type(my_param.f_get())),
                                     str(type(other_param.f_get()))))

    @not_in_run
    def f_backup(self, **kwargs):
        """Backs up the trajectory with the given storage service.

        Arguments of ``kwargs`` are directly passed to the storage service,
        for the HDF5StorageService you can provide the following argument:

        :param backup_filename:

            Name of file where to store the backup.

            In case you use the standard HDF5 storage service and `backup_filename=None`,
            the file will be chosen automatically.
            The backup file will be in the same folder as your hdf5 file and
            named 'backup_XXXXX.hdf5' where 'XXXXX' is the name of your current trajectory.

        """
        self._storage_service.store(pypetconstants.BACKUP, self, trajectory_name=self.v_name,
                                    **kwargs)

    def _make_reversed_wildcards(self, old_length=-1):
        """Creates a full mapping from all wildcard translations to the corresponding wildcards"""
        if len(self._reversed_wildcards) > 0:
            # We already created reversed wildcards, so we don't need to do all of them
            # again
            start = old_length
        else:
            start = -1
        for wildcards, func in self._wildcard_functions.items():
            for irun in range(start, len(self)):
                translated_name = func(irun)
                if not translated_name in self._reversed_wildcards:
                    self._reversed_wildcards[translated_name] = ([], wildcards)
                self._reversed_wildcards[translated_name][0].append(irun)

    @not_in_run
    def f_merge_many(self, other_trajectories,
                ignore_data=(),
                move_data=False,
                delete_other_trajectory=False,
                keep_info=True,
                keep_other_trajectory_info=True,
                merge_config=True,
                backup=True):
        """Can be used to merge several `other_trajectories` into your current one.

        IMPORTANT `backup=True` only backs up the current trajectory not any of
        the `other_trajectories`. If you need a backup of these, do it manually.

        Parameters as for :func:`~pypet.trajectory.Trajectory.f_merge`.

        """
        other_length = len(other_trajectories)
        self._logger.info('Merging %d trajectories into the current one.' % other_length)
        self.f_load_skeleton()

        if backup:
            self.f_backup()

        for idx, other in enumerate(other_trajectories):
            self.f_merge(other, ignore_data=ignore_data,
                         move_data=move_data,
                         delete_other_trajectory=delete_other_trajectory,
                         keep_info=keep_info,
                         keep_other_trajectory_info=keep_other_trajectory_info,
                         merge_config=merge_config,
                         backup=False,
                         consecutive_merge=True)
            self._logger.log(21,'Merged %d out of %d' % (idx + 1, other_length))
        self._logger.info('Storing data to disk')
        self._reversed_wildcards = {}
        self.f_store()
        self._logger.info('Finished final storage')

    @not_in_run
    @kwargs_api_change('backup_filename', 'backup')
    @kwargs_api_change('move_nodes', 'move_data')
    @kwargs_api_change('ignore_trajectory_derived_parameters')
    @kwargs_api_change('ignore_trajectory_results')
    def f_merge(self, other_trajectory, trial_parameter=None, remove_duplicates=False,
                ignore_data=(),
                backup=True,
                move_data=False,
                delete_other_trajectory=False,
                keep_info=True,
                keep_other_trajectory_info=True,
                merge_config=True,
                consecutive_merge=False,
                slow_merge=False):
        """Merges another trajectory into the current trajectory.

        Both trajectories must live in the same space. This means both need to have the same
        parameters with similar types of values.

        Note that links are also merged. There are exceptions: Links found under
        a generic run group called `run_ALL` or links linking to a node under such a
        group are NOT merged and simply skipped, because there is no straightforward
        way to resolve the link.

        :param other_trajectory: Other trajectory instance to merge into the current one.

        :param trial_parameter:

            If you have a particular parameter that specifies only the trial
            number, i.e. an integer parameter running form 0 to T1 and
            0 to T2, the parameter is modified such that after merging it will
            cover the range 0 to T1+T2+1. T1 is the number of individual trials in the current
            trajectory and T2 number of trials in the other trajectory.

        :param remove_duplicates:

            Whether you want to remove duplicate parameter points.
            Requires N1 * N2 (quadratic complexity in single runs).
            A ValueError is raised if no runs would be merged.

        :param ignore_data:

            List of full names of data that should be ignored and not merged.

        :param backup:

            If ``True``, backs up both trajectories into files chosen automatically
            by the storage services. If you want to customize your backup use
            the `f_backup` function instead.

        :param move_data:

           Tells the storage service to move data from one trajectory to the other
           instead of copying it.

           If you use the HDF5 storage service and both trajectories are
           stored in the same file, merging is performed fast directly within
           the file. You can choose if you want to copy nodes ('move_nodes=False`)
           from the other trajectory to the current one, or if you want to move them.
           Accordingly, the stored data is no longer accessible in the other trajectory.

        :param delete_other_trajectory:

            If you want to delete the other trajectory after merging.

        :param keep_info:

            If `True`, information about the merge is added to the trajectory `config` tree under
            `config.merge`.

        :param merge_config:

            Whether or not to merge all config parameters under `.config.git`,
            `.config.environment`, and `.config.merge` of the other trajectory
            into the current one.

        :param keep_other_trajectory_info:

            Whether to keep information like length, name, etc. of the other trajectory
            in case you want to keep all the information. Setting of `keep_other_trajectory_info`
            is irrelevant in case `keep_info=False`.

        :param consecutive_merge:
            Can be set to `True` if you are about to merge several trajectories into the current
            one within a loop to avoid quadratic complexity.
            But remember to store your trajectory manually after
            all merges. Also make sure that all parameters and derived parameters are available
            in your current trajectory and load them before the consecutive merging.
            Also avoid specifying a `trial_parameter` and set `backup=False`
            to avoid quadratic complexity in case of consecutive merges.

        :param slow_merge:
            Enforces a slow merging. This means all data is loaded one after the other to
            memory and stored to disk. Otherwise it is tried to directly copy the data
            from one file into another without explicitly loading the data.

        If you cannot directly merge trajectories within one HDF5 file, a slow merging process
        is used. Results are loaded, stored, and emptied again one after the other. Might take
        some time!

        Annotations of parameters and derived parameters under `.derived_parameters.trajectory`
        are NOT merged. If you wish to extract the annotations of these parameters you have to
        do that manually before merging. Note that annotations of results and derived parameters
        of single runs are copied, so you don't have to worry about these.

        """
        if consecutive_merge and trial_parameter is not None:
            self._logger.warning('If you do a consecutive merge and specify a trial parameter, '
                                 'your merging will still suffer from quadratic time complexity!')
        if consecutive_merge and backup:
            self._logger.warning('If you do a consecutive merge and backup, '
                                 'your merging will still suffer from quadratic time complexity!')

        # Keep the timestamp of the merge
        timestamp = time.time()
        original_ignore_data = set(ignore_data)
        ignore_data = original_ignore_data.copy()

        old_len = len(self)

        # Check if trajectories can be merged
        self._check_if_both_have_same_parameters(other_trajectory, ignore_data, consecutive_merge)

        # Create a full mapping set for renaming
        self._make_reversed_wildcards(old_length=old_len)
        other_trajectory._make_reversed_wildcards()

        # BACKUP if merge is possible
        if backup:
            other_trajectory.f_backup()
            self.f_backup()

        # Merge parameters and keep track which runs where used and which parameters need
        # to be updated
        self._logger.info('Merging the parameters')
        used_runs, changed_parameters = self._merge_parameters(
            other_trajectory,
            remove_duplicates,
            trial_parameter,
            ignore_data)

        ignore_data.update(set(changed_parameters))

        if len(used_runs) == 0:
            raise ValueError('Your merge discards all runs of the other trajectory, maybe you '
                             'try to merge a trajectory with itself?')

        # Dictionary containing the mappings between run names in the other trajectory
        # and their new names in the current trajectory
        rename_dict = {}

        self._logger.info('Merging run information')
        allowed_translations = set([translation for translation, pair in
                                    other_trajectory._reversed_wildcards.items() if
                                        any(x in used_runs for x in pair[0])])
        self._merge_single_runs(other_trajectory, used_runs)

        self._logger.info('Merging derived parameters')
        self._merge_derived_parameters(other_trajectory=other_trajectory,
                                         used_runs=used_runs,
                                         rename_dict=rename_dict,
                                         allowed_translations=allowed_translations,
                                         ignore_data=ignore_data)

        # Keep track of all trajectory results that should be merged and put
        # information into `rename_dict`
        self._merge_results(other_trajectory=other_trajectory,
                             used_runs=used_runs,
                             rename_dict=rename_dict,
                             allowed_translations=allowed_translations,
                             ignore_data=ignore_data)



        # The storage service needs to prepare the file for merging.
        # This includes updating meta information and already storing the merged parameters
        self._logger.info('Start copying results and single run derived parameters')
        self._logger.info('Updating Trajectory information and changed parameters in storage')
        self._storage_service.store(pypetconstants.PREPARE_MERGE, self,
                                    trajectory_name=self.v_name,
                                    changed_parameters=changed_parameters,
                                    old_length=old_len)

        if not slow_merge:
            try:
                # Merge the single run derived parameters and all results
                # within the same hdf5 file based on `renamed_dict`
                try:
                    # try to get the other file
                    other_filename = other_trajectory.v_storage_service.filename
                except AttributeError:
                    self._logger.warning('Could not determine the filename of the other '
                                         'trajectory, I will assume it`s in the same file.')
                    other_filename = None
                self._storage_service.store(pypetconstants.MERGE, 'FAST MERGE', trajectory_name=self.v_name,
                                            other_trajectory_name=other_trajectory.v_name,
                                            rename_dict=rename_dict, move_nodes=move_data,
                                            delete_trajectory=delete_other_trajectory,
                                            other_filename=other_filename)

            except pex.NoSuchServiceError as exc:
                # If the storage service does not support merge we end up here
                self._logger.exception('My storage service does not support merging of trajectories, '
                                     'I will use the f_load mechanism of the other trajectory and '
                                     'store the results slowly item by item. '
                                     'Note that thereby the other '
                                     'trajectory will be altered (in RAM).')
                slow_merge = True

            except ValueError as exc:
                # If both trajectories are stored in separate files we end up here
                self._logger.exception('Could not perfom fast merging. '
                                     'I will use the `f_load` method of the other trajectory and '
                                     'store the results slowly item by item. '
                                     'Note that thereby the other '
                                     'trajectory will be altered (in RAM).')
                slow_merge = True

        if slow_merge:
            self._merge_slowly(other_trajectory, rename_dict)

        # We will merge the git commits and other config data
        if merge_config:
            self._merge_config(other_trajectory)

        # Finally merging links
        self._merge_links(other_trajectory=other_trajectory,
                         used_runs=used_runs,
                         allowed_translations=allowed_translations,
                         ignore_data=original_ignore_data)

        # Add meta information about the merge to the current trajectory
        self._logger.info('Adding merge information')
        formatted_time = datetime.datetime.fromtimestamp(timestamp).strftime('%Y_%m_%d_%Hh%Mm%Ss')
        hexsha = hashlib.sha1((self.v_name +
                                             str(self.v_timestamp) +
                                             other_trajectory.v_name +
                                             str(other_trajectory.v_timestamp) +
                                             VERSION).encode('utf-8')).hexdigest()

        short_hexsha = hexsha[0:7]

        if keep_info:
            merge_name = 'merge_%s_%s' % (short_hexsha, formatted_time)

            config_name = 'merge.%s.merged_runs' % merge_name
            self.f_add_config(config_name, len(used_runs),
                              comment='Number of merged runs')

            config_name = 'merge.%s.timestamp' % merge_name
            self.f_add_config(config_name, timestamp,
                              comment='Timestamp of merge')

            config_name = 'merge.%s.hexsha' % merge_name
            self.f_add_config(config_name, hexsha,
                              comment='SHA-1 identifier of the merge')

            config_name = 'merge.%s.remove_duplicates' % merge_name
            self.f_add_config(config_name, remove_duplicates,
                              comment='Option to remove duplicate entries')

            if original_ignore_data:
                config_name = 'merge.%s.ignore_data' % merge_name
                self.f_add_config(config_name, tuple(original_ignore_data),
                                  comment='Data to ignore during merge')

            config_name = 'merge.%s.length_before_merge' % merge_name
            self.f_add_config(config_name, len(self),
                              comment='Length of trajectory before merge')

            self.config.merge.v_comment = 'Settings and information of the different merges'

            if self.v_version != VERSION:
                config_name = 'merge.%s.version' % merge_name
                self.f_add_config(config_name, self.v_version,
                                  comment='Pypet version if it differs from the version'
                                          ' of the trajectory')

            if trial_parameter is not None:
                config_name = 'merge.%s.trial_parameter' % merge_name
                self.f_add_config(config_name, len(other_trajectory),
                                  comment='Name of trial parameter')

            if keep_other_trajectory_info:

                if other_trajectory.v_version != self.v_version:
                    config_name = 'merge.%s.other_trajectory.version' % merge_name
                    self.f_add_config(config_name, other_trajectory.v_version,
                                      comment='The version of pypet you used to manage the other'
                                              ' trajectory. Only added if other trajectory\'s'
                                              ' version differs from current trajectory version.')

                config_name = 'merge.%s.other_trajectory.name' % merge_name
                self.f_add_config(config_name, other_trajectory.v_name,
                                  comment='Name of other trajectory merged into the current one')

                config_name = 'merge.%s.other_trajectory.timestamp' % merge_name
                self.f_add_config(config_name, other_trajectory.v_timestamp,
                                  comment='Timestamp of creation of other trajectory '
                                          'merged into the current one')

                config_name = 'merge.%s.other_trajectory.length' % merge_name
                self.f_add_config(config_name, len(other_trajectory),
                                  comment='Length of other trajectory')

                if other_trajectory.v_comment:
                    config_name = 'merge.%s.other_trajectory.comment' % merge_name
                    self.f_add_config(config_name, other_trajectory.v_comment,
                                      comment='Comment of other trajectory')

        # Write out the merged data to disk
        if not consecutive_merge:
            self._logger.info('Writing merged data to disk')
            self.f_store(store_data=pypetconstants.STORE_DATA)
            self._reversed_wildcards = {}
        other_trajectory._reversed_wildcards = {}

        self._logger.info('Finished Merging!')

    def _merge_single_runs(self, other_trajectory, used_runs):
        """  Updates the `run_information` of the current trajectory."""
        count = len(self)  # Variable to count the increasing new run indices and create
        # new run names

        run_indices = range(len(other_trajectory))

        run_name_dict = OrderedDict()

        to_store_groups_with_annotations = []

        for idx in run_indices:
            # Iterate through all used runs and store annotated groups and mark results and
            # derived parameters for merging
            if idx in used_runs:
                # Update the run information dict of the current trajectory
                other_info_dict = other_trajectory.f_get_run_information(idx)
                time_ = other_info_dict['time']
                timestamp = other_info_dict['timestamp']
                completed = other_info_dict['completed']
                short_environment_hexsha = other_info_dict['short_environment_hexsha']
                finish_timestamp = other_info_dict['finish_timestamp']
                runtime = other_info_dict['runtime']

                new_idx = used_runs[idx]
                new_runname = self.f_wildcard('$', new_idx)

                run_name_dict[idx] = new_runname

                info_dict = dict(
                    idx=new_idx,
                    time=time_,
                    timestamp=timestamp,
                    completed=completed,
                    short_environment_hexsha=short_environment_hexsha,
                    finish_timestamp=finish_timestamp,
                    runtime=runtime)

                self._add_run_info(**info_dict)

    def _rename_full_name(self, full_name, other_trajectory, used_runs=None, new_run_idx=None):
        """Renames a full name based on the wildcards and a particular run"""
        split_name = full_name.split('.')
        for idx, name in enumerate(split_name):
            if name in other_trajectory._reversed_wildcards:
                run_indices, wildcards = other_trajectory._reversed_wildcards[name]
                if new_run_idx is None:
                    # We can safely take the first index of the index list that matches
                    run_idx = None
                    for run_jdx in run_indices:
                        if run_jdx in used_runs:
                            run_idx = used_runs[run_jdx]
                            break
                        elif run_jdx == -1:
                            run_idx = -1
                            break
                    if run_idx is None:
                        raise RuntimeError('You shall not pass!')
                else:
                    run_idx = new_run_idx
                new_name = self.f_wildcard(wildcards[0], run_idx)
                split_name[idx] = new_name
        full_name = '.'.join(split_name)
        return full_name

    def _merge_derived_parameters(self,
                                    other_trajectory,
                                    used_runs,
                                    rename_dict,
                                    allowed_translations,
                                    ignore_data):
        """ Merges derived parameters that have the `run_ALL` in a name.

        Creates a new parameter with the name of the first new run and links to this
        parameter to avoid copying in all other runs.

        """
        other_derived_parameters = other_trajectory._derived_parameters.copy()
        # get first run_idx
        new_first_run_idx = min(used_runs.values())

        run_name_dummy = other_trajectory.f_wildcard('$', -1)
        for param_name in other_derived_parameters:
            if param_name in ignore_data:
                continue
            split_name = param_name.split('.')
            if not any(x in run_name_dummy for x in split_name):
                continue
            ignore_data.add(param_name)

            param = other_derived_parameters[param_name]
            new_param_name = self._rename_full_name(param_name, other_trajectory,
                                                    used_runs=used_runs)
            if new_param_name in self:
                my_param = self.f_get(new_param_name, fast_access=False)
                if (my_param._equal_values(my_param.f_get(), param.f_get()) and
                        not (my_param.f_has_range() or param.f_has_range())):
                    continue

            first_new_param_name = self._rename_full_name(param_name,
                                                          other_trajectory,
                                                          new_run_idx=new_first_run_idx)

            rename_dict[param_name] = first_new_param_name
            comment = param.v_comment
            param_type = param.f_get_class_name()
            param_type = self._create_class(param_type)
            first_param = self.f_add_leaf(param_type,
                                          first_new_param_name,
                                          comment=comment)
            for run_idx in used_runs.values():
                if run_idx == new_first_run_idx:
                    continue
                next_name = self._rename_full_name(param_name, other_trajectory,
                                                    new_run_idx=run_idx)
                split_name = next_name.split('.')
                link_name = split_name.pop()
                location_name = '.'.join(split_name)
                if not self.f_contains(location_name, shortcuts=False):
                    the_group = self.f_add_group(location_name)
                else:
                    the_group = self.f_get(location_name)

                the_group.f_add_link(link_name, first_param)

        for param_name in other_derived_parameters:
            if param_name in ignore_data:
                continue
            split_name = param_name.split('.')
            ignore_data.add(param_name)
            if any(x in other_trajectory._reversed_wildcards and x not in allowed_translations
                        for x in split_name):
                continue
            new_name = self._rename_full_name(param_name, other_trajectory,
                                                    used_runs=used_runs)
            if self.f_contains(new_name):
                my_param = self.f_get(new_name, fast_access=False)
                param = other_derived_parameters[param_name]
                if (my_param._equal_values(my_param.f_get(), param.f_get()) and
                        not (my_param.f_has_range() or param.f_has_range())):
                    continue
                else:
                    self._logger.error('Could not merge parameter `%s`. '
                                       'I will ignore it!' % new_name)
            rename_dict[param_name] = new_name


    def _merge_links(self, other_trajectory, used_runs, allowed_translations, ignore_data):
        """ Merges all links"""
        linked_items = other_trajectory._linked_by
        run_name_dummys = set([f(-1) for f in other_trajectory._wildcard_functions.values()])
        if len(linked_items) > 0:
            self._logger.info('Merging potential links!')
            for old_linked_name in other_trajectory._linked_by:
                if old_linked_name in ignore_data:
                    continue
                split_name = old_linked_name.split('.')
                if any(x in run_name_dummys for x in split_name):
                    self._logger.warning('Ignoring all links linking to `%s` because '
                                     'I don`t know how to resolve links under `%s` nodes.' %
                                     (old_linked_name, str(run_name_dummys)))
                    continue
                old_link_dict = other_trajectory._linked_by[old_linked_name]
                split_name = old_linked_name.split('.')
                if all(x in allowed_translations for x in split_name):
                    new_linked_full_name = self._rename_full_name(old_linked_name,
                                                                  other_trajectory,
                                                                  used_runs=used_runs)
                else:
                    new_linked_full_name = old_linked_name

                for linking_node, link_set in old_link_dict.values():

                    linking_full_name = linking_node.v_full_name
                    split_name = linking_full_name .split('.')
                    if any(x in run_name_dummys for x in split_name):
                        self._logger.warning('Ignoring links under `%s` because '
                                             'I don`t know how to resolve links '
                                             'under a `%s` node.' %
                                             (linking_full_name, str(run_name_dummys)))

                    split_name = linking_full_name .split('.')
                    if any(x in allowed_translations for x in split_name):
                        new_linking_full_name = self._rename_full_name(linking_full_name,
                                                                      other_trajectory,
                                                                      used_runs=used_runs)
                    else:
                        new_linking_full_name = linking_full_name

                    for link in link_set:
                        if (linking_full_name + '.' + link) in ignore_data:
                            continue
                        if link in run_name_dummys:
                            self._logger.warning('Ignoring link `%s` under `%s` because '
                                                 'I don`t know how to resolve '
                                                 'links named as `%s`.' %
                                                 (link,
                                                  linking_full_name,
                                                  str(run_name_dummys)))
                            continue

                        try:
                            new_linked_item = self.f_get(new_linked_full_name,
                                                         shortcuts=False)
                            if self.f_contains(new_linking_full_name):
                                new_linking_item = self.f_get(new_linking_full_name,
                                                              shortcuts=False)
                            else:
                                new_linking_item =  self.f_add_group(new_linking_full_name)
                            if link in allowed_translations:
                                run_indices, wildcards = other_trajectory._reversed_wildcards[link]
                                link = self.f_wildcard(wildcards[0], used_runs[run_indices[0]])
                            if not link in new_linking_item._links:
                                new_linking_item.f_add_link(link, new_linked_item)
                            else:
                                self._logger.debug('Link `%s` exists already under `%s`.' %
                                                    (link, new_linked_item.v_full_name))
                        except (AttributeError, ValueError) as exc:
                            self._logger.error('Could not copy link `%s` under `%s` linking '
                                               'to `%s` due to `%s`' %
                                               (link, linking_full_name, old_linked_name,
                                                repr(exc)))

    def _merge_config(self, other_trajectory):
        """Merges meta data about previous merges, git commits, and environment settings
        of the other trajectory into the current one.

        """
        self._logger.info('Merging config!')

        # Merge git commit meta data
        if 'config.git' in other_trajectory:

            self._logger.info('Merging git commits!')
            git_node = other_trajectory.f_get('config.git')
            param_list = []
            for param in git_node.f_iter_leaves(with_links=False):
                if not self.f_contains(param.v_full_name, shortcuts=False):
                    param_list.append(self.f_add_config(param))

            if param_list:
                self.f_store_items(param_list)

            self._logger.info('Merging git commits successful!')

        # Merge environment meta data
        if 'config.environment' in other_trajectory:

            self._logger.info('Merging environment config!')
            env_node = other_trajectory.f_get('config.environment')
            param_list = []
            for param in env_node.f_iter_leaves(with_links=False):
                if not self.f_contains(param.v_full_name, shortcuts=False):
                    param_list.append(self.f_add_config(param))

            if param_list:
                self.f_store_items(param_list)

            self._logger.info('Merging config successful!')

        # Merge meta data of previous merges
        if 'config.merge' in other_trajectory:

            self._logger.info('Merging merge config!')
            merge_node = other_trajectory.f_get('config.merge')
            param_list = []
            for param in merge_node.f_iter_leaves(with_links=False):
                if not self.f_contains(param.v_full_name, shortcuts=False):
                    param_list.append(self.f_add_config(param))

            if param_list:
                self.f_store_items(param_list)

            self._logger.info('Merging config successful!')

    def _merge_slowly(self, other_trajectory, rename_dict):
        """Merges trajectories by loading iteratively items of the other trajectory and
        store it into the current trajectory.

        :param rename_dict:

            Dictionary containing mappings from the old result names in the `other_trajectory`
            to the new names in the current trajectory.

        """
        for other_key in rename_dict:
            new_key = rename_dict[other_key]

            other_instance = other_trajectory.f_get(other_key)

            if other_instance.f_is_empty():
                # To suppress warnings if nothing needs to be loaded
                with self._nn_interface._disable_logging:
                    other_trajectory.f_load_item(other_instance)

            if not self.f_contains(new_key):
                class_name = other_instance.f_get_class_name()
                class_ = self._create_class(class_name)
                my_instance = self.f_add_leaf(class_, new_key)
            else:
                my_instance = self.f_get(new_key, shortcuts=False)

            if not my_instance.f_is_empty():
                raise RuntimeError('Something is wrong! Your item `%s` should be empty.' % new_key)

            load_dict = other_instance._store()
            my_instance._load(load_dict)
            my_instance.f_set_annotations(**other_instance.v_annotations.f_to_dict(copy=False))
            my_instance.v_comment = other_instance.v_comment

            self.f_store_item(my_instance)

            # We do not want to blow up the RAM Memory
            if other_instance.v_is_parameter:
                other_instance.f_unlock()
                my_instance.f_unlock()
            other_instance.f_empty()
            my_instance.f_empty()

    def _merge_results(self, other_trajectory, rename_dict, used_runs, allowed_translations,
                       ignore_data):
        """Merges all results.

        :param rename_dict:

            Dictionary that is filled with the names of results in the `other_trajectory`
            as keys and the corresponding new names in the current trajectory as values.
            Note for results kept under trajectory run branch there is actually no need to
            change the names. So we will simply keep the original name.

        """

        other_results = other_trajectory._results.copy()

        for result_name in other_results:
            if result_name in ignore_data:
                continue
            split_name = result_name.split('.')
            ignore_data.add(result_name)
            if any(x in other_trajectory._reversed_wildcards and x not in allowed_translations
                        for x in split_name):
                continue
            new_name = self._rename_full_name(result_name, other_trajectory,
                                                    used_runs=used_runs)
            if self.f_contains(new_name):
                self._logger.warning('I found result `%s` already, I will ignore it.' % new_name)
                continue
            rename_dict[result_name] = new_name

    def _merge_parameters(self, other_trajectory, remove_duplicates=False,
                          trial_parameter_name=None,
                          ignore_data=()):
        """Merges parameters from the other trajectory into the current one.

        The explored parameters in the current trajectory are directly enlarged (in RAM),
        no storage service is needed here. Later on in `f_merge` the storage service
        will be requested to store the enlarge parameters to disk.

        Note explored parameters are always enlarged. Unexplored parameters might become
        new explored parameters if they differ in their default values
        in the current and the other trajectory, respectively.

        :return: A tuple with two elements:

            1.

                Dictionary of run index mappings from old trajectroy to the new one.

            2.

                List of names of parameters that were altered.

        """

        if trial_parameter_name:
            if remove_duplicates:
                self._logger.warning('You have given a trial parameter and you want to '
                                     'remove_items duplicates. There cannot be any duplicates '
                                     'when adding trials, I will not look for duplicates.')
                remove_duplicates = False

        # Dictionary containing full parameter names as keys
        # and pairs of parameters from both trajectories as values.
        # Parameters kept in this dictionary are marked for merging and will be enlarged
        # with ranges and values of corresponding parameters in the other trajectory
        params_to_change = {}

        if trial_parameter_name:
            # We want to merge a trial parameter
            # First make some sanity checks
            my_trial_parameter = self.f_get(trial_parameter_name)
            other_trial_parameter = other_trajectory.f_get(trial_parameter_name)
            if not isinstance(my_trial_parameter, BaseParameter):
                raise TypeError('Your trial_parameter `%s` does not evaluate to a real parameter'
                                ' in the trajectory' % trial_parameter_name)

            # Extract the ranges of both trial parameters
            if my_trial_parameter.f_has_range():
                my_trial_list = my_trial_parameter.f_get_range(copy=False)
            else:
                # If we only have a single trial, we need to make a range of length 1
                # This is probably a very exceptional case
                my_trial_list = [my_trial_parameter.f_get()]

            if other_trial_parameter.f_has_range():
                other_trial_list = other_trial_parameter.f_get_range(copy=False)
            else:
                other_trial_list = [other_trial_parameter.f_get()]

            # Make sanity checks if both ranges contain all numbers from 0 to T1
            # for the current trajectory and 0 to T2 for the other trajectory
            mytrialset = set(my_trial_list)
            mymaxtrial_T1 = max(mytrialset)  # maximum trial index in current trajectory aka T1

            if mytrialset != set(range(mymaxtrial_T1 + 1)):
                raise TypeError('In order to specify a trial parameter, this parameter must '
                                'contain integers from 0 to %d, but it in fact it '
                                'contains `%s`.' % (mymaxtrial_T1, str(mytrialset)))

            othertrialset = set(other_trial_list)
            othermaxtrial_T2 = max(othertrialset)  # maximum trial index in other trajectory aka T2
            if othertrialset != set(range(othermaxtrial_T2 + 1)):
                raise TypeError('In order to specify a trial parameter, this parameter must '
                                'contain integers from 0 to %d, but it infact it contains `%s` '
                                'in the other trajectory.' %
                                (othermaxtrial_T2, str(othertrialset)))

            # If the trial parameter's name was just given in parts we update it here
            # to the full name
            trial_parameter_name = my_trial_parameter.v_full_name

            # If we had the very exceptional case, that our trial parameter was not explored,
            # aka we only had 1 trial, we have to add it to the explored parameters
            if not trial_parameter_name in self._explored_parameters:
                self._explored_parameters[trial_parameter_name] = my_trial_parameter

            # We need to mark the trial parameter for merging
            params_to_change[trial_parameter_name] = (my_trial_parameter, other_trial_parameter)

        # Dictionary containing all parameters of the other trajectory, we will iterate through it
        # to spot parameters that need to be enlarge or become new explored parameters
        params_to_merge = other_trajectory._parameters.copy()

        params_to_merge.update(other_trajectory._derived_parameters)
        for ignore in ignore_data:
            if ignore in params_to_merge:
                del params_to_merge[ignore]

        run_name_dummys = set([f(-1) for f in other_trajectory._wildcard_functions.values()])
        # Iterate through all parameters of the other trajectory
        # and check which differ from the parameters of the current trajectory
        for key in params_to_merge:
            other_param = params_to_merge[key]

            # We don't need to merge anything based on wildcards
            split_key = key.split('.')
            if any(x in other_trajectory._reversed_wildcards for x in split_key):
                continue

            my_param = self.f_get(key)
            if not my_param._values_of_same_type(my_param.f_get(), other_param.f_get()):
                raise TypeError('The parameters with name `%s` are not of the same type, cannot '
                                'merge trajectory.' % key)

            # We have taken care about the trial parameter before, it is already
            # marked for merging
            if my_param.v_full_name == trial_parameter_name:
                continue

            # If a parameter was explored in one of the trajectories or two unexplored
            # parameters differ, we need to mark them for merge
            if (my_param.f_has_range() or
                    other_param.f_has_range() or
                    not my_param._equal_values(my_param.f_get(), other_param.f_get())):

                # If two unexplored parameters differ, that means they differ in every run,
                # accordingly we do not need to check for duplicate runs anymore
                params_to_change[key] = (my_param, other_param)
                if not my_param.f_has_range() and not other_param.f_has_range():
                    remove_duplicates = False

        # Check if we use all runs or remove duplicates:
        used_runs = {}
        for idx in range(len(other_trajectory)):
            used_runs[idx] = idx
        if remove_duplicates:

            # We need to compare all parameter combinations in the current trajectory
            # to all parameter combinations in the other trajectory to spot duplicate points.
            # Quadratic Complexity!
            for irun in range(len(other_trajectory)):
                for jrun in range(len(self)):
                    change = True

                    # Check all marked parameters
                    for my_param, other_param in params_to_change.values():
                        if other_param.f_has_range():
                            other_param._set_parameter_access(irun)

                        if my_param.f_has_range():
                            my_param._set_parameter_access(jrun)

                        val1 = my_param.f_get()
                        val2 = other_param.f_get()

                        # If only one parameter differs, the parameter space point differs
                        # and we can skip the rest of the parameters
                        if not my_param._equal_values(val1, val2):
                            change = False
                            break

                    # If we found one parameter space point in the current trajectory
                    # that matches the ith point in the other, we do not need the ith
                    # point. We can also skip comparing to the rest of the points in the
                    # current trajectory
                    if change:
                        del used_runs[irun]
                        break

            # Restore changed default values
            for my_param, other_param in params_to_change.values():
                other_param._restore_default()
                my_param._restore_default()


        # Merge parameters into the current trajectory
        adding_length = len(used_runs)
        starting_length = len(self)
        if adding_length == 0:
            return used_runs, []
        count = 0
        for key in sorted(used_runs.keys()):
            used_runs[key] = starting_length + count
            count += 1

        for my_param, other_param in params_to_change.values():
            fullname = my_param.v_full_name

            # We need new ranges to enlarge all parameters marked for merging
            if fullname == trial_parameter_name:
                # The trial parameter now has to cover the range 0 to T1+T2+1
                other_range = [x + mymaxtrial_T1 + 1 for x in other_trial_list]
            else:
                # In case we do not use all runs we need to filter the ranges of the
                # parameters of the other trajectory
                if other_param.f_has_range():
                    other_range = (x for jdx, x in enumerate(other_param.f_get_range(copy=False))
                                                             if jdx in used_runs)
                else:
                    other_range = (other_param.f_get() for _ in range(adding_length))

            # If a parameter in the current trajectory was marked for merging but was not
            # explored before, we need to explore it first, simply by creating the range of
            # the current trajectory's length containing only it's default value
            if not my_param.f_has_range():
                my_param.f_unlock()
                my_param._explore((my_param.f_get() for _ in  range(len(self))))

            # After determining the new range extension `other_range`,
            # expand the parameters
            my_param.f_unlock()
            my_param._expand(other_range)

            if not fullname in self._explored_parameters:
                self._explored_parameters[fullname] = my_param

        return used_runs, list(params_to_change.keys())

    @not_in_run
    def f_migrate(self, new_name=None, in_store=False,
                  new_storage_service=None, **kwargs):
        """Can be called to rename and relocate the trajectory.

        :param new_name: New name of the trajectory, None if you do not want to change the name.

        :param in_store:

            Set this to True if the trajectory has been stored with the new name at the new
            file before  and you just want to "switch back" to the location.
            If you migrate to a store used before and you do not set `in_store=True`,
            the storage service will throw a RuntimeError in case you store the Trajectory
            because it will assume that you try to store a new trajectory that accidentally has
            the very same name as another trajectory. If set to `True` and trajectory is not found
            in the file, the trajectory is simply stored to the file.

        :param new_storage_service:

            New service where you want to migrate to. Leave none if you want to keep the olde one.

        :param kwargs:

            Additional keyword arguments passed to the service.
            For instance, to change the file of the trajectory use ``filename='my_new_file.hdf5``.

        """
        if new_name is not None:
            self._name = new_name

        unused_kwargs = set(kwargs.keys())
        if new_storage_service is not None or len(kwargs) > 0:
            self._storage_service, unused_kwargs = storage_factory(
                                                    storage_service=new_storage_service,
                                                    trajectory=self, **kwargs)
        if len(unused_kwargs) > 0:
            raise ValueError('The following keyword arguments were not used: `%s`' %
                             str(unused_kwargs))

        self._stored = in_store

    @kwargs_api_change('new_filename')
    @kwargs_api_change('new_name')
    @kwargs_mutual_exclusive('only_init', 'recursive', lambda x: not x)
    def f_store(self, only_init=False, store_data=pypetconstants.STORE_DATA,
                max_depth=None):
        """ Stores the trajectory to disk and recursively all data in the tree.

        :param only_init:

            If you just want to initialise the store. If yes, only meta information about
            the trajectory is stored and none of the groups/leaves within the trajectory.
            Alternatively, you can pass `recursive=False`.

        :param store_data:

            Only considered if ``only_init=False``. Choose of the following:

                * :const:`pypet.pypetconstants.STORE_NOTHING`: (0)

                    Nothing is store.

                * :const:`pypet.pypetconstants.STORE_DATA_SKIPPING`: (1)

                    Speedy version of normal ``STORE_DATA`` will entirely skip groups
                    (but not their children) and leaves if they have been stored before.
                    No new data is added in this case.

                * :const:`pypet.pypetconstants.STORE_DATA`: (2)

                    Stores every group and leave node. If they contain data that is not yet stored
                    to disk it is added.

                * :const:`pypet.pypetconstants.OVERWRITE_DATA`: (3)

                    Stores all groups and leave nodes and will delete all data on disk
                    and overwrite it with the current data in RAM.

                    **NOT RECOMMENDED**! Overwriting data on disk fragments the HDF5 file and
                    yields badly compressed large files. Better stick to the concept
                    write once and read many!


        If you use the HDF5 Storage Service usually (STORE_DATA (2)) only novel data
        is stored to disk.
        If you have results that have been stored to disk before only new data items are added and
        already present data is NOT overwritten.

        Overwriting (OVERWRITE_DATA (3)) existing data with the HDF5 storage service
        is not recommended due to fragmentation of the HDF5 file. Better stick to the concept
        write once, but read often.

        If you want to store individual parameters or results, you might want to
        take a look at :func:`~pypet.Trajectory.f_store_items`.
        To store whole subtrees of your trajectory check out
        :func:`~pypet.naturalnaming.NNGroupNode.f_store_child`.
        Note both functions require that your trajectory was stored to disk with `f_store`
        at least once before.


        **ATTENTION**: Calling `f_store` during a single run the behavior is different.

        To avoid re-storing the full trajectory in every single run, which is redundant,
        only sub-trees of the trajectory are really stored.

        The storage serivce looks for new data that is added below groups called `run_XXXXXXXXXX`
        and stores it where `XXXXXXXXX` is the index of this run. The `only_init` parameter is
        ignored in this case. You can avoid this behavior by using the argument from below.

        :param max_depth:

            Maximum depth to store tree (inclusive). During single runs `max_depth` is also counted
            from root.


        """
        if self._is_run:
            if self._new_nodes or self._new_links:
                self._storage_service.store(pypetconstants.SINGLE_RUN, self,
                                        trajectory_name=self.v_name,
                                        recursive=not only_init,
                                        store_data=store_data,
                                        max_depth=max_depth)
        else:
            self._storage_service.store(pypetconstants.TRAJECTORY, self,
                                        trajectory_name=self.v_name,
                                        only_init=only_init,
                                        store_data=store_data,
                                        max_depth=max_depth)

            self._stored = True # We do this in case the storage service forgot to tell the
            # Trajectory that it was actually stored

    @not_in_run
    def f_is_empty(self):
        """ Whether no results nor parameters have been added yet to the trajectory
        (ignores config)."""
        return (len(self._parameters) == 0 and
                len(self._derived_parameters) == 0 and
                len(self._results) == 0 and
                len(self._other_leaves) == 0)

    @not_in_run
    def f_restore_default(self):
        """ Restores the default value in all explored parameters and sets the
        v_idx property back to -1 and v_crun to None."""
        self._idx = -1
        self._crun = None
        for param in self._explored_parameters.values():
            if param is not None:
                param._restore_default()

    def _set_explored_parameters_to_idx(self, idx):
        """ Notifies the explored parameters what current point in the parameter space
        they should represent.

        """
        for param in self._explored_parameters.values():
            if param is not None:
                param._set_parameter_access(idx)

    def _make_single_run(self):
        """ Modifies the trajectory for single runs executed by the environment """
        self._is_run = False # to be able to use f_set_crun
        self._new_nodes = OrderedDict()
        self._new_links = OrderedDict()
        self._is_run = True
        return self

    def f_get_run_names(self, sort=True):
        """ Returns a list of run names.

        ONLY useful for a single run during multiprocessing if ``v_full_copy` was set to ``True``.
        Otherwise only the current run is available.

        :param sort:

            Whether to get them sorted, will only require O(N) [and not O(N*log N)] since we
            use (sort of) bucket sort.

        """
        if sort:
            return [self.f_idx_to_run(idx) for idx in range(len(self))]
        else:
            return list(self._run_information.keys())

    def f_get_run_information(self, name_or_idx=None, copy=True):
        """ Returns a dictionary containing information about a single run.

        ONLY useful during a single run if ``v_full_copy` was set to ``True``.
        Otherwise only the current run is available.

        The information dictionaries have the following key, value pairings:

            * completed: Boolean, whether a run was completed

            * idx: Index of a run

            * timestamp: Timestamp of the run as a float

            * time: Formatted time string

            * finish_timestamp: Timestamp of the finishing of the run

            * runtime: Total runtime of the run in human readable format

            * name: Name of the run

            * parameter_summary:

                A string summary of the explored parameter settings for the particular run

            * short_environment_hexsha: The short version of the environment SHA-1 code


        If no name or idx is given then a nested dictionary with keys as run names and
        info dictionaries as values is returned.

        :param name_or_idx: str or int

        :param copy:

            Whether you want the dictionary used by the trajectory or a copy. Note if
            you want the real thing, please do not modify it, i.e. popping or adding stuff. This
            could mess up your whole trajectory.

        :return:

            A run information dictionary or a nested dictionary of information dictionaries
            with the run names as keys.

        """
        if name_or_idx is None:
            if copy:
                return cp.deepcopy(self._run_information)
            else:
                return self._run_information
        try:
            if copy:
                # Since the information dictionaries only contain immutable items
                # (float, int, str)
                # the normal copy operation is sufficient
                return self._run_information[name_or_idx].copy()
            else:
                return self._run_information[name_or_idx]
        except KeyError:
            # Maybe the user provided an idx, this would yield a key error and we
            # have to convert it to a run name
            name_or_idx = self.f_idx_to_run(name_or_idx)
            if copy:
                return self._run_information[name_or_idx].copy()
            else:
                return self._run_information[name_or_idx]

    def f_find_idx(self, name_list, predicate):
        """ Finds a single run index given a particular condition on parameters.

        ONLY useful for a single run if ``v_full_copy` was set to ``True``.
        Otherwise a TypeError is thrown.

        :param name_list:

            A list of parameter names the predicate applies to, if you have only a single
            parameter name you can omit the list brackets.

        :param predicate:

            A lambda predicate for filtering that evaluates to either ``True`` or  ``False``

        :return: A generator yielding the matching single run indices

        Example:

        >>> predicate = lambda param1, param2: param1==4 and param2 in [1.0, 2.0]
        >>> iterator = traj.f_find_idx(['groupA.param1', 'groupA.param2'], predicate)
        >>> [x for x in iterator]
        [0, 2, 17, 36]

        """
        if self._is_run and not self.v_full_copy:
            raise TypeError('You cannot use this function during a multiprocessing signle run and '
                            'not having ``v_full_copy=True``.')
        if isinstance(name_list, str):
            name_list = [name_list]

        # First create a list of iterators, each over the range of the matched parameters
        iter_list = []
        for name in name_list:
            param = self.f_get(name)
            if not param.v_is_parameter:
                raise TypeError('`%s` is not a parameter it is a %s, find idx is not applicable' %
                                (name, str(type(param))))

            if param.f_has_range():
                iter_list.append(iter(param.f_get_range(copy=False)))
            else:
                iter_list.append(itools.repeat(param.f_get(), len(self)))

        # Create a logical iterator returning `True` or `False`
        # whether the user's predicate matches the parameter data
        logic_iter = map(predicate, *iter_list)

        # Now the run indices are the the indices where `logic_iter` evaluates to `True`
        for idx, item in enumerate(logic_iter):
            if item:
                yield idx

    def f_idx_to_run(self, name_or_idx):
        """Converts an integer idx to the corresponding single run name and vice versa.

        Note during a single run ONLY useful if ``v_full_copy`` was set to True.

        :param name_or_idx: Name of a single run or an integer index

        :return: The corresponding idx or name of the single run

        Example usage:

        >>> traj.f_idx_to_run(4)
        'run_00000004'
        >>> traj.f_idx_to_run('run_00000000')
        0

        """
        return self._single_run_ids[name_or_idx]

    def f_start_run(self, run_name_or_idx=None, turn_into_run=True):
        """ Can be used to manually allow running of an experiment without using an environment.

        :param run_name_or_idx:

            Can manually set a trajectory to a particular run. If `None` the current run
            the trajectory is set to is used.

        :param turn_into_run:

            Turns the trajectory into a run, i.e. reduces functionality but makes storing
            more efficient.

        """
        if self._run_started:
            return self

        if run_name_or_idx is None:
            if self.v_idx == -1:
                raise ValueError('Cannot start run if trajectory is not set to a particular run')
        else:
            self.f_set_crun(run_name_or_idx)

        self._run_started = True

        if turn_into_run:
            self._make_single_run()

        self._set_start()

        return self

    def f_finalize_run(self, store_meta_data=True, clean_up=True):
        """ Can be called to finish a run if manually started.

        Does NOT reset the index of the run,
        i.e. ``f_restore_default`` should be called manually if desired.

        Does NOT store any data (except meta data) so you have to call
        ``f_store`` manually before to avoid data loss.

        :param store_meta_data:

            If meta data like the runtime should be stored

        :param clean_up:

            If data added during the run should be cleaned up.
            Only works if ``turn_into_run`` was set to ``True``.

        """
        if not self._run_started:
            return self

        self._set_finish()

        if clean_up and self._is_run:
            self._finalize_run()

        self._is_run = False
        self._run_started = False

        self._updated_run_information.add(self.v_idx)

        if store_meta_data:
            self.f_store(only_init=True)

        return self

    def _set_start(self):
        """ Sets the start timestamp and formatted time to the current time. """
        init_time = time.time()
        formatted_time = datetime.datetime.fromtimestamp(init_time).strftime('%Y_%m_%d_%Hh%Mm%Ss')
        run_info_dict = self._run_information[self.v_crun]
        run_info_dict['timestamp'] = init_time
        run_info_dict['time'] = formatted_time
        if self._environment_hexsha is not None:
            run_info_dict['short_environment_hexsha'] = self._environment_hexsha[0:7]

    def _summarize_explored_parameters(self):
        """Summarizes the parameter settings.

        :param run_name: Name of the single run

        :param paramlist: List of explored parameters

        :param add_table: Whether to add the overview table

        :param create_run_group:

            If a group with the particular name should be created if it does not exist.
            Might be necessary when trajectories are merged.

        """

        runsummary = ''
        for idx, expparam in enumerate(self._explored_parameters.values()):

            # Create the run summary for the `run` overview
            if idx > 0:
                runsummary += ',   '

            valstr = expparam.f_val_to_str()

            if len(valstr) >= pypetconstants.HDF5_STRCOL_MAX_COMMENT_LENGTH:
                valstr = valstr[0:pypetconstants.HDF5_STRCOL_MAX_COMMENT_LENGTH - 3]
                valstr += '...'

            if expparam.v_name in runsummary:
                param_name = expparam.v_full_name
            else:
                param_name = expparam.v_name

            runsummary = runsummary + param_name + ': ' + valstr

        return runsummary

    def _set_finish(self):
        """ Sets the finish time and computes the runtime in human readable format """

        run_info_dict = self._run_information[self.v_crun]
        timestamp_run = run_info_dict['timestamp']

        run_summary = self._summarize_explored_parameters()

        finish_timestamp_run = time.time()

        findatetime = datetime.datetime.fromtimestamp(finish_timestamp_run)
        startdatetime = datetime.datetime.fromtimestamp(timestamp_run)

        runtime_run = str(findatetime - startdatetime)

        run_info_dict['parameter_summary'] = run_summary
        run_info_dict['completed'] = 1
        run_info_dict['finish_timestamp'] = finish_timestamp_run
        run_info_dict['runtime'] = runtime_run

    def _construct_instance(self, constructor, full_name, *args, **kwargs):
        """ Creates a new node. Checks if the new node needs to know the trajectory.

        :param constructor:  The constructor to use
        :param full_name: Full name of node
        :param args: Arguments passed to constructor
        :param kwargs: Keyword arguments passed to the constructor
        :return:
        """
        if getattr(constructor, 'KNOWS_TRAJECTORY', False):
            return constructor(full_name, self, *args, **kwargs)
        else:
            return constructor(full_name, *args, **kwargs)

    def _create_class(self, class_name):
        """Dynamically creates a class.

        It is tried if the class can be created by the already given imports.
        If not the list of the dynamically loaded classes is used.

        """
        return dynamicimports.create_class(class_name, self._dynamic_imports)

    @property
    def v_shortcuts(self):
        """Whether shortcuts are allowed if accessing data via natural naming or
        squared bracket indexing."""
        return self._shortcuts

    @v_shortcuts.setter
    def v_shortcuts(self, shortcuts):
        self._shortcuts = bool(shortcuts)

    @property
    def v_max_depth(self):
        """The maximum depth the tree should be searched if shortcuts are allowed.

        Set to `None` if there should be no depth limit.
        """
        return self._max_depth

    @v_max_depth.setter
    def v_max_depth(self, max_depth):
        self._max_depth = max_depth

    @property
    def v_iter_recursive(self):
        """Whether using `__iter__` should iterate only immediate children or
        recursively all nodes."""
        return self._iter_recursive

    @v_iter_recursive.setter
    def v_iter_recursive(self, iter_recursive):
        self._iter_recursive = bool(iter_recursive)


    @property
    def v_auto_load(self):
        """Whether the trajectory should attempt to load data on the fly.
        """
        return self._auto_load

    @v_auto_load.setter
    def v_auto_load(self, auto_load):
        self._auto_load = bool(auto_load)

    @property
    def v_timestamp(self):
        """Float timestamp of creation time"""
        return self._timestamp

    @property
    def v_time(self):
        """Formatted time string of the time the trajectory or run was created."""
        return self._time

    @property
    def v_standard_parameter(self):
        """ The standard parameter used for parameter creation"""
        return self._standard_parameter

    @v_standard_parameter.setter
    def v_standard_parameter(self, parameter):
        """Sets the standard parameter"""
        self._standard_parameter = parameter

    @property
    def v_standard_result(self):
        """The standard result class used for result creation """
        return self._standard_result

    @v_standard_result.setter
    def v_standard_result(self, result):
        """Sets standard result"""
        self._standard_result = result

    @property
    def v_standard_leaf(self):
        """The standard constructor used if you add a generic leaf.

        The constructor is only used if you do not add items under the usual four subtrees
        (`parameters`, `derived_parameters`, `config`, `results`).

        """
        return self._standard_leaf

    @v_standard_leaf.setter
    def v_standard_leaf(self, leaf):
        """Sets standard result"""
        self._standard_leaf = leaf

    @property
    def v_no_clobber(self):
        """If `f_add_leaf` should not throw an error in case something is added
        that is already part of the Trajectory. If `True` no error is thrown and
        the new data is ignored."""
        return self._no_clobber

    @v_no_clobber.setter
    def v_no_clobber(self, value):
        """Sets no_clobber"""
        self._no_clobber = bool(value)

    @property
    def v_auto_run_prepend(self):
        """If during run the `runs.run_XXXXXXXX` should be prepended if it is missing.

        Is not considered for `f_add_leaf` and `f_add_group` which never prepend.

        """
        return self._auto_run_prepend

    @v_auto_run_prepend.setter
    def v_auto_run_prepend(self, value):
        """Sets fast access"""
        self._auto_run_prepend = bool(value)

    @property
    def v_fast_access(self):
        """Whether parameter instances (False) or their values (True) are returned via natural
        naming.

        Works also for results if they contain a single item with the name of the result.

        Default is True.

        """
        return self._fast_access

    @v_fast_access.setter
    def v_fast_access(self, value):
        """Sets fast access"""
        self._fast_access = bool(value)

    # @property
    # def v_fast_adding(self):
    #     """If you can add items to the trajectory simply by calling ``setattr``
    #
    #     For example, ``traj.par.a = 4`` would add a parameter named `a` with the value 4.
    #
    #     """
    #     return self._fast_adding
    #
    # @v_fast_adding.setter
    # def v_fast_adding(self, value):
    #     """Sets fast adding"""
    #     self._fast_adding = bool(value)

    @property
    def v_environment_hexsha(self):
        """If the trajectory is used with an environment this returns
        the SHA-1 code of the environment.

        """
        return self._environment_hexsha

    @property
    def v_environment_name(self):
        """If the trajectory is used with an environment this returns
        the name of the environment.

        """
        return self._environment_name

    @staticmethod
    def _return_item_dictionary(param_dict, fast_access, copy):
        """Returns a dictionary containing either all parameters, all explored parameters,
        all config, all derived parameters, or all results.

        :param param_dict: The dictionary which is about to be returned
        :param fast_access: Whether to use fast access
        :param copy: If the original dict should be returned or a shallow copy

        :return: The dictionary

        :raises: ValueError if `copy=False` and fast_access=True`

        """

        if not copy and fast_access:
            raise ValueError('You cannot access the original dictionary and use fast access at the'
                             ' same time!')
        if not fast_access:
            if copy:
                return param_dict.copy()
            else:
                return param_dict
        else:
            resdict = {}
            for key in param_dict:
                param = param_dict[key]

                val = param.f_get()
                resdict[key] = val

            return resdict

    def _finalize_run(self):
        """Called by the environment after storing to perform some rollback operations.

        All results and derived parameters created in the current run are removed.

        Important for single processing to not blow up the parent trajectory with the results
        of all runs.

        """
        self._run_information[self.v_crun]['completed'] = 1
        while len(self._new_links):
            name_pair, child_parent_pair = self._new_links.popitem(last=False)
            parent_node, _ = child_parent_pair
            _, link = name_pair
            parent_node.f_remove_child(link)

        while len(self._new_nodes):
            _, child_parent_pair = self._new_nodes.popitem(last=False)
            parent, child = child_parent_pair
            child_name = child.v_name
            parent.f_remove_child(child_name, recursive=True)

    def f_to_dict(self, fast_access=False, short_names=False, nested=False,
                  copy=True, with_links=True):
        """Returns a dictionary with pairings of (full) names as keys and instances/values.


        :param fast_access:

            If True, parameter values are returned instead of the instances.
            Works also for results if they contain a single item with the name of the result.

        :param short_names:

            If true, keys are not full names but only the names. Raises a ValueError
            if the names are not unique.

        :param nested:

            If true, a nested dictionary is returned.

        :param copy:

            If `fast_access=False` and `short_names=False` you can access the original
            data dictionary if you set `copy=False`. If you do that, please do not
            modify anything! Raises ValueError if `copy=False` and `fast_access=True`
            or `short_names=True`.

        :param with_links:

            If links should be ignored

        :return: dictionary

        :raises: ValueError

        """
        return self._nn_interface._to_dict(self, fast_access=fast_access,
                                           short_names=short_names,
                                           nested=nested,
                                           copy=copy, with_links=with_links)

    def f_get_config(self, fast_access=False, copy=True):
        """Returns a dictionary containing the full config names as keys and the config parameters
        or the config parameter data items as values.


        :param fast_access:

            Determines whether the parameter objects or their values are returned
            in the dictionary.

        :param copy:

            Whether the original dictionary or a shallow copy is returned.
            If you want the real dictionary please do not modify it at all!
            Not Copying and fast access do not work at the same time! Raises ValueError
            if fast access is true and copy false.

        :return: Dictionary containing the config data

        :raises: ValueError

        """
        return self._return_item_dictionary(self._config, fast_access, copy)

    def f_get_parameters(self, fast_access=False, copy=True):
        """ Returns a dictionary containing the full parameter names as keys and the parameters
         or the parameter data items as values.


        :param fast_access:

            Determines whether the parameter objects or their values are returned
            in the dictionary.

        :param copy:

            Whether the original dictionary or a shallow copy is returned.
            If you want the real dictionary please do not modify it at all!
            Not Copying and fast access do not work at the same time! Raises ValueError
            if fast access is true and copy false.

        :return: Dictionary containing the parameters.

        :raises: ValueError

        """
        return self._return_item_dictionary(self._parameters, fast_access, copy)


    def f_get_explored_parameters(self, fast_access=False, copy=True):
        """ Returns a dictionary containing the full parameter names as keys and the parameters
         or the parameter data items as values.

         IMPORTANT: This dictionary always contains all explored parameters as keys.
         Even when they are not loaded, in this case the value is simply `None`.
         `fast_access` only works if all explored parameters are loaded.


        :param fast_access:

            Determines whether the parameter objects or their values are returned
            in the dictionary.

        :param copy:

            Whether the original dictionary or a shallow copy is returned.
            If you want the real dictionary please do not modify it at all!
            Not Copying and fast access do not work at the same time! Raises ValueError
            if fast access is true and copy false.

        :return: Dictionary containing the parameters.

        :raises: ValueError

        """
        return self._return_item_dictionary(self._explored_parameters, fast_access, copy)

    def f_get_derived_parameters(self, fast_access=False, copy=True):
        """ Returns a dictionary containing the full parameter names as keys and the parameters
         or the parameter data items as values.


        :param fast_access:

            Determines whether the parameter objects or their values are returned
            in the dictionary.

        :param copy:

            Whether the original dictionary or a shallow copy is returned.
            If you want the real dictionary please do not modify it at all!
            Not Copying and fast access do not work at the same time! Raises ValueError
            if fast access is true and copy false.

        :return: Dictionary containing the parameters.

        :raises: ValueError

        """
        return self._return_item_dictionary(self._derived_parameters, fast_access, copy)

    def f_get_results(self, fast_access=False, copy=True):
        """ Returns a dictionary containing the full result names as keys and the corresponding
        result objects or result data items as values.


        :param fast_access:

            Determines whether the result objects or their values are returned
            in the dictionary. Works only for results if they contain a single item with
            the name of the result.

        :param copy:

            Whether the original dictionary or a shallow copy is returned.
            If you want the real dictionary please do not modify it at all!
            Not Copying and fast access do not work at the same time! Raises ValueError
            if fast access is true and copy false.

        :return: Dictionary containing the results.

        :raises: ValueError

        """
        return self._return_item_dictionary(self._results, fast_access, copy)

    def f_store_item(self, item, *args, **kwargs):
        """Stores a single item, see also :func:`~pypet.trajectory.Trajectory.f_store_items`."""
        self.f_store_items([item], *args, **kwargs)

    def f_store_items(self, iterator, *args, **kwargs):
        """Stores individual items to disk.

        This function is useful if you calculated very large results (or large derived parameters)
        during runtime and you want to write these to disk immediately and empty them afterwards
        to free some memory.

        Instead of storing individual parameters or results you can also store whole subtrees with
        :func:`~pypet.naturalnaming.NNGroupNode.f_store_child`.


        You can pass the following arguments to `f_store_items`:

        :param iterator:

            An iterable containing the parameters or results to store, either their
            names or the instances. You can also pass group instances or names here
            to store the annotations of the groups.

        :param non_empties:

            Optional keyword argument (boolean),
            if `True` will only store the subset of provided items that are not empty.
            Empty parameters or results found in `iterator` are simply ignored.

        :param args: Additional arguments passed to the storage service

        :param kwargs:

            If you use the standard hdf5 storage service, you can pass the following additional
            keyword argument:

            :param overwrite:

                List names of parts of your item that should
                be erased and overwritten by the new data in your leaf.
                You can also set `overwrite=True`
                to overwrite all parts.

                For instance:

                    >>> traj.f_add_result('mygroup.myresult', partA=42, partB=44, partC=46)
                    >>> traj.f_store()
                    >>> traj.mygroup.myresult.partA = 333
                    >>> traj.mygroup.myresult.partB = 'I am going to change to a string'
                    >>> traj.f_store_item('mygroup.myresult', overwrite=['partA', 'partB'])

                Will store `'mygroup.myresult'` to disk again and overwrite the parts
                `'partA'` and `'partB'` with the new values `333` and
                `'I am going to change to a string'`.
                The data stored as `partC` is not changed.

                Be aware that you need to specify the names of parts as they were stored
                to HDF5. Depending on how your leaf construction works, this may differ
                from the names the data might have in your leaf in the trajectory container.

                Note that massive overwriting will fragment and blow up your HDF5 file.
                Try to avoid changing data on disk whenever you can.

        :raises:

            TypeError:

                If the (parent) trajectory has never been stored to disk. In this case
                use :func:`pypet.trajectory.f_store` first.

            ValueError: If no item could be found to be stored.

        Note if you use the standard hdf5 storage service, there are no additional arguments
        or keyword arguments to pass!

        """

        if not self._stored:
            raise TypeError('Cannot store stuff for a trajectory that has never been '
                            'stored to disk. Please call traj.f_store(only_init=True) first.')

        fetched_items = self._nn_interface._fetch_items(STORE, iterator, args, kwargs)

        if fetched_items:
            self._storage_service.store(pypetconstants.LIST, fetched_items,
                                        trajectory_name=self.v_name)
        else:
            raise ValueError('Your storage was not successful, could not find a single item '
                             'to store.')

    def f_load_item(self, item, *args, **kwargs):
        """Loads a single item, see also :func:`~pypet.trajectory.Trajectory.f_load_items`"""
        self.f_load_items([item], *args, **kwargs)

    def f_load_items(self, iterator, *args, **kwargs):
        """Loads parameters and results specified in `iterator`.

        You can directly list the Parameter objects or just their names.

        If names are given the `~pypet.naturalnaming.NNGroupNode.f_get` method is applied to find the
        parameters or results in the trajectory. Accordingly, the parameters and results
        you want to load must already exist in your trajectory (in RAM), probably they
        are just empty skeletons waiting desperately to handle data.
        If they do not exist in RAM yet, but have been stored to disk before,
        you can call :func:`~pypet.trajectory.Trajectory.f_load_skeleton` in order to
        bring your trajectory tree skeleton up to date. In case of a single run you can
        use the :func:`~pypet.naturalnaming.NNGroupNode.f_load_child` method to recursively
        load a subtree without any data.
        Then you can load the data of individual results or parameters one by one.

        If want to load the whole trajectory at once or ALL results and parameters that are
        still empty take a look at :func:`~pypet.trajectory.Trajectory.f_load`.
        As mentioned before, to load subtrees of your trajectory you might want to check out
        :func:`~pypet.naturalnaming.NNGroupNode.f_load_child`.

        To load a list of parameters or results with `f_load_items` you can pass
        the following arguments:

        :param iterator: A list with parameters or results to be loaded.

        :param only_empties:

            Optional keyword argument (boolean),
            if `True` only empty parameters or results are passed
            to the storage service to get loaded. Non-empty parameters or results found in
            `iterator` are simply ignored.

        :param args: Additional arguments directly passed to the storage service

        :param kwargs:

            Additional keyword arguments directly passed to the storage service
            (except the kwarg `only_empties`)

            If you use the standard hdf5 storage service, you can pass the following additional
            keyword arguments:

            :param load_only:

                If you load a result, you can partially load it and ignore the rest of data items.
                Just specify the name of the data you want to load. You can also provide a list,
                for example `load_only='spikes'`, `load_only=['spikes','membrane_potential']`.

                Be aware that you need to specify the names of parts as they were stored
                to HDF5. Depending on how your leaf construction works, this may differ
                from the names the data might have in your leaf in the trajectory container.

                A warning is issued if data specified in `load_only` cannot be found in the
                instances specified in `iterator`.

            :param load_except:

                Analogous to the above, but everything is loaded except names or parts
                specified in `load_except`.
                You cannot use `load_only` and `load_except` at the same time. If you do
                a ValueError is thrown.

                A warning is issued if names listed in `load_except` are not part of the
                items to load.

        """

        if not self._stored:
            raise TypeError(
                'Cannot load stuff from disk for a trajectory that has never been stored.')

        fetched_items = self._nn_interface._fetch_items(LOAD, iterator, args, kwargs)
        if fetched_items:
            self._storage_service.load(pypetconstants.LIST, fetched_items,
                                       trajectory_name=self.v_name)
        else:
            self._logger.warning('Your loading was not successful, could not find a single item '
                                 'to load.')

    @kwargs_api_change('remove_empty_groups')
    def f_remove_item(self, item, recursive=False):
        """Removes a single item, see :func:`~pypet.trajectory.Trajectory.f_remove_items`"""
        self.f_remove_items([item], recursive=recursive)

    @kwargs_api_change('remove_empty_groups')
    def f_remove_items(self, iterator, recursive=False):
        """Removes parameters, results or groups from the trajectory.

        This function ONLY removes items from your current trajectory and does not delete
        data stored to disk. If you want to delete data from disk, take a look at
        :func:`~pypet.trajectory.Trajectory.f_delete_items`.

        This will also remove all links if items are linked.

        :param iterator:

            A sequence of items you want to remove. Either the instances themselves
            or strings with the names of the items.

        :param recursive:

            In case you want to remove group nodes, if the children should be removed, too.

        """

        # Will format the request in a form that is understood by the storage service
        # aka (msg, item, args, kwargs)
        fetched_items = self._nn_interface._fetch_items(REMOVE, iterator, (), {})

        if fetched_items:
            for _, item, dummy1, dummy2 in fetched_items:
                self._nn_interface._remove_node_or_leaf(item, recursive=recursive)

        else:
            self._logger.warning('Your removal was not successful, could not find a single '
                                 'item to remove.')

    def f_delete_link(self, link, remove_from_trajectory=False):
        """Deletes a single link see :func:`~pypet.trajectory.Trajectory.f_delete_links`"""
        self.f_delete_links((link,), remove_from_trajectory)

    def f_delete_links(self, iterator_of_links, remove_from_trajectory=False):
        """Deletes several links from the hard disk.

        Links can be passed as a string ``'groupA.groupB.linkA'``
        or as a tuple  containing the node from which the link should be removed and the
        name of the link ``(groupWithLink, 'linkA')``.

        """

        to_delete_links = []

        group_link_pairs = []

        for elem in iterator_of_links:
            if isinstance(elem, str):
                split_names = elem.split('.')
                parent_name = '.'.join(split_names[:-1])
                link = split_names[-1]
                parent_node = self.f_get(parent_name) if parent_name != '' else self
                link_name = parent_node.v_full_name + '.' + link if parent_name != '' else link
                to_delete_links.append((pypetconstants.DELETE_LINK, link_name))
                group_link_pairs.append((parent_node, link))
            else:
                link_name = elem[0].v_full_name + '.' + elem[1]
                to_delete_links.append((pypetconstants.DELETE_LINK, link_name))
                group_link_pairs.append(elem)
        try:
            self._storage_service.store(pypetconstants.LIST, to_delete_links,
                                        trajectory_name=self.v_name)
        except:
            self._logger.error('Could not remove `%s` from the trajectory. Maybe the'
                               ' item(s) was/were never stored to disk.' % str(to_delete_links))
            raise

        if remove_from_trajectory:
            for group, link in group_link_pairs:
                group.f_remove_link(link)

    def f_remove(self, recursive=True, predicate=None):
        """Recursively removes all children of the trajectory

        :param recursive:

            Only here for consistency with signature of parent method. Cannot be set
            to `False` because the trajectory root node cannot be removed.

        :param predicate:

            Predicate which can evaluate for each node to ``True`` in order to remove the node or
            ``False`` if the node should be kept. Leave ``None`` if you want to remove all nodes.

        """
        if not recursive:
            raise ValueError('Nice try ;-)')

        for child in list(self._children.keys()):
            self.f_remove_child(child, recursive=True, predicate=predicate)

    @kwargs_api_change('remove_empty_groups')
    def f_delete_item(self, item, *args, **kwargs):
        """Deletes a single item, see :func:`~pypet.trajectory.Trajectory.f_delete_items`"""
        self.f_delete_items([item], *args, **kwargs)

    @kwargs_api_change('remove_empty_groups')
    def f_delete_items(self, iterator, *args, **kwargs):
        """Deletes items from storage on disk.

        Per default the item is NOT removed from the trajectory.

        Links are NOT deleted on the hard disk, please delete links manually before deleting
        data!

        :param iterator:

            A sequence of items you want to remove. Either the instances themselves
            or strings with the names of the items.

        :param remove_from_trajectory:

            If items should also be removed from trajectory. Default is `False`.


        :param args: Additional arguments passed to the storage service

        :param kwargs: Additional keyword arguments passed to the storage service

            If you use the standard hdf5 storage service, you can pass the following additional
            keyword argument:

            :param delete_only:

                You can partially delete leaf nodes. Specify a list of parts of the result node
                that should be deleted like `delete_only=['mystuff','otherstuff']`.
                This wil only delete the hdf5 sub parts `mystuff` and `otherstuff` from disk.
                BE CAREFUL,
                erasing data partly happens at your own risk. Depending on how complex the
                loading process of your result node is, you might not be able to reconstruct
                any data due to partially deleting some of it.

                Be aware that you need to specify the names of parts as they were stored
                to HDF5. Depending on how your leaf construction works, this may differ
                from the names the data might have in your leaf in the trajectory container.

                If the hdf5 nodes you specified in `delete_only` cannot be found a warning
                is issued.

                Note that massive deletion will fragment your HDF5 file.
                Try to avoid changing data on disk whenever you can.

                If you want to erase a full node, simply ignore this argument or set to `None`.

            :param remove_from_item:

                If data that you want to delete from storage should also be removed from
                the items in `iterator` if they contain these. Default is `False`.

            :param recursive:

                If you want to delete a group node and it has children you need to
                set `recursive` to `True. Default is `False`.

        """

        remove_from_trajectory = kwargs.pop('remove_from_trajectory', False)
        recursive = kwargs.get('recursive', False)

        # Will format the request in a form that is understood by the storage service
        # aka (msg, item, args, kwargs)
        fetched_items = self._nn_interface._fetch_items(REMOVE, iterator, args, kwargs)

        if fetched_items:

            try:
                self._storage_service.store(pypetconstants.LIST, fetched_items,
                                            trajectory_name=self.v_name)
            except:
                self._logger.error('Could not remove `%s` from the trajectory. Maybe the'
                                   ' item(s) was/were never stored to disk.' % str(fetched_items))
                raise


            for _, item, dummy1, dummy2 in fetched_items:
                if remove_from_trajectory:
                    self._nn_interface._remove_node_or_leaf(item, recursive=recursive)
                else:
                    item._stored = False

        else:
            self._logger.warning('Your removal was not successful, could not find a single '
                                 'item to remove.')

    @not_in_run
    @copydoc(ConfigGroup.f_add_config)
    def f_add_config(self, *args, **kwargs):
        return super(Trajectory, self).f_add_config(*args, **kwargs)


    @not_in_run
    @copydoc(ConfigGroup.f_add_config_group)
    def f_add_config_group(self, *args, **kwargs):
        return super(Trajectory, self).f_add_config_group(*args, **kwargs)

    @not_in_run
    @copydoc(ParameterGroup.f_add_parameter)
    def f_add_parameter(self, *args, **kwargs):
        return super(Trajectory, self).f_add_parameter(*args, **kwargs)

    @not_in_run
    @copydoc(ParameterGroup.f_add_parameter_group)
    def f_add_parameter_group(self, *args, **kwargs):
        return super(Trajectory, self).f_add_parameter_group(*args, **kwargs)

    def __dir__(self):
        """Adds all children to auto-completion

        In case of a single run it spares all non-available functions

        """
        result = super(Trajectory, self).__dir__()
        if self._is_run:
            result = [x for x in result if not x.startswith('f_') or
                      not getattr(getattr(self, x), '_not_in_run', False)]
        if not is_debug():
            result.extend(self._children.keys())
        return result class HDF5StorageService(StorageService, HasLogger):
    """Storage Service to handle the storage of a trajectory/parameters/results into hdf5 files.

    Normally you do not interact with the storage service directly but via the trajectory,
    see :func:`pypet.trajectory.Trajectory.f_store` and :func:`pypet.trajectory.Trajectory.f_load`.

    The service is not thread safe. For multiprocessing the service needs to be wrapped either
    by the :class:`~pypet.storageservice.LockWrapper` or with a combination of
    :class:`~pypet.storageservice.QueueStorageServiceSender` and
    :class:`~pypet.storageservice.QueueStorageServiceWriter`.

    The storage service supports two operations *store* and *load*.

    Requests for these two are always passed as
    `msg, what_to_store_or_load, *args, **kwargs`

    For example:

    >>> HDF5StorageService.load(pypetconstants.LEAF, myresult, load_only=['spikestimes','nspikes'])

    For a list of supported items see :func:`~pypet.storageservice.HDF5StorageService.store`
    and :func:`~pypet.storageservice.HDF5StorageService.load`.

    The service accepts the following parameters

    :param filename:

        The name of the hdf5 file. If none is specified the default
        `./hdf5/the_name_of_your_trajectory.hdf5` is chosen. If `filename` contains only a path
        like `filename='./myfolder/', it is changed to
        `filename='./myfolder/the_name_of_your_trajectory.hdf5'`.

    :param file_title: Title of the hdf5 file (only important if file is created new)

    :param overwrite_file:

        If the file already exists it will be overwritten. Otherwise
        the trajectory will simply be added to the file and already
        existing trajectories are not deleted.

    :param encoding:

        Format to encode and decode unicode strings stored to disk.
        The default ``'utf8'`` is highly recommended.

    :param complevel:

        If you use HDF5, you can specify your compression level. 0 means no compression
        and 9 is the highest compression level. See `PyTables Compression`_ for a detailed
        description.

        .. _`PyTables Compression`: http://pytables.github.io/usersguide/optimization.html#compression-issues

    :param complib:

        The library used for compression. Choose between *zlib*, *blosc*, and *lzo*.
        Note that 'blosc' and 'lzo' are usually faster than 'zlib' but it may be the case that
        you can no longer open your hdf5 files with third-party applications that do not rely
        on PyTables.

    :param shuffle:

        Whether or not to use the shuffle filters in the HDF5 library.
        This normally improves the compression ratio.

    :param fletcher32:

        Whether or not to use the *Fletcher32* filter in the HDF5 library.
        This is used to add a checksum on hdf5 data.

    :param pandas_format:

        How to store pandas data frames. Either in 'fixed' ('f') or 'table' ('t') format.
        Fixed format allows fast reading and writing but disables querying the hdf5 data and
        appending to the store (with other 3rd party software other than *pypet*).

    :param purge_duplicate_comments:

        If you add a result via :func:`~pypet.naturalnaming.ResultGroup.f_add_result` or a derived
        parameter :func:`~pypet.naturalnaming.DerivedParameterGroup.f_add_derived_parameter` and
        you set a comment, normally that comment would be attached to each and every instance.
        This can produce a lot of unnecessary overhead if the comment is the same for every
        instance over all runs. If `purge_duplicate_comments=1` than only the comment of the
        first result or derived parameter instance created in a run is stored or comments
        that differ from this first comment.

        For instance, during a single run you call
        `traj.f_add_result('my_result`,42, comment='Mostly harmless!')`
        and the result will be renamed to `results.run_00000000.my_result`. After storage
        in the node associated with this result in your hdf5 file, you will find the comment
        `'Mostly harmless!'` there. If you call
        `traj.f_add_result('my_result',-43, comment='Mostly harmless!')`
        in another run again, let's say run 00000001, the name will be mapped to
        `results.run_00000001.my_result`. But this time the comment will not be saved to disk
        since `'Mostly harmless!'` is already part of the very first result with the name
        'results.run_00000000.my_result'.
        Note that the comments will be compared and storage will only be discarded if the strings
        are exactly the same.

        If you use multiprocessing, the storage service will take care that the comment for
        the result or derived parameter with the lowest run index will be considered regardless
        of the order of the finishing of your runs. Note that this only works properly if all
        comments are the same. Otherwise the comment in the overview table might not be the one
        with the lowest run index.

        You need summary tables (see below) to be able to purge duplicate comments.

        This feature only works for comments in *leaf* nodes (aka Results and Parameters).
        So try to avoid to add comments in *group* nodes within single runs.

    :param summary_tables:

        Whether the summary tables should be created, i.e. the 'derived_parameters_runs_summary',
        and the `results_runs_summary`.

        The 'XXXXXX_summary' tables give a summary about all results or derived parameters.
        It is assumed that results and derived parameters with equal names in individual runs
        are similar and only the first result or derived parameter that was created
        is shown as an example.

        The summary table can be used in combination with `purge_duplicate_comments` to only store
        a single comment for every result with the same name in each run, see above.

    :param small_overview_tables:

        Whether the small overview tables should be created.
        Small tables are giving overview about 'config','parameters',
        'derived_parameters_trajectory', ,
        'results_trajectory','results_runs_summary'.

        Note that these tables create some overhead. If you want very small hdf5 files set
        `small_overview_tables` to False.

    :param large_overview_tables:

        Whether to add large overview tables. This encompasses information about every derived
        parameter, result, and the explored parameter in every single run.
        If you want small hdf5 files, this is the first option to set to false.

    :param results_per_run:

        Expected results you store per run. If you give a good/correct estimate
        storage to hdf5 file is much faster in case you store LARGE overview tables.

        Default is 0, i.e. the number of results is not estimated!

    :param derived_parameters_per_run:

        Analogous to the above.

    :param display_time:

        How often status messages about loading and storing time should be displayed.
        Interval in seconds.

    :param trajectory:

        A trajectory container, the storage service will add the used parameter to
        the trajectory container.

    """

    ADD_ROW = 'ADD'
    ''' Adds a row to an overview table'''
    REMOVE_ROW = 'REMOVE'
    ''' Removes a row from an overview table'''
    MODIFY_ROW = 'MODIFY'
    ''' Changes a row of an overview table'''

    COLL_TYPE = 'COLL_TYPE'
    '''Type of a container stored to hdf5, like list,tuple,dict,etc

    Must be stored in order to allow perfect reconstructions.
    '''

    COLL_LIST = 'COLL_LIST'
    ''' Container was a list'''
    COLL_TUPLE = 'COLL_TUPLE'
    ''' Container was a tuple'''
    COLL_NDARRAY = 'COLL_NDARRAY'
    ''' Container was a numpy array'''
    COLL_MATRIX = 'COLL_MATRIX'
    ''' Container was a numpy matrix'''
    COLL_DICT = 'COLL_DICT'
    ''' Container was a dictionary'''
    COLL_EMPTY_DICT = 'COLL_EMPTY_DICT'
    ''' Container was an empty dictionary'''
    COLL_SCALAR = 'COLL_SCALAR'
    ''' No container, but the thing to store was a scalar'''

    SCALAR_TYPE = 'SCALAR_TYPE'
    ''' Type of scalars stored into a container'''

    ### Overview Table constants
    CONFIG = 'config'
    PARAMETERS = 'parameters'
    RESULTS = 'results'
    EXPLORED_PARAMETERS = 'explored_parameters'
    DERIVED_PARAMETERS = 'derived_parameters'

    NAME_TABLE_MAPPING = {
        '_overview_config': 'config_overview',
        '_overview_parameters': 'parameters_overview',
        '_overview_derived_parameters': 'derived_parameters_overview',
        '_overview_results': 'results_overview',
        '_overview_explored_parameters': 'explored_parameters_overview',
        '_overview_derived_parameters_summary': 'derived_parameters_summary',
        '_overview_results_summary': 'results_summary'
    }
    ''' Mapping of trajectory config names to the tables'''

    PR_ATTR_NAME_MAPPING = {
        '_derived_parameters_per_run': 'derived_parameters_per_run',
        '_results_per_run': 'results_per_run',
        '_purge_duplicate_comments': 'purge_duplicate_comments'
    }
    '''Mapping of Attribute names for hdf5_settings table'''

    ATTR_LIST = [
        'complevel',
        'complib',
        'shuffle',
        'fletcher32',
        'pandas_format',
        'encoding'
    ]
    '''List of HDF5StorageService Attributes that have to be stored into the hdf5_settings table'''

    ### Storing Data Constants
    STORAGE_TYPE = 'SRVC_STORE'
    '''Flag, how data was stored'''

    SHARED_DATA_TYPE = 'SRVC_SHARED_TYPE'

    ARRAY = pypetconstants.ARRAY
    '''Stored as array_

    .. _array: http://pytables.github.io/usersguide/libref/homogenous_storage.html#the-array-class

    '''
    CARRAY = pypetconstants.CARRAY
    '''Stored as carray_

    .. _carray: http://pytables.github.io/usersguide/libref/homogenous_storage.html#the-carray-class

    '''
    EARRAY = pypetconstants.EARRAY
    ''' Stored as earray_e.

    .. _earray: http://pytables.github.io/usersguide/libref/homogenous_storage.html#the-earray-class

    '''

    VLARRAY = pypetconstants.VLARRAY
    '''Stored as vlarray_

    .. _vlarray: http://pytables.github.io/usersguide/libref/homogenous_storage.html#the-vlarray-class

    '''

    TABLE = pypetconstants.TABLE
    '''Stored as pytable_

    .. _pytable: http://pytables.github.io/usersguide/libref/structured_storage.html#the-table-class

    '''

    DICT = pypetconstants.DICT
    ''' Stored as dict.

    In fact, stored as pytable, but the dictionary wil be reconstructed.
    '''

    FRAME = pypetconstants.FRAME
    ''' Stored as pandas DataFrame_

    .. _DataFrame: http://pandas.pydata.org/pandas-docs/dev/io.html#hdf5-pytables

    '''

    SERIES = pypetconstants.SERIES
    ''' Store data as pandas Series '''

    SPLIT_TABLE = pypetconstants.SPLIT_TABLE
    ''' If a table was split due to too many columns'''

    DATATYPE_TABLE = pypetconstants.DATATYPE_TABLE
    '''If a table contains the data types instead of the attrs'''

    SHARED_DATA = pypetconstants.SHARED_DATA
    ''' An HDF5 data object for direct interaction '''

    NESTED_GROUP = pypetconstants.NESTED_GROUP
    ''' An HDF5 data object containing nested data '''

    TYPE_FLAG_MAPPING = {
        ObjectTable: TABLE,
        list: ARRAY,
        tuple: ARRAY,
        dict: DICT,
        np.ndarray: CARRAY,
        np.matrix: CARRAY,
        DataFrame: FRAME,
        Series: SERIES,
        shared.SharedTable: SHARED_DATA,
        shared.SharedArray: SHARED_DATA,
        shared.SharedPandasFrame: SHARED_DATA,
        shared.SharedCArray: SHARED_DATA,
        shared.SharedEArray: SHARED_DATA,
        shared.SharedVLArray: SHARED_DATA,
    }
    ''' Mapping from object type to storage flag'''

    # Python native data should always be stored as an ARRAY
    for item in pypetconstants.PARAMETER_SUPPORTED_DATA:
        TYPE_FLAG_MAPPING[item] = ARRAY

    FORMATTED_COLUMN_PREFIX = 'SRVC_COLUMN_%s_'
    ''' Stores data type of a specific pytables column for perfect reconstruction'''
    DATA_PREFIX = 'SRVC_DATA_'
    ''' Stores data type of a pytables carray or array for perfect reconstruction'''


    # ANNOTATION CONSTANTS
    ANNOTATION_PREFIX = 'SRVC_AN_'
    ''' Prefix to store annotations as node attributes_

    .. _attributes: http://pytables.github.io/usersguide/libref/declarative_classes.html#the-attributeset-class

    '''
    ANNOTATED = 'SRVC_ANNOTATED'
    ''' Whether an item was annotated'''


    # Stuff necessary to construct parameters and result
    INIT_PREFIX = 'SRVC_INIT_'
    ''' Hdf5 attribute prefix to store class name of parameter or result'''
    CLASS_NAME = INIT_PREFIX + 'CLASS_NAME'
    ''' Name of a parameter or result class, is converted to a constructor'''
    COMMENT = INIT_PREFIX + 'COMMENT'
    ''' Comment of parameter or result'''
    LENGTH = INIT_PREFIX + 'LENGTH'
    ''' Length of a parameter if it is explored, no longer in use, only for backwards
    compatibility'''
    LEAF = 'SRVC_LEAF'
    ''' Whether an hdf5 node is a leaf node'''

    def __init__(self, filename=None,
                 file_title=None,
                 overwrite_file=False,
                 encoding='utf8',
                 complevel=9,
                 complib='zlib',
                 shuffle=True,
                 fletcher32=False,
                 pandas_format='fixed',
                 purge_duplicate_comments=True,
                 summary_tables=True,
                 small_overview_tables=True,
                 large_overview_tables=False,
                 results_per_run=0,
                 derived_parameters_per_run=0,
                 display_time=20,
                 trajectory=None):

        self._set_logger()

        if purge_duplicate_comments and not summary_tables:
            raise ValueError('You cannot purge duplicate comments without having the'
                             ' small overview tables.')

        # Prepare file names and log folder
        if file_title is None and trajectory is not None:
            file_title = trajectory.v_name
        elif file_title is None:
            file_title = 'Experiments'
        else:
            file_title = file_title

        if filename is None and trajectory is not None:
            # If no filename is supplied and the filename cannot be extracted from the
            # trajectory, create the default filename
            filename = os.path.join(os.getcwd(), 'hdf5', trajectory.v_name + '.hdf5')
        elif filename is None:
            filename = 'Experiments.hdf5'

        if os.path.isdir(filename):
            head, tail = (filename, '')
        else:
            head, tail = os.path.split(filename)
        if not head:
            # If the filename contains no path information,
            # we put it into the current working directory
            filename = os.path.join(os.getcwd(), filename)
        if not tail and trajectory is not None:
            filename = os.path.join(filename, trajectory.v_name + '.hdf5')
        elif not tail and trajectory is None:
            filename = os.path.join(filename, 'Experiments.hdf5')

        # Print which file we use for storage
        self._logger.info('I will use the hdf5 file `%s`.' % filename)

        self._filename = filename
        self._file_title = file_title
        self._trajectory_name = None if trajectory is None else trajectory.v_name
        self._trajectory_index = None
        self._hdf5file = None
        self._hdf5store = None
        self._trajectory_group = None  # link to the top group in hdf5 file which is the start
        # node of a trajectory

        self._filters = None
        self._complevel = complevel
        self._complib = complib
        self._fletcher32 = fletcher32
        self._shuffle = shuffle
        self._encoding = encoding

        self._node_processing_timer = None

        self._display_time = display_time

        self._pandas_format = pandas_format

        self._purge_duplicate_comments = purge_duplicate_comments
        self._results_per_run = results_per_run
        self._derived_parameters_per_run = derived_parameters_per_run

        self._overview_parameters = small_overview_tables
        self._overview_config = small_overview_tables
        self._overview_explored_parameters = small_overview_tables
        self._overview_derived_parameters = large_overview_tables
        self._overview_derived_parameters_summary = summary_tables
        self._overview_results = large_overview_tables
        self._overview_results_summary = summary_tables

        self._overview_group_ = None  # to cache link to overview

        self._disable_logger = DisableAllLogging()


        self._mode = None
        self._keep_open = False

        if trajectory is not None and not trajectory.v_stored:
            self._srvc_set_config(trajectory=trajectory)

        if overwrite_file:
            try:
                os.remove(filename)
                self._logger.info('You specified ``overwrite_file=True``, so I deleted the '
                                  'file `%s`.' % filename)
            except OSError:
                # File not found, we're good
                pass

        # We don't want the NN warnings of Pytables to display because they can be
        # annoying as hell
        warnings.simplefilter('ignore', pt.NaturalNameWarning)

    def __repr__(self):
        return '<%s (filename:`%s`)>' % (self.__class__.__name__, str(self._filename))

    @property
    def is_open(self):
        """ Normally the file is opened and closed after each insertion.

        However, the storage service may provide the option to keep the store open and signals
        this via this property.

        """
        return self._hdf5file is not None and self._hdf5file.isopen

    @property
    def encoding(self):
        """ How unicode strings are encoded"""
        return self._encoding

    @encoding.setter
    def encoding(self, encoding):
        self._encoding = encoding


    @property
    def display_time(self):
        """Time interval in seconds, when to display the storage or loading of nodes"""
        return self._display_time

    @display_time.setter
    def display_time(self, display_time):
        self._display_time = display_time

    @property
    def complib(self):
        """Compression library used"""
        return self._complib

    @complib.setter
    def complib(self, complib):
        self._complib = complib
        self._filters = None

    @property
    def complevel(self):
        """Compression level used"""
        return self._complevel

    @complevel.setter
    def complevel(self, complevel):
        self._complevel = complevel
        self._filters = None

    @property
    def fletcher32(self):
        """ Whether fletcher 32 should be used """
        return self._fletcher32

    @fletcher32.setter
    def fletcher32(self, fletcher32):
        self._fletcher32 = bool(fletcher32)
        self._filters = None

    @property
    def shuffle(self):
        """ Whether shuffle filtering should be used"""
        return self._shuffle

    @shuffle.setter
    def shuffle(self, shuffle):
        self._shuffle = bool(shuffle)
        self._filters = None

    @property
    def pandas_format(self):
        """Format of pandas data. Applicable formats are 'table' (or 't') and 'fixed' (or 'f')"""
        return self._pandas_format

    @pandas_format.setter
    def pandas_format(self, pandas_format):
        if pandas_format not in ('f', 'fixed', 'table', 't'):
            raise ValueError('''Pandas format can only be 'table' (or 't') and 'fixed' (or 'f')
                            not `%s`.''' % pandas_format)
        self._pandas_format = pandas_format

    @property
    def filename(self):
        """The name and path of the underlying hdf5 file."""
        return self._filename

    @filename.setter
    def filename(self, filename):
        self._filename = filename

    @property
    def _overview_group(self):
        """Direct link to the overview group"""
        if self._overview_group_ is None:
            self._overview_group_ = self._all_create_or_get_groups('overview')[0]
        return self._overview_group_

    def _all_get_filters(self, kwargs=None):
        """Makes filters

        Pops filter arguments from `kwargs` such that they are not passed
        on to other functions also using kwargs.

        """
        if kwargs is None:
            kwargs = {}
        complib = kwargs.pop('complib', None)
        complevel = kwargs.pop('complevel', None)
        shuffle = kwargs.pop('shuffle', None)
        fletcher32 = kwargs.pop('fletcher32', None)
        if complib is not None:
            self._filters = None
        else:
            complib = self._complib
        if complevel is not None:
            self._filters = None
        else:
            complevel = self._complevel
        if shuffle is not None:
            self._filters = None
        else:
            shuffle = self._shuffle
        if fletcher32 is not None:
            self._filters = None
        else:
            fletcher32 = self._fletcher32

        if self._filters is None:
            # Recreate the filters if something was changed
            self._filters = pt.Filters(complib=complib, complevel=complevel,
                                       shuffle=shuffle, fletcher32=fletcher32)
            self._hdf5file.filters = self._filters
            self._hdf5store._filters = self._filters
            self._hdf5store._complevel = complevel
            self._hdf5store._complib = complib
            self._hdf5store._fletcher32 = fletcher32

        return self._filters

    def _srvc_set_config(self, trajectory):
        """Sets a config value to the Trajectory or changes it if the trajectory was loaded
        a the settings no longer match"""
        def _set_config(name, value, comment):
            if not trajectory.f_contains('config.'+name, shortcuts=False):
                trajectory.f_add_config(Parameter, name, value, comment=comment)

        for attr_name in HDF5StorageService.NAME_TABLE_MAPPING:
            table_name = HDF5StorageService.NAME_TABLE_MAPPING[attr_name]
            value = getattr(self, attr_name)
            _set_config('hdf5.overview.' + table_name,
                         value,
                         comment='Whether or not to have an overview '
                                 'table with that name')

        _set_config('hdf5.purge_duplicate_comments',
                    self._purge_duplicate_comments,
                    comment='Whether comments of results and'
                            ' derived parameters should only'
                            ' be stored for the very first instance.'
                            ' Works only if the summary tables are'
                            ' active.')

        _set_config('hdf5.results_per_run', self._results_per_run,
                    comment='Expected number of results per run,'
                            ' a good guess can increase storage performance')

        _set_config('hdf5.derived_parameters_per_run',
                    self._derived_parameters_per_run,
                    comment='Expected number of derived parameters per run,'
                            ' a good guess can increase storage performance')

        _set_config('hdf5.complevel', self._complevel,
                    comment='Compression Level (0 no compression '
                            'to 9 highest compression)')

        _set_config('hdf5.complib', self._complib,
                    comment='Compression Algorithm')

        _set_config('hdf5.encoding', self._encoding,
                    comment='Encoding for unicode characters')

        _set_config('hdf5.fletcher32', self._fletcher32,
                    comment='Whether to use fletcher 32 checksum')

        _set_config('hdf5.shuffle', self._shuffle,
                    comment='Whether to use shuffle filtering.')

        _set_config('hdf5.pandas_format', self._pandas_format,
                    comment='''How to store pandas data frames, either'''
                            ''' 'fixed' ('f') or 'table' ('t').''')

        if trajectory.f_contains('config.hdf5', shortcuts=False):
            if trajectory.config.hdf5.v_comment == '':
                # If this has not happened yet, add a description of the hdf5 config group
                trajectory.config.hdf5.v_comment = 'Settings for the standard HDF5 storage service'

        trajectory.v_storage_service = self # And add the storage service

    def load(self, msg, stuff_to_load, *args, **kwargs):
        """Loads a particular item from disk.

        The storage service always accepts these parameters:

        :param trajectory_name: Name of current trajectory and name of top node in hdf5 file.

        :param trajectory_index:

            If no `trajectory_name` is provided, you can specify an integer index.
            The trajectory at the index position in the hdf5 file is considered to loaded.
            Negative indices are also possible for reverse indexing.

        :param filename: Name of the hdf5 file


        The following messages (first argument msg) are understood and the following arguments
        can be provided in combination with the message:

            * :const:`pypet.pypetconstants.TRAJECTORY` ('TRAJECTORY')

                Loads a trajectory.

                :param stuff_to_load: The trajectory

                :param as_new: Whether to load trajectory as new

                :param load_parameters: How to load parameters and config

                :param load_derived_parameters: How to load derived parameters

                :param load_results: How to load results

                :param force: Force load in case there is a pypet version mismatch

                You can specify how to load the parameters, derived parameters and results
                as follows:

                :const:`pypet.pypetconstants.LOAD_NOTHING`: (0)

                    Nothing is loaded

                :const:`pypet.pypetconstants.LOAD_SKELETON`: (1)

                    The skeleton including annotations are loaded, i.e. the items are empty.
                    Non-empty items in RAM are left untouched.

                :const:`pypet.pypetconstants.LOAD_DATA`: (2)

                    The whole data is loaded.
                    Only empty or in RAM non-existing instance are filled with the
                    data found on disk.

                :const:`pypet.pypetconstants.OVERWRITE_DATA`: (3)

                    The whole data is loaded.
                    If items that are to be loaded are already in RAM and not empty,
                    they are emptied and new data is loaded from disk.

            * :const:`pypet.pypetconstants.LEAF` ('LEAF')

                Loads a parameter or result.

                :param stuff_to_load: The item to be loaded

                :param load_data: How to load data

                :param load_only:

                    If you load a result, you can partially load it and ignore the
                    rest of the data. Just specify the name of the data you want to load.
                    You can also provide a list,
                    for example `load_only='spikes'`, `load_only=['spikes','membrane_potential']`.

                    Issues a warning if items cannot be found.

                :param load_except:

                    If you load a result you can partially load in and specify items
                    that should NOT be loaded here. You cannot use `load_except` and
                    `load_only` at the same time.

            * :const:`pypet.pyetconstants.GROUP`

                Loads a group a node (comment and annotations)

                :param recursive:

                    Recursively loads everything below

                :param load_data:

                    How to load stuff if ``recursive=True``
                    accepted values as above for loading the trajectory

                :param max_depth:

                    Maximum depth in case of recursion. `None` for no limit.

            * :const:`pypet.pypetconstants.TREE` ('TREE')

                Loads a whole subtree

                :param stuff_to_load: The parent node (!) not the one where loading starts!

                :param child_name: Name of child node that should be loaded

                :param recursive: Whether to load recursively the subtree below child

                :param load_data:

                    How to load stuff, accepted values as above for loading the trajectory

                :param max_depth:

                    Maximum depth in case of recursion. `None` for no limit.

                :param trajectory: The trajectory object

            * :const:`pypet.pypetconstants.LIST` ('LIST')

                Analogous to :ref:`storing lists <store-lists>`

        :raises:

            NoSuchServiceError if message or data is not understood

            DataNotInStorageError if data to be loaded cannot be found on disk

        """
        opened = True
        try:

            opened = self._srvc_opening_routine('r', kwargs=kwargs)

            if msg == pypetconstants.TRAJECTORY:
                self._trj_load_trajectory(stuff_to_load, *args, **kwargs)

            elif msg == pypetconstants.LEAF:
                self._prm_load_parameter_or_result(stuff_to_load, *args, **kwargs)

            elif msg == pypetconstants.GROUP:
                self._grp_load_group(stuff_to_load, *args, **kwargs)

            elif msg == pypetconstants.TREE:
                self._tree_load_sub_branch(stuff_to_load, *args, **kwargs)

            elif msg == pypetconstants.LIST:
                self._srvc_load_several_items(stuff_to_load, *args, **kwargs)

            else:
                raise pex.NoSuchServiceError('I do not know how to handle `%s`' % msg)

        except pt.NoSuchNodeError as exc:
            self._logger.error('Failed loading  `%s`' % str(stuff_to_load))
            raise pex.DataNotInStorageError(repr(exc))
        except:
            self._logger.error('Failed loading  `%s`' % str(stuff_to_load))
            raise
        finally:
            self._srvc_closing_routine(opened)

    def store(self, msg, stuff_to_store, *args, **kwargs):
        """ Stores a particular item to disk.

        The storage service always accepts these parameters:

        :param trajectory_name: Name or current trajectory and name of top node in hdf5 file

        :param filename: Name of the hdf5 file

        :param file_title: If file needs to be created, assigns a title to the file.


        The following messages (first argument msg) are understood and the following arguments
        can be provided in combination with the message:

            * :const:`pypet.pypetconstants.PREPARE_MERGE` ('PREPARE_MERGE'):

                Called to prepare a trajectory for merging, see also 'MERGE' below.

                Will also be called if merging cannot happen within the same hdf5 file.
                Stores already enlarged parameters and updates meta information.

                :param stuff_to_store: Trajectory that is about to be extended by another one

                :param changed_parameters:

                    List containing all parameters that were enlarged due to merging

                :param old_length:

                    Old length of trajectory before merge

            * :const:`pypet.pypetconstants.MERGE` ('MERGE')

                Note that before merging within HDF5 file, the storage service will be called
                with msg='PREPARE_MERGE' before, see above.

                Raises a ValueError if the two trajectories are not stored within the very
                same hdf5 file. Then the current trajectory needs to perform the merge slowly
                item by item.

                Merges two trajectories, parameters are:

                :param stuff_to_store: The trajectory data is merged into

                :param other_trajectory_name: Name of the other trajectory

                :param rename_dict:

                    Dictionary containing the old result and derived parameter names in the
                    other trajectory and their new names in the current trajectory.

                :param move_nodes:

                    Whether to move the nodes from the other to the current trajectory

                :param delete_trajectory:

                    Whether to delete the other trajectory after merging.

            * :const:`pypet.pypetconstants.BACKUP` ('BACKUP')

                :param stuff_to_store: Trajectory to be backed up

                :param backup_filename:

                    Name of file where to store the backup. If None the backup file will be in
                    the same folder as your hdf5 file and named 'backup_XXXXX.hdf5'
                    where 'XXXXX' is the name of your current trajectory.

            * :const:`pypet.pypetconstants.TRAJECTORY` ('TRAJECTORY')

                Stores the whole trajectory

                :param stuff_to_store: The trajectory to be stored

                :param only_init:

                    If you just want to initialise the store. If yes, only meta information about
                    the trajectory is stored and none of the nodes/leaves within the trajectory.

                :param store_data:

                    How to store data, the following settings are understood:

                     :const:`pypet.pypetconstants.STORE_NOTHING`: (0)

                        Nothing is stored

                    :const:`pypet.pypetconstants.STORE_DATA_SKIPPING`: (1)

                        Data of not already stored nodes is stored

                    :const:`pypet.pypetconstants.STORE_DATA`: (2)

                        Data of all nodes is stored. However, existing data on disk is left
                        untouched.

                    :const:`pypet.pypetconstants.OVERWRITE_DATA`: (3)

                        Data of all nodes is stored and data on disk is overwritten.
                        May lead to fragmentation of the HDF5 file. The user is adviced
                        to recompress the file manually later on.

            * :const:`pypet.pypetconstants.SINGLE_RUN` ('SINGLE_RUN')

                :param stuff_to_store: The trajectory

                :param store_data: How to store data see above

                :param store_final: If final meta info should be stored

            * :const:`pypet.pypetconstants.LEAF`

                Stores a parameter or result

                Note that everything that is supported by the storage service and that is
                stored to disk will be perfectly recovered.
                For instance, you store a tuple of numpy 32 bit integers, you will get a tuple
                of numpy 32 bit integers after loading independent of the platform!

                :param stuff_to_sore: Result or parameter to store

                    In order to determine what to store, the function '_store' of the parameter or
                    result is called. This function returns a dictionary with name keys and data to
                    store as values. In order to determine how to store the data, the storage flags
                    are considered, see below.

                    The function '_store' has to return a dictionary containing values only from
                    the following objects:

                        * python natives (int, long, str, bool, float, complex),

                        *
                            numpy natives, arrays and matrices of type np.int8-64, np.uint8-64,
                            np.float32-64, np.complex, np.str

                        *

                            python lists and tuples of the previous types
                            (python natives + numpy natives and arrays)
                            Lists and tuples are not allowed to be nested and must be
                            homogeneous, i.e. only contain data of one particular type.
                            Only integers, or only floats, etc.

                        *

                            python dictionaries of the previous types (not nested!), data can be
                            heterogeneous, keys must be strings. For example, one key-value-pair
                            of string and int and one key-value pair of string and float, and so
                            on.

                        * pandas DataFrames_

                        * :class:`~pypet.parameter.ObjectTable`

                    .. _DataFrames: http://pandas.pydata.org/pandas-docs/dev/dsintro.html#dataframe

                    The keys from the '_store' dictionaries determine how the data will be named
                    in the hdf5 file.

                :param store_data:

                    How to store the data, see above for a descitpion.

                :param store_flags: Flags describing how to store data.

                        :const:`~pypet.HDF5StorageService.ARRAY` ('ARRAY')

                            Store stuff as array

                        :const:`~pypet.HDF5StorageService.CARRAY` ('CARRAY')

                            Store stuff as carray

                        :const:`~pypet.HDF5StorageService.TABLE` ('TABLE')

                            Store stuff as pytable

                        :const:`~pypet.HDF5StorageService.DICT` ('DICT')

                            Store stuff as pytable but reconstructs it later as dictionary
                            on loading

                        :const:`~pypet.HDF%StorageService.FRAME` ('FRAME')

                            Store stuff as pandas data frame

                    Storage flags can also be provided by the parameters and results themselves
                    if they implement a function '_store_flags' that returns a dictionary
                    with the names of the data to store as keys and the flags as values.

                    If no storage flags are provided, they are automatically inferred from the
                    data. See :const:`pypet.HDF5StorageService.TYPE_FLAG_MAPPING` for the mapping
                    from type to flag.

                :param overwrite:

                    Can be used if parts of a leaf should be replaced. Either a list of
                    HDF5 names or `True` if this should account for all.

            * :const:`pypet.pypetconstants.DELETE` ('DELETE')

                Removes an item from disk. Empty group nodes, results and non-explored
                parameters can be removed.

                :param stuff_to_store: The item to be removed.

                :param delete_only:

                    Potential list of parts of a leaf node that should be deleted.

                :param remove_from_item:

                    If `delete_only` is used, whether deleted nodes should also be erased
                    from the leaf nodes themseleves.

                :param recursive:

                    If you want to delete a group node you can recursively delete all its
                    children.

            * :const:`pypet.pypetconstants.GROUP` ('GROUP')

                :param stuff_to_store: The group to store

                :param store_data: How to store data

                :param recursive: To recursively load everything below.

                :param max_depth:

                    Maximum depth in case of recursion. `None` for no limit.

            * :const:`pypet.pypetconstants.TREE`

                Stores a single node or a full subtree

                :param stuff_to_store: Node to store

                :param store_data: How to store data

                :param recursive: Whether to store recursively the whole sub-tree

                :param max_depth:

                    Maximum depth in case of recursion. `None` for no limit.

            * :const:`pypet.pypetconstants.DELETE_LINK`

                Deletes a link from hard drive

                :param name: The full colon separated name of the link

            * :const:`pypet.pypetconstants.LIST`

                .. _store-lists:

                Stores several items at once

                :param stuff_to_store:

                    Iterable whose items are to be stored. Iterable must contain tuples,
                    for example `[(msg1,item1,arg1,kwargs1),(msg2,item2,arg2,kwargs2),...]`

            * :const:`pypet.pypetconstants.ACCESS_DATA`

                Requests and manipulates data within the storage.
                Storage must be open.

                :param stuff_to_store:

                    A colon separated name to the data path

                :param item_name:

                    The name of the data item to interact with

                :param request:

                    A functional request in form of a string

                :param args:

                    Positional arguments passed to the reques

                :param kwargs:

                    Keyword arguments passed to the request

            * :const:`pypet.pypetconstants.OPEN_FILE`

                Opens the HDF5 file and keeps it open

                :param stuff_to_store: ``None``

            * :const:`pypet.pypetconstants.CLOSE_FILE`

                Closes an HDF5 file that was kept open, must be open before.

                :param stuff_to_store: ``None``

            * :const:`pypet.pypetconstants.FLUSH`

                Flushes an open file, must be open before.

                :param stuff_to_store: ``None``

        :raises: NoSuchServiceError if message or data is not understood

        """
        opened = True
        try:

            opened = self._srvc_opening_routine('a', msg, kwargs)

            if msg == pypetconstants.MERGE:
                self._trj_merge_trajectories(*args, **kwargs)

            elif msg == pypetconstants.BACKUP:
                self._trj_backup_trajectory(stuff_to_store, *args, **kwargs)

            elif msg == pypetconstants.PREPARE_MERGE:
                self._trj_prepare_merge(stuff_to_store, *args, **kwargs)

            elif msg == pypetconstants.TRAJECTORY:
                self._trj_store_trajectory(stuff_to_store, *args, **kwargs)

            elif msg == pypetconstants.SINGLE_RUN:
                self._srn_store_single_run(stuff_to_store, *args, **kwargs)

            elif msg in pypetconstants.LEAF:
                self._prm_store_parameter_or_result(stuff_to_store, *args, **kwargs)

            elif msg == pypetconstants.DELETE:
                self._all_delete_parameter_or_result_or_group(stuff_to_store, *args, **kwargs)

            elif msg == pypetconstants.GROUP:
                self._grp_store_group(stuff_to_store, *args, **kwargs)

            elif msg == pypetconstants.TREE:
                self._tree_store_sub_branch(stuff_to_store, *args, **kwargs)

            elif msg == pypetconstants.DELETE_LINK:
                self._lnk_delete_link(stuff_to_store, *args, **kwargs)

            elif msg == pypetconstants.LIST:
                self._srvc_store_several_items(stuff_to_store, *args, **kwargs)

            elif msg == pypetconstants.ACCESS_DATA:
                return self._hdf5_interact_with_data(stuff_to_store, *args, **kwargs)

            elif msg == pypetconstants.OPEN_FILE:
                opened = False  # Wee need to keep the file open to allow later interaction
                self._keep_open = True
                self._node_processing_timer.active = False # This might be open quite long
                # so we don't want to display horribly long opening times

            elif msg == pypetconstants.CLOSE_FILE:
                opened = True  # Simply conduct the closing routine afterwards
                self._keep_open = False

            elif msg == pypetconstants.FLUSH:
                self._hdf5file.flush()

            else:
                raise pex.NoSuchServiceError('I do not know how to handle `%s`' % msg)

        except:
            self._logger.error('Failed storing `%s`' % str(stuff_to_store))
            raise
        finally:
            self._srvc_closing_routine(opened)

    def _srvc_load_several_items(self, iterable, *args, **kwargs):
        """Loads several items from an iterable

        Iterables are supposed to be of a format like `[(msg, item, args, kwarg),...]`
        If `args` and `kwargs` are not part of a tuple, they are taken from the
        current `args` and `kwargs` provided to this function.

        """
        for input_tuple in iterable:
            msg = input_tuple[0]
            item = input_tuple[1]
            if len(input_tuple) > 2:
                args = input_tuple[2]
            if len(input_tuple) > 3:
                kwargs = input_tuple[3]
            if len(input_tuple) > 4:
                raise RuntimeError('You shall not pass!')

            self.load(msg, item, *args, **kwargs)

    def _srvc_check_hdf_properties(self, traj):
        """Reads out the properties for storing new data into the hdf5file

        :param traj:

            The trajectory

        """

        for attr_name in HDF5StorageService.ATTR_LIST:
            try:
                config = traj.f_get('config.hdf5.' + attr_name).f_get()
                setattr(self, attr_name, config)
            except AttributeError:
                self._logger.debug('Could not find `%s` in traj config, '
                                     'using (default) value `%s`.' %
                                     (attr_name, str(getattr(self, attr_name))))

        for attr_name, table_name in HDF5StorageService.NAME_TABLE_MAPPING.items():
            try:
                if table_name in ('parameters', 'config'):
                    table_name += '_overview'
                config = traj.f_get('config.hdf5.overview.' + table_name).f_get()
                setattr(self, attr_name, config)
            except AttributeError:
                self._logger.debug('Could not find `%s` in traj config, '
                                     'using (default) value `%s`.' %
                                     (table_name, str(getattr(self, attr_name))))

        for attr_name, name in HDF5StorageService.PR_ATTR_NAME_MAPPING.items():
            try:
                config = traj.f_get('config.hdf5.' + name).f_get()
                setattr(self, attr_name, config)
            except AttributeError:
                self._logger.debug('Could not find `%s` in traj config, '
                                     'using (default) value `%s`.' %
                                     (name, str(getattr(self, attr_name))))

        if ((not self._overview_results_summary or
                    not self._overview_derived_parameters_summary) and
                    self._purge_duplicate_comments):
            raise RuntimeError('You chose to purge duplicate comments but disabled a summary '
                                   'table. You can only use the purging if you enable '
                                   'the summary tables.')

        self._filters = None

    def _srvc_store_several_items(self, iterable, *args, **kwargs):
        """Stores several items from an iterable

        Iterables are supposed to be of a format like `[(msg, item, args, kwarg),...]`
        If `args` and `kwargs` are not part of a tuple, they are taken from the
        current `args` and `kwargs` provided to this function.

        """
        for input_tuple in iterable:
            msg = input_tuple[0]
            item = input_tuple[1]
            if len(input_tuple) > 2:
                args = input_tuple[2]
            if len(input_tuple) > 3:
                kwargs = input_tuple[3]
            if len(input_tuple) > 4:
                raise RuntimeError('You shall not pass!')

            self.store(msg, item, *args, **kwargs)

    def _srvc_opening_routine(self, mode, msg=None, kwargs=()):
        """Opens an hdf5 file for reading or writing

        The file is only opened if it has not been opened before (i.e. `self._hdf5file is None`).

        :param mode:

            'a' for appending

            'r' for reading

                Unfortunately, pandas currently does not work with read-only mode.
                Thus, if mode is chosen to be 'r', the file will still be opened in
                append mode.

        :param msg:

            Message provided to `load` or `store`. Only considered to check if a trajectory
            was stored before.

        :param kwargs:

            Arguments to extract file information from

        :return:

            `True` if file is opened

            `False` if the file was already open before calling this function

        """
        self._mode = mode
        self._srvc_extract_file_information(kwargs)

        if not self.is_open:

            if 'a' in mode:
                (path, filename) = os.path.split(self._filename)
                racedirs(os.path.abspath(path))

                self._hdf5store = HDFStore(self._filename, mode=self._mode, complib=self._complib,
                                           complevel=self._complevel, fletcher32=self._fletcher32)
                self._hdf5file = self._hdf5store._handle
                self._hdf5file.title = self._file_title

                if self._trajectory_name is not None:
                    if not '/' + self._trajectory_name in self._hdf5file:
                        # If we want to store individual items we we have to check if the
                        # trajectory has been stored before
                        if not msg == pypetconstants.TRAJECTORY:
                            raise ValueError('Your trajectory cannot be found in the hdf5file, '
                                             'please use >>traj.f_store()<< '
                                             'before storing anything else.')

                    else:
                        # Keep a reference to the top trajectory node
                        self._trajectory_group = self._hdf5file.get_node('/' +
                                                                         self._trajectory_name)
                else:
                    raise ValueError('I don`t know which trajectory to load')
                self._logger.debug('Opening HDF5 file `%s` in mode `a` with trajectory `%s`' %
                                   (self._filename, self._trajectory_name))

            elif mode == 'r':

                if self._trajectory_name is not None and self._trajectory_index is not None:
                    raise ValueError('Please specify either a name of a trajectory or an index, '
                                     'but not both at the same time.')

                if not os.path.isfile(self._filename):
                    raise ValueError('File `' + self._filename + '` does not exist.')

                self._hdf5store = HDFStore(self._filename, mode=self._mode, complib=self._complib,
                                           complevel=self._complevel, fletcher32=self._fletcher32)
                self._hdf5file = self._hdf5store._handle

                if self._trajectory_index is not None:
                    # If an index is provided pick the trajectory at the corresponding
                    # position in the trajectory node list
                    nodelist = self._hdf5file.list_nodes(where='/')

                    if (self._trajectory_index >= len(nodelist) or
                            self._trajectory_index < -len(nodelist)):
                        raise ValueError('Trajectory No. %d does not exists, there are only '
                                         '%d trajectories in %s.'
                                         % (self._trajectory_index, len(nodelist), self._filename))

                    self._trajectory_group = nodelist[self._trajectory_index]
                    self._trajectory_name = self._trajectory_group._v_name

                elif self._trajectory_name is not None:
                    # Otherwise pick the trajectory group by name
                    if not '/' + self._trajectory_name in self._hdf5file:
                        raise ValueError('File %s does not contain trajectory %s.'
                                         % (self._filename, self._trajectory_name))

                    self._trajectory_group = self._hdf5file.get_node('/' + self._trajectory_name)
                else:
                    raise ValueError('Please specify a name of a trajectory to load or its '
                                     'index, otherwise I cannot open one.')

                self._logger.debug('Opening HDF5 file `%s` in mode `r` with trajectory `%s`' %
                                   (self._filename, self._trajectory_name))

            else:
                raise RuntimeError('You shall not pass!')

            self._node_processing_timer = NodeProcessingTimer(display_time=self._display_time,
                                                              logger_name=self._logger.name)
            self._overview_group_ = None

            return True
        else:
            return False

    def _srvc_closing_routine(self, closing):
        """Routine to close an hdf5 file

        The file is closed only when `closing=True`. `closing=True` means that
        the file was opened in the current highest recursion level. This prevents re-opening
        and closing of the file if `store` or `load` are called recursively.

        """
        if (not self._keep_open and
                closing and
                    self.is_open):

            f_fd = self._hdf5file.fileno()
            self._hdf5file.flush()
            try:
                os.fsync(f_fd)
                try:
                    self._hdf5store.flush(fsync=True)
                except TypeError:
                    f_fd = self._hdf5store._handle.fileno()
                    self._hdf5store.flush()
                    os.fsync(f_fd)
            except OSError as exc:
                # This seems to be the only way to avoid an OSError under Windows
                errmsg = ('Encountered OSError while flushing file.'
                                   'If you are using Windows, don`t worry! '
                                   'I will ignore the error and try to close the file. '
                                   'Original error: %s' % repr(exc))
                self._logger.debug(errmsg)

            self._hdf5store.close()
            if self._hdf5file.isopen:
                self._logger.error('Could not close HDF5 file!')
            self._hdf5file = None
            self._hdf5store = None
            self._trajectory_group = None
            self._trajectory_name = None
            self._trajectory_index = None
            self._overview_group_ = None
            self._logger.debug('Closing HDF5 file')
            return True
        else:
            return False

    def _srvc_extract_file_information(self, kwargs):
        """Extracts file information from kwargs.

        Note that `kwargs` is not passed as `**kwargs` in order to also
        `pop` the elements on the level of the function calling `_srvc_extract_file_information`.

        """
        if 'filename' in kwargs:
            self._filename = kwargs.pop('filename')

        if 'file_title' in kwargs:
            self._file_title = kwargs.pop('file_title')

        if 'trajectory_name' in kwargs:
            self._trajectory_name = kwargs.pop('trajectory_name')

        if 'trajectory_index' in kwargs:
            self._trajectory_index = kwargs.pop('trajectory_index')


    ########################### Merging ###########################################################

    def _trj_backup_trajectory(self, traj, backup_filename=None):
        """Backs up a trajectory.

        :param traj: Trajectory that should be backed up

        :param backup_filename:

            Path and filename of backup file. If None is specified the storage service
            defaults to `path_to_trajectory_hdf5_file/backup_trajectory_name.hdf`.

        """
        self._logger.info('Storing backup of %s.' % traj.v_name)

        mypath, _ = os.path.split(self._filename)

        if backup_filename is None:
            backup_filename = os.path.join('%s' % mypath, 'backup_%s.hdf5' % traj.v_name)

        backup_hdf5file = pt.open_file(filename=backup_filename,
                                             mode='a', title=backup_filename)

        if '/' + self._trajectory_name in backup_hdf5file:
            raise ValueError('I cannot backup  `%s` into file `%s`, there is already a '
                             'trajectory with that name.' % (traj.v_name, backup_filename))

        backup_root = backup_hdf5file.root

        self._trajectory_group._f_copy(newparent=backup_root, recursive=True)

        backup_hdf5file.flush()
        backup_hdf5file.close()

        self._logger.info('Finished backup of %s.' % traj.v_name)

    @staticmethod
    def _trj_read_out_row(colnames, row):
        """Reads out a row and returns a dictionary containing the row content.

        :param colnames: List of column names
        :param row:  A pytables table row
        :return: A dictionary with colnames as keys and content as values

        """
        result_dict = {}
        for colname in colnames:
            result_dict[colname] = row[colname]

        return result_dict


    def _trj_merge_trajectories(self, other_trajectory_name, rename_dict, move_nodes=False,
                                delete_trajectory=False, other_filename=None):
        """Merges another trajectory into the current trajectory (as in self._trajectory_name).

        :param other_trajectory_name: Name of other trajectory
        :param rename_dict: Dictionary with old names (keys) and new names (values).
        :param move_nodes: Whether to move hdf5 nodes or copy them
        :param delete_trajectory: Whether to delete the other trajectory

        """
        if other_filename is None or other_filename == self.filename:
            other_filename = self.filename
            other_file = self._hdf5file
            other_is_different = False
        else:
            other_file = pt.open_file(filename=other_filename, mode='r+')
            other_is_different = True

        try:
            if not '/' + other_trajectory_name in other_file:
                raise ValueError('Cannot merge `%s` and `%s`, because the second trajectory cannot '
                                 'be found in file: %s.' % (self._trajectory_name,
                                                            other_trajectory_name,
                                                            other_filename))
            for old_name in rename_dict:
                new_name = rename_dict[old_name]

                # Iterate over all items that need to be merged
                split_name = old_name.split('.')
                old_location = '/' + other_trajectory_name + '/' + '/'.join(split_name)

                split_name = new_name.split('.')
                new_parent_location = '/' + self._trajectory_name + '/' + '/'.join(split_name[:-1])

                new_short_name = split_name[-1]

                # Get the data from the other trajectory
                old_node = other_file.get_node(old_location)

                # Now move or copy the data
                if move_nodes:
                    self._hdf5file.move_node( where=old_node, newparent=new_parent_location,
                                       newname=new_short_name, createparents=True)
                else:
                    if other_is_different:
                        new_parent_dot_location = '.'.join(split_name[:-1])
                        new_parent_or_loc, _ = self._all_create_or_get_groups(
                            new_parent_dot_location)
                        create_parents = False
                    else:
                        new_parent_or_loc = new_parent_location
                        create_parents = True
                    self._hdf5file.copy_node(where=old_node, newparent=new_parent_or_loc,
                                       newname=new_short_name, createparents=create_parents,
                                       recursive=True)


            if delete_trajectory:
                other_file.remove_node(where='/', name=other_trajectory_name, recursive=True)
        finally:
            if other_is_different:
                other_file.flush()
                other_file.close()

    def _trj_prepare_merge(self, traj, changed_parameters, old_length):
        """Prepares a trajectory for merging.

        This function will already store extended parameters.

        :param traj: Target of merge
        :param changed_parameters: List of extended parameters (i.e. their names).

        """

        if not traj._stored:
            traj.f_store()

        # Update meta information
        infotable = getattr(self._overview_group, 'info')
        insert_dict = self._all_extract_insert_dict(traj, infotable.colnames)
        self._all_add_or_modify_row(traj.v_name, insert_dict, infotable, index=0,
                                    flags=(HDF5StorageService.MODIFY_ROW,))

        # Store extended parameters
        for param_name in changed_parameters:
            param = traj.f_get(param_name)

            try:
                self._all_delete_parameter_or_result_or_group(param)
            except pt.NoSuchNodeError:
                pass  # We are fine and the node did not exist in the first place

        # Increase the run table by the number of new runs
        run_table = getattr(self._overview_group, 'runs')
        actual_rows = run_table.nrows
        self._trj_fill_run_table(traj, actual_rows, len(traj))

        # Extract parameter summary and if necessary create new explored parameter tables
        # in the result groups
        for idx in range(old_length, len(traj)):
            run_name = traj.f_idx_to_run(idx)
            run_info = traj.f_get_run_information(run_name)
            run_info['name'] = run_name

            traj._set_explored_parameters_to_idx(idx)

            run_summary = self._srn_summarize_explored_parameters(list(
                                                            traj._explored_parameters.values()))

            run_info['parameter_summary'] = run_summary

            self._all_add_or_modify_row(run_name, run_info, run_table, index=idx,
                                        flags=(HDF5StorageService.MODIFY_ROW,))

        traj.f_restore_default()


    ######################## Loading a Trajectory #################################################

    def _trj_load_trajectory(self, traj, as_new, load_parameters, load_derived_parameters,
                             load_results, load_other_data, recursive, max_depth,
                             with_run_information, with_meta_data, force):
        """Loads a single trajectory from a given file.


        :param traj: The trajectory

        :param as_new: Whether to load trajectory as new

        :param load_parameters: How to load parameters and config

        :param load_derived_parameters: How to load derived parameters

        :param load_results: How to load results

        :param load_other_data: How to load anything not within the four subbranches

        :param recursive: If data should be loaded recursively

        :param max_depth: Maximum depth of loading

        :param with_run_information:

            If run information should be loaded

        :param with_meta_data:

            If meta data infor should be loaded

        :param force: Force load in case there is a pypet version mismatch

        You can specify how to load the parameters, derived parameters and results
        as follows:

        :const:`pypet.pypetconstants.LOAD_NOTHING`: (0)

            Nothing is loaded

        :const:`pypet.pypetconstants.LOAD_SKELETON`: (1)

            The skeleton including annotations are loaded, i.e. the items are empty.
            Non-empty items in RAM are left untouched.

        :const:`pypet.pypetconstants.LOAD_DATA`: (2)

            The whole data is loaded.
            Only empty or in RAM non-existing instance are filled with the
            data found on disk.

        :const:`pypet.pypetconstants.OVERWRITE_DATA`: (3)

            The whole data is loaded.
            If items that are to be loaded are already in RAM and not empty,
            they are emptied and new data is loaded from disk.


        If `as_new=True` the old trajectory is loaded into the new one, only parameters can be
        loaded. If `as_new=False` the current trajectory is completely replaced by the one
        on disk, i.e. the name from disk, the timestamp, etc. are assigned to `traj`.

        """
        # Some validity checks, if `as_new` is used correctly
        if (as_new and (load_derived_parameters != pypetconstants.LOAD_NOTHING or
                                load_results != pypetconstants.LOAD_NOTHING or
                                load_other_data != pypetconstants.LOAD_NOTHING)):
            raise ValueError('You cannot load a trajectory as new and load the derived '
                             'parameters and results. Only parameters are allowed.')

        if as_new and load_parameters != pypetconstants.LOAD_DATA:
            raise ValueError('You cannot load the trajectory as new and not load the data of '
                             'the parameters.')

        loadconstants = (pypetconstants.LOAD_NOTHING, pypetconstants.LOAD_SKELETON,
                         pypetconstants.LOAD_DATA, pypetconstants.OVERWRITE_DATA)

        if not (load_parameters in loadconstants and load_derived_parameters in loadconstants and
                        load_results in loadconstants and load_other_data in loadconstants):
            raise ValueError('Please give a valid option on how to load data. Options for '
                             '`load_parameter`, `load_derived_parameters`, `load_results`, '
                             'and `load_other_data` are %s. See function documentation for '
                             'the semantics of the values.' % str(loadconstants))

        traj._stored = not as_new

        # Loads meta data like the name, timestamps etc.
        # load_data is only used here to determine how to load the annotations
        load_data = max(load_parameters, load_derived_parameters, load_results, load_other_data)
        if with_meta_data:
            self._trj_load_meta_data(traj, load_data, as_new, with_run_information, force)

        if (load_parameters != pypetconstants.LOAD_NOTHING or
                load_derived_parameters != pypetconstants.LOAD_NOTHING or
                load_results != pypetconstants.LOAD_NOTHING or
                load_other_data != pypetconstants.LOAD_NOTHING):
            self._logger.info('Loading trajectory `%s`.' % traj.v_name)
        else:
            self._logger.info('Checked meta data of trajectory `%s`.' % traj.v_name)
            return

        maximum_display_other = 10
        counter = 0

        for children in [self._trajectory_group._v_groups, self._trajectory_group._v_links]:
            for hdf5_group_name in children:
                hdf5_group = children[hdf5_group_name]
                child_name = hdf5_group._v_name

                load_subbranch = True
                if child_name == 'config':
                    if as_new:
                        loading = pypetconstants.LOAD_NOTHING
                    else:
                        # If the trajectory is loaded as new, we don't care about old config stuff
                        # and only load the parameters
                        loading = load_parameters
                elif child_name == 'parameters':
                    loading = load_parameters
                elif child_name == 'results':
                    loading = load_results
                elif child_name == 'derived_parameters':
                    loading = load_derived_parameters
                elif child_name == 'overview':
                    continue
                else:
                    loading = load_other_data
                    load_subbranch = False

                if loading == pypetconstants.LOAD_NOTHING:
                    continue

                if load_subbranch:
                    # Load the subbranches recursively
                    self._logger.info('Loading branch `%s` in mode `%s`.' %
                                          (child_name, str(loading)))
                else:
                    if counter < maximum_display_other:
                        self._logger.info(
                            'Loading branch/node `%s` in mode `%s`.' % (child_name, str(loading)))
                    elif counter == maximum_display_other:
                        self._logger.info('To many branchs or nodes at root for display. '
                                          'I will not inform you about loading anymore. '
                                          'Branches are loaded silently '
                                          'in the background. Do not worry, '
                                          'I will not freeze! Pinky promise!!!')
                    counter += 1

                self._tree_load_sub_branch(traj, child_name, load_data=loading, with_links=True,
                                     recursive=recursive,
                                     max_depth=max_depth,
                                     _trajectory=traj, _as_new=as_new,
                                     _hdf5_group=self._trajectory_group)

    def _trj_load_meta_data(self, traj, load_data, as_new, with_run_information, force):
        """Loads meta information about the trajectory

        Checks if the version number does not differ from current pypet version
        Loads, comment, timestamp, name, version from disk in case trajectory is not loaded
        as new. Updates the run information as well.

        """

        metatable = self._overview_group.info
        metarow = metatable[0]

        try:
            version = metarow['version'].decode('utf-8')
        except (IndexError, ValueError) as ke:
            self._logger.error('Could not check version due to: %s' % str(ke))
            version = '`COULD NOT BE LOADED`'

        try:
            python = metarow['python'].decode('utf-8')
        except (IndexError, ValueError) as ke:
            self._logger.error('Could not check version due to: %s' % str(ke))
            python = '`COULD NOT BE LOADED`'

        self._trj_check_version(version, python, force)

        # Load the skeleton information
        self._grp_load_group(traj, load_data=load_data,
                             with_links=False, recursive=False, _traj=traj,
                             _as_new=as_new, _hdf5_group=self._trajectory_group)

        if as_new:
            length = int(metarow['length'])
            for irun in range(length):
                traj._add_run_info(irun)
        else:
            traj._comment =  metarow['comment'].decode('utf-8')
            traj._timestamp = float(metarow['timestamp'])
            traj._trajectory_timestamp = traj._timestamp
            traj._time = metarow['time'].decode('utf-8')
            traj._trajectory_time = traj._time
            traj._name = metarow['name'].decode('utf-8')
            traj._trajectory_name = traj._name
            traj._version = version
            traj._python = python

            single_run_table = self._overview_group.runs

            if with_run_information:
                for row in single_run_table.iterrows():
                    name = row['name'].decode('utf-8')
                    idx = int(row['idx'])
                    timestamp = float(row['timestamp'])
                    time_ = row['time'].decode('utf-8')
                    completed = int(row['completed'])
                    summary = row['parameter_summary'].decode('utf-8')
                    hexsha = row['short_environment_hexsha'].decode('utf-8')


                    # To allow backwards compatibility we need this try catch block
                    try:
                        runtime = row['runtime'].decode('utf-8')
                        finish_timestamp = float(row['finish_timestamp'])
                    except (IndexError, ValueError) as ke:
                        runtime = ''
                        finish_timestamp = 0.0
                        self._logger.debug('Could not load runtime, ' + repr(ke))

                    info_dict = {'idx': idx,
                                 'timestamp': timestamp,
                                 'finish_timestamp': finish_timestamp,
                                 'runtime': runtime,
                                 'time': time_,
                                 'completed': completed,
                                 'name': name,
                                 'parameter_summary': summary,
                                 'short_environment_hexsha': hexsha}

                    traj._add_run_info(**info_dict)
            else:
                traj._length = single_run_table.nrows

        # Load explorations
        self._trj_load_exploration(traj)
        # Load the hdf5 config data:
        self._srvc_load_hdf5_settings()

    def _srvc_load_hdf5_settings(self):

        def _extract_meta_data(attr_name, row, name_in_row, conversion_function):
            try:
                setattr(self, attr_name, conversion_function(row[name_in_row]))
            except (IndexError, ValueError) as exc:
                self._logger.error('Using default hdf5 setting, '
                                   'could not extract `%s` hdf5 setting because of: %s' %
                                        (name_in_row, repr(exc)))

        if 'hdf5_settings' in self._overview_group:
            hdf5_table = self._overview_group.hdf5_settings
            hdf5_row = hdf5_table[0]

            tostr = lambda x: x.decode('utf-8')
            _extract_meta_data('complib', hdf5_row, 'complib', tostr)
            _extract_meta_data('complevel', hdf5_row, 'complevel', int)
            _extract_meta_data('shuffle', hdf5_row, 'shuffle', bool)
            _extract_meta_data('fletcher32', hdf5_row, 'fletcher32', bool)
            _extract_meta_data('pandas_format', hdf5_row, 'pandas_format', tostr)
            _extract_meta_data('encoding', hdf5_row, 'encoding', tostr)

            _extract_meta_data('_results_per_run', hdf5_row,
                               'results_per_run', int)
            _extract_meta_data('_derived_parameters_per_run', hdf5_row,
                               'derived_parameters_per_run', int)
            _extract_meta_data('_purge_duplicate_comments', hdf5_row,
                               'purge_duplicate_comments', bool)

            for attr_name, table_name in self.NAME_TABLE_MAPPING.items():
                _extract_meta_data(attr_name, hdf5_row, table_name, bool)
        else:
            self._logger.warning('Could not find `hdf5_settings` overview table. I will use the '
                                 'standard settings (for `complib`, `complevel` etc.) instead.')

    def _tree_load_sub_branch(self, traj_node, branch_name,
                              load_data=pypetconstants.LOAD_DATA,
                              with_links=True, recursive=False,
                              max_depth=None, _trajectory=None,
                              _as_new=False, _hdf5_group=None):
        """Loads data starting from a node along a branch and starts recursively loading
        all data at end of branch.

        :param traj_node: The node from where loading starts

        :param branch_name:

            A branch along which loading progresses. Colon Notation is used:
            'group1.group2.group3' loads 'group1', then 'group2', then 'group3' and then finally
            recursively all children and children's children below 'group3'

        :param load_data:

            How to load the data


        :param with_links:

            If links should be loaded

        :param recursive:

            If loading recursively

        :param max_depth:

            The maximum depth to load the tree

        :param _trajectory:

            The trajectory

        :param _as_new:

            If trajectory is loaded as new

        :param _hdf5_group:

            HDF5 node in the file corresponding to `traj_node`.

        """
        if load_data == pypetconstants.LOAD_NOTHING:
            return

        if max_depth is None:
            max_depth = float('inf')

        if _trajectory is None:
            _trajectory = traj_node.v_root

        if _hdf5_group is None:
            hdf5_group_name = traj_node.v_full_name.replace('.', '/')

            # Get child node to load
            if hdf5_group_name == '':
                _hdf5_group = self._trajectory_group
            else:
                try:
                    _hdf5_group = self._hdf5file.get_node(where=self._trajectory_group,
                                                  name=hdf5_group_name)
                except pt.NoSuchNodeError:
                    self._logger.error('Cannot find `%s` the hdf5 node `%s` does not exist!'
                                       % (traj_node.v_full_name, hdf5_group_name))
                    raise

        split_names = branch_name.split('.')

        final_group_name = split_names.pop()

        current_depth = 1

        for name in split_names:
            if current_depth > max_depth:
                return
            # First load along the branch
            _hdf5_group = getattr(_hdf5_group, name)

            self._tree_load_nodes_dfs(traj_node, load_data=load_data, with_links=with_links,
                                  recursive=False, max_depth=max_depth, current_depth=current_depth,
                                  trajectory=_trajectory, as_new=_as_new,
                                  hdf5_group=_hdf5_group)

            current_depth += 1

            traj_node = traj_node._children[name]

        if current_depth <= max_depth:
            # Then load recursively all data in the last group and below
            _hdf5_group = getattr(_hdf5_group, final_group_name)
            self._tree_load_nodes_dfs(traj_node, load_data=load_data, with_links=with_links,
                                  recursive=recursive, max_depth=max_depth,
                                  current_depth=current_depth, trajectory=_trajectory,
                                  as_new=_as_new, hdf5_group=_hdf5_group)

    def _trj_check_version(self, version, python, force):
        """Checks for version mismatch

        Raises a VersionMismatchError if version of loaded trajectory and current pypet version
        do not match. In case of `force=True` error is not raised only a warning is emitted.

        """
        curr_python = pypetconstants.python_version_string

        if (version != VERSION or curr_python != python) and not force:
            raise pex.VersionMismatchError('Current pypet version is %s used under python %s '
                                           '  but your trajectory'
                                           ' was created with version %s and python %s.'
                                           ' Use >>force=True<< to perform your load regardless'
                                           ' of version mismatch.' %
                                           (VERSION, curr_python, version, python))
        elif version != VERSION or curr_python != python:
            self._logger.warning('Current pypet version is %s with python %s but your trajectory'
                                 ' was created with version %s under python %s.'
                                 ' Yet, you enforced the load, so I will'
                                 ' handle the trajectory despite the'
                                 ' version mismatch.' %
                                 (VERSION, curr_python, version, python))


    #################################### Storing a Trajectory ####################################

    def _trj_fill_run_table(self, traj, start, stop):
        """Fills the `run` overview table with information.

        Will also update new information.

        """

        def _make_row(info_dict):
            row = (info_dict['idx'],
                   info_dict['name'],
                   info_dict['time'],
                   info_dict['timestamp'],
                   info_dict['finish_timestamp'],
                   info_dict['runtime'],
                   info_dict['parameter_summary'],
                   info_dict['short_environment_hexsha'],
                   info_dict['completed'])
            return row

        runtable = getattr(self._overview_group, 'runs')

        rows = []
        updated_run_information = traj._updated_run_information
        for idx in range(start, stop):
            info_dict = traj._run_information[traj._single_run_ids[idx]]
            rows.append(_make_row(info_dict))
            updated_run_information.discard(idx)

        if rows:
            runtable.append(rows)
            runtable.flush()

        # Store all runs that are updated and that have not been stored yet
        rows = []
        indices = []
        for idx in updated_run_information:
            info_dict = traj.f_get_run_information(idx, copy=False)
            rows.append(_make_row(info_dict))
            indices.append(idx)

        if rows:
            runtable.modify_coordinates(indices, rows)

        traj._updated_run_information = set()

    def _trj_store_meta_data(self, traj):
        """ Stores general information about the trajectory in the hdf5file.

        The `info` table will contain the name of the trajectory, it's timestamp, a comment,
        the length (aka the number of single runs), and the current version number of pypet.

        Also prepares the desired overview tables and fills the `run` table with dummies.

        """

        # Description of the `info` table
        descriptiondict = {'name': pt.StringCol(pypetconstants.HDF5_STRCOL_MAX_LOCATION_LENGTH,
                                                pos=0),
                           'time': pt.StringCol(len(traj.v_time), pos=1),
                           'timestamp': pt.FloatCol(pos=3),
                           'comment': pt.StringCol(pypetconstants.HDF5_STRCOL_MAX_COMMENT_LENGTH,
                                                   pos=4),
                           'length': pt.IntCol(pos=2),
                           'version': pt.StringCol(pypetconstants.HDF5_STRCOL_MAX_NAME_LENGTH,
                                                   pos=5),
                           'python': pt.StringCol(pypetconstants.HDF5_STRCOL_MAX_NAME_LENGTH,
                                                  pos=5)}
        # 'loaded_from' : pt.StringCol(pypetconstants.HDF5_STRCOL_MAX_LOCATION_LENGTH)}

        infotable = self._all_get_or_create_table(where=self._overview_group, tablename='info',
                                                  description=descriptiondict,
                                                  expectedrows=len(traj))

        insert_dict = self._all_extract_insert_dict(traj, infotable.colnames)
        self._all_add_or_modify_row(traj.v_name, insert_dict, infotable, index=0,
                                    flags=(HDF5StorageService.ADD_ROW,
                                           HDF5StorageService.MODIFY_ROW))

        # Description of the `run` table
        rundescription_dict = {'name': pt.StringCol(pypetconstants.HDF5_STRCOL_MAX_NAME_LENGTH,
                                                    pos=1),
                               'time': pt.StringCol(len(traj.v_time), pos=2),
                               'timestamp': pt.FloatCol(pos=3),
                               'idx': pt.IntCol(pos=0),
                               'completed': pt.IntCol(pos=8),
                               'parameter_summary': pt.StringCol(
                                   pypetconstants.HDF5_STRCOL_MAX_COMMENT_LENGTH,
                                   pos=6),
                               'short_environment_hexsha': pt.StringCol(7, pos=7),
                               'finish_timestamp': pt.FloatCol(pos=4),
                               'runtime': pt.StringCol(
                                   pypetconstants.HDF5_STRCOL_MAX_RUNTIME_LENGTH,
                                   pos=5)}

        runtable = self._all_get_or_create_table(where=self._overview_group,
                                                 tablename='runs',
                                                 description=rundescription_dict)

        hdf5_description_dict = {'complib': pt.StringCol(7, pos=0),
                                 'complevel': pt.IntCol(pos=1),
                                 'shuffle': pt.BoolCol(pos=2),
                                 'fletcher32': pt.BoolCol(pos=3),
                                 'pandas_format': pt.StringCol(7, pos=4),
                                 'encoding': pt.StringCol(11, pos=5)}

        pos = 7
        for name, table_name in HDF5StorageService.NAME_TABLE_MAPPING.items():
            hdf5_description_dict[table_name] = pt.BoolCol(pos=pos)
            pos += 1

        # Store the hdf5 properties in an overview table
        hdf5_description_dict.update({'purge_duplicate_comments': pt.BoolCol(pos=pos + 2),
                                      'results_per_run': pt.IntCol(pos=pos + 3),
                                      'derived_parameters_per_run': pt.IntCol(pos=pos + 4)})

        hdf5table = self._all_get_or_create_table(where=self._overview_group,
                                                  tablename='hdf5_settings',
                                                  description=hdf5_description_dict)

        insert_dict = {}
        for attr_name in self.ATTR_LIST:
            insert_dict[attr_name] = getattr(self, attr_name)

        for attr_name, table_name in self.NAME_TABLE_MAPPING.items():
            insert_dict[table_name] = getattr(self, attr_name)

        for attr_name, name in self.PR_ATTR_NAME_MAPPING.items():
            insert_dict[name] = getattr(self, attr_name)

        self._all_add_or_modify_row(traj.v_name, insert_dict, hdf5table, index=0,
                                    flags=(HDF5StorageService.ADD_ROW,
                                           HDF5StorageService.MODIFY_ROW))


        # Fill table with dummy entries starting from the current table size
        actual_rows = runtable.nrows
        self._trj_fill_run_table(traj, actual_rows, len(traj._run_information))
        # stop != len(traj) to allow immediate post-proc with QUEUE wrapping

        # Store the annotations and comment of the trajectory node
        self._grp_store_group(traj, store_data=pypetconstants.STORE_DATA,
                                  with_links=False,
                                  recursive=False,
                                  _hdf5_group=self._trajectory_group)

        # Store the list of explored paramters
        self._trj_store_explorations(traj)

        # Prepare the exploration tables
        # Prepare the overview tables
        tostore_tables = []

        for name, table_name in HDF5StorageService.NAME_TABLE_MAPPING.items():

            # Check if we want the corresponding overview table
            # If the trajectory does not contain information about the table
            # we assume it should be created.

            if getattr(self, name):
                tostore_tables.append(table_name)

        self._srvc_make_overview_tables(tostore_tables, traj)

    def _trj_load_exploration(self, traj):
        """Recalls names of all explored parameters"""
        if hasattr(self._overview_group, 'explorations'):
            explorations_table = self._overview_group._f_get_child( 'explorations')
            for row in explorations_table.iterrows():
                param_name = row['explorations'].decode('utf-8')
                if param_name not in traj._explored_parameters:
                    traj._explored_parameters[param_name] = None
        else:
            # This is for backwards compatibility
            for what in ('parameters', 'derived_parameters'):
                if hasattr(self._trajectory_group, what):
                    parameters = self._trajectory_group._f_get_child(what)
                    for group in parameters._f_walk_groups():
                        if self._all_get_from_attrs(group, HDF5StorageService.LENGTH):
                            group_location = group._v_pathname
                            full_name = '.'.join(group_location.split('/')[2:])
                            traj._explored_parameters[full_name] = None

    def _trj_store_explorations(self, traj):
        """Stores a all explored parameter names for internal recall"""
        nexplored = len(traj._explored_parameters)
        if nexplored > 0:
            if hasattr(self._overview_group, 'explorations'):
                explorations_table = self._overview_group._f_get_child('explorations')
                if len(explorations_table) != nexplored:
                    self._hdf5file.remove_node(where=self._overview_group,
                                         name='explorations')
        if not hasattr(self._overview_group, 'explorations'):
            explored_list = list(traj._explored_parameters.keys())
            if explored_list:
                string_col = self._all_get_table_col('explorations',
                                                      explored_list,
                                                      'overview.explorations')
            else:
                string_col = pt.StringCol(1)
            description = {'explorations': string_col}
            explorations_table = self._hdf5file.create_table(where=self._overview_group,
                                                       name='explorations',
                                                       description=description)
            rows = [(x.encode('utf-8'),) for x in explored_list]
            if rows:
                explorations_table.append(rows)
                explorations_table.flush()

    def _srvc_make_overview_tables(self, tables_to_make, traj=None):
        """Creates the overview tables in overview group"""
        for table_name in tables_to_make:
            # Prepare the tables desciptions, depending on which overview table we create
            # we need different columns
            paramdescriptiondict = {}
            expectedrows = 0

            # Every overview table has a name and location column
            paramdescriptiondict['location'] = pt.StringCol(
                pypetconstants.HDF5_STRCOL_MAX_LOCATION_LENGTH,
                pos=0)
            paramdescriptiondict['name'] = pt.StringCol(pypetconstants.HDF5_STRCOL_MAX_NAME_LENGTH,
                                                        pos=1)

            paramdescriptiondict['comment'] = pt.StringCol(
                    pypetconstants.HDF5_STRCOL_MAX_COMMENT_LENGTH)

            paramdescriptiondict['value'] = pt.StringCol(
                    pypetconstants.HDF5_STRCOL_MAX_VALUE_LENGTH, pos=2)

            if table_name == 'config_overview':
                if traj is not None:
                    expectedrows = len(traj._config)

            if table_name == 'parameters_overview':
                if traj is not None:
                    expectedrows = len(traj._parameters)

            if table_name == 'explored_parameters_overview':
                paramdescriptiondict['range'] = pt.StringCol(
                    pypetconstants.HDF5_STRCOL_MAX_RANGE_LENGTH)
                paramdescriptiondict['length'] = pt.IntCol()
                if traj is not None:
                    expectedrows = len(traj._explored_parameters)

            if table_name.endswith('summary'):
                paramdescriptiondict['hexdigest'] = pt.StringCol(64, pos=10)

            # Check if the user provided an estimate of the amount of results per run
            # This can help to speed up storing
            if table_name == 'derived_parameters_overview':

                expectedrows = self._derived_parameters_per_run

                if traj is not None:
                    expectedrows *= len(traj)
                    expectedrows += len(traj._derived_parameters)

            if table_name == 'results_overview':
                expectedrows = self._results_per_run

                if traj is not None:
                    expectedrows *= len(traj)
                    expectedrows += len(traj._results)

            if expectedrows > 0:
                paramtable = self._all_get_or_create_table(where=self._overview_group,
                                                           tablename=table_name,
                                                           description=paramdescriptiondict,
                                                           expectedrows=expectedrows)
            else:
                paramtable = self._all_get_or_create_table(where=self._overview_group,
                                                           tablename=table_name,
                                                           description=paramdescriptiondict)

            paramtable.flush()

    def _trj_store_trajectory(self, traj, only_init=False, store_data=pypetconstants.STORE_DATA,
                              max_depth=None):
        """ Stores a trajectory to an hdf5 file

        Stores all groups, parameters and results

        """
        if not only_init:
            self._logger.info('Start storing Trajectory `%s`.' % self._trajectory_name)
        else:
            self._logger.info('Initialising storage or updating meta data of Trajectory `%s`.' %
                              self._trajectory_name)
            store_data = pypetconstants.STORE_NOTHING

        # In case we accidentally chose a trajectory name that already exist
        # We do not want to mess up the stored trajectory but raise an Error
        if not traj._stored and self._trajectory_group is not None:
            raise RuntimeError('You want to store a completely new trajectory with name'
                               ' `%s` but this trajectory is already found in file `%s`.'
                               'Did you try to accidentally overwrite existing data? If '
                               'you DO want to override existing data, use `overwrite_file=True`.'
                               'Note that this deletes the whole HDF5 file not just the particular '
                               'trajectroy therein! ' %
                               (traj.v_name, self._filename))

        # Extract HDF5 settings from the trajectory
        self._srvc_check_hdf_properties(traj)

        # Store the trajectory for the first time if necessary:
        if self._trajectory_group is None:
            self._trajectory_group = self._hdf5file.create_group(where='/',
                                                           name=self._trajectory_name,
                                                           title=self._trajectory_name,
                                                           filters=self._all_get_filters())
        # Store meta information
        self._trj_store_meta_data(traj)

        # # Store recursively the config subtree
        # self._tree_store_recursively(pypetconstants.LEAF,traj.config,self._trajectory_group)

        if store_data in (pypetconstants.STORE_DATA_SKIPPING,
                          pypetconstants.STORE_DATA,
                          pypetconstants.OVERWRITE_DATA):

            counter = 0
            maximum_display_other = 10
            name_set = set(['parameters', 'config', 'derived_parameters', 'results'])

            for child_name in traj._children:

                if child_name in name_set:
                    self._logger.info('Storing branch `%s`.' % child_name)
                else:
                    if counter < maximum_display_other:
                        self._logger.info('Storing branch/node `%s`.' % child_name)
                    elif counter == maximum_display_other:
                        self._logger.info('To many branches or nodes at root for display. '
                                          'I will not inform you about storing anymore. '
                                          'Branches are stored silently in the background. '
                                          'Do not worry, I will not freeze! Pinky promise!!!')
                    counter += 1

                # Store recursively the elements
                self._tree_store_sub_branch(traj, child_name, store_data=store_data,
                                            with_links=True,
                                            recursive=True, max_depth=max_depth,
                                            hdf5_group=self._trajectory_group)

            self._logger.info('Finished storing Trajectory `%s`.' % self._trajectory_name)
        else:
            self._logger.info('Finished init or meta data update for `%s`.' %
                              self._trajectory_name)
        traj._stored = True

    def _tree_store_sub_branch(self, traj_node, branch_name,
                               store_data=pypetconstants.STORE_DATA,
                               with_links=True,
                               recursive=False,
                               max_depth=None,
                               hdf5_group=None):
        """Stores data starting from a node along a branch and starts recursively loading
        all data at end of branch.

        :param traj_node: The node where storing starts

        :param branch_name:

            A branch along which storing progresses. Colon Notation is used:
            'group1.group2.group3' loads 'group1', then 'group2', then 'group3', and then finally
            recursively all children and children's children below 'group3'.

        :param store_data: How data should be stored

        :param with_links: If links should be stored

        :param recursive:

            If the rest of the tree should be recursively stored

        :param max_depth:

            Maximum depth to store

        :param hdf5_group:

            HDF5 node in the file corresponding to `traj_node`

        """
        if store_data == pypetconstants.STORE_NOTHING:
            return

        if max_depth is None:
            max_depth = float('inf')

        if hdf5_group is None:
            # Get parent hdf5 node
            location = traj_node.v_full_name
            hdf5_location = location.replace('.', '/')
            try:
                if location == '':
                    hdf5_group = self._trajectory_group
                else:
                    hdf5_group = self._hdf5file.get_node( where=self._trajectory_group,
                                                         name=hdf5_location)
            except pt.NoSuchNodeError:
                self._logger.debug('Cannot store `%s` the parental hdf5 node with path `%s` does '
                                     'not exist on disk.' %
                                     (traj_node.v_name, hdf5_location))

                if traj_node.v_is_leaf:
                    self._logger.error('Cannot store `%s` the parental hdf5 '
                                       'node with path `%s` does '
                                       'not exist on disk! The child '
                                       'you want to store is a leaf node,'
                                       'that cannot be stored without '
                                       'the parental node existing on '
                                       'disk.' % (traj_node.v_name, hdf5_location))
                    raise
                else:
                    self._logger.debug('I will try to store the path from trajectory root to '
                                         'the child now.')

                    self._tree_store_sub_branch(traj_node._nn_interface._root_instance,
                                                traj_node.v_full_name + '.' + branch_name,
                                                store_data=store_data, with_links=with_links,
                                                recursive=recursive,
                                                max_depth=max_depth + traj_node.v_depth,
                                                hdf5_group=self._trajectory_group)
                    return

        current_depth = 1

        split_names = branch_name.split('.')

        leaf_name = split_names.pop()

        for name in split_names:
            if current_depth > max_depth:
                return
            # Store along a branch
            self._tree_store_nodes_dfs(traj_node, name, store_data=store_data, with_links=with_links,
                                   recursive=False, max_depth=max_depth,
                                   current_depth=current_depth, parent_hdf5_group=hdf5_group)
            current_depth += 1

            traj_node = traj_node._children[name]

            hdf5_group = getattr(hdf5_group, name)

        # Store final group and recursively everything below it
        if current_depth <= max_depth:
            self._tree_store_nodes_dfs(traj_node, leaf_name, store_data=store_data,
                               with_links=with_links, recursive=recursive,
                               max_depth=max_depth, current_depth=current_depth,
                               parent_hdf5_group=hdf5_group)


    ########################  Storing and Loading Sub Trees #######################################

    def _tree_create_leaf(self, name, trajectory, hdf5_group):
        """ Creates a new pypet leaf instance.

        Returns the leaf and if it is an explored parameter the length of the range.

        """
        class_name = self._all_get_from_attrs(hdf5_group, HDF5StorageService.CLASS_NAME)

        # Create the instance with the appropriate constructor
        class_constructor = trajectory._create_class(class_name)

        instance = trajectory._construct_instance(class_constructor, name)

        return instance

    def _tree_load_nodes_dfs(self, parent_traj_node, load_data, with_links, recursive,
                         max_depth, current_depth, trajectory, as_new, hdf5_group):
        """Loads a node from hdf5 file and if desired recursively everything below

        :param parent_traj_node: The parent node whose child should be loaded
        :param load_data: How to load the data
        :param with_links: If links should be loaded
        :param recursive: Whether loading recursively below hdf5_group
        :param max_depth: Maximum depth
        :param current_depth: Current depth
        :param trajectory: The trajectory object
        :param as_new: If trajectory is loaded as new
        :param hdf5_group: The hdf5 group containing the child to be loaded

        """
        if max_depth is None:
            max_depth = float('inf')

        loading_list = [(parent_traj_node, current_depth, hdf5_group)]

        while loading_list:
            parent_traj_node, current_depth, hdf5_group = loading_list.pop()

            if isinstance(hdf5_group, pt.link.SoftLink):
                if with_links:
                    # We end up here when auto-loading a soft link
                    self._tree_load_link(parent_traj_node, load_data=load_data, traj=trajectory,
                                         as_new=as_new, hdf5_soft_link=hdf5_group)
                continue


            name = hdf5_group._v_name
            is_leaf = self._all_get_from_attrs(hdf5_group, HDF5StorageService.LEAF)
            in_trajectory = name in parent_traj_node._children

            if is_leaf:
                # In case we have a leaf node, we need to check if we have to create a new
                # parameter or result

                if in_trajectory:
                    instance = parent_traj_node._children[name]
                # Otherwise we need to create a new instance
                else:
                    instance = self._tree_create_leaf(name, trajectory, hdf5_group)

                    # Add the instance to the trajectory tree
                    parent_traj_node._add_leaf_from_storage(args=(instance,), kwargs={})

                self._prm_load_parameter_or_result(instance, load_data=load_data,
                                                   _hdf5_group=hdf5_group)
                if as_new:
                    instance._stored = False

            else:
                if in_trajectory:
                    traj_group = parent_traj_node._children[name]

                    if load_data == pypetconstants.OVERWRITE_DATA:
                        traj_group.v_annotations.f_empty()
                        traj_group.v_comment = ''
                else:
                    if HDF5StorageService.CLASS_NAME in hdf5_group._v_attrs:
                        class_name = self._all_get_from_attrs(hdf5_group,
                                                              HDF5StorageService.CLASS_NAME)
                        class_constructor = trajectory._create_class(class_name)
                        instance = trajectory._construct_instance(class_constructor, name)
                        args = (instance,)
                    else:
                        args = (name,)
                    # If the group does not exist create it'
                    traj_group = parent_traj_node._add_group_from_storage(args=args, kwargs={})

                # Load annotations and comment
                self._grp_load_group(traj_group, load_data=load_data, with_links=with_links,
                                     recursive=False, max_depth=max_depth,
                                     _traj=trajectory, _as_new=as_new,
                                     _hdf5_group=hdf5_group)

                if recursive and current_depth < max_depth:
                    new_depth = current_depth + 1
                    for children in (hdf5_group._v_groups, hdf5_group._v_links):
                        for new_hdf5_group_name in children:
                            new_hdf5_group = children[new_hdf5_group_name]
                            loading_list.append((traj_group, new_depth, new_hdf5_group))

    def _tree_load_link(self, new_traj_node, load_data, traj, as_new, hdf5_soft_link):
        """ Loads a link
        
        :param new_traj_node: Node in traj containing link 
        :param load_data: How to load data in the linked node
        :param traj: The trajectory
        :param as_new: If data in linked node should be loaded as new
        :param hdf5_soft_link: The hdf5 soft link

        """
        try:
            linked_group = hdf5_soft_link()
            link_name = hdf5_soft_link._v_name

            if (not link_name in new_traj_node._links or
                        load_data==pypetconstants.OVERWRITE_DATA):

                link_location = linked_group._v_pathname
                full_name = '.'.join(link_location.split('/')[2:])
                if not full_name in traj:
                    self._tree_load_sub_branch(traj, full_name,
                                               load_data=pypetconstants.LOAD_SKELETON,
                                               with_links=False, recursive=False, _trajectory=traj,
                                               _as_new=as_new, _hdf5_group=self._trajectory_group)

                if (load_data == pypetconstants.OVERWRITE_DATA and
                            link_name in new_traj_node._links):
                    new_traj_node.f_remove_link(link_name)
                if not link_name in new_traj_node._links:
                    new_traj_node._nn_interface._add_generic(new_traj_node,
                                                                type_name=nn.LINK,
                                                                group_type_name=nn.GROUP,
                                                                args=(link_name,
                                                                      traj.f_get(full_name)),
                                                                kwargs={},
                                                                add_prefix=False,
                                                                check_naming=False)
                else:
                    raise RuntimeError('You shall not pass!')
        except pt.NoSuchNodeError:
            self._logger.error('Link `%s` under `%s` is broken, cannot load it, '
                               'I will ignore it, you have to '
                               'manually delete it!' %
                               (hdf5_soft_link._v_name, new_traj_node.v_full_name))

    def _tree_store_nodes_dfs(self, parent_traj_node, name, store_data, with_links, recursive,
                          max_depth, current_depth,
                          parent_hdf5_group):
        """Stores a node to hdf5 and if desired stores recursively everything below it.

        :param parent_traj_node: The parental node
        :param name: Name of node to be stored
        :param store_data: How to store data
        :param with_links: If links should be stored
        :param recursive: Whether to store recursively the subtree
        :param max_depth: Maximum recursion depth in tree
        :param current_depth: Current depth
        :param parent_hdf5_group: Parent hdf5 group

        """
        if max_depth is None:
            max_depth = float('inf')

        store_list = [(parent_traj_node, name, current_depth, parent_hdf5_group)]

        while store_list:
            parent_traj_node, name, current_depth, parent_hdf5_group = store_list.pop()

            # Check if we create a link
            if name in parent_traj_node._links:
                if with_links:
                    self._tree_store_link(parent_traj_node, name, parent_hdf5_group)
                continue

            traj_node = parent_traj_node._children[name]

            # If the node does not exist in the hdf5 file create it
            if not hasattr(parent_hdf5_group, name):
                newly_created = True
                new_hdf5_group = self._hdf5file.create_group(where=parent_hdf5_group,
                                                       name=name, filters=self._all_get_filters())
            else:
                newly_created = False
                new_hdf5_group = getattr(parent_hdf5_group, name)

            if traj_node.v_is_leaf:
                self._prm_store_parameter_or_result(traj_node, store_data=store_data,
                                                     _hdf5_group=new_hdf5_group,
                                                    _newly_created=newly_created)

            else:
                self._grp_store_group(traj_node, store_data=store_data, with_links=with_links,
                                      recursive=False, max_depth=max_depth,
                                      _hdf5_group=new_hdf5_group,
                                      _newly_created=newly_created)

                if recursive and current_depth < max_depth:
                    for child in traj_node._children.keys():
                        store_list.append((traj_node, child, current_depth + 1, new_hdf5_group))

    def _tree_store_link(self, node_in_traj, link, hdf5_group):
        """Creates a soft link.
        
        :param node_in_traj: parental node
        :param store_data: how to store data
        :param link: name of link
        :param hdf5_group: current parental hdf5 group
        """

        if hasattr(hdf5_group, link):
            return

        linked_traj_node = node_in_traj._links[link]
        linking_name = linked_traj_node.v_full_name.replace('.','/')
        linking_name = '/' + self._trajectory_name + '/' + linking_name
        try:
            to_link_hdf5_group = self._hdf5file.get_node(where=linking_name)
        except pt.NoSuchNodeError:
            self._logger.debug('Could not store link `%s` under `%s` immediately, '
                               'need to store `%s` first. '
                               'Will store the link right after.' % (link,
                                                                     node_in_traj.v_full_name,
                                                                     linked_traj_node.v_full_name))
            root = node_in_traj._nn_interface._root_instance
            self._tree_store_sub_branch(root, linked_traj_node.v_full_name,
                                        store_data=pypetconstants.STORE_DATA_SKIPPING,
                                        with_links=False, recursive=False,
                                        hdf5_group=self._trajectory_group)
            to_link_hdf5_group = self._hdf5file.get_node(where=linking_name)
        self._hdf5file.create_soft_link(where=hdf5_group,
                                  name=link,
                                  target=to_link_hdf5_group)

    ######################## Storing a Single Run ##########################################

    def _srn_store_single_run(self, traj,
                              recursive=True,
                              store_data=pypetconstants.STORE_DATA,
                              max_depth=None):
        """ Stores a single run instance to disk (only meta data)"""

        if store_data != pypetconstants.STORE_NOTHING:
            self._logger.debug('Storing Data of single run `%s`.' % traj.v_crun)
            if max_depth is None:
                max_depth = float('inf')
            for name_pair in traj._new_nodes:
                _, name = name_pair
                parent_group, child_node = traj._new_nodes[name_pair]
                if not child_node._stored:
                    self._tree_store_sub_branch(parent_group, name,
                                          store_data=store_data,
                                          with_links=True,
                                          recursive=recursive,
                                          max_depth=max_depth - child_node.v_depth,
                                          hdf5_group=None)
            for name_pair in traj._new_links:
                _, link = name_pair
                parent_group, _ = traj._new_links[name_pair]
                self._tree_store_sub_branch(parent_group, link,
                                            store_data=store_data,
                                            with_links=True,
                                            recursive=recursive,
                                            max_depth=max_depth - parent_group.v_depth - 1,
                                            hdf5_group=None)

    def _srn_summarize_explored_parameters(self, paramlist):
        """Summarizes the parameter settings.

        :param run_name: Name of the single run

        :param paramlist: List of explored parameters

        :param add_table: Whether to add the overview table

        :param create_run_group:

            If a group with the particular name should be created if it does not exist.
            Might be necessary when trajectories are merged.

        """

        runsummary = ''
        paramlist = sorted(paramlist, key=lambda name: name.v_name + name.v_location)
        for idx, expparam in enumerate(paramlist):

            # Create the run summary for the `run` overview
            if idx > 0:
                runsummary += ',   '

            valstr = expparam.f_val_to_str()

            if len(valstr) >= pypetconstants.HDF5_STRCOL_MAX_COMMENT_LENGTH:
                valstr = valstr[0:pypetconstants.HDF5_STRCOL_MAX_COMMENT_LENGTH - 3]
                valstr += '...'

            if expparam.v_name in runsummary:
                param_name = expparam.v_full_name
            else:
                param_name = expparam.v_name

            runsummary = runsummary + param_name + ': ' + valstr

        return runsummary


    ################# Methods used across Storing and Loading different Items ##################

    def _all_store_param_or_result_table_entry(self, instance, table, flags,
                                               additional_info=None):
        """Stores a single row into an overview table

        :param instance: A parameter or result instance

        :param table: Table where row will be inserted

        :param flags:

            Flags how to insert into the table. Potential Flags are
            `ADD_ROW`, `REMOVE_ROW`, `MODIFY_ROW`

        :param additional_info:

            Dictionary containing information that cannot be extracted from
            `instance`, but needs to be inserted, too.


        """
        # assert isinstance(table, pt.Table)

        location = instance.v_location
        name = instance.v_name
        fullname = instance.v_full_name

        if (flags == (HDF5StorageService.ADD_ROW,) and table.nrows < 2
                and 'location' in table.colnames):
            # We add the modify row option here because you cannot delete the very first
            # row of the table, so there is the rare condition, that the row might already
            # exist.
            # We also need to check if 'location' is in the columns in order to avoid
            # confusion with the smaller explored parameter overviews
            flags = (HDF5StorageService.ADD_ROW, HDF5StorageService.MODIFY_ROW)

        if flags == (HDF5StorageService.ADD_ROW,):
            # If we are sure we only want to add a row we do not need to search!
            condvars = None
            condition = None
        else:
            # Condition to search for an entry
            condvars = {'namecol': table.cols.name, 'locationcol': table.cols.location,
                        'name': name, 'location': location}

            condition = """(namecol == name) & (locationcol == location)"""

        if HDF5StorageService.REMOVE_ROW in flags:
            # If we want to remove a row, we don't need to extract information
            insert_dict = {}
        else:
            # Extract information to insert from the instance and the additional info dict
            colnames = set(table.colnames)
            insert_dict = self._all_extract_insert_dict(instance, colnames, additional_info)

        # Write the table entry
        self._all_add_or_modify_row(fullname, insert_dict, table, condition=condition,
                                    condvars=condvars, flags=flags)


    def _all_get_or_create_table(self, where, tablename, description, expectedrows=None):
        """Creates a new table, or if the table already exists, returns it."""
        where_node = self._hdf5file.get_node(where)

        if not tablename in where_node:
            if not expectedrows is None:
                table = self._hdf5file.create_table(where=where_node, name=tablename,
                                              description=description, title=tablename,
                                              expectedrows=expectedrows,
                                              filters=self._all_get_filters())
            else:
                table = self._hdf5file.create_table(where=where_node, name=tablename,
                                              description=description, title=tablename,
                                              filters=self._all_get_filters())
        else:
            table = where_node._f_get_child(tablename)

        return table

    def _all_get_node_by_name(self, name):
        """Returns an HDF5 node by the path specified in `name`"""
        path_name = name.replace('.', '/')
        where = '/%s/%s' % (self._trajectory_name, path_name)
        return self._hdf5file.get_node(where=where)

    @staticmethod
    def _all_get_from_attrs(ptitem, name):
        """Gets an attribute `name` from `ptitem`, returns None if attribute does not exist."""
        try:
            return getattr(ptitem._v_attrs, name)
        except AttributeError:
            return None

    @staticmethod
    def _all_set_attr(ptitem, name, value):
        """Sets an attribute `name` from `ptitem`"""
        return setattr(ptitem._v_attrs, name, value)

    @staticmethod
    def _all_set_attributes_to_recall_natives(data, ptitem, prefix):
        """Stores original data type to hdf5 node attributes for preserving the data type.

        :param data:

            Data to be stored

        :param ptitem:

            HDF5 node to store data types as attributes. Can also be just a PTItemMock.

        :param prefix:

            String prefix to label and name data in HDF5 attributes

        """

        # If `data` is a container, remember the container type
        if type(data) is tuple:
            HDF5StorageService._all_set_attr(ptitem, prefix + HDF5StorageService.COLL_TYPE,
                                   HDF5StorageService.COLL_TUPLE)

        elif type(data) is list:
            HDF5StorageService._all_set_attr(ptitem, prefix + HDF5StorageService.COLL_TYPE,
                                   HDF5StorageService.COLL_LIST)

        elif type(data) is np.ndarray:
            HDF5StorageService._all_set_attr(ptitem, prefix + HDF5StorageService.COLL_TYPE,
                                   HDF5StorageService.COLL_NDARRAY)

        elif type(data) is np.matrix:
            HDF5StorageService._all_set_attr(ptitem, prefix + HDF5StorageService.COLL_TYPE,
                                   HDF5StorageService.COLL_MATRIX)

        elif type(data) in pypetconstants.PARAMETER_SUPPORTED_DATA:
            HDF5StorageService._all_set_attr(ptitem, prefix + HDF5StorageService.COLL_TYPE,
                                   HDF5StorageService.COLL_SCALAR)

            strtype = type(data).__name__

            if not strtype in pypetconstants.PARAMETERTYPEDICT:
                raise TypeError('I do not know how to handle `%s` its type is `%s`.' %
                                (str(data), repr(type(data))))

            HDF5StorageService._all_set_attr(ptitem, prefix + HDF5StorageService.SCALAR_TYPE, strtype)

        elif type(data) is dict:
            if len(data) > 0:
                HDF5StorageService._all_set_attr(ptitem, prefix + HDF5StorageService.COLL_TYPE,
                                   HDF5StorageService.COLL_DICT)
            else:
                HDF5StorageService._all_set_attr(ptitem, prefix + HDF5StorageService.COLL_TYPE,
                                   HDF5StorageService.COLL_EMPTY_DICT)
        else:
            raise TypeError('I do not know how to handle `%s` its type is `%s`.' %
                            (str(data), repr(type(data))))

        if type(data) in (list, tuple):
            # If data is a list or tuple we need to remember the data type of the elements
            # in the list or tuple.
            # We do NOT need to remember the elements of `dict` explicitly, though.
            # `dict` is stored
            # as an `ObjectTable` and thus types are already conserved.
            if len(data) > 0:
                strtype = type(data[0]).__name__

                if not strtype in pypetconstants.PARAMETERTYPEDICT:
                    raise TypeError('I do not know how to handle `%s` its type is '
                                    '`%s`.' % (str(data), strtype))

                HDF5StorageService._all_set_attr(ptitem, prefix +
                                       HDF5StorageService.SCALAR_TYPE, strtype)
        elif (type(data) in (np.ndarray, np.matrix) and
                  np.issubdtype(data.dtype, str)):
            HDF5StorageService._all_set_attr(ptitem, prefix + HDF5StorageService.SCALAR_TYPE,
                                   str.__name__)

    def _all_recall_native_type(self, data, ptitem, prefix):
        """Checks if loaded data has the type it was stored in. If not converts it.

        :param data: Data item to be checked and converted
        :param ptitem: HDf5 Node or Leaf from where data was loaded
        :param prefix: Prefix for recalling the data type from the hdf5 node attributes

        :return:

            Tuple, first item is the (converted) `data` item, second boolean whether
            item was converted or not.

        """
        typestr = self._all_get_from_attrs(ptitem, prefix + HDF5StorageService.SCALAR_TYPE)
        colltype = self._all_get_from_attrs(ptitem, prefix + HDF5StorageService.COLL_TYPE)
        type_changed = False

        # Check what the original data type was from the hdf5 node attributes
        if colltype == HDF5StorageService.COLL_SCALAR:
            # Here data item was a scalar

            if isinstance(data, np.ndarray):
                # If we recall a numpy scalar, pytables loads a 1d array :-/
                # So we have to change it to a real scalar value
                data = np.array([data])[0]
                type_changed = True

            if not typestr is None:
                # Check if current type and stored type match
                # if not convert the data
                if typestr != type(data).__name__:

                    if typestr == str.__name__:
                        data = data.decode(self._encoding)
                    else:
                        try:
                            data = pypetconstants.PARAMETERTYPEDICT[typestr](data)
                        except KeyError:
                            # For compatibility with files from older pypet versions
                            data = pypetconstants.COMPATPARAMETERTYPEDICT[typestr](data)

                    type_changed = True

        elif (colltype == HDF5StorageService.COLL_TUPLE or
                        colltype == HDF5StorageService.COLL_LIST):
            # Here data item was originally a tuple or a list

            if type(data) is not list and type is not tuple:
                # If the original type cannot be recalled, first convert it to a list
                type_changed = True
                data = list(data)

            if len(data) > 0:
                first_item = data[0]
                # Check if the type of the first item was conserved
                if not typestr == type(first_item).__name__:

                    if not isinstance(data, list):
                        data = list(data)

                    # If type was not conserved we need to convert all items
                    # in the list or tuple
                    for idx, item in enumerate(data):
                        if typestr == str.__name__:
                            data[idx] = data[idx].decode(self._encoding)
                        else:
                            try:
                                data[idx] = pypetconstants.PARAMETERTYPEDICT[typestr](item)
                            except KeyError:
                                # For compatibility with files from older pypet versions:
                                data[idx] = pypetconstants.COMPATPARAMETERTYPEDICT[typestr](item)
                        type_changed = True

            if colltype == HDF5StorageService.COLL_TUPLE:
                # If it was originally a tuple we need to convert it back to tuple
                if type(data) is not tuple:
                    data = tuple(data)
                    type_changed = True
        elif colltype == HDF5StorageService.COLL_EMPTY_DICT:
            data = {}
            type_changed = True
        elif isinstance(data, np.ndarray):

            if typestr == str.__name__:
                data = np.core.defchararray.decode(data, self._encoding)
                type_changed = True

            if colltype == HDF5StorageService.COLL_MATRIX:
                # Here data item was originally a matrix
                data = np.matrix(data)
                type_changed = True

        return data, type_changed

    @staticmethod
    def _all_kill_iterator(iterator):
        if iterator is not None:
            try:
                while True:
                    next(iterator)
            except StopIteration:
                pass

    def _all_add_or_modify_row(self, item_name, insert_dict, table, index=None, condition=None,
                               condvars=None,
                               flags=(ADD_ROW, MODIFY_ROW,)):
        """Adds or changes a row in a pytable.

        :param item_name: Name of item, the row is about, only important for throwing errors.

        :param insert_dict:

            Dictionary of data that is about to be inserted into the pytables row.

        :param table:

            The table to insert or modify a row in

        :param index:

            Index of row to be modified. Instead of an index a search condition can be
            used as well, see below.

        :param condition:

            Condition to search for in the table

        :param condvars:

            Variables for the search condition

        :param flags:

            Flags whether to add, modify, or remove a row in the table


        """
        if len(flags) == 0:
            # No flags means no-op
            return

        # You can only specify either an index or a condition not both
        if index is not None and condition is not None:
            raise ValueError('Please give either a condition or an index or none!')
        elif condition is not None:
            row_iterator = table.where(condition, condvars=condvars)
        elif index is not None:
            row_iterator = table.iterrows(index, index + 1)
        else:
            row_iterator = None

        try:
            row = next(row_iterator)
        except TypeError:
            row = None
        except StopIteration:
            row = None

        # multiple_entries = []

        if ((HDF5StorageService.MODIFY_ROW in flags or HDF5StorageService.ADD_ROW in flags) and
                    HDF5StorageService.REMOVE_ROW in flags):
            # You cannot remove and modify or add at the same time
            raise ValueError('You cannot add or modify and remove a row at the same time.')

        if row is None and HDF5StorageService.ADD_ROW in flags:
            # Here we add a new row

            row = table.row

            self._all_insert_into_row(row, insert_dict)

            row.append()

        elif row is not None and HDF5StorageService.MODIFY_ROW in flags:
            # Here we modify an existing row

            self._all_insert_into_row(row, insert_dict)

            row.update()

        elif HDF5StorageService.REMOVE_ROW in flags:
            # Here we delete an existing row

            if row is not None:
                # Only delete if the row does exist otherwise we do not have to do anything
                row_number = row.nrow
                try:
                    table.remove_rows(start=row_number, stop=row_number+1)
                except NotImplementedError:
                    pass
                    # We get here if we try to remove the last row of a table
                    # there is nothing we can do but keep it :-(
        else:
            raise ValueError('Something is wrong, you might not have found '
                             'a row, or your flags are not set appropriately')

        self._all_kill_iterator(row_iterator)
        table.flush()

        if HDF5StorageService.REMOVE_ROW not in flags and row is None:
            raise RuntimeError('Could not add or modify entries of `%s` in '
                               'table %s' % (item_name, table._v_name))

    def _all_insert_into_row(self, row, insert_dict):
        """Copies data from `insert_dict` into a pytables `row`."""
        for key, val in insert_dict.items():
            try:
                row[key] = val
            except KeyError as ke:
                self._logger.warning('Could not write `%s` into a table, ' % key + repr(ke))

    def _all_extract_insert_dict(self, item, colnames, additional_info=None):
        """Extracts information from a given item to be stored into a pytable row.

        Items can be a variety of things here, trajectories, single runs, group node,
        parameters, results.

        :param item: Item from which data should be extracted

        :param colnames: Names of the columns in the pytable

        :param additional_info: (dict)

            Additional information that should be stored into the pytable row that cannot be
            read out from `item`.

        :return: Dictionary containing the data to be inserted into a row

        """
        insert_dict = {}

        if 'length' in colnames:
            insert_dict['length'] = len(item)

        if 'comment' in colnames:
            comment = self._all_cut_string(item.v_comment.encode('utf-8'),
                                           pypetconstants.HDF5_STRCOL_MAX_COMMENT_LENGTH,
                                           self._logger)

            insert_dict['comment'] = comment

        if 'location' in colnames:
            insert_dict['location'] = item.v_location.encode('utf-8')

        if 'name' in colnames:
            name = item._name if (not item.v_is_root or not item.v_is_run) else item._crun
            insert_dict['name'] = name.encode('utf-8')

        if 'class_name' in colnames:
            insert_dict['class_name'] = item.f_get_class_name().encode('utf-8')

        if 'value' in colnames:
            insert_dict['value'] = self._all_cut_string(
                item.f_val_to_str().encode('utf-8'),
                pypetconstants.HDF5_STRCOL_MAX_VALUE_LENGTH,
                self._logger)

        if 'hexdigest' in colnames:
            insert_dict['hexdigest'] = additional_info['hexdigest']

        if 'idx' in colnames:
            insert_dict['idx'] = item.v_idx

        if 'time' in colnames:
            time_ = item._time
            insert_dict['time'] = time_.encode('utf-8')

        if 'timestamp' in colnames:
            timestamp = item._timestamp
            insert_dict['timestamp'] = timestamp

        if 'range' in colnames:
            third_length = pypetconstants.HDF5_STRCOL_MAX_RANGE_LENGTH // 3 + 10
            item_range = itools.islice(item.f_get_range(copy=False), 0, third_length)
            range_string = ', '.join([repr(x) for x in item_range])
            insert_dict['range'] = self._all_cut_string(
                range_string.encode('utf-8'),
                pypetconstants.HDF5_STRCOL_MAX_RANGE_LENGTH,
                self._logger)

        # To allow backwards compatibility
        if 'array' in colnames:
            third_length = pypetconstants.HDF5_STRCOL_MAX_RANGE_LENGTH // 3 + 10
            item_range = itools.islice(item.f_get_range(copy=False), 0, third_length)
            range_string = ', '.join([repr(x) for x in item_range])
            insert_dict['array'] = self._all_cut_string(
                range_string.encode('utf-8'),
                pypetconstants.HDF5_STRCOL_MAX_RANGE_LENGTH,
                self._logger)

        if 'version' in colnames:
            insert_dict['version'] = item.v_version.encode('utf-8')

        if 'python' in colnames:
            insert_dict['python'] = item.v_python.encode('utf-8')

        if 'finish_timestamp' in colnames:
            insert_dict['finish_timestamp'] = item._finish_timestamp_run

        return insert_dict

    @staticmethod
    def _all_cut_string(string, max_length, logger):
        """Cuts string data to the maximum length allowed in a pytables column
        if string is too long.

        :param string: String to be cut
        :param max_length: Maximum allowed string length
        :param logger: Logger where messages about truncating should be written

        :return: String, cut if too long

        """
        if len(string) > max_length:
            logger.debug('The string `%s` was too long I truncated it to'
                         ' %d characters' %
                         (string, max_length))
            string = string[0:max_length - 3] + '...'.encode('utf-8')

        return string

    def _all_create_or_get_group(self, name, parent_hdf5_group=None):
        """Creates or returns a group"""
        if not name in parent_hdf5_group:
            new_hdf5_group = self._hdf5file.create_group(where=parent_hdf5_group,
                                                      name=name,
                                                      title=name,
                                                      filters=self._all_get_filters())
            return new_hdf5_group, True
        else:
            new_hdf5_group = parent_hdf5_group._f_get_child(name)
            return new_hdf5_group, False

    def _all_create_or_get_groups(self, key, start_hdf5_group=None):
        """Creates new or follows existing group nodes along a given colon separated `key`.

        :param key:

            Colon separated path along hdf5 file, e.g. `parameters.mobiles.cars`.

        :param start_hdf5_group:

            HDF5 group from where to start, leave `None` for the trajectory group.

        :return:

            Final group node, e.g. group node with name `cars`.

        """
        if start_hdf5_group is None:
            newhdf5_group = self._trajectory_group
        else:
            newhdf5_group = start_hdf5_group
        created = False
        if key == '':
            return newhdf5_group, created
        split_key = key.split('.')
        for name in split_key:
            newhdf5_group, created = self._all_create_or_get_group(name, newhdf5_group)

        return newhdf5_group, created


    ################# Storing and loading Annotations ###########################################

    def _ann_store_annotations(self, item_with_annotations, node, overwrite=False):
        """Stores annotations into an hdf5 file."""

        # If we overwrite delete all annotations first
        if overwrite is True or overwrite == 'v_annotations':
            annotated = self._all_get_from_attrs(node, HDF5StorageService.ANNOTATED)
            if annotated:
                current_attrs = node._v_attrs
                for attr_name in current_attrs._v_attrnames:
                    if attr_name.startswith(HDF5StorageService.ANNOTATION_PREFIX):
                        delattr(current_attrs, attr_name)
                delattr(current_attrs, HDF5StorageService.ANNOTATED)
                self._hdf5file.flush()

        # Only store annotations if the item has some
        if not item_with_annotations.v_annotations.f_is_empty():

            anno_dict = item_with_annotations.v_annotations._dict

            current_attrs = node._v_attrs

            changed = False

            for field_name in anno_dict:
                val = anno_dict[field_name]

                field_name_with_prefix = HDF5StorageService.ANNOTATION_PREFIX + field_name
                if field_name_with_prefix not in current_attrs:
                    # Only store *new* annotations, if they already exist on disk, skip storage
                    setattr(current_attrs, field_name_with_prefix, val)
                    changed = True

            if changed:
                setattr(current_attrs, HDF5StorageService.ANNOTATED, True)
                self._hdf5file.flush()

    def _ann_load_annotations(self, item_with_annotations, node):
        """Loads annotations from disk."""

        annotated = self._all_get_from_attrs(node, HDF5StorageService.ANNOTATED)

        if annotated:

            annotations = item_with_annotations.v_annotations

            # You can only load into non-empty annotations, to prevent overwriting data in RAM
            if not annotations.f_is_empty():
                raise TypeError('Loading into non-empty annotations!')

            current_attrs = node._v_attrs

            for attr_name in current_attrs._v_attrnames:

                if attr_name.startswith(HDF5StorageService.ANNOTATION_PREFIX):
                    key = attr_name
                    key = key.replace(HDF5StorageService.ANNOTATION_PREFIX, '')

                    data = getattr(current_attrs, attr_name)
                    setattr(annotations, key, data)


    ########################## Storing/Loading Groups ################################

    def _grp_store_group(self, traj_group, store_data=pypetconstants.STORE_DATA,
                         with_links=True, recursive=False, max_depth=None,
                         _hdf5_group=None, _newly_created=False):
        """Stores a group node.

        For group nodes only annotations and comments need to be stored.

        """
        if store_data == pypetconstants.STORE_NOTHING:
            return
        elif store_data == pypetconstants.STORE_DATA_SKIPPING and traj_group._stored:
            self._logger.debug('Already found `%s` on disk I will not store it!' %
                                   traj_group.v_full_name)
        elif not recursive:
            if _hdf5_group is None:
                _hdf5_group, _newly_created = self._all_create_or_get_groups(traj_group.v_full_name)

            overwrite = store_data == pypetconstants.OVERWRITE_DATA

            if (traj_group.v_comment != '' and
                    (HDF5StorageService.COMMENT not in _hdf5_group._v_attrs or overwrite)):
                setattr(_hdf5_group._v_attrs, HDF5StorageService.COMMENT, traj_group.v_comment)

            if ((_newly_created or overwrite) and
                type(traj_group) not in (nn.NNGroupNode, nn.ConfigGroup, nn.ParameterGroup,
                                             nn.DerivedParameterGroup, nn.ResultGroup)):
                # We only store the name of the class if it is not one of the standard groups,
                # that are always used.
                setattr(_hdf5_group._v_attrs, HDF5StorageService.CLASS_NAME,
                        traj_group.f_get_class_name())

            self._ann_store_annotations(traj_group, _hdf5_group, overwrite=overwrite)
            self._hdf5file.flush()
            traj_group._stored = True

            # Signal completed node loading
            self._node_processing_timer.signal_update()

        if recursive:
            parent_traj_group = traj_group.f_get_parent()
            parent_hdf5_group = self._all_create_or_get_groups(parent_traj_group.v_full_name)[0]

            self._tree_store_nodes_dfs(parent_traj_group, traj_group.v_name, store_data=store_data,
                                       with_links=with_links, recursive=recursive,
                                       max_depth=max_depth, current_depth=0,
                                       parent_hdf5_group=parent_hdf5_group)

    def _grp_load_group(self, traj_group, load_data=pypetconstants.LOAD_DATA, with_links=True,
                        recursive=False, max_depth=None,
                        _traj=None, _as_new=False, _hdf5_group=None):
        """Loads a group node and potentially everything recursively below"""
        if _hdf5_group is None:
            _hdf5_group = self._all_get_node_by_name(traj_group.v_full_name)
            _traj = traj_group.v_root

        if recursive:
            parent_traj_node = traj_group.f_get_parent()
            self._tree_load_nodes_dfs(parent_traj_node, load_data=load_data, with_links=with_links,
                                  recursive=recursive, max_depth=max_depth,
                                  current_depth=0,
                                  trajectory=_traj, as_new=_as_new,
                                  hdf5_group=_hdf5_group)
        else:
            if load_data == pypetconstants.LOAD_NOTHING:
                return

            elif load_data == pypetconstants.OVERWRITE_DATA:
                traj_group.v_annotations.f_empty()
                traj_group.v_comment = ''

            self._all_load_skeleton(traj_group, _hdf5_group)
            traj_group._stored = not _as_new

            # Signal completed node loading
            self._node_processing_timer.signal_update()

    def _all_load_skeleton(self, traj_node, hdf5_group):
        """Reloads skeleton data of a tree node"""
        if traj_node.v_annotations.f_is_empty():
            self._ann_load_annotations(traj_node, hdf5_group)
        if traj_node.v_comment == '':
            comment = self._all_get_from_attrs(hdf5_group, HDF5StorageService.COMMENT)
            if comment is None:
                comment = ''
            traj_node.v_comment = comment

    ################# Storing and Loading Parameters ############################################

    @staticmethod
    def _prm_extract_missing_flags(data_dict, flags_dict):
        """Extracts storage flags for data in `data_dict`
        if they were not specified in `flags_dict`.

        See :const:`~pypet.storageservice.HDF5StorageService.TYPE_FLAG_MAPPING`
        for how to store different types of data per default.

        """
        for key, data in data_dict.items():
            if not key in flags_dict:
                dtype = type(data)
                if (dtype is np.ndarray or dtype is dict) and len(data) == 0:
                        # Empty containers are stored as an Array
                        # No need to ask for tuple or list, because they are always
                        # stored as arrays.
                        flags_dict[key] = HDF5StorageService.ARRAY
                        continue
                else:
                    try:
                        flags_dict[key] = HDF5StorageService.TYPE_FLAG_MAPPING[dtype]
                    except KeyError:
                        raise pex.NoSuchServiceError('I cannot store `%s`, I do not understand the'
                                                     'type `%s`.' % (key, str(dtype)))

    def _prm_meta_add_summary(self, instance):
        """Adds data to the summary tables and returns if `instance`s comment has to be stored.

        Also moves comments upwards in the hierarchy if purge_duplicate_comments is true
        and a lower index run has completed. Only necessary for *multiprocessing*.

        :return: Tuple

            * String specifying the subtree

            * Boolean whether to store the comment to `instance`s hdf5 node

        """
        if instance.v_comment == '':
            return False

        where = instance.v_branch
        definitely_store_comment = True

        # Get the hexdigest of the comment to see if such a comment has been stored before
        bytes_comment = instance.v_comment.encode('utf-8')
        hexdigest = hashlib.sha1(bytes_comment).hexdigest()
        hexdigest = hexdigest.encode('utf-8')

        # Get the overview table
        table_name = where + '_summary'

        # Check if the overview table exists, otherwise skip the rest of
        # the meta adding
        if table_name in self._overview_group:
            table = getattr(self._overview_group, table_name)
        else:
            return definitely_store_comment


        try:
            condvars = {'hexdigestcol': table.cols.hexdigest,
                'hexdigest': hexdigest}

            condition = """(hexdigestcol == hexdigest)"""

            row_iterator = table.where(condition, condvars=condvars)

            row = None
            try:
                row = next(row_iterator)
            except StopIteration:
                pass

            if row is None:
                self._all_store_param_or_result_table_entry(instance, table,
                                                            flags=(
                                                                HDF5StorageService.ADD_ROW,),
                                                            additional_info={
                                                                'hexdigest': hexdigest})

                definitely_store_comment = True
            else:
                definitely_store_comment = False
                self._all_kill_iterator(row_iterator)

        except pt.NoSuchNodeError:
            definitely_store_comment = True

        return definitely_store_comment

    def _prm_add_meta_info(self, instance, group, overwrite=False):
        """Adds information to overview tables and meta information to
        the `instance`s hdf5 `group`.

        :param instance: Instance to store meta info about
        :param group: HDF5 group of instance
        :param overwrite: If data should be explicitly overwritten

        """

        if overwrite:
            flags = ()
        else:
            flags = (HDF5StorageService.ADD_ROW,)

        definitely_store_comment = True
        try:
            # Check if we need to store the comment. Maybe update the overview tables
            # accordingly if the current run index is lower than the one in the table.
            definitely_store_comment = self._prm_meta_add_summary(instance)

            try:
                # Update the summary overview table
                table_name = instance.v_branch + '_overview'

                table = getattr(self._overview_group, table_name)
                if len(table) < pypetconstants.HDF5_MAX_OVERVIEW_TABLE_LENGTH:

                    self._all_store_param_or_result_table_entry(instance, table,
                                                                flags=flags)
            except pt.NoSuchNodeError:
                pass
        except Exception as exc:
            self._logger.error('Could not store information table due to `%s`.' % repr(exc))

        if ((not self._purge_duplicate_comments or definitely_store_comment) and
                    instance.v_comment != ''):
            # Only add the comment if necessary
            setattr(group._v_attrs, HDF5StorageService.COMMENT, instance.v_comment)

        # Add class name and whether node is a leaf to the HDF5 attributes
        setattr(group._v_attrs, HDF5StorageService.CLASS_NAME, instance.f_get_class_name())
        setattr(group._v_attrs, HDF5StorageService.LEAF, True)

        if instance.v_is_parameter and instance.v_explored:
            # If the stored parameter was an explored one we need to mark this in the
            # explored overview table
            try:
                tablename = 'explored_parameters_overview'
                table = getattr(self._overview_group, tablename)

                if len(table) < pypetconstants.HDF5_MAX_OVERVIEW_TABLE_LENGTH:
                    self._all_store_param_or_result_table_entry(instance, table,
                                                                flags=flags)
            except pt.NoSuchNodeError:
                pass
            except Exception as exc:
                self._logger.error('Could not store information '
                                   'table due to `%s`.' % repr(exc))

    def _prm_store_from_dict(self, fullname, store_dict, hdf5_group, store_flags, kwargs):
        """Stores a `store_dict`"""
        for key, data_to_store in store_dict.items():
            # self._logger.log(1, 'SUB-Storing %s [%s]', key, str(store_dict[key]))
            original_hdf5_group = None

            flag = store_flags[key]

            if '.' in key:
                original_hdf5_group = hdf5_group
                split_key = key.split('.')
                key = split_key.pop()
                for inner_key in split_key:
                    hdf5_group, newly_created =  self._all_create_or_get_group(inner_key,
                                                                                hdf5_group)
                    if newly_created:
                        setattr(hdf5_group._v_attrs, HDF5StorageService.STORAGE_TYPE,
                                HDF5StorageService.NESTED_GROUP)
                    else:
                        store_type = self._all_get_from_attrs(hdf5_group, HDF5StorageService.STORAGE_TYPE)
                        if store_type != HDF5StorageService.NESTED_GROUP:
                            raise ValueError('You want to nested results but `%s` is already '
                                             'of type `%s`!' % (hdf5_group._v_name, store_type))

            # Iterate through the data and store according to the storage flags
            if key in hdf5_group:
                # We won't change any data that is found on disk
                self._logger.debug(
                    'Found %s already in hdf5 node of %s, so I will ignore it.' %
                    (key, fullname))
                continue

            if flag == HDF5StorageService.TABLE:
                # self._logger.log(1, 'SUB-Storing %s TABLE', key)
                self._prm_write_into_pytable(key, data_to_store, hdf5_group, fullname,
                                             **kwargs)
            elif flag == HDF5StorageService.DICT:
                # self._logger.log(1, 'SUB-Storing %s DICT', key)
                self._prm_write_dict_as_table(key, data_to_store, hdf5_group, fullname,
                                              **kwargs)
            elif flag == HDF5StorageService.ARRAY:
                # self._logger.log(1, 'SUB-Storing %s ARRAY', key)
                self._prm_write_into_array(key, data_to_store, hdf5_group, fullname,
                                           **kwargs)
            elif flag in (HDF5StorageService.CARRAY,
                          HDF5StorageService.EARRAY,
                          HDF5StorageService.VLARRAY):
                self._prm_write_into_other_array(key, data_to_store,
                                                 hdf5_group, fullname,
                                                 flag=flag, **kwargs)
            elif flag in (HDF5StorageService.SERIES,
                          HDF5StorageService.FRAME,
                          #  HDF5StorageService.PANEL
                          ):
                # self._logger.log(1, 'SUB-Storing %s PANDAS', key)
                self._prm_write_pandas_data(key, data_to_store, hdf5_group, fullname,
                                            flag, **kwargs)
            elif flag == HDF5StorageService.SHARED_DATA:
                pass  # Shared data needs to be explicitly created and is not stored on
                # the fly
            else:
                raise RuntimeError('You shall not pass!')

            if original_hdf5_group is not None:
                hdf5_group = original_hdf5_group

    def _prm_store_parameter_or_result(self,
                                       instance,
                                       store_data=pypetconstants.STORE_DATA,
                                       store_flags=None,
                                       overwrite=None,
                                       with_links=False,
                                       recursive=False,
                                       _hdf5_group=None,
                                       _newly_created=False,
                                       **kwargs):
        """Stores a parameter or result to hdf5.

        :param instance:

            The instance to be stored

        :param store_data:

            How to store data

        :param store_flags:

            Dictionary containing how to store individual data, usually empty.

        :param overwrite:

            Instructions how to overwrite data

        :param with_links:

            Placeholder because leaves have no links

        :param recursive:

            Placeholder, because leaves have no children

        :param _hdf5_group:

            The hdf5 group for storing the parameter or result

        :param _newly_created:

            If should be created in a new form

        """
        if store_data == pypetconstants.STORE_NOTHING:
            return
        elif store_data == pypetconstants.STORE_DATA_SKIPPING and instance._stored:
            self._logger.debug('Already found `%s` on disk I will not store it!' %
                                   instance.v_full_name)
            return
        elif store_data == pypetconstants.OVERWRITE_DATA:
            if not overwrite:
                overwrite = True

        fullname = instance.v_full_name
        self._logger.debug('Storing `%s`.' % fullname)

        if _hdf5_group is None:
            # If no group is provided we might need to create one
            _hdf5_group, _newly_created = self._all_create_or_get_groups(fullname)

        # kwargs_flags = {} # Dictionary to change settings
        # old_kwargs = {}
        store_dict = {}
        # If the user did not supply storage flags, we need to set it to the empty dictionary
        if store_flags is None:
            store_flags = {}

        try:
            # Get the data to store from the instance
            if not instance.f_is_empty():
                store_dict = instance._store()
            try:
                # Ask the instance for storage flags
                instance_flags = instance._store_flags().copy() # copy to avoid modifying the
                # original data
            except AttributeError:
                # If it does not provide any, set it to the empty dictionary
                instance_flags = {}

            # User specified flags have priority over the flags from the instance
            instance_flags.update(store_flags)
            store_flags = instance_flags

            # If we still have data in `store_dict` about which we do not know how to store
            # it, pick default storage flags
            self._prm_extract_missing_flags(store_dict, store_flags)

            if overwrite:
                if isinstance(overwrite, str):
                    overwrite = [overwrite]

                if overwrite is True:
                    to_delete = [key for key in store_dict.keys() if key in _hdf5_group]
                    self._all_delete_parameter_or_result_or_group(instance,
                                                                  delete_only=to_delete,
                                                                  _hdf5_group=_hdf5_group)
                elif isinstance(overwrite, (list, tuple)):
                    overwrite_set = set(overwrite)
                    key_set = set(store_dict.keys())

                    stuff_not_to_be_overwritten = overwrite_set - key_set

                    if overwrite!='v_annotations' and len(stuff_not_to_be_overwritten) > 0:
                        self._logger.warning('Cannot overwrite `%s`, these items are not supposed to '
                                             'be stored by the leaf node.' %
                                             str(stuff_not_to_be_overwritten))

                    stuff_to_overwrite = overwrite_set & key_set
                    if len(stuff_to_overwrite) > 0:
                        self._all_delete_parameter_or_result_or_group(instance,
                                                                      delete_only=list(
                                                                          stuff_to_overwrite))
                else:
                    raise ValueError('Your value of overwrite `%s` is not understood. '
                                     'Please pass `True` of a list of strings to fine grain '
                                     'overwriting.' % str(overwrite))

            self._prm_store_from_dict(fullname, store_dict, _hdf5_group, store_flags, kwargs)

            # Store annotations
            self._ann_store_annotations(instance, _hdf5_group, overwrite=overwrite)

            if _newly_created or overwrite is True:
                # If we created a new group or the parameter was extended we need to
                # update the meta information and summary tables
                self._prm_add_meta_info(instance, _hdf5_group,
                                        overwrite=not _newly_created)

            instance._stored = True

            #self._logger.debug('Finished Storing `%s`.' % fullname)
            # Signal completed node loading
            self._node_processing_timer.signal_update()

        except:
            # I anything fails, we want to remove the data of the parameter again
            self._logger.error(
                'Failed storing leaf `%s`. I will remove the hdf5 data I added  again.' % fullname)
            # Delete data
            for key in store_dict.keys():
                if key in  _hdf5_group:
                    hdf5_child = _hdf5_group._f_get_child(key)
                    hdf5_child._f_remove(recursive=True)
            # If no data left delete the whole parameter
            if _hdf5_group._v_nchildren == 0:
                _hdf5_group._f_remove(recursive=True)
            raise

    def _shared_write_shared_data(self, key, hdf5_group, full_name, **kwargs):
        try:
            data = kwargs.pop('obj', None)
            flag = kwargs.pop('flag', None)

            if flag is None and data is None:
                raise RuntimeError('You must provide at least data or a flag')

            if flag is None:
                flags_dict={}
                self._prm_extract_missing_flags({key: data}, flags_dict)
                flag = flags_dict[key]

            if flag == HDF5StorageService.TABLE:
                self._prm_write_shared_table(key, hdf5_group,
                                           full_name, **kwargs)

            elif flag in (HDF5StorageService.FRAME,
                          HDF5StorageService.SERIES,
                          #  HDF5StorageService.PANEL
                          ) :

                self._prm_write_pandas_data(key, data,
                                            hdf5_group,
                                            full_name, flag, **kwargs)
            else:
                self._prm_write_shared_array(key, data,
                                             hdf5_group, full_name, flag, **kwargs)

            hdf5data = hdf5_group._f_get_child(key)
            setattr(hdf5data._v_attrs, HDF5StorageService.STORAGE_TYPE,
                HDF5StorageService.SHARED_DATA)
            setattr(hdf5data._v_attrs, HDF5StorageService.SHARED_DATA_TYPE, flag)
        except:
            self._logger.error('Failed storing shared data `%s` of `%s`.' % (key, full_name))
            raise

    def _prm_select_shared_pandas_data(self, pd_node, full_name, **kwargs):
        """Reads a DataFrame from dis.

        :param pd_node:

            hdf5 node storing the pandas DataFrame

        :param full_name:

            Full name of the parameter or result whose data is to be loaded

        :param kwargs:

            Arguments passed to pandas' select method

        """
        try:
            pathname = pd_node._v_pathname
            pandas_store = self._hdf5store
            return pandas_store.select(pathname, **kwargs)
        except:
            self._logger.error('Failed loading `%s` of `%s`.' % (pd_node._v_name, full_name))
            raise

    def _prm_write_shared_array(self, key, data, hdf5_group, full_name, flag, **kwargs):
        """Creates and array that can be used with an HDF5 array object"""

        if flag == HDF5StorageService.ARRAY:
            self._prm_write_into_array(key, data, hdf5_group, full_name, **kwargs)
        elif flag in (HDF5StorageService.CARRAY,
                    HDF5StorageService.EARRAY,
                    HDF5StorageService.VLARRAY):
            self._prm_write_into_other_array(key, data, hdf5_group, full_name,
                                             flag=flag, **kwargs)
        else:
            raise RuntimeError('Flag `%s` of hdf5 data `%s` of `%s` not understood' %
                               (flag, key, full_name))

        self._hdf5file.flush()

    def _prm_write_shared_table(self, key, hdf5_group, fullname, **kwargs):
        """Creates a new empty table"""
        first_row = None
        description = None
        if 'first_row' in kwargs:
            first_row = kwargs.pop('first_row')
            if not 'description' in kwargs:
                description = {}
                for colname in first_row:
                    data = first_row[colname]
                    column = self._all_get_table_col(key, [data], fullname)
                    description[colname] = column

        if 'description' in kwargs:
            description = kwargs.pop('description')

        if 'filters' in kwargs:
            filters = kwargs.pop('filters')
        else:
            filters = self._all_get_filters(kwargs)

        table = self._hdf5file.create_table(where=hdf5_group, name=key,
                                          description=description,
                                          filters=filters,
                                          **kwargs)
        table.flush()

        if first_row is not None:
            row = table.row
            for key in description:
                row[key] = first_row[key]

            row.append()
            table.flush()

    def _prm_write_dict_as_table(self, key, data_to_store, group, fullname, **kwargs):
        """Stores a python dictionary as pytable

        :param key:

            Name of data item to store

        :param data_to_store:

            Dictionary to store

        :param group:

            Group node where to store data in hdf5 file

        :param fullname:

            Full name of the `data_to_store`s original container, only needed for throwing errors.

        """
        if key in group:
            raise ValueError(
                'Dictionary `%s` already exists in `%s`. Appending is not supported (yet).')

        if key in group:
            raise ValueError('Dict `%s` already exists in `%s`. Appending is not supported (yet).')

        temp_dict = {}
        for innerkey in data_to_store:
            val = data_to_store[innerkey]
            temp_dict[innerkey] = [val]

        # Convert dictionary to object table
        objtable = ObjectTable(data=temp_dict)

        # Then store the object table
        self._prm_write_into_pytable(key, objtable, group, fullname, **kwargs)

        new_table = group._f_get_child(key)

        # Remember that the Object Table represents a dictionary
        self._all_set_attributes_to_recall_natives(temp_dict, new_table,
                                                   HDF5StorageService.DATA_PREFIX)

        setattr(new_table._v_attrs, HDF5StorageService.STORAGE_TYPE,
                HDF5StorageService.DICT)

        self._hdf5file.flush()

    def _prm_write_pandas_data(self, key, data, group, fullname, flag, **kwargs):
        """Stores a pandas DataFrame into hdf5.

        :param key:

            Name of data item to store

        :param data:

            Pandas Data to Store

        :param group:

            Group node where to store data in hdf5 file

        :param fullname:

            Full name of the `data_to_store`s original container, only needed for throwing errors.

        :param flag:

            If it is a series, frame or panel

        """
        try:
            if 'filters' not in kwargs:
                self._logger.debug(
                    'filters are no longer supported by pandas')
            if 'format' not in kwargs:
                kwargs['format'] = self.pandas_format
            if 'encoding' not in kwargs:
                kwargs['encoding'] = self.encoding

            overwrite = kwargs.pop('overwrite', False)

            if key in group and not (overwrite or kwargs.get('append', False)):
                raise ValueError(
                    'DataFrame `%s` already exists in `%s`. '
                    'To append pass ``append=`True```.' % (key, fullname))
            else:
                self._logger.debug('Appending to pandas data `%s` in `%s`' % (key, fullname))

            if 'expectedrows' in kwargs:
                self._logger.debug('expectedrows no longer supported by pandas, will '
                                   'ignore the option')

            name = group._v_pathname + '/' + key
            self._hdf5store.put(name, data, **kwargs)
            self._hdf5store.flush()
            self._hdf5file.flush()

            frame_group = group._f_get_child(key)
            setattr(frame_group._v_attrs, HDF5StorageService.STORAGE_TYPE, flag)
            self._hdf5file.flush()

        except:
            self._logger.error('Failed storing pandas data `%s` of `%s`.' % (key, fullname))
            raise

    def _prm_write_into_other_array(self, key, data, group, fullname,
                                    flag, **kwargs):
        """Stores data as carray, earray or vlarray depending on `flag`.

        :param key:

            Name of data item to store

        :param data:

            Data to store

        :param group:

            Group node where to store data in hdf5 file

        :param fullname:

            Full name of the `data_to_store`s original container, only needed for throwing errors.

        :param recall:

            If container type and data type for perfect recall should be stored

        :param flag:

            How to store:
                CARRAY, EARRAY, VLARRAY

        """
        try:

            if flag == HDF5StorageService.CARRAY:
                factory = self._hdf5file.create_carray
            elif flag == HDF5StorageService.EARRAY:
                factory = self._hdf5file.create_earray
            elif flag == HDF5StorageService.VLARRAY:
                factory = self._hdf5file.create_vlarray
            else:
                raise RuntimeError('You shall not pass!')

            if key in group:
                raise ValueError(
                    'CArray `%s` already exists in `%s`. Appending is not supported (yet).')

            if 'filters' in kwargs:
                filters = kwargs.pop('filters')
            else:
                filters = self._all_get_filters(kwargs)

            try:
                other_array = factory(where=group, name=key, obj=data,
                                                filters=filters, **kwargs)
            except (ValueError, TypeError) as exc:
                try:
                    conv_data = data[:]
                    conv_data = np.core.defchararray.encode(conv_data, self.encoding)
                    other_array = factory(where=group, name=key,
                                                obj=conv_data,
                                                filters=filters, **kwargs)
                except Exception:
                    # Re-raise original Error
                    raise exc

            if data is not None:
                # Remember the types of the original data to recall them on loading
                self._all_set_attributes_to_recall_natives(data, other_array,
                                                       HDF5StorageService.DATA_PREFIX)
            setattr(other_array._v_attrs, HDF5StorageService.STORAGE_TYPE, flag)
            self._hdf5file.flush()
        except:
            self._logger.error('Failed storing %s `%s` of `%s`.' % (flag, key, fullname))
            raise

    def _prm_write_into_array(self, key, data, group, fullname, **kwargs):
        """Stores data as array.

        :param key:

            Name of data item to store

        :param data:

            Data to store

        :param group:

            Group node where to store data in hdf5 file

        :param fullname:

            Full name of the `data_to_store`s original container, only needed for throwing errors.

        :param recall:

            If container type and data type for perfect recall should be stored

        """

        try:
            if key in group:
                raise ValueError(
                    'Array `%s` already exists in `%s`. Appending is not supported (yet).')

            try:

                array = self._hdf5file.create_array(where=group,
                                              name=key, obj=data, **kwargs)
            except (TypeError, ValueError) as exc:
                try:
                    if type(data) is dict and len(data) == 0:
                        # We cannot store an empty dictionary,
                        # but we can use an empty tuple as a dummy.
                        conv_data = ()
                    elif isinstance(data, str):
                        conv_data = data.encode(self._encoding)
                    elif isinstance(data, int):
                        conv_data = np.int64(data)
                    else:
                        conv_data = []
                        for string in data:
                            conv_data.append(string.encode(self._encoding))
                    array = self._hdf5file.create_array(where=group,
                                                  name=key, obj=conv_data, **kwargs)
                except Exception:
                    # Re-raise original error
                    raise exc

            if data is not None:
                # Remember the types of the original data to recall them on loading
                self._all_set_attributes_to_recall_natives(data, array,
                                                           HDF5StorageService.DATA_PREFIX)
            setattr(array._v_attrs, HDF5StorageService.STORAGE_TYPE,
                    HDF5StorageService.ARRAY)
            self._hdf5file.flush()
        except:
            self._logger.error('Failed storing array `%s` of `%s`.' % (key, fullname))
            raise

    def _lnk_delete_link(self, link_name):
        """Removes a link from disk"""
        translated_name = '/' + self._trajectory_name + '/' + link_name.replace('.','/')
        link = self._hdf5file.get_node(where=translated_name)
        link._f_remove()

    def _all_delete_parameter_or_result_or_group(self, instance,
                                                 delete_only=None,
                                                 remove_from_item=False,
                                                 recursive=False,
                                                 _hdf5_group=None):
        """Removes a parameter or result or group from the hdf5 file.

        :param instance: Instance to be removed

        :param delete_only:

            List of elements if you only want to delete parts of a leaf node. Note that this
            needs to list the names of the hdf5 subnodes. BE CAREFUL if you erase parts of a leaf.
            Erasing partly happens at your own risk, it might be the case that you can
            no longer reconstruct the leaf from the leftovers!

        :param remove_from_item:

            If using `delete_only` and `remove_from_item=True` after deletion the data item is
            also removed from the `instance`.

        :param recursive:

            If a group node has children, you will can delete it if recursive is True.


        """
        split_name = instance.v_location.split('.')
        if _hdf5_group is None:
            where = '/' + self._trajectory_name + '/' + '/'.join(split_name)
            node_name = instance.v_name
            _hdf5_group = self._hdf5file.get_node(where=where, name=node_name)

        if delete_only is None:
            if instance.v_is_group and not recursive and len(_hdf5_group._v_children) != 0:
                    raise TypeError('You cannot remove the group `%s`, it has children, please '
                                    'use `recursive=True` to enforce removal.' %
                                    instance.v_full_name)
            _hdf5_group._f_remove(recursive=True)
        else:
            if not instance.v_is_leaf:
                raise ValueError('You can only choose `delete_only` mode for leafs.')

            if isinstance(delete_only, str):
                delete_only = [delete_only]

            for delete_item in delete_only:
                if (remove_from_item and
                        hasattr(instance, '__contains__') and
                        hasattr(instance, '__delattr__') and
                            delete_item in instance):
                    delattr(instance, delete_item)
                try:
                    _hdf5_sub_group = self._hdf5file.get_node(where=_hdf5_group,
                                                        name=delete_item)
                    _hdf5_sub_group._f_remove(recursive=True)
                except pt.NoSuchNodeError:
                    self._logger.warning('Could not delete `%s` from `%s`. HDF5 node not found!' %
                                         (delete_item, instance.v_full_name))

    def _prm_write_into_pytable(self, tablename, data, hdf5_group, fullname, **kwargs):
        """Stores data as pytable.

        :param tablename:

            Name of the data table

        :param data:

            Data to store

        :param hdf5_group:

            Group node where to store data in hdf5 file

        :param fullname:

            Full name of the `data_to_store`s original container, only needed for throwing errors.

        """

        datasize = data.shape[0]

        try:

            # Get a new pytables description from the data and create a new table
            description_dict, data_type_dict = self._prm_make_description(data, fullname)
            description_dicts = [{}]

            if len(description_dict) > ptpa.MAX_COLUMNS:
                # For optimization we want to store the original data types into another table
                # and split the tables into several ones
                new_table_group = self._hdf5file.create_group(where=hdf5_group,
                                                    name=tablename,
                                                    filters=self._all_get_filters(kwargs.copy()))

                count = 0
                for innerkey in description_dict:
                    val = description_dict[innerkey]
                    if count == ptpa.MAX_COLUMNS:
                        description_dicts.append({})
                        count = 0
                    description_dicts[-1][innerkey] = val
                    count += 1

                setattr(new_table_group._v_attrs, HDF5StorageService.STORAGE_TYPE,
                        HDF5StorageService.TABLE)
                setattr(new_table_group._v_attrs, HDF5StorageService.SPLIT_TABLE, 1)

                hdf5_group = new_table_group
            else:
                description_dicts = [description_dict]

            for idx, descr_dict in enumerate(description_dicts):

                if idx == 0:
                    tblname = tablename
                else:
                    tblname = tablename + '_%d' % idx

                table = self._hdf5file.create_table(where=hdf5_group, name=tblname,
                                              description=descr_dict,
                                              title=tblname,
                                              expectedrows=datasize,
                                              filters=self._all_get_filters(kwargs.copy()))

                row = table.row
                for n in range(datasize):
                    # Fill the columns with data, note if the parameter was extended nstart!=0

                    for key in descr_dict:
                        row[key] = data[key][n]

                    row.append()

                # Remember the original types of the data for perfect recall
                if idx == 0 and len(description_dict) <= ptpa.MAX_COLUMNS:
                    # We only have a single table and
                    # we can store the original data types as attributes
                    for field_name in data_type_dict:
                        type_description = data_type_dict[field_name]
                        self._all_set_attr(table, field_name, type_description)

                    setattr(table._v_attrs, HDF5StorageService.STORAGE_TYPE,
                            HDF5StorageService.TABLE)

                table.flush()
                self._hdf5file.flush()

            if len(description_dict) > ptpa.MAX_COLUMNS:
                # We have potentially many split tables and the data types are
                # stored into an additional table for performance reasons
                tblname = tablename + '__' + HDF5StorageService.STORAGE_TYPE
                field_names, data_types = list(zip(*data_type_dict.items()))
                data_type_table_dict = {'field_name': field_names, 'data_type': data_types}
                descr_dict, _ = self._prm_make_description(data_type_table_dict, fullname)

                table = self._hdf5file.create_table(where=hdf5_group, name=tblname,
                                              description=descr_dict,
                                              title=tblname,
                                              expectedrows=len(field_names),
                                              filters=self._all_get_filters(kwargs))

                row = table.row

                for n in range(len(field_names)):
                    # Fill the columns with data

                    for key in data_type_table_dict:
                        row[key] = data_type_table_dict[key][n]

                    row.append()

                setattr(table._v_attrs, HDF5StorageService.DATATYPE_TABLE, 1)

                table.flush()
                self._hdf5file.flush()

        except:
            self._logger.error('Failed storing table `%s` of `%s`.' % (tablename, fullname))
            raise

    def _prm_make_description(self, data, fullname):
        """ Returns a description dictionary for pytables table creation"""

        def _convert_lists_and_tuples(series_of_data):
            """Converts lists and tuples to numpy arrays"""
            if isinstance(series_of_data[0],
                          (list, tuple)):  # and not isinstance(series_of_data[0], np.ndarray):
                # If the first data item is a list, the rest must be as well, since
                # data has to be homogeneous
                for idx, item in enumerate(series_of_data):
                    series_of_data[idx] = np.array(item)

        descriptiondict = {}  # dictionary containing the description to build a pytables table
        original_data_type_dict = {}  # dictionary containing the original data types

        for key in data:
            val = data[key]


            # remember the original data types
            self._all_set_attributes_to_recall_natives(val[0], PTItemMock(original_data_type_dict),
                                                       HDF5StorageService.FORMATTED_COLUMN_PREFIX %
                                                       key)

            _convert_lists_and_tuples(val)

            # get a pytables column from the data
            col = self._all_get_table_col(key, val, fullname)

            descriptiondict[key] = col

        return descriptiondict, original_data_type_dict

    def _all_get_table_col(self, key, column, fullname):
        """ Creates a pytables column instance.

        The type of column depends on the type of `column[0]`.
        Note that data in `column` must be homogeneous!

        """
        val = column[0]

        try:

            # # We do not want to loose int_
            if type(val) is int:
                return pt.IntCol()

            if isinstance(val, (str, bytes)):
                itemsize = int(self._prm_get_longest_stringsize(column))
                return pt.StringCol(itemsize)

            if isinstance(val, np.ndarray):
                if (np.issubdtype(val.dtype, str) or
                        np.issubdtype(val.dtype, bytes)):
                    itemsize = int(self._prm_get_longest_stringsize(column))
                    return pt.StringCol(itemsize, shape=val.shape)
                else:
                    return pt.Col.from_dtype(np.dtype((val.dtype, val.shape)))
            else:
                return pt.Col.from_dtype(np.dtype(type(val)))
        except Exception:
            self._logger.error('Failure in storing `%s` of Parameter/Result `%s`.'
                               ' Its type was `%s`.' % (key, fullname, repr(type(val))))
            raise

    @staticmethod
    def _prm_get_longest_stringsize(string_list):
        """ Returns the longest string size for a string entry across data."""
        maxlength = 1

        for stringar in string_list:
            if isinstance(stringar, np.ndarray):
                if stringar.ndim > 0:
                    for string in stringar.ravel():
                        maxlength = max(len(string), maxlength)
                else:
                    maxlength = max(len(stringar.tolist()), maxlength)
            else:
                maxlength = max(len(stringar), maxlength)

        # Make the string Col longer than needed in order to allow later on slightly larger strings
        return int(maxlength * 1.5)

    def _prm_load_into_dict(self, full_name, load_dict, hdf5_group, instance,
                            load_only, load_except, load_flags, _prefix = ''):
        """Loads into dictionary"""
        for node in hdf5_group:

            load_type = self._all_get_from_attrs(node, HDF5StorageService.STORAGE_TYPE)

            if _prefix:
                load_name = '%s.%s' % (_prefix, node._v_name)
            else:
                load_name = node._v_name

            if load_type == HDF5StorageService.NESTED_GROUP:
                self._prm_load_into_dict(full_name=full_name,
                                         load_dict=load_dict,
                                         hdf5_group=node,
                                         instance=instance,
                                         load_only=load_only,
                                         load_except=load_except,
                                         load_flags=load_flags,
                                         _prefix=load_name)
                continue

            if load_only is not None:

                if load_name not in load_only:
                    continue
                else:
                    load_only.remove(load_name)

            elif load_except is not None:
                if load_name in load_except:
                    load_except.remove(load_name)
                    continue

            # Recall from the hdf5 node attributes how the data was stored and reload accordingly
            if load_name in load_flags:
                load_type = load_flags[load_name]

            if load_type == HDF5StorageService.DICT:
                to_load = self._prm_read_dictionary(node, full_name)
            elif load_type == HDF5StorageService.TABLE:
                to_load = self._prm_read_table(node, full_name)
            elif load_type in (HDF5StorageService.ARRAY, HDF5StorageService.CARRAY,
                                HDF5StorageService.EARRAY, HDF5StorageService.VLARRAY):
                to_load = self._prm_read_array(node, full_name)
            elif load_type in (HDF5StorageService.FRAME,
                               HDF5StorageService.SERIES,
                               #  HDF5StorageService.PANEL
                               ):
                to_load = self._prm_read_pandas(node, full_name)
            elif load_type.startswith(HDF5StorageService.SHARED_DATA):
                to_load = self._prm_read_shared_data(node, instance)
            else:
                raise pex.NoSuchServiceError('Cannot load %s, do not understand the hdf5 file '
                                             'structure of %s [%s].' %
                                             (full_name, str(node), str(load_type)))

            if to_load is None:
                raise RuntimeError('You shall not pass!')

            load_dict[load_name] = to_load

    def _prm_load_parameter_or_result(self, instance,
                                      load_data=pypetconstants.LOAD_DATA,
                                      load_only=None,
                                      load_except=None,
                                      load_flags=None,
                                      with_links=False,
                                      recursive=False,
                                      max_depth=None,
                                      _hdf5_group=None,):
        """Loads a parameter or result from disk.

        :param instance:

            Empty parameter or result instance

        :param load_data:

            How to load stuff

        :param load_only:

            List of data keys if only parts of a result should be loaded

        :param load_except:

            List of data key that should NOT be loaded.

        :param load_flags:

            Dictionary to determine how something is loaded

        :param with_links:

            Placeholder, because leaves have no links

        :param recursive:

            Dummy variable, no-op because leaves have no children

        :param max_depth:

            Dummy variable, no-op because leaves have no children

        :param _hdf5_group:

            The corresponding hdf5 group of the instance

        """
        if load_data == pypetconstants.LOAD_NOTHING:
            return

        if _hdf5_group is None:
            _hdf5_group = self._all_get_node_by_name(instance.v_full_name)

        if load_data == pypetconstants.OVERWRITE_DATA:
            if instance.v_is_parameter and instance.v_locked:
                self._logger.debug('Parameter `%s` is locked, I will skip loading.' %
                                     instance.v_full_name)
                return
            instance.f_empty()
            instance.v_annotations.f_empty()
            instance.v_comment = ''

        self._all_load_skeleton(instance, _hdf5_group)
        instance._stored = True

        # If load only is just a name and not a list of names, turn it into a 1 element list
        if isinstance(load_only, str):
            load_only = [load_only]
        if isinstance(load_except, str):
            load_except = [load_except]

        if load_data == pypetconstants.LOAD_SKELETON:
            # We only load skeleton if asked for it and thus only
            # signal completed node loading
            self._node_processing_timer.signal_update()
            return
        elif load_only is not None:
            if load_except is not None:
                raise ValueError('Please use either `load_only` or `load_except` and not '
                             'both at the same time.')
            elif instance.v_is_parameter and instance.v_locked:
                raise pex.ParameterLockedException('Parameter `%s` is locked, '
                                                   'I will skip loading.' %
                                                    instance.v_full_name)
            self._logger.debug('I am in load only mode, I will only load %s.' %
                               str(load_only))
            load_only = set(load_only)
        elif load_except is not None:
            if instance.v_is_parameter and instance.v_locked:
                raise pex.ParameterLockedException('Parameter `%s` is locked, '
                                                   'I will skip loading.' %
                                                    instance.v_full_name)
            self._logger.debug('I am in load except mode, I will load everything except %s.' %
                               str(load_except))
            # We do not want to modify the original list
            load_except = set(load_except)
        elif not instance.f_is_empty():
            # We only load data if the instance is empty or we specified load_only or
            # load_except and thus only
            # signal completed node loading
            self._node_processing_timer.signal_update()
            return

        full_name = instance.v_full_name
        self._logger.debug('Loading data of %s' % full_name)

        load_dict = {}  # Dict that will be used to keep all data for loading the parameter or
        # result

        if load_flags is None:
            load_flags = {}
        try:
            # Ask the instance for load flags
            instance_flags = instance._load_flags().copy() # copy to avoid modifying the
            # original data
        except AttributeError:
            # If it does not provide any, set it to the empty dictionary
            instance_flags = {}
        # User specified flags have priority over the flags from the instance
        instance_flags.update(load_flags)
        load_flags = instance_flags

        self._prm_load_into_dict(full_name=full_name,
                                 load_dict=load_dict,
                                 hdf5_group=_hdf5_group,
                                 instance=instance,
                                 load_only=load_only,
                                 load_except=load_except,
                                 load_flags=load_flags)

        if load_only is not None:
            # Check if all data in `load_only` was actually found in the hdf5 file
            if len(load_only) > 0:
                self._logger.warning('You marked %s for load only, '
                                     'but I cannot find these for `%s`' %
                                     (str(load_only), full_name))
        elif load_except is not None:
            if len(load_except) > 0:
                self._logger.warning(('You marked `%s` for not loading, but these were not part '
                                      'of `%s` anyway.' % (str(load_except), full_name)))

        # Finally tell the parameter or result to load the data, if there was any ;-)
        if load_dict:
            try:
                instance._load(load_dict)
                if instance.v_is_parameter:
                    # Lock parameter as soon as data is loaded
                    instance.f_lock()
            except:
                self._logger.error(
                    'Error while reconstructing data of leaf `%s`.' % full_name)
                raise

        # Signal completed node loading
        self._node_processing_timer.signal_update()

    def _prm_read_dictionary(self, leaf, full_name):
        """Loads data that was originally a dictionary when stored

        :param leaf:

            PyTables table containing the dictionary data

        :param full_name:

            Full name of the parameter or result whose data is to be loaded

        :return:

            Data to be loaded

        """
        try:
            # Load as Pbject Table
            temp_table = self._prm_read_table(leaf, full_name)
            # Turn the ObjectTable into a dictionary of lists (with length 1).
            temp_dict = temp_table.to_dict('list')

            innder_dict = {}

            # Turn the dictionary of lists into a normal dictionary
            for innerkey, vallist in temp_dict.items():
                innder_dict[innerkey] = vallist[0]

            return innder_dict
        except:
            self._logger.error('Failed loading `%s` of `%s`.' % (leaf._v_name, full_name))
            raise

    def _prm_read_shared_data(self, shared_node, instance):
        """Reads shared data and constructs the appropraite class.

        :param shared_node:

            hdf5 node storing the pandas DataFrame

        :param full_name:

            Full name of the parameter or result whose data is to be loaded

        :return:

            Data to load

        """
        try:
            data_type = self._all_get_from_attrs(shared_node,
                                                 HDF5StorageService.SHARED_DATA_TYPE)
            constructor = shared.FLAG_CLASS_MAPPING[data_type]
            name = shared_node._v_name
            result = constructor(name=name, parent=instance)
            return result
        except:
            self._logger.error('Failed loading `%s` of `%s`.' % (shared_node._v_name,
                                                                 instance.v_full_name))
            raise

    def _prm_read_pandas(self, pd_node, full_name):
        """Reads a DataFrame from dis.

        :param pd_node:

            hdf5 node storing the pandas DataFrame

        :param full_name:

            Full name of the parameter or result whose data is to be loaded

        :return:

            Data to load

        """
        try:
            name = pd_node._v_name
            pathname = pd_node._v_pathname
            pandas_store = self._hdf5store
            pandas_data = pandas_store.get(pathname)
            return pandas_data
        except:
            self._logger.error('Failed loading `%s` of `%s`.' % (pd_node._v_name, full_name))
            raise

    def _prm_read_table(self, table_or_group, full_name):
        """Reads a non-nested PyTables table column by column and created a new ObjectTable for
        the loaded data.

        :param table_or_group:

            PyTables table to read from or a group containing subtables.

        :param full_name:

            Full name of the parameter or result whose data is to be loaded

        :return:

            Data to be loaded

        """
        try:
            result_table = None

            if self._all_get_from_attrs(table_or_group, HDF5StorageService.SPLIT_TABLE):
                table_name = table_or_group._v_name

                data_type_table_name = table_name + '__' + HDF5StorageService.STORAGE_TYPE

                data_type_table = table_or_group._v_children[data_type_table_name]
                data_type_dict = {}


                for row in data_type_table:
                    fieldname = row['field_name'].decode('utf-8')
                    data_type_dict[fieldname] = row['data_type'].decode('utf-8')

                for sub_table in table_or_group:
                    sub_table_name = sub_table._v_name

                    if sub_table_name == data_type_table_name:
                        continue

                    for colname in sub_table.colnames:
                        # Read Data column by column
                        col = sub_table.col(colname)
                        data_list = list(col)

                        prefix = HDF5StorageService.FORMATTED_COLUMN_PREFIX % colname
                        for idx, data in enumerate(data_list):
                            # Recall original type of data
                            data, type_changed = self._all_recall_native_type(data,
                                                                              PTItemMock(
                                                                                  data_type_dict),
                                                                              prefix)
                            if type_changed:
                                data_list[idx] = data
                            else:
                                break

                        # Construct or insert into an ObjectTable
                        if result_table is None:
                            result_table = ObjectTable(data={colname: data_list})
                        else:
                            result_table[colname] = data_list

            else:
                for colname in table_or_group.colnames:
                    # Read Data column by column
                    col = table_or_group.col(colname)
                    data_list = list(col)

                    prefix = HDF5StorageService.FORMATTED_COLUMN_PREFIX % colname
                    for idx, data in enumerate(data_list):
                        # Recall original type of data
                        data, type_changed = self._all_recall_native_type(data, table_or_group,
                                                                          prefix)
                        if type_changed:
                            data_list[idx] = data
                        else:
                            break

                    # Construct or insert into an ObjectTable
                    if result_table is None:
                        result_table = ObjectTable(data={colname: data_list})
                    else:
                        result_table[colname] = data_list

            return result_table
        except:
            self._logger.error(
                'Failed loading `%s` of `%s`.' % (table_or_group._v_name, full_name))
            raise

    @staticmethod
    def _svrc_read_array(array):
        EMPTY_ARRAY_FIX_PT_2 = 'PTCOMPAT__empty__dtype'
        empty_array_val = 0
        none_type = 0
        res = array.read()
        try:
            if (res.shape == (1,) and res[0] == empty_array_val and
                        EMPTY_ARRAY_FIX_PT_2 in array._v_attrs):
                # If the array was stored with pytables 2 we end up here
                dtype = array._v_attrs[EMPTY_ARRAY_FIX_PT_2]
                if dtype == none_type:
                    res = np.array([])
                else:
                    res = np.array([], dtype=np.dtype(dtype))
        except (AttributeError, TypeError):
            pass  # has no size or getitem, we don't need to worry
        return res

    def _prm_read_array(self, array, full_name):
        """Reads data from an array or carray

        :param array:

            PyTables array or carray to read from

        :param full_name:

            Full name of the parameter or result whose data is to be loaded

        :return:

            Data to load

        """
        try:
            result = self._svrc_read_array(array)
            # Recall original data types
            result, dummy = self._all_recall_native_type(result, array,
                                                         HDF5StorageService.DATA_PREFIX)

            return result
        except:
            self._logger.error('Failed loading `%s` of `%s`.' % (array._v_name, full_name))
            raise

    def _hdf5_interact_with_data(self, path_to_data, item_name, request, args, kwargs):

        hdf5_group = self._all_get_node_by_name(path_to_data)

        if request == 'create_shared_data' or request == 'pandas_put':
            return self._shared_write_shared_data(key=item_name, hdf5_group=hdf5_group,
                                        full_name=path_to_data, **kwargs)

        hdf5data = hdf5_group._f_get_child(item_name)

        if request == 'make_shared':
            hdf5data = hdf5_group._f_get_child(item_name)
            flag = getattr(hdf5data._v_attrs, HDF5StorageService.STORAGE_TYPE)
            setattr(hdf5data._v_attrs, HDF5StorageService.SHARED_DATA_TYPE, flag)
            setattr(hdf5data._v_attrs, HDF5StorageService.STORAGE_TYPE,
                    HDF5StorageService.SHARED_DATA)
            return
        elif request == '__thenode__':
            return hdf5data
        elif request == 'pandas_get':
            pandas_data = self._prm_read_pandas(hdf5data, path_to_data)
            return pandas_data
        elif request == 'pandas_select':
            return self._prm_select_shared_pandas_data(hdf5data, path_to_data, **kwargs)
        elif request == 'make_ordinary':
            flag = getattr(hdf5data._v_attrs, HDF5StorageService.SHARED_DATA_TYPE)
            delattr(hdf5data._v_attrs, HDF5StorageService.SHARED_DATA_TYPE)
            setattr(hdf5data._v_attrs, HDF5StorageService.STORAGE_TYPE, flag)
            return

        if kwargs and kwargs.pop('_col_func', False):
            colname = kwargs.pop('colname', None)
            if colname is None:
                colname = args[0]
            name_list = colname.split('/')
            curr = hdf5data.cols
            for name in name_list:
                curr = getattr(curr, name)
            hdf5data = curr

        what = getattr(hdf5data, request)
        if args is None and kwargs is None:
            return what
        else:
            if args is None:
                args=()
            if kwargs is None:
                kwargs = {}
            result = what(*args, **kwargs)
            return result class LazyStorageService(StorageService):
    """This lazy guy does nothing! Only for debugging purposes.

    Ignores all storage and loading requests and simply executes `pass` instead.

    """
    def __init__(self, *args, **kwargs):
        """Swallows all arguments for easier debugging"""
        pass

    def load(self, *args, **kwargs):
        """Nope, I won't care, dude!"""
        pass

    def store(self, *args, **kwargs):
        """Do whatever you want, I won't store anything!"""
        pass class QueueStorageServiceWriter(StorageServiceDataHandler):
    """Wrapper class that listens to the queue and stores queue items via the storage service."""

    def __init__(self, storage_service, storage_queue, gc_interval=None):
        super(QueueStorageServiceWriter, self).__init__(storage_service, gc_interval=gc_interval)
        self.queue = storage_queue

    @retry(9, Exception, 0.01, 'pypet.retry')
    def _receive_data(self):
        """Gets data from queue"""
        result = self.queue.get(block=True)
        if hasattr(self.queue, 'task_done'):
            self.queue.task_done()
        return result class LockWrapper(MultiprocWrapper, LockAcquisition):
    """For multiprocessing in :const:`~pypet.pypetconstants.WRAP_MODE_LOCK` mode,
    augments a storage service with a lock.

    The lock is acquired before storage or loading and released afterwards.

    """

    def __init__(self, storage_service, lock=None):
        self._storage_service = storage_service
        self.lock = lock
        self.is_locked = False
        self.pickle_lock = True
        self._set_logger()

    def __getstate__(self):
        result = super(LockWrapper, self).__getstate__()
        if not self.pickle_lock:
            result['lock'] = None
        return result

    def __repr__(self):
        return '<%s wrapping Storage Service %s>' % (self.__class__.__name__,
                                                     repr(self._storage_service))

    @property
    def is_open(self):
        """ Normally the file is opened and closed after each insertion.

        However, the storage service may provide the option to keep the store open and signals
        this via this property.

        """
        return self._storage_service.is_open

    @property
    def multiproc_safe(self):
        """Usually storage services are not supposed to be multiprocessing safe"""
        return True

    def store(self, *args, **kwargs):
        """Acquires a lock before storage and releases it afterwards."""
        try:
            self.acquire_lock()
            return self._storage_service.store(*args, **kwargs)
        finally:
            if self.lock is not None:
                try:
                    self.release_lock()
                except RuntimeError:
                    self._logger.error('Could not release lock `%s`!' % str(self.lock))

    def __del__(self):
        # In order to prevent a dead-lock in case of error,
        # we release the lock once again
        self.release_lock()

    def load(self, *args, **kwargs):
        """Acquires a lock before loading and releases it afterwards."""
        try:
            self.acquire_lock()
            return self._storage_service.load(*args, **kwargs)
        finally:
            if self.lock is not None:
                try:
                    self.release_lock()
                except RuntimeError:
                    self._logger.error('Could not release lock `%s`!' % str(self.lock)) class PipeStorageServiceSender(MultiprocWrapper, LockAcquisition):
    def __init__(self, storage_connection=None, lock=None):
        self.conn = storage_connection
        self.lock = lock
        self.is_locked = False
        self._set_logger()

    def __getstate__(self):
        # result = super(PipeStorageServiceSender, self).__getstate__()
        result = self.__dict__.copy()
        result['conn'] = None
        result['lock'] = None
        return result

    def load(self, *args, **kwargs):
        raise NotImplementedError('Pipe wrapping does not support loading. If you want to '
                                  'load data in a multiprocessing environment, use the Lock '
                                  'wrapping.')

    @retry(9, Exception, 0.01, 'pypet.retry')
    def _put_on_pipe(self, to_put):
        """Puts data on queue"""
        self.acquire_lock()
        self._send_chunks(to_put)
        self.release_lock()

    def _make_chunk_iterator(self, to_chunk, chunksize):
        return (to_chunk[i:i + chunksize] for i in range(0, len(to_chunk), chunksize))

    def _send_chunks(self, to_put):
        put_dump = pickle.dumps(to_put)
        data_size = sys.getsizeof(put_dump)
        nchunks = data_size / 20000000.   # chunks with size 20 MB
        chunksize = int(len(put_dump) / nchunks)
        chunk_iterator = self._make_chunk_iterator(put_dump, chunksize)
        for chunk in chunk_iterator:
            # print('S: sending False')
            self.conn.send(False)
            # print('S: sent False')
            # print('S: sending chunk')
            self.conn.send_bytes(chunk)
            # print('S: sent chunk %s' % chunk[0:10])
            # print('S: recv signal')
            self.conn.recv()  # wait for signal that message was received
            # print('S: read signal')
        # print('S: sending True')
        self.conn.send(True)
        # print('S: sent True')
        # print('S: recving last signal')
        self.conn.recv()  # wait for signal that message was received
        # print('S: read last signal')
        # print('S; DONE SENDING data')

    def store(self, *args, **kwargs):
        """Puts data to store on queue.

        Note that the queue will no longer be pickled if the Sender is pickled.

        """
        self._put_on_pipe(('STORE', args, kwargs))

    def send_done(self):
        """Signals the writer that it can stop listening to the queue"""
        self._put_on_pipe(('DONE', [], {})) class PipeStorageServiceWriter(StorageServiceDataHandler):
    """Wrapper class that listens to the queue and stores queue items via the storage service."""

    def __init__(self, storage_service, storage_connection, max_buffer_size=10, gc_interval=None):
        super(PipeStorageServiceWriter, self).__init__(storage_service, gc_interval=gc_interval)
        self.conn = storage_connection
        if max_buffer_size == 0:
            # no maximum buffer size
            max_buffer_size = float('inf')
        self.max_size = max_buffer_size
        self._buffer = deque()
        self._set_logger()

    def _read_chunks(self):
        chunks = []
        stop = False
        while not stop:
            # print('W: recving stop')
            stop = self.conn.recv()
            # print('W: read stop = %s' % str(stop))
            if not stop:
                # print('W: recving chunk')
                chunk = self.conn.recv_bytes()
                chunks.append(chunk)
                # print('W: read chunk')
            # print('W: sending True')
            self.conn.send(True)
            # print('W: sent True')
        # print('W: reconstructing data')
        to_load = b''.join(chunks)
        del chunks  # free unnecessary memory
        try:
            data = pickle.loads(to_load)
        except Exception:
            # We don't want to crash the storage service if reconstruction
            # due to errors fails
            self._logger.exception('Could not reconstruct pickled data.')
            data = None
        return data

    @retry(9, Exception, 0.01, 'pypet.retry')
    def _receive_data(self):
        """Gets data from pipe"""
        while True:
            while len(self._buffer) < self.max_size and self.conn.poll():
                data = self._read_chunks()
                if data is not None:
                    self._buffer.append(data)
            if len(self._buffer) > 0:
                return self._buffer.popleft() class ReferenceWrapper(MultiprocWrapper):
    """Wrapper that just keeps references to data to be stored."""
    def __init__(self):
        self.references = {}

    def store(self, msg, stuff_to_store, *args, **kwargs):
        """Simply keeps a reference to the stored data """
        trajectory_name = kwargs['trajectory_name']
        if trajectory_name not in self.references:
            self.references[trajectory_name] = []
        self.references[trajectory_name].append((msg, cp.copy(stuff_to_store), args, kwargs))

    def load(self, *args, **kwargs):
        """Not implemented"""
        raise NotImplementedError('Reference wrapping does not support loading. If you want to '
                                  'load data in a multiprocessing environment, use a Lock '
                                  'wrapping.')

    def free_references(self):
        self.references = {} class ReferenceStore(HasLogger):
    """Class that can store references"""
    def __init__(self, storage_service, gc_interval=None):
        self._storage_service = storage_service
        self.gc_interval = gc_interval
        self.operation_counter = 0
        self._set_logger()

    def _check_and_collect_garbage(self):
        if self.gc_interval and self.operation_counter % self.gc_interval == 0:
            collected = gc.collect()
            self._logger.debug('Garbage Collection: Found %d unreachable items.' % collected)
        self.operation_counter += 1

    def store_references(self, references):
        """Stores references to disk and may collect garbage."""
        for trajectory_name in references:
            self._storage_service.store(pypetconstants.LIST, references[trajectory_name], trajectory_name=trajectory_name)
        self._check_and_collect_garbage() class QueueStorageServiceSender(MultiprocWrapper, HasLogger):
    """ For multiprocessing with :const:`~pypet.pypetconstants.WRAP_MODE_QUEUE`, replaces the
        original storage service.

        All storage requests are send over a queue to the process running the
        :class:`~pypet.storageservice.QdebugueueStorageServiceWriter`.

        Does not support loading of data!

    """

    def __init__(self, storage_queue=None):
        self.queue = storage_queue
        self.pickle_queue = True
        self._set_logger()

    def __getstate__(self):
        result = super(QueueStorageServiceSender, self).__getstate__()
        if not self.pickle_queue:
            result['queue'] = None
        return result

    def load(self, *args, **kwargs):
        raise NotImplementedError('Queue wrapping does not support loading. If you want to '
                                  'load data in a multiprocessing environment, use a Lock '
                                  'wrapping.')

    @retry(9, Exception, 0.01, 'pypet.retry')
    def _put_on_queue(self, to_put):
        """Puts data on queue"""
        old = self.pickle_queue
        self.pickle_queue = False
        try:
            self.queue.put(to_put, block=True)
        finally:
            self.pickle_queue = old

    def store(self, *args, **kwargs):
        """Puts data to store on queue.

        Note that the queue will no longer be pickled if the Sender is pickled.

        """
        self._put_on_queue(('STORE', args, kwargs))

    def send_done(self):
        """Signals the writer that it can stop listening to the queue"""
        self._put_on_queue(('DONE', [], {})) class LockerServer(ZMQServer):
    """ Manages a database of locks """

    LOCK = 'LOCK'  # command for locking a lock
    RELEASE_ERROR = 'RELEASE_ERROR'  # signals unsuccessful attempt to unlock
    MSG_ERROR = 'MSG_ERROR'  # signals error in decoding client request
    UNLOCK = 'UNLOCK'  # command for unlocking a lock
    RELEASED = 'RELEASED'  # signals successful unlocking
    LOCK_ERROR = 'LOCK_ERROR'  # signals unsuccessful attempt to lock
    GO = 'GO'  # signals successful locking and and allwos continuing of client
    WAIT = 'WAIT'  # signals lock is already in use and client has to wait for release
    DELIMITER = ':::'  # delimiter to split messages
    DEFAULT_LOCK = '_DEFAULT_'  # default lock name

    def __init__(self, url="tcp://127.0.0.1:7777"):
        super(LockerServer, self).__init__(url)
        self._locks = {}  # lock DB, format 'lock_name': ('client_id', 'request_id')

    def _pre_respond_hook(self, response):
        """ Hook that can be used to temper with the server before responding

        :param response: Response to be send

        :return: Boolean value if response should be send or not

        """
        return True

    def _lock(self, name, client_id, request_id):
        """Hanldes locking of locks

        If a lock is already locked sends a WAIT command,
        else LOCKs it and sends GO.

        Complains if a given client re-locks a lock without releasing it before.

        """
        if name in self._locks:
            other_client_id, other_request_id = self._locks[name]
            if other_client_id == client_id:
                response = (self.LOCK_ERROR + self.DELIMITER +
                            'Re-request of lock `%s` (old request id `%s`) by `%s` '
                            '(request id `%s`)' % (name, client_id, other_request_id, request_id))
                self._logger.warning(response)
                return response
            else:
                return self.WAIT
        else:
            self._locks[name] = (client_id, request_id)
            return self.GO

    def _unlock(self, name, client_id, request_id):
        """Handles unlocking

        Complains if a non-existent lock should be released or
        if a lock should be released that was acquired by
        another client before.

        """
        if name in self._locks:
            other_client_id, other_request_id = self._locks[name]
            if other_client_id != client_id:
                response = (self.RELEASE_ERROR + self.DELIMITER +
                            'Lock `%s` was acquired by `%s` (old request id `%s`) and not by '
                            '`%s` (request id `%s`)' % (name,
                                                        other_client_id,
                                                        other_request_id,
                                                        client_id,
                                                        request_id))
                self._logger.error(response)
                return response
            else:
                del self._locks[name]
                return self.RELEASED
        else:
            response = (self.RELEASE_ERROR + self.DELIMITER +
                        'Lock `%s` cannot be found in database (client id `%s`, '
                        'request id `%s`)' % (name, client_id, request_id))
            self._logger.error(response)
            return response

    def run(self):
        """Runs server"""
        try:
            self._start()

            running = True
            while running:
                msg = ''
                name = ''
                client_id = ''
                request_id = ''

                request = self._socket.recv_string()
                self._logger.log(1, 'Recevied REQ `%s`', request)

                split_msg = request.split(self.DELIMITER)

                if len(split_msg) == 4:
                    msg, name, client_id, request_id = split_msg

                if msg == self.LOCK:
                    response = self._lock(name, client_id, request_id)

                elif msg == self.UNLOCK:
                    response = self._unlock(name, client_id, request_id)

                elif msg == self.PING:
                    response = self.PONG

                elif msg == self.DONE:
                    response = self.CLOSED
                    running = False

                else:
                    response = (self.MSG_ERROR + self.DELIMITER +
                                'Request `%s` not understood '
                                '(or wrong number of delimiters)' % request)
                    self._logger.error(response)

                respond = self._pre_respond_hook(response)
                if respond:
                    self._logger.log(1, 'Sending REP `%s` to `%s` (request id `%s`)',
                                     response, client_id, request_id)
                    self._socket.send_string(response)

            # Close everything in the end
            self._close()
        except Exception:
            self._logger.exception('Crashed Lock Server!')
            raise class LockerClient(ReliableClient):
    """ Implements a Lock by requesting lock information from LockServer"""

    def __init__(self, url='tcp://127.0.0.1:7777', lock_name=LockerServer.DEFAULT_LOCK):
        super(LockerClient, self).__init__(url)
        self.lock_name = lock_name
        self.id = None

    def __getstate__(self):
        result_dict = super(LockerClient, self).__getstate__()
        result_dict['id'] = None
        return result_dict

    def start(self, test_connection=True):
        if self._context is None:
            self.id = self._get_id()
            cls = self.__class__
            self._set_logger('%s.%s_%s' % (cls.__module__, cls.__name__, self.id))
            super(LockerClient, self).start(test_connection)

    @staticmethod
    def _get_id():
        return socket.getfqdn().replace(LockerServer.DELIMITER, '-') + '__' + str(os.getpid())

    @staticmethod
    def _get_request_id():
        return str(time.time()).replace(LockerServer.DELIMITER, '-')

    def _compose_request(self, request_sketch):
        request = (request_sketch + LockerServer.DELIMITER +
                   self.lock_name + LockerServer.DELIMITER + self.id +
                   LockerServer.DELIMITER + self._get_request_id())
        return request

    def acquire(self):
        """Acquires lock and returns `True`

        Blocks until lock is available.

        """
        self.start(test_connection=False)
        while True:
            str_response, retries = self._req_rep_retry(LockerServer.LOCK)
            response = str_response.split(LockerServer.DELIMITER)
            if response[0] == LockerServer.GO:
                return True
            elif response[0] == LockerServer.LOCK_ERROR and retries > 0:
                # Message was sent but Server response was lost and we tried again
                self._logger.error(str_response + '; Probably due to retry')
                return True
            elif response[0] == LockerServer.WAIT:
                time.sleep(self.SLEEP)
            else:
                raise RuntimeError('Response `%s` not understood' % response)

    def release(self):
        """Releases lock"""
        # self.start(test_connection=False)
        str_response, retries = self._req_rep_retry(LockerServer.UNLOCK)
        response = str_response.split(LockerServer.DELIMITER)
        if response[0] == LockerServer.RELEASED:
            pass  # Everything is fine
        elif response[0] == LockerServer.RELEASE_ERROR and retries > 0:
            # Message was sent but Server response was lost and we tried again
            self._logger.error(str_response + '; Probably due to retry')
        else:
            raise RuntimeError('Response `%s` not understood' % response)

    def _req_rep_retry(self, request):
        request = self._compose_request(request)
        return super(LockerClient, self)._req_rep_retry(request) class ForkAwareLockerClient(LockerClient, ForkDetector):
    """Locker Client that can detect forking of processes.

    In this case the context and socket are restarted.

    """

    def __init__(self, url='tcp://127.0.0.1:7777', lock_name=LockerServer.DEFAULT_LOCK):
        super(ForkAwareLockerClient, self).__init__(url, lock_name)
        self._pid = None

    def __getstate__(self):
        result_dict = super(ForkAwareLockerClient, self).__getstate__()
        result_dict['_pid'] = None
        return result_dict

    def start(self, test_connection=True):
        """Checks for forking and starts/restarts if desired"""
        self._detect_fork()
        super(ForkAwareLockerClient, self).start(test_connection) class TimeOutLockerServer(LockerServer):
    """ Lock Server where each lock is valid only for a fixed period of time. """

    def __init__(self, url, timeout):
        super(TimeOutLockerServer, self).__init__(url)
        self._timeout = timeout
        self._timeout_locks = {}

    def _lock(self, name, client_id, request_id):
        """Handles locking

        Locking time is stored to determine time out.
        If a lock is timed out it can be acquired by a different client.

        """
        if name in self._locks:
            other_client_id, other_request_id, lock_time = self._locks[name]
            if other_client_id == client_id:
                response = (self.LOCK_ERROR + self.DELIMITER +
                            'Re-request of lock `%s` (old request id `%s`) by `%s` '
                            '(request id `%s`)' % (name, client_id, other_request_id, request_id))
                self._logger.warning(response)
                return response
            else:
                current_time = time.time()
                if current_time - lock_time < self._timeout:
                    return self.WAIT
                else:
                    response = (self.GO + self.DELIMITER + 'Lock `%s` by `%s` (old request id `%s) '
                                                          'timed out' % (name,
                                                                         other_client_id,
                                                                         other_request_id))
                    self._logger.info(response)
                    self._locks[name] = (client_id, request_id, time.time())
                    self._timeout_locks[(name, other_client_id)] = (request_id, lock_time)
                    return response
        else:
            self._locks[name] = (client_id, request_id, time.time())
            return self.GO

    def _unlock(self, name, client_id, request_id):
        """Handles unlocking"""
        if name in self._locks:
            other_client_id, other_request_id, lock_time = self._locks[name]
            if other_client_id != client_id:
                response = (self.RELEASE_ERROR + self.DELIMITER +
                            'Lock `%s` was acquired by `%s` (old request id `%s`) and not by '
                            '`%s` (request id `%s`)' % (name,
                                                        other_client_id,
                                                        other_request_id,
                                                        client_id,
                                                        request_id))
                self._logger.error(response)
                return response
            else:
                del self._locks[name]
                return self.RELEASED
        elif (name, client_id) in self._timeout_locks:
            other_request_id, lock_time = self._timeout_locks[(name, client_id)]
            timeout = time.time() - lock_time - self._timeout
            response = (self.RELEASE_ERROR + self.DELIMITER +
                        'Lock `%s` timed out %f seconds ago (client id `%s`, '
                        'old request id `%s`)' % (name, timeout, client_id, other_request_id))
            return response
        else:
            response = (self.RELEASE_ERROR + self.DELIMITER +
                        'Lock `%s` cannot be found in database (client id `%s`, '
                        'request id `%s`)' % (name, client_id, request_id))
            self._logger.warning(response)
            return response class QueuingClient(ReliableClient):
    """ Manages the returning requests"""

    def put(self, data, block=True):
        """ If there is space it sends data to server

        If no space in the queue

        It returns the request in every 10 millisecond

        until there will be space in the queue.

        """

        self.start(test_connection=False)
        while True:
            response = self._req_rep(QueuingServerMessageListener.SPACE)
            if response == QueuingServerMessageListener.SPACE_AVAILABLE:
                self._req_rep((QueuingServerMessageListener.DATA, data))
                break
            else:
                time.sleep(0.01)

    def _send_request(self, request):
        return self._socket.send_pyobj(request) class QueuingServer(HasLogger):
    """ Implements server architecture for Queueing"""

    def __init__(self, url, storage_service, queue_maxsize, gc_interval):
        self._url = url
        self._storage_service = storage_service
        self._queue_maxsize = queue_maxsize
        self._gc_interval = gc_interval

    def run(self):
        main_queue = queue.Queue(maxsize=self._queue_maxsize)
        server_message_listener = QueuingServerMessageListener(self._url, main_queue, self._queue_maxsize)
        storage_writer = QueueStorageServiceWriter(self._storage_service, main_queue, self._gc_interval)

        server_queue = Thread(target=server_message_listener.listen, args=())
        server_queue.start()

        storage_writer.run()
        server_queue.join() class ForkAwareQueuingClient(QueuingClient, ForkDetector):
    """ Queuing Client can detect forking of process.

    In this case the context and socket are restarted.

    """

    def __init__(self, url='tcp://127.0.0.1:22334'):
        super(ForkAwareQueuingClient, self).__init__(url)
        self._pid = None

    def __getstate__(self):
        result_dict = super(ForkAwareQueuingClient, self).__getstate__()
        result_dict['_pid'] = None
        return result_dict

    def start(self, test_connection=True):
        self._detect_fork()
        super(ForkAwareQueuingClient, self).start(test_connection) def deprecated(msg=''):
    """This is a decorator which can be used to mark functions
    as deprecated. It will result in a warning being emitted
    when the function is used.

    :param msg:

        Additional message added to the warning.

    """

    def wrapper(func):
        @functools.wraps(func)
        def new_func(*args, **kwargs):
            warning_string = "Call to deprecated function or property `%s`." % func.__name__
            warning_string = warning_string + ' ' + msg
            warnings.warn(
                warning_string,
                category=DeprecationWarning,
            )
            return func(*args, **kwargs)

        return new_func

    return wrapper def kwargs_api_change(old_name, new_name=None):
    """This is a decorator which can be used if a kwarg has changed
    its name over versions to also support the old argument name.

    Issues a warning if the old keyword argument is detected and
    converts call to new API.

    :param old_name:

        Old name of the keyword argument

    :param new_name:

        New name of keyword argument

    """

    def wrapper(func):
        @functools.wraps(func)
        def new_func(*args, **kwargs):

            if old_name in kwargs:
                if new_name is None:
                    warning_string = 'Using deprecated keyword argument `%s` in function `%s`. ' \
                                 'This keyword is no longer supported, please don`t use it ' \
                                 'anymore.' % (old_name, func.__name__)
                else:
                    warning_string = 'Using deprecated keyword argument `%s` in function `%s`, ' \
                                 'please use keyword `%s` instead.' % \
                                 (old_name, func.__name__, new_name)
                warnings.warn(warning_string, category=DeprecationWarning)
                value = kwargs.pop(old_name)
                if new_name is not None:
                    kwargs[new_name] = value

            return func(*args, **kwargs)

        return new_func

    return wrapper def prefix_naming(cls):
    """Decorate that adds the prefix naming scheme"""
    if hasattr(cls, '__getattr__'):
        raise TypeError('__getattr__ already defined')
    cls.__getattr__ = _prfx_getattr_
    cls.__setattr__ = _prfx_setattr_
    return cls def is_debug():
    """Checks if user is currently debugging.

    Debugging is checked via ``'pydevd' in sys.modules``.

    :return: True of False

    """
    return 'pydevd' in sys.modules def result_sort(result_list, start_index=0):
    """Sorts a list of results in O(n) in place (since every run is unique)

    :param result_list: List of tuples [(run_idx, res), ...]
    :param start_index: Index with which to start, every entry before `start_index` is ignored

    """
    if len(result_list) < 2:
        return result_list
    to_sort = result_list[start_index:]
    minmax = [x[0] for x in to_sort]
    minimum = min(minmax)
    maximum = max(minmax)
    #print minimum, maximum
    sorted_list = [None for _ in range(minimum, maximum + 1)]
    for elem in to_sort:
        key = elem[0] - minimum
        sorted_list[key] = elem
    idx_count = start_index
    for elem in sorted_list:
        if elem is not None:
            result_list[idx_count] = elem
            idx_count += 1
    return result_list def format_time(timestamp):
    """Formats timestamp to human readable format"""
    format_string = '%Y_%m_%d_%Hh%Mm%Ss'
    formatted_time = datetime.datetime.fromtimestamp(timestamp).strftime(format_string)
    return formatted_time def port_to_tcp(port=None):
    """Returns local tcp address for a given `port`, automatic port if `None`"""
    #address = 'tcp://' + socket.gethostbyname(socket.getfqdn())
    domain_name = socket.getfqdn()
    try:
        addr_list = socket.getaddrinfo(domain_name, None)
    except Exception:
        addr_list = socket.getaddrinfo('127.0.0.1', None)
    family, socktype, proto, canonname, sockaddr = addr_list[0]
    host = convert_ipv6(sockaddr[0])
    address =  'tcp://' + host
    if port is None:
        port = ()
    if not isinstance(port, int):
        # determine port automatically
        context = zmq.Context()
        try:
            socket_ = context.socket(zmq.REP)
            socket_.ipv6 = is_ipv6(address)
            port = socket_.bind_to_random_port(address, *port)
        except Exception:
            print('Could not connect to {} using {}'.format(address, addr_list))
            pypet_root_logger = logging.getLogger('pypet')
            pypet_root_logger.exception('Could not connect to {}'.format(address))
            raise
        socket_.close()
        context.term()
    return address + ':' + str(port) def racedirs(path):
    """Like os.makedirs but takes care about race conditions"""
    if os.path.isfile(path):
        raise IOError('Path `%s` is already a file not a directory')
    while True:
        try:
            if os.path.isdir(path):
                # only break if full path has been created or exists
                break
            os.makedirs(path)
        except EnvironmentError as exc:
            # Part of the directory path already exist
            if exc.errno != 17:
                # This error won't be any good
                raise def storage_factory(storage_service, trajectory=None, **kwargs):
    """Creates a storage service, to be extended if new storage services are added

    :param storage_service:

        Storage Service instance of constructor or a string pointing to a file

    :param trajectory:

        A trajectory instance

    :param kwargs:

        Arguments passed to the storage service

    :return:

        A storage service and a set of not used keyword arguments from kwargs

    """

    if 'filename' in kwargs and storage_service is None:
        filename = kwargs['filename']
        _, ext = os.path.splitext(filename)
        if ext in ('.hdf', '.h4', '.hdf4', '.he2', '.h5', '.hdf5', '.he5'):
            storage_service = HDF5StorageService
        else:
            raise ValueError('Extension `%s` of filename `%s` not understood.' %
                             (ext, filename))
    elif isinstance(storage_service, str):
        class_name = storage_service.split('.')[-1]
        storage_service = create_class(class_name, [storage_service, HDF5StorageService])

    if inspect.isclass(storage_service):
        return _create_storage(storage_service, trajectory, **kwargs)
    else:
        return storage_service, set(kwargs.keys()) def parse_config(init_func):
    """Decorator wrapping the environment to use a config file"""
    @functools.wraps(init_func)
    def new_func(env, *args, **kwargs):
        config_interpreter = ConfigInterpreter(kwargs)
        # Pass the config data to the kwargs
        new_kwargs = config_interpreter.interpret()
        init_func(env, *args, **new_kwargs)
        # Add parameters and config data from the `.ini` file
        config_interpreter.add_parameters(env.traj)
    return new_func class Parameter(BaseParameter):
    """The standard container that handles access to simulation parameters.

    Parameters are simple container objects for data values. They handle single values as well as
    the so called exploration range. An array containing multiple values which are accessed
    one after the other in individual simulation runs.

    Parameter exploration is usually initiated through the trajectory see
    `:func:~pypet.trajectory.Trajectory.f_explore` and
    `:func:~pypet.trajectory.Trajectory.f_expand`.

    To access the parameter's data value one can call the :func:`~pypet.parameter.Parameter.f_get`
    method.

    Parameters support the concept of locking. Once a value of the parameter has been accessed,
    the parameter cannot be changed anymore unless it is explicitly unlocked using
    :func:`~pypet.parameter.Parameter.f_unlock`.
    Locking prevents parameters from being changed during runtime of a simulation.

    Supported data values for the parameter are

    * python natives (int, long, str, bool, float, complex),

    * numpy natives, arrays and matrices of type
      np.int8-64, np.uint8-64, np.float32-64, np.complex, np.str

    * python homogeneous non-nested tuples and lists

    Note that for larger numpy arrays it is recommended to use the
    :class:`~pypet.parameter.ArrayParameter`.


    In case you create a new parameter you can pass the following arguments:

    :param full_name: The full name of the parameter. Grouping can be achieved by using colons.

    :param data:

        A data value that is handled by the parameter. It is checked whether the parameter
        :func:`~pypet.parameter.Parameter.f_supports` the data. If not a TypeError is thrown.
        If the parameter becomes explored, the data value is kept as a default. After
        simulation the default value will be restored.

        The data can be accessed as follows:

        >>> param.f_get()
        42

        Or using
        >>> param.data
        42

        [It is not `v_data` because the data is supposed to be part of the trajectory tree
        or extension of the natural naming scheme and not considered as an
        attribute/variable of the parameter container.]

        To change the data after parameter creation one can call
        :func:`~pypet.parameter.Parameter.f_set`:

        >>> param.f_set(43)
        >>> param.f_get()
        43

    :param comment:

        A useful comment describing the parameter.
        The comment can be changed later on using the 'v_comment' variable.

        >>> param.v_comment = 'Example comment'
        >>> print param.v_comment
        'Example comment'

    :raises: TypeError: If `data` is not supported by the parameter.

    Example usage:

    >>> param = Parameter('traffic.mobiles.ncars',data=42, comment='I am a neat example')

    """

    __slots__ = ("_data", "_default", "_explored_range")

    def __init__(self, full_name, data=None, comment=""):
        super(Parameter, self).__init__(full_name, comment)
        self._data = None

        self._default = (
            None  # The default value, which is the same as _data in the beginning,
        )
        # but it is necessary to keep a reference to it to restore the original value
        # after exploration

        self._explored_range = (
            []
        )  # List that will changed later on if parameter is explored
        self._set_logger()

        if data is not None:
            self.f_set(data)

    def _restore_default(self):
        """Restores the default data that was set with the
        `:func:`~pypet.parameter.Parameter.f_set` method (or at initialisation).

        If the parameter is explored during the runtime of a simulation,
        the actual value of the parameter is changed and taken from the exploration range.
        Calling :func:`~pypet.parameter.Parameter._restore_default` sets the parameter's value
        back to it's original value.

        Example usage:

        >>> param = Parameter('supergroup1.subgroup2.', data=44, comment='Im a comment!')
        >>> param._explore([1,2,3,4])
        >>> param._set_parameter_access(2)
        >>> param.f_get()
        3
        >>> param._restore_default()
        >>> param.f_get()
        44

        """
        self._data = self._default

    @copydoc(BaseParameter.f_is_empty)
    def f_is_empty(self):
        """True if no data has been assigned to the parameter.

        Example usage:

        >>> param = Parameter('myname.is.example', comment='I am _empty!')
        >>> param.f_is_empty()
        True
        >>> param.f_set(444)
        >>> param.f_is_empty()
        False

        """
        return self._data is None

    def __len__(self):
        """Returns length of parameter.

        :return:

            0 if empty

            1 if not explored

            length of range if explored and has range

        """
        if self._data is None:
            return 0
        elif len(self._explored_range) > 0:
            return len(self._explored_range)
        else:
            return 1

    def f_has_range(self):
        """If the parameter has a range.

        Does not have to be `True` if the parameter is explored.
        The range might be removed during pickling to save memory.
        Accordingly, `v_explored` remains `True` whereas `f_has_range` is `False`.

        """
        return len(self._explored_range) > 0

    def __getstate__(self):
        """Returns the actual state of the parameter for pickling.

        If `v_full_copy` is true the exploration range is also pickled, otherwise it is omitted.

        """
        result = super(Parameter, self).__getstate__()

        # If we don't need a full copy of the Parameter (because a single process needs
        # only access to a single point in the parameter space) we can delete the rest
        if not self._full_copy:
            result["_explored_range"] = []

        return result

    # @no_prefix_getattr
    def __getattr__(self, item):
        """Allows to query for `.data` as an attribute"""
        if item == "data":
            return self.f_get()
        elif item == "default":
            return self.f_get_default()
        else:
            raise AttributeError(
                "`%s` object has no attribute `%s`." % (self.f_get_class_name(), item)
            )

    def __getitem__(self, item):
        """Equivalent to `f_get_range[item] if item is integer`

        Allows also ['data'] access which is equivalent to `f_get`

        :raises: TypeError if parameter has no range

        """
        if item == "data":
            return self.f_get()
        elif item == "default" or item == -1:
            return self.f_get_default()
        else:
            return super(Parameter, self).__getitem__(item)

    @copydoc(BaseParameter._set_parameter_access)
    def _set_parameter_access(self, idx=0):
        if idx >= len(self) and self.f_has_range():
            raise ValueError(
                "You try to access data item No. %d in the parameter range, "
                "yet there are only %d potential items." % (idx, len(self))
            )
        elif self.f_has_range():
            self._data = self._explored_range[idx]
        else:
            self._logger.warning(
                "You try to change the access to a parameter range of parameter"
                " `%s`. The parameter has no range, your setting has no"
                " effect."
            )

    def f_supports(self, data):
        """Checks if input data is supported by the parameter."""
        dtype = type(data)
        if dtype is tuple or dtype is list:

            # Parameters cannot handle empty tuples
            if len(data) == 0:
                return False

            old_type = None

            # Check if the data in the tuple is homogeneous
            for item in data:
                if not type(item) in pypetconstants.PARAMETER_SUPPORTED_DATA:
                    return False
                if not old_type is None and old_type != type(item):
                    return False
                old_type = type(item)
            return True

        elif dtype is np.ndarray or dtype is np.matrix:

            if data.size == 0:
                return False  # Parameters cannot handle empty arrays and matrices

            # Numpy has many string types that depend on the length of the string,
            # We allow all of them
            dtype = data.dtype
            if np.issubdtype(dtype, np.str):
                dtype = np.str

        return dtype in pypetconstants.PARAMETER_SUPPORTED_DATA

    def _values_of_same_type(self, val1, val2):
        """Checks if two values agree in type.

        Raises a TypeError if both values are not supported by the parameter.
        Returns false if only one of the two values is supported by the parameter.

        Example usage:

        >>>param._values_of_same_type(42,43)
        True

        >>>param._values_of_same_type(42,'43')
        False

        :raises: TypeError

        """
        if self.f_supports(val1) != self.f_supports(val2):
            return False

        if not self.f_supports(val1) and not self.f_supports(val2):
            raise TypeError(
                "I do not support the types of both inputs (`%s` and `%s`),"
                " therefore I cannot judge whether the two are of same type."
                % str(type(val1)),
                str(type(val2)),
            )

        if not type(val1) is type(val2):
            return False

        # Numpy arrays must agree in data type and shape
        if type(val1) is np.array:
            if not val1.dtype is val2.dtype:
                return False

            if not np.shape(val1) == np.shape(val2):
                return False

        # For tuples we now from earlier checks that the data is homogeneous.
        # Thus, only the type of the first item and the length must agree.
        if type(val1) is tuple:
            return (type(val1[0]) is type(val2[0])) and (len(val1) == len(val2))

        return True

    @copydoc(BaseParameter.f_set)
    def f_set(self, data):

        if self.v_locked:
            raise pex.ParameterLockedException(
                "Parameter `%s` is locked!" % self._full_name
            )

        if self.f_has_range():
            raise AttributeError(
                "Your Parameter is an explored array can no longer change values!"
            )

        if self.v_stored:
            self._logger.debug(
                "You are changing an already stored parameter. If "
                "you not explicitly overwrite the data on disk, this change "
                "might be lost and not propagated to disk."
            )

        if not self.f_supports(data):
            raise TypeError(
                "Unsupported data `%s` of type `%s`. "
                "If you passed a tuple or list, this error might also be caused "
                "by heterogeneous data within the "
                "tuple or list." % (str(data), str(type(data)))
            )

        self._data = data
        self._default = self._data

    @copydoc(BaseParameter.f_get_default)
    def f_get_default(self):
        if self.f_is_empty():
            raise TypeError(
                "Parameter `%s` is empty cannot access data" % self.v_full_name
            )

        self.f_lock()  # As soon as someone accesses an entry the parameter gets locked
        return self._default

    def f_get_range(self, copy=True):
        """Returns a python iterable containing the exploration range.

        :param copy:

            If the range should be copied before handed over to avoid tempering with data

        Example usage:

        >>> param = Parameter('groupA.groupB.myparam',data=22, comment='I am a neat example')
        >>> param._explore([42,43,43])
        >>> param.f_get_range()
        (42,43,44)

        :raises: TypeError: If parameter is not explored.

        """
        if not self.f_has_range():
            raise TypeError(
                "Your parameter `%s` is not array, so cannot return array."
                % self.v_full_name
            )
        elif copy:
            return self._explored_range[:]
        else:
            return self._explored_range

    def _explore(self, explore_iterable):
        """Explores the parameter according to the iterable.

        Raises ParameterLockedException if the parameter is locked.
        Raises TypeError if the parameter does not support the data,
        the types of the data in the iterable are not the same as the type of the default value,
        or the parameter has already an exploration range.

        Note that the parameter will iterate over the whole iterable once and store
        the individual data values into a tuple. Thus, the whole exploration range is
        explicitly stored in memory.

        :param explore_iterable: An iterable specifying the exploration range

        For example:

        >>> param._explore([3.0,2.0,1.0])
        >>> param.f_get_range()
        (3.0, 2.0, 1.0)

        :raises TypeError,ParameterLockedException

        """
        if self.v_locked:
            raise pex.ParameterLockedException(
                "Parameter `%s` is locked!" % self.v_full_name
            )

        if self.f_has_range():
            raise TypeError(
                "Your parameter `%s` is already explored, "
                "cannot _explore it further!" % self._name
            )

        if self._data is None:
            raise TypeError(
                "Your parameter `%s` has no default value, please specify one "
                "via `f_set` before exploration. " % self.v_full_name
            )

        data_list = self._data_sanity_checks(explore_iterable)

        self._explored_range = data_list
        self._explored = True
        self.f_lock()

    def _expand(self, explore_iterable):
        """Explores the parameter according to the iterable and appends to the exploration range.

        Raises ParameterLockedException if the parameter is locked.
        Raises TypeError if the parameter does not support the data,
        the types of the data in the iterable are not the same as the type of the default value,
        or the parameter did not have an array before.

        Note that the parameter will iterate over the whole iterable once and store
        the individual data values into a tuple. Thus, the whole exploration range is
        explicitly stored in memory.

        :param explore_iterable: An iterable specifying the exploration range

         For example:

         >>> param = Parameter('Im.an.example', data=33.33, comment='Wooohoo!')
         >>> param._explore([3.0,2.0,1.0])
         >>> param._expand([42.0, 43.42])
         >>> param.f_get_range()
         >>> [3.0, 2.0, 1.0, 42.0, 43.42]

        :raises TypeError, ParameterLockedException

        """
        if self.v_locked:
            raise pex.ParameterLockedException(
                "Parameter `%s` is locked!" % self.v_full_name
            )

        if not self.f_has_range():
            raise TypeError(
                "Your Parameter `%s` is not an array and can therefore "
                "not be expanded." % self._name
            )

        data_list = self._data_sanity_checks(explore_iterable)

        self._explored_range.extend(data_list)
        self.f_lock()

    def _data_sanity_checks(self, explore_iterable):
        """Checks if data values are  valid.

        Checks if the data values are supported by the parameter and if the values are of the same
        type as the default value.

        """
        data_list = []

        for val in explore_iterable:

            if not self.f_supports(val):
                raise TypeError(
                    "%s is of not supported type %s." % (repr(val), str(type(val)))
                )

            if not self._values_of_same_type(val, self._default):
                raise TypeError(
                    "Data of `%s` is not of the same type as the original entry value, "
                    "new type is %s vs old type %s."
                    % (self.v_full_name, str(type(val)), str(type(self._default)))
                )

            data_list.append(val)

        if len(data_list) == 0:
            raise ValueError("Cannot explore an empty list!")

        return data_list

    def _store(self):
        """Returns a dictionary of formatted data understood by the storage service.

        The data is put into an :class:`~pypet.parameter.ObjectTable` named 'data'.
        If the parameter is explored, the exploration range is also put into another table
        named 'explored_data'.

        :return: Dictionary containing the data and optionally the exploration range.

        """
        if self._data is not None:
            store_dict = {"data": ObjectTable(data={"data": [self._data]})}

        if self.f_has_range():
            store_dict["explored_data"] = ObjectTable(
                data={"data": self._explored_range}
            )

        self._locked = True

        return store_dict

    def _load(self, load_dict):
        """Loads the data and exploration range from the `load_dict`.

        The `load_dict` needs to be in the same format as the result of the
        :func:`~pypet.parameter.Parameter._store` method.

        """
        if self.v_locked:
            raise pex.ParameterLockedException(
                "Parameter `%s` is locked!" % self.v_full_name
            )

        if "data" in load_dict:
            self._data = load_dict["data"]["data"][0]
            self._default = self._data
        else:
            self._logger.warning(
                "Your parameter `%s` is empty, "
                "I did not find any data on disk." % self.v_full_name
            )

        if "explored_data" in load_dict:
            self._explored_range = [
                x for x in load_dict["explored_data"]["data"].tolist()
            ]
            self._explored = True

        self._locked = True

    @copydoc(BaseParameter.f_get)
    def f_get(self):
        if self.f_is_empty():
            raise TypeError(
                "Parameter `%s` is empty cannot access data" % self.v_full_name
            )

        self.f_lock()  # As soon as someone accesses an entry the parameter gets locked
        return self._data

    @copydoc(BaseParameter._shrink)
    def _shrink(self):

        if self.v_locked:
            raise pex.ParameterLockedException(
                "Parameter %s is locked!" % self.v_full_name
            )

        if not self.f_has_range():
            raise TypeError("Cannot shrink Parameter without a range.")

        if self.f_is_empty():
            raise TypeError("Cannot shrink empty Parameter.")

        del self._explored_range
        self._explored_range = []
        self._explored = False

    @copydoc(BaseParameter.f_empty)
    def f_empty(self):

        if self.v_locked:
            raise pex.ParameterLockedException(
                "Parameter %s is locked!" % self.v_full_name
            )

        if self.f_has_range():
            self._shrink()

        del self._data
        del self._default
        self._data = None
        self._default = None
        self._explored = False
    import __main__ as main
import os
import sys
import logging
import shutil
import multiprocessing as multip
import traceback
import hashlib
import time
import datetime
import inspect
    import dill
    import psutil
    import scoop
    import git
    import zmq
import pypet.pypetconstants as pypetconstants
    from sumatra.projects import load_project
    from sumatra.programs import PythonExecutable
    from scoop import futures, shared
from pypet.pypetlogging import LoggingManager, HasLogger, simple_logging_config
from pypet.trajectory import Trajectory
from pypet.storageservice import HDF5StorageService, LazyStorageService
from pypet.utils.mpwrappers import QueueStorageServiceWriter, LockWrapper, \
    PipeStorageServiceSender, PipeStorageServiceWriter, ReferenceWrapper, \
    ReferenceStore, QueueStorageServiceSender, LockerServer, LockerClient, \
    ForkAwareLockerClient, TimeOutLockerServer, QueuingClient, QueuingServer, \
    ForkAwareQueuingClient
from pypet.utils.siginthandling import sigint_handling
from pypet.utils.gitintegration import make_git_commit
from pypet._version import __version__ as VERSION
from pypet.utils.decorators import deprecated, kwargs_api_change, prefix_naming
from pypet.utils.helpful_functions import is_debug, result_sort, format_time, port_to_tcp, \
    racedirs
from pypet.utils.storagefactory import storage_factory
from pypet.utils.configparsing import parse_config
from pypet.parameter import Parameter
            self._traj = trajectory
            # If no new trajectory is created the time of the environment differs
            # from the trajectory and must be computed from the current time.
            init_time = time.time()
            formatted_time = format_time(init_time)
            self._timestamp = init_time
            self._time = formatted_time

        # In case the user provided a git repository path, a git commit is performed
        # and the environment's hexsha is taken from the commit if the commit was triggered by
        # this particular environment, otherwise a new one is generated
        if self._git_repository is not None:
            new_commit, self._hexsha = make_git_commit(self, self._git_repository,
                                                       self._git_message, self._git_fail)
            # Identifier hexsha
        else:
            new_commit = False

        if not new_commit:
            # Otherwise we need to create a novel hexsha
            self._hexsha = hashlib.sha1((self.trajectory.v_name +
                                                       str(self.trajectory.v_timestamp) +
                                                       str(self.timestamp) +
                                                           VERSION).encode('utf-8')).hexdigest()

        # Create the name of the environment
        short_hexsha = self._hexsha[0:7]
        name = 'environment'
        self._name = name + '_' + str(short_hexsha) + '_' + self._time  # Name of environment

        # The trajectory should know the hexsha of the current environment.
        # Thus, for all runs, one can identify by which environment they were run.
        self._traj._environment_hexsha = self._hexsha
        self._traj._environment_name = self._name

        self._logging_manager.extract_replacements(self._traj)
        self._logging_manager.remove_null_handler()
        self._logging_manager.make_logging_handlers_and_tools()

        # Drop a message if we made a commit. We cannot drop the message directly after the
        # commit, because the logging files do not exist yet,
        # and we want this commit to be tracked
        if self._git_repository is not None:
            if new_commit:
                self._logger.info('Triggered NEW GIT commit `%s`.' % str(self._hexsha))
            else:
                self._logger.info('No changes detected, added PREVIOUS GIT commit `%s`.' %
                                  str(self._hexsha))

        # Create the storage service
        if storage_service is True: # to allow compatibility with older python versions, i.e. old
            # keyword use_hdf5
            storage_service = HDF5StorageService
        if self._traj.v_storage_service is not None:
            # Use the service of the trajectory
            self._logger.info('Found storage service attached to Trajectory. Will use '
                              'this storage service.')
            self._storage_service = self.trajectory.v_storage_service
        else:
            # Create a new service
            self._storage_service, unused_factory_kwargs = storage_factory(storage_service,
                                                                        self._traj, **kwargs)
            unused_kwargs = unused_kwargs - (set(kwargs.keys()) - unused_factory_kwargs)

        if lazy_debug and is_debug():
            self._storage_service = LazyStorageService()

        self._traj.v_storage_service = self._storage_service

        # Create resume path if desired
        self._resumable = resumable

        if self._resumable:
            if resume_folder is None:
                resume_folder = os.path.join(os.getcwd(), 'resume')
            resume_path = os.path.join(resume_folder, self._traj.v_name)
        else:
            resume_path = None

        self._resume_folder = resume_folder
        self._resume_path = resume_path
        self._delete_resume = delete_resume

        # Check multiproc
        self._multiproc = multiproc
        if ncores == 0:
            # Let *pypet* detect CPU count via psutil
            ncores = psutil.cpu_count()
            self._logger.info('Determined CPUs automatically, found `%d` cores.' % ncores)
        self._ncores = ncores
        if queue_maxsize == -1:
            # Educated guess of queue size
            queue_maxsize = 2 * ncores
        self._queue_maxsize = queue_maxsize
        if wrap_mode is None:
            # None cannot be used in HDF5 files, accordingly we need a string representation
            wrap_mode = pypetconstants.WRAP_MODE_NONE
        self._wrap_mode = wrap_mode
        # Whether to use a pool of processes
        self._use_pool = use_pool
        self._use_scoop = use_scoop
        self._freeze_input = freeze_input
        self._gc_interval = gc_interval
        self._multiproc_wrapper = None # The wrapper Service

        self._do_single_runs = do_single_runs
        self._automatic_storing = automatic_storing
        self._clean_up_runs = clean_up_runs

        if (wrap_mode == pypetconstants.WRAP_MODE_NETLOCK and
                not isinstance(port, str)):
                url = port_to_tcp(port)
                self._logger.info('Determined lock-server URL automatically, it is `%s`.' % url)
        else:
            url = port
        self._url = url
        self._timeout = timeout
        # self._deep_copy_data = False  # deep_copy_data # For future reference deep_copy_arguments

        # Notify that in case of lazy debuggin we won't record anythin

        if lazy_debug and is_debug():