class NaturalNamingInterface(HasLogger):
    """Class to manage the tree structure of a trajectory.

    Handles search, insertion, etc.

    """

    def __init__(self, root_instance):

        # The root instance is a reference to the top node of the tree. This is either
        # a single run or the parent trajectory. This can change during runtime!
        self._root_instance = root_instance

        self._set_logger()

        # Dictionary containing ALL leaves. Keys are the full names and values the parameter
        # and result instances.
        self._flat_leaf_storage_dict = {}

        # Nested dictionary containing names (not full names) as keys. Values are dictionaries
        # containing the full names as keys and the parameters and results as values.
        self._nodes_and_leaves = {}

        # Twofold nested dictionary: Outer dictionary has the creator name
        # (e.g. trajectory or run_00000000) as keys.
        # Values are dictionaries containing names (not full names) as keys and dictionaries
        # of parameter and result instances as values and their full names as keys (as above).
        # This dictionary is used for fast search in case a trajectory is told to behave like
        # a particular run (by setting the v_crun property).
        self._nodes_and_leaves_runs_sorted = {}
        self._links_count =  {} # Dictionary of how often a link exists

        # Context Manager to disable logging for auto-loading
        self._disable_logging = DisableAllLogging()

        # List of names that are taboo. The user cannot create parameters or results that
        # contain these names.
        self._not_admissible_names = set(dir(self)) | set(dir(self._root_instance))
        self._python_keywords = set(keyword.kwlist)


    def _map_type_to_dict(self, type_name):
        """ Maps a an instance type representation string (e.g. 'RESULT')
        to the corresponding dictionary in root.

        """
        root = self._root_instance

        if type_name == RESULT:
            return root._results
        elif type_name == PARAMETER:
            return root._parameters
        elif type_name == DERIVED_PARAMETER:
            return root._derived_parameters
        elif type_name == CONFIG:
            return root._config
        elif type_name == LEAF:
            return root._other_leaves
        else:
            raise RuntimeError('You shall not pass!')

    def _fetch_from_string(self, store_load, name, args, kwargs):
        """Method used by f_store/load/remove_items to find a corresponding item in the tree.

        :param store_load:

            String constant specifying if we want to store, load or remove.
            The corresponding constants are defined at the top of this module.

        :param name: String name of item to store, load or remove.

        :param args: Additional arguments passed to the storage service

        :param kwargs: Additional keyword arguments passed to the storage service

        :return:

            A formatted request that can be handled by the storage service, aka
            a tuple: (msg, item_to_store_load_or_remove, args, kwargs)

        """
        if not isinstance(name, str):
            raise TypeError('No string!')

        node = self._root_instance.f_get(name)

        return self._fetch_from_node(store_load, node, args, kwargs)

    def _fetch_from_node(self, store_load, node, args, kwargs):
        """Method used by f_store/load/remove_items to find a corresponding item in the tree.

        :param store_load: String constant specifying if we want to store, load or remove
        :param node: A group, parameter or result instance.
        :param args: Additional arguments passed to the storage service
        :param kwargs: Additional keyword arguments passed to the storage service

        :return:

            A formatted request that can be handled by the storage service, aka
            a tuple: (msg, item_to_store_load_or_remove, args, kwargs)

        """
        msg = self._node_to_msg(store_load, node)

        return msg, node, args, kwargs

    def _fetch_from_tuple(self, store_load, store_tuple, args, kwargs):
        """ Method used by f_store/load/remove_items to find a corresponding item in the tree.

        The input to the method should already be in the correct format, this method only
        checks for sanity.

        :param store_load: String constant specifying if we want to store, load or remove

        :param store_tuple:

            Tuple already in correct format (msg, item, args, kwargs). If args and kwargs
            are not given, they are taken from the supplied parameters

        :param args: Additional arguments passed to the storage service if len(store_tuple)<3

        :param kwargs:

            Additional keyword arguments passed to the storage service if
            ``len(store_tuple)<4``


        :return:

            A formatted request that can be handled by the storage service, aka
            a tuple: (msg, item_to_store_load_or_remove, args, kwargs)

        """

        node = store_tuple[1]
        msg = store_tuple[0]
        if len(store_tuple) > 2:
            args = store_tuple[2]
        if len(store_tuple) > 3:
            kwargs = store_tuple[3]
        if len(store_tuple) > 4:
            raise ValueError('Your argument tuple %s has to many entries, please call '
                             'store with [(msg,item,args,kwargs),...]' % str(store_tuple))

        # #dummy test
        _ = self._fetch_from_node(store_load, node, args, kwargs)

        return msg, node, args, kwargs

    @staticmethod
    def _node_to_msg(store_load, node):
        """Maps a given node and a store_load constant to the message that is understood by
        the storage service.

        """
        if node.v_is_leaf:
            if store_load == STORE:
                return pypetconstants.LEAF
            elif store_load == LOAD:
                return pypetconstants.LEAF
            elif store_load == REMOVE:
                return pypetconstants.DELETE
        else:
            if store_load == STORE:
                return pypetconstants.GROUP
            elif store_load == LOAD:
                return pypetconstants.GROUP
            elif store_load == REMOVE:
                return pypetconstants.DELETE

    def _fetch_items(self, store_load, iterable, args, kwargs):
        """ Method used by f_store/load/remove_items to find corresponding items in the tree.


        :param store_load:

            String constant specifying if we want to store, load or remove.
            The corresponding constants are defined at the top of this module.

        :param iterable:

            Iterable over items to look for in the tree. Can be strings specifying names,
            can be the item instances themselves or already correctly formatted tuples.

        :param args: Additional arguments passed to the storage service

        :param kwargs:

            Additional keyword arguments passed to the storage service.
            Two optional keyword arguments are popped and used by this method.

            only_empties:

                Can be in kwargs if only empty parameters and results should be considered.

            non_empties:

                Can be in kwargs if only non-empty parameters and results should be considered.


        :return:

            A list containing formatted tuples.
            These tuples can be handled by the storage service, they have
            the following format: (msg, item_to_store_load_or_remove, args, kwargs)

        """
        only_empties = kwargs.pop('only_empties', False)

        non_empties = kwargs.pop('non_empties', False)

        item_list = []
        # Iterate through the iterable and apply the appropriate fetching method via try and error.
        for iter_item in iterable:

            try:
                item_tuple = self._fetch_from_string(store_load, iter_item, args, kwargs)
            except TypeError:
                try:
                    item_tuple = self._fetch_from_node(store_load, iter_item, args, kwargs)
                except AttributeError:
                    item_tuple = self._fetch_from_tuple(store_load, iter_item, args, kwargs)

            item = item_tuple[1]
            msg = item_tuple[0]

            if item.v_is_leaf:
                if only_empties and not item.f_is_empty():
                    continue
                if non_empties and item.f_is_empty():
                    continue

            # # Explored Parameters cannot be deleted, this would break the underlying hdf5 file
            # # structure
            # if (msg == pypetconstants.DELETE and
            #             item.v_full_name in self._root_instance._explored_parameters and
            #             len(self._root_instance._explored_parameters) == 1):
            #     raise TypeError('You cannot the last explored parameter of a trajectory stored '
            #                     'into an hdf5 file.')

            item_list.append(item_tuple)

        return item_list

    def _remove_subtree(self, start_node, name, predicate=None):
        """Removes a subtree from the trajectory tree.

        Does not delete stuff from disk only from RAM.

        :param start_node: The parent node from where to start
        :param name: Name of child which will be deleted and recursively all nodes below the child
        :param predicate:

            Predicate that can be used to compute for individual nodes if they should be removed
            ``True`` or kept ``False``.

        """
        def _delete_from_children(node, child_name):
            del node._children[child_name]
            if child_name in node._groups:
                del node._groups[child_name]
            elif child_name in node._leaves:
                del node._leaves[child_name]
            else:
                raise RuntimeError('You shall not pass!')

        def _remove_subtree_inner(node, predicate):

            if not predicate(node):
                return False
            elif node.v_is_group:
                for name_ in itools.chain(list(node._leaves.keys()),
                                          list(node._groups.keys())):
                    child_ = node._children[name_]
                    child_deleted = _remove_subtree_inner(child_, predicate)
                    if child_deleted:
                        _delete_from_children(node, name_)
                        del child_

                for link_ in list(node._links.keys()):
                    node.f_remove_link(link_)

                if len(node._children) == 0:
                    self._delete_node(node)
                    return True
                else:
                    return False
            else:
                self._delete_node(node)
                return True

        if name in start_node._links:
            start_node.f_remove_link(name)
        else:
            child = start_node._children[name]
            if predicate is None:
                predicate = lambda x: True

            if _remove_subtree_inner(child, predicate):
                _delete_from_children(start_node, name)
                del child
                return True
            else:
                return False

    def _delete_node(self, node):
        """Deletes a single node from the tree.

        Removes all references to the node.

        Note that the 'parameters', 'results', 'derived_parameters', and 'config' groups
        hanging directly below root cannot be deleted. Also the root node itself cannot be
        deleted. (This would cause a tremendous wave of uncontrollable self destruction, which
        would finally lead to the Apocalypse!)

        """
        full_name = node.v_full_name
        root = self._root_instance

        if full_name == '':
            # You cannot delete root
            return

        if node.v_is_leaf:
            if full_name in root._parameters:
                del root._parameters[full_name]

            elif full_name in root._config:
                del root._config[full_name]

            elif full_name in root._derived_parameters:
                del root._derived_parameters[full_name]

            elif full_name in root._results:
                del root._results[full_name]

            elif full_name in root._other_leaves:
                del root._other_leaves[full_name]

            if full_name in root._explored_parameters:

                if root._stored:
                    # We always keep the explored parameters in case the trajectory was stored
                    root._explored_parameters[full_name] = None
                else:
                    del root._explored_parameters[full_name]
                if len(root._explored_parameters) == 0:
                    root.f_shrink()

            del self._flat_leaf_storage_dict[full_name]
        else:
            del root._all_groups[full_name]

            if full_name in root._run_parent_groups:
                del root._run_parent_groups[full_name]

        # Delete all links to the node
        if full_name in root._linked_by:
            linking = root._linked_by[full_name]
            for linking_name in list(linking.keys()):
                linking_group, link_set = linking[linking_name]
                for link in list(link_set):
                    linking_group.f_remove_link(link)
        
        if (node.v_location, node.v_name) in self._root_instance._new_nodes:
            del self._root_instance._new_nodes[(node.v_location, node.v_name)]

        # Finally remove all references in the dictionaries for fast search
        self._remove_from_nodes_and_leaves(node)

        # Remove circular references
        node._vars = None
        node._func = None

    def _remove_from_nodes_and_leaves(self, node):

        run_name = node.v_run_branch
        full_name = node.v_full_name
        name = node.v_name

        del self._nodes_and_leaves[name][full_name]
        if len(self._nodes_and_leaves[name]) == 0:
            del self._nodes_and_leaves[name]

        del self._nodes_and_leaves_runs_sorted[name][run_name][full_name]
        if len(self._nodes_and_leaves_runs_sorted[name][run_name]) == 0:
            del self._nodes_and_leaves_runs_sorted[name][run_name]
            if len(self._nodes_and_leaves_runs_sorted[name]) == 0:
                del self._nodes_and_leaves_runs_sorted[name]

    def _remove_node_or_leaf(self, instance, recursive=False):
        """Removes a single node from the tree.

        Only from RAM not from hdf5 file!

        :param instance: The node to be deleted

        :param recursive: If group nodes with children should be deleted

        """
        full_name = instance.v_full_name
        split_name = deque(full_name.split('.'))
        self._remove_along_branch(self._root_instance, split_name, recursive)

    def _remove_along_branch(self, actual_node, split_name, recursive=False):
        """Removes a given node from the tree.

        Starts from a given node and walks recursively down the tree to the location of the node
        we want to remove.

        We need to walk from a start node in case we want to check on the way back whether we got
        empty group nodes due to deletion.

        :param actual_node: Current node

        :param split_name: DEQUE of names to get the next nodes.

        :param recursive:

            To also delete all children of a group node

        :return: True if node was deleted, otherwise False

        """

        # If the names list is empty, we have reached the node we want to delete.

        if len(split_name) == 0:
            if actual_node.v_is_group and actual_node.f_has_children():
                if recursive:
                    for child in list(actual_node._children.keys()):
                        actual_node.f_remove_child(child, recursive=True)
                else:
                    raise TypeError('Cannot remove group `%s` it contains children. Please '
                                    'remove with `recursive=True`.' % actual_node.v_full_name)
            self._delete_node(actual_node)
            return True

        # Otherwise get the next node by using the first name in the list
        name = split_name.popleft()

        if name in actual_node._links:
            if len(split_name)>0:
                raise RuntimeError('You cannot remove nodes while hopping over links!')
            actual_node.f_remove_link(name)
        else:
            child = actual_node._children[name]

            if self._remove_along_branch(child, split_name, recursive=recursive):

                del actual_node._children[name]
                if name in actual_node._groups:
                    del actual_node._groups[name]
                elif name in actual_node._leaves:
                    del actual_node._leaves[name]
                else:
                    raise RuntimeError('You shall not pass!')
                del child
                return False

    def _translate_shortcut(self, name):
        """Maps a given shortcut to corresponding name

        * 'run_X' or 'r_X' to 'run_XXXXXXXXX'

        * 'crun' to the current run name in case of a
          single run instance if trajectory is used via `v_crun`

        * 'par' 'parameters'

        * 'dpar' to 'derived_parameters'

        * 'res' to 'results'

        * 'conf' to 'config'

        :return: True or False and the mapped name.

        """

        if isinstance(name, int):
            return True, self._root_instance.f_wildcard('$', name)

        if name.startswith('run_') or name.startswith('r_'):
            split_name = name.split('_')
            if len(split_name) == 2:
                index = split_name[1]
                if index.isdigit():
                    return True, self._root_instance.f_wildcard('$', int(index))
                elif index == 'A':
                    return True, self._root_instance.f_wildcard('$', -1)

        if name.startswith('runtoset_') or name.startswith('rts_'):
            split_name = name.split('_')
            if len(split_name) == 2:
                index = split_name[1]
                if index.isdigit():
                    return True, self._root_instance.f_wildcard('$set', int(index))
                elif index == 'A':
                    return True, self._root_instance.f_wildcard('$set', -1)

        if name in SHORTCUT_SET:
            if name == 'par':
                return True, 'parameters'
            elif name == 'dpar':
                return True, 'derived_parameters'
            elif name == 'res':
                return True, 'results'
            elif name == 'conf':
                return True, 'config'
            else:
                raise RuntimeError('You shall not pass!')

        return False, name

    def _add_prefix(self, split_names, start_node, group_type_name):
        """Adds the correct sub branch prefix to a given name.

        Usually the prefix is the full name of the parent node. In case items are added
        directly to the trajectory the prefixes are chosen according to the matching subbranch.

        For example, this could be 'parameters' for parameters or 'results.run_00000004' for
        results added to the fifth single run.

        :param split_names:

            List of names of the new node (e.g. ``['mynewgroupA', 'mynewgroupB', 'myresult']``).

        :param start_node:

            Parent node under which the new node should be added.

        :param group_type_name:

            Type name of subbranch the item belongs to
            (e.g. 'PARAMETER_GROUP', 'RESULT_GROUP' etc).


        :return: The name with the added prefix.

        """
        root = self._root_instance

        # If the start node of our insertion is root or one below root
        # we might need to add prefixes.
        # In case of derived parameters and results we also need to add prefixes containing the
        # subbranch and the current run in case of a single run.
        # For instance, a prefix could be 'results.runs.run_00000007'.
        prepend = []
        if start_node.v_depth < 3 and not group_type_name == GROUP:
            if start_node.v_depth == 0:

                if group_type_name == DERIVED_PARAMETER_GROUP:
                    if split_names[0] == 'derived_parameters':
                        return split_names
                    else:
                        prepend += ['derived_parameters']

                elif group_type_name == RESULT_GROUP:
                    if split_names[0] == 'results':
                        return split_names
                    else:
                        prepend += ['results']

                elif group_type_name == CONFIG_GROUP:
                    if split_names[0] == 'config':
                        return split_names
                    else:
                        prepend += ['config']

                elif group_type_name == PARAMETER_GROUP:
                    if split_names[0] == 'parameters':
                        return split_names[0]
                    else:
                        prepend += ['parameters']
                else:
                    raise RuntimeError('Why are you here?')

            # Check if we have to add a prefix containing the current run
            if root._is_run and root._auto_run_prepend:
                dummy = root.f_wildcard('$', -1)
                crun = root.f_wildcard('$')
                if any(name in root._run_information for name in split_names):
                    pass
                elif any(name == dummy for name in split_names):
                    pass
                elif (group_type_name == RESULT_GROUP or
                        group_type_name == DERIVED_PARAMETER_GROUP):

                    if start_node.v_depth == 0:
                        prepend += ['runs', crun]

                    elif start_node.v_depth == 1:

                        if len(split_names) == 1 and split_names[0] == 'runs':
                            return split_names
                        else:
                            prepend += ['runs', crun]

                    elif start_node.v_depth == 2 and start_node.v_name == 'runs':
                        prepend += [crun]

        if prepend:
            split_names = prepend + split_names

        return split_names

    @staticmethod
    def _determine_types(start_node, first_name, add_leaf, add_link):
        """Determines types for generic additions"""
        if start_node.v_is_root:
            where = first_name
        else:
            where = start_node._branch

        if where in SUBTREE_MAPPING:
            type_tuple = SUBTREE_MAPPING[where]
        else:
            type_tuple = (GROUP, LEAF)

        if add_link:
            return type_tuple[0], LINK
        if add_leaf:
            return type_tuple
        else:
            return type_tuple[0], type_tuple[0]

    def _add_generic(self, start_node, type_name, group_type_name, args, kwargs,
                     add_prefix=True, check_naming=True):
        """Adds a given item to the tree irrespective of the subtree.

        Infers the subtree from the arguments.

        :param start_node: The parental node the adding was initiated from

        :param type_name:

            The type of the new instance. Whether it is a parameter, parameter group, config,
            config group, etc. See the name of the corresponding constants at the top of this
            python module.


        :param group_type_name:

            Type of the subbranch. i.e. whether the item is added to the 'parameters',
            'results' etc. These subbranch types are named as the group names
            (e.g. 'PARAMETER_GROUP') in order to have less constants.
            For all constants used see beginning of this python module.

        :param args:

            Arguments specifying how the item is added.

            If len(args)==1 and the argument is the a given instance of a result or parameter,
            this one is added to the tree.

            Otherwise it is checked if the first argument is a class specifying how to
            construct a new item and the second argument is the name of the new class.

            If the first argument is not a class but a string, the string is assumed to be
            the name of the new instance.

            Additional args are later on used for the construction of the instance.

        :param kwargs:

            Additional keyword arguments that might be handed over to the instance constructor.

        :param add_prefix:

            If a prefix group, i.e. `results`, `config`, etc. should be added

        :param check_naming:

            If it should be checked for correct namings, can be set to ``False`` if data is loaded
            and we know that all names are correct.

        :return: The new added instance

        """
        args = list(args)
        create_new = True
        name = ''
        instance = None
        constructor = None

        add_link = type_name == LINK

        # First check if the item is already a given instance or we want to add a link
        if add_link:
            name = args[0]
            instance = args[1]
            create_new = False
        elif len(args) == 1 and len(kwargs) == 0:
            item = args[0]
            try:
                name = item.v_full_name
                instance = item

                create_new = False
            except AttributeError:
                pass

        # If the item is not an instance yet, check if args[0] is a class and args[1] is
        # a string describing the new name of the instance.
        # If args[0] is not a class it is assumed to be the name of the new instance.
        if create_new:
            if len(args) > 0 and inspect.isclass(args[0]):
                constructor = args.pop(0)
            if len(args) > 0 and isinstance(args[0], str):
                name = args.pop(0)
            elif 'name' in kwargs:
                name = kwargs.pop('name')
            elif 'full_name' in kwargs:
                name = kwargs.pop('full_name')
            else:
                raise ValueError('Could not determine a name of the new item you want to add. '
                                 'Either pass the name as positional argument or as a keyword '
                                 'argument `name`.')

        split_names = name.split('.')
        if check_naming:

            for idx, name in enumerate(split_names):
                translated_shortcut, name = self._translate_shortcut(name)
                replaced, name = self._replace_wildcards(name)
                if translated_shortcut or replaced:
                    split_names[idx] = name

            # First check if the naming of the new item is appropriate
            faulty_names = self._check_names(split_names, start_node)

            if faulty_names:
                full_name = '.'.join(split_names)
                raise ValueError(
                    'Your Parameter/Result/Node `%s` contains the following not admissible names: '
                    '%s please choose other names.' % (full_name, faulty_names))

            if add_link:
                if instance is None:
                    raise ValueError('You must provide an instance to link to!')
                if instance.v_is_root:
                    raise ValueError('You cannot create a link to the root node')
                if start_node.v_is_root and name in SUBTREE_MAPPING:
                    raise ValueError('`%s` is a reserved name for a group under root.' % name)
                if not self._root_instance.f_contains(instance, with_links=False, shortcuts=False):
                    raise ValueError('You can only link to items within the trajectory tree!')

        # Check if the name fulfils the prefix conditions, if not change the name accordingly.
        if add_prefix:
            split_names = self._add_prefix(split_names, start_node, group_type_name)

        if group_type_name == GROUP:
            add_leaf = type_name != group_type_name and not add_link
            # If this is equal we add a group node
            group_type_name, type_name = self._determine_types(start_node, split_names[0],
                                                               add_leaf, add_link)

        # Check if we are allowed to add the data
        if self._root_instance._is_run and type_name in SENSITIVE_TYPES:
            raise TypeError('You are not allowed to add config or parameter data or groups '
                            'during a single run.')

        return self._add_to_tree(start_node, split_names, type_name, group_type_name, instance,
                                 constructor, args, kwargs)

    def _replace_wildcards(self, name, run_idx=None):
        """Replaces the $ wildcards and returns True/False in case it was replaced"""
        if self._root_instance.f_is_wildcard(name):
            return True, self._root_instance.f_wildcard(name, run_idx)
        else:
            return False, name

    def _add_to_nodes_and_leaves(self, new_node):

        full_name = new_node.v_full_name
        name = new_node.v_name
        run_name = new_node._run_branch
        if not name in self._nodes_and_leaves:
            self._nodes_and_leaves[name] = {full_name: new_node}
        else:
            self._nodes_and_leaves[name][full_name] = new_node

        if not name in self._nodes_and_leaves_runs_sorted:
            self._nodes_and_leaves_runs_sorted[name] = {run_name:
                                                            {full_name:
                                                                 new_node}}
        else:
            if not run_name in self._nodes_and_leaves_runs_sorted[name]:
                self._nodes_and_leaves_runs_sorted[name][run_name] = \
                    {full_name: new_node}
            else:
                self._nodes_and_leaves_runs_sorted[name][run_name]\
                    [full_name] = new_node

    def _add_to_tree(self, start_node, split_names, type_name, group_type_name,
                     instance, constructor, args, kwargs):
        """Adds a new item to the tree.

        The item can be an already given instance or it is created new.

        :param start_node:

            Parental node the adding of the item was initiated from.

        :param split_names:

            List of names of the new item

        :param type_name:

            Type of item 'RESULT', 'RESULT_GROUP', 'PARAMETER', etc. See name of constants
            at beginning of the python module.

        :param group_type_name:

            Name of the subbranch the item is added to 'RESULT_GROUP', 'PARAMETER_GROUP' etc.
            See name of constants at beginning of this python module.

        :param instance:

            Here an already given instance can be passed. If instance should be created new
            pass None.

        :param constructor:

            If instance should be created new pass a constructor class. If None is passed
            the standard constructor for the instance is chosen.

        :param args:

            Additional arguments passed to instance construction

        :param kwargs:

            Additional keyword arguments passed to instance construction

        :return: The new added instance

        :raises: ValueError if naming of the new item is invalid

        """

        # Then walk iteratively from the start node as specified by the new name and create
        # new empty groups on the fly
        try:
            act_node = start_node
            last_idx = len(split_names) - 1
            add_link = type_name == LINK
            link_added = False
            # last_name = start_node.v_crun
            for idx, name in enumerate(split_names):
                if name not in act_node._children:
                    if idx == last_idx:
                        if add_link:
                            new_node = self._create_link(act_node, name, instance)
                            link_added = True
                        elif group_type_name != type_name:
                            # We are at the end of the chain and we add a leaf node

                            new_node = self._create_any_param_or_result(act_node,
                                                                        name,
                                                                        type_name,
                                                                        instance,
                                                                        constructor,
                                                                        args, kwargs)

                            self._flat_leaf_storage_dict[new_node.v_full_name] = new_node
                        else:
                            # We add a group as desired
                            new_node = self._create_any_group(act_node, name,
                                                              group_type_name,
                                                              instance,
                                                              constructor,
                                                              args, kwargs)
                    else:
                        # We add a group on the fly
                        new_node = self._create_any_group(act_node, name,
                                                          group_type_name)


                    if name in self._root_instance._run_information:
                        self._root_instance._run_parent_groups[act_node.v_full_name] = act_node
                    if self._root_instance._is_run:
                        if link_added:
                            self._root_instance._new_links[(act_node.v_full_name, name)] = \
                                (act_node, new_node)
                        else:
                            self._root_instance._new_nodes[(act_node.v_full_name, name)] = \
                                (act_node, new_node)
                else:
                    if name in act_node._links:
                        raise AttributeError('You cannot hop over links when adding '
                                             'data to the tree. '
                                             'There is a link called `%s` under `%s`.' %
                                             (name, act_node.v_full_name))
                    if idx == last_idx:
                        if self._root_instance._no_clobber:
                            self._logger.warning('You already have a group/instance/link `%s` '
                                                 'under `%s`. '
                                                 'However, you set `v_no_clobber=True`, '
                                                 'so I will ignore your addition of '
                                                 'data.' % (name, act_node.v_full_name))
                        else:
                            raise AttributeError('You already have a group/instance/link `%s` '
                                                 'under `%s`' % (name, act_node.v_full_name))
                act_node = act_node._children[name]
            return act_node
        except:
            self._logger.error('Failed adding `%s` under `%s`.' %
                               (name, start_node.v_full_name))
            raise

    def _remove_link(self, act_node, name):
        linked_node = act_node._links[name]
        full_name = linked_node.v_full_name
        linking = self._root_instance._linked_by[full_name]
        link_set = linking[act_node.v_full_name][1]
        link_set.remove(name)
        if len(link_set) == 0:
            del linking[act_node.v_full_name]
        if len(linking) == 0:
            del self._root_instance._linked_by[full_name]
            del linking
        del act_node._links[name]
        del act_node._children[name]
        self._links_count[name] = self._links_count[name] - 1
        if self._links_count[name] < 1:
            del self._links_count[name]
        if (act_node.v_full_name, name) in self._root_instance._new_links:
            del self._root_instance._new_links[(act_node.v_full_name, name)]

    def _create_link(self, act_node, name, instance):
        """Creates a link and checks if names are appropriate
        """

        act_node._links[name] = instance
        act_node._children[name] = instance

        full_name = instance.v_full_name
        if full_name not in self._root_instance._linked_by:
            self._root_instance._linked_by[full_name] = {}
        linking = self._root_instance._linked_by[full_name]
        if act_node.v_full_name not in linking:
            linking[act_node.v_full_name] = (act_node, set())
        linking[act_node.v_full_name][1].add(name)

        if name not in self._links_count:
            self._links_count[name] = 0
        self._links_count[name] = self._links_count[name] + 1

        self._logger.debug('Added link `%s` under `%s` pointing '
                           'to `%s`.' % (name, act_node.v_full_name,
                                         instance.v_full_name))
        return instance

    def _check_names(self, split_names, parent_node=None):
        """Checks if a list contains strings with invalid names.

        Returns a description of the name violations. If names are correct the empty
        string is returned.

        :param split_names: List of strings

        :param parent_node:

            The parental node from where to start (only applicable for node names)

        """

        faulty_names = ''

        if parent_node is not None and parent_node.v_is_root and split_names[0] == 'overview':
            faulty_names = '%s `overview` cannot be added directly under the root node ' \
                           'this is a reserved keyword,' % (faulty_names)

        for split_name in split_names:

            if len(split_name) == 0:
                faulty_names = '%s `%s` contains no characters, please use at least 1,' % (
                    faulty_names, split_name)

            elif split_name.startswith('_'):
                faulty_names = '%s `%s` starts with a leading underscore,' % (
                    faulty_names, split_name)

            elif re.match(CHECK_REGEXP, split_name) is None:
                faulty_names = '%s `%s` contains non-admissible characters ' \
                               '(use only [A-Za-z0-9_-]),' % \
                               (faulty_names, split_name)

            elif '$' in split_name:
                if split_name not in self._root_instance._wildcard_keys:
                    faulty_names = '%s `%s` contains `$` but has no associated ' \
                                   'wildcard function,' % (faulty_names, split_name)

            elif split_name in self._not_admissible_names:
                warnings.warn('`%s` is a method/attribute of the '
                              'trajectory/treenode/naminginterface, you may not be '
                              'able to access it via natural naming but only by using '
                              '`[]` square bracket notation. ' % split_name,
                              category=SyntaxWarning)

            elif split_name in self._python_keywords:
                warnings.warn('`%s` is a python keyword, you may not be '
                              'able to access it via natural naming but only by using '
                              '`[]` square bracket notation. ' % split_name,
                              category=SyntaxWarning)

        name = split_names[-1]
        if len(name) >= pypetconstants.HDF5_STRCOL_MAX_NAME_LENGTH:
            faulty_names = '%s `%s` is too long the name can only have %d characters but it has ' \
                           '%d,' % \
                           (faulty_names, name, len(name),
                            pypetconstants.HDF5_STRCOL_MAX_NAME_LENGTH)

        return faulty_names

    def _create_any_group(self, parent_node, name, type_name, instance=None, constructor=None,
                          args=None, kwargs=None):
        """Generically creates a new group inferring from the `type_name`."""

        if args is None:
            args = []

        if kwargs is None:
            kwargs = {}

        full_name = self._make_full_name(parent_node.v_full_name, name)

        if instance is None:
            if constructor is None:
                if type_name == RESULT_GROUP:
                    constructor = ResultGroup
                elif type_name == PARAMETER_GROUP:
                    constructor = ParameterGroup
                elif type_name == CONFIG_GROUP:
                    constructor = ConfigGroup
                elif type_name == DERIVED_PARAMETER_GROUP:
                    constructor = DerivedParameterGroup
                elif type_name == GROUP:
                    constructor = NNGroupNode
                else:
                    raise RuntimeError('You shall not pass!')
            instance = self._root_instance._construct_instance(constructor, full_name,
                                                               *args, **kwargs)
        else:
            instance._rename(full_name)
            # Check if someone tries to add a particular standard group to a branch where
            # it does not belong:
            if type_name == RESULT_GROUP:
                if type(instance) in (NNGroupNode,
                                   ParameterGroup,
                                   ConfigGroup,
                                   DerivedParameterGroup):
                    raise TypeError('You cannot add a `%s` type of group under results' %
                                    str(type(instance)))
            elif type_name == PARAMETER_GROUP:
                if type(instance) in (NNGroupNode,
                                   ResultGroup,
                                   ConfigGroup,
                                   DerivedParameterGroup):
                    raise TypeError('You cannot add a `%s` type of group under parameters' %
                                    str(type(instance)))
            elif type_name == CONFIG_GROUP:
                if type(instance) in (NNGroupNode,
                                   ParameterGroup,
                                   ResultGroup,
                                   DerivedParameterGroup):
                    raise TypeError('You cannot add a `%s` type of group under config' %
                                    str(type(instance)))
            elif type_name == DERIVED_PARAMETER_GROUP:
                if type(instance) in (NNGroupNode,
                                   ParameterGroup,
                                   ConfigGroup,
                                   ResultGroup):
                    raise TypeError('You cannot add a `%s` type of group under derived '
                                    'parameters' % str(type(instance)))
            elif type_name == GROUP:
                if type(instance) in (ResultGroup,
                                   ParameterGroup,
                                   ConfigGroup,
                                   DerivedParameterGroup):
                    raise TypeError('You cannot add a `%s` type of group under other data' %
                                    str(type(instance)))
            else:
                raise RuntimeError('You shall not pass!')

        self._set_details_tree_node(parent_node, name, instance)
        instance._nn_interface = self
        self._root_instance._all_groups[instance.v_full_name] = instance
        self._add_to_nodes_and_leaves(instance)
        parent_node._children[name] = instance
        parent_node._groups[name] = instance

        return instance

    def _create_any_param_or_result(self, parent_node, name, type_name, instance, constructor,
                                    args, kwargs):
        """Generically creates a novel parameter or result instance inferring from the `type_name`.

        If the instance is already supplied it is NOT constructed new.

        :param parent_node:

            Parent trajectory node

        :param name:

            Name of the new result or parameter. Here the name no longer contains colons.

        :param type_name:

            Whether it is a parameter below parameters, config, derived parameters or whether
            it is a result.

        :param instance:

            The instance if it has been constructed somewhere else, otherwise None.

        :param constructor:

            A constructor used if instance needs to be constructed. If None the current standard
            constructor is chosen.

        :param args:

            Additional arguments passed to the constructor

        :param kwargs:

            Additional keyword arguments passed to the constructor

        :return: The new instance

        """
        root = self._root_instance
        full_name = self._make_full_name(parent_node.v_full_name, name)
        if instance is None:
            if constructor is None:
                if type_name == RESULT:
                    constructor = root._standard_result
                elif type_name in [PARAMETER, CONFIG, DERIVED_PARAMETER]:
                    constructor = root._standard_parameter
                else:
                    constructor = root._standard_leaf
            instance = root._construct_instance(constructor, full_name, *args, **kwargs)
        else:
            instance._rename(full_name)

        self._set_details_tree_node(parent_node, name, instance)

        where_dict = self._map_type_to_dict(type_name)
        full_name = instance._full_name

        if full_name in where_dict:
            raise AttributeError(full_name + ' is already part of trajectory,')

        if type_name != RESULT and full_name in root._changed_default_parameters:
            self._logger.info(
                'You have marked parameter %s for change before, so here you go!' %
                full_name)

            change_args, change_kwargs = root._changed_default_parameters.pop(full_name)
            instance.f_set(*change_args, **change_kwargs)

        where_dict[full_name] = instance
        self._add_to_nodes_and_leaves(instance)
        parent_node._children[name] = instance
        parent_node._leaves[name] = instance

        if full_name in self._root_instance._explored_parameters:
            instance._explored = True  # Mark this parameter as explored.
            self._root_instance._explored_parameters[full_name] = instance

        self._logger.debug('Added `%s` to trajectory.' % full_name)

        return instance

    @staticmethod
    def _make_full_name(location, name):
        if location:
            return '%s.%s' % (location, name)
        else:
            return name

    def _set_details_tree_node(self, parent_node, name, instance):
        """Renames a given `instance` based on `parent_node` and `name`.

        Adds meta information like depth as well.

        """
        depth = parent_node._depth + 1
        if parent_node.v_is_root:
            branch = name  # We add below root
        else:
            branch = parent_node._branch
        if name in self._root_instance._run_information:
            run_branch = name
        else:
            run_branch = parent_node._run_branch

        instance._set_details(depth, branch, run_branch)

    @staticmethod
    def _apply_fast_access(data, fast_access):
        """Method that checks if fast access is possible and applies it if desired"""
        if fast_access and data.f_supports_fast_access():
            return data.f_get()
        else:
            return data

    def _iter_nodes(self, node, recursive=False, max_depth=float('inf'),
                    with_links=True, in_search=False, predicate=None):
        """Returns an iterator over nodes hanging below a given start node.

        :param node:

            Start node

        :param recursive:

            Whether recursively also iterate over the children of the start node's children

        :param max_depth:

            Maximum depth to search for

        :param in_search:

            if it is used during get search and if detailed info should be returned

        :param with_links:

            If links should be considered

        :param predicate:

            A predicate to filter nodes

        :return: Iterator

        """
        def _run_predicate(x, run_name_set):
            branch = x.v_run_branch
            return branch == 'trajectory' or branch in run_name_set

        if max_depth is None:
            max_depth = float('inf')

        if predicate is None:
            predicate = lambda x: True
        elif isinstance(predicate, (tuple, list)):
            # Create a predicate from a list of run names or run indices
            run_list = predicate
            run_name_set = set()
            for item in run_list:
                if item == -1:
                    run_name_set.add(self._root_instance.f_wildcard('$', -1))
                elif isinstance(item, int):
                    run_name_set.add(self._root_instance.f_idx_to_run(item))
                else:
                    run_name_set.add(item)
            predicate = lambda x: _run_predicate(x, run_name_set)

        if recursive:
            return NaturalNamingInterface._recursive_traversal_bfs(node,
                                            self._root_instance._linked_by,
                                            max_depth, with_links,
                                            in_search, predicate)
        else:
            iterator = (x for x in self._make_child_iterator(node, with_links) if
                        predicate(x[2]))
            if in_search:
                return iterator # Here we return tuples: (depth, name, object)
            else:
                return (x[2] for x in iterator) # Here we only want the objects themselves

    def _to_dict(self, node, fast_access=True, short_names=False, nested=False,
                 copy=True, with_links=True):
        """ Returns a dictionary with pairings of (full) names as keys and instances as values.

        :param fast_access:

            If true parameter or result values are returned instead of the
            instances.

        :param short_names:

            If true keys are not full names but only the names.
            Raises a ValueError if the names are not unique.

        :param nested:

            If true returns a nested dictionary.

        :param with_links:

            If links should be considered

        :return: dictionary

        :raises: ValueError

        """

        if (fast_access or short_names or nested) and not copy:
            raise ValueError('You can not request the original data with >>fast_access=True<< or'
                             ' >>short_names=True<< of >>nested=True<<.')

        if nested and short_names:
            raise ValueError('You cannot use short_names and nested at the '
                             'same time.')

        # First, let's check if we can return the `flat_leaf_storage_dict` or a copy of that, this
        # is faster than creating a novel dictionary by tree traversal.
        if node.v_is_root:
            temp_dict = self._flat_leaf_storage_dict

            if not fast_access and not short_names:
                if copy:
                    return temp_dict.copy()
                else:
                    return temp_dict

            else:
                iterator = temp_dict.values()
        else:
            iterator = node.f_iter_leaves(with_links=with_links)

        # If not we need to build the dict by iterating recursively over all leaves:
        result_dict = {}
        for val in iterator:
            if short_names:
                new_key = val.v_name
            else:
                new_key = val.v_full_name

            if new_key in result_dict:
                raise ValueError('Your short names are not unique. '
                                 'Duplicate key `%s`!' % new_key)

            new_val = self._apply_fast_access(val, fast_access)
            result_dict[new_key] = new_val

        if nested:
            if node.v_is_root:
                nest_dict = result_dict
            else:
                # remove the name of the current node
                # such that the nested dictionary starts with the children
                strip = len(node.v_full_name) + 1
                nest_dict = {key[strip:]: val for key, val in result_dict.items()}
            result_dict = nest_dictionary(nest_dict, '.')

        return result_dict

    @staticmethod
    def _make_child_iterator(node, with_links, current_depth=0):
        """Returns an iterator over a node's children.

        In case of using a trajectory as a run (setting 'v_crun') some sub branches
        that do not belong to the run are blinded out.

        """
        cdp1 = current_depth + 1
        if with_links:
            iterator = ((cdp1, x[0], x[1]) for x in node._children.items())
        else:
            leaves = ((cdp1, x[0], x[1]) for x in node._leaves.items())
            groups = ((cdp1, y[0], y[1]) for y in node._groups.items())
            iterator = itools.chain(groups, leaves)
        return iterator

    @staticmethod
    def _recursive_traversal_bfs(node, linked_by=None,
                                 max_depth=float('inf'),
                                 with_links=True, in_search=False, predicate=None):
        """Iterator function traversing the tree below `node` in breadth first search manner.

        If `run_name` is given only sub branches of this run are considered and the rest is
        blinded out.

        """

        if predicate is None:
            predicate = lambda x: True

        iterator_queue = IteratorChain([(0, node.v_name, node)])
        #iterator_queue = iter([(0, node.v_name, node)])
        start = True
        visited_linked_nodes = set([])

        while True:
            try:
                depth, name, item = next(iterator_queue)
                full_name = item._full_name
                if start or predicate(item):
                    if full_name in visited_linked_nodes:
                        if in_search:
                            # We need to return the node again to check if a link to the node
                            # has to be found
                            yield depth, name, item
                    elif depth <= max_depth:
                        if start:
                            start = False
                        else:
                            if in_search:
                                yield depth, name, item
                            else:
                                yield item

                        if full_name in linked_by:
                            visited_linked_nodes.add(full_name)

                        if not item._is_leaf and depth < max_depth:
                            child_iterator = NaturalNamingInterface._make_child_iterator(item,
                                                                        with_links,
                                                                        current_depth=depth)
                            iterator_queue.add(child_iterator)
                            #iterator_queue = itools.chain(iterator_queue, child_iterator)
            except StopIteration:
                break

    def _get_candidate_dict(self, key, crun, use_upper_bound=True):
        # First find all nodes where the key matches the (short) name of the node
        try:
            if crun is None:
                return self._nodes_and_leaves[key]
            # This can be false in case of links which are not added to the run sorted nodes and leaves
            else:
                temp_dict = {}
                if crun in self._nodes_and_leaves_runs_sorted[key]:
                    temp_dict = self._nodes_and_leaves_runs_sorted[key][crun]
                    if use_upper_bound and len(temp_dict) > FAST_UPPER_BOUND:
                        raise pex.TooManyGroupsError('Too many nodes')

                temp_dict2 = {}
                if 'trajectory' in self._nodes_and_leaves_runs_sorted[key]:
                    temp_dict2 = self._nodes_and_leaves_runs_sorted[key]['trajectory']
                    if use_upper_bound and len(temp_dict) + len(temp_dict2) > FAST_UPPER_BOUND:
                        raise pex.TooManyGroupsError('Too many nodes')

                return ChainMap(temp_dict, temp_dict2)
        except KeyError:
            # We end up here if `key` is actually a link
            return {}

    def _very_fast_search(self, node, key, max_depth, with_links, crun):
        """Fast search for a node in the tree.

        The tree is not traversed but the reference dictionaries are searched.

        :param node:

            Parent node to start from

        :param key:

            Name of node to find

        :param max_depth:

            Maximum depth.

        :param with_links:

            If we work with links than we can only be sure to found the node in case we
            have a single match. Otherwise the other match might have been linked as well.

        :param crun:

            If given only nodes belonging to this particular run are searched and the rest
            is blinded out.

        :return: The found node and its depth

        :raises:

            TooManyGroupsError:

                If search cannot performed fast enough, an alternative search method is needed.

            NotUniqueNodeError:

                If several nodes match the key criterion

        """

        if key in self._links_count:
            return

        parent_full_name = node.v_full_name
        starting_depth = node.v_depth
        candidate_dict = self._get_candidate_dict(key, crun)

        # If there are to many potential candidates sequential search might be too slow
        if with_links:
            upper_bound = 1
        else:
            upper_bound = FAST_UPPER_BOUND
        if len(candidate_dict) > upper_bound:
            raise pex.TooManyGroupsError('Too many nodes')

        # Next check if the found candidates could be reached from the parent node
        result_node = None
        for goal_name in candidate_dict:

            # Check if we have found a matching node
            if goal_name.startswith(parent_full_name):
                candidate = candidate_dict[goal_name]
                if candidate.v_depth - starting_depth <= max_depth:
                    # In case of several solutions raise an error:
                    if result_node is not None:
                        raise pex.NotUniqueNodeError('Node `%s` has been found more than once, '
                                                     'full name of first occurrence is `%s` and of'
                                                     'second `%s`'
                                                     % (key, goal_name, result_node.v_full_name))
                    result_node = candidate

        if result_node is not None:
            return result_node, result_node.v_depth

    def _search(self, node, key, max_depth=float('inf'), with_links=True, crun=None):
        """ Searches for an item in the tree below `node`

        :param node:

            The parent node below which the search is performed

        :param key:

            Name to search for. Can be the short name, the full name or parts of it

        :param max_depth:

            maximum search depth.

        :param with_links:

            If links should be considered

        :param crun:

            Used for very fast search if we know we operate in a single run branch

        :return: The found node and the depth it was found for

        """

        # If we find it directly there is no need for an exhaustive search
        if key in node._children and (with_links or key not in node._links):
            return node._children[key], 1

        # First the very fast search is tried that does not need tree traversal.
        try:
            result = self._very_fast_search(node, key, max_depth, with_links, crun)
            if result:
                return result
        except pex.TooManyGroupsError:
            pass
        except pex.NotUniqueNodeError:
            pass

        # Slowly traverse the entire tree
        nodes_iterator = self._iter_nodes(node, recursive=True,
                                          max_depth=max_depth, in_search=True,
                                          with_links=with_links)
        result_node = None
        result_depth = float('inf')
        for depth, name, child in nodes_iterator:

            if depth > result_depth:
                # We can break here because we enter a deeper stage of the tree and we
                # cannot find matching node of the same depth as the one we found
                break

            if key == name:
                # If result_node is not None means that we care about uniqueness and the search
                # has found more than a single solution.
                if result_node is not None:
                    raise pex.NotUniqueNodeError('Node `%s` has been found more than once within '
                                                 'the same depth %d. '
                                                 'Full name of first occurrence is `%s` and of '
                                                 'second `%s`'
                                                 % (key, child.v_depth, result_node.v_full_name,
                                                    child.v_full_name))

                result_node = child
                result_depth = depth

        return result_node, result_depth

    def _backwards_search(self, start_node, split_name, max_depth=float('inf'), shortcuts=True):
        """ Performs a backwards search from the terminal node back to the start node

        :param start_node:

            The node from where search starts, or here better way where backwards search should
            end.

        :param split_name:

            List of names

        :param max_depth:

            Maximum search depth where to look for

        :param shortcuts:

            If shortcuts are allowed

        """

        result_list = [] # Result list of all found items
        full_name_set = set() # Set containing full names of all found items to avoid finding items
        # twice due to links

        colon_name = '.'.join(split_name)
        key = split_name[-1]
        candidate_dict = self._get_candidate_dict(key, None, use_upper_bound=False)
        parent_full_name = start_node.v_full_name

        split_length = len(split_name)

        for candidate_name in candidate_dict:
            # Check if candidate startswith the parent's name
            candidate = candidate_dict[candidate_name]
            if key != candidate.v_name or candidate.v_full_name in full_name_set:
                # If this is not the case we do have link, that we need to skip
                continue

            if candidate_name.startswith(parent_full_name):
                if parent_full_name != '':
                    reduced_candidate_name = candidate_name[len(parent_full_name) + 1:]
                else:
                    reduced_candidate_name = candidate_name

                candidate_split_name = reduced_candidate_name.split('.')

                if len(candidate_split_name) > max_depth:
                    break

                if len(split_name) == 1 or reduced_candidate_name.endswith(colon_name):
                    result_list.append(candidate)
                    full_name_set.add(candidate.v_full_name)

                elif shortcuts:

                    candidate_set = set(candidate_split_name)
                    climbing = True
                    for name in split_name:
                        if name not in candidate_set:
                            climbing = False
                            break

                    if climbing:
                        count = 0
                        candidate_length = len(candidate_split_name)
                        for idx in range(candidate_length):

                            if idx + split_length - count > candidate_length:
                                break

                            if split_name[count] == candidate_split_name[idx]:
                                count += 1
                                if count == len(split_name):
                                    result_list.append(candidate)
                                    full_name_set.add(candidate.v_full_name)
                                    break

        return result_list

    def _get_all(self, node, name, max_depth, shortcuts):
        """ Searches for all occurrences of `name` under `node`.

        :param node:

            Start node

        :param name:

            Name what to look for can be longer and separated by colons, i.e.
            `mygroupA.mygroupB.myparam`.

        :param max_depth:

            Maximum depth to search for relative to start node.

        :param shortcuts:

            If shortcuts are allowed

        :return:

            List of nodes that match the name, empty list if nothing was found.

        """

        if max_depth is None:
            max_depth = float('inf')

        if isinstance(name, list):
            split_name = name
        elif isinstance(name, tuple):
            split_name = list(name)
        elif isinstance(name, int):
            split_name = [name]
        else:
            split_name = name.split('.')

        for idx, key in enumerate(split_name):
            _, key = self._translate_shortcut(key)
            _, key = self._replace_wildcards(key)
            split_name[idx] = key

        return self._backwards_search(node, split_name, max_depth, shortcuts)

    def _check_flat_dicts(self, node, split_name):

        # Check in O(d) first if a full parameter/result name is given and
        # we might be able to find it in the flat storage dictionary or the group dictionary.
        # Here `d` refers to the depth of the tree
        fullname = '.'.join(split_name)

        if fullname.startswith(node.v_full_name):
            new_name = fullname
        else:
            new_name = node.v_full_name + '.' + fullname

        if new_name in self._flat_leaf_storage_dict:
            return self._flat_leaf_storage_dict[new_name]

        if new_name in self._root_instance._all_groups:
            return self._root_instance._all_groups[new_name]

        return None

    def _get(self, node, name, fast_access,
             shortcuts, max_depth, auto_load, with_links):
        """Searches for an item (parameter/result/group node) with the given `name`.

        :param node: The node below which the search is performed

        :param name: Name of the item (full name or parts of the full name)

        :param fast_access: If the result is a parameter, whether fast access should be applied.

        :param max_depth:

            Maximum search depth relative to start node.

        :param auto_load:

            If data should be automatically loaded

        :param with_links

            If links should be considered

        :return:

            The found instance (result/parameter/group node) or if fast access is True and you
            found a parameter or result that supports fast access, the contained value is returned.

        :raises:

            AttributeError if no node with the given name can be found.
            Raises errors that are raised by the storage service if `auto_load=True` and
            item cannot be found.

        """
        if auto_load and not with_links:
            raise ValueError('If you allow auto-loading you mus allow links.')

        if isinstance(name, list):
            split_name = name
        elif isinstance(name, tuple):
            split_name = list(name)
        elif isinstance(name, int):
            split_name = [name]
        else:
            split_name = name.split('.')

        if node.v_is_root:
            # We want to add `parameters`, `config`, `derived_parameters` and `results`
            # on the fly if they don't exist
            if len(split_name) == 1 and split_name[0] == '':
                return node
            key = split_name[0]
            _, key = self._translate_shortcut(key)
            if key in SUBTREE_MAPPING and key not in node._children:
                node.f_add_group(key)

        if max_depth is None:
            max_depth = float('inf')

        if len(split_name) > max_depth and shortcuts:
            raise ValueError(
                'Name of node to search for (%s) is longer thant the maximum depth %d' %
                (str(name), max_depth))

        try_auto_load_directly1 = False
        try_auto_load_directly2 = False
        wildcard_positions = []

        root = self._root_instance
        # # Rename shortcuts and check keys:
        for idx, key in enumerate(split_name):
            translated_shortcut, key = self._translate_shortcut(key)
            if translated_shortcut:
                split_name[idx] = key

            if key[0] == '_':
                raise AttributeError('Leading underscores are not allowed '
                                     'for group or parameter '
                                     'names. Cannot return %s.' % key)

            is_wildcard = self._root_instance.f_is_wildcard(key)
            if (not is_wildcard and key not in self._nodes_and_leaves and
                    key not in self._links_count):
                try_auto_load_directly1 = True
                try_auto_load_directly2 = True

            if is_wildcard:
                wildcard_positions.append((idx, key))
                if root.f_wildcard(key) not in self._nodes_and_leaves:
                    try_auto_load_directly1 = True
                if root.f_wildcard(key, -1) not in self._nodes_and_leaves:
                    try_auto_load_directly2 = True

        run_idx = root.v_idx
        wildcard_exception = None # Helper variable to store the exception thrown in case
        # of using a wildcard, to be re-thrown later on.

        if try_auto_load_directly1 and try_auto_load_directly2 and not auto_load:
            for wildcard_pos, wildcard in wildcard_positions:
                split_name[wildcard_pos] = root.f_wildcard(wildcard, run_idx)
            raise AttributeError('%s is not part of your trajectory or it\'s tree. ' %
                                 str('.'.join(split_name)))

        if run_idx > -1:
            # If we count the wildcard we have to perform the search twice,
            # one with a run name and one with the dummy:
            with self._disable_logging:
                try:
                    for wildcard_pos, wildcard in wildcard_positions:
                        split_name[wildcard_pos] = root.f_wildcard(wildcard, run_idx)

                    result = self._perform_get(node, split_name, fast_access,
                                               shortcuts, max_depth, auto_load, with_links,
                                               try_auto_load_directly1)
                    return result
                except (pex.DataNotInStorageError, AttributeError) as exc:
                    wildcard_exception = exc

        if wildcard_positions:
            for wildcard_pos, wildcard in wildcard_positions:
                split_name[wildcard_pos] = root.f_wildcard(wildcard, -1)
        try:
            return self._perform_get(node, split_name, fast_access,
                                     shortcuts, max_depth, auto_load, with_links,
                                     try_auto_load_directly2)
        except (pex.DataNotInStorageError, AttributeError):
            # Re-raise the old error in case it was thronw already
            if wildcard_exception is not None:
                raise wildcard_exception
            else:
                raise

    def _perform_get(self, node, split_name, fast_access,
                     shortcuts, max_depth, auto_load, with_links,
                     try_auto_load_directly):
        """Searches for an item (parameter/result/group node) with the given `name`.

        :param node: The node below which the search is performed

        :param split_name: Name split into list according to '.'

        :param fast_access: If the result is a parameter, whether fast access should be applied.

        :param max_depth:

            Maximum search depth relative to start node.

        :param auto_load:

            If data should be automatically loaded

        :param with_links:

            If links should be considered.

        :param try_auto_load_directly:

            If one should skip search and directly try auto_loading

        :return:

            The found instance (result/parameter/group node) or if fast access is True and you
            found a parameter or result that supports fast access, the contained value is returned.

        :raises:

            AttributeError if no node with the given name can be found
            Raises errors that are raiesd by the storage service if `auto_load=True`

        """

        result = None
        name = '.'.join(split_name)

        if len(split_name) > max_depth:
            raise AttributeError('The node or param/result `%s`, cannot be found under `%s`'
                                 'The name you are looking for is larger than the maximum '
                                 'search depth.' %
                                 (name, node.v_full_name))

        if shortcuts and not try_auto_load_directly:
            first = split_name[0]

            if len(split_name) == 1 and first in node._children and (with_links or
                                                                    first not in node._links):
                result = node._children[first]
            else:

                result = self._check_flat_dicts(node, split_name)

                if result is None:
                    # Check in O(N) with `N` number of groups and nodes
                    # [Worst Case O(N), average case is better
                    # since looking into a single dict costs O(1)].
                    result = node
                    crun = None
                    for key in split_name:
                        if key in self._root_instance._run_information:
                            crun = key
                        result, depth = self._search(result, key, max_depth, with_links, crun)
                        max_depth -= depth
                        if result is None:
                            break

        elif not try_auto_load_directly:
            result = node
            for name in split_name:
                if name in result._children and (with_links or not name in result._links):
                    result = result._children[name]
                else:
                    raise AttributeError(
                        'You did not allow for shortcuts and `%s` was not directly '
                        'found  under node `%s`.' % (name, result.v_full_name))

        if result is None and auto_load:
            try:
                result = node.f_load_child('.'.join(split_name),
                                           load_data=pypetconstants.LOAD_DATA)
                if (self._root_instance.v_idx != -1 and
                            result.v_is_leaf and
                            result.v_is_parameter and
                            result.v_explored):
                    result._set_parameter_access(self._root_instance.v_idx)
            except:
                self._logger.error('Error while auto-loading `%s` under `%s`.' %
                                   (name, node.v_full_name))
                raise

        if result is None:
            raise AttributeError('The node or param/result `%s`, cannot be found under `%s`' %
                                 (name, node.v_full_name))
        if result.v_is_leaf:
            if auto_load and result.f_is_empty():

                try:
                    self._root_instance.f_load_item(result)
                    if (self._root_instance.v_idx != -1 and
                            result.v_is_parameter and
                            result.v_explored):
                        result._set_parameter_access(self._root_instance.v_idx)
                except:
                    self._logger.error('Error while auto-loading `%s` under `%s`. I found the '
                                       'item but I could not load the data.' %
                                       (name, node.v_full_name))
                    raise

            return self._apply_fast_access(result, fast_access)
        else:
            return result class ResultGroup(NNGroupNode):
    """Group node in your trajectory, hanging below `traj.results`.

    You can add other groups or results to it.

    """

    __slots__ = ()

    def f_add_result_group(self, *args, **kwargs):
        """Adds an empty result group under the current node.

        Adds the full name of the current node as prefix to the name of the group.
        If current node is a single run (root) adds the prefix `'results.runs.run_08%d%'` to the
        full name where `'08%d'` is replaced by the index of the current run.

        The `name` can also contain subgroups separated via colons, for example:
        `name=subgroup1.subgroup2.subgroup3`. These other parent groups will be automatically
        be created.

        """

        return self._nn_interface._add_generic(self, type_name=RESULT_GROUP,
                                               group_type_name=RESULT_GROUP,
                                               args=args, kwargs=kwargs)

    def f_add_result(self, *args, **kwargs):
        """Adds a result under the current node.

        There are two ways to add a new result either by adding a result instance:

        >>> new_result = Result('group1.group2.myresult', 1666, x=3, y=4, comment='Example!')
        >>> traj.f_add_result(new_result)

        Or by passing the values directly to the function, with the name being the first
        (non-keyword!) argument:

        >>> traj.f_add_result('group1.group2.myresult', 1666, x=3, y=3,comment='Example!')


        If you want to create a different result than the standard result, you can
        give the constructor as the first (non-keyword!) argument followed by the name
        (non-keyword!):

        >>> traj.f_add_result(PickleResult,'group1.group2.myresult', 1666, x=3, y=3, comment='Example!')

        Additional arguments (here `1666`) or keyword arguments (here `x=3, y=3`) are passed
        onto the constructor of the result.


        Adds the full name of the current node as prefix to the name of the result.
        If current node is a single run (root) adds the prefix `'results.runs.run_08%d%'` to the
        full name where `'08%d'` is replaced by the index of the current run.

        """
        return self._nn_interface._add_generic(self, type_name=RESULT,
                                               group_type_name=RESULT_GROUP,
                                               args=args, kwargs=kwargs)

    f_ares = f_add_result  # Abbreviation of the function class ParameterGroup(NNGroupNode):
    """ Group node in your trajectory, hanging below `traj.parameters`.

    You can add other groups or parameters to it.

    """
    __slots__ = ()

    def f_add_parameter_group(self, *args, **kwargs):
        """Adds an empty parameter group under the current node.

        Can be called with ``f_add_parameter_group('MyName', 'this is an informative comment')``
        or ``f_add_parameter_group(name='MyName', comment='This is an informative comment')``
        or with a given new group instance:
        ``f_add_parameter_group(ParameterGroup('MyName', comment='This is a comment'))``.

        Adds the full name of the current node as prefix to the name of the group.
        If current node is the trajectory (root), the prefix `'parameters'`
        is added to the full name.

        The `name` can also contain subgroups separated via colons, for example:
        `name=subgroup1.subgroup2.subgroup3`. These other parent groups will be automatically
        created.

        """
        return self._nn_interface._add_generic(self, type_name=PARAMETER_GROUP,
                                               group_type_name=PARAMETER_GROUP,
                                               args=args, kwargs=kwargs)

    def f_add_parameter(self, *args, **kwargs):
        """ Adds a parameter under the current node.

        There are two ways to add a new parameter either by adding a parameter instance:

        >>> new_parameter = Parameter('group1.group2.myparam', data=42, comment='Example!')
        >>> traj.f_add_parameter(new_parameter)

        Or by passing the values directly to the function, with the name being the first
        (non-keyword!) argument:

        >>> traj.f_add_parameter('group1.group2.myparam', 42, comment='Example!')

        If you want to create a different parameter than the standard parameter, you can
        give the constructor as the first (non-keyword!) argument followed by the name
        (non-keyword!):

        >>> traj.f_add_parameter(PickleParameter,'group1.group2.myparam', data=42, comment='Example!')

        The full name of the current node is added as a prefix to the given parameter name.
        If the current node is the trajectory the prefix `'parameters'` is added to the name.

        Note, all non-keyword and keyword parameters apart from the optional constructor
        are passed on as is to the constructor.

        Moreover, you always should specify a default data value of a parameter,
        even if you want to explore it later.

        """
        return self._nn_interface._add_generic(self, type_name=PARAMETER,
                                               group_type_name=PARAMETER_GROUP,
                                               args=args, kwargs=kwargs)

    f_apar = f_add_parameter  # Abbreviation of the function class DerivedParameterGroup(NNGroupNode):
    """Group node in your trajectory, hanging below `traj.derived_parameters`.

    You can add other groups or parameters to it.

    """

    __slots__ = ()

    def f_add_derived_parameter_group(self, *args, **kwargs):
        """Adds an empty derived parameter group under the current node.

        Adds the full name of the current node as prefix to the name of the group.
        If current node is a single run (root) adds the prefix `'derived_parameters.runs.run_08%d%'`
        to the full name where `'08%d'` is replaced by the index of the current run.

        The `name` can also contain subgroups separated via colons, for example:
        `name=subgroup1.subgroup2.subgroup3`. These other parent groups will be automatically
        be created.

        """

        return self._nn_interface._add_generic(self, type_name=DERIVED_PARAMETER_GROUP,
                                               group_type_name=DERIVED_PARAMETER_GROUP,
                                               args=args, kwargs=kwargs)

    def f_add_derived_parameter(self, *args, **kwargs):
        """Adds a derived parameter under the current group.

        Similar to
        :func:`~pypet.naturalnaming.ParameterGroup.f_add_parameter`

        Naming prefixes are added as in
        :func:`~pypet.naturalnaming.DerivedParameterGroup.f_add_derived_parameter_group`

        """
        return self._nn_interface._add_generic(self, type_name=DERIVED_PARAMETER,
                                               group_type_name=DERIVED_PARAMETER_GROUP,
                                               args=args, kwargs=kwargs)

    f_adpar = f_add_derived_parameter  # Abbreviation of the function class ConfigGroup(NNGroupNode):
    """Group node in your trajectory, hanging below `traj.config`.

    You can add other groups or parameters to it.

    """

    __slots__ = ()

    def f_add_config_group(self, *args, **kwargs):
        """Adds an empty config group under the current node.

        Adds the full name of the current node as prefix to the name of the group.
        If current node is the trajectory (root), the prefix `'config'` is added to the full name.

        The `name` can also contain subgroups separated via colons, for example:
        `name=subgroup1.subgroup2.subgroup3`. These other parent groups will be automatically
        be created.

        """
        return self._nn_interface._add_generic(self, type_name=CONFIG_GROUP,
                                               group_type_name=CONFIG_GROUP,
                                               args=args, kwargs=kwargs)

    def f_add_config(self, *args, **kwargs):
        """Adds a config parameter under the current group.

        Similar to
        :func:`~pypet.naturalnaming.ParameterGroup.f_add_parameter`.

        If current group is the trajectory the prefix `'config'` is added to the name.

        """
        return self._nn_interface._add_generic(self, type_name=CONFIG,
                                               group_type_name=CONFIG_GROUP,
                                               args=args, kwargs=kwargs)

    f_aconf=f_add_config  # Abbreviation of the function STORE = 'STORE'  # We want to store stuff with the storage service LOAD = 'LOAD'  # We want to load stuff with the storage service REMOVE = 'REMOVE'  # We want to remove stuff, potentially from disk class BaseParameter(NNLeafNode):
    """Abstract class that specifies the methods for a trajectory parameter.

    Parameters are simple container objects for data values. They handle single values as well as
    ranges of potential values. These range arrays contain multiple values which are accessed
    one after the other in individual simulation runs.

    Parameter exploration is usually initiated through the trajectory see
    :func:`~pypet.trajectory.Trajectory.f_explore` and
    :func:`~pypet.trajectory.Trajectory.f_expand`.

    To access the parameter's data value one can call the
    :func:`~pypet.parameter.BaseParameter.f_get` method.

    Parameters support the concept of locking. Once a value of the parameter has been accessed,
    the parameter cannot be changed anymore unless it is explicitly unlocked using
    :func:`~pypet.parameter.BaseParameter.f_unlock`.
    This prevents parameters from being changed during runtime of a simulation.

    If multiprocessing is desired the parameter must be picklable!

    :param full_name:

        The full name of the parameter in the trajectory tree, groupings are
        separated by a colon: `fullname = 'supergroup.subgroup.paramname'`

    :param comment:

        A useful comment describing the parameter:
        `comment = 'Some useful text, dude!'`

    """

    __slots__ = ("_locked", "_full_copy", "_explored")

    def __init__(self, full_name, comment=""):
        super(BaseParameter, self).__init__(full_name, comment, is_parameter=True)

        self._locked = False

        # Whether to keep the full range array when pickled or not
        self._full_copy = False
        self._explored = False  # If explored or not

    def f_supports(self, data):
        """Checks whether the data is supported by the parameter."""
        return type(data) in pypetconstants.PARAMETER_SUPPORTED_DATA

    @property
    def v_locked(self):
        """Whether or not the parameter is locked and prevents further modification"""
        return self._locked

    def f_supports_fast_access(self):
        """Checks if parameter supports fast access.

        A parameter supports fast access if it is NOT empty!

        """
        return not self.f_is_empty()

    @property
    def v_explored(self):
        """Whether parameter is explored.

        Does not necessarily have to be similar to
        :func:`~pypet.parameter.BaseParameter.f_has_range` since the range can be
        deleted on pickling and the parameter remains explored.

        """
        return self._explored

    @property
    def v_full_copy(self):
        """Whether or not the full parameter including the range or only the current
        data is copied during pickling.

        If you run your simulations in multiprocessing mode, the whole trajectory and all
        parameters need to be pickled and are sent to the individual processes.
        Each process than runs an individual point in the parameter space.
        As a consequence, you do not need the full ranges during these calculations.
        Thus, if the full copy mode is set to `False` the parameter is pickled without
        the range array and you can save memory.

        If you want to access the full range during individual runs, you need to set
        `v_full_copy` to `True`.

        It is recommended NOT to do that in order to save memory and also do obey the
        philosophy that individual simulation runs are independent.

        Example usage:

        >>> import pickle
        >>> param = Parameter('examples.fullcopy', data=333, comment='I show you how the copy mode works!')
        >>> param._explore([1,2,3,4])
        >>> dump=pickle.dumps(param)
        >>> newparam = pickle.loads(dump)
        >>> newparam.f_get_range()
        TypeError

        >>> param.v_full_copy=True
        >>> dump = pickle.dumps(param)
        >>> newparam=pickle.loads(dump)
        >>> newparam.f_get_range()
        (1,2,3,4)

        """
        return self._full_copy

    @v_full_copy.setter
    def v_full_copy(self, val):
        """Sets the full copy mode"""
        val = bool(val)
        self._full_copy = val

    def f_has_range(self):
        """Returns true if the parameter contains a range array.

        Not necessarily equal to `v_explored` if the range is removed on
        pickling due to `v_full_copy=False`.

        ABSTRACT: Needs to be defined in subclass

        """
        raise NotImplementedError("Should have implemented this.")

    def _restore_default(self):
        """Restores original data if changed due to exploration.

        If a Parameter is explored, the actual data is changed over the course of different
        simulations. This method restores the original data assigned before exploration.

        ABSTRACT: Needs to be defined in subclass

        """
        raise NotImplementedError("Should have implemented this.")

    def f_get_range_length(self):
        """Returns the length of the parameter range.

        Raises TypeError if the parameter has no range.

        Does not need to be implemented if the parameter supports
        ``__len__`` appropriately.

        """
        if not self.f_has_range():
            raise TypeError("Not applicable, parameter does not have a range")
        elif hasattr(self, "__len__"):
            return len(self)
        else:
            raise NotImplementedError("Should have implemented this.")

    def f_val_to_str(self):
        """String summary of the value handled by the parameter.

        Note that representing the parameter as a string accesses its value,
        but for simpler debugging, this does not lock the parameter or counts as usage!

        Calls `__repr__` of the contained value.

        """
        old_locked = self._locked
        try:
            return repr(self.f_get())
        except Exception:
            return "No Evaluation possible (yet)!"
        finally:
            self._locked = old_locked

    def _equal_values(self, val1, val2):
        """Checks if the parameter considers two values as equal.

        This is important for the trajectory in case of merging. In case you want to delete
        duplicate parameter points, the trajectory needs to know when two parameters
        are equal. Since equality is not always implemented by values handled by
        parameters in the same way, the parameters need to judge whether their values are equal.

        The straightforward example here is a numpy array.
        Checking for equality of two numpy arrays yields
        a third numpy array containing truth values of a piecewise comparison.
        Accordingly, the parameter could judge two numpy arrays equal if ALL of the numpy
        array elements are equal.

        In this BaseParameter class values are considered to be equal if they obey
        the function :func:`~pypet.utils.comparisons.nested_equal`.
        You might consider implementing a different equality comparison in your subclass.

        :raises: TypeError: If both values are not supported by the parameter.

        """
        if self.f_supports(val1) != self.f_supports(val2):
            return False

        if not self.f_supports(val1) and not self.f_supports(val2):
            raise TypeError(
                "I do not support the types of both inputs (`%s` and `%s`), "
                "therefore I cannot judge whether "
                "the two are equal." % (str(type(val1)), str(type(val2)))
            )

        if not self._values_of_same_type(val1, val2):
            return False

        return comparisons.nested_equal(val1, val2)

    def _values_of_same_type(self, val1, val2):
        """Checks if two values agree in type.

        For example, two 32 bit integers would be of same type, but not a string and an integer,
        nor a 64 bit and a 32 bit integer.

        This is important for exploration. You are only allowed to explore data that
        is of the same type as the default value.

        One could always come up with a trivial solution of `type(val1) is type(val2)`.
        But sometimes your parameter does want even more strict equality or
        less type equality.

        For example, the :class:`~pypet.parameter.Parameter` has a stricter sense of
        type equality regarding numpy arrays. In order to have two numpy arrays of the same type,
        they must also agree in shape. However, the :class:`~pypet.parameter.ArrayParameter`,
        considers all numpy arrays as of being of same type regardless of their shape.

        Moreover, the :class:`~pypet.parameter.SparseParameter` considers all supported
        sparse matrices (csc, csr, bsr, dia) as being of the same type. You can make
        explorations using all these four types at once.

        The difference in how strict types are treated arises from the way parameter data
        is stored to disk and how the parameters hand over their data to the storage service
        (see :func:`pypet.parameter.BaseParameter._store`).

        The :class:`~pypet.parameter.Parameter` puts all it's data in an
        :class:`~pypet.parameter.ObjectTable` which
        has strict constraints on the column sizes. This means that numpy array columns only
        accept numpy arrays with a particular size. In contrast, the array and sparse
        parameter hand over their data as individual items which yield individual entries
        in the hdf5 node. In order to see what I mean simply run an experiment with all 3
        parameters, explore all of them, and take a look at the resulting hdf5 file!

        However, this BaseParameter class implements the straightforward version of
        `type(val1) is type(val2)` to consider data to be of the same type.

        :raises: TypeError: if both values are not supported by the parameter.

        """
        if self.f_supports(val1) != self.f_supports(val2):
            return False

        if not self.f_supports(val1) and not self.f_supports(val2):
            raise TypeError(
                "I do not support the types of both inputs (`%s` and `%s`),"
                " therefore I cannot judge whether the two are of same type."
                % str(type(val1)),
                str(type(val2)),
            )

        return type(val1) is type(val2)

    def __repr__(self):
        return "<%s>" % self.__str__()

    def __str__(self):
        """String representation of the Parameter

        Output format is:`<class_name> full_name (len:X, `comment`): value`.
        If comment is the empty string, the comment is omitted.
        If the parameter is not explored the length is omitted.

        """
        if self.f_has_range():
            lenstr = "len:%d" % self.f_get_range_length()
        else:
            lenstr = ""

        if self.v_comment:
            commentstr = "`%s`" % self.v_comment
        else:
            commentstr = ""

        if commentstr or lenstr:
            if commentstr and lenstr:
                combined_str = "%s, %s" % (lenstr, commentstr)
            elif commentstr:
                combined_str = commentstr
            elif lenstr:
                combined_str = lenstr
            else:
                raise RuntimeError("You shall not pass!")

            infostr = " (%s)" % combined_str

        else:
            infostr = ""

        return_string = "%s %s%s" % (self.f_get_class_name(), self.v_full_name, infostr)

        if not self.f_is_empty():
            return_string += ": " + self.f_val_to_str()

        return return_string

    def f_unlock(self):
        """Unlocks the locked parameter.

        Please use it very carefully, or best do not use this function at all.
        There should better be no reason to unlock a locked parameter!
        The only exception I can think of is to unlock a large derived parameter
        after usage to subsequently call :func:`~pypet.parameter.BaseParameter.f_empty`
        to clear memory.

        """
        self._locked = False

    def f_lock(self):
        """Locks the parameter and forbids further manipulation.

        Changing the data value or exploration range of the parameter are no longer allowed.

        """
        self._locked = True

    def f_set(self, data):
        """Sets a data value for a parameter.

        Example usage:

        >>> param = Parameter('groupA.groupB.myparam', comment='I am a neat example')
        >>> param.f_set(44.0)
        >>> param.f_get()
        44.0

        :raises:

                ParameterLockedException: If parameter is locked

                TypeError: If the type of the data value is not supported by the parameter

        ABSTRACT: Needs to be defined in subclass

        """
        raise NotImplementedError("Should have implemented this.")

    def __getitem__(self, item):
        """Equivalent to `f_get_range()[idx]`

        :raises: TypeError if parameter has no range

        """
        return self.f_get_range().__getitem__(item)

    def f_get(self):
        """Returns the current data value of the parameter and locks the parameter.

        :raises: TypeError if the parameter is empty

        Example usage:

        >>> param = Parameter('groupA.groupB.myparam', comment='I am a neat example')
        >>> param.f_set(44.0)
        >>> param.f_get()
        44.0:

        ABSTRACT: Needs to be defined in subclass

        """
        raise NotImplementedError("Should have implemented this.")

    def f_get_default(self):
        """Returns the default value of the parameter and locks it."""
        raise NotImplementedError("Should have implemented this.")

    def f_get_range(self, copy=True):
        """Returns an iterable to iterate over the values of the exploration range.

        Note that the returned values should be either a copy of the exploration range
        unless explicetly requested otherwise.

        :param copy:

            If range should be copied to avoid tempering with data.

        :return: Iterable

        :raises: TypeError if the parameter is not explored

        Example usage:

        >>> param = Parameter('groupA.groupB.myparam',data=22, comment='I am a neat example')
        >>> param._explore([42,43,43])
        >>> param.f_get_range()
        (42,43,44)

        ABSTRACT: Needs to be defined in subclass

        """
        raise NotImplementedError("Should have implemented this.")

    def _explore(self, iterable):
        """The method to explore a parameter and create a range of entries.

        :param iterable: An iterable specifying the exploration range

             For example:

             >>> param = Parameter('groupA.groupB.myparam',data=22.33,\
              comment='I am a neat example')
             >>> param._explore([3.0,2.0,1.0])

        :raises:

            ParameterLockedException: If the parameter is locked

            TypeError: If the parameter is already explored

        ABSTRACT: Needs to be defined in subclass

        """
        raise NotImplementedError("Should have implemented this.")

    def _expand(self, iterable):
        """Similar to :func:`~pypet.parameter.BaseParameter._explore` but appends to
        the exploration range.

        :param iterable: An iterable specifying the exploration range.

        :raises:

            ParameterLockedException: If the parameter is locked

            TypeError: If the parameter did not have a range before

        Example usage:

        >>> param = Parameter('groupA.groupB.myparam', data=3.13, comment='I am a neat example')
        >>> param._explore([3.0,2.0,1.0])
        >>> param._expand([42.0,43.0])
        >>> param.f_get_range()
        (3.0,2.0,1.0,42.0,43.0)

        ABSTRACT: Needs to be defined in subclass

        """
        raise NotImplementedError("Should have implemented this.")

    def _set_parameter_access(self, idx=0):
        """Sets the current value according to the `idx` in the exploration range.

        Prepares the parameter for further usage, and tells it which point in the parameter
        space should be accessed by calls to :func:`~pypet.parameter.BaseParameter.f_get`.

        :param idx: The index within the exploration range.

            If the parameter has no range, the single data value is considered
            regardless of the value of `idx`.
            Raises ValueError if the parameter is explored and `idx>=len(param)`.

        :raises: ValueError:

            If the parameter has a range and `idx` is larger or equal to the
            length of the parameter.

        Example usage:

        >>> param = Parameter('groupA.groupB.myparam',data=22.33, comment='I am a neat example')
        >>> param._explore([42.0,43.0,44.0])
        >>> param._set_parameter_access(idx=1)
        >>> param.f_get()
        43.0

        ABSTRACT: Needs to be defined in subclass

        """
        raise NotImplementedError("Should have implemented this.")

    def f_get_class_name(self):
        """ Returns the name of the class i.e. `return self.__class__.__name__`"""
        return self.__class__.__name__

    def f_is_empty(self):
        """True if no data has been assigned to the parameter.

        Example usage:

        >>> param = Parameter('myname.is.example', comment='I am _empty!')
        >>> param.f_is_empty()
        True
        >>> param.f_set(444)
        >>> param.f_is_empty()
        False

        """
        raise NotImplementedError("Implement this!")

    def _shrink(self):
        """If a parameter is explored, i.e. it has a range, the whole exploration range is deleted.

        Note that this function does not erase data from disk. So if the parameter has
        been stored with a service to disk and is shrunk, it can be restored by loading from disk.

        :raises:

            ParameterLockedException: If the parameter is locked

            TypeError: If the parameter has no range


        ABSTRACT: Needs to be defined in subclass

        """
        raise NotImplementedError("Should have implemented this.")

    def f_empty(self):
        """Erases all data in the parameter.

        Does not erase data from disk. So if the parameter has
        been stored with a service to disk and is emptied,
        it can be restored by loading from disk.

        :raises: ParameterLockedException: If the parameter is locked.

        ABSTRACT: Needs to be defined in subclass

        """
        raise NotImplementedError("Should have implemented this.") class Parameter(BaseParameter):
    """The standard container that handles access to simulation parameters.

    Parameters are simple container objects for data values. They handle single values as well as
    the so called exploration range. An array containing multiple values which are accessed
    one after the other in individual simulation runs.

    Parameter exploration is usually initiated through the trajectory see
    `:func:~pypet.trajectory.Trajectory.f_explore` and
    `:func:~pypet.trajectory.Trajectory.f_expand`.

    To access the parameter's data value one can call the :func:`~pypet.parameter.Parameter.f_get`
    method.

    Parameters support the concept of locking. Once a value of the parameter has been accessed,
    the parameter cannot be changed anymore unless it is explicitly unlocked using
    :func:`~pypet.parameter.Parameter.f_unlock`.
    Locking prevents parameters from being changed during runtime of a simulation.

    Supported data values for the parameter are

    * python natives (int, long, str, bool, float, complex),

    * numpy natives, arrays and matrices of type
      np.int8-64, np.uint8-64, np.float32-64, np.complex, np.str

    * python homogeneous non-nested tuples and lists

    Note that for larger numpy arrays it is recommended to use the
    :class:`~pypet.parameter.ArrayParameter`.


    In case you create a new parameter you can pass the following arguments:

    :param full_name: The full name of the parameter. Grouping can be achieved by using colons.

    :param data:

        A data value that is handled by the parameter. It is checked whether the parameter
        :func:`~pypet.parameter.Parameter.f_supports` the data. If not a TypeError is thrown.
        If the parameter becomes explored, the data value is kept as a default. After
        simulation the default value will be restored.

        The data can be accessed as follows:

        >>> param.f_get()
        42

        Or using
        >>> param.data
        42

        [It is not `v_data` because the data is supposed to be part of the trajectory tree
        or extension of the natural naming scheme and not considered as an
        attribute/variable of the parameter container.]

        To change the data after parameter creation one can call
        :func:`~pypet.parameter.Parameter.f_set`:

        >>> param.f_set(43)
        >>> param.f_get()
        43

    :param comment:

        A useful comment describing the parameter.
        The comment can be changed later on using the 'v_comment' variable.

        >>> param.v_comment = 'Example comment'
        >>> print param.v_comment
        'Example comment'

    :raises: TypeError: If `data` is not supported by the parameter.

    Example usage:

    >>> param = Parameter('traffic.mobiles.ncars',data=42, comment='I am a neat example')

    """

    __slots__ = ("_data", "_default", "_explored_range")

    def __init__(self, full_name, data=None, comment=""):
        super(Parameter, self).__init__(full_name, comment)
        self._data = None

        self._default = (
            None  # The default value, which is the same as _data in the beginning,
        )
        # but it is necessary to keep a reference to it to restore the original value
        # after exploration

        self._explored_range = (
            []
        )  # List that will changed later on if parameter is explored
        self._set_logger()

        if data is not None:
            self.f_set(data)

    def _restore_default(self):
        """Restores the default data that was set with the
        `:func:`~pypet.parameter.Parameter.f_set` method (or at initialisation).

        If the parameter is explored during the runtime of a simulation,
        the actual value of the parameter is changed and taken from the exploration range.
        Calling :func:`~pypet.parameter.Parameter._restore_default` sets the parameter's value
        back to it's original value.

        Example usage:

        >>> param = Parameter('supergroup1.subgroup2.', data=44, comment='Im a comment!')
        >>> param._explore([1,2,3,4])
        >>> param._set_parameter_access(2)
        >>> param.f_get()
        3
        >>> param._restore_default()
        >>> param.f_get()
        44

        """
        self._data = self._default

    @copydoc(BaseParameter.f_is_empty)
    def f_is_empty(self):
        """True if no data has been assigned to the parameter.

        Example usage:

        >>> param = Parameter('myname.is.example', comment='I am _empty!')
        >>> param.f_is_empty()
        True
        >>> param.f_set(444)
        >>> param.f_is_empty()
        False

        """
        return self._data is None

    def __len__(self):
        """Returns length of parameter.

        :return:

            0 if empty

            1 if not explored

            length of range if explored and has range

        """
        if self._data is None:
            return 0
        elif len(self._explored_range) > 0:
            return len(self._explored_range)
        else:
            return 1

    def f_has_range(self):
        """If the parameter has a range.

        Does not have to be `True` if the parameter is explored.
        The range might be removed during pickling to save memory.
        Accordingly, `v_explored` remains `True` whereas `f_has_range` is `False`.

        """
        return len(self._explored_range) > 0

    def __getstate__(self):
        """Returns the actual state of the parameter for pickling.

        If `v_full_copy` is true the exploration range is also pickled, otherwise it is omitted.

        """
        result = super(Parameter, self).__getstate__()

        # If we don't need a full copy of the Parameter (because a single process needs
        # only access to a single point in the parameter space) we can delete the rest
        if not self._full_copy:
            result["_explored_range"] = []

        return result

    # @no_prefix_getattr
    def __getattr__(self, item):
        """Allows to query for `.data` as an attribute"""
        if item == "data":
            return self.f_get()
        elif item == "default":
            return self.f_get_default()
        else:
            raise AttributeError(
                "`%s` object has no attribute `%s`." % (self.f_get_class_name(), item)
            )

    def __getitem__(self, item):
        """Equivalent to `f_get_range[item] if item is integer`

        Allows also ['data'] access which is equivalent to `f_get`

        :raises: TypeError if parameter has no range

        """
        if item == "data":
            return self.f_get()
        elif item == "default" or item == -1:
            return self.f_get_default()
        else:
            return super(Parameter, self).__getitem__(item)

    @copydoc(BaseParameter._set_parameter_access)
    def _set_parameter_access(self, idx=0):
        if idx >= len(self) and self.f_has_range():
            raise ValueError(
                "You try to access data item No. %d in the parameter range, "
                "yet there are only %d potential items." % (idx, len(self))
            )
        elif self.f_has_range():
            self._data = self._explored_range[idx]
        else:
            self._logger.warning(
                "You try to change the access to a parameter range of parameter"
                " `%s`. The parameter has no range, your setting has no"
                " effect."
            )

    def f_supports(self, data):
        """Checks if input data is supported by the parameter."""
        dtype = type(data)
        if dtype is tuple or dtype is list:

            # Parameters cannot handle empty tuples
            if len(data) == 0:
                return False

            old_type = None

            # Check if the data in the tuple is homogeneous
            for item in data:
                if not type(item) in pypetconstants.PARAMETER_SUPPORTED_DATA:
                    return False
                if not old_type is None and old_type != type(item):
                    return False
                old_type = type(item)
            return True

        elif dtype is np.ndarray or dtype is np.matrix:

            if data.size == 0:
                return False  # Parameters cannot handle empty arrays and matrices

            # Numpy has many string types that depend on the length of the string,
            # We allow all of them
            dtype = data.dtype
            if np.issubdtype(dtype, np.str):
                dtype = np.str

        return dtype in pypetconstants.PARAMETER_SUPPORTED_DATA

    def _values_of_same_type(self, val1, val2):
        """Checks if two values agree in type.

        Raises a TypeError if both values are not supported by the parameter.
        Returns false if only one of the two values is supported by the parameter.

        Example usage:

        >>>param._values_of_same_type(42,43)
        True

        >>>param._values_of_same_type(42,'43')
        False

        :raises: TypeError

        """
        if self.f_supports(val1) != self.f_supports(val2):
            return False

        if not self.f_supports(val1) and not self.f_supports(val2):
            raise TypeError(
                "I do not support the types of both inputs (`%s` and `%s`),"
                " therefore I cannot judge whether the two are of same type."
                % str(type(val1)),
                str(type(val2)),
            )

        if not type(val1) is type(val2):
            return False

        # Numpy arrays must agree in data type and shape
        if type(val1) is np.array:
            if not val1.dtype is val2.dtype:
                return False

            if not np.shape(val1) == np.shape(val2):
                return False

        # For tuples we now from earlier checks that the data is homogeneous.
        # Thus, only the type of the first item and the length must agree.
        if type(val1) is tuple:
            return (type(val1[0]) is type(val2[0])) and (len(val1) == len(val2))

        return True

    @copydoc(BaseParameter.f_set)
    def f_set(self, data):

        if self.v_locked:
            raise pex.ParameterLockedException(
                "Parameter `%s` is locked!" % self._full_name
            )

        if self.f_has_range():
            raise AttributeError(
                "Your Parameter is an explored array can no longer change values!"
            )

        if self.v_stored:
            self._logger.debug(
                "You are changing an already stored parameter. If "
                "you not explicitly overwrite the data on disk, this change "
                "might be lost and not propagated to disk."
            )

        if not self.f_supports(data):
            raise TypeError(
                "Unsupported data `%s` of type `%s`. "
                "If you passed a tuple or list, this error might also be caused "
                "by heterogeneous data within the "
                "tuple or list." % (str(data), str(type(data)))
            )

        self._data = data
        self._default = self._data

    @copydoc(BaseParameter.f_get_default)
    def f_get_default(self):
        if self.f_is_empty():
            raise TypeError(
                "Parameter `%s` is empty cannot access data" % self.v_full_name
            )

        self.f_lock()  # As soon as someone accesses an entry the parameter gets locked
        return self._default

    def f_get_range(self, copy=True):
        """Returns a python iterable containing the exploration range.

        :param copy:

            If the range should be copied before handed over to avoid tempering with data

        Example usage:

        >>> param = Parameter('groupA.groupB.myparam',data=22, comment='I am a neat example')
        >>> param._explore([42,43,43])
        >>> param.f_get_range()
        (42,43,44)

        :raises: TypeError: If parameter is not explored.

        """
        if not self.f_has_range():
            raise TypeError(
                "Your parameter `%s` is not array, so cannot return array."
                % self.v_full_name
            )
        elif copy:
            return self._explored_range[:]
        else:
            return self._explored_range

    def _explore(self, explore_iterable):
        """Explores the parameter according to the iterable.

        Raises ParameterLockedException if the parameter is locked.
        Raises TypeError if the parameter does not support the data,
        the types of the data in the iterable are not the same as the type of the default value,
        or the parameter has already an exploration range.

        Note that the parameter will iterate over the whole iterable once and store
        the individual data values into a tuple. Thus, the whole exploration range is
        explicitly stored in memory.

        :param explore_iterable: An iterable specifying the exploration range

        For example:

        >>> param._explore([3.0,2.0,1.0])
        >>> param.f_get_range()
        (3.0, 2.0, 1.0)

        :raises TypeError,ParameterLockedException

        """
        if self.v_locked:
            raise pex.ParameterLockedException(
                "Parameter `%s` is locked!" % self.v_full_name
            )

        if self.f_has_range():
            raise TypeError(
                "Your parameter `%s` is already explored, "
                "cannot _explore it further!" % self._name
            )

        if self._data is None:
            raise TypeError(
                "Your parameter `%s` has no default value, please specify one "
                "via `f_set` before exploration. " % self.v_full_name
            )

        data_list = self._data_sanity_checks(explore_iterable)

        self._explored_range = data_list
        self._explored = True
        self.f_lock()

    def _expand(self, explore_iterable):
        """Explores the parameter according to the iterable and appends to the exploration range.

        Raises ParameterLockedException if the parameter is locked.
        Raises TypeError if the parameter does not support the data,
        the types of the data in the iterable are not the same as the type of the default value,
        or the parameter did not have an array before.

        Note that the parameter will iterate over the whole iterable once and store
        the individual data values into a tuple. Thus, the whole exploration range is
        explicitly stored in memory.

        :param explore_iterable: An iterable specifying the exploration range

         For example:

         >>> param = Parameter('Im.an.example', data=33.33, comment='Wooohoo!')
         >>> param._explore([3.0,2.0,1.0])
         >>> param._expand([42.0, 43.42])
         >>> param.f_get_range()
         >>> [3.0, 2.0, 1.0, 42.0, 43.42]

        :raises TypeError, ParameterLockedException

        """
        if self.v_locked:
            raise pex.ParameterLockedException(
                "Parameter `%s` is locked!" % self.v_full_name
            )

        if not self.f_has_range():
            raise TypeError(
                "Your Parameter `%s` is not an array and can therefore "
                "not be expanded." % self._name
            )

        data_list = self._data_sanity_checks(explore_iterable)

        self._explored_range.extend(data_list)
        self.f_lock()

    def _data_sanity_checks(self, explore_iterable):
        """Checks if data values are  valid.

        Checks if the data values are supported by the parameter and if the values are of the same
        type as the default value.

        """
        data_list = []

        for val in explore_iterable:

            if not self.f_supports(val):
                raise TypeError(
                    "%s is of not supported type %s." % (repr(val), str(type(val)))
                )

            if not self._values_of_same_type(val, self._default):
                raise TypeError(
                    "Data of `%s` is not of the same type as the original entry value, "
                    "new type is %s vs old type %s."
                    % (self.v_full_name, str(type(val)), str(type(self._default)))
                )

            data_list.append(val)

        if len(data_list) == 0:
            raise ValueError("Cannot explore an empty list!")

        return data_list

    def _store(self):
        """Returns a dictionary of formatted data understood by the storage service.

        The data is put into an :class:`~pypet.parameter.ObjectTable` named 'data'.
        If the parameter is explored, the exploration range is also put into another table
        named 'explored_data'.

        :return: Dictionary containing the data and optionally the exploration range.

        """
        if self._data is not None:
            store_dict = {"data": ObjectTable(data={"data": [self._data]})}

        if self.f_has_range():
            store_dict["explored_data"] = ObjectTable(
                data={"data": self._explored_range}
            )

        self._locked = True

        return store_dict

    def _load(self, load_dict):
        """Loads the data and exploration range from the `load_dict`.

        The `load_dict` needs to be in the same format as the result of the
        :func:`~pypet.parameter.Parameter._store` method.

        """
        if self.v_locked:
            raise pex.ParameterLockedException(
                "Parameter `%s` is locked!" % self.v_full_name
            )

        if "data" in load_dict:
            self._data = load_dict["data"]["data"][0]
            self._default = self._data
        else:
            self._logger.warning(
                "Your parameter `%s` is empty, "
                "I did not find any data on disk." % self.v_full_name
            )

        if "explored_data" in load_dict:
            self._explored_range = [
                x for x in load_dict["explored_data"]["data"].tolist()
            ]
            self._explored = True

        self._locked = True

    @copydoc(BaseParameter.f_get)
    def f_get(self):
        if self.f_is_empty():
            raise TypeError(
                "Parameter `%s` is empty cannot access data" % self.v_full_name
            )

        self.f_lock()  # As soon as someone accesses an entry the parameter gets locked
        return self._data

    @copydoc(BaseParameter._shrink)
    def _shrink(self):

        if self.v_locked:
            raise pex.ParameterLockedException(
                "Parameter %s is locked!" % self.v_full_name
            )

        if not self.f_has_range():
            raise TypeError("Cannot shrink Parameter without a range.")

        if self.f_is_empty():
            raise TypeError("Cannot shrink empty Parameter.")

        del self._explored_range
        self._explored_range = []
        self._explored = False

    @copydoc(BaseParameter.f_empty)
    def f_empty(self):

        if self.v_locked:
            raise pex.ParameterLockedException(
                "Parameter %s is locked!" % self.v_full_name
            )

        if self.f_has_range():
            self._shrink()

        del self._data
        del self._default
        self._data = None
        self._default = None
        self._explored = False class Result(BaseResult):
    """Light Container that stores basic python and numpy data.

    Note that no sanity checks on individual data is made (only on outer data structure)
    and you have to take care, that your data is understood by the storage service.
    It is assumed that results tend to be large and therefore sanity checks would be too expensive.

    Data that can safely be stored into a Result are:

        *   python natives (int, long, str, bool, float, complex),

        *
            numpy natives, arrays and matrices of type
            np.int8-64, np.uint8-64, np.float32-64, np.complex, np.str

        *
            python lists and tuples of the previous types
            (python natives + numpy natives and arrays)
            Lists and tuples are not allowed to be nested and must be
            homogeneous, i.e. only contain data of one particular type.
            Only integers, or only floats, etc.

        *
            python dictionaries of the previous types (not nested!), data can be
            heterogeneous, keys must be strings. For example, one key-value pair
            of string and int and one key-value pair of string and float, and so
            on.

        *   pandas DataFrames_

        *   :class:`~pypet.parameter.ObjectTable`

    .. _DataFrames: http://pandas.pydata.org/pandas-docs/dev/dsintro.html#dataframe

    Note that containers should NOT be empty (like empty dicts or lists) at the time
    they are saved to disk. The standard HDF5 storage service cannot store empty containers!
    The Result emits a warning if you hand over an empty container.

    Data is set on initialisation or with :func:`~pypet.parameter.Result.f_set`

    Example usage:

    >>> res = Result('supergroup.subgroup.myresult', comment='I am a neat example!' \
        [1000,2000], {'a':'b','c':333}, hitchhiker='Arthur Dent')


    In case you create a new result you can pass the following arguments:

    :param fullanme: The fullname of the result, grouping can be achieved by colons,

    :param comment:

        A useful comment describing the result.
        The comment can later on be changed using the `v_comment` variable

        >>> param.v_comment
        'I am a neat example!'

    :param args:

        Data that is handled by the result.
        The first positional argument is stored with the name of the result.
        Following arguments are stored with `name_X` where `X` is the position
        of the argument.

    :param kwargs:

        Data that is handled by the result, it is kept by the result under the names
        specified by the keys of kwargs.

        >>> res.f_get(0)
        [1000,2000]
        >>> res.f_get(1)
        {'a':'b','c':'d'}
        >>> res.f_get('myresult')
        [1000,2000]
        >>> res.f_get('hitchhiker')
        'ArthurDent'
        >>> res.f_get('myresult','hitchhiker')
        ([1000,2000], 'ArthurDent')

        Can be changed or more can be added via :func:`~pypet.parameter.Result.f_set`

        >>> result.f_set('Uno',x='y')
        >>> result.f_get(0)
        'Uno'
        >>> result.f_get('x')
        'y'


        Alternative method to put and retrieve data from the result container is via
        ``__getattr__`` and ``__setattr__``:

        >>> res.ford = 'prefect'
        >>> res.ford
        'prefect'

    :raises: TypeError:

        If the data format in args or kwargs is not known to the result. Checks type of
        outer data structure, i.e. checks if you have a list or dictionary.
        But it does not check on individual values within dicts or lists.

    """

    __slots__ = ("_data_",)

    SUPPORTED_DATA = set(
        (np.ndarray, ObjectTable, DataFrame, Series, dict, tuple, list, np.matrix)
        + pypetconstants.PARAMETER_SUPPORTED_DATA
    )

    def __init__(self, full_name, *args, **kwargs):
        comment = kwargs.pop("comment", "")
        super(Result, self).__init__(full_name, comment)
        self._data_ = None
        self._set_logger()
        self.f_set(*args, **kwargs)

    @property
    def _data(self):
        """To avoid the overhead of producing an empty dictionary"""
        if self._data_ is None:
            self._data_ = {}
        return self._data_

    def __dir__(self):
        """Adds all data to auto-completion"""
        result = super(Result, self).__dir__()
        if self._data_ is not None:
            result.extend(self._data.keys())
        return result

    def f_translate_key(self, key):
        """Translates integer indices into the appropriate names"""
        if isinstance(key, int):
            if key == 0:
                key = self.v_name
            else:
                key = self.v_name + "_%d" % key
        return key

    def __contains__(self, key):
        key = self.f_translate_key(key)
        return key in self._data

    def f_val_to_str(self):
        """Summarizes data handled by the result as a string.

        Calls `__repr__` on all handled data. Data is NOT ordered.

        Truncates the string if it is longer than
        :const:`pypetconstants.HDF5_STRCOL_MAX_VALUE_LENGTH`

        :return: string

        """

        resstrlist = []
        strlen = 0

        for key in self._data:
            val = self._data[key]
            resstr = "%s=%s, " % (key, repr(val))
            resstrlist.append(resstr)

            strlen += len(resstr)
            if strlen > pypetconstants.HDF5_STRCOL_MAX_VALUE_LENGTH:
                break

        return_string = "".join(resstrlist)
        if len(return_string) > pypetconstants.HDF5_STRCOL_MAX_VALUE_LENGTH:
            return_string = (
                return_string[0 : pypetconstants.HDF5_STRCOL_MAX_VALUE_LENGTH - 3]
                + "..."
            )
        else:
            return_string = return_string[0:-2]  # Delete the last `, `

        return return_string

    def __repr__(self):
        return "<%s>" % self.__str__()

    def __str__(self):
        """String representation of the result.

        Output format is '<class_name> name (`comment`): value_string'

        The `value_string` is obtained from :func:`~pypet.parameter.Result.f_val_to_str`.

        If the comment is the empty string, the comment is omitted.

        """
        datastr = self.f_val_to_str()
        return_string = "%s %s" % (self.f_get_class_name(), self.v_full_name)
        if self.v_comment:
            return_string += " (`%s`)" % self.v_comment
        if datastr:
            return_string += ": " + datastr

        return return_string

    def f_to_dict(self, copy=True):
        """Returns all handled data as a dictionary.

        :param copy:

            Whether the original dictionary or a shallow copy is returned.

        :return: Data dictionary

        """
        if copy:
            return self._data.copy()
        else:
            return self._data

    def f_is_empty(self):
        """True if no data has been put into the result.

        Also True if all data has been erased via :func:`~pypet.parameter.Result.f_empty`.

        """
        return len(self._data) == 0

    @copydoc(BaseResult.f_empty)
    def f_empty(self):
        self._data_ = None

    def f_set(self, *args, **kwargs):
        """Method to put data into the result.

        :param args:

            The first positional argument is stored with the name of the result.
            Following arguments are stored with `name_X` where `X` is the position
            of the argument.

        :param kwargs: Arguments are stored with the key as name.

        :raises: TypeError if outer data structure is not understood.

        Example usage:

        >>> res = Result('supergroup.subgroup.myresult', comment='I am a neat example!')
        >>> res.f_set(333,42.0, mystring='String!')
        >>> res.f_get('myresult')
        333
        >>> res.f_get('myresult_1')
        42.0
        >>> res.f_get(1)
        42.0
        >>> res.f_get('mystring')
        'String!'

        """
        if args and self.v_name is None:
            raise AttributeError(
                "Cannot set positional value because I do not have a name!"
            )
        for idx, arg in enumerate(args):
            valstr = self.f_translate_key(idx)
            self.f_set_single(valstr, arg)

        for key, arg in kwargs.items():
            self.f_set_single(key, arg)

    def __getitem__(self, name):
        """ Equivalent to calling `f_get`"""
        return self.f_get(name)

    def __setitem__(self, key, value):
        """Almost equivalent to calling __setattr__.

        Treats integer values as `f_get`.

        """
        key = self.f_translate_key(key)

        setattr(self, key, value)

    def __iter__(self):
        """Equivalent to iterating over the keys of the data dictionary."""
        return self._data.__iter__()

    def f_get(self, *args):
        """Returns items handled by the result.

         If only a single name is given, a single data item is returned. If several names are
         given, a list is returned. For integer inputs the result returns `resultname_X`.

         If the result contains only a single entry you can call `f_get()` without arguments.
         If you call `f_get()` and the result contains more than one element a ValueError is
         thrown.

         If the requested item(s) cannot be found an AttributeError is thrown.

        :param args: strings-names or integers

        :return: Single data item or tuple of data

        Example:

        >>> res = Result('supergroup.subgroup.myresult', comment='I am a neat example!' \
        [1000,2000], {'a':'b','c':333}, hitchhiker='Arthur Dent')
        >>> res.f_get('hitchhiker')
        'Arthur Dent'
        >>> res.f_get(0)
        [1000,2000]
        >>> res.f_get('hitchhiker', 'myresult')
        ('Arthur Dent', [1000,2000])

        """

        if len(args) == 0:
            if len(self._data) == 1:
                return list(self._data.values())[0]
            elif len(self._data) > 1:
                raise ValueError(
                    "Your result `%s` contains more than one entry: "
                    "`%s` Please use >>f_get<< with one of these."
                    % (self.v_full_name, str(list(self._data.keys())))
                )
            else:
                raise AttributeError(
                    "Your result `%s` is empty, cannot access data." % self.v_full_name
                )

        result_list = []
        for name in args:
            name = self.f_translate_key(name)
            if not name in self._data:
                if name == "data" and len(self._data) == 1:
                    return self._data[list(self._data.keys())[0]]
                else:
                    raise AttributeError(
                        "`%s` is not part of your result `%s`."
                        % (name, self.v_full_name)
                    )

            result_list.append(self._data[name])

        if len(args) == 1:
            return result_list[0]
        else:
            return result_list

    def f_set_single(self, name, item):
        """Sets a single data item of the result.

        Raises TypeError if the type of the outer data structure is not understood.
        Note that the type check is shallow. For example, if the data item is a list,
        the individual list elements are NOT checked whether their types are appropriate.

        :param name: The name of the data item

        :param item: The data item

        :raises: TypeError

        Example usage:

        >>> res.f_set_single('answer', 42)
        >>> res.f_get('answer')
        42

        """

        if self.v_stored:
            self._logger.debug(
                "You are changing an already stored result. If "
                "you not explicitly overwrite the data on disk, this change "
                "might be lost and not propagated to disk."
            )

        if self._supports(item):

            # self._check_if_empty(item, name) # No longer needed

            if name in self._data:
                self._logger.debug(
                    "Replacing `%s` in result `%s`." % (name, self.v_full_name)
                )

            self._data[name] = item
        else:
            raise TypeError(
                "Your result `%s` of type `%s` is not supported."
                % (name, str(type(item)))
            )

    def _supports(self, item):
        """Checks if outer data structure is supported."""
        return type(item) in Result.SUPPORTED_DATA

    def f_supports_fast_access(self):
        """Whether or not the result supports fast access.

        A result supports fast access if it contains exactly one item with the name of the result.

        """
        return len(self._data) == 1 and self.v_name in self._data

    def _store(self):
        """Returns a storage dictionary understood by the storage service.

        Simply returns a shallow copy of its own data dictionary.

        """
        store_dict = {}
        store_dict.update(self._data)
        return store_dict

    def _load(self, load_dict):
        """Loads data from load_dict"""
        self._data_ = load_dict

    def f_remove(self, *args):
        """Removes `*args` from the result"""
        for arg in args:
            arg = self.f_translate_key(arg)
            if arg in self._data:
                del self._data[arg]
            else:
                raise AttributeError(
                    "Your result `%s` does not contain %s." % (self.name_, arg)
                )

    def __delitem__(self, key):
        """ Deletes an item, see also __delattr__"""
        self.f_remove(key)

    def __delattr__(self, item):
        """Deletes an item from the result.

        If the item has been stored to disk before with a storage service, this storage is not
        deleted!

        :param item: Name of item to delete

        :raises: AttributeError if the item does not exist

        Example usage:

        >>> res = Result('Iam.an.example', comment='And a neat one, indeed!', fortytwo=42)
        >>> 'fortytwo' in res
        True
        >>> del res.fortytwo
        >>> 'fortytwo' in res
        False

        """
        if item[0] == "_":
            super(Result, self).__delattr__(item)
        else:
            self.f_remove(item)

    def __setattr__(self, key, value):
        if key[0] == "_":
            # We set a private attribute
            super(Result, self).__setattr__(key, value)
        elif hasattr(self.__class__, key):
            # Work around for python properties
            python_property = getattr(self.__class__, key)
            if python_property.fset is None:
                raise AttributeError("%s is read only!" % key)
            else:
                python_property.fset(self, value)
        else:
            self.f_set_single(key, value)

    # @no_prefix_getattr
    def __getattr__(self, name):
        return self.f_get(name) def kwargs_api_change(old_name, new_name=None):
    """This is a decorator which can be used if a kwarg has changed
    its name over versions to also support the old argument name.

    Issues a warning if the old keyword argument is detected and
    converts call to new API.

    :param old_name:

        Old name of the keyword argument

    :param new_name:

        New name of keyword argument

    """

    def wrapper(func):
        @functools.wraps(func)
        def new_func(*args, **kwargs):

            if old_name in kwargs:
                if new_name is None:
                    warning_string = 'Using deprecated keyword argument `%s` in function `%s`. ' \
                                 'This keyword is no longer supported, please don`t use it ' \
                                 'anymore.' % (old_name, func.__name__)
                else:
                    warning_string = 'Using deprecated keyword argument `%s` in function `%s`, ' \
                                 'please use keyword `%s` instead.' % \
                                 (old_name, func.__name__, new_name)
                warnings.warn(warning_string, category=DeprecationWarning)
                value = kwargs.pop(old_name)
                if new_name is not None:
                    kwargs[new_name] = value

            return func(*args, **kwargs)

        return new_func

    return wrapper def not_in_run(func):
    """This is a decorator that signaling that a function is not available during a single run.

    """
    doc = func.__doc__
    na_string = '''\nATTENTION: This function is not available during a single run!\n'''

    if doc is not None:
        func.__doc__ = '\n'.join([doc, na_string])
    func._not_in_run = True

    @functools.wraps(func)
    def new_func(self, *args, **kwargs):

        if self._is_run:
            raise TypeError('Function `%s` is not available during a single run.' %
                            func.__name__)

        return func(self, *args, **kwargs)

    return new_func def copydoc(fromfunc, sep="\n"):
    """Decorator: Copy the docstring of `fromfunc`

    If the doc contains a line with the keyword `ABSTRACT`,
    like `ABSTRACT: Needs to be defined in subclass`, this line and the line after are removed.

    """

    def _decorator(func):
        sourcedoc = fromfunc.__doc__

        # Remove the ABSTRACT line:
        split_doc = sourcedoc.split('\n')
        split_doc_no_abstract = [line for line in split_doc if not 'ABSTRACT' in line]

        # If the length is different we have found an ABSTRACT line
        # Finally we want to remove the final blank line, otherwise
        # we would have three blank lines at the end
        if len(split_doc) != len(split_doc_no_abstract):
            sourcedoc = '\n'.join(split_doc_no_abstract[:-1])

        if func.__doc__ is None:
            func.__doc__ = sourcedoc
        else:
            func.__doc__ = sep.join([sourcedoc, func.__doc__])
        return func

    return _decorator def deprecated(msg=''):
    """This is a decorator which can be used to mark functions
    as deprecated. It will result in a warning being emitted
    when the function is used.

    :param msg:

        Additional message added to the warning.

    """

    def wrapper(func):
        @functools.wraps(func)
        def new_func(*args, **kwargs):
            warning_string = "Call to deprecated function or property `%s`." % func.__name__
            warning_string = warning_string + ' ' + msg
            warnings.warn(
                warning_string,
                category=DeprecationWarning,
            )
            return func(*args, **kwargs)

        return new_func

    return wrapper def kwargs_mutual_exclusive(param1_name, param2_name, map2to1=None):
    """ If there exist mutually exclusive parameters checks for them and maps param2 to 1."""
    def wrapper(func):
        @functools.wraps(func)
        def new_func(*args, **kwargs):
            if param2_name in kwargs:
                if param1_name in kwargs:
                    raise ValueError('You cannot specify `%s` and `%s` at the same time, '
                                     'they are mutually exclusive.' % (param1_name, param2_name))
                param2 = kwargs.pop(param2_name)
                if map2to1 is not None:
                    param1 = map2to1(param2)
                else:
                    param1 = param2
                kwargs[param1_name] = param1

            return func(*args, **kwargs)

        return new_func

    return wrapper def manual_run(turn_into_run=True, store_meta_data=True, clean_up=True):
    """Can be used to decorate a function as a manual run function.

    This can be helpful if you want the run functionality without using an environment.

    :param turn_into_run:

        If the trajectory should become a `single run` with more specialized functionality
        during a single run.

    :param store_meta_data:

        If meta-data like runtime should be automatically stored

    :param clean_up:

        If all data added during the single run should be removed, only works
        if ``turn_into_run=True``.

    """
    def wrapper(func):
        @functools.wraps(func)
        def new_func(traj, *args, **kwargs):
            do_wrap = not traj._run_by_environment
            if do_wrap:
                traj.f_start_run(turn_into_run=turn_into_run)
            result = func(traj, *args, **kwargs)
            if do_wrap:
                traj.f_finalize_run(store_meta_data=store_meta_data,
                                    clean_up=clean_up)
            return result

        return new_func

    return wrapper def is_debug():
    """Checks if user is currently debugging.

    Debugging is checked via ``'pydevd' in sys.modules``.

    :return: True of False

    """
    return 'pydevd' in sys.modules def format_time(timestamp):
    """Formats timestamp to human readable format"""
    format_string = '%Y_%m_%d_%Hh%Mm%Ss'
    formatted_time = datetime.datetime.fromtimestamp(timestamp).strftime(format_string)
    return formatted_time def storage_factory(storage_service, trajectory=None, **kwargs):
    """Creates a storage service, to be extended if new storage services are added

    :param storage_service:

        Storage Service instance of constructor or a string pointing to a file

    :param trajectory:

        A trajectory instance

    :param kwargs:

        Arguments passed to the storage service

    :return:

        A storage service and a set of not used keyword arguments from kwargs

    """

    if 'filename' in kwargs and storage_service is None:
        filename = kwargs['filename']
        _, ext = os.path.splitext(filename)
        if ext in ('.hdf', '.h4', '.hdf4', '.he2', '.h5', '.hdf5', '.he5'):
            storage_service = HDF5StorageService
        else:
            raise ValueError('Extension `%s` of filename `%s` not understood.' %
                             (ext, filename))
    elif isinstance(storage_service, str):
        class_name = storage_service.split('.')[-1]
        storage_service = create_class(class_name, [storage_service, HDF5StorageService])

    if inspect.isclass(storage_service):
        return _create_storage(storage_service, trajectory, **kwargs)
    else:
        return storage_service, set(kwargs.keys())
import datetime
import time
import hashlib
import itertools as itools
import inspect
import sys
import copy as cp
import pypet.pypetexceptions as pex
import pypet.pypetconstants as pypetconstants
import pypet.storageservice as storage
import pypet.utils.dynamicimports as dynamicimports
from collections import OrderedDict
from pypet._version import __version__ as VERSION
from pypet.naturalnaming import NaturalNamingInterface, ResultGroup, ParameterGroup, \
    DerivedParameterGroup, ConfigGroup, STORE, LOAD, REMOVE
from pypet.parameter import BaseParameter, Parameter, Result
from pypet.utils.decorators import kwargs_api_change, not_in_run, copydoc, deprecated,\
    kwargs_mutual_exclusive, manual_run
from pypet.utils.helpful_functions import is_debug, format_time
from pypet.utils.storagefactory import storage_factory
                for jrun in range(len(self)):
                    change = True

                    # Check all marked parameters
                    for my_param, other_param in params_to_change.values():
                        if other_param.f_has_range():
                            other_param._set_parameter_access(irun)

                        if my_param.f_has_range():
                            my_param._set_parameter_access(jrun)

                        val1 = my_param.f_get()
                        val2 = other_param.f_get()

                        # If only one parameter differs, the parameter space point differs
                        # and we can skip the rest of the parameters
                        if not my_param._equal_values(val1, val2):
                            change = False
                            break

                    # If we found one parameter space point in the current trajectory
                    # that matches the ith point in the other, we do not need the ith
                    # point. We can also skip comparing to the rest of the points in the
                    # current trajectory
                    if change:
                        del used_runs[irun]
                        break

            # Restore changed default values
            for my_param, other_param in params_to_change.values():
                other_param._restore_default()
                my_param._restore_default()


        # Merge parameters into the current trajectory
        adding_length = len(used_runs)
        starting_length = len(self)
        if adding_length == 0:
            return used_runs, []
        count = 0
        for key in sorted(used_runs.keys()):
            used_runs[key] = starting_length + count
            count += 1

        for my_param, other_param in params_to_change.values():
            fullname = my_param.v_full_name

            # We need new ranges to enlarge all parameters marked for merging
            if fullname == trial_parameter_name:
                # The trial parameter now has to cover the range 0 to T1+T2+1
                other_range = [x + mymaxtrial_T1 + 1 for x in other_trial_list]
            else:
                # In case we do not use all runs we need to filter the ranges of the
                # parameters of the other trajectory
                if other_param.f_has_range():
                    other_range = (x for jdx, x in enumerate(other_param.f_get_range(copy=False))
                                                             if jdx in used_runs)
                else:
                    other_range = (other_param.f_get() for _ in range(adding_length))

            # If a parameter in the current trajectory was marked for merging but was not
            # explored before, we need to explore it first, simply by creating the range of
            # the current trajectory's length containing only it's default value
            if not my_param.f_has_range():
                my_param.f_unlock()
                my_param._explore((my_param.f_get() for _ in  range(len(self))))

            # After determining the new range extension `other_range`,
            # expand the parameters
            my_param.f_unlock()
            my_param._expand(other_range)

            if not fullname in self._explored_parameters:
                self._explored_parameters[fullname] = my_param

        return used_runs, list(params_to_change.keys())

    @not_in_run
    def f_migrate(self, new_name=None, in_store=False,
                  new_storage_service=None, **kwargs):
        """Can be called to rename and relocate the trajectory.

        :param new_name: New name of the trajectory, None if you do not want to change the name.

        :param in_store:

            Set this to True if the trajectory has been stored with the new name at the new
            file before  and you just want to "switch back" to the location.
            If you migrate to a store used before and you do not set `in_store=True`,
            the storage service will throw a RuntimeError in case you store the Trajectory
            because it will assume that you try to store a new trajectory that accidentally has
            the very same name as another trajectory. If set to `True` and trajectory is not found
            in the file, the trajectory is simply stored to the file.

        :param new_storage_service:

            New service where you want to migrate to. Leave none if you want to keep the olde one.

        :param kwargs:

            Additional keyword arguments passed to the service.
            For instance, to change the file of the trajectory use ``filename='my_new_file.hdf5``.

        """
        if new_name is not None:
            self._name = new_name

        unused_kwargs = set(kwargs.keys())
        if new_storage_service is not None or len(kwargs) > 0:
            self._storage_service, unused_kwargs = storage_factory(
                                                    storage_service=new_storage_service,
                                                    trajectory=self, **kwargs)
        if len(unused_kwargs) > 0:
            raise ValueError('The following keyword arguments were not used: `%s`' %
                             str(unused_kwargs))

        self._stored = in_store

    @kwargs_api_change('new_filename')
    @kwargs_api_change('new_name')

    @kwargs_mutual_exclusive('only_init', 'recursive', lambda x: not x)