def _trj_store_explorations(self, traj):
        """Stores a all explored parameter names for internal recall"""
        nexplored = len(traj._explored_parameters)
        if nexplored > 0:
            if hasattr(self._overview_group, 'explorations'):
                explorations_table = self._overview_group._f_get_child('explorations')
                if len(explorations_table) != nexplored:
                    self._hdf5file.remove_node(where=self._overview_group,
                                         name='explorations')
        if not hasattr(self._overview_group, 'explorations'):
            explored_list = list(traj._explored_parameters.keys())
            if explored_list:
                string_col = self._all_get_table_col('explorations',
                                                      explored_list,
                                                      'overview.explorations')
            else:
                string_col = pt.StringCol(1)
            description = {'explorations': string_col}
            explorations_table = self._hdf5file.create_table(where=self._overview_group,
                                                       name='explorations',
                                                       description=description)
            rows = [(x.encode('utf-8'),) for x in explored_list]
            if rows:
                explorations_table.append(rows)
                explorations_table.flush()

def _trj_store_explorations ( self , traj ) : nexplored = len ( traj . _explored_parameters ) if nexplored > 0 : if hasattr ( self . _overview_group , 'explorations' ) : explorations_table = self . _overview_group . _f_get_child ( 'explorations' ) if len ( explorations_table ) != nexplored : self . _hdf5file . remove_node ( where = self . _overview_group , name = 'explorations' ) if not hasattr ( self . _overview_group , 'explorations' ) : explored_list = list ( traj . _explored_parameters . keys ( ) ) if explored_list : string_col = self . _all_get_table_col ( 'explorations' , explored_list , 'overview.explorations' ) else : string_col = pt . StringCol ( 1 ) description = { 'explorations' : string_col } explorations_table = self . _hdf5file . create_table ( where = self . _overview_group , name = 'explorations' , description = description ) rows = [ ( x . encode ( 'utf-8' ) , ) for x in explored_list ] if rows : explorations_table . append ( rows ) explorations_table . flush ( )


def _srvc_make_overview_tables(self, tables_to_make, traj=None):
        """Creates the overview tables in overview group"""
        for table_name in tables_to_make:
            # Prepare the tables desciptions, depending on which overview table we create
            # we need different columns
            paramdescriptiondict = {}
            expectedrows = 0

            # Every overview table has a name and location column
            paramdescriptiondict['location'] = pt.StringCol(
                pypetconstants.HDF5_STRCOL_MAX_LOCATION_LENGTH,
                pos=0)
            paramdescriptiondict['name'] = pt.StringCol(pypetconstants.HDF5_STRCOL_MAX_NAME_LENGTH,
                                                        pos=1)

            paramdescriptiondict['comment'] = pt.StringCol(
                    pypetconstants.HDF5_STRCOL_MAX_COMMENT_LENGTH)

            paramdescriptiondict['value'] = pt.StringCol(
                    pypetconstants.HDF5_STRCOL_MAX_VALUE_LENGTH, pos=2)

            if table_name == 'config_overview':
                if traj is not None:
                    expectedrows = len(traj._config)

            if table_name == 'parameters_overview':
                if traj is not None:
                    expectedrows = len(traj._parameters)

            if table_name == 'explored_parameters_overview':
                paramdescriptiondict['range'] = pt.StringCol(
                    pypetconstants.HDF5_STRCOL_MAX_RANGE_LENGTH)
                paramdescriptiondict['length'] = pt.IntCol()
                if traj is not None:
                    expectedrows = len(traj._explored_parameters)

            if table_name.endswith('summary'):
                paramdescriptiondict['hexdigest'] = pt.StringCol(64, pos=10)

            # Check if the user provided an estimate of the amount of results per run
            # This can help to speed up storing
            if table_name == 'derived_parameters_overview':

                expectedrows = self._derived_parameters_per_run

                if traj is not None:
                    expectedrows *= len(traj)
                    expectedrows += len(traj._derived_parameters)

            if table_name == 'results_overview':
                expectedrows = self._results_per_run

                if traj is not None:
                    expectedrows *= len(traj)
                    expectedrows += len(traj._results)

            if expectedrows > 0:
                paramtable = self._all_get_or_create_table(where=self._overview_group,
                                                           tablename=table_name,
                                                           description=paramdescriptiondict,
                                                           expectedrows=expectedrows)
            else:
                paramtable = self._all_get_or_create_table(where=self._overview_group,
                                                           tablename=table_name,
                                                           description=paramdescriptiondict)

            paramtable.flush()


def _trj_store_trajectory(self, traj, only_init=False, store_data=pypetconstants.STORE_DATA,
                              max_depth=None):
        """ Stores a trajectory to an hdf5 file

        Stores all groups, parameters and results

        """
        if not only_init:
            self._logger.info('Start storing Trajectory `%s`.' % self._trajectory_name)
        else:
            self._logger.info('Initialising storage or updating meta data of Trajectory `%s`.' %
                              self._trajectory_name)
            store_data = pypetconstants.STORE_NOTHING

        # In case we accidentally chose a trajectory name that already exist
        # We do not want to mess up the stored trajectory but raise an Error
        if not traj._stored and self._trajectory_group is not None:
            raise RuntimeError('You want to store a completely new trajectory with name'
                               ' `%s` but this trajectory is already found in file `%s`.'
                               'Did you try to accidentally overwrite existing data? If '
                               'you DO want to override existing data, use `overwrite_file=True`.'
                               'Note that this deletes the whole HDF5 file not just the particular '
                               'trajectroy therein! ' %
                               (traj.v_name, self._filename))

        # Extract HDF5 settings from the trajectory
        self._srvc_check_hdf_properties(traj)

        # Store the trajectory for the first time if necessary:
        if self._trajectory_group is None:
            self._trajectory_group = self._hdf5file.create_group(where='/',
                                                           name=self._trajectory_name,
                                                           title=self._trajectory_name,
                                                           filters=self._all_get_filters())
        # Store meta information
        self._trj_store_meta_data(traj)

        # # Store recursively the config subtree
        # self._tree_store_recursively(pypetconstants.LEAF,traj.config,self._trajectory_group)

        if store_data in (pypetconstants.STORE_DATA_SKIPPING,
                          pypetconstants.STORE_DATA,
                          pypetconstants.OVERWRITE_DATA):

            counter = 0
            maximum_display_other = 10
            name_set = set(['parameters', 'config', 'derived_parameters', 'results'])

            for child_name in traj._children:

                if child_name in name_set:
                    self._logger.info('Storing branch `%s`.' % child_name)
                else:
                    if counter < maximum_display_other:
                        self._logger.info('Storing branch/node `%s`.' % child_name)
                    elif counter == maximum_display_other:
                        self._logger.info('To many branches or nodes at root for display. '
                                          'I will not inform you about storing anymore. '
                                          'Branches are stored silently in the background. '
                                          'Do not worry, I will not freeze! Pinky promise!!!')
                    counter += 1

                # Store recursively the elements
                self._tree_store_sub_branch(traj, child_name, store_data=store_data,
                                            with_links=True,
                                            recursive=True, max_depth=max_depth,
                                            hdf5_group=self._trajectory_group)

            self._logger.info('Finished storing Trajectory `%s`.' % self._trajectory_name)
        else:
            self._logger.info('Finished init or meta data update for `%s`.' %
                              self._trajectory_name)
        traj._stored = True


def _tree_store_sub_branch(self, traj_node, branch_name,
                               store_data=pypetconstants.STORE_DATA,
                               with_links=True,
                               recursive=False,
                               max_depth=None,
                               hdf5_group=None):
        """Stores data starting from a node along a branch and starts recursively loading
        all data at end of branch.

        :param traj_node: The node where storing starts

        :param branch_name:

            A branch along which storing progresses. Colon Notation is used:
            'group1.group2.group3' loads 'group1', then 'group2', then 'group3', and then finally
            recursively all children and children's children below 'group3'.

        :param store_data: How data should be stored

        :param with_links: If links should be stored

        :param recursive:

            If the rest of the tree should be recursively stored

        :param max_depth:

            Maximum depth to store

        :param hdf5_group:

            HDF5 node in the file corresponding to `traj_node`

        """
        if store_data == pypetconstants.STORE_NOTHING:
            return

        if max_depth is None:
            max_depth = float('inf')

        if hdf5_group is None:
            # Get parent hdf5 node
            location = traj_node.v_full_name
            hdf5_location = location.replace('.', '/')
            try:
                if location == '':
                    hdf5_group = self._trajectory_group
                else:
                    hdf5_group = self._hdf5file.get_node( where=self._trajectory_group,
                                                         name=hdf5_location)
            except pt.NoSuchNodeError:
                self._logger.debug('Cannot store `%s` the parental hdf5 node with path `%s` does '
                                     'not exist on disk.' %
                                     (traj_node.v_name, hdf5_location))

                if traj_node.v_is_leaf:
                    self._logger.error('Cannot store `%s` the parental hdf5 '
                                       'node with path `%s` does '
                                       'not exist on disk! The child '
                                       'you want to store is a leaf node,'
                                       'that cannot be stored without '
                                       'the parental node existing on '
                                       'disk.' % (traj_node.v_name, hdf5_location))
                    raise
                else:
                    self._logger.debug('I will try to store the path from trajectory root to '
                                         'the child now.')

                    self._tree_store_sub_branch(traj_node._nn_interface._root_instance,
                                                traj_node.v_full_name + '.' + branch_name,
                                                store_data=store_data, with_links=with_links,
                                                recursive=recursive,
                                                max_depth=max_depth + traj_node.v_depth,
                                                hdf5_group=self._trajectory_group)
                    return

        current_depth = 1

        split_names = branch_name.split('.')

        leaf_name = split_names.pop()

        for name in split_names:
            if current_depth > max_depth:
                return
            # Store along a branch
            self._tree_store_nodes_dfs(traj_node, name, store_data=store_data, with_links=with_links,
                                   recursive=False, max_depth=max_depth,
                                   current_depth=current_depth, parent_hdf5_group=hdf5_group)
            current_depth += 1

            traj_node = traj_node._children[name]

            hdf5_group = getattr(hdf5_group, name)

        # Store final group and recursively everything below it
        if current_depth <= max_depth:
            self._tree_store_nodes_dfs(traj_node, leaf_name, store_data=store_data,
                               with_links=with_links, recursive=recursive,
                               max_depth=max_depth, current_depth=current_depth,
                               parent_hdf5_group=hdf5_group)

Stores data starting from a node along a branch and starts recursively loading all data at end of branch .


def _tree_create_leaf(self, name, trajectory, hdf5_group):
        """ Creates a new pypet leaf instance.

        Returns the leaf and if it is an explored parameter the length of the range.

        """
        class_name = self._all_get_from_attrs(hdf5_group, HDF5StorageService.CLASS_NAME)

        # Create the instance with the appropriate constructor
        class_constructor = trajectory._create_class(class_name)

        instance = trajectory._construct_instance(class_constructor, name)

        return instance


def _tree_load_nodes_dfs(self, parent_traj_node, load_data, with_links, recursive,
                         max_depth, current_depth, trajectory, as_new, hdf5_group):
        """Loads a node from hdf5 file and if desired recursively everything below

        :param parent_traj_node: The parent node whose child should be loaded
        :param load_data: How to load the data
        :param with_links: If links should be loaded
        :param recursive: Whether loading recursively below hdf5_group
        :param max_depth: Maximum depth
        :param current_depth: Current depth
        :param trajectory: The trajectory object
        :param as_new: If trajectory is loaded as new
        :param hdf5_group: The hdf5 group containing the child to be loaded

        """
        if max_depth is None:
            max_depth = float('inf')

        loading_list = [(parent_traj_node, current_depth, hdf5_group)]

        while loading_list:
            parent_traj_node, current_depth, hdf5_group = loading_list.pop()

            if isinstance(hdf5_group, pt.link.SoftLink):
                if with_links:
                    # We end up here when auto-loading a soft link
                    self._tree_load_link(parent_traj_node, load_data=load_data, traj=trajectory,
                                         as_new=as_new, hdf5_soft_link=hdf5_group)
                continue

            name = hdf5_group._v_name
            is_leaf = self._all_get_from_attrs(hdf5_group, HDF5StorageService.LEAF)
            in_trajectory = name in parent_traj_node._children

            if is_leaf:
                # In case we have a leaf node, we need to check if we have to create a new
                # parameter or result

                if in_trajectory:
                    instance = parent_traj_node._children[name]
                # Otherwise we need to create a new instance
                else:
                    instance = self._tree_create_leaf(name, trajectory, hdf5_group)

                    # Add the instance to the trajectory tree
                    parent_traj_node._add_leaf_from_storage(args=(instance,), kwargs={})

                self._prm_load_parameter_or_result(instance, load_data=load_data,
                                                   _hdf5_group=hdf5_group)
                if as_new:
                    instance._stored = False

            else:
                if in_trajectory:
                    traj_group = parent_traj_node._children[name]

                    if load_data == pypetconstants.OVERWRITE_DATA:
                        traj_group.v_annotations.f_empty()
                        traj_group.v_comment = ''
                else:
                    if HDF5StorageService.CLASS_NAME in hdf5_group._v_attrs:
                        class_name = self._all_get_from_attrs(hdf5_group,
                                                              HDF5StorageService.CLASS_NAME)
                        class_constructor = trajectory._create_class(class_name)
                        instance = trajectory._construct_instance(class_constructor, name)
                        args = (instance,)
                    else:
                        args = (name,)
                    # If the group does not exist create it'
                    traj_group = parent_traj_node._add_group_from_storage(args=args, kwargs={})

                # Load annotations and comment
                self._grp_load_group(traj_group, load_data=load_data, with_links=with_links,
                                     recursive=False, max_depth=max_depth,
                                     _traj=trajectory, _as_new=as_new,
                                     _hdf5_group=hdf5_group)

                if recursive and current_depth < max_depth:
                    new_depth = current_depth + 1
                    for children in (hdf5_group._v_groups, hdf5_group._v_links):
                        for new_hdf5_group_name in children:
                            new_hdf5_group = children[new_hdf5_group_name]
                            loading_list.append((traj_group, new_depth, new_hdf5_group))


def _tree_store_nodes_dfs(self, parent_traj_node, name, store_data, with_links, recursive,
                          max_depth, current_depth,
                          parent_hdf5_group):
        """Stores a node to hdf5 and if desired stores recursively everything below it.

        :param parent_traj_node: The parental node
        :param name: Name of node to be stored
        :param store_data: How to store data
        :param with_links: If links should be stored
        :param recursive: Whether to store recursively the subtree
        :param max_depth: Maximum recursion depth in tree
        :param current_depth: Current depth
        :param parent_hdf5_group: Parent hdf5 group

        """
        if max_depth is None:
            max_depth = float('inf')

        store_list = [(parent_traj_node, name, current_depth, parent_hdf5_group)]

        while store_list:
            parent_traj_node, name, current_depth, parent_hdf5_group = store_list.pop()

            # Check if we create a link
            if name in parent_traj_node._links:
                if with_links:
                    self._tree_store_link(parent_traj_node, name, parent_hdf5_group)
                continue

            traj_node = parent_traj_node._children[name]

            # If the node does not exist in the hdf5 file create it
            if not hasattr(parent_hdf5_group, name):
                newly_created = True
                new_hdf5_group = self._hdf5file.create_group(where=parent_hdf5_group,
                                                       name=name, filters=self._all_get_filters())
            else:
                newly_created = False
                new_hdf5_group = getattr(parent_hdf5_group, name)

            if traj_node.v_is_leaf:
                self._prm_store_parameter_or_result(traj_node, store_data=store_data,
                                                     _hdf5_group=new_hdf5_group,
                                                    _newly_created=newly_created)

            else:
                self._grp_store_group(traj_node, store_data=store_data, with_links=with_links,
                                      recursive=False, max_depth=max_depth,
                                      _hdf5_group=new_hdf5_group,
                                      _newly_created=newly_created)

                if recursive and current_depth < max_depth:
                    for child in traj_node._children.keys():
                        store_list.append((traj_node, child, current_depth + 1, new_hdf5_group))

Stores a node to hdf5 and if desired stores recursively everything below it .


def _all_store_param_or_result_table_entry(self, instance, table, flags,
                                               additional_info=None):
        """Stores a single row into an overview table

        :param instance: A parameter or result instance

        :param table: Table where row will be inserted

        :param flags:

            Flags how to insert into the table. Potential Flags are
            `ADD_ROW`, `REMOVE_ROW`, `MODIFY_ROW`

        :param additional_info:

            Dictionary containing information that cannot be extracted from
            `instance`, but needs to be inserted, too.

        """
        # assert isinstance(table, pt.Table)

        location = instance.v_location
        name = instance.v_name
        fullname = instance.v_full_name

        if (flags == (HDF5StorageService.ADD_ROW,) and table.nrows < 2
                and 'location' in table.colnames):
            # We add the modify row option here because you cannot delete the very first
            # row of the table, so there is the rare condition, that the row might already
            # exist.
            # We also need to check if 'location' is in the columns in order to avoid
            # confusion with the smaller explored parameter overviews
            flags = (HDF5StorageService.ADD_ROW, HDF5StorageService.MODIFY_ROW)

        if flags == (HDF5StorageService.ADD_ROW,):
            # If we are sure we only want to add a row we do not need to search!
            condvars = None
            condition = None
        else:
            # Condition to search for an entry
            condvars = {'namecol': table.cols.name, 'locationcol': table.cols.location,
                        'name': name, 'location': location}

            condition = """(namecol == name) & (locationcol == location)"""

        if HDF5StorageService.REMOVE_ROW in flags:
            # If we want to remove a row, we don't need to extract information
            insert_dict = {}
        else:
            # Extract information to insert from the instance and the additional info dict
            colnames = set(table.colnames)
            insert_dict = self._all_extract_insert_dict(instance, colnames, additional_info)

        # Write the table entry
        self._all_add_or_modify_row(fullname, insert_dict, table, condition=condition,
                                    condvars=condvars, flags=flags)


def _all_get_or_create_table(self, where, tablename, description, expectedrows=None):
        """Creates a new table, or if the table already exists, returns it."""
        where_node = self._hdf5file.get_node(where)

        if not tablename in where_node:
            if not expectedrows is None:
                table = self._hdf5file.create_table(where=where_node, name=tablename,
                                              description=description, title=tablename,
                                              expectedrows=expectedrows,
                                              filters=self._all_get_filters())
            else:
                table = self._hdf5file.create_table(where=where_node, name=tablename,
                                              description=description, title=tablename,
                                              filters=self._all_get_filters())
        else:
            table = where_node._f_get_child(tablename)

        return table

def _all_get_or_create_table ( self , where , tablename , description , expectedrows = None ) : where_node = self . _hdf5file . get_node ( where ) if not tablename in where_node : if not expectedrows is None : table = self . _hdf5file . create_table ( where = where_node , name = tablename , description = description , title = tablename , expectedrows = expectedrows , filters = self . _all_get_filters ( ) ) else : table = self . _hdf5file . create_table ( where = where_node , name = tablename , description = description , title = tablename , filters = self . _all_get_filters ( ) ) else : table = where_node . _f_get_child ( tablename ) return table

Creates a new table or if the table already exists returns it .


def _all_get_node_by_name(self, name):
        """Returns an HDF5 node by the path specified in `name`"""
        path_name = name.replace('.', '/')
        where = '/%s/%s' % (self._trajectory_name, path_name)
        return self._hdf5file.get_node(where=where)

def _all_get_node_by_name ( self , name ) : path_name = name . replace ( '.' , '/' ) where = '/%s/%s' % ( self . _trajectory_name , path_name ) return self . _hdf5file . get_node ( where = where )

Returns an HDF5 node by the path specified in name


def _all_set_attributes_to_recall_natives(data, ptitem, prefix):
        """Stores original data type to hdf5 node attributes for preserving the data type.

        :param data:

            Data to be stored

        :param ptitem:

            HDF5 node to store data types as attributes. Can also be just a PTItemMock.

        :param prefix:

            String prefix to label and name data in HDF5 attributes

        """

        # If `data` is a container, remember the container type
        if type(data) is tuple:
            HDF5StorageService._all_set_attr(ptitem, prefix + HDF5StorageService.COLL_TYPE,
                                   HDF5StorageService.COLL_TUPLE)

        elif type(data) is list:
            HDF5StorageService._all_set_attr(ptitem, prefix + HDF5StorageService.COLL_TYPE,
                                   HDF5StorageService.COLL_LIST)

        elif type(data) is np.ndarray:
            HDF5StorageService._all_set_attr(ptitem, prefix + HDF5StorageService.COLL_TYPE,
                                   HDF5StorageService.COLL_NDARRAY)

        elif type(data) is np.matrix:
            HDF5StorageService._all_set_attr(ptitem, prefix + HDF5StorageService.COLL_TYPE,
                                   HDF5StorageService.COLL_MATRIX)

        elif type(data) in pypetconstants.PARAMETER_SUPPORTED_DATA:
            HDF5StorageService._all_set_attr(ptitem, prefix + HDF5StorageService.COLL_TYPE,
                                   HDF5StorageService.COLL_SCALAR)

            strtype = type(data).__name__

            if not strtype in pypetconstants.PARAMETERTYPEDICT:
                raise TypeError('I do not know how to handle `%s` its type is `%s`.' %
                                (str(data), repr(type(data))))

            HDF5StorageService._all_set_attr(ptitem, prefix + HDF5StorageService.SCALAR_TYPE, strtype)

        elif type(data) is dict:
            if len(data) > 0:
                HDF5StorageService._all_set_attr(ptitem, prefix + HDF5StorageService.COLL_TYPE,
                                   HDF5StorageService.COLL_DICT)
            else:
                HDF5StorageService._all_set_attr(ptitem, prefix + HDF5StorageService.COLL_TYPE,
                                   HDF5StorageService.COLL_EMPTY_DICT)
        else:
            raise TypeError('I do not know how to handle `%s` its type is `%s`.' %
                            (str(data), repr(type(data))))

        if type(data) in (list, tuple):
            # If data is a list or tuple we need to remember the data type of the elements
            # in the list or tuple.
            # We do NOT need to remember the elements of `dict` explicitly, though.
            # `dict` is stored
            # as an `ObjectTable` and thus types are already conserved.
            if len(data) > 0:
                strtype = type(data[0]).__name__

                if not strtype in pypetconstants.PARAMETERTYPEDICT:
                    raise TypeError('I do not know how to handle `%s` its type is '
                                    '`%s`.' % (str(data), strtype))

                HDF5StorageService._all_set_attr(ptitem, prefix +
                                       HDF5StorageService.SCALAR_TYPE, strtype)
        elif (type(data) in (np.ndarray, np.matrix) and
                  np.issubdtype(data.dtype, str)):
            HDF5StorageService._all_set_attr(ptitem, prefix + HDF5StorageService.SCALAR_TYPE,
                                   str.__name__)

Stores original data type to hdf5 node attributes for preserving the data type .


def _all_recall_native_type(self, data, ptitem, prefix):
        """Checks if loaded data has the type it was stored in. If not converts it.

        :param data: Data item to be checked and converted
        :param ptitem: HDf5 Node or Leaf from where data was loaded
        :param prefix: Prefix for recalling the data type from the hdf5 node attributes

        :return:

            Tuple, first item is the (converted) `data` item, second boolean whether
            item was converted or not.

        """
        typestr = self._all_get_from_attrs(ptitem, prefix + HDF5StorageService.SCALAR_TYPE)
        colltype = self._all_get_from_attrs(ptitem, prefix + HDF5StorageService.COLL_TYPE)
        type_changed = False

        # Check what the original data type was from the hdf5 node attributes
        if colltype == HDF5StorageService.COLL_SCALAR:
            # Here data item was a scalar

            if isinstance(data, np.ndarray):
                # If we recall a numpy scalar, pytables loads a 1d array :-/
                # So we have to change it to a real scalar value
                data = np.array([data])[0]
                type_changed = True

            if not typestr is None:
                # Check if current type and stored type match
                # if not convert the data
                if typestr != type(data).__name__:

                    if typestr == str.__name__:
                        data = data.decode(self._encoding)
                    else:
                        try:
                            data = pypetconstants.PARAMETERTYPEDICT[typestr](data)
                        except KeyError:
                            # For compatibility with files from older pypet versions
                            data = pypetconstants.COMPATPARAMETERTYPEDICT[typestr](data)

                    type_changed = True

        elif (colltype == HDF5StorageService.COLL_TUPLE or
                        colltype == HDF5StorageService.COLL_LIST):
            # Here data item was originally a tuple or a list

            if type(data) is not list and type is not tuple:
                # If the original type cannot be recalled, first convert it to a list
                type_changed = True
                data = list(data)

            if len(data) > 0:
                first_item = data[0]
                # Check if the type of the first item was conserved
                if not typestr == type(first_item).__name__:

                    if not isinstance(data, list):
                        data = list(data)

                    # If type was not conserved we need to convert all items
                    # in the list or tuple
                    for idx, item in enumerate(data):
                        if typestr == str.__name__:
                            data[idx] = data[idx].decode(self._encoding)
                        else:
                            try:
                                data[idx] = pypetconstants.PARAMETERTYPEDICT[typestr](item)
                            except KeyError:
                                # For compatibility with files from older pypet versions:
                                data[idx] = pypetconstants.COMPATPARAMETERTYPEDICT[typestr](item)
                        type_changed = True

            if colltype == HDF5StorageService.COLL_TUPLE:
                # If it was originally a tuple we need to convert it back to tuple
                if type(data) is not tuple:
                    data = tuple(data)
                    type_changed = True
        elif colltype == HDF5StorageService.COLL_EMPTY_DICT:
            data = {}
            type_changed = True
        elif isinstance(data, np.ndarray):

            if typestr == str.__name__:
                data = np.core.defchararray.decode(data, self._encoding)
                type_changed = True

            if colltype == HDF5StorageService.COLL_MATRIX:
                # Here data item was originally a matrix
                data = np.matrix(data)
                type_changed = True

        return data, type_changed

Checks if loaded data has the type it was stored in . If not converts it .


def _all_add_or_modify_row(self, item_name, insert_dict, table, index=None, condition=None,
                               condvars=None,
                               flags=(ADD_ROW, MODIFY_ROW,)):
        """Adds or changes a row in a pytable.

        :param item_name: Name of item, the row is about, only important for throwing errors.

        :param insert_dict:

            Dictionary of data that is about to be inserted into the pytables row.

        :param table:

            The table to insert or modify a row in

        :param index:

            Index of row to be modified. Instead of an index a search condition can be
            used as well, see below.

        :param condition:

            Condition to search for in the table

        :param condvars:

            Variables for the search condition

        :param flags:

            Flags whether to add, modify, or remove a row in the table

        """
        if len(flags) == 0:
            # No flags means no-op
            return

        # You can only specify either an index or a condition not both
        if index is not None and condition is not None:
            raise ValueError('Please give either a condition or an index or none!')
        elif condition is not None:
            row_iterator = table.where(condition, condvars=condvars)
        elif index is not None:
            row_iterator = table.iterrows(index, index + 1)
        else:
            row_iterator = None

        try:
            row = next(row_iterator)
        except TypeError:
            row = None
        except StopIteration:
            row = None

        # multiple_entries = []

        if ((HDF5StorageService.MODIFY_ROW in flags or HDF5StorageService.ADD_ROW in flags) and
                    HDF5StorageService.REMOVE_ROW in flags):
            # You cannot remove and modify or add at the same time
            raise ValueError('You cannot add or modify and remove a row at the same time.')

        if row is None and HDF5StorageService.ADD_ROW in flags:
            # Here we add a new row

            row = table.row

            self._all_insert_into_row(row, insert_dict)

            row.append()

        elif row is not None and HDF5StorageService.MODIFY_ROW in flags:
            # Here we modify an existing row

            self._all_insert_into_row(row, insert_dict)

            row.update()

        elif HDF5StorageService.REMOVE_ROW in flags:
            # Here we delete an existing row

            if row is not None:
                # Only delete if the row does exist otherwise we do not have to do anything
                row_number = row.nrow
                try:
                    table.remove_rows(start=row_number, stop=row_number+1)
                except NotImplementedError:
                    pass
                    # We get here if we try to remove the last row of a table
                    # there is nothing we can do but keep it :-(
        else:
            raise ValueError('Something is wrong, you might not have found '
                             'a row, or your flags are not set appropriately')

        self._all_kill_iterator(row_iterator)
        table.flush()

        if HDF5StorageService.REMOVE_ROW not in flags and row is None:
            raise RuntimeError('Could not add or modify entries of `%s` in '
                               'table %s' % (item_name, table._v_name))


def _all_insert_into_row(self, row, insert_dict):
        """Copies data from `insert_dict` into a pytables `row`."""
        for key, val in insert_dict.items():
            try:
                row[key] = val
            except KeyError as ke:
                self._logger.warning('Could not write `%s` into a table, ' % key + repr(ke))

def
_all_insert_into_row
(
self
,
row
,
insert_dict
)
:
for
key
,
val
in
insert_dict
.
items
(
)
:
try
:
row
[
key
]
=
val
except
KeyError
as
ke
:
self
.
_logger
.
warning
(
'Could not write `%s` into a table, '
%
key
+
repr
(
ke
)
)

Copies data from insert_dict into a pytables row .


def _all_extract_insert_dict(self, item, colnames, additional_info=None):
        """Extracts information from a given item to be stored into a pytable row.

        Items can be a variety of things here, trajectories, single runs, group node,
        parameters, results.

        :param item: Item from which data should be extracted

        :param colnames: Names of the columns in the pytable

        :param additional_info: (dict)

            Additional information that should be stored into the pytable row that cannot be
            read out from `item`.

        :return: Dictionary containing the data to be inserted into a row

        """
        insert_dict = {}

        if 'length' in colnames:
            insert_dict['length'] = len(item)

        if 'comment' in colnames:
            comment = self._all_cut_string(item.v_comment.encode('utf-8'),
                                           pypetconstants.HDF5_STRCOL_MAX_COMMENT_LENGTH,
                                           self._logger)

            insert_dict['comment'] = comment

        if 'location' in colnames:
            insert_dict['location'] = item.v_location.encode('utf-8')

        if 'name' in colnames:
            name = item._name if (not item.v_is_root or not item.v_is_run) else item._crun
            insert_dict['name'] = name.encode('utf-8')

        if 'class_name' in colnames:
            insert_dict['class_name'] = item.f_get_class_name().encode('utf-8')

        if 'value' in colnames:
            insert_dict['value'] = self._all_cut_string(
                item.f_val_to_str().encode('utf-8'),
                pypetconstants.HDF5_STRCOL_MAX_VALUE_LENGTH,
                self._logger)

        if 'hexdigest' in colnames:
            insert_dict['hexdigest'] = additional_info['hexdigest']

        if 'idx' in colnames:
            insert_dict['idx'] = item.v_idx

        if 'time' in colnames:
            time_ = item._time
            insert_dict['time'] = time_.encode('utf-8')

        if 'timestamp' in colnames:
            timestamp = item._timestamp
            insert_dict['timestamp'] = timestamp

        if 'range' in colnames:
            third_length = pypetconstants.HDF5_STRCOL_MAX_RANGE_LENGTH // 3 + 10
            item_range = itools.islice(item.f_get_range(copy=False), 0, third_length)
            range_string = ', '.join([repr(x) for x in item_range])
            insert_dict['range'] = self._all_cut_string(
                range_string.encode('utf-8'),
                pypetconstants.HDF5_STRCOL_MAX_RANGE_LENGTH,
                self._logger)

        # To allow backwards compatibility
        if 'array' in colnames:
            third_length = pypetconstants.HDF5_STRCOL_MAX_RANGE_LENGTH // 3 + 10
            item_range = itools.islice(item.f_get_range(copy=False), 0, third_length)
            range_string = ', '.join([repr(x) for x in item_range])
            insert_dict['array'] = self._all_cut_string(
                range_string.encode('utf-8'),
                pypetconstants.HDF5_STRCOL_MAX_RANGE_LENGTH,
                self._logger)

        if 'version' in colnames:
            insert_dict['version'] = item.v_version.encode('utf-8')

        if 'python' in colnames:
            insert_dict['python'] = item.v_python.encode('utf-8')

        if 'finish_timestamp' in colnames:
            insert_dict['finish_timestamp'] = item._finish_timestamp_run

        return insert_dict

def
_all_extract_insert_dict
(
self
,
item
,
colnames
,
additional_info
=
)
:
insert_dict
=
{
}
if
'length'
in
colnames
:
insert_dict
[
'length'
]
=
len
(
item
)
if
'comment'
in
colnames
:
comment
=
self
.
_all_cut_string
(
item
.
v_comment
.
encode
(
'utf-8'
)
,
pypetconstants
.
HDF5_STRCOL_MAX_COMMENT_LENGTH
,
self
.
_logger
)
insert_dict
[
'comment'
]
=
comment
if
'location'
in
colnames
:
insert_dict
[
'location'
]
=
item
.
v_location
.
encode
(
'utf-8'
)
if
'name'
in
colnames
:
name
=
item
.
_name
if
(
not
item
.
v_is_root
or
not
item
.
v_is_run
)
else
item
.
_crun
insert_dict
[
'name'
]
=
name
.
encode
(
'utf-8'
)
if
'class_name'
in
colnames
:
insert_dict
[
'class_name'
]
=
item
.
f_get_class_name
(
)
.
encode
(
'utf-8'
)
if
'value'
in
colnames
:
insert_dict
[
'value'
]
=
self
.
_all_cut_string
(
item
.
f_val_to_str
(
)
.
encode
(
'utf-8'
)
,
pypetconstants
.
HDF5_STRCOL_MAX_VALUE_LENGTH
,
self
.
_logger
)
if
'hexdigest'
in
colnames
:
insert_dict
[
'hexdigest'
]
=
additional_info
[
'hexdigest'
]
if
'idx'
in
colnames
:
insert_dict
[
'idx'
]
=
item
.
v_idx
if
'time'
in
colnames
:
time_
=
item
.
_time
insert_dict
[
'time'
]
=
time_
.
encode
(
'utf-8'
)
if
'timestamp'
in
colnames
:
timestamp
=
item
.
_timestamp
insert_dict
[
'timestamp'
]
=
timestamp
if
'range'
in
colnames
:
third_length
=
pypetconstants
.
HDF5_STRCOL_MAX_RANGE_LENGTH
//
3
+
10
item_range
=
itools
.
islice
(
item
.
f_get_range
(
copy
=
False
)
,
0
,
third_length
)
range_string
=
', '
.
join
(
[
repr
(
x
)
for
x
in
item_range
]
)
insert_dict
[
'range'
]
=
self
.
_all_cut_string
(
range_string
.
encode
(
'utf-8'
)
,
pypetconstants
.
HDF5_STRCOL_MAX_RANGE_LENGTH
,
self
.
_logger
)
# To allow backwards compatibility
if
'array'
in
colnames
:
third_length
=
pypetconstants
.
HDF5_STRCOL_MAX_RANGE_LENGTH
//
3
+
10
item_range
=
itools
.
islice
(
item
.
f_get_range
(
copy
=
False
)
,
0
,
third_length
)
range_string
=
', '
.
join
(
[
repr
(
x
)
for
x
in
item_range
]
)
insert_dict
[
'array'
]
=
self
.
_all_cut_string
(
range_string
.
encode
(
'utf-8'
)
,
pypetconstants
.
HDF5_STRCOL_MAX_RANGE_LENGTH
,
self
.
_logger
)
if
'version'
in
colnames
:
insert_dict
[
'version'
]
=
item
.
v_version
.
encode
(
'utf-8'
)
if
'python'
in
colnames
:
insert_dict
[
'python'
]
=
item
.
v_python
.
encode
(
'utf-8'
)
if
'finish_timestamp'
in
colnames
:
insert_dict
[
'finish_timestamp'
]
=
item
.
_finish_timestamp_run
return
insert_dict

Extracts information from a given item to be stored into a pytable row .


def _all_cut_string(string, max_length, logger):
        """Cuts string data to the maximum length allowed in a pytables column
        if string is too long.

        :param string: String to be cut
        :param max_length: Maximum allowed string length
        :param logger: Logger where messages about truncating should be written

        :return: String, cut if too long

        """
        if len(string) > max_length:
            logger.debug('The string `%s` was too long I truncated it to'
                         ' %d characters' %
                         (string, max_length))
            string = string[0:max_length - 3] + '...'.encode('utf-8')

        return string

Cuts string data to the maximum length allowed in a pytables column if string is too long .


def _all_create_or_get_group(self, name, parent_hdf5_group=None):
        """Creates or returns a group"""
        if not name in parent_hdf5_group:
            new_hdf5_group = self._hdf5file.create_group(where=parent_hdf5_group,
                                                      name=name,
                                                      title=name,
                                                      filters=self._all_get_filters())
            return new_hdf5_group, True
        else:
            new_hdf5_group = parent_hdf5_group._f_get_child(name)
            return new_hdf5_group, False

def _all_create_or_get_group ( self , name , parent_hdf5_group = None ) : if not name in parent_hdf5_group : new_hdf5_group = self . _hdf5file . create_group ( where = parent_hdf5_group , name = name , title = name , filters = self . _all_get_filters ( ) ) return new_hdf5_group , True else : new_hdf5_group = parent_hdf5_group . _f_get_child ( name ) return new_hdf5_group , False


def _all_create_or_get_groups(self, key, start_hdf5_group=None):
        """Creates new or follows existing group nodes along a given colon separated `key`.

        :param key:

            Colon separated path along hdf5 file, e.g. `parameters.mobiles.cars`.

        :param start_hdf5_group:

            HDF5 group from where to start, leave `None` for the trajectory group.

        :return:

            Final group node, e.g. group node with name `cars`.

        """
        if start_hdf5_group is None:
            newhdf5_group = self._trajectory_group
        else:
            newhdf5_group = start_hdf5_group
        created = False
        if key == '':
            return newhdf5_group, created
        split_key = key.split('.')
        for name in split_key:
            newhdf5_group, created = self._all_create_or_get_group(name, newhdf5_group)

        return newhdf5_group, created

def _all_create_or_get_groups ( self , key , start_hdf5_group = None ) : if start_hdf5_group is None : newhdf5_group = self . _trajectory_group else : newhdf5_group = start_hdf5_group created = False if key == '' : return newhdf5_group , created split_key = key . split ( '.' ) for name in split_key : newhdf5_group , created = self . _all_create_or_get_group ( name , newhdf5_group ) return newhdf5_group , created

Creates new or follows existing group nodes along a given colon separated key .


def _ann_store_annotations(self, item_with_annotations, node, overwrite=False):
        """Stores annotations into an hdf5 file."""

        # If we overwrite delete all annotations first
        if overwrite is True or overwrite == 'v_annotations':
            annotated = self._all_get_from_attrs(node, HDF5StorageService.ANNOTATED)
            if annotated:
                current_attrs = node._v_attrs
                for attr_name in current_attrs._v_attrnames:
                    if attr_name.startswith(HDF5StorageService.ANNOTATION_PREFIX):
                        delattr(current_attrs, attr_name)
                delattr(current_attrs, HDF5StorageService.ANNOTATED)
                self._hdf5file.flush()

        # Only store annotations if the item has some
        if not item_with_annotations.v_annotations.f_is_empty():

            anno_dict = item_with_annotations.v_annotations._dict

            current_attrs = node._v_attrs

            changed = False

            for field_name in anno_dict:
                val = anno_dict[field_name]

                field_name_with_prefix = HDF5StorageService.ANNOTATION_PREFIX + field_name
                if field_name_with_prefix not in current_attrs:
                    # Only store *new* annotations, if they already exist on disk, skip storage
                    setattr(current_attrs, field_name_with_prefix, val)
                    changed = True

            if changed:
                setattr(current_attrs, HDF5StorageService.ANNOTATED, True)
                self._hdf5file.flush()


def _ann_load_annotations(self, item_with_annotations, node):
        """Loads annotations from disk."""

        annotated = self._all_get_from_attrs(node, HDF5StorageService.ANNOTATED)

        if annotated:

            annotations = item_with_annotations.v_annotations

            # You can only load into non-empty annotations, to prevent overwriting data in RAM
            if not annotations.f_is_empty():
                raise TypeError('Loading into non-empty annotations!')

            current_attrs = node._v_attrs

            for attr_name in current_attrs._v_attrnames:

                if attr_name.startswith(HDF5StorageService.ANNOTATION_PREFIX):
                    key = attr_name
                    key = key.replace(HDF5StorageService.ANNOTATION_PREFIX, '')

                    data = getattr(current_attrs, attr_name)
                    setattr(annotations, key, data)


def _grp_store_group(self, traj_group, store_data=pypetconstants.STORE_DATA,
                         with_links=True, recursive=False, max_depth=None,
                         _hdf5_group=None, _newly_created=False):
        """Stores a group node.

        For group nodes only annotations and comments need to be stored.

        """
        if store_data == pypetconstants.STORE_NOTHING:
            return
        elif store_data == pypetconstants.STORE_DATA_SKIPPING and traj_group._stored:
            self._logger.debug('Already found `%s` on disk I will not store it!' %
                                   traj_group.v_full_name)
        elif not recursive:
            if _hdf5_group is None:
                _hdf5_group, _newly_created = self._all_create_or_get_groups(traj_group.v_full_name)

            overwrite = store_data == pypetconstants.OVERWRITE_DATA

            if (traj_group.v_comment != '' and
                    (HDF5StorageService.COMMENT not in _hdf5_group._v_attrs or overwrite)):
                setattr(_hdf5_group._v_attrs, HDF5StorageService.COMMENT, traj_group.v_comment)

            if ((_newly_created or overwrite) and
                type(traj_group) not in (nn.NNGroupNode, nn.ConfigGroup, nn.ParameterGroup,
                                             nn.DerivedParameterGroup, nn.ResultGroup)):
                # We only store the name of the class if it is not one of the standard groups,
                # that are always used.
                setattr(_hdf5_group._v_attrs, HDF5StorageService.CLASS_NAME,
                        traj_group.f_get_class_name())

            self._ann_store_annotations(traj_group, _hdf5_group, overwrite=overwrite)
            self._hdf5file.flush()
            traj_group._stored = True

            # Signal completed node loading
            self._node_processing_timer.signal_update()

        if recursive:
            parent_traj_group = traj_group.f_get_parent()
            parent_hdf5_group = self._all_create_or_get_groups(parent_traj_group.v_full_name)[0]

            self._tree_store_nodes_dfs(parent_traj_group, traj_group.v_name, store_data=store_data,
                                       with_links=with_links, recursive=recursive,
                                       max_depth=max_depth, current_depth=0,
                                       parent_hdf5_group=parent_hdf5_group)


def _grp_load_group(self, traj_group, load_data=pypetconstants.LOAD_DATA, with_links=True,
                        recursive=False, max_depth=None,
                        _traj=None, _as_new=False, _hdf5_group=None):
        """Loads a group node and potentially everything recursively below"""
        if _hdf5_group is None:
            _hdf5_group = self._all_get_node_by_name(traj_group.v_full_name)
            _traj = traj_group.v_root

        if recursive:
            parent_traj_node = traj_group.f_get_parent()
            self._tree_load_nodes_dfs(parent_traj_node, load_data=load_data, with_links=with_links,
                                  recursive=recursive, max_depth=max_depth,
                                  current_depth=0,
                                  trajectory=_traj, as_new=_as_new,
                                  hdf5_group=_hdf5_group)
        else:
            if load_data == pypetconstants.LOAD_NOTHING:
                return

            elif load_data == pypetconstants.OVERWRITE_DATA:
                traj_group.v_annotations.f_empty()
                traj_group.v_comment = ''

            self._all_load_skeleton(traj_group, _hdf5_group)
            traj_group._stored = not _as_new

            # Signal completed node loading
            self._node_processing_timer.signal_update()


def _all_load_skeleton(self, traj_node, hdf5_group):
        """Reloads skeleton data of a tree node"""
        if traj_node.v_annotations.f_is_empty():
            self._ann_load_annotations(traj_node, hdf5_group)
        if traj_node.v_comment == '':
            comment = self._all_get_from_attrs(hdf5_group, HDF5StorageService.COMMENT)
            if comment is None:
                comment = ''
            traj_node.v_comment = comment

def _all_load_skeleton ( self , traj_node , hdf5_group ) : if traj_node . v_annotations . f_is_empty ( ) : self . _ann_load_annotations ( traj_node , hdf5_group ) if traj_node . v_comment == '' : comment = self . _all_get_from_attrs ( hdf5_group , HDF5StorageService . COMMENT ) if comment is None : comment = '' traj_node . v_comment = comment


def _prm_extract_missing_flags(data_dict, flags_dict):
        """Extracts storage flags for data in `data_dict`
        if they were not specified in `flags_dict`.

        See :const:`~pypet.storageservice.HDF5StorageService.TYPE_FLAG_MAPPING`
        for how to store different types of data per default.

        """
        for key, data in data_dict.items():
            if not key in flags_dict:
                dtype = type(data)
                if (dtype is np.ndarray or dtype is dict) and len(data) == 0:
                        # Empty containers are stored as an Array
                        # No need to ask for tuple or list, because they are always
                        # stored as arrays.
                        flags_dict[key] = HDF5StorageService.ARRAY
                        continue
                else:
                    try:
                        flags_dict[key] = HDF5StorageService.TYPE_FLAG_MAPPING[dtype]
                    except KeyError:
                        raise pex.NoSuchServiceError('I cannot store `%s`, I do not understand the'
                                                     'type `%s`.' % (key, str(dtype)))

Extracts storage flags for data in data_dict if they were not specified in flags_dict .


def _prm_meta_add_summary(self, instance):
        """Adds data to the summary tables and returns if `instance`s comment has to be stored.

        Also moves comments upwards in the hierarchy if purge_duplicate_comments is true
        and a lower index run has completed. Only necessary for *multiprocessing*.

        :return: Tuple

            * String specifying the subtree

            * Boolean whether to store the comment to `instance`s hdf5 node

        """
        if instance.v_comment == '':
            return False

        where = instance.v_branch
        definitely_store_comment = True

        # Get the hexdigest of the comment to see if such a comment has been stored before
        bytes_comment = instance.v_comment.encode('utf-8')
        hexdigest = hashlib.sha1(bytes_comment).hexdigest()
        hexdigest = hexdigest.encode('utf-8')

        # Get the overview table
        table_name = where + '_summary'

        # Check if the overview table exists, otherwise skip the rest of
        # the meta adding
        if table_name in self._overview_group:
            table = getattr(self._overview_group, table_name)
        else:
            return definitely_store_comment

        try:
            condvars = {'hexdigestcol': table.cols.hexdigest,
                'hexdigest': hexdigest}

            condition = """(hexdigestcol == hexdigest)"""

            row_iterator = table.where(condition, condvars=condvars)

            row = None
            try:
                row = next(row_iterator)
            except StopIteration:
                pass

            if row is None:
                self._all_store_param_or_result_table_entry(instance, table,
                                                            flags=(
                                                                HDF5StorageService.ADD_ROW,),
                                                            additional_info={
                                                                'hexdigest': hexdigest})

                definitely_store_comment = True
            else:
                definitely_store_comment = False
                self._all_kill_iterator(row_iterator)

        except pt.NoSuchNodeError:
            definitely_store_comment = True

        return definitely_store_comment

Adds data to the summary tables and returns if instance s comment has to be stored .


def _prm_add_meta_info(self, instance, group, overwrite=False):
        """Adds information to overview tables and meta information to
        the `instance`s hdf5 `group`.

        :param instance: Instance to store meta info about
        :param group: HDF5 group of instance
        :param overwrite: If data should be explicitly overwritten

        """

        if overwrite:
            flags = ()
        else:
            flags = (HDF5StorageService.ADD_ROW,)

        definitely_store_comment = True
        try:
            # Check if we need to store the comment. Maybe update the overview tables
            # accordingly if the current run index is lower than the one in the table.
            definitely_store_comment = self._prm_meta_add_summary(instance)

            try:
                # Update the summary overview table
                table_name = instance.v_branch + '_overview'

                table = getattr(self._overview_group, table_name)
                if len(table) < pypetconstants.HDF5_MAX_OVERVIEW_TABLE_LENGTH:

                    self._all_store_param_or_result_table_entry(instance, table,
                                                                flags=flags)
            except pt.NoSuchNodeError:
                pass
        except Exception as exc:
            self._logger.error('Could not store information table due to `%s`.' % repr(exc))

        if ((not self._purge_duplicate_comments or definitely_store_comment) and
                    instance.v_comment != ''):
            # Only add the comment if necessary
            setattr(group._v_attrs, HDF5StorageService.COMMENT, instance.v_comment)

        # Add class name and whether node is a leaf to the HDF5 attributes
        setattr(group._v_attrs, HDF5StorageService.CLASS_NAME, instance.f_get_class_name())
        setattr(group._v_attrs, HDF5StorageService.LEAF, True)

        if instance.v_is_parameter and instance.v_explored:
            # If the stored parameter was an explored one we need to mark this in the
            # explored overview table
            try:
                tablename = 'explored_parameters_overview'
                table = getattr(self._overview_group, tablename)

                if len(table) < pypetconstants.HDF5_MAX_OVERVIEW_TABLE_LENGTH:
                    self._all_store_param_or_result_table_entry(instance, table,
                                                                flags=flags)
            except pt.NoSuchNodeError:
                pass
            except Exception as exc:
                self._logger.error('Could not store information '
                                   'table due to `%s`.' % repr(exc))

Adds information to overview tables and meta information to the instance s hdf5 group .


def _prm_store_from_dict(self, fullname, store_dict, hdf5_group, store_flags, kwargs):
        """Stores a `store_dict`"""
        for key, data_to_store in store_dict.items():
            # self._logger.log(1, 'SUB-Storing %s [%s]', key, str(store_dict[key]))
            original_hdf5_group = None

            flag = store_flags[key]

            if '.' in key:
                original_hdf5_group = hdf5_group
                split_key = key.split('.')
                key = split_key.pop()
                for inner_key in split_key:
                    hdf5_group, newly_created =  self._all_create_or_get_group(inner_key,
                                                                                hdf5_group)
                    if newly_created:
                        setattr(hdf5_group._v_attrs, HDF5StorageService.STORAGE_TYPE,
                                HDF5StorageService.NESTED_GROUP)
                    else:
                        store_type = self._all_get_from_attrs(hdf5_group, HDF5StorageService.STORAGE_TYPE)
                        if store_type != HDF5StorageService.NESTED_GROUP:
                            raise ValueError('You want to nested results but `%s` is already '
                                             'of type `%s`!' % (hdf5_group._v_name, store_type))

            # Iterate through the data and store according to the storage flags
            if key in hdf5_group:
                # We won't change any data that is found on disk
                self._logger.debug(
                    'Found %s already in hdf5 node of %s, so I will ignore it.' %
                    (key, fullname))
                continue

            if flag == HDF5StorageService.TABLE:
                # self._logger.log(1, 'SUB-Storing %s TABLE', key)
                self._prm_write_into_pytable(key, data_to_store, hdf5_group, fullname,
                                             **kwargs)
            elif flag == HDF5StorageService.DICT:
                # self._logger.log(1, 'SUB-Storing %s DICT', key)
                self._prm_write_dict_as_table(key, data_to_store, hdf5_group, fullname,
                                              **kwargs)
            elif flag == HDF5StorageService.ARRAY:
                # self._logger.log(1, 'SUB-Storing %s ARRAY', key)
                self._prm_write_into_array(key, data_to_store, hdf5_group, fullname,
                                           **kwargs)
            elif flag in (HDF5StorageService.CARRAY,
                          HDF5StorageService.EARRAY,
                          HDF5StorageService.VLARRAY):
                self._prm_write_into_other_array(key, data_to_store,
                                                 hdf5_group, fullname,
                                                 flag=flag, **kwargs)
            elif flag in (HDF5StorageService.SERIES,
                          HDF5StorageService.FRAME,
                          #  HDF5StorageService.PANEL
                          ):
                # self._logger.log(1, 'SUB-Storing %s PANDAS', key)
                self._prm_write_pandas_data(key, data_to_store, hdf5_group, fullname,
                                            flag, **kwargs)
            elif flag == HDF5StorageService.SHARED_DATA:
                pass  # Shared data needs to be explicitly created and is not stored on
                # the fly
            else:
                raise RuntimeError('You shall not pass!')

            if original_hdf5_group is not None:
                hdf5_group = original_hdf5_group


def _prm_store_parameter_or_result(self,
                                       instance,
                                       store_data=pypetconstants.STORE_DATA,
                                       store_flags=None,
                                       overwrite=None,
                                       with_links=False,
                                       recursive=False,
                                       _hdf5_group=None,
                                       _newly_created=False,
                                       **kwargs):
        """Stores a parameter or result to hdf5.

        :param instance:

            The instance to be stored

        :param store_data:

            How to store data

        :param store_flags:

            Dictionary containing how to store individual data, usually empty.

        :param overwrite:

            Instructions how to overwrite data

        :param with_links:

            Placeholder because leaves have no links

        :param recursive:

            Placeholder, because leaves have no children

        :param _hdf5_group:

            The hdf5 group for storing the parameter or result

        :param _newly_created:

            If should be created in a new form

        """
        if store_data == pypetconstants.STORE_NOTHING:
            return
        elif store_data == pypetconstants.STORE_DATA_SKIPPING and instance._stored:
            self._logger.debug('Already found `%s` on disk I will not store it!' %
                                   instance.v_full_name)
            return
        elif store_data == pypetconstants.OVERWRITE_DATA:
            if not overwrite:
                overwrite = True

        fullname = instance.v_full_name
        self._logger.debug('Storing `%s`.' % fullname)

        if _hdf5_group is None:
            # If no group is provided we might need to create one
            _hdf5_group, _newly_created = self._all_create_or_get_groups(fullname)

        # kwargs_flags = {} # Dictionary to change settings
        # old_kwargs = {}
        store_dict = {}
        # If the user did not supply storage flags, we need to set it to the empty dictionary
        if store_flags is None:
            store_flags = {}

        try:
            # Get the data to store from the instance
            if not instance.f_is_empty():
                store_dict = instance._store()
            try:
                # Ask the instance for storage flags
                instance_flags = instance._store_flags().copy() # copy to avoid modifying the
                # original data
            except AttributeError:
                # If it does not provide any, set it to the empty dictionary
                instance_flags = {}

            # User specified flags have priority over the flags from the instance
            instance_flags.update(store_flags)
            store_flags = instance_flags

            # If we still have data in `store_dict` about which we do not know how to store
            # it, pick default storage flags
            self._prm_extract_missing_flags(store_dict, store_flags)

            if overwrite:
                if isinstance(overwrite, str):
                    overwrite = [overwrite]

                if overwrite is True:
                    to_delete = [key for key in store_dict.keys() if key in _hdf5_group]
                    self._all_delete_parameter_or_result_or_group(instance,
                                                                  delete_only=to_delete,
                                                                  _hdf5_group=_hdf5_group)
                elif isinstance(overwrite, (list, tuple)):
                    overwrite_set = set(overwrite)
                    key_set = set(store_dict.keys())

                    stuff_not_to_be_overwritten = overwrite_set - key_set

                    if overwrite!='v_annotations' and len(stuff_not_to_be_overwritten) > 0:
                        self._logger.warning('Cannot overwrite `%s`, these items are not supposed to '
                                             'be stored by the leaf node.' %
                                             str(stuff_not_to_be_overwritten))

                    stuff_to_overwrite = overwrite_set & key_set
                    if len(stuff_to_overwrite) > 0:
                        self._all_delete_parameter_or_result_or_group(instance,
                                                                      delete_only=list(
                                                                          stuff_to_overwrite))
                else:
                    raise ValueError('Your value of overwrite `%s` is not understood. '
                                     'Please pass `True` of a list of strings to fine grain '
                                     'overwriting.' % str(overwrite))

            self._prm_store_from_dict(fullname, store_dict, _hdf5_group, store_flags, kwargs)

            # Store annotations
            self._ann_store_annotations(instance, _hdf5_group, overwrite=overwrite)

            if _newly_created or overwrite is True:
                # If we created a new group or the parameter was extended we need to
                # update the meta information and summary tables
                self._prm_add_meta_info(instance, _hdf5_group,
                                        overwrite=not _newly_created)

            instance._stored = True

            #self._logger.debug('Finished Storing `%s`.' % fullname)
            # Signal completed node loading
            self._node_processing_timer.signal_update()

        except:
            # I anything fails, we want to remove the data of the parameter again
            self._logger.error(
                'Failed storing leaf `%s`. I will remove the hdf5 data I added  again.' % fullname)
            # Delete data
            for key in store_dict.keys():
                if key in  _hdf5_group:
                    hdf5_child = _hdf5_group._f_get_child(key)
                    hdf5_child._f_remove(recursive=True)
            # If no data left delete the whole parameter
            if _hdf5_group._v_nchildren == 0:
                _hdf5_group._f_remove(recursive=True)
            raise


def _prm_write_shared_array(self, key, data, hdf5_group, full_name, flag, **kwargs):
        """Creates and array that can be used with an HDF5 array object"""

        if flag == HDF5StorageService.ARRAY:
            self._prm_write_into_array(key, data, hdf5_group, full_name, **kwargs)
        elif flag in (HDF5StorageService.CARRAY,
                    HDF5StorageService.EARRAY,
                    HDF5StorageService.VLARRAY):
            self._prm_write_into_other_array(key, data, hdf5_group, full_name,
                                             flag=flag, **kwargs)
        else:
            raise RuntimeError('Flag `%s` of hdf5 data `%s` of `%s` not understood' %
                               (flag, key, full_name))

        self._hdf5file.flush()


def _prm_write_shared_table(self, key, hdf5_group, fullname, **kwargs):
        """Creates a new empty table"""
        first_row = None
        description = None
        if 'first_row' in kwargs:
            first_row = kwargs.pop('first_row')
            if not 'description' in kwargs:
                description = {}
                for colname in first_row:
                    data = first_row[colname]
                    column = self._all_get_table_col(key, [data], fullname)
                    description[colname] = column

        if 'description' in kwargs:
            description = kwargs.pop('description')

        if 'filters' in kwargs:
            filters = kwargs.pop('filters')
        else:
            filters = self._all_get_filters(kwargs)

        table = self._hdf5file.create_table(where=hdf5_group, name=key,
                                          description=description,
                                          filters=filters,
                                          **kwargs)
        table.flush()

        if first_row is not None:
            row = table.row
            for key in description:
                row[key] = first_row[key]

            row.append()
            table.flush()

def _prm_write_shared_table ( self , key , hdf5_group , fullname , * * kwargs ) : first_row = None description = None if 'first_row' in kwargs : first_row = kwargs . pop ( 'first_row' ) if not 'description' in kwargs : description = { } for colname in first_row : data = first_row [ colname ] column = self . _all_get_table_col ( key , [ data ] , fullname ) description [ colname ] = column if 'description' in kwargs : description = kwargs . pop ( 'description' ) if 'filters' in kwargs : filters = kwargs . pop ( 'filters' ) else : filters = self . _all_get_filters ( kwargs ) table = self . _hdf5file . create_table ( where = hdf5_group , name = key , description = description , filters = filters , * * kwargs ) table . flush ( ) if first_row is not None : row = table . row for key in description : row [ key ] = first_row [ key ] row . append ( ) table . flush ( )


def _prm_write_dict_as_table(self, key, data_to_store, group, fullname, **kwargs):
        """Stores a python dictionary as pytable

        :param key:

            Name of data item to store

        :param data_to_store:

            Dictionary to store

        :param group:

            Group node where to store data in hdf5 file

        :param fullname:

            Full name of the `data_to_store`s original container, only needed for throwing errors.

        """
        if key in group:
            raise ValueError(
                'Dictionary `%s` already exists in `%s`. Appending is not supported (yet).')

        if key in group:
            raise ValueError('Dict `%s` already exists in `%s`. Appending is not supported (yet).')

        temp_dict = {}
        for innerkey in data_to_store:
            val = data_to_store[innerkey]
            temp_dict[innerkey] = [val]

        # Convert dictionary to object table
        objtable = ObjectTable(data=temp_dict)

        # Then store the object table
        self._prm_write_into_pytable(key, objtable, group, fullname, **kwargs)

        new_table = group._f_get_child(key)

        # Remember that the Object Table represents a dictionary
        self._all_set_attributes_to_recall_natives(temp_dict, new_table,
                                                   HDF5StorageService.DATA_PREFIX)

        setattr(new_table._v_attrs, HDF5StorageService.STORAGE_TYPE,
                HDF5StorageService.DICT)

        self._hdf5file.flush()


def _prm_write_pandas_data(self, key, data, group, fullname, flag, **kwargs):
        """Stores a pandas DataFrame into hdf5.

        :param key:

            Name of data item to store

        :param data:

            Pandas Data to Store

        :param group:

            Group node where to store data in hdf5 file

        :param fullname:

            Full name of the `data_to_store`s original container, only needed for throwing errors.

        :param flag:

            If it is a series, frame or panel

        """
        try:
            if 'filters' not in kwargs:
                filters = self._all_get_filters(kwargs)
                kwargs['filters'] = filters
            if 'format' not in kwargs:
                kwargs['format'] = self.pandas_format
            if 'encoding' not in kwargs:
                kwargs['encoding'] = self.encoding

            overwrite = kwargs.pop('overwrite', False)

            if key in group and not (overwrite or kwargs.get('append', False)):
                raise ValueError(
                    'DataFrame `%s` already exists in `%s`. '
                    'To append pass ``append=`True```.' % (key, fullname))
            else:
                self._logger.debug('Appending to pandas data `%s` in `%s`' % (key, fullname))

            if data is not None and (kwargs['format'] == 'f' or kwargs['format'] == 'fixed'):
                kwargs['expectedrows'] = data.shape[0]

            name = group._v_pathname + '/' + key
            self._hdf5store.put(name, data, **kwargs)
            self._hdf5store.flush()
            self._hdf5file.flush()

            frame_group = group._f_get_child(key)
            setattr(frame_group._v_attrs, HDF5StorageService.STORAGE_TYPE, flag)
            self._hdf5file.flush()

        except:
            self._logger.error('Failed storing pandas data `%s` of `%s`.' % (key, fullname))
            raise


def _prm_write_into_other_array(self, key, data, group, fullname,
                                    flag, **kwargs):
        """Stores data as carray, earray or vlarray depending on `flag`.

        :param key:

            Name of data item to store

        :param data:

            Data to store

        :param group:

            Group node where to store data in hdf5 file

        :param fullname:

            Full name of the `data_to_store`s original container, only needed for throwing errors.

        :param recall:

            If container type and data type for perfect recall should be stored

        :param flag:

            How to store:
                CARRAY, EARRAY, VLARRAY

        """
        try:

            if flag == HDF5StorageService.CARRAY:
                factory = self._hdf5file.create_carray
            elif flag == HDF5StorageService.EARRAY:
                factory = self._hdf5file.create_earray
            elif flag == HDF5StorageService.VLARRAY:
                factory = self._hdf5file.create_vlarray
            else:
                raise RuntimeError('You shall not pass!')

            if key in group:
                raise ValueError(
                    'CArray `%s` already exists in `%s`. Appending is not supported (yet).')

            if 'filters' in kwargs:
                filters = kwargs.pop('filters')
            else:
                filters = self._all_get_filters(kwargs)

            try:
                other_array = factory(where=group, name=key, obj=data,
                                                filters=filters, **kwargs)
            except (ValueError, TypeError) as exc:
                try:
                    conv_data = data[:]
                    conv_data = np.core.defchararray.encode(conv_data, self.encoding)
                    other_array = factory(where=group, name=key,
                                                obj=conv_data,
                                                filters=filters, **kwargs)
                except Exception:
                    # Re-raise original Error
                    raise exc

            if data is not None:
                # Remember the types of the original data to recall them on loading
                self._all_set_attributes_to_recall_natives(data, other_array,
                                                       HDF5StorageService.DATA_PREFIX)
            setattr(other_array._v_attrs, HDF5StorageService.STORAGE_TYPE, flag)
            self._hdf5file.flush()
        except:
            self._logger.error('Failed storing %s `%s` of `%s`.' % (flag, key, fullname))
            raise

Stores data as carray earray or vlarray depending on flag .


def _prm_write_into_array(self, key, data, group, fullname, **kwargs):
        """Stores data as array.

        :param key:

            Name of data item to store

        :param data:

            Data to store

        :param group:

            Group node where to store data in hdf5 file

        :param fullname:

            Full name of the `data_to_store`s original container, only needed for throwing errors.

        :param recall:

            If container type and data type for perfect recall should be stored

        """

        try:
            if key in group:
                raise ValueError(
                    'Array `%s` already exists in `%s`. Appending is not supported (yet).')

            try:

                array = self._hdf5file.create_array(where=group,
                                              name=key, obj=data, **kwargs)
            except (TypeError, ValueError) as exc:
                try:
                    if type(data) is dict and len(data) == 0:
                        # We cannot store an empty dictionary,
                        # but we can use an empty tuple as a dummy.
                        conv_data = ()
                    elif isinstance(data, str):
                        conv_data = data.encode(self._encoding)
                    elif isinstance(data, int):
                        conv_data = np.int64(data)
                    else:
                        conv_data = []
                        for string in data:
                            conv_data.append(string.encode(self._encoding))
                    array = self._hdf5file.create_array(where=group,
                                                  name=key, obj=conv_data, **kwargs)
                except Exception:
                    # Re-raise original error
                    raise exc

            if data is not None:
                # Remember the types of the original data to recall them on loading
                self._all_set_attributes_to_recall_natives(data, array,
                                                           HDF5StorageService.DATA_PREFIX)
            setattr(array._v_attrs, HDF5StorageService.STORAGE_TYPE,
                    HDF5StorageService.ARRAY)
            self._hdf5file.flush()
        except:
            self._logger.error('Failed storing array `%s` of `%s`.' % (key, fullname))
            raise


def _lnk_delete_link(self, link_name):
        """Removes a link from disk"""
        translated_name = '/' + self._trajectory_name + '/' + link_name.replace('.','/')
        link = self._hdf5file.get_node(where=translated_name)
        link._f_remove()

def _lnk_delete_link ( self , link_name ) : translated_name = '/' + self . _trajectory_name + '/' + link_name . replace ( '.' , '/' ) link = self . _hdf5file . get_node ( where = translated_name ) link . _f_remove ( )


def _all_delete_parameter_or_result_or_group(self, instance,
                                                 delete_only=None,
                                                 remove_from_item=False,
                                                 recursive=False,
                                                 _hdf5_group=None):
        """Removes a parameter or result or group from the hdf5 file.

        :param instance: Instance to be removed

        :param delete_only:

            List of elements if you only want to delete parts of a leaf node. Note that this
            needs to list the names of the hdf5 subnodes. BE CAREFUL if you erase parts of a leaf.
            Erasing partly happens at your own risk, it might be the case that you can
            no longer reconstruct the leaf from the leftovers!

        :param remove_from_item:

            If using `delete_only` and `remove_from_item=True` after deletion the data item is
            also removed from the `instance`.

        :param recursive:

            If a group node has children, you will can delete it if recursive is True.

        """
        split_name = instance.v_location.split('.')
        if _hdf5_group is None:
            where = '/' + self._trajectory_name + '/' + '/'.join(split_name)
            node_name = instance.v_name
            _hdf5_group = self._hdf5file.get_node(where=where, name=node_name)

        if delete_only is None:
            if instance.v_is_group and not recursive and len(_hdf5_group._v_children) != 0:
                    raise TypeError('You cannot remove the group `%s`, it has children, please '
                                    'use `recursive=True` to enforce removal.' %
                                    instance.v_full_name)
            _hdf5_group._f_remove(recursive=True)
        else:
            if not instance.v_is_leaf:
                raise ValueError('You can only choose `delete_only` mode for leafs.')

            if isinstance(delete_only, str):
                delete_only = [delete_only]

            for delete_item in delete_only:
                if (remove_from_item and
                        hasattr(instance, '__contains__') and
                        hasattr(instance, '__delattr__') and
                            delete_item in instance):
                    delattr(instance, delete_item)
                try:
                    _hdf5_sub_group = self._hdf5file.get_node(where=_hdf5_group,
                                                        name=delete_item)
                    _hdf5_sub_group._f_remove(recursive=True)
                except pt.NoSuchNodeError:
                    self._logger.warning('Could not delete `%s` from `%s`. HDF5 node not found!' %
                                         (delete_item, instance.v_full_name))

Removes a parameter or result or group from the hdf5 file .


def _prm_write_into_pytable(self, tablename, data, hdf5_group, fullname, **kwargs):
        """Stores data as pytable.

        :param tablename:

            Name of the data table

        :param data:

            Data to store

        :param hdf5_group:

            Group node where to store data in hdf5 file

        :param fullname:

            Full name of the `data_to_store`s original container, only needed for throwing errors.

        """

        datasize = data.shape[0]

        try:

            # Get a new pytables description from the data and create a new table
            description_dict, data_type_dict = self._prm_make_description(data, fullname)
            description_dicts = [{}]

            if len(description_dict) > ptpa.MAX_COLUMNS:
                # For optimization we want to store the original data types into another table
                # and split the tables into several ones
                new_table_group = self._hdf5file.create_group(where=hdf5_group,
                                                    name=tablename,
                                                    filters=self._all_get_filters(kwargs.copy()))

                count = 0
                for innerkey in description_dict:
                    val = description_dict[innerkey]
                    if count == ptpa.MAX_COLUMNS:
                        description_dicts.append({})
                        count = 0
                    description_dicts[-1][innerkey] = val
                    count += 1

                setattr(new_table_group._v_attrs, HDF5StorageService.STORAGE_TYPE,
                        HDF5StorageService.TABLE)
                setattr(new_table_group._v_attrs, HDF5StorageService.SPLIT_TABLE, 1)

                hdf5_group = new_table_group
            else:
                description_dicts = [description_dict]

            for idx, descr_dict in enumerate(description_dicts):

                if idx == 0:
                    tblname = tablename
                else:
                    tblname = tablename + '_%d' % idx

                table = self._hdf5file.create_table(where=hdf5_group, name=tblname,
                                              description=descr_dict,
                                              title=tblname,
                                              expectedrows=datasize,
                                              filters=self._all_get_filters(kwargs.copy()))

                row = table.row
                for n in range(datasize):
                    # Fill the columns with data, note if the parameter was extended nstart!=0

                    for key in descr_dict:
                        row[key] = data[key][n]

                    row.append()

                # Remember the original types of the data for perfect recall
                if idx == 0 and len(description_dict) <= ptpa.MAX_COLUMNS:
                    # We only have a single table and
                    # we can store the original data types as attributes
                    for field_name in data_type_dict:
                        type_description = data_type_dict[field_name]
                        self._all_set_attr(table, field_name, type_description)

                    setattr(table._v_attrs, HDF5StorageService.STORAGE_TYPE,
                            HDF5StorageService.TABLE)

                table.flush()
                self._hdf5file.flush()

            if len(description_dict) > ptpa.MAX_COLUMNS:
                # We have potentially many split tables and the data types are
                # stored into an additional table for performance reasons
                tblname = tablename + '__' + HDF5StorageService.STORAGE_TYPE
                field_names, data_types = list(zip(*data_type_dict.items()))
                data_type_table_dict = {'field_name': field_names, 'data_type': data_types}
                descr_dict, _ = self._prm_make_description(data_type_table_dict, fullname)

                table = self._hdf5file.create_table(where=hdf5_group, name=tblname,
                                              description=descr_dict,
                                              title=tblname,
                                              expectedrows=len(field_names),
                                              filters=self._all_get_filters(kwargs))

                row = table.row

                for n in range(len(field_names)):
                    # Fill the columns with data

                    for key in data_type_table_dict:
                        row[key] = data_type_table_dict[key][n]

                    row.append()

                setattr(table._v_attrs, HDF5StorageService.DATATYPE_TABLE, 1)

                table.flush()
                self._hdf5file.flush()

        except:
            self._logger.error('Failed storing table `%s` of `%s`.' % (tablename, fullname))
            raise


def _prm_make_description(self, data, fullname):
        """ Returns a description dictionary for pytables table creation"""

        def _convert_lists_and_tuples(series_of_data):
            """Converts lists and tuples to numpy arrays"""
            if isinstance(series_of_data[0],
                          (list, tuple)):  # and not isinstance(series_of_data[0], np.ndarray):
                # If the first data item is a list, the rest must be as well, since
                # data has to be homogeneous
                for idx, item in enumerate(series_of_data):
                    series_of_data[idx] = np.array(item)

        descriptiondict = {}  # dictionary containing the description to build a pytables table
        original_data_type_dict = {}  # dictionary containing the original data types

        for key in data:
            val = data[key]

            # remember the original data types
            self._all_set_attributes_to_recall_natives(val[0], PTItemMock(original_data_type_dict),
                                                       HDF5StorageService.FORMATTED_COLUMN_PREFIX %
                                                       key)

            _convert_lists_and_tuples(val)

            # get a pytables column from the data
            col = self._all_get_table_col(key, val, fullname)

            descriptiondict[key] = col

        return descriptiondict, original_data_type_dict


def _all_get_table_col(self, key, column, fullname):
        """ Creates a pytables column instance.

        The type of column depends on the type of `column[0]`.
        Note that data in `column` must be homogeneous!

        """
        val = column[0]

        try:

            # # We do not want to loose int_
            if type(val) is int:
                return pt.IntCol()

            if isinstance(val, (str, bytes)):
                itemsize = int(self._prm_get_longest_stringsize(column))
                return pt.StringCol(itemsize)

            if isinstance(val, np.ndarray):
                if (np.issubdtype(val.dtype, str) or
                        np.issubdtype(val.dtype, bytes)):
                    itemsize = int(self._prm_get_longest_stringsize(column))
                    return pt.StringCol(itemsize, shape=val.shape)
                else:
                    return pt.Col.from_dtype(np.dtype((val.dtype, val.shape)))
            else:
                return pt.Col.from_dtype(np.dtype(type(val)))
        except Exception:
            self._logger.error('Failure in storing `%s` of Parameter/Result `%s`.'
                               ' Its type was `%s`.' % (key, fullname, repr(type(val))))
            raise


def _prm_get_longest_stringsize(string_list):
        """ Returns the longest string size for a string entry across data."""
        maxlength = 1

        for stringar in string_list:
            if isinstance(stringar, np.ndarray):
                if stringar.ndim > 0:
                    for string in stringar.ravel():
                        maxlength = max(len(string), maxlength)
                else:
                    maxlength = max(len(stringar.tolist()), maxlength)
            else:
                maxlength = max(len(stringar), maxlength)

        # Make the string Col longer than needed in order to allow later on slightly larger strings
        return int(maxlength * 1.5)

Returns the longest string size for a string entry across data .


def _prm_load_into_dict(self, full_name, load_dict, hdf5_group, instance,
                            load_only, load_except, load_flags, _prefix = ''):
        """Loads into dictionary"""
        for node in hdf5_group:

            load_type = self._all_get_from_attrs(node, HDF5StorageService.STORAGE_TYPE)

            if _prefix:
                load_name = '%s.%s' % (_prefix, node._v_name)
            else:
                load_name = node._v_name

            if load_type == HDF5StorageService.NESTED_GROUP:
                self._prm_load_into_dict(full_name=full_name,
                                         load_dict=load_dict,
                                         hdf5_group=node,
                                         instance=instance,
                                         load_only=load_only,
                                         load_except=load_except,
                                         load_flags=load_flags,
                                         _prefix=load_name)
                continue

            if load_only is not None:

                if load_name not in load_only:
                    continue
                else:
                    load_only.remove(load_name)

            elif load_except is not None:
                if load_name in load_except:
                    load_except.remove(load_name)
                    continue

            # Recall from the hdf5 node attributes how the data was stored and reload accordingly
            if load_name in load_flags:
                load_type = load_flags[load_name]

            if load_type == HDF5StorageService.DICT:
                to_load = self._prm_read_dictionary(node, full_name)
            elif load_type == HDF5StorageService.TABLE:
                to_load = self._prm_read_table(node, full_name)
            elif load_type in (HDF5StorageService.ARRAY, HDF5StorageService.CARRAY,
                                HDF5StorageService.EARRAY, HDF5StorageService.VLARRAY):
                to_load = self._prm_read_array(node, full_name)
            elif load_type in (HDF5StorageService.FRAME,
                               HDF5StorageService.SERIES,
                               #  HDF5StorageService.PANEL
                               ):
                to_load = self._prm_read_pandas(node, full_name)
            elif load_type.startswith(HDF5StorageService.SHARED_DATA):
                to_load = self._prm_read_shared_data(node, instance)
            else:
                raise pex.NoSuchServiceError('Cannot load %s, do not understand the hdf5 file '
                                             'structure of %s [%s].' %
                                             (full_name, str(node), str(load_type)))

            if to_load is None:
                raise RuntimeError('You shall not pass!')

            load_dict[load_name] = to_load


def _prm_read_dictionary(self, leaf, full_name):
        """Loads data that was originally a dictionary when stored

        :param leaf:

            PyTables table containing the dictionary data

        :param full_name:

            Full name of the parameter or result whose data is to be loaded

        :return:

            Data to be loaded

        """
        try:
            # Load as Pbject Table
            temp_table = self._prm_read_table(leaf, full_name)
            # Turn the ObjectTable into a dictionary of lists (with length 1).
            temp_dict = temp_table.to_dict('list')

            innder_dict = {}

            # Turn the dictionary of lists into a normal dictionary
            for innerkey, vallist in temp_dict.items():
                innder_dict[innerkey] = vallist[0]

            return innder_dict
        except:
            self._logger.error('Failed loading `%s` of `%s`.' % (leaf._v_name, full_name))
            raise


def _prm_read_shared_data(self, shared_node, instance):
        """Reads shared data and constructs the appropraite class.

        :param shared_node:

            hdf5 node storing the pandas DataFrame

        :param full_name:

            Full name of the parameter or result whose data is to be loaded

        :return:

            Data to load

        """
        try:
            data_type = self._all_get_from_attrs(shared_node,
                                                 HDF5StorageService.SHARED_DATA_TYPE)
            constructor = shared.FLAG_CLASS_MAPPING[data_type]
            name = shared_node._v_name
            result = constructor(name=name, parent=instance)
            return result
        except:
            self._logger.error('Failed loading `%s` of `%s`.' % (shared_node._v_name,
                                                                 instance.v_full_name))
            raise

def
_prm_read_shared_data
(
self
,
shared_node
,
instance
)
:
try
:
data_type
=
self
.
_all_get_from_attrs
(
shared_node
,
HDF5StorageService
.
SHARED_DATA_TYPE
)
constructor
=
shared
.
FLAG_CLASS_MAPPING
[
data_type
]
name
=
shared_node
.
_v_name
result
=
constructor
(
name
=
name
,
parent
=
instance
)
return
result
except
:
self
.
_logger
.
error
(
'Failed loading `%s` of `%s`.'
%
(
shared_node
.
_v_name
,
instance
.
v_full_name
)
)
raise

Reads shared data and constructs the appropraite class .