def f_add_result(self, *args, **kwargs):
        """Adds a result under the current node.

        There are two ways to add a new result either by adding a result instance:

        >>> new_result = Result('group1.group2.myresult', 1666, x=3, y=4, comment='Example!')
        >>> traj.f_add_result(new_result)

        Or by passing the values directly to the function, with the name being the first
        (non-keyword!) argument:

        >>> traj.f_add_result('group1.group2.myresult', 1666, x=3, y=3,comment='Example!')

        If you want to create a different result than the standard result, you can
        give the constructor as the first (non-keyword!) argument followed by the name
        (non-keyword!):

        >>> traj.f_add_result(PickleResult,'group1.group2.myresult', 1666, x=3, y=3, comment='Example!')

        Additional arguments (here `1666`) or keyword arguments (here `x=3, y=3`) are passed
        onto the constructor of the result.

        Adds the full name of the current node as prefix to the name of the result.
        If current node is a single run (root) adds the prefix `'results.runs.run_08%d%'` to the
        full name where `'08%d'` is replaced by the index of the current run.

        """
        return self._nn_interface._add_generic(self, type_name=RESULT,
                                               group_type_name=RESULT_GROUP,
                                               args=args, kwargs=kwargs)

def f_add_result ( self , * args , * * kwargs ) : return self . _nn_interface . _add_generic ( self , type_name = RESULT , group_type_name = RESULT_GROUP , args = args , kwargs = kwargs )


def f_add_derived_parameter_group(self, *args, **kwargs):
        """Adds an empty derived parameter group under the current node.

        Adds the full name of the current node as prefix to the name of the group.
        If current node is a single run (root) adds the prefix `'derived_parameters.runs.run_08%d%'`
        to the full name where `'08%d'` is replaced by the index of the current run.

        The `name` can also contain subgroups separated via colons, for example:
        `name=subgroup1.subgroup2.subgroup3`. These other parent groups will be automatically
        be created.

        """

        return self._nn_interface._add_generic(self, type_name=DERIVED_PARAMETER_GROUP,
                                               group_type_name=DERIVED_PARAMETER_GROUP,
                                               args=args, kwargs=kwargs)

def f_add_derived_parameter_group ( self , * args , * * kwargs ) : return self . _nn_interface . _add_generic ( self , type_name = DERIVED_PARAMETER_GROUP , group_type_name = DERIVED_PARAMETER_GROUP , args = args , kwargs = kwargs )

Adds an empty derived parameter group under the current node .


def f_add_derived_parameter(self, *args, **kwargs):
        """Adds a derived parameter under the current group.

        Similar to
        :func:`~pypet.naturalnaming.ParameterGroup.f_add_parameter`

        Naming prefixes are added as in
        :func:`~pypet.naturalnaming.DerivedParameterGroup.f_add_derived_parameter_group`

        """
        return self._nn_interface._add_generic(self, type_name=DERIVED_PARAMETER,
                                               group_type_name=DERIVED_PARAMETER_GROUP,
                                               args=args, kwargs=kwargs)

def f_add_derived_parameter ( self , * args , * * kwargs ) : return self . _nn_interface . _add_generic ( self , type_name = DERIVED_PARAMETER , group_type_name = DERIVED_PARAMETER_GROUP , args = args , kwargs = kwargs )

Adds a derived parameter under the current group .


def f_add_config_group(self, *args, **kwargs):
        """Adds an empty config group under the current node.

        Adds the full name of the current node as prefix to the name of the group.
        If current node is the trajectory (root), the prefix `'config'` is added to the full name.

        The `name` can also contain subgroups separated via colons, for example:
        `name=subgroup1.subgroup2.subgroup3`. These other parent groups will be automatically
        be created.

        """
        return self._nn_interface._add_generic(self, type_name=CONFIG_GROUP,
                                               group_type_name=CONFIG_GROUP,
                                               args=args, kwargs=kwargs)

def f_add_config_group ( self , * args , * * kwargs ) : return self . _nn_interface . _add_generic ( self , type_name = CONFIG_GROUP , group_type_name = CONFIG_GROUP , args = args , kwargs = kwargs )

Adds an empty config group under the current node .


def f_add_config(self, *args, **kwargs):
        """Adds a config parameter under the current group.

        Similar to
        :func:`~pypet.naturalnaming.ParameterGroup.f_add_parameter`.

        If current group is the trajectory the prefix `'config'` is added to the name.

        """
        return self._nn_interface._add_generic(self, type_name=CONFIG,
                                               group_type_name=CONFIG_GROUP,
                                               args=args, kwargs=kwargs)

def f_add_config ( self , * args , * * kwargs ) : return self . _nn_interface . _add_generic ( self , type_name = CONFIG , group_type_name = CONFIG_GROUP , args = args , kwargs = kwargs )

Adds a config parameter under the current group .


def eval_one_max(traj, individual):
    """The fitness function"""
    traj.f_add_result('$set.$.individual', list(individual))
    fitness = sum(individual)
    traj.f_add_result('$set.$.fitness', fitness)
    traj.f_store()
    return (fitness,)

def eval_one_max ( traj , individual ) : traj . f_add_result ( '$set.$.individual' , list ( individual ) ) fitness = sum ( individual ) traj . f_add_result ( '$set.$.fitness' , fitness ) traj . f_store ( ) return ( fitness , )


def add_commit_variables(traj, commit):
    """Adds commit information to the trajectory."""

    git_time_value = time.strftime('%Y_%m_%d_%Hh%Mm%Ss', time.localtime(commit.committed_date))

    git_short_name = str(commit.hexsha[0:7])
    git_commit_name = 'commit_%s_' % git_short_name
    git_commit_name = 'git.' + git_commit_name + git_time_value

    if not traj.f_contains('config.'+git_commit_name, shortcuts=False):

        git_commit_name += '.'
        # Add the hexsha
        traj.f_add_config(git_commit_name+'hexsha', commit.hexsha,
                          comment='SHA-1 hash of commit')

        # Add the description string
        traj.f_add_config(git_commit_name+'name_rev', commit.name_rev,
                          comment='String describing the commits hex sha based on '
                                  'the closest Reference')

        # Add unix epoch
        traj.f_add_config(git_commit_name+'committed_date',
                          commit.committed_date, comment='Date of commit as unix epoch seconds')

        # Add commit message
        traj.f_add_config(git_commit_name+'message', str(commit.message),
                          comment='The commit message')

Adds commit information to the trajectory .


def make_git_commit(environment, git_repository, user_message, git_fail):
    """ Makes a commit and returns if a new commit was triggered and the SHA_1 code of the commit.

    If `git_fail` is `True` program fails instead of triggering a new commit given
    not committed changes. Then a `GitDiffError` is raised.

    """

    # Import GitPython, we do it here to allow also users not having GitPython installed
    # to use the normal environment

    # Open the repository
    repo = git.Repo(git_repository)
    index = repo.index

    traj = environment.v_trajectory

    # Create the commit message and append the trajectory name and comment
    if traj.v_comment:
        commentstr = ', Comment: `%s`' % traj.v_comment
    else:
        commentstr = ''

    if user_message:
        user_message += ' -- '

    message = '%sTrajectory: `%s`, Time: `%s`, %s' % \
              (user_message, traj.v_name, traj.v_time, commentstr)

    # Detect changes:
    diff = index.diff(None)

    if diff:
        if git_fail:
            # User requested fail instead of a new commit
            raise pex.GitDiffError('Found not committed changes!')
        # Make the commit
        repo.git.add('-u')
        commit = index.commit(message)
        new_commit = True

    else:
        # Take old commit
        commit = repo.commit(None)
        new_commit = False

    # Add the commit info to the trajectory
    add_commit_variables(traj, commit)

    return new_commit, commit.hexsha

Makes a commit and returns if a new commit was triggered and the SHA_1 code of the commit .


def flatten_dictionary(nested_dict, separator):
    """Flattens a nested dictionary.

    New keys are concatenations of nested keys with the `separator` in between.

    """
    flat_dict = {}
    for key, val in nested_dict.items():
        if isinstance(val, dict):
            new_flat_dict = flatten_dictionary(val, separator)
            for flat_key, inval in new_flat_dict.items():
                new_key = key + separator + flat_key
                flat_dict[new_key] = inval
        else:
            flat_dict[key] = val

    return flat_dict

def flatten_dictionary ( nested_dict , separator ) : flat_dict = { } for key , val in nested_dict . items ( ) : if isinstance ( val , dict ) : new_flat_dict = flatten_dictionary ( val , separator ) for flat_key , inval in new_flat_dict . items ( ) : new_key = key + separator + flat_key flat_dict [ new_key ] = inval else : flat_dict [ key ] = val return flat_dict


def nest_dictionary(flat_dict, separator):
    """ Nests a given flat dictionary.

    Nested keys are created by splitting given keys around the `separator`.

    """
    nested_dict = {}
    for key, val in flat_dict.items():
        split_key = key.split(separator)
        act_dict = nested_dict
        final_key = split_key.pop()
        for new_key in split_key:
            if not new_key in act_dict:
                act_dict[new_key] = {}

            act_dict = act_dict[new_key]

        act_dict[final_key] = val
    return nested_dict

def nest_dictionary ( flat_dict , separator ) : nested_dict = { } for key , val in flat_dict . items ( ) : split_key = key . split ( separator ) act_dict = nested_dict final_key = split_key . pop ( ) for new_key in split_key : if not new_key in act_dict : act_dict [ new_key ] = { } act_dict = act_dict [ new_key ] act_dict [ final_key ] = val return nested_dict


def progressbar(index, total, percentage_step=10, logger='print', log_level=logging.INFO,
                 reprint=True, time=True, length=20, fmt_string=None, reset=False):
    """Plots a progress bar to the given `logger` for large for loops.

    To be used inside a for-loop at the end of the loop:

    .. code-block:: python

        for irun in range(42):
            my_costly_job() # Your expensive function
            progressbar(index=irun, total=42, reprint=True) # shows a growing progressbar

    There is no initialisation of the progressbar necessary before the for-loop.
    The progressbar will be reset automatically if used in another for-loop.

    :param index: Current index of for-loop
    :param total: Total size of for-loop
    :param percentage_step: Steps with which the bar should be plotted
    :param logger:

        Logger to write to - with level INFO. If string 'print' is given, the print statement is
        used. Use ``None`` if you don't want to print or log the progressbar statement.

    :param log_level: Log level with which to log.
    :param reprint:

        If no new line should be plotted but carriage return (works only for printing)

    :param time: If the remaining time should be estimated and displayed
    :param length: Length of the bar in `=` signs.
    :param fmt_string:

        A string which contains exactly one `%s` in order to incorporate the progressbar.
        If such a string is given, ``fmt_string % progressbar`` is printed/logged.

    :param reset:

        If the progressbar should be restarted. If progressbar is called with a lower
        index than the one before, the progressbar is automatically restarted.

    :return:

        The progressbar string or `None` if the string has not been updated.

    """
    return _progressbar(index=index, total=total, percentage_step=percentage_step,
                        logger=logger, log_level=log_level, reprint=reprint,
                        time=time, length=length, fmt_string=fmt_string, reset=reset)

def progressbar ( index , total , percentage_step = 10 , logger = 'print' , log_level = logging . INFO , reprint = True , time = True , length = 20 , fmt_string = None , reset = False ) : return _progressbar ( index = index , total = total , percentage_step = percentage_step , logger = logger , log_level = log_level , reprint = reprint , time = time , length = length , fmt_string = fmt_string , reset = reset )

Plots a progress bar to the given logger for large for loops .


def _get_argspec(func):
    """Helper function to support both Python versions"""
    if inspect.isclass(func):
        func = func.__init__
    if not inspect.isfunction(func):
        # Init function not existing
        return [], False
    parameters = inspect.signature(func).parameters
    args = []
    uses_starstar = False
    for par in parameters.values():
        if (par.kind == inspect.Parameter.POSITIONAL_OR_KEYWORD or
                    par.kind == inspect.Parameter.KEYWORD_ONLY):
            args.append(par.name)
        elif par.kind == inspect.Parameter.VAR_KEYWORD:
            uses_starstar = True
    return args, uses_starstar

def
_get_argspec
(
func
)
:
if
inspect
.
isclass
(
func
)
:
func
=
func
.
__init__
if
not
inspect
.
isfunction
(
func
)
:
# Init function not existing
return
[
]
,
False
parameters
=
inspect
.
signature
(
func
)
.
parameters
args
=
[
]
uses_starstar
=
False
for
par
in
parameters
.
values
(
)
:
if
(
par
.
kind
==
inspect
.
Parameter
.
POSITIONAL_OR_KEYWORD
or
par
.
kind
==
inspect
.
Parameter
.
KEYWORD_ONLY
)
:
args
.
append
(
par
.
name
)
elif
par
.
kind
==
inspect
.
Parameter
.
VAR_KEYWORD
:
uses_starstar
=
True
return
args
,
uses_starstar


def get_matching_kwargs(func, kwargs):
    """Takes a function and keyword arguments and returns the ones that can be passed."""
    args, uses_startstar = _get_argspec(func)
    if uses_startstar:
        return kwargs.copy()
    else:
        matching_kwargs = dict((k, kwargs[k]) for k in args if k in kwargs)
        return matching_kwargs

def get_matching_kwargs ( func , kwargs ) : args , uses_startstar = _get_argspec ( func ) if uses_startstar : return kwargs . copy ( ) else : matching_kwargs = dict ( ( k , kwargs [ k ] ) for k in args if k in kwargs ) return matching_kwargs

Takes a function and keyword arguments and returns the ones that can be passed .


def format_time(timestamp):
    """Formats timestamp to human readable format"""
    format_string = '%Y_%m_%d_%Hh%Mm%Ss'
    formatted_time = datetime.datetime.fromtimestamp(timestamp).strftime(format_string)
    return formatted_time

def format_time ( timestamp ) : format_string = '%Y_%m_%d_%Hh%Mm%Ss' formatted_time = datetime . datetime . fromtimestamp ( timestamp ) . strftime ( format_string ) return formatted_time


def port_to_tcp(port=None):
    """Returns local tcp address for a given `port`, automatic port if `None`"""
    #address = 'tcp://' + socket.gethostbyname(socket.getfqdn())
    domain_name = socket.getfqdn()
    try:
        addr_list = socket.getaddrinfo(domain_name, None)
    except Exception:
        addr_list = socket.getaddrinfo('127.0.0.1', None)
    family, socktype, proto, canonname, sockaddr = addr_list[0]
    host = convert_ipv6(sockaddr[0])
    address =  'tcp://' + host
    if port is None:
        port = ()
    if not isinstance(port, int):
        # determine port automatically
        context = zmq.Context()
        try:
            socket_ = context.socket(zmq.REP)
            socket_.ipv6 = is_ipv6(address)
            port = socket_.bind_to_random_port(address, *port)
        except Exception:
            print('Could not connect to {} using {}'.format(address, addr_list))
            pypet_root_logger = logging.getLogger('pypet')
            pypet_root_logger.exception('Could not connect to {}'.format(address))
            raise
        socket_.close()
        context.term()
    return address + ':' + str(port)

Returns local tcp address for a given port automatic port if None


def racedirs(path):
    """Like os.makedirs but takes care about race conditions"""
    if os.path.isfile(path):
        raise IOError('Path `%s` is already a file not a directory')
    while True:
        try:
            if os.path.isdir(path):
                # only break if full path has been created or exists
                break
            os.makedirs(path)
        except EnvironmentError as exc:
            # Part of the directory path already exist
            if exc.errno != 17:
                # This error won't be any good
                raise

Like os . makedirs but takes care about race conditions


def _reset(self, index, total, percentage_step, length):
        """Resets to the progressbar to start a new one"""
        self._start_time = datetime.datetime.now()
        self._start_index = index
        self._current_index = index
        self._percentage_step = percentage_step
        self._total = float(total)
        self._total_minus_one = total - 1
        self._length = length
        self._norm_factor = total * percentage_step / 100.0
        self._current_interval = int((index + 1.0) / self._norm_factor)

def _reset ( self , index , total , percentage_step , length ) : self . _start_time = datetime . datetime . now ( ) self . _start_index = index self . _current_index = index self . _percentage_step = percentage_step self . _total = float ( total ) self . _total_minus_one = total - 1 self . _length = length self . _norm_factor = total * percentage_step / 100.0 self . _current_interval = int ( ( index + 1.0 ) / self . _norm_factor )


def _get_remaining(self, index):
        """Calculates remaining time as a string"""
        try:
            current_time = datetime.datetime.now()
            time_delta = current_time - self._start_time
            try:
                total_seconds = time_delta.total_seconds()
            except AttributeError:
                # for backwards-compatibility
                # Python 2.6 does not support `total_seconds`
                total_seconds = ((time_delta.microseconds +
                                    (time_delta.seconds +
                                     time_delta.days * 24 * 3600) * 10 ** 6) / 10.0 ** 6)
            remaining_seconds = int((self._total - self._start_index - 1.0) *
                                    total_seconds / float(index - self._start_index) -
                                    total_seconds)
            remaining_delta = datetime.timedelta(seconds=remaining_seconds)
            remaining_str = ', remaining: ' + str(remaining_delta)
        except ZeroDivisionError:
            remaining_str = ''
        return remaining_str


def f_to_dict(self, copy=True):
        """Returns annotations as dictionary.

        :param copy: Whether to return a shallow copy or the real thing (aka _dict).

        """
        if copy:
            return self._dict.copy()
        else:
            return self._dict

def f_to_dict ( self , copy = True ) : if copy : return self . _dict . copy ( ) else : return self . _dict


def f_remove(self, key):
        """Removes `key` from annotations"""
        key = self._translate_key(key)
        try:
            del self._dict[key]
        except KeyError:
            raise AttributeError('Your annotations do not contain %s' % key)

def
f_remove
(
self
,
key
)
:
key
=
self
.
_translate_key
(
key
)
try
:
del
self
.
_dict
[
key
]
except
KeyError
:
raise
AttributeError
(
'Your annotations do not contain %s'
%
key
)


def f_ann_to_str(self):
        """Returns all annotations lexicographically sorted as a concatenated string."""
        resstr = ''
        for key in sorted(self._dict.keys()):
            resstr += '%s=%s; ' % (key, str(self._dict[key]))
        return resstr[:-2]

def
f_ann_to_str
(
self
)
:
resstr
=
''
for
key
in
sorted
(
self
.
_dict
.
keys
(
)
)
:
resstr
+=
'%s=%s; '
%
(
key
,
str
(
self
.
_dict
[
key
]
)
)
return
resstr
[
:
2
]

Returns all annotations lexicographically sorted as a concatenated string .


def make_ordinary_result(result, key, trajectory=None, reload=True):
    """Turns a given shared data item into a an ordinary one.

    :param result: Result container with shared data
    :param key: The name of the shared data
    :param trajectory:

        The trajectory, only needed if shared data has
        no access to the trajectory, yet.

    :param reload: If data should be reloaded after conversion

    :return: The result

    """
    shared_data = result.f_get(key)
    if trajectory is not None:
        shared_data.traj = trajectory
    shared_data._request_data('make_ordinary')
    result.f_remove(key)
    if reload:
        trajectory.f_load_item(result, load_data=pypetconstants.OVERWRITE_DATA)
    return result

def make_ordinary_result ( result , key , trajectory = None , reload = True ) : shared_data = result . f_get ( key ) if trajectory is not None : shared_data . traj = trajectory shared_data . _request_data ( 'make_ordinary' ) result . f_remove ( key ) if reload : trajectory . f_load_item ( result , load_data = pypetconstants . OVERWRITE_DATA ) return result

Turns a given shared data item into a an ordinary one .


def make_shared_result(result, key, trajectory, new_class=None):
    """Turns an ordinary data item into a shared one.

    Removes the old result from the trajectory and replaces it.
    Empties the given result.

    :param result: The result containing ordinary data
    :param key: Name of ordinary data item
    :param trajectory: Trajectory container
    :param new_class:

        Class of new shared data item.
        Leave `None` for automatic detection.

    :return: The `result`

    """
    data = result.f_get(key)
    if new_class is None:
        if isinstance(data, ObjectTable):
            new_class = SharedTable
        elif isinstance(data, pd.DataFrame):
            new_class = SharedPandasFrame
        elif isinstance(data, (tuple, list)):
            new_class = SharedArray
        elif isinstance(data, (np.ndarray, np.matrix)):
            new_class = SharedCArray
        else:
            raise RuntimeError('Your data `%s` is not understood.' % key)
    shared_data = new_class(result.f_translate_key(key), result, trajectory=trajectory)
    result[key] = shared_data
    shared_data._request_data('make_shared')
    return result

def
make_shared_result
(
result
,
key
,
trajectory
,
new_class
=
)
:
data
=
result
.
f_get
(
key
)
if
new_class
is
:
if
isinstance
(
data
,
ObjectTable
)
:
new_class
=
SharedTable
elif
isinstance
(
data
,
pd
.
DataFrame
)
:
new_class
=
SharedPandasFrame
elif
isinstance
(
data
,
(
tuple
,
list
)
)
:
new_class
=
SharedArray
elif
isinstance
(
data
,
(
np
.
ndarray
,
np
.
matrix
)
)
:
new_class
=
SharedCArray
else
:
raise
RuntimeError
(
'Your data `%s` is not understood.'
%
key
)
shared_data
=
new_class
(
result
.
f_translate_key
(
key
)
,
result
,
trajectory
=
trajectory
)
result
[
key
]
=
shared_data
shared_data
.
_request_data
(
'make_shared'
)
return
result

Turns an ordinary data item into a shared one .


def create_shared_data(self, **kwargs):
        """Creates shared data on disk with a StorageService on disk.

        Needs to be called before shared data can be used later on.

        Actual arguments of ``kwargs`` depend on the type of data to be
        created. For instance, creating an array one can use the keyword
        ``obj`` to pass a numpy array (``obj=np.zeros((10,20,30))``).
        Whereas for a PyTables table may need a description dictionary
        (``description={'column_1': pt.StringCol(2, pos=0),'column_2': pt.FloatCol( pos=1)}``)
        Refer to the PyTables documentation on how to create tables.

        """
        if 'flag' not in kwargs:
            kwargs['flag'] = self.FLAG
        if 'data' in kwargs:
            kwargs['obj'] = kwargs.pop('data')
        if 'trajectory' in kwargs:
            self.traj = kwargs.pop('trajectory')
        if 'traj' in kwargs:
            self.traj = kwargs.pop('traj')
        if 'name' in kwargs:
            self.name = kwargs.pop['name']
        if 'parent' in kwargs:
            self.parent = kwargs.pop('parent')
            if self.name is not None:
                self.parent[self.name] = self
        return self._request_data('create_shared_data', kwargs=kwargs)

def create_shared_data ( self , * * kwargs ) : if 'flag' not in kwargs : kwargs [ 'flag' ] = self . FLAG if 'data' in kwargs : kwargs [ 'obj' ] = kwargs . pop ( 'data' ) if 'trajectory' in kwargs : self . traj = kwargs . pop ( 'trajectory' ) if 'traj' in kwargs : self . traj = kwargs . pop ( 'traj' ) if 'name' in kwargs : self . name = kwargs . pop [ 'name' ] if 'parent' in kwargs : self . parent = kwargs . pop ( 'parent' ) if self . name is not None : self . parent [ self . name ] = self return self . _request_data ( 'create_shared_data' , kwargs = kwargs )

Creates shared data on disk with a StorageService on disk .


def _request_data(self, request, args=None, kwargs=None):
        """Interface with the underlying storage.

        Passes request to the StorageService that performs the appropriate action.
        For example, given a shared table ``t``.
        ``t.remove_row(4)`` is parsed into ``request='remove_row', args=(4,)`` and
        passed onto the storage service. In case of the HDF5StorageService,
        this is again translated back into ``hdf5_table_node.remove_row(4)``.

        """
        return self._storage_service.store(pypetconstants.ACCESS_DATA,
                                           self.parent.v_full_name,
                                           self.name,
                                           request, args, kwargs,
                                           trajectory_name=self.traj.v_name)

def _request_data ( self , request , args = None , kwargs = None ) : return self . _storage_service . store ( pypetconstants . ACCESS_DATA , self . parent . v_full_name , self . name , request , args , kwargs , trajectory_name = self . traj . v_name )


def get_data_node(self):
        """Returns the actula node of the underlying data.

        In case one uses HDF5 this will be the HDF5 leaf node.

        """
        if not self._storage_service.is_open:
            warnings.warn('You requesting the data item but your store is not open, '
                          'the item itself will be closed, too!',
                          category=RuntimeWarning)
        return self._request_data('__thenode__')

Returns the actula node of the underlying data .


def _supports(self, item):
        """Checks if outer data structure is supported."""
        result = super(SharedResult, self)._supports(item)
        result = result or type(item) in SharedResult.SUPPORTED_DATA
        return result

def _supports ( self , item ) : result = super ( SharedResult , self ) . _supports ( item ) result = result or type ( item ) in SharedResult . SUPPORTED_DATA return result

Checks if outer data structure is supported .


def create_shared_data(self, name=None, **kwargs):
        """Calls the corresponding function of the shared data item"""
        if name is None:
            item = self.f_get()
        else:
            item = self.f_get(name)
        return item.create_shared_data(**kwargs)

def create_shared_data ( self , name = None , * * kwargs ) : if name is None : item = self . f_get ( ) else : item = self . f_get ( name ) return item . create_shared_data ( * * kwargs )


def manipulate_multiproc_safe(traj):
    """ Target function that manipulates the trajectory.

    Stores the current name of the process into the trajectory and
    **overwrites** previous settings.

    :param traj:

        Trajectory container with multiprocessing safe storage service

    """

    # Manipulate the data in the trajectory
    traj.last_process_name = mp.current_process().name
    # Store the manipulated data
    traj.results.f_store(store_data=3)

def
manipulate_multiproc_safe
(
traj
)
:
# Manipulate the data in the trajectory
traj
.
last_process_name
=
mp
.
current_process
(
)
.
name
# Store the manipulated data
traj
.
results
.
f_store
(
store_data
=
3
)

Target function that manipulates the trajectory .


def _lock(self, name, client_id, request_id):
        """Hanldes locking of locks

        If a lock is already locked sends a WAIT command,
        else LOCKs it and sends GO.

        Complains if a given client re-locks a lock without releasing it before.

        """
        if name in self._locks:
            other_client_id, other_request_id = self._locks[name]
            if other_client_id == client_id:
                response = (self.LOCK_ERROR + self.DELIMITER +
                            'Re-request of lock `%s` (old request id `%s`) by `%s` '
                            '(request id `%s`)' % (name, client_id, other_request_id, request_id))
                self._logger.warning(response)
                return response
            else:
                return self.WAIT
        else:
            self._locks[name] = (client_id, request_id)
            return self.GO


def send_done(self):
        """Notifies the Server to shutdown"""
        self.start(test_connection=False)
        self._logger.debug('Sending shutdown signal')
        self._req_rep(ZMQServer.DONE)

def
send_done
(
self
)
:
self
.
start
(
test_connection
=
False
)
self
.
_logger
.
debug
(
'Sending shutdown signal'
)
self
.
_req_rep
(
ZMQServer
.
DONE
)


def finalize(self):
        """Closes socket and terminates context

        NO-OP if already closed.

        """
        if self._context is not None:
            if self._socket is not None:
                self._close_socket(confused=False)
            self._context.term()
            self._context = None
            self._poll = None

def finalize ( self ) : if self . _context is not None : if self . _socket is not None : self . _close_socket ( confused = False ) self . _context . term ( ) self . _context = None self . _poll = None


def start(self, test_connection=True):
        """Starts connection to server if not existent.

        NO-OP if connection is already established.
        Makes ping-pong test as well if desired.

        """
        if self._context is None:
            self._logger.debug('Starting Client')
            self._context = zmq.Context()
            self._poll = zmq.Poller()
            self._start_socket()
            if test_connection:
                self.test_ping()

def
start
(
self
,
test_connection
=
True
)
:
if
self
.
_context
is
:
self
.
_logger
.
debug
(
'Starting Client'
)
self
.
_context
=
zmq
.
Context
(
)
self
.
_poll
=
zmq
.
Poller
(
)
self
.
_start_socket
(
)
if
test_connection
:
self
.
test_ping
(
)

Starts connection to server if not existent .


def _req_rep_retry(self, request):
        """Returns response and number of retries"""
        retries_left = self.RETRIES
        while retries_left:
            self._logger.log(1, 'Sending REQ `%s`', request)
            self._send_request(request)
            socks = dict(self._poll.poll(self.TIMEOUT))
            if socks.get(self._socket) == zmq.POLLIN:
                response = self._receive_response()
                self._logger.log(1, 'Received REP `%s`', response)
                return response, self.RETRIES - retries_left
            else:
                self._logger.debug('No response from server (%d retries left)' %
                                   retries_left)
                self._close_socket(confused=True)
                retries_left -= 1
                if retries_left == 0:
                    raise RuntimeError('Server seems to be offline!')
                time.sleep(self.SLEEP)
                self._start_socket()


def acquire(self):
        """Acquires lock and returns `True`

        Blocks until lock is available.

        """
        self.start(test_connection=False)
        while True:
            str_response, retries = self._req_rep_retry(LockerServer.LOCK)
            response = str_response.split(LockerServer.DELIMITER)
            if response[0] == LockerServer.GO:
                return True
            elif response[0] == LockerServer.LOCK_ERROR and retries > 0:
                # Message was sent but Server response was lost and we tried again
                self._logger.error(str_response + '; Probably due to retry')
                return True
            elif response[0] == LockerServer.WAIT:
                time.sleep(self.SLEEP)
            else:
                raise RuntimeError('Response `%s` not understood' % response)


def listen(self):
        """ Handles listening requests from the client.

        There are 4 types of requests:

        1- Check space in the queue
        2- Tests the socket
        3- If there is a space, it sends data
        4- after data is sent, puts it to queue for storing

        """
        count = 0
        self._start()
        while True:
            result = self._socket.recv_pyobj()

            if isinstance(result, tuple):
                request, data = result
            else:
                request = result
                data = None

            if request == self.SPACE:
                if self.queue.qsize() + count < self.queue_maxsize:
                    self._socket.send_string(self.SPACE_AVAILABLE)
                    count += 1
                else:
                    self._socket.send_string(self.SPACE_NOT_AVAILABLE)

            elif request == self.PING:
                self._socket.send_string(self.PONG)

            elif request == self.DATA:
                self._socket.send_string(self.STORING)
                self.queue.put(data)
                count -= 1

            elif request == self.DONE:
                self._socket.send_string(ZMQServer.CLOSED)
                self.queue.put(('DONE', [], {}))
                self._close()
                break

            else:
                raise RuntimeError('I did not understand your request %s' % request)

def
listen
(
self
)
:
count
=
0
self
.
_start
(
)
while
True
:
result
=
self
.
_socket
.
recv_pyobj
(
)
if
isinstance
(
result
,
tuple
)
:
request
,
data
=
result
else
:
request
=
result
data
=
if
request
==
self
.
SPACE
:
if
self
.
queue
.
qsize
(
)
+
count
<
self
.
queue_maxsize
:
self
.
_socket
.
send_string
(
self
.
SPACE_AVAILABLE
)
count
+=
1
else
:
self
.
_socket
.
send_string
(
self
.
SPACE_NOT_AVAILABLE
)
elif
request
==
self
.
PING
:
self
.
_socket
.
send_string
(
self
.
PONG
)
elif
request
==
self
.
DATA
:
self
.
_socket
.
send_string
(
self
.
STORING
)
self
.
queue
.
put
(
data
)
count
-=
1
elif
request
==
self
.
DONE
:
self
.
_socket
.
send_string
(
ZMQServer
.
CLOSED
)
self
.
queue
.
put
(
(
'DONE'
,
[
]
,
{
}
)
)
self
.
_close
(
)
break
else
:
raise
RuntimeError
(
'I did not understand your request %s'
%
request
)

Handles listening requests from the client .


def put(self, data, block=True):
        """ If there is space it sends data to server

        If no space in the queue

        It returns the request in every 10 millisecond

        until there will be space in the queue.

        """

        self.start(test_connection=False)
        while True:
            response = self._req_rep(QueuingServerMessageListener.SPACE)
            if response == QueuingServerMessageListener.SPACE_AVAILABLE:
                self._req_rep((QueuingServerMessageListener.DATA, data))
                break
            else:
                time.sleep(0.01)

def put ( self , data , block = True ) : self . start ( test_connection = False ) while True : response = self . _req_rep ( QueuingServerMessageListener . SPACE ) if response == QueuingServerMessageListener . SPACE_AVAILABLE : self . _req_rep ( ( QueuingServerMessageListener . DATA , data ) ) break else : time . sleep ( 0.01 )


def _detect_fork(self):
        """Detects if lock client was forked.

        Forking is detected by comparing the PID of the current
        process with the stored PID.

        """
        if self._pid is None:
            self._pid = os.getpid()
        if self._context is not None:
            current_pid = os.getpid()
            if current_pid != self._pid:
                self._logger.debug('Fork detected: My pid `%s` != os pid `%s`. '
                                   'Restarting connection.' % (str(self._pid), str(current_pid)))
                self._context = None
                self._pid = current_pid


def _handle_data(self, msg, args, kwargs):
        """Handles data and returns `True` or `False` if everything is done."""
        stop = False
        try:
            if msg == 'DONE':
                stop = True
            elif msg == 'STORE':
                if 'msg' in kwargs:
                    store_msg = kwargs.pop('msg')
                else:
                    store_msg = args[0]
                    args = args[1:]
                if 'stuff_to_store' in kwargs:
                    stuff_to_store = kwargs.pop('stuff_to_store')
                else:
                    stuff_to_store = args[0]
                    args = args[1:]
                trajectory_name = kwargs['trajectory_name']
                if self._trajectory_name != trajectory_name:
                    if self._storage_service.is_open:
                        self._close_file()
                    self._trajectory_name = trajectory_name
                    self._open_file()
                self._storage_service.store(store_msg, stuff_to_store, *args, **kwargs)
                self._storage_service.store(pypetconstants.FLUSH, None)
                self._check_and_collect_garbage()
            else:
                raise RuntimeError('You queued something that was not '
                                   'intended to be queued. I did not understand message '
                                   '`%s`.' % msg)
        except Exception:
            self._logger.exception('ERROR occurred during storing!')
            time.sleep(0.01)
            pass  # We don't want to kill the queue process in case of an error

        return stop

Handles data and returns True or False if everything is done .


def run(self):
        """Starts listening to the queue."""
        try:
            while True:
                msg, args, kwargs = self._receive_data()
                stop = self._handle_data(msg, args, kwargs)
                if stop:
                    break
        finally:
            if self._storage_service.is_open:
                self._close_file()
            self._trajectory_name = ''

def run ( self ) : try : while True : msg , args , kwargs = self . _receive_data ( ) stop = self . _handle_data ( msg , args , kwargs ) if stop : break finally : if self . _storage_service . is_open : self . _close_file ( ) self . _trajectory_name = ''


def _receive_data(self):
        """Gets data from queue"""
        result = self.queue.get(block=True)
        if hasattr(self.queue, 'task_done'):
            self.queue.task_done()
        return result

def _receive_data ( self ) : result = self . queue . get ( block = True ) if hasattr ( self . queue , 'task_done' ) : self . queue . task_done ( ) return result


def _receive_data(self):
        """Gets data from pipe"""
        while True:
            while len(self._buffer) < self.max_size and self.conn.poll():
                data = self._read_chunks()
                if data is not None:
                    self._buffer.append(data)
            if len(self._buffer) > 0:
                return self._buffer.popleft()

def _receive_data ( self ) : while True : while len ( self . _buffer ) < self . max_size and self . conn . poll ( ) : data = self . _read_chunks ( ) if data is not None : self . _buffer . append ( data ) if len ( self . _buffer ) > 0 : return self . _buffer . popleft ( )


def store(self, *args, **kwargs):
        """Acquires a lock before storage and releases it afterwards."""
        try:
            self.acquire_lock()
            return self._storage_service.store(*args, **kwargs)
        finally:
            if self.lock is not None:
                try:
                    self.release_lock()
                except RuntimeError:
                    self._logger.error('Could not release lock `%s`!' % str(self.lock))

def
store
(
self
,
*
args
,
*
*
kwargs
)
:
try
:
self
.
acquire_lock
(
)
return
self
.
_storage_service
.
store
(
*
args
,
*
*
kwargs
)
finally
:
if
self
.
lock
is
not
:
try
:
self
.
release_lock
(
)
except
RuntimeError
:
self
.
_logger
.
error
(
'Could not release lock `%s`!'
%
str
(
self
.
lock
)
)

Acquires a lock before storage and releases it afterwards .


def store(self, msg, stuff_to_store, *args, **kwargs):
        """Simply keeps a reference to the stored data """
        trajectory_name = kwargs['trajectory_name']
        if trajectory_name not in self.references:
            self.references[trajectory_name] = []
        self.references[trajectory_name].append((msg, cp.copy(stuff_to_store), args, kwargs))

def store ( self , msg , stuff_to_store , * args , * * kwargs ) : trajectory_name = kwargs [ 'trajectory_name' ] if trajectory_name not in self . references : self . references [ trajectory_name ] = [ ] self . references [ trajectory_name ] . append ( ( msg , cp . copy ( stuff_to_store ) , args , kwargs ) )


def store_references(self, references):
        """Stores references to disk and may collect garbage."""
        for trajectory_name in references:
            self._storage_service.store(pypetconstants.LIST, references[trajectory_name], trajectory_name=trajectory_name)
        self._check_and_collect_garbage()

def store_references ( self , references ) : for trajectory_name in references : self . _storage_service . store ( pypetconstants . LIST , references [ trajectory_name ] , trajectory_name = trajectory_name ) self . _check_and_collect_garbage ( )

Stores references to disk and may collect garbage .


def parse_config(init_func):
    """Decorator wrapping the environment to use a config file"""
    @functools.wraps(init_func)
    def new_func(env, *args, **kwargs):
        config_interpreter = ConfigInterpreter(kwargs)
        # Pass the config data to the kwargs
        new_kwargs = config_interpreter.interpret()
        init_func(env, *args, **new_kwargs)
        # Add parameters and config data from the `.ini` file
        config_interpreter.add_parameters(env.traj)
    return new_func


def _collect_section(self, section):
        """Collects all settings within a section"""
        kwargs = {}
        try:
            if self.parser.has_section(section):
                options = self.parser.options(section)
                for option in options:
                    str_val = self.parser.get(section, option)
                    val = ast.literal_eval(str_val)
                    kwargs[option] = val
            return kwargs
        except:
            raise

def _collect_section ( self , section ) : kwargs = { } try : if self . parser . has_section ( section ) : options = self . parser . options ( section ) for option in options : str_val = self . parser . get ( section , option ) val = ast . literal_eval ( str_val ) kwargs [ option ] = val return kwargs except : raise


def _collect_config(self):
        """Collects all info from three sections"""
        kwargs = {}
        sections = ('storage_service', 'trajectory', 'environment')
        for section in sections:
            kwargs.update(self._collect_section(section))
        return kwargs

def _collect_config ( self ) : kwargs = { } sections = ( 'storage_service' , 'trajectory' , 'environment' ) for section in sections : kwargs . update ( self . _collect_section ( section ) ) return kwargs


def interpret(self):
        """Copies parsed arguments into the kwargs passed to the environment"""
        if self.config_file:
            new_kwargs = self._collect_config()
            for key in new_kwargs:
                # Already specified kwargs take precedence over the ini file
                if key not in self.kwargs:
                    self.kwargs[key] = new_kwargs[key]
            if not use_simple_logging(self.kwargs) and 'log_config' not in self.kwargs:
                self.kwargs['log_config'] = self.config_file
        return self.kwargs


def add_parameters(self, traj):
        """Adds parameters and config from the `.ini` file to the trajectory"""
        if self.config_file:
            parameters = self._collect_section('parameters')
            for name in parameters:
                value = parameters[name]
                if not isinstance(value, tuple):
                    value = (value,)
                traj.f_add_parameter(name, *value)
            config = self._collect_section('config')
            for name in config:
                value = config[name]
                if not isinstance(value, tuple):
                    value = (value,)
                traj.f_add_config(name, *value)

def add_parameters ( self , traj ) : if self . config_file : parameters = self . _collect_section ( 'parameters' ) for name in parameters : value = parameters [ name ] if not isinstance ( value , tuple ) : value = ( value , ) traj . f_add_parameter ( name , * value ) config = self . _collect_section ( 'config' ) for name in config : value = config [ name ] if not isinstance ( value , tuple ) : value = ( value , ) traj . f_add_config ( name , * value )

Adds parameters and config from the . ini file to the trajectory


def convert_rule(rule_number):
    """ Converts a rule given as an integer into a binary list representation.

    It reads from left to right (contrary to the Wikipedia article given below),
    i.e. the 2**0 is found on the left hand side and 2**7 on the right.

    For example:

        ``convert_rule(30)`` returns [0, 1, 1, 1, 1, 0, 0, 0]

    The resulting binary list can be interpreted as
    the following transition table:

         neighborhood  new cell state
                000     0
                001     1
                010     1
                011     1
                100     1
                101     0
                110     0
                111     0

    For more information about this rule
    see: http://en.wikipedia.org/wiki/Rule_30

    """
    binary_rule = [(rule_number // pow(2,i)) % 2 for i in range(8)]
    return np.array(binary_rule)

def convert_rule ( rule_number ) : binary_rule = [ ( rule_number // pow ( 2 , i ) ) % 2 for i in range ( 8 ) ] return np . array ( binary_rule )

Converts a rule given as an integer into a binary list representation .


def make_initial_state(name, ncells, seed=42):
    """ Creates an initial state for the automaton.

    :param name:

        Either ``'single'`` for a single live cell in the middle of the cell ring,
        or ``'random'`` for uniformly distributed random pattern of zeros and ones.

    :param ncells: Number of cells in the automaton

    :param seed: Random number seed for the ``#random'`` condition

    :return: Numpy array of zeros and ones (or just a one lonely one surrounded by zeros)

    :raises: ValueError if the ``name`` is unknown

    """
    if name == 'single':
        just_one_cell = np.zeros(ncells)
        just_one_cell[int(ncells/2)] = 1.0
        return just_one_cell
    elif name == 'random':
        np.random.seed(seed)
        random_init = np.random.randint(2, size=ncells)
        return random_init
    else:
        raise ValueError('I cannot handel your initial state `%s`.' % name)

Creates an initial state for the automaton .


def plot_pattern(pattern, rule_number, filename):
    """ Plots an automaton ``pattern`` and stores the image under a given ``filename``.

    For axes labels the ``rule_number`` is also required.

    """
    plt.figure()
    plt.imshow(pattern)
    plt.xlabel('Cell No.')
    plt.ylabel('Time Step')
    plt.title('CA with Rule %s' % str(rule_number))
    plt.savefig(filename)
    #plt.show()
    plt.close()

def
plot_pattern
(
pattern
,
rule_number
,
filename
)
:
plt
.
figure
(
)
plt
.
imshow
(
pattern
)
plt
.
xlabel
(
'Cell No.'
)
plt
.
ylabel
(
'Time Step'
)
plt
.
title
(
'CA with Rule %s'
%
str
(
rule_number
)
)
plt
.
savefig
(
filename
)
#plt.show()
plt
.
close
(
)

Plots an automaton pattern and stores the image under a given filename .


def cellular_automaton_1D(initial_state, rule_number, steps):
    """ Simulates a 1 dimensional cellular automaton.

    :param initial_state:

        The initial state of *dead* and *alive* cells as a 1D numpy array.
        It's length determines the size of the simulation.

    :param rule_number:

        The update rule as an integer from 0 to 255.

    :param steps:

        Number of cell iterations

    :return:

        A 2D numpy array (steps x len(initial_state)) containing zeros and ones representing
        the automaton development over time.

    """

    ncells = len(initial_state)
    # Create an array for the full pattern
    pattern = np.zeros((steps, ncells))

    # Pass initial state:
    pattern[0,:] = initial_state

    # Get the binary rule list
    binary_rule = convert_rule(rule_number)

    # Conversion list to get the position in the binary rule list
    neighbourhood_factors = np.array([1, 2, 4])

    # Iterate over all steps to compute the CA
    all_cells = range(ncells)
    for step in range(steps-1):
        current_row = pattern[step, :]
        next_row = pattern[step+1, :]
        for irun in all_cells:
            # Get the neighbourhood
            neighbour_indices = range(irun - 1, irun + 2)
            neighbourhood = np.take(current_row, neighbour_indices, mode='wrap')
            # Convert neighborhood to decimal
            decimal_neighborhood = int(np.sum(neighbourhood * neighbourhood_factors))
            # Get next state from rule book
            next_state = binary_rule[decimal_neighborhood]
            # Update next state of cell
            next_row[irun] = next_state

    return pattern

Simulates a 1 dimensional cellular automaton .


def main():
    """ Main simulation function """
    rules_to_test = [10, 30, 90, 110, 184]  # rules we want to explore:
    steps = 250  # cell iterations
    ncells = 400  # number of cells
    seed = 100042  # RNG seed
    initial_states = ['single', 'random']  # Initial states we want to explore

    # create a folder for the plots and the data
    folder = os.path.join(os.getcwd(), 'experiments', 'ca_patterns_original')
    if not os.path.isdir(folder):
        os.makedirs(folder)
    filename = os.path.join(folder, 'all_patterns.p')

    print('Computing all patterns')
    all_patterns = []  # list containing the simulation results
    for idx, rule_number in enumerate(rules_to_test):
        # iterate over all rules
        for initial_name in initial_states:
            # iterate over the initial states

            # make the initial state
            initial_state = make_initial_state(initial_name, ncells, seed=seed)
            # simulate the automaton
            pattern = cellular_automaton_1D(initial_state, rule_number, steps)
            # keep the resulting pattern
            all_patterns.append((rule_number, initial_name, pattern))

        # Print a progressbar, because I am always impatient
        #  (ok that's already from pypet, but it's really handy!)
        progressbar(idx, len(rules_to_test), reprint=True)

    # Store all patterns to disk
    with open(filename, 'wb') as file:
        pickle.dump(all_patterns, file=file)

    # Finally print all patterns
    print('Plotting all patterns')
    for idx, pattern_tuple in enumerate(all_patterns):
        rule_number, initial_name, pattern = pattern_tuple
        # Plot the pattern
        filename = os.path.join(folder, 'rule_%s_%s.png' % (str(rule_number), initial_name))
        plot_pattern(pattern, rule_number, filename)
        progressbar(idx, len(all_patterns), reprint=True)


def signal_update(self):
        """Signals the process timer.

        If more time than the display time has passed a message is emitted.

        """
        if not self.active:
            return

        self._updates += 1
        current_time = time.time()
        dt = current_time - self._last_time
        if dt > self._display_time:
            dfullt = current_time - self._start_time
            seconds = int(dfullt) % 60
            minutes = int(dfullt) / 60
            if minutes == 0:
                formatted_time = '%ds' % seconds
            else:
                formatted_time = '%dm%02ds' % (minutes, seconds)
            nodespersecond = self._updates / dfullt
            message = 'Processed %d nodes in %s (%.2f nodes/s).' % \
                      (self._updates, formatted_time, nodespersecond)
            self._logger.info(message)
            self._last_time = current_time


def _overview_group(self):
        """Direct link to the overview group"""
        if self._overview_group_ is None:
            self._overview_group_ = self._all_create_or_get_groups('overview')[0]
        return self._overview_group_

def _overview_group ( self ) : if self . _overview_group_ is None : self . _overview_group_ = self . _all_create_or_get_groups ( 'overview' ) [ 0 ] return self . _overview_group_


def load(self, msg, stuff_to_load, *args, **kwargs):
        """Loads a particular item from disk.

        The storage service always accepts these parameters:

        :param trajectory_name: Name of current trajectory and name of top node in hdf5 file.

        :param trajectory_index:

            If no `trajectory_name` is provided, you can specify an integer index.
            The trajectory at the index position in the hdf5 file is considered to loaded.
            Negative indices are also possible for reverse indexing.

        :param filename: Name of the hdf5 file

        The following messages (first argument msg) are understood and the following arguments
        can be provided in combination with the message:

            * :const:`pypet.pypetconstants.TRAJECTORY` ('TRAJECTORY')

                Loads a trajectory.

                :param stuff_to_load: The trajectory

                :param as_new: Whether to load trajectory as new

                :param load_parameters: How to load parameters and config

                :param load_derived_parameters: How to load derived parameters

                :param load_results: How to load results

                :param force: Force load in case there is a pypet version mismatch

                You can specify how to load the parameters, derived parameters and results
                as follows:

                :const:`pypet.pypetconstants.LOAD_NOTHING`: (0)

                    Nothing is loaded

                :const:`pypet.pypetconstants.LOAD_SKELETON`: (1)

                    The skeleton including annotations are loaded, i.e. the items are empty.
                    Non-empty items in RAM are left untouched.

                :const:`pypet.pypetconstants.LOAD_DATA`: (2)

                    The whole data is loaded.
                    Only empty or in RAM non-existing instance are filled with the
                    data found on disk.

                :const:`pypet.pypetconstants.OVERWRITE_DATA`: (3)

                    The whole data is loaded.
                    If items that are to be loaded are already in RAM and not empty,
                    they are emptied and new data is loaded from disk.

            * :const:`pypet.pypetconstants.LEAF` ('LEAF')

                Loads a parameter or result.

                :param stuff_to_load: The item to be loaded

                :param load_data: How to load data

                :param load_only:

                    If you load a result, you can partially load it and ignore the
                    rest of the data. Just specify the name of the data you want to load.
                    You can also provide a list,
                    for example `load_only='spikes'`, `load_only=['spikes','membrane_potential']`.

                    Issues a warning if items cannot be found.

                :param load_except:

                    If you load a result you can partially load in and specify items
                    that should NOT be loaded here. You cannot use `load_except` and
                    `load_only` at the same time.

            * :const:`pypet.pyetconstants.GROUP`

                Loads a group a node (comment and annotations)

                :param recursive:

                    Recursively loads everything below

                :param load_data:

                    How to load stuff if ``recursive=True``
                    accepted values as above for loading the trajectory

                :param max_depth:

                    Maximum depth in case of recursion. `None` for no limit.

            * :const:`pypet.pypetconstants.TREE` ('TREE')

                Loads a whole subtree

                :param stuff_to_load: The parent node (!) not the one where loading starts!

                :param child_name: Name of child node that should be loaded

                :param recursive: Whether to load recursively the subtree below child

                :param load_data:

                    How to load stuff, accepted values as above for loading the trajectory

                :param max_depth:

                    Maximum depth in case of recursion. `None` for no limit.

                :param trajectory: The trajectory object

            * :const:`pypet.pypetconstants.LIST` ('LIST')

                Analogous to :ref:`storing lists <store-lists>`

        :raises:

            NoSuchServiceError if message or data is not understood

            DataNotInStorageError if data to be loaded cannot be found on disk

        """
        opened = True
        try:

            opened = self._srvc_opening_routine('r', kwargs=kwargs)

            if msg == pypetconstants.TRAJECTORY:
                self._trj_load_trajectory(stuff_to_load, *args, **kwargs)

            elif msg == pypetconstants.LEAF:
                self._prm_load_parameter_or_result(stuff_to_load, *args, **kwargs)

            elif msg == pypetconstants.GROUP:
                self._grp_load_group(stuff_to_load, *args, **kwargs)

            elif msg == pypetconstants.TREE:
                self._tree_load_sub_branch(stuff_to_load, *args, **kwargs)

            elif msg == pypetconstants.LIST:
                self._srvc_load_several_items(stuff_to_load, *args, **kwargs)

            else:
                raise pex.NoSuchServiceError('I do not know how to handle `%s`' % msg)

        except pt.NoSuchNodeError as exc:
            self._logger.error('Failed loading  `%s`' % str(stuff_to_load))
            raise pex.DataNotInStorageError(repr(exc))
        except:
            self._logger.error('Failed loading  `%s`' % str(stuff_to_load))
            raise
        finally:
            self._srvc_closing_routine(opened)

def
load
(
self
,
msg
,
stuff_to_load
,
*
args
,
*
*
kwargs
)
:
opened
=
True
try
:
opened
=
self
.
_srvc_opening_routine
(
'r'
,
kwargs
=
kwargs
)
if
msg
==
pypetconstants
.
TRAJECTORY
:
self
.
_trj_load_trajectory
(
stuff_to_load
,
*
args
,
*
*
kwargs
)
elif
msg
==
pypetconstants
.
LEAF
:
self
.
_prm_load_parameter_or_result
(
stuff_to_load
,
*
args
,
*
*
kwargs
)
elif
msg
==
pypetconstants
.
GROUP
:
self
.
_grp_load_group
(
stuff_to_load
,
*
args
,
*
*
kwargs
)
elif
msg
==
pypetconstants
.
TREE
:
self
.
_tree_load_sub_branch
(
stuff_to_load
,
*
args
,
*
*
kwargs
)
elif
msg
==
pypetconstants
.
LIST
:
self
.
_srvc_load_several_items
(
stuff_to_load
,
*
args
,
*
*
kwargs
)
else
:
raise
pex
.
NoSuchServiceError
(
'I do not know how to handle `%s`'
%
msg
)
except
pt
.
NoSuchNodeError
as
exc
:
self
.
_logger
.
error
(
'Failed loading  `%s`'
%
str
(
stuff_to_load
)
)
raise
pex
.
DataNotInStorageError
(
repr
(
exc
)
)
except
:
self
.
_logger
.
error
(
'Failed loading  `%s`'
%
str
(
stuff_to_load
)
)
raise
finally
:
self
.
_srvc_closing_routine
(
opened
)


def store(self, msg, stuff_to_store, *args, **kwargs):
        """ Stores a particular item to disk.

        The storage service always accepts these parameters:

        :param trajectory_name: Name or current trajectory and name of top node in hdf5 file

        :param filename: Name of the hdf5 file

        :param file_title: If file needs to be created, assigns a title to the file.

        The following messages (first argument msg) are understood and the following arguments
        can be provided in combination with the message:

            * :const:`pypet.pypetconstants.PREPARE_MERGE` ('PREPARE_MERGE'):

                Called to prepare a trajectory for merging, see also 'MERGE' below.

                Will also be called if merging cannot happen within the same hdf5 file.
                Stores already enlarged parameters and updates meta information.

                :param stuff_to_store: Trajectory that is about to be extended by another one

                :param changed_parameters:

                    List containing all parameters that were enlarged due to merging

                :param old_length:

                    Old length of trajectory before merge

            * :const:`pypet.pypetconstants.MERGE` ('MERGE')

                Note that before merging within HDF5 file, the storage service will be called
                with msg='PREPARE_MERGE' before, see above.

                Raises a ValueError if the two trajectories are not stored within the very
                same hdf5 file. Then the current trajectory needs to perform the merge slowly
                item by item.

                Merges two trajectories, parameters are:

                :param stuff_to_store: The trajectory data is merged into

                :param other_trajectory_name: Name of the other trajectory

                :param rename_dict:

                    Dictionary containing the old result and derived parameter names in the
                    other trajectory and their new names in the current trajectory.

                :param move_nodes:

                    Whether to move the nodes from the other to the current trajectory

                :param delete_trajectory:

                    Whether to delete the other trajectory after merging.

            * :const:`pypet.pypetconstants.BACKUP` ('BACKUP')

                :param stuff_to_store: Trajectory to be backed up

                :param backup_filename:

                    Name of file where to store the backup. If None the backup file will be in
                    the same folder as your hdf5 file and named 'backup_XXXXX.hdf5'
                    where 'XXXXX' is the name of your current trajectory.

            * :const:`pypet.pypetconstants.TRAJECTORY` ('TRAJECTORY')

                Stores the whole trajectory

                :param stuff_to_store: The trajectory to be stored

                :param only_init:

                    If you just want to initialise the store. If yes, only meta information about
                    the trajectory is stored and none of the nodes/leaves within the trajectory.

                :param store_data:

                    How to store data, the following settings are understood:

                     :const:`pypet.pypetconstants.STORE_NOTHING`: (0)

                        Nothing is stored

                    :const:`pypet.pypetconstants.STORE_DATA_SKIPPING`: (1)

                        Data of not already stored nodes is stored

                    :const:`pypet.pypetconstants.STORE_DATA`: (2)

                        Data of all nodes is stored. However, existing data on disk is left
                        untouched.

                    :const:`pypet.pypetconstants.OVERWRITE_DATA`: (3)

                        Data of all nodes is stored and data on disk is overwritten.
                        May lead to fragmentation of the HDF5 file. The user is adviced
                        to recompress the file manually later on.

            * :const:`pypet.pypetconstants.SINGLE_RUN` ('SINGLE_RUN')

                :param stuff_to_store: The trajectory

                :param store_data: How to store data see above

                :param store_final: If final meta info should be stored

            * :const:`pypet.pypetconstants.LEAF`

                Stores a parameter or result

                Note that everything that is supported by the storage service and that is
                stored to disk will be perfectly recovered.
                For instance, you store a tuple of numpy 32 bit integers, you will get a tuple
                of numpy 32 bit integers after loading independent of the platform!

                :param stuff_to_sore: Result or parameter to store

                    In order to determine what to store, the function '_store' of the parameter or
                    result is called. This function returns a dictionary with name keys and data to
                    store as values. In order to determine how to store the data, the storage flags
                    are considered, see below.

                    The function '_store' has to return a dictionary containing values only from
                    the following objects:

                        * python natives (int, long, str, bool, float, complex),

                        *
                            numpy natives, arrays and matrices of type np.int8-64, np.uint8-64,
                            np.float32-64, np.complex, np.str

                        *

                            python lists and tuples of the previous types
                            (python natives + numpy natives and arrays)
                            Lists and tuples are not allowed to be nested and must be
                            homogeneous, i.e. only contain data of one particular type.
                            Only integers, or only floats, etc.

                        *

                            python dictionaries of the previous types (not nested!), data can be
                            heterogeneous, keys must be strings. For example, one key-value-pair
                            of string and int and one key-value pair of string and float, and so
                            on.

                        * pandas DataFrames_

                        * :class:`~pypet.parameter.ObjectTable`

                    .. _DataFrames: http://pandas.pydata.org/pandas-docs/dev/dsintro.html#dataframe

                    The keys from the '_store' dictionaries determine how the data will be named
                    in the hdf5 file.

                :param store_data:

                    How to store the data, see above for a descitpion.

                :param store_flags: Flags describing how to store data.

                        :const:`~pypet.HDF5StorageService.ARRAY` ('ARRAY')

                            Store stuff as array

                        :const:`~pypet.HDF5StorageService.CARRAY` ('CARRAY')

                            Store stuff as carray

                        :const:`~pypet.HDF5StorageService.TABLE` ('TABLE')

                            Store stuff as pytable

                        :const:`~pypet.HDF5StorageService.DICT` ('DICT')

                            Store stuff as pytable but reconstructs it later as dictionary
                            on loading

                        :const:`~pypet.HDF%StorageService.FRAME` ('FRAME')

                            Store stuff as pandas data frame

                    Storage flags can also be provided by the parameters and results themselves
                    if they implement a function '_store_flags' that returns a dictionary
                    with the names of the data to store as keys and the flags as values.

                    If no storage flags are provided, they are automatically inferred from the
                    data. See :const:`pypet.HDF5StorageService.TYPE_FLAG_MAPPING` for the mapping
                    from type to flag.

                :param overwrite:

                    Can be used if parts of a leaf should be replaced. Either a list of
                    HDF5 names or `True` if this should account for all.

            * :const:`pypet.pypetconstants.DELETE` ('DELETE')

                Removes an item from disk. Empty group nodes, results and non-explored
                parameters can be removed.

                :param stuff_to_store: The item to be removed.

                :param delete_only:

                    Potential list of parts of a leaf node that should be deleted.

                :param remove_from_item:

                    If `delete_only` is used, whether deleted nodes should also be erased
                    from the leaf nodes themseleves.

                :param recursive:

                    If you want to delete a group node you can recursively delete all its
                    children.

            * :const:`pypet.pypetconstants.GROUP` ('GROUP')

                :param stuff_to_store: The group to store

                :param store_data: How to store data

                :param recursive: To recursively load everything below.

                :param max_depth:

                    Maximum depth in case of recursion. `None` for no limit.

            * :const:`pypet.pypetconstants.TREE`

                Stores a single node or a full subtree

                :param stuff_to_store: Node to store

                :param store_data: How to store data

                :param recursive: Whether to store recursively the whole sub-tree

                :param max_depth:

                    Maximum depth in case of recursion. `None` for no limit.

            * :const:`pypet.pypetconstants.DELETE_LINK`

                Deletes a link from hard drive

                :param name: The full colon separated name of the link

            * :const:`pypet.pypetconstants.LIST`

                .. _store-lists:

                Stores several items at once

                :param stuff_to_store:

                    Iterable whose items are to be stored. Iterable must contain tuples,
                    for example `[(msg1,item1,arg1,kwargs1),(msg2,item2,arg2,kwargs2),...]`

            * :const:`pypet.pypetconstants.ACCESS_DATA`

                Requests and manipulates data within the storage.
                Storage must be open.

                :param stuff_to_store:

                    A colon separated name to the data path

                :param item_name:

                    The name of the data item to interact with

                :param request:

                    A functional request in form of a string

                :param args:

                    Positional arguments passed to the reques

                :param kwargs:

                    Keyword arguments passed to the request

            * :const:`pypet.pypetconstants.OPEN_FILE`

                Opens the HDF5 file and keeps it open

                :param stuff_to_store: ``None``

            * :const:`pypet.pypetconstants.CLOSE_FILE`

                Closes an HDF5 file that was kept open, must be open before.

                :param stuff_to_store: ``None``

            * :const:`pypet.pypetconstants.FLUSH`

                Flushes an open file, must be open before.

                :param stuff_to_store: ``None``

        :raises: NoSuchServiceError if message or data is not understood

        """
        opened = True
        try:

            opened = self._srvc_opening_routine('a', msg, kwargs)

            if msg == pypetconstants.MERGE:
                self._trj_merge_trajectories(*args, **kwargs)

            elif msg == pypetconstants.BACKUP:
                self._trj_backup_trajectory(stuff_to_store, *args, **kwargs)

            elif msg == pypetconstants.PREPARE_MERGE:
                self._trj_prepare_merge(stuff_to_store, *args, **kwargs)

            elif msg == pypetconstants.TRAJECTORY:
                self._trj_store_trajectory(stuff_to_store, *args, **kwargs)

            elif msg == pypetconstants.SINGLE_RUN:
                self._srn_store_single_run(stuff_to_store, *args, **kwargs)

            elif msg in pypetconstants.LEAF:
                self._prm_store_parameter_or_result(stuff_to_store, *args, **kwargs)

            elif msg == pypetconstants.DELETE:
                self._all_delete_parameter_or_result_or_group(stuff_to_store, *args, **kwargs)

            elif msg == pypetconstants.GROUP:
                self._grp_store_group(stuff_to_store, *args, **kwargs)

            elif msg == pypetconstants.TREE:
                self._tree_store_sub_branch(stuff_to_store, *args, **kwargs)

            elif msg == pypetconstants.DELETE_LINK:
                self._lnk_delete_link(stuff_to_store, *args, **kwargs)

            elif msg == pypetconstants.LIST:
                self._srvc_store_several_items(stuff_to_store, *args, **kwargs)

            elif msg == pypetconstants.ACCESS_DATA:
                return self._hdf5_interact_with_data(stuff_to_store, *args, **kwargs)

            elif msg == pypetconstants.OPEN_FILE:
                opened = False  # Wee need to keep the file open to allow later interaction
                self._keep_open = True
                self._node_processing_timer.active = False # This might be open quite long
                # so we don't want to display horribly long opening times

            elif msg == pypetconstants.CLOSE_FILE:
                opened = True  # Simply conduct the closing routine afterwards
                self._keep_open = False

            elif msg == pypetconstants.FLUSH:
                self._hdf5file.flush()

            else:
                raise pex.NoSuchServiceError('I do not know how to handle `%s`' % msg)

        except:
            self._logger.error('Failed storing `%s`' % str(stuff_to_store))
            raise
        finally:
            self._srvc_closing_routine(opened)


def _srvc_load_several_items(self, iterable, *args, **kwargs):
        """Loads several items from an iterable

        Iterables are supposed to be of a format like `[(msg, item, args, kwarg),...]`
        If `args` and `kwargs` are not part of a tuple, they are taken from the
        current `args` and `kwargs` provided to this function.

        """
        for input_tuple in iterable:
            msg = input_tuple[0]
            item = input_tuple[1]
            if len(input_tuple) > 2:
                args = input_tuple[2]
            if len(input_tuple) > 3:
                kwargs = input_tuple[3]
            if len(input_tuple) > 4:
                raise RuntimeError('You shall not pass!')

            self.load(msg, item, *args, **kwargs)

def
_srvc_load_several_items
(
self
,
iterable
,
*
args
,
*
*
kwargs
)
:
for
input_tuple
in
iterable
:
msg
=
input_tuple
[
0
]
item
=
input_tuple
[
1
]
if
len
(
input_tuple
)
>
2
:
args
=
input_tuple
[
2
]
if
len
(
input_tuple
)
>
3
:
kwargs
=
input_tuple
[
3
]
if
len
(
input_tuple
)
>
4
:
raise
RuntimeError
(
'You shall not pass!'
)
self
.
load
(
msg
,
item
,
*
args
,
*
*
kwargs
)


def _srvc_check_hdf_properties(self, traj):
        """Reads out the properties for storing new data into the hdf5file

        :param traj:

            The trajectory

        """

        for attr_name in HDF5StorageService.ATTR_LIST:
            try:
                config = traj.f_get('config.hdf5.' + attr_name).f_get()
                setattr(self, attr_name, config)
            except AttributeError:
                self._logger.debug('Could not find `%s` in traj config, '
                                     'using (default) value `%s`.' %
                                     (attr_name, str(getattr(self, attr_name))))

        for attr_name, table_name in HDF5StorageService.NAME_TABLE_MAPPING.items():
            try:
                if table_name in ('parameters', 'config'):
                    table_name += '_overview'
                config = traj.f_get('config.hdf5.overview.' + table_name).f_get()
                setattr(self, attr_name, config)
            except AttributeError:
                self._logger.debug('Could not find `%s` in traj config, '
                                     'using (default) value `%s`.' %
                                     (table_name, str(getattr(self, attr_name))))

        for attr_name, name in HDF5StorageService.PR_ATTR_NAME_MAPPING.items():
            try:
                config = traj.f_get('config.hdf5.' + name).f_get()
                setattr(self, attr_name, config)
            except AttributeError:
                self._logger.debug('Could not find `%s` in traj config, '
                                     'using (default) value `%s`.' %
                                     (name, str(getattr(self, attr_name))))

        if ((not self._overview_results_summary or
                    not self._overview_derived_parameters_summary) and
                    self._purge_duplicate_comments):
            raise RuntimeError('You chose to purge duplicate comments but disabled a summary '
                                   'table. You can only use the purging if you enable '
                                   'the summary tables.')

        self._filters = None


def _srvc_store_several_items(self, iterable, *args, **kwargs):
        """Stores several items from an iterable

        Iterables are supposed to be of a format like `[(msg, item, args, kwarg),...]`
        If `args` and `kwargs` are not part of a tuple, they are taken from the
        current `args` and `kwargs` provided to this function.

        """
        for input_tuple in iterable:
            msg = input_tuple[0]
            item = input_tuple[1]
            if len(input_tuple) > 2:
                args = input_tuple[2]
            if len(input_tuple) > 3:
                kwargs = input_tuple[3]
            if len(input_tuple) > 4:
                raise RuntimeError('You shall not pass!')

            self.store(msg, item, *args, **kwargs)

def
_srvc_store_several_items
(
self
,
iterable
,
*
args
,
*
*
kwargs
)
:
for
input_tuple
in
iterable
:
msg
=
input_tuple
[
0
]
item
=
input_tuple
[
1
]
if
len
(
input_tuple
)
>
2
:
args
=
input_tuple
[
2
]
if
len
(
input_tuple
)
>
3
:
kwargs
=
input_tuple
[
3
]
if
len
(
input_tuple
)
>
4
:
raise
RuntimeError
(
'You shall not pass!'
)
self
.
store
(
msg
,
item
,
*
args
,
*
*
kwargs
)


def _srvc_closing_routine(self, closing):
        """Routine to close an hdf5 file

        The file is closed only when `closing=True`. `closing=True` means that
        the file was opened in the current highest recursion level. This prevents re-opening
        and closing of the file if `store` or `load` are called recursively.

        """
        if (not self._keep_open and
                closing and
                    self.is_open):

            f_fd = self._hdf5file.fileno()
            self._hdf5file.flush()
            try:
                os.fsync(f_fd)
                try:
                    self._hdf5store.flush(fsync=True)
                except TypeError:
                    f_fd = self._hdf5store._handle.fileno()
                    self._hdf5store.flush()
                    os.fsync(f_fd)
            except OSError as exc:
                # This seems to be the only way to avoid an OSError under Windows
                errmsg = ('Encountered OSError while flushing file.'
                                   'If you are using Windows, don`t worry! '
                                   'I will ignore the error and try to close the file. '
                                   'Original error: %s' % repr(exc))
                self._logger.debug(errmsg)

            self._hdf5store.close()
            if self._hdf5file.isopen:
                self._logger.error('Could not close HDF5 file!')
            self._hdf5file = None
            self._hdf5store = None
            self._trajectory_group = None
            self._trajectory_name = None
            self._trajectory_index = None
            self._overview_group_ = None
            self._logger.debug('Closing HDF5 file')
            return True
        else:
            return False


def _srvc_extract_file_information(self, kwargs):
        """Extracts file information from kwargs.

        Note that `kwargs` is not passed as `**kwargs` in order to also
        `pop` the elements on the level of the function calling `_srvc_extract_file_information`.

        """
        if 'filename' in kwargs:
            self._filename = kwargs.pop('filename')

        if 'file_title' in kwargs:
            self._file_title = kwargs.pop('file_title')

        if 'trajectory_name' in kwargs:
            self._trajectory_name = kwargs.pop('trajectory_name')

        if 'trajectory_index' in kwargs:
            self._trajectory_index = kwargs.pop('trajectory_index')

def _srvc_extract_file_information ( self , kwargs ) : if 'filename' in kwargs : self . _filename = kwargs . pop ( 'filename' ) if 'file_title' in kwargs : self . _file_title = kwargs . pop ( 'file_title' ) if 'trajectory_name' in kwargs : self . _trajectory_name = kwargs . pop ( 'trajectory_name' ) if 'trajectory_index' in kwargs : self . _trajectory_index = kwargs . pop ( 'trajectory_index' )


def _trj_backup_trajectory(self, traj, backup_filename=None):
        """Backs up a trajectory.

        :param traj: Trajectory that should be backed up

        :param backup_filename:

            Path and filename of backup file. If None is specified the storage service
            defaults to `path_to_trajectory_hdf5_file/backup_trajectory_name.hdf`.

        """
        self._logger.info('Storing backup of %s.' % traj.v_name)

        mypath, _ = os.path.split(self._filename)

        if backup_filename is None:
            backup_filename = os.path.join('%s' % mypath, 'backup_%s.hdf5' % traj.v_name)

        backup_hdf5file = pt.open_file(filename=backup_filename,
                                             mode='a', title=backup_filename)

        if '/' + self._trajectory_name in backup_hdf5file:
            raise ValueError('I cannot backup  `%s` into file `%s`, there is already a '
                             'trajectory with that name.' % (traj.v_name, backup_filename))

        backup_root = backup_hdf5file.root

        self._trajectory_group._f_copy(newparent=backup_root, recursive=True)

        backup_hdf5file.flush()
        backup_hdf5file.close()

        self._logger.info('Finished backup of %s.' % traj.v_name)


def _trj_read_out_row(colnames, row):
        """Reads out a row and returns a dictionary containing the row content.

        :param colnames: List of column names
        :param row:  A pytables table row
        :return: A dictionary with colnames as keys and content as values

        """
        result_dict = {}
        for colname in colnames:
            result_dict[colname] = row[colname]

        return result_dict

def _trj_read_out_row ( colnames , row ) : result_dict = { } for colname in colnames : result_dict [ colname ] = row [ colname ] return result_dict

Reads out a row and returns a dictionary containing the row content .


def _trj_prepare_merge(self, traj, changed_parameters, old_length):
        """Prepares a trajectory for merging.

        This function will already store extended parameters.

        :param traj: Target of merge
        :param changed_parameters: List of extended parameters (i.e. their names).

        """

        if not traj._stored:
            traj.f_store()

        # Update meta information
        infotable = getattr(self._overview_group, 'info')
        insert_dict = self._all_extract_insert_dict(traj, infotable.colnames)
        self._all_add_or_modify_row(traj.v_name, insert_dict, infotable, index=0,
                                    flags=(HDF5StorageService.MODIFY_ROW,))

        # Store extended parameters
        for param_name in changed_parameters:
            param = traj.f_get(param_name)

            try:
                self._all_delete_parameter_or_result_or_group(param)
            except pt.NoSuchNodeError:
                pass  # We are fine and the node did not exist in the first place

        # Increase the run table by the number of new runs
        run_table = getattr(self._overview_group, 'runs')
        actual_rows = run_table.nrows
        self._trj_fill_run_table(traj, actual_rows, len(traj))

        # Extract parameter summary and if necessary create new explored parameter tables
        # in the result groups
        for idx in range(old_length, len(traj)):
            run_name = traj.f_idx_to_run(idx)
            run_info = traj.f_get_run_information(run_name)
            run_info['name'] = run_name

            traj._set_explored_parameters_to_idx(idx)

            run_summary = self._srn_summarize_explored_parameters(list(
                                                            traj._explored_parameters.values()))

            run_info['parameter_summary'] = run_summary

            self._all_add_or_modify_row(run_name, run_info, run_table, index=idx,
                                        flags=(HDF5StorageService.MODIFY_ROW,))

        traj.f_restore_default()


def _trj_load_meta_data(self, traj, load_data, as_new, with_run_information, force):
        """Loads meta information about the trajectory

        Checks if the version number does not differ from current pypet version
        Loads, comment, timestamp, name, version from disk in case trajectory is not loaded
        as new. Updates the run information as well.

        """

        metatable = self._overview_group.info
        metarow = metatable[0]

        try:
            version = metarow['version'].decode('utf-8')
        except (IndexError, ValueError) as ke:
            self._logger.error('Could not check version due to: %s' % str(ke))
            version = '`COULD NOT BE LOADED`'

        try:
            python = metarow['python'].decode('utf-8')
        except (IndexError, ValueError) as ke:
            self._logger.error('Could not check version due to: %s' % str(ke))
            python = '`COULD NOT BE LOADED`'

        self._trj_check_version(version, python, force)

        # Load the skeleton information
        self._grp_load_group(traj, load_data=load_data,
                             with_links=False, recursive=False, _traj=traj,
                             _as_new=as_new, _hdf5_group=self._trajectory_group)

        if as_new:
            length = int(metarow['length'])
            for irun in range(length):
                traj._add_run_info(irun)
        else:
            traj._comment =  metarow['comment'].decode('utf-8')
            traj._timestamp = float(metarow['timestamp'])
            traj._trajectory_timestamp = traj._timestamp
            traj._time = metarow['time'].decode('utf-8')
            traj._trajectory_time = traj._time
            traj._name = metarow['name'].decode('utf-8')
            traj._trajectory_name = traj._name
            traj._version = version
            traj._python = python

            single_run_table = self._overview_group.runs

            if with_run_information:
                for row in single_run_table.iterrows():
                    name = row['name'].decode('utf-8')
                    idx = int(row['idx'])
                    timestamp = float(row['timestamp'])
                    time_ = row['time'].decode('utf-8')
                    completed = int(row['completed'])
                    summary = row['parameter_summary'].decode('utf-8')
                    hexsha = row['short_environment_hexsha'].decode('utf-8')

                    # To allow backwards compatibility we need this try catch block
                    try:
                        runtime = row['runtime'].decode('utf-8')
                        finish_timestamp = float(row['finish_timestamp'])
                    except (IndexError, ValueError) as ke:
                        runtime = ''
                        finish_timestamp = 0.0
                        self._logger.debug('Could not load runtime, ' + repr(ke))

                    info_dict = {'idx': idx,
                                 'timestamp': timestamp,
                                 'finish_timestamp': finish_timestamp,
                                 'runtime': runtime,
                                 'time': time_,
                                 'completed': completed,
                                 'name': name,
                                 'parameter_summary': summary,
                                 'short_environment_hexsha': hexsha}

                    traj._add_run_info(**info_dict)
            else:
                traj._length = single_run_table.nrows

        # Load explorations
        self._trj_load_exploration(traj)
        # Load the hdf5 config data:
        self._srvc_load_hdf5_settings()


def _tree_load_sub_branch(self, traj_node, branch_name,
                              load_data=pypetconstants.LOAD_DATA,
                              with_links=True, recursive=False,
                              max_depth=None, _trajectory=None,
                              _as_new=False, _hdf5_group=None):
        """Loads data starting from a node along a branch and starts recursively loading
        all data at end of branch.

        :param traj_node: The node from where loading starts

        :param branch_name:

            A branch along which loading progresses. Colon Notation is used:
            'group1.group2.group3' loads 'group1', then 'group2', then 'group3' and then finally
            recursively all children and children's children below 'group3'

        :param load_data:

            How to load the data

        :param with_links:

            If links should be loaded

        :param recursive:

            If loading recursively

        :param max_depth:

            The maximum depth to load the tree

        :param _trajectory:

            The trajectory

        :param _as_new:

            If trajectory is loaded as new

        :param _hdf5_group:

            HDF5 node in the file corresponding to `traj_node`.

        """
        if load_data == pypetconstants.LOAD_NOTHING:
            return

        if max_depth is None:
            max_depth = float('inf')

        if _trajectory is None:
            _trajectory = traj_node.v_root

        if _hdf5_group is None:
            hdf5_group_name = traj_node.v_full_name.replace('.', '/')

            # Get child node to load
            if hdf5_group_name == '':
                _hdf5_group = self._trajectory_group
            else:
                try:
                    _hdf5_group = self._hdf5file.get_node(where=self._trajectory_group,
                                                  name=hdf5_group_name)
                except pt.NoSuchNodeError:
                    self._logger.error('Cannot find `%s` the hdf5 node `%s` does not exist!'
                                       % (traj_node.v_full_name, hdf5_group_name))
                    raise

        split_names = branch_name.split('.')

        final_group_name = split_names.pop()

        current_depth = 1

        for name in split_names:
            if current_depth > max_depth:
                return
            # First load along the branch
            _hdf5_group = getattr(_hdf5_group, name)

            self._tree_load_nodes_dfs(traj_node, load_data=load_data, with_links=with_links,
                                  recursive=False, max_depth=max_depth, current_depth=current_depth,
                                  trajectory=_trajectory, as_new=_as_new,
                                  hdf5_group=_hdf5_group)

            current_depth += 1

            traj_node = traj_node._children[name]

        if current_depth <= max_depth:
            # Then load recursively all data in the last group and below
            _hdf5_group = getattr(_hdf5_group, final_group_name)
            self._tree_load_nodes_dfs(traj_node, load_data=load_data, with_links=with_links,
                                  recursive=recursive, max_depth=max_depth,
                                  current_depth=current_depth, trajectory=_trajectory,
                                  as_new=_as_new, hdf5_group=_hdf5_group)

Loads data starting from a node along a branch and starts recursively loading all data at end of branch .


def _trj_check_version(self, version, python, force):
        """Checks for version mismatch

        Raises a VersionMismatchError if version of loaded trajectory and current pypet version
        do not match. In case of `force=True` error is not raised only a warning is emitted.

        """
        curr_python = pypetconstants.python_version_string

        if (version != VERSION or curr_python != python) and not force:
            raise pex.VersionMismatchError('Current pypet version is %s used under python %s '
                                           '  but your trajectory'
                                           ' was created with version %s and python %s.'
                                           ' Use >>force=True<< to perform your load regardless'
                                           ' of version mismatch.' %
                                           (VERSION, curr_python, version, python))
        elif version != VERSION or curr_python != python:
            self._logger.warning('Current pypet version is %s with python %s but your trajectory'
                                 ' was created with version %s under python %s.'
                                 ' Yet, you enforced the load, so I will'
                                 ' handle the trajectory despite the'
                                 ' version mismatch.' %
                                 (VERSION, curr_python, version, python))


def _trj_fill_run_table(self, traj, start, stop):
        """Fills the `run` overview table with information.

        Will also update new information.

        """

        def _make_row(info_dict):
            row = (info_dict['idx'],
                   info_dict['name'],
                   info_dict['time'],
                   info_dict['timestamp'],
                   info_dict['finish_timestamp'],
                   info_dict['runtime'],
                   info_dict['parameter_summary'],
                   info_dict['short_environment_hexsha'],
                   info_dict['completed'])
            return row

        runtable = getattr(self._overview_group, 'runs')

        rows = []
        updated_run_information = traj._updated_run_information
        for idx in range(start, stop):
            info_dict = traj._run_information[traj._single_run_ids[idx]]
            rows.append(_make_row(info_dict))
            updated_run_information.discard(idx)

        if rows:
            runtable.append(rows)
            runtable.flush()

        # Store all runs that are updated and that have not been stored yet
        rows = []
        indices = []
        for idx in updated_run_information:
            info_dict = traj.f_get_run_information(idx, copy=False)
            rows.append(_make_row(info_dict))
            indices.append(idx)

        if rows:
            runtable.modify_coordinates(indices, rows)

        traj._updated_run_information = set()

Fills the run overview table with information .


def _trj_load_exploration(self, traj):
        """Recalls names of all explored parameters"""
        if hasattr(self._overview_group, 'explorations'):
            explorations_table = self._overview_group._f_get_child( 'explorations')
            for row in explorations_table.iterrows():
                param_name = row['explorations'].decode('utf-8')
                if param_name not in traj._explored_parameters:
                    traj._explored_parameters[param_name] = None
        else:
            # This is for backwards compatibility
            for what in ('parameters', 'derived_parameters'):
                if hasattr(self._trajectory_group, what):
                    parameters = self._trajectory_group._f_get_child(what)
                    for group in parameters._f_walk_groups():
                        if self._all_get_from_attrs(group, HDF5StorageService.LENGTH):
                            group_location = group._v_pathname
                            full_name = '.'.join(group_location.split('/')[2:])
                            traj._explored_parameters[full_name] = None

def
_trj_load_exploration
(
self
,
traj
)
:
if
hasattr
(
self
.
_overview_group
,
'explorations'
)
:
explorations_table
=
self
.
_overview_group
.
_f_get_child
(
'explorations'
)
for
row
in
explorations_table
.
iterrows
(
)
:
param_name
=
row
[
'explorations'
]
.
decode
(
'utf-8'
)
if
param_name
not
in
traj
.
_explored_parameters
:
traj
.
_explored_parameters
[
param_name
]
=
else
:
# This is for backwards compatibility
for
what
in
(
'parameters'
,
'derived_parameters'
)
:
if
hasattr
(
self
.
_trajectory_group
,
what
)
:
parameters
=
self
.
_trajectory_group
.
_f_get_child
(
what
)
for
group
in
parameters
.
_f_walk_groups
(
)
:
if
self
.
_all_get_from_attrs
(
group
,
HDF5StorageService
.
LENGTH
)
:
group_location
=
group
.
_v_pathname
full_name
=
'.'
.
join
(
group_location
.
split
(
'/'
)
[
2
:
]
)
traj
.
_explored_parameters
[
full_name
]
=