def f_store_items(self, iterator, *args, **kwargs):
        """Stores individual items to disk.

        This function is useful if you calculated very large results (or large derived parameters)
        during runtime and you want to write these to disk immediately and empty them afterwards
        to free some memory.

        Instead of storing individual parameters or results you can also store whole subtrees with
        :func:`~pypet.naturalnaming.NNGroupNode.f_store_child`.

        You can pass the following arguments to `f_store_items`:

        :param iterator:

            An iterable containing the parameters or results to store, either their
            names or the instances. You can also pass group instances or names here
            to store the annotations of the groups.

        :param non_empties:

            Optional keyword argument (boolean),
            if `True` will only store the subset of provided items that are not empty.
            Empty parameters or results found in `iterator` are simply ignored.

        :param args: Additional arguments passed to the storage service

        :param kwargs:

            If you use the standard hdf5 storage service, you can pass the following additional
            keyword argument:

            :param overwrite:

                List names of parts of your item that should
                be erased and overwritten by the new data in your leaf.
                You can also set `overwrite=True`
                to overwrite all parts.

                For instance:

                    >>> traj.f_add_result('mygroup.myresult', partA=42, partB=44, partC=46)
                    >>> traj.f_store()
                    >>> traj.mygroup.myresult.partA = 333
                    >>> traj.mygroup.myresult.partB = 'I am going to change to a string'
                    >>> traj.f_store_item('mygroup.myresult', overwrite=['partA', 'partB'])

                Will store `'mygroup.myresult'` to disk again and overwrite the parts
                `'partA'` and `'partB'` with the new values `333` and
                `'I am going to change to a string'`.
                The data stored as `partC` is not changed.

                Be aware that you need to specify the names of parts as they were stored
                to HDF5. Depending on how your leaf construction works, this may differ
                from the names the data might have in your leaf in the trajectory container.

                Note that massive overwriting will fragment and blow up your HDF5 file.
                Try to avoid changing data on disk whenever you can.

        :raises:

            TypeError:

                If the (parent) trajectory has never been stored to disk. In this case
                use :func:`pypet.trajectory.f_store` first.

            ValueError: If no item could be found to be stored.

        Note if you use the standard hdf5 storage service, there are no additional arguments
        or keyword arguments to pass!

        """

        if not self._stored:
            raise TypeError('Cannot store stuff for a trajectory that has never been '
                            'stored to disk. Please call traj.f_store(only_init=True) first.')

        fetched_items = self._nn_interface._fetch_items(STORE, iterator, args, kwargs)

        if fetched_items:
            self._storage_service.store(pypetconstants.LIST, fetched_items,
                                        trajectory_name=self.v_name)
        else:
            raise ValueError('Your storage was not successful, could not find a single item '
                             'to store.')


def f_load_items(self, iterator, *args, **kwargs):
        """Loads parameters and results specified in `iterator`.

        You can directly list the Parameter objects or just their names.

        If names are given the `~pypet.naturalnaming.NNGroupNode.f_get` method is applied to find the
        parameters or results in the trajectory. Accordingly, the parameters and results
        you want to load must already exist in your trajectory (in RAM), probably they
        are just empty skeletons waiting desperately to handle data.
        If they do not exist in RAM yet, but have been stored to disk before,
        you can call :func:`~pypet.trajectory.Trajectory.f_load_skeleton` in order to
        bring your trajectory tree skeleton up to date. In case of a single run you can
        use the :func:`~pypet.naturalnaming.NNGroupNode.f_load_child` method to recursively
        load a subtree without any data.
        Then you can load the data of individual results or parameters one by one.

        If want to load the whole trajectory at once or ALL results and parameters that are
        still empty take a look at :func:`~pypet.trajectory.Trajectory.f_load`.
        As mentioned before, to load subtrees of your trajectory you might want to check out
        :func:`~pypet.naturalnaming.NNGroupNode.f_load_child`.

        To load a list of parameters or results with `f_load_items` you can pass
        the following arguments:

        :param iterator: A list with parameters or results to be loaded.

        :param only_empties:

            Optional keyword argument (boolean),
            if `True` only empty parameters or results are passed
            to the storage service to get loaded. Non-empty parameters or results found in
            `iterator` are simply ignored.

        :param args: Additional arguments directly passed to the storage service

        :param kwargs:

            Additional keyword arguments directly passed to the storage service
            (except the kwarg `only_empties`)

            If you use the standard hdf5 storage service, you can pass the following additional
            keyword arguments:

            :param load_only:

                If you load a result, you can partially load it and ignore the rest of data items.
                Just specify the name of the data you want to load. You can also provide a list,
                for example `load_only='spikes'`, `load_only=['spikes','membrane_potential']`.

                Be aware that you need to specify the names of parts as they were stored
                to HDF5. Depending on how your leaf construction works, this may differ
                from the names the data might have in your leaf in the trajectory container.

                A warning is issued if data specified in `load_only` cannot be found in the
                instances specified in `iterator`.

            :param load_except:

                Analogous to the above, but everything is loaded except names or parts
                specified in `load_except`.
                You cannot use `load_only` and `load_except` at the same time. If you do
                a ValueError is thrown.

                A warning is issued if names listed in `load_except` are not part of the
                items to load.

        """

        if not self._stored:
            raise TypeError(
                'Cannot load stuff from disk for a trajectory that has never been stored.')

        fetched_items = self._nn_interface._fetch_items(LOAD, iterator, args, kwargs)
        if fetched_items:
            self._storage_service.load(pypetconstants.LIST, fetched_items,
                                       trajectory_name=self.v_name)
        else:
            self._logger.warning('Your loading was not successful, could not find a single item '
                                 'to load.')

Loads parameters and results specified in iterator .


def f_remove_items(self, iterator, recursive=False):
        """Removes parameters, results or groups from the trajectory.

        This function ONLY removes items from your current trajectory and does not delete
        data stored to disk. If you want to delete data from disk, take a look at
        :func:`~pypet.trajectory.Trajectory.f_delete_items`.

        This will also remove all links if items are linked.

        :param iterator:

            A sequence of items you want to remove. Either the instances themselves
            or strings with the names of the items.

        :param recursive:

            In case you want to remove group nodes, if the children should be removed, too.

        """

        # Will format the request in a form that is understood by the storage service
        # aka (msg, item, args, kwargs)
        fetched_items = self._nn_interface._fetch_items(REMOVE, iterator, (), {})

        if fetched_items:
            for _, item, dummy1, dummy2 in fetched_items:
                self._nn_interface._remove_node_or_leaf(item, recursive=recursive)

        else:
            self._logger.warning('Your removal was not successful, could not find a single '
                                 'item to remove.')

Removes parameters results or groups from the trajectory .


def f_delete_links(self, iterator_of_links, remove_from_trajectory=False):
        """Deletes several links from the hard disk.

        Links can be passed as a string ``'groupA.groupB.linkA'``
        or as a tuple  containing the node from which the link should be removed and the
        name of the link ``(groupWithLink, 'linkA')``.

        """

        to_delete_links = []

        group_link_pairs = []

        for elem in iterator_of_links:
            if isinstance(elem, str):
                split_names = elem.split('.')
                parent_name = '.'.join(split_names[:-1])
                link = split_names[-1]
                parent_node = self.f_get(parent_name) if parent_name != '' else self
                link_name = parent_node.v_full_name + '.' + link if parent_name != '' else link
                to_delete_links.append((pypetconstants.DELETE_LINK, link_name))
                group_link_pairs.append((parent_node, link))
            else:
                link_name = elem[0].v_full_name + '.' + elem[1]
                to_delete_links.append((pypetconstants.DELETE_LINK, link_name))
                group_link_pairs.append(elem)
        try:
            self._storage_service.store(pypetconstants.LIST, to_delete_links,
                                        trajectory_name=self.v_name)
        except:
            self._logger.error('Could not remove `%s` from the trajectory. Maybe the'
                               ' item(s) was/were never stored to disk.' % str(to_delete_links))
            raise

        if remove_from_trajectory:
            for group, link in group_link_pairs:
                group.f_remove_link(link)

Deletes several links from the hard disk .


def f_remove(self, recursive=True, predicate=None):
        """Recursively removes all children of the trajectory

        :param recursive:

            Only here for consistency with signature of parent method. Cannot be set
            to `False` because the trajectory root node cannot be removed.

        :param predicate:

            Predicate which can evaluate for each node to ``True`` in order to remove the node or
            ``False`` if the node should be kept. Leave ``None`` if you want to remove all nodes.

        """
        if not recursive:
            raise ValueError('Nice try ;-)')

        for child in list(self._children.keys()):
            self.f_remove_child(child, recursive=True, predicate=predicate)

def
f_remove
(
self
,
recursive
=
True
,
predicate
=
)
:
if
not
recursive
:
raise
ValueError
(
'Nice try ;-)'
)
for
child
in
list
(
self
.
_children
.
keys
(
)
)
:
self
.
f_remove_child
(
child
,
recursive
=
True
,
predicate
=
predicate
)


def f_delete_items(self, iterator, *args, **kwargs):
        """Deletes items from storage on disk.

        Per default the item is NOT removed from the trajectory.

        Links are NOT deleted on the hard disk, please delete links manually before deleting
        data!

        :param iterator:

            A sequence of items you want to remove. Either the instances themselves
            or strings with the names of the items.

        :param remove_from_trajectory:

            If items should also be removed from trajectory. Default is `False`.

        :param args: Additional arguments passed to the storage service

        :param kwargs: Additional keyword arguments passed to the storage service

            If you use the standard hdf5 storage service, you can pass the following additional
            keyword argument:

            :param delete_only:

                You can partially delete leaf nodes. Specify a list of parts of the result node
                that should be deleted like `delete_only=['mystuff','otherstuff']`.
                This wil only delete the hdf5 sub parts `mystuff` and `otherstuff` from disk.
                BE CAREFUL,
                erasing data partly happens at your own risk. Depending on how complex the
                loading process of your result node is, you might not be able to reconstruct
                any data due to partially deleting some of it.

                Be aware that you need to specify the names of parts as they were stored
                to HDF5. Depending on how your leaf construction works, this may differ
                from the names the data might have in your leaf in the trajectory container.

                If the hdf5 nodes you specified in `delete_only` cannot be found a warning
                is issued.

                Note that massive deletion will fragment your HDF5 file.
                Try to avoid changing data on disk whenever you can.

                If you want to erase a full node, simply ignore this argument or set to `None`.

            :param remove_from_item:

                If data that you want to delete from storage should also be removed from
                the items in `iterator` if they contain these. Default is `False`.

            :param recursive:

                If you want to delete a group node and it has children you need to
                set `recursive` to `True. Default is `False`.

        """

        remove_from_trajectory = kwargs.pop('remove_from_trajectory', False)
        recursive = kwargs.get('recursive', False)

        # Will format the request in a form that is understood by the storage service
        # aka (msg, item, args, kwargs)
        fetched_items = self._nn_interface._fetch_items(REMOVE, iterator, args, kwargs)

        if fetched_items:

            try:
                self._storage_service.store(pypetconstants.LIST, fetched_items,
                                            trajectory_name=self.v_name)
            except:
                self._logger.error('Could not remove `%s` from the trajectory. Maybe the'
                                   ' item(s) was/were never stored to disk.' % str(fetched_items))
                raise

            for _, item, dummy1, dummy2 in fetched_items:
                if remove_from_trajectory:
                    self._nn_interface._remove_node_or_leaf(item, recursive=recursive)
                else:
                    item._stored = False

        else:
            self._logger.warning('Your removal was not successful, could not find a single '
                                 'item to remove.')


def _pool_single_run(kwargs):
    """Starts a pool single run and passes the storage service"""
    wrap_mode = kwargs['wrap_mode']
    traj = kwargs['traj']
    traj.v_storage_service = _pool_single_run.storage_service
    if wrap_mode == pypetconstants.WRAP_MODE_LOCAL:
        # Free references from previous runs
        traj.v_storage_service.free_references()
    return _sigint_handling_single_run(kwargs)

def
_pool_single_run
(
kwargs
)
:
wrap_mode
=
kwargs
[
'wrap_mode'
]
traj
=
kwargs
[
'traj'
]
traj
.
v_storage_service
=
_pool_single_run
.
storage_service
if
wrap_mode
==
pypetconstants
.
WRAP_MODE_LOCAL
:
# Free references from previous runs
traj
.
v_storage_service
.
free_references
(
)
return
_sigint_handling_single_run
(
kwargs
)


def _frozen_pool_single_run(kwargs):
    """Single run wrapper for the frozen pool, makes a single run and passes kwargs"""
    idx = kwargs.pop('idx')
    frozen_kwargs = _frozen_pool_single_run.kwargs
    frozen_kwargs.update(kwargs)  # in case of `run_map`
    # we need to update job's args and kwargs
    traj = frozen_kwargs['traj']
    traj.f_set_crun(idx)
    return _sigint_handling_single_run(frozen_kwargs)

Single run wrapper for the frozen pool makes a single run and passes kwargs


def _configure_pool(kwargs):
    """Configures the pool and keeps the storage service"""
    _pool_single_run.storage_service = kwargs['storage_service']
    _configure_niceness(kwargs)
    _configure_logging(kwargs, extract=False)

def _configure_pool ( kwargs ) : _pool_single_run . storage_service = kwargs [ 'storage_service' ] _configure_niceness ( kwargs ) _configure_logging ( kwargs , extract = False )


def _configure_frozen_pool(kwargs):
    """Configures the frozen pool and keeps all kwargs"""
    _frozen_pool_single_run.kwargs = kwargs
    _configure_niceness(kwargs)
    _configure_logging(kwargs, extract=False)
    # Reset full copy to it's old value
    traj = kwargs['traj']
    traj.v_full_copy = kwargs['full_copy']

def
_configure_frozen_pool
(
kwargs
)
:
_frozen_pool_single_run
.
kwargs
=
kwargs
_configure_niceness
(
kwargs
)
_configure_logging
(
kwargs
,
extract
=
False
)
# Reset full copy to it's old value
traj
=
kwargs
[
'traj'
]
traj
.
v_full_copy
=
kwargs
[
'full_copy'
]


def _process_single_run(kwargs):
    """Wrapper function that first configures logging and starts a single run afterwards."""
    _configure_niceness(kwargs)
    _configure_logging(kwargs)
    result_queue = kwargs['result_queue']
    result = _sigint_handling_single_run(kwargs)
    result_queue.put(result)
    result_queue.close()

def _process_single_run ( kwargs ) : _configure_niceness ( kwargs ) _configure_logging ( kwargs ) result_queue = kwargs [ 'result_queue' ] result = _sigint_handling_single_run ( kwargs ) result_queue . put ( result ) result_queue . close ( )

Wrapper function that first configures logging and starts a single run afterwards .


def _configure_frozen_scoop(kwargs):
    """Wrapper function that configures a frozen SCOOP set up.

    Deletes of data if necessary.

    """
    def _delete_old_scoop_rev_data(old_scoop_rev):
        if old_scoop_rev is not None:
            try:
                elements = shared.elements
                for key in elements:
                    var_dict = elements[key]
                    if old_scoop_rev in var_dict:
                        del var_dict[old_scoop_rev]
                logging.getLogger('pypet.scoop').debug('Deleted old SCOOP data from '
                                                       'revolution `%s`.' % old_scoop_rev)
            except AttributeError:
                logging.getLogger('pypet.scoop').error('Could not delete old SCOOP data from '
                                                       'revolution `%s`.' % old_scoop_rev)
    scoop_rev = kwargs.pop('scoop_rev')
    # Check if we need to reconfigure SCOOP
    try:
        old_scoop_rev = _frozen_scoop_single_run.kwargs['scoop_rev']
        configured = old_scoop_rev == scoop_rev
    except (AttributeError, KeyError):
        old_scoop_rev = None
        configured = False
    if not configured:
        _frozen_scoop_single_run.kwargs = shared.getConst(scoop_rev, timeout=424.2)
        frozen_kwargs = _frozen_scoop_single_run.kwargs
        frozen_kwargs['scoop_rev'] = scoop_rev
        frozen_kwargs['traj'].v_full_copy = frozen_kwargs['full_copy']
        if not scoop.IS_ORIGIN:
            _configure_niceness(frozen_kwargs)
            _configure_logging(frozen_kwargs, extract=False)
        _delete_old_scoop_rev_data(old_scoop_rev)
        logging.getLogger('pypet.scoop').info('Configured Worker %s' % str(scoop.worker))

def
_configure_frozen_scoop
(
kwargs
)
:
def
_delete_old_scoop_rev_data
(
old_scoop_rev
)
:
if
old_scoop_rev
is
not
:
try
:
elements
=
shared
.
elements
for
key
in
elements
:
var_dict
=
elements
[
key
]
if
old_scoop_rev
in
var_dict
:
del
var_dict
[
old_scoop_rev
]
logging
.
getLogger
(
'pypet.scoop'
)
.
debug
(
'Deleted old SCOOP data from '
'revolution `%s`.'
%
old_scoop_rev
)
except
AttributeError
:
logging
.
getLogger
(
'pypet.scoop'
)
.
error
(
'Could not delete old SCOOP data from '
'revolution `%s`.'
%
old_scoop_rev
)
scoop_rev
=
kwargs
.
pop
(
'scoop_rev'
)
# Check if we need to reconfigure SCOOP
try
:
old_scoop_rev
=
_frozen_scoop_single_run
.
kwargs
[
'scoop_rev'
]
configured
=
old_scoop_rev
==
scoop_rev
except
(
AttributeError
,
KeyError
)
:
old_scoop_rev
=
configured
=
False
if
not
configured
:
_frozen_scoop_single_run
.
kwargs
=
shared
.
getConst
(
scoop_rev
,
timeout
=
424.2
)
frozen_kwargs
=
_frozen_scoop_single_run
.
kwargs
frozen_kwargs
[
'scoop_rev'
]
=
scoop_rev
frozen_kwargs
[
'traj'
]
.
v_full_copy
=
frozen_kwargs
[
'full_copy'
]
if
not
scoop
.
IS_ORIGIN
:
_configure_niceness
(
frozen_kwargs
)
_configure_logging
(
frozen_kwargs
,
extract
=
False
)
_delete_old_scoop_rev_data
(
old_scoop_rev
)
logging
.
getLogger
(
'pypet.scoop'
)
.
info
(
'Configured Worker %s'
%
str
(
scoop
.
worker
)
)

Wrapper function that configures a frozen SCOOP set up .


def _scoop_single_run(kwargs):
    """Wrapper function for scoop, that does not configure logging"""
    try:
        try:
            is_origin = scoop.IS_ORIGIN
        except AttributeError:
            # scoop is not properly started, i.e. with `python -m scoop...`
            # in this case scoop uses default `map` function, i.e.
            # the main process
            is_origin = True
        if not is_origin:
            # configure logging and niceness if not the main process:
            _configure_niceness(kwargs)
            _configure_logging(kwargs)
        return _single_run(kwargs)
    except Exception:
        scoop.logger.exception('ERROR occurred during a single run!')
        raise

Wrapper function for scoop that does not configure logging


def _configure_logging(kwargs, extract=True):
    """Requests the logging manager to configure logging.

    :param extract:

        If naming data should be extracted from the trajectory

    """
    try:
        logging_manager = kwargs['logging_manager']
        if extract:
            logging_manager.extract_replacements(kwargs['traj'])
        logging_manager.make_logging_handlers_and_tools(multiproc=True)
    except Exception as exc:
        sys.stderr.write('Could not configure logging system because of: %s' % repr(exc))
        traceback.print_exc()

Requests the logging manager to configure logging .


def _configure_niceness(kwargs):
    """Sets niceness of a process"""
    niceness = kwargs['niceness']
    if niceness is not None:
        try:
            try:
                current = os.nice(0)
                if niceness - current > 0:
                    # Under Linux you cannot decrement niceness if set elsewhere
                    os.nice(niceness-current)
            except AttributeError:
                # Fall back on psutil under Windows
                psutil.Process().nice(niceness)
        except Exception as exc:
            sys.stderr.write('Could not configure niceness because of: %s' % repr(exc))
            traceback.print_exc()


def _sigint_handling_single_run(kwargs):
    """Wrapper that allow graceful exits of single runs"""
    try:
        graceful_exit = kwargs['graceful_exit']

        if graceful_exit:
            sigint_handling.start()
            if sigint_handling.hit:
                result = (sigint_handling.SIGINT, None)
            else:
                result = _single_run(kwargs)
                if sigint_handling.hit:
                    result = (sigint_handling.SIGINT, result)
            return result
        return _single_run(kwargs)

    except:
        # Log traceback of exception
        pypet_root_logger = logging.getLogger('pypet')
        pypet_root_logger.exception('ERROR occurred during a single run! ')
        raise

def
_sigint_handling_single_run
(
kwargs
)
:
try
:
graceful_exit
=
kwargs
[
'graceful_exit'
]
if
graceful_exit
:
sigint_handling
.
start
(
)
if
sigint_handling
.
hit
:
result
=
(
sigint_handling
.
SIGINT
,
)
else
:
result
=
_single_run
(
kwargs
)
if
sigint_handling
.
hit
:
result
=
(
sigint_handling
.
SIGINT
,
result
)
return
result
return
_single_run
(
kwargs
)
except
:
# Log traceback of exception
pypet_root_logger
=
logging
.
getLogger
(
'pypet'
)
pypet_root_logger
.
exception
(
'ERROR occurred during a single run! '
)
raise


def _single_run(kwargs):
    """ Performs a single run of the experiment.

    :param kwargs: Dict of arguments

        traj: The trajectory containing all parameters set to the corresponding run index.

        runfunc: The user's job function

        runargs: The arguments handed to the user's job function (as *args)

        runkwargs: The keyword arguments handed to the user's job function (as **kwargs)

        clean_up_after_run: Whether to clean up after the run

        automatic_storing: Whether or not the data should be automatically stored

        result_queue: A queue object to store results into in case a pool is used, otherwise None

    :return:

        Results computed by the user's job function which are not stored into the trajectory.
        Returns a nested tuple of run index and result and run information:
        ``((traj.v_idx, result), run_information_dict)``

    """
    pypet_root_logger = logging.getLogger('pypet')
    traj = kwargs['traj']
    runfunc = kwargs['runfunc']
    runargs = kwargs['runargs']
    kwrunparams = kwargs['runkwargs']
    clean_up_after_run = kwargs['clean_up_runs']
    automatic_storing = kwargs['automatic_storing']
    wrap_mode = kwargs['wrap_mode']

    idx = traj.v_idx
    total_runs = len(traj)

    pypet_root_logger.info('\n=========================================\n '
              'Starting single run #%d of %d '
              '\n=========================================\n' % (idx, total_runs))

    # Measure start time
    traj.f_start_run(turn_into_run=True)

    # Run the job function of the user
    result = runfunc(traj, *runargs, **kwrunparams)

    # Store data if desired
    if automatic_storing:
        traj.f_store()

    # Add the index to the result and the run information
    if wrap_mode == pypetconstants.WRAP_MODE_LOCAL:
        result = ((traj.v_idx, result),
                   traj.f_get_run_information(traj.v_idx, copy=False),
                   traj.v_storage_service.references)
        traj.v_storage_service.free_references()
    else:
        result = ((traj.v_idx, result),
                   traj.f_get_run_information(traj.v_idx, copy=False))

    # Measure time of finishing
    traj.f_finalize_run(store_meta_data=False,
                        clean_up=clean_up_after_run)

    pypet_root_logger.info('\n=========================================\n '
              'Finished single run #%d of %d '
              '\n=========================================\n' % (idx, total_runs))

    return result

Performs a single run of the experiment .


def _wrap_handling(kwargs):
    """ Starts running a queue handler and creates a log file for the queue."""
    _configure_logging(kwargs, extract=False)
    # Main job, make the listener to the queue start receiving message for writing to disk.
    handler=kwargs['handler']
    graceful_exit = kwargs['graceful_exit']
    # import cProfile as profile
    # profiler = profile.Profile()
    # profiler.enable()
    if graceful_exit:
        sigint_handling.start()
    handler.run()

Starts running a queue handler and creates a log file for the queue .


def load_class(full_class_string):
    """Loads a class from a string naming the module and class name.

    For example:
    >>> load_class(full_class_string = 'pypet.brian.parameter.BrianParameter')
    <BrianParameter>

    """

    class_data = full_class_string.split(".")
    module_path = ".".join(class_data[:-1])
    class_str = class_data[-1]
    module = importlib.import_module(module_path)

    # We retrieve the Class from the module
    return getattr(module, class_str)

def
load_class
(
full_class_string
)
:
class_data
=
full_class_string
.
split
(
"."
)
module_path
=
"."
.
join
(
class_data
[
:
1
]
)
class_str
=
class_data
[
1
]
module
=
importlib
.
import_module
(
module_path
)
# We retrieve the Class from the module
return
getattr
(
module
,
class_str
)

Loads a class from a string naming the module and class name .


def create_class(class_name, dynamic_imports):
    """Dynamically creates a class.

    It is tried if the class can be created by the already given imports.
    If not the list of the dynamically loaded classes is used.

    """
    try:
        new_class = globals()[class_name]

        if not inspect.isclass(new_class):
            raise TypeError('Not a class!')

        return new_class
    except (KeyError, TypeError):
        for dynamic_class in dynamic_imports:
            # Dynamic classes can be provided directly as a Class instance,
            # for example as `MyCustomParameter`,
            # or as a string describing where to import the class from,
            # for instance as `'mypackage.mymodule.MyCustomParameter'`.
            if inspect.isclass(dynamic_class):
                if class_name == dynamic_class.__name__:
                    return dynamic_class
            else:
                # The class name is always the last in an import string,
                # e.g. `'mypackage.mymodule.MyCustomParameter'`
                class_name_to_test = dynamic_class.split('.')[-1]
                if class_name == class_name_to_test:
                    new_class = load_class(dynamic_class)
                    return new_class
        raise ImportError('Could not create the class named `%s`.' % class_name)


def f_get_range_length(self):
        """Returns the length of the parameter range.

        Raises TypeError if the parameter has no range.

        Does not need to be implemented if the parameter supports
        ``__len__`` appropriately.

        """
        if not self.f_has_range():
            raise TypeError('Not applicable, parameter does not have a range')
        elif hasattr(self, '__len__'):
            return len(self)
        else:
            raise NotImplementedError("Should have implemented this.")

Returns the length of the parameter range .


def f_val_to_str(self):
        """String summary of the value handled by the parameter.

        Note that representing the parameter as a string accesses its value,
        but for simpler debugging, this does not lock the parameter or counts as usage!

        Calls `__repr__` of the contained value.

        """
        old_locked = self._locked
        try:
            return repr(self.f_get())
        except Exception:
            return 'No Evaluation possible (yet)!'
        finally:
            self._locked = old_locked

def
f_val_to_str
(
self
)
:
old_locked
=
self
.
_locked
try
:
return
repr
(
self
.
f_get
(
)
)
except
Exception
:
return
'No Evaluation possible (yet)!'
finally
:
self
.
_locked
=
old_locked

String summary of the value handled by the parameter .


def _equal_values(self, val1, val2):
        """Checks if the parameter considers two values as equal.

        This is important for the trajectory in case of merging. In case you want to delete
        duplicate parameter points, the trajectory needs to know when two parameters
        are equal. Since equality is not always implemented by values handled by
        parameters in the same way, the parameters need to judge whether their values are equal.

        The straightforward example here is a numpy array.
        Checking for equality of two numpy arrays yields
        a third numpy array containing truth values of a piecewise comparison.
        Accordingly, the parameter could judge two numpy arrays equal if ALL of the numpy
        array elements are equal.

        In this BaseParameter class values are considered to be equal if they obey
        the function :func:`~pypet.utils.comparisons.nested_equal`.
        You might consider implementing a different equality comparison in your subclass.

        :raises: TypeError: If both values are not supported by the parameter.

        """
        if self.f_supports(val1) != self.f_supports(val2):
            return False

        if not self.f_supports(val1) and not self.f_supports(val2):
            raise TypeError('I do not support the types of both inputs (`%s` and `%s`), '
                            'therefore I cannot judge whether '
                            'the two are equal.' % (str(type(val1)), str(type(val2))))

        if not self._values_of_same_type(val1, val2):
            return False

        return comparisons.nested_equal(val1, val2)

Checks if the parameter considers two values as equal .


def f_get_range(self, copy=True):
        """Returns a python iterable containing the exploration range.

        :param copy:

            If the range should be copied before handed over to avoid tempering with data

        Example usage:

        >>> param = Parameter('groupA.groupB.myparam',data=22, comment='I am a neat example')
        >>> param._explore([42,43,43])
        >>> param.f_get_range()
        (42,43,44)

        :raises: TypeError: If parameter is not explored.

        """
        if not self.f_has_range():
            raise TypeError('Your parameter `%s` is not array, so cannot return array.' %
                            self.v_full_name)
        elif copy:
            return self._explored_range[:]
        else:
            return self._explored_range

Returns a python iterable containing the exploration range .


def _explore(self, explore_iterable):
        """Explores the parameter according to the iterable.

        Raises ParameterLockedException if the parameter is locked.
        Raises TypeError if the parameter does not support the data,
        the types of the data in the iterable are not the same as the type of the default value,
        or the parameter has already an exploration range.

        Note that the parameter will iterate over the whole iterable once and store
        the individual data values into a tuple. Thus, the whole exploration range is
        explicitly stored in memory.

        :param explore_iterable: An iterable specifying the exploration range

        For example:

        >>> param._explore([3.0,2.0,1.0])
        >>> param.f_get_range()
        (3.0, 2.0, 1.0)

        :raises TypeError,ParameterLockedException

        """
        if self.v_locked:
            raise pex.ParameterLockedException('Parameter `%s` is locked!' % self.v_full_name)

        if self.f_has_range():
            raise TypeError('Your parameter `%s` is already explored, '
                            'cannot _explore it further!' % self._name)

        if self._data is None:
            raise TypeError('Your parameter `%s` has no default value, please specify one '
                            'via `f_set` before exploration. ' % self.v_full_name)

        data_list = self._data_sanity_checks(explore_iterable)

        self._explored_range = data_list
        self._explored = True
        self.f_lock()

Explores the parameter according to the iterable .


def _expand(self, explore_iterable):
        """Explores the parameter according to the iterable and appends to the exploration range.

        Raises ParameterLockedException if the parameter is locked.
        Raises TypeError if the parameter does not support the data,
        the types of the data in the iterable are not the same as the type of the default value,
        or the parameter did not have an array before.

        Note that the parameter will iterate over the whole iterable once and store
        the individual data values into a tuple. Thus, the whole exploration range is
        explicitly stored in memory.

        :param explore_iterable: An iterable specifying the exploration range

         For example:

         >>> param = Parameter('Im.an.example', data=33.33, comment='Wooohoo!')
         >>> param._explore([3.0,2.0,1.0])
         >>> param._expand([42.0, 43.42])
         >>> param.f_get_range()
         >>> [3.0, 2.0, 1.0, 42.0, 43.42]

        :raises TypeError, ParameterLockedException

        """
        if self.v_locked:
            raise pex.ParameterLockedException('Parameter `%s` is locked!' % self.v_full_name)

        if not self.f_has_range():
            raise TypeError('Your Parameter `%s` is not an array and can therefore '
                            'not be expanded.' % self._name)

        data_list = self._data_sanity_checks(explore_iterable)

        self._explored_range.extend(data_list)
        self.f_lock()

Explores the parameter according to the iterable and appends to the exploration range .


def _data_sanity_checks(self, explore_iterable):
        """Checks if data values are  valid.

        Checks if the data values are supported by the parameter and if the values are of the same
        type as the default value.

        """
        data_list = []

        for val in explore_iterable:

            if not self.f_supports(val):
                raise TypeError('%s is of not supported type %s.' % (repr(val), str(type(val))))

            if not self._values_of_same_type(val, self._default):
                raise TypeError(
                    'Data of `%s` is not of the same type as the original entry value, '
                    'new type is %s vs old type %s.' %
                    (self.v_full_name, str(type(val)), str(type(self._default))))

            data_list.append(val)

        if len(data_list) == 0:
            raise ValueError('Cannot explore an empty list!')

        return data_list


def _store(self):
        """Returns a dictionary of formatted data understood by the storage service.

        The data is put into an :class:`~pypet.parameter.ObjectTable` named 'data'.
        If the parameter is explored, the exploration range is also put into another table
        named 'explored_data'.

        :return: Dictionary containing the data and optionally the exploration range.

        """
        if self._data is not None:
            store_dict = {'data': ObjectTable(data={'data': [self._data]})}

        if self.f_has_range():
            store_dict['explored_data'] = ObjectTable(data={'data': self._explored_range})

        self._locked = True

        return store_dict

def _store ( self ) : if self . _data is not None : store_dict = { 'data' : ObjectTable ( data = { 'data' : [ self . _data ] } ) } if self . f_has_range ( ) : store_dict [ 'explored_data' ] = ObjectTable ( data = { 'data' : self . _explored_range } ) self . _locked = True return store_dict

Returns a dictionary of formatted data understood by the storage service .


def _load(self, load_dict):
        """Loads the data and exploration range from the `load_dict`.

        The `load_dict` needs to be in the same format as the result of the
        :func:`~pypet.parameter.Parameter._store` method.

        """
        if self.v_locked:
            raise pex.ParameterLockedException('Parameter `%s` is locked!' % self.v_full_name)

        if 'data' in load_dict:
            self._data = load_dict['data']['data'][0]
            self._default = self._data
        else:
            self._logger.warning('Your parameter `%s` is empty, '
                                 'I did not find any data on disk.' % self.v_full_name)

        if 'explored_data' in load_dict:
            self._explored_range = [x for x in load_dict['explored_data']['data'].tolist()]
            self._explored = True

        self._locked = True

def
_load
(
self
,
load_dict
)
:
if
self
.
v_locked
:
raise
pex
.
ParameterLockedException
(
'Parameter `%s` is locked!'
%
self
.
v_full_name
)
if
'data'
in
load_dict
:
self
.
_data
=
load_dict
[
'data'
]
[
'data'
]
[
0
]
self
.
_default
=
self
.
_data
else
:
self
.
_logger
.
warning
(
'Your parameter `%s` is empty, '
'I did not find any data on disk.'
%
self
.
v_full_name
)
if
'explored_data'
in
load_dict
:
self
.
_explored_range
=
[
x
for
x
in
load_dict
[
'explored_data'
]
[
'data'
]
.
tolist
(
)
]
self
.
_explored
=
True
self
.
_locked
=
True

Loads the data and exploration range from the load_dict .


def _load(self, load_dict):
        """Reconstructs the data and exploration array.

        Checks if it can find the array identifier in the `load_dict`, i.e. '__rr__'.
        If not calls :class:`~pypet.parameter.Parameter._load` of the parent class.

        If the parameter is explored, the exploration range of arrays is reconstructed
        as it was stored in :func:`~pypet.parameter.ArrayParameter._store`.

        """
        if self.v_locked:
            raise pex.ParameterLockedException('Parameter `%s` is locked!' % self.v_full_name)

        try:
            self._data = load_dict['data' + ArrayParameter.IDENTIFIER]

            if 'explored_data' + ArrayParameter.IDENTIFIER in load_dict:
                explore_table = load_dict['explored_data' + ArrayParameter.IDENTIFIER]

                idx = explore_table['idx']

                explore_list = []

                # Recall the arrays in the order stored in the ObjectTable 'explored_data__rr__'
                for name_idx in idx:
                    arrayname = self._build_name(name_idx)
                    explore_list.append(load_dict[arrayname])

                self._explored_range = [x for x in explore_list]
                self._explored = True

        except KeyError:
            super(ArrayParameter, self)._load(load_dict)

        self._default = self._data
        self._locked = True

Reconstructs the data and exploration array .


def _equal_values(self, val1, val2):
        """Matrices are equal if they hash to the same value."""
        if self._is_supported_matrix(val1):
            if self._is_supported_matrix(val2):

                _, _, hash_tuple_1 = self._serialize_matrix(val1)
                _, _, hash_tuple_2 = self._serialize_matrix(val2)

                return hash(hash_tuple_1) == hash(hash_tuple_2)
            else:
                return False
        else:
            return super(SparseParameter, self)._equal_values(val1, val2)

def _equal_values ( self , val1 , val2 ) : if self . _is_supported_matrix ( val1 ) : if self . _is_supported_matrix ( val2 ) : _ , _ , hash_tuple_1 = self . _serialize_matrix ( val1 ) _ , _ , hash_tuple_2 = self . _serialize_matrix ( val2 ) return hash ( hash_tuple_1 ) == hash ( hash_tuple_2 ) else : return False else : return super ( SparseParameter , self ) . _equal_values ( val1 , val2 )

Matrices are equal if they hash to the same value .


def _is_supported_matrix(data):
        """Checks if a data is csr, csc, bsr, or dia Scipy sparse matrix"""
        return (spsp.isspmatrix_csc(data) or
                spsp.isspmatrix_csr(data) or
                spsp.isspmatrix_bsr(data) or
                spsp.isspmatrix_dia(data))

def _is_supported_matrix ( data ) : return ( spsp . isspmatrix_csc ( data ) or spsp . isspmatrix_csr ( data ) or spsp . isspmatrix_bsr ( data ) or spsp . isspmatrix_dia ( data ) )

Checks if a data is csr csc bsr or dia Scipy sparse matrix


def _serialize_matrix(matrix):
        """Extracts data from a sparse matrix to make it serializable in a human readable format.

        :return: Tuple with following elements:

            1.

                A list containing data that is necessary to reconstruct the matrix.
                For csr, csc, and bsr matrices the following attributes are extracted:
                `format`, `data`, `indices`, `indptr`, `shape`.
                Where format is simply one of the strings 'csr', 'csc', or 'bsr'.

                For dia matrices the following attributes are extracted:
                `format`, `data`, `offsets`, `shape`.
                Where `format` is simply the string 'dia'.

            2.

                A list containing the names of the extracted attributes.
                For csr, csc, and bsr:

                    [`format`, `data`, `indices`, `indptr`, `shape`]

                For dia:

                    [`format`, `data`, `offsets`, `shape`]

            3.

                A tuple containing the hashable parts of (1) in order to use the tuple as
                a key for a dictionary. Accordingly, the numpy arrays of (1) are
                changed to read-only.

        """
        if (spsp.isspmatrix_csc(matrix) or
                spsp.isspmatrix_csr(matrix) or
                spsp.isspmatrix_bsr(matrix)):
            if matrix.size > 0:
                return_list = [matrix.data, matrix.indices, matrix.indptr, matrix.shape]
            else:
                # For empty matrices we only need the shape
                return_list = ['__empty__', (), (), matrix.shape]

            return_names = SparseParameter.OTHER_NAME_LIST

            if spsp.isspmatrix_csc(matrix):
                return_list = ['csc'] + return_list
            elif spsp.isspmatrix_csr(matrix):
                return_list = ['csr'] + return_list
            elif spsp.isspmatrix_bsr(matrix):
                return_list = ['bsr'] + return_list
            else:
                raise RuntimeError('You shall not pass!')

        elif spsp.isspmatrix_dia(matrix):
            if matrix.size > 0:
                return_list = ['dia', matrix.data, matrix.offsets, matrix.shape]
            else:
                # For empty matrices we only need the shape
                return_list = ['dia', '__empty__', (), matrix.shape]

            return_names = SparseParameter.DIA_NAME_LIST
        else:
            raise RuntimeError('You shall not pass!')

        hash_list = []

        # Extract the `data` property of a read-only numpy array in order to have something
        # hashable.
        for item in return_list:
            if type(item) is np.ndarray:
                # item.flags.writeable = False
                hash_list.append(HashArray(item))
            else:
                hash_list.append(item)

        return return_list, return_names, tuple(hash_list)

Extracts data from a sparse matrix to make it serializable in a human readable format .


def _build_names(self, name_idx, is_dia):
        """Formats a name for storage

        :return: A tuple of names with the following format:

            `xspm__spsp__XXXX__spsp__XXXXXXXX` where the first 'XXXX' refer to the property and
            the latter 'XXXXXXX' to the sparse matrix index.

        """
        name_list = self._get_name_list(is_dia)
        return tuple(['explored%s.set_%05d.xspm_%s_%08d' % (SparseParameter.IDENTIFIER,
                                                         name_idx // 200, name, name_idx)
                                                                        for name in name_list])

def _build_names ( self , name_idx , is_dia ) : name_list = self . _get_name_list ( is_dia ) return tuple ( [ 'explored%s.set_%05d.xspm_%s_%08d' % ( SparseParameter . IDENTIFIER , name_idx // 200 , name , name_idx ) for name in name_list ] )


def _reconstruct_matrix(data_list):
        """Reconstructs a matrix from a list containing sparse matrix extracted properties

        `data_list` needs to be formatted as the first result of
        :func:`~pypet.parameter.SparseParameter._serialize_matrix`

        """
        matrix_format = data_list[0]
        data = data_list[1]
        is_empty = isinstance(data, str) and data == '__empty__'

        if matrix_format == 'csc':
            if is_empty:
                return spsp.csc_matrix(data_list[4])
            else:
                return spsp.csc_matrix(tuple(data_list[1:4]), shape=data_list[4])
        elif matrix_format == 'csr':
            if is_empty:
                return spsp.csr_matrix(data_list[4])
            else:
                return spsp.csr_matrix(tuple(data_list[1:4]), shape=data_list[4])
        elif matrix_format == 'bsr':
            if is_empty:
                # We have an empty matrix, that cannot be build as in elee case
                return spsp.bsr_matrix(data_list[4])
            else:
                return spsp.bsr_matrix(tuple(data_list[1:4]), shape=data_list[4])
        elif matrix_format == 'dia':
            if is_empty:
                return spsp.dia_matrix(data_list[3])
            else:
                return spsp.dia_matrix(tuple(data_list[1:3]), shape=data_list[3])
        else:
            raise RuntimeError('You shall not pass!')


def _load(self, load_dict):
        """Reconstructs the data and exploration array

        Checks if it can find the array identifier in the `load_dict`, i.e. '__spsp__'.
        If not, calls :class:`~pypet.parameter.ArrayParameter._load` of the parent class.

        If the parameter is explored, the exploration range of matrices is reconstructed
        as it was stored in :func:`~pypet.parameter.SparseParameter._store`.

        """
        if self.v_locked:
            raise pex.ParameterLockedException('Parameter `%s` is locked!' % self.v_full_name)

        try:
            is_dia = load_dict['data%sis_dia' % SparseParameter.IDENTIFIER]

            name_list = self._get_name_list(is_dia)
            rename_list = ['data%s%s' % (SparseParameter.IDENTIFIER, name)
                           for name in name_list]

            data_list = [load_dict[name] for name in rename_list]
            self._data = self._reconstruct_matrix(data_list)

            if 'explored_data' + SparseParameter.IDENTIFIER in load_dict:
                explore_table = load_dict['explored_data' + SparseParameter.IDENTIFIER]

                idx_col = explore_table['idx']
                dia_col = explore_table['is_dia']

                explore_list = []
                for irun, name_id in enumerate(idx_col):
                    is_dia = dia_col[irun]

                    # To make everything work with the old format we have the try catch block
                    try:
                        name_list = self._build_names(name_id, is_dia)
                        data_list = [load_dict[name] for name in name_list]
                    except KeyError:
                        name_list = self._build_names_old(name_id, is_dia)
                        data_list = [load_dict[name] for name in name_list]

                    matrix = self._reconstruct_matrix(data_list)
                    explore_list.append(matrix)

                self._explored_range = explore_list
                self._explored = True

        except KeyError:
            super(SparseParameter, self)._load(load_dict)

        self._default = self._data
        self._locked = True


def _store(self):
        """Returns a dictionary for storage.

        Every element in the dictionary except for 'explored_data' is a pickle dump.

        Reusage of objects is identified over the object id, i.e. python's built-in id function.

        'explored_data' contains the references to the objects to be able to recall the
        order of objects later on.

        """
        store_dict = {}

        if self._data is not None:
            dump = pickle.dumps(self._data, protocol=self.v_protocol)
            store_dict['data'] = dump
            store_dict[PickleParameter.PROTOCOL] = self.v_protocol

        if self.f_has_range():

            store_dict['explored_data'] = \
                ObjectTable(columns=['idx'], index=list(range(len(self))))

            smart_dict = {}
            count = 0

            for idx, val in enumerate(self._explored_range):

                obj_id = id(val)

                if obj_id in smart_dict:
                    name_id = smart_dict[obj_id]
                    add = False
                else:
                    name_id = count
                    add = True

                name = self._build_name(name_id)
                store_dict['explored_data']['idx'][idx] = name_id

                if add:
                    store_dict[name] = pickle.dumps(val, protocol=self.v_protocol)
                    smart_dict[obj_id] = name_id
                    count += 1

        self._locked = True

        return store_dict

def _store ( self ) : store_dict = { } if self . _data is not None : dump = pickle . dumps ( self . _data , protocol = self . v_protocol ) store_dict [ 'data' ] = dump store_dict [ PickleParameter . PROTOCOL ] = self . v_protocol if self . f_has_range ( ) : store_dict [ 'explored_data' ] = ObjectTable ( columns = [ 'idx' ] , index = list ( range ( len ( self ) ) ) ) smart_dict = { } count = 0 for idx , val in enumerate ( self . _explored_range ) : obj_id = id ( val ) if obj_id in smart_dict : name_id = smart_dict [ obj_id ] add = False else : name_id = count add = True name = self . _build_name ( name_id ) store_dict [ 'explored_data' ] [ 'idx' ] [ idx ] = name_id if add : store_dict [ name ] = pickle . dumps ( val , protocol = self . v_protocol ) smart_dict [ obj_id ] = name_id count += 1 self . _locked = True return store_dict


def _load(self, load_dict):
        """Reconstructs objects from the pickle dumps in `load_dict`.

        The 'explored_data' entry in `load_dict` is used to reconstruct
        the exploration range in the correct order.

        Sets the `v_protocol` property to the protocol used to store 'data'.

        """
        if self.v_locked:
            raise pex.ParameterLockedException('Parameter `%s` is locked!' % self.v_full_name)

        if 'data' in load_dict:
            dump = load_dict['data']
            self._data = pickle.loads(dump)
        else:
            self._logger.warning('Your parameter `%s` is empty, '
                                 'I did not find any data on disk.' % self.v_full_name)

        try:
            self.v_protocol = load_dict[PickleParameter.PROTOCOL]
        except KeyError:
            # For backwards compatibility
            self.v_protocol = PickleParameter._get_protocol(dump)

        if 'explored_data' in load_dict:
            explore_table = load_dict['explored_data']

            name_col = explore_table['idx']

            explore_list = []
            for name_id in name_col:
                arrayname = self._build_name(name_id)
                loaded = pickle.loads(load_dict[arrayname])
                explore_list.append(loaded)

            self._explored_range = explore_list
            self._explored = True

        self._default = self._data
        self._locked = True

def
_load
(
self
,
load_dict
)
:
if
self
.
v_locked
:
raise
pex
.
ParameterLockedException
(
'Parameter `%s` is locked!'
%
self
.
v_full_name
)
if
'data'
in
load_dict
:
dump
=
load_dict
[
'data'
]
self
.
_data
=
pickle
.
loads
(
dump
)
else
:
self
.
_logger
.
warning
(
'Your parameter `%s` is empty, '
'I did not find any data on disk.'
%
self
.
v_full_name
)
try
:
self
.
v_protocol
=
load_dict
[
PickleParameter
.
PROTOCOL
]
except
KeyError
:
# For backwards compatibility
self
.
v_protocol
=
PickleParameter
.
_get_protocol
(
dump
)
if
'explored_data'
in
load_dict
:
explore_table
=
load_dict
[
'explored_data'
]
name_col
=
explore_table
[
'idx'
]
explore_list
=
[
]
for
name_id
in
name_col
:
arrayname
=
self
.
_build_name
(
name_id
)
loaded
=
pickle
.
loads
(
load_dict
[
arrayname
]
)
explore_list
.
append
(
loaded
)
self
.
_explored_range
=
explore_list
self
.
_explored
=
True
self
.
_default
=
self
.
_data
self
.
_locked
=
True

Reconstructs objects from the pickle dumps in load_dict .


def f_translate_key(self, key):
        """Translates integer indices into the appropriate names"""
        if isinstance(key, int):
            if key == 0:
                key = self.v_name
            else:
                key = self.v_name + '_%d' % key
        return key

def f_translate_key ( self , key ) : if isinstance ( key , int ) : if key == 0 : key = self . v_name else : key = self . v_name + '_%d' % key return key


def f_val_to_str(self):
        """Summarizes data handled by the result as a string.

        Calls `__repr__` on all handled data. Data is NOT ordered.

        Truncates the string if it is longer than
        :const:`pypetconstants.HDF5_STRCOL_MAX_VALUE_LENGTH`

        :return: string

        """

        resstrlist = []
        strlen = 0

        for key in self._data:
            val = self._data[key]
            resstr = '%s=%s, ' % (key, repr(val))
            resstrlist.append(resstr)

            strlen += len(resstr)
            if strlen > pypetconstants.HDF5_STRCOL_MAX_VALUE_LENGTH:
                break

        return_string = "".join(resstrlist)
        if len(return_string) > pypetconstants.HDF5_STRCOL_MAX_VALUE_LENGTH:
            return_string =\
                return_string[0:pypetconstants.HDF5_STRCOL_MAX_VALUE_LENGTH - 3] + '...'
        else:
            return_string = return_string[0:-2] # Delete the last `, `

        return return_string

def
f_val_to_str
(
self
)
:
resstrlist
=
[
]
strlen
=
0
for
key
in
self
.
_data
:
val
=
self
.
_data
[
key
]
resstr
=
'%s=%s, '
%
(
key
,
repr
(
val
)
)
resstrlist
.
append
(
resstr
)
strlen
+=
len
(
resstr
)
if
strlen
>
pypetconstants
.
HDF5_STRCOL_MAX_VALUE_LENGTH
:
break
return_string
=
""
.
join
(
resstrlist
)
if
len
(
return_string
)
>
pypetconstants
.
HDF5_STRCOL_MAX_VALUE_LENGTH
:
return_string
=
return_string
[
0
:
pypetconstants
.
HDF5_STRCOL_MAX_VALUE_LENGTH
3
]
+
'...'
else
:
return_string
=
return_string
[
0
:
2
]
# Delete the last `, `
return
return_string

Summarizes data handled by the result as a string .


def f_to_dict(self, copy=True):
        """Returns all handled data as a dictionary.

        :param copy:

            Whether the original dictionary or a shallow copy is returned.

        :return: Data dictionary

        """
        if copy:
            return self._data.copy()
        else:
            return self._data

def f_to_dict ( self , copy = True ) : if copy : return self . _data . copy ( ) else : return self . _data

Returns all handled data as a dictionary .


def f_set(self, *args, **kwargs):
        """ Method to put data into the result.

        :param args:

            The first positional argument is stored with the name of the result.
            Following arguments are stored with `name_X` where `X` is the position
            of the argument.

        :param kwargs: Arguments are stored with the key as name.

        :raises: TypeError if outer data structure is not understood.

        Example usage:

        >>> res = Result('supergroup.subgroup.myresult', comment='I am a neat example!')
        >>> res.f_set(333,42.0, mystring='String!')
        >>> res.f_get('myresult')
        333
        >>> res.f_get('myresult_1')
        42.0
        >>> res.f_get(1)
        42.0
        >>> res.f_get('mystring')
        'String!'

        """
        if args and self.v_name is None:
            raise AttributeError('Cannot set positional value because I do not have a name!')
        for idx, arg in enumerate(args):
            valstr = self.f_translate_key(idx)
            self.f_set_single(valstr, arg)

        for key, arg in kwargs.items():
            self.f_set_single(key, arg)


def f_get(self, *args):
        """Returns items handled by the result.

         If only a single name is given, a single data item is returned. If several names are
         given, a list is returned. For integer inputs the result returns `resultname_X`.

         If the result contains only a single entry you can call `f_get()` without arguments.
         If you call `f_get()` and the result contains more than one element a ValueError is
         thrown.

         If the requested item(s) cannot be found an AttributeError is thrown.

        :param args: strings-names or integers

        :return: Single data item or tuple of data

        Example:

        >>> res = Result('supergroup.subgroup.myresult', comment='I am a neat example!' \
        [1000,2000], {'a':'b','c':333}, hitchhiker='Arthur Dent')
        >>> res.f_get('hitchhiker')
        'Arthur Dent'
        >>> res.f_get(0)
        [1000,2000]
        >>> res.f_get('hitchhiker', 'myresult')
        ('Arthur Dent', [1000,2000])

        """

        if len(args) == 0:
            if len(self._data) == 1:
                return list(self._data.values())[0]
            elif len(self._data) > 1:
                raise ValueError('Your result `%s` contains more than one entry: '
                                 '`%s` Please use >>f_get<< with one of these.' %
                                 (self.v_full_name, str(list(self._data.keys()))))
            else:
                raise AttributeError('Your result `%s` is empty, cannot access data.' %
                                     self.v_full_name)

        result_list = []
        for name in args:
            name = self.f_translate_key(name)
            if not name in self._data:
                if name == 'data' and len(self._data) == 1:
                    return self._data[list(self._data.keys())[0]]
                else:
                    raise AttributeError('`%s` is not part of your result `%s`.' %
                                         (name, self.v_full_name))

            result_list.append(self._data[name])

        if len(args) == 1:
            return result_list[0]
        else:
            return result_list


def f_set_single(self, name, item):
        """Sets a single data item of the result.

        Raises TypeError if the type of the outer data structure is not understood.
        Note that the type check is shallow. For example, if the data item is a list,
        the individual list elements are NOT checked whether their types are appropriate.

        :param name: The name of the data item

        :param item: The data item

        :raises: TypeError

        Example usage:

        >>> res.f_set_single('answer', 42)
        >>> res.f_get('answer')
        42

        """

        if self.v_stored:
            self._logger.debug('You are changing an already stored result. If '
                                 'you not explicitly overwrite the data on disk, this change '
                                 'might be lost and not propagated to disk.')

        if self._supports(item):

            # self._check_if_empty(item, name) # No longer needed

            if name in self._data:
                self._logger.debug('Replacing `%s` in result `%s`.' % (name, self.v_full_name))

            self._data[name] = item
        else:
            raise TypeError('Your result `%s` of type `%s` is not supported.' %
                            (name, str(type(item))))


def _supports(self, item):
        """Supports everything of parent class and csr, csc, bsr, and dia sparse matrices."""
        if SparseParameter._is_supported_matrix(item):
            return True
        else:
            return super(SparseResult, self)._supports(item)

def _supports ( self , item ) : if SparseParameter . _is_supported_matrix ( item ) : return True else : return super ( SparseResult , self ) . _supports ( item )

Supports everything of parent class and csr csc bsr and dia sparse matrices .


def _store(self):
        """Returns a storage dictionary understood by the storage service.

        Sparse matrices are extracted similar to the :class:`~pypet.parameter.SparseParameter` and
        marked with the identifier `__spsp__`.

        """
        store_dict = {}
        for key in self._data:
            val = self._data[key]
            if SparseParameter._is_supported_matrix(val):

                data_list, name_list, hash_tuple = SparseParameter._serialize_matrix(val)
                rename_list = ['%s%s%s' % (key, SparseParameter.IDENTIFIER, name)
                               for name in name_list]

                is_dia = int(len(rename_list) == 4)
                store_dict[key + SparseResult.IDENTIFIER + 'is_dia'] = is_dia

                for idx, name in enumerate(rename_list):
                    store_dict[name] = data_list[idx]

            else:
                store_dict[key] = val

        return store_dict

def _store ( self ) : store_dict = { } for key in self . _data : val = self . _data [ key ] if SparseParameter . _is_supported_matrix ( val ) : data_list , name_list , hash_tuple = SparseParameter . _serialize_matrix ( val ) rename_list = [ '%s%s%s' % ( key , SparseParameter . IDENTIFIER , name ) for name in name_list ] is_dia = int ( len ( rename_list ) == 4 ) store_dict [ key + SparseResult . IDENTIFIER + 'is_dia' ] = is_dia for idx , name in enumerate ( rename_list ) : store_dict [ name ] = data_list [ idx ] else : store_dict [ key ] = val return store_dict

Returns a storage dictionary understood by the storage service .


def _load(self, load_dict):
        """Loads data from `load_dict`

        Reconstruction of sparse matrices similar to the :class:`~pypet.parameter.SparseParameter`.

        """
        for key in list(load_dict.keys()):
            # We delete keys over time:
            if key in load_dict:
                if SparseResult.IDENTIFIER in key:
                    new_key = key.split(SparseResult.IDENTIFIER)[0]

                    is_dia = load_dict.pop(new_key + SparseResult.IDENTIFIER + 'is_dia')

                    name_list = SparseParameter._get_name_list(is_dia)
                    rename_list = ['%s%s%s' % (new_key, SparseResult.IDENTIFIER, name)
                                   for name in name_list]

                    data_list = [load_dict.pop(name) for name in rename_list]
                    matrix = SparseParameter._reconstruct_matrix(data_list)
                    self._data[new_key] = matrix
                else:
                    self._data[key] = load_dict[key]

def
_load
(
self
,
load_dict
)
:
for
key
in
list
(
load_dict
.
keys
(
)
)
:
# We delete keys over time:
if
key
in
load_dict
:
if
SparseResult
.
IDENTIFIER
in
key
:
new_key
=
key
.
split
(
SparseResult
.
IDENTIFIER
)
[
0
]
is_dia
=
load_dict
.
pop
(
new_key
+
SparseResult
.
IDENTIFIER
+
'is_dia'
)
name_list
=
SparseParameter
.
_get_name_list
(
is_dia
)
rename_list
=
[
'%s%s%s'
%
(
new_key
,
SparseResult
.
IDENTIFIER
,
name
)
for
name
in
name_list
]
data_list
=
[
load_dict
.
pop
(
name
)
for
name
in
rename_list
]
matrix
=
SparseParameter
.
_reconstruct_matrix
(
data_list
)
self
.
_data
[
new_key
]
=
matrix
else
:
self
.
_data
[
key
]
=
load_dict
[
key
]


def f_set_single(self, name, item):
        """Adds a single data item to the pickle result.

         Note that it is NOT checked if the item can be pickled!

        """

        if self.v_stored:
            self._logger.debug('You are changing an already stored result. If '
                                 'you not explicitly overwrite the data on disk, this change '
                                 'might be lost and not propagated to disk.')

        if name == PickleResult.PROTOCOL:
            raise AttributeError('You cannot name an entry `%s`' % PickleResult.PROTOCOL)

        self._data[name] = item

Adds a single data item to the pickle result .


def _store(self):
        """Returns a dictionary containing pickle dumps"""
        store_dict = {}
        for key, val in self._data.items():
            store_dict[key] = pickle.dumps(val, protocol=self.v_protocol)
        store_dict[PickleResult.PROTOCOL] = self.v_protocol
        return store_dict

def _store ( self ) : store_dict = { } for key , val in self . _data . items ( ) : store_dict [ key ] = pickle . dumps ( val , protocol = self . v_protocol ) store_dict [ PickleResult . PROTOCOL ] = self . v_protocol return store_dict


def _load(self, load_dict):
        """Reconstructs all items from the pickle dumps in `load_dict`.

        Sets the `v_protocol` property to the protocol of the first reconstructed item.

        """
        try:
            self.v_protocol = load_dict.pop(PickleParameter.PROTOCOL)
        except KeyError:
            # For backwards compatibility
            dump = next(load_dict.values())
            self.v_protocol = PickleParameter._get_protocol(dump)
        for key in load_dict:
            val = load_dict[key]
            self._data[key] = pickle.loads(val)

def
_load
(
self
,
load_dict
)
:
try
:
self
.
v_protocol
=
load_dict
.
pop
(
PickleParameter
.
PROTOCOL
)
except
KeyError
:
# For backwards compatibility
dump
=
next
(
load_dict
.
values
(
)
)
self
.
v_protocol
=
PickleParameter
.
_get_protocol
(
dump
)
for
key
in
load_dict
:
val
=
load_dict
[
key
]
self
.
_data
[
key
]
=
pickle
.
loads
(
val
)

Reconstructs all items from the pickle dumps in load_dict .


def main():
    """Simply merge all trajectories in the working directory"""
    folder = os.getcwd()
    print('Merging all files')
    merge_all_in_folder(folder,
                        delete_other_files=True,  # We will only keep one trajectory
                        dynamic_imports=FunctionParameter,
                        backup=False)
    print('Done')

def
main
(
)
:
folder
=
os
.
getcwd
(
)
print
(
'Merging all files'
)
merge_all_in_folder
(
folder
,
delete_other_files
=
True
,
# We will only keep one trajectory
dynamic_imports
=
FunctionParameter
,
backup
=
False
)
print
(
'Done'
)


def upload_file(filename, session):
    """ Uploads a file """
    print('Uploading file %s' % filename)
    outfilesource = os.path.join(os.getcwd(), filename)
    outfiletarget = 'sftp://' + ADDRESS + WORKING_DIR
    out = saga.filesystem.File(outfilesource, session=session, flags=OVERWRITE)
    out.copy(outfiletarget)
    print('Transfer of `%s` to `%s` successful' % (filename, outfiletarget))

def
upload_file
(
filename
,
session
)
:
print
(
'Uploading file %s'
%
filename
)
outfilesource
=
os
.
path
.
join
(
os
.
getcwd
(
)
,
filename
)
outfiletarget
=
'sftp://'
+
ADDRESS
+
WORKING_DIR
out
=
saga
.
filesystem
.
File
(
outfilesource
,
session
=
session
,
flags
=
OVERWRITE
)
out
.
copy
(
outfiletarget
)
print
(
'Transfer of `%s` to `%s` successful'
%
(
filename
,
outfiletarget
)
)


def download_file(filename, session):
    """ Downloads a file """
    print('Downloading file %s' % filename)
    infilesource = os.path.join('sftp://' + ADDRESS + WORKING_DIR,
                                 filename)
    infiletarget = os.path.join(os.getcwd(), filename)
    incoming = saga.filesystem.File(infilesource, session=session, flags=OVERWRITE)
    incoming.copy(infiletarget)
    print('Transfer of `%s` to `%s` successful' % (filename, infiletarget))

def
download_file
(
filename
,
session
)
:
print
(
'Downloading file %s'
%
filename
)
infilesource
=
os
.
path
.
join
(
'sftp://'
+
ADDRESS
+
WORKING_DIR
,
filename
)
infiletarget
=
os
.
path
.
join
(
os
.
getcwd
(
)
,
filename
)
incoming
=
saga
.
filesystem
.
File
(
infilesource
,
session
=
session
,
flags
=
OVERWRITE
)
incoming
.
copy
(
infiletarget
)
print
(
'Transfer of `%s` to `%s` successful'
%
(
filename
,
infiletarget
)
)


def create_session():
    """ Creates and returns a new SAGA session """
    ctx = saga.Context("UserPass")
    ctx.user_id = USER
    ctx.user_pass = PASSWORD

    session = saga.Session()
    session.add_context(ctx)

    return session

def create_session ( ) : ctx = saga . Context ( "UserPass" ) ctx . user_id = USER ctx . user_pass = PASSWORD session = saga . Session ( ) session . add_context ( ctx ) return session


def merge_trajectories(session):
    """ Merges all trajectories found in the working directory """
    jd = saga.job.Description()

    jd.executable      = 'python'
    jd.arguments       = ['merge_trajs.py']
    jd.output          = "mysagajob_merge.stdout"
    jd.error           = "mysagajob_merge.stderr"
    jd.working_directory = WORKING_DIR

    js = saga.job.Service('ssh://' + ADDRESS, session=session)
    myjob = js.create_job(jd)
    print("\n...starting job...\n")

    # Now we can start our job.
    myjob.run()
    print("Job ID    : %s" % (myjob.id))
    print("Job State : %s" % (myjob.state))

    print("\n...waiting for job...\n")
    # wait for the job to either finish or fail
    myjob.wait()

    print("Job State : %s" % (myjob.state))
    print("Exitcode  : %s" % (myjob.exit_code))


def start_jobs(session):
    """ Starts all jobs and runs `the_task.py` in batches. """

    js = saga.job.Service('ssh://' + ADDRESS, session=session)

    batches = range(3)
    jobs = []

    for batch in batches:
        print('Starting batch %d' % batch)

        jd = saga.job.Description()

        jd.executable      = 'python'
        jd.arguments       = ['the_task.py --batch=' + str(batch)]
        jd.output          = "mysagajob.stdout" + str(batch)
        jd.error           = "mysagajob.stderr" + str(batch)
        jd.working_directory = WORKING_DIR

        myjob = js.create_job(jd)

        print("Job ID    : %s" % (myjob.id))
        print("Job State : %s" % (myjob.state))

        print("\n...starting job...\n")

        myjob.run()
        jobs.append(myjob)

    for myjob in jobs:
        print("Job ID    : %s" % (myjob.id))
        print("Job State : %s" % (myjob.state))

        print("\n...waiting for job...\n")
        # wait for the job to either finish or fail
        myjob.wait()

        print("Job State : %s" % (myjob.state))
        print("Exitcode  : %s" % (myjob.exit_code))

Starts all jobs and runs the_task . py in batches .


def multiply(traj):
    """Sophisticated simulation of multiplication"""
    z=traj.x*traj.y
    traj.f_add_result('z',z=z, comment='I am the product of two reals!')

def
multiply
(
traj
)
:
z
=
traj
.
x
*
traj
.
y
traj
.
f_add_result
(
'z'
,
z
=
z
,
comment
=
'I am the product of two reals!'
)


def run_neuron(traj):
    """Runs a simulation of a model neuron.

    :param traj:

        Container with all parameters.

    :return:

        An estimate of the firing rate of the neuron

    """

    # Extract all parameters from `traj`
    V_init = traj.par.neuron.V_init
    I = traj.par.neuron.I
    tau_V = traj.par.neuron.tau_V
    tau_ref = traj.par.neuron.tau_ref
    dt = traj.par.simulation.dt
    duration = traj.par.simulation.duration

    steps = int(duration / float(dt))
    # Create some containers for the Euler integration
    V_array = np.zeros(steps)
    V_array[0] = V_init
    spiketimes = []  # List to collect all times of action potentials

    # Do the Euler integration:
    print('Starting Euler Integration')
    for step in range(1, steps):
        if V_array[step-1] >= 1:
            # The membrane potential crossed the threshold and we mark this as
            # an action potential
            V_array[step] = 0
            spiketimes.append((step-1)*dt)
        elif spiketimes and step * dt - spiketimes[-1] <= tau_ref:
            # We are in the refractory period, so we simply clamp the voltage
            # to 0
            V_array[step] = 0
        else:
            # Euler Integration step:
            dV = -1/tau_V * V_array[step-1] + I
            V_array[step] = V_array[step-1] + dV*dt

    print('Finished Euler Integration')

    # Add the voltage trace and spike times
    traj.f_add_result('neuron.$', V=V_array, nspikes=len(spiketimes),
                      comment='Contains the development of the membrane potential over time '
                              'as well as the number of spikes.')
    # This result will be renamed to `traj.results.neuron.run_XXXXXXXX`.

    # And finally we return the estimate of the firing rate
    return len(spiketimes) / float(traj.par.simulation.duration) *1000


def neuron_postproc(traj, result_list):
    """Postprocessing, sorts computed firing rates into a table

    :param traj:

        Container for results and parameters

    :param result_list:

        List of tuples, where first entry is the run index and second is the actual
        result of the corresponding run.

    :return:
    """

    # Let's create a pandas DataFrame to sort the computed firing rate according to the
    # parameters. We could have also used a 2D numpy array.
    # But a pandas DataFrame has the advantage that we can index into directly with
    # the parameter values without translating these into integer indices.
    I_range = traj.par.neuron.f_get('I').f_get_range()
    ref_range = traj.par.neuron.f_get('tau_ref').f_get_range()

    I_index = sorted(set(I_range))
    ref_index = sorted(set(ref_range))
    rates_frame = pd.DataFrame(columns=ref_index, index=I_index)
    # This frame is basically a two dimensional table that we can index with our
    # parameters

    # Now iterate over the results. The result list is a list of tuples, with the
    # run index at first position and our result at the second
    for result_tuple in result_list:
        run_idx = result_tuple[0]
        firing_rates = result_tuple[1]
        I_val = I_range[run_idx]
        ref_val = ref_range[run_idx]
        rates_frame.loc[I_val, ref_val] = firing_rates # Put the firing rate into the
        # data frame

    # Finally we going to store our new firing rate table into the trajectory
    traj.f_add_result('summary.firing_rates', rates_frame=rates_frame,
                      comment='Contains a pandas data frame with all firing rates.')

Postprocessing sorts computed firing rates into a table


def add_parameters(traj):
    """Adds all parameters to `traj`"""
    print('Adding Parameters')

    traj.f_add_parameter('neuron.V_init', 0.0,
                         comment='The initial condition for the '
                                    'membrane potential')
    traj.f_add_parameter('neuron.I', 0.0,
                         comment='The externally applied current.')
    traj.f_add_parameter('neuron.tau_V', 10.0,
                         comment='The membrane time constant in milliseconds')
    traj.f_add_parameter('neuron.tau_ref', 5.0,
                        comment='The refractory period in milliseconds '
                                'where the membrane potnetial '
                                'is clamped.')

    traj.f_add_parameter('simulation.duration', 1000.0,
                         comment='The duration of the experiment in '
                                'milliseconds.')
    traj.f_add_parameter('simulation.dt', 0.1,
                         comment='The step size of an Euler integration step.')


def add_exploration(traj):
    """Explores different values of `I` and `tau_ref`."""

    print('Adding exploration of I and tau_ref')

    explore_dict = {'neuron.I': np.arange(0, 1.01, 0.01).tolist(),
                    'neuron.tau_ref': [5.0, 7.5, 10.0]}

    explore_dict = cartesian_product(explore_dict, ('neuron.tau_ref', 'neuron.I'))
    # The second argument, the tuple, specifies the order of the cartesian product,
    # The variable on the right most side changes fastest and defines the
    # 'inner for-loop' of the cartesian product

    traj.f_explore(explore_dict)

Explores different values of I and tau_ref .


def execute_network_pre_run(self, traj, network, network_dict, component_list, analyser_list):
        """Runs a network before the actual experiment.

        Called by a :class:`~pypet.brian2.network.NetworkManager`.
        Similar to :func:`~pypet.brian2.network.NetworkRunner.run_network`.

        Subruns and their durations are extracted from the trajectory. All
        :class:`~pypet.brian2.parameter.Brian2Parameter` instances found under
        `traj.parameters.simulation.pre_durations` (default, you can change the
        name of the group where to search for durations at runner initialisation).
        The order is determined from
        the `v_annotations.order` attributes. There must be at least one subrun in the trajectory,
        otherwise an AttributeError is thrown. If two subruns equal in their order
        property a RuntimeError is thrown.

        :param traj: Trajectory container

        :param network: BRIAN2 network

        :param network_dict: Dictionary of items shared among all components

        :param component_list: List of :class:`~pypet.brian2.network.NetworkComponent` objects

        :param analyser_list: List of :class:`~pypet.brian2.network.NetworkAnalyser` objects

        """
        self._execute_network_run(traj, network, network_dict, component_list, analyser_list,
                                  pre_run=True)

def execute_network_pre_run ( self , traj , network , network_dict , component_list , analyser_list ) : self . _execute_network_run ( traj , network , network_dict , component_list , analyser_list , pre_run = True )

Runs a network before the actual experiment .


def execute_network_run(self, traj, network, network_dict, component_list, analyser_list):
        """Runs a network in an experimental run.

        Called by a :class:`~pypet.brian2.network.NetworkManager`.

        A network run is divided into several subruns which are defined as
        :class:`~pypet.brian2.parameter.Brian2Parameter` instances.

        These subruns are extracted from the trajectory. All
        :class:`~pypet.brian2.parameter.Brian2Parameter` instances found under
        `traj.parameters.simulation.durations` (default, you can change the
        name of the group where to search for durations at runner initialisation).
        The order is determined from
        the `v_annotations.order` attributes. An error is thrown if no orders attribute
        can be found or if two parameters have the same order.

        There must be at least one subrun in the trajectory,
        otherwise an AttributeError is thrown. If two subruns equal in their order
        property a RuntimeError is thrown.

        For every subrun the following steps are executed:

        1.  Calling :func:`~pypet.brian2.network.NetworkComponent.add_to_network` for every
            every :class:`~pypet.brian2.network.NetworkComponent` in the order as
            they were passed to the :class:`~pypet.brian2.network.NetworkManager`.

        2.  Calling :func:`~pypet.brian2.network.NetworkComponent.add_to_network` for every
            every :class:`~pypet.brian2.network.NetworkAnalyser` in the order as
            they were passed to the :class:`~pypet.brian2.network.NetworkManager`.

        3.  Calling :func:`~pypet.brian2.network.NetworkComponent.add_to_network` of the
            NetworkRunner itself (usually the network runner should not add or remove
            anything from the network, but this step is executed for completeness).

        4.  Running the BRIAN2 network for the duration of the current subrun by calling
            the network's `run` function.

        5.  Calling :func:`~pypet.brian2.network.NetworkAnalyser.analyse` for every
            every :class:`~pypet.brian2.network.NetworkAnalyser` in the order as
            they were passed to the :class:`~pypet.brian2.network.NetworkManager`.

        6.  Calling :func:`~pypet.brian2.network.NetworkComponent.remove_from_network` of the
            NetworkRunner itself (usually the network runner should not add or remove
            anything from the network, but this step is executed for completeness).

        7.  Calling :func:`~pypet.brian2.network.NetworkComponent.remove_from_network` for every
            every :class:`~pypet.brian2.network.NetworkAnalyser` in the order as
            they were passed to the :class:`~pypet.brian2.network.NetworkManager`

        8.  Calling :func:`~pypet.brian2.network.NetworkComponent.remove_from_network` for every
            every :class:`~pypet.brian2.network.NetworkComponent` in the order as
            they were passed to the :class:`~pypet.brian2.network.NetworkManager`.

        These 8 steps are repeated for every subrun in the `subrun_list`.
        The `subrun_list` passed to all `add_to_network`, `analyse` and
        `remove_from_network` methods can be modified
        within these functions to potentially alter the order of execution or
        even erase or add upcoming subruns if necessary.

        For example, a NetworkAnalyser checks
        for epileptic pathological activity and cancels all coming subruns in case
        of undesired network dynamics.

        :param traj: Trajectory container

        :param network: BRIAN2 network

        :param network_dict: Dictionary of items shared among all components

        :param component_list: List of :class:`~pypet.brian2.network.NetworkComponent` objects

        :param analyser_list: List of :class:`~pypet.brian2.network.NetworkAnalyser` objects

        """
        self._execute_network_run(traj, network, network_dict, component_list, analyser_list,
                                  pre_run=False)

def execute_network_run ( self , traj , network , network_dict , component_list , analyser_list ) : self . _execute_network_run ( traj , network , network_dict , component_list , analyser_list , pre_run = False )


def _extract_subruns(self, traj, pre_run=False):
        """Extracts subruns from the trajectory.

        :param traj: Trajectory container

        :param pre_run: Boolean whether current run is regular or a pre-run

        :raises: RuntimeError if orders are duplicates or even missing

        """
        if pre_run:
            durations_list = traj.f_get_all(self._pre_durations_group_name)
        else:
            durations_list = traj.f_get_all(self._durations_group_name)

        subruns = {}
        orders = []

        for durations in durations_list:
            for duration_param in durations.f_iter_leaves(with_links=False):

                if 'order' in duration_param.v_annotations:
                    order = duration_param.v_annotations.order
                else:
                    raise RuntimeError('Your duration parameter %s has no order. Please add '
                                       'an order in `v_annotations.order`.' %
                                       duration_param.v_full_name)

                if order in subruns:
                    raise RuntimeError('Your durations must differ in their order, there are two '
                                       'with order %d.' % order)
                else:
                    subruns[order] = duration_param
                    orders.append(order)

        return [subruns[order] for order in sorted(orders)]


def _execute_network_run(self, traj, network, network_dict, component_list,
                             analyser_list, pre_run=False):
        """Generic `execute_network_run` function, handles experimental runs as well as pre-runs.

        See also :func:`~pypet.brian2.network.NetworkRunner.execute_network_run` and
         :func:`~pypet.brian2.network.NetworkRunner.execute_network_pre_run`.

        """

        # Initially extract the `subrun_list`
        subrun_list = self._extract_subruns(traj, pre_run=pre_run)

        # counter for subruns
        subrun_number = 0

        # Execute all subruns in order
        while len(subrun_list) > 0:

            # Get the next subrun
            current_subrun = subrun_list.pop(0)

            # 1. Call `add` of all normal components
            for component in component_list:
                component.add_to_network(traj, network, current_subrun, subrun_list,
                                         network_dict)

            # 2. Call `add` of all analyser components
            for analyser in analyser_list:
                analyser.add_to_network(traj, network, current_subrun, subrun_list,
                                        network_dict)

            # 3. Call `add` of the network runner itself
            self.add_to_network(traj, network, current_subrun, subrun_list,
                                network_dict)

            # 4. Run the network
            self._logger.info('STARTING subrun `%s` (#%d) lasting %s.' %
                             (current_subrun.v_name, subrun_number, str(current_subrun.f_get())))
            network.run(duration=current_subrun.f_get(), report=self._report,
                              report_period=self._report_period)

            # 5. Call `analyse` of all analyser components
            for analyser in analyser_list:
                analyser.analyse(traj, network, current_subrun, subrun_list,
                                 network_dict)

            # 6. Call `remove` of the network runner itself
            self.remove_from_network(traj, network, current_subrun, subrun_list,
                                     network_dict)

            # 7. Call `remove` for all analyser components
            for analyser in analyser_list:
                analyser.remove_from_network(traj, network, current_subrun, subrun_list,
                                             network_dict)

            # 8. Call `remove` for all normal components
            for component in component_list:
                component.remove_from_network(traj, network, current_subrun, subrun_list,
                                              network_dict)

            subrun_number += 1

Generic execute_network_run function handles experimental runs as well as pre - runs .


def add_parameters(self, traj):
        """Adds parameters for a network simulation.

        Calls :func:`~pypet.brian2.network.NetworkComponent.add_parameters` for all components,
        analyser, and the network runner (in this order).

        :param traj:  Trajectory container

        """
        self._logger.info('Adding Parameters of Components')

        for component in self.components:
            component.add_parameters(traj)

        if self.analysers:
            self._logger.info('Adding Parameters of Analysers')

            for analyser in self.analysers:
                analyser.add_parameters(traj)

        self._logger.info('Adding Parameters of Runner')

        self.network_runner.add_parameters(traj)

def
add_parameters
(
self
,
traj
)
:
self
.
_logger
.
info
(
'Adding Parameters of Components'
)
for
component
in
self
.
components
:
component
.
add_parameters
(
traj
)
if
self
.
analysers
:
self
.
_logger
.
info
(
'Adding Parameters of Analysers'
)
for
analyser
in
self
.
analysers
:
analyser
.
add_parameters
(
traj
)
self
.
_logger
.
info
(
'Adding Parameters of Runner'
)
self
.
network_runner
.
add_parameters
(
traj
)

Adds parameters for a network simulation .


def pre_run_network(self, traj):
        """Starts a network run before the individual run.

        Useful if a network needs an initial run that can be shared by all individual
        experimental runs during parameter exploration.

        Needs to be called by the user. If `pre_run_network` is started by the user,
        :func:`~pypet.brian2.network.NetworkManager.pre_build` will be automatically called
        from this function.

        This function will create a new BRIAN2 network which is run by
        the :class:`~pypet.brian2.network.NetworkRunner` and it's
        :func:`~pypet.brian2.network.NetworkRunner.execute_network_pre_run`.

        To see how a network run is structured also take a look at
        :func:`~pypet.brian2.network.NetworkRunner.run_network`.

        :param traj: Trajectory container

        """
        self.pre_build(traj)

        self._logger.info('\n------------------------\n'
                          'Pre-Running the Network\n'
                          '------------------------')

        self._network = self._network_constructor(*self._brian_list)
        self.network_runner.execute_network_pre_run(traj, self._network, self._network_dict,
                                                    self.components, self.analysers)

        self._logger.info('\n-----------------------------\n'
                          'Network Simulation successful\n'
                          '-----------------------------')

        self._pre_run = True
        if hasattr(self._network, 'store'):
            self._network.store('pre_run')

def
pre_run_network
(
self
,
traj
)
:
self
.
pre_build
(
traj
)
self
.
_logger
.
info
(
'\n------------------------\n'
'Pre-Running the Network\n'
'------------------------'
)
self
.
_network
=
self
.
_network_constructor
(
*
self
.
_brian_list
)
self
.
network_runner
.
execute_network_pre_run
(
traj
,
self
.
_network
,
self
.
_network_dict
,
self
.
components
,
self
.
analysers
)
self
.
_logger
.
info
(
'\n-----------------------------\n'
'Network Simulation successful\n'
'-----------------------------'
)
self
.
_pre_run
=
True
if
hasattr
(
self
.
_network
,
'store'
)
:
self
.
_network
.
store
(
'pre_run'
)

Starts a network run before the individual run .


def run_network(self, traj):
        """Top-level simulation function, pass this to the environment

        Performs an individual network run during parameter exploration.

        `run_network` does not need to be called by the user. If this
        method (not this one of the NetworkManager)
        is passed to an :class:`~pypet.environment.Environment` with this NetworkManager,
        `run_network` and :func:`~pypet.brian2.network.NetworkManager.build`
        are automatically called for each individual experimental run.

        This function will create a new BRIAN2 network in case one was not pre-run.
        The execution of the network run is carried out by
        the :class:`~pypet.brian2.network.NetworkRunner` and it's
        :func:`~pypet.brian2.network.NetworkRunner.execute_network_run` (also take
        a look at this function's documentation to see the structure of a network run).

        :param traj: Trajectory container

        """

        # Check if the network was pre-built
        if self._pre_built:
            if self._pre_run and hasattr(self._network, 'restore'):
                self._network.restore('pre_run')
                # Temprorary fix for https://github.com/brian-team/brian2/issues/681
                self._network.store('pre_run')
            self._run_network(traj)
        else:
            self._run_network(traj)

Top - level simulation function pass this to the environment


def _run_network(self, traj):
        """Starts a single run carried out by a NetworkRunner.

        Called from the public function :func:`~pypet.brian2.network.NetworkManger.run_network`.

        :param traj: Trajectory container

        """
        self.build(traj)

        self._pretty_print_explored_parameters(traj)

        # We need to construct a network object in case one was not pre-run
        if not self._pre_run:

            self._network = self._network_constructor(*self._brian_list)

        # Start the experimental run
        self.network_runner.execute_network_run(traj, self._network, self._network_dict,
                                                self.components, self.analysers)

        self._logger.info('\n-----------------------------\n'
                          'Network Simulation successful\n'
                          '-----------------------------')

Starts a single run carried out by a NetworkRunner .


def make_filename(traj):
    """ Function to create generic filenames based on what has been explored """
    explored_parameters = traj.f_get_explored_parameters()
    filename = ''
    for param in explored_parameters.values():
        short_name = param.v_name
        val = param.f_get()
        filename += '%s_%s__' % (short_name, str(val))

    return filename[:-2] + '.png'

def make_filename ( traj ) : explored_parameters = traj . f_get_explored_parameters ( ) filename = '' for param in explored_parameters . values ( ) : short_name = param . v_name val = param . f_get ( ) filename += '%s_%s__' % ( short_name , str ( val ) ) return filename [ : - 2 ] + '.png'


def next(self):
        """Returns next element from chain.

        More precisely, it returns the next element of the
        foremost iterator. If this iterator is empty it moves iteratively
        along the chain of available iterators to pick the new foremost one.

        Raises StopIteration if there are no elements left.

        """
        while True:
            # We need this loop because some iterators may already be empty.
            # We keep on popping from the left until next succeeds and as long
            # as there are iterators available
            try:
                return next(self._current)
            except StopIteration:
                try:
                    self._current = iter(self._chain.popleft())
                except IndexError:
                    # If we run out of iterators we are sure that
                    # there can be no more element
                    raise StopIteration('Reached end of iterator chain')


def merge_all_in_folder(folder, ext='.hdf5',
                        dynamic_imports=None,
                        storage_service=None,
                        force=False,
                        ignore_data=(),
                        move_data=False,
                        delete_other_files=False,
                        keep_info=True,
                        keep_other_trajectory_info=True,
                        merge_config=True,
                        backup=True):
    """Merges all files in a given folder.

    IMPORTANT: Does not check if there are more than 1 trajectory in a file. Always
    uses the last trajectory in file and ignores the other ones.

    Trajectories are merged according to the alphabetical order of the files,
    i.e. the resulting merged trajectory is found in the first file
    (according to lexicographic ordering).

    :param folder: folder (not recursive) where to look for files
    :param ext: only files with the given extension are used
    :param dynamic_imports: Dynamic imports for loading
    :param storage_service: storage service to use, leave `None` to use the default one
    :param force: If loading should be forced.
    :param delete_other_files: Deletes files of merged trajectories

    All other parameters as in `f_merge_many` of the trajectory.

    :return: The merged traj

    """
    in_dir = os.listdir(folder)
    all_files = []
    # Find all files with matching extension
    for file in in_dir:
        full_file = os.path.join(folder, file)
        if os.path.isfile(full_file):
            _, extension = os.path.splitext(full_file)
            if extension == ext:
                all_files.append(full_file)
    all_files = sorted(all_files)

    # Open all trajectories
    trajs = []
    for full_file in all_files:
        traj = load_trajectory(index=-1,
                               storage_service=storage_service,
                               filename=full_file,
                               load_data=0,
                               force=force,
                               dynamic_imports=dynamic_imports)
        trajs.append(traj)

    # Merge all trajectories
    first_traj = trajs.pop(0)
    first_traj.f_merge_many(trajs,
                ignore_data=ignore_data,
                move_data=move_data,
                delete_other_trajectory=False,
                keep_info=keep_info,
                keep_other_trajectory_info=keep_other_trajectory_info,
                merge_config=merge_config,
                backup=backup)

    if delete_other_files:
        # Delete all but the first file
        for file in all_files[1:]:
            os.remove(file)

    return first_traj