""" An example showing how to use DEAP optimization (http://pythonhosted.org/deap/).

DEAP can be combined with *pypet* to keep track of all the data and the full trajectory
of points created by a genetic algorithm.

Note that *pypet* adds quite some overhead to the optimization algorithm.
Using *pypet* in combination with DEAP is only suitable in case the
evaluation of an individual (i.e. a single run) takes a considerable amount of time
(i.e. 1 second or longer) and, thus, pypet's overhead is only marginal.

This *OneMax* problem serves only as an example and is not a well suited problem.
Suitable would be the genetic optimization of neural networks where running and evaluating
the network may take a few seconds.

Here we avoid using an Environment and *manually* execute runs using multiprocessing.

"""

__author__ = 'Robert Meyer'

import random

import os
import multiprocessing as multip
try:
    from itertools import izip
except ImportError:
    # For Python 3
    izip = zip

from deap import base
from deap import creator
from deap import tools

from pypet import Trajectory, cartesian_product, manual_run, MultiprocContext

@manual_run(store_meta_data=True)   # Important decorator for manual execution of runs
def eval_one_max(traj, individual):
    """The fitness function"""
    traj.f_add_result('$set.$.individual', list(individual))
    fitness = sum(individual)
    traj.f_add_result('$set.$.fitness', fitness)
    traj.f_store()
    return (fitness,)  # DEAP wants a tuple here!

def eval_wrapper(the_tuple):
    """Wrapper function that unpacks a single tuple as arguments to the fitness function.

    The pool's map function only allows a single iterable so we need to zip it first
    and then unpack it here.

    """
    return eval_one_max(*the_tuple)

def main():

    # No environment here ;-)
    filename = os.path.join('experiments', 'example_20.hdf5')
    traj = Trajectory('onemax', filename=filename, overwrite_file=True)

    # ------- Add parameters ------- #
    traj.f_add_parameter('popsize', 100)
    traj.f_add_parameter('CXPB', 0.5)
    traj.f_add_parameter('MUTPB', 0.2)
    traj.f_add_parameter('NGEN', 20)

    traj.f_add_parameter('generation', 0)
    traj.f_add_parameter('ind_idx', 0)
    traj.f_add_parameter('ind_len', 50)

    traj.f_add_parameter('indpb', 0.005)
    traj.f_add_parameter('tournsize', 3)

    traj.f_add_parameter('seed', 42)
    traj.f_store(only_init=True)

    # ------- Create and register functions with DEAP ------- #
    creator.create("FitnessMax", base.Fitness, weights=(1.0,))
    creator.create("Individual", list, fitness=creator.FitnessMax)

    toolbox = base.Toolbox()
    # Attribute generator
    toolbox.register("attr_bool", random.randint, 0, 1)
    # Structure initializers
    toolbox.register("individual", tools.initRepeat, creator.Individual,
        toolbox.attr_bool, traj.ind_len)
    toolbox.register("population", tools.initRepeat, list, toolbox.individual)

    # Operator registering
    toolbox.register("mate", tools.cxTwoPoint)
    toolbox.register("mutate", tools.mutFlipBit, indpb=traj.indpb)
    toolbox.register("select", tools.selTournament, tournsize=traj.tournsize)
    toolbox.register("evaluate", eval_wrapper)

    pool = multip.Pool(4)
    toolbox.register("map", pool.map)  # We use the pool's map function!

    # ------- Initialize Population -------- #
    random.seed(traj.seed)

    pop = toolbox.population(n=traj.popsize)
    CXPB, MUTPB, NGEN = traj.CXPB, traj.MUTPB, traj.NGEN

    start_idx = 0  # We need to count executed runs

    print("Start of evolution")
    for g in range(traj.NGEN):
        print("-- Generation %i --" % g)

        # Determine individuals that need to be evaluated
        eval_pop = [ind for ind in pop if not ind.fitness.valid]

        # Add as many explored runs as individuals that need to be evaluated
        traj.f_expand(cartesian_product({'generation': [g], 'ind_idx': range(len(eval_pop))}))

        # We need to make the storage service multiprocessing safe
        mc = MultiprocContext(traj, wrap_mode='QUEUE')
        mc.f_start()

        # Create a single iterable to be passed to our fitness function (wrapper).
        # `yields='copy'` is important, the pool's `map` function will
        # go over the whole iterator at once and store it in memory.
        # So for every run we need a copy of the trajectory.
        # Alternatively, you could use `yields='self'` and use the pool's `imap` function.
        zip_iterable = izip(traj.f_iter_runs(start_idx, yields='copy'), eval_pop)

        fitnesses = toolbox.map(eval_wrapper, zip_iterable)
        # fitnesses is just a list of tuples [(fitness,), ...]
        for idx, fitness in enumerate(fitnesses):
            # Update fitnesses
            eval_pop[idx].fitness.values = fitness

        # Finalize the multiproc wrapper
        mc.f_finalize()

        # Update start index
        start_idx += len(eval_pop)

        print("  Evaluated %i individuals" % len(eval_pop))

        # Gather all the fitnesses in one list and print the stats
        fits = [ind.fitness.values[0] for ind in pop]

        length = len(pop)
        mean = sum(fits) / length
        sum2 = sum(x*x for x in fits)
        std = abs(sum2 / length - mean**2)**0.5

        print("  Min %s" % min(fits))
        print("  Max %s" % max(fits))
        print("  Avg %s" % mean)
        print("  Std %s" % std)

        # ------- Create the next generation by crossover and mutation -------- #
        if g < traj.NGEN -1:   # not necessary for the last generation
            # Select the next generation individuals
            offspring = toolbox.select(pop, len(pop))
            # Clone the selected individuals
            offspring = list(map(toolbox.clone, offspring))

            # Apply crossover and mutation on the offspring
            for child1, child2 in zip(offspring[::2], offspring[1::2]):
                if random.random() < CXPB:
                    toolbox.mate(child1, child2)
                    del child1.fitness.values
                    del child2.fitness.values

            for mutant in offspring:
                if random.random() < MUTPB:
                    toolbox.mutate(mutant)
                    del mutant.fitness.values

            # The population is entirely replaced by the offspring
            pop[:] = offspring

    # Stop the multiprocessing pool
    pool.close()
    pool.join()

    print("-- End of (successful) evolution --")
    best_ind = tools.selBest(pop, 1)[0]
    print("Best individual is %s, %s" % (best_ind, best_ind.fitness.values))

    traj.f_store()  # And store all the rest of the data

if __name__ == "__main__":
    main()


"""Module containing results and parameters that can be used to store `BRIAN2 data`_.

Parameters handling BRIAN2 data are instantiated by the
:class:`~pypet.brian2.parameter.Brian2Parameter` class for any BRIAN2 Quantity.

The :class:`~pypet.brian2.parameter.Brian2Result` can store BRIAN2 Quantities
and the :class:`~pypet.brian2.parameter.Brian2MonitorResult` extracts data from
BRIAN2 Monitors.

.. _`BRIAN2 data`: http://brian2.readthedocs.org/

"""

__author__ = ['Henri Bunting', 'Robert Meyer']

import ast

import brian2.numpy_ as np
from brian2.units.fundamentalunits import Quantity, get_dimensions
from brian2.monitors import SpikeMonitor, StateMonitor, PopulationRateMonitor
import brian2.units.allunits as allunits

import pypet.pypetexceptions as pex
from pypet.parameter import Parameter, Result, ObjectTable

ALLUNITS = {}
for name in allunits.__all__:
    ALLUNITS[name] =  getattr(allunits, name)

def get_unit_fast(x):
    """ Return a `Quantity` with value 1 and the same dimensions. """
    return Quantity.with_dimensions(1, get_dimensions(x))

def unit_from_expression(expr):
    """Takes a unit string like ``'1. * volt'`` and returns the BRIAN2 unit."""
    if expr == '1':
        return get_unit_fast(1)
    elif isinstance(expr, str):
        mod = ast.parse(expr, mode='eval')
        expr = mod.body
        return unit_from_expression(expr)
    elif expr.__class__ is ast.Name:
        return ALLUNITS[expr.id]
    elif expr.__class__ is ast.Num:
        return expr.n
    elif expr.__class__ is ast.UnaryOp:
        op = expr.op.__class__.__name__
        operand = unit_from_expression(expr.operand)
        if op=='USub':
            return -operand
        else:
            raise SyntaxError("Unsupported operation "+op)
    elif expr.__class__ is ast.BinOp:
        op = expr.op.__class__.__name__
        left = unit_from_expression(expr.left)
        right = unit_from_expression(expr.right)
        if op=='Add':
            u = left+right
        elif op=='Sub':
            u = left-right
        elif op=='Mult':
            u = left*right
        elif op=='Div':
            u = left/right
        elif op=='Pow':
            n = unit_from_expression(expr.right)
            u = left**n
        elif op=='Mod':
            u = left % right
        else:
            raise SyntaxError("Unsupported operation "+op)
        return u
    else:
        raise RuntimeError('You shall not pass')

class Brian2Parameter(Parameter):
    """A Parameter class that supports BRIAN2 Quantities.

    Note that only scalar BRIAN2 quantities are supported, lists, tuples or dictionaries
    of BRIAN2 quantities cannot be handled.

    Supports data for the standard :class:`~pypet.parameter.Parameter`, too.

    """

    IDENTIFIER = '__brn2__'
    ''' Identification string stored into column title of hdf5 table'''

    __slots__ = ()

    def __init__(self, full_name, data=None, comment=''):
        super(Brian2Parameter, self).__init__(full_name, data, comment)

    def f_supports(self, data):
        """ Simply checks if data is supported """
        if isinstance(data, Quantity):
            return True
        elif super(Brian2Parameter, self).f_supports(data):
            return True
        return False

    def _values_of_same_type(self, val1, val2):

        if isinstance(val1, Quantity):
            try:
                if not val1.has_same_dimensions(val2):
                    return False
            except TypeError:
                return False

        elif isinstance(val2, Quantity):
            try:
                if not val2.has_same_dimensions(val1):
                    return False
            except TypeError:
                return False

        elif not super(Brian2Parameter, self)._values_of_same_type(val1, val2):
            return False

        return True

    def _store(self):

        if not isinstance(self._data, Quantity):
            return super(Brian2Parameter, self)._store()
        else:
            store_dict = {}
            unit = get_unit_fast(self._data)

            value = self._data/unit
            store_dict['data' + Brian2Parameter.IDENTIFIER] = ObjectTable(data={'value': [value], 'unit': [repr(unit)]})

            if self.f_has_range():
                value_list = [value_with_unit/unit for value_with_unit in self._explored_range]
                store_dict['explored_data' + Brian2Parameter.IDENTIFIER] = ObjectTable(data={'value': value_list})

            self._locked = True

            return store_dict

    def _load(self, load_dict):
        if self.v_locked:
            raise pex.ParameterLockedException('Parameter `%s` is locked!' % self.v_full_name)

        try:
            data_table = load_dict['data' + Brian2Parameter.IDENTIFIER]

            unitstring = data_table['unit'][0]
            unit = unit_from_expression(unitstring)
            value = data_table['value'][0]
            self._data = value * unit

            if 'explored_data' + Brian2Parameter.IDENTIFIER in load_dict:
                explore_table = load_dict['explored_data' + Brian2Parameter.IDENTIFIER]

                value_col = explore_table['value']
                explore_list = [value * unit for value in value_col]

                self._explored_range = explore_list
                self._explored = True

        except KeyError:
            super(Brian2Parameter, self)._load(load_dict)

        self._default = self._data
        self._locked = True

class Brian2Result(Result):
    """ A result class that can handle BRIAN2 quantities.

    Note that only scalar BRIAN2 quantities are supported, lists, tuples or dictionaries
    of BRIAN2 quantities cannot be handled.

    Supports also all data supported by the standard :class:`~pypet.parameter.Result`.

    """

    IDENTIFIER = Brian2Parameter.IDENTIFIER
    ''' Identifier String to label brian result '''

    __slots__ = ()

    def __init__(self, full_name, *args, **kwargs):
        super(Brian2Result, self).__init__(full_name, *args, **kwargs)

    def f_set_single(self, name, item):
        if Brian2Result.IDENTIFIER in name:
            raise AttributeError('Your result name contains the identifier for brian data,'
                                 ' please do not use %s in your result names.' %
                                 Brian2Result.IDENTIFIER)
        else:
            super(Brian2Result, self).f_set_single(name, item)

    def _supports(self, data):
        """ Simply checks if data is supported """
        if isinstance(data, Quantity):
            return True
        elif super(Brian2Result, self)._supports(data):
            return True
        return False

    def _store(self):

        store_dict = {}

        for key in self._data:
            val = self._data[key]
            if isinstance(val, Quantity):
                unit = get_unit_fast(val)
                value = val/unit
                # Potentially the results are very big in contrast to small parameters
                # Accordingly, an ObjectTable might not be the best choice after all for a result
                if isinstance(val, np.ndarray) and len(val.shape) == 0:
                    # Convert 0-dimensional arrays to regular numpy floats
                    value = np.float(value)
                store_dict[key + Brian2Result.IDENTIFIER + 'value'] = value
                store_dict[key + Brian2Result.IDENTIFIER + 'unit'] = repr(unit)

            else:
                store_dict[key] = val

        return store_dict

    def _load(self, load_dict):

        for key in load_dict:
            if Brian2Result.IDENTIFIER in key:

                new_key = key.split(Brian2Result.IDENTIFIER)[0]

                if new_key in self._data:
                    # We already extracted the unit/value pair
                    continue

                # Recreate the brain units from the vale as float and unit as string:
                unitstring = load_dict[new_key + Brian2Result.IDENTIFIER + 'unit']
                unit = unit_from_expression(unitstring)
                value = load_dict[new_key + Brian2Result.IDENTIFIER +'value']
                self._data[new_key] = value * unit
            else:
                self._data[key] = load_dict[key]

class Brian2MonitorResult(Brian2Result):
    """ A Result class that supports BRIAN2 monitors.

    Monitor attributes are extracted and added as results with the attribute names.
    Note the original monitors are NOT stored, only their attribute/property values are kept.

    Add monitor on `__init__` via `monitor=` or via `f_set(monitor=brian_monitor)`

    **IMPORTANT**: You can only use 1 result per monitor. Otherwise a 'TypeError' is thrown.

    Example:

    >>> brian_result = BrianMonitorResult('example.brian_test_test.mymonitor',
                                            monitor=SpikeMonitor(...),
                                            comment='Im a SpikeMonitor Example!')
    >>> brian_result.num_spikes
    1337

    Following monitors are supported and the following values are extraced:

    * PopulationRateMonitor

        * t

            The times of the bins.

        * rate

            An array of rates in Hz

        * source

            String representation of source

        * name

            The name of the monitor

    * SpikeMonitor

        * t

            The times of the spikes

        * i

            The neuron indices of the spikes

        * num_spikes

            Total number of spikes

        * record_variables

            If variables are recorded at spike time, this is the list of their names

        * "varname"

            Array of variable recorded at spike time

        * source

            String representation of source

        * name

            The name of the monitor

    * StateMonitor

        * t

            Recording times

        * record_variables

            List of recorded variable names

        * "varname"

            Array of variable recorded

        * source

            String representation of source

        * name

            The name of the monitor

    """

    __slots__ = ('_monitor_type',)

    def __init__(self, full_name, *args, **kwargs):
        self._monitor_type = None
        super(Brian2MonitorResult, self).__init__(full_name, *args, **kwargs)

    def _store(self):
        store_dict = super(Brian2MonitorResult, self)._store()

        if self._monitor_type is not None:
            store_dict['monitor_type'] = self._monitor_type

        return store_dict

    def _load(self, load_dict):
        if 'monitor_type' in load_dict:
            self._monitor_type = load_dict.pop('monitor_type')
        super(Brian2MonitorResult, self)._load(load_dict)

    @property
    def v_monitor_type(self):
        """ The type of the stored monitor. Each MonitorResult can only manage a single Monitor.
        """
        return self._monitor_type

    def f_set_single(self, name, item):
        """ To add a monitor use `f_set_single('monitor', brian_monitor)`.

        Otherwise `f_set_single` works similar to :func:`~pypet.parameter.Result.f_set_single`.
        """
        if type(item) in [SpikeMonitor, StateMonitor, PopulationRateMonitor]:
            if self.v_stored:
                self._logger.warning('You are changing an already stored result. If '
                                     'you not explicitly overwrite the data on disk, '
                                     'this change might be lost and not propagated to disk.')

            self._extract_monitor_data(item)
        else:
            super(Brian2MonitorResult, self).f_set_single(name, item)

    def _extract_monitor_data(self, monitor):

        if self._monitor_type is not None:
            raise TypeError('Your result `%s` already extracted data from a `%s` monitor.'
                             ' Please use a new empty result for a new monitor.')

        self._monitor_type = monitor.__class__.__name__

        if isinstance(monitor, SpikeMonitor):
            self._extract_spike_monitor(monitor)

        elif isinstance(monitor, StateMonitor):
            self._extract_state_monitor(monitor)

        elif isinstance(monitor, PopulationRateMonitor):
            self._extract_population_rate_monitor(monitor)

        else:
            raise ValueError('Monitor Type %s is not supported (yet)' % str(type(monitor)))

    def _extract_state_monitor(self, monitor):

        self.f_set(record_variables=monitor.record_variables)
        self.f_set(record=monitor.record)
        self.f_set(when=monitor.when)
        self.f_set(source=str(monitor.source))
        self.f_set(name=monitor.name)

        times=np.array(monitor.t[:])
        if len(times) > 0:
            self.f_set(t=times)

            for varname in monitor.record_variables:
                val = getattr(monitor, varname)
                self.f_set(**{varname: val})

    def _extract_spike_monitor(self, monitor):

        self.f_set(source=str(monitor.source))
        self.f_set(num_spikes=monitor.num_spikes)
        self.f_set(when=monitor.when)
        self.f_set(name=monitor.name)

        if len(monitor.record_variables) > 0:
            self.f_set(record_variables=list(monitor.record_variables))

        times = monitor.t[:]

        if len(times) > 0:
            self.f_set(t=times)
            self.f_set(i=monitor.i[:])

            for varname in monitor.record_variables:
                val = getattr(monitor, varname)
                self.f_set(**{varname: val[:]})

    def _extract_population_rate_monitor(self, monitor):

        times = monitor.t[:]
        self.f_set(source=str(monitor.source))
        self.f_set(name=monitor.name)

        if len(times) > 0:
            self.f_set(t=times)
            self.f_set(rate=monitor.rate[:])


""" Module providing the functionality to allow automatic git commits.

* :func:`~pypet.gitintegration.make_git_commit` performs the git commit

* :func:`~pypet.gitintegration.add_commit_variables` adds some information about the commit to a
  :class:`~pypet.trajectory.Trajectory`.

"""

__author__ = 'Robert Meyer'

import time

try:
    import git
except ImportError:
    git = None

import pypet.pypetexceptions as pex

def add_commit_variables(traj, commit):
    """Adds commit information to the trajectory."""

    git_time_value = time.strftime('%Y_%m_%d_%Hh%Mm%Ss', time.localtime(commit.committed_date))

    git_short_name = str(commit.hexsha[0:7])
    git_commit_name = 'commit_%s_' % git_short_name
    git_commit_name = 'git.' + git_commit_name + git_time_value

    if not traj.f_contains('config.'+git_commit_name, shortcuts=False):

        git_commit_name += '.'
        # Add the hexsha
        traj.f_add_config(git_commit_name+'hexsha', commit.hexsha,
                          comment='SHA-1 hash of commit')

        # Add the description string
        traj.f_add_config(git_commit_name+'name_rev', commit.name_rev,
                          comment='String describing the commits hex sha based on '
                                  'the closest Reference')

        # Add unix epoch
        traj.f_add_config(git_commit_name+'committed_date',
                          commit.committed_date, comment='Date of commit as unix epoch seconds')

        # Add commit message
        traj.f_add_config(git_commit_name+'message', str(commit.message),
                          comment='The commit message')

        # # Add commit author
        # traj.f_add_config(git_commit_name+'committer', str(commit.committer.name),
        #                   comment='The committer of the commit')
        #
        # # Add author's email
        # traj.f_add_config(git_commit_name+'committer_email', str(commit.committer.email),
        #                   comment='Email of committer')

def make_git_commit(environment, git_repository, user_message, git_fail):
    """ Makes a commit and returns if a new commit was triggered and the SHA_1 code of the commit.

    If `git_fail` is `True` program fails instead of triggering a new commit given
    not committed changes. Then a `GitDiffError` is raised.

    """

    # Import GitPython, we do it here to allow also users not having GitPython installed
    # to use the normal environment

    # Open the repository
    repo = git.Repo(git_repository)
    index = repo.index

    traj = environment.v_trajectory

    # Create the commit message and append the trajectory name and comment
    if traj.v_comment:
        commentstr = ', Comment: `%s`' % traj.v_comment
    else:
        commentstr = ''

    if user_message:
        user_message += ' -- '

    message = '%sTrajectory: `%s`, Time: `%s`, %s' % \
              (user_message, traj.v_name, traj.v_time, commentstr)

    # Detect changes:
    diff = index.diff(None)

    if diff:
        if git_fail:
            # User requested fail instead of a new commit
            raise pex.GitDiffError('Found not committed changes!')
        # Make the commit
        repo.git.add('-u')
        commit = index.commit(message)
        new_commit = True

    else:
        # Take old commit
        commit = repo.commit(None)
        new_commit = False

    # Add the commit info to the trajectory
    add_commit_variables(traj, commit)

    return new_commit, commit.hexsha


__author__ = 'Robert Meyer'

import sys
import os
import datetime
import numpy as np
import inspect
import logging
import socket
try:
    import zmq
except ImportError:
    zmq = None

def is_debug():
    """Checks if user is currently debugging.

    Debugging is checked via ``'pydevd' in sys.modules``.

    :return: True of False

    """
    return 'pydevd' in sys.modules

def flatten_dictionary(nested_dict, separator):
    """Flattens a nested dictionary.

    New keys are concatenations of nested keys with the `separator` in between.

    """
    flat_dict = {}
    for key, val in nested_dict.items():
        if isinstance(val, dict):
            new_flat_dict = flatten_dictionary(val, separator)
            for flat_key, inval in new_flat_dict.items():
                new_key = key + separator + flat_key
                flat_dict[new_key] = inval
        else:
            flat_dict[key] = val

    return flat_dict

def nest_dictionary(flat_dict, separator):
    """ Nests a given flat dictionary.

    Nested keys are created by splitting given keys around the `separator`.

    """
    nested_dict = {}
    for key, val in flat_dict.items():
        split_key = key.split(separator)
        act_dict = nested_dict
        final_key = split_key.pop()
        for new_key in split_key:
            if not new_key in act_dict:
                act_dict[new_key] = {}

            act_dict = act_dict[new_key]

        act_dict[final_key] = val
    return nested_dict

class _Progressbar(object):
    """Implements a progress bar.

    This class is supposed to be a singleton. Do not
    import the class itself but use the `progressbar` function from this module.

    """
    def __init__(self):
        self._start_time = None   # Time of start/reset
        self._start_index = None  # Index of start/reset
        self._current_index = np.inf  # Current index
        self._percentage_step = None  # Percentage step for bar update
        self._total = None  # Total steps of the bas (float) not to be mistaken for length
        self._total_minus_one = None  # (int) the above minus 1
        self._length = None  # Length of the percentage bar in `=` signs
        self._norm_factor = None  # Normalization factor
        self._current_interval = None  # The current interval,
        # to check if bar needs to be updated

    def _reset(self, index, total, percentage_step, length):
        """Resets to the progressbar to start a new one"""
        self._start_time = datetime.datetime.now()
        self._start_index = index
        self._current_index = index
        self._percentage_step = percentage_step
        self._total = float(total)
        self._total_minus_one = total - 1
        self._length = length
        self._norm_factor = total * percentage_step / 100.0
        self._current_interval = int((index + 1.0) / self._norm_factor)

    def _get_remaining(self, index):
        """Calculates remaining time as a string"""
        try:
            current_time = datetime.datetime.now()
            time_delta = current_time - self._start_time
            try:
                total_seconds = time_delta.total_seconds()
            except AttributeError:
                # for backwards-compatibility
                # Python 2.6 does not support `total_seconds`
                total_seconds = ((time_delta.microseconds +
                                    (time_delta.seconds +
                                     time_delta.days * 24 * 3600) * 10 ** 6) / 10.0 ** 6)
            remaining_seconds = int((self._total - self._start_index - 1.0) *
                                    total_seconds / float(index - self._start_index) -
                                    total_seconds)
            remaining_delta = datetime.timedelta(seconds=remaining_seconds)
            remaining_str = ', remaining: ' + str(remaining_delta)
        except ZeroDivisionError:
            remaining_str = ''
        return remaining_str

    def __call__(self, index, total, percentage_step=5, logger='print', log_level=logging.INFO,
                 reprint=False, time=True, length=20, fmt_string=None,  reset=False):
        """Plots a progress bar to the given `logger` for large for loops.

        To be used inside a for-loop at the end of the loop.

        :param index: Current index of for-loop
        :param total: Total size of for-loop
        :param percentage_step: Percentage step with which the bar should be updated
        :param logger:

            Logger to write to, if string 'print' is given, the print statement is
            used. Use None if you don't want to print or log the progressbar statement.

        :param log_level: Log level with which to log.
        :param reprint:

            If no new line should be plotted but carriage return (works only for printing)

        :param time: If the remaining time should be calculated and displayed
        :param length: Length of the bar in `=` signs.
        :param fmt_string:

            A string which contains exactly one `%s` in order to incorporate the progressbar.
            If such a string is given, ``fmt_string % progressbar`` is printed/logged.

        :param reset:

            If the progressbar should be restarted. If progressbar is called with a lower
            index than the one before, the progressbar is automatically restarted.

        :return:

            The progressbar string or None if the string has not been updated.

        """
        reset = (reset or
                 index <= self._current_index or
                 total != self._total)
        if reset:
            self._reset(index, total, percentage_step, length)

        statement = None
        indexp1 = index + 1.0
        next_interval = int(indexp1 / self._norm_factor)
        ending = index >= self._total_minus_one

        if next_interval > self._current_interval or ending or reset:
            if time:
                remaining_str = self._get_remaining(index)
            else:
                remaining_str = ''

            if ending:
                statement = '[' + '=' * self._length +']100.0%'
            else:
                bars = int((indexp1 / self._total) * self._length)
                spaces = self._length - bars
                percentage = indexp1 / self._total * 100.0
                if reset:
                    statement = ('[' + '=' * bars +
                                 ' ' * spaces + ']' + ' %4.1f' % percentage + '%')
                else:
                    statement = ('[' + '=' * bars +
                                 ' ' * spaces + ']' + ' %4.1f' % percentage + '%' +
                                 remaining_str)

            if fmt_string:
                statement = fmt_string % statement
            if logger == 'print':
                if reprint:
                    print('\r' + statement, end='', flush=True)
                else:
                    print(statement)
            elif logger is not None:
                if isinstance(logger, str):
                    logger = logging.getLogger(logger)
                logger.log(msg=statement, level=log_level)

        self._current_interval = next_interval
        self._current_index = index

        return statement

_progressbar = _Progressbar()

def progressbar(index, total, percentage_step=10, logger='print', log_level=logging.INFO,
                 reprint=True, time=True, length=20, fmt_string=None, reset=False):
    """Plots a progress bar to the given `logger` for large for loops.

    To be used inside a for-loop at the end of the loop:

    .. code-block:: python

        for irun in range(42):
            my_costly_job() # Your expensive function
            progressbar(index=irun, total=42, reprint=True) # shows a growing progressbar

    There is no initialisation of the progressbar necessary before the for-loop.
    The progressbar will be reset automatically if used in another for-loop.

    :param index: Current index of for-loop
    :param total: Total size of for-loop
    :param percentage_step: Steps with which the bar should be plotted
    :param logger:

        Logger to write to - with level INFO. If string 'print' is given, the print statement is
        used. Use ``None`` if you don't want to print or log the progressbar statement.

    :param log_level: Log level with which to log.
    :param reprint:

        If no new line should be plotted but carriage return (works only for printing)

    :param time: If the remaining time should be estimated and displayed
    :param length: Length of the bar in `=` signs.
    :param fmt_string:

        A string which contains exactly one `%s` in order to incorporate the progressbar.
        If such a string is given, ``fmt_string % progressbar`` is printed/logged.

    :param reset:

        If the progressbar should be restarted. If progressbar is called with a lower
        index than the one before, the progressbar is automatically restarted.

    :return:

        The progressbar string or `None` if the string has not been updated.

    """
    return _progressbar(index=index, total=total, percentage_step=percentage_step,
                        logger=logger, log_level=log_level, reprint=reprint,
                        time=time, length=length, fmt_string=fmt_string, reset=reset)

def _get_argspec(func):
    """Helper function to support both Python versions"""
    if inspect.isclass(func):
        func = func.__init__
    if not inspect.isfunction(func):
        # Init function not existing
        return [], False
    parameters = inspect.signature(func).parameters
    args = []
    uses_starstar = False
    for par in parameters.values():
        if (par.kind == inspect.Parameter.POSITIONAL_OR_KEYWORD or
                    par.kind == inspect.Parameter.KEYWORD_ONLY):
            args.append(par.name)
        elif par.kind == inspect.Parameter.VAR_KEYWORD:
            uses_starstar = True
    return args, uses_starstar

def get_matching_kwargs(func, kwargs):
    """Takes a function and keyword arguments and returns the ones that can be passed."""
    args, uses_startstar = _get_argspec(func)
    if uses_startstar:
        return kwargs.copy()
    else:
        matching_kwargs = dict((k, kwargs[k]) for k in args if k in kwargs)
        return matching_kwargs

def result_sort(result_list, start_index=0):
    """Sorts a list of results in O(n) in place (since every run is unique)

    :param result_list: List of tuples [(run_idx, res), ...]
    :param start_index: Index with which to start, every entry before `start_index` is ignored

    """
    if len(result_list) < 2:
        return result_list
    to_sort = result_list[start_index:]
    minmax = [x[0] for x in to_sort]
    minimum = min(minmax)
    maximum = max(minmax)
    #print minimum, maximum
    sorted_list = [None for _ in range(minimum, maximum + 1)]
    for elem in to_sort:
        key = elem[0] - minimum
        sorted_list[key] = elem
    idx_count = start_index
    for elem in sorted_list:
        if elem is not None:
            result_list[idx_count] = elem
            idx_count += 1
    return result_list

def format_time(timestamp):
    """Formats timestamp to human readable format"""
    format_string = '%Y_%m_%d_%Hh%Mm%Ss'
    formatted_time = datetime.datetime.fromtimestamp(timestamp).strftime(format_string)
    return formatted_time

def convert_ipv6(host):
    if ':' in host:
        #IP 6 address
        host = host.split('%')[0]
        host = '[%s]' % host

    return  host

def is_ipv6(url):
    return '[' in url

def port_to_tcp(port=None):
    """Returns local tcp address for a given `port`, automatic port if `None`"""
    #address = 'tcp://' + socket.gethostbyname(socket.getfqdn())
    domain_name = socket.getfqdn()
    try:
        addr_list = socket.getaddrinfo(domain_name, None)
    except Exception:
        addr_list = socket.getaddrinfo('127.0.0.1', None)
    family, socktype, proto, canonname, sockaddr = addr_list[0]
    host = convert_ipv6(sockaddr[0])
    address =  'tcp://' + host
    if port is None:
        port = ()
    if not isinstance(port, int):
        # determine port automatically
        context = zmq.Context()
        try:
            socket_ = context.socket(zmq.REP)
            socket_.ipv6 = is_ipv6(address)
            port = socket_.bind_to_random_port(address, *port)
        except Exception:
            print('Could not connect to {} using {}'.format(address, addr_list))
            pypet_root_logger = logging.getLogger('pypet')
            pypet_root_logger.exception('Could not connect to {}'.format(address))
            raise
        socket_.close()
        context.term()
    return address + ':' + str(port)

def racedirs(path):
    """Like os.makedirs but takes care about race conditions"""
    if os.path.isfile(path):
        raise IOError('Path `%s` is already a file not a directory')
    while True:
        try:
            if os.path.isdir(path):
                # only break if full path has been created or exists
                break
            os.makedirs(path)
        except EnvironmentError as exc:
            # Part of the directory path already exist
            if exc.errno != 17:
                # This error won't be any good
                raise


""" Module defining annotations.

This module contains two classes:

    1. :class:`~pypet.annotations.Annotations`

        Container class for annotations. There are no restrictions regarding what is considered
        to be an annotation. In principle, this can be any python object. However,
        if using the standard :class:`~pypet.storageservice.HDF5StorageService`,
        these annotations are stored as hdf5 node attributes.
        Accordingly, annotations should be rather small since reading
        and writing these attributes to disk is slow.

    2. :class:`~pypet.annotations.WithAnnotations`

        Abstract class to subclass from. Instances that subclass `WithAnnotations`
        have the `v_annotations` property which is an instance of the `Annotations` class
        to handle annotations. They also acquire some functions to put annotations directly
        into the `v_annotations` object like `f_set_annotations`.

"""

__author__ = 'Robert Meyer'

from pypet.pypetlogging import HasLogger
from pypet.slots import HasSlots

class Annotations(HasSlots):
    """ Simple container class for annotations.

    Every tree node (*leaves* and *group* nodes) can be annotated.
    In case you use the standard :class:`~pypet.storageservice.HDF5StorageService`,
    these annotations are stored in the attributes of the hdf5 nodes in the hdf5 file,
    you might wanna take a look at pytables attributes_.

    Annotations should be small (short strings or basic python data types) since their storage
    and retrieval is quite slow!

    .. _attributes: http://pytables.github.io/usersguide/libref/declarative_classes.html#the-attributeset-class

    """
    __slots__ = ('_dict_',)

    def __init__(self):
        self._dict_ = None

    @property
    def _dict(self):
        if self._dict_ is None:
            self._dict_ = {}
        return self._dict_

    def __iter__(self):
        return self._dict.__iter__()

    def __getitem__(self, item):
        """Equivalent to calling f_get()"""
        return self.f_get(item)

    def __setitem__(self, key, value):
        """Almost equivalent to calling __setattr__.

        Treats integer values as `f_get`.

        """
        self.f_set(**{key: value})

    def __delitem__(self, key):
        self.f_remove(key)

    def f_to_dict(self, copy=True):
        """Returns annotations as dictionary.

        :param copy: Whether to return a shallow copy or the real thing (aka _dict).

        """
        if copy:
            return self._dict.copy()
        else:
            return self._dict

    def f_is_empty(self):
        """Checks if annotations are empty"""
        return len(self._dict) == 0

    def f_empty(self):
        """Removes all annotations from RAM """
        self._dict_ = None

    def __setattr__(self, key, value):
        if key.startswith('_'):
            # We set a private attribute
            super(Annotations, self).__setattr__(key, value)
        else:
            self.f_set_single(key, value)

    def __getattr__(self, item):
        return self.f_get(item)

    def __delattr__(self, item):
        self.f_remove(item)

    def _translate_key(self, key):
        if isinstance(key, int):
            if key == 0:
                key = 'annotation'
            else:
                key = 'annotation_%d' % key
        return key

    def f_get(self, *args):
        """Returns annotations

        If len(args)>1, then returns a list of annotations.

        `f_get(X)` with *X* integer will return the annotation with name `annotation_X`.

        If the annotation contains only a single entry you can call `f_get()` without arguments.
        If you call `f_get()` and the annotation contains more than one element a ValueError is
        thrown.

        """

        if len(args) == 0:
            if len(self._dict) == 1:
                return self._dict[list(self._dict.keys())[0]]
            elif len(self._dict) > 1:
                raise ValueError('Your annotation contains more than one entry: '
                                 '`%s` Please use >>f_get<< with one of these.' %
                                 (str(list(self._dict.keys()))))
            else:
                raise AttributeError('Your annotation is empty, cannot access data.')

        result_list = []
        for name in args:
            name = self._translate_key(name)
            try:
                result_list.append(self._dict[name])
            except KeyError:
                raise AttributeError('Your annotation does not contain %s.' % name)

        if len(args) == 1:
            return result_list[0]
        else:
            return tuple(result_list)

    def f_set(self, *args, **kwargs):
        """Sets annotations

        Items in args are added as `annotation` and `annotation_X` where
        'X' is the position in args for following arguments.

        """
        for idx, arg in enumerate(args):
            valstr = self._translate_key(idx)
            self.f_set_single(valstr, arg)

        for key, arg in kwargs.items():
            self.f_set_single(key, arg)

    def f_remove(self, key):
        """Removes `key` from annotations"""
        key = self._translate_key(key)
        try:
            del self._dict[key]
        except KeyError:
            raise AttributeError('Your annotations do not contain %s' % key)

    def f_set_single(self, name, data):
        """ Sets a single annotation. """
        self._dict[name] = data

    def f_ann_to_str(self):
        """Returns all annotations lexicographically sorted as a concatenated string."""
        resstr = ''
        for key in sorted(self._dict.keys()):
            resstr += '%s=%s; ' % (key, str(self._dict[key]))
        return resstr[:-2]

    def __str__(self):
        return self.f_ann_to_str()

class WithAnnotations(HasLogger):

    __slots__ = ('_annotations',)

    def __init__(self):
        self._annotations = Annotations()  # The annotation object to handle annotations

    @property
    def v_annotations(self):
        """ Annotation feature of a trajectory node.

        Store some short additional information about your nodes here.
        If you use the standard HDF5 storage service, they will be stored as hdf5 node
        attributes_.

        .. _attributes: http://pytables.github.io/usersguide/libref/declarative_classes.html#the-attributeset-class

        """
        return self._annotations

    def f_set_annotations(self, *args, **kwargs):
        """Sets annotations

        Equivalent to calling `v_annotations.f_set(*args,**kwargs)`

        """
        self._annotations.f_set(*args, **kwargs)

    def f_get_annotations(self, *args):
        """Returns annotations

        Equivalent to `v_annotations.f_get(*args)`

        """
        return self._annotations.f_get(*args)

    def f_ann_to_str(self):
        """Returns annotations as string

        Equivalent to `v_annotations.f_ann_to_str()`

        """
        return self._annotations.f_ann_to_str()


"""
New module allowing to share data across processes in a multiprocessing environment.

Already functional but not yet documented or thoroughly tested.

Use at your own risk!

"""

__author__ = 'Robert Meyer'

import warnings

import pandas as pd
import numpy as np

import pypet.pypetconstants as pypetconstants
from pypet.pypetlogging import HasLogger
from pypet.utils.decorators import with_open_store, prefix_naming
from pypet.parameter import ObjectTable, Result
from pypet.naturalnaming import KnowsTrajectory

@prefix_naming
class StorageContextManager(HasLogger):
    """Context manager that can be used to open a storage file with multiprocessing.

    For example:

    .. code-block:: python

        with StorageContextManager(traj) as scm:
            # This is a block with open storage,
            # this block is also multiprocessing safe,
            # i.e. everything happens sequentially in here
            do_stuff_to_open_storage_sequentially()

    """
    def __init__(self, trajectory):
        self._traj = trajectory

    def __enter__(self):
        self.open_store()
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        try:
            self.close_store()
        except Exception as exc:
            self._logger.error('Could not close file because of `%s`' % repr(exc))
            if exc_type is None:
                raise
            else:
                return False

    def close_store(self):
        """Closes store manually not needed if used with `with`"""
        service = self._traj.v_storage_service
        if not service.is_open:
            raise RuntimeError('The storage service is not open, '
                               'please open via `f_open_storage`.')
        service.store(pypetconstants.CLOSE_FILE, None)

    def open_store(self):
        """Opens store manually not needed if used with `with`"""
        service = self._traj.v_storage_service
        if service.is_open:
            raise RuntimeError('Your service is already open, there is no need to re-open it.')
        service.store(pypetconstants.OPEN_FILE, None,
                      trajectory_name=self._traj.v_name)

    def flush_store(self):
        """Flushes data to the storage can be called at any time when storage is open."""
        service = self._traj.v_storage_service
        if not service.is_open:
            raise RuntimeError('The storage service is not open, '
                               'please open via `f_open_storage`.')
        service.store(pypetconstants.FLUSH, None)

def make_ordinary_result(result, key, trajectory=None, reload=True):
    """Turns a given shared data item into a an ordinary one.

    :param result: Result container with shared data
    :param key: The name of the shared data
    :param trajectory:

        The trajectory, only needed if shared data has
        no access to the trajectory, yet.

    :param reload: If data should be reloaded after conversion

    :return: The result

    """
    shared_data = result.f_get(key)
    if trajectory is not None:
        shared_data.traj = trajectory
    shared_data._request_data('make_ordinary')
    result.f_remove(key)
    if reload:
        trajectory.f_load_item(result, load_data=pypetconstants.OVERWRITE_DATA)
    return result

def make_shared_result(result, key, trajectory, new_class=None):
    """Turns an ordinary data item into a shared one.

    Removes the old result from the trajectory and replaces it.
    Empties the given result.

    :param result: The result containing ordinary data
    :param key: Name of ordinary data item
    :param trajectory: Trajectory container
    :param new_class:

        Class of new shared data item.
        Leave `None` for automatic detection.

    :return: The `result`

    """
    data = result.f_get(key)
    if new_class is None:
        if isinstance(data, ObjectTable):
            new_class = SharedTable
        elif isinstance(data, pd.DataFrame):
            new_class = SharedPandasFrame
        elif isinstance(data, (tuple, list)):
            new_class = SharedArray
        elif isinstance(data, (np.ndarray, np.matrix)):
            new_class = SharedCArray
        else:
            raise RuntimeError('Your data `%s` is not understood.' % key)
    shared_data = new_class(result.f_translate_key(key), result, trajectory=trajectory)
    result[key] = shared_data
    shared_data._request_data('make_shared')
    return result

class SharedData(HasLogger):
    """ Piece of data that can be shared among many processes.

    Could be a PyTables table or array or pandas DataFrame.
    The data can be accessed and modified by all processes.

    Can be added as follows:

        >>> res = traj.f_add_result(SharedResult, 'shared_result')
        >>> my_array = SharedArray(name='array', parent=res, trajectory=traj, add_to_parent=True)

    Next, the shared array can be accessed via:

        >>> my_array = traj.shared_result.array

    So far the shared array is empty and we need to create shared data on the disk.
    In order to do this we call

        >>> my_array.create_shared_data(obj=np.zeros((100,100)))

    Thus, within the HDF5 file, there exist now a 100 by 100 array that can be
    accessed and modified by any process.

    For example:

    .. code-block:: python

        with StorageContextManager(traj) as scm:
            my_array = traj.shared_result
            my_array[42, 42] = 1001.array

    Now the value 1001 can be accessed from all processes at
    position ``[42, 42]``.

    :param name:

        Name of shared data, must be the same name as in the parent node.

    :param parent:

        *pypet* result that will contain the shared data item.

    :param trajectory:

        Trajectory container

    :param add_to_parent:

        If shared data should automatically be added to the parent
        with the given `name`.

        >>> my_array = SharedArray(name='array', parent=res, trajectory=traj, add_to_parent=True)

        is equivalent to

        >>> my_array = SharedArray(name='array', parent=res, trajectory=traj, add_to_parent=False)
        >>> res['array'] = my_array

    """

    FLAG = None

    def __init__(self, name=None, parent=None, trajectory=None, add_to_parent=False):
        self._set_logger()
        self.name = name
        self.parent = parent
        self.traj = trajectory
        if add_to_parent:
            self.parent[name] = self

    def _check_state(self):
        if self.traj is None:
            raise TypeError('Please pass the trajectory to the shared data, via '
                            '`shared_data.traj = trajectory`, otherwise you cannot '
                            'access the data.')
        if self.parent is None:
            raise TypeError('Please pass the parent to the shared data, via'
                            '`shared_data.parent = parent`, otherwise you cannot '
                            'access the data.')
        if self.name is None:
            raise TypeError('Please pass the name of the shared data, via'
                            '`shared_data.name = name`, otherwise you cannot '
                            'access the data.')

        if self.name not in self.parent or self.parent.f_get(self.name) is not self:
            raise TypeError('Please make sure that the SharedData is added to the '
                            'parent with the name of the SharedData.')

    def _store_parent(self):
        self._check_state()
        if not self.parent._stored:
            self.traj.f_store_item(self.parent)

    def create_shared_data(self, **kwargs):
        """Creates shared data on disk with a StorageService on disk.

        Needs to be called before shared data can be used later on.

        Actual arguments of ``kwargs`` depend on the type of data to be
        created. For instance, creating an array one can use the keyword
        ``obj`` to pass a numpy array (``obj=np.zeros((10,20,30))``).
        Whereas for a PyTables table may need a description dictionary
        (``description={'column_1': pt.StringCol(2, pos=0),'column_2': pt.FloatCol( pos=1)}``)
        Refer to the PyTables documentation on how to create tables.

        """
        if 'flag' not in kwargs:
            kwargs['flag'] = self.FLAG
        if 'data' in kwargs:
            kwargs['obj'] = kwargs.pop('data')
        if 'trajectory' in kwargs:
            self.traj = kwargs.pop('trajectory')
        if 'traj' in kwargs:
            self.traj = kwargs.pop('traj')
        if 'name' in kwargs:
            self.name = kwargs.pop['name']
        if 'parent' in kwargs:
            self.parent = kwargs.pop('parent')
            if self.name is not None:
                self.parent[self.name] = self
        return self._request_data('create_shared_data', kwargs=kwargs)

    def _request_data(self, request, args=None, kwargs=None):
        """Interface with the underlying storage.

        Passes request to the StorageService that performs the appropriate action.
        For example, given a shared table ``t``.
        ``t.remove_row(4)`` is parsed into ``request='remove_row', args=(4,)`` and
        passed onto the storage service. In case of the HDF5StorageService,
        this is again translated back into ``hdf5_table_node.remove_row(4)``.

        """
        return self._storage_service.store(pypetconstants.ACCESS_DATA,
                                           self.parent.v_full_name,
                                           self.name,
                                           request, args, kwargs,
                                           trajectory_name=self.traj.v_name)

    def get_data_node(self):
        """Returns the actula node of the underlying data.

        In case one uses HDF5 this will be the HDF5 leaf node.

        """
        if not self._storage_service.is_open:
            warnings.warn('You requesting the data item but your store is not open, '
                          'the item itself will be closed, too!',
                          category=RuntimeWarning)
        return self._request_data('__thenode__')

    @property
    def _storage_service(self):
        self._store_parent()
        return self.traj.v_storage_service

class SharedArray(SharedData):

    FLAG = pypetconstants.ARRAY

    @property
    def rowsize(self):
        return self._request_data('rowsize')

    @property
    def nrows(self):
        return self._request_data('nrows')

    @property
    def atom(self):
        return self._request_data('atom')

    def get_enum(self):
        return self._request_data('get_enum', args=())

    def read(self, start=None, stop=None, step=None, out=None):
        kwargs = dict(start=start, stop=stop, step=step, out=out)
        return self._request_data('read', kwargs=kwargs)

    def __iter__(self):
        if self._storage_service.is_open:
            return self.iterrows()
        else:
            return iter(self.read())

    def __getitem__(self, item):
        return self._request_data('__getitem__', args=(item,))

    def __setitem__(self, key, value):
        return self._request_data('__setitem__', args=(key, value))

    def __len__(self):
        return self.nrows

    @with_open_store
    def iterrows(self, start=None, stop=None, step=None):
        return self._request_data('iterrows', kwargs=dict(start=start, stop=stop, step=step))

    @with_open_store
    def next(self):
        return self._request_data('next', args=())

class SharedCArray(SharedArray):

    FLAG = pypetconstants.CARRAY

class SharedEArray(SharedArray):

    FLAG = pypetconstants.EARRAY

    def append(self, sequence):
        return self._request_data('append', args=(sequence,))

class SharedVLArray(SharedEArray):

    FLAG = pypetconstants.VLARRAY

class SharedTable(SharedData):

    FLAG = pypetconstants.TABLE

    @property
    def coldescrs(self):
        return self._request_data('coldescrs')

    @property
    def coldtypes(self):
        return self._request_data('coldtypes')

    @property
    def colindexed(self):
        return self._request_data('colindexed')

    @property
    def colnames(self):
        return self._request_data('colnames')

    @property
    def colpathnames(self):
        return self._request_data('colpathnames')

    @property
    def coltypes(self):
        return self._request_data('coltypes')

    @property
    def description(self):
        return self._request_data('description')

    @property
    def indexed(self):
        return self._request_data('indexed')

    @property
    def extdim(self):
        return self._request_data('extdim')

    @property
    def nrows(self):
        return self._request_data('nrows')

    @property
    def rowsize(self):
        return self._request_data('rowsize')

    @property
    def autoindex(self):
        return self._request_data('autoindex')

    @autoindex.setter
    def autoindex(self, value):
        self._request_data('__setattr__', args=('autoindex', value))

    @property
    def indexedcolpathnames(self):
        return self._request_data('indexedcolpathnames')

    def col(self, name):
        return self._request_data('col', args=(name,))

    def read(self, start=None, stop=None, step=None, field=None, out=None):
        kwargs = dict(start=start, stop=stop, step=step, field=field, out=out)
        return self._request_data('read', kwargs=kwargs)

    def read_coordinates(self, coords, field=None):
        return self._request_data('read_coordinates', args=(coords,), kwargs=dict(filed=field))

    def read_sorted(self, sortby, checkCSI=False, field=None, start=None, stop=None, step=None):
        kwargs = dict(checkCSI=checkCSI, field=field, start=start, stop=stop, step=step)
        return self._request_data('read_sorted', args=(sortby,), kwargs=kwargs)

    def __iter__(self):
        if self._storage_service.is_open:
            return self._request_data('__iter__', args=())
        else:
            return iter(self.read())

    def __getitem__(self, key):
        return self._request_data('__getitem__', args=(key,))

    def __len__(self):
        return self.nrows

    def append(self, rows):
        return self._request_data('append', args=(rows,))

    # def append_row(self, row):
    #     return self.append([row])

    def modify_column(self, start=None, stop=None, step=None, column=None, colname=None):
        kwargs = dict(start=start, stop=stop, step=step, column=column, colname=colname)
        return self._request_data('modify_column', kwargs=kwargs)

    def modify_columns(self, start=None, stop=None, step=None, column=None, colname=None):
        kwargs = dict(start=start, stop=stop, step=step, column=column, colname=colname)
        return self._request_data('modify_columns', kwargs=kwargs)

    def modify_coordinates(self, coords, rows):
        self._request_data('modify_coordinates', args=(coords, rows))

    def modify_rows(self, start=None, stop=None, step=None, rows=None):
        kwargs = dict(start=start, stop=stop, step=step, rows=rows)
        self._request_data('modify_rows', kwargs=kwargs)

    def remove_rows(self, start=None, stop=None, step=None):
        kwargs = dict(start=start, stop=stop, step=step)
        self._request_data('remove_rows', kwargs=kwargs)

    def remove_row(self, n):
        self._request_data('remove_row', args=(n,))

    def __setitem__(self, key, value):
        return self._request_data('__setitem__', args=(key, value))

    def get_where_list(self, condition, condvars=None,
                         sort=False, start=None, stop=None, step=None):
        kwargs = dict(condvars=condvars, sort=sort, start=start, stop=stop, step=step)
        return self._request_data('get_where_list', args=(condition,), kwargs=kwargs)

    def read_where(self, condition, condvars=None, field=None,
                     start=None, stop=None, step=None):
        kwargs = dict(condvars=condvars, field=field, start=start, stop=stop, step=step)
        return self._request_data('read_where', args=(condition,), kwargs=kwargs)

    def will_query_use_indexing(self, condition, condvars=None):
        return self._request_data('will_query_use_indexing', args=(condition,),
                                  kwargs=dict(condvars=condvars))

    def get_enum(self, colname):
        return self._request_data('get_enum', args=(colname,))

    def reindex(self, colname=None):
        return self._request_data('reindex', kwargs=dict(colname=colname,
                                                         _col_func=colname is not None))

    def reindex_dirty(self, colname=None):
        return self._request_data('reindex_dirty', kwargs=dict(colname=colname,
                                                                _col_func=colname is not None))

    def remove_index(self, colname):
        return self._request_data('remove_index', args=(colname,),
                                  kwargs=dict(_col_func=True))

    def create_index(self, colname, optlevel=6,
                       kind='medium', filters=None,
                       tmp_dir=None, _blocksizes=None,
                       _testmode=False, _verbose=False):
        kwargs = dict(optlevel=optlevel, kind=kind, filters=filters, tmp_dir=tmp_dir,
                      _blocksizes=_blocksizes, _testmode=_testmode, _verbose=_verbose,
                      _col_func=True, colname=colname)
        return self._request_data('create_index', kwargs=kwargs)

    def create_csindex(self, colname, filters=None, tmp_dir=None,
                         _blocksizes=None, _testmode=False, _verbose=False):
        kwargs = dict(filters=filters, tmp_dir=tmp_dir, _blocksizes=_blocksizes,
                      _testmode=_testmode, _verbose=_verbose,
                      _col_func=True, colname=colname)

        return self._request_data('create_csindex', kwargs=kwargs)

    @property
    @with_open_store
    def colindexes(self):
        return self._request_data('colindexes')

    @property
    @with_open_store
    def cols(self):
        return self._request_data('cols')

    @property
    @with_open_store
    def row(self):
        return self._request_data('row')

    @with_open_store
    def iterrows(self, start=None, stop=None, step=None):
        return self._request_data('iterrows', kwargs=dict(start=start, stop=stop, step=step))

    @with_open_store
    def itersequence(self, sequence):
        return self._request_data('itersequence', args=(sequence,))

    @with_open_store
    def itersorted(self, sortby, checkCSI=False, start=None, stop=None, step=None):
        kwargs = dict(checkCSI=checkCSI, start=start, stop=stop, step=step)
        return self._request_data('itersorted', args=(sortby,), kwargs=kwargs)

    @with_open_store
    def where(self, condition, condvars=None, start=None, stop=None, step=None):
        kwargs = dict(condvars=condvars, start=start, stop=stop, step=step)
        return self._request_data('where', args=(condition,), kwargs=kwargs)

    @with_open_store
    def append_where(self, dstTable, condition, condvars=None, start=None, stop=None, step=None):
        kwargs = dict(condvars=condvars, start=start, stop=stop, step=step)
        return self._request_data('append_where', args=(dstTable, condition), kwargs=kwargs)

    @with_open_store
    def flush(self):
        return self._request_data('flush', args=())

    @with_open_store
    def flush_rows_to_index(self, _lastrow=True):
        return self._request_data('flush_rows_to_index', kwargs=dict(_lastrow=_lastrow))

class SharedPandasFrame(SharedData):

    FLAG = pypetconstants.FRAME

    def create_shared_data(self, **kwargs):
        if 'format' not in kwargs:
            kwargs['format'] = 'table'
        return super(SharedPandasFrame, self).create_shared_data(**kwargs)

    def put(self, data=None, format='table', append=False, **kwargs):
        # data = self._extract_data(data)
        kwargs['format'] = format
        kwargs['append'] = append
        kwargs['obj'] = data
        return self._request_data('pandas_put', kwargs=kwargs)

    def append(self, data=None, **kwargs):
        # data = self._extract_data(data)
        return self.put(data, format='table', append=True, **kwargs)

    def select(self, where=None, start = None, stop=None, columns=None,
               chunksize=None, **kwargs):
        kwargs['where'] = where,
        kwargs['start'] = start
        kwargs['stop'] = stop
        kwargs['columns'] = columns
        kwargs['chunksize'] = chunksize
        if not self._storage_service.is_open:
            kwargs['iterator'] = False
        return self._request_data('pandas_select', kwargs=kwargs)
        # return self.data

    def read(self):
        return self._request_data('pandas_get', args=())
        # return self.data

FLAG_CLASS_MAPPING = {
    pypetconstants.ARRAY: SharedArray,
    pypetconstants.CARRAY: SharedCArray,
    pypetconstants.EARRAY: SharedEArray,
    pypetconstants.VLARRAY: SharedVLArray,
    pypetconstants.TABLE: SharedTable,
    pypetconstants.FRAME: SharedPandasFrame,
}

class SharedResult(Result, KnowsTrajectory):
    """Behaves exactly like the normal `Result` but accepts `SharedData` subclasses as data."""

    __slots__ = ('_traj',)

    SUPPORTED_DATA = set(FLAG_CLASS_MAPPING.values())

    def __init__(self, full_name, trajectory, *args, **kwargs):
        self._traj = trajectory
        super(SharedResult, self).__init__(full_name, *args, **kwargs)

    @property
    def v_traj(self):
        return self._traj

    def _supports(self, item):
        """Checks if outer data structure is supported."""
        result = super(SharedResult, self)._supports(item)
        result = result or type(item) in SharedResult.SUPPORTED_DATA
        return result

    def _pass_to_shared(self, name, item):
        try:
            item.traj = self.v_traj
            item.name = name
            item.parent = self
        except AttributeError:
            pass

    def f_set_single(self, name, item):
        super(SharedResult, self).f_set_single(name, item)
        self._pass_to_shared(name, item)

    def _load(self, load_dict):
        super(SharedResult, self)._load(load_dict)
        for key in self:
            item = self[key]
            self._pass_to_shared(key, item)

    def create_shared_data(self, name=None, **kwargs):
        """Calls the corresponding function of the shared data item"""
        if name is None:
            item = self.f_get()
        else:
            item = self.f_get(name)
        return item.create_shared_data(**kwargs)


__author__ = 'Robert Meyer'

import os
import multiprocessing as mp
import logging

from pypet import Trajectory, MultiprocContext

def manipulate_multiproc_safe(traj):
    """ Target function that manipulates the trajectory.

    Stores the current name of the process into the trajectory and
    **overwrites** previous settings.

    :param traj:

        Trajectory container with multiprocessing safe storage service

    """

    # Manipulate the data in the trajectory
    traj.last_process_name = mp.current_process().name
    # Store the manipulated data
    traj.results.f_store(store_data=3)  # Overwrites data on disk
    # Not recommended, here only for demonstration purposes :-)

def main():
    # We don't use an environment so we enable logging manually
    logging.basicConfig(level=logging.INFO)

    filename = os.path.join('hdf5','example_16.hdf5')
    traj = Trajectory(filename=filename, overwrite_file=True)

    # The result that will be manipulated
    traj.f_add_result('last_process_name', 'N/A',
                      comment='Name of the last process that manipulated the trajectory')

    with MultiprocContext(trajectory=traj, wrap_mode='LOCK') as mc:
        # The multiprocessing context manager wraps the storage service of the trajectory
        # and passes the wrapped service to the trajectory.
        # Also restores the original storage service in the end.
        # Moreover, wee need to use the `MANAGER_LOCK` wrapping because the locks
        # are pickled and send to the pool for all function executions

        # Start a pool of processes manipulating the trajectory
        iterable = (traj for x in range(50))
        pool = mp.Pool(processes=4)
        # Pass the trajectory and the function to the pool and execute it 20 times
        pool.map_async(manipulate_multiproc_safe, iterable)
        pool.close()
        # Wait for all processes to join
        pool.join()

    # Reload the data from disk and overwrite the existing result in RAM
    traj.results.f_load(load_data=3)
    # Print the name of the last process the trajectory was manipulated by
    print('The last process to manipulate the trajectory was: `%s`' % traj.last_process_name)

if __name__ == '__main__':
    main()


__author__ = 'Robert Meyer'

import multiprocessing as mp
import numpy as np
import os # For path names working under Windows and Linux

from pypet import Environment, cartesian_product

def multiply(traj, result_list):
    """Example of a sophisticated simulation that involves multiplying two values.

    This time we will store tha value in a shared list and only in the end add the result.

    :param traj:

        Trajectory containing
        the parameters in a particular combination,
        it also serves as a container for results.

    """
    z=traj.x*traj.y
    result_list[traj.v_idx] = z

def main():
    # Create an environment that handles running
    filename = os.path.join('hdf5', 'example_12.hdf5')
    env = Environment(trajectory='Multiplication',
                      filename=filename,
                      file_title='Example_12_Sharing_Data',
                      overwrite_file=True,
                      comment='The first example!',
                      continuable=False, # We have shared data in terms of a multiprocessing list,
                      # so we CANNOT use the continue feature.
                      multiproc=True,
                      ncores=2)

    # The environment has created a trajectory container for us
    traj = env.trajectory

    # Add both parameters
    traj.f_add_parameter('x', 1, comment='I am the first dimension!')
    traj.f_add_parameter('y', 1, comment='I am the second dimension!')

    # Explore the parameters with a cartesian product
    traj.f_explore(cartesian_product({'x':[1,2,3,4], 'y':[6,7,8]}))

    # We want a shared list where we can put all out results in. We use a manager for this:
    result_list = mp.Manager().list()
    # Let's make some space for potential results
    result_list[:] =[0 for _dummy in range(len(traj))]

    # Run the simulation
    env.run(multiply, result_list)

    # Now we want to store the final list as numpy array
    traj.f_add_result('z', np.array(result_list))

    # Finally let's print the result to see that it worked
    print(traj.z)

    #Disable logging and close all log-files
    env.disable_logging()

if __name__ == '__main__':
    main()