# -*- coding: utf-8 -*-
from __future__ import unicode_literals
import six
import random
import string
import sys
import uuid
import re
import copy
import datetime
from collections import OrderedDict
from six.moves import urllib_parse as urlparse, range
from IPy import IP, MAX_IPV4_ADDRESS, MAX_IPV6_ADDRESS
from . import exceptions
from .utils import force_text
from .translation import gettext as _

__all__ = [
    # Don't need to add field to here by hand,
    # BaseFieldMetaClass will auto add field to here.
]

FIELDS_NAME_MAP = {
    # Don't need to add field to here by hand,
    # BaseFieldMetaClass will auto add field to here.
}

def create_field(field_info):
    """
    Create a field by field info dict.
    """
    field_type = field_info.get('type')
    if field_type not in FIELDS_NAME_MAP:
        raise ValueError(_('not support this field: {}').format(field_type))
    field_class = FIELDS_NAME_MAP.get(field_type)
    params = dict(field_info)
    params.pop('type')
    return field_class.from_dict(params)

class EmptyValue(object):
    """
    a data type replace None
    """

    def __init__(self):
        pass

    def __str__(self):
        return '__empty_value__'

    def __repr__(self):
        return '<{}>'.format(self.__class__.__name__)

EMPTY_VALUE = EmptyValue()

class BaseFieldMetaClass(type):

    def __new__(cls, name, bases, attrs):
        __all__.append(name)
        clazz = super(BaseFieldMetaClass, cls).__new__(cls, name, bases, attrs)
        field_name = attrs.get('FIELD_TYPE_NAME')
        if field_name is not None and field_name != 'object':
            FIELDS_NAME_MAP[field_name] = clazz
        return clazz

@six.add_metaclass(BaseFieldMetaClass)
class BaseField(object):
    """
    BaseField
    """

    """
    INTERNAL_TYPE is the type of the field in python internal, like str, int, list, dict
    INTERNAL_TYPE can be a type list, such as [int, long]
    INTERNAL_TYPE used to validate field's type by isinstance(value, INTERNAL_TYPE)
    """
    INTERNAL_TYPE = object

    FIELD_TYPE_NAME = 'object'

    PARAMS = [
        'strict', 'default', 'validators', 'required'
    ]

    def __init__(self, strict=True, default=EMPTY_VALUE, validators=None, required=False, **kwargs):
        """
        :param strict: bool, if strict is True, value must be an instance of INTERVAL_TYPE,
                        otherwise, value should be convert to INTERNAL_TYPE
        :param default: default value, defaults to EMPTY_VALUE
        :param validators: a validator list, validator can be function, other callable object or object that have method named validate
        :param required: bool, indicate that this field is whether required
        """
        self.strict = strict
        self.default = default

        if validators is None:
            validators = []
        elif not isinstance(validators, (tuple, list)):
            validators = [validators]
        self.validators = validators

        self.required = required

    def __str__(self):
        return self.__class__.__name__

    @classmethod
    def _check_value_range(cls, min_value, max_value):
        if max_value is not None and max_value < min_value:
            raise ValueError(_('the max value must greater than or equals the min value, got min value={min}, max value={max}').format(
                min=min_value, max=max_value))

    def _convert_type(self, value):
        if isinstance(self.INTERNAL_TYPE, (tuple, list)):
            for t in self.INTERNAL_TYPE:
                try:
                    value = t(value)
                    break
                except TypeError as e:
                    pass
            else:
                raise ValueError()
        else:
            value = self.INTERNAL_TYPE(value)
        return value

    @classmethod
    def _get_all_params(cls):
        """
        Collect all PARAMS from this class and its parent class.
        """
        params = list(cls.PARAMS)
        bases = cls.__bases__
        for base in bases:
            if issubclass(base, BaseField):
                params.extend(base._get_all_params())
        return params

    def validate(self, value):
        """
        return validated value or raise FieldValidationError.
        """
        value = self._validate(value)
        for v in self.validators:
            v(value)
        return value

    def _validate(self, value):
        """
        return validated value or raise FieldValidationError.
        sub-class should override this method.
        """
        return self._validate_type(value)

    def _validate_type(self, value):
        """
        validate the type of value
        """
        if not isinstance(value, self.INTERNAL_TYPE):
            if self.strict:
                raise exceptions.FieldValidationError(
                    _('got a wrong type: {0}, expect {1}').format(type(value).__name__, self.FIELD_TYPE_NAME))
            else:
                try:
                    value = self._convert_type(value)
                except (ValueError, TypeError) as e:
                    raise exceptions.FieldValidationError(
                        _('type convertion({0} -> {1}) is failed: {2}').format(type(value).__name__, self.FIELD_TYPE_NAME, str(e)))
        return value

    def is_required(self):
        return self.required

    def get_default(self):
        """
        return default value
        """
        if callable(self.default):
            return self.default()
        else:
            return self.default

    def to_presentation(self, value):
        """
        value: must be a internal value
        """
        return value

    def to_internal(self, value):
        """
        value: must be a validated value
        """
        return value

    def to_dict(self):
        """
        to dict presentation
        """
        d = {
            'type': self.FIELD_TYPE_NAME,
        }
        params = self._get_all_params()
        for name in params:
            if hasattr(self, name):
                value = getattr(self, name)
                # 处理特殊值
                if value is EMPTY_VALUE:
                    value = '__empty__'
                d[name] = value
        return d

    @classmethod
    def from_dict(cls, params):
        """
        Create a field from params.
        sub-class can override this method.
        """
        if params.get('default') == '__empty__':
            params['default'] = EMPTY_VALUE
        return cls(**params)

    def mock_data(self):
        """
        reutrn mocking data
        sub-class should override this method
        """
        return 'this field doesnt implement mock_data method'

class StringField(BaseField):
    """
    StringField
    internal: six.string_types
    presentation: string
    """
    if six.PY2:
        INTERNAL_TYPE = (unicode, str)
    else:
        INTERNAL_TYPE = str
    FIELD_TYPE_NAME = 'string'
    PARAMS = ['min_length', 'max_length', 'regex']

    def __init__(self, min_length=0, max_length=None, regex=None, **kwargs):
        if min_length < 0:
            min_length = 0
        self._check_value_range(min_length, max_length)
        self.min_length = min_length
        self.max_length = max_length

        if isinstance(regex, six.string_types):
            regex = re.compile(regex)
        self.regex = regex

        super(StringField, self).__init__(**kwargs)

    def _validate(self, value):
        value = self._validate_type(value)

        if len(value) < self.min_length:
            raise exceptions.FieldValidationError(
                _('string is too short, min-lenght is {}').format(self.min_length))
        if self.max_length and len(value) > self.max_length:
            raise exceptions.FieldValidationError(
                _('string is too long, max-lenght is {}').format(self.max_length))

        if not self._match(value):
            raise exceptions.FieldValidationError(
                _('{0} not match {1}').format(self.regex.pattern, value))

        return value

    def _match(self, value):
        if self.regex is None:
            return True
        else:
            return self.regex.match(value) is not None

    def to_internal(self, value):
        return six.text_type(value)

    def mock_data(self):
        min_ = self.min_length
        max_ = self.max_length
        if max_ is None:
            max_ = min_ + 100
        size = random.randint(min_, max_)
        random_str = ''.join(
            [random.choice(string.ascii_letters + string.digits) for _ in range(size)])
        random_str = self.to_internal(random_str)
        return random_str

class NumberField(BaseField):
    if six.PY2:
        INTERNAL_TYPE = (int, long, float)
    else:
        INTERNAL_TYPE = (int, float)
    FIELD_TYPE_NAME = 'number'
    PARAMS = ['min_value', 'max_value']

    def __init__(self, min_value=None, max_value=None, **kwargs):
        self._check_value_range(min_value, max_value)
        self.min_value = min_value
        self.max_value = max_value

        super(NumberField, self).__init__(**kwargs)

    def _validate(self, value):
        value = self._validate_type(value)

        if self.min_value is not None and value < self.min_value:
            raise exceptions.FieldValidationError(
                _('value is too small, min-value is {}').format(self.min_value))

        if self.max_value is not None and value > self.max_value:
            raise exceptions.FieldValidationError(
                _('value is too big, max-value is {}').format(self.max_value))

        return value

    def mock_data(self):
        min_ = self.min_value
        if min_ is None:
            min_ = 0
        max_ = self.max_value
        if max_ is None:
            max_ = min_ + 1000
        return random.uniform(min_, max_)

class IntegerField(NumberField):
    INTERNAL_TYPE = int
    FIELD_TYPE_NAME = 'integer'
    PARAMS = []

    def mock_data(self):
        d = super(IntegerField, self).mock_data()
        return int(d)

class FloatField(NumberField):
    INTERNAL_TYPE = float
    FIELD_TYPE_NAME = 'float'
    PARAMS = []

class BoolField(BaseField):
    INTERNAL_TYPE = bool
    FIELD_TYPE_NAME = 'bool'
    PARAMS = []

    def mock_data(self):
        return random.choice([True, False])

class UUIDField(BaseField):
    INTERNAL_TYPE = uuid.UUID
    FIELD_TYPE_NAME = 'UUID'
    PARAMS = ['format']
    SUPPORT_FORMATS = {
        'hex': 'hex',
        'str': '__str__',
        'int': 'int',
        'bytes': 'bytes',
        'bytes_le': 'bytes_le'
    }

    def __init__(self, format='hex', **kwargs):
        """
        format: what format used when to_presentation, supports 'hex', 'str', 'int', 'bytes', 'bytes_le'
        """
        if format not in self.SUPPORT_FORMATS:
            raise ValueError(_('not supports format: {}').format(format))
        self.format = format

        kwargs.setdefault('strict', False)
        super(UUIDField, self).__init__(**kwargs)

    def _validate(self, value):
        value = self._validate_type(value)
        return value

    def to_presentation(self, value):
        assert isinstance(value, self.INTERNAL_TYPE)
        attr = getattr(value, self.SUPPORT_FORMATS[self.format])
        if callable(attr):
            return attr()
        return attr

    def mock_data(self):
        return uuid.uuid4()

class MD5Field(StringField):
    FIELD_TYPE_NAME = 'md5'
    PARAMS = []
    REGEX = r'[\da-fA-F]{32}'

    def __init__(self, **kwargs):
        kwargs['strict'] = True
        super(MD5Field, self).__init__(min_length=32,
                                       max_length=32,
                                       regex=self.REGEX,
                                       **kwargs)

    def _validate(self, value):
        try:
            return super(MD5Field, self)._validate(value)
        except exceptions.FieldValidationError as e:
            raise exceptions.FieldValidationError(
                _('Got wrong md5 value: {}').format(value))

    def mock_data(self):
        return ''.join([random.choice(string.hexdigits) for i in range(32)])

class SHAField(StringField):
    FIELD_TYPE_NAME = 'sha'
    SUPPORT_VERSION = [1, 224, 256, 384, 512]
    PARAMS = ['version']

    def __init__(self, version=256, **kwargs):
        if version not in self.SUPPORT_VERSION:
            raise ValueError(_('{0} not support, support versions are: {1}').format(
                version, self.SUPPORT_VERSION))
        if version == 1:
            length = 40
        else:
            length = int(version / 8 * 2)
        self.version = version
        self.length = length
        kwargs['strict'] = True
        super(SHAField, self).__init__(min_length=length,
                                       max_length=length,
                                       regex=r'[\da-fA-F]{' +
                                       str(length) + '}',
                                       **kwargs)

    def _validate(self, value):
        try:
            return super(SHAField, self)._validate(value)
        except exceptions.FieldValidationError as e:
            raise exceptions.FieldValidationError(
                _('Got wrong sha{0} value: {1}').format(self.version, value))

    def mock_data(self):
        return ''.join([random.choice(string.hexdigits) for i in range(self.length)])

class EmailField(StringField):
    FIELD_TYPE_NAME = 'email'
    REGEX = r'^[a-zA-Z0-9.!#$%&\'*+/=?^_`{|}~-]+@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*$'
    PARAMS = []

    def __init__(self, **kwargs):
        kwargs['strict'] = True
        super(EmailField, self).__init__(regex=self.REGEX, **kwargs)

    def _validate(self, value):
        try:
            return super(EmailField, self)._validate(value)
        except exceptions.FieldValidationError as e:
            raise exceptions.FieldValidationError(
                _('Got wrong email value: {}').format(value))

    def mock_data(self):
        name = ''.join(random.sample(string.ascii_lowercase, 5))
        domain = '{0}.com'.format(
            ''.join(random.sample(string.ascii_lowercase, 3)))
        return '{0}@{1}'.format(name, domain)

class IPAddressField(BaseField):
    INTERNAL_TYPE = IP
    FIELD_TYPE_NAME = 'ip_address'
    PARAMS = ['version']
    SUPPORT_VERSIONS = ['ipv4', 'ipv6', 'both']

    def __init__(self, version='both', **kwargs):
        if version not in self.SUPPORT_VERSIONS:
            raise ValueError(_('{} version is not supported').format(version))
        self.version = version

        kwargs.setdefault('strict', False)
        super(IPAddressField, self).__init__(**kwargs)

    def _validate(self, value):
        try:
            value = IP(value)
        except ValueError as e:
            raise exceptions.FieldValidationError(str(e))
        if self.version == 'ipv4' and value.version() != 4:
            raise exceptions.FieldValidationError(
                _('expected an ipv4 address, got {}').format(value.strNormal()))
        if self.version == 'ipv6' and value.version() != 6:
            raise exceptions.FieldValidationError(
                -('expected an ipv6 address, got {}').format(value.strNormal()))
        return value

    def to_presentation(self, value):
        return value.strNormal()

    def mock_data(self):
        v = self.version
        if v == 'both':
            v = random.choice(['ipv4', 'ipv6'])

        if v == 'ipv4':
            ip = random.randint(0, MAX_IPV4_ADDRESS)
            return IP(ip)
        else:
            ip = random.randint(0, MAX_IPV6_ADDRESS)
            return IP(ip)

class URLField(StringField):
    FIELD_TYPE_NAME = 'url'
    PARAMS = []
    SCHEMAS = ('http', 'https')

    def __init__(self, **kwargs):
        kwargs['strict'] = True
        super(URLField, self).__init__(min_length=0, **kwargs)

    def _validate(self, value):
        value = self._validate_type(value)
        url = urlparse.urlparse(value)
        if url.scheme not in self.SCHEMAS:
            raise exceptions.FieldValidationError(_('schema is lost'))
        if url.hostname == '':
            raise exceptions.FieldValidationError(_('hostname is lost'))
        return url.geturl()

    def mock_data(self):
        return 'http://www.example.com/media/image/demo.jpg'

class EnumField(BaseField):
    INTERNAL_TYPE = object
    FIELD_TYPE_NAME = 'enum'
    PARAMS = ['choices']

    def __init__(self, choices=None, **kwargs):
        if choices is None or len(choices) == 0:
            raise ValueError('choices cant be empty or None')
        self.choices = choices

        super(EnumField, self).__init__(**kwargs)

    def _validate(self, value):
        if value not in self.choices:
            raise exceptions.FieldValidationError(
                _('{!r} not in the choices').format(value))
        return value

    def mock_data(self):
        return random.choice(self.choices)

class DictField(BaseField):
    INTERNAL_TYPE = dict
    FIELD_TYPE_NAME = 'dict'
    PARAMS = ['validator']

    def __init__(self, validator=None, **kwargs):
        """
        :param validator: Validator object
        """
        self.validator = validator
        super(DictField, self).__init__(**kwargs)

    def _validate(self, value):
        value = self._validate_type(value)

        if self.validator:
            v = self.validator(value)
            if v.is_valid():
                value = v.validated_data
            else:
                raise exceptions.FieldValidationError(v.errors)
        else:
            value = copy.deepcopy(value)
        return value

    def to_dict(self):
        d = super(DictField, self).to_dict()
        if d['validator'] is not None:
            d['validator'] = d['validator'].to_dict()
        return d

    def mock_data(self):
        if self.validator:
            return self.validator.mock_data()
        else:
            return {}

class ListField(BaseField):
    INTERNAL_TYPE = (list, tuple)
    FIELD_TYPE_NAME = 'list'
    PARAMS = ['field', 'min_length', 'max_length']

    def __init__(self, field=None, min_length=0, max_length=None, **kwargs):
        if field is not None and not isinstance(field, BaseField):
            raise ValueError(
                _('field param expect a instance of BaseField, but got {!r}').format(field))
        self.field = field

        self._check_value_range(min_length, max_length)
        self.min_length = min_length
        self.max_length = max_length

        super(ListField, self).__init__(**kwargs)

    def _validate(self, value):
        value = self._validate_type(value)
        if self.min_length is not None and len(value) < self.min_length:
            raise exceptions.FieldValidationError(
                _('this list has too few elements, min length is {}').format(self.min_length))

        if self.max_length is not None and len(value) > self.max_length:
            raise exceptions.FieldValidationError(
                _('this list has too many elements, max length is {}').format(self.max_length))

        if self.field:
            new_value = []
            for item in value:
                new_item = self.field.validate(item)
                new_value.append(new_item)
            value = new_value
        else:
            value = copy.deepcopy(value)
        return value

    def to_dict(self):
        d = super(ListField, self).to_dict()
        if d['field'] is not None:
            d['field'] = d['field'].to_dict()
        return d

    @classmethod
    def from_dict(cls, params):
        if 'field' in params and isinstance(params['field'], dict):
            params['field'] = create_field(params['field'])
        return super(ListField, cls).from_dict(params)

    def mock_data(self):
        min_ = self.min_length
        if min_ is None:
            min_ = 0
        max_ = self.max_length
        if max_ is None:
            max_ = 10
        length = random.choice(range(min_, max_))

        data = [None] * length
        if self.field:
            for i in range(length):
                data[i] = self.field.mock_data()
        return data

class TimestampField(IntegerField):
    FIELD_TYPE_NAME = 'timestamp'
    PARAMS = []

    def __init__(self, **kwargs):
        super(TimestampField, self).__init__(
            min_value=0, max_value=2 ** 32 - 1, **kwargs)

    def _validate(self, value):
        try:
            return super(TimestampField, self)._validate(value)
        except exceptions.FieldValidationError as e:
            raise exceptions.FieldValidationError(
                _('Got wrong timestamp: {}').format(value))

class DatetimeField(BaseField):
    INTERNAL_TYPE = datetime.datetime
    FIELD_TYPE_NAME = 'datetime'
    PARAMS = ['dt_format', 'tzinfo']
    DEFAULT_FORMAT = '%Y/%m/%d %H:%M:%S'

    def __init__(self, dt_format=None, tzinfo=None, **kwargs):
        if dt_format is None:
            dt_format = self.DEFAULT_FORMAT
        self.dt_format = dt_format
        if isinstance(tzinfo, six.string_types):
            try:
                import pytz
            except ImportError as e:
                raise ValueError(
                    _('Cant create DatetimeField instance with tzinfo {}, please install pytz and try again').format(params['tzinfo']))
            tzinfo = pytz.timezone(tzinfo)
        self.tzinfo = tzinfo
        kwargs.setdefault('strict', False)
        super(DatetimeField, self).__init__(**kwargs)

    def _convert_type(self, value):
        # override
        if isinstance(value, six.string_types):
            if value.isdigit():
                value = int(value)
                return self.INTERNAL_TYPE.fromtimestamp(value, tz=self.tzinfo)
            else:
                dt = self.INTERNAL_TYPE.strptime(value, self.dt_format)
                if self.tzinfo:
                    dt = dt.replace(tzinfo=self.tzinfo)
                return dt
        elif isinstance(value, six.integer_types):
            return self.INTERNAL_TYPE.fromtimestamp(value, tz=self.tzinfo)
        else:
            raise ValueError(_('Got wrong datetime value: {}').format(value))

    def _validate(self, value):
        value = self._validate_type(value)
        return copy.copy(value)

    def to_presentation(self, value):
        return value.strftime(self.dt_format)

    def to_dict(self):
        d = super(DatetimeField, self).to_dict()
        if d['tzinfo'] is not None:
            d['tzinfo'] = force_text(d['tzinfo'])
        return d

    def mock_data(self):
        return self.INTERNAL_TYPE.fromtimestamp(random.randint(0, 2 ** 32 - 1))

class DateField(BaseField):
    INTERNAL_TYPE = datetime.date
    FIELD_TYPE_NAME = 'date'
    PARAMS = ['dt_format']
    DEFAULT_FORMAT = '%Y/%m/%d'

    def __init__(self, dt_format=None, **kwargs):
        if dt_format is None:
            dt_format = self.DEFAULT_FORMAT
        self.dt_format = dt_format
        kwargs.setdefault('strict', False)
        super(DateField, self).__init__(**kwargs)

    def _convert_type(self, value):
        # override
        if isinstance(value, six.string_types):
            if value.isdigit():
                value = int(value)
                return self.INTERNAL_TYPE.fromtimestamp(value)
            else:
                dt = datetime.datetime.strptime(value, self.dt_format)
                return dt.date()
        elif isinstance(value, six.integer_types):
            return self.INTERNAL_TYPE.fromtimestamp(value)
        else:
            raise ValueError()

    def _validate(self, value):
        value = self._validate_type(value)
        return copy.copy(value)

    def to_presentation(self, value):
        return value.strftime(self.dt_format)

    def mock_data(self):
        return self.INTERNAL_TYPE.fromtimestamp(random.randint(0, 2 ** 32 - 1))


# -*- coding: utf-8 -*-
from __future__ import unicode_literals
import six
from . import exceptions
from .fields import BaseField, EMPTY_VALUE, create_field, DictField
from .utils import force_str

class ValidatorMetaClass(type):

    def __new__(cls, cls_name, bases, attrs):
        fields_map = dict()
        parent_fields_map = dict()
        for parent in bases:
            if hasattr(parent, '_FIELDS_MAP'):
                parent_fields_map.update(parent._FIELDS_MAP)

        for name, value in six.iteritems(attrs):
            if isinstance(value, BaseField):
                fields_map[name] = value

        for name in fields_map:
            attrs.pop(name, None)

        parent_fields_map.update(fields_map)

        attrs['_FIELDS_MAP'] = parent_fields_map

        return super(ValidatorMetaClass, cls).__new__(cls, cls_name, bases, attrs)

@six.add_metaclass(ValidatorMetaClass)
class Validator(object):
    """ a data validator like Django ORM
    """

    def __init__(self, raw_data):
        """
        :param raw_data: unvalidate data
        """
        assert isinstance(raw_data, dict), '"raw_data" must be a dict, not "{}"'.format(type(raw_data).__name__)
        self.raw_data = raw_data
        self.validated_data = None
        self.errors = {}

    def _validate(self):
        data = {}
        for name, field in six.iteritems(self._FIELDS_MAP):
            value = self.raw_data.get(name, field.get_default())
            if value is EMPTY_VALUE:
                if field.is_required():
                    self.errors[name] = exceptions.FieldRequiredError()
                continue

            # dont need to validate None
            if value is None:
                data[name] = None
                continue

            try:
                validated_value = field.validate(value)
                internal_value = field.to_internal(validated_value)
                field_validator = getattr(
                    self, 'validate_{}'.format(name), None)
                if field_validator and callable(field_validator):
                    field_validator(internal_value)
                data[name] = internal_value
            except exceptions.FieldValidationError as e:
                self.errors[name] = e

        if self.errors:
            return
        try:
            data = self.validate(data)
        except exceptions.ValidationError as e:
            self.errors['__data_error__'] = [e]

        if not self.errors:
            self.validated_data = data

    def is_valid(self, raise_error=False):
        self._validate()
        if raise_error and self.errors:
            raise exceptions.ValidationError(self.errors)
        return False if self.errors else True

    def validate(self, data):
        """
        model-level validate.
        sub-class can override this method to validate data, return modified data
        """
        return data

    @property
    def str_errors(self):
        errors = dict()
        for name, error in six.iteritems(self.errors):
            errors[name] = error.get_detail()
        return errors

    @classmethod
    def to_dict(cls):
        """
        format Validator to dict
        """
        d = dict()
        for name, field in six.iteritems(cls._FIELDS_MAP):
            field_info = field.to_dict()
            d[name] = field_info
        return d

    @classmethod
    def mock_data(cls):
        """
        return random mocking data.
        mocking data will be valid in most case, but it maybe can't pass from your own `validate` method or `validator`
        """
        mocking_data = {}
        for name, field in six.iteritems(cls._FIELDS_MAP):
            mocking_data[name] = field.mock_data()
        return mocking_data

    def _format(self):
        fields = []
        for name, field in six.iteritems(self._FIELDS_MAP):
            fields.append('{0}:{1}'.format(name, field.FIELD_TYPE_NAME))
        fields = ','.join(fields)
        if len(fields) > 103:
            fields = fields[:100]
        return '<{0}: {1}>'.format(self.__class__.__name__, fields)

    def __str__(self):
        return self._format()

    def __repr__(self):
        return self._format()

def create_validator(data_struct_dict, name=None):
    """
    create a Validator instance from data_struct_dict

    :param data_struct_dict: a dict describe validator's fields, like the dict `to_dict()` method returned.
    :param name: name of Validator class

    :return: Validator instance
    """

    if name is None:
        name = 'FromDictValidator'
    attrs = {}
    for field_name, field_info in six.iteritems(data_struct_dict):
        field_type = field_info['type']
        if field_type == DictField.FIELD_TYPE_NAME and isinstance(field_info.get('validator'), dict):
            field_info['validator'] = create_validator(field_info['validator'])
        attrs[field_name] = create_field(field_info)
    name = force_str(name)
    return type(name, (Validator, ), attrs)


"""Module containing factory functions for parameter exploration"""

import logging
import sys
import itertools as itools
from collections import OrderedDict

def cartesian_product(parameter_dict, combined_parameters=()):
    """ Generates a Cartesian product of the input parameter dictionary.

    For example:

    >>> print cartesian_product({'param1':[1,2,3], 'param2':[42.0, 52.5]})
    {'param1':[1,1,2,2,3,3],'param2': [42.0,52.5,42.0,52.5,42.0,52.5]}

    :param parameter_dict:

        Dictionary containing parameter names as keys and iterables of data to explore.

    :param combined_parameters:

        Tuple of tuples. Defines the order of the parameters and parameters that are
        linked together.
        If an inner tuple contains only a single item, you can spare the
        inner tuple brackets.

        For example:

        >>> print cartesian_product( {'param1': [42.0, 52.5], 'param2':['a', 'b'], 'param3' : [1,2,3]}, ('param3',('param1', 'param2')))
        {param3':[1,1,2,2,3,3],'param1' : [42.0,52.5,42.0,52.5,42.0,52.5], 'param2':['a','b','a','b','a','b']}

    :returns: Dictionary with cartesian product lists.

    """
    if not combined_parameters:
        combined_parameters = list(parameter_dict)
    else:
        combined_parameters = list(combined_parameters)

    for idx, item in enumerate(combined_parameters):
        if isinstance(item, str):
            combined_parameters[idx] = (item,)

    iterator_list = []
    for item_tuple in combined_parameters:
        inner_iterator_list = [parameter_dict[key] for key in item_tuple]
        zipped_iterator = zip(*inner_iterator_list)
        iterator_list.append(zipped_iterator)

    result_dict = {}
    for key in parameter_dict:
        result_dict[key] = []

    cartesian_iterator = itools.product(*iterator_list)

    for cartesian_tuple in cartesian_iterator:
        for idx, item_tuple in enumerate(combined_parameters):
            for inneridx, key in enumerate(item_tuple):
                result_dict[key].append(cartesian_tuple[idx][inneridx])

    return result_dict

def find_unique_points(explored_parameters):
    """Takes a list of explored parameters and finds unique parameter combinations.

    If parameter ranges are hashable operates in O(N), otherwise O(N**2).

    :param explored_parameters:

        List of **explored** parameters

    :return:

        List of tuples, first entry being the parameter values, second entry a list
        containing the run position of the unique combination.

    """
    ranges = [param.f_get_range(copy=False) for param in explored_parameters]
    zipped_tuples = list(zip(*ranges))
    try:
        unique_elements = OrderedDict()
        for idx, val_tuple in enumerate(zipped_tuples):
            if val_tuple not in unique_elements:
                unique_elements[val_tuple] = []
            unique_elements[val_tuple].append(idx)
        return list(unique_elements.items())
    except TypeError:
        logger = logging.getLogger('pypet.find_unique')
        logger.error('Your parameter entries could not be hashed, '
                     'now I am sorting slowly in O(N**2).')
        unique_elements = []
        for idx, val_tuple in enumerate(zipped_tuples):
            matches = False
            for added_tuple, pos_list in unique_elements:
                matches = True
                for idx2, val in enumerate(added_tuple):
                    if not explored_parameters[idx2]._equal_values(val_tuple[idx2], val):
                        matches = False
                        break
                if matches:
                    pos_list.append(idx)
                    break
            if not matches:
                unique_elements.append((val_tuple, [idx]))
        return unique_elements


"""Module containing utilities for logging."""

__author__ = 'Robert Meyer'

try:
    # Python3
    FileNotFoundError
except NameError:
    FileNotFoundError = IOError

import configparser as cp
from io import StringIO
import logging
from logging.config import fileConfig
from logging.config import dictConfig
from logging import NullHandler
import os
import math
import sys
import ast
import copy
import multiprocessing as multip
import functools
import socket

import pypet.pypetconstants as pypetconstants
from pypet.utils.helpful_functions import progressbar, racedirs
from pypet.utils.decorators import retry
from pypet.slots import HasSlots

FILENAME_INDICATORS = set([pypetconstants.LOG_ENV,
                           pypetconstants.LOG_PROC,
                           pypetconstants.LOG_TRAJ,
                           pypetconstants.LOG_RUN,
                           pypetconstants.LOG_HOST,
                           pypetconstants.LOG_SET,
                           '.log',
                           '.txt'])
"""Set of strings that mark a log file"""

LOGGING_DICT = {
    'version': 1,
    'disable_existing_loggers': False,
    'formatters': {
        'file': {
            'format': '%(asctime)s %(name)s %(levelname)-8s %(message)s'
        },
        'stream': {
            'format': '%(processName)-10s %(name)s %(levelname)-8s %(message)s'
        }
    },
    'handlers': {
        'stream': {
            'class': 'logging.StreamHandler',
            'formatter': 'stream'
        },
        'file_main': {
            'class': 'logging.FileHandler',
            'formatter': 'file',
            'filename': os.path.join(pypetconstants.LOG_TRAJ,
                                     pypetconstants.LOG_ENV,
                                     'LOG.txt')
        },
        'file_error': {
            'class': 'logging.FileHandler',
            'formatter': 'file',
            'filename': os.path.join(pypetconstants.LOG_TRAJ,
                                     pypetconstants.LOG_ENV,
                                     'ERROR.txt'),
            'level': 'ERROR'
        }
    },
    'multiproc_formatters': {
        'file': {
            'format': '%(asctime)s %(name)s %(levelname)-8s %(message)s'
        },
    },
    'multiproc_handlers': {
        'file_main': {
            'class': 'logging.FileHandler',
            'formatter': 'file',
            'filename': os.path.join(pypetconstants.LOG_TRAJ,
                                     pypetconstants.LOG_ENV,
                                     '%s_%s_%s_LOG.txt' % (pypetconstants.LOG_RUN,
                                                           pypetconstants.LOG_HOST,
                                                           pypetconstants.LOG_PROC))
        },
        'file_error': {
            'class': 'logging.FileHandler',
            'formatter': 'file',
            'filename': os.path.join(pypetconstants.LOG_TRAJ,
                                     pypetconstants.LOG_ENV,
                                     '%s_%s_%s_ERROR.txt' % (pypetconstants.LOG_RUN,
                                                             pypetconstants.LOG_HOST,
                                                             pypetconstants.LOG_PROC)),
            'level': 'ERROR'
        }
    }
}
"""Dictionary containing the default configuration."""

def _change_logging_kwargs(kwargs):
    """ Helper function to turn the simple logging kwargs into a `log_config`."""
    log_levels = kwargs.pop('log_level', None)
    log_folder = kwargs.pop('log_folder', 'logs')
    logger_names = kwargs.pop('logger_names', '')
    if log_levels is None:
        log_levels = kwargs.pop('log_levels', logging.INFO)
    log_multiproc = kwargs.pop('log_multiproc', True)

    if not isinstance(logger_names, (tuple, list)):
        logger_names = [logger_names]
    if not isinstance(log_levels, (tuple, list)):
        log_levels = [log_levels]
    if len(log_levels) == 1:
        log_levels = [log_levels[0] for _ in logger_names]

    # We don't want to manipulate the original dictionary
    dictionary = copy.deepcopy(LOGGING_DICT)
    prefixes = ['']
    if not log_multiproc:
        for key in list(dictionary.keys()):
            if key.startswith('multiproc_'):
                del dictionary[key]
    else:
        prefixes.append('multiproc_')

    # Add all handlers to all loggers
    for prefix in prefixes:
        for handler_dict in dictionary[prefix + 'handlers'].values():
            if 'filename' in handler_dict:
                filename = os.path.join(log_folder, handler_dict['filename'])
                filename = os.path.normpath(filename)
                handler_dict['filename'] = filename
        dictionary[prefix + 'loggers'] = {}
        logger_dict = dictionary[prefix + 'loggers']
        for idx, logger_name in enumerate(logger_names):
            logger_dict[logger_name] = {
                'level': log_levels[idx],
                'handlers': list(dictionary[prefix + 'handlers'].keys())
            }

    kwargs['log_config'] = dictionary

def use_simple_logging(kwargs):
    """Checks if simple logging is requested"""
    return any([x in kwargs for x in ('log_folder', 'logger_names',
                                      'log_levels', 'log_multiproc', 'log_level')])

def simple_logging_config(func):
    """Decorator to allow a simple logging configuration.

    This encompasses giving a `log_folder`, `logger_names` as well as `log_levels`.

    """

    @functools.wraps(func)
    def new_func(self, *args, **kwargs):
        if use_simple_logging(kwargs):
            if 'log_config' in kwargs:
                raise ValueError('Please do not specify `log_config` '
                                 'if you want to use the simple '
                                 'way of providing logging configuration '
                                 '(i.e using `log_folder`, `logger_names` and/or `log_levels`).')
            _change_logging_kwargs(kwargs)

        return func(self, *args, **kwargs)

    return new_func

def try_make_dirs(filename):
    """ Tries to make directories for a given `filename`.

    Ignores any error but notifies via stderr.

    """
    try:
        dirname = os.path.dirname(os.path.normpath(filename))
        racedirs(dirname)
    except Exception as exc:
        sys.stderr.write('ERROR during log config file handling, could not create dirs for '
                         'filename `%s` because of: %s' % (filename, repr(exc)))

def get_strings(args):
    """Returns all valid python strings inside a given argument string."""
    string_list = []
    for elem in ast.walk(ast.parse(args)):
        if isinstance(elem, ast.Str):
            string_list.append(elem.s)
    return string_list

def rename_log_file(filename, trajectory=None,
                    env_name=None,
                    traj_name=None,
                    set_name=None,
                    run_name=None,
                    process_name=None,
                    host_name=None):
    """ Renames a given `filename` with valid wildcard placements.

    :const:`~pypet.pypetconstants.LOG_ENV` ($env) is replaces by the name of the
    trajectory`s environment.

    :const:`~pypet.pypetconstants.LOG_TRAJ` ($traj) is replaced by the name of the
    trajectory.

    :const:`~pypet.pypetconstants.LOG_RUN` ($run) is replaced by the name of the current
    run. If the trajectory is not set to a run 'run_ALL' is used.

    :const:`~pypet.pypetconstants.LOG_SET` ($set) is replaced by the name of the current
    run set. If the trajectory is not set to a run 'run_set_ALL' is used.

    :const:`~pypet.pypetconstants.LOG_PROC` ($proc) is replaced by the name fo the
    current process.

    :const:`~pypet.pypetconstant.LOG_HOST` ($host) is replaced by the name of the current host.

    :param filename:  A filename string
    :param traj:  A trajectory container, leave `None` if you provide all the parameters below
    :param env_name: Name of environemnt, leave `None` to get it from `traj`
    :param traj_name: Name of trajectory, leave `None` to get it from `traj`
    :param set_name: Name of run set, leave `None` to get it from `traj`
    :param run_name: Name of run, leave `None` to get it from `traj`
    :param process_name:

        The name of the desired process. If `None` the name of the current process is
        taken determined by the multiprocessing module.

    :param host_name:

        Name of host, leave `None` to determine it automatically with the platform module.

    :return: The new filename

    """
    if pypetconstants.LOG_ENV in filename:
        if env_name is None:
            env_name = trajectory.v_environment_name
        filename = filename.replace(pypetconstants.LOG_ENV, env_name)
    if pypetconstants.LOG_TRAJ in filename:
        if traj_name is None:
            traj_name = trajectory.v_name
        filename = filename.replace(pypetconstants.LOG_TRAJ, traj_name)
    if pypetconstants.LOG_RUN in filename:
        if run_name is None:
            run_name = trajectory.f_wildcard('$')
        filename = filename.replace(pypetconstants.LOG_RUN, run_name)
    if pypetconstants.LOG_SET in filename:
        if set_name is None:
            set_name = trajectory.f_wildcard('$set')
        filename = filename.replace(pypetconstants.LOG_SET, set_name)
    if pypetconstants.LOG_PROC in filename:
        if process_name is None:
            process_name = multip.current_process().name + '-' + str(os.getpid())
        filename = filename.replace(pypetconstants.LOG_PROC, process_name)
    if pypetconstants.LOG_HOST in filename:
        if host_name is None:
            host_name = socket.getfqdn().replace('.', '-')
        filename = filename.replace(pypetconstants.LOG_HOST, host_name)
    return filename

class HasLogger(HasSlots):
    """Abstract super class that automatically adds a logger to a class.

    To add a logger to a sub-class of yours simply call ``myobj._set_logger(name)``.
    If ``name=None`` the logger is chosen as follows:

        ``self._logger = logging.getLogger(self.__class.__.__module__ + '.' + self.__class__.__name__)``

    The logger can be accessed via ``myobj._logger``.

    """

    __slots__ = ('_logger',)

    def __getstate__(self):
        """Called for pickling.

        Removes the logger to allow pickling and returns a copy of `__dict__`.

        """
        state_dict = super(HasLogger, self).__getstate__()
        if '_logger' in state_dict:
            # Pickling does not work with loggers objects,
            # so we just keep the logger's name:
            state_dict['_logger'] = self._logger.name
        return state_dict

    def __setstate__(self, statedict):
        """Called after loading a pickle dump.

        Restores `__dict__` from `statedict` and adds a new logger.

        """
        super(HasLogger, self).__setstate__(statedict)
        if '_logger' in statedict:
            # If we re-instantiate the component the
            # logger attribute only contains a name,
            # so we also need to re-create the logger:
            self._set_logger(statedict['_logger'])

    def _set_logger(self, name=None):
        """Adds a logger with a given `name`.

        If no name is given, name is constructed as
        `type(self).__name__`.

        """
        if name is None:
            cls = self.__class__
            name = '%s.%s' % (cls.__module__, cls.__name__)
        self._logger = logging.getLogger(name)

class LoggingManager(object):
    """ Manager taking care of all logging related issues.

    :param trajectory: Trajectory container of Mock
    :param log_config: Logging configuration

        Can be a a full name of an `ini` file.
        An already instantiated config parser,
        or a logging dictionary.

    :param log_stdout: If `stdout` should be logged.

    :param report_progress:

        How to report progress.

    """
    def __init__(self, log_config=None, log_stdout=False,
                 report_progress=False):
        self.log_config = log_config
        self._sp_config = None
        self._mp_config = None
        self.log_stdout = log_stdout
        self.report_progress = report_progress
        self._tools = []
        self._null_handler = NullHandler()
        self._format_string = 'PROGRESS: Finished %d/%d runs '
        self._stdout_to_logger = None

        self.env_name = None
        self.traj_name = None
        self.set_name = None
        self.run_name = None
        # Format string for the progressbar

    def extract_replacements(self, trajectory):
        """Extracts the wildcards and file replacements from the `trajectory`"""
        self.env_name = trajectory.v_environment_name
        self.traj_name = trajectory.v_name
        self.set_name =  trajectory.f_wildcard('$set')
        self.run_name = trajectory.f_wildcard('$')

    def __getstate__(self):
        """ConfigParsers are not guaranteed to
        be picklable so we need to remove these."""
        state_dict = self.__dict__.copy()
        if isinstance(state_dict['log_config'], cp.RawConfigParser):
            # Config Parsers are not guaranteed to be picklable
            state_dict['log_config'] = True
        return state_dict

    def show_progress(self, n, total_runs):
        """Displays a progressbar"""
        if self.report_progress:
            percentage, logger_name, log_level = self.report_progress
            if logger_name == 'print':
                logger = 'print'
            else:
                logger = logging.getLogger(logger_name)

            if n == -1:
                # Compute the number of digits and avoid log10(0)
                digits = int(math.log10(total_runs + 0.1)) + 1
                self._format_string = 'PROGRESS: Finished %' + '%d' % digits + 'd/%d runs '

            fmt_string = self._format_string % (n + 1, total_runs) + '%s'
            reprint = log_level == 0
            progressbar(n, total_runs, percentage_step=percentage,
                        logger=logger, log_level=log_level,
                        fmt_string=fmt_string, reprint=reprint)

    def add_null_handler(self):
        """Adds a NullHandler to the root logger.

        This is simply added to avoid warnings that no
        logger has been configured.

        """
        root = logging.getLogger()
        root.addHandler(self._null_handler)

    def remove_null_handler(self):
        """Removes the NullHandler from the root logger."""
        root = logging.getLogger()
        root.removeHandler(self._null_handler)

    @staticmethod
    def tabula_rasa():
        """Removes all loggers and logging handlers. """
        erase_dict = {'disable_existing_loggers': False, 'version': 1}
        dictConfig(erase_dict)

    @staticmethod
    def _check_and_replace_parser_args(parser, section, option, rename_func, make_dirs=True):
        """ Searches for parser settings that define filenames.

        If such settings are found, they are renamed according to the wildcard
        rules. Moreover, it is also tried to create the corresponding folders.

        :param parser:  A config parser
        :param section: A config section
        :param option: The section option
        :param rename_func: A function to rename found files
        :param make_dirs: If the directories of the file should be created.

        """
        args = parser.get(section, option, raw=True)
        strings = get_strings(args)
        replace = False
        for string in strings:
            isfilename = any(x in string for x in FILENAME_INDICATORS)
            if isfilename:
                newstring = rename_func(string)
                if make_dirs:
                    try_make_dirs(newstring)
                # To work with windows path specifications we need this replacement:
                raw_string = string.replace('\\', '\\\\')
                raw_newstring = newstring.replace('\\', '\\\\')
                args = args.replace(raw_string, raw_newstring)
                replace = True
        if replace:
            parser.set(section, option, args)

    @staticmethod
    def _parser_to_string_io(parser):
        """Turns a ConfigParser into a StringIO stream."""
        memory_file = StringIO()
        parser.write(memory_file)
        memory_file.flush()
        memory_file.seek(0)
        return memory_file

    @staticmethod
    def _find_multiproc_options(parser):
        """ Searches for multiprocessing options within a ConfigParser.

        If such options are found, they are copied (without the `'multiproc_'` prefix)
        into a new parser.

        """
        sections = parser.sections()
        if not any(section.startswith('multiproc_') for section in sections):
            return None
        mp_parser = NoInterpolationParser()
        for section in sections:
            if section.startswith('multiproc_'):
                new_section = section.replace('multiproc_', '')
                mp_parser.add_section(new_section)
                options = parser.options(section)
                for option in options:
                    val = parser.get(section, option, raw=True)
                    mp_parser.set(new_section, option, val)
        return mp_parser

    @staticmethod
    def _find_multiproc_dict(dictionary):
        """ Searches for multiprocessing options in a given `dictionary`.

        If found they are copied (without the `'multiproc_'` prefix)
        into a new dictionary

        """
        if not any(key.startswith('multiproc_') for key in dictionary.keys()):
            return None
        mp_dictionary = {}
        for key in dictionary.keys():
            if key.startswith('multiproc_'):
                new_key = key.replace('multiproc_', '')
                mp_dictionary[new_key] = dictionary[key]
        mp_dictionary['version'] = dictionary['version']
        if 'disable_existing_loggers' in dictionary:
            mp_dictionary['disable_existing_loggers'] = dictionary['disable_existing_loggers']
        return mp_dictionary

    def check_log_config(self):
        """ Checks and converts all settings if necessary passed to the Manager.

        Searches for multiprocessing options as well.

        """
        if self.report_progress:
            if self.report_progress is True:
                self.report_progress = (5, 'pypet', logging.INFO)
            elif isinstance(self.report_progress, (int, float)):
                self.report_progress = (self.report_progress, 'pypet', logging.INFO)
            elif isinstance(self.report_progress, str):
                self.report_progress = (5, self.report_progress, logging.INFO)
            elif len(self.report_progress) == 2:
                self.report_progress = (self.report_progress[0], self.report_progress[1],
                                        logging.INFO)

        if self.log_config:
            if self.log_config == pypetconstants.DEFAULT_LOGGING:
                pypet_path = os.path.abspath(os.path.dirname(__file__))
                init_path = os.path.join(pypet_path, 'logging')
                self.log_config = os.path.join(init_path, 'default.ini')

            if isinstance(self.log_config, str):
                if not os.path.isfile(self.log_config):
                    raise ValueError('Could not find the logger init file '
                                     '`%s`.' % self.log_config)
                parser = NoInterpolationParser()
                parser.read(self.log_config)
            elif isinstance(self.log_config, cp.RawConfigParser):
                parser = self.log_config
            else:
                parser = None

            if parser is not None:
                self._sp_config = self._parser_to_string_io(parser)
                self._mp_config = self._find_multiproc_options(parser)
                if self._mp_config is not None:
                    self._mp_config = self._parser_to_string_io(self._mp_config)

            elif isinstance(self.log_config, dict):
                self._sp_config = self.log_config
                self._mp_config = self._find_multiproc_dict(self._sp_config)

        if self.log_stdout:
            if self.log_stdout is True:
                self.log_stdout = ('STDOUT', logging.INFO)
            if isinstance(self.log_stdout, str):
                self.log_stdout = (self.log_stdout, logging.INFO)
            if isinstance(self.log_stdout, int):
                self.log_stdout = ('STDOUT', self.log_stdout)

    def _handle_config_parsing(self, log_config):
        """ Checks for filenames within a config file and translates them.

        Moreover, directories for the files are created as well.

        :param log_config: Config file as a stream (like StringIO)

        """
        parser = NoInterpolationParser()
        parser.readfp(log_config)

        rename_func = lambda string: rename_log_file(string,
                                                     env_name=self.env_name,
                                                     traj_name=self.traj_name,
                                                     set_name=self.set_name,
                                                     run_name=self.run_name)

        sections = parser.sections()
        for section in sections:
            options = parser.options(section)
            for option in options:
                if option == 'args':
                    self._check_and_replace_parser_args(parser, section, option,
                                                        rename_func=rename_func)
        return parser

    def _handle_dict_config(self, log_config):
        """Recursively walks and copies the `log_config` dict and searches for filenames.

        Translates filenames and creates directories if necessary.

        """
        new_dict = dict()
        for key in log_config.keys():
            if key == 'filename':
                filename = log_config[key]
                filename = rename_log_file(filename,
                                           env_name=self.env_name,
                                           traj_name=self.traj_name,
                                           set_name=self.set_name,
                                           run_name=self.run_name)
                new_dict[key] = filename
                try_make_dirs(filename)
            elif isinstance(log_config[key], dict):
                inner_dict = self._handle_dict_config(log_config[key])
                new_dict[key] = inner_dict
            else:
                new_dict[key] = log_config[key]
        return new_dict

    def make_logging_handlers_and_tools(self, multiproc=False):
        """Creates logging handlers and redirects stdout."""

        log_stdout = self.log_stdout
        if sys.stdout is self._stdout_to_logger:
            # If we already redirected stdout we don't neet to redo it again
            log_stdout = False

        if self.log_config:
            if multiproc:
                proc_log_config = self._mp_config
            else:
                proc_log_config = self._sp_config

            if proc_log_config:
                if isinstance(proc_log_config, dict):
                    new_dict = self._handle_dict_config(proc_log_config)
                    dictConfig(new_dict)
                else:
                    parser = self._handle_config_parsing(proc_log_config)
                    memory_file = self._parser_to_string_io(parser)
                    fileConfig(memory_file, disable_existing_loggers=False)

        if log_stdout:
            #  Create a logging mock for stdout
            std_name, std_level = self.log_stdout

            stdout = StdoutToLogger(std_name, log_level=std_level)
            stdout.start()
            self._tools.append(stdout)

    def finalize(self, remove_all_handlers=True):
        """Finalizes the manager, closes and removes all handlers if desired."""
        for tool in self._tools:
            tool.finalize()
        self._tools = []
        self._stdout_to_logger = None
        for config in (self._sp_config, self._mp_config):
            if hasattr(config, 'close'):
                config.close()
        self._sp_config = None
        self._mp_config = None
        if remove_all_handlers:
            self.tabula_rasa()

class NoInterpolationParser(cp.ConfigParser):
    """Dummy class to solve a Python 3 bug/feature in configparser.py"""
    def __init__(self):
        try:
            # Needed for Python 3, see [http://bugs.python.org/issue21265]
            super(NoInterpolationParser, self).__init__(interpolation=None)
        except TypeError:
            # Python 2.x
            cp.ConfigParser.__init__(self)

class DisableAllLogging(object):
    """Context Manager that disables logging"""

    def __enter__(self):
        logging.disable(logging.CRITICAL)

    def __exit__(self, exc_type, exc_val, exc_tb):
        logging.disable(logging.NOTSET)

class StdoutToLogger(HasLogger):
    """Fake file-like stream object that redirects writes to a logger instance."""
    def __init__(self, logger_name, log_level=logging.INFO):
        self._logger_name = logger_name
        self._log_level = log_level
        self._linebuf = ''
        self._recursion = False
        self._redirection = False
        self._original_steam = None
        #self._logger = logging.getLogger(self._logger_name)
        self._set_logger(name=self._logger_name)

    def __getstate__(self):
        state_dict = super(StdoutToLogger, self).__getstate__()
        # The original stream cannot be pickled
        state_dict['_original_stream'] = None

    def start(self):
        """Starts redirection of `stdout`"""
        if sys.stdout is not self:
            self._original_steam = sys.stdout
            sys.stdout = self
            self._redirection = True
        if self._redirection:
            print('Established redirection of `stdout`.')

    def write(self, buf):
        """Writes data from buffer to logger"""
        if not self._recursion:
            self._recursion = True
            try:
                for line in buf.rstrip().splitlines():
                    self._logger.log(self._log_level, line.rstrip())
            finally:
                self._recursion = False
        else:
            # If stderr is redirected to stdout we can avoid further recursion by
            sys.__stderr__.write('ERROR: Recursion in Stream redirection!')

    def flush(self):
        """No-op to fulfil API"""
        pass

    def finalize(self):
        """Disables redirection"""
        if self._original_steam is not None and self._redirection:
            sys.stdout = self._original_steam
            print('Disabled redirection of `stdout`.')
            self._redirection = False
            self._original_steam = None

# class PypetTestFileHandler(logging.FileHandler):
#     """Takes care that data is flushed using fsync"""
#     def flush(self):
#         """
#         Flushes the stream.
#         """
#         self.acquire()
#         try:
#             if self.stream and hasattr(self.stream, "flush"):
#                 self.stream.flush()
#                 try:
#                     os.fsync(self.stream.fileno())
#                 except OSError:
#                     pass
#         finally:
#             self.release()
#
#     @retry(9, FileNotFoundError, 0.01, 'pypet.retry')
#     def _open(self):
#         try:
#             return logging.FileHandler._open(self)
#         except FileNotFoundError:
#             old_mode = self.mode
#             try:
#                 self.mode = 'w'
#                 try_make_dirs(self.baseFilename)
#                 return logging.FileHandler._open(self)
#             finally:
#                 self.mode = old_mode


"""Module containing utility functions to compare parameters and results"""

__author__ = 'Robert Meyer'

from collections import Sequence, Mapping

import numpy as np
import pandas as pd
import scipy.sparse as spsp

import pypet.slots as slots

def results_equal(a, b):
    """Compares two result instances

    Checks full name and all data. Does not consider the comment.

    :return: True or False

    :raises: ValueError if both inputs are no result instances

    """
    if a.v_is_parameter and b.v_is_parameter:
        raise ValueError('Both inputs are not results.')

    if a.v_is_parameter or b.v_is_parameter:
        return False

    if a.v_full_name != b.v_full_name:
        return False

    if hasattr(a, '_data') and not hasattr(b, '_data'):
        return False

    if hasattr(a, '_data'):
        akeyset = set(a._data.keys())
        bkeyset = set(b._data.keys())

        if akeyset != bkeyset:
            return False

        for key in a._data:
            val = a._data[key]
            bval = b._data[key]

            if not nested_equal(val, bval):
                return False

    return True

def parameters_equal(a, b):
    """Compares two parameter instances

    Checks full name, data, and ranges. Does not consider the comment.

    :return: True or False

    :raises: ValueError if both inputs are no parameter instances

    """
    if (not b.v_is_parameter and
            not a.v_is_parameter):
        raise ValueError('Both inputs are not parameters')

    if (not b.v_is_parameter or
            not a.v_is_parameter):
        return False

    if a.v_full_name != b.v_full_name:
        return False

    if a.f_is_empty() and b.f_is_empty():
        return True

    if a.f_is_empty() != b.f_is_empty():
        return False

    if not a._values_of_same_type(a.f_get(), b.f_get()):
        return False

    if not a._equal_values(a.f_get(), b.f_get()):
        return False

    if a.f_has_range() != b.f_has_range():
        return False

    if a.f_has_range():
        if a.f_get_range_length() != b.f_get_range_length():
            return False

        for myitem, bitem in zip(a.f_get_range(copy=False), b.f_get_range(copy=False)):
            if not a._values_of_same_type(myitem, bitem):
                return False
            if not a._equal_values(myitem, bitem):
                return False

    return True

def get_all_attributes(instance):
    """Returns an attribute value dictionary much like `__dict__` but incorporates `__slots__`"""
    try:
        result_dict = instance.__dict__.copy()
    except AttributeError:
        result_dict = {}

    if hasattr(instance, '__all_slots__'):
        all_slots = instance.__all_slots__
    else:
        all_slots = slots.get_all_slots(instance.__class__)

    for slot in all_slots:
        result_dict[slot] = getattr(instance, slot)

    result_dict.pop('__dict__', None)
    result_dict.pop('__weakref__', None)

    return result_dict

def nested_equal(a, b):
    """Compares two objects recursively by their elements.

    Also handles numpy arrays, pandas data and sparse matrices.

    First checks if the data falls into the above categories.
    If not, it is checked if a or b are some type of sequence or mapping and
    the contained elements are compared.
    If this is not the case, it is checked if a or b do provide a custom `__eq__` that
    evaluates to a single boolean value.
    If this is not the case, the attributes of a and b are compared.
    If this does not help either, normal `==` is used.

    Assumes hashable items are not mutable in a way that affects equality.
    Based on the suggestion from HERE_, thanks again Lauritz V. Thaulow :-)

    .. _HERE: http://stackoverflow.com/questions/18376935/best-practice-for-equality-in-python

    """
    if a is b:
        return True

    if a is None or b is None:
        return False

    a_sparse = spsp.isspmatrix(a)
    b_sparse = spsp.isspmatrix(b)
    if a_sparse != b_sparse:
        return False
    if a_sparse:
        if a.nnz == 0:
            return b.nnz == 0
        else:
            return not np.any((a != b).data)

    a_series = isinstance(a, pd.Series)
    b_series = isinstance(b, pd.Series)
    if a_series != b_series:
        return False
    if a_series:
        try:
            eq = (a == b).all()
            return eq
        except (TypeError, ValueError):
            # If Sequence itself contains numpy arrays we get here
            if not len(a) == len(b):
                return False
            for idx, itema in enumerate(a):
                itemb = b[idx]
                if not nested_equal(itema, itemb):
                    return False
            return True

    a_frame = isinstance(a, pd.DataFrame)
    b_frame = isinstance(b, pd.DataFrame)
    if a_frame != b_frame:
        return False
    if a_frame:
        try:
            if a.empty and b.empty:
                return True
            new_frame = a == b
            new_frame = new_frame | (pd.isnull(a) & pd.isnull(b))
            if isinstance(new_frame, pd.DataFrame):
                return np.all(new_frame.as_matrix())
        except (ValueError, TypeError):
            # The Value Error can happen if the data frame is of dtype=object and contains
            # numpy arrays. Numpy array comparisons do not evaluate to a single truth value
            for name in a:
                cola = a[name]
                if not name in b:
                    return False
                colb = b[name]
                if not len(cola) == len(colb):
                    return False
                for idx, itema in enumerate(cola):
                    itemb = colb[idx]
                    if not nested_equal(itema, itemb):
                        return False
            return True

    a_array = isinstance(a, np.ndarray)
    b_array = isinstance(b, np.ndarray)
    if a_array != b_array:
        return False
    if a_array:
        if a.shape != b.shape:
            return False
        return np.all(a == b)

    a_list = isinstance(a, (Sequence, list, tuple))
    b_list = isinstance(b, (Sequence, list, tuple))
    if a_list != b_list:
        return False
    if a_list:
        return all(nested_equal(x, y) for x, y in zip(a, b))

    a_mapping = isinstance(a, (Mapping, dict))
    b_mapping = isinstance(b, (Mapping, dict))
    if a_mapping != b_mapping:
        return False
    if a_mapping:
        keys_a = a.keys()
        if set(keys_a) != set(b.keys()):
            return False
        return all(nested_equal(a[k], b[k]) for k in keys_a)

    # Equality for general objects
    # for types that support __eq__ or __cmp__
    equality = NotImplemented
    try:
        equality = a.__eq__(b)
    except (AttributeError, NotImplementedError, TypeError, ValueError):
        pass
    if equality is NotImplemented:
        try:
            equality = b.__eq__(a)
        except (AttributeError, NotImplementedError, TypeError, ValueError):
            pass
    if equality is NotImplemented:
        try:
            cmp = a.__cmp__(b)
            if cmp is not NotImplemented:
                equality = cmp == 0
        except (AttributeError, NotImplementedError, TypeError, ValueError):
            pass
    if equality is NotImplemented:
        try:
            cmp = b.__cmp__(a)
            if cmp is not NotImplemented:
                equality = cmp == 0
        except (AttributeError, NotImplementedError, TypeError, ValueError):
            pass
    if equality is not NotImplemented:
        try:
            return bool(equality)
        except (AttributeError, NotImplementedError, TypeError, ValueError):
            pass

    # Compare objects based on their attributes
    attributes_a = get_all_attributes(a)
    attributes_b = get_all_attributes(b)
    if len(attributes_a) != len(attributes_b):
        return False
    if len(attributes_a) > 0:
        keys_a = list(attributes_a.keys())
        if set(keys_a) != set(attributes_b.keys()):
            return False

        return all(nested_equal(attributes_a[k], attributes_b[k]) for k in keys_a)

    # Ok they are really not equal
    return False


"""Module containing decorators"""

__author__ = 'Robert Meyer'

import functools
import warnings
import logging
import time

def manual_run(turn_into_run=True, store_meta_data=True, clean_up=True):
    """Can be used to decorate a function as a manual run function.

    This can be helpful if you want the run functionality without using an environment.

    :param turn_into_run:

        If the trajectory should become a `single run` with more specialized functionality
        during a single run.

    :param store_meta_data:

        If meta-data like runtime should be automatically stored

    :param clean_up:

        If all data added during the single run should be removed, only works
        if ``turn_into_run=True``.

    """
    def wrapper(func):
        @functools.wraps(func)
        def new_func(traj, *args, **kwargs):
            do_wrap = not traj._run_by_environment
            if do_wrap:
                traj.f_start_run(turn_into_run=turn_into_run)
            result = func(traj, *args, **kwargs)
            if do_wrap:
                traj.f_finalize_run(store_meta_data=store_meta_data,
                                    clean_up=clean_up)
            return result

        return new_func

    return wrapper

def deprecated(msg=''):
    """This is a decorator which can be used to mark functions
    as deprecated. It will result in a warning being emitted
    when the function is used.

    :param msg:

        Additional message added to the warning.

    """

    def wrapper(func):
        @functools.wraps(func)
        def new_func(*args, **kwargs):
            warning_string = "Call to deprecated function or property `%s`." % func.__name__
            warning_string = warning_string + ' ' + msg
            warnings.warn(
                warning_string,
                category=DeprecationWarning,
            )
            return func(*args, **kwargs)

        return new_func

    return wrapper

def copydoc(fromfunc, sep="\n"):
    """Decorator: Copy the docstring of `fromfunc`

    If the doc contains a line with the keyword `ABSTRACT`,
    like `ABSTRACT: Needs to be defined in subclass`, this line and the line after are removed.

    """

    def _decorator(func):
        sourcedoc = fromfunc.__doc__

        # Remove the ABSTRACT line:
        split_doc = sourcedoc.split('\n')
        split_doc_no_abstract = [line for line in split_doc if not 'ABSTRACT' in line]

        # If the length is different we have found an ABSTRACT line
        # Finally we want to remove the final blank line, otherwise
        # we would have three blank lines at the end
        if len(split_doc) != len(split_doc_no_abstract):
            sourcedoc = '\n'.join(split_doc_no_abstract[:-1])

        if func.__doc__ is None:
            func.__doc__ = sourcedoc
        else:
            func.__doc__ = sep.join([sourcedoc, func.__doc__])
        return func

    return _decorator

def kwargs_mutual_exclusive(param1_name, param2_name, map2to1=None):
    """ If there exist mutually exclusive parameters checks for them and maps param2 to 1."""
    def wrapper(func):
        @functools.wraps(func)
        def new_func(*args, **kwargs):
            if param2_name in kwargs:
                if param1_name in kwargs:
                    raise ValueError('You cannot specify `%s` and `%s` at the same time, '
                                     'they are mutually exclusive.' % (param1_name, param2_name))
                param2 = kwargs.pop(param2_name)
                if map2to1 is not None:
                    param1 = map2to1(param2)
                else:
                    param1 = param2
                kwargs[param1_name] = param1

            return func(*args, **kwargs)

        return new_func

    return wrapper

def kwargs_api_change(old_name, new_name=None):
    """This is a decorator which can be used if a kwarg has changed
    its name over versions to also support the old argument name.

    Issues a warning if the old keyword argument is detected and
    converts call to new API.

    :param old_name:

        Old name of the keyword argument

    :param new_name:

        New name of keyword argument

    """

    def wrapper(func):
        @functools.wraps(func)
        def new_func(*args, **kwargs):

            if old_name in kwargs:
                if new_name is None:
                    warning_string = 'Using deprecated keyword argument `%s` in function `%s`. ' \
                                 'This keyword is no longer supported, please don`t use it ' \
                                 'anymore.' % (old_name, func.__name__)
                else:
                    warning_string = 'Using deprecated keyword argument `%s` in function `%s`, ' \
                                 'please use keyword `%s` instead.' % \
                                 (old_name, func.__name__, new_name)
                warnings.warn(warning_string, category=DeprecationWarning)
                value = kwargs.pop(old_name)
                if new_name is not None:
                    kwargs[new_name] = value

            return func(*args, **kwargs)

        return new_func

    return wrapper

def not_in_run(func):
    """This is a decorator that signaling that a function is not available during a single run.

    """
    doc = func.__doc__
    na_string = '''\nATTENTION: This function is not available during a single run!\n'''

    if doc is not None:
        func.__doc__ = '\n'.join([doc, na_string])
    func._not_in_run = True

    @functools.wraps(func)
    def new_func(self, *args, **kwargs):

        if self._is_run:
            raise TypeError('Function `%s` is not available during a single run.' %
                            func.__name__)

        return func(self, *args, **kwargs)

    return new_func

def with_open_store(func):
    """This is a decorator that signaling that a function is only available if the storage is open.

    """
    doc = func.__doc__
    na_string = '''\nATTENTION: This function can only be used if the store is open!\n'''

    if doc is not None:
        func.__doc__ = '\n'.join([doc, na_string])
    func._with_open_store = True

    @functools.wraps(func)
    def new_func(self, *args, **kwargs):

        if not self.traj.v_storage_service.is_open:
            raise TypeError('Function `%s` is only available if the storage is open.' %
                            func.__name__)

        return func(self, *args, **kwargs)

    return new_func

def retry(n, errors, wait=0.0, logger_name=None):
    """This is a decorator that retries a function.

    Tries `n` times and catches a given tuple of `errors`.

    If the `n` retries are not enough, the error is reraised.

    If desired `waits` some seconds.

    Optionally takes a 'logger_name' of a given logger to print the caught error.

    """

    def wrapper(func):
        @functools.wraps(func)
        def new_func(*args, **kwargs):
            retries = 0
            while True:
                try:
                    result = func(*args, **kwargs)
                    if retries and logger_name:
                        logger = logging.getLogger(logger_name)
                        logger.debug('Retry of `%s` successful' % func.__name__)
                    return result
                except errors:
                    if retries >= n:
                        if logger_name:
                            logger = logging.getLogger(logger_name)
                            logger.exception('I could not execute `%s` with args %s and kwargs %s, '
                                             'starting next try. ' % (func.__name__,
                                                                      str(args),
                                                                      str(kwargs)))
                        raise
                    elif logger_name:
                        logger = logging.getLogger(logger_name)
                        logger.debug('I could not execute `%s` with args %s and kwargs %s, '
                                         'starting next try. ' % (func.__name__,
                                                                  str(args),
                                                                  str(kwargs)))
                    retries += 1
                    if wait:
                        time.sleep(wait)
        return new_func

    return wrapper

def _prfx_getattr_(obj, item):
    """Replacement of __getattr__"""
    if item.startswith('f_') or item.startswith('v_'):
        return getattr(obj, item[2:])
    raise AttributeError('`%s` object has no attribute `%s`' % (obj.__class__.__name__, item))

def _prfx_setattr_(obj, item, value):
    """Replacement of __setattr__"""
    if item.startswith('v_'):
        return setattr(obj, item[2:], value)
    else:
        return super(obj.__class__, obj).__setattr__(item, value)

def prefix_naming(cls):
    """Decorate that adds the prefix naming scheme"""
    if hasattr(cls, '__getattr__'):
        raise TypeError('__getattr__ already defined')
    cls.__getattr__ = _prfx_getattr_
    cls.__setattr__ = _prfx_setattr_
    return cls


__author__ = 'Robert Meyer'

import logging
import os # For path names being viable under Windows and Linux

from pypet.environment import Environment
from pypet.brian2.parameter import Brian2Parameter, Brian2MonitorResult
from pypet.utils.explore import cartesian_product
# Don't do this at home:
from brian2 import pF, nS, mV, ms, nA, NeuronGroup, SpikeMonitor, StateMonitor, linspace,\
    Network

# We define a function to set all parameter
def add_params(traj):
    """Adds all necessary parameters to `traj`."""

    # We set the BrianParameter to be the standard parameter
    traj.v_standard_parameter=Brian2Parameter
    traj.v_fast_access=True

    # Add parameters we need for our network
    traj.f_add_parameter('Net.C',281*pF)
    traj.f_add_parameter('Net.gL',30*nS)
    traj.f_add_parameter('Net.EL',-70.6*mV)
    traj.f_add_parameter('Net.VT',-50.4*mV)
    traj.f_add_parameter('Net.DeltaT',2*mV)
    traj.f_add_parameter('Net.tauw',40*ms)
    traj.f_add_parameter('Net.a',4*nS)
    traj.f_add_parameter('Net.b',0.08*nA)
    traj.f_add_parameter('Net.I',.8*nA)
    traj.f_add_parameter('Net.Vcut','vm > 0*mV') # practical threshold condition
    traj.f_add_parameter('Net.N',50)

    eqs='''
    dvm/dt=(gL*(EL-vm)+gL*DeltaT*exp((vm-VT)/DeltaT)+I-w)/C : volt
    dw/dt=(a*(vm-EL)-w)/tauw : amp
    Vr:volt
    '''
    traj.f_add_parameter('Net.eqs', eqs)
    traj.f_add_parameter('reset', 'vm=Vr;w+=b')

# This is our job that we will execute
def run_net(traj):
    """Creates and runs BRIAN network based on the parameters in `traj`."""

    eqs=traj.eqs

    # Create a namespace dictionairy
    namespace = traj.Net.f_to_dict(short_names=True, fast_access=True)
    # Create the Neuron Group
    neuron=NeuronGroup(traj.N, model=eqs, threshold=traj.Vcut, reset=traj.reset,
                       namespace=namespace)
    neuron.vm=traj.EL
    neuron.w=traj.a*(neuron.vm-traj.EL)
    neuron.Vr=linspace(-48.3*mV,-47.7*mV,traj.N) # bifurcation parameter

    # Run the network initially for 100 milliseconds
    print('Initial Run')
    net = Network(neuron)
    net.run(100*ms, report='text') # we discard the first spikes

    # Create a Spike Monitor
    MSpike=SpikeMonitor(neuron)
    net.add(MSpike)
    # Create a State Monitor for the membrane voltage, record from neurons 1-3
    MStateV = StateMonitor(neuron, variables=['vm'],record=[1,2,3])
    net.add(MStateV)

    # Now record for 500 milliseconds
    print('Measurement run')
    net.run(500*ms,report='text')

    # Add the BRAIN monitors
    traj.v_standard_result = Brian2MonitorResult
    traj.f_add_result('SpikeMonitor',MSpike)
    traj.f_add_result('StateMonitorV', MStateV)

def main():
    # Let's be very verbose!
    logging.basicConfig(level = logging.INFO)

    # Let's do multiprocessing this time with a lock (which is default)
    filename = os.path.join('hdf5', 'example_23.hdf5')
    env = Environment(trajectory='Example_23_BRIAN2',
                      filename=filename,
                      file_title='Example_23_Brian2',
                      comment = 'Go Brian2!',
                      dynamically_imported_classes=[Brian2MonitorResult, Brian2Parameter])

    traj = env.trajectory

    # 1st a) add the parameters
    add_params(traj)

    # 1st b) prepare, we want to explore the different network sizes and different tauw time scales
    traj.f_explore(cartesian_product({traj.f_get('N').v_full_name:[50,60],
                           traj.f_get('tauw').v_full_name:[30*ms,40*ms]}))

    # 2nd let's run our experiment
    env.run(run_net)

    # You can take a look at the results in the hdf5 file if you want!

    # Finally disable logging and close all log-files
    env.disable_logging()

if __name__ == '__main__':
    main()


__author__ = 'Robert Meyer'

import numpy as np
import inspect
import os # For path names being viable under Windows and Linux

from pypet import Environment, Parameter, ArrayParameter, Trajectory
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D

# Here we will see how we can write our own custom parameters and how we can use
# it with a trajectory.

# Now we want to do a more sophisticated simulations, we will integrate a differential equation
# with an Euler scheme

# Let's first define our job to do
def euler_scheme(traj, diff_func):
    """Simulation function for Euler integration.

    :param traj:

        Container for parameters and results

    :param diff_func:

        The differential equation we want to integrate

    """

    steps = traj.steps
    initial_conditions = traj.initial_conditions
    dimension = len(initial_conditions)

    # This array will collect the results
    result_array = np.zeros((steps,dimension))
    # Get the function parameters stored into `traj` as a dictionary
    # with the (short) names as keys :
    func_params_dict = traj.func_params.f_to_dict(short_names=True, fast_access=True)
    # Take initial conditions as first result
    result_array[0] = initial_conditions

    # Now we compute the Euler Scheme steps-1 times
    for idx in range(1,steps):
        result_array[idx] = diff_func(result_array[idx-1], **func_params_dict) * traj.dt + \
                            result_array[idx-1]
    # Note the **func_params_dict unzips the dictionary, it's the reverse of **kwargs in function
    # definitions!

    #Finally we want to keep the results
    traj.f_add_result('euler_evolution', data=result_array, comment='Our time series data!')

# Ok, now we want to make our own (derived) parameter that stores source code of python functions.
# We do NOT want a parameter that stores an executable function. This would complicate
# the problem a lot. If you have something like that in mind, you might wanna take a look
# at the marshal (http://docs.python.org/2/library/marshal) module
# or dill (https://pypi.python.org/pypi/dill) package.
# Our intention here is to define a parameter that we later on use as a derived parameter
# to simply keep track of the source code we use ('git' would be, of course, the better solution
# but this is just an illustrative example)
class FunctionParameter(Parameter):
    # We need to override the `f_set` function and simply extract the the source code if our
    # item is callable and store this instead.
    def f_set(self, data):
        if callable(data):
            data = inspect.getsource(data)
        return super(FunctionParameter, self).f_set(data)

    # For more complicate parameters you might consider implementing:
    # `f_supports` (we do not need it since we convert the data to stuff the parameter already
    #    supports, and that is strings!)
    #
    # and
    # the private functions
    #
    # `_values_of_same_type` (to tell whether data is similar, i.e. of two data items agree in their
    #   type, this is important to only allow exploration within the same dimension.
    #   For instance, a parameter that stores integers, should only explore integers etc.)
    #
    # and
    #
    # `_equal_values` (to tell if two data items are equal. This is important for merging if you
    #       want to erase duplicate parameter points. The trajectory needs to know when a
    #       parameter space point was visited before.)
    #
    # and
    #
    # `_store` (to be able to turn complex data into basic types understood by the storage service)
    #
    # and
    #
    # `_load` (to be able to recover your complex data form the basic types understood by the storage
    # service)
    #
    # But for now we will rely on the parent functions and hope for the best!

# Ok now let's follow the ideas in the final section of the cookbook and let's
# have a part in our simulation that only defines the parameters.
def add_parameters(traj):
    """Adds all necessary parameters to the `traj` container"""

    traj.f_add_parameter('steps', 10000, comment='Number of time steps to simulate')
    traj.f_add_parameter('dt', 0.01, comment='Step size')

    # Here we want to add the initial conditions as an array parameter. We will simulate
    # a 3-D differential equation, the Lorenz attractor.
    traj.f_add_parameter(ArrayParameter,'initial_conditions', np.array([0.0,0.0,0.0]),
                         comment = 'Our initial conditions, as default we will start from'
                                   ' origin!')

    # We will group all parameters of the Lorenz differential equation into the group 'func_params'
    traj.f_add_parameter('func_params.sigma', 10.0)
    traj.f_add_parameter('func_params.beta', 8.0/3.0)
    traj.f_add_parameter('func_params.rho', 28.0)

    #For the fun of it we will annotate the  group
    traj.func_params.v_annotations.info='This group contains as default the original values chosen ' \
                                   'by Edward Lorenz in 1963. Check it out on wikipedia ' \
                                   '(https://en.wikipedia.org/wiki/Lorenz_attractor)!'

# We need to define the lorenz function, we will assume that the value array is 3 dimensional,
# First dimension contains the x-component, second y-component, and third the z-component
def diff_lorenz(value_array, sigma, beta, rho):
    """The Lorenz attractor differential equation

    :param value_array: 3d array containing the x,y, and z component values.
    :param sigma: Constant attractor parameter
    :param beta: FConstant attractor parameter
    :param rho: Constant attractor parameter

    :return: 3d array of the Lorenz system evaluated at `value_array`

    """
    diff_array = np.zeros(3)
    diff_array[0] = sigma * (value_array[1]-value_array[0])
    diff_array[1] = value_array[0] * (rho - value_array[2]) - value_array[1]
    diff_array[2] = value_array[0] * value_array[1] - beta * value_array[2]

    return diff_array

# And here goes our main function
def main():

    filename = os.path.join('hdf5', 'example_05.hdf5')
    env = Environment(trajectory='Example_05_Euler_Integration',
                      filename=filename,
                      file_title='Example_05_Euler_Integration',
                      overwrite_file=True,
                      comment='Go for Euler!')

    traj = env.trajectory
    trajectory_name = traj.v_name

    # 1st a) phase parameter addition
    add_parameters(traj)

    # 1st b) phase preparation
    # We will add the differential equation (well, its source code only) as a derived parameter
    traj.f_add_derived_parameter(FunctionParameter,'diff_eq', diff_lorenz,
                                 comment='Source code of our equation!')

    # We want to explore some initial conditions
    traj.f_explore({'initial_conditions' : [
        np.array([0.01,0.01,0.01]),
        np.array([2.02,0.02,0.02]),
        np.array([42.0,4.2,0.42])
    ]})
    # 3 different conditions are enough for an illustrative example

    # 2nd phase let's run the experiment
    # We pass `euler_scheme` as our top-level simulation function and
    # the Lorenz equation 'diff_lorenz' as an additional argument
    env.run(euler_scheme, diff_lorenz)

    # We don't have a 3rd phase of post-processing here

    # 4th phase analysis.
    # I would recommend to do post-processing completely independent from the simulation,
    # but for simplicity let's do it here.

    # Let's assume that we start all over again and load the entire trajectory new.
    # Yet, there is an error within this approach, do you spot it?
    del traj
    traj = Trajectory(filename=filename)

    # We will only fully load parameters and derived parameters.
    # Results will be loaded manually later on.
    try:
        # However, this will fail because our trajectory does not know how to
        # build the FunctionParameter. You have seen this coming, right?
        traj.f_load(name=trajectory_name, load_parameters=2, load_derived_parameters=2,
                    load_results=1)
    except ImportError as e:

        print('That did\'nt work, I am sorry: %s ' % str(e))

        # Ok, let's try again but this time with adding our parameter to the imports
        traj = Trajectory(filename=filename,
                           dynamically_imported_classes=FunctionParameter)

        # Now it works:
        traj.f_load(name=trajectory_name, load_parameters=2, load_derived_parameters=2,
                    load_results=1)

    #For the fun of it, let's print the source code
    print('\n ---------- The source code of your function ---------- \n %s' % traj.diff_eq)

    # Let's get the exploration array:
    initial_conditions_exploration_array = traj.f_get('initial_conditions').f_get_range()
    # Now let's plot our simulated equations for the different initial conditions:
    # We will iterate through the run names
    for idx, run_name in enumerate(traj.f_get_run_names()):

        #Get the result of run idx from the trajectory
        euler_result = traj.results.f_get(run_name).euler_evolution
        # Now we manually need to load the result. Actually the results are not so large and we
        # could load them all at once. But for demonstration we do as if they were huge:
        traj.f_load_item(euler_result)
        euler_data = euler_result.data

        #Plot fancy 3d plot
        fig = plt.figure(idx)
        ax = fig.gca(projection='3d')
        x = euler_data[:,0]
        y = euler_data[:,1]
        z = euler_data[:,2]
        ax.plot(x, y, z, label='Initial Conditions: %s' % str(initial_conditions_exploration_array[idx]))
        plt.legend()
        plt.show()

        # Now we free the data again (because we assume its huuuuuuge):
        del euler_data
        euler_result.f_empty()

    # You have to click through the images to stop the example_05 module!

    # Finally disable logging and close all log-files
    env.disable_logging()

if __name__ == '__main__':
    main()


""" Module to create storage service from given settings.

Currently only the HDF5StorageService is supported.
But to be extended in the future.

"""

__author__ = 'Robert Meyer'

import inspect
import os

from pypet.utils.helpful_functions import get_matching_kwargs
from pypet.storageservice import HDF5StorageService
from pypet.utils.dynamicimports import create_class

def _create_storage(storage_service, trajectory=None, **kwargs):
    """Creates a service from a constructor and checks which kwargs are not used"""
    kwargs_copy = kwargs.copy()
    kwargs_copy['trajectory'] = trajectory
    matching_kwargs = get_matching_kwargs(storage_service, kwargs_copy)
    storage_service = storage_service(**matching_kwargs)
    unused_kwargs = set(kwargs.keys()) - set(matching_kwargs.keys())
    return storage_service, unused_kwargs

def storage_factory(storage_service, trajectory=None, **kwargs):
    """Creates a storage service, to be extended if new storage services are added

    :param storage_service:

        Storage Service instance of constructor or a string pointing to a file

    :param trajectory:

        A trajectory instance

    :param kwargs:

        Arguments passed to the storage service

    :return:

        A storage service and a set of not used keyword arguments from kwargs

    """

    if 'filename' in kwargs and storage_service is None:
        filename = kwargs['filename']
        _, ext = os.path.splitext(filename)
        if ext in ('.hdf', '.h4', '.hdf4', '.he2', '.h5', '.hdf5', '.he5'):
            storage_service = HDF5StorageService
        else:
            raise ValueError('Extension `%s` of filename `%s` not understood.' %
                             (ext, filename))
    elif isinstance(storage_service, str):
        class_name = storage_service.split('.')[-1]
        storage_service = create_class(class_name, [storage_service, HDF5StorageService])

    if inspect.isclass(storage_service):
        return _create_storage(storage_service, trajectory, **kwargs)
    else:
        return storage_service, set(kwargs.keys())


__author__ = 'Robert Meyer'

import os # To allow file paths working under Windows and Linux

from pypet import Environment, Result, Parameter

def multiply(traj):
    """Example of a sophisticated simulation that involves multiplying two values.

    :param traj:

        Trajectory containing
        the parameters in a particular combination,
        it also serves as a container for results.

    """
    z=traj.mylink1*traj.mylink2 # And again we now can also use the different names
    # due to the creation of links
    traj.f_add_result('runs.$.z', z, comment='Result of our simulation!')

# Create an environment that handles running
filename = os.path.join('hdf5','example_14.hdf5')
env = Environment(trajectory='Multiplication',
                  filename=filename,
                  file_title='Example_14_Links',
                  overwrite_file=True,
                  comment='How to use links')

# The environment has created a trajectory container for us
traj = env.trajectory

# Add both parameters
traj.par.x = Parameter('x', 1, 'I am the first dimension!')
traj.par.y = Parameter('y', 1, 'I am the second dimension!')

# Explore just two points
traj.f_explore({'x': [3, 4]})

# So far everything was as in the first example. However now we add links:
traj.f_add_link('mylink1', traj.f_get('x'))
# Note the `f_get` here to ensure to get the parameter instance, not the value 1
# This allows us now to access x differently:
print('x=' + str(traj.mylink1))
# We can try to avoid fast access as well, and recover the original parameter
print(str(traj.f_get('mylink1')))
# And also colon notation is allowed that creates new groups on the fly
traj.f_add_link('parameters.mynewgroup.mylink2', traj.f_get('y'))

# And, of course, we can also use the links during run:
env.run(multiply)

# Finally disable logging and close all log-files
env.disable_logging()