# -*- coding: utf-8 -*-

# Copyright 2018, IBM.
#
# This source code is licensed under the Apache License, Version 2.0 found in
# the LICENSE.txt file in the root directory of this source tree.

"""Polymorphic fields that represent one of several schemas or types."""

from collections.abc import Iterable
from functools import partial

from marshmallow.exceptions import ValidationError
from marshmallow.utils import is_collection
from marshmallow_polyfield import PolyField

from qiskit.validation import ModelTypeValidator

class BasePolyField(PolyField, ModelTypeValidator):
    """Base class for polymorphic fields.

    Defines a Field that can contain data fitting different ``BaseSchema``.
    Deciding the type is performed by the ``to_dict_selector()`` and
    ``from_dict_selector()`` functions, that act on ``choices``.

    Subclasses are recommended to customize the ``to_dict_selector()`` and
    ``from_dict_selector()``, adding the necessary logic for inspecting
    ``choices`` and the data, and returning one of the Schemas.

     Args:
        choices (dict or iterable): iterable or dict containing the schema
        instances and the information needed for performing disambiguation.
        many (bool): whether the field is a collection of objects.
        metadata (dict): the same keyword arguments that ``PolyField`` receives.
    """

    def __init__(self, choices, many=False, **metadata):

        if isinstance(choices, dict):
            self._choices = choices.values()
        elif isinstance(choices, (list, tuple)):
            self._choices = list(choices)
        else:
            raise ValueError(
                '`choices` parameter must be a dict, a list or a tuple')

        to_dict_selector = partial(self.to_dict_selector, choices)
        from_dict_selector = partial(self.from_dict_selector, choices)

        super().__init__(to_dict_selector, from_dict_selector, many=many, **metadata)

    def to_dict_selector(self, choices, *args, **kwargs):
        """Return an schema in ``choices`` for serialization."""
        raise NotImplementedError

    def from_dict_selector(self, choices, *args, **kwargs):
        """Return an schema in ``choices`` for deserialization."""
        raise NotImplementedError

    def _deserialize(self, value, attr, data):
        """Override ``_deserialize`` for customizing the exception raised."""
        try:
            return super()._deserialize(value, attr, data)
        except ValidationError as ex:
            if 'deserialization_schema_selector' in ex.messages[0]:
                ex.messages[0] = 'Cannot find a valid schema among the choices'
            raise

    def _serialize(self, value, key, obj):
        """Override ``_serialize`` for customizing the exception raised."""
        try:
            return super()._serialize(value, key, obj)
        except TypeError as ex:
            if 'serialization_schema_selector' in str(ex):
                raise ValidationError('Data from an invalid schema')
            raise

    def _expected_types(self):
        return tuple(schema.model_cls for schema in self._choices)

    def check_type(self, value, attr, data):
        """Check if the type of the value is one of the possible choices.

        Possible choices are the model classes bound to the possible schemas.
        """
        if self.many and not is_collection(value):
            raise self._not_expected_type(
                value, Iterable, fields=[self], field_names=attr, data=data)

        _check_type = super().check_type

        errors = []
        values = value if self.many else [value]
        for idx, v in enumerate(values):
            try:
                _check_type(v, idx, values)
            except ValidationError as err:
                errors.append(err.messages)

        if errors:
            errors = errors if self.many else errors[0]
            raise ValidationError(errors)

        return value

class TryFrom(BasePolyField):
    """Polymorphic field that returns the first candidate schema that matches.

    Polymorphic field that accepts a list of candidate schemas, and iterates
    through them, returning the first Schema that matches the data. Note that
    the list of choices is traversed in order, and an attempt to match the
    data is performed until a valid Schema is found, which might have
    performance implications.

    Examples:
        class PetOwnerSchema(BaseSchema):
            pet = TryFrom([CatSchema, DogSchema])

    Args:
        choices (list[class]): list of BaseSchema classes that are iterated in
            order for for performing disambiguation.
        many (bool): whether the field is a collection of objects.
        metadata (dict): the same keyword arguments that ``PolyField`` receives.
    """

    def to_dict_selector(self, choices, base_object, *_):
        # pylint: disable=arguments-differ
        if getattr(base_object, 'schema'):
            if base_object.schema.__class__ in choices:
                return base_object.schema

        return None

    def from_dict_selector(self, choices, base_dict, *_):
        # pylint: disable=arguments-differ
        for schema_cls in choices:
            try:
                schema = schema_cls(strict=True)
                schema.load(base_dict)

                return schema_cls()
            except ValidationError:
                pass
        return None

class ByAttribute(BasePolyField):
    """Polymorphic field that disambiguates based on an attribute's existence.

    Polymorphic field that accepts a dictionary of (``'attribute': schema``)
    entries, and checks for the existence of ``attribute`` in the data for
    disambiguating.

    Examples:
        class PetOwnerSchema(BaseSchema):
            pet = ByAttribute({'fur_density': CatSchema,
                               'barking_power': DogSchema)}

    Args:
        choices (dict[string: class]): dictionary with attribute names as
            keys, and BaseSchema classes as values.
        many (bool): whether the field is a collection of objects.
        metadata (dict): the same keyword arguments that ``PolyField`` receives.
    """

    def to_dict_selector(self, choices, base_object, *_):
        # pylint: disable=arguments-differ
        if getattr(base_object, 'schema'):
            if base_object.schema.__class__ in choices.values():
                return base_object.schema

        return None

    def from_dict_selector(self, choices, base_dict, *_):
        # pylint: disable=arguments-differ
        for attribute, schema_cls in choices.items():
            if attribute in base_dict:
                return schema_cls()

        return None

class ByType(ModelTypeValidator):
    """Polymorphic field that disambiguates based on an attribute's type.

    Polymorphic field that accepts a list of ``Fields``, and checks that the
    data belongs to any of those types. Note this Field does not inherit from
    ``BasePolyField``, as it operates directly on ``Fields`` instead of
    operating in ``Schemas``.

    Examples:
        class PetOwnerSchema(BaseSchema):
            contact_method = ByType([fields.Email(), fields.Url()])

    Args:
        choices (list[Field]): list of accepted `Fields` instances.
        *args (tuple): args for Field.
        **kwargs (dict): kwargs for Field.
    """

    default_error_messages = {
        'invalid': 'Value {value} does not fit any of the types {types}.'
    }

    def __init__(self, choices, *args, **kwargs):
        self.choices = choices
        super().__init__(*args, **kwargs)

    def _serialize(self, value, attr, obj):
        for field in self.choices:
            try:
                return field._serialize(value, attr, obj)
            except ValidationError:
                pass

        self.fail('invalid', value=value, types=self.choices)

    def _deserialize(self, value, attr, data):
        for field in self.choices:
            try:
                return field._deserialize(value, attr, data)
            except ValidationError:
                pass

        self.fail('invalid', value=value, types=self.choices)

    def check_type(self, value, attr, data):
        """Check if at least one of the possible choices validates the value.

        Possible choices are assumed to be ``ModelTypeValidator`` fields.
        """
        for field in self.choices:
            if isinstance(field, ModelTypeValidator):
                try:
                    return field.check_type(value, attr, data)
                except ValidationError:
                    pass

        raise self._not_expected_type(
            value, [field.__class__ for field in self.choices],
            fields=[self], field_names=attr, data=data)


# -*- coding: utf-8 -*-

# Copyright 2017, IBM.
#
# This source code is licensed under the Apache License, Version 2.0 found in
# the LICENSE.txt file in the root directory of this source tree.

# pylint: disable=invalid-name,anomalous-backslash-in-string

"""
A collection of useful quantum information functions for states.

"""

import numpy as np
import scipy.linalg as la

def state_fidelity(state1, state2):
    """Return the state fidelity between two quantum states.

    Either input may be a state vector, or a density matrix. The state
    fidelity (F) for two density matrices is defined as::

        F(rho1, rho2) = Tr[sqrt(sqrt(rho1).rho2.sqrt(rho1))] ^ 2

    For a pure state and mixed state the fidelity is given by::

        F(|psi1>, rho2) = <psi1|rho2|psi1>

    For two pure states the fidelity is given by::

        F(|psi1>, |psi2>) = |<psi1|psi2>|^2

    Args:
        state1 (array_like): a quantum state vector or density matrix.
        state2 (array_like): a quantum state vector or density matrix.

    Returns:
        array_like: The state fidelity F(state1, state2).
    """
    # convert input to numpy arrays
    s1 = np.array(state1)
    s2 = np.array(state2)

    # fidelity of two state vectors
    if s1.ndim == 1 and s2.ndim == 1:
        return np.abs(s2.conj().dot(s1)) ** 2
    # fidelity of vector and density matrix
    elif s1.ndim == 1:
        # psi = s1, rho = s2
        return np.abs(s1.conj().dot(s2).dot(s1))
    elif s2.ndim == 1:
        # psi = s2, rho = s1
        return np.abs(s2.conj().dot(s1).dot(s2))
    # fidelity of two density matrices
    s1sq = _funm_svd(s1, np.sqrt)
    s2sq = _funm_svd(s2, np.sqrt)
    return np.linalg.norm(s1sq.dot(s2sq), ord='nuc') ** 2

def _funm_svd(a, func):
    """Apply real scalar function to singular values of a matrix.

    Args:
        a (array_like): (N, N) Matrix at which to evaluate the function.
        func (callable): Callable object that evaluates a scalar function f.

    Returns:
        ndarray: funm (N, N) Value of the matrix function specified by func
        evaluated at `A`.
    """
    U, s, Vh = la.svd(a, lapack_driver='gesvd')
    S = np.diag(func(s))
    return U.dot(S).dot(Vh)


# -*- coding: utf-8 -*-

# Copyright 2018, IBM.
#
# This source code is licensed under the Apache License, Version 2.0 found in
# the LICENSE.txt file in the root directory of this source tree.

"""This pass checks if a DAG circuit is already mapped to a coupling map.

It checks that all 2-qubit interactions are laid out to be physically close.
"""

from qiskit.transpiler.basepasses import AnalysisPass
from qiskit.transpiler import Layout

class CheckMap(AnalysisPass):
    """
    Checks if a DAGCircuit is mapped to `coupling_map`, setting `is_swap_mapped`
    in the property set as True if mapped. False otherwise.
    """

    def __init__(self, coupling_map, initial_layout=None):
        """
        Checks if a DAGCircuit is mapped to `coupling_map`.
        Args:
            coupling_map (CouplingMap): Directed graph representing a coupling map.
            initial_layout (Layout): The initial layout of the DAG to analyze.
        """
        super().__init__()
        self.layout = initial_layout
        self.coupling_map = coupling_map

    def run(self, dag):
        """
        If `dag` is mapped to `coupling_map`, the property
        `is_swap_mapped` is set to True (or to False otherwise).

        Args:
            dag (DAGCircuit): DAG to map.
        """
        if self.layout is None:
            if self.property_set["layout"]:
                self.layout = self.property_set["layout"]
            else:
                self.layout = Layout.generate_trivial_layout(*dag.qregs.values())

        self.property_set['is_swap_mapped'] = True

        for gate in dag.twoQ_gates():
            physical_q0 = self.layout[gate.qargs[0]]
            physical_q1 = self.layout[gate.qargs[1]]

            if self.coupling_map.distance(physical_q0, physical_q1) != 1:
                self.property_set['is_swap_mapped'] = False
                return


# -*- coding: utf-8 -*-

# Copyright 2017, IBM.
#
# This source code is licensed under the Apache License, Version 2.0 found in
# the LICENSE.txt file in the root directory of this source tree.
"""
Simulator command to snapshot internal simulator representation.
"""

import warnings

from qiskit import QuantumCircuit
from qiskit.circuit import CompositeGate
from qiskit import QuantumRegister
from qiskit.circuit import Instruction
from qiskit.extensions.exceptions import ExtensionError

class Snapshot(Instruction):
    """Simulator snapshot instruction."""

    def __init__(self,
                 label,
                 snapshot_type='statevector',
                 num_qubits=0,
                 num_clbits=0,
                 params=None):
        """Create new snapshot instruction.

        Args:
            label (str): the snapshot label for result data.
            snapshot_type (str): the type of the snapshot.
            num_qubits (int): the number of qubits for the snapshot type [Default: 0].
            num_clbits (int): the number of classical bits for the snapshot type [Default: 0].
            params (list or None): the parameters for snapshot_type [Default: None].

        Raises:
            ExtensionError: if snapshot label is invalid.
        """
        if not isinstance(label, str):
            raise ExtensionError('Snapshot label must be a string.')
        self._label = label
        self._snapshot_type = snapshot_type
        if params is None:
            params = []
        super().__init__('snapshot', num_qubits, num_clbits, params)

    def assemble(self):
        """Assemble a QasmQobjInstruction"""
        instruction = super().assemble()
        instruction.label = self._label
        instruction.snapshot_type = self._snapshot_type
        return instruction

    def inverse(self):
        """Special case. Return self."""
        return Snapshot(self.num_qubits, self.num_clbits, self.params[0],
                        self.params[1])

    @property
    def snapshot_type(self):
        """Return snapshot type"""
        return self._snapshot_type

    @property
    def label(self):
        """Return snapshot label"""
        return self._label

    @label.setter
    def label(self, name):
        """Set snapshot label to name

        Args:
            name (str or None): label to assign unitary

        Raises:
            TypeError: name is not string or None.
        """
        if isinstance(name, str):
            self._label = name
        else:
            raise TypeError('label expects a string')

def snapshot(self,
             label,
             snapshot_type='statevector',
             qubits=None,
             params=None):
    """Take a statevector snapshot of the internal simulator representation.
    Works on all qubits, and prevents reordering (like barrier).

    For other types of snapshots use the Snapshot extension directly.

    Args:
        label (str): a snapshot label to report the result
        snapshot_type (str): the type of the snapshot.
        qubits (list or None): the qubits to apply snapshot to [Default: None].
        params (list or None): the parameters for snapshot_type [Default: None].

    Returns:
        QuantumCircuit: with attached command

    Raises:
        ExtensionError: malformed command
    """
    # Convert label to string for backwards compatibility
    if not isinstance(label, str):
        warnings.warn(
            "Snapshot label should be a string, "
            "implicit conversion is depreciated.", DeprecationWarning)
        label = str(label)
    # If no qubits are specified we add all qubits so it acts as a barrier
    # This is needed for full register snapshots like statevector
    if isinstance(qubits, QuantumRegister):
        qubits = qubits[:]
    if not qubits:
        tuples = []
        if isinstance(self, QuantumCircuit):
            for register in self.qregs:
                tuples.append(register)
        if not tuples:
            raise ExtensionError('no qubits for snapshot')
        qubits = []
        for tuple_element in tuples:
            if isinstance(tuple_element, QuantumRegister):
                for j in range(tuple_element.size):
                    qubits.append((tuple_element, j))
            else:
                qubits.append(tuple_element)
    return self.append(
        Snapshot(
            label,
            snapshot_type=snapshot_type,
            num_qubits=len(qubits),
            params=params), qubits)

# Add to QuantumCircuit and CompositeGate classes
QuantumCircuit.snapshot = snapshot
CompositeGate.snapshot = snapshot


# -*- coding: utf-8 -*-

# Copyright 2019, IBM.
#
# This source code is licensed under the Apache License, Version 2.0 found in
# the LICENSE.txt file in the root directory of this source tree.

# pylint:disable=abstract-method
"""
Abstract base class for Quantum Channels.
"""

import numpy as np

from qiskit.qiskiterror import QiskitError
from qiskit.quantum_info.operators.base_operator import BaseOperator
from qiskit.quantum_info.operators.operator import Operator
from qiskit.quantum_info.operators.predicates import is_identity_matrix
from qiskit.quantum_info.operators.predicates import is_positive_semidefinite_matrix
from qiskit.quantum_info.operators.channel.transformations import _to_choi
from qiskit.quantum_info.operators.channel.transformations import _to_kraus
from qiskit.quantum_info.operators.channel.transformations import _to_operator

class QuantumChannel(BaseOperator):
    """Quantum channel representation base class."""

    def is_cptp(self, atol=None, rtol=None):
        """Return True if completely-positive trace-preserving (CPTP)."""
        choi = _to_choi(self.rep, self._data, *self.dim)
        return self._is_cp_helper(choi, atol, rtol) and self._is_tp_helper(
            choi, atol, rtol)

    def is_tp(self, atol=None, rtol=None):
        """Test if a channel is completely-positive (CP)"""
        choi = _to_choi(self.rep, self._data, *self.dim)
        return self._is_tp_helper(choi, atol, rtol)

    def is_cp(self, atol=None, rtol=None):
        """Test if Choi-matrix is completely-positive (CP)"""
        choi = _to_choi(self.rep, self._data, *self.dim)
        return self._is_cp_helper(choi, atol, rtol)

    def is_unitary(self, atol=None, rtol=None):
        """Return True if QuantumChannel is a unitary channel."""
        try:
            op = self.to_operator()
            return op.is_unitary(atol=atol, rtol=rtol)
        except QiskitError:
            return False

    def to_operator(self):
        """Try to convert channel to a unitary representation Operator."""
        mat = _to_operator(self.rep, self._data, *self.dim)
        return Operator(mat, self.input_dims(), self.output_dims())

    def to_instruction(self):
        """Convert to a Kraus or UnitaryGate circuit instruction.

        If the channel is unitary it will be added as a unitary gate,
        otherwise it will be added as a kraus simulator instruction.

        Returns:
            Instruction: A kraus instruction for the channel.

        Raises:
            QiskitError: if input data is not an N-qubit CPTP quantum channel.
        """
        from qiskit.circuit.instruction import Instruction
        # Check if input is an N-qubit CPTP channel.
        n_qubits = int(np.log2(self._input_dim))
        if self._input_dim != self._output_dim or 2**n_qubits != self._input_dim:
            raise QiskitError(
                'Cannot convert QuantumChannel to Instruction: channel is not an N-qubit channel.'
            )
        if not self.is_cptp():
            raise QiskitError(
                'Cannot convert QuantumChannel to Instruction: channel is not CPTP.'
            )
        # Next we convert to the Kraus representation. Since channel is CPTP we know
        # that there is only a single set of Kraus operators
        kraus, _ = _to_kraus(self.rep, self._data, *self.dim)
        # If we only have a single Kraus operator then the channel is
        # a unitary channel so can be converted to a UnitaryGate. We do this by
        # converting to an Operator and using its to_instruction method
        if len(kraus) == 1:
            return Operator(kraus[0]).to_instruction()
        return Instruction('kraus', n_qubits, 0, kraus)

    def _is_cp_helper(self, choi, atol, rtol):
        """Test if a channel is completely-positive (CP)"""
        if atol is None:
            atol = self._atol
        if rtol is None:
            rtol = self._rtol
        return is_positive_semidefinite_matrix(choi, rtol=rtol, atol=atol)

    def _is_tp_helper(self, choi, atol, rtol):
        """Test if Choi-matrix is trace-preserving (TP)"""
        if atol is None:
            atol = self._atol
        if rtol is None:
            rtol = self._rtol
        # Check if the partial trace is the identity matrix
        d_in, d_out = self.dim
        mat = np.trace(
            np.reshape(choi, (d_in, d_out, d_in, d_out)), axis1=1, axis2=3)
        return is_identity_matrix(mat, rtol=rtol, atol=atol)

    def _format_state(self, state, density_matrix=False):
        """Format input state so it is statevector or density matrix"""
        state = np.array(state)
        shape = state.shape
        ndim = state.ndim
        if ndim > 2:
            raise QiskitError('Input state is not a vector or matrix.')
        # Flatten column-vector to vector
        if ndim == 2:
            if shape[1] != 1 and shape[1] != shape[0]:
                raise QiskitError('Input state is not a vector or matrix.')
            if shape[1] == 1:
                # flatten colum-vector to vector
                state = np.reshape(state, shape[0])
        # Convert statevector to density matrix if required
        if density_matrix and ndim == 1:
            state = np.outer(state, np.transpose(np.conj(state)))
        return state

    @classmethod
    def _init_transformer(cls, data):
        """Convert input into a QuantumChannel subclass object or Operator object"""
        # This handles common conversion for all QuantumChannel subclasses.
        # If the input is already a QuantumChannel subclass it will return
        # the original object
        if isinstance(data, QuantumChannel):
            return data
        if hasattr(data, 'to_quantumchannel'):
            # If the data object is not a QuantumChannel it will give
            # preference to a 'to_quantumchannel' attribute that allows
            # an arbitrary object to define its own conversion to any
            # quantum channel subclass.
            return data.to_channel()
        if hasattr(data, 'to_channel'):
            # TODO: this 'to_channel' method is the same case as the above
            # but is used by current version of Aer. It should be removed
            # once Aer is nupdated to use `to_quantumchannel`
            # instead of `to_channel`,
            return data.to_channel()
        # Finally if the input is not a QuantumChannel and doesn't have a
        # 'to_quantumchannel' conversion method we try and initialize it as a
        # regular matrix Operator which can be converted into a QuantumChannel.
        return Operator(data)


# -*- coding: utf-8 -*-

# Copyright 2018, IBM.
#
# This source code is licensed under the Apache License, Version 2.0 found in
# the LICENSE.txt file in the root directory of this source tree.

"""
This pass checks if the CNOTs (or any other 2Q) in the DAG follow the right
direction with respect to thecoupling map.
"""

from qiskit.transpiler.basepasses import AnalysisPass
from qiskit.transpiler import Layout
from qiskit.extensions.standard.cx import CnotGate
from qiskit.extensions.standard.cxbase import CXBase

class CheckCnotDirection(AnalysisPass):
    """
    Checks if the CNOTs in the DAG circuit follow the right
    direction with respect to the coupling map.
    """

    def __init__(self, coupling_map, initial_layout=None):
        """
        Checks if the CNOTs in DAGCircuit are in the allowed direction with
        respect to `coupling_map`.
        Args:
            coupling_map (CouplingMap): Directed graph representing a coupling map.
            initial_layout (Layout): The initial layout of the DAG to analyze.
        """
        super().__init__()
        self.layout = initial_layout
        self.coupling_map = coupling_map

    def run(self, dag):
        """
        If `dag` is mapped and the direction is correct the property
        `is_direction_mapped` is set to True (or to False otherwise).

        Args:
            dag (DAGCircuit): DAG to check.
        """
        if self.layout is None:
            if self.property_set["layout"]:
                self.layout = self.property_set["layout"]
            else:
                self.layout = Layout.generate_trivial_layout(*dag.qregs.values())

        self.property_set['is_direction_mapped'] = True
        edges = self.coupling_map.get_edges()

        for gate in dag.twoQ_gates():
            physical_q0 = self.layout[gate.qargs[0]]
            physical_q1 = self.layout[gate.qargs[1]]

            if isinstance(gate.op, (CXBase, CnotGate)) and (
                    physical_q0, physical_q1) not in edges:
                self.property_set['is_direction_mapped'] = False
                return


from sqlalchemy.exc import ArgumentError
from sqlalchemy.inspection import inspect
from sqlalchemy.orm import class_mapper, object_mapper
from sqlalchemy.orm.exc import UnmappedClassError, UnmappedInstanceError

from graphene import Argument, Enum, List

def get_session(context):
    return context.get("session")

def get_query(model, context):
    query = getattr(model, "query", None)
    if not query:
        session = get_session(context)
        if not session:
            raise Exception(
                "A query in the model Base or a session in the schema is required for querying.\n"
                "Read more http://docs.graphene-python.org/projects/sqlalchemy/en/latest/tips/#querying"
            )
        query = session.query(model)
    return query

def is_mapped_class(cls):
    try:
        class_mapper(cls)
    except (ArgumentError, UnmappedClassError):
        return False
    else:
        return True

def is_mapped_instance(cls):
    try:
        object_mapper(cls)
    except (ArgumentError, UnmappedInstanceError):
        return False
    else:
        return True

def _symbol_name(column_name, is_asc):
    return column_name + ("_asc" if is_asc else "_desc")

class EnumValue(str):
    """Subclass of str that stores a string and an arbitrary value in the "value" property"""

    def __new__(cls, str_value, value):
        return super(EnumValue, cls).__new__(cls, str_value)

    def __init__(self, str_value, value):
        super(EnumValue, self).__init__()
        self.value = value

# Cache for the generated enums, to avoid name clash
_ENUM_CACHE = {}

def _sort_enum_for_model(cls, name=None, symbol_name=_symbol_name):
    name = name or cls.__name__ + "SortEnum"
    if name in _ENUM_CACHE:
        return _ENUM_CACHE[name]
    items = []
    default = []
    for column in inspect(cls).columns.values():
        asc_name = symbol_name(column.name, True)
        asc_value = EnumValue(asc_name, column.asc())
        desc_name = symbol_name(column.name, False)
        desc_value = EnumValue(desc_name, column.desc())
        if column.primary_key:
            default.append(asc_value)
        items.extend(((asc_name, asc_value), (desc_name, desc_value)))
    enum = Enum(name, items)
    _ENUM_CACHE[name] = (enum, default)
    return enum, default

def sort_enum_for_model(cls, name=None, symbol_name=_symbol_name):
    """Create Graphene Enum for sorting a SQLAlchemy class query

    Parameters
    - cls : Sqlalchemy model class
        Model used to create the sort enumerator
    - name : str, optional, default None
        Name to use for the enumerator. If not provided it will be set to `cls.__name__ + 'SortEnum'`
    - symbol_name : function, optional, default `_symbol_name`
        Function which takes the column name and a boolean indicating if the sort direction is ascending,
        and returns the symbol name for the current column and sort direction.
        The default function will create, for a column named 'foo', the symbols 'foo_asc' and 'foo_desc'

    Returns
    - Enum
        The Graphene enumerator
    """
    enum, _ = _sort_enum_for_model(cls, name, symbol_name)
    return enum

def sort_argument_for_model(cls, has_default=True):
    """Returns a Graphene argument for the sort field that accepts a list of sorting directions for a model.
    If `has_default` is True (the default) it will sort the result by the primary key(s)
    """
    enum, default = _sort_enum_for_model(cls)
    if not has_default:
        default = None
    return Argument(List(enum), default_value=default)


# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from dateparser.search.search import DateSearchWithDetection
from dateparser.utils import normalize_unicode

_search_with_detection = DateSearchWithDetection()

def search_dates(text, languages=None, settings=None, add_detected_language=False):
    """Find all substrings of the given string which represent date and/or time and parse them.

        :param text:
            A string in a natural language which may contain date and/or time expressions.
        :type text: str|unicode

        :param languages:
            A list of two letters language codes.e.g. ['en', 'es']. If languages are given, it will
            not attempt to detect the language.
        :type languages: list

        :param settings:
               Configure customized behavior using settings defined in :mod:`dateparser.conf.Settings`.
        :type settings: dict

        :param add_detected_language:
               Indicates if we want the detected language returned in the tuple.
        :type add_detected_language: bool

        :return: Returns list of tuples containing:
                 substrings representing date and/or time, corresponding :mod:`datetime.datetime`
                 object and detected language if *add_detected_language* is True.
                 Returns None if no dates that can be parsed are found.
        :rtype: list
        :raises: ValueError - Unknown Language

        >>> from dateparser.search import search_dates
        >>> search_dates('The first artificial Earth satellite was launched on 4 October 1957.')
        [('on 4 October 1957', datetime.datetime(1957, 10, 4, 0, 0))]

        >>> search_dates('The first artificial Earth satellite was launched on 4 October 1957.', add_detected_language=True)
        [('on 4 October 1957', datetime.datetime(1957, 10, 4, 0, 0), 'en')]

        >>> search_dates("The client arrived to the office for the first time in March 3rd, 2004 and got serviced, after a couple of months, on May 6th 2004, the customer returned indicating a defect on the part")
        [('in March 3rd, 2004 and', datetime.datetime(2004, 3, 3, 0, 0)),
         ('on May 6th 2004', datetime.datetime(2004, 5, 6, 0, 0))]

        """
    result = _search_with_detection.search_dates(
        text=text, languages=languages, settings=settings
    )
    language, dates = result.get('Language'), result.get('Dates')
    if dates:
        if add_detected_language:
            dates = [date + (language, ) for date in dates]
        return dates


import imp
import regex as re

from datetime import datetime

TIME_MATCHER = re.compile(
    r'.*?'
    r'(?P<hour>2[0-3]|[0-1]\d|\d):'
    '(?P<minute>[0-5]\d|\d):'
    '(?P<second>6[0-1]|[0-5]\d|\d)'
    '\.(?P<microsecond>[0-9]{1,6})'
)

MS_SEARCHER = re.compile(r'\.(?P<microsecond>[0-9]{1,6})')

def patch_strptime():
    """Monkey patching _strptime to avoid problems related with non-english
    locale changes on the system.

    For example, if system's locale is set to fr_FR. Parser won't recognize
    any date since all languages are translated to english dates.
    """

    _strptime = imp.load_module(
        'strptime_patched', *imp.find_module('_strptime')
    )

    _calendar = imp.load_module(
        'calendar_patched', *imp.find_module('_strptime')
    )

    _strptime._getlang = lambda: ('en_US', 'UTF-8')
    _strptime.calendar = _calendar
    _strptime.calendar.day_abbr = [
        'mon', 'tue', 'wed', 'thu', 'fri', 'sat', 'sun'
    ]
    _strptime.calendar.day_name = [
        'monday', 'tuesday', 'wednesday', 'thursday',
        'friday', 'saturday', 'sunday'
    ]
    _strptime.calendar.month_abbr = [
        '', 'jan', 'feb', 'mar', 'apr', 'may', 'jun',
        'jul', 'aug', 'sep', 'oct', 'nov', 'dec'
    ]
    _strptime.calendar.month_name = [
        '', 'january', 'february', 'march', 'april',
        'may', 'june', 'july', 'august', 'september',
        'october', 'november', 'december'
    ]

    return _strptime._strptime_time

__strptime = patch_strptime()

def strptime(date_string, format):
    obj = datetime(*__strptime(date_string, format)[:-3])

    if '%f' in format:
        try:
            match_groups = TIME_MATCHER.match(date_string).groupdict()
            ms = match_groups['microsecond']
            ms = ms + ((6 - len(ms)) * '0')
            obj = obj.replace(microsecond=int(ms))
        except AttributeError:
            match_groups = MS_SEARCHER.search(date_string).groupdict()
            ms = match_groups['microsecond']
            ms = ms + ((6 - len(ms)) * '0')
            obj = obj.replace(microsecond=int(ms))

    return obj


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from collections import OrderedDict
from importlib import import_module
from six.moves import zip_longest
import regex as re
from copy import deepcopy

from ..data import language_order, language_locale_dict
from .locale import Locale
from ..utils import convert_to_unicode

LOCALE_SPLIT_PATTERN = re.compile(r'-(?=[A-Z0-9]+$)')

def _isvalidlocale(locale):
    language = LOCALE_SPLIT_PATTERN.split(locale)[0]
    if language not in language_order:
        return False
    else:
        locales_list = language_locale_dict[language]
        if locale == language or locale in locales_list:
            return True
        else:
            return False

def _filter_valid_locales(locales):
    return [locale for locale in locales if _isvalidlocale(locale)]

def _construct_locales(languages, region):
    if region:
        possible_locales = [language + '-' + region for language in languages]
        locales = _filter_valid_locales(possible_locales)
    else:
        locales = languages
    return locales

class LocaleDataLoader(object):
    """Class that handles loading of locale instances."""

    _loaded_languages = {}
    _loaded_locales = {}

    def get_locale_map(self, languages=None, locales=None, region=None,
                       use_given_order=False, allow_conflicting_locales=False):
        """
        Get an ordered mapping with locale codes as keys
        and corresponding locale instances as values.

        :param languages:
            A list of language codes, e.g. ['en', 'es', 'zh-Hant'].
            If locales are not given, languages and region are
            used to construct locales to load.
        :type languages: list

        :param locales:
            A list of codes of locales which are to be loaded,
            e.g. ['fr-PF', 'qu-EC', 'af-NA']
        :type locales: list

        :param region:
            A region code, e.g. 'IN', '001', 'NE'.
            If locales are not given, languages and region are
            used to construct locales to load.
        :type region: str|unicode

        :param use_given_order:
            If True, the returned mapping is ordered in the order locales are given.
        :type allow_redetect_language: bool

        :param allow_conflicting_locales:
            if True, locales with same language and different region can be loaded.
        :type allow_conflicting_locales: bool

        :return: ordered locale code to locale instance mapping
        """
        return OrderedDict(self._load_data(
            languages=languages, locales=locales, region=region, use_given_order=use_given_order,
            allow_conflicting_locales=allow_conflicting_locales))

    def get_locales(self, languages=None, locales=None, region=None,
                    use_given_order=False, allow_conflicting_locales=False):
        """
        Yield locale instances.

        :param languages:
            A list of language codes, e.g. ['en', 'es', 'zh-Hant'].
            If locales are not given, languages and region are
            used to construct locales to load.
        :type languages: list

        :param locales:
            A list of codes of locales which are to be loaded,
            e.g. ['fr-PF', 'qu-EC', 'af-NA']
        :type locales: list

        :param region:
            A region code, e.g. 'IN', '001', 'NE'.
            If locales are not given, languages and region are
            used to construct locales to load.
        :type region: str|unicode

        :param use_given_order:
            If True, the returned mapping is ordered in the order locales are given.
        :type allow_redetect_language: bool

        :param allow_conflicting_locales:
            if True, locales with same language and different region can be loaded.
        :type allow_conflicting_locales: bool

        :yield: locale instances
        """
        for _, locale in self._load_data(
                languages=languages, locales=locales, region=region,
                use_given_order=use_given_order,
                allow_conflicting_locales=allow_conflicting_locales):
            yield locale

    def get_locale(self, shortname):
        """
        Get a locale instance.

        :param shortname:
            A locale code, e.g. 'fr-PF', 'qu-EC', 'af-NA'.
        :type shortname: str|unicode

        :return: locale instance
        """
        return list(self.get_locales(locales=[shortname]))[0]

    def _load_data(self, languages=None, locales=None, region=None,
                   use_given_order=False, allow_conflicting_locales=False):
        locale_dict = OrderedDict()
        if locales:
            invalid_locales = []
            for locale in locales:
                lang_reg = LOCALE_SPLIT_PATTERN.split(locale)
                if len(lang_reg) == 1:
                    lang_reg.append('')
                locale_dict[locale] = tuple(lang_reg)
                if not _isvalidlocale(locale):
                    invalid_locales.append(locale)
            if invalid_locales:
                raise ValueError("Unknown locale(s): %s"
                                 % ', '.join(map(repr, invalid_locales)))

            if not allow_conflicting_locales:
                if len(set(locales)) > len(set([t[0] for t in locale_dict.values()])):
                    raise ValueError("Locales should not have same language and different region")

        else:
            if languages is None:
                languages = language_order
            unsupported_languages = set(languages) - set(language_order)
            if unsupported_languages:
                raise ValueError("Unknown language(s): %s"
                                 % ', '.join(map(repr, unsupported_languages)))
            if region is None:
                region = ''
            locales = _construct_locales(languages, region)
            locale_dict.update(zip_longest(locales,
                               tuple(zip_longest(languages, [], fillvalue=region))))

        if not use_given_order:
            locale_dict = OrderedDict(sorted(locale_dict.items(),
                                      key=lambda x: language_order.index(x[1][0])))

        for shortname, lang_reg in locale_dict.items():
            if shortname not in self._loaded_locales:
                lang, reg = lang_reg
                if lang in self._loaded_languages:
                    locale = Locale(shortname, language_info=deepcopy(self._loaded_languages[lang]))
                    self._loaded_locales[shortname] = locale
                else:
                    language_info = getattr(
                        import_module('dateparser.data.date_translation_data.' + lang), 'info')
                    language_info = convert_to_unicode(language_info)
                    locale = Locale(shortname, language_info=deepcopy(language_info))
                    self._loaded_languages[lang] = language_info
                    self._loaded_locales[shortname] = locale
            yield shortname, self._loaded_locales[shortname]

default_loader = LocaleDataLoader()


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from itertools import chain
from operator import methodcaller
import regex as re
from six.moves import zip_longest

from dateparser.utils import normalize_unicode

PARSER_HARDCODED_TOKENS = [":", ".", " ", "-", "/"]
PARSER_KNOWN_TOKENS = ["am", "pm", "UTC", "GMT", "Z"]
ALWAYS_KEEP_TOKENS = ["+"] + PARSER_HARDCODED_TOKENS
KNOWN_WORD_TOKENS = ['monday', 'tuesday', 'wednesday', 'thursday', 'friday',
                     'saturday', 'sunday', 'january', 'february', 'march',
                     'april', 'may', 'june', 'july', 'august', 'september',
                     'october', 'november', 'december', 'year', 'month', 'week',
                     'day', 'hour', 'minute', 'second', 'ago', 'in', 'am', 'pm']

PARENTHESES_PATTERN = re.compile(r'[\(\)]')
NUMERAL_PATTERN = re.compile(r'(\d+)')
KEEP_TOKEN_PATTERN = re.compile(r"^.*[^\W_].*$", flags=re.U)

class UnknownTokenError(Exception):
    pass

class Dictionary(object):
    """
    Class that modifies and stores translations and handles splitting of date string.

    :param locale_info:
        Locale info (translation data) of the locale.
    :type language_info: dict

    :param settings:
        Configure customized behavior using settings defined in :mod:`dateparser.conf.Settings`.
    :type settings: dict

    :return: a Dictionary instance.
    """

    _split_regex_cache = {}
    _sorted_words_cache = {}
    _split_relative_regex_cache = {}
    _sorted_relative_strings_cache = {}
    _match_relative_regex_cache = {}

    def __init__(self, locale_info, settings=None):
        dictionary = {}
        self._settings = settings
        self.info = locale_info

        if 'skip' in locale_info:
            skip = map(methodcaller('lower'), locale_info['skip'])
            dictionary.update(zip_longest(skip, [], fillvalue=None))
        if 'pertain' in locale_info:
            pertain = map(methodcaller('lower'), locale_info['pertain'])
            dictionary.update(zip_longest(pertain, [], fillvalue=None))
        for word in KNOWN_WORD_TOKENS:
            if word in locale_info:
                translations = map(methodcaller('lower'), locale_info[word])
                dictionary.update(zip_longest(translations, [], fillvalue=word))
        dictionary.update(zip_longest(ALWAYS_KEEP_TOKENS, ALWAYS_KEEP_TOKENS))
        dictionary.update(zip_longest(map(methodcaller('lower'),
                                          PARSER_KNOWN_TOKENS),
                                          PARSER_KNOWN_TOKENS))

        relative_type = locale_info.get('relative-type', {})
        for key, value in relative_type.items():
            relative_translations = map(methodcaller('lower'), value)
            dictionary.update(zip_longest(relative_translations, [], fillvalue=key))

        self._dictionary = dictionary

        no_word_spacing = locale_info.get('no_word_spacing', 'False')
        self._no_word_spacing = bool(eval(no_word_spacing))

        relative_type_regex = locale_info.get("relative-type-regex", {})
        self._relative_strings = list(chain(*relative_type_regex.values()))

    def __contains__(self, key):
        if key in self._settings.SKIP_TOKENS:
            return True
        return self._dictionary.__contains__(key)

    def __getitem__(self, key):
        if key in self._settings.SKIP_TOKENS:
            return None
        return self._dictionary.__getitem__(key)

    def __iter__(self):
        return chain(self._settings.SKIP_TOKENS, iter(self._dictionary))

    def are_tokens_valid(self, tokens):
        """
        Check if tokens are valid tokens for the locale.

        :param tokens:
            a list of string or unicode tokens.
        :type tokens: list

        :return: True if tokens are valid, False otherwise.
        """
        match_relative_regex = self._get_match_relative_regex_cache()
        for token in tokens:
            if any([match_relative_regex.match(token),
                    token in self, token.isdigit()]):
                continue
            else:
                return False
        else:
            return True

    def split(self, string, keep_formatting=False):
        """
        Split the date string using translations in locale info.

        :param string:
            Date string to be splitted.
        :type string:
            str|unicode

        :param keep_formatting:
            If True, retain formatting of the date string.
        :type keep_formatting: bool

        :return: A list of string tokens formed after splitting the date string.
        """
        if not string:
            return string

        split_relative_regex = self._get_split_relative_regex_cache()
        match_relative_regex = self._get_match_relative_regex_cache()

        tokens = split_relative_regex.split(string)

        for i, token in enumerate(tokens):
            if match_relative_regex.match(token):
                tokens[i] = [token]
                continue
            tokens[i] = self._split_by_known_words(token, keep_formatting)

        return list(filter(bool, chain(*tokens)))

    def _split_by_known_words(self, string, keep_formatting):
        if not string:
            return string

        regex = self._get_split_regex_cache()
        match = regex.match(string)
        if not match:
            return (self._split_by_numerals(string, keep_formatting)
                    if self._should_capture(string, keep_formatting) else [])

        unparsed, known, unknown = match.groups()
        splitted = [known] if self._should_capture(known, keep_formatting) else []
        if unparsed and self._should_capture(unparsed, keep_formatting):
            splitted = self._split_by_numerals(unparsed, keep_formatting) + splitted
        if unknown:
            splitted.extend(self._split_by_known_words(unknown, keep_formatting))

        return splitted

    def _split_by_numerals(self, string, keep_formatting):
        return [token for token in NUMERAL_PATTERN.split(string)
                if self._should_capture(token, keep_formatting)]

    def _should_capture(self, token, keep_formatting):
        return (
            keep_formatting or
            (token in ALWAYS_KEEP_TOKENS) or
            KEEP_TOKEN_PATTERN.match(token)
        )

    def _get_sorted_words_from_cache(self):
        if (
                self._settings.registry_key not in self._sorted_words_cache or
                self.info['name'] not in self._sorted_words_cache[self._settings.registry_key]
           ):
            self._sorted_words_cache.setdefault(
                self._settings.registry_key, {})[self.info['name']] = \
                sorted([key for key in self], key=len, reverse=True)
        return self._sorted_words_cache[self._settings.registry_key][self.info['name']]

    def _get_split_regex_cache(self):
        if (
                self._settings.registry_key not in self._split_regex_cache or
                self.info['name'] not in self._split_regex_cache[self._settings.registry_key]
           ):
            self._construct_split_regex()
        return self._split_regex_cache[self._settings.registry_key][self.info['name']]

    def _construct_split_regex(self):
        known_words_group = "|".join(map(re.escape, self._get_sorted_words_from_cache()))
        if self._no_word_spacing:
            regex = r"^(.*?)({})(.*)$".format(known_words_group)
        else:
            regex = r"^(.*?(?:\A|\W|_|\d))({})((?:\Z|\W|_|\d).*)$".format(known_words_group)
        self._split_regex_cache.setdefault(
            self._settings.registry_key, {})[self.info['name']] = \
            re.compile(regex, re.UNICODE | re.IGNORECASE)

    def _get_sorted_relative_strings_from_cache(self):
        if (
            self._settings.registry_key not in self._sorted_relative_strings_cache or
            self.info['name'] not in self._sorted_relative_strings_cache[self._settings.registry_key]
           ):

            self._sorted_relative_strings_cache.setdefault(
                self._settings.registry_key, {})[self.info['name']] = \
                sorted([PARENTHESES_PATTERN.sub('', key) for key in
                        self._relative_strings], key=len, reverse=True)
        return self._sorted_relative_strings_cache[self._settings.registry_key][self.info['name']]

    def _get_split_relative_regex_cache(self):
        if (
            self._settings.registry_key not in self._split_relative_regex_cache or
            self.info['name'] not in self._split_relative_regex_cache[self._settings.registry_key]
           ):

            self._construct_split_relative_regex()
        return self._split_relative_regex_cache[self._settings.registry_key][self.info['name']]

    def _construct_split_relative_regex(self):
        known_relative_strings_group = "|".join(self._get_sorted_relative_strings_from_cache())
        if self._no_word_spacing:
            regex = "({})".format(known_relative_strings_group)
        else:
            regex = "(?<=(?:\\A|\\W|_))({})(?=(?:\\Z|\\W|_))".format(known_relative_strings_group)
        self._split_relative_regex_cache.setdefault(
            self._settings.registry_key, {})[self.info['name']] = \
            re.compile(regex, re.UNICODE | re.IGNORECASE)

    def _get_match_relative_regex_cache(self):
        if (
            self._settings.registry_key not in self._match_relative_regex_cache or
            self.info['name'] not in self._match_relative_regex_cache[self._settings.registry_key]
           ):

            self._construct_match_relative_regex()
        return self._match_relative_regex_cache[self._settings.registry_key][self.info['name']]

    def _construct_match_relative_regex(self):
        known_relative_strings_group = "|".join(self._get_sorted_relative_strings_from_cache())
        regex = "^({})$".format(known_relative_strings_group)
        self._match_relative_regex_cache.setdefault(
            self._settings.registry_key, {})[self.info['name']] = \
            re.compile(regex, re.UNICODE | re.IGNORECASE)

class NormalizedDictionary(Dictionary):

    def __init__(self, locale_info, settings=None):
        super(NormalizedDictionary, self).__init__(locale_info, settings)
        self._normalize()

    def _normalize(self):
        new_dict = {}
        conflicting_keys = []
        for key, value in self._dictionary.items():
            normalized = normalize_unicode(key)
            if key != normalized and normalized in self._dictionary:
                conflicting_keys.append(key)
            else:
                new_dict[normalized] = value
        for key in conflicting_keys:
            normalized = normalize_unicode(key)
            if key in (self.info.get('skip', []) + self.info.get('pertain', [])):
                new_dict[normalized] = self._dictionary[key]
        self._dictionary = new_dict
        self._relative_strings = list(map(normalize_unicode, self._relative_strings))


# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from dateparser.languages.loader import LocaleDataLoader
from dateparser.conf import apply_settings, Settings
from dateparser.date import DateDataParser
from dateparser.search.text_detection import FullTextLanguageDetector
import regex as re

try:
    # Python 3
    from collections.abc import Set
except ImportError:
    # Python 2.7
    from collections import Set

RELATIVE_REG = re.compile("(ago|in|from now|tomorrow|today|yesterday)")

def date_is_relative(translation):
    if re.search(RELATIVE_REG, translation):

        return True
    else:
        return False

class ExactLanguageSearch:
    def __init__(self, loader):
        self.loader = loader
        self.language = None

    def get_current_language(self, shortname):
        if self.language is None or self.language.shortname != shortname:
            self.language = self.loader.get_locale(shortname)

    def search(self, shortname, text, settings):
        self.get_current_language(shortname)
        result = self.language.translate_search(text, settings=settings)
        return result

    @staticmethod
    def set_relative_base(substring, already_parsed):
        if len(already_parsed) == 0:
            return substring, None
        else:
            i = len(already_parsed) - 1
            while already_parsed[i]['is_relative']:
                i -= 1
                if i == -1:
                    return substring, None
            relative_base = already_parsed[i]['date_obj']
            return substring,  relative_base

    def choose_best_split(self, possible_parsed_splits, possible_substrings_splits):
        rating = []
        for i in range(len(possible_parsed_splits)):
            num_substrings = len(possible_substrings_splits[i])
            num_substrings_without_digits = 0
            not_parsed = 0
            for j, item in enumerate(possible_parsed_splits[i]):
                if item['date_obj'] is None:
                    not_parsed += 1
                if not any(char.isdigit() for char in possible_substrings_splits[i][j]):
                    num_substrings_without_digits += 1
            rating.append([
                num_substrings,
                0 if not_parsed == 0 else (float(not_parsed)/float(num_substrings)),
                0 if num_substrings_without_digits == 0 else (
                    float(num_substrings_without_digits)/float(num_substrings))])
            best_index, best_rating = min(enumerate(rating), key=lambda p: (p[1][1], p[1][0], p[1][2]))
        return possible_parsed_splits[best_index], possible_substrings_splits[best_index]

    def split_by(self, item, original, splitter):
        if item.count(splitter) <= 2:
            return [[item.split(splitter), original.split(splitter)]]
        else:
            item_all_split = item.split(splitter)
            original_all_split = original.split(splitter)
            all_possible_splits = [[item_all_split, original_all_split]]
            for i in range(2, 4):
                item_partially_split = []
                original_partially_split = []
                for j in range(0, len(item_all_split), i):
                    item_join = splitter.join(item_all_split[j:j+i])
                    original_join = splitter.join(original_all_split[j:j+i])
                    item_partially_split.append(item_join)
                    original_partially_split.append(original_join)
                all_possible_splits.append([item_partially_split, original_partially_split])
            return all_possible_splits

    def split_if_not_parsed(self, item, original):
        splitters = [',', '،', '——', '—', '–', '.', ' ']
        possible_splits = []
        for splitter in splitters:
            if splitter in item and item.count(splitter) == original.count(splitter):
                possible_splits.extend(self.split_by(item, original, splitter))
        return possible_splits

    def parse_item(self, parser, item, translated_item, parsed, need_relative_base):
        item = item.replace('ngày', '')
        item = item.replace('am', '')
        pre_parsed_item = parser.get_date_data(item)
        is_relative = date_is_relative(translated_item)
        if need_relative_base:
            item, relative_base = self.set_relative_base(item, parsed)
        else:
            relative_base = None

        if relative_base:
            parser._settings.RELATIVE_BASE = relative_base
            parsed_item = parser.get_date_data(item)
        else:
            parsed_item = pre_parsed_item
        parsed_item['is_relative'] = is_relative
        return parsed_item

    def parse_found_objects(self, parser, to_parse, original, translated, settings):
        parsed = []
        substrings = []
        need_relative_base = True
        if settings.RELATIVE_BASE:
            need_relative_base = False
        for i, item in enumerate(to_parse):
            if len(item) > 2:
                parsed_item = self.parse_item(parser, item, translated[i], parsed, need_relative_base)
                if parsed_item['date_obj']:
                    parsed.append(parsed_item)
                    substrings.append(original[i].strip(' .,:()[]-'))
                    pass
                else:
                    possible_splits = self.split_if_not_parsed(item, original[i])
                    if possible_splits:
                        possible_parsed = []
                        possible_substrings = []
                        for split_translated, split_original in possible_splits:
                            current_parsed = []
                            current_substrings = []
                            if split_translated:
                                for j, jtem in enumerate(split_translated):
                                    if len(jtem) > 2:
                                        parsed_jtem = self.parse_item(parser, jtem, split_translated[j],
                                                                      current_parsed, need_relative_base)
                                        current_parsed.append(parsed_jtem)
                                        current_substrings.append(split_original[j].strip(' .,:()[]-'))
                            else:
                                pass
                            possible_parsed.append(current_parsed)
                            possible_substrings.append(current_substrings)
                        parsed_best, substrings_best = self.choose_best_split(possible_parsed, possible_substrings)
                        for k in range(len(parsed_best)):
                            if parsed_best[k]['date_obj']:
                                parsed.append(parsed_best[k])
                                substrings.append(substrings_best[k])
        return parsed, substrings

    def search_parse(self, shortname, text, settings):
        translated, original = self.search(shortname, text, settings)
        bad_translate_with_search = ['vi', 'hu']   # splitting done by spaces and some dictionary items contain spaces
        if shortname not in bad_translate_with_search:
            parser = DateDataParser(languages=['en'], settings=settings)
            parsed, substrings = self.parse_found_objects(parser=parser, to_parse=translated,
                                                          original=original, translated=translated, settings=settings)
        else:
            parser = DateDataParser(languages=[shortname], settings=settings)
            parsed, substrings = self.parse_found_objects(parser=parser, to_parse=original,
                                                          original=original, translated=translated, settings=settings)
        parser._settings = Settings()
        return list(zip(substrings, [i['date_obj'] for i in parsed]))

class DateSearchWithDetection:
    """
    Class which executes language detection of string in a natural language, translation of a given string,
    search of substrings which represent date and/or time and parsing of these substrings.

    """
    def __init__(self):
        self.loader = LocaleDataLoader()
        self.available_language_map = self.loader.get_locale_map()
        self.search = ExactLanguageSearch(self.loader)

    def detect_language(self, text, languages):
        if isinstance(languages, (list, tuple, Set)):

            if all([language in self.available_language_map for language in languages]):
                languages = [self.available_language_map[language] for language in languages]
            else:
                unsupported_languages = set(languages) - set(self.available_language_map.keys())
                raise ValueError(
                    "Unknown language(s): %s" % ', '.join(map(repr, unsupported_languages)))
        elif languages is not None:
            raise TypeError("languages argument must be a list (%r given)" % type(languages))

        if languages:
            self.language_detector = FullTextLanguageDetector(languages=languages)
        else:
            self.language_detector = FullTextLanguageDetector(list(self.available_language_map.values()))

        return self.language_detector._best_language(text)

    @apply_settings
    def search_dates(self, text, languages=None, settings=None):
        """
        Find all substrings of the given string which represent date and/or time and parse them.

        :param text:
            A string in a natural language which may contain date and/or time expressions.
        :type text: str|unicode
        :param languages:
            A list of two letters language codes.e.g. ['en', 'es']. If languages are given, it will not attempt
            to detect the language.
        :type languages: list
        :param settings:
               Configure customized behavior using settings defined in :mod:`dateparser.conf.Settings`.
        :type settings: dict

        :return: a dict mapping keys to two letter language code and a list of tuples of pairs:
                substring representing date expressions and corresponding :mod:`datetime.datetime` object.
            For example:
            {'Language': 'en', 'Dates': [('on 4 October 1957', datetime.datetime(1957, 10, 4, 0, 0))]}
            If language of the string isn't recognised returns:
            {'Language': None, 'Dates': None}
        :raises: ValueError - Unknown Language
        """

        language_shortname = self.detect_language(text=text, languages=languages)
        if not language_shortname:
            return {'Language': None, 'Dates': None}
        return {'Language': language_shortname, 'Dates': self.search.search_parse(language_shortname, text,
                                                                                settings=settings)}


# -*- coding: utf-8 -*-
__version__ = '0.7.1'

from .date import DateDataParser
from .conf import apply_settings

_default_parser = DateDataParser()

@apply_settings
def parse(date_string, date_formats=None, languages=None, locales=None, region=None, settings=None):
    """Parse date and time from given date string.

    :param date_string:
        A string representing date and/or time in a recognizably valid format.
    :type date_string: str|unicode

    :param date_formats:
        A list of format strings using directives as given
        `here <https://docs.python.org/2/library/datetime.html#strftime-and-strptime-behavior>`_.
        The parser applies formats one by one, taking into account the detected languages/locales.
    :type date_formats: list

    :param languages:
        A list of language codes, e.g. ['en', 'es', 'zh-Hant'].
        If locales are not given, languages and region are used to construct locales for translation.
    :type languages: list

    :param locales:
        A list of locale codes, e.g. ['fr-PF', 'qu-EC', 'af-NA'].
        The parser uses locales to translate date string.
    :type locales: list

    :param region:
        A region code, e.g. 'IN', '001', 'NE'.
        If locales are not given, languages and region are used to construct locales for translation.
    :type region: str|unicode

    :param settings:
        Configure customized behavior using settings defined in :mod:`dateparser.conf.Settings`.
    :type settings: dict

    :return: Returns :class:`datetime <datetime.datetime>` representing parsed date if successful, else returns None
    :rtype: :class:`datetime <datetime.datetime>`.
    :raises: ValueError - Unknown Language
    """
    parser = _default_parser

    if any([languages, locales, region, not settings._default]):
        parser = DateDataParser(languages=languages, locales=locales,
                                region=region, settings=settings)

    data = parser.get_date_data(date_string, date_formats)

    if data:
        return data['date_obj']


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import regex as re
from datetime import datetime
from datetime import time
from tzlocal import get_localzone

from dateutil.relativedelta import relativedelta

from dateparser.utils import apply_timezone, localize_timezone, strip_braces
from .parser import time_parser
from .timezone_parser import pop_tz_offset_from_string

_UNITS = r'year|month|week|day|hour|minute|second'
PATTERN = re.compile(r'(\d+)\s*(%s)\b' % _UNITS, re.I | re.S | re.U)

class FreshnessDateDataParser(object):
    """ Parses date string like "1 year, 2 months ago" and "3 hours, 50 minutes ago" """
    def __init__(self):
        self.now = None

    def _are_all_words_units(self, date_string):
        skip = [_UNITS,
                r'ago|in|\d+',
                r':|[ap]m']

        date_string = re.sub(r'\s+', ' ', date_string.strip())

        words = filter(lambda x: x if x else False, re.split(r'\W', date_string))
        words = filter(lambda x: not re.match(r'%s' % '|'.join(skip), x), words)
        return not list(words)

    def _parse_time(self, date_string, settings):
        """Attemps to parse time part of date strings like '1 day ago, 2 PM' """
        date_string = PATTERN.sub('', date_string)
        date_string = re.sub(r'\b(?:ago|in)\b', '', date_string)
        try:
            return time_parser(date_string)
        except:
            pass

    def get_local_tz(self):
        return get_localzone()

    def parse(self, date_string, settings):

        _time = self._parse_time(date_string, settings)

        date_string = strip_braces(date_string)
        date_string, ptz = pop_tz_offset_from_string(date_string)

        _settings_tz = settings.TIMEZONE.lower()

        def apply_time(dateobj, timeobj):
            if not isinstance(_time, time):
                return dateobj

            return dateobj.replace(
                hour=timeobj.hour, minute=timeobj.minute,
                second=timeobj.second, microsecond=timeobj.microsecond
            )

        if settings.RELATIVE_BASE:
            self.now = settings.RELATIVE_BASE

            if 'local' not in _settings_tz:
                self.now = localize_timezone(self.now, settings.TIMEZONE)

            if ptz:
                if self.now.tzinfo:
                    self.now = self.now.astimezone(ptz)
                else:
                    self.now = ptz.localize(self.now)

            if not self.now.tzinfo:
                self.now = self.get_local_tz().localize(self.now)

        elif ptz:
            _now = datetime.now(ptz)

            if 'local' in _settings_tz:
                self.now = _now
            else:
                self.now = apply_timezone(_now, settings.TIMEZONE)

        else:
            if 'local' not in _settings_tz:
                utc_dt = datetime.utcnow()
                self.now = apply_timezone(utc_dt, settings.TIMEZONE)
            else:
                self.now = datetime.now(self.get_local_tz())

        date, period = self._parse_date(date_string)

        if date:
            date = apply_time(date, _time)
            if settings.TO_TIMEZONE:
                date = apply_timezone(date, settings.TO_TIMEZONE)

            if (
                not settings.RETURN_AS_TIMEZONE_AWARE or
                (settings.RETURN_AS_TIMEZONE_AWARE and
                 'default' == settings.RETURN_AS_TIMEZONE_AWARE and not ptz)
            ):
                date = date.replace(tzinfo=None)

        self.now = None
        return date, period

    def _parse_date(self, date_string):
        if not self._are_all_words_units(date_string):
            return None, None

        kwargs = self.get_kwargs(date_string)
        if not kwargs:
            return None, None

        period = 'day'
        if 'days' not in kwargs:
            for k in ['weeks', 'months', 'years']:
                if k in kwargs:
                    period = k[:-1]
                    break

        td = relativedelta(**kwargs)
        if re.search(r'\bin\b', date_string):
            date = self.now + td
        else:
            date = self.now - td
        return date, period

    def get_kwargs(self, date_string):
        m = PATTERN.findall(date_string)
        if not m:
            return {}

        kwargs = {}
        for num, unit in m:
            kwargs[unit + 's'] = int(num)

        return kwargs

    def get_date_data(self, date_string, settings=None):
        date, period = self.parse(date_string, settings)
        return dict(date_obj=date, period=period)

freshness_date_parser = FreshnessDateDataParser()


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from itertools import chain

import regex as re
from collections import OrderedDict

from dateutil import parser

from dateparser.timezone_parser import pop_tz_offset_from_string
from dateparser.utils import normalize_unicode, combine_dicts

from .dictionary import Dictionary, NormalizedDictionary, ALWAYS_KEEP_TOKENS

DIGIT_GROUP_PATTERN = re.compile(r'\\d\+')
NUMERAL_PATTERN = re.compile(r'(\d+)', re.U)

class Locale(object):
    """
    Class that deals with applicability and translation from a locale.

    :param shortname:
        A locale code, e.g. 'fr-PF', 'qu-EC', 'af-NA'.
    :type shortname: str|unicode

    :param language_info:
        Language info (translation data) of the language the locale belongs to.
    :type language_info: dict

    :return: A Locale instance
    """

    _dictionary = None
    _normalized_dictionary = None
    _simplifications = None
    _normalized_simplifications = None
    _splitters = None
    _wordchars = None
    _relative_translations = None
    _normalized_relative_translations = None
    _abbreviations = None
    _split_dictionary = None
    _wordchars_for_detection = None

    def __init__(self, shortname, language_info):
        self.shortname = shortname
        locale_specific_info = language_info.get("locale_specific", {}).get(shortname, {})
        self.info = combine_dicts(language_info, locale_specific_info)
        self.info.pop("locale_specific", None)

    def is_applicable(self, date_string, strip_timezone=False, settings=None):
        """
        Check if the locale is applicable to translate date string.

        :param date_string:
            A string representing date and/or time in a recognizably valid format.
        :type date_string: str|unicode

        :param strip_timezone:
            If True, timezone is stripped from date string.
        :type strip_timezone: bool

        :return: boolean value representing if the locale is applicable for the date string or not.
        """
        if strip_timezone:
            date_string, _ = pop_tz_offset_from_string(date_string, as_offset=False)

        date_string = self._translate_numerals(date_string)
        if settings.NORMALIZE:
            date_string = normalize_unicode(date_string)
        date_string = self._simplify(date_string, settings=settings)
        dictionary = self._get_dictionary(settings)
        date_tokens = dictionary.split(date_string)

        return dictionary.are_tokens_valid(date_tokens)

    def count_applicability(self, text, strip_timezone=False, settings=None):
        if strip_timezone:
            text, _ = pop_tz_offset_from_string(text, as_offset=False)

        text = self._simplify(text, settings=settings)
        sentences = self._sentence_split(text, settings=settings)
        tokens = []
        for sent in sentences:
            tokens.extend(self._split(sent, keep_formatting=False, settings=settings))
        return self._count_words_present_in_the_dictionary(tokens, settings)

    def _count_words_present_in_the_dictionary(self, words, settings=None):
        dictionary = self.clean_dictionary(self._get_split_dictionary(settings=settings))
        dict_cnt = 0
        skip_cnt = 0
        for word in set(words):
            if word in dictionary:
                if dictionary[word]:
                    dict_cnt += 1
                else:
                    skip_cnt += 1
            elif word.isdigit():
                skip_cnt += 1
        return [dict_cnt, skip_cnt]

    @staticmethod
    def clean_dictionary(dictionary, threshold=2):
        del_keys = []
        for key in dictionary:
            if len(key) < threshold:
                del_keys.append(key)
        for del_key in del_keys:
            del dictionary[del_key]
        return dictionary

    def translate(self, date_string, keep_formatting=False, settings=None):
        """
        Translate the date string to its English equivalent.

        :param date_string:
            A string representing date and/or time in a recognizably valid format.
        :type date_string: str|unicode

        :param keep_formatting:
            If True, retain formatting of the date string after translation.
        :type keep_formatting: bool

        :return: translated date string.
        """
        date_string = self._translate_numerals(date_string)
        if settings.NORMALIZE:
            date_string = normalize_unicode(date_string)
        date_string = self._simplify(date_string, settings=settings)
        dictionary = self._get_dictionary(settings)
        date_string_tokens = dictionary.split(date_string, keep_formatting)

        relative_translations = self._get_relative_translations(settings=settings)

        for i, word in enumerate(date_string_tokens):
            word = word.lower()
            for pattern, replacement in relative_translations.items():
                if pattern.match(word):
                    date_string_tokens[i] = pattern.sub(replacement, word)
            else:
                if word in dictionary:
                    date_string_tokens[i] = dictionary[word] or ''
        if "in" in date_string_tokens:
            date_string_tokens = self._clear_future_words(date_string_tokens)

        return self._join(list(filter(bool, date_string_tokens)),
                          separator="" if keep_formatting else " ", settings=settings)

    def _translate_numerals(self, date_string):
        date_string_tokens = NUMERAL_PATTERN.split(date_string)
        for i, token in enumerate(date_string_tokens):
            if token.isdigit():
                date_string_tokens[i] = str(int(token)).zfill(len(token))
                if isinstance(date_string_tokens[i], bytes):
                    date_string_tokens[i] = date_string_tokens[i].decode('utf-8')
        return u''.join(date_string_tokens)

    def _get_relative_translations(self, settings=None):
        if settings.NORMALIZE:
            if self._normalized_relative_translations is None:
                self._normalized_relative_translations = (
                        self._generate_relative_translations(normalize=True))
            return self._normalized_relative_translations
        else:
            if self._relative_translations is None:
                self._relative_translations = self._generate_relative_translations(normalize=False)
            return self._relative_translations

    def _generate_relative_translations(self, normalize=False):
        relative_translations = self.info.get('relative-type-regex', {})
        relative_dictionary = OrderedDict()
        for key, value in relative_translations.items():
            if normalize:
                value = list(map(normalize_unicode, value))
            pattern = '|'.join(sorted(value, key=len, reverse=True))
            pattern = DIGIT_GROUP_PATTERN.sub(r'?P<n>\d+', pattern)
            pattern = re.compile(r'^(?:{})$'.format(pattern), re.UNICODE | re.IGNORECASE)
            relative_dictionary[pattern] = key
        return relative_dictionary

    def translate_search(self, search_string, settings=None):
        dashes = ['-', '——', '—', '～']
        sentences = self._sentence_split(search_string, settings=settings)
        dictionary = self._get_dictionary(settings=settings)
        translated = []
        original = []
        for sentence in sentences:
            original_tokens, simplified_tokens = self._simplify_split_align(sentence, settings=settings)
            translated_chunk = []
            original_chunk = []
            for i, word in enumerate(simplified_tokens):
                if word == '' or word == ' ':
                    translated_chunk.append(word)
                    original_chunk.append(original_tokens[i])
                elif word in dictionary and word not in dashes:
                    translated_chunk.append(dictionary[word])
                    original_chunk.append(original_tokens[i])
                elif word.strip('()\"\'{}[],.،') in dictionary and word not in dashes:
                    punct = word[len(word.strip('()\"\'{}[],.،')):]
                    if punct and dictionary[word.strip('()\"\'{}[],.،')]:
                        translated_chunk.append(dictionary[word.strip('()\"\'{}[],.،')]+punct)
                    else:
                        translated_chunk.append(dictionary[word.strip('()\"\'{}[],.،')])
                    original_chunk.append(original_tokens[i])
                elif self._token_with_digits_is_ok(word):
                    translated_chunk.append(word)
                    original_chunk.append(original_tokens[i])
                else:
                    if translated_chunk:
                        translated.append(translated_chunk)
                        translated_chunk = []
                        original.append(original_chunk)
                        original_chunk = []
            if translated_chunk:
                translated.append(translated_chunk)
                original.append(original_chunk)
        for i in range(len(translated)):
            if "in" in translated[i]:
                translated[i] = self._clear_future_words(translated[i])
            translated[i] = self._join_chunk(list(filter(bool, translated[i])), settings=settings)
            original[i] = self._join_chunk(list(filter(bool, original[i])), settings=settings)
        return translated, original

    def _get_abbreviations(self, settings):
        dictionary = self._get_dictionary(settings=settings)
        abbreviations = []
        if self._abbreviations is None:
            for item in dictionary:
                if item.endswith('.') and len(item) > 1:
                    abbreviations.append(item)
            self._abbreviations = abbreviations
        return self._abbreviations

    def _sentence_split(self, string, settings):
        abbreviations = self._get_abbreviations(settings=settings)
        digit_abbreviations = ['[0-9]']  # numeric date with full stop
        abbreviation_string = ''

        for abbreviation in abbreviations:
            abbreviation_string += '(?<! ' + abbreviation[:-1] + ')'  # negative lookbehind
        if self.shortname in ['fi', 'cs', 'hu', 'de', 'da']:
            for digit_abbreviation in digit_abbreviations:
                abbreviation_string += '(?<!' + digit_abbreviation + ')'  # negative lookbehind

        splitters_dict = {1: '[\.!?;…\r\n]+(?:\s|$)*',  # most European, Tagalog, Hebrew, Georgian,
                          # Indonesian, Vietnamese
                          2: '(?:[¡¿]+|[\.!?;…\r\n]+(?:\s|$))+',  # Spanish
                          3: '[|!?;\r\n]+(?:\s|$)+',  # Hindi and Bangla
                          4: '[。…‥\.!?？！;\r\n]+(?:\s|$)+',  # Japanese and Chinese
                          5: '[\r\n]+',  # Thai
                          6: '[\r\n؟!\.…]+(?:\s|$)+'}  # Arabic and Farsi
        if 'sentence_splitter_group' not in self.info:
            split_reg = abbreviation_string + splitters_dict[1]
            sentences = re.split(split_reg, string)
        else:
            split_reg = abbreviation_string + splitters_dict[self.info['sentence_splitter_group']]
            sentences = re.split(split_reg, string)

        for i in sentences:
            if not i:
                sentences.remove(i)
        return sentences

    def _simplify_split_align(self, original, settings):
        # TODO: Switch to new split method.
        original_tokens = self._word_split(original, settings=settings)
        simplified_tokens = self._word_split(self._simplify(normalize_unicode(original), settings=settings),
                                             settings=settings)
        if len(original_tokens) == len(simplified_tokens):
            return original_tokens, simplified_tokens

        elif len(original_tokens) < len(simplified_tokens):
            add_empty = False
            for i, token in enumerate(simplified_tokens):
                if i < len(original_tokens):
                    if token == normalize_unicode(original_tokens[i].lower()):
                        add_empty = False
                    else:
                        if not add_empty:
                            add_empty = True
                            continue
                        else:
                            original_tokens.insert(i, '')
                else:
                    original_tokens.insert(i, '')
        else:
            add_empty = False
            for i, token in enumerate(original_tokens):
                if i < len(simplified_tokens):
                    if normalize_unicode(token.lower()) == simplified_tokens[i]:
                        add_empty = False
                    else:
                        if not add_empty:
                            add_empty = True
                            continue
                        else:
                            simplified_tokens.insert(i, '')
                else:
                    simplified_tokens.insert(i, '')

        while len(original_tokens) != len(simplified_tokens):
            if len(original_tokens) > len(simplified_tokens):
                original_tokens.remove('')
            else:
                simplified_tokens.remove('')
        return original_tokens, simplified_tokens

    def _get_split_dictionary(self, settings):
        if self._split_dictionary is None:
            settings.NORMALIZE = True
            dictionary = self._get_dictionary(settings=settings)
            self._split_dictionary = self._split_dict(dictionary)
        return self._split_dictionary

    def _split_dict(self, dictionary):
        newdict = {}
        for item in dictionary:
            if ' ' in item:
                items = item.split()
                for i in items:
                    newdict[i] = dictionary[item]
            else:
                newdict[item] = dictionary[item]
        return newdict

    def _word_split(self, string, settings):
        if 'no_word_spacing' in self.info:
            return self._split(string, keep_formatting=True, settings=settings)
        else:
            return string.split()

    def _split(self, date_string, keep_formatting, settings=None):
        tokens = [date_string]
        tokens = list(self._split_tokens_with_regex(tokens, r"(\d+)"))
        tokens = list(
            self._split_tokens_by_known_words(tokens, keep_formatting, settings=settings))
        return tokens

    def _split_tokens_with_regex(self, tokens, regex):
        tokens = tokens[:]
        for i, token in enumerate(tokens):
            tokens[i] = re.split(regex, token)
        return filter(bool, chain(*tokens))

    def _split_tokens_by_known_words(self, tokens, keep_formatting, settings=None):
        dictionary = self._get_dictionary(settings)
        for i, token in enumerate(tokens):
            tokens[i] = dictionary.split(token, keep_formatting)
        return list(chain(*tokens))

    def _join_chunk(self, chunk, settings):
        if 'no_word_spacing' in self.info:
            return self._join(chunk, separator="", settings=settings)
        else:
            return re.sub('\s{2,}', ' ', " ".join(chunk))

    def _token_with_digits_is_ok(self, token):
        if 'no_word_spacing' in self.info:
            if re.search('[\d\.:\-/]+', token) is not None:
                return True
            else:
                return False

        else:
            if re.search('\d+', token) is not None:
                return True
            else:
                return False

    def _simplify(self, date_string, settings=None):
        date_string = date_string.lower()
        simplifications = self._get_simplifications(settings=settings)
        for simplification in simplifications:
            pattern, replacement = list(simplification.items())[0]
            date_string = pattern.sub(replacement, date_string).lower()
        return date_string

    def _get_simplifications(self, settings=None):
        no_word_spacing = eval(self.info.get('no_word_spacing', 'False'))
        if settings.NORMALIZE:
            if self._normalized_simplifications is None:
                self._normalized_simplifications = []
                simplifications = self._generate_simplifications(normalize=True)
                for simplification in simplifications:
                    pattern, replacement = list(simplification.items())[0]
                    if not no_word_spacing:
                        pattern = r'(?<=\A|\W|_)%s(?=\Z|\W|_)' % pattern
                    pattern = re.compile(pattern, flags=re.I | re.U)
                    self._normalized_simplifications.append({pattern: replacement})
            return self._normalized_simplifications

        else:
            if self._simplifications is None:
                self._simplifications = []
                simplifications = self._generate_simplifications(normalize=False)
                for simplification in simplifications:
                    pattern, replacement = list(simplification.items())[0]
                    if not no_word_spacing:
                        pattern = r'(?<=\A|\W|_)%s(?=\Z|\W|_)' % pattern
                    pattern = re.compile(pattern, flags=re.I | re.U)
                    self._simplifications.append({pattern: replacement})
            return self._simplifications

    def _generate_simplifications(self, normalize=False):
        simplifications = []
        for simplification in self.info.get('simplifications', []):
            c_simplification = {}
            key, value = list(simplification.items())[0]
            if normalize:
                key = normalize_unicode(key)

            if isinstance(value, int):
                c_simplification[key] = str(value)
            else:
                c_simplification[key] = normalize_unicode(value) if normalize else value

            simplifications.append(c_simplification)
        return simplifications

    def _clear_future_words(self, words):
        freshness_words = {'day', 'week', 'month', 'year', 'hour', 'minute', 'second'}
        if set(words).isdisjoint(freshness_words):
            words.remove("in")
        return words

    def _join(self, tokens, separator=" ", settings=None):
        if not tokens:
            return ""

        capturing_splitters = self._get_splitters(settings)['capturing']
        joined = tokens[0]
        for i in range(1, len(tokens)):
            left, right = tokens[i - 1], tokens[i]
            if left not in capturing_splitters and right not in capturing_splitters:
                joined += separator
            joined += right

        return joined

    def _get_dictionary(self, settings=None):
        if not settings.NORMALIZE:
            if self._dictionary is None:
                self._generate_dictionary()
            self._dictionary._settings = settings
            return self._dictionary
        else:
            if self._normalized_dictionary is None:
                self._generate_normalized_dictionary()
            self._normalized_dictionary._settings = settings
            return self._normalized_dictionary

    def _get_wordchars(self, settings=None):
        if self._wordchars is None:
            self._set_wordchars(settings)
        return self._wordchars

    def _get_splitters(self, settings=None):
        if self._splitters is None:
            self._set_splitters(settings)
        return self._splitters

    def _set_splitters(self, settings=None):
        splitters = {
            'wordchars': set(),  # The ones that split string only if they are not surrounded by letters from both sides
            'capturing': set(),  # The ones that are not filtered out from tokens after split
        }
        splitters['capturing'] |= set(ALWAYS_KEEP_TOKENS)

        wordchars = self._get_wordchars(settings)
        skip = set(self.info.get('skip', [])) | splitters['capturing']
        for token in skip:
            if not re.match(r'^\W+$', token, re.UNICODE):
                continue
            if token in wordchars:
                splitters['wordchars'].add(token)

        self._splitters = splitters

    def _set_wordchars(self, settings=None):
        wordchars = set()
        for word in self._get_dictionary(settings):
            if re.match(r'^[\W\d_]+$', word, re.UNICODE):
                continue
            for char in word:
                wordchars.add(char.lower())

        self._wordchars = wordchars - {" "} | {"0", "1", "2", "3", "4", "5", "6", "7", "8", "9"}

    def get_wordchars_for_detection(self, settings):
        if self._wordchars_for_detection is None:
            wordchars = set()
            for word in self._get_dictionary(settings):
                if re.match(r'^[\W\d_]+$', word, re.UNICODE):
                    continue
                for char in word:
                    wordchars.add(char.lower())
            self._wordchars_for_detection = wordchars - {"0", "1", "2", "3", "4", "5", "6", "7", "8", "9",
                                                         ":", "(", ")", "'", "q", "a", "m", "p", " "}
        return self._wordchars_for_detection

    def _generate_dictionary(self, settings=None):
        self._dictionary = Dictionary(self.info, settings=settings)

    def _generate_normalized_dictionary(self, settings=None):
        self._normalized_dictionary = NormalizedDictionary(self.info, settings=settings)

    def to_parserinfo(self, base_cls=parser.parserinfo):
        attributes = {
            'JUMP': self.info.get('skip', []),
            'PERTAIN': self.info.get('pertain', []),
            'WEEKDAYS': [self.info['monday'],
                         self.info['tuesday'],
                         self.info['wednesday'],
                         self.info['thursday'],
                         self.info['friday'],
                         self.info['saturday'],
                         self.info['sunday']],
            'MONTHS': [self.info['january'],
                       self.info['february'],
                       self.info['march'],
                       self.info['april'],
                       self.info['may'],
                       self.info['june'],
                       self.info['july'],
                       self.info['august'],
                       self.info['september'],
                       self.info['october'],
                       self.info['november'],
                       self.info['december']],
            'HMS': [self.info['hour'],
                    self.info['minute'],
                    self.info['second']],
        }
        name = '{language}ParserInfo'.format(language=self.info['name'])
        return type(name, bases=[base_cls], dict=attributes)