import io

import boto3
from ieml.dictionary.dictionary import Dictionary

from ieml.constants import LANGUAGES
from ieml.dictionary.tools import term
from ieml.dictionary.version import get_available_dictionary_version, create_dictionary_version, \
    DictionaryVersion, latest_dictionary_version
from ieml.dictionary.script import script, m, NullScript, AdditiveScript

from ieml.dictionary.script.tools import factorize

def find_and_add_script_to_current_version(to_find):
    t = reversed(get_available_dictionary_version())
    for version in t:
        version.load()
        if to_find in version.terms:
            version = create_dictionary_version(merge=[version])
            version.upload_to_s3()
            break

"""
s.-S:.U:.-'l.-S:.O:.-'n.-S:.U:.-',+M:.-'M:.-'n.-S:.U:.-',  => n.-S:.U:.-'s.-S:.U:.-'l.-S:.O:.-',+n.-S:.U:.-‘M:.-‘M:.-‘,   (continents)
M:.-',M:.-',S:.-'B:.-'n.-S:.U:.-',_    =>     n.-S:.U:.-’S:.-’B:.-‘,M:.-‘,M:.-‘,_   (Pays Européens)
M:.-',M:.-',S:.-'S:.-'n.-S:.U:.-',_    =>     n.-S:.U:.-’S:.-’S:.-‘,M:.-‘,M:.-‘,_   (Pays d’Amérique du Nord)
M:.-',M:.-',B:.-'S:.-'n.-S:.U:.-',_    =>     n.-S:.U:.-’B:.-’S:.-‘,M:.-‘,M:.-‘,_  (Pays d’Amérique centrale et des Caraïbes)
M:.-',M:.-',T:.-'S:.-'n.-S:.U:.-',_    =>     n.-S:.U:.-’T:.-’S:.-‘,M:.-‘,M:.-‘,_   (Pays d’Amérique du Sud)
1 Comment

Pour avoir le VIDE en substance et donner une étymologie aux auxiliaires qui n’en n’ont pas
E:O:.M:O:.-  => E:.O:.M:O:.- (pour récupérer l’étymologie avec le vide en substance)
E:M:.M:O:.-  => E:.M:.M:O:.-  (idem)
E:F:.O:O:.- => E:.F:.O:O:.- (idem)
E:F:.O:M:.t.- => E:.-F:.O:M:.-t.-‘  (changement de couche)
E:F:.M:M:.l.- => E:.-F:.M:M:.-l.-‘   (changement de couche)
E:O:.-M:O:.-t.o.-' => E:.-O:.M:O:.-t.o.-‘
Pour donner une étymologie aux verbes qui n’en n’ont pas
O:O:.O:O:.-  => O:O:.O:O:.t.-
O:O:.F:.- => O:O:.f.F:.-
noms
i.B:.-+u.M:.-O:.-' => i.f.B:.-+u.f.M:.-O:.-‘  (pour tenir compte du changement précédent)
M:.-O:.-'M:.-wa.e.-'t.x.-s.y.-',  => M:.-O:.-'M:.-wa.e.-'t.-x.-s.y.-', (Pour réparer une erreur)
"""

def _rotate_sc(s):
    subst, attr, mode = s
    return m(mode, subst, attr)

def _rotate_sc_additive(s):
    """
    s.-S:.U:.-'l.-S:.O:.-'n.-S:.U:.-',+M:.-'M:.-'n.-S:.U:.-',  =>
    n.-S:.U:.-'s.-S:.U:.-'l.-S:.O:.-',+n.-S:.U:.-‘M:.-‘M:.-‘,"""

    if isinstance(s, AdditiveScript):
        return AdditiveScript([_rotate_sc(_s) for _s in s])
    else:
        return _rotate_sc(s)

def _double_rotate(s):
    subst, attr, mode = s
    mode = _rotate_sc(mode)
    return m(mode, subst, attr)

def _split_subst(s):
    subst, attr, mode = s
    assert isinstance(mode, NullScript)
    subst0, subst1, mode = subst
    assert isinstance(mode, NullScript)

    return m(m(subst0), m(subst1), attr)

def _promote_and_split(s):
    """
    E:F:.O:M:.t.- => E:.-F:.O:M:.-t.-‘
    E:F:.M:M:.l.- => E:.-F:.M:M:.-l.-‘
    """
    subst, attr, mode = s
    subst0, subst1, _mode = subst
    assert isinstance(_mode, NullScript)

    return m(m(m(subst0)) ,m(m(subst1), attr) ,m(mode))

def _transfer_substance(s):
    """
    E:O:.-M:O:.-t.o.-' => E:.-O:.M:O:.-t.o.-‘
    """
    subst, attr, mode = s
    attr0, attr1, attr2 = attr
    assert isinstance(attr1, NullScript) and isinstance(attr2, NullScript)
    subst, subst1, subst2 = subst
    assert isinstance(subst1, NullScript) and isinstance(subst2, NullScript)

    subst0, subst1, subst2 = subst
    assert isinstance(subst2, NullScript)
    return m(m(m(subst0)), m(m(subst1), attr0), mode)

"""
 (Pour réparer une erreur)
"""
def _add_mode_t(s):
    """
O:O:.O:O:.-  => O:O:.O:O:.t.-
    """
    subst, attr, mode = s
    assert isinstance(mode, NullScript)
    return m(subst, attr, script('t.'))

def _insert_attr_f(s):
    """O:O:.F:.- => O:O:.f.F:.-"""
    subst, attr, mode = s
    assert isinstance(mode, NullScript)
    return m(subst, script('f.'), attr)

def _insert_f_additive(s):
    """i.B:.-+u.M:.-O:.-' => i.f.B:.-+u.f.M:.-O:.-'"""
    subst, attr, mode = s
    assert isinstance(mode, NullScript)

    if isinstance(subst, AdditiveScript):
        subst = AdditiveScript([_insert_attr_f(_s) for _s in subst])
    else:
        subst = _insert_attr_f(subst)

    return m(subst ,attr)

def _fix_typo(s):
    """M:.-O:.-'M:.-wa.e.-'t.x.-s.y.-',  => M:.-O:.-'M:.-wa.e.-'t.-x.-s.y.-',"""
    subst, attr, mode = s
    return m(subst, attr, script("t.-x.-s.y.-'"))

to_translate = {
    "s.-S:.U:.-'l.-S:.O:.-'n.-S:.U:.-',+M:.-'M:.-'n.-S:.U:.-',": _rotate_sc_additive,
    "M:.-',M:.-',S:.-'B:.-'n.-S:.U:.-',_": _double_rotate,
    "M:.-',M:.-',S:.-'S:.-'n.-S:.U:.-',_": _double_rotate,
    "M:.-',M:.-',B:.-'S:.-'n.-S:.U:.-',_": _double_rotate,
    "M:.-',M:.-',T:.-'S:.-'n.-S:.U:.-',_": _double_rotate,
    "E:O:.M:O:.-": _split_subst,
    "E:M:.M:O:.-": _split_subst,
    "E:F:.O:O:.-": _split_subst,
    "E:F:.O:M:.t.-": _promote_and_split,
    "E:F:.M:M:.l.-": _promote_and_split,
    "E:O:.-M:O:.-t.o.-'": _transfer_substance,
    "O:O:.O:O:.-": _add_mode_t,
    "O:O:.F:.-": _insert_attr_f,
    "i.B:.-+u.M:.-O:.-'":_insert_f_additive,
    "M:.-O:.-'M:.-wa.e.-'t.x.-s.y.-',":_fix_typo
}

result = {
    "s.-S:.U:.-'l.-S:.O:.-'n.-S:.U:.-',+M:.-'M:.-'n.-S:.U:.-',": "n.-S:.U:.-'s.-S:.U:.-'l.-S:.O:.-',+n.-S:.U:.-'M:.-'M:.-',",
    "M:.-',M:.-',S:.-'B:.-'n.-S:.U:.-',_": "n.-S:.U:.-'S:.-'B:.-',M:.-',M:.-',_",
    "M:.-',M:.-',S:.-'S:.-'n.-S:.U:.-',_": "n.-S:.U:.-'S:.-'S:.-',M:.-',M:.-',_",
    "M:.-',M:.-',B:.-'S:.-'n.-S:.U:.-',_": "n.-S:.U:.-'B:.-'S:.-',M:.-',M:.-',_",
    "M:.-',M:.-',T:.-'S:.-'n.-S:.U:.-',_": "n.-S:.U:.-'T:.-'S:.-',M:.-',M:.-',_",
    "E:O:.M:O:.-": "E:.O:.M:O:.-",
    "E:M:.M:O:.-": "E:.M:.M:O:.-",
    "E:F:.O:O:.-": "E:.F:.O:O:.-",
    "E:F:.O:M:.t.-": "E:.-F:.O:M:.-t.-'",
    "E:F:.M:M:.l.-": "E:.-F:.M:M:.-l.-'",
    "E:O:.-M:O:.-t.o.-'": "E:.-O:.M:O:.-t.o.-'",
    "O:O:.O:O:.-": "O:O:.O:O:.t.-",
    "O:O:.F:.-": "O:O:.f.F:.-",
    "i.B:.-+u.M:.-O:.-'": "i.f.B:.-+u.f.M:.-O:.-'",
    "M:.-O:.-'M:.-wa.e.-'t.x.-s.y.-',": "M:.-O:.-'M:.-wa.e.-'t.-x.-s.y.-',"
}

# def _test_rules():
#     for s, r in to_translate.items():
#         if str(r(script(s))) != result[s]:
#             print("error for %s != %s"%(str(r(script(s))), result[s]))

def translate_script(to_translate):
    """
    translate the root paradigms in key in argument, with the function in value
    :param to_translate:
    :return:
    """
    version = DictionaryVersion(latest_dictionary_version())
    version.load()
    to_remove = []
    to_add = {
        'terms': [],
        'roots': [],
        'inhibitions': {},
        'translations': {l: {} for l in LANGUAGES}
    }

    for root, func in to_translate.items():
        root = script(root)
        terms = list(filter(lambda s: s in root, map(script, version.terms)))

        new_root = func(root)
        new_terms = [func(s) for s in terms]

        to_add['terms'].extend(map(str, new_terms))
        to_add['roots'].append(str(new_root))
        to_add['inhibitions'].update({str(new_root): version.inhibitions[root]})
        for l in LANGUAGES:
            to_add['translations'][l].update({str(func(s)): version.translations[l][s] for s in terms})

        to_remove.extend(map(str, terms))

    return create_dictionary_version(version, add=to_add, remove=to_remove)

"""
s.-S:.U:.-'l.-S:.O:.-'n.-T:.A:.-',+M:.-'M:.-'n.-T:.A:.-',
s.-S:.U:.-'n.-T:.A:.-'l.-S:.O:.-',+n.-T:.A:.-'M:.-'M:.-',
"""
def translate_ocean(s):
    if s in script("M:.-'M:.-'n.-T:.A:.-',"):
        return _rotate_sc(s)
    elif s in script("s.-S:.U:.-'l.-S:.O:.-'n.-T:.A:.-',"):
        subst, attr, mode = s
        return m(subst, mode, attr)
    elif isinstance(s, AdditiveScript):
        return AdditiveScript(children=[translate_ocean(c) for c in s])

def translate_body_parts(s):
    def _level_3(s):
        s2, a2, m2 = s
        s1, a1, m1 = a2
        return m(s2, m(s1, m(a1.children[0], m1.children[0])), m2)

    s4, a4, _ = s
    s3, a3, _ = a4
    return m(s4, m(_level_3(s3), _level_3(a3)))

body_parts = {
    "f.o.-f.o.-'E:.-U:.n.-l.-',E:.-U:.M:.T:.-l.-'E:.-A:.M:.T:.-l.-',_" : "f.o.-f.o.-'E:.-U:.n.-l.-',E:.-U:.M:T:.-l.-'E:.-A:.M:T:.-l.-',_",
    "f.o.-f.o.-'E:.-U:.t.-l.-',E:.-U:.M:.T:.-l.-'E:.-A:.M:.T:.-l.-',_" : "f.o.-f.o.-'E:.-U:.t.-l.-',E:.-U:.M:T:.-l.-'E:.-A:.M:T:.-l.-',_",
}
to_translate_body = {
        "f.o.-f.o.-'E:.-U:.n.-l.-',E:.-U:.M:.T:.-l.-'E:.-A:.M:.T:.-l.-',_": translate_body_parts,
        "f.o.-f.o.-'E:.-U:.t.-l.-',E:.-U:.M:.T:.-l.-'E:.-A:.M:.T:.-l.-',_": translate_body_parts
    }

body_parts_root = factorize(list(map(script, [
    "f.o.-f.o.-'E:.-U:.n.-l.-',E:.-U:.M:T:.-l.-'E:.-A:.M:T:.-l.-',_",
    "f.o.-f.o.-'E:.-U:.t.-l.-',E:.-U:.M:T:.-l.-'E:.-A:.M:T:.-l.-',_",
    "f.o.-f.o.-',n.i.-f.i.-',M:O:.-O:.-',_"
])))

def upload_to_s3(dictionary_version):
    s3 = boto3.resource('s3')
    bucket_name = 'ieml-dictionary-versions'
    bucket = s3.Bucket(bucket_name)
    obj = bucket.Object("%s.json" % str(dictionary_version))

    obj.upload_fileobj(io.BytesIO(bytes(dictionary_version.json(), 'utf-8')))
    obj.Acl().put(ACL='public-read')

    assert dictionary_version in get_available_dictionary_version()

animals_and_vegetebals = {"B:.S:.n.-k.-+n.-n.S:.U:.-+n.B:.U:.-+n.T:.A:.-'+B:.B:.n.-k.-+n.-u.S:.-+u.B:.-+u.T:.-'":
                          "B:.S:.n.-B:S:+T:.-n.T:.A:.-+n.S:+B:.U:.-'+     B:.B:.n.-B:S:+T:.-u.M:.-'"}

def factorize_root(root):
    factorized_root = factorize(root.script)

    to_remove = []
    to_add = {
        'terms': [],
        'roots': [str(factorized_root)],
        'inhibitions': {
            str(factorized_root): root.inhibitions
        },
        'translations': {
            'fr': {},
            'en': {}
        }
    }

    for t in Dictionary().roots[root]:
        to_remove.append(str(t.script))
        s = factorize(t.script)
        to_add['terms'].append(str(s))
        to_add['translations']['fr'][str(s)] = t.translations.fr
        to_add['translations']['en'][str(s)] = t.translations.en

    return to_remove, to_add

def _remove_attr_f(s):
    """O:O:.f.F:.- -> O:O:.F:.- """
    subst, attr, mode = s
    return m(subst, mode)

def translate_mouvements_et_milieux(s):
    """i.f.B:.-+u.f.M:.-O:.-' -> i.B:.-+u.M:.-O:.-'"""
    subst, attr, mode = s
    assert isinstance(mode, NullScript)

    if isinstance(subst, AdditiveScript):
        subst = AdditiveScript([_remove_attr_f(_s) for _s in subst])
    else:
        subst = _remove_attr_f(subst)

    return m(subst, attr)

def translate_competence_en_curr_data(s):
    """M:.-O:.-'M:.-wa.e.-'t.-x.-s.y.-',  => t.-x.-s.y.-' wa.e.-', M:M:.-',O:.-',_"""
    subst, attr, mode = s
    attr_s, attr_a, attr_m = attr
    assert isinstance(attr_m, NullScript)

    subst_s, subst_a, subst_m = subst
    assert isinstance(subst_m, NullScript)
    first_M = subst_s.children[0].children[0]

    return m(m(mode, m(attr_a)), m(m(m(m(first_M, attr_s.children[0].children[0])))), m(m(subst_a)))

def get_competence_en_curr_data_diff():
    update = {}
    remove = []
    for t in term("M:.-O:.-'M:.-wa.e.-'t.-x.-s.y.-',").relations.contains:
        if len(t) == 1 or len(t) == 2 or len(t) == 18:
            update[str(t.script)] = str(translate_competence_en_curr_data(t.script))
        else:
            remove.append(str(t.script))
    return update, remove

def translate_noetic(s):
    """M:.O:.-O:.O:.-B:.T:.n.-' => s.M:O:.O:O:.-"""
    subst, attr, mode = s
    return m(script('s.'),
             m(subst.children[0].children[0], subst.children[1].children[0]),
             m(attr.children[0].children[0], attr.children[1].children[0]))

def translate_tisse_intl_col(s):
    """O:M:.-O:M:.-we.h.-' => O:M:.-'O:M:.-'s.o.-k.o.-',"""
    subst, attr, mode = s
    return m(m(subst), m(attr), script("s.o.-k.o.-'"))

def translate_formes_visuelles(s):
    """s.u.-'O:M:.-'O:.-',+s.u.-'M:O:.-O:.-'M:.-', => b.-S:.U:.-'O:M:.-'O:.-', + b.-S:.U:.-'M:O:.-O:.-'M:.-',"""

    def set_bSU_subst(s):
        subst, attr, mode = s
        return m(script("b.-S:.U:.-'"), attr, mode)

    if isinstance(s, AdditiveScript):
        return AdditiveScript([set_bSU_subst(i) for i in s.children])
    else:
        return set_bSU_subst(s)

def translate_update(root, f):
    update = {'terms':
                  {str(s.script): str(f(s.script)) for s in term(root).relations.contains}
              }
    version = create_dictionary_version(latest_dictionary_version(), update=update)
    upload_to_s3(version)
    print(version)

def translate_ecosystem_intl_col(s):
    """O:.M:.- => s.o.-k.o.-'M:O:.-',"""
    subst, attr, mode = s

    return m(script("s.o.-k.o.-'"), m(m(m(attr.children[0], subst.children[0]))))

def translate_ecosystem_intl_col_tern(s):
    """O:.M:.-M:.-' => s.o.-k.o.-‘M:O:.-‘,M:.-',_"""
    subst, attr, mode = s

    return m(translate_ecosystem_intl_col(subst), m(m(attr)))

if __name__ == "__main__":
    # to_add = {
    #     'terms': ["f.o.-f.o.-'E:.-U:.n.-l.-',E:.-U:.M:T:.-l.-'E:.-A:.M:T:.-l.-',_",
    #               "f.o.-f.o.-'E:.-U:.t.-l.-',E:.-U:.M:T:.-l.-'E:.-A:.M:T:.-l.-',_",
    #               "f.o.-f.o.-',n.i.-f.i.-',M:O:.-O:.-',_",
    #               "f.o.-f.o.-',n.i.-f.i.-',M:O:.-O:.-',_+f.o.-f.o.-'E:.-U:.S:+B:T:.-l.-',E:.-U:.M:T:.-l.-'E:.-A:.M:T:.-l.-',_",
    #               "f.o.-f.o.-'E:.-U:.S:+B:T:.-l.-',E:.-U:.M:T:.-l.-'E:.-A:.M:T:.-l.-',_"
    #              ],
    #     'roots': ["f.o.-f.o.-',n.i.-f.i.-',M:O:.-O:.-',_+f.o.-f.o.-'E:.-U:.S:+B:T:.-l.-',E:.-U:.M:T:.-l.-'E:.-A:.M:T:.-l.-',_"],
    #     'inhibitions': {
    #          "f.o.-f.o.-',n.i.-f.i.-',M:O:.-O:.-',_+f.o.-f.o.-'E:.-U:.S:+B:T:.-l.-',E:.-U:.M:T:.-l.-'E:.-A:.M:T:.-l.-',_": ['father_mode']
    #     },
    #     'translations': {
    #         'fr': {
    #             "f.o.-f.o.-',n.i.-f.i.-',M:O:.-O:.-',_+f.o.-f.o.-'E:.-U:.S:+B:T:.-l.-',E:.-U:.M:T:.-l.-'E:.-A:.M:T:.-l.-',_": "Parties du corps",
    #             "f.o.-f.o.-'E:.-U:.n.-l.-',E:.-U:.M:T:.-l.-'E:.-A:.M:T:.-l.-',_": "Parties du corps: le tronc",
    #             "f.o.-f.o.-'E:.-U:.t.-l.-',E:.-U:.M:T:.-l.-'E:.-A:.M:T:.-l.-',_": "Parties du corps: la tête",
    #             "f.o.-f.o.-',n.i.-f.i.-',M:O:.-O:.-',_": "Parties du corps: membres",
    #             "f.o.-f.o.-'E:.-U:.S:+B:T:.-l.-',E:.-U:.M:T:.-l.-'E:.-A:.M:T:.-l.-',_": "Parties du corps: la tête et le tronc"
    #         },
    #         'en': {
    #             "f.o.-f.o.-',n.i.-f.i.-',M:O:.-O:.-',_+f.o.-f.o.-'E:.-U:.S:+B:T:.-l.-',E:.-U:.M:T:.-l.-'E:.-A:.M:T:.-l.-',_": "body parts",
    #             "f.o.-f.o.-'E:.-U:.n.-l.-',E:.-U:.M:T:.-l.-'E:.-A:.M:T:.-l.-',_": "Body parts: trunk",
    #             "f.o.-f.o.-'E:.-U:.t.-l.-',E:.-U:.M:T:.-l.-'E:.-A:.M:T:.-l.-',_": "Body parts: head",
    #             "f.o.-f.o.-',n.i.-f.i.-',M:O:.-O:.-',_": "body parts: limbs",
    #             "f.o.-f.o.-'E:.-U:.S:+B:T:.-l.-',E:.-U:.M:T:.-l.-'E:.-A:.M:T:.-l.-',_": "Body parts: head and trunk"
    #
    #         }
    #     }
    # }
    # to_remove = ["f.o.-f.o.-'E:.-U:.n.-l.-',E:.-U:.M:T:.-l.-'E:.-A:.M:T:.-l.-',_",
    #               "f.o.-f.o.-'E:.-U:.t.-l.-',E:.-U:.M:T:.-l.-'E:.-A:.M:T:.-l.-',_",
    #               "f.o.-f.o.-',n.i.-f.i.-',M:O:.-O:.-',_"
    # ]

    # to_remove, to_add = factorize_root(term("b.i.-n.i.-'l.i.-n.i.-'+m.+l.i.-f.i.-'+wu.f.A:.-+a.S:.-+i.U:.-+e.O:.-+S:+T:T:.i.-'n.o.-n.o.-'+f.o.-f.o.-'+n.-B:.A:.-+S:+B:.U:.-',"))
    #
    #
    # version = create_dictionary_version(latest_dictionary_version(),
    #                           add=to_add, remove=to_remove)
    # upload_to_s3(version)
    # print(version)
    # update = {str(s.script): str(translate_mouvements_et_milieux(s.script)) for s in term("i.f.B:.-+u.f.M:.-O:.-'").relations.contains}
    # version = create_dictionary_version(latest_dictionary_version(), update=update)
    # upload_to_s3(version)
    # print(version)
    # _up, _rem = get_competence_en_curr_data_diff()
    # update = {'terms': {
    #         **{str(s.script): str(translate_mouvements_et_milieux(s.script)) for s in term("i.f.B:.-+u.f.M:.-O:.-'").relations.contains},
    #         **_up
    #     },
    #     'remove': {
    #         *_rem
    #     }
    # }
    # print('\n'.join("{} => {}".format(a, b) for a, b in _up.items()))
    # version = create_dictionary_version(latest_dictionary_version(), update=update, remove=_rem)
    # upload_to_s3(version)
    # print(version)

    root = "O:.M:.-M:.-'"
    translator = translate_ecosystem_intl_col_tern
    "dictionary_2018-06-08_17:07:06"
    print(str(translator(script(root))))

    d = Dictionary("dictionary_2018-06-08_17:07:06")
    # translate_update("s.u.-'O:M:.-'O:.-',+s.u.-'M:O:.-O:.-'M:.-',", translate_formes_visuelles)
    diff = {
        **{str(s.script): str(translate_competence_en_curr_data(s.script)) for s in term("M:.-O:.-'M:.-wa.e.-'t.-x.-s.y.-',", d).relations.contains},
        # **{str(s.script): str(translate_ecosystem_intl_col_tern(s.script)) for s in term("O:.M:.-M:.-'", d).relations.contains},
    }
    import json
    print(json.dumps(diff, indent=True))

    version = create_dictionary_version(None, diff=diff)
    # upload_to_s3(version)
    print(version)


import logging
import os
from functools import partial
import ply.yacc as yacc

from ieml.dictionary.dictionary import Dictionary
from ieml.exceptions import TermNotFoundInDictionary, InvalidIEMLObjectArgument
from ieml.grammar import Text
from ieml.grammar.fact import Fact
from ieml.grammar.theory import Theory
from ieml.grammar.topic import Topic
from ieml.grammar.word import Word
from ... import parser_folder
from ...exceptions import CannotParse

from .lexer import get_lexer, tokens
import threading

def _add(lp1, p2):
    return lp1[0] + [p2[0]], lp1[1] + p2[1]

def _build(*arg):
    if len(arg) > 1:
        _h = []
        for e in arg[1:]:
            for h in e[1]:
                h[1].insert(0, arg[0])
                _h.append(h)
        return arg[0], _h
    return arg[0], []

def _hyperlink(node, text_list):
    return node[0], node[1] + [(text, [node[0]]) for text in text_list[0]]

class IEMLParserSingleton(type):
    _instances = {}

    def __call__(cls, *args, **kwargs):
        dictionary = args[0] if len(args) > 0 else \
            kwargs['dictionary'] if 'dictionary' in kwargs else None

        if dictionary is None:
            dictionary = Dictionary()

        if not isinstance(dictionary, Dictionary):
            dictionary = Dictionary(dictionary)

        if dictionary.version not in cls._instances:
            # this code is to clean up duplicate class if we reload modules
            cls._instances[dictionary.version] = \
                super(IEMLParserSingleton, cls).__call__(dictionary=dictionary)

        return cls._instances[dictionary.version]

class IEMLParser(metaclass=IEMLParserSingleton):
    tokens = tokens
    lock = threading.Lock()

    def __init__(self, dictionary=None):
        from ieml.dictionary.tools import term

        self._get_term = partial(term, dictionary=dictionary)

        # Build the lexer and parser
        self.lexer = get_lexer()
        self.parser = yacc.yacc(module=self, errorlog=logging, start='proposition',
                                debug=False, optimize=True, picklefile=os.path.join(parser_folder, "ieml_parser.pickle"))
        self._ieml = None

    def parse(self, s):
        """Parses the input string, and returns a reference to the created AST's root"""
        with self.lock:
            try:
                return self.parser.parse(s, lexer=self.lexer)
            except InvalidIEMLObjectArgument as e:
                raise CannotParse(s, str(e))
            except CannotParse as e:
                e.s = s
                raise e

    # Parsing rules
    def p_ieml_proposition(self, p):
        """proposition :  word
                        | topic
                        | fact
                        | theory
                        | text"""

        p[0] = p[1]

    def p_literal_list(self, p):
        """literal_list : literal_list LITERAL
                        | LITERAL"""

        if len(p) == 3:
            p[0] = p[1] + [p[2][1:-1]]
        else:
            p[0] = [p[1][1:-1]]

    def p_word(self, p):
        """word : TERM
                | LBRACKET TERM RBRACKET
                | LBRACKET TERM RBRACKET literal_list"""
        try:
            term = self._get_term(p[1 if len(p) == 2 else 2])
        except TermNotFoundInDictionary as e:
            raise CannotParse(self._ieml, str(e))

        if len(p) == 5:
            p[0] = Word(term, literals=p[4])
        else:
            p[0] = Word(term)

    def p_proposition_sum(self, p):
        """word_sum : word_sum PLUS word
                    | word
            clauses_sum : clauses_sum PLUS clause
                    | clause
            superclauses_sum : superclauses_sum PLUS superclause
                    | superclause"""

        # closed_proposition_list : closed_proposition_list closed_proposition
        #                         | closed_proposition"""
        if len(p) == 4:
            p[0] = p[1] + [p[3]]
        elif len(p) == 3:
            p[0] = p[1] + [p[2]]
        else:
            p[0] = [p[1]]

    def p_morpheme(self, p):
        """morpheme : LPAREN word_sum RPAREN"""
        p[0] = sorted(p[2])

    def p_topic(self, p):
        """topic : LBRACKET morpheme RBRACKET
                | LBRACKET morpheme RBRACKET literal_list
                | LBRACKET morpheme TIMES morpheme RBRACKET
                | LBRACKET morpheme TIMES morpheme RBRACKET literal_list"""

        if len(p) == 4:
            p[0] = Topic(root=tuple(p[2]), flexing=())
        elif len(p) == 5:
            p[0] = Topic(root=tuple(p[2]), flexing=(), literals=p[4])
        elif len(p) == 6:
            p[0] = Topic(root=tuple(p[2]), flexing=tuple(p[4]))
        else:
            p[0] = Topic(root=tuple(p[2]), flexing=tuple(p[4]), literals=p[6])

    # def p_decorated(self, p):
    #     """decorated_word : word
    #                        | word text_list
    #         decorated_sentence : sentence
    #                             | sentence text_list
    #         decorated_supersentence : supersentence
    #                                 | supersentence text_list"""
    #     if len(p) == 2:
    #         p[0] = p[1]
    #     else:
    #         p[0] = _hyperlink(p[1], p[2])

    def p_clause(self, p):
        """clause : LPAREN topic TIMES topic TIMES topic RPAREN"""

        # """clause : LPAREN decorated_word TIMES decorated_word TIMES decorated_word RPAREN"""
        p[0] = (p[2], p[4], p[6])

    def p_fact(self, p):
        """fact : LBRACKET clauses_sum RBRACKET
                | LBRACKET clauses_sum RBRACKET literal_list"""
        if len(p) == 4:
            p[0] = Fact(p[2])
        else:
            p[0] = Fact(p[2], literals=p[4])

    def p_superclause(self, p):
        """superclause : LPAREN fact TIMES fact TIMES fact RPAREN"""
        # """theory : LPAREN decorated_sentence TIMES decorated_sentence TIMES decorated_sentence RPAREN"""
        p[0] = (p[2], p[4], p[6])

    def p_theory(self, p):
        """theory : LBRACKET superclauses_sum RBRACKET
                  | LBRACKET superclauses_sum RBRACKET literal_list"""
        if len(p) == 4:
            p[0] = Theory(p[2])
        else:
            p[0] = Theory(p[2], literals=p[4])

    def p_closed_proposition(self, p):
        """ closed_proposition : topic
                               | fact
                               | theory"""
        p[0] = p[1]

    def p_closed_proposition_list(self, p):
        """ closed_proposition_list :  closed_proposition_list SLASH SLASH closed_proposition
                                    | closed_proposition"""
        if len(p) == 2:
            p[0] = [p[1]]
        else:
            p[0] = p[1] + [p[4]]

    def p_text(self, p):
        """text : SLASH closed_proposition_list SLASH"""
        p[0] = Text(p[2])

        # if p[2][1]:
        #     raise NotImplementedError("Ieml doesn't support hypertext parsing for the moment.")
            # tuple of the hyperlink (end text, path)
            # self.hyperlinks += [Hyperlink(p[0][0], e[0], PropositionPath(e[1])) for e in p[2][1]]

    # def p_text_list(self, p):
    #     """text_list : text_list text
    #                 | text"""
    #     if len(p) == 3:
    #         p[0] = _add(p[1], p[2])
    #     else:
    #         p[0] = _add(([], []), p[1])

    def p_error(self, p):
        if p:
            msg = "Syntax error at '%s' (%d, %d)" % (p.value, p.lineno, p.lexpos)
        else:
            msg = "Syntax error at EOF"

        raise CannotParse(None, msg)


import itertools as it
import numpy as np
from bidict import bidict

from .script import MultiplicativeScript, Script, AdditiveScript, remarkable_multiplication_lookup_table

def old_canonical(script_ast):
    result = ''
    for byte in script_ast.canonical:
        result += chr(byte + ord('a') - 1)
    return [result]

def factor(sequences):
    layer = next(iter(sequences)).layer

    if layer == 0:
        return list(sequences)

    if len(sequences) == 1:
        return list(sequences)

    # holds the attributes/substances/modes as individual sets in primitives[0]/primitives[1]/primitives[2] respectively
    primitives = (set(seme) for seme in zip(*sequences))

    # same but now there is a bijection between the coordinate system and the primitives semes
    primitives = [bidict({i: s for i, s in enumerate(p_set)}) for p_set in primitives]

    # hold the mapping coordinate -> parser
    scripts = {tuple(primitives[i].inv[seme] for i, seme in enumerate(s)):s for s in sequences}

    # hold the primitive as coodinate described in scripts keys
    shape = tuple(len(p) for p in primitives)
    topology = np.full(shape, False, dtype=bool)
    for s in scripts:
        topology[s[0]][s[1]][s[2]] = True

    # calculate the relations, ie for a seq, the others seq that can be factorized with it
    relations = {}
    _computed = set()
    for seq in scripts:
        if not topology[seq[0]][seq[1]][seq[2]]:
            continue

        cubes = {e for e in _computed if
                 topology[e[0]][seq[1]][seq[2]] and
                 topology[seq[0]][e[1]][seq[2]] and
                 topology[seq[0]][seq[1]][e[2]]}

        for c in cubes:
            relations[c].add(seq)

        relations[seq] = cubes
        _computed.add(seq)

    def _neighbours(t1, t2):
        x1, y1, z1 = t1
        x2, y2, z2 = t2
        yield x1, y1, z1
        yield x1, y1, z2
        yield x1, y2, z1
        yield x1, y2, z2
        yield x2, y1, z1
        yield x2, y1, z2
        yield x2, y2, z1
        yield x2, y2, z2

    def _factors(candidate, factorisation):
        # sorting the list of candidate to get the one with the most of potential factors
        candidate.sort(key=lambda e: len(relations[e]), reverse=True)

        for r in candidate:
            _facto = set(it.chain.from_iterable(_neighbours(t, r) for t in factorisation))
            _candidate = set(candidate)
            for i in _facto:
                _candidate &= set(relations[i])

            if _candidate:
                yield from _factors(list(_candidate), _facto)
            else:
                yield _facto

        yield factorisation

    _candidate = [r for r in relations]
    _candidate.sort(key=lambda e: len(relations[e]))

    e = _candidate.pop()
    factorisations = next(iter(_factors(list(relations[e]), [e])))

    remaining = set(sequences) - set(scripts[f] for f in factorisations)
    factorisations = tuple(factor({primitives[i][seme] for seme in semes}) for i, semes in enumerate(zip(*factorisations)))

    if remaining:
        return [factorisations] + factor(remaining)
    else:
        return [factorisations]

def pack_factorisation(facto_list):
    """
    :param facto_list: list of parser or tuple of factorisation
    :return:
    """
    _sum = []
    for f in facto_list:
        if isinstance(f, Script):
            _sum.append(f)
        else:
            # tuple of factorisation
            _sum.append(MultiplicativeScript(children=(pack_factorisation(l_f) for l_f in f)))

    if len(_sum) == 1:
        return _sum[0]
    else:
        return AdditiveScript(children=_sum)

def factorize(script):
    if isinstance(script, Script):
        seqs = script.singular_sequences
    elif isinstance(script, list) or hasattr(script, '__iter__'):
        seqs = list(it.chain.from_iterable(s.singular_sequences for s in script))
    else:
        raise ValueError

    result = pack_factorisation(factor(seqs))
    return result

def inverse_relation(relation_name):
    from ..relations import INVERSE_RELATIONS
    return INVERSE_RELATIONS[relation_name]


import logging
from collections import OrderedDict, defaultdict
from itertools import groupby, combinations, permutations, chain, repeat

import numpy as np
from scipy.sparse.coo import coo_matrix
from scipy.sparse.csr import csr_matrix
from scipy.sparse.dok import dok_matrix

from ieml.commons import cached_property
from ieml.dictionary.script.script import MultiplicativeScript, AdditiveScript, NullScript

logger = logging.getLogger(__name__)
RELATIONS = [
            'contains',         # 0
            'contained',        # 1
            'father_substance', # 2
            'child_substance',  # 3
            'father_attribute', # 4
            'child_attribute',  # 5
            'father_mode',      # 6
            'child_mode',       # 7
            'opposed',          # 8
            'associated',       # 9
            'crossed',          # 10
            'twin',             # 11
            'table_0',
            'table_1',
            'table_2',
            'table_3',
            'table_4',
            'table_5',
            'identity',  # -1

             # 'inclusion',        # 12
             # 'father',           # 13
             # 'child',            # 14
             # 'etymology',        # 15
             # 'siblings',         # 16
             # 'table'             # 17
             ]

INVERSE_RELATIONS = {
    'father_substance': 'child_substance',
    'child_substance': 'father_substance',  # 3
    'father_attribute': 'child_attribute', # 4
    'child_attribute': 'father_attribute',  # 5
    'father_mode': 'child_mode',      # 6
    'child_mode': 'father_mode',
    'contains': 'contained',
    'contained': 'contains',
    'opposed':'opposed',          # 8
    'associated':'associated',       # 9
    'crossed': 'crossed',        # 10
    'twin': 'twin',
    'table_0': 'table_0',
    'table_1': 'table_1',
    'table_2': 'table_2',
    'table_3': 'table_3',
    'table_4': 'table_4',
    'table_5': 'table_5',
    'father': 'child',
    'child': 'father',
    'inclusion': 'inclusion',
    'etymology': 'etymology',        # 15
    'siblings': 'siblings',         # 16
    'table': 'table',
    'identity': 'identity'
}

class RelationsGraph:
    def __init__(self, dictionary):
        super().__init__()

        self.dictionary = dictionary
        self.relations = None
        self._compute_relations()

    def __getitem__(self, item):
        if isinstance(item, str) and item in RELATIONS:
            return self.relations[item]

        if isinstance(item, int):
            return self.relations[RELATIONS[item]]

        from .terms import Term
        if isinstance(item, Term):
            if item.dictionary != self.dictionary:
                raise ValueError("Invalid dictionary, (%s/%s)"%(str(item.dictionary), str(self.dictionary)))
            return Relations(term=item, relations_graph=self)

        raise NotImplemented

    @cached_property
    def connexity(self):
        """
        A boolean matrix, m[i, j] == True if there is a relation term(i) -> term(j)
        :return: a np.matrix (len(dictionary), len(dictionary)) of boolean
        """
        return np.matrix(sum(self.relations.values()).todense(), dtype=bool)

    def relation_type(self, term, relation_type):
        return [self.dictionary.index[j] for j in
                self.relations[relation_type][term.index, :].indices]

    def neighbours(self, term):
        return {
            reltype: self.relation_type(term, reltype) for reltype in RELATIONS
        }

    def _compute_relations(self):
        logger.log(logging.INFO, "Computing relations")

        self.relations = {}
        contains = self._compute_contains()
        self.relations['contains'] = csr_matrix(contains)
        self.relations['contained'] = csr_matrix(self.relations['contains'].transpose())

        father = self._compute_father()

        for i, r in enumerate(['_substance', '_attribute', '_mode']):
            self.relations['father' + r] = dok_matrix(father[i])

        siblings = self._compute_siblings()
        self.relations['opposed'] = dok_matrix(siblings[0])
        self.relations['associated'] = dok_matrix(siblings[1])
        self.relations['crossed'] = dok_matrix(siblings[2])
        self.relations['twin'] = dok_matrix(siblings[3])

        # self._do_inhibitions()

        for i, r in enumerate(['_substance', '_attribute', '_mode']):
            self.relations['child' + r] = self.relations['father' + r].transpose()

        # self.relations['siblings'] = sum(siblings)
        # self.relations['inclusion'] = np.clip(self.relations['contains'] + self.relations['contained'], 0, 1)
        # self.relations['father'] = self.relations['father_substance'] + \
        #                            self.relations['father_attribute'] + \
        #                            self.relations['father_mode']
        # self.relations['child'] = self.relations['child_substance'] + \
        #                           self.relations['child_attribute'] + \
        #                           self.relations['child_mode']
        # self.relations['etymology'] = self.relations['father'] + self.relations['child']

        table = self._compute_table_rank(self.relations['contained'])
        for i in range(6):
            self.relations['table_%d'%i] = table[i]

        self.relations['identity'] = csr_matrix(np.eye(len(self.dictionary)))

        missing = {s for s in RELATIONS if s not in self.relations}
        if missing:
            raise ValueError("Missing relations : {%s}"%", ".join(missing))

        self.relations = {reltype: csr_matrix(self.relations[reltype]) for reltype in RELATIONS}

    def _compute_table_rank(self, contained):
        logger.log(logging.DEBUG, "Computing tables relations")

        tables_rank = [([], []) for _ in range(6)]

        indices = [
            set(l) for l in np.split(contained.indices, contained.indptr)[1:-1]
        ]

        for root in self.dictionary.roots:
            for t0, t1 in combinations(self.dictionary.roots[root], 2):
                commons = [self.dictionary.index[i] for i in indices[t0.index] & indices[t1.index]]

                ranks = set(map(lambda t: t.rank, commons))
                for rank in ranks:
                    tables_rank[rank][0].extend((t0.index, t1.index))
                    tables_rank[rank][1].extend((t1.index, t0.index))

        for t in self.dictionary:
            ranks = {self.dictionary.index[i].rank for i in indices[t.index]} - {6}
            for rank in ranks:
                tables_rank[rank][0].append(t.index)
                tables_rank[rank][1].append(t.index)

        return [coo_matrix(([True]*len(i), (i, j)), shape=self.shape, dtype=np.bool) for i, j in tables_rank]

    def _compute_contains(self):
        logger.log(logging.DEBUG, "Computing contains/contained relations")
        # contain/contained

        i = list(range(len(self.dictionary)))
        j = list(range(len(self.dictionary)))
        for r_p, v in self.dictionary.roots.items():
            paradigms = {t for t in v if t.script.paradigm}

            for p in paradigms:
                _contains = [self.dictionary.terms[ss].index for ss in p.script.singular_sequences] + \
                            [k.index for k in paradigms if k.script in p.script]
                i.extend(repeat(p.index, len(_contains)))
                j.extend(_contains)

        return coo_matrix(([True] * len(i), (i, j)), shape=self.shape, dtype=np.bool)

    def _compute_father(self):
        logger.log(logging.DEBUG, "Computing father/child relations")

        def _recurse_script(script):
            result = []
            for sub_s in script.children if isinstance(script, AdditiveScript) else [script]:
                if isinstance(sub_s, NullScript):
                    continue

                if sub_s in self.dictionary.terms:
                    result.append(self.dictionary.terms[sub_s].index)
                else:
                    if sub_s.layer > 0:
                        result.extend(chain.from_iterable(_recurse_script(c) for c in sub_s.children))

            return result

        # father = coo_matrix((3, len(self.dictionary), len(self.dictionary)), dtype=np.bool)

        father = [([], []) for _ in range(3)]

        for t in self.dictionary.terms.values():
            s = t.script

            for sub_s in s if isinstance(s, AdditiveScript) else [s]:
                if len(sub_s.children) == 0 or isinstance(sub_s, NullScript):
                    continue

                for i, s in enumerate(('father_substance', 'father_attribute', 'father_mode')):
                    if s in t.inhibitions:
                        continue

                    fathers_indexes = _recurse_script(sub_s.children[i])
                    father[i][0].extend(repeat(t.index, len(fathers_indexes)))
                    father[i][1].extend(fathers_indexes)

        return [coo_matrix(([True] * len(i), (i, j)), shape=self.shape, dtype=np.bool) for i, j in father]

    def _compute_siblings(self):
        # siblings
        # 1 dim => the sibling type
        #  -0 opposed
        #  -1 associated
        #  -2 crossed
        #  -3 twin
        def _opposed_sibling(s0, s1):
            return not s0.empty and not s1.empty and\
                   s0.cardinal == s1.cardinal and\
                   s0.children[0] == s1.children[1] and s0.children[1] == s1.children[0]

        def _associated_sibling(s0, s1):
            return s0.cardinal == s1.cardinal and\
                   s0.children[0] == s1.children[0] and \
                   s0.children[1] == s1.children[1] and \
                   s0.children[2] != s1.children[2]

        def _crossed_sibling(s0, s1):
            return s0.layer >= 2 and \
                   s0.cardinal == s1.cardinal and \
                   _opposed_sibling(s0.children[0], s1.children[0]) and \
                   _opposed_sibling(s0.children[1], s1.children[1])

        siblings = [([], []) for _ in range(4)]

        logger.log(logging.DEBUG, "Computing siblings relations")

        for root in self.dictionary.roots:
            _inhib_opposed = 'opposed' not in root.inhibitions
            _inhib_associated = 'associated' not in root.inhibitions
            _inhib_crossed = 'crossed' not in root.inhibitions
            _inhib_twin = 'twin' not in root.inhibitions

            if root.script.layer == 0:
                continue
            _twins = []

            for i, t0 in enumerate(self.dictionary.roots[root]):
                if not isinstance(t0.script, MultiplicativeScript):
                    continue

                if t0.script.children[0] == t0.script.children[1]:
                    _twins.append(t0)

                for t1 in [t for j, t in enumerate(self.dictionary.roots[root])
                           if j > i and isinstance(t.script, MultiplicativeScript)]:

                    if _inhib_opposed and _opposed_sibling(t0.script, t1.script):
                        siblings[0][0].extend((t0.index, t1.index))
                        siblings[0][1].extend((t1.index, t0.index))

                    if _inhib_associated and _associated_sibling(t0.script, t1.script):
                        siblings[1][0].extend((t0.index, t1.index))
                        siblings[1][1].extend((t1.index, t0.index))

                    if _inhib_crossed and _crossed_sibling(t0.script, t1.script):
                        siblings[2][0].extend((t0.index, t1.index))
                        siblings[2][1].extend((t1.index, t0.index))

            if _inhib_twin:
                _twins = sorted(_twins, key=lambda t: t.script.cardinal)
                for card, g in groupby(_twins, key=lambda t: t.script.cardinal):
                    twin_indexes = [t.index for t in g]

                    if len(twin_indexes) > 1:
                        index0, index1 = list(zip(*permutations(twin_indexes, r=2)))
                        siblings[3][0].extend(index0)
                        siblings[3][1].extend(index1)

        return [coo_matrix(([True]*len(i), (i, j)), shape=self.shape, dtype=np.bool) for i, j in siblings]

    @property
    def shape(self):
        return (len(self.dictionary), len(self.dictionary))

    def __setstate__(self, state):
        self.relations = state['relations']
        self.dictionary = state['dictionary']

    def __getstate__(self):
        return {
            'relations': self.relations,
            'dictionary': self.dictionary
        }

class Relations:
    def __init__(self, term, relations_graph):
        super().__init__()

        self.relations_graph = relations_graph
        self.term = term

    @cached_property
    def neighbours(self):
        rels = defaultdict(list)
        for reltype in RELATIONS:
            for t in self[reltype]:
                rels[t].append(reltype)

        neighbours = OrderedDict()

        for t in sorted(rels):
            neighbours[t] = rels[t]

        return neighbours

    def to(self, term, relations_types=None):
        from ieml.grammar.word import Word
        if isinstance(term, Word):
            term = term.term

        if relations_types is None:
            relations_types = RELATIONS

        if term not in self.neighbours:
            return []

        return [reltype for reltype in self.neighbours[term] if reltype in relations_types]

    def __iter__(self):
        return self.neighbours.__iter__()

    def __len__(self):
        return len(self.neighbours)

    def __contains__(self, item):
        return item in self.neighbours

    def __getitem__(self, item):
        if isinstance(item, int):
            item = RELATIONS[item]

        if isinstance(item, str):
            return getattr(self, item)

        raise NotImplemented

    @property
    def father(self):
        return {
            's': self.father_substance,
            'a': self.father_attribute,
            'm': self.father_mode,
        }

    @property
    def child(self):
        return {
            's': self.child_substance,
            'a': self.child_attribute,
            'm': self.child_mode,
        }

def get_relation(reltype):
    def getter(self):
        return tuple(self.relations_graph.relation_type(term=self.term, relation_type=reltype))

    getter.__name__ = reltype
    return getter

for reltype in {'contains',
                'contained',
                'father_substance',
                'child_substance',
                'father_attribute',
                'child_attribute',
                'father_mode',
                'child_mode',
                'opposed',
                'associated',
                'crossed',
                'twin',
                'table_0',
                'table_1',
                'table_2',
                'table_3',
                'table_4',
                'table_5',
                'identity'}:
    setattr(Relations, reltype, cached_property(get_relation(reltype)))


from collections import defaultdict

import numpy

from ieml.exceptions import InvalidPathException
from ieml.grammar import topic, text, fact, theory
from ieml.tools import ieml
from ...exceptions import InvalidIEMLObjectArgument
from ...grammar import Theory, Fact, Text, Word, Topic
from ...exceptions import IEMLObjectResolutionError, ResolveError
from .parser import PathParser
from .paths import Path, Coordinate, MultiplicativePath, AdditivePath, ContextPath

def path(p):
    if isinstance(p, Path):
        return p
    if isinstance(p, str):
        return PathParser().parse(p)

    try:
        l = list(p)
        if all(isinstance(e, Path) for e in l):
            res = l[0]
            for p in l[1:]:
                res *= p
            return res

    except TypeError:
        pass

    raise ValueError("Invalid argument to create a path.")

def _resolve_path_tree_graph(tree_graph, path):
    if isinstance(path, Coordinate):
        coords = [path]
    else:
        coords = list(path.children)

    c0 = coords[0]
    if c0.kind == 's':
        stack = {tree_graph.root}
    elif c0.kind == 'a':
        # tous les attributs
        stack = {attr[0] for vals in tree_graph.transitions.values() for attr in vals}
    else:
        # tous les modes
        return {attr[1][2] for vals in tree_graph.transitions.values() for attr in vals}

    for c in coords[1:]:
        _stack = set()
        for s in stack:
            if c.kind == 's':
                raise InvalidPathException(tree_graph, path, "double substance 's' (root node).")

            if c.kind == 'm':
                if c.index is not None:
                    _stack.add(tree_graph.transitions[s][c.index][1][2])
                else:
                    _stack |= {vals[1][2] for vals in tree_graph.transitions[s]}

            if c.kind == 'a':
                if c.index is not None:
                    _stack.add(tree_graph.transitions[s][c.index][0])
                else:
                    _stack |= {vals[0] for vals in tree_graph.transitions[s]}

        if not _stack:
            return set()

        stack = _stack

    return stack

def _resolve_path(obj, path):
    """path is a mul of coord or a coord"""
    if obj.__class__ not in path.context.accept:
        result = set()
        for ctx in path.context.accept:
            result |= {e for u in obj[ctx] for e in _resolve_path(u, path)}

        return result

    if isinstance(obj, Text):
        if path.index is not None:
            return {obj.children[path.index]}

        return set(obj.children)

    if isinstance(obj, (Fact, Theory)):
        return _resolve_path_tree_graph(obj.tree_graph, path)

    if isinstance(obj, Topic):
        if path.kind == 'r':
            if path.index is not None:
                return {obj.root[path.index]}
            return set(obj.root)
        else:
            if path.index is not None:
                return {obj.flexing[path.index]}
            return set(obj.flexing)

def resolve(usl, path):
    """"""

    # if usl.__class__ not in path.context.accept:
    #     raise InvalidPathException(usl, path, "invalid path context, expected types: {%s}."%', '.join(path.context.accept))

    result = set()
    for d in path.develop:
        if isinstance(d, (Coordinate, MultiplicativePath)):
            result |= _resolve_path(usl, d)
        else:
            # context path
            stack = {usl}
            for c in d.children:
                _stack = set()
                for s in stack:
                    _stack |= _resolve_path(s, c)

                if not _stack:
                    break

                stack = _stack
            else:
                result |= stack

    return result

def _enumerate_paths(usl, level):
    if isinstance(usl, level):
        yield [], usl

    if isinstance(usl, Text):
        for i, t in enumerate(usl.children):
            for p, e in _enumerate_paths(t, level=level):
                yield [path('t%d'%i)] + p, e

    if isinstance(usl, (Theory)):
        for node in usl.facts:
            for p, e in _enumerate_paths(node, level=level):
                yield [_tree_graph_path_of_node(usl.tree_graph, node)] + p, e

    if isinstance(usl, (Fact)):
        for node in usl.topics:
            for p, e in _enumerate_paths(node, level=level):
                yield [_tree_graph_path_of_node(usl.tree_graph, node)] + p, e

    if isinstance(usl, Topic):
        for i, t in enumerate(usl.root):
            for p, e in _enumerate_paths(t, level=level):
                yield [path('r%d'%i)] + p, e

        if usl.flexing:
            for i, t in enumerate(usl.flexing):
                for p, e in _enumerate_paths(t, level=level):
                    yield [path('f%d' % i)] + p, e

    return

def _tree_graph_path_of_node(tree_graph, node):
    if node in tree_graph.nodes:
        nodes = [(node, False)]
    else:
        nodes = []

    # can be a mode
    nodes += [(c[0], True) for c_list in tree_graph.transitions.values() for c in c_list if c[1][2] == node]
    if not nodes:
        raise ValueError("Node not in tree graph : %s" % str(node))

    def _build_coord(node, mode=False):
        if node == tree_graph.root:
            return [Coordinate(kind='s')]

        parent = tree_graph.nodes[numpy.where(tree_graph.array[:, tree_graph.nodes_index[node]])[0][0]]

        return _build_coord(parent) + \
               [Coordinate(index=[c[0] for c in tree_graph.transitions[parent]].index(node), kind='m' if mode else 'a')]

    return AdditivePath([MultiplicativePath(_build_coord(node, mode)) for node, mode in nodes])

def enumerate_paths(ieml_obj, level=Word):
    for p, t in _enumerate_paths(ieml_obj, level=level):
        if len(p) == 1:
            yield p[0], t
        elif not p:
            yield None, t
        else:
            yield ContextPath(p), t

def _build_deps_tree_graph(rules):
    def _node():
        return {
            'resolve': defaultdict(_node),  # the indexed coordinate
            'context': [],
            'rules': defaultdict(_node)
        }

    # contain s, a et m (relative path to the root)
    roots = defaultdict(_node)

    s = Coordinate(kind='s')

    for p, e in rules:
        if isinstance(p, ContextPath):
            actual_P = p.children[0]  # multiplicative or coordinate
            ctx_P = path(p.children[1:])
        else:
            actual_P = p
            ctx_P = None

        if isinstance(actual_P, MultiplicativePath):
            actual_P = list(actual_P.children)
        else:
            actual_P = [actual_P]

        # actual_p is the list of coordinate to navigate
        # ctx_p is the rest of the path or None

        current_node = roots

        # replacing s0 -> s
        if actual_P[0].kind == 's' and actual_P[0].index == 0:
            actual_P[0] = s

        first = True
        for c in actual_P:
            if c.index is not None:
                categorie = 'resolve'
            else:
                categorie = 'rules'

            if first:
                current_node = current_node[c]
                first = False
            else:
                current_node = current_node[categorie][c]

        # the nodes with ctx_P is None will be instanciate
        current_node['context'].append((ctx_P, e))

    result = []

    def _merge_nodes(n0, n1):
        for cat in ('resolve', 'rules'):
            for r in n1[cat]:
                if r in n0[cat]:
                    _merge_nodes(n0[cat][r], n1[cat][r])
                else:
                    n0[cat][r] = n1[cat][r]

        # merge ctx
        n0['context'] += n1['context']

    coord_a = Coordinate('a')
    coord_m = Coordinate('m')

    # apply the rules on the tree starting from s
    def _apply_rule(node, _path, mode=False):
        # we add the globals rules to its own
        _merge_nodes(node['rules'][coord_a], roots[coord_a])
        _merge_nodes(node['rules'][coord_m], roots[coord_m])

        obj = _resolve_ctx(_path, node['context'])
        error = obj is None

        if mode or not node['resolve']:
            return error, obj

        # resolve the children
        max_i = max(n.index for n in node['resolve'])

        _result = {
            'a': [],
            'm': []
        }

        for r in ('a', 'm'):
            indexed = {k.index: node['resolve'][k] for k in node['resolve'] if k.kind == r}
            rules = [k for k in node['rules'] if k.kind == r]

            if max(indexed, default=-1) + 1 > len(indexed) + (1 if rules else 0):
                raise ResolveError("Not enough information to instantiate all the %s rules. At %s."%(r, _path))

            for i in range(max_i + 1):
                if i in indexed:
                    child = indexed[i]
                else:
                    child = _node()

                _merge_nodes(child, node['rules'][r])

                e, n = _apply_rule(child, _path + "%s%d"%(r, i), mode=r == 'm')
                error |= e
                _result[r].append(n)

        for a, m in zip(_result['a'], _result['m']):
            result.append((obj, a, m))

        return error, obj
    # 'context' attribute is now the sub element
    # generating the triplet from the root s

    error, o = _apply_rule(roots[s], 's0')

    return error, result[::-1]

def _build_deps_text(rules):
    indexes = defaultdict(list)
    generals = []

    for p, e in rules:
        if isinstance(p, ContextPath):
            actual_P = p.children[0]  # coordinate
            ctx_P = path(p.children[1:])
        else:
            actual_P = p
            ctx_P = None

        if not isinstance(actual_P, Coordinate):
            raise ResolveError("A text must be defined by a coordinate, not %s"%actual_P)

        if actual_P.index is not None:
            indexes[actual_P.index].append((ctx_P, e))
        else:
            generals.append((ctx_P, e))

    if not indexes:
        return False, []

    if max(indexes, default=-1) + 1 - len(indexes) > 1:
        # if there is more than one missing
        raise ResolveError("Index missing on text definition.")

    error = False
    i = 0
    result = []
    while i <= max(indexes):
        ctx_rules = generals[:]
        if i in indexes:
            ctx_rules.extend(indexes[i])

        node = _resolve_ctx("t%d"%i, ctx_rules)
        error |= node is None
        result.append(node)
        i += 1

    return error, result

def _build_deps_topic(rules):
    result = {
        'r': {
            'indexes': defaultdict(list),
            'generals': []
        },
        'f': {
            'indexes': defaultdict(list),
            'generals': []
        }
    }

    for p, e in rules:
        if isinstance(p, ContextPath):
            # should be an error but support it
            actual_P = p.children[0]  # coordinate
            ctx_P = path(p.children[1:])
        else:
            actual_P = p
            ctx_P = None

        if not isinstance(actual_P, Coordinate):
            raise ResolveError("A topic must be defined by a single coordinate.")

        if actual_P.index is not None:
            result[actual_P.kind]['indexes'][actual_P.index].append((ctx_P, e))
        else:
            result[actual_P.kind]['generals'].append((ctx_P, e))

    error = False
    for k in result:
        # k for kind (r or f)

        indexes = result[k]['indexes']
        generals = result[k]['generals']

        if not indexes and not generals:
            result[k] = []
            continue

        if max(indexes, default=-1) + 1 > len(indexes) + len(generals):
            # if there is more than one missing
            raise ResolveError("Index missing on topic definition.")

        current = []
        length = len(indexes) + len(generals)
        generals = generals.__iter__()
        for i in range(length):
            if i in indexes:
                ctx_rules = indexes[i]
            else:
                ctx_rules = [next(generals)]

            node = _resolve_ctx("%s%d"%(k, i), ctx_rules)
            error |= node is None

            current.append(node)

        result[k] = current

    return error, result

def _inferred_types(path, e):
    result = set()
    for inf in path.context.switch:
        if isinstance(e, tuple(path.context.switch[inf])):
            result.add(inf)

    if result:
        return result

    raise ResolveError("No compatible type found with the path %s and the ieml object of type %s"%
                     (str(path), e.__class__.__name__))

_errors = []
_context_stack = []
debug = False

def _context_error_handler(func):
    def wrapper(_path, rules):
        _context_stack.append(_path)

        result = None
        if debug:
            result = func(rules)
        else:
            try:
                result = func(rules)
            except ResolveError as e:
                _errors.append((':'.join(_context_stack[1:]), str(e)))
            except InvalidIEMLObjectArgument as e:
                _errors.append((':'.join(_context_stack[1:]), str(e)))

        _context_stack.pop()

        return result

    return wrapper

@_context_error_handler
def _resolve_ctx(rules):
    """
    Resolve the context of the rules (the type of this element), and building the ieml element.
    :param rules:
    :return:
    """
    if not rules:
        raise ResolveError("Missing node definition.")

    # if rules == [(None, e)] --> e
    if len(rules) == 1 and rules[0][0] is None:
        return rules[0][1]

    if any(r[0] is None for r in rules):
        raise ResolveError("Multiple definition, multiple ieml object provided for the same node.")

    if any(not isinstance(r[0], Path) for r in rules):
        raise ResolveError("Must have only path instance.")

    # resolve all the possible types for this element
    r0 = rules[0]
    types = _inferred_types(*r0)
    for r in rules[1:]:
        types = types.intersection(_inferred_types(*r))

    if not types:
        raise ResolveError("No definition, no type inferred on rules list.")

    if len(types) > 1:
        raise ResolveError("Multiple definition, multiple type inferred on rules list.")

    type = next(types.__iter__())

    if type == Topic:
        error, deps = _build_deps_topic(rules)
        if error:
            return

        flexing = None
        if deps['f']:
            flexing = deps['f']
        if not deps['r']:
            raise ResolveError("No root for the topic node.")

        return topic(deps['r'], flexing)

    if type == Text:
        error, deps = _build_deps_text(rules)
        if error:
            return

        return text(deps)

    if type in (Theory, Fact):
        error, deps = _build_deps_tree_graph(rules)
        if error:
            return

        if type == Fact:
            clauses = []
            for s, a, m in deps:
                clauses.append((s, a, m))
            return fact(clauses)
        else:
            clauses = []
            for s, a, m in deps:
                clauses.append((s, a, m))
            return theory(clauses)

    raise ResolveError("Invalid type inferred %s"%type.__name__)

def resolve_ieml_object(paths, elements=None):
    global _errors, _context_stack
    _errors = []
    _context_stack = []

    if isinstance(paths, dict):
        paths = list(paths.items())

    if elements is None:
        result = _resolve_ctx('', [(d, ieml(e)) for p, e in paths for d in path(p).develop])
    else:
        result = _resolve_ctx('', [(d, ieml(e)) for e, p in zip(elements, paths) for d in path(p).develop])

    if _errors:
        raise IEMLObjectResolutionError(_errors)

    return result


from collections import defaultdict
from itertools import chain

from ieml.dictionary.table import Cell, Table3D, TableSet
from ieml.dictionary.terms import Term

# from ..tools import ieml
# from .template import Template

# template = Template(ieml("[([M:M:.a.-]+[M:.-',M:.-',S:.-'B:.-'n.-S:.U:.-',_])*([E:U:T:.]+[E:U:.wa.-])]"), ['r0', 'r1'])
# collection = list(template)
# inverse_terms = defaultdict(list)
#
# for u in collection:
#     for t in u.paths.values():
#         inverse_terms[t].append(u)
#
# inverse_root = defaultdict(list)
# for t in inverse_terms:
#     inverse_root[t.root].extend(inverse_terms[t])
#
# roots = sorted(inverse_root, key=lambda t: len(inverse_root[t]), reverse=True)

def project_usls_on_dictionary(usls, allowed_terms=None):
    """`usls` is an iterable of usl.

    return a mapping term -> usl list
    """

    cells_to_usls = defaultdict(set)
    tables = set()

    for u in usls:
        for t in u.objects(Term):
            for c in t.singular_sequences:
                # This is the first time we meet the cell c
                if not cells_to_usls[c]:
                    tables.update(c.relations.contained)

                cells_to_usls[c].add(u)

    if allowed_terms:
        allowed_terms = set(allowed_terms)
        tables = tables & allowed_terms
        cells_to_usls = {c: l for c, l in cells_to_usls.items() if c in allowed_terms}

    tables_to_usls = {
        table: list(set(u for c in table.singular_sequences for u in cells_to_usls[c]))
            for table in tables if not isinstance(table, TableSet)
    }

    return tables_to_usls

def project_usl_with_data(usls_data, metric=None):
    """
    usls_data: usl => data[]
    :param usls_data:
    :return:
    """

    projection = project_usls_on_dictionary(usls_data)
    all_terms = set(c for u in usls_data for t in u.objects(Term) for c in t.singular_sequences)
    if metric is None:
        metric = lambda e: len(e['posts']) * len(all_terms.intersection(e['table'].singular_sequences))

    return sorted(({
        'table': table,
        'usls': usls,
        'posts': list(set(chain.from_iterable(usls_data[u] for u in usls)))
    } for table, usls in projection.items()), key=metric, reverse=True)


import logging
import types
from functools import lru_cache
import os
import ply.yacc as yacc

from ieml.exceptions import InvalidScript
from ....exceptions import CannotParse
from ..script import AdditiveScript, MultiplicativeScript, NullScript
from ....constants import REMARKABLE_ADDITION
from ....commons import Singleton

from .lexer import get_script_lexer, tokens

from .... import parser_folder
import threading

class ScriptParser(metaclass=Singleton):
    tokens = tokens

    # ply have an internal state , then we forbid two thread try to parse a string simultaneously
    lock = threading.Lock()

    def __init__(self):
        self.t_add_rules()

        self.lexer = get_script_lexer()
        self.parser = yacc.yacc(module=self, errorlog=logging, start='term',
                                debug=False, optimize=True, picklefile=os.path.join(parser_folder, "script_parser.pickle"))
        # rename the parsing method (can't name it directly parse with lru_cache due to ply checking)
        self.parse = self.t_parse

    @lru_cache(maxsize=10000)
    def t_parse(self, s):
        with self.lock:
            try:
                return self.parser.parse(s, lexer=self.lexer)
            except InvalidScript as e:
                raise CannotParse(s, str(e))

    def p_error(self, p):
        if p:
            msg = "Syntax error at '%s' (%d, %d)" % (p.value, p.lineno, p.lexpos)
        else:
            msg = "Syntax error at EOF"

        raise InvalidScript(msg)

    # Rules
    def p_term(self, p):
        """ term : script_lvl_0
                | additive_script_lvl_0
                | script_lvl_1
                | additive_script_lvl_1
                | script_lvl_2
                | additive_script_lvl_2
                | script_lvl_3
                | additive_script_lvl_3
                | script_lvl_4
                | additive_script_lvl_4
                | script_lvl_5
                | additive_script_lvl_5
                | script_lvl_6
                | additive_script_lvl_6 """
        p[0] = p[1]

    def p_script_lvl_0(self, p):
        """ script_lvl_0 : PRIMITIVE LAYER0_MARK
                            | REMARKABLE_ADDITION LAYER0_MARK"""
        if p[1] == 'E':
            p[0] = NullScript(layer=0)
        elif p[1] in REMARKABLE_ADDITION:
            p[0] = AdditiveScript(character=p[1])
        else:
            p[0] = MultiplicativeScript(character=p[1])

    def p_additive_script_lvl_0(self, p):
        """ additive_script_lvl_0 : sum_lvl_0"""
        p[0] = AdditiveScript(children=p[1])

    def p_sum_lvl_0(self, p):
        """ sum_lvl_0 : script_lvl_0
                    | script_lvl_0 PLUS sum_lvl_0"""
        if len(p) == 4:
            p[3].append(p[1])
            p[0] = p[3]
        else:
            p[0] = [p[1]]

    def p_script_lvl_1(self, p):
        """ script_lvl_1 : additive_script_lvl_0 LAYER1_MARK
                        | additive_script_lvl_0 additive_script_lvl_0 LAYER1_MARK
                        | additive_script_lvl_0 additive_script_lvl_0 additive_script_lvl_0 LAYER1_MARK
                        | REMARKABLE_MULTIPLICATION LAYER1_MARK"""
        if isinstance(p[1], AdditiveScript):
            if len(p) == 3:
                p[0] = MultiplicativeScript(substance=p[1])
            elif len(p) == 4:
                p[0] = MultiplicativeScript(substance=p[1],
                                            attribute=p[2])
            else:
                p[0] = MultiplicativeScript(substance=p[1],
                                            attribute=p[2],
                                            mode=p[3])
        else:
            p[0] = MultiplicativeScript(character=p[1])

    def p_sum_lvl_1(self, p):
        """ sum_lvl_1 : script_lvl_1
                    |  script_lvl_1 PLUS sum_lvl_1"""
        if len(p) == 4:
            p[3].append(p[1])
            p[0] = p[3]
        else:
            p[0] = [p[1]]

    def p_additive_script_lvl_1(self, p):
        """ additive_script_lvl_1 : sum_lvl_1 """
        p[0] = AdditiveScript(children=p[1])

    def t_add_rules(self):
        def rule_layer(layer):
            def _sum(self, p):
                if len(p) == 4:
                    p[3].append(p[1])
                    p[0] = p[3]
                else:
                    p[0] = [p[1]]

            def _additive(self, p):
                p[0] = AdditiveScript(children=p[1])

            def _script(self, p):
                if len(p) == 3:
                    p[0] = MultiplicativeScript(substance=AdditiveScript(p[1]))
                elif len(p) == 4:
                    p[0] = MultiplicativeScript(substance=AdditiveScript(p[1]),
                                                attribute=AdditiveScript(p[2]))
                else:
                    p[0] = MultiplicativeScript(substance=AdditiveScript(p[1]),
                                                attribute=AdditiveScript(p[2]),
                                                mode=AdditiveScript(p[3]))

            result = []
            # sum rule
            function = types.FunctionType(_sum.__code__, _sum.__globals__)
            function.__name__ = 'p_sum_lvl_%d'%layer
            function.__doc__ = """sum_lvl_%d : script_lvl_%d
                                            | script_lvl_%d PLUS sum_lvl_%d"""%((layer,)*4)
            result.append(function)

            # additive parser rule
            function = types.FunctionType(_additive.__code__, _additive.__globals__)
            function.__name__ = 'p_additive_script_lvl_%d'%layer
            function.__doc__ = "additive_script_lvl_%d : sum_lvl_%d "%((layer,)*2)
            result.append(function)

            function = types.FunctionType(_script.__code__, _script.__globals__)
            function.__name__ = 'p_script_lvl_%d'%layer
            function.__doc__ = """script_lvl_%d : sum_lvl_%d LAYER%d_MARK
                                    | sum_lvl_%d sum_lvl_%d LAYER%d_MARK
                                    | sum_lvl_%d sum_lvl_%d sum_lvl_%d LAYER%d_MARK """%(
                (layer, layer - 1, layer) +
                (layer-1,)*2 + (layer,) +
                (layer-1,)*3 + (layer,))

            result.append(function)

            return result

        for i in range(2, 7):
            for f in rule_layer(i):
                setattr(self.__class__, f.__name__, f)


from ieml.grammar import Usl
import numpy as np

def square_order_matrix(usl_list):
    """
    Compute the ordering of a list of usls from each usl and return the matrix m s.t.
    for each u in usl_list at index i, [usl_list[j] for j in m[i, :]] is the list sorted
    by proximity from u.
     of the result
    :param usl_list: a list of usls
    :return: a (len(usl_list), len(usl_list)) np.array

    """
    usl_list = list(usl_list)
    indexes = {
        u: i for i, u in enumerate(usl_list)
    }

    order_mat = np.zeros(shape=(len(usl_list), len(usl_list)), dtype=int)

    for u in usl_list:
        sorted_list = QuerySort(u).sort(collection=usl_list)
        for i, u_s in enumerate(sorted_list):
            order_mat[indexes[u], indexes[u_s]] = i

    return order_mat

class QuerySort:
    """order a collection of usl from a specific usl"""
    def __init__(self, usl):
        self.usl = usl

        assert isinstance(usl, Usl)

    def sort(self, collection):
        def sort_key(u):
            return self._proximity(u, lambda u: u.words)
            # self._proximity(u, lambda u: u.topics)

        return sorted(collection, key=sort_key)

    def _proximity(self, u, get_set):
        s0 = get_set(self.usl)
        s1 = get_set(u)
        sym_diff = len(s0 ^ s1)

        if sym_diff != 0:
            return len(s0.intersection(s1)) / sym_diff
        else:
            # same subject
            # use max of ( len(g0.intersection(g1)) / sym_diff ) + 1
            return len(s0) + 1


import numpy as np

from ieml.commons import cached_property
from ieml.dictionary.script import script
from .terms import Term
from .script import Script, factorize

class Table(Term):
    def __init__(self, script, index, dictionary, parent, regular=False):
        super().__init__(script, index, dictionary, parent)
        self.regular = regular

    @property
    def rank(self):
        if self.parent is None:
            return 0

        if isinstance(self.parent, TableSet):
            return max(self.parent.rank, 1)

        if self.regular:
            return max(self.parent.rank, 1) + 2
        else:
            return max(self.parent.rank, 1) + 1

    @cached_property
    def partitions(self):
        return {t for t in self.relations.contains if isinstance(t, Table) and t.parent == self}

class Cell(Term):
    @property
    def rank(self):
        return 6

class Table2D(Table):
    def __init__(self, script, index, dictionary, parent, regular=False):
        super().__init__(script, index, dictionary, parent, regular)

        if script.tables_script[0] != script or self.script.cells[0].shape[2] != 1 or self.script.cells[0].shape[1] == 1:
            raise ValueError("Invalid script for Table creation: %s. Expected a script that lead a 2d table"%str(script))

        self._index = None

    @property
    def shape(self):
        return self.cells.shape

    @cached_property
    def rows(self):
        return [self.dictionary.terms[row] if row in self.dictionary else None
                for row in self.script_rows]

    @cached_property
    def script_rows(self):
        return [factorize([t.script for t in line]) for line in self.cells]

    @cached_property
    def columns(self):
        return [self.dictionary.terms[column] if column in self.dictionary else None
                for column in self.script_columns]

    @cached_property
    def script_columns(self):
        return [factorize([t.script for t in line]) for line in self.cells.transpose()]

    @cached_property
    def cells(self):
        return np.vectorize(lambda sc: self.dictionary.terms[sc])(self.script.cells[0][:, :, 0])

    def __getitem__(self, item):
        return self.cells[item]

    def index_of(self, item):
        if self._index is None:
            self._index = {
                t.script: index for index, t in np.ndenumerate(self.cells)
            }

        return self._index[script(item)]

    def accept_script(self, script):
        """

        :param term:
        :return: is_accepting, is_regular
        """
        def is_connexe_tilling(coords):
            shape_t = self.shape
            # test if the table table is a connexe tiling of the table t
            size = 1
            for i, indexes in enumerate(zip(*coords)):

                indexes = sorted(set(indexes))
                size *= len(indexes)

                missing = [j for j in range(shape_t[i]) if j not in indexes]
                # more than one transition from missing -> included
                transitions = [j for j in missing if (j + 1) % shape_t[i] not in missing]
                if len(transitions) > 1:
                    step = int(shape_t[i] / len(transitions))

                    # check if there is a periodicity
                    if any((j + step)%shape_t[i] not in transitions for j in transitions):
                        return False

                # at least a square of 2x2x1
                if len(indexes) < 2 and i != 2:
                    return False

            if len(coords) == size:
                return True

            return False

        def is_dim_subset(coords):
            """
            Return if it is a dim subset (only one dimension, the others are not touched)
            If True, return (True, nb_dim)
            else (False, 0)

            :param coords:
            :param t:
            :return:
            """
            shape_ts0 = tuple(len(set(indexes)) for indexes in zip(*coords))
            shape_t = self.shape

            if shape_ts0[0] == shape_t[0]:
                return True, shape_ts0[1]

            if shape_ts0[1] == shape_t[1]:
                return True, shape_ts0[0]

            return False, 0

        if script not in self.script:
            return False, False

        if self.rank != 0 and self.rank % 2 == 0:
            return False, False

        coords = sorted([self.index_of(ss) for ss in script.singular_sequences])

        is_dim, count = is_dim_subset(coords)
        if is_dim and count == 1:
            # one dimension subset, it is a rank 3/5 paradigm
            return True, True

        # the coordinates sorted of the ss of s0 in the table t
        # rank is then 2/4
        if is_connexe_tilling(coords) or is_dim:
            return True, False

        return False, False

class Table1D(Table):
    def __init__(self, script, index, dictionary, parent, regular=False):
        super().__init__(script, index, dictionary, parent, regular)
        self._index = None

    @property
    def shape(self):
        return self.cells.shape

    @cached_property
    def cells(self):
        return np.vectorize(lambda sc: self.dictionary.terms[sc])(self.script.cells[0][:, 0, 0])

    def __getitem__(self, item):
        return self.cells[item]

    def index_of(self, item):
        if self._index is None:
            self._index = {
                t.script: index for index, t in np.ndenumerate(self.cells)
            }

        return self._index[script(item)]

    def accept_script(self, script):
        if script not in self.script:
            return False, False

        coords = sorted([self.index_of(ss) for ss in script.singular_sequences])

        if len(coords) == coords[-1][0] - coords[0][0] + 1:
            return True, True
        else:
            return False, False

class TableSet(Table):
    def __init__(self, script, index, dictionary, parent, regular=False):
        super().__init__(script, index, dictionary, parent, regular)

        if len(self.script.tables_script) == 1:
            raise ValueError("Invalid script for TableSet creation: %s. Expected a script that generate multiple "
                             "Tables"%str(script))

    @property
    def tables(self):
        return [self.dictionary.terms[s] for s in self.script.tables_script]

    def accept_script(self, script):
        """
        True when the term is a subset of this term tables. If the parent of this term is already a TableSet,return
        always false (only one main tableset)
        :param term:
        :return:
        """
        if isinstance(self.parent, TableSet):
            return False, False

        tables = [table for table in self.script.tables_script if table in script]

        if len(tables) >= 1 and {ss for t in tables for ss in t.singular_sequences} == set(script.singular_sequences):
            return True, False

        return False, False

class Table3D(TableSet):
    def __init__(self, script, index, dictionary, parent, regular=False):
        super().__init__(script, index, dictionary, parent, regular)

    def __getitem__(self, item):
        return self.tables[item[0]][item[1:]]

def table_class(script):
    if len(script) == 1:
        return Cell

    if len(script.tables_script) == 1:
        dim = sum(1 for s in script.cells[0].shape if s != 1)
        if dim == 1:
            return Table1D
        if dim == 2:
            return Table2D

        raise ValueError("Invalid dim %d for script %s"%(dim, str(script)))
    else:
        if len(script.cells) == 1:
            return Table3D
        else:
            return TableSet


import xml.etree.ElementTree as ET
import random
import functools
from itertools import chain

from urllib.request import urlopen

from ieml.exceptions import CannotParse
from ieml.grammar.text import Text, text
from ieml.grammar.usl import Usl
from ieml.grammar.fact import Fact, fact
from ieml.grammar.parser import IEMLParser
from ieml.grammar.theory import Theory, theory
from ieml.grammar.topic import Topic, topic
from ieml.grammar.word import Word

from ieml.dictionary.version import get_default_dictionary_version
from .exceptions import InvalidIEMLObjectArgument
from .exceptions import CantGenerateElement
from .dictionary import Term, Dictionary

def _loop_result(max_try):
    def decorator(func):
        @functools.wraps(func)
        def wrapper(*args, **kwargs):
            ex = None
            for i in range(max_try):
                try:
                    return func(*args, **kwargs)
                except InvalidIEMLObjectArgument as e:
                    ex = e
                    continue

            raise CantGenerateElement(str(ex))
        return wrapper
    return decorator

class RandomPoolIEMLObjectGenerator:
    def __init__(self, level=Theory, pool_size=20, dictionary_version=None):
        self.level = level
        self.pool_size = pool_size
        self.dictionary_version = dictionary_version

        if level > Text:
            raise ValueError('Cannot generate object higher than a Text.')

        self._build_pools()

        self.type_to_method = {
            Word: self.word,
            Topic: self.topic,
            Fact: self.fact,
            Theory: self.theory,
            Text: self.text
        }

    def _build_pools(self):
        """
        Slow method, retrieve all the terms from the database.
        :return:
        """
        if self.level >= Topic:
            # words
            self.topics_pool = set(self.topic() for i in range(self.pool_size))

        if self.level >= Fact:
            # sentences
            self.facts_pool = set(self.fact() for i in range(self.pool_size))

        if self.level >= Theory:
            self.theories_pool = set(self.theory() for i in range(self.pool_size))

        if self.level >= Text:
            self.propositions_pool = set(chain.from_iterable((self.topics_pool, self.facts_pool, self.theories_pool)))

        # self.hypertext_pool = set(self.hypertext() for i in range(self.pool_size))

    @_loop_result(10)
    def word(self):
        return Word(random.sample(Dictionary(self.dictionary_version).index, 1)[0])

    @_loop_result(10)
    def uniterm_topic(self):
        return topic([random.sample(Dictionary(self.dictionary_version).index, 1)])

    @_loop_result(10)
    def topic(self):
        return topic([Word(t) for t in random.sample(Dictionary(self.dictionary_version).index, 3)],
                     [Word(t) for t in random.sample(Dictionary(self.dictionary_version).index, 2)])

    def _build_graph_object(self, primitive, mode, max_nodes=6):
        nodes = {primitive()}
        modes = set()

        if max_nodes < 2:
            raise ValueError('Max nodes >= 2.')

        result = set()

        for i in range(random.randint(2, max_nodes)):
            while True:
                s, a, m = random.sample(nodes, 1)[0], primitive(), mode()
                if a in nodes or m in nodes or a in modes:
                    continue

                nodes.add(a)
                modes.add(m)

                result.add((s, a, m))
                break
        return result

    @_loop_result(10)
    def fact(self, max_clause=6):
        def p():
            return random.sample(self.topics_pool, 1)[0]

        return fact(self._build_graph_object(p, p, max_nodes=max_clause))

    @_loop_result(10)
    def theory(self, max_clause=4):
        def p():
            return random.sample(self.facts_pool, 1)[0]

        return theory(self._build_graph_object(p, p, max_nodes=max_clause))

    @_loop_result(10)
    def text(self):
        return text(random.sample(self.propositions_pool, random.randint(1, 8)))

    def from_type(self, type):
        try:
            return self.type_to_method[type]()
        except KeyError:
            raise ValueError("Can't generate that type or not an IEMLObject : %s"%str(type))

def list_bucket(url):
    root_node = ET.fromstring(urlopen(url).read())
    all_versions_entry = ({k.tag: k.text for k in list(t)} for t in root_node
                          if t.tag == '{http://s3.amazonaws.com/doc/2006-03-01/}Contents')

    # sort by date
    all_versions = sorted(all_versions_entry,
                          key=lambda t: t['{http://s3.amazonaws.com/doc/2006-03-01/}LastModified'], reverse=True)

    return [v['{http://s3.amazonaws.com/doc/2006-03-01/}Key'] for v in all_versions]

def ieml(arg, dictionary_version=None):
    if not dictionary_version:
        dictionary_version = get_default_dictionary_version()

    if isinstance(arg, Usl):
        if arg.dictionary_version != dictionary_version:
            arg.set_dictionary_version(dictionary_version)

        return arg

    if isinstance(arg, str):
        try:
            return IEMLParser(Dictionary(dictionary_version)).parse(arg)
        except CannotParse as e:
            raise InvalidIEMLObjectArgument(Usl, str(e))

    if isinstance(arg, Term):
        arg = Word(arg)
        if arg.dictionary_version != dictionary_version:
            arg.set_dictionary_version(dictionary_version)

        return arg

    raise NotImplemented


from __future__ import division

import sys

from atomic import AtomicLong, AtomicLongArray

from metrology.stats.sample import UniformSample, ExponentiallyDecayingSample

class Histogram(object):
    """
    A histogram measures the statistical distribution of values in a stream of
    data. In addition to minimum, maximum, mean, it also measures median,
    75th, 90th, 95th, 98th, 99th, and 99.9th percentiles ::

      histogram = Metrology.histogram('response-sizes')
      histogram.update(len(response.content))

    Metrology provides two types of histograms: uniform and exponentially
    decaying.
    """
    DEFAULT_SAMPLE_SIZE = 1028
    DEFAULT_ALPHA = 0.015

    def __init__(self, sample):
        self.sample = sample
        self.counter = AtomicLong(0)
        self.minimum = AtomicLong(sys.maxsize)
        self.maximum = AtomicLong(-sys.maxsize - 1)
        self.sum = AtomicLong(0)
        self.var = AtomicLongArray([-1, 0])

    def clear(self):
        self.sample.clear()
        self.counter.value = 0
        self.minimum.value = sys.maxsize
        self.maximum.value = (-sys.maxsize - 1)
        self.sum.value = 0
        self.var.value = [-1, 0]

    def update(self, value):
        self.counter += 1
        self.sample.update(value)
        self.max = value
        self.min = value
        self.sum += value
        self.update_variance(value)

    @property
    def snapshot(self):
        return self.sample.snapshot()

    @property
    def count(self):
        """Return number of values."""
        return self.counter.value

    def get_max(self):
        if self.counter.value > 0:
            return self.maximum.value
        return 0.0

    def set_max(self, potential_max):
        done = False
        while not done:
            current_max = self.maximum.value
            done = (current_max is not None and current_max >= potential_max) \
                or self.maximum.compare_and_swap(current_max, potential_max)

    max = property(get_max, set_max, doc="""Returns the maximun value.""")

    def get_min(self):
        if self.counter.value > 0:
            return self.minimum.value
        return 0.0

    def set_min(self, potential_min):
        done = False
        while not done:
            current_min = self.minimum.value
            done = (current_min is not None and current_min <= potential_min) \
                or self.minimum.compare_and_swap(current_min, potential_min)

    min = property(get_min, set_min, doc="""Returns the minimum value.""")

    @property
    def total(self):
        """Returns the total value."""
        return self.sum.value

    @property
    def mean(self):
        """Returns the mean value."""
        if self.counter.value > 0:
            return self.sum.value / self.counter.value
        return 0.0

    @property
    def stddev(self):
        """Returns the standard deviation."""
        if self.counter.value > 0:
            return self.variance ** .5
        return 0.0

    @property
    def variance(self):
        """Returns variance"""
        if self.counter.value <= 1:
            return 0.0
        return self.var.value[1] / (self.counter.value - 1)

    def update_variance(self, value):
        def variance(old_values):
            if old_values[0] == -1:
                new_values = (value, 0)
            else:
                old_m = old_values[0]
                old_s = old_values[1]

                new_m = old_m + ((value - old_m) / self.counter.value)
                new_s = old_s + ((value - old_m) * (value - new_m))

                new_values = (new_m, new_s)
            return new_values
        self.var.value = variance(self.var.value)

class HistogramUniform(Histogram):
    """
    A uniform histogram produces quantiles which are valid for the entirely of
    the histogram's lifetime. It will return a median value, for example, which
    is the median of all the values the histogram has ever been updated with.

    Use a uniform histogram when you're interested in long-term measurements.
    Don't use one where you'd want to know if the distribution of the
    underlying data stream has changed recently.
    """
    def __init__(self):
        sample = UniformSample(self.DEFAULT_SAMPLE_SIZE)
        super(HistogramUniform, self).__init__(sample)

class HistogramExponentiallyDecaying(Histogram):
    """
    A exponentially decaying histogram produces quantiles which are
    representative of approximately the last five minutes of data.

    Unlike the uniform histogram, a biased histogram represents recent data,
    allowing you to know very quickly if the distribution of the data has
    changed.

    """
    def __init__(self):
        sample = ExponentiallyDecayingSample(self.DEFAULT_SAMPLE_SIZE,
                                             self.DEFAULT_ALPHA)
        super(HistogramExponentiallyDecaying, self).__init__(sample)


from functools import wraps
from time import time

from atomic import AtomicLong

from metrology.stats import EWMA
from metrology.utils import now

def ticker(method):
    @wraps(method)
    def wrapper(self, *args, **kwargs):
        self._tick()
        return method(self, *args, **kwargs)
    return wrapper

class Meter(object):
    """A meter measures the rate of events over time
    (e.g., "requests per second").

    In addition to the mean rate, you can also track 1, 5 and 15 minutes moving
    averages ::

      meter = Metrology.meter('requests')
      meter.mark()
      meter.count

    """
    def __init__(self, average_class=EWMA):
        self.counter = AtomicLong(0)
        self.start_time = now()
        self.last_tick = AtomicLong(self.start_time)

        self.interval = EWMA.INTERVAL
        self.m1_rate = EWMA.m1()
        self.m5_rate = EWMA.m5()
        self.m15_rate = EWMA.m15()

    def _tick(self):
        old_tick, new_tick = self.last_tick.value, time()
        age = new_tick - old_tick
        ticks = int(age / self.interval)
        new_tick = old_tick + int(ticks * self.interval)
        if ticks and self.last_tick.compare_and_swap(old_tick, new_tick):
            for _ in range(ticks):
                self.tick()

    def __call__(self, *args, **kwargs):
        if args and hasattr(args[0], '__call__'):
            _orig_func = args[0]

            def _decorator(*args, **kwargs):
                with self:
                    return _orig_func(*args, **kwargs)
            return _decorator

    def __enter__(self):
        pass

    def __exit__(self, exc, exv, trace):
        self.mark()

    @property
    def count(self):
        """Returns the total number of events that have been recorded."""
        return self.counter.value

    def clear(self):
        self.counter.value = 0
        self.start_time = time()

        self.m1_rate.clear()
        self.m5_rate.clear()
        self.m15_rate.clear()

    @ticker
    def mark(self, value=1):
        """Record an event with the meter. By default it will record one event.

        :param value: number of event to record
        """
        self.counter += value
        self.m1_rate.update(value)
        self.m5_rate.update(value)
        self.m15_rate.update(value)

    def tick(self):
        self.m1_rate.tick()
        self.m5_rate.tick()
        self.m15_rate.tick()

    @property
    @ticker
    def one_minute_rate(self):
        """Returns the one-minute average rate."""
        return self.m1_rate.rate

    @property
    @ticker
    def five_minute_rate(self):
        """Returns the five-minute average rate."""
        return self.m5_rate.rate

    @property
    @ticker
    def fifteen_minute_rate(self):
        """Returns the fifteen-minute average rate."""
        return self.m15_rate.rate

    @property
    def mean_rate(self):
        """
        Returns the mean rate of the events since the start of the process.
        """
        if self.counter.value == 0:
            return 0.0
        else:
            elapsed = time() - self.start_time
            return self.counter.value / elapsed

    def stop(self):
        pass


from atomic import AtomicLong

from metrology.instruments.meter import Meter
from metrology.stats import EWMA

class Derive(Meter):
    """
    A derive is like a meter but accepts an absolute counter as input.

      derive = Metrology.derive('network.io')
      derive.mark()
      derive.count

    """
    def __init__(self, average_class=EWMA):
        self.last = AtomicLong(0)
        super(Derive, self).__init__(average_class)

    def mark(self, value=1):
        """Record an event with the derive.

        :param value: counter value to record
        """
        last = self.last.get_and_set(value)
        if last <= value:
            value = value - last
        super(Derive, self).mark(value)


import socket
import sys

from metrology.instruments import (
        Counter,
        Gauge,
        Histogram,
        Meter,
        Timer,
        UtilizationTimer
)
from metrology.reporter.base import Reporter

def class_name(obj):
    return obj.__name__ if isinstance(obj, type) else type(obj).__name__

def mmap(func, iterable):
    """Wrapper to make map() behave the same on Py2 and Py3."""

    if sys.version_info[0] > 2:
        return [i for i in map(func, iterable)]
    else:
        return map(func, iterable)

# NOTE(romcheg): This dictionary maps metric types to specific configuration
#                of the metric serializer.
#                Format:
#                    {
#                     'metric_type':
#                          {
#                             'serialized_type': str,
#                             'keys': list(str),
#                             'snapshot_keys': list(str)
#                          }
#                    }
SERIALIZER_CONFIG = {
    class_name(Meter): {
        'serialized_type': 'm',
        'keys': [
            'count', 'one_minute_rate', 'five_minute_rate', 'mean_rate',
            'fifteen_minute_rate'
            ],
        'snapshot_keys': None
        },

    class_name(Gauge): {
        'serialized_type': 'g',
        'keys': ['value'],
        'snapshot_keys': None
        },

    class_name(UtilizationTimer): {
        'serialized_type': 'ms',
        'keys': [
            'count', 'one_minute_rate', 'five_minute_rate', 'min', 'max',
            'fifteen_minute_rate', 'mean_rate', 'mean', 'stddev',
            'one_minute_utilization', 'five_minute_utilization',
            'fifteen_minute_utilization', 'mean_utilization'
            ],
        'snapshot_keys': [
            'median', 'percentile_95th', 'percentile_99th', 'percentile_999th'
            ]
        },

    class_name(Timer): {
        'serialized_type': 'ms',
        'keys': [
            'count', 'total_time', 'one_minute_rate', 'five_minute_rate',
            'fifteen_minute_rate', 'mean_rate', 'min', 'max', 'mean', 'stddev'
            ],
        'snapshot_keys': [
            'median', 'percentile_95th', 'percentile_99th', 'percentile_999th'
            ]
        },

    class_name(Counter): {
        'serialized_type': 'c',
        'keys': ['count'],
        'snapshot_keys': None
        },

    class_name(Histogram): {
        'serialized_type': 'h',
        'keys': ['count', 'min', 'max', 'mean', 'stddev'],
        'snapshot_keys': [
            'median', 'percentile_95th', 'percentile_99th', 'percentile_999th'
            ]
        }
}

class StatsDReporter(Reporter):
    """
    A statsd reporter that sends metrics to statsd daemon ::

      reporter = StatsDReporter('statsd.local', 8125)
      reporter.start()

    :param host: hostname of statsd daemon
    :param port: port of daemon
    :param interval: time between each reports
    :param prefix: metrics name prefix

    """
    def __init__(self, host, port, conn_type='udp', **options):
        self.host = host
        self.port = port
        self.conn_type = conn_type

        self.prefix = options.get('prefix')
        self.batch_size = options.get('batch_size', 100)
        self.batch_buffer = ''
        if self.batch_size <= 0:
            self.batch_size = 1
        self._socket = None
        super(StatsDReporter, self).__init__(**options)
        self.batch_count = 0
        if conn_type == 'tcp':
            self._send = self._send_tcp
        else:
            self._send = self._send_udp

    @property
    def socket(self):
        if not self._socket:
            if self.conn_type == 'tcp':
                self._socket = socket.socket(socket.AF_INET,
                                             socket.SOCK_STREAM)
                self._socket.connect((self.host, self.port))
            else:
                self._socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
        return self._socket

    def write(self):
        for name, metric in self.registry:

            if self._is_metric_supported(metric):
                self.send_metric(name, metric)

        self._send()

    def send_metric(self, name, metric):
        """Send metric and its snapshot."""
        config = SERIALIZER_CONFIG[class_name(metric)]

        mmap(
            self._buffered_send_metric,
            self.serialize_metric(
                metric,
                name,
                config['keys'],
                config['serialized_type']
            )
        )

        if hasattr(metric, 'snapshot') and config.get('snapshot_keys'):
            mmap(
                self._buffered_send_metric,
                self.serialize_metric(
                    metric.snapshot,
                    name,
                    config['snapshot_keys'],
                    config['serialized_type']
                )
            )

    def serialize_metric(self, metric, m_name, keys, m_type):
        """Serialize and send available measures of a metric."""

        return [
            self.format_metric_string(m_name, getattr(metric, key), m_type)
            for key in keys
        ]

    def format_metric_string(self, name, value, m_type):
        """Compose a statsd compatible string for a metric's measurement."""

        # NOTE(romcheg): This serialized metric template is based on
        #                statsd's documentation.
        template = '{name}:{value}|{m_type}\n'

        if self.prefix:
            name = "{prefix}.{m_name}".format(prefix=self.prefix, m_name=name)

        return template.format(name=name, value=value, m_type=m_type)

    def _buffered_send_metric(self, metric_str):
        """Add a metric to the buffer."""

        self.batch_count += 1

        self.batch_buffer += metric_str

        # NOTE(romcheg): Send metrics if the number of metrics in the buffer
        #                has reached the threshold for sending.
        if self.batch_count >= self.batch_size:
            self._send()

    def _is_metric_supported(self, metric):
        return class_name(metric) in SERIALIZER_CONFIG

    def _send_tcp(self):
        if len(self.batch_buffer):
            if sys.version_info[0] > 2:
                self.socket.sendall(bytes(self.batch_buffer, 'ascii'))
            else:
                self.socket.sendall(self.batch_buffer)

            self.batch_count = 0
            self.batch_buffer = ''

    def _send_udp(self):
        if len(self.batch_buffer):
            if sys.version_info[0] > 2:
                self.socket.sendto(bytes(self.batch_buffer, 'ascii'),
                                   (self.host, self.port))
            else:
                self.socket.sendto(self.batch_buffer,
                                   (self.host, self.port))

            self.batch_count = 0
            self.batch_buffer = ''