# inference mechanisms
from pyrser.error import *
from pyrser.type_system.scope import Scope
from pyrser.type_system.evalctx import EvalCtx
from pyrser.type_system.val import Val
from pyrser.passes.to_yml import *

class InferNode:
    """
    An instance of this class is automatically attach on each
    AST Nodes by the inference algorithm during the inference process
    into the '.infer_node' attribute.
    The aim is to keep track of the correct Scope and the final EvalCtx
    for each AST Nodes.
    """
    def __init__(self, init_scope: Scope=None, parent: Scope=None):
        if init_scope is not None:
            self.scope_node = init_scope
        else:
            self.scope_node = Scope(is_namespace=False)
        if parent is not None:
            self.scope_node.set_parent(parent.scope_node)
        self.type_node = None
        self.need_feedback = False

class Inference:
    # TO REDEFINE IN AST
    def type_algos(self) -> ('self.infer_algo',
                             'self.feedback_algo', ['args']):
        """
        Sub class must return a Tuple of 3 elements:
            - the method to use to infer type.
            - the list of params to used when infer a type.
            - the method to use when feedback a type.

        This is useful to connect AST members to generic algo or
        to overload some specific semantic for your language.
        """
        raise Exception(("%s must inherit from Inference and define" +
                        " type_algos(self) method to support" +
                        " Pyrser Type Systems.") % type(self).__name__)

    def infer_type(self, init_scope: Scope=None, diagnostic=None):
        """
        Do inference. Write infos into diagnostic object, if this parameter
        is not provide and self is a AST (has is own diagnostic object),
        use the diagnostic of self.
        """
        # create the first .infer_node
        if not hasattr(self, 'infer_node'):
            self.infer_node = InferNode(init_scope)
        elif init_scope is not None:
            # only change the root scope
            self.infer_node.scope_node = init_scope
        # get algo
        type_algo = self.type_algos()
        if diagnostic is None and hasattr(self, 'diagnostic'):
            diagnostic = self.diagnostic
        type_algo[0](type_algo[1], diagnostic)

    def feedback(self, diagnostic=None):
        """
        Do feedback. Write infos into diagnostic object, if this parameter
        is not provide and self is a AST (has is own diagnostic object),
        use the diagnostic of self.
        """
        # get algo
        type_algo = self.type_algos()
        if diagnostic is None and hasattr(self, 'diagnostic'):
            diagnostic = self.diagnostic
        type_algo[2](diagnostic)

    ## INFER ALGOS

    def infer_block(self, body, diagnostic=None):
        """
        Infer type on block is to type each of is sub-element
        """
        # RootBlockStmt has his own .infer_node (created via infer_type)
        for e in body:
            e.infer_node = InferNode(parent=self.infer_node)
            e.infer_type(diagnostic=diagnostic)
        # TODO: result in .infer_node.type_node

    def infer_subexpr(self, expr, diagnostic=None):
        """
        Infer type on the subexpr
        """
        expr.infer_node = InferNode(parent=self.infer_node)
        expr.infer_type(diagnostic=diagnostic)
        # TODO: result in .infer_node.type_node

    def infer_fun(self, args, diagnostic=None):
        # 0 - get function
        call_expr, arguments = args
        diagnostic.notify(
            Severity.INFO,
            "Infer Function call '%s'" % call_expr.value,
            self.info
        )
        # 1 - fetch all possible types for the call expr
        call_expr.infer_node = InferNode(parent=self.infer_node)
        call_expr.infer_type(diagnostic=diagnostic)
        f = call_expr.infer_node.scope_node
        # 2 - fetch all possible types for each parameters
        tparams = []
        i = 0
        for p in arguments:
            p.infer_node = InferNode(parent=self.infer_node)
            p.infer_type(diagnostic=diagnostic)
            i += 1
            tparams.append(p.infer_node.scope_node)
        # 3 - select overloads
        (final_call_expr, final_tparams) = f.get_by_params(tparams)
        # 4 - record overloads
        self.infer_node.scope_node.clear()
        self.infer_node.scope_node.update(final_call_expr)
        nversion = len(final_call_expr)
        if nversion > 1:
            # too many choice
            self.infer_node.need_feedback = True
            return
        elif nversion == 0:
            # TODO: Try type reconstruction
            # if final_tparams[x][0].is_polymorphic
            # ...
            # type error
            details = "\ndetails:\n"
            details += "overloads:\n"
            for fun in f.values():
                details += str(fun.get_compute_sig()) + "\n"
            details += "parameters:\n"
            i = 0
            # TODO: change by final_tparams
            for p in tparams:
                details += "- param[%d] type " % i
                ptypes = []
                for alt in p.values():
                    ptypes.append(alt.tret)
                details += '|'.join(ptypes) + "\n"
                i += 1
            # TODO: when f are empty
            diagnostic.notify(
                Severity.ERROR,
                ("can't match overloads "
                 + "with parameters for function '%s'") % str(f.first().name),
                self.info,
                details
            )
            return
        # here len(self.type_node) == 1 && len(final_tparams) == 1
        # 5 - handle polymorphism
        # TODO: INSIDE A CLASS?
        my_map = dict()
        my_type = self.infer_node.scope_node.first()
        if my_type.tret.is_polymorphic:
            diagnostic.notify(
                Severity.INFO,
                "polymorphic return type %s" % str(my_type.get_compute_sig()),
                self.info
            )
            # if tret is polymorphic, take type from call_expr if unique,
            # else type come from parameter resolution
            sig = call_expr.infer_node.scope_node.first()
            if (len(call_expr.infer_node.scope_node) == 1
                and not sig.tret.is_polymorphic
            ):
                diagnostic.notify(
                    Severity.INFO,
                    "call_expr have real type '%s'" % sig.tret,
                    self.info
                )
                my_map.update(sig.resolution)
                my_type.set_resolved_name(
                    sig.resolution,
                    my_type.tret,
                    sig.tret
                )
        arity = len(my_type.tparams)
        for i in range(arity):
            p = my_type.tparams[i]
            # use AST Injector
            if final_tparams[0][i].first()._translate_to is not None:
                t = final_tparams[0][i].first()._translate_to
                old = arguments[i]
                n = t.notify
                diagnostic.notify(
                    n.severity,
                    n.msg,
                    old.info,
                    details=n.details
                )
                arguments[i] = self.infer_node.scope_node.callInjector(old, t)
                infer_node = InferNode(parent=self.infer_node)
                infer_node.scope_node.add(t.fun)
                old.infer_node.scope_node.set_parent(infer_node.scope_node)
                arguments[i].infer_node = infer_node
            if p.is_polymorphic:
                # take type in final_tparams
                diagnostic.notify(
                    Severity.INFO,
                    "- param[%d] polymorphic type" % i, arguments[i].info
                )
                # here len(final_tparams) == 1, only one set for parameters
                sig = final_tparams[0][i].first()
                if not sig.tret.is_polymorphic:
                    diagnostic.notify(
                        Severity.INFO,
                        "- argument[%d] real type '%s'" % (i, sig.tret),
                        arguments[i].info
                    )
                    my_map[p.value] = sig.resolution[sig.tret.value]
                    my_type.set_resolved_name(sig.resolution, p, sig.tret)
        diagnostic.notify(
            Severity.INFO,
            "after resolution %s" % str(my_type.get_compute_sig()),
            self.info
        )
        # 6 - feedback
        self.map_type = my_map
        self.final_type = my_type.tret
        self.feedback(diagnostic)
        # 7 - Are we finish? Global type reconstruction
        # TODO: result in .infer_node.type_node

    def infer_id(self, ident, diagnostic=None):
        """
        Infer type from an ID!
        - check if ID is declarated in the scope
        - if no ID is polymorphic type
        """
        # check if ID is declared
        #defined = self.type_node.get_by_symbol_name(ident)
        defined = self.infer_node.scope_node.get_by_symbol_name(ident)
        if len(defined) > 0:
            # set from matchings declarations
            #self.type_node.update(defined)
            self.infer_node.scope_node.update(defined)
        else:
            diagnostic.notify(
                Severity.ERROR,
                "%s never declared" % self.value,
                self.info
            )

    # TODO: uncomment/fix when finish
    #def infer_id_self_type(self, ident, diagnostic=None):
    #    """
    #    Infer type from an ID!
    #    - check if ID is declarated in the scope
    #    - if no ID is polymorphic type
    #    """
    #    # check if ID is declared
    #    defined = self.type_node.get_by_symbol_name(ident)
    #    if len(defined) > 0:
    #        # set from matchings declarations
    #        self.type_node.update(defined)
    #    else:
    #        # TODO: est de type polymorphique local ... pas tout le temps ?1
    #        # a faire dans une declaration ... ou a l'affectation
    #        # mais pas la
    #        self.type_node.add(Var(ident, '?1'))
    #        self.type_node.need_feedback = True

    def infer_literal(self, args, diagnostic=None):
        """
        Infer type from an LITERAL!
        Type of literal depend of language.
        We adopt a basic convention
        """
        literal, t = args
        #self.type_node.add(EvalCtx.from_sig(Val(literal, t)))
        self.infer_node.scope_node.add(EvalCtx.from_sig(Val(literal, t)))

    ## FEEDBACK ALGOS

    def feedback_block(self, diagnostic=None):
        # normally nothing to do!?!?!
        type_algo = self.type_algos()
        # body in type_algo[1]
        for e in type_algo[1]:
            if e.infer_node.need_feedback:
                e.feedback(diagnostic)

    def feedback_subexpr(self, diagnostic=None):
        final_scn = self.infer_node.scope_node
        final_scn = final_scn.get_by_return_type(self.final_type)
        self.infer_node.scope_node = final_scn
        self.infer_node.need_feedback = False
        if self.expr.infer_node.need_feedback:
            self.expr.feedback(diagnostic)

    def feedback_leaf(self, diagnostic=None):
        final_scn = self.infer_node.scope_node
        final_scn = final_scn.get_by_return_type(self.final_type)
        self.infer_node.scope_node = final_scn
        self.infer_node.need_feedback = False

    def feedback_fun(self, diagnostic=None):
        final_scn = self.infer_node.scope_node
        final_scn = final_scn.get_by_return_type(self.final_type)
        self.infer_node.scope_node = final_scn
        self.infer_node.need_feedback = False
        arguments = self.infer_node.scope_node.first().tparams
        nargs = len(arguments)
        type_algo = self.type_algos()
        call_expr, args = type_algo[1]
        if call_expr.infer_node.need_feedback:
            diagnostic.notify(
                Severity.INFO,
                "feed back call_expr = %s" % self.final_type,
                call_expr.info
            )
            call_expr.map_type = self.map_type
            # TODO: could be infer_node.type_node
            call_expr.final_type = self.final_type
            call_expr.feedback()
        for i in range(nargs):
            p = arguments[i]
            if args[i].infer_node.need_feedback:
                diagnostic.notify(
                    Severity.INFO,
                    "feed back p[%d] = %s" % (i, p),
                    args[i].info
                )
                diagnostic.notify(
                    Severity.INFO,
                    "feed map %s" % self.map_type,
                    args[i].info
                )
                args[i].map_type = self.map_type
                args[i].final_type = p
                args[i].feedback(diagnostic)

    def feedback_id(self, diagnostic=None):
        # instancie META!!!
        if len(self.infer_node.scope_node) > 1:
            final_scn = self.infer_node.scope_node
            final_scn = final_scn.get_by_return_type(self.final_type)
            self.infer_node.scope_node = final_scn
            if len(self.infer_node.scope_node) != 1:
                # ERROR TYPE !!!?!?
                diagnostic.notify(
                    Severity.ERROR,
                    ("Type error: too many candidates %s"
                        % str(self.infer_node.scope_node)),
                    self.info
                )
        else:
            the_sig = list(self.infer_node.scope_node.values())[0]
            if the_sig.tret.is_polymorphic:
                #                                 self.final_type)
                self.infer_node.type_node = EvalCtx.from_sig(the_sig)
                final_tn = self.infer_node.type_node
                final_tn.set_resolved_name(
                    self.map_type,
                    the_sig.tret,
                    self.final_type
                )


from pyrser import meta
from pyrser.parsing.base import BasicParser

@meta.hook(BasicParser, "dump_nodes")
def dump_nodes(self):
    """
    Dump tag,rule,id and value cache. For debug.

    example::

        R = [
            #dump_nodes
        ]

    """
    print("DUMP NODE LOCAL INFOS")
    try:
        print("map Id->node name")
        for k, v in self.id_cache.items():
            print("[%d]=%s" % (k, v))
        print("map tag->capture infos")
        for k, v in self.tag_cache.items():
            print("[%s]=%s" % (k, v))
        print("map nodes->tag resolution")
        for k, v in self.rule_nodes.items():
            txt = "['%s']=%d" % (k, id(v))
            if k in self.tag_cache:
                tag = self.tag_cache[k]
                txt += " tag <%s>" % tag
                k = "%d:%d" % (tag._begin, tag._end)
                if k in self._stream.value_cache:
                    txt += " cache <%s>" % self._stream.value_cache[k]
            print(txt)
    except Exception as err:
        print("RECV Exception %s" % err)
    import sys
    sys.stdout.flush()
    return True


import os
import json
import time

try:
    from urlparse import urlunparse
except ImportError:
    from urllib.parse import urlunparse

import boto3
from botocore.errorfactory import ClientError

from dtoolcore.utils import (
    generate_identifier,
    get_config_value,
    mkdir_parents,
    generous_parse_uri,
    DEFAULT_CACHE_PATH,
)

from dtoolcore.filehasher import FileHasher, md5sum_hexdigest

from dtoolcore.storagebroker import BaseStorageBroker

from dtool_s3 import __version__

# We update the dataset_registration_key when we have the UUID for the dataset
_STRUCTURE_PARAMETERS = {
    "dataset_registration_key": None,
    "data_key_infix": "data",
    "fragment_key_infix": "fragments",
    "overlays_key_infix": "overlays",
    "structure_key_suffix": "structure.json",
    "dtool_readme_key_suffix": "README.txt",
    "dataset_readme_key_suffix": "README.yml",
    "manifest_key_suffix": "manifest.json",
    "admin_metadata_key_suffix": "dtool",
    "http_manifest_key": "http_manifest.json",
    "storage_broker_version": __version__,
}

_DTOOL_README_TXT = """README
======
This is a Dtool dataset stored in S3 accessible storage.

Content provided during the dataset creation process
----------------------------------------------------

Dataset registration key (at top level of bucket): dtool-$UUID

Where UUID is the unique identifier for the dataset. The file is empty and
used to aid fast enumeration of unique datasets in the bucket.

The UUID is used as a prefix for all other keys in the dataset.

Dataset descriptive metadata: $UUID/README.yml
Dataset items prefixed by: $UUID/data/

The item identifiers are used to name the files using the data
prefix.

An item identifier is the sha1sum hexdigest of the relative path
used to represent the file on traditional file system disk.

Automatically generated files and directories
---------------------------------------------

This file: $UUID/README.txt
Administrative metadata describing the dataset: $UUID/dtool
Structural metadata describing the dataset: $UUID/structure.json
Structural metadata describing the data items: $UUID/manifest.json
Per item descriptive metadata prefixed by: $UUID/overlays/
"""

class S3StorageBroker(BaseStorageBroker):

    #: Attribute used to define the type of storage broker.
    key = "s3"

    #: Attribute used by :class:`dtoolcore.ProtoDataSet` to write the hash
    #: function name to the manifest.
    hasher = FileHasher(md5sum_hexdigest)

    _dtool_readme_txt = _DTOOL_README_TXT

    def __init__(self, uri, config_path=None):

        parse_result = generous_parse_uri(uri)

        self.bucket = parse_result.netloc
        uuid = parse_result.path[1:]

        self.uuid = uuid
        self.s3resource = boto3.resource('s3')
        self.s3client = boto3.client('s3')

        self._structure_parameters = _STRUCTURE_PARAMETERS
        self.dataset_registration_key = 'dtool-{}'.format(self.uuid)
        self._structure_parameters["dataset_registration_key"] = self.dataset_registration_key  # NOQA

        self.data_key_prefix = self._generate_key_prefix("data_key_infix")
        self.fragments_key_prefix = self._generate_key_prefix(
            "fragment_key_infix"
        )
        self.overlays_key_prefix = self._generate_key_prefix(
            "overlays_key_infix"
        )

        self.http_manifest_key = self._generate_key("http_manifest_key")

        self._s3_cache_abspath = get_config_value(
            "DTOOL_CACHE_DIRECTORY",
            config_path=config_path,
            default=DEFAULT_CACHE_PATH
        )

    # Generic helper functions.

    def _generate_key(self, structure_dict_key):
        return self.uuid + '/' + self._structure_parameters[structure_dict_key]

    def _generate_key_prefix(self, structure_dict_key):
        return self._generate_key(structure_dict_key) + '/'

    def _get_item_object(self, handle):
        identifier = generate_identifier(handle)
        item_key = self.data_key_prefix + identifier
        obj = self.s3resource.Object(self.bucket, item_key)
        return obj

    def _make_key_public(self, key):
        acl = self.s3resource.ObjectAcl(self.bucket, key)
        acl.put(ACL='public-read')

    def _generate_http_manifest(self):

        readme_url = self.generate_key_url(self.get_readme_key())
        manifest_url = self.generate_key_url(self.get_manifest_key())

        overlays = {}
        for overlay_name in self.list_overlay_names():
            overlay_fpath = self.overlays_key_prefix + overlay_name + '.json'
            overlays[overlay_name] = self.generate_key_url(overlay_fpath)

        manifest = self.get_manifest()
        item_urls = {}
        for identifier in manifest["items"]:
            item_urls[identifier] = self.generate_key_url(
                self.data_key_prefix + identifier
            )

        http_manifest = {
            "admin_metadata": self.get_admin_metadata(),
            "item_urls": item_urls,
            "overlays": overlays,
            "readme_url": readme_url,
            "manifest_url": manifest_url
        }

        return http_manifest

    def _write_http_manifest(self, http_manifest):

        self.s3resource.Object(self.bucket, self.http_manifest_key).put(
            Body=json.dumps(http_manifest)
        )

        self._make_key_public(self.http_manifest_key)

    # Class methods to override.

    @classmethod
    def list_dataset_uris(cls, base_uri, config_path):
        """Return list containing URIs with base URI."""
        uri_list = []

        parse_result = generous_parse_uri(base_uri)
        bucket_name = parse_result.netloc
        bucket = boto3.resource('s3').Bucket(bucket_name)

        for obj in bucket.objects.filter(Prefix='dtool').all():
            uuid = obj.key.split('-', 1)[1]
            uri = cls.generate_uri(None, uuid, base_uri)

            storage_broker = cls(uri, config_path)
            if storage_broker.has_admin_metadata():
                uri_list.append(uri)

        return uri_list

    @classmethod
    def generate_uri(cls, name, uuid, base_uri):

        scheme, netloc, path, _, _, _ = generous_parse_uri(base_uri)
        assert scheme == 's3'

        # Force path (third component of tuple) to be the dataset UUID
        uri = urlunparse((scheme, netloc, uuid, _, _, _))

        return uri

    # Methods to override.

    def _create_structure(self):
        self.s3resource.Object(self.bucket, self.dataset_registration_key).put(
            Body=''
        )

    def put_text(self, key, content):
        self.s3resource.Object(self.bucket, key).put(
            Body=content
        )

    def get_text(self, key):

        response = self.s3resource.Object(
            self.bucket,
            key
        ).get()

        return response['Body'].read().decode('utf-8')

    def get_structure_key(self):
        return self._generate_key("structure_key_suffix")

    def get_dtool_readme_key(self):
        return self._generate_key("dtool_readme_key_suffix")

    def get_readme_key(self):
        return self._generate_key("dataset_readme_key_suffix")

    def get_overlay_key(self, overlay_name):
        return os.path.join(self.overlays_key_prefix, overlay_name + '.json')

    def get_manifest_key(self):
        return self._generate_key("manifest_key_suffix")

    def get_admin_metadata_key(self):
        return self._generate_key("admin_metadata_key_suffix")

    def put_admin_metadata(self, admin_metadata):

        str_admin_metadata = {}
        for k, v in admin_metadata.items():
            str_admin_metadata[k] = str(v)

        self.s3resource.Object(self.bucket, self.get_admin_metadata_key()).put(
            Body=json.dumps(admin_metadata),
            Metadata=str_admin_metadata
        )

    def get_admin_metadata(self):

        response = self.s3resource.Object(
            self.bucket,
            self.get_admin_metadata_key()
        ).get()

        return response['Metadata']

    def get_size_in_bytes(self, handle):
        obj = self._get_item_object(handle)
        return int(obj.content_length)

    def get_utc_timestamp(self, handle):
        obj = self._get_item_object(handle)
        return time.mktime(obj.last_modified.timetuple())

    def get_hash(self, handle):

        # Here the calculated MD5 checksum is retrieved from the
        # metadata. This is needed as the AWS etag is
        # not the md5 sum of the uploaded object for items that are uploaded
        # using multipart uploads (large files).
        # See: https://stackoverflow.com/a/43067788
        obj = self._get_item_object(handle)
        return obj.get()['Metadata']['checksum']

# According to the tests the below is not needed.
#   def get_relpath(self, handle):
#       obj = self._get_item_object(handle)
#       return obj.get()['Metadata']['handle']

    def has_admin_metadata(self):
        """Return True if the administrative metadata exists.

        This is the definition of being a "dataset".
        """

        try:
            self.get_admin_metadata()
            return True
        except ClientError:
            return False

    def get_item_abspath(self, identifier):
        """Return absolute path at which item content can be accessed.

        :param identifier: item identifier
        :returns: absolute path from which the item content can be accessed
        """
        admin_metadata = self.get_admin_metadata()
        uuid = admin_metadata["uuid"]
        # Create directory for the specific dataset.
        dataset_cache_abspath = os.path.join(self._s3_cache_abspath, uuid)
        mkdir_parents(dataset_cache_abspath)

        bucket_fpath = self.data_key_prefix + identifier
        obj = self.s3resource.Object(self.bucket, bucket_fpath)
        relpath = obj.get()['Metadata']['handle']
        _, ext = os.path.splitext(relpath)

        local_item_abspath = os.path.join(
            dataset_cache_abspath,
            identifier + ext
        )
        if not os.path.isfile(local_item_abspath):

            tmp_local_item_abspath = local_item_abspath + ".tmp"
            self.s3resource.Bucket(self.bucket).download_file(
                bucket_fpath,
                tmp_local_item_abspath
            )
            os.rename(tmp_local_item_abspath, local_item_abspath)

        return local_item_abspath

    def list_overlay_names(self):
        """Return list of overlay names."""

        bucket = self.s3resource.Bucket(self.bucket)

        overlay_names = []
        for obj in bucket.objects.filter(
            Prefix=self.overlays_key_prefix
        ).all():

            overlay_file = obj.key.rsplit('/', 1)[-1]
            overlay_name, ext = overlay_file.split('.')
            overlay_names.append(overlay_name)

        return overlay_names

    def put_item(self, fpath, relpath):

        # Here the MD5 checksum is calculated so that it can be uploaded with
        # the item as a piece of metadata. This is needed as the AWS etag is
        # not the md5 sum of the uploaded object for items that are uploaded
        # using multipart uploads (large files).
        # See: https://stackoverflow.com/a/43067788
        checksum = S3StorageBroker.hasher(fpath)

        fname = generate_identifier(relpath)
        dest_path = self.data_key_prefix + fname
        self.s3client.upload_file(
            fpath,
            self.bucket,
            dest_path,
            ExtraArgs={'Metadata': {
                'handle': relpath,
                'checksum': checksum,
                }
            }
        )

        return relpath

    def add_item_metadata(self, handle, key, value):
        """Store the given key:value pair for the item associated with handle.

        :param handle: handle for accessing an item before the dataset is
                       frozen
        :param key: metadata key
        :param value: metadata value
        """

        identifier = generate_identifier(handle)
        suffix = '{}.{}.json'.format(identifier, key)
        bucket_fpath = self.fragments_key_prefix + suffix

        self.s3resource.Object(self.bucket, bucket_fpath).put(
            Body=json.dumps(value)
        )

    def iter_item_handles(self):
        """Return iterator over item handles."""

        bucket = self.s3resource.Bucket(self.bucket)

        for obj in bucket.objects.filter(Prefix=self.data_key_prefix).all():
            relpath = obj.get()['Metadata']['handle']

            yield relpath

    def pre_freeze_hook(self):
        pass

    def post_freeze_hook(self):

        # Delete the temporary fragment metadata objects from the bucket.

        # Get the keys of the fragment metadata objects.
        bucket = self.s3resource.Bucket(self.bucket)
        prefix_object_keys = [
            obj.key for obj in
            bucket.objects.filter(Prefix=self.fragments_key_prefix).all()
        ]

        def _chunks(l, n):
            """Yield successive n-sized chunks from l."""

            for i in range(0, len(l), n):
                yield l[i:i + n]

        # Delete the chunks of 500 fragment metadata objects.
        for keys in _chunks(prefix_object_keys, 500):
            keys_as_list_of_dicts = [{'Key': k} for k in keys]
            bucket.objects.delete(
                Delete={'Objects': keys_as_list_of_dicts}
            )

    def get_item_metadata(self, handle):
        """Return dictionary containing all metadata associated with handle.

        In other words all the metadata added using the ``add_item_metadata``
        method.

        :param handle: handle for accessing an item before the dataset is
                       frozen
        :returns: dictionary containing item metadata
        """

        bucket = self.s3resource.Bucket(self.bucket)

        metadata = {}

        identifier = generate_identifier(handle)
        prefix = self.fragments_key_prefix + '{}'.format(identifier)
        for obj in bucket.objects.filter(Prefix=prefix).all():
            metadata_key = obj.key.split('.')[-2]
            response = obj.get()
            value_as_string = response['Body'].read().decode('utf-8')
            value = json.loads(value_as_string)

            metadata[metadata_key] = value

        return metadata

    # HTTP enabling functions

    def generate_key_url(self, key):

        url = "https://{}.s3.amazonaws.com/{}".format(
            self.bucket,
            key
        )

        return url

    def http_enable(self):

        self._make_key_public(self.get_readme_key())
        self._make_key_public(self.get_manifest_key())

        for overlay_name in self.list_overlay_names():
            overlay_fpath = self.overlays_key_prefix + overlay_name + '.json'
            self._make_key_public(overlay_fpath)

        manifest = self.get_manifest()
        for identifier in manifest["items"]:
            self._make_key_public(self.data_key_prefix + identifier)

        http_manifest = self._generate_http_manifest()
        self._write_http_manifest(http_manifest)

        access_url = "https://{}.s3.amazonaws.com/{}".format(
            self.bucket,
            self.uuid
        )

        return access_url

    def _list_historical_readme_keys(self):
        # This method is used to test the
        # BaseStorageBroker.readme_update method.
        prefix = self.get_readme_key() + "-"
        historical_readme_keys = []

        bucket = boto3.resource('s3').Bucket(self.bucket)
        for obj in bucket.objects.filter(Prefix=prefix).all():
            historical_readme_keys.append(obj.key)

        return historical_readme_keys


import ast

from pyrser import meta
from pyrser import parsing

class RuleVisitor(ast.NodeVisitor):
    def __init__(self):
        self.in_optional, self.in_loop, self.in_try = 0, 0, 0

    def generic_visit(self, node):
        raise TypeError("Unhandled {} node".format(node.__class__.__name__))

    def __exit_scope(self) -> ast.stmt:
        """Create the appropriate scope exiting statement.

        The documentation only shows one level and always uses
        'return False' in examples.

        'raise AltFalse()' within a try.
        'break' within a loop.
        'return False' otherwise.
        """
        if self.in_optional:
            return ast.Pass()
        if self.in_try:
            return ast.Raise(
                ast.Call(ast.Name('AltFalse', ast.Load()), [], [], None, None),
                None)
        if self.in_loop:
            return ast.Break()
        return ast.Return(ast.Name('False', ast.Load()))

    #TODO(bps): find a better name to describe what this does
    def _clause(self, pt: parsing.ParserTree) -> [ast.stmt]:
        """Normalize a test expression into a statements list.

        Statements list are returned as-is.
        Expression is packaged as:
        if not expr:
            return False
        """
        if isinstance(pt, list):
            return pt
        return [ast.If(ast.UnaryOp(ast.Not(), pt),
                       [self.__exit_scope()],
                       [])]

    def visit_Call(self, node: parsing.Call) -> ast.expr:
        """Generates python code calling the function.

        fn(*args)
        """
        return ast.Call(
            ast.Attribute(
                ast.Name('self', ast.Load),
                node.callObject.__name__,
                ast.Load()),
            [ast.Str(param) for param in node.params],
            [],
            None,
            None)

    def visit_CallTrue(self, node: parsing.CallTrue) -> ast.expr:
        """Generates python code calling the function and returning True.

        lambda: fn(*args) or True
        """
        return ast.Lambda(
            ast.arguments([], None, None, [], None, None, [], []),
            ast.BoolOp(
                ast.Or(),
                [
                    self.visit_Call(node),
                    ast.Name('True', ast.Load())]))

    def visit_Hook(self, node: parsing.Hook) -> ast.expr:
        """Generates python code calling a hook.

        self.evalHook('hookname', self.ruleNodes[-1])
        """
        return ast.Call(
            ast.Attribute(
                ast.Name('self', ast.Load()), 'evalHook', ast.Load()),
            [
                ast.Str(node.name),
                ast.Subscript(
                    ast.Attribute(
                        ast.Name('self', ast.Load()), 'ruleNodes', ast.Load()),
                    ast.Index(ast.UnaryOp(ast.USub(), ast.Num(1))),
                    ast.Load())],
            [],
            None,
            None)

    def visit_Rule(self, node: parsing.Rule) -> ast.expr:
        """Generates python code calling a rule.

        self.evalRule('rulename')
        """
        return ast.Call(
            ast.Attribute(ast.Name('self', ast.Load()),
                          'evalRule', ast.Load()),
            [ast.Str(node.name)], [], None, None)

    def visit_Capture(self, node: parsing.Capture) -> [ast.stmt] or ast.expr:
        """Generates python code to capture text consumed by a clause.

        #If all clauses can be inlined
        self.beginTag('tagname') and clause and self.endTag('tagname')

        if not self.beginTag('tagname'):
            return False
        <code for the clause>
        if not self.endTag('tagname'):
            return False
        """
        begintag = ast.Attribute(
            ast.Name('self', ast.Load()), 'beginTag', ast.Load())
        endtag = ast.Attribute(
            ast.Name('self', ast.Load()), 'endTag', ast.Load())
        begin = ast.Call(begintag, [ast.Str(node.tagname)], [], None, None)
        end = ast.Call(endtag, [ast.Str(node.tagname)], [], None, None)
        result = [begin, self.visit(node.pt), end]
        for clause in result:
            if not isinstance(clause, ast.expr):
                break
        else:
            return ast.BoolOp(ast.And(), result)
        res = []
        for stmt in map(self._clause, result):
            res.extend(stmt)
        return res

    def visit_Scope(self, node: parsing.Capture) -> [ast.stmt] or ast.expr:
        """Generates python code for a scope.

        if not self.begin():
            return False
        res = self.pt()
        if not self.end():
            return False
        return res
        """
        return ast.Name('scope_not_implemented', ast.Load())
        raise NotImplementedError()

    #TODO(bps): forbid nested alt
    def visit_Alt(self, node: parsing.Alt) -> [ast.stmt]:
        """Generates python code for alternatives.

        try:
            try:
                <code for clause>  #raise AltFalse when alternative is False
                raise AltTrue()
            except AltFalse:
                pass
            return False
        except AltTrue:
            pass
        """
        clauses = [self.visit(clause) for clause in node.ptlist]
        for clause in clauses:
            if not isinstance(clause, ast.expr):
                break
        else:
            return ast.BoolOp(ast.Or(), clauses)
        res = ast.Try([], [ast.ExceptHandler(
            ast.Name('AltTrue', ast.Load()), None, [ast.Pass()])], [], [])
        alt_true = [ast.Raise(ast.Call(
            ast.Name('AltTrue', ast.Load()), [], [], None, None), None)]
        alt_false = [ast.ExceptHandler(
            ast.Name('AltFalse', ast.Load()), None, [ast.Pass()])]
        self.in_try += 1
        for clause in node.ptlist:
            res.body.append(
                ast.Try(self._clause(self.visit(clause)) + alt_true,
                        alt_false, [], []))
        self.in_try -= 1
        res.body.append(self.__exit_scope())
        return [res]

    def combine_exprs_for_clauses(self, exprs: [ast.expr]) -> [ast.stmt]:
        expr = exprs[0] if len(exprs) == 1 else ast.BoolOp(ast.And(), exprs)
        return self._clause(expr)

    def visit_Seq(self, node: parsing.Seq) -> [ast.stmt] or ast.expr:
        """Generates python code for clauses.

        #Continuous clauses which can can be inlined are combined with and
        clause and clause

        if not clause:
            return False
        if not clause:
            return False
        """
        exprs, stmts = [], []
        for clause in node.ptlist:
            clause_ast = self.visit(clause)
            if isinstance(clause_ast, ast.expr):
                exprs.append(clause_ast)
            else:
                if exprs:
                    stmts.extend(self.combine_exprs_for_clauses(exprs))
                    exprs = []
                stmts.extend(self._clause(clause_ast))
        if not stmts:
            return ast.BoolOp(ast.And(), exprs)
        if exprs:
            stmts.extend(self.combine_exprs_for_clauses(exprs))
        return stmts

    def visit_RepOptional(self, node: parsing.RepOptional) -> ([ast.stmt] or
                                                               ast.expr):
        """Generates python code for an optional clause.

        <code for the clause>
        """
        cl_ast = self.visit(node.pt)
        if isinstance(cl_ast, ast.expr):
            return ast.BoolOp(ast.Or(), [cl_ast, ast.Name('True', ast.Load())])
        self.in_optional += 1
        cl_ast = self.visit(node.pt)
        self.in_optional -= 1
        return cl_ast

    def visit_Rep0N(self, node: parsing.Rep0N) -> [ast.stmt]:
        """Generates python code for a clause repeated 0 or more times.

        #If all clauses can be inlined
        while clause:
            pass

        while True:
            <code for the clause>
        """
        cl_ast = self.visit(node.pt)
        if isinstance(cl_ast, ast.expr):
            return [ast.While(cl_ast, [ast.Pass()], [])]
        self.in_loop += 1
        clause = self._clause(self.visit(node.pt))
        self.in_loop -= 1
        return [ast.While(ast.Name('True', ast.Load()), clause, [])]

    def visit_Rep1N(self, node: parsing.Rep0N) -> [ast.stmt]:
        """Generates python code for a clause repeated 1 or more times.

        <code for the clause>
        while True:
            <code for the clause>
        """
        clause = self.visit(node.pt)
        if isinstance(clause, ast.expr):
            return (self._clause(clause) + self.visit_Rep0N(node))
        self.in_loop += 1
        clause = self._clause(self.visit(node.pt))
        self.in_loop -= 1
        return self._clause(self.visit(node.pt)) + [
            ast.While(ast.Name('True', ast.Load()), clause, [])]

def rule_topython(rule: parsing.ParserTree) -> [ast.stmt]:
    return RuleVisitor().visit(rule)

def parserrule_topython(parser: parsing.BasicParser,
                        rulename: str) -> ast.FunctionDef:
    """Generates code for a rule.

    def rulename(self):
        <code for the rule>
        return True
    """
    visitor = RuleVisitor()
    rule = parser._rules[rulename]
    fn_args = ast.arguments([ast.arg('self', None)], None, None, [], None,
                            None, [], [])
    body = visitor._clause(rule_topython(rule))
    body.append(ast.Return(ast.Name('True', ast.Load())))
    return ast.FunctionDef(rulename, fn_args, body, [], None)


# -*- coding: utf-8 -*-

from operator import itemgetter
import requests
import warnings

with warnings.catch_warnings():
    warnings.filterwarnings("ignore", category=UserWarning)
    from fuzzywuzzy.process import extractBests

__all__ = ['CPCApi']

def memoize(f):
    cache = {}

    def aux(*args, **kargs):
        k = (args, tuple(sorted(kargs.items())))
        if k not in cache:
            cache[k] = f(*args, **kargs)
        return cache[k]
    return aux

class CPCApi(object):
    format = 'json'

    def __init__(self, ptype='depute', legislature=None):
        """
        type: depute or senateur
        legislature: 2007-2012 or None
        """

        assert(ptype in ['depute', 'senateur'])
        assert(legislature in ['2007-2012', '2012-2017', None])
        self.legislature = legislature
        self.ptype = ptype
        self.ptype_plural = ptype + 's'
        self.base_url = 'https://%s.nos%s.fr' % (legislature or 'www', self.ptype_plural)

    def synthese(self, month=None):
        """
        month format: YYYYMM
        """
        if month is None and self.legislature == '2012-2017':
            raise AssertionError('Global Synthesis on legislature does not work, see https://github.com/regardscitoyens/nosdeputes.fr/issues/69')

        if month is None:
            month = 'data'

        url = '%s/synthese/%s/%s' % (self.base_url, month, self.format)

        data = requests.get(url).json()
        return [depute[self.ptype] for depute in data[self.ptype_plural]]

    def parlementaire(self, slug_name):
        url = '%s/%s/%s' % (self.base_url, slug_name, self.format)
        return requests.get(url).json()[self.ptype]

    def picture(self, slug_name, pixels='60'):
        return requests.get(self.picture_url(slug_name, pixels=pixels))

    def picture_url(self, slug_name, pixels='60'):
        return '%s/%s/photo/%s/%s' % (self.base_url, self.ptype, slug_name, pixels)

    def search(self, q, page=1):
        # XXX : the response with json format is not a valid json :'(
        # Temporary return csv raw data
        url = '%s/recherche/%s?page=%s&format=%s' % (self.base_url, q, page, 'csv')
        return requests.get(url).content

    @memoize
    def parlementaires(self, active=None):
        if active is None:
            url = '%s/%s/%s' % (self.base_url, self.ptype_plural, self.format)
        else:
            url = '%s/%s/enmandat/%s' % (self.base_url, self.ptype_plural, self.format)

        data = requests.get(url).json()
        return [depute[self.ptype] for depute in data[self.ptype_plural]]

    def search_parlementaires(self, q, field='nom', limit=5):
        return extractBests(q, self.parlementaires(), processor=lambda x: x[field] if type(x) == dict else x, limit=limit)


## Helping class for pretty printing

class indentable:
    """
    base of all fmt objects
    """
    char_indent = " "
    num_indent = 4

    def __init__(self):
        self._indent = 0
        self._is_indented = False
        self._lsdata = None

    def to_str(self, res: str) -> str:
        pass

    def __str__(self):
        self.set_indent()
        strinit = (self.char_indent * self.num_indent) * (self._indent - 1)
        return self.to_str(strinit, self._indent)

    @property
    def lsdata(self) -> list:
        return self._lsdata

    def set_indent(self, indent: int=1):
        if self._is_indented:
            return
        self._indent = indent
        if isinstance(self._lsdata, indentable):
            self._lsdata.set_indent(self._indent)
        if isinstance(self._lsdata, list):
            list_set_indent(self._lsdata, self._indent)
        self._is_indented = True

def catend(dst: str, src: str, indent) -> str:
    """cat two strings but handle \n for tabulation"""
    res = dst
    txtsrc = src
    if not isinstance(src, str):
        txtsrc = str(src)
    for c in list(txtsrc):
        if len(res) > 0 and res[-1] == '\n':
            res += (indentable.char_indent * indentable.num_indent) * \
                   (indent - 1) + c
        else:
            res += c
    return res

def list_set_indent(lst: list, indent: int=1):
    """recurs into list for indentation"""
    for i in lst:
        if isinstance(i, indentable):
            i.set_indent(indent)
        if isinstance(i, list):
            list_set_indent(i, indent)

def list_to_str(lst: list, content: str, indent: int=1):
    """recurs into list for string computing """
    for i in lst:
        if isinstance(i, indentable):
            content = i.to_str(content, indent)
        elif isinstance(i, list):
            content = list_to_str(i, content, indent)
        elif isinstance(i, str):
            content = catend(content, i, indent)
    return content

class block(indentable):
    """
    for {}, (), []
    """
    def __init__(self, beginby: str, endby: str, lsdata: list):
        indentable.__init__(self)
        self._beginby = beginby
        self._endby = endby
        if lsdata is None:
            raise Exception("lsdata can't be None")
        self._lsdata = lsdata

    def to_str(self, res: str, parent_indent) -> str:
        self.set_indent()
        content = catend(res, self._beginby, parent_indent)
        if isinstance(self._lsdata, indentable):
            return catend(self._lsdata.to_str(content, self._indent),
                          self._endby, parent_indent)
        if isinstance(self._lsdata, list):
            content = list_to_str(self._lsdata, content, self._indent)
        return catend(content, self._endby, parent_indent)

class sep(indentable):
    """
    for all list seperated by a char
    """
    def __init__(self, ch: str, lsdata: list):
        indentable.__init__(self)
        self._ch = ch
        if lsdata is None:
            raise Exception("lsdata can't be None")
        self._lsdata = lsdata

    def to_str(self, res: str, parent_indent) -> str:
        self.set_indent()
        content = res
        if isinstance(self._lsdata, indentable):
            return self._lsdata.to_str(res, self._indent)
        if isinstance(self._lsdata, list):
            sz = len(self._lsdata)
            for i in range(sz):
                if isinstance(self._lsdata[i], indentable):
                    content = self._lsdata[i].to_str(content, self._indent)
                elif isinstance(self._lsdata[i], list):
                    content = list_to_str(self._lsdata[i], content,
                                          self._indent)
                else:
                    content = catend(content, self._lsdata[i], self._indent)
                if i < sz - 1:
                    content = catend(content, self._ch, self._indent)
        return content

class end(indentable):
    """
    for all list that end by a char
    """
    def __init__(self, ch: str, lsdata: list):
        indentable.__init__(self)
        self._ch = ch
        if lsdata is None:
            raise Exception("lsdata can't be None")
        self._lsdata = lsdata

    def to_str(self, res: str, parent_indent) -> str:
        self.set_indent()
        content = res
        if isinstance(self._lsdata, indentable):
            return catend(self._lsdata.to_str(res, self._indent), self._ch,
                          self._indent)
        if isinstance(self._lsdata, list):
            for i in self._lsdata:
                if isinstance(i, indentable):
                    content = i.to_str(content, self._indent)
                elif isinstance(i, list):
                    content = list_to_str(i, content, self._indent)
                else:
                    content = catend(content, i, self._indent)
                content = catend(content, self._ch, self._indent)
        return content

class tab(indentable):
    """
    to handle indentation level
    """
    def __init__(self, lsdata: indentable):
        indentable.__init__(self)
        if lsdata is None:
            raise Exception("lsdata can't be None")
        self._lsdata = lsdata

    def set_indent(self, indent: int=1):
        if self._is_indented:
            return
        self._indent = indent + 1
        if isinstance(self._lsdata, indentable):
            self._lsdata.set_indent(self._indent)
        if isinstance(self._lsdata, list):
            list_set_indent(self._lsdata, self._indent)
        self._is_indented = True

    def to_str(self, res: str, parent_indent) -> str:
        self.set_indent()
        if isinstance(self._lsdata, indentable):
            return self._lsdata.to_str(res, self._indent)
        if isinstance(self._lsdata, list):
            content = res
            for i in self._lsdata:
                if isinstance(i, indentable):
                    content = i.to_str(content, self._indent)
                elif isinstance(i, list):
                    content = list_to_str(i, content, self._indent)
                else:
                    content = catend(content, i, self._indent)
        return content


from pyrser import meta
from pyrser.parsing import Node
from pyrser.parsing.base import BasicParser

@meta.hook(BasicParser, "echo")
def echo_nodes(self, *rest):
    """
    Print nodes.

    example::

        R = [
            In : node #echo("coucou", 12, node)
        ]

    """
    txt = ""
    for thing in rest:
        if isinstance(thing, Node):
            txt += self.value(thing)
        else:
            txt += str(thing)
    print(txt)
    return True


"""
This module provide classes for PSL (Pyrser Selectors Language) implementation.
"""

from weakref import *
from pyrser import fmt
from pyrser.ast import state

################

class MatchExpr:
    """
    Ast Node for all match expression.
    """
    pass

class MatchIndice(MatchExpr):
    """
    Ast Node for matching one indice.
    """
    def __init__(self, idx: int, v=None):
        self.idx = idx
        if v is None:
            v = MatchValue()
        self.v = v

    def __eq__(self, other) -> bool:
        return self.idx == other.idx

    def is_in_state(self, s: state.State):
        if self.idx is None and '*' in s.indices:
            return s.indices['*']
        elif self.idx in s.indices:
            return s.indices[self.idx]
        return None

    def attach(
        self,
        s1: state.State,
        s2: state.State,
        sr: state.StateRegister
    ):
        s1.matchIndice(self.idx, s2)

    def build_state_tree(self, tree: list):
        # go deeper
        self.v.build_state_tree(tree)
        # add ourself
        tree.append(self)

    def to_fmt(self) -> fmt.indentable:
        index = '*'
        if self.idx is not None:
            index = str(self.idx)
        res = fmt.block('[' + index + ': ', ']', [])
        res.lsdata.append(self.v.to_fmt())
        return res

    def __repr__(self) -> str:
        return str(self.to_fmt())

class MatchKey(MatchExpr):
    """
    Ast Node for matching one key.
    """
    def __init__(self, key: str, v=None):
        self.key = key
        if v is None:
            v = MatchValue()
        self.v = v

    def __eq__(self, other) -> bool:
        return self.key == other.key

    def is_in_state(self, s: state.State):
        if self.key in s.keys:
            return s.keys[self.key]
        return None

    def attach(
        self,
        s1: state.State,
        s2: state.State,
        sr: state.StateRegister
    ):
        s1.matchKey(self.key, s2)

    def build_state_tree(self, tree: list):
        # go deeper
        self.v.build_state_tree(tree)
        # add ourself
        tree.append(self)

    def to_fmt(self) -> fmt.indentable:
        key = '*'
        if self.key is not None:
            key = repr(self.key)
        res = fmt.block('{' + key + ': ', '}', [])
        res.lsdata.append(self.v.to_fmt())
        return res

    def __repr__(self) -> str:
        return str(self.to_fmt())

class MatchAttr(MatchExpr):
    """
    Ast Node for matching one attribute.
    """
    def __init__(self, name: str=None, v=None):
        self.name = name
        if v is None:
            v = MatchValue()
        self.v = v

    def __eq__(self, other) -> bool:
        return self.name == other.name

    def is_in_state(self, s: state.State):
        if self.name in s.attrs:
            return s.attrs[self.name]
        return None

    def attach(
        self,
        s1: state.State,
        s2: state.State,
        sr: state.StateRegister
    ):
        if self.name is not None:
            s1.matchAttr(self.name, s2)
        else:
            s1.matchDefault(s2)

    def build_state_tree(self, tree: list):
        # go deeper
        self.v.build_state_tree(tree)
        #TODO: match all attr
        # add ourself
        tree.append(self)
        return tree

    def to_fmt(self) -> fmt.indentable:
        n = '*'
        if self.name is not None:
            n = self.name
        n = '.' + n
        v = '*'
        if self.v is not None:
            v = self.v.to_fmt()
        res = fmt.sep('=', [n, v])
        return res

    def __repr__(self) -> str:
        return str(self.to_fmt())

class MatchValue(MatchExpr):
    """
    Ast Node for matching one value.
    """
    def __init__(self, v=None):
        # if v is None -> wildcard
        self.v = v

    def __eq__(self, other) -> bool:
        return self.v == other.v

    def is_in_state(self, s: state.State):
        if self.v is None and '*' in s.values:
            return s.values['*']
        elif str(self.v) in s.values:
            return s.values[str(self.v)]
        return None

    def attach(
        self,
        s1: state.State,
        s2: state.State,
        sr: state.StateRegister
    ):
        s1.matchValue(self.v, s2)

    def build_state_tree(self, tree: list):
        # we are typically a leaf
        tree.append(self)

    def to_fmt(self) -> fmt.indentable:
        res = fmt.sep('', [])
        if self.v is None:
            res.lsdata.append('*')
        else:
            res.lsdata.append(repr(self.v))
        return res

    def __repr__(self) -> str:
        return str(self.to_fmt())

class MatchType(MatchExpr):
    """
    Ast Node for matching exactly one type.
    """
    def __init__(
        self,
        t: type=object,
        attrs: [MatchExpr]=None,
        strict=True,
        iskindof=False
    ):
        self.t = t
        self.attrs = attrs
        self.strict = strict
        self.iskindof = iskindof

    def __eq__(self, other) -> bool:
        return self.t is other.t

    def is_in_state(self, s: state.State):
        if self.iskindof and self.t in s.ktypes:
            return s.ktypes[self.t]
        if not self.iskindof and self.t in s.types:
            return s.types[self.t]
        return None

    def attach(
        self,
        s1: state.State,
        s2: state.State,
        sr: state.StateRegister
    ):
        if self.t is not object:
            if self.iskindof:
                s1.matchKindType(self.t, s2)
            else:
                s1.matchType(self.t, s2)
            # to avoid artefact, store the minimal subelement to match
            s1.minsubelmt = len(self.attrs)

    def build_state_tree(self, tree: list):
        precond = None
        if self.attrs is not None:
            # TODO: attr with unknown name... after all other attr
            if self.strict:
                # go deeper
                for a in self.attrs:
                    a.build_state_tree(tree)
            else:
                # for non-strict match of type, we used unamed events...
                list_alt = []
                list_ev = []
                idx = 0
                for a in self.attrs:
                    # add a new event...
                    ev_name = '.E' + str(id(self)) + '_' + str(idx)
                    # MatchEvent instance will set the event when is reached
                    ev = MatchEvent(ev_name, a)
                    b = list()
                    ev.build_state_tree(b)
                    list_alt.append(b)
                    # add the unamed event name
                    list_ev.append(ev_name)
                    idx += 1
                # add match event for the computed precond...
                list_expr = []
                for e in list_ev:
                    list_expr.append(state.EventNamed(e))
                precond = MatchPrecond(
                    state.EventSeq(list_expr),
                    MatchType(
                        self.t,
                        attrs=[],
                        strict=True,
                        iskindof=self.iskindof
                    ),
                    clean_event=True
                )
                # add all alternative states
                tree.append(list_alt)
        # if needed, add a precondition
        if precond is not None:
            precond.build_state_tree(tree)
        # append ourself to match the type
        elif self.t is not object:
            tree.append(self)

    def to_fmt(self) -> fmt.indentable:
        res = fmt.sep('', [])
        if self.t is not object:
            res.lsdata.append(self.t.__name__)
        else:
            res.lsdata.append('*')
        iparen = []
        if self.attrs is not None:
            # TODO: render unknown attr (.?) at the end after ...,
            # also unknown attr implie 'unstrict' mode
            iparen = fmt.sep(', ', [])
            for a in self.attrs:
                iparen.lsdata.append(a.to_fmt())
        if not self.strict:
            iparen.lsdata.append('...')
        if self.iskindof:
            paren = fmt.block('^(', ')', iparen)
        else:
            paren = fmt.block('(', ')', iparen)
        res.lsdata.append(paren)
        return res

    def __repr__(self) -> str:
        return str(self.to_fmt())

class MatchPrecond(MatchExpr):
    """
    Ast Node for matching a precondition expression.
    """
    def __init__(
        self,
        precond: state.EventExpr,
        v: MatchValue=None,
        clean_event=True
    ):
        self.precond = precond
        self.v = v
        self.clean_event = clean_event

    def __eq__(self, other) -> bool:
        return id(self.precond) == id(other.precond)

    def is_in_state(self, s: state.State):
        if self.precond in s.events_expr:
            return s.events_expr[self.precond]
        return None

    def attach(
        self,
        s1: state.State,
        s2: state.State,
        sr: state.StateRegister
    ):
        s1.matchEventExpr(self.precond, s2, self.clean_event)

    def build_state_tree(self, tree: list):
        # go deeper
        if self.v is not None:
            self.v.build_state_tree(tree)
        # add ourself
        tree.append(self)

    def to_fmt(self) -> fmt.indentable:
        res = fmt.sep(' ', [])
        if self.v is not None:
            res.lsdata.append(self.v.to_fmt())
        if self.clean_event:
            res.lsdata.append('?')
        else:
            res.lsdata.append('??')
        res.lsdata.append('<' + repr(self.precond) + '>')
        return res

    def __repr__(self) -> str:
        return str(self.to_fmt())

class MatchEvent(MatchExpr):
    """
    Ast Node for a Resulting Event.
    """
    def __init__(self, n: str, v: MatchValue):
        self.n = n
        self.v = v

    def __eq__(self, other) -> bool:
        return self.n == other.n

    def attach(
        self,
        s1: state.State,
        s2: state.State,
        sr: state.StateRegister
    ):
        # TODO:!!! the event must be store in the LivingState
        s1.matchEvent(self.n, s2)

    def build_state_tree(self, tree: list):
        # go deeper
        self.v.build_state_tree(tree)
        # add ourself
        tree.append(self)

    def to_fmt(self) -> fmt.indentable:
        res = fmt.sep(' -> ', [self.v.to_fmt(), self.n + ';'])
        return res

    def __repr__(self) -> str:
        return str(self.to_fmt())

class MatchHook(MatchExpr):
    """
    Ast Node for a Resulting Hook.
    """
    def __init__(self, call: callable, v: MatchValue):
        if not callable(call):
            raise ValueError("argument 'call' must be callable")
        self.call = call
        self.n = call.__name__
        self.v = v

    def __eq__(self, other) -> bool:
        return id(self.call) == id(other.call)

    def attach(
        self,
        s1: state.State,
        s2: state.State,
        sr: state.StateRegister
    ):
        s1.matchHook(self.call, s2)

    def build_state_tree(self, tree: list):
        # go deeper
        self.v.build_state_tree(tree)
        # add ourself
        tree.append(self)

    def to_fmt(self) -> fmt.indentable:
        res = fmt.sep(' -> ', [self.v.to_fmt(), '#' + self.n + ';'])
        return res

    def __repr__(self) -> str:
        return str(self.to_fmt())

class MatchBlock(MatchExpr):
    """ Ast Node for a block of PSL statement. """
    def __init__(self, stmts: [MatchExpr]):
        self.stmts = stmts
        self.root_edge = None

    def build_state_tree(self, tree: list, sr: state.StateRegister):
        """ main function for creating a bottom-up tree automata
            for a block of matching statements.
        """
        all_seq = []
        # for all statements populate a list
        # from deeper to nearer of MatchExpr instances.
        for stmt in self.stmts:
            part_seq = list()
            stmt.build_state_tree(part_seq)
            all_seq.append(part_seq)
        # Walk on all MatchExpr instance
        # and create State instance into the StateRegister
        self.root_edge = populate_state_register(all_seq, sr)

    def to_fmt(self) -> fmt.indentable:
        res = fmt.block('{\n', '}', [fmt.tab([])])
        lines = res.lsdata[0].lsdata
        for stmt in self.stmts:
            lines.append(fmt.end('\n', stmt.to_fmt()))
        return res

    def __repr__(self) -> str:
        return str(self.to_fmt())

class Edge:
    """ Class that implement the state transition
        used for state construction.
    """
    def __init__(self, s: state.State):
        self.s = s
        self.next_edge = {}

    def get_next_edge(self, m: MatchExpr):
        # check if the MatchExpr is already handle by the state
        if hasattr(m, 'is_in_state'):
            sX = m.is_in_state(self.s)
            if sX is not None:
                # return the corresponding edge
                return self.next_edge[id(sX)]
        return None

def populate_from_sequence(seq: list, r: ref(Edge), sr: state.StateRegister):
    """ function that connect each other one sequence of MatchExpr. """
    base_state = r
    # we need to detect the last state of the sequence
    idxlast = len(seq) - 1
    idx = 0
    for m in seq:
        # alternatives are represented by builtin list
        if isinstance(m, list):
            # so recursively connect all states of each alternative sequences.
            for item in m:
                populate_from_sequence(item, r, sr)
        elif isinstance(m, MatchExpr):
            # from the current state, have we a existing edge for this event?
            eX = r().get_next_edge(m)
            if eX is None:
                sX = None
                if idx != idxlast:
                    sX = state.State(sr)
                    sX.matchDefault(base_state().s)
                else:
                    # last state of sequence return to the base
                    sX = base_state().s
                eX = Edge(sX)
                r().next_edge[id(sX)] = eX
                m.attach(r().s, sX, sr)
            r = ref(eX)
        idx += 1

def populate_state_register(all_seq: [list], sr: state.StateRegister) -> Edge:
    """ function that create a state for all instance
        of MatchExpr in the given list and connect each others.
    """
    # Basic State
    s0 = state.State(sr)
    # loop on himself
    s0.matchDefault(s0)
    # this is default
    sr.set_default_state(s0)
    # use Edge to store connection
    e0 = Edge(s0)
    for seq in all_seq:
        r = ref(e0)
        # merge all sequences into one tree automata
        populate_from_sequence(seq, r, sr)
    # return edge for debug purpose
    return e0


from pyrser import meta
from pyrser.parsing.base import BasicParser

@meta.hook(BasicParser, "false")
def pred_false(self):
    """
    False in parser. Go to alternative...

    example::

        R = [
            #false | R2
        ]

    """
    return False

@meta.hook(BasicParser, "true")
def pred_true(self):
    """
    True in parser. Do nothing

    example::

        R = [
            R2 | #true
        ]

    """
    return True

@meta.hook(BasicParser, "eq")
def pred_eq(self, n, val):
    """
    Test if a node set with setint or setstr equal a certain value

    example::

        R = [
            __scope__:n
            ['a' #setint(n, 12) | 'b' #setint(n, 14)]
            C
            [#eq(n, 12) D]
        ]

    """
    v1 = n.value
    v2 = val
    if hasattr(val, 'value'):
        v2 = val.value
    if isinstance(v1, int) and not isinstance(v2, int):
        return v1 == int(v2)
    return v1 == v2

@meta.hook(BasicParser, "neq")
def pred_neq(self, n, val):
    """
    Test if a node set with setint or setstr not equal a certain value

    example::

        R = [
            __scope__:n
            ['a' #setint(n, 12) | 'b' #setint(n, 14)]
            C
            [#neq(n, 12) D]
        ]

    """
    v1 = n.value
    v2 = val
    if hasattr(val, 'value'):
        v2 = val.value
    if isinstance(v1, int) and not isinstance(v2, int):
        return v1 != int(v2)
    return v1 != v2


from pyrser import dsl
from pyrser import parsing
from pyrser import meta
from pyrser import error
from collections import ChainMap

class MetaGrammar(parsing.MetaBasicParser):
    """Metaclass for all grammars."""
    def __new__(metacls, name, bases, namespace):
        # for multi heritance we have a simple inheritance relation
        # from the first class in declaration order.
        metabp = parsing.MetaBasicParser
        if len(bases) <= 1:
            cls = metabp.__new__(metacls, name, bases, namespace)
        else:
            b = tuple([bases[0]])
            cls = metabp.__new__(metacls, name, b, namespace)
        # lookup for the metaclass of parsing.
        # Grammar magically inherit rules&hooks from Parser
        if 'Parser' in parsing.base._MetaBasicParser:
            clsbase = parsing.base._MetaBasicParser['Parser']
            # link rules&hooks
            cls._rules = clsbase._rules.new_child()
            cls._hooks = clsbase._hooks.new_child()
            # add rules from DSL
            if 'grammar' in namespace and namespace['grammar'] is not None:
                sname = None
                if 'source' in namespace and namespace['source'] is not None:
                    sname = namespace['source']
                rules = cls.dsl_parser(namespace['grammar'], sname).get_rules()
                if not rules:
                    return rules
                # namespace rules with module/classe name
                for rule_name, rule_pt in rules.items():
                    if '.' not in rule_name:
                        rule_name = cls.__module__ \
                            + '.' + cls.__name__ \
                            + '.' + rule_name
                    meta.set_one(cls._rules, rule_name, rule_pt)
            # add localy define rules (and thus overloads)
            if '_rules' in namespace and namespace['_rules'] is not None:
                cls._rules.update(namespace['_rules'])
            # add localy define hooks
            if '_hooks' in namespace and namespace['_hooks'] is not None:
                cls._hooks.update(namespace['_hooks'])
        # Manage Aggregation
        if len(bases) > 1:
            aggreg_rules = ChainMap()
            aggreg_hooks = ChainMap()
            for subgrammar in bases:
                if hasattr(subgrammar, '_rules'):
                    aggreg_rules = ChainMap(*(aggreg_rules.maps
                                            + subgrammar._rules.maps))
                if hasattr(subgrammar, '_hooks'):
                    aggreg_hooks = ChainMap(*(aggreg_hooks.maps
                                            + subgrammar._hooks.maps))
            # aggregate at toplevel the branch grammar
            cls._rules = ChainMap(*(cls._rules.maps + aggreg_rules.maps))
            cls._hooks = ChainMap(*(cls._hooks.maps + aggreg_hooks.maps))
            # clean redondant in chain for rules
            orderedunique_rules = []
            tocpy_rules = set([id(_) for _ in cls._rules.maps])
            for ch in cls._rules.maps:
                idch = id(ch)
                if idch in tocpy_rules:
                    orderedunique_rules.append(ch)
                    tocpy_rules.remove(idch)
            cls._rules = ChainMap(*orderedunique_rules)
            # clean redondant in chain for hooks
            orderedunique_hooks = []
            tocpy_hooks = set([id(_) for _ in cls._hooks.maps])
            for ch in cls._hooks.maps:
                idch = id(ch)
                if idch in tocpy_hooks:
                    orderedunique_hooks.append(ch)
                    tocpy_hooks.remove(idch)
            cls._hooks = ChainMap(*orderedunique_hooks)
        return cls

class Grammar(parsing.Parser, metaclass=MetaGrammar):
    """
    Base class for all grammars.

    This class turn any class A that inherit it into a grammar.
    Taking the description of the grammar in parameter it will add
    all what is what is needed for A to parse it.
    """
    # Text grammar to generate parsing rules for this class.
    grammar = None
    # Name of the default rule to parse the grammar.
    entry = None
    # DSL parsing class
    dsl_parser = dsl.EBNF

    def after_parse(self, node: parsing.Node) -> parsing.Node:
        """
        If you want to do some stuff after parsing, overload this...
        """
        return node

    def _do_parse(self, entry: str) -> parsing.Node:
        if self.nstream == 0:
            raise ValueError("No opened stream for reading."
                             + " Check if you provide something "
                             + "to read for method parse()"
                             + " or a correct file for method parse_file()"
                             )
        res = None
        self.diagnostic = error.Diagnostic()
        try:
            res = self.eval_rule(entry)
        except error.Diagnostic as d:
            # User put an error rule
            d.notify(
                error.Severity.ERROR,
                "Exception during the evaluation of '%s'" % self._lastRule,
                error.LocationInfo.from_stream(
                    self._stream,
                    is_error=self.from_string
                )
            )
            self.diagnostic = d
        if not res:
            # we fail to parse, but error is not set on the last rule
            self.diagnostic.notify(
                error.Severity.ERROR,
                "Parse error in '%s'" % self._lastRule,
                error.LocationInfo.from_maxstream(
                    self._stream,
                    is_error=self.from_string
                )
            )
            if self.raise_diagnostic:
                raise self.diagnostic
            else:
                return self
        # clear contexted variables
        self.rule_nodes.clear()
        # create a new Diagnostic object for the node result
        res.diagnostic = error.Diagnostic()
        # all is ok
        return self.after_parse(res)

    def parse(self, source: str=None, entry: str=None) -> parsing.Node:
        """Parse source using the grammar"""
        self.from_string = True
        if source is not None:
            self.parsed_stream(source)
        if entry is None:
            entry = self.entry
        if entry is None:
            raise ValueError("No entry rule name defined for {}".format(
                self.__class__.__name__))
        return self._do_parse(entry)

    def parse_file(self, filename: str, entry: str=None) -> parsing.Node:
        """Parse filename using the grammar"""
        self.from_string = False
        import os.path
        with open(filename, 'r') as f:
            self.parsed_stream(f.read(), os.path.abspath(filename))
        if entry is None:
            entry = self.entry
        if entry is None:
            raise ValueError("No entry rule name defined for {}".format(
                self.__class__.__name__))
        return self._do_parse(entry)

generated_class = 0

def build_grammar(inherit: tuple, scope: dict) -> Grammar:
    global generated_class
    class_name = "gen_class_" + str(generated_class)
    generated_class += 1
    return type(class_name, inherit, scope)

def from_string(bnf: str, entry=None, *optional_inherit) -> Grammar:
    """
    Create a Grammar from a string
    """
    inherit = [Grammar] + list(optional_inherit)
    scope = {'grammar': bnf, 'entry': entry}
    return build_grammar(tuple(inherit), scope)

def from_file(fn: str, entry=None, *optional_inherit) -> Grammar:
    """
    Create a Grammar from a file
    """
    import os.path
    if os.path.exists(fn):
        f = open(fn, 'r')
        bnf = f.read()
        f.close()
        inherit = [Grammar] + list(optional_inherit)
        scope = {'grammar': bnf, 'entry': entry, 'source': fn}
        return build_grammar(tuple(inherit), scope)
    raise Exception("File not Found!")


from pyrser import meta
from pyrser.parsing import Node
from pyrser.parsing.base import BasicParser

@meta.hook(BasicParser, "set")
def set_node(self, dst, src):
    """
        Basically copy one node to another.
        usefull to transmit a node from a terminal
        rule as result of the current rule.

        example::

            R = [
                In : node #set(_, node)
            ]

        here the node return by the rule In is
        also the node return by the rule R
    """
    if not isinstance(src, Node):
        dst.value = src
    else:
        dst.set(src)
        idsrc = id(src)
        iddst = id(dst)
        if iddst not in self.id_cache:
            print("DST: %s" % repr(dst))
            print("RULE_NODES %s" % repr(self.rule_nodes))
            print("IDCACHE %s" % repr(self.id_cache))
        if idsrc in self.id_cache:
            k = self.id_cache[idsrc]
            k2 = self.id_cache[iddst]
            if k in self.rule_nodes:
                self.tag_cache[k2] = self.tag_cache[k]
    return True

@meta.hook(BasicParser, "setcapture")
def set_node_as_int(self, dst, src):
    """
        Set a node to a value captured from another node

        example::

            R = [
                In : node #setcapture(_, node)
            ]
    """
    dst.value = self.value(src)
    return True

@meta.hook(BasicParser, "setint")
def set_node_as_int(self, dst, src):
    """
        Set a int literal to a node

        example::

            R = [
                In : node #setint(node, 12)
            ]
    """
    dst.value = int(src)
    return True

@meta.hook(BasicParser, "setstr")
def set_node_as_str(self, dst, src):
    """
        Set a str literal to a node

        example::

            R = [
                In : node #setstr(node, "toto")
            ]
    """
    dst.value = src
    return True

@meta.hook(BasicParser, "get")
def get_subnode(self, dst, ast, expr):
    """
        get the value of subnode

        example::

            R = [
                __scope__:big  getsomethingbig:>big
                #get(_, big, '.val') // copy big.val into _
            ]
    """
    dst.value = eval('ast' + expr)
    return True


# -*- coding: utf-8 -*-
#
# This file is part of Invenio.
# Copyright (C) 2016 CERN.
#
# Invenio is free software; you can redistribute it
# and/or modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 2 of the
# License, or (at your option) any later version.
#
# Invenio is distributed in the hope that it will be
# useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Invenio; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston,
# MA 02111-1307, USA.
#
# In applying this license, CERN does not
# waive the privileges and immunities granted to it by virtue of its status
# as an Intergovernmental Organization or submit itself to any jurisdiction.

"""Invenio-Deposit migration."""

from __future__ import absolute_import, print_function

import datetime
import json
from itertools import islice

from .utils import datetime_toutc

def dt2utc_timestamp(dt):
    """Serialize the datetime to UTC timestamp."""
    return datetime_toutc(dt).isoformat()

def default_serializer(o):
    """Default serializer for json."""
    defs = (
        ((datetime.date, datetime.time),
         lambda x: x.isoformat(), ),
        ((datetime.datetime, ),
         lambda x: dt2utc_timestamp(x), ),
    )
    for types, fun in defs:
        if isinstance(o, types):
            return fun(o)

def _get_depositions(user=None, type=None):
    """Get list of depositions (as iterator).

    This is redefined Deposition.get_depositions classmethod without order-by
    for better performance.
    """
    from invenio.modules.workflows.models import BibWorkflowObject, Workflow
    from invenio.modules.deposit.models import InvalidDepositionType
    from flask import current_app
    from invenio.ext.sqlalchemy import db
    from invenio.modules.deposit.models import Deposition
    params = [
        Workflow.module_name == 'webdeposit',
    ]

    if user:
        params.append(BibWorkflowObject.id_user == user.get_id())
    else:
        params.append(BibWorkflowObject.id_user != 0)

    if type:
        params.append(Workflow.name == type.get_identifier())

    objects = BibWorkflowObject.query.join("workflow").options(
        db.contains_eager('workflow')).filter(*params)

    def _create_obj(o):
        try:
            obj = Deposition(o)
        except InvalidDepositionType as err:
            current_app.logger.exception(err)
            return None
        if type is None or obj.type == type:
            return obj
        return None

    def mapper_filter(objs):
        for o in objs:
            o = _create_obj(o)
            if o is not None:
                yield o

    return mapper_filter(objects)

def get(query, from_date, limit=0, **kwargs):
    """Get deposits."""
    dep_generator = _get_depositions()
    total_depids = 1  # Count of depositions is hard to determine

    # If limit provided, serve only first n=limit items
    if limit > 0:
        dep_generator = islice(dep_generator, limit)
        total_depids = limit
    return total_depids, dep_generator

def dump(deposition, from_date, with_json=True, latest_only=False, **kwargs):
    """Dump the deposition object as dictionary."""
    # Serialize the __getstate__ and fall back to default serializer
    dep_json = json.dumps(deposition.__getstate__(),
                          default=default_serializer)
    dep_dict = json.loads(dep_json)
    dep_dict['_p'] = {}
    dep_dict['_p']['id'] = deposition.id
    dep_dict['_p']['created'] = dt2utc_timestamp(deposition.created)
    dep_dict['_p']['modified'] = dt2utc_timestamp(deposition.modified)
    dep_dict['_p']['user_id'] = deposition.user_id
    dep_dict['_p']['state'] = deposition.state
    dep_dict['_p']['has_sip'] = deposition.has_sip()
    dep_dict['_p']['submitted'] = deposition.submitted
    return dep_dict


# -*- coding: utf-8 -*-
#
# This file is part of Invenio.
# Copyright (C) 2016 CERN.
#
# Invenio is free software; you can redistribute it
# and/or modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 2 of the
# License, or (at your option) any later version.
#
# Invenio is distributed in the hope that it will be
# useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Invenio; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston,
# MA 02111-1307, USA.
#
# In applying this license, CERN does not
# waive the privileges and immunities granted to it by virtue of its status
# as an Intergovernmental Organization or submit itself to any jurisdiction.

"""Bibdocfile metadata dumper."""

from __future__ import absolute_import, print_function

import datetime

import click

from .utils import datetime_toutc

def _get_recids_invenio12(from_date):
    """Get BibDocs for Invenio 1."""
    from invenio.dbquery import run_sql
    return (id[0] for id in run_sql(
        'select id_bibrec from '
        'bibrec_bibdoc as r join bibdoc as d on r.id_bibdoc=d.id '
        'where d.modification_date >=%s',
        (from_date, ), run_on_slave=True))

def _get_recids_invenio2(from_date):
    """Get BibDocs for Invenio 2."""
    from invenio.legacy.dbquery import run_sql
    return (id[0] for id in run_sql(
        'select id_bibrec from '
        'bibrec_bibdoc as r join bibdoc as d on r.id_bibdoc=d.id '
        'where d.modification_date >=%s',
        (from_date, ), run_on_slave=True))

def get_modified_bibdoc_recids(from_date):
    """Get bibdocs."""
    try:
        return set(_get_recids_invenio12(from_date))
    except ImportError:
        return set(_get_recids_invenio2(from_date))

def _import_bibdoc():
    """Import BibDocFile."""
    try:
        from invenio.bibdocfile import BibRecDocs, BibDoc
    except ImportError:
        from invenio.legacy.bibdocfile.api import BibRecDocs, BibDoc
    return BibRecDocs, BibDoc

def dump_bibdoc(recid, from_date, **kwargs):
    """Dump all BibDoc metadata.

    :param docid: BibDoc ID
    :param from_date: Dump only BibDoc revisions newer than this date.

    :returns: List of version of the BibDoc formatted as a dict
    """
    BibRecDocs, BibDoc = _import_bibdoc()

    bibdocfile_dump = []

    date = datetime.datetime.strptime(from_date, '%Y-%m-%d %H:%M:%S')
    for bibdoc in BibRecDocs(recid).list_bibdocs():
        for version in bibdoc.list_versions():
            bibdoc_version = bibdoc.list_version_files(version)
            for f in bibdoc_version:
                if f.is_icon() or f.md < date:
                    # Don't care about icons
                    # Don't care about files not modified since from_date
                    continue
                bibdocfile_dump.append(dict(
                    bibdocid=f.get_bibdocid(),
                    checksum=f.get_checksum(),
                    comment=f.get_comment(),
                    copyright=(
                        f.get_copyright() if hasattr(f, 'get_copyright')
                        else None),
                    creation_date=datetime_toutc(f.cd).isoformat(),
                    description=f.get_description(),
                    encoding=f.encoding,
                    etag=f.etag,
                    flags=f.flags,
                    format=f.get_format(),
                    full_name=f.get_full_name(),
                    full_path=f.get_full_path(),
                    hidden=f.hidden,
                    license=(
                        f.get_license()if hasattr(f, 'get_license') else None),
                    modification_date=datetime_toutc(f.md).isoformat(),
                    name=f.get_name(),
                    mime=f.mime,
                    path=f.get_path(),
                    recid=f.get_recid(),
                    recids_doctype=f.recids_doctypes,
                    size=f.get_size(),
                    status=f.get_status(),
                    subformat=f.get_subformat(),
                    superformat=f.get_superformat(),
                    type=f.get_type(),
                    url=f.get_url(),
                    version=f.get_version(),
                ))

    return bibdocfile_dump

def get_check():
    """Get bibdocs to check."""
    try:
        from invenio.dbquery import run_sql
    except ImportError:
        from invenio.legacy.dbquery import run_sql

    return (
        run_sql('select count(id) from bibdoc', run_on_slave=True)[0][0],
        [id[0] for id in run_sql('select id from bibdoc', run_on_slave=True)],
    )

def check(id_):
    """Check bibdocs."""
    BibRecDocs, BibDoc = _import_bibdoc()

    try:
        BibDoc(id_).list_all_files()
    except Exception:
        click.secho("BibDoc {0} failed check.".format(id_), fg='red')


# -*- coding: utf-8 -*-
#
# This file is part of Invenio.
# Copyright (C) 2016 CERN.
#
# Invenio is free software; you can redistribute it
# and/or modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 2 of the
# License, or (at your option) any later version.
#
# Invenio is distributed in the hope that it will be
# useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Invenio; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston,
# MA 02111-1307, USA.
#
# In applying this license, CERN does not
# waive the privileges and immunities granted to it by virtue of its status
# as an Intergovernmental Organization or submit itself to any jurisdiction.

"""Invenio OAuth2Server Token dump functions."""

from __future__ import absolute_import, print_function

from .utils import dt2iso_or_empty

def get(*args, **kwargs):
    """Get users."""
    from invenio.modules.oauth2server.models import Token
    q = Token.query
    return q.count(), q.all()

def dump(obj, from_date, with_json=True, latest_only=False, **kwargs):
    """Dump the oauth2server tokens."""
    return dict(id=obj.id,
                client_id=obj.client_id,
                user_id=obj.user_id,
                token_type=obj.token_type,
                access_token=obj.access_token,
                refresh_token=obj.refresh_token,
                expires=dt2iso_or_empty(obj.expires),
                _scopes=obj._scopes,
                is_personal=obj.is_personal,
                is_internal=obj.is_internal)


# -*- coding: utf-8 -*-
#
# This file is part of Invenio.
# Copyright (C) 2016 CERN.
#
# Invenio is free software; you can redistribute it
# and/or modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 2 of the
# License, or (at your option) any later version.
#
# Invenio is distributed in the hope that it will be
# useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Invenio; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston,
# MA 02111-1307, USA.
#
# In applying this license, CERN does not
# waive the privileges and immunities granted to it by virtue of its status
# as an Intergovernmental Organization or submit itself to any jurisdiction.

"""Community dump functions."""

from __future__ import absolute_import, print_function

def get(*args, **kwargs):
    """Get UserEXT objects."""
    try:
        from invenio.modules.accounts.models import UserEXT
    except ImportError:
        from invenio_accounts.models import UserEXT
    q = UserEXT.query
    return q.count(), q.all()

def dump(u, from_date, with_json=True, latest_only=False, **kwargs):
    """Dump the UserEXt objects as a list of dictionaries.

    :param u: UserEXT to be dumped.
    :type u: `invenio_accounts.models.UserEXT [Invenio2.x]`
    :returns: User serialized to dictionary.
    :rtype: dict
    """
    return dict(id=u.id, method=u.method, id_user=u.id_user)


# -*- coding: utf-8 -*-
#
# This file is part of Invenio.
# Copyright (C) 2016 CERN.
#
# Invenio is free software; you can redistribute it
# and/or modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 2 of the
# License, or (at your option) any later version.
#
# Invenio is distributed in the hope that it will be
# useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Invenio; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston,
# MA 02111-1307, USA.
#
# In applying this license, CERN does not
# waive the privileges and immunities granted to it by virtue of its status
# as an Intergovernmental Organization or submit itself to any jurisdiction.

"""Community dump functions."""

from __future__ import absolute_import, print_function

def get(*args, **kwargs):
    """Get communities."""
    from invenio.modules.communities.models import FeaturedCommunity
    q = FeaturedCommunity.query
    return q.count(), q.all()

def dump(fc, from_date, with_json=True, latest_only=False, **kwargs):
    """Dump the community object as dictionary.

    :param fc: Community featuring to be dumped.
    :type fc: `invenio_communities.models.FeaturedCommunity [Invenio2.x]`
    :returns: Community serialized to dictionary.
    :rtype: dict
    """
    return dict(id=fc.id,
                id_community=fc.id_community,
                start_date=fc.start_date.isoformat())


# -*- coding: utf-8 -*-
#
# This file is part of Invenio.
# Copyright (C) 2016 CERN.
#
# Invenio is free software; you can redistribute it
# and/or modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 2 of the
# License, or (at your option) any later version.
#
# Invenio is distributed in the hope that it will be
# useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Invenio; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston,
# MA 02111-1307, USA.
#
# In applying this license, CERN does not
# waive the privileges and immunities granted to it by virtue of its status
# as an Intergovernmental Organization or submit itself to any jurisdiction.
"""Record and BibDocFile dump functions."""

from __future__ import absolute_import, print_function

import datetime
import zlib

from .bibdocfile import dump_bibdoc, get_modified_bibdoc_recids
from .utils import datetime_toutc, memoize

def _get_modified_recids_invenio12(from_date):
    """Get record ids for Invenio 1."""
    from invenio.search_engine import search_pattern
    from invenio.dbquery import run_sql
    return set((id[0] for id in run_sql(
        'select id from bibrec where modification_date >= %s',
        (from_date, ), run_on_slave=True))), search_pattern

def _get_modified_recids_invenio2(from_date):
    """Get record ids for Invenio 2."""
    from invenio.legacy.search_engine import search_pattern
    from invenio.modules.records.models import Record

    date = datetime.datetime.strptime(from_date, '%Y-%m-%d %H:%M:%S')
    return set(
        (x[0]
         for x in Record.query.filter(Record.modification_date >= date).values(
             Record.id))), search_pattern

@memoize
def _get_collection_restrictions(collection):
    """Get all restrictions for a given collection, users and fireroles."""
    try:
        from invenio.dbquery import run_sql
        from invenio.access_control_firerole import compile_role_definition
    except ImportError:
        from invenio.modules.access.firerole import compile_role_definition
        from invenio.legacy.dbquery import run_sql

    res = run_sql(
        'SELECT r.firerole_def_src, email '
        'FROM accROLE as r '
        'JOIN accROLE_accACTION_accARGUMENT ON r.id=id_accROLE '
        'JOIN accARGUMENT AS a ON a.id=id_accARGUMENT '
        'JOIN user_accROLE AS u ON r.id=u.id_accROLE '
        'JOIN user ON user.id=u.id_user '
        'WHERE a.keyword="collection" AND '
        'a.value=%s AND '
        'id_accACTION=(select id from accACTION where name="viewrestrcoll")',
        (collection, ), run_on_slave=True
    )
    fireroles = set()
    users = set()

    for f, u in res:
        fireroles.add(compile_role_definition(f))
        users.add(u)

    return {'fireroles': list(fireroles), 'users': users}

def get_modified_recids(from_date):
    """Get modified recids."""
    try:
        return _get_modified_recids_invenio12(from_date)
    except ImportError:
        return _get_modified_recids_invenio2(from_date)

def get_record_revisions(recid, from_date):
    """Get record revisions."""
    try:
        from invenio.dbquery import run_sql
    except ImportError:
        from invenio.legacy.dbquery import run_sql

    return run_sql(
        'SELECT job_date, marcxml '
        'FROM hstRECORD WHERE id_bibrec = %s AND job_date >= %s '
        'ORDER BY job_date ASC', (recid, from_date),
        run_on_slave=True)

def get_record_collections(recid):
    """Get all collections the record belong to."""
    try:
        from invenio.search_engine import (
            get_all_collections_of_a_record,
            get_restricted_collections_for_recid)
    except ImportError:
        from invenio.legacy.search_engine import (
            get_all_collections_of_a_record,
            get_restricted_collections_for_recid)

    collections = {
        'all':
        get_all_collections_of_a_record(recid, recreate_cache_if_needed=False),
    }
    collections['restricted'] = dict(
        (coll, _get_collection_restrictions(coll))
        for coll in get_restricted_collections_for_recid(
                recid, recreate_cache_if_needed=False))

    return collections

def dump_record_json(marcxml):
    """Dump JSON of record."""
    try:
        from invenio.modules.records.api import Record
        d = Record.create(marcxml, 'marc')
        return d.dumps(clean=True)
    except ImportError:
        from invenio.bibfield import create_record
        d = create_record(marcxml, master_format='marc')
        return d.dumps()

def get(query, from_date, **kwargs):
    """Get recids matching query and with changes."""
    recids, search_pattern = get_modified_recids(from_date)
    recids = recids.union(get_modified_bibdoc_recids(from_date))

    if query:
        recids = recids.intersection(
            set(search_pattern(p=query.encode('utf-8'))))

    return len(recids), recids

def dump(recid,
         from_date,
         with_json=False,
         latest_only=False,
         with_collections=False,
         **kwargs):
    """Dump MARCXML and JSON representation of a record.

    :param recid: Record identifier
    :param from_date: Dump only revisions from this date onwards.
    :param with_json: If ``True`` use old ``Record.create`` to generate the
        JSON representation of the record.
    :param latest_only: Dump only the last revision of the record metadata.
    :param with_collections: If ``True`` dump the list of collections that the
        record belongs to.
    :returns: List of versions of the record.
    """
    # Grab latest only
    if latest_only:
        revision_iter = [get_record_revisions(recid, from_date)[-1]]
    else:
        revision_iter = get_record_revisions(recid, from_date)

    # Dump revisions
    record_dump = dict(
        record=[],
        files=[],
        recid=recid,
        collections=get_record_collections(recid)
        if with_collections else None, )

    for revision_date, revision_marcxml in revision_iter:
        marcxml = zlib.decompress(revision_marcxml)
        record_dump['record'].append(
            dict(
                modification_datetime=datetime_toutc(revision_date)
                .isoformat(),
                marcxml=marcxml,
                json=dump_record_json(marcxml) if with_json else None, ))

    record_dump['files'] = dump_bibdoc(recid, from_date)

    return record_dump


# -*- coding: utf-8 -*-
#
# This file is part of Invenio.
# Copyright (C) 2016 CERN.
#
# Invenio is free software; you can redistribute it
# and/or modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 2 of the
# License, or (at your option) any later version.
#
# Invenio is distributed in the hope that it will be
# useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Invenio; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston,
# MA 02111-1307, USA.
#
# In applying this license, CERN does not
# waive the privileges and immunities granted to it by virtue of its status
# as an Intergovernmental Organization or submit itself to any jurisdiction.

"""Remote account dump functions."""

from __future__ import absolute_import, print_function

def get(*args, **kwargs):
    """Get users."""
    from invenio.modules.oauthclient.models import RemoteAccount
    q = RemoteAccount.query
    return q.count(), q.all()

def dump(ra, from_date, with_json=True, latest_only=False, **kwargs):
    """Dump the remote accounts as a list of dictionaries.

    :param ra: Remote account to be dumped.
    :type ra: `invenio_oauthclient.models.RemoteAccount [Invenio2.x]`
    :returns: Remote accounts serialized to dictionary.
    :rtype: dict
    """
    return dict(id=ra.id, user_id=ra.user_id, client_id=ra.client_id,
                extra_data=ra.extra_data)