# encoding: utf-8
# Copyright (C) 2017 John Törnblom
#
# This file is part of pyxtuml.
#
# pyxtuml is free software: you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation, either
# version 3 of the License, or (at your option) any later version.
#
# pyxtuml is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with pyxtuml. If not, see <http://www.gnu.org/licenses/>.

import collections
import functools
import os
import logging
import keyword
import xtuml

from xtuml import navigate_one as one
from xtuml import navigate_many as many
from xtuml import navigate_subtype as subtype
from xtuml import where_eq as where

from bridgepoint import interpret
from bridgepoint import external_entities as builtin_ee
from bridgepoint import schema

logger = logging.getLogger(__name__)

class OoaOfOoaException(Exception):
    pass

class Domain(xtuml.MetaModel):
    symbols = None

    def __init__(self, id_generator=None):
        self.symbols = dict()
        xtuml.MetaModel.__init__(self, id_generator)

    def add_symbol(self, name, handle):
        self.symbols[name] = handle

    def find_symbol(self, name):
        if name in self.symbols:
            return self.symbols[name]

        try:
            return self.find_class(name)
        except:
            raise OoaOfOoaException('Unknown symbol %s' % name)

def is_contained_in(pe_pe, root):
    '''
    Determine if a PE_PE is contained within a EP_PKG or a C_C.
    '''
    if not pe_pe:
        return False

    if type(pe_pe).__name__ != 'PE_PE':
        pe_pe = one(pe_pe).PE_PE[8001]()

    ep_pkg = one(pe_pe).EP_PKG[8000]()
    c_c = one(pe_pe).C_C[8003]()

    if root in [ep_pkg, c_c]:
        return True

    elif is_contained_in(ep_pkg, root):
        return True

    elif is_contained_in(c_c, root):
        return True

    else:
        return False

def is_global(pe_pe):
    '''
    Check if a PE_PE is globally defined, i.e. not inside a C_C
    '''
    if type(pe_pe).__name__ != 'PE_PE':
        pe_pe = one(pe_pe).PE_PE[8001]()

    if one(pe_pe).C_C[8003]():
        return False

    pe_pe = one(pe_pe).EP_PKG[8000].PE_PE[8001]()
    if not pe_pe:
        return True

    return is_global(pe_pe)

def get_defining_component(pe_pe):
    '''
    Get the BridgePoint component (C_C) that defines the packeable element
    *pe_pe*.
    '''
    if pe_pe is None:
        return None

    if type(pe_pe).__name__ != 'PE_PE':
        pe_pe = one(pe_pe).PE_PE[8001]()

    ep_pkg = one(pe_pe).EP_PKG[8000]()
    if ep_pkg:
        return get_defining_component(ep_pkg)

    return one(pe_pe).C_C[8003]()

def get_attribute_type(o_attr):
    '''
    Get the base data type (S_DT) associated with a BridgePoint attribute.
    '''
    ref_o_attr = one(o_attr).O_RATTR[106].O_BATTR[113].O_ATTR[106]()
    if ref_o_attr:
        return get_attribute_type(ref_o_attr)
    else:
        return one(o_attr).S_DT[114]()

def _get_data_type_name(s_dt):
    '''
    Convert a BridgePoint data type to a pyxtuml meta model type.
    '''
    s_cdt = one(s_dt).S_CDT[17]()
    if s_cdt and s_cdt.Core_Typ in range(1, 6):
        return s_dt.Name.upper()

    if one(s_dt).S_EDT[17]():
        return 'INTEGER'

    s_dt = one(s_dt).S_UDT[17].S_DT[18]()
    if s_dt:
        return _get_data_type_name(s_dt)

def _get_related_attributes(r_rgo, r_rto):
    '''
    The two lists of attributes which relates two classes in an association.
    '''
    l1 = list()
    l2 = list()

    ref_filter = lambda ref: ref.OIR_ID == r_rgo.OIR_ID
    for o_ref in many(r_rto).O_RTIDA[110].O_REF[111](ref_filter):
        o_attr = one(o_ref).O_RATTR[108].O_ATTR[106]()
        l1.append(o_attr.Name)

        o_attr = one(o_ref).O_RTIDA[111].O_OIDA[110].O_ATTR[105]()
        l2.append(o_attr.Name)

    return l1, l2

def mk_enum(s_edt):
    '''
    Create a named tuple from a BridgePoint enumeration.
    '''
    s_dt = one(s_edt).S_DT[17]()
    enums = list()
    kwlist =['False', 'None', 'True'] + keyword.kwlist
    for enum in many(s_edt).S_ENUM[27]():
        if enum.Name in kwlist:
            enums.append(enum.Name + '_')
        else:
            enums.append(enum.Name)

    Enum = collections.namedtuple(s_dt.Name, enums)
    return Enum(*range(len(enums)))

def mk_bridge(metamodel, s_brg):
    '''
    Create a python function from a BridgePoint bridge.
    '''
    action = s_brg.Action_Semantics_internal
    label = s_brg.Name
    return lambda **kwargs: interpret.run_function(metamodel, label,
                                                   action, kwargs)

def mk_external_entity(metamodel, s_ee):
    '''
    Create a python object from a BridgePoint external entity with bridges
    realized as python member functions.
    '''
    bridges = many(s_ee).S_BRG[19]()
    names = [brg.Name for brg in bridges]
    EE = collections.namedtuple(s_ee.Key_Lett, names)

    funcs = list()
    for s_brg in many(s_ee).S_BRG[19]():
        fn = mk_bridge(metamodel, s_brg)
        funcs.append(fn)

    return EE(*funcs)

def mk_function(metamodel, s_sync):
    '''
    Create a python function from a BridgePoint function.
    '''
    action = s_sync.Action_Semantics_internal
    label = s_sync.Name
    return lambda **kwargs: interpret.run_function(metamodel, label,
                                                   action, kwargs)

def mk_constant(cnst_syc):
    '''
    Create a python value from a BridgePoint constant.
    '''
    s_dt = one(cnst_syc).S_DT[1500]()
    cnst_lsc = one(cnst_syc).CNST_LFSC[1502].CNST_LSC[1503]()

    if s_dt.Name == 'boolean':
        return cnst_lsc.Value.lower() == 'true'

    if s_dt.Name == 'integer':
        return int(cnst_lsc.Value)

    if s_dt.Name == 'real':
        return float(cnst_lsc.Value)

    if s_dt.Name == 'string':
        return str(cnst_lsc.Value)

def mk_operation(metaclass, o_tfr):
    '''
    Create a python function that interprets that action of a BridgePoint class
    operation.
    '''
    o_obj = one(o_tfr).O_OBJ[115]()
    action = o_tfr.Action_Semantics_internal
    label = '%s::%s' % (o_obj.Name, o_tfr.Name)
    run = interpret.run_operation

    if o_tfr.Instance_Based:
        return lambda self, **kwargs: run(metaclass, label, action, kwargs, self)
    else:
        fn = lambda cls, **kwargs: run(metaclass, label, action, kwargs, None)
        return classmethod(fn)

def mk_derived_attribute(metaclass, o_dbattr):
    '''
    Create a python property that interprets that action of a BridgePoint derived
    attribute.
    '''
    o_attr = one(o_dbattr).O_BATTR[107].O_ATTR[106]()
    o_obj = one(o_attr).O_OBJ[102]()
    action = o_dbattr.Action_Semantics_internal
    label = '%s::%s' % (o_obj.Name, o_attr.Name)
    fget = functools.partial(interpret.run_derived_attribute, metaclass,
                             label, action, o_attr.Name)
    return property(fget)

def mk_class(m, o_obj, derived_attributes=False):
    '''
    Create a pyxtuml class from a BridgePoint class.
    '''
    first_filter = lambda selected: not one(selected).O_ATTR[103, 'succeeds']()
    o_attr = one(o_obj).O_ATTR[102](first_filter)
    attributes = list()

    while o_attr:
        s_dt = get_attribute_type(o_attr)
        ty = _get_data_type_name(s_dt)
        if not derived_attributes and one(o_attr).O_BATTR[106].O_DBATTR[107]():
            pass
#            logger.warning('Omitting derived attribute %s.%s ' %
#                           (o_obj.Key_Lett, o_attr.Name))
        elif not ty:
            logger.warning('Omitting unsupported attribute %s.%s ' %
                           (o_obj.Key_Lett, o_attr.Name))
        else:
            attributes.append((o_attr.Name, ty))

        o_attr = one(o_attr).O_ATTR[103, 'precedes']()

    metaclass = m.define_class(o_obj.Key_Lett, list(attributes), o_obj.Descrip)

    for o_id in many(o_obj).O_ID[104]():
        o_oida = many(o_id).O_OIDA[105]()
        o_attrs = many(o_oida).O_ATTR[105]()
        if not derived_attributes and one(o_attrs).O_BATTR[106].O_DBATTR[107]():
            logger.warning('Omitting unique identifier %s.I%d' %
                           (o_obj.Key_Lett, o_id.Oid_ID + 1))
            continue

        names = [o_attr.Name for o_attr in o_attrs]
        m.define_unique_identifier(o_obj.Key_Lett, o_id.Oid_ID + 1, *names)

    for o_tfr in many(o_obj).O_TFR[115]():
        fn = mk_operation(metaclass, o_tfr)
        setattr(metaclass.clazz, o_tfr.Name, fn)

    for o_dbattr in many(o_obj).O_ATTR[102].O_BATTR[106].O_DBATTR[107]():
        o_attr = one(o_dbattr).O_BATTR[107].O_ATTR[106]()
        fn = mk_derived_attribute(metaclass, o_dbattr)
        setattr(metaclass.clazz, o_attr.Name, fn)

    return metaclass

def mk_simple_association(m, r_simp):
    '''
    Create a pyxtuml association from a simple association in BridgePoint.
    '''
    r_rel = one(r_simp).R_REL[206]()

    r_form = one(r_simp).R_FORM[208]()
    r_part = one(r_simp).R_PART[207]()

    r_rgo = one(r_form).R_RGO[205]()
    r_rto = one(r_part).R_RTO[204]()

    if not r_form:
        logger.info('unformalized association R%s' % (r_rel.Numb))
        r_form = one(r_simp).R_PART[207](lambda sel: sel != r_part)
        r_rgo = one(r_form).R_RTO[204]()

    source_o_obj = one(r_rgo).R_OIR[203].O_OBJ[201]()
    target_o_obj = one(r_rto).R_OIR[203].O_OBJ[201]()
    source_ids, target_ids = _get_related_attributes(r_rgo, r_rto)

    if source_o_obj.Obj_ID != target_o_obj.Obj_ID:
        source_phrase = target_phrase = ''
    else:
        source_phrase = r_part.Txt_Phrs
        target_phrase = r_form.Txt_Phrs

    m.define_association(rel_id=r_rel.Numb,
                         source_kind=source_o_obj.Key_Lett,
                         target_kind=target_o_obj.Key_Lett,
                         source_keys=source_ids,
                         target_keys=target_ids,
                         source_conditional=r_form.Cond,
                         target_conditional=r_part.Cond,
                         source_phrase=source_phrase,
                         target_phrase=target_phrase,
                         source_many=r_form.Mult,
                         target_many=r_part.Mult)

def mk_linked_association(m, r_assoc):
    '''
    Create pyxtuml associations from a linked association in BridgePoint.
    '''
    r_rel = one(r_assoc).R_REL[206]()
    r_rgo = one(r_assoc).R_ASSR[211].R_RGO[205]()
    source_o_obj = one(r_rgo).R_OIR[203].O_OBJ[201]()

    def _mk_assoc(side1, side2):
        r_rto = one(side1).R_RTO[204]()

        target_o_obj = one(r_rto).R_OIR[203].O_OBJ[201]()
        source_ids, target_ids = _get_related_attributes(r_rgo, r_rto)
        if side1.Obj_ID != side2.Obj_ID:
            source_phrase = target_phrase = ''
        else:
            source_phrase = side1.Txt_Phrs
            target_phrase = side2.Txt_Phrs

        m.define_association(rel_id=r_rel.Numb,
                             source_kind=source_o_obj.Key_Lett,
                             target_kind=target_o_obj.Key_Lett,
                             source_keys=source_ids,
                             target_keys=target_ids,
                             source_conditional=side2.Cond,
                             target_conditional=False,
                             source_phrase=source_phrase,
                             target_phrase=target_phrase,
                             source_many=side2.Mult,
                             target_many=False)

    r_aone = one(r_assoc).R_AONE[209]()
    r_aoth = one(r_assoc).R_AOTH[210]()

    _mk_assoc(r_aone, r_aoth)
    _mk_assoc(r_aoth, r_aone)

def mk_subsuper_association(m, r_subsup):
    '''
    Create pyxtuml associations from a sub/super association in BridgePoint.
    '''
    r_rel = one(r_subsup).R_REL[206]()
    r_rto = one(r_subsup).R_SUPER[212].R_RTO[204]()
    target_o_obj = one(r_rto).R_OIR[203].O_OBJ[201]()

    for r_sub in many(r_subsup).R_SUB[213]():
        r_rgo = one(r_sub).R_RGO[205]()

        source_o_obj = one(r_rgo).R_OIR[203].O_OBJ[201]()
        source_ids, target_ids = _get_related_attributes(r_rgo, r_rto)
        m.define_association(rel_id=r_rel.Numb,
                             source_kind=source_o_obj.Key_Lett,
                             target_kind=target_o_obj.Key_Lett,
                             source_keys=source_ids,
                             target_keys=target_ids,
                             source_conditional=True,
                             target_conditional=False,
                             source_phrase='',
                             target_phrase='',
                             source_many=False,
                             target_many=False)

def mk_derived_association(m, r_comp):
    '''
    Create a pyxtuml association from a derived association in BridgePoint.
    '''
    pass

def mk_association(m, r_rel):
    '''
    Create a pyxtuml association from a R_REL in ooaofooa.
    '''
    handler = {
        'R_SIMP': mk_simple_association,
        'R_ASSOC': mk_linked_association,
        'R_SUBSUP': mk_subsuper_association,
        'R_COMP': mk_derived_association,
    }
    inst = subtype(r_rel, 206)
    fn = handler.get(type(inst).__name__)
    return fn(m, inst)

def mk_component(bp_model, c_c=None, derived_attributes=False):
    '''
    Create a pyxtuml meta model from a BridgePoint model.
    Optionally, restrict to classes and associations contained in the
    component c_c.
    '''
    target = Domain()

    c_c_filt = lambda sel: c_c is None or is_contained_in(sel, c_c)

    for o_obj in bp_model.select_many('O_OBJ', c_c_filt):
        mk_class(target, o_obj, derived_attributes)

    for r_rel in bp_model.select_many('R_REL', c_c_filt):
        mk_association(target, r_rel)

    for s_sync in bp_model.select_many('S_SYNC', c_c_filt):
        fn = mk_function(target, s_sync)
        target.add_symbol(s_sync.Name, fn)

    for s_dt in bp_model.select_many('S_DT', c_c_filt):
        s_edt = one(s_dt).S_EDT[17]()
        if s_edt:
            enum = mk_enum(s_edt)
            target.add_symbol(s_dt.Name, enum)

    for cnst_csp in bp_model.select_many('CNST_CSP', c_c_filt):
        for cnst_syc in many(cnst_csp).CNST_SYC[1504]():
            value = mk_constant(cnst_syc)
            target.add_symbol(cnst_syc.Name, value)

    for ass in target.associations:
        ass.formalize()

    for s_ee in bp_model.select_many('S_EE', c_c_filt):
        if s_ee.Key_Lett in ['LOG', 'ARCH', 'TIM', 'NVS', 'PERSIST']:
            target.add_symbol(s_ee.Key_Lett, getattr(builtin_ee, s_ee.Key_Lett))

        else:
            ee = mk_external_entity(target, s_ee)
            target.add_symbol(s_ee.Key_Lett, ee)

    return target

class ModelLoader(xtuml.ModelLoader):
    '''
    A *xtuml.MetaModel* loader with ooaofooa schema and globals pre-defined.
    '''

    def __init__(self, load_globals=True):
        xtuml.ModelLoader.__init__(self)
        self.input(schema.classes, 'ooaofooa classes (v%02.1f)' % schema.__version__)
        self.input(schema.associations, 'ooaofooa associations (v%02.1f)' % schema.__version__)
        self.input(schema.indices, 'ooaofooa indices (v%02.1f)' % schema.__version__)
        if load_globals:
            self.input(schema.globals, 'predefined ooaofooa globals (v%02.1f)' % schema.__version__)

    def filename_input(self, path_or_filename):
        '''
        Open and read input from a *path or filename*, and parse its content.

        If the filename is a directory, files that ends with .xtuml located
        somewhere in the directory or sub directories will be loaded as well.
        '''
        if os.path.isdir(path_or_filename):
            for path, _, files in os.walk(path_or_filename):
                for name in files:
                    if name.endswith('.xtuml'):
                        xtuml.ModelLoader.filename_input(self, os.path.join(path, name))
        else:
            xtuml.ModelLoader.filename_input(self, path_or_filename)

    def build_component(self, name=None, derived_attributes=False):
        '''
        Instantiate and build a component from ooaofooa named *name* as a
        pyxtuml model. Classes, associations, attributes and unique identifers,
        i.e. O_OBJ, R_REL, O_ATTR in ooaofooa, are defined in the resulting
        pyxtuml model.

        Optionally, control whether *derived attributes* shall be mapped into
        the resulting pyxtuml model as attributes or not.

        Futhermore, if no *name* is provided, the entire content of the ooaofooa
        model is instantiated into the pyxtuml model.
        '''
        mm = self.build_metamodel()
        c_c = mm.select_any('C_C', where(Name=name))
        if c_c:
            return mk_component(mm, c_c, derived_attributes)
        elif name:
            raise OoaOfOoaException('Unable to find the component %s' % name)
        else:
            return mk_component(mm, c_c, derived_attributes)

def _mk_loader(resource, load_globals):
    resource = resource or list()

    if isinstance(resource, str):
        resource = [resource]

    loader = Loader(load_globals)
    for filename in resource:
        loader.filename_input(filename)

    return loader

def load_metamodel(resource=None, load_globals=True):
    '''
    Load and return a metamodel expressed in ooaofooa from a *resource*.
    The resource may be either a filename, a path, or a list of filenames
    and/or paths.
    '''
    loader = _mk_loader(resource, load_globals)
    return loader.build_metamodel()

def load_component(resource, name=None, load_globals=True):
    '''
    Load and return a model from a *resource*. The resource may be either a
    filename, a path, or a list of filenames and/or paths.
    '''
    loader = _mk_loader(resource, load_globals)
    return loader.build_component()

def delete_globals(m, disconnect=False):
    '''
    Remove global instances, e.g. the core data type integer.
    '''
    filt = lambda sel: (247728914420827907967735776184937480192 <=
                        sel.DT_ID <=
                        247728914420827907967735776184937480208)

    for s_dt in m.select_many('S_DT', filt):
        xtuml.delete(one(s_dt).PE_PE[8001](), disconnect)
        xtuml.delete(subtype(s_dt, 17), disconnect)
        xtuml.delete(s_dt, disconnect)

# Backwards compatibility with older versions of pyxtuml
Loader = ModelLoader
empty_model = load_metamodel


# -*- coding: utf-8 -*-
"""
   zeronimo.core
   ~~~~~~~~~~~~~

   :copyright: (c) 2013-2017 by Heungsub Lee
   :license: BSD, see LICENSE for more details.

"""
from __future__ import absolute_import

from collections import Iterator
from contextlib import contextmanager
import sys
import traceback
from warnings import warn

from gevent import Greenlet, GreenletExit, Timeout
from gevent.pool import Group
from gevent.queue import Queue
from six import int2byte, reraise, StringIO, string_types, viewvalues
try:
    from libuuid import uuid4_bytes
except ImportError:
    import uuid
    uuid4_bytes = lambda: uuid.uuid4().bytes
import zmq.green as zmq

from zeronimo.application import NULL_RPC_SPEC, rpc_spec_table
from zeronimo.exceptions import (
    EmissionError, MalformedMessage, Rejected, TaskClosed, Undelivered,
    WorkerNotFound)
from zeronimo.helpers import (
    class_name, eintr_retry_zmq as safe, FALSE_RETURNER, repr_socket)
from zeronimo.messaging import (
    ACCEPT, ACK, BREAK, Call, PACK, parse_call, RAISE, recv, REJECT, Reply,
    RETURN, send, UNPACK, YIELD)
from zeronimo.results import RemoteException, RemoteResult

__all__ = ['Worker', 'Customer', 'Fanout', 'Collector']

# Compatible zmq constants:
try:
    ZMQ_XPUB = zmq.XPUB
    ZMQ_XSUB = zmq.XSUB
except AttributeError:
    ZMQ_XPUB = -1
    ZMQ_XSUB = -1
try:
    ZMQ_STREAM = zmq.STREAM
except AttributeError:
    ZMQ_STREAM = -1

# Default timeouts:
CUSTOMER_TIMEOUT = 5
FANOUT_TIMEOUT = 0.1

# Used for a value for `reply_to`:
NO_REPLY = b'\x00'
DUPLEX = b'\x01'

ENCODING = 'utf-8'

class Background(object):
    """A background object spawns only one greenlet at a time.  The greenlet
    will call its :meth:`__call__`.
    """

    #: The greenlet class to be spawned.
    greenlet_class = Greenlet

    #: The current running greenlet.
    greenlet = None

    def __call__(self):
        # should be implemented by subclass.
        raise NotImplementedError('%s has no __call__ implementation'
                                  '' % class_name(self))

    def run(self):
        try:
            self()
        except GreenletExit:
            pass
        finally:
            del self.greenlet

    def start(self, silent=False):
        if self.is_running():
            if silent:
                return
            raise RuntimeError('%s already running' % class_name(self))
        self.greenlet = self.greenlet_class.spawn(self.run)
        self.greenlet.join(0)
        return self.greenlet

    def stop(self, block=True, silent=False):
        if not self.is_running():
            if silent:
                return
            raise RuntimeError('%s not running' % class_name(self))
        self.greenlet.kill(block=block)

    def wait(self, timeout=None):
        if not self.is_running():
            raise RuntimeError('%s not running' % class_name(self))
        self.greenlet.join(timeout)

    def is_running(self):
        return self.greenlet is not None

    def running(self):
        warn(DeprecationWarning('use is_running() instead'))
        return self.is_running()

    def close(self, block=True):
        self.stop(block, silent=True)

def default_exception_handler(worker, exc_info):
    """The default exception handler for :class:`Worker`.  It just raises
    the given ``exc_info``.
    """
    reraise(*exc_info)

def default_malformed_message_handler(worker, exc_info, message_parts):
    """The default malformed message handler for :class:`Worker`.  It warns
    as a :exc:`MalformedMessage`.
    """
    exc_type, exc, tb = exc_info
    exc_strs = traceback.format_exception_only(exc_type, exc)
    exc_str = exc_strs[0].strip()
    if len(exc_strs) > 1:
        exc_str += '...'
    warn('<%s> occurred by %r' % (exc_str, message_parts), MalformedMessage)

class Worker(Background):
    """A worker runs an RPC service of an object through ZeroMQ sockets:

    ::

       import os
       worker = Worker(os, [sock1, sock2], sock3, info='doctor')
       worker.run()

    :param app: an application to be shared by an RPC service.
    :param sockets: ZeroMQ sockets to receive RPC calls.
    :param reply_socket: a ZeroMQ socket to send RPC replies.
    :param info: (optional) a worker will send this value to callers at
                 accepting an call.  It might be the identity of the worker to
                 let the callers know which worker accepted.
    """

    sockets = None
    reply_socket = None
    info = None
    greenlet_group = None
    exception_handler = None
    malformed_message_handler = None
    pack = None
    unpack = None

    def __init__(self, app, sockets, reply_socket=None,
                 info=None, greenlet_group=None,
                 exception_handler=default_exception_handler,
                 malformed_message_handler=default_malformed_message_handler,
                 reject_if=FALSE_RETURNER, require_rpc_specs=False,
                 pack=PACK, unpack=UNPACK):
        super(Worker, self).__init__()
        self.app = app
        self.sockets = sockets
        self.reply_socket = reply_socket
        self.info = info
        if greenlet_group is None:
            greenlet_group = Group()
        self.greenlet_group = greenlet_group
        self.exception_handler = exception_handler
        self.malformed_message_handler = malformed_message_handler
        self.reject_if = reject_if
        self.require_rpc_specs = require_rpc_specs
        self.pack = pack
        self.unpack = unpack

    @property
    def app(self):
        return self._app

    @app.setter
    def app(self, app):
        self._app = app
        self.rpc_spec_table = rpc_spec_table(app)

    @app.deleter
    def app(self):
        del self._app
        del self.rpc_spec_table

    @property
    def obj(self):
        warn('use app instead', DeprecationWarning)
        return self.app

    def __call__(self):
        """Runs the worker.  While running, an RPC service is online."""
        poller = zmq.Poller()
        for socket in self.sockets:
            poller.register(socket, zmq.POLLIN)
        group = self.greenlet_group
        msgs = []
        capture = msgs.extend
        def accept(socket, call, args, kwargs, topics):
            group.spawn(self.work, socket, call, args, kwargs, topics)
            group.join(0)
        def reject(socket, call, topics):
            __, call_id, reply_to, __, __ = call
            reply_socket, topics = self.replier(socket, topics, reply_to)
            self.reject(reply_socket, call_id, topics)
        def reject_if(call, topics):
            if self.reject_if(call, topics):
                return True
            try:
                __, rpc_spec = self.find_call_target(call)
            except KeyError:
                return True
            return rpc_spec.reject_if.__get__(self.app)(call, topics)
        try:
            while True:
                for socket, event in safe(poller.poll):
                    assert event & zmq.POLLIN
                    del msgs[:]
                    try:
                        header, payload, topics = recv(socket, capture=capture)
                        call = parse_call(header)
                        if group.full() or reject_if(call, topics):
                            reject(socket, call, topics)
                            continue
                        args, kwargs = self.unpack(payload)
                    except:
                        # If any exception occurs in the above block,
                        # the messages are treated as malformed.
                        handle = self.malformed_message_handler
                        if handle is not None:
                            exc_info = sys.exc_info()
                            handle(self, exc_info, msgs[:])
                        del handle
                        continue
                    # Accept the call.
                    accept(socket, call, args, kwargs, topics)
                # Release memory.
                try:
                    del header, payload, topics
                    del call
                    del args, kwargs
                except UnboundLocalError:
                    # Stop at the first error.
                    pass
        finally:
            group.kill()

    def work(self, socket, call, args, kwargs, topics=()):
        """Calls a function and send results to the collector.  It supports
        all of function actions.  A function could return, yield, raise any
        packable objects.
        """
        task_id = uuid4_bytes()
        reply_socket, topics = self.replier(socket, topics, call.reply_to)
        if reply_socket:
            channel = (call.call_id, task_id, topics)
        else:
            channel = (None, None, None)
        f, rpc_spec = self.find_call_target(call)
        if rpc_spec.reject_if.__get__(self.app)(call, topics):
            reply_socket and self.reject(reply_socket, call.call_id, topics)
            return
        reply_socket and self.accept(reply_socket, channel)
        success = False
        with self.catch_exceptions():
            try:
                val = self.call(call, args, kwargs, f, rpc_spec)
            except:
                exc_info = sys.exc_info()
                self.raise_(reply_socket, channel, exc_info)
                reraise(*exc_info)
            success = True
        if not success:
            # catch_exceptions() hides exceptions.
            return
        if isinstance(val, Iterator):
            vals = val
            with self.catch_exceptions():
                try:
                    try:
                        val = next(vals)
                    except StopIteration:
                        pass
                    else:
                        self.send_reply(reply_socket, YIELD, val, *channel)
                        for val in vals:
                            self.send_reply(reply_socket, YIELD, val, *channel)
                    self.send_reply(reply_socket, BREAK, None, *channel)
                except:
                    exc_info = sys.exc_info()
                    self.raise_(reply_socket, channel, exc_info)
                    reraise(*exc_info)
        else:
            self.send_reply(reply_socket, RETURN, val, *channel)

    def find_call_target(self, call):
        try:
            return self.rpc_spec_table[call.name]
        except KeyError:
            if self.require_rpc_specs:
                raise
            return getattr(self.app, call.name), NULL_RPC_SPEC

    def call(self, call, args, kwargs, f=None, rpc_spec=None):
        if f is None and rpc_spec is None:
            f, rpc_spec = self.find_call_target(call)
        if rpc_spec.pass_call:
            args = (call,) + args
        return f(*args, **kwargs)

    def accept(self, reply_socket, channel):
        """Sends ACCEPT reply."""
        info = self.info or b''
        self.send_raw(reply_socket, ACCEPT, info, *channel)

    def reject(self, reply_socket, call_id, topics=()):
        """Sends REJECT reply."""
        info = self.info or b''
        self.send_raw(reply_socket, REJECT, info, call_id, b'', topics)

    def raise_(self, reply_socket, channel, exc_info=None):
        """Sends RAISE reply."""
        if not reply_socket:
            return
        if exc_info is None:
            exc_info = sys.exc_info()
        exc_type, exc, tb = exc_info
        while tb.tb_next is not None:
            tb = tb.tb_next
        if issubclass(exc_type, RemoteException):
            exc_type = exc_type.exc_type
        filename, lineno = tb.tb_frame.f_code.co_filename, tb.tb_lineno
        val = (exc_type, str(exc), filename, lineno)
        try:
            state = exc.__getstate__()
        except AttributeError:
            pass
        else:
            val += (state,)
        self.send_reply(reply_socket, RAISE, val, *channel)

    @contextmanager
    def catch_exceptions(self):
        try:
            yield
        except:
            if self.exception_handler is not None:
                exc_info = sys.exc_info()
                self.exception_handler(self, exc_info)

    def replier(self, socket, topics, reply_to):
        if reply_to == NO_REPLY:
            return None, ()
        elif reply_to == DUPLEX:
            return socket, topics
        else:
            return self.reply_socket, (reply_to,)

    def send_raw(self, socket, method, payload, call_id, task_id, topics=()):
        if not socket:
            return
        # normal tuple is faster than namedtuple.
        header = [int2byte(method), call_id, task_id]
        try:
            safe(send, socket, header, payload, topics, zmq.NOBLOCK)
        except (zmq.Again, zmq.ZMQError):
            pass  # ignore.

    def send_reply(self, socket, method, value, call_id, task_id, topics=()):
        if not socket:
            return
        payload = self.pack(value)
        return self.send_raw(socket, method, payload, call_id, task_id, topics)

    def close(self):
        super(Worker, self).close()
        if self.reply_socket is not None:
            self.reply_socket.close()
        for socket in self.sockets:
            socket.close()

    def join(self, timeout=None, raise_error=False):
        return self.greenlet_group.join(timeout, raise_error)

    def __repr__(self):
        buf = StringIO()
        buf.write('<%s app=%r' % (class_name(self), self.app))
        if self.info is not None:
            buf.write(' info=%r' % self.info)
        sockets = ', '.join(repr_socket(s) for s in self.sockets)
        buf.write(' sockets=[%s]' % sockets)
        if self.reply_socket is not None:
            buf.write(' reply_socket=%s' % repr_socket(self.reply_socket))
        buf.write('>')
        return buf.getvalue()

class _Caller(object):
    """A caller sends RPC calls to workers.  But it could not receive results
    from the workers by itself.  To receive the results, it should work with
    :class:`Collector` together.
    """

    timeout = NotImplemented

    socket = None
    collector = None
    info = None
    pack = None

    def __init__(self, socket, collector=None, info=None,
                 timeout=None, pack=PACK):
        self.socket = socket
        self.collector = collector
        self.info = info
        self.pack = pack
        if timeout is not None:
            self.timeout = timeout

    def _make_header(self, name, call_id, reply_to, hints):
        header = [name.encode(ENCODING), call_id, reply_to]
        header.extend(hints)
        header.append(self.info or b'')
        return header

    def _call_nowait(self, hints, name, args, kwargs, topics=(), raw=False):
        header = self._make_header(name, b'', NO_REPLY, hints)
        payload = self._pack(args, kwargs, raw)
        try:
            safe(send, self.socket, header, payload, topics, zmq.NOBLOCK)
        except zmq.Again:
            pass  # ignore.

    def _call_wait(self, hints, name, args, kwargs, topics=(), raw=False,
                   limit=None, retry=False, max_retries=None):
        """Allocates a call id and emit."""
        col = self.collector
        if not col.is_running():
            col.start()
        call_id = uuid4_bytes()
        reply_to = (DUPLEX if self.socket is col.socket else col.topic)
        # Normal tuple is faster than namedtuple.
        header = self._make_header(name, call_id, reply_to, hints)
        payload = self._pack(args, kwargs, raw)
        # Use short names.
        def send_call():
            try:
                safe(send, self.socket, header, payload, topics, zmq.NOBLOCK)
            except zmq.Again:
                raise Undelivered('emission was not delivered')
        col.prepare(call_id, self, name, args, kwargs)
        send_call()
        return col.establish(call_id, self.timeout, limit,
                             send_call if retry else None,
                             max_retries=max_retries)

    def _pack(self, args, kwargs, raw=False):
        if raw:
            if kwargs or len(args) != 1:
                raise TypeError('too many arguments')
            return args[0]
        return self.pack((args, kwargs))

    def close(self):
        self.socket.close()

    def __repr__(self):
        buf = StringIO()
        buf.write('<%s' % class_name(self))
        if self.info:
            buf.write(' info=%r' % (self.info,))
        buf.write(' socket=%s' % repr_socket(self.socket))
        if self.collector is not None:
            buf.write(' collector=%r' % self.collector)
        if self.timeout is not None:
            buf.write(' timeout=%.3f' % self.timeout)
        buf.write('>')
        return buf.getvalue()

def split_call_args(args, start=0):
    if isinstance(args[start], string_types):
        name, args = args[start], args[start + 1:]
        hints = ()
    else:
        hints, name, args = args[start], args[start + 1], args[start + 2:]
    return hints, name, args

class Customer(_Caller):
    """A customer is a caller that sends an RPC call to one of workers at once.
    """

    timeout = CUSTOMER_TIMEOUT
    max_retries = None

    def __init__(self, *args, **kwargs):
        max_retries = kwargs.pop('max_retries', None)
        if max_retries is not None:
            self.max_retries = max_retries
        super(Customer, self).__init__(*args, **kwargs)

    def _call(self, raw, *args, **kwargs):
        hints, name, args = split_call_args(args)
        if self.collector is None:
            self._call_nowait(hints, name, args, kwargs, raw=raw)
            return
        results = self._call_wait(hints, name, args, kwargs,
                                  raw=raw, limit=1, retry=True,
                                  max_retries=self.max_retries)
        return results[0]

    def call(self, *args, **kwargs):
        return self._call(False, *args, **kwargs)

    def call_raw(self, *args, **kwargs):
        return self._call(True, *args, **kwargs)

class Fanout(_Caller):
    """A fanout is a caller that sends an RPC call to all workers subscribing
    the topic of the call at once.

    :param drop_if: (optional) a function which determines to drop RPC calls by
                    the topics it takes.  This parameter allows only as a
                    keyword-argument.

                    If you already know the list of topics which have one or
                    more subscribers, drop unnecessary RPC calls to reduce
                    serialization cost.  The XPUB socket type may can help you
                    to detect necessary topics.
    """

    timeout = FANOUT_TIMEOUT
    drop_if = None

    def __init__(self, *args, **kwargs):
        self.drop_if = kwargs.pop('drop_if', FALSE_RETURNER)
        super(Fanout, self).__init__(*args, **kwargs)

    def _emit(self, raw, *args, **kwargs):
        topic = args[0]
        if self.drop_if(topic):
            # Drop the call without emission.
            return None if self.collector is None else []
        hints, name, args = split_call_args(args, start=1)
        topics = (topic,) if topic else ()
        if self.collector is None:
            self._call_nowait(hints, name, args, kwargs, topics, raw=raw)
            return
        try:
            return self._call_wait(hints, name, args, kwargs, topics, raw=raw)
        except EmissionError:
            return []

    def emit(self, *args, **kwargs):
        return self._emit(False, *args, **kwargs)

    def emit_raw(self, *args, **kwargs):
        return self._emit(True, *args, **kwargs)

class Collector(Background):
    """A collector receives RPC results from workers."""

    socket = None
    topic = None
    trace = None
    unpack = None

    def __init__(self, socket, topic=b'', trace=None, unpack=UNPACK):
        super(Collector, self).__init__()
        self.socket = socket
        self.topic = topic
        self.trace = trace
        self.unpack = unpack
        self.results = {}
        self.result_queues = {}

    def prepare(self, call_id, caller, name, args, kwargs):
        if call_id in self.results:
            raise KeyError('call-%s already prepared' % call_id)
        self.results[call_id] = {}
        self.result_queues[call_id] = Queue()
        self.trace and self.trace(0, (call_id, caller, name, args, kwargs))

    def establish(self, call_id, timeout, limit=None,
                  retry=None, max_retries=None):
        """Waits for the call is accepted by workers and starts to collect the
        results.
        """
        rejected = 0
        retried = 0
        results = []
        result_queue = self.result_queues[call_id]
        try:
            with Timeout(timeout, False):
                while True:
                    result = result_queue.get()
                    if result is None:
                        rejected += 1
                        if retry is not None:
                            if retried == max_retries:
                                break
                            retry()
                            retried += 1
                        continue
                    results.append(result)
                    if len(results) == limit:
                        break
        finally:
            del result_queue
            self.remove_result_queue(call_id)
        if not results:
            if rejected:
                raise Rejected('%d workers rejected' % rejected
                               if rejected != 1 else
                               'A worker rejected')
            else:
                raise WorkerNotFound('failed to find worker')
        return results

    def __call__(self):
        while True:
            try:
                header, payload, __ = recv(self.socket)
            except GreenletExit:
                break
            except zmq.ZMQError:
                exc = TaskClosed('Collector socket closed')
                for results in viewvalues(self.results):
                    for result in viewvalues(results):
                        result.set_exception(exc)
                break
            except:
                # TODO: warn MalformedMessage
                continue
            method, call_id, task_id = header
            method = ord(method)
            reply = Reply(method, call_id, task_id)
            if method & ACK:
                value = payload or None
            else:
                value = self.unpack(payload)
            self.trace and self.trace(method, (call_id, task_id, value))
            del header, payload, method, call_id, task_id
            try:
                self.dispatch_reply(reply, value)
            except KeyError:
                # TODO: warning
                continue
            finally:
                del reply, value

    def dispatch_reply(self, reply, value):
        """Dispatches the reply to the proper queue."""
        method = reply.method
        call_id = reply.call_id
        task_id = reply.task_id
        if method & ACK:
            try:
                result_queue = self.result_queues[call_id]
            except KeyError:
                raise KeyError('already established or unprepared call')
            if method == ACCEPT:
                worker_info = value
                result = RemoteResult(self, call_id, task_id, worker_info)
                self.results[call_id][task_id] = result
                result_queue.put_nowait(result)
            elif method == REJECT:
                result_queue.put_nowait(None)
        else:
            result = self.results[call_id][task_id]
            result.set_reply(reply.method, value)

    def remove_result(self, result):
        call_id = result.call_id
        task_id = result.task_id
        assert self.results[call_id][task_id] is result
        del self.results[call_id][task_id]
        if call_id not in self.result_queues and not self.results[call_id]:
            del self.results[call_id]

    def remove_result_queue(self, call_id):
        del self.result_queues[call_id]
        if not self.results[call_id]:
            del self.results[call_id]

    def close(self):
        super(Collector, self).close()
        self.socket.close()

    def __repr__(self):
        buf = StringIO()
        buf.write('<%s socket=%s' % (class_name(self),
                                     repr_socket(self.socket)))
        if self.topic is not None:
            buf.write(' topic=%r' % self.topic)
        buf.write('>')
        return buf.getvalue()


# encoding: utf-8
# Copyright (C) 2017 John Törnblom
#
# This file is part of pyxtuml.
#
# pyxtuml is free software: you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation, either
# version 3 of the License, or (at your option) any later version.
#
# pyxtuml is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with pyxtuml. If not, see <http://www.gnu.org/licenses/>.

'''
Loading support for xtUML models (based on sql).
'''

import uuid
import logging
import os
import re

from ply import lex
from ply import yacc

import xtuml

logger = logging.getLogger(__name__)

def guess_type_name(value):
    '''
    Guess the type name of a serialized value.
    '''
    value = str(value)

    if value.upper() in ['TRUE', 'FALSE']:
        return 'BOOLEAN'

    elif re.match(r'(-)?(\d+)(\.\d+)', value):
        return 'REAL'

    elif re.match(r'(-)?(\d+)', value):
        return 'INTEGER'

    elif re.match(r'\'((\'\')|[^\'])*\'', value):
        return 'STRING'

    elif re.match(r'\"([^\\\n]|(\\.))*?\"', value):
        return 'UNIQUE_ID'

def deserialize_value(ty, value):
    '''
    Deserialize a value of some type
    '''
    uty = ty.upper()

    if uty == 'BOOLEAN':
        if value.isdigit():
            return bool(int(value))
        elif value.upper() == 'FALSE':
            return False
        elif value.upper() == 'TRUE':
            return True
        else:
            return None

    elif uty == 'INTEGER':
        if '"' in value:
            return uuid.UUID(value[1:-1]).int
        else:
            return int(value)

    elif uty == 'REAL':
        return float(value)

    elif uty == 'STRING':
        return value[1:-1].replace("''", "'")

    elif uty == 'UNIQUE_ID':
        if '"' in value:
            return uuid.UUID(value[1:-1]).int
        else:
            return int(value)

class ParsingException(Exception):
    '''
    An exception that may be thrown while loading (and parsing) a metamodel.
    '''
    pass

class Stmt(object):
    offset = None
    lineno = None
    filename = None

class CreateInstanceStmt(Stmt):

    def __init__(self, kind, values, names):
        self.kind = kind
        self.values = values
        self.names = names

class CreateClassStmt(Stmt):

    def __init__(self, kind, attributes):
        self.kind = kind
        self.attributes = attributes

class CreateAssociationStmt(Stmt):
    def __init__(self, rel_id, source_kind, source_cardinality, source_keys,
                 source_phrase, target_kind, target_cardinality, target_keys,
                target_phrase):
        self.rel_id = rel_id
        self.source_kind = source_kind
        self.target_kind = target_kind
        self.source_keys = source_keys
        self.target_keys = target_keys
        self.source_cardinality = source_cardinality
        self.target_cardinality = target_cardinality
        self.source_phrase = source_phrase
        self.target_phrase = target_phrase

class CreateUniqueStmt(Stmt):

    def __init__(self, kind, name, attributes):
        self.kind = kind
        self.name = name
        self.attributes = attributes

class ModelLoader(object):
    '''
    Class for loading metamodels previously persisted to disk.

    Data may be provided in any order, e.g. instances followed by associations,
    followed by class definitions. One single loader may be used to build
    several *xtuml.MetaModel* objects, and additional data may be provided at
    any time.

    **Note:** Additional data will not affect previosly built metamodels.

    Usage example:

    >>> l = xtuml.ModelLoader()
    >>> l.filename_input('data.sql')
    >>> l.filename_input('schema.sql')
    >>> m1 = l.build_metamodel()
    >>> l.filename_input('additional_data.sql')
    >>> m2 = l.build_metamodel()
    '''
    reserved = (
        'CREATE',
        'FALSE',
        'FROM',
        'INDEX',
        'INSERT',
        'INTO',
        'ON',
        'PHRASE',
        'REF_ID',
        'ROP',
        'TABLE',
        'TO',
        'TRUE',
        'UNIQUE',
        'VALUES'
    )

    tokens = reserved + (
        'CARDINALITY',
        'COMMA',
        'FRACTION',
        'GUID',
        'ID',
        'LPAREN',
        'MINUS',
        'NUMBER',
        'RPAREN',
        'RELID',
        'SEMICOLON',
        'STRING'
    )

    # A string containing ignored characters (spaces and tabs).
    t_ignore = ' \t\r\x0c'

    parser = None
    lexer = None
    statements = None

    def __init__(self):
        self.statements = list()
        self.parser = yacc.yacc(debuglog=logger,
                                errorlog=logger,
                                optimize=1,
                                module=self,
                                outputdir=os.path.dirname(__file__),
                                tabmodule='xtuml.__xtuml_parsetab')

    def input(self, data, name='<string>'):
        '''
        Parse *data* directly from a string. The *name* is used when reporting
        positional information if the parser encounter syntax errors.
        '''
        lexer = lex.lex(debuglog=logger,
                        errorlog=logger,
                        optimize=1,
                        module=self,
                        outputdir=os.path.dirname(__file__),
                        lextab="xtuml.__xtuml_lextab")
        lexer.filename = name
        logger.debug('parsing %s' % name)
        s = self.parser.parse(lexer=lexer, input=data, tracking=1)
        self.statements.extend(s)

    def filename_input(self, filename):
        '''
        Open and read from a *filename* on disk, and parse its content.
        '''
        with open(filename, 'r') as f:
            return self.file_input(f)

    def file_input(self, file_object):
        '''
        Read and parse data from a *file object*, i.e. the type of object
        returned by the builtin python function *open()*.
        '''
        return self.input(file_object.read(), name=file_object.name)

    def populate_classes(self, metamodel):
        '''
        Populate a *metamodel* with classes previously encountered from input.
        '''
        for stmt in self.statements:
            if isinstance(stmt, CreateClassStmt):
                metamodel.define_class(stmt.kind, stmt.attributes)

    def populate_associations(self, metamodel):
        '''
        Populate a *metamodel* with associations previously encountered from
        input.
        '''
        for stmt in self.statements:
            if not isinstance(stmt, CreateAssociationStmt):
                continue

            ass = metamodel.define_association(stmt.rel_id,
                                         stmt.source_kind,
                                         stmt.source_keys,
                                         'M' in stmt.source_cardinality,
                                         'C' in stmt.source_cardinality,
                                         stmt.source_phrase,
                                         stmt.target_kind,
                                         stmt.target_keys,
                                         'M' in stmt.target_cardinality,
                                         'C' in stmt.target_cardinality,
                                         stmt.target_phrase)
            ass.formalize()

    def populate_unique_identifiers(self, metamodel):
        '''
        Populate a *metamodel* with class unique identifiers previously
        encountered from input.
        '''
        for stmt in self.statements:
            if isinstance(stmt, CreateUniqueStmt):
                metamodel.define_unique_identifier(stmt.kind, stmt.name,
                                                   *stmt.attributes)

    @staticmethod
    def _populate_matching_class(metamodel, kind, names, values):
        '''
        Populate a *metamodel* with a class that matches the given *insert
        statement*.
        '''
        attributes = list()
        for name, value in zip(names, values):
            ty = guess_type_name(value)
            attr = (name, ty)
            attributes.append(attr)

        return metamodel.define_class(kind, attributes)

    @staticmethod
    def _populate_instance_with_positional_arguments(metamodel, stmt):
        '''
        Populate a *metamodel* with an instance previously encountered from
        input that was defined using positional arguments.
        '''
        if stmt.kind.upper() not in metamodel.metaclasses:
            names = ['_%s' % idx for idx in range(len(stmt.values))]
            ModelLoader._populate_matching_class(metamodel, stmt.kind,
                                                 names, stmt.values)

        metaclass = metamodel.find_metaclass(stmt.kind)
        if len(metaclass.attributes) != len(stmt.values):
            logger.warn('%s:%d:schema mismatch' % (stmt.filename, stmt.lineno))

        inst = metamodel.new(stmt.kind)
        for attr, value in zip(metaclass.attributes, stmt.values):
            name, ty = attr
            py_value = deserialize_value(ty, value)
            if py_value is None:
                raise ParsingException("%s:%d:unable to deserialize "\
                                       "%s to a %s" % (stmt.filename,
                                                       stmt.lineno,
                                                       value,
                                                       ty))

            inst.__dict__[name] = py_value

        return inst

    @staticmethod
    def _populate_instance_with_named_arguments(metamodel, stmt):
        '''
        Populate a *metamodel* with an instance previously encountered from
        input that was defined using named arguments.
        '''
        if stmt.kind.upper() not in metamodel.metaclasses:
            ModelLoader._populate_matching_class(metamodel, stmt.kind,
                                                 stmt.names, stmt.values)

        metaclass = metamodel.find_metaclass(stmt.kind)

        schema_unames = [name.upper() for name in metaclass.attribute_names]
        inst_unames = [name.upper() for name in stmt.names]

        if set(inst_unames) - set(schema_unames):
            logger.warn('%s:%d:schema mismatch' % (stmt.filename, stmt.lineno))

        inst = metamodel.new(stmt.kind)
        for name, ty in metaclass.attributes:
            uname = name.upper()
            if uname in inst_unames:
                idx = inst_unames.index(uname)
                value = deserialize_value(ty, stmt.values[idx])
                if value is None:
                    raise ParsingException("%s:%d:unable to deserialize "\
                                           "%s to a %s" % (stmt.filename,
                                                           stmt.lineno,
                                                           value,
                                                           ty))
            else:
                value = None

            inst.__dict__[name] = value

        return inst

    def populate_instances(self, metamodel):
        '''
        Populate a *metamodel* with instances previously encountered from
        input.
        '''
        for stmt in self.statements:
            if not isinstance(stmt, CreateInstanceStmt):
                continue

            if stmt.names:
                fn = self._populate_instance_with_named_arguments
            else:
                fn = self._populate_instance_with_positional_arguments

            fn(metamodel, stmt)

    def populate_connections(self, metamodel):
        '''
        Populate links in a *metamodel* with connections between them.
        '''
        storage = dict()
        for ass in metamodel.associations:
            source_class = ass.source_link.to_metaclass
            target_class = ass.target_link.to_metaclass

            if target_class not in storage:
                storage[target_class] = dict()

            link_key = frozenset(ass.source_link.key_map.values())
            if link_key not in storage[target_class]:
                storage[target_class][link_key] = dict()
                for other_inst in target_class.storage:
                    inst_key = ass.source_link.compute_index_key(other_inst)
                    if inst_key is None:
                        continue

                    if inst_key not in storage[target_class][link_key]:
                        storage[target_class][link_key][inst_key] = xtuml.OrderedSet()

                    storage[target_class][link_key][inst_key].add(other_inst)

            for inst in source_class.storage:
                inst_key = ass.source_link.compute_lookup_key(inst)
                if inst_key is None:
                    continue

                if inst_key not in storage[target_class][link_key]:
                    continue

                for other_inst in storage[target_class][link_key][inst_key]:
                    ass.source_link.connect(other_inst, inst, check=False)
                    ass.target_link.connect(inst, other_inst, check=False)

        for inst in metamodel.instances:
            metaclass = xtuml.get_metaclass(inst)
            for attr in metaclass.referential_attributes:
                if attr in inst.__dict__:
                    delattr(inst, attr)

    def populate(self, metamodel):
        '''
        Populate a *metamodel* with entities previously encountered from input.
        '''
        self.populate_classes(metamodel)
        self.populate_unique_identifiers(metamodel)
        self.populate_associations(metamodel)
        self.populate_instances(metamodel)
        self.populate_connections(metamodel)

    def build_metamodel(self, id_generator=None):
        '''
        Build and return a *xtuml.MetaModel* containing previously loaded input.
        '''
        m = xtuml.MetaModel(id_generator)

        self.populate(m)

        return m

    def t_comment(self, t):
        r'\-\-([^\n]*\n?)'
        t.lexer.lineno += (t.value.count("\n"))
        t.endlexpos = t.lexpos + len(t.value)

    def t_COMMA(self, t):
        r','
        t.endlexpos = t.lexpos + len(t.value)
        return t

    def t_FRACTION(self, t):
        r'(\d+)(\.\d+)'
        t.endlexpos = t.lexpos + len(t.value)
        return t

    def t_RELID(self, t):
        r'R[0-9]+'
        t.endlexpos = t.lexpos + len(t.value)
        return t

    def t_CARDINALITY(self, t):
        r'(1C)'
        t.endlexpos = t.lexpos + len(t.value)
        return t

    def t_ID(self, t):
        r'[A-Za-z_][\w_]*'
        vup = t.value.upper()
        if vup in self.reserved:
            t.type = vup
        t.endlexpos = t.lexpos + len(t.value)
        return t

    def t_LPAREN(self, t):
        r'\('
        t.endlexpos = t.lexpos + len(t.value)
        return t

    def t_MINUS(self, t):
        r'-'
        t.endlexpos = t.lexpos + len(t.value)
        return t

    def t_NUMBER(self, t):
        r'[0-9]+'
        t.endlexpos = t.lexpos + len(t.value)
        return t

    def t_RPAREN(self, t):
        r'\)'
        t.endlexpos = t.lexpos + len(t.value)
        return t

    def t_SEMICOLON(self, t):
        r';'
        t.endlexpos = t.lexpos + len(t.value)
        return t

    def t_STRING(self, t):
        r'\'((\'\')|[^\'])*\''
        t.lexer.lineno += (t.value.count("\n"))
        t.endlexpos = t.lexpos + len(t.value)
        return t

    def t_GUID(self, t):
        r'\"([^\\\n]|(\\.))*?\"'
        t.endlexpos = t.lexpos + len(t.value)
        return t

    def t_newline(self, t):
        r'\n+'
        t.lexer.lineno += len(t.value)
        t.endlexpos = t.lexpos + len(t.value)

    def t_error(self, t):
        raise ParsingException("illegal character '%s' at %s:%d" % (t.value[0],
                               t.lexer.filename, t.lineno))

    def p_empty_translation_unit(self, p):
        '''translation_unit :'''
        p[0] = list()

    def p_translation_unit(self, p):
        '''translation_unit : statement_sequence'''
        p[0] = p[1]

    def p_statement_sequence(self, p):
        '''statement_sequence : statement_sequence statement'''
        p[0] = p[1]
        p[0].append(p[2])

    def p_statement_sequence_endpoint(self, p):
        '''statement_sequence : statement'''
        p[0] = [p[1]]

    def p_statement(self, p):
        '''
        statement : create_table_statement SEMICOLON
                  | insert_into_statement SEMICOLON
                  | create_rop_statement SEMICOLON
                  | create_index_statement SEMICOLON
        '''
        p[0] = p[1]
        p[0].offset = p.lexpos(1)
        p[0].lineno = p.lineno(1)
        p[0].filename = p.lexer.filename

    def p_create_table_statement(self, p):
        '''create_table_statement : CREATE TABLE identifier LPAREN attribute_sequence RPAREN'''
        p[0] = CreateClassStmt(p[3], p[5])

    def p_empty_attribute_sequence(self, p):
        '''attribute_sequence : '''
        p[0] = []

    def p_attribute_sequence_endpoint(self, p):
        '''attribute_sequence : attribute'''
        p[0] = [p[1]]

    def p_attribute_sequence(self, p):
        '''attribute_sequence : attribute_sequence COMMA attribute'''
        p[0] = p[1]
        p[0].append(p[3])

    def p_attribute(self, p):
        '''attribute : identifier identifier'''
        p[0] = (p[1], p[2])

    def p_ordered_insert_into_statement(self, p):
        '''
        insert_into_statement : INSERT INTO identifier VALUES LPAREN value_sequence RPAREN
        '''
        p[0] = CreateInstanceStmt(p[3], p[6], None)

    def p_named_insert_into_statement(self, p):
        '''
        insert_into_statement : INSERT INTO identifier LPAREN identifier_sequence RPAREN VALUES LPAREN value_sequence RPAREN
        '''
        p[0] = CreateInstanceStmt(p[3], p[9], p[5])

    def p_empty_value_sequence(self, p):
        '''value_sequence : '''
        p[0] = []

    def p_value_sequence(self, p):
        '''value_sequence : value_sequence COMMA value'''
        p[0] = p[1]
        p[0].append(p[3])

    def p_value_sequence_endpoint(self, p):
        '''value_sequence : value'''
        p[0] = [p[1]]

    def p_value(self, p):
        '''
        value : FRACTION
              | NUMBER
              | STRING
              | GUID
              | TRUE
              | FALSE
        '''
        p[0] = p[1]

    def p_negative_value(self, p):
        '''
        value : MINUS FRACTION
              | MINUS NUMBER
        '''
        p[0] = p[1] + p[2]

    def p_create_rop_statement(self, p):
        '''create_rop_statement : CREATE ROP REF_ID RELID FROM association_end TO association_end'''
        args = [p[4]]
        args.extend(p[6])
        args.extend(p[8])
        p[0] = CreateAssociationStmt(*args)

    def p_association_end(self, p):
        '''association_end : cardinality identifier LPAREN identifier_sequence RPAREN'''
        p[0] = (p[2], p[1], p[4], '')

    def p_phrased_association_end(self, p):
        '''association_end : cardinality identifier LPAREN identifier_sequence RPAREN PHRASE STRING'''
        p[0] = (p[2], p[1], p[4], p[7][1:-1])

    def p_cardinality_1(self, p):
        '''cardinality : NUMBER'''
        if p[1] != '1':
            raise ParsingException("illegal cardinality (%s) at %s:%d" % (p[1],
                                   p.lexer.filename, p.lineno(1)))
        p[0] = p[1]

    def p_cardinality_many(self, p):
        '''cardinality : ID'''
        if p[1] not in ['M', 'MC']:
            raise ParsingException("illegal cardinality (%s) at %s:%d" % (p[1],
                                   p.lexer.filename, p.lineno(1)))
        p[0] = p[1]

    def p_cardinality_01(self, p):
        '''cardinality : CARDINALITY'''
        p[0] = p[1]

    def p_empty_identifier_sequence(self, p):
        '''identifier_sequence : '''
        p[0] = []

    def p_identifier_sequence(self, p):
        '''identifier_sequence : identifier_sequence COMMA identifier'''
        p[0] = p[1]
        p[0].append(p[3])

    def p_identifier_sequence_endpoint(self, p):
        '''identifier_sequence : identifier'''
        p[0] = [p[1]]

    def p_identifier(self, p):
        '''
        identifier : ID
                   | CREATE
                   | INSERT
                   | INTO
                   | VALUES
                   | TABLE
                   | ROP
                   | REF_ID
                   | FROM
                   | TO
                   | PHRASE
                   | UNIQUE
                   | INDEX
                   | ON
                   | TRUE
                   | FALSE
        '''
        p[0] = p[1]

    def p_create_index_statement(self, p):
        '''
        create_index_statement : CREATE UNIQUE INDEX identifier ON identifier LPAREN identifier_sequence RPAREN
        '''
        p[0] = CreateUniqueStmt(p[6], p[4], p[8])

    def p_error(self, p):
        if p:
            raise ParsingException("illegal token %s (%s) at %s:%d" % (p.type,
                                   p.value, p.lexer.filename, p.lineno))
        else:
            raise ParsingException("unknown error")

def load_metamodel(resource):
    '''
    Load and return a metamodel from a *resource*. The *resource* may be either
    a filename, or a list of filenames.

    Usage example:

    >>> metamodel = xtuml.load_metamodel(['schema.sql', 'data.sql'])
    '''
    if isinstance(resource, str):
        resource = [resource]

    loader = ModelLoader()
    for filename in resource:
        loader.filename_input(filename)

    return loader.build_metamodel()


# coding=utf-8
"""
3 ways to find secrets I know of:

- grep for THING = "xyzzy" and the like. dodgy and pylint does this.
- search for high entropy strings. detect-secrets does this.
- look up known secrets, e.g. values from an known .ini file, or values
  currently set in the environment

"""
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
import tabulate
import os
from typing import List, Tuple, Optional, Set, Dict

from colorama import init, Fore, Back, Style

_ = List

try:
    import configparser
except ImportError:
    # Python 2.x fallback
    import ConfigParser as configparser

import sys

if sys.version_info.major == 3:
    unicode = str

init(autoreset=True)  # cross plat now

class Searcher(object):
    def __init__(self, source, files=None):  # type: (str,Optional[str]) -> None
        self.source = source

        self.files = ["~/.aws/credentials"]
        if files:
            for file in files.split(","):
                self.files.append(file)

        # TODO: merge in more .ini files

        self.false_positives = ["localhost", "admin", "0.0.0.0"]

        self.skip_files = []  # type: List[str]
        # TODO: merge in an ignore file

        self.secrets = []  # type: List[str]

        self.found = {}  # type: Dict[str, List[Tuple[str,str]]]

    def append_known_secrets(self):  # type: () -> None
        """
        Read key-value pair files with secrets. For example, .conf and .ini files.
        :return:
        """
        for file_name in self.files:
            if "~" in file_name:
                file_name = os.path.expanduser(file_name)
            if not os.path.isfile(file_name):
                print(
                    "Don't have "
                    + Back.BLACK
                    + Fore.YELLOW
                    + file_name
                    + ", won't use."
                )
                continue
            with open(os.path.expanduser(file_name), "r") as file:
                for line in file:
                    if line and "=" in line:
                        possible = line.split("=")[1].strip(" \"'\n")
                        if len(possible) > 4 and possible not in self.false_positives:
                            self.secrets.append(possible)

    def search_known_secrets(self):  # type: () -> None
        """
        Search a path for known secrets, outputing text and file when found
        :return:
        """
        count = 0
        here = os.path.abspath(self.source)
        # python 3 only!
        # for file in glob.glob(here + "/" + "**/*.*", recursive=True):

        # py 2
        matches = []
        for root, dirnames, filenames in os.walk(here + "/"):
            for filename in filenames:
                matches.append(os.path.join(root, filename))

        for file in matches:
            if os.path.isdir(file):
                continue
            with open(file) as f:
                try:
                    contents = f.read()
                except UnicodeDecodeError:
                    continue
                except Exception as e:
                    print(e)
                    print(file)
                    raise
            for secret in self.secrets:
                if secret in contents:
                    for line in contents.split("\n"):
                        if secret in line:
                            self.found.setdefault(file, []).append(
                                (
                                    secret,
                                    line.replace(
                                        secret,
                                        Fore.RED
                                        + Back.YELLOW
                                        + secret
                                        + Style.RESET_ALL,
                                    ),
                                )
                            )
                            count += 1

    def report(self):  # type: ()-> None
        current_directory = os.getcwd()
        count = len(self.found)
        if count > 0:
            print(Fore.RED + "Found {0} secrets. Failing this run.".format(count))

            data = [
                (
                    key.replace(current_directory, ""),
                    tabulate.tabulate(tabular_data=value, tablefmt="plain"),
                )
                for key, value in self.found.items()
            ]
            result = tabulate.tabulate(
                tabular_data=data,
                headers=("File", "Secret Found - Secret Text"),
                tablefmt="grid",
            )
            print(result)

            exit(-1)
        else:
            print(
                "No known secrets found. Consider trying out detect-secrets and git-secrets, too."
            )

    def go(self):  # type: () -> None
        """
        Entry point method
        :return:
        """
        self.append_known_secrets()
        self.search_known_secrets()
        self.report()

if __name__ == "__main__":
    searcher = Searcher("")
    searcher.go()


'''
    sowrapper: Utility functions to locate and load shared libraries

    Copyright (C) 2016 Sundar Nagarajan

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.

    For details of the GNU General Pulic License version 3, see the
    LICENSE.txt file that accompanied this program

    Recommended usage:

    Should only need to use get_lib_ffi_shared() or get_lib_ffi_resource()

    Use get_lib_ffi_shared to load a system-wide shared library with a known
    library filename and / or path

    Use get_lib_ffi_resource to load a module-specific shared library where
    library filename _MAY_ be mangled as per PEP3149 and path _MAY_ need to
    be looked up using pkg_resources. Internally, get_lib_ffi_resource()
    calls get_lib_ffi_shared()

    Both return a tuple: (ffi, lib):
        ffi-->FFIExt - should behave like cffi.FFI with some additional
                utility methods
        lib-->SharedLibWrapper instance - use methods on this object to
            call methods in the shared library
'''
import sys
import os
from pkg_resources import resource_filename
import sysconfig
from .py2to3 import PYPY
from .ffi import FFIExt

def get_lib_ffi_resource(module_name, libpath, c_hdr):
    '''
    module_name-->str: module name to retrieve resource
    libpath-->str: shared library filename with optional path
    c_hdr-->str: C-style header definitions for functions to wrap
    Returns-->(ffi, lib)

    Use this method when you are loading a package-specific shared library
    If you want to load a system-wide shared library, use get_lib_ffi_shared
    instead
    '''
    lib = SharedLibWrapper(libpath, c_hdr, module_name=module_name)
    ffi = lib.ffi
    return (ffi, lib)

def get_lib_ffi_shared(libpath, c_hdr):
    '''
    libpath-->str: shared library filename with optional path
    c_hdr-->str: C-style header definitions for functions to wrap
    Returns-->(ffi, lib)
    '''
    lib = SharedLibWrapper(libpath, c_hdr)
    ffi = lib.ffi
    return (ffi, lib)

class SharedLibWrapper(object):
    def __init__(self, libpath, c_hdr, module_name=None):
        '''
        libpath-->str: library name; can also be full path
        c_hdr-->str: C-style header definitions for functions to wrap
        ffi-->FFIExt or cffi.FFI
        '''
        self._libpath = libpath
        self._c_hdr = c_hdr
        self._module_name = module_name
        self.ffi = FFIExt()
        self.lib = None

        self.ffi.cdef(self._c_hdr)
        self._libloaded = False

    def load(self):
        '''
        Returns-->boolean: whether library could be loaded
        When this is called, the library would have been loaded if possible
        If this is not called, the first invocation of a method in lib
        CAN raise OSError if shared library cannot be loaded
        '''
        try:
            self.__openlib()
            return self.lib is not None
        except OSError:
            return False

    def __openlib(self):
        '''
        Actual (lazy) dlopen() only when an attribute is accessed
        '''
        if self.__getattribute__('_libloaded'):
            return
        libpath_list = self.__get_libres()
        for p in libpath_list:
            try:
                libres = resource_filename(self._module_name, p)
                self.lib = self.ffi.dlopen(libres)
                return
            except:
                continue
        # Try self._libpath if nothing in libpath_list worked - will work
        # only if self._module_name is set
        try:
            libres = resource_filename(self._module_name, self._libpath)
            self.lib = self.ffi.dlopen(libres)
        except:
            # If self._module_name is in sys.modules, try self._libpath
            # in the same dir as sys.modules[self._module_name].__file__
            # This is allows get_lib_ffi_shared to work in REPL
            try:
                # We set _libloaded to indicate all options have been tried
                self._libloaded = True

                libdir = ''
                if self._module_name is not None:
                    mod = sys.modules.get(self._module_name, None)
                    if mod is not None:
                        libdir = os.path.dirname(mod.__file__) or os.getcwd()

                libres = os.path.join(libdir, self._libpath)
                self.lib = self.ffi.dlopen(libres)
            except:
                return None

    def __getattr__(self, name):
        if not self.__getattribute__('_libloaded'):
            self.__openlib()
        if self.__getattribute__('lib') is not None:
            try:
                return getattr(self.lib, name)
            except AttributeError:
                return self.__getattribute__(name)
        else:
            return self.__getattribute__(name)

    def __get_libres(self):
        '''
        Computes libpath based on whether module_name is set or not
        Returns-->list of str lib paths to try

        PEP3140: ABI version tagged .so files:
            https://www.python.org/dev/peps/pep-3149/

        There's still one unexplained bit: pypy adds '-' + sys._multiarch()
        at the end (looks like 'x86_64-linux-gnu'), but neither py2 or py3 do

        Additionally, in older releases of pypy (e.g. build f3ad1e1e1d62
        Aug-28-2015), sysconfig.get_config_var('SOABI') returns '' but
        shared library still has '.pypy-26' in the name!

        So for pypy we try this this variant anyway!

        _I_ think Py2 and Py3 _MAY_ start adding sys._multiarch at some time

        So, we generate three forms:
            1. With sys._multiarch
            2. Without sys._multiarch
            3. libpath as-is - always tried by self.__openlib anyway
        For different versions we try in different order (for efficiency):
            Python2                 Python3                 Pypy

            2 --> 1 --> 3           2 --> 1 --> 3           1 --> 2 --> 3
        '''
        if self._module_name is None:
            return []
        ending = '.so'
        base = self._libpath.rsplit(ending, 1)[0]
        abi = sysconfig.get_config_var('SOABI')
        if abi is not None:
            abi = '.' + abi
        else:
            abi = ''

        multi_arch = sysconfig.get_config_var('MULTIARCH')
        if multi_arch is None:
            multi_arch = ''
        else:
            multi_arch = '-' + multi_arch

        if PYPY:
            n1 = base + abi + multi_arch + ending
            n2 = base + abi + ending
        else:
            n1 = base + abi + ending
            n2 = base + abi + multi_arch + ending
        if PYPY:
            n3 = base + '.pypy-26' + ending
            return [n1, n2, n3]
        else:
            return [n1, n2]

    def get_extension(self):
        return [self.ffi.verifier.get_extension()]


# coding=utf-8
"""
Find Known Secrets. Are you about to check in passwords and keys into your public source control and lose your job?

Usage:
  find_known_secrets here
  find_known_secrets [--source=DIR] --secrets=<ini>
  find_known_secrets -h | --help
  find_known_secrets --version

Options:
  --secrets=<secrets> List of ini or ini-like files with known secrets
  --source=DIR        Source folder. e.g. src/ [default: .]
  -h --help           Show this screen.
  --version           Show version.
  --debug=<debug>     Show diagnostic info [default: False]
"""
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import logging

from docopt import docopt

from find_known_secrets.searcher import Searcher
from find_known_secrets.__version__ import __version__

logger = logging.getLogger(__name__)

def go():  # type: () -> None
    """
    Default scenario
    """
    searcher = Searcher(source="")
    searcher.go()

def process_docopts():  # type: ()->None
    """
    Take care of command line options
    """

    arguments = docopt(__doc__, version="Find Known Secrets {0}".format(__version__))

    logger.debug(arguments)
    # print(arguments)
    if arguments["here"]:
        # all default
        go()
    else:
        # user config
        files = arguments["--secrets"]

        searcher = Searcher(source=arguments["--source"], files=files)
        searcher.go()

if __name__ == "__main__":
    process_docopts()


from typing import Optional

from aiobreaker import CircuitBreakerError
from aiohttp import web

from hyperion.fetch import ApiError
from hyperion.fetch.wikipedia import fetch_nearby
from hyperion.models import Postcode, CachingError
from hyperion.models.util import get_postcode, get_postcode_random
from .util import str_json_response

async def api_postcode(request):
    """
    Gets data from a postcode.
    :param request: The aiohttp request.
    """
    postcode: Optional[str] = request.match_info.get('postcode', None)

    try:
        coroutine = get_postcode_random() if postcode == "random" else get_postcode(postcode)
        postcode: Optional[Postcode] = await coroutine
    except CachingError as e:
        return web.HTTPInternalServerError(body=e.status)
    except CircuitBreakerError as e:
        pass
    else:
        if postcode is not None:
            return str_json_response(postcode.serialize())
        else:
            return web.HTTPNotFound(body="Invalid Postcode")

async def api_nearby(request):
    """
    Gets wikipedia articles near a given postcode.
    :param request: The aiohttp request.
    """
    postcode: Optional[str] = request.match_info.get('postcode', None)

    try:
        limit = int(request.match_info.get('limit', 10))
    except ValueError:
        raise web.HTTPBadRequest(text="Invalid Limit")

    try:
        coroutine = get_postcode_random() if postcode == "random" else get_postcode(postcode)
        postcode: Optional[Postcode] = await coroutine
    except CachingError as e:
        raise web.HTTPInternalServerError(body=e.status)

    if postcode is None:
        raise web.HTTPNotFound(text="Invalid Postcode")

    try:
        nearby_items = await fetch_nearby(postcode.lat, postcode.long, limit)
    except ApiError:
        return web.HTTPInternalServerError(text=f"No nearby locations cached, and can't be retrieved.")

    if nearby_items is None:
        raise web.HTTPNotFound(text="No Results")
    else:
        return str_json_response(nearby_items)


import collections
import formencode.htmlfill
import jinja2
import jinja2.ext
from jinja2 import nodes

__all__ = ['FormFillExtension']

class FormFillExtension(jinja2.ext.Extension):
    """Jinja2 extension for filling HTML forms via :mod:`formencode.htmlfill`.

    For example, this code:

    .. code-block:: html+jinja

       {% formfill {'username': 'robert', 'email': 'robert153@usrobots.com'}
              with {'username': 'This name is invalid'} %}
       <form action="/register" method="POST">
           <input type="text" name="username" />
           <form:error name="username">
           <input type="password" name="password" />
           <input type="email" name="email" />
       </form>
       {% endformfill %}

    will be rendered like below:

    .. code-block:: html

       <form action="/register" method="POST">
           <input type="text" name="username" class="error" value="robert" />
           This name is invalid
           <input type="password" name="password" value="" />
           <input type="email" name="email" value="robert153@usrobots.com" />
       </form>

    **Syntax:**

    .. code-block:: jinja

       {% formfill <defaults> [with <errors>] %}
           body
       {% endformfill %}

    :param defaults: a :term:`mapping` that contains default values of the
                     input field (including ``select`` and ``textarea``)
                     surrounded in the template tag.
                     Keys contain a value of ``name`` attribute of the input
                     field, and values contain its default value.
    :param errors: a :term:`mapping` that contains error messages of the
                   input fields. this value will also effect ``class``
                   attribute of the input field.
    :returns: rendered forms

    This extension provides the additional variables in the Jinja2 environment:

    .. attribute:: jinja2.Environment.formfill_config

       The default rendering configuration of the ``formfill`` tag.
       This property accepts the same arguments of
       :func:`formencode.htmlfill.render`, except ``form``, ``defaults``,
       ``errors`` and ``error_formatters``.

    .. attribute:: jinja2.Environment.formfill_error_formatters

       The :term:`mapping` of error formatters and its name.
       Formatters are functions or callable objects that take the error text
       as a single argument, and returns a formatted text as a string.

       .. seealso:: http://www.formencode.org/en/latest/htmlfill.html#errors

    """
    tags = frozenset(['formfill'])

    def __init__(self, environment):
        super(FormFillExtension, self).__init__(environment)
        environment.extend(
            formfill_config={},
            formfill_error_formatters=dict(DEFAULT_ERROR_FORMATTERS),
        )

    def parse(self, parser):
        token = next(parser.stream)
        defaults = parser.parse_expression()
        if parser.stream.skip_if('name:with'):
            errors = parser.parse_expression()
        else:
            errors = nodes.Const({})
        body = parser.parse_statements(['name:endformfill'], drop_needle=True)
        return nodes.CallBlock(
            self.call_method('_formfill_support', [defaults, errors]),
            [], [], body).set_lineno(token.lineno)

    def _formfill_support(self, defaults, errors, caller):
        if isinstance(defaults, jinja2.runtime.Undefined):
            defaults = {}
        if isinstance(errors, jinja2.runtime.Undefined):
            errors = {}
        if not isinstance(defaults, collections.Mapping):
            raise TypeError("argument 'defaults' should be "
                            "collections.Mapping, not {0!r}".format(defaults))
        if not isinstance(errors, collections.Mapping):
            raise TypeError("argument 'errors' should be collections.Mapping, "
                            "not {0!r}".format(errors))
        rv = caller()
        return formencode.htmlfill.render(
            rv, defaults, errors,
            error_formatters=self.environment.formfill_error_formatters,
            **self.environment.formfill_config)

def default_formatter(error):
    """Escape the error, and wrap it in a span with class ``error-message``"""
    quoted = formencode.htmlfill.escape_formatter(error)
    return u'{0}'.format(quoted)

DEFAULT_ERROR_FORMATTERS = dict(formencode.htmlfill.default_formatter_dict)
DEFAULT_ERROR_FORMATTERS.update(
    default=default_formatter,
)


# encoding: utf-8
# Copyright (C) 2017 John Törnblom
#
# This file is part of pyxtuml.
#
# pyxtuml is free software: you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation, either
# version 3 of the License, or (at your option) any later version.
#
# pyxtuml is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with pyxtuml. If not, see <http://www.gnu.org/licenses/>.
'''
Check an xtuml model for association constraint violations in its metamodel.
'''

import logging
import optparse
import sys
import xtuml

logger = logging.getLogger('consistency_check')

def pretty_to_link(inst, link):
    '''
    Create a human-readable representation of a link on the 'TO'-side
    '''
    values = ''
    prefix = ''
    metaclass = xtuml.get_metaclass(inst)

    for name, ty in metaclass.attributes:
        if name in link.key_map:
            value = getattr(inst, name)
            value = xtuml.serialize_value(value, ty)
            name = link.key_map[name]
            values += '%s%s=%s' % (prefix, name, value)
            prefix = ', '

    return '%s(%s)' % (link.kind, values)

def pretty_from_link(inst, link):
    '''
    Create a human-readable representation of a link on the 'FROM'-side
    '''
    values = ''
    prefix = ''
    metaclass = xtuml.get_metaclass(inst)

    for name, ty in metaclass.attributes:
        if name in link.key_map:
            value = getattr(inst, name)
            value = xtuml.serialize_value(value, ty)
            values += '%s%s=%s' % (prefix, name, value)
            prefix = ', '

    return '%s(%s)' % (metaclass.kind, values)

def pretty_unique_identifier(inst, identifier):
    '''
    Create a human-readable representation a unique identifier.
    '''
    values = ''
    prefix = ''
    metaclass = xtuml.get_metaclass(inst)

    for name, ty in metaclass.attributes:
        if name in metaclass.identifying_attributes:
            value = getattr(inst, name)
            value = xtuml.serialize_value(value, ty)
            values += '%s%s=%s' % (prefix, name, value)
            prefix = ', '

    return '%s(%s)' % (identifier, values)

def check_uniqueness_constraint(m, kind=None):
    '''
    Check the model for uniqueness constraint violations.
    '''
    if kind is None:
        metaclasses = m.metaclasses.values()
    else:
        metaclasses = [m.find_metaclass(kind)]

    res = 0
    for metaclass in metaclasses:
        id_map = dict()
        for identifier in metaclass.indices:
            id_map[identifier] = dict()

        for inst in metaclass.select_many():
            # Check for null-values
            for name, ty in metaclass.attributes:
                if name not in metaclass.identifying_attributes:
                    continue

                value = getattr(inst, name)
                isnull = value is None
                isnull |= (ty == 'UNIQUE_ID' and not value)
                if isnull:
                    res += 1
                    logger.warning('%s.%s is part of an identifier and is null'
                                   % (metaclass.kind, name))

            # Check uniqueness
            for identifier in metaclass.indices:
                kwargs = dict()
                for name in metaclass.indices[identifier]:
                    kwargs[name] = getattr(inst, name)

                index_key = frozenset(kwargs.items())
                if index_key in id_map[identifier]:
                    res += 1
                    id_string = pretty_unique_identifier(inst, identifier)
                    logger.warning('uniqueness constraint violation in %s, %s'
                                   % (metaclass.kind, id_string))

                id_map[identifier][index_key] = inst

    return res

def check_link_integrity(m, link):
    '''
    Check the model for integrity violations on an association in a particular direction.
    '''
    res = 0
    for inst in link.from_metaclass.select_many():
        q_set = list(link.navigate(inst))

        if(len(q_set) < 1 and not link.conditional) or (
          (len(q_set) > 1 and not link.many)):
            res += 1
            logger.warning('integrity violation in '
                           '%s --(%s)--> %s' % (pretty_from_link(inst, link),
                                                link.rel_id,
                                                pretty_to_link(inst, link)))

    return res

def check_subtype_integrity(m, super_kind, rel_id):
    '''
    Check the model for integrity violations across a subtype association.
    '''
    if isinstance(rel_id, int):
        rel_id = 'R%d' % rel_id

    res = 0
    for inst in m.select_many(super_kind):
        if not xtuml.navigate_subtype(inst, rel_id):
            res += 1
            logger.warning('integrity violation across '
                           '%s[%s]' % (super_kind, rel_id))

    return res

def check_association_integrity(m, rel_id=None):
    '''
    Check the model for integrity violations on association(s).
    '''
    if isinstance(rel_id, int):
        rel_id = 'R%d' % rel_id

    res = 0
    for ass in m.associations:
        if rel_id in [ass.rel_id, None]:
            res += check_link_integrity(m, ass.source_link)
            res += check_link_integrity(m, ass.target_link)

    return res

def main(args):
    parser = optparse.OptionParser(usage="%prog [options] <sql_file> [another_sql_file...].",
                                   version=xtuml.version.complete_string,
                                   formatter=optparse.TitledHelpFormatter())

    parser.set_description(__doc__.strip())

    parser.add_option("-r", "-R", dest="rel_ids", type='int', metavar="<number>",
                      help="limit consistency check to one or more associations",
                      action="append", default=[])

    parser.add_option("-k", dest="kinds", type='string', metavar="<key letter>",
                      help="limit check for uniqueness constraint violations to one or more classes",
                      action="append", default=[])

    parser.add_option("-v", "--verbosity", dest='verbosity', action="count",
                      help="increase debug logging level", default=1)

    (opts, args) = parser.parse_args(args)
    if len(args) == 0:
        parser.print_help()
        sys.exit(1)

    levels = {
              0: logging.ERROR,
              1: logging.WARNING,
              2: logging.INFO,
              3: logging.DEBUG,
    }
    logging.basicConfig(level=levels.get(opts.verbosity, logging.DEBUG))

    loader = xtuml.ModelLoader()
    for filename in args:
        loader.filename_input(filename)

    m = loader.build_metamodel()

    error = 0
    for rel_id in opts.rel_ids:
        error += xtuml.check_association_integrity(m, rel_id)

    if not opts.rel_ids:
        error += xtuml.check_association_integrity(m)

    for kind in opts.kinds:
        error += xtuml.check_uniqueness_constraint(m, kind)

    if not opts.kinds:
        error += xtuml.check_uniqueness_constraint(m)

    return error

if __name__ == '__main__':
    num_errors = main(sys.argv[1:])
    sys.exit(num_errors > 0)


"""
    This will override how traceback works so that it skips certain modules
"""

__author__ = 'Ben Christenson'
__date__ = "11/5/15"
import inspect
import linecache
import traceback
import sys

SKIPPED_PATHS = []
SKIPPED_MODULES = []
ONLY_MODULES = []

def skip_module(*modules):
    """
        This will exclude all of the "modules" from the traceback
    :param modules: list of modules to exclude
    :return:        None
    """
    modules = (modules and isinstance(modules[0], list)) and \
              modules[0] or modules

    for module in modules:
        if not module in SKIPPED_MODULES:
            SKIPPED_MODULES.append(module)
    traceback.extract_tb = _new_extract_tb

def only_module(*modules):
    """
        This will exclude all modules from the traceback except these "modules"
    :param modules: list of modules to report in traceback
    :return:        None
    """
    modules = (modules and isinstance(modules[0], list)) and \
              modules[0] or modules
    for module in modules:
        if not module in ONLY_MODULES:
            ONLY_MODULES.append(module)
    traceback.extract_tb = _new_extract_tb

def skip_path(*paths):
    """
        This will exclude all modules that start from this path
    :param paths: list of str of the path of modules to exclude
    :return:      None
    """
    paths = (paths and isinstance(paths[0], list)) and paths[0] or paths
    for path in paths:
        if not path in SKIPPED_PATHS:
            SKIPPED_PATHS.append(path)
    traceback.extract_tb = _new_extract_tb

def _test_skip_frame(frame):
    try:
        module = inspect.getmodule(frame)
        if ONLY_MODULES and not module in ONLY_MODULES:
            return True
        for path in SKIPPED_PATHS:
            if path in frame.f_code.co_filename.replace('\\', '/'):
                return True
        if module in SKIPPED_MODULES:
            return True
        return False
    except Exception as e:
        return False

def _tb_skipper(tbnext):
    while tbnext is not None:
        if not _test_skip_frame(tbnext.tb_frame):
            yield tbnext
        if tbnext.tb_next is tbnext: break
        tbnext = tbnext.tb_next

    yield None

def _new_extract_tb(tb, limit=None):
    if limit is None:
        if hasattr(sys, 'tracebacklimit'):
            limit = sys.tracebacklimit
    list_ = []
    n = 0
    new_tb_order = _tb_skipper(tb)  # <-- this line added
    tb = next(new_tb_order)
    while tb is not None and (limit is None or n < limit):
        f = tb.tb_frame
        lineno = tb.tb_lineno
        co = f.f_code
        filename = co.co_filename
        name = co.co_name
        linecache.checkcache(filename)
        line = linecache.getline(filename, lineno, f.f_globals)
        if line:
            line = line.strip()
        else:
            line = None
        list_.append((filename, lineno, name, line))
        tb = next(new_tb_order)  # <-- this line modified, was tb = tb.tb_next
        n += 1
    return list_


from datetime import timedelta
from typing import List

from aiohttp import ClientSession, ClientConnectionError
from feedparser import parse
from aiobreaker import CircuitBreaker

from hyperion import logger
from . import ApiError

twitrss_breaker = CircuitBreaker(3, timedelta(hours=1))

@twitrss_breaker
async def fetch_twitter(handle: str) -> List:
    """
    Gets the twitter feed for a given handle.
    :param handle: The twitter handle.
    :return: A list of entries in a user's feed.
    :raises ApiError: When the api couldn't connect.
    :raises CircuitBreakerError: When the circuit breaker is open.
    """
    async with ClientSession() as session:
        try:
            async with session.get(f"http://twitrss.me/twitter_user_to_rss/?user={handle}") as request:
                text = await request.text()
        except ClientConnectionError as con_err:
            logger.debug(f"Could not connect to {con_err.host}")
            raise ApiError(f"Could not connect to {con_err.host}")
        else:
            feed = parse(text)
            for x in feed.entries:
                x["image"] = feed.feed["image"]["href"]
            return feed.entries


from json import JSONDecodeError
from typing import Dict, List, Optional

import aiohttp
from aiohttp import ClientConnectionError, ClientSession
from aiobreaker import CircuitBreaker

from hyperion import logger
from . import ApiError

wikipedia_breaker = CircuitBreaker()

@wikipedia_breaker
async def fetch_nearby(lat: float, long: float, limit: int = 10) -> Optional[List[Dict]]:
    """
    Gets wikipedia articles near a given set of coordinates.
    :raise ApiError: When there was an error connecting to the API.

    todo cache
    """
    request_url = f"https://en.wikipedia.org/w/api.php?action=query" \
                  f"&list=geosearch" \
                  f"&gscoord={lat}%7C{long}" \
                  f"&gsradius=10000" \
                  f"&gslimit={limit}" \
                  f"&format=json"

    async with ClientSession() as session:
        try:
            async with session.get(request_url) as request:
                if request.status == 404:
                    return None
                data = (await request.json())["query"]["geosearch"]

        except ClientConnectionError as con_err:
            logger.debug(f"Could not connect to {con_err.host}")
            raise ApiError(f"Could not connect to {con_err.host}")
        except JSONDecodeError as dec_err:
            logger.error(f"Could not decode data: {dec_err}")
            raise ApiError(f"Could not decode data: {dec_err}")
        except KeyError:
            return None
        else:
            for location in data:
                location.pop("ns")
                location.pop("primary")
            return data