# encoding: utf-8

"""
.. codeauthor:: Tsuyoshi Hombashi <tsuyoshi.hombashi@gmail.com>
"""

from __future__ import absolute_import, unicode_literals

import os.path
import re
import sqlite3
import warnings
from collections import OrderedDict
from textwrap import dedent

import typepy

from ._const import MAX_VERBOSITY_LEVEL, SQLITE_SYSTEM_TABLES, SchemaHeader
from ._error import DataNotFoundError, OperationalError
from ._logger import logger
from ._schema import SQLiteTableSchema

class SQLiteSchemaExtractor(object):
    """A SQLite database file schema extractor class.

    Args:
        database_source (str or simplesqlite.SimpleSQLite or sqlite3.Connection):
            SQLite database source to extract schema information.
    """

    global_debug_query = False

    _SQLITE_MASTER_TABLE_NAME = "master"
    _SQLITE_MASTER_ATTR_NAME_LIST = ["tbl_name", "sql", "type", "name", "rootpage"]

    _RE_ATTR_DESCRIPTION = re.compile("[(].*[)]")
    _RE_FOREIGN_KEY = re.compile("FOREIGN KEY")
    _RE_ATTR_NAME = re.compile(r"^'.+?'|^\".+?\"|^\[.+?\]")

    _RE_NOT_NULL = re.compile("NOT NULL", re.IGNORECASE)
    _RE_PRIMARY_KEY = re.compile("PRIMARY KEY", re.IGNORECASE)
    _RE_UNIQUE = re.compile("UNIQUE", re.IGNORECASE)
    _RE_AUTO_INC = re.compile("AUTOINCREMENT", re.IGNORECASE)

    def __init__(self, database_source):
        is_connection_required = True

        try:
            if database_source.is_connected():
                # datasource is a SimpleSQLite instance
                self.__con = database_source.connection
                is_connection_required = False
        except AttributeError:
            pass

        if isinstance(database_source, sqlite3.Connection):
            self.__con = database_source
            is_connection_required = False

        if is_connection_required:
            if not os.path.isfile(database_source):
                raise IOError("file not found: {}".format(database_source))

            try:
                self.__con = sqlite3.connect(database_source)
            except sqlite3.OperationalError as e:
                raise OperationalError(e)

        self.__cur = self.__con.cursor()
        self.__con_sqlite_master = None
        self.__total_changes = None

    def fetch_table_names(self, include_system_table=False):
        """
        :return: List of table names in the database.
        :rtype: list
        """

        result = self.__cur.execute("SELECT name FROM sqlite_master WHERE TYPE='table'")
        if result is None:
            return []

        table_names = [record[0] for record in result.fetchall()]

        if include_system_table:
            return table_names

        return [table for table in table_names if table not in SQLITE_SYSTEM_TABLES]

    def fetch_table_name_list(self, include_system_table=False):
        warnings.warn(
            "'fetch_table_name_list()' has moved to 'fetch_table_names()'", DeprecationWarning
        )

        return self.fetch_table_names(include_system_table)

    def fetch_table_schema(self, table_name):
        return SQLiteTableSchema(table_name, schema_map=self.__fetch_table_metadata(table_name))

    def fetch_database_schema(self):
        for table_name in self.fetch_table_names():
            yield self.fetch_table_schema(table_name)

    def fetch_database_schema_as_dict(self):
        database_schema = {}
        for table_schema in self.fetch_database_schema():
            database_schema.update(table_schema.as_dict())

        return database_schema

    def fetch_sqlite_master(self):
        """
        Get sqlite_master table information as a list of dictionaries.

        :return: sqlite_master table information.
        :rtype: list

        :Sample Code:
            .. code:: python

                from sqliteschema import SQLiteSchemaExtractor

                print(json.dumps(SQLiteSchemaExtractor("sample.sqlite").fetch_sqlite_master(), indent=4))

        :Output:
            .. code-block:: json

                [
                    {
                        "tbl_name": "sample_table",
                        "sql": "CREATE TABLE 'sample_table' ('a' INTEGER, 'b' REAL, 'c' TEXT, 'd' REAL, 'e' TEXT)",
                        "type": "table",
                        "name": "sample_table",
                        "rootpage": 2
                    },
                    {
                        "tbl_name": "sample_table",
                        "sql": "CREATE INDEX sample_table_a_index ON sample_table('a')",
                        "type": "index",
                        "name": "sample_table_a_index",
                        "rootpage": 3
                    }
                ]
        """

        sqlite_master_record_list = []
        result = self.__cur.execute(
            "SELECT {:s} FROM sqlite_master".format(", ".join(self._SQLITE_MASTER_ATTR_NAME_LIST))
        )

        for record in result.fetchall():
            sqlite_master_record_list.append(
                dict(
                    [
                        [attr_name, item]
                        for attr_name, item in zip(self._SQLITE_MASTER_ATTR_NAME_LIST, record)
                    ]
                )
            )

        return sqlite_master_record_list

    def dumps(self, output_format=None, verbosity_level=MAX_VERBOSITY_LEVEL):
        dump_list = []

        for table_schema in self.fetch_database_schema():
            dump_list.append(
                table_schema.dumps(output_format=output_format, verbosity_level=verbosity_level)
            )

        return "\n".join(dump_list)

    def _extract_attr_name(self, schema):
        _RE_SINGLE_QUOTES = re.compile("^'.+?'")
        _RE_DOUBLE_QUOTES = re.compile('^".+?"')
        _RE_BRACKETS = re.compile(r"^\[.+?\]")

        match_attr_name = self._RE_ATTR_NAME.search(schema)
        if match_attr_name is None:
            attr_name = schema.split()[0]
        else:
            attr_name = match_attr_name.group()

        if _RE_SINGLE_QUOTES.search(attr_name):
            return attr_name.strip("'")

        if _RE_DOUBLE_QUOTES.search(attr_name):
            return attr_name.strip('"')

        if _RE_BRACKETS.search(attr_name):
            return attr_name.strip("[]")

        return attr_name

    def _extract_attr_type(self, schema):
        match_attr_name = self._RE_ATTR_NAME.search(schema)
        if match_attr_name is None:
            return schema.split()[1]

        schema_wo_name = self._RE_ATTR_NAME.sub("", schema).strip()

        return schema_wo_name.split()[0]

    def _extract_attr_constraints(self, schema):
        attr_name_match = self._RE_ATTR_NAME.search(schema)
        if attr_name_match is None:
            return " ".join(schema.split()[2:])

        schema_wo_name = self._RE_ATTR_NAME.sub("", schema).strip()

        return " ".join(schema_wo_name.split()[1:])

    def _fetch_attr_schema(self, table_name, schema_type):
        if table_name in SQLITE_SYSTEM_TABLES:
            logger.debug("skip fetching sqlite system table: {:s}".format(table_name))
            return []

        self.__update_sqlite_master_db()

        result = self.__execute_sqlite_master(
            "SELECT {:s} FROM {:s} WHERE {:s} AND {:s}".format(
                "sql",
                self._SQLITE_MASTER_TABLE_NAME,
                "{:s} = '{:s}'".format("tbl_name", table_name),
                "{:s} = '{:s}'".format("type", schema_type),
            ),
            self.global_debug_query,
        )
        error_message_format = "data not found in '{}' table"

        try:
            table_schema = result.fetchone()[0]
        except TypeError:
            raise DataNotFoundError(error_message_format.format(self._SQLITE_MASTER_TABLE_NAME))

        match = self._RE_ATTR_DESCRIPTION.search(table_schema)
        if match is None:
            raise DataNotFoundError(error_message_format.format(table_schema))

        return [
            attr.strip()
            for attr in match.group().strip("()").split(",")
            if self._RE_FOREIGN_KEY.search(attr) is None
        ]

    def _fetch_index_schema(self, table_name):
        self.__update_sqlite_master_db()

        result = self.__execute_sqlite_master(
            "SELECT {:s} FROM {:s} WHERE {:s} AND {:s}".format(
                "sql",
                self._SQLITE_MASTER_TABLE_NAME,
                "{:s} = '{:s}'".format("tbl_name", table_name),
                "{:s} = '{:s}'".format("type", "index"),
            ),
            self.global_debug_query,
        )

        try:
            return [
                record[0] for record in result.fetchall() if typepy.is_not_empty_sequence(record[0])
            ]
        except TypeError:
            raise DataNotFoundError("index not found in '{}'".format(table_name))

    def __fetch_table_metadata(self, table_name):
        index_query_list = self._fetch_index_schema(table_name)
        metadata = OrderedDict()

        for attr_schema in self._fetch_attr_schema(table_name, "table"):
            values = OrderedDict()
            attr_name = self._extract_attr_name(attr_schema)
            re_index = re.compile(re.escape(attr_name))

            values[SchemaHeader.ATTR_NAME] = attr_name
            values[SchemaHeader.INDEX] = False

            try:
                values[SchemaHeader.DATA_TYPE] = self._extract_attr_type(attr_schema)
            except IndexError:
                continue

            try:
                constraint = self._extract_attr_constraints(attr_schema)
            except IndexError:
                continue

            values[SchemaHeader.NULL] = (
                "NO" if self._RE_NOT_NULL.search(constraint) is not None else "YES"
            )
            values[SchemaHeader.KEY] = self.__extract_key_constraint(constraint)
            values[SchemaHeader.DEFAULT] = self.__extract_default_value(constraint)

            if values[SchemaHeader.KEY] in ("PRI", "UNI"):
                values[SchemaHeader.INDEX] = True
            else:
                for index_query in index_query_list:
                    if re_index.search(index_query) is not None:
                        values[SchemaHeader.INDEX] = True
                        break

            values[SchemaHeader.EXTRA] = ", ".join(self.__extract_extra(constraint))

            metadata.setdefault(table_name, []).append(values)

        if not metadata:
            pass

        return metadata

    def __extract_key_constraint(self, constraint):
        if self._RE_PRIMARY_KEY.search(constraint):
            return "PRI"

        if self._RE_UNIQUE.search(constraint):
            return "UNI"

        return ""

    def __extract_default_value(self, constraint):
        regexp_default = re.compile("DEFAULT (?P<value>.+)", re.IGNORECASE)
        match = regexp_default.search(constraint)

        if match:
            return match.group("value")

        if self._RE_NOT_NULL.search(constraint):
            return ""

        return "NULL"

    def __extract_extra(self, constraint):
        extra_list = []
        if self._RE_AUTO_INC.search(constraint):
            extra_list.append("AUTOINCREMENT")

        return extra_list

    def __execute_sqlite_master(self, query, is_logging=True):
        if is_logging:
            logger.debug(query)

        return self.__con_sqlite_master.execute(query)

    def __update_sqlite_master_db(self):
        try:
            if self.__total_changes == self.__con.total_changes:
                """
                logger.debug(
                    "skipping the {} table update. updates not found after the last update.".format(
                        self._SQLITE_MASTER_TABLE_NAME
                    )
                )
                """
                return
        except AttributeError:
            pass

        if self.__con_sqlite_master:
            self.__con_sqlite_master.close()

        self.__con_sqlite_master = sqlite3.connect(":memory:")
        sqlite_master = self.fetch_sqlite_master()

        if typepy.is_empty_sequence(sqlite_master):
            return

        sqlite_master_records = [
            [record[attr] for attr in self._SQLITE_MASTER_ATTR_NAME_LIST]
            for record in sqlite_master
        ]
        self.__execute_sqlite_master(
            dedent(
                """\
                CREATE TABLE {:s} (
                    tbl_name TEXT NOT NULL,
                    sql TEXT,
                    type TEXT NOT NULL,
                    name TEXT NOT NULL,
                    rootpage INTEGER NOT NULL
                )
                """
            ).format(self._SQLITE_MASTER_TABLE_NAME),
            False,
        )
        self.__con_sqlite_master.executemany(
            "INSERT INTO {:s} VALUES (?,?,?,?,?)".format(self._SQLITE_MASTER_TABLE_NAME),
            sqlite_master_records,
        )

        if self.global_debug_query:
            logger.debug(
                "insert {:d} records into {:s}".format(
                    len(sqlite_master_records), self._SQLITE_MASTER_TABLE_NAME
                )
            )

        self.__con_sqlite_master.commit()

        self.__total_changes = self.__con.total_changes


"""
jsonselect

Public interface:
    select
    take a selector and an object. return matched node(s)

Exceptions:
    SelectorSyntaxError
    Raised by Parser when parsing cannot continue.

TODO:
T:expr(E)           3   A node of type T with a value that satisfies
                        the expression E
"""
from __future__ import division
import re
import numbers
import collections
import logging
import json
import sys

if sys.version_info[0] >= 3:
    basestring = str

log = logging.getLogger(__name__)

S_TYPE = lambda x, token: ('type', token)
S_IDENTIFIER = lambda x, token: ('identifier', token[1:])
S_QUOTED_IDENTIFIER = lambda x, token: S_IDENTIFIER(None,
                                                    token.replace('"', ''))
S_PCLASS = lambda x, token: ('pclass', token[1:])
S_PCLASS_FUNC = lambda x, token: ('pclass_func', token[1:])
S_NTH_FUNC = lambda x, token: ('nth_func', token[1:])
S_OPER = lambda x, token: ('operator', token)
S_EMPTY = lambda x, token:  ('empty', ' ')
S_UNK = lambda x, token: ('unknown', token)
S_FLOAT = lambda x, token: ('float', float(token))
S_WORD = lambda x, token: ('word', token[1:-1])
S_BINOP = lambda x, token: ('binop', token)
S_VALS = lambda x, token: ('val', token)
S_KEYWORD = lambda x, token: ('keyword', token)
S_PVAR = lambda x, token: ('pvar', token)
S_EXPR = lambda x, token: ('expr', token)
S_NUMBER = lambda x, token: ('number', token)
S_STRING = lambda x, token: ('string', token)
S_PAREN = lambda x, token: ('paren', token)

SCANNER = re.Scanner([
    (r"\([^\)]+\)", S_EXPR),
    (r"[~*,>]", S_OPER),
    (r"\s", S_EMPTY),
    (r"(-?\d+(\.\d*)([eE][+\-]?\d+)?)", S_FLOAT),
    (r"string|boolean|null|array|object|number", S_TYPE),
    (u"\"([_a-zA-Z]|[^\0-\0177]|\\\\[^\s0-9a-fA-F])([_a-zA-Z0-9\-]"
     u"|[^\u0000-\u0177]|(\\\\[^\s0-9a-fA-F]))*\"", S_WORD),
    (r'\.?\"([^"\\]|\\[^"])*\"', S_QUOTED_IDENTIFIER),
    (u"\.([_a-zA-Z]|[^\0-\0177]|\\\\[^\s0-9a-fA-F])([_a-zA-Z0-9\-]"
     u"|[^\u0000-\u0177]|(\\\\[^\s0-9a-fA-F]))*", S_IDENTIFIER),
    (r":(root|empty|first-child|last-child|only-child)", S_PCLASS),
    (r":(has|expr|val|contains)", S_PCLASS_FUNC),
    (r":(nth-child|nth-last-child)", S_NTH_FUNC),
    (r"(&&|\|\||[\$\^<>!\*]=|[=+\-*/%<>])", S_BINOP),
    (r"true|false|null", S_VALS),
    (r"n", S_PVAR),
    (r"odd|even", S_KEYWORD),
])

EXPR_SCANNER = re.Scanner([
    (r"\s", S_EMPTY),
    (r"true|false|null", S_VALS),
    (r"-?\d+(\.\d*)?([eE][+\-]?\d+)?", S_NUMBER),
    (r"\"([^\]|\[^\"])*\"", S_STRING),
    (r"x", S_PVAR),
    (r"(&&|\|\||[\$\^<>!\*]=|[=+\-*/%<>])", S_BINOP),
    (r"\(|\)", S_PAREN)
])

class SelectorSyntaxError(Exception):
    pass

class LexingError(SelectorSyntaxError):
    pass

# metadata about a node in the target object graph
Node = collections.namedtuple('Node', [
    'value',
    'parent',       # parent Node. None if root.
    'parent_key',   # if parent is a dict, key which indexes current Node.
    'idx',          # if parent is an array, index of curr Node. starts at 1.
    'siblings'      # if parent is an array, number of elements in it
])

def object_iter(obj, parent=None, parent_key=None, idx=None,
                siblings=None):
    """Yields each node of object graph in postorder."""

    obj_node = Node(value=obj, parent=parent, parent_key=parent_key,
                siblings=siblings, idx=idx)

    if isinstance(obj, list):
        _siblings = len(obj)
        for i, elem in enumerate(obj):
            for node in object_iter(elem, obj_node, None, i + 1, _siblings):
                yield node
    elif isinstance(obj, collections.Mapping):
        for key in obj:
            for node in object_iter(obj[key], obj_node, key):
                yield node
    yield obj_node

def lex(input, scanner=SCANNER):
    tokens, rest = scanner.scan(input)
    if not len(tokens):
        raise LexingError("no input parsed.")
    if len(rest):
        raise LexingError("found leftover tokens: (%s, %s)" % (tokens, rest))
    return [tok for tok in tokens if tok[0] != 'empty']

def lex_expr(expression):
    tokens = lex(expression, scanner=EXPR_SCANNER)
    for i, token in enumerate(tokens):
        if token[0] in ('number', 'string', 'val'):
            tokens[i] = (token[0], json.loads(tokens[i][1]))
    return tokens

class Parser(object):

    """
    Parse jsonselect queries.

    A simple top-down recursive-descendant parser of jsonselect selectors.
    Initialize with the object you wish to match against.

    Clients should only need to call Parser.parse, which accepts
    a list of tokens as generated by jsonselect.lex
    """

    nth_child_pat = re.compile(
        r"^\s*\(\s*(?:([+\-]?)([0-9]*)n\s*(?:([+\-])\s*([0-9]))?"
        r"|(odd|even)|([+\-]?[0-9]+))\s*\)"
    )

    def __init__(self, obj):
        """Create a parser for a particular object."""
        self.obj = obj

    def parse(self, selector):
        """Accept a list of tokens. Returns matched nodes of self.obj."""
        log.debug(self.obj)
        tokens = lex(selector)

        if self.peek(tokens, 'operator') == '*':
            self.match(tokens, 'operator')
            results = list(object_iter(self.obj))
        else:
            results = self.selector_production(tokens)

        results = [node.value for node in results]
        # single results should be returned as a primitive
        if len(results) == 1:
            return results[0]
        elif not len(results):
            return None
        return results

    def selector_production(self, tokens):
        """Production for a full selector."""

        validators = []
        # the following productions should return predicate functions.

        if self.peek(tokens, 'type'):
            type_ = self.match(tokens, 'type')
            validators.append(self.type_production(type_))

        if self.peek(tokens, 'identifier'):
            key = self.match(tokens, 'identifier')
            validators.append(self.key_production(key))

        if self.peek(tokens, 'pclass'):
            pclass = self.match(tokens, 'pclass')
            validators.append(self.pclass_production(pclass))

        if self.peek(tokens, 'nth_func'):
            nth_func = self.match(tokens, 'nth_func')
            validators.append(self.nth_child_production(nth_func, tokens))

        if self.peek(tokens, 'pclass_func'):
            pclass_func = self.match(tokens, 'pclass_func')
            validators.append(self.pclass_func_production(pclass_func, tokens))

        if not len(validators):
            raise SelectorSyntaxError('no selector recognized.')

        # apply validators from a selector expression to self.obj
        results = self._match_nodes(validators, self.obj)

        if self.peek(tokens, 'operator'):
            operator = self.match(tokens, 'operator')
            rvals = self.selector_production(tokens)
            if operator == ',':
                results.extend(rvals)
            elif operator == '>':
                results = self.parents(results, rvals)
            elif operator == '~':
                results = self.siblings(results, rvals)
            elif operator == ' ':
                results = self.ancestors(results, rvals)
            else:
                raise SelectorSyntaxError("unrecognized operator '%s'"
                                          % operator)
        else:
            if len(tokens):
                rvals = self.selector_production(tokens)
                results = self.ancestors(results, rvals)

        return results

    def parents(self, lhs, rhs):
        """Find nodes in rhs which have parents in lhs."""

        return [node for node in rhs if node.parent in lhs]

    def ancestors(self, lhs, rhs):
        """Return nodes from rhs which have ancestors in lhs."""

        def _search(node):
            if node in lhs:
                return True
            if not node.parent:
                return False
            return _search(node.parent)

        return [node for node in rhs if _search(node)]

    def siblings(self, lhs, rhs):
        """Find nodes in rhs having common parents in lhs."""
        parents = [node.parent for node in lhs]

        return [node for node in rhs if node.parent in parents]

    # The following productions should return predicate functions

    def type_production(self, type_):
        assert type_

        type_map = {
            'string': basestring,
            'number': numbers.Number,
            'object': collections.Mapping,
            'array': list,
            'boolean': bool,
            'null': type(None)
        }
        return lambda node: isinstance(node.value, type_map[type_])

    def key_production(self, key):
        assert key

        def validate(node):
            if not node.parent_key:
                return False
            return node.parent_key == key
        return validate

    def pclass_production(self, pclass):
        pclass_map = {
            'first-child': lambda node: node.idx == 1,
            'last-child': lambda node: (node.siblings and
                                        node.idx == node.siblings),
            'only-child': lambda node: node.siblings == 1,
            'root': lambda node: not node.parent,
            'empty': lambda node: (isinstance(node.value, list) and
                                   not len(node.value))
        }

        try:
            return pclass_map[pclass]
        except KeyError:
            raise SelectorSyntaxError("unrecognized pclass %s" % pclass)

    def parse_expr(self, tokens, node):
        def types_eq(type_, *args):
            return all([isinstance(arg, type_) for arg in args])

        cmpf_map = {
            '*': lambda lhs, rhs: (types_eq(numbers.Number, lhs, rhs) and
                                   lhs * rhs),
            '/': lambda lhs, rhs: (types_eq(numbers.Number, lhs, rhs) and
                                   lhs / rhs),
            '%': lambda lhs, rhs: (types_eq(numbers.Number, lhs, rhs) and
                                   lhs % rhs),
            '+': lambda lhs, rhs: (types_eq(numbers.Number, lhs, rhs) and
                                   lhs + rhs),
            '-': lambda lhs, rhs: (types_eq(numbers.Number, lhs, rhs) and
                                   lhs - rhs),
            '<=': lambda lhs, rhs: (types_eq(numbers.Number, lhs, rhs) and
                                    lhs <= rhs),
            '<': lambda lhs, rhs: (types_eq(basestring, lhs, rhs) and
                                   lhs < rhs),
            '>=': lambda lhs, rhs: (types_eq(numbers.Number, lhs, rhs) and
                                    lhs >= rhs),
            '>': lambda lhs, rhs: (types_eq(basestring, lhs, rhs) and
                                   lhs > rhs),
            '$=': lambda lhs, rhs: (types_eq(basestring, lhs, rhs) and
                                    lhs.rfind(rhs) == len(lhs) - len(rhs)),
            '^=': lambda lhs, rhs: (types_eq(basestring, lhs, rhs) and
                                    lhs.find(rhs) == 0),
            '*=': lambda lhs, rhs: (types_eq(basestring, lhs, rhs) and
                                    lhs.find(rhs) != 0),
            '=': lambda lhs, rhs: lhs == rhs,
            '!=': lambda lhs, rhs: lhs != rhs,
            '&&': lambda lhs, rhs: lhs and rhs,
            '||': lambda lhs, rhs: lhs or rhs
        }

# ((12 % 10) + 40 = x)

        def parse(tokens):
            if not len(tokens):
                raise Exception

            if self.peek(tokens, 'paren') == '(':
                self.match(tokens, 'paren')
                lhs = parse(tokens)
                return lhs
            elif self.peek(tokens, 'pvar') is not None:
                self.match(tokens, 'pvar')
                lhs = node.value
            else:
                for tok in ('string', 'val', 'number'):
                    if self.peek(tokens, tok) is not None:
                        lhs = self.match(tokens, tok)
                        break

            if self.peek(tokens, 'paren') == ')':
                self.match(tokens, 'paren')
                return lhs

            op = self.match(tokens, 'binop')
            cf = cmpf_map[op]
            rhs = parse(tokens)

            return cf(lhs, rhs)

        return parse(tokens)

    def expr_production(self, args):
        tokens = lex_expr(args)
        return lambda node: self.parse_expr(list(tokens), node)

    def pclass_func_production(self, pclass, tokens):
        args = self.match(tokens, 'expr')

        if pclass == 'expr':
            return self.expr_production(args)

        args = lex(args[1:-1])

        if pclass == 'has':
            # T:has(S)
            # A node of type T which has a child node satisfying the selector S
            for i, token in enumerate(args):
                if token[1] == '>':
                    args[i] = (token[0], ' ')
            rvals = self.selector_production(args)
            ancestors = [node.parent for node in rvals]
            return lambda node: node in ancestors

        if pclass == 'contains':
            return lambda node: (isinstance(node.value, basestring) and
                                 node.value.find(args[0][1]) >= 0)

        if pclass == 'val':
            return lambda node: (isinstance(node.value, basestring) and
                                 node.value == args[0][1])

        raise SelectorSyntaxError("unsupported pclass function %s" % pclass)

    def nth_child_production(self, lexeme, tokens):
        """Parse args and pass them to pclass_func_validator."""

        args = self.match(tokens, 'expr')

        pat = self.nth_child_pat.match(args)

        if pat.group(5):
            a = 2
            b = 1 if pat.group(5) == 'odd' else 0
        elif pat.group(6):
            a = 0
            b = int(pat.group(6))
        else:
            sign = pat.group(1) if pat.group(1) else '+'
            coef = pat.group(2) if pat.group(2) else '1'
            a = eval(sign + coef)
            b = eval(pat.group(3) + pat.group(4)) if pat.group(3) else 0

        reverse = False
        if lexeme == 'nth-last-child':
            reverse = True

        def validate(node):
            """This crazy function taken from jsonselect.js:444."""

            if not node.siblings:
                return False

            idx = node.idx - 1
            tot = node.siblings

            if reverse:
                idx = tot - idx
            else:
                idx += 1

            if a == 0:
                m = b == idx
            else:
                mod = (idx - b) % a
                m = not mod and (idx * a + b) >= 0
            return m

        return validate

    def _match_nodes(self, validators, obj):
        """Apply each validator in validators to each node in obj.

        Return each node in obj which matches all validators.
        """

        results = []
        for node in object_iter(obj):
            if all([validate(node) for validate in validators]):
                results.append(node)
        return results

    @staticmethod
    def match(tokens, type_):
        if Parser.peek(tokens, type_) is None:
            raise Exception('match not successful (%s, %s)' % (type_, tokens))

        token = tokens.pop(0)
        return token[1]

    @staticmethod
    def peek(tokens, type_):
        if not tokens:
            return None

        if isinstance(type_, list) and tokens[0][0] in type_:
            return tokens[0][1]
        elif tokens[0][0] == type_:
            return tokens[0][1]
        else:
            return None

def select(selector, obj):
    """Appy selector to obj and return matching nodes.

    If only one node is found, return it, otherwise return a list of matches.
    Returns False on syntax error. None if no results found.
    """

    parser = Parser(obj)
    try:
        return parser.parse(selector)
    except SelectorSyntaxError as e:
        log.exception(e)
        return False


import socket
import time
import fcntl
import random
import struct

from zope.interface import implementer

from twisted.internet import task, defer, reactor, udp
from twisted.internet.protocol import DatagramProtocol
from twisted.internet.interfaces import ISystemHandle

# OMG SHUT UP
class STFU(object):
    msg = lambda x, y: None
udp.log = STFU()

class IP(object):
    """IP header decoder
    """
    def __init__(self, packet):
        self.readPacket(packet)

    def readPacket(self, packet):
        vl = struct.unpack('!b', packet[0])[0]
        l = (vl & 0xf) * 4

        head = packet[:l]
        self.offset = struct.unpack('!H', packet[6:8])

        self.payload = packet[l:]

class EchoPacket(object):
    """ICMP Echo packet encoder and decoder
    """
    def __init__(self, seq=0, id=None, data=None, packet=None):
        if packet:
            self.decodePacket(packet)
            self.packet = packet
        else:
            self.id = id
            self.seq = seq
            self.data = data
            self.encodePacket()

    def calculateChecksum(self, buffer):
        nleft = len(buffer)
        sum = 0
        pos = 0
        while nleft > 1:
            sum = ord(buffer[pos]) * 256 + (ord(buffer[pos + 1]) + sum)
            pos = pos + 2
            nleft = nleft - 2
        if nleft == 1:
            sum = sum + ord(buffer[pos]) * 256

        sum = (sum >> 16) + (sum & 0xFFFF)
        sum += (sum >> 16)
        sum = (~sum & 0xFFFF)

        return sum

    def encodePacket(self):
        head = struct.pack('!bb', 8, 0)

        echo = struct.pack('!HH', self.seq, self.id)

        chk = self.calculateChecksum(
            head + '\x00\x00' + echo + self.data)

        chk = struct.pack('!H', chk)

        self.packet = head + chk + echo + self.data

    def decodePacket(self, packet):
        self.type, self.code, self.chk, self.seq, self.id = struct.unpack(
            '!bbHHH', packet[:8])

        self.data = packet[8:]

        rc = '%s\x00\x00%s' % (packet[:2], packet[4:])
        mychk = self.calculateChecksum(rc)

        if mychk == self.chk:
            self.valid = True
        else:
            self.valid = False

    def __repr__(self):
        return "<type=%s code=%s chk=%s seq=%s data=%s valid=%s>" % (
            self.type, self.code, self.chk, self.seq, len(self.data), self.valid)

class ICMPPing(DatagramProtocol):
    """ICMP Ping implementation
    """
    noisy=False
    def __init__(self, d, dst, count, inter=0.2, maxwait=1000, size=64):
        self.deferred = d
        self.dst = dst
        self.size = size - 36
        self.count = count
        self.seq = 0
        self.start = 0
        self.id_base = random.randint(0, 40000)
        self.maxwait = maxwait
        self.inter = inter

        self.t = task.LoopingCall(self.ping)
        self.recv = []

    def datagramReceived(self, datagram, address):
        now = int(time.time()*1000000)
        host, port = address

        packet = IP(datagram)

        icmp = EchoPacket(packet=packet.payload)

        if icmp.valid and icmp.code==0 and icmp.type==0:
            # Check ID is from this pinger
            if (icmp.id-icmp.seq) == self.id_base:
                ts = icmp.data[:8]
                data = icmp.data[8:]
                delta = (now - struct.unpack('!Q', ts)[0])/1000.0

                self.maxwait = (self.maxwait + delta)/2.0

                self.recv.append((icmp.seq, delta))

    def createData(self, n):
        s = ""
        c = 33
        for i in range(n):
            s += chr(c)
            if c < 126:
                c += 1
            else:
                c = 33
        return s

    def sendEchoRequest(self):
        # Pack the packet with an ascii table
        md = self.createData(self.size)

        us = int(time.time()*1000000)
        data = '%s%s' % (struct.pack('!Q', us), md)

        pkt = EchoPacket(seq=self.seq, id=self.id_base+self.seq, data=data)

        self.transport.write(pkt.packet)
        self.seq += 1

    def ping(self):
        if self.seq < self.count:
            self.sendEchoRequest()
        else:
            self.t.stop()

            tdelay = (self.maxwait * self.count)/1000.0
            elapsed = time.time() - self.start
            remaining = tdelay - elapsed
            if remaining < 0.05:
                remaining = 0.05

            reactor.callLater(remaining, self.endPing)

    def endPing(self):
        r = len(self.recv)
        loss = (self.count - r) / float(self.count)
        loss = int(100*loss)
        if r:
            avgLatency = sum([i[1] for i in self.recv]) / float(r)
        else:
            avgLatency = None

        self.deferred.callback((loss, avgLatency))

    def startPing(self):
        self.transport.connect(self.dst, random.randint(33434, 33534))
        self.start = time.time()
        self.t.start(self.inter)

    def startProtocol(self):
        self.startPing()

@implementer(ISystemHandle)
class ICMPPort(udp.Port):
    """Raw socket listener for ICMP
    """
    maxThroughput = 256 * 1024

    def createInternetSocket(self):
        s = socket.socket(
            socket.AF_INET, socket.SOCK_RAW, socket.IPPROTO_ICMP)

        s.setblocking(0)

        fd = s.fileno()

        # Set close-on-exec

        flags = fcntl.fcntl(fd, fcntl.F_GETFD)
        flags = flags | fcntl.FD_CLOEXEC
        fcntl.fcntl(fd, fcntl.F_SETFD, flags)

        return s

def ping(dst, count, inter=0.2, maxwait=1000, size=64):
    """Sends ICMP echo requests to destination `dst` `count` times.
    Returns a deferred which fires when responses are finished.
    """
    def _then(result, p):
        p.stopListening()
        return result

    d = defer.Deferred()
    p = ICMPPort(0, ICMPPing(d, dst, count, inter, maxwait, size), "", 8192, reactor)
    p.startListening()

    return d.addCallback(_then, p)


import signal
import json
import time
import urllib
import os

try:
    from StringIO import StringIO
except ImportError:
    from io import StringIO

try:
    from exceptions import IOError
except ImportError:
    pass

from zope.interface import implementer

from twisted.internet import reactor, protocol, defer, error
from twisted.web.http_headers import Headers
from twisted.web.iweb import IBodyProducer
from twisted.web.client import Agent
from twisted.names import client
from twisted.python import log

from twisted.internet.endpoints import clientFromString

class SocketyAgent(Agent):
    def __init__(self, reactor, path, **kwargs):
        self.path = path
        Agent.__init__(self, reactor, **kwargs)

    def _getEndpoint(self, scheme, host, port):
        client = clientFromString(reactor, self.path)
        return client

class Timeout(Exception):
    """
    Raised to notify that an operation exceeded its timeout.
    """

class Resolver(object):
    """Helper class for DNS resolution
    """

    def __init__(self):
        self.recs = {}

        self.resolver = client.getResolver()

    def reverseNameFromIPAddress(self, address):
        return '.'.join(reversed(address.split('.'))) + '.in-addr.arpa'

    def reverse(self, ip):
        def _ret(result, ip):
            host = ip
            if isinstance(result, tuple):
                answers, authority, additional = result
                if isinstance(answers, list):
                    ttl = answers[0].payload.ttl
                    host = answers[0].payload.name.name
                    self.recs[ip] = (host, ttl, time.time())

            return host

        if ip in self.recs:
            host, ttl, t = self.recs[ip]

            if (time.time() - t) < ttl:
                return defer.maybeDeferred(lambda x: x, host)

        return self.resolver.lookupPointer(
            name=self.reverseNameFromIPAddress(address=ip)
        ).addCallback(_ret, ip).addErrback(_ret, ip)

class BodyReceiver(protocol.Protocol):
    """ Simple buffering consumer for body objects """
    def __init__(self, finished):
        self.finished = finished
        self.data = StringIO()

    def dataReceived(self, data):
        self.data.write(data.decode())

    def connectionLost(self, reason):
        self.data.seek(0)
        self.finished.callback(self.data)

@implementer(IBodyProducer)
class StringProducer(object):
    """String producer for writing to HTTP requests
    """
    def __init__(self, body):
        self.body = body
        self.length = len(body)

    def startProducing(self, consumer):
        consumer.write(self.body)
        return defer.succeed(None)

    def pauseProducing(self):
        pass

    def stopProducing(self):
        pass

class ProcessProtocol(protocol.ProcessProtocol):
    """ProcessProtocol which supports timeouts"""
    def __init__(self, deferred, timeout):
        self.timeout = timeout
        self.timer = None

        self.deferred = deferred
        self.outBuf = StringIO()
        self.errBuf = StringIO()

    def outReceived(self, data):
        self.outBuf.write(data.decode())

    def errReceived(self, data):
        self.errBuf.write(data.decode())

    def processEnded(self, reason):
        if self.timer and (not self.timer.called):
            self.timer.cancel()

        out = self.outBuf.getvalue()
        err = self.errBuf.getvalue()

        e = reason.value
        code = e.exitCode

        if e.signal:
            self.deferred.errback(reason)
        else:
            self.deferred.callback((out, err, code))

    def connectionMade(self):
        @defer.inlineCallbacks
        def killIfAlive():
            try:
                yield self.transport.signalProcess('KILL')
                log.msg('Killed source proccess: Timeout %s exceeded' % self.timeout)
            except error.ProcessExitedAlready:
                pass

        self.timer = reactor.callLater(self.timeout, killIfAlive)

def fork(executable, args=(), env={}, path=None, timeout=3600):
    """fork
    Provides a deferred wrapper function with a timeout function

    :param executable: Executable
    :type executable: str.
    :param args: Tupple of arguments
    :type args: tupple.
    :param env: Environment dictionary
    :type env: dict.
    :param timeout: Kill the child process if timeout is exceeded
    :type timeout: int.
    """
    d = defer.Deferred()
    p = ProcessProtocol(d, timeout)
    reactor.spawnProcess(p, executable, (executable,)+tuple(args), env, path)
    return d

try:
    from twisted.internet.ssl import ClientContextFactory

    class WebClientContextFactory(ClientContextFactory):
        def getContext(self, hostname, port):
            return ClientContextFactory.getContext(self)
    SSL=True
except:
    SSL=False

try:
    from twisted.web import client
    client._HTTP11ClientFactory.noisy = False
    client.HTTPClientFactory.noisy = False
except:
    pass

class HTTPRequest(object):
    def __init__(self, timeout=120):
        self.timeout = timeout

    def abort_request(self, request):
        """Called to abort request on timeout"""
        self.timedout = True
        if not request.called:
            try:
                request.cancel()
            except error.AlreadyCancelled:
                return

    @defer.inlineCallbacks
    def response(self, request):
        if request.length:
            d = defer.Deferred()
            request.deliverBody(BodyReceiver(d))
            b = yield d
            body = b.read()
        else:
            body = ""

        defer.returnValue(body)

    def request(self, url, method='GET', headers={}, data=None, socket=None):
        self.timedout = False

        if socket:
            agent = SocketyAgent(reactor, socket)
        else:
            if url[:5] == 'https':
                if SSL:
                    agent = Agent(reactor, WebClientContextFactory())
                else:
                    raise Exception('HTTPS requested but not supported')
            else:
                agent = Agent(reactor)

        request = agent.request(method.encode(), url.encode(),
            Headers(headers),
            StringProducer(data) if data else None
        )

        if self.timeout:
            timer = reactor.callLater(self.timeout, self.abort_request,
                request)

            def timeoutProxy(request):
                if timer.active():
                    timer.cancel()
                return self.response(request)

            def requestAborted(failure):
                if timer.active():
                    timer.cancel()

                failure.trap(defer.CancelledError,
                             error.ConnectingCancelledError)

                raise Timeout(
                    "Request took longer than %s seconds" % self.timeout)

            request.addCallback(timeoutProxy).addErrback(requestAborted)
        else:
            request.addCallback(self.response)

        return request

    def getBody(self, url, method='GET', headers={}, data=None, socket=None):
        """Make an HTTP request and return the body
        """

        if not 'User-Agent' in headers:
            headers['User-Agent'] = ['Tensor HTTP checker']

        return self.request(url, method, headers, data, socket)

    @defer.inlineCallbacks
    def getJson(self, url, method='GET', headers={}, data=None, socket=None):
        """Fetch a JSON result via HTTP
        """
        if not 'Content-Type' in headers:
            headers['Content-Type'] = ['application/json']

        body = yield self.getBody(url, method, headers, data, socket)

        defer.returnValue(json.loads(body))

class PersistentCache(object):
    """A very basic dictionary cache abstraction. Not to be used
    for large amounts of data or high concurrency"""

    def __init__(self, location='/var/lib/tensor/cache'):
        self.store = {}
        self.location = location
        self.mtime = 0
        self._read()

    def _changed(self):
        if os.path.exists(self.location):
            mtime = os.stat(self.location).st_mtime

            return self.mtime != mtime
        else:
            return False

    def _acquire_cache(self):
        try:
            cache_file = open(self.location, 'r')
        except IOError:
            return {}

        cache = json.loads(cache_file.read())
        cache_file.close()
        return cache

    def _write_cache(self, d):
        cache_file = open(self.location, 'w')
        cache_file.write(json.dumps(d))
        cache_file.close()

    def _persist(self):
        cache = self._acquire_cache()

        for k, v in self.store.items():
            cache[k] = v

        self._write_cache(cache)

    def _read(self):
        cache = self._acquire_cache()
        for k, v in cache.items():
            self.store[k] = v

    def _remove_key(self, k):
        cache = self._acquire_cache()
        if k in cache:
            if k in cache:
                del cache[k]
            if k in self.store:
                del self.store[k]
            self._write_cache(cache)

    def expire(self, age):
        """Expire any items in the cache older than `age` seconds"""
        now = time.time()
        cache = self._acquire_cache()

        expired = [k for k, v in cache.items() if (now - v[0]) > age]

        for k in expired:
            if k in cache:
                del cache[k]
            if k in self.store:
                del self.store[k]

        self._write_cache(cache)

    def set(self, k, v):
        """Set a key `k` to value `v`"""
        self.store[k] = (time.time(), v)
        self._persist()

    def get(self, k):
        """Returns key contents, and modify time"""
        if self._changed():
            self._read()

        if k in self.store:
            return tuple(self.store[k])
        else:
            return None

    def contains(self, k):
        """Return True if key `k` exists"""
        if self._changed():
            self._read()
        return k in self.store.keys()

    def delete(self, k):
        """Remove key `k` from the cache"""
        self._remove_key(k)


"""
Copyright 2015-2016 Peter Urda

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

import requests
from requests.exceptions import RequestException

from nistbeacon.nistbeaconvalue import NistBeaconValue

class NistBeacon(object):
    """
    The NistBeacon object is used to query the NIST Randomness Beacon.
    For the most part, it returns actual NistBeaconValue objects for the
    consumer to use as they please.
    """

    _NIST_API_URL = "https://beacon.nist.gov/rest/record"

    # noinspection SpellCheckingInspection
    _INIT_RECORD = NistBeaconValue(
        version="Version 1.0",
        frequency=60,
        timestamp=1378395540,
        seed_value='87F49DB997D2EED0B4FDD93BAA4CDFCA49095AF98E54B81F2C39B5C400'
                   '2EC04B8C9E31FA537E64AC35FA2F038AA80730B054CFCF371AB5584CFB'
                   '4EFD293280EE',
        previous_output_value="00000000000000000000000000000000000000000000000"
                              "00000000000000000000000000000000000000000000000"
                              "0000000000000000000000000000000000",
        signature_value="F93BBE5714944F31983AE8187D5D94F87FFEC2F98185F9EB4FE5D"
                        "B61A9E5119FB0756E9AF4B7112DEBF541E9E53D05346B7358C12F"
                        "A43A8E0D695BFFAF193B1C3FFC4AE7BCF6651812B6D60190DB8FF"
                        "23C9364374737F45F1A89F22E1E492B0F373E4DB523274E9D31C8"
                        "6987C64A26F507008828A358B0E166A197D433597480895E9298C"
                        "60D079673879C3C1AEDA6306C3201991D0A6778B21462BDEBB8D3"
                        "776CD3D0FA0325AFE99B2D88A7C357E62170320EFB51F9749B5C7"
                        "B9E7347178AB051BDD097B226664A2D64AF1557BB31540601849F"
                        "0BE3AAF31D7A25E2B358EEF5A346937ADE34A110722DA8C037F97"
                        "3350B3846DCAB16C9AA125F2027C246FDB3",
        output_value="17070B49DBF3BA12BEA427CB6651ECF7860FDC3792268031B77711D6"
                     "3A8610F4CDA551B7FB331103889A62E2CB23C0F85362BBA49B9E0086"
                     "D1DA0830E4389AB1",
        status_code="1",
    )

    @classmethod
    def _query_nist(cls, url_data: str) -> NistBeaconValue:
        try:
            nist_response = requests.get(
                "{0}/{1}".format(
                    cls._NIST_API_URL,
                    url_data,
                )
            )

            if (
                    isinstance(nist_response, requests.Response) and
                    nist_response.status_code is requests.codes.OK
            ):
                return NistBeaconValue.from_xml(nist_response.text)
            else:
                return None
        except RequestException:
            return None

    @classmethod
    def chain_check(cls, timestamp: int) -> bool:
        """
        Given a record timestamp, verify the chain integrity.

        :param timestamp: UNIX time / POSIX time / Epoch time
        :return: 'True' if the timestamp fits the chain. 'False' otherwise.
        """

        # Creation is messy.
        # You want genius, you get madness; two sides of the same coin.
        # ... I'm sure this can be cleaned up. However, let's test it first.

        record = cls.get_record(timestamp)

        if isinstance(record, NistBeaconValue) is False:
            # Don't you dare try to play me
            return False

        prev_record = cls.get_previous(record.timestamp)
        next_record = cls.get_next(record.timestamp)

        if prev_record is None and next_record is None:
            # Uh, how did you manage to do this?
            # I'm not even mad, that's amazing.
            return False

        if (
                isinstance(prev_record, NistBeaconValue) and
                isinstance(next_record, NistBeaconValue)
        ):
            # Majority case, somewhere in the middle of the chain
            # True if:
            #   - All three records have proper signatures
            #   - The requested record's previous output equals previous
            #   - The next possible record's previous output equals the record
            return (
                record.valid_signature and
                prev_record.valid_signature and
                next_record.valid_signature and
                record.previous_output_value == prev_record.output_value and
                next_record.previous_output_value == record.output_value
            )

        if (
                prev_record is None and
                isinstance(next_record, NistBeaconValue)
        ):
            # Edge case, this was potentially the first record of all time
            return (
                record.valid_signature and
                next_record.valid_signature and
                cls._INIT_RECORD == record and
                next_record.previous_output_value == record.output_value
            )

        if (
                isinstance(prev_record, NistBeaconValue) and
                next_record is None
        ):
            # Edge case, this was potentially the latest and greatest
            return (
                record.valid_signature and
                prev_record.valid_signature and
                record.previous_output_value == prev_record.output_value
            )

    @classmethod
    def get_first_record(
            cls,
            download: bool=True
    ) -> NistBeaconValue:
        """
        Get the first (oldest) record available. Since the first record
        IS a known value in the system we can load it from constants.

        :param download: 'True' will always reach out to NIST to get the
                         first record. 'False' returns a local copy.
        :return: The first beacon value. 'None' otherwise.
        """

        if download:
            return NistBeacon.get_record(cls._INIT_RECORD.timestamp)
        else:
            return NistBeaconValue.from_json(cls._INIT_RECORD.json)

    @classmethod
    def get_last_record(cls) -> NistBeaconValue:
        """
        Get the last (newest) record available.

        :return: The last beacon value. 'None' otherwise.
        """

        return cls._query_nist("last")

    @classmethod
    def get_next(cls, timestamp: int) -> NistBeaconValue:
        """
        Given a record timestamp, get the next record available.

        :param timestamp: UNIX time / POSIX time / Epoch time
        :return: The next beacon value if available. 'None' otherwise.
        """

        return cls._query_nist("next/{}".format(timestamp))

    @classmethod
    def get_previous(cls, timestamp: int) -> NistBeaconValue:
        """
        Given a record timestamp, get the previous record available.

        :param timestamp: UNIX time / POSIX time / Epoch time
        :return: The previous beacon value if available. 'None; otherwise
        """

        return cls._query_nist("previous/{}".format(timestamp))

    @classmethod
    def get_record(cls, timestamp: int) -> NistBeaconValue:
        """
        Get a specific record (or next closest)

        :param timestamp: UNIX time / POSIX time / Epoch time
        :return: The requested beacon value if available. 'None' otherwise.
        """

        return cls._query_nist(str(timestamp))


"""
Copyright 2015-2017 Peter Urda

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

import binascii
import hashlib
import json
from random import Random
from xml.etree import ElementTree

from nistbeacon.nistbeaconcrypto import NistBeaconCrypto

class NistBeaconValue(object):
    """
    A single NIST Beacon Value object represents one beacon value.
    It has all the normal properties of a NIST beacon API call,
    but stored as a python object
    """

    _xml_template = (
        '<?xml version="1.0" encoding="UTF-8" standalone="yes"?>'
        '<record xmlns="http://beacon.nist.gov/record/0.1/">'
        '<version>{0}</version>'
        '<frequency>{1}</frequency>'
        '<timeStamp>{2}</timeStamp>'
        '<seedValue>{3}</seedValue>'
        '<previousOutputValue>{4}</previousOutputValue>'
        '<signatureValue>{5}</signatureValue>'
        '<outputValue>{6}</outputValue>'
        '<statusCode>{7}</statusCode>'
        '</record>'
    )

    _KEY_FREQUENCY = 'frequency'
    _KEY_OUTPUT_VALUE = 'outputValue'
    _KEY_PREVIOUS_OUTPUT_VALUE = 'previousOutputValue'
    _KEY_SEED_VALUE = 'seedValue'
    _KEY_SIGNATURE_VALUE = 'signatureValue'
    _KEY_STATUS_CODE = 'statusCode'
    _KEY_TIMESTAMP = 'timeStamp'
    _KEY_VERSION = 'version'

    def __init__(
            self,
            version: str,
            frequency: int,
            timestamp: int,
            seed_value: str,
            previous_output_value: str,
            signature_value: str,
            output_value: str,
            status_code: str,
    ):
        """
        :param version:
            Reported NIST randomness beacon version

        :param frequency:
            The time interval, in seconds, between expected records

        :param timestamp:
            The time the seed value was generated as the number of
            seconds since January 1, 1970

        :param seed_value:
            A seed value represented as a 64 byte (512-bit) hex string
            value

        :param previous_output_value:
            The SHA-512 hash value for the previous record - 64 byte hex
            string

        :param signature_value:
            A digital signature (RSA) computed over (in order): version,
            frequency, timeStamp, seedValue, previousHashValue, statusCode

            Note: Except for version, the hash is on the byte
            representations and not the string representations of the data
            values

        :param output_value:
            The SHA-512 hash of the signatureValue as a 64 byte hex string

        :param status_code:
            The status code value:
                0 - Chain intact, values all good
                1 - Start of a new chain of values, previous hash value
                    will be all zeroes
                2 - Time between values is greater than the frequency, but
                    the chain is still intact
        """

        self._version = version
        self._frequency = frequency
        self._timestamp = timestamp
        self._seed_value = seed_value
        self._previous_output_value = previous_output_value
        self._signature_value = signature_value
        self._output_value = output_value
        self._status_code = status_code

        # Compute JSON, XML strings
        self._json = json.dumps(
            {
                self._KEY_VERSION: self.version,
                self._KEY_FREQUENCY: self.frequency,
                self._KEY_TIMESTAMP: self.timestamp,
                self._KEY_SEED_VALUE: self.seed_value,
                self._KEY_PREVIOUS_OUTPUT_VALUE: self.previous_output_value,
                self._KEY_SIGNATURE_VALUE: self.signature_value,
                self._KEY_OUTPUT_VALUE: self.output_value,
                self._KEY_STATUS_CODE: self.status_code,
            },
            sort_keys=True,
        )

        self._xml = self._xml_template.format(
            self.version,
            self.frequency,
            self.timestamp,
            self.seed_value,
            self.previous_output_value,
            self.signature_value,
            self.output_value,
            self.status_code,
        )

        # Signature checking
        sha512_hash = NistBeaconCrypto.get_hash(
            self.version,
            self.frequency,
            self.timestamp,
            self.seed_value,
            self.previous_output_value,
            self.status_code,
        )

        sig_check_result = NistBeaconCrypto.verify(
            timestamp=self.timestamp,
            message_hash=sha512_hash,
            signature=binascii.a2b_hex(self.signature_value)[::-1],
        )

        # The signature sha512'd again should equal the output value
        expected_signature = hashlib.sha512(
            binascii.a2b_hex(self.signature_value)
        ).hexdigest().upper()

        sig_hash_check = (expected_signature == self.output_value)

        # Store the valid signature state after computation
        self._valid_signature = sig_check_result and sig_hash_check

        # Using the output value from the beacon, let's seed a personal
        # python random.Random object
        self._pseudo_random = Random(self.output_value)

    def __eq__(self, other):
        try:
            return self.version == other.version \
                   and self.frequency == other.frequency \
                   and self.timestamp == other.timestamp \
                   and self.seed_value == other.seed_value \
                   and self.previous_output_value == \
                   other.previous_output_value \
                   and self.signature_value == other.signature_value \
                   and self.output_value == other.output_value \
                   and self.status_code == other.status_code \
                   and self.valid_signature == other.valid_signature
        except (AttributeError, TypeError):
            return False

    def __ne__(self, other):
        return not self.__eq__(other)

    @property
    def frequency(self) -> int:
        """
        :return: The time interval, in seconds, between expected records
        """

        return self._frequency

    @property
    def json(self) -> str:
        """
        :return: The JSON representation of the beacon, as a string
        """

        return self._json

    @property
    def output_value(self) -> str:
        """
        :return: The SHA-512 hash of the signatureValue as a 64 byte hex string
        """

        return self._output_value

    @property
    def previous_output_value(self) -> str:
        """
        :return:
            The SHA-512 hash value for the previous record - 64 byte hex
            string
        """

        return self._previous_output_value

    @property
    def pseudo_random(self) -> Random:
        """
        :return:
            A python `random.Random` object that has been seeded with
            the value's `output_value`. This is a pseudo-random
            number generator
        """

        return self._pseudo_random

    @property
    def seed_value(self) -> str:
        """
        :return:
            A seed value represented as a 64 byte (512-bit) hex string
            value
        """

        return self._seed_value

    @property
    def signature_value(self) -> str:
        """
        :return:
            A digital signature (RSA) computed over (in order): version,
            frequency, timeStamp, seedValue, previousHashValue, statusCode

            Note: Except for version, the hash is on the byte
            representations and not the string representations of the data
            values
        """

        return self._signature_value

    @property
    def status_code(self) -> str:
        """
        :return:
            The status code value:
                0 - Chain intact, values all good
                1 - Start of a new chain of values, previous hash value
                    will be all zeroes
                2 - Time between values is greater than the frequency, but
                    the chain is still intact
        """

        return self._status_code

    @property
    def timestamp(self) -> int:
        """
        :return:
            The time the seed value was generated as the number of
            seconds since January 1, 1970
        """

        return self._timestamp

    @property
    def valid_signature(self) -> bool:
        """
        Shows the result of signature verification

        First, required records (version, frequency, timestamp,
        seed_value, previous_output_value) are packed together to form
        a message. This message is then checked against the record's reported
        signature field WITH the known NIST public key.

        Second, the signature value is independently ran through a SHA512
        hash. The result of this operation SHOULD equal the record's reported
        output_value field.

        As long as the result of the 'First' step and'ed with the 'Second'
        step, the record is considered valid.

        :return: 'True' if this record is valid. 'False' otherwise
        """

        return self._valid_signature

    @property
    def version(self) -> str:
        """
        :return: Reported NIST randomness beacon version
        """

        return self._version

    @property
    def xml(self) -> str:
        """
        :return: The XML representation of the beacon, as a string
        """

        return self._xml

    @classmethod
    def from_json(cls, input_json: str) -> 'NistBeaconValue':
        """
        Convert a string of JSON which represents a NIST randomness beacon
        value into a 'NistBeaconValue' object.

        :param input_json: JSON to build a 'Nist RandomnessBeaconValue' from
        :return: A 'NistBeaconValue' object, 'None' otherwise
        """

        try:
            data_dict = json.loads(input_json)
        except ValueError:
            return None

        # Our required values are "must haves". This makes it simple
        # to verify we loaded everything out of JSON correctly.
        required_values = {
            cls._KEY_FREQUENCY: None,
            cls._KEY_OUTPUT_VALUE: None,
            cls._KEY_PREVIOUS_OUTPUT_VALUE: None,
            cls._KEY_SEED_VALUE: None,
            cls._KEY_SIGNATURE_VALUE: None,
            cls._KEY_STATUS_CODE: None,
            cls._KEY_TIMESTAMP: None,
            cls._KEY_VERSION: None,
        }

        for key in required_values:
            if key in data_dict:
                required_values[key] = data_dict[key]

        # Confirm that the required values are set, and not 'None'
        if None in required_values.values():
            return None

        # We have all the required values, return a node object
        return cls(
            version=required_values[cls._KEY_VERSION],
            frequency=int(required_values[cls._KEY_FREQUENCY]),
            timestamp=int(required_values[cls._KEY_TIMESTAMP]),
            seed_value=required_values[cls._KEY_SEED_VALUE],
            previous_output_value=required_values[
                cls._KEY_PREVIOUS_OUTPUT_VALUE
            ],
            signature_value=required_values[cls._KEY_SIGNATURE_VALUE],
            output_value=required_values[cls._KEY_OUTPUT_VALUE],
            status_code=required_values[cls._KEY_STATUS_CODE],
        )

    @classmethod
    def from_xml(cls, input_xml: str) -> 'NistBeaconValue':
        """
        Convert a string of XML which represents a NIST Randomness Beacon value
        into a 'NistBeaconValue' object.

        :param input_xml: XML to build a 'NistBeaconValue' from
        :return: A 'NistBeaconValue' object, 'None' otherwise
        """

        invalid_result = None

        understood_namespaces = {
            'nist-0.1': 'http://beacon.nist.gov/record/0.1/',
        }

        # Our required values are "must haves". This makes it simple
        # to verify we loaded everything out of XML correctly.
        required_values = {
            cls._KEY_FREQUENCY: None,
            cls._KEY_OUTPUT_VALUE: None,
            cls._KEY_PREVIOUS_OUTPUT_VALUE: None,
            cls._KEY_SEED_VALUE: None,
            cls._KEY_SIGNATURE_VALUE: None,
            cls._KEY_STATUS_CODE: None,
            cls._KEY_TIMESTAMP: None,
            cls._KEY_VERSION: None,
        }

        # First attempt to load the xml, return 'None' on ParseError
        try:
            tree = ElementTree.ElementTree(ElementTree.fromstring(input_xml))
        except ElementTree.ParseError:
            return invalid_result

        # Using the required values, let's load the xml values in
        for key in required_values:
            discovered_element = tree.find(
                "{0}:{1}".format('nist-0.1', key),
                namespaces=understood_namespaces,
            )

            if not isinstance(discovered_element, ElementTree.Element):
                continue

            # Bad pylint message - https://github.com/PyCQA/pylint/issues/476
            # pylint: disable=no-member
            required_values[key] = discovered_element.text

        # Confirm that the required values are set, and not 'None'
        if None in required_values.values():
            return invalid_result

        # We have all the required values, return a node object
        return cls(
            version=required_values[cls._KEY_VERSION],
            frequency=int(required_values[cls._KEY_FREQUENCY]),
            timestamp=int(required_values[cls._KEY_TIMESTAMP]),
            seed_value=required_values[cls._KEY_SEED_VALUE],
            previous_output_value=required_values[
                cls._KEY_PREVIOUS_OUTPUT_VALUE
            ],
            signature_value=required_values[cls._KEY_SIGNATURE_VALUE],
            output_value=required_values[cls._KEY_OUTPUT_VALUE],
            status_code=required_values[cls._KEY_STATUS_CODE],
        )


import os
import re
import warnings

from datetime import datetime, timedelta
from urlparse import urlparse

import jsmin

import django

from django.conf import settings
from django.core.urlresolvers import reverse_lazy
from django.http.response import (
    HttpResponseForbidden,
    HttpResponseBadRequest,
    HttpResponse
)
from django.template.response import TemplateResponse
from django.views.generic.base import View, TemplateView

from . import models

TRACKING_PIXEL_PATH = os.path.join(os.path.dirname(__file__), 'templates')

class ValidatedViewMixin(object):

    parsed_url = None
    client = None
    client_ip = None

    def is_valid(self, request):
        self.parsed_url = urlparse(request.META.get('HTTP_REFERER', ''))
        origin = self.parsed_url.hostname or ''
        client_id = request.GET.get('dja_id')

        try:
            self.client = models.Client.objects.get(uuid=client_id)
        except models.Client.DoesNotExist:
            return HttpResponseForbidden(content='Client not found')

        if not self.client.domain_set.exists():
            return HttpResponseBadRequest(content='No domains found for client')

        domain_found = self.client.domain_set.filter(name=origin).exists()
        if not domain_found:
            warnings.warn(
                "Use of domain patterns will be removed in a future version.",
                DeprecationWarning
            )
            for domain in self.client.domain_set.all():
                if (
                    domain.pattern and
                    re.match('.*%s$' % domain.pattern, origin, re.IGNORECASE)
                ):
                    domain_found = True
                    break
        if not domain_found:
            return HttpResponseForbidden(content='Invalid domain for client')

        if settings.USE_X_FORWARDED_HOST:
            self.client_ip = request.META.get(
                'HTTP_X_FORWARDED_FOR',
                request.META.get('REMOTE_ADDR', '')
            )
        else:
            self.client_ip = request.META.get('REMOTE_ADDR')

class CaptureEventView(View, ValidatedViewMixin):

    def get(self, request):
        invalid_response = self.is_valid(request)

        if invalid_response:
            return invalid_response

        if not self.client.path_valid(
            request.GET.get('pth', '')
        ) or not self.client.ip_valid(self.client_ip):
            return HttpResponse(status=204) # NO_CONTENT

        tracking_id = request.GET.get(
            'dti', request.session.get('dja_tracking_id')
        )
        user_id = request.GET.get(
            'du', request.COOKIES.get('dja_uuid')
        )
        data = {
            'domain': self.parsed_url.netloc,
            'protocol': self.parsed_url.scheme,
            'client': self.client,
            'ip_address': self.client_ip,
            'user_agent': request.META.get('HTTP_USER_AGENT', 'None'),
            'path': request.GET.get('pth', ''),
            'query_string': request.GET.get('qs', ''),
            'referrer': request.GET.get('rf', '')[:2083],
            'screen_width': request.GET.get('sw'),
            'screen_height': request.GET.get('sh'),
        }
        status = 201 # CREATED
        if tracking_id:
            data['tracking_key'] = tracking_id
            status = 202 # ACCEPTED
        if user_id:
            data['tracking_user_id'] = user_id
            status = 202 # ACCEPTED
        new_event = models.RequestEvent.objects.create(**data)
        if not tracking_id:
            request.session['dja_tracking_id'] = new_event.tracking_key
        img_data = file(os.path.join(TRACKING_PIXEL_PATH, 'tracking_pixel.png')).read()
        response = HttpResponse(content=img_data, status=status, content_type='image/png')
        response.set_cookie('dja_uuid', new_event.tracking_user_id,
                            expires=datetime.now() + timedelta(days=365))
        return response

capture_event = CaptureEventView.as_view()

class MinifiedJsTemplateResponse(TemplateResponse):

    @property
    def rendered_content(self):
        """Returns a 'minified' version of the javascript content"""
        template = self.resolve_template(self.template_name)
        if django.VERSION[1] < 8:
            if template.name.endswith('.min'):
                return super(MinifiedJsTemplateResponse, self).rendered_content
        else:
            if template.template.name.endswith('.min'):
                return super(MinifiedJsTemplateResponse, self).rendered_content
        # if no minified template exists, minify the response
        content = super(MinifiedJsTemplateResponse, self).rendered_content
        content = jsmin.jsmin(content)
        return content

class DjanalyticsJs(TemplateView):

    response_class = MinifiedJsTemplateResponse
    content_type = 'application/javascript'

    def get_template_names(self):
        return ['djanalytics.js.min', 'djanalytics.js']

    def get_context_data(self, **kwargs):
        parsed_url = urlparse(self.request.META.get('HTTP_REFERER', ''))
        client_id = self.request.GET.get('dja_id')
        context_data = super(DjanalyticsJs, self).get_context_data(**kwargs)
        tl_domain = '.'.join(parsed_url.netloc.split('.')[-2:])
        # the js will check for existing cookie values for uuid and tracking_id
        # and will only use the generated values below if cookie values
        # don't exist
        context_data.update(
            {
                'uuid': models.generate_uuid(),
                'tracking_id': models.generate_uuid(),
                'domain': '.%s' % tl_domain,
                'dja_id': client_id,
                'capture_img_url': '%s%s' % (
                    self.request.META.get('HTTP_HOST', ''),
                    reverse_lazy('dja_capture')
                ),
            }
        )
        return context_data

djanalytics_js = DjanalyticsJs.as_view()


import os

class LogFollower(object):
    """Provides a class for following log files between runs

    :param logfile: Full path to logfile
    :type logfile: str
    :param parser: Optional parser method for log lines
    :type parser: str
    """

    def __init__(self, logfile, parser=None, tmp_path="/var/lib/tensor/", history=False):
        self.logfile = logfile
        self.tmp = os.path.join(tmp_path,
            '%s.lf' % self.logfile.lstrip('/').replace('/','-'))

        self.history = history

        self.readLast()

        self.parser = parser

    def cleanStore(self):
        os.unlink(self.tmp)

    def storeLast(self):
        fi = open(self.tmp, 'wt')
        fi.write('%s:%s' % (self.lastSize, self.lastInode))
        fi.close()

    def readLast(self):
        if os.path.exists(self.tmp):
            fi = open(self.tmp, 'rt')
            ls, li = fi.read().split(':')
            self.lastSize = int(ls)
            self.lastInode = int(li)
        else:
            if self.history:
                self.lastSize = 0
                self.lastInode = 0
            else:
                # Don't re-read the entire file
                stat = os.stat(self.logfile)
                self.lastSize = stat.st_size
                self.lastInode = stat.st_ino

    def get_fn(self, fn, max_lines=None):
        """Passes each parsed log line to `fn`
        This is a better idea than storing a giant log file in memory
        """
        stat = os.stat(self.logfile)

        if (stat.st_ino == self.lastInode) and (stat.st_size == self.lastSize):
            # Nothing new
            return []

        # Handle rollover and rotations vaguely
        if (stat.st_ino != self.lastInode) or (stat.st_size < self.lastSize):
            self.lastSize = 0

        fi = open(self.logfile, 'rt')
        fi.seek(self.lastSize)

        self.lastInode = stat.st_ino

        lines = 0

        for i in fi:
            lines += 1
            if max_lines and (lines > max_lines):
                self.storeLast()
                fi.close()
                return

            if '\n' in i:
                self.lastSize += len(i)
                if self.parser:
                    line = self.parser(i.strip('\n'))
                else:
                    line = i.strip('\n')

                fn(line)

        self.storeLast()

        fi.close()

    def get(self, max_lines=None):
        """Returns a big list of all log lines since the last run
        """
        rows = []

        self.get_fn(lambda row: rows.append(row), max_lines=max_lines)

        return rows


# -*- coding: utf-8 -*-
#
# This file is part of Zenodo.
# Copyright (C) 2015, 2016 CERN.
#
# Zenodo is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Zenodo is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Zenodo. If not, see <http://www.gnu.org/licenses/>.
#
# In applying this licence, CERN does not waive the privileges and immunities
# granted to it by virtue of its status as an Intergovernmental Organization
# or submit itself to any jurisdiction.

"""Token serializers."""

from __future__ import absolute_import, print_function

import binascii
import os
from base64 import urlsafe_b64encode
from datetime import datetime

from flask import current_app
from itsdangerous import BadData, JSONWebSignatureSerializer, \
    SignatureExpired, TimedJSONWebSignatureSerializer

SUPPORTED_DIGEST_ALGORITHMS = ('HS256', 'HS512')

class TokenMixin(object):
    """Mix-in class for token serializers."""

    def create_token(self, obj_id, extra_data):
        """Create a token referencing the object id with extra data.

        Note random data is added to ensure that no two tokens are identical.
        """
        return self.dumps(
            dict(
                id=obj_id,
                data=extra_data,
                rnd=binascii.hexlify(os.urandom(4)).decode('utf-8')
            )
        )

    def validate_token(self, token, expected_data=None):
        """Validate secret link token.

        :param token: Token value.
        :param expected_data: A dictionary of key/values that must be present
            in the data part of the token (i.e. included via ``extra_data`` in
            ``create_token``).
        """
        try:
            # Load token and remove random data.
            data = self.load_token(token)

            # Compare expected data with data in token.
            if expected_data:
                for k in expected_data:
                    if expected_data[k] != data["data"].get(k):
                        return None
            return data
        except BadData:
            return None

    def load_token(self, token, force=False):
        """Load data in a token.

        :param token: Token to load.
        :param force: Load token data even if signature expired.
                      Default: False.
        """
        try:
            data = self.loads(token)
        except SignatureExpired as e:
            if not force:
                raise
            data = e.payload

        del data["rnd"]
        return data

class EncryptedTokenMixIn(TokenMixin):
    """Mix-in class for token serializers that generate encrypted tokens."""

    @property
    def engine(self):
        """Get cryptographic engine."""
        if not hasattr(self, '_engine'):
            from cryptography.fernet import Fernet
            from cryptography.hazmat.backends import default_backend
            from cryptography.hazmat.primitives import hashes

            digest = hashes.Hash(hashes.SHA256(), backend=default_backend())
            digest.update(current_app.config['SECRET_KEY'].encode('utf8'))
            fernet_key = urlsafe_b64encode(digest.finalize())
            self._engine = Fernet(fernet_key)
        return self._engine

    def create_token(self, obj_id, extra_data):
        """Create a token referencing the object id with extra data."""
        return self.engine.encrypt(
            super(EncryptedTokenMixIn, self).create_token(obj_id, extra_data)
        )

    def load_token(self, token, force=False):
        """Load data in a token.

        :param token: Token to load.
        :param force: Load token data even if signature expired.
                      Default: False.
        """
        return super(EncryptedTokenMixIn, self).load_token(
            self.engine.decrypt(token), force=force
        )

class EmailConfirmationSerializer(TimedJSONWebSignatureSerializer, TokenMixin):
    """Serializer for email confirmation link tokens.

    Depends upon the secrecy of ``SECRET_KEY``. Tokens expire after a specific
    time (defaults to ``ACCESSREQUESTS_CONFIRMLINK_EXPIRES_IN``). The
    access request id as well as the email address is stored in the token
    together with a random bit to ensure all tokens are unique.
    """

    def __init__(self, expires_in=None, **kwargs):
        """Initialize underlying TimedJSONWebSignatureSerializer."""
        dt = expires_in or \
            current_app.config['ACCESSREQUESTS_CONFIRMLINK_EXPIRES_IN']

        super(EmailConfirmationSerializer, self).__init__(
            current_app.config['SECRET_KEY'],
            expires_in=dt,
            salt='accessrequests-email',
            **kwargs
        )

    @classmethod
    def compat_validate_token(cls, *args, **kwargs):
        """Multiple algorithm-compatible token validation."""
        data = None
        for algorithm in SUPPORTED_DIGEST_ALGORITHMS:
            data = cls(algorithm_name=algorithm).validate_token(
                *args, **kwargs)
            if not data:  # move to next algorithm
                continue
        return data

class SecretLinkSerializer(JSONWebSignatureSerializer, TokenMixin):
    """Serializer for secret links."""

    def __init__(self, **kwargs):
        """Initialize underlying JSONWebSignatureSerializer."""
        super(SecretLinkSerializer, self).__init__(
            current_app.config['SECRET_KEY'],
            salt='accessrequests-link',
            **kwargs
        )

class TimedSecretLinkSerializer(TimedJSONWebSignatureSerializer,
                                TokenMixin):
    """Serializer for expiring secret links."""

    def __init__(self, expires_at=None, **kwargs):
        """Initialize underlying TimedJSONWebSignatureSerializer."""
        assert isinstance(expires_at, datetime) or expires_at is None

        dt = expires_at - datetime.now() if expires_at else None

        super(TimedSecretLinkSerializer, self).__init__(
            current_app.config['SECRET_KEY'],
            expires_in=int(dt.total_seconds()) if dt else None,
            salt='accessrequests-timedlink',
            **kwargs
        )

class SecretLinkFactory(object):
    """Functions for creating and validating any secret link tokens."""

    @classmethod
    def create_token(cls, obj_id, data, expires_at=None):
        """Create the secret link token."""
        if expires_at:
            s = TimedSecretLinkSerializer(expires_at=expires_at)
        else:
            s = SecretLinkSerializer()

        return s.create_token(obj_id, data)

    @classmethod
    def validate_token(cls, token, expected_data=None):
        """Validate a secret link token (non-expiring + expiring)."""
        for algorithm in SUPPORTED_DIGEST_ALGORITHMS:
            s = SecretLinkSerializer(algorithm_name=algorithm)
            st = TimedSecretLinkSerializer(algorithm_name=algorithm)

            try:
                for serializer in (s, st):
                    data = serializer.validate_token(
                        token, expected_data=expected_data)
                    if data:
                        return data
            except SignatureExpired:  # move to next algorithm
                raise
            except BadData:
                continue  # move to next serializer/algorithm

    @classmethod
    def load_token(cls, token, force=False):
        """Validate a secret link token (non-expiring + expiring)."""
        for algorithm in SUPPORTED_DIGEST_ALGORITHMS:
            s = SecretLinkSerializer(algorithm_name=algorithm)
            st = TimedSecretLinkSerializer(algorithm_name=algorithm)
            for serializer in (s, st):
                try:
                    data = serializer.load_token(token, force=force)
                    if data:
                        return data
                except SignatureExpired:
                    raise  # signature was parsed and is expired
                except BadData:
                    continue  # move to next serializer/algorithm


def Counter32(a, b, delta):
    """32bit counter aggregator with wrapping
    """
    if b < a:
        c = 4294967295 - a
        return (c + b) / float(delta)

    return (b - a) / float(delta)

def Counter64(a, b, delta):
    """64bit counter aggregator with wrapping
    """
    if b < a:
        c = 18446744073709551615 - a
        return (c + b) / float(delta)

    return (b - a) / float(delta)

def Counter(a, b, delta):
    """Counter derivative
    """
    if b < a:
        return None

    return (b - a) / float(delta)


from datetime import timedelta
from decimal import Decimal

def average(numerator, denominator):
    """ Method to calculate and format an average safely """
    if not denominator:
        return "0.00"
    return "%.2f" % (numerator / Decimal(denominator))

def average_duration(total_duration, visits):
    """ Method to calculate and format an average duration safely """
    if not visits:
        seconds = 0
    else:
        seconds = int(round(total_duration / Decimal(visits)))
    duration = timedelta(seconds=seconds)
    return str(duration)

def percentage(numerator, denominator):
    """ Method to calculate and format a rate (%) safely """
    if not denominator:
        return "0.00%"
    val = (numerator / Decimal(denominator)) * 100
    return "%.2f%%" % val

def format_date(date_obj):
    return date_obj.strftime('%m/%d/%y')


import time
import sys
import os
import importlib
import re
import copy

import yaml

from twisted.application import service
from twisted.internet import task, reactor, defer
from twisted.python import log

from tensor.protocol import riemann

class TensorService(service.Service):
    """ Tensor service

    Runs timers, configures sources and and manages the queue
    """
    def __init__(self, config):
        self.running = 0
        self.sources = []
        self.lastEvents = {}
        self.outputs = {}

        self.evCache = {}
        self.critical = {}
        self.warn = {}

        self.hostConnectorCache = {}

        self.eventCounter = 0

        self.factory = None
        self.protocol = None
        self.watchdog = None

        self.config = config

        both = lambda i1, i2, t: isinstance(i1, t) and isinstance(i2, t)

        if os.path.exists('/var/lib/tensor'):
            sys.path.append('/var/lib/tensor')

        if 'include_path' in config:
            ipath = config['include_path']
            if os.path.exists(ipath):
                files = [
                    os.path.join(ipath, f) for f in os.listdir(ipath)
                        if f[-4:] == '.yml'
                ]

                for f in files:
                    conf = yaml.load(open(f, 'rt'))
                    for k,v in conf.items():
                        if k in self.config:
                            if both(v, self.config[k], dict):
                                # Merge dicts
                                for k2, v2 in v.items():
                                    self.config[k][j2] = v2

                            elif both(v, self.config[k], list):
                                # Extend lists
                                self.config[k].extend(v)
                            else:
                                # Overwrite
                                self.config[k] = v
                        else:
                            self.config[k] = v
                    log.msg('Loadded additional configuration from %s' % f)
            else:
                log.msg('Config Error: include_path %s does not exist' % ipath)

        # Read some config stuff
        self.debug = float(self.config.get('debug', False))
        self.ttl = float(self.config.get('ttl', 60.0))
        self.stagger = float(self.config.get('stagger', 0.2))

        # Backward compatibility
        self.server = self.config.get('server', 'localhost')
        self.port = int(self.config.get('port', 5555))
        self.proto = self.config.get('proto', 'tcp')
        self.inter = self.config.get('interval', 60.0)

        if self.debug:
            print("config:", repr(config))

        self.setupSources(self.config)

    def setupOutputs(self, config):
        """Setup output processors"""

        if self.proto == 'tcp':
            defaultOutput = {
                'output': 'tensor.outputs.riemann.RiemannTCP',
                'server': self.server,
                'port': self.port
            }
        else:
            defaultOutput = {
                'output': 'tensor.outputs.riemann.RiemannUDP',
                'server': self.server,
                'port': self.port
            }

        outputs = config.get('outputs', [defaultOutput])

        for output in outputs:
            if not ('debug' in output):
                output['debug'] = self.debug

            cl = output['output'].split('.')[-1]                # class
            path = '.'.join(output['output'].split('.')[:-1])   # import path

            # Import the module and construct the output object
            outputObj = getattr(
                importlib.import_module(path), cl)(output, self)

            name = output.get('name', None)

            # Add the output to our routing hash
            if name in self.outputs:
                self.outputs[name].append(outputObj)
            else:
                self.outputs[name] = [outputObj]

            # connect the output
            reactor.callLater(0, outputObj.createClient)

    def createSource(self, source):
        #
        if source.get('path'):
            path = source['path']
            if path not in sys.path:
                sys.path.append(path)

        # Resolve the source
        cl = source['source'].split('.')[-1]                # class
        path = '.'.join(source['source'].split('.')[:-1])   # import path

        # Import the module and get the object source we care about
        sourceObj = getattr(importlib.import_module(path), cl)

        if not ('debug' in source):
            source['debug'] = self.debug

        if not ('ttl' in source.keys()):
            source['ttl'] = self.ttl

        if not ('interval' in source.keys()):
            source['interval'] = self.inter

        return sourceObj(source, self.sendEvent, self)

    def setupTriggers(self, source, sobj):
        if source.get('critical'):
            self.critical[sobj] = [
                (re.compile(k), v) for k, v in source['critical'].items()
            ]

        if source.get('warning'):
            self.warn[sobj] = [
                (re.compile(k), v) for k, v in source['warning'].items()
            ]

    def setupSources(self, config):
        """Sets up source objects from the given config"""
        sources = config.get('sources', [])

        for source in sources:
            src = self.createSource(source)
            self.setupTriggers(source, src)

            self.sources.append(src)

    def _aggregateQueue(self, events):
        # Handle aggregation for each event
        queue = []
        for ev in events:
            if ev.aggregation:
                id = ev.id()
                thisM = ev.metric

                if id in self.evCache:
                    lastM, lastTime = self.evCache[id]
                    tDelta = ev.time - lastTime
                    m = ev.aggregation(
                        lastM, ev.metric, tDelta)
                    if m:
                        ev.metric = m
                        queue.append(ev)

                self.evCache[id] = (thisM, ev.time)
            else:
                queue.append(ev)

        return queue

    def setStates(self, source, queue):
        for ev in queue:
            if ev.state == 'ok':
                for k, v in self.warn.get(source, []):
                    if k.match(ev.service):
                        s = eval("service %s" % v, {'service': ev.metric})
                        if s:
                            ev.state = 'warning'

                for k, v in self.critical.get(source, []):
                    if k.match(ev.service):
                        s = eval("service %s" % v, {'service': ev.metric})
                        if s:
                            ev.state = 'critical'

    def routeEvent(self, source, events):
        routes = source.config.get('route', None)

        if not isinstance(routes, list):
            routes = [routes]

        for route in routes:
            if self.debug:
                log.msg("Sending events %s to %s" % (events, route))

            if not route in self.outputs:
                # Non existant route
                log.msg('Could not route %s -> %s.' % (
                    source.config['service'], route))
            else:
                for output in self.outputs[route]:
                   reactor.callLater(0, output.eventsReceived, events)

    def sendEvent(self, source, events):
        """Callback that all event sources call when they have a new event
        or list of events
        """

        if isinstance(events, list):
            self.eventCounter += len(events)
        else:
            self.eventCounter += 1
            events = [events]

        queue = self._aggregateQueue(events)

        if queue:
            if (source in self.critical) or (source in self.warn):
                self.setStates(source, queue)

            self.routeEvent(source, queue)

        queue = []

        self.lastEvents[source] = time.time()

    def _startSource(self, source):
        source.startTimer()

    @defer.inlineCallbacks
    def startService(self):
        yield self.setupOutputs(self.config)

        if self.debug:
            log.msg("Starting service")

        stagger = 0
        # Start sources internal timers
        for source in self.sources:
            if self.debug:
                log.msg("Starting source " + source.config['service'])
            # Stagger source timers, or use per-source start_delay
            start_delay = float(source.config.get('start_delay', stagger))
            reactor.callLater(start_delay, self._startSource, source)
            stagger += self.stagger

        reactor.callLater(stagger, self.startWatchdog)
        self.running = 1

    def startWatchdog(self):
        # Start source watchdog
        self.watchdog = task.LoopingCall(self.sourceWatchdog)
        self.watchdog.start(10)

    def sourceWatchdog(self):
        """Watchdog timer function.

        Recreates sources which have not generated events in 10*interval if
        they have watchdog set to true in their configuration
        """
        for i, source in enumerate(self.sources):
            if not source.config.get('watchdog', False):
                continue
            sn = repr(source)
            last = self.lastEvents.get(source, None)
            if last:
                try:
                    if last < (time.time()-(source.inter*10)):
                        log.msg("Trying to restart stale source %s: %ss" % (
                            sn, int(time.time() - last)
                        ))

                        s = self.sources.pop(i)
                        try:
                            s.t.stop()
                        except Exception as e:
                            log.msg("Could not stop timer for %s: %s" % (
                                sn, e))

                        config = copy.deepcopy(s.config)

                        del self.lastEvents[source]
                        del s, source

                        source = self.createSource(config)

                        reactor.callLater(0, self._startSource, source)
                except Exception as e:
                    log.msg("Could not reset source %s: %s" % (
                        sn, e))

    @defer.inlineCallbacks
    def stopService(self):
        self.running = 0

        if self.watchdog:
            self.watchdog.stop()

        for n, outputs in self.outputs.items():
            for output in outputs:
                yield defer.maybeDeferred(output.stop)