import re
from typing import Generic, Sequence, List, Union, Callable, Optional, Any
from types import MethodType

from . import options
from .state import Input, Output, Convert, Reader, StringReader, Result, Status, Success, Failure, Continue, Backtrack

class Parser(Generic[Input, Output]):
    """Abstract base class for all parser combinators.

    Inheritors of this class must:

    1. Implement the ``consume`` method
    2. Implement the ``__str__`` method
    3. Call super().__init__() in their constructor to get the parse method from
       the context.

    Attributes:
        protected (bool): The metaclasses set this flag to true whenever a
            parser is assigned to a name. Operators that flatten the parsers
            they receive (``__or__`` and ``__and__``) will not flatten parsers
            with a ``True`` value here. This is most important for ``__and__``
            because there is no other way to tell that these two should be
            different:

            ```
            abc = a & b & c  # returns [a, b, c]
            ```
            ```
            temp = a & b
            abc = temp & c  # returns [[a, b], c]
            ```

            The fundamental limitation is that python does not handle linked
            lists well or have unpacking that would let one unpack abc as
            [temp, c].
        name (Optional[str]): A name used by ``__str__`` and ``__repr__``.
            It is set by the context classes when a parser is assigned to a
            name.
    """

    def __init__(self):
        self.parse = MethodType(options.parse_method, self)

    def consume(self, reader: Reader[Input]) -> Status[Input, Output]:
        """Abstract method for matching this parser at the current location.

        This is the critical method of every parser combinator.

        Args:
            reader: The current state of the parser.

        Returns:
            If the pattern matches, a ``Continue`` is returned. If the pattern
            does not match, a ``Failure`` is returned. In either case, if
            multiple branches are explored, the error from the farthest point is
            merged with the returned status.
        """
        raise NotImplementedError()

    def parse(self, source: Sequence[Input]) -> Result[Output]:
        """Abstract method for completely parsing a source.

        While ``parse`` is a method on every parser for convenience, it
        is really a function of the context. It is the duty of the context
        to set the correct ``Reader`` to use and to handle whitespace
        not handled by the parsers themselves. This method is pulled from the
        context when the parser is initialized.

        Args:
            source: What will be parsed.

        Returns:
            If the parser succeeded in matching and consumed the entire output,
            the value from ``Continue`` is copied to make a ``Success``. If the
            parser failed in matching, the error message is copied to a
            ``Failure``. If the parser succeeded but the source was not
            completelt consumed, a ``Failure`` with a message indicating this
            is returned.
        """
        raise NotImplementedError()

    name = None  # type: Optional[str]

    protected = False  # type: bool

    def name_or_repr(self) -> str:
        if self.name is None:
            return self.__repr__()
        else:
            return self.name

    def name_or_nothing(self) -> Optional[str]:
        if self.name is None:
            return ''
        else:
            return self.name + ' = '

    @staticmethod
    def handle_other(obj: Any) -> 'Parser':
        if isinstance(obj, Parser):
            return obj
        else:
            return options.handle_literal(obj)

    def __or__(self, other) -> 'AlternativeParser':
        other = self.handle_other(other)
        parsers = []
        if isinstance(self, AlternativeParser) and not self.protected:
            parsers.extend(self.parsers)
        else:
            parsers.append(self)
        if isinstance(other, AlternativeParser) and not other.protected:
            parsers.extend(other.parsers)
        else:
            parsers.append(other)
        return AlternativeParser(*parsers)

    def __ror__(self, other) -> 'AlternativeParser':
        other = self.handle_other(other)
        return other.__or__(self)

    def __and__(self, other) -> 'SequentialParser':
        other = self.handle_other(other)
        if isinstance(self, SequentialParser) and not self.protected:
            return SequentialParser(*(self.parsers + (other,)))
        else:
            return SequentialParser(self, other)

    def __rand__(self, other) -> 'SequentialParser':
        other = self.handle_other(other)
        return other.__and__(self)

    def __rshift__(self, other) -> 'DiscardLeftParser':
        other = self.handle_other(other)
        return DiscardLeftParser(self, other)

    def __rrshift__(self, other) -> 'DiscardLeftParser':
        other = self.handle_other(other)
        return other.__rshift__(self)

    def __lshift__(self, other) -> 'DiscardRightParser':
        other = self.handle_other(other)
        return DiscardRightParser(self, other)

    def __rlshift__(self, other) -> 'DiscardRightParser':
        other = self.handle_other(other)
        return other.__lshift__(self)

    def __gt__(self, other) -> 'ConversionParser':
        return ConversionParser(self, other)

def completely_parse_reader(parser: Parser[Input, Output], reader: Reader[Input]) -> Result[Output]:
    """Consume reader and return Success only on complete consumption.

    This is a helper function for ``parse`` methods, which return ``Success``
    when the input is completely consumed and ``Failure`` with an appropriate
    message otherwise.

    Args:
        parser: The parser doing the consuming
        reader: The input being consumed

    Returns:
        A parsing ``Result``
    """
    result = (parser << eof).consume(reader)

    if isinstance(result, Continue):
        return Success(result.value)
    else:
        used = set()
        unique_expected = []
        for expected_lambda in result.expected:
            expected = expected_lambda()
            if expected not in used:
                used.add(expected)
                unique_expected.append(expected)

        return Failure(result.farthest.expected_error(' or '.join(unique_expected)))

class LiteralParser(Generic[Input], Parser[Input, Input]):
    def __init__(self, pattern: Sequence[Input]):
        super().__init__()
        self.pattern = pattern

    def consume(self, reader: Reader[Input]):
        remainder = reader
        for elem in self.pattern:
            if remainder.finished:
                return Backtrack(remainder, lambda: elem)
            elif elem == remainder.first:
                remainder = remainder.rest
            else:
                return Backtrack(remainder, lambda: elem)

        return Continue(remainder, self.pattern)

    def __repr__(self):
        return self.name_or_nothing() + repr(self.pattern)

class LiteralStringParser(Parser[str, str]):
    def __init__(self, pattern: str, whitespace: Parser[str, None] = None):
        super().__init__()
        self.whitespace = whitespace
        self.pattern = pattern

    def consume(self, reader: StringReader):
        if self.whitespace is not None:
            status = self.whitespace.consume(reader)
            reader = status.remainder

        if reader.source.startswith(self.pattern, reader.position):
            reader = reader.drop(len(self.pattern))

            if self.whitespace is not None:
                status = self.whitespace.consume(reader)
                reader = status.remainder

            return Continue(reader, self.pattern)
        else:
            return Backtrack(reader, lambda: repr(self.pattern))

    def __repr__(self):
        return self.name_or_nothing() + repr(self.pattern)

def lit(literal: Sequence[Input], *literals: Sequence[Sequence[Input]]) -> Parser:
    """Match a literal sequence.

    In the `TextParsers`` context, this matches the literal string
    provided. In the ``GeneralParsers`` context, this matches a sequence of
    input.

    If multiple literals are provided, they are treated as alternatives. e.g.
    ``lit('+', '-')`` is the same as ``lit('+') | lit('-')``.

    Args:
        literal: A literal to match
        *literals: Alternative literals to match

    Returns:
        A ``LiteralParser`` in the ``GeneralContext``, a ``LiteralStringParser``
        in the ``TextParsers`` context, and an ``AlternativeParser`` if multiple
        arguments are provided.
    """
    if len(literals) > 0:
        return AlternativeParser(options.handle_literal(literal), *map(options.handle_literal, literals))
    else:
        return options.handle_literal(literal)

class RegexParser(Parser[str, str]):
    def __init__(self, pattern: str, whitespace: Parser[str, None] = None):  # Python lacks type of compiled regex
        super().__init__()
        self.whitespace = whitespace
        self.pattern = re.compile(pattern)

    def consume(self, reader: StringReader):
        if self.whitespace is not None:
            status = self.whitespace.consume(reader)
            reader = status.remainder

        match = self.pattern.match(reader.source, reader.position)

        if match is None:
            return Backtrack(reader, lambda: "r'" + self.pattern.pattern + "'")
        else:
            value = reader.source[match.start():match.end()]
            reader = reader.drop(len(value))

            if self.whitespace is not None:
                status = self.whitespace.consume(reader)
                reader = status.remainder

            return Continue(reader, value)

    def __repr__(self):
        return "reg(r'{}')".format(self.pattern.pattern)

def reg(pattern: str) -> RegexParser:
    """Match with a regular expression.

    This matches the text with a regular expression. The regular expressions is
    treated as greedy. Backtracking in the parser combinators does not flow into
    regular expression backtracking. This is only valid in the ``TextParsers``
    context and not in the ``GeneralParsers`` context because regular
    expressions only operate on text.

    Args:
        pattern: str or python regular expression.
    """
    return RegexParser(pattern, options.whitespace)

class OptionalParser(Generic[Input, Output], Parser[Input, List[Output]]):
    def __init__(self, parser: Parser[Input, Output]):
        super().__init__()
        self.parser = parser

    def consume(self, reader: Reader[Input]) -> Status[Input, Sequence[Output]]:
        status = self.parser.consume(reader)

        if isinstance(status, Continue):
            return Continue(status.remainder, [status.value]).merge(status)
        else:
            return Continue(reader, []).merge(status)

    def __repr__(self):
        return self.name_or_nothing() + 'opt({})'.format(self.parser.name_or_repr())

def opt(parser: Union[Parser, Sequence[Input]]) -> OptionalParser:
    """Optionally match a parser.

    An ``OptionalParser`` attempts to match ``parser``. If it succeeds, it
    returns a list of length one with the value returned by the parser as the
    only element. If it fails, it returns an empty list.

    Args:
        parser: Parser or literal
    """
    if isinstance(parser, str):
        parser = lit(parser)
    return OptionalParser(parser)

class AlternativeParser(Generic[Input, Output], Parser[Input, Output]):
    def __init__(self, parser: Parser[Input, Output], *parsers: Sequence[Parser[Input, Output]]):
        super().__init__()
        self.parsers = (parser,) + tuple(parsers)

    def consume(self, reader: Reader[Input]):
        best_failure = None
        for parser in self.parsers:
            status = parser.consume(reader)
            if isinstance(status, Continue):
                return status.merge(best_failure)
            else:
                best_failure = status.merge(best_failure)

        return best_failure

    def __repr__(self):
        names = []
        for parser in self.parsers:
            names.append(parser.name_or_repr())

        return self.name_or_nothing() + ' | '.join(names)

class SequentialParser(Generic[Input], Parser[Input, List[Any]]):  # Type of this class is inexpressible
    def __init__(self, parser: Parser[Input, Any], *parsers: Sequence[Parser[Input, Any]]):
        super().__init__()
        self.parsers = (parser,) + tuple(parsers)

    def consume(self, reader: Reader[Input]):
        output = []
        status = None
        remainder = reader

        for parser in self.parsers:
            status = parser.consume(remainder).merge(status)
            if isinstance(status, Continue):
                output.append(status.value)
                remainder = status.remainder
            else:
                return status

        return Continue(remainder, output).merge(status)

    def __repr__(self):
        names = []
        for parser in self.parsers:
            names.append(parser.name_or_repr())

        return self.name_or_nothing() + ' & '.join(names)

class DiscardLeftParser(Generic[Input, Output], Parser[Input, Output]):
    def __init__(self, left: Parser[Input, Any], right: Parser[Input, Output]):
        super().__init__()
        self.left = left
        self.right = right

    def consume(self, reader: Reader[Input]):
        status = self.left.consume(reader)
        if isinstance(status, Continue):
            return self.right.consume(status.remainder).merge(status)
        else:
            return status

    def __repr__(self):
        return self.name_or_nothing() + '{} >> {}'.format(self.left.name_or_repr(), self.right.name_or_repr())

class DiscardRightParser(Generic[Input, Output], Parser[Input, Output]):
    def __init__(self, left: Parser[Input, Output], right: Parser[Input, Any]):
        super().__init__()
        self.left = left
        self.right = right

    def consume(self, reader: Reader[Input]):
        status1 = self.left.consume(reader)
        if isinstance(status1, Continue):
            status2 = self.right.consume(status1.remainder).merge(status1)
            if isinstance(status2, Continue):
                return Continue(status2.remainder, status1.value).merge(status2)
            else:
                return status2
        else:
            return status1

    def __repr__(self):
        return self.name_or_nothing() + '{} << {}'.format(self.left.name_or_repr(), self.right.name_or_repr())

class RepeatedOnceParser(Generic[Input, Output], Parser[Input, Sequence[Output]]):
    def __init__(self, parser: Parser[Input, Output]):
        super().__init__()
        self.parser = parser

    def consume(self, reader: Reader[Input]):
        status = self.parser.consume(reader)

        if not isinstance(status, Continue):
            return status
        else:
            output = [status.value]
            remainder = status.remainder
            while True:
                status = self.parser.consume(remainder).merge(status)
                if isinstance(status, Continue):
                    if remainder.position == status.remainder.position:
                        raise RuntimeError(remainder.recursion_error(str(self)))

                    remainder = status.remainder
                    output.append(status.value)
                else:
                    return Continue(remainder, output).merge(status)

    def __repr__(self):
        return self.name_or_nothing() + 'rep1({})'.format(self.parser.name_or_repr())

def rep1(parser: Union[Parser, Sequence[Input]]) -> RepeatedOnceParser:
    """Match a parser one or more times repeatedly.

    This matches ``parser`` multiple times in a row. If it matches as least
    once, it returns a list of values from each time ``parser`` matched. If it
    does not match ``parser`` at all, it fails.

    Args:
        parser: Parser or literal
    """
    if isinstance(parser, str):
        parser = lit(parser)
    return RepeatedOnceParser(parser)

class RepeatedParser(Generic[Input, Output], Parser[Input, Sequence[Output]]):
    def __init__(self, parser: Parser[Input, Output]):
        super().__init__()
        self.parser = parser

    def consume(self, reader: Reader[Input]):
        output = []
        status = None
        remainder = reader

        while True:
            status = self.parser.consume(remainder).merge(status)
            if isinstance(status, Continue):
                if remainder.position == status.remainder.position:
                    raise RuntimeError(remainder.recursion_error(str(self)))

                remainder = status.remainder
                output.append(status.value)
            else:
                return Continue(remainder, output).merge(status)

    def __repr__(self):
        return self.name_or_nothing() + 'rep({})'.format(self.parser.name_or_repr())

def rep(parser: Union[Parser, Sequence[Input]]) -> RepeatedParser:
    """Match a parser zero or more times repeatedly.

    This matches ``parser`` multiple times in a row. A list is returned
    containing the value from each match. If there are no matches, an empty list
    is returned.

    Args:
        parser: Parser or literal
    """
    if isinstance(parser, str):
        parser = lit(parser)
    return RepeatedParser(parser)

class RepeatedOnceSeparatedParser(Generic[Input, Output], Parser[Input, Sequence[Output]]):
    def __init__(self, parser: Parser[Input, Output], separator: Parser[Input, Output]):
        super().__init__()
        self.parser = parser
        self.separator = separator

    def consume(self, reader: Reader[Input]):
        status = self.parser.consume(reader)

        if not isinstance(status, Continue):
            return status
        else:
            output = [status.value]
            remainder = status.remainder
            while True:
                # If the separator matches, but the parser does not, the remainder from the last successful parser step
                # must be used, not the remainder from any separator. That is why the parser starts from the remainder
                # on the status, but remainder is not updated until after the parser succeeds.
                status = self.separator.consume(remainder).merge(status)
                if isinstance(status, Continue):
                    status = self.parser.consume(status.remainder).merge(status)
                    if isinstance(status, Continue):
                        if remainder.position == status.remainder.position:
                            raise RuntimeError(remainder.recursion_error(str(self)))

                        remainder = status.remainder
                        output.append(status.value)
                    else:
                        return Continue(remainder, output).merge(status)
                else:
                    return Continue(remainder, output).merge(status)

    def __repr__(self):
        return self.name_or_nothing() + 'rep1sep({}, {})'.format(self.parser.name_or_repr(),
                                                                 self.separator.name_or_repr())

def rep1sep(parser: Union[Parser, Sequence[Input]], separator: Union[Parser, Sequence[Input]]) \
        -> RepeatedOnceSeparatedParser:
    """Match a parser one or more times separated by another parser.

    This matches repeated sequences of ``parser`` separated by ``separator``.
    If there is at least one match, a list containing the values of the
    ``parser`` matches is returned. The values from ``separator`` are discarded.
    If it does not match ``parser`` at all, it fails.

    Args:
        parser: Parser or literal
        separator: Parser or literal
    """
    if isinstance(parser, str):
        parser = lit(parser)
    if isinstance(separator, str):
        separator = lit(separator)
    return RepeatedOnceSeparatedParser(parser, separator)

class RepeatedSeparatedParser(Generic[Input, Output], Parser[Input, Sequence[Output]]):
    def __init__(self, parser: Parser[Input, Output], separator: Parser[Input, Output]):
        super().__init__()
        self.parser = parser
        self.separator = separator

    def consume(self, reader: Reader[Input]):
        status = self.parser.consume(reader)

        if not isinstance(status, Continue):
            return Continue(reader, []).merge(status)
        else:
            output = [status.value]
            remainder = status.remainder
            while True:
                # If the separator matches, but the parser does not, the remainder from the last successful parser step
                # must be used, not the remainder from any separator. That is why the parser starts from the remainder
                # on the status, but remainder is not updated until after the parser succeeds.
                status = self.separator.consume(remainder).merge(status)
                if isinstance(status, Continue):
                    status = self.parser.consume(status.remainder).merge(status)
                    if isinstance(status, Continue):
                        if remainder.position == status.remainder.position:
                            raise RuntimeError(remainder.recursion_error(str(self)))

                        remainder = status.remainder
                        output.append(status.value)
                    else:
                        return Continue(remainder, output).merge(status)
                else:
                    return Continue(remainder, output).merge(status)

    def __repr__(self):
        return self.name_or_nothing() + 'repsep({}, {})'.format(self.parser.name_or_repr(),
                                                                self.separator.name_or_repr())

def repsep(parser: Union[Parser, Sequence[Input]], separator: Union[Parser, Sequence[Input]]) \
        -> RepeatedSeparatedParser:
    """Match a parser zero or more times separated by another parser.

    This matches repeated sequences of ``parser`` separated by ``separator``. A
    list is returned containing the value from each match of ``parser``. The
    values from ``separator`` are discarded. If there are no matches, an empty
    list is returned.

    Args:
        parser: Parser or literal
        separator: Parser or literal
    """
    if isinstance(parser, str):
        parser = lit(parser)
    if isinstance(separator, str):
        separator = lit(separator)
    return RepeatedSeparatedParser(parser, separator)

class ConversionParser(Generic[Input, Output, Convert], Parser[Input, Convert]):
    def __init__(self, parser: Parser[Input, Output], converter: Callable[[Output], Convert]):
        super().__init__()
        self.parser = parser
        self.converter = converter

    def consume(self, reader: Reader[Input]) -> Status[Input, Convert]:
        status = self.parser.consume(reader)

        if isinstance(status, Continue):
            return Continue(status.remainder, self.converter(status.value)).merge(status)
        else:
            return status

    def __repr__(self):
        return self.name_or_nothing() + repr(self.parser)

class EndOfSourceParser(Generic[Input], Parser[Input, None]):
    def __init__(self):
        super().__init__()

    def consume(self, reader: Reader[Input]) -> Status[Input, None]:
        if reader.finished:
            return Continue(reader, None)
        else:
            return Backtrack(reader, lambda: 'end of source')

    def __repr__(self):
        return self.name_or_nothing() + 'eof'

eof = EndOfSourceParser()

class SuccessParser(Generic[Input, Output], Parser[Input, Output]):
    def __init__(self, value: Output):
        super().__init__()
        self.value = value

    def consume(self, reader: Reader[Input]) -> Status[Input, None]:
        return Continue(reader, self.value)

    def __repr__(self):
        return self.name_or_nothing() + 'success({})'.format(repr(self.value))

def success(value: Any):
    """Always succeed in matching and return a value.

    This parser always succeeds and returns the given ``value``. No input is
    consumed. This is useful for inserting arbitrary values into complex
    parsers.

    Args:
        value: Any value
    """
    return SuccessParser(value)

class FailureParser(Generic[Input, Output], Parser[Input, Output]):
    def __init__(self, expected: str):
        super().__init__()
        self.expected = expected

    def consume(self, reader: Reader[Input]) -> Status[Input, None]:
        return Backtrack(reader, lambda: self.expected)

    def __repr__(self):
        return self.name_or_nothing() + 'failure({})'.format(repr(self.expected))

def failure(expected: str = ''):
    """Always fail in matching with a given message.

    This parser always backtracks with a message that it is expecting the
    given ``expected``. This is useful for appending to parsers that are
    expected to be seen in the input, but are not valid so that a more useful
    error message can be given.

    Args:
        expected: Message to be conveyed
    """
    return FailureParser(expected)

__all__ = ['Parser', 'LiteralParser', 'LiteralStringParser', 'lit', 'RegexParser', 'reg', 'OptionalParser', 'opt',
           'AlternativeParser', 'SequentialParser', 'DiscardLeftParser', 'DiscardRightParser', 'RepeatedOnceParser',
           'rep1', 'RepeatedParser', 'rep', 'RepeatedOnceSeparatedParser', 'rep1sep', 'RepeatedSeparatedParser',
           'repsep', 'ConversionParser', 'EndOfSourceParser', 'eof', 'SuccessParser', 'success', 'FailureParser',
           'failure', 'completely_parse_reader']


# Copyright (c) Moshe Zadka
# See LICENSE for details.

"""Check HTTP server for responsiveness"""

import collections
import functools
import json
import sys

import six

import twisted

from twisted.application import internet as tainternet
from twisted.internet import defer, reactor
from twisted.web import client

import ncolony
from ncolony import beatcheck
from ncolony.client import heart

class _ScoreCard(object):

    def __init__(self, maxBad=0):
        self.maxBad = maxBad
        self.bad = 0

    def isBad(self):
        """Too many unsuccessful checks"""
        return self.bad > self.maxBad

    def markBad(self, dummyValue):
        """Note an unsuccessful check"""
        self.bad += 1

    def markGood(self, dummyValue):
        """Note a successful check"""
        self.bad = 0

    def __repr__(self):
        return ('<%(klass)s:%(id)s:maxBad=%(maxBad)s,bad=%(bad)s>' %
                dict(klass=self.__class__.__name__,
                     id=hex(id(self)),
                     maxBad=self.maxBad,
                     bad=self.bad))

Settings = collections.namedtuple('Settings', 'reactor agent')

_USER_AGENT = ('NColony HTTP Check ('
               'NColony/' + str(ncolony.__version__) + ', '
               'Twisted/' + twisted.__version__ + ', '
               'Python ' + sys.version.replace('\n', '') + ')')

_standardHeaders = client.Headers({'User-Agent': [_USER_AGENT]})

# pylint: disable=too-many-instance-attributes

class State(object):

    """State of an HTTP check"""

    KEY = 'ncolony.httpcheck'

    def __init__(self, location, settings):
        self.location = location
        self.settings = settings
        self.closed = False
        self.content = None
        self.call = None
        self.card = None
        self.url = None
        self.nextCheck = None
        self.period = None
        self.timeout = None

    def __repr__(self):
        return ('<%(klass)s:%(id)s:location=%(location)s,'
                'settings=%(settings)s,'
                'closed=%(closed)s,content=%(content)s,call=%(call)s,'
                'card=%(card)s>' %
                dict(klass=self.__class__.__name__, id=hex(id(self)),
                     location=self.location,
                     settings=self.settings,
                     closed=self.closed,
                     content=self.content,
                     call=self.call,
                     card=self.card))

    def _reset(self):
        self.content = None
        self._maybeReset()

    def close(self):
        """Discard data and cancel all calls.

        Instance cannot be reused after closing.
        """
        if self.closed:
            raise ValueError("Cannot close a closed state")
        if self.call is not None:
            self.call.cancel()
        self.closed = True

    def check(self):
        """Check the state of HTTP"""
        if self.closed:
            raise ValueError("Cannot check a closed state")
        self._maybeReset()
        if self.url is None:
            return False
        return self._maybeCheck()

    def _maybeReset(self):
        content = self.location.getContent().decode('utf-8')
        if content == self.content:
            return
        self.content = content
        if self.call is not None:
            self.call.cancel()
            self.call = None
        parsed = json.loads(self.content)
        config = parsed.get(self.KEY)
        if config is None:
            self.url = None
            self.card = _ScoreCard()
            return
        self.url = config['url']
        self.period = config['period']
        self.card = _ScoreCard(config['maxBad'])
        self.timeout = min(self.period, config['timeout'])
        self.nextCheck = (self.settings.reactor.seconds() +
                          config['grace'] * self.period)

    def _maybeCheck(self):
        if self.settings.reactor.seconds() <= self.nextCheck:
            return False
        assert self.call is None, 'Timeout reached, call still outstanding'
        if self.card.isBad():
            self._reset()
            return True
        self.nextCheck = self.settings.reactor.seconds() + self.period
        self.call = self.settings.agent.request('GET', self.url,
                                                _standardHeaders, None)
        delayedCall = self.settings.reactor.callLater(self.timeout,
                                                      self.call.cancel)

        def _gotResult(result):
            if delayedCall.active():
                delayedCall.cancel()
            return result
        self.call.addBoth(_gotResult)
        self.call.addErrback(defer.logError)
        self.call.addCallbacks(callback=self.card.markGood,
                               errback=self.card.markBad)

        def _removeCall(dummy):
            self.call = None
        self.call.addCallback(_removeCall)
        return False

# pylint: enable=too-many-instance-attributes

def check(settings, states, location):
    """Check all processes"""
    children = {child.basename(): child for child in location.children()}
    last = set(states)
    current = set(children)
    gone = last - current
    added = current - last
    for name in gone:
        states[name].close()
        del states[name]
    for name in added:
        states[name] = State(location=children[name], settings=settings)
    return [name for name, state in six.iteritems(states) if state.check()]

def run(restarter, checker):
    """Run restarter on the checker's output

    :params restarter: something to run on the output of the checker
    :params checker: a function expected to get one argument (current time)
                     and return a list of stale names
    :params timer: a function of zero arguments, intended to return current
                   time
    :returns: None
    """
    for bad in checker():
        restarter(bad)

def makeService(opt):
    """Make a service

    :params opt: dictionary-like object with 'freq', 'config' and 'messages'
    :returns: twisted.application.internet.TimerService that at opt['freq']
              checks for stale processes in opt['config'], and sends
              restart messages through opt['messages']
    """
    restarter, path = beatcheck.parseConfig(opt)
    pool = client.HTTPConnectionPool(reactor)
    agent = client.Agent(reactor=reactor, pool=pool)
    settings = Settings(reactor=reactor, agent=agent)
    states = {}
    checker = functools.partial(check, settings, states, path)
    httpcheck = tainternet.TimerService(opt['freq'], run, restarter, checker)
    httpcheck.setName('httpcheck')
    return heart.wrapHeart(httpcheck)

Options = beatcheck.Options


# Copyright (c) Moshe Zadka
# See LICENSE for details.

"""ncolony.client.heart
=====================

A heart beater.
"""
from __future__ import division

import json
import os

from twisted.python import filepath
from twisted.application import internet as tainternet, service as taservice

class Heart(object):

    """A Heart.

    Each beat touches a file.
    """
    def __init__(self, path):
        self.path = path

    def getFile(self):
        """Get the file being touched"""
        return self.path

    def beat(self):
        """Touch the file"""
        self.path.touch()

def makeService():
    """Make a service

    :returns: an IService
    """
    configJSON = os.environ.get('NCOLONY_CONFIG')
    if configJSON is None:
        return None
    config = json.loads(configJSON)
    params = config.get('ncolony.beatcheck')
    if params is None:
        return None
    myFilePath = filepath.FilePath(params['status'])
    if myFilePath.isdir():
        name = os.environ['NCOLONY_NAME']
        myFilePath = myFilePath.child(name)
    heart = Heart(myFilePath)
    ret = tainternet.TimerService(params['period']/3, heart.beat)
    return ret

def maybeAddHeart(master):
    """Add a heart to a service collection

    Add a heart to a service.IServiceCollector if
    the heart is not None.

    :params master: a service.IServiceCollector
    """
    heartSer = makeService()
    if heartSer is None:
        return
    heartSer.setName('heart')
    heartSer.setServiceParent(master)

def wrapHeart(service):
    """Wrap a service in a MultiService with a heart"""
    master = taservice.MultiService()
    service.setServiceParent(master)
    maybeAddHeart(master)
    return master


#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""Utility to freeze TensorFlow graphs."""

from __future__ import absolute_import, division, print_function, unicode_literals

import os

import tensorflow as tf
from tensorflow.python.framework import graph_io, graph_util
from tensorflow.python.tools import freeze_graph

from benderthon.util import check_input_checkpoint, output_node_names_string_as_list, restore_from_checkpoint,\
    TemporaryDirectory

def freeze_from_checkpoint(input_checkpoint, output_file_path, output_node_names):
    """Freeze and shrink the graph based on a checkpoint and the output node names."""
    check_input_checkpoint(input_checkpoint)

    output_node_names = output_node_names_string_as_list(output_node_names)

    with tf.Session() as sess:
        restore_from_checkpoint(sess, input_checkpoint)
        freeze_graph.freeze_graph_with_def_protos(input_graph_def=sess.graph_def, input_saver_def=None,
                                                  input_checkpoint=input_checkpoint,
                                                  output_node_names=','.join(output_node_names),
                                                  restore_op_name='save/restore_all',
                                                  filename_tensor_name='save/Const:0', output_graph=output_file_path,
                                                  clear_devices=True, initializer_nodes='')

def freeze(sess, output_file_path, output_node_names):
    """Freeze and shrink the graph based on a session and the output node names."""
    with TemporaryDirectory() as temp_dir_name:
        checkpoint_path = os.path.join(temp_dir_name, 'model.ckpt')
        tf.train.Saver().save(sess, checkpoint_path)

        freeze_from_checkpoint(checkpoint_path, output_file_path, output_node_names)

def save_graph_only(sess, output_file_path, output_node_names, as_text=False):
    """Save a small version of the graph based on a session and the output node names."""
    for node in sess.graph_def.node:
        node.device = ''
    graph_def = graph_util.extract_sub_graph(sess.graph_def, output_node_names)
    output_dir, output_filename = os.path.split(output_file_path)
    graph_io.write_graph(graph_def, output_dir, output_filename, as_text=as_text)

def save_graph_only_from_checkpoint(input_checkpoint, output_file_path, output_node_names, as_text=False):
    """Save a small version of the graph based on a checkpoint and the output node names."""
    check_input_checkpoint(input_checkpoint)

    output_node_names = output_node_names_string_as_list(output_node_names)

    with tf.Session() as sess:
        restore_from_checkpoint(sess, input_checkpoint)
        save_graph_only(sess, output_file_path, output_node_names, as_text=as_text)

def save_weights(sess, output_path, conv_var_names=None, conv_transpose_var_names=None):
    """Save the weights of the trainable variables, each one in a different file in output_path."""
    if not conv_var_names:
        conv_var_names = []

    if not conv_transpose_var_names:
        conv_transpose_var_names = []

    for var in tf.trainable_variables():
        filename = '{}-{}'.format(output_path, var.name.replace(':', '-').replace('/', '-'))

        if var.name in conv_var_names:
            var = tf.transpose(var, perm=[3, 0, 1, 2])
        elif var.name in conv_transpose_var_names:
            var = tf.transpose(var, perm=[3, 1, 0, 2])

        value = sess.run(var)

        # noinspection PyTypeChecker
        with open(filename, 'w') as file_:
            value.tofile(file_)

def save_weights_from_checkpoint(input_checkpoint, output_path, conv_var_names=None, conv_transpose_var_names=None):
    """Save the weights of the trainable variables given a checkpoint, each one in a different file in output_path."""
    check_input_checkpoint(input_checkpoint)

    with tf.Session() as sess:
        restore_from_checkpoint(sess, input_checkpoint)
        save_weights(sess, output_path, conv_var_names=conv_var_names,
                     conv_transpose_var_names=conv_transpose_var_names)


# -*- coding: utf-8 -*-
"""Util functions used by Bender subcommands."""

from __future__ import absolute_import, division, print_function, unicode_literals

import shutil
import tempfile
import warnings

from six import string_types
import tensorflow as tf
from tensorflow.python.training import saver as saver_lib

def check_input_checkpoint(input_checkpoint):
    """Check if input_checkpoint is a valid path or path prefix."""
    if not saver_lib.checkpoint_exists(input_checkpoint):
        print("Input checkpoint '{}' doesn't exist!".format(input_checkpoint))
        exit(-1)

def restore_from_checkpoint(sess, input_checkpoint):
    """Return a TensorFlow saver from a checkpoint containing the metagraph."""
    saver = tf.train.import_meta_graph('{}.meta'.format(input_checkpoint))
    saver.restore(sess, input_checkpoint)
    return saver

def output_node_names_string_as_list(output_node_names):
    """Return a list of containing output_node_names if it's a string, otherwise return just output_node_names."""
    if isinstance(output_node_names, string_types):
        return [output_node_names]
    else:
        return output_node_names

class TemporaryDirectory(object):
    """Create and return a temporary directory.  This has the same
    behavior as mkdtemp but can be used as a context manager.  For
    example:

        with TemporaryDirectory() as tmpdir:
            ...

    Upon exiting the context, the directory and everything contained
    in it are removed.

    Inspired from https://hg.python.org/cpython/file/3.6/Lib/tempfile.py
    """

    def __init__(self, suffix='', prefix='', dir_=None):
        self.name = tempfile.mkdtemp(suffix, prefix, dir_)

    @classmethod
    def _cleanup(cls, name, warn_message):
        shutil.rmtree(name)
        warnings.warn(warn_message, ResourceWarning)

    def __repr__(self):
        return "<{} {!r}>".format(self.__class__.__name__, self.name)

    def __enter__(self):
        return self.name

    def __exit__(self, exc, value, tb):
        shutil.rmtree(self.name)


import django
from django.core.exceptions import ImproperlyConfigured
from django.template.base import Template, Parser
from django.template import Node, Context, TemplateSyntaxError
from django.template.loader import get_template, select_template
from django.utils import six
from django.utils.itercompat import is_iterable
from tag_parser.parser import parse_token_kwargs, parse_as_var

__all__ = (
    'BaseNode', 'BaseInclusionNode'
)

class BaseNode(Node):
    """
    Base class for template tag nodes.

    Plain style:

    .. code-block:: python

        class MyTag(BaseNode):
            def render_tag(self, context, *args, **kwargs):
                return "Tag output"

        register.tag('my_tag', MyTag.parse)

    or classical style:

    .. code-block:: python

        @register.tag
        def my_tag(parser, token):
            return MyTag.parse(parser, token)

    or using the decorator:

    .. code-block:: python

        @template_tag(register, 'my_tag')
        class MyTag(BaseNode):
            def render_tag(self, context, *args, **kwargs):
                return "Tag output"
    """
    #: The names of the allowed keyword arguments in the template tag.
    #: Using ``None`` to disable this check, thus allowing all keyword arguments.
    allowed_kwargs = ()

    #: If you want that it this will also parse the text until the end tag,
    #: and store that in ``self.nodelist``.
    end_tag_name = None

    #: The minimum number of required positional arguments. Use ``None`` to disable the check.
    min_args = 0

    #: The maximum number of allowed positional arguments. Use ``None`` for infinite.
    max_args = 0

    #: Whether the parser should treat the arguments as filter expressions.
    compile_args = True

    #: Whether the parser should treat the keyword arguments as filter expressions.
    compile_kwargs = True

    @staticmethod
    def __new__(cls, *args, **kwargs):
        if not kwargs and len(args) == 2 and isinstance(args[0], Parser):
            # Reasons for this code:
            # * When @register.tag() is used as class decorator,
            #   this class itself is called as compile_function(parser, token).
            # * By intercepting cls() calls, we can call parse() instead.
            # * We want to keep __init__() clean for unit testing and inheritance reuse.
            # * We must inherit from cls so template tag detection works as expected.
            #
            # Python note: When __new__ returns an instance of cls,
            #              the __init__ method will be called on that object afterwards.
            class CompileWrapper(cls):
                __real_class = cls

                def __init__(self, parser, token):
                    self.__real_node = self.__real_class.parse(parser, token)

                def __getattribute__(self, item):
                    # Proxy all accessed attributes to the real object type.
                    # Note this even intercepts __class__, etc.. type(self) gives the real type.
                    if item.startswith('_CompileWrapper_'):
                        return super(CompileWrapper, self).__getattribute__(item)

                    return getattr(self.__real_node, item)

            return object.__new__(CompileWrapper)
            # return _CompileWrapper(cls, *args)
        else:
            # Standard object construction (typically done from parse())
            # The created object instance will be receiving the __init__(*args, **kwargs) call on return.
            return object.__new__(cls)

    def __init__(self, tag_name, *args, **kwargs):
        """
        The constructor receives the parsed arguments.
        The values are stored in :attr:`tagname`, :attr:`args`, :attr:`kwargs`.
        """
        if isinstance(tag_name, Parser):
            raise TypeError("Unexpected call of {0}.__init__(parser, token)!".format(self.__class__.__name__))

        if self.end_tag_name and 'nodelist' in kwargs:
            self.nodelist = kwargs.pop('nodelist')
        self.tag_name = tag_name  # May differ from cls.tag_name, and doesn't affect the 'cls' attribute at all.
        self.args = args
        self.kwargs = kwargs

    def __repr__(self):
        show_as_is = lambda text: text
        show_as_token = lambda expr: expr.token
        format_arg = show_as_token if self.compile_args else show_as_is
        format_kwarg = show_as_token if self.compile_kwargs else show_as_is

        return u'<{0}: {{% {1}{2}{3} %}}>'.format(
            self.__class__.__name__,
            self.tag_name,
            u''.join(u" {0}".format(format_arg(a)) for a in self.args),
            u''.join(u" {0}={1}".format(k, format_kwarg(v)) for k, v in six.iteritems(self.kwargs)),
        )

    @classmethod
    def parse(cls, parser, token):
        """
        Parse the tag, instantiate the class.

        :type parser: django.template.base.Parser
        :type token: django.template.base.Token
        """
        tag_name, args, kwargs = parse_token_kwargs(
            parser, token,
            allowed_kwargs=cls.allowed_kwargs,
            compile_args=cls.compile_args,
            compile_kwargs=cls.compile_kwargs
        )
        cls.validate_args(tag_name, *args, **kwargs)
        if cls.end_tag_name:
            kwargs['nodelist'] = parser.parse((cls.end_tag_name,))
            parser.delete_first_token()

        return cls(tag_name, *args, **kwargs)

    def render(self, context):
        """
        The default Django render() method for the tag.

        This method resolves the filter expressions, and calls :func:`render_tag`.
        """
        # Resolve token kwargs
        tag_args = [expr.resolve(context) for expr in self.args] if self.compile_args else self.args
        tag_kwargs = dict([(name, expr.resolve(context)) for name, expr in six.iteritems(self.kwargs)]) if self.compile_kwargs else self.kwargs

        return self.render_tag(context, *tag_args, **tag_kwargs)

    def render_tag(self, context, *tag_args, **tag_kwargs):
        """
        Render the tag, with all arguments resolved to their actual values.
        """
        raise NotImplementedError("{0}.render_tag() is not implemented!".format(self.__class__.__name__))

    @classmethod
    def validate_args(cls, tag_name, *args, **kwargs):
        """
        Validate the syntax of the template tag.
        """
        if cls.min_args is not None and len(args) < cls.min_args:
            if cls.min_args == 1:
                raise TemplateSyntaxError("'{0}' tag requires at least {1} argument".format(tag_name, cls.min_args))
            else:
                raise TemplateSyntaxError("'{0}' tag requires at least {1} arguments".format(tag_name, cls.min_args))

        if cls.max_args is not None and len(args) > cls.max_args:
            if cls.max_args == 0:
                if cls.allowed_kwargs:
                    raise TemplateSyntaxError("'{0}' tag only allows keywords arguments, for example {1}=\"...\".".format(tag_name, cls.allowed_kwargs[0]))
                else:
                    raise TemplateSyntaxError("'{0}' tag doesn't support any arguments".format(tag_name))
            elif cls.max_args == 1:
                raise TemplateSyntaxError("'{0}' tag only allows {1} argument.".format(tag_name, cls.max_args))
            else:
                raise TemplateSyntaxError("'{0}' tag only allows {1} arguments.".format(tag_name, cls.max_args))

    def get_request(self, context):
        """
        Fetch the request from the context.

        This enforces the use of a RequestProcessor, e.g.
        ::

            render_to_response("page.html", context, context_instance=RequestContext(request))
        """
        if 'request' not in context:
            # This error message is issued to help newcomers find solutions faster!
            raise ImproperlyConfigured(
                "The '{0}' tag requires a 'request' variable in the template context.\n"
                "Make sure 'RequestContext' is used and 'TEMPLATE_CONTEXT_PROCESSORS' includes 'django.core.context_processors.request'.".format(self.tag_name)
            )

        return context['request']

class BaseInclusionNode(BaseNode):
    """
    Base class to render a template tag with a template.

    This class allows more flexibility then Django's default :func:`django.template.Library.inclusion_tag` decorator.

    It allows specify the template name via:

    * a static ``template_name`` property.
    * a ``get_template_name()`` method
    * a 'template' kwarg in the HTML.

    The :func:`get_context_data` function should be overwritten to provide the required context.
    """
    template_name = None
    allowed_kwargs = ('template',)

    def render_tag(self, context, *tag_args, **tag_kwargs):
        data = self.get_context_data(context, *tag_args, **tag_kwargs)
        new_context = self.get_context(context, data)

        if django.VERSION >= (1, 8):
            t = context.render_context.get(self)
            if t is None:
                file_name = self.get_template_name(*tag_args, **tag_kwargs)

                if isinstance(file_name, Template):
                    t = file_name
                elif isinstance(getattr(file_name, 'template', None), Template):
                    t = file_name.template
                elif not isinstance(file_name, six.string_types) and is_iterable(file_name):
                    t = context.template.engine.select_template(file_name)
                else:
                    t = context.template.engine.get_template(file_name)
                context.render_context[self] = t

            return t.render(new_context)
        else:
            # Get template nodes, and cache it.
            # Note that self.nodelist has a special meaning in the Node base class.
            if not getattr(self, 'nodelist', None):
                file_name = self.get_template_name(*tag_args, **tag_kwargs)

                if not isinstance(file_name, six.string_types) and is_iterable(file_name):
                    tpl = select_template(file_name)
                else:
                    tpl = get_template(file_name)

                self.nodelist = tpl.nodelist

            # Render the node
            return self.nodelist.render(new_context)

    def get_template_name(self, *tag_args, **tag_kwargs):
        """
        Get the template name, by default using the :attr:`template_name` attribute.
        """
        return tag_kwargs.get('template', self.template_name)

    def get_context_data(self, parent_context, *tag_args, **tag_kwargs):
        """
        Return the context data for the included template.
        """
        raise NotImplementedError("{0}.get_context_data() is not implemented.".format(self.__class__.__name__))

    def get_context(self, parent_context, data):
        """
        Wrap the context data in a :class:`~django.template.Context` object.

        :param parent_context: The context of the parent template.
        :type parent_context: :class:`~django.template.Context`
        :param data: The result from :func:`get_context_data`
        :type data: dict
        :return: Context data.
        :rtype: :class:`~django.template.Context`
        """
        if django.VERSION >= (1, 8):
            new_context = parent_context.new(data)
        else:
            settings = {
                'autoescape': parent_context.autoescape,
                'current_app': parent_context.current_app,
                'use_l10n': parent_context.use_l10n,
                'use_tz': parent_context.use_tz,
            }
            new_context = Context(data, **settings)

        # Pass CSRF token for same reasons as @register.inclusion_tag does.
        csrf_token = parent_context.get('csrf_token', None)
        if csrf_token is not None:
            new_context['csrf_token'] = csrf_token

        return new_context

class BaseAssignmentNode(BaseNode):
    """
    Base assignment node for an ``as var`` syntax.
    """

    def __init__(self, tag_name, as_var, *args, **kwargs):
        super(BaseAssignmentNode, self).__init__(tag_name, *args, **kwargs)
        self.as_var = as_var

    @classmethod
    def parse(cls, parser, token):
        """
        Parse the "as var" syntax.
        """
        bits, as_var = parse_as_var(parser, token)
        tag_name, args, kwargs = parse_token_kwargs(parser, bits, cls.allowed_kwargs, compile_args=cls.compile_args, compile_kwargs=cls.compile_kwargs)

        # Pass through standard chain
        cls.validate_args(tag_name, *args)
        return cls(tag_name, as_var, *args, **kwargs)

    def render_tag(self, context, *tag_args, **tag_kwargs):
        """
        Rendering of the tag. It either assigns the value as variable, or renders it.
        """
        if self.as_var:
            # Assign the value in the parent context
            context[self.as_var] = self.get_value(context, *tag_args, **tag_kwargs)

        return u''

    def get_value(self, context, *tag_args, **tag_kwargs):
        """
        Return the value for the tag.

        :param tag_args:
        :param tag_kwargs:
        """
        raise NotImplementedError("{0}.get_value() is not implemented'.".format(self.__class__.__name__))

class BaseAssignmentOrOutputNode(BaseAssignmentNode):
    """
    Base class to assign a value, or output it.
    This works like ``{% trans .. %}`` and ``{% url .. %}`` which both optionally
    have an ``as var`` syntax; ``{% trans .. as .. %}`` and ``{% url .. as .. %}``.
    """

    def render_tag(self, context, *tag_args, **tag_kwargs):
        if self.as_var:
            return super(BaseAssignmentOrOutputNode, self).render_tag(context, *tag_args, **tag_kwargs)
        else:
            return self.get_value(context, *tag_args, **tag_kwargs)

class BaseAssignmentOrInclusionNode(BaseInclusionNode, BaseAssignmentNode):
    """
    Base class to either assign a tag, or render it using a template.
    """
    context_value_name = 'value'

    @classmethod
    def parse(cls, parser, token):
        """
        Parse the "as var" syntax.
        """
        bits, as_var = parse_as_var(parser, token)
        tag_name, args, kwargs = parse_token_kwargs(parser, bits, ('template',) + cls.allowed_kwargs, compile_args=cls.compile_args, compile_kwargs=cls.compile_kwargs)

        # Pass through standard chain
        cls.validate_args(tag_name, *args)
        return cls(tag_name, as_var, *args, **kwargs)

    def render_tag(self, context, *tag_args, **tag_kwargs):
        """
        Rendering of the tag. It either assigns the value as variable, or renders it.
        """
        # Be very explicit about which base functionality is used:
        # Using super() for mixin support will not work nicely anyway here.
        if self.as_var:
            # Assign the value in the parent context
            return BaseAssignmentNode.render_tag(self, context, *tag_args, **tag_kwargs)
        else:
            # Render the output using the BaseInclusionNode features
            return BaseInclusionNode.render_tag(self, context, *tag_args, **tag_kwargs)

    def get_context_data(self, parent_context, *tag_args, **tag_kwargs):
        """
        Return the context data for the inclusion tag.

        Returns ``{'value': self.get_value(parent_context, *tag_args, **tag_kwargs)}`` by default.
        """
        if 'template' not in self.allowed_kwargs:
            # The overwritten get_value() doesn't have to take care of our customly inserted tag parameters,
            # It can safely assume passing **tag_kwargs to another function.
            tag_kwargs.pop('template', None)

        return {
            self.context_value_name: self.get_value(parent_context, *tag_args, **tag_kwargs)
        }


#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""Utility to freeze Caffe models."""

from __future__ import absolute_import, division, print_function, unicode_literals

import contextlib
import imp
import os

import tensorflow as tf

from benderthon import tf_freeze, util

@contextlib.contextmanager
def dummy_context_mgr(obj):
    yield obj

def caffe_to_tensorflow_session(caffe_def_path, caffemodel_path, inputs, graph_name='Graph',
                                conversion_out_dir_path=None, use_padding_same=False):
    """Create a TensorFlow Session from a Caffe model."""
    try:
        # noinspection PyUnresolvedReferences
        from caffeflow import convert
    except ImportError:
        raise Exception("caffeflow package needs to be installed to freeze Caffe models. Check out the README file.")

    with (dummy_context_mgr(conversion_out_dir_path) or util.TemporaryDirectory()) as dir_path:
        params_values_output_path = os.path.join(dir_path, 'params_values.npy')
        network_output_path = os.path.join(dir_path, 'network.py')

        convert.convert(caffe_def_path, caffemodel_path, params_values_output_path, network_output_path, False,
                        use_padding_same=use_padding_same)

        network_module = imp.load_source('module.name', network_output_path)
        network_class = getattr(network_module, graph_name)
        network = network_class(inputs)

        sess = tf.Session()

        network.load(params_values_output_path, sess)

        return sess

def freeze(caffe_def_path, caffemodel_path, inputs, output_file_path, output_node_names, graph_name='Graph',
           conversion_out_dir_path=None, checkpoint_out_path=None, use_padding_same=False):
    """Freeze and shrink the graph based on a Caffe model, the input tensors and the output node names."""
    with caffe_to_tensorflow_session(caffe_def_path, caffemodel_path, inputs, graph_name=graph_name,
                                     conversion_out_dir_path=conversion_out_dir_path,
                                     use_padding_same=use_padding_same) as sess:
        saver = tf.train.Saver()

        with (dummy_context_mgr(checkpoint_out_path) or util.TemporaryDirectory()) as temp_dir_path:
            checkpoint_path = checkpoint_out_path or os.path.join(temp_dir_path, 'pose.ckpt')
            saver.save(sess, checkpoint_path)

            output_node_names = util.output_node_names_string_as_list(output_node_names)

            tf_freeze.freeze_from_checkpoint(checkpoint_path, output_file_path, output_node_names)

def save_graph_only(caffe_def_path, caffemodel_path, inputs, output_file_path, output_node_names, graph_name='Graph',
                    use_padding_same=False):
    """Save a small version of the graph based on a Caffe model, the input tensors and the output node names."""
    with caffe_to_tensorflow_session(caffe_def_path, caffemodel_path, inputs, graph_name=graph_name,
                                     use_padding_same=use_padding_same) as sess:
        tf_freeze.save_graph_only(sess, output_file_path, output_node_names)

def save_weights(caffe_def_path, caffemodel_path, inputs, output_path, graph_name='Graph', conv_var_names=None,
                 conv_transpose_var_names=None, use_padding_same=False):
    """Save the weights of the trainable variables, each one in a different file in output_path."""
    with caffe_to_tensorflow_session(caffe_def_path, caffemodel_path, inputs, graph_name=graph_name,
                                     use_padding_same=use_padding_same) as sess:
        tf_freeze.save_weights(sess, output_path, conv_var_names=conv_var_names,
                               conv_transpose_var_names=conv_transpose_var_names)


# -*- coding: UTF-8 -*-

"""
jaraco.itertools
Tools for working with iterables.  Complements itertools and more_itertools.
"""

from __future__ import absolute_import, unicode_literals, print_function

import operator
import itertools
import collections
import math
import warnings
import functools

try:
	import collections.abc
except ImportError:
	collections.abc = collections

import six
from six.moves import queue, xrange as range

import inflect
import more_itertools

def make_rows(num_columns, seq):
	"""
	Make a sequence into rows of num_columns columns.

	>>> tuple(make_rows(2, [1, 2, 3, 4, 5]))
	((1, 4), (2, 5), (3, None))
	>>> tuple(make_rows(3, [1, 2, 3, 4, 5]))
	((1, 3, 5), (2, 4, None))
	"""
	# calculate the minimum number of rows necessary to fit the list in
	# num_columns Columns
	num_rows, partial = divmod(len(seq), num_columns)
	if partial:
		num_rows += 1
	# break the seq into num_columns of length num_rows
	try:
		result = more_itertools.grouper(seq, num_rows)
	except TypeError:
		# more_itertools before 6.x
		result = more_itertools.grouper(num_rows, seq)
	# result is now a list of columns... transpose it to return a list
	# of rows
	return zip(*result)

def bisect(seq, func=bool):
	"""
	Split a sequence into two sequences:  the first is elements that
	return False for func(element) and the second for True for
	func(element).
	By default, func is ``bool``, so uses the truth value of the object.

	>>> is_odd = lambda n: n%2
	>>> even, odd = bisect(range(5), is_odd)
	>>> list(odd)
	[1, 3]
	>>> list(even)
	[0, 2, 4]

	>>> other, zeros = bisect(reversed(range(5)))
	>>> list(zeros)
	[0]
	>>> list(other)
	[4, 3, 2, 1]

	"""
	queues = GroupbySaved(seq, func)
	return queues.get_first_n_queues(2)

class GroupbySaved(object):
	"""
	Split a sequence into n sequences where n is determined by the
	number of distinct values returned by a key function applied to each
	element in the sequence.

	>>> truthsplit = GroupbySaved(['Test', '', 30, None], bool)
	>>> truthsplit['x']
	Traceback (most recent call last):
	...
	KeyError: 'x'
	>>> true_items = truthsplit[True]
	>>> false_items = truthsplit[False]
	>>> tuple(iter(false_items))
	('', None)
	>>> tuple(iter(true_items))
	('Test', 30)

	>>> every_third_split = GroupbySaved(range(99), lambda n: n%3)
	>>> zeros = every_third_split[0]
	>>> ones = every_third_split[1]
	>>> twos = every_third_split[2]
	>>> next(zeros)
	0
	>>> next(zeros)
	3
	>>> next(ones)
	1
	>>> next(twos)
	2
	>>> next(ones)
	4
	"""

	def __init__(self, sequence, func=lambda x: x):
		self.sequence = iter(sequence)
		self.func = func
		self.queues = collections.OrderedDict()

	def __getitem__(self, key):
		try:
			return self.queues[key]
		except KeyError:
			return self.__find_queue__(key)

	def __fetch__(self):
		"get the next item from the sequence and queue it up"
		item = next(self.sequence)
		key = self.func(item)
		queue = self.queues.setdefault(key, FetchingQueue(self.__fetch__))
		queue.enqueue(item)

	def __find_queue__(self, key):
		"search for the queue indexed by key"
		try:
			while key not in self.queues:
				self.__fetch__()
			return self.queues[key]
		except StopIteration:
			raise KeyError(key)

	def get_first_n_queues(self, n):
		"""
		Run through the sequence until n queues are created and return
		them. If fewer are created, return those plus empty iterables to
		compensate.
		"""
		try:
			while len(self.queues) < n:
				self.__fetch__()
		except StopIteration:
			pass
		values = list(self.queues.values())
		missing = n - len(values)
		values.extend(iter([]) for n in range(missing))
		return values

class FetchingQueue(queue.Queue):
	"""
	A FIFO Queue that is supplied with a function to inject more into
	the queue if it is empty.

	>>> values = iter(range(10))
	>>> get_value = lambda: globals()['q'].enqueue(next(values))
	>>> q = FetchingQueue(get_value)
	>>> [x for x in q] == list(range(10))
	True

	Note that tuple(q) or list(q) would not have worked above because
	tuple(q) just copies the elements in the list (of which there are
	none).
	"""

	def __init__(self, fetcher):
		if six.PY3:
			super(FetchingQueue, self).__init__()
		else:
			queue.Queue.__init__(self)
		self._fetcher = fetcher

	def __next__(self):
		while self.empty():
			self._fetcher()
		return self.get()
	next = __next__

	def __iter__(self):
		while True:
			try:
				yield next(self)
			except StopIteration:
				return

	def enqueue(self, item):
		self.put_nowait(item)

class Count(object):
	"""
	A stop object that will count how many times it's been called and return
	False on the N+1st call.  Useful for use with takewhile.

	>>> tuple(itertools.takewhile(Count(5), range(20)))
	(0, 1, 2, 3, 4)

	>>> print('catch', Count(5))
	catch at most 5

	It's possible to construct a Count with no limit or infinite limit.

	>>> unl_c = Count(None)
	>>> inf_c = Count(float('Inf'))

	Unlimited or limited by infinity are equivalent.

	>>> unl_c == inf_c
	True

	An unlimited counter is useful for wrapping an iterable to get the
	count after it's consumed.

	>>> tuple(itertools.takewhile(unl_c, range(20)))[-3:]
	(17, 18, 19)
	>>> unl_c.count
	20

	If all you need is the count of items, consider :class:`Counter` instead.
	"""

	def __init__(self, limit):
		self.count = 0
		self.limit = limit if limit is not None else float('Inf')

	def __call__(self, arg):
		if self.count > self.limit:
			raise ValueError("Should not call count stop more anymore.")
		self.count += 1
		return self.count <= self.limit

	def __str__(self):
		if self.limit:
			return 'at most %d' % self.limit
		else:
			return 'all'

	def __eq__(self, other):
		return vars(self) == vars(other)

class islice(object):
	"""May be applied to an iterable to limit the number of items returned.
	Works similarly to count, except is called only once on an iterable.
	Functionality is identical to islice, except for __str__ and reusability.

	>>> tuple(islice(5).apply(range(20)))
	(0, 1, 2, 3, 4)

	>>> tuple(islice(None).apply(range(20)))
	(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19)

	>>> print(islice(3, 10))
	items 3 to 9

	>>> print(islice(3, 10, 2))
	every 2nd item from 3 to 9
	"""

	def __init__(self, *sliceArgs):
		self.sliceArgs = sliceArgs

	def apply(self, i):
		return itertools.islice(i, *self.sliceArgs)

	def __str__(self):
		if self.sliceArgs == (None,):
			result = 'all'
		else:
			result = self._formatArgs()
		return result

	def _formatArgs(self):

		def slice_range(a_b):
			return '%d to %d' % (a_b[0], a_b[1] - 1)

		if len(self.sliceArgs) == 1:
			result = 'at most %d' % self.sliceArgs
		if len(self.sliceArgs) == 2:
			result = 'items %s' % slice_range(self.sliceArgs)
		if len(self.sliceArgs) == 3:
			ord = inflect.engine().ordinal(self.sliceArgs[2])
			range = slice_range(self.sliceArgs[0:2])
			result = 'every %(ord)s item from %(range)s' % locals()
		return result

class LessThanNBlanks(object):
	"""
	An object that when called will return True until n false elements
	are encountered.

	Can be used with filter or itertools.ifilter, for example:

	>>> import itertools
	>>> sampleData = ['string 1', 'string 2', '', 'string 3', '',
	...     'string 4', '', '', 'string 5']
	>>> first = itertools.takewhile(LessThanNBlanks(2), sampleData)
	>>> tuple(first)
	('string 1', 'string 2', '', 'string 3')
	>>> first = itertools.takewhile(LessThanNBlanks(3), sampleData)
	>>> tuple(first)
	('string 1', 'string 2', '', 'string 3', '', 'string 4')
	"""

	def __init__(self, nBlanks):
		self.limit = nBlanks
		self.count = 0

	def __call__(self, arg):
		self.count += not arg
		if self.count > self.limit:
			raise ValueError("Should not call this object anymore.")
		return self.count < self.limit

class LessThanNConsecutiveBlanks(object):
	"""
	An object that when called will return True until n consecutive
	false elements are encountered.

	Can be used with filter or itertools.ifilter, for example:

	>>> import itertools
	>>> sampleData = ['string 1', 'string 2', '', 'string 3', '', 'string 4',
	...     '', '', 'string 5']
	>>> first = itertools.takewhile(LessThanNConsecutiveBlanks(2), sampleData)
	>>> tuple(first)
	('string 1', 'string 2', '', 'string 3', '', 'string 4', '')
	"""

	def __init__(self, nBlanks):
		self.limit = nBlanks
		self.count = 0
		self.last = False

	def __call__(self, arg):
		self.count += not arg
		if arg:
			self.count = 0
		self.last = operator.truth(arg)
		if self.count > self.limit:
			raise ValueError("Should not call this object anymore.")
		return self.count < self.limit

class splitter(object):
	"""
	object that will split a string with the given arguments for each call.

	>>> s = splitter(',')
	>>> list(s('hello, world, this is your, master calling'))
	['hello', ' world', ' this is your', ' master calling']
	"""

	def __init__(self, sep=None):
		self.sep = sep

	def __call__(self, s):
		lastIndex = 0
		while True:
			nextIndex = s.find(self.sep, lastIndex)
			if nextIndex != -1:
				yield s[lastIndex:nextIndex]
				lastIndex = nextIndex + 1
			else:
				yield s[lastIndex:]
				break

def grouper_nofill_str(n, iterable):
	"""
	Take a sequence and break it up into chunks of the specified size.
	The last chunk may be smaller than size.

	This works very similar to grouper_nofill, except
	it works with strings as well.

	>>> tuple(grouper_nofill_str(3, 'foobarbaz'))
	('foo', 'bar', 'baz')

	You can still use it on non-strings too if you like.

	>>> tuple(grouper_nofill_str(42, []))
	()

	>>> tuple(grouper_nofill_str(3, list(range(10))))
	([0, 1, 2], [3, 4, 5], [6, 7, 8], [9])
	"""
	res = more_itertools.chunked(iterable, n)
	if isinstance(iterable, six.string_types):
		res = (''.join(item) for item in res)
	return res

def infinite_call(f):
	"""
	Perpetually yield the result of calling function f.

	>>> counter = itertools.count()
	>>> get_next = functools.partial(next, counter)
	>>> numbers = infinite_call(get_next)
	>>> next(numbers)
	0
	>>> next(numbers)
	1
	"""
	return (f() for _ in itertools.repeat(None))

def infiniteCall(f, *args):
	warnings.warn("Use infinite_call")
	return infinite_call(functools.partial(f, *args))

class Counter(object):
	"""
	Wrap an iterable in an object that stores the count of items
	that pass through it.

	>>> items = Counter(range(20))
	>>> values = list(items)
	>>> items.count
	20
	"""

	def __init__(self, i):
		self.count = 0
		self._orig_iter = iter(i)

	def __iter__(self):
		return self

	def __next__(self):
		result = next(self._orig_iter)
		self.count += 1
		return result
	next = __next__

	def GetCount(self):
		warnings.warn(
			"Use count attribute directly", DeprecationWarning,
			stacklevel=2)
		return self.count

# todo, factor out caching capability

class iterable_test(dict):
	def __init__(self, ignore_classes=six.string_types + (six.binary_type,)):
		"""ignore_classes must include str, because if a string
		is iterable, so is a single character, and the routine runs
		into an infinite recursion"""
		warnings.warn(
			"Slated for removal", DeprecationWarning, stacklevel=2)
		assert set(six.string_types) <= set(ignore_classes), (
			'str must be in ignore_classes')
		self.ignore_classes = ignore_classes

	def __getitem__(self, candidate):
		return dict.get(self, type(candidate)) or self._test(candidate)

	def _test(self, candidate):
		try:
			if isinstance(candidate, tuple(self.ignore_classes)):
				raise TypeError
			iter(candidate)
			result = True
		except TypeError:
			result = False
		self[type(candidate)] = result
		return result

def iflatten(subject, test=None):
	if test is None:
		test = iterable_test()
	if not test[subject]:
		yield subject
	else:
		for elem in subject:
			for subelem in iflatten(elem, test):
				yield subelem

def flatten(subject, test=None):
	"""
	*Deprecated*: Use more_itertools.collapse instead.
	"""
	warnings.warn(
		"Use more_itertools.collapse instead",
		DeprecationWarning,
		stacklevel=2)
	return list(more_itertools.collapse(subject, base_type=(bytes,)))

def empty():
	"""
	An empty iterator.
	"""
	return iter(tuple())

def is_empty(iterable):
	"""
	Return whether the iterable is empty or not. Consumes at most one item
	from the iterator to test.

	>>> is_empty(iter(range(0)))
	True
	>>> is_empty(iter(range(1)))
	False
	"""
	try:
		next(iter(iterable))
	except StopIteration:
		return True
	return False

class Reusable(object):
	"""
	An iterator that may be reset and reused.

	>>> ri = Reusable(range(3))
	>>> tuple(ri)
	(0, 1, 2)
	>>> next(ri)
	0
	>>> tuple(ri)
	(1, 2)
	>>> next(ri)
	0
	>>> ri.reset()
	>>> tuple(ri)
	(0, 1, 2)
	"""

	def __init__(self, iterable):
		self.__saved = iterable
		self.reset()

	def __iter__(self):
		return self

	def reset(self):
		"""
		Resets the iterator to the start.

		Any remaining values in the current iteration are discarded.
		"""
		self.__iterator, self.__saved = itertools.tee(self.__saved)

	def __next__(self):
		try:
			return next(self.__iterator)
		except StopIteration:
			# we're still going to raise the exception, but first
			#  reset the iterator so it's good for next time
			self.reset()
			raise
	next = __next__

def every_other(iterable):
	"""
	Yield every other item from the iterable

	>>> ' '.join(every_other('abcdefg'))
	'a c e g'
	"""
	items = iter(iterable)
	while True:
		try:
			yield next(items)
			next(items)
		except StopIteration:
			return

def remove_duplicates(iterable, key=None):
	"""
	Given an iterable with items that may come in as sequential duplicates,
	remove those duplicates.

	Unlike unique_justseen, this function does not remove triplicates.

	>>> ' '.join(remove_duplicates('abcaabbccaaabbbcccbcbc'))
	'a b c a b c a a b b c c b c b c'
	>>> ' '.join(remove_duplicates('aaaabbbbb'))
	'a a b b b'
	"""
	return itertools.chain.from_iterable(six.moves.map(
		every_other, six.moves.map(
			operator.itemgetter(1),
			itertools.groupby(iterable, key)
		)))

def skip_first(iterable):
	"""
	Skip the first element of an iterable

	>>> tuple(skip_first(range(10)))
	(1, 2, 3, 4, 5, 6, 7, 8, 9)
	"""
	return itertools.islice(iterable, 1, None)

def peek(iterable):
	"""
	Get the next value from an iterable, but also return an iterable
	that will subsequently return that value and the rest of the
	original iterable.

	>>> l = iter([1,2,3])
	>>> val, l = peek(l)
	>>> val
	1
	>>> list(l)
	[1, 2, 3]
	"""
	peeker, original = itertools.tee(iterable)
	return next(peeker), original

class Peekable(object):
	"""
	Wrapper for a traditional iterable to give it a peek attribute.

	>>> nums = Peekable(range(2))
	>>> nums.peek()
	0
	>>> nums.peek()
	0
	>>> next(nums)
	0
	>>> nums.peek()
	1
	>>> next(nums)
	1
	>>> nums.peek()
	Traceback (most recent call last):
	...
	StopIteration

	Peekable should accept an iterable and not just an iterator.

	>>> list(Peekable(range(2)))
	[0, 1]
	"""
	def __new__(cls, iterator):
		# if the iterator is already 'peekable', return it; otherwise
		# wrap it
		if hasattr(iterator, 'peek'):
			return iterator
		else:
			return object.__new__(cls)

	def __init__(self, iterator):
		self.iterator = iter(iterator)

	def __iter__(self):
		return self

	def __next__(self):
		return next(self.iterator)
	next = __next__

	def peek(self):
		result, self.iterator = peek(self.iterator)
		return result

def takewhile_peek(predicate, iterable):
	"""
	Like takewhile, but takes a peekable iterable and doesn't
	consume the non-matching item.

	>>> items = Peekable(range(10))
	>>> is_small = lambda n: n < 4

	>>> small_items = takewhile_peek(is_small, items)

	>>> list(small_items)
	[0, 1, 2, 3]

	>>> list(items)
	[4, 5, 6, 7, 8, 9]

	>>> empty = takewhile_peek(is_small, Peekable([]))
	>>> list(empty)
	[]

	>>> items = Peekable([3])
	>>> small_items = takewhile_peek(is_small, items)
	>>> list(small_items)
	[3]
	>>> list(items)
	[]

	>>> items = Peekable([4])
	>>> small_items = takewhile_peek(is_small, items)
	>>> list(small_items)
	[]
	>>> list(items)
	[4]
	"""
	while True:
		try:
			if not predicate(iterable.peek()):
				break
			yield next(iterable)
		except StopIteration:
			break

def first(iterable, *args):
	"""
	Return the first item from the iterable.

	>>> first(range(11))
	0
	>>> first([3,2,1])
	3
	>>> iter = range(11)
	>>> first(iter)
	0

	Raises StopIteration if no value is present.

	>>> first([])
	Traceback (most recent call last):
	...
	StopIteration

	Pass a default to be used when iterable is empty.

	>>> first([], None)

	"""
	iterable = iter(iterable)
	return next(iterable, *args)

def last(iterable):
	"""
	Return the last item from the iterable, discarding the rest.

	>>> last(range(20))
	19
	>>> last([])
	Traceback (most recent call last):
	...
	ValueError: Iterable contains no items
	"""
	for item in iterable:
		pass
	try:
		return item
	except NameError:
		raise ValueError("Iterable contains no items")

def one(item):
	"""
	Return the first element from the iterable, but raise an exception
	if elements remain in the iterable after the first.

	>>> one(['val'])
	'val'

	>>> one(['val', 'other'])
	Traceback (most recent call last):
	...
	ValueError: ...values to unpack...

	>>> one([])
	Traceback (most recent call last):
	...
	ValueError: ...values to unpack...

	>>> numbers = itertools.count()
	>>> one(numbers)
	Traceback (most recent call last):
	...
	ValueError: ...values to unpack...
	>>> next(numbers)
	2
	"""
	result, = item
	return result

def nwise(iter, n):
	"""
	Like pairwise, except returns n-tuples of adjacent items.
	s -> (s0,s1,...,sn), (s1,s2,...,s(n+1)), ...
	"""
	iterset = [iter]
	while len(iterset) < n:
		iterset[-1:] = itertools.tee(iterset[-1])
		next(iterset[-1], None)
	return six.moves.zip(*iterset)

def window(iter, pre_size=1, post_size=1):
	"""
	Given an iterable, return a new iterable which yields triples of
	(pre, item, post), where pre and post are the items preceeding and
	following the item (or None if no such item is appropriate). pre
	and post will always be pre_size and post_size in length.

	>>> example = window(range(10), pre_size=2)
	>>> pre, item, post = next(example)
	>>> pre
	(None, None)
	>>> post
	(1,)
	>>> next(example)
	((None, 0), 1, (2,))
	>>> list(example)[-1]
	((7, 8), 9, (None,))
	"""
	pre_iter, iter = itertools.tee(iter)
	pre_iter = itertools.chain((None,) * pre_size, pre_iter)
	pre_iter = nwise(pre_iter, pre_size)
	post_iter, iter = itertools.tee(iter)
	post_iter = itertools.chain(post_iter, (None,) * post_size)
	post_iter = nwise(post_iter, post_size)
	next(post_iter, None)
	return six.moves.zip(pre_iter, iter, post_iter)

class IterSaver(object):
	def __init__(self, n, iterable):
		self.n = n
		self.iterable = iterable
		self.buffer = collections.deque()

	def __next__(self):
		while len(self.buffer) <= self.n:
			self.buffer.append(next(self.iterable))
		return self.buffer.popleft()
	next = __next__

def partition_items(count, bin_size):
	"""
	Given the total number of items, determine the number of items that
	can be added to each bin with a limit on the bin size.

	So if you want to partition 11 items into groups of 3, you'll want
	three of three and one of two.

	>>> partition_items(11, 3)
	[3, 3, 3, 2]

	But if you only have ten items, you'll have two groups of three and
	two of two.

	>>> partition_items(10, 3)
	[3, 3, 2, 2]
	"""
	num_bins = int(math.ceil(count / float(bin_size)))
	bins = [0] * num_bins
	for i in range(count):
		bins[i % num_bins] += 1
	return bins

def balanced_rows(n, iterable, fillvalue=None):
	"""
	Like grouper, but balance the rows to minimize fill per row.
	balanced_rows(3, 'ABCDEFG', 'x') --> ABC DEx FGx"
	"""
	iterable, iterable_copy = itertools.tee(iterable)
	count = len(tuple(iterable_copy))
	for allocation in partition_items(count, n):
		row = itertools.islice(iterable, allocation)
		if allocation < n:
			row = itertools.chain(row, [fillvalue])
		yield tuple(row)

def reverse_lists(lists):
	"""
	>>> reverse_lists([[1,2,3], [4,5,6]])
	[[3, 2, 1], [6, 5, 4]]
	"""

	return list(map(list, map(reversed, lists)))

def always_iterable(item):
	"""
	Given an object, always return an iterable. If the item is not
	already iterable, return a tuple containing only the item. If item is
	None, an empty iterable is returned.

	>>> always_iterable([1,2,3])
	<list_iterator...>
	>>> always_iterable('foo')
	<tuple_iterator...>
	>>> always_iterable(None)
	<tuple_iterator...>
	>>> always_iterable(range(10))
	<range_iterator...>
	>>> def _test_func(): yield "I'm iterable"
	>>> print(next(always_iterable(_test_func())))
	I'm iterable

	Although mappings are iterable, treat each like a singleton, as
	it's more like an object than a sequence.

	>>> next(always_iterable(dict(a=1)))
	{'a': 1}
	"""
	base_types = six.text_type, bytes, collections.abc.Mapping
	return more_itertools.always_iterable(item, base_type=base_types)

def suppress_exceptions(callables, *exceptions):
	"""
	Call each callable in callables, suppressing any exceptions supplied. If
	no exception classes are supplied, all Exceptions will be suppressed.

	>>> import functools
	>>> c1 = functools.partial(int, 'a')
	>>> c2 = functools.partial(int, '10')
	>>> list(suppress_exceptions((c1, c2)))
	[10]
	>>> list(suppress_exceptions((c1, c2), KeyError))
	Traceback (most recent call last):
	...
	ValueError: invalid literal for int() with base 10: 'a'
	"""
	if not exceptions:
		exceptions = Exception,
	for callable in callables:
		try:
			yield callable()
		except exceptions:
			pass

def apply(func, iterable):
	"""
	Like 'map', invoking func on each item in the iterable,
	except return the original item and not the return
	value from the function.

	Useful when the side-effect of the func is what's desired.

	>>> res = apply(print, range(1, 4))
	>>> list(res)
	1
	2
	3
	[1, 2, 3]
	"""
	for item in iterable:
		func(item)
		yield item

def list_or_single(iterable):
	"""
	Given an iterable, return the items as a list. If the iterable contains
	exactly one item, return that item. Correlary function to always_iterable.
	"""
	warnings.warn("Use maybe_single", DeprecationWarning, stacklevel=2)
	return maybe_single(list(iterable))

def maybe_single(sequence):
	"""
	Given a sequence, if it contains exactly one item,
	return that item, otherwise return the sequence.
	Correlary function to always_iterable.

	>>> maybe_single(tuple('abcd'))
	('a', 'b', 'c', 'd')
	>>> maybe_single(['a'])
	'a'
	"""
	try:
		single, = sequence
	except ValueError:
		return sequence
	return single

def self_product(iterable):
	"""
	Return the cross product of the iterable with itself.

	>>> list(self_product([1, 2, 3]))
	[(1, 1), (1, 2), ..., (3, 3)]
	"""
	return itertools.product(*itertools.tee(iterable))

def duplicates(*iterables, **kwargs):
	"""
	Yield duplicate items from any number of sorted iterables of items

	>>> items_a = [1, 2, 3]
	>>> items_b = [0, 3, 4, 5, 6]
	>>> list(duplicates(items_a, items_b))
	[(3, 3)]

	It won't behave as you expect if the iterables aren't ordered

	>>> items_b.append(1)
	>>> list(duplicates(items_a, items_b))
	[(3, 3)]
	>>> list(duplicates(items_a, sorted(items_b)))
	[(1, 1), (3, 3)]

	This function is most interesting when it's operating on a key
	of more complex objects.

	>>> items_a = [dict(email='joe@example.com', id=1)]
	>>> items_b = [dict(email='joe@example.com', id=2), dict(email='other')]
	>>> dupe, = duplicates(items_a, items_b, key=operator.itemgetter('email'))
	>>> dupe[0]['email'] == dupe[1]['email'] == 'joe@example.com'
	True
	>>> dupe[0]['id']
	1
	>>> dupe[1]['id']
	2
	"""
	key = kwargs.pop('key', lambda x: x)
	assert not kwargs
	zipped = more_itertools.collate(*iterables, key=key)
	grouped = itertools.groupby(zipped, key=key)
	groups = (
		tuple(g)
		for k, g in grouped
	)

	def has_dupes(group):
		return len(group) > 1
	return filter(has_dupes, groups)

def assert_ordered(iterable, key=lambda x: x, comp=operator.le):
	"""
	Assert that for all items in the iterable, they're in order based on comp

	>>> list(assert_ordered(range(5)))
	[0, 1, 2, 3, 4]
	>>> list(assert_ordered(range(5), comp=operator.ge))
	Traceback (most recent call last):
	...
	AssertionError: 0 < 1
	>>> list(assert_ordered(range(5, 0, -1), key=operator.neg))
	[5, 4, 3, 2, 1]
	"""
	err_tmpl = (
		"{pair[0]} > {pair[1]}" if comp is operator.le else
		"{pair[0]} < {pair[1]}" if comp is operator.ge else
		"not {comp} {pair}"
	)
	for pair in more_itertools.pairwise(iterable):
		keyed = tuple(map(key, pair))
		assert comp(*keyed), err_tmpl.format(**locals())
		yield pair[0]
	yield pair[1]

def collate_revs(old, new, key=lambda x: x, merge=lambda old, new: new):
	"""
	Given revision sets old and new, each containing a series
	of revisions of some set of objects, collate them based on
	these rules:

	- all items from each set are yielded in stable order
	- items in old are yielded first
	- items in new are yielded last
	- items that match are yielded in the order in which they
	appear, giving preference to new

	Items match based on the 'key' parameter (identity by default).

	Items are merged using the 'merge' function, which accepts the old
	and new items to be merged (returning new by default).

	This algorithm requires fully materializing both old and new in memory.

	>>> rev1 = ['a', 'b', 'c']
	>>> rev2 = ['a', 'd', 'c']
	>>> result = list(collate_revs(rev1, rev2))

	'd' must appear before 'c'
	>>> result.index('d') < result.index('c')
	True

	'b' must appear before 'd' because it came chronologically
	first.
	>>> result.index('b') < result.index('d')
	True

	>>> result
	['a', 'b', 'd', 'c']

	>>> list(collate_revs(['a', 'b', 'c'], ['d']))
	['a', 'b', 'c', 'd']

	>>> list(collate_revs(['b', 'a'], ['a', 'b']))
	['a', 'b']

	>>> list(collate_revs(['a', 'c'], ['a', 'b', 'c']))
	['a', 'b', 'c']

	Given two sequences of things out of order, regardless
	of which order in which the items are merged, all
	keys should always be merged.

	>>> from more_itertools import consume
	>>> left_items = ['a', 'b', 'c']
	>>> right_items = ['a', 'c', 'b']
	>>> consume(collate_revs(left_items, right_items, merge=print))
	a a
	c c
	b b
	>>> consume(collate_revs(right_items, left_items, merge=print))
	a a
	b b
	c c

	The merge should not suppress non-True items:

	>>> consume(collate_revs([0, 1, 2, None, ''], [0, None, ''], merge=print))
	None None
	<BLANKLINE>
	0 0

	"""
	missing = object()

	def maybe_merge(*items):
		"""
		Merge any non-null items
		"""
		def not_missing(ob):
			return ob is not missing

		return functools.reduce(merge, filter(not_missing, items))

	new_items = collections.OrderedDict(
		(key(el), el)
		for el in new
	)
	old_items = collections.OrderedDict(
		(key(el), el)
		for el in old
	)

	# use the old_items as a reference
	for old_key, old_item in _mutable_iter(old_items):
		if old_key not in new_items:
			yield old_item
			continue

		# yield all new items that appear before the matching key
		before, match_new, new_items = _swap_on_miss(
			partition_dict(new_items, old_key))
		for new_key, new_item in before.items():
			# ensure any new keys are merged with previous items if
			# they exist
			yield maybe_merge(new_item, old_items.pop(new_key, missing))
		yield merge(old_item, match_new)

	# finally, yield whatever is leftover
	# yield from new_items.values()
	for item in new_items.values():
		yield item

def _mutable_iter(dict):
	"""
	Iterate over items in the dict, yielding the first one, but allowing
	it to be mutated during the process.
	>>> d = dict(a=1)
	>>> it = _mutable_iter(d)
	>>> next(it)
	('a', 1)
	>>> d
	{}
	>>> d.update(b=2)
	>>> list(it)
	[('b', 2)]
	"""
	while dict:
		prev_key = next(iter(dict))
		yield prev_key, dict.pop(prev_key)

def _swap_on_miss(partition_result):
	"""
	Given a partition_dict result, if the partition missed, swap
	the before and after.
	"""
	before, item, after = partition_result
	return (before, item, after) if item else (after, item, before)

def partition_dict(items, key):
	"""
	Given an ordered dictionary of items and a key in that dict,
	return an ordered dict of items before, the keyed item, and
	an ordered dict of items after.

	>>> od = collections.OrderedDict(zip(range(5), 'abcde'))
	>>> before, item, after = partition_dict(od, 3)
	>>> before
	OrderedDict([(0, 'a'), (1, 'b'), (2, 'c')])
	>>> item
	'd'
	>>> after
	OrderedDict([(4, 'e')])

	Like string.partition, if the key is not found in the items,
	the before will contain all items, item will be None, and
	after will be an empty iterable.

	>>> before, item, after = partition_dict(od, -1)
	>>> before
	OrderedDict([(0, 'a'), ..., (4, 'e')])
	>>> item
	>>> list(after)
	[]
	"""
	def unmatched(pair):
		test_key, item, = pair
		return test_key != key

	items_iter = iter(items.items())
	item = items.get(key)
	left = collections.OrderedDict(itertools.takewhile(unmatched, items_iter))
	right = collections.OrderedDict(items_iter)
	return left, item, right


from django.template import TemplateSyntaxError
from django.template.base import Token
import re

__all__ = (
    'parse_as_var', 'parse_token_kwargs'
)

kwarg_re = re.compile('^(?P<name>\w+)=')

def parse_as_var(parser, token):
    """
    Parse the remainder of the token, to find a "as varname" statement.

    :param parser: The "parser" object that ``@register.tag`` provides.
    :type parser: :class:`~django.template.Parser`
    :param token: The "token" object that ``@register.tag`` provides.
    :type token: :class:`~django.template.Token` or splitted bits
    """
    if isinstance(token, Token):
        bits = token.split_contents()
    else:
        bits = token

    as_var = None
    if len(bits) > 2 and bits[-2] == 'as':
        bits = bits[:]
        as_var = bits.pop()
        bits.pop()  # as keyword

    return bits, as_var

def parse_token_kwargs(parser, token, allowed_kwargs=None, compile_args=True, compile_kwargs=True):
    """
    Allow the template tag arguments to be like a normal Python function, with *args and **kwargs.

    :param parser: The "parser" object that ``@register.tag`` provides.
    :type parser: :class:`~django.template.Parser`
    :param token: The "token" object that ``@register.tag`` provides.
    :type token: :class:`~django.template.Token` or splitted bits
    :param compile_args: Whether the arguments should be compiled using :func:`parser.compile_filter <django.template.Parser.compile_filter>`.
    :param compile_kwargs: Whether the keyword arguments should be compiled using :func:`parser.compile_filter <django.template.Parser.compile_filter>`.
    :param allowed_kwargs: A list of allowed keyword arguments. A value of ``None`` disables the check.
    :type allowed_kwargs: tuple
    :return: The tag name, arguments and keyword arguments.
    :rtype: tuple(tag_name, args, kwargs)
    """
    if isinstance(token, Token):
        bits = token.split_contents()
    else:
        bits = token

    expect_kwarg = False
    args = []
    kwargs = {}
    prev_bit = None

    tag_name = bits[0]

    for bit in bits[1::]:
        kwarg_match = kwarg_re.match(bit)
        if kwarg_match:
            # Keyword argument
            expect_kwarg = True
            (name, expr) = bit.split('=', 2)
            kwargs[name] = parser.compile_filter(expr) if compile_kwargs else expr
        else:
            # Still at positioned arguments.
            if expect_kwarg:
                raise TemplateSyntaxError("{0} tag may not have a non-keyword argument ({1}) after a keyword argument ({2}).".format(bits[0], bit, prev_bit))
            args.append(parser.compile_filter(bit) if compile_args else bit)

        prev_bit = bit

    # Validate the allowed arguments, to make things easier for template developers
    if allowed_kwargs is not None and kwargs:
        if not allowed_kwargs:
            raise TemplateSyntaxError("The option %s=... cannot be used in '%s'.\nNo keyword arguments are allowed.")

        for name in kwargs:
            if name not in allowed_kwargs:
                raise TemplateSyntaxError("The option %s=... cannot be used in '%s'.\nPossible options are: %s." % (name, bits[0], ", ".join(allowed_kwargs)))

    return tag_name, args, kwargs


"""
Decorators to write template tags
"""

def template_tag(library, name):
    """
    Decorator to register class tags

    :param library: The template tag library, typically instantiated as ``register = Library()``.
    :type library: :class:`~django.template.Library`
    :param name: The name of the template tag
    :type name: str

    Example:

    .. code-block:: python

        @template_tag(register, 'my_tag')
        class MyTag(BaseNode):
            pass
    """
    def _inner(cls):
        if hasattr(cls, 'parse'):
            compile_function = cls.parse
        else:
            # Hope that it's either a function, or cls with __init__(self, parser, token) method.
            compile_function = cls

        library.tag(name, compile_function)
        return cls  # Return the class body to keep it in the namespace of the module

    return _inner


from bitmerchant.wallet import (
    Wallet as HDWallet
)
from binascii import hexlify, unhexlify
from .utils import extract_bin_chain_path, bip32_serialize
from .configs import EXTENDED_PUBLIC_KEY_VERSION_BYTES as version_bytes

import keylib
from keylib.key_formatting import encode_pubkey as encode_public_key

class PublicKeychain():
    def __init__(self, public_keychain):
        if isinstance(public_keychain, HDWallet):
            self.hdkeychain = public_keychain
        elif isinstance(public_keychain, (str, unicode)):
            self.hdkeychain = HDWallet.deserialize(public_keychain)
        else:
            raise ValueError('public keychain must be a string')

    def __str__(self):
        return self.hdkeychain.serialize_b58(private=False)

    def child(self, index):
        child_keychain = self.hdkeychain.get_child(
            index, is_prime=False, as_private=False)
        return PublicKeychain(child_keychain)

    def descendant(self, chain_path):
        """ A descendant is a child many steps down.
        """
        public_child = self.hdkeychain
        chain_step_bytes = 4
        max_bits_per_step = 2**31
        chain_steps = [
            int(chain_path[i:i+chain_step_bytes*2], 16) % max_bits_per_step
            for i in range(0, len(chain_path), chain_step_bytes*2)
        ]
        for step in chain_steps:
            public_child = public_child.get_child(step)

        return PublicKeychain(public_child)

    def public_key(self, compressed=True):
        return self.hdkeychain.get_public_key_hex(compressed=compressed)

    def address(self):
        return str(self.hdkeychain.to_address())

    @classmethod
    def from_public_key(cls, public_key, chain_path='\x00'*32, depth=0,
                        fingerprint='\x00'*4, child_index=0):
        public_key_bytes = encode_public_key(public_key, 'bin_compressed')
        chain_path = extract_bin_chain_path(chain_path)
        keychain_parts = (version_bytes, depth, fingerprint,
                          child_index, chain_path, public_key_bytes)
        public_keychain_string = bip32_serialize(keychain_parts)
        return PublicKeychain(public_keychain_string)


from binascii import unhexlify
from keylib.key_formatting import encode, decode, from_int_to_byte, from_byte_to_int, changebase, bin_dbl_sha256

MAINNET_PRIVATE = b'\x04\x88\xAD\xE4'
MAINNET_PUBLIC = b'\x04\x88\xB2\x1E'
TESTNET_PRIVATE = b'\x04\x35\x83\x94'
TESTNET_PUBLIC = b'\x04\x35\x87\xCF'
PRIVATE = [MAINNET_PRIVATE, TESTNET_PRIVATE]
PUBLIC = [MAINNET_PUBLIC, TESTNET_PUBLIC]

def extract_bin_chain_path(chain_path):
    if len(chain_path) == 64:
        return unhexlify(chain_path)
    elif len(chain_path) == 32:
        return chain_path
    else:
        raise ValueError('Invalid chain path')

def hash_to_int(x):
    if len(x) in [40, 64]:
        # decode as hex string
        return decode(x, 16)

    # decode as byte string
    return decode(x, 256)

def bip32_serialize(rawtuple):
    """
    Derived from code from pybitcointools (https://github.com/vbuterin/pybitcointools)
    by Vitalik Buterin
    """
    vbytes, depth, fingerprint, i, chaincode, key = rawtuple
    i = encode(i, 256, 4)
    chaincode = encode(hash_to_int(chaincode), 256, 32)
    keydata = b'\x00'  +key[:-1] if vbytes in PRIVATE else key
    bindata = vbytes + from_int_to_byte(depth % 256) + fingerprint + i + chaincode + keydata
    return changebase(bindata + bin_dbl_sha256(bindata)[:4], 256, 58)

def bip32_deserialize(data):
    """
    Derived from code from pybitcointools (https://github.com/vbuterin/pybitcointools)
    by Vitalik Buterin
    """
    dbin = changebase(data, 58, 256)
    if bin_dbl_sha256(dbin[:-4])[:4] != dbin[-4:]:
        raise Exception("Invalid checksum")
    vbytes = dbin[0:4]
    depth = from_byte_to_int(dbin[4])
    fingerprint = dbin[5:9]
    i = decode(dbin[9:13], 256)
    chaincode = dbin[13:45]
    key = dbin[46:78]+b'\x01' if vbytes in PRIVATE else dbin[45:78]
    return (vbytes, depth, fingerprint, i, chaincode, key)