# Copyright (c) 2015 Heiko Hees

from collections import Counter

from bitcoin.main import encode_privkey
from ethereum.blocks import Block
from ethereum.utils import big_endian_to_int, zpad, int_to_32bytearray
from bitcoin import encode_pubkey, N, P
import rlp
from rlp.sedes import big_endian_int, binary
from rlp.sedes import CountableList
from rlp.utils import encode_hex
from ethereum.blocks import BlockHeader
from ethereum.transactions import Transaction
from secp256k1 import PrivateKey, PublicKey, ALL_FLAGS

from hydrachain.utils import sha3, phx

def ishash(h):
    return isinstance(h, bytes) and len(h) == 32

def isaddress(a):
    return isinstance(a, bytes) and len(a) == 20

class InvalidSignature(Exception):
    pass

class RLPHashable(rlp.Serializable):

    @property
    def hash(self):
        return sha3(rlp.encode(self))

    def __eq__(self, other):
        return isinstance(other, self.__class__) and self.hash == other.hash

    def __hash__(self):
        return big_endian_to_int(self.hash)

    def __ne__(self, other):
        return not self.__eq__(other)

    def __repr__(self):
        try:
            return '<%s(%s)>' % (self.__class__.__name__, encode_hex(self.hash)[:4])
        except:
            return '<%s>' % (self.__class__.__name__)

class MissingSignatureError(Exception):
    pass

class Signed(RLPHashable):

    fields = [
        ('v', big_endian_int),
        ('r', big_endian_int),
        ('s', big_endian_int),
    ]

    _sender = None

    def __init__(self, *args, **kargs):
        super(Signed, self).__init__(*args, **kargs)

    def sign(self, privkey):
        """Sign this with a private key"""
        if self.v:
            raise InvalidSignature("already signed")

        if privkey in (0, '', '\x00' * 32):
            raise InvalidSignature("Zero privkey cannot sign")
        rawhash = sha3(rlp.encode(self, self.__class__.exclude(['v', 'r', 's'])))

        if len(privkey) == 64:
            privkey = encode_privkey(privkey, 'bin')

        pk = PrivateKey(privkey, raw=True)
        signature = pk.ecdsa_recoverable_serialize(pk.ecdsa_sign_recoverable(rawhash, raw=True))

        signature = signature[0] + chr(signature[1])

        self.v = ord(signature[64]) + 27
        self.r = big_endian_to_int(signature[0:32])
        self.s = big_endian_to_int(signature[32:64])

        self._sender = None
        return self

    @property
    def sender(self):
        if not self._sender:
            self._sender = self.recover_sender()
        return self._sender

    def recover_sender(self):
        if self.v:
            if self.r >= N or self.s >= P or self.v < 27 or self.v > 28 \
               or self.r == 0 or self.s == 0:
                raise InvalidSignature()
            rlpdata = rlp.encode(self, self.__class__.exclude(['v', 'r', 's']))
            rawhash = sha3(rlpdata)
            pk = PublicKey(flags=ALL_FLAGS)
            try:
                pk.public_key = pk.ecdsa_recover(
                    rawhash,
                    pk.ecdsa_recoverable_deserialize(
                        zpad(
                            "".join(chr(c) for c in int_to_32bytearray(self.r)),
                            32
                        ) + zpad(
                            "".join(chr(c) for c in int_to_32bytearray(self.s)),
                            32
                        ),
                        self.v - 27
                    ),
                    raw=True
                )
                pub = pk.serialize(compressed=False)
            except Exception:
                raise InvalidSignature()
            if pub[1:] == "\x00" * 32:
                raise InvalidSignature()
            pub = encode_pubkey(pub, 'bin')
            return sha3(pub[1:])[-20:]

    @property
    def hash(self):
        "signatures are non deterministic"
        if self.sender is None:
            raise MissingSignatureError()

        class HashSerializable(rlp.Serializable):
            fields = [(field, sedes) for field, sedes in self.fields
                      if field not in ('v', 'r', 's')] + [('_sender', binary)]
            _sedes = None
        return sha3(rlp.encode(self, HashSerializable))

# Votes

class Vote(Signed):

    """A signed Vote"""

    fields = [
        ('height', big_endian_int),
        ('round', big_endian_int),
        ('blockhash', binary),
    ] + Signed.fields

    def __init__(self, height, round, blockhash='', v=0, r=0, s=0):
        super(Vote, self).__init__(height, round, blockhash, v=v, r=r, s=s)

        # restore class when deserialized
        if blockhash:
            assert ishash(blockhash)
            self.blockhash = blockhash
            self.__class__ = VoteBlock
        else:
            self.__class__ = VoteNil

    def __repr__(self):
        return '<%s(S:%s BH:%s)>' % (self.__class__.__name__,
                                     phx(self.sender), phx(self.blockhash))

    @property
    def hr(self):
        return self.height, self.round

class VoteBlock(Vote):
    pass

class VoteNil(Vote):
    pass

# LockSets

class InvalidVoteError(Exception):
    pass

class DoubleVotingError(InvalidVoteError):
    pass

class LockSet(RLPHashable):  # careful, is mutable!

    fields = [
        ('num_eligible_votes', big_endian_int),
        ('votes', CountableList(Vote))
    ]

    processed = False

    def __init__(self, num_eligible_votes, votes=None):
        self.num_eligible_votes = num_eligible_votes
        self.votes = []
        for v in votes or []:
            self.add(v)

    # @property
    # def size(self):
    #     return len(self.votes) * 67 + 5

    def copy(self):
        return LockSet(self.num_eligible_votes, self.votes)

    @property
    def state(self):
        if not self.is_valid:
            s = 'I'
        elif self.has_quorum:
            s = 'Q'
        elif self.has_quorum_possible:
            s = 'P'
        elif self.has_noquorum:
            s = 'N'
        else:
            raise Exception('no valid state')
        return '%s:%d' % (s, len(self))

    def __repr__(self):
        if self.votes:
            return '<LockSet(%s H:%d R:%d)>' % (self.state, self.height, self.round)
        return '<LockSet(I:0)>'

    def add(self, vote, force_replace=False):
        assert isinstance(vote, Vote)
        if not vote.sender:
            raise InvalidVoteError('no signature')
        if vote not in self.votes:
            if len(self) and self.hr != vote.hr:
                raise InvalidVoteError('inconsistent height, round')
            signee = self.signee
            if vote.sender in signee:
                if not force_replace:
                    raise DoubleVotingError(vote.sender)  # different votes on the same H,R
                self.votes.remove(self.votes[signee.index(vote.sender)])
            self.votes.append(vote)
            return True

    def __len__(self):
        return len(self.votes)

    def __iter__(self):
        return iter(self.votes)

    @property
    def signee(self):
        return [v.sender for v in self.votes]

    def blockhashes(self):
        assert self.is_valid
        c = Counter(v.blockhash for v in self.votes if isinstance(v, VoteBlock))
        # deterministc sort necessary
        return sorted(c.most_common(), cmp=lambda a, b: cmp((b[1], b[0]), (a[1], a[0])))

    @property
    def hr(self):
        """compute (height,round)
        We might have multiple rounds before we see consensus for a certain height.
        If everything is good, round should always be 0.
        """
        assert len(self), 'no votes, can not determine height'
        h = set([(v.height, v.round) for v in self.votes])
        assert len(h) == 1, len(h)
        return h.pop()

    height = property(lambda self: self.hr[0])
    round = property(lambda self: self.hr[1])

    @property
    def is_valid(self):
        return len(self) > 2 / 3. * self.num_eligible_votes and self.hr

    @property
    def has_quorum(self):
        """
        we've seen +2/3 of all eligible votes voting for one block.
        there is a quorum.
        """
        assert self.is_valid
        bhs = self.blockhashes()
        if bhs and bhs[0][1] > 2 / 3. * self.num_eligible_votes:
            return bhs[0][0]

    @property
    def has_noquorum(self):
        """
        less than 1/3 of the known votes are on the same block
        """
        assert self.is_valid
        bhs = self.blockhashes()
        if not bhs or bhs[0][1] <= 1 / 3. * self.num_eligible_votes:
            assert not self.has_quorum_possible
            return True

    @property
    def has_quorum_possible(self):
        """
        we've seen +1/3 of all eligible votes voting for one block.
        at least one vote was from a honest node.
        we can assume that this block is agreeable.
        """
        if self.has_quorum:
            return
        assert self.is_valid  # we could tell that earlier
        bhs = self.blockhashes()
        if bhs and bhs[0][1] > 1 / 3. * self.num_eligible_votes:
            return bhs[0][0]

    def check(self):
        "either invalid or one of quorum, noquorum, quorumpossible"
        if not self.is_valid:
            return True
        test = (self.has_quorum, self.has_quorum_possible, self.has_noquorum)
        assert 1 == len([x for x in test if x is not None])
        return True

############

def genesis_signing_lockset(genesis, privkey):
    """
    in order to avoid a complicated bootstrapping, we define
    the genesis_signing_lockset as a lockset with one vote by any validator.
    """
    v = VoteBlock(0, 0, genesis.hash)
    v.sign(privkey)
    ls = LockSet(num_eligible_votes=1)
    ls.add(v)
    assert ls.has_quorum
    return ls

########

class Ready(Signed):

    """
    Used to sync during the startup sequence
    """
    fields = [
        ('nonce', big_endian_int),
        ('current_lockset', LockSet)
    ] + Signed.fields

    def __init__(self, nonce, current_lockset, v=0, r=0, s=0):
        super(Ready, self).__init__(nonce, current_lockset, v, r, s)

    def __repr__(self):
        return '<Ready(n:{})>'.format(self.nonce)

class InvalidProposalError(Exception):
    pass

class HDCBlockHeader(BlockHeader):

    def check_pow(self, nonce=None):
        return True

class HDCBlock(Block):
    pass

class TransientBlock(rlp.Serializable):

    """A partially decoded, unvalidated block."""

    fields = [
        ('header', HDCBlockHeader),
        ('transaction_list', rlp.sedes.CountableList(Transaction)),
        ('uncles', rlp.sedes.CountableList(BlockHeader))
    ]

    def __init__(self, header, transaction_list, uncles):
        self.header = header
        self.transaction_list = transaction_list
        self.uncles = uncles

    def to_block(self, env, parent=None):
        """Convert the transient block to a :class:`ethereum.blocks.Block`"""
        return Block(self.header, self.transaction_list, self.uncles, env=env, parent=parent)

    @property
    def hash(self):
        """The binary block hash
        This is equivalent to ``header.hash``.
        """
        return sha3(rlp.encode(self.header))

    @property
    def number(self):
        return self.header.number

    @property
    def prevhash(self):
        return self.header.prevhash

class Proposal(Signed):
    pass

class BlockProposal(Proposal):

    fields = [
        ('height', big_endian_int),
        ('round', big_endian_int),
        ('block', TransientBlock),
        ('signing_lockset', LockSet),
        ('round_lockset', LockSet)
    ] + Signed.fields

    def __init__(self, height, round, block, signing_lockset,
                 round_lockset=None, v=0, r=0, s=0):
        """
        if round == 0 the signing_lockset also proves,
        that proposal is eligible and we need not round_lockset
        """
        assert isinstance(block, (Block, TransientBlock))
        assert isinstance(signing_lockset, LockSet)
        assert round_lockset is None or isinstance(round_lockset, LockSet)
        assert round >= 0
        assert height > 0
        if round > 0 and not round_lockset:
            raise InvalidProposalError('R>0 needs a round lockset')
        if round == 0 and round_lockset:
            raise InvalidProposalError('R0 must not have a round lockset')
        self.height = height
        self.round = round
        self.block = block
        self.signing_lockset = signing_lockset
        self.round_lockset = round_lockset or LockSet(0)

        super(BlockProposal, self).__init__(height, round, block, signing_lockset,
                                            self.round_lockset, v, r, s)

        if block.header.number != self.height:
            raise InvalidProposalError('lockset.height / block.number mismatch')
        if self.round_lockset and height != self.round_lockset.height:
            raise InvalidProposalError('height mismatch')
        if not (round > 0 or self.lockset.has_quorum):
            raise InvalidProposalError('R0 lockset == signing lockset needs quorum')
        if not (round > 0 or self.lockset.height == block.header.number - 1):
            raise InvalidProposalError('R0 round lockset must be from previous height')
        if not (round == 0 or round == self.lockset.round + 1):
            raise InvalidProposalError('Rn round lockset must be from previous round')
        if not self.signing_lockset.has_quorum:
            raise InvalidProposalError('signing lockset needs quorum')
        if not (self.signing_lockset.height == self.height - 1):
            raise InvalidProposalError('signing lockset height mismatch')
        if self.round_lockset and not round_lockset.has_noquorum:
            raise InvalidProposalError('at R>0 can only propose if there is a NoQuorum for R-1')

        self.rawhash = sha3(rlp.encode(self, self.__class__.exclude(['v', 'r', 's'])))
        if self.v:  # validate sender == block.coinbase
            assert self.sender

    @property
    def lockset(self):
        return self.round_lockset or self.signing_lockset

    @property
    def sender(self):
        # double check unmutable
        s = super(BlockProposal, self).sender
        if not s:
            raise InvalidProposalError('signature missing')
        assert self.rawhash
        assert self.v
        _rawhash = sha3(rlp.encode(self, self.__class__.exclude(['v', 'r', 's'])))
        assert self.rawhash == _rawhash
        assert len(s) == 20
        assert len(self.block.header.coinbase) == 20
        if s != self.block.header.coinbase:
            raise InvalidProposalError('signature does not match coinbase')
        return s

    def sign(self, privkey):
        super(BlockProposal, self).sign(privkey)
        if self.sender != self.block.header.coinbase:
            raise InvalidProposalError('signature does not match coinbase')

    def validate_votes(self, validators_H, validators_prevH):
        "set of validators may change between heights"
        assert self.sender

        def check(lockset, validators):
            if not lockset.num_eligible_votes == len(validators):
                raise InvalidProposalError('lockset num_eligible_votes mismatch')
            for v in lockset:
                if v.sender not in validators:
                    raise InvalidProposalError('invalid signer')
        if self.round_lockset:
            check(self.round_lockset, validators_H)
        check(self.signing_lockset, validators_prevH)

        return True

    def __repr__(self):
        return "<%s S:%r H:%d R:%d BH:%s>" % (self.__class__.__name__, phx(self._sender),
                                              self.height, self.round, phx(self.blockhash))

    @property
    def blockhash(self):
        return self.block.hash

class VotingInstruction(Proposal):

    fields = [
        ('height', big_endian_int),
        ('round', big_endian_int),
        ('round_lockset', LockSet)
    ] + Signed.fields

    def __init__(self, height, round, round_lockset, v=0, r=0, s=0):
        super(VotingInstruction, self).__init__(height, round, round_lockset, v, r, s)
        if not round > 0:
            raise InvalidProposalError('VotingInstructions must have R>0')
        if not self.lockset.has_quorum_possible:
            raise InvalidProposalError('VotingInstruction requires quorum possible')
        if not (round == self.lockset.round + 1):
            raise InvalidProposalError('Rn round lockset must be from previous round')
        if not (height == self.lockset.height):
            raise InvalidProposalError('height mismatch')
        if not round > 0:
            raise InvalidProposalError('VotingInstructions must have R>0')
        assert round == round_lockset.round + 1
        assert height == round_lockset.height

    @property
    def blockhash(self):
        return self.round_lockset.has_quorum_possible

    @property
    def lockset(self):
        return self.round_lockset

    def __repr__(self):
        return "<%s %r BH:%s>" % (self.__class__.__name__, phx(self.sender), phx(self.blockhash))

    def validate_votes(self, validators_H):
        "set of validators may change between heights"
        assert self.sender
        if not self.round_lockset.num_eligible_votes == len(validators_H):
            raise InvalidProposalError('round_lockset num_eligible_votes mismatch')
        for v in self.round_lockset:
            if v.sender not in validators_H:
                raise InvalidProposalError('invalid signer')


import ethereum.utils as utils
import ethereum.slogging as slogging
import hydrachain.native_contracts as nc
from hydrachain.nc_utils import isaddress, STATUS, FORBIDDEN, OK, INSUFFICIENTFUNDS
log = slogging.get_logger('contracts.fungible')

class Transfer(nc.ABIEvent):

    """Triggered when tokens are transferred."""
    args = [dict(name='from', type='address', indexed=True),
            dict(name='to', type='address', indexed=True),
            dict(name='value', type='uint256', indexed=True)]

class Approval(nc.ABIEvent):

    """Triggered when Fungible.approved is called."""
    args = [dict(name='owner', type='address', indexed=True),
            dict(name='spender', type='address', indexed=True),
            dict(name='value', type='uint256', indexed=True)]

class Fungible(nc.NativeContract):

    """
    based on
    https://github.com/ethereum/wiki/wiki/Standardized_Contract_APIs
    """
    address = utils.int_to_addr(5000)
    events = [Transfer, Approval]

    owner = nc.Scalar('address')
    supply = nc.Scalar('uint256')
    # mapping (address => uint256)
    #   here mapping betw address => balances
    accounts = nc.IterableDict('uint256')
    allowances = nc.Dict(nc.Dict('uint256'))

    def init(ctx, _supply='uint256', returns=STATUS):
        log.DEV('In Fungible.init')
        if isaddress(ctx.owner):
            return FORBIDDEN
        ctx.owner = ctx.tx_origin
        ctx.accounts[ctx.tx_origin] = _supply
        ctx.supply = _supply
        return OK

    def transfer(ctx, _to='address', _value='uint256', returns=STATUS):
        """ Standardized Contract API:
        function transfer(address _to, uint256 _value) returns (bool _success)
        """
        log.DEV('In Fungible.transfer')
        if ctx.accounts[ctx.msg_sender] >= _value:
            ctx.accounts[ctx.msg_sender] -= _value
            ctx.accounts[_to] += _value
            ctx.Transfer(ctx.msg_sender, _to, _value)
            return OK
        else:
            return INSUFFICIENTFUNDS

    def transferFrom(ctx, _from='address', _to='address', _value='uint256', returns=STATUS):
        """ Standardized Contract API:
        function transferFrom(address _from, address _to, uint256 _value) returns (bool success)
        """
        auth = ctx.allowances[_from][ctx.msg_sender]
        if ctx.accounts[_from] >= _value and auth >= _value:
            ctx.allowances[_from][ctx.msg_sender] -= _value
            ctx.accounts[_from] -= _value
            ctx.accounts[_to] += _value
            ctx.Transfer(_from, _to, _value)
            return OK
        else:
            return INSUFFICIENTFUNDS

    @nc.constant
    def totalSupply(ctx, returns='uint256'):
        """ Standardized Contract API:
        function totalSupply() constant returns (uint256 supply)
        """
        return ctx.supply

    @nc.constant
    def balanceOf(ctx, _address='address', returns='uint256'):
        """ Standardized Contract API:
        function balanceOf(address _address) constant returns (uint256 balance)
        """
        return ctx.accounts[_address]

    def approve(ctx, _spender='address', _value='uint256', returns=STATUS):
        """ Standardized Contract API:
        function approve(address _spender, uint256 _value) returns (bool success)
        """
        ctx.allowances[ctx.msg_sender][_spender] += _value
        ctx.Approval(ctx.msg_sender, _spender, _value)
        return OK

    @nc.constant
    def allowance(ctx, _spender='address', returns='uint256'):
        """ Standardized Contract API:
        function allowance(address _owner, address _spender) constant returns (uint256 remaining)
        """
        return ctx.allowances[ctx.msg_sender][_spender]

    @nc.constant
    def allowanceFrom(ctx, _from='address', _spender='address', returns='uint256'):
        return ctx.allowances[_from][_spender]

    # Other Functions
    @nc.constant
    def get_creator(ctx, returns='address'):
        return ctx.owner

    @nc.constant
    def num_accounts(ctx, returns='uint32'):
        return len(ctx.accounts)

    @nc.constant
    def get_accounts(ctx, returns='address[]'):
        return list(ctx.accounts.keys())

class Token(Fungible):
    address = utils.int_to_addr(5001)

class Coin(Fungible):
    address = utils.int_to_addr(5002)

class Currency(Fungible):
    address = utils.int_to_addr(5003)

class Issuance(nc.ABIEvent):

    "Triggered when IOU.issue is called."
    args = [dict(name='issuer', type='address', indexed=True),
            dict(name='rtgs_hash', type='bytes32', indexed=True),
            dict(name='amount', type='uint256', indexed=True)]

class IOU(Fungible):
    """
    IOU fungible, can Issue its supply
    """

    address = utils.int_to_addr(5004)
    events = [Transfer, Approval, Issuance]
    issued_amounts = nc.IterableDict('uint256')

    def init(ctx, returns=STATUS):
        log.DEV('In IOU.init')
        return super(IOU, ctx).init(0)

    def issue_funds(ctx, amount='uint256', rtgs_hash='bytes32', returns=STATUS):
        "In the IOU fungible the supply is set by Issuer, who issue funds."
        # allocate new issue as result of a new cash entry
        ctx.accounts[ctx.msg_sender] += amount
        ctx.issued_amounts[ctx.msg_sender] += amount
        # Store hash(rtgs)
        ctx.Issuance(ctx.msg_sender, rtgs_hash, amount)
        return OK

    # Other Functions
    @nc.constant
    def get_issued_amount(ctx, issuer='address', returns='uint256'):
        return ctx.issued_amounts[issuer]


# Copyright (c) 2015 Heiko Hees
import sys
import rlp
from .base import LockSet, Vote, VoteBlock, VoteNil, Signed, Ready
from .base import BlockProposal, VotingInstruction, DoubleVotingError, InvalidVoteError
from .base import Block, Proposal, HDCBlockHeader, InvalidProposalError
from .protocol import HDCProtocol
from .utils import cstr, phx
from .synchronizer import Synchronizer
from ethereum.slogging import get_logger
log = get_logger('hdc.consensus')

class ManagerDict(object):

    def __init__(self, dklass, parent):
        self.d = dict()
        self.dklass = dklass
        self.parent = parent

    def __getitem__(self, k):
        if k not in self.d:
            self.d[k] = self.dklass(self.parent, k)
        return self.d[k]

    def __iter__(self):
        return iter(sorted(self.d, reverse=True))

    def pop(self, k):
        self.d.pop(k)

class MissingParent(Exception):
    pass

class ProtocolFailureEvidence(object):
    protocol = None
    evidence = None

    def __repr__(self):
        return '<%s protocol=%r evidence=%r>' % (self.__class__.__name__,
                                                 self.protocol, self.evidence)

class InvalidProposalEvidence(ProtocolFailureEvidence):

    def __init__(self, protocol, proposal):
        self.protocol = protocol
        self.evidence = proposal

class DoubleVotingEvidence(ProtocolFailureEvidence):

    def __init__(self, protocol, vote, othervote):
        self.protocol = protocol
        self.evidence = (vote, othervote)

class InvalidVoteEvidence(ProtocolFailureEvidence):

    def __init__(self, protocol, vote):
        self.protocol = protocol
        self.evidence = vote

class FailedToProposeEvidence(ProtocolFailureEvidence):

    def __init__(self, protocol, round_lockset):
        self.protocol = protocol
        self.evidence = round_lockset

class ForkDetectedEvidence(ProtocolFailureEvidence):

    def __init__(self, protocol, prevblock, block, committing_lockset):
        self.protocol = protocol
        self.evidence = (prevblock, block, committing_lockset)

class ConsensusManager(object):

    allow_empty_blocks = False
    num_initial_blocks = 10
    round_timeout = 3  # timeout when waiting for proposal
    round_timeout_factor = 1.5  # timeout increase per round
    transaction_timeout = 0.5  # delay when waiting for new transaction

    def __init__(self, chainservice, consensus_contract, privkey):
        self.chainservice = chainservice
        self.chain = chainservice.chain
        self.contract = consensus_contract
        self.privkey = privkey

        self.synchronizer = Synchronizer(self)
        self.heights = ManagerDict(HeightManager, self)
        self.block_candidates = dict()  # blockhash : BlockProposal

        self.tracked_protocol_failures = list()

        # wait for enough validators in order to start
        self.ready_validators = set()  # addresses
        self.ready_nonce = 0

        assert self.contract.isvalidator(self.coinbase)
        self.initialize_locksets()

        self.ready_validators = set([self.coinbase])  # old votes dont count

    def initialize_locksets(self):
        log.debug('initializing locksets')
        # sign genesis
        v = self.sign(VoteBlock(0, 0, self.chainservice.chain.genesis.hash))
        self.add_vote(v)

        # add initial lockset
        head_proposal = self.load_proposal(self.head.hash)
        # assert head_proposal
        if head_proposal:
            assert head_proposal.blockhash == self.head.hash
            for v in head_proposal.signing_lockset:
                self.add_vote(v)  # head - 1 , height -2
            assert self.heights[self.head.header.number - 1].has_quorum
        last_committing_lockset = self.load_last_committing_lockset()
        if last_committing_lockset:
            assert last_committing_lockset.has_quorum == self.head.hash
            for v in last_committing_lockset.votes:
                self.add_vote(v)  # head  , height - 1
            assert self.heights[self.head.header.number].has_quorum
        else:
            assert self.head.header.number == 0
        assert self.highest_committing_lockset
        assert self.last_committing_lockset
        assert self.last_valid_lockset

    # persist proposals and last committing lockset

    def store_last_committing_lockset(self, ls):
        assert isinstance(ls, LockSet)
        assert ls.has_quorum
        self.chainservice.db.put('last_committing_lockset', rlp.encode(ls))

    def load_last_committing_lockset(self):
        try:
            data = self.chainservice.db.get('last_committing_lockset')
        except KeyError:
            self.log('no last_committing_lockset could be loaded')
            return
        return rlp.decode(data, sedes=LockSet)

    def store_proposal(self, p):
        assert isinstance(p, BlockProposal)
        self.chainservice.db.put('blockproposal:%s' % p.blockhash, rlp.encode(p))

    def load_proposal_rlp(self, blockhash):
        try:
            prlp = self.chainservice.db.get('blockproposal:%s' % blockhash)
            assert isinstance(prlp, bytes)
            return prlp
        except KeyError:
            return None

    def load_proposal(self, blockhash):
        prlp = self.load_proposal_rlp(blockhash)
        if prlp:
            return rlp.decode(prlp, sedes=BlockProposal)

    def get_blockproposal(self, blockhash):
        return self.block_candidates.get(blockhash) or self.load_proposal(blockhash)

    def has_blockproposal(self, blockhash):
        return bool(self.load_proposal_rlp(blockhash))

    def get_blockproposal_rlp_by_height(self, height):
        assert 0 < height < self.height
        bh = self.chainservice.chain.index.get_block_by_number(height)
        return self.load_proposal_rlp(bh)

    @property
    def coinbase(self):
        return self.chain.coinbase

    def set_proposal_lock(self, block):
        self.chainservice.set_proposal_lock(block)

    def __repr__(self):
        return '<CP A:%r H:%d R:%d L:%r %s>' % (phx(self.coinbase), self.height, self.round,
                                                self.active_round.lock,
                                                self.active_round.lockset.state)

    def log(self, tag, **kargs):
        # if self.coinbase != 0: return
        t = int(self.chainservice.now)
        c = lambda x: cstr(self.coinbase, x)
        msg = ' '.join([str(t), c(repr(self)), tag, (' %r' % kargs if kargs else '')])
        log.debug(msg)

    @property
    def head(self):
        return self.chain.head

    @property
    def height(self):
        return self.head.number + 1

    @property
    def round(self):
        return self.heights[self.height].round

    # message handling

    def broadcast(self, m):
        self.log('broadcasting', message=m)
        self.chainservice.broadcast(m)

    # validator ready handling

    @property
    def is_ready(self):
        return len(self.ready_validators) > len(self.contract.validators) * 2 / 3.

    def send_ready(self):
        self.log('cm.send_ready')
        assert not self.is_ready
        r = Ready(self.ready_nonce, self.active_round.lockset)
        self.sign(r)
        self.broadcast(r)
        self.ready_nonce += 1

    def add_ready(self, ready, proto=None):
        assert isinstance(ready, Ready)
        assert self.contract.isvalidator(ready.sender)
        self.ready_validators.add(ready.sender)
        self.log('cm.add_ready', validator=ready.sender)
        if self.is_ready:
            self.log('cm.add_ready, sufficient count of validators ready',
                     num=len(self.ready_validators))
        else:
            self.send_ready()

    def add_vote(self, v, proto=None):
        assert isinstance(v, Vote)
        assert self.contract.isvalidator(v.sender)
        self.ready_validators.add(v.sender)
        # exception for externaly received votes signed by self, necessary for resyncing
        is_own_vote = bool(v.sender == self.coinbase)
        try:
            success = self.heights[v.height].add_vote(v, force_replace=is_own_vote)
        except DoubleVotingError:
            ls = self.heights[v.height].rounds[v.round].lockset
            self.tracked_protocol_failures.append(DoubleVotingEvidence(proto, v, ls))
            log.warn('double voting detected', vote=v, ls=ls)
        return success

    def add_proposal(self, p, proto=None):
        assert isinstance(p, Proposal)
        assert proto is None or isinstance(proto, HDCProtocol)

        def check(valid):
            if not valid:
                self.tracked_protocol_failures.append(InvalidProposalEvidence(None, p))
                log.warn('invalid proposal', p=p)
                raise InvalidProposalError()
            return True

        self.log('cm.add_proposal', p=p)
        if p.height < self.height:
            self.log('proposal from the past')
            return

        if not check(self.contract.isvalidator(p.sender) and self.contract.isproposer(p)):
            return
        self.ready_validators.add(p.sender)

        if not check(p.lockset.is_valid):
            return
        if not check(p.lockset.height == p.height or p.round == 0):
            return
        if not check(p.round - p.lockset.round == 1 or p.round == 0):
            return

        # proposal is valid
        if proto is not None:  # inactive proto is False
            self.synchronizer.on_proposal(p, proto)

        for v in p.lockset:
            self.add_vote(v)  # implicitly checks their validity
        if isinstance(p, BlockProposal):
            if not check(p.block.number == p.height):
                return
            if not check(p.lockset.has_noquorum or p.round == 0):
                return
            # validation
            if p.height > self.height:
                self.log('proposal from the future, not in sync', p=p)
                return  # note: we are not broadcasting this, as we could not validate
            blk = self.chainservice.link_block(p.block)
            if not check(blk):
                # safeguard for forks:
                # if there is a quorum on a block which can not be applied: panic!
                ls = self.heights[p.height].last_quorum_lockset
                if ls and ls.has_quorum == p.blockhash:
                    raise ForkDetectedEvidence(proto, (self.head, p, ls))
                    sys.exit(1)
                return
            p._mutable = True
            p._cached_rlp = None
            p.block = blk  # block linked to chain
            self.log('successfully linked block')
            self.add_block_proposal(p)  # implicitly checks the votes validity
        else:
            assert isinstance(p, VotingInstruction)
            assert p.lockset.round == p.round - 1 and p.height == p.lockset.height
            assert p.round > 0
            assert p.lockset.has_quorum_possible
            assert not p.lockset.has_quorum
            if not check(p.lockset.has_quorum_possible and not p.lockset.has_quorum):
                return
        is_valid = self.heights[p.height].add_proposal(p)
        return is_valid  # can be broadcasted

    def add_lockset(self, ls, proto=None):
        assert ls.is_valid
        for v in ls:
            self.add_vote(v)  # implicitly checks their validity

    def add_block_proposal(self, p):
        assert isinstance(p, BlockProposal)
        if self.has_blockproposal(p.blockhash):
            self.log('known block_proposal')
            return
        assert p.signing_lockset.has_quorum  # on previous block
        assert p.signing_lockset.height == p.height - 1
        for v in p.signing_lockset:
            self.add_vote(v)
        self.block_candidates[p.blockhash] = p

    @property
    def last_committing_lockset(self):
        return self.heights[self.height - 1].last_quorum_lockset

    @property
    def highest_committing_lockset(self):
        for height in self.heights:
            ls = self.heights[height].last_quorum_lockset
            if ls:
                return ls

    @property
    def last_valid_lockset(self):
        return self.heights[self.height].last_valid_lockset or self.last_committing_lockset

    @property
    def last_lock(self):
        return self.heights[self.height].last_lock

    @property
    def last_blockproposal(self):
        # valid block proposal on currrent height
        p = self.heights[self.height].last_voted_blockproposal
        if p:
            return p
        elif self.height > 1:  # or last block
            return self.get_blockproposal(self.head.hash)

    @property
    def active_round(self):
        hm = self.heights[self.height]
        return hm.rounds[hm.round]

    def setup_alarm(self):
        ar = self.active_round
        delay = ar.get_timeout()
        self.log('in set up alarm', delay=delay)
        if self.is_waiting_for_proposal:
            if delay is not None:
                self.chainservice.setup_alarm(delay, self.on_alarm, ar)
                self.log('set up alarm on timeout', now=self.chainservice.now,
                         delay=delay, triggered=delay + self.chainservice.now)
        else:
            self.chainservice.setup_transaction_alarm(self.on_alarm, ar)
            self.log('set up alarm on tx', now=self.chainservice.now)

    def on_alarm(self, ar):
        assert isinstance(ar, RoundManager)
        if self.active_round == ar:
            self.log('on alarm, matched', ts=self.chainservice.now)
            if not self.is_ready:
                # defer alarm if not ready
                self.log('not ready defering alarm', ts=self.chainservice.now)
                self.setup_alarm()
            elif not self.is_waiting_for_proposal:
                # defer alarm if there are no pending transactions
                self.log('no txs defering alarm', ts=self.chainservice.now)
                self.setup_alarm()
            else:
                self.process()

    @property
    def is_waiting_for_proposal(self):
        return self.allow_empty_blocks \
            or self.has_pending_transactions \
            or self.height <= self.num_initial_blocks

    @property
    def has_pending_transactions(self):
        return self.chain.head_candidate.num_transactions() > 0

    def process(self):
        r = self._process()
        return r

    def _process(self):
        self.log('-' * 40)
        self.log('in process')
        if not self.is_ready:
            self.log('not ready ')
            self.setup_alarm()
            return
        self.commit()
        self.heights[self.height].process()
        if self.commit():  # re enter process if we did commit (e.g. to immediately propose)
            return self._process()
        self.cleanup()
        self.synchronizer.process()
        self.setup_alarm()

        for f in self.tracked_protocol_failures:
            if not isinstance(f, FailedToProposeEvidence):
                log.warn('protocol failure', incident=f)

    start = process

    def commit(self):
        self.log('in commit')
        for p in [c for c in self.block_candidates.values() if c.block.prevhash == self.head.hash]:
            assert isinstance(p, BlockProposal)
            ls = self.heights[p.height].last_quorum_lockset
            if ls and ls.has_quorum == p.blockhash:
                self.store_proposal(p)
                self.store_last_committing_lockset(ls)
                success = self.chainservice.commit_block(p.block)
                assert success
                if success:
                    self.log('commited', p=p, hash=phx(p.blockhash))
                    assert self.head == p.block
                    self.commit()  # commit all possible
                    return True
                else:
                    self.log('could not commit', p=p)
            else:
                self.log('no quorum for', p=p)
                if ls:
                    self.log('votes', votes=ls.votes)

    def cleanup(self):
        self.log('in cleanup')
        for p in self.block_candidates.values():
            if self.head.number >= p.height:
                self.block_candidates.pop(p.blockhash)
        for h in list(self.heights):
            if self.heights[h].height < self.head.number:
                self.heights.pop(h)

    def mk_lockset(self, height):
        return LockSet(num_eligible_votes=self.contract.num_eligible_votes(height))

    def sign(self, o):
        assert isinstance(o, Signed)
        return o.sign(self.privkey)

class HeightManager(object):

    def __init__(self, consensusmanager, height=0):
        self.cm = consensusmanager
        self.log = self.cm.log
        self.height = height
        self.rounds = ManagerDict(RoundManager, self)
        log.debug('A:%s Created HeightManager H:%d' % (phx(self.cm.coinbase), self.height))

    @property
    def round(self):
        l = self.last_valid_lockset
        if l:
            return l.round + 1
        return 0

    @property
    def last_lock(self):
        "highest lock on height"
        rs = list(self.rounds)
        assert len(rs) < 2 or rs[0] > rs[1]  # FIXME REMOVE
        for r in self.rounds:  # is sorted highest to lowest
            if self.rounds[r].lock is not None:
                return self.rounds[r].lock

    @property
    def last_voted_blockproposal(self):
        "the last block proposal node voted on"
        for r in self.rounds:
            if isinstance(self.rounds[r].proposal, BlockProposal):
                assert isinstance(self.rounds[r].lock, Vote)
                if self.rounds[r].proposal.blockhash == self.rounds[r].lock.blockhash:
                    return self.rounds[r].proposal

    @property
    def last_valid_lockset(self):
        "highest valid lockset on height"
        for r in self.rounds:
            ls = self.rounds[r].lockset
            if ls.is_valid:
                return ls
        return None

    @property
    def last_quorum_lockset(self):
        found = None
        for r in sorted(self.rounds):  # search from lowest round first
            ls = self.rounds[r].lockset
            if ls.is_valid and ls.has_quorum:
                if found is not None:  # consistency check, only one quorum on block allowed
                    for r in sorted(self.rounds):  # dump all locksets
                        self.log('multiple valid locksets', round=r, ls=self.rounds[r].lockset,
                                 votes=self.rounds[r].lockset.votes)
                    if found.has_quorum != ls.has_quorum:
                        log.error('FATAL: multiple valid locksets on different proposals')
                        import sys
                        sys.exit(1)
                found = ls
        return found

    @property
    def has_quorum(self):
        ls = self.last_quorum_lockset
        if ls:
            return ls.has_quorum

    def add_vote(self, v, force_replace=False):
        return self.rounds[v.round].add_vote(v, force_replace)

    def add_proposal(self, p):
        assert p.height == self.height
        assert p.lockset.is_valid
        if p.round > self.round:
            self.round = p.round
        return self.rounds[p.round].add_proposal(p)

    def process(self):
        self.log('in hm.process', height=self.height)
        self.rounds[self.round].process()

class RoundManager(object):

    def __init__(self, heightmanager, round_=0):

        assert isinstance(round_, int)
        self.round = round_

        self.hm = heightmanager
        self.cm = heightmanager.cm
        self.log = self.hm.log
        self.height = heightmanager.height
        self.lockset = self.cm.mk_lockset(self.height)
        self.proposal = None
        self.lock = None
        self.timeout_time = None
        log.debug('A:%s Created RoundManager H:%d R:%d' %
                  (phx(self.cm.coinbase), self.hm.height, self.round))

    def get_timeout(self):
        "setup a timeout for waiting for a proposal"
        if self.timeout_time is not None or self.proposal:
            return
        now = self.cm.chainservice.now
        round_timeout = ConsensusManager.round_timeout
        round_timeout_factor = ConsensusManager.round_timeout_factor
        delay = round_timeout * round_timeout_factor ** self.round
        self.timeout_time = now + delay
        return delay

    def add_vote(self, v, force_replace=False):
        if v in self.lockset:
            return
        self.log('rm.adding', vote=v, received_proposal=self.proposal)
        try:
            success = self.lockset.add(v, force_replace)
        except InvalidVoteError:
            self.cm.tracked_protocol_failures.append(InvalidVoteEvidence(None, v))
            return
        # report failed proposer
        if self.lockset.is_valid:
            self.log('lockset is valid', ls=self.lockset)
            if not self.proposal and self.lockset.has_noquorum:
                self.cm.tracked_protocol_failures.append(
                    FailedToProposeEvidence(None, self.lockset))
        return success

    def add_proposal(self, p):
        self.log('rm.adding', proposal=p, old=self.proposal)
        assert isinstance(p, Proposal)
        assert isinstance(p, VotingInstruction) or isinstance(p.block, Block)  # already linked
        assert not self.proposal or self.proposal == p
        self.proposal = p
        return True

    def process(self):
        self.log('in rm.process', height=self.hm.height, round=self.round)

        assert self.cm.round == self.round
        assert self.cm.height == self.hm.height == self.height
        p = self.propose()
        if isinstance(p, BlockProposal):
            self.cm.add_block_proposal(p)
        if p:
            self.cm.broadcast(p)
        v = self.vote()
        if v:
            self.cm.broadcast(v)
        assert not self.proposal or self.lock

    def mk_proposal(self, round_lockset=None):
        signing_lockset = self.cm.last_committing_lockset.copy()  # quorum which signs prev block
        if self.round > 0:
            round_lockset = self.cm.last_valid_lockset.copy()
            assert round_lockset.has_noquorum
        else:
            round_lockset = None
        assert signing_lockset.has_quorum
        # for R0 (std case) we only need one lockset!
        assert round_lockset is None or self.round > 0
        block = self.cm.chain.head_candidate
        # fix pow
        block.header.__class__ = HDCBlockHeader
        block.should_be_locked = True
        bp = BlockProposal(self.height, self.round, block, signing_lockset, round_lockset)
        self.cm.sign(bp)
        self.cm.set_proposal_lock(block)
        assert self.cm.chainservice.proposal_lock.locked()
        return bp

    def propose(self):
        if not self.cm.is_waiting_for_proposal:
            return
        proposer = self.cm.contract.proposer(self.height, self.round)
        self.log('in propose', proposer=phx(proposer), proposal=self.proposal, lock=self.lock)
        if proposer != self.cm.coinbase:
            return
        self.log('is proposer')
        if self.proposal:
            assert self.proposal.sender == self.cm.coinbase
            assert self.lock
            return

        round_lockset = self.cm.last_valid_lockset
        if not round_lockset:
            self.log('no valid round lockset for height')
            return

        self.log('in creating proposal', round_lockset=round_lockset)

        if round_lockset.height == self.height and round_lockset.has_quorum:
            self.log('have quorum on height, not proposing')
            return
        elif self.round == 0 or round_lockset.has_noquorum:
            proposal = self.mk_proposal()
        elif round_lockset.has_quorum_possible:
            proposal = VotingInstruction(self.height, self.round, round_lockset.copy())
            self.cm.sign(proposal)
        else:
            raise Exception('invalid round_lockset')

        self.log('created proposal', p=proposal, bh=phx(proposal.blockhash))
        self.proposal = proposal
        return proposal

    def vote(self):
        if self.lock:
            return  # voted in this round
        self.log('in vote', proposal=self.proposal, pid=id(self.proposal))

        # get last lock on height
        last_lock = self.hm.last_lock

        if self.proposal:
            if isinstance(self.proposal, VotingInstruction):
                assert self.proposal.lockset.has_quorum_possible
                self.log('voting on instruction')
                v = VoteBlock(self.height, self.round, self.proposal.blockhash)
            elif not isinstance(last_lock, VoteBlock):
                assert isinstance(self.proposal, BlockProposal)
                assert isinstance(self.proposal.block, Block)  # already linked to chain
                assert self.proposal.lockset.has_noquorum or self.round == 0
                assert self.proposal.block.prevhash == self.cm.head.hash
                self.log('voting proposed block')
                v = VoteBlock(self.height, self.round, self.proposal.blockhash)
            else:  # repeat vote
                self.log('voting on last vote')
                v = VoteBlock(self.height, self.round, last_lock.blockhash)
        elif self.timeout_time is not None and self.cm.chainservice.now >= self.timeout_time:
            if isinstance(last_lock, VoteBlock):  # repeat vote
                self.log('timeout voting on last vote')
                v = VoteBlock(self.height, self.round, last_lock.blockhash)
            else:
                self.log('timeout voting not locked')
                v = VoteNil(self.height, self.round)
        else:
            return
        self.cm.sign(v)

        self.log('voted', vote=v)
        self.lock = v
        assert self.hm.last_lock == self.lock
        self.lockset.add(v)
        return v


import gevent
from .base import Proposal
from .protocol import HDCProtocol

class Synchronizer(object):

    timeout = 5
    max_getproposals_count = HDCProtocol.max_getproposals_count
    max_queued = 3 * max_getproposals_count

    def __init__(self, consensusmanager):
        self.cm = consensusmanager
        self.requested = set()
        self.received = set()
        self.last_active_protocol = None  # last protocol (peer) which sent a proposal
        self.add_proposals_lock = gevent.lock.Semaphore()

    def __repr__(self):
        status = 'syncing' if self.is_syncing else 'insync'
        return '<Synchronizer(%s missing=%d requested=%d received=%d)>' \
            % (status, len(self.missing), len(self.requested), len(self.received))

    @property
    def is_syncing(self):
        return len(self.requested)

    @property
    def missing(self):
        ls = self.cm.highest_committing_lockset
        if not ls:
            return []
        max_height = ls.height
        if not max_height or max_height <= self.cm.head.number:
            return []
        return range(self.cm.head.number + 1, max_height + 1)

    def request(self):
        """
        sync the missing blocks between:
            head
            highest height with signing lockset

        we get these locksets by collecting votes on all heights
        """
        missing = self.missing
        self.cm.log('sync.request', missing=len(missing), requested=len(self.requested),
                    received=len(self.received))
        if self.requested:
            self.cm.log('waiting for requested')
            return
        if len(self.received) + self.max_getproposals_count >= self.max_queued:
            self.cm.log('queue is full')
            return
        if not missing:
            self.cm.log('insync')
            return
        if self.last_active_protocol is None:  # FIXME, check if it is active
            self.cm.log('no active protocol', last_active_protocol=self.last_active_protocol)
            return
        self.cm.log('collecting')
        blocknumbers = []
        for h in missing:
            if h not in self.received and h not in self.requested:
                blocknumbers.append(h)
                self.requested.add(h)
                if len(blocknumbers) == self.max_getproposals_count:
                    break
        self.cm.log('collected', num=len(blocknumbers))
        if not blocknumbers:
            return
        self.cm.log('requesting', num=len(blocknumbers),
                    requesting_range=(blocknumbers[0], blocknumbers[-1]))
        self.last_active_protocol.send_getblockproposals(*blocknumbers)
        # setup alarm
        self.cm.chainservice.setup_alarm(self.timeout, self.on_alarm, blocknumbers)

    def on_proposal(self, proposal, proto):
        "called to inform about synced peers"
        assert isinstance(proto, HDCProtocol)
        assert isinstance(proposal, Proposal)
        if proposal.height >= self.cm.height:
            assert proposal.lockset.is_valid
            self.last_active_protocol = proto

    def on_alarm(self, requested):
        # remove requested, so they can be rerequested
        self.requested.difference_update(set(self.requested))
        self.request()

    def receive_blockproposals(self, proposals):
        self.cm.log('receive_blockproposals', p=proposals, received=self.received)
        for p in proposals:
            self.received.add(p.height)
            self.requested.remove(p.height)
            for v in p.signing_lockset:  # add all votes, so we have locksets ready for committing
                self.cm.add_vote(v)

        # commit after we added new votes to commit a block from the last sync
        self.cm.process()

        # request next round
        self.request()
        self.add_proposals_lock.acquire()
        for p in proposals:
            self.cm.add_proposal(p)
            self.cm.process()
        self.cleanup()
        self.add_proposals_lock.release()

        not_added = []
        for p in proposals:
            if p.height > self.cm.head.number:
                not_added.append(p)
                if p.height == self.cm.head.number:
                    assert p.blockhash in self.cm.block_candidates

        # print 'not added', not_added
        # print 'received', self.received
        # for h in self.cm.heights:
        #     if self.cm.heights[h].last_quorum_lockset:
        #         print 'quorum @', h

        assert self.cm.height >= max(p.height for p in proposals)

        self.cm.log('done receive_blockproposals', sync=self)
        if len(not_added) > 1:
            raise Exception('more than one proposal not added')

    def cleanup(self):
        height = self.cm.height
        for h in list(self.received):
            if h < height:
                self.received.remove(h)
        for h in list(self.requested):
            if h < height:
                self.requested.remove(h)

    def process(self):
        self.request()


from ethereum.utils import zpad
import gevent.event
import ethereum.slogging as slogging
from ethereum import transactions
from pyethapp.rpc_client import ABIContract
from hydrachain import native_contracts as nc
log = slogging.get_logger('nc.utils')

STATUS = 'uint16'
FORBIDDEN = 403
NOTFOUND = 404
OK = 200
ERROR = 500
BADREQEST = 400
INSUFFICIENTFUNDS = PAYMENTREQUIRED = 402

def isaddress(x):
    return len(x) == 20 and x != '\0' * 20

def lhexenc(lst):
    return [l.encode('hex') for l in lst]

def transact(app, sender, to_, value=0, data=''):
    head_candidate = app.services.chain.chain.head_candidate
    default_gasprice = 1
    default_startgas = head_candidate.gas_limit - head_candidate.gas_used
    nonce = head_candidate.get_nonce(sender)
    tx = transactions.Transaction(nonce=nonce, gasprice=default_gasprice,
                                  startgas=default_startgas, to=to_, value=value, data=data)
    assert sender in app.services.accounts, 'no account for sender'
    app.services.accounts.sign_tx(sender, tx)
    success = app.services.chain.add_transaction(tx)
    assert success
    return tx

def wait_next_block_factory(app, timeout=None):
    """Creates a `wait_next_block` function, that
    will wait `timeout` seconds (`None` = indefinitely)
    for a new block to appear.

    :param app: the app-instance the function should work for
    :param timeout: timeout in seconds
    """

    chain = app.services.chain

    # setup new block callbacks and events
    new_block_evt = gevent.event.Event()

    def _on_new_block(app):
        log.DEV('new block mined')
        new_block_evt.set()
    chain.on_new_head_cbs.append(_on_new_block)

    def wait_next_block():
        bn = chain.chain.head.number
        chain.consensus_manager.log('waiting for new block', block=bn)
        new_block_evt.wait(timeout)
        new_block_evt.clear()
        if chain.chain.head.number > bn:
            chain.consensus_manager.log('new block event', block=chain.chain.head.number)
        elif chain.chain.head.number == bn:
            chain.consensus_manager.log('wait_next_block timed out', block=bn)

    return wait_next_block

def create_contract_instance(app, sender, contract_template):
    log.DEV("creating instance", klass=contract_template)
    to_ = nc.CreateNativeContractInstance.address
    call_data = contract_template.address[-4:]
    tx = transact(app, sender, to_, data=call_data)
    instance_address = nc.registry.mk_instance_address(contract_template, sender, tx.nonce)
    return instance_address

def decode_log(log_, events):
    cls = None
    for e_class in events:
        if e_class.event_id() == log_.topics[0]:
            cls = e_class
    if not cls:
        log.DEV('unknown eventclass for log_')
        return None
    res = []
    cls.listen(log_, address=None, callback=res.append)
    e = res[0]
    e['contract'] = log_.address
    cn = nc.registry.address_to_native_contract_class(log_.address).im_self.__name__
    e['contract_class'] = cn
    return e

def get_logs(app, events):
    log.DEV('getting logs!')
    chain = app.services.chain.chain
    logs = []
    for i in range(0, chain.head.number + 1):
        block = chain.get(chain.index.get_block_by_number(i))
        receipts = block.get_receipts()
        # log.DEV('with', bnum=i, num_receipts=len(receipts))
        for r_idx, receipt in enumerate(receipts):  # one receipt per tx
            # log.DEV('with', r_num=r_idx, num_logs=len(receipt.logs))
            for l_idx, log_ in enumerate(receipt.logs):
                dlog = decode_log(log_, events)
                # log.DEV('log', rlog=log_, log=dlog)
                logs.append(dlog)
    return logs

def hexify_dict(dict_):
    for k, v in dict_.items():
        if isinstance(v, bytes) and len(v) in (20, 32):
            dict_[k] = v.encode('hex')
    return dict_

def contract_args_from_kargs(contract_class, method, kargs):
    for obj in contract_class.abi():
        if obj['type'] == 'function' and obj['name'] == method:
            arg_names = [i['name'] for i in obj['inputs']]
            diff = set(kargs.keys()).symmetric_difference(set(arg_names))
            assert not diff, 'doesnt match signature:{} error:{}'.format(arg_names, diff)
            return [kargs[name] for name in arg_names]
    raise Exception('method not found')

class User():

    def __init__(self, app, address):
        self.app = app
        self.address = address

    def add_proxy(self, name, address):
        template_address = zpad(address[-4:], 20)
        klass = nc.registry[template_address].im_self
        assert issubclass(klass, nc.NativeABIContract)

        def _transact_func(sender, to, value, data):
            return transact(self.app, sender, to, value, data)

        def _call_func(sender, to, value, data):
            block = self.app.services.chain.chain.head_candidate
            return nc.test_call(block, sender, to, data=data, gasprice=0, value=value)

        proxy = ABIContract(self.address, klass.abi(), address, _call_func, _transact_func)
        setattr(self, name, proxy)


# Copyright (c) 2015 Heiko Hees
import copy
import random
import tempfile
import time
from collections import Counter

import ethereum.keys
import gevent
import simpy
from ethereum import slogging
from ethereum.db import EphemDB
from ethereum.utils import big_endian_to_int, sha3, privtoaddr
from pyethapp.accounts import Account, AccountsService

from hydrachain import hdc_service
from hydrachain.consensus import protocol as hdc_protocol
from hydrachain.consensus.base import Block
from hydrachain.consensus.manager import ConsensusManager
from hydrachain.consensus.utils import num_colors, phx
from hydrachain.hdc_service import ChainService

log = slogging.get_logger('hdc.sim')

# stop on exception
gevent.get_hub().SYSTEM_ERROR = BaseException

# reduce key derivation iterations
ethereum.keys.PBKDF2_CONSTANTS['c'] = 100

privkeys = [chr(i) * 32 for i in range(1, 11)]
validators = privkeys[:]

empty = object()

random.seed(42)

def mk_privkeys(num):
    "make privkeys that support coloring, see utils.cstr"
    privkeys = []
    assert num <= num_colors
    for i in range(num):
        j = 0
        while True:
            k = sha3(str(j))
            a = privtoaddr(k)
            an = big_endian_to_int(a)
            if an % num_colors == i:
                break
            j += 1
        privkeys.append(k)
    return privkeys

class Transport(object):

    def __init__(self, simenv=None):
        self.simenv = simenv

    def delay(self, sender, receiver, packet, add_delay=0):
        """
        bandwidths are inaccurate, as we don't account for parallel transfers here
        """
        bw = min(sender.ul_bandwidth, receiver.dl_bandwidth)
        delay = sender.base_latency + receiver.base_latency
        delay += len(packet) / bw
        delay += add_delay
        return delay

    def deliver(self, sender, receiver, packet, add_delay=0):
        delay = self.delay(sender, receiver, packet, add_delay)
        if self.simenv:
            self.simenv_deliver(sender, receiver, packet, delay)
        else:
            self.gevent_deliver(sender, receiver, packet, delay)

    def gevent_deliver(self, sender, receiver, packet, delay):
        assert sender != receiver

        def transfer():
            gevent.sleep(delay)
            receiver.receive_packet(sender, packet)
        gevent.spawn(transfer)

    def simenv_deliver(self, sender, receiver, packet, delay):
        def transfer():
            yield self.simenv.timeout(delay)
            receiver.receive_packet(sender, packet)

        self.simenv.process(transfer())

class NoTransport(Transport):

    def deliver(self, sender, receiver, packet):
        pass

class SlowTransport(Transport):

    def deliver(self, sender, receiver, packet):
        "deliver on edge of timeout_window"
        to = ConsensusManager.round_timeout
        assert to > 0
        print "in slow transport deliver"
        super(SlowTransport, self).deliver(sender, receiver, packet, add_delay=to)

class PeerMock(object):

    ul_bandwidth = 1 * 10 ** 6  # bytes/s net bandwidth
    dl_bandwidth = 1 * 10 ** 6  # bytes/s net bandwidth
    base_latency = 0.05  # secs
    ingress_bytes = 0
    egress_bytes = 0

    def __init__(self, app, transport):
        self.app = app
        self.config = app.config
        self.remote_client_version = empty
        self.peer = None
        self.protocol = None
        self.transport = transport

    def __repr__(self):
        return "<PeerMock(A:%s > R:%s)>" % (phx(self.coinbase), phx(self.peer.coinbase))

    @property
    def coinbase(self):
        return self.app.services.chainservice.chain.coinbase

    def send_packet(self, packet):
        assert self.peer
        # log.debug('send_packet', sender=self, receiver=self.peer, len=len(packet))
        self.egress_bytes += len(packet)
        self.transport.deliver(self, self.peer, packet)

    def receive_packet(self, sender, packet):
        assert self.app.isactive
        assert sender != self
        # log.debug('receive_packet', sender=sender, receiver=self, len=len(packet))
        self.ingress_bytes += len(packet)
        self.protocol.receive_packet(packet)

    def __eq__(self, other):
        return repr(self) == repr(other)

    def __hash__(self):
        return hash(repr(self))

class PeerManagerMock(object):

    def __init__(self, app):
        self.app = app
        self.peers = []

    def __repr__(self):
        return "<PeerManager(A:%s)>" % phx(self.coinbase)

    @property
    def coinbase(self):
        return self.app.services.chainservice.chain.coinbase

    def broadcast(self, protocol, command_name, args=[], kargs={},
                  num_peers=None, exclude_peers=[]):
        for p in self.peers:
            assert p.coinbase == self.coinbase
            assert p.peer.coinbase != self.coinbase
            assert isinstance(p.protocol, (protocol, hdc_protocol.HDCProtocol))
            if not exclude_peers or p not in exclude_peers:
                log.debug('broadcasting', sender=self, receiver=p, obj=args)
                func = getattr(p.protocol, 'send_' + command_name)
                func(*args, **kargs)

class SimChainService(ChainService):

    processing_time = 0.1  # time to validate block

    def __init__(self, *args, **kargs):
        self.simenv = kargs.pop('simenv')
        super(SimChainService, self).__init__(*args, **kargs)

    @property
    def now(self):
        assert self.simenv
        return self.simenv.now

    def setup_alarm(self, delay, cb, *args):
        assert self.simenv

        def _trigger():
            yield self.simenv.timeout(delay)
            cb(*args)
        self.simenv.process(_trigger())

    def on_receive_newblockproposal(self, proto, proposal):

        def process():
            yield self.simenv.timeout(self.processing_time)
            super(SimChainService, self).on_receive_newblockproposal(proto, proposal)

        self.simenv.process(process())

class AppMock(object):

    class Services(dict):
        pass

    def __init__(self, privkey, validators, simenv=None):
        self.config = copy.deepcopy(hdc_service.ChainService.default_config)
        self.config['db'] = dict(path='_db')
        self.config['data_dir'] = tempfile.mkdtemp()
        self.config['hdc']['validators'] = validators

        initial_alloc = dict((a, dict(wei=2 ** 200)) for a in validators)
        self.config['eth']['block']['GENESIS_INITIAL_ALLOC'] = initial_alloc

        self.simenv = simenv
        self.services = self.Services()
        self.services.db = EphemDB()
        self.services.accounts = AccountsService(self)
        self.services.peermanager = PeerManagerMock(self)
        account = Account.new(password='', key=privkey)
        self.services.accounts.add_account(account, store=False)
        if simenv:
            self.services.chainservice = SimChainService(self, simenv=simenv)
        else:
            self.services.chainservice = hdc_service.ChainService(self)
        self.isactive = True

    def __repr__(self):
        return '<AppMock(%s)>' % phx(self.services.chainservice.chain.coinbase)

    def add_peer(self, peer):
        if peer in self.services.peermanager.peers:
            return
        self.services.peermanager.peers.append(peer)
        proto = hdc_protocol.HDCProtocol(peer, self.services.chainservice)
        peer.protocol = proto
        return True

    def connect_app(self, other):
        log.debug('connecting', node=self, other=other)
        transport = Transport(self.simenv)
        p = PeerMock(self, transport)
        op = PeerMock(other, transport)
        p.peer = op
        op.peer = p
        if self.add_peer(p):
            self.services.chainservice.on_wire_protocol_start(p.protocol)
        if other.add_peer(op):
            other.services.chainservice.on_wire_protocol_start(op.protocol)

class Network(object):

    starttime = None

    def __init__(self, num_nodes=2, simenv=None):
        if simenv:
            self.simenv = simpy.Environment()
        else:
            self.simenv = None
        privkeys = mk_privkeys(num_nodes)
        validators = [privtoaddr(p) for p in privkeys]
        self.nodes = []
        for i in range(num_nodes):
            app = AppMock(privkeys[i], validators, self.simenv)
            self.nodes.append(app)

    def connect_nodes(self):
        # connect nodes
        for i, n in enumerate(self.nodes):
            for o in self.nodes[i + 1:]:
                if n.isactive and o.isactive:
                    n.connect_app(o)

    def start(self):
        # start nodes
        for n in self.nodes:
            if n.isactive:
                cm = n.services.chainservice.consensus_manager
                if True or self.simenv:   # set dummy ready!"
                    cm.ready_validators = set(range(int(len(cm.contract.validators))))
                    assert cm.is_ready
                cm.process()

    def run(self, duration):
        if self.simenv:
            self.simenv.run(until=self.elapsed + duration)
        else:
            if not self.starttime:
                self.starttime = time.time()
            gevent.sleep(duration)

    @property
    def elapsed(self):
        if self.simenv:
            return self.simenv.now
        else:
            return time.time() - self.starttime

    def disable_validators(self, num):
        assert num <= len(self.nodes)
        for i in range(num):
            n = self.nodes[i]
            for p in n.services.peermanager.peers:
                p.transport = NoTransport(self.simenv)

    def throttle_validators(self, num):
        assert num <= len(self.nodes)
        for i in reversed(range(num)):
            n = self.nodes[i]
            for p in n.services.peermanager.peers:
                p.transport = SlowTransport(self.simenv)

    def normvariate_base_latencies(self, sigma_factor=0.5, base_latency=None):
        min_latency = 0.001
        for n in self.nodes:
            for p in n.services.peermanager.peers:
                p.base_latency = base_latency or p.base_latency
                sigma = p.base_latency * sigma_factor
                p.base_latency = max(min_latency, random.normalvariate(p.base_latency, sigma))
                assert p.base_latency > 0

    def consensus_managers(self):
        return [n.services.chainservice.consensus_manager for n in self.nodes]

    def check_consistency(self):
        if self.simenv:
            elapsed = self.simenv.now
        else:
            elapsed = time.time() - self.starttime
        cs = self.consensus_managers()
        # check they are all on the same block or the previous one
        heights = Counter(c.chain.head.number for c in cs)
        height_distance = max(heights.keys()) - min(heights.keys())
        max_height = height = max(heights)

        # check they are all using the same block
        while height > 0:
            bs = list(set(c.chain.index.get_block_by_number(height) for c in cs
                          if c.chain.index.has_block_by_number(height)))
            assert len(bs) == 1 or (len(bs) == 2 and None in bs), bs
            height -= 1

        # highest round seen (i.e. number of failed proposers)
        max_rounds = 0
        for c in cs:
            blk = c.chain.head
            while blk.number > 0:
                p = c.load_proposal(blk.hash)
                max_rounds = max(max_rounds, p.signing_lockset.round)
                bh = c.chain.index.get_block_by_number(blk.number - 1)
                blk = c.chain.get(bh)
                assert isinstance(blk, Block)
        max_rounds += 1

        # messages
        ingress_bytes_transfered = 0
        egress_bytes_transfered = 0

        for n in self.nodes:
            for p in n.services.peermanager.peers:
                ingress_bytes_transfered += p.ingress_bytes
                egress_bytes_transfered += p.egress_bytes

        r = dict(max_height=max_height,
                 max_rounds=max_rounds,
                 heights=heights,
                 ingress_bytes_transfered=ingress_bytes_transfered,
                 egress_bytes_transfered=egress_bytes_transfered,
                 elapsed=elapsed,
                 height_distance=height_distance)
        log.debug('checked consistency', r=r)
        return r

# funcs making assumptions on the return value of Network.check_consistency

def assert_blocktime(r, max_avg_blocktime=1):
    print r
    assert r['elapsed'] / r['max_height'] < max_avg_blocktime

def assert_maxrounds(r, max_rounds=1):
    assert max_rounds > 0
    assert r['max_rounds'] == max_rounds

def assert_heightdistance(r, max_distance=0):
    assert r['max_height'] > 0
    assert r['height_distance'] <= max_distance

def main(num_nodes=10, sim_duration=10, timeout=0.5,
         base_latency=0.05, latency_sigma_factor=0.5,
         num_faulty_nodes=3, num_slow_nodes=0):

    slogging.configure(config_string=':debug')

    orig_timeout = ConsensusManager.round_timeout
    ConsensusManager.round_timeout = timeout
    network = Network(num_nodes, simenv=True)
    network.connect_nodes()
    network.normvariate_base_latencies(latency_sigma_factor, base_latency)
    network.disable_validators(num_faulty_nodes)
    network.throttle_validators(num_slow_nodes)
    network.start()
    network.run(sim_duration)
    network.check_consistency()
    ConsensusManager.round_timeout = orig_timeout
    return network

if __name__ == '__main__':
    num_nodes = 10
    faulty_fraction = 1 / 3. * 0  # nodes not sending anything
    # nodes sending votes and proposals at the edge of the timeout window
    slow_fraction = 1 / 3. * 0

    network = main(num_nodes=num_nodes,
                   sim_duration=10,
                   timeout=0.5,
                   base_latency=0.05,
                   latency_sigma_factor=0.5,
                   num_faulty_nodes=int(num_nodes * faulty_fraction),
                   num_slow_nodes=int(num_nodes * slow_fraction)
                   )