MINRELAYTXFEE = 5000 DIFF_RETARGET_INTERVAL = 2016 def get_block_timestamp(blockheight):
    '''Get the timestamp of a block specified by height.'''
    block = proxy.getblock(proxy.getblockhash(blockheight))
    return block.nTime def get_hashesperblock(blockheight):
    '''Get the expected number of hashes required per block.'''
    block = proxy.getblock(proxy.getblockhash(blockheight))
    nbits = hex(block.nBits)[2:]
    assert len(nbits) == 8
    significand = int(nbits[2:], base=16)
    exponent = (int(nbits[:2], base=16)-3)*8
    logtarget = log(significand, 2) + exponent

    # This is not technically correct because the target is fulfilled in the
    # case of equality. It should more correctly be:
    # 2**256 / (2**logtarget + 1),
    # but of course the difference is negligible.
    return 2**(256 - logtarget) class BlockMetadata(object):
    """Block metadata.

    Coinbase addresses / tag, block size, mean hashes per block
    (function of nBits).
    """

    def __init__(self, height):
        self.height = height
        self.addrs, self.tag = get_coinbase_info(height)
        self.size = get_block_size(height)
        self.hashes = get_hashesperblock(height)

    def get_poolname(self):
        pooltags = feemodel.config.pooltags
        assigned_name = None
        for name, taglist in pooltags.items():
            if any([tag in self.tag for tag in taglist]):
                if assigned_name is not None:
                    logger.warning("Multiple name assignment in block {}.".
                                   format(self.height))
                else:
                    assigned_name = name
        if assigned_name is None:
            for addr in self.addrs:
                if addr is not None:
                    assigned_name = addr[:12] + "_"
                    break
        if assigned_name is None:
            assigned_name = "UNKNOWN_" + str(self.height)

        return assigned_name

    def __repr__(self):
        return ("BlockMetaData(height: {}, size: {})".
                format(self.height, self.size)) class SimPool(object):

    def __init__(self, hashrate, maxblocksize, minfeerate):
        self.hashrate = hashrate
        self.maxblocksize = maxblocksize
        self.minfeerate = minfeerate

    def __cmp__(self, other):
        # TODO: deprecate this.
        raise NotImplementedError

    def __repr__(self):
        return ("SimPool{hashrate: %.2f, maxblocksize: %d, minfeerate: %.0f}" %
                (self.hashrate, self.maxblocksize, self.minfeerate))

    def __eq__(self, other):
        relevant_attrs = ['hashrate', 'maxblocksize', 'minfeerate']
        return all([
            getattr(self, attr) == getattr(other, attr)
            for attr in relevant_attrs])

    def __ne__(self, other):
        relevant_attrs = ['hashrate', 'maxblocksize', 'minfeerate']
        return any([
            getattr(self, attr) != getattr(other, attr)
            for attr in relevant_attrs]) class SimPools(object):

    def __init__(self, pools, blockrate=DEFAULT_BLOCKRATE):
        self.pools = pools
        self.blockrate = blockrate

    def check(self):
        if not self.pools:
            raise ValueError("No pools.")
        if any([pool.hashrate <= 0 or
                pool.maxblocksize <= 0 or
                pool.minfeerate < 0
                for pool in self.pools.values()]):
            raise ValueError("Bad pool stats.")
        if not any([pool.minfeerate < float("inf")
                    for pool in self.pools.values()]):
            raise ValueError("Zero pools capacity.")

    def blockgen(self):
        self.check()
        poolitems = sorted(self.pools.items(),
                           key=lambda item: item[1].hashrate)
        cumhashrates = list(
            cumsum_gen([pool.hashrate for name, pool in poolitems]))
        totalhashrate = cumhashrates[-1]
        prop_table = map(lambda hashrate: hashrate/totalhashrate,
                         cumhashrates)
        while True:
            poolidx = bisect_left(prop_table, random())
            simblock = SimBlock(*poolitems[poolidx])
            blockinterval = expovariate(self.blockrate)
            yield simblock, blockinterval

    def get_capacityfn(self):
        """Get cumulative capacity as function of minfeerate."""
        self.check()
        totalhashrate = self.calc_totalhashrate()

        def byterate_groupsum(grouptuple):
            return sum(
                [pool.hashrate*pool.maxblocksize
                 for pool in grouptuple[1]])*self.blockrate/totalhashrate

        pools = filter(
            lambda pool: pool.minfeerate < float("inf"),
            sorted(self.pools.values(), key=attrgetter("minfeerate")))
        feerates = sorted(set(map(attrgetter("minfeerate"), pools)))
        caps = list(cumsum_gen(
            groupby(pools, attrgetter("minfeerate")), mapfn=byterate_groupsum))

        feerates.insert(0, feerates[0]-1)
        caps.insert(0, 0)

        return StepFunction(feerates, caps)

    def get_hashratefn(self):
        """Get cumulative hashrate as function of minfeerate."""
        self.check()

        def hashrate_groupsum(grouptuple):
            return sum(map(attrgetter("hashrate"), grouptuple[1]))

        pools = filter(
            lambda pool: pool.minfeerate < float("inf"),
            sorted(self.pools.values(), key=attrgetter("minfeerate")))
        feerates = sorted(set(map(attrgetter("minfeerate"), pools)))
        hashrates = list(cumsum_gen(
            groupby(pools, attrgetter("minfeerate")), mapfn=hashrate_groupsum))

        feerates.insert(0, feerates[0]-1)
        hashrates.insert(0, 0)

        return StepFunction(feerates, hashrates)

    def calc_totalhashrate(self):
        return sum([pool.hashrate for pool in self.pools.values()])

    def __nonzero__(self):
        try:
            self.check()
        except ValueError:
            return False
        return True

    def __repr__(self):
        try:
            capfn = self.get_capacityfn()
        except ValueError:
            maxcap = 0
        else:
            maxcap = capfn[-1][1]
        return ("SimPools(Num: {}, MaxCap: {})".
                format(len(self.pools), maxcap))

    def __str__(self):
        try:
            self.check()
        except ValueError as e:
            return e.message
        poolitems = sorted(self.pools.items(),
                           key=lambda pitem: pitem[1].hashrate, reverse=True)
        totalhashrate = self.calc_totalhashrate()
        headers = ["Name", "HR", "Prop", "MBS", "MFR"]
        table = [(name,
                  pool.hashrate,
                  pool.hashrate / totalhashrate,
                  pool.maxblocksize,
                  pool.minfeerate)
                 for name, pool in poolitems]
        poolstats = tabulate(table, headers=headers)
        meanblocksize = sum([prop*mbs for _0, _1, prop, mbs, _2 in table])
        maxcap = meanblocksize*self.blockrate

        table = [
            ("Block interval (s)", 1 / self.blockrate),
            ("Total hashrate", totalhashrate),
            ("Max capacity (bytes/s)", maxcap)
        ]
        miscstats = tabulate(table)
        return poolstats + '\n' + miscstats

    def __eq__(self, other):
        return self.pools == other.pools

    def __ne__(self, other):
        return self.pools != other.pools class SimPoolsNP(object):

    def __init__(self, maxblocksizes, minfeerates,
                 blockrate=DEFAULT_BLOCKRATE):
        self.maxblocksizes = maxblocksizes
        self.minfeerates = minfeerates
        self.blockrate = blockrate

    def check(self):
        for attr in ['maxblocksizes', 'minfeerates', 'blockrate']:
            if not getattr(self, attr):
                raise ValueError("{} must be nonzero.".format(attr))

    def blockgen(self):
        self.check()
        while True:
            blockinterval = expovariate(self.blockrate)
            maxblocksize = choice(self.maxblocksizes)
            minfeerate = choice(self.minfeerates)
            pool = SimPool(1, maxblocksize, minfeerate)
            simblock = SimBlock('', pool)
            yield simblock, blockinterval

    def get_capacityfn(self):
        def cap_mapfn(y):
            return (y*expected_maxblocksize*self.blockrate /
                    len(self.minfeerates))

        expected_maxblocksize = (
            sum(self.maxblocksizes) / len(self.maxblocksizes))
        cap_fn = self.get_hashratefn()
        cap_fn._y = map(cap_mapfn, cap_fn._y)

        return cap_fn

    def get_hashratefn(self):
        self.check()
        feerates_all = filter(lambda f: f < float("inf"), self.minfeerates)
        feerates_count = Counter(feerates_all)
        feerates, counts = map(list, zip(*sorted(feerates_count.items())))

        caps = list(cumsum_gen(counts))
        feerates.insert(0, feerates[0]-1)
        caps.insert(0, 0)

        return StepFunction(feerates, caps)

    def calc_totalhashrate(self):
        return len(self.minfeerates)

    def __nonzero__(self):
        try:
            self.check()
        except ValueError:
            return False
        return True

    def __eq__(self, other):
        return all([
            getattr(self, attr) == getattr(other, attr)
            for attr in ['maxblocksizes', 'minfeerates', 'blockrate']]) class MemBlock(BaseMemBlock):
    '''The mempool state at the time a block was discovered.'''

    def write(self, dbfile, blocks_to_keep):
        '''Write MemBlock to disk.

        blocks_to_keep specifies how many blocks of information should be
        retained. All MemBlocks older (with respect to this block) than
        blocks_to_keep will be deleted.
        '''
        if not self:
            raise ValueError("Failed write: empty memblock.")

        # Temp tables
        NONREMOVED = "nonremoved"
        MEMBLOCKTXS = "memblocktxs"

        db = None
        memblocktxids = self.entries.keys()

        try:
            with db_lock:
                db = sqlite3.connect(dbfile)
                for key, val in MEMBLOCK_SCHEMA.items():
                    db.execute('CREATE TABLE IF NOT EXISTS %s (%s)' %
                               (key, ','.join(val)))
                db.execute('CREATE INDEX IF NOT EXISTS heightidx '
                           'ON txs (heightremoved)')
                db.execute('CREATE INDEX IF NOT EXISTS block_heightidx '
                           'ON blocktxs (blockheight)')

                # Enter into blocks
                db.execute(
                    'INSERT INTO blocks VALUES (?,?,?)',
                    (self.blockheight, self.blocksize, self.time))

                # Temporary tables for data manipulation
                db.execute(
                    "CREATE TEMP TABLE {} (id INTEGER, txid TEXT)".
                    format(NONREMOVED))
                db.execute(
                    "CREATE TEMP TABLE {} "
                    "(txid TEXT, isconflict INTEGER, inblock INTEGER)".
                    format(MEMBLOCKTXS))
                # Fetch the nonremoved txs
                db.execute(
                    "INSERT INTO {} "
                    "SELECT id, txid FROM txs "
                    "WHERE heightremoved IS NULL".format(NONREMOVED)
                )
                # Table the memblocktxs
                db.executemany(
                    "INSERT INTO {} VALUES (?,?,?)".format(MEMBLOCKTXS),
                    [(txid,
                      self.entries[txid].isconflict,
                      self.entries[txid].inblock)
                     for txid in memblocktxids])
                # Update the heightremoved
                db.execute(
                    "UPDATE txs SET heightremoved=? "
                    "WHERE id IN "
                    "(SELECT id FROM {0} LEFT JOIN {1} "
                    " ON {0}.txid={1}.txid WHERE "
                    " {1}.isconflict=1 OR "
                    " {1}.inblock=1 OR "
                    " {1}.inblock is NULL)".
                    format(NONREMOVED, MEMBLOCKTXS),
                    (self.blockheight,)
                )
                # Get the new txs to table
                txidstoenter = db.execute(
                    "SELECT txid FROM {} EXCEPT SELECT txid FROM {}".
                    format(MEMBLOCKTXS, NONREMOVED)
                )
                txstoenter = [
                    (
                        txid,
                        self.entries[txid].size,
                        str(self.entries[txid].fee),
                        str(self.entries[txid].startingpriority),
                        self.entries[txid].time,
                        self.entries[txid].height,
                        ','.join(self.entries[txid].depends),
                        self.entries[txid].feerate,
                        self.blockheight if (
                            self.entries[txid].isconflict or
                            self.entries[txid].inblock)
                        else None
                    )
                    for txid in map(itemgetter(0), txidstoenter)
                ]
                # Enter new txs. There might be duplicate txid,
                # but that's OK!
                db.executemany(
                    "INSERT INTO txs(txid, size, fee, startingpriority, "
                    "time, height, depends, feerate, heightremoved) "
                    "VALUES (?,?,?,?,?,?,?,?,?)", txstoenter)

                # Get the rowids, to enter into blocktxs
                finaltxs = db.execute(
                    "SELECT id, txid FROM txs WHERE "
                    "heightremoved IS NULL OR "
                    "heightremoved=?",
                    (self.blockheight,)
                ).fetchall()
                rowidmap = {txid: rowid for rowid, txid in finaltxs}
                # Assert that there are no duplicate txids
                assert len(finaltxs) == len(set(map(itemgetter(1), finaltxs)))
                # Enter into blocktxs
                blocktxstoenter = [(
                    self.blockheight,
                    rowidmap[txid],
                    str(self.entries[txid].currentpriority),
                    self.entries[txid].isconflict,
                    self.entries[txid].inblock)
                    for txid in memblocktxids
                ]
                db.executemany("INSERT INTO blocktxs VALUES (?,?,?,?,?)",
                               blocktxstoenter)

                # Remove old blocks
                if blocks_to_keep > 0:
                    height_thresh = self.blockheight - blocks_to_keep
                    db.execute("DELETE FROM txs WHERE heightremoved<=?",
                               (height_thresh,))
                    db.execute("DELETE FROM blocks WHERE height<=?",
                               (height_thresh,))
                    db.execute("DELETE FROM blocktxs WHERE blockheight<=?",
                               (height_thresh,))
                db.commit()
        finally:
            if db is not None:
                db.close()

    @classmethod
    def read(cls, blockheight, dbfile=MEMBLOCK_DBFILE):
        '''Read MemBlock from disk.

        Returns the memblock with specified blockheight.
        Returns None if no record exists for that block.
        Raises one of the sqlite3 errors if there are other problems.
        '''
        if not os.path.exists(dbfile):
            return None
        db = None
        try:
            db = sqlite3.connect(dbfile)
            with db_lock:
                block = db.execute('SELECT size, time FROM blocks '
                                   'WHERE height=?',
                                   (blockheight,)).fetchall()
                txlist = db.execute(
                    "SELECT "
                    "   txid,"
                    "   size,"
                    "   fee,"
                    "   startingpriority,"
                    "   currentpriority,"
                    "   time,"
                    "   height,"
                    "   depends,"
                    "   feerate,"
                    "   isconflict,"
                    "   inblock "
                    "FROM blocktxs LEFT JOIN txs ON blocktxs.txrowid=txs.id "
                    "WHERE blockheight=?",
                    (blockheight,)).fetchall()
        finally:
            if db is not None:
                db.close()

        # Make sure there are no missing txs.
        txids = map(itemgetter(0), txlist)
        assert not any([txid is None for txid in txids])

        if block:
            blocksize, blocktime = block[0]
        else:
            return None
        memblock = cls()
        memblock.height = blockheight - 1
        entries = {}
        for tx in txlist:
            entry = MemEntry()
            entry.size = tx[1]
            entry.fee = decimal.Decimal(tx[2])
            entry.startingpriority = decimal.Decimal(tx[3])
            entry.currentpriority = decimal.Decimal(tx[4])
            entry.time = tx[5]
            entry.height = tx[6]
            entry.depends = tx[7].split(",") if tx[7] else []
            # We need to do this because depends is recorded upon first sight
            # of the tx; some deps might have confirmed in the meantime
            entry.depends = filter(lambda dep: dep in txids, entry.depends)
            entry.feerate = tx[8]
            entry.isconflict = bool(tx[9])
            entry.inblock = bool(tx[10])
            entry.leadtime = blocktime - tx[5]
            entries[tx[0]] = entry
        memblock.entries = entries
        memblock.time = blocktime
        memblock.blockheight = blockheight
        memblock.blocksize = blocksize
        return memblock

    @staticmethod
    def get_heights(blockrangetuple=None, dbfile=MEMBLOCK_DBFILE):
        '''Get the list of MemBlocks stored on disk.

        Returns a list of heights of all MemBlocks on disk within
        range(*blockrangetuple)
        '''
        if not os.path.exists(dbfile):
            return []
        if blockrangetuple is None:
            blockrangetuple = (0, float("inf"))
        db = None
        try:
            db = sqlite3.connect(dbfile)
            with db_lock:
                heights = db.execute(
                    'SELECT height FROM blocks '
                    'where height>=? and height <?',
                    blockrangetuple).fetchall()
        finally:
            if db is not None:
                db.close()
        return [r[0] for r in heights] MEMBLOCK_DBFILE = os.path.join(datadir, 'memblock.db')
import logging
import feemodel.config
from time import time
from math import ceil
from collections import defaultdict
from operator import attrgetter, itemgetter
from tabulate import tabulate
from feemodel.config import MINRELAYTXFEE, DIFF_RETARGET_INTERVAL
from feemodel.util import (get_block_timestamp, get_hashesperblock,
                           BlockMetadata)
from feemodel.stranding import tx_preprocess, calc_stranding_feerate
from feemodel.simul.pools import SimPool, SimPools, SimPoolsNP
from feemodel.txmempool import MemBlock, MEMBLOCK_DBFILE
from __future__ import division




logger = logging.getLogger(__name__)

MAX_TXS = 50000


class PoolsEstimatorNP(SimPoolsNP):

    def __init__(self):
        super(PoolsEstimatorNP, self).__init__(None, None, blockrate=None)
        self.blockstats = {}

    def start(self, blockrangetuple, stopflag=None, dbfile=MEMBLOCK_DBFILE):
        logger.info("Beginning NP pool estimation "
                    "from blockrange({}, {})".format(*blockrangetuple))
        self._clear_window(blockrangetuple[0])
        for height in range(*blockrangetuple):
            if height in self.blockstats:
                continue
            if stopflag and stopflag.is_set():
                raise StopIteration("Stop flag set.")

            memblock = MemBlock.read(height, dbfile=dbfile)