class Taker(object):
    class _MakerTxData(NamedTuple):
        nick: Any
        utxo_data: Any
        total_input: Any
        change_amount: Any
        real_cjfee: Any
        utxo_list: Any = None
        cj_addr: Any = None
        change_addr: Any = None

    def __init__(self,
                 wallet_service,
                 schedule,
                 max_cj_fee,
                 order_chooser=fidelity_bond_weighted_order_choose,
                 callbacks=None,
                 tdestaddrs=None,
                 custom_change_address=None,
                 ignored_makers=None):
        """`schedule`` must be a list of tuples: (see sample_schedule_for_testnet
        for explanation of syntax, also schedule.py module in this directory),
        which will be a sequence of joins to do.
        `max_cj_fee` must be a tuple of form: (float, int or float) where the first
        is the maximum relative fee as a decimal and the second is the maximum
        absolute fee in satoshis.
        Callbacks:
        External callers set the 3 callbacks for filtering orders,
        sending info messages to client, and action on completion.
        "None" is allowable for taker_info_callback, defaults to log msg.
        Callback function definitions:
        =====================
        filter_orders_callback
        =====================
        args:
        1. orders_fees - a list of two items 1. orders dict 2 total cjfee
        2. cjamount - coinjoin amount in satoshis
        returns:
        False - offers rejected OR
        True - offers accepted OR
        'retry' - offers not accepted but try again
        =======================
        on_finished_callback
        =======================
        args:
        1. res - True means tx successful, False means tx unsucessful
        2. fromtx - True means not the final transaction, False means final
         (end of schedule), 'unconfirmed' means tx seen on the network only.
        3. waittime - passed in minutes, time to wait after confirmation before
         continuing to next tx (thus, only used if fromtx is True).
        4. txdetails - a tuple (txd, txid) - only to be used when fromtx
         is 'unconfirmed', used for monitoring.
        returns:
        None
        ========================
        taker_info_callback
        ========================
        args:
        1. type - one of 'ABORT' or 'INFO', the former signals the client that
         processing of this transaction is aborted, the latter is only an update.
        2. message - an information message.
        returns:
        None
        """
        self.aborted = False
        assert isinstance(wallet_service, WalletService)
        self.wallet_service = wallet_service
        self.schedule = schedule
        self.order_chooser = order_chooser
        self.max_cj_fee = max_cj_fee
        self.custom_change_address = custom_change_address

        #List (which persists between transactions) of makers
        #who have not responded or behaved maliciously at any
        #stage of the protocol.
        self.ignored_makers = [] if not ignored_makers else ignored_makers

        #Used in attempts to complete with subset after second round failure:
        self.honest_makers = []
        #Toggle: if set, only honest makers will be used from orderbook
        self.honest_only = False

        #Temporary (per transaction) list of makers that keeps track of
        #which have responded, both in Stage 1 and Stage 2. Before each
        #stage, the list is set to the full set of expected responders,
        #and entries are removed when honest responses are received;
        #emptiness of the list can be used to trigger completion of
        #processing.
        self.nonrespondants = []

        self.waiting_for_conf = False
        self.txid = None
        self.schedule_index = -1
        self.utxos = {}
        self.maker_utxo_data = {}
        self.tdestaddrs = [] if not tdestaddrs else tdestaddrs
        self.filter_orders_callback = callbacks[0]
        self.taker_info_callback = callbacks[1]
        if not self.taker_info_callback:
            self.taker_info_callback = self.default_taker_info_callback
        self.on_finished_callback = callbacks[2]

    def default_taker_info_callback(self, infotype, msg):
        jlog.info(infotype + ":" + msg)

    def add_ignored_makers(self, makers):
        """Makers should be added to this list when they have refused to
        complete the protocol honestly, and should remain in this set
        for the duration of the Taker run (so, the whole schedule).
        """
        self.ignored_makers.extend(makers)
        self.ignored_makers = list(set(self.ignored_makers))

    def add_honest_makers(self, makers):
        """A maker who has shown willigness to complete the protocol
        by returning a valid signature for a coinjoin can be added to
        this list, the taker can optionally choose to only source
        offers from thus-defined "honest" makers.
        """
        self.honest_makers.extend(makers)
        self.honest_makers = list(set(self.honest_makers))

    def set_honest_only(self, truefalse):
        """Toggle; if set, offers will only be accepted
        from makers in the self.honest_makers list.
        This should not be called unless we already have
        a list of such honest makers (see add_honest_makers()).
        """
        if truefalse:
            if not len(self.honest_makers):
                jlog.debug("Attempt to set honest-only without "
                           "any honest makers; ignored.")
                return
        self.honest_only = truefalse

    def initialize(self, orderbook, fidelity_bonds_info):
        """Once the daemon is active and has returned the current orderbook,
        select offers, re-initialize variables and prepare a commitment,
        then send it to the protocol to fill offers.
        """
        if self.aborted:
            return (False,)
        self.taker_info_callback("INFO", "Received offers from joinmarket pit")
        #choose the next item in the schedule
        self.schedule_index += 1
        if self.schedule_index == len(self.schedule):
            self.taker_info_callback("INFO", "Finished all scheduled transactions")
            self.on_finished_callback(True)
            return (False,)
        else:
            #read the settings from the schedule entry
            si = self.schedule[self.schedule_index]
            self.mixdepth = si[0]
            self.cjamount = si[1]
            rounding = si[5]
            #non-integer coinjoin amounts are treated as fractions
            #this is currently used by the tumbler algo
            if isinstance(self.cjamount, float):
                #the mixdepth balance is fixed at the *start* of each new
                #mixdepth in tumble schedules:
                if self.schedule_index == 0 or si[0] != self.schedule[
                    self.schedule_index - 1]:
                    self.mixdepthbal = self.wallet_service.get_balance_by_mixdepth(
                        )[self.mixdepth]
                #reset to satoshis
                self.cjamount = int(self.cjamount * self.mixdepthbal)
                if rounding != NO_ROUNDING:
                    self.cjamount = round_to_significant_figures(self.cjamount,
                        rounding)
                if self.cjamount < jm_single().mincjamount:
                    jlog.info("Coinjoin amount too low, bringing up to: " +
                        btc.amount_to_str(jm_single().mincjamount))
                    self.cjamount = jm_single().mincjamount
            self.n_counterparties = si[2]
            self.my_cj_addr = si[3]
            # for sweeps to external addresses we need an in-wallet import
            # for the transaction monitor (this will be a no-op for txs to
            # in-wallet addresses).
            if self.cjamount == 0 and self.my_cj_addr != "INTERNAL":
                self.wallet_service.import_non_wallet_address(self.my_cj_addr)

            #if destination is flagged "INTERNAL", choose a destination
            #from the next mixdepth modulo the maxmixdepth
            if self.my_cj_addr == "INTERNAL":
                next_mixdepth = (self.mixdepth + 1) % (
                    self.wallet_service.mixdepth + 1)
                jlog.info("Choosing a destination from mixdepth: " + str(
                    next_mixdepth))
                self.my_cj_addr = self.wallet_service.get_internal_addr(next_mixdepth)
                jlog.info("Chose destination address: " + self.my_cj_addr)
            self.outputs = []
            self.cjfee_total = 0
            self.maker_txfee_contributions = 0
            self.latest_tx = None
            self.txid = None

        fidelity_bond_values = calculate_fidelity_bond_values(fidelity_bonds_info)
        for offer in orderbook:
            #having no fidelity bond is like having a zero value fidelity bond
            offer["fidelity_bond_value"] = fidelity_bond_values.get(offer["counterparty"], 0)

        sweep = True if self.cjamount == 0 else False
        if not self.filter_orderbook(orderbook, sweep):
            return (False,)
        #choose coins to spend
        self.taker_info_callback("INFO", "Preparing bitcoin data..")
        if not self.prepare_my_bitcoin_data():
            return (False,)
        #Prepare a commitment
        commitment, revelation, errmsg = self.make_commitment()
        if not commitment:
            utxo_pairs, to, ts = revelation
            if len(to) == 0:
                #If any utxos are too new, then we can continue retrying
                #until they get old enough; otherwise, we have to abort
                #(TODO, it's possible for user to dynamically add more coins,
                #consider if this option means we should stay alive).
                self.taker_info_callback("ABORT", errmsg)
                return ("commitment-failure",)
            else:
                self.taker_info_callback("INFO", errmsg)
                return (False,)
        else:
            self.taker_info_callback("INFO", errmsg)

        #Initialization has been successful. We must set the nonrespondants
        #now to keep track of what changed when we receive the utxo data
        self.nonrespondants = list(self.orderbook.keys())
        return (True, self.cjamount, commitment, revelation, self.orderbook)

    def filter_orderbook(self, orderbook, sweep=False):
        #If honesty filter is set, we immediately filter to only the prescribed
        #honest makers before continuing. In this case, the number of
        #counterparties should already match, and this has to be set by the
        #script instantiating the Taker.
        #Note: If one or more of the honest makers has dropped out in the meantime,
        #we will just have insufficient offers and it will fail in the usual way
        #for insufficient liquidity.
        if self.honest_only:
            orderbook = [o for o in orderbook if o['counterparty'] in self.honest_makers]
        if sweep:
            self.orderbook = orderbook #offers choosing deferred to next step
        else:
            if self.wallet_service.get_txtype() == "p2pkh":
                allowed_types = ["reloffer", "absoffer"]
            elif self.wallet_service.get_txtype() == "p2sh-p2wpkh":
                allowed_types = ["swreloffer", "swabsoffer"]
            elif self.wallet_service.get_txtype() == "p2wpkh":
                allowed_types = ["sw0reloffer", "sw0absoffer"]
            else:
                jlog.error("Unrecognized wallet type, taker cannot continue.")
                return False
            self.orderbook, self.total_cj_fee = choose_orders(
                orderbook, self.cjamount, self.n_counterparties, self.order_chooser,
                self.ignored_makers, allowed_types=allowed_types,
                max_cj_fee=self.max_cj_fee)
            if self.orderbook is None:
                #Failure to get an orderbook means order selection failed
                #for some reason; no action is taken, we let the stallMonitor
                # + the finished callback decide whether to retry.
                return False
            if self.filter_orders_callback:
                accepted = self.filter_orders_callback([self.orderbook,
                                                        self.total_cj_fee],
                                                       self.cjamount)
                if accepted == "retry":
                    #Special condition if Taker is "determined to continue"
                    #(such as tumbler); even though these offers are rejected,
                    #we don't trigger the finished callback; see above note on
                    #`if self.orderbook is None`
                    return False
                if not accepted:
                    return False
        return True

    def prepare_my_bitcoin_data(self):
        """Get a coinjoin address and a change address; prepare inputs
        appropriate for this transaction"""
        if not self.my_cj_addr:
            #previously used for donations; TODO reimplement?
            raise NotImplementedError
        if self.cjamount != 0:
            if self.custom_change_address:
                self.my_change_addr = self.custom_change_address
            else:
                try:
                    self.my_change_addr = self.wallet_service.get_internal_addr(self.mixdepth)
                except:
                    self.taker_info_callback("ABORT", "Failed to get a change address")
                    return False
            #adjust the required amount upwards to anticipate an increase in
            #transaction fees after re-estimation; this is sufficiently conservative
            #to make failures unlikely while keeping the occurence of failure to
            #find sufficient utxos extremely rare. Indeed, a doubling of 'normal'
            #txfee indicates undesirable behaviour on maker side anyway.
            self.total_txfee = estimate_tx_fee(3, 2,
                txtype=self.wallet_service.get_txtype()) * (self.n_counterparties - 1) + \
                estimate_tx_fee(3, 2, txtype=self.wallet_service.get_txtype(),
                                outtype=self.wallet_service.get_outtype(self.coinjoin_address()))
            total_amount = self.cjamount + self.total_cj_fee + self.total_txfee
            jlog.info('total estimated amount spent = ' + btc.amount_to_str(total_amount))
            try:
                self.input_utxos = self.wallet_service.select_utxos(self.mixdepth, total_amount,
                                                        minconfs=1)
            except Exception as e:
                self.taker_info_callback("ABORT",
                                    "Unable to select sufficient coins: " + repr(e))
                return False
        else:
            #sweep
            self.input_utxos = self.wallet_service.get_utxos_by_mixdepth()[self.mixdepth]
            self.my_change_addr = None
            #do our best to estimate the fee based on the number of
            #our own utxos; this estimate may be significantly higher
            #than the default set in option.txfee * makercount, where
            #we have a large number of utxos to spend. If it is smaller,
            #we'll be conservative and retain the original estimate.
            est_ins = len(self.input_utxos)+3*self.n_counterparties
            jlog.debug("Estimated ins: "+str(est_ins))
            est_outs = 2*self.n_counterparties + 1
            jlog.debug("Estimated outs: "+str(est_outs))
            self.total_txfee = estimate_tx_fee(
                est_ins, est_outs, txtype=self.wallet_service.get_txtype(),
                outtype=self.wallet_service.get_outtype(self.coinjoin_address()))
            jlog.debug("We have a fee estimate: "+str(self.total_txfee))
            total_value = sum([va['value'] for va in self.input_utxos.values()])
            if self.wallet_service.get_txtype() == "p2pkh":
                allowed_types = ["reloffer", "absoffer"]
            elif self.wallet_service.get_txtype() == "p2sh-p2wpkh":
                allowed_types = ["swreloffer", "swabsoffer"]
            elif self.wallet_service.get_txtype() == "p2wpkh":
                allowed_types = ["sw0reloffer", "sw0absoffer"]
            else:
                jlog.error("Unrecognized wallet type, taker cannot continue.")
                return False
            self.orderbook, self.cjamount, self.total_cj_fee = choose_sweep_orders(
                self.orderbook, total_value, self.total_txfee,
                self.n_counterparties, self.order_chooser,
                self.ignored_makers, allowed_types=allowed_types,
                max_cj_fee=self.max_cj_fee)
            if not self.orderbook:
                self.taker_info_callback("ABORT",
                                "Could not find orders to complete transaction")
                return False
            if self.filter_orders_callback:
                if not self.filter_orders_callback((self.orderbook,
                                                    self.total_cj_fee),
                                                   self.cjamount):
                    return False

        self.utxos = {None: list(self.input_utxos.keys())}
        return True

    @hexbin
    def receive_utxos(self, ioauth_data):
        """Triggered when the daemon returns utxo data from
        makers who responded; this is the completion of phase 1
        of the protocol
        """
        if self.aborted:
            return (False, "User aborted")

        self.maker_utxo_data = {}

        verified_data = self._verify_ioauth_data(ioauth_data)
        for maker_inputs in verified_data:
            # We have succesfully processed the data from this nick
            self.utxos[maker_inputs.nick] = maker_inputs.utxo_list
            self.outputs.append({'address': maker_inputs.change_addr,
                                 'value': maker_inputs.change_amount})
            self.outputs.append({'address': maker_inputs.cj_addr,
                                 'value': self.cjamount})
            self.cjfee_total += maker_inputs.real_cjfee
            self.maker_txfee_contributions +=\
                self.orderbook[maker_inputs.nick]['txfee']
            self.maker_utxo_data[maker_inputs.nick] = maker_inputs.utxo_data
            jlog.info(
                f"fee breakdown for {maker_inputs.nick} "
                f"totalin={maker_inputs.total_input:d} "
                f"cjamount={self.cjamount:d} "
                f"txfee={self.orderbook[maker_inputs.nick]['txfee']:d} "
                f"realcjfee={maker_inputs.real_cjfee:d}")

            try:
                self.nonrespondants.remove(maker_inputs.nick)
            except Exception as e:
                jlog.warn(
                    "Failure to remove counterparty from nonrespondants list:"
                    f" {maker_inputs.nick}), error message: {repr(e)})")

        #Apply business logic of how many counterparties are enough; note that
        #this must occur after the above ioauth data processing, since we only now
        #know for sure that the data meets all business-logic requirements.
        if len(self.maker_utxo_data) < jm_single().config.getint(
                "POLICY", "minimum_makers"):
            self.taker_info_callback("INFO", "Not enough counterparties, aborting.")
            return (False,
                    "Not enough counterparties responded to fill, giving up")

        self.taker_info_callback("INFO", "Got all parts, enough to build a tx")

        #The list self.nonrespondants is now reset and
        #used to track return of signatures for phase 2
        self.nonrespondants = list(self.maker_utxo_data.keys())

        my_total_in = sum([va['value'] for u, va in self.input_utxos.items()])
        if self.my_change_addr:
            #Estimate fee per choice of next/3/6 blocks targetting.
            estimated_fee = estimate_tx_fee(
                len(sum(self.utxos.values(), [])), len(self.outputs) + 2,
                txtype=self.wallet_service.get_txtype(),
                outtype=self.wallet_service.get_outtype(self.coinjoin_address()))
            jlog.info("Based on initial guess: " +
                      btc.amount_to_str(self.total_txfee) +
                      ", we estimated a miner fee of: " +
                      btc.amount_to_str(estimated_fee))
            #reset total
            self.total_txfee = estimated_fee
        my_txfee = max(self.total_txfee - self.maker_txfee_contributions, 0)
        my_change_value = (
            my_total_in - self.cjamount - self.cjfee_total - my_txfee)
        #Since we could not predict the maker's inputs, we may end up needing
        #too much such that the change value is negative or small. Note that
        #we have tried to avoid this based on over-estimating the needed amount
        #in SendPayment.create_tx(), but it is still a possibility if one maker
        #uses a *lot* of inputs.
        if self.my_change_addr:
            if my_change_value < -1:
                raise ValueError("Calculated transaction fee of: " +
                    btc.amount_to_str(self.total_txfee) +
                    " is too large for our inputs; Please try again.")
            if my_change_value <= jm_single().BITCOIN_DUST_THRESHOLD:
                jlog.info("Dynamically calculated change lower than dust: " +
                    btc.amount_to_str(my_change_value) + "; dropping.")
                self.my_change_addr = None
                my_change_value = 0
        jlog.info(
            'fee breakdown for me totalin=%d my_txfee=%d makers_txfee=%d cjfee_total=%d => changevalue=%d'
            % (my_total_in, my_txfee, self.maker_txfee_contributions,
               self.cjfee_total, my_change_value))
        if self.my_change_addr is None:
            if my_change_value != 0 and abs(my_change_value) != 1:
                # seems you wont always get exactly zero because of integer
                # rounding so 1 satoshi extra or fewer being spent as miner
                # fees is acceptable
                jlog.info(
                ('WARNING CHANGE NOT BEING USED\nCHANGEVALUE = {}').format(
                    my_change_value))
            # we need to check whether the *achieved* txfee-rate is outside
            # the range allowed by the user in config; if not, abort the tx.
            # this is done with using the same estimate fee function and comparing
            # the totals; this ratio will correspond to the ratio of the feerates.
            num_ins = len([u for u in sum(self.utxos.values(), [])])
            num_outs = len(self.outputs) + 1
            new_total_fee = estimate_tx_fee(num_ins, num_outs,
                                    txtype=self.wallet_service.get_txtype(),
                                    outtype=self.wallet_service.get_outtype(self.coinjoin_address()))
            feeratio = new_total_fee/self.total_txfee
            jlog.debug("Ratio of actual to estimated sweep fee: {}".format(
                feeratio))
            sweep_delta = float(jm_single().config.get("POLICY",
                                                       "max_sweep_fee_change"))
            if feeratio < 1 - sweep_delta or feeratio > 1 + sweep_delta:
                jlog.warn("Transaction fee for sweep: {} too far from expected:"
                          " {}; check the setting 'max_sweep_fee_change'"
                          " in joinmarket.cfg. Aborting this attempt.".format(
                              new_total_fee, self.total_txfee))
                return (False, "Unacceptable feerate for sweep, giving up.")
        else:
            self.outputs.append({'address': self.my_change_addr,
                                 'value': my_change_value})
        self.utxo_tx = [u for u in sum(self.utxos.values(), [])]
        self.outputs.append({'address': self.coinjoin_address(),
                             'value': self.cjamount})
        # pre-Nov-2020/v0.8.0: transactions used ver 1 and nlocktime 0
        # so only the new "pit" (using native segwit) will use the updated
        # version 2 and nlocktime ~ current block as per normal payments.
        # TODO makers do not check this; while there is no security risk,
        # it might be better for them to sanity check.
        if self.wallet_service.get_txtype() == "p2wpkh":
            n_version = 2
            locktime = compute_tx_locktime()
        else:
            n_version = 1
            locktime = 0
        self.latest_tx = btc.make_shuffled_tx(self.utxo_tx, self.outputs,
                                              version=n_version, locktime=locktime)
        jlog.info('obtained tx\n' + btc.human_readable_transaction(
            self.latest_tx))

        self.taker_info_callback("INFO", "Built tx, sending to counterparties.")
        return (True, list(self.maker_utxo_data.keys()),
                self.latest_tx.serialize())

    def _verify_ioauth_data(self, ioauth_data):
        verified_data = []
        # Need to authorize against the btc pubkey first.
        for nick, nickdata in ioauth_data.items():
            utxo_list, auth_pub, cj_addr, change_addr, btc_sig, maker_pk = nickdata
            if not self.auth_counterparty(btc_sig, auth_pub, maker_pk):
                jlog.debug(
                    "Counterparty encryption verification failed, aborting: " + nick)
                # This counterparty must be rejected
                continue

            if not validate_address(cj_addr)[0]\
                    or not validate_address(change_addr)[0]:
                jlog.warn("Counterparty provided invalid address: {}".format(
                    (cj_addr, change_addr)))
                # Interpreted as malicious
                self.add_ignored_makers([nick])
                continue

            try:
                maker_inputs_data = self._verify_ioauth_inputs(
                    nick, utxo_list, auth_pub)
            except IoauthInputVerificationError as e:
                for msg in e.messages:
                    jlog.warning(msg)
                continue

            verified_data.append(maker_inputs_data._replace(
                utxo_list=utxo_list, cj_addr=cj_addr, change_addr=change_addr))
        return verified_data

    def _verify_ioauth_inputs(self, nick, utxo_list, auth_pub):
        utxo_data = jm_single().bc_interface.query_utxo_set(utxo_list)
        if None in utxo_data:
            raise IoauthInputVerificationError([
                "ERROR: outputs unconfirmed or already spent. utxo_data="
                f"{pprint.pformat(utxo_data)}",
                "Disregarding this counterparty."])

        # Complete maker authorization:
        # Extract the address fields from the utxos
        # Construct the Bitcoin address for the auth_pub field
        # Ensure that at least one address from utxos corresponds.
        for inp in utxo_data:
            try:
                if self.wallet_service.pubkey_has_script(
                        auth_pub, inp['script']):
                    break
            except EngineError as e:
                pass
        else:
            raise IoauthInputVerificationError([
                f"ERROR maker's ({nick}) authorising pubkey is not included "
                "in the transaction!"])

        total_input = sum([d['value'] for d in utxo_data])
        real_cjfee = calc_cj_fee(self.orderbook[nick]['ordertype'],
                                 self.orderbook[nick]['cjfee'],
                                 self.cjamount)
        change_amount = (total_input - self.cjamount -
                         self.orderbook[nick]['txfee'] + real_cjfee)

        # certain malicious and/or incompetent liquidity providers send
        # inputs totalling less than the coinjoin amount! this leads to
        # a change output of zero satoshis; this counterparty must be removed.
        if change_amount < jm_single().DUST_THRESHOLD:
            raise IoauthInputVerificationError([
                f"ERROR counterparty requires sub-dust change. nick={nick} "
                f"totalin={total_input:d} cjamount={self.cjamount:d} "
                f"change={change_amount:d}",
                f"Invalid change, too small, nick={nick}"])
        return self._MakerTxData(nick, utxo_data, total_input, change_amount,
                                 real_cjfee)

    @hexbin
    def auth_counterparty(self, btc_sig, auth_pub, maker_pk):
        """Validate the counterpartys claim to own the btc
        address/pubkey that will be used for coinjoining
        with an ecdsa verification.
        """
        try:
            # maker pubkey as message is in hex format:
            if not btc.ecdsa_verify(bintohex(maker_pk), btc_sig, auth_pub):
                jlog.debug('signature didnt match pubkey and message')
                return False
        except Exception as e:
            jlog.info("Failed ecdsa verify for maker pubkey: " + bintohex(maker_pk))
            jlog.info("Exception was: " + repr(e))
            return False
        return True

    def on_sig(self, nick, sigb64):
        """Processes transaction signatures from counterparties.
        If all signatures received correctly, returns the result
        of self.self_sign_and_push() (i.e. we complete the signing
        and broadcast); else returns False (thus returns False for
        all but last signature).
        """
        if self.aborted:
            return False
        if nick not in self.nonrespondants:
            jlog.debug(('add_signature => nick={} '
                       'not in nonrespondants {}').format(nick, self.nonrespondants))
            return False
        sig = base64.b64decode(sigb64)
        inserted_sig = False

        # batch retrieval of utxo data
        utxo = {}
        ctr = 0
        for index, ins in enumerate(self.latest_tx.vin):
            if self._is_our_input(ins) or ins.scriptSig != b"":
                continue
            utxo_for_checking = (ins.prevout.hash[::-1], ins.prevout.n)
            utxo[ctr] = [index, utxo_for_checking]
            ctr += 1
        utxo_data = jm_single().bc_interface.query_utxo_set([x[
            1] for x in utxo.values()])
        # insert signatures
        for i, u in utxo.items():
            if utxo_data[i] is None:
                continue
            # Check if the sender included the scriptCode in the sig message;
            # if so, also pick up the amount from the utxo data retrieved
            # from the blockchain to verify the segwit-style signature.
            # Note that this allows a mixed SW/non-SW transaction as each utxo
            # is interpreted separately.
            try:
                sig_deserialized = [a for a in iter(btc.CScript(sig))]
            except Exception as e:
                jlog.debug("Failed to parse junk sig message, ignoring.")
                break
            # abort in case we were given a junk sig (note this previously had
            # to check to avoid crashes in verify_tx_input, no longer (Feb 2020)):
            if not all([x for x in sig_deserialized]):
                jlog.debug("Junk signature: " + str(sig_deserialized) + \
                          ", not attempting to verify")
                break
            # The second case here is kept for backwards compatibility.
            if len(sig_deserialized) == 2:
                ver_sig, ver_pub = sig_deserialized
            elif len(sig_deserialized) == 3:
                ver_sig, ver_pub, _ = sig_deserialized
            else:
                jlog.debug("Invalid signature message - not 2 or 3 items")
                break

            scriptPubKey = btc.CScript(utxo_data[i]['script'])
            is_witness_input = scriptPubKey.is_p2sh() or scriptPubKey.is_witness_v0_keyhash()
            ver_amt = utxo_data[i]['value'] if is_witness_input else None
            witness = btc.CScriptWitness(
                [ver_sig, ver_pub]) if is_witness_input else None

            # don't attempt to parse `pub` as pubkey unless it's valid.
            if scriptPubKey.is_p2sh():
                try:
                    s = btc.pubkey_to_p2wpkh_script(ver_pub)
                except:
                    jlog.debug("Junk signature message, invalid pubkey, ignoring.")
                    break

            if scriptPubKey.is_witness_v0_keyhash():
                scriptSig = btc.CScript(b'')
            elif scriptPubKey.is_p2sh():
                scriptSig = btc.CScript([s])
            else:
                scriptSig = btc.CScript([ver_sig, ver_pub])

            sig_good = btc.verify_tx_input(self.latest_tx, u[0], scriptSig,
                    scriptPubKey, amount=ver_amt, witness=witness)

            if sig_good:
                jlog.debug('found good sig at index=%d' % (u[0]))

                # Note that, due to the complexity of handling multisig or other
                # arbitrary script (considering sending multiple signatures OTW),
                # there is an assumption of p2sh-p2wpkh or p2wpkh, for the segwit
                # case.
                self.latest_tx.vin[u[0]].scriptSig = scriptSig
                if is_witness_input:
                    self.latest_tx.wit.vtxinwit[u[0]] = btc.CTxInWitness(
                    btc.CScriptWitness(witness))
                inserted_sig = True

                # check if maker has sent everything possible
                try:
                    self.utxos[nick].remove(u[1])
                except ValueError:
                    pass
                if len(self.utxos[nick]) == 0:
                    jlog.debug(('nick = {} sent all sigs, removing from '
                               'nonrespondant list').format(nick))
                    try:
                        self.nonrespondants.remove(nick)
                    except ValueError:
                        pass
                break
        if not inserted_sig:
            jlog.debug('signature did not match anything in the tx')
            # TODO what if the signature doesnt match anything
            # nothing really to do except drop it, carry on and wonder why the
            # other guy sent a failed signature

        tx_signed = True
        for ins, witness in zip(self.latest_tx.vin, self.latest_tx.wit.vtxinwit):
            if ins.scriptSig == b"" \
                    and not self._is_our_input(ins) \
                    and witness == btc.CTxInWitness(btc.CScriptWitness([])):
                tx_signed = False
        if not tx_signed:
            return False
        assert not len(self.nonrespondants)
        jlog.info('all makers have sent their signatures')
        self.taker_info_callback("INFO", "Transaction is valid, signing..")
        jlog.debug("schedule item was: " + str(self.schedule[self.schedule_index]))
        return self.self_sign_and_push()

    def make_commitment(self):
        """The Taker default commitment function, which uses PoDLE.
        Alternative commitment types should use a different commit type byte.
        This will allow future upgrades to provide different style commitments
        by subclassing Taker and changing the commit_type_byte; existing makers
        will simply not accept this new type of commitment.
        In case of success, return the (commitment, commitment opening).
        In case of failure returns (None, None, err) where 'err' is a detailed
        error string for the user to read and discern the reason.
        """

        def filter_by_coin_age_amt(utxos, age, amt):
            results = jm_single().bc_interface.query_utxo_set(utxos,
                                                    includeconf=True)
            newresults = []
            too_new = []
            too_small = []
            for i, r in enumerate(results):
                #results return "None" if txo is spent; drop this
                if not r:
                    continue
                valid_age = r['confirms'] >= age
                valid_amt = r['value'] >= amt
                if not valid_age:
                    too_new.append(utxos[i])
                if not valid_amt:
                    too_small.append(utxos[i])
                if valid_age and valid_amt:
                    newresults.append(utxos[i])
            return newresults, too_new, too_small

        def priv_utxo_pairs_from_utxos(utxos, age, amt):
            #returns pairs list of (priv, utxo) for each valid utxo;
            #also returns lists "too_new" and "too_small" for any
            #utxos that did not satisfy the criteria for debugging.
            priv_utxo_pairs = []
            new_utxos, too_new, too_small = filter_by_coin_age_amt(list(utxos.keys()),
                                                                   age, amt)
            new_utxos_dict = {k: v for k, v in utxos.items() if k in new_utxos}
            for k, v in new_utxos_dict.items():
                # filter out any non-standard utxos:
                path = self.wallet_service.script_to_path(v["script"])
                if not self.wallet_service.is_standard_wallet_script(path):
                    continue
                addr = self.wallet_service.script_to_addr(v["script"])
                priv = self.wallet_service.get_key_from_addr(addr)
                if priv:  #can be null from create-unsigned
                    priv_utxo_pairs.append((priv, k))
            return priv_utxo_pairs, too_new, too_small

        commit_type_byte = "P"
        tries = jm_single().config.getint("POLICY", "taker_utxo_retries")
        age = jm_single().config.getint("POLICY", "taker_utxo_age")
        #Minor rounding errors don't matter here
        amt = int(self.cjamount *
                  jm_single().config.getint("POLICY",
                                            "taker_utxo_amtpercent") / 100.0)
        priv_utxo_pairs, to, ts = priv_utxo_pairs_from_utxos(self.input_utxos,
                                                             age, amt)

        #For podle data format see: podle.PoDLE.reveal()
        #In first round try, don't use external commitments
        podle_data = generate_podle(priv_utxo_pairs, tries)
        if not podle_data:
            #Pre-filter the set of external commitments that work for this
            #transaction according to its size and age.
            dummy, extdict = get_podle_commitments()
            if len(extdict) > 0:
                ext_valid, ext_to, ext_ts = filter_by_coin_age_amt(
                    list(extdict.keys()), age, amt)
            else:
                ext_valid = None
            #We defer to a second round to try *all* utxos in spending mixdepth;
            #this is because it's much cleaner to use the utxos involved
            #in the transaction, about to be consumed, rather than use
            #random utxos that will persist after. At this step we also
            #allow use of external utxos in the json file.
            mixdepth_utxos = self.wallet_service.get_utxos_by_mixdepth()[self.mixdepth]
            if len(self.input_utxos) == len(mixdepth_utxos):
                # Already tried the whole mixdepth
                podle_data = generate_podle([], tries, ext_valid)
            else:
                priv_utxo_pairs, to, ts = priv_utxo_pairs_from_utxos(mixdepth_utxos, age, amt)
                podle_data = generate_podle(priv_utxo_pairs, tries, ext_valid)
        if podle_data:
            jlog.debug("Generated PoDLE: " + repr(podle_data))
            return (commit_type_byte + bintohex(podle_data.commitment),
                    podle_data.serialize_revelation(),
                    "Commitment sourced OK")
        else:
            errmsgheader, errmsg = generate_podle_error_string(priv_utxo_pairs,
                        to, ts, self.wallet_service, self.cjamount,
                        jm_single().config.get("POLICY", "taker_utxo_age"),
                        jm_single().config.get("POLICY", "taker_utxo_amtpercent"))

            with open("commitments_debug.txt", "wb") as f:
                errmsgfileheader = (b"THIS IS A TEMPORARY FILE FOR DEBUGGING; "
                        b"IT CAN BE SAFELY DELETED ANY TIME.\n")
                errmsgfileheader += (b"***\n")
                f.write(errmsgfileheader + errmsg.encode('utf-8'))

            return (None, (priv_utxo_pairs, to, ts), errmsgheader + errmsg)

    def coinjoin_address(self):
        if self.my_cj_addr:
            return self.my_cj_addr
        else:
            #Note: donation code removed (possibly temporarily)
            raise NotImplementedError

    def self_sign(self):
        # now sign it ourselves
        our_inputs = {}
        for index, ins in enumerate(self.latest_tx.vin):
            if not self._is_our_input(ins):
                continue
            utxo = (ins.prevout.hash[::-1], ins.prevout.n)
            self.latest_tx.vin[index].scriptSig = btc.CScript(b'')
            script = self.input_utxos[utxo]['script']
            amount = self.input_utxos[utxo]['value']
            our_inputs[index] = (script, amount)
        success, msg = self.wallet_service.sign_tx(self.latest_tx, our_inputs)
        if not success:
            jlog.error("Failed to sign transaction: " + msg)

    def handle_unbroadcast_transaction(self, txid, tx):
        """ The wallet service will handle dangling
        callbacks for transactions but we want to reattempt
        broadcast in case the cause of the problem is a
        counterparty who refused to broadcast it for us.
        """
        if not self.wallet_service.check_callback_called(
            self.txid, self.unconfirm_callback, "unconfirmed",
            "transaction with txid: " + str(self.txid) + " not broadcast."):
            # we now know the transaction was not pushed, so we reinstigate
            # the cancelledcallback with the same logic as explained
            # in Taker.push():
            self.wallet_service.register_callbacks([self.unconfirm_callback],
                                                   txid, "unconfirmed")
            if jm_single().config.get('POLICY', 'tx_broadcast') == "not-self":
                warnmsg = ("You have chosen not to broadcast from your own "
                          "node. The transaction is NOT broadcast.")
                self.taker_info_callback("ABORT", warnmsg + "\nSee log for details.")
                # warning is arguably not correct but it will stand out more:
                jlog.warn(warnmsg)
                jlog.info(btc.human_readable_transaction(tx))
                return
            if not self.push_ourselves():
                jlog.error("Failed to broadcast transaction: ")
                jlog.info(btc.human_readable_transaction(tx))

    def push_ourselves(self):
        return jm_single().bc_interface.pushtx(self.latest_tx.serialize())

    def push(self):
        jlog.debug('\n' + bintohex(self.latest_tx.serialize()))
        self.txid = bintohex(self.latest_tx.GetTxid()[::-1])
        jlog.info('txid = ' + self.txid)
        #If we are sending to a bech32 address, in case of sweep, will
        #need to use that bech32 for address import, which requires
        #converting to script (Core does not allow import of bech32)
        if self.my_cj_addr.lower()[:2] in ['bc', 'tb']:
            notify_addr = btc.CCoinAddress(self.my_cj_addr).to_scriptPubKey()
        else:
            notify_addr = self.my_cj_addr
        #add the callbacks *before* pushing to ensure triggering;
        #this does leave a dangling notify callback if the push fails, but
        #that doesn't cause problems.
        self.wallet_service.register_callbacks([self.unconfirm_callback], self.txid,
                                   "unconfirmed")
        self.wallet_service.register_callbacks([self.confirm_callback], self.txid,
                                   "confirmed")
        task.deferLater(reactor,
                        float(jm_single().config.getint(
                            "TIMEOUT", "unconfirm_timeout_sec")),
                        self.handle_unbroadcast_transaction, self.txid, self.latest_tx)

        tx_broadcast = jm_single().config.get('POLICY', 'tx_broadcast')
        nick_to_use = None
        if tx_broadcast == 'self':
            pushed = self.push_ourselves()
        elif tx_broadcast in ['random-peer', 'not-self']:
            n = len(self.maker_utxo_data)
            if tx_broadcast == 'random-peer':
                i = random.randrange(n + 1)
            else:
                i = random.randrange(n)
            if i == n:
                pushed = self.push_ourselves()
            else:
                nick_to_use = list(self.maker_utxo_data.keys())[i]
                pushed = True
        else:
            jlog.info("Only self, random-peer and not-self broadcast "
                      "methods supported. Reverting to self-broadcast.")
            pushed = self.push_ourselves()
        if not pushed:
            self.on_finished_callback(False, fromtx=True)
        else:
            if nick_to_use:
                return (nick_to_use, self.latest_tx.serialize())
        #if push was not successful, return None

    def self_sign_and_push(self):
        self.self_sign()
        return self.push()

    def tx_match(self, txd):
        # Takers process only in series, so this should not occur:
        assert self.latest_tx is not None
        # check if the transaction matches our created tx:
        if txd.vout != self.latest_tx.vout:
            return False
        return True

    def unconfirm_callback(self, txd, txid):
        if not self.tx_match(txd):
            return False
        jlog.info("Transaction seen on network, waiting for confirmation")
        #To allow client to mark transaction as "done" (e.g. by persisting state)
        self.on_finished_callback(True, fromtx="unconfirmed")
        self.waiting_for_conf = True
        confirm_timeout_sec = float(jm_single().config.get(
            "TIMEOUT", "confirm_timeout_hours")) * 3600
        task.deferLater(reactor, confirm_timeout_sec,
                        self.wallet_service.check_callback_called,
                        txid, self.confirm_callback, "confirmed",
                        "transaction with txid " + str(txid) + " not confirmed.")
        return True

    def confirm_callback(self, txd, txid, confirmations):
        if not self.tx_match(txd):
            return False
        self.waiting_for_conf = False
        if self.aborted:
            #do not trigger on_finished processing (abort whole schedule),
            # but we still return True as we have finished our listening
            # for this tx:
            return True
        jlog.debug("Confirmed callback in taker, confs: " + str(confirmations))
        fromtx=False if self.schedule_index + 1 == len(self.schedule) else True
        waittime = self.schedule[self.schedule_index][4]
        self.on_finished_callback(True, fromtx=fromtx, waittime=waittime,
                                  txdetails=(txd, txid))
        return True

    def _is_our_input(self, tx_input):
        utxo = (tx_input.prevout.hash[::-1], tx_input.prevout.n)
        return utxo in self.input_utxos def estimate_tx_fee(ins, outs, txtype='p2pkh', outtype=None, extra_bytes=0):
    '''Returns an estimate of the number of satoshis required
    for a transaction with the given number of inputs and outputs,
    based on information from the blockchain interface.
    '''
    if jm_single().bc_interface is None:
        raise RuntimeError("Cannot estimate transaction fee " +
            "without blockchain source.")
    fee_per_kb = jm_single().bc_interface.estimate_fee_per_kb(
                jm_single().config.getint("POLICY","tx_fees"))
    if fee_per_kb is None:
        raise RuntimeError("Cannot estimate fee per kB, possibly" +
                           " a failure of connection to the blockchain.")
    absurd_fee = jm_single().config.getint("POLICY", "absurd_fee_per_kb")
    if fee_per_kb > absurd_fee:
        #This error is considered critical; for safety reasons, shut down.
        raise ValueError("Estimated fee " +
            btc.fee_per_kb_to_str(fee_per_kb) +
            " greater than absurd value " +
            btc.fee_per_kb_to_str(absurd_fee) + ", quitting.")
    if txtype in ['p2pkh', 'p2shMofN']:
        tx_estimated_bytes = btc.estimate_tx_size(ins, outs, txtype, outtype) + extra_bytes
        return int((tx_estimated_bytes * fee_per_kb)/Decimal(1000.0))
    elif txtype in ['p2wpkh', 'p2sh-p2wpkh']:
        witness_estimate, non_witness_estimate = btc.estimate_tx_size(
            ins, outs, txtype, outtype)
        non_witness_estimate += extra_bytes
        return int(int((
            non_witness_estimate + 0.25*witness_estimate)*fee_per_kb)/Decimal(1000.0))
    else:
        raise NotImplementedError("Txtype: " + txtype + " not implemented.") def load_program_config(config_path="", bs=None, plugin_services=[]):
    global_singleton.config.readfp(io.StringIO(defaultconfig))
    if not config_path:
        config_path = lookup_appdata_folder(global_singleton.APPNAME)
    # we set the global home directory, but keep the config_path variable
    # for callers of this function:
    global_singleton.datadir = config_path
    jmprint("User data location: " + global_singleton.datadir, "info")
    if not os.path.exists(global_singleton.datadir):
        os.makedirs(global_singleton.datadir)
    # prepare folders for wallets and logs
    if not os.path.exists(os.path.join(global_singleton.datadir, "wallets")):
        os.makedirs(os.path.join(global_singleton.datadir, "wallets"))
    if not os.path.exists(os.path.join(global_singleton.datadir, "logs")):
        os.makedirs(os.path.join(global_singleton.datadir, "logs"))
    if not os.path.exists(os.path.join(global_singleton.datadir, "cmtdata")):
        os.makedirs(os.path.join(global_singleton.datadir, "cmtdata"))
    global_singleton.config_location = os.path.join(
        global_singleton.datadir, global_singleton.config_location)

    remove_unwanted_default_settings(global_singleton.config)
    try:
        loadedFiles = global_singleton.config.read(
            [global_singleton.config_location])
    except UnicodeDecodeError:
        jmprint("Error loading `joinmarket.cfg`, invalid file format.",
            "info")
        sys.exit(EXIT_FAILURE)

    #Hack required for electrum; must be able to enforce a different
    #blockchain interface even in default/new load.
    if bs:
        global_singleton.config.set("BLOCKCHAIN", "blockchain_source", bs)
    # Create default config file if not found
    if len(loadedFiles) != 1:
        with open(global_singleton.config_location, "w") as configfile:
            configfile.write(defaultconfig)
        jmprint("Created a new `joinmarket.cfg`. Please review and adopt the "
              "settings and restart joinmarket.", "info")
        sys.exit(EXIT_FAILURE)

    #These are left as sanity checks but currently impossible
    #since any edits are overlays to the default, these sections/options will
    #always exist.
    # FIXME: This check is a best-effort attempt. Certain incorrect section
    # names can pass and so can non-first invalid sections.
    for s in required_options: #pragma: no cover
        # check for sections
        avail = None
        if not global_singleton.config.has_section(s):
            for avail in global_singleton.config.sections():
                if avail.startswith(s):
                    break
            else:
                raise Exception(
                    "Config file does not contain the required section: " + s)
        # then check for specific options
        k = avail or s
        for o in required_options[s]:
            if not global_singleton.config.has_option(k, o):
                raise Exception("Config file does not contain the required "
                                "option '{}' in section '{}'.".format(o, k))

    loglevel = global_singleton.config.get("LOGGING", "console_log_level")
    try:
        set_logging_level(loglevel)
    except:
        jmprint("Failed to set logging level, must be DEBUG, INFO, WARNING, ERROR",
                "error")

    # Logs to the console are color-coded if user chooses (file is unaffected)
    if global_singleton.config.get("LOGGING", "color") == "true":
        set_logging_color(True)
    else:
        set_logging_color(False)

    try:
        global_singleton.maker_timeout_sec = global_singleton.config.getint(
            'TIMEOUT', 'maker_timeout_sec')
    except NoOptionError: #pragma: no cover
        log.debug('TIMEOUT/maker_timeout_sec not found in .cfg file, '
                  'using default value')

    # configure the interface to the blockchain on startup
    global_singleton.bc_interface = get_blockchain_interface_instance(
        global_singleton.config)

    # set the location of the commitments file; for non-mainnet a different
    # file is used to avoid conflict
    try:
        global_singleton.commit_file_location = global_singleton.config.get(
            "POLICY", "commit_file_location")
    except NoOptionError: #pragma: no cover
        if get_network() == "mainnet":
            log.debug("No commitment file location in config, using default "
                  "location cmtdata/commitments.json")
    if get_network() != "mainnet":
        # no need to be flexible for tests; note this is used
        # for regtest, signet and testnet3
        global_singleton.commit_file_location = "cmtdata/" + get_network() + \
            "_commitments.json"
    set_commitment_file(os.path.join(config_path,
                                         global_singleton.commit_file_location))

    for p in plugin_services:
        # for now, at this config level, the only significance
        # of a "plugin" is that it keeps its own separate log.
        # We require that a section exists in the config file,
        # and that it has enabled=true:
        assert isinstance(p, JMPluginService)
        if not (global_singleton.config.has_section(p.name) and \
                global_singleton.config.has_option(p.name, "enabled") and \
                global_singleton.config.get(p.name, "enabled") == "true"):
            break
        if p.requires_logging:
            # make sure the environment can accept a logfile by
            # creating the directory in the correct place,
            # and setting that in the plugin object; the plugin
            # itself will switch on its own logging when ready,
            # attaching a filehandler to the global log.
            plogsdir = os.path.join(os.path.dirname(
                global_singleton.config_location), "logs", p.name)
            if not os.path.exists(plogsdir):
                os.makedirs(plogsdir)
            p.set_log_dir(plogsdir) def jm_single():
    return global_singleton def validate_address(addr):
    try:
        # automatically respects the network
        # as set in btc.select_chain_params(...)
        dummyaddr = btc.CCoinAddress(addr)
    except Exception as e:
        return False, repr(e)
    # additional check necessary because python-bitcointx
    # does not check hash length on p2sh construction.
    try:
        dummyaddr.to_scriptPubKey()
    except Exception as e:
        return False, repr(e)
    return True, "address validated" class JMClientProtocolFactory(protocol.ClientFactory):
    protocol = JMTakerClientProtocol

    def __init__(self, client, proto_type="TAKER"):
        self.client = client
        self.proto_client = None
        self.proto_type = proto_type
        if self.proto_type == "MAKER":
            self.protocol = JMMakerClientProtocol

    def setClient(self, client):
        self.proto_client = client
    def getClient(self):
        return self.proto_client

    def buildProtocol(self, addr):
        return self.protocol(self, self.client) def start_reactor(host, port, factory=None, snickerfactory=None,
                  bip78=False, jm_coinjoin=True, ish=True,
                  daemon=False, rs=True, gui=False): #pragma: no cover
    #(Cannot start the reactor in tests)
    #Not used in prod (twisted logging):
    #startLogging(stdout)
    global daemon_serving_host
    global daemon_serving_port

    # in case we are starting connections but not the
    # reactor, we can return a handle to the connections so
    # that they can be cleaned up properly.
    # TODO: currently *only* used in tests, with only one
    # server protocol listening.
    serverconn = None
    clientconn = None

    usessl = jm_single().config.get("DAEMON", "use_ssl") != 'false'
    jmcport, snickerport, bip78port = [port]*3
    if daemon:
        try:
            from jmdaemon import JMDaemonServerProtocolFactory, start_daemon, \
                 SNICKERDaemonServerProtocolFactory, BIP78ServerProtocolFactory
        except ImportError:
            jlog.error("Cannot start daemon without jmdaemon package; "
                       "either install it, and restart, or, if you want "
                       "to run the daemon separately, edit the DAEMON "
                       "section of the config. Quitting.")
            return
        if jm_coinjoin:
            dfactory = JMDaemonServerProtocolFactory()
        if snickerfactory:
            sdfactory = SNICKERDaemonServerProtocolFactory()
        if bip78:
            bip78factory = BIP78ServerProtocolFactory()
        # ints are immutable in python, to pass by ref we use
        # an array object:
        port_a = [port]
        def start_daemon_on_port(p, f, name, port_offset):
            orgp = p[0]
            while True:
                try:
                    serverconn = start_daemon(host, p[0] - port_offset, f, usessl,
                        './ssl/key.pem', './ssl/cert.pem')
                    jlog.info("{} daemon listening on port {}".format(
                        name, str(p[0] - port_offset)))
                    break
                except Exception:
                    jlog.warn("Cannot listen on port " + str(
                        p[0] - port_offset) + ", trying next port")
                    if p[0] >= (orgp + 100):
                        jlog.error("Tried 100 ports but cannot "
                                   "listen on any of them. Quitting.")
                        sys.exit(EXIT_FAILURE)
                    p[0] += 1
            return (p[0], serverconn)

        if jm_coinjoin:
            # TODO either re-apply this port incrementing logic
            # to other protocols, or re-work how the ports work entirely.
            jmcport, serverconn = start_daemon_on_port(port_a, dfactory,
                                                       "Joinmarket", 0)
            daemon_serving_port = jmcport
            daemon_serving_host = host
        # (See above) For now these other two are just on ports that are 1K offsets.
        if snickerfactory:
            snickerport, serverconn = start_daemon_on_port(port_a, sdfactory,
                                                    "SNICKER", 1000)
            snickerport = snickerport - 1000
        if bip78:
            start_daemon_on_port(port_a, bip78factory, "BIP78", 2000)

        # if the port had to be incremented due to conflict above, we should update
        # it in the config var so e.g. bip78 connections choose the port we actually
        # used.
        # This is specific to the daemon-in-same-process case; for the external daemon
        # the user must just set the right value.
        jm_single().config.set("DAEMON", "daemon_port", str(port_a[0]))

    # Note the reactor.connect*** entries do not include BIP78 which
    # starts in jmclient.payjoin:
    if usessl:
        if factory:
            reactor.connectSSL(host, jmcport, factory, ClientContextFactory())
        if snickerfactory:
            reactor.connectSSL(host, snickerport, snickerfactory,
                               ClientContextFactory())
    else:
        if factory:
            clientconn = reactor.connectTCP(host, jmcport, factory)
        if snickerfactory:
            reactor.connectTCP(host, snickerport, snickerfactory)
    if rs:
        if not gui:
            reactor.run(installSignalHandlers=ish)
        if isinstance(jm_single().bc_interface, RegtestBitcoinCoreInterface):
            jm_single().bc_interface.shutdown_signal = True
    return (serverconn, clientconn) def get_schedule(filename):
    with open(filename, "rb") as f:
        schedule = []
        schedule_lines = f.readlines()
        for sl in schedule_lines:
            sl = sl.decode('utf-8')
            if sl.startswith("#"):
                continue
            try:
                (mixdepth, amount, makercount, destaddr, waittime,
                    rounding, completed) = sl.split(',')
            except ValueError as e:
                return (False, "Failed to parse schedule line: " + sl)
            try:
                mixdepth = int(mixdepth)
                #TODO this isn't the right way, but floats must be allowed
                #for any persisted tumbler-style schedule
                if "." in amount:
                    amount = float(amount)
                else:
                    amount = int(amount)
                makercount = int(makercount)
                destaddr = destaddr.strip()
                waittime = float(waittime)
                rounding = int(rounding)
                completed = completed.strip()
                if not len(completed) == 64:
                    completed = int(completed)
            except ValueError as e:
                return (False, "Failed to parse schedule line: " + sl)
            if destaddr not in ["INTERNAL", "addrask"]:
                success, errmsg = validate_address(destaddr)
                if not success:
                    return (False, "Invalid address: " + destaddr + "," + errmsg)
            schedule.append([mixdepth, amount, makercount, destaddr,
                             waittime, rounding, completed])
    return (True, schedule) def get_tumble_schedule(options, destaddrs, mixdepth_balance_dict):
    """for the general intent and design of the tumbler algo, see the docs in
    joinmarket-org/joinmarket.
    Alterations:
    Donation removed for now.
    Default final setting for "amount_fraction" is zero, for each mixdepth.
    This is because we now use a general "schedule" syntax for both tumbler and
    any other taker algo; it interprets floats as fractions and integers as satoshis,
    and zero as sweep (as before).
    This is a modified version of tumbler.py/generate_tumbler_tx()
    """
    #if options['mixdepthsrc'] != 0:
    #    raise NotImplementedError("Non-zero mixdepth source not supported; "
    #                              "restart the tumbler with --restart instead")

    def lower_bounded_int(thelist, lowerbound):
        return [int(l) if int(l) >= lowerbound else lowerbound for l in thelist]

    txcounts = rand_norm_array(options['txcountparams'][0],
                               options['txcountparams'][1], options['mixdepthcount'])
    txcounts = lower_bounded_int(txcounts, options['mintxcount'])
    tx_list = []
    ### stage 1 coinjoins, which sweep the entire mixdepth without creating change
    lowest_initial_filled_mixdepth = sys.maxsize
    sweep_mixdepths = []
    for mixdepth, balance in mixdepth_balance_dict.items():
        if balance > 0:
            lowest_initial_filled_mixdepth = min(mixdepth,
                lowest_initial_filled_mixdepth)
            sweep_mixdepths.append(mixdepth)
    waits = rand_exp_array(options['timelambda']*options[
        'stage1_timelambda_increase'], len(sweep_mixdepths))
    makercounts = rand_norm_array(options['makercountrange'][0],
        options['makercountrange'][1], len(sweep_mixdepths))
    makercounts = lower_bounded_int(makercounts, options['minmakercount'])
    sweep_mixdepths = sorted(sweep_mixdepths)[::-1]
    for mixdepth, wait, makercount in zip(sweep_mixdepths, waits, makercounts):
        tx = {'amount_fraction': 0,
              'wait': round(wait, 2),
              'srcmixdepth': mixdepth,
              'makercount': makercount,
              'destination': 'INTERNAL',
              'rounding': NO_ROUNDING
        }
        tx_list.append(tx)
    ### stage 2 coinjoins, which create a number of random-amount coinjoins from each mixdepth
    for m, txcount in enumerate(txcounts):
        if options['mixdepthcount'] - options['addrcount'] <= m and m < \
                options['mixdepthcount'] - 1:
            #these mixdepths send to a destination address, so their
            # amount_fraction cant be 1.0, some coins must be left over
            if txcount == 1:
                txcount = 2
        amount_fractions = get_amount_fractions(txcount)
        # transaction times are uncorrelated
        # time between events in a poisson process followed exp
        waits = rand_exp_array(options['timelambda'], txcount)
        # number of makers to use follows a normal distribution
        makercounts = rand_norm_array(options['makercountrange'][0],
                                      options['makercountrange'][1], txcount)
        makercounts = lower_bounded_int(makercounts, options['minmakercount'])
        do_rounds = [random.random() < options['rounding_chance'] for _ in range(txcount)]
        for amount_fraction, wait, makercount, do_round in zip(amount_fractions, waits,
                                                     makercounts, do_rounds):
            rounding = NO_ROUNDING
            if do_round:
                weight_sum = 1.0*sum(options['rounding_sigfig_weights'])
                weight_prob = [a/weight_sum for a in options['rounding_sigfig_weights']]
                rounding = rand_weighted_choice(len(weight_prob), weight_prob) + 1
            tx = {'amount_fraction': amount_fraction,
                  'wait': round(wait, 2),
                  'srcmixdepth': lowest_initial_filled_mixdepth + m + options['mixdepthsrc'] + 1,
                  'makercount': makercount,
                  'destination': 'INTERNAL',
                  'rounding': rounding
            }
            tx_list.append(tx)
        #reset the final amt_frac to zero, as it's the last one for this mixdepth:
        tx_list[-1]['amount_fraction'] = 0
        tx_list[-1]['rounding'] = NO_ROUNDING

    addrask = options['addrcount'] - len(destaddrs)
    external_dest_addrs = ['addrask'] * addrask + destaddrs[::-1]
    for mix_offset in range(options['addrcount']):
        srcmix = (lowest_initial_filled_mixdepth + options['mixdepthsrc']
            + options['mixdepthcount'] - mix_offset)
        for tx in reversed(tx_list):
            if tx['srcmixdepth'] == srcmix:
                tx['destination'] = external_dest_addrs[mix_offset]
                break
        if mix_offset == 0:
            # setting last mixdepth to send all to dest
            tx_list_remove = []
            for tx in tx_list:
                if tx['srcmixdepth'] == srcmix:
                    if tx['destination'] == 'INTERNAL':
                        tx_list_remove.append(tx)
                    else:
                        tx['amount_fraction'] = 0
            [tx_list.remove(t) for t in tx_list_remove]
    schedule = []
    for t in tx_list:
        schedule.append([t['srcmixdepth'], t['amount_fraction'],
                  t['makercount'], t['destination'], t['wait'],
                  t['rounding'], 0])
    return schedule def schedule_to_text(schedule):
    return "\n".join([",".join([str(y) for y in x]) for x in schedule]).encode('utf-8') def tumbler_taker_finished_update(taker, schedulefile, tumble_log, options,
                   res, fromtx=False, waittime=0.0, txdetails=None):
    """on_finished_callback processing for tumbler.
    Note that this is *not* the full callback, but provides common
    processing across command line and other GUI versions.
    """

    if fromtx == "unconfirmed":
        #unconfirmed event means transaction has been propagated,
        #we update state to prevent accidentally re-creating it in
        #any crash/restart condition
        unconf_update(taker, schedulefile, tumble_log, True)
        return

    if fromtx:
        if res:
            #this has no effect except in the rare case that confirmation
            #is immediate; also it does not repeat the log entry.
            unconf_update(taker, schedulefile, tumble_log, False)
            #note that Qt does not yet support 'addrask', so this is only
            #for command line script TODO
            if taker.schedule[taker.schedule_index+1][3] == 'addrask':
                jm_single().debug_silence[0] = True
                jmprint('\n'.join(['=' * 60] * 3))
                jmprint('Tumbler requires more addresses to stop amount correlation')
                jmprint('Obtain a new destination address from your bitcoin recipient')
                jmprint(' for example click the button that gives a new deposit address')
                jmprint('\n'.join(['=' * 60] * 1))
                while True:
                    destaddr = input('insert new address: ')
                    addr_valid, errormsg = validate_address(destaddr)
                    if addr_valid:
                        break
                    jmprint(
                    'Address ' + destaddr + ' invalid. ' + errormsg + ' try again',
                    "warning")
                jm_single().debug_silence[0] = False
                taker.schedule[taker.schedule_index+1][3] = destaddr
                taker.tdestaddrs.append(destaddr)

            waiting_message = "Waiting for: " + str(waittime) + " minutes."
            tumble_log.info(waiting_message)
            log.info(waiting_message)
        else:
            # a transaction failed, either because insufficient makers
            # (acording to minimum_makers) responded in Phase 1, or not all
            # makers responded in Phase 2, or the tx was a mempool conflict.
            # If the tx was a mempool conflict, we should restart with random
            # maker choice as usual. If someone didn't respond, we'll try to
            # repeat without the troublemakers.
            log.info("Schedule entry: " + str(
                taker.schedule[taker.schedule_index]) + \
                     " failed after timeout, trying again")
            taker.add_ignored_makers(taker.nonrespondants)
            #Is the failure in Phase 2?
            if not taker.latest_tx is None:
                if len(taker.nonrespondants) == 0:
                    # transaction was created validly but conflicted in the
                    # mempool; just try again without honest settings;
                    # i.e. fallback to same as Phase 1 failure.
                    log.info("Invalid transaction; possible mempool conflict.")
                else:
                    #Now we have to set the specific group we want to use, and hopefully
                    #they will respond again as they showed honesty last time.
                    #Note that we must wipe the list first; other honest makers needn't
                    #have the right settings (e.g. max cjamount), so can't be carried
                    #over from earlier transactions.
                    taker.honest_makers = []
                    taker.add_honest_makers(list(set(
                        taker.maker_utxo_data.keys()).symmetric_difference(
                            set(taker.nonrespondants))))
                    #If insufficient makers were honest, we can only tweak the schedule.
                    #If enough were, we prefer to restart with them only:
                    log.info("Inside a Phase 2 failure; number of honest "
                             "respondants was: " + str(len(taker.honest_makers)))
                    log.info("They were: " + str(taker.honest_makers))
                    if len(taker.honest_makers) >= jm_single().config.getint(
                        "POLICY", "minimum_makers"):
                        tumble_log.info("Transaction attempt failed, attempting to "
                                        "restart with subset.")
                        tumble_log.info("The paramaters of the failed attempt: ")
                        tumble_log.info(str(taker.schedule[taker.schedule_index]))
                        #we must reset the number of counterparties, as well as fix who they
                        #are; this is because the number is used to e.g. calculate fees.
                        #cleanest way is to reset the number in the schedule before restart.
                        taker.schedule[taker.schedule_index][2] = len(taker.honest_makers)
                        retry_str = "Retrying with: " + str(taker.schedule[
                            taker.schedule_index][2]) + " counterparties."
                        tumble_log.info(retry_str)
                        log.info(retry_str)
                        taker.set_honest_only(True)
                        taker.schedule_index -= 1
                        return

            #There were not enough honest counterparties.
            #Tumbler is aggressive in trying to complete; we tweak the schedule
            #from this point in the mixdepth, then try again.
            tumble_log.info("Transaction attempt failed, tweaking schedule"
                            " and trying again.")
            tumble_log.info("The paramaters of the failed attempt: ")
            tumble_log.info(str(taker.schedule[taker.schedule_index]))
            taker.schedule_index -= 1
            taker.schedule = tweak_tumble_schedule(options, taker.schedule,
                                                   taker.schedule_index,
                                                   taker.tdestaddrs)
            tumble_log.info("We tweaked the schedule, the new schedule is:")
            tumble_log.info(pprint.pformat(taker.schedule))
    else:
        if not res:
            failure_msg = "Did not complete successfully, shutting down"
            tumble_log.info(failure_msg)
            log.info(failure_msg)
        else:
            log.info("All transactions completed correctly")
            tumble_log.info("Completed successfully the last entry:")
            #Whether sweep or not, the amt is not in satoshis; use taker data
            hramt = taker.cjamount
            tumble_log.info(human_readable_schedule_entry(
                taker.schedule[taker.schedule_index], hramt))
            #copy of above, TODO refactor out
            taker.schedule[taker.schedule_index][5] = 1
            with open(schedulefile, "wb") as f:
                f.write(schedule_to_text(taker.schedule)) def restart_waiter(txid):
    """Given a txid, wait for confirmation by polling the blockchain
    interface instance. Note that this is currently blocking, so only used
    by the CLI version; the Qt/GUI uses the underlying restart_wait() fn.
    """
    ctr = 0
    log.info("Waiting for confirmation of last transaction: " + str(txid))
    while True:
        time.sleep(10)
        ctr += 1
        if not (ctr % 12):
            log.debug("Still waiting for confirmation of last transaction ...")
        if restart_wait(txid):
            break
    log.info("The previous transaction is now in a block; continuing.") def get_tumble_log(logsdir):
    tumble_log = logging.getLogger('tumbler')
    tumble_log.setLevel(logging.DEBUG)
    logFormatter = logging.Formatter(
        ('%(asctime)s %(message)s'))
    fileHandler = logging.FileHandler(os.path.join(logsdir, 'TUMBLE.log'))
    fileHandler.setFormatter(logFormatter)
    tumble_log.addHandler(fileHandler)
    return tumble_log def tumbler_filter_orders_callback(orders_fees, cjamount, taker):
    """Since the tumbler does not use interactive fee checking,
    we use the -x values from the command line instead.
    """
    orders, total_cj_fee = orders_fees
    abs_cj_fee = 1.0 * total_cj_fee / taker.n_counterparties
    rel_cj_fee = abs_cj_fee / cjamount
    log.info('rel/abs average fee = ' + str(rel_cj_fee) + ' / ' + str(
            abs_cj_fee))

    if rel_cj_fee > taker.max_cj_fee[0] and abs_cj_fee > taker.max_cj_fee[1]:
        log.info("Rejected fees as too high according to options, will "
                 "retry.")
        return "retry"
    return True def get_tumbler_parser():
    parser = OptionParser(
            usage='usage: %prog [options] [wallet file] [destaddr(s)...]',
            description=
            'Sends bitcoins to many different addresses using coinjoin in'
            ' an attempt to break the link between them. Sending to multiple '
            ' addresses is highly recommended for privacy. This tumbler can'
            ' be configured to ask for more address mid-run, giving the user'
            ' a chance to click `Generate New Deposit Address` on whatever service'
            ' they are using.')
    parser.add_option(
            '-m',
            '--mixdepthsource',
            type='int',
            dest='mixdepthsrc',
            help=
            'Mixing depth to start tumble process from. default=0.',
            default=0)
    parser.add_option('--restart',
        action='store_true',
        dest='restart',
        default=False,
        help=('Restarts the schedule currently found in the schedule file in the '
              'logs directory, with name TUMBLE.schedule or what is set in the '
              'schedulefile option.'))
    parser.add_option('--schedulefile',
            type='string',
            dest='schedulefile',
            default='TUMBLE.schedule',
            help=('Name of schedule file for tumbler, useful for restart, default '
                  'TUMBLE.schedule'))
    parser.add_option(
            '-a',
            '--addrcount',
            type='int',
            dest='addrcount',
            default=3,
            help=
            'How many destination addresses in total should be used. If not enough are given'
            ' as command line arguments, the script will ask for more. This parameter is required'
            ' to stop amount correlation. default=3')
    parser.add_option(
            '-N',
            '--makercountrange',
            type='float',
            nargs=2,
            action='store',
            dest='makercountrange',
            help=
            'Input the mean and spread of number of makers to use. e.g. 9 1 will be a normal distribution '
            'with mean 9 and standard deviation 1 inclusive, default=9 1 (floats are also OK)',
            default=(9, 1))
    parser.add_option(
            '--minmakercount',
            type='int',
            dest='minmakercount',
            default=4,
            help=
            'The minimum maker count in a transaction, random values below this are clamped at this number. default=4')
    parser.add_option(
            '-M',
            '--mixdepthcount',
            type='int',
            dest='mixdepthcount',
            help='How many mixing depths to mix through',
            default=4)
    parser.add_option(
            '-c',
            '--txcountparams',
            type='float',
            nargs=2,
            dest='txcountparams',
            default=(2, 1),
            help=
            'The number of transactions to take coins from one mixing depth to the next, it is'
            ' randomly chosen following a normal distribution. Should be similar to --addrask. '
            'This option controls the parameters of the normal distribution curve. (mean, standard deviation). default=2 1')
    parser.add_option(
            '--mintxcount',
            type='int',
            dest='mintxcount',
            default=2,
            help='The minimum transaction count per mixing level, default=2')
    parser.add_option(
            '--donateamount',
            type='float',
            dest='donateamount',
            default=0,
            help=
            'percent of funds to donate to joinmarket development, or zero to opt out (default=0%)')
    parser.add_option(
            '-l',
            '--timelambda',
            type='float',
            dest='timelambda',
            default=60,
            help=
            'Average the number of minutes to wait between transactions. Randomly chosen '
            ' following an exponential distribution, which describes the time between uncorrelated'
            ' events. default=60')
    parser.add_option(
            '--stage1-timelambda-increase',
            type='float',
            dest='stage1_timelambda_increase',
            default=3,
            help=
            'Stage 1 sweep coinjoins have a longer wait time. This parameter'
            ' controls by what factor longer is this average wait time compared to stage2 coinjoins'
            ' which are controlled by `--timelambda`, default=3')
    parser.add_option(
            '-w',
            '--wait-time',
            action='store',
            type='float',
            dest='waittime',
            help='wait time in seconds to allow orders to arrive, default=20',
            default=20)
    parser.add_option(
            '-s',
            '--mincjamount',
            type='int',
            dest='mincjamount',
            default=100000,
            help='minimum coinjoin amount in transaction in satoshi, default 100k')
    parser.add_option(
            '-q',
            '--liquiditywait',
            type='int',
            dest='liquiditywait',
            default=60,
            help=
            'amount of seconds to wait after failing to choose suitable orders before trying again, default 60')
    parser.add_option(
            '--maxbroadcasts',
            type='int',
            dest='maxbroadcasts',
            default=4,
            help=
            'maximum amount of times to broadcast a transaction before giving up and re-creating it, default 4')
    parser.add_option(
            '--maxcreatetx',
            type='int',
            dest='maxcreatetx',
            default=9,
            help=
            'maximum amount of times to re-create a transaction before giving up, default 9')
    # note that this is used slightly differently in tumbler from sendpayment,
    # hence duplicated:
    parser.add_option('-A',
            '--amtmixdepths',
            action='store',
            type='int',
            dest='amtmixdepths',
            help='number of mixdepths ever used in wallet, '
                 'only to be used if mixdepths higher than '
                 'mixdepthsrc + number of mixdepths to tumble '
                 'have been used.',
            default=-1)
    parser.add_option(
            '--rounding-chance',
            action='store',
            type='float',
            dest='rounding_chance',
            help='probability of non-sweep coinjoin amount being rounded, default=0.25 (25%)',
            default=0.25)
    parser.add_option(
            '--rounding-sigfig-weights',
            type='float',
            nargs=5,
            dest='rounding_sigfig_weights',
            default=(55, 15, 25, 65, 40),
            help=
            "If rounding happens (determined by --rounding-chance) then the weights of how many"
            " significant figures to round to. The five values refer to the probability of"
            " rounding to one, two, three, four and five significant figures respectively."
            " default=(55, 15, 25, 65, 40)")
    add_common_options(parser)
    return parser def get_max_cj_fee_values(config, parser_options,
                          user_callback=prompt_user_for_cj_fee):
    """ Given a config object, retrieve the chosen maximum absolute
    and relative coinjoin fees chosen by the user, or prompt
    the user via the user_callback function, if not present in
    the config.

    user_callback:
    Arguments: relative value(default None), absolute value (default None)
    Returns: relative value (float), absolute value (int, satoshis)
    """

    CONFIG_SECTION = 'POLICY'
    CONFIG_OPTION = 'max_cj_fee_'
    # rel, abs
    fee_values = [None, None]
    fee_types = [float, int]

    for i, option in enumerate(('rel', 'abs')):
        if parser_options is not None:
            value = getattr(parser_options, CONFIG_OPTION + option, None)
            if value is not None:
                fee_values[i] = fee_types[i](value)
                continue
        try:
            fee_values[i] = config.get(CONFIG_SECTION, CONFIG_OPTION + option)
        except NoOptionError:
            pass

    if any(x is None for x in fee_values):
        fee_values = user_callback(*fee_values)

    return tuple(map(lambda j: fee_types[j](fee_values[j]),
                     range(len(fee_values)))) def check_regtest(blockchain_start=True):
    """ Applies any regtest-specific configuration
    """
    if not isinstance(jm_single().bc_interface,
                      RegtestBitcoinCoreInterface):
        return
    if blockchain_start:
        #to allow testing of confirm/unconfirm callback for multiple txs
        jm_single().bc_interface.tick_forward_chain_interval = 10
        jm_single().bc_interface.simulating = True
    # handles the custom regtest hrp for bech32
    cryptoengine.BTC_P2WPKH.VBYTE = 100 def open_test_wallet_maybe(path, seed, max_mixdepth,
                           test_wallet_cls=SegwitWallet, wallet_password_stdin=False, **kwargs):
    """
    Create a volatile test wallet if path is a hex-encoded string of length 64,
    otherwise run open_wallet().

    params:
        path: path to wallet file, ignored for test wallets
        seed: hex-encoded test seed
        max_mixdepth: maximum mixdepth to use
        kwargs: see open_wallet()

    returns:
        wallet object
    """
    # If the native flag is set in the config, it overrides the argument
    # test_wallet_cls
    if jm_single().config.get("POLICY", "native") == "false":
        test_wallet_cls = SegwitLegacyWallet
    if len(seed) == test_wallet_cls.ENTROPY_BYTES * 2:
        try:
            seed = binascii.unhexlify(seed)
        except binascii.Error:
            pass
        else:
            if max_mixdepth is None:
                max_mixdepth = DEFAULT_MIXDEPTH

            storage = VolatileStorage()
            test_wallet_cls.initialize(
                storage, get_network(), max_mixdepth=max_mixdepth,
                entropy=seed)
            #wallet instantiation insists on no unexpected kwargs,
            #but Qt caller opens both test and mainnet with same args,
            #hence these checks/deletes of unwanted args for tests.
            if 'ask_for_password' in kwargs:
                del kwargs['ask_for_password']
            if 'password' in kwargs:
                del kwargs['password']
            if 'read_only' in kwargs:
                del kwargs['read_only']
            return test_wallet_cls(storage, **kwargs)

    if wallet_password_stdin is True:
        password = read_password_stdin()
        return open_wallet(path, ask_for_password=False, password=password, mixdepth=max_mixdepth, **kwargs)

    return open_wallet(path, mixdepth=max_mixdepth, **kwargs) def get_wallet_path(file_name, wallet_dir=None):
    if not wallet_dir:
        wallet_dir = os.path.join(jm_single().datadir, 'wallets')
    return os.path.join(wallet_dir, file_name) class WalletService(Service):
    EXTERNAL_WALLET_LABEL = "joinmarket-notify"

    def __init__(self, wallet):
        # The two principal member variables
        # are the blockchaininterface instance,
        # which is currently global in JM but
        # could be more flexible in future, and
        # the JM wallet object.
        self.bci = jm_single().bc_interface

        # main loop used to check for transactions, instantiated
        # after wallet is synced:
        self.monitor_loop = None
        self.wallet = wallet
        self.synced = False

        # used to flag RPC failure at construction of object:
        self.rpc_error = False

        # keep track of the quasi-real-time blockheight
        # (updated in main monitor loop)
        self.current_blockheight = None

        if self.bci is not None:
            if not self.update_blockheight():
                # this accounts for the unusual case
                # where the application started up with
                # a functioning blockchain interface, but
                # that bci is now failing when we are starting
                # the wallet service.
                jlog.error("Failure of RPC connection to Bitcoin Core in "
                          "wallet service startup. Application cannot "
                          "continue, shutting down.")
                self.rpc_error = ("Failure of RPC connection to Bitcoin "
                                  "Core in wallet service startup.")
                # no need to call stopService as it has not yet been started.
                stop_reactor()
        else:
            jlog.warning("No blockchain source available, " +
                "wallet tools will not show correct balances.")

        # Dicts of registered callbacks, by type
        # and then by txinfo, for events
        # on transactions.
        self.callbacks = {}
        self.callbacks["all"] = []
        self.callbacks["unconfirmed"] = {}
        self.callbacks["confirmed"] = {}

        self.restart_callback = None

        # transactions we are actively monitoring,
        # i.e. they are not new but we want to track:
        self.active_txids = []
        # to ensure transactions are only processed once:
        self.processed_txids = []

        self.set_autofreeze_warning_cb()

    def update_blockheight(self):
        """ Can be called manually (on startup, or for tests)
        but will be called as part of main monitoring
        loop to ensure new transactions are added at
        the right height.
        Any failure of the RPC call must result in this returning
        False, otherwise return True (means self.current_blockheight
        has been correctly updated).
        """

        def critical_error():
            jlog.error("Critical error updating blockheight.")
            # this cleanup (a) closes the wallet, removing the lock
            # and (b) signals to clients that the service is no longer
            # in a running state, both of which can be useful
            # post reactor shutdown.
            self.stopService()
            stop_reactor()
            return False

        if self.current_blockheight:
            old_blockheight = self.current_blockheight
        else:
            old_blockheight = -1
        try:
            self.current_blockheight = self.bci.get_current_block_height()
        except Exception as e:
            # This should never happen now, as we are catching every
            # possible Exception in jsonrpc or bci.rpc:
            return critical_error()
        if not self.current_blockheight:
            return critical_error()

        # We have received a new blockheight from Core, sanity check it:
        assert isinstance(self.current_blockheight, Integral)
        assert self.current_blockheight >= 0
        if self.current_blockheight < old_blockheight:
            jlog.warn("Bitcoin Core is reporting a lower blockheight, "
                      "possibly a reorg.")
        return True

    def startService(self):
        """ Encapsulates start up actions.
        Here wallet sync.
        """
        super().startService()
        self.request_sync_wallet()

    def stopService(self):
        """ Encapsulates shut down actions.
        Note that after the service is stopped, it
        should *not* be restarted, instead a new
        WalletService instance should be created.
        """
        if self.monitor_loop and self.monitor_loop.running:
            self.monitor_loop.stop()
        self.wallet.close()
        super().stopService()

    def isRunning(self):
        if self.running == 1:
            return True
        return False

    def add_restart_callback(self, callback):
        """ Sets the function that will be
        called in the event that the wallet
        sync completes with a restart warning.
        The only argument is a message string,
        which the calling function displays to
        the user before quitting gracefully.
        """
        self.restart_callback = callback

    def request_sync_wallet(self):
        """ Ensures wallet sync is complete
        before the main event loop starts.
        """
        if self.bci is not None:
            d = task.deferLater(reactor, 0.0, self.sync_wallet)
            d.addCallback(self.start_wallet_monitoring)

    def register_callbacks(self, callbacks, txinfo, cb_type="all"):
        """ Register callbacks that will be called by the
        transaction monitor loop, on transactions stored under
        our wallet label (see WalletService.get_wallet_name()).
        Callback arguments are currently (txd, txid) and return
        is boolean, except "confirmed" callbacks which have
        arguments (txd, txid, confirmations).
        Note that callbacks MUST correctly return True if they
        recognized the transaction and processed it, and False
        if not. The True return value will be used to remove
        the callback from the list.
        Arguments:
        `callbacks` - a list of functions with signature as above
        and return type boolean.
        `txinfo` - either a txid expected for the transaction, if
        known, or a tuple of the ordered output set, of the form
        ((CScript, int), ..). This is be constructed from the
        CMutableTransaction vout list.
        See WalletService.transaction_monitor().
        `cb_type` - must be one of "all", "unconfirmed", "confirmed";
        the first type will be called back once for every new
        transaction, the second only once when the number of
        confirmations is 0, and the third only once when the number
        of confirmations is > 0.
        """
        if cb_type == "all":
            # note that in this case, txid is ignored.
            self.callbacks["all"].extend(callbacks)
        elif cb_type in ["unconfirmed", "confirmed"]:
            if txinfo not in self.callbacks[cb_type]:
                self.callbacks[cb_type][txinfo] = []
            self.callbacks[cb_type][txinfo].extend(callbacks)
        else:
            assert False, "Invalid argument: " + cb_type


    def start_wallet_monitoring(self, syncresult):
        """ Once the initialization of the service
        (currently, means: wallet sync) is complete,
        we start the main monitoring jobs of the
        wallet service (currently, means: monitoring
        all new transactions on the blockchain that
        are recognised as belonging to the Bitcoin
        Core wallet with the JM wallet's label).
        """
        if not syncresult:
            jlog.error("Failed to sync the bitcoin wallet. Shutting down.")
            self.stopService()
            if reactor.running:
                reactor.stop()
            return
        jlog.info("Starting transaction monitor in walletservice")
        self.monitor_loop = task.LoopingCall(
            self.transaction_monitor)
        self.monitor_loop.start(5.0)

    def import_non_wallet_address(self, address):
        """ Used for keeping track of transactions which
        have no in-wallet destinations. External wallet
        label is used to avoid breaking fast sync (which
        assumes label => wallet)
        """
        if not self.bci.is_address_imported(address):
            self.bci.import_addresses([address], self.EXTERNAL_WALLET_LABEL,
                                  restart_cb=self.restart_callback)

    def default_autofreeze_warning_cb(self, utxo):
        success, utxostr = utxo_to_utxostr(utxo)
        assert success, "Autofreeze warning cb called with invalid utxo."
        jlog.warning("WARNING: new utxo has been automatically "
             "frozen to prevent forced address reuse: ")
        jlog.warning(utxostr)
        jlog.warning("You can unfreeze this utxo with the method "
                 "'freeze' of wallet-tool.py or the Coins tab "
                 "of Joinmarket-Qt.")

    def set_autofreeze_warning_cb(self, cb=None):
        """ This callback takes a single argument, the
        string representation of a utxo in form txid:index,
        and informs the user that the utxo has been frozen.
        It returns nothing (the user is not deciding in this case,
        as the decision was already taken by the configuration).
        """
        if cb is None:
            self.autofreeze_warning_cb = self.default_autofreeze_warning_cb
        else:
            self.autofreeze_warning_cb = cb

    def check_for_reuse(self, added_utxos):
        """ (a) Check if addresses in new utxos are already in
        used address list, (b) record new addresses as now used
        (c) disable the new utxo if it returned as true for (a),
        and it passes the filter set in the configuration.
        """
        to_be_frozen = set()
        for au in added_utxos:
            if self.has_address_been_used(added_utxos[au]["address"]):
                to_be_frozen.add(au)
        # any utxos actually added must have their destination address
        # added to the used address list for this program run:
        for au in added_utxos.values():
            self.used_addresses.add(au["address"])

        # disable those that passed the first check, before the addition,
        # if they satisfy configured logic
        for utxo in to_be_frozen:
            freeze_threshold = jm_single().config.getint("POLICY",
                                        "max_sats_freeze_reuse")
            if freeze_threshold == -1 or added_utxos[
                utxo]["value"] <= freeze_threshold:
                # freezing of coins must be communicated to user:
                self.autofreeze_warning_cb(utxo)
                self.disable_utxo(*utxo)

    def transaction_monitor(self):
        """Keeps track of any changes in the wallet (new transactions).
        Intended to be run as a twisted task.LoopingCall so that this
        Service is constantly in near-realtime sync with the blockchain.
        """

        if not self.update_blockheight():
            return

        txlist = self.bci.list_transactions(100)
        if not txlist:
            return

        new_txs = []
        for x in txlist:
            # process either (a) a completely new tx or
            # (b) a tx that reached unconf status but we are still
            # waiting for conf (active_txids)
            if "txid" not in x:
                continue
            if x['txid'] in self.active_txids or x['txid'] not in self.old_txs:
                new_txs.append(x)
        # reset for next polling event:
        self.old_txs = set(x['txid'] for x in txlist if "txid" in x)
        # for this invocation of transaction_monitor, we *don't* want
        # to call `gettransaction` more than once per txid, even if the
        # `listtransactions` result has multiple instances for different
        # wallet labels; so we use a temporary variable to cache.
        gettx_results = {}
        for tx in new_txs:
            txid = tx["txid"]
            if txid not in gettx_results:
                res = self.bci.get_transaction(hextobin(txid))
                gettx_results[txid] = res
            else:
                res = gettx_results[txid]
            if not res:
                continue
            confs = res["confirmations"]
            if not isinstance(confs, Integral):
                jlog.warning("Malformed gettx result: " + str(res))
                continue
            if confs < 0:
                jlog.info(
                    "Transaction: " + txid + " has a conflict, abandoning.")
                continue
            if confs == 0:
                height = None
            else:
                height = self.current_blockheight - confs + 1

            txd = self.bci.get_deser_from_gettransaction(res)
            if txd is None:
                continue
            removed_utxos, added_utxos = self.wallet.process_new_tx(txd, height)
            if txid not in self.processed_txids:
                # apply checks to disable/freeze utxos to reused addrs if needed:
                self.check_for_reuse(added_utxos)
                # TODO note that this log message will be missed if confirmation
                # is absurdly fast, this is considered acceptable compared with
                # additional complexity.
                self.log_new_tx(removed_utxos, added_utxos, txid)
                self.processed_txids.append(txid)

            # first fire 'all' type callbacks, irrespective of if the
            # transaction pertains to anything known (but must
            # have correct label per above); filter on this Joinmarket wallet label,
            # or the external monitoring label:
            if (self.bci.is_address_labeled(tx, self.get_wallet_name()) or
                self.bci.is_address_labeled(tx, self.EXTERNAL_WALLET_LABEL)):
                for f in self.callbacks["all"]:
                    # note we need no return value as we will never
                    # remove these from the list
                    f(txd, txid)

            # The tuple given as the second possible key for the dict
            # is such because txid is not always available
            # at the time of callback registration).
            possible_keys = [txid, tuple((x.scriptPubKey, x.nValue) for x in txd.vout)]

            # note that len(added_utxos) > 0 is not a sufficient condition for
            # the tx being new, since wallet.add_new_utxos will happily re-add
            # a utxo that already exists; but this does not cause re-firing
            # of callbacks since we in these cases delete the callback after being
            # called once.
            # Note also that it's entirely possible that there are only removals,
            # not additions, to the utxo set, specifically in sweeps to external
            # addresses. In this case, since removal can by definition only
            # happen once, we must allow entries in self.active_txids through the
            # filter.
            if len(added_utxos) > 0 or len(removed_utxos) > 0 \
               or txid in self.active_txids:
                if confs == 0:
                    for k in possible_keys:
                        if k in self.callbacks["unconfirmed"]:
                            for f in self.callbacks["unconfirmed"][k]:
                                # True implies success, implies removal:
                                if f(txd, txid):
                                    self.callbacks["unconfirmed"][k].remove(f)
                                    # keep monitoring for conf > 0:
                                    self.active_txids.append(txid)
                elif confs > 0:
                    for k in possible_keys:
                        if k in self.callbacks["confirmed"]:
                            for f in self.callbacks["confirmed"][k]:
                                if f(txd, txid, confs):
                                    self.callbacks["confirmed"][k].remove(f)
                                    if txid in self.active_txids:
                                        self.active_txids.remove(txid)

    def check_callback_called(self, txinfo, callback, cbtype, msg):
        """ Intended to be a deferred Task to be scheduled some
        set time after the callback was registered. "all" type
        callbacks do not expire and are not included.
        If the callback was previously called, return True, otherwise False.
        """
        assert cbtype in ["unconfirmed", "confirmed"]
        if txinfo in self.callbacks[cbtype]:
            if callback in self.callbacks[cbtype][txinfo]:
                # the callback was not called, drop it and warn
                self.callbacks[cbtype][txinfo].remove(callback)
                # TODO - dangling txids in self.active_txids will
                # be caused by this, but could also happen for
                # other reasons; possibly add logic to ensure that
                # this never occurs, although their presence should
                # not cause a functional error.
                jlog.info("Timed out: " + msg)
                return False
            # if callback is not in the list, it was already
            # processed and so do nothing.
        return True

    def log_new_tx(self, removed_utxos, added_utxos, txid):
        """ Changes to the wallet are logged at INFO level by
        the WalletService.
        """
        def report_changed(x, utxos):
            if len(utxos.keys()) > 0:
                jlog.info(x + ' utxos=\n{}'.format('\n'.join(
                    '{} - {}'.format(utxo_to_utxostr(u)[1],
                        fmt_tx_data(tx_data, self)) for u,
                    tx_data in utxos.items())))

        report_changed("Removed", removed_utxos)
        report_changed("Added", added_utxos)


        """ Wallet syncing code
        """

    def sync_wallet(self, fast=True):
        """ Syncs wallet; note that if slow sync
        requires multiple rounds this must be called
        until self.synced is True.
        Before starting the event loop, we cache
        the current most recent transactions as
        reported by the blockchain interface, since
        we are interested in deltas.
        """
        # If this is called when syncing already complete:
        if self.synced:
            return True
        if fast:
            self.sync_wallet_fast()
        else:
            self.sync_addresses()
            self.sync_unspent()
        # Don't attempt updates on transactions that existed
        # before startup
        self.old_txs = [x['txid'] for x in self.bci.list_transactions(100)
            if "txid" in x]
        if isinstance(self.bci, BitcoinCoreNoHistoryInterface):
            self.bci.set_wallet_no_history(self.wallet)
        return self.synced

    def resync_wallet(self, fast=True):
        """ The self.synced state is generally
        updated to True, once, at the start of
        a run of a particular program. Here we
        can manually force re-sync.
        """
        self.synced = False
        self.sync_wallet(fast=fast)

    def sync_wallet_fast(self):
        """Exploits the fact that given an index_cache,
        all addresses necessary should be imported, so we
        can just list all used addresses to find the right
        index values.
        """
        self.sync_addresses_fast()
        self.sync_unspent()

    def has_address_been_used(self, address):
        """ Once wallet has been synced, the set of used
        addresses includes those identified at sync time,
        plus any used during operation. This is stored in
        the WalletService object as self.used_addresses.
        """
        return address in self.used_addresses

    def get_address_usages(self):
        """ sets, at time of sync, the list of addresses that
        have been used in our Core wallet with the specific label
        for our JM wallet. This operation is generally immediate.
        """
        agd = self.bci.listaddressgroupings()
        # flatten all groups into a single list; then, remove duplicates
        fagd = (tuple(item) for sublist in agd for item in sublist)
        # "deduplicated flattened address grouping data" = dfagd
        dfagd = set(fagd)
        used_addresses = set()
        for addr_info in dfagd:
            if len(addr_info) < 3 or addr_info[2] != self.get_wallet_name():
                continue
            used_addresses.add(addr_info[0])
        self.used_addresses = used_addresses

    def sync_addresses_fast(self):
        """Locates all used addresses in the account (whether spent or
        unspent outputs), and then, assuming that all such usages must be
        related to our wallet, calculates the correct wallet indices and
        does any needed imports.

        This will not result in a full sync if working with a new
        Bitcoin Core instance, in which case "recoversync" should have
        been specifically chosen by the user.
        """
        self.get_address_usages()
        # for a first run, import first chunk
        if not self.used_addresses:
            jlog.info("Detected new wallet, performing initial import")
            # delegate inital address import to sync_addresses
            # this should be fast because "getaddressesbyaccount" should return
            # an empty list in this case
            self.sync_addresses()
            self.synced = True
            return

        # Wallet has been used; scan forwards.
        jlog.debug("Fast sync in progress. Got this many used addresses: " + str(
            len(self.used_addresses)))
        # Need to have wallet.index point to the last used address
        # Algo:
        #    1. Scan batch 1 of each branch, record matched wallet addresses.
        #    2. Check if all addresses in 'used addresses' have been matched, if
        #       so, break.
        #    3. Repeat the above for batch 2, 3.. up to max 20 batches.
        #    4. If after all 20 batches not all used addresses were matched,
        #       quit with error.
        #    5. Calculate used indices.
        #    6. If all used addresses were matched, set wallet index to highest
        #       found index in each branch and mark wallet sync complete.
        # Rationale for this algo:
        #    Retrieving addresses is a non-zero computational load, so batching
        #    and then caching allows a small sync to complete *reasonably*
        #    quickly while a larger one is not really negatively affected.
        #    The downside is another free variable, batch size, but this need
        #    not be exposed to the user; it is not the same as gap limit.
        #    The assumption is that usage of addresses occurs only if already
        #    imported, either through in-app usage during coinjoins, or because
        #    deposit by user will be based on wallet_display() which is only
        #    showing imported addresses. Hence the gap-limit import at the end
        #    to ensure this is always true.
        remaining_used_addresses = self.used_addresses.copy()
        addresses, saved_indices = self.collect_addresses_init()
        for addr in addresses:
            remaining_used_addresses.discard(addr)

        BATCH_SIZE = 100
        MAX_ITERATIONS = 20
        current_indices = deepcopy(saved_indices)
        for j in range(MAX_ITERATIONS):
            if not remaining_used_addresses:
                break
            gap_addrs = self.collect_addresses_gap(gap_limit=BATCH_SIZE)
            # note: gap addresses *not* imported here; we are still trying
            # to find the highest-index used address, and assume that imports
            # are up to that index (at least) - see above main rationale.
            for addr in gap_addrs:
                remaining_used_addresses.discard(addr)

            # increase wallet indices for next iteration
            for md in current_indices:
                current_indices[md][0] += BATCH_SIZE
                current_indices[md][1] += BATCH_SIZE
            self.rewind_wallet_indices(current_indices, current_indices)
        else:
            self.rewind_wallet_indices(saved_indices, saved_indices)
            raise Exception("Failed to sync in fast mode after 20 batches; "
                            "please re-try wallet sync with --recoversync flag.")

        # creating used_indices on-the-fly would be more efficient, but the
        # overall performance gain is probably negligible
        used_indices = self.get_used_indices(self.used_addresses)
        self.rewind_wallet_indices(used_indices, saved_indices)
        # at this point we have the correct up to date index at each branch;
        # we ensure that all addresses that will be displayed (see wallet_utils.py,
        # function wallet_display()) are imported by importing gap limit beyond current
        # index:
        self.bci.import_addresses(self.collect_addresses_gap(), self.get_wallet_name(),
                                  self.restart_callback)

        if isinstance(self.wallet, FidelityBondMixin):
            mixdepth = FidelityBondMixin.FIDELITY_BOND_MIXDEPTH
            address_type = FidelityBondMixin.BIP32_BURN_ID

            burner_outputs = self.wallet.get_burner_outputs()
            max_index = 0
            for path_repr in burner_outputs:
                index = self.wallet.path_repr_to_path(path_repr.decode())[-1]
                max_index = max(index+1, max_index)
            self.wallet.set_next_index(mixdepth, address_type, max_index,
                force=True)

        self.synced = True

    def display_rescan_message_and_system_exit(self, restart_cb):
        #TODO using system exit here should be avoided as it makes the code
        # harder to understand and reason about
        #theres also a sys.exit() in BitcoinCoreInterface.import_addresses()
        #perhaps have sys.exit() placed inside the restart_cb that only
        # CLI scripts will use
        if self.bci.__class__ == BitcoinCoreInterface:
            #Exit conditions cannot be included in tests
            restart_msg = ("restart Bitcoin Core with -rescan or use "
                           "`bitcoin-cli rescanblockchain` if you're "
                           "recovering an existing wallet from backup seed\n"
                           "Otherwise just restart this joinmarket application.")
            if restart_cb:
                restart_cb(restart_msg)
            else:
                jmprint(restart_msg, "important")
                sys.exit(EXIT_SUCCESS)

    def sync_burner_outputs(self, burner_txes):
        mixdepth = FidelityBondMixin.FIDELITY_BOND_MIXDEPTH
        address_type = FidelityBondMixin.BIP32_BURN_ID
        self.wallet.set_next_index(mixdepth, address_type, self.wallet.gap_limit,
            force=True)
        highest_used_index = 0
        known_burner_outputs = self.wallet.get_burner_outputs()

        index = -1
        while index - highest_used_index < self.wallet.gap_limit:
            index += 1
            self.wallet.set_next_index(mixdepth, address_type, index, force=True)
            path = self.wallet.get_path(mixdepth, address_type, index)
            path_privkey, engine = self.wallet._get_key_from_path(path)
            path_pubkey = engine.privkey_to_pubkey(path_privkey)
            path_pubkeyhash = btc.Hash160(path_pubkey)
            for burner_tx in burner_txes:
                burner_pubkeyhash, gettx = burner_tx
                if burner_pubkeyhash != path_pubkeyhash:
                    continue
                highest_used_index = index
                path_repr = self.wallet.get_path_repr(path)
                if path_repr.encode() in known_burner_outputs:
                    continue
                txid = gettx["txid"]
                jlog.info("Found a burner transaction txid=" + txid + " path = "
                    + path_repr)
                try:
                    merkle_branch = self.bci.get_tx_merkle_branch(txid, gettx["blockhash"])
                except ValueError as e:
                    jlog.warning(repr(e))
                    jlog.warning("Merkle branch likely not available, use "
                        + "wallet-tool `addtxoutproof`")
                    merkle_branch = None
                block_height = self.bci.get_block_height(gettx["blockhash"])
                if merkle_branch:
                    assert self.bci.verify_tx_merkle_branch(txid, block_height, merkle_branch)
                self.wallet.add_burner_output(path_repr, gettx["hex"], block_height,
                    merkle_branch, gettx["blockindex"])

        self.wallet.set_next_index(mixdepth, address_type, highest_used_index + 1)

    def get_all_transactions(self):
        """ Get all transactions (spending or receiving) that
        are currently recorded by our blockchain interface as relating
        to this wallet, as a list.
        """
        res = []
        processed_txids = []
        for r in self.bci._yield_transactions(
            self.get_wallet_name()):
            txid = r["txid"]
            if txid not in processed_txids:
                tx = self.bci.get_transaction(hextobin(txid))
                res.append(self.bci.get_deser_from_gettransaction(tx))
                processed_txids.append(txid)
        return res

    def get_transaction(self, txid):
        """ If the transaction for txid is an in-wallet
        transaction, will return a CTransaction object for it;
        if not, will return None.
        """
        tx = self.bci.get_transaction(txid)
        if not tx:
            return None
        return self.bci.get_deser_from_gettransaction(tx)

    def get_block_height(self, blockhash):
        return self.bci.get_block_height(blockhash)

    def get_transaction_block_height(self, tx):
        """ Given a CTransaction object tx, return
        the block height at which it was mined, or False
        if it is not found in the blockchain (including if
        it is not a wallet tx and so can't be queried).
        """
        txid = tx.GetTxid()[::-1]
        return self.get_block_height(self.bci.get_transaction(
            txid)["blockhash"])

    def sync_addresses(self):
        """ Triggered by use of --recoversync option in scripts,
        attempts a full scan of the blockchain without assuming
        anything about past usages of addresses (does not use
        wallet.index_cache as hint).
        """
        jlog.debug("requesting detailed wallet history")
        wallet_name = self.get_wallet_name()
        addresses, saved_indices = self.collect_addresses_init()

        import_needed = self.bci.import_addresses_if_needed(addresses,
            wallet_name)
        if import_needed:
            self.display_rescan_message_and_system_exit(self.restart_callback)
            return

        if isinstance(self.wallet, FidelityBondMixin):
            tx_receive = []
            burner_txes = []
            for tx in self.bci._yield_transactions(wallet_name):
                if tx['category'] == 'receive':
                    tx_receive.append(tx)
                elif tx["category"] == "send":
                    gettx = self.bci.get_transaction(hextobin(tx["txid"]))
                    txd = self.bci.get_deser_from_gettransaction(gettx)
                    if len(txd.vout) > 1:
                        continue
                    #must be mined into a block to sync
                    #otherwise there's no merkleproof or block index
                    if gettx["confirmations"] < 1:
                        continue
                    script = txd.vout[0].scriptPubKey
                    if script[0] != 0x6a: #OP_RETURN
                        continue
                    pubkeyhash = script[2:]
                    burner_txes.append((pubkeyhash, gettx))

            self.sync_burner_outputs(burner_txes)
            used_addresses_gen = set(tx["address"] for tx in tx_receive)
        else:
            #not fidelity bond wallet, significantly faster sync
            used_addresses_gen = set(tx['address']
                                  for tx in self.bci._yield_transactions(wallet_name)
                                  if tx['category'] == 'receive')
        # needed for address-reuse check:
        self.used_addresses = used_addresses_gen
        used_indices = self.get_used_indices(used_addresses_gen)
        jlog.debug("got used indices: {}".format(used_indices))
        gap_limit_used = not self.check_gap_indices(used_indices)
        self.rewind_wallet_indices(used_indices, saved_indices)

        new_addresses = self.collect_addresses_gap()
        if self.bci.import_addresses_if_needed(new_addresses, wallet_name):
            jlog.debug("Syncing iteration finished, additional step required (more address import required)")
            self.synced = False
            self.display_rescan_message_and_system_exit(self.restart_callback)
        elif gap_limit_used:
            jlog.debug("Syncing iteration finished, additional step required (gap limit used)")
            self.synced = False
        else:
            jlog.debug("Wallet successfully synced")
            self.rewind_wallet_indices(used_indices, saved_indices)
            self.synced = True

    def sync_unspent(self):
        st = time.time()
        # block height needs to be real time for addition to our utxos:
        current_blockheight = self.bci.get_current_block_height()
        if not current_blockheight:
            # this failure will shut down the application elsewhere, here
            # just give up:
            return
        wallet_name = self.get_wallet_name()
        self.reset_utxos()

        unspent_list = self.bci.listunspent()
        # filter on label, but note (a) in certain circumstances (in-
        # wallet transfer) it is possible for the utxo to be labeled
        # with the external label, and (b) the wallet will know if it
        # belongs or not anyway (is_known_addr):
        our_unspent_list = [x for x in unspent_list if (
            self.bci.is_address_labeled(x, wallet_name) or
            self.bci.is_address_labeled(x, self.EXTERNAL_WALLET_LABEL))]
        for utxo in our_unspent_list:
            if not self.is_known_addr(utxo['address']):
                continue
            # The result of bitcoin core's listunspent RPC call does not have
            # a "height" field, only "confirmations".
            # But the result of scantxoutset used in no-history sync does
            # have "height".
            if "height" in utxo:
                height = utxo["height"]
            else:
                height = None
                # wallet's utxo database needs to store an absolute rather
                # than relative height measure:
                confs = int(utxo['confirmations'])
                if confs < 0:
                    jlog.warning("Utxo not added, has a conflict: " + str(utxo))
                    continue
                if confs >= 1:
                    height = current_blockheight - confs + 1
            self._add_unspent_txo(utxo, height)
        et = time.time()
        jlog.debug('bitcoind sync_unspent took ' + str((et - st)) + 'sec')

    def _add_unspent_txo(self, utxo, height):
        """
        Add a UTXO as returned by rpc's listunspent call to the wallet.
        Note that these are returned as little endian outpoint txids, so
        are converted.
        params:
            utxo: single utxo dict as returned by listunspent
            current_blockheight: blockheight as integer, used to
            set the block in which a confirmed utxo is included.
        """
        txid = hextobin(utxo['txid'])
        script = hextobin(utxo['scriptPubKey'])
        value = int(Decimal(str(utxo['amount'])) * Decimal('1e8'))
        self.add_utxo(txid, int(utxo['vout']), script, value, height)


    """ The following functions mostly are not pure
    pass through to the underlying wallet, so declared
    here; the remainder are covered by the __getattr__
    fallback.
    """

    def save_wallet(self):
        self.wallet.save()

    def get_utxos_by_mixdepth(self, include_disabled=False,
                              verbose=False, includeconfs=False):
        """ Returns utxos by mixdepth in a dict, optionally including
        information about how many confirmations each utxo has.
        """
        def height_to_confs(x):
            # convert height entries to confirmations:
            ubym_conv = collections.defaultdict(dict)
            for m, i in x.items():
                for u, d in i.items():
                    ubym_conv[m][u] = d
                    h = ubym_conv[m][u].pop("height")
                    if h == INF_HEIGHT:
                        confs = 0
                    else:
                        confs = self.current_blockheight - h + 1
                    ubym_conv[m][u]["confs"] = confs
            return ubym_conv
        ubym = self.wallet.get_utxos_by_mixdepth(
            include_disabled=include_disabled, includeheight=includeconfs)
        if not includeconfs:
            return ubym
        else:
            return height_to_confs(ubym)

    def minconfs_to_maxheight(self, minconfs):
        if minconfs is None:
            return None
        else:
            return self.current_blockheight - minconfs + 1

    def select_utxos(self, mixdepth, amount, utxo_filter=None, select_fn=None,
                     minconfs=None, includeaddr=False, require_auth_address=False):
        """ Request utxos from the wallet in a particular mixdepth to satisfy
        a certain total amount, optionally set the selector function (or use
        the currently configured function set by the wallet, and optionally
        require a minimum of minconfs confirmations (default none means
        unconfirmed are allowed).
        """
        return self.wallet.select_utxos(
            mixdepth, amount, utxo_filter=utxo_filter, select_fn=select_fn,
            maxheight=self.minconfs_to_maxheight(minconfs),
            includeaddr=includeaddr, require_auth_address=require_auth_address)

    def get_balance_by_mixdepth(self, verbose=True,
                                include_disabled=False,
                                minconfs=None):
        if minconfs is None:
            maxheight = None
        else:
            maxheight = self.current_blockheight - minconfs + 1
        return self.wallet.get_balance_by_mixdepth(verbose=verbose,
                                                   include_disabled=include_disabled,
                                                   maxheight=maxheight)

    def import_addr(self, addr):
        if self.bci is not None and hasattr(self.bci, 'import_addresses'):
            self.bci.import_addresses([addr], self.wallet.get_wallet_name())

    def get_internal_addr(self, mixdepth):
        addr = self.wallet.get_internal_addr(mixdepth)
        self.import_addr(addr)
        return addr

    def collect_addresses_init(self):
        """ Collects the "current" set of addresses,
        as defined by the indices recorded in the wallet's
        index cache (persisted in the wallet file usually).
        Note that it collects up to the current indices plus
        the gap limit.
        """
        addresses = set()
        saved_indices = dict()

        for md in range(self.max_mixdepth + 1):
            saved_indices[md] = [0, 0]
            for address_type in (BaseWallet.ADDRESS_TYPE_EXTERNAL,
                                 BaseWallet.ADDRESS_TYPE_INTERNAL):
                next_unused = self.get_next_unused_index(md, address_type)
                for index in range(next_unused):
                    addresses.add(self.get_addr(md, address_type, index))
                for index in range(self.gap_limit):
                    addresses.add(self.get_new_addr(md, address_type))
                # reset the indices to the value we had before the
                # new address calls:
                self.set_next_index(md, address_type, next_unused)
                saved_indices[md][address_type] = next_unused
            # include any imported addresses
            for path in self.yield_imported_paths(md):
                addresses.add(self.get_address_from_path(path))

        if isinstance(self.wallet, FidelityBondMixin):
            md = FidelityBondMixin.FIDELITY_BOND_MIXDEPTH
            address_type = FidelityBondMixin.BIP32_TIMELOCK_ID
            for timenumber in range(FidelityBondMixin.TIMENUMBER_COUNT):
                addresses.add(self.get_addr(md, address_type, timenumber))

        return addresses, saved_indices

    def collect_addresses_gap(self, gap_limit=None):
        gap_limit = gap_limit or self.gap_limit
        addresses = set()

        for md in range(self.max_mixdepth + 1):
            for address_type in (BaseWallet.ADDRESS_TYPE_INTERNAL,
                                 BaseWallet.ADDRESS_TYPE_EXTERNAL):
                old_next = self.get_next_unused_index(md, address_type)
                for index in range(gap_limit):
                    addresses.add(self.get_new_addr(md, address_type))
                self.set_next_index(md, address_type, old_next)

        return addresses

    def get_external_addr(self, mixdepth):
        addr = self.wallet.get_external_addr(mixdepth)
        self.import_addr(addr)
        return addr

    def __getattr__(self, attr):
        # any method not present here is passed
        # to the wallet:
        return getattr(self.wallet, attr)
import sys
import os
import pprint
from twisted.internet import reactor
from twisted.python.log import startLogging
from jmclient import Taker, load_program_config, get_schedule,\
    JMClientProtocolFactory, start_reactor, jm_single, get_wallet_path,\
    open_test_wallet_maybe, get_tumble_schedule,\
    schedule_to_text, estimate_tx_fee, restart_waiter, WalletService,\
    get_tumble_log, tumbler_taker_finished_update, check_regtest, \
    tumbler_filter_orders_callback, validate_address, get_tumbler_parser, \
    get_max_cj_fee_values
from jmbase.support import get_log, jmprint, EXIT_SUCCESS, \
    EXIT_FAILURE, EXIT_ARGERROR
    jm_single().mincjamount = options['mincjamount']
    destaddrs = args[1:]
    for daddr in destaddrs:
        success, errmsg = validate_address(daddr)
        if not success:
            jmprint("Invalid destination address: " + daddr, "error")
            sys.exit(EXIT_ARGERROR)
    jmprint("Destination addresses: " + str(destaddrs), "important")
    #If the --restart flag is set we read the schedule
    #from the file, and filter out entries that are
    #already complete
    if options['restart']:
        res, schedule = get_schedule(os.path.join(logsdir,
                                                  options['schedulefile']))
        if not res:
            jmprint("Failed to load schedule, name: " + str(
                options['schedulefile']), "error")
            jmprint("Error was: " + str(schedule), "error")
            sys.exit(EXIT_FAILURE)
        #This removes all entries that are marked as done
        schedule = [s for s in schedule if s[-1] != 1]
        # remaining destination addresses must be stored in Taker.tdestaddrs
        # in case of tweaks; note we can't change, so any passed on command
        # line must be ignored:
        if len(destaddrs) > 0:
            jmprint("For restarts, destinations are taken from schedule file,"
                    " so passed destinations on the command line were ignored.",
                    "important")
            if input("OK? (y/n)") != "y":
                sys.exit(EXIT_SUCCESS)
        destaddrs = [s[3] for s in schedule if s[3] not in ["INTERNAL", "addrask"]]
        jmprint("Remaining destination addresses in restart: " + ",".join(destaddrs),
                "important")
        if isinstance(schedule[0][-1], str) and len(schedule[0][-1]) == 64:
            #ensure last transaction is confirmed before restart
            tumble_log.info("WAITING TO RESTART...")
            txid = schedule[0][-1]
            restart_waiter(txid)
            #remove the already-done entry (this connects to the other TODO,
            #probably better *not* to truncate the done-already txs from file,
            #but simplest for now.
            schedule = schedule[1:]
        elif schedule[0][-1] != 0:
            print("Error: first schedule entry is invalid.")
            sys.exit(EXIT_FAILURE)
        with open(os.path.join(logsdir, options['schedulefile']), "wb") as f:
                    f.write(schedule_to_text(schedule))
        tumble_log.info("TUMBLE RESTARTING")
    else:
        #Create a new schedule from scratch
        schedule = get_tumble_schedule(options, destaddrs,
            wallet.get_balance_by_mixdepth())
        tumble_log.info("TUMBLE STARTING")
        with open(os.path.join(logsdir, options['schedulefile']), "wb") as f:
            f.write(schedule_to_text(schedule))
        print("Schedule written to logs/" + options['schedulefile'])
    tumble_log.info("With this schedule: ")
    tumble_log.info(pprint.pformat(schedule))

    # If tx_fees are set manually by CLI argument, override joinmarket.cfg:
    if int(options['txfee']) > 0:
        jm_single().config.set("POLICY", "tx_fees", str(options['txfee']))

    # Dynamically estimate an expected tx fee for the whole tumbling run.
    # This is very rough: we guess with 2 inputs and 2 outputs each.
    fee_per_cp_guess = estimate_tx_fee(2, 2, txtype=wallet_service.get_txtype())
    log.debug("Estimated miner/tx fee for each cj participant: " + str(
            fee_per_cp_guess))

    # From the estimated tx fees, check if the expected amount is a
    # significant value compared the the cj amount
    involved_parties = len(schedule)    # own participation in each CJ
    for item in schedule:
        involved_parties += item[2] #  number of total tumble counterparties
    total_tumble_amount = int(0)
    max_mix_to_tumble = min(options['mixdepthsrc']+options['mixdepthcount'], \
                            max_mix_depth)
    for i in range(options['mixdepthsrc'], max_mix_to_tumble):
        total_tumble_amount += wallet_service.get_balance_by_mixdepth()[i]
        if total_tumble_amount == 0:
            raise ValueError("No confirmed coins in the selected mixdepth(s). Quitting")
    exp_tx_fees_ratio = (involved_parties * fee_per_cp_guess) \
        / total_tumble_amount
    if exp_tx_fees_ratio > 0.05:
        jmprint('WARNING: Expected bitcoin network miner fees for the whole '
            'tumbling run are roughly {:.1%}'.format(exp_tx_fees_ratio), "warning")
        if not options['restart'] and input('You might want to modify your tx_fee'
            ' settings in joinmarket.cfg. Still continue? (y/n):')[0] != 'y':
            sys.exit('Aborted by user.')
    else:
        log.info("Estimated miner/tx fees for this coinjoin amount for the "
            "whole tumbling run: {:.1%}".format(exp_tx_fees_ratio))

    print("Progress logging to logs/TUMBLE.log")

    def filter_orders_callback(orders_fees, cjamount):
        """Decide whether to accept fees
        """
        return tumbler_filter_orders_callback(orders_fees, cjamount, taker)

    def taker_finished(res, fromtx=False, waittime=0.0, txdetails=None):
        """on_finished_callback for tumbler; processing is almost entirely
        deferred to generic taker_finished in tumbler_support module, except
        here reactor signalling.
        """
        sfile = os.path.join(logsdir, options['schedulefile'])
        tumbler_taker_finished_update(taker, sfile, tumble_log, options,
                                      res, fromtx, waittime, txdetails)
        if not fromtx:
            reactor.stop()
        elif fromtx != "unconfirmed":
            reactor.callLater(waittime*60, clientfactory.getClient().clientStart)

    #instantiate Taker with given schedule and run
    taker = Taker(wallet_service,
                  schedule,
                  maxcjfee,
                  order_chooser=options['order_choose_fn'],
                  callbacks=(filter_orders_callback, None, taker_finished),
                  tdestaddrs=destaddrs)

    clientfactory = JMClientProtocolFactory(taker)