Inside Bitcoin

501            if miner:
502                mined = miner.getblock(self.generate(miner, 1)[0], True)["tx"]
503                self.update_utxo(mined)
504
505    def send_transactions(self, utxos, fee_rate, target_vsize):
506        for utxo in utxos:
507            self.wallet.send_self_transfer(
508                from_node=self.nodes[0],
509                utxo_to_spend=utxo,
510                fee_rate=fee_rate,
511                target_vsize=target_vsize,
512            )
513
514    def test_estimation_modes(self):
515        low_feerate = Decimal("0.001")
516        high_feerate = Decimal("0.005")
517        # Broadcast and mine high fee transactions for the first 12 blocks.
518        self.broadcast_and_maybe_mine(self.nodes[1], high_feerate, TXS_COUNT, 12, self.nodes[2])
519        check_fee_estimates_btw_modes(self.nodes[0], high_feerate, high_feerate)
520        # We now track 12 blocks; short horizon stats will start decaying.
521        # Broadcast and mine low fee transactions for the next 4 blocks.
522        self.broadcast_and_maybe_mine(self.nodes[1], low_feerate, TXS_COUNT, 4, self.nodes[2])
523        # conservative mode will consider longer time horizons while economical mode does not
524        # Check the fee estimates for both modes after mining low fee transactions.
525        check_fee_estimates_btw_modes(self.nodes[0], high_feerate, low_feerate)
526
527    def test_sub_1s_per_vb_estimates(self):
528        feerate_0_5_s_per_vb = MIN_BUCKET_FEERATE * 5
529        feerate_1_s_per_vb = Decimal(1000) / Decimal(COIN)
530        for i in range(6):
531            self.broadcast_and_maybe_mine(self.nodes[1], feerate_0_5_s_per_vb, TXS_COUNT)
532            self.broadcast_and_maybe_mine(self.nodes[1], feerate_1_s_per_vb, TXS_COUNT, 1, self.nodes[2])
533        assert_equal(feerate_0_5_s_per_vb, self.nodes[0].estimatesmartfee(1, "economical", {"fee_rate_estimator": "block_policy"})["feerate"])
534
535
536    def test_estimatesmartfee_return_mempool_estimates(self):
537        node0 = self.nodes[0]
538        miner = self.nodes[1]
539        self.log.info("Ensure node0's mempool is empty at the start")
540        assert_equal(node0.getmempoolinfo()['size'], 0)
541        self.log.info("Test estimatesmartfee with empty mempool and no block policy estimator data")
542        estimate_after_restart = node0.estimatesmartfee(1, "economical", {"fee_rate_estimator": "auto"})
543        verify_estimate_response(estimate_after_restart, None, [BLOCK_POLICY_ESTIMATOR_ERROR])
544        self.log.info("Populate block policy estimator with high-feerate history")
545        # Generate high-feerate transactions and mine them over 6 blocks to give block policy data.
546        high_feerate = Decimal("0.004")
547        self.broadcast_and_maybe_mine(node0, high_feerate, TXS_COUNT, 6, miner)
548        self.log.info("Test estimatesmartfee returns block policy estimator estimate when mempool is higher")
549        # Add 10 large insane-feerate transactions enough to generate a block template
550        num_txs = 10
551        target_vsize = int(((MAX_BLOCK_WEIGHT - DEFAULT_BLOCK_RESERVED_WEIGHT) / WITNESS_SCALE_FACTOR) / num_txs)
552        utxos = [self.wallet.get_utxo(confirmed_only=True) for _ in range(num_txs)]
553        insane_feerate = Decimal("0.01")
554        self.send_transactions(utxos, insane_feerate, target_vsize)
555        estimate_after_spike = node0.estimatesmartfee(1, "economical", {"verbosity": 2, "fee_rate_estimator": "auto"})
556        assert_equal(len(estimate_after_spike["mempool_health_statistics"]), 6)
557        current_height = node0.getchaintips()[0]['height']
558        for block_stat in estimate_after_spike["mempool_health_statistics"]:
559            assert_equal(block_stat['block_height'], current_height)
560            current_height -= 1
561            assert block_stat['block_weight']
562            assert block_stat['mempool_txs_weight']
563        verify_estimate_response(estimate_after_spike, high_feerate, [])
564        assert_equal(estimate_after_spike["estimator"], "block_policy")
565        mempool_policy_estimate = node0.estimatesmartfee(1, "economical", {"fee_rate_estimator": "mempool_policy"})
566        verify_estimate_response(mempool_policy_estimate, insane_feerate, [])
567        # Confirm the spike transactions so they leave the mempool; the mined block
568        # keeps the mempool representation healthy. Then broadcast fresh low-feerate
569        # transactions so the mempool estimate is now the lower of the two.
570        self.generate(node0, 1, sync_fun=lambda: None)
571        assert_equal(node0.getmempoolinfo()['size'], 0)
572        low_feerate = Decimal("0.00004")
573        low_utxos = [self.wallet.get_utxo(confirmed_only=True) for _ in range(num_txs)]
574        self.send_transactions(low_utxos, low_feerate, target_vsize)
575        lower_estimate = node0.estimatesmartfee(1, "economical", {"fee_rate_estimator": "auto"})
576        verify_estimate_response(lower_estimate, low_feerate, [])
577        # The mempool block stats are persisted across restarts, so the mempool
578        # stays healthy and the lower mempool estimate is still returned after a
579        # restart. Without persistence, the combined estimate would fall back to
580        # the block policy estimate until enough new blocks are observed.
581        self.restart_node(0)
582        estimate_post_restart = node0.estimatesmartfee(1, "economical", {"fee_rate_estimator": "auto"})
583        verify_estimate_response(estimate_post_restart, low_feerate, [])
584
585        self.log.info("Test estimatesmartfee returns the fee rate floor when the mempool is empty but healthy")
586        self.generate(node0, 1, sync_fun=lambda: None)
587        assert_equal(node0.getmempoolinfo()['size'], 0)
588        block_policy_estimate = node0.estimatesmartfee(1, "economical", {"fee_rate_estimator": "block_policy"})
589        assert "feerate" in block_policy_estimate
590        # With an empty but healthy mempool the mempool estimator has no percentile data,
591        # so it falls back to the fee rate floor: the max of minrelaytxfee and mempoolminfee.
592        # That floor is lower than the block policy estimate, so the combined estimator returns it.
593        mempool_info = node0.getmempoolinfo()
594        floor = max(mempool_info["minrelaytxfee"], mempool_info["mempoolminfee"])
595        combined_estimate = node0.estimatesmartfee(1, "economical", {"fee_rate_estimator": "auto"})
596        verify_estimate_response(combined_estimate, floor, [])
597        assert_equal(combined_estimate["estimator"], "mempool_policy")
598
599    def test_fallback_to_block_policy_on_low_coverage(self):
600        self.clear_estimates()
601        self.broadcast_and_maybe_mine(self.nodes[0], Decimal("0.004"), TXS_COUNT, MEMPOOL_HEALTH_WINDOW, self.nodes[1])
602        assert "feerate" in self.nodes[0].estimatesmartfee(1, "economical", {"fee_rate_estimator": "block_policy"})
603        assert_equal(self.nodes[0].estimatesmartfee(1, "economical", {"fee_rate_estimator": "auto"})["estimator"], "mempool_policy")
604        anyone_can_spend_script = CScript([OP_TRUE]).hex()
605        quarter_block_vsize = int(MAX_BLOCK_WEIGHT / WITNESS_SCALE_FACTOR / 4)
606        expected_stats = []
607        for _ in range(MEMPOOL_HEALTH_WINDOW):
608            unseen_txs = [make_tx(self.wallet, self.wallet.get_utxo(confirmed_only=True), 10, quarter_block_vsize) for _ in range(3)]
609            mempool_txs = [self.wallet.send_self_transfer(from_node=self.nodes[1]) for _ in range(3)]
610            self.sync_mempools([self.nodes[0], self.nodes[1]])
611            self.generateblock(self.nodes[1], output=f"raw({anyone_can_spend_script})",
612                               transactions=[tx["hex"] for tx in unseen_txs] + [tx["txid"] for tx in mempool_txs],
613                               sync_fun=lambda: self.sync_blocks([self.nodes[0], self.nodes[1]]))
614            expected_stats.append({
615                "block_height": self.nodes[0].getblockcount(),
616                "block_weight": sum(tx["tx"].get_weight() for tx in unseen_txs + mempool_txs),
617                "mempool_txs_weight": sum(tx["tx"].get_weight() for tx in mempool_txs),
618            })
619        combined = self.nodes[0].estimatesmartfee(1, "economical", {"fee_rate_estimator": "auto", "verbosity": 2})
620        assert_equal(combined["estimator"], "block_policy")
621        assert_equal(combined["feerate"], self.nodes[0].estimatesmartfee(1, "economical", {"fee_rate_estimator": "block_policy"})["feerate"])
622        assert_equal(combined["mempool_health_statistics"], list(reversed(expected_stats)))
623        mempool_policy = self.nodes[0].estimatesmartfee(1, "economical", {"fee_rate_estimator": "mempool_policy"})
624        assert_equal(mempool_policy["errors"], ["mempool_policy: Mempool is unreliable for fee rate estimation"])
625        self.restart_node(0)
626
627    def test_mined_block_stats_cleared_on_failed_mempool_load(self):
628        assert_equal(len(self.nodes[0].estimatesmartfee(1, "economical", {"fee_rate_estimator": "auto", "verbosity": 2})["mempool_health_statistics"]), MEMPOOL_HEALTH_WINDOW)
629        estimator_dat = self.nodes[0].chain_path / "fees" / "mempool_policy_estimator.dat"
630        mempool_dat = self.nodes[0].chain_path / "mempool.dat"
631        estimator_bak = estimator_dat.with_name(estimator_dat.name + ".bak")
632        mempool_bak = mempool_dat.with_name(mempool_dat.name + ".bak")
633        self.stop_node(0)
634        shutil.copyfile(estimator_dat, estimator_bak)
635        shutil.copyfile(mempool_dat, mempool_bak)
636
637        def assert_failed_load_clears_window(break_mempool_load, **start_kwargs):
638            shutil.copyfile(estimator_bak, estimator_dat)
639            shutil.copyfile(mempool_bak, mempool_dat)
640            break_mempool_load()
641            self.start_node(0, **start_kwargs)
642            self.wait_until(lambda: self.nodes[0].getmempoolinfo()["loaded"])
643            after = self.nodes[0].estimatesmartfee(1, "economical", {"fee_rate_estimator": "auto", "verbosity": 2})
644            assert_equal(after["mempool_health_statistics"], [])
645            assert_equal(after["estimator"], "block_policy")
646            assert_equal(self.nodes[0].estimatesmartfee(1, "economical", {"fee_rate_estimator": "mempool_policy"})["errors"],
647                         ["mempool_policy: Not enough recent block data for fee rate estimation"])
648            self.stop_node(0)
649
650        assert_failed_load_clears_window(mempool_dat.unlink)
651        assert_failed_load_clears_window(lambda: mempool_dat.write_bytes(b"not a valid mempool.dat"))
652        assert_failed_load_clears_window(lambda: None, extra_args=["-persistmempool=0"])
653
654        shutil.copyfile(estimator_bak, estimator_dat)
655        shutil.copyfile(mempool_bak, mempool_dat)
656        estimator_bak.unlink()
657        mempool_bak.unlink()
658        self.start_node(0)
659
660    def test_stale_mempool_block_stats_are_rejected_on_load(self):
661        # Persisted mempool block stats must be tied to the best block hash,
662        # not just height, because a reorg can replace the tip without
663        # changing the height.
664        node0 = self.nodes[0]
665        miner = self.nodes[1]
666        mempool_policy_dat = node0.chain_path / "fees/mempool_policy_estimator.dat"
667        healthy_feerate = Decimal("0.004")
668        self.connect_nodes(0, 1)
669        self.connect_nodes(0, 2)
670        self.sync_all()
671        # Build a full, healthy window whose tracked heights match the current tip.
672        self.broadcast_and_maybe_mine(node0, healthy_feerate, TXS_COUNT, 6, miner)
673        stale_stats = node0.estimatesmartfee(
674            1,
675            "economical",
676            {"verbosity": 2, "fee_rate_estimator": "auto"},
677        )["mempool_health_statistics"]
678        assert_equal(len(stale_stats), 6)
679        stale_height = node0.getblockcount()
680        assert_equal(stale_stats[0]["block_height"], stale_height)
681        stale_tip = node0.getbestblockhash()
682        self.stop_node(0)
683        stale_stats_snapshot = open(mempool_policy_dat, "rb").read()
684        self.start_node(0)
685        node0.invalidateblock(stale_tip)
686        assert_equal(node0.getblockcount(), stale_height - 1)
687        reorged_tip = self.generate(node0, 1, sync_fun=lambda: None)[0]
688        assert_equal(node0.getblockcount(), stale_height)
689        assert_not_equal(reorged_tip, stale_tip)
690        self.stop_node(0)
691        with open(mempool_policy_dat, "wb") as f:
692            f.write(stale_stats_snapshot)
693        self.start_node(0)
694        assert_equal(node0.getblockcount(), stale_height)
695        assert_equal(node0.getbestblockhash(), reorged_tip)
696        stats_after_restart = node0.estimatesmartfee(
697            1,
698            "economical",
699            {"verbosity": 2, "fee_rate_estimator": "auto"},
700        )["mempool_health_statistics"]
701        assert_equal(stats_after_restart, [])
702
703    def run_test(self):
704        self.log.info("This test is time consuming, please be patient")
705        self.log.info("Splitting inputs so we can generate tx's")
706
707        # Split two coinbases into many small utxos
708        self.start_node(0)
709        self.wallet = MiniWallet(self.nodes[0])
710        self.initial_split(self.nodes[0])
711        self.log.info("Finished splitting")
712
713        # Now we can connect the other nodes, didn't want to connect them earlier
714        # so the estimates would not be affected by the splitting transactions
715        self.start_node(1)
716        self.start_node(2)
717        self.connect_nodes(1, 0)
718        self.connect_nodes(0, 2)
719        self.connect_nodes(2, 1)
720        self.sync_all()
721
722        self.log.info("Testing estimates with single transactions.")
723        self.sanity_check_estimates_range()
724
725        self.log.info("Test fees/block_policy_estimates.dat is flushed periodically")
726        self.test_estimate_dat_is_flushed_periodically()
727
728        # check that estimatesmartfee feerate is greater than or equal to maximum of mempoolminfee and minrelaytxfee
729        self.log.info(
730            "Test fee rate estimation after restarting node with high minrelaytxfee"
731        )
732        self.test_estimates_with_highminrelaytxfee()
733
734        self.log.info("Test acceptstalefeeestimates option")
735        self.test_acceptstalefeeestimates_option()
736
737        self.log.info("Test fallback to block policy when the mempool estimator has insufficient data")
738        self.test_fallback_when_mempool_estimator_has_insufficient_data()
739
740        self.log.info("Test reading old block policy estimator file")
741        self.test_old_fee_estimate_file()
742
743        self.clear_estimates()
744
745        self.log.info("Testing estimates with RBF.")
746        self.sanity_check_rbf_estimates()
747
748        self.clear_estimates()
749        self.log.info("Test estimatesmartfee modes")
750        self.test_estimation_modes()