Inside Bitcoin

251
252            self.log.info(
253                "Creating transactions and mining them at a block size that is just big enough"
254            )
255            # Generate transactions while mining 10 more blocks, this time with node1
256            # which mines blocks with capacity just above the rate that transactions are being created
257            self.transact_and_mine(10, self.nodes[1])
258            check_estimates(self.nodes[1], self.fees_per_kb)
259
260        # Finish by mining a normal-sized block:
261        while len(self.nodes[1].getrawmempool()) > 0:
262            self.generate(self.nodes[1], 1)
263
264        self.log.info("Final estimates after emptying mempools")
265        check_estimates(self.nodes[1], self.fees_per_kb)
266
267    def test_estimates_with_highminrelaytxfee(self):
268        high_val = 3 * self.nodes[1].estimatesmartfee(2, "economical", {"fee_rate_estimator": "block_policy"})["feerate"]
269        self.restart_node(1, extra_args=[f"-minrelaytxfee={high_val}"])
270        check_smart_estimates(self.nodes[1], self.fees_per_kb)
271        self.restart_node(1)
272
273    def sanity_check_rbf_estimates(self):
274        """During 5 blocks, broadcast low fee transactions. Only 10% of them get
275        confirmed and the remaining ones get RBF'd with a high fee transaction at
276        the next block.
277        The block policy estimator should return the high feerate.
278        """
279        # The broadcaster and block producer
280        node = self.nodes[0]
281        miner = self.nodes[1]
282        # In sat/vb
283        low_feerate = 1
284        high_feerate = 10
285        # Cache the utxos of which to replace the spender after it failed to get
286        # confirmed
287        utxos_to_respend = []
288        txids_to_replace = []
289
290        assert_greater_than_or_equal(len(self.confutxo), 250)
291        for _ in range(5):
292            # Broadcast 45 low fee transactions that will need to be RBF'd
293            txs = []
294            for _ in range(45):
295                u = self.confutxo.pop(0)
296                tx = make_tx(self.wallet, u, low_feerate)
297                utxos_to_respend.append(u)
298                txids_to_replace.append(tx["txid"])
299                txs.append(tx)
300            # Broadcast 5 low fee transaction which don't need to
301            for _ in range(5):
302                tx = make_tx(self.wallet, self.confutxo.pop(0), low_feerate)
303                self.memutxo.append(tx["new_utxo"])
304                txs.append(tx)
305            batch_send_tx = [node.sendrawtransaction.get_request(tx["hex"]) for tx in txs]
306            for n in self.nodes:
307                n.batch(batch_send_tx)
308            # Mine the transactions on another node
309            self.sync_mempools(wait=0.1, nodes=[node, miner])
310            for txid in txids_to_replace:
311                miner.prioritisetransaction(txid=txid, fee_delta=-COIN)
312            mined = miner.getblock(self.generate(miner, 1)[0], True)["tx"]
313            self.update_utxo(mined)
314            # RBF the low-fee transactions
315            while len(utxos_to_respend) > 0:
316                u = utxos_to_respend.pop(0)
317                tx = make_tx(self.wallet, u, high_feerate)
318                self.memutxo.append(tx["new_utxo"])
319                node.sendrawtransaction(tx["hex"])
320                txs.append(tx)
321            dec_txs = [res["result"] for res in node.batch([node.decoderawtransaction.get_request(tx["hex"]) for tx in txs])]
322            self.wallet.scan_txs(dec_txs)
323
324
325        # Mine the last replacement txs
326        self.sync_mempools(wait=0.1, nodes=[node, miner])
327        mined = miner.getblock(self.generate(miner, 1)[0], True)["tx"]
328        self.update_utxo(mined)
329
330        # Only 10% of the transactions were really confirmed with a low feerate,
331        # the rest needed to be RBF'd. We must return the 90% conf rate feerate.
332        high_feerate_kvb = Decimal(high_feerate) / COIN * 10 ** 3
333        est_feerate = node.estimatesmartfee(2, "economical", {"fee_rate_estimator": "block_policy"})["feerate"]
334        assert_equal(est_feerate, high_feerate_kvb)
335
336    def test_fallback_when_mempool_estimator_has_insufficient_data(self):
337        assert_equal(self.nodes[0].estimatesmartfee(1, "economical", {"fee_rate_estimator": "auto"})["estimator"], "mempool_policy")
338        self.stop_node(0)
339        # Move the mempool estimator data aside so the mempool policy estimator has insufficient data.
340        original_path = self.nodes[0].chain_path / "fees" / "mempool_policy_estimator.dat"
341        temp_path = self.nodes[0].chain_path / "fees" / "mempool_policy_estimator.dat.bak"
342        original_path.replace(temp_path)
343        self.start_node(0)
344        self.wait_until(lambda: self.nodes[0].getmempoolinfo()["loaded"])
345        assert_equal(self.nodes[0].estimatesmartfee(1, "economical", {"fee_rate_estimator": "mempool_policy"})["errors"],
346                     ["mempool_policy: Not enough recent block data for fee rate estimation"])
347        assert_equal(self.nodes[0].estimatesmartfee(1, "economical", {"fee_rate_estimator": "auto"})["estimator"], "block_policy")
348
349        # Restore the mempool estimator data.
350        self.stop_node(0)
351        temp_path.replace(original_path)
352        self.start_node(0)
353
354    def test_old_fee_estimate_file(self):
355        # Get the initial fee rate while node is running
356        fee_rate = self.nodes[0].estimatesmartfee(1, "economical", {"fee_rate_estimator": "block_policy"})["feerate"]
357
358        # Restart node to ensure block policy estimator file is read
359        self.restart_node(0)
360        assert_equal(self.nodes[0].estimatesmartfee(1, "economical", {"fee_rate_estimator": "block_policy"})["feerate"], fee_rate)
361
362        block_policy_fee_dat = self.nodes[0].chain_path / BLOCK_POLICY_ESTIMATOR_FILE_PATH
363        legacy_fee_dat = self.nodes[0].chain_path / "fee_estimates.dat"
364
365        # If only the legacy fee_estimates.dat file exists, it is migrated to
366        # the new block policy estimator path.
367        self.stop_node(0)
368        os.rename(block_policy_fee_dat, legacy_fee_dat)
369        self.start_node(0)
370        assert_equal(self.nodes[0].estimatesmartfee(1, "economical", {"fee_rate_estimator": "block_policy"})["feerate"], fee_rate)
371        self.stop_node(0)
372        assert_equal(os.path.isfile(block_policy_fee_dat), True)
373        assert_equal(os.path.isfile(legacy_fee_dat), False)
374
375        # If both files exist, the new block policy estimator path is used and
376        # the obsolete legacy file is removed.
377        with open(legacy_fee_dat, "wb") as f:
378            f.write(b"ignored legacy fee estimates")
379        self.start_node(0)
380        assert_equal(self.nodes[0].estimatesmartfee(1, "economical", {"fee_rate_estimator": "block_policy"})["feerate"], fee_rate)
381        self.stop_node(0)
382        assert_equal(os.path.isfile(legacy_fee_dat), False)
383
384        # Stop the node and backdate the block policy estimator file more than MAX_FILE_AGE
385        last_modified_time = time.time() - (MAX_FILE_AGE + 1) * SECONDS_PER_HOUR
386        os.utime(block_policy_fee_dat, (last_modified_time, last_modified_time))
387
388        # Start node and ensure the block policy estimator file was not read
389        self.start_node(0)
390        assert_equal(self.nodes[0].estimatesmartfee(1, "economical", {"fee_rate_estimator": "block_policy"})["errors"], [BLOCK_POLICY_ESTIMATOR_ERROR])
391
392
393    def test_estimate_dat_is_flushed_periodically(self):
394        block_policy_fees_dat = self.nodes[0].chain_path / BLOCK_POLICY_ESTIMATOR_FILE_PATH
395        mempool_policy_dat = self.nodes[0].chain_path / "fees/mempool_policy_estimator.dat"
396        mempool_estimator_name_str = "mempool_policy"
397        os.remove(block_policy_fees_dat) if os.path.exists(block_policy_fees_dat) else None
398        os.remove(mempool_policy_dat) if os.path.exists(mempool_policy_dat) else None
399        if os.path.isdir(block_policy_fees_dat.parent):
400            os.rmdir(block_policy_fees_dat.parent)
401
402        # Verify that estimator data files and their parent directory do not exist
403        assert_equal(os.path.isfile(block_policy_fees_dat), False)
404        assert_equal(os.path.isfile(mempool_policy_dat), False)
405        assert_equal(os.path.isdir(block_policy_fees_dat.parent), False)
406        # Verify if the string "Flushed fee estimates to block_policy_estimates.dat." is present in the debug log file.
407        # If present, it indicates that fee estimator data has been successfully flushed to disk.
408        block_policy_estimator_message = f"Flushed fee estimates to {block_policy_fees_dat}."
409        mempool_policy_estimator_message = (
410            f"{mempool_estimator_name_str}: mined-block stats flushed to {mempool_policy_dat}."
411        )
412        expected_messages = [block_policy_estimator_message, mempool_policy_estimator_message]
413        with self.nodes[0].assert_debug_log(expected_msgs=expected_messages, timeout=1):
414            # Mock the scheduler for an hour to flush estimator data.
415            self.nodes[0].mockscheduler(SECONDS_PER_HOUR)
416
417        # Verify that estimator data was flushed and the estimator directory and files are created
418        assert_equal(os.path.isdir(block_policy_fees_dat.parent), True)
419        assert_equal(os.path.isfile(block_policy_fees_dat), True)
420        assert_equal(os.path.isfile(mempool_policy_dat), True)
421        # Verify that estimator data remains the same if there are no blocks in the flush interval
422        block_hash_before = self.nodes[0].getbestblockhash()
423        block_policy_fees_dat_initial_content = open(block_policy_fees_dat, "rb").read()
424        mempool_policy_dat_initial_content = open(mempool_policy_dat, "rb").read()
425        with self.nodes[0].assert_debug_log(expected_msgs=expected_messages, timeout=1):
426            # Mock the scheduler for an hour to flush estimator data.
427            self.nodes[0].mockscheduler(SECONDS_PER_HOUR)
428        # Verify that there were no blocks in between the flush interval
429        assert_equal(block_hash_before, self.nodes[0].getbestblockhash())
430        block_policy_fees_dat_current_content = open(block_policy_fees_dat, "rb").read()
431        mempool_policy_dat_current_content = open(mempool_policy_dat, "rb").read()
432        assert_equal(block_policy_fees_dat_initial_content, block_policy_fees_dat_current_content)
433        assert_equal(mempool_policy_dat_initial_content, mempool_policy_dat_current_content)
434        # Verify that estimator data remains the same after shutdown with no blocks before shutdown
435        self.restart_node(0)
436        block_policy_fees_dat_current_content = open(block_policy_fees_dat, "rb").read()
437        mempool_policy_dat_current_content = open(mempool_policy_dat, "rb").read()
438        assert_equal(block_policy_fees_dat_initial_content, block_policy_fees_dat_current_content)
439        assert_equal(mempool_policy_dat_initial_content, mempool_policy_dat_current_content)
440        # Verify that estimator data changes if new blocks were produced in the flush interval
441        with self.nodes[0].assert_debug_log(expected_msgs=expected_messages, timeout=1):
442            # Mock the scheduler for an hour to flush estimator data.
443            self.generate(self.nodes[0], 5, sync_fun=self.no_op)
444            self.nodes[0].mockscheduler(SECONDS_PER_HOUR)
445        block_policy_fees_dat_current_content = open(block_policy_fees_dat, "rb").read()
446        assert_not_equal(block_policy_fees_dat_current_content, block_policy_fees_dat_initial_content)
447        block_policy_fees_dat_initial_content = block_policy_fees_dat_current_content
448        mempool_policy_dat_current_content = open(mempool_policy_dat, "rb").read()
449        assert_not_equal(mempool_policy_dat_current_content, mempool_policy_dat_initial_content)
450        mempool_policy_dat_initial_content = mempool_policy_dat_current_content
451        # Generate blocks before shutdown and verify that estimator data changes
452        self.generate(self.nodes[0], 5, sync_fun=self.no_op)
453        self.restart_node(0)
454        block_policy_fees_dat_current_content = open(block_policy_fees_dat, "rb").read()
455        mempool_policy_dat_current_content = open(mempool_policy_dat, "rb").read()
456        assert_not_equal(block_policy_fees_dat_initial_content, block_policy_fees_dat_current_content)
457        assert_not_equal(mempool_policy_dat_initial_content, mempool_policy_dat_current_content)
458
459
460    def test_acceptstalefeeestimates_option(self):
461        # Get the initial fee rate while node is running
462        fee_rate = self.nodes[0].estimatesmartfee(1, "economical", {"fee_rate_estimator": "block_policy"})["feerate"]
463
464        self.stop_node(0)
465
466        fee_dat = self.nodes[0].chain_path / BLOCK_POLICY_ESTIMATOR_FILE_PATH
467
468        # Stop the node and backdate the block policy estimator file more than MAX_FILE_AGE
469        last_modified_time = time.time() - (MAX_FILE_AGE + 1) * SECONDS_PER_HOUR
470        os.utime(fee_dat, (last_modified_time, last_modified_time))
471
472        # Restart node with -acceptstalefeeestimates option to ensure block policy estimator file is read
473        self.start_node(0,extra_args=["-acceptstalefeeestimates"])
474        assert_equal(self.nodes[0].estimatesmartfee(1, "economical", {"fee_rate_estimator": "block_policy"})["feerate"], fee_rate)
475
476    def clear_estimates(self):
477        self.log.info("Restarting node with fresh estimation")
478        self.stop_node(0)
479        fee_dat = self.nodes[0].chain_path / BLOCK_POLICY_ESTIMATOR_FILE_PATH
480        os.remove(fee_dat)
481        self.start_node(0)
482        self.connect_nodes(0, 1)
483        self.connect_nodes(0, 2)
484        self.sync_blocks()
485        assert_equal(self.nodes[0].estimatesmartfee(1, "economical", {"fee_rate_estimator": "block_policy"})["errors"], ["Insufficient data or no feerate found"])
486
487    def broadcast_and_maybe_mine(self, broadcaster, feerate, txs, blocks=1, miner=None):
488        """Broadcast and maybe mine some number of transactions with a specified fee rate."""
489        for _ in range(blocks):
490            tx_batch = []
491            for _ in range(txs):
492                tx = self.wallet.create_self_transfer(fee_rate=feerate, utxo_to_spend=self.confutxo.pop(0))
493                self.memutxo.append(tx["new_utxo"])
494                tx_batch.append(tx)
495            # To speed up the test, submit the transactions in batches to the nodes directly
496            # avoiding having to wait for p2p to propagate them between the nodes.
497            batch_send_tx = [broadcaster.sendrawtransaction.get_request(hexstring=tx["hex"]) for tx in tx_batch]
498            for node in self.nodes:
499                node.batch(batch_send_tx)
500            self.sync_mempools(wait=0.1, nodes=[self.nodes[0], self.nodes[1], self.nodes[2]])