Inside Bitcoin

1#!/usr/bin/env python3
2# Copyright (c) 2014-present The Bitcoin Core developers
3# Distributed under the MIT software license, see the accompanying
4# file COPYING or http://www.opensource.org/licenses/mit-license.php.
5"""Test the pruning code.
6
7WARNING:
8This test uses 4GB of disk space.
9"""
10import os
11
12from test_framework.blocktools import (
13    MIN_BLOCKS_TO_KEEP,
14    create_block,
15    create_coinbase,
16)
17from test_framework.script import (
18    CScript,
19    OP_NOP,
20    OP_RETURN,
21)
22from test_framework.test_framework import BitcoinTestFramework
23from test_framework.util import (
24    assert_equal,
25    assert_greater_than,
26    assert_raises_rpc_error,
27    try_rpc,
28)
29
30# Rescans start at the earliest block up to 2 hours before a key timestamp, so
31# the manual prune RPC avoids pruning blocks in the same window to be
32# compatible with pruning based on key creation time.
33TIMESTAMP_WINDOW = 2 * 60 * 60
34
35def mine_large_blocks(node, n):
36    # Make a large scriptPubKey for the coinbase transaction. This is OP_RETURN
37    # followed by 950k of OP_NOP. This would be non-standard in a non-coinbase
38    # transaction but is consensus valid.
39
40    # Set the nTime if this is the first time this function has been called.
41    # A static variable ensures that time is monotonicly increasing and is therefore
42    # different for each block created => blockhash is unique.
43    if "nTime" not in mine_large_blocks.__dict__:
44        mine_large_blocks.nTime = 0
45
46    # Get the block parameters for the first block
47    big_script = CScript([OP_RETURN] + [OP_NOP] * 950000)
48    best_block = node.getblock(node.getbestblockhash())
49    height = int(best_block["height"]) + 1
50    mine_large_blocks.nTime = max(mine_large_blocks.nTime, int(best_block["time"])) + 1
51    previousblockhash = int(best_block["hash"], 16)
52
53    for _ in range(n):
54        block = create_block(hashprev=previousblockhash, ntime=mine_large_blocks.nTime, coinbase=create_coinbase(height, script_pubkey=big_script))
55        block.solve()
56
57        # Submit to the node
58        node.submitblock(block.serialize().hex())
59
60        previousblockhash = block.hash_int
61        height += 1
62        mine_large_blocks.nTime += 1
63
64def calc_usage(blockdir):
65    return sum(os.path.getsize(blockdir + f) for f in os.listdir(blockdir) if os.path.isfile(os.path.join(blockdir, f))) / (1024. * 1024.)
66
67class PruneTest(BitcoinTestFramework):
68    def set_test_params(self):
69        self.setup_clean_chain = True
70        self.num_nodes = 6
71        self.uses_wallet = None
72
73        # Create nodes 0 and 1 to mine.
74        # Create node 2 to test pruning.
75        self.full_node_default_args = ["-maxreceivebuffer=20000", "-checkblocks=5"]
76        # Create nodes 3 and 4 to test manual pruning (they will be re-started with manual pruning later)
77        # Create nodes 5 to test wallet in prune mode, but do not connect
78        self.extra_args = [
79            self.full_node_default_args,
80            self.full_node_default_args,
81            ["-maxreceivebuffer=20000", "-prune=550"],
82            ["-maxreceivebuffer=20000"],
83            ["-maxreceivebuffer=20000"],
84            ["-prune=550", "-blockfilterindex=1"],
85        ]
86        self.rpc_timeout = 120
87
88    def setup_network(self):
89        self.setup_nodes()
90
91        self.prunedir = os.path.join(self.nodes[2].blocks_path, '')
92
93        self.connect_nodes(0, 1)
94        self.connect_nodes(1, 2)
95        self.connect_nodes(0, 2)
96        self.connect_nodes(0, 3)
97        self.connect_nodes(0, 4)
98        self.sync_blocks(self.nodes[0:5])
99
100    def setup_nodes(self):
101        self.add_nodes(self.num_nodes, self.extra_args)
102        self.start_nodes()
103        if self.is_wallet_compiled():
104            self.import_deterministic_coinbase_privkeys()
105
106    def create_big_chain(self):
107        # Start by creating some coinbases we can spend later
108        self.generate(self.nodes[1], 200, sync_fun=lambda: self.sync_blocks(self.nodes[0:2]))
109        self.generate(self.nodes[0], 150, sync_fun=self.no_op)
110
111        # Then mine enough full blocks to create more than 550MiB of data
112        mine_large_blocks(self.nodes[0], 645)
113
114        self.sync_blocks(self.nodes[0:5])
115
116    def test_invalid_command_line_options(self):
117        self.stop_node(0)
118        self.nodes[0].assert_start_raises_init_error(
119            expected_msg='Error: Prune cannot be configured with a negative value.',
120            extra_args=['-prune=-1'],
121        )
122        self.nodes[0].assert_start_raises_init_error(
123            expected_msg='Error: Prune configured below the minimum of 550 MiB.  Please use a higher number.',
124            extra_args=['-prune=549'],
125        )
126        self.nodes[0].assert_start_raises_init_error(
127            expected_msg='Error: Prune mode is incompatible with -txindex.',
128            extra_args=['-prune=550', '-txindex'],
129        )
130        self.nodes[0].assert_start_raises_init_error(
131            expected_msg='Error: Prune mode is incompatible with -reindex-chainstate. Use full -reindex instead.',
132            extra_args=['-prune=550', '-reindex-chainstate'],
133        )
134
135    def test_rescan_blockchain(self):
136        self.restart_node(0, ["-prune=550"])
137        assert_raises_rpc_error(-1, "Can't rescan beyond pruned data. Use RPC call getblockchaininfo to determine your pruned height.", self.nodes[0].rescanblockchain)
138
139    def test_height_min(self):
140        assert os.path.isfile(os.path.join(self.prunedir, "blk00000.dat")), "blk00000.dat is missing, pruning too early"
141        self.log.info("Success")
142        self.log.info(f"Though we're already using more than 550MiB, current usage: {calc_usage(self.prunedir)}")
143        self.log.info("Mining 25 more blocks should cause the first block file to be pruned")
144        # Pruning doesn't run until we're allocating another chunk, 20 full blocks past the height cutoff will ensure this
145        mine_large_blocks(self.nodes[0], 25)
146
147        # Wait for blk00000.dat to be pruned
148        self.wait_until(lambda: not os.path.isfile(os.path.join(self.prunedir, "blk00000.dat")), timeout=30)
149
150        self.log.info("Success")
151        usage = calc_usage(self.prunedir)
152        self.log.info(f"Usage should be below target: {usage}")
153        assert_greater_than(550, usage)
154
155    def create_chain_with_staleblocks(self):
156        # Create stale blocks in manageable sized chunks
157        self.log.info("Mine 24 (stale) blocks on Node 1, followed by 25 (main chain) block reorg from Node 0, for 12 rounds")
158
159        for _ in range(12):
160            # Disconnect node 0 so it can mine a longer reorg chain without knowing about node 1's soon-to-be-stale chain
161            # Node 2 stays connected, so it hears about the stale blocks and then reorg's when node0 reconnects
162            self.disconnect_nodes(0, 1)
163            self.disconnect_nodes(0, 2)
164            # Mine 24 blocks in node 1
165            mine_large_blocks(self.nodes[1], 24)
166
167            # Reorg back with 25 block chain from node 0
168            mine_large_blocks(self.nodes[0], 25)
169
170            # Create connections in the order so both nodes can see the reorg at the same time
171            self.connect_nodes(0, 1)
172            self.connect_nodes(0, 2)
173            self.sync_blocks(self.nodes[0:3])
174
175        self.log.info(f"Usage can be over target because of high stale rate: {calc_usage(self.prunedir)}")
176
177    def reorg_test(self):
178        # Node 1 will mine a 300 block chain starting 287 blocks back from Node 0 and Node 2's tip
179        # This will cause Node 2 to do a reorg requiring 288 blocks of undo data to the reorg_test chain
180
181        height = self.nodes[1].getblockcount()
182        self.log.info(f"Current block height: {height}")
183
184        self.forkheight = height - 287
185        self.forkhash = self.nodes[1].getblockhash(self.forkheight)
186        self.log.info(f"Invalidating block {self.forkhash} at height {self.forkheight}")
187        self.nodes[1].invalidateblock(self.forkhash)
188
189        # We've now switched to our previously mined-24 block fork on node 1, but that's not what we want
190        # So invalidate that fork as well, until we're on the same chain as node 0/2 (but at an ancestor 288 blocks ago)
191        mainchainhash = self.nodes[0].getblockhash(self.forkheight - 1)
192        curhash = self.nodes[1].getblockhash(self.forkheight - 1)
193        while curhash != mainchainhash:
194            self.nodes[1].invalidateblock(curhash)
195            curhash = self.nodes[1].getblockhash(self.forkheight - 1)
196
197        assert_equal(self.nodes[1].getblockcount(), self.forkheight - 1)
198        self.log.info(f"New best height: {self.nodes[1].getblockcount()}")
199
200        # Disconnect node1 and generate the new chain
201        self.disconnect_nodes(0, 1)
202        self.disconnect_nodes(1, 2)
203
204        self.log.info("Generating new longer chain of 300 more blocks")
205        self.generate(self.nodes[1], 300, sync_fun=self.no_op)
206
207        self.log.info("Reconnect nodes")
208        self.connect_nodes(0, 1)
209        self.connect_nodes(1, 2)
210        self.sync_blocks(self.nodes[0:3], timeout=120)
211
212        self.log.info(f"Verify height on node 2: {self.nodes[2].getblockcount()}")
213        self.log.info(f"Usage possibly still high because of stale blocks in block files: {calc_usage(self.prunedir)}")
214
215        self.log.info("Mine 220 more large blocks so we have requisite history")
216
217        mine_large_blocks(self.nodes[0], 220)
218        self.sync_blocks(self.nodes[0:3], timeout=120)
219
220        usage = calc_usage(self.prunedir)
221        self.log.info(f"Usage should be below target: {usage}")
222        assert_greater_than(550, usage)
223
224    def reorg_back(self):
225        # Verify that a block on the old main chain fork has been pruned away
226        assert_raises_rpc_error(-1, "Block not available (pruned data)", self.nodes[2].getblock, self.forkhash)
227        with self.nodes[2].assert_debug_log(expected_msgs=["Block verification stopping at height", "(no data)"]):
228            assert not self.nodes[2].verifychain(checklevel=4, nblocks=0)
229        self.log.info(f"Will need to redownload block {self.forkheight}")
230
231        # Verify that we have enough history to reorg back to the fork point
232        # Although this is more than 288 blocks, because this chain was written more recently
233        # and only its other 299 small and 220 large blocks are in the block files after it,
234        # it is expected to still be retained
235        self.nodes[2].getblock(self.nodes[2].getblockhash(self.forkheight))
236
237        first_reorg_height = self.nodes[2].getblockcount()
238        block_hash_1295 = self.nodes[2].getblockhash(1295)
239        self.nodes[2].invalidateblock(block_hash_1295)
240        goalbestheight = self.mainchainheight
241        goalbesthash = self.mainchainhash2
242
243        # As of 0.10 the current block download logic is not able to reorg to the original chain created in
244        # create_chain_with_stale_blocks because it doesn't know of any peer that's on that chain from which to
245        # redownload its missing blocks.
246        # Invalidate the reorg_test chain in node 0 as well, it can successfully switch to the original chain
247        # because it has all the block data.
248        # However it must mine enough blocks to have a more work chain than the reorg_test chain in order
249        # to trigger node 2's block download logic.
250        # At this point node 2 is within 288 blocks of the fork point so it will preserve its ability to reorg