READ THE CODE

Inside Bitcoin

The code behind the network, with room to understand it.

Stable release v32.1bitcoin/bitcoin · f0de87f04062 ↗3,020 source files · 47.5 MB stored on the server440 file guides

Release published 2026-10-28 · Snapshot fetched 2026-11-01. Latest stable release at fetch time; development branches and Git history are excluded. Only the selected code window and visible folders load here.

6,617 bytes · SHA-256 e7661963ffc8bdd8… · View upstream ↗

1#!/usr/bin/env python3
2# Copyright (c) 2024-present The Bitcoin Core developers
3# Distributed under the MIT software license, see the accompanying
4# file COPYING or http://www.opensource.org/licenses/mit-license.php.
5"""Test utxo-to-sqlite conversion tool"""
6from itertools import product
7import os
8import platform
9try:
10    import sqlite3
11except ImportError:
12    pass
13import subprocess
14import sys
15
16from test_framework.key import ECKey
17from test_framework.messages import (
18    COutPoint,
19    CTxOut,
20    uint256_from_str,
21)
22from test_framework.crypto.muhash import MuHash3072
23from test_framework.script import (
24    CScript,
25    CScriptOp,
26)
27from test_framework.script_util import (
28    PAY_TO_ANCHOR,
29    key_to_p2pk_script,
30    key_to_p2pkh_script,
31    key_to_p2wpkh_script,
32    keys_to_multisig_script,
33    output_key_to_p2tr_script,
34    script_to_p2sh_script,
35    script_to_p2wsh_script,
36)
37from test_framework.test_framework import BitcoinTestFramework
38from test_framework.util import (
39    assert_equal,
40)
41from test_framework.wallet import MiniWallet
42
43
44def calculate_muhash_from_sqlite_utxos(filename, txid_format, spk_format):
45    muhash = MuHash3072()
46    con = sqlite3.connect(filename)
47    cur = con.cursor()
48    for (txid, vout, value, coinbase, height, spk) in cur.execute("SELECT * FROM utxos"):
49        match txid_format:
50            case "hex":
51                assert type(txid) is str
52                txid_bytes = bytes.fromhex(txid)[::-1]
53            case "raw":
54                assert type(txid) is bytes
55                txid_bytes = txid
56            case "rawle":
57                assert type(txid) is bytes
58                txid_bytes = txid[::-1]
59        match spk_format:
60            case "hex":
61                assert type(spk) is str
62                spk_bytes = bytes.fromhex(spk)
63            case "raw":
64                assert type(spk) is bytes
65                spk_bytes = spk
66
67        # serialize UTXO for MuHash (see function `TxOutSer` in the  coinstats module)
68        utxo_ser = COutPoint(uint256_from_str(txid_bytes), vout).serialize()
69        utxo_ser += (height * 2 + coinbase).to_bytes(4, 'little')
70        utxo_ser += CTxOut(value, spk_bytes).serialize()
71        muhash.insert(utxo_ser)
72    con.close()
73    return muhash.digest()[::-1].hex()
74
75
76class UtxoToSqliteTest(BitcoinTestFramework):
77    def set_test_params(self):
78        self.num_nodes = 1
79        # we want to create some UTXOs with non-standard output scripts
80        self.extra_args = [['-acceptnonstdtxn=1', '-coinstatsindex=1']]
81
82    def skip_test_if_missing_module(self):
83        self.skip_if_no_py_sqlite3()
84
85    def run_test(self):
86        node = self.nodes[0]
87        wallet = MiniWallet(node)
88        key = ECKey()
89
90        self.log.info('Create UTXOs with various output script types')
91        for i in range(1, 10+1):
92            key.generate(compressed=False)
93            uncompressed_pubkey = key.get_pubkey().get_bytes()
94            key.generate(compressed=True)
95            pubkey = key.get_pubkey().get_bytes()
96
97            # add output scripts for compressed script type 0 (P2PKH), type 1 (P2SH),
98            # types 2-3 (P2PK compressed), types 4-5 (P2PK uncompressed) and
99            # for uncompressed scripts (bare multisig, segwit, etc.)
100            output_scripts = (
101                key_to_p2pkh_script(pubkey),
102                script_to_p2sh_script(key_to_p2pkh_script(pubkey)),
103                key_to_p2pk_script(pubkey),
104                key_to_p2pk_script(uncompressed_pubkey),
105
106                keys_to_multisig_script([pubkey]*i),
107                keys_to_multisig_script([uncompressed_pubkey]*i),
108                key_to_p2wpkh_script(pubkey),
109                script_to_p2wsh_script(key_to_p2pkh_script(pubkey)),
110                output_key_to_p2tr_script(pubkey[1:]),
111                PAY_TO_ANCHOR,
112                CScript([CScriptOp.encode_op_n(i)]*(1000*i)),  # large script (up to 10000 bytes)
113            )
114
115            # create outputs and mine them in a block
116            for output_script in output_scripts:
117                wallet.send_to(from_node=node, scriptPubKey=output_script, amount=i, fee=20000)
118            self.generate(wallet, 1)
119
120        self.log.info('Dump UTXO set via `dumptxoutset` RPC')
121        input_filename = os.path.join(self.options.tmpdir, "utxos.dat")
122        node.dumptxoutset(input_filename, "latest")
123
124        for i, (txid_format, spk_format) in enumerate(product(["hex", "raw", "rawle"], ["hex", "raw"])):
125            self.log.info(f'Test utxo-to-sqlite script using txid format "{txid_format}" and spk format "{spk_format}" ({i+1})')
126            self.log.info('-> Convert UTXO set from compact-serialized format to sqlite format')
127            output_filename = os.path.join(self.options.tmpdir, f"utxos_{i+1}.sqlite")
128            base_dir = self.config["environment"]["SRCDIR"]
129            utxo_to_sqlite_path = os.path.join(base_dir, "contrib", "utxo-tools", "utxo_to_sqlite.py")
130            arguments = [input_filename, output_filename, f'--txid={txid_format}', f'--spk={spk_format}']
131            subprocess.run([sys.executable, utxo_to_sqlite_path] + arguments, check=True, stderr=subprocess.STDOUT)
132
133            self.log.info('-> Verify that both UTXO sets match by comparing their MuHash')
134            muhash_sqlite = calculate_muhash_from_sqlite_utxos(output_filename, txid_format, spk_format)
135            muhash_compact_serialized = node.gettxoutsetinfo('muhash')['muhash']
136            assert_equal(muhash_sqlite, muhash_compact_serialized)
137            self.log.info('')
138
139        if platform.system() != "Windows":  # FIFOs are not available on Windows
140            self.log.info('Convert UTXO set directly (without intermediate dump) via named pipe')
141            fifo_filename = os.path.join(self.options.tmpdir, "utxos.fifo")
142            os.mkfifo(fifo_filename)
143            output_direct_filename = os.path.join(self.options.tmpdir, "utxos_direct.sqlite")
144            p = subprocess.Popen([sys.executable, utxo_to_sqlite_path, fifo_filename, output_direct_filename],
145                                 stderr=subprocess.STDOUT)
146            target_height = node.getblockcount() - 10
147            node.dumptxoutset(fifo_filename, "rollback", {"rollback": target_height})
148            p.wait(timeout=10)
149            muhash_direct_sqlite = calculate_muhash_from_sqlite_utxos(output_direct_filename, "hex", "hex")
150            muhash_index = node.gettxoutsetinfo('muhash', target_height)['muhash']
151            assert_equal(muhash_index, muhash_direct_sqlite)
152            os.remove(fifo_filename)
153
154
155if __name__ == "__main__":
156    UtxoToSqliteTest(__file__).main()
157

src/wallet/receive.cpp

src/wallet/coinselection.cpp

src/consensus/validation.h

src/primitives/transaction.cpp

src/script/descriptor.cpp

src/bitcoin-cli.cpp

src/rpc/blockchain.cpp

src/rpc/rawtransaction.cpp

src/init.cpp

src/crypto/sha256.cpp

src/consensus/merkle.h

src/consensus/tx_check.h

src/consensus/tx_verify.h

src/policy/feerate.h

src/policy/feerate.cpp

src/primitives/transaction_identifier.h

src/chainparamsbase.h

src/chainparamsbase.cpp

src/util/moneystr.cpp

src/crypto/ripemd160.h

src/crypto/sha1.h

src/crypto/sha512.h

src/crypto/hmac_sha256.h

src/crypto/hmac_sha512.h

src/crypto/hmac_sha256.cpp

src/crypto/hmac_sha512.cpp

src/crypto/hkdf_sha256_32.h

src/crypto/hkdf_sha256_32.cpp

src/support/cleanse.h

src/support/cleanse.cpp

src/crypto/siphash.cpp

src/crypto/muhash.h

src/crypto/chacha20.h

src/crypto/poly1305.h

src/crypto/chacha20poly1305.h

src/crypto/chacha20poly1305.cpp

src/util/chaintype.cpp

src/util/obfuscation.h

src/util/byte_units.h

src/util/overflow.h

src/blockencodings.cpp

src/net_permissions.cpp

src/policy/packages.cpp

src/policy/rbf.cpp

src/policy/truc_policy.cpp

src/policy/ephemeral_policy.cpp

src/policy/settings.cpp

src/policy/policy.cpp

src/node/transaction.cpp

src/node/coin.cpp

src/rpc/client.cpp

src/rpc/external_signer.cpp

src/rpc/request.cpp

src/rpc/server_util.cpp

src/rpc/fees.cpp

src/rpc/output_script.cpp

src/rpc/rawtransaction_util.cpp

src/rpc/rawtransaction_util.h

src/rpc/register.h

src/rpc/request.h

src/rpc/server_util.h

src/rpc/protocol.h

src/rpc/server.h

src/rpc/signmessage.cpp

src/rpc/txoutproof.cpp

src/rpc/util.cpp

src/rpc/util.h

src/wallet/rpc/addresses.cpp

src/wallet/rpc/backup.cpp

src/wallet/rpc/coins.cpp

src/wallet/rpc/encrypt.cpp

src/wallet/rpc/signmessage.cpp

src/wallet/rpc/util.cpp

src/wallet/rpc/util.h

src/wallet/coincontrol.cpp

src/wallet/coincontrol.h

src/wallet/context.cpp

src/wallet/context.h

src/wallet/crypter.cpp

src/wallet/crypter.h

src/wallet/db.cpp

src/wallet/export.cpp

src/wallet/external_signer_scriptpubkeyman.cpp

src/wallet/rpc/spend.cpp

src/wallet/rpc/transactions.cpp

src/wallet/rpc/wallet.cpp

src/wallet/feebumper.cpp

src/wallet/feebumper.h

src/index/base.h

src/index/base.cpp

src/index/txindex.h

src/index/txindex.cpp

src/index/txindex_key.h

src/index/txospenderindex.h

src/index/txospenderindex.cpp

src/index/blockfilterindex.h

src/index/blockfilterindex.cpp

src/index/coinstatsindex.h

src/index/coinstatsindex.cpp

src/index/db_key.h

src/index/disktxpos.h

src/kernel/chain.cpp

src/kernel/chain.h

src/kernel/checks.cpp

src/kernel/checks.h

src/kernel/context.cpp

src/kernel/context.h

src/kernel/cs_main.cpp

src/kernel/cs_main.h

src/kernel/disconnected_transactions.cpp

src/kernel/disconnected_transactions.h

src/kernel/mempool_removal_reason.cpp

src/kernel/mempool_removal_reason.h

src/kernel/coinstats.cpp

src/kernel/coinstats.h

src/kernel/chainparams.cpp

src/kernel/chainparams.h

src/wallet/sqlite.cpp

src/wallet/sqlite.h

src/wallet/transaction.cpp

src/wallet/transaction.h

src/wallet/dump.cpp

src/wallet/dump.h

src/wallet/init.cpp

src/wallet/walletutil.cpp

src/wallet/walletutil.h

src/wallet/wallettool.cpp

src/wallet/wallettool.h

src/wallet/export.h

src/wallet/migrate.cpp

src/wallet/migrate.h

src/wallet/db.h

src/wallet/walletdb.cpp

src/wallet/interfaces.cpp

src/wallet/scriptpubkeyman.cpp

src/wallet/scriptpubkeyman.h

src/kernel/bitcoinkernel.cpp

src/kernel/bitcoinkernel.h

src/kernel/bitcoinkernel_wrapper.h

src/node/chainstate.cpp

src/node/chainstate.h

src/node/blockstorage.cpp

src/node/blockstorage.h

src/node/mempool_persist.cpp

src/node/mempool_persist.h

src/node/utxo_snapshot.cpp

src/node/utxo_snapshot.h

src/node/kernel_notifications.cpp

src/node/kernel_notifications.h

src/node/blockmanager_args.cpp

src/kernel/blockmanager_opts.h

src/util/expected.cpp

src/policy/settings.h

src/util/types.h

src/kernel/messagestartchars.h

src/kernel/warning.h

src/util/exception.h

src/common/license_info.h

src/node/coins_view_args.h

src/node/database_args.h

src/util/batchpriority.h

src/util/ui_change_type.h

src/node/coins_view_args.cpp

src/node/peerman_args.h

src/util/serfloat.h

src/node/blockmanager_args.h

src/node/database_args.cpp

src/util/overloaded.h

src/interfaces/types.h

src/node/chainstatemanager_args.h

src/node/minisketchwrapper.h

src/util/macros.h

src/util/syserror.h

src/common/run_command.h

src/util/bytevectorhash.cpp

src/node/mempool_persist_args.cpp

src/util/exec.h

src/util/rbf.h

src/crypto/hex_base.h

src/node/mempool_persist_args.h

src/interfaces/echo.h

src/util/any.h

src/util/batchpriority.cpp

src/util/threadnames.h

src/node/coin.h

src/node/mempool_args.h

src/util/asmap.h

src/util/bytevectorhash.h

src/node/context.cpp

src/util/insert.h

src/util/readwritefile.h

src/node/peerman_args.cpp

src/node/interface_ui.cpp

src/interfaces/handler.h

src/node/mining_args.h

src/util/hasher.cpp

src/common/init.h

src/kernel/mempool_limits.h

src/common/system.h

src/crypto/sha3.h

src/util/threadinterrupt.cpp

src/node/protocol_version.h

src/util/exception.cpp

src/common/messages.h

src/node/connection_types.cpp

src/util/bip32.h

src/common/license_info.cpp

src/util/stdmutex.h

src/common/run_command.cpp

src/util/check.cpp

src/util/golombrice.h

src/node/types.h

src/node/timeoffsets.h

src/util/readwritefile.cpp

src/util/fastrange.h

src/util/signalinterrupt.h

src/util/task_runner.h

src/util/syserror.cpp

src/common/interfaces.cpp

src/common/types.h

src/util/threadinterrupt.h

src/kernel/chainstatemanager_opts.h

src/node/psbt.h

src/policy/ephemeral_policy.h

src/util/trace.h

src/common/url.cpp

src/kernel/mempool_options.h

src/util/serfloat.cpp

src/kernel/notifications_interface.h

src/node/timeoffsets.cpp

src/node/warnings.cpp

src/util/hash_type.h

src/crypto/hex_base.cpp

src/node/eviction.h

src/util/tokenbucket.h

src/crypto/aes.h

src/common/pcp.h

src/util/feefrac.cpp

src/node/transaction.h

src/util/threadnames.cpp

src/common/signmessage.h

src/util/exec.cpp

src/util/signalinterrupt.cpp

src/clientversion.cpp

src/common/signmessage.cpp

src/compat/assumptions.h

src/compat/byteswap.h

src/deploymentinfo.cpp

src/deploymentinfo.h

src/deploymentstatus.cpp

src/deploymentstatus.h

src/dummywallet.cpp

src/external_signer.h

src/init/bitcoin-gui.cpp

src/init/bitcoin-node.cpp

src/init/bitcoin-qt.cpp

src/init/bitcoin-wallet.cpp

src/init/bitcoind.cpp

src/init/common.h

src/interfaces/init.h

src/interfaces/ipc.h

src/key_io.h

src/logging/categories.h

src/logging/timer.h

src/mapport.h

src/musig.h

src/net_permissions.h

src/netgroup.h

src/netmessagemaker.h

src/node/chainstatemanager_args.cpp

src/node/connection_types.h

src/node/context.h

src/node/interface_ui.h

src/node/mempool_args.cpp

src/node/mining_args.cpp

src/node/minisketchwrapper.cpp

src/node/txreconciliation.h

src/policy/fees/estimator_args.cpp

src/policy/fees/estimator_args.h

src/policy/fees/estimator_man.cpp

src/policy/fees/estimator_man.h

src/policy/packages.h

src/policy/rbf.h

src/policy/truc_policy.h

src/pow.h

src/primitives/block.cpp

src/randomenv.h

src/rest.h

src/script/keyorigin.h

src/script/parsing.cpp

src/script/parsing.h

src/script/script_error.h

src/script/sigcache.cpp

src/script/sigcache.h

src/script/solver.h

src/script/verify_flags.h

src/semaphore_grant.h

src/signet.h

src/span.h

src/support/allocators/secure.h

src/support/allocators/zeroafterfree.h

src/util/tokenpipe.cpp

src/util/translation.h

src/util/vector.h

src/versionbits.h

src/versionbits_impl.h

src/wallet/external_signer_scriptpubkeyman.h

src/wallet/fees.h

src/wallet/load.h

src/wallet/receive.h

src/wallet/rpc/wallet.h

src/wallet/types.h

src/walletinitinterface.h

src/zmq/zmqabstractnotifier.cpp

Independent explanations by MyBitcoinExplorer. Not affiliated with or endorsed by Bitcoin Core. Guides are educational and do not replace the source or its tests.

Bitcoin Core source © its contributors, distributed under the MIT license. Original notices and bundled third-party licenses are retained in the source tree.