Inside Bitcoin

1#!/usr/bin/env python3
2#
3# Copyright (c) 2018-present The Bitcoin Core developers
4# Distributed under the MIT software license, see the accompanying
5# file COPYING or http://www.opensource.org/licenses/mit-license.php.
6
7import argparse
8import contextlib
9from fnmatch import fnmatch
10import hashlib
11import os
12from pathlib import Path
13import platform
14import re
15import shutil
16import subprocess
17import sys
18import time
19import zipfile
20
21sys.path.append(str(Path(__file__).resolve().parent))
22from download_utils import download_from_url
23
24TAR = os.getenv('TAR', 'tar')
25
26SHA256_SUMS = {
27    "0e2819135366f150d9906e294b61dff58fd1996ebd26c2f8e979d6c0b7a79580": {"tag": "v0.14.3", "archive": "bitcoin-0.14.3-aarch64-linux-gnu.tar.gz"},
28    "d86fc90824a85c38b25c8488115178d5785dbc975f5ff674f9f5716bc8ad6e65": {"tag": "v0.14.3", "archive": "bitcoin-0.14.3-arm-linux-gnueabihf.tar.gz"},
29    "1b0a7408c050e3d09a8be8e21e183ef7ee570385dc41216698cc3ab392a484e7": {"tag": "v0.14.3", "archive": "bitcoin-0.14.3-osx64.tar.gz"},
30    "706e0472dbc933ed2757650d54cbcd780fd3829ebf8f609b32780c7eedebdbc9": {"tag": "v0.14.3", "archive": "bitcoin-0.14.3-x86_64-linux-gnu.tar.gz"},
31    "5ea84b6a4c1b03d5f4d1a718fbed215e2fd0e66ca7d59ca6141218d163ce1301": {"tag": "v0.14.3", "archive": "bitcoin-0.14.3-win64.zip"},
32
33    "60c93e3462c303eb080be7cf623f1a7684b37fd47a018ad3848bc23e13c84e1c": {"tag": "v0.20.1", "archive": "bitcoin-0.20.1-aarch64-linux-gnu.tar.gz"},
34    "55b577e0fb306fb429d4be6c9316607753e8543e5946b542d75d876a2f08654c": {"tag": "v0.20.1", "archive": "bitcoin-0.20.1-arm-linux-gnueabihf.tar.gz"},
35    "b9024dde373ea7dad707363e07ec7e265383204127539ae0c234bff3a61da0d1": {"tag": "v0.20.1", "archive": "bitcoin-0.20.1-osx64.tar.gz"},
36    "fa71cb52ee5e0459cbf5248cdec72df27995840c796f58b304607a1ed4c165af": {"tag": "v0.20.1", "archive": "bitcoin-0.20.1-riscv64-linux-gnu.tar.gz"},
37    "376194f06596ecfa40331167c39bc70c355f960280bd2a645fdbf18f66527397": {"tag": "v0.20.1", "archive": "bitcoin-0.20.1-x86_64-linux-gnu.tar.gz"},
38    "e59fba67afce011d32b5d723a3a0be12da1b8a34f5d7966e504520c48d64716d": {"tag": "v0.20.1", "archive": "bitcoin-0.20.1-win64.zip"},
39
40    "43416854330914992bbba2d0e9adf2a6fff4130be9af8ae2ef1186e743d9a3fe": {"tag": "v0.21.0", "archive": "bitcoin-0.21.0-aarch64-linux-gnu.tar.gz"},
41    "f028af308eda45a3c4c90f9332f96b075bf21e3495c945ebce48597151808176": {"tag": "v0.21.0", "archive": "bitcoin-0.21.0-arm-linux-gnueabihf.tar.gz"},
42    "695fb624fa6423f5da4f443b60763dd1d77488bfe5ef63760904a7b54e91298d": {"tag": "v0.21.0", "archive": "bitcoin-0.21.0-osx64.tar.gz"},
43    "f8b2adfeae021a672effbc7bd40d5c48d6b94e53b2dd660f787340bf1a52e4e9": {"tag": "v0.21.0", "archive": "bitcoin-0.21.0-riscv64-linux-gnu.tar.gz"},
44    "da7766775e3f9c98d7a9145429f2be8297c2672fe5b118fd3dc2411fb48e0032": {"tag": "v0.21.0", "archive": "bitcoin-0.21.0-x86_64-linux-gnu.tar.gz"},
45    "1d0052c4ce80227fb6d0bc1c4e673ba21033e219c1f935d25f130ef7f43360d4": {"tag": "v0.21.0", "archive": "bitcoin-0.21.0-win64.zip"},
46
47    "ac718fed08570a81b3587587872ad85a25173afa5f9fbbd0c03ba4d1714cfa3e": {"tag": "v22.0", "archive": "bitcoin-22.0-aarch64-linux-gnu.tar.gz"},
48    "b8713c6c5f03f5258b54e9f436e2ed6d85449aa24c2c9972f91963d413e86311": {"tag": "v22.0", "archive": "bitcoin-22.0-arm-linux-gnueabihf.tar.gz"},
49    "2744d199c3343b2d94faffdfb2c94d75a630ba27301a70e47b0ad30a7e0155e9": {"tag": "v22.0", "archive": "bitcoin-22.0-osx64.tar.gz"},
50    "2cca5f99007d060aca9d8c7cbd035dfe2f040dd8200b210ce32cdf858479f70d": {"tag": "v22.0", "archive": "bitcoin-22.0-powerpc64-linux-gnu.tar.gz"},
51    "91b1e012975c5a363b5b5fcc81b5b7495e86ff703ec8262d4b9afcfec633c30d": {"tag": "v22.0", "archive": "bitcoin-22.0-powerpc64le-linux-gnu.tar.gz"},
52    "9cc3a62c469fe57e11485fdd32c916f10ce7a2899299855a2e479256ff49ff3c": {"tag": "v22.0", "archive": "bitcoin-22.0-riscv64-linux-gnu.tar.gz"},
53    "59ebd25dd82a51638b7a6bb914586201e67db67b919b2a1ff08925a7936d1b16": {"tag": "v22.0", "archive": "bitcoin-22.0-x86_64-linux-gnu.tar.gz"},
54    "9485e4b52ed6cebfe474ab4d7d0c1be6d0bb879ba7246a8239326b2230a77eb1": {"tag": "v22.0", "archive": "bitcoin-22.0-win64.zip"},
55
56    "06f4c78271a77752ba5990d60d81b1751507f77efda1e5981b4e92fd4d9969fb": {"tag": "v23.0", "archive": "bitcoin-23.0-aarch64-linux-gnu.tar.gz"},
57    "952c574366aff76f6d6ad1c9ee45a361d64fa04155e973e926dfe7e26f9703a3": {"tag": "v23.0", "archive": "bitcoin-23.0-arm-linux-gnueabihf.tar.gz"},
58    "7c8bc63731aa872b7b334a8a7d96e33536ad77d49029bad179b09dca32cd77ac": {"tag": "v23.0", "archive": "bitcoin-23.0-arm64-apple-darwin.tar.gz"},
59    "2caa5898399e415f61d9af80a366a3008e5856efa15aaff74b88acf429674c99": {"tag": "v23.0", "archive": "bitcoin-23.0-powerpc64-linux-gnu.tar.gz"},
60    "217dd0469d0f4962d22818c368358575f6a0abcba8804807bb75325eb2f28b19": {"tag": "v23.0", "archive": "bitcoin-23.0-powerpc64le-linux-gnu.tar.gz"},
61    "078f96b1e92895009c798ab827fb3fde5f6719eee886bd0c0e93acab18ea4865": {"tag": "v23.0", "archive": "bitcoin-23.0-riscv64-linux-gnu.tar.gz"},
62    "c816780583009a9dad426dc0c183c89be9da98906e1e2c7ebae91041c1aaaaf3": {"tag": "v23.0", "archive": "bitcoin-23.0-x86_64-apple-darwin.tar.gz"},
63    "2cca490c1f2842884a3c5b0606f179f9f937177da4eadd628e3f7fd7e25d26d0": {"tag": "v23.0", "archive": "bitcoin-23.0-x86_64-linux-gnu.tar.gz"},
64    "004b2e25b21e0f14cbcce6acec37f221447abbb3ea7931c689e508054bfc6cf6": {"tag": "v23.0", "archive": "bitcoin-23.0-win64.zip"},
65
66    "0b48b9e69b30037b41a1e6b78fb7cbcc48c7ad627908c99686e81f3802454609": {"tag": "v24.0.1", "archive": "bitcoin-24.0.1-aarch64-linux-gnu.tar.gz"},
67    "37d7660f0277301744e96426bbb001d2206b8d4505385dfdeedf50c09aaaef60": {"tag": "v24.0.1", "archive": "bitcoin-24.0.1-arm-linux-gnueabihf.tar.gz"},
68    "90ed59e86bfda1256f4b4cad8cc1dd77ee0efec2492bcb5af61402709288b62c": {"tag": "v24.0.1", "archive": "bitcoin-24.0.1-arm64-apple-darwin.tar.gz"},
69    "7590645e8676f8b5fda62dc20174474c4ac8fd0defc83a19ed908ebf2e94dc11": {"tag": "v24.0.1", "archive": "bitcoin-24.0.1-powerpc64-linux-gnu.tar.gz"},
70    "79e89a101f23ff87816675b98769cd1ee91059f95c5277f38f48f21a9f7f8509": {"tag": "v24.0.1", "archive": "bitcoin-24.0.1-powerpc64le-linux-gnu.tar.gz"},
71    "6b163cef7de4beb07b8cb3347095e0d76a584019b1891135cd1268a1f05b9d88": {"tag": "v24.0.1", "archive": "bitcoin-24.0.1-riscv64-linux-gnu.tar.gz"},
72    "e2f751512f3c0f00eb68ba946d9c829e6cf99422a61e8f5e0a7c109c318674d0": {"tag": "v24.0.1", "archive": "bitcoin-24.0.1-x86_64-apple-darwin.tar.gz"},
73    "49df6e444515d457ea0b885d66f521f2a26ca92ccf73d5296082e633544253bf": {"tag": "v24.0.1", "archive": "bitcoin-24.0.1-x86_64-linux-gnu.tar.gz"},
74    "8784ce304f22c495392d3adfd7fc2c645d093db9bd4d42666c41adf540539fff": {"tag": "v24.0.1", "archive": "bitcoin-24.0.1-win64.zip"},
75
76    "3a7bdd959a0b426624f63f394f25e5b7769a5a2f96f8126dcc2ea53f3fa5212b": {"tag": "v25.0", "archive": "bitcoin-25.0-aarch64-linux-gnu.tar.gz"},
77    "e537c8630b05e63242d979c3004f851fd73c2a10b5b4fdbb161788427c7b3c0f": {"tag": "v25.0", "archive": "bitcoin-25.0-arm-linux-gnueabihf.tar.gz"},
78    "3b35075d6c1209743611c705a13575be2668bc069bc6301ce78a2e1e53ebe7cc": {"tag": "v25.0", "archive": "bitcoin-25.0-arm64-apple-darwin.tar.gz"},
79    "0c8e135a6fd297270d3b65196042d761453493a022b5ff7fb847fc911e938214": {"tag": "v25.0", "archive": "bitcoin-25.0-powerpc64-linux-gnu.tar.gz"},
80    "fa8af160782f5adfcea570f72b947073c1663b3e9c3cd0f82b216b609fe47573": {"tag": "v25.0", "archive": "bitcoin-25.0-powerpc64le-linux-gnu.tar.gz"},
81    "fe6e347a66043946920c72c9c4afca301968101e6b82fb90a63d7885ebcceb32": {"tag": "v25.0", "archive": "bitcoin-25.0-riscv64-linux-gnu.tar.gz"},
82    "5708fc639cdfc27347cccfd50db9b73b53647b36fb5f3a4a93537cbe8828c27f": {"tag": "v25.0", "archive": "bitcoin-25.0-x86_64-apple-darwin.tar.gz"},
83    "33930d432593e49d58a9bff4c30078823e9af5d98594d2935862788ce8a20aec": {"tag": "v25.0", "archive": "bitcoin-25.0-x86_64-linux-gnu.tar.gz"},
84    "7154b35ecc8247589070ae739b7c73c4dee4794bea49eb18dc66faed65b819e7": {"tag": "v25.0", "archive": "bitcoin-25.0-win64.zip"},
85
86    "10581b6d4aeb091c08c9e69eb4e4ced000038c009286dc1edb8a876656ccf6df": {"tag": "v28.2", "archive": "bitcoin-28.2-aarch64-linux-gnu.tar.gz"},
87    "c08135c249688438b1ed6f48a2f192ffaae7c7111ffee5cfad59fdae355bb47f": {"tag": "v28.2", "archive": "bitcoin-28.2-arm-linux-gnueabihf.tar.gz"},
88    "c0270ed50effc174f7ff3332dba5183a8693999dac2ba78b37d8c8797b3ea2b2": {"tag": "v28.2", "archive": "bitcoin-28.2-arm64-apple-darwin.tar.gz"},
89    "21c54bc7520e873c8b52c817af257a8a4aee15d81bb2492e5bd51af055ae4469": {"tag": "v28.2", "archive": "bitcoin-28.2-powerpc64-linux-gnu.tar.gz"},
90    "8ad2bf5dd0a7fd04afa9cd2b1309621f8662f00c799f1971cd44ba9eb9d18ef1": {"tag": "v28.2", "archive": "bitcoin-28.2-riscv64-linux-gnu.tar.gz"},
91    "866a4b703a2095301151c17dcc753e19e4dba61ec68d19709ec4f81ff4320103": {"tag": "v28.2", "archive": "bitcoin-28.2-x86_64-apple-darwin.tar.gz"},
92    "98add5f220c01b387343b70edeb6273403fe081e22cd85fda132704cdcaa98aa": {"tag": "v28.2", "archive": "bitcoin-28.2-x86_64-linux-gnu.tar.gz"},
93    "da0869639c323bbf6f264f1829083b9514e10179b90c34b09d8cbcab8a1897e3": {"tag": "v28.2", "archive": "bitcoin-28.2-win64.zip"},
94
95    "73e76c14edc79808a0511c744d102ffbb494807ee90cbcba176568243254b532": {"tag": "v30.2", "archive": "bitcoin-30.2-aarch64-linux-gnu.tar.gz"},
96    "d510542842318ea34d87cb2c93d6a7fe091dcac2e8684460be2b3c44843fb502": {"tag": "v30.2", "archive": "bitcoin-30.2-arm-linux-gnueabihf.tar.gz"},
97    "c2ecab62891de22228043815cb6211549a32272be3d5d052ff19847d3420bd10": {"tag": "v30.2", "archive": "bitcoin-30.2-arm64-apple-darwin.tar.gz"},
98    "db8803f11f8259794864b8b0d2ef8a1a27d01a5943ff4f525bc26a325031fa87": {"tag": "v30.2", "archive": "bitcoin-30.2-powerpc64-linux-gnu.tar.gz"},
99    "b0302e4d9579d19a9a501f1278e5d2c56d33fd9583040f34802d8567a1f81ace": {"tag": "v30.2", "archive": "bitcoin-30.2-riscv64-linux-gnu.tar.gz"},
100    "99d5cee9b9c37be506396c30837a4b98e320bfea71c474d6120a7e8eb6075c7b": {"tag": "v30.2", "archive": "bitcoin-30.2-x86_64-apple-darwin.tar.gz"},
101    "6aa7bb4feb699c4c6262dd23e4004191f6df7f373b5d5978b5bcdd4bb72f75d8": {"tag": "v30.2", "archive": "bitcoin-30.2-x86_64-linux-gnu.tar.gz"},
102    "0d7e1f16f8823aa26d29b44855ff6dbac11c03d75631a6c1d2ea5fab3a84fdf8": {"tag": "v30.2", "archive": "bitcoin-30.2-win64.zip"},
103
104    "4de1d568dedd48604f75132421bc0abeca432639589b49a3909c81db3a813112": {"tag": "v31.0", "archive": "bitcoin-31.0-aarch64-linux-gnu.tar.gz"},
105    "8c19d007bfc73502625095ea4073af3a98ceb722d500556ab173bac5bcadd0d6": {"tag": "v31.0", "archive": "bitcoin-31.0-arm-linux-gnueabihf.tar.gz"},
106    "a2d7a13b4da53d4a3e4c517f3a0269e2429813417bb320d3b268993cfdc545d0": {"tag": "v31.0", "archive": "bitcoin-31.0-arm64-apple-darwin.tar.gz"},
107    "1d9c865aa0ccf675fc068e79d9fa57a5a70b59132fca38bb322a7d44ce2f0ff2": {"tag": "v31.0", "archive": "bitcoin-31.0-powerpc64-linux-gnu.tar.gz"},
108    "7ece4ea365bba9b2008b27f0717ef6a518598a572edaa2815e775faadc53c136": {"tag": "v31.0", "archive": "bitcoin-31.0-riscv64-linux-gnu.tar.gz"},
109    "56824dd705bc2a3b22d42e8aa02ed53498d491ff7c2c8aa96831333871887ead": {"tag": "v31.0", "archive": "bitcoin-31.0-x86_64-apple-darwin.tar.gz"},
110    "d3e4c58a35b1d0a97a457462c94f55501ad167c660c245cb1ffa565641c65074": {"tag": "v31.0", "archive": "bitcoin-31.0-x86_64-linux-gnu.tar.gz"},
111    "82fd2c504a0f20a31d4d13bd407783d6fc7bf17622d0ce85228a9b92694e03f0": {"tag": "v31.0", "archive": "bitcoin-31.0-win64.zip"},
112
113    "dcf1873f2208ba4f962f3398d47e154c39c0084be8f4553e05c940d0ace3d004": {"tag": "v31.1", "archive": "bitcoin-31.1-aarch64-linux-gnu.tar.gz"},
114    "66b2b45359efa161031a49898f96aa7cf1455db46ca6102acd16a7197dc3b96f": {"tag": "v31.1", "archive": "bitcoin-31.1-arm-linux-gnueabihf.tar.gz"},
115    "16a097c09fbd7eb78b240ce1dae123663ea2e5e377cfd6a951e71e227e23cf2f": {"tag": "v31.1", "archive": "bitcoin-31.1-arm64-apple-darwin.tar.gz"},
116    "f81dd017a551c5af7fa2d6fa67b885077a8353322a8019e8fd538366bae1eff7": {"tag": "v31.1", "archive": "bitcoin-31.1-powerpc64-linux-gnu.tar.gz"},
117    "8a9213348a111438472653b8bd46c12184c60cc35ce0c2af02b853de4297cf94": {"tag": "v31.1", "archive": "bitcoin-31.1-riscv64-linux-gnu.tar.gz"},
118    "bc506958d0f387c1ea770bdc7c7192a505fa645ff62cabcc7761fa7eb89e867e": {"tag": "v31.1", "archive": "bitcoin-31.1-x86_64-apple-darwin.tar.gz"},
119    "b80d9c3e04da78fb6f0569685673418cf686fadba9042d926d13fb87ff503f9e": {"tag": "v31.1", "archive": "bitcoin-31.1-x86_64-linux-gnu.tar.gz"},
120    "c99ef173471c58e6766d9eebd12e6c35349082eeed3939bc99eed58ef57db587": {"tag": "v31.1", "archive": "bitcoin-31.1-win64.zip"},
121}
122
123
124@contextlib.contextmanager
125def pushd(new_dir) -> None:
126    previous_dir = os.getcwd()
127    os.chdir(new_dir)
128    try:
129        yield
130    finally:
131        os.chdir(previous_dir)
132
133
134def download_binary(tag, args) -> int:
135    if Path(tag).is_dir():
136        if not args.remove_dir:
137            print('Using cached {}'.format(tag))
138            return 0
139        shutil.rmtree(tag)
140
141    bin_path = 'bin/bitcoin-core-{}'.format(tag[1:])
142
143    match = re.compile('v(.*)(rc[0-9]+)$').search(tag)
144    if match:
145        bin_path = 'bin/bitcoin-core-{}/test.{}'.format(
146            match.group(1), match.group(2))
147
148    host = args.host
149    if tag < "v23" and host in ["x86_64-apple-darwin", "arm64-apple-darwin"]:
150        host = "osx64"
151
152    archive_format = 'tar.gz'
153    if host == 'win64':
154        archive_format = 'zip'
155
156    archive = f'bitcoin-{tag[1:]}-{host}.{archive_format}'
157    archive_url = f'https://bitcoincore.org/{bin_path}/{archive}'
158
159    try:
160        download_from_url(archive_url, archive)
161    except Exception as e:
162        print(f"\nDownload failed: {e}", file=sys.stderr)
163        print("Retrying download after failure ...", file=sys.stderr)
164        time.sleep(12)
165        try:
166            download_from_url(archive_url, archive)
167        except Exception as e2:
168            print(f"\nDownload failed a second time: {e2}", file=sys.stderr)
169            return 1
170
171    hasher = hashlib.sha256()
172    with open(archive, "rb") as afile:
173        hasher.update(afile.read())
174    archiveHash = hasher.hexdigest()
175
176    if archiveHash not in SHA256_SUMS or SHA256_SUMS[archiveHash]['archive'] != archive:
177        if archive in [v['archive'] for v in SHA256_SUMS.values()]:
178            print(f"Checksum {archiveHash} did not match", file=sys.stderr)
179        else:
180            print("Checksum for given version doesn't exist", file=sys.stderr)
181        return 1
182
183    print("Checksum matched")
184    Path(tag).mkdir()
185
186    # Extract archive
187    if host == 'win64':
188        try:
189            with zipfile.ZipFile(archive, 'r') as zip:
190                zip.extractall(tag)
191            # Remove the top level directory to match tar's --strip-components=1
192            extracted_items = os.listdir(tag)
193            top_level_dir = os.path.join(tag, extracted_items[0])
194            # Move all files & subdirectories up one level
195            for item in os.listdir(top_level_dir):
196                shutil.move(os.path.join(top_level_dir, item), tag)
197            # Remove the now-empty top-level directory
198            os.rmdir(top_level_dir)
199        except Exception as e:
200            print(f"Zip extraction failed: {e}", file=sys.stderr)
201            return 1
202    else:
203        ret = subprocess.run([TAR, '-zxf', archive, '-C', tag,
204                              '--strip-components=1',
205                              'bitcoin-{tag}'.format(tag=tag[1:])]).returncode
206        if ret != 0:
207            print(f"Failed to extract the {tag} tarball", file=sys.stderr)
208            return ret
209
210    Path(archive).unlink()
211
212    if tag >= "v23" and tag < "v28.2" and args.host == "arm64-apple-darwin":
213        # Starting with v23 there are arm64 binaries for ARM (e.g. M1, M2) mac.
214        # Until v28.2 they had to be signed to run.
215        binary_path = f'{os.getcwd()}/{tag}/bin/'
216
217        for arm_binary in os.listdir(binary_path):
218            # Is it already signed?
219            ret = subprocess.run(
220                ['codesign', '-v', binary_path + arm_binary],
221                stderr=subprocess.DEVNULL,  # Suppress expected stderr output
222            ).returncode
223            if ret == 1:
224                # Have to self-sign the binary
225                ret = subprocess.run(
226                    ['codesign', '-s', '-', binary_path + arm_binary]
227                ).returncode
228                if ret != 0:
229                    print(f"Failed to self-sign {tag} {arm_binary} arm64 binary", file=sys.stderr)
230                    return 1
231
232                # Confirm success
233                ret = subprocess.run(
234                    ['codesign', '-v', binary_path + arm_binary]
235                ).returncode
236                if ret != 0:
237                    print(f"Failed to verify the self-signed {tag} {arm_binary} arm64 binary", file=sys.stderr)
238                    return 1
239
240    return 0
241
242
243def set_host(args) -> int:
244    if platform.system().lower() == 'windows':
245        if platform.machine() != 'AMD64':
246            print('Only 64bit Windows supported', file=sys.stderr)
247            return 1
248        args.host = 'win64'
249        return 0
250    host = os.environ.get('HOST', subprocess.check_output(