Inside Bitcoin

1#!/usr/bin/env python3
2# Copyright (c) 2018-present The Bitcoin Core developers
3# Distributed under the MIT software license, see the accompanying
4# file COPYING or http://www.opensource.org/licenses/mit-license.php.
5"""Verify commits against a trusted keys list."""
6import argparse
7import hashlib
8import logging
9import os
10from pathlib import Path
11import subprocess
12import sys
13import time
14
15GIT = os.getenv('GIT', 'git')
16
17def is_ancestor(older, newer, root_name):
18    """Return whether older is an ancestor of newer, rejecting Git errors."""
19    result = subprocess.run([GIT, "merge-base", "--is-ancestor", older, newer])
20    if result.returncode not in (0, 1):
21        print(f'Failed to determine ancestry between "{older}" and "{newer}" for the {root_name} (git merge-base exited with {result.returncode}).', file=sys.stderr)
22        sys.exit(1)
23    return result.returncode == 0
24
25def predates(commit, root, root_name):
26    """Return whether commit is provably older than root, rejecting divergent history."""
27    if is_ancestor(root, commit, root_name):
28        return False
29    elif is_ancestor(commit, root, root_name):
30        return True
31    print(f'"{commit}" diverges from the {root_name} "{root}", refusing to verify.', file=sys.stderr)
32    sys.exit(1)
33
34def tree_sha512sum(commit='HEAD'):
35    """Calculate the Tree-sha512 for the commit.
36
37    This is copied from github-merge.py. See https://github.com/bitcoin-core/bitcoin-maintainer-tools."""
38
39    # request metadata for entire tree, recursively
40    files = []
41    blob_by_name = {}
42    for line in subprocess.check_output([GIT, 'ls-tree', '--full-tree', '-r', commit]).splitlines():
43        name_sep = line.index(b'\t')
44        metadata = line[:name_sep].split()  # perms, 'blob', blobid
45        assert metadata[1] == b'blob'
46        name = line[name_sep + 1:]
47        files.append(name)
48        blob_by_name[name] = metadata[2]
49
50    files.sort()
51    # open connection to git-cat-file in batch mode to request data for all blobs
52    # this is much faster than launching it per file
53    p = subprocess.Popen([GIT, 'cat-file', '--batch'], stdout=subprocess.PIPE, stdin=subprocess.PIPE)
54    overall = hashlib.sha512()
55    for f in files:
56        blob = blob_by_name[f]
57        # request blob
58        p.stdin.write(blob + b'\n')
59        p.stdin.flush()
60        # read header: blob, "blob", size
61        reply = p.stdout.readline().split()
62        assert reply[0] == blob and reply[1] == b'blob'
63        size = int(reply[2])
64        # hash the blob data
65        intern = hashlib.sha512()
66        ptr = 0
67        while ptr < size:
68            bs = min(65536, size - ptr)
69            piece = p.stdout.read(bs)
70            if len(piece) == bs:
71                intern.update(piece)
72            else:
73                raise IOError('Premature EOF reading git cat-file output')
74            ptr += bs
75        dig = intern.hexdigest()
76        assert p.stdout.read(1) == b'\n'  # ignore LF that follows blob data
77        # update overall hash with file hash
78        overall.update(dig.encode("utf-8"))
79        overall.update("  ".encode("utf-8"))
80        overall.update(f)
81        overall.update("\n".encode("utf-8"))
82    p.stdin.close()
83    if p.wait():
84        raise IOError('Non-zero return value executing git cat-file')
85    return overall.hexdigest()
86
87def main():
88
89    # Enable debug logging if running in CI
90    if 'CI' in os.environ and os.environ['CI'].lower() == "true":
91        logging.getLogger().setLevel(logging.DEBUG)
92
93    # Parse arguments
94    parser = argparse.ArgumentParser(usage='%(prog)s [options] [commit id]')
95    parser.add_argument('--disable-tree-check', action='store_false', dest='verify_tree', help='disable SHA-512 tree check')
96    parser.add_argument('--clean-merge', type=float, dest='clean_merge', default=float('inf'), help='Only check clean merge after <NUMBER> days ago (default: %(default)s)', metavar='NUMBER')
97    parser.add_argument('commit', nargs='?', default='HEAD', help='Check clean merge up to commit <commit>')
98    args = parser.parse_args()
99
100    # get directory of this program and read data files
101    dirname = Path(__file__).absolute().parent
102    print(f"Using verify-commits data from {dirname}")
103    verified_root = (dirname / "trusted-git-root").read_text().splitlines()[0]
104    verified_sha512_root = (dirname / "trusted-sha512-root-commit").read_text().splitlines()[0]
105    revsig_allowed = (dirname / "allow-revsig-commits").read_text().splitlines()
106    unclean_merge_allowed = (dirname / "allow-unclean-merge-commits").read_text().splitlines()
107    incorrect_sha512_allowed = (dirname / "allow-incorrect-sha512-commits").read_text().splitlines()
108    trusted_keys = (dirname / "trusted-keys").read_text().splitlines()
109
110    # Set commit and variables
111    current_commit = args.commit
112    if ' ' in current_commit:
113        print("Commit must not contain spaces", file=sys.stderr)
114        sys.exit(1)
115    verify_tree = args.verify_tree
116    no_sha1 = True
117    prev_commit = ""
118    initial_commit = current_commit
119
120    # Iterate through commits
121    while True:
122
123        # Log a message to prevent Travis from timing out
124        logging.debug("verify-commits: [in-progress] processing commit {}".format(current_commit[:8]))
125
126        if current_commit == verified_root:
127            # Ensure the trusted root identifies an existing commit.
128            is_ancestor(verified_root, current_commit, "trusted Git root")
129            print('There is a valid path from "{}" to {} where all commits are signed!'.format(initial_commit, verified_root))
130            sys.exit(0)
131        elif predates(current_commit, verified_root, "trusted Git root"):
132            print(f"\"{current_commit}\" predates the trusted root, stopping!")
133            sys.exit(0)
134
135        if verify_tree:
136            if current_commit == verified_sha512_root:
137                print("All Tree-SHA512s matched up to {}".format(verified_sha512_root), file=sys.stderr)
138                verify_tree = False
139                no_sha1 = False
140            elif predates(current_commit, verified_sha512_root, "trusted Tree-SHA512 root"):
141                print(f"\"{current_commit}\" predates the trusted SHA512 root, disabling tree verification.")
142                verify_tree = False
143                no_sha1 = False
144
145
146        os.environ['BITCOIN_VERIFY_COMMITS_ALLOW_SHA1'] = "0" if no_sha1 else "1"
147        allow_revsig = current_commit in revsig_allowed
148
149        # Check that the commit (and parents) was signed with a trusted key
150        valid_sig = False
151        verify_res = subprocess.run([GIT, '-c', 'gpg.program={}/gpg.sh'.format(dirname), 'verify-commit', "--raw", current_commit], text=True, capture_output=True)
152        for line in verify_res.stderr.splitlines():
153            if line.startswith("[GNUPG:] VALIDSIG "):
154                key = line.split(" ")[-1]
155                valid_sig = key in trusted_keys
156            elif (line.startswith("[GNUPG:] REVKEYSIG ") or line.startswith("[GNUPG:] EXPKEYSIG ")) and not allow_revsig:
157                valid_sig = False
158                break
159        if not valid_sig:
160            if prev_commit != "":
161                print("No parent of {} was signed with a trusted key!".format(prev_commit), file=sys.stderr)
162                print("Parents are:", file=sys.stderr)
163                parents = subprocess.check_output([GIT, 'show', '-s', '--format=format:%P', prev_commit], text=True).splitlines()[0].split(' ')
164                for parent in parents:
165                    subprocess.call([GIT, 'show', '-s', parent], stdout=sys.stderr)
166            else:
167                print("{} was not signed with a trusted key!".format(current_commit), file=sys.stderr)
168            sys.exit(1)
169
170        # Check the Tree-SHA512
171        if (verify_tree or prev_commit == "") and current_commit not in incorrect_sha512_allowed:
172            tree_hash = tree_sha512sum(current_commit)
173            if ("Tree-SHA512: {}".format(tree_hash)) not in subprocess.check_output([GIT, 'show', '-s', '--format=format:%B', current_commit], text=True).splitlines():
174                print("Tree-SHA512 did not match for commit " + current_commit, file=sys.stderr)
175                sys.exit(1)
176
177        # Merge commits should only have two parents
178        parents = subprocess.check_output([GIT, 'show', '-s', '--format=format:%P', current_commit], text=True).splitlines()[0].split(' ')
179        if len(parents) > 2:
180            print("Commit {} is an octopus merge".format(current_commit), file=sys.stderr)
181            sys.exit(1)
182
183        # Check that the merge commit is clean
184        commit_time = int(subprocess.check_output([GIT, 'show', '-s', '--format=format:%ct', current_commit], text=True).splitlines()[0])
185        check_merge = commit_time > time.time() - args.clean_merge * 24 * 60 * 60  # Only check commits in clean_merge days
186        allow_unclean = current_commit in unclean_merge_allowed
187        if len(parents) == 2 and check_merge and not allow_unclean:
188            current_tree = subprocess.check_output([GIT, 'show', '--format=%T', current_commit], text=True).splitlines()[0]
189
190            # This merge-tree functionality requires git >= 2.38. The
191            # --write-tree option was added in order to opt-in to the new
192            # behavior. Older versions of git will not recognize the option and
193            # will instead exit with code 128.
194            try:
195                recreated_tree = subprocess.check_output([GIT, "merge-tree", "--write-tree", parents[0], parents[1]], text=True).splitlines()[0]
196            except subprocess.CalledProcessError as e:
197                if e.returncode == 128:
198                    print("git v2.38+ is required for this functionality.", file=sys.stderr)
199                    sys.exit(1)
200                else:
201                    raise e
202
203            if current_tree != recreated_tree:
204                print("Merge commit {} is not clean".format(current_commit), file=sys.stderr)
205                subprocess.call([GIT, 'diff', recreated_tree, current_tree])
206                sys.exit(1)
207
208        prev_commit = current_commit
209        current_commit = parents[0]
210
211if __name__ == '__main__':
212    main()
213