Inside Bitcoin

1#!/usr/bin/env python3
2# Copyright (c) The Bitcoin Core developers
3# Distributed under the MIT software license, see the accompanying
4# file COPYING or https://opensource.org/license/mit/.
5
6import ipaddress
7from pathlib import Path
8import os
9import shlex
10import subprocess
11import sys
12import time
13
14
15def run(cmd, **kwargs):
16    print("+ " + shlex.join(cmd), flush=True)
17    kwargs.setdefault("check", True)
18    try:
19        return subprocess.run(cmd, **kwargs)
20    except Exception as e:
21        sys.exit(str(e))
22
23
24def main():
25    print("Export only allowed settings:")
26    settings = run(
27        ["bash", "-c", "grep export ./ci/test/00_setup_env*.sh"],
28        stdout=subprocess.PIPE,
29        text=True,
30    ).stdout.splitlines()
31    settings = set(l.split("=")[0].split("export ")[1] for l in settings)
32    # Add "hidden" settings, which are never exported, manually. Otherwise,
33    # they will not be passed on.
34    settings.update([
35        "BASE_BUILD_DIR",
36        "BIND_TEST_ROUTABLE_IPV4",
37        "BIND_TEST_ROUTABLE_IPV6",
38        "CI_FAILFAST_TEST_LEAVE_DANGLING",
39    ])
40
41    ci_idx = sorted(path.name for path in Path(__file__).parent.glob("00_setup_env_*.sh")).index(Path(os.environ["FILE_ENV"]).name)
42    subnet_id = ci_idx % 256
43    ip4_subnet = ipaddress.IPv4Network(f"11.11.{subnet_id}.0/24")
44    ip4_addr = ip4_subnet[5]
45    ip6_subnet = ipaddress.IPv6Network(f"1111:1111:{subnet_id:x}::/112")
46    ip6_addr = ip6_subnet[5]
47
48    if not os.getenv("DANGER_RUN_CI_ON_HOST"):
49        os.environ["BIND_TEST_ROUTABLE_IPV4"] = str(ip4_addr)
50        os.environ["BIND_TEST_ROUTABLE_IPV6"] = str(ip6_addr)
51
52    # Append $USER to /tmp/env to support multi-user systems and $CONTAINER_NAME
53    # to allow support starting multiple runs simultaneously by the same user.
54    env_file = "/tmp/env-{u}-{c}".format(
55        u=os.environ["USER"],
56        c=os.environ["CONTAINER_NAME"],
57    )
58    with open(env_file, "w") as file:
59        for k, v in os.environ.items():
60            if k in settings:
61                file.write(f"{k}={v}\n")
62    run(["cat", env_file])
63
64    if os.getenv("DANGER_RUN_CI_ON_HOST"):
65        print("Running on host system without docker wrapper")
66        print("Create missing folders")
67        for create_dir in [
68                os.environ["CCACHE_DIR"],
69                os.environ["PREVIOUS_RELEASES_DIR"],
70        ]:
71            Path(create_dir).mkdir(parents=True, exist_ok=True)
72
73        # Modify PATH to prepend the retry script, needed for CI_RETRY_EXE
74        os.environ["PATH"] = f"{os.environ['BASE_ROOT_DIR']}/ci/retry:{os.environ['PATH']}"
75    else:
76        CI_IMAGE_LABEL = "bitcoin-ci-test"
77
78        # Use buildx unconditionally
79        # Using buildx is required to properly load the correct driver, for use with registry caching. Neither build, nor BUILDKIT=1 currently do this properly
80        cmd_build = ["docker", "buildx", "build"]
81        cmd_build += [
82            f"--file={os.environ['BASE_READ_ONLY_DIR']}/ci/test_imagefile",
83            f"--build-arg=CI_IMAGE_NAME_TAG={os.environ['CI_IMAGE_NAME_TAG']}",
84            f"--build-arg=FILE_ENV={os.environ['FILE_ENV']}",
85            f"--build-arg=BASE_ROOT_DIR={os.environ['BASE_ROOT_DIR']}",
86            f"--platform={os.environ['CI_IMAGE_PLATFORM']}",
87            f"--label={CI_IMAGE_LABEL}",
88            f"--tag={os.environ['CONTAINER_NAME']}",
89        ]
90        cmd_build += shlex.split(os.getenv("DOCKER_BUILD_CACHE_ARG", ""))
91        cmd_build += [os.environ["BASE_READ_ONLY_DIR"]]
92
93        print(f"Building {os.environ['CONTAINER_NAME']} image tag to run in")
94        if run(cmd_build, check=False).returncode != 0:
95            print(f"Retry building {os.environ['CONTAINER_NAME']} image tag after failure")
96            time.sleep(3)
97            run(cmd_build)
98
99        for suffix in ["ccache", "depends", "depends_sources", "previous_releases"]:
100            run(["docker", "volume", "create", f"{os.environ['CONTAINER_NAME']}_{suffix}"], check=False)
101
102        CI_CCACHE_MOUNT = f"type=volume,src={os.environ['CONTAINER_NAME']}_ccache,dst={os.environ['CCACHE_DIR']}"
103        CI_DEPENDS_MOUNT = f"type=volume,src={os.environ['CONTAINER_NAME']}_depends,dst={os.environ['DEPENDS_DIR']}/built"
104        CI_DEPENDS_SOURCES_MOUNT = f"type=volume,src={os.environ['CONTAINER_NAME']}_depends_sources,dst={os.environ['DEPENDS_DIR']}/sources"
105        CI_PREVIOUS_RELEASES_MOUNT = f"type=volume,src={os.environ['CONTAINER_NAME']}_previous_releases,dst={os.environ['PREVIOUS_RELEASES_DIR']}"
106        CI_BUILD_MOUNT = []
107
108        if os.getenv("DANGER_CI_ON_HOST_FOLDERS"):
109            # ensure the directories exist
110            for create_dir in [
111                    os.environ["CCACHE_DIR"],
112                    f"{os.environ['DEPENDS_DIR']}/built",
113                    f"{os.environ['DEPENDS_DIR']}/sources",
114                    os.environ["PREVIOUS_RELEASES_DIR"],
115                    os.environ["BASE_BUILD_DIR"],  # Unset by default, must be defined externally
116            ]:
117                Path(create_dir).mkdir(parents=True, exist_ok=True)
118
119            CI_CCACHE_MOUNT = f"type=bind,src={os.environ['CCACHE_DIR']},dst={os.environ['CCACHE_DIR']}"
120            CI_DEPENDS_MOUNT = f"type=bind,src={os.environ['DEPENDS_DIR']}/built,dst={os.environ['DEPENDS_DIR']}/built"
121            CI_DEPENDS_SOURCES_MOUNT = f"type=bind,src={os.environ['DEPENDS_DIR']}/sources,dst={os.environ['DEPENDS_DIR']}/sources"
122            CI_PREVIOUS_RELEASES_MOUNT = f"type=bind,src={os.environ['PREVIOUS_RELEASES_DIR']},dst={os.environ['PREVIOUS_RELEASES_DIR']}"
123            CI_BUILD_MOUNT = [f"--mount=type=bind,src={os.environ['BASE_BUILD_DIR']},dst={os.environ['BASE_BUILD_DIR']}"]
124
125        if os.getenv("DANGER_CI_ON_HOST_CCACHE_FOLDER"):
126            if not os.path.isdir(os.environ["CCACHE_DIR"]):
127                print(f"Error: Directory '{os.environ['CCACHE_DIR']}' must be created in advance.")
128                sys.exit(1)
129            CI_CCACHE_MOUNT = f"type=bind,src={os.environ['CCACHE_DIR']},dst={os.environ['CCACHE_DIR']}"
130
131        network_name = f"{os.environ['CONTAINER_NAME']}-net"
132        run(["docker", "network", "create", "--ipv6", f"--subnet={ip6_subnet}", f"--subnet={ip4_subnet}", network_name])
133
134        if os.getenv("RESTART_CI_DOCKER_BEFORE_RUN"):
135            print("Restart docker before run to stop and clear all containers started with --rm")
136            run(["podman", "container", "rm", "--force", "--all"])  # Similar to "systemctl restart docker"
137
138            # Still prune everything in case the filtered pruning doesn't work, or if labels were not set
139            # on a previous run. Belt and suspenders approach, should be fine to remove in the future.
140            # Prune images used by --external containers (e.g. build containers) when
141            # using podman.
142            print("Prune all dangling images")
143            run(["podman", "image", "prune", "--force", "--external"])
144
145        print(f"Prune all dangling {CI_IMAGE_LABEL} images")
146        # When detecting podman-docker, `--external` should be added.
147        run(["docker", "image", "prune", "--force", "--filter", f"label={CI_IMAGE_LABEL}"])
148
149        cmd_run = ["docker", "run", "--rm", "--interactive", "--detach", "--tty"]
150        cmd_run += [
151            "--cap-add=LINUX_IMMUTABLE",
152            *shlex.split(os.getenv("CI_CONTAINER_CAP", "")),
153            f"--mount=type=bind,src={os.environ['BASE_READ_ONLY_DIR']},dst={os.environ['BASE_READ_ONLY_DIR']},readonly",
154            f"--mount={CI_CCACHE_MOUNT}",
155            f"--mount={CI_DEPENDS_MOUNT}",
156            f"--mount={CI_DEPENDS_SOURCES_MOUNT}",
157            f"--mount={CI_PREVIOUS_RELEASES_MOUNT}",
158            *CI_BUILD_MOUNT,
159            f"--env-file={env_file}",
160            f"--name={os.environ['CONTAINER_NAME']}",
161            f"--network={network_name}",
162            f"--ip={ip4_addr}",
163            f"--ip6={ip6_addr}",
164            f"--platform={os.environ['CI_IMAGE_PLATFORM']}",
165            os.environ["CONTAINER_NAME"],
166        ]
167
168        container_id = run(
169            cmd_run,
170            stdout=subprocess.PIPE,
171            text=True,
172        ).stdout.strip()
173
174    def ci_exec(cmd_inner, **kwargs):
175        if os.getenv("DANGER_RUN_CI_ON_HOST"):
176            prefix = []
177        else:
178            prefix = [
179                "docker",
180                "exec",
181                "--env",
182                "DANGER_RUN_CI_ON_HOST=1",  # Safe to set *inside* the container
183                container_id,
184            ]
185
186        return run([*prefix, *cmd_inner], **kwargs)
187
188    # Normalize all folders to BASE_ROOT_DIR
189    ci_exec([
190        "rsync",
191        "--recursive",
192        "--perms",
193        "--stats",
194        "--human-readable",
195        f"{os.environ['BASE_READ_ONLY_DIR']}/",
196        f"{os.environ['BASE_ROOT_DIR']}",
197    ])
198    ci_exec([f"{os.environ['BASE_ROOT_DIR']}/ci/test/01_base_install.sh"])
199    test_script = f"{os.environ['BASE_ROOT_DIR']}/ci/test/03_test_script.sh"
200    if os.environ.get("HOST", "").startswith("x86_64-w64-mingw32"):
201        ci_exec([
202            "env",
203            "NIX_BUILD_SHELL=bash",
204            "nix-shell",
205            f"{os.environ['BASE_ROOT_DIR']}/contrib/devtools/shell-win64-cross.nix",
206            "--run",
207            f"echo 'Windows cross compiler:' && \"$CXX\" -v && exec bash {shlex.quote(test_script)}",
208        ])
209    else:
210        ci_exec([test_script])
211
212    if not os.getenv("DANGER_RUN_CI_ON_HOST"):
213        print("Stop and remove CI container by ID")
214        run(["docker", "container", "kill", container_id])
215        run(["docker", "network", "rm", network_name])
216
217
218if __name__ == "__main__":
219    main()
220