1#!/usr/bin/env bash
2#
3# Copyright (c) 2018-present The Bitcoin Core developers
4# Distributed under the MIT software license, see the accompanying
5# file COPYING or http://www.opensource.org/licenses/mit-license.php.
6
7export LC_ALL=C.UTF-8
8
9set -o errexit -o xtrace -o pipefail
10
11if [ "${DANGER_RUN_CI_ON_HOST}" != "1" ]; then
12  echo "This script will make unsafe local and global modifications, so it can only be run inside a container and requires DANGER_RUN_CI_ON_HOST=1"
13  exit 1
14fi
15
16cd "${BASE_ROOT_DIR}"
17
18export PATH="/path_with space:${PATH}"
19export ASAN_OPTIONS="detect_leaks=1:detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1"
20export LSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/lsan:print_suppressions=0"
21export TSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/tsan:halt_on_error=1:second_deadlock_stack=1"
22export UBSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/ubsan:print_stacktrace=1:halt_on_error=1:report_error_type=1"
23
24echo "Number of available processing units: $(nproc)"
25if [ "$CI_OS_NAME" == "macos" ]; then
26  top -l 1 -s 0 | awk ' /PhysMem/ {print}'
27else
28  free -m -h
29  echo "System info: $(uname --kernel-name --kernel-release)"
30  lscpu
31fi
32echo "Free disk space:"
33df -h
34
35# We force an install of linux-headers again here via $PACKAGES to fix any
36# kernel mismatch between a cached docker image and the underlying host.
37# This can happen occasionally on hosted runners if the runner image is updated.
38if [[ "$CONTAINER_NAME" == "ci_native_asan" ]]; then
39  $CI_RETRY_EXE apt-get update
40  ${CI_RETRY_EXE} bash -c "apt-get install --no-install-recommends --no-upgrade -y $PACKAGES"
41fi
42
43# What host to compile for. See also ./depends/README.md
44# Tests that need cross-compilation export the appropriate HOST.
45# Tests that run natively guess the host
46export HOST=${HOST:-$("$BASE_ROOT_DIR/depends/config.guess")}
47
48echo "=== BEGIN env ==="
49env
50echo "=== END env ==="
51
52# The CI framework should be flexible where it is run from. For example, from
53# a git-archive, a git-worktree, or a normal git repo.
54# The iwyu task requires a working git repo, which may not always be
55# available, so initialize one with force.
56if [[ "${RUN_IWYU}" == true ]]; then
57  mv .git .git_ci_backup || true
58  git init
59  git add ./src  # the git diff command used later for iwyu only cares about ./src
60  git config user.email "ci@ci"
61  git config user.name "CI"
62  git commit -m "dummy CI ./src init for IWYU"
63fi
64
65if [ "$RUN_FUZZ_TESTS" = "true" ]; then
66  export DIR_FUZZ_IN=${DIR_QA_ASSETS}/fuzz_corpora/
67  if [ ! -d "$DIR_FUZZ_IN" ]; then
68    ${CI_RETRY_EXE} git clone --depth=1 https://github.com/bitcoin-core/qa-assets "${DIR_QA_ASSETS}"
69  fi
70  (
71    cd "${DIR_QA_ASSETS}"
72    echo "Using qa-assets repo from commit ..."
73    git log -1
74  )
75elif [ "$RUN_UNIT_TESTS" = "true" ]; then
76  export DIR_UNIT_TEST_DATA=${DIR_QA_ASSETS}/unit_test_data/
77  if [ ! -d "$DIR_UNIT_TEST_DATA" ]; then
78    mkdir -p "$DIR_UNIT_TEST_DATA"
79    ${CI_RETRY_EXE} curl --location --fail https://github.com/bitcoin-core/qa-assets/raw/main/unit_test_data/script_assets_test.json -o "${DIR_UNIT_TEST_DATA}/script_assets_test.json"
80  fi
81fi
82
83# Make sure default datadir does not exist and is never read by creating a dummy file
84if [ "$CI_OS_NAME" == "macos" ]; then
85  echo > "${HOME}/Library/Application Support/Bitcoin"
86else
87  echo > "${HOME}/.bitcoin"
88fi
89
90if [ -z "$NO_DEPENDS" ]; then
91  if [[ $CI_IMAGE_NAME_TAG == *alpine* ]]; then
92    SHELL_OPTS="CONFIG_SHELL=/usr/bin/dash"
93  else
94    SHELL_OPTS="CONFIG_SHELL="
95  fi
96  bash -c "$SHELL_OPTS make $MAKEJOBS -C depends HOST=$HOST $DEP_OPTS LOG=1"
97fi
98if [ "$DOWNLOAD_PREVIOUS_RELEASES" = "true" ]; then
99  test/get_previous_releases.py
100fi
101
102BITCOIN_CONFIG_ALL="-DCMAKE_COMPILE_WARNING_AS_ERROR=ON -DBUILD_BENCH=ON -DBUILD_FUZZ_BINARY=ON"
103if [ -z "$NO_DEPENDS" ]; then
104  BITCOIN_CONFIG_ALL="${BITCOIN_CONFIG_ALL} -DCMAKE_TOOLCHAIN_FILE=$DEPENDS_DIR/$HOST/toolchain.cmake"
105fi
106
107ccache --zero-stats
108
109# Folder where the build is done.
110BASE_BUILD_DIR=${BASE_BUILD_DIR:-$BASE_SCRATCH_DIR/build-$HOST}
111
112BITCOIN_CONFIG_ALL="$BITCOIN_CONFIG_ALL '-DCMAKE_INSTALL_PREFIX=$BASE_OUTDIR' -Werror=dev"
113
114if [[ "${RUN_IWYU}" == true || "${RUN_TIDY}" == true ]]; then
115  BITCOIN_CONFIG_ALL="$BITCOIN_CONFIG_ALL -DCMAKE_EXPORT_COMPILE_COMMANDS=ON"
116fi
117
118eval "CMAKE_ARGS=($BITCOIN_CONFIG_ALL $BITCOIN_CONFIG)"
119cmake -S "$BASE_ROOT_DIR" -B "$BASE_BUILD_DIR" "${CMAKE_ARGS[@]}" || (
120  cd "${BASE_BUILD_DIR}"
121  # shellcheck disable=SC2046
122  cat $(cmake -P "${BASE_ROOT_DIR}/ci/test/GetCMakeLogFiles.cmake")
123  false
124)
125
126BUILD_TARGETS="${GOAL}"
127if [[ "${GOAL}" != all && "${GOAL}" != *codegen* ]]; then
128  BUILD_TARGETS="all ${GOAL}"
129fi
130
131# shellcheck disable=SC2086
132cmake --build "${BASE_BUILD_DIR}" "$MAKEJOBS" --target $BUILD_TARGETS || (
133  echo "Build failure. Verbose build follows."
134  # shellcheck disable=SC2086
135  cmake --build "${BASE_BUILD_DIR}" -j1 --target $BUILD_TARGETS --verbose
136  false
137)
138
139ccache --version | head -n 1 && ccache --show-stats --verbose
140ccache --print-stats | python3 -c '
141import os
142import sys
143
144for line in sys.stdin:
145    key, value = line.split("\t", 1)
146    # "primary storage" fallback only needed for ccache version 4.5.1
147    if key in ("local_storage_hit", "primary_storage_hit"):
148        hits = int(value)
149    elif key in ("local_storage_miss", "primary_storage_miss"):
150        miss = int(value)
151
152calls = hits + miss
153# codegen has no calls, so skip that here
154if calls:
155    rate = (hits / calls * 100)
156    print(f"{rate:.2f}")
157    if rate < 75:
158        container = os.environ["CONTAINER_NAME"]
159        print(
160            "::notice title=low ccache hitrate::"
161            f"Ccache hit-rate in {container} was {rate:.2f}%"
162        )
163'
164du -sh "${DEPENDS_DIR}"/*/
165du -sh "${PREVIOUS_RELEASES_DIR}"
166
167if [ -n "${CI_LIMIT_STACK_SIZE}" ]; then
168  ulimit -s 512
169fi
170
171if [[ ${BARE_METAL_RISCV} == "true" ]]; then
172    export BASE_BUILD_DIR
173    "${BASE_ROOT_DIR}/ci/test/link-riscv.sh"
174fi
175
176if [ -n "$USE_VALGRIND" ]; then
177  "${BASE_ROOT_DIR}/ci/test/wrap-valgrind.py"
178fi
179
180if [ "$RUN_CHECK_DEPS" = "true" ]; then
181  "${BASE_ROOT_DIR}/contrib/devtools/check-deps.sh" "${BASE_BUILD_DIR}"
182fi
183
184if [[ "$CI_OS_NAME" == "macos" && "${GOAL}" = "install deploy" ]]; then
185  unzip "${BASE_BUILD_DIR}/bitcoin-macos-app.zip" -d "${BASE_BUILD_DIR}/deploy"
186  if ! ( codesign --verify --deep --strict "${BASE_BUILD_DIR}/deploy/Bitcoin-Qt.app" ); then
187    echo "Codesigning failed."
188    false
189  fi
190fi
191
192if [ "$RUN_UNIT_TESTS" = "true" ]; then
193  DIR_UNIT_TEST_DATA="${DIR_UNIT_TEST_DATA}" \
194  LD_LIBRARY_PATH="${DEPENDS_DIR}/${HOST}/lib" \
195  CTEST_OUTPUT_ON_FAILURE=ON \
196  ctest --test-dir "${BASE_BUILD_DIR}" \
197    --stop-on-failure \
198    "${MAKEJOBS}" \
199    --timeout $(( TEST_RUNNER_TIMEOUT_FACTOR * 60 ))
200fi
201
202if [ "$RUN_FUNCTIONAL_TESTS" = "true" ]; then
203  # parses TEST_RUNNER_EXTRA as an array which allows for multiple arguments such as TEST_RUNNER_EXTRA='--exclude "rpc_bind.py --ipv6"'
204  eval "TEST_RUNNER_EXTRA=($TEST_RUNNER_EXTRA)"
205  LD_LIBRARY_PATH="${DEPENDS_DIR}/${HOST}/lib" \
206  "${BASE_BUILD_DIR}/test/functional/test_runner.py" \
207    "${MAKEJOBS}" \
208    --tmpdirprefix "${BASE_SCRATCH_DIR}/test_runner/" \
209    --ansi \
210    --combinedlogslen=99999999 \
211    --timeout-factor="${TEST_RUNNER_TIMEOUT_FACTOR}" \
212    "${TEST_RUNNER_EXTRA[@]}" \
213    --quiet \
214    --failfast
215fi
216
217if [ "${RUN_TIDY}" = "true" ]; then
218  cmake -B /tidy-build -DLLVM_DIR=/usr/lib/llvm-"${TIDY_LLVM_V}"/cmake -DCMAKE_BUILD_TYPE=Release -S "${BASE_ROOT_DIR}"/contrib/devtools/bitcoin-tidy
219  cmake --build /tidy-build "$MAKEJOBS"
220  cmake --build /tidy-build --target bitcoin-tidy-tests "$MAKEJOBS"
221
222  set -eo pipefail
223  # Filter out:
224  # * qt qrc and moc generated files
225  jq 'map(select(.file | test("src/qt/.*_autogen/.*\\.cpp$") | not))' "${BASE_BUILD_DIR}/compile_commands.json" > tmp.json
226  mv tmp.json "${BASE_BUILD_DIR}/compile_commands.json"
227
228  cd "${BASE_BUILD_DIR}/src/"
229  if ! ( run-clang-tidy-"${TIDY_LLVM_V}" -config-file="${BASE_ROOT_DIR}/src/.clang-tidy" -quiet -load="/tidy-build/libbitcoin-tidy.so" "${MAKEJOBS}" | tee tmp.tidy-out.txt ); then
230    grep -C5 "error: " tmp.tidy-out.txt
231    echo "^^^ ⚠️ Failure generated from clang-tidy"
232    false
233  fi
234fi
235
236if [[ "${RUN_IWYU}" == true ]]; then
237  # TODO: Consider enforcing IWYU across the entire codebase.
238  FILES_WITH_ENFORCED_IWYU='/src/((bench|common|consensus|crypto|index|init|kernel|primitives|rpc|script|univalue/(lib|test)|util|zmq)/.*|node/(blockstorage|interfaces|miner|mining_args|utxo_snapshot)|test/fuzz/(kitchen_sink|minisketch|parse_univalue)|clientversion|core_io|rest|signet|init)\.cpp'
239  jq --arg patterns "$FILES_WITH_ENFORCED_IWYU" 'map(select(.file | test($patterns)))' "${BASE_BUILD_DIR}/compile_commands.json" > "${BASE_BUILD_DIR}/compile_commands_iwyu_errors.json"
240  jq --arg patterns "$FILES_WITH_ENFORCED_IWYU" 'map(select(.file | test($patterns) | not))' "${BASE_BUILD_DIR}/compile_commands.json" > "${BASE_BUILD_DIR}/compile_commands_iwyu_warnings.json"
241
242  cd "${BASE_ROOT_DIR}"
243
244  run_iwyu() {
245    mv "${BASE_BUILD_DIR}/$1" "${BASE_BUILD_DIR}/compile_commands.json"
246    {
247      python3 /include-what-you-use/mapgen/iwyu-mapgen-clang-intrin.py --lang imp "$("clang-${IWYU_LLVM_V}" -print-resource-dir)/include" > "${BASE_BUILD_DIR}/clang.intrinsics.imp"
248      python3 /include-what-you-use/iwyu_tool.py \
249             -p "${BASE_BUILD_DIR}" "${MAKEJOBS}" -- \
250             -Xiwyu --cxx17ns \