Inside Bitcoin

501        #   `test/functional/test_runner.py test/functional/wallet*`
502        #   `test/functional/test_runner.py ./test/functional/wallet*`
503        #   `test_runner.py wallet*`
504        #   but not:
505        #   `test/functional/test_runner.py wallet*`
506        # Multiple wildcards can be passed:
507        #   `test_runner.py tool* mempool*`
508        for test in tests:
509            script = test.split("/")[-1]
510            script = script + ".py" if ".py" not in script else script
511            matching_scripts = [s for s in ALL_SCRIPTS if s.startswith(script)]
512            if matching_scripts:
513                test_list += matching_scripts
514            else:
515                print("{}WARNING!{} Test '{}' not found in full test list.".format(BOLD[1], BOLD[0], test))
516    elif args.extended:
517        # Include extended tests
518        test_list += ALL_SCRIPTS
519    else:
520        # Run base tests only
521        test_list += BASE_SCRIPTS
522
523    # Remove the test cases that the user has explicitly asked to exclude.
524    # The user can specify a test case with or without the .py extension.
525    if args.exclude:
526
527        def print_warning_missing_test(test_name):
528            print("{}WARNING!{} Test '{}' not found in current test list. Check the --exclude options.".format(BOLD[1], BOLD[0], test_name))
529            if fail_on_warn:
530                sys.exit(1)
531
532        def remove_tests(exclude_list):
533            if not exclude_list:
534                print_warning_missing_test(exclude_test)
535            for exclude_item in exclude_list:
536                print("Excluding %s" % exclude_item)
537                test_list.remove(exclude_item)
538
539        for exclude_test in args.exclude:
540            if ',' in exclude_test:
541                print("{}WARNING!{} --exclude '{}' contains a comma. Use --exclude for each test.".format(BOLD[1], BOLD[0], exclude_test))
542                if fail_on_warn:
543                    sys.exit(1)
544
545        for exclude_test in args.exclude:
546            # A space in the name indicates it has arguments such as "rpc_bind.py --ipv4"
547            if ' ' in exclude_test:
548                remove_tests([test for test in test_list if test.replace('.py', '') == exclude_test.replace('.py', '')])
549            else:
550                # Exclude all variants of a test
551                remove_tests([test for test in test_list if test.split('.py')[0] == exclude_test.split('.py')[0]])
552
553    if config.getboolean("components", "BUILD_BENCH") and TOOL_BENCH_SANITY_CHECK in test_list:
554        # Remove it, and expand it for each bench in the list
555        test_list.remove(TOOL_BENCH_SANITY_CHECK)
556        bench_cmd = Binaries(get_binary_paths(config), bin_dir=None).bench_argv() + ["-list"]
557        bench_list = subprocess.check_output(bench_cmd, text=True).splitlines()
558        bench_list = [f"{TOOL_BENCH_SANITY_CHECK} --bench={b}" for b in bench_list]
559        # Start with special scripts (variable, unknown runtime)
560        test_list.extendleft(reversed(bench_list))
561
562    if args.filter:
563        test_list = deque(filter(re.compile(args.filter).search, test_list))
564
565    if not test_list:
566        print("No valid test scripts specified. Check that your test is in one "
567              "of the test lists in test_runner.py, or run test_runner.py with no arguments to run all tests")
568        sys.exit(1)
569
570    if args.help:
571        # Print help for test_runner.py, then print help of the first script (with args removed) and exit.
572        parser.print_help()
573        subprocess.check_call([sys.executable, os.path.join(config["environment"]["SRCDIR"], 'test', 'functional', test_list[0].split()[0]), '-h'])
574        sys.exit(0)
575
576    # Warn if there is not enough space on tmpdir to run the tests with --nocleanup
577    if args.nocleanup:
578        if shutil.disk_usage(tmpdir).free < MIN_NO_CLEANUP_SPACE:
579            print(f"{BOLD[1]}WARNING!{BOLD[0]} There may be insufficient free space in {tmpdir} to run the functional test suite with --nocleanup. "
580                  f"A minimum of {MIN_NO_CLEANUP_SPACE // (1024 * 1024 * 1024)} GB of free space is required.")
581        passon_args.append("--nocleanup")
582
583    check_script_list(src_dir=config["environment"]["SRCDIR"], fail_on_warn=fail_on_warn)
584    check_script_prefixes()
585
586    run_tests(
587        test_list=test_list,
588        build_dir=config["environment"]["BUILDDIR"],
589        tmpdir=tmpdir,
590        jobs=args.jobs,
591        enable_coverage=args.coverage,
592        args=passon_args,
593        combined_logs_len=args.combinedlogslen,
594        failfast=args.failfast,
595        use_term_control=args.ansi,
596        results_filepath=results_filepath,
597    )
598
599def run_tests(*, test_list, build_dir, tmpdir, jobs=1, enable_coverage=False, args=None, combined_logs_len=0, failfast=False, use_term_control, results_filepath=None):
600    args = args or []
601
602    # Some optional Python dependencies (e.g. pycapnp) may emit warnings or fail under
603    # CPython free-threaded builds when the GIL is disabled. Force it on for all
604    # functional tests so every child process inherits PYTHON_GIL=1.
605    os.environ["PYTHON_GIL"] = "1"
606
607    # Warn if bitcoind is already running
608    try:
609        # pgrep exits with code zero when one or more matching processes found
610        if subprocess.run(["pgrep", "-x", "bitcoind"], stdout=subprocess.DEVNULL).returncode == 0:
611            print("%sWARNING!%s There is already a bitcoind process running on this system. Tests may fail unexpectedly due to resource contention!" % (BOLD[1], BOLD[0]))
612    except OSError:
613        # pgrep not supported
614        pass
615
616    # Warn if there is not enough space on the testing dir
617    min_space = MIN_FREE_SPACE + (jobs - 1) * ADDITIONAL_SPACE_PER_JOB
618    if shutil.disk_usage(tmpdir).free < min_space:
619        print(f"{BOLD[1]}WARNING!{BOLD[0]} There may be insufficient free space in {tmpdir} to run the Bitcoin functional test suite. "
620              f"Running the test suite with fewer than {min_space // (1024 * 1024)} MB of free space might cause tests to fail.")
621
622    tests_dir = f"{build_dir}/test/functional/"
623    # This allows `test_runner.py` to work from an out-of-source build directory using a symlink,
624    # a hard link or a copy on any platform. See https://github.com/bitcoin/bitcoin/pull/27561.
625    sys.path.append(tests_dir)
626
627    cache_tmp_dir = tempfile.TemporaryDirectory(prefix="functional_test_cache")
628    flags = [f"--cachedir={cache_tmp_dir.name}"] + args
629
630    if enable_coverage:
631        coverage = RPCCoverage()
632        flags.append(coverage.flag)
633        logging.debug("Initializing coverage directory at %s" % coverage.dir)
634    else:
635        coverage = None
636
637    if len(test_list) > 1 and jobs > 1:
638        # Populate cache
639        try:
640            subprocess.check_output([sys.executable, tests_dir + 'create_cache.py'] + flags + ["--tmpdir=%s/cache" % tmpdir])
641        except subprocess.CalledProcessError as e:
642            sys.stdout.buffer.write(e.output)
643            raise
644
645    # Run Tests
646    job_queue = TestHandler(
647        num_tests_parallel=jobs,
648        tests_dir=tests_dir,
649        tmpdir=tmpdir,
650        test_list=test_list,
651        flags=flags,
652        use_term_control=use_term_control,
653    )
654    start_time = time.time()
655    test_results = []
656
657    max_len_name = len(max(test_list, key=len))
658    test_count = len(test_list)
659    all_passed = True
660    while not job_queue.done():
661        if failfast and not all_passed:
662            break
663        for test_result, testdir, stdout, stderr, exit_code, skip_reason in job_queue.get_next():
664            test_results.append(test_result)
665            done_str = f"{len(test_results)}/{test_count} - {BOLD[1]}{test_result.name}{BOLD[0]}"
666            if test_result.status == "Passed":
667                logging.debug("%s passed, Duration: %s s" % (done_str, test_result.time))
668            elif test_result.status == "Skipped":
669                logging.debug(f"{done_str} skipped ({skip_reason})")
670            else:
671                all_passed = False
672                print(f"{done_str} failed (exit code {exit_code}), Duration: {test_result.time} s\n")
673                print(BOLD[1] + 'stdout:\n' + BOLD[0] + stdout + '\n')
674                print(BOLD[1] + 'stderr:\n' + BOLD[0] + stderr + '\n')
675                if combined_logs_len and os.path.isdir(testdir):
676                    # Print the final `combinedlogslen` lines of the combined logs
677                    print('{}Combine the logs and print the last {} lines ...{}'.format(BOLD[1], combined_logs_len, BOLD[0]))
678                    print('\n============')
679                    print('{}Combined log for {}:{}'.format(BOLD[1], testdir, BOLD[0]))
680                    print('============\n')
681                    combined_logs_args = [sys.executable, os.path.join(tests_dir, 'combine_logs.py'), testdir]
682                    if BOLD[0]:
683                        combined_logs_args += ['--color']
684                    combined_logs, _ = subprocess.Popen(combined_logs_args, text=True, stdout=subprocess.PIPE).communicate()
685                    print("\n".join(deque(combined_logs.splitlines(), combined_logs_len)))
686
687                if failfast:
688                    logging.debug("Early exiting after test failure")
689                    break
690
691                if "[Errno 28] No space left on device" in stdout:
692                    sys.exit(f"Early exiting after test failure due to insufficient free space in {tmpdir}\n"
693                             f"Test execution data left in {tmpdir}.\n"
694                             f"Additional storage is needed to execute testing.")
695
696    runtime = int(time.time() - start_time)
697    print_results(test_results, max_len_name, runtime)
698    if results_filepath:
699        write_results(test_results, results_filepath, runtime)
700
701    if coverage:
702        coverage_passed = coverage.report_rpc_coverage()
703    else:
704        coverage_passed = True
705
706    # Clear up the temp directory if all subdirectories are gone
707    if not os.listdir(tmpdir):
708        os.rmdir(tmpdir)
709
710    all_passed = all_passed and coverage_passed
711
712    # Clean up dangling processes if any. This may only happen with --failfast option.
713    # Killing the process group will also terminate the current process but that is
714    # not an issue
715    if not os.getenv("CI_FAILFAST_TEST_LEAVE_DANGLING") and len(job_queue.jobs):
716        os.killpg(os.getpgid(0), signal.SIGKILL)
717
718    sys.exit(not all_passed)
719
720
721def print_results(test_results, max_len_name, runtime):
722    results = "\n" + BOLD[1] + "%s | %s | %s\n\n" % ("TEST".ljust(max_len_name), "STATUS   ", "DURATION") + BOLD[0]
723
724    test_results.sort(key=TestResult.sort_key)
725    all_passed = True
726    time_sum = 0
727
728    for test_result in test_results:
729        all_passed = all_passed and test_result.was_successful
730        time_sum += test_result.time
731        test_result.padding = max_len_name
732        results += str(test_result)
733
734    status = TICK + "Passed" if all_passed else CROSS + "Failed"
735    if not all_passed:
736        results += RED[1]
737    results += BOLD[1] + "\n%s | %s | %s s (accumulated) \n" % ("ALL".ljust(max_len_name), status.ljust(9), time_sum) + BOLD[0]
738    if not all_passed:
739        results += RED[0]
740    results += "Runtime: %s s\n" % (runtime)
741    print(results)
742
743
744def write_results(test_results, filepath, total_runtime):
745    with open(filepath, mode="w") as results_file:
746        results_writer = csv.writer(results_file)
747        results_writer.writerow(['test', 'status', 'duration(seconds)'])
748        all_passed = True
749        for test_result in test_results:
750            all_passed = all_passed and test_result.was_successful