Inside Bitcoin

751            results_writer.writerow([test_result.name, test_result.status, str(test_result.time)])
752        results_writer.writerow(['ALL', ("Passed" if all_passed else "Failed"), str(total_runtime)])
753
754class TestHandler:
755    """
756    Trigger the test scripts passed in via the list.
757    """
758    def __init__(self, *, num_tests_parallel, tests_dir, tmpdir, test_list, flags, use_term_control):
759        assert num_tests_parallel >= 1
760        self.executor = futures.ThreadPoolExecutor(max_workers=num_tests_parallel)
761        self.num_jobs = num_tests_parallel
762        self.tests_dir = tests_dir
763        self.tmpdir = tmpdir
764        self.test_list = test_list
765        self.flags = flags
766        self.jobs = {}
767        self.use_term_control = use_term_control
768
769    def done(self):
770        return not (self.jobs or self.test_list)
771
772    def get_next(self):
773        while len(self.jobs) < self.num_jobs and self.test_list:
774            # Add tests
775            test = self.test_list.popleft()
776            portseed = len(self.test_list)
777            portseed_arg = ["--portseed={}".format(portseed)]
778            log_stdout = tempfile.SpooledTemporaryFile(max_size=2**16)
779            log_stderr = tempfile.SpooledTemporaryFile(max_size=2**16)
780            test_argv = test.split()
781            testdir = "{}/{}_{}".format(self.tmpdir, re.sub(".py$", "", test_argv[0]), portseed)
782            tmpdir_arg = ["--tmpdir={}".format(testdir)]
783
784            def proc_wait(task):
785                task[2].wait()
786                return task
787
788            task = [
789                test,
790                time.time(),
791                subprocess.Popen(
792                    [sys.executable, self.tests_dir + test_argv[0]] + test_argv[1:] + self.flags + portseed_arg + tmpdir_arg,
793                    text=True,
794                    stdout=log_stdout,
795                    stderr=log_stderr,
796                ),
797                testdir,
798                log_stdout,
799                log_stderr,
800            ]
801            fut = self.executor.submit(proc_wait, task)
802            self.jobs[fut] = test
803        assert self.jobs  # Must not be empty here
804
805        # Print remaining running jobs when all jobs have been started.
806        if not self.test_list:
807            print("Remaining jobs: [{}]".format(", ".join(sorted(self.jobs.values()))))
808
809        dot_count = 0
810        while True:
811            # Return all procs that have finished, if any. Otherwise sleep until there is one.
812            procs = futures.wait(self.jobs.keys(), timeout=.5, return_when=futures.FIRST_COMPLETED)
813            self.jobs = {fut: self.jobs[fut] for fut in procs.not_done}
814            ret = []
815            for job in procs.done:
816                (name, start_time, proc, testdir, log_out, log_err) = job.result()
817
818                log_out.seek(0), log_err.seek(0)
819                [stdout, stderr] = [log_file.read().decode('utf-8') for log_file in (log_out, log_err)]
820                log_out.close(), log_err.close()
821                skip_reason = None
822                if proc.returncode == TEST_EXIT_PASSED and stderr == "":
823                    status = "Passed"
824                elif proc.returncode == TEST_EXIT_SKIPPED:
825                    status = "Skipped"
826                    skip_reason = re.search(r"Test Skipped: (.*)", stdout).group(1).strip()
827                else:
828                    status = "Failed"
829
830                if self.use_term_control:
831                    clearline = '\r' + (' ' * dot_count) + '\r'
832                    print(clearline, end='', flush=True)
833                dot_count = 0
834                ret.append((TestResult(name, status, int(time.time() - start_time)), testdir, stdout, stderr, proc.returncode, skip_reason))
835            if ret:
836                return ret
837            if self.use_term_control:
838                print('.', end='', flush=True)
839            dot_count += 1
840
841
842class TestResult():
843    def __init__(self, name, status, time):
844        self.name = name
845        self.status = status
846        self.time = time
847        self.padding = 0
848
849    def sort_key(self):
850        if self.status == "Passed":
851            return 0, self.name.lower()
852        elif self.status == "Failed":
853            return 2, self.name.lower()
854        elif self.status == "Skipped":
855            return 1, self.name.lower()
856
857    def __repr__(self):
858        if self.status == "Passed":
859            color = GREEN
860            glyph = TICK
861        elif self.status == "Failed":
862            color = RED
863            glyph = CROSS
864        elif self.status == "Skipped":
865            color = DEFAULT
866            glyph = CIRCLE
867
868        return color[1] + "%s | %s%s | %s s\n" % (self.name.ljust(self.padding), glyph, self.status.ljust(7), self.time) + color[0]
869
870    @property
871    def was_successful(self):
872        return self.status != "Failed"
873
874
875def check_script_prefixes():
876    """Check that test scripts start with one of the allowed name prefixes."""
877
878    good_prefixes_re = re.compile("^(example|feature|interface|mempool|mining|p2p|rpc|wallet|tool)_")
879    bad_script_names = [script for script in ALL_SCRIPTS if good_prefixes_re.match(script) is None]
880
881    if bad_script_names:
882        print("%sERROR:%s %d tests not meeting naming conventions:" % (BOLD[1], BOLD[0], len(bad_script_names)))
883        print("  %s" % ("\n  ".join(sorted(bad_script_names))))
884        raise AssertionError("Some tests are not following naming convention!")
885
886
887def check_script_list(*, src_dir, fail_on_warn):
888    """Check scripts directory.
889
890    Check that all python files in this directory are categorized
891    as a test script or meta script."""
892    script_dir = src_dir + '/test/functional/'
893    python_files = set([test_file for test_file in os.listdir(script_dir) if test_file.endswith(".py")])
894    missed_tests = list(python_files - set(map(lambda x: x.split()[0], ALL_SCRIPTS + NON_SCRIPTS)))
895    if len(missed_tests) != 0:
896        print("%sWARNING!%s The following scripts are not being run: %s. Check the test lists in test_runner.py." % (BOLD[1], BOLD[0], str(missed_tests)))
897        if fail_on_warn:
898            sys.exit(1)
899
900
901class RPCCoverage():
902    """
903    Coverage reporting utilities for test_runner.
904
905    Coverage calculation works by having each test script subprocess write
906    coverage files into a particular directory. These files contain the RPC
907    commands invoked during testing, as well as a complete listing of RPC
908    commands per `bitcoin-cli help` (`rpc_interface.txt`).
909
910    After all tests complete, the commands run are combined and diff'd against
911    the complete list to calculate uncovered RPC commands.
912
913    See also: test/functional/test_framework/coverage.py
914
915    """
916    def __init__(self):
917        self.temp_dir = tempfile.TemporaryDirectory(prefix="coverage")
918        self.dir = self.temp_dir.name
919        self.flag = '--coveragedir=%s' % self.dir
920
921    def report_rpc_coverage(self):
922        """
923        Print out RPC commands that were unexercised by tests.
924
925        """
926        uncovered = self._get_uncovered_rpc_commands()
927
928        if uncovered:
929            print("Uncovered RPC commands:")
930            print("".join(("  - %s\n" % command) for command in sorted(uncovered)))
931            return False
932        else:
933            print("All RPC commands covered.")
934            return True
935
936    def _get_uncovered_rpc_commands(self):
937        """
938        Return a set of currently untested RPC commands.
939
940        """
941        # This is shared from `test/functional/test_framework/coverage.py`
942        reference_filename = 'rpc_interface.txt'
943        coverage_file_prefix = 'coverage.'
944
945        coverage_ref_filename = os.path.join(self.dir, reference_filename)
946        coverage_filenames = set()
947        all_cmds = set()
948        # Consider RPC generate covered, because it is overloaded in
949        # test_framework/test_node.py and not seen by the coverage check.
950        covered_cmds = set({'generate'})
951
952        if not os.path.isfile(coverage_ref_filename):
953            raise RuntimeError("No coverage reference found")
954
955        with open(coverage_ref_filename, 'r') as coverage_ref_file:
956            all_cmds.update([line.strip() for line in coverage_ref_file.readlines()])
957
958        for root, _, files in os.walk(self.dir):
959            for filename in files:
960                if filename.startswith(coverage_file_prefix):
961                    coverage_filenames.add(os.path.join(root, filename))
962
963        for filename in coverage_filenames:
964            with open(filename, 'r') as coverage_file:
965                covered_cmds.update([line.strip() for line in coverage_file.readlines()])
966
967        return all_cmds - covered_cmds
968
969
970if __name__ == '__main__':
971    main()
972