Inside Bitcoin

1// Copyright (c) 2015-present The Bitcoin Core developers
2// Distributed under the MIT software license, see the accompanying
3// file COPYING or http://www.opensource.org/licenses/mit-license.php.
4
5#include <bench/bench.h>
6
7#include <test/util/setup_common.h> // IWYU pragma: keep
8#include <util/check.h>
9#include <util/fs.h>
10#include <util/time.h>
11
12#include <compare>
13#include <fstream>
14#include <functional>
15#include <iostream>
16#include <regex>
17#include <string>
18#include <string_view>
19#include <utility>
20#include <vector>
21
22/**
23 * Retrieves the available test setup command line arguments that may be used
24 * in the benchmark. They will be used only if the benchmark utilizes a
25 * 'BasicTestingSetup' or any child of it.
26 */
27static std::function<std::vector<const char*>()> g_bench_command_line_args{};
28const std::function<std::vector<const char*>()> G_TEST_COMMAND_LINE_ARGUMENTS = []() {
29    return g_bench_command_line_args();
30};
31
32/**
33 * Retrieve the name of the currently in-use benchmark.
34 * This is applicable only to benchmarks that utilize the unit test
35 * framework context setup (e.g. ones using 'MakeNoLogFileContext<TestingSetup>()').
36 * It places the datadir of each benchmark run within their respective benchmark name.
37 */
38static std::string g_running_benchmark_name;
39const std::function<std::string()> G_TEST_GET_FULL_NAME = []() {
40    return g_running_benchmark_name;
41};
42
43namespace {
44
45void GenerateTemplateResults(const std::vector<ankerl::nanobench::Result>& benchmarkResults, const fs::path& file, const char* tpl)
46{
47    if (benchmarkResults.empty() || file.empty()) {
48        // nothing to write, bail out
49        return;
50    }
51    std::ofstream fout{file.std_path()};
52    if (fout.is_open()) {
53        ankerl::nanobench::render(tpl, benchmarkResults, fout);
54        std::cout << "Created " << file << std::endl;
55    } else {
56        std::cout << "Could not write to file " << file << std::endl;
57    }
58}
59
60} // namespace
61
62namespace benchmark {
63
64BenchRunner::BenchmarkMap& BenchRunner::benchmarks()
65{
66    static BenchmarkMap benchmarks_map;
67    return benchmarks_map;
68}
69
70BenchRunner::BenchRunner(std::string_view name, BenchFunction func)
71{
72    Assert(benchmarks().try_emplace(std::string{name}, std::move(func)).second);
73}
74
75void BenchRunner::RunAll(const Args& args)
76{
77    std::regex reFilter(args.regex_filter);
78    std::smatch baseMatch;
79
80    if (args.sanity_check) {
81        std::cout << "Running with -sanity-check option, output is being suppressed as benchmark results will be useless." << std::endl;
82    }
83
84    // Load inner test setup args
85    g_bench_command_line_args = [&args]() {
86        std::vector<const char*> ret;
87        ret.reserve(args.setup_args.size());
88        for (const auto& arg : args.setup_args) ret.emplace_back(arg.c_str());
89        return ret;
90    };
91
92    std::vector<ankerl::nanobench::Result> benchmarkResults;
93    for (const auto& [name, func] : benchmarks()) {
94
95        if (!std::regex_match(name, baseMatch, reFilter)) {
96            continue;
97        }
98
99        if (args.is_list_only) {
100            std::cout << name << std::endl;
101            continue;
102        }
103
104        Bench bench;
105        if (args.sanity_check) {
106            bench.epochs(1).epochIterations(1);
107            bench.output(nullptr);
108        }
109        bench.name(name);
110        g_running_benchmark_name = name;
111        if (args.min_time > 0ms) {
112            // convert to nanos before dividing to reduce rounding errors
113            std::chrono::nanoseconds min_time_ns = args.min_time;
114            bench.minEpochTime(min_time_ns / bench.epochs());
115        }
116
117        if (args.asymptote.empty()) {
118            func(bench);
119        } else {
120            for (auto n : args.asymptote) {
121                bench.complexityN(n);
122                func(bench);
123            }
124            std::cout << bench.complexityBigO() << std::endl;
125        }
126
127        if (!bench.results().empty()) {
128            benchmarkResults.push_back(bench.results().back());
129        }
130    }
131
132    GenerateTemplateResults(benchmarkResults, args.output_csv, "# Benchmark, evals, iterations, total, min, max, median\n"
133                                                               "{{#result}}{{name}}, {{epochs}}, {{average(iterations)}}, {{sumProduct(iterations, elapsed)}}, {{minimum(elapsed)}}, {{maximum(elapsed)}}, {{median(elapsed)}}\n"
134                                                               "{{/result}}");
135    GenerateTemplateResults(benchmarkResults, args.output_json, ankerl::nanobench::templates::json());
136}
137
138} // namespace benchmark
139