Inside Bitcoin

1// Copyright (c) The Bitcoin Core developers
2// Distributed under the MIT software license, see the accompanying
3// file COPYING or https://opensource.org/license/mit/.
4
5use std::collections::VecDeque;
6use std::env;
7use std::fs::{read_dir, DirEntry, File};
8use std::path::{Path, PathBuf};
9use std::process::{Command, ExitCode};
10use std::str;
11use std::thread;
12
13/// A type for a complete and readable error message.
14type AppError = String;
15type AppResult = Result<(), AppError>;
16
17const LLVM_PROFDATA: &str = "llvm-profdata";
18const LLVM_COV: &str = "llvm-cov";
19const GIT: &str = "git";
20
21const DEFAULT_PAR: usize = 1;
22
23fn exit_help(err: &str) -> AppError {
24    format!(
25        r#"
26Error: {err}
27
28Usage: program ./build_dir ./qa-assets/fuzz_corpora fuzz_target_name [parallelism={DEFAULT_PAR}]
29
30Refer to the devtools/README.md for more details."#
31    )
32}
33
34fn sanity_check(corpora_dir: &Path, fuzz_exe: &Path) -> AppResult {
35    for tool in [LLVM_PROFDATA, LLVM_COV, GIT] {
36        let output = Command::new(tool).arg("--help").output();
37        match output {
38            Ok(output) if output.status.success() => {}
39            _ => Err(exit_help(&format!("The tool {tool} is not installed")))?,
40        }
41    }
42    if !corpora_dir.is_dir() {
43        Err(exit_help(&format!(
44            "Fuzz corpora path ({}) must be a directory",
45            corpora_dir.display()
46        )))?;
47    }
48    if !fuzz_exe.exists() {
49        Err(exit_help(&format!(
50            "Fuzz executable ({}) not found",
51            fuzz_exe.display()
52        )))?;
53    }
54    Ok(())
55}
56
57fn app() -> AppResult {
58    // Parse args
59    let args = env::args().collect::<Vec<_>>();
60    let build_dir = args.get(1).ok_or(exit_help("Must set build dir"))?;
61    if build_dir == "--help" {
62        Err(exit_help("--help requested"))?;
63    }
64    let corpora_dir = args.get(2).ok_or(exit_help("Must set fuzz corpora dir"))?;
65    let fuzz_target = args
66        .get(3)
67        // Require fuzz target for now. In the future it could be optional and the tool could
68        // iterate over all compiled fuzz targets
69        .ok_or(exit_help("Must set fuzz target"))?;
70    let par = match args.get(4) {
71        Some(s) => s
72            .parse::<usize>()
73            .map_err(|e| exit_help(&format!("Could not parse parallelism as usize ({s}): {e}")))?,
74        None => DEFAULT_PAR,
75    }
76    .max(1);
77    if args.get(5).is_some() {
78        Err(exit_help("Too many args"))?;
79    }
80
81    let build_dir = Path::new(build_dir);
82    let corpora_dir = Path::new(corpora_dir);
83    let fuzz_exe = build_dir.join("bin/fuzz");
84
85    sanity_check(corpora_dir, &fuzz_exe)?;
86
87    deterministic_coverage(build_dir, corpora_dir, &fuzz_exe, fuzz_target, par)
88}
89
90fn using_libfuzzer(fuzz_exe: &Path) -> Result<bool, AppError> {
91    println!("Check if using libFuzzer ...");
92    let stderr = Command::new(fuzz_exe)
93        .arg("-help=1") // Will be interpreted as option (libfuzzer) or as input file
94        .env("FUZZ", "addition_overflow") // Any valid target
95        .output()
96        .map_err(|e| format!("fuzz failed with {e}"))?
97        .stderr;
98    let help_output = str::from_utf8(&stderr)
99        .map_err(|e| format!("The libFuzzer -help=1 output must be valid text ({e})"))?;
100    Ok(help_output.contains("libFuzzer"))
101}
102
103fn deterministic_coverage(
104    build_dir: &Path,
105    corpora_dir: &Path,
106    fuzz_exe: &Path,
107    fuzz_target: &str,
108    par: usize,
109) -> AppResult {
110    let using_libfuzzer = using_libfuzzer(fuzz_exe)?;
111    if using_libfuzzer {
112        println!("Warning: The fuzz executable was compiled with libFuzzer as sanitizer.");
113        println!("This tool may be tripped by libFuzzer misbehavior.");
114        println!("It is recommended to compile without libFuzzer.");
115    }
116    let corpus_dir = corpora_dir.join(fuzz_target);
117    let mut entries = read_dir(&corpus_dir)
118        .map_err(|err| {
119            exit_help(&format!(
120                "The fuzz target's input directory must exist! ({}; {})",
121                corpus_dir.display(),
122                err
123            ))
124        })?
125        .map(|entry| entry.expect("IO error"))
126        .collect::<Vec<_>>();
127    entries.sort_by_key(|entry| entry.file_name());
128    let run_single = |run_id: char, entry: &Path, thread_id: usize| -> Result<PathBuf, AppError> {
129        let cov_txt_path = build_dir.join(format!("fuzz_det_cov.show.t{thread_id}.{run_id}.txt"));
130        let profraw_file = build_dir.join(format!("fuzz_det_cov.t{thread_id}.{run_id}.profraw"));
131        let profdata_file = build_dir.join(format!("fuzz_det_cov.t{thread_id}.{run_id}.profdata"));
132        {
133            let output = {
134                let mut cmd = Command::new(fuzz_exe);
135                if using_libfuzzer {
136                    cmd.args(["-runs=1", "-shuffle=1", "-prefer_small=0"]);
137                }
138                cmd
139            }
140            .env("LLVM_PROFILE_FILE", &profraw_file)
141            .env("FUZZ", fuzz_target)
142            .arg(entry)
143            .output()
144            .map_err(|e| format!("fuzz failed: {e}"))?;
145            if !output.status.success() {
146                Err(format!(
147                    "fuzz failed!\nstdout:\n{}\nstderr:\n{}\n",
148                    String::from_utf8_lossy(&output.stdout),
149                    String::from_utf8_lossy(&output.stderr)
150                ))?;
151            }
152        }
153        if !Command::new(LLVM_PROFDATA)
154            .arg("merge")
155            .arg("--sparse")
156            .arg(&profraw_file)
157            .arg("-o")
158            .arg(&profdata_file)
159            .status()
160            .map_err(|e| format!("{LLVM_PROFDATA} merge failed with {e}"))?
161            .success()
162        {
163            Err(format!("{LLVM_PROFDATA} merge failed. This can be a sign of compiling without code coverage support."))?;
164        }
165        let cov_file = File::create(&cov_txt_path)
166            .map_err(|e| format!("Failed to create coverage txt file ({e})"))?;
167        if !Command::new(LLVM_COV)
168            .args([
169                "show",
170                "--show-line-counts-or-regions",
171                "--show-branches=count",
172                "--show-expansions",
173                "--show-instantiation-summary",
174                "-Xdemangler=llvm-cxxfilt",
175                &format!("--instr-profile={}", profdata_file.display()),
176            ])
177            .arg(fuzz_exe)
178            .stdout(cov_file)
179            .spawn()
180            .map_err(|e| format!("{LLVM_COV} show failed with {e}"))?
181            .wait()
182            .map_err(|e| format!("{LLVM_COV} show failed with {e}"))?
183            .success()
184        {
185            Err(format!("{LLVM_COV} show failed"))?;
186        };
187        Ok(cov_txt_path)
188    };
189    let check_diff = |a: &Path, b: &Path, err: &str| -> AppResult {
190        let same = Command::new(GIT)
191            .args(["--no-pager", "diff", "--no-index"])
192            .arg(a)
193            .arg(b)
194            .status()
195            .map_err(|e| format!("{GIT} diff failed with {e}"))?
196            .success();
197        if !same {
198            Err(format!(
199                r#"
200The coverage was not deterministic between runs.
201{err}"#
202            ))?;
203        }
204        Ok(())
205    };
206    // First, check that each fuzz input is deterministic running by itself in a process.
207    //
208    // This can catch issues and isolate where a single fuzz input triggers non-determinism, but
209    // all other fuzz inputs are deterministic.
210    //
211    // Also, This can catch issues where several fuzz inputs are non-deterministic, but the sum of
212    // their overall coverage trace remains the same across runs and thus remains undetected.
213    println!(
214        "Check each fuzz input individually ... ({} inputs with parallelism {par})",
215        entries.len()
216    );
217    let check_individual = |entry: &DirEntry, thread_id: usize| -> AppResult {
218        let entry = entry.path();
219        if !entry.is_file() {
220            Err(format!("{} should be a file", entry.display()))?;
221        }
222        let cov_txt_base = run_single('a', &entry, thread_id)?;
223        let cov_txt_repeat = run_single('b', &entry, thread_id)?;
224        check_diff(
225            &cov_txt_base,
226            &cov_txt_repeat,
227            &format!("The fuzz target input was {}.", entry.display()),
228        )?;
229        Ok(())
230    };
231    thread::scope(|s| -> AppResult {
232        let mut handles = VecDeque::with_capacity(par);
233        let mut res = Ok(());
234        for (i, entry) in entries.iter().enumerate() {
235            println!("[{}/{}]", i + 1, entries.len());
236            handles.push_back(s.spawn(move || check_individual(entry, i % par)));
237            while handles.len() >= par || i == (entries.len() - 1) || res.is_err() {
238                if let Some(th) = handles.pop_front() {
239                    let thread_result = match th.join() {
240                        Err(_e) => Err("A scoped thread panicked".to_string()),
241                        Ok(r) => r,
242                    };
243                    if thread_result.is_err() {
244                        res = thread_result;
245                    }
246                } else {
247                    return res;
248                }
249            }
250        }