Inside Bitcoin

251    targets = transform_rpc_target(targets, Path(src_dir))
252
253    def job(command, t, t_env):
254        logging.debug(f"Running '{command}'")
255        logging.debug("Command '{}' output:\n'{}'\n".format(
256            command,
257            subprocess.run(
258                command,
259                env={
260                    **t_env,
261                    **get_fuzz_env(target=t, source_dir=src_dir),
262                },
263                check=True,
264                stderr=subprocess.PIPE,
265                text=True,
266            ).stderr,
267        ))
268
269    futures = []
270    for target, t_env in targets:
271        target_corpus_dir = corpus_dir / target
272        os.makedirs(target_corpus_dir, exist_ok=True)
273        use_value_profile = int(random.random() < .3)
274        command = [
275            fuzz_bin,
276            "-rss_limit_mb=8000",
277            "-max_total_time=6000",
278            "-reload=0",
279            f"-use_value_profile={use_value_profile}",
280            target_corpus_dir,
281        ]
282        futures.append(fuzz_pool.submit(job, command, target, t_env))
283
284    for future in as_completed(futures):
285        future.result()
286
287
288def merge_inputs(*, fuzz_pool, corpus, test_list, src_dir, fuzz_bin, merge_dirs):
289    logging.info(f"Merge the inputs from the passed dir into the corpus_dir. Passed dirs {merge_dirs}")
290    jobs = []
291    for t in test_list:
292        args = [
293            fuzz_bin,
294            '-rss_limit_mb=8000',
295            '-set_cover_merge=1',
296            # set_cover_merge is used instead of -merge=1 to reduce the overall
297            # size of the qa-assets git repository a bit, but more importantly,
298            # to cut the runtime to iterate over all fuzz inputs [0].
299            # [0] https://github.com/bitcoin-core/qa-assets/issues/130#issuecomment-1761760866
300            '-shuffle=0',
301            '-prefer_small=1',
302            '-use_value_profile=0',
303            # use_value_profile is enabled by oss-fuzz [0], but disabled for
304            # now to avoid bloating the qa-assets git repository [1].
305            # [0] https://github.com/google/oss-fuzz/issues/1406#issuecomment-387790487
306            # [1] https://github.com/bitcoin-core/qa-assets/issues/130#issuecomment-1749075891
307            os.path.join(corpus, t),
308        ] + [str(m_dir / t) for m_dir in merge_dirs]
309        os.makedirs(os.path.join(corpus, t), exist_ok=True)
310        for m_dir in merge_dirs:
311            (m_dir / t).mkdir(exist_ok=True)
312
313        def job(t, args):
314            output = 'Run {} with args {}\n'.format(t, " ".join(args))
315            output += subprocess.run(
316                args,
317                env=get_fuzz_env(target=t, source_dir=src_dir),
318                check=True,
319                stderr=subprocess.PIPE,
320                text=True,
321            ).stderr
322            logging.debug(output)
323
324        jobs.append(fuzz_pool.submit(job, t, args))
325
326    for future in as_completed(jobs):
327        future.result()
328
329
330def run_once(*, fuzz_pool, corpus, test_list, src_dir, fuzz_bin, using_libfuzzer, use_valgrind, empty_min_time):
331    jobs = []
332    for t in test_list:
333        corpus_path = corpus / t
334        os.makedirs(corpus_path, exist_ok=True)
335        args = [
336            fuzz_bin,
337        ]
338        empty_dir = not any(corpus_path.iterdir())
339        if using_libfuzzer:
340            if empty_min_time and empty_dir:
341                args += [f"-max_total_time={empty_min_time}"]
342            else:
343                args += [
344                    "-runs=1",
345                    corpus_path,
346                ]
347        else:
348            args += [corpus_path]
349        if use_valgrind:
350            args = ['valgrind', '--quiet', '--error-exitcode=1'] + args
351
352        def job(t, args):
353            output = 'Run {} with args {}'.format(t, args)
354            result = subprocess.run(
355                args,
356                env=get_fuzz_env(target=t, source_dir=src_dir),
357                stderr=subprocess.PIPE,
358                text=True,
359            )
360            output += result.stderr
361            return output, result, t
362
363        jobs.append(fuzz_pool.submit(job, t, args))
364
365    stats = []
366    for future in as_completed(jobs):
367        output, result, target = future.result()
368        logging.debug(output)
369        try:
370            result.check_returncode()
371        except subprocess.CalledProcessError as e:
372            if e.stdout:
373                logging.info(e.stdout)
374            if e.stderr:
375                logging.info(e.stderr)
376            logging.info(f"⚠️ Failure generated from target with exit code {e.returncode}: {result.args}")
377            sys.exit(1)
378        if using_libfuzzer:
379            done_stat = [l for l in output.splitlines() if "DONE" in l]
380            assert len(done_stat) == 1
381            stats.append((target, done_stat[0]))
382
383    if using_libfuzzer:
384        print("Summary:")
385        max_len = max(len(t[0]) for t in stats)
386        for t, s in sorted(stats):
387            t = t.ljust(max_len + 1)
388            print(f"{t}{s}")
389
390
391def parse_test_list(*, fuzz_bin, source_dir):
392    test_list_all = subprocess.run(
393        fuzz_bin,
394        env={
395            'PRINT_ALL_FUZZ_TARGETS_AND_ABORT': '',
396            **get_fuzz_env(target="", source_dir=source_dir)
397        },
398        stdout=subprocess.PIPE,
399        text=True,
400        check=True,
401    ).stdout.splitlines()
402    return test_list_all
403
404
405if __name__ == '__main__':
406    main()
407