fold/tests/integration/test_fuzz_run.cpp

// End-to-end integration test for `fold fuzz-run`. Chains
// fuzz-prep → fuzz-run on a null-deref-behind-magic-byte-guard
// fixture and asserts that the augmented report contains a
// ConcreteWitness Finding whose id matches the predicted Finding.id.
//
// Gated on libFuzzer being linkable by clang++ at runtime — the test
// probes for it and SKIPs cleanly if the environment can't build the
// harness (e.g. base-system clang without compiler-rt-fuzzer).

#include <algorithm>
#include <catch2/catch_test_macros.hpp>
#include <chrono>
#include <cstdlib>
#include <filesystem>
#include <fstream>
#include <nlohmann/json.hpp>
#include <string>

#include "fold/cli/fuzz_prep_runner.hpp"
#include "fold/cli/fuzz_run_runner.hpp"
#include "fold/cli/scan_router.hpp"

namespace fs = std::filesystem;

namespace {

constexpr const char* kHybridFixtureUrl = "https://fold.example.invalid/hybrid-fuzz-r2-token";

bool commandExists(const char* command) {
    std::string check = "command -v ";
    check += command;
    check += " >/dev/null 2>&1";
    return std::system(check.c_str()) == 0;
}

std::string shellQuote(const std::string& value) {
    std::string quoted = "'";
    for (char ch : value) {
        if (ch == '\'')
            quoted += "'\\''";
        else
            quoted.push_back(ch);
    }
    quoted.push_back('\'');
    return quoted;
}

// Probe the sanitizer set used by the production fuzz runner. Returns the
// compiler path that worked, or empty when none did.
std::string findLibFuzzerCompiler() {
    static const char* candidates[] = {
        "/opt/homebrew/opt/llvm@22/bin/clang++",
        "/opt/homebrew/opt/llvm/bin/clang++",
        "/usr/local/opt/llvm/bin/clang++",
        "clang++",
    };

    fs::path probe_dir = fs::temp_directory_path() / "fold-libfuzzer-probe";
    fs::create_directories(probe_dir);
    fs::path src = probe_dir / "p.cc";
    std::ofstream(src) << "#include <cstddef>\n#include <cstdint>\n"
                          "extern \"C\" int LLVMFuzzerTestOneInput(const std::uint8_t*, "
                          "std::size_t) { return 0; }\n";
    fs::path bin = probe_dir / "p";

    for (const char* compiler : candidates) {
        const std::string cmd =
            std::string(compiler) +
            " -fsanitize=fuzzer,address,integer-divide-by-zero,signed-integer-overflow "
            "-fno-sanitize-recover=integer-divide-by-zero,signed-integer-overflow -O1 " +
            src.string() + " -o " + bin.string() + " 2>/dev/null";
        if (std::system(cmd.c_str()) == 0)
            return compiler;
    }
    return {};
}

fs::path writeFixture(const std::string& source) {
    fs::path tmp = fs::temp_directory_path() / "fold-fuzz-run-test";
    fs::remove_all(tmp);
    fs::create_directories(tmp);

    fs::path csrc = tmp / "fixture.c";
    std::ofstream(csrc) << source;

    nlohmann::json db = nlohmann::json::array();
    db.push_back({
        {"directory", tmp.string()},
        {"command", "clang -c fixture.c -o fixture.o"},
        {"file", csrc.string()},
    });
    std::ofstream(tmp / "compile_commands.json") << db.dump(2);
    return tmp;
}

bool buildFixtureBinary(const fs::path& tmp, std::string& binary) {
    if (!commandExists("cc")) {
        WARN("C compiler not found — skipping hybrid binary assertion");
        return false;
    }
    fs::path source = tmp / "fixture.c";
    fs::path main_source = tmp / "binary_main.c";
    fs::path output = tmp / "fixture-bin";
    std::ofstream(main_source) << "#include <stdio.h>\n"
                               << "const char *fold_marker(void);\n"
                               << "int main(void) { puts(fold_marker()); return 0; }\n";
    const std::string cmd = "cc -O0 " + shellQuote(source.string()) + " " +
                            shellQuote(main_source.string()) + " -o " +
                            shellQuote(output.string()) + " 2>/dev/null";
    if (std::system(cmd.c_str()) != 0 || !fs::exists(output)) {
        WARN("failed to compile hybrid fixture binary");
        return false;
    }
    binary = output.string();
    return true;
}

bool hasConcreteNullDerefPromotion(const nlohmann::json& findings) {
    for (const auto& f : findings) {
        if (f.value("confidence", "") != "concrete-witness")
            continue;
        if (f.value("kind", "") != "null-deref")
            continue;
        if (f.value("id", "").find("null-deref:") != 0)
            continue;
        if (!f.contains("evidence") || !f["evidence"].contains("fuzz_witness"))
            continue;
        const auto& witness = f["evidence"]["fuzz_witness"];
        if (!witness.value("seed_path", "").empty() && !witness.value("asan_kind", "").empty())
            return true;
    }
    return false;
}

} // namespace

TEST_CASE("fuzz-run promotes a predicted Finding to ConcreteWitness", "[fuzz][fuzz-run][e2e]") {
    const std::string compiler = findLibFuzzerCompiler();
    if (compiler.empty()) {
        WARN("libFuzzer not linkable in this environment — skipping");
        SUCCEED();
        return;
    }

    auto tmp = writeFixture(R"c(
int leaky(int *p, int flag) {
    if (flag == 0xAB) {
        return *p;
    }
    return 0;
}
)c");

    // Stage 1 — fuzz-prep
    fold::cli::FuzzPrepOptions prep_opts;
    prep_opts.scan.compile_db = (tmp / "compile_commands.json").string();
    prep_opts.output_dir = (tmp / "prep").string();
    REQUIRE(fold::cli::runFuzzPrep(std::move(prep_opts)) == 0);

    // Stage 2 — fuzz-run
    fold::cli::FuzzRunOptions run_opts;
    run_opts.prep_dir = (tmp / "prep").string();
    run_opts.target_sources = {(tmp / "fixture.c").string()};
    run_opts.compiler = compiler;
    run_opts.time_budget_per_fn = std::chrono::seconds(8);
    run_opts.report_path = (tmp / "augmented.json").string();
    REQUIRE(fold::cli::runFuzzRun(std::move(run_opts)) == 0);

    // The augmented report must contain at least one ConcreteWitness
    // Finding whose id matches the prep-time prediction.
    REQUIRE(fs::exists(tmp / "augmented.json"));
    auto j = nlohmann::json::parse(std::ifstream(tmp / "augmented.json"));
    REQUIRE(j.is_array());
    REQUIRE_FALSE(j.empty());

    bool found_promotion = false;
    for (const auto& f : j) {
        if (f.value("confidence", "") != "concrete-witness")
            continue;
        const std::string id = f.value("id", "");
        if (id.find("null-deref:") != 0)
            continue;
        REQUIRE(f.contains("evidence"));
        const auto& ev = f["evidence"];
        REQUIRE(ev.contains("fuzz_witness"));
        const auto& w = ev["fuzz_witness"];
        REQUIRE(!w.value("seed_path", "").empty());
        REQUIRE(!w.value("asan_kind", "").empty());
        // Some ASan configurations emit only `function+offset`. The runtime
        // witness must not invent coordinates; the promoted finding still
        // carries the prediction's exact source sink for triage.
        if (!w.value("asan_file", "").empty())
            REQUIRE(w.value("asan_line", 0U) > 0U);
        else
            REQUIRE(w.value("asan_line", 0U) == 0U);
        REQUIRE(f.value("evidence", nlohmann::json::object()).value("backend", "") == "source");
        REQUIRE(ev.value("correlation_kind", "") == (!w.value("asan_file", "").empty()
                                                         ? "source-location+hazard"
                                                         : "unique-function+hazard"));
        REQUIRE(f.contains("locations"));
        REQUIRE_FALSE(f["locations"].empty());
        REQUIRE(f["locations"][0].value("line", 0) > 0);
        found_promotion = true;
        break;
    }
    REQUIRE(found_promotion);

    REQUIRE(fs::exists(tmp / "prep" / "confirmation.results.json"));
    auto results = nlohmann::json::parse(std::ifstream(tmp / "prep" / "confirmation.results.json"));
    REQUIRE(results.is_array());
    REQUIRE_FALSE(results.empty());
    bool found_confirmation_result = false;
    for (const auto& result : results) {
        if (result.value("status", "") == "confirmed" &&
            result.value("adapter", "") == "libfuzzer-asan" &&
            result.value("promoted_confidence", "") == "concrete-witness") {
            const auto progress = result.value("progress", nlohmann::json::object());
            REQUIRE(progress.value("reached", false));
            REQUIRE(progress.value("triggered", false));
            REQUIRE(progress.value("detected", false));
            REQUIRE(progress.value("replayed", false));
            found_confirmation_result = true;
        }
    }
    REQUIRE(found_confirmation_result);

    fs::remove_all(tmp);
}

TEST_CASE("fuzz-run promotes a UBSan division prediction to ConcreteWitness",
          "[fuzz][fuzz-run][ubsan][e2e]") {
    const std::string compiler = findLibFuzzerCompiler();
    if (compiler.empty()) {
        WARN("libFuzzer + arithmetic UBSan not linkable in this environment — skipping");
        SUCCEED();
        return;
    }

    auto tmp = writeFixture(R"c(
int divide(int numerator, int divisor) {
    return numerator / divisor;
}
)c");

    fold::cli::FuzzPrepOptions prep_opts;
    prep_opts.scan.compile_db = (tmp / "compile_commands.json").string();
    prep_opts.output_dir = (tmp / "prep").string();
    REQUIRE(fold::cli::runFuzzPrep(std::move(prep_opts)) == 0);

    fold::cli::FuzzRunOptions run_opts;
    run_opts.prep_dir = (tmp / "prep").string();
    run_opts.target_sources = {(tmp / "fixture.c").string()};
    run_opts.compiler = compiler;
    run_opts.time_budget_per_fn = std::chrono::seconds(8);
    run_opts.report_path = (tmp / "augmented.json").string();
    REQUIRE(fold::cli::runFuzzRun(std::move(run_opts)) == 0);

    auto findings = nlohmann::json::parse(std::ifstream(tmp / "augmented.json"));
    const auto confirmed = std::find_if(findings.begin(), findings.end(), [](const auto& finding) {
        if (finding.value("kind", "") != "div-by-zero" ||
            finding.value("confidence", "") != "concrete-witness")
            return false;
        const auto& evidence = finding.value("evidence", nlohmann::json::object());
        if (!evidence.contains("fuzz_witness"))
            return false;
        return evidence["fuzz_witness"].value("asan_kind", "") == "division-by-zero";
    });
    REQUIRE(confirmed != findings.end());

    fs::remove_all(tmp);
}

TEST_CASE("fuzz-run promotes a UBSan signed-overflow prediction to ConcreteWitness",
          "[fuzz][fuzz-run][ubsan][e2e]") {
    const std::string compiler = findLibFuzzerCompiler();
    if (compiler.empty()) {
        WARN("libFuzzer + arithmetic UBSan not linkable in this environment — skipping");
        SUCCEED();
        return;
    }

    auto tmp = writeFixture(R"c(
int add_one(int value) {
    return value + 1;
}
)c");

    fold::cli::FuzzPrepOptions prep_opts;
    prep_opts.scan.compile_db = (tmp / "compile_commands.json").string();
    prep_opts.output_dir = (tmp / "prep").string();
    REQUIRE(fold::cli::runFuzzPrep(std::move(prep_opts)) == 0);

    fold::cli::FuzzRunOptions run_opts;
    run_opts.prep_dir = (tmp / "prep").string();
    run_opts.target_sources = {(tmp / "fixture.c").string()};
    run_opts.compiler = compiler;
    run_opts.time_budget_per_fn = std::chrono::seconds(8);
    run_opts.report_path = (tmp / "augmented.json").string();
    REQUIRE(fold::cli::runFuzzRun(std::move(run_opts)) == 0);

    auto findings = nlohmann::json::parse(std::ifstream(tmp / "augmented.json"));
    const auto confirmed = std::find_if(findings.begin(), findings.end(), [](const auto& finding) {
        if (finding.value("kind", "") != "int-overflow" ||
            finding.value("confidence", "") != "concrete-witness")
            return false;
        const auto evidence = finding.value("evidence", nlohmann::json::object());
        if (!evidence.contains("fuzz_witness"))
            return false;
        return evidence["fuzz_witness"].value("asan_kind", "") == "signed-integer-overflow";
    });
    REQUIRE(confirmed != findings.end());

    fs::remove_all(tmp);
}

TEST_CASE("hybrid scan preserves fuzz/source symbolic and r2 binary evidence",
          "[fuzz][binary][hybrid][e2e]") {
    const std::string compiler = findLibFuzzerCompiler();
    if (compiler.empty()) {
        WARN("libFuzzer not linkable in this environment — skipping");
        SUCCEED();
        return;
    }
    if (!commandExists("r2")) {
        WARN("radare2 CLI not found — skipping hybrid r2 assertion");
        SUCCEED();
        return;
    }

    auto tmp = writeFixture(R"c(
#include <stdio.h>

int leaky(int *p, int flag) {
    if (flag == 0xAB) {
        return *p;
    }
    return 0;
}

const char *fold_marker(void) {
    return "https://fold.example.invalid/hybrid-fuzz-r2-token";
}
)c");

    std::string binary;
    if (!buildFixtureBinary(tmp, binary)) {
        fs::remove_all(tmp);
        return;
    }

    // Stage 1 — source scanner + symbolic engine produce a Z3 witness,
    // then libFuzzer/ASan confirms it as a ConcreteWitness finding.
    fold::cli::FuzzPrepOptions prep_opts;
    prep_opts.scan.compile_db = (tmp / "compile_commands.json").string();
    prep_opts.output_dir = (tmp / "prep").string();
    REQUIRE(fold::cli::runFuzzPrep(std::move(prep_opts)) == 0);

    fold::cli::FuzzRunOptions run_opts;
    run_opts.prep_dir = (tmp / "prep").string();
    run_opts.target_sources = {(tmp / "fixture.c").string()};
    run_opts.compiler = compiler;
    run_opts.time_budget_per_fn = std::chrono::seconds(8);
    run_opts.report_path = (tmp / "augmented.json").string();
    REQUIRE(fold::cli::runFuzzRun(std::move(run_opts)) == 0);

    auto augmented = nlohmann::json::parse(std::ifstream(tmp / "augmented.json"));
    REQUIRE(augmented.is_array());
    REQUIRE(hasConcreteNullDerefPromotion(augmented));

    // Stage 2 — the router runs source/Z3 and r2/binary independently.
    // This validates the product loop without implying binary lifting into
    // the symbolic IR: backend provenance must remain distinct.
    fold::cli::ScanOptions scan_opts;
    scan_opts.mode = "hybrid";
    scan_opts.compile_db = (tmp / "compile_commands.json").string();
    scan_opts.binary_path = binary;
    scan_opts.detectors = {"security"};
    scan_opts.min_body_stmts = 0;
    scan_opts.processes = 1;

    auto hybrid = fold::cli::runScanRouter(std::move(scan_opts));
    REQUIRE(hybrid.context.source_ran);
    REQUIRE(hybrid.context.binary_ran);

    const auto source_symbolic = std::find_if(
        hybrid.findings.begin(), hybrid.findings.end(), [](const fold::Finding& finding) {
            return finding.kind == fold::FindingKind::NullDeref &&
                   finding.evidence.backend == "source" &&
                   finding.evidence.detector_name == "null-deref";
        });
    REQUIRE(source_symbolic != hybrid.findings.end());

    const auto binary_string = std::find_if(
        hybrid.findings.begin(), hybrid.findings.end(), [](const fold::Finding& finding) {
            return finding.evidence.backend == "binary" &&
                   finding.evidence.detector_name == "binary-security" &&
                   finding.evidence.binary_string == kHybridFixtureUrl;
        });
    REQUIRE(binary_string != hybrid.findings.end());
    REQUIRE(binary_string->confidence == fold::Confidence::Heuristic);
    REQUIRE(binary_string->evidence.binary_offset.has_value());
    REQUIRE(binary_string->evidence.binary_vaddr.has_value());

    fs::remove_all(tmp);
}