fold/tests/unit/test_scan_coverage.cpp

#include <catch2/catch_test_macros.hpp>
#include <filesystem>
#include <nlohmann/json.hpp>
#include <sstream>
#include <string>
#include <vector>

#include "fold/cli/clang_shard_tu.hpp"
#include "fold/report/findings_json_report.hpp"
#include "fold/report/sarif.hpp"
#include "fold/report/scan_coverage.hpp"

// A TU the front end could not parse used to vanish: the report had no trace
// of it and the scan exited 0, indistinguishable from a clean scan. Coverage
// now records every requested TU that was not analyzed on a clean AST.

namespace {

std::string tmp(const std::string& name) {
    return (std::filesystem::temp_directory_path() / name).string();
}

} // namespace

TEST_CASE("tuCoverage classifies every requested TU", "[coverage]") {
    const std::vector<std::string> requested{tmp("ok.cpp"), tmp("broken.cpp"), tmp("missing.cpp"),
                                             tmp("slow.cpp")};
    const std::vector<fold::cli::ClangShardTuOutcome> outcomes{
        {tmp("ok.cpp"), ""},
        {tmp("broken.cpp"), "compile-errors"},
        {tmp("slow.cpp"), "timeout"},
    };
    const auto coverage = fold::cli::tuCoverage(requested, outcomes);
    CHECK(coverage.tracked);
    CHECK(coverage.tus_attempted == 4);
    CHECK(coverage.tus_analyzed == 1);
    REQUIRE(coverage.gaps.size() == 3);
    CHECK(coverage.gaps[0].file == tmp("broken.cpp"));
    CHECK(coverage.gaps[0].reason == "compile-errors");
    CHECK(coverage.gaps[1].file == tmp("missing.cpp"));
    CHECK(coverage.gaps[1].reason == "frontend-failed");
    CHECK(coverage.gaps[2].reason == "timeout");
    CHECK(coverage.incomplete());
}

TEST_CASE("tuCoverage matches paths across spellings", "[coverage]") {
    // The compile DB may spell a path differently than the front end does.
    const auto dir = std::filesystem::temp_directory_path();
    const std::string dotted = (dir / "sub" / ".." / "a.cpp").string();
    const auto coverage = fold::cli::tuCoverage({dotted}, {{(dir / "a.cpp").string(), ""}});
    CHECK(coverage.tus_analyzed == 1);
    CHECK_FALSE(coverage.incomplete());
}

TEST_CASE("coverage: complete, untracked and empty scans", "[coverage]") {
    SECTION("every TU analyzed is complete (control)") {
        const auto coverage = fold::cli::tuCoverage({tmp("a.cpp")}, {{tmp("a.cpp"), ""}});
        CHECK_FALSE(coverage.incomplete());
    }
    SECTION("an untracked lane is never incomplete") {
        fold::ScanCoverage coverage;
        coverage.tus_attempted = 3;
        CHECK_FALSE(coverage.incomplete());
    }
    SECTION("attempted TUs with none analyzed is incomplete even without gaps") {
        fold::ScanCoverage coverage;
        coverage.tracked = true;
        coverage.tus_attempted = 2;
        CHECK(coverage.incomplete());
    }
    SECTION("merge sums counts and keeps gaps; an untracked side is ignored") {
        auto a = fold::cli::tuCoverage({tmp("a.cpp")}, {{tmp("a.cpp"), ""}});
        const auto b = fold::cli::tuCoverage({tmp("b.cpp")}, {});
        a.merge(b);
        a.merge(fold::ScanCoverage{});
        CHECK(a.tus_attempted == 2);
        CHECK(a.tus_analyzed == 1);
        CHECK(a.gaps.size() == 1);
    }
}

TEST_CASE("reports carry coverage", "[coverage][report]") {
    const auto coverage =
        fold::cli::tuCoverage({tmp("broken.cpp")}, {{tmp("broken.cpp"), "compile-errors"}});
    SECTION("SARIF: invocation is unsuccessful and names the TU") {
        std::ostringstream out;
        fold::writeFilteredSarif(out, {}, &coverage);
        const auto sarif = nlohmann::json::parse(out.str());
        const auto& invocation = sarif["runs"][0]["invocations"][0];
        CHECK(invocation["executionSuccessful"] == false);
        const auto& note = invocation["toolExecutionNotifications"][0];
        CHECK(note["level"] == "error");
        CHECK(note["locations"][0]["physicalLocation"]["artifactLocation"]["uri"] ==
              tmp("broken.cpp"));
        CHECK(invocation["properties"]["coverage"]["tus_analyzed"] == 0);
    }
    SECTION("findings JSON: coverage object") {
        std::ostringstream out;
        fold::writeFindingsJson(out, {}, &coverage);
        const auto report = nlohmann::json::parse(out.str());
        CHECK(report["coverage"]["complete"] == false);
        CHECK(report["coverage"]["gaps"][0]["reason"] == "compile-errors");
    }
    SECTION("no coverage argument leaves the documents unchanged") {
        std::ostringstream sarif_out;
        fold::writeFilteredSarif(sarif_out, {});
        CHECK_FALSE(nlohmann::json::parse(sarif_out.str())["runs"][0].contains("invocations"));
        std::ostringstream json_out;
        fold::writeFindingsJson(json_out, {});
        CHECK_FALSE(nlohmann::json::parse(json_out.str()).contains("coverage"));
    }
}