Fix audit findings: soundness, coverage reporting, docs, and review follow-ups #1
24 changed files with 435 additions and 50 deletions
fix(scan): report unanalyzed translation units and exit 2
Problem: a TU the front end could not parse left no trace. ClangTool::run's status was discarded, TUs with compile errors were analyzed on a partial AST and counted as done, and the isolated-worker path dropped TUs that never produced an AST. A scan of only broken files printed no findings and exited 0, the same as a clean scan. Separately, the scan logger wrote to stdout, so any warning corrupted a SARIF/JSON report written to stdout. Fix: every TU now yields an outcome (clean, compile-errors, analysis-failed); workers carry it on their file_finished record, and the fan-out parent adds timeout/crash/worker-exit for the TU a worker died on. tuCoverage() turns the requested file list plus outcomes into a ScanCoverage (requested TUs with no outcome are frontend-failed). SARIF gets a run `invocations` entry (executionSuccessful, one toolExecutionNotification per gap); findings JSON gets an additive `coverage` object. A scan with any gap, or with nothing analyzed, exits 2 after writing its report. CLI diagnostics now go to stderr, as docs/internal/cli-philosophy.md principle 12 requires. USAGE.md and AGENTS.md document the coverage fields and the exit code. Test: tests/unit/test_scan_coverage.cpp (classification, path spelling, complete/untracked controls, merge, both writers, and unchanged output without coverage); test_cli_paths "[coverage]" covers a clean TU (exit 0, complete) and a TU with a missing include (exit 2, gap in JSON, SARIF on stdout parseable with executionSuccessful=false). Before: both probe files exited 0 with no coverage information. Full suite: 139/139.
commit
c9811b7d41
|
|
@ -42,7 +42,8 @@ Compatibility requirements:
|
|||
- Existing `fold scan --input-type binary --binary <path>` commands must keep
|
||||
working.
|
||||
- Exit codes stay stable: `0` no actionable findings, `1` actionable findings,
|
||||
`2` input/config/report failure.
|
||||
`2` input/config/report failure, including a scan that left any requested
|
||||
translation unit unanalyzed (see `coverage` in docs/USAGE.md).
|
||||
- Reports must preserve per-finding backend provenance.
|
||||
|
||||
Router implementation rules:
|
||||
|
|
|
|||
|
|
@ -89,6 +89,16 @@ report. Requested report/profile paths are written atomically; failures return
|
|||
exit `2`. Console counts and both machine formats use the same filtered and
|
||||
deduplicated finding set.
|
||||
|
||||
Both machine formats record which C/C++ translation units were analyzed: the
|
||||
native JSON gains a `coverage` object (`tus_attempted`, `tus_analyzed`, and one
|
||||
`gaps` entry per unanalyzed TU with a reason: `compile-errors`,
|
||||
`frontend-failed`, `analysis-failed`, `timeout`, `crash`, or `worker-exit`),
|
||||
and SARIF carries the same data as an `invocations` entry with
|
||||
`executionSuccessful` and one `toolExecutionNotifications` entry per gap. A
|
||||
scan with any gap, or one that analyzed no TU, exits `2` after writing its
|
||||
report: no findings from an incomplete scan is not evidence of no bugs.
|
||||
Diagnostics go to stderr, so a report written to stdout stays parseable.
|
||||
|
||||
`fold --version` reports the Meson semantic version plus Git revision and a
|
||||
`.dirty` suffix for builds configured from modified trees. SARIF, native JSON,
|
||||
profiles, and benchmark run artifacts retain the same provenance.
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ class ASTContext;
|
|||
namespace fold {
|
||||
|
||||
struct Finding;
|
||||
struct ScanCoverage;
|
||||
struct FunctionSummary;
|
||||
struct SerializedSummary;
|
||||
|
||||
|
|
@ -34,8 +35,26 @@ struct ClangShardTuHooks {
|
|||
|
||||
[[nodiscard]] std::string currentClangTuPath(const clang::ASTContext& ctx);
|
||||
|
||||
void runClangShardTranslationUnit(const ScanOptions& opts, clang::ASTContext& ctx,
|
||||
ClangShardTuState state, ClangShardTuHooks hooks);
|
||||
// Absolute, symlink-resolved form used to match TU paths across the compile
|
||||
// DB, the front end, and worker records.
|
||||
[[nodiscard]] std::string normalizeClangTuPath(const std::string& path);
|
||||
|
||||
// `gap_reason` is empty when the TU was analyzed on a clean AST; otherwise it
|
||||
// is a CoverageGap reason ("compile-errors", "analysis-failed").
|
||||
struct ClangShardTuOutcome {
|
||||
std::string path;
|
||||
std::string gap_reason;
|
||||
};
|
||||
|
||||
// Workers report the outcome on their `file_finished` record as `gap`.
|
||||
ClangShardTuOutcome runClangShardTranslationUnit(const ScanOptions& opts, clang::ASTContext& ctx,
|
||||
ClangShardTuState state, ClangShardTuHooks hooks);
|
||||
|
||||
// Coverage of `requested` TUs given the outcomes recorded for them. A
|
||||
// requested TU with no outcome never reached the detectors: the front end
|
||||
// produced no AST for it ("frontend-failed").
|
||||
[[nodiscard]] ScanCoverage tuCoverage(const std::vector<std::string>& requested,
|
||||
const std::vector<ClangShardTuOutcome>& outcomes);
|
||||
|
||||
} // namespace cli
|
||||
} // namespace fold
|
||||
|
|
|
|||
|
|
@ -33,6 +33,8 @@ struct ScanRouterResult {
|
|||
ScanContext context;
|
||||
// RF-8: Rust fuzzable function interfaces carried through the router.
|
||||
std::vector<frontend::RustInterface> rust_interfaces;
|
||||
// Which requested C/C++ TUs were actually analyzed.
|
||||
ScanCoverage coverage;
|
||||
};
|
||||
|
||||
std::optional<ScanProfile> mergeScanProfilesForRouter(const std::optional<ScanProfile>& source,
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@
|
|||
#include "fold/frontend/rust_backend.hpp"
|
||||
#include "fold/ir/summary.hpp"
|
||||
#include "fold/report/finding.hpp"
|
||||
#include "fold/report/scan_coverage.hpp"
|
||||
#include "fold/symbolic/serialize.hpp"
|
||||
|
||||
namespace fold::ir {
|
||||
|
|
@ -31,6 +32,8 @@ struct ShardResult {
|
|||
// runs detectEquivalentFunctionsCrossTU on the complete merged set.
|
||||
std::vector<SerializedSummary> func_summaries;
|
||||
std::optional<ScanProfile> profile;
|
||||
// Per-TU coverage of the clang source lane; untracked elsewhere.
|
||||
ScanCoverage coverage;
|
||||
};
|
||||
|
||||
struct LanguageIrSafetyDetectors {
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@
|
|||
#include <unordered_set>
|
||||
#include <vector>
|
||||
|
||||
#include "fold/cli/clang_shard_tu.hpp"
|
||||
#include "fold/cli/detector_plan.hpp"
|
||||
#include "fold/cli/profile.hpp"
|
||||
#include "fold/cli/scan_options.hpp"
|
||||
|
|
@ -59,6 +60,8 @@ struct SourceTuRunState {
|
|||
std::size_t security_candidate_functions = 0;
|
||||
std::size_t security_graph_expanded_functions = 0;
|
||||
std::size_t source_tus_completed = 0;
|
||||
// One entry per TU the front end handed over, in completion order.
|
||||
std::vector<ClangShardTuOutcome> tu_outcomes;
|
||||
// Non-incremental profiling counts definitions during the existing AST
|
||||
// traversal instead of launching a second fingerprinting parse.
|
||||
std::unordered_set<std::string> profile_function_names;
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@
|
|||
|
||||
#include "fold/cli/scan_options.hpp"
|
||||
#include "fold/report/finding.hpp"
|
||||
#include "fold/report/scan_coverage.hpp"
|
||||
|
||||
namespace fold::cli {
|
||||
|
||||
|
|
@ -35,6 +36,10 @@ class SubprocessFanout {
|
|||
std::vector<Finding> runSync(const ScanOptions& opts, const std::vector<std::string>& files,
|
||||
int n_workers);
|
||||
|
||||
// Per-TU coverage of the last runSync: TUs with compile errors, TUs the
|
||||
// front end never produced an AST for, and TUs a worker died on.
|
||||
[[nodiscard]] const ScanCoverage& coverage() const { return coverage_; }
|
||||
|
||||
// Cancel any in-flight workers. Used by MCP/LSP when the client
|
||||
// disconnects mid-scan. Safe to call from any thread.
|
||||
void cancel();
|
||||
|
|
@ -45,6 +50,7 @@ class SubprocessFanout {
|
|||
std::mutex mtx_;
|
||||
std::vector<pid_t> children_; // tracked PIDs, cleared as they exit
|
||||
bool cancelled_ = false;
|
||||
ScanCoverage coverage_;
|
||||
};
|
||||
|
||||
} // namespace fold::cli
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@
|
|||
#include <vector>
|
||||
|
||||
#include "finding.hpp"
|
||||
#include "scan_coverage.hpp"
|
||||
|
||||
namespace fold {
|
||||
|
||||
|
|
@ -11,6 +12,9 @@ inline constexpr int kFindingsJsonOutputVersion = 1;
|
|||
|
||||
// Native, lossless report envelope for Fold findings. This is distinct from
|
||||
// SARIF and is versioned so readers can reject incompatible future shapes.
|
||||
void writeFindingsJson(std::ostream& out, const std::vector<Finding>& findings);
|
||||
// With `coverage`, the envelope gains a `coverage` object (additive; the
|
||||
// output_version is unchanged).
|
||||
void writeFindingsJson(std::ostream& out, const std::vector<Finding>& findings,
|
||||
const ScanCoverage* coverage = nullptr);
|
||||
|
||||
} // namespace fold
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@
|
|||
#include <vector>
|
||||
|
||||
#include "finding.hpp"
|
||||
#include "scan_coverage.hpp"
|
||||
|
||||
namespace fold {
|
||||
|
||||
|
|
@ -13,7 +14,10 @@ const char* sarifRuleId(FindingKind kind);
|
|||
// Serialize findings as SARIF 2.1.0 JSON. Single run with tool name "fold".
|
||||
void writeSarif(std::ostream& out, const std::vector<Finding>& findings);
|
||||
|
||||
// Serialize a finding set that has already passed filterAndDedup.
|
||||
void writeFilteredSarif(std::ostream& out, const std::vector<Finding>& findings);
|
||||
// Serialize a finding set that has already passed filterAndDedup. With
|
||||
// `coverage`, the run carries an `invocations` entry recording which
|
||||
// translation units were not analyzed.
|
||||
void writeFilteredSarif(std::ostream& out, const std::vector<Finding>& findings,
|
||||
const ScanCoverage* coverage = nullptr);
|
||||
|
||||
} // namespace fold
|
||||
|
|
|
|||
43
include/fold/report/scan_coverage.hpp
Normal file
43
include/fold/report/scan_coverage.hpp
Normal file
|
|
@ -0,0 +1,43 @@
|
|||
#pragma once
|
||||
|
||||
#include <cstddef>
|
||||
#include <nlohmann/json.hpp>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
namespace fold {
|
||||
|
||||
// A translation unit the scan attempted but did not fully analyze.
|
||||
struct CoverageGap {
|
||||
std::string file;
|
||||
// compile-errors: the front end reported errors, so the AST is partial.
|
||||
// frontend-failed: no AST was produced (missing file, bad command line).
|
||||
// analysis-failed: the detectors threw on this TU.
|
||||
// timeout / crash / worker-exit: the isolated worker died on this TU.
|
||||
std::string reason;
|
||||
};
|
||||
|
||||
// What part of the requested input a scan actually analyzed. A report with
|
||||
// no findings is only evidence of absence when the coverage is complete.
|
||||
struct ScanCoverage {
|
||||
// False when the lane does not report per-TU coverage (binary, Rust,
|
||||
// Swift, reachability-only); such a scan is never marked incomplete.
|
||||
bool tracked = false;
|
||||
std::size_t tus_attempted = 0;
|
||||
std::size_t tus_analyzed = 0;
|
||||
std::vector<CoverageGap> gaps;
|
||||
|
||||
[[nodiscard]] bool incomplete() const {
|
||||
return tracked && (!gaps.empty() || (tus_attempted > 0 && tus_analyzed == 0));
|
||||
}
|
||||
|
||||
void merge(const ScanCoverage& other);
|
||||
};
|
||||
|
||||
[[nodiscard]] nlohmann::json coverageToJson(const ScanCoverage& coverage);
|
||||
|
||||
// SARIF 2.1.0 `invocations` entry: executionSuccessful plus one
|
||||
// toolExecutionNotification per gap.
|
||||
[[nodiscard]] nlohmann::json coverageToSarifInvocation(const ScanCoverage& coverage);
|
||||
|
||||
} // namespace fold
|
||||
|
|
@ -1,6 +1,7 @@
|
|||
#include "fold/cli/clang_shard_tu.hpp"
|
||||
|
||||
#include <clang/AST/ASTContext.h>
|
||||
#include <clang/Basic/Diagnostic.h>
|
||||
#include <clang/Basic/SourceManager.h>
|
||||
#include <spdlog/spdlog.h>
|
||||
|
||||
|
|
@ -8,33 +9,22 @@
|
|||
#include <filesystem>
|
||||
#include <iostream>
|
||||
#include <nlohmann/json.hpp>
|
||||
#include <unordered_map>
|
||||
|
||||
#include "fold/cli/scan_options.hpp"
|
||||
#include "fold/cli/signal_isolation.hpp"
|
||||
#include "fold/report/finding.hpp"
|
||||
#include "fold/report/finding_json.hpp"
|
||||
#include "fold/report/scan_coverage.hpp"
|
||||
#include "fold/symbolic/serialize.hpp"
|
||||
|
||||
namespace fold::cli {
|
||||
namespace {
|
||||
|
||||
std::string normalizeTuPath(const std::string& path) {
|
||||
if (path.empty())
|
||||
return path;
|
||||
std::error_code ec;
|
||||
std::filesystem::path p(path);
|
||||
std::filesystem::path abs = p.is_absolute() ? p : std::filesystem::absolute(p, ec);
|
||||
if (ec)
|
||||
return p.lexically_normal().string();
|
||||
auto canonical = std::filesystem::weakly_canonical(abs, ec);
|
||||
if (!ec)
|
||||
return canonical.string();
|
||||
return abs.lexically_normal().string();
|
||||
}
|
||||
|
||||
void emitWorkerTuPayload(std::ostream& out, const ClangShardTuState& state,
|
||||
std::size_t start_summaries, std::size_t start_func_summaries,
|
||||
std::size_t start_findings, const std::string& tu_path) {
|
||||
std::size_t start_findings, const std::string& tu_path,
|
||||
const std::string& gap_reason) {
|
||||
for (std::size_t i = start_summaries; i < state.summaries->size(); ++i) {
|
||||
out << nlohmann::json{{"kind", "summary"},
|
||||
{"data", fold::summaryToJson((*state.summaries)[i])}}
|
||||
|
|
@ -62,12 +52,29 @@ void emitWorkerTuPayload(std::ostream& out, const ClangShardTuState& state,
|
|||
state.findings->erase(state.findings->begin() + static_cast<std::ptrdiff_t>(start_findings),
|
||||
state.findings->end());
|
||||
|
||||
out << nlohmann::json{{"kind", "file_finished"}, {"data", {{"path", tu_path}}}}.dump() << '\n';
|
||||
out << nlohmann::json{{"kind", "file_finished"},
|
||||
{"data", {{"path", tu_path}, {"gap", gap_reason}}}}
|
||||
.dump()
|
||||
<< '\n';
|
||||
out.flush();
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
std::string normalizeClangTuPath(const std::string& path) {
|
||||
if (path.empty())
|
||||
return path;
|
||||
std::error_code ec;
|
||||
std::filesystem::path p(path);
|
||||
std::filesystem::path abs = p.is_absolute() ? p : std::filesystem::absolute(p, ec);
|
||||
if (ec)
|
||||
return p.lexically_normal().string();
|
||||
auto canonical = std::filesystem::weakly_canonical(abs, ec);
|
||||
if (!ec)
|
||||
return canonical.string();
|
||||
return abs.lexically_normal().string();
|
||||
}
|
||||
|
||||
std::string currentClangTuPath(const clang::ASTContext& ctx) {
|
||||
const auto& sm = ctx.getSourceManager();
|
||||
auto fid = sm.getMainFileID();
|
||||
|
|
@ -76,17 +83,18 @@ std::string currentClangTuPath(const clang::ASTContext& ctx) {
|
|||
return "<unknown>";
|
||||
}
|
||||
|
||||
void runClangShardTranslationUnit(const ScanOptions& opts, clang::ASTContext& ctx,
|
||||
ClangShardTuState state, ClangShardTuHooks hooks) {
|
||||
const std::string tu_path = normalizeTuPath(currentClangTuPath(ctx));
|
||||
ClangShardTuOutcome runClangShardTranslationUnit(const ScanOptions& opts, clang::ASTContext& ctx,
|
||||
ClangShardTuState state, ClangShardTuHooks hooks) {
|
||||
const std::string tu_path = normalizeClangTuPath(currentClangTuPath(ctx));
|
||||
ClangShardTuOutcome outcome{tu_path, "analysis-failed"};
|
||||
if (!state.summaries || !state.func_summaries || !state.findings ||
|
||||
!state.quarantined_this_run) {
|
||||
spdlog::warn("scan: skipped TU {} because shard state was not initialized", tu_path);
|
||||
return;
|
||||
return outcome;
|
||||
}
|
||||
if (state.quarantined_this_run->count(tu_path) != 0u) {
|
||||
spdlog::warn("scan: skipped TU {} already quarantined in this run", tu_path);
|
||||
return;
|
||||
return outcome;
|
||||
}
|
||||
|
||||
const std::size_t start_summaries = state.summaries->size();
|
||||
|
|
@ -99,9 +107,15 @@ void runClangShardTranslationUnit(const ScanOptions& opts, clang::ASTContext& ct
|
|||
hooks.handle_translation_unit(ctx);
|
||||
},
|
||||
opts.per_tu_timeout_seconds);
|
||||
// The front end recovers from errors and still hands over an AST,
|
||||
// but it is missing whatever failed to parse.
|
||||
const bool compile_errors = ctx.getDiagnostics().hasErrorOccurred();
|
||||
outcome.gap_reason = compile_errors ? "compile-errors" : "";
|
||||
if (compile_errors)
|
||||
spdlog::error("scan: {} has compile errors; its analysis is incomplete", tu_path);
|
||||
if (opts.is_worker) {
|
||||
emitWorkerTuPayload(std::cout, state, start_summaries, start_func_summaries,
|
||||
start_findings, tu_path);
|
||||
start_findings, tu_path, outcome.gap_reason);
|
||||
}
|
||||
} catch (const std::exception& e) {
|
||||
spdlog::warn("scan: skipped TU {} after exception: {}", tu_path, e.what());
|
||||
|
|
@ -112,6 +126,27 @@ void runClangShardTranslationUnit(const ScanOptions& opts, clang::ASTContext& ct
|
|||
if (opts.is_worker)
|
||||
::_Exit(1);
|
||||
}
|
||||
return outcome;
|
||||
}
|
||||
|
||||
ScanCoverage tuCoverage(const std::vector<std::string>& requested,
|
||||
const std::vector<ClangShardTuOutcome>& outcomes) {
|
||||
std::unordered_map<std::string, std::string> gap_by_path;
|
||||
for (const auto& outcome : outcomes)
|
||||
gap_by_path.insert_or_assign(normalizeClangTuPath(outcome.path), outcome.gap_reason);
|
||||
ScanCoverage coverage;
|
||||
coverage.tracked = true;
|
||||
coverage.tus_attempted = requested.size();
|
||||
for (const auto& file : requested) {
|
||||
const auto it = gap_by_path.find(normalizeClangTuPath(file));
|
||||
if (it == gap_by_path.end())
|
||||
coverage.gaps.push_back({file, "frontend-failed"});
|
||||
else if (!it->second.empty())
|
||||
coverage.gaps.push_back({file, it->second});
|
||||
else
|
||||
++coverage.tus_analyzed;
|
||||
}
|
||||
return coverage;
|
||||
}
|
||||
|
||||
} // namespace fold::cli
|
||||
|
|
|
|||
|
|
@ -402,6 +402,11 @@ int main(int argc, char** argv) {
|
|||
return rc == 0 ? 0 : 2;
|
||||
}
|
||||
|
||||
// Diagnostics go to stderr so a report written to stdout stays parseable
|
||||
// (docs/internal/cli-philosophy.md principle 12).
|
||||
spdlog::default_logger()->sinks().clear();
|
||||
spdlog::default_logger()->sinks().push_back(
|
||||
std::make_shared<spdlog::sinks::stderr_color_sink_mt>());
|
||||
if (quiet) {
|
||||
spdlog::set_level(spdlog::level::warn);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -107,16 +107,17 @@ bool writeStreamAtomically(const std::string& output_path, Writer&& writer) {
|
|||
}
|
||||
|
||||
int writeReport(const std::vector<fold::Finding>& report_findings, const std::string& output_path,
|
||||
const std::string& format, bool redundancy_secondary = false) {
|
||||
const std::string& format, const fold::ScanCoverage& coverage,
|
||||
bool redundancy_secondary = false) {
|
||||
const auto serialize = [&](std::ostream& out) {
|
||||
if (format == "sarif") {
|
||||
fold::writeFilteredSarif(out, report_findings);
|
||||
fold::writeFilteredSarif(out, report_findings, &coverage);
|
||||
} else if (format == "json") {
|
||||
spdlog::warn("report format 'json' is deprecated as a SARIF alias; use 'sarif' or "
|
||||
"'findings-json'");
|
||||
fold::writeFilteredSarif(out, report_findings);
|
||||
fold::writeFilteredSarif(out, report_findings, &coverage);
|
||||
} else if (format == "findings-json") {
|
||||
fold::writeFindingsJson(out, report_findings);
|
||||
fold::writeFindingsJson(out, report_findings, &coverage);
|
||||
} else if (format == "markdown" || format == "md") {
|
||||
fold::writeMarkdown(out, report_findings, redundancy_secondary);
|
||||
} else {
|
||||
|
|
@ -318,8 +319,9 @@ int runScan(const ScanCommand& cli) {
|
|||
}
|
||||
|
||||
const auto report_started = std::chrono::steady_clock::now();
|
||||
const int report_rc = writeReport(report_findings, cli.output_path, cli.format,
|
||||
opts.freestanding.redundancy_lane_secondary);
|
||||
const int report_rc =
|
||||
writeReport(report_findings, cli.output_path, cli.format, router_result.coverage,
|
||||
opts.freestanding.redundancy_lane_secondary);
|
||||
const auto report_finished = std::chrono::steady_clock::now();
|
||||
if (report_rc == 2)
|
||||
return 2;
|
||||
|
|
@ -338,6 +340,17 @@ int runScan(const ScanCommand& cli) {
|
|||
return 2;
|
||||
}
|
||||
|
||||
// A scan that skipped part of its input cannot vouch for the absence of
|
||||
// findings there, so it is an input failure (exit 2) even when it found
|
||||
// nothing.
|
||||
if (router_result.coverage.incomplete()) {
|
||||
const auto& coverage = router_result.coverage;
|
||||
spdlog::error("scan incomplete: analyzed {} of {} translation unit(s)",
|
||||
coverage.tus_analyzed, coverage.tus_attempted);
|
||||
for (const auto& gap : coverage.gaps)
|
||||
spdlog::error(" not analyzed ({}): {}", gap.reason, gap.file);
|
||||
return 2;
|
||||
}
|
||||
return report_rc;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -34,6 +34,7 @@ struct SourceRun {
|
|||
std::optional<ShardResult> shard;
|
||||
std::optional<ScanProfile> profile;
|
||||
std::vector<std::string> sources;
|
||||
ScanCoverage coverage;
|
||||
};
|
||||
|
||||
struct BinaryRun {
|
||||
|
|
@ -434,6 +435,7 @@ SourceRun runSource(ScanOptions opts) {
|
|||
fold::cli::SubprocessFanout fanout;
|
||||
const auto fanout_start = Clock::now();
|
||||
run.findings = fanout.runSync(opts, sources, processes);
|
||||
run.coverage = fanout.coverage();
|
||||
fanout_ms = durationMs(fanout_start, Clock::now());
|
||||
} else {
|
||||
spdlog::info("serial: running on {} files", sources.size());
|
||||
|
|
@ -443,6 +445,7 @@ SourceRun runSource(ScanOptions opts) {
|
|||
serial_ms = durationMs(serial_start, Clock::now());
|
||||
run.findings = run.shard->findings;
|
||||
run.profile = run.shard->profile;
|
||||
run.coverage = run.shard->coverage;
|
||||
}
|
||||
// Stamp the per-phase timings onto the profile so the perf-audit
|
||||
// doc can quote them. When --profile isn't on, run.profile is
|
||||
|
|
@ -762,6 +765,7 @@ ScanRouterResult runScanRouter(ScanOptions opts) {
|
|||
result.source_shard = std::move(source.shard);
|
||||
result.profile = std::move(source.profile);
|
||||
result.findings = std::move(source.findings);
|
||||
result.coverage = std::move(source.coverage);
|
||||
} else if (mode == "rust") {
|
||||
auto rust = runRust(opts);
|
||||
// Re-use the source-side context slots; findings are tagged
|
||||
|
|
@ -814,6 +818,7 @@ ScanRouterResult runScanRouter(ScanOptions opts) {
|
|||
result.context.binary_findings = binary.findings.size();
|
||||
result.source_shard = std::move(source.shard);
|
||||
result.binary_shard = std::move(binary.shard);
|
||||
result.coverage = std::move(source.coverage);
|
||||
result.profile = mergeScanProfilesForRouter(source.profile, binary.profile, opts);
|
||||
result.findings.reserve(source.findings.size() + binary.findings.size());
|
||||
result.findings.insert(result.findings.end(),
|
||||
|
|
|
|||
|
|
@ -583,14 +583,14 @@ ShardResult runClangDetectorsOnShard(clang::tooling::CompilationDatabase& db,
|
|||
clone_cache.get(),
|
||||
graph_security_candidates.empty() ? nullptr : &graph_security_candidates};
|
||||
auto factory = makeSourceTuFrontendFactory(tu_config, tu_state);
|
||||
(void)tool.run(factory.get());
|
||||
(void)tool.run(factory.get()); // which TUs failed comes from tu_outcomes
|
||||
out.coverage = tuCoverage(files, tu_state.tu_outcomes);
|
||||
}
|
||||
if (out.profile) {
|
||||
out.profile->source_tus_completed = tu_state.source_tus_completed;
|
||||
out.profile->source_tus_frontend_failed =
|
||||
out.profile->source_tus_attempted >= out.profile->source_tus_completed
|
||||
? out.profile->source_tus_attempted - out.profile->source_tus_completed
|
||||
: 0;
|
||||
out.profile->source_tus_attempted -
|
||||
std::min(out.profile->source_tus_attempted, out.profile->source_tus_completed);
|
||||
}
|
||||
|
||||
if (detector_plan.production_reachability)
|
||||
|
|
|
|||
|
|
@ -222,10 +222,11 @@ class ShardFrontend final : public clang::ASTFrontendAction {
|
|||
handleTranslationUnitImpl(tu_ctx);
|
||||
};
|
||||
hooks.quarantine_writer = quarantine_writer;
|
||||
runClangShardTranslationUnit(opts, ctx,
|
||||
ClangShardTuState{&summaries, &func_summaries,
|
||||
&findings, &quarantined_this_run},
|
||||
std::move(hooks));
|
||||
state.tu_outcomes.push_back(runClangShardTranslationUnit(
|
||||
opts, ctx,
|
||||
ClangShardTuState{&summaries, &func_summaries, &findings,
|
||||
&quarantined_this_run},
|
||||
std::move(hooks)));
|
||||
++state.source_tus_completed;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@
|
|||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
#include "fold/cli/clang_shard_tu.hpp"
|
||||
#include "fold/cli/tu_quarantine.hpp"
|
||||
#include "fold/detect/clone.hpp"
|
||||
#include "fold/detect/equiv.hpp"
|
||||
|
|
@ -83,6 +84,8 @@ struct ShardOutput {
|
|||
std::vector<FunctionSummary> summaries;
|
||||
std::vector<Finding> findings;
|
||||
std::vector<SerializedSummary> func_summaries;
|
||||
// file_finished records plus the TU each dead worker was processing.
|
||||
std::vector<ClangShardTuOutcome> tu_outcomes;
|
||||
bool success = true;
|
||||
};
|
||||
|
||||
|
|
@ -168,6 +171,8 @@ ShardOutput runShardWithRecovery(const std::string& binary,
|
|||
local_func_summaries.push_back(fromJson(data));
|
||||
} else if (kind == "file_finished") {
|
||||
finished_files.insert(normalizePath(data.value("path", "")));
|
||||
result.tu_outcomes.push_back(
|
||||
{data.value("path", ""), data.value("gap", "")});
|
||||
}
|
||||
} catch (const std::exception& e) {
|
||||
spdlog::warn("subprocess_fanout: bad NDJSON line: {}", e.what());
|
||||
|
|
@ -217,6 +222,8 @@ ShardOutput runShardWithRecovery(const std::string& binary,
|
|||
bool is_signal = WIFSIGNALED(status);
|
||||
const bool is_timeout =
|
||||
worker_read.timed_out || (is_signal && sig_or_status == SIGALRM);
|
||||
result.tu_outcomes.push_back(
|
||||
{failed_file, is_timeout ? "timeout" : (is_signal ? "crash" : "worker-exit")});
|
||||
if (is_timeout) {
|
||||
spdlog::warn("worker pid {} timed out after {}s while processing TU: {}", kid.pid,
|
||||
opts.per_tu_timeout_seconds, failed_file);
|
||||
|
|
@ -579,6 +586,7 @@ std::vector<Finding> SubprocessFanout::runSync(const ScanOptions& opts,
|
|||
std::vector<FunctionSummary> all_summaries;
|
||||
std::vector<Finding> all_findings;
|
||||
std::vector<SerializedSummary> all_func_summaries;
|
||||
std::vector<ClangShardTuOutcome> all_tu_outcomes;
|
||||
bool all_shards_succeeded = true;
|
||||
for (auto& t : tasks) {
|
||||
auto res = t.get();
|
||||
|
|
@ -590,6 +598,9 @@ std::vector<Finding> SubprocessFanout::runSync(const ScanOptions& opts,
|
|||
all_func_summaries.insert(all_func_summaries.end(),
|
||||
std::make_move_iterator(res.func_summaries.begin()),
|
||||
std::make_move_iterator(res.func_summaries.end()));
|
||||
all_tu_outcomes.insert(all_tu_outcomes.end(),
|
||||
std::make_move_iterator(res.tu_outcomes.begin()),
|
||||
std::make_move_iterator(res.tu_outcomes.end()));
|
||||
}
|
||||
{
|
||||
std::lock_guard<std::mutex> lk(mtx_);
|
||||
|
|
@ -597,6 +608,7 @@ std::vector<Finding> SubprocessFanout::runSync(const ScanOptions& opts,
|
|||
}
|
||||
if (!all_shards_succeeded)
|
||||
throw std::runtime_error("source worker failed before completing its shard");
|
||||
coverage_ = tuCoverage(files, all_tu_outcomes);
|
||||
|
||||
// Global clone clustering on the merged summary set. Multiple workers
|
||||
// re-#include the same headers and emit summaries for those header
|
||||
|
|
|
|||
|
|
@ -7,15 +7,17 @@
|
|||
|
||||
namespace fold {
|
||||
|
||||
void writeFindingsJson(std::ostream& out, const std::vector<Finding>& findings) {
|
||||
void writeFindingsJson(std::ostream& out, const std::vector<Finding>& findings,
|
||||
const ScanCoverage* coverage) {
|
||||
json encoded = json::array();
|
||||
for (const auto& finding : findings)
|
||||
encoded.push_back(findingToJson(finding));
|
||||
out << json{{"output_version", kFindingsJsonOutputVersion},
|
||||
{"tool", {{"name", "fold"}, {"version", foldVersion()}}},
|
||||
{"findings", encoded}}
|
||||
.dump(2)
|
||||
<< '\n';
|
||||
json document{{"output_version", kFindingsJsonOutputVersion},
|
||||
{"tool", {{"name", "fold"}, {"version", foldVersion()}}},
|
||||
{"findings", encoded}};
|
||||
if (coverage != nullptr)
|
||||
document["coverage"] = coverageToJson(*coverage);
|
||||
out << document.dump(2) << '\n';
|
||||
}
|
||||
|
||||
} // namespace fold
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ fold_core_sources += files(
|
|||
'findings_json_report.cpp',
|
||||
'markdown.cpp',
|
||||
'patch_writer.cpp',
|
||||
'scan_coverage.cpp',
|
||||
'sarif.cpp',
|
||||
'triage.cpp',
|
||||
)
|
||||
|
|
|
|||
|
|
@ -404,7 +404,8 @@ void writeSarif(std::ostream& out, const std::vector<Finding>& findings) {
|
|||
writeFilteredSarif(out, deduped);
|
||||
}
|
||||
|
||||
void writeFilteredSarif(std::ostream& out, const std::vector<Finding>& findings) {
|
||||
void writeFilteredSarif(std::ostream& out, const std::vector<Finding>& findings,
|
||||
const ScanCoverage* coverage) {
|
||||
const json tool = {
|
||||
{"driver",
|
||||
{
|
||||
|
|
@ -421,7 +422,10 @@ void writeFilteredSarif(std::ostream& out, const std::vector<Finding>& findings)
|
|||
|
||||
out << R"({"$schema":"https://json.schemastore.org/sarif-2.1.0.json",)"
|
||||
R"("version":"2.1.0","runs":[{"tool":)"
|
||||
<< tool.dump() << R"(,"results":[)";
|
||||
<< tool.dump();
|
||||
if (coverage != nullptr)
|
||||
out << R"(,"invocations":)" << json::array({coverageToSarifInvocation(*coverage)}).dump();
|
||||
out << R"(,"results":[)";
|
||||
for (std::size_t index = 0; index < findings.size(); ++index) {
|
||||
if (index != 0)
|
||||
out << ',';
|
||||
|
|
|
|||
47
src/report/scan_coverage.cpp
Normal file
47
src/report/scan_coverage.cpp
Normal file
|
|
@ -0,0 +1,47 @@
|
|||
#include "fold/report/scan_coverage.hpp"
|
||||
|
||||
namespace fold {
|
||||
|
||||
void ScanCoverage::merge(const ScanCoverage& other) {
|
||||
if (!other.tracked)
|
||||
return;
|
||||
tracked = true;
|
||||
tus_attempted += other.tus_attempted;
|
||||
tus_analyzed += other.tus_analyzed;
|
||||
gaps.insert(gaps.end(), other.gaps.begin(), other.gaps.end());
|
||||
}
|
||||
|
||||
nlohmann::json coverageToJson(const ScanCoverage& coverage) {
|
||||
nlohmann::json gaps = nlohmann::json::array();
|
||||
for (const auto& gap : coverage.gaps)
|
||||
gaps.push_back({{"file", gap.file}, {"reason", gap.reason}});
|
||||
return {{"tracked", coverage.tracked},
|
||||
{"complete", !coverage.incomplete()},
|
||||
{"tus_attempted", coverage.tus_attempted},
|
||||
{"tus_analyzed", coverage.tus_analyzed},
|
||||
{"gaps", gaps}};
|
||||
}
|
||||
|
||||
nlohmann::json coverageToSarifInvocation(const ScanCoverage& coverage) {
|
||||
nlohmann::json notifications = nlohmann::json::array();
|
||||
for (const auto& gap : coverage.gaps) {
|
||||
notifications.push_back(
|
||||
{{"level", "error"},
|
||||
{"descriptor", {{"id", "fold/coverage/" + gap.reason}}},
|
||||
{"message", {{"text", "translation unit not analyzed (" + gap.reason + ")"}}},
|
||||
{"locations",
|
||||
nlohmann::json::array(
|
||||
{{{"physicalLocation", {{"artifactLocation", {{"uri", gap.file}}}}}}})}});
|
||||
}
|
||||
if (coverage.tracked && coverage.tus_attempted > 0 && coverage.tus_analyzed == 0 &&
|
||||
coverage.gaps.empty()) {
|
||||
notifications.push_back({{"level", "error"},
|
||||
{"descriptor", {{"id", "fold/coverage/none-analyzed"}}},
|
||||
{"message", {{"text", "no translation unit was analyzed"}}}});
|
||||
}
|
||||
return {{"executionSuccessful", !coverage.incomplete()},
|
||||
{"toolExecutionNotifications", notifications},
|
||||
{"properties", {{"coverage", coverageToJson(coverage)}}}};
|
||||
}
|
||||
|
||||
} // namespace fold
|
||||
|
|
@ -476,3 +476,47 @@ TEST_CASE("fold scan formats have truthful distinct contracts", "[cli][contract]
|
|||
REQUIRE(log_text.find(input_type_warning, log_text.find(input_type_warning) + 1) ==
|
||||
std::string::npos);
|
||||
}
|
||||
|
||||
TEST_CASE("fold scan fails when a translation unit is not analyzed", "[cli][coverage]") {
|
||||
// A TU that failed to parse used to leave no trace: no findings, exit 0.
|
||||
const std::string fold_bin = locateFoldBinary();
|
||||
if (fold_bin.empty()) {
|
||||
WARN("could not locate fold binary; skipping CLI coverage test");
|
||||
return;
|
||||
}
|
||||
|
||||
TempDir dir;
|
||||
const auto good = dir.path / "good.cpp";
|
||||
const auto broken = dir.path / "broken.cpp";
|
||||
writeFile(good, "int add(int a, int b) { return a + b; }\n");
|
||||
writeFile(broken, "#include \"does_not_exist.h\"\nint f(int x) { return x; }\n");
|
||||
|
||||
const auto scan = [&](const std::filesystem::path& source, const std::string& format,
|
||||
const std::filesystem::path& report) {
|
||||
const std::string command = "\"" + fold_bin + "\" --quiet scan --detectors div-by-zero " +
|
||||
"--format " + format + " \"" + source.string() + "\" > \"" +
|
||||
report.string() + "\" 2>/dev/null";
|
||||
return commandExitCode(std::system(command.c_str()));
|
||||
};
|
||||
|
||||
SECTION("clean TU: complete coverage, exit 0 (control)") {
|
||||
const auto report = dir.path / "good.json";
|
||||
REQUIRE(scan(good, "findings-json", report) == 0);
|
||||
const auto json = nlohmann::json::parse(readFile(report));
|
||||
CHECK(json["coverage"]["complete"] == true);
|
||||
CHECK(json["coverage"]["tus_analyzed"] == 1);
|
||||
}
|
||||
SECTION("broken TU: gap in findings JSON, exit 2") {
|
||||
const auto report = dir.path / "broken.json";
|
||||
REQUIRE(scan(broken, "findings-json", report) == 2);
|
||||
const auto json = nlohmann::json::parse(readFile(report));
|
||||
CHECK(json["coverage"]["complete"] == false);
|
||||
CHECK(json["coverage"]["gaps"][0]["reason"] == "compile-errors");
|
||||
}
|
||||
SECTION("broken TU: SARIF on stdout stays parseable and records the failure") {
|
||||
const auto report = dir.path / "broken.sarif";
|
||||
REQUIRE(scan(broken, "sarif", report) == 2);
|
||||
const auto sarif = nlohmann::json::parse(readFile(report));
|
||||
CHECK(sarif["runs"][0]["invocations"][0]["executionSuccessful"] == false);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -213,6 +213,15 @@ test_proved_requires_exact_model = executable(
|
|||
)
|
||||
test('proved_requires_exact_model', test_proved_requires_exact_model)
|
||||
|
||||
test_scan_coverage = executable(
|
||||
'test_scan_coverage',
|
||||
'test_scan_coverage.cpp',
|
||||
dependencies: [fold_core_dep, catch2_dep, json_dep],
|
||||
install: false,
|
||||
build_by_default: false,
|
||||
)
|
||||
test('scan_coverage', test_scan_coverage)
|
||||
|
||||
test_batch_safety_observer = executable(
|
||||
'test_batch_safety_observer',
|
||||
'test_batch_safety_observer.cpp',
|
||||
|
|
|
|||
112
tests/unit/test_scan_coverage.cpp
Normal file
112
tests/unit/test_scan_coverage.cpp
Normal file
|
|
@ -0,0 +1,112 @@
|
|||
#include <catch2/catch_test_macros.hpp>
|
||||
#include <filesystem>
|
||||
#include <nlohmann/json.hpp>
|
||||
#include <sstream>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "fold/cli/clang_shard_tu.hpp"
|
||||
#include "fold/report/findings_json_report.hpp"
|
||||
#include "fold/report/sarif.hpp"
|
||||
#include "fold/report/scan_coverage.hpp"
|
||||
|
||||
// A TU the front end could not parse used to vanish: the report had no trace
|
||||
// of it and the scan exited 0, indistinguishable from a clean scan. Coverage
|
||||
// now records every requested TU that was not analyzed on a clean AST.
|
||||
|
||||
namespace {
|
||||
|
||||
std::string tmp(const std::string& name) {
|
||||
return (std::filesystem::temp_directory_path() / name).string();
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
TEST_CASE("tuCoverage classifies every requested TU", "[coverage]") {
|
||||
const std::vector<std::string> requested{tmp("ok.cpp"), tmp("broken.cpp"), tmp("missing.cpp"),
|
||||
tmp("slow.cpp")};
|
||||
const std::vector<fold::cli::ClangShardTuOutcome> outcomes{
|
||||
{tmp("ok.cpp"), ""},
|
||||
{tmp("broken.cpp"), "compile-errors"},
|
||||
{tmp("slow.cpp"), "timeout"},
|
||||
};
|
||||
const auto coverage = fold::cli::tuCoverage(requested, outcomes);
|
||||
CHECK(coverage.tracked);
|
||||
CHECK(coverage.tus_attempted == 4);
|
||||
CHECK(coverage.tus_analyzed == 1);
|
||||
REQUIRE(coverage.gaps.size() == 3);
|
||||
CHECK(coverage.gaps[0].file == tmp("broken.cpp"));
|
||||
CHECK(coverage.gaps[0].reason == "compile-errors");
|
||||
CHECK(coverage.gaps[1].file == tmp("missing.cpp"));
|
||||
CHECK(coverage.gaps[1].reason == "frontend-failed");
|
||||
CHECK(coverage.gaps[2].reason == "timeout");
|
||||
CHECK(coverage.incomplete());
|
||||
}
|
||||
|
||||
TEST_CASE("tuCoverage matches paths across spellings", "[coverage]") {
|
||||
// The compile DB may spell a path differently than the front end does.
|
||||
const auto dir = std::filesystem::temp_directory_path();
|
||||
const std::string dotted = (dir / "sub" / ".." / "a.cpp").string();
|
||||
const auto coverage = fold::cli::tuCoverage({dotted}, {{(dir / "a.cpp").string(), ""}});
|
||||
CHECK(coverage.tus_analyzed == 1);
|
||||
CHECK_FALSE(coverage.incomplete());
|
||||
}
|
||||
|
||||
TEST_CASE("coverage: complete, untracked and empty scans", "[coverage]") {
|
||||
SECTION("every TU analyzed is complete (control)") {
|
||||
const auto coverage = fold::cli::tuCoverage({tmp("a.cpp")}, {{tmp("a.cpp"), ""}});
|
||||
CHECK_FALSE(coverage.incomplete());
|
||||
}
|
||||
SECTION("an untracked lane is never incomplete") {
|
||||
fold::ScanCoverage coverage;
|
||||
coverage.tus_attempted = 3;
|
||||
CHECK_FALSE(coverage.incomplete());
|
||||
}
|
||||
SECTION("attempted TUs with none analyzed is incomplete even without gaps") {
|
||||
fold::ScanCoverage coverage;
|
||||
coverage.tracked = true;
|
||||
coverage.tus_attempted = 2;
|
||||
CHECK(coverage.incomplete());
|
||||
}
|
||||
SECTION("merge sums counts and keeps gaps; an untracked side is ignored") {
|
||||
auto a = fold::cli::tuCoverage({tmp("a.cpp")}, {{tmp("a.cpp"), ""}});
|
||||
const auto b = fold::cli::tuCoverage({tmp("b.cpp")}, {});
|
||||
a.merge(b);
|
||||
a.merge(fold::ScanCoverage{});
|
||||
CHECK(a.tus_attempted == 2);
|
||||
CHECK(a.tus_analyzed == 1);
|
||||
CHECK(a.gaps.size() == 1);
|
||||
}
|
||||
}
|
||||
|
||||
TEST_CASE("reports carry coverage", "[coverage][report]") {
|
||||
const auto coverage =
|
||||
fold::cli::tuCoverage({tmp("broken.cpp")}, {{tmp("broken.cpp"), "compile-errors"}});
|
||||
SECTION("SARIF: invocation is unsuccessful and names the TU") {
|
||||
std::ostringstream out;
|
||||
fold::writeFilteredSarif(out, {}, &coverage);
|
||||
const auto sarif = nlohmann::json::parse(out.str());
|
||||
const auto& invocation = sarif["runs"][0]["invocations"][0];
|
||||
CHECK(invocation["executionSuccessful"] == false);
|
||||
const auto& note = invocation["toolExecutionNotifications"][0];
|
||||
CHECK(note["level"] == "error");
|
||||
CHECK(note["locations"][0]["physicalLocation"]["artifactLocation"]["uri"] ==
|
||||
tmp("broken.cpp"));
|
||||
CHECK(invocation["properties"]["coverage"]["tus_analyzed"] == 0);
|
||||
}
|
||||
SECTION("findings JSON: coverage object") {
|
||||
std::ostringstream out;
|
||||
fold::writeFindingsJson(out, {}, &coverage);
|
||||
const auto report = nlohmann::json::parse(out.str());
|
||||
CHECK(report["coverage"]["complete"] == false);
|
||||
CHECK(report["coverage"]["gaps"][0]["reason"] == "compile-errors");
|
||||
}
|
||||
SECTION("no coverage argument leaves the documents unchanged") {
|
||||
std::ostringstream sarif_out;
|
||||
fold::writeFilteredSarif(sarif_out, {});
|
||||
CHECK_FALSE(nlohmann::json::parse(sarif_out.str())["runs"][0].contains("invocations"));
|
||||
std::ostringstream json_out;
|
||||
fold::writeFindingsJson(json_out, {});
|
||||
CHECK_FALSE(nlohmann::json::parse(json_out.str()).contains("coverage"));
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue