trevon/fold
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions
fold/tests/integration/test_cli_paths.cpp
trvon c9811b7d41
 fix(scan): report unanalyzed translation units and exit 2 
Problem: a TU the front end could not parse left no trace. ClangTool::run's
status was discarded, TUs with compile errors were analyzed on a partial
AST and counted as done, and the isolated-worker path dropped TUs that
never produced an AST. A scan of only broken files printed no findings and
exited 0, the same as a clean scan. Separately, the scan logger wrote to
stdout, so any warning corrupted a SARIF/JSON report written to stdout.

Fix: every TU now yields an outcome (clean, compile-errors,
analysis-failed); workers carry it on their file_finished record, and the
fan-out parent adds timeout/crash/worker-exit for the TU a worker died on.
tuCoverage() turns the requested file list plus outcomes into a
ScanCoverage (requested TUs with no outcome are frontend-failed). SARIF
gets a run `invocations` entry (executionSuccessful, one
toolExecutionNotification per gap); findings JSON gets an additive
`coverage` object. A scan with any gap, or with nothing analyzed, exits 2
after writing its report. CLI diagnostics now go to stderr, as
docs/internal/cli-philosophy.md principle 12 requires. USAGE.md and
AGENTS.md document the coverage fields and the exit code.

Test: tests/unit/test_scan_coverage.cpp (classification, path spelling,
complete/untracked controls, merge, both writers, and unchanged output
without coverage); test_cli_paths "[coverage]" covers a clean TU (exit 0,
complete) and a TU with a missing include (exit 2, gap in JSON, SARIF on
stdout parseable with executionSuccessful=false). Before: both probe files
exited 0 with no coverage information. Full suite: 139/139.
2026-09-24 20:37:51 -06:00

522 lines
24 KiB
C++
Raw Permalink Blame History

#include <sys/wait.h>
#include <catch2/catch_test_macros.hpp>
#include <cctype>
#include <chrono>
#include <cstdlib>
#include <filesystem>
#include <fstream>
#include <nlohmann/json.hpp>
#include <sstream>
#include <string>
#include "../../include/fold/cli/scan_option_catalog.hpp"
#include "../../include/fold/mcp/scan_cache.hpp"
#include "../../include/fold/mcp/tool_registry.hpp"
#include "../../include/fold/report/findings_json_report.hpp"
namespace {
struct TempDir {
std::filesystem::path path;
TempDir()
: path(std::filesystem::temp_directory_path() /
("fold-cli-paths-" +
std::to_string(std::chrono::steady_clock::now().time_since_epoch().count()))) {
std::filesystem::create_directories(path);
}
~TempDir() { std::filesystem::remove_all(path); }
};
void writeFile(const std::filesystem::path& path, const std::string& content) {
std::ofstream(path) << content;
}
std::string readFile(const std::filesystem::path& path) {
std::ifstream in(path);
std::ostringstream out;
out << in.rdbuf();
return out.str();
}
int commandExitCode(int status) { return WIFEXITED(status) ? WEXITSTATUS(status) : -1; }
std::string locateFoldBinary() {
if (const char* env = std::getenv("FOLD_BINARY")) {
if (*env && std::filesystem::exists(env)) {
return std::filesystem::canonical(env).string();
}
}
auto cwd = std::filesystem::current_path();
for (const auto& rel : {"../../src/cli/fold", "../../../src/cli/fold", "../src/cli/fold"}) {
auto candidate = cwd / rel;
if (std::filesystem::exists(candidate)) {
return std::filesystem::canonical(candidate).string();
}
}
return "";
}
} // namespace
TEST_CASE("fold help advertises embedded skill command", "[cli][help][skill]") {
const std::string fold_bin = locateFoldBinary();
if (fold_bin.empty()) {
WARN("could not locate fold binary; skipping CLI help test");
return;
}
TempDir dir;
const auto help = dir.path / "help.txt";
const auto skill = dir.path / "skill.md";
REQUIRE(std::system(("\"" + fold_bin + "\" --help > \"" + help.string() + "\" 2>&1").c_str()) ==
0);
REQUIRE(std::system(("\"" + fold_bin + "\" skill -o \"" + skill.string() + "\"").c_str()) == 0);
const auto help_text = readFile(help);
REQUIRE(help_text.find("skill") != std::string::npos);
REQUIRE(help_text.find("bounded symbolic SAST") != std::string::npos);
const auto skill_text = readFile(skill);
REQUIRE(skill_text.find("name: skill-fold-sast") != std::string::npos);
REQUIRE(skill_text.find("concrete-witness") != std::string::npos);
REQUIRE(skill_text.find("fold fuzz prep") != std::string::npos);
const char* canonical_skill = std::getenv("FOLD_SKILL_SOURCE");
REQUIRE(canonical_skill != nullptr);
REQUIRE(skill_text == readFile(canonical_skill));
}
TEST_CASE("fold scan honors positional paths after repeatable detector flags", "[cli][paths]") {
const std::string fold_bin = locateFoldBinary();
if (fold_bin.empty()) {
WARN("could not locate fold binary; skipping CLI paths test");
return;
}
TempDir dir;
const auto first = dir.path / "first.cpp";
const auto second = dir.path / "second.cpp";
const auto db = dir.path / "compile_commands.json";
const auto out = dir.path / "scan.json";
const auto log = dir.path / "scan.log";
writeFile(first, "int first(int x) {\n"
" int y = x + 1;\n"
" return y * 2;\n"
"}\n");
writeFile(second, "int second(int x) {\n"
" int y = x + 1;\n"
" return y * 2;\n"
"}\n");
writeFile(db, nlohmann::json::array(
{
{{"directory", dir.path.string()},
{"command", std::string("clang++ -std=c++20 -c ") + first.string()},
{"file", first.string()}},
{{"directory", dir.path.string()},
{"command", std::string("clang++ -std=c++20 -c ") + second.string()},
{"file", second.string()}},
})
.dump(2));
const std::string cmd = "\"" + fold_bin + "\" scan --compile-db \"" + db.string() +
"\" --format json --output \"" + out.string() +
"\" --processes 1 --detectors clone \"" + first.string() + "\" > \"" +
log.string() + "\" 2>&1";
REQUIRE(std::system(cmd.c_str()) == 0);
std::ifstream in(out);
REQUIRE(in.good());
const auto report = nlohmann::json::parse(in);
REQUIRE(report["runs"].size() == 1);
REQUIRE(report["runs"][0]["results"].empty());
}
TEST_CASE("fold scan exclude globs support globstar and brace alternatives",
"[cli][paths][exclude-glob]") {
const std::string fold_bin = locateFoldBinary();
if (fold_bin.empty()) {
WARN("could not locate fold binary; skipping CLI exclude-glob test");
return;
}
TempDir dir;
const auto safe = dir.path / "safe.cpp";
const auto vendor_dir = dir.path / "nested" / "third_party";
const auto generated_dir = dir.path / "nested" / "generated";
std::filesystem::create_directories(vendor_dir);
std::filesystem::create_directories(generated_dir);
const auto vendor = vendor_dir / "unsafe.cpp";
const auto generated = generated_dir / "unsafe.pb.cc";
const auto db = dir.path / "compile_commands.json";
const auto out = dir.path / "scan.json";
const auto log = dir.path / "scan.log";
const auto profile = dir.path / "scan.profile.json";
writeFile(safe, "int safe() { return 1; }\n");
writeFile(vendor, "int vendor_bad() { return 1 / 0; }\n");
writeFile(generated, "int generated_bad() { return 1 / 0; }\n");
nlohmann::json commands = nlohmann::json::array();
for (const auto& file : {safe, vendor, generated}) {
commands.push_back({
{"directory", dir.path.string()},
{"command", std::string("clang++ -std=c++20 -c ") + file.string()},
{"file", file.string()},
});
}
writeFile(db, commands.dump(2));
const std::string cmd =
"\"" + fold_bin + "\" scan --compile-db \"" + db.string() + "\" --format json --output \"" +
out.string() +
"\" --processes 1 --detectors div-by-zero --exclude-glob 'third_party/**' "
"--exclude-glob '**/*.pb.{h,cc}' --profile --profile-output \"" +
profile.string() + "\" > \"" + log.string() + "\" 2>&1";
const int status = std::system(cmd.c_str());
INFO(readFile(log));
REQUIRE(status == 0);
const auto report = nlohmann::json::parse(std::ifstream(out));
REQUIRE(report["runs"][0]["results"].empty());
REQUIRE(readFile(log).find("3 -> 1 files (2 skipped)") != std::string::npos);
const auto profile_json = nlohmann::json::parse(std::ifstream(profile));
REQUIRE(profile_json["scan"]["source_tus_selected"] == 1);
REQUIRE(profile_json["scan"]["source_tus_excluded"] == 2);
REQUIRE(profile_json["scan"]["source_tus_attempted"] == 1);
REQUIRE(profile_json["scan"]["source_tus_completed"] == 1);
REQUIRE(profile_json["scan"]["source_tus_frontend_failed"] == 0);
}
TEST_CASE("fold scan artifacts share report-visible counts and commit atomically",
"[cli][artifacts][counts]") {
const std::string fold_bin = locateFoldBinary();
if (fold_bin.empty()) {
WARN("could not locate fold binary; skipping artifact contract test");
return;
}
TempDir dir;
const auto source = dir.path / "unsafe.cpp";
const auto db = dir.path / "compile_commands.json";
const auto sarif = dir.path / "serial.sarif";
const auto native = dir.path / "fanout.json";
const auto profile = dir.path / "serial.profile.json";
const auto serial_log = dir.path / "serial.log";
const auto fanout_log = dir.path / "fanout.log";
const auto version_path = dir.path / "version.txt";
REQUIRE(std::system(
("\"" + fold_bin + "\" --version > \"" + version_path.string() + "\"").c_str()) ==
0);
std::string fold_version = readFile(version_path);
while (!fold_version.empty() && std::isspace(static_cast<unsigned char>(fold_version.back())))
fold_version.pop_back();
REQUIRE_FALSE(fold_version.empty());
writeFile(source, "int unsafe_divisor(int value) { return 10 / value; }\n");
writeFile(db, nlohmann::json::array({{{"directory", dir.path.string()},
{"command", "clang++ -std=c++20 -c " + source.string()},
{"file", source.string()}}})
.dump(2));
const std::string common =
"\"" + fold_bin + "\" scan --compile-db \"" + db.string() + "\" --detectors div-by-zero ";
const std::string serial_command =
common + "--processes 1 --format sarif --output \"" + sarif.string() +
"\" --profile-output \"" + profile.string() + "\" > \"" + serial_log.string() + "\" 2>&1";
REQUIRE(commandExitCode(std::system(serial_command.c_str())) == 1);
const auto sarif_report = nlohmann::json::parse(std::ifstream(sarif));
REQUIRE(sarif_report["runs"][0]["results"].size() == 1);
REQUIRE(sarif_report["runs"][0]["tool"]["driver"]["version"] == fold_version);
REQUIRE(std::filesystem::is_regular_file(profile));
const auto profile_report = nlohmann::json::parse(std::ifstream(profile));
REQUIRE(profile_report["version"] == fold_version);
REQUIRE(profile_report["scan"]["findings_before_filter"] >= 1);
REQUIRE(profile_report["scan"]["findings_report_visible"] == 1);
REQUIRE(readFile(serial_log).find("emitted 1 report-visible findings total") !=
std::string::npos);
const std::string fanout_command = common + "--processes 2 --format findings-json --output \"" +
native.string() + "\" > \"" + fanout_log.string() +
"\" 2>&1";
REQUIRE(commandExitCode(std::system(fanout_command.c_str())) == 1);
const auto native_report = nlohmann::json::parse(std::ifstream(native));
REQUIRE(native_report["findings"].size() == 1);
REQUIRE(native_report["tool"]["version"] == fold_version);
REQUIRE(readFile(fanout_log).find("emitted 1 report-visible findings total") !=
std::string::npos);
const auto invalid_target = dir.path / "report-directory";
std::filesystem::create_directories(invalid_target);
const std::string failed_command = common + "--format findings-json --output \"" +
invalid_target.string() + "\" >/dev/null 2>&1";
REQUIRE(commandExitCode(std::system(failed_command.c_str())) == 2);
for (const auto& entry : std::filesystem::directory_iterator(dir.path)) {
REQUIRE(entry.path().filename().string().find("report-directory.tmp.") != 0);
}
const auto invalid_profile = dir.path / "profile-directory";
const auto valid_report = dir.path / "profile-failure-report.json";
std::filesystem::create_directories(invalid_profile);
const std::string profile_failure_command = common + "--format findings-json --output \"" +
valid_report.string() + "\" --profile-output \"" +
invalid_profile.string() + "\" >/dev/null 2>&1";
REQUIRE(commandExitCode(std::system(profile_failure_command.c_str())) == 2);
REQUIRE(std::filesystem::is_directory(invalid_profile));
for (const auto& entry : std::filesystem::directory_iterator(dir.path)) {
REQUIRE(entry.path().filename().string().find("profile-directory.tmp.") != 0);
}
const auto collision = dir.path / "collision.json";
const std::string collision_command = common + "--output \"" + collision.string() +
"\" --profile-output \"" + collision.string() +
"\" >/dev/null 2>&1";
REQUIRE(commandExitCode(std::system(collision_command.c_str())) == 2);
REQUIRE_FALSE(std::filesystem::exists(collision));
}
TEST_CASE("fold scan help separates common and advanced options", "[cli][contract][help]") {
const std::string fold_bin = locateFoldBinary();
if (fold_bin.empty()) {
WARN("could not locate fold binary; skipping CLI help contract test");
return;
}
TempDir dir;
const auto help = dir.path / "scan-help.txt";
const auto advanced = dir.path / "scan-advanced-help.txt";
REQUIRE(std::system(
("\"" + fold_bin + "\" scan --help > \"" + help.string() + "\" 2>&1").c_str()) ==
0);
REQUIRE(std::system(
("\"" + fold_bin + "\" scan --advanced-help > \"" + advanced.string() + "\" 2>&1")
.c_str()) == 0);
const auto help_text = readFile(help);
const auto advanced_text = readFile(advanced);
REQUIRE(help_text.find("--paths") == std::string::npos);
REQUIRE(help_text.find("--advanced-help") != std::string::npos);
REQUIRE(advanced_text.find("Advanced scan options") != std::string::npos);
for (const auto& option : fold::cli::scanOptionCatalog()) {
if (!option.cli_exposed || !option.cli_flag.starts_with("--"))
continue;
const auto& expected = option.advanced ? advanced_text : help_text;
INFO("catalog option missing from expected help surface: " << option.cli_flag);
REQUIRE(expected.find(option.cli_flag) != std::string::npos);
}
}
TEST_CASE("fold scan rejects catalog-invalid numeric options", "[cli][contract][bounds]") {
const std::string fold_bin = locateFoldBinary();
if (fold_bin.empty()) {
WARN("could not locate fold binary; skipping CLI bounds test");
return;
}
for (const auto& arguments : {"--clone-type 1", "--clone-similarity 2", "--budget-seconds -1",
"--binary-max-functions -1"}) {
INFO(arguments);
const std::string command =
"\"" + fold_bin + "\" scan --advanced-help " + arguments + " >/dev/null 2>&1";
REQUIRE(commandExitCode(std::system(command.c_str())) != 0);
}
}
TEST_CASE("fold grep queries finding reports with machine-readable metadata",
"[cli][grep][contract]") {
const std::string fold_bin = locateFoldBinary();
if (fold_bin.empty()) {
WARN("could not locate fold binary; skipping CLI grep test");
return;
}
TempDir dir;
const auto report_path = dir.path / "findings.json";
const auto result_path = dir.path / "query.json";
const auto log_path = dir.path / "query.log";
fold::Finding source;
source.id = "finding-a";
source.kind = fold::FindingKind::NullDeref;
source.confidence = fold::Confidence::Proved;
source.message = "Null token reaches parser";
source.evidence.backend = "source";
source.locations.push_back({"src/auth.cpp", 10, 2, "parse"});
fold::Finding suppressed;
suppressed.id = "finding-b";
suppressed.kind = fold::FindingKind::HardcodedCredential;
suppressed.confidence = fold::Confidence::Heuristic;
suppressed.message = "Embedded token";
suppressed.evidence.backend = "binary";
suppressed.locations.push_back({"bin/app", 1, 1, "sym.auth"});
suppressed.suppression_reason = "accepted fixture";
{
std::ofstream report(report_path);
fold::writeFindingsJson(report, {source, suppressed});
}
const auto command = "\"" + fold_bin + "\" grep TOKEN \"" + report_path.string() +
"\" --ignore-case --field message --format json --output \"" +
result_path.string() + "\" > \"" + log_path.string() + "\" 2>&1";
REQUIRE(commandExitCode(std::system(command.c_str())) == 1);
const auto result = nlohmann::json::parse(std::ifstream(result_path));
REQUIRE(result["output_version"] == 1);
REQUIRE(result["total_input"] == 2);
REQUIRE(result["total_matches"] == 2);
REQUIRE(result["total_actionable_matches"] == 1);
REQUIRE(result["findings"][0]["id"] == "finding-a");
REQUIRE(result["findings"][1]["id"] == "finding-b");
fold::mcp::ScanCache cache;
cache.replaceAll({source, suppressed});
fold::mcp::ToolRegistry registry(cache);
const auto mcp_result =
registry.call("fold.query_findings", {{"pattern", "TOKEN"},
{"ignore_case", true},
{"fields", nlohmann::json::array({"message"})},
{"limit", 100}});
const auto& mcp_query = mcp_result["structuredContent"];
REQUIRE(mcp_query["total_input"] == result["total_input"]);
REQUIRE(mcp_query["total_matches"] == result["total_matches"]);
REQUIRE(mcp_query["total_actionable_matches"] == result["total_actionable_matches"]);
REQUIRE(mcp_query["findings"][0]["id"] == result["findings"][0]["id"]);
REQUIRE(mcp_query["findings"][1]["id"] == result["findings"][1]["id"]);
const auto suppressed_command = "\"" + fold_bin + "\" grep token \"" + report_path.string() +
"\" --ignore-case --suppressed --format json --output \"" +
result_path.string() + "\" > \"" + log_path.string() +
"\" 2>&1";
REQUIRE(commandExitCode(std::system(suppressed_command.c_str())) == 0);
const auto suppressed_result = nlohmann::json::parse(std::ifstream(result_path));
REQUIRE(suppressed_result["total_matches"] == 1);
REQUIRE(suppressed_result["total_actionable_matches"] == 0);
REQUIRE(suppressed_result["findings"][0]["id"] == "finding-b");
}
TEST_CASE("fold scan rejects unimplemented hybrid strategies", "[cli][contract][hybrid]") {
const std::string fold_bin = locateFoldBinary();
if (fold_bin.empty()) {
WARN("could not locate fold binary; skipping CLI hybrid contract test");
return;
}
TempDir dir;
const auto log = dir.path / "hybrid.log";
const auto command = "\"" + fold_bin +
"\" scan --mode hybrid --hybrid-strategy binary-first > \"" +
log.string() + "\" 2>&1";
REQUIRE(std::system(command.c_str()) != 0);
const auto output = readFile(log);
REQUIRE(output.find("binary-first") != std::string::npos);
REQUIRE(output.find("merge-only") != std::string::npos);
}
TEST_CASE("fold scan formats have truthful distinct contracts", "[cli][contract][format]") {
const std::string fold_bin = locateFoldBinary();
if (fold_bin.empty()) {
WARN("could not locate fold binary; skipping CLI format contract test");
return;
}
TempDir dir;
const auto source = dir.path / "safe.cpp";
const auto sarif_path = dir.path / "report.sarif.json";
const auto json_path = dir.path / "report.json";
const auto native_path = dir.path / "report.findings.json";
const auto markdown_path = dir.path / "report.md";
const auto log = dir.path / "scan.log";
writeFile(source, "int safe(int value) { return value + 1; }\n");
const std::string common = "\"" + fold_bin + "\" scan --mode source --processes 1 " +
"--detectors complexity \"" + source.string() + "\" ";
REQUIRE(std::system((common + "--format sarif --output \"" + sarif_path.string() + "\" > \"" +
log.string() + "\" 2>&1")
.c_str()) == 0);
REQUIRE(std::system((common + "--format json --output \"" + json_path.string() + "\" >> \"" +
log.string() + "\" 2>&1")
.c_str()) == 0);
REQUIRE(std::system((common + "--format findings-json --output \"" + native_path.string() +
"\" >> \"" + log.string() + "\" 2>&1")
.c_str()) == 0);
REQUIRE(std::system((common + "--format markdown --output \"" + markdown_path.string() +
"\" >> \"" + log.string() + "\" 2>&1")
.c_str()) == 0);
REQUIRE(std::system((common + "--null-deref-nullable-allocations --format sarif --output \"" +
sarif_path.string() + "\" >> \"" + log.string() + "\" 2>&1")
.c_str()) == 0);
const std::string legacy_mode =
"\"" + fold_bin + "\" scan --input-type source --processes 1 --detectors complexity \"" +
source.string() + "\" --format sarif --output \"" + sarif_path.string() + "\" >> \"" +
log.string() + "\" 2>&1";
REQUIRE(std::system(legacy_mode.c_str()) == 0);
const auto sarif = nlohmann::json::parse(std::ifstream(sarif_path));
const auto json_alias = nlohmann::json::parse(std::ifstream(json_path));
const auto native = nlohmann::json::parse(std::ifstream(native_path));
REQUIRE(sarif == json_alias);
REQUIRE(json_alias["version"] == "2.1.0");
REQUIRE(native["output_version"] == 1);
REQUIRE(native["findings"].is_array());
REQUIRE_FALSE(native.contains("runs"));
REQUIRE(readFile(markdown_path).find("# fold report") != std::string::npos);
const auto log_text = readFile(log);
const std::string warning = "report format 'json' is deprecated as a SARIF alias";
REQUIRE(log_text.find(warning) != std::string::npos);
REQUIRE(log_text.find(warning, log_text.find(warning) + 1) == std::string::npos);
const std::string nullable_warning =
"'--null-deref-nullable-allocations' is deprecated; use '--nullable-allocations'";
REQUIRE(log_text.find(nullable_warning) != std::string::npos);
REQUIRE(log_text.find(nullable_warning, log_text.find(nullable_warning) + 1) ==
std::string::npos);
const std::string input_type_warning = "'--input-type' is deprecated; use '--mode source'";
REQUIRE(log_text.find(input_type_warning) != std::string::npos);
REQUIRE(log_text.find(input_type_warning, log_text.find(input_type_warning) + 1) ==
std::string::npos);
}
TEST_CASE("fold scan fails when a translation unit is not analyzed", "[cli][coverage]") {
// A TU that failed to parse used to leave no trace: no findings, exit 0.
const std::string fold_bin = locateFoldBinary();
if (fold_bin.empty()) {
WARN("could not locate fold binary; skipping CLI coverage test");
return;
}
TempDir dir;
const auto good = dir.path / "good.cpp";
const auto broken = dir.path / "broken.cpp";
writeFile(good, "int add(int a, int b) { return a + b; }\n");
writeFile(broken, "#include \"does_not_exist.h\"\nint f(int x) { return x; }\n");
const auto scan = [&](const std::filesystem::path& source, const std::string& format,
const std::filesystem::path& report) {
const std::string command = "\"" + fold_bin + "\" --quiet scan --detectors div-by-zero " +
"--format " + format + " \"" + source.string() + "\" > \"" +
report.string() + "\" 2>/dev/null";
return commandExitCode(std::system(command.c_str()));
};
SECTION("clean TU: complete coverage, exit 0 (control)") {
const auto report = dir.path / "good.json";
REQUIRE(scan(good, "findings-json", report) == 0);
const auto json = nlohmann::json::parse(readFile(report));
CHECK(json["coverage"]["complete"] == true);
CHECK(json["coverage"]["tus_analyzed"] == 1);
}
SECTION("broken TU: gap in findings JSON, exit 2") {
const auto report = dir.path / "broken.json";
REQUIRE(scan(broken, "findings-json", report) == 2);
const auto json = nlohmann::json::parse(readFile(report));
CHECK(json["coverage"]["complete"] == false);
CHECK(json["coverage"]["gaps"][0]["reason"] == "compile-errors");
}
SECTION("broken TU: SARIF on stdout stays parseable and records the failure") {
const auto report = dir.path / "broken.sarif";
REQUIRE(scan(broken, "sarif", report) == 2);
const auto sarif = nlohmann::json::parse(readFile(report));
CHECK(sarif["runs"][0]["invocations"][0]["executionSuccessful"] == false);
}
}
Reference in a new issue View git blame Copy permalink