#include #include #include #include #include #include #include #include #include #include #include "../../include/fold/cli/scan_option_catalog.hpp" #include "../../include/fold/mcp/scan_cache.hpp" #include "../../include/fold/mcp/tool_registry.hpp" #include "../../include/fold/report/findings_json_report.hpp" namespace { struct TempDir { std::filesystem::path path; TempDir() : path(std::filesystem::temp_directory_path() / ("fold-cli-paths-" + std::to_string(std::chrono::steady_clock::now().time_since_epoch().count()))) { std::filesystem::create_directories(path); } ~TempDir() { std::filesystem::remove_all(path); } }; void writeFile(const std::filesystem::path& path, const std::string& content) { std::ofstream(path) << content; } std::string readFile(const std::filesystem::path& path) { std::ifstream in(path); std::ostringstream out; out << in.rdbuf(); return out.str(); } int commandExitCode(int status) { return WIFEXITED(status) ? WEXITSTATUS(status) : -1; } std::string locateFoldBinary() { if (const char* env = std::getenv("FOLD_BINARY")) { if (*env && std::filesystem::exists(env)) { return std::filesystem::canonical(env).string(); } } auto cwd = std::filesystem::current_path(); for (const auto& rel : {"../../src/cli/fold", "../../../src/cli/fold", "../src/cli/fold"}) { auto candidate = cwd / rel; if (std::filesystem::exists(candidate)) { return std::filesystem::canonical(candidate).string(); } } return ""; } } // namespace TEST_CASE("fold help advertises embedded skill command", "[cli][help][skill]") { const std::string fold_bin = locateFoldBinary(); if (fold_bin.empty()) { WARN("could not locate fold binary; skipping CLI help test"); return; } TempDir dir; const auto help = dir.path / "help.txt"; const auto skill = dir.path / "skill.md"; REQUIRE(std::system(("\"" + fold_bin + "\" --help > \"" + help.string() + "\" 2>&1").c_str()) == 0); REQUIRE(std::system(("\"" + fold_bin + "\" skill -o \"" + skill.string() + "\"").c_str()) == 0); const auto help_text = readFile(help); REQUIRE(help_text.find("skill") != std::string::npos); REQUIRE(help_text.find("bounded symbolic SAST") != std::string::npos); const auto skill_text = readFile(skill); REQUIRE(skill_text.find("name: skill-fold-sast") != std::string::npos); REQUIRE(skill_text.find("concrete-witness") != std::string::npos); REQUIRE(skill_text.find("fold fuzz prep") != std::string::npos); const char* canonical_skill = std::getenv("FOLD_SKILL_SOURCE"); REQUIRE(canonical_skill != nullptr); REQUIRE(skill_text == readFile(canonical_skill)); } TEST_CASE("fold scan honors positional paths after repeatable detector flags", "[cli][paths]") { const std::string fold_bin = locateFoldBinary(); if (fold_bin.empty()) { WARN("could not locate fold binary; skipping CLI paths test"); return; } TempDir dir; const auto first = dir.path / "first.cpp"; const auto second = dir.path / "second.cpp"; const auto db = dir.path / "compile_commands.json"; const auto out = dir.path / "scan.json"; const auto log = dir.path / "scan.log"; writeFile(first, "int first(int x) {\n" " int y = x + 1;\n" " return y * 2;\n" "}\n"); writeFile(second, "int second(int x) {\n" " int y = x + 1;\n" " return y * 2;\n" "}\n"); writeFile(db, nlohmann::json::array( { {{"directory", dir.path.string()}, {"command", std::string("clang++ -std=c++20 -c ") + first.string()}, {"file", first.string()}}, {{"directory", dir.path.string()}, {"command", std::string("clang++ -std=c++20 -c ") + second.string()}, {"file", second.string()}}, }) .dump(2)); const std::string cmd = "\"" + fold_bin + "\" scan --compile-db \"" + db.string() + "\" --format json --output \"" + out.string() + "\" --processes 1 --detectors clone \"" + first.string() + "\" > \"" + log.string() + "\" 2>&1"; REQUIRE(std::system(cmd.c_str()) == 0); std::ifstream in(out); REQUIRE(in.good()); const auto report = nlohmann::json::parse(in); REQUIRE(report["runs"].size() == 1); REQUIRE(report["runs"][0]["results"].empty()); } TEST_CASE("fold scan exclude globs support globstar and brace alternatives", "[cli][paths][exclude-glob]") { const std::string fold_bin = locateFoldBinary(); if (fold_bin.empty()) { WARN("could not locate fold binary; skipping CLI exclude-glob test"); return; } TempDir dir; const auto safe = dir.path / "safe.cpp"; const auto vendor_dir = dir.path / "nested" / "third_party"; const auto generated_dir = dir.path / "nested" / "generated"; std::filesystem::create_directories(vendor_dir); std::filesystem::create_directories(generated_dir); const auto vendor = vendor_dir / "unsafe.cpp"; const auto generated = generated_dir / "unsafe.pb.cc"; const auto db = dir.path / "compile_commands.json"; const auto out = dir.path / "scan.json"; const auto log = dir.path / "scan.log"; const auto profile = dir.path / "scan.profile.json"; writeFile(safe, "int safe() { return 1; }\n"); writeFile(vendor, "int vendor_bad() { return 1 / 0; }\n"); writeFile(generated, "int generated_bad() { return 1 / 0; }\n"); nlohmann::json commands = nlohmann::json::array(); for (const auto& file : {safe, vendor, generated}) { commands.push_back({ {"directory", dir.path.string()}, {"command", std::string("clang++ -std=c++20 -c ") + file.string()}, {"file", file.string()}, }); } writeFile(db, commands.dump(2)); const std::string cmd = "\"" + fold_bin + "\" scan --compile-db \"" + db.string() + "\" --format json --output \"" + out.string() + "\" --processes 1 --detectors div-by-zero --exclude-glob 'third_party/**' " "--exclude-glob '**/*.pb.{h,cc}' --profile --profile-output \"" + profile.string() + "\" > \"" + log.string() + "\" 2>&1"; const int status = std::system(cmd.c_str()); INFO(readFile(log)); REQUIRE(status == 0); const auto report = nlohmann::json::parse(std::ifstream(out)); REQUIRE(report["runs"][0]["results"].empty()); REQUIRE(readFile(log).find("3 -> 1 files (2 skipped)") != std::string::npos); const auto profile_json = nlohmann::json::parse(std::ifstream(profile)); REQUIRE(profile_json["scan"]["source_tus_selected"] == 1); REQUIRE(profile_json["scan"]["source_tus_excluded"] == 2); REQUIRE(profile_json["scan"]["source_tus_attempted"] == 1); REQUIRE(profile_json["scan"]["source_tus_completed"] == 1); REQUIRE(profile_json["scan"]["source_tus_frontend_failed"] == 0); } TEST_CASE("fold scan artifacts share report-visible counts and commit atomically", "[cli][artifacts][counts]") { const std::string fold_bin = locateFoldBinary(); if (fold_bin.empty()) { WARN("could not locate fold binary; skipping artifact contract test"); return; } TempDir dir; const auto source = dir.path / "unsafe.cpp"; const auto db = dir.path / "compile_commands.json"; const auto sarif = dir.path / "serial.sarif"; const auto native = dir.path / "fanout.json"; const auto profile = dir.path / "serial.profile.json"; const auto serial_log = dir.path / "serial.log"; const auto fanout_log = dir.path / "fanout.log"; const auto version_path = dir.path / "version.txt"; REQUIRE(std::system( ("\"" + fold_bin + "\" --version > \"" + version_path.string() + "\"").c_str()) == 0); std::string fold_version = readFile(version_path); while (!fold_version.empty() && std::isspace(static_cast(fold_version.back()))) fold_version.pop_back(); REQUIRE_FALSE(fold_version.empty()); writeFile(source, "int unsafe_divisor(int value) { return 10 / value; }\n"); writeFile(db, nlohmann::json::array({{{"directory", dir.path.string()}, {"command", "clang++ -std=c++20 -c " + source.string()}, {"file", source.string()}}}) .dump(2)); const std::string common = "\"" + fold_bin + "\" scan --compile-db \"" + db.string() + "\" --detectors div-by-zero "; const std::string serial_command = common + "--processes 1 --format sarif --output \"" + sarif.string() + "\" --profile-output \"" + profile.string() + "\" > \"" + serial_log.string() + "\" 2>&1"; REQUIRE(commandExitCode(std::system(serial_command.c_str())) == 1); const auto sarif_report = nlohmann::json::parse(std::ifstream(sarif)); REQUIRE(sarif_report["runs"][0]["results"].size() == 1); REQUIRE(sarif_report["runs"][0]["tool"]["driver"]["version"] == fold_version); REQUIRE(std::filesystem::is_regular_file(profile)); const auto profile_report = nlohmann::json::parse(std::ifstream(profile)); REQUIRE(profile_report["version"] == fold_version); REQUIRE(profile_report["scan"]["findings_before_filter"] >= 1); REQUIRE(profile_report["scan"]["findings_report_visible"] == 1); REQUIRE(readFile(serial_log).find("emitted 1 report-visible findings total") != std::string::npos); const std::string fanout_command = common + "--processes 2 --format findings-json --output \"" + native.string() + "\" > \"" + fanout_log.string() + "\" 2>&1"; REQUIRE(commandExitCode(std::system(fanout_command.c_str())) == 1); const auto native_report = nlohmann::json::parse(std::ifstream(native)); REQUIRE(native_report["findings"].size() == 1); REQUIRE(native_report["tool"]["version"] == fold_version); REQUIRE(readFile(fanout_log).find("emitted 1 report-visible findings total") != std::string::npos); const auto invalid_target = dir.path / "report-directory"; std::filesystem::create_directories(invalid_target); const std::string failed_command = common + "--format findings-json --output \"" + invalid_target.string() + "\" >/dev/null 2>&1"; REQUIRE(commandExitCode(std::system(failed_command.c_str())) == 2); for (const auto& entry : std::filesystem::directory_iterator(dir.path)) { REQUIRE(entry.path().filename().string().find("report-directory.tmp.") != 0); } const auto invalid_profile = dir.path / "profile-directory"; const auto valid_report = dir.path / "profile-failure-report.json"; std::filesystem::create_directories(invalid_profile); const std::string profile_failure_command = common + "--format findings-json --output \"" + valid_report.string() + "\" --profile-output \"" + invalid_profile.string() + "\" >/dev/null 2>&1"; REQUIRE(commandExitCode(std::system(profile_failure_command.c_str())) == 2); REQUIRE(std::filesystem::is_directory(invalid_profile)); for (const auto& entry : std::filesystem::directory_iterator(dir.path)) { REQUIRE(entry.path().filename().string().find("profile-directory.tmp.") != 0); } const auto collision = dir.path / "collision.json"; const std::string collision_command = common + "--output \"" + collision.string() + "\" --profile-output \"" + collision.string() + "\" >/dev/null 2>&1"; REQUIRE(commandExitCode(std::system(collision_command.c_str())) == 2); REQUIRE_FALSE(std::filesystem::exists(collision)); } TEST_CASE("fold scan help separates common and advanced options", "[cli][contract][help]") { const std::string fold_bin = locateFoldBinary(); if (fold_bin.empty()) { WARN("could not locate fold binary; skipping CLI help contract test"); return; } TempDir dir; const auto help = dir.path / "scan-help.txt"; const auto advanced = dir.path / "scan-advanced-help.txt"; REQUIRE(std::system( ("\"" + fold_bin + "\" scan --help > \"" + help.string() + "\" 2>&1").c_str()) == 0); REQUIRE(std::system( ("\"" + fold_bin + "\" scan --advanced-help > \"" + advanced.string() + "\" 2>&1") .c_str()) == 0); const auto help_text = readFile(help); const auto advanced_text = readFile(advanced); REQUIRE(help_text.find("--paths") == std::string::npos); REQUIRE(help_text.find("--advanced-help") != std::string::npos); REQUIRE(advanced_text.find("Advanced scan options") != std::string::npos); for (const auto& option : fold::cli::scanOptionCatalog()) { if (!option.cli_exposed || !option.cli_flag.starts_with("--")) continue; const auto& expected = option.advanced ? advanced_text : help_text; INFO("catalog option missing from expected help surface: " << option.cli_flag); REQUIRE(expected.find(option.cli_flag) != std::string::npos); } } TEST_CASE("fold scan rejects catalog-invalid numeric options", "[cli][contract][bounds]") { const std::string fold_bin = locateFoldBinary(); if (fold_bin.empty()) { WARN("could not locate fold binary; skipping CLI bounds test"); return; } for (const auto& arguments : {"--clone-type 1", "--clone-similarity 2", "--budget-seconds -1", "--binary-max-functions -1"}) { INFO(arguments); const std::string command = "\"" + fold_bin + "\" scan --advanced-help " + arguments + " >/dev/null 2>&1"; REQUIRE(commandExitCode(std::system(command.c_str())) != 0); } } TEST_CASE("fold grep queries finding reports with machine-readable metadata", "[cli][grep][contract]") { const std::string fold_bin = locateFoldBinary(); if (fold_bin.empty()) { WARN("could not locate fold binary; skipping CLI grep test"); return; } TempDir dir; const auto report_path = dir.path / "findings.json"; const auto result_path = dir.path / "query.json"; const auto log_path = dir.path / "query.log"; fold::Finding source; source.id = "finding-a"; source.kind = fold::FindingKind::NullDeref; source.confidence = fold::Confidence::Proved; source.message = "Null token reaches parser"; source.evidence.backend = "source"; source.locations.push_back({"src/auth.cpp", 10, 2, "parse"}); fold::Finding suppressed; suppressed.id = "finding-b"; suppressed.kind = fold::FindingKind::HardcodedCredential; suppressed.confidence = fold::Confidence::Heuristic; suppressed.message = "Embedded token"; suppressed.evidence.backend = "binary"; suppressed.locations.push_back({"bin/app", 1, 1, "sym.auth"}); suppressed.suppression_reason = "accepted fixture"; { std::ofstream report(report_path); fold::writeFindingsJson(report, {source, suppressed}); } const auto command = "\"" + fold_bin + "\" grep TOKEN \"" + report_path.string() + "\" --ignore-case --field message --format json --output \"" + result_path.string() + "\" > \"" + log_path.string() + "\" 2>&1"; REQUIRE(commandExitCode(std::system(command.c_str())) == 1); const auto result = nlohmann::json::parse(std::ifstream(result_path)); REQUIRE(result["output_version"] == 1); REQUIRE(result["total_input"] == 2); REQUIRE(result["total_matches"] == 2); REQUIRE(result["total_actionable_matches"] == 1); REQUIRE(result["findings"][0]["id"] == "finding-a"); REQUIRE(result["findings"][1]["id"] == "finding-b"); fold::mcp::ScanCache cache; cache.replaceAll({source, suppressed}); fold::mcp::ToolRegistry registry(cache); const auto mcp_result = registry.call("fold.query_findings", {{"pattern", "TOKEN"}, {"ignore_case", true}, {"fields", nlohmann::json::array({"message"})}, {"limit", 100}}); const auto& mcp_query = mcp_result["structuredContent"]; REQUIRE(mcp_query["total_input"] == result["total_input"]); REQUIRE(mcp_query["total_matches"] == result["total_matches"]); REQUIRE(mcp_query["total_actionable_matches"] == result["total_actionable_matches"]); REQUIRE(mcp_query["findings"][0]["id"] == result["findings"][0]["id"]); REQUIRE(mcp_query["findings"][1]["id"] == result["findings"][1]["id"]); const auto suppressed_command = "\"" + fold_bin + "\" grep token \"" + report_path.string() + "\" --ignore-case --suppressed --format json --output \"" + result_path.string() + "\" > \"" + log_path.string() + "\" 2>&1"; REQUIRE(commandExitCode(std::system(suppressed_command.c_str())) == 0); const auto suppressed_result = nlohmann::json::parse(std::ifstream(result_path)); REQUIRE(suppressed_result["total_matches"] == 1); REQUIRE(suppressed_result["total_actionable_matches"] == 0); REQUIRE(suppressed_result["findings"][0]["id"] == "finding-b"); } TEST_CASE("fold scan rejects unimplemented hybrid strategies", "[cli][contract][hybrid]") { const std::string fold_bin = locateFoldBinary(); if (fold_bin.empty()) { WARN("could not locate fold binary; skipping CLI hybrid contract test"); return; } TempDir dir; const auto log = dir.path / "hybrid.log"; const auto command = "\"" + fold_bin + "\" scan --mode hybrid --hybrid-strategy binary-first > \"" + log.string() + "\" 2>&1"; REQUIRE(std::system(command.c_str()) != 0); const auto output = readFile(log); REQUIRE(output.find("binary-first") != std::string::npos); REQUIRE(output.find("merge-only") != std::string::npos); } TEST_CASE("fold scan formats have truthful distinct contracts", "[cli][contract][format]") { const std::string fold_bin = locateFoldBinary(); if (fold_bin.empty()) { WARN("could not locate fold binary; skipping CLI format contract test"); return; } TempDir dir; const auto source = dir.path / "safe.cpp"; const auto sarif_path = dir.path / "report.sarif.json"; const auto json_path = dir.path / "report.json"; const auto native_path = dir.path / "report.findings.json"; const auto markdown_path = dir.path / "report.md"; const auto log = dir.path / "scan.log"; writeFile(source, "int safe(int value) { return value + 1; }\n"); const std::string common = "\"" + fold_bin + "\" scan --mode source --processes 1 " + "--detectors complexity \"" + source.string() + "\" "; REQUIRE(std::system((common + "--format sarif --output \"" + sarif_path.string() + "\" > \"" + log.string() + "\" 2>&1") .c_str()) == 0); REQUIRE(std::system((common + "--format json --output \"" + json_path.string() + "\" >> \"" + log.string() + "\" 2>&1") .c_str()) == 0); REQUIRE(std::system((common + "--format findings-json --output \"" + native_path.string() + "\" >> \"" + log.string() + "\" 2>&1") .c_str()) == 0); REQUIRE(std::system((common + "--format markdown --output \"" + markdown_path.string() + "\" >> \"" + log.string() + "\" 2>&1") .c_str()) == 0); REQUIRE(std::system((common + "--null-deref-nullable-allocations --format sarif --output \"" + sarif_path.string() + "\" >> \"" + log.string() + "\" 2>&1") .c_str()) == 0); const std::string legacy_mode = "\"" + fold_bin + "\" scan --input-type source --processes 1 --detectors complexity \"" + source.string() + "\" --format sarif --output \"" + sarif_path.string() + "\" >> \"" + log.string() + "\" 2>&1"; REQUIRE(std::system(legacy_mode.c_str()) == 0); const auto sarif = nlohmann::json::parse(std::ifstream(sarif_path)); const auto json_alias = nlohmann::json::parse(std::ifstream(json_path)); const auto native = nlohmann::json::parse(std::ifstream(native_path)); REQUIRE(sarif == json_alias); REQUIRE(json_alias["version"] == "2.1.0"); REQUIRE(native["output_version"] == 1); REQUIRE(native["findings"].is_array()); REQUIRE_FALSE(native.contains("runs")); REQUIRE(readFile(markdown_path).find("# fold report") != std::string::npos); const auto log_text = readFile(log); const std::string warning = "report format 'json' is deprecated as a SARIF alias"; REQUIRE(log_text.find(warning) != std::string::npos); REQUIRE(log_text.find(warning, log_text.find(warning) + 1) == std::string::npos); const std::string nullable_warning = "'--null-deref-nullable-allocations' is deprecated; use '--nullable-allocations'"; REQUIRE(log_text.find(nullable_warning) != std::string::npos); REQUIRE(log_text.find(nullable_warning, log_text.find(nullable_warning) + 1) == std::string::npos); const std::string input_type_warning = "'--input-type' is deprecated; use '--mode source'"; REQUIRE(log_text.find(input_type_warning) != std::string::npos); REQUIRE(log_text.find(input_type_warning, log_text.find(input_type_warning) + 1) == std::string::npos); } TEST_CASE("fold scan fails when a translation unit is not analyzed", "[cli][coverage]") { // A TU that failed to parse used to leave no trace: no findings, exit 0. const std::string fold_bin = locateFoldBinary(); if (fold_bin.empty()) { WARN("could not locate fold binary; skipping CLI coverage test"); return; } TempDir dir; const auto good = dir.path / "good.cpp"; const auto broken = dir.path / "broken.cpp"; writeFile(good, "int add(int a, int b) { return a + b; }\n"); writeFile(broken, "#include \"does_not_exist.h\"\nint f(int x) { return x; }\n"); const auto scan = [&](const std::filesystem::path& source, const std::string& format, const std::filesystem::path& report) { const std::string command = "\"" + fold_bin + "\" --quiet scan --detectors div-by-zero " + "--format " + format + " \"" + source.string() + "\" > \"" + report.string() + "\" 2>/dev/null"; return commandExitCode(std::system(command.c_str())); }; SECTION("clean TU: complete coverage, exit 0 (control)") { const auto report = dir.path / "good.json"; REQUIRE(scan(good, "findings-json", report) == 0); const auto json = nlohmann::json::parse(readFile(report)); CHECK(json["coverage"]["complete"] == true); CHECK(json["coverage"]["tus_analyzed"] == 1); } SECTION("broken TU: gap in findings JSON, exit 2") { const auto report = dir.path / "broken.json"; REQUIRE(scan(broken, "findings-json", report) == 2); const auto json = nlohmann::json::parse(readFile(report)); CHECK(json["coverage"]["complete"] == false); CHECK(json["coverage"]["gaps"][0]["reason"] == "compile-errors"); } SECTION("broken TU: SARIF on stdout stays parseable and records the failure") { const auto report = dir.path / "broken.sarif"; REQUIRE(scan(broken, "sarif", report) == 2); const auto sarif = nlohmann::json::parse(readFile(report)); CHECK(sarif["runs"][0]["invocations"][0]["executionSuccessful"] == false); } }