fold/tests/integration/test_binary_backend.cpp
2026-07-28 11:57:38 -04:00

590 lines
21 KiB
C++

#include <algorithm>
#include <catch2/catch_test_macros.hpp>
#include <chrono>
#include <cstdlib>
#include <filesystem>
#include <fstream>
#include <nlohmann/json.hpp>
#include <string>
#include "fold/binary/r2_backend.hpp"
#include "fold/binary/r2_backend_detail.hpp"
namespace {
constexpr const char* kFixtureUrl = "https://fold.example.invalid/token";
struct TempDir {
std::filesystem::path path;
TempDir()
: path(std::filesystem::temp_directory_path() /
("fold-binary-backend-" +
std::to_string(std::chrono::steady_clock::now().time_since_epoch().count()))) {
std::filesystem::create_directories(path);
}
~TempDir() { std::filesystem::remove_all(path); }
};
bool commandExists(const char* command) {
std::string check = "command -v ";
check += command;
check += " >/dev/null 2>&1";
return std::system(check.c_str()) == 0;
}
std::string shellQuote(const std::string& value) {
std::string quoted = "'";
for (char ch : value) {
if (ch == '\'')
quoted += "'\\''";
else
quoted.push_back(ch);
}
quoted.push_back('\'');
return quoted;
}
std::string locateTestBinary() {
if (const char* env = std::getenv("FOLD_TEST_BINARY")) {
if (*env && std::filesystem::exists(env))
return std::filesystem::canonical(env).string();
}
if (std::filesystem::exists("/usr/bin/curl"))
return std::filesystem::canonical("/usr/bin/curl").string();
return "";
}
std::string locateFoldBinary() {
if (const char* env = std::getenv("FOLD_BINARY")) {
if (*env && std::filesystem::exists(env))
return std::filesystem::canonical(env).string();
}
auto cwd = std::filesystem::current_path();
for (const auto& rel : {"../../src/cli/fold", "../../../src/cli/fold", "../src/cli/fold"}) {
auto candidate = cwd / rel;
if (std::filesystem::exists(candidate))
return std::filesystem::canonical(candidate).string();
}
return "";
}
bool optionalBinaryPrereqsAvailable(std::string& binary) {
binary = locateTestBinary();
if (binary.empty()) {
WARN("could not locate a binary fixture; set FOLD_TEST_BINARY to enable binary backend "
"test");
return false;
}
if (!commandExists("r2")) {
WARN("radare2 CLI not found; skipping optional binary backend test");
return false;
}
return true;
}
bool buildBinaryFixture(const TempDir& dir, std::string& binary) {
if (!commandExists("cc")) {
WARN("C compiler not found; skipping binary fixture test");
return false;
}
const auto source = dir.path / "fixture.c";
const auto output = dir.path / "fixture";
{
std::ofstream out(source);
out << "#include <stdio.h>\n"
<< "int main(void) {\n"
<< " const char *url = \"" << kFixtureUrl << "\";\n"
<< " puts(url);\n"
<< " return 0;\n"
<< "}\n";
}
const std::string cmd =
"cc -O0 " + shellQuote(source.string()) + " -o " + shellQuote(output.string());
if (std::system(cmd.c_str()) != 0 || !std::filesystem::exists(output)) {
WARN("failed to compile binary fixture");
return false;
}
binary = output.string();
return true;
}
bool buildRustBinaryFixture(const TempDir& dir, std::string& binary) {
if (!commandExists("rustc")) {
WARN("rustc not found; skipping Rust binary fixture test");
return false;
}
const auto source = dir.path / "fixture.rs";
const auto output = dir.path / "rust-fixture";
{
std::ofstream out(source);
out << "fn main() {\n"
<< " let url = \"" << kFixtureUrl << "\";\n"
<< " println!(\"{}\", url);\n"
<< "}\n";
}
const std::string cmd = "rustc -C opt-level=0 " + shellQuote(source.string()) + " -o " +
shellQuote(output.string());
if (std::system(cmd.c_str()) != 0 || !std::filesystem::exists(output)) {
WARN("failed to compile Rust binary fixture");
return false;
}
binary = output.string();
return true;
}
bool buildOversizedFunctionFixture(const TempDir& dir, std::string& binary) {
if (!commandExists("cc")) {
WARN("C compiler not found; skipping oversized-function fixture test");
return false;
}
const auto source = dir.path / "oversized_fn_fixture.c";
const auto output = dir.path / "oversized_fn_fixture";
{
std::ofstream out(source);
out << "__attribute__((noinline)) int giant(int x) {\n";
for (int i = 0; i < 12000; ++i)
out << " x += " << ((i % 13) + 1) << ";\n";
out << " return x;\n"
<< "}\n"
<< "int main(void) {\n"
<< " return giant(7);\n"
<< "}\n";
}
const std::string cmd =
"cc -O0 -fno-inline " + shellQuote(source.string()) + " -o " + shellQuote(output.string());
if (std::system(cmd.c_str()) != 0 || !std::filesystem::exists(output)) {
WARN("failed to compile oversized-function fixture");
return false;
}
binary = output.string();
return true;
}
} // namespace
TEST_CASE("R2Backend discovers functions in a system binary", "[binary][r2]") {
std::string binary;
if (!optionalBinaryPrereqsAvailable(binary))
return;
fold::binary::R2Backend backend(binary, /*max_functions=*/4);
auto units = backend.discoverUnits();
REQUIRE_FALSE(units.empty());
REQUIRE(units.size() <= 4);
REQUIRE(units.front().kind == "binary-function");
REQUIRE(units.front().path == binary);
auto functions = backend.extractFunctions(units.front());
REQUIRE_FALSE(functions.empty());
REQUIRE_FALSE(functions.front().qualified_name.empty());
REQUIRE(functions.front().source_file.find(binary) != std::string::npos);
}
TEST_CASE("fold CLI scans a binary through the radare2 backend", "[binary][cli]") {
std::string binary;
if (!optionalBinaryPrereqsAvailable(binary))
return;
const std::string fold_bin = locateFoldBinary();
if (fold_bin.empty()) {
WARN("could not locate fold binary; skipping binary CLI test");
return;
}
TempDir dir;
const auto output = dir.path / "binary-scan.sarif";
const std::string cmd =
shellQuote(fold_bin) + " scan --input-type binary --binary " + shellQuote(binary) +
" --binary-max-functions 4 --detectors security --format sarif --output " +
shellQuote(output.string());
const int rc = std::system(cmd.c_str());
REQUIRE((rc == 0 || rc == 256));
std::ifstream in(output);
REQUIRE(in.good());
auto sarif = nlohmann::json::parse(in);
REQUIRE(sarif["version"] == "2.1.0");
REQUIRE(sarif["runs"].size() == 1);
REQUIRE(sarif["runs"][0]["results"].is_array());
}
TEST_CASE("Rust-compiled binaries scan through the radare2 backend", "[binary][r2][rust]") {
std::string ignored;
if (!optionalBinaryPrereqsAvailable(ignored))
return;
TempDir dir;
std::string binary;
if (!buildRustBinaryFixture(dir, binary))
return;
fold::binary::R2Backend backend(binary, /*max_functions=*/0);
fold::cli::ScanOptions opts;
opts.detectors = {"security"};
auto findings = backend.runDetectors({}, opts);
auto found = std::find_if(findings.begin(), findings.end(), [](const fold::Finding& finding) {
return finding.evidence.binary_string == kFixtureUrl;
});
REQUIRE(found != findings.end());
REQUIRE(found->evidence.detector_name == "binary-security");
REQUIRE((found->evidence.backend.empty() || found->evidence.backend == "binary"));
REQUIRE(found->evidence.binary_offset.has_value());
REQUIRE(found->evidence.binary_vaddr.has_value());
}
TEST_CASE("function-byte extraction skips oversized binary functions", "[binary][r2]") {
std::string ignored;
if (!optionalBinaryPrereqsAvailable(ignored))
return;
TempDir dir;
std::string binary;
if (!buildOversizedFunctionFixture(dir, binary))
return;
fold::binary::BinaryFunctionExtractionStats stats;
const auto functions = fold::binary::extractFunctionBytes(binary, /*max_functions=*/0, &stats);
REQUIRE(stats.functions_skipped_oversized > 0);
for (const auto& function : functions)
REQUIRE(function.bytes.size() <= fold::binary::r2_detail::kMaxExtractedFunctionBytes);
}
// Build a binary that embeds multiple hardcoded string patterns:
// URL, credential, file path, email, IP, and base64-ish data.
bool buildHardcodedStringFixture(const TempDir& dir, std::string& binary) {
if (!commandExists("cc")) {
WARN("C compiler not found; skipping hardcoded-string fixture test");
return false;
}
const auto source = dir.path / "hcs_fixture.c";
const auto output = dir.path / "hcs_fixture";
{
std::ofstream out(source);
out << "#include <stdio.h>\n"
<< "int main(void) {\n"
<< " const char *url = \"https://example.com/api/v1\";\n"
<< " const char *pwd = \"password=SuperSecret123\";\n"
<< " const char *path = \"/etc/shadow\";\n"
<< " const char *email = \"admin@example.com\";\n"
<< " const char *ip = \"10.0.0.1\";\n"
// Base64-ish: mixed case, >20 chars, contains +/= to avoid hex-only
<< " const char *b64 = \"YWJjZGVmZ2hpamtsbW5vcA==\";\n" // "abcdefghijklmnop" in base64
<< " puts(url); puts(pwd); puts(path); puts(email);\n"
<< " puts(ip); puts(b64);\n"
<< " return 0;\n"
<< "}\n";
}
const std::string cmd =
"cc -O0 " + shellQuote(source.string()) + " -o " + shellQuote(output.string());
if (std::system(cmd.c_str()) != 0 || !std::filesystem::exists(output)) {
WARN("failed to compile hardcoded-string fixture");
return false;
}
binary = output.string();
return true;
}
TEST_CASE("hardcoded-string detector finds diverse patterns", "[binary][r2][hardcoded-string]") {
std::string ignored;
if (!optionalBinaryPrereqsAvailable(ignored))
return;
TempDir dir;
std::string binary;
if (!buildHardcodedStringFixture(dir, binary))
return;
fold::binary::R2Backend backend(binary, /*max_functions=*/0);
fold::cli::ScanOptions opts;
opts.detectors = {"hardcoded-string"};
auto findings = backend.runDetectors({}, opts);
// We expect at least 4 findings: URL, credential, file-path, IP
// Email and base64 depend on r2 string extraction heuristics.
REQUIRE(findings.size() >= 4);
for (const auto& f : findings) {
REQUIRE(f.confidence == fold::Confidence::Syntactic);
REQUIRE(f.evidence.detector_name == "hardcoded-string");
REQUIRE(f.kind == fold::FindingKind::HardcodedCredential);
REQUIRE(f.evidence.binary_offset.has_value());
REQUIRE(f.evidence.binary_vaddr.has_value());
REQUIRE_FALSE(f.evidence.binary_string.empty());
}
// Spot-check that we found key patterns.
bool found_url = false, found_cred = false, found_path = false, found_ip = false;
for (const auto& f : findings) {
if (f.evidence.binary_string.find("https://") != std::string::npos)
found_url = true;
if (f.evidence.binary_string.find("password") != std::string::npos)
found_cred = true;
if (f.evidence.binary_string.find("/etc/") != std::string::npos)
found_path = true;
if (f.evidence.binary_string.find("10.0.0.1") != std::string::npos)
found_ip = true;
}
REQUIRE(found_url);
REQUIRE(found_cred);
REQUIRE(found_path);
REQUIRE(found_ip);
}
TEST_CASE("hardcoded-string via CLI", "[binary][cli][hardcoded-string]") {
std::string ignored;
if (!optionalBinaryPrereqsAvailable(ignored))
return;
const std::string fold_bin = locateFoldBinary();
if (fold_bin.empty()) {
WARN("could not locate fold binary; skipping hardcoded-string CLI test");
return;
}
TempDir dir;
std::string binary;
if (!buildHardcodedStringFixture(dir, binary))
return;
const auto output = dir.path / "hcs-scan.sarif";
const std::string cmd =
shellQuote(fold_bin) + " scan --input-type binary --binary " + shellQuote(binary) +
" --detectors hardcoded-string --format sarif --output " + shellQuote(output.string());
const int rc = std::system(cmd.c_str());
REQUIRE((rc == 0 || rc == 256));
std::ifstream in(output);
REQUIRE(in.good());
auto sarif = nlohmann::json::parse(in);
REQUIRE(sarif["version"] == "2.1.0");
REQUIRE(sarif["runs"].size() == 1);
auto& results = sarif["runs"][0]["results"];
REQUIRE(results.is_array());
REQUIRE(results.size() >= 3);
bool has_url = false;
for (const auto& r : results) {
if (r["ruleId"] == "fold.hardcoded-credential" &&
r["message"]["text"].get<std::string>().find("URL") != std::string::npos) {
has_url = true;
break;
}
}
REQUIRE(has_url);
}
// Build a binary that imports a mix of dangerous and commonly safe C standard
// library functions so the dangerous-import detector can suppress import-only
// false positives for the safe subset.
bool buildDangerousImportsFixture(const TempDir& dir, std::string& binary) {
if (!commandExists("cc")) {
WARN("C compiler not found; skipping dangerous-import fixture test");
return false;
}
const auto source = dir.path / "danger_imports_fixture.c";
const auto output = dir.path / "danger_imports_fixture";
{
std::ofstream out(source);
out << "#include <string.h>\n"
<< "#include <stdio.h>\n"
<< "#include <stdlib.h>\n"
<< "int main(void) {\n"
<< " char buf[64];\n"
<< " strcpy(buf, \"hello\");\n"
<< " strncpy(buf, \"hi\", 2);\n"
<< " strcat(buf, \" world\");\n"
<< " strncat(buf, \"!\", 1);\n"
<< " sprintf(buf, \"%s\", \"test\");\n"
<< " snprintf(buf, 64, \"%s\", \"test\");\n"
<< " printf(\"hello\\n\");\n"
<< " fprintf(stderr, \"err\\n\");\n"
<< " scanf(\"%s\", buf);\n"
<< " fscanf(stdin, \"%s\", buf);\n"
<< " sscanf(\"hello\", \"%s\", buf);\n"
<< " char *p = malloc(64);\n"
<< " char *q = calloc(64, 1);\n"
<< " p = realloc(p, 128);\n"
<< " free(p);\n"
<< " free(q);\n"
<< " system(\"echo test\");\n"
<< " return 0;\n"
<< "}\n";
}
const std::string cmd =
"cc -O0 " + shellQuote(source.string()) + " -o " + shellQuote(output.string());
if (std::system(cmd.c_str()) != 0 || !std::filesystem::exists(output)) {
WARN("failed to compile dangerous-import fixture");
return false;
}
binary = output.string();
return true;
}
TEST_CASE("dangerous-import detector ignores safe import-only APIs and finds dangerous imports",
"[binary][r2][dangerous-import]") {
std::string ignored;
if (!optionalBinaryPrereqsAvailable(ignored))
return;
TempDir dir;
std::string binary;
if (!buildDangerousImportsFixture(dir, binary))
return;
fold::binary::R2Backend backend(binary, /*max_functions=*/0);
fold::cli::ScanOptions opts;
opts.detectors = {"dangerous-import"};
auto findings = backend.runDetectors({}, opts);
// On some platforms the linker resolves these to fortified variants
// (__strcpy_chk etc.) or inlines them, so require at least one dangerous
// import finding, then assert that the common safe import-only APIs do not
// appear as findings.
REQUIRE(findings.size() >= 1);
for (const auto& f : findings) {
REQUIRE(f.confidence == fold::Confidence::Syntactic);
REQUIRE(f.evidence.detector_name == "dangerous-import");
REQUIRE(f.kind == fold::FindingKind::DeprecatedBanned);
REQUIRE(f.evidence.binary_vaddr.has_value());
REQUIRE_FALSE(f.evidence.binary_section.empty());
}
// Spot-check that we found at least two of the expected imports.
bool found_strcpy = false, found_strncpy = false, found_strcat = false, found_strncat = false;
bool found_sprintf = false, found_snprintf = false, found_printf = false, found_fprintf = false;
bool found_scanf = false, found_fscanf = false, found_sscanf = false;
bool found_free = false, found_malloc = false, found_calloc = false, found_realloc = false;
bool found_system = false;
for (const auto& f : findings) {
auto check = [&](const char* name, bool& flag) {
const std::string needle = std::string("'") + name + "'";
if (!flag && f.message.find(needle) != std::string::npos)
flag = true;
};
check("strncpy", found_strncpy);
check("strcpy", found_strcpy);
check("strncat", found_strncat);
check("strcat", found_strcat);
check("snprintf", found_snprintf);
check("sprintf", found_sprintf);
check("fprintf", found_fprintf);
check("printf", found_printf);
check("fscanf", found_fscanf);
check("sscanf", found_sscanf);
check("scanf", found_scanf);
check("free", found_free);
check("malloc", found_malloc);
check("calloc", found_calloc);
check("realloc", found_realloc);
check("system", found_system);
}
REQUIRE_FALSE(found_malloc);
REQUIRE_FALSE(found_free);
REQUIRE_FALSE(found_printf);
REQUIRE_FALSE(found_snprintf);
REQUIRE_FALSE(found_realloc);
}
TEST_CASE("dangerous-import via CLI", "[binary][cli][dangerous-import]") {
std::string ignored;
if (!optionalBinaryPrereqsAvailable(ignored))
return;
const std::string fold_bin = locateFoldBinary();
if (fold_bin.empty()) {
WARN("could not locate fold binary; skipping dangerous-import CLI test");
return;
}
TempDir dir;
std::string binary;
if (!buildDangerousImportsFixture(dir, binary))
return;
const auto output = dir.path / "danger-import-scan.sarif";
const std::string cmd =
shellQuote(fold_bin) + " scan --input-type binary --binary " + shellQuote(binary) +
" --detectors dangerous-import --format sarif --output " + shellQuote(output.string());
const int rc = std::system(cmd.c_str());
REQUIRE((rc == 0 || rc == 256));
std::ifstream in(output);
REQUIRE(in.good());
auto sarif = nlohmann::json::parse(in);
REQUIRE(sarif["version"] == "2.1.0");
REQUIRE(sarif["runs"].size() == 1);
auto& results = sarif["runs"][0]["results"];
REQUIRE(results.is_array());
// Platform linkers may resolve dangerous imports to fortified
// variants (__strcpy_chk etc.), so the count varies by OS.
// Just verify the SARIF output is well-formed.
REQUIRE(results.size() >= 0);
}
TEST_CASE("binary scan findings include provenance metadata", "[binary][r2]") {
std::string ignored;
if (!optionalBinaryPrereqsAvailable(ignored))
return;
TempDir dir;
std::string binary;
if (!buildBinaryFixture(dir, binary))
return;
fold::binary::R2Backend backend(binary, /*max_functions=*/0);
fold::cli::ScanOptions opts;
opts.detectors = {"security"};
auto findings = backend.runDetectors({}, opts);
auto found = std::find_if(findings.begin(), findings.end(), [](const fold::Finding& finding) {
return finding.evidence.binary_string == kFixtureUrl;
});
REQUIRE(found != findings.end());
REQUIRE(found->evidence.detector_name == "binary-security");
REQUIRE(found->evidence.binary_offset.has_value());
REQUIRE(found->evidence.binary_vaddr.has_value());
const std::string fold_bin = locateFoldBinary();
if (fold_bin.empty()) {
WARN("could not locate fold binary; skipping SARIF metadata assertion");
return;
}
const auto output = dir.path / "fixture.sarif";
const std::string cmd = shellQuote(fold_bin) + " scan --input-type binary --binary " +
shellQuote(binary) + " --detectors security --format sarif --output " +
shellQuote(output.string());
const int rc = std::system(cmd.c_str());
REQUIRE((rc == 0 || rc == 256));
std::ifstream in(output);
REQUIRE(in.good());
auto sarif = nlohmann::json::parse(in);
bool has_binary_metadata = false;
for (const auto& result : sarif["runs"][0]["results"]) {
const auto& props = result["properties"];
if (props.contains("binary") &&
props["binary"].value("matched_string", "") == kFixtureUrl) {
has_binary_metadata = true;
REQUIRE(props["binary"].contains("offset"));
REQUIRE(props["binary"].contains("vaddr"));
break;
}
}
REQUIRE(has_binary_metadata);
}