strawberry/tests/test_fault_injection.cpp
trvon 74aa9664ad
Pipeline: keep oversized cache refusals unscanned
Do not turn an empty cached digest into an unbounded descriptor hash. Distinguish size refusals from YARA results so the exec guard applies its own bounded scan/degrade policy. Cover cached read budgets across size boundaries, known hashes, inventory/live events and real guard-cache reuse. Fold/live/KPI and post-rollout CPU acceptance remain pending; OpenSnitch checksum work is separate.
2026-11-03 06:57:35 -07:00

547 lines
22 KiB
C++

// Fault-injection properties (built with STRAWBERRY_FAULT_INJECTION=1).
//
// The 2026-10-25 freeze began with a /proc read failing mid-file (ESRCH) and an
// exception escaping. Unit tests, property tests and fuzzers all missed it
// because they fed in-memory strings and never made I/O fail. These properties
// make every read/open/fstat fail, shorten or pass at random, seeded, and
// assert that nothing throws, nothing lies, and the pipeline keeps going.
//
// A failure prints the seed; reproduce with STRAWBERRY_PROPERTY_SEED=<seed>.
#include "strawberry/detection_pipeline.hpp"
#include "strawberry/correlation_engine.hpp"
#include "strawberry/hash_utils.hpp"
#include "strawberry/proc_read.hpp"
#include "strawberry/policy.hpp"
#include "strawberry/resource_governor.hpp"
#include "strawberry/sys_io.hpp"
#include "strawberry/thread_entry.hpp"
#include "../src/image_file.hpp"
#include <array>
#include <chrono>
#include <atomic>
#include <cassert>
#include <cerrno>
#include <cstdlib>
#include <cstring>
#include <filesystem>
#include <fstream>
#include <iostream>
#include <random>
#include <string>
#include <thread>
#include <vector>
#include <unistd.h>
#if !defined(STRAWBERRY_FAULT_INJECTION) || !STRAWBERRY_FAULT_INJECTION
#error "test_fault_injection must be built with STRAWBERRY_FAULT_INJECTION=1"
#endif
namespace fs = std::filesystem;
using strawberry::io::FaultDecision;
using strawberry::io::Op;
namespace {
uint64_t seed_from_env() {
if (const char* text = std::getenv("STRAWBERRY_PROPERTY_SEED")) {
return std::strtoull(text, nullptr, 10);
}
return std::random_device{}();
}
const uint64_t kSeed = seed_from_env();
#define PROPERTY(cond) \
do { \
if (!(cond)) { \
std::cerr << "PROPERTY FAILED: " #cond " (seed " << kSeed << ", " << __FILE__ << ":" \
<< __LINE__ << ")" << std::endl; \
std::abort(); \
} \
} while (0)
// Seeded random faults: each I/O call fails with probability `fail_rate`
// (errno drawn from the realistic set) and reads are shortened at random.
class RandomFaults final : public strawberry::io::FaultInjector {
public:
RandomFaults(uint64_t seed, double fail_rate) : rng_(seed), fail_rate_(fail_rate) {}
FaultDecision decide(Op op) noexcept override {
++calls_;
FaultDecision d;
std::uniform_real_distribution<double> unit(0.0, 1.0);
if (unit(rng_) < fail_rate_) {
static constexpr std::array<int, 6> errors{ESRCH, EIO, ENOENT, EINTR, EAGAIN, EACCES};
d.error = errors[rng_() % errors.size()];
++faults_;
} else if ((op == Op::read || op == Op::pread) && unit(rng_) < 0.3) {
d.short_bytes = static_cast<long>(rng_() % 7); // includes 0: a premature EOF
}
return d;
}
uint64_t calls() const { return calls_; }
uint64_t faults() const { return faults_; }
private:
std::mt19937_64 rng_;
double fail_rate_;
uint64_t calls_ = 0;
uint64_t faults_ = 0;
};
fs::path make_root(const char* name) {
const auto root = fs::temp_directory_path() / (std::string("sb-fi-") + name + "-" + std::to_string(::getpid()));
std::error_code ec;
fs::remove_all(root, ec);
fs::create_directories(root);
return root;
}
void write_file(const fs::path& path, const std::string& content) {
fs::create_directories(path.parent_path());
std::ofstream(path, std::ios::binary) << content;
}
std::string random_content(std::mt19937_64& rng, std::size_t max_len) {
std::string out(rng() % (max_len + 1), '\0');
for (auto& c : out) c = static_cast<char>(rng() & 0xff);
return out;
}
// P1: read_small_file is never wrong: nullopt, or an exact prefix of the file
// (exactly the file when it fits), and it never throws.
void property_read_small_file() {
std::cout << "P1 read_small_file under faults" << std::endl;
const auto root = make_root("p1");
std::mt19937_64 rng(kSeed);
for (int round = 0; round < 2000; ++round) {
const auto content = random_content(rng, 20000);
const auto path = root / "f";
write_file(path, content);
const std::size_t max = 1 + rng() % 30000;
RandomFaults faults(rng(), 0.2);
strawberry::io::ScopedInjector scope(&faults);
std::optional<std::string> got;
try {
got = strawberry::read_small_file(path.string(), max);
} catch (...) {
PROPERTY(!"read_small_file threw");
}
if (got) {
PROPERTY(got->size() <= max);
// Short reads are retried until EOF/limit, so a returned value is
// either the whole file (capped) or ... it is never a silent
// truncation: a premature 0-byte read is indistinguishable from EOF
// for /proc, so we only require a prefix.
PROPERTY(content.compare(0, got->size(), *got) == 0);
}
}
fs::remove_all(root);
std::cout << " PASSED (seed " << kSeed << ")" << std::endl;
}
// P2: read_fd_bounded is exact or fails with an error message.
void property_read_fd_bounded() {
std::cout << "P2 read_fd_bounded under faults" << std::endl;
const auto root = make_root("p2");
std::mt19937_64 rng(kSeed ^ 0x2);
for (int round = 0; round < 2000; ++round) {
const auto content = random_content(rng, 70000);
const auto path = root / "img";
write_file(path, content);
const int fd = ::open(path.c_str(), O_RDONLY | O_CLOEXEC);
PROPERTY(fd >= 0);
RandomFaults faults(rng(), 0.15);
std::vector<unsigned char> data;
std::string error;
bool ok = false;
std::size_t pace_calls = 0;
const std::size_t allowed_reads = round % 2 == 0 ? rng() % 4 : 128;
{
strawberry::io::ScopedInjector scope(&faults);
try {
ok = strawberry::detail::read_fd_bounded(fd, 1u << 20, data, error,
[&](std::size_t bytes) {
PROPERTY(bytes <= 65536);
return ++pace_calls <= allowed_reads;
});
} catch (...) {
PROPERTY(!"read_fd_bounded threw");
}
}
::close(fd);
PROPERTY(pace_calls <= allowed_reads + 1);
if (ok) {
// Success means the exact file: never a silent prefix.
PROPERTY(data.size() == content.size());
PROPERTY(std::memcmp(data.data(), content.data(), data.size()) == 0);
} else {
PROPERTY(!error.empty());
}
}
// Hashing never throws either.
for (int round = 0; round < 500; ++round) {
const auto path = root / "h";
write_file(path, random_content(rng, 30000));
RandomFaults faults(rng(), 0.3);
strawberry::io::ScopedInjector scope(&faults);
try {
(void) strawberry::HashUtils::sha256_file(path.string());
(void) strawberry::HashUtils::md5_file(path.string());
} catch (...) {
PROPERTY(!"hashing threw");
}
}
fs::remove_all(root);
std::cout << " PASSED" << std::endl;
}
#if STRAWBERRY_HAVE_YARA
// P3: the detection pipeline under random I/O faults: no exception escapes,
// a failed image read never produces a cached (wrong) result, and a later
// fault-free pass detects what it should.
void property_pipeline_under_faults() {
std::cout << "P3 detection pipeline under faults" << std::endl;
const auto root = make_root("p3");
write_file(root / "rules" / "r.yar",
"rule FI_Marker { meta: severity = \"high\" strings: $m = \"fault-injection-marker\" condition: $m }\n");
std::mt19937_64 rng(kSeed ^ 0x3);
std::vector<fs::path> images;
for (int i = 0; i < 12; ++i) {
const auto path = root / "bin" / ("img" + std::to_string(i));
const bool malicious = i % 3 == 0;
write_file(path, std::string("\x7f" "ELF") + random_content(rng, 4000) + (malicious ? "fault-injection-marker" : ""));
images.push_back(path);
}
strawberry::Config config;
config.db_path = (root / "fi.db").string();
config.yara_rules_path = (root / "rules").string();
strawberry::DetectionPipeline pipeline;
PROPERTY(pipeline.initialize(config));
pipeline.set_detection_callback([](const strawberry::DetectionEvent&) { throw 42; }); // must be contained
RandomFaults faults(rng(), 0.35);
for (int round = 0; round < 600; ++round) {
const auto& image = images[rng() % images.size()];
strawberry::ProcessInfo info;
info.pid = 10000 + round;
info.start_time_ticks = 1 + static_cast<uint64_t>(round);
info.executable_path = image.string();
info.image_path = image.string();
info.image_identity = strawberry::detail::stat_image_identity(image.string());
strawberry::io::ScopedInjector scope(&faults);
try {
if (rng() % 2) {
pipeline.handle_process(info);
} else {
pipeline.handle_inventory_process(info);
}
} catch (...) {
PROPERTY(!"handle_process threw under I/O faults");
}
}
PROPERTY(faults.faults() > 0);
// Fault-free pass: every malicious image is detected (no poisoned cache
// entry from a failed read hides it), every clean one is not.
const auto before = pipeline.stats().detections;
for (std::size_t i = 0; i < images.size(); ++i) {
strawberry::ProcessInfo info;
info.pid = 90000 + static_cast<int>(i);
info.start_time_ticks = 90000 + i;
info.executable_path = images[i].string();
info.image_path = images[i].string();
info.image_identity = strawberry::detail::stat_image_identity(images[i].string());
pipeline.handle_process(info);
}
const auto found = pipeline.stats().detections - before;
PROPERTY(found == 4); // images 0, 3, 6, 9
// A failed promoted identity check must not poison the rule-scan cache.
const strawberry::ProcessRef gone{2147483647, "process_2147483647_abcd1234_1", "/missing"};
for (int retry = 0; retry < 2; ++retry) {
const auto result = pipeline.scan_process_memory(gone, "identity regression", false, true);
PROPERTY(!result.ok && !result.skipped_duplicate);
}
pipeline.shutdown();
fs::remove_all(root);
std::cout << " PASSED (" << faults.faults() << " injected faults over " << faults.calls() << " I/O calls)"
<< std::endl;
}
class CountCachedImageReads final : public strawberry::io::FaultInjector {
public:
std::atomic<uint64_t> reads{0};
FaultDecision decide(Op op) noexcept override {
if (op == Op::read || op == Op::pread) ++reads;
return {};
}
};
// P9: across the size boundary, a cached image outcome never re-reads bytes.
// In particular an oversized/unhashed refusal is not permission to hash to EOF.
void property_cached_image_does_not_rehash() {
std::cout << "P9 cached image byte-read budget" << std::endl;
const auto root = make_root("p9");
write_file(root / "rules" / "r.yar", "rule CacheBudget { condition: false }\n");
std::mt19937_64 rng(kSeed ^ 0x9);
for (int round = 0; round < 24; ++round) {
const uint64_t limit = 32 + rng() % 64;
const int delta = round % 3 - 1;
const auto size = static_cast<std::size_t>(static_cast<int64_t>(limit) + delta);
const auto image = root / "bin" / ("image" + std::to_string(round));
write_file(image, std::string(size, 'x'));
strawberry::Config config;
config.db_path = (root / ("db" + std::to_string(round))).string();
config.yara_rules_path = (root / "rules").string();
config.process_scan_max_bytes = limit;
strawberry::DetectionPipeline pipeline;
PROPERTY(pipeline.initialize(config));
strawberry::ProcessInfo info;
info.pid = 73000 + 2 * round;
info.start_time_ticks = 1;
info.executable_path = image.string();
info.image_path = image.string();
info.image_identity = strawberry::detail::stat_image_identity(image.string());
pipeline.handle_process(info);
CountCachedImageReads count;
{
strawberry::io::ScopedInjector scope(&count);
++info.pid;
++info.start_time_ticks;
if (round % 2) pipeline.handle_inventory_process(info);
else pipeline.handle_process(info);
}
std::cout << "cache-io: seed=" << kSeed << " round=" << round << " delta=" << delta
<< " reads=" << count.reads.load() << std::endl;
PROPERTY(count.reads == 0);
PROPERTY(pipeline.stats().images_scanned == (delta > 0 ? 0u : 1u));
PROPERTY(pipeline.stats().images_too_large == (delta > 0 ? 1u : 0u));
PROPERTY(pipeline.stats().scan_cache_hits == 1);
pipeline.shutdown();
}
fs::remove_all(root);
std::cout << " PASSED" << std::endl;
}
#endif
// P6: a fault schedule that only ever returns EINTR (a signal storm) must not
// hang any read loop: each one gives up after a bounded number of retries.
// fuzz_io_faults found this as a timeout.
class AlwaysEintr final : public strawberry::io::FaultInjector {
public:
FaultDecision decide(Op op) noexcept override {
FaultDecision d;
if (op == Op::read || op == Op::pread) d.error = EINTR;
return d;
}
};
#if STRAWBERRY_HAVE_YARA
// P7: storage failure never silences an alert. Every write/fsync fails (the
// evidence store cannot persist anything) while image reads work: a malicious
// exec is still counted and notified, and the failure is visible as
// persist_failures.
class FailWrites final : public strawberry::io::FaultInjector {
public:
FaultDecision decide(Op op) noexcept override {
FaultDecision d;
if (op == Op::write || op == Op::pwrite || op == Op::fsync || op == Op::fdatasync) d.error = ENOSPC;
return d;
}
};
void property_correlation_rebuild_write_failure() {
std::cout << "P8 correlation rebuild rollback and retry" << std::endl;
const auto root = make_root("p8");
strawberry::Config config;
config.db_path = (root / "evidence.db").string();
config.policy_path = (root / "policy.json").string();
config.feeds_dir = (root / "feeds").string();
config.yara_rules_path = (root / "absent").string();
config.require_rules = false;
strawberry::IndicatorEntry a;
a.id = "a"; a.hash = std::string(64, 'a'); a.rule_name = "Shared"; a.severity = "high"; a.timestamp = 100;
a.metadata = {{"source", "yara"}, {"pid", "101"}, {"process_node_id", "process_101_boot_1"},
{"process_path", "/synthetic/a"}};
auto b = a; b.id = "b"; b.hash = std::string(64, 'b'); b.metadata["pid"] = "102";
b.metadata["process_node_id"] = "process_102_boot_1";
{
strawberry::Database seed;
PROPERTY(seed.open(config.db_path));
PROPERTY(seed.store_indicator(a) && seed.store_indicator(b));
}
strawberry::PolicyStore policy(config.policy_path);
PROPERTY(policy.load());
const auto policy_bytes = strawberry::read_small_file(config.policy_path);
const auto policy_json = policy.to_json();
class FailMetadata final : public strawberry::io::FaultInjector {
public:
explicit FailMetadata(int error) : error_(error) {}
FaultDecision decide(Op op) noexcept override { return {.error = op == Op::stat ? error_ : 0}; }
private:
int error_;
};
for (int error : {EACCES, EIO, EINTR, ENOENT}) {
FailMetadata denied(error);
std::string why;
{ strawberry::io::ScopedInjector scope(&denied); PROPERTY(!policy.load_read_only(&why)); }
PROPERTY(!why.empty() && policy.to_json() == policy_json);
PROPERTY(strawberry::read_small_file(config.policy_path) == policy_bytes);
}
strawberry::DetectionPipeline pipeline;
PROPERTY(pipeline.initialize(config, &policy));
PROPERTY(pipeline.run_pending_correlation());
PROPERTY(pipeline.finding_count() == 3);
std::string error;
PROPERTY(policy.apply("allowlist-add", R"({"kind":"sha256","value":")" + a.hash + R"("})", 0, &error));
pipeline.request_correlation();
FailWrites fail;
{
strawberry::io::ScopedInjector scope(&fail);
PROPERTY(!pipeline.run_pending_correlation());
}
PROPERTY(pipeline.finding_count() == 3); // no partial retirement in the materialized view
PROPERTY(pipeline.has_pending_correlation());
PROPERTY(pipeline.persist_failures() >= 1);
// The store has its own healing backoff; reopening must also retain enough
// evidence to finish the rebuild, rather than forgetting consumed deltas.
pipeline.shutdown();
PROPERTY(pipeline.initialize(config, &policy));
PROPERTY(pipeline.run_pending_correlation());
PROPERTY(pipeline.finding_count() == 1);
pipeline.shutdown();
strawberry::Database direct;
PROPERTY(direct.open(config.db_path));
{
strawberry::io::ScopedInjector scope(&fail);
PROPERTY(strawberry::CorrelationEngine::correlate(direct).empty());
}
PROPERTY(direct.list_findings().size() == 1); // standalone API owns and rolls back its failed view transaction
direct.close();
PROPERTY(direct.open(config.db_path));
PROPERTY(strawberry::CorrelationEngine::correlate(direct).size() == 3);
PROPERTY(direct.list_findings().size() == 3 && direct.list_indicators().size() == 2);
direct.close();
fs::remove_all(root);
std::cout << " PASSED" << std::endl;
}
void property_alert_without_storage() {
std::cout << "P7 alerts survive storage failure" << std::endl;
const auto root = make_root("p7");
write_file(root / "rules" / "r.yar",
"rule FI_Marker { meta: severity = \"high\" strings: $m = \"fault-injection-marker\" condition: $m }\n");
const auto image = root / "bin" / "bad";
write_file(image, std::string("\x7f" "ELF") + "padding fault-injection-marker padding");
strawberry::Config config;
config.db_path = (root / "fi.db").string();
config.yara_rules_path = (root / "rules").string();
strawberry::DetectionPipeline pipeline;
PROPERTY(pipeline.initialize(config));
int notified = 0;
pipeline.set_detection_callback([&](const strawberry::DetectionEvent&) { ++notified; });
FailWrites fail;
{
strawberry::io::ScopedInjector scope(&fail);
strawberry::ProcessInfo info;
info.pid = 4242;
info.start_time_ticks = 7;
info.executable_path = image.string();
info.image_path = image.string();
info.image_identity = strawberry::detail::stat_image_identity(image.string());
pipeline.handle_process(info);
}
const auto stats = pipeline.stats();
PROPERTY(stats.detections == 1);
PROPERTY(notified == 1);
PROPERTY(stats.persist_failures >= 1);
pipeline.shutdown();
fs::remove_all(root);
std::cout << " PASSED (persist_failures " << stats.persist_failures << ")" << std::endl;
}
#endif
void property_eintr_storm_terminates() {
std::cout << "P6 EINTR storm terminates" << std::endl;
const auto root = make_root("p6");
write_file(root / "f", std::string(5000, 'x'));
AlwaysEintr storm;
strawberry::io::ScopedInjector scope(&storm);
const auto start = std::chrono::steady_clock::now();
PROPERTY(!strawberry::read_small_file((root / "f").string()));
const int fd = ::open((root / "f").c_str(), O_RDONLY | O_CLOEXEC);
std::vector<unsigned char> data;
std::string error;
PROPERTY(!strawberry::detail::read_fd_bounded(fd, 1u << 20, data, error));
PROPERTY(strawberry::HashUtils::sha256_fd(fd).empty());
::close(fd);
PROPERTY(strawberry::HashUtils::md5_file((root / "f").string()).empty());
PROPERTY(std::chrono::steady_clock::now() - start < std::chrono::seconds(2));
fs::remove_all(root);
std::cout << " PASSED" << std::endl;
}
// P4: a thread body that throws anything (not just std::exception) is contained.
void property_guarded_threads() {
std::cout << "P4 guarded threads contain any exception" << std::endl;
const auto errors_before = strawberry::thread_errors().load();
std::thread a(strawberry::run_guarded("fi-int", [] { throw 42; }));
std::thread b(strawberry::run_guarded("fi-std", [] { throw std::runtime_error("x"); }));
std::thread c(strawberry::run_guarded("fi-ok", [] {}));
a.join();
b.join();
c.join();
PROPERTY(strawberry::thread_errors().load() == errors_before + 2);
std::cout << " PASSED" << std::endl;
}
// P5: the governor's sampler never throws and yields sentinels under faults.
void property_sampler_under_faults() {
std::cout << "P5 system sampler under faults" << std::endl;
std::mt19937_64 rng(kSeed ^ 0x5);
strawberry::SystemSampler sampler;
for (int round = 0; round < 300; ++round) {
RandomFaults faults(rng(), 0.5);
strawberry::io::ScopedInjector scope(&faults);
try {
const auto s = sampler.sample();
PROPERTY(s.cpu_capacity > 0);
PROPERTY(s.own_cpu_pct <= 100.0);
} catch (...) {
PROPERTY(!"sampler threw");
}
}
std::cout << " PASSED" << std::endl;
}
} // namespace
int main(int argc, char** argv) {
std::cout << "Running fault-injection properties (seed " << kSeed << ")..." << std::endl << std::endl;
if (argc == 2) {
#if STRAWBERRY_HAVE_YARA
if (std::string(argv[1]) == "cache-io") { property_cached_image_does_not_rehash(); return 0; }
#endif
return 2;
}
property_read_small_file();
property_read_fd_bounded();
#if STRAWBERRY_HAVE_YARA
property_pipeline_under_faults();
property_cached_image_does_not_rehash();
#endif
#if STRAWBERRY_HAVE_YARA
property_alert_without_storage();
property_correlation_rebuild_write_failure();
#endif
property_eintr_storm_terminates();
property_guarded_threads();
property_sampler_under_faults();
std::cout << std::endl << "All properties held." << std::endl;
return 0;
}