// Fault-injection properties (built with STRAWBERRY_FAULT_INJECTION=1). // // The 2026-10-25 freeze began with a /proc read failing mid-file (ESRCH) and an // exception escaping. Unit tests, property tests and fuzzers all missed it // because they fed in-memory strings and never made I/O fail. These properties // make every read/open/fstat fail, shorten or pass at random, seeded, and // assert that nothing throws, nothing lies, and the pipeline keeps going. // // A failure prints the seed; reproduce with STRAWBERRY_PROPERTY_SEED=. #include "strawberry/detection_pipeline.hpp" #include "strawberry/correlation_engine.hpp" #include "strawberry/hash_utils.hpp" #include "strawberry/proc_read.hpp" #include "strawberry/policy.hpp" #include "strawberry/resource_governor.hpp" #include "strawberry/sys_io.hpp" #include "strawberry/thread_entry.hpp" #include "../src/image_file.hpp" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #if !defined(STRAWBERRY_FAULT_INJECTION) || !STRAWBERRY_FAULT_INJECTION #error "test_fault_injection must be built with STRAWBERRY_FAULT_INJECTION=1" #endif namespace fs = std::filesystem; using strawberry::io::FaultDecision; using strawberry::io::Op; namespace { uint64_t seed_from_env() { if (const char* text = std::getenv("STRAWBERRY_PROPERTY_SEED")) { return std::strtoull(text, nullptr, 10); } return std::random_device{}(); } const uint64_t kSeed = seed_from_env(); #define PROPERTY(cond) \ do { \ if (!(cond)) { \ std::cerr << "PROPERTY FAILED: " #cond " (seed " << kSeed << ", " << __FILE__ << ":" \ << __LINE__ << ")" << std::endl; \ std::abort(); \ } \ } while (0) // Seeded random faults: each I/O call fails with probability `fail_rate` // (errno drawn from the realistic set) and reads are shortened at random. class RandomFaults final : public strawberry::io::FaultInjector { public: RandomFaults(uint64_t seed, double fail_rate) : rng_(seed), fail_rate_(fail_rate) {} FaultDecision decide(Op op) noexcept override { ++calls_; FaultDecision d; std::uniform_real_distribution unit(0.0, 1.0); if (unit(rng_) < fail_rate_) { static constexpr std::array errors{ESRCH, EIO, ENOENT, EINTR, EAGAIN, EACCES}; d.error = errors[rng_() % errors.size()]; ++faults_; } else if ((op == Op::read || op == Op::pread) && unit(rng_) < 0.3) { d.short_bytes = static_cast(rng_() % 7); // includes 0: a premature EOF } return d; } uint64_t calls() const { return calls_; } uint64_t faults() const { return faults_; } private: std::mt19937_64 rng_; double fail_rate_; uint64_t calls_ = 0; uint64_t faults_ = 0; }; fs::path make_root(const char* name) { const auto root = fs::temp_directory_path() / (std::string("sb-fi-") + name + "-" + std::to_string(::getpid())); std::error_code ec; fs::remove_all(root, ec); fs::create_directories(root); return root; } void write_file(const fs::path& path, const std::string& content) { fs::create_directories(path.parent_path()); std::ofstream(path, std::ios::binary) << content; } std::string random_content(std::mt19937_64& rng, std::size_t max_len) { std::string out(rng() % (max_len + 1), '\0'); for (auto& c : out) c = static_cast(rng() & 0xff); return out; } // P1: read_small_file is never wrong: nullopt, or an exact prefix of the file // (exactly the file when it fits), and it never throws. void property_read_small_file() { std::cout << "P1 read_small_file under faults" << std::endl; const auto root = make_root("p1"); std::mt19937_64 rng(kSeed); for (int round = 0; round < 2000; ++round) { const auto content = random_content(rng, 20000); const auto path = root / "f"; write_file(path, content); const std::size_t max = 1 + rng() % 30000; RandomFaults faults(rng(), 0.2); strawberry::io::ScopedInjector scope(&faults); std::optional got; try { got = strawberry::read_small_file(path.string(), max); } catch (...) { PROPERTY(!"read_small_file threw"); } if (got) { PROPERTY(got->size() <= max); // Short reads are retried until EOF/limit, so a returned value is // either the whole file (capped) or ... it is never a silent // truncation: a premature 0-byte read is indistinguishable from EOF // for /proc, so we only require a prefix. PROPERTY(content.compare(0, got->size(), *got) == 0); } } fs::remove_all(root); std::cout << " PASSED (seed " << kSeed << ")" << std::endl; } // P2: read_fd_bounded is exact or fails with an error message. void property_read_fd_bounded() { std::cout << "P2 read_fd_bounded under faults" << std::endl; const auto root = make_root("p2"); std::mt19937_64 rng(kSeed ^ 0x2); for (int round = 0; round < 2000; ++round) { const auto content = random_content(rng, 70000); const auto path = root / "img"; write_file(path, content); const int fd = ::open(path.c_str(), O_RDONLY | O_CLOEXEC); PROPERTY(fd >= 0); RandomFaults faults(rng(), 0.15); std::vector data; std::string error; bool ok = false; std::size_t pace_calls = 0; const std::size_t allowed_reads = round % 2 == 0 ? rng() % 4 : 128; { strawberry::io::ScopedInjector scope(&faults); try { ok = strawberry::detail::read_fd_bounded(fd, 1u << 20, data, error, [&](std::size_t bytes) { PROPERTY(bytes <= 65536); return ++pace_calls <= allowed_reads; }); } catch (...) { PROPERTY(!"read_fd_bounded threw"); } } ::close(fd); PROPERTY(pace_calls <= allowed_reads + 1); if (ok) { // Success means the exact file: never a silent prefix. PROPERTY(data.size() == content.size()); PROPERTY(std::memcmp(data.data(), content.data(), data.size()) == 0); } else { PROPERTY(!error.empty()); } } // Hashing never throws either. for (int round = 0; round < 500; ++round) { const auto path = root / "h"; write_file(path, random_content(rng, 30000)); RandomFaults faults(rng(), 0.3); strawberry::io::ScopedInjector scope(&faults); try { (void) strawberry::HashUtils::sha256_file(path.string()); (void) strawberry::HashUtils::md5_file(path.string()); } catch (...) { PROPERTY(!"hashing threw"); } } fs::remove_all(root); std::cout << " PASSED" << std::endl; } #if STRAWBERRY_HAVE_YARA // P3: the detection pipeline under random I/O faults: no exception escapes, // a failed image read never produces a cached (wrong) result, and a later // fault-free pass detects what it should. void property_pipeline_under_faults() { std::cout << "P3 detection pipeline under faults" << std::endl; const auto root = make_root("p3"); write_file(root / "rules" / "r.yar", "rule FI_Marker { meta: severity = \"high\" strings: $m = \"fault-injection-marker\" condition: $m }\n"); std::mt19937_64 rng(kSeed ^ 0x3); std::vector images; for (int i = 0; i < 12; ++i) { const auto path = root / "bin" / ("img" + std::to_string(i)); const bool malicious = i % 3 == 0; write_file(path, std::string("\x7f" "ELF") + random_content(rng, 4000) + (malicious ? "fault-injection-marker" : "")); images.push_back(path); } strawberry::Config config; config.db_path = (root / "fi.db").string(); config.yara_rules_path = (root / "rules").string(); strawberry::DetectionPipeline pipeline; PROPERTY(pipeline.initialize(config)); pipeline.set_detection_callback([](const strawberry::DetectionEvent&) { throw 42; }); // must be contained RandomFaults faults(rng(), 0.35); for (int round = 0; round < 600; ++round) { const auto& image = images[rng() % images.size()]; strawberry::ProcessInfo info; info.pid = 10000 + round; info.start_time_ticks = 1 + static_cast(round); info.executable_path = image.string(); info.image_path = image.string(); info.image_identity = strawberry::detail::stat_image_identity(image.string()); strawberry::io::ScopedInjector scope(&faults); try { if (rng() % 2) { pipeline.handle_process(info); } else { pipeline.handle_inventory_process(info); } } catch (...) { PROPERTY(!"handle_process threw under I/O faults"); } } PROPERTY(faults.faults() > 0); // Fault-free pass: every malicious image is detected (no poisoned cache // entry from a failed read hides it), every clean one is not. const auto before = pipeline.stats().detections; for (std::size_t i = 0; i < images.size(); ++i) { strawberry::ProcessInfo info; info.pid = 90000 + static_cast(i); info.start_time_ticks = 90000 + i; info.executable_path = images[i].string(); info.image_path = images[i].string(); info.image_identity = strawberry::detail::stat_image_identity(images[i].string()); pipeline.handle_process(info); } const auto found = pipeline.stats().detections - before; PROPERTY(found == 4); // images 0, 3, 6, 9 // A failed promoted identity check must not poison the rule-scan cache. const strawberry::ProcessRef gone{2147483647, "process_2147483647_abcd1234_1", "/missing"}; for (int retry = 0; retry < 2; ++retry) { const auto result = pipeline.scan_process_memory(gone, "identity regression", false, true); PROPERTY(!result.ok && !result.skipped_duplicate); } pipeline.shutdown(); fs::remove_all(root); std::cout << " PASSED (" << faults.faults() << " injected faults over " << faults.calls() << " I/O calls)" << std::endl; } class CountCachedImageReads final : public strawberry::io::FaultInjector { public: std::atomic reads{0}; FaultDecision decide(Op op) noexcept override { if (op == Op::read || op == Op::pread) ++reads; return {}; } }; // P9: across the size boundary, a cached image outcome never re-reads bytes. // In particular an oversized/unhashed refusal is not permission to hash to EOF. void property_cached_image_does_not_rehash() { std::cout << "P9 cached image byte-read budget" << std::endl; const auto root = make_root("p9"); write_file(root / "rules" / "r.yar", "rule CacheBudget { condition: false }\n"); std::mt19937_64 rng(kSeed ^ 0x9); for (int round = 0; round < 24; ++round) { const uint64_t limit = 32 + rng() % 64; const int delta = round % 3 - 1; const auto size = static_cast(static_cast(limit) + delta); const auto image = root / "bin" / ("image" + std::to_string(round)); write_file(image, std::string(size, 'x')); strawberry::Config config; config.db_path = (root / ("db" + std::to_string(round))).string(); config.yara_rules_path = (root / "rules").string(); config.process_scan_max_bytes = limit; strawberry::DetectionPipeline pipeline; PROPERTY(pipeline.initialize(config)); strawberry::ProcessInfo info; info.pid = 73000 + 2 * round; info.start_time_ticks = 1; info.executable_path = image.string(); info.image_path = image.string(); info.image_identity = strawberry::detail::stat_image_identity(image.string()); pipeline.handle_process(info); CountCachedImageReads count; { strawberry::io::ScopedInjector scope(&count); ++info.pid; ++info.start_time_ticks; if (round % 2) pipeline.handle_inventory_process(info); else pipeline.handle_process(info); } std::cout << "cache-io: seed=" << kSeed << " round=" << round << " delta=" << delta << " reads=" << count.reads.load() << std::endl; PROPERTY(count.reads == 0); PROPERTY(pipeline.stats().images_scanned == (delta > 0 ? 0u : 1u)); PROPERTY(pipeline.stats().images_too_large == (delta > 0 ? 1u : 0u)); PROPERTY(pipeline.stats().scan_cache_hits == 1); pipeline.shutdown(); } fs::remove_all(root); std::cout << " PASSED" << std::endl; } #endif // P6: a fault schedule that only ever returns EINTR (a signal storm) must not // hang any read loop: each one gives up after a bounded number of retries. // fuzz_io_faults found this as a timeout. class AlwaysEintr final : public strawberry::io::FaultInjector { public: FaultDecision decide(Op op) noexcept override { FaultDecision d; if (op == Op::read || op == Op::pread) d.error = EINTR; return d; } }; #if STRAWBERRY_HAVE_YARA // P7: storage failure never silences an alert. Every write/fsync fails (the // evidence store cannot persist anything) while image reads work: a malicious // exec is still counted and notified, and the failure is visible as // persist_failures. class FailWrites final : public strawberry::io::FaultInjector { public: FaultDecision decide(Op op) noexcept override { FaultDecision d; if (op == Op::write || op == Op::pwrite || op == Op::fsync || op == Op::fdatasync) d.error = ENOSPC; return d; } }; void property_correlation_rebuild_write_failure() { std::cout << "P8 correlation rebuild rollback and retry" << std::endl; const auto root = make_root("p8"); strawberry::Config config; config.db_path = (root / "evidence.db").string(); config.policy_path = (root / "policy.json").string(); config.feeds_dir = (root / "feeds").string(); config.yara_rules_path = (root / "absent").string(); config.require_rules = false; strawberry::IndicatorEntry a; a.id = "a"; a.hash = std::string(64, 'a'); a.rule_name = "Shared"; a.severity = "high"; a.timestamp = 100; a.metadata = {{"source", "yara"}, {"pid", "101"}, {"process_node_id", "process_101_boot_1"}, {"process_path", "/synthetic/a"}}; auto b = a; b.id = "b"; b.hash = std::string(64, 'b'); b.metadata["pid"] = "102"; b.metadata["process_node_id"] = "process_102_boot_1"; { strawberry::Database seed; PROPERTY(seed.open(config.db_path)); PROPERTY(seed.store_indicator(a) && seed.store_indicator(b)); } strawberry::PolicyStore policy(config.policy_path); PROPERTY(policy.load()); const auto policy_bytes = strawberry::read_small_file(config.policy_path); const auto policy_json = policy.to_json(); class FailMetadata final : public strawberry::io::FaultInjector { public: explicit FailMetadata(int error) : error_(error) {} FaultDecision decide(Op op) noexcept override { return {.error = op == Op::stat ? error_ : 0}; } private: int error_; }; for (int error : {EACCES, EIO, EINTR, ENOENT}) { FailMetadata denied(error); std::string why; { strawberry::io::ScopedInjector scope(&denied); PROPERTY(!policy.load_read_only(&why)); } PROPERTY(!why.empty() && policy.to_json() == policy_json); PROPERTY(strawberry::read_small_file(config.policy_path) == policy_bytes); } strawberry::DetectionPipeline pipeline; PROPERTY(pipeline.initialize(config, &policy)); PROPERTY(pipeline.run_pending_correlation()); PROPERTY(pipeline.finding_count() == 3); std::string error; PROPERTY(policy.apply("allowlist-add", R"({"kind":"sha256","value":")" + a.hash + R"("})", 0, &error)); pipeline.request_correlation(); FailWrites fail; { strawberry::io::ScopedInjector scope(&fail); PROPERTY(!pipeline.run_pending_correlation()); } PROPERTY(pipeline.finding_count() == 3); // no partial retirement in the materialized view PROPERTY(pipeline.has_pending_correlation()); PROPERTY(pipeline.persist_failures() >= 1); // The store has its own healing backoff; reopening must also retain enough // evidence to finish the rebuild, rather than forgetting consumed deltas. pipeline.shutdown(); PROPERTY(pipeline.initialize(config, &policy)); PROPERTY(pipeline.run_pending_correlation()); PROPERTY(pipeline.finding_count() == 1); pipeline.shutdown(); strawberry::Database direct; PROPERTY(direct.open(config.db_path)); { strawberry::io::ScopedInjector scope(&fail); PROPERTY(strawberry::CorrelationEngine::correlate(direct).empty()); } PROPERTY(direct.list_findings().size() == 1); // standalone API owns and rolls back its failed view transaction direct.close(); PROPERTY(direct.open(config.db_path)); PROPERTY(strawberry::CorrelationEngine::correlate(direct).size() == 3); PROPERTY(direct.list_findings().size() == 3 && direct.list_indicators().size() == 2); direct.close(); fs::remove_all(root); std::cout << " PASSED" << std::endl; } void property_alert_without_storage() { std::cout << "P7 alerts survive storage failure" << std::endl; const auto root = make_root("p7"); write_file(root / "rules" / "r.yar", "rule FI_Marker { meta: severity = \"high\" strings: $m = \"fault-injection-marker\" condition: $m }\n"); const auto image = root / "bin" / "bad"; write_file(image, std::string("\x7f" "ELF") + "padding fault-injection-marker padding"); strawberry::Config config; config.db_path = (root / "fi.db").string(); config.yara_rules_path = (root / "rules").string(); strawberry::DetectionPipeline pipeline; PROPERTY(pipeline.initialize(config)); int notified = 0; pipeline.set_detection_callback([&](const strawberry::DetectionEvent&) { ++notified; }); FailWrites fail; { strawberry::io::ScopedInjector scope(&fail); strawberry::ProcessInfo info; info.pid = 4242; info.start_time_ticks = 7; info.executable_path = image.string(); info.image_path = image.string(); info.image_identity = strawberry::detail::stat_image_identity(image.string()); pipeline.handle_process(info); } const auto stats = pipeline.stats(); PROPERTY(stats.detections == 1); PROPERTY(notified == 1); PROPERTY(stats.persist_failures >= 1); pipeline.shutdown(); fs::remove_all(root); std::cout << " PASSED (persist_failures " << stats.persist_failures << ")" << std::endl; } #endif void property_eintr_storm_terminates() { std::cout << "P6 EINTR storm terminates" << std::endl; const auto root = make_root("p6"); write_file(root / "f", std::string(5000, 'x')); AlwaysEintr storm; strawberry::io::ScopedInjector scope(&storm); const auto start = std::chrono::steady_clock::now(); PROPERTY(!strawberry::read_small_file((root / "f").string())); const int fd = ::open((root / "f").c_str(), O_RDONLY | O_CLOEXEC); std::vector data; std::string error; PROPERTY(!strawberry::detail::read_fd_bounded(fd, 1u << 20, data, error)); PROPERTY(strawberry::HashUtils::sha256_fd(fd).empty()); ::close(fd); PROPERTY(strawberry::HashUtils::md5_file((root / "f").string()).empty()); PROPERTY(std::chrono::steady_clock::now() - start < std::chrono::seconds(2)); fs::remove_all(root); std::cout << " PASSED" << std::endl; } // P4: a thread body that throws anything (not just std::exception) is contained. void property_guarded_threads() { std::cout << "P4 guarded threads contain any exception" << std::endl; const auto errors_before = strawberry::thread_errors().load(); std::thread a(strawberry::run_guarded("fi-int", [] { throw 42; })); std::thread b(strawberry::run_guarded("fi-std", [] { throw std::runtime_error("x"); })); std::thread c(strawberry::run_guarded("fi-ok", [] {})); a.join(); b.join(); c.join(); PROPERTY(strawberry::thread_errors().load() == errors_before + 2); std::cout << " PASSED" << std::endl; } // P5: the governor's sampler never throws and yields sentinels under faults. void property_sampler_under_faults() { std::cout << "P5 system sampler under faults" << std::endl; std::mt19937_64 rng(kSeed ^ 0x5); strawberry::SystemSampler sampler; for (int round = 0; round < 300; ++round) { RandomFaults faults(rng(), 0.5); strawberry::io::ScopedInjector scope(&faults); try { const auto s = sampler.sample(); PROPERTY(s.cpu_capacity > 0); PROPERTY(s.own_cpu_pct <= 100.0); } catch (...) { PROPERTY(!"sampler threw"); } } std::cout << " PASSED" << std::endl; } } // namespace int main(int argc, char** argv) { std::cout << "Running fault-injection properties (seed " << kSeed << ")..." << std::endl << std::endl; if (argc == 2) { #if STRAWBERRY_HAVE_YARA if (std::string(argv[1]) == "cache-io") { property_cached_image_does_not_rehash(); return 0; } #endif return 2; } property_read_small_file(); property_read_fd_bounded(); #if STRAWBERRY_HAVE_YARA property_pipeline_under_faults(); property_cached_image_does_not_rehash(); #endif #if STRAWBERRY_HAVE_YARA property_alert_without_storage(); property_correlation_rebuild_write_failure(); #endif property_eintr_storm_terminates(); property_guarded_threads(); property_sampler_under_faults(); std::cout << std::endl << "All properties held." << std::endl; return 0; }