strawberry/tests/test_fault_injection.cpp
trvon 6628ea9aea
Harden against crashes and I/O faults
Fix for the 2026-10-25 host freeze (an uncaught iostream exception on a
/proc read aborted the daemon; its core dump deadlocked with the exec
guard): non-throwing /proc reads, crash handlers that release fanotify
permission fds, guarded thread entries, from_chars parsing.

Adds the io fault-injection seam with property tests, source invariant
checks (R1-R6) with self-test fixtures, libFuzzer targets and seeds
(including I/O-fault fuzzing), the fold changed-files scan scripts and
the agent workflow docs (CLAUDE.md, prompts).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-25 18:17:17 -06:00

393 lines
15 KiB
C++

// Fault-injection properties (built with STRAWBERRY_FAULT_INJECTION=1).
//
// The 2026-10-25 freeze began with a /proc read failing mid-file (ESRCH) and an
// exception escaping. Unit tests, property tests and fuzzers all missed it
// because they fed in-memory strings and never made I/O fail. These properties
// make every read/open/fstat fail, shorten or pass at random, seeded, and
// assert that nothing throws, nothing lies, and the pipeline keeps going.
//
// A failure prints the seed; reproduce with STRAWBERRY_PROPERTY_SEED=<seed>.
#include "strawberry/detection_pipeline.hpp"
#include "strawberry/hash_utils.hpp"
#include "strawberry/proc_read.hpp"
#include "strawberry/resource_governor.hpp"
#include "strawberry/sys_io.hpp"
#include "strawberry/thread_entry.hpp"
#include "../src/image_file.hpp"
#include <array>
#include <chrono>
#include <atomic>
#include <cassert>
#include <cerrno>
#include <cstdlib>
#include <cstring>
#include <filesystem>
#include <fstream>
#include <iostream>
#include <random>
#include <string>
#include <thread>
#include <vector>
#include <unistd.h>
#if !defined(STRAWBERRY_FAULT_INJECTION) || !STRAWBERRY_FAULT_INJECTION
#error "test_fault_injection must be built with STRAWBERRY_FAULT_INJECTION=1"
#endif
namespace fs = std::filesystem;
using strawberry::io::FaultDecision;
using strawberry::io::Op;
namespace {
uint64_t seed_from_env() {
if (const char* text = std::getenv("STRAWBERRY_PROPERTY_SEED")) {
return std::strtoull(text, nullptr, 10);
}
return std::random_device{}();
}
const uint64_t kSeed = seed_from_env();
#define PROPERTY(cond) \
do { \
if (!(cond)) { \
std::cerr << "PROPERTY FAILED: " #cond " (seed " << kSeed << ", " << __FILE__ << ":" \
<< __LINE__ << ")" << std::endl; \
std::abort(); \
} \
} while (0)
// Seeded random faults: each I/O call fails with probability `fail_rate`
// (errno drawn from the realistic set) and reads are shortened at random.
class RandomFaults final : public strawberry::io::FaultInjector {
public:
RandomFaults(uint64_t seed, double fail_rate) : rng_(seed), fail_rate_(fail_rate) {}
FaultDecision decide(Op op) noexcept override {
++calls_;
FaultDecision d;
std::uniform_real_distribution<double> unit(0.0, 1.0);
if (unit(rng_) < fail_rate_) {
static constexpr std::array<int, 6> errors{ESRCH, EIO, ENOENT, EINTR, EAGAIN, EACCES};
d.error = errors[rng_() % errors.size()];
++faults_;
} else if ((op == Op::read || op == Op::pread) && unit(rng_) < 0.3) {
d.short_bytes = static_cast<long>(rng_() % 7); // includes 0: a premature EOF
}
return d;
}
uint64_t calls() const { return calls_; }
uint64_t faults() const { return faults_; }
private:
std::mt19937_64 rng_;
double fail_rate_;
uint64_t calls_ = 0;
uint64_t faults_ = 0;
};
fs::path make_root(const char* name) {
const auto root = fs::temp_directory_path() / (std::string("sb-fi-") + name + "-" + std::to_string(::getpid()));
std::error_code ec;
fs::remove_all(root, ec);
fs::create_directories(root);
return root;
}
void write_file(const fs::path& path, const std::string& content) {
fs::create_directories(path.parent_path());
std::ofstream(path, std::ios::binary) << content;
}
std::string random_content(std::mt19937_64& rng, std::size_t max_len) {
std::string out(rng() % (max_len + 1), '\0');
for (auto& c : out) c = static_cast<char>(rng() & 0xff);
return out;
}
// P1: read_small_file is never wrong: nullopt, or an exact prefix of the file
// (exactly the file when it fits), and it never throws.
void property_read_small_file() {
std::cout << "P1 read_small_file under faults" << std::endl;
const auto root = make_root("p1");
std::mt19937_64 rng(kSeed);
for (int round = 0; round < 2000; ++round) {
const auto content = random_content(rng, 20000);
const auto path = root / "f";
write_file(path, content);
const std::size_t max = 1 + rng() % 30000;
RandomFaults faults(rng(), 0.2);
strawberry::io::ScopedInjector scope(&faults);
std::optional<std::string> got;
try {
got = strawberry::read_small_file(path.string(), max);
} catch (...) {
PROPERTY(!"read_small_file threw");
}
if (got) {
PROPERTY(got->size() <= max);
// Short reads are retried until EOF/limit, so a returned value is
// either the whole file (capped) or ... it is never a silent
// truncation: a premature 0-byte read is indistinguishable from EOF
// for /proc, so we only require a prefix.
PROPERTY(content.compare(0, got->size(), *got) == 0);
}
}
fs::remove_all(root);
std::cout << " PASSED (seed " << kSeed << ")" << std::endl;
}
// P2: read_fd_bounded is exact or fails with an error message.
void property_read_fd_bounded() {
std::cout << "P2 read_fd_bounded under faults" << std::endl;
const auto root = make_root("p2");
std::mt19937_64 rng(kSeed ^ 0x2);
for (int round = 0; round < 2000; ++round) {
const auto content = random_content(rng, 70000);
const auto path = root / "img";
write_file(path, content);
const int fd = ::open(path.c_str(), O_RDONLY | O_CLOEXEC);
PROPERTY(fd >= 0);
RandomFaults faults(rng(), 0.15);
std::vector<unsigned char> data;
std::string error;
bool ok = false;
{
strawberry::io::ScopedInjector scope(&faults);
try {
ok = strawberry::detail::read_fd_bounded(fd, 1u << 20, data, error);
} catch (...) {
PROPERTY(!"read_fd_bounded threw");
}
}
::close(fd);
if (ok) {
// Success means the exact file: never a silent prefix.
PROPERTY(data.size() == content.size());
PROPERTY(std::memcmp(data.data(), content.data(), data.size()) == 0);
} else {
PROPERTY(!error.empty());
}
}
// Hashing never throws either.
for (int round = 0; round < 500; ++round) {
const auto path = root / "h";
write_file(path, random_content(rng, 30000));
RandomFaults faults(rng(), 0.3);
strawberry::io::ScopedInjector scope(&faults);
try {
(void) strawberry::HashUtils::sha256_file(path.string());
(void) strawberry::HashUtils::md5_file(path.string());
} catch (...) {
PROPERTY(!"hashing threw");
}
}
fs::remove_all(root);
std::cout << " PASSED" << std::endl;
}
#if STRAWBERRY_HAVE_YARA
// P3: the detection pipeline under random I/O faults: no exception escapes,
// a failed image read never produces a cached (wrong) result, and a later
// fault-free pass detects what it should.
void property_pipeline_under_faults() {
std::cout << "P3 detection pipeline under faults" << std::endl;
const auto root = make_root("p3");
write_file(root / "rules" / "r.yar",
"rule FI_Marker { meta: severity = \"high\" strings: $m = \"fault-injection-marker\" condition: $m }\n");
std::mt19937_64 rng(kSeed ^ 0x3);
std::vector<fs::path> images;
for (int i = 0; i < 12; ++i) {
const auto path = root / "bin" / ("img" + std::to_string(i));
const bool malicious = i % 3 == 0;
write_file(path, std::string("\x7f" "ELF") + random_content(rng, 4000) + (malicious ? "fault-injection-marker" : ""));
images.push_back(path);
}
strawberry::Config config;
config.db_path = (root / "fi.db").string();
config.yara_rules_path = (root / "rules").string();
strawberry::DetectionPipeline pipeline;
PROPERTY(pipeline.initialize(config));
pipeline.set_detection_callback([](const strawberry::DetectionEvent&) { throw 42; }); // must be contained
RandomFaults faults(rng(), 0.35);
for (int round = 0; round < 600; ++round) {
const auto& image = images[rng() % images.size()];
strawberry::ProcessInfo info;
info.pid = 10000 + round;
info.start_time_ticks = 1 + static_cast<uint64_t>(round);
info.executable_path = image.string();
info.image_path = image.string();
info.image_identity = strawberry::detail::stat_image_identity(image.string());
strawberry::io::ScopedInjector scope(&faults);
try {
if (rng() % 2) {
pipeline.handle_process(info);
} else {
pipeline.handle_inventory_process(info);
}
} catch (...) {
PROPERTY(!"handle_process threw under I/O faults");
}
}
PROPERTY(faults.faults() > 0);
// Fault-free pass: every malicious image is detected (no poisoned cache
// entry from a failed read hides it), every clean one is not.
const auto before = pipeline.stats().detections;
for (std::size_t i = 0; i < images.size(); ++i) {
strawberry::ProcessInfo info;
info.pid = 90000 + static_cast<int>(i);
info.start_time_ticks = 90000 + i;
info.executable_path = images[i].string();
info.image_path = images[i].string();
info.image_identity = strawberry::detail::stat_image_identity(images[i].string());
pipeline.handle_process(info);
}
const auto found = pipeline.stats().detections - before;
PROPERTY(found == 4); // images 0, 3, 6, 9
pipeline.shutdown();
fs::remove_all(root);
std::cout << " PASSED (" << faults.faults() << " injected faults over " << faults.calls() << " I/O calls)"
<< std::endl;
}
#endif
// P6: a fault schedule that only ever returns EINTR (a signal storm) must not
// hang any read loop: each one gives up after a bounded number of retries.
// fuzz_io_faults found this as a timeout.
class AlwaysEintr final : public strawberry::io::FaultInjector {
public:
FaultDecision decide(Op op) noexcept override {
FaultDecision d;
if (op == Op::read || op == Op::pread) d.error = EINTR;
return d;
}
};
#if STRAWBERRY_HAVE_YARA
// P7: storage failure never silences an alert. Every write/fsync fails (the
// evidence store cannot persist anything) while image reads work: a malicious
// exec is still counted and notified, and the failure is visible as
// persist_failures.
class FailWrites final : public strawberry::io::FaultInjector {
public:
FaultDecision decide(Op op) noexcept override {
FaultDecision d;
if (op == Op::write || op == Op::pwrite || op == Op::fsync || op == Op::fdatasync) d.error = ENOSPC;
return d;
}
};
void property_alert_without_storage() {
std::cout << "P7 alerts survive storage failure" << std::endl;
const auto root = make_root("p7");
write_file(root / "rules" / "r.yar",
"rule FI_Marker { meta: severity = \"high\" strings: $m = \"fault-injection-marker\" condition: $m }\n");
const auto image = root / "bin" / "bad";
write_file(image, std::string("\x7f" "ELF") + "padding fault-injection-marker padding");
strawberry::Config config;
config.db_path = (root / "fi.db").string();
config.yara_rules_path = (root / "rules").string();
strawberry::DetectionPipeline pipeline;
PROPERTY(pipeline.initialize(config));
int notified = 0;
pipeline.set_detection_callback([&](const strawberry::DetectionEvent&) { ++notified; });
FailWrites fail;
{
strawberry::io::ScopedInjector scope(&fail);
strawberry::ProcessInfo info;
info.pid = 4242;
info.start_time_ticks = 7;
info.executable_path = image.string();
info.image_path = image.string();
info.image_identity = strawberry::detail::stat_image_identity(image.string());
pipeline.handle_process(info);
}
const auto stats = pipeline.stats();
PROPERTY(stats.detections == 1);
PROPERTY(notified == 1);
PROPERTY(stats.persist_failures >= 1);
pipeline.shutdown();
fs::remove_all(root);
std::cout << " PASSED (persist_failures " << stats.persist_failures << ")" << std::endl;
}
#endif
void property_eintr_storm_terminates() {
std::cout << "P6 EINTR storm terminates" << std::endl;
const auto root = make_root("p6");
write_file(root / "f", std::string(5000, 'x'));
AlwaysEintr storm;
strawberry::io::ScopedInjector scope(&storm);
const auto start = std::chrono::steady_clock::now();
PROPERTY(!strawberry::read_small_file((root / "f").string()));
const int fd = ::open((root / "f").c_str(), O_RDONLY | O_CLOEXEC);
std::vector<unsigned char> data;
std::string error;
PROPERTY(!strawberry::detail::read_fd_bounded(fd, 1u << 20, data, error));
PROPERTY(strawberry::HashUtils::sha256_fd(fd).empty());
::close(fd);
PROPERTY(strawberry::HashUtils::md5_file((root / "f").string()).empty());
PROPERTY(std::chrono::steady_clock::now() - start < std::chrono::seconds(2));
fs::remove_all(root);
std::cout << " PASSED" << std::endl;
}
// P4: a thread body that throws anything (not just std::exception) is contained.
void property_guarded_threads() {
std::cout << "P4 guarded threads contain any exception" << std::endl;
const auto errors_before = strawberry::thread_errors().load();
std::thread a(strawberry::run_guarded("fi-int", [] { throw 42; }));
std::thread b(strawberry::run_guarded("fi-std", [] { throw std::runtime_error("x"); }));
std::thread c(strawberry::run_guarded("fi-ok", [] {}));
a.join();
b.join();
c.join();
PROPERTY(strawberry::thread_errors().load() == errors_before + 2);
std::cout << " PASSED" << std::endl;
}
// P5: the governor's sampler never throws and yields sentinels under faults.
void property_sampler_under_faults() {
std::cout << "P5 system sampler under faults" << std::endl;
std::mt19937_64 rng(kSeed ^ 0x5);
strawberry::SystemSampler sampler;
for (int round = 0; round < 300; ++round) {
RandomFaults faults(rng(), 0.5);
strawberry::io::ScopedInjector scope(&faults);
try {
const auto s = sampler.sample();
PROPERTY(s.cpu_capacity > 0);
PROPERTY(s.own_cpu_pct <= 100.0);
} catch (...) {
PROPERTY(!"sampler threw");
}
}
std::cout << " PASSED" << std::endl;
}
} // namespace
int main() {
std::cout << "Running fault-injection properties (seed " << kSeed << ")..." << std::endl << std::endl;
property_read_small_file();
property_read_fd_bounded();
#if STRAWBERRY_HAVE_YARA
property_pipeline_under_faults();
#endif
#if STRAWBERRY_HAVE_YARA
property_alert_without_storage();
#endif
property_eintr_storm_terminates();
property_guarded_threads();
property_sampler_under_faults();
std::cout << std::endl << "All properties held." << std::endl;
return 0;
}