strawberry/tests/test_doctor.cpp
trvon 5685600e57
Daemon memory: budget scan buffers, stream large images on the launch path, report throttling
2026-11-14 host hang: a yams ASan test run (~110 MiB binaries) pushed
strawberry.service past MemoryHigh=768M; the kernel throttled the daemon
(own memory.pressure full ~72%), so the exec guard stalled launches (589
1-second prevention timeouts in 80 min, even start-swayidle.sh), scans hit
p95 67 s and the control socket stopped answering.

- Exec guard (assess_exec) no longer copies an image before deciding: a cached
  scan answers with no read; an image over guard_full_scan_max_bytes, or when
  the scan-buffer budget is exhausted, is hashed by streaming (1 MiB chunks,
  new HashUtils::sha256_md5_fd) and decided on hash / IOC / quarantine
  (degraded; the pipeline still scans it). Only small budgeted images are copied.
- Scan-buffer budget (ResourceGovernor::acquire_scan_buffer, RAII
  ScanBufferLease): in-memory image copies bounded to memory budget / 4
  (balanced 192 MiB); the pipeline waits up to 30 s, on-demand scans 60 s;
  an oversized request is granted only alone, so no deadlock or starvation.
- Balanced pipeline_max_image_bytes 128 -> 64 MiB: larger images get hash/IOC
  checks only (as on the efficient profile).
- Response hashing streams instead of copying the whole executable.
- Status: resources.cgroup memory_high_mb / memory_high_events /
  memory_full_pct / throttled, and resources.scan_buffers; doctor check
  resources.memory (problem when throttled at MemoryHigh, with the fix).

Tests: budget grant/refuse/exclusive/wait/timeout/move, PSI full + memory.events
parsers, streaming hash = buffer hash across chunks, guard streams a 33 MiB
image with no YARA pass and no buffer grant, streams when the budget is
exhausted, full-scans + caches when budget is free, doctor throttle check.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-14 15:36:14 -07:00

245 lines
11 KiB
C++

// Doctor checks and maintenance actions.
//
// D1 checks: each status path (store failed/soft/ok, growth warning with
// unlimited retention, leftovers, superseded findings, thread errors)
// D2 superseded findings: old hash/rule/lineage findings with a current
// stable-id replacement are listed; a lineage finding for the same pid
// but another parent process (pid reuse) is not; nothing without a
// replacement is
// D3 retire_finding removes a finding, and stays removed after reopening
// D4 clean_store_leftovers: old orphans and temp files go, recent orphans
// and listed segments stay; dry run removes nothing
#include "strawberry/correlation_engine.hpp"
#include "strawberry/database.hpp"
#include "strawberry/doctor.hpp"
#include "strawberry/hash_utils.hpp"
#include "strawberry/store/log_store.hpp"
#include <fcntl.h>
#include <sys/stat.h>
#include <unistd.h>
#include <cstdlib>
#include <ctime>
#include <filesystem>
#include <fstream>
#include <algorithm>
#include <iostream>
#include <set>
#include <string>
namespace fs = std::filesystem;
using namespace strawberry;
namespace {
#define CHECK(cond) \
do { \
if (!(cond)) { \
std::cerr << "CHECK FAILED: " #cond " (" << __FILE__ << ":" << __LINE__ << ")" << std::endl; \
std::abort(); \
} \
} while (0)
const DoctorCheck& find(const std::vector<DoctorCheck>& checks, const std::string& id) {
for (const auto& c : checks) {
if (c.id == id) return c;
}
std::cerr << "no check " << id << std::endl;
std::abort();
}
std::string suffix(const std::string& s) {
return HashUtils::sha256_data(reinterpret_cast<const unsigned char*>(s.data()), s.size()).substr(0, 16);
}
void test_checks() {
DoctorInputs in;
StoreHealth h;
h.floor = "ok";
h.free_bytes = 10ull << 30;
h.soft_floor_bytes = 2ull << 30;
in.store = h;
auto checks = run_doctor_checks(in);
CHECK(find(checks, "store.health").status == CheckStatus::ok);
CHECK(find(checks, "store.growth").status == CheckStatus::info); // no telemetry yet
CHECK(find(checks, "store.leftovers").status == CheckStatus::ok);
CHECK(find(checks, "findings.allowlisted").status == CheckStatus::ok);
CHECK(doctor_json(checks).find("\"healthy\":true") != std::string::npos);
// 1 GiB/day with 8 GiB of room and unlimited retention: warn, with advice.
in.files.telemetry_first_ns = 1;
in.files.telemetry_last_ns = 1 + 86'400'000'000'000LL;
in.files.telemetry_sealed_bytes = 1ull << 30;
checks = run_doctor_checks(in);
CHECK(find(checks, "store.growth").status == CheckStatus::warn);
CHECK(find(checks, "store.growth").advice.find("store_retention_days") != std::string::npos);
in.store_retention_days = 30;
CHECK(find(run_doctor_checks(in), "store.growth").status == CheckStatus::info);
in.files.orphan_files = 2;
in.superseded_findings = 3;
in.allowlisted_findings = 2;
in.findings = 10;
in.thread_errors = 1;
h.floor = "soft";
in.store = h;
checks = run_doctor_checks(in);
CHECK(find(checks, "store.health").status == CheckStatus::warn);
CHECK(find(checks, "store.leftovers").action == kActionCleanStore);
CHECK(find(checks, "findings.superseded").action == kActionRetireSuperseded);
CHECK(find(checks, "findings.allowlisted").status == CheckStatus::warn);
CHECK(find(checks, "findings.allowlisted").action == kActionRetireAllowlisted);
CHECK(find(checks, "daemon.errors").status == CheckStatus::problem);
CHECK(doctor_json(checks).find("\"healthy\":false") != std::string::npos);
h.telemetry.failed = true;
in.store = h;
CHECK(find(run_doctor_checks(in), "store.health").status == CheckStatus::problem);
in.store.reset();
CHECK(find(run_doctor_checks(in), "store.health").status == CheckStatus::problem);
DoctorInputs models;
CHECK(find(run_doctor_checks(models), "model").summary.find("no inspection model") != std::string::npos);
{
DoctorInputs rarity;
const auto none = run_doctor_checks(rarity);
CHECK(std::none_of(none.begin(), none.end(), [](const auto& c) { return c.id == "model.rarity_state"; }));
rarity.rarity_state_error = "save failed: mkdir /var/lib/strawberry/model: Permission denied";
CHECK(find(run_doctor_checks(rarity), "model.rarity_state").status == CheckStatus::warn);
}
models.models.push_back({"shadow", true, true, "", 4, 1});
CHECK(find(run_doctor_checks(models), "model").summary.find("shadow generation 4") != std::string::npos);
models.models.push_back({"promoted", true, false, "checksum mismatch", 0, 0});
const auto bad = find(run_doctor_checks(models), "model");
CHECK(bad.status == CheckStatus::problem && bad.summary.find("checksum mismatch") != std::string::npos);
std::cout << "D1 checks: PASSED" << std::endl;
}
FindingRecord finding(const std::string& id, const std::string& rule, std::map<std::string, std::string> meta,
std::vector<std::string> nodes = {}) {
FindingRecord f;
f.id = id;
f.title = rule;
f.metadata = std::move(meta);
f.metadata["rule"] = rule;
f.related_node_ids = std::move(nodes);
return f;
}
void test_superseded() {
const std::string H(64, 'a');
std::vector<FindingRecord> fs{
// hash: two old copies and the current one
finding("finding_hash_yara_old1", "same_hash_multiple_process_matches", {{"hash", H}}),
finding("finding_hash_yara_old2", "same_hash_multiple_process_matches", {{"hash", H}}),
finding("finding_hash_yara_" + suffix("hash\n" + H), "same_hash_multiple_process_matches", {{"hash", H}}),
// rule: an old copy without a current one stays
finding("finding_rule_yara_lonely", "same_yara_rule_multiple_processes", {{"matched_rule", "Lonely"}}),
finding("finding_rule_yara_old", "same_yara_rule_multiple_processes", {{"matched_rule", "R"}}),
finding("finding_rule_yara_" + suffix("rule\nR"), "same_yara_rule_multiple_processes", {{"matched_rule", "R"}}),
// lineage: current for parent node P (pid 50); an old one naming P; an
// old one for pid 50 but another boot's parent
finding("finding_lineage_yara_" + suffix("process_50_bootA_7"), "lineage_multiple_yara_matches",
{{"parent_pid", "50"}, {"parent_process_node_id", "process_50_bootA_7"}}, {"process_50_bootA_7"}),
finding("finding_lineage_yara_old", "lineage_multiple_yara_matches", {{"parent_pid", "50"}},
{"process_50_bootA_7", "process_51_bootA_9"}),
finding("finding_lineage_yara_reused", "lineage_multiple_yara_matches", {{"parent_pid", "50"}},
{"process_50_bootB_3"}),
// a process finding is never superseded
finding("finding_process_yara_x", "process_executable_yara_match", {{"pid", "1"}}),
};
const auto pairs = CorrelationEngine::superseded_findings(fs);
std::set<std::string> old;
for (const auto& [o, n] : pairs) {
old.insert(o);
CHECK(o != n);
}
CHECK(old == (std::set<std::string>{"finding_hash_yara_old1", "finding_hash_yara_old2", "finding_rule_yara_old",
"finding_lineage_yara_old"}));
std::cout << "D2 superseded findings: PASSED" << std::endl;
}
void touch_old(const fs::path& p, int days) {
std::ofstream(p) << "x";
struct timespec times[2];
times[0].tv_sec = times[1].tv_sec = ::time(nullptr) - days * 86400;
times[0].tv_nsec = times[1].tv_nsec = 0;
CHECK(::utimensat(AT_FDCWD, p.c_str(), times, 0) == 0);
}
void test_store() {
const auto dir = fs::temp_directory_path() / ("strawberry_doctor_" + std::to_string(::getpid()));
std::error_code ec;
fs::remove_all(dir, ec);
{
Database db;
CHECK(db.open(dir.string()));
FindingRecord f;
f.id = "finding_hash_yara_old1";
f.title = "t";
CHECK(db.store_finding(f));
f.id = "keep";
CHECK(db.store_finding(f));
CHECK(db.retire_finding("finding_hash_yara_old1", "new", "superseded"));
CHECK(!db.retire_finding("absent", "new", "superseded"));
CHECK(db.list_findings().size() == 1);
db.close();
}
{
Database db; // D3: replayed from the stream
CHECK(db.open_read_only(dir.string()));
const auto list = db.list_findings();
CHECK(list.size() == 1 && list[0].id == "keep");
db.close();
}
std::cout << "D3 retire_finding: PASSED" << std::endl;
fs::create_directories(dir / "orphan");
touch_old(dir / "orphan" / "telemetry-seg-1.log.123", 30);
touch_old(dir / "orphan" / "evidence-seg-2.log.456", 1);
touch_old(dir / "telemetry" / "seg-9.log.compact", 0);
std::size_t listed_before = 0;
for (const auto& e : fs::directory_iterator(dir / "evidence")) listed_before += e.path().extension() == ".log";
auto files = store::inspect_store_files(dir.string());
CHECK(files.orphan_files == 2 && files.temp_files == 1);
const auto dry = store::clean_store_leftovers(dir.string(), 7 * 86'400'000'000'000LL, true);
CHECK(dry.files == 2 && fs::exists(dir / "orphan" / "telemetry-seg-1.log.123")); // reported, not removed
const auto done = store::clean_store_leftovers(dir.string(), 7 * 86'400'000'000'000LL, false);
CHECK(done.files == 2 && done.problems.empty());
CHECK(!fs::exists(dir / "orphan" / "telemetry-seg-1.log.123"));
CHECK(fs::exists(dir / "orphan" / "evidence-seg-2.log.456")); // too recent
CHECK(!fs::exists(dir / "telemetry" / "seg-9.log.compact"));
std::size_t listed_after = 0;
for (const auto& e : fs::directory_iterator(dir / "evidence")) listed_after += e.path().extension() == ".log";
CHECK(listed_after == listed_before && listed_before > 0);
CHECK(store::verify_store(dir.string()).problems.empty());
fs::remove_all(dir, ec);
std::cout << "D4 clean_store_leftovers: PASSED" << std::endl;
}
} // namespace
void test_memory_throttle() {
DoctorInputs in;
auto checks = run_doctor_checks(in);
for (const auto& c : checks) CHECK(c.id != "resources.memory"); // unknown (-1): no check
in.memory_full_pct = 2;
in.memory_current_bytes = 500ull << 20;
in.memory_high_bytes = 768ull << 20;
CHECK(find(run_doctor_checks(in), "resources.memory").status == CheckStatus::ok);
in.memory_full_pct = 72.35;
in.memory_current_bytes = 822ull << 20;
const auto bad = find(run_doctor_checks(in), "resources.memory");
CHECK(bad.status == CheckStatus::problem);
CHECK(bad.summary.find("throttled") != std::string::npos && bad.summary.find("72%") != std::string::npos);
CHECK(bad.advice.find("MemoryHigh") != std::string::npos);
}
int main() {
test_checks();
test_memory_throttle();
test_superseded();
test_store();
return 0;
}