2026-11-14 host hang: a yams ASan test run (~110 MiB binaries) pushed strawberry.service past MemoryHigh=768M; the kernel throttled the daemon (own memory.pressure full ~72%), so the exec guard stalled launches (589 1-second prevention timeouts in 80 min, even start-swayidle.sh), scans hit p95 67 s and the control socket stopped answering. - Exec guard (assess_exec) no longer copies an image before deciding: a cached scan answers with no read; an image over guard_full_scan_max_bytes, or when the scan-buffer budget is exhausted, is hashed by streaming (1 MiB chunks, new HashUtils::sha256_md5_fd) and decided on hash / IOC / quarantine (degraded; the pipeline still scans it). Only small budgeted images are copied. - Scan-buffer budget (ResourceGovernor::acquire_scan_buffer, RAII ScanBufferLease): in-memory image copies bounded to memory budget / 4 (balanced 192 MiB); the pipeline waits up to 30 s, on-demand scans 60 s; an oversized request is granted only alone, so no deadlock or starvation. - Balanced pipeline_max_image_bytes 128 -> 64 MiB: larger images get hash/IOC checks only (as on the efficient profile). - Response hashing streams instead of copying the whole executable. - Status: resources.cgroup memory_high_mb / memory_high_events / memory_full_pct / throttled, and resources.scan_buffers; doctor check resources.memory (problem when throttled at MemoryHigh, with the fix). Tests: budget grant/refuse/exclusive/wait/timeout/move, PSI full + memory.events parsers, streaming hash = buffer hash across chunks, guard streams a 33 MiB image with no YARA pass and no buffer grant, streams when the budget is exhausted, full-scans + caches when budget is free, doctor throttle check. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
245 lines
11 KiB
C++
245 lines
11 KiB
C++
// Doctor checks and maintenance actions.
|
|
//
|
|
// D1 checks: each status path (store failed/soft/ok, growth warning with
|
|
// unlimited retention, leftovers, superseded findings, thread errors)
|
|
// D2 superseded findings: old hash/rule/lineage findings with a current
|
|
// stable-id replacement are listed; a lineage finding for the same pid
|
|
// but another parent process (pid reuse) is not; nothing without a
|
|
// replacement is
|
|
// D3 retire_finding removes a finding, and stays removed after reopening
|
|
// D4 clean_store_leftovers: old orphans and temp files go, recent orphans
|
|
// and listed segments stay; dry run removes nothing
|
|
|
|
#include "strawberry/correlation_engine.hpp"
|
|
#include "strawberry/database.hpp"
|
|
#include "strawberry/doctor.hpp"
|
|
#include "strawberry/hash_utils.hpp"
|
|
#include "strawberry/store/log_store.hpp"
|
|
|
|
#include <fcntl.h>
|
|
#include <sys/stat.h>
|
|
#include <unistd.h>
|
|
|
|
#include <cstdlib>
|
|
#include <ctime>
|
|
#include <filesystem>
|
|
#include <fstream>
|
|
#include <algorithm>
|
|
#include <iostream>
|
|
#include <set>
|
|
#include <string>
|
|
|
|
namespace fs = std::filesystem;
|
|
using namespace strawberry;
|
|
|
|
namespace {
|
|
|
|
#define CHECK(cond) \
|
|
do { \
|
|
if (!(cond)) { \
|
|
std::cerr << "CHECK FAILED: " #cond " (" << __FILE__ << ":" << __LINE__ << ")" << std::endl; \
|
|
std::abort(); \
|
|
} \
|
|
} while (0)
|
|
|
|
const DoctorCheck& find(const std::vector<DoctorCheck>& checks, const std::string& id) {
|
|
for (const auto& c : checks) {
|
|
if (c.id == id) return c;
|
|
}
|
|
std::cerr << "no check " << id << std::endl;
|
|
std::abort();
|
|
}
|
|
|
|
std::string suffix(const std::string& s) {
|
|
return HashUtils::sha256_data(reinterpret_cast<const unsigned char*>(s.data()), s.size()).substr(0, 16);
|
|
}
|
|
|
|
void test_checks() {
|
|
DoctorInputs in;
|
|
StoreHealth h;
|
|
h.floor = "ok";
|
|
h.free_bytes = 10ull << 30;
|
|
h.soft_floor_bytes = 2ull << 30;
|
|
in.store = h;
|
|
auto checks = run_doctor_checks(in);
|
|
CHECK(find(checks, "store.health").status == CheckStatus::ok);
|
|
CHECK(find(checks, "store.growth").status == CheckStatus::info); // no telemetry yet
|
|
CHECK(find(checks, "store.leftovers").status == CheckStatus::ok);
|
|
CHECK(find(checks, "findings.allowlisted").status == CheckStatus::ok);
|
|
CHECK(doctor_json(checks).find("\"healthy\":true") != std::string::npos);
|
|
|
|
// 1 GiB/day with 8 GiB of room and unlimited retention: warn, with advice.
|
|
in.files.telemetry_first_ns = 1;
|
|
in.files.telemetry_last_ns = 1 + 86'400'000'000'000LL;
|
|
in.files.telemetry_sealed_bytes = 1ull << 30;
|
|
checks = run_doctor_checks(in);
|
|
CHECK(find(checks, "store.growth").status == CheckStatus::warn);
|
|
CHECK(find(checks, "store.growth").advice.find("store_retention_days") != std::string::npos);
|
|
in.store_retention_days = 30;
|
|
CHECK(find(run_doctor_checks(in), "store.growth").status == CheckStatus::info);
|
|
|
|
in.files.orphan_files = 2;
|
|
in.superseded_findings = 3;
|
|
in.allowlisted_findings = 2;
|
|
in.findings = 10;
|
|
in.thread_errors = 1;
|
|
h.floor = "soft";
|
|
in.store = h;
|
|
checks = run_doctor_checks(in);
|
|
CHECK(find(checks, "store.health").status == CheckStatus::warn);
|
|
CHECK(find(checks, "store.leftovers").action == kActionCleanStore);
|
|
CHECK(find(checks, "findings.superseded").action == kActionRetireSuperseded);
|
|
CHECK(find(checks, "findings.allowlisted").status == CheckStatus::warn);
|
|
CHECK(find(checks, "findings.allowlisted").action == kActionRetireAllowlisted);
|
|
CHECK(find(checks, "daemon.errors").status == CheckStatus::problem);
|
|
CHECK(doctor_json(checks).find("\"healthy\":false") != std::string::npos);
|
|
h.telemetry.failed = true;
|
|
in.store = h;
|
|
CHECK(find(run_doctor_checks(in), "store.health").status == CheckStatus::problem);
|
|
in.store.reset();
|
|
CHECK(find(run_doctor_checks(in), "store.health").status == CheckStatus::problem);
|
|
|
|
DoctorInputs models;
|
|
CHECK(find(run_doctor_checks(models), "model").summary.find("no inspection model") != std::string::npos);
|
|
{
|
|
DoctorInputs rarity;
|
|
const auto none = run_doctor_checks(rarity);
|
|
CHECK(std::none_of(none.begin(), none.end(), [](const auto& c) { return c.id == "model.rarity_state"; }));
|
|
rarity.rarity_state_error = "save failed: mkdir /var/lib/strawberry/model: Permission denied";
|
|
CHECK(find(run_doctor_checks(rarity), "model.rarity_state").status == CheckStatus::warn);
|
|
}
|
|
models.models.push_back({"shadow", true, true, "", 4, 1});
|
|
CHECK(find(run_doctor_checks(models), "model").summary.find("shadow generation 4") != std::string::npos);
|
|
models.models.push_back({"promoted", true, false, "checksum mismatch", 0, 0});
|
|
const auto bad = find(run_doctor_checks(models), "model");
|
|
CHECK(bad.status == CheckStatus::problem && bad.summary.find("checksum mismatch") != std::string::npos);
|
|
std::cout << "D1 checks: PASSED" << std::endl;
|
|
}
|
|
|
|
FindingRecord finding(const std::string& id, const std::string& rule, std::map<std::string, std::string> meta,
|
|
std::vector<std::string> nodes = {}) {
|
|
FindingRecord f;
|
|
f.id = id;
|
|
f.title = rule;
|
|
f.metadata = std::move(meta);
|
|
f.metadata["rule"] = rule;
|
|
f.related_node_ids = std::move(nodes);
|
|
return f;
|
|
}
|
|
|
|
void test_superseded() {
|
|
const std::string H(64, 'a');
|
|
std::vector<FindingRecord> fs{
|
|
// hash: two old copies and the current one
|
|
finding("finding_hash_yara_old1", "same_hash_multiple_process_matches", {{"hash", H}}),
|
|
finding("finding_hash_yara_old2", "same_hash_multiple_process_matches", {{"hash", H}}),
|
|
finding("finding_hash_yara_" + suffix("hash\n" + H), "same_hash_multiple_process_matches", {{"hash", H}}),
|
|
// rule: an old copy without a current one stays
|
|
finding("finding_rule_yara_lonely", "same_yara_rule_multiple_processes", {{"matched_rule", "Lonely"}}),
|
|
finding("finding_rule_yara_old", "same_yara_rule_multiple_processes", {{"matched_rule", "R"}}),
|
|
finding("finding_rule_yara_" + suffix("rule\nR"), "same_yara_rule_multiple_processes", {{"matched_rule", "R"}}),
|
|
// lineage: current for parent node P (pid 50); an old one naming P; an
|
|
// old one for pid 50 but another boot's parent
|
|
finding("finding_lineage_yara_" + suffix("process_50_bootA_7"), "lineage_multiple_yara_matches",
|
|
{{"parent_pid", "50"}, {"parent_process_node_id", "process_50_bootA_7"}}, {"process_50_bootA_7"}),
|
|
finding("finding_lineage_yara_old", "lineage_multiple_yara_matches", {{"parent_pid", "50"}},
|
|
{"process_50_bootA_7", "process_51_bootA_9"}),
|
|
finding("finding_lineage_yara_reused", "lineage_multiple_yara_matches", {{"parent_pid", "50"}},
|
|
{"process_50_bootB_3"}),
|
|
// a process finding is never superseded
|
|
finding("finding_process_yara_x", "process_executable_yara_match", {{"pid", "1"}}),
|
|
};
|
|
const auto pairs = CorrelationEngine::superseded_findings(fs);
|
|
std::set<std::string> old;
|
|
for (const auto& [o, n] : pairs) {
|
|
old.insert(o);
|
|
CHECK(o != n);
|
|
}
|
|
CHECK(old == (std::set<std::string>{"finding_hash_yara_old1", "finding_hash_yara_old2", "finding_rule_yara_old",
|
|
"finding_lineage_yara_old"}));
|
|
std::cout << "D2 superseded findings: PASSED" << std::endl;
|
|
}
|
|
|
|
void touch_old(const fs::path& p, int days) {
|
|
std::ofstream(p) << "x";
|
|
struct timespec times[2];
|
|
times[0].tv_sec = times[1].tv_sec = ::time(nullptr) - days * 86400;
|
|
times[0].tv_nsec = times[1].tv_nsec = 0;
|
|
CHECK(::utimensat(AT_FDCWD, p.c_str(), times, 0) == 0);
|
|
}
|
|
|
|
void test_store() {
|
|
const auto dir = fs::temp_directory_path() / ("strawberry_doctor_" + std::to_string(::getpid()));
|
|
std::error_code ec;
|
|
fs::remove_all(dir, ec);
|
|
{
|
|
Database db;
|
|
CHECK(db.open(dir.string()));
|
|
FindingRecord f;
|
|
f.id = "finding_hash_yara_old1";
|
|
f.title = "t";
|
|
CHECK(db.store_finding(f));
|
|
f.id = "keep";
|
|
CHECK(db.store_finding(f));
|
|
CHECK(db.retire_finding("finding_hash_yara_old1", "new", "superseded"));
|
|
CHECK(!db.retire_finding("absent", "new", "superseded"));
|
|
CHECK(db.list_findings().size() == 1);
|
|
db.close();
|
|
}
|
|
{
|
|
Database db; // D3: replayed from the stream
|
|
CHECK(db.open_read_only(dir.string()));
|
|
const auto list = db.list_findings();
|
|
CHECK(list.size() == 1 && list[0].id == "keep");
|
|
db.close();
|
|
}
|
|
std::cout << "D3 retire_finding: PASSED" << std::endl;
|
|
|
|
fs::create_directories(dir / "orphan");
|
|
touch_old(dir / "orphan" / "telemetry-seg-1.log.123", 30);
|
|
touch_old(dir / "orphan" / "evidence-seg-2.log.456", 1);
|
|
touch_old(dir / "telemetry" / "seg-9.log.compact", 0);
|
|
std::size_t listed_before = 0;
|
|
for (const auto& e : fs::directory_iterator(dir / "evidence")) listed_before += e.path().extension() == ".log";
|
|
auto files = store::inspect_store_files(dir.string());
|
|
CHECK(files.orphan_files == 2 && files.temp_files == 1);
|
|
const auto dry = store::clean_store_leftovers(dir.string(), 7 * 86'400'000'000'000LL, true);
|
|
CHECK(dry.files == 2 && fs::exists(dir / "orphan" / "telemetry-seg-1.log.123")); // reported, not removed
|
|
const auto done = store::clean_store_leftovers(dir.string(), 7 * 86'400'000'000'000LL, false);
|
|
CHECK(done.files == 2 && done.problems.empty());
|
|
CHECK(!fs::exists(dir / "orphan" / "telemetry-seg-1.log.123"));
|
|
CHECK(fs::exists(dir / "orphan" / "evidence-seg-2.log.456")); // too recent
|
|
CHECK(!fs::exists(dir / "telemetry" / "seg-9.log.compact"));
|
|
std::size_t listed_after = 0;
|
|
for (const auto& e : fs::directory_iterator(dir / "evidence")) listed_after += e.path().extension() == ".log";
|
|
CHECK(listed_after == listed_before && listed_before > 0);
|
|
CHECK(store::verify_store(dir.string()).problems.empty());
|
|
fs::remove_all(dir, ec);
|
|
std::cout << "D4 clean_store_leftovers: PASSED" << std::endl;
|
|
}
|
|
|
|
} // namespace
|
|
|
|
void test_memory_throttle() {
|
|
DoctorInputs in;
|
|
auto checks = run_doctor_checks(in);
|
|
for (const auto& c : checks) CHECK(c.id != "resources.memory"); // unknown (-1): no check
|
|
in.memory_full_pct = 2;
|
|
in.memory_current_bytes = 500ull << 20;
|
|
in.memory_high_bytes = 768ull << 20;
|
|
CHECK(find(run_doctor_checks(in), "resources.memory").status == CheckStatus::ok);
|
|
in.memory_full_pct = 72.35;
|
|
in.memory_current_bytes = 822ull << 20;
|
|
const auto bad = find(run_doctor_checks(in), "resources.memory");
|
|
CHECK(bad.status == CheckStatus::problem);
|
|
CHECK(bad.summary.find("throttled") != std::string::npos && bad.summary.find("72%") != std::string::npos);
|
|
CHECK(bad.advice.find("MemoryHigh") != std::string::npos);
|
|
}
|
|
|
|
int main() {
|
|
test_checks();
|
|
test_memory_throttle();
|
|
test_superseded();
|
|
test_store();
|
|
return 0;
|
|
}
|