// Doctor checks and maintenance actions. // // D1 checks: each status path (store failed/soft/ok, growth warning with // unlimited retention, leftovers, superseded findings, thread errors) // D2 superseded findings: old hash/rule/lineage findings with a current // stable-id replacement are listed; a lineage finding for the same pid // but another parent process (pid reuse) is not; nothing without a // replacement is // D3 retire_finding removes a finding, and stays removed after reopening // D4 clean_store_leftovers: old orphans and temp files go, recent orphans // and listed segments stay; dry run removes nothing #include "strawberry/correlation_engine.hpp" #include "strawberry/database.hpp" #include "strawberry/doctor.hpp" #include "strawberry/hash_utils.hpp" #include "strawberry/store/log_store.hpp" #include #include #include #include #include #include #include #include #include #include #include namespace fs = std::filesystem; using namespace strawberry; namespace { #define CHECK(cond) \ do { \ if (!(cond)) { \ std::cerr << "CHECK FAILED: " #cond " (" << __FILE__ << ":" << __LINE__ << ")" << std::endl; \ std::abort(); \ } \ } while (0) const DoctorCheck& find(const std::vector& checks, const std::string& id) { for (const auto& c : checks) { if (c.id == id) return c; } std::cerr << "no check " << id << std::endl; std::abort(); } std::string suffix(const std::string& s) { return HashUtils::sha256_data(reinterpret_cast(s.data()), s.size()).substr(0, 16); } void test_checks() { DoctorInputs in; StoreHealth h; h.floor = "ok"; h.free_bytes = 10ull << 30; h.soft_floor_bytes = 2ull << 30; in.store = h; auto checks = run_doctor_checks(in); CHECK(find(checks, "store.health").status == CheckStatus::ok); CHECK(find(checks, "store.growth").status == CheckStatus::info); // no telemetry yet CHECK(find(checks, "store.leftovers").status == CheckStatus::ok); CHECK(find(checks, "findings.allowlisted").status == CheckStatus::ok); CHECK(doctor_json(checks).find("\"healthy\":true") != std::string::npos); // 1 GiB/day with 8 GiB of room and unlimited retention: warn, with advice. in.files.telemetry_first_ns = 1; in.files.telemetry_last_ns = 1 + 86'400'000'000'000LL; in.files.telemetry_sealed_bytes = 1ull << 30; checks = run_doctor_checks(in); CHECK(find(checks, "store.growth").status == CheckStatus::warn); CHECK(find(checks, "store.growth").advice.find("store_retention_days") != std::string::npos); in.store_retention_days = 30; CHECK(find(run_doctor_checks(in), "store.growth").status == CheckStatus::info); in.files.orphan_files = 2; in.superseded_findings = 3; in.allowlisted_findings = 2; in.findings = 10; in.thread_errors = 1; h.floor = "soft"; in.store = h; checks = run_doctor_checks(in); CHECK(find(checks, "store.health").status == CheckStatus::warn); CHECK(find(checks, "store.leftovers").action == kActionCleanStore); CHECK(find(checks, "findings.superseded").action == kActionRetireSuperseded); CHECK(find(checks, "findings.allowlisted").status == CheckStatus::warn); CHECK(find(checks, "findings.allowlisted").action == kActionRetireAllowlisted); CHECK(find(checks, "daemon.errors").status == CheckStatus::problem); CHECK(doctor_json(checks).find("\"healthy\":false") != std::string::npos); h.telemetry.failed = true; in.store = h; CHECK(find(run_doctor_checks(in), "store.health").status == CheckStatus::problem); in.store.reset(); CHECK(find(run_doctor_checks(in), "store.health").status == CheckStatus::problem); DoctorInputs models; CHECK(find(run_doctor_checks(models), "model").summary.find("no inspection model") != std::string::npos); { DoctorInputs rarity; const auto none = run_doctor_checks(rarity); CHECK(std::none_of(none.begin(), none.end(), [](const auto& c) { return c.id == "model.rarity_state"; })); rarity.rarity_state_error = "save failed: mkdir /var/lib/strawberry/model: Permission denied"; CHECK(find(run_doctor_checks(rarity), "model.rarity_state").status == CheckStatus::warn); } models.models.push_back({"shadow", true, true, "", 4, 1}); CHECK(find(run_doctor_checks(models), "model").summary.find("shadow generation 4") != std::string::npos); models.models.push_back({"promoted", true, false, "checksum mismatch", 0, 0}); const auto bad = find(run_doctor_checks(models), "model"); CHECK(bad.status == CheckStatus::problem && bad.summary.find("checksum mismatch") != std::string::npos); std::cout << "D1 checks: PASSED" << std::endl; } FindingRecord finding(const std::string& id, const std::string& rule, std::map meta, std::vector nodes = {}) { FindingRecord f; f.id = id; f.title = rule; f.metadata = std::move(meta); f.metadata["rule"] = rule; f.related_node_ids = std::move(nodes); return f; } void test_superseded() { const std::string H(64, 'a'); std::vector fs{ // hash: two old copies and the current one finding("finding_hash_yara_old1", "same_hash_multiple_process_matches", {{"hash", H}}), finding("finding_hash_yara_old2", "same_hash_multiple_process_matches", {{"hash", H}}), finding("finding_hash_yara_" + suffix("hash\n" + H), "same_hash_multiple_process_matches", {{"hash", H}}), // rule: an old copy without a current one stays finding("finding_rule_yara_lonely", "same_yara_rule_multiple_processes", {{"matched_rule", "Lonely"}}), finding("finding_rule_yara_old", "same_yara_rule_multiple_processes", {{"matched_rule", "R"}}), finding("finding_rule_yara_" + suffix("rule\nR"), "same_yara_rule_multiple_processes", {{"matched_rule", "R"}}), // lineage: current for parent node P (pid 50); an old one naming P; an // old one for pid 50 but another boot's parent finding("finding_lineage_yara_" + suffix("process_50_bootA_7"), "lineage_multiple_yara_matches", {{"parent_pid", "50"}, {"parent_process_node_id", "process_50_bootA_7"}}, {"process_50_bootA_7"}), finding("finding_lineage_yara_old", "lineage_multiple_yara_matches", {{"parent_pid", "50"}}, {"process_50_bootA_7", "process_51_bootA_9"}), finding("finding_lineage_yara_reused", "lineage_multiple_yara_matches", {{"parent_pid", "50"}}, {"process_50_bootB_3"}), // a process finding is never superseded finding("finding_process_yara_x", "process_executable_yara_match", {{"pid", "1"}}), }; const auto pairs = CorrelationEngine::superseded_findings(fs); std::set old; for (const auto& [o, n] : pairs) { old.insert(o); CHECK(o != n); } CHECK(old == (std::set{"finding_hash_yara_old1", "finding_hash_yara_old2", "finding_rule_yara_old", "finding_lineage_yara_old"})); std::cout << "D2 superseded findings: PASSED" << std::endl; } void touch_old(const fs::path& p, int days) { std::ofstream(p) << "x"; struct timespec times[2]; times[0].tv_sec = times[1].tv_sec = ::time(nullptr) - days * 86400; times[0].tv_nsec = times[1].tv_nsec = 0; CHECK(::utimensat(AT_FDCWD, p.c_str(), times, 0) == 0); } void test_store() { const auto dir = fs::temp_directory_path() / ("strawberry_doctor_" + std::to_string(::getpid())); std::error_code ec; fs::remove_all(dir, ec); { Database db; CHECK(db.open(dir.string())); FindingRecord f; f.id = "finding_hash_yara_old1"; f.title = "t"; CHECK(db.store_finding(f)); f.id = "keep"; CHECK(db.store_finding(f)); CHECK(db.retire_finding("finding_hash_yara_old1", "new", "superseded")); CHECK(!db.retire_finding("absent", "new", "superseded")); CHECK(db.list_findings().size() == 1); db.close(); } { Database db; // D3: replayed from the stream CHECK(db.open_read_only(dir.string())); const auto list = db.list_findings(); CHECK(list.size() == 1 && list[0].id == "keep"); db.close(); } std::cout << "D3 retire_finding: PASSED" << std::endl; fs::create_directories(dir / "orphan"); touch_old(dir / "orphan" / "telemetry-seg-1.log.123", 30); touch_old(dir / "orphan" / "evidence-seg-2.log.456", 1); touch_old(dir / "telemetry" / "seg-9.log.compact", 0); std::size_t listed_before = 0; for (const auto& e : fs::directory_iterator(dir / "evidence")) listed_before += e.path().extension() == ".log"; auto files = store::inspect_store_files(dir.string()); CHECK(files.orphan_files == 2 && files.temp_files == 1); const auto dry = store::clean_store_leftovers(dir.string(), 7 * 86'400'000'000'000LL, true); CHECK(dry.files == 2 && fs::exists(dir / "orphan" / "telemetry-seg-1.log.123")); // reported, not removed const auto done = store::clean_store_leftovers(dir.string(), 7 * 86'400'000'000'000LL, false); CHECK(done.files == 2 && done.problems.empty()); CHECK(!fs::exists(dir / "orphan" / "telemetry-seg-1.log.123")); CHECK(fs::exists(dir / "orphan" / "evidence-seg-2.log.456")); // too recent CHECK(!fs::exists(dir / "telemetry" / "seg-9.log.compact")); std::size_t listed_after = 0; for (const auto& e : fs::directory_iterator(dir / "evidence")) listed_after += e.path().extension() == ".log"; CHECK(listed_after == listed_before && listed_before > 0); CHECK(store::verify_store(dir.string()).problems.empty()); fs::remove_all(dir, ec); std::cout << "D4 clean_store_leftovers: PASSED" << std::endl; } } // namespace void test_memory_throttle() { DoctorInputs in; auto checks = run_doctor_checks(in); for (const auto& c : checks) CHECK(c.id != "resources.memory"); // unknown (-1): no check in.memory_full_pct = 2; in.memory_current_bytes = 500ull << 20; in.memory_high_bytes = 768ull << 20; CHECK(find(run_doctor_checks(in), "resources.memory").status == CheckStatus::ok); in.memory_full_pct = 72.35; in.memory_current_bytes = 822ull << 20; const auto bad = find(run_doctor_checks(in), "resources.memory"); CHECK(bad.status == CheckStatus::problem); CHECK(bad.summary.find("throttled") != std::string::npos && bad.summary.find("72%") != std::string::npos); CHECK(bad.advice.find("MemoryHigh") != std::string::npos); } int main() { test_checks(); test_memory_throttle(); test_superseded(); test_store(); return 0; }