strawberry/tests/test_resource_governor.cpp
trvon 728b2462e7
Some checks failed
Build and Test / build (debug, clang) (push) Has been cancelled
Build and Test / build (debug, gcc) (push) Has been cancelled
Build and Test / build (release, clang) (push) Has been cancelled
Build and Test / build (release, gcc) (push) Has been cancelled
Build and Test / macos (push) Has been cancelled
Build and Test / docker (push) Has been cancelled
Governor: background work no longer starves under sustained pressure
On a host that sits at critical/emergency for a long time (its own apps holding
most of RAM, or the daemon slightly over its soft budget), the starvation guard
was disabled (`!hard`), so correlation, retro-hunt and inventory deferred
forever — observed deferred 153 min with forced=0, which also starves the
retro-hunt delayed labels that Stage 4 training needs.

- Tiered starvation guard: it now fires under critical/emergency too, but after a
  level-scaled window (1x warning, 4x critical, 12x emergency) so background work
  is rare under load, not zero. A true host-memory crunch (host_severe_,
  MemAvailable below the severe floor) still holds everything, since forcing work
  then could tip the host toward OOM. exec guard/scan paths unchanged.
- Raise the balanced own-memory soft budget 512 -> 768 MiB: the daemon's steady
  footprint grew (telemetry buffers, the ingest server, the shadow model), so 512
  left it permanently over budget -> pinned critical/emergency. 768 fits the real
  baseline, still well under the 1.5 GiB cgroup cap.

Tests updated for the 768 budget and the tiered guard (critical forces after the
4x window; severe holds it). Suite 51/51; analyzer + clang-tidy clean.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-10-28 22:33:25 -06:00

363 lines
15 KiB
C++

#include "strawberry/resource_governor.hpp"
#include <cassert>
#include <chrono>
#include <cmath>
#include <iostream>
#include <vector>
using namespace strawberry;
using namespace std::chrono_literals;
namespace {
bool near(double a, double b) { return std::fabs(a - b) < 1e-6; }
void test_parsers() {
std::cout << "Test parsers:" << std::endl;
assert(near(*parse_psi_some_avg10("some avg10=12.34 avg60=1.00 avg300=0.50 total=123\n"
"full avg10=3.00 avg60=0.00 avg300=0.00 total=0\n"), 12.34));
assert(!parse_psi_some_avg10("full avg10=3.00\n"));
assert(!parse_psi_some_avg10("some avg10=abc avg60=0\n"));
assert(!parse_psi_some_avg10(""));
assert(near(*parse_loadavg_1m("2.50 1.00 0.50 3/900 12345\n"), 2.5));
assert(!parse_loadavg_1m("garbage"));
const auto mem = parse_meminfo("MemTotal: 16000000 kB\nMemFree: 100 kB\nMemAvailable: 8000000 kB\n");
assert(mem && mem->total_bytes == 16000000ull * 1024 && mem->available_bytes == 8000000ull * 1024);
assert(!parse_meminfo("MemTotal: 100 kB\n")); // no MemAvailable
const auto stat = parse_proc_stat("cpu 100 0 50 1000 10 0 5 0 0 0\ncpu0 50 0 25 500 5 0 2 0 0 0\n"
"cpu1 50 0 25 500 5 0 3 0 0 0\nintr 1\n");
assert(stat && stat->total_jiffies == 1165 && stat->online_cpus == 2);
assert(!parse_proc_stat("intr 1\n"));
// comm with spaces and ')' must not shift fields; utime=11 stime=7.
assert(*parse_self_stat_cpu("1234 (we ird) name)) S 1 1 1 0 -1 4194560 100 0 0 0 11 7 0 0 20 0 1 0 5 0 0") == 18);
assert(!parse_self_stat_cpu("no paren here"));
assert(*parse_status_rss("Name:\tstrawberry\nVmRSS:\t 204800 kB\n") == 204800ull * 1024);
assert(*parse_cgroup_v2_path("0::/system.slice/strawberry.service\n") == "/system.slice/strawberry.service");
assert(!parse_cgroup_v2_path("1:name=systemd:/x\n"));
assert(!parse_cgroup_v2_path("0::/../etc\n"));
assert(!parse_cgroup_memory_max("max\n"));
assert(*parse_cgroup_memory_max("1610612736\n") == 1610612736ull);
assert(!parse_cgroup_cpu_max("max 100000\n"));
assert(near(*parse_cgroup_cpu_max("200000 100000\n"), 2.0));
std::cout << " PASSED" << std::endl;
}
void test_profiles_are_ordered() {
std::cout << "Test profiles are ordered:" << std::endl;
const auto e = *scan_profile_named("efficient");
const auto b = *scan_profile_named("balanced");
const auto a = *scan_profile_named("aggressive");
assert(!scan_profile_named("turbo"));
// Efficient <= Balanced <= Aggressive on every "how hard may we work" knob.
assert(e.pipeline_workers <= b.pipeline_workers && b.pipeline_workers <= a.pipeline_workers);
assert(e.guard_workers <= b.guard_workers && b.guard_workers <= a.guard_workers);
assert(e.pipeline_max_image_bytes <= b.pipeline_max_image_bytes && b.pipeline_max_image_bytes <= a.pipeline_max_image_bytes);
assert(e.guard_full_scan_max_bytes <= b.guard_full_scan_max_bytes && b.guard_full_scan_max_bytes <= a.guard_full_scan_max_bytes);
assert(e.yara_timeout_s <= b.yara_timeout_s && b.yara_timeout_s <= a.yara_timeout_s);
assert(e.memory_queue_max <= b.memory_queue_max && b.memory_queue_max <= a.memory_queue_max);
assert(e.own_cpu_high_pct <= b.own_cpu_high_pct && b.own_cpu_high_pct <= a.own_cpu_high_pct);
assert(e.host_cpu_some_pct <= b.host_cpu_some_pct && b.host_cpu_some_pct <= a.host_cpu_some_pct);
assert(e.memory_budget_bytes <= b.memory_budget_bytes && b.memory_budget_bytes <= a.memory_budget_bytes);
assert(e.inventory_interval >= b.inventory_interval && b.inventory_interval >= a.inventory_interval);
assert(e.background_nice >= b.background_nice && b.background_nice >= a.background_nice);
// Read rate: 0 means unlimited (aggressive).
assert(e.background_read_bytes_per_s < b.background_read_bytes_per_s && a.background_read_bytes_per_s == 0);
std::cout << " PASSED" << std::endl;
}
struct Fake {
ResourceSample sample;
std::chrono::steady_clock::time_point now = std::chrono::steady_clock::time_point{} + 1h;
std::vector<std::chrono::milliseconds> sleeps;
};
GovernorOptions options_for(Fake& fake, const char* profile = "balanced") {
GovernorOptions o;
o.profile = *scan_profile_named(profile);
o.max_deferral = 300s;
o.sampler = [&fake]() { return fake.sample; };
o.clock = [&fake]() { return fake.now; };
o.sleeper = [&fake](std::chrono::milliseconds d) { fake.sleeps.push_back(d); };
return o;
}
ResourceSample calm() {
ResourceSample s;
s.host_source = "psi";
s.cpu_some_pct = 1;
s.io_some_pct = 0;
s.memory_some_pct = 0;
s.own_cpu_pct = 1;
s.rss_bytes = 50ull << 20;
s.mem_total_bytes = 16ull << 30;
s.mem_available_bytes = 8ull << 30;
return s;
}
void test_level_hysteresis() {
std::cout << "Test level hysteresis:" << std::endl;
Fake fake;
fake.sample = calm();
ResourceGovernor gov(options_for(fake));
gov.tick();
assert(gov.level() == PressureLevel::normal);
// Own CPU above balanced cpu_high (25%): warning, but only after 500 ms.
fake.sample.own_cpu_pct = 30;
gov.tick();
assert(gov.level() == PressureLevel::normal);
fake.now += 600ms;
gov.tick();
assert(gov.level() == PressureLevel::warning);
// Memory over 88.5% of the 768 MiB budget: critical.
fake.sample.rss_bytes = 705ull << 20;
gov.tick();
fake.now += 600ms;
gov.tick();
assert(gov.level() == PressureLevel::critical);
// Memory released: holds 1000 ms while memory is still above warning...
fake.sample.rss_bytes = 600ull << 20; // 78%: still >= warning threshold
fake.sample.own_cpu_pct = 1;
gov.tick();
fake.now += 600ms;
gov.tick();
assert(gov.level() == PressureLevel::critical);
fake.now += 500ms;
gov.tick();
assert(gov.level() == PressureLevel::warning);
// ...and CPU-only pressure clears after 150 ms.
fake.sample.rss_bytes = 50ull << 20;
gov.tick();
fake.now += 200ms;
gov.tick();
assert(gov.level() == PressureLevel::normal);
// A cgroup memory.max below the profile budget becomes the budget.
fake.sample.cgroup_memory_max = 100ull << 20;
fake.sample.rss_bytes = 96ull << 20;
gov.tick();
fake.now += 600ms;
gov.tick();
assert(gov.level() == PressureLevel::emergency);
assert(gov.snapshot().memory_budget_bytes == 100ull << 20);
// Live exec scans are held only briefly per emergency episode, then run
// (forced): memory that stays must not stop scanning for good.
const auto exec = static_cast<std::size_t>(WorkClass::exec_scan);
assert(gov.admit(WorkClass::exec_scan) == Admission::defer);
fake.now += 3s;
assert(gov.admit(WorkClass::exec_scan) == Admission::defer);
fake.now += 3s;
assert(gov.admit(WorkClass::exec_scan) == Admission::admit);
assert(gov.admit(WorkClass::exec_scan) == Admission::admit);
assert(gov.snapshot().forced[exec] == 2);
assert(gov.admit(WorkClass::inventory) == Admission::defer); // background still waits
std::cout << " PASSED" << std::endl;
}
void test_host_release_band_and_deferral() {
std::cout << "Test host release band and deferral:" << std::endl;
Fake fake;
fake.sample = calm();
ResourceGovernor gov(options_for(fake));
gov.tick();
assert(!gov.host_busy());
assert(gov.admit(WorkClass::inventory) == Admission::admit);
// PSI cpu some 45% >= 40%: busy. Exec checks are never deferred.
fake.sample.cpu_some_pct = 45;
gov.tick();
assert(gov.host_busy());
assert(gov.level() == PressureLevel::normal); // host load never raises our level
assert(gov.admit(WorkClass::exec_guard) == Admission::admit);
assert(gov.admit(WorkClass::exec_scan) == Admission::admit);
assert(gov.admit(WorkClass::inventory) == Admission::defer);
assert(gov.admit(WorkClass::memory_scan) == Admission::defer);
// Release band: 35% is below 40 but above 40*0.75=30, so still busy.
fake.sample.cpu_some_pct = 35;
gov.tick();
assert(gov.host_busy());
fake.sample.cpu_some_pct = 25;
gov.tick();
assert(!gov.host_busy());
assert(gov.admit(WorkClass::inventory) == Admission::admit);
// Starvation guard: busy for longer than max_deferral -> one forced pass.
fake.sample.cpu_some_pct = 60;
gov.tick();
assert(gov.admit(WorkClass::correlation) == Admission::defer);
fake.now += 299s;
assert(gov.admit(WorkClass::correlation) == Admission::defer);
fake.now += 2s;
assert(gov.admit(WorkClass::correlation) == Admission::admit);
assert(gov.admit(WorkClass::correlation) == Admission::defer); // window restarts
const auto snap = gov.snapshot();
assert(snap.forced[static_cast<std::size_t>(WorkClass::correlation)] == 1);
assert(snap.deferring[static_cast<std::size_t>(WorkClass::correlation)]);
// Under our own critical memory pressure the guard still fires, but only
// after a longer (4x) window, so background work is rare but not starved.
fake.sample.rss_bytes = 705ull << 20; // 91.8% of 768 MiB budget: critical
gov.tick();
fake.now += 600ms;
gov.tick();
assert(gov.level() == PressureLevel::critical);
assert(gov.admit(WorkClass::correlation) == Admission::defer); // window starts
fake.now += 1100s; // < 4x max_deferral (1200s)
assert(gov.admit(WorkClass::correlation) == Admission::defer);
fake.now += 200s; // now past the 4x window
assert(gov.admit(WorkClass::correlation) == Admission::admit); // forced pass
// A true host-memory crunch (severe) holds even the scaled guard.
Fake severe;
severe.sample = calm();
severe.sample.mem_available_bytes = severe.sample.mem_total_bytes / 50; // 2%: host_severe
ResourceGovernor gov_sev(options_for(severe));
gov_sev.tick();
assert(gov_sev.snapshot().host_severe);
assert(gov_sev.admit(WorkClass::correlation) == Admission::defer);
gov_sev.tick(); // advance would not help: severe holds the guard
severe.now += 10h;
assert(gov_sev.admit(WorkClass::correlation) == Admission::defer);
// Loadavg fallback when PSI is unavailable.
Fake load;
load.sample = calm();
load.sample.host_source = "loadavg";
load.sample.load_per_cpu = 2.0; // balanced threshold 1.5
ResourceGovernor gov2(options_for(load));
gov2.tick();
assert(gov2.host_busy());
// Very low MemAvailable is severe regardless of PSI.
Fake lowmem;
lowmem.sample = calm();
lowmem.sample.mem_available_bytes = lowmem.sample.mem_total_bytes / 50; // 2%
ResourceGovernor gov3(options_for(lowmem));
gov3.tick();
assert(gov3.host_busy() && gov3.snapshot().host_severe);
std::cout << " PASSED" << std::endl;
}
void test_guard_degrade() {
std::cout << "Test exec guard degrade:" << std::endl;
Fake fake;
fake.sample = calm();
ResourceGovernor balanced(options_for(fake, "balanced"));
balanced.tick();
assert(!balanced.guard_should_degrade(1 << 20));
assert(balanced.guard_should_degrade(64ull << 20)); // > 32 MiB: hash/IOC only
fake.sample.cpu_some_pct = 50; // busy, not severe (needs 80)
balanced.tick();
assert(!balanced.guard_should_degrade(1 << 20));
fake.sample.cpu_some_pct = 90; // severe
balanced.tick();
assert(balanced.guard_should_degrade(1 << 20));
Fake e;
e.sample = calm();
e.sample.cpu_some_pct = 25; // efficient busy threshold is 20
ResourceGovernor efficient(options_for(e, "efficient"));
efficient.tick();
assert(efficient.guard_should_degrade(1 << 20));
Fake a;
a.sample = calm();
a.sample.cpu_some_pct = 99;
ResourceGovernor aggressive(options_for(a, "aggressive"));
aggressive.tick();
assert(!aggressive.guard_should_degrade(1 << 20)); // never degrades on load
std::cout << " PASSED" << std::endl;
}
void test_read_pacing() {
std::cout << "Test read pacing:" << std::endl;
Fake fake;
fake.sample = calm();
auto options = options_for(fake);
options.background_read_bytes_per_s = 10ull << 20; // 10 MiB/s, 10 MiB burst
ResourceGovernor gov(options);
gov.tick();
fake.now += 10s; // bucket full
gov.pace_read(4ull << 20);
assert(fake.sleeps.empty());
gov.pace_read(26ull << 20); // 30 MiB total vs 10 MiB burst: ~2 s of waiting
std::chrono::milliseconds total{0};
for (const auto d : fake.sleeps) {
total += d;
assert(d <= 200ms); // sliced so shutdown is never held up
}
assert(total >= 1900ms && total <= 2100ms);
Fake unlimited;
unlimited.sample = calm();
ResourceGovernor fast(options_for(unlimited, "aggressive"));
fast.pace_read(1ull << 30);
assert(unlimited.sleeps.empty());
std::cout << " PASSED" << std::endl;
}
void test_wait_admitted_stops() {
std::cout << "Test wait_admitted honors stop and timeout:" << std::endl;
Fake fake;
fake.sample = calm();
fake.sample.cpu_some_pct = 90;
auto options = options_for(fake);
options.tick = 20ms;
ResourceGovernor gov(options);
gov.tick();
const auto start = std::chrono::steady_clock::now();
assert(!gov.wait_admitted(WorkClass::inventory, 100ms));
assert(std::chrono::steady_clock::now() - start < 2s);
int calls = 0;
assert(!gov.wait_admitted(WorkClass::inventory, 10s, [&]() { return ++calls > 2; }));
assert(gov.wait_admitted(WorkClass::exec_scan, 10ms));
std::cout << " PASSED" << std::endl;
}
void test_system_sampler_reads_this_host() {
std::cout << "Test system sampler on this host:" << std::endl;
SystemSampler sampler;
sampler.sample();
const auto s = sampler.sample();
#if defined(__linux__)
assert(s.rss_bytes > 0);
assert(s.mem_total_bytes > 0);
assert(s.cpu_capacity >= 0.1);
assert(s.host_source == "psi" || s.host_source == "loadavg");
#elif defined(__APPLE__)
assert(s.rss_bytes > 0);
assert(s.mem_total_bytes > 0);
assert(s.mem_available_bytes > 0 && s.mem_available_bytes <= s.mem_total_bytes);
assert(s.cpu_capacity >= 1);
assert(s.host_source == "loadavg" && s.load_per_cpu >= 0);
assert(s.own_cpu_pct >= 0 && s.own_cpu_pct <= 100); // second sample has a delta
#endif
(void) s;
std::cout << " PASSED" << std::endl;
}
} // namespace
int main() {
std::cout << "Running ResourceGovernor tests..." << std::endl << std::endl;
test_parsers();
test_profiles_are_ordered();
test_level_hysteresis();
test_host_release_band_and_deferral();
test_guard_degrade();
test_read_pacing();
test_wait_admitted_stops();
test_system_sampler_reads_this_host();
std::cout << std::endl << "All tests passed!" << std::endl;
return 0;
}