#include "strawberry/resource_governor.hpp" #include #include #include #include #include #include #include #include #include #include using namespace strawberry; using namespace std::chrono_literals; namespace { bool near(double a, double b) { return std::fabs(a - b) < 1e-6; } void test_parsers() { std::cout << "Test parsers:" << std::endl; assert(near(*parse_psi_some_avg10("some avg10=12.34 avg60=1.00 avg300=0.50 total=123\n" "full avg10=3.00 avg60=0.00 avg300=0.00 total=0\n"), 12.34)); assert(!parse_psi_some_avg10("full avg10=3.00\n")); assert(!parse_psi_some_avg10("some avg10=abc avg60=0\n")); assert(!parse_psi_some_avg10("")); // memory.pressure "full" line and memory.events "high" counter (throttle detection). assert(near(*parse_psi_full_avg10("some avg10=81.00 avg60=1.00 avg300=0.50 total=1\n" "full avg10=73.83 avg60=0.00 avg300=0.00 total=0\n"), 73.83)); assert(!parse_psi_full_avg10("some avg10=3.00 avg60=0 avg300=0 total=0\n")); assert(*parse_cgroup_memory_events_high("low 0\nhigh 4211\nmax 3\noom 0\noom_kill 0\n") == 4211); assert(!parse_cgroup_memory_events_high("low 0\nmax 3\n")); assert(near(*parse_loadavg_1m("2.50 1.00 0.50 3/900 12345\n"), 2.5)); assert(!parse_loadavg_1m("garbage")); const auto mem = parse_meminfo("MemTotal: 16000000 kB\nMemFree: 100 kB\nMemAvailable: 8000000 kB\n"); assert(mem && mem->total_bytes == 16000000ull * 1024 && mem->available_bytes == 8000000ull * 1024); assert(!parse_meminfo("MemTotal: 100 kB\n")); // no MemAvailable const auto stat = parse_proc_stat("cpu 100 0 50 1000 10 0 5 0 0 0\ncpu0 50 0 25 500 5 0 2 0 0 0\n" "cpu1 50 0 25 500 5 0 3 0 0 0\nintr 1\n"); assert(stat && stat->total_jiffies == 1165 && stat->online_cpus == 2); assert(!parse_proc_stat("intr 1\n")); const auto disks = parse_proc_diskstats("259 0 nvme0n1 1 0 2 0 3 0 4 0 0 54 60 0 0 0 0\n" "259 1 nvme0n1p1 1 0 2 0 3 0 4 0 0 7 8 0 0 0 0\n"); assert(disks && disks->size() == 2); assert((*disks)[0].name == "nvme0n1" && (*disks)[0].io_time_ms == 54); assert((*disks)[1].name == "nvme0n1p1" && (*disks)[1].io_time_ms == 7); assert(!parse_proc_diskstats("not diskstats\n")); assert(!parse_proc_diskstats("")); // comm with spaces and ')' must not shift fields; utime=11 stime=7. assert(*parse_self_stat_cpu("1234 (we ird) name)) S 1 1 1 0 -1 4194560 100 0 0 0 11 7 0 0 20 0 1 0 5 0 0") == 18); assert(!parse_self_stat_cpu("no paren here")); assert(*parse_status_rss("Name:\tstrawberry\nVmRSS:\t 204800 kB\n") == 204800ull * 1024); assert(*parse_cgroup_v2_path("0::/system.slice/strawberry.service\n") == "/system.slice/strawberry.service"); assert(!parse_cgroup_v2_path("1:name=systemd:/x\n")); assert(!parse_cgroup_v2_path("0::/../etc\n")); assert(!parse_cgroup_memory_max("max\n")); assert(*parse_cgroup_memory_max("1610612736\n") == 1610612736ull); assert(!parse_cgroup_cpu_max("max 100000\n")); assert(near(*parse_cgroup_cpu_max("200000 100000\n"), 2.0)); std::cout << " PASSED" << std::endl; } void test_profiles_are_ordered() { std::cout << "Test profiles are ordered:" << std::endl; const auto e = *scan_profile_named("efficient"); const auto b = *scan_profile_named("balanced"); const auto a = *scan_profile_named("aggressive"); assert(!scan_profile_named("turbo")); // Efficient <= Balanced <= Aggressive on every "how hard may we work" knob. assert(e.pipeline_workers <= b.pipeline_workers && b.pipeline_workers <= a.pipeline_workers); assert(e.guard_workers <= b.guard_workers && b.guard_workers <= a.guard_workers); assert(e.pipeline_max_image_bytes <= b.pipeline_max_image_bytes && b.pipeline_max_image_bytes <= a.pipeline_max_image_bytes); assert(e.guard_full_scan_max_bytes <= b.guard_full_scan_max_bytes && b.guard_full_scan_max_bytes <= a.guard_full_scan_max_bytes); assert(e.yara_timeout_s <= b.yara_timeout_s && b.yara_timeout_s <= a.yara_timeout_s); assert(e.memory_queue_max <= b.memory_queue_max && b.memory_queue_max <= a.memory_queue_max); assert(e.own_cpu_high_pct <= b.own_cpu_high_pct && b.own_cpu_high_pct <= a.own_cpu_high_pct); assert(e.host_cpu_some_pct <= b.host_cpu_some_pct && b.host_cpu_some_pct <= a.host_cpu_some_pct); assert(e.memory_budget_bytes <= b.memory_budget_bytes && b.memory_budget_bytes <= a.memory_budget_bytes); assert(e.inventory_interval >= b.inventory_interval && b.inventory_interval >= a.inventory_interval); assert(e.background_nice >= b.background_nice && b.background_nice >= a.background_nice); // Read rate: 0 means unlimited (aggressive). assert(e.background_read_bytes_per_s < b.background_read_bytes_per_s && a.background_read_bytes_per_s == 0); std::cout << " PASSED" << std::endl; } struct Fake { ResourceSample sample; std::chrono::steady_clock::time_point now = std::chrono::steady_clock::time_point{} + 1h; std::vector sleeps; }; GovernorOptions options_for(Fake& fake, const char* profile = "balanced") { GovernorOptions o; o.profile = *scan_profile_named(profile); o.max_deferral = 300s; o.sampler = [&fake]() { return fake.sample; }; o.clock = [&fake]() { return fake.now; }; o.sleeper = [&fake](std::chrono::milliseconds d) { fake.sleeps.push_back(d); }; return o; } ResourceSample calm() { ResourceSample s; s.host_source = "psi"; s.cpu_some_pct = 1; s.io_some_pct = 0; s.memory_some_pct = 0; s.own_cpu_pct = 1; s.rss_bytes = 50ull << 20; s.mem_total_bytes = 16ull << 30; s.mem_available_bytes = 8ull << 30; return s; } void test_level_hysteresis() { std::cout << "Test level hysteresis:" << std::endl; Fake fake; fake.sample = calm(); ResourceGovernor gov(options_for(fake)); gov.tick(); assert(gov.level() == PressureLevel::normal); // Own CPU above balanced cpu_high (25%): warning, but only after 500 ms. fake.sample.own_cpu_pct = 30; gov.tick(); assert(gov.level() == PressureLevel::normal); fake.now += 600ms; gov.tick(); assert(gov.level() == PressureLevel::warning); // Memory over 88.5% of the 768 MiB budget: critical. fake.sample.rss_bytes = 705ull << 20; gov.tick(); fake.now += 600ms; gov.tick(); assert(gov.level() == PressureLevel::critical); // Memory released: holds 1000 ms while memory is still above warning... fake.sample.rss_bytes = 600ull << 20; // 78%: still >= warning threshold fake.sample.own_cpu_pct = 1; gov.tick(); fake.now += 600ms; gov.tick(); assert(gov.level() == PressureLevel::critical); fake.now += 500ms; gov.tick(); assert(gov.level() == PressureLevel::warning); // Memory released: the episode keeps the 1000 ms memory hold even once RSS // is already below warning, because it was memory that raised it (picking // the hold from the new RSS made warning -> calm release in 150 ms). fake.sample.rss_bytes = 50ull << 20; gov.tick(); fake.now += 200ms; gov.tick(); assert(gov.level() == PressureLevel::warning); fake.now += 799ms; // 999 ms since the release was proposed gov.tick(); assert(gov.level() == PressureLevel::warning); fake.now += 1ms; // 1000 ms: the memory hold has elapsed gov.tick(); assert(gov.level() == PressureLevel::normal); // A CPU-driven episode still releases quickly (150 ms), not on the memory hold. fake.sample.own_cpu_pct = 30; gov.tick(); fake.now += 600ms; gov.tick(); assert(gov.level() == PressureLevel::warning); fake.sample.own_cpu_pct = 1; gov.tick(); fake.now += 200ms; gov.tick(); assert(gov.level() == PressureLevel::normal); // A cgroup memory.max below the profile budget becomes the budget. fake.sample.cgroup_memory_max = 100ull << 20; fake.sample.rss_bytes = 96ull << 20; gov.tick(); fake.now += 600ms; gov.tick(); assert(gov.level() == PressureLevel::emergency); assert(gov.snapshot().memory_budget_bytes == 100ull << 20); // Live exec scans are held only briefly per emergency episode, then run // (forced): memory that stays must not stop scanning for good. const auto exec = static_cast(WorkClass::exec_scan); assert(gov.admit(WorkClass::exec_scan) == Admission::defer); fake.now += 3s; assert(gov.admit(WorkClass::exec_scan) == Admission::defer); fake.now += 3s; assert(gov.admit(WorkClass::exec_scan) == Admission::admit); assert(gov.admit(WorkClass::exec_scan) == Admission::admit); assert(gov.snapshot().forced[exec] == 2); assert(gov.admit(WorkClass::inventory) == Admission::defer); // background still waits std::cout << " PASSED" << std::endl; } void test_host_release_band_and_deferral() { std::cout << "Test host release band and deferral:" << std::endl; Fake fake; fake.sample = calm(); ResourceGovernor gov(options_for(fake)); gov.tick(); assert(!gov.host_busy()); assert(gov.admit(WorkClass::inventory) == Admission::admit); // PSI cpu some 45% >= 40%: busy. Exec checks are never deferred. fake.sample.cpu_some_pct = 45; gov.tick(); assert(gov.host_busy()); assert(gov.level() == PressureLevel::normal); // host load never raises our level assert(gov.admit(WorkClass::exec_guard) == Admission::admit); assert(gov.admit(WorkClass::exec_scan) == Admission::admit); assert(gov.admit(WorkClass::inventory) == Admission::defer); assert(gov.admit(WorkClass::memory_scan) == Admission::defer); // Release band: 35% is below 40 but above 40*0.75=30, so still busy. fake.sample.cpu_some_pct = 35; gov.tick(); assert(gov.host_busy()); fake.sample.cpu_some_pct = 25; gov.tick(); assert(!gov.host_busy()); assert(gov.admit(WorkClass::inventory) == Admission::admit); // Starvation guard: busy for longer than max_deferral -> one forced pass. fake.sample.cpu_some_pct = 60; gov.tick(); assert(gov.admit(WorkClass::correlation) == Admission::defer); fake.now += 299s; assert(gov.admit(WorkClass::correlation) == Admission::defer); fake.now += 2s; assert(gov.admit(WorkClass::correlation) == Admission::admit); assert(gov.admit(WorkClass::correlation) == Admission::defer); // window restarts const auto snap = gov.snapshot(); assert(snap.forced[static_cast(WorkClass::correlation)] == 1); assert(snap.deferring[static_cast(WorkClass::correlation)]); // Under our own critical memory pressure the guard still fires, but only // after a longer (4x) window, so background work is rare but not starved. fake.sample.rss_bytes = 705ull << 20; // 91.8% of 768 MiB budget: critical gov.tick(); fake.now += 600ms; gov.tick(); assert(gov.level() == PressureLevel::critical); assert(gov.admit(WorkClass::correlation) == Admission::defer); // window starts fake.now += 1100s; // < 4x max_deferral (1200s) assert(gov.admit(WorkClass::correlation) == Admission::defer); fake.now += 200s; // now past the 4x window assert(gov.admit(WorkClass::correlation) == Admission::admit); // forced pass // A true host-memory crunch (severe) holds even the scaled guard. Fake severe; severe.sample = calm(); severe.sample.mem_available_bytes = severe.sample.mem_total_bytes / 50; // 2%: host_severe ResourceGovernor gov_sev(options_for(severe)); gov_sev.tick(); assert(gov_sev.snapshot().host_severe); assert(gov_sev.admit(WorkClass::correlation) == Admission::defer); gov_sev.tick(); // advance would not help: severe holds the guard severe.now += 10h; assert(gov_sev.admit(WorkClass::correlation) == Admission::defer); // Loadavg fallback when PSI is unavailable. Fake load; load.sample = calm(); load.sample.host_source = "loadavg"; load.sample.load_per_cpu = 2.0; // balanced threshold 1.5 ResourceGovernor gov2(options_for(load)); gov2.tick(); assert(gov2.host_busy()); // Very low MemAvailable is severe regardless of PSI. Fake lowmem; lowmem.sample = calm(); lowmem.sample.mem_available_bytes = lowmem.sample.mem_total_bytes / 50; // 2% ResourceGovernor gov3(options_for(lowmem)); gov3.tick(); assert(gov3.host_busy() && gov3.snapshot().host_severe); std::cout << " PASSED" << std::endl; } void test_io_pressure_is_fail_safe() { std::cout << "Test host I/O PSI stays fail-safe (diskstats is diagnostic):" << std::endl; Fake fake; fake.sample = calm(); fake.sample.io_some_pct = 95; ResourceGovernor gov(options_for(fake)); // Any diskstat reading, including idle storage, must never clear I/O PSI: // global PSI also counts waits without a block counter (FUSE/NFS-backed, // display page flips). for (const double util : {-1.0, 0.0, 1.8, 4.99, 5.0, 40.0, 100.0}) { fake.sample.block_io_util_pct = util; gov.tick(); assert(gov.host_busy() && gov.snapshot().host_severe); assert(gov.admit(WorkClass::inventory) == Admission::defer); } // Just below the balanced I/O threshold with idle storage: still not busy. fake.sample.block_io_util_pct = 0.0; fake.sample.io_some_pct = 20; gov.tick(); assert(!gov.host_busy() && !gov.snapshot().host_severe); assert(gov.admit(WorkClass::inventory) == Admission::admit); // CPU, PSI-memory and MemAvailable remain independent and effective. fake.sample.cpu_some_pct = 90; gov.tick(); assert(gov.host_busy() && gov.snapshot().host_severe); fake.sample.cpu_some_pct = 1; fake.sample.memory_some_pct = 50; gov.tick(); assert(gov.host_busy() && gov.snapshot().host_severe); fake.sample.memory_some_pct = 0; fake.sample.mem_available_bytes = fake.sample.mem_total_bytes / 50; gov.tick(); assert(gov.host_busy() && gov.snapshot().host_severe); std::cout << " PASSED" << std::endl; } #if defined(__linux__) void test_sampler_diskstats_identity() { std::cout << "Test diskstats sampler identity and new-device handling:" << std::endl; namespace fs = std::filesystem; const auto dir = fs::temp_directory_path() / ("sb-diskstats-" + std::to_string(::getpid())); std::error_code ec; fs::remove_all(dir, ec); fs::create_directories(dir); const auto write = [&](const std::string& text) { std::ofstream out(dir / "diskstats", std::ios::trunc); out << text; }; // First sample has no baseline: unknown, never a false 0%. write("259 0 nvme0n1 1 0 2 0 3 0 4 0 0 1000 60 0 0 0 0\n"); { SystemSampler sampler(dir.string(), dir.string()); assert(sampler.sample().block_io_util_pct < 0.0); // Same devices again: the delta is measurable. assert(sampler.sample().block_io_util_pct >= 0.0); } // A new device with no baseline makes the aggregate unknown, even when // every previously known device is idle (reviewers' P1 regression). { SystemSampler sampler(dir.string(), dir.string()); sampler.sample(); write("259 0 nvme0n1 1 0 2 0 3 0 4 0 0 1000 60 0 0 0 0\n" "253 0 dm-0 1 0 2 0 3 0 4 0 0 5000 60 0 0 0 0\n"); const auto with_new_device = sampler.sample(); assert(with_new_device.block_io_util_pct < 0.0); // Once the baseline exists, the delta is measurable again. write("259 0 nvme0n1 1 0 2 0 3 0 4 0 0 1000 60 0 0 0 0\n" "253 0 dm-0 1 0 2 0 3 0 4 0 0 5000 60 0 0 0 0\n"); assert(sampler.sample().block_io_util_pct >= 0.0); } // A truncation-sized read is unknown rather than a silently short list. { std::string huge; for (int i = 0; huge.size() < 64 * 1024 + 100; ++i) { huge += "259 " + std::to_string(i) + " dev" + std::to_string(i) + " 1 0 2 0 3 0 4 0 0 7 8 0 0 0 0\n"; } write(huge); SystemSampler sampler(dir.string(), dir.string()); sampler.sample(); assert(sampler.sample().block_io_util_pct < 0.0); } fs::remove_all(dir, ec); std::cout << " PASSED" << std::endl; } #endif void test_guard_degrade() { std::cout << "Test exec guard degrade:" << std::endl; Fake fake; fake.sample = calm(); ResourceGovernor balanced(options_for(fake, "balanced")); balanced.tick(); assert(!balanced.guard_should_degrade(1 << 20)); assert(balanced.guard_should_degrade(64ull << 20)); // > 32 MiB: hash/IOC only fake.sample.cpu_some_pct = 50; // busy, not severe (needs 80) balanced.tick(); assert(!balanced.guard_should_degrade(1 << 20)); fake.sample.cpu_some_pct = 90; // severe balanced.tick(); assert(balanced.guard_should_degrade(1 << 20)); Fake e; e.sample = calm(); e.sample.cpu_some_pct = 25; // efficient busy threshold is 20 ResourceGovernor efficient(options_for(e, "efficient")); efficient.tick(); assert(efficient.guard_should_degrade(1 << 20)); Fake a; a.sample = calm(); a.sample.cpu_some_pct = 99; ResourceGovernor aggressive(options_for(a, "aggressive")); aggressive.tick(); assert(!aggressive.guard_should_degrade(1 << 20)); // never degrades on load std::cout << " PASSED" << std::endl; } void test_read_pacing() { std::cout << "Test read pacing:" << std::endl; Fake fake; fake.sample = calm(); auto options = options_for(fake); options.background_read_bytes_per_s = 10ull << 20; // 10 MiB/s, 10 MiB burst ResourceGovernor gov(options); gov.tick(); fake.now += 10s; // bucket full gov.pace_read(4ull << 20); assert(fake.sleeps.empty()); gov.pace_read(26ull << 20); // 30 MiB total vs 10 MiB burst: ~2 s of waiting std::chrono::milliseconds total{0}; for (const auto d : fake.sleeps) { total += d; assert(d <= 200ms); // sliced so shutdown is never held up } assert(total >= 1900ms && total <= 2100ms); Fake unlimited; unlimited.sample = calm(); ResourceGovernor fast(options_for(unlimited, "aggressive")); fast.pace_read(1ull << 30); assert(unlimited.sleeps.empty()); std::cout << " PASSED" << std::endl; } void test_wait_admitted_stops() { std::cout << "Test wait_admitted honors stop and timeout:" << std::endl; Fake fake; fake.sample = calm(); fake.sample.cpu_some_pct = 90; auto options = options_for(fake); options.tick = 20ms; ResourceGovernor gov(options); gov.tick(); const auto start = std::chrono::steady_clock::now(); assert(!gov.wait_admitted(WorkClass::inventory, 100ms)); assert(std::chrono::steady_clock::now() - start < 2s); int calls = 0; assert(!gov.wait_admitted(WorkClass::inventory, 10s, [&]() { return ++calls > 2; })); assert(gov.wait_admitted(WorkClass::exec_scan, 10ms)); std::cout << " PASSED" << std::endl; } void test_system_sampler_reads_this_host() { std::cout << "Test system sampler on this host:" << std::endl; SystemSampler sampler; sampler.sample(); const auto s = sampler.sample(); #if defined(__linux__) assert(s.rss_bytes > 0); assert(s.mem_total_bytes > 0); assert(s.cpu_capacity >= 0.1); assert(s.host_source == "psi" || s.host_source == "loadavg"); #elif defined(__APPLE__) assert(s.rss_bytes > 0); assert(s.mem_total_bytes > 0); assert(s.mem_available_bytes > 0 && s.mem_available_bytes <= s.mem_total_bytes); assert(s.cpu_capacity >= 1); assert(s.host_source == "loadavg" && s.load_per_cpu >= 0); assert(s.own_cpu_pct >= 0 && s.own_cpu_pct <= 100); // second sample has a delta #endif (void) s; std::cout << " PASSED" << std::endl; } } // namespace void test_scan_buffer_budget() { std::cout << "Test scan buffer budget:" << std::endl; Fake fake; fake.sample = calm(); ResourceGovernor gov(options_for(fake)); // balanced: 768 MiB budget -> 192 MiB of scan buffers const uint64_t mib = 1ull << 20; assert(gov.scan_buffer_budget() == 192 * mib); { auto a = gov.acquire_scan_buffer(110 * mib, 0ms); assert(a); auto b = gov.acquire_scan_buffer(110 * mib, 0ms); // would exceed 192: refused, no wait assert(!b); auto c = gov.acquire_scan_buffer(32 * mib, 0ms); // 142 <= 192: fits alongside assert(c); auto snap = gov.snapshot(); assert(snap.scan_buffer_in_flight == 142 * mib && snap.scan_buffer_refused == 1); } assert(gov.snapshot().scan_buffer_in_flight == 0); // leases released on scope exit { auto big = gov.acquire_scan_buffer(500 * mib, 0ms); // larger than the budget, alone: exclusive assert(big); auto other = gov.acquire_scan_buffer(1 * mib, 0ms); assert(!other); } // A waiter is granted when the holder releases (bounded wait, counted). auto holder = gov.acquire_scan_buffer(150 * mib, 0ms); assert(holder); std::thread release([&] { std::this_thread::sleep_for(50ms); holder.release(); }); auto waited = gov.acquire_scan_buffer(150 * mib, 5000ms); release.join(); assert(waited && gov.snapshot().scan_buffer_waits == 1); // A wait that times out is refused, never granted over budget. auto timeout = gov.acquire_scan_buffer(150 * mib, 30ms); assert(!timeout); // Moving a lease transfers it; the moved-from lease releases nothing. ScanBufferLease moved = std::move(waited); assert(moved && !waited); moved.release(); assert(gov.snapshot().scan_buffer_in_flight == 0 && gov.snapshot().scan_buffer_peak >= 150 * mib); std::cout << " PASSED" << std::endl; } void test_scan_buffer_budget_cgroup_capped() { std::cout << "Test scan buffer budget follows the cgroup-capped memory budget:" << std::endl; const uint64_t mib = 1ull << 20; Fake fake; fake.sample = calm(); fake.sample.cgroup_memory_max = 100 * mib; // balanced profile asks for 768 MiB ResourceGovernor gov(options_for(fake)); gov.tick(); // The pressure level and the scan buffers read the same effective budget, so status cannot // report a 100 MiB budget while letting 192 MiB of image copies into memory. assert(gov.snapshot().memory_budget_bytes == 100 * mib); assert(gov.scan_buffer_budget() == 32 * mib); // 100/4 = 25 MiB, raised to the 32 MiB floor assert(gov.snapshot().scan_buffer_budget == 32 * mib); { auto a = gov.acquire_scan_buffer(20 * mib, 0ms); assert(a); auto b = gov.acquire_scan_buffer(20 * mib, 0ms); // 40 > 32: refused, no wait assert(!b); auto c = gov.acquire_scan_buffer(12 * mib, 0ms); // 32 <= 32: fits alongside assert(c); assert(gov.snapshot().scan_buffer_in_flight == 32 * mib); assert(gov.snapshot().scan_buffer_refused == 1); } assert(gov.snapshot().scan_buffer_in_flight == 0); // Before the first tick there is no sample, so only the profile budget applies. Fake unticked; unticked.sample = calm(); ResourceGovernor gov_unticked(options_for(unticked)); assert(gov_unticked.scan_buffer_budget() == 192 * mib); std::cout << " PASSED" << std::endl; } int main() { std::cout << "Running ResourceGovernor tests..." << std::endl << std::endl; test_parsers(); test_profiles_are_ordered(); test_level_hysteresis(); test_host_release_band_and_deferral(); test_io_pressure_is_fail_safe(); #if defined(__linux__) test_sampler_diskstats_identity(); #endif test_guard_degrade(); test_read_pacing(); test_scan_buffer_budget(); test_scan_buffer_budget_cgroup_capped(); test_wait_admitted_stops(); test_system_sampler_reads_this_host(); std::cout << std::endl << "All tests passed!" << std::endl; return 0; }