// CPU Governance Enforcement Tests (Catch2) // Tests OnnxConcurrencyRegistry slot enforcement, TuningManager profile configuration, // and ResourceGovernor admission control. #include #include #include #include #include "../../common/test_helpers_catch2.h" #include #include #include #include #include #include #include "../../common/env_compat.h" #include using namespace yams::daemon; using namespace std::chrono_literals; // ============================================================================= // Test Helpers // ============================================================================= namespace { /// RAII guard for TuneAdvisor profile restoration class ProfileGuard { TuneAdvisor::Profile prev_; public: explicit ProfileGuard(TuneAdvisor::Profile profile) : prev_(TuneAdvisor::tuningProfile()) { TuneAdvisor::setTuningProfile(profile); } ~ProfileGuard() { TuneAdvisor::setTuningProfile(prev_); } ProfileGuard(const ProfileGuard&) = delete; ProfileGuard& operator=(const ProfileGuard&) = delete; }; using yams::test::ScopedEnvVar; /// RAII guard for OnnxConcurrencyRegistry max slots (restore original on destruction) class SlotConfigGuard { uint32_t prevMax_; public: explicit SlotConfigGuard(uint32_t newMax) { auto& reg = OnnxConcurrencyRegistry::instance(); prevMax_ = reg.totalSlots(); reg.setMaxSlots(newMax); } ~SlotConfigGuard() { OnnxConcurrencyRegistry::instance().setMaxSlots(prevMax_); } SlotConfigGuard(const SlotConfigGuard&) = delete; SlotConfigGuard& operator=(const SlotConfigGuard&) = delete; }; } // namespace // ============================================================================= // OnnxConcurrencyRegistry Basic Tests // ============================================================================= TEST_CASE("OnnxConcurrencyRegistry singleton is accessible", "[daemon][governance][catch2]") { auto& reg1 = OnnxConcurrencyRegistry::instance(); auto& reg2 = OnnxConcurrencyRegistry::instance(); CHECK(®1 == ®2); } TEST_CASE("OnnxConcurrencyRegistry slot metrics are observable", "[daemon][governance][catch2]") { auto& reg = OnnxConcurrencyRegistry::instance(); auto snap = reg.snapshot(); CHECK(snap.totalSlots > 0); CHECK(snap.availableSlots <= snap.totalSlots); CHECK(snap.usedSlots <= snap.totalSlots); } TEST_CASE("OnnxConcurrencyRegistry lane metrics are observable", "[daemon][governance][catch2]") { auto& reg = OnnxConcurrencyRegistry::instance(); auto glinerMetrics = reg.laneMetrics(OnnxLane::Gliner); auto embedMetrics = reg.laneMetrics(OnnxLane::Embedding); auto rerankerMetrics = reg.laneMetrics(OnnxLane::Reranker); // Default reserved slots static_cast(glinerMetrics.reserved); static_cast(embedMetrics.reserved); static_cast(rerankerMetrics.reserved); } namespace { /// Captures messages delivered through OnnxConcurrencyRegistry::setLogSink(). struct CapturedRegistryLogs { static inline std::vector> messages; static void sink(OnnxRegistryLogLevel level, const char* message) noexcept { messages.emplace_back(level, message); } }; /// Restores the registry log sink to "none" when a test case ends. struct LogSinkGuard { explicit LogSinkGuard(OnnxRegistryLogSink sink) { OnnxConcurrencyRegistry::setLogSink(sink); } ~LogSinkGuard() { OnnxConcurrencyRegistry::setLogSink(nullptr); } LogSinkGuard(const LogSinkGuard&) = delete; LogSinkGuard& operator=(const LogSinkGuard&) = delete; }; } // namespace TEST_CASE("OnnxConcurrencyRegistry routes diagnostics through the host log sink", "[daemon][governance][catch2]") { // The registry is a private shared library that links no logger; the host // binary installs a sink so registry messages land in the host's log. auto& reg = OnnxConcurrencyRegistry::instance(); SlotConfigGuard configGuard(reg.totalSlots()); CapturedRegistryLogs::messages.clear(); { LogSinkGuard sinkGuard(&CapturedRegistryLogs::sink); reg.setMaxSlots(reg.totalSlots() + 1); } REQUIRE(CapturedRegistryLogs::messages.size() == 1); CHECK(CapturedRegistryLogs::messages[0].first == OnnxRegistryLogLevel::Info); CHECK(CapturedRegistryLogs::messages[0].second.find("Max slots changed") != std::string::npos); // Without a sink the registry stays silent instead of falling back to a // private default logger (which would write to the host's stdout). reg.setMaxSlots(reg.totalSlots() + 1); CHECK(CapturedRegistryLogs::messages.size() == 1); } // ============================================================================= // SlotGuard RAII Tests // ============================================================================= TEST_CASE("SlotGuard acquires and releases slot", "[daemon][governance][catch2]") { auto& reg = OnnxConcurrencyRegistry::instance(); SlotConfigGuard configGuard(8); // Ensure enough slots uint32_t beforeUsed = reg.usedSlots(); { OnnxConcurrencyRegistry::SlotGuard guard(reg, OnnxLane::Embedding, 100ms); CHECK(guard.acquired()); CHECK(reg.usedSlots() == beforeUsed + 1); } CHECK(reg.usedSlots() == beforeUsed); } TEST_CASE("SlotGuard converts to bool", "[daemon][governance][catch2]") { auto& reg = OnnxConcurrencyRegistry::instance(); SlotConfigGuard configGuard(8); OnnxConcurrencyRegistry::SlotGuard guard(reg, OnnxLane::Embedding, 100ms); if (guard) { CHECK(guard.acquired()); } else { FAIL("SlotGuard should have acquired a slot"); } } TEST_CASE("Multiple slots can be acquired up to limit", "[daemon][governance][catch2]") { auto& reg = OnnxConcurrencyRegistry::instance(); SlotConfigGuard configGuard(4); // Small limit for test uint32_t beforeUsed = reg.usedSlots(); // Acquire multiple slots OnnxConcurrencyRegistry::SlotGuard slot1(reg, OnnxLane::Embedding, 100ms); OnnxConcurrencyRegistry::SlotGuard slot2(reg, OnnxLane::Embedding, 100ms); CHECK(slot1.acquired()); CHECK(slot2.acquired()); CHECK(reg.usedSlots() >= beforeUsed + 2); } // ============================================================================= // Slot Enforcement Tests // ============================================================================= TEST_CASE("Slot acquisition respects limits", "[daemon][governance][catch2]") { auto& reg = OnnxConcurrencyRegistry::instance(); // Just verify we can acquire and release slots normally // Full capacity testing is complex due to reserved slots SlotConfigGuard configGuard(8); uint32_t beforeUsed = reg.usedSlots(); { OnnxConcurrencyRegistry::SlotGuard slot1(reg, OnnxLane::Other, 100ms); OnnxConcurrencyRegistry::SlotGuard slot2(reg, OnnxLane::Embedding, 100ms); CHECK(slot1.acquired()); CHECK(slot2.acquired()); CHECK(reg.usedSlots() >= beforeUsed + 2); } // Slots should be released CHECK(reg.usedSlots() == beforeUsed); } // ============================================================================= // ResourceGovernor Admission Control Tests // ============================================================================= TEST_CASE("ResourceGovernor admission control methods exist", "[daemon][governance][catch2]") { auto& governor = ResourceGovernor::instance(); // These methods should be callable bool canWork = governor.canAdmitWork(); bool canLoad = governor.canLoadModel(1024 * 1024); // 1 MB bool canScale = governor.canScaleUp("test", 1); // At normal pressure, all should be true (unless governor is disabled) // Just verify they're callable without crashing (void)canWork; (void)canLoad; (void)canScale; } TEST_CASE("ResourceGovernor pressure level is Normal by default", "[daemon][governance][catch2]") { auto& governor = ResourceGovernor::instance(); // At startup with no memory pressure, level should be Normal auto level = governor.getPressureLevel(); // This may vary based on actual system memory usage, so just check it's valid CHECK((level == ResourcePressureLevel::Normal || level == ResourcePressureLevel::Warning || level == ResourcePressureLevel::Critical || level == ResourcePressureLevel::Emergency)); } TEST_CASE("ResourceGovernor scaling caps are queryable", "[daemon][governance][catch2]") { auto& governor = ResourceGovernor::instance(); uint32_t maxIngest = governor.maxIngestWorkers(); uint32_t maxSearch = governor.maxSearchConcurrency(); uint32_t maxEmbed = governor.maxEmbedConcurrency(); uint32_t maxExtraction = governor.maxExtractionConcurrency(); uint32_t maxKg = governor.maxKgConcurrency(); // All should have some positive value at Normal pressure CHECK(maxIngest > 0); CHECK(maxSearch > 0); // Embed may be 0 at Emergency level (void)maxEmbed; (void)maxExtraction; (void)maxKg; } TEST_CASE("ResourceGovernor Warning caps honor configurable scale percent", "[daemon][governance][catch2]") { auto& governor = ResourceGovernor::instance(); TuneAdvisor::resetGovernorWarningScalePercentOverride(); auto expectedScaled = [](uint32_t current, uint32_t percent) -> uint32_t { if (current == 0) { return 0; } uint32_t scaled = static_cast((static_cast(current) * percent + 99u) / 100u); return std::min(current, std::max(1u, scaled)); }; governor.testing_updateScalingCaps(ResourcePressureLevel::Normal); auto normal = governor.getScalingCaps(); TuneAdvisor::setGovernorWarningScalePercent(90u); governor.testing_updateScalingCaps(ResourcePressureLevel::Warning); auto warning90 = governor.getScalingCaps(); CHECK(warning90.ingestWorkers == expectedScaled(normal.ingestWorkers, 90u)); CHECK(warning90.searchConcurrency == expectedScaled(normal.searchConcurrency, 90u)); CHECK(warning90.extractionConcurrency == expectedScaled(normal.extractionConcurrency, 90u)); CHECK(warning90.kgConcurrency == expectedScaled(normal.kgConcurrency, 90u)); CHECK(warning90.enrichConcurrency == expectedScaled(normal.enrichConcurrency, 90u)); CHECK(warning90.embedConcurrency == expectedScaled(normal.embedConcurrency, 90u)); CHECK_FALSE(warning90.allowModelLoads); CHECK(warning90.allowNewIngest); TuneAdvisor::setGovernorWarningScalePercent(70u); governor.testing_updateScalingCaps(ResourcePressureLevel::Warning); auto warning70 = governor.getScalingCaps(); CHECK(warning70.ingestWorkers == expectedScaled(normal.ingestWorkers, 70u)); CHECK(warning70.searchConcurrency == expectedScaled(normal.searchConcurrency, 70u)); governor.testing_setPressureState(ResourcePressureLevel::Warning, std::chrono::steady_clock::now()); TuneAdvisor::setGovernorWarningScalePercent(80u); governor.refreshScalingCaps(); auto refreshed = governor.getScalingCaps(); CHECK(refreshed.ingestWorkers == expectedScaled(normal.ingestWorkers, 80u)); CHECK(refreshed.searchConcurrency == expectedScaled(normal.searchConcurrency, 80u)); governor.testing_setPressureState(ResourcePressureLevel::Normal, std::chrono::steady_clock::now()); governor.testing_updateScalingCaps(ResourcePressureLevel::Normal); TuneAdvisor::resetGovernorWarningScalePercentOverride(); } TEST_CASE("ResourceGovernor tuning refresh preserves raw Emergency caps during startup grace", "[daemon][governance][catch2][reload]") { auto& governor = ResourceGovernor::instance(); governor.testing_setPressureState(ResourcePressureLevel::Emergency, std::chrono::steady_clock::now()); governor.refreshScalingCaps(); const auto emergency = governor.getScalingCaps(); CHECK_FALSE(emergency.allowNewIngest); CHECK_FALSE(emergency.allowModelLoads); CHECK((emergency.extractionConcurrency == 0)); CHECK((emergency.embedConcurrency == 0)); governor.testing_setPressureState(ResourcePressureLevel::Normal, std::chrono::steady_clock::now()); governor.testing_updateScalingCaps(ResourcePressureLevel::Normal); } TEST_CASE("ResourceGovernor de-escalation hysteresis differentiates CPU and memory pressure", "[daemon][governance][catch2]") { auto& governor = ResourceGovernor::instance(); // Save and restore tunables touched by this test. const double prevWarning = TuneAdvisor::memoryWarningThreshold(); const double prevCritical = TuneAdvisor::memoryCriticalThreshold(); const double prevEmergency = TuneAdvisor::memoryEmergencyThreshold(); const uint32_t prevMemHys = TuneAdvisor::memoryHysteresisMs(); const uint32_t prevCpuHys = TuneAdvisor::cpuLevelHysteresisMs(); TuneAdvisor::setMemoryWarningThreshold(0.70); TuneAdvisor::setMemoryCriticalThreshold(0.90); TuneAdvisor::setMemoryEmergencyThreshold(0.97); TuneAdvisor::setMemoryHysteresisMs(500); TuneAdvisor::setCpuLevelHysteresisMs(100); const auto t0 = std::chrono::steady_clock::now(); SECTION("CPU-driven de-escalation uses cpu-level hysteresis") { governor.testing_setPressureState(ResourcePressureLevel::Warning, t0); ResourceSnapshot snap{}; snap.memoryPressure = 0.30; // no memory pressure snap.cpuUsagePercent = 0.0; snap.embedQueued = 0; snap.timestamp = t0; CHECK(governor.testing_computeLevel(snap) == ResourcePressureLevel::Warning); snap.timestamp = t0 + std::chrono::milliseconds(150); CHECK(governor.testing_computeLevel(snap) == ResourcePressureLevel::Normal); } SECTION("Memory-driven de-escalation keeps conservative 2x hysteresis") { governor.testing_setPressureState(ResourcePressureLevel::Critical, t0); ResourceSnapshot snap{}; snap.memoryPressure = 0.80; // warning-level memory pressure (below critical) snap.cpuUsagePercent = 0.0; snap.embedQueued = 0; snap.timestamp = t0; CHECK(governor.testing_computeLevel(snap) == ResourcePressureLevel::Critical); snap.timestamp = t0 + std::chrono::milliseconds(600); CHECK(governor.testing_computeLevel(snap) == ResourcePressureLevel::Critical); snap.timestamp = t0 + std::chrono::milliseconds(1100); CHECK(governor.testing_computeLevel(snap) == ResourcePressureLevel::Warning); } // Restore original values. TuneAdvisor::setMemoryWarningThreshold(prevWarning); TuneAdvisor::setMemoryCriticalThreshold(prevCritical); TuneAdvisor::setMemoryEmergencyThreshold(prevEmergency); TuneAdvisor::setMemoryHysteresisMs(prevMemHys); TuneAdvisor::setCpuLevelHysteresisMs(prevCpuHys); governor.testing_setPressureState(ResourcePressureLevel::Normal, std::chrono::steady_clock::now()); } TEST_CASE("ResourceGovernor snapshot contains valid metrics", "[daemon][governance][catch2]") { auto& governor = ResourceGovernor::instance(); auto snapshot = governor.getSnapshot(); // RSS should be positive (we're running) // Note: May be 0 if RSS reading failed on this platform static_cast(snapshot.rssBytes); // Timestamp should be valid CHECK(snapshot.timestamp.time_since_epoch().count() >= 0); } TEST_CASE("ResourceGovernor::recommendVectorRebuildBudgetBytes scales with budget", "[daemon][governance][catch2]") { auto& governor = ResourceGovernor::instance(); const std::uint64_t prev = TuneAdvisor::memoryBudgetBytes(); SECTION("8 GiB budget yields 1 GiB cap (12.5%)") { TuneAdvisor::setMemoryBudgetBytes(8ULL * 1024ULL * 1024ULL * 1024ULL); const auto cap = governor.recommendVectorRebuildBudgetBytes(); CHECK(cap == 1024ULL * 1024ULL * 1024ULL); } SECTION("64 GiB budget clamps at 6 GiB ceiling") { TuneAdvisor::setMemoryBudgetBytes(64ULL * 1024ULL * 1024ULL * 1024ULL); const auto cap = governor.recommendVectorRebuildBudgetBytes(); CHECK(cap == 6ULL * 1024ULL * 1024ULL * 1024ULL); } SECTION("1 GiB budget clamps at 256 MiB floor") { TuneAdvisor::setMemoryBudgetBytes(1ULL * 1024ULL * 1024ULL * 1024ULL); const auto cap = governor.recommendVectorRebuildBudgetBytes(); CHECK(cap == 256ULL * 1024ULL * 1024ULL); } TuneAdvisor::setMemoryBudgetBytes(prev); } // ============================================================================= // Profile-Aware Configuration Tests // ============================================================================= TEST_CASE("TuneAdvisor profile affects resource settings", "[daemon][governance][catch2]") { SECTION("Efficient profile has lower thresholds") { ProfileGuard guard(TuneAdvisor::Profile::Efficient); auto scale = TuneAdvisor::profileScale(); CHECK(scale < 1.0); } SECTION("Balanced profile has moderate thresholds") { ProfileGuard guard(TuneAdvisor::Profile::Balanced); auto scale = TuneAdvisor::profileScale(); // Balanced profile now uses 0.5 scale CHECK(scale == 0.5); } SECTION("Aggressive profile has default thresholds") { ProfileGuard guard(TuneAdvisor::Profile::Aggressive); auto scale = TuneAdvisor::profileScale(); // Aggressive profile now uses 1.0 scale (reduced from 1.5) CHECK(scale == 1.0); } } TEST_CASE("OnnxConcurrencyRegistry setMaxSlots updates total", "[daemon][governance][catch2]") { auto& reg = OnnxConcurrencyRegistry::instance(); uint32_t originalTotal = reg.totalSlots(); reg.setMaxSlots(16); CHECK(reg.totalSlots() == 16); reg.setMaxSlots(4); CHECK(reg.totalSlots() == 4); // Restore original reg.setMaxSlots(originalTotal); CHECK(reg.totalSlots() == originalTotal); } // ============================================================================= // Concurrency Stress Tests // ============================================================================= TEST_CASE("Concurrent slot acquisition is thread-safe", "[daemon][governance][catch2]") { auto& reg = OnnxConcurrencyRegistry::instance(); SlotConfigGuard configGuard(8); std::atomic successCount{0}; std::atomic failCount{0}; auto worker = [&]() { OnnxConcurrencyRegistry::SlotGuard guard(reg, OnnxLane::Embedding, 50ms); if (guard.acquired()) { successCount++; std::this_thread::sleep_for(10ms); } else { failCount++; } }; std::vector threads; for (int i = 0; i < 16; ++i) { threads.emplace_back(worker); } for (auto& t : threads) { t.join(); } // Some should succeed, some may timeout CHECK(successCount > 0); // Total attempts should be 16 CHECK(successCount + failCount == 16); } TEST_CASE("Slots are released correctly after use", "[daemon][governance][catch2]") { auto& reg = OnnxConcurrencyRegistry::instance(); SlotConfigGuard configGuard(8); uint32_t beforeUsed = reg.usedSlots(); // Acquire and release in sequence { OnnxConcurrencyRegistry::SlotGuard guard1(reg, OnnxLane::Embedding, 100ms); CHECK(guard1.acquired()); CHECK(reg.usedSlots() == beforeUsed + 1); } CHECK(reg.usedSlots() == beforeUsed); // Acquire again - should work since slot was released { OnnxConcurrencyRegistry::SlotGuard guard2(reg, OnnxLane::Embedding, 100ms); CHECK(guard2.acquired()); } CHECK(reg.usedSlots() == beforeUsed); } // ============================================================================= // TuningManager Configuration Tests // ============================================================================= TEST_CASE("TuneAdvisor ONNX settings are profile-aware", "[daemon][governance][catch2]") { SECTION("onnxMaxConcurrent returns reasonable values") { auto maxConcurrent = TuneAdvisor::onnxMaxConcurrent(); // Should be between 4 and 16 (clamped from hw_threads/2) CHECK(maxConcurrent >= 2); CHECK(maxConcurrent <= 64); } SECTION("Reserved slot settings are available") { auto glinerReserved = TuneAdvisor::onnxGlinerReserved(); auto embedReserved = TuneAdvisor::onnxEmbedReserved(); auto rerankerReserved = TuneAdvisor::onnxRerankerReserved(); // Defaults: 1, 1, 1 static_cast(glinerReserved); static_cast(embedReserved); static_cast(rerankerReserved); } SECTION("Profile scale affects computed max slots") { // Calculate what TuningManager would set auto maxConcurrent = TuneAdvisor::onnxMaxConcurrent(); { ProfileGuard guard(TuneAdvisor::Profile::Efficient); double scale = TuneAdvisor::profileScale(); uint32_t scaledMax = static_cast(maxConcurrent * scale); CHECK(scaledMax < maxConcurrent); // Efficient (0.0) uses fewer slots } { ProfileGuard guard(TuneAdvisor::Profile::Balanced); double scale = TuneAdvisor::profileScale(); uint32_t scaledMax = static_cast(maxConcurrent * scale); CHECK(scaledMax < maxConcurrent); // Balanced (0.5) uses fewer slots than max } { ProfileGuard guard(TuneAdvisor::Profile::Aggressive); double scale = TuneAdvisor::profileScale(); uint32_t scaledMax = static_cast(maxConcurrent * scale); CHECK(scaledMax == maxConcurrent); // Aggressive (1.0) uses full slots } } } // ============================================================================= // Integration Tests (Governor + Registry) // ============================================================================= TEST_CASE("Governor and Registry are independently accessible", "[daemon][governance][catch2]") { auto& governor = ResourceGovernor::instance(); auto& registry = OnnxConcurrencyRegistry::instance(); // Both should be singletons CHECK(&governor == &ResourceGovernor::instance()); CHECK(®istry == &OnnxConcurrencyRegistry::instance()); // Both should have valid state CHECK(registry.totalSlots() > 0); auto caps = governor.getScalingCaps(); (void)caps; // Just verify it's callable } // ============================================================================= // Exact profileScale() Values // ============================================================================= TEST_CASE("profileScale returns exact values per profile", "[daemon][governance][catch2]") { SECTION("Efficient profile scale is exactly 0.0") { ProfileGuard guard(TuneAdvisor::Profile::Efficient); CHECK(TuneAdvisor::profileScale() == 0.0); } SECTION("Balanced profile scale is exactly 0.5") { ProfileGuard guard(TuneAdvisor::Profile::Balanced); CHECK(TuneAdvisor::profileScale() == 0.5); } SECTION("Aggressive profile scale is exactly 1.0") { ProfileGuard guard(TuneAdvisor::Profile::Aggressive); CHECK(TuneAdvisor::profileScale() == 1.0); } } // ============================================================================= // ONNX Reserved Slot Defaults Per Profile // ============================================================================= TEST_CASE("ONNX reserved slot defaults match profile", "[daemon][governance][catch2]") { SECTION("All profiles: gliner=1, embed=1") { for (auto profile : {TuneAdvisor::Profile::Efficient, TuneAdvisor::Profile::Balanced, TuneAdvisor::Profile::Aggressive}) { ProfileGuard guard(profile); CHECK(TuneAdvisor::onnxGlinerReserved() == 1); CHECK(TuneAdvisor::onnxEmbedReserved() == 1); } } SECTION("Efficient: reranker=0") { ProfileGuard guard(TuneAdvisor::Profile::Efficient); CHECK(TuneAdvisor::onnxRerankerReserved() == 0); } SECTION("Balanced: reranker=1") { ProfileGuard guard(TuneAdvisor::Profile::Balanced); CHECK(TuneAdvisor::onnxRerankerReserved() == 1); } SECTION("Aggressive: reranker=1") { ProfileGuard guard(TuneAdvisor::Profile::Aggressive); CHECK(TuneAdvisor::onnxRerankerReserved() == 1); } } // ============================================================================= // Slot Arithmetic Invariant // ============================================================================= TEST_CASE("Reserved slots never exceed max concurrent", "[daemon][governance][catch2]") { for (auto profile : {TuneAdvisor::Profile::Efficient, TuneAdvisor::Profile::Balanced, TuneAdvisor::Profile::Aggressive}) { ProfileGuard guard(profile); uint32_t totalReserved = TuneAdvisor::onnxGlinerReserved() + TuneAdvisor::onnxEmbedReserved() + TuneAdvisor::onnxRerankerReserved(); CHECK(totalReserved <= TuneAdvisor::onnxMaxConcurrent()); } } // ============================================================================= // Env Var Override for Reserved Slots // ============================================================================= TEST_CASE("ONNX reserved slot env var overrides", "[daemon][governance][catch2]") { ProfileGuard profileGuard(TuneAdvisor::Profile::Balanced); SECTION("YAMS_ONNX_EMBED_RESERVED overrides default") { ScopedEnvVar envGuard("YAMS_ONNX_EMBED_RESERVED", "3"); CHECK(TuneAdvisor::onnxEmbedReserved() == 3); } SECTION("YAMS_ONNX_RERANKER_RESERVED overrides profile-aware default") { ScopedEnvVar envGuard("YAMS_ONNX_RERANKER_RESERVED", "4"); CHECK(TuneAdvisor::onnxRerankerReserved() == 4); } } // ============================================================================= // Lane-Specific Slot Acquisition Across Lanes // ============================================================================= TEST_CASE("Slots acquired across multiple lanes", "[daemon][governance][catch2]") { auto& reg = OnnxConcurrencyRegistry::instance(); SlotConfigGuard configGuard(8); uint32_t beforeUsed = reg.usedSlots(); { OnnxConcurrencyRegistry::SlotGuard glinerSlot(reg, OnnxLane::Gliner, 100ms); OnnxConcurrencyRegistry::SlotGuard embedSlot(reg, OnnxLane::Embedding, 100ms); OnnxConcurrencyRegistry::SlotGuard rerankerSlot(reg, OnnxLane::Reranker, 100ms); CHECK(glinerSlot.acquired()); CHECK(embedSlot.acquired()); CHECK(rerankerSlot.acquired()); CHECK(reg.usedSlots() == beforeUsed + 3); } CHECK(reg.usedSlots() == beforeUsed); } // ============================================================================= // Memory Threshold Ordering Across All Profiles // ============================================================================= TEST_CASE("Memory thresholds maintain ordering invariant", "[daemon][governance][catch2]") { TuneAdvisor::resetModelEvictThresholdOverrides(); for (auto profile : {TuneAdvisor::Profile::Efficient, TuneAdvisor::Profile::Balanced, TuneAdvisor::Profile::Aggressive}) { ProfileGuard guard(profile); CHECK(TuneAdvisor::modelEvictWarningThreshold() < TuneAdvisor::modelEvictCriticalThreshold()); CHECK(TuneAdvisor::modelEvictCriticalThreshold() < TuneAdvisor::modelEvictEmergencyThreshold()); } } // ============================================================================= // Scaling Caps Pressure Level Tests (testing_updateScalingCaps) // ============================================================================= TEST_CASE("ResourceGovernor scaling caps change at Warning level", "[daemon][governance][catch2]") { auto& governor = ResourceGovernor::instance(); // Record caps at Normal governor.testing_updateScalingCaps(ResourcePressureLevel::Normal); auto normalCaps = governor.getScalingCaps(); // Update to Warning governor.testing_updateScalingCaps(ResourcePressureLevel::Warning); auto warningCaps = governor.getScalingCaps(); // Warning should have equal or lower caps CHECK(warningCaps.ingestWorkers <= normalCaps.ingestWorkers); CHECK(warningCaps.searchConcurrency <= normalCaps.searchConcurrency); CHECK(warningCaps.extractionConcurrency <= normalCaps.extractionConcurrency); CHECK(warningCaps.kgConcurrency <= normalCaps.kgConcurrency); CHECK(warningCaps.embedConcurrency <= normalCaps.embedConcurrency); // Warning blocks model loads but still allows ingest CHECK_FALSE(warningCaps.allowModelLoads); CHECK(warningCaps.allowNewIngest); // Restore Normal governor.testing_updateScalingCaps(ResourcePressureLevel::Normal); } TEST_CASE("ResourceGovernor scaling caps at Critical level", "[daemon][governance][catch2]") { auto& governor = ResourceGovernor::instance(); // Record Normal caps for comparison governor.testing_updateScalingCaps(ResourcePressureLevel::Normal); auto normalCaps = governor.getScalingCaps(); // Update to Critical governor.testing_updateScalingCaps(ResourcePressureLevel::Critical); auto criticalCaps = governor.getScalingCaps(); // Critical should have even lower caps than Warning CHECK(criticalCaps.ingestWorkers <= normalCaps.ingestWorkers); CHECK(criticalCaps.searchConcurrency <= normalCaps.searchConcurrency); // Critical blocks model loads CHECK_FALSE(criticalCaps.allowModelLoads); // Critical still allows ingest (only Emergency blocks it) CHECK(criticalCaps.allowNewIngest); // Restore Normal governor.testing_updateScalingCaps(ResourcePressureLevel::Normal); } TEST_CASE("ResourceGovernor scaling caps at Emergency level", "[daemon][governance][catch2]") { auto& governor = ResourceGovernor::instance(); // Update to Emergency governor.testing_updateScalingCaps(ResourcePressureLevel::Emergency); auto emergencyCaps = governor.getScalingCaps(); // Emergency should halt new ingest CHECK_FALSE(emergencyCaps.allowNewIngest); CHECK_FALSE(emergencyCaps.allowModelLoads); // Embed concurrency should be 0 CHECK(emergencyCaps.embedConcurrency == 0); // Extraction concurrency should be 0 CHECK(emergencyCaps.extractionConcurrency == 0); // Restore Normal governor.testing_updateScalingCaps(ResourcePressureLevel::Normal); } TEST_CASE("ResourceGovernor caps restore to full after returning to Normal", "[daemon][governance][catch2]") { auto& governor = ResourceGovernor::instance(); // Record Normal caps governor.testing_updateScalingCaps(ResourcePressureLevel::Normal); auto normalCaps = governor.getScalingCaps(); // Go through Emergency and back to Normal governor.testing_updateScalingCaps(ResourcePressureLevel::Emergency); governor.testing_updateScalingCaps(ResourcePressureLevel::Normal); auto restoredCaps = governor.getScalingCaps(); // Should be back to full values CHECK(restoredCaps.ingestWorkers == normalCaps.ingestWorkers); CHECK(restoredCaps.searchConcurrency == normalCaps.searchConcurrency); CHECK(restoredCaps.extractionConcurrency == normalCaps.extractionConcurrency); CHECK(restoredCaps.kgConcurrency == normalCaps.kgConcurrency); CHECK(restoredCaps.embedConcurrency == normalCaps.embedConcurrency); CHECK(restoredCaps.allowModelLoads); CHECK(restoredCaps.allowNewIngest); } TEST_CASE("ResourceGovernor caps ordering: Normal >= Emergency (endpoints)", "[daemon][governance][catch2]") { auto& governor = ResourceGovernor::instance(); governor.testing_updateScalingCaps(ResourcePressureLevel::Normal); auto normal = governor.getScalingCaps(); governor.testing_updateScalingCaps(ResourcePressureLevel::Emergency); auto emergency = governor.getScalingCaps(); // Normal should always be >= Emergency across all dimensions CHECK(normal.ingestWorkers >= emergency.ingestWorkers); CHECK(normal.embedConcurrency >= emergency.embedConcurrency); CHECK(normal.extractionConcurrency >= emergency.extractionConcurrency); CHECK(normal.searchConcurrency >= emergency.searchConcurrency); CHECK(normal.kgConcurrency >= emergency.kgConcurrency); // Normal allows everything, Emergency blocks CHECK(normal.allowModelLoads); CHECK(normal.allowNewIngest); CHECK_FALSE(emergency.allowModelLoads); CHECK_FALSE(emergency.allowNewIngest); // Restore Normal governor.testing_updateScalingCaps(ResourcePressureLevel::Normal); } // ============================================================================= // ResourceGovernor Tick and Admission Control Tests // ============================================================================= TEST_CASE("ResourceGovernor tick is callable without ServiceManager", "[daemon][governance][catch2]") { auto& governor = ResourceGovernor::instance(); // tick(nullptr) should not crash — it just skips SM metric collection auto snap = governor.tick(nullptr); CHECK(snap.timestamp.time_since_epoch().count() > 0); // RSS should be readable (we're a running process) CHECK(snap.rssBytes > 0); } TEST_CASE("ResourceGovernor canAdmitWork blocked at Emergency", "[daemon][governance][catch2]") { auto& governor = ResourceGovernor::instance(); // At Normal, work should be admitted governor.testing_updateScalingCaps(ResourcePressureLevel::Normal); CHECK(governor.getScalingCaps().allowNewIngest); // At Emergency, allowNewIngest is false governor.testing_updateScalingCaps(ResourcePressureLevel::Emergency); CHECK_FALSE(governor.getScalingCaps().allowNewIngest); // Restore governor.testing_updateScalingCaps(ResourcePressureLevel::Normal); }