/** * @file post_ingest_queue_validation_test.cpp * @brief Integration tests to validate PostIngestQueue and repair indexing service * * These tests verify that: * 1. PostIngestQueue is properly initialized * 2. Documents are actually enqueued after ingestion * 3. Background indexing (FTS5, KG, embeddings) happens * 4. Background repair detects and fixes missing indexes */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "common/fixture_manager.h" #include "common/test_helpers_catch2.h" #include "service_manager_test_helper.h" // Windows daemon tests are currently unstable - ServiceManager async initialization // has socket path length issues and shutdown race conditions #ifdef _WIN32 #define SKIP_DAEMON_TEST_ON_WINDOWS() \ SKIP("Daemon tests unstable on Windows - see windows-daemon-ipc-plan.md") #else #define SKIP_DAEMON_TEST_ON_WINDOWS() ((void)0) #endif namespace fs = std::filesystem; using namespace yams; using namespace yams::daemon; using namespace yams::app::services; namespace { class WorkCoordinatorThreadsGuard { public: explicit WorkCoordinatorThreadsGuard(std::uint32_t threads) : previous_(TuneAdvisor::workCoordinatorThreads()) { TuneAdvisor::setWorkCoordinatorThreads(threads); } ~WorkCoordinatorThreadsGuard() { TuneAdvisor::setWorkCoordinatorThreads(previous_); } private: std::uint32_t previous_; }; struct DrainSignal { std::promise promise; std::atomic signalled{false}; void notify() { if (!signalled.exchange(true, std::memory_order_acq_rel)) { promise.set_value(); } } }; /** * @brief Test fixture for PostIngestQueue validation */ class PostIngestQueueFixture { public: PostIngestQueueFixture() { setupTestEnvironment(); setupDaemonComponents(); } ~PostIngestQueueFixture() { cleanupDaemonComponents(); cleanupTestEnvironment(); } void setupTestEnvironment() { auto pid = std::to_string(::getpid()); auto timestamp = std::to_string(std::chrono::system_clock::now().time_since_epoch().count()); testDir_ = fs::temp_directory_path() / ("post_ingest_test_" + pid + "_" + timestamp); fs::create_directories(testDir_); // Keep all daemon/session state inside the temp test directory and restore the exact host // values after every fixture. environment_.emplace_back("YAMS_STORAGE", testDir_.string()); environment_.emplace_back("XDG_DATA_HOME", testDir_.string()); environment_.emplace_back("XDG_STATE_HOME", testDir_.string()); environment_.emplace_back("HOME", testDir_.string()); fixtureManager_ = std::make_shared(testDir_); } void setupDaemonComponents() { // Create daemon config config_.dataDir = testDir_; config_.socketPath = testDir_ / "yams.sock"; // Enable post-ingest queue with minimal threads for testing config_.tuning.postIngestThreadsMin = 2; config_.tuning.postIngestThreadsMax = 4; config_.tuning.postIngestCapacity = 4; // Create state and lifecycle components state_ = std::make_unique(); lifecycleFsm_ = std::make_unique(); // Create service manager serviceManager_ = std::make_shared(config_, *state_, *lifecycleFsm_); // Initialize service manager (both sync and async phases) bool initialized = yams::test::initializeServiceManagerFully(serviceManager_); REQUIRE(initialized); } void cleanupDaemonComponents() { if (serviceManager_) { serviceManager_->shutdown(); serviceManager_.reset(); } lifecycleFsm_.reset(); state_.reset(); } void cleanupTestEnvironment() { fixtureManager_.reset(); environment_.clear(); if (fs::exists(testDir_)) { fs::remove_all(testDir_); } } // Helper: Wait for post-ingest queue to drain (both channel AND in-flight work) bool waitForQueueDrain(std::chrono::milliseconds timeout = std::chrono::seconds(30)) { auto start = std::chrono::steady_clock::now(); while (std::chrono::steady_clock::now() - start < timeout) { if (auto queue = serviceManager_->getPostIngestQueue()) { auto size = queue->size(); auto inFlight = queue->totalInFlight(); spdlog::info("PostIngestQueue status: size={}, inFlight={}", size, inFlight); // Must wait for BOTH channel to be empty AND all in-flight work to complete if (size == 0 && inFlight == 0) { // Extra delay for FTS5 index to finalize writes std::this_thread::sleep_for(std::chrono::milliseconds(500)); return true; } } std::this_thread::sleep_for(std::chrono::milliseconds(100)); } return false; } bool waitForProcessed(std::size_t target, std::chrono::milliseconds timeout = std::chrono::seconds(30)) { auto start = std::chrono::steady_clock::now(); while (std::chrono::steady_clock::now() - start < timeout) { if (auto queue = serviceManager_->getPostIngestQueue()) { if (queue->processed() >= target) { return true; } } std::this_thread::sleep_for(std::chrono::milliseconds(100)); } return false; } bool waitForKgQueued(std::uint64_t target, std::chrono::milliseconds timeout = std::chrono::seconds(30)) { const auto deadline = std::chrono::steady_clock::now() + timeout; while (std::chrono::steady_clock::now() < deadline) { if (InternalEventBus::instance().kgQueued() >= target) { return true; } std::this_thread::sleep_for(std::chrono::milliseconds(10)); } return false; } bool waitForFailed(std::size_t target, std::chrono::milliseconds timeout = std::chrono::seconds(30)) { const auto deadline = std::chrono::steady_clock::now() + timeout; while (std::chrono::steady_clock::now() < deadline) { if (auto queue = serviceManager_->getPostIngestQueue(); queue && queue->failed() >= target) { return true; } std::this_thread::sleep_for(std::chrono::milliseconds(10)); } return false; } bool waitForExtractionIdle(std::chrono::milliseconds timeout = std::chrono::seconds(30)) { const auto deadline = std::chrono::steady_clock::now() + timeout; while (std::chrono::steady_clock::now() < deadline) { if (auto queue = serviceManager_->getPostIngestQueue(); queue && queue->size() == 0 && queue->totalInFlight() == 0) { return true; } std::this_thread::sleep_for(std::chrono::milliseconds(10)); } return false; } // Helper: Create and store a document std::string storeDocument(const std::string& filename, const std::string& content) { auto fixture = fixtureManager_->createTextFixture(filename, content); // Get document service from service manager's app context auto appContext = serviceManager_->getAppContext(); auto docService = makeDocumentService(appContext); REQUIRE(docService); StoreDocumentRequest req; req.path = fixture.path.string(); auto result = docService->store(req); REQUIRE(result); return result.value().hash; } fs::path testDir_; std::shared_ptr fixtureManager_; std::vector environment_; DaemonConfig config_; std::unique_ptr state_; std::unique_ptr lifecycleFsm_; std::shared_ptr serviceManager_; }; } // anonymous namespace // ============================================================================ // PostIngestQueue Initialization Tests // ============================================================================ TEST_CASE("PostIngestQueue - Initialization", "[daemon][post-ingest][init]") { SKIP_DAEMON_TEST_ON_WINDOWS(); PostIngestQueueFixture fixture; SECTION("PostIngestQueue is created") { auto queue = fixture.serviceManager_->getPostIngestQueue(); REQUIRE((queue != nullptr)); INFO("Queue should have capacity"); REQUIRE((queue->capacity() > 0)); } SECTION("PostIngestQueue metrics are accessible") { auto queue = fixture.serviceManager_->getPostIngestQueue(); REQUIRE((queue != nullptr)); INFO("Fresh queue metrics should be empty"); CHECK((queue->size() == 0)); CHECK((queue->processed() == 0)); CHECK((queue->failed() == 0)); } } // ============================================================================ // Document Ingestion and Queueing Tests // ============================================================================ TEST_CASE("PostIngestQueue - Document Enqueuing", "[daemon][post-ingest][enqueue]") { SKIP_DAEMON_TEST_ON_WINDOWS(); PostIngestQueueFixture fixture; SECTION("Documents are enqueued after storage") { auto queue = fixture.serviceManager_->getPostIngestQueue(); REQUIRE((queue != nullptr)); const auto initialProcessed = queue->processed(); // Store a document auto hash = fixture.storeDocument("test.txt", "Hello World"); REQUIRE(!hash.empty()); // Manually enqueue to post-ingest (simulating what IngestService does) fixture.serviceManager_->enqueuePostIngest(hash, "text/plain"); INFO("PostIngestQueue should process the document before the deadline"); REQUIRE(fixture.waitForProcessed(initialProcessed + 1, std::chrono::seconds(30))); } SECTION("Multiple documents are processed") { auto queue = fixture.serviceManager_->getPostIngestQueue(); REQUIRE((queue != nullptr)); auto initialProcessed = queue->processed(); // Store multiple documents std::vector hashes; for (int i = 0; i < 5; i++) { auto hash = fixture.storeDocument("doc" + std::to_string(i) + ".txt", "Content " + std::to_string(i)); hashes.push_back(hash); fixture.serviceManager_->enqueuePostIngest(hash, "text/plain"); } INFO("All documents should be processed"); REQUIRE(fixture.waitForProcessed(initialProcessed + 5, std::chrono::seconds(30))); } } // ============================================================================ // FTS5 Indexing Verification Tests // ============================================================================ TEST_CASE("PostIngestQueue - FTS5 Indexing", "[daemon][post-ingest][fts5]") { SKIP_DAEMON_TEST_ON_WINDOWS(); PostIngestQueueFixture fixture; SECTION("Stored document is searchable after post-ingest") { // Store a document with unique content // NOTE: Use spaces, not underscores! FTS5 tokenizer treats underscores as token chars // (tokenchars '_-'), so "xyzzy_foo" is ONE token. Spaces create separate tokens. const std::string uniqueContent = "xyzzy unique test content 12345"; auto hash = fixture.storeDocument("searchable.txt", uniqueContent); // Enqueue for post-ingest processing fixture.serviceManager_->enqueuePostIngest(hash, "text/plain"); // Wait for processing bool drained = fixture.waitForQueueDrain(std::chrono::seconds(30)); REQUIRE(drained); // Try to search for the document auto appContext = fixture.serviceManager_->getAppContext(); auto searchService = makeSearchService(appContext); REQUIRE(searchService); app::services::SearchRequest req; req.query = "xyzzy"; // Search for unique token req.type = "keyword"; req.limit = 10; req.showHash = true; // REQUIRED: populate hash field in results req.globalSearch = true; // Bypass session isolation for test // Run search using async helper boost::asio::io_context ioc; auto fut = boost::asio::co_spawn(ioc, searchService->search(req), boost::asio::use_future); ioc.run(); auto result = fut.get(); REQUIRE(result); INFO("Document should be found in FTS5 search"); REQUIRE((result.value().results.size() > 0)); // Verify the document was actually indexed bool found = false; for (const auto& item : result.value().results) { if (item.hash == hash) { found = true; break; } } REQUIRE(found); } } // ============================================================================ // Sync Indexing Tests (PBI-040 feature) // ============================================================================ TEST_CASE("PostIngestQueue - Synchronous Indexing", "[daemon][post-ingest][sync]") { SKIP_DAEMON_TEST_ON_WINDOWS(); PostIngestQueueFixture fixture; SECTION("Documents are indexed via async channel") { auto queue = fixture.serviceManager_->getPostIngestQueue(); REQUIRE((queue != nullptr)); auto hash = fixture.storeDocument("sync_test.txt", "Sync content for async test"); // Enqueue for post-ingest processing fixture.serviceManager_->enqueuePostIngest(hash, "text/plain"); REQUIRE(fixture.waitForQueueDrain(std::chrono::seconds(30))); auto appContext = fixture.serviceManager_->getAppContext(); auto searchService = makeSearchService(appContext); REQUIRE(searchService); app::services::SearchRequest req; req.query = "Sync"; req.type = "keyword"; req.limit = 10; req.globalSearch = true; // Bypass session isolation for test boost::asio::io_context ioc; auto fut = boost::asio::co_spawn(ioc, searchService->search(req), boost::asio::use_future); ioc.run(); auto result = fut.get(); REQUIRE(result); REQUIRE((result.value().results.size() > 0)); } } TEST_CASE("PostIngestQueue - continues processing when KG stage is paused", "[daemon][post-ingest][sync][kg-paused]") { SKIP_DAEMON_TEST_ON_WINDOWS(); PostIngestQueueFixture fixture; auto queue = fixture.serviceManager_->getPostIngestQueue(); REQUIRE((queue != nullptr)); auto& bus = InternalEventBus::instance(); const auto processedBefore = queue->processed(); const auto kgQueuedBefore = bus.kgQueued(); const auto kgConsumedBefore = bus.kgConsumed(); const auto kgDroppedBefore = bus.kgDropped(); queue->pauseStage(PostIngestQueue::Stage::KnowledgeGraph); auto hash = fixture.storeDocument("kg_paused_test.txt", "Pipeline should still index this content when KG is paused"); fixture.serviceManager_->enqueuePostIngest(hash, "text/plain"); REQUIRE(fixture.waitForProcessed(processedBefore + 1, std::chrono::seconds(30))); REQUIRE(fixture.waitForKgQueued(kgQueuedBefore + 1, std::chrono::seconds(30))); CHECK((bus.kgQueued() == kgQueuedBefore + 1)); CHECK((bus.kgConsumed() == kgConsumedBefore)); CHECK((bus.kgDropped() == kgDroppedBefore)); auto appContext = fixture.serviceManager_->getAppContext(); auto searchService = makeSearchService(appContext); REQUIRE(searchService); app::services::SearchRequest req; req.query = "Pipeline should still index"; req.type = "keyword"; req.limit = 10; req.globalSearch = true; boost::asio::io_context ioc; auto fut = boost::asio::co_spawn(ioc, searchService->search(req), boost::asio::use_future); ioc.run(); auto result = fut.get(); REQUIRE(result); REQUIRE((result.value().results.size() > 0)); auto drainSignal = std::make_shared(); auto drainedFuture = drainSignal->promise.get_future(); queue->setDrainCallback([drainSignal]() { drainSignal->notify(); }); queue->resumeStage(PostIngestQueue::Stage::KnowledgeGraph); REQUIRE((drainedFuture.wait_for(std::chrono::seconds(10)) == std::future_status::ready)); queue->setDrainCallback({}); CHECK((bus.kgConsumed() == kgConsumedBefore + 1)); CHECK((bus.kgDropped() == kgDroppedBefore)); } // ============================================================================ // Queue Capacity and Backpressure Tests // ============================================================================ TEST_CASE("PostIngestQueue - Capacity and Backpressure", "[daemon][post-ingest][capacity]") { SKIP_DAEMON_TEST_ON_WINDOWS(); PostIngestQueueFixture fixture; SECTION("Runtime tuning does not misreport construction-time channel capacity") { auto queue = fixture.serviceManager_->getPostIngestQueue(); REQUIRE((queue != nullptr)); const auto constructedCapacity = queue->capacity(); auto tuning = fixture.serviceManager_->getTuningConfig(); tuning.postIngestCapacity = static_cast(constructedCapacity + 32); fixture.serviceManager_->setTuningConfig(tuning); CHECK((queue->capacity() == constructedCapacity)); CHECK( (fixture.serviceManager_->getTuningConfig().postIngestCapacity == constructedCapacity)); } SECTION("Paused extraction applies deterministic bounded backpressure") { auto queue = fixture.serviceManager_->getPostIngestQueue(); REQUIRE((queue != nullptr)); queue->pauseStage(PostIngestQueue::Stage::Extraction); REQUIRE(queue->isStagePaused(PostIngestQueue::Stage::Extraction)); const auto capacity = queue->capacity(); INFO("Queue capacity: " << capacity); REQUIRE((capacity > 1)); std::size_t enqueued = 0; std::size_t rejected = 0; for (std::size_t i = 0; i < capacity + 10; ++i) { PostIngestQueue::Task task; task.hash = "backpressure_hash_" + std::to_string(i); task.mime = "text/plain"; if (queue->tryEnqueue(std::move(task))) { ++enqueued; } else { ++rejected; } } INFO("Enqueued: " << enqueued << ", Rejected: " << rejected); CHECK((enqueued == capacity - 1)); CHECK((rejected == 11)); CHECK((queue->size() == enqueued)); const auto failedBefore = queue->failed(); queue->resumeStage(PostIngestQueue::Stage::Extraction); REQUIRE(fixture.waitForFailed(failedBefore + enqueued, std::chrono::seconds(10))); CHECK((queue->size() == 0)); } } TEST_CASE("PostIngestQueue - KG disable and saturation semantics", "[daemon][post-ingest][backpressure]") { SKIP_DAEMON_TEST_ON_WINDOWS(); WorkCoordinatorThreadsGuard workerGuard(1); PostIngestQueueFixture fixture; auto queue = fixture.serviceManager_->getPostIngestQueue(); REQUIRE((queue != nullptr)); auto& bus = InternalEventBus::instance(); SECTION("Explicit disable is outside the pipeline and does not count as a drop") { const auto processedBefore = queue->processed(); const auto queuedBefore = bus.kgQueued(); const auto consumedBefore = bus.kgConsumed(); const auto droppedBefore = bus.kgDropped(); queue->setKnowledgeGraphEnabled(false); const auto hash = fixture.storeDocument("kg-disabled.txt", "explicitly disabled KG"); REQUIRE(queue->tryEnqueue(PostIngestQueue::Task{ .hash = hash, .mime = "text/plain", .filePath = "kg-disabled.txt"})); REQUIRE(fixture.waitForProcessed(processedBefore + 1, std::chrono::seconds(10))); CHECK((bus.kgQueued() == queuedBefore)); CHECK((bus.kgConsumed() == consumedBefore)); CHECK((bus.kgDropped() == droppedBefore)); } SECTION("Disable racing a saturated dispatch cancels without recording a drop") { const auto queuedBefore = bus.kgQueued(); const auto consumedBefore = bus.kgConsumed(); const auto droppedBefore = bus.kgDropped(); queue->pauseStage(PostIngestQueue::Stage::KnowledgeGraph); std::size_t accepted = 0; for (std::size_t i = 0; i < 32; ++i) { const auto name = "kg-disable-race-" + std::to_string(i) + ".txt"; const auto hash = fixture.storeDocument(name, "KG disable race " + std::to_string(i)); if (queue->tryEnqueue( PostIngestQueue::Task{.hash = hash, .mime = "text/plain", .filePath = name})) { ++accepted; } } REQUIRE((accepted > 3)); REQUIRE((accepted < 32)); REQUIRE(fixture.waitForKgQueued(queuedBefore + 3, std::chrono::seconds(10))); queue->setKnowledgeGraphEnabled(false); REQUIRE(fixture.waitForExtractionIdle(std::chrono::seconds(10))); CHECK((bus.kgDropped() == droppedBefore)); const auto queuedAfterDisable = bus.kgQueued(); queue->setKnowledgeGraphEnabled(true); auto drainSignal = std::make_shared(); auto drainedFuture = drainSignal->promise.get_future(); queue->setDrainCallback([drainSignal]() { drainSignal->notify(); }); queue->resumeStage(PostIngestQueue::Stage::KnowledgeGraph); REQUIRE((drainedFuture.wait_for(std::chrono::seconds(30)) == std::future_status::ready)); queue->setDrainCallback({}); CHECK((bus.kgConsumed() == consumedBefore + (queuedAfterDisable - queuedBefore))); CHECK((bus.kgDropped() == droppedBefore)); } SECTION("Saturated paused KG preserves every accepted job with one requested worker") { const auto processedBefore = queue->processed(); const auto queuedBefore = bus.kgQueued(); const auto consumedBefore = bus.kgConsumed(); const auto droppedBefore = bus.kgDropped(); queue->pauseStage(PostIngestQueue::Stage::KnowledgeGraph); std::size_t accepted = 0; for (std::size_t i = 0; i < 32; ++i) { const auto name = "kg-saturated-" + std::to_string(i) + ".txt"; const auto hash = fixture.storeDocument(name, "saturated KG document " + std::to_string(i)); if (queue->tryEnqueue( PostIngestQueue::Task{.hash = hash, .mime = "text/plain", .filePath = name})) { ++accepted; } } REQUIRE((accepted > 0)); REQUIRE((accepted < 32)); REQUIRE(fixture.waitForKgQueued(queuedBefore + 1, std::chrono::seconds(10))); CHECK((bus.kgDropped() == droppedBefore)); auto drainSignal = std::make_shared(); auto drainedFuture = drainSignal->promise.get_future(); queue->setDrainCallback([drainSignal]() { drainSignal->notify(); }); queue->resumeStage(PostIngestQueue::Stage::KnowledgeGraph); REQUIRE((drainedFuture.wait_for(std::chrono::seconds(30)) == std::future_status::ready)); queue->setDrainCallback({}); REQUIRE(fixture.waitForProcessed(processedBefore + accepted, std::chrono::seconds(10))); CHECK((bus.kgQueued() == queuedBefore + accepted)); CHECK((bus.kgConsumed() == consumedBefore + accepted)); CHECK((bus.kgDropped() == droppedBefore)); } } // ============================================================================ // Error Handling Tests // ============================================================================ TEST_CASE("PostIngestQueue - Error Handling", "[daemon][post-ingest][errors]") { SKIP_DAEMON_TEST_ON_WINDOWS(); PostIngestQueueFixture fixture; SECTION("Missing content records extraction failure before the queue drains") { auto queue = fixture.serviceManager_->getPostIngestQueue(); REQUIRE((queue != nullptr)); queue->pauseStage(PostIngestQueue::Stage::Extraction); REQUIRE(queue->isStagePaused(PostIngestQueue::Stage::Extraction)); const auto failedBefore = queue->failed(); PostIngestQueue::Task task; task.hash = "nonexistent_hash_12345"; task.mime = "text/plain"; REQUIRE(queue->tryEnqueue(std::move(task))); queue->resumeStage(PostIngestQueue::Stage::Extraction); REQUIRE(fixture.waitForFailed(failedBefore + 1, std::chrono::seconds(10))); INFO("failed before=" << failedBefore << ", after=" << queue->failed()); CHECK((queue->failed() == failedBefore + 1)); CHECK((queue->size() == 0)); CHECK((queue->totalInFlight() == 0)); } } // ============================================================================ // Thread Scaling Tests // ============================================================================ // NOTE: Thread scaling test removed - resizePostIngestThreads API does not exist // in the current ServiceManager implementation. If this functionality is added // in the future, tests should be re-enabled.