#include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include using namespace yams; using namespace yams::app::services; using namespace yams::metadata; using namespace yams::content; using namespace yams::detection; using namespace yams::api; namespace { struct IndexingFixture { IndexingFixture() { initializeGlobalSingletons(); setupTestEnvironment(); setupDatabase(); setupServices(); } ~IndexingFixture() { indexingService_.reset(); searchEngine_.reset(); appContext_.metadataRepo.reset(); appContext_.store.reset(); appContext_.searchEngine.reset(); contentStore_.reset(); metadataRepo_.reset(); if (pool_) { pool_->shutdown(); pool_.reset(); } if (database_) { database_->close(); database_.reset(); } std::error_code ec; if (!testDir_.empty() && std::filesystem::exists(testDir_)) { std::filesystem::remove_all(testDir_, ec); } if (!infraDir_.empty() && std::filesystem::exists(infraDir_)) { std::filesystem::remove_all(infraDir_, ec); } } void initializeGlobalSingletons() { static bool initialized = false; if (initialized) return; auto& detector = FileTypeDetector::instance(); auto detectorInit = detector.initializeWithMagicNumbers(); REQUIRE(detectorInit); auto& registry = ContentHandlerRegistry::instance(); registry.clear(); initialized = true; } void setupTestEnvironment() { auto pid = std::to_string(::getpid()); auto timestamp = std::to_string(std::chrono::system_clock::now().time_since_epoch().count()); testDir_ = std::filesystem::temp_directory_path() / ("indexing_test_" + pid + "_" + timestamp); infraDir_ = std::filesystem::temp_directory_path() / ("indexing_infra_" + pid + "_" + timestamp); std::filesystem::create_directories(testDir_); std::filesystem::create_directories(infraDir_); } void setupDatabase() { dbPath_ = std::filesystem::absolute(infraDir_ / "test.db"); database_ = std::make_unique(); REQUIRE(database_->open(dbPath_.string(), ConnectionMode::Create)); pool_ = std::make_unique(dbPath_.string(), ConnectionPoolConfig{.maxConnections = 4}); metadataRepo_ = std::make_shared(*pool_); MigrationManager mm(*database_); REQUIRE(mm.initialize()); mm.registerMigrations(YamsMetadataMigrations::getAllMigrations()); REQUIRE(mm.migrate()); } void setupServices() { ContentStoreBuilder builder; auto storeResult = builder.withStoragePath(infraDir_ / "storage") .withChunkSize(65536) .withCompression(true) .withDeduplication(true) .build(); REQUIRE(storeResult); contentStore_ = std::shared_ptr( const_cast&>(storeResult.value()).release()); KnowledgeGraphStoreConfig kgConfig; auto kgResult = makeSqliteKnowledgeGraphStore(*pool_, kgConfig); REQUIRE(kgResult.has_value()); kgStore_ = std::shared_ptr(std::move(kgResult).value()); metadataRepo_->setKnowledgeGraphStore(kgStore_); appContext_.service_manager = nullptr; appContext_.store = contentStore_; appContext_.metadataRepo = metadataRepo_; appContext_.kgStore = kgStore_; appContext_.searchEngine = nullptr; indexingService_ = makeIndexingService(appContext_); } std::filesystem::path createFile(std::string_view filename, std::string_view content) { auto filePath = testDir_ / filename; std::filesystem::create_directories(filePath.parent_path()); std::ofstream{filePath} << content; return filePath; } std::filesystem::path createBinaryFile(std::string_view filename, size_t size = 1000) { auto filePath = testDir_ / filename; std::filesystem::create_directories(filePath.parent_path()); std::ofstream file(filePath, std::ios::binary); for (size_t i = 0; i < size; ++i) { file.put(static_cast(i % 256)); } return filePath; } AddDirectoryRequest createRequest(const std::filesystem::path& path, std::string_view collection = "") { AddDirectoryRequest req; req.directoryPath = path.string(); req.collection = std::string(collection); req.recursive = false; return req; } std::filesystem::path testDir_; std::filesystem::path infraDir_; std::filesystem::path dbPath_; std::unique_ptr database_; std::unique_ptr pool_; std::shared_ptr metadataRepo_; std::shared_ptr kgStore_; std::shared_ptr contentStore_; std::shared_ptr searchEngine_; AppContext appContext_; std::shared_ptr indexingService_; }; } // namespace TEST_CASE("IndexingService - Basic Operations", "[indexing][service][basic]") { IndexingFixture fixture; SECTION("Index single file in directory") { fixture.createFile("docs/test.txt", "This is a test document with some content."); auto request = fixture.createRequest(fixture.testDir_ / "docs", "test-collection"); auto result = fixture.indexingService_->addDirectory(request); REQUIRE(result); CHECK(result.value().filesIndexed == 1); CHECK(result.value().filesSkipped == 0); CHECK(result.value().filesFailed == 0); CHECK(result.value().filesProcessed == 1); REQUIRE(result.value().results.size() == 1); auto document = fixture.metadataRepo_->getDocumentByHash(result.value().results[0].hash); REQUIRE(document); REQUIRE(document.value().has_value()); CHECK(document.value()->contentExtracted); } SECTION("Indexing populates canonical doc and blob KG nodes") { auto filePath = fixture.createFile("docs/kg.txt", "Knowledge graph ingest coverage."); auto request = fixture.createRequest(fixture.testDir_ / "docs", "kg-collection"); auto result = fixture.indexingService_->addDirectory(request); REQUIRE(result); REQUIRE(result.value().filesIndexed == 1); auto docInfo = fixture.metadataRepo_->findDocumentByExactPath(filePath.string()); REQUIRE(docInfo); REQUIRE(docInfo.value().has_value()); const auto& document = *docInfo.value(); auto docNode = fixture.kgStore_->getNodeByKey("doc:" + document.sha256Hash); REQUIRE(docNode); REQUIRE(docNode.value().has_value()); CHECK(docNode.value()->type == std::optional{"document"}); auto blobNode = fixture.kgStore_->getNodeByKey("blob:" + document.sha256Hash); REQUIRE(blobNode); REQUIRE(blobNode.value().has_value()); CHECK(blobNode.value()->type == std::optional{"blob"}); auto edges = fixture.kgStore_->getEdgesFrom(docNode.value()->id, "has_blob", 10, 0); REQUIRE(edges); REQUIRE(edges.value().size() == 1); CHECK(edges.value().front().dstNodeId == blobNode.value()->id); } SECTION("Index multiple files in directory") { fixture.createFile("multi/file1.txt", "Content of first file"); fixture.createFile("multi/file2.md", "# Markdown content\n\nSome **bold** text"); fixture.createFile("multi/file3.cpp", "#include \nint main() { return 0; }"); AddDirectoryRequest request; request.directoryPath = (fixture.testDir_ / "multi").string(); request.collection = "test-batch"; request.recursive = false; auto result = fixture.indexingService_->addDirectory(request); REQUIRE(result); CHECK(result.value().filesIndexed == 3); CHECK(result.value().filesSkipped == 0); CHECK(result.value().filesProcessed == 3); } SECTION("Index directory recursively") { fixture.createFile("recursive/root.txt", "Root level file"); fixture.createFile("recursive/subdir/nested.txt", "Nested file content"); fixture.createFile("recursive/subdir/another.md", "# Nested markdown"); AddDirectoryRequest request; request.directoryPath = (fixture.testDir_ / "recursive").string(); request.recursive = true; request.includePatterns = {"*.txt", "*.md"}; auto result = fixture.indexingService_->addDirectory(request); REQUIRE(result); CHECK(result.value().filesIndexed == 3); CHECK(result.value().filesSkipped == 0); } SECTION("Handle non-existent directory") { auto request = fixture.createRequest(fixture.testDir_ / "does_not_exist"); auto result = fixture.indexingService_->addDirectory(request); REQUIRE_FALSE(result); CHECK(result.error().code == ErrorCode::InvalidArgument); CHECK(result.error().message.find("does not exist") != std::string::npos); } SECTION("Handle empty directory") { std::filesystem::create_directories(fixture.testDir_ / "empty"); auto request = fixture.createRequest(fixture.testDir_ / "empty"); auto result = fixture.indexingService_->addDirectory(request); REQUIRE(result); CHECK(result.value().filesIndexed == 0); CHECK(result.value().filesProcessed == 0); } } TEST_CASE("IndexingService defers content indexing when post-ingest is available", "[indexing][service][post-ingest]") { IndexingFixture fixture; std::vector> dispatchedBatches; fixture.appContext_.enqueuePostIngestBatch = [&](const std::vector& hashes, const std::string&) { dispatchedBatches.push_back(hashes); }; fixture.indexingService_ = makeIndexingService(fixture.appContext_); constexpr std::size_t kDocumentCount = 5; for (std::size_t i = 0; i < kDocumentCount; ++i) { fixture.createFile("deferred/content-" + std::to_string(i) + ".txt", "deferred indexing content " + std::to_string(i)); } auto request = fixture.createRequest(fixture.testDir_ / "deferred"); auto result = fixture.indexingService_->addDirectory(request); REQUIRE(result); REQUIRE(result.value().filesIndexed == kDocumentCount); REQUIRE(dispatchedBatches.size() == 1); REQUIRE(dispatchedBatches.front().size() == kDocumentCount); for (const auto& hash : dispatchedBatches.front()) { auto document = fixture.metadataRepo_->getDocumentByHash(hash); REQUIRE(document); REQUIRE(document.value().has_value()); CHECK_FALSE(document.value()->contentExtracted); } } TEST_CASE("IndexingService submits directory content through the batch boundary", "[indexing][service][batch]") { IndexingFixture fixture; constexpr std::size_t kDocumentCount = 33; for (std::size_t i = 0; i < kDocumentCount; ++i) { fixture.createFile("batched/file-" + std::to_string(i) + ".txt", "batched content " + std::to_string(i)); } yams::api::resetContentStorePhaseTimings(); auto request = fixture.createRequest(fixture.testDir_ / "batched"); auto result = fixture.indexingService_->addDirectory(request); REQUIRE(result.has_value()); CHECK(result.value().filesIndexed == kDocumentCount); CHECK(result.value().filesFailed == 0); const auto timings = yams::api::getContentStorePhaseTimingsSnapshot(); REQUIRE(timings.contains("batch_ref_commit")); CHECK(timings.at("batch_ref_commit").calls == 2); CHECK(timings.at("ref_commit").calls == 0); } TEST_CASE("IndexingService - File Handling", "[indexing][service][files]") { IndexingFixture fixture; SECTION("Handle binary files") { fixture.createBinaryFile("binaries/binary.exe"); auto request = fixture.createRequest(fixture.testDir_ / "binaries"); auto result = fixture.indexingService_->addDirectory(request); REQUIRE(result); CHECK(result.value().filesIndexed >= 0); } SECTION("Apply collection and metadata") { fixture.createFile("tagged/tagged.txt", "Content with tags and metadata"); AddDirectoryRequest request; request.directoryPath = (fixture.testDir_ / "tagged").string(); request.collection = "important-collection"; request.metadata = { {"project", "test-project"}, {"author", "test-author"}, {"priority", "high"}}; auto result = fixture.indexingService_->addDirectory(request); REQUIRE(result); CHECK(result.value().filesIndexed == 1); CHECK(result.value().collection == "important-collection"); } SECTION("Handle duplicate content") { constexpr auto content = "Identical content for deduplication test"; fixture.createFile("duplicates/file1.txt", content); fixture.createFile("duplicates/file2.txt", content); auto request = fixture.createRequest(fixture.testDir_ / "duplicates"); auto result = fixture.indexingService_->addDirectory(request); REQUIRE(result); CHECK(result.value().filesProcessed == 2); CHECK(result.value().filesIndexed == 2); } SECTION("Handle large file") { std::string largeContent(1024 * 1024, 'A'); // 1MB of 'A' characters fixture.createFile("large/huge.txt", largeContent); auto request = fixture.createRequest(fixture.testDir_ / "large"); auto result = fixture.indexingService_->addDirectory(request); REQUIRE(result); CHECK(result.value().filesIndexed == 1); } SECTION("Handle special characters in filenames") { fixture.createFile("special/file with spaces.txt", "Content with spaces"); fixture.createFile("special/file-with-dashes.txt", "Content with dashes"); fixture.createFile("special/file_with_underscores.txt", "Content with underscores"); auto request = fixture.createRequest(fixture.testDir_ / "special"); auto result = fixture.indexingService_->addDirectory(request); REQUIRE(result); CHECK(result.value().filesIndexed == 3); } } TEST_CASE("IndexingService - Symlink Handling", "[indexing][service][symlinks]") { IndexingFixture fixture; SECTION("Handle circular symlink") { fixture.createFile("file.txt", "content"); auto circular = fixture.testDir_ / "circular.txt"; std::error_code ec; std::filesystem::create_symlink("circular.txt", circular, ec); auto request = fixture.createRequest(fixture.testDir_); auto result = fixture.indexingService_->addDirectory(request); REQUIRE(result); CHECK(result.value().filesIndexed == 1); CHECK(result.value().filesFailed == 0); } SECTION("Handle broken symlink") { auto broken = fixture.testDir_ / "broken.txt"; std::error_code ec; std::filesystem::create_symlink("nonexistent.txt", broken, ec); auto request = fixture.createRequest(fixture.testDir_); auto result = fixture.indexingService_->addDirectory(request); REQUIRE(result); CHECK(result.value().filesIndexed == 0); } SECTION("Handle valid symlink to file") { fixture.createFile("target.txt", "target content"); auto link = fixture.testDir_ / "link.txt"; std::error_code ec; std::filesystem::create_symlink("target.txt", link, ec); auto request = fixture.createRequest(fixture.testDir_); auto result = fixture.indexingService_->addDirectory(request); REQUIRE(result); CHECK(result.value().filesIndexed >= 1); } SECTION("Handle nested directory with circular symlink") { std::filesystem::create_directories(fixture.testDir_ / "nested"); fixture.createFile("nested/file.txt", "content"); auto circular = fixture.testDir_ / "nested/loop"; std::error_code ec; std::filesystem::create_directory_symlink("..", circular, ec); AddDirectoryRequest request; request.directoryPath = fixture.testDir_.string(); request.recursive = true; auto result = fixture.indexingService_->addDirectory(request); REQUIRE(result); CHECK(result.value().filesIndexed >= 1); } SECTION("Handle symlink to directory") { std::filesystem::create_directories(fixture.testDir_ / "target_dir"); fixture.createFile("target_dir/file.txt", "content"); auto link = fixture.testDir_ / "link_dir"; std::error_code ec; std::filesystem::create_directory_symlink("target_dir", link, ec); AddDirectoryRequest request; request.directoryPath = fixture.testDir_.string(); request.recursive = true; auto result = fixture.indexingService_->addDirectory(request); REQUIRE(result); CHECK(result.value().filesIndexed >= 1); } SECTION("Handle symlink with special characters") { fixture.createFile("target.txt", "content"); auto link = fixture.testDir_ / "link with spaces & special.txt"; std::error_code ec; std::filesystem::create_symlink("target.txt", link, ec); auto request = fixture.createRequest(fixture.testDir_); auto result = fixture.indexingService_->addDirectory(request); REQUIRE(result); CHECK(result.value().filesIndexed >= 1); } } TEST_CASE("IndexingService - Pattern Matching", "[indexing][service][patterns]") { IndexingFixture fixture; SECTION("Include pattern with wildcards") { fixture.createFile("file.txt", "text file"); fixture.createFile("file.md", "markdown file"); fixture.createFile("file.cpp", "c++ file"); AddDirectoryRequest request; request.directoryPath = fixture.testDir_.string(); request.includePatterns = {"*.txt", "*.md"}; auto result = fixture.indexingService_->addDirectory(request); REQUIRE(result); CHECK(result.value().filesIndexed == 2); } SECTION("Include multiple patterns") { fixture.createFile("doc.txt", "text"); fixture.createFile("readme.md", "markdown"); fixture.createFile("script.py", "python"); fixture.createFile("code.cpp", "c++"); AddDirectoryRequest request; request.directoryPath = fixture.testDir_.string(); request.includePatterns = {"*.txt", "*.md", "*.py"}; auto result = fixture.indexingService_->addDirectory(request); REQUIRE(result); CHECK(result.value().filesIndexed == 3); } SECTION("Exclude pattern overrides include") { fixture.createFile("file1.txt", "text1"); fixture.createFile("file2.txt", "text2"); fixture.createFile("exclude_me.txt", "excluded"); AddDirectoryRequest request; request.directoryPath = fixture.testDir_.string(); request.includePatterns = {"*.txt"}; request.excludePatterns = {"exclude_*"}; auto result = fixture.indexingService_->addDirectory(request); REQUIRE(result); CHECK(result.value().filesIndexed == 2); } SECTION("No include patterns indexes all") { fixture.createFile("file.txt", "text"); fixture.createFile("file.md", "markdown"); fixture.createFile("file.cpp", "c++"); auto request = fixture.createRequest(fixture.testDir_); auto result = fixture.indexingService_->addDirectory(request); REQUIRE(result); CHECK(result.value().filesIndexed == 3); } SECTION("Pattern with question mark wildcard") { fixture.createFile("file1.txt", "1"); fixture.createFile("file2.txt", "2"); fixture.createFile("file10.txt", "10"); AddDirectoryRequest request; request.directoryPath = fixture.testDir_.string(); request.includePatterns = {"file?.txt"}; auto result = fixture.indexingService_->addDirectory(request); REQUIRE(result); CHECK(result.value().filesIndexed == 2); // Matches file1.txt and file2.txt only } SECTION("Case insensitive pattern matching") { fixture.createFile("file.txt", "content1"); fixture.createFile("readme.txt", "content2"); AddDirectoryRequest request; request.directoryPath = fixture.testDir_.string(); request.includePatterns = {"*.txt"}; auto result = fixture.indexingService_->addDirectory(request); REQUIRE(result); CHECK(result.value().filesIndexed == 2); } } TEST_CASE("IndexingService - Recursion and Directory Traversal", "[indexing][service][recursion]") { IndexingFixture fixture; SECTION("Non-recursive skips subdirectories") { fixture.createFile("root.txt", "root"); fixture.createFile("subdir/nested.txt", "nested"); AddDirectoryRequest request; request.directoryPath = fixture.testDir_.string(); request.recursive = false; auto result = fixture.indexingService_->addDirectory(request); REQUIRE(result); CHECK(result.value().filesIndexed == 1); } SECTION("Recursive indexes nested directories") { fixture.createFile("root.txt", "root"); fixture.createFile("sub1/file1.txt", "file1"); fixture.createFile("sub1/sub2/file2.txt", "file2"); AddDirectoryRequest request; request.directoryPath = fixture.testDir_.string(); request.recursive = true; auto result = fixture.indexingService_->addDirectory(request); REQUIRE(result); CHECK(result.value().filesIndexed == 3); } SECTION("Recursive with patterns in nested dirs") { fixture.createFile("sub1/file.txt", "txt"); fixture.createFile("sub1/file.md", "md"); fixture.createFile("sub2/file.txt", "txt2"); fixture.createFile("sub2/file.cpp", "cpp"); AddDirectoryRequest request; request.directoryPath = fixture.testDir_.string(); request.recursive = true; request.includePatterns = {"*.txt"}; auto result = fixture.indexingService_->addDirectory(request); REQUIRE(result); CHECK(result.value().filesIndexed == 2); } SECTION("Recursive indexing honors root gitignore") { fixture.createFile(".gitignore", "ignored/\n"); fixture.createFile("ignored/duplicate.cpp", "ignored"); fixture.createFile("src/visible.cpp", "visible"); AddDirectoryRequest request; request.directoryPath = fixture.testDir_.string(); request.recursive = true; request.includePatterns = {"*.cpp"}; auto result = fixture.indexingService_->addDirectory(request); REQUIRE(result); CHECK(result.value().filesIndexed == 1); } SECTION("Directory with only subdirectories") { std::filesystem::create_directories(fixture.testDir_ / "sub1"); std::filesystem::create_directories(fixture.testDir_ / "sub2"); auto request = fixture.createRequest(fixture.testDir_); auto result = fixture.indexingService_->addDirectory(request); REQUIRE(result); CHECK(result.value().filesIndexed == 0); } SECTION("Deep directory hierarchy") { fixture.createFile("l1/l2/l3/l4/l5/deep.txt", "deep content"); AddDirectoryRequest request; request.directoryPath = fixture.testDir_.string(); request.recursive = true; auto result = fixture.indexingService_->addDirectory(request); REQUIRE(result); CHECK(result.value().filesIndexed == 1); } } TEST_CASE("IndexingService - Metadata and Collections", "[indexing][service][metadata]") { IndexingFixture fixture; SECTION("Tags propagated to all files") { fixture.createFile("file1.txt", "content1"); fixture.createFile("file2.txt", "content2"); AddDirectoryRequest request; request.directoryPath = fixture.testDir_.string(); request.tags = {"important", "project-x"}; auto result = fixture.indexingService_->addDirectory(request); REQUIRE(result); CHECK(result.value().filesIndexed == 2); } SECTION("Collection assigned to all files") { fixture.createFile("file1.txt", "content1"); fixture.createFile("file2.txt", "content2"); AddDirectoryRequest request; request.directoryPath = fixture.testDir_.string(); request.collection = "my-collection"; auto result = fixture.indexingService_->addDirectory(request); REQUIRE(result); CHECK(result.value().filesIndexed == 2); CHECK(result.value().collection == "my-collection"); } SECTION("Metadata propagated to files") { fixture.createFile("file.txt", "content"); AddDirectoryRequest request; request.directoryPath = fixture.testDir_.string(); request.metadata = {{"project", "test"}, {"version", "1.0"}, {"author", "tester"}}; auto result = fixture.indexingService_->addDirectory(request); REQUIRE(result); CHECK(result.value().filesIndexed == 1); } } TEST_CASE("IndexingService - Response Validation", "[indexing][service][response]") { IndexingFixture fixture; SECTION("Response contains correct directory path") { fixture.createFile("file.txt", "content"); auto request = fixture.createRequest(fixture.testDir_); auto result = fixture.indexingService_->addDirectory(request); REQUIRE(result); CHECK(result.value().directoryPath == fixture.testDir_.string()); } SECTION("Response contains collection name") { fixture.createFile("file.txt", "content"); AddDirectoryRequest request; request.directoryPath = fixture.testDir_.string(); request.collection = "test-collection"; auto result = fixture.indexingService_->addDirectory(request); REQUIRE(result); CHECK(result.value().collection == "test-collection"); } SECTION("Individual file results populated") { fixture.createFile("file1.txt", "content1"); fixture.createFile("file2.txt", "content2"); auto request = fixture.createRequest(fixture.testDir_); auto result = fixture.indexingService_->addDirectory(request); REQUIRE(result); CHECK_FALSE(result.value().results.empty()); CHECK(result.value().results.size() == 2); } }