#include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "tests/common/test_helpers_catch2.h" using namespace yams; using namespace yams::app::services; using namespace yams::metadata; using namespace yams::api; struct DocumentFixture { DocumentFixture() { setupTestEnvironment(); setupDatabase(); setupServices(); setupTestDocuments(); } ~DocumentFixture() { documentService_.reset(); searchEngine_.reset(); contentStore_.reset(); // Clear all shared_ptr refs to metadataRepo BEFORE pool/database // destruction. appContext_ holds its own copy of the shared_ptr, // and C++ destroys members in reverse declaration order — but // appContext_ is declared AFTER pool_, so pool_ would be destroyed // while appContext_ still owns a MetadataRepository that references it. metadataRepo_.reset(); appContext_ = {}; // drops shared_ptr copies held by AppContext if (pool_) { pool_->shutdown(); pool_.reset(); } if (database_) { database_->close(); database_.reset(); } if (!testDir_.empty() && std::filesystem::exists(testDir_)) { std::error_code ec; std::filesystem::remove_all(testDir_, ec); } } void setupTestEnvironment() { auto pid = std::to_string(::getpid()); auto timestamp = std::to_string(std::chrono::system_clock::now().time_since_epoch().count()); testDir_ = std::filesystem::temp_directory_path() / ("document_service_test_" + pid + "_" + timestamp); std::error_code ec; std::filesystem::create_directories(testDir_, ec); REQUIRE_FALSE(ec); } void setupDatabase() { dbPath_ = std::filesystem::absolute(testDir_ / "test.db"); database_ = std::make_unique(); auto openResult = database_->open(dbPath_.string(), ConnectionMode::Create); REQUIRE(openResult); ConnectionPoolConfig poolConfig; poolConfig.maxConnections = 4; pool_ = std::make_unique(dbPath_.string(), poolConfig); metadataRepo_ = std::make_shared(*pool_); MigrationManager mm(*database_); auto initResult = mm.initialize(); REQUIRE(initResult); mm.registerMigrations(YamsMetadataMigrations::getAllMigrations()); auto migrateResult = mm.migrate(); REQUIRE(migrateResult); } void setupServices() { ContentStoreBuilder builder; auto storeResult = builder.withStoragePath(testDir_ / "storage") .withChunkSize(65536) .withCompression(true) .withDeduplication(true) .build(); REQUIRE(storeResult); auto& uniqueStore = storeResult.value(); contentStore_ = std::shared_ptr( const_cast&>(uniqueStore).release()); searchEngine_ = nullptr; appContext_.store = contentStore_; appContext_.metadataRepo = metadataRepo_; appContext_.searchEngine = searchEngine_; documentService_ = makeDocumentService(appContext_); } void setupTestDocuments() { auto testFile1 = testDir_ / "test1.txt"; auto testFile2 = testDir_ / "test2.md"; auto testFile3 = testDir_ / "libsample.so"; std::ofstream{testFile1} << "Test document content for retrieval testing"; std::ofstream{testFile2} << "Another test document with different content"; { std::ofstream binary(testFile3, std::ios::binary); const unsigned char elfBytes[] = {0x7F, 0x45, 0x4C, 0x46, 0x02, 0x01, 0x01, 0x00}; binary.write(reinterpret_cast(elfBytes), sizeof(elfBytes)); } StoreDocumentRequest storeReq1; storeReq1.path = testFile1.string(); storeReq1.tags = {"test", "document"}; StoreDocumentRequest storeReq2; storeReq2.path = testFile2.string(); storeReq2.tags = {"test", "markdown"}; StoreDocumentRequest storeReq3; storeReq3.path = testFile3.string(); storeReq3.tags = {"test", "binary"}; auto result1 = documentService_->store(storeReq1); auto result2 = documentService_->store(storeReq2); auto result3 = documentService_->store(storeReq3); REQUIRE(result1); REQUIRE(result2); REQUIRE(result3); testHash1_ = result1.value().hash; testHash2_ = result2.value().hash; } std::string createTestContent(std::string_view identifier) { return std::string("Test content for document: ") + std::string(identifier) + " created at " + std::to_string(std::chrono::system_clock::now().time_since_epoch().count()); } std::filesystem::path testDir_; std::filesystem::path dbPath_; std::unique_ptr database_; std::unique_ptr pool_; std::shared_ptr metadataRepo_; std::shared_ptr contentStore_; std::shared_ptr searchEngine_; AppContext appContext_; std::shared_ptr documentService_; std::string testHash1_; std::string testHash2_; }; class RemoveFailingContentStore : public IContentStore { public: RemoveFailingContentStore(std::shared_ptr inner, std::string failHash, Error error) : inner_(std::move(inner)), failHash_(std::move(failHash)), error_(std::move(error)) {} Result store(const std::filesystem::path& path, const ContentMetadata& metadata, yams::api::ProgressCallback progress) override { return inner_->store(path, metadata, progress); } Result retrieve(const std::string& hash, const std::filesystem::path& outputPath, yams::api::ProgressCallback progress) override { return inner_->retrieve(hash, outputPath, progress); } Result storeStream(std::istream& stream, const ContentMetadata& metadata, yams::api::ProgressCallback progress) override { return inner_->storeStream(stream, metadata, progress); } Result retrieveStream(const std::string& hash, std::ostream& output, yams::api::ProgressCallback progress) override { return inner_->retrieveStream(hash, output, progress); } Result storeBytes(std::span data, const ContentMetadata& metadata) override { return inner_->storeBytes(data, metadata); } Result> retrieveBytes(const std::string& hash) override { return inner_->retrieveBytes(hash); } Result> retrieveBytesPrefix(const std::string& hash, std::size_t maxBytes) override { return inner_->retrieveBytesPrefix(hash, maxBytes); } Result retrieveRaw(const std::string& hash) override { return inner_->retrieveRaw(hash); } std::future> retrieveRawAsync(const std::string& hash) override { return inner_->retrieveRawAsync(hash); } Result exists(const std::string& hash) const override { return inner_->exists(hash); } Result remove(const std::string& hash) override { if (hash == failHash_) { return error_; } return inner_->remove(hash); } Result getMetadata(const std::string& hash) const override { return inner_->getMetadata(hash); } Result updateMetadata(const std::string& hash, const ContentMetadata& metadata) override { return inner_->updateMetadata(hash, metadata); } std::vector> storeBatch(const std::vector& paths, const std::vector& metadata) override { return inner_->storeBatch(paths, metadata); } std::vector> removeBatch(const std::vector& hashes) override { return inner_->removeBatch(hashes); } ContentStoreStats getStats() const override { return inner_->getStats(); } HealthStatus checkHealth() const override { return inner_->checkHealth(); } Result verify(yams::api::ProgressCallback progress) override { return inner_->verify(progress); } Result compact(yams::api::ProgressCallback progress) override { return inner_->compact(progress); } Result garbageCollect(yams::api::ProgressCallback progress) override { return inner_->garbageCollect(progress); } private: std::shared_ptr inner_; std::string failHash_; Error error_; }; TEST_CASE("DocumentService - Listing", "[document][service][listing]") { DocumentFixture fixture; SECTION("List all documents") { ListDocumentsRequest request; request.limit = 100; auto result = fixture.documentService_->list(request); REQUIRE(result); CHECK(result.value().documents.size() >= 2); } SECTION("List with limit") { ListDocumentsRequest request; request.limit = 1; auto result = fixture.documentService_->list(request); REQUIRE(result); CHECK(result.value().documents.size() == 1); } SECTION("List with offset") { ListDocumentsRequest request; request.limit = 100; request.offset = 1; auto result = fixture.documentService_->list(request); REQUIRE(result); CHECK(result.value().documents.size() >= 1); } SECTION("List bare directory returns descendants") { namespace fs = std::filesystem; auto normalizedPathString = [](const fs::path& path) { std::error_code ec; auto resolved = fs::weakly_canonical(path, ec); if (ec) { resolved = path.lexically_normal(); } auto normalized = resolved.generic_string(); #ifdef _WIN32 std::transform(normalized.begin(), normalized.end(), normalized.begin(), [](unsigned char c) { return static_cast(std::tolower(c)); }); #endif return normalized; }; fs::create_directories(fixture.testDir_ / "nested" / "deep"); std::ofstream(fixture.testDir_ / "nested" / "deep" / "descendant.txt") << "descendant content"; StoreDocumentRequest nestedReq; nestedReq.path = (fixture.testDir_ / "nested" / "deep" / "descendant.txt").string(); nestedReq.tags = {"nested"}; auto nestedStore = fixture.documentService_->store(nestedReq); REQUIRE(nestedStore); const auto expectedPath = normalizedPathString(fixture.testDir_ / "nested" / "deep" / "descendant.txt"); ListDocumentsRequest request; request.pattern = (fixture.testDir_ / "nested").string(); request.limit = 100; auto result = fixture.documentService_->list(request); REQUIRE(result); bool foundDescendant = false; for (const auto& doc : result.value().documents) { if (normalizedPathString(doc.path) == expectedPath) { foundDescendant = true; break; } } CHECK(foundDescendant); } SECTION("List with tag filter") { ListDocumentsRequest request; request.limit = 100; request.tags = {"markdown"}; auto result = fixture.documentService_->list(request); REQUIRE(result); bool foundMarkdown = false; for (const auto& doc : result.value().documents) { if (std::find(doc.tags.begin(), doc.tags.end(), "markdown") != doc.tags.end()) { foundMarkdown = true; break; } } CHECK(foundMarkdown); } SECTION("List with type filter") { ListDocumentsRequest request; request.limit = 100; request.type = "text"; auto result = fixture.documentService_->list(request); REQUIRE(result); } SECTION("List with executable type filter") { ListDocumentsRequest request; request.limit = 100; request.type = "executable"; auto result = fixture.documentService_->list(request); REQUIRE(result); REQUIRE_FALSE(result.value().documents.empty()); bool foundSharedObject = false; for (const auto& doc : result.value().documents) { CHECK(doc.fileType == "executable"); if (doc.fileName == "libsample.so") { foundSharedObject = true; CHECK(doc.mimeType == "application/x-sharedlib"); } } CHECK(foundSharedObject); } SECTION("List recent documents") { ListDocumentsRequest request; request.limit = 5; request.recent = 5; auto result = fixture.documentService_->list(request); REQUIRE(result); CHECK(result.value().documents.size() >= 2); } } TEST_CASE("DocumentService - Retrieval", "[document][service][retrieval]") { DocumentFixture fixture; SECTION("Retrieve document by hash") { REQUIRE_FALSE(fixture.testHash1_.empty()); RetrieveDocumentRequest request; request.hash = fixture.testHash1_; auto result = fixture.documentService_->retrieve(request); REQUIRE(result); REQUIRE(result.value().document.has_value()); CHECK(result.value().document->hash == fixture.testHash1_); } SECTION("Retrieve with compressed content and metadata") { REQUIRE_FALSE(fixture.testHash1_.empty()); RetrieveDocumentRequest request; request.hash = fixture.testHash1_; request.includeContent = true; auto result = fixture.documentService_->retrieve(request); REQUIRE(result); REQUIRE(result.value().document.has_value()); CHECK(result.value().document->hash == fixture.testHash1_); CHECK(result.value().document->content.has_value()); } SECTION("Retrieve document by name") { RetrieveDocumentRequest request; request.name = (fixture.testDir_ / "test1.txt").string(); auto result = fixture.documentService_->retrieve(request); REQUIRE(result); REQUIRE(result.value().document.has_value()); CHECK_FALSE(result.value().document->hash.empty()); } SECTION("Retrieve non-existent document") { RetrieveDocumentRequest request; request.hash = "nonexistent_hash_12345"; auto result = fixture.documentService_->retrieve(request); CHECK_FALSE(result); } } TEST_CASE("DocumentService - Name Resolution", "[document][service][resolve]") { DocumentFixture fixture; SECTION("Resolve name to hash via tree lookup") { auto path = (fixture.testDir_ / "test1.txt").string(); RetrieveDocumentRequest request; request.name = path; auto result = fixture.documentService_->retrieve(request); REQUIRE(result); REQUIRE(result.value().document.has_value()); CHECK_FALSE(result.value().document->hash.empty()); } SECTION("Resolve by hash directly") { REQUIRE_FALSE(fixture.testHash1_.empty()); RetrieveDocumentRequest request; request.hash = fixture.testHash1_; auto result = fixture.documentService_->retrieve(request); REQUIRE(result); REQUIRE(result.value().document.has_value()); CHECK(result.value().document->hash == fixture.testHash1_); } SECTION("Resolve path suffix") { RetrieveDocumentRequest request; request.name = "test1.txt"; auto result = fixture.documentService_->retrieve(request); REQUIRE(result); REQUIRE(result.value().document.has_value()); CHECK_FALSE(result.value().document->hash.empty()); } SECTION("Resolve not found") { RetrieveDocumentRequest request; request.name = "nonexistent_file.txt"; auto result = fixture.documentService_->retrieve(request); CHECK_FALSE(result); } SECTION("Resolve with normalized path separators") { auto pathWithMixedSeparators = fixture.testDir_.string() + "/test1.txt"; RetrieveDocumentRequest request; request.name = pathWithMixedSeparators; auto result = fixture.documentService_->retrieve(request); REQUIRE(result); REQUIRE(result.value().document.has_value()); } SECTION("Resolve with empty name and hash") { RetrieveDocumentRequest request; auto result = fixture.documentService_->retrieve(request); CHECK_FALSE(result); } SECTION("Resolve relative path from cwd") { auto originalCwd = std::filesystem::current_path(); std::filesystem::current_path(fixture.testDir_); RetrieveDocumentRequest request; request.name = "test1.txt"; auto result = fixture.documentService_->retrieve(request); std::filesystem::current_path(originalCwd); REQUIRE(result); REQUIRE(result.value().document.has_value()); CHECK_FALSE(result.value().document->hash.empty()); } } TEST_CASE("DocumentService - Deletion", "[document][service][deletion]") { DocumentFixture fixture; SECTION("Delete document by name") { DeleteByNameRequest request; request.name = (fixture.testDir_ / "test2.md").string(); auto result = fixture.documentService_->deleteByName(request); REQUIRE(result); CHECK_FALSE(result.value().deleted.empty()); } SECTION("Delete document removes vector rows") { yams::test::ScopedEnvVar enableVectors("YAMS_DISABLE_VECTORS", std::optional("0")); yams::test::ScopedEnvVar enableSqliteVecInit("YAMS_SQLITE_VEC_SKIP_INIT", std::optional("0")); yams::vector::VectorDatabaseConfig config; config.database_path = ":memory:"; config.embedding_dim = 4; config.create_if_missing = true; config.use_in_memory = true; auto vectorDb = std::make_shared(config); if (!vectorDb->initialize()) { SKIP(std::string("Vector database unavailable: ") + vectorDb->getLastError()); } yams::vector::VectorRecord record; record.chunk_id = "delete_cleanup_chunk"; record.document_hash = fixture.testHash2_; record.embedding = {1.0f, 0.0f, 0.0f, 0.0f}; record.content = "vector cleanup content"; record.start_offset = 0; record.end_offset = 22; REQUIRE(vectorDb->insertVector(record)); REQUIRE(vectorDb->hasEmbedding(fixture.testHash2_)); fixture.appContext_.vectorDatabase = vectorDb; fixture.documentService_ = makeDocumentService(fixture.appContext_); DeleteByNameRequest request; request.name = (fixture.testDir_ / "test2.md").string(); auto result = fixture.documentService_->deleteByName(request); REQUIRE(result); CHECK_FALSE(result.value().deleted.empty()); CHECK_FALSE(vectorDb->hasEmbedding(fixture.testHash2_)); } SECTION("Delete document cleans orphan metadata when content is already missing") { auto removed = fixture.contentStore_->remove(fixture.testHash2_); REQUIRE(removed); auto before = fixture.metadataRepo_->getDocumentByHash(fixture.testHash2_); REQUIRE(before); REQUIRE(before.value().has_value()); DeleteByNameRequest request; request.name = (fixture.testDir_ / "test2.md").string(); auto result = fixture.documentService_->deleteByName(request); REQUIRE(result); CHECK(result.value().errors.empty()); REQUIRE(result.value().deleted.size() == 1); CHECK(result.value().deleted.front().deleted); auto after = fixture.metadataRepo_->getDocumentByHash(fixture.testHash2_); REQUIRE(after); CHECK_FALSE(after.value().has_value()); } SECTION("Delete document force-cleans metadata when store removal reports corruption") { auto failingStore = std::make_shared( fixture.contentStore_, fixture.testHash2_, Error{ErrorCode::IOError, "Corrupted data while removing object"}); fixture.appContext_.store = failingStore; fixture.documentService_ = makeDocumentService(fixture.appContext_); DeleteByNameRequest request; request.name = (fixture.testDir_ / "test2.md").string(); auto failed = fixture.documentService_->deleteByName(request); REQUIRE(failed); CHECK(failed.value().deleted.empty()); REQUIRE(failed.value().errors.size() == 1); auto stillPresent = fixture.metadataRepo_->getDocumentByHash(fixture.testHash2_); REQUIRE(stillPresent); REQUIRE(stillPresent.value().has_value()); request.force = true; auto forced = fixture.documentService_->deleteByName(request); REQUIRE(forced); CHECK(forced.value().errors.empty()); REQUIRE(forced.value().deleted.size() == 1); CHECK(forced.value().deleted.front().deleted); auto after = fixture.metadataRepo_->getDocumentByHash(fixture.testHash2_); REQUIRE(after); CHECK_FALSE(after.value().has_value()); } SECTION("Delete document by raw hash removes content even without metadata") { auto rawPath = fixture.testDir_ / "raw-content.txt"; std::ofstream{rawPath} << "raw content without metadata"; auto stored = fixture.contentStore_->store(rawPath, ContentMetadata{}); REQUIRE(stored); const auto hash = stored.value().contentHash; auto existsBefore = fixture.contentStore_->exists(hash); REQUIRE(existsBefore); REQUIRE(existsBefore.value()); DeleteByNameRequest request; request.hash = hash; auto result = fixture.documentService_->deleteByName(request); REQUIRE(result); CHECK(result.value().errors.empty()); REQUIRE(result.value().deleted.size() == 1); CHECK(result.value().deleted.front().deleted); auto removedAgain = fixture.contentStore_->remove(hash); REQUIRE(removedAgain); CHECK_FALSE(removedAgain.value()); } } TEST_CASE("DocumentService - Error Handling", "[document][service][errors]") { DocumentFixture fixture; SECTION("Emit helpful message on file not found with commas") { class NotFoundStore : public IContentStore { public: Result store(const std::filesystem::path& path, const ContentMetadata&, api::ProgressCallback) override { return Error{ErrorCode::FileNotFound, std::string("File not found: ") + path.string()}; } Result retrieve(const std::string&, const std::filesystem::path&, api::ProgressCallback) override { return ErrorCode::NotImplemented; } Result storeStream(std::istream&, const ContentMetadata&, api::ProgressCallback) override { return ErrorCode::NotImplemented; } Result retrieveStream(const std::string&, std::ostream&, api::ProgressCallback) override { return ErrorCode::NotImplemented; } Result storeBytes(std::span, const ContentMetadata&) override { return ErrorCode::NotImplemented; } Result> retrieveBytes(const std::string&) override { return ErrorCode::NotImplemented; } Result> retrieveBytesPrefix(const std::string&, std::size_t) override { return ErrorCode::NotImplemented; } Result retrieveRaw(const std::string&) override { return ErrorCode::NotImplemented; } std::future> retrieveRawAsync(const std::string&) override { return std::async(std::launch::deferred, [] { return Result(ErrorCode::NotImplemented); }); } Result exists(const std::string&) const override { return ErrorCode::NotImplemented; } Result remove(const std::string&) override { return ErrorCode::NotImplemented; } Result getMetadata(const std::string&) const override { return ErrorCode::NotImplemented; } Result updateMetadata(const std::string&, const ContentMetadata&) override { return ErrorCode::NotImplemented; } std::vector> storeBatch(const std::vector&, const std::vector&) override { return {}; } std::vector> removeBatch(const std::vector&) override { return {}; } ContentStoreStats getStats() const override { return {}; } HealthStatus checkHealth() const override { return {}; } Result verify(api::ProgressCallback) override { return ErrorCode::NotImplemented; } Result compact(api::ProgressCallback) override { return ErrorCode::NotImplemented; } Result garbageCollect(api::ProgressCallback) override { return ErrorCode::NotImplemented; } }; AppContext ctx; ctx.store = std::make_shared(); auto svc = makeDocumentService(ctx); StoreDocumentRequest req; req.path = "a,b,c"; auto r = svc->store(req); REQUIRE_FALSE(r); CHECK((r.error().code == ErrorCode::FileNotFound)); auto msg = r.error().message; CHECK((msg.find("a,b,c") != std::string::npos)); CHECK((msg.find("commas") != std::string::npos)); } } TEST_CASE("DocumentService - Performance", "[document][service][performance]") { DocumentFixture fixture; SECTION("List performance") { auto start = std::chrono::steady_clock::now(); ListDocumentsRequest request; request.limit = 100; auto result = fixture.documentService_->list(request); auto end = std::chrono::steady_clock::now(); auto duration = std::chrono::duration_cast(end - start); REQUIRE(result); CHECK((duration.count() < 1000)); } SECTION("Retrieval performance") { REQUIRE_FALSE(fixture.testHash1_.empty()); auto start = std::chrono::steady_clock::now(); RetrieveDocumentRequest request; request.hash = fixture.testHash1_; auto result = fixture.documentService_->retrieve(request); auto end = std::chrono::steady_clock::now(); auto duration = std::chrono::duration_cast(end - start); REQUIRE(result); CHECK((duration.count() < 1000)); } } TEST_CASE("DocumentService - Integration", "[document][service][integration]") { DocumentFixture fixture; SECTION("Search integration") { ListDocumentsRequest request; request.limit = 100; auto result = fixture.documentService_->list(request); REQUIRE(result); CHECK((result.value().documents.size() >= 2)); } SECTION("Service context integration") { REQUIRE(fixture.documentService_); CHECK(fixture.appContext_.store); CHECK(fixture.appContext_.metadataRepo); } SECTION("Inline extensionless binary content is not forced to text") { StoreDocumentRequest request; request.name = "88f051b8c01feda0b07cc00112233445"; request.content = "?<store(request); REQUIRE(stored); auto doc = fixture.metadataRepo_->getDocumentByHash(stored.value().hash); REQUIRE(doc); REQUIRE(doc.value().has_value()); CHECK((doc.value()->mimeType == "application/octet-stream")); CHECK_FALSE(doc.value()->contentExtracted); } }