trevon/yams
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 2
yams/tests/unit/app/services/indexing_service_test.cpp
trvon 4df44e0c2f
 fix(tests): use portable getpid shim
2026-05-20 15:54:13 -04:00

690 lines
24 KiB
C++
Raw Permalink Blame History

#include <catch2/catch_test_macros.hpp>
#include <yams/api/content_store_builder.h>
#include <yams/app/services/services.hpp>
#include <yams/compat/unistd.h>
#include <yams/content/content_handler_registry.h>
#include <yams/core/cpp23_features.hpp>
#include <yams/detection/file_type_detector.h>
#include <yams/metadata/connection_pool.h>
#include <yams/metadata/database.h>
#include <yams/metadata/knowledge_graph_store.h>
#include <yams/metadata/metadata_repository.h>
#include <yams/metadata/migration.h>
#include <yams/search/search_engine.h>
#include <chrono>
#include <filesystem>
#include <fstream>
using namespace yams;
using namespace yams::app::services;
using namespace yams::metadata;
using namespace yams::content;
using namespace yams::detection;
using namespace yams::api;
namespace {
struct IndexingFixture {
IndexingFixture() {
initializeGlobalSingletons();
setupTestEnvironment();
setupDatabase();
setupServices();
}
~IndexingFixture() {
indexingService_.reset();
searchEngine_.reset();
appContext_.metadataRepo.reset();
appContext_.store.reset();
appContext_.searchEngine.reset();
contentStore_.reset();
metadataRepo_.reset();
if (pool_) {
pool_->shutdown();
pool_.reset();
}
if (database_) {
database_->close();
database_.reset();
}
std::error_code ec;
if (!testDir_.empty() && std::filesystem::exists(testDir_)) {
std::filesystem::remove_all(testDir_, ec);
}
if (!infraDir_.empty() && std::filesystem::exists(infraDir_)) {
std::filesystem::remove_all(infraDir_, ec);
}
}
void initializeGlobalSingletons() {
static bool initialized = false;
if (initialized)
return;
auto& detector = FileTypeDetector::instance();
auto detectorInit = detector.initializeWithMagicNumbers();
REQUIRE(detectorInit);
auto& registry = ContentHandlerRegistry::instance();
registry.clear();
initialized = true;
}
void setupTestEnvironment() {
auto pid = std::to_string(::getpid());
auto timestamp =
std::to_string(std::chrono::system_clock::now().time_since_epoch().count());
testDir_ =
std::filesystem::temp_directory_path() / ("indexing_test_" + pid + "_" + timestamp);
infraDir_ =
std::filesystem::temp_directory_path() / ("indexing_infra_" + pid + "_" + timestamp);
std::filesystem::create_directories(testDir_);
std::filesystem::create_directories(infraDir_);
}
void setupDatabase() {
dbPath_ = std::filesystem::absolute(infraDir_ / "test.db");
database_ = std::make_unique<Database>();
REQUIRE(database_->open(dbPath_.string(), ConnectionMode::Create));
pool_ = std::make_unique<ConnectionPool>(dbPath_.string(),
ConnectionPoolConfig{.maxConnections = 4});
metadataRepo_ = std::make_shared<MetadataRepository>(*pool_);
MigrationManager mm(*database_);
REQUIRE(mm.initialize());
mm.registerMigrations(YamsMetadataMigrations::getAllMigrations());
REQUIRE(mm.migrate());
}
void setupServices() {
ContentStoreBuilder builder;
auto storeResult = builder.withStoragePath(infraDir_ / "storage")
.withChunkSize(65536)
.withCompression(true)
.withDeduplication(true)
.build();
REQUIRE(storeResult);
contentStore_ = std::shared_ptr<IContentStore>(
const_cast<std::unique_ptr<IContentStore>&>(storeResult.value()).release());
KnowledgeGraphStoreConfig kgConfig;
auto kgResult = makeSqliteKnowledgeGraphStore(*pool_, kgConfig);
REQUIRE(kgResult.has_value());
kgStore_ = std::shared_ptr<KnowledgeGraphStore>(std::move(kgResult).value());
metadataRepo_->setKnowledgeGraphStore(kgStore_);
appContext_.service_manager = nullptr;
appContext_.store = contentStore_;
appContext_.metadataRepo = metadataRepo_;
appContext_.kgStore = kgStore_;
appContext_.searchEngine = nullptr;
indexingService_ = makeIndexingService(appContext_);
}
std::filesystem::path createFile(std::string_view filename, std::string_view content) {
auto filePath = testDir_ / filename;
std::filesystem::create_directories(filePath.parent_path());
std::ofstream{filePath} << content;
return filePath;
}
std::filesystem::path createBinaryFile(std::string_view filename, size_t size = 1000) {
auto filePath = testDir_ / filename;
std::filesystem::create_directories(filePath.parent_path());
std::ofstream file(filePath, std::ios::binary);
for (size_t i = 0; i < size; ++i) {
file.put(static_cast<char>(i % 256));
}
return filePath;
}
AddDirectoryRequest createRequest(const std::filesystem::path& path,
std::string_view collection = "") {
AddDirectoryRequest req;
req.directoryPath = path.string();
req.collection = std::string(collection);
req.recursive = false;
return req;
}
std::filesystem::path testDir_;
std::filesystem::path infraDir_;
std::filesystem::path dbPath_;
std::unique_ptr<Database> database_;
std::unique_ptr<ConnectionPool> pool_;
std::shared_ptr<MetadataRepository> metadataRepo_;
std::shared_ptr<KnowledgeGraphStore> kgStore_;
std::shared_ptr<IContentStore> contentStore_;
std::shared_ptr<search::SearchEngine> searchEngine_;
AppContext appContext_;
std::shared_ptr<IIndexingService> indexingService_;
};
} // namespace
TEST_CASE("IndexingService - Basic Operations", "[indexing][service][basic]") {
IndexingFixture fixture;
SECTION("Index single file in directory") {
fixture.createFile("docs/test.txt", "This is a test document with some content.");
auto request = fixture.createRequest(fixture.testDir_ / "docs", "test-collection");
auto result = fixture.indexingService_->addDirectory(request);
REQUIRE(result);
CHECK(result.value().filesIndexed == 1);
CHECK(result.value().filesSkipped == 0);
CHECK(result.value().filesFailed == 0);
CHECK(result.value().filesProcessed == 1);
}
SECTION("Indexing populates canonical doc and blob KG nodes") {
auto filePath = fixture.createFile("docs/kg.txt", "Knowledge graph ingest coverage.");
auto request = fixture.createRequest(fixture.testDir_ / "docs", "kg-collection");
auto result = fixture.indexingService_->addDirectory(request);
REQUIRE(result);
REQUIRE(result.value().filesIndexed == 1);
auto docInfo = fixture.metadataRepo_->findDocumentByExactPath(filePath.string());
REQUIRE(docInfo);
REQUIRE(docInfo.value().has_value());
const auto& document = *docInfo.value();
auto docNode = fixture.kgStore_->getNodeByKey("doc:" + document.sha256Hash);
REQUIRE(docNode);
REQUIRE(docNode.value().has_value());
CHECK(docNode.value()->type == std::optional<std::string>{"document"});
auto blobNode = fixture.kgStore_->getNodeByKey("blob:" + document.sha256Hash);
REQUIRE(blobNode);
REQUIRE(blobNode.value().has_value());
CHECK(blobNode.value()->type == std::optional<std::string>{"blob"});
auto edges = fixture.kgStore_->getEdgesFrom(docNode.value()->id, "has_blob", 10, 0);
REQUIRE(edges);
REQUIRE(edges.value().size() == 1);
CHECK(edges.value().front().dstNodeId == blobNode.value()->id);
}
SECTION("Index multiple files in directory") {
fixture.createFile("multi/file1.txt", "Content of first file");
fixture.createFile("multi/file2.md", "# Markdown content\n\nSome **bold** text");
fixture.createFile("multi/file3.cpp", "#include <iostream>\nint main() { return 0; }");
AddDirectoryRequest request;
request.directoryPath = (fixture.testDir_ / "multi").string();
request.collection = "test-batch";
request.recursive = false;
auto result = fixture.indexingService_->addDirectory(request);
REQUIRE(result);
CHECK(result.value().filesIndexed == 3);
CHECK(result.value().filesSkipped == 0);
CHECK(result.value().filesProcessed == 3);
}
SECTION("Index directory recursively") {
fixture.createFile("recursive/root.txt", "Root level file");
fixture.createFile("recursive/subdir/nested.txt", "Nested file content");
fixture.createFile("recursive/subdir/another.md", "# Nested markdown");
AddDirectoryRequest request;
request.directoryPath = (fixture.testDir_ / "recursive").string();
request.recursive = true;
request.includePatterns = {"*.txt", "*.md"};
auto result = fixture.indexingService_->addDirectory(request);
REQUIRE(result);
CHECK(result.value().filesIndexed == 3);
CHECK(result.value().filesSkipped == 0);
}
SECTION("Handle non-existent directory") {
auto request = fixture.createRequest(fixture.testDir_ / "does_not_exist");
auto result = fixture.indexingService_->addDirectory(request);
REQUIRE_FALSE(result);
CHECK(result.error().code == ErrorCode::InvalidArgument);
CHECK(result.error().message.find("does not exist") != std::string::npos);
}
SECTION("Handle empty directory") {
std::filesystem::create_directories(fixture.testDir_ / "empty");
auto request = fixture.createRequest(fixture.testDir_ / "empty");
auto result = fixture.indexingService_->addDirectory(request);
REQUIRE(result);
CHECK(result.value().filesIndexed == 0);
CHECK(result.value().filesProcessed == 0);
}
}
TEST_CASE("IndexingService - File Handling", "[indexing][service][files]") {
IndexingFixture fixture;
SECTION("Handle binary files") {
fixture.createBinaryFile("binaries/binary.exe");
auto request = fixture.createRequest(fixture.testDir_ / "binaries");
auto result = fixture.indexingService_->addDirectory(request);
REQUIRE(result);
CHECK(result.value().filesIndexed >= 0);
}
SECTION("Apply collection and metadata") {
fixture.createFile("tagged/tagged.txt", "Content with tags and metadata");
AddDirectoryRequest request;
request.directoryPath = (fixture.testDir_ / "tagged").string();
request.collection = "important-collection";
request.metadata = {
{"project", "test-project"}, {"author", "test-author"}, {"priority", "high"}};
auto result = fixture.indexingService_->addDirectory(request);
REQUIRE(result);
CHECK(result.value().filesIndexed == 1);
CHECK(result.value().collection == "important-collection");
}
SECTION("Handle duplicate content") {
constexpr auto content = "Identical content for deduplication test";
fixture.createFile("duplicates/file1.txt", content);
fixture.createFile("duplicates/file2.txt", content);
auto request = fixture.createRequest(fixture.testDir_ / "duplicates");
auto result = fixture.indexingService_->addDirectory(request);
REQUIRE(result);
CHECK(result.value().filesProcessed == 2);
CHECK(result.value().filesIndexed == 2);
}
SECTION("Handle large file") {
std::string largeContent(1024 * 1024, 'A'); // 1MB of 'A' characters
fixture.createFile("large/huge.txt", largeContent);
auto request = fixture.createRequest(fixture.testDir_ / "large");
auto result = fixture.indexingService_->addDirectory(request);
REQUIRE(result);
CHECK(result.value().filesIndexed == 1);
}
SECTION("Handle special characters in filenames") {
fixture.createFile("special/file with spaces.txt", "Content with spaces");
fixture.createFile("special/file-with-dashes.txt", "Content with dashes");
fixture.createFile("special/file_with_underscores.txt", "Content with underscores");
auto request = fixture.createRequest(fixture.testDir_ / "special");
auto result = fixture.indexingService_->addDirectory(request);
REQUIRE(result);
CHECK(result.value().filesIndexed == 3);
}
}
TEST_CASE("IndexingService - Symlink Handling", "[indexing][service][symlinks]") {
IndexingFixture fixture;
SECTION("Handle circular symlink") {
fixture.createFile("file.txt", "content");
auto circular = fixture.testDir_ / "circular.txt";
std::error_code ec;
std::filesystem::create_symlink("circular.txt", circular, ec);
auto request = fixture.createRequest(fixture.testDir_);
auto result = fixture.indexingService_->addDirectory(request);
REQUIRE(result);
CHECK(result.value().filesIndexed == 1);
CHECK(result.value().filesFailed == 0);
}
SECTION("Handle broken symlink") {
auto broken = fixture.testDir_ / "broken.txt";
std::error_code ec;
std::filesystem::create_symlink("nonexistent.txt", broken, ec);
auto request = fixture.createRequest(fixture.testDir_);
auto result = fixture.indexingService_->addDirectory(request);
REQUIRE(result);
CHECK(result.value().filesIndexed == 0);
}
SECTION("Handle valid symlink to file") {
fixture.createFile("target.txt", "target content");
auto link = fixture.testDir_ / "link.txt";
std::error_code ec;
std::filesystem::create_symlink("target.txt", link, ec);
auto request = fixture.createRequest(fixture.testDir_);
auto result = fixture.indexingService_->addDirectory(request);
REQUIRE(result);
CHECK(result.value().filesIndexed >= 1);
}
SECTION("Handle nested directory with circular symlink") {
std::filesystem::create_directories(fixture.testDir_ / "nested");
fixture.createFile("nested/file.txt", "content");
auto circular = fixture.testDir_ / "nested/loop";
std::error_code ec;
std::filesystem::create_directory_symlink("..", circular, ec);
AddDirectoryRequest request;
request.directoryPath = fixture.testDir_.string();
request.recursive = true;
auto result = fixture.indexingService_->addDirectory(request);
REQUIRE(result);
CHECK(result.value().filesIndexed >= 1);
}
SECTION("Handle symlink to directory") {
std::filesystem::create_directories(fixture.testDir_ / "target_dir");
fixture.createFile("target_dir/file.txt", "content");
auto link = fixture.testDir_ / "link_dir";
std::error_code ec;
std::filesystem::create_directory_symlink("target_dir", link, ec);
AddDirectoryRequest request;
request.directoryPath = fixture.testDir_.string();
request.recursive = true;
auto result = fixture.indexingService_->addDirectory(request);
REQUIRE(result);
CHECK(result.value().filesIndexed >= 1);
}
SECTION("Handle symlink with special characters") {
fixture.createFile("target.txt", "content");
auto link = fixture.testDir_ / "link with spaces & special.txt";
std::error_code ec;
std::filesystem::create_symlink("target.txt", link, ec);
auto request = fixture.createRequest(fixture.testDir_);
auto result = fixture.indexingService_->addDirectory(request);
REQUIRE(result);
CHECK(result.value().filesIndexed >= 1);
}
}
TEST_CASE("IndexingService - Pattern Matching", "[indexing][service][patterns]") {
IndexingFixture fixture;
SECTION("Include pattern with wildcards") {
fixture.createFile("file.txt", "text file");
fixture.createFile("file.md", "markdown file");
fixture.createFile("file.cpp", "c++ file");
AddDirectoryRequest request;
request.directoryPath = fixture.testDir_.string();
request.includePatterns = {"*.txt", "*.md"};
auto result = fixture.indexingService_->addDirectory(request);
REQUIRE(result);
CHECK(result.value().filesIndexed == 2);
}
SECTION("Include multiple patterns") {
fixture.createFile("doc.txt", "text");
fixture.createFile("readme.md", "markdown");
fixture.createFile("script.py", "python");
fixture.createFile("code.cpp", "c++");
AddDirectoryRequest request;
request.directoryPath = fixture.testDir_.string();
request.includePatterns = {"*.txt", "*.md", "*.py"};
auto result = fixture.indexingService_->addDirectory(request);
REQUIRE(result);
CHECK(result.value().filesIndexed == 3);
}
SECTION("Exclude pattern overrides include") {
fixture.createFile("file1.txt", "text1");
fixture.createFile("file2.txt", "text2");
fixture.createFile("exclude_me.txt", "excluded");
AddDirectoryRequest request;
request.directoryPath = fixture.testDir_.string();
request.includePatterns = {"*.txt"};
request.excludePatterns = {"exclude_*"};
auto result = fixture.indexingService_->addDirectory(request);
REQUIRE(result);
CHECK(result.value().filesIndexed == 2);
}
SECTION("No include patterns indexes all") {
fixture.createFile("file.txt", "text");
fixture.createFile("file.md", "markdown");
fixture.createFile("file.cpp", "c++");
auto request = fixture.createRequest(fixture.testDir_);
auto result = fixture.indexingService_->addDirectory(request);
REQUIRE(result);
CHECK(result.value().filesIndexed == 3);
}
SECTION("Pattern with question mark wildcard") {
fixture.createFile("file1.txt", "1");
fixture.createFile("file2.txt", "2");
fixture.createFile("file10.txt", "10");
AddDirectoryRequest request;
request.directoryPath = fixture.testDir_.string();
request.includePatterns = {"file?.txt"};
auto result = fixture.indexingService_->addDirectory(request);
REQUIRE(result);
CHECK(result.value().filesIndexed == 2); // Matches file1.txt and file2.txt only
}
SECTION("Case insensitive pattern matching") {
fixture.createFile("file.txt", "content1");
fixture.createFile("readme.txt", "content2");
AddDirectoryRequest request;
request.directoryPath = fixture.testDir_.string();
request.includePatterns = {"*.txt"};
auto result = fixture.indexingService_->addDirectory(request);
REQUIRE(result);
CHECK(result.value().filesIndexed == 2);
}
}
TEST_CASE("IndexingService - Recursion and Directory Traversal", "[indexing][service][recursion]") {
IndexingFixture fixture;
SECTION("Non-recursive skips subdirectories") {
fixture.createFile("root.txt", "root");
fixture.createFile("subdir/nested.txt", "nested");
AddDirectoryRequest request;
request.directoryPath = fixture.testDir_.string();
request.recursive = false;
auto result = fixture.indexingService_->addDirectory(request);
REQUIRE(result);
CHECK(result.value().filesIndexed == 1);
}
SECTION("Recursive indexes nested directories") {
fixture.createFile("root.txt", "root");
fixture.createFile("sub1/file1.txt", "file1");
fixture.createFile("sub1/sub2/file2.txt", "file2");
AddDirectoryRequest request;
request.directoryPath = fixture.testDir_.string();
request.recursive = true;
auto result = fixture.indexingService_->addDirectory(request);
REQUIRE(result);
CHECK(result.value().filesIndexed == 3);
}
SECTION("Recursive with patterns in nested dirs") {
fixture.createFile("sub1/file.txt", "txt");
fixture.createFile("sub1/file.md", "md");
fixture.createFile("sub2/file.txt", "txt2");
fixture.createFile("sub2/file.cpp", "cpp");
AddDirectoryRequest request;
request.directoryPath = fixture.testDir_.string();
request.recursive = true;
request.includePatterns = {"*.txt"};
auto result = fixture.indexingService_->addDirectory(request);
REQUIRE(result);
CHECK(result.value().filesIndexed == 2);
}
SECTION("Directory with only subdirectories") {
std::filesystem::create_directories(fixture.testDir_ / "sub1");
std::filesystem::create_directories(fixture.testDir_ / "sub2");
auto request = fixture.createRequest(fixture.testDir_);
auto result = fixture.indexingService_->addDirectory(request);
REQUIRE(result);
CHECK(result.value().filesIndexed == 0);
}
SECTION("Deep directory hierarchy") {
fixture.createFile("l1/l2/l3/l4/l5/deep.txt", "deep content");
AddDirectoryRequest request;
request.directoryPath = fixture.testDir_.string();
request.recursive = true;
auto result = fixture.indexingService_->addDirectory(request);
REQUIRE(result);
CHECK(result.value().filesIndexed == 1);
}
}
TEST_CASE("IndexingService - Metadata and Collections", "[indexing][service][metadata]") {
IndexingFixture fixture;
SECTION("Tags propagated to all files") {
fixture.createFile("file1.txt", "content1");
fixture.createFile("file2.txt", "content2");
AddDirectoryRequest request;
request.directoryPath = fixture.testDir_.string();
request.tags = {"important", "project-x"};
auto result = fixture.indexingService_->addDirectory(request);
REQUIRE(result);
CHECK(result.value().filesIndexed == 2);
}
SECTION("Collection assigned to all files") {
fixture.createFile("file1.txt", "content1");
fixture.createFile("file2.txt", "content2");
AddDirectoryRequest request;
request.directoryPath = fixture.testDir_.string();
request.collection = "my-collection";
auto result = fixture.indexingService_->addDirectory(request);
REQUIRE(result);
CHECK(result.value().filesIndexed == 2);
CHECK(result.value().collection == "my-collection");
}
SECTION("Metadata propagated to files") {
fixture.createFile("file.txt", "content");
AddDirectoryRequest request;
request.directoryPath = fixture.testDir_.string();
request.metadata = {{"project", "test"}, {"version", "1.0"}, {"author", "tester"}};
auto result = fixture.indexingService_->addDirectory(request);
REQUIRE(result);
CHECK(result.value().filesIndexed == 1);
}
}
TEST_CASE("IndexingService - Response Validation", "[indexing][service][response]") {
IndexingFixture fixture;
SECTION("Response contains correct directory path") {
fixture.createFile("file.txt", "content");
auto request = fixture.createRequest(fixture.testDir_);
auto result = fixture.indexingService_->addDirectory(request);
REQUIRE(result);
CHECK(result.value().directoryPath == fixture.testDir_.string());
}
SECTION("Response contains collection name") {
fixture.createFile("file.txt", "content");
AddDirectoryRequest request;
request.directoryPath = fixture.testDir_.string();
request.collection = "test-collection";
auto result = fixture.indexingService_->addDirectory(request);
REQUIRE(result);
CHECK(result.value().collection == "test-collection");
}
SECTION("Individual file results populated") {
fixture.createFile("file1.txt", "content1");
fixture.createFile("file2.txt", "content2");
auto request = fixture.createRequest(fixture.testDir_);
auto result = fixture.indexingService_->addDirectory(request);
REQUIRE(result);
CHECK_FALSE(result.value().results.empty());
CHECK(result.value().results.size() == 2);
}
}
Reference in a new issue View git blame Copy permalink