08322369cb* test(storage): add crash-replay and handle-failure coverage for ReferenceCounter Adds 6 test cases: - Recovery after forced close preserves committed data - Uncommitted transaction rolled back after forced close - Corrupted database file rejected on open - Consistent stats after interrupted commit recovery - Multiple PENDING transactions survive force close - WAL checkpoint survives forced close * fix(ci): remove ccache — conflicts with OpenBLAS ASM as direct compiler * fix(ci): use explicit CC/CXX paths to prevent Conan from using ccache directly ccache was picked up by Conan/CMake as the compiler itself (/usr/bin/ccache) rather than as a transparent wrapper, causing OpenBLAS ASM compilation to fail with 'ccache: invalid option -- D'. Setting explicit compiler paths prevents this. * fix(ci): add YAMS_SKIP_CCACHE to avoid OpenBLAS ASM ccache conflict setup.sh wraps CC/CXX with ccache for faster incremental builds, but this causes Conan/CMake to use ccache directly as the ASM compiler for OpenBLAS, where it fails on -D flags. Add an opt-out gate in setup.sh and set YAMS_SKIP_CCACHE=1 in .build.yml for sourcehut CI VMs. * fix(ci): add libomp-dev, libfaiss-dev, libopenblas-dev for FAISS build * fix(ci): limit meson compile to -j1 to prevent OOM on sourcehut VMs * fix(build): restore hooks and shrink srht Compile the storage library with YAMS_TESTING in test builds so debug-only hooks link again. Include the POSIX chmod header used by the storage utility Catch2 coverage cases. Prune SourceHut fallback stage archives to runtime files so branch artifacts stay under the upload limit. * refactor(storage): drop dead code, add coverage * refactor(cli): polish grep/search graph UX * fix(daemon): stabilize shutdown and CI paths - force vector-backed tests to opt out of CI-wide disable flags - reset reference prune statements between deletes - make optional-plugin and POSIX-only test assumptions portable - extract ServiceManager open/shutdown helpers for safer WAL recovery work * test(daemon): add db vacuum coverage * test(daemon): add WAL startup recovery - cover stale WAL startup recovery in ServiceManager - cover repeated restart cycles with seeded WAL sidecars - assert daemon recovery reporting for corrupt data-dir DBs - verify oversized metadata DBs shrink after WAL recovery shutdown * refactor(grep): consolidate indexed content scan * fix(grep): remove pre-scan candidate truncation * refactor(runtime): split startup, harden storage * feat(bench): add manifest retrieval corpus * fix(bench): run fusion experiment on simeon * fix(cli): sharpen search graph hints * fix(build): refresh version metadata * fix(ci): stabilize sync search and srht packaging * fix(ci): unblock windows and shrink artifacts * fix(ci): stabilize linux runtime checks * fix(build): shrink srht manifest
1621 lines
57 KiB
C++
1621 lines
57 KiB
C++
// Catch2 migration of content_store_test.cpp with expanded test coverage
|
|
// Epic: yams-3s4 | Migration: api tests
|
|
|
|
#include <catch2/catch_approx.hpp>
|
|
#include <catch2/catch_test_macros.hpp>
|
|
#include <catch2/matchers/catch_matchers_floating_point.hpp>
|
|
#include <catch2/matchers/catch_matchers_string.hpp>
|
|
|
|
#include <yams/api/async_content_store.h>
|
|
#include <yams/api/content_metadata.h>
|
|
#include <yams/api/content_store.h>
|
|
#include <yams/api/content_store_builder.h>
|
|
#include <yams/api/content_store_error.h>
|
|
#include <yams/api/progress_reporter.h>
|
|
#include <yams/crypto/hasher.h>
|
|
#include <yams/storage/storage_engine.h>
|
|
|
|
#include <algorithm>
|
|
#include <atomic>
|
|
#include <filesystem>
|
|
#include <fstream>
|
|
#include <future>
|
|
#include <mutex>
|
|
#include <random>
|
|
#include <sstream>
|
|
#include <thread>
|
|
#include <unordered_map>
|
|
|
|
namespace yams::api::test {
|
|
|
|
namespace fs = std::filesystem;
|
|
using namespace std::chrono_literals;
|
|
|
|
// Test fixture helper
|
|
struct ContentStoreFixture {
|
|
fs::path testDir_;
|
|
std::unique_ptr<IContentStore> store_;
|
|
|
|
ContentStoreFixture() {
|
|
testDir_ =
|
|
fs::temp_directory_path() / ("kronos_test_" + std::to_string(std::random_device{}()));
|
|
fs::create_directories(testDir_);
|
|
|
|
auto result = ContentStoreBuilder::createDefault(testDir_);
|
|
REQUIRE(result.has_value());
|
|
store_ = std::move(result).value();
|
|
}
|
|
|
|
~ContentStoreFixture() {
|
|
store_.reset();
|
|
std::error_code ec;
|
|
fs::remove_all(testDir_, ec);
|
|
}
|
|
|
|
fs::path createTestFile(const std::string& name, const std::string& content) {
|
|
auto path = testDir_ / name;
|
|
std::ofstream file(path);
|
|
file << content;
|
|
file.close();
|
|
return path;
|
|
}
|
|
|
|
std::string readFile(const fs::path& path) {
|
|
std::ifstream file(path);
|
|
return std::string(std::istreambuf_iterator<char>(file), std::istreambuf_iterator<char>());
|
|
}
|
|
};
|
|
|
|
class ScriptedStorageEngine final : public yams::storage::IStorageEngine {
|
|
public:
|
|
bool failStoresAsNotInitialized{false};
|
|
bool failManifestStore{false};
|
|
bool failManifestStoreAmbiguously{false};
|
|
bool failManifestStoreWithMismatchedDurableBytes{false};
|
|
size_t storeCalls{0};
|
|
size_t removeCalls{0};
|
|
std::vector<std::string> removedKeys;
|
|
|
|
Result<void> store(std::string_view hash, std::span<const std::byte> data) override {
|
|
++storeCalls;
|
|
const std::string key(hash);
|
|
if (failStoresAsNotInitialized) {
|
|
return Error{ErrorCode::NotInitialized, "scripted storage is not ready"};
|
|
}
|
|
if (failManifestStore && key.ends_with(".manifest")) {
|
|
if (failManifestStoreAmbiguously) {
|
|
objects_[key] = std::vector<std::byte>(data.begin(), data.end());
|
|
return Error{ErrorCode::NetworkError, "scripted ambiguous manifest write"};
|
|
}
|
|
if (failManifestStoreWithMismatchedDurableBytes) {
|
|
auto durableBytes = std::vector<std::byte>(data.begin(), data.end());
|
|
if (durableBytes.empty()) {
|
|
durableBytes.push_back(std::byte{0});
|
|
} else {
|
|
durableBytes.front() ^= std::byte{0x01};
|
|
}
|
|
objects_[key] = std::move(durableBytes);
|
|
return Error{ErrorCode::NetworkError,
|
|
"scripted ambiguous manifest write with mismatched durable bytes"};
|
|
}
|
|
return Error{ErrorCode::WriteError, "scripted manifest write failure"};
|
|
}
|
|
objects_[key] = std::vector<std::byte>(data.begin(), data.end());
|
|
return {};
|
|
}
|
|
|
|
Result<std::vector<std::byte>> retrieve(std::string_view hash) const override {
|
|
auto it = objects_.find(std::string(hash));
|
|
if (it == objects_.end()) {
|
|
return Error{ErrorCode::NotFound, "scripted object missing"};
|
|
}
|
|
return it->second;
|
|
}
|
|
|
|
Result<RawObject> retrieveRaw(std::string_view hash) const override {
|
|
auto data = retrieve(hash);
|
|
if (!data) {
|
|
return data.error();
|
|
}
|
|
RawObject raw;
|
|
raw.data = std::move(data).value();
|
|
return raw;
|
|
}
|
|
|
|
Result<bool> exists(std::string_view hash) const noexcept override {
|
|
return objects_.contains(std::string(hash));
|
|
}
|
|
|
|
Result<void> remove(std::string_view hash) override {
|
|
++removeCalls;
|
|
removedKeys.emplace_back(hash.data(), hash.size());
|
|
objects_.erase(std::string(hash));
|
|
return {};
|
|
}
|
|
|
|
Result<uint64_t> getBlockSize(std::string_view hash) const override {
|
|
auto it = objects_.find(std::string(hash));
|
|
if (it == objects_.end()) {
|
|
return Error{ErrorCode::NotFound, "scripted object missing"};
|
|
}
|
|
return static_cast<uint64_t>(it->second.size());
|
|
}
|
|
|
|
std::future<Result<void>> storeAsync(std::string_view hash,
|
|
std::span<const std::byte> data) override {
|
|
return std::async(std::launch::deferred,
|
|
[this, key = std::string(hash),
|
|
copy = std::vector<std::byte>(data.begin(), data.end())]() {
|
|
return store(key, copy);
|
|
});
|
|
}
|
|
|
|
std::future<Result<std::vector<std::byte>>>
|
|
retrieveAsync(std::string_view hash) const override {
|
|
return std::async(std::launch::deferred,
|
|
[this, key = std::string(hash)]() { return retrieve(key); });
|
|
}
|
|
|
|
std::future<Result<RawObject>> retrieveRawAsync(std::string_view hash) const override {
|
|
return std::async(std::launch::deferred,
|
|
[this, key = std::string(hash)]() { return retrieveRaw(key); });
|
|
}
|
|
|
|
std::vector<Result<void>>
|
|
storeBatch(const std::vector<std::pair<std::string, std::vector<std::byte>>>& items) override {
|
|
std::vector<Result<void>> results;
|
|
results.reserve(items.size());
|
|
for (const auto& [hash, data] : items) {
|
|
results.push_back(store(hash, data));
|
|
}
|
|
return results;
|
|
}
|
|
|
|
yams::storage::StorageStats getStats() const noexcept override {
|
|
yams::storage::StorageStats stats;
|
|
stats.totalObjects = objects_.size();
|
|
uint64_t totalBytes = 0;
|
|
for (const auto& [_, data] : objects_) {
|
|
totalBytes += data.size();
|
|
}
|
|
stats.totalBytes = totalBytes;
|
|
return stats;
|
|
}
|
|
|
|
Result<uint64_t> getStorageSize() const override {
|
|
uint64_t totalBytes = 0;
|
|
for (const auto& [_, data] : objects_) {
|
|
totalBytes += data.size();
|
|
}
|
|
return totalBytes;
|
|
}
|
|
|
|
Result<std::vector<std::string>> list(std::string_view = "") const override {
|
|
std::vector<std::string> keys;
|
|
for (const auto& [k, _] : objects_) {
|
|
keys.push_back(k);
|
|
}
|
|
return keys;
|
|
}
|
|
|
|
bool empty() const { return objects_.empty(); }
|
|
|
|
private:
|
|
std::unordered_map<std::string, std::vector<std::byte>> objects_;
|
|
};
|
|
|
|
class FailingFileChunker final : public yams::chunking::IChunker {
|
|
public:
|
|
size_t chunkFileCalls{0};
|
|
|
|
const yams::chunking::ChunkingConfig& getConfig() const override { return config_; }
|
|
|
|
std::vector<yams::chunking::Chunk> chunkFile(const fs::path&) override {
|
|
++chunkFileCalls;
|
|
throw std::runtime_error("scripted file chunk failure");
|
|
}
|
|
|
|
std::vector<yams::chunking::Chunk> chunkData(yams::span<const std::byte> data) override {
|
|
std::vector<std::byte> copy(data.begin(), data.end());
|
|
return {yams::chunking::Chunk{.data = std::move(copy),
|
|
.hash = yams::crypto::SHA256Hasher::hash(data),
|
|
.offset = 0,
|
|
.size = data.size()}};
|
|
}
|
|
|
|
std::vector<yams::chunking::Chunk> chunkDataLazy(yams::span<const std::byte> data) override {
|
|
return chunkData(data);
|
|
}
|
|
|
|
std::future<Result<std::vector<yams::chunking::Chunk>>>
|
|
chunkFileAsync(const fs::path&) override {
|
|
return std::async(std::launch::deferred,
|
|
[]() -> Result<std::vector<yams::chunking::Chunk>> {
|
|
return Error{ErrorCode::InternalError, "scripted file chunk failure"};
|
|
});
|
|
}
|
|
|
|
void setProgressCallback(ProgressCallback) override {}
|
|
|
|
private:
|
|
yams::chunking::ChunkingConfig config_{};
|
|
};
|
|
|
|
class ScriptedReferenceCounter final : public yams::storage::IReferenceCounter {
|
|
class Transaction;
|
|
|
|
public:
|
|
bool failCommit{false};
|
|
size_t beginCalls{0};
|
|
size_t commitCalls{0};
|
|
size_t rollbackCalls{0};
|
|
size_t queuedIncrements{0};
|
|
|
|
Result<void> increment(std::string_view, size_t, size_t) override { return {}; }
|
|
Result<void> decrement(std::string_view) override { return {}; }
|
|
Result<uint64_t> getRefCount(std::string_view) const override { return uint64_t{0}; }
|
|
Result<bool> hasReferences(std::string_view) const override { return false; }
|
|
|
|
Result<yams::storage::RefCountStats> getStats() const override {
|
|
return yams::storage::RefCountStats{};
|
|
}
|
|
|
|
std::unique_ptr<ITransaction> beginTransaction() override;
|
|
|
|
private:
|
|
class Transaction final : public ITransaction {
|
|
public:
|
|
explicit Transaction(ScriptedReferenceCounter& parent) : parent_(parent) {}
|
|
|
|
void increment(std::string_view, size_t, size_t) override {
|
|
if (!active_) {
|
|
throw std::runtime_error("scripted transaction inactive");
|
|
}
|
|
++parent_.queuedIncrements;
|
|
}
|
|
|
|
void decrement(std::string_view) override {}
|
|
void pruneReference(std::string_view) override {}
|
|
|
|
Result<void> commit() override {
|
|
++parent_.commitCalls;
|
|
if (parent_.failCommit) {
|
|
return Error{ErrorCode::TransactionFailed, "scripted commit failure"};
|
|
}
|
|
active_ = false;
|
|
return {};
|
|
}
|
|
|
|
void rollback() override {
|
|
if (!active_) {
|
|
return;
|
|
}
|
|
++parent_.rollbackCalls;
|
|
active_ = false;
|
|
}
|
|
|
|
bool isActive() const override { return active_; }
|
|
|
|
private:
|
|
ScriptedReferenceCounter& parent_;
|
|
bool active_{true};
|
|
};
|
|
};
|
|
|
|
std::unique_ptr<yams::storage::IReferenceCounter::ITransaction>
|
|
ScriptedReferenceCounter::beginTransaction() {
|
|
++beginCalls;
|
|
return std::make_unique<Transaction>(*this);
|
|
}
|
|
|
|
fs::path makeTempDir(std::string_view prefix) {
|
|
auto path = fs::temp_directory_path() /
|
|
(std::string(prefix) + "_" + std::to_string(std::random_device{}()));
|
|
fs::create_directories(path);
|
|
return path;
|
|
}
|
|
|
|
ContentStoreConfig readinessConfig(const fs::path& path) {
|
|
ContentStoreConfig config;
|
|
config.storagePath = path;
|
|
config.chunkSize = MIN_CHUNK_SIZE;
|
|
config.enableCompression = false;
|
|
return config;
|
|
}
|
|
|
|
// =============================================================================
|
|
// Basic Store and Retrieve Tests
|
|
// =============================================================================
|
|
|
|
TEST_CASE("ContentStore: Basic store and retrieve", "[api][content-store]") {
|
|
ContentStoreFixture fixture;
|
|
|
|
SECTION("Store and retrieve simple text file") {
|
|
std::string content = "Hello, Kronos!";
|
|
auto inputPath = fixture.createTestFile("test.txt", content);
|
|
|
|
ContentMetadata metadata;
|
|
metadata.mimeType = "text/plain";
|
|
metadata.name = "test.txt";
|
|
|
|
auto storeResult = fixture.store_->store(inputPath, metadata);
|
|
REQUIRE(storeResult.has_value());
|
|
CHECK_FALSE(storeResult.value().contentHash.empty());
|
|
CHECK(storeResult.value().bytesStored == content.size());
|
|
|
|
auto outputPath = fixture.testDir_ / "retrieved.txt";
|
|
auto retrieveResult = fixture.store_->retrieve(storeResult.value().contentHash, outputPath);
|
|
REQUIRE(retrieveResult.has_value());
|
|
CHECK(retrieveResult.value().found);
|
|
CHECK(retrieveResult.value().size == content.size());
|
|
CHECK(fixture.readFile(outputPath) == content);
|
|
}
|
|
|
|
SECTION("Store file without metadata") {
|
|
auto file = fixture.createTestFile("bare.txt", "Bare content");
|
|
auto result = fixture.store_->store(file);
|
|
REQUIRE(result.has_value());
|
|
CHECK_FALSE(result.value().contentHash.empty());
|
|
}
|
|
|
|
SECTION("Store empty file") {
|
|
auto file = fixture.createTestFile("empty.txt", "");
|
|
auto result = fixture.store_->store(file);
|
|
REQUIRE(result.has_value());
|
|
CHECK(result.value().bytesStored == 0);
|
|
}
|
|
|
|
SECTION("Store binary content") {
|
|
std::string binaryContent;
|
|
for (int i = 0; i < 256; ++i) {
|
|
binaryContent.push_back(static_cast<char>(i));
|
|
}
|
|
auto file = fixture.createTestFile("binary.bin", binaryContent);
|
|
|
|
ContentMetadata metadata;
|
|
metadata.mimeType = "application/octet-stream";
|
|
metadata.name = "binary.bin";
|
|
|
|
auto result = fixture.store_->store(file, metadata);
|
|
REQUIRE(result.has_value());
|
|
// Note: bytesStored may include additional metadata or vary by implementation
|
|
CHECK(result.value().bytesStored >= 256);
|
|
|
|
// Retrieve and verify binary content intact
|
|
auto outPath = fixture.testDir_ / "binary_out.bin";
|
|
auto retrieved = fixture.store_->retrieve(result.value().contentHash, outPath);
|
|
REQUIRE(retrieved.has_value());
|
|
// Binary round-trip may have implementation-specific behavior
|
|
// Just verify we can retrieve the content
|
|
CHECK(retrieved.value().found);
|
|
}
|
|
}
|
|
|
|
// =============================================================================
|
|
// Deduplication Tests
|
|
// =============================================================================
|
|
|
|
TEST_CASE("ContentStore: Deduplication", "[api][content-store][dedup]") {
|
|
ContentStoreFixture fixture;
|
|
|
|
SECTION("Identical content produces same hash") {
|
|
std::string content = "Duplicate content for testing";
|
|
|
|
auto file1 = fixture.createTestFile("dup1.txt", content);
|
|
auto result1 = fixture.store_->store(file1);
|
|
REQUIRE(result1.has_value());
|
|
|
|
auto file2 = fixture.createTestFile("dup2.txt", content);
|
|
auto result2 = fixture.store_->store(file2);
|
|
REQUIRE(result2.has_value());
|
|
|
|
CHECK(result1.value().contentHash == result2.value().contentHash);
|
|
CHECK(result2.value().bytesDeduped > 0);
|
|
CHECK(result2.value().dedupRatio() == Catch::Approx(1.0));
|
|
}
|
|
|
|
SECTION("Different content produces different hash") {
|
|
auto file1 = fixture.createTestFile("unique1.txt", "Content A");
|
|
auto file2 = fixture.createTestFile("unique2.txt", "Content B");
|
|
|
|
auto result1 = fixture.store_->store(file1);
|
|
auto result2 = fixture.store_->store(file2);
|
|
|
|
REQUIRE(result1.has_value());
|
|
REQUIRE(result2.has_value());
|
|
CHECK(result1.value().contentHash != result2.value().contentHash);
|
|
}
|
|
|
|
SECTION("Case sensitive content") {
|
|
auto lower = fixture.createTestFile("lower.txt", "hello world");
|
|
auto upper = fixture.createTestFile("upper.txt", "HELLO WORLD");
|
|
|
|
auto r1 = fixture.store_->store(lower);
|
|
auto r2 = fixture.store_->store(upper);
|
|
|
|
REQUIRE(r1.has_value());
|
|
REQUIRE(r2.has_value());
|
|
CHECK(r1.value().contentHash != r2.value().contentHash);
|
|
}
|
|
|
|
SECTION("Whitespace matters") {
|
|
auto normal = fixture.createTestFile("normal.txt", "hello world");
|
|
auto extra = fixture.createTestFile("extra.txt", "hello world");
|
|
|
|
auto r1 = fixture.store_->store(normal);
|
|
auto r2 = fixture.store_->store(extra);
|
|
|
|
REQUIRE(r1.has_value());
|
|
REQUIRE(r2.has_value());
|
|
CHECK(r1.value().contentHash != r2.value().contentHash);
|
|
}
|
|
}
|
|
|
|
// =============================================================================
|
|
// Metadata Operations Tests
|
|
// =============================================================================
|
|
|
|
TEST_CASE("ContentStore: Metadata operations", "[api][content-store][metadata]") {
|
|
ContentStoreFixture fixture;
|
|
|
|
SECTION("Store and retrieve metadata") {
|
|
auto file = fixture.createTestFile("meta.txt", "Content with metadata");
|
|
|
|
ContentMetadata metadata;
|
|
metadata.mimeType = "text/plain";
|
|
metadata.name = "meta.txt";
|
|
metadata.tags["category"] = "test";
|
|
metadata.tags["type"] = "sample";
|
|
metadata.tags["author"] = "test_suite";
|
|
|
|
auto storeResult = fixture.store_->store(file, metadata);
|
|
REQUIRE(storeResult.has_value());
|
|
|
|
auto hash = storeResult.value().contentHash;
|
|
auto metaResult = fixture.store_->getMetadata(hash);
|
|
REQUIRE(metaResult.has_value());
|
|
|
|
auto& retrieved = metaResult.value();
|
|
CHECK(retrieved.mimeType == metadata.mimeType);
|
|
CHECK(retrieved.name == metadata.name);
|
|
CHECK(retrieved.tags == metadata.tags);
|
|
CHECK(retrieved.tags.at("author") == "test_suite");
|
|
}
|
|
|
|
SECTION("Update metadata") {
|
|
auto file = fixture.createTestFile("update.txt", "Update test");
|
|
auto storeResult = fixture.store_->store(file);
|
|
REQUIRE(storeResult.has_value());
|
|
|
|
auto hash = storeResult.value().contentHash;
|
|
|
|
ContentMetadata updated;
|
|
updated.tags["status"] = "updated";
|
|
|
|
auto updateResult = fixture.store_->updateMetadata(hash, updated);
|
|
REQUIRE(updateResult.has_value());
|
|
|
|
auto updatedMeta = fixture.store_->getMetadata(hash);
|
|
REQUIRE(updatedMeta.has_value());
|
|
CHECK(updatedMeta.value().tags.at("status") == "updated");
|
|
}
|
|
|
|
SECTION("Multiple tags") {
|
|
auto file = fixture.createTestFile("tagged.txt", "Tagged content");
|
|
|
|
ContentMetadata metadata;
|
|
metadata.tags["tag1"] = "value1";
|
|
metadata.tags["tag2"] = "value2";
|
|
metadata.tags["tag3"] = "value3";
|
|
metadata.tags["tag4"] = "value4";
|
|
metadata.tags["tag5"] = "value5";
|
|
|
|
auto result = fixture.store_->store(file, metadata);
|
|
REQUIRE(result.has_value());
|
|
|
|
auto retrieved = fixture.store_->getMetadata(result.value().contentHash);
|
|
REQUIRE(retrieved.has_value());
|
|
CHECK(retrieved.value().tags.size() == 5);
|
|
}
|
|
|
|
SECTION("Special characters in tags") {
|
|
auto file = fixture.createTestFile("special.txt", "Special");
|
|
|
|
ContentMetadata metadata;
|
|
metadata.tags["path/with/slashes"] = "/some/path";
|
|
metadata.tags["unicode_tag"] = "日本語";
|
|
metadata.tags["spaces in key"] = "spaces in value";
|
|
|
|
auto result = fixture.store_->store(file, metadata);
|
|
REQUIRE(result.has_value());
|
|
|
|
auto retrieved = fixture.store_->getMetadata(result.value().contentHash);
|
|
REQUIRE(retrieved.has_value());
|
|
CHECK(retrieved.value().tags.at("unicode_tag") == "日本語");
|
|
}
|
|
|
|
SECTION("Internal hash hint tags are stripped from persisted file metadata") {
|
|
const std::string content = "trusted hash hint";
|
|
auto file = fixture.createTestFile("trusted.txt", content);
|
|
|
|
ContentMetadata metadata;
|
|
metadata.name = "trusted.txt";
|
|
metadata.tags["visible"] = "keep";
|
|
metadata.tags["__yams_trusted_hash_hint"] = "1";
|
|
metadata.tags["__yams_hash_hint_mtime_ns"] = "123";
|
|
|
|
auto result = fixture.store_->store(file, metadata);
|
|
REQUIRE(result.has_value());
|
|
|
|
auto retrieved = fixture.store_->getMetadata(result.value().contentHash);
|
|
REQUIRE(retrieved.has_value());
|
|
CHECK(retrieved.value().tags.contains("visible"));
|
|
CHECK_FALSE(retrieved.value().tags.contains("__yams_trusted_hash_hint"));
|
|
CHECK_FALSE(retrieved.value().tags.contains("__yams_hash_hint_mtime_ns"));
|
|
CHECK(retrieved.value().contentHash == result.value().contentHash);
|
|
CHECK(retrieved.value().size == content.size());
|
|
}
|
|
}
|
|
|
|
// =============================================================================
|
|
// Stream Operations Tests
|
|
// =============================================================================
|
|
|
|
TEST_CASE("ContentStore: Stream operations", "[api][content-store][stream]") {
|
|
ContentStoreFixture fixture;
|
|
|
|
SECTION("Store and retrieve via stream") {
|
|
std::string content = "Stream test content";
|
|
|
|
std::istringstream input(content);
|
|
ContentMetadata metadata;
|
|
metadata.name = "stream.txt";
|
|
|
|
auto storeResult = fixture.store_->storeStream(input, metadata);
|
|
REQUIRE(storeResult.has_value());
|
|
|
|
std::ostringstream output;
|
|
auto retrieveResult =
|
|
fixture.store_->retrieveStream(storeResult.value().contentHash, output);
|
|
REQUIRE(retrieveResult.has_value());
|
|
|
|
CHECK(output.str() == content);
|
|
}
|
|
|
|
SECTION("Store large stream") {
|
|
std::string largeContent(100000, 'X');
|
|
std::istringstream input(largeContent);
|
|
|
|
auto result = fixture.store_->storeStream(input);
|
|
REQUIRE(result.has_value());
|
|
CHECK(result.value().bytesStored == 100000);
|
|
}
|
|
|
|
SECTION("Empty stream") {
|
|
std::istringstream empty("");
|
|
auto result = fixture.store_->storeStream(empty);
|
|
REQUIRE(result.has_value());
|
|
CHECK(result.value().bytesStored == 0);
|
|
}
|
|
}
|
|
|
|
// =============================================================================
|
|
// Memory Operations Tests
|
|
// =============================================================================
|
|
|
|
TEST_CASE("ContentStore: Memory operations", "[api][content-store][memory]") {
|
|
ContentStoreFixture fixture;
|
|
|
|
SECTION("Store and retrieve bytes") {
|
|
std::string content = "Memory test";
|
|
std::vector<std::byte> data;
|
|
data.reserve(content.size());
|
|
for (char c : content) {
|
|
data.push_back(static_cast<std::byte>(c));
|
|
}
|
|
|
|
ContentMetadata metadata;
|
|
metadata.name = "memory.bin";
|
|
|
|
auto storeResult = fixture.store_->storeBytes(data, metadata);
|
|
REQUIRE(storeResult.has_value());
|
|
|
|
auto retrieveResult = fixture.store_->retrieveBytes(storeResult.value().contentHash);
|
|
REQUIRE(retrieveResult.has_value());
|
|
|
|
std::string retrieved(reinterpret_cast<const char*>(retrieveResult.value().data()),
|
|
retrieveResult.value().size());
|
|
CHECK(retrieved == content);
|
|
}
|
|
|
|
SECTION("Store binary bytes with null characters") {
|
|
std::vector<std::byte> data = {std::byte{0x00}, std::byte{0x01}, std::byte{0x00},
|
|
std::byte{0xFF}, std::byte{0x00}, std::byte{0xFE}};
|
|
|
|
auto result = fixture.store_->storeBytes(data);
|
|
REQUIRE(result.has_value());
|
|
|
|
auto retrieved = fixture.store_->retrieveBytes(result.value().contentHash);
|
|
REQUIRE(retrieved.has_value());
|
|
CHECK(retrieved.value().size() == 6);
|
|
CHECK(retrieved.value()[0] == std::byte{0x00});
|
|
CHECK(retrieved.value()[3] == std::byte{0xFF});
|
|
}
|
|
|
|
SECTION("Retrieve raw returns stored direct bytes without compression header") {
|
|
const std::string content = "raw direct bytes";
|
|
std::vector<std::byte> data;
|
|
data.reserve(content.size());
|
|
for (char c : content) {
|
|
data.push_back(static_cast<std::byte>(c));
|
|
}
|
|
|
|
auto stored = fixture.store_->storeBytes(data);
|
|
REQUIRE(stored.has_value());
|
|
|
|
auto raw = fixture.store_->retrieveRaw(stored.value().contentHash);
|
|
REQUIRE(raw.has_value());
|
|
CHECK(raw.value().data == data);
|
|
CHECK_FALSE(raw.value().header.has_value());
|
|
}
|
|
|
|
SECTION("Retrieve raw async returns stored direct bytes") {
|
|
const std::string content = "async raw bytes";
|
|
std::vector<std::byte> data;
|
|
data.reserve(content.size());
|
|
for (char c : content) {
|
|
data.push_back(static_cast<std::byte>(c));
|
|
}
|
|
|
|
auto stored = fixture.store_->storeBytes(data);
|
|
REQUIRE(stored.has_value());
|
|
|
|
auto rawFuture = fixture.store_->retrieveRawAsync(stored.value().contentHash);
|
|
auto raw = rawFuture.get();
|
|
REQUIRE(raw.has_value());
|
|
CHECK(raw.value().data == data);
|
|
CHECK_FALSE(raw.value().header.has_value());
|
|
}
|
|
|
|
SECTION("Retrieve bytes prefix returns empty vector when max bytes is zero") {
|
|
std::vector<std::byte> data = {std::byte{0x41}, std::byte{0x42}, std::byte{0x43}};
|
|
auto stored = fixture.store_->storeBytes(data);
|
|
REQUIRE(stored.has_value());
|
|
|
|
auto prefix = fixture.store_->retrieveBytesPrefix(stored.value().contentHash, 0);
|
|
REQUIRE(prefix.has_value());
|
|
CHECK(prefix.value().empty());
|
|
}
|
|
}
|
|
|
|
// =============================================================================
|
|
// Batch Operations Tests
|
|
// =============================================================================
|
|
|
|
TEST_CASE("ContentStore: Batch operations", "[api][content-store][batch]") {
|
|
ContentStoreFixture fixture;
|
|
|
|
SECTION("Batch store multiple files") {
|
|
std::vector<fs::path> files;
|
|
std::vector<ContentMetadata> metadata;
|
|
|
|
for (int i = 0; i < 5; ++i) {
|
|
std::string name = "batch" + std::to_string(i) + ".txt";
|
|
std::string content = "Batch content " + std::to_string(i);
|
|
files.push_back(fixture.createTestFile(name, content));
|
|
|
|
ContentMetadata meta;
|
|
meta.name = name;
|
|
metadata.push_back(meta);
|
|
}
|
|
|
|
auto results = fixture.store_->storeBatch(files, metadata);
|
|
REQUIRE(results.size() == files.size());
|
|
|
|
std::vector<std::string> hashes;
|
|
for (const auto& result : results) {
|
|
REQUIRE(result.has_value());
|
|
hashes.push_back(result.value().contentHash);
|
|
}
|
|
|
|
// All hashes should be unique (different content)
|
|
std::sort(hashes.begin(), hashes.end());
|
|
auto last = std::unique(hashes.begin(), hashes.end());
|
|
CHECK(std::distance(hashes.begin(), last) == static_cast<int>(hashes.size()));
|
|
}
|
|
|
|
SECTION("Batch remove") {
|
|
std::vector<std::string> hashes;
|
|
for (int i = 0; i < 3; ++i) {
|
|
auto file = fixture.createTestFile("rm" + std::to_string(i) + ".txt",
|
|
"Remove " + std::to_string(i));
|
|
auto result = fixture.store_->store(file);
|
|
REQUIRE(result.has_value());
|
|
hashes.push_back(result.value().contentHash);
|
|
}
|
|
|
|
auto removeResults = fixture.store_->removeBatch(hashes);
|
|
REQUIRE(removeResults.size() == hashes.size());
|
|
|
|
for (const auto& result : removeResults) {
|
|
REQUIRE(result.has_value());
|
|
CHECK(result.value());
|
|
}
|
|
}
|
|
}
|
|
|
|
// =============================================================================
|
|
// Progress Reporting Tests
|
|
// =============================================================================
|
|
|
|
TEST_CASE("ContentStore: Progress reporting", "[api][content-store][progress]") {
|
|
ContentStoreFixture fixture;
|
|
|
|
SECTION("Progress callback invoked") {
|
|
std::string content(1024 * 1024, 'X'); // 1MB
|
|
auto file = fixture.createTestFile("large.bin", content);
|
|
|
|
std::atomic<int> progressCalls{0};
|
|
std::atomic<uint64_t> lastBytes{0};
|
|
|
|
ProgressCallback progressCallback = [&](const Progress& progress) {
|
|
progressCalls++;
|
|
lastBytes = progress.bytesProcessed;
|
|
REQUIRE(progress.bytesProcessed <= progress.totalBytes);
|
|
};
|
|
|
|
ContentMetadata metadata;
|
|
metadata.name = "large.bin";
|
|
auto result = fixture.store_->store(file, metadata, progressCallback);
|
|
REQUIRE(result.has_value());
|
|
|
|
CHECK(progressCalls.load() > 0);
|
|
CHECK(lastBytes.load() == content.size());
|
|
}
|
|
|
|
SECTION("Progress percentage calculation") {
|
|
std::string content(10000, 'Y');
|
|
auto file = fixture.createTestFile("progress.bin", content);
|
|
|
|
std::vector<double> percentages;
|
|
ProgressCallback callback = [&](const Progress& p) { percentages.push_back(p.percentage); };
|
|
|
|
fixture.store_->store(file, ContentMetadata{}, callback);
|
|
|
|
CHECK_FALSE(percentages.empty());
|
|
// Last percentage should be 100%
|
|
CHECK(percentages.back() == Catch::Approx(100.0));
|
|
}
|
|
}
|
|
|
|
// =============================================================================
|
|
// Error Handling Tests
|
|
// =============================================================================
|
|
|
|
TEST_CASE("ContentStore: Error handling", "[api][content-store][error]") {
|
|
ContentStoreFixture fixture;
|
|
|
|
SECTION("Store non-existent file") {
|
|
auto result = fixture.store_->store(fixture.testDir_ / "nonexistent.txt");
|
|
CHECK_FALSE(result.has_value());
|
|
CHECK(result.error() == ErrorCode::FileNotFound);
|
|
}
|
|
|
|
SECTION("Retrieve with invalid hash") {
|
|
auto result = fixture.store_->retrieve("invalid_hash", fixture.testDir_ / "out.txt");
|
|
CHECK_FALSE(result.has_value());
|
|
}
|
|
|
|
SECTION("Remove non-existent content returns false not error") {
|
|
auto result = fixture.store_->remove("nonexistent_hash");
|
|
REQUIRE(result.has_value());
|
|
CHECK_FALSE(result.value());
|
|
}
|
|
|
|
SECTION("Store to read-only directory") {
|
|
// This test is platform-specific and may not work on all systems
|
|
// Just verify we handle errors gracefully
|
|
}
|
|
}
|
|
|
|
TEST_CASE("ContentStore: Storage readiness failures", "[api][content-store][readiness]") {
|
|
SECTION("Injected not-ready storage is used and not replaced by local fallback") {
|
|
auto tempDir = makeTempDir("content_store_injected_not_ready");
|
|
auto storage = std::make_shared<ScriptedStorageEngine>();
|
|
storage->failStoresAsNotInitialized = true;
|
|
|
|
{
|
|
ContentStoreBuilder builder;
|
|
auto built =
|
|
builder.withConfig(readinessConfig(tempDir)).withStorageEngine(storage).build();
|
|
REQUIRE(built.has_value());
|
|
auto store = std::move(built).value();
|
|
|
|
std::vector<std::byte> data(128, std::byte{0x61});
|
|
auto stored = store->storeBytes(data);
|
|
REQUIRE_FALSE(stored.has_value());
|
|
CHECK(stored.error().code == ErrorCode::NotInitialized);
|
|
CHECK(storage->storeCalls == 1);
|
|
CHECK(storage->empty());
|
|
}
|
|
|
|
std::error_code ec;
|
|
fs::remove_all(tempDir, ec);
|
|
}
|
|
|
|
SECTION("Explicit null storage injection is rejected") {
|
|
auto tempDir = makeTempDir("content_store_null_storage");
|
|
{
|
|
ContentStoreBuilder builder;
|
|
auto built = builder.withConfig(readinessConfig(tempDir))
|
|
.withStorageEngine(std::shared_ptr<yams::storage::IStorageEngine>{})
|
|
.build();
|
|
REQUIRE_FALSE(built.has_value());
|
|
CHECK(built.error().code == ErrorCode::InvalidArgument);
|
|
}
|
|
|
|
std::error_code ec;
|
|
fs::remove_all(tempDir, ec);
|
|
}
|
|
}
|
|
|
|
TEST_CASE("ContentStore: Chunked add rollback leaves no committed manifest or metadata",
|
|
"[api][content-store][readiness][rollback]") {
|
|
auto tempDir = makeTempDir("content_store_partial_rollback");
|
|
auto storage = std::make_shared<ScriptedStorageEngine>();
|
|
storage->failManifestStore = true;
|
|
|
|
{
|
|
ContentStoreBuilder builder;
|
|
auto built =
|
|
builder.withConfig(readinessConfig(tempDir)).withStorageEngine(storage).build();
|
|
REQUIRE(built.has_value());
|
|
auto store = std::move(built).value();
|
|
|
|
std::vector<std::byte> data(MIN_CHUNK_SIZE + 1024);
|
|
for (size_t i = 0; i < data.size(); ++i) {
|
|
data[i] = static_cast<std::byte>(i & 0xff);
|
|
}
|
|
const auto hash =
|
|
yams::crypto::SHA256Hasher::hash(std::span<const std::byte>(data.data(), data.size()));
|
|
|
|
auto stored = store->storeBytes(data);
|
|
REQUIRE_FALSE(stored.has_value());
|
|
CHECK(stored.error().code == ErrorCode::WriteError);
|
|
CHECK(storage->storeCalls > 0);
|
|
|
|
auto manifestExists = storage->exists(hash + ".manifest");
|
|
REQUIRE(manifestExists.has_value());
|
|
CHECK_FALSE(manifestExists.value());
|
|
|
|
auto metadata = store->getMetadata(hash);
|
|
REQUIRE_FALSE(metadata.has_value());
|
|
CHECK(metadata.error().code == ErrorCode::FileNotFound);
|
|
|
|
auto stats = store->getStats();
|
|
CHECK(stats.storeOperations == 0);
|
|
CHECK(stats.uniqueBlocks == 0);
|
|
CHECK(stats.totalUncompressedBytes == 0);
|
|
}
|
|
|
|
std::error_code ec;
|
|
fs::remove_all(tempDir, ec);
|
|
}
|
|
|
|
TEST_CASE("ContentStore: File chunker failures return typed errors without durable writes",
|
|
"[api][content-store][readiness][rollback]") {
|
|
auto tempDir = makeTempDir("content_store_file_chunk_failure");
|
|
auto storage = std::make_shared<ScriptedStorageEngine>();
|
|
auto chunker = std::make_shared<FailingFileChunker>();
|
|
|
|
{
|
|
ContentStoreBuilder builder;
|
|
auto built = builder.withConfig(readinessConfig(tempDir))
|
|
.withStorageEngine(storage)
|
|
.withChunker(chunker)
|
|
.build();
|
|
REQUIRE(built.has_value());
|
|
auto store = std::move(built).value();
|
|
|
|
auto path = tempDir / "input.bin";
|
|
std::ofstream file(path, std::ios::binary);
|
|
file << "chunker failure input";
|
|
file.close();
|
|
|
|
auto stored = store->store(path);
|
|
REQUIRE_FALSE(stored.has_value());
|
|
CHECK(stored.error().code == ErrorCode::InternalError);
|
|
CHECK(chunker->chunkFileCalls == 1);
|
|
CHECK(storage->storeCalls == 0);
|
|
CHECK(storage->empty());
|
|
|
|
auto stats = store->getStats();
|
|
CHECK(stats.storeOperations == 0);
|
|
CHECK(stats.totalObjects == 0);
|
|
CHECK(stats.uniqueBlocks == 0);
|
|
}
|
|
|
|
std::error_code ec;
|
|
fs::remove_all(tempDir, ec);
|
|
}
|
|
|
|
TEST_CASE("ContentStore: File add commit failure leaves no committed manifest metadata",
|
|
"[api][content-store][readiness][rollback]") {
|
|
auto tempDir = makeTempDir("content_store_file_commit_failure");
|
|
auto storage = std::make_shared<ScriptedStorageEngine>();
|
|
auto refCounter = std::make_shared<ScriptedReferenceCounter>();
|
|
refCounter->failCommit = true;
|
|
|
|
{
|
|
ContentStoreBuilder builder;
|
|
auto built = builder.withConfig(readinessConfig(tempDir))
|
|
.withStorageEngine(storage)
|
|
.withReferenceCounter(refCounter)
|
|
.build();
|
|
REQUIRE(built.has_value());
|
|
auto store = std::move(built).value();
|
|
|
|
std::vector<std::byte> data(MIN_CHUNK_SIZE + 2048);
|
|
for (size_t i = 0; i < data.size(); ++i) {
|
|
data[i] = static_cast<std::byte>((i * 17) & 0xff);
|
|
}
|
|
const auto hash =
|
|
yams::crypto::SHA256Hasher::hash(std::span<const std::byte>(data.data(), data.size()));
|
|
|
|
auto path = tempDir / "commit-fail.bin";
|
|
std::ofstream file(path, std::ios::binary);
|
|
file.write(reinterpret_cast<const char*>(data.data()),
|
|
static_cast<std::streamsize>(data.size()));
|
|
file.close();
|
|
|
|
auto stored = store->store(path);
|
|
REQUIRE_FALSE(stored.has_value());
|
|
CHECK(stored.error().code == ErrorCode::TransactionFailed);
|
|
CHECK(refCounter->beginCalls == 1);
|
|
CHECK(refCounter->commitCalls == 1);
|
|
CHECK(refCounter->rollbackCalls == 1);
|
|
CHECK(refCounter->queuedIncrements > 0);
|
|
CHECK(storage->removeCalls > 0);
|
|
CHECK(std::any_of(storage->removedKeys.begin(), storage->removedKeys.end(),
|
|
[](const std::string& key) { return key.ends_with(".manifest"); }));
|
|
|
|
auto manifestExists = storage->exists(hash + ".manifest");
|
|
REQUIRE(manifestExists.has_value());
|
|
CHECK_FALSE(manifestExists.value());
|
|
|
|
auto metadata = store->getMetadata(hash);
|
|
REQUIRE_FALSE(metadata.has_value());
|
|
CHECK(metadata.error().code == ErrorCode::FileNotFound);
|
|
|
|
auto stats = store->getStats();
|
|
CHECK(stats.storeOperations == 0);
|
|
CHECK(stats.uniqueBlocks == 0);
|
|
CHECK(stats.totalUncompressedBytes == 0);
|
|
}
|
|
|
|
std::error_code ec;
|
|
fs::remove_all(tempDir, ec);
|
|
}
|
|
|
|
TEST_CASE("ContentStore: Ambiguous manifest store is reconciled without rollback",
|
|
"[api][content-store][readiness][reconcile]") {
|
|
auto tempDir = makeTempDir("content_store_ambiguous_manifest");
|
|
auto storage = std::make_shared<ScriptedStorageEngine>();
|
|
storage->failManifestStore = true;
|
|
storage->failManifestStoreAmbiguously = true;
|
|
|
|
{
|
|
ContentStoreBuilder builder;
|
|
auto built =
|
|
builder.withConfig(readinessConfig(tempDir)).withStorageEngine(storage).build();
|
|
REQUIRE(built.has_value());
|
|
auto store = std::move(built).value();
|
|
|
|
std::vector<std::byte> data(MIN_CHUNK_SIZE + 2048);
|
|
for (size_t i = 0; i < data.size(); ++i) {
|
|
data[i] = static_cast<std::byte>((i * 13) & 0xff);
|
|
}
|
|
const auto hash =
|
|
yams::crypto::SHA256Hasher::hash(std::span<const std::byte>(data.data(), data.size()));
|
|
|
|
auto stored = store->storeBytes(data, ContentMetadata{});
|
|
REQUIRE(stored.has_value());
|
|
CHECK(stored.value().contentHash == hash);
|
|
CHECK(stored.value().bytesStored > 0);
|
|
|
|
CHECK(storage->storeCalls > 0);
|
|
CHECK_FALSE(storage->empty());
|
|
|
|
auto metadata = store->getMetadata(hash);
|
|
REQUIRE(metadata.has_value());
|
|
|
|
auto stats = store->getStats();
|
|
CHECK(stats.storeOperations == 1);
|
|
CHECK(stats.totalObjects > 0);
|
|
CHECK(stats.uniqueBlocks > 0);
|
|
}
|
|
|
|
std::error_code ec;
|
|
fs::remove_all(tempDir, ec);
|
|
}
|
|
|
|
TEST_CASE("ContentStore: Ambiguous manifest store with mismatched bytes rolls back",
|
|
"[api][content-store][readiness][reconcile]") {
|
|
auto tempDir = makeTempDir("content_store_ambiguous_mismatch");
|
|
auto storage = std::make_shared<ScriptedStorageEngine>();
|
|
storage->failManifestStore = true;
|
|
storage->failManifestStoreWithMismatchedDurableBytes = true;
|
|
|
|
{
|
|
ContentStoreBuilder builder;
|
|
auto built =
|
|
builder.withConfig(readinessConfig(tempDir)).withStorageEngine(storage).build();
|
|
REQUIRE(built.has_value());
|
|
auto store = std::move(built).value();
|
|
|
|
std::vector<std::byte> data(MIN_CHUNK_SIZE + 2048);
|
|
for (size_t i = 0; i < data.size(); ++i) {
|
|
data[i] = static_cast<std::byte>((i * 17) & 0xff);
|
|
}
|
|
const auto hash =
|
|
yams::crypto::SHA256Hasher::hash(std::span<const std::byte>(data.data(), data.size()));
|
|
|
|
auto stored = store->storeBytes(data, ContentMetadata{});
|
|
REQUIRE_FALSE(stored.has_value());
|
|
CHECK(stored.error().code == ErrorCode::NetworkError);
|
|
CHECK(storage->removeCalls > 0);
|
|
|
|
auto manifestExists = storage->exists(hash + ".manifest");
|
|
REQUIRE(manifestExists.has_value());
|
|
CHECK_FALSE(manifestExists.value());
|
|
|
|
auto metadata = store->getMetadata(hash);
|
|
REQUIRE_FALSE(metadata.has_value());
|
|
|
|
auto stats = store->getStats();
|
|
CHECK(stats.storeOperations == 0);
|
|
}
|
|
|
|
std::error_code ec;
|
|
fs::remove_all(tempDir, ec);
|
|
}
|
|
|
|
// =============================================================================
|
|
// Exists Operation Tests
|
|
// =============================================================================
|
|
|
|
TEST_CASE("ContentStore: Exists operation", "[api][content-store][exists]") {
|
|
ContentStoreFixture fixture;
|
|
|
|
SECTION("Exists returns true for stored content") {
|
|
auto file = fixture.createTestFile("exists.txt", "Test content");
|
|
auto storeResult = fixture.store_->store(file);
|
|
REQUIRE(storeResult.has_value());
|
|
|
|
auto hash = storeResult.value().contentHash;
|
|
|
|
auto exists = fixture.store_->exists(hash);
|
|
REQUIRE(exists.has_value());
|
|
CHECK(exists.value());
|
|
}
|
|
|
|
SECTION("Exists returns false for non-existent content") {
|
|
auto exists = fixture.store_->exists("nonexistent_hash");
|
|
REQUIRE(exists.has_value());
|
|
CHECK_FALSE(exists.value());
|
|
}
|
|
|
|
SECTION("Exists after remove") {
|
|
auto file = fixture.createTestFile("temp.txt", "Temporary");
|
|
auto storeResult = fixture.store_->store(file);
|
|
REQUIRE(storeResult.has_value());
|
|
|
|
auto hash = storeResult.value().contentHash;
|
|
auto removeResult = fixture.store_->remove(hash);
|
|
REQUIRE(removeResult.has_value());
|
|
|
|
// Note: Due to content-addressable storage with reference counting,
|
|
// content may still exist after remove if referenced elsewhere.
|
|
// This test verifies remove operation completes successfully.
|
|
auto exists = fixture.store_->exists(hash);
|
|
REQUIRE(exists.has_value());
|
|
// Content may or may not exist depending on reference count
|
|
}
|
|
}
|
|
|
|
// =============================================================================
|
|
// Health Check Tests
|
|
// =============================================================================
|
|
|
|
TEST_CASE("ContentStore: Health check", "[api][content-store][health]") {
|
|
ContentStoreFixture fixture;
|
|
|
|
SECTION("Healthy store") {
|
|
auto health = fixture.store_->checkHealth();
|
|
|
|
CHECK(health.isHealthy);
|
|
CHECK_FALSE(health.status.empty());
|
|
CHECK(health.errors.empty());
|
|
}
|
|
}
|
|
|
|
TEST_CASE("ContentStore: Verify detects storage integrity issues",
|
|
"[api][content-store][verify][integrity]") {
|
|
ContentStoreFixture fixture;
|
|
|
|
SECTION("Verify succeeds for direct and chunked content") {
|
|
std::vector<std::byte> small(1024, std::byte{0x21});
|
|
auto smallStore = fixture.store_->storeBytes(small);
|
|
REQUIRE(smallStore.has_value());
|
|
|
|
std::vector<std::byte> large(DEFAULT_CHUNK_SIZE + 32768, std::byte{0x42});
|
|
auto largeStore = fixture.store_->storeBytes(large);
|
|
REQUIRE(largeStore.has_value());
|
|
|
|
auto verify = fixture.store_->verify();
|
|
REQUIRE(verify.has_value());
|
|
}
|
|
|
|
SECTION("Verify detects direct object corruption") {
|
|
std::vector<std::byte> data(1024, std::byte{0x33});
|
|
auto stored = fixture.store_->storeBytes(data);
|
|
REQUIRE(stored.has_value());
|
|
|
|
const auto& hash = stored.value().contentHash;
|
|
auto objectPath = fixture.testDir_ / "objects" / hash.substr(0, 2) / hash.substr(2);
|
|
{
|
|
std::fstream file(objectPath, std::ios::binary | std::ios::in | std::ios::out);
|
|
REQUIRE(static_cast<bool>(file));
|
|
file.seekp(0);
|
|
const char corrupt = static_cast<char>(0xff);
|
|
file.write(&corrupt, 1);
|
|
REQUIRE(static_cast<bool>(file));
|
|
}
|
|
|
|
auto verify = fixture.store_->verify();
|
|
REQUIRE_FALSE(verify.has_value());
|
|
CHECK(verify.error() == ErrorCode::CorruptedData);
|
|
}
|
|
|
|
SECTION("Verify detects missing chunk from manifest") {
|
|
std::vector<std::byte> large(DEFAULT_CHUNK_SIZE + 32768, std::byte{0x44});
|
|
for (size_t i = 0; i < large.size(); ++i) {
|
|
large[i] = static_cast<std::byte>(i & 0xff);
|
|
}
|
|
auto stored = fixture.store_->storeBytes(large);
|
|
REQUIRE(stored.has_value());
|
|
|
|
bool removedObject = false;
|
|
const auto objectsRoot = fixture.testDir_ / "objects";
|
|
for (const auto& entry : fs::recursive_directory_iterator(objectsRoot)) {
|
|
if (entry.is_regular_file()) {
|
|
fs::remove(entry.path());
|
|
removedObject = true;
|
|
break;
|
|
}
|
|
}
|
|
REQUIRE(removedObject);
|
|
|
|
auto verify = fixture.store_->verify();
|
|
REQUIRE_FALSE(verify.has_value());
|
|
CHECK(verify.error() == ErrorCode::CorruptedData);
|
|
}
|
|
}
|
|
|
|
// =============================================================================
|
|
// Statistics Tests
|
|
// =============================================================================
|
|
|
|
TEST_CASE("ContentStore: Statistics", "[api][content-store][stats]") {
|
|
ContentStoreFixture fixture;
|
|
|
|
SECTION("Statistics update after operations") {
|
|
auto initialStats = fixture.store_->getStats();
|
|
|
|
for (int i = 0; i < 3; ++i) {
|
|
auto file = fixture.createTestFile("stats" + std::to_string(i) + ".txt",
|
|
"Content " + std::to_string(i));
|
|
fixture.store_->store(file);
|
|
}
|
|
|
|
auto finalStats = fixture.store_->getStats();
|
|
|
|
CHECK(finalStats.totalObjects > initialStats.totalObjects);
|
|
CHECK(finalStats.totalBytes > initialStats.totalBytes);
|
|
CHECK(finalStats.storeOperations > initialStats.storeOperations);
|
|
}
|
|
}
|
|
|
|
// =============================================================================
|
|
// Progress Reporter Unit Tests
|
|
// =============================================================================
|
|
|
|
TEST_CASE("ProgressReporter: Basic functionality", "[api][progress]") {
|
|
SECTION("Basic progress") {
|
|
ProgressReporter reporter(1000);
|
|
|
|
std::vector<Progress> updates;
|
|
reporter.setCallback([&updates](const Progress& p) { updates.push_back(p); });
|
|
|
|
reporter.reportProgress(250);
|
|
reporter.reportProgress(500);
|
|
reporter.reportProgress(750);
|
|
reporter.reportProgress(1000);
|
|
|
|
REQUIRE(updates.size() >= 4);
|
|
CHECK(updates.back().bytesProcessed == 1000);
|
|
CHECK(updates.back().totalBytes == 1000);
|
|
CHECK(updates.back().percentage == Catch::Approx(100.0));
|
|
}
|
|
|
|
SECTION("Cancellation") {
|
|
ProgressReporter reporter;
|
|
|
|
CHECK_FALSE(reporter.isCancelled());
|
|
reporter.cancel();
|
|
CHECK(reporter.isCancelled());
|
|
|
|
CHECK_THROWS_AS(reporter.throwIfCancelled(), OperationCancelledException);
|
|
}
|
|
|
|
SECTION("Sub-reporter") {
|
|
ProgressReporter parent(1000);
|
|
auto sub = parent.createSubReporter(500);
|
|
|
|
sub->reportProgress(250);
|
|
CHECK(parent.getBytesProcessed() == 125); // 250/500 * 500/2 = 125
|
|
}
|
|
}
|
|
|
|
// =============================================================================
|
|
// Content Metadata Unit Tests
|
|
// =============================================================================
|
|
|
|
TEST_CASE("ContentMetadata: Serialization", "[api][metadata]") {
|
|
SECTION("JSON round-trip") {
|
|
ContentMetadata original;
|
|
original.mimeType = "application/json";
|
|
original.name = "data.json";
|
|
original.tags["category"] = "test";
|
|
original.tags["format"] = "json";
|
|
original.tags["version"] = "1.0";
|
|
|
|
auto json = original.toJson();
|
|
REQUIRE_FALSE(json.empty());
|
|
|
|
auto result = ContentMetadata::fromJson(json);
|
|
REQUIRE(result.has_value());
|
|
|
|
auto& deserialized = result.value();
|
|
CHECK(deserialized.mimeType == original.mimeType);
|
|
CHECK(deserialized.name == original.name);
|
|
CHECK(deserialized.tags == original.tags);
|
|
CHECK(deserialized.tags.at("version") == "1.0");
|
|
}
|
|
|
|
SECTION("Empty metadata serialization") {
|
|
ContentMetadata empty;
|
|
auto json = empty.toJson();
|
|
auto result = ContentMetadata::fromJson(json);
|
|
REQUIRE(result.has_value());
|
|
}
|
|
}
|
|
|
|
TEST_CASE("ContentMetadata: Tag operations", "[api][metadata][tags]") {
|
|
ContentMetadata metadata;
|
|
|
|
SECTION("Add and check tag") {
|
|
CHECK_FALSE(metadata.tags.count("test"));
|
|
metadata.tags["test"] = "true";
|
|
CHECK(metadata.tags.count("test"));
|
|
}
|
|
|
|
SECTION("Duplicate tag key overwrites") {
|
|
metadata.tags["key"] = "value1";
|
|
metadata.tags["key"] = "value2";
|
|
CHECK(metadata.tags.size() == 1);
|
|
CHECK(metadata.tags["key"] == "value2");
|
|
}
|
|
|
|
SECTION("Remove tag") {
|
|
metadata.tags["test"] = "value";
|
|
metadata.tags.erase("test");
|
|
CHECK_FALSE(metadata.tags.count("test"));
|
|
}
|
|
}
|
|
|
|
TEST_CASE("MetadataQuery: Matching", "[api][metadata][query]") {
|
|
ContentMetadata metadata;
|
|
metadata.mimeType = "text/plain";
|
|
metadata.name = "test.txt";
|
|
metadata.tags["priority"] = "important";
|
|
metadata.tags["type"] = "document";
|
|
metadata.tags["author"] = "john";
|
|
|
|
SECTION("Match by MIME type") {
|
|
MetadataQuery query;
|
|
query.mimeType = "text/plain";
|
|
CHECK(query.matches(metadata));
|
|
}
|
|
|
|
SECTION("No match for wrong MIME type") {
|
|
MetadataQuery query;
|
|
query.mimeType = "image/png";
|
|
CHECK_FALSE(query.matches(metadata));
|
|
}
|
|
|
|
SECTION("Match by required tags") {
|
|
MetadataQuery query;
|
|
query.requiredTags = {"priority"};
|
|
CHECK(query.matches(metadata));
|
|
}
|
|
|
|
SECTION("Match by any tags") {
|
|
MetadataQuery query;
|
|
query.anyTags = {"priority", "type"};
|
|
CHECK(query.matches(metadata));
|
|
}
|
|
|
|
SECTION("Exclude tags") {
|
|
MetadataQuery query;
|
|
query.excludeTags = {"priority"};
|
|
CHECK_FALSE(query.matches(metadata));
|
|
}
|
|
|
|
SECTION("Exclude non-existent tag passes") {
|
|
MetadataQuery query;
|
|
query.excludeTags = {"nonexistent"};
|
|
CHECK(query.matches(metadata));
|
|
}
|
|
}
|
|
|
|
// =============================================================================
|
|
// Builder Tests
|
|
// =============================================================================
|
|
|
|
TEST_CASE("ContentStoreBuilder: Construction", "[api][builder]") {
|
|
SECTION("Default build") {
|
|
auto tempDir =
|
|
fs::temp_directory_path() / ("builder_test_" + std::to_string(std::random_device{}()));
|
|
fs::create_directories(tempDir);
|
|
|
|
{
|
|
auto result = ContentStoreBuilder::createDefault(tempDir);
|
|
REQUIRE(result.has_value());
|
|
auto store = std::move(result).value();
|
|
CHECK(store != nullptr);
|
|
}
|
|
|
|
std::error_code ec;
|
|
fs::remove_all(tempDir, ec);
|
|
}
|
|
|
|
SECTION("Custom config") {
|
|
ContentStoreConfig config;
|
|
config.storagePath =
|
|
fs::temp_directory_path() / ("custom_test_" + std::to_string(std::random_device{}()));
|
|
config.chunkSize = 128 * 1024;
|
|
config.enableCompression = true;
|
|
config.compressionType = "zstd";
|
|
config.compressionLevel = 5;
|
|
|
|
{
|
|
ContentStoreBuilder builder;
|
|
auto result = builder.withConfig(config).build();
|
|
REQUIRE(result.has_value());
|
|
}
|
|
|
|
std::error_code ec;
|
|
fs::remove_all(config.storagePath, ec);
|
|
}
|
|
|
|
SECTION("Validation error for empty storage path") {
|
|
ContentStoreConfig config;
|
|
// Invalid: empty storage path
|
|
|
|
auto result = ContentStoreBuilder::createFromConfig(config);
|
|
CHECK_FALSE(result.has_value());
|
|
CHECK(result.error() == ErrorCode::InvalidArgument);
|
|
}
|
|
}
|
|
|
|
// =============================================================================
|
|
// Edge Cases and Stress Tests
|
|
// =============================================================================
|
|
|
|
TEST_CASE("ContentStore: Edge cases", "[api][content-store][edge]") {
|
|
ContentStoreFixture fixture;
|
|
|
|
#ifndef _WIN32
|
|
// Skip on Windows due to MAX_PATH limitations
|
|
SECTION("Very long filename") {
|
|
std::string longName(200, 'x');
|
|
longName += ".txt";
|
|
auto file = fixture.createTestFile(longName, "Long filename content");
|
|
|
|
ContentMetadata meta;
|
|
meta.name = longName;
|
|
auto result = fixture.store_->store(file, meta);
|
|
REQUIRE(result.has_value());
|
|
}
|
|
#endif
|
|
|
|
SECTION("Unicode filename") {
|
|
auto file = fixture.createTestFile("日本語ファイル.txt", "Unicode content");
|
|
auto result = fixture.store_->store(file);
|
|
REQUIRE(result.has_value());
|
|
}
|
|
|
|
SECTION("Concurrent reads of same content") {
|
|
auto file = fixture.createTestFile("concurrent.txt", "Concurrent content");
|
|
auto storeResult = fixture.store_->store(file);
|
|
REQUIRE(storeResult.has_value());
|
|
|
|
auto hash = storeResult.value().contentHash;
|
|
std::vector<std::thread> threads;
|
|
std::atomic<int> successCount{0};
|
|
|
|
for (int i = 0; i < 5; ++i) {
|
|
threads.emplace_back([&, i]() {
|
|
auto outPath = fixture.testDir_ / ("out_" + std::to_string(i) + ".txt");
|
|
auto result = fixture.store_->retrieve(hash, outPath);
|
|
if (result.has_value() && result.value().found) {
|
|
successCount++;
|
|
}
|
|
});
|
|
}
|
|
|
|
for (auto& t : threads) {
|
|
t.join();
|
|
}
|
|
|
|
CHECK(successCount.load() == 5);
|
|
}
|
|
|
|
SECTION("Concurrent stores of same file") {
|
|
auto file = fixture.createTestFile("concurrent_store.txt", "Concurrent store content");
|
|
|
|
constexpr int kThreadCount = 6;
|
|
constexpr int kStoresPerThread = 4;
|
|
std::vector<std::thread> threads;
|
|
std::atomic<int> successCount{0};
|
|
std::mutex resultMutex;
|
|
std::vector<std::string> hashes;
|
|
std::vector<std::string> errors;
|
|
hashes.reserve(kThreadCount * kStoresPerThread);
|
|
|
|
for (int i = 0; i < kThreadCount; ++i) {
|
|
threads.emplace_back([&, i]() {
|
|
ContentMetadata metadata;
|
|
metadata.name = "concurrent_store_" + std::to_string(i) + ".txt";
|
|
metadata.mimeType = "text/plain";
|
|
|
|
for (int j = 0; j < kStoresPerThread; ++j) {
|
|
auto result = fixture.store_->store(file, metadata);
|
|
if (result.has_value()) {
|
|
successCount.fetch_add(1, std::memory_order_relaxed);
|
|
std::lock_guard<std::mutex> lock(resultMutex);
|
|
hashes.push_back(result.value().contentHash);
|
|
} else {
|
|
std::lock_guard<std::mutex> lock(resultMutex);
|
|
errors.push_back("store failed in thread " + std::to_string(i) + " iter " +
|
|
std::to_string(j));
|
|
}
|
|
}
|
|
});
|
|
}
|
|
|
|
for (auto& t : threads) {
|
|
t.join();
|
|
}
|
|
|
|
INFO("errors=" << errors.size());
|
|
CHECK(successCount.load() == kThreadCount * kStoresPerThread);
|
|
REQUIRE_FALSE(hashes.empty());
|
|
for (const auto& hash : hashes) {
|
|
CHECK(hash == hashes.front());
|
|
}
|
|
}
|
|
}
|
|
|
|
TEST_CASE("ContentStore: Health check reports status", "[api][content-store][health]") {
|
|
ContentStoreFixture fixture;
|
|
|
|
auto health = fixture.store_->checkHealth();
|
|
CHECK(health.isHealthy);
|
|
CHECK(health.status.find("Healthy") != std::string::npos);
|
|
}
|
|
|
|
TEST_CASE("ContentStore: Compact operation succeeds", "[api][content-store][compact]") {
|
|
ContentStoreFixture fixture;
|
|
|
|
auto file = fixture.createTestFile("compact_test.txt", "compact me");
|
|
auto stored = fixture.store_->store(file);
|
|
REQUIRE(stored.has_value());
|
|
|
|
auto result = fixture.store_->compact(nullptr);
|
|
CHECK(result.has_value());
|
|
}
|
|
|
|
TEST_CASE("ContentStore: GarbageCollect operates on stored content", "[api][content-store][gc]") {
|
|
ContentStoreFixture fixture;
|
|
|
|
auto file = fixture.createTestFile("gc_test.txt", "content for garbage collection");
|
|
ContentMetadata metadata;
|
|
metadata.name = "gc_test.txt";
|
|
|
|
auto stored = fixture.store_->store(file, metadata);
|
|
REQUIRE(stored.has_value());
|
|
auto hash = stored.value().contentHash;
|
|
|
|
REQUIRE(fixture.store_->remove(hash).has_value());
|
|
|
|
auto result = fixture.store_->garbageCollect(nullptr);
|
|
CHECK(result.has_value());
|
|
}
|
|
TEST_CASE("ContentStore: Verify reports success on clean store", "[api][content-store][verify]") {
|
|
ContentStoreFixture fixture;
|
|
|
|
auto file = fixture.createTestFile("verify_test.txt", "verify me");
|
|
auto stored = fixture.store_->store(file);
|
|
REQUIRE(stored.has_value());
|
|
|
|
auto result = fixture.store_->verify(nullptr);
|
|
CHECK(result.has_value());
|
|
}
|
|
|
|
TEST_CASE("ContentStore: Retrieve bytes with non-existent hash returns error",
|
|
"[api][content-store][memory]") {
|
|
ContentStoreFixture fixture;
|
|
|
|
std::string nonExistentHash(64, '0');
|
|
auto result = fixture.store_->retrieveBytes(nonExistentHash);
|
|
CHECK_FALSE(result.has_value());
|
|
}
|
|
|
|
TEST_CASE("ContentStore: Retrieve bytes prefix returns subset", "[api][content-store][memory]") {
|
|
ContentStoreFixture fixture;
|
|
|
|
std::string content = "Hello, prefix test content!";
|
|
std::vector<std::byte> data;
|
|
data.reserve(content.size());
|
|
for (char c : content) {
|
|
data.push_back(static_cast<std::byte>(c));
|
|
}
|
|
|
|
ContentMetadata metadata;
|
|
metadata.name = "prefix.bin";
|
|
|
|
auto storeResult = fixture.store_->storeBytes(data, metadata);
|
|
REQUIRE(storeResult.has_value());
|
|
|
|
auto partial = fixture.store_->retrieveBytesPrefix(storeResult.value().contentHash, 5);
|
|
REQUIRE(partial.has_value());
|
|
CHECK(partial.value().size() == 5);
|
|
}
|
|
|
|
TEST_CASE("ContentStore: Retrieve stream with progress callback", "[api][content-store][stream]") {
|
|
ContentStoreFixture fixture;
|
|
|
|
std::string content = "stream with progress";
|
|
std::istringstream input(content);
|
|
ContentMetadata metadata;
|
|
metadata.name = "progress_stream.txt";
|
|
|
|
auto storeResult = fixture.store_->storeStream(input, metadata);
|
|
REQUIRE(storeResult.has_value());
|
|
|
|
std::atomic<int> progressCalls{0};
|
|
ProgressCallback progressCallback = [&](const Progress& p) {
|
|
progressCalls++;
|
|
(void)p;
|
|
};
|
|
|
|
auto outPath = fixture.testDir_ / "progress_out.txt";
|
|
auto retrieveResult =
|
|
fixture.store_->retrieve(storeResult.value().contentHash, outPath, progressCallback);
|
|
REQUIRE(retrieveResult.has_value());
|
|
CHECK(retrieveResult.value().found);
|
|
}
|
|
|
|
} // namespace yams::api::test
|