strawberry/tests/test_store_manifest.cpp
trvon adb40e5b62
Add compressed event store
zstd block codec (CRC32C, varint tagged records), segment files with
sealed footers and block index, crash-atomic MANIFEST, LogStore
(lock, recovery, rolls, orphaning, heal with backoff), the evidence
view, and the legacy unqlite store kept for import. Property tests
under injected faults and crashes, a differential test against the old
store, and fuzz targets for every decoder.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-25 18:17:17 -06:00

256 lines
10 KiB
C++

// MANIFEST encoding and atomic publish under injected faults (built with
// STRAWBERRY_FAULT_INJECTION=1). A failure prints the seed; re-run with
// STRAWBERRY_PROPERTY_SEED=<seed> to reproduce.
//
// M1 a crash at any I/O step of publish leaves exactly the old or the new
// generation loadable (never neither, never a mix); a publish that
// returned true always loads as the new one.
// M2 encode/decode round-trips; truncation, splicing and unsafe file names
// are rejected.
// M3 an unreadable manifest is reported as an I/O error, never as "absent".
#include "strawberry/store/manifest.hpp"
#include "strawberry/sys_io.hpp"
#include <cstdint>
#include <cstdlib>
#include <filesystem>
#include <iostream>
#include <random>
#include <string>
#include <unistd.h>
using namespace strawberry;
using namespace strawberry::store;
namespace fs = std::filesystem;
namespace {
uint64_t seed_from_env() {
if (const char* text = std::getenv("STRAWBERRY_PROPERTY_SEED")) {
return std::strtoull(text, nullptr, 10);
}
return std::random_device{}();
}
const uint64_t kSeed = seed_from_env();
#define PROPERTY(cond) \
do { \
if (!(cond)) { \
std::cerr << "PROPERTY FAILED: " #cond " (seed " << kSeed << ", " << __FILE__ << ":" \
<< __LINE__ << ")" << std::endl; \
std::abort(); \
} \
} while (0)
/// Fails every I/O operation from the k-th on (a crash at step k), except
/// close, which the seam never fails.
class CrashAt final : public io::FaultInjector {
public:
explicit CrashAt(uint64_t k, int error) : k_(k), error_(error) {}
io::FaultDecision decide(io::Op) noexcept override {
io::FaultDecision d;
if (count_++ >= k_) d.error = error_;
return d;
}
uint64_t count() const noexcept { return count_; }
private:
uint64_t k_;
int error_;
uint64_t count_ = 0;
};
class FailReads final : public io::FaultInjector {
public:
io::FaultDecision decide(io::Op op) noexcept override {
io::FaultDecision d;
if (op == io::Op::read) d.error = EIO;
return d;
}
};
Manifest random_manifest(std::mt19937_64& rng, uint64_t generation) {
Manifest m;
m.generation = generation;
m.created_unix_ns = rng();
m.next_segment_id = 1 + rng() % 100000;
const std::size_t n = rng() % 4 == 0 ? rng() % 3000 : rng() % 20; // sometimes multi-block
for (std::size_t i = 0; i < n; ++i) {
ManifestSegment s;
s.stream = rng() % 2 ? Stream::evidence : Stream::telemetry;
s.id = 1 + rng() % 1000000;
s.file = segment_file_name(s.id);
s.sealed = rng() % 2;
s.sealed_unix_ns = s.sealed ? rng() : 0;
s.bytes = rng() % (1ull << 34);
s.records = rng() % 1000000;
s.min_ts = rng() % 3 == 0 ? kNoTimestamp : static_cast<int64_t>(rng() >> 1);
s.max_ts = s.min_ts == kNoTimestamp ? kNoTimestamp : s.min_ts + static_cast<int64_t>(rng() % 1000);
s.dictionary_id = static_cast<uint32_t>(rng() % 3);
m.segments.push_back(s);
}
for (std::size_t i = 0, d = rng() % 4; i < d; ++i) {
ManifestDictionary dict;
dict.id = 1 + static_cast<uint32_t>(rng() % 100000);
dict.stream = rng() % 2 ? Stream::evidence : Stream::telemetry;
dict.file = dictionary_file_name(dict.id);
m.dictionaries.push_back(dict);
}
return m;
}
struct TempDir {
fs::path path;
TempDir() {
path = fs::temp_directory_path() / ("sb-manifest-test-" + std::to_string(::getpid()));
fs::remove_all(path);
fs::create_directories(path);
}
~TempDir() {
std::error_code ec;
fs::remove_all(path, ec);
}
};
uint64_t g_boundary_cuts = 0, g_crash_points = 0, g_loaded_new = 0, g_loaded_old = 0, g_from_temp = 0, g_from_prev = 0;
void property_crash_atomicity(std::mt19937_64& rng, const TempDir& dir) { // M1
const std::string d = dir.path.string();
fs::remove_all(dir.path);
fs::create_directories(dir.path);
const uint64_t old_gen = 1 + rng() % 1000;
const Manifest old_m = random_manifest(rng, old_gen);
std::string error;
const bool have_old = rng() % 8 != 0; // sometimes the very first publish
if (have_old) PROPERTY(publish_manifest(d, old_m, error));
const Manifest new_m = random_manifest(rng, old_gen + 1);
// Count the operations of a clean publish, then crash at each step.
uint64_t total_ops = 0;
{
const fs::path probe = dir.path / "probe";
fs::create_directories(probe);
if (have_old) PROPERTY(publish_manifest(probe.string(), old_m, error));
CrashAt never(UINT64_MAX, 0);
io::ScopedInjector scope(&never);
PROPERTY(publish_manifest(probe.string(), new_m, error));
total_ops = never.count();
fs::remove_all(probe);
}
const uint64_t k = rng() % (total_ops + 1);
bool ok = false;
{
static constexpr int errors[] = {EIO, ENOSPC, EINTR};
CrashAt crash(k, errors[rng() % 3]);
io::ScopedInjector scope(&crash);
ok = publish_manifest(d, new_m, error);
}
++g_crash_points;
const ManifestLoad load = load_manifest(d);
PROPERTY(load.io_error.empty());
if (ok) {
PROPERTY(load.manifest == new_m);
}
if (!have_old) {
// First publish: either nothing yet or the new manifest.
PROPERTY(load.source == ManifestSource::none || load.manifest == new_m);
} else {
PROPERTY(load.source != ManifestSource::none);
PROPERTY(load.manifest == old_m || load.manifest == new_m);
}
if (load.source != ManifestSource::none) {
(load.manifest == new_m ? g_loaded_new : g_loaded_old)++;
if (load.source == ManifestSource::temp) ++g_from_temp;
if (load.source == ManifestSource::previous) ++g_from_prev;
}
// The next publish after a crash (generation bumped past what loaded)
// succeeds and wins.
Manifest next = random_manifest(rng, std::max(old_gen, load.manifest.generation) + 1);
PROPERTY(publish_manifest(d, next, error));
PROPERTY(load_manifest(d).manifest == next);
}
void property_encoding(std::mt19937_64& rng) { // M2
for (int i = 0; i < 200; ++i) {
Manifest m = random_manifest(rng, rng());
if (i % 10 == 0) { // force multi-block manifests
while (m.segments.size() < 3000) {
ManifestSegment s;
s.id = m.segments.size() + 1;
s.file = segment_file_name(s.id);
m.segments.push_back(s);
}
}
const std::string bytes = encode_manifest(m);
Manifest back;
PROPERTY(decode_manifest(bytes, back) && back == m);
// Any truncation is rejected.
PROPERTY(!decode_manifest(std::string_view(bytes).substr(0, rng() % bytes.size()), back));
// Truncation exactly at a block boundary (only the counts catch it).
for (std::size_t off = kManifestHeaderBytes; off < bytes.size();) {
BlockInfo info;
std::size_t total = 0;
PROPERTY(peek_block(std::string_view(bytes).substr(off), info, total) == BlockError::ok);
off += total;
if (off < bytes.size()) {
++g_boundary_cuts;
PROPERTY(!decode_manifest(std::string_view(bytes).substr(0, off), back));
}
}
// Splicing two manifests (header of one, blocks of another) is rejected.
const Manifest other = random_manifest(rng, m.generation + 1);
std::string spliced = encode_manifest(other).substr(0, kManifestHeaderBytes) + bytes.substr(kManifestHeaderBytes);
PROPERTY(!decode_manifest(spliced, back));
}
for (const char* bad : {"../seg-1.log", "seg-1.log/..", "seg-01.log", "seg-0.log", "seg-.log", "seg-1.logx",
"/etc/passwd", "dict-0.zdict", "dict-4294967296.zdict", "seg-1 .log", "seg--1.log"}) {
PROPERTY(!valid_store_file_name(bad));
Manifest m;
m.generation = 3;
ManifestSegment s;
s.id = 1;
s.file = bad;
m.segments.push_back(s);
Manifest back;
PROPERTY(!decode_manifest(encode_manifest(m), back));
}
PROPERTY(valid_store_file_name("seg-1.log") && valid_store_file_name("dict-7.zdict"));
// A segment whose file name disagrees with its id is rejected.
Manifest m;
m.generation = 1;
ManifestSegment s;
s.id = 2;
s.file = "seg-3.log";
m.segments.push_back(s);
Manifest back;
PROPERTY(!decode_manifest(encode_manifest(m), back));
}
void property_unreadable(std::mt19937_64& rng, const TempDir& dir) { // M3
const std::string d = dir.path.string();
std::string error;
PROPERTY(publish_manifest(d, random_manifest(rng, 50), error));
FailReads fail;
io::ScopedInjector scope(&fail);
const ManifestLoad load = load_manifest(d);
PROPERTY(!load.io_error.empty() && load.source == ManifestSource::none);
}
} // namespace
int main(int argc, char** argv) {
const int iterations = argc > 1 ? std::atoi(argv[1]) : 300;
std::cout << "store manifest properties, seed " << kSeed << std::endl;
std::mt19937_64 rng(kSeed);
TempDir dir;
property_encoding(rng);
for (int i = 0; i < iterations; ++i) property_crash_atomicity(rng, dir);
property_unreadable(rng, dir);
std::cout << "store manifest: all properties hold (crash points " << g_crash_points << ", loaded new/old "
<< g_loaded_new << "/" << g_loaded_old << ", block-boundary cuts " << g_boundary_cuts << ", from temp " << g_from_temp << ", from prev " << g_from_prev
<< ")" << std::endl;
return 0;
}