strawberry/tests/test_store_segment.cpp
trvon adb40e5b62
Add compressed event store
zstd block codec (CRC32C, varint tagged records), segment files with
sealed footers and block index, crash-atomic MANIFEST, LogStore
(lock, recovery, rolls, orphaning, heal with backoff), the evidence
view, and the legacy unqlite store kept for import. Property tests
under injected faults and crashes, a differential test against the old
store, and fuzz targets for every decoder.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-25 18:17:17 -06:00

439 lines
19 KiB
C++

// Segment writer/reader/recovery under injected I/O faults (built with
// STRAWBERRY_FAULT_INJECTION=1). A failure prints the seed; re-run with
// STRAWBERRY_PROPERTY_SEED=<seed> to reproduce.
//
// S1 crash-prefix: after write/fsync failures, short writes, a crash at
// any point and loss/garbage past the last fdatasync, recovery yields
// exactly a prefix of the acknowledged groups, whole groups only, and
// never loses a group covered by a successful sync.
// S2 recovery is idempotent, and appending after recovery keeps S1.
// S3 bit-flip isolation: a corrupt payload costs only its own block; a
// corrupt block header stops the scan of an active segment there, while
// a sealed segment's index skips it (only that block is lost).
// S4 recovery under read faults never truncates synced data.
// S5 sealing under faults/crash: either the segment is sealed with every
// group and a footer that matches its contents, or it recovers as an
// active segment satisfying S1; a recovered segment can be sealed later
// and its footer covers the blocks written before the crash.
#include "strawberry/store/segment.hpp"
#include "strawberry/sys_io.hpp"
#include <cstdint>
#include <cstdlib>
#include <filesystem>
#include <fstream>
#include <iostream>
#include <random>
#include <string>
#include <vector>
#include <unistd.h>
using namespace strawberry;
using namespace strawberry::store;
namespace fs = std::filesystem;
namespace {
uint64_t seed_from_env() {
if (const char* text = std::getenv("STRAWBERRY_PROPERTY_SEED")) {
return std::strtoull(text, nullptr, 10);
}
return std::random_device{}();
}
const uint64_t kSeed = seed_from_env();
#define PROPERTY(cond) \
do { \
if (!(cond)) { \
std::cerr << "PROPERTY FAILED: " #cond " (seed " << kSeed << ", " << __FILE__ << ":" \
<< __LINE__ << ")" << std::endl; \
std::abort(); \
} \
} while (0)
class RandomFaults final : public io::FaultInjector {
public:
RandomFaults(uint64_t seed, unsigned fail_per_mille, unsigned short_per_mille, bool reads)
: rng_(seed), fail_(fail_per_mille), short_(short_per_mille), reads_(reads) {}
io::FaultDecision decide(io::Op op) noexcept override {
io::FaultDecision d;
const bool is_read = op == io::Op::read || op == io::Op::pread || op == io::Op::fstat || op == io::Op::open;
if (is_read != reads_) return d;
const unsigned roll = static_cast<unsigned>(rng_() % 1000);
if (roll < fail_) {
static constexpr int errors[] = {EIO, ENOSPC, EINTR, EAGAIN};
d.error = errors[rng_() % 4];
++faults;
} else if (roll < fail_ + short_ && (op == io::Op::write || op == io::Op::pread || op == io::Op::read)) {
d.short_bytes = static_cast<long>(rng_() % 64);
++faults;
}
return d;
}
uint64_t faults = 0;
private:
std::mt19937_64 rng_;
unsigned fail_;
unsigned short_;
bool reads_;
};
struct Stats {
uint64_t write_faults = 0, read_faults = 0, writer_failed = 0, truncations = 0, groups_recovered = 0,
groups_lost_unsynced = 0, synced_groups = 0, header_flips = 0, payload_flips = 0, seals_ok = 0,
seals_failed = 0, sealed_after_crash = 0;
} g_stats;
struct TempDir {
fs::path path;
TempDir() {
path = fs::temp_directory_path() / ("sb-segment-test-" + std::to_string(::getpid()));
fs::create_directories(path);
}
~TempDir() {
std::error_code ec;
fs::remove_all(path, ec);
}
};
/// Group g holds `count` records whose payload is "g:<g>:<i>:<filler>".
std::vector<PendingRecord> make_group(std::mt19937_64& rng, uint64_t g) {
std::vector<PendingRecord> group;
const std::size_t count = 1 + rng() % 6;
for (std::size_t i = 0; i < count; ++i) {
std::string payload = "g:" + std::to_string(g) + ":" + std::to_string(i) + ":" + std::to_string(count) + ":";
payload.append(rng() % 3 == 0 ? rng() % 3000 : rng() % 80, static_cast<char>('a' + g % 26));
group.push_back({RecordType::process_exec, std::move(payload)});
}
return group;
}
struct Parsed {
uint64_t group;
std::size_t index;
std::size_t count;
};
Parsed parse(std::string_view p) {
Parsed out{};
std::size_t a = p.find(':', 2), b = p.find(':', a + 1), c = p.find(':', b + 1);
out.group = std::stoull(std::string(p.substr(2, a - 2))); // invariant-allow(sto): test-only, our own payloads
out.index = std::stoull(std::string(p.substr(a + 1, b - a - 1))); // invariant-allow(sto): test-only
out.count = std::stoull(std::string(p.substr(b + 1, c - b - 1))); // invariant-allow(sto): test-only
return out;
}
/// Reads all records and checks they form whole groups 0..k-1 in order.
/// Returns k.
uint64_t read_groups_checked(const std::string& path, ScanReport* out = nullptr) {
std::vector<Parsed> recs;
ScanOptions options;
const ScanReport report = scan_segment(path, options, [&](uint32_t type, std::string_view payload) {
PROPERTY(type == static_cast<uint32_t>(RecordType::process_exec));
recs.push_back(parse(payload));
});
PROPERTY(report.header_ok && report.io_error.empty());
PROPERTY(report.bad_blocks == 0 && report.malformed_records == 0);
uint64_t next_group = 0;
std::size_t i = 0;
while (i < recs.size()) {
PROPERTY(recs[i].group == next_group && recs[i].index == 0);
const std::size_t count = recs[i].count;
for (std::size_t k = 0; k < count; ++k) {
PROPERTY(i + k < recs.size()); // whole groups only
PROPERTY(recs[i + k].group == next_group && recs[i + k].index == k && recs[i + k].count == count);
}
i += count;
++next_group;
}
if (out) *out = report;
return next_group;
}
/// Whole groups readable right now without recovery, reading the way
/// recovery does (stop at the first bad block). Tolerates a torn tail.
uint64_t read_groups_checked_lenient(const std::string& path) {
std::vector<Parsed> recs;
ScanOptions strict;
strict.skip_corrupt_blocks = false;
const ScanReport report = scan_segment(path, strict, [&](uint32_t, std::string_view payload) {
recs.push_back(parse(payload));
});
PROPERTY(report.header_ok && report.io_error.empty());
uint64_t next_group = 0;
for (std::size_t i = 0; i < recs.size(); i += recs[i].count) {
PROPERTY(recs[i].group == next_group && recs[i].index == 0 && i + recs[i].count <= recs.size());
++next_group;
}
return next_group;
}
void truncate_to(const std::string& path, uint64_t size) {
PROPERTY(::truncate(path.c_str(), static_cast<off_t>(size)) == 0);
}
void append_bytes(const std::string& path, const std::string& bytes) {
std::ofstream f(path, std::ios::binary | std::ios::app);
f.write(bytes.data(), static_cast<std::streamsize>(bytes.size()));
}
uint64_t file_size(const std::string& path) {
return static_cast<uint64_t>(fs::file_size(path));
}
void property_crash_prefix(std::mt19937_64& rng, const TempDir& dir, int iteration) { // S1, S2, S4
const std::string path = (dir.path / ("s1-" + std::to_string(iteration) + ".log")).string();
WriterOptions options;
options.target_block_raw = 512 + rng() % 8192;
std::string error;
auto w = SegmentWriter::create(path, {Stream::telemetry, static_cast<uint64_t>(iteration), 1}, options, error);
PROPERTY(w != nullptr);
uint64_t acked = 0; // groups append_group accepted
uint64_t synced_groups = 0; // groups covered by a successful sync
uint64_t synced_bytes = w->synced_bytes();
{
RandomFaults faults(rng(), rng() % 40, rng() % 80, /*reads=*/false);
io::ScopedInjector scope(&faults);
const uint64_t groups = 5 + rng() % 120;
for (uint64_t g = 0; g < groups && !w->failed(); ++g) {
if (!w->append_group(make_group(rng, acked))) {
break;
}
++acked;
if (rng() % 5 == 0) {
if (w->flush(rng() % 2 == 0) && w->synced_bytes() == w->file_bytes() && w->pending_raw_bytes() == 0) {
synced_groups = acked;
synced_bytes = w->synced_bytes();
}
}
}
bool sealed_ok = false;
if (!w->failed() && rng() % 3 == 0) {
sealed_ok = w->seal(99);
if (sealed_ok) {
synced_groups = acked;
synced_bytes = w->synced_bytes();
++g_stats.seals_ok;
} else {
++g_stats.seals_failed;
}
}
if (w->failed()) ++g_stats.writer_failed;
g_stats.write_faults += faults.faults;
// Crash: pending records vanish with the process.
w->abandon();
}
// Power loss: anything past the last fdatasync may be gone or garbage.
const uint64_t size = file_size(path);
PROPERTY(size >= synced_bytes);
switch (rng() % 4) {
case 0: break;
case 1: truncate_to(path, synced_bytes + rng() % (size - synced_bytes + 1)); break;
case 2: {
truncate_to(path, synced_bytes + rng() % (size - synced_bytes + 1));
append_bytes(path, std::string(rng() % 300, '\0')); // delayed-allocation zeros
break;
}
default: {
std::string junk(rng() % 300, '\0');
for (auto& c : junk) c = static_cast<char>(rng());
append_bytes(path, junk);
break;
}
}
// S4: recovery under read faults may give up, but never cuts synced data.
{
RandomFaults faults(rng(), 100 + rng() % 300, rng() % 200, /*reads=*/true);
io::ScopedInjector scope(&faults);
const ScanReport r = recover_segment(path);
(void) r;
g_stats.read_faults += faults.faults;
}
// The guarantee is about records, not bytes: a sealed segment with junk
// after its trailer reverts to active (its footer is cut and rebuilt on
// the next seal), which shrinks the file below synced_bytes without
// losing a record.
PROPERTY(read_groups_checked_lenient(path) >= synced_groups);
const ScanReport rec = recover_segment(path);
PROPERTY(rec.header_ok && rec.io_error.empty());
PROPERTY(file_size(path) == rec.valid_bytes);
ScanReport after;
const uint64_t k = read_groups_checked(path, &after);
PROPERTY(k >= synced_groups); // nothing synced was lost
PROPERTY(k <= acked); // nothing unacknowledged appeared
PROPERTY(!after.torn_tail);
if (rec.truncated_bytes > 0) ++g_stats.truncations;
g_stats.groups_recovered += k;
g_stats.groups_lost_unsynced += acked - k;
g_stats.synced_groups += synced_groups;
// S2: idempotent, and appending after recovery continues the prefix.
const ScanReport again = recover_segment(path);
PROPERTY(again.truncated_bytes == 0 && again.valid_bytes == rec.valid_bytes);
if (rec.sealed) {
// S5: a sealed segment is complete, final, and its footer is accurate.
++g_stats.sealed_after_crash;
PROPERTY(k == acked && after.records == rec.footer.records);
PROPERTY(SegmentWriter::open_append(path, rec, options, error) == nullptr);
uint64_t sum = 0;
for (const auto& b : rec.footer.index) sum += b.records;
PROPERTY(!rec.footer.index_complete || sum == rec.footer.records);
return;
}
auto w2 = SegmentWriter::open_append(path, rec, options, error);
PROPERTY(w2 != nullptr);
std::mt19937_64 rng2(rng());
const uint64_t extra = 1 + rng() % 10;
for (uint64_t g = 0; g < extra; ++g) PROPERTY(w2->append_group(make_group(rng2, k + g)));
const bool seal_now = rng() % 2 == 0;
PROPERTY(seal_now ? w2->seal(100) : w2->flush(true));
w2.reset();
ScanReport final_scan;
PROPERTY(read_groups_checked(path, &final_scan) == k + extra);
PROPERTY(final_scan.sealed == seal_now);
if (seal_now) {
// S5: the footer index covers blocks from before the crash too.
uint64_t sum = 0;
for (const auto& b : final_scan.footer.index) sum += b.records;
PROPERTY(final_scan.footer.index_complete && sum == final_scan.records &&
final_scan.footer.records == final_scan.records);
}
}
void property_bitflip(std::mt19937_64& rng, const TempDir& dir, int iteration) { // S3
const std::string path = (dir.path / ("s3-" + std::to_string(iteration) + ".log")).string();
std::string error;
auto w = SegmentWriter::create(path, {Stream::evidence, 7, 1}, WriterOptions{}, error);
PROPERTY(w != nullptr);
const uint64_t groups = 3 + rng() % 20;
std::vector<uint64_t> block_start; // one group per block
for (uint64_t g = 0; g < groups; ++g) {
block_start.push_back(w->file_bytes());
PROPERTY(w->append_group(make_group(rng, g)));
PROPERTY(w->flush(g + 1 == groups));
}
const uint64_t end = w->file_bytes();
const bool sealed = rng() % 2 == 0;
if (sealed) PROPERTY(w->seal(5));
w.reset();
block_start.push_back(end);
const uint64_t victim = rng() % groups;
const uint64_t begin = block_start[victim];
const uint64_t block_len = block_start[victim + 1] - begin;
const bool in_header = rng() % 3 == 0;
++(in_header ? g_stats.header_flips : g_stats.payload_flips);
const uint64_t pos = in_header ? begin + rng() % kBlockHeaderBytes
: begin + kBlockHeaderBytes + rng() % (block_len - kBlockHeaderBytes);
{
std::fstream f(path, std::ios::binary | std::ios::in | std::ios::out);
f.seekg(static_cast<std::streamoff>(pos));
char c = 0;
f.read(&c, 1);
c = static_cast<char>(c ^ (1 << (rng() % 8)));
f.seekp(static_cast<std::streamoff>(pos));
f.write(&c, 1);
}
std::vector<uint64_t> seen;
ScanOptions options; // skip_corrupt_blocks = true (sealed segment)
const ScanReport r = scan_segment(path, options, [&](uint32_t, std::string_view payload) {
const Parsed p = parse(payload);
if (p.index == 0) seen.push_back(p.group);
});
PROPERTY(r.header_ok && r.sealed == sealed);
if (sealed) {
// The index resyncs past any corrupt block, header or payload.
PROPERTY(r.bad_blocks == 1 && !r.torn_tail);
PROPERTY(seen.size() == groups - 1);
for (uint64_t g = 0, i = 0; g < groups; ++g) {
if (g == victim) continue;
PROPERTY(seen[i++] == g);
}
} else if (in_header) {
// No way to find the next block without its header: stop there.
PROPERTY(r.torn_tail && r.bad_blocks == 0);
PROPERTY(seen.size() == victim);
} else {
PROPERTY(r.bad_blocks == 1 && !r.torn_tail);
PROPERTY(seen.size() == groups - 1);
for (uint64_t g = 0, i = 0; g < groups; ++g) {
if (g == victim) continue;
PROPERTY(seen[i++] == g);
}
}
}
void test_basics(const TempDir& dir) {
const std::string path = (dir.path / "basic.log").string();
std::string error;
auto w = SegmentWriter::create(path, {Stream::telemetry, 42, 123}, WriterOptions{}, error);
PROPERTY(w != nullptr);
PROPERTY(SegmentWriter::create(path, {Stream::telemetry, 42, 123}, WriterOptions{}, error) == nullptr); // O_EXCL
PROPERTY(w->append(RecordType::process_exec, "hello"));
PROPERTY(!w->append(RecordType::process_exec, std::string(kMaxBlockRaw, 'x'))); // too big, rejected
PROPERTY(!w->failed()); // ...but still usable
PROPERTY(w->flush(true));
PROPERTY(w->synced_bytes() == w->file_bytes());
w.reset();
ScanReport r = scan_segment(path, ScanOptions{}, nullptr);
PROPERTY(r.header_ok && r.header.segment_id == 42 && r.header.created_unix_ns == 123 &&
r.header.stream == Stream::telemetry && r.records == 1);
// A file that is not a segment is left alone by recovery.
const std::string junk = (dir.path / "junk.log").string();
append_bytes(junk, "not a segment at all, definitely longer than 32 bytes");
const uint64_t before = file_size(junk);
r = recover_segment(junk);
PROPERTY(!r.header_ok && file_size(junk) == before);
// open_append refuses a size that recovery did not produce.
ScanReport fake = scan_segment(path, ScanOptions{}, nullptr);
fake.valid_bytes = fake.file_bytes = 33;
PROPERTY(SegmentWriter::open_append(path, fake, WriterOptions{}, error) == nullptr);
// Footer record round-trip, including the timestamp convention.
SegmentFooter f;
f.sealed_unix_ns = 7;
f.segment_id = 9;
f.stream = Stream::evidence;
f.records = 3;
f.min_ts = -5;
f.max_ts = 1700000000000000000;
f.index_complete = true;
f.index = {{32, 100, 3, 1, 2}};
SegmentFooter g;
PROPERTY(decode_footer_record(encode_footer_record(f), g));
PROPERTY(g.segment_id == 9 && g.stream == Stream::evidence && g.min_ts == -5 && g.index.size() == 1 &&
g.index[0].bytes == 100 && g.index[0].max_ts == 2);
int64_t ts = 0;
PROPERTY(record_timestamp(encode_footer_record(f), ts) && ts == 7);
}
} // namespace
int main(int argc, char** argv) {
const int iterations = argc > 1 ? std::atoi(argv[1]) : 300;
std::cout << "store segment properties, seed " << kSeed << std::endl;
std::mt19937_64 rng(kSeed);
TempDir dir;
test_basics(dir);
for (int i = 0; i < iterations; ++i) {
property_crash_prefix(rng, dir, i);
property_bitflip(rng, dir, i);
}
std::cout << "store segment: all properties hold (" << iterations << " iterations; write faults "
<< g_stats.write_faults << ", read faults " << g_stats.read_faults << ", writers failed "
<< g_stats.writer_failed << ", recoveries that cut " << g_stats.truncations << ", groups recovered "
<< g_stats.groups_recovered << " (synced " << g_stats.synced_groups << ", unsynced lost "
<< g_stats.groups_lost_unsynced << "), flips header/payload " << g_stats.header_flips << "/"
<< g_stats.payload_flips << ", seals ok/failed " << g_stats.seals_ok << "/" << g_stats.seals_failed
<< ", sealed after crash " << g_stats.sealed_after_crash << ")" << std::endl;
return 0;
}