strawberry/tests/test_store_log.cpp
trvon 833ae7c423
Some checks failed
Build and Test / build (debug, clang) (push) Has been cancelled
Build and Test / build (debug, gcc) (push) Has been cancelled
Build and Test / build (release, clang) (push) Has been cancelled
Build and Test / build (release, gcc) (push) Has been cancelled
Build and Test / docker (push) Has been cancelled
Add the store's disk floor, retention, verify and compact
Stage 1 increment 7.
- Disk floor: below max(store_min_free_mb, store_min_free_pct) free,
  telemetry is refused and counted (a gap record with the count and
  time range is written once space returns) or, with
  store_floor_action=evict_oldest_telemetry, the oldest sealed
  telemetry is deleted; evidence continues until store_hard_min_free_mb.
- Retention (telemetry only, off by default): store_retention_days and
  store_max_mb, applied when a segment is sealed. Evidence is never
  deleted by retention.
- Status gains a store block (floor, free space, per-stream sizes,
  compression ratio, drops, deletions); health reports a low disk,
  a failed evidence store and unsaved detections.
- strawberry --store-verify checks the manifest and every segment
  (read-only, safe while the daemon runs; root-only telemetry is noted,
  not flagged). strawberry --store-compact rewrites sealed segments at
  zstd level 19 with the daemon stopped, never touching a damaged one.
- The manifest records each sealed segment's uncompressed size.

Tests L7-L11 cover the floor, gaps, retention by age and size,
eviction, compaction and verify.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-26 06:19:29 -06:00

362 lines
16 KiB
C++

// LogStore (store directory) under injected I/O faults and crashes (built
// with STRAWBERRY_FAULT_INJECTION=1). A failure prints the seed; re-run with
// STRAWBERRY_PROPERTY_SEED=<seed> to reproduce.
//
// L1 after random appends/flushes/rolls on both streams with write, fsync,
// open and rename faults, then a crash: per stream, the recovered groups
// are whole, in increasing order, include every synced group, and none
// was never acknowledged.
// L2 a second open after recovery changes nothing (same generation, same data).
// L3 an open under faults may fail, but never costs synced data.
// L4 with every manifest copy lost, open rebuilds from the segment files
// and L1 still holds.
// L5 a second writer cannot open a locked store.
// L6 sealed segments' manifest stats match their contents (records, time range).
#include "strawberry/store/log_store.hpp"
#include "strawberry/sys_io.hpp"
#include <cstdint>
#include <cstdlib>
#include <filesystem>
#include <iostream>
#include <map>
#include <random>
#include <string>
#include <vector>
#include <unistd.h>
using namespace strawberry;
using namespace strawberry::store;
namespace fs = std::filesystem;
namespace {
uint64_t seed_from_env() {
if (const char* text = std::getenv("STRAWBERRY_PROPERTY_SEED")) {
return std::strtoull(text, nullptr, 10);
}
return std::random_device{}();
}
const uint64_t kSeed = seed_from_env();
#define PROPERTY(cond) \
do { \
if (!(cond)) { \
std::cerr << "PROPERTY FAILED: " #cond " (seed " << kSeed << ", " << __FILE__ << ":" \
<< __LINE__ << ")" << std::endl; \
std::abort(); \
} \
} while (0)
class RandomFaults final : public io::FaultInjector {
public:
RandomFaults(uint64_t seed, unsigned per_mille, bool reads) : rng_(seed), per_mille_(per_mille), reads_(reads) {}
io::FaultDecision decide(io::Op op) noexcept override {
io::FaultDecision d;
const bool is_read = op == io::Op::read || op == io::Op::pread || op == io::Op::fstat;
if (is_read && !reads_) return d;
if (rng_() % 1000 < per_mille_) {
static constexpr int errors[] = {EIO, ENOSPC, EINTR};
d.error = errors[rng_() % 3];
++faults;
} else if ((op == io::Op::write || op == io::Op::pread) && rng_() % 1000 < per_mille_) {
d.short_bytes = static_cast<long>(rng_() % 100);
++faults;
}
return d;
}
uint64_t faults = 0;
private:
std::mt19937_64 rng_;
unsigned per_mille_;
bool reads_;
};
struct TempDir {
fs::path path;
TempDir() {
path = fs::temp_directory_path() / ("sb-logstore-test-" + std::to_string(::getpid()));
fs::remove_all(path);
fs::create_directories(path);
}
~TempDir() {
std::error_code ec;
fs::remove_all(path, ec);
}
};
uint64_t g_clock = 1'700'000'000'000'000'000ull;
StoreOptions test_options(const std::string& dir, std::mt19937_64& rng) {
StoreOptions o = default_store_options(dir);
o.seal_bytes = 2048 + rng() % 30000;
o.seal_blocks = 2 + rng() % 30;
o.seal_age = std::chrono::seconds(rng() % 2 ? 3600 : 5);
o.evidence.target_block_raw = 256 + rng() % 4096;
o.telemetry.target_block_raw = 256 + rng() % 4096;
o.now_ns = [] { return g_clock += 1'000'000'000ull; }; // 1 s per call
// Hermetic: never trip the disk floor on a small /tmp (L7 covers it).
o.disk_space = [](uint64_t& free_bytes, uint64_t& total_bytes) {
free_bytes = 1ull << 40;
total_bytes = 2ull << 40;
return true;
};
return o;
}
std::vector<PendingRecord> make_group(std::mt19937_64& rng, uint64_t group) {
std::vector<PendingRecord> out;
const uint64_t count = 1 + rng() % 4;
for (uint64_t i = 0; i < count; ++i) {
FieldWriter w;
w.u64(1, g_clock + i); // event time (field-1 convention)
w.u64(2, group);
w.u64(3, i);
w.u64(4, count);
w.bytes(5, std::string(rng() % 400, static_cast<char>('a' + group % 26)));
out.push_back({RecordType::process_exec, w.take()});
}
return out;
}
struct Model {
uint64_t next_group = 1; // next id to append
std::vector<uint64_t> acked; // every group append_group accepted (sorted)
std::vector<uint64_t> required; // acked and covered by a successful sync/seal
std::vector<uint64_t> pending; // acked since the last sync; lost on a failure
void synced() {
required.insert(required.end(), pending.begin(), pending.end());
pending.clear();
}
void failed() { pending.clear(); } // buffered groups may be gone (heal = crash)
};
struct Stats {
uint64_t write_faults = 0, failed_flushes = 0, heals = 0, rolls = 0, opens_under_faults = 0,
opens_failed = 0, manifests_lost = 0, orphaned = 0, truncated = 0, groups = 0, synced = 0;
} g_stats;
/// Reads a stream and checks L1. Returns the recovered group ids.
std::vector<uint64_t> check_stream(const LogStore& store, Stream stream, const Model& m) {
struct Rec { uint64_t ts, group, index, count; };
std::vector<Rec> recs;
const ScanReport r = store.scan(stream, [&](uint32_t type, std::string_view payload) {
PROPERTY(type == static_cast<uint32_t>(RecordType::process_exec));
Rec rec{};
FieldReader fr(payload);
Field f;
while (fr.next(f)) {
if (f.tag == 1) rec.ts = f.value;
if (f.tag == 2) rec.group = f.value;
if (f.tag == 3) rec.index = f.value;
if (f.tag == 4) rec.count = f.value;
}
PROPERTY(!fr.error());
recs.push_back(rec);
});
PROPERTY(r.io_error.empty() && r.header_ok && r.bad_blocks == 0 && r.malformed_records == 0);
std::vector<uint64_t> groups;
for (std::size_t i = 0; i < recs.size();) {
const uint64_t g = recs[i].group;
PROPERTY(recs[i].index == 0);
if (!(groups.empty() || g > groups.back())) {
std::cerr << "order: stream " << (stream == Stream::evidence ? "evidence" : "telemetry") << " group " << g
<< " after " << groups.back() << "; tail:";
for (std::size_t k = groups.size() > 8 ? groups.size() - 8 : 0; k < groups.size(); ++k) std::cerr << ' ' << groups[k];
std::cerr << "\nsegments:";
for (const auto& seg : store.manifest().segments)
if (seg.stream == stream) std::cerr << ' ' << seg.file << (seg.sealed ? "(s," : "(a,") << seg.records << ')';
std::cerr << std::endl;
for (const auto& seg : store.manifest().segments) {
if (seg.stream != stream) continue;
std::vector<uint64_t> gs;
scan_segment(store.dir() + (stream == Stream::evidence ? "/evidence/" : "/telemetry/") + seg.file, ScanOptions{},
[&](uint32_t, std::string_view payload) {
FieldReader fr(payload); Field f;
while (fr.next(f)) if (f.tag == 2) gs.push_back(f.value);
});
std::cerr << " " << seg.file << ":";
for (auto x : gs) std::cerr << ' ' << x;
std::cerr << std::endl;
}
}
PROPERTY(groups.empty() || g > groups.back()); // increasing, no duplicates
PROPERTY(std::binary_search(m.acked.begin(), m.acked.end(), g)); // no phantom
for (uint64_t k = 0; k < recs[i].count; ++k) {
PROPERTY(i + k < recs.size() && recs[i + k].group == g && recs[i + k].index == k); // whole
}
groups.push_back(g);
i += recs[i].count;
}
for (uint64_t g : m.required) {
PROPERTY(std::binary_search(groups.begin(), groups.end(), g)); // nothing synced lost
}
return groups;
}
void check_manifest_stats(const LogStore& store) { // L6
for (const auto& seg : store.manifest().segments) {
if (!seg.sealed) continue;
uint64_t records = 0;
int64_t lo = kNoTimestamp, hi = kNoTimestamp;
const ScanReport r = scan_segment(store.dir() + "/" + (seg.stream == Stream::evidence ? "evidence/" : "telemetry/") + seg.file,
ScanOptions{}, [&](uint32_t, std::string_view payload) {
++records;
int64_t ts = 0;
if (record_timestamp(payload, ts)) {
if (lo == kNoTimestamp || ts < lo) lo = ts;
if (hi == kNoTimestamp || ts > hi) hi = ts;
}
});
PROPERTY(r.sealed && r.bad_blocks == 0);
PROPERTY(seg.records == records && seg.min_ts == lo && seg.max_ts == hi && seg.bytes == r.file_bytes);
}
}
void run_iteration(std::mt19937_64& rng, const TempDir& tmp, int iteration) {
const std::string dir = (tmp.path / ("s" + std::to_string(iteration))).string();
StoreOptions options = test_options(dir, rng);
std::map<Stream, Model> model{{Stream::evidence, {}}, {Stream::telemetry, {}}};
std::string error;
StoreOpenReport report;
auto store = LogStore::open(options, report, error);
PROPERTY(store != nullptr && report.created);
{ // L5
StoreOpenReport r2;
std::string e2;
PROPERTY(LogStore::open(options, r2, e2) == nullptr && e2.find("locked") != std::string::npos);
}
const int rounds = 1 + static_cast<int>(rng() % 4); // crash/reopen cycles
for (int round = 0; round < rounds; ++round) {
{
RandomFaults faults(rng(), rng() % 3 == 0 ? 0 : 5 + rng() % 25, /*reads=*/false);
io::ScopedInjector scope(&faults);
const int ops = 20 + static_cast<int>(rng() % 200);
for (int op = 0; op < ops; ++op) {
const Stream stream = rng() % 2 ? Stream::evidence : Stream::telemetry;
Model& m = model[stream];
const uint32_t pick = static_cast<uint32_t>(rng() % 10);
if (pick < 6) {
const uint64_t g = m.next_group++;
if (store->append_group(stream, make_group(rng, g))) {
m.acked.push_back(g);
m.pending.push_back(g);
} else if (store->status(stream).failed) {
m.failed();
}
} else if (pick < 9) {
const bool sync = rng() % 2 == 0;
if (store->flush(stream, sync)) {
if (sync) m.synced();
} else {
++g_stats.failed_flushes;
m.failed();
}
} else {
if (store->roll(stream)) {
m.synced(); // seal = flush + fdatasync
++g_stats.rolls;
} else {
m.failed();
}
}
}
g_stats.write_faults += faults.faults;
g_stats.heals += store->status(Stream::evidence).heals + store->status(Stream::telemetry).heals;
store->simulate_crash();
store.reset();
}
if (rng() % 2 == 0) { // a torn write: partial block at the end of an active segment
const ManifestLoad load = load_manifest(dir);
for (const auto& seg : load.manifest.segments) {
if (seg.sealed || rng() % 2) continue;
const std::string path = dir + (seg.stream == Stream::evidence ? "/evidence/" : "/telemetry/") + seg.file;
if (!fs::exists(path)) continue;
std::string junk;
encode_block(junk, std::string(100 + rng() % 2000, 'z'), 1);
junk.resize(rng() % junk.size()); // cut mid-block
if (FILE* f = std::fopen(path.c_str(), "ab")) {
std::fwrite(junk.data(), 1, junk.size(), f);
std::fclose(f);
}
}
}
if (rng() % 6 == 0) { // L4: lose every manifest copy
for (const char* name : {"/MANIFEST", "/MANIFEST.tmp", "/MANIFEST.prev"}) fs::remove(dir + name);
++g_stats.manifests_lost;
}
if (rng() % 3 == 0) { // L3: open under faults
++g_stats.opens_under_faults;
RandomFaults faults(rng(), 20 + rng() % 100, /*reads=*/true);
io::ScopedInjector scope(&faults);
StoreOpenReport r;
std::string e;
auto s = LogStore::open(options, r, e);
if (s) {
s->simulate_crash();
} else {
++g_stats.opens_failed;
}
}
StoreOpenReport r;
store = LogStore::open(options, r, error);
if (!store) std::cerr << "open failed: " << error << std::endl;
PROPERTY(store != nullptr);
g_stats.orphaned += r.orphaned.size();
g_stats.truncated += r.bytes_truncated;
for (auto& [stream, m] : model) {
const auto groups = check_stream(*store, stream, m);
g_stats.groups += groups.size();
g_stats.synced += m.required.size();
// The model follows what is on disk now: survivors are durable
// (recovery synced them), the rest are gone for good.
m.acked = groups;
m.required = groups;
m.pending.clear();
}
check_manifest_stats(*store);
// L2: reopening changes nothing.
const uint64_t gen = store->manifest().generation;
store.reset(); // clean close (flush + sync)
StoreOpenReport r3;
store = LogStore::open(options, r3, error);
if (store && !(store->manifest().generation == gen && r3.orphaned.empty() && r3.bytes_truncated == 0 &&
r3.problems.empty())) {
std::cerr << "L2: gen " << gen << " -> " << store->manifest().generation << ", orphaned "
<< r3.orphaned.size() << ", truncated " << r3.bytes_truncated << ", sealed on open "
<< r3.segments_sealed_on_open << ", problems:";
for (const auto& p : r3.problems) std::cerr << " [" << p << "]";
std::cerr << std::endl;
}
PROPERTY(store != nullptr && store->manifest().generation == gen && r3.orphaned.empty() &&
r3.bytes_truncated == 0 && r3.problems.empty());
for (auto& [stream, m] : model) PROPERTY(check_stream(*store, stream, m) == m.acked);
}
store.reset();
}
} // namespace
int main(int argc, char** argv) {
const int iterations = argc > 1 ? std::atoi(argv[1]) : 60;
std::cout << "log store properties, seed " << kSeed << std::endl;
std::mt19937_64 rng(kSeed);
TempDir tmp;
for (int i = 0; i < iterations; ++i) run_iteration(rng, tmp, i);
std::cout << "log store: all properties hold (" << iterations << " stores; faults " << g_stats.write_faults
<< ", failed flushes " << g_stats.failed_flushes << ", heals " << g_stats.heals << ", rolls "
<< g_stats.rolls << ", opens under faults " << g_stats.opens_under_faults << " (failed "
<< g_stats.opens_failed << "), manifests lost " << g_stats.manifests_lost << ", orphaned "
<< g_stats.orphaned << ", bytes truncated " << g_stats.truncated << ", groups recovered "
<< g_stats.groups << ", synced checked " << g_stats.synced << ")" << std::endl;
return 0;
}