Some checks failed
Build and Test / build (debug, clang) (push) Has been cancelled
Build and Test / build (debug, gcc) (push) Has been cancelled
Build and Test / build (release, clang) (push) Has been cancelled
Build and Test / build (release, gcc) (push) Has been cancelled
Build and Test / docker (push) Has been cancelled
Stage 1 increment 7. - Disk floor: below max(store_min_free_mb, store_min_free_pct) free, telemetry is refused and counted (a gap record with the count and time range is written once space returns) or, with store_floor_action=evict_oldest_telemetry, the oldest sealed telemetry is deleted; evidence continues until store_hard_min_free_mb. - Retention (telemetry only, off by default): store_retention_days and store_max_mb, applied when a segment is sealed. Evidence is never deleted by retention. - Status gains a store block (floor, free space, per-stream sizes, compression ratio, drops, deletions); health reports a low disk, a failed evidence store and unsaved detections. - strawberry --store-verify checks the manifest and every segment (read-only, safe while the daemon runs; root-only telemetry is noted, not flagged). strawberry --store-compact rewrites sealed segments at zstd level 19 with the daemon stopped, never touching a damaged one. - The manifest records each sealed segment's uncompressed size. Tests L7-L11 cover the floor, gaps, retention by age and size, eviction, compaction and verify. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
362 lines
16 KiB
C++
362 lines
16 KiB
C++
// LogStore (store directory) under injected I/O faults and crashes (built
|
|
// with STRAWBERRY_FAULT_INJECTION=1). A failure prints the seed; re-run with
|
|
// STRAWBERRY_PROPERTY_SEED=<seed> to reproduce.
|
|
//
|
|
// L1 after random appends/flushes/rolls on both streams with write, fsync,
|
|
// open and rename faults, then a crash: per stream, the recovered groups
|
|
// are whole, in increasing order, include every synced group, and none
|
|
// was never acknowledged.
|
|
// L2 a second open after recovery changes nothing (same generation, same data).
|
|
// L3 an open under faults may fail, but never costs synced data.
|
|
// L4 with every manifest copy lost, open rebuilds from the segment files
|
|
// and L1 still holds.
|
|
// L5 a second writer cannot open a locked store.
|
|
// L6 sealed segments' manifest stats match their contents (records, time range).
|
|
|
|
#include "strawberry/store/log_store.hpp"
|
|
#include "strawberry/sys_io.hpp"
|
|
|
|
#include <cstdint>
|
|
#include <cstdlib>
|
|
#include <filesystem>
|
|
#include <iostream>
|
|
#include <map>
|
|
#include <random>
|
|
#include <string>
|
|
#include <vector>
|
|
|
|
#include <unistd.h>
|
|
|
|
using namespace strawberry;
|
|
using namespace strawberry::store;
|
|
namespace fs = std::filesystem;
|
|
|
|
namespace {
|
|
|
|
uint64_t seed_from_env() {
|
|
if (const char* text = std::getenv("STRAWBERRY_PROPERTY_SEED")) {
|
|
return std::strtoull(text, nullptr, 10);
|
|
}
|
|
return std::random_device{}();
|
|
}
|
|
|
|
const uint64_t kSeed = seed_from_env();
|
|
|
|
#define PROPERTY(cond) \
|
|
do { \
|
|
if (!(cond)) { \
|
|
std::cerr << "PROPERTY FAILED: " #cond " (seed " << kSeed << ", " << __FILE__ << ":" \
|
|
<< __LINE__ << ")" << std::endl; \
|
|
std::abort(); \
|
|
} \
|
|
} while (0)
|
|
|
|
class RandomFaults final : public io::FaultInjector {
|
|
public:
|
|
RandomFaults(uint64_t seed, unsigned per_mille, bool reads) : rng_(seed), per_mille_(per_mille), reads_(reads) {}
|
|
io::FaultDecision decide(io::Op op) noexcept override {
|
|
io::FaultDecision d;
|
|
const bool is_read = op == io::Op::read || op == io::Op::pread || op == io::Op::fstat;
|
|
if (is_read && !reads_) return d;
|
|
if (rng_() % 1000 < per_mille_) {
|
|
static constexpr int errors[] = {EIO, ENOSPC, EINTR};
|
|
d.error = errors[rng_() % 3];
|
|
++faults;
|
|
} else if ((op == io::Op::write || op == io::Op::pread) && rng_() % 1000 < per_mille_) {
|
|
d.short_bytes = static_cast<long>(rng_() % 100);
|
|
++faults;
|
|
}
|
|
return d;
|
|
}
|
|
uint64_t faults = 0;
|
|
|
|
private:
|
|
std::mt19937_64 rng_;
|
|
unsigned per_mille_;
|
|
bool reads_;
|
|
};
|
|
|
|
struct TempDir {
|
|
fs::path path;
|
|
TempDir() {
|
|
path = fs::temp_directory_path() / ("sb-logstore-test-" + std::to_string(::getpid()));
|
|
fs::remove_all(path);
|
|
fs::create_directories(path);
|
|
}
|
|
~TempDir() {
|
|
std::error_code ec;
|
|
fs::remove_all(path, ec);
|
|
}
|
|
};
|
|
|
|
uint64_t g_clock = 1'700'000'000'000'000'000ull;
|
|
|
|
StoreOptions test_options(const std::string& dir, std::mt19937_64& rng) {
|
|
StoreOptions o = default_store_options(dir);
|
|
o.seal_bytes = 2048 + rng() % 30000;
|
|
o.seal_blocks = 2 + rng() % 30;
|
|
o.seal_age = std::chrono::seconds(rng() % 2 ? 3600 : 5);
|
|
o.evidence.target_block_raw = 256 + rng() % 4096;
|
|
o.telemetry.target_block_raw = 256 + rng() % 4096;
|
|
o.now_ns = [] { return g_clock += 1'000'000'000ull; }; // 1 s per call
|
|
// Hermetic: never trip the disk floor on a small /tmp (L7 covers it).
|
|
o.disk_space = [](uint64_t& free_bytes, uint64_t& total_bytes) {
|
|
free_bytes = 1ull << 40;
|
|
total_bytes = 2ull << 40;
|
|
return true;
|
|
};
|
|
return o;
|
|
}
|
|
|
|
std::vector<PendingRecord> make_group(std::mt19937_64& rng, uint64_t group) {
|
|
std::vector<PendingRecord> out;
|
|
const uint64_t count = 1 + rng() % 4;
|
|
for (uint64_t i = 0; i < count; ++i) {
|
|
FieldWriter w;
|
|
w.u64(1, g_clock + i); // event time (field-1 convention)
|
|
w.u64(2, group);
|
|
w.u64(3, i);
|
|
w.u64(4, count);
|
|
w.bytes(5, std::string(rng() % 400, static_cast<char>('a' + group % 26)));
|
|
out.push_back({RecordType::process_exec, w.take()});
|
|
}
|
|
return out;
|
|
}
|
|
|
|
struct Model {
|
|
uint64_t next_group = 1; // next id to append
|
|
std::vector<uint64_t> acked; // every group append_group accepted (sorted)
|
|
std::vector<uint64_t> required; // acked and covered by a successful sync/seal
|
|
std::vector<uint64_t> pending; // acked since the last sync; lost on a failure
|
|
void synced() {
|
|
required.insert(required.end(), pending.begin(), pending.end());
|
|
pending.clear();
|
|
}
|
|
void failed() { pending.clear(); } // buffered groups may be gone (heal = crash)
|
|
};
|
|
|
|
struct Stats {
|
|
uint64_t write_faults = 0, failed_flushes = 0, heals = 0, rolls = 0, opens_under_faults = 0,
|
|
opens_failed = 0, manifests_lost = 0, orphaned = 0, truncated = 0, groups = 0, synced = 0;
|
|
} g_stats;
|
|
|
|
/// Reads a stream and checks L1. Returns the recovered group ids.
|
|
std::vector<uint64_t> check_stream(const LogStore& store, Stream stream, const Model& m) {
|
|
struct Rec { uint64_t ts, group, index, count; };
|
|
std::vector<Rec> recs;
|
|
const ScanReport r = store.scan(stream, [&](uint32_t type, std::string_view payload) {
|
|
PROPERTY(type == static_cast<uint32_t>(RecordType::process_exec));
|
|
Rec rec{};
|
|
FieldReader fr(payload);
|
|
Field f;
|
|
while (fr.next(f)) {
|
|
if (f.tag == 1) rec.ts = f.value;
|
|
if (f.tag == 2) rec.group = f.value;
|
|
if (f.tag == 3) rec.index = f.value;
|
|
if (f.tag == 4) rec.count = f.value;
|
|
}
|
|
PROPERTY(!fr.error());
|
|
recs.push_back(rec);
|
|
});
|
|
PROPERTY(r.io_error.empty() && r.header_ok && r.bad_blocks == 0 && r.malformed_records == 0);
|
|
std::vector<uint64_t> groups;
|
|
for (std::size_t i = 0; i < recs.size();) {
|
|
const uint64_t g = recs[i].group;
|
|
PROPERTY(recs[i].index == 0);
|
|
if (!(groups.empty() || g > groups.back())) {
|
|
std::cerr << "order: stream " << (stream == Stream::evidence ? "evidence" : "telemetry") << " group " << g
|
|
<< " after " << groups.back() << "; tail:";
|
|
for (std::size_t k = groups.size() > 8 ? groups.size() - 8 : 0; k < groups.size(); ++k) std::cerr << ' ' << groups[k];
|
|
std::cerr << "\nsegments:";
|
|
for (const auto& seg : store.manifest().segments)
|
|
if (seg.stream == stream) std::cerr << ' ' << seg.file << (seg.sealed ? "(s," : "(a,") << seg.records << ')';
|
|
std::cerr << std::endl;
|
|
for (const auto& seg : store.manifest().segments) {
|
|
if (seg.stream != stream) continue;
|
|
std::vector<uint64_t> gs;
|
|
scan_segment(store.dir() + (stream == Stream::evidence ? "/evidence/" : "/telemetry/") + seg.file, ScanOptions{},
|
|
[&](uint32_t, std::string_view payload) {
|
|
FieldReader fr(payload); Field f;
|
|
while (fr.next(f)) if (f.tag == 2) gs.push_back(f.value);
|
|
});
|
|
std::cerr << " " << seg.file << ":";
|
|
for (auto x : gs) std::cerr << ' ' << x;
|
|
std::cerr << std::endl;
|
|
}
|
|
}
|
|
PROPERTY(groups.empty() || g > groups.back()); // increasing, no duplicates
|
|
PROPERTY(std::binary_search(m.acked.begin(), m.acked.end(), g)); // no phantom
|
|
for (uint64_t k = 0; k < recs[i].count; ++k) {
|
|
PROPERTY(i + k < recs.size() && recs[i + k].group == g && recs[i + k].index == k); // whole
|
|
}
|
|
groups.push_back(g);
|
|
i += recs[i].count;
|
|
}
|
|
for (uint64_t g : m.required) {
|
|
PROPERTY(std::binary_search(groups.begin(), groups.end(), g)); // nothing synced lost
|
|
}
|
|
return groups;
|
|
}
|
|
|
|
void check_manifest_stats(const LogStore& store) { // L6
|
|
for (const auto& seg : store.manifest().segments) {
|
|
if (!seg.sealed) continue;
|
|
uint64_t records = 0;
|
|
int64_t lo = kNoTimestamp, hi = kNoTimestamp;
|
|
const ScanReport r = scan_segment(store.dir() + "/" + (seg.stream == Stream::evidence ? "evidence/" : "telemetry/") + seg.file,
|
|
ScanOptions{}, [&](uint32_t, std::string_view payload) {
|
|
++records;
|
|
int64_t ts = 0;
|
|
if (record_timestamp(payload, ts)) {
|
|
if (lo == kNoTimestamp || ts < lo) lo = ts;
|
|
if (hi == kNoTimestamp || ts > hi) hi = ts;
|
|
}
|
|
});
|
|
PROPERTY(r.sealed && r.bad_blocks == 0);
|
|
PROPERTY(seg.records == records && seg.min_ts == lo && seg.max_ts == hi && seg.bytes == r.file_bytes);
|
|
}
|
|
}
|
|
|
|
void run_iteration(std::mt19937_64& rng, const TempDir& tmp, int iteration) {
|
|
const std::string dir = (tmp.path / ("s" + std::to_string(iteration))).string();
|
|
StoreOptions options = test_options(dir, rng);
|
|
std::map<Stream, Model> model{{Stream::evidence, {}}, {Stream::telemetry, {}}};
|
|
std::string error;
|
|
StoreOpenReport report;
|
|
auto store = LogStore::open(options, report, error);
|
|
PROPERTY(store != nullptr && report.created);
|
|
|
|
{ // L5
|
|
StoreOpenReport r2;
|
|
std::string e2;
|
|
PROPERTY(LogStore::open(options, r2, e2) == nullptr && e2.find("locked") != std::string::npos);
|
|
}
|
|
|
|
const int rounds = 1 + static_cast<int>(rng() % 4); // crash/reopen cycles
|
|
for (int round = 0; round < rounds; ++round) {
|
|
{
|
|
RandomFaults faults(rng(), rng() % 3 == 0 ? 0 : 5 + rng() % 25, /*reads=*/false);
|
|
io::ScopedInjector scope(&faults);
|
|
const int ops = 20 + static_cast<int>(rng() % 200);
|
|
for (int op = 0; op < ops; ++op) {
|
|
const Stream stream = rng() % 2 ? Stream::evidence : Stream::telemetry;
|
|
Model& m = model[stream];
|
|
const uint32_t pick = static_cast<uint32_t>(rng() % 10);
|
|
if (pick < 6) {
|
|
const uint64_t g = m.next_group++;
|
|
if (store->append_group(stream, make_group(rng, g))) {
|
|
m.acked.push_back(g);
|
|
m.pending.push_back(g);
|
|
} else if (store->status(stream).failed) {
|
|
m.failed();
|
|
}
|
|
} else if (pick < 9) {
|
|
const bool sync = rng() % 2 == 0;
|
|
if (store->flush(stream, sync)) {
|
|
if (sync) m.synced();
|
|
} else {
|
|
++g_stats.failed_flushes;
|
|
m.failed();
|
|
}
|
|
} else {
|
|
if (store->roll(stream)) {
|
|
m.synced(); // seal = flush + fdatasync
|
|
++g_stats.rolls;
|
|
} else {
|
|
m.failed();
|
|
}
|
|
}
|
|
}
|
|
g_stats.write_faults += faults.faults;
|
|
g_stats.heals += store->status(Stream::evidence).heals + store->status(Stream::telemetry).heals;
|
|
store->simulate_crash();
|
|
store.reset();
|
|
}
|
|
|
|
if (rng() % 2 == 0) { // a torn write: partial block at the end of an active segment
|
|
const ManifestLoad load = load_manifest(dir);
|
|
for (const auto& seg : load.manifest.segments) {
|
|
if (seg.sealed || rng() % 2) continue;
|
|
const std::string path = dir + (seg.stream == Stream::evidence ? "/evidence/" : "/telemetry/") + seg.file;
|
|
if (!fs::exists(path)) continue;
|
|
std::string junk;
|
|
encode_block(junk, std::string(100 + rng() % 2000, 'z'), 1);
|
|
junk.resize(rng() % junk.size()); // cut mid-block
|
|
if (FILE* f = std::fopen(path.c_str(), "ab")) {
|
|
std::fwrite(junk.data(), 1, junk.size(), f);
|
|
std::fclose(f);
|
|
}
|
|
}
|
|
}
|
|
if (rng() % 6 == 0) { // L4: lose every manifest copy
|
|
for (const char* name : {"/MANIFEST", "/MANIFEST.tmp", "/MANIFEST.prev"}) fs::remove(dir + name);
|
|
++g_stats.manifests_lost;
|
|
}
|
|
if (rng() % 3 == 0) { // L3: open under faults
|
|
++g_stats.opens_under_faults;
|
|
RandomFaults faults(rng(), 20 + rng() % 100, /*reads=*/true);
|
|
io::ScopedInjector scope(&faults);
|
|
StoreOpenReport r;
|
|
std::string e;
|
|
auto s = LogStore::open(options, r, e);
|
|
if (s) {
|
|
s->simulate_crash();
|
|
} else {
|
|
++g_stats.opens_failed;
|
|
}
|
|
}
|
|
|
|
StoreOpenReport r;
|
|
store = LogStore::open(options, r, error);
|
|
if (!store) std::cerr << "open failed: " << error << std::endl;
|
|
PROPERTY(store != nullptr);
|
|
g_stats.orphaned += r.orphaned.size();
|
|
g_stats.truncated += r.bytes_truncated;
|
|
for (auto& [stream, m] : model) {
|
|
const auto groups = check_stream(*store, stream, m);
|
|
g_stats.groups += groups.size();
|
|
g_stats.synced += m.required.size();
|
|
// The model follows what is on disk now: survivors are durable
|
|
// (recovery synced them), the rest are gone for good.
|
|
m.acked = groups;
|
|
m.required = groups;
|
|
m.pending.clear();
|
|
}
|
|
check_manifest_stats(*store);
|
|
|
|
// L2: reopening changes nothing.
|
|
const uint64_t gen = store->manifest().generation;
|
|
store.reset(); // clean close (flush + sync)
|
|
StoreOpenReport r3;
|
|
store = LogStore::open(options, r3, error);
|
|
if (store && !(store->manifest().generation == gen && r3.orphaned.empty() && r3.bytes_truncated == 0 &&
|
|
r3.problems.empty())) {
|
|
std::cerr << "L2: gen " << gen << " -> " << store->manifest().generation << ", orphaned "
|
|
<< r3.orphaned.size() << ", truncated " << r3.bytes_truncated << ", sealed on open "
|
|
<< r3.segments_sealed_on_open << ", problems:";
|
|
for (const auto& p : r3.problems) std::cerr << " [" << p << "]";
|
|
std::cerr << std::endl;
|
|
}
|
|
PROPERTY(store != nullptr && store->manifest().generation == gen && r3.orphaned.empty() &&
|
|
r3.bytes_truncated == 0 && r3.problems.empty());
|
|
for (auto& [stream, m] : model) PROPERTY(check_stream(*store, stream, m) == m.acked);
|
|
}
|
|
store.reset();
|
|
}
|
|
|
|
} // namespace
|
|
|
|
int main(int argc, char** argv) {
|
|
const int iterations = argc > 1 ? std::atoi(argv[1]) : 60;
|
|
std::cout << "log store properties, seed " << kSeed << std::endl;
|
|
std::mt19937_64 rng(kSeed);
|
|
TempDir tmp;
|
|
for (int i = 0; i < iterations; ++i) run_iteration(rng, tmp, i);
|
|
std::cout << "log store: all properties hold (" << iterations << " stores; faults " << g_stats.write_faults
|
|
<< ", failed flushes " << g_stats.failed_flushes << ", heals " << g_stats.heals << ", rolls "
|
|
<< g_stats.rolls << ", opens under faults " << g_stats.opens_under_faults << " (failed "
|
|
<< g_stats.opens_failed << "), manifests lost " << g_stats.manifests_lost << ", orphaned "
|
|
<< g_stats.orphaned << ", bytes truncated " << g_stats.truncated << ", groups recovered "
|
|
<< g_stats.groups << ", synced checked " << g_stats.synced << ")" << std::endl;
|
|
return 0;
|
|
}
|