// LogStore (store directory) under injected I/O faults and crashes (built // with STRAWBERRY_FAULT_INJECTION=1). A failure prints the seed; re-run with // STRAWBERRY_PROPERTY_SEED= to reproduce. // // L1 after random appends/flushes/rolls on both streams with write, fsync, // open and rename faults, then a crash: per stream, the recovered groups // are whole, in increasing order, include every synced group, and none // was never acknowledged. // L2 a second open after recovery changes nothing (same generation, same data). // L3 an open under faults may fail, but never costs synced data. // L4 with every manifest copy lost, open rebuilds from the segment files // and L1 still holds. // L5 a second writer cannot open a locked store. // L6 sealed segments' manifest stats match their contents (records, time range). #include "strawberry/store/log_store.hpp" #include "strawberry/sys_io.hpp" #include #include #include #include #include #include #include #include #include using namespace strawberry; using namespace strawberry::store; namespace fs = std::filesystem; namespace { uint64_t seed_from_env() { if (const char* text = std::getenv("STRAWBERRY_PROPERTY_SEED")) { return std::strtoull(text, nullptr, 10); } return std::random_device{}(); } const uint64_t kSeed = seed_from_env(); #define PROPERTY(cond) \ do { \ if (!(cond)) { \ std::cerr << "PROPERTY FAILED: " #cond " (seed " << kSeed << ", " << __FILE__ << ":" \ << __LINE__ << ")" << std::endl; \ std::abort(); \ } \ } while (0) class RandomFaults final : public io::FaultInjector { public: RandomFaults(uint64_t seed, unsigned per_mille, bool reads) : rng_(seed), per_mille_(per_mille), reads_(reads) {} io::FaultDecision decide(io::Op op) noexcept override { io::FaultDecision d; const bool is_read = op == io::Op::read || op == io::Op::pread || op == io::Op::fstat; if (is_read && !reads_) return d; if (rng_() % 1000 < per_mille_) { static constexpr int errors[] = {EIO, ENOSPC, EINTR}; d.error = errors[rng_() % 3]; ++faults; } else if ((op == io::Op::write || op == io::Op::pread) && rng_() % 1000 < per_mille_) { d.short_bytes = static_cast(rng_() % 100); ++faults; } return d; } uint64_t faults = 0; private: std::mt19937_64 rng_; unsigned per_mille_; bool reads_; }; struct TempDir { fs::path path; TempDir() { path = fs::temp_directory_path() / ("sb-logstore-test-" + std::to_string(::getpid())); fs::remove_all(path); fs::create_directories(path); } ~TempDir() { std::error_code ec; fs::remove_all(path, ec); } }; uint64_t g_clock = 1'700'000'000'000'000'000ull; StoreOptions test_options(const std::string& dir, std::mt19937_64& rng) { StoreOptions o = default_store_options(dir); o.seal_bytes = 2048 + rng() % 30000; o.seal_blocks = 2 + rng() % 30; o.seal_age = std::chrono::seconds(rng() % 2 ? 3600 : 5); o.evidence.target_block_raw = 256 + rng() % 4096; o.telemetry.target_block_raw = 256 + rng() % 4096; o.now_ns = [] { return g_clock += 1'000'000'000ull; }; // 1 s per call // Hermetic: never trip the disk floor on a small /tmp (L7 covers it). o.disk_space = [](uint64_t& free_bytes, uint64_t& total_bytes) { free_bytes = 1ull << 40; total_bytes = 2ull << 40; return true; }; return o; } std::vector make_group(std::mt19937_64& rng, uint64_t group) { std::vector out; const uint64_t count = 1 + rng() % 4; for (uint64_t i = 0; i < count; ++i) { FieldWriter w; w.u64(1, g_clock + i); // event time (field-1 convention) w.u64(2, group); w.u64(3, i); w.u64(4, count); w.bytes(5, std::string(rng() % 400, static_cast('a' + group % 26))); out.push_back({RecordType::process_exec, w.take()}); } return out; } struct Model { uint64_t next_group = 1; // next id to append std::vector acked; // every group append_group accepted (sorted) std::vector required; // acked and covered by a successful sync/seal std::vector pending; // acked since the last sync; lost on a failure void synced() { required.insert(required.end(), pending.begin(), pending.end()); pending.clear(); } void failed() { pending.clear(); } // buffered groups may be gone (heal = crash) }; struct Stats { uint64_t write_faults = 0, failed_flushes = 0, heals = 0, rolls = 0, opens_under_faults = 0, opens_failed = 0, manifests_lost = 0, orphaned = 0, truncated = 0, groups = 0, synced = 0; } g_stats; /// Reads a stream and checks L1. Returns the recovered group ids. std::vector check_stream(const LogStore& store, Stream stream, const Model& m) { struct Rec { uint64_t ts, group, index, count; }; std::vector recs; const ScanReport r = store.scan(stream, [&](uint32_t type, std::string_view payload) { PROPERTY(type == static_cast(RecordType::process_exec)); Rec rec{}; FieldReader fr(payload); Field f; while (fr.next(f)) { if (f.tag == 1) rec.ts = f.value; if (f.tag == 2) rec.group = f.value; if (f.tag == 3) rec.index = f.value; if (f.tag == 4) rec.count = f.value; } PROPERTY(!fr.error()); recs.push_back(rec); }); PROPERTY(r.io_error.empty() && r.header_ok && r.bad_blocks == 0 && r.malformed_records == 0); std::vector groups; for (std::size_t i = 0; i < recs.size();) { const uint64_t g = recs[i].group; PROPERTY(recs[i].index == 0); if (!(groups.empty() || g > groups.back())) { std::cerr << "order: stream " << (stream == Stream::evidence ? "evidence" : "telemetry") << " group " << g << " after " << groups.back() << "; tail:"; for (std::size_t k = groups.size() > 8 ? groups.size() - 8 : 0; k < groups.size(); ++k) std::cerr << ' ' << groups[k]; std::cerr << "\nsegments:"; for (const auto& seg : store.manifest().segments) if (seg.stream == stream) std::cerr << ' ' << seg.file << (seg.sealed ? "(s," : "(a,") << seg.records << ')'; std::cerr << std::endl; for (const auto& seg : store.manifest().segments) { if (seg.stream != stream) continue; std::vector gs; scan_segment(store.dir() + (stream == Stream::evidence ? "/evidence/" : "/telemetry/") + seg.file, ScanOptions{}, [&](uint32_t, std::string_view payload) { FieldReader fr(payload); Field f; while (fr.next(f)) if (f.tag == 2) gs.push_back(f.value); }); std::cerr << " " << seg.file << ":"; for (auto x : gs) std::cerr << ' ' << x; std::cerr << std::endl; } } PROPERTY(groups.empty() || g > groups.back()); // increasing, no duplicates PROPERTY(std::binary_search(m.acked.begin(), m.acked.end(), g)); // no phantom for (uint64_t k = 0; k < recs[i].count; ++k) { PROPERTY(i + k < recs.size() && recs[i + k].group == g && recs[i + k].index == k); // whole } groups.push_back(g); i += recs[i].count; } for (uint64_t g : m.required) { PROPERTY(std::binary_search(groups.begin(), groups.end(), g)); // nothing synced lost } return groups; } void check_manifest_stats(const LogStore& store) { // L6 for (const auto& seg : store.manifest().segments) { if (!seg.sealed) continue; uint64_t records = 0; int64_t lo = kNoTimestamp, hi = kNoTimestamp; const ScanReport r = scan_segment(store.dir() + "/" + (seg.stream == Stream::evidence ? "evidence/" : "telemetry/") + seg.file, ScanOptions{}, [&](uint32_t, std::string_view payload) { ++records; int64_t ts = 0; if (record_timestamp(payload, ts)) { if (lo == kNoTimestamp || ts < lo) lo = ts; if (hi == kNoTimestamp || ts > hi) hi = ts; } }); PROPERTY(r.sealed && r.bad_blocks == 0); PROPERTY(seg.records == records && seg.min_ts == lo && seg.max_ts == hi && seg.bytes == r.file_bytes); } } void run_iteration(std::mt19937_64& rng, const TempDir& tmp, int iteration) { const std::string dir = (tmp.path / ("s" + std::to_string(iteration))).string(); StoreOptions options = test_options(dir, rng); std::map model{{Stream::evidence, {}}, {Stream::telemetry, {}}}; std::string error; StoreOpenReport report; auto store = LogStore::open(options, report, error); PROPERTY(store != nullptr && report.created); { // L5 StoreOpenReport r2; std::string e2; PROPERTY(LogStore::open(options, r2, e2) == nullptr && e2.find("locked") != std::string::npos); } const int rounds = 1 + static_cast(rng() % 4); // crash/reopen cycles for (int round = 0; round < rounds; ++round) { { RandomFaults faults(rng(), rng() % 3 == 0 ? 0 : 5 + rng() % 25, /*reads=*/false); io::ScopedInjector scope(&faults); const int ops = 20 + static_cast(rng() % 200); for (int op = 0; op < ops; ++op) { const Stream stream = rng() % 2 ? Stream::evidence : Stream::telemetry; Model& m = model[stream]; const uint32_t pick = static_cast(rng() % 10); if (pick < 6) { const uint64_t g = m.next_group++; if (store->append_group(stream, make_group(rng, g))) { m.acked.push_back(g); m.pending.push_back(g); } else if (store->status(stream).failed) { m.failed(); } } else if (pick < 9) { const bool sync = rng() % 2 == 0; if (store->flush(stream, sync)) { if (sync) m.synced(); } else { ++g_stats.failed_flushes; m.failed(); } } else { if (store->roll(stream)) { m.synced(); // seal = flush + fdatasync ++g_stats.rolls; } else { m.failed(); } } } g_stats.write_faults += faults.faults; g_stats.heals += store->status(Stream::evidence).heals + store->status(Stream::telemetry).heals; store->simulate_crash(); store.reset(); } if (rng() % 2 == 0) { // a torn write: partial block at the end of an active segment const ManifestLoad load = load_manifest(dir); for (const auto& seg : load.manifest.segments) { if (seg.sealed || rng() % 2) continue; const std::string path = dir + (seg.stream == Stream::evidence ? "/evidence/" : "/telemetry/") + seg.file; if (!fs::exists(path)) continue; std::string junk; encode_block(junk, std::string(100 + rng() % 2000, 'z'), 1); junk.resize(rng() % junk.size()); // cut mid-block if (FILE* f = std::fopen(path.c_str(), "ab")) { std::fwrite(junk.data(), 1, junk.size(), f); std::fclose(f); } } } if (rng() % 6 == 0) { // L4: lose every manifest copy for (const char* name : {"/MANIFEST", "/MANIFEST.tmp", "/MANIFEST.prev"}) fs::remove(dir + name); ++g_stats.manifests_lost; } if (rng() % 3 == 0) { // L3: open under faults ++g_stats.opens_under_faults; RandomFaults faults(rng(), 20 + rng() % 100, /*reads=*/true); io::ScopedInjector scope(&faults); StoreOpenReport r; std::string e; auto s = LogStore::open(options, r, e); if (s) { s->simulate_crash(); } else { ++g_stats.opens_failed; } } StoreOpenReport r; store = LogStore::open(options, r, error); if (!store) std::cerr << "open failed: " << error << std::endl; PROPERTY(store != nullptr); g_stats.orphaned += r.orphaned.size(); g_stats.truncated += r.bytes_truncated; for (auto& [stream, m] : model) { const auto groups = check_stream(*store, stream, m); g_stats.groups += groups.size(); g_stats.synced += m.required.size(); // The model follows what is on disk now: survivors are durable // (recovery synced them), the rest are gone for good. m.acked = groups; m.required = groups; m.pending.clear(); } check_manifest_stats(*store); // L2: reopening changes nothing. const uint64_t gen = store->manifest().generation; store.reset(); // clean close (flush + sync) StoreOpenReport r3; store = LogStore::open(options, r3, error); if (store && !(store->manifest().generation == gen && r3.orphaned.empty() && r3.bytes_truncated == 0 && r3.problems.empty())) { std::cerr << "L2: gen " << gen << " -> " << store->manifest().generation << ", orphaned " << r3.orphaned.size() << ", truncated " << r3.bytes_truncated << ", sealed on open " << r3.segments_sealed_on_open << ", problems:"; for (const auto& p : r3.problems) std::cerr << " [" << p << "]"; std::cerr << std::endl; } PROPERTY(store != nullptr && store->manifest().generation == gen && r3.orphaned.empty() && r3.bytes_truncated == 0 && r3.problems.empty()); for (auto& [stream, m] : model) PROPERTY(check_stream(*store, stream, m) == m.acked); } store.reset(); } } // namespace int main(int argc, char** argv) { const int iterations = argc > 1 ? std::atoi(argv[1]) : 60; std::cout << "log store properties, seed " << kSeed << std::endl; std::mt19937_64 rng(kSeed); TempDir tmp; for (int i = 0; i < iterations; ++i) run_iteration(rng, tmp, i); std::cout << "log store: all properties hold (" << iterations << " stores; faults " << g_stats.write_faults << ", failed flushes " << g_stats.failed_flushes << ", heals " << g_stats.heals << ", rolls " << g_stats.rolls << ", opens under faults " << g_stats.opens_under_faults << " (failed " << g_stats.opens_failed << "), manifests lost " << g_stats.manifests_lost << ", orphaned " << g_stats.orphaned << ", bytes truncated " << g_stats.truncated << ", groups recovered " << g_stats.groups << ", synced checked " << g_stats.synced << ")" << std::endl; return 0; }