// Segment writer/reader/recovery under injected I/O faults (built with // STRAWBERRY_FAULT_INJECTION=1). A failure prints the seed; re-run with // STRAWBERRY_PROPERTY_SEED= to reproduce. // // S1 crash-prefix: after write/fsync failures, short writes, a crash at // any point and loss/garbage past the last fdatasync, recovery yields // exactly a prefix of the acknowledged groups, whole groups only, and // never loses a group covered by a successful sync. // S2 recovery is idempotent, and appending after recovery keeps S1. // S3 bit-flip isolation: a corrupt payload costs only its own block; a // corrupt block header stops the scan of an active segment there, while // a sealed segment's index skips it (only that block is lost). // S4 recovery under read faults never truncates synced data. // S5 sealing under faults/crash: either the segment is sealed with every // group and a footer that matches its contents, or it recovers as an // active segment satisfying S1; a recovered segment can be sealed later // and its footer covers the blocks written before the crash. #include "strawberry/store/segment.hpp" #include "strawberry/sys_io.hpp" #include #include #include #include #include #include #include #include #include using namespace strawberry; using namespace strawberry::store; namespace fs = std::filesystem; namespace { uint64_t seed_from_env() { if (const char* text = std::getenv("STRAWBERRY_PROPERTY_SEED")) { return std::strtoull(text, nullptr, 10); } return std::random_device{}(); } const uint64_t kSeed = seed_from_env(); #define PROPERTY(cond) \ do { \ if (!(cond)) { \ std::cerr << "PROPERTY FAILED: " #cond " (seed " << kSeed << ", " << __FILE__ << ":" \ << __LINE__ << ")" << std::endl; \ std::abort(); \ } \ } while (0) class RandomFaults final : public io::FaultInjector { public: RandomFaults(uint64_t seed, unsigned fail_per_mille, unsigned short_per_mille, bool reads) : rng_(seed), fail_(fail_per_mille), short_(short_per_mille), reads_(reads) {} io::FaultDecision decide(io::Op op) noexcept override { io::FaultDecision d; const bool is_read = op == io::Op::read || op == io::Op::pread || op == io::Op::fstat || op == io::Op::open; if (is_read != reads_) return d; const unsigned roll = static_cast(rng_() % 1000); if (roll < fail_) { static constexpr int errors[] = {EIO, ENOSPC, EINTR, EAGAIN}; d.error = errors[rng_() % 4]; ++faults; } else if (roll < fail_ + short_ && (op == io::Op::write || op == io::Op::pread || op == io::Op::read)) { d.short_bytes = static_cast(rng_() % 64); ++faults; } return d; } uint64_t faults = 0; private: std::mt19937_64 rng_; unsigned fail_; unsigned short_; bool reads_; }; struct Stats { uint64_t write_faults = 0, read_faults = 0, writer_failed = 0, truncations = 0, groups_recovered = 0, groups_lost_unsynced = 0, synced_groups = 0, header_flips = 0, payload_flips = 0, seals_ok = 0, seals_failed = 0, sealed_after_crash = 0; } g_stats; struct TempDir { fs::path path; TempDir() { path = fs::temp_directory_path() / ("sb-segment-test-" + std::to_string(::getpid())); fs::create_directories(path); } ~TempDir() { std::error_code ec; fs::remove_all(path, ec); } }; /// Group g holds `count` records whose payload is "g:::". std::vector make_group(std::mt19937_64& rng, uint64_t g) { std::vector group; const std::size_t count = 1 + rng() % 6; for (std::size_t i = 0; i < count; ++i) { std::string payload = "g:" + std::to_string(g) + ":" + std::to_string(i) + ":" + std::to_string(count) + ":"; payload.append(rng() % 3 == 0 ? rng() % 3000 : rng() % 80, static_cast('a' + g % 26)); group.push_back({RecordType::process_exec, std::move(payload)}); } return group; } struct Parsed { uint64_t group; std::size_t index; std::size_t count; }; Parsed parse(std::string_view p) { Parsed out{}; std::size_t a = p.find(':', 2), b = p.find(':', a + 1), c = p.find(':', b + 1); out.group = std::stoull(std::string(p.substr(2, a - 2))); // invariant-allow(sto): test-only, our own payloads out.index = std::stoull(std::string(p.substr(a + 1, b - a - 1))); // invariant-allow(sto): test-only out.count = std::stoull(std::string(p.substr(b + 1, c - b - 1))); // invariant-allow(sto): test-only return out; } /// Reads all records and checks they form whole groups 0..k-1 in order. /// Returns k. uint64_t read_groups_checked(const std::string& path, ScanReport* out = nullptr) { std::vector recs; ScanOptions options; const ScanReport report = scan_segment(path, options, [&](uint32_t type, std::string_view payload) { PROPERTY(type == static_cast(RecordType::process_exec)); recs.push_back(parse(payload)); }); PROPERTY(report.header_ok && report.io_error.empty()); PROPERTY(report.bad_blocks == 0 && report.malformed_records == 0); uint64_t next_group = 0; std::size_t i = 0; while (i < recs.size()) { PROPERTY(recs[i].group == next_group && recs[i].index == 0); const std::size_t count = recs[i].count; for (std::size_t k = 0; k < count; ++k) { PROPERTY(i + k < recs.size()); // whole groups only PROPERTY(recs[i + k].group == next_group && recs[i + k].index == k && recs[i + k].count == count); } i += count; ++next_group; } if (out) *out = report; return next_group; } /// Whole groups readable right now without recovery, reading the way /// recovery does (stop at the first bad block). Tolerates a torn tail. uint64_t read_groups_checked_lenient(const std::string& path) { std::vector recs; ScanOptions strict; strict.skip_corrupt_blocks = false; const ScanReport report = scan_segment(path, strict, [&](uint32_t, std::string_view payload) { recs.push_back(parse(payload)); }); PROPERTY(report.header_ok && report.io_error.empty()); uint64_t next_group = 0; for (std::size_t i = 0; i < recs.size(); i += recs[i].count) { PROPERTY(recs[i].group == next_group && recs[i].index == 0 && i + recs[i].count <= recs.size()); ++next_group; } return next_group; } void truncate_to(const std::string& path, uint64_t size) { PROPERTY(::truncate(path.c_str(), static_cast(size)) == 0); } void append_bytes(const std::string& path, const std::string& bytes) { std::ofstream f(path, std::ios::binary | std::ios::app); f.write(bytes.data(), static_cast(bytes.size())); } uint64_t file_size(const std::string& path) { return static_cast(fs::file_size(path)); } void property_crash_prefix(std::mt19937_64& rng, const TempDir& dir, int iteration) { // S1, S2, S4 const std::string path = (dir.path / ("s1-" + std::to_string(iteration) + ".log")).string(); WriterOptions options; options.target_block_raw = 512 + rng() % 8192; std::string error; auto w = SegmentWriter::create(path, {Stream::telemetry, static_cast(iteration), 1}, options, error); PROPERTY(w != nullptr); uint64_t acked = 0; // groups append_group accepted uint64_t synced_groups = 0; // groups covered by a successful sync uint64_t synced_bytes = w->synced_bytes(); { RandomFaults faults(rng(), rng() % 40, rng() % 80, /*reads=*/false); io::ScopedInjector scope(&faults); const uint64_t groups = 5 + rng() % 120; for (uint64_t g = 0; g < groups && !w->failed(); ++g) { if (!w->append_group(make_group(rng, acked))) { break; } ++acked; if (rng() % 5 == 0) { if (w->flush(rng() % 2 == 0) && w->synced_bytes() == w->file_bytes() && w->pending_raw_bytes() == 0) { synced_groups = acked; synced_bytes = w->synced_bytes(); } } } bool sealed_ok = false; if (!w->failed() && rng() % 3 == 0) { sealed_ok = w->seal(99); if (sealed_ok) { synced_groups = acked; synced_bytes = w->synced_bytes(); ++g_stats.seals_ok; } else { ++g_stats.seals_failed; } } if (w->failed()) ++g_stats.writer_failed; g_stats.write_faults += faults.faults; // Crash: pending records vanish with the process. w->abandon(); } // Power loss: anything past the last fdatasync may be gone or garbage. const uint64_t size = file_size(path); PROPERTY(size >= synced_bytes); switch (rng() % 4) { case 0: break; case 1: truncate_to(path, synced_bytes + rng() % (size - synced_bytes + 1)); break; case 2: { truncate_to(path, synced_bytes + rng() % (size - synced_bytes + 1)); append_bytes(path, std::string(rng() % 300, '\0')); // delayed-allocation zeros break; } default: { std::string junk(rng() % 300, '\0'); for (auto& c : junk) c = static_cast(rng()); append_bytes(path, junk); break; } } // S4: recovery under read faults may give up, but never cuts synced data. { RandomFaults faults(rng(), 100 + rng() % 300, rng() % 200, /*reads=*/true); io::ScopedInjector scope(&faults); const ScanReport r = recover_segment(path); (void) r; g_stats.read_faults += faults.faults; } // The guarantee is about records, not bytes: a sealed segment with junk // after its trailer reverts to active (its footer is cut and rebuilt on // the next seal), which shrinks the file below synced_bytes without // losing a record. PROPERTY(read_groups_checked_lenient(path) >= synced_groups); const ScanReport rec = recover_segment(path); PROPERTY(rec.header_ok && rec.io_error.empty()); PROPERTY(file_size(path) == rec.valid_bytes); ScanReport after; const uint64_t k = read_groups_checked(path, &after); PROPERTY(k >= synced_groups); // nothing synced was lost PROPERTY(k <= acked); // nothing unacknowledged appeared PROPERTY(!after.torn_tail); if (rec.truncated_bytes > 0) ++g_stats.truncations; g_stats.groups_recovered += k; g_stats.groups_lost_unsynced += acked - k; g_stats.synced_groups += synced_groups; // S2: idempotent, and appending after recovery continues the prefix. const ScanReport again = recover_segment(path); PROPERTY(again.truncated_bytes == 0 && again.valid_bytes == rec.valid_bytes); if (rec.sealed) { // S5: a sealed segment is complete, final, and its footer is accurate. ++g_stats.sealed_after_crash; PROPERTY(k == acked && after.records == rec.footer.records); PROPERTY(SegmentWriter::open_append(path, rec, options, error) == nullptr); uint64_t sum = 0; for (const auto& b : rec.footer.index) sum += b.records; PROPERTY(!rec.footer.index_complete || sum == rec.footer.records); return; } auto w2 = SegmentWriter::open_append(path, rec, options, error); PROPERTY(w2 != nullptr); std::mt19937_64 rng2(rng()); const uint64_t extra = 1 + rng() % 10; for (uint64_t g = 0; g < extra; ++g) PROPERTY(w2->append_group(make_group(rng2, k + g))); const bool seal_now = rng() % 2 == 0; PROPERTY(seal_now ? w2->seal(100) : w2->flush(true)); w2.reset(); ScanReport final_scan; PROPERTY(read_groups_checked(path, &final_scan) == k + extra); PROPERTY(final_scan.sealed == seal_now); if (seal_now) { // S5: the footer index covers blocks from before the crash too. uint64_t sum = 0; for (const auto& b : final_scan.footer.index) sum += b.records; PROPERTY(final_scan.footer.index_complete && sum == final_scan.records && final_scan.footer.records == final_scan.records); } } void property_bitflip(std::mt19937_64& rng, const TempDir& dir, int iteration) { // S3 const std::string path = (dir.path / ("s3-" + std::to_string(iteration) + ".log")).string(); std::string error; auto w = SegmentWriter::create(path, {Stream::evidence, 7, 1}, WriterOptions{}, error); PROPERTY(w != nullptr); const uint64_t groups = 3 + rng() % 20; std::vector block_start; // one group per block for (uint64_t g = 0; g < groups; ++g) { block_start.push_back(w->file_bytes()); PROPERTY(w->append_group(make_group(rng, g))); PROPERTY(w->flush(g + 1 == groups)); } const uint64_t end = w->file_bytes(); const bool sealed = rng() % 2 == 0; if (sealed) PROPERTY(w->seal(5)); w.reset(); block_start.push_back(end); const uint64_t victim = rng() % groups; const uint64_t begin = block_start[victim]; const uint64_t block_len = block_start[victim + 1] - begin; const bool in_header = rng() % 3 == 0; ++(in_header ? g_stats.header_flips : g_stats.payload_flips); const uint64_t pos = in_header ? begin + rng() % kBlockHeaderBytes : begin + kBlockHeaderBytes + rng() % (block_len - kBlockHeaderBytes); { std::fstream f(path, std::ios::binary | std::ios::in | std::ios::out); f.seekg(static_cast(pos)); char c = 0; f.read(&c, 1); c = static_cast(c ^ (1 << (rng() % 8))); f.seekp(static_cast(pos)); f.write(&c, 1); } std::vector seen; ScanOptions options; // skip_corrupt_blocks = true (sealed segment) const ScanReport r = scan_segment(path, options, [&](uint32_t, std::string_view payload) { const Parsed p = parse(payload); if (p.index == 0) seen.push_back(p.group); }); PROPERTY(r.header_ok && r.sealed == sealed); if (sealed) { // The index resyncs past any corrupt block, header or payload. PROPERTY(r.bad_blocks == 1 && !r.torn_tail); PROPERTY(seen.size() == groups - 1); for (uint64_t g = 0, i = 0; g < groups; ++g) { if (g == victim) continue; PROPERTY(seen[i++] == g); } } else if (in_header) { // No way to find the next block without its header: stop there. PROPERTY(r.torn_tail && r.bad_blocks == 0); PROPERTY(seen.size() == victim); } else { PROPERTY(r.bad_blocks == 1 && !r.torn_tail); PROPERTY(seen.size() == groups - 1); for (uint64_t g = 0, i = 0; g < groups; ++g) { if (g == victim) continue; PROPERTY(seen[i++] == g); } } } void test_basics(const TempDir& dir) { const std::string path = (dir.path / "basic.log").string(); std::string error; auto w = SegmentWriter::create(path, {Stream::telemetry, 42, 123}, WriterOptions{}, error); PROPERTY(w != nullptr); PROPERTY(SegmentWriter::create(path, {Stream::telemetry, 42, 123}, WriterOptions{}, error) == nullptr); // O_EXCL PROPERTY(w->append(RecordType::process_exec, "hello")); PROPERTY(w->append(RecordType::process_exec, std::string(600 * 1024, 'y'))); // one big record: its own block PROPERTY(!w->append(RecordType::process_exec, std::string(kMaxRecordBytes + 1, 'x'))); // too big, rejected PROPERTY(!w->failed()); // ...but still usable PROPERTY(w->flush(true)); PROPERTY(w->synced_bytes() == w->file_bytes()); w.reset(); ScanReport r = scan_segment(path, ScanOptions{}, nullptr); PROPERTY(r.header_ok && r.header.segment_id == 42 && r.header.created_unix_ns == 123 && r.header.stream == Stream::telemetry && r.records == 2); // A file that is not a segment is left alone by recovery. const std::string junk = (dir.path / "junk.log").string(); append_bytes(junk, "not a segment at all, definitely longer than 32 bytes"); const uint64_t before = file_size(junk); r = recover_segment(junk); PROPERTY(!r.header_ok && file_size(junk) == before); // open_append refuses a size that recovery did not produce. ScanReport fake = scan_segment(path, ScanOptions{}, nullptr); fake.valid_bytes = fake.file_bytes = 33; PROPERTY(SegmentWriter::open_append(path, fake, WriterOptions{}, error) == nullptr); // Footer record round-trip, including the timestamp convention. SegmentFooter f; f.sealed_unix_ns = 7; f.segment_id = 9; f.stream = Stream::evidence; f.records = 3; f.min_ts = -5; f.max_ts = 1700000000000000000; f.index_complete = true; f.index = {{32, 100, 3, 1, 2}}; SegmentFooter g; PROPERTY(decode_footer_record(encode_footer_record(f), g)); PROPERTY(g.segment_id == 9 && g.stream == Stream::evidence && g.min_ts == -5 && g.index.size() == 1 && g.index[0].bytes == 100 && g.index[0].max_ts == 2); int64_t ts = 0; PROPERTY(record_timestamp(encode_footer_record(f), ts) && ts == 7); } } // namespace int main(int argc, char** argv) { const int iterations = argc > 1 ? std::atoi(argv[1]) : 300; std::cout << "store segment properties, seed " << kSeed << std::endl; std::mt19937_64 rng(kSeed); TempDir dir; test_basics(dir); for (int i = 0; i < iterations; ++i) { property_crash_prefix(rng, dir, i); property_bitflip(rng, dir, i); } std::cout << "store segment: all properties hold (" << iterations << " iterations; write faults " << g_stats.write_faults << ", read faults " << g_stats.read_faults << ", writers failed " << g_stats.writer_failed << ", recoveries that cut " << g_stats.truncations << ", groups recovered " << g_stats.groups_recovered << " (synced " << g_stats.synced_groups << ", unsynced lost " << g_stats.groups_lost_unsynced << "), flips header/payload " << g_stats.header_flips << "/" << g_stats.payload_flips << ", seals ok/failed " << g_stats.seals_ok << "/" << g_stats.seals_failed << ", sealed after crash " << g_stats.sealed_after_crash << ")" << std::endl; return 0; }