Some checks failed
Build and Test / build (debug, clang) (push) Has been cancelled
Build and Test / build (debug, gcc) (push) Has been cancelled
Build and Test / build (release, clang) (push) Has been cancelled
Build and Test / build (release, gcc) (push) Has been cancelled
Build and Test / docker (push) Has been cancelled
Found by the KPI detect phase (2-3 persist failures per run):
- a finding for one rule across thousands of processes listed every
related id and outgrew the 256 KiB block limit, so the correlation
commit was refused; related lists are now capped at 200 (the most
recent indicators) with the totals in metadata
- a single record up to the 1 MiB record limit may now form a block on
its own (the decoder bound only loosens)
- a refused append records its reason ("rejected: ...") instead of
logging an empty error
- the Database no longer logs every indicator, node, edge and finding it
stores (175k lines in a 4-minute run); failures are still logged
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
440 lines
19 KiB
C++
440 lines
19 KiB
C++
// Segment writer/reader/recovery under injected I/O faults (built with
|
|
// STRAWBERRY_FAULT_INJECTION=1). A failure prints the seed; re-run with
|
|
// STRAWBERRY_PROPERTY_SEED=<seed> to reproduce.
|
|
//
|
|
// S1 crash-prefix: after write/fsync failures, short writes, a crash at
|
|
// any point and loss/garbage past the last fdatasync, recovery yields
|
|
// exactly a prefix of the acknowledged groups, whole groups only, and
|
|
// never loses a group covered by a successful sync.
|
|
// S2 recovery is idempotent, and appending after recovery keeps S1.
|
|
// S3 bit-flip isolation: a corrupt payload costs only its own block; a
|
|
// corrupt block header stops the scan of an active segment there, while
|
|
// a sealed segment's index skips it (only that block is lost).
|
|
// S4 recovery under read faults never truncates synced data.
|
|
// S5 sealing under faults/crash: either the segment is sealed with every
|
|
// group and a footer that matches its contents, or it recovers as an
|
|
// active segment satisfying S1; a recovered segment can be sealed later
|
|
// and its footer covers the blocks written before the crash.
|
|
|
|
#include "strawberry/store/segment.hpp"
|
|
#include "strawberry/sys_io.hpp"
|
|
|
|
#include <cstdint>
|
|
#include <cstdlib>
|
|
#include <filesystem>
|
|
#include <fstream>
|
|
#include <iostream>
|
|
#include <random>
|
|
#include <string>
|
|
#include <vector>
|
|
|
|
#include <unistd.h>
|
|
|
|
using namespace strawberry;
|
|
using namespace strawberry::store;
|
|
namespace fs = std::filesystem;
|
|
|
|
namespace {
|
|
|
|
uint64_t seed_from_env() {
|
|
if (const char* text = std::getenv("STRAWBERRY_PROPERTY_SEED")) {
|
|
return std::strtoull(text, nullptr, 10);
|
|
}
|
|
return std::random_device{}();
|
|
}
|
|
|
|
const uint64_t kSeed = seed_from_env();
|
|
|
|
#define PROPERTY(cond) \
|
|
do { \
|
|
if (!(cond)) { \
|
|
std::cerr << "PROPERTY FAILED: " #cond " (seed " << kSeed << ", " << __FILE__ << ":" \
|
|
<< __LINE__ << ")" << std::endl; \
|
|
std::abort(); \
|
|
} \
|
|
} while (0)
|
|
|
|
class RandomFaults final : public io::FaultInjector {
|
|
public:
|
|
RandomFaults(uint64_t seed, unsigned fail_per_mille, unsigned short_per_mille, bool reads)
|
|
: rng_(seed), fail_(fail_per_mille), short_(short_per_mille), reads_(reads) {}
|
|
|
|
io::FaultDecision decide(io::Op op) noexcept override {
|
|
io::FaultDecision d;
|
|
const bool is_read = op == io::Op::read || op == io::Op::pread || op == io::Op::fstat || op == io::Op::open;
|
|
if (is_read != reads_) return d;
|
|
const unsigned roll = static_cast<unsigned>(rng_() % 1000);
|
|
if (roll < fail_) {
|
|
static constexpr int errors[] = {EIO, ENOSPC, EINTR, EAGAIN};
|
|
d.error = errors[rng_() % 4];
|
|
++faults;
|
|
} else if (roll < fail_ + short_ && (op == io::Op::write || op == io::Op::pread || op == io::Op::read)) {
|
|
d.short_bytes = static_cast<long>(rng_() % 64);
|
|
++faults;
|
|
}
|
|
return d;
|
|
}
|
|
uint64_t faults = 0;
|
|
|
|
private:
|
|
std::mt19937_64 rng_;
|
|
unsigned fail_;
|
|
unsigned short_;
|
|
bool reads_;
|
|
};
|
|
|
|
struct Stats {
|
|
uint64_t write_faults = 0, read_faults = 0, writer_failed = 0, truncations = 0, groups_recovered = 0,
|
|
groups_lost_unsynced = 0, synced_groups = 0, header_flips = 0, payload_flips = 0, seals_ok = 0,
|
|
seals_failed = 0, sealed_after_crash = 0;
|
|
} g_stats;
|
|
|
|
struct TempDir {
|
|
fs::path path;
|
|
TempDir() {
|
|
path = fs::temp_directory_path() / ("sb-segment-test-" + std::to_string(::getpid()));
|
|
fs::create_directories(path);
|
|
}
|
|
~TempDir() {
|
|
std::error_code ec;
|
|
fs::remove_all(path, ec);
|
|
}
|
|
};
|
|
|
|
/// Group g holds `count` records whose payload is "g:<g>:<i>:<filler>".
|
|
std::vector<PendingRecord> make_group(std::mt19937_64& rng, uint64_t g) {
|
|
std::vector<PendingRecord> group;
|
|
const std::size_t count = 1 + rng() % 6;
|
|
for (std::size_t i = 0; i < count; ++i) {
|
|
std::string payload = "g:" + std::to_string(g) + ":" + std::to_string(i) + ":" + std::to_string(count) + ":";
|
|
payload.append(rng() % 3 == 0 ? rng() % 3000 : rng() % 80, static_cast<char>('a' + g % 26));
|
|
group.push_back({RecordType::process_exec, std::move(payload)});
|
|
}
|
|
return group;
|
|
}
|
|
|
|
struct Parsed {
|
|
uint64_t group;
|
|
std::size_t index;
|
|
std::size_t count;
|
|
};
|
|
|
|
Parsed parse(std::string_view p) {
|
|
Parsed out{};
|
|
std::size_t a = p.find(':', 2), b = p.find(':', a + 1), c = p.find(':', b + 1);
|
|
out.group = std::stoull(std::string(p.substr(2, a - 2))); // invariant-allow(sto): test-only, our own payloads
|
|
out.index = std::stoull(std::string(p.substr(a + 1, b - a - 1))); // invariant-allow(sto): test-only
|
|
out.count = std::stoull(std::string(p.substr(b + 1, c - b - 1))); // invariant-allow(sto): test-only
|
|
return out;
|
|
}
|
|
|
|
/// Reads all records and checks they form whole groups 0..k-1 in order.
|
|
/// Returns k.
|
|
uint64_t read_groups_checked(const std::string& path, ScanReport* out = nullptr) {
|
|
std::vector<Parsed> recs;
|
|
ScanOptions options;
|
|
const ScanReport report = scan_segment(path, options, [&](uint32_t type, std::string_view payload) {
|
|
PROPERTY(type == static_cast<uint32_t>(RecordType::process_exec));
|
|
recs.push_back(parse(payload));
|
|
});
|
|
PROPERTY(report.header_ok && report.io_error.empty());
|
|
PROPERTY(report.bad_blocks == 0 && report.malformed_records == 0);
|
|
uint64_t next_group = 0;
|
|
std::size_t i = 0;
|
|
while (i < recs.size()) {
|
|
PROPERTY(recs[i].group == next_group && recs[i].index == 0);
|
|
const std::size_t count = recs[i].count;
|
|
for (std::size_t k = 0; k < count; ++k) {
|
|
PROPERTY(i + k < recs.size()); // whole groups only
|
|
PROPERTY(recs[i + k].group == next_group && recs[i + k].index == k && recs[i + k].count == count);
|
|
}
|
|
i += count;
|
|
++next_group;
|
|
}
|
|
if (out) *out = report;
|
|
return next_group;
|
|
}
|
|
|
|
/// Whole groups readable right now without recovery, reading the way
|
|
/// recovery does (stop at the first bad block). Tolerates a torn tail.
|
|
uint64_t read_groups_checked_lenient(const std::string& path) {
|
|
std::vector<Parsed> recs;
|
|
ScanOptions strict;
|
|
strict.skip_corrupt_blocks = false;
|
|
const ScanReport report = scan_segment(path, strict, [&](uint32_t, std::string_view payload) {
|
|
recs.push_back(parse(payload));
|
|
});
|
|
PROPERTY(report.header_ok && report.io_error.empty());
|
|
uint64_t next_group = 0;
|
|
for (std::size_t i = 0; i < recs.size(); i += recs[i].count) {
|
|
PROPERTY(recs[i].group == next_group && recs[i].index == 0 && i + recs[i].count <= recs.size());
|
|
++next_group;
|
|
}
|
|
return next_group;
|
|
}
|
|
|
|
void truncate_to(const std::string& path, uint64_t size) {
|
|
PROPERTY(::truncate(path.c_str(), static_cast<off_t>(size)) == 0);
|
|
}
|
|
|
|
void append_bytes(const std::string& path, const std::string& bytes) {
|
|
std::ofstream f(path, std::ios::binary | std::ios::app);
|
|
f.write(bytes.data(), static_cast<std::streamsize>(bytes.size()));
|
|
}
|
|
|
|
uint64_t file_size(const std::string& path) {
|
|
return static_cast<uint64_t>(fs::file_size(path));
|
|
}
|
|
|
|
void property_crash_prefix(std::mt19937_64& rng, const TempDir& dir, int iteration) { // S1, S2, S4
|
|
const std::string path = (dir.path / ("s1-" + std::to_string(iteration) + ".log")).string();
|
|
WriterOptions options;
|
|
options.target_block_raw = 512 + rng() % 8192;
|
|
std::string error;
|
|
auto w = SegmentWriter::create(path, {Stream::telemetry, static_cast<uint64_t>(iteration), 1}, options, error);
|
|
PROPERTY(w != nullptr);
|
|
|
|
uint64_t acked = 0; // groups append_group accepted
|
|
uint64_t synced_groups = 0; // groups covered by a successful sync
|
|
uint64_t synced_bytes = w->synced_bytes();
|
|
{
|
|
RandomFaults faults(rng(), rng() % 40, rng() % 80, /*reads=*/false);
|
|
io::ScopedInjector scope(&faults);
|
|
const uint64_t groups = 5 + rng() % 120;
|
|
for (uint64_t g = 0; g < groups && !w->failed(); ++g) {
|
|
if (!w->append_group(make_group(rng, acked))) {
|
|
break;
|
|
}
|
|
++acked;
|
|
if (rng() % 5 == 0) {
|
|
if (w->flush(rng() % 2 == 0) && w->synced_bytes() == w->file_bytes() && w->pending_raw_bytes() == 0) {
|
|
synced_groups = acked;
|
|
synced_bytes = w->synced_bytes();
|
|
}
|
|
}
|
|
}
|
|
bool sealed_ok = false;
|
|
if (!w->failed() && rng() % 3 == 0) {
|
|
sealed_ok = w->seal(99);
|
|
if (sealed_ok) {
|
|
synced_groups = acked;
|
|
synced_bytes = w->synced_bytes();
|
|
++g_stats.seals_ok;
|
|
} else {
|
|
++g_stats.seals_failed;
|
|
}
|
|
}
|
|
if (w->failed()) ++g_stats.writer_failed;
|
|
g_stats.write_faults += faults.faults;
|
|
// Crash: pending records vanish with the process.
|
|
w->abandon();
|
|
}
|
|
|
|
// Power loss: anything past the last fdatasync may be gone or garbage.
|
|
const uint64_t size = file_size(path);
|
|
PROPERTY(size >= synced_bytes);
|
|
switch (rng() % 4) {
|
|
case 0: break;
|
|
case 1: truncate_to(path, synced_bytes + rng() % (size - synced_bytes + 1)); break;
|
|
case 2: {
|
|
truncate_to(path, synced_bytes + rng() % (size - synced_bytes + 1));
|
|
append_bytes(path, std::string(rng() % 300, '\0')); // delayed-allocation zeros
|
|
break;
|
|
}
|
|
default: {
|
|
std::string junk(rng() % 300, '\0');
|
|
for (auto& c : junk) c = static_cast<char>(rng());
|
|
append_bytes(path, junk);
|
|
break;
|
|
}
|
|
}
|
|
|
|
// S4: recovery under read faults may give up, but never cuts synced data.
|
|
{
|
|
RandomFaults faults(rng(), 100 + rng() % 300, rng() % 200, /*reads=*/true);
|
|
io::ScopedInjector scope(&faults);
|
|
const ScanReport r = recover_segment(path);
|
|
(void) r;
|
|
g_stats.read_faults += faults.faults;
|
|
}
|
|
// The guarantee is about records, not bytes: a sealed segment with junk
|
|
// after its trailer reverts to active (its footer is cut and rebuilt on
|
|
// the next seal), which shrinks the file below synced_bytes without
|
|
// losing a record.
|
|
PROPERTY(read_groups_checked_lenient(path) >= synced_groups);
|
|
|
|
const ScanReport rec = recover_segment(path);
|
|
PROPERTY(rec.header_ok && rec.io_error.empty());
|
|
PROPERTY(file_size(path) == rec.valid_bytes);
|
|
ScanReport after;
|
|
const uint64_t k = read_groups_checked(path, &after);
|
|
PROPERTY(k >= synced_groups); // nothing synced was lost
|
|
PROPERTY(k <= acked); // nothing unacknowledged appeared
|
|
PROPERTY(!after.torn_tail);
|
|
if (rec.truncated_bytes > 0) ++g_stats.truncations;
|
|
g_stats.groups_recovered += k;
|
|
g_stats.groups_lost_unsynced += acked - k;
|
|
g_stats.synced_groups += synced_groups;
|
|
|
|
// S2: idempotent, and appending after recovery continues the prefix.
|
|
const ScanReport again = recover_segment(path);
|
|
PROPERTY(again.truncated_bytes == 0 && again.valid_bytes == rec.valid_bytes);
|
|
if (rec.sealed) {
|
|
// S5: a sealed segment is complete, final, and its footer is accurate.
|
|
++g_stats.sealed_after_crash;
|
|
PROPERTY(k == acked && after.records == rec.footer.records);
|
|
PROPERTY(SegmentWriter::open_append(path, rec, options, error) == nullptr);
|
|
uint64_t sum = 0;
|
|
for (const auto& b : rec.footer.index) sum += b.records;
|
|
PROPERTY(!rec.footer.index_complete || sum == rec.footer.records);
|
|
return;
|
|
}
|
|
auto w2 = SegmentWriter::open_append(path, rec, options, error);
|
|
PROPERTY(w2 != nullptr);
|
|
std::mt19937_64 rng2(rng());
|
|
const uint64_t extra = 1 + rng() % 10;
|
|
for (uint64_t g = 0; g < extra; ++g) PROPERTY(w2->append_group(make_group(rng2, k + g)));
|
|
const bool seal_now = rng() % 2 == 0;
|
|
PROPERTY(seal_now ? w2->seal(100) : w2->flush(true));
|
|
w2.reset();
|
|
ScanReport final_scan;
|
|
PROPERTY(read_groups_checked(path, &final_scan) == k + extra);
|
|
PROPERTY(final_scan.sealed == seal_now);
|
|
if (seal_now) {
|
|
// S5: the footer index covers blocks from before the crash too.
|
|
uint64_t sum = 0;
|
|
for (const auto& b : final_scan.footer.index) sum += b.records;
|
|
PROPERTY(final_scan.footer.index_complete && sum == final_scan.records &&
|
|
final_scan.footer.records == final_scan.records);
|
|
}
|
|
}
|
|
|
|
void property_bitflip(std::mt19937_64& rng, const TempDir& dir, int iteration) { // S3
|
|
const std::string path = (dir.path / ("s3-" + std::to_string(iteration) + ".log")).string();
|
|
std::string error;
|
|
auto w = SegmentWriter::create(path, {Stream::evidence, 7, 1}, WriterOptions{}, error);
|
|
PROPERTY(w != nullptr);
|
|
const uint64_t groups = 3 + rng() % 20;
|
|
std::vector<uint64_t> block_start; // one group per block
|
|
for (uint64_t g = 0; g < groups; ++g) {
|
|
block_start.push_back(w->file_bytes());
|
|
PROPERTY(w->append_group(make_group(rng, g)));
|
|
PROPERTY(w->flush(g + 1 == groups));
|
|
}
|
|
const uint64_t end = w->file_bytes();
|
|
const bool sealed = rng() % 2 == 0;
|
|
if (sealed) PROPERTY(w->seal(5));
|
|
w.reset();
|
|
block_start.push_back(end);
|
|
|
|
const uint64_t victim = rng() % groups;
|
|
const uint64_t begin = block_start[victim];
|
|
const uint64_t block_len = block_start[victim + 1] - begin;
|
|
const bool in_header = rng() % 3 == 0;
|
|
++(in_header ? g_stats.header_flips : g_stats.payload_flips);
|
|
const uint64_t pos = in_header ? begin + rng() % kBlockHeaderBytes
|
|
: begin + kBlockHeaderBytes + rng() % (block_len - kBlockHeaderBytes);
|
|
{
|
|
std::fstream f(path, std::ios::binary | std::ios::in | std::ios::out);
|
|
f.seekg(static_cast<std::streamoff>(pos));
|
|
char c = 0;
|
|
f.read(&c, 1);
|
|
c = static_cast<char>(c ^ (1 << (rng() % 8)));
|
|
f.seekp(static_cast<std::streamoff>(pos));
|
|
f.write(&c, 1);
|
|
}
|
|
|
|
std::vector<uint64_t> seen;
|
|
ScanOptions options; // skip_corrupt_blocks = true (sealed segment)
|
|
const ScanReport r = scan_segment(path, options, [&](uint32_t, std::string_view payload) {
|
|
const Parsed p = parse(payload);
|
|
if (p.index == 0) seen.push_back(p.group);
|
|
});
|
|
PROPERTY(r.header_ok && r.sealed == sealed);
|
|
if (sealed) {
|
|
// The index resyncs past any corrupt block, header or payload.
|
|
PROPERTY(r.bad_blocks == 1 && !r.torn_tail);
|
|
PROPERTY(seen.size() == groups - 1);
|
|
for (uint64_t g = 0, i = 0; g < groups; ++g) {
|
|
if (g == victim) continue;
|
|
PROPERTY(seen[i++] == g);
|
|
}
|
|
} else if (in_header) {
|
|
// No way to find the next block without its header: stop there.
|
|
PROPERTY(r.torn_tail && r.bad_blocks == 0);
|
|
PROPERTY(seen.size() == victim);
|
|
} else {
|
|
PROPERTY(r.bad_blocks == 1 && !r.torn_tail);
|
|
PROPERTY(seen.size() == groups - 1);
|
|
for (uint64_t g = 0, i = 0; g < groups; ++g) {
|
|
if (g == victim) continue;
|
|
PROPERTY(seen[i++] == g);
|
|
}
|
|
}
|
|
}
|
|
|
|
void test_basics(const TempDir& dir) {
|
|
const std::string path = (dir.path / "basic.log").string();
|
|
std::string error;
|
|
auto w = SegmentWriter::create(path, {Stream::telemetry, 42, 123}, WriterOptions{}, error);
|
|
PROPERTY(w != nullptr);
|
|
PROPERTY(SegmentWriter::create(path, {Stream::telemetry, 42, 123}, WriterOptions{}, error) == nullptr); // O_EXCL
|
|
PROPERTY(w->append(RecordType::process_exec, "hello"));
|
|
PROPERTY(w->append(RecordType::process_exec, std::string(600 * 1024, 'y'))); // one big record: its own block
|
|
PROPERTY(!w->append(RecordType::process_exec, std::string(kMaxRecordBytes + 1, 'x'))); // too big, rejected
|
|
PROPERTY(!w->failed()); // ...but still usable
|
|
PROPERTY(w->flush(true));
|
|
PROPERTY(w->synced_bytes() == w->file_bytes());
|
|
w.reset();
|
|
ScanReport r = scan_segment(path, ScanOptions{}, nullptr);
|
|
PROPERTY(r.header_ok && r.header.segment_id == 42 && r.header.created_unix_ns == 123 &&
|
|
r.header.stream == Stream::telemetry && r.records == 2);
|
|
// A file that is not a segment is left alone by recovery.
|
|
const std::string junk = (dir.path / "junk.log").string();
|
|
append_bytes(junk, "not a segment at all, definitely longer than 32 bytes");
|
|
const uint64_t before = file_size(junk);
|
|
r = recover_segment(junk);
|
|
PROPERTY(!r.header_ok && file_size(junk) == before);
|
|
// open_append refuses a size that recovery did not produce.
|
|
ScanReport fake = scan_segment(path, ScanOptions{}, nullptr);
|
|
fake.valid_bytes = fake.file_bytes = 33;
|
|
PROPERTY(SegmentWriter::open_append(path, fake, WriterOptions{}, error) == nullptr);
|
|
// Footer record round-trip, including the timestamp convention.
|
|
SegmentFooter f;
|
|
f.sealed_unix_ns = 7;
|
|
f.segment_id = 9;
|
|
f.stream = Stream::evidence;
|
|
f.records = 3;
|
|
f.min_ts = -5;
|
|
f.max_ts = 1700000000000000000;
|
|
f.index_complete = true;
|
|
f.index = {{32, 100, 3, 1, 2}};
|
|
SegmentFooter g;
|
|
PROPERTY(decode_footer_record(encode_footer_record(f), g));
|
|
PROPERTY(g.segment_id == 9 && g.stream == Stream::evidence && g.min_ts == -5 && g.index.size() == 1 &&
|
|
g.index[0].bytes == 100 && g.index[0].max_ts == 2);
|
|
int64_t ts = 0;
|
|
PROPERTY(record_timestamp(encode_footer_record(f), ts) && ts == 7);
|
|
}
|
|
|
|
} // namespace
|
|
|
|
int main(int argc, char** argv) {
|
|
const int iterations = argc > 1 ? std::atoi(argv[1]) : 300;
|
|
std::cout << "store segment properties, seed " << kSeed << std::endl;
|
|
std::mt19937_64 rng(kSeed);
|
|
TempDir dir;
|
|
test_basics(dir);
|
|
for (int i = 0; i < iterations; ++i) {
|
|
property_crash_prefix(rng, dir, i);
|
|
property_bitflip(rng, dir, i);
|
|
}
|
|
std::cout << "store segment: all properties hold (" << iterations << " iterations; write faults "
|
|
<< g_stats.write_faults << ", read faults " << g_stats.read_faults << ", writers failed "
|
|
<< g_stats.writer_failed << ", recoveries that cut " << g_stats.truncations << ", groups recovered "
|
|
<< g_stats.groups_recovered << " (synced " << g_stats.synced_groups << ", unsynced lost "
|
|
<< g_stats.groups_lost_unsynced << "), flips header/payload " << g_stats.header_flips << "/"
|
|
<< g_stats.payload_flips << ", seals ok/failed " << g_stats.seals_ok << "/" << g_stats.seals_failed
|
|
<< ", sealed after crash " << g_stats.sealed_after_crash << ")" << std::endl;
|
|
return 0;
|
|
}
|