// Event store maintenance: disk-space floor, retention, eviction, compaction // and verify. // // L7 soft floor: telemetry refused and counted, evidence continues; a gap // record (count, time range) is written when space returns; hard floor // refuses everything // L8 retention by age and by size deletes sealed telemetry only (files and // manifest entries), never evidence; a reopen agrees // L9 floor eviction deletes the oldest sealed telemetry until above the floor // L10 compaction keeps every record in order, shrinks or keeps size, leaves // a damaged segment untouched // L11 verify reports a corrupt block, a missing file and an unlisted file #include "strawberry/store/log_store.hpp" #include #include #include #include #include #include #include #include using namespace strawberry::store; namespace fs = std::filesystem; namespace { uint64_t g_clock = 1'800'000'000'000'000'000ull; uint64_t g_free = 100ull << 30; const uint64_t kTotal = 1000ull << 30; StoreOptions options_for(const std::string& dir) { StoreOptions o = default_store_options(dir); o.now_ns = [] { return g_clock; }; o.disk_space = [](uint64_t& free_bytes, uint64_t& total_bytes) { free_bytes = g_free; total_bytes = kTotal; return true; }; o.min_free_bytes = 2ull << 30; o.min_free_pct = 5; // 50 GiB of 1000 GiB: the soft floor o.hard_min_free_bytes = 256ull << 20; return o; } void advance(uint64_t seconds) { g_clock += seconds * 1'000'000'000ull; } std::string payload(uint64_t n, std::size_t filler = 200) { FieldWriter w; w.u64(1, g_clock); w.u64(2, n); w.bytes(3, std::string(filler, static_cast('a' + n % 26))); return w.take(); } std::vector read_numbers(const LogStore& store, Stream stream, uint64_t* gaps = nullptr, uint64_t* gap_count = nullptr) { std::vector out; store.scan(stream, [&](uint32_t type, std::string_view p) { FieldReader r(p); Field f; if (type == static_cast(RecordType::gap)) { if (gaps) ++*gaps; while (r.next(f)) { if (f.tag == 2 && gap_count) *gap_count += f.value; } return; } while (r.next(f)) { if (f.tag == 2) out.push_back(f.value); } }); return out; } fs::path fresh(const std::string& name) { const fs::path p = fs::temp_directory_path() / ("sb-maint-" + std::to_string(::getpid())) / name; fs::remove_all(p); fs::create_directories(p.parent_path()); return p; } void test_floor() { // L7 const auto dir = fresh("floor").string(); g_free = 100ull << 30; StoreOpenReport report; std::string error; auto store = LogStore::open(options_for(dir), report, error); assert(store); assert(store->append(Stream::telemetry, RecordType::process_exec, payload(1))); g_free = 10ull << 30; // below 5% of 1000 GiB advance(5); assert(store->store_status().floor == FloorState::soft); assert(!store->append(Stream::telemetry, RecordType::process_exec, payload(2))); assert(!store->append_group(Stream::telemetry, {{RecordType::process_exec, payload(3)}, {RecordType::process_exec, payload(4)}})); assert(store->append(Stream::evidence, RecordType::finding_upsert, payload(100))); // evidence continues auto st = store->store_status(); assert(st.telemetry.dropped_records == 3 && !st.telemetry.failed && st.telemetry.error.starts_with("refused:")); g_free = 100ull << 20; // below the hard floor advance(5); assert(store->store_status().floor == FloorState::hard); assert(!store->append(Stream::evidence, RecordType::finding_upsert, payload(101))); g_free = 100ull << 30; // space is back advance(5); assert(store->append(Stream::telemetry, RecordType::process_exec, payload(5))); assert(store->flush(Stream::telemetry, true) && store->flush(Stream::evidence, true)); uint64_t gaps = 0, gap_count = 0; const auto telemetry = read_numbers(*store, Stream::telemetry, &gaps, &gap_count); assert((telemetry == std::vector{1, 5})); assert(gaps == 1 && gap_count == 3); // the drop is accounted, once assert(read_numbers(*store, Stream::evidence) == std::vector{100}); st = store->store_status(); assert(st.telemetry.gaps_recorded == 1 && st.telemetry.error.empty() && st.evidence.dropped_records == 1); std::cout << "L7 floor: PASSED" << std::endl; } void test_retention() { // L8 const auto dir = fresh("retention").string(); g_free = 100ull << 30; StoreOptions o = options_for(dir); o.telemetry_retention_days = 2; o.seal_age = std::chrono::hours(24 * 30); // only the explicit rolls below StoreOpenReport report; std::string error; auto store = LogStore::open(o, report, error); assert(store); uint64_t n = 0; for (int day = 0; day < 4; ++day) { // one sealed telemetry + evidence segment per day assert(store->append(Stream::telemetry, RecordType::process_exec, payload(n++))); assert(store->append(Stream::evidence, RecordType::finding_upsert, payload(n++))); assert(store->flush(Stream::telemetry, true) && store->flush(Stream::evidence, true)); assert(store->roll(Stream::evidence) && store->roll(Stream::telemetry)); advance(86'400); } // Rolled at days 0..3, now day 4. Retention ran at each roll: by the // day-3 roll only day 0 had expired; at day 4 day 1 has too. assert(store->maintain() == 1); assert(store->maintain() == 0); // idempotent auto st = store->store_status(); assert(st.telemetry.sealed_segments == 2 && st.telemetry.deleted_segments == 2); assert(st.evidence.sealed_segments == 4 && st.evidence.deleted_segments == 0); assert((read_numbers(*store, Stream::telemetry) == std::vector{4, 6})); assert((read_numbers(*store, Stream::evidence) == std::vector{1, 3, 5, 7})); std::size_t files = 0; for (const auto& e : fs::directory_iterator(dir + "/telemetry")) files += e.path().extension() == ".log"; assert(files == 3); // 2 sealed + active store.reset(); auto again = LogStore::open(o, report, error); assert(again && report.orphaned.empty()); assert((read_numbers(*again, Stream::telemetry) == std::vector{4, 6})); again.reset(); // By size: keep at most ~2 sealed telemetry segments' worth. const auto dir2 = fresh("retention-size").string(); StoreOptions o2 = options_for(dir2); auto s2 = LogStore::open(o2, report, error); assert(s2); for (uint64_t i = 0; i < 5; ++i) { assert(s2->append(Stream::telemetry, RecordType::process_exec, payload(i, 20000))); assert(s2->roll(Stream::telemetry)); } const uint64_t per_segment = s2->store_status().telemetry.sealed_bytes / 5; s2.reset(); o2.telemetry_max_bytes = per_segment * 2 + per_segment / 2; s2 = LogStore::open(o2, report, error); assert(s2); assert(s2->maintain() >= 2); const auto sizes = s2->store_status().telemetry; assert(sizes.sealed_bytes + sizes.active_bytes <= o2.telemetry_max_bytes); const auto left = read_numbers(*s2, Stream::telemetry); assert(!left.empty() && left.back() == 4); // the oldest went first std::cout << "L8 retention: PASSED" << std::endl; } void test_eviction() { // L9 const auto dir = fresh("evict").string(); g_free = 100ull << 30; StoreOptions o = options_for(dir); o.floor_action = FloorAction::evict_oldest_telemetry; // Each deleted segment "frees" 20 GiB in this fake filesystem. uint64_t deleted_seen = 0; StoreOpenReport report; std::string error; auto store = LogStore::open(o, report, error); assert(store); for (uint64_t i = 0; i < 4; ++i) { assert(store->append(Stream::telemetry, RecordType::process_exec, payload(i))); assert(store->roll(Stream::telemetry)); } store.reset(); o.disk_space = [&deleted_seen, dir](uint64_t& free_bytes, uint64_t& total_bytes) { std::size_t files = 0; for (const auto& e : fs::directory_iterator(dir + "/telemetry")) files += e.path().extension() == ".log"; deleted_seen = 5 - files; // 4 sealed + active at the start free_bytes = (10ull << 30) + deleted_seen * (20ull << 30); total_bytes = kTotal; return true; }; store = LogStore::open(o, report, error); assert(store); advance(5); // Below the floor: the append evicts the two oldest (10 + 2*20 = 50 GiB) and goes through. assert(store->append(Stream::telemetry, RecordType::process_exec, payload(9))); assert(store->store_status().telemetry.deleted_segments == 2); assert((read_numbers(*store, Stream::telemetry) == std::vector{2, 3})); // 9 is still pending std::cout << "L9 eviction: PASSED" << std::endl; } void test_compact_and_verify() { // L10, L11 const auto dir = fresh("compact").string(); g_free = 100ull << 30; StoreOpenReport report; std::string error; auto store = LogStore::open(options_for(dir), report, error); assert(store); std::vector expect; for (uint64_t seg = 0; seg < 3; ++seg) { for (uint64_t i = 0; i < 300; ++i) { const uint64_t n = seg * 1000 + i; assert(store->append(Stream::evidence, RecordType::finding_upsert, payload(n, 300))); expect.push_back(n); } assert(store->roll(Stream::evidence)); } VerifyReport clean = verify_store(dir); assert(clean.manifest_ok && clean.problems.empty() && clean.sealed == 3 && clean.records == 900); CompactReport compact; assert(store->compact(19, compact)); assert(compact.segments == 3 && compact.bytes_after <= compact.bytes_before && compact.problems.empty()); assert(read_numbers(*store, Stream::evidence) == expect); clean = verify_store(dir); assert(clean.problems.empty() && clean.records == 900); // Damage one sealed segment: verify reports it, compaction leaves it alone. const auto& segs = store->manifest().segments; std::string victim; for (const auto& s : segs) { if (s.stream == Stream::evidence && s.sealed) { victim = dir + "/evidence/" + s.file; break; } } const auto size = fs::file_size(victim); { std::fstream f(victim, std::ios::in | std::ios::out | std::ios::binary); f.seekp(static_cast(size / 2)); f.put('\x5a'); } const auto damaged_size = fs::file_size(victim); VerifyReport bad = verify_store(dir); assert(!bad.problems.empty() && bad.bad_blocks >= 1); CompactReport again; assert(!store->compact(19, again) && !again.problems.empty()); assert(fs::file_size(victim) == damaged_size); // untouched // A listed file missing, and a stray file. store.reset(); const fs::path stray = fs::path(dir) / "evidence" / "seg-999.log"; std::ofstream(stray) << "junk"; fs::rename(victim, victim + ".moved"); VerifyReport missing = verify_store(dir); bool saw_missing = false, saw_stray = false; for (const auto& p : missing.problems) saw_missing |= p.find("listed but missing") != std::string::npos; for (const auto& n : missing.notes) saw_stray |= n.find("seg-999.log") != std::string::npos; assert(saw_missing && saw_stray); std::cout << "L10 compaction + L11 verify: PASSED" << std::endl; } } // namespace int main() { test_floor(); test_retention(); test_eviction(); test_compact_and_verify(); fs::remove_all(fs::temp_directory_path() / ("sb-maint-" + std::to_string(::getpid()))); std::cout << "store maintenance: all properties hold" << std::endl; return 0; }