strawberry/tests/test_store_maintenance.cpp
trvon 833ae7c423
Some checks failed
Build and Test / build (debug, clang) (push) Has been cancelled
Build and Test / build (debug, gcc) (push) Has been cancelled
Build and Test / build (release, clang) (push) Has been cancelled
Build and Test / build (release, gcc) (push) Has been cancelled
Build and Test / docker (push) Has been cancelled
Add the store's disk floor, retention, verify and compact
Stage 1 increment 7.
- Disk floor: below max(store_min_free_mb, store_min_free_pct) free,
  telemetry is refused and counted (a gap record with the count and
  time range is written once space returns) or, with
  store_floor_action=evict_oldest_telemetry, the oldest sealed
  telemetry is deleted; evidence continues until store_hard_min_free_mb.
- Retention (telemetry only, off by default): store_retention_days and
  store_max_mb, applied when a segment is sealed. Evidence is never
  deleted by retention.
- Status gains a store block (floor, free space, per-stream sizes,
  compression ratio, drops, deletions); health reports a low disk,
  a failed evidence store and unsaved detections.
- strawberry --store-verify checks the manifest and every segment
  (read-only, safe while the daemon runs; root-only telemetry is noted,
  not flagged). strawberry --store-compact rewrites sealed segments at
  zstd level 19 with the daemon stopped, never touching a damaged one.
- The manifest records each sealed segment's uncompressed size.

Tests L7-L11 cover the floor, gaps, retention by age and size,
eviction, compaction and verify.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-26 06:19:29 -06:00

287 lines
12 KiB
C++

// Event store maintenance: disk-space floor, retention, eviction, compaction
// and verify.
//
// L7 soft floor: telemetry refused and counted, evidence continues; a gap
// record (count, time range) is written when space returns; hard floor
// refuses everything
// L8 retention by age and by size deletes sealed telemetry only (files and
// manifest entries), never evidence; a reopen agrees
// L9 floor eviction deletes the oldest sealed telemetry until above the floor
// L10 compaction keeps every record in order, shrinks or keeps size, leaves
// a damaged segment untouched
// L11 verify reports a corrupt block, a missing file and an unlisted file
#include "strawberry/store/log_store.hpp"
#include <cassert>
#include <cstdint>
#include <filesystem>
#include <fstream>
#include <iostream>
#include <string>
#include <vector>
#include <unistd.h>
using namespace strawberry::store;
namespace fs = std::filesystem;
namespace {
uint64_t g_clock = 1'800'000'000'000'000'000ull;
uint64_t g_free = 100ull << 30;
const uint64_t kTotal = 1000ull << 30;
StoreOptions options_for(const std::string& dir) {
StoreOptions o = default_store_options(dir);
o.now_ns = [] { return g_clock; };
o.disk_space = [](uint64_t& free_bytes, uint64_t& total_bytes) {
free_bytes = g_free;
total_bytes = kTotal;
return true;
};
o.min_free_bytes = 2ull << 30;
o.min_free_pct = 5; // 50 GiB of 1000 GiB: the soft floor
o.hard_min_free_bytes = 256ull << 20;
return o;
}
void advance(uint64_t seconds) { g_clock += seconds * 1'000'000'000ull; }
std::string payload(uint64_t n, std::size_t filler = 200) {
FieldWriter w;
w.u64(1, g_clock);
w.u64(2, n);
w.bytes(3, std::string(filler, static_cast<char>('a' + n % 26)));
return w.take();
}
std::vector<uint64_t> read_numbers(const LogStore& store, Stream stream, uint64_t* gaps = nullptr,
uint64_t* gap_count = nullptr) {
std::vector<uint64_t> out;
store.scan(stream, [&](uint32_t type, std::string_view p) {
FieldReader r(p);
Field f;
if (type == static_cast<uint32_t>(RecordType::gap)) {
if (gaps) ++*gaps;
while (r.next(f)) {
if (f.tag == 2 && gap_count) *gap_count += f.value;
}
return;
}
while (r.next(f)) {
if (f.tag == 2) out.push_back(f.value);
}
});
return out;
}
fs::path fresh(const std::string& name) {
const fs::path p = fs::temp_directory_path() / ("sb-maint-" + std::to_string(::getpid())) / name;
fs::remove_all(p);
fs::create_directories(p.parent_path());
return p;
}
void test_floor() { // L7
const auto dir = fresh("floor").string();
g_free = 100ull << 30;
StoreOpenReport report;
std::string error;
auto store = LogStore::open(options_for(dir), report, error);
assert(store);
assert(store->append(Stream::telemetry, RecordType::process_exec, payload(1)));
g_free = 10ull << 30; // below 5% of 1000 GiB
advance(5);
assert(store->store_status().floor == FloorState::soft);
assert(!store->append(Stream::telemetry, RecordType::process_exec, payload(2)));
assert(!store->append_group(Stream::telemetry, {{RecordType::process_exec, payload(3)},
{RecordType::process_exec, payload(4)}}));
assert(store->append(Stream::evidence, RecordType::finding_upsert, payload(100))); // evidence continues
auto st = store->store_status();
assert(st.telemetry.dropped_records == 3 && !st.telemetry.failed && st.telemetry.error.starts_with("refused:"));
g_free = 100ull << 20; // below the hard floor
advance(5);
assert(store->store_status().floor == FloorState::hard);
assert(!store->append(Stream::evidence, RecordType::finding_upsert, payload(101)));
g_free = 100ull << 30; // space is back
advance(5);
assert(store->append(Stream::telemetry, RecordType::process_exec, payload(5)));
assert(store->flush(Stream::telemetry, true) && store->flush(Stream::evidence, true));
uint64_t gaps = 0, gap_count = 0;
const auto telemetry = read_numbers(*store, Stream::telemetry, &gaps, &gap_count);
assert((telemetry == std::vector<uint64_t>{1, 5}));
assert(gaps == 1 && gap_count == 3); // the drop is accounted, once
assert(read_numbers(*store, Stream::evidence) == std::vector<uint64_t>{100});
st = store->store_status();
assert(st.telemetry.gaps_recorded == 1 && st.telemetry.error.empty() && st.evidence.dropped_records == 1);
std::cout << "L7 floor: PASSED" << std::endl;
}
void test_retention() { // L8
const auto dir = fresh("retention").string();
g_free = 100ull << 30;
StoreOptions o = options_for(dir);
o.telemetry_retention_days = 2;
o.seal_age = std::chrono::hours(24 * 30); // only the explicit rolls below
StoreOpenReport report;
std::string error;
auto store = LogStore::open(o, report, error);
assert(store);
uint64_t n = 0;
for (int day = 0; day < 4; ++day) { // one sealed telemetry + evidence segment per day
assert(store->append(Stream::telemetry, RecordType::process_exec, payload(n++)));
assert(store->append(Stream::evidence, RecordType::finding_upsert, payload(n++)));
assert(store->flush(Stream::telemetry, true) && store->flush(Stream::evidence, true));
assert(store->roll(Stream::evidence) && store->roll(Stream::telemetry));
advance(86'400);
}
// Rolled at days 0..3, now day 4. Retention ran at each roll: by the
// day-3 roll only day 0 had expired; at day 4 day 1 has too.
assert(store->maintain() == 1);
assert(store->maintain() == 0); // idempotent
auto st = store->store_status();
assert(st.telemetry.sealed_segments == 2 && st.telemetry.deleted_segments == 2);
assert(st.evidence.sealed_segments == 4 && st.evidence.deleted_segments == 0);
assert((read_numbers(*store, Stream::telemetry) == std::vector<uint64_t>{4, 6}));
assert((read_numbers(*store, Stream::evidence) == std::vector<uint64_t>{1, 3, 5, 7}));
std::size_t files = 0;
for (const auto& e : fs::directory_iterator(dir + "/telemetry")) files += e.path().extension() == ".log";
assert(files == 3); // 2 sealed + active
store.reset();
auto again = LogStore::open(o, report, error);
assert(again && report.orphaned.empty());
assert((read_numbers(*again, Stream::telemetry) == std::vector<uint64_t>{4, 6}));
again.reset();
// By size: keep at most ~2 sealed telemetry segments' worth.
const auto dir2 = fresh("retention-size").string();
StoreOptions o2 = options_for(dir2);
auto s2 = LogStore::open(o2, report, error);
assert(s2);
for (uint64_t i = 0; i < 5; ++i) {
assert(s2->append(Stream::telemetry, RecordType::process_exec, payload(i, 20000)));
assert(s2->roll(Stream::telemetry));
}
const uint64_t per_segment = s2->store_status().telemetry.sealed_bytes / 5;
s2.reset();
o2.telemetry_max_bytes = per_segment * 2 + per_segment / 2;
s2 = LogStore::open(o2, report, error);
assert(s2);
assert(s2->maintain() >= 2);
const auto sizes = s2->store_status().telemetry;
assert(sizes.sealed_bytes + sizes.active_bytes <= o2.telemetry_max_bytes);
const auto left = read_numbers(*s2, Stream::telemetry);
assert(!left.empty() && left.back() == 4); // the oldest went first
std::cout << "L8 retention: PASSED" << std::endl;
}
void test_eviction() { // L9
const auto dir = fresh("evict").string();
g_free = 100ull << 30;
StoreOptions o = options_for(dir);
o.floor_action = FloorAction::evict_oldest_telemetry;
// Each deleted segment "frees" 20 GiB in this fake filesystem.
uint64_t deleted_seen = 0;
StoreOpenReport report;
std::string error;
auto store = LogStore::open(o, report, error);
assert(store);
for (uint64_t i = 0; i < 4; ++i) {
assert(store->append(Stream::telemetry, RecordType::process_exec, payload(i)));
assert(store->roll(Stream::telemetry));
}
store.reset();
o.disk_space = [&deleted_seen, dir](uint64_t& free_bytes, uint64_t& total_bytes) {
std::size_t files = 0;
for (const auto& e : fs::directory_iterator(dir + "/telemetry")) files += e.path().extension() == ".log";
deleted_seen = 5 - files; // 4 sealed + active at the start
free_bytes = (10ull << 30) + deleted_seen * (20ull << 30);
total_bytes = kTotal;
return true;
};
store = LogStore::open(o, report, error);
assert(store);
advance(5);
// Below the floor: the append evicts the two oldest (10 + 2*20 = 50 GiB) and goes through.
assert(store->append(Stream::telemetry, RecordType::process_exec, payload(9)));
assert(store->store_status().telemetry.deleted_segments == 2);
assert((read_numbers(*store, Stream::telemetry) == std::vector<uint64_t>{2, 3})); // 9 is still pending
std::cout << "L9 eviction: PASSED" << std::endl;
}
void test_compact_and_verify() { // L10, L11
const auto dir = fresh("compact").string();
g_free = 100ull << 30;
StoreOpenReport report;
std::string error;
auto store = LogStore::open(options_for(dir), report, error);
assert(store);
std::vector<uint64_t> expect;
for (uint64_t seg = 0; seg < 3; ++seg) {
for (uint64_t i = 0; i < 300; ++i) {
const uint64_t n = seg * 1000 + i;
assert(store->append(Stream::evidence, RecordType::finding_upsert, payload(n, 300)));
expect.push_back(n);
}
assert(store->roll(Stream::evidence));
}
VerifyReport clean = verify_store(dir);
assert(clean.manifest_ok && clean.problems.empty() && clean.sealed == 3 && clean.records == 900);
CompactReport compact;
assert(store->compact(19, compact));
assert(compact.segments == 3 && compact.bytes_after <= compact.bytes_before && compact.problems.empty());
assert(read_numbers(*store, Stream::evidence) == expect);
clean = verify_store(dir);
assert(clean.problems.empty() && clean.records == 900);
// Damage one sealed segment: verify reports it, compaction leaves it alone.
const auto& segs = store->manifest().segments;
std::string victim;
for (const auto& s : segs) {
if (s.stream == Stream::evidence && s.sealed) {
victim = dir + "/evidence/" + s.file;
break;
}
}
const auto size = fs::file_size(victim);
{
std::fstream f(victim, std::ios::in | std::ios::out | std::ios::binary);
f.seekp(static_cast<std::streamoff>(size / 2));
f.put('\x5a');
}
const auto damaged_size = fs::file_size(victim);
VerifyReport bad = verify_store(dir);
assert(!bad.problems.empty() && bad.bad_blocks >= 1);
CompactReport again;
assert(!store->compact(19, again) && !again.problems.empty());
assert(fs::file_size(victim) == damaged_size); // untouched
// A listed file missing, and a stray file.
store.reset();
const fs::path stray = fs::path(dir) / "evidence" / "seg-999.log";
std::ofstream(stray) << "junk";
fs::rename(victim, victim + ".moved");
VerifyReport missing = verify_store(dir);
bool saw_missing = false, saw_stray = false;
for (const auto& p : missing.problems) saw_missing |= p.find("listed but missing") != std::string::npos;
for (const auto& n : missing.notes) saw_stray |= n.find("seg-999.log") != std::string::npos;
assert(saw_missing && saw_stray);
std::cout << "L10 compaction + L11 verify: PASSED" << std::endl;
}
} // namespace
int main() {
test_floor();
test_retention();
test_eviction();
test_compact_and_verify();
fs::remove_all(fs::temp_directory_path() / ("sb-maint-" + std::to_string(::getpid())));
std::cout << "store maintenance: all properties hold" << std::endl;
return 0;
}