Some checks failed
Build and Test / build (debug, clang) (push) Has been cancelled
Build and Test / build (debug, gcc) (push) Has been cancelled
Build and Test / build (release, clang) (push) Has been cancelled
Build and Test / build (release, gcc) (push) Has been cancelled
Build and Test / docker (push) Has been cancelled
Stage 1 increment 7. - Disk floor: below max(store_min_free_mb, store_min_free_pct) free, telemetry is refused and counted (a gap record with the count and time range is written once space returns) or, with store_floor_action=evict_oldest_telemetry, the oldest sealed telemetry is deleted; evidence continues until store_hard_min_free_mb. - Retention (telemetry only, off by default): store_retention_days and store_max_mb, applied when a segment is sealed. Evidence is never deleted by retention. - Status gains a store block (floor, free space, per-stream sizes, compression ratio, drops, deletions); health reports a low disk, a failed evidence store and unsaved detections. - strawberry --store-verify checks the manifest and every segment (read-only, safe while the daemon runs; root-only telemetry is noted, not flagged). strawberry --store-compact rewrites sealed segments at zstd level 19 with the daemon stopped, never touching a damaged one. - The manifest records each sealed segment's uncompressed size. Tests L7-L11 cover the floor, gaps, retention by age and size, eviction, compaction and verify. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
287 lines
12 KiB
C++
287 lines
12 KiB
C++
// Event store maintenance: disk-space floor, retention, eviction, compaction
|
|
// and verify.
|
|
//
|
|
// L7 soft floor: telemetry refused and counted, evidence continues; a gap
|
|
// record (count, time range) is written when space returns; hard floor
|
|
// refuses everything
|
|
// L8 retention by age and by size deletes sealed telemetry only (files and
|
|
// manifest entries), never evidence; a reopen agrees
|
|
// L9 floor eviction deletes the oldest sealed telemetry until above the floor
|
|
// L10 compaction keeps every record in order, shrinks or keeps size, leaves
|
|
// a damaged segment untouched
|
|
// L11 verify reports a corrupt block, a missing file and an unlisted file
|
|
|
|
#include "strawberry/store/log_store.hpp"
|
|
|
|
#include <cassert>
|
|
#include <cstdint>
|
|
#include <filesystem>
|
|
#include <fstream>
|
|
#include <iostream>
|
|
#include <string>
|
|
#include <vector>
|
|
|
|
#include <unistd.h>
|
|
|
|
using namespace strawberry::store;
|
|
namespace fs = std::filesystem;
|
|
|
|
namespace {
|
|
|
|
uint64_t g_clock = 1'800'000'000'000'000'000ull;
|
|
uint64_t g_free = 100ull << 30;
|
|
const uint64_t kTotal = 1000ull << 30;
|
|
|
|
StoreOptions options_for(const std::string& dir) {
|
|
StoreOptions o = default_store_options(dir);
|
|
o.now_ns = [] { return g_clock; };
|
|
o.disk_space = [](uint64_t& free_bytes, uint64_t& total_bytes) {
|
|
free_bytes = g_free;
|
|
total_bytes = kTotal;
|
|
return true;
|
|
};
|
|
o.min_free_bytes = 2ull << 30;
|
|
o.min_free_pct = 5; // 50 GiB of 1000 GiB: the soft floor
|
|
o.hard_min_free_bytes = 256ull << 20;
|
|
return o;
|
|
}
|
|
|
|
void advance(uint64_t seconds) { g_clock += seconds * 1'000'000'000ull; }
|
|
|
|
std::string payload(uint64_t n, std::size_t filler = 200) {
|
|
FieldWriter w;
|
|
w.u64(1, g_clock);
|
|
w.u64(2, n);
|
|
w.bytes(3, std::string(filler, static_cast<char>('a' + n % 26)));
|
|
return w.take();
|
|
}
|
|
|
|
std::vector<uint64_t> read_numbers(const LogStore& store, Stream stream, uint64_t* gaps = nullptr,
|
|
uint64_t* gap_count = nullptr) {
|
|
std::vector<uint64_t> out;
|
|
store.scan(stream, [&](uint32_t type, std::string_view p) {
|
|
FieldReader r(p);
|
|
Field f;
|
|
if (type == static_cast<uint32_t>(RecordType::gap)) {
|
|
if (gaps) ++*gaps;
|
|
while (r.next(f)) {
|
|
if (f.tag == 2 && gap_count) *gap_count += f.value;
|
|
}
|
|
return;
|
|
}
|
|
while (r.next(f)) {
|
|
if (f.tag == 2) out.push_back(f.value);
|
|
}
|
|
});
|
|
return out;
|
|
}
|
|
|
|
fs::path fresh(const std::string& name) {
|
|
const fs::path p = fs::temp_directory_path() / ("sb-maint-" + std::to_string(::getpid())) / name;
|
|
fs::remove_all(p);
|
|
fs::create_directories(p.parent_path());
|
|
return p;
|
|
}
|
|
|
|
void test_floor() { // L7
|
|
const auto dir = fresh("floor").string();
|
|
g_free = 100ull << 30;
|
|
StoreOpenReport report;
|
|
std::string error;
|
|
auto store = LogStore::open(options_for(dir), report, error);
|
|
assert(store);
|
|
|
|
assert(store->append(Stream::telemetry, RecordType::process_exec, payload(1)));
|
|
g_free = 10ull << 30; // below 5% of 1000 GiB
|
|
advance(5);
|
|
assert(store->store_status().floor == FloorState::soft);
|
|
assert(!store->append(Stream::telemetry, RecordType::process_exec, payload(2)));
|
|
assert(!store->append_group(Stream::telemetry, {{RecordType::process_exec, payload(3)},
|
|
{RecordType::process_exec, payload(4)}}));
|
|
assert(store->append(Stream::evidence, RecordType::finding_upsert, payload(100))); // evidence continues
|
|
auto st = store->store_status();
|
|
assert(st.telemetry.dropped_records == 3 && !st.telemetry.failed && st.telemetry.error.starts_with("refused:"));
|
|
|
|
g_free = 100ull << 20; // below the hard floor
|
|
advance(5);
|
|
assert(store->store_status().floor == FloorState::hard);
|
|
assert(!store->append(Stream::evidence, RecordType::finding_upsert, payload(101)));
|
|
|
|
g_free = 100ull << 30; // space is back
|
|
advance(5);
|
|
assert(store->append(Stream::telemetry, RecordType::process_exec, payload(5)));
|
|
assert(store->flush(Stream::telemetry, true) && store->flush(Stream::evidence, true));
|
|
uint64_t gaps = 0, gap_count = 0;
|
|
const auto telemetry = read_numbers(*store, Stream::telemetry, &gaps, &gap_count);
|
|
assert((telemetry == std::vector<uint64_t>{1, 5}));
|
|
assert(gaps == 1 && gap_count == 3); // the drop is accounted, once
|
|
assert(read_numbers(*store, Stream::evidence) == std::vector<uint64_t>{100});
|
|
st = store->store_status();
|
|
assert(st.telemetry.gaps_recorded == 1 && st.telemetry.error.empty() && st.evidence.dropped_records == 1);
|
|
std::cout << "L7 floor: PASSED" << std::endl;
|
|
}
|
|
|
|
void test_retention() { // L8
|
|
const auto dir = fresh("retention").string();
|
|
g_free = 100ull << 30;
|
|
StoreOptions o = options_for(dir);
|
|
o.telemetry_retention_days = 2;
|
|
o.seal_age = std::chrono::hours(24 * 30); // only the explicit rolls below
|
|
StoreOpenReport report;
|
|
std::string error;
|
|
auto store = LogStore::open(o, report, error);
|
|
assert(store);
|
|
uint64_t n = 0;
|
|
for (int day = 0; day < 4; ++day) { // one sealed telemetry + evidence segment per day
|
|
assert(store->append(Stream::telemetry, RecordType::process_exec, payload(n++)));
|
|
assert(store->append(Stream::evidence, RecordType::finding_upsert, payload(n++)));
|
|
assert(store->flush(Stream::telemetry, true) && store->flush(Stream::evidence, true));
|
|
assert(store->roll(Stream::evidence) && store->roll(Stream::telemetry));
|
|
advance(86'400);
|
|
}
|
|
// Rolled at days 0..3, now day 4. Retention ran at each roll: by the
|
|
// day-3 roll only day 0 had expired; at day 4 day 1 has too.
|
|
assert(store->maintain() == 1);
|
|
assert(store->maintain() == 0); // idempotent
|
|
auto st = store->store_status();
|
|
assert(st.telemetry.sealed_segments == 2 && st.telemetry.deleted_segments == 2);
|
|
assert(st.evidence.sealed_segments == 4 && st.evidence.deleted_segments == 0);
|
|
assert((read_numbers(*store, Stream::telemetry) == std::vector<uint64_t>{4, 6}));
|
|
assert((read_numbers(*store, Stream::evidence) == std::vector<uint64_t>{1, 3, 5, 7}));
|
|
std::size_t files = 0;
|
|
for (const auto& e : fs::directory_iterator(dir + "/telemetry")) files += e.path().extension() == ".log";
|
|
assert(files == 3); // 2 sealed + active
|
|
store.reset();
|
|
auto again = LogStore::open(o, report, error);
|
|
assert(again && report.orphaned.empty());
|
|
assert((read_numbers(*again, Stream::telemetry) == std::vector<uint64_t>{4, 6}));
|
|
again.reset();
|
|
|
|
// By size: keep at most ~2 sealed telemetry segments' worth.
|
|
const auto dir2 = fresh("retention-size").string();
|
|
StoreOptions o2 = options_for(dir2);
|
|
auto s2 = LogStore::open(o2, report, error);
|
|
assert(s2);
|
|
for (uint64_t i = 0; i < 5; ++i) {
|
|
assert(s2->append(Stream::telemetry, RecordType::process_exec, payload(i, 20000)));
|
|
assert(s2->roll(Stream::telemetry));
|
|
}
|
|
const uint64_t per_segment = s2->store_status().telemetry.sealed_bytes / 5;
|
|
s2.reset();
|
|
o2.telemetry_max_bytes = per_segment * 2 + per_segment / 2;
|
|
s2 = LogStore::open(o2, report, error);
|
|
assert(s2);
|
|
assert(s2->maintain() >= 2);
|
|
const auto sizes = s2->store_status().telemetry;
|
|
assert(sizes.sealed_bytes + sizes.active_bytes <= o2.telemetry_max_bytes);
|
|
const auto left = read_numbers(*s2, Stream::telemetry);
|
|
assert(!left.empty() && left.back() == 4); // the oldest went first
|
|
std::cout << "L8 retention: PASSED" << std::endl;
|
|
}
|
|
|
|
void test_eviction() { // L9
|
|
const auto dir = fresh("evict").string();
|
|
g_free = 100ull << 30;
|
|
StoreOptions o = options_for(dir);
|
|
o.floor_action = FloorAction::evict_oldest_telemetry;
|
|
// Each deleted segment "frees" 20 GiB in this fake filesystem.
|
|
uint64_t deleted_seen = 0;
|
|
StoreOpenReport report;
|
|
std::string error;
|
|
auto store = LogStore::open(o, report, error);
|
|
assert(store);
|
|
for (uint64_t i = 0; i < 4; ++i) {
|
|
assert(store->append(Stream::telemetry, RecordType::process_exec, payload(i)));
|
|
assert(store->roll(Stream::telemetry));
|
|
}
|
|
store.reset();
|
|
o.disk_space = [&deleted_seen, dir](uint64_t& free_bytes, uint64_t& total_bytes) {
|
|
std::size_t files = 0;
|
|
for (const auto& e : fs::directory_iterator(dir + "/telemetry")) files += e.path().extension() == ".log";
|
|
deleted_seen = 5 - files; // 4 sealed + active at the start
|
|
free_bytes = (10ull << 30) + deleted_seen * (20ull << 30);
|
|
total_bytes = kTotal;
|
|
return true;
|
|
};
|
|
store = LogStore::open(o, report, error);
|
|
assert(store);
|
|
advance(5);
|
|
// Below the floor: the append evicts the two oldest (10 + 2*20 = 50 GiB) and goes through.
|
|
assert(store->append(Stream::telemetry, RecordType::process_exec, payload(9)));
|
|
assert(store->store_status().telemetry.deleted_segments == 2);
|
|
assert((read_numbers(*store, Stream::telemetry) == std::vector<uint64_t>{2, 3})); // 9 is still pending
|
|
std::cout << "L9 eviction: PASSED" << std::endl;
|
|
}
|
|
|
|
void test_compact_and_verify() { // L10, L11
|
|
const auto dir = fresh("compact").string();
|
|
g_free = 100ull << 30;
|
|
StoreOpenReport report;
|
|
std::string error;
|
|
auto store = LogStore::open(options_for(dir), report, error);
|
|
assert(store);
|
|
std::vector<uint64_t> expect;
|
|
for (uint64_t seg = 0; seg < 3; ++seg) {
|
|
for (uint64_t i = 0; i < 300; ++i) {
|
|
const uint64_t n = seg * 1000 + i;
|
|
assert(store->append(Stream::evidence, RecordType::finding_upsert, payload(n, 300)));
|
|
expect.push_back(n);
|
|
}
|
|
assert(store->roll(Stream::evidence));
|
|
}
|
|
VerifyReport clean = verify_store(dir);
|
|
assert(clean.manifest_ok && clean.problems.empty() && clean.sealed == 3 && clean.records == 900);
|
|
|
|
CompactReport compact;
|
|
assert(store->compact(19, compact));
|
|
assert(compact.segments == 3 && compact.bytes_after <= compact.bytes_before && compact.problems.empty());
|
|
assert(read_numbers(*store, Stream::evidence) == expect);
|
|
clean = verify_store(dir);
|
|
assert(clean.problems.empty() && clean.records == 900);
|
|
|
|
// Damage one sealed segment: verify reports it, compaction leaves it alone.
|
|
const auto& segs = store->manifest().segments;
|
|
std::string victim;
|
|
for (const auto& s : segs) {
|
|
if (s.stream == Stream::evidence && s.sealed) {
|
|
victim = dir + "/evidence/" + s.file;
|
|
break;
|
|
}
|
|
}
|
|
const auto size = fs::file_size(victim);
|
|
{
|
|
std::fstream f(victim, std::ios::in | std::ios::out | std::ios::binary);
|
|
f.seekp(static_cast<std::streamoff>(size / 2));
|
|
f.put('\x5a');
|
|
}
|
|
const auto damaged_size = fs::file_size(victim);
|
|
VerifyReport bad = verify_store(dir);
|
|
assert(!bad.problems.empty() && bad.bad_blocks >= 1);
|
|
CompactReport again;
|
|
assert(!store->compact(19, again) && !again.problems.empty());
|
|
assert(fs::file_size(victim) == damaged_size); // untouched
|
|
|
|
// A listed file missing, and a stray file.
|
|
store.reset();
|
|
const fs::path stray = fs::path(dir) / "evidence" / "seg-999.log";
|
|
std::ofstream(stray) << "junk";
|
|
fs::rename(victim, victim + ".moved");
|
|
VerifyReport missing = verify_store(dir);
|
|
bool saw_missing = false, saw_stray = false;
|
|
for (const auto& p : missing.problems) saw_missing |= p.find("listed but missing") != std::string::npos;
|
|
for (const auto& n : missing.notes) saw_stray |= n.find("seg-999.log") != std::string::npos;
|
|
assert(saw_missing && saw_stray);
|
|
std::cout << "L10 compaction + L11 verify: PASSED" << std::endl;
|
|
}
|
|
|
|
} // namespace
|
|
|
|
int main() {
|
|
test_floor();
|
|
test_retention();
|
|
test_eviction();
|
|
test_compact_and_verify();
|
|
fs::remove_all(fs::temp_directory_path() / ("sb-maint-" + std::to_string(::getpid())));
|
|
std::cout << "store maintenance: all properties hold" << std::endl;
|
|
return 0;
|
|
}
|