Root cause, from the Tracy capture the CPU watcher took when the tray spun (2026-11-15 08:15, 67% for 15 s): database.reload_read_only rebuilt the whole evidence view from every segment — mean 1.4 s, max 1.7 s, 4x in 20 s — because every 3 s poll that saw the daemon append evidence triggered a full reload. - store::scan_store_read_only_from + ReadCursor (per segment: id, sealed, bytes, records delivered). When the previously seen segments are an unchanged prefix (same ids, sealed ones byte-identical), only records appended since are delivered: the rest of the previously last segment (skipping what was applied) and new segments. Any other change — compaction, retention, reduction, a rewritten or shrunk segment — delivers everything and the caller rebuilds. - Database read-only refresh uses it; records apply in the same order a full rebuild uses, so the view is identical. read_only_reload_counts() reports full vs incremental refreshes. Live, on the real evidence store: the change-driven reload went from ~1.4 s to 2.8 ms; the tray did ~97 ms of work in a 60 s window. Test (test_incremental_view, seeded property): a long-lived reader refreshing across random appends, upserts, seals and a compaction always equals a freshly opened reader; appends refresh incrementally (42-49 per run), the compaction forces a rebuild, and no change reports no change. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
143 lines
6.1 KiB
C++
143 lines
6.1 KiB
C++
// Incremental read-only refresh (2026-11-15 tray spin: a full evidence-view rebuild of ~1.4 s on
|
|
// every poll that saw a change). Property, seeded: a long-lived read-only Database that refreshes
|
|
// after random appends, segment seals and a compaction always holds exactly the view a freshly
|
|
// opened read-only Database builds from scratch.
|
|
//
|
|
// V1 after every refresh: findings and indicators equal a fresh reader's (ids and contents)
|
|
// V2 pure appends (incl. seals) refresh incrementally; a compaction falls back to a full rebuild
|
|
// V3 nothing changed -> refresh reports no change
|
|
// A failure prints the seed; re-run with STRAWBERRY_PROPERTY_SEED=<seed>.
|
|
#include "strawberry/database.hpp"
|
|
#include "strawberry/store/log_store.hpp"
|
|
|
|
#include "../src/store/evidence_records.hpp"
|
|
|
|
#include <algorithm>
|
|
#include <cstdlib>
|
|
#include <filesystem>
|
|
#include <iostream>
|
|
#include <random>
|
|
#include <string>
|
|
#include <vector>
|
|
|
|
#include <unistd.h>
|
|
|
|
namespace fs = std::filesystem;
|
|
using namespace strawberry;
|
|
|
|
namespace {
|
|
|
|
uint64_t seed_from_env() {
|
|
if (const char* text = std::getenv("STRAWBERRY_PROPERTY_SEED")) return std::strtoull(text, nullptr, 10);
|
|
return std::random_device{}();
|
|
}
|
|
const uint64_t kSeed = seed_from_env();
|
|
|
|
#define PROPERTY(cond) \
|
|
do { \
|
|
if (!(cond)) { \
|
|
std::cerr << "PROPERTY FAILED: " #cond " (seed " << kSeed << ", " << __FILE__ << ":" \
|
|
<< __LINE__ << ")" << std::endl; \
|
|
std::abort(); \
|
|
} \
|
|
} while (0)
|
|
|
|
std::string snapshot(Database& db) {
|
|
std::vector<std::string> rows;
|
|
for (const auto& f : db.list_findings()) {
|
|
rows.push_back("F|" + f.id + "|" + f.title + "|" + f.severity + "|" + std::to_string(f.score));
|
|
}
|
|
for (const auto& i : db.list_indicators()) {
|
|
rows.push_back("I|" + i.id + "|" + i.rule_name + "|" + i.severity);
|
|
}
|
|
std::sort(rows.begin(), rows.end());
|
|
std::string out;
|
|
for (const auto& r : rows) out += r + "\n";
|
|
return out;
|
|
}
|
|
|
|
std::string fresh_snapshot(const std::string& dir) {
|
|
Database fresh;
|
|
PROPERTY(fresh.open_read_only(dir));
|
|
return snapshot(fresh);
|
|
}
|
|
|
|
} // namespace
|
|
|
|
int main() {
|
|
std::cout << "incremental view, seed " << kSeed << std::endl;
|
|
const fs::path root = fs::temp_directory_path() / ("sb-incremental-view-" + std::to_string(::getpid()));
|
|
fs::remove_all(root);
|
|
fs::create_directories(root);
|
|
const std::string dir = (root / "db").string();
|
|
|
|
std::mt19937_64 rng(kSeed);
|
|
store::StoreOpenReport open_report;
|
|
std::string error;
|
|
auto options = store::default_store_options(dir);
|
|
auto writer = store::LogStore::open(options, open_report, error);
|
|
PROPERTY(writer != nullptr);
|
|
// Seed some evidence so the reader starts with a real view.
|
|
const auto put_finding = [&](int n, int score) {
|
|
FindingRecord f;
|
|
f.id = "finding-" + std::to_string(n);
|
|
f.title = "title " + std::to_string(n);
|
|
f.severity = score > 70 ? "high" : "low";
|
|
f.score = score;
|
|
PROPERTY(writer->append(store::Stream::evidence, store::RecordType::finding_upsert, store::encode_finding(f)));
|
|
};
|
|
const auto put_indicator = [&](int n) {
|
|
IndicatorEntry e;
|
|
e.id = "ind-" + std::to_string(n);
|
|
e.rule_name = "Rule_" + std::to_string(n % 7);
|
|
e.hash = std::string(64, static_cast<char>('a' + n % 6));
|
|
e.timestamp = n;
|
|
e.severity = (n % 2) ? "high" : "medium";
|
|
PROPERTY(writer->append(store::Stream::evidence, store::RecordType::indicator, store::encode_indicator(e)));
|
|
};
|
|
for (int i = 0; i < 20; ++i) put_finding(i, 50);
|
|
PROPERTY(writer->flush(store::Stream::evidence, true));
|
|
|
|
Database reader;
|
|
PROPERTY(reader.open_read_only(dir));
|
|
PROPERTY(snapshot(reader) == fresh_snapshot(dir));
|
|
PROPERTY(!reader.refresh()); // V3
|
|
|
|
int next_id = 20;
|
|
bool compacted = false;
|
|
for (int round = 0; round < 60; ++round) {
|
|
// A burst of appends: new findings, upserts of old ones (score changes), indicators.
|
|
const int ops = 1 + static_cast<int>(rng() % 25);
|
|
for (int k = 0; k < ops; ++k) {
|
|
switch (rng() % 3) {
|
|
case 0: put_finding(next_id++, static_cast<int>(rng() % 100)); break;
|
|
case 1: put_finding(static_cast<int>(rng() % next_id), static_cast<int>(rng() % 100)); break;
|
|
default: put_indicator(static_cast<int>(rng() % 200)); break;
|
|
}
|
|
}
|
|
PROPERTY(writer->flush(store::Stream::evidence, true));
|
|
if (rng() % 5 == 0) PROPERTY(writer->roll(store::Stream::evidence)); // seal: a new active segment
|
|
if (!compacted && round == 40) {
|
|
// A non-append change (segments rewritten): the reader must notice and rebuild.
|
|
store::CompactReport cr;
|
|
PROPERTY(writer->compact(19, cr));
|
|
compacted = true;
|
|
}
|
|
if (rng() % 4 == 0) continue; // sometimes several rounds pile up before a refresh
|
|
PROPERTY(reader.refresh());
|
|
PROPERTY(snapshot(reader) == fresh_snapshot(dir)); // V1
|
|
PROPERTY(!reader.refresh()); // V3
|
|
}
|
|
PROPERTY(writer->flush(store::Stream::evidence, true));
|
|
reader.refresh();
|
|
PROPERTY(snapshot(reader) == fresh_snapshot(dir));
|
|
const auto counts = reader.read_only_reload_counts();
|
|
std::cout << " reloads: " << counts.first << " full, " << counts.second << " incremental" << std::endl;
|
|
PROPERTY(counts.second > 10); // V2: appends are incremental
|
|
PROPERTY(counts.first >= 2); // the first load + the compaction rebuild
|
|
|
|
writer.reset();
|
|
fs::remove_all(root);
|
|
std::cout << "incremental view: PASSED" << std::endl;
|
|
return 0;
|
|
}
|