strawberry/tests/test_incremental_view.cpp
trvon 1c9b005645
store: incremental read-only refresh (fixes the tray spinning a core)
Root cause, from the Tracy capture the CPU watcher took when the tray spun
(2026-11-15 08:15, 67% for 15 s): database.reload_read_only rebuilt the whole
evidence view from every segment — mean 1.4 s, max 1.7 s, 4x in 20 s — because
every 3 s poll that saw the daemon append evidence triggered a full reload.

- store::scan_store_read_only_from + ReadCursor (per segment: id, sealed,
  bytes, records delivered). When the previously seen segments are an
  unchanged prefix (same ids, sealed ones byte-identical), only records
  appended since are delivered: the rest of the previously last segment
  (skipping what was applied) and new segments. Any other change —
  compaction, retention, reduction, a rewritten or shrunk segment — delivers
  everything and the caller rebuilds.
- Database read-only refresh uses it; records apply in the same order a full
  rebuild uses, so the view is identical. read_only_reload_counts() reports
  full vs incremental refreshes.

Live, on the real evidence store: the change-driven reload went from ~1.4 s
to 2.8 ms; the tray did ~97 ms of work in a 60 s window.

Test (test_incremental_view, seeded property): a long-lived reader
refreshing across random appends, upserts, seals and a compaction always
equals a freshly opened reader; appends refresh incrementally (42-49 per run),
the compaction forces a rebuild, and no change reports no change.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-15 08:46:00 -07:00

143 lines
6.1 KiB
C++

// Incremental read-only refresh (2026-11-15 tray spin: a full evidence-view rebuild of ~1.4 s on
// every poll that saw a change). Property, seeded: a long-lived read-only Database that refreshes
// after random appends, segment seals and a compaction always holds exactly the view a freshly
// opened read-only Database builds from scratch.
//
// V1 after every refresh: findings and indicators equal a fresh reader's (ids and contents)
// V2 pure appends (incl. seals) refresh incrementally; a compaction falls back to a full rebuild
// V3 nothing changed -> refresh reports no change
// A failure prints the seed; re-run with STRAWBERRY_PROPERTY_SEED=<seed>.
#include "strawberry/database.hpp"
#include "strawberry/store/log_store.hpp"
#include "../src/store/evidence_records.hpp"
#include <algorithm>
#include <cstdlib>
#include <filesystem>
#include <iostream>
#include <random>
#include <string>
#include <vector>
#include <unistd.h>
namespace fs = std::filesystem;
using namespace strawberry;
namespace {
uint64_t seed_from_env() {
if (const char* text = std::getenv("STRAWBERRY_PROPERTY_SEED")) return std::strtoull(text, nullptr, 10);
return std::random_device{}();
}
const uint64_t kSeed = seed_from_env();
#define PROPERTY(cond) \
do { \
if (!(cond)) { \
std::cerr << "PROPERTY FAILED: " #cond " (seed " << kSeed << ", " << __FILE__ << ":" \
<< __LINE__ << ")" << std::endl; \
std::abort(); \
} \
} while (0)
std::string snapshot(Database& db) {
std::vector<std::string> rows;
for (const auto& f : db.list_findings()) {
rows.push_back("F|" + f.id + "|" + f.title + "|" + f.severity + "|" + std::to_string(f.score));
}
for (const auto& i : db.list_indicators()) {
rows.push_back("I|" + i.id + "|" + i.rule_name + "|" + i.severity);
}
std::sort(rows.begin(), rows.end());
std::string out;
for (const auto& r : rows) out += r + "\n";
return out;
}
std::string fresh_snapshot(const std::string& dir) {
Database fresh;
PROPERTY(fresh.open_read_only(dir));
return snapshot(fresh);
}
} // namespace
int main() {
std::cout << "incremental view, seed " << kSeed << std::endl;
const fs::path root = fs::temp_directory_path() / ("sb-incremental-view-" + std::to_string(::getpid()));
fs::remove_all(root);
fs::create_directories(root);
const std::string dir = (root / "db").string();
std::mt19937_64 rng(kSeed);
store::StoreOpenReport open_report;
std::string error;
auto options = store::default_store_options(dir);
auto writer = store::LogStore::open(options, open_report, error);
PROPERTY(writer != nullptr);
// Seed some evidence so the reader starts with a real view.
const auto put_finding = [&](int n, int score) {
FindingRecord f;
f.id = "finding-" + std::to_string(n);
f.title = "title " + std::to_string(n);
f.severity = score > 70 ? "high" : "low";
f.score = score;
PROPERTY(writer->append(store::Stream::evidence, store::RecordType::finding_upsert, store::encode_finding(f)));
};
const auto put_indicator = [&](int n) {
IndicatorEntry e;
e.id = "ind-" + std::to_string(n);
e.rule_name = "Rule_" + std::to_string(n % 7);
e.hash = std::string(64, static_cast<char>('a' + n % 6));
e.timestamp = n;
e.severity = (n % 2) ? "high" : "medium";
PROPERTY(writer->append(store::Stream::evidence, store::RecordType::indicator, store::encode_indicator(e)));
};
for (int i = 0; i < 20; ++i) put_finding(i, 50);
PROPERTY(writer->flush(store::Stream::evidence, true));
Database reader;
PROPERTY(reader.open_read_only(dir));
PROPERTY(snapshot(reader) == fresh_snapshot(dir));
PROPERTY(!reader.refresh()); // V3
int next_id = 20;
bool compacted = false;
for (int round = 0; round < 60; ++round) {
// A burst of appends: new findings, upserts of old ones (score changes), indicators.
const int ops = 1 + static_cast<int>(rng() % 25);
for (int k = 0; k < ops; ++k) {
switch (rng() % 3) {
case 0: put_finding(next_id++, static_cast<int>(rng() % 100)); break;
case 1: put_finding(static_cast<int>(rng() % next_id), static_cast<int>(rng() % 100)); break;
default: put_indicator(static_cast<int>(rng() % 200)); break;
}
}
PROPERTY(writer->flush(store::Stream::evidence, true));
if (rng() % 5 == 0) PROPERTY(writer->roll(store::Stream::evidence)); // seal: a new active segment
if (!compacted && round == 40) {
// A non-append change (segments rewritten): the reader must notice and rebuild.
store::CompactReport cr;
PROPERTY(writer->compact(19, cr));
compacted = true;
}
if (rng() % 4 == 0) continue; // sometimes several rounds pile up before a refresh
PROPERTY(reader.refresh());
PROPERTY(snapshot(reader) == fresh_snapshot(dir)); // V1
PROPERTY(!reader.refresh()); // V3
}
PROPERTY(writer->flush(store::Stream::evidence, true));
reader.refresh();
PROPERTY(snapshot(reader) == fresh_snapshot(dir));
const auto counts = reader.read_only_reload_counts();
std::cout << " reloads: " << counts.first << " full, " << counts.second << " incremental" << std::endl;
PROPERTY(counts.second > 10); // V2: appends are incremental
PROPERTY(counts.first >= 2); // the first load + the compaction rebuild
writer.reset();
fs::remove_all(root);
std::cout << "incremental view: PASSED" << std::endl;
return 0;
}