strawberry/tests/test_replay.cpp
trvon d81ccbe7c8
gate: a bounded evaluation range, so out-of-sample evidence is possible
The in-sample check landed without any way to bound an evaluation, so every cell
was in-sample and every promotion path refused. Cutting the training bundles
earlier does not help: training spans the bundles, evaluation replays the whole
store with no lower bound, and the store keeps everything (retention 0), so
eval_first <= training_last always held.

replay --policy-eval gains --eval-after NS|auto (0 = unbounded). The bound is
applied where windows are ranked and where injected traces are planned, so the
periods partition only the out-of-sample span; auto derives it as the latest
training-range end over the loaded --eval-model files. Comparisons are written
without first + warm-up arithmetic so INT64_MAX timestamps cannot overflow. A
bound past the end of the store does not produce a silent empty cell: the cell
carries eval_error and promotion_gate refuses naming it.

Provenance also gets honest about width: training_last_ns is the last training
window's END (a window spans window_ns, so using its start left up to 60 s of
trained-on data looking disjoint), and the single-model path (model promote) now
uses the ranked span instead of the whole store, which had guaranteed a refusal.

Verified: R11c passes with 128 out-of-sample windows and a derived bound, and
unbounded returns to in-sample. Refuses an explicit bound later than all history.
2026-11-19 17:27:59 -07:00

1962 lines
97 KiB
C++

// Replay: windows, joins and scoring over telemetry records. A failure
// prints the seed; re-run with STRAWBERRY_PROPERTY_SEED=<seed>.
//
// R1 deterministic: the same records give the same report
// R2 order within the reorder horizon does not matter
// R3 a process chain within 60 s is one window; its behavior joins it
// C3 PID reuse: behavior joins the exec of that pid at its time
// R4 an unusual chain ranks above routine ones; detections mark windows
// R5 replay_store reads a real store directory
// R7 warm-up windows are learned from but not ranked; a never-seen event
// scores the same early and late; identical patterns are grouped
// R8 injected traces: every copy is found and scored; injecting changes
// nothing about real windows (scored, never learned from); a novel
// attack chain beats routine activity at a 10/day budget
// R9 late labels (a binary found bad after it ran) mark the windows that
// ran it, read from the evidence stream by replay_store
// R10 training examples: one per ranked window, labelled by detections;
// training traces become positives (not evaluated); per-program cap;
// ranking by a model sets each window's score from its value
// R11 policy evaluation: extra positives exclude rule-flagged windows;
// rarity-greedy inspects within budget; logged shadow decisions are
// joined to their windows; the promotion gate checks each criterion
// R6 paths that are not UTF-8 (any byte but '/' and NUL is legal) give
// valid JSON, in the report and in json_escape
// R12 label retraction: allowlist-remove revokes the named entry's negative
// (detected windows revert to rule-detected positives); quarantine-remove
// revokes only the quarantine positive, preserving independent positives;
// multiple entries and re-adds resolve in stored order
// R13 positive entity cap: max_entity_share caps the combined (positive +
// negative) per-program count; a late positive displaces a negative of
// the same entity rather than being starved; concentration rejections
// are counted, not silently dropped
#include "strawberry/database.hpp"
#include "strawberry/learn/replay.hpp"
#include "strawberry/learn/model_file.hpp"
#include "strawberry/learn/inspection.hpp"
#include "strawberry/learn/trace.hpp"
#include "strawberry/telemetry.hpp"
#include "strawberry/store/log_store.hpp"
#include "strawberry/sys_io.hpp"
#include "../src/json_util.hpp"
#include <nlohmann/json.hpp>
#include <algorithm>
#include <cmath>
#include <memory>
#include <cstdlib>
#include <filesystem>
#include <iostream>
#include <random>
#include <string>
#include <unordered_map>
#include <vector>
#include <fcntl.h>
#include <unistd.h>
using namespace strawberry;
using namespace strawberry::learn;
namespace {
uint64_t seed_from_env() {
if (const char* text = std::getenv("STRAWBERRY_PROPERTY_SEED")) return std::strtoull(text, nullptr, 10);
return std::random_device{}();
}
const uint64_t kSeed = seed_from_env();
#define PROPERTY(cond) \
do { \
if (!(cond)) { \
std::cerr << "PROPERTY FAILED: " #cond " (seed " << kSeed << ", " << __FILE__ << ":" \
<< __LINE__ << ")" << std::endl; \
std::abort(); \
} \
} while (0)
constexpr int64_t kSec = 1'000'000'000;
constexpr int64_t kT0 = 1'800'000'000 * kSec;
TelemetryEvent exec(int64_t ts, int pid, const std::string& node, const std::string& parent, const std::string& exe,
const std::string& cmdline) {
ProcessExecRecord r;
r.timestamp_ns = ts;
r.pid = pid;
r.process_node_id = node;
r.parent_node_id = parent;
r.exe = exe;
r.cmdline = cmdline;
r.source = "live";
return {store::RecordType::process_exec, encode_process_exec(r)};
}
TelemetryEvent connect(int64_t ts, int pid, const std::string& address, int port) {
BehaviorEvent b;
b.type = BehaviorEvent::Type::connect;
b.timestamp_ns = static_cast<uint64_t>(ts);
b.pid = pid;
b.family = 2;
b.address = address;
b.port = static_cast<uint16_t>(port);
return {store::RecordType::connect, encode_behavior(b)};
}
TelemetryEvent scan(int64_t ts, int pid, const std::string& node, std::vector<std::string> rules) {
ScanResultRecord s;
s.timestamp_ns = ts;
s.pid = pid;
s.process_node_id = node;
s.rules = std::move(rules);
return {store::RecordType::scan_result, encode_scan_result(s)};
}
TelemetryEvent write(int64_t ts, int pid, const std::string& path) {
FileWriteEvent w;
w.pid = pid;
w.path = path;
w.category = "drop";
w.timestamp_ns = static_cast<uint64_t>(ts);
return {store::RecordType::file_write, encode_file_write(w)};
}
ReplayOptions no_warmup() {
ReplayOptions o;
o.warmup_ns = 0;
return o;
}
ReplayReport run(const std::vector<TelemetryEvent>& events, ReplayOptions options = no_warmup()) {
Replayer r(options);
for (const auto& e : events) r.feed(e.type, e.payload);
return r.finish();
}
const ReplayWindow* window_rooted(const ReplayReport& report, const std::string& node) {
for (const auto& w : report.top_windows) {
if (w.root_node == node) return &w;
}
return nullptr;
}
// A routine background: many bash -> ls windows, a minute apart.
std::vector<TelemetryEvent> routine(int count, int64_t start) {
std::vector<TelemetryEvent> out;
for (int i = 0; i < count; ++i) {
const auto ts = start + i * 61 * kSec;
const std::string sh = "process_" + std::to_string(1000 + i) + "_b_1";
out.push_back(exec(ts, 1000 + i, sh, "process_1_b_1", "/usr/bin/bash", "bash"));
out.push_back(exec(ts + kSec, 20000 + i, "process_" + std::to_string(20000 + i) + "_b_1", sh, "/usr/bin/ls",
"ls -la /home/a"));
}
return out;
}
void test_chain_and_pid_reuse() {
std::vector<TelemetryEvent> ev;
// firefox -> sh -> curl, curl connects: one window.
ev.push_back(exec(kT0, 100, "process_100_b_1", "process_1_b_1", "/usr/lib/firefox/firefox", "firefox"));
ev.push_back(exec(kT0 + 2 * kSec, 101, "process_101_b_1", "process_100_b_1", "/usr/bin/sh", "sh -c x"));
ev.push_back(exec(kT0 + 3 * kSec, 102, "process_102_b_1", "process_101_b_1", "/usr/bin/curl",
"curl -fsSL https://x/a.sh"));
ev.push_back(connect(kT0 + 4 * kSec, 102, "203.0.113.9", 443));
// PID 500 reused: two different programs, each with a connection.
ev.push_back(exec(kT0 + 10 * kSec, 500, "process_500_b_10", "process_1_b_1", "/usr/bin/a", "a"));
ev.push_back(connect(kT0 + 11 * kSec, 500, "10.0.0.1", 22));
ev.push_back(exec(kT0 + 200 * kSec, 500, "process_500_b_20", "process_1_b_1", "/usr/bin/b", "b"));
ev.push_back(connect(kT0 + 201 * kSec, 500, "10.0.0.2", 80));
ev.push_back(connect(kT0 + 202 * kSec, 777, "10.0.0.3", 80)); // no exec for 777
const auto report = run(ev);
PROPERTY(report.execs == 5 && report.behavior_events == 4 && report.unjoined_events == 1);
PROPERTY(report.windows == 3);
const auto* chain = window_rooted(report, "process_100_b_1");
PROPERTY(chain != nullptr && chain->execs == 3 && chain->connects == 1); // R3
const auto* a = window_rooted(report, "process_500_b_10");
const auto* b = window_rooted(report, "process_500_b_20");
PROPERTY(a != nullptr && b != nullptr && a->connects == 1 && b->connects == 1); // C3
std::cout << "R3 chain window, C3 pid reuse: PASSED" << std::endl;
}
void test_order_and_determinism() {
std::mt19937_64 rng(kSeed);
auto ev = routine(50, kT0);
ev.push_back(exec(kT0 + 5 * kSec + 7, 9, "process_9_b_1", "process_1_b_1", "/usr/bin/odd", "odd --x"));
const auto base = replay_report_json(run(ev));
PROPERTY(replay_report_json(run(ev)) == base); // R1
// R2: shuffle within small time buckets (records arrive batched, a few
// seconds apart at most; the horizon is 2 min).
for (int round = 0; round < 20; ++round) {
auto shuffled = ev;
for (std::size_t i = 0; i + 1 < shuffled.size(); i += 2) {
if (rng() % 2) std::swap(shuffled[i], shuffled[i + 1]);
}
PROPERTY(replay_report_json(run(shuffled)) == base);
}
std::cout << "R1 deterministic, R2 order-insensitive: PASSED" << std::endl;
}
void test_ranking_and_detection() {
auto ev = routine(300, kT0);
const int64_t t = kT0 + 301 * 61 * kSec;
ev.push_back(exec(t, 4000, "process_4000_b_1", "process_1_b_1", "/usr/bin/evince", "evince /home/a/x.pdf"));
ev.push_back(exec(t + kSec, 4001, "process_4001_b_1", "process_4000_b_1", "/usr/bin/python3", "python3 -c x"));
ev.push_back(connect(t + 2 * kSec, 4001, "198.51.100.7", 4444));
ev.push_back(scan(t + kSec, 4001, "process_4001_b_1", {"Suspicious_Python"}));
ReplayOptions options;
options.warmup_ns = 3600 * kSec; // the first routine windows are new too: that is what warm-up is for
options.top = 5;
const auto report = run(ev, options);
PROPERTY(!report.top_windows.empty());
const auto& top = report.top_windows.front();
PROPERTY(top.root_node == "process_4000_b_1"); // R4
PROPERTY(top.detected && top.rules == std::vector<std::string>{"Suspicious_Python"});
PROPERTY(report.detected_windows == 1 && report.detected.size() == 1);
PROPERTY(top.bits > report.top_windows.back().bits);
std::cout << "R4 unusual chain ranks first (" << top.bits << " bits), detection marked: PASSED" << std::endl;
}
void test_store_round_trip() {
namespace fs = std::filesystem;
const auto dir = fs::temp_directory_path() / ("strawberry_replay_" + std::to_string(::getpid()));
std::error_code ec;
fs::remove_all(dir, ec);
auto ev = routine(20, kT0);
{
Database db;
PROPERTY(db.open(dir.string()));
auto copy = ev;
PROPERTY(db.append_telemetry(copy, true));
db.close();
}
std::string error;
const auto report = replay_store(dir.string(), no_warmup(), error);
PROPERTY(error.empty() && report.execs == 40 && report.windows == 20);
PROPERTY(replay_report_json(report) == replay_report_json(run(ev))); // R5
fs::remove_all(dir, ec);
std::cout << "R5 store round trip: PASSED" << std::endl;
}
void test_warmup_and_time_invariance() {
// Two novel programs, one at the start and one 20 h later after lots
// of routine activity: same score (capped novelty), not "later wins".
auto ev = routine(1200, kT0 + 60 * kSec);
ev.insert(ev.begin(), scan(kT0, 7, "process_7_b_1", {"EarlyRule"}));
ev.insert(ev.begin(), exec(kT0, 7, "process_7_b_1", "process_1_b_1", "/usr/bin/early-novel", "early-novel"));
const int64_t late = kT0 + 1201 * 61 * kSec;
ev.push_back(exec(late, 8, "process_8_b_1", "process_1_b_1", "/usr/bin/late-novel", "late-novel"));
auto late_scan = ev; // a record far older than the reorder horizon: counted as late
late_scan.push_back(scan(kT0 + 5, 7, "process_7_b_1", {"Late"}));
PROPERTY(run(late_scan).late_events == 1);
const auto all = run(ev);
const ReplayWindow* early = nullptr;
const ReplayWindow* late_w = nullptr;
for (const auto& w : all.top_windows) {
if (w.root_exe == "/usr/bin/early-novel") early = &w;
if (w.root_exe == "/usr/bin/late-novel") late_w = &w;
}
PROPERTY(early != nullptr && late_w != nullptr && early->bits == late_w->bits);
PROPERTY(early->bits == kNovelBits); // only exe_cmdline: its parent was never recorded
// The routine bash -> ls windows collapse into one line.
const auto routine_line = std::find_if(all.top_windows.begin(), all.top_windows.end(),
[](const auto& w) { return w.root_exe == "/usr/bin/bash" && w.similar > 100; });
PROPERTY(routine_line != all.top_windows.end());
ReplayOptions warm; // default 12 h warm-up
const auto report = run(ev, warm);
PROPERTY(report.warmup_windows > 0 && report.warmup_windows < report.windows);
for (const auto& w : report.top_windows) PROPERTY(w.start_ns >= kT0 + warm.warmup_ns);
PROPERTY(report.detected.size() == 1 && report.detected[0].rules == std::vector<std::string>{"EarlyRule"});
std::cout << "R7 warm-up, time invariance, grouping: PASSED" << std::endl;
}
void test_injection_evaluation() {
namespace fs = std::filesystem;
const auto dir = fs::temp_directory_path() / ("strawberry_inject_" + std::to_string(::getpid()));
std::error_code ec;
fs::remove_all(dir, ec);
// Two days of routine activity, one window a minute.
auto ev = routine(2 * 24 * 60, kT0);
{
Database db;
PROPERTY(db.open(dir.string()));
auto copy = ev;
PROPERTY(db.append_telemetry(copy, true));
db.close();
}
std::vector<Trace> traces;
std::string error;
PROPERTY(compile_traces("trace pdf_python\n"
"+0s exec viewer /usr/bin/evince \"evince /home/u/x.pdf\"\n"
"+3s exec py /usr/bin/python3 \"python3 -c import socket\" parent=viewer\n"
"+4s connect py 198.51.100.50 4444\n"
"trace quiet\n"
"+0s exec sh /usr/bin/bash \"bash\"\n",
traces, error));
ReplayOptions options;
options.warmup_ns = 6 * 3600 * kSec;
const auto plain = replay_store(dir.string(), options, error);
PROPERTY(error.empty());
options.inject = traces;
options.inject_copies = 4;
const auto report = replay_store(dir.string(), options, error);
PROPERTY(error.empty());
const auto& eval = report.evaluation;
PROPERTY(eval.traces.size() == 2 && eval.copies == 8);
PROPERTY(eval.traces[0].found == 4 && eval.traces[1].found == 4);
// Real activity is untouched: same records, windows and ranking.
PROPERTY(report.records == plain.records && eval.real_windows + report.warmup_windows == plain.windows);
PROPERTY(replay_report_json(plain).find("\"top_windows\"") != std::string::npos);
PROPERTY(nlohmann::json::parse(replay_report_json(report))["top_windows"] ==
nlohmann::json::parse(replay_report_json(plain))["top_windows"]);
// The novel chain is caught at 10/day; a bare bash window is not.
PROPERTY(eval.traces[0].caught.back() == 4);
PROPERTY(eval.traces[1].caught.back() < 4);
for (const auto r : eval.traces[0].ranks) PROPERTY(r == 1);
fs::remove_all(dir, ec);
std::cout << "R8 injection evaluation: PASSED (pdf_python ranks 1; cut at 10/day "
<< eval.cut_scores.back() << " bits)" << std::endl;
}
void test_late_labels() {
namespace fs = std::filesystem;
const std::string bad_hash(64, 'b');
auto ev = routine(10, kT0);
ProcessExecRecord r;
r.timestamp_ns = kT0 + 5 * kSec + 3;
r.pid = 600;
r.process_node_id = "process_600_b_1";
r.exe = "/home/u/.local/bin/tool";
r.cmdline = "tool";
r.sha256 = bad_hash;
r.source = "live";
ev.push_back({store::RecordType::process_exec, encode_process_exec(r)});
auto options = no_warmup();
options.late_labels[bad_hash] = {"BetaRule"};
const auto direct = run(ev, options);
PROPERTY(direct.late_detected_windows == 1 && direct.detected_windows == 0);
PROPERTY(direct.detected.size() == 1 && direct.detected[0].late_rules == std::vector<std::string>{"BetaRule"});
const auto dir = fs::temp_directory_path() / ("strawberry_late_" + std::to_string(::getpid()));
std::error_code ec;
fs::remove_all(dir, ec);
{
Database db;
PROPERTY(db.open(dir.string()));
auto copy = ev;
PROPERTY(db.append_telemetry(copy, true));
LabelRecord label;
label.timestamp_ns = kT0 + 3600 * kSec;
label.polarity = LabelPolarity::positive;
label.source = "retro_hunt";
label.action = "rules_update";
label.sha256 = bad_hash;
label.rule = "BetaRule";
PROPERTY(db.append_evidence(store::RecordType::label, encode_label(label)));
label.polarity = LabelPolarity::negative; // negatives are not late detections
label.sha256 = std::string(64, 'c');
PROPERTY(db.append_evidence(store::RecordType::label, encode_label(label)));
db.close();
}
std::string error;
const auto stored = replay_store(dir.string(), no_warmup(), error);
PROPERTY(error.empty() && stored.late_labels == 1 && stored.late_detected_windows == 1);
fs::remove_all(dir, ec);
std::cout << "R9 late labels: PASSED" << std::endl;
}
// Stage 3.5c: a window flagged late (retro-hunt) is attributed to the node that WROTE the flagged
// binary (the dropper) via the global write->exec linkage, not the exec leaf that ran it.
void test_attribution() {
const std::string bad_hash(64, 'd');
auto ev = routine(10, kT0);
// Dropper D (its own root) execs, then writes /tmp/evil.
ev.push_back(exec(kT0 + 2 * kSec, 500, "process_500_d_1", "process_1_b_1", "/usr/bin/curl", "curl x"));
ev.push_back(write(kT0 + 2 * kSec + 1, 500, "/tmp/evil"));
// Victim V (a SEPARATE root) execs the dropped binary; flagged bad later.
ProcessExecRecord r;
r.timestamp_ns = kT0 + 3 * kSec;
r.pid = 600;
r.process_node_id = "process_600_v_1";
r.parent_node_id = "process_1_b_1";
r.exe = "/tmp/evil";
r.cmdline = "evil";
r.sha256 = bad_hash;
r.source = "live";
ev.push_back({store::RecordType::process_exec, encode_process_exec(r)});
auto options = no_warmup();
options.late_labels[bad_hash] = {"DropperRule"};
auto victim = [](const ReplayReport& rep) -> const ReplayWindow* {
for (const auto& w : rep.detected) {
if (w.root_node == "process_600_v_1") return &w;
}
return nullptr;
};
const auto rep = run(ev, options);
const ReplayWindow* v = victim(rep);
PROPERTY(v != nullptr && v->late_detected);
PROPERTY(v->origin_node == "process_500_d_1"); // the dropper, not the exec leaf process_600_v_1
PROPERTY(v->origin_exe == "/usr/bin/curl");
// Determinism: same stream -> same attribution.
// Keep the report alive: victim() returns a pointer into it (a temporary here dangled; it passed
// on Linux by allocator luck and failed deterministically on macOS).
const auto rep2 = run(ev, options);
const ReplayWindow* v2 = victim(rep2);
PROPERTY(v2 != nullptr && v2->origin_node == "process_500_d_1");
// A flagged binary with no in-set write has no origin.
auto ev3 = routine(10, kT0);
ProcessExecRecord r3 = r;
r3.exe = "/usr/bin/true"; // never written in this stream
ev3.push_back({store::RecordType::process_exec, encode_process_exec(r3)});
auto opt3 = no_warmup();
opt3.late_labels[bad_hash] = {"DropperRule"};
const auto rep3 = run(ev3, opt3); // keep alive: victim() points into it
const ReplayWindow* v3 = victim(rep3);
PROPERTY(v3 != nullptr && v3->late_detected && v3->origin_node.empty());
std::cout << "R10 attribution: late_detected traced to dropper node: PASSED" << std::endl;
}
void test_examples_and_model_ranking() {
namespace fs = std::filesystem;
auto ev = routine(200, kT0);
const int64_t t = kT0 + 201 * 61 * kSec;
ev.push_back(exec(t, 4000, "process_4000_b_1", "process_1_b_1", "/usr/bin/evince", "evince x.pdf"));
ev.push_back(scan(t, 4000, "process_4000_b_1", {"SomeRule"}));
const auto dir = fs::temp_directory_path() / ("strawberry_examples_" + std::to_string(::getpid()));
std::error_code ec;
fs::remove_all(dir, ec);
{
Database db;
PROPERTY(db.open(dir.string()));
auto copy = ev;
PROPERTY(db.append_telemetry(copy, true));
db.close();
}
std::vector<Trace> traces;
std::string error;
PROPERTY(compile_traces("trace train_chain\n+0s exec a /tmp/x/dropper \"dropper\"\n"
"+1s connect a 198.51.100.9 4444\n", traces, error));
ReplayOptions o = no_warmup();
o.collect_examples = true;
o.inject = traces;
o.inject_copies = 3;
o.injected_are_positives = true;
const auto r = replay_store(dir.string(), o, error);
PROPERTY(error.empty());
PROPERTY(r.example_windows == 201 + 3 && r.example_positives == 1 + 3);
PROPERTY(r.evaluation.copies == 0); // training traces are not evaluated
PROPERTY(r.examples.size() == r.example_windows);
std::size_t positives = 0;
for (const auto& e : r.examples) positives += e.label > 0.5f;
PROPERTY(positives == 4);
o.max_examples = 100;
o.max_entity_share = 0.05; // at most 5 bash windows
const auto capped = replay_store(dir.string(), o, error);
std::size_t bash = 0;
for (const auto& e : capped.examples) bash += e.entity == "/usr/bin/bash" ? 1 : 0;
PROPERTY(bash == 5 && capped.examples.size() == 5 + 4 + 0); // 5 bash + 4 positives (evince is positive)
auto model = std::make_shared<Model>(Model::initialized(ModelShape{}, 3));
ReplayOptions ranked = no_warmup();
ranked.model = model;
const auto m = replay_store(dir.string(), ranked, error);
PROPERTY(!m.top_windows.empty());
for (const auto& w : m.top_windows) {
PROPERTY(w.model_value >= 0.0f && w.model_value <= 1.0f);
PROPERTY(std::fabs(w.bits - w.model_value * 48.0) < 1e-3);
}
fs::remove_all(dir, ec);
std::cout << "R10 examples and model ranking: PASSED" << std::endl;
}
void test_positive_sampling_under_pressure() {
// Fill every slot with distinct negative entities before any teacher
// positive arrives. Late labels must still get their class quota.
auto events = std::vector<TelemetryEvent>{};
for (int i = 0; i < 100; ++i) {
const auto ts = kT0 + i * 61 * kSec;
const auto node = "process_" + std::to_string(5000 + i) + "_b_1";
const std::string entity = std::string("/usr/bin/sample_") +
static_cast<char>('a' + i / 26) + static_cast<char>('a' + i % 26);
events.push_back(exec(ts, 5000 + i, node, "", entity, entity));
if (i >= 60) events.push_back(scan(ts, 5000 + i, node, {"Teacher"}));
}
ReplayOptions o = no_warmup();
o.collect_examples = true;
o.max_examples = 20;
o.max_entity_share = 0.05; // one negative per entity
o.sample_seed = kSeed;
const auto sampled = run(events, o);
PROPERTY(sampled.example_windows == 100 && sampled.example_positives == 40);
PROPERTY(sampled.examples.size() == o.max_examples);
std::size_t positives = 0;
bool retained_later_positive = false;
for (const auto& e : sampled.examples) {
positives += e.label > 0.5f;
retained_later_positive |= e.label > 0.5f && e.start_ns >= kT0 + 64 * 61 * kSec;
}
PROPERTY(positives == 4); // the positive quota survives a full negative sample
// Check reproducibility including examples, which report JSON does not emit.
const auto again = run(events, o);
PROPERTY(again.examples.size() == sampled.examples.size());
for (std::size_t i = 0; i < sampled.examples.size(); ++i) {
PROPERTY(sampled.examples[i].start_ns == again.examples[i].start_ns);
PROPERTY(sampled.examples[i].entity == again.examples[i].entity);
PROPERTY(sampled.examples[i].label == again.examples[i].label);
PROPERTY(sampled.examples[i].x == again.examples[i].x);
}
// Do not assert that an arbitrary seed must choose a later example:
// reservoir sampling may legitimately retain the first four. Across
// fixed seeds, however, the implementation cannot keep only the first.
for (uint64_t seed = 1; seed <= 8; ++seed) {
o.sample_seed = seed;
for (const auto& e : run(events, o).examples) {
retained_later_positive |= e.label > 0.5f && e.start_ns >= kT0 + 64 * 61 * kSec;
}
}
PROPERTY(retained_later_positive);
// Tiny configured samples must retain a teacher example too.
for (std::size_t capacity = 1; capacity < 5; ++capacity) {
o.max_examples = capacity;
const auto tiny = run(events, o);
PROPERTY(tiny.examples.size() == capacity);
PROPERTY(std::count_if(tiny.examples.begin(), tiny.examples.end(),
[](const auto& e) { return e.label > 0.5f; }) == 1);
}
std::cout << "R10 positive sampling under pressure: PASSED" << std::endl;
}
void test_policy_evaluation() {
// Gate as a pure function. Checks: positives, baseline health, efficiency,
// recall, out-of-sample.
PolicyEvaluation pe;
pe.extra_positives = 12;
pe.policies = {{"model", 10, 20.0, 9, 9.0, 0}, {"rarity_greedy", 20, 40.0, 8, 8.0, 0}};
pe.provenance.model = "seed1";
pe.provenance.model_sha256 = "0123456789abcdef";
pe.provenance.training_range_known = true;
pe.provenance.training_first_ns = 1000;
pe.provenance.training_last_ns = 2000;
pe.provenance.eval_first_ns = 3000;
pe.provenance.eval_last_ns = 4000;
auto g = promotion_gate(pe); // 0.45 vs 0.2 hits/CPU-s, recall 9 >= 8
PROPERTY(g.passed && g.checks.size() == 5);
PROPERTY(g.checks[4].rfind("ok: out-of-sample", 0) == 0);
pe.policies[0].hits = 7; // efficiency 0.35 (still >= 0.3) but lower recall
pe.policies[0].weighted_hits = 7;
g = promotion_gate(pe);
PROPERTY(!g.passed && g.checks[3].rfind("FAIL", 0) == 0 && g.checks[2].rfind("ok", 0) == 0);
pe.policies[0] = {"model", 10, 40.0, 9, 9.0, 0}; // 0.225 < 1.5 x 0.2
PROPERTY(!promotion_gate(pe).passed && promotion_gate(pe).checks[2].rfind("FAIL", 0) == 0);
// Degenerate baseline: a rarity-greedy baseline with no hits or no CPU-seconds
// makes criteria 2 and 3 pass for free, so the cell must fail and say why.
PolicyEvaluation degenerate = pe;
degenerate.extra_positives = 12;
degenerate.policies = {{"model", 10, 20.0, 9, 9.0, 0}, {"rarity_greedy", 10, 0.0, 0, 0.0, 7}};
auto dg = promotion_gate(degenerate);
PROPERTY(!dg.passed);
PROPERTY(dg.checks[1].rfind("FAIL", 0) == 0 && dg.checks[1].find("degenerate baseline") != std::string::npos);
PROPERTY(dg.checks[1].find("7 budget refusals") != std::string::npos);
degenerate.policies[1] = {"rarity_greedy", 10, 5.0, 0, 0.0, 0}; // 0 hits, CPU > 0
PROPERTY(!promotion_gate(degenerate).passed &&
promotion_gate(degenerate).checks[1].find("degenerate baseline") != std::string::npos);
degenerate.policies[1] = {"rarity_greedy", 10, 0.0, 3, 3.0, 0}; // hits, no CPU
PROPERTY(!promotion_gate(degenerate).passed &&
promotion_gate(degenerate).checks[1].find("degenerate baseline") != std::string::npos);
// In-sample evaluation: the gate says so and refuses.
PolicyEvaluation in_sample = pe;
in_sample.extra_positives = 12;
in_sample.policies = {{"model", 10, 20.0, 9, 9.0, 0}, {"rarity_greedy", 20, 40.0, 8, 8.0, 0}};
in_sample.provenance.eval_first_ns = 1500; // inside the training range
in_sample.provenance.in_sample = true;
auto is = promotion_gate(in_sample);
PROPERTY(!is.passed);
PROPERTY(is.checks[4].rfind("FAIL", 0) == 0 && is.checks[4].find("in-sample evaluation") != std::string::npos);
PROPERTY(is.checks[4].find("0123456789abcdef") != std::string::npos);
// Unprovable provenance never counts as out-of-sample.
PolicyEvaluation unknown = in_sample;
unknown.provenance.in_sample = false;
unknown.provenance.training_range_known = false;
PROPERTY(!promotion_gate(unknown).passed &&
promotion_gate(unknown).checks[4].find("no recorded training range") != std::string::npos);
PolicyEvaluation unbound = in_sample;
unbound.provenance.in_sample = false;
unbound.provenance.model_sha256.clear();
PROPERTY(!promotion_gate(unbound).passed &&
promotion_gate(unbound).checks[4].find("no model digest") != std::string::npos);
pe.extra_positives = 4;
pe.policies[0] = {"model", 10, 20.0, 4, 4.0, 0};
PROPERTY(!promotion_gate(pe).passed && promotion_gate(pe).checks[0].rfind("FAIL", 0) == 0);
pe.policies.pop_back();
PROPERTY(!promotion_gate(pe).passed);
// Replay: routine history, injected evaluation traces, a rule detection,
// and logged shadow decisions (one on an injected window).
auto ev = routine(240, kT0);
const int64_t t = kT0 + 30 * 61 * kSec + 7;
ev.push_back(exec(t, 4000, "process_4000_b_1", "process_1_b_1", "/usr/bin/evince", "evince x.pdf"));
ev.push_back(scan(t, 4000, "process_4000_b_1", {"SomeRule"}));
InspectionDecision logged;
logged.timestamp_ns = t + 200 * kSec;
logged.root_node = "process_4000_b_1";
logged.action = InspectAction::memory_scan;
ev.push_back({store::RecordType::inspection_action, encode_inspection_action(logged)});
logged.shadow = false;
ev.push_back({store::RecordType::inspection_action, encode_inspection_action(logged)});
std::vector<Trace> traces;
std::string error;
PROPERTY(compile_traces("trace odd\n+0s exec a /tmp/q/odd \"odd --x\"\n+1s connect a 198.51.100.4 4444\n",
traces, error));
ReplayOptions o;
o.warmup_ns = 600 * kSec; // routine windows are known by then
o.policy_eval = true;
o.policy.budget_seconds_per_hour = 4; // two memory scans an hour
// Even an apparently perfect free unsupported action cannot influence
// the deployment gate. Without the executable mask this evades budgets.
o.policy.reach[static_cast<std::size_t>(InspectAction::watch_subtree)] = 1000;
o.policy.cost_seconds[static_cast<std::size_t>(InspectAction::watch_subtree)] = 0;
o.inject = traces;
o.inject_copies = 4;
// Through a store, so injection is planned over the history's time range.
namespace fs = std::filesystem;
const auto dir = fs::temp_directory_path() / ("strawberry_policy_" + std::to_string(::getpid()));
std::error_code ec;
fs::remove_all(dir, ec);
{
Database db;
PROPERTY(db.open(dir.string()));
auto copy = ev;
PROPERTY(db.append_telemetry(copy, true));
db.close();
}
const auto r = replay_store(dir.string(), o, error);
PROPERTY(error.empty());
const auto& p = r.policy_eval;
PROPERTY(p.extra_positives == 4 && p.rule_detected == 1); // the evince window is rules' already
PROPERTY(p.policies.size() == 2 && p.policies[0].name == "rarity_greedy" && p.policies[1].name == "shadow_logged");
const auto& rarity = p.policies[0];
// Structure only: with a few hundred windows rarity has not seen enough
// (its smoothing wants ~30k observations) to separate anything.
PROPERTY(rarity.hits <= p.extra_positives && rarity.inspections + rarity.budget_refusals <= p.windows);
PROPERTY(rarity.budget_refusals > 0);
PROPERTY(rarity.cpu_seconds <= 4.0 * (static_cast<double>(r.last_ns - r.first_ns) / 3.6e12 + 1.0));
const auto& shadow = p.policies[1];
PROPERTY(shadow.inspections == 1 && shadow.hits == 0); // it inspected a rule-flagged window: not an extra hit
fs::remove_all(dir, ec);
std::cout << "R11 policy evaluation and gate: PASSED (rarity-greedy " << rarity.hits << "/" << p.extra_positives
<< " in " << rarity.cpu_seconds << " CPU-s)" << std::endl;
}
// R11b fleet gate cells: with one eval model equal to `model` and one period, the cell is exactly
// the single-model evaluation; with N periods, cells partition the windows and positives.
void test_gate_cells() {
auto ev = routine(240, kT0);
std::vector<Trace> traces;
std::string error;
PROPERTY(compile_traces("trace odd\n+0s exec a /tmp/q/odd \"odd --x\"\n+1s connect a 198.51.100.4 4444\n",
traces, error));
namespace fs = std::filesystem;
const auto dir = fs::temp_directory_path() / ("strawberry_gate_cells_" + std::to_string(::getpid()));
std::error_code ec;
fs::remove_all(dir, ec);
{
Database db;
PROPERTY(db.open(dir.string()));
auto copy = ev;
PROPERTY(db.append_telemetry(copy, true));
db.close();
}
auto ma = std::make_shared<Model>(Model::initialized(ModelShape{}, 11));
auto mb = std::make_shared<Model>(Model::initialized(ModelShape{}, 12));
ReplayOptions o;
o.warmup_ns = 600 * kSec;
o.policy_eval = true;
o.policy.budget_seconds_per_hour = 4;
o.inject = traces;
o.inject_copies = 9;
o.model = ma;
o.eval_models = {{"a", ma}};
o.eval_periods = 1;
const auto r1 = replay_store(dir.string(), o, error);
PROPERTY(error.empty());
const auto& pe = r1.policy_eval;
PROPERTY(pe.cells.size() == 1 && pe.cells[0].model == "a" && pe.cells[0].period == 0);
const auto& cell = pe.cells[0];
PROPERTY(cell.windows == pe.windows && cell.extra_positives == pe.extra_positives && cell.rule_detected == pe.rule_detected);
const auto find = [](const std::vector<PolicyResult>& ps, const std::string& n) -> const PolicyResult* {
for (const auto& p : ps) if (p.name == n) return &p;
return nullptr;
};
for (const char* name : {"model", "rarity_greedy"}) {
const auto* agg = find(pe.policies, name);
const auto* c = find(cell.policies, name);
PROPERTY(agg && c && agg->hits == c->hits && agg->inspections == c->inspections &&
agg->budget_refusals == c->budget_refusals && agg->cpu_seconds == c->cpu_seconds);
}
// The gate applies to a cell unchanged -- and now also refuses to count an
// unproven or in-sample evaluation as evidence.
PolicyEvaluation as_eval;
as_eval.extra_positives = cell.extra_positives;
as_eval.policies = cell.policies;
as_eval.provenance = cell.provenance;
PROPERTY(promotion_gate(as_eval).checks.size() == 5);
// The cell carries the model's content identity and its (here: absent) range.
PROPERTY(!cell.provenance.model_sha256.empty());
PROPERTY(cell.provenance.model_sha256 == model_fingerprint(*ma));
PROPERTY(!cell.provenance.training_range_known && !cell.provenance.in_sample);
PROPERTY(!promotion_gate(as_eval).passed &&
promotion_gate(as_eval).checks[4].find("no recorded training range") != std::string::npos);
// Two models x three periods: 6 cells; per model the periods partition windows and positives;
// the periods are ordered and cover the ranked history.
o.eval_models = {{"a", ma}, {"b", mb}};
o.eval_periods = 3;
const auto r3 = replay_store(dir.string(), o, error);
PROPERTY(error.empty() && r3.policy_eval.cells.size() == 6);
for (const char* m : {"a", "b"}) {
uint64_t windows = 0, positives = 0;
int64_t prev_end = INT64_MIN;
for (const auto& c : r3.policy_eval.cells) {
if (c.model != m) continue;
windows += c.windows;
positives += c.extra_positives;
PROPERTY(c.start_ns <= c.end_ns && c.start_ns >= prev_end);
prev_end = c.end_ns;
}
PROPERTY(windows == r3.policy_eval.windows && positives == r3.policy_eval.extra_positives);
}
// Injected copies spread over the history: every period gets positives.
for (const auto& c : r3.policy_eval.cells) PROPERTY(c.extra_positives > 0);
const auto json = replay_report_json(r3);
PROPERTY(json.find("\"cells\"") != std::string::npos);
// A model trained on the replayed history makes every cell in-sample: the
// cell says so, and the gate refuses it naming the reason.
auto trained = std::make_shared<Model>(Model::initialized(ModelShape{}, 11));
trained->training_first_ns = kT0;
trained->training_last_ns = kT0 + 240 * 61 * kSec;
o.eval_models = {{"trained", trained}};
o.eval_periods = 2;
const auto ri = replay_store(dir.string(), o, error);
PROPERTY(error.empty() && ri.policy_eval.cells.size() == 2);
for (const auto& c : ri.policy_eval.cells) {
PROPERTY(c.provenance.training_range_known && c.provenance.in_sample);
PROPERTY(c.provenance.model_sha256 == model_fingerprint(*trained));
PolicyEvaluation ev;
ev.extra_positives = c.extra_positives;
ev.policies = c.policies;
ev.provenance = c.provenance;
const auto gate_in = promotion_gate(ev);
PROPERTY(!gate_in.passed && gate_in.checks[4].find("in-sample evaluation") != std::string::npos);
}
const auto in_sample_json = replay_report_json(ri);
PROPERTY(in_sample_json.find("\"model_sha256\"") != std::string::npos);
PROPERTY(in_sample_json.find("\"in_sample\": true") != std::string::npos);
// A model trained on an earlier, disjoint period is not flagged in-sample.
auto earlier = std::make_shared<Model>(Model::initialized(ModelShape{}, 11));
earlier->training_first_ns = kT0 - 1000 * kSec;
earlier->training_last_ns = kT0 - 500 * kSec;
o.eval_models = {{"earlier", earlier}};
const auto rd = replay_store(dir.string(), o, error);
PROPERTY(error.empty() && !rd.policy_eval.cells.empty());
for (const auto& c : rd.policy_eval.cells) PROPERTY(c.provenance.training_range_known && !c.provenance.in_sample);
fs::remove_all(dir, ec);
std::cout << "R11b fleet gate cells: PASSED (" << r3.policy_eval.cells.size() << " cells, "
<< r3.policy_eval.extra_positives << " positives)" << std::endl;
}
void test_extreme_timestamps() {
// On-disk timestamps are nonnegative int64 values, not necessarily sane
// wall-clock dates. Interval comparisons must not overflow near the end.
ReplayOptions options;
options.warmup_ns = 3600 * kSec;
Replayer replay(options);
const auto first = exec(INT64_MAX - 100 * kSec, 42, "process_42_b_1", "", "/tmp/a", "a");
const auto last = exec(INT64_MAX - 1, 43, "process_43_b_1", "", "/tmp/b", "b");
replay.feed(first.type, first.payload);
replay.feed(last.type, last.payload);
const auto report = replay.finish();
PROPERTY(report.warmup_windows == 2 && report.top_windows.empty());
std::string error;
PROPERTY(compile_traces("trace edge\n+0s exec a /tmp/edge edge\n", options.inject, error));
options.inject_copies = 1;
// Field iteration accepts repeated tags. The store indexes the first
// timestamp while the typed decoder retains the last, so the window
// may start after report.last_ns even though both times are nonnegative.
auto duplicate = exec(1, 42, "process_42_b_1", "", "/tmp/a", "a");
store::FieldWriter suffix;
suffix.u64(1, INT64_MAX - 1);
duplicate.payload += suffix.data();
Replayer mismatched(options);
mismatched.feed(duplicate.type, duplicate.payload);
PROPERTY(mismatched.finish().evaluation.days == 1.0 / 24);
namespace fs = std::filesystem;
const auto dir = fs::temp_directory_path() / ("strawberry_replay_edge_" + std::to_string(::getpid()));
std::error_code ec;
fs::remove_all(dir, ec);
{
Database db;
PROPERTY(db.open(dir.string()));
std::vector<TelemetryEvent> events{first, last};
PROPERTY(db.append_telemetry(events, true));
}
(void) replay_store(dir.string(), options, error);
PROPERTY(error.find("not enough history") != std::string::npos);
fs::remove_all(dir, ec);
}
void test_invalid_utf8() {
// Split literals: "\x80" "evil", or the escape would swallow the 'e'.
const std::string bad = std::string("/tmp/\xff\xfe\xc0\x80" "evil\xed\xa0\x80") + "\xf4\x90\x80\x80";
std::vector<TelemetryEvent> ev{exec(kT0, 1, "process_1_b_1", "", bad, bad + " -x")};
const auto json = replay_report_json(run(ev));
const auto parsed = nlohmann::json::parse(json, nullptr, false);
PROPERTY(!parsed.is_discarded());
PROPERTY(!valid_utf8(bad) && valid_utf8(json));
std::mt19937_64 rng(kSeed);
for (int i = 0; i < 20000; ++i) {
std::string s;
const auto n = rng() % 12;
for (std::size_t k = 0; k < n; ++k) s.push_back(static_cast<char>(rng() % 256));
const auto quoted = json_string(s);
const auto back = nlohmann::json::parse(quoted, nullptr, false);
PROPERTY(!back.is_discarded() && back.is_string());
if (valid_utf8(s)) PROPERTY(back.get<std::string>() == s); // valid text is unchanged
}
PROPERTY(valid_utf8("plain ascii") && valid_utf8("\xc3\xa9t\xc3\xa9") && !valid_utf8("\xc3"));
std::cout << "R6 invalid UTF-8 stays valid JSON: PASSED" << std::endl;
}
} // namespace
// A window running an allowlisted (negative-labelled) binary is a confirmed
// benign: kept as a negative training example, never a teacher positive, even
// though a (now-cleared) rule detected it. This is what keeps a false-positive
// feed hit the operator allowlisted from poisoning the labels.
void test_allowlisted_window_is_negative() {
namespace fs = std::filesystem;
const std::string sha(64, 'd');
auto ev = routine(10, kT0); // benign windows (label 0)
ProcessExecRecord r;
r.timestamp_ns = kT0 + 5 * kSec + 7;
r.pid = 700;
r.process_node_id = "process_700_d_1";
r.exe = "/usr/bin/ss";
r.cmdline = "ss -tulpn";
r.sha256 = sha;
r.source = "live";
ev.push_back({store::RecordType::process_exec, encode_process_exec(r)});
ev.push_back(scan(r.timestamp_ns + 1, 700, "process_700_d_1", {"IOC_KnownMalwareHash"})); // detected
const auto dir = fs::temp_directory_path() / ("strawberry_allow_neg_" + std::to_string(::getpid()));
std::error_code ec;
fs::remove_all(dir, ec);
{
Database db;
PROPERTY(db.open(dir.string()));
auto copy = ev;
PROPERTY(db.append_telemetry(copy, true));
db.close();
}
std::string error;
ReplayOptions o = no_warmup();
o.collect_examples = true;
const auto before = replay_store(dir.string(), o, error);
std::size_t pos_before = 0;
for (const auto& e : before.examples) pos_before += e.label > 0.5f;
PROPERTY(error.empty() && pos_before == 1); // the detected ss window is a positive
{ // the operator allowlists the hash -> a negative label in the stream
Database db;
PROPERTY(db.open(dir.string()));
LabelRecord label;
label.timestamp_ns = kT0 + 6 * kSec;
label.polarity = LabelPolarity::negative;
label.source = "allowlist";
label.action = "suppressed";
label.sha256 = sha;
PROPERTY(db.append_evidence(store::RecordType::label, encode_label(label)));
db.close();
}
const auto after = replay_store(dir.string(), o, error);
std::size_t pos_after = 0;
for (const auto& e : after.examples) pos_after += e.label > 0.5f;
PROPERTY(error.empty() && pos_after == 0); // no longer a positive
PROPERTY(after.examples.size() == before.examples.size()); // kept as a negative, not dropped
fs::remove_all(dir, ec);
std::cout << "allowlisted window is a negative: PASSED" << std::endl;
}
// ---- label retraction (R12) ----
void append_label(const std::string& dir, LabelPolarity polarity, const std::string& action,
const std::string& sha, const std::string& entry, const std::string& source,
const std::string& rule, int64_t timestamp = kT0 + 3600 * kSec) {
Database db;
PROPERTY(db.open(dir));
LabelRecord label;
label.timestamp_ns = timestamp;
label.polarity = polarity;
label.source = source;
label.action = action;
label.sha256 = sha;
label.allowlist_entry = entry;
label.rule = rule;
PROPERTY(db.append_evidence(store::RecordType::label, encode_label(label)));
db.close();
}
std::size_t positives_in(const ReplayReport& r) {
std::size_t n = 0;
for (const auto& e : r.examples) n += e.label > 0.5f;
return n;
}
const ReplayWindow* detected_rooted(const ReplayReport& report, const std::string& node) {
for (const auto& w : report.detected) {
if (w.root_node == node) return &w;
}
return nullptr;
}
// A store with a few routine windows plus one window running `sha` (`exe`),
// optionally rule-detected.
std::filesystem::path make_retract_store(const std::string& tag, const std::string& sha, const std::string& exe,
bool detected) {
namespace fs = std::filesystem;
const auto dir = fs::temp_directory_path() / ("strawberry_" + tag + "_" + std::to_string(::getpid()));
std::error_code ec;
fs::remove_all(dir, ec);
auto ev = routine(5, kT0);
ProcessExecRecord r;
r.timestamp_ns = kT0 + 3 * kSec + 7;
r.pid = 700;
r.process_node_id = "process_700_b_1";
r.exe = exe;
r.cmdline = "x";
r.sha256 = sha;
r.source = "live";
ev.push_back({store::RecordType::process_exec, encode_process_exec(r)});
if (detected) ev.push_back(scan(r.timestamp_ns + 1, 700, "process_700_b_1", {"IOC_KnownMalwareHash"}));
Database db;
PROPERTY(db.open(dir.string()));
auto copy = ev;
PROPERTY(db.append_telemetry(copy, true));
db.close();
return dir;
}
void test_retraction_allowlist_remove() {
namespace fs = std::filesystem;
const std::string sha(64, 'd');
const auto dir = make_retract_store("retract_al", sha, "/usr/bin/ss", true);
std::string error;
ReplayOptions o = no_warmup();
o.collect_examples = true;
append_label(dir.string(), LabelPolarity::negative, "allowlist-add", sha, "allowlist-e1", "operator", "");
const auto after_add = replay_store(dir.string(), o, error);
PROPERTY(error.empty() && positives_in(after_add) == 0); // cleared false positive
append_label(dir.string(), LabelPolarity::retracted, "allowlist-remove", "", "allowlist-e1", "operator", "");
const auto after_remove = replay_store(dir.string(), o, error);
// The removal reverts the hash to its pre-label state: the rule detection
// stands again (it was the only judgment). It is not a new teacher positive.
PROPERTY(error.empty() && positives_in(after_remove) == 1);
fs::remove_all(dir);
std::cout << "R12 allowlist-remove retraction: PASSED" << std::endl;
}
void test_retraction_quarantine_remove() {
namespace fs = std::filesystem;
const std::string sha(64, 'b');
const auto dir = make_retract_store("retract_q", sha, "/tmp/x/dropper", false);
std::string error;
ReplayOptions o = no_warmup();
o.collect_examples = true;
append_label(dir.string(), LabelPolarity::positive, "quarantine", sha, "", "operator", "");
const auto after_quarantine = replay_store(dir.string(), o, error);
PROPERTY(error.empty() && positives_in(after_quarantine) == 1); // late-detected via quarantine
append_label(dir.string(), LabelPolarity::retracted, "quarantine-remove", sha, "", "operator", "");
const auto after_remove = replay_store(dir.string(), o, error);
PROPERTY(error.empty() && positives_in(after_remove) == 0); // the quarantine reason is gone
fs::remove_all(dir);
std::cout << "R12 quarantine-remove retraction: PASSED" << std::endl;
}
void test_retraction_independent_positives() {
namespace fs = std::filesystem;
const std::string sha(64, 'c');
const auto dir = make_retract_store("retract_indep", sha, "/tmp/x/dropper", false);
std::string error;
ReplayOptions o = no_warmup();
o.collect_examples = true;
// Two independent positives on one hash: an operator quarantine and a
// retro-hunt rule. Removing the quarantine must not erase the rule reason.
append_label(dir.string(), LabelPolarity::positive, "quarantine", sha, "", "operator", "");
append_label(dir.string(), LabelPolarity::positive, "rules_update", sha, "", "rules_update", "BetaRule");
const auto before = replay_store(dir.string(), o, error);
PROPERTY(error.empty() && positives_in(before) == 1);
const auto* before_w = detected_rooted(before, "process_700_b_1");
PROPERTY(before_w != nullptr && before_w->late_rules.size() == 2);
append_label(dir.string(), LabelPolarity::retracted, "quarantine-remove", sha, "", "operator", "");
const auto after = replay_store(dir.string(), o, error);
PROPERTY(error.empty() && positives_in(after) == 1); // still late-detected via BetaRule
const auto* after_w = detected_rooted(after, "process_700_b_1");
PROPERTY(after_w != nullptr && after_w->late_rules == std::vector<std::string>{"BetaRule"});
fs::remove_all(dir);
std::cout << "R12 independent positives survive quarantine-remove: PASSED" << std::endl;
}
void test_retraction_multiple_entries() {
namespace fs = std::filesystem;
const std::string sha(64, 'e');
const auto dir = make_retract_store("retract_multi", sha, "/usr/bin/ss", true);
std::string error;
ReplayOptions o = no_warmup();
o.collect_examples = true;
append_label(dir.string(), LabelPolarity::negative, "allowlist-add", sha, "allowlist-e1", "operator", "");
append_label(dir.string(), LabelPolarity::negative, "allowlist-add", sha, "allowlist-e2", "operator", "");
const auto both = replay_store(dir.string(), o, error);
PROPERTY(error.empty() && positives_in(both) == 0);
append_label(dir.string(), LabelPolarity::retracted, "allowlist-remove", "", "allowlist-e1", "operator", "");
const auto one_left = replay_store(dir.string(), o, error);
PROPERTY(error.empty() && positives_in(one_left) == 0); // e2 still covers the hash
append_label(dir.string(), LabelPolarity::retracted, "allowlist-remove", "", "allowlist-e2", "operator", "");
const auto none = replay_store(dir.string(), o, error);
PROPERTY(error.empty() && positives_in(none) == 1); // both entries gone
fs::remove_all(dir);
std::cout << "R12 multiple allowlist entries resolve independently: PASSED" << std::endl;
}
void test_retraction_readd() {
namespace fs = std::filesystem;
const std::string sha(64, 'f');
const auto dir = make_retract_store("retract_readd", sha, "/usr/bin/ss", true);
std::string error;
ReplayOptions o = no_warmup();
o.collect_examples = true;
append_label(dir.string(), LabelPolarity::negative, "allowlist-add", sha, "allowlist-e1", "operator", "");
PROPERTY(error.empty() && positives_in(replay_store(dir.string(), o, error)) == 0);
append_label(dir.string(), LabelPolarity::retracted, "allowlist-remove", "", "allowlist-e1", "operator", "");
PROPERTY(error.empty() && positives_in(replay_store(dir.string(), o, error)) == 1);
append_label(dir.string(), LabelPolarity::negative, "allowlist-add", sha, "allowlist-e1", "operator", "");
PROPERTY(error.empty() && positives_in(replay_store(dir.string(), o, error)) == 0); // re-added
fs::remove_all(dir);
std::cout << "R12 add/remove/re-add resolves in stored order: PASSED" << std::endl;
}
// ---- positive per-entity cap (R13) ----
void test_positive_entity_cap() {
std::vector<TelemetryEvent> ev;
// One hot entity with many detected (positive) windows, plus many distinct
// negative entities to fill the sample.
for (int i = 0; i < 60; ++i) {
const auto ts = kT0 + i * 61 * kSec;
const auto node = "process_" + std::to_string(6000 + i) + "_b_1";
ev.push_back(exec(ts, 6000 + i, node, "", "/usr/bin/hot", "hot --x"));
ev.push_back(scan(ts, 6000 + i, node, {"Teacher"}));
}
for (int i = 0; i < 200; ++i) {
const auto ts = kT0 + 1000 * 61 * kSec + i * 61 * kSec;
const auto node = "process_" + std::to_string(8000 + i) + "_b_1";
const std::string entity =
std::string("/usr/bin/neg_") + static_cast<char>('a' + i / 26) + static_cast<char>('a' + i % 26);
ev.push_back(exec(ts, 8000 + i, node, "", entity, entity));
}
ReplayOptions o = no_warmup();
o.collect_examples = true;
o.max_examples = 50;
o.max_entity_share = 0.1; // combined cap = 5
o.sample_seed = kSeed;
const auto r = run(ev, o);
// Combined per-entity cap holds for every entity.
std::unordered_map<std::string, std::size_t> per_entity;
for (const auto& e : r.examples) ++per_entity[e.entity];
for (const auto& [entity, n] : per_entity) PROPERTY(n <= 5);
// The hot entity's 60 raw positives are capped to at most 5, and the
// rejection is counted (distinguishable from accidental starvation).
PROPERTY(per_entity["/usr/bin/hot"] <= 5);
// Extra candidates may refresh existing slots, so not every arrival
// beyond the cap is a rejection.
PROPERTY(r.example_positive_entity_rejections <= 55);
// The positive quota still bounds the retained positives.
PROPERTY(positives_in(r) <= o.max_examples / 5);
PROPERTY(r.examples.size() <= o.max_examples);
// Deterministic per seed.
const auto again = run(ev, o);
PROPERTY(again.examples.size() == r.examples.size());
for (std::size_t i = 0; i < r.examples.size(); ++i) {
PROPERTY(r.examples[i].start_ns == again.examples[i].start_ns);
PROPERTY(r.examples[i].entity == again.examples[i].entity);
PROPERTY(r.examples[i].label == again.examples[i].label);
}
std::cout << "R13 positive entity cap: PASSED (hot entity " << per_entity["/usr/bin/hot"]
<< " examples, " << r.example_positive_entity_rejections << " concentration rejections)" << std::endl;
}
void test_positive_not_starved_by_entity_negatives() {
std::vector<TelemetryEvent> ev;
// Entity "/usr/bin/mixed" first contributes many negatives (filling its
// combined cap), then one detected positive. The positive must displace a
// negative of the same entity rather than being starved by the negatives.
for (int i = 0; i < 10; ++i) {
const auto ts = kT0 + i * 61 * kSec;
const auto node = "process_" + std::to_string(7000 + i) + "_b_1";
ev.push_back(exec(ts, 7000 + i, node, "", "/usr/bin/mixed", "mixed"));
}
const int64_t tpos = kT0 + 100 * 61 * kSec;
ev.push_back(exec(tpos, 7999, "process_7999_b_1", "", "/usr/bin/mixed", "mixed"));
ev.push_back(scan(tpos, 7999, "process_7999_b_1", {"Teacher"}));
ReplayOptions o = no_warmup();
o.collect_examples = true;
o.max_examples = 8;
o.max_entity_share = 0.5; // combined cap = 4
o.sample_seed = kSeed;
const auto r = run(ev, o);
PROPERTY(positives_in(r) == 1); // the late positive is retained
std::size_t mixed = 0;
for (const auto& e : r.examples) mixed += e.entity == "/usr/bin/mixed";
PROPERTY(mixed <= 4); // combined cap held even though negatives preceded the positive
bool retained_positive = false;
for (const auto& e : r.examples) retained_positive |= e.label > 0.5f && e.entity == "/usr/bin/mixed";
PROPERTY(retained_positive);
std::cout << "R13 positive displaces same-entity negatives: PASSED" << std::endl;
}
void test_sampler_capacity_lifetime() {
// The debug sampler invariant checks ALLOCATED per-entity capacity, not
// just logical slot counts. Empty class vectors must release old storage.
auto o = no_warmup();
o.collect_examples = true;
o.max_examples = 1;
o.max_entity_share = 1;
o.sample_seed = 1;
auto events = std::vector<TelemetryEvent>{
exec(kT0, 9400, "process_9400_b_1", "", "/usr/bin/kept", "kept"),
exec(kT0 + 61 * kSec, 9401, "process_9401_b_1", "", "/usr/bin/kept", "kept"),
scan(kT0 + 61 * kSec, 9401, "process_9401_b_1", {"Teacher"})};
PROPERTY(positives_in(run(events, o)) == 1);
// Keep one negative of A while its positive reservoir grows and shrinks.
// Historic nonempty capacities must also shrink geometrically, otherwise
// many such retained entities can accumulate O(max_examples^2) storage.
o.max_examples = 128;
for (int i = 0; i < 532; ++i) {
const auto ts = kT0 + (i + 2) * 61 * kSec;
const auto node = "process_" + std::to_string(9500 + i) + "_b_1";
const auto entity = i < 20 ? "/usr/bin/kept" : "/usr/bin/other";
events.push_back(exec(ts, 9500 + i, node, "", entity, entity));
events.push_back(scan(ts, 9500 + i, node, {"Teacher"}));
}
const auto result = run(events, o);
PROPERTY(positives_in(result) == 25 && result.examples.size() == 26);
std::cout << "R15 allocated sampler capacity lifetime: PASSED" << std::endl;
}
void test_parent_audit_regressions() {
namespace fs = std::filesystem;
std::size_t failures = 0;
const auto verify = [&](bool ok, const char* reason) {
if (!ok) {
++failures;
std::cerr << "AUDIT REGRESSION FAILED: " << reason << std::endl;
}
};
ReplayOptions o = no_warmup();
o.collect_examples = true;
const std::string sha(64, 'a'), other(64, 'b');
std::string error;
auto dir = make_retract_store("audit_multi_hash", sha, "/usr/bin/ss", true);
// A path/rule entry can suppress several binaries. Removal must retire
// every binding, not just the last hash observed for that entry.
append_label(dir.string(), LabelPolarity::negative, "suppressed", sha, "path-entry", "allowlist", "");
append_label(dir.string(), LabelPolarity::negative, "suppressed", other, "path-entry", "allowlist", "");
append_label(dir.string(), LabelPolarity::retracted, "allowlist-remove", "", "path-entry", "operator", "");
verify(positives_in(replay_store(dir.string(), o, error)) == 1 && error.empty(), "remove all entry hash bindings");
fs::remove_all(dir);
dir = make_retract_store("audit_delayed_suppression", sha, "/usr/bin/ss", true);
append_label(dir.string(), LabelPolarity::negative, "allowlist-add", sha, "entry", "operator", "", kT0 + 100);
// Stored order, not wall time, controls policy judgments.
append_label(dir.string(), LabelPolarity::retracted, "allowlist-remove", "", "entry", "operator", "", kT0 + 50);
append_label(dir.string(), LabelPolarity::negative, "suppressed", sha, "entry", "allowlist", "");
verify(positives_in(replay_store(dir.string(), o, error)) == 1 && error.empty(), "late observation cannot resurrect removed entry");
// Path/rule adds carry no hash; they still explicitly start a new lifetime.
append_label(dir.string(), LabelPolarity::negative, "allowlist-add", "", "entry", "operator", "");
verify(positives_in(replay_store(dir.string(), o, error)) == 1 && error.empty(), "new lifetime has no old hash bindings");
append_label(dir.string(), LabelPolarity::negative, "suppressed", sha, "entry", "allowlist", "");
verify(positives_in(replay_store(dir.string(), o, error)) == 0 && error.empty(), "explicit re-add permits new observations");
fs::remove_all(dir);
dir = make_retract_store("audit_provenance", sha, "/tmp/x/dropper", false);
append_label(dir.string(), LabelPolarity::positive, "quarantine", sha, "", "operator", "");
append_label(dir.string(), LabelPolarity::retracted, "quarantine-remove", sha, "", "teacher", "");
verify(positives_in(replay_store(dir.string(), o, error)) == 1 && error.empty(), "unrelated source cannot retract operator judgment");
// Display strings are not identities: an independent reason with the
// same rendered text must survive an actual quarantine removal.
append_label(dir.string(), LabelPolarity::positive, "rules_update", sha, "", "teacher", "operator:quarantine");
append_label(dir.string(), LabelPolarity::retracted, "quarantine-remove", sha, "", "operator", "");
verify(positives_in(replay_store(dir.string(), o, error)) == 1 && error.empty(), "independent same-text positive survives");
fs::remove_all(dir);
dir = make_retract_store("audit_bad_label", sha, "/usr/bin/ss", true);
{
Database db;
PROPERTY(db.open(dir.string()));
PROPERTY(db.append_evidence(store::RecordType::label, std::string(1, static_cast<char>(0x80))));
}
(void) replay_store(dir.string(), o, error);
verify(!error.empty(), "malformed labels fail visibly instead of silently joining partial evidence");
fs::remove_all(dir);
// A capped entity must refresh its retained positives across time,
// rather than freezing the first cap examples forever.
std::vector<TelemetryEvent> events;
for (int i = 0; i < 100; ++i) {
const auto ts = kT0 + i * 61 * kSec;
const auto node = "process_" + std::to_string(9000 + i) + "_b_1";
events.push_back(exec(ts, 9000 + i, node, "", "/usr/bin/hot", "hot"));
events.push_back(scan(ts, 9000 + i, node, {"Teacher"}));
}
o.max_examples = 50;
o.max_entity_share = 0.1;
o.sample_seed = 1;
const auto capped = run(events, o);
verify(positives_in(capped) == 5, "achievable per-entity positive capacity is retained");
verify(std::any_of(capped.examples.begin(), capped.examples.end(), [](const auto& e) {
return e.start_ns >= kT0 + 5 * 61 * kSec;
}), "capped positives are sampled across time");
// With the positive quota full, an entity whose capacity is occupied
// by negatives must still have a chance to supply a teacher positive.
events.clear();
for (int i = 0; i < 4; ++i) {
events.push_back(exec(kT0 + i * 61 * kSec, 9100 + i,
"process_" + std::to_string(9100 + i) + "_b_1", "", "/usr/bin/mixed", "mixed"));
}
events.push_back(exec(kT0 + 300 * kSec, 9200, "process_9200_b_1", "", "/usr/bin/first", "first"));
events.push_back(scan(kT0 + 300 * kSec, 9200, "process_9200_b_1", {"Teacher"}));
events.push_back(exec(kT0 + 400 * kSec, 9201, "process_9201_b_1", "", "/usr/bin/mixed", "mixed"));
events.push_back(scan(kT0 + 400 * kSec, 9201, "process_9201_b_1", {"Teacher"}));
o.max_examples = 8;
o.max_entity_share = 0.5;
bool selected_mixed = false;
for (uint64_t seed = 1; seed <= 16; ++seed) {
o.sample_seed = seed;
const auto sample = run(events, o);
verify(positives_in(sample) == 1 && sample.examples.size() <= 8, "class quota survives cross-entity replacement");
std::unordered_map<std::string, std::size_t> counts;
for (const auto& e : sample.examples) {
++counts[e.entity];
selected_mixed |= e.label > 0.5f && e.entity == "/usr/bin/mixed";
}
for (const auto& [entity, count] : counts) verify(count <= 4, "combined cap survives cross-entity replacement");
}
verify(selected_mixed, "negative entity capacity cannot starve a positive at full global quota");
// Seeded mutation pressure exercises slot compaction, cross-class changes,
// entity eviction/re-entry and both full and partially occupied samples.
std::mt19937_64 rng(kSeed);
for (std::size_t trial = 0; trial < 48; ++trial) {
events.clear();
o.max_examples = 1 + trial % 24;
o.max_entity_share = trial % 3 == 0 ? 0.01 : trial % 3 == 1 ? 0.25 : 1.0;
o.sample_seed = kSeed + trial;
for (int i = 0; i < 72; ++i) {
const auto ts = kT0 + i * 61 * kSec;
const auto node = "process_" + std::to_string(9300 + i) + "_b_1";
const auto entity = std::string("/usr/bin/entity_") + static_cast<char>('a' + rng() % 7);
events.push_back(exec(ts, 9300 + i, node, "", entity, entity));
if (rng() % 3 == 0) events.push_back(scan(ts, 9300 + i, node, {"Teacher"}));
}
const auto sample = run(events, o);
const auto again = run(events, o);
const auto cap = std::max<std::size_t>(1, static_cast<std::size_t>(o.max_entity_share * o.max_examples));
verify(sample.examples.size() <= o.max_examples, "sample bound under mutation pressure");
verify(positives_in(sample) <= std::max<std::size_t>(1, o.max_examples / 5), "positive quota under mutation pressure");
verify(sample.example_positives == 0 || positives_in(sample) != 0, "positive cannot be lost to negatives");
PROPERTY(again.examples.size() == sample.examples.size());
std::unordered_map<std::string, std::size_t> counts;
for (std::size_t i = 0; i < sample.examples.size(); ++i) {
const auto& e = sample.examples[i];
verify(++counts[e.entity] <= cap, "combined entity cap under mutation pressure");
verify(e.start_ns == again.examples[i].start_ns && e.entity == again.examples[i].entity &&
e.label == again.examples[i].label, "same seed is deterministic");
}
}
PROPERTY(failures == 0);
std::cout << "R14 parent audit regressions: PASSED" << std::endl;
}
void no_partial_replay(const ReplayReport& r, const std::string& error, ReplayFailure failure) {
PROPERTY(!error.empty() && r.failure == failure);
PROPERTY(r.records == 0 && r.windows == 0 && r.examples.empty());
PROPERTY(r.top_windows.empty() && r.detected.empty() && !r.examples_collected && !r.snapshot.checked);
PROPERTY(replay_report_json(r).empty() && replay_training_summary_json(r).empty());
}
void append_test_tail(const std::filesystem::path& path) {
const int fd = io::open(path.c_str(), O_WRONLY | O_APPEND | O_CLOEXEC);
PROPERTY(fd >= 0);
const char byte = 'x';
PROPERTY(io::write(fd, &byte, 1) == 1);
PROPERTY(io::close(fd) == 0);
}
void test_replay_completeness() {
namespace fs = std::filesystem;
const std::string sha(64, 'a');
auto o = no_warmup();
o.collect_examples = true;
std::string error;
for (const auto stream : {store::Stream::evidence, store::Stream::telemetry}) {
const auto dir = make_retract_store("completeness_" + std::to_string(static_cast<int>(stream)), sha, "/tmp/private/a", true);
append_label(dir.string(), LabelPolarity::positive, "quarantine", sha, "", "operator", "private-rule");
const auto load = store::load_manifest(dir.string());
const auto it = std::find_if(load.manifest.segments.begin(), load.manifest.segments.end(),
[&](const auto& seg) { return seg.stream == stream; });
PROPERTY(it != load.manifest.segments.end());
const auto path = dir / (stream == store::Stream::evidence ? "evidence" : "telemetry") / it->file;
fs::rename(path, path.string() + ".saved");
const auto scan = store::scan_store_read_only(dir.string(), stream, [](uint32_t, std::string_view) {});
PROPERTY(scan.exists && scan.listed_segments != 0 && scan.skipped_segments == 1);
PROPERTY(scan.report.io_error.empty()); // ordinary readers still skip vanished segments
const auto missing = replay_store(dir.string(), o, error);
no_partial_replay(missing, error, ReplayFailure::unavailable);
fs::remove_all(dir);
}
const auto dir = make_retract_store("sealed_completeness", sha, "/tmp/private/a", true);
{
store::StoreOptions so;
so.dir = dir.string();
store::StoreOpenReport opened;
auto log = store::LogStore::open(so, opened, error);
PROPERTY(log && log->roll(store::Stream::telemetry));
const auto event = exec(kT0 + 1000 * kSec, 9900, "process_9900_b_1", "", "/usr/bin/new", "new");
PROPERTY(log->append(store::Stream::telemetry, event.type, event.payload));
PROPERTY(log->flush(store::Stream::telemetry, true));
}
auto load = store::load_manifest(dir.string());
auto sealed = std::find_if(load.manifest.segments.begin(), load.manifest.segments.end(),
[](const auto& seg) { return seg.stream == store::Stream::telemetry && seg.sealed; });
PROPERTY(sealed != load.manifest.segments.end());
const auto path = dir / "telemetry" / sealed->file;
// Count mismatches and absent sealed footers invalidate otherwise readable
// records. An ordinary active tail is tested separately below.
++sealed->records;
++load.manifest.generation;
PROPERTY(store::publish_manifest(dir.string(), load.manifest, error));
no_partial_replay(replay_store(dir.string(), o, error), error, ReplayFailure::invalid);
--sealed->records;
++load.manifest.generation;
PROPERTY(store::publish_manifest(dir.string(), load.manifest, error));
fs::resize_file(path, fs::file_size(path) - 1);
no_partial_replay(replay_store(dir.string(), o, error), error, ReplayFailure::invalid);
// Best-effort readers continue delivering a valid later segment even when
// the earlier manifest-listed segment cannot be opened.
fs::rename(path, path.string() + ".saved");
uint64_t delivered = 0;
const auto partial = store::scan_store_read_only(dir.string(), store::Stream::telemetry,
[&](uint32_t, std::string_view) { ++delivered; });
PROPERTY(delivered != 0 && partial.skipped_segments == 1 && partial.report.io_error.empty());
no_partial_replay(replay_store(dir.string(), o, error), error, ReplayFailure::unavailable);
fs::remove_all(dir);
std::cout << "R16 incomplete inputs export no partial replay: PASSED" << std::endl;
}
void test_snapshot_summary() {
namespace fs = std::filesystem;
const std::string sha(64, 'b');
std::string error;
// Fixed-bucket observations must not alter resolution, seeded sampling or
// leak host-controlled strings. Include zero/extreme clocks and overlap.
auto dir = make_retract_store("summary_labels", sha, "/tmp/host-secret/program", true);
append_label(dir.string(), LabelPolarity::positive, "quarantine", sha, "", "operator", "host-secret-rule");
append_label(dir.string(), LabelPolarity::negative, "host-secret-action", sha, "", "host-secret-source", "");
append_label(dir.string(), LabelPolarity::negative, "allowlist-add", "", "", "operator", "", 0);
append_label(dir.string(), LabelPolarity::retracted, "allowlist-remove", "", "", "host-secret-source", "", INT64_MIN);
append_label(dir.string(), LabelPolarity::positive, "host-secret-action", sha, "", "host-secret-source", "", INT64_MAX);
auto o = no_warmup();
o.collect_examples = true;
o.sample_seed = kSeed;
const auto ordinary = replay_store(dir.string(), o, error);
PROPERTY(error.empty());
o.training_summary = true;
const auto summary = replay_store(dir.string(), o, error);
PROPERTY(error.empty() && summary.snapshot.checked && summary.examples_collected);
PROPERTY(summary.examples.size() == ordinary.examples.size());
for (std::size_t i = 0; i < summary.examples.size(); ++i) {
PROPERTY(summary.examples[i].x == ordinary.examples[i].x && summary.examples[i].label == ordinary.examples[i].label);
PROPERTY(summary.examples[i].start_ns == ordinary.examples[i].start_ns && summary.examples[i].entity == ordinary.examples[i].entity);
}
PROPERTY(summary.labels.records == 5 && summary.labels.missing_hash == 2 && summary.labels.idless_allowlist == 2);
PROPERTY((summary.labels.polarity == std::array<uint64_t, 3>{2, 2, 1}));
PROPERTY((summary.labels.provenance == std::array<uint64_t, 3>{2, 0, 3}));
PROPERTY(summary.labels.timestamp_known == 3 && summary.labels.timestamp_unknown == 2);
PROPERTY(summary.labels.last_ns == INT64_MAX && summary.labels.resolved_positive_hashes == 1);
PROPERTY(summary.labels.resolved_negative_hashes == 1 && summary.labels.overlap_hashes == 1);
const auto json = replay_training_summary_json(summary);
PROPERTY(json.find("host-secret") == std::string::npos && json.find(sha) == std::string::npos);
PROPERTY(json == replay_training_summary_json(replay_store(dir.string(), o, error)) && error.empty());
fs::remove_all(dir);
for (const bool change_manifest : {false, true}) {
dir = make_retract_store(change_manifest ? "snapshot_generation" : "snapshot_extent", sha, "/tmp/private/a", false);
auto load = store::load_manifest(dir.string());
const auto telemetry = std::find_if(load.manifest.segments.begin(), load.manifest.segments.end(),
[](const auto& seg) { return seg.stream == store::Stream::telemetry; });
PROPERTY(telemetry != load.manifest.segments.end());
const auto path = dir / "telemetry" / telemetry->file;
bool changed = false;
// A trusted in-process observation seam, never exposed by the CLI,
// gives deterministic extent/generation changes without a racing writer.
o.before_snapshot_recheck = [&] {
if (changed) return;
changed = true;
if (change_manifest) {
++load.manifest.generation;
PROPERTY(store::publish_manifest(dir.string(), load.manifest, error));
} else append_test_tail(path);
};
const auto changed_report = replay_store(dir.string(), o, error);
PROPERTY(changed);
no_partial_replay(changed_report, error, ReplayFailure::changed);
fs::remove_all(dir);
}
o.before_snapshot_recheck = [] { throw 42; };
dir = make_retract_store("snapshot_observer_fault", sha, "/tmp/private/a", false);
no_partial_replay(replay_store(dir.string(), o, error), error, ReplayFailure::unavailable);
fs::remove_all(dir);
o.before_snapshot_recheck = {};
dir = make_retract_store("active_tail_summary", sha, "/tmp/private/a", false);
const auto load = store::load_manifest(dir.string());
const auto telemetry = std::find_if(load.manifest.segments.begin(), load.manifest.segments.end(),
[](const auto& seg) { return seg.stream == store::Stream::telemetry; });
PROPERTY(telemetry != load.manifest.segments.end());
append_test_tail(dir / "telemetry" / telemetry->file);
o.training_summary = false;
const auto active = replay_store(dir.string(), o, error);
PROPERTY(error.empty() && active.records != 0);
o.training_summary = true;
no_partial_replay(replay_store(dir.string(), o, error), error, ReplayFailure::unavailable);
fs::remove_all(dir);
dir = make_retract_store("fragment_summary", sha, "/tmp/private/a", false);
for (const char* name : {"MANIFEST", "MANIFEST.prev", "MANIFEST.tmp"}) {
const auto path = dir / name;
if (fs::exists(path)) fs::rename(path, path.string() + ".saved");
}
o.training_summary = false;
const auto fragment = replay_store(dir.string(), o, error);
PROPERTY(error.empty() && fragment.records != 0);
o.training_summary = true;
no_partial_replay(replay_store(dir.string(), o, error), error, ReplayFailure::unavailable);
fs::remove_all(dir);
std::cout << "R17 aggregate labels and strict cooperative snapshots: PASSED" << std::endl;
}
std::string read_whole_file(const std::filesystem::path& path) {
const int fd = io::open(path.c_str(), O_RDONLY | O_CLOEXEC);
PROPERTY(fd >= 0);
struct stat st {};
PROPERTY(io::fstat(fd, &st) == 0);
std::string out(static_cast<std::size_t>(st.st_size), '\0');
std::size_t got = 0;
while (got < out.size()) {
const ssize_t n = io::read(fd, out.data() + got, out.size() - got);
PROPERTY(n > 0);
got += static_cast<std::size_t>(n);
}
PROPERTY(io::close(fd) == 0);
return out;
}
void write_whole_file(const std::filesystem::path& path, std::string_view bytes) {
const int fd = io::open(path.c_str(), O_WRONLY | O_TRUNC | O_CLOEXEC);
PROPERTY(fd >= 0);
std::size_t done = 0;
while (done < bytes.size()) {
const ssize_t n = io::write(fd, bytes.data() + done, bytes.size() - done);
PROPERTY(n > 0);
done += static_cast<std::size_t>(n);
}
PROPERTY(io::close(fd) == 0);
}
uint32_t block_le32(std::string_view bytes, std::size_t at) {
uint32_t value = 0;
for (int i = 0; i < 4; ++i)
value |= static_cast<uint32_t>(static_cast<unsigned char>(bytes[at + i])) << (8 * i);
return value;
}
void block_put_le32(std::string& bytes, std::size_t at, uint32_t value) {
for (int i = 0; i < 4; ++i) bytes[at + i] = static_cast<char>((value >> (8 * i)) & 0xFFu);
}
// The first block starts right after the segment header.
constexpr std::size_t kFirstBlock = store::kSegmentHeaderBytes;
/// Makes the first block of `file` need a dictionary this build does not have
/// (the header stays self-consistent, so it is not corruption).
void require_missing_dictionary(std::string& file) {
PROPERTY(file.size() > kFirstBlock + store::kBlockHeaderBytes);
PROPERTY(block_le32(file, kFirstBlock) == store::kBlockMagic);
PROPERTY(file[kFirstBlock + 1 + 4] == static_cast<char>(store::BlockCodec::zstd));
block_put_le32(file, kFirstBlock + 16, 0x5EEDu);
block_put_le32(file, kFirstBlock + 24, store::crc32c(file.data() + kFirstBlock, 24));
}
/// Corrupts the magic of the first block: a complete header that is not valid.
void break_block_header(std::string& file) {
PROPERTY(file.size() > kFirstBlock + store::kBlockHeaderBytes);
file[kFirstBlock] = static_cast<char>(file[kFirstBlock] ^ 0x80);
}
std::filesystem::path stream_segment(const std::string& dir, const std::string& name, bool sealed) {
const auto load = store::load_manifest(dir);
const auto stream = name == "evidence" ? store::Stream::evidence : store::Stream::telemetry;
for (const auto& seg : load.manifest.segments) {
if (seg.stream == stream && seg.sealed == sealed) return std::filesystem::path(dir) / name / seg.file;
}
return {};
}
void test_decoder_availability() {
namespace fs = std::filesystem;
const std::string sha(64, 'd');
auto o = no_warmup();
o.collect_examples = true;
std::string error;
// A block needing a dictionary this build lacks is a missing decoder, not
// an empty stream and not corruption (ordinary replay and summary alike).
{
const auto dir = make_retract_store("decoder_active", sha, "/tmp/private/a", true);
const auto path = stream_segment(dir.string(), "telemetry", false);
PROPERTY(!path.empty());
auto file = read_whole_file(path);
require_missing_dictionary(file);
write_whole_file(path, file);
const auto scan = store::scan_store_read_only(dir.string(), store::Stream::telemetry,
[](uint32_t, std::string_view) {});
PROPERTY(scan.unavailable_segments == 1 && scan.malformed_tail_segments == 0);
PROPERTY(scan.invalid_segments == 0 && scan.report.torn_tail == false);
no_partial_replay(replay_store(dir.string(), o, error), error, ReplayFailure::unavailable);
o.training_summary = true;
no_partial_replay(replay_store(dir.string(), o, error), error, ReplayFailure::unavailable);
o.training_summary = false;
fs::remove_all(dir);
}
// The same block in a sealed segment must not be misread as bad data: the
// footer/manifest counts it explains are not comparable without the block.
{
const auto dir = make_retract_store("decoder_sealed", sha, "/tmp/private/a", true);
{
store::StoreOptions so;
so.dir = dir.string();
store::StoreOpenReport opened;
auto log = store::LogStore::open(so, opened, error);
PROPERTY(log && log->roll(store::Stream::telemetry));
const auto event = exec(kT0 + 1000 * kSec, 9900, "process_9900_b_1", "", "/usr/bin/new", "new");
PROPERTY(log->append(store::Stream::telemetry, event.type, event.payload));
PROPERTY(log->flush(store::Stream::telemetry, true));
}
const auto path = stream_segment(dir.string(), "telemetry", true);
PROPERTY(!path.empty());
auto file = read_whole_file(path);
require_missing_dictionary(file);
write_whole_file(path, file);
const auto scan = store::scan_store_read_only(dir.string(), store::Stream::telemetry,
[](uint32_t, std::string_view) {});
PROPERTY(scan.unavailable_segments == 1 && scan.malformed_tail_segments == 0);
PROPERTY(scan.invalid_segments == 0); // counts are not checkable here
no_partial_replay(replay_store(dir.string(), o, error), error, ReplayFailure::unavailable);
fs::remove_all(dir);
}
// A complete block header that fails validation is corrupt data: unlike a
// short in-flight tail it is not tolerated by ordinary replay.
{
const auto dir = make_retract_store("decoder_badheader", sha, "/tmp/private/a", true);
const auto path = stream_segment(dir.string(), "telemetry", false);
PROPERTY(!path.empty());
auto file = read_whole_file(path);
break_block_header(file);
write_whole_file(path, file);
const auto scan = store::scan_store_read_only(dir.string(), store::Stream::telemetry,
[](uint32_t, std::string_view) {});
PROPERTY(scan.malformed_tail_segments == 1 && scan.unavailable_segments == 0);
no_partial_replay(replay_store(dir.string(), o, error), error, ReplayFailure::invalid);
o.training_summary = true;
no_partial_replay(replay_store(dir.string(), o, error), error, ReplayFailure::invalid);
o.training_summary = false;
fs::remove_all(dir);
}
// An earlier unusable segment must not be masked by a later short tail
// (the aggregate tail_error keeps only the last one).
for (const bool missing_dictionary : {false, true}) {
const auto dir = make_retract_store(missing_dictionary ? "masked_decoder" : "masked_header", sha,
"/tmp/private/a", true);
{
store::StoreOptions so;
so.dir = dir.string();
store::StoreOpenReport opened;
auto log = store::LogStore::open(so, opened, error);
PROPERTY(log && log->roll(store::Stream::telemetry));
const auto event = exec(kT0 + 1000 * kSec, 9900, "process_9900_b_1", "", "/usr/bin/new", "new");
PROPERTY(log->append(store::Stream::telemetry, event.type, event.payload));
PROPERTY(log->flush(store::Stream::telemetry, true));
}
const auto sealed = stream_segment(dir.string(), "telemetry", true);
const auto active = stream_segment(dir.string(), "telemetry", false);
PROPERTY(!sealed.empty() && !active.empty());
auto file = read_whole_file(sealed);
if (missing_dictionary) require_missing_dictionary(file);
else break_block_header(file);
write_whole_file(sealed, file);
append_test_tail(active); // the later, tolerated short tail
const auto scan = store::scan_store_read_only(dir.string(), store::Stream::telemetry,
[](uint32_t, std::string_view) {});
PROPERTY(scan.report.tail_error == store::BlockError::truncated);
if (missing_dictionary) {
PROPERTY(scan.unavailable_segments == 1 && scan.malformed_tail_segments == 0);
no_partial_replay(replay_store(dir.string(), o, error), error, ReplayFailure::unavailable);
} else {
no_partial_replay(replay_store(dir.string(), o, error), error, ReplayFailure::invalid);
}
fs::remove_all(dir);
}
// Positive control: a genuine in-flight tail stays readable to ordinary
// replay (and is still refused for a summary snapshot).
{
const auto dir = make_retract_store("decoder_inflight", sha, "/tmp/private/a", true);
const auto path = stream_segment(dir.string(), "telemetry", false);
PROPERTY(!path.empty());
append_test_tail(path);
const auto scan = store::scan_store_read_only(dir.string(), store::Stream::telemetry,
[](uint32_t, std::string_view) {});
PROPERTY(scan.malformed_tail_segments == 0 && scan.unavailable_segments == 0);
PROPERTY(scan.report.torn_tail && scan.report.tail_error == store::BlockError::truncated);
const auto ordinary = replay_store(dir.string(), o, error);
PROPERTY(error.empty() && ordinary.records != 0 && ordinary.failure == ReplayFailure::none);
o.training_summary = true;
no_partial_replay(replay_store(dir.string(), o, error), error, ReplayFailure::unavailable);
o.training_summary = false;
fs::remove_all(dir);
}
std::cout << "R18 missing decoder and malformed tails are not empty inputs: PASSED" << std::endl;
}
// Stage 7a: LogGC must not change what replay detects. Replays a full store and the same store after
// reduce_telemetry(): dead benign subtrees are dropped (fewer execs), detections are identical.
void test_reduce_replay_equivalence() {
namespace fs = std::filesystem;
constexpr int64_t kDayNs = 86'400 * kSec;
auto ev = routine(20, kT0);
ev.push_back(exec(kT0 + 40 * 61 * kSec, 700, "process_700_b_1", "process_1_b_1", "/tmp/evil", "evil"));
ev.push_back(scan(kT0 + 40 * 61 * kSec + kSec, 700, "process_700_b_1", {"Evil_Rule"}));
for (int i = 0; i < 3; ++i) { // live (distinct nodes, days later): sets the horizon
const auto ts = kT0 + 5 * kDayNs + i * 61 * kSec;
const std::string sh = "process_" + std::to_string(5000 + i) + "_b_1";
ev.push_back(exec(ts, 5000 + i, sh, "process_1_b_1", "/usr/bin/zsh", "zsh"));
ev.push_back(exec(ts + kSec, 6000 + i, "process_" + std::to_string(6000 + i) + "_b_1", sh, "/usr/bin/cat", "cat x"));
}
const auto root = fs::temp_directory_path() / ("strawberry_reduce_replay_" + std::to_string(::getpid()));
std::error_code ec;
fs::remove_all(root, ec);
fs::create_directories(root, ec);
const std::string full_dir = (root / "full").string();
const std::string reduced_dir = (root / "reduced").string();
const std::string skeleton_dir = (root / "skeleton").string();
const std::string merge_dir = (root / "merge").string();
const std::string cpr_dir = (root / "cpr").string();
for (const auto& dir : {full_dir, reduced_dir, skeleton_dir, merge_dir, cpr_dir}) {
Database db;
PROPERTY(db.open(dir));
auto copy = ev;
PROPERTY(db.append_telemetry(copy, true));
db.close();
}
store::ReduceReport reduce;
{
store::StoreOpenReport open_report;
std::string error;
auto log = store::LogStore::open(store::default_store_options(reduced_dir), open_report, error);
PROPERTY(log != nullptr);
PROPERTY(log->roll(store::Stream::telemetry)); // seal: only sealed telemetry is reduced
PROPERTY(log->reduce_telemetry(1, reduce));
}
PROPERTY(reduce.problems.empty() && reduce.subtrees_dropped > 0 && reduce.records_after < reduce.records_before);
store::ReduceReport skeleton; // Stage 7b: also skeletonize dead benign nodes (exec kept, leaf telemetry dropped)
{
store::StoreOpenReport open_report;
std::string error;
auto log = store::LogStore::open(store::default_store_options(skeleton_dir), open_report, error);
PROPERTY(log != nullptr);
PROPERTY(log->roll(store::Stream::telemetry));
store::ReduceOptions skeleton_options;
skeleton_options.seed = 1;
skeleton_options.skeletonize_dead_benign = true;
PROPERTY(log->reduce_telemetry(skeleton_options, skeleton));
}
PROPERTY(skeleton.problems.empty() && skeleton.records_after <= reduce.records_after);
store::ReduceReport merged; // Stage 7c: also merge old repeated benign leaf accesses (7a path otherwise)
{
store::StoreOpenReport open_report;
std::string merge_error;
auto log = store::LogStore::open(store::default_store_options(merge_dir), open_report, merge_error);
PROPERTY(log != nullptr);
PROPERTY(log->roll(store::Stream::telemetry));
store::ReduceOptions merge_options;
merge_options.seed = 1;
merge_options.merge_redundant = true;
PROPERTY(log->reduce_telemetry(merge_options, merged));
}
PROPERTY(merged.problems.empty() && merged.records_after <= reduce.records_after);
store::ReduceReport cpr_merged; // Stage 7d: CPR merge with a 1 h horizon
{
store::StoreOpenReport open_report;
std::string cpr_error;
auto log = store::LogStore::open(store::default_store_options(cpr_dir), open_report, cpr_error);
PROPERTY(log != nullptr);
PROPERTY(log->roll(store::Stream::telemetry));
store::ReduceOptions cpr_options;
cpr_options.seed = 1;
cpr_options.merge_redundant = true;
cpr_options.merge_mode = store::MergeMode::cpr;
cpr_options.merge_horizon_ns = 3'600ll * 1'000'000'000ll;
PROPERTY(log->reduce_telemetry(cpr_options, cpr_merged));
}
PROPERTY(cpr_merged.problems.empty() && cpr_merged.records_after <= reduce.records_after);
std::string error;
const auto full = replay_store(full_dir, no_warmup(), error);
PROPERTY(error.empty());
const auto reduced = replay_store(reduced_dir, no_warmup(), error);
PROPERTY(error.empty());
PROPERTY(reduced.execs < full.execs); // the compute proxy went down
PROPERTY(reduced.detected_windows == full.detected_windows && full.detected_windows == 1);
PROPERTY(reduced.late_detected_windows == full.late_detected_windows);
PROPERTY(reduced.detected.size() == full.detected.size());
for (std::size_t i = 0; i < full.detected.size(); ++i) {
PROPERTY(reduced.detected[i].root_node == full.detected[i].root_node);
}
const auto skeletonized = replay_store(skeleton_dir, no_warmup(), error);
PROPERTY(error.empty());
PROPERTY(skeletonized.execs == reduced.execs); // skeletonization never drops an exec
PROPERTY(skeletonized.detected_windows == full.detected_windows);
PROPERTY(skeletonized.late_detected_windows == full.late_detected_windows);
PROPERTY(skeletonized.detected.size() == full.detected.size());
for (std::size_t i = 0; i < full.detected.size(); ++i) {
PROPERTY(skeletonized.detected[i].root_node == full.detected[i].root_node);
}
// 7c recall gate: merge touches only old benign far-from-label repeats, so detections are unchanged.
const auto merged_replay = replay_store(merge_dir, no_warmup(), error);
PROPERTY(error.empty());
PROPERTY(merged_replay.execs == reduced.execs); // merge never drops an exec
PROPERTY(merged_replay.detected_windows == full.detected_windows);
PROPERTY(merged_replay.late_detected_windows == full.late_detected_windows);
PROPERTY(merged_replay.detected.size() == full.detected.size());
for (std::size_t i = 0; i < full.detected.size(); ++i) {
PROPERTY(merged_replay.detected[i].root_node == full.detected[i].root_node);
}
const auto cpr_replay = replay_store(cpr_dir, no_warmup(), error);
PROPERTY(error.empty());
PROPERTY(cpr_replay.execs == reduced.execs);
PROPERTY(cpr_replay.detected_windows == full.detected_windows);
PROPERTY(cpr_replay.late_detected_windows == full.late_detected_windows);
PROPERTY(cpr_replay.detected.size() == full.detected.size());
for (std::size_t i = 0; i < full.detected.size(); ++i) {
PROPERTY(cpr_replay.detected[i].root_node == full.detected[i].root_node);
}
fs::remove_all(root, ec);
std::cout << "R-7a LogGC replay equivalence (+7b skeleton, +7c merge, +7d cpr): PASSED" << std::endl;
}
// R11c evaluation bound (--eval-after): a bounded replay is out-of-sample and
// can pass the gate; the same run unbounded is in-sample and refused; a bound
// past the end of the history fails explicitly instead of emitting a silent
// 0-window cell; `auto` derives the bound from the models' training range.
// Without the bound these assertions cannot hold: the bounded cells would be
// in-sample (their windows overlap the training range) and the gate refuses.
void test_eval_bound() {
auto ev = routine(240, kT0);
std::vector<Trace> traces;
std::string error;
PROPERTY(compile_traces(
"trace odd\n+0s exec a /tmp/q/odd \"odd --x\"\n+1s connect a 198.51.100.4 4444\n", traces, error));
namespace fs = std::filesystem;
const auto dir = fs::temp_directory_path() / ("strawberry_eval_bound_" + std::to_string(::getpid()));
std::error_code ec;
fs::remove_all(dir, ec);
{
Database db;
PROPERTY(db.open(dir.string()));
auto copy = ev;
PROPERTY(db.append_telemetry(copy, true));
db.close();
}
// Training ranges are [first window start, last window END].
auto whole = std::make_shared<Model>(Model::initialized(ModelShape{}, 11));
whole->training_first_ns = kT0;
whole->training_last_ns = kT0 + 240 * 61 * kSec; // covers the whole store
auto earlier = std::make_shared<Model>(Model::initialized(ModelShape{}, 12));
earlier->training_first_ns = kT0;
earlier->training_last_ns = kT0 + 120 * 61 * kSec; // the first half
const auto healthy = [](const PolicyEvaluation::Provenance& p) {
PolicyEvaluation e;
e.extra_positives = 12;
e.policies = {{"model", 10, 20.0, 9, 9.0, 0}, {"rarity_greedy", 20, 40.0, 8, 8.0, 0}};
e.provenance = p;
return promotion_gate(e);
};
ReplayOptions o;
o.warmup_ns = 600 * kSec;
o.policy_eval = true;
o.policy.budget_seconds_per_hour = 4;
o.inject = traces;
o.inject_copies = 9;
o.eval_models = {{"whole", whole}};
o.eval_periods = 2;
// Unbounded: the ranked windows are inside the training range, every cell is
// in-sample, and the gate refuses it (today's "always refuses" behaviour).
const auto unbounded = replay_store(dir.string(), o, error);
PROPERTY(error.empty());
PROPERTY(unbounded.policy_eval.cells.size() == 2);
for (const auto& c : unbounded.policy_eval.cells) {
PROPERTY(c.provenance.training_range_known && c.provenance.in_sample);
PROPERTY(!healthy(c.provenance).passed);
PROPERTY(healthy(c.provenance).checks[4].find("in-sample evaluation") != std::string::npos);
}
// Bounded at the model's training end the same history is out-of-sample, so
// the identical evidence can pass the pre-registered gate.
o.eval_models = {{"earlier", earlier}};
o.eval_after_ns = earlier->training_last_ns;
const auto bounded = replay_store(dir.string(), o, error);
PROPERTY(error.empty() && bounded.policy_eval.cells.size() == 2);
uint64_t windows = 0, positives = 0;
for (const auto& c : bounded.policy_eval.cells) {
windows += c.windows;
positives += c.extra_positives;
PROPERTY(c.provenance.training_range_known && !c.provenance.in_sample);
PROPERTY(c.provenance.eval_first_ns >= o.eval_after_ns);
PROPERTY(healthy(c.provenance).passed);
}
// Periods partition only the out-of-sample span: the routine windows precede
// the bound, so what is left is the injected evaluation traces.
PROPERTY(windows == bounded.policy_eval.windows && windows > 0 && positives > 0);
// The single-model policy_eval provenance uses the same bound, so `model
// promote` is judged out-of-sample instead of always refusing.
ReplayOptions single;
single.warmup_ns = 600 * kSec;
single.policy_eval = true;
single.policy.budget_seconds_per_hour = 4;
single.inject = traces;
single.inject_copies = 9;
single.model = earlier;
single.model_name = "earlier";
single.eval_after_ns = earlier->training_last_ns;
const auto promoted = replay_store(dir.string(), single, error);
PROPERTY(error.empty());
PROPERTY(promoted.policy_eval.provenance.training_range_known);
PROPERTY(!promoted.policy_eval.provenance.in_sample);
PROPERTY(promoted.policy_eval.provenance.eval_first_ns >= single.eval_after_ns);
single.eval_after_ns = 0; // unbounded: in-sample again
const auto refused = replay_store(dir.string(), single, error);
PROPERTY(error.empty() && refused.policy_eval.provenance.in_sample);
// A bound past the end of the history: the run says why it has no evidence.
ReplayOptions late = o;
late.eval_after_ns = kT0 + 500 * 61 * kSec;
replay_store(dir.string(), late, error);
PROPERTY(!error.empty());
PROPERTY(error.find("no out-of-sample history yet") != std::string::npos);
PROPERTY(error.find("store ends") != std::string::npos);
// Without injected traces the report still carries an explicit cell failure.
late.inject.clear();
const auto empty = replay_store(dir.string(), late, error);
PROPERTY(error.empty());
PROPERTY(empty.policy_eval.cells.size() == 2);
for (const auto& c : empty.policy_eval.cells) {
PROPERTY(c.windows == 0 && c.extra_positives == 0);
PROPERTY(c.provenance.eval_error.find("no out-of-sample history yet") != std::string::npos);
const auto g = healthy(c.provenance);
PROPERTY(!g.passed && g.checks[4].find("no out-of-sample history yet") != std::string::npos);
}
// `auto` = the latest training-range end over the loaded eval models.
PROPERTY(eval_after_from_models({}) == 0);
PROPERTY(eval_after_from_models({{"earlier", earlier}}) == earlier->training_last_ns);
PROPERTY(eval_after_from_models({{"earlier", earlier}, {"none", nullptr}}) == earlier->training_last_ns);
PROPERTY(eval_after_from_models({{"earlier", earlier}, {"whole", whole}}) == whole->training_last_ns);
auto unknown = std::make_shared<Model>(Model::initialized(ModelShape{}, 13));
PROPERTY(eval_after_from_models({{"unknown", unknown}}) == 0);
// The derived bound is what the CLI hands to replay: still out-of-sample.
o.eval_after_ns = eval_after_from_models(o.eval_models);
PROPERTY(o.eval_after_ns == earlier->training_last_ns);
const auto auto_bounded = replay_store(dir.string(), o, error);
PROPERTY(error.empty() && !auto_bounded.policy_eval.cells.empty());
for (const auto& c : auto_bounded.policy_eval.cells) PROPERTY(!c.provenance.in_sample);
// The cut cells carry the reason in the report JSON too (not just in memory),
// so a pooled fleet gate can see why a node contributed no out-of-sample run.
const auto empty_json = replay_report_json(empty);
PROPERTY(empty_json.find("no out-of-sample history yet") != std::string::npos);
const auto json = replay_report_json(auto_bounded);
PROPERTY(json.find("\"eval_error\": \"\"") != std::string::npos);
fs::remove_all(dir, ec);
std::cout << "R11c evaluation bound: PASSED (" << windows << " out-of-sample windows, bound "
<< (o.eval_after_ns - kT0) / kSec << " s after the store start)" << std::endl;
}
int main() {
std::cout << "replay properties, seed " << kSeed << std::endl;
test_reduce_replay_equivalence();
test_sampler_capacity_lifetime();
test_parent_audit_regressions();
test_replay_completeness();
test_snapshot_summary();
test_decoder_availability();
test_chain_and_pid_reuse();
test_order_and_determinism();
test_ranking_and_detection();
test_store_round_trip();
test_invalid_utf8();
test_warmup_and_time_invariance();
test_injection_evaluation();
test_late_labels();
test_attribution();
test_allowlisted_window_is_negative();
test_retraction_allowlist_remove();
test_retraction_quarantine_remove();
test_retraction_independent_positives();
test_retraction_multiple_entries();
test_retraction_readd();
test_examples_and_model_ranking();
test_positive_sampling_under_pressure();
test_positive_entity_cap();
test_positive_not_starved_by_entity_negatives();
test_policy_evaluation();
test_gate_cells();
test_eval_bound();
test_extreme_timestamps();
return 0;
}