// Replay: windows, joins and scoring over telemetry records. A failure // prints the seed; re-run with STRAWBERRY_PROPERTY_SEED=. // // R1 deterministic: the same records give the same report // R2 order within the reorder horizon does not matter // R3 a process chain within 60 s is one window; its behavior joins it // C3 PID reuse: behavior joins the exec of that pid at its time // R4 an unusual chain ranks above routine ones; detections mark windows // R5 replay_store reads a real store directory // R7 warm-up windows are learned from but not ranked; a never-seen event // scores the same early and late; identical patterns are grouped // R8 injected traces: every copy is found and scored; injecting changes // nothing about real windows (scored, never learned from); a novel // attack chain beats routine activity at a 10/day budget // R9 late labels (a binary found bad after it ran) mark the windows that // ran it, read from the evidence stream by replay_store // R10 training examples: one per ranked window, labelled by detections; // training traces become positives (not evaluated); per-program cap; // ranking by a model sets each window's score from its value // R11 policy evaluation: extra positives exclude rule-flagged windows; // rarity-greedy inspects within budget; logged shadow decisions are // joined to their windows; the promotion gate checks each criterion // R6 paths that are not UTF-8 (any byte but '/' and NUL is legal) give // valid JSON, in the report and in json_escape // R12 label retraction: allowlist-remove revokes the named entry's negative // (detected windows revert to rule-detected positives); quarantine-remove // revokes only the quarantine positive, preserving independent positives; // multiple entries and re-adds resolve in stored order // R13 positive entity cap: max_entity_share caps the combined (positive + // negative) per-program count; a late positive displaces a negative of // the same entity rather than being starved; concentration rejections // are counted, not silently dropped #include "strawberry/database.hpp" #include "strawberry/learn/replay.hpp" #include "strawberry/learn/model_file.hpp" #include "strawberry/learn/inspection.hpp" #include "strawberry/learn/trace.hpp" #include "strawberry/telemetry.hpp" #include "strawberry/store/log_store.hpp" #include "strawberry/sys_io.hpp" #include "../src/json_util.hpp" #include #include #include #include #include #include #include #include #include #include #include #include #include using namespace strawberry; using namespace strawberry::learn; namespace { uint64_t seed_from_env() { if (const char* text = std::getenv("STRAWBERRY_PROPERTY_SEED")) return std::strtoull(text, nullptr, 10); return std::random_device{}(); } const uint64_t kSeed = seed_from_env(); #define PROPERTY(cond) \ do { \ if (!(cond)) { \ std::cerr << "PROPERTY FAILED: " #cond " (seed " << kSeed << ", " << __FILE__ << ":" \ << __LINE__ << ")" << std::endl; \ std::abort(); \ } \ } while (0) constexpr int64_t kSec = 1'000'000'000; constexpr int64_t kT0 = 1'800'000'000 * kSec; TelemetryEvent exec(int64_t ts, int pid, const std::string& node, const std::string& parent, const std::string& exe, const std::string& cmdline) { ProcessExecRecord r; r.timestamp_ns = ts; r.pid = pid; r.process_node_id = node; r.parent_node_id = parent; r.exe = exe; r.cmdline = cmdline; r.source = "live"; return {store::RecordType::process_exec, encode_process_exec(r)}; } TelemetryEvent connect(int64_t ts, int pid, const std::string& address, int port) { BehaviorEvent b; b.type = BehaviorEvent::Type::connect; b.timestamp_ns = static_cast(ts); b.pid = pid; b.family = 2; b.address = address; b.port = static_cast(port); return {store::RecordType::connect, encode_behavior(b)}; } TelemetryEvent scan(int64_t ts, int pid, const std::string& node, std::vector rules) { ScanResultRecord s; s.timestamp_ns = ts; s.pid = pid; s.process_node_id = node; s.rules = std::move(rules); return {store::RecordType::scan_result, encode_scan_result(s)}; } TelemetryEvent write(int64_t ts, int pid, const std::string& path) { FileWriteEvent w; w.pid = pid; w.path = path; w.category = "drop"; w.timestamp_ns = static_cast(ts); return {store::RecordType::file_write, encode_file_write(w)}; } ReplayOptions no_warmup() { ReplayOptions o; o.warmup_ns = 0; return o; } ReplayReport run(const std::vector& events, ReplayOptions options = no_warmup()) { Replayer r(options); for (const auto& e : events) r.feed(e.type, e.payload); return r.finish(); } const ReplayWindow* window_rooted(const ReplayReport& report, const std::string& node) { for (const auto& w : report.top_windows) { if (w.root_node == node) return &w; } return nullptr; } // A routine background: many bash -> ls windows, a minute apart. std::vector routine(int count, int64_t start) { std::vector out; for (int i = 0; i < count; ++i) { const auto ts = start + i * 61 * kSec; const std::string sh = "process_" + std::to_string(1000 + i) + "_b_1"; out.push_back(exec(ts, 1000 + i, sh, "process_1_b_1", "/usr/bin/bash", "bash")); out.push_back(exec(ts + kSec, 20000 + i, "process_" + std::to_string(20000 + i) + "_b_1", sh, "/usr/bin/ls", "ls -la /home/a")); } return out; } void test_chain_and_pid_reuse() { std::vector ev; // firefox -> sh -> curl, curl connects: one window. ev.push_back(exec(kT0, 100, "process_100_b_1", "process_1_b_1", "/usr/lib/firefox/firefox", "firefox")); ev.push_back(exec(kT0 + 2 * kSec, 101, "process_101_b_1", "process_100_b_1", "/usr/bin/sh", "sh -c x")); ev.push_back(exec(kT0 + 3 * kSec, 102, "process_102_b_1", "process_101_b_1", "/usr/bin/curl", "curl -fsSL https://x/a.sh")); ev.push_back(connect(kT0 + 4 * kSec, 102, "203.0.113.9", 443)); // PID 500 reused: two different programs, each with a connection. ev.push_back(exec(kT0 + 10 * kSec, 500, "process_500_b_10", "process_1_b_1", "/usr/bin/a", "a")); ev.push_back(connect(kT0 + 11 * kSec, 500, "10.0.0.1", 22)); ev.push_back(exec(kT0 + 200 * kSec, 500, "process_500_b_20", "process_1_b_1", "/usr/bin/b", "b")); ev.push_back(connect(kT0 + 201 * kSec, 500, "10.0.0.2", 80)); ev.push_back(connect(kT0 + 202 * kSec, 777, "10.0.0.3", 80)); // no exec for 777 const auto report = run(ev); PROPERTY(report.execs == 5 && report.behavior_events == 4 && report.unjoined_events == 1); PROPERTY(report.windows == 3); const auto* chain = window_rooted(report, "process_100_b_1"); PROPERTY(chain != nullptr && chain->execs == 3 && chain->connects == 1); // R3 const auto* a = window_rooted(report, "process_500_b_10"); const auto* b = window_rooted(report, "process_500_b_20"); PROPERTY(a != nullptr && b != nullptr && a->connects == 1 && b->connects == 1); // C3 std::cout << "R3 chain window, C3 pid reuse: PASSED" << std::endl; } void test_order_and_determinism() { std::mt19937_64 rng(kSeed); auto ev = routine(50, kT0); ev.push_back(exec(kT0 + 5 * kSec + 7, 9, "process_9_b_1", "process_1_b_1", "/usr/bin/odd", "odd --x")); const auto base = replay_report_json(run(ev)); PROPERTY(replay_report_json(run(ev)) == base); // R1 // R2: shuffle within small time buckets (records arrive batched, a few // seconds apart at most; the horizon is 2 min). for (int round = 0; round < 20; ++round) { auto shuffled = ev; for (std::size_t i = 0; i + 1 < shuffled.size(); i += 2) { if (rng() % 2) std::swap(shuffled[i], shuffled[i + 1]); } PROPERTY(replay_report_json(run(shuffled)) == base); } std::cout << "R1 deterministic, R2 order-insensitive: PASSED" << std::endl; } void test_ranking_and_detection() { auto ev = routine(300, kT0); const int64_t t = kT0 + 301 * 61 * kSec; ev.push_back(exec(t, 4000, "process_4000_b_1", "process_1_b_1", "/usr/bin/evince", "evince /home/a/x.pdf")); ev.push_back(exec(t + kSec, 4001, "process_4001_b_1", "process_4000_b_1", "/usr/bin/python3", "python3 -c x")); ev.push_back(connect(t + 2 * kSec, 4001, "198.51.100.7", 4444)); ev.push_back(scan(t + kSec, 4001, "process_4001_b_1", {"Suspicious_Python"})); ReplayOptions options; options.warmup_ns = 3600 * kSec; // the first routine windows are new too: that is what warm-up is for options.top = 5; const auto report = run(ev, options); PROPERTY(!report.top_windows.empty()); const auto& top = report.top_windows.front(); PROPERTY(top.root_node == "process_4000_b_1"); // R4 PROPERTY(top.detected && top.rules == std::vector{"Suspicious_Python"}); PROPERTY(report.detected_windows == 1 && report.detected.size() == 1); PROPERTY(top.bits > report.top_windows.back().bits); std::cout << "R4 unusual chain ranks first (" << top.bits << " bits), detection marked: PASSED" << std::endl; } void test_store_round_trip() { namespace fs = std::filesystem; const auto dir = fs::temp_directory_path() / ("strawberry_replay_" + std::to_string(::getpid())); std::error_code ec; fs::remove_all(dir, ec); auto ev = routine(20, kT0); { Database db; PROPERTY(db.open(dir.string())); auto copy = ev; PROPERTY(db.append_telemetry(copy, true)); db.close(); } std::string error; const auto report = replay_store(dir.string(), no_warmup(), error); PROPERTY(error.empty() && report.execs == 40 && report.windows == 20); PROPERTY(replay_report_json(report) == replay_report_json(run(ev))); // R5 fs::remove_all(dir, ec); std::cout << "R5 store round trip: PASSED" << std::endl; } void test_warmup_and_time_invariance() { // Two novel programs, one at the start and one 20 h later after lots // of routine activity: same score (capped novelty), not "later wins". auto ev = routine(1200, kT0 + 60 * kSec); ev.insert(ev.begin(), scan(kT0, 7, "process_7_b_1", {"EarlyRule"})); ev.insert(ev.begin(), exec(kT0, 7, "process_7_b_1", "process_1_b_1", "/usr/bin/early-novel", "early-novel")); const int64_t late = kT0 + 1201 * 61 * kSec; ev.push_back(exec(late, 8, "process_8_b_1", "process_1_b_1", "/usr/bin/late-novel", "late-novel")); auto late_scan = ev; // a record far older than the reorder horizon: counted as late late_scan.push_back(scan(kT0 + 5, 7, "process_7_b_1", {"Late"})); PROPERTY(run(late_scan).late_events == 1); const auto all = run(ev); const ReplayWindow* early = nullptr; const ReplayWindow* late_w = nullptr; for (const auto& w : all.top_windows) { if (w.root_exe == "/usr/bin/early-novel") early = &w; if (w.root_exe == "/usr/bin/late-novel") late_w = &w; } PROPERTY(early != nullptr && late_w != nullptr && early->bits == late_w->bits); PROPERTY(early->bits == kNovelBits); // only exe_cmdline: its parent was never recorded // The routine bash -> ls windows collapse into one line. const auto routine_line = std::find_if(all.top_windows.begin(), all.top_windows.end(), [](const auto& w) { return w.root_exe == "/usr/bin/bash" && w.similar > 100; }); PROPERTY(routine_line != all.top_windows.end()); ReplayOptions warm; // default 12 h warm-up const auto report = run(ev, warm); PROPERTY(report.warmup_windows > 0 && report.warmup_windows < report.windows); for (const auto& w : report.top_windows) PROPERTY(w.start_ns >= kT0 + warm.warmup_ns); PROPERTY(report.detected.size() == 1 && report.detected[0].rules == std::vector{"EarlyRule"}); std::cout << "R7 warm-up, time invariance, grouping: PASSED" << std::endl; } void test_injection_evaluation() { namespace fs = std::filesystem; const auto dir = fs::temp_directory_path() / ("strawberry_inject_" + std::to_string(::getpid())); std::error_code ec; fs::remove_all(dir, ec); // Two days of routine activity, one window a minute. auto ev = routine(2 * 24 * 60, kT0); { Database db; PROPERTY(db.open(dir.string())); auto copy = ev; PROPERTY(db.append_telemetry(copy, true)); db.close(); } std::vector traces; std::string error; PROPERTY(compile_traces("trace pdf_python\n" "+0s exec viewer /usr/bin/evince \"evince /home/u/x.pdf\"\n" "+3s exec py /usr/bin/python3 \"python3 -c import socket\" parent=viewer\n" "+4s connect py 198.51.100.50 4444\n" "trace quiet\n" "+0s exec sh /usr/bin/bash \"bash\"\n", traces, error)); ReplayOptions options; options.warmup_ns = 6 * 3600 * kSec; const auto plain = replay_store(dir.string(), options, error); PROPERTY(error.empty()); options.inject = traces; options.inject_copies = 4; const auto report = replay_store(dir.string(), options, error); PROPERTY(error.empty()); const auto& eval = report.evaluation; PROPERTY(eval.traces.size() == 2 && eval.copies == 8); PROPERTY(eval.traces[0].found == 4 && eval.traces[1].found == 4); // Real activity is untouched: same records, windows and ranking. PROPERTY(report.records == plain.records && eval.real_windows + report.warmup_windows == plain.windows); PROPERTY(replay_report_json(plain).find("\"top_windows\"") != std::string::npos); PROPERTY(nlohmann::json::parse(replay_report_json(report))["top_windows"] == nlohmann::json::parse(replay_report_json(plain))["top_windows"]); // The novel chain is caught at 10/day; a bare bash window is not. PROPERTY(eval.traces[0].caught.back() == 4); PROPERTY(eval.traces[1].caught.back() < 4); for (const auto r : eval.traces[0].ranks) PROPERTY(r == 1); fs::remove_all(dir, ec); std::cout << "R8 injection evaluation: PASSED (pdf_python ranks 1; cut at 10/day " << eval.cut_scores.back() << " bits)" << std::endl; } void test_late_labels() { namespace fs = std::filesystem; const std::string bad_hash(64, 'b'); auto ev = routine(10, kT0); ProcessExecRecord r; r.timestamp_ns = kT0 + 5 * kSec + 3; r.pid = 600; r.process_node_id = "process_600_b_1"; r.exe = "/home/u/.local/bin/tool"; r.cmdline = "tool"; r.sha256 = bad_hash; r.source = "live"; ev.push_back({store::RecordType::process_exec, encode_process_exec(r)}); auto options = no_warmup(); options.late_labels[bad_hash] = {"BetaRule"}; const auto direct = run(ev, options); PROPERTY(direct.late_detected_windows == 1 && direct.detected_windows == 0); PROPERTY(direct.detected.size() == 1 && direct.detected[0].late_rules == std::vector{"BetaRule"}); const auto dir = fs::temp_directory_path() / ("strawberry_late_" + std::to_string(::getpid())); std::error_code ec; fs::remove_all(dir, ec); { Database db; PROPERTY(db.open(dir.string())); auto copy = ev; PROPERTY(db.append_telemetry(copy, true)); LabelRecord label; label.timestamp_ns = kT0 + 3600 * kSec; label.polarity = LabelPolarity::positive; label.source = "retro_hunt"; label.action = "rules_update"; label.sha256 = bad_hash; label.rule = "BetaRule"; PROPERTY(db.append_evidence(store::RecordType::label, encode_label(label))); label.polarity = LabelPolarity::negative; // negatives are not late detections label.sha256 = std::string(64, 'c'); PROPERTY(db.append_evidence(store::RecordType::label, encode_label(label))); db.close(); } std::string error; const auto stored = replay_store(dir.string(), no_warmup(), error); PROPERTY(error.empty() && stored.late_labels == 1 && stored.late_detected_windows == 1); fs::remove_all(dir, ec); std::cout << "R9 late labels: PASSED" << std::endl; } // Stage 3.5c: a window flagged late (retro-hunt) is attributed to the node that WROTE the flagged // binary (the dropper) via the global write->exec linkage, not the exec leaf that ran it. void test_attribution() { const std::string bad_hash(64, 'd'); auto ev = routine(10, kT0); // Dropper D (its own root) execs, then writes /tmp/evil. ev.push_back(exec(kT0 + 2 * kSec, 500, "process_500_d_1", "process_1_b_1", "/usr/bin/curl", "curl x")); ev.push_back(write(kT0 + 2 * kSec + 1, 500, "/tmp/evil")); // Victim V (a SEPARATE root) execs the dropped binary; flagged bad later. ProcessExecRecord r; r.timestamp_ns = kT0 + 3 * kSec; r.pid = 600; r.process_node_id = "process_600_v_1"; r.parent_node_id = "process_1_b_1"; r.exe = "/tmp/evil"; r.cmdline = "evil"; r.sha256 = bad_hash; r.source = "live"; ev.push_back({store::RecordType::process_exec, encode_process_exec(r)}); auto options = no_warmup(); options.late_labels[bad_hash] = {"DropperRule"}; auto victim = [](const ReplayReport& rep) -> const ReplayWindow* { for (const auto& w : rep.detected) { if (w.root_node == "process_600_v_1") return &w; } return nullptr; }; const auto rep = run(ev, options); const ReplayWindow* v = victim(rep); PROPERTY(v != nullptr && v->late_detected); PROPERTY(v->origin_node == "process_500_d_1"); // the dropper, not the exec leaf process_600_v_1 PROPERTY(v->origin_exe == "/usr/bin/curl"); // Determinism: same stream -> same attribution. // Keep the report alive: victim() returns a pointer into it (a temporary here dangled; it passed // on Linux by allocator luck and failed deterministically on macOS). const auto rep2 = run(ev, options); const ReplayWindow* v2 = victim(rep2); PROPERTY(v2 != nullptr && v2->origin_node == "process_500_d_1"); // A flagged binary with no in-set write has no origin. auto ev3 = routine(10, kT0); ProcessExecRecord r3 = r; r3.exe = "/usr/bin/true"; // never written in this stream ev3.push_back({store::RecordType::process_exec, encode_process_exec(r3)}); auto opt3 = no_warmup(); opt3.late_labels[bad_hash] = {"DropperRule"}; const auto rep3 = run(ev3, opt3); // keep alive: victim() points into it const ReplayWindow* v3 = victim(rep3); PROPERTY(v3 != nullptr && v3->late_detected && v3->origin_node.empty()); std::cout << "R10 attribution: late_detected traced to dropper node: PASSED" << std::endl; } void test_examples_and_model_ranking() { namespace fs = std::filesystem; auto ev = routine(200, kT0); const int64_t t = kT0 + 201 * 61 * kSec; ev.push_back(exec(t, 4000, "process_4000_b_1", "process_1_b_1", "/usr/bin/evince", "evince x.pdf")); ev.push_back(scan(t, 4000, "process_4000_b_1", {"SomeRule"})); const auto dir = fs::temp_directory_path() / ("strawberry_examples_" + std::to_string(::getpid())); std::error_code ec; fs::remove_all(dir, ec); { Database db; PROPERTY(db.open(dir.string())); auto copy = ev; PROPERTY(db.append_telemetry(copy, true)); db.close(); } std::vector traces; std::string error; PROPERTY(compile_traces("trace train_chain\n+0s exec a /tmp/x/dropper \"dropper\"\n" "+1s connect a 198.51.100.9 4444\n", traces, error)); ReplayOptions o = no_warmup(); o.collect_examples = true; o.inject = traces; o.inject_copies = 3; o.injected_are_positives = true; const auto r = replay_store(dir.string(), o, error); PROPERTY(error.empty()); PROPERTY(r.example_windows == 201 + 3 && r.example_positives == 1 + 3); PROPERTY(r.evaluation.copies == 0); // training traces are not evaluated PROPERTY(r.examples.size() == r.example_windows); std::size_t positives = 0; for (const auto& e : r.examples) positives += e.label > 0.5f; PROPERTY(positives == 4); o.max_examples = 100; o.max_entity_share = 0.05; // at most 5 bash windows const auto capped = replay_store(dir.string(), o, error); std::size_t bash = 0; for (const auto& e : capped.examples) bash += e.entity == "/usr/bin/bash" ? 1 : 0; PROPERTY(bash == 5 && capped.examples.size() == 5 + 4 + 0); // 5 bash + 4 positives (evince is positive) auto model = std::make_shared(Model::initialized(ModelShape{}, 3)); ReplayOptions ranked = no_warmup(); ranked.model = model; const auto m = replay_store(dir.string(), ranked, error); PROPERTY(!m.top_windows.empty()); for (const auto& w : m.top_windows) { PROPERTY(w.model_value >= 0.0f && w.model_value <= 1.0f); PROPERTY(std::fabs(w.bits - w.model_value * 48.0) < 1e-3); } fs::remove_all(dir, ec); std::cout << "R10 examples and model ranking: PASSED" << std::endl; } void test_positive_sampling_under_pressure() { // Fill every slot with distinct negative entities before any teacher // positive arrives. Late labels must still get their class quota. auto events = std::vector{}; for (int i = 0; i < 100; ++i) { const auto ts = kT0 + i * 61 * kSec; const auto node = "process_" + std::to_string(5000 + i) + "_b_1"; const std::string entity = std::string("/usr/bin/sample_") + static_cast('a' + i / 26) + static_cast('a' + i % 26); events.push_back(exec(ts, 5000 + i, node, "", entity, entity)); if (i >= 60) events.push_back(scan(ts, 5000 + i, node, {"Teacher"})); } ReplayOptions o = no_warmup(); o.collect_examples = true; o.max_examples = 20; o.max_entity_share = 0.05; // one negative per entity o.sample_seed = kSeed; const auto sampled = run(events, o); PROPERTY(sampled.example_windows == 100 && sampled.example_positives == 40); PROPERTY(sampled.examples.size() == o.max_examples); std::size_t positives = 0; bool retained_later_positive = false; for (const auto& e : sampled.examples) { positives += e.label > 0.5f; retained_later_positive |= e.label > 0.5f && e.start_ns >= kT0 + 64 * 61 * kSec; } PROPERTY(positives == 4); // the positive quota survives a full negative sample // Check reproducibility including examples, which report JSON does not emit. const auto again = run(events, o); PROPERTY(again.examples.size() == sampled.examples.size()); for (std::size_t i = 0; i < sampled.examples.size(); ++i) { PROPERTY(sampled.examples[i].start_ns == again.examples[i].start_ns); PROPERTY(sampled.examples[i].entity == again.examples[i].entity); PROPERTY(sampled.examples[i].label == again.examples[i].label); PROPERTY(sampled.examples[i].x == again.examples[i].x); } // Do not assert that an arbitrary seed must choose a later example: // reservoir sampling may legitimately retain the first four. Across // fixed seeds, however, the implementation cannot keep only the first. for (uint64_t seed = 1; seed <= 8; ++seed) { o.sample_seed = seed; for (const auto& e : run(events, o).examples) { retained_later_positive |= e.label > 0.5f && e.start_ns >= kT0 + 64 * 61 * kSec; } } PROPERTY(retained_later_positive); // Tiny configured samples must retain a teacher example too. for (std::size_t capacity = 1; capacity < 5; ++capacity) { o.max_examples = capacity; const auto tiny = run(events, o); PROPERTY(tiny.examples.size() == capacity); PROPERTY(std::count_if(tiny.examples.begin(), tiny.examples.end(), [](const auto& e) { return e.label > 0.5f; }) == 1); } std::cout << "R10 positive sampling under pressure: PASSED" << std::endl; } void test_policy_evaluation() { // Gate as a pure function. Checks: positives, baseline health, efficiency, // recall, out-of-sample. PolicyEvaluation pe; pe.extra_positives = 12; pe.policies = {{"model", 10, 20.0, 9, 9.0, 0}, {"rarity_greedy", 20, 40.0, 8, 8.0, 0}}; pe.provenance.model = "seed1"; pe.provenance.model_sha256 = "0123456789abcdef"; pe.provenance.training_range_known = true; pe.provenance.training_first_ns = 1000; pe.provenance.training_last_ns = 2000; pe.provenance.eval_first_ns = 3000; pe.provenance.eval_last_ns = 4000; auto g = promotion_gate(pe); // 0.45 vs 0.2 hits/CPU-s, recall 9 >= 8 PROPERTY(g.passed && g.checks.size() == 5); PROPERTY(g.checks[4].rfind("ok: out-of-sample", 0) == 0); pe.policies[0].hits = 7; // efficiency 0.35 (still >= 0.3) but lower recall pe.policies[0].weighted_hits = 7; g = promotion_gate(pe); PROPERTY(!g.passed && g.checks[3].rfind("FAIL", 0) == 0 && g.checks[2].rfind("ok", 0) == 0); pe.policies[0] = {"model", 10, 40.0, 9, 9.0, 0}; // 0.225 < 1.5 x 0.2 PROPERTY(!promotion_gate(pe).passed && promotion_gate(pe).checks[2].rfind("FAIL", 0) == 0); // Degenerate baseline: a rarity-greedy baseline with no hits or no CPU-seconds // makes criteria 2 and 3 pass for free, so the cell must fail and say why. PolicyEvaluation degenerate = pe; degenerate.extra_positives = 12; degenerate.policies = {{"model", 10, 20.0, 9, 9.0, 0}, {"rarity_greedy", 10, 0.0, 0, 0.0, 7}}; auto dg = promotion_gate(degenerate); PROPERTY(!dg.passed); PROPERTY(dg.checks[1].rfind("FAIL", 0) == 0 && dg.checks[1].find("degenerate baseline") != std::string::npos); PROPERTY(dg.checks[1].find("7 budget refusals") != std::string::npos); degenerate.policies[1] = {"rarity_greedy", 10, 5.0, 0, 0.0, 0}; // 0 hits, CPU > 0 PROPERTY(!promotion_gate(degenerate).passed && promotion_gate(degenerate).checks[1].find("degenerate baseline") != std::string::npos); degenerate.policies[1] = {"rarity_greedy", 10, 0.0, 3, 3.0, 0}; // hits, no CPU PROPERTY(!promotion_gate(degenerate).passed && promotion_gate(degenerate).checks[1].find("degenerate baseline") != std::string::npos); // In-sample evaluation: the gate says so and refuses. PolicyEvaluation in_sample = pe; in_sample.extra_positives = 12; in_sample.policies = {{"model", 10, 20.0, 9, 9.0, 0}, {"rarity_greedy", 20, 40.0, 8, 8.0, 0}}; in_sample.provenance.eval_first_ns = 1500; // inside the training range in_sample.provenance.in_sample = true; auto is = promotion_gate(in_sample); PROPERTY(!is.passed); PROPERTY(is.checks[4].rfind("FAIL", 0) == 0 && is.checks[4].find("in-sample evaluation") != std::string::npos); PROPERTY(is.checks[4].find("0123456789abcdef") != std::string::npos); // Unprovable provenance never counts as out-of-sample. PolicyEvaluation unknown = in_sample; unknown.provenance.in_sample = false; unknown.provenance.training_range_known = false; PROPERTY(!promotion_gate(unknown).passed && promotion_gate(unknown).checks[4].find("no recorded training range") != std::string::npos); PolicyEvaluation unbound = in_sample; unbound.provenance.in_sample = false; unbound.provenance.model_sha256.clear(); PROPERTY(!promotion_gate(unbound).passed && promotion_gate(unbound).checks[4].find("no model digest") != std::string::npos); pe.extra_positives = 4; pe.policies[0] = {"model", 10, 20.0, 4, 4.0, 0}; PROPERTY(!promotion_gate(pe).passed && promotion_gate(pe).checks[0].rfind("FAIL", 0) == 0); pe.policies.pop_back(); PROPERTY(!promotion_gate(pe).passed); // Replay: routine history, injected evaluation traces, a rule detection, // and logged shadow decisions (one on an injected window). auto ev = routine(240, kT0); const int64_t t = kT0 + 30 * 61 * kSec + 7; ev.push_back(exec(t, 4000, "process_4000_b_1", "process_1_b_1", "/usr/bin/evince", "evince x.pdf")); ev.push_back(scan(t, 4000, "process_4000_b_1", {"SomeRule"})); InspectionDecision logged; logged.timestamp_ns = t + 200 * kSec; logged.root_node = "process_4000_b_1"; logged.action = InspectAction::memory_scan; ev.push_back({store::RecordType::inspection_action, encode_inspection_action(logged)}); logged.shadow = false; ev.push_back({store::RecordType::inspection_action, encode_inspection_action(logged)}); std::vector traces; std::string error; PROPERTY(compile_traces("trace odd\n+0s exec a /tmp/q/odd \"odd --x\"\n+1s connect a 198.51.100.4 4444\n", traces, error)); ReplayOptions o; o.warmup_ns = 600 * kSec; // routine windows are known by then o.policy_eval = true; o.policy.budget_seconds_per_hour = 4; // two memory scans an hour // Even an apparently perfect free unsupported action cannot influence // the deployment gate. Without the executable mask this evades budgets. o.policy.reach[static_cast(InspectAction::watch_subtree)] = 1000; o.policy.cost_seconds[static_cast(InspectAction::watch_subtree)] = 0; o.inject = traces; o.inject_copies = 4; // Through a store, so injection is planned over the history's time range. namespace fs = std::filesystem; const auto dir = fs::temp_directory_path() / ("strawberry_policy_" + std::to_string(::getpid())); std::error_code ec; fs::remove_all(dir, ec); { Database db; PROPERTY(db.open(dir.string())); auto copy = ev; PROPERTY(db.append_telemetry(copy, true)); db.close(); } const auto r = replay_store(dir.string(), o, error); PROPERTY(error.empty()); const auto& p = r.policy_eval; PROPERTY(p.extra_positives == 4 && p.rule_detected == 1); // the evince window is rules' already PROPERTY(p.policies.size() == 2 && p.policies[0].name == "rarity_greedy" && p.policies[1].name == "shadow_logged"); const auto& rarity = p.policies[0]; // Structure only: with a few hundred windows rarity has not seen enough // (its smoothing wants ~30k observations) to separate anything. PROPERTY(rarity.hits <= p.extra_positives && rarity.inspections + rarity.budget_refusals <= p.windows); PROPERTY(rarity.budget_refusals > 0); PROPERTY(rarity.cpu_seconds <= 4.0 * (static_cast(r.last_ns - r.first_ns) / 3.6e12 + 1.0)); const auto& shadow = p.policies[1]; PROPERTY(shadow.inspections == 1 && shadow.hits == 0); // it inspected a rule-flagged window: not an extra hit fs::remove_all(dir, ec); std::cout << "R11 policy evaluation and gate: PASSED (rarity-greedy " << rarity.hits << "/" << p.extra_positives << " in " << rarity.cpu_seconds << " CPU-s)" << std::endl; } // R11b fleet gate cells: with one eval model equal to `model` and one period, the cell is exactly // the single-model evaluation; with N periods, cells partition the windows and positives. void test_gate_cells() { auto ev = routine(240, kT0); std::vector traces; std::string error; PROPERTY(compile_traces("trace odd\n+0s exec a /tmp/q/odd \"odd --x\"\n+1s connect a 198.51.100.4 4444\n", traces, error)); namespace fs = std::filesystem; const auto dir = fs::temp_directory_path() / ("strawberry_gate_cells_" + std::to_string(::getpid())); std::error_code ec; fs::remove_all(dir, ec); { Database db; PROPERTY(db.open(dir.string())); auto copy = ev; PROPERTY(db.append_telemetry(copy, true)); db.close(); } auto ma = std::make_shared(Model::initialized(ModelShape{}, 11)); auto mb = std::make_shared(Model::initialized(ModelShape{}, 12)); ReplayOptions o; o.warmup_ns = 600 * kSec; o.policy_eval = true; o.policy.budget_seconds_per_hour = 4; o.inject = traces; o.inject_copies = 9; o.model = ma; o.eval_models = {{"a", ma}}; o.eval_periods = 1; const auto r1 = replay_store(dir.string(), o, error); PROPERTY(error.empty()); const auto& pe = r1.policy_eval; PROPERTY(pe.cells.size() == 1 && pe.cells[0].model == "a" && pe.cells[0].period == 0); const auto& cell = pe.cells[0]; PROPERTY(cell.windows == pe.windows && cell.extra_positives == pe.extra_positives && cell.rule_detected == pe.rule_detected); const auto find = [](const std::vector& ps, const std::string& n) -> const PolicyResult* { for (const auto& p : ps) if (p.name == n) return &p; return nullptr; }; for (const char* name : {"model", "rarity_greedy"}) { const auto* agg = find(pe.policies, name); const auto* c = find(cell.policies, name); PROPERTY(agg && c && agg->hits == c->hits && agg->inspections == c->inspections && agg->budget_refusals == c->budget_refusals && agg->cpu_seconds == c->cpu_seconds); } // The gate applies to a cell unchanged -- and now also refuses to count an // unproven or in-sample evaluation as evidence. PolicyEvaluation as_eval; as_eval.extra_positives = cell.extra_positives; as_eval.policies = cell.policies; as_eval.provenance = cell.provenance; PROPERTY(promotion_gate(as_eval).checks.size() == 5); // The cell carries the model's content identity and its (here: absent) range. PROPERTY(!cell.provenance.model_sha256.empty()); PROPERTY(cell.provenance.model_sha256 == model_fingerprint(*ma)); PROPERTY(!cell.provenance.training_range_known && !cell.provenance.in_sample); PROPERTY(!promotion_gate(as_eval).passed && promotion_gate(as_eval).checks[4].find("no recorded training range") != std::string::npos); // Two models x three periods: 6 cells; per model the periods partition windows and positives; // the periods are ordered and cover the ranked history. o.eval_models = {{"a", ma}, {"b", mb}}; o.eval_periods = 3; const auto r3 = replay_store(dir.string(), o, error); PROPERTY(error.empty() && r3.policy_eval.cells.size() == 6); for (const char* m : {"a", "b"}) { uint64_t windows = 0, positives = 0; int64_t prev_end = INT64_MIN; for (const auto& c : r3.policy_eval.cells) { if (c.model != m) continue; windows += c.windows; positives += c.extra_positives; PROPERTY(c.start_ns <= c.end_ns && c.start_ns >= prev_end); prev_end = c.end_ns; } PROPERTY(windows == r3.policy_eval.windows && positives == r3.policy_eval.extra_positives); } // Injected copies spread over the history: every period gets positives. for (const auto& c : r3.policy_eval.cells) PROPERTY(c.extra_positives > 0); const auto json = replay_report_json(r3); PROPERTY(json.find("\"cells\"") != std::string::npos); // A model trained on the replayed history makes every cell in-sample: the // cell says so, and the gate refuses it naming the reason. auto trained = std::make_shared(Model::initialized(ModelShape{}, 11)); trained->training_first_ns = kT0; trained->training_last_ns = kT0 + 240 * 61 * kSec; o.eval_models = {{"trained", trained}}; o.eval_periods = 2; const auto ri = replay_store(dir.string(), o, error); PROPERTY(error.empty() && ri.policy_eval.cells.size() == 2); for (const auto& c : ri.policy_eval.cells) { PROPERTY(c.provenance.training_range_known && c.provenance.in_sample); PROPERTY(c.provenance.model_sha256 == model_fingerprint(*trained)); PolicyEvaluation ev; ev.extra_positives = c.extra_positives; ev.policies = c.policies; ev.provenance = c.provenance; const auto gate_in = promotion_gate(ev); PROPERTY(!gate_in.passed && gate_in.checks[4].find("in-sample evaluation") != std::string::npos); } const auto in_sample_json = replay_report_json(ri); PROPERTY(in_sample_json.find("\"model_sha256\"") != std::string::npos); PROPERTY(in_sample_json.find("\"in_sample\": true") != std::string::npos); // A model trained on an earlier, disjoint period is not flagged in-sample. auto earlier = std::make_shared(Model::initialized(ModelShape{}, 11)); earlier->training_first_ns = kT0 - 1000 * kSec; earlier->training_last_ns = kT0 - 500 * kSec; o.eval_models = {{"earlier", earlier}}; const auto rd = replay_store(dir.string(), o, error); PROPERTY(error.empty() && !rd.policy_eval.cells.empty()); for (const auto& c : rd.policy_eval.cells) PROPERTY(c.provenance.training_range_known && !c.provenance.in_sample); fs::remove_all(dir, ec); std::cout << "R11b fleet gate cells: PASSED (" << r3.policy_eval.cells.size() << " cells, " << r3.policy_eval.extra_positives << " positives)" << std::endl; } void test_extreme_timestamps() { // On-disk timestamps are nonnegative int64 values, not necessarily sane // wall-clock dates. Interval comparisons must not overflow near the end. ReplayOptions options; options.warmup_ns = 3600 * kSec; Replayer replay(options); const auto first = exec(INT64_MAX - 100 * kSec, 42, "process_42_b_1", "", "/tmp/a", "a"); const auto last = exec(INT64_MAX - 1, 43, "process_43_b_1", "", "/tmp/b", "b"); replay.feed(first.type, first.payload); replay.feed(last.type, last.payload); const auto report = replay.finish(); PROPERTY(report.warmup_windows == 2 && report.top_windows.empty()); std::string error; PROPERTY(compile_traces("trace edge\n+0s exec a /tmp/edge edge\n", options.inject, error)); options.inject_copies = 1; // Field iteration accepts repeated tags. The store indexes the first // timestamp while the typed decoder retains the last, so the window // may start after report.last_ns even though both times are nonnegative. auto duplicate = exec(1, 42, "process_42_b_1", "", "/tmp/a", "a"); store::FieldWriter suffix; suffix.u64(1, INT64_MAX - 1); duplicate.payload += suffix.data(); Replayer mismatched(options); mismatched.feed(duplicate.type, duplicate.payload); PROPERTY(mismatched.finish().evaluation.days == 1.0 / 24); namespace fs = std::filesystem; const auto dir = fs::temp_directory_path() / ("strawberry_replay_edge_" + std::to_string(::getpid())); std::error_code ec; fs::remove_all(dir, ec); { Database db; PROPERTY(db.open(dir.string())); std::vector events{first, last}; PROPERTY(db.append_telemetry(events, true)); } (void) replay_store(dir.string(), options, error); PROPERTY(error.find("not enough history") != std::string::npos); fs::remove_all(dir, ec); } void test_invalid_utf8() { // Split literals: "\x80" "evil", or the escape would swallow the 'e'. const std::string bad = std::string("/tmp/\xff\xfe\xc0\x80" "evil\xed\xa0\x80") + "\xf4\x90\x80\x80"; std::vector ev{exec(kT0, 1, "process_1_b_1", "", bad, bad + " -x")}; const auto json = replay_report_json(run(ev)); const auto parsed = nlohmann::json::parse(json, nullptr, false); PROPERTY(!parsed.is_discarded()); PROPERTY(!valid_utf8(bad) && valid_utf8(json)); std::mt19937_64 rng(kSeed); for (int i = 0; i < 20000; ++i) { std::string s; const auto n = rng() % 12; for (std::size_t k = 0; k < n; ++k) s.push_back(static_cast(rng() % 256)); const auto quoted = json_string(s); const auto back = nlohmann::json::parse(quoted, nullptr, false); PROPERTY(!back.is_discarded() && back.is_string()); if (valid_utf8(s)) PROPERTY(back.get() == s); // valid text is unchanged } PROPERTY(valid_utf8("plain ascii") && valid_utf8("\xc3\xa9t\xc3\xa9") && !valid_utf8("\xc3")); std::cout << "R6 invalid UTF-8 stays valid JSON: PASSED" << std::endl; } } // namespace // A window running an allowlisted (negative-labelled) binary is a confirmed // benign: kept as a negative training example, never a teacher positive, even // though a (now-cleared) rule detected it. This is what keeps a false-positive // feed hit the operator allowlisted from poisoning the labels. void test_allowlisted_window_is_negative() { namespace fs = std::filesystem; const std::string sha(64, 'd'); auto ev = routine(10, kT0); // benign windows (label 0) ProcessExecRecord r; r.timestamp_ns = kT0 + 5 * kSec + 7; r.pid = 700; r.process_node_id = "process_700_d_1"; r.exe = "/usr/bin/ss"; r.cmdline = "ss -tulpn"; r.sha256 = sha; r.source = "live"; ev.push_back({store::RecordType::process_exec, encode_process_exec(r)}); ev.push_back(scan(r.timestamp_ns + 1, 700, "process_700_d_1", {"IOC_KnownMalwareHash"})); // detected const auto dir = fs::temp_directory_path() / ("strawberry_allow_neg_" + std::to_string(::getpid())); std::error_code ec; fs::remove_all(dir, ec); { Database db; PROPERTY(db.open(dir.string())); auto copy = ev; PROPERTY(db.append_telemetry(copy, true)); db.close(); } std::string error; ReplayOptions o = no_warmup(); o.collect_examples = true; const auto before = replay_store(dir.string(), o, error); std::size_t pos_before = 0; for (const auto& e : before.examples) pos_before += e.label > 0.5f; PROPERTY(error.empty() && pos_before == 1); // the detected ss window is a positive { // the operator allowlists the hash -> a negative label in the stream Database db; PROPERTY(db.open(dir.string())); LabelRecord label; label.timestamp_ns = kT0 + 6 * kSec; label.polarity = LabelPolarity::negative; label.source = "allowlist"; label.action = "suppressed"; label.sha256 = sha; PROPERTY(db.append_evidence(store::RecordType::label, encode_label(label))); db.close(); } const auto after = replay_store(dir.string(), o, error); std::size_t pos_after = 0; for (const auto& e : after.examples) pos_after += e.label > 0.5f; PROPERTY(error.empty() && pos_after == 0); // no longer a positive PROPERTY(after.examples.size() == before.examples.size()); // kept as a negative, not dropped fs::remove_all(dir, ec); std::cout << "allowlisted window is a negative: PASSED" << std::endl; } // ---- label retraction (R12) ---- void append_label(const std::string& dir, LabelPolarity polarity, const std::string& action, const std::string& sha, const std::string& entry, const std::string& source, const std::string& rule, int64_t timestamp = kT0 + 3600 * kSec) { Database db; PROPERTY(db.open(dir)); LabelRecord label; label.timestamp_ns = timestamp; label.polarity = polarity; label.source = source; label.action = action; label.sha256 = sha; label.allowlist_entry = entry; label.rule = rule; PROPERTY(db.append_evidence(store::RecordType::label, encode_label(label))); db.close(); } std::size_t positives_in(const ReplayReport& r) { std::size_t n = 0; for (const auto& e : r.examples) n += e.label > 0.5f; return n; } const ReplayWindow* detected_rooted(const ReplayReport& report, const std::string& node) { for (const auto& w : report.detected) { if (w.root_node == node) return &w; } return nullptr; } // A store with a few routine windows plus one window running `sha` (`exe`), // optionally rule-detected. std::filesystem::path make_retract_store(const std::string& tag, const std::string& sha, const std::string& exe, bool detected) { namespace fs = std::filesystem; const auto dir = fs::temp_directory_path() / ("strawberry_" + tag + "_" + std::to_string(::getpid())); std::error_code ec; fs::remove_all(dir, ec); auto ev = routine(5, kT0); ProcessExecRecord r; r.timestamp_ns = kT0 + 3 * kSec + 7; r.pid = 700; r.process_node_id = "process_700_b_1"; r.exe = exe; r.cmdline = "x"; r.sha256 = sha; r.source = "live"; ev.push_back({store::RecordType::process_exec, encode_process_exec(r)}); if (detected) ev.push_back(scan(r.timestamp_ns + 1, 700, "process_700_b_1", {"IOC_KnownMalwareHash"})); Database db; PROPERTY(db.open(dir.string())); auto copy = ev; PROPERTY(db.append_telemetry(copy, true)); db.close(); return dir; } void test_retraction_allowlist_remove() { namespace fs = std::filesystem; const std::string sha(64, 'd'); const auto dir = make_retract_store("retract_al", sha, "/usr/bin/ss", true); std::string error; ReplayOptions o = no_warmup(); o.collect_examples = true; append_label(dir.string(), LabelPolarity::negative, "allowlist-add", sha, "allowlist-e1", "operator", ""); const auto after_add = replay_store(dir.string(), o, error); PROPERTY(error.empty() && positives_in(after_add) == 0); // cleared false positive append_label(dir.string(), LabelPolarity::retracted, "allowlist-remove", "", "allowlist-e1", "operator", ""); const auto after_remove = replay_store(dir.string(), o, error); // The removal reverts the hash to its pre-label state: the rule detection // stands again (it was the only judgment). It is not a new teacher positive. PROPERTY(error.empty() && positives_in(after_remove) == 1); fs::remove_all(dir); std::cout << "R12 allowlist-remove retraction: PASSED" << std::endl; } void test_retraction_quarantine_remove() { namespace fs = std::filesystem; const std::string sha(64, 'b'); const auto dir = make_retract_store("retract_q", sha, "/tmp/x/dropper", false); std::string error; ReplayOptions o = no_warmup(); o.collect_examples = true; append_label(dir.string(), LabelPolarity::positive, "quarantine", sha, "", "operator", ""); const auto after_quarantine = replay_store(dir.string(), o, error); PROPERTY(error.empty() && positives_in(after_quarantine) == 1); // late-detected via quarantine append_label(dir.string(), LabelPolarity::retracted, "quarantine-remove", sha, "", "operator", ""); const auto after_remove = replay_store(dir.string(), o, error); PROPERTY(error.empty() && positives_in(after_remove) == 0); // the quarantine reason is gone fs::remove_all(dir); std::cout << "R12 quarantine-remove retraction: PASSED" << std::endl; } void test_retraction_independent_positives() { namespace fs = std::filesystem; const std::string sha(64, 'c'); const auto dir = make_retract_store("retract_indep", sha, "/tmp/x/dropper", false); std::string error; ReplayOptions o = no_warmup(); o.collect_examples = true; // Two independent positives on one hash: an operator quarantine and a // retro-hunt rule. Removing the quarantine must not erase the rule reason. append_label(dir.string(), LabelPolarity::positive, "quarantine", sha, "", "operator", ""); append_label(dir.string(), LabelPolarity::positive, "rules_update", sha, "", "rules_update", "BetaRule"); const auto before = replay_store(dir.string(), o, error); PROPERTY(error.empty() && positives_in(before) == 1); const auto* before_w = detected_rooted(before, "process_700_b_1"); PROPERTY(before_w != nullptr && before_w->late_rules.size() == 2); append_label(dir.string(), LabelPolarity::retracted, "quarantine-remove", sha, "", "operator", ""); const auto after = replay_store(dir.string(), o, error); PROPERTY(error.empty() && positives_in(after) == 1); // still late-detected via BetaRule const auto* after_w = detected_rooted(after, "process_700_b_1"); PROPERTY(after_w != nullptr && after_w->late_rules == std::vector{"BetaRule"}); fs::remove_all(dir); std::cout << "R12 independent positives survive quarantine-remove: PASSED" << std::endl; } void test_retraction_multiple_entries() { namespace fs = std::filesystem; const std::string sha(64, 'e'); const auto dir = make_retract_store("retract_multi", sha, "/usr/bin/ss", true); std::string error; ReplayOptions o = no_warmup(); o.collect_examples = true; append_label(dir.string(), LabelPolarity::negative, "allowlist-add", sha, "allowlist-e1", "operator", ""); append_label(dir.string(), LabelPolarity::negative, "allowlist-add", sha, "allowlist-e2", "operator", ""); const auto both = replay_store(dir.string(), o, error); PROPERTY(error.empty() && positives_in(both) == 0); append_label(dir.string(), LabelPolarity::retracted, "allowlist-remove", "", "allowlist-e1", "operator", ""); const auto one_left = replay_store(dir.string(), o, error); PROPERTY(error.empty() && positives_in(one_left) == 0); // e2 still covers the hash append_label(dir.string(), LabelPolarity::retracted, "allowlist-remove", "", "allowlist-e2", "operator", ""); const auto none = replay_store(dir.string(), o, error); PROPERTY(error.empty() && positives_in(none) == 1); // both entries gone fs::remove_all(dir); std::cout << "R12 multiple allowlist entries resolve independently: PASSED" << std::endl; } void test_retraction_readd() { namespace fs = std::filesystem; const std::string sha(64, 'f'); const auto dir = make_retract_store("retract_readd", sha, "/usr/bin/ss", true); std::string error; ReplayOptions o = no_warmup(); o.collect_examples = true; append_label(dir.string(), LabelPolarity::negative, "allowlist-add", sha, "allowlist-e1", "operator", ""); PROPERTY(error.empty() && positives_in(replay_store(dir.string(), o, error)) == 0); append_label(dir.string(), LabelPolarity::retracted, "allowlist-remove", "", "allowlist-e1", "operator", ""); PROPERTY(error.empty() && positives_in(replay_store(dir.string(), o, error)) == 1); append_label(dir.string(), LabelPolarity::negative, "allowlist-add", sha, "allowlist-e1", "operator", ""); PROPERTY(error.empty() && positives_in(replay_store(dir.string(), o, error)) == 0); // re-added fs::remove_all(dir); std::cout << "R12 add/remove/re-add resolves in stored order: PASSED" << std::endl; } // ---- positive per-entity cap (R13) ---- void test_positive_entity_cap() { std::vector ev; // One hot entity with many detected (positive) windows, plus many distinct // negative entities to fill the sample. for (int i = 0; i < 60; ++i) { const auto ts = kT0 + i * 61 * kSec; const auto node = "process_" + std::to_string(6000 + i) + "_b_1"; ev.push_back(exec(ts, 6000 + i, node, "", "/usr/bin/hot", "hot --x")); ev.push_back(scan(ts, 6000 + i, node, {"Teacher"})); } for (int i = 0; i < 200; ++i) { const auto ts = kT0 + 1000 * 61 * kSec + i * 61 * kSec; const auto node = "process_" + std::to_string(8000 + i) + "_b_1"; const std::string entity = std::string("/usr/bin/neg_") + static_cast('a' + i / 26) + static_cast('a' + i % 26); ev.push_back(exec(ts, 8000 + i, node, "", entity, entity)); } ReplayOptions o = no_warmup(); o.collect_examples = true; o.max_examples = 50; o.max_entity_share = 0.1; // combined cap = 5 o.sample_seed = kSeed; const auto r = run(ev, o); // Combined per-entity cap holds for every entity. std::unordered_map per_entity; for (const auto& e : r.examples) ++per_entity[e.entity]; for (const auto& [entity, n] : per_entity) PROPERTY(n <= 5); // The hot entity's 60 raw positives are capped to at most 5, and the // rejection is counted (distinguishable from accidental starvation). PROPERTY(per_entity["/usr/bin/hot"] <= 5); // Extra candidates may refresh existing slots, so not every arrival // beyond the cap is a rejection. PROPERTY(r.example_positive_entity_rejections <= 55); // The positive quota still bounds the retained positives. PROPERTY(positives_in(r) <= o.max_examples / 5); PROPERTY(r.examples.size() <= o.max_examples); // Deterministic per seed. const auto again = run(ev, o); PROPERTY(again.examples.size() == r.examples.size()); for (std::size_t i = 0; i < r.examples.size(); ++i) { PROPERTY(r.examples[i].start_ns == again.examples[i].start_ns); PROPERTY(r.examples[i].entity == again.examples[i].entity); PROPERTY(r.examples[i].label == again.examples[i].label); } std::cout << "R13 positive entity cap: PASSED (hot entity " << per_entity["/usr/bin/hot"] << " examples, " << r.example_positive_entity_rejections << " concentration rejections)" << std::endl; } void test_positive_not_starved_by_entity_negatives() { std::vector ev; // Entity "/usr/bin/mixed" first contributes many negatives (filling its // combined cap), then one detected positive. The positive must displace a // negative of the same entity rather than being starved by the negatives. for (int i = 0; i < 10; ++i) { const auto ts = kT0 + i * 61 * kSec; const auto node = "process_" + std::to_string(7000 + i) + "_b_1"; ev.push_back(exec(ts, 7000 + i, node, "", "/usr/bin/mixed", "mixed")); } const int64_t tpos = kT0 + 100 * 61 * kSec; ev.push_back(exec(tpos, 7999, "process_7999_b_1", "", "/usr/bin/mixed", "mixed")); ev.push_back(scan(tpos, 7999, "process_7999_b_1", {"Teacher"})); ReplayOptions o = no_warmup(); o.collect_examples = true; o.max_examples = 8; o.max_entity_share = 0.5; // combined cap = 4 o.sample_seed = kSeed; const auto r = run(ev, o); PROPERTY(positives_in(r) == 1); // the late positive is retained std::size_t mixed = 0; for (const auto& e : r.examples) mixed += e.entity == "/usr/bin/mixed"; PROPERTY(mixed <= 4); // combined cap held even though negatives preceded the positive bool retained_positive = false; for (const auto& e : r.examples) retained_positive |= e.label > 0.5f && e.entity == "/usr/bin/mixed"; PROPERTY(retained_positive); std::cout << "R13 positive displaces same-entity negatives: PASSED" << std::endl; } void test_sampler_capacity_lifetime() { // The debug sampler invariant checks ALLOCATED per-entity capacity, not // just logical slot counts. Empty class vectors must release old storage. auto o = no_warmup(); o.collect_examples = true; o.max_examples = 1; o.max_entity_share = 1; o.sample_seed = 1; auto events = std::vector{ exec(kT0, 9400, "process_9400_b_1", "", "/usr/bin/kept", "kept"), exec(kT0 + 61 * kSec, 9401, "process_9401_b_1", "", "/usr/bin/kept", "kept"), scan(kT0 + 61 * kSec, 9401, "process_9401_b_1", {"Teacher"})}; PROPERTY(positives_in(run(events, o)) == 1); // Keep one negative of A while its positive reservoir grows and shrinks. // Historic nonempty capacities must also shrink geometrically, otherwise // many such retained entities can accumulate O(max_examples^2) storage. o.max_examples = 128; for (int i = 0; i < 532; ++i) { const auto ts = kT0 + (i + 2) * 61 * kSec; const auto node = "process_" + std::to_string(9500 + i) + "_b_1"; const auto entity = i < 20 ? "/usr/bin/kept" : "/usr/bin/other"; events.push_back(exec(ts, 9500 + i, node, "", entity, entity)); events.push_back(scan(ts, 9500 + i, node, {"Teacher"})); } const auto result = run(events, o); PROPERTY(positives_in(result) == 25 && result.examples.size() == 26); std::cout << "R15 allocated sampler capacity lifetime: PASSED" << std::endl; } void test_parent_audit_regressions() { namespace fs = std::filesystem; std::size_t failures = 0; const auto verify = [&](bool ok, const char* reason) { if (!ok) { ++failures; std::cerr << "AUDIT REGRESSION FAILED: " << reason << std::endl; } }; ReplayOptions o = no_warmup(); o.collect_examples = true; const std::string sha(64, 'a'), other(64, 'b'); std::string error; auto dir = make_retract_store("audit_multi_hash", sha, "/usr/bin/ss", true); // A path/rule entry can suppress several binaries. Removal must retire // every binding, not just the last hash observed for that entry. append_label(dir.string(), LabelPolarity::negative, "suppressed", sha, "path-entry", "allowlist", ""); append_label(dir.string(), LabelPolarity::negative, "suppressed", other, "path-entry", "allowlist", ""); append_label(dir.string(), LabelPolarity::retracted, "allowlist-remove", "", "path-entry", "operator", ""); verify(positives_in(replay_store(dir.string(), o, error)) == 1 && error.empty(), "remove all entry hash bindings"); fs::remove_all(dir); dir = make_retract_store("audit_delayed_suppression", sha, "/usr/bin/ss", true); append_label(dir.string(), LabelPolarity::negative, "allowlist-add", sha, "entry", "operator", "", kT0 + 100); // Stored order, not wall time, controls policy judgments. append_label(dir.string(), LabelPolarity::retracted, "allowlist-remove", "", "entry", "operator", "", kT0 + 50); append_label(dir.string(), LabelPolarity::negative, "suppressed", sha, "entry", "allowlist", ""); verify(positives_in(replay_store(dir.string(), o, error)) == 1 && error.empty(), "late observation cannot resurrect removed entry"); // Path/rule adds carry no hash; they still explicitly start a new lifetime. append_label(dir.string(), LabelPolarity::negative, "allowlist-add", "", "entry", "operator", ""); verify(positives_in(replay_store(dir.string(), o, error)) == 1 && error.empty(), "new lifetime has no old hash bindings"); append_label(dir.string(), LabelPolarity::negative, "suppressed", sha, "entry", "allowlist", ""); verify(positives_in(replay_store(dir.string(), o, error)) == 0 && error.empty(), "explicit re-add permits new observations"); fs::remove_all(dir); dir = make_retract_store("audit_provenance", sha, "/tmp/x/dropper", false); append_label(dir.string(), LabelPolarity::positive, "quarantine", sha, "", "operator", ""); append_label(dir.string(), LabelPolarity::retracted, "quarantine-remove", sha, "", "teacher", ""); verify(positives_in(replay_store(dir.string(), o, error)) == 1 && error.empty(), "unrelated source cannot retract operator judgment"); // Display strings are not identities: an independent reason with the // same rendered text must survive an actual quarantine removal. append_label(dir.string(), LabelPolarity::positive, "rules_update", sha, "", "teacher", "operator:quarantine"); append_label(dir.string(), LabelPolarity::retracted, "quarantine-remove", sha, "", "operator", ""); verify(positives_in(replay_store(dir.string(), o, error)) == 1 && error.empty(), "independent same-text positive survives"); fs::remove_all(dir); dir = make_retract_store("audit_bad_label", sha, "/usr/bin/ss", true); { Database db; PROPERTY(db.open(dir.string())); PROPERTY(db.append_evidence(store::RecordType::label, std::string(1, static_cast(0x80)))); } (void) replay_store(dir.string(), o, error); verify(!error.empty(), "malformed labels fail visibly instead of silently joining partial evidence"); fs::remove_all(dir); // A capped entity must refresh its retained positives across time, // rather than freezing the first cap examples forever. std::vector events; for (int i = 0; i < 100; ++i) { const auto ts = kT0 + i * 61 * kSec; const auto node = "process_" + std::to_string(9000 + i) + "_b_1"; events.push_back(exec(ts, 9000 + i, node, "", "/usr/bin/hot", "hot")); events.push_back(scan(ts, 9000 + i, node, {"Teacher"})); } o.max_examples = 50; o.max_entity_share = 0.1; o.sample_seed = 1; const auto capped = run(events, o); verify(positives_in(capped) == 5, "achievable per-entity positive capacity is retained"); verify(std::any_of(capped.examples.begin(), capped.examples.end(), [](const auto& e) { return e.start_ns >= kT0 + 5 * 61 * kSec; }), "capped positives are sampled across time"); // With the positive quota full, an entity whose capacity is occupied // by negatives must still have a chance to supply a teacher positive. events.clear(); for (int i = 0; i < 4; ++i) { events.push_back(exec(kT0 + i * 61 * kSec, 9100 + i, "process_" + std::to_string(9100 + i) + "_b_1", "", "/usr/bin/mixed", "mixed")); } events.push_back(exec(kT0 + 300 * kSec, 9200, "process_9200_b_1", "", "/usr/bin/first", "first")); events.push_back(scan(kT0 + 300 * kSec, 9200, "process_9200_b_1", {"Teacher"})); events.push_back(exec(kT0 + 400 * kSec, 9201, "process_9201_b_1", "", "/usr/bin/mixed", "mixed")); events.push_back(scan(kT0 + 400 * kSec, 9201, "process_9201_b_1", {"Teacher"})); o.max_examples = 8; o.max_entity_share = 0.5; bool selected_mixed = false; for (uint64_t seed = 1; seed <= 16; ++seed) { o.sample_seed = seed; const auto sample = run(events, o); verify(positives_in(sample) == 1 && sample.examples.size() <= 8, "class quota survives cross-entity replacement"); std::unordered_map counts; for (const auto& e : sample.examples) { ++counts[e.entity]; selected_mixed |= e.label > 0.5f && e.entity == "/usr/bin/mixed"; } for (const auto& [entity, count] : counts) verify(count <= 4, "combined cap survives cross-entity replacement"); } verify(selected_mixed, "negative entity capacity cannot starve a positive at full global quota"); // Seeded mutation pressure exercises slot compaction, cross-class changes, // entity eviction/re-entry and both full and partially occupied samples. std::mt19937_64 rng(kSeed); for (std::size_t trial = 0; trial < 48; ++trial) { events.clear(); o.max_examples = 1 + trial % 24; o.max_entity_share = trial % 3 == 0 ? 0.01 : trial % 3 == 1 ? 0.25 : 1.0; o.sample_seed = kSeed + trial; for (int i = 0; i < 72; ++i) { const auto ts = kT0 + i * 61 * kSec; const auto node = "process_" + std::to_string(9300 + i) + "_b_1"; const auto entity = std::string("/usr/bin/entity_") + static_cast('a' + rng() % 7); events.push_back(exec(ts, 9300 + i, node, "", entity, entity)); if (rng() % 3 == 0) events.push_back(scan(ts, 9300 + i, node, {"Teacher"})); } const auto sample = run(events, o); const auto again = run(events, o); const auto cap = std::max(1, static_cast(o.max_entity_share * o.max_examples)); verify(sample.examples.size() <= o.max_examples, "sample bound under mutation pressure"); verify(positives_in(sample) <= std::max(1, o.max_examples / 5), "positive quota under mutation pressure"); verify(sample.example_positives == 0 || positives_in(sample) != 0, "positive cannot be lost to negatives"); PROPERTY(again.examples.size() == sample.examples.size()); std::unordered_map counts; for (std::size_t i = 0; i < sample.examples.size(); ++i) { const auto& e = sample.examples[i]; verify(++counts[e.entity] <= cap, "combined entity cap under mutation pressure"); verify(e.start_ns == again.examples[i].start_ns && e.entity == again.examples[i].entity && e.label == again.examples[i].label, "same seed is deterministic"); } } PROPERTY(failures == 0); std::cout << "R14 parent audit regressions: PASSED" << std::endl; } void no_partial_replay(const ReplayReport& r, const std::string& error, ReplayFailure failure) { PROPERTY(!error.empty() && r.failure == failure); PROPERTY(r.records == 0 && r.windows == 0 && r.examples.empty()); PROPERTY(r.top_windows.empty() && r.detected.empty() && !r.examples_collected && !r.snapshot.checked); PROPERTY(replay_report_json(r).empty() && replay_training_summary_json(r).empty()); } void append_test_tail(const std::filesystem::path& path) { const int fd = io::open(path.c_str(), O_WRONLY | O_APPEND | O_CLOEXEC); PROPERTY(fd >= 0); const char byte = 'x'; PROPERTY(io::write(fd, &byte, 1) == 1); PROPERTY(io::close(fd) == 0); } void test_replay_completeness() { namespace fs = std::filesystem; const std::string sha(64, 'a'); auto o = no_warmup(); o.collect_examples = true; std::string error; for (const auto stream : {store::Stream::evidence, store::Stream::telemetry}) { const auto dir = make_retract_store("completeness_" + std::to_string(static_cast(stream)), sha, "/tmp/private/a", true); append_label(dir.string(), LabelPolarity::positive, "quarantine", sha, "", "operator", "private-rule"); const auto load = store::load_manifest(dir.string()); const auto it = std::find_if(load.manifest.segments.begin(), load.manifest.segments.end(), [&](const auto& seg) { return seg.stream == stream; }); PROPERTY(it != load.manifest.segments.end()); const auto path = dir / (stream == store::Stream::evidence ? "evidence" : "telemetry") / it->file; fs::rename(path, path.string() + ".saved"); const auto scan = store::scan_store_read_only(dir.string(), stream, [](uint32_t, std::string_view) {}); PROPERTY(scan.exists && scan.listed_segments != 0 && scan.skipped_segments == 1); PROPERTY(scan.report.io_error.empty()); // ordinary readers still skip vanished segments const auto missing = replay_store(dir.string(), o, error); no_partial_replay(missing, error, ReplayFailure::unavailable); fs::remove_all(dir); } const auto dir = make_retract_store("sealed_completeness", sha, "/tmp/private/a", true); { store::StoreOptions so; so.dir = dir.string(); store::StoreOpenReport opened; auto log = store::LogStore::open(so, opened, error); PROPERTY(log && log->roll(store::Stream::telemetry)); const auto event = exec(kT0 + 1000 * kSec, 9900, "process_9900_b_1", "", "/usr/bin/new", "new"); PROPERTY(log->append(store::Stream::telemetry, event.type, event.payload)); PROPERTY(log->flush(store::Stream::telemetry, true)); } auto load = store::load_manifest(dir.string()); auto sealed = std::find_if(load.manifest.segments.begin(), load.manifest.segments.end(), [](const auto& seg) { return seg.stream == store::Stream::telemetry && seg.sealed; }); PROPERTY(sealed != load.manifest.segments.end()); const auto path = dir / "telemetry" / sealed->file; // Count mismatches and absent sealed footers invalidate otherwise readable // records. An ordinary active tail is tested separately below. ++sealed->records; ++load.manifest.generation; PROPERTY(store::publish_manifest(dir.string(), load.manifest, error)); no_partial_replay(replay_store(dir.string(), o, error), error, ReplayFailure::invalid); --sealed->records; ++load.manifest.generation; PROPERTY(store::publish_manifest(dir.string(), load.manifest, error)); fs::resize_file(path, fs::file_size(path) - 1); no_partial_replay(replay_store(dir.string(), o, error), error, ReplayFailure::invalid); // Best-effort readers continue delivering a valid later segment even when // the earlier manifest-listed segment cannot be opened. fs::rename(path, path.string() + ".saved"); uint64_t delivered = 0; const auto partial = store::scan_store_read_only(dir.string(), store::Stream::telemetry, [&](uint32_t, std::string_view) { ++delivered; }); PROPERTY(delivered != 0 && partial.skipped_segments == 1 && partial.report.io_error.empty()); no_partial_replay(replay_store(dir.string(), o, error), error, ReplayFailure::unavailable); fs::remove_all(dir); std::cout << "R16 incomplete inputs export no partial replay: PASSED" << std::endl; } void test_snapshot_summary() { namespace fs = std::filesystem; const std::string sha(64, 'b'); std::string error; // Fixed-bucket observations must not alter resolution, seeded sampling or // leak host-controlled strings. Include zero/extreme clocks and overlap. auto dir = make_retract_store("summary_labels", sha, "/tmp/host-secret/program", true); append_label(dir.string(), LabelPolarity::positive, "quarantine", sha, "", "operator", "host-secret-rule"); append_label(dir.string(), LabelPolarity::negative, "host-secret-action", sha, "", "host-secret-source", ""); append_label(dir.string(), LabelPolarity::negative, "allowlist-add", "", "", "operator", "", 0); append_label(dir.string(), LabelPolarity::retracted, "allowlist-remove", "", "", "host-secret-source", "", INT64_MIN); append_label(dir.string(), LabelPolarity::positive, "host-secret-action", sha, "", "host-secret-source", "", INT64_MAX); auto o = no_warmup(); o.collect_examples = true; o.sample_seed = kSeed; const auto ordinary = replay_store(dir.string(), o, error); PROPERTY(error.empty()); o.training_summary = true; const auto summary = replay_store(dir.string(), o, error); PROPERTY(error.empty() && summary.snapshot.checked && summary.examples_collected); PROPERTY(summary.examples.size() == ordinary.examples.size()); for (std::size_t i = 0; i < summary.examples.size(); ++i) { PROPERTY(summary.examples[i].x == ordinary.examples[i].x && summary.examples[i].label == ordinary.examples[i].label); PROPERTY(summary.examples[i].start_ns == ordinary.examples[i].start_ns && summary.examples[i].entity == ordinary.examples[i].entity); } PROPERTY(summary.labels.records == 5 && summary.labels.missing_hash == 2 && summary.labels.idless_allowlist == 2); PROPERTY((summary.labels.polarity == std::array{2, 2, 1})); PROPERTY((summary.labels.provenance == std::array{2, 0, 3})); PROPERTY(summary.labels.timestamp_known == 3 && summary.labels.timestamp_unknown == 2); PROPERTY(summary.labels.last_ns == INT64_MAX && summary.labels.resolved_positive_hashes == 1); PROPERTY(summary.labels.resolved_negative_hashes == 1 && summary.labels.overlap_hashes == 1); const auto json = replay_training_summary_json(summary); PROPERTY(json.find("host-secret") == std::string::npos && json.find(sha) == std::string::npos); PROPERTY(json == replay_training_summary_json(replay_store(dir.string(), o, error)) && error.empty()); fs::remove_all(dir); for (const bool change_manifest : {false, true}) { dir = make_retract_store(change_manifest ? "snapshot_generation" : "snapshot_extent", sha, "/tmp/private/a", false); auto load = store::load_manifest(dir.string()); const auto telemetry = std::find_if(load.manifest.segments.begin(), load.manifest.segments.end(), [](const auto& seg) { return seg.stream == store::Stream::telemetry; }); PROPERTY(telemetry != load.manifest.segments.end()); const auto path = dir / "telemetry" / telemetry->file; bool changed = false; // A trusted in-process observation seam, never exposed by the CLI, // gives deterministic extent/generation changes without a racing writer. o.before_snapshot_recheck = [&] { if (changed) return; changed = true; if (change_manifest) { ++load.manifest.generation; PROPERTY(store::publish_manifest(dir.string(), load.manifest, error)); } else append_test_tail(path); }; const auto changed_report = replay_store(dir.string(), o, error); PROPERTY(changed); no_partial_replay(changed_report, error, ReplayFailure::changed); fs::remove_all(dir); } o.before_snapshot_recheck = [] { throw 42; }; dir = make_retract_store("snapshot_observer_fault", sha, "/tmp/private/a", false); no_partial_replay(replay_store(dir.string(), o, error), error, ReplayFailure::unavailable); fs::remove_all(dir); o.before_snapshot_recheck = {}; dir = make_retract_store("active_tail_summary", sha, "/tmp/private/a", false); const auto load = store::load_manifest(dir.string()); const auto telemetry = std::find_if(load.manifest.segments.begin(), load.manifest.segments.end(), [](const auto& seg) { return seg.stream == store::Stream::telemetry; }); PROPERTY(telemetry != load.manifest.segments.end()); append_test_tail(dir / "telemetry" / telemetry->file); o.training_summary = false; const auto active = replay_store(dir.string(), o, error); PROPERTY(error.empty() && active.records != 0); o.training_summary = true; no_partial_replay(replay_store(dir.string(), o, error), error, ReplayFailure::unavailable); fs::remove_all(dir); dir = make_retract_store("fragment_summary", sha, "/tmp/private/a", false); for (const char* name : {"MANIFEST", "MANIFEST.prev", "MANIFEST.tmp"}) { const auto path = dir / name; if (fs::exists(path)) fs::rename(path, path.string() + ".saved"); } o.training_summary = false; const auto fragment = replay_store(dir.string(), o, error); PROPERTY(error.empty() && fragment.records != 0); o.training_summary = true; no_partial_replay(replay_store(dir.string(), o, error), error, ReplayFailure::unavailable); fs::remove_all(dir); std::cout << "R17 aggregate labels and strict cooperative snapshots: PASSED" << std::endl; } std::string read_whole_file(const std::filesystem::path& path) { const int fd = io::open(path.c_str(), O_RDONLY | O_CLOEXEC); PROPERTY(fd >= 0); struct stat st {}; PROPERTY(io::fstat(fd, &st) == 0); std::string out(static_cast(st.st_size), '\0'); std::size_t got = 0; while (got < out.size()) { const ssize_t n = io::read(fd, out.data() + got, out.size() - got); PROPERTY(n > 0); got += static_cast(n); } PROPERTY(io::close(fd) == 0); return out; } void write_whole_file(const std::filesystem::path& path, std::string_view bytes) { const int fd = io::open(path.c_str(), O_WRONLY | O_TRUNC | O_CLOEXEC); PROPERTY(fd >= 0); std::size_t done = 0; while (done < bytes.size()) { const ssize_t n = io::write(fd, bytes.data() + done, bytes.size() - done); PROPERTY(n > 0); done += static_cast(n); } PROPERTY(io::close(fd) == 0); } uint32_t block_le32(std::string_view bytes, std::size_t at) { uint32_t value = 0; for (int i = 0; i < 4; ++i) value |= static_cast(static_cast(bytes[at + i])) << (8 * i); return value; } void block_put_le32(std::string& bytes, std::size_t at, uint32_t value) { for (int i = 0; i < 4; ++i) bytes[at + i] = static_cast((value >> (8 * i)) & 0xFFu); } // The first block starts right after the segment header. constexpr std::size_t kFirstBlock = store::kSegmentHeaderBytes; /// Makes the first block of `file` need a dictionary this build does not have /// (the header stays self-consistent, so it is not corruption). void require_missing_dictionary(std::string& file) { PROPERTY(file.size() > kFirstBlock + store::kBlockHeaderBytes); PROPERTY(block_le32(file, kFirstBlock) == store::kBlockMagic); PROPERTY(file[kFirstBlock + 1 + 4] == static_cast(store::BlockCodec::zstd)); block_put_le32(file, kFirstBlock + 16, 0x5EEDu); block_put_le32(file, kFirstBlock + 24, store::crc32c(file.data() + kFirstBlock, 24)); } /// Corrupts the magic of the first block: a complete header that is not valid. void break_block_header(std::string& file) { PROPERTY(file.size() > kFirstBlock + store::kBlockHeaderBytes); file[kFirstBlock] = static_cast(file[kFirstBlock] ^ 0x80); } std::filesystem::path stream_segment(const std::string& dir, const std::string& name, bool sealed) { const auto load = store::load_manifest(dir); const auto stream = name == "evidence" ? store::Stream::evidence : store::Stream::telemetry; for (const auto& seg : load.manifest.segments) { if (seg.stream == stream && seg.sealed == sealed) return std::filesystem::path(dir) / name / seg.file; } return {}; } void test_decoder_availability() { namespace fs = std::filesystem; const std::string sha(64, 'd'); auto o = no_warmup(); o.collect_examples = true; std::string error; // A block needing a dictionary this build lacks is a missing decoder, not // an empty stream and not corruption (ordinary replay and summary alike). { const auto dir = make_retract_store("decoder_active", sha, "/tmp/private/a", true); const auto path = stream_segment(dir.string(), "telemetry", false); PROPERTY(!path.empty()); auto file = read_whole_file(path); require_missing_dictionary(file); write_whole_file(path, file); const auto scan = store::scan_store_read_only(dir.string(), store::Stream::telemetry, [](uint32_t, std::string_view) {}); PROPERTY(scan.unavailable_segments == 1 && scan.malformed_tail_segments == 0); PROPERTY(scan.invalid_segments == 0 && scan.report.torn_tail == false); no_partial_replay(replay_store(dir.string(), o, error), error, ReplayFailure::unavailable); o.training_summary = true; no_partial_replay(replay_store(dir.string(), o, error), error, ReplayFailure::unavailable); o.training_summary = false; fs::remove_all(dir); } // The same block in a sealed segment must not be misread as bad data: the // footer/manifest counts it explains are not comparable without the block. { const auto dir = make_retract_store("decoder_sealed", sha, "/tmp/private/a", true); { store::StoreOptions so; so.dir = dir.string(); store::StoreOpenReport opened; auto log = store::LogStore::open(so, opened, error); PROPERTY(log && log->roll(store::Stream::telemetry)); const auto event = exec(kT0 + 1000 * kSec, 9900, "process_9900_b_1", "", "/usr/bin/new", "new"); PROPERTY(log->append(store::Stream::telemetry, event.type, event.payload)); PROPERTY(log->flush(store::Stream::telemetry, true)); } const auto path = stream_segment(dir.string(), "telemetry", true); PROPERTY(!path.empty()); auto file = read_whole_file(path); require_missing_dictionary(file); write_whole_file(path, file); const auto scan = store::scan_store_read_only(dir.string(), store::Stream::telemetry, [](uint32_t, std::string_view) {}); PROPERTY(scan.unavailable_segments == 1 && scan.malformed_tail_segments == 0); PROPERTY(scan.invalid_segments == 0); // counts are not checkable here no_partial_replay(replay_store(dir.string(), o, error), error, ReplayFailure::unavailable); fs::remove_all(dir); } // A complete block header that fails validation is corrupt data: unlike a // short in-flight tail it is not tolerated by ordinary replay. { const auto dir = make_retract_store("decoder_badheader", sha, "/tmp/private/a", true); const auto path = stream_segment(dir.string(), "telemetry", false); PROPERTY(!path.empty()); auto file = read_whole_file(path); break_block_header(file); write_whole_file(path, file); const auto scan = store::scan_store_read_only(dir.string(), store::Stream::telemetry, [](uint32_t, std::string_view) {}); PROPERTY(scan.malformed_tail_segments == 1 && scan.unavailable_segments == 0); no_partial_replay(replay_store(dir.string(), o, error), error, ReplayFailure::invalid); o.training_summary = true; no_partial_replay(replay_store(dir.string(), o, error), error, ReplayFailure::invalid); o.training_summary = false; fs::remove_all(dir); } // An earlier unusable segment must not be masked by a later short tail // (the aggregate tail_error keeps only the last one). for (const bool missing_dictionary : {false, true}) { const auto dir = make_retract_store(missing_dictionary ? "masked_decoder" : "masked_header", sha, "/tmp/private/a", true); { store::StoreOptions so; so.dir = dir.string(); store::StoreOpenReport opened; auto log = store::LogStore::open(so, opened, error); PROPERTY(log && log->roll(store::Stream::telemetry)); const auto event = exec(kT0 + 1000 * kSec, 9900, "process_9900_b_1", "", "/usr/bin/new", "new"); PROPERTY(log->append(store::Stream::telemetry, event.type, event.payload)); PROPERTY(log->flush(store::Stream::telemetry, true)); } const auto sealed = stream_segment(dir.string(), "telemetry", true); const auto active = stream_segment(dir.string(), "telemetry", false); PROPERTY(!sealed.empty() && !active.empty()); auto file = read_whole_file(sealed); if (missing_dictionary) require_missing_dictionary(file); else break_block_header(file); write_whole_file(sealed, file); append_test_tail(active); // the later, tolerated short tail const auto scan = store::scan_store_read_only(dir.string(), store::Stream::telemetry, [](uint32_t, std::string_view) {}); PROPERTY(scan.report.tail_error == store::BlockError::truncated); if (missing_dictionary) { PROPERTY(scan.unavailable_segments == 1 && scan.malformed_tail_segments == 0); no_partial_replay(replay_store(dir.string(), o, error), error, ReplayFailure::unavailable); } else { no_partial_replay(replay_store(dir.string(), o, error), error, ReplayFailure::invalid); } fs::remove_all(dir); } // Positive control: a genuine in-flight tail stays readable to ordinary // replay (and is still refused for a summary snapshot). { const auto dir = make_retract_store("decoder_inflight", sha, "/tmp/private/a", true); const auto path = stream_segment(dir.string(), "telemetry", false); PROPERTY(!path.empty()); append_test_tail(path); const auto scan = store::scan_store_read_only(dir.string(), store::Stream::telemetry, [](uint32_t, std::string_view) {}); PROPERTY(scan.malformed_tail_segments == 0 && scan.unavailable_segments == 0); PROPERTY(scan.report.torn_tail && scan.report.tail_error == store::BlockError::truncated); const auto ordinary = replay_store(dir.string(), o, error); PROPERTY(error.empty() && ordinary.records != 0 && ordinary.failure == ReplayFailure::none); o.training_summary = true; no_partial_replay(replay_store(dir.string(), o, error), error, ReplayFailure::unavailable); o.training_summary = false; fs::remove_all(dir); } std::cout << "R18 missing decoder and malformed tails are not empty inputs: PASSED" << std::endl; } // Stage 7a: LogGC must not change what replay detects. Replays a full store and the same store after // reduce_telemetry(): dead benign subtrees are dropped (fewer execs), detections are identical. void test_reduce_replay_equivalence() { namespace fs = std::filesystem; constexpr int64_t kDayNs = 86'400 * kSec; auto ev = routine(20, kT0); ev.push_back(exec(kT0 + 40 * 61 * kSec, 700, "process_700_b_1", "process_1_b_1", "/tmp/evil", "evil")); ev.push_back(scan(kT0 + 40 * 61 * kSec + kSec, 700, "process_700_b_1", {"Evil_Rule"})); for (int i = 0; i < 3; ++i) { // live (distinct nodes, days later): sets the horizon const auto ts = kT0 + 5 * kDayNs + i * 61 * kSec; const std::string sh = "process_" + std::to_string(5000 + i) + "_b_1"; ev.push_back(exec(ts, 5000 + i, sh, "process_1_b_1", "/usr/bin/zsh", "zsh")); ev.push_back(exec(ts + kSec, 6000 + i, "process_" + std::to_string(6000 + i) + "_b_1", sh, "/usr/bin/cat", "cat x")); } const auto root = fs::temp_directory_path() / ("strawberry_reduce_replay_" + std::to_string(::getpid())); std::error_code ec; fs::remove_all(root, ec); fs::create_directories(root, ec); const std::string full_dir = (root / "full").string(); const std::string reduced_dir = (root / "reduced").string(); const std::string skeleton_dir = (root / "skeleton").string(); const std::string merge_dir = (root / "merge").string(); const std::string cpr_dir = (root / "cpr").string(); for (const auto& dir : {full_dir, reduced_dir, skeleton_dir, merge_dir, cpr_dir}) { Database db; PROPERTY(db.open(dir)); auto copy = ev; PROPERTY(db.append_telemetry(copy, true)); db.close(); } store::ReduceReport reduce; { store::StoreOpenReport open_report; std::string error; auto log = store::LogStore::open(store::default_store_options(reduced_dir), open_report, error); PROPERTY(log != nullptr); PROPERTY(log->roll(store::Stream::telemetry)); // seal: only sealed telemetry is reduced PROPERTY(log->reduce_telemetry(1, reduce)); } PROPERTY(reduce.problems.empty() && reduce.subtrees_dropped > 0 && reduce.records_after < reduce.records_before); store::ReduceReport skeleton; // Stage 7b: also skeletonize dead benign nodes (exec kept, leaf telemetry dropped) { store::StoreOpenReport open_report; std::string error; auto log = store::LogStore::open(store::default_store_options(skeleton_dir), open_report, error); PROPERTY(log != nullptr); PROPERTY(log->roll(store::Stream::telemetry)); store::ReduceOptions skeleton_options; skeleton_options.seed = 1; skeleton_options.skeletonize_dead_benign = true; PROPERTY(log->reduce_telemetry(skeleton_options, skeleton)); } PROPERTY(skeleton.problems.empty() && skeleton.records_after <= reduce.records_after); store::ReduceReport merged; // Stage 7c: also merge old repeated benign leaf accesses (7a path otherwise) { store::StoreOpenReport open_report; std::string merge_error; auto log = store::LogStore::open(store::default_store_options(merge_dir), open_report, merge_error); PROPERTY(log != nullptr); PROPERTY(log->roll(store::Stream::telemetry)); store::ReduceOptions merge_options; merge_options.seed = 1; merge_options.merge_redundant = true; PROPERTY(log->reduce_telemetry(merge_options, merged)); } PROPERTY(merged.problems.empty() && merged.records_after <= reduce.records_after); store::ReduceReport cpr_merged; // Stage 7d: CPR merge with a 1 h horizon { store::StoreOpenReport open_report; std::string cpr_error; auto log = store::LogStore::open(store::default_store_options(cpr_dir), open_report, cpr_error); PROPERTY(log != nullptr); PROPERTY(log->roll(store::Stream::telemetry)); store::ReduceOptions cpr_options; cpr_options.seed = 1; cpr_options.merge_redundant = true; cpr_options.merge_mode = store::MergeMode::cpr; cpr_options.merge_horizon_ns = 3'600ll * 1'000'000'000ll; PROPERTY(log->reduce_telemetry(cpr_options, cpr_merged)); } PROPERTY(cpr_merged.problems.empty() && cpr_merged.records_after <= reduce.records_after); std::string error; const auto full = replay_store(full_dir, no_warmup(), error); PROPERTY(error.empty()); const auto reduced = replay_store(reduced_dir, no_warmup(), error); PROPERTY(error.empty()); PROPERTY(reduced.execs < full.execs); // the compute proxy went down PROPERTY(reduced.detected_windows == full.detected_windows && full.detected_windows == 1); PROPERTY(reduced.late_detected_windows == full.late_detected_windows); PROPERTY(reduced.detected.size() == full.detected.size()); for (std::size_t i = 0; i < full.detected.size(); ++i) { PROPERTY(reduced.detected[i].root_node == full.detected[i].root_node); } const auto skeletonized = replay_store(skeleton_dir, no_warmup(), error); PROPERTY(error.empty()); PROPERTY(skeletonized.execs == reduced.execs); // skeletonization never drops an exec PROPERTY(skeletonized.detected_windows == full.detected_windows); PROPERTY(skeletonized.late_detected_windows == full.late_detected_windows); PROPERTY(skeletonized.detected.size() == full.detected.size()); for (std::size_t i = 0; i < full.detected.size(); ++i) { PROPERTY(skeletonized.detected[i].root_node == full.detected[i].root_node); } // 7c recall gate: merge touches only old benign far-from-label repeats, so detections are unchanged. const auto merged_replay = replay_store(merge_dir, no_warmup(), error); PROPERTY(error.empty()); PROPERTY(merged_replay.execs == reduced.execs); // merge never drops an exec PROPERTY(merged_replay.detected_windows == full.detected_windows); PROPERTY(merged_replay.late_detected_windows == full.late_detected_windows); PROPERTY(merged_replay.detected.size() == full.detected.size()); for (std::size_t i = 0; i < full.detected.size(); ++i) { PROPERTY(merged_replay.detected[i].root_node == full.detected[i].root_node); } const auto cpr_replay = replay_store(cpr_dir, no_warmup(), error); PROPERTY(error.empty()); PROPERTY(cpr_replay.execs == reduced.execs); PROPERTY(cpr_replay.detected_windows == full.detected_windows); PROPERTY(cpr_replay.late_detected_windows == full.late_detected_windows); PROPERTY(cpr_replay.detected.size() == full.detected.size()); for (std::size_t i = 0; i < full.detected.size(); ++i) { PROPERTY(cpr_replay.detected[i].root_node == full.detected[i].root_node); } fs::remove_all(root, ec); std::cout << "R-7a LogGC replay equivalence (+7b skeleton, +7c merge, +7d cpr): PASSED" << std::endl; } // R11c evaluation bound (--eval-after): a bounded replay is out-of-sample and // can pass the gate; the same run unbounded is in-sample and refused; a bound // past the end of the history fails explicitly instead of emitting a silent // 0-window cell; `auto` derives the bound from the models' training range. // Without the bound these assertions cannot hold: the bounded cells would be // in-sample (their windows overlap the training range) and the gate refuses. void test_eval_bound() { auto ev = routine(240, kT0); std::vector traces; std::string error; PROPERTY(compile_traces( "trace odd\n+0s exec a /tmp/q/odd \"odd --x\"\n+1s connect a 198.51.100.4 4444\n", traces, error)); namespace fs = std::filesystem; const auto dir = fs::temp_directory_path() / ("strawberry_eval_bound_" + std::to_string(::getpid())); std::error_code ec; fs::remove_all(dir, ec); { Database db; PROPERTY(db.open(dir.string())); auto copy = ev; PROPERTY(db.append_telemetry(copy, true)); db.close(); } // Training ranges are [first window start, last window END]. auto whole = std::make_shared(Model::initialized(ModelShape{}, 11)); whole->training_first_ns = kT0; whole->training_last_ns = kT0 + 240 * 61 * kSec; // covers the whole store auto earlier = std::make_shared(Model::initialized(ModelShape{}, 12)); earlier->training_first_ns = kT0; earlier->training_last_ns = kT0 + 120 * 61 * kSec; // the first half const auto healthy = [](const PolicyEvaluation::Provenance& p) { PolicyEvaluation e; e.extra_positives = 12; e.policies = {{"model", 10, 20.0, 9, 9.0, 0}, {"rarity_greedy", 20, 40.0, 8, 8.0, 0}}; e.provenance = p; return promotion_gate(e); }; ReplayOptions o; o.warmup_ns = 600 * kSec; o.policy_eval = true; o.policy.budget_seconds_per_hour = 4; o.inject = traces; o.inject_copies = 9; o.eval_models = {{"whole", whole}}; o.eval_periods = 2; // Unbounded: the ranked windows are inside the training range, every cell is // in-sample, and the gate refuses it (today's "always refuses" behaviour). const auto unbounded = replay_store(dir.string(), o, error); PROPERTY(error.empty()); PROPERTY(unbounded.policy_eval.cells.size() == 2); for (const auto& c : unbounded.policy_eval.cells) { PROPERTY(c.provenance.training_range_known && c.provenance.in_sample); PROPERTY(!healthy(c.provenance).passed); PROPERTY(healthy(c.provenance).checks[4].find("in-sample evaluation") != std::string::npos); } // Bounded at the model's training end the same history is out-of-sample, so // the identical evidence can pass the pre-registered gate. o.eval_models = {{"earlier", earlier}}; o.eval_after_ns = earlier->training_last_ns; const auto bounded = replay_store(dir.string(), o, error); PROPERTY(error.empty() && bounded.policy_eval.cells.size() == 2); uint64_t windows = 0, positives = 0; for (const auto& c : bounded.policy_eval.cells) { windows += c.windows; positives += c.extra_positives; PROPERTY(c.provenance.training_range_known && !c.provenance.in_sample); PROPERTY(c.provenance.eval_first_ns >= o.eval_after_ns); PROPERTY(healthy(c.provenance).passed); } // Periods partition only the out-of-sample span: the routine windows precede // the bound, so what is left is the injected evaluation traces. PROPERTY(windows == bounded.policy_eval.windows && windows > 0 && positives > 0); // The single-model policy_eval provenance uses the same bound, so `model // promote` is judged out-of-sample instead of always refusing. ReplayOptions single; single.warmup_ns = 600 * kSec; single.policy_eval = true; single.policy.budget_seconds_per_hour = 4; single.inject = traces; single.inject_copies = 9; single.model = earlier; single.model_name = "earlier"; single.eval_after_ns = earlier->training_last_ns; const auto promoted = replay_store(dir.string(), single, error); PROPERTY(error.empty()); PROPERTY(promoted.policy_eval.provenance.training_range_known); PROPERTY(!promoted.policy_eval.provenance.in_sample); PROPERTY(promoted.policy_eval.provenance.eval_first_ns >= single.eval_after_ns); single.eval_after_ns = 0; // unbounded: in-sample again const auto refused = replay_store(dir.string(), single, error); PROPERTY(error.empty() && refused.policy_eval.provenance.in_sample); // A bound past the end of the history: the run says why it has no evidence. ReplayOptions late = o; late.eval_after_ns = kT0 + 500 * 61 * kSec; replay_store(dir.string(), late, error); PROPERTY(!error.empty()); PROPERTY(error.find("no out-of-sample history yet") != std::string::npos); PROPERTY(error.find("store ends") != std::string::npos); // Without injected traces the report still carries an explicit cell failure. late.inject.clear(); const auto empty = replay_store(dir.string(), late, error); PROPERTY(error.empty()); PROPERTY(empty.policy_eval.cells.size() == 2); for (const auto& c : empty.policy_eval.cells) { PROPERTY(c.windows == 0 && c.extra_positives == 0); PROPERTY(c.provenance.eval_error.find("no out-of-sample history yet") != std::string::npos); const auto g = healthy(c.provenance); PROPERTY(!g.passed && g.checks[4].find("no out-of-sample history yet") != std::string::npos); } // `auto` = the latest training-range end over the loaded eval models. PROPERTY(eval_after_from_models({}) == 0); PROPERTY(eval_after_from_models({{"earlier", earlier}}) == earlier->training_last_ns); PROPERTY(eval_after_from_models({{"earlier", earlier}, {"none", nullptr}}) == earlier->training_last_ns); PROPERTY(eval_after_from_models({{"earlier", earlier}, {"whole", whole}}) == whole->training_last_ns); auto unknown = std::make_shared(Model::initialized(ModelShape{}, 13)); PROPERTY(eval_after_from_models({{"unknown", unknown}}) == 0); // The derived bound is what the CLI hands to replay: still out-of-sample. o.eval_after_ns = eval_after_from_models(o.eval_models); PROPERTY(o.eval_after_ns == earlier->training_last_ns); const auto auto_bounded = replay_store(dir.string(), o, error); PROPERTY(error.empty() && !auto_bounded.policy_eval.cells.empty()); for (const auto& c : auto_bounded.policy_eval.cells) PROPERTY(!c.provenance.in_sample); // The cut cells carry the reason in the report JSON too (not just in memory), // so a pooled fleet gate can see why a node contributed no out-of-sample run. const auto empty_json = replay_report_json(empty); PROPERTY(empty_json.find("no out-of-sample history yet") != std::string::npos); const auto json = replay_report_json(auto_bounded); PROPERTY(json.find("\"eval_error\": \"\"") != std::string::npos); fs::remove_all(dir, ec); std::cout << "R11c evaluation bound: PASSED (" << windows << " out-of-sample windows, bound " << (o.eval_after_ns - kT0) / kSec << " s after the store start)" << std::endl; } int main() { std::cout << "replay properties, seed " << kSeed << std::endl; test_reduce_replay_equivalence(); test_sampler_capacity_lifetime(); test_parent_audit_regressions(); test_replay_completeness(); test_snapshot_summary(); test_decoder_availability(); test_chain_and_pid_reuse(); test_order_and_determinism(); test_ranking_and_detection(); test_store_round_trip(); test_invalid_utf8(); test_warmup_and_time_invariance(); test_injection_evaluation(); test_late_labels(); test_attribution(); test_allowlisted_window_is_negative(); test_retraction_allowlist_remove(); test_retraction_quarantine_remove(); test_retraction_independent_positives(); test_retraction_multiple_entries(); test_retraction_readd(); test_examples_and_model_ranking(); test_positive_sampling_under_pressure(); test_positive_entity_cap(); test_positive_not_starved_by_entity_negatives(); test_policy_evaluation(); test_gate_cells(); test_eval_bound(); test_extreme_timestamps(); return 0; }