1586 lines
78 KiB
C++
1586 lines
78 KiB
C++
// Replay: windows, joins and scoring over telemetry records. A failure
|
|
// prints the seed; re-run with STRAWBERRY_PROPERTY_SEED=<seed>.
|
|
//
|
|
// R1 deterministic: the same records give the same report
|
|
// R2 order within the reorder horizon does not matter
|
|
// R3 a process chain within 60 s is one window; its behavior joins it
|
|
// C3 PID reuse: behavior joins the exec of that pid at its time
|
|
// R4 an unusual chain ranks above routine ones; detections mark windows
|
|
// R5 replay_store reads a real store directory
|
|
// R7 warm-up windows are learned from but not ranked; a never-seen event
|
|
// scores the same early and late; identical patterns are grouped
|
|
// R8 injected traces: every copy is found and scored; injecting changes
|
|
// nothing about real windows (scored, never learned from); a novel
|
|
// attack chain beats routine activity at a 10/day budget
|
|
// R9 late labels (a binary found bad after it ran) mark the windows that
|
|
// ran it, read from the evidence stream by replay_store
|
|
// R10 training examples: one per ranked window, labelled by detections;
|
|
// training traces become positives (not evaluated); per-program cap;
|
|
// ranking by a model sets each window's score from its value
|
|
// R11 policy evaluation: extra positives exclude rule-flagged windows;
|
|
// rarity-greedy inspects within budget; logged shadow decisions are
|
|
// joined to their windows; the promotion gate checks each criterion
|
|
// R6 paths that are not UTF-8 (any byte but '/' and NUL is legal) give
|
|
// valid JSON, in the report and in json_escape
|
|
// R12 label retraction: allowlist-remove revokes the named entry's negative
|
|
// (detected windows revert to rule-detected positives); quarantine-remove
|
|
// revokes only the quarantine positive, preserving independent positives;
|
|
// multiple entries and re-adds resolve in stored order
|
|
// R13 positive entity cap: max_entity_share caps the combined (positive +
|
|
// negative) per-program count; a late positive displaces a negative of
|
|
// the same entity rather than being starved; concentration rejections
|
|
// are counted, not silently dropped
|
|
|
|
#include "strawberry/database.hpp"
|
|
#include "strawberry/learn/replay.hpp"
|
|
#include "strawberry/learn/inspection.hpp"
|
|
#include "strawberry/learn/trace.hpp"
|
|
#include "strawberry/telemetry.hpp"
|
|
#include "strawberry/store/log_store.hpp"
|
|
#include "strawberry/sys_io.hpp"
|
|
#include "../src/json_util.hpp"
|
|
|
|
#include <nlohmann/json.hpp>
|
|
|
|
#include <algorithm>
|
|
#include <cmath>
|
|
#include <memory>
|
|
#include <cstdlib>
|
|
#include <filesystem>
|
|
#include <iostream>
|
|
#include <random>
|
|
#include <string>
|
|
#include <unordered_map>
|
|
#include <vector>
|
|
|
|
#include <fcntl.h>
|
|
#include <unistd.h>
|
|
|
|
using namespace strawberry;
|
|
using namespace strawberry::learn;
|
|
|
|
namespace {
|
|
|
|
uint64_t seed_from_env() {
|
|
if (const char* text = std::getenv("STRAWBERRY_PROPERTY_SEED")) return std::strtoull(text, nullptr, 10);
|
|
return std::random_device{}();
|
|
}
|
|
const uint64_t kSeed = seed_from_env();
|
|
|
|
#define PROPERTY(cond) \
|
|
do { \
|
|
if (!(cond)) { \
|
|
std::cerr << "PROPERTY FAILED: " #cond " (seed " << kSeed << ", " << __FILE__ << ":" \
|
|
<< __LINE__ << ")" << std::endl; \
|
|
std::abort(); \
|
|
} \
|
|
} while (0)
|
|
|
|
constexpr int64_t kSec = 1'000'000'000;
|
|
constexpr int64_t kT0 = 1'800'000'000 * kSec;
|
|
|
|
TelemetryEvent exec(int64_t ts, int pid, const std::string& node, const std::string& parent, const std::string& exe,
|
|
const std::string& cmdline) {
|
|
ProcessExecRecord r;
|
|
r.timestamp_ns = ts;
|
|
r.pid = pid;
|
|
r.process_node_id = node;
|
|
r.parent_node_id = parent;
|
|
r.exe = exe;
|
|
r.cmdline = cmdline;
|
|
r.source = "live";
|
|
return {store::RecordType::process_exec, encode_process_exec(r)};
|
|
}
|
|
|
|
TelemetryEvent connect(int64_t ts, int pid, const std::string& address, int port) {
|
|
BehaviorEvent b;
|
|
b.type = BehaviorEvent::Type::connect;
|
|
b.timestamp_ns = static_cast<uint64_t>(ts);
|
|
b.pid = pid;
|
|
b.family = 2;
|
|
b.address = address;
|
|
b.port = static_cast<uint16_t>(port);
|
|
return {store::RecordType::connect, encode_behavior(b)};
|
|
}
|
|
|
|
TelemetryEvent scan(int64_t ts, int pid, const std::string& node, std::vector<std::string> rules) {
|
|
ScanResultRecord s;
|
|
s.timestamp_ns = ts;
|
|
s.pid = pid;
|
|
s.process_node_id = node;
|
|
s.rules = std::move(rules);
|
|
return {store::RecordType::scan_result, encode_scan_result(s)};
|
|
}
|
|
|
|
TelemetryEvent write(int64_t ts, int pid, const std::string& path) {
|
|
FileWriteEvent w;
|
|
w.pid = pid;
|
|
w.path = path;
|
|
w.category = "drop";
|
|
w.timestamp_ns = static_cast<uint64_t>(ts);
|
|
return {store::RecordType::file_write, encode_file_write(w)};
|
|
}
|
|
|
|
ReplayOptions no_warmup() {
|
|
ReplayOptions o;
|
|
o.warmup_ns = 0;
|
|
return o;
|
|
}
|
|
|
|
ReplayReport run(const std::vector<TelemetryEvent>& events, ReplayOptions options = no_warmup()) {
|
|
Replayer r(options);
|
|
for (const auto& e : events) r.feed(e.type, e.payload);
|
|
return r.finish();
|
|
}
|
|
|
|
const ReplayWindow* window_rooted(const ReplayReport& report, const std::string& node) {
|
|
for (const auto& w : report.top_windows) {
|
|
if (w.root_node == node) return &w;
|
|
}
|
|
return nullptr;
|
|
}
|
|
|
|
// A routine background: many bash -> ls windows, a minute apart.
|
|
std::vector<TelemetryEvent> routine(int count, int64_t start) {
|
|
std::vector<TelemetryEvent> out;
|
|
for (int i = 0; i < count; ++i) {
|
|
const auto ts = start + i * 61 * kSec;
|
|
const std::string sh = "process_" + std::to_string(1000 + i) + "_b_1";
|
|
out.push_back(exec(ts, 1000 + i, sh, "process_1_b_1", "/usr/bin/bash", "bash"));
|
|
out.push_back(exec(ts + kSec, 20000 + i, "process_" + std::to_string(20000 + i) + "_b_1", sh, "/usr/bin/ls",
|
|
"ls -la /home/a"));
|
|
}
|
|
return out;
|
|
}
|
|
|
|
void test_chain_and_pid_reuse() {
|
|
std::vector<TelemetryEvent> ev;
|
|
// firefox -> sh -> curl, curl connects: one window.
|
|
ev.push_back(exec(kT0, 100, "process_100_b_1", "process_1_b_1", "/usr/lib/firefox/firefox", "firefox"));
|
|
ev.push_back(exec(kT0 + 2 * kSec, 101, "process_101_b_1", "process_100_b_1", "/usr/bin/sh", "sh -c x"));
|
|
ev.push_back(exec(kT0 + 3 * kSec, 102, "process_102_b_1", "process_101_b_1", "/usr/bin/curl",
|
|
"curl -fsSL https://x/a.sh"));
|
|
ev.push_back(connect(kT0 + 4 * kSec, 102, "203.0.113.9", 443));
|
|
// PID 500 reused: two different programs, each with a connection.
|
|
ev.push_back(exec(kT0 + 10 * kSec, 500, "process_500_b_10", "process_1_b_1", "/usr/bin/a", "a"));
|
|
ev.push_back(connect(kT0 + 11 * kSec, 500, "10.0.0.1", 22));
|
|
ev.push_back(exec(kT0 + 200 * kSec, 500, "process_500_b_20", "process_1_b_1", "/usr/bin/b", "b"));
|
|
ev.push_back(connect(kT0 + 201 * kSec, 500, "10.0.0.2", 80));
|
|
ev.push_back(connect(kT0 + 202 * kSec, 777, "10.0.0.3", 80)); // no exec for 777
|
|
const auto report = run(ev);
|
|
PROPERTY(report.execs == 5 && report.behavior_events == 4 && report.unjoined_events == 1);
|
|
PROPERTY(report.windows == 3);
|
|
const auto* chain = window_rooted(report, "process_100_b_1");
|
|
PROPERTY(chain != nullptr && chain->execs == 3 && chain->connects == 1); // R3
|
|
const auto* a = window_rooted(report, "process_500_b_10");
|
|
const auto* b = window_rooted(report, "process_500_b_20");
|
|
PROPERTY(a != nullptr && b != nullptr && a->connects == 1 && b->connects == 1); // C3
|
|
std::cout << "R3 chain window, C3 pid reuse: PASSED" << std::endl;
|
|
}
|
|
|
|
void test_order_and_determinism() {
|
|
std::mt19937_64 rng(kSeed);
|
|
auto ev = routine(50, kT0);
|
|
ev.push_back(exec(kT0 + 5 * kSec + 7, 9, "process_9_b_1", "process_1_b_1", "/usr/bin/odd", "odd --x"));
|
|
const auto base = replay_report_json(run(ev));
|
|
PROPERTY(replay_report_json(run(ev)) == base); // R1
|
|
// R2: shuffle within small time buckets (records arrive batched, a few
|
|
// seconds apart at most; the horizon is 2 min).
|
|
for (int round = 0; round < 20; ++round) {
|
|
auto shuffled = ev;
|
|
for (std::size_t i = 0; i + 1 < shuffled.size(); i += 2) {
|
|
if (rng() % 2) std::swap(shuffled[i], shuffled[i + 1]);
|
|
}
|
|
PROPERTY(replay_report_json(run(shuffled)) == base);
|
|
}
|
|
std::cout << "R1 deterministic, R2 order-insensitive: PASSED" << std::endl;
|
|
}
|
|
|
|
void test_ranking_and_detection() {
|
|
auto ev = routine(300, kT0);
|
|
const int64_t t = kT0 + 301 * 61 * kSec;
|
|
ev.push_back(exec(t, 4000, "process_4000_b_1", "process_1_b_1", "/usr/bin/evince", "evince /home/a/x.pdf"));
|
|
ev.push_back(exec(t + kSec, 4001, "process_4001_b_1", "process_4000_b_1", "/usr/bin/python3", "python3 -c x"));
|
|
ev.push_back(connect(t + 2 * kSec, 4001, "198.51.100.7", 4444));
|
|
ev.push_back(scan(t + kSec, 4001, "process_4001_b_1", {"Suspicious_Python"}));
|
|
ReplayOptions options;
|
|
options.warmup_ns = 3600 * kSec; // the first routine windows are new too: that is what warm-up is for
|
|
options.top = 5;
|
|
const auto report = run(ev, options);
|
|
PROPERTY(!report.top_windows.empty());
|
|
const auto& top = report.top_windows.front();
|
|
PROPERTY(top.root_node == "process_4000_b_1"); // R4
|
|
PROPERTY(top.detected && top.rules == std::vector<std::string>{"Suspicious_Python"});
|
|
PROPERTY(report.detected_windows == 1 && report.detected.size() == 1);
|
|
PROPERTY(top.bits > report.top_windows.back().bits);
|
|
std::cout << "R4 unusual chain ranks first (" << top.bits << " bits), detection marked: PASSED" << std::endl;
|
|
}
|
|
|
|
void test_store_round_trip() {
|
|
namespace fs = std::filesystem;
|
|
const auto dir = fs::temp_directory_path() / ("strawberry_replay_" + std::to_string(::getpid()));
|
|
std::error_code ec;
|
|
fs::remove_all(dir, ec);
|
|
auto ev = routine(20, kT0);
|
|
{
|
|
Database db;
|
|
PROPERTY(db.open(dir.string()));
|
|
auto copy = ev;
|
|
PROPERTY(db.append_telemetry(copy, true));
|
|
db.close();
|
|
}
|
|
std::string error;
|
|
const auto report = replay_store(dir.string(), no_warmup(), error);
|
|
PROPERTY(error.empty() && report.execs == 40 && report.windows == 20);
|
|
PROPERTY(replay_report_json(report) == replay_report_json(run(ev))); // R5
|
|
fs::remove_all(dir, ec);
|
|
std::cout << "R5 store round trip: PASSED" << std::endl;
|
|
}
|
|
|
|
void test_warmup_and_time_invariance() {
|
|
// Two novel programs, one at the start and one 20 h later after lots
|
|
// of routine activity: same score (capped novelty), not "later wins".
|
|
auto ev = routine(1200, kT0 + 60 * kSec);
|
|
ev.insert(ev.begin(), scan(kT0, 7, "process_7_b_1", {"EarlyRule"}));
|
|
ev.insert(ev.begin(), exec(kT0, 7, "process_7_b_1", "process_1_b_1", "/usr/bin/early-novel", "early-novel"));
|
|
const int64_t late = kT0 + 1201 * 61 * kSec;
|
|
ev.push_back(exec(late, 8, "process_8_b_1", "process_1_b_1", "/usr/bin/late-novel", "late-novel"));
|
|
auto late_scan = ev; // a record far older than the reorder horizon: counted as late
|
|
late_scan.push_back(scan(kT0 + 5, 7, "process_7_b_1", {"Late"}));
|
|
PROPERTY(run(late_scan).late_events == 1);
|
|
const auto all = run(ev);
|
|
const ReplayWindow* early = nullptr;
|
|
const ReplayWindow* late_w = nullptr;
|
|
for (const auto& w : all.top_windows) {
|
|
if (w.root_exe == "/usr/bin/early-novel") early = &w;
|
|
if (w.root_exe == "/usr/bin/late-novel") late_w = &w;
|
|
}
|
|
PROPERTY(early != nullptr && late_w != nullptr && early->bits == late_w->bits);
|
|
PROPERTY(early->bits == kNovelBits); // only exe_cmdline: its parent was never recorded
|
|
// The routine bash -> ls windows collapse into one line.
|
|
const auto routine_line = std::find_if(all.top_windows.begin(), all.top_windows.end(),
|
|
[](const auto& w) { return w.root_exe == "/usr/bin/bash" && w.similar > 100; });
|
|
PROPERTY(routine_line != all.top_windows.end());
|
|
|
|
ReplayOptions warm; // default 12 h warm-up
|
|
const auto report = run(ev, warm);
|
|
PROPERTY(report.warmup_windows > 0 && report.warmup_windows < report.windows);
|
|
for (const auto& w : report.top_windows) PROPERTY(w.start_ns >= kT0 + warm.warmup_ns);
|
|
PROPERTY(report.detected.size() == 1 && report.detected[0].rules == std::vector<std::string>{"EarlyRule"});
|
|
std::cout << "R7 warm-up, time invariance, grouping: PASSED" << std::endl;
|
|
}
|
|
|
|
void test_injection_evaluation() {
|
|
namespace fs = std::filesystem;
|
|
const auto dir = fs::temp_directory_path() / ("strawberry_inject_" + std::to_string(::getpid()));
|
|
std::error_code ec;
|
|
fs::remove_all(dir, ec);
|
|
// Two days of routine activity, one window a minute.
|
|
auto ev = routine(2 * 24 * 60, kT0);
|
|
{
|
|
Database db;
|
|
PROPERTY(db.open(dir.string()));
|
|
auto copy = ev;
|
|
PROPERTY(db.append_telemetry(copy, true));
|
|
db.close();
|
|
}
|
|
std::vector<Trace> traces;
|
|
std::string error;
|
|
PROPERTY(compile_traces("trace pdf_python\n"
|
|
"+0s exec viewer /usr/bin/evince \"evince /home/u/x.pdf\"\n"
|
|
"+3s exec py /usr/bin/python3 \"python3 -c import socket\" parent=viewer\n"
|
|
"+4s connect py 198.51.100.50 4444\n"
|
|
"trace quiet\n"
|
|
"+0s exec sh /usr/bin/bash \"bash\"\n",
|
|
traces, error));
|
|
ReplayOptions options;
|
|
options.warmup_ns = 6 * 3600 * kSec;
|
|
const auto plain = replay_store(dir.string(), options, error);
|
|
PROPERTY(error.empty());
|
|
options.inject = traces;
|
|
options.inject_copies = 4;
|
|
const auto report = replay_store(dir.string(), options, error);
|
|
PROPERTY(error.empty());
|
|
const auto& eval = report.evaluation;
|
|
PROPERTY(eval.traces.size() == 2 && eval.copies == 8);
|
|
PROPERTY(eval.traces[0].found == 4 && eval.traces[1].found == 4);
|
|
// Real activity is untouched: same records, windows and ranking.
|
|
PROPERTY(report.records == plain.records && eval.real_windows + report.warmup_windows == plain.windows);
|
|
PROPERTY(replay_report_json(plain).find("\"top_windows\"") != std::string::npos);
|
|
PROPERTY(nlohmann::json::parse(replay_report_json(report))["top_windows"] ==
|
|
nlohmann::json::parse(replay_report_json(plain))["top_windows"]);
|
|
// The novel chain is caught at 10/day; a bare bash window is not.
|
|
PROPERTY(eval.traces[0].caught.back() == 4);
|
|
PROPERTY(eval.traces[1].caught.back() < 4);
|
|
for (const auto r : eval.traces[0].ranks) PROPERTY(r == 1);
|
|
fs::remove_all(dir, ec);
|
|
std::cout << "R8 injection evaluation: PASSED (pdf_python ranks 1; cut at 10/day "
|
|
<< eval.cut_scores.back() << " bits)" << std::endl;
|
|
}
|
|
|
|
void test_late_labels() {
|
|
namespace fs = std::filesystem;
|
|
const std::string bad_hash(64, 'b');
|
|
auto ev = routine(10, kT0);
|
|
ProcessExecRecord r;
|
|
r.timestamp_ns = kT0 + 5 * kSec + 3;
|
|
r.pid = 600;
|
|
r.process_node_id = "process_600_b_1";
|
|
r.exe = "/home/u/.local/bin/tool";
|
|
r.cmdline = "tool";
|
|
r.sha256 = bad_hash;
|
|
r.source = "live";
|
|
ev.push_back({store::RecordType::process_exec, encode_process_exec(r)});
|
|
|
|
auto options = no_warmup();
|
|
options.late_labels[bad_hash] = {"BetaRule"};
|
|
const auto direct = run(ev, options);
|
|
PROPERTY(direct.late_detected_windows == 1 && direct.detected_windows == 0);
|
|
PROPERTY(direct.detected.size() == 1 && direct.detected[0].late_rules == std::vector<std::string>{"BetaRule"});
|
|
|
|
const auto dir = fs::temp_directory_path() / ("strawberry_late_" + std::to_string(::getpid()));
|
|
std::error_code ec;
|
|
fs::remove_all(dir, ec);
|
|
{
|
|
Database db;
|
|
PROPERTY(db.open(dir.string()));
|
|
auto copy = ev;
|
|
PROPERTY(db.append_telemetry(copy, true));
|
|
LabelRecord label;
|
|
label.timestamp_ns = kT0 + 3600 * kSec;
|
|
label.polarity = LabelPolarity::positive;
|
|
label.source = "retro_hunt";
|
|
label.action = "rules_update";
|
|
label.sha256 = bad_hash;
|
|
label.rule = "BetaRule";
|
|
PROPERTY(db.append_evidence(store::RecordType::label, encode_label(label)));
|
|
label.polarity = LabelPolarity::negative; // negatives are not late detections
|
|
label.sha256 = std::string(64, 'c');
|
|
PROPERTY(db.append_evidence(store::RecordType::label, encode_label(label)));
|
|
db.close();
|
|
}
|
|
std::string error;
|
|
const auto stored = replay_store(dir.string(), no_warmup(), error);
|
|
PROPERTY(error.empty() && stored.late_labels == 1 && stored.late_detected_windows == 1);
|
|
fs::remove_all(dir, ec);
|
|
std::cout << "R9 late labels: PASSED" << std::endl;
|
|
}
|
|
|
|
// Stage 3.5c: a window flagged late (retro-hunt) is attributed to the node that WROTE the flagged
|
|
// binary (the dropper) via the global write->exec linkage, not the exec leaf that ran it.
|
|
void test_attribution() {
|
|
const std::string bad_hash(64, 'd');
|
|
auto ev = routine(10, kT0);
|
|
// Dropper D (its own root) execs, then writes /tmp/evil.
|
|
ev.push_back(exec(kT0 + 2 * kSec, 500, "process_500_d_1", "process_1_b_1", "/usr/bin/curl", "curl x"));
|
|
ev.push_back(write(kT0 + 2 * kSec + 1, 500, "/tmp/evil"));
|
|
// Victim V (a SEPARATE root) execs the dropped binary; flagged bad later.
|
|
ProcessExecRecord r;
|
|
r.timestamp_ns = kT0 + 3 * kSec;
|
|
r.pid = 600;
|
|
r.process_node_id = "process_600_v_1";
|
|
r.parent_node_id = "process_1_b_1";
|
|
r.exe = "/tmp/evil";
|
|
r.cmdline = "evil";
|
|
r.sha256 = bad_hash;
|
|
r.source = "live";
|
|
ev.push_back({store::RecordType::process_exec, encode_process_exec(r)});
|
|
|
|
auto options = no_warmup();
|
|
options.late_labels[bad_hash] = {"DropperRule"};
|
|
auto victim = [](const ReplayReport& rep) -> const ReplayWindow* {
|
|
for (const auto& w : rep.detected) {
|
|
if (w.root_node == "process_600_v_1") return &w;
|
|
}
|
|
return nullptr;
|
|
};
|
|
const auto rep = run(ev, options);
|
|
const ReplayWindow* v = victim(rep);
|
|
PROPERTY(v != nullptr && v->late_detected);
|
|
PROPERTY(v->origin_node == "process_500_d_1"); // the dropper, not the exec leaf process_600_v_1
|
|
PROPERTY(v->origin_exe == "/usr/bin/curl");
|
|
// Determinism: same stream -> same attribution.
|
|
const ReplayWindow* v2 = victim(run(ev, options));
|
|
PROPERTY(v2 != nullptr && v2->origin_node == "process_500_d_1");
|
|
// A flagged binary with no in-set write has no origin.
|
|
auto ev3 = routine(10, kT0);
|
|
ProcessExecRecord r3 = r;
|
|
r3.exe = "/usr/bin/true"; // never written in this stream
|
|
ev3.push_back({store::RecordType::process_exec, encode_process_exec(r3)});
|
|
auto opt3 = no_warmup();
|
|
opt3.late_labels[bad_hash] = {"DropperRule"};
|
|
const ReplayWindow* v3 = victim(run(ev3, opt3));
|
|
PROPERTY(v3 != nullptr && v3->late_detected && v3->origin_node.empty());
|
|
std::cout << "R10 attribution: late_detected traced to dropper node: PASSED" << std::endl;
|
|
}
|
|
|
|
void test_examples_and_model_ranking() {
|
|
namespace fs = std::filesystem;
|
|
auto ev = routine(200, kT0);
|
|
const int64_t t = kT0 + 201 * 61 * kSec;
|
|
ev.push_back(exec(t, 4000, "process_4000_b_1", "process_1_b_1", "/usr/bin/evince", "evince x.pdf"));
|
|
ev.push_back(scan(t, 4000, "process_4000_b_1", {"SomeRule"}));
|
|
const auto dir = fs::temp_directory_path() / ("strawberry_examples_" + std::to_string(::getpid()));
|
|
std::error_code ec;
|
|
fs::remove_all(dir, ec);
|
|
{
|
|
Database db;
|
|
PROPERTY(db.open(dir.string()));
|
|
auto copy = ev;
|
|
PROPERTY(db.append_telemetry(copy, true));
|
|
db.close();
|
|
}
|
|
std::vector<Trace> traces;
|
|
std::string error;
|
|
PROPERTY(compile_traces("trace train_chain\n+0s exec a /tmp/x/dropper \"dropper\"\n"
|
|
"+1s connect a 198.51.100.9 4444\n", traces, error));
|
|
ReplayOptions o = no_warmup();
|
|
o.collect_examples = true;
|
|
o.inject = traces;
|
|
o.inject_copies = 3;
|
|
o.injected_are_positives = true;
|
|
const auto r = replay_store(dir.string(), o, error);
|
|
PROPERTY(error.empty());
|
|
PROPERTY(r.example_windows == 201 + 3 && r.example_positives == 1 + 3);
|
|
PROPERTY(r.evaluation.copies == 0); // training traces are not evaluated
|
|
PROPERTY(r.examples.size() == r.example_windows);
|
|
std::size_t positives = 0;
|
|
for (const auto& e : r.examples) positives += e.label > 0.5f;
|
|
PROPERTY(positives == 4);
|
|
|
|
o.max_examples = 100;
|
|
o.max_entity_share = 0.05; // at most 5 bash windows
|
|
const auto capped = replay_store(dir.string(), o, error);
|
|
std::size_t bash = 0;
|
|
for (const auto& e : capped.examples) bash += e.entity == "/usr/bin/bash" ? 1 : 0;
|
|
PROPERTY(bash == 5 && capped.examples.size() == 5 + 4 + 0); // 5 bash + 4 positives (evince is positive)
|
|
|
|
auto model = std::make_shared<Model>(Model::initialized(ModelShape{}, 3));
|
|
ReplayOptions ranked = no_warmup();
|
|
ranked.model = model;
|
|
const auto m = replay_store(dir.string(), ranked, error);
|
|
PROPERTY(!m.top_windows.empty());
|
|
for (const auto& w : m.top_windows) {
|
|
PROPERTY(w.model_value >= 0.0f && w.model_value <= 1.0f);
|
|
PROPERTY(std::fabs(w.bits - w.model_value * 48.0) < 1e-3);
|
|
}
|
|
fs::remove_all(dir, ec);
|
|
std::cout << "R10 examples and model ranking: PASSED" << std::endl;
|
|
}
|
|
|
|
void test_positive_sampling_under_pressure() {
|
|
// Fill every slot with distinct negative entities before any teacher
|
|
// positive arrives. Late labels must still get their class quota.
|
|
auto events = std::vector<TelemetryEvent>{};
|
|
for (int i = 0; i < 100; ++i) {
|
|
const auto ts = kT0 + i * 61 * kSec;
|
|
const auto node = "process_" + std::to_string(5000 + i) + "_b_1";
|
|
const std::string entity = std::string("/usr/bin/sample_") +
|
|
static_cast<char>('a' + i / 26) + static_cast<char>('a' + i % 26);
|
|
events.push_back(exec(ts, 5000 + i, node, "", entity, entity));
|
|
if (i >= 60) events.push_back(scan(ts, 5000 + i, node, {"Teacher"}));
|
|
}
|
|
ReplayOptions o = no_warmup();
|
|
o.collect_examples = true;
|
|
o.max_examples = 20;
|
|
o.max_entity_share = 0.05; // one negative per entity
|
|
o.sample_seed = kSeed;
|
|
const auto sampled = run(events, o);
|
|
PROPERTY(sampled.example_windows == 100 && sampled.example_positives == 40);
|
|
PROPERTY(sampled.examples.size() == o.max_examples);
|
|
std::size_t positives = 0;
|
|
bool retained_later_positive = false;
|
|
for (const auto& e : sampled.examples) {
|
|
positives += e.label > 0.5f;
|
|
retained_later_positive |= e.label > 0.5f && e.start_ns >= kT0 + 64 * 61 * kSec;
|
|
}
|
|
PROPERTY(positives == 4); // the positive quota survives a full negative sample
|
|
// Check reproducibility including examples, which report JSON does not emit.
|
|
const auto again = run(events, o);
|
|
PROPERTY(again.examples.size() == sampled.examples.size());
|
|
for (std::size_t i = 0; i < sampled.examples.size(); ++i) {
|
|
PROPERTY(sampled.examples[i].start_ns == again.examples[i].start_ns);
|
|
PROPERTY(sampled.examples[i].entity == again.examples[i].entity);
|
|
PROPERTY(sampled.examples[i].label == again.examples[i].label);
|
|
PROPERTY(sampled.examples[i].x == again.examples[i].x);
|
|
}
|
|
// Do not assert that an arbitrary seed must choose a later example:
|
|
// reservoir sampling may legitimately retain the first four. Across
|
|
// fixed seeds, however, the implementation cannot keep only the first.
|
|
for (uint64_t seed = 1; seed <= 8; ++seed) {
|
|
o.sample_seed = seed;
|
|
for (const auto& e : run(events, o).examples) {
|
|
retained_later_positive |= e.label > 0.5f && e.start_ns >= kT0 + 64 * 61 * kSec;
|
|
}
|
|
}
|
|
PROPERTY(retained_later_positive);
|
|
|
|
// Tiny configured samples must retain a teacher example too.
|
|
for (std::size_t capacity = 1; capacity < 5; ++capacity) {
|
|
o.max_examples = capacity;
|
|
const auto tiny = run(events, o);
|
|
PROPERTY(tiny.examples.size() == capacity);
|
|
PROPERTY(std::count_if(tiny.examples.begin(), tiny.examples.end(),
|
|
[](const auto& e) { return e.label > 0.5f; }) == 1);
|
|
}
|
|
std::cout << "R10 positive sampling under pressure: PASSED" << std::endl;
|
|
}
|
|
|
|
void test_policy_evaluation() {
|
|
// Gate as a pure function.
|
|
PolicyEvaluation pe;
|
|
pe.extra_positives = 12;
|
|
pe.policies = {{"model", 10, 20.0, 9, 9.0, 0}, {"rarity_greedy", 20, 40.0, 8, 8.0, 0}};
|
|
auto g = promotion_gate(pe); // 0.45 vs 0.2 hits/CPU-s, recall 9 >= 8
|
|
PROPERTY(g.passed && g.checks.size() == 3);
|
|
pe.policies[0].hits = 7; // efficiency 0.35 (still >= 0.3) but lower recall
|
|
pe.policies[0].weighted_hits = 7;
|
|
g = promotion_gate(pe);
|
|
PROPERTY(!g.passed && g.checks[2].rfind("FAIL", 0) == 0 && g.checks[1].rfind("ok", 0) == 0);
|
|
pe.policies[0] = {"model", 10, 40.0, 9, 9.0, 0}; // 0.225 < 1.5 x 0.2
|
|
PROPERTY(!promotion_gate(pe).passed && promotion_gate(pe).checks[1].rfind("FAIL", 0) == 0);
|
|
pe.extra_positives = 4;
|
|
pe.policies[0] = {"model", 10, 20.0, 4, 4.0, 0};
|
|
PROPERTY(!promotion_gate(pe).passed && promotion_gate(pe).checks[0].rfind("FAIL", 0) == 0);
|
|
pe.policies.pop_back();
|
|
PROPERTY(!promotion_gate(pe).passed);
|
|
|
|
// Replay: routine history, injected evaluation traces, a rule detection,
|
|
// and logged shadow decisions (one on an injected window).
|
|
auto ev = routine(240, kT0);
|
|
const int64_t t = kT0 + 30 * 61 * kSec + 7;
|
|
ev.push_back(exec(t, 4000, "process_4000_b_1", "process_1_b_1", "/usr/bin/evince", "evince x.pdf"));
|
|
ev.push_back(scan(t, 4000, "process_4000_b_1", {"SomeRule"}));
|
|
InspectionDecision logged;
|
|
logged.timestamp_ns = t + 200 * kSec;
|
|
logged.root_node = "process_4000_b_1";
|
|
logged.action = InspectAction::memory_scan;
|
|
ev.push_back({store::RecordType::inspection_action, encode_inspection_action(logged)});
|
|
logged.shadow = false;
|
|
ev.push_back({store::RecordType::inspection_action, encode_inspection_action(logged)});
|
|
std::vector<Trace> traces;
|
|
std::string error;
|
|
PROPERTY(compile_traces("trace odd\n+0s exec a /tmp/q/odd \"odd --x\"\n+1s connect a 198.51.100.4 4444\n",
|
|
traces, error));
|
|
ReplayOptions o;
|
|
o.warmup_ns = 600 * kSec; // routine windows are known by then
|
|
o.policy_eval = true;
|
|
o.policy.budget_seconds_per_hour = 4; // two memory scans an hour
|
|
// Even an apparently perfect free unsupported action cannot influence
|
|
// the deployment gate. Without the executable mask this evades budgets.
|
|
o.policy.reach[static_cast<std::size_t>(InspectAction::watch_subtree)] = 1000;
|
|
o.policy.cost_seconds[static_cast<std::size_t>(InspectAction::watch_subtree)] = 0;
|
|
|
|
o.inject = traces;
|
|
o.inject_copies = 4;
|
|
// Through a store, so injection is planned over the history's time range.
|
|
namespace fs = std::filesystem;
|
|
const auto dir = fs::temp_directory_path() / ("strawberry_policy_" + std::to_string(::getpid()));
|
|
std::error_code ec;
|
|
fs::remove_all(dir, ec);
|
|
{
|
|
Database db;
|
|
PROPERTY(db.open(dir.string()));
|
|
auto copy = ev;
|
|
PROPERTY(db.append_telemetry(copy, true));
|
|
db.close();
|
|
}
|
|
const auto r = replay_store(dir.string(), o, error);
|
|
PROPERTY(error.empty());
|
|
const auto& p = r.policy_eval;
|
|
PROPERTY(p.extra_positives == 4 && p.rule_detected == 1); // the evince window is rules' already
|
|
PROPERTY(p.policies.size() == 2 && p.policies[0].name == "rarity_greedy" && p.policies[1].name == "shadow_logged");
|
|
const auto& rarity = p.policies[0];
|
|
// Structure only: with a few hundred windows rarity has not seen enough
|
|
// (its smoothing wants ~30k observations) to separate anything.
|
|
PROPERTY(rarity.hits <= p.extra_positives && rarity.inspections + rarity.budget_refusals <= p.windows);
|
|
PROPERTY(rarity.budget_refusals > 0);
|
|
PROPERTY(rarity.cpu_seconds <= 4.0 * (static_cast<double>(r.last_ns - r.first_ns) / 3.6e12 + 1.0));
|
|
const auto& shadow = p.policies[1];
|
|
PROPERTY(shadow.inspections == 1 && shadow.hits == 0); // it inspected a rule-flagged window: not an extra hit
|
|
fs::remove_all(dir, ec);
|
|
std::cout << "R11 policy evaluation and gate: PASSED (rarity-greedy " << rarity.hits << "/" << p.extra_positives
|
|
<< " in " << rarity.cpu_seconds << " CPU-s)" << std::endl;
|
|
}
|
|
|
|
void test_extreme_timestamps() {
|
|
// On-disk timestamps are nonnegative int64 values, not necessarily sane
|
|
// wall-clock dates. Interval comparisons must not overflow near the end.
|
|
ReplayOptions options;
|
|
options.warmup_ns = 3600 * kSec;
|
|
Replayer replay(options);
|
|
const auto first = exec(INT64_MAX - 100 * kSec, 42, "process_42_b_1", "", "/tmp/a", "a");
|
|
const auto last = exec(INT64_MAX - 1, 43, "process_43_b_1", "", "/tmp/b", "b");
|
|
replay.feed(first.type, first.payload);
|
|
replay.feed(last.type, last.payload);
|
|
const auto report = replay.finish();
|
|
PROPERTY(report.warmup_windows == 2 && report.top_windows.empty());
|
|
std::string error;
|
|
PROPERTY(compile_traces("trace edge\n+0s exec a /tmp/edge edge\n", options.inject, error));
|
|
options.inject_copies = 1;
|
|
// Field iteration accepts repeated tags. The store indexes the first
|
|
// timestamp while the typed decoder retains the last, so the window
|
|
// may start after report.last_ns even though both times are nonnegative.
|
|
auto duplicate = exec(1, 42, "process_42_b_1", "", "/tmp/a", "a");
|
|
store::FieldWriter suffix;
|
|
suffix.u64(1, INT64_MAX - 1);
|
|
duplicate.payload += suffix.data();
|
|
Replayer mismatched(options);
|
|
mismatched.feed(duplicate.type, duplicate.payload);
|
|
PROPERTY(mismatched.finish().evaluation.days == 1.0 / 24);
|
|
namespace fs = std::filesystem;
|
|
const auto dir = fs::temp_directory_path() / ("strawberry_replay_edge_" + std::to_string(::getpid()));
|
|
std::error_code ec;
|
|
fs::remove_all(dir, ec);
|
|
{
|
|
Database db;
|
|
PROPERTY(db.open(dir.string()));
|
|
std::vector<TelemetryEvent> events{first, last};
|
|
PROPERTY(db.append_telemetry(events, true));
|
|
}
|
|
(void) replay_store(dir.string(), options, error);
|
|
PROPERTY(error.find("not enough history") != std::string::npos);
|
|
fs::remove_all(dir, ec);
|
|
}
|
|
|
|
void test_invalid_utf8() {
|
|
// Split literals: "\x80" "evil", or the escape would swallow the 'e'.
|
|
const std::string bad = std::string("/tmp/\xff\xfe\xc0\x80" "evil\xed\xa0\x80") + "\xf4\x90\x80\x80";
|
|
std::vector<TelemetryEvent> ev{exec(kT0, 1, "process_1_b_1", "", bad, bad + " -x")};
|
|
const auto json = replay_report_json(run(ev));
|
|
const auto parsed = nlohmann::json::parse(json, nullptr, false);
|
|
PROPERTY(!parsed.is_discarded());
|
|
PROPERTY(!valid_utf8(bad) && valid_utf8(json));
|
|
|
|
std::mt19937_64 rng(kSeed);
|
|
for (int i = 0; i < 20000; ++i) {
|
|
std::string s;
|
|
const auto n = rng() % 12;
|
|
for (std::size_t k = 0; k < n; ++k) s.push_back(static_cast<char>(rng() % 256));
|
|
const auto quoted = json_string(s);
|
|
const auto back = nlohmann::json::parse(quoted, nullptr, false);
|
|
PROPERTY(!back.is_discarded() && back.is_string());
|
|
if (valid_utf8(s)) PROPERTY(back.get<std::string>() == s); // valid text is unchanged
|
|
}
|
|
PROPERTY(valid_utf8("plain ascii") && valid_utf8("\xc3\xa9t\xc3\xa9") && !valid_utf8("\xc3"));
|
|
std::cout << "R6 invalid UTF-8 stays valid JSON: PASSED" << std::endl;
|
|
}
|
|
|
|
} // namespace
|
|
|
|
// A window running an allowlisted (negative-labelled) binary is a confirmed
|
|
// benign: kept as a negative training example, never a teacher positive, even
|
|
// though a (now-cleared) rule detected it. This is what keeps a false-positive
|
|
// feed hit the operator allowlisted from poisoning the labels.
|
|
void test_allowlisted_window_is_negative() {
|
|
namespace fs = std::filesystem;
|
|
const std::string sha(64, 'd');
|
|
auto ev = routine(10, kT0); // benign windows (label 0)
|
|
ProcessExecRecord r;
|
|
r.timestamp_ns = kT0 + 5 * kSec + 7;
|
|
r.pid = 700;
|
|
r.process_node_id = "process_700_d_1";
|
|
r.exe = "/usr/bin/ss";
|
|
r.cmdline = "ss -tulpn";
|
|
r.sha256 = sha;
|
|
r.source = "live";
|
|
ev.push_back({store::RecordType::process_exec, encode_process_exec(r)});
|
|
ev.push_back(scan(r.timestamp_ns + 1, 700, "process_700_d_1", {"IOC_KnownMalwareHash"})); // detected
|
|
|
|
const auto dir = fs::temp_directory_path() / ("strawberry_allow_neg_" + std::to_string(::getpid()));
|
|
std::error_code ec;
|
|
fs::remove_all(dir, ec);
|
|
{
|
|
Database db;
|
|
PROPERTY(db.open(dir.string()));
|
|
auto copy = ev;
|
|
PROPERTY(db.append_telemetry(copy, true));
|
|
db.close();
|
|
}
|
|
std::string error;
|
|
ReplayOptions o = no_warmup();
|
|
o.collect_examples = true;
|
|
const auto before = replay_store(dir.string(), o, error);
|
|
std::size_t pos_before = 0;
|
|
for (const auto& e : before.examples) pos_before += e.label > 0.5f;
|
|
PROPERTY(error.empty() && pos_before == 1); // the detected ss window is a positive
|
|
|
|
{ // the operator allowlists the hash -> a negative label in the stream
|
|
Database db;
|
|
PROPERTY(db.open(dir.string()));
|
|
LabelRecord label;
|
|
label.timestamp_ns = kT0 + 6 * kSec;
|
|
label.polarity = LabelPolarity::negative;
|
|
label.source = "allowlist";
|
|
label.action = "suppressed";
|
|
label.sha256 = sha;
|
|
PROPERTY(db.append_evidence(store::RecordType::label, encode_label(label)));
|
|
db.close();
|
|
}
|
|
const auto after = replay_store(dir.string(), o, error);
|
|
std::size_t pos_after = 0;
|
|
for (const auto& e : after.examples) pos_after += e.label > 0.5f;
|
|
PROPERTY(error.empty() && pos_after == 0); // no longer a positive
|
|
PROPERTY(after.examples.size() == before.examples.size()); // kept as a negative, not dropped
|
|
fs::remove_all(dir, ec);
|
|
std::cout << "allowlisted window is a negative: PASSED" << std::endl;
|
|
}
|
|
|
|
// ---- label retraction (R12) ----
|
|
|
|
void append_label(const std::string& dir, LabelPolarity polarity, const std::string& action,
|
|
const std::string& sha, const std::string& entry, const std::string& source,
|
|
const std::string& rule, int64_t timestamp = kT0 + 3600 * kSec) {
|
|
Database db;
|
|
PROPERTY(db.open(dir));
|
|
LabelRecord label;
|
|
label.timestamp_ns = timestamp;
|
|
label.polarity = polarity;
|
|
label.source = source;
|
|
label.action = action;
|
|
label.sha256 = sha;
|
|
label.allowlist_entry = entry;
|
|
label.rule = rule;
|
|
PROPERTY(db.append_evidence(store::RecordType::label, encode_label(label)));
|
|
db.close();
|
|
}
|
|
|
|
std::size_t positives_in(const ReplayReport& r) {
|
|
std::size_t n = 0;
|
|
for (const auto& e : r.examples) n += e.label > 0.5f;
|
|
return n;
|
|
}
|
|
|
|
const ReplayWindow* detected_rooted(const ReplayReport& report, const std::string& node) {
|
|
for (const auto& w : report.detected) {
|
|
if (w.root_node == node) return &w;
|
|
}
|
|
return nullptr;
|
|
}
|
|
|
|
// A store with a few routine windows plus one window running `sha` (`exe`),
|
|
// optionally rule-detected.
|
|
std::filesystem::path make_retract_store(const std::string& tag, const std::string& sha, const std::string& exe,
|
|
bool detected) {
|
|
namespace fs = std::filesystem;
|
|
const auto dir = fs::temp_directory_path() / ("strawberry_" + tag + "_" + std::to_string(::getpid()));
|
|
std::error_code ec;
|
|
fs::remove_all(dir, ec);
|
|
auto ev = routine(5, kT0);
|
|
ProcessExecRecord r;
|
|
r.timestamp_ns = kT0 + 3 * kSec + 7;
|
|
r.pid = 700;
|
|
r.process_node_id = "process_700_b_1";
|
|
r.exe = exe;
|
|
r.cmdline = "x";
|
|
r.sha256 = sha;
|
|
r.source = "live";
|
|
ev.push_back({store::RecordType::process_exec, encode_process_exec(r)});
|
|
if (detected) ev.push_back(scan(r.timestamp_ns + 1, 700, "process_700_b_1", {"IOC_KnownMalwareHash"}));
|
|
Database db;
|
|
PROPERTY(db.open(dir.string()));
|
|
auto copy = ev;
|
|
PROPERTY(db.append_telemetry(copy, true));
|
|
db.close();
|
|
return dir;
|
|
}
|
|
|
|
void test_retraction_allowlist_remove() {
|
|
namespace fs = std::filesystem;
|
|
const std::string sha(64, 'd');
|
|
const auto dir = make_retract_store("retract_al", sha, "/usr/bin/ss", true);
|
|
std::string error;
|
|
ReplayOptions o = no_warmup();
|
|
o.collect_examples = true;
|
|
|
|
append_label(dir.string(), LabelPolarity::negative, "allowlist-add", sha, "allowlist-e1", "operator", "");
|
|
const auto after_add = replay_store(dir.string(), o, error);
|
|
PROPERTY(error.empty() && positives_in(after_add) == 0); // cleared false positive
|
|
|
|
append_label(dir.string(), LabelPolarity::retracted, "allowlist-remove", "", "allowlist-e1", "operator", "");
|
|
const auto after_remove = replay_store(dir.string(), o, error);
|
|
// The removal reverts the hash to its pre-label state: the rule detection
|
|
// stands again (it was the only judgment). It is not a new teacher positive.
|
|
PROPERTY(error.empty() && positives_in(after_remove) == 1);
|
|
fs::remove_all(dir);
|
|
std::cout << "R12 allowlist-remove retraction: PASSED" << std::endl;
|
|
}
|
|
|
|
void test_retraction_quarantine_remove() {
|
|
namespace fs = std::filesystem;
|
|
const std::string sha(64, 'b');
|
|
const auto dir = make_retract_store("retract_q", sha, "/tmp/x/dropper", false);
|
|
std::string error;
|
|
ReplayOptions o = no_warmup();
|
|
o.collect_examples = true;
|
|
|
|
append_label(dir.string(), LabelPolarity::positive, "quarantine", sha, "", "operator", "");
|
|
const auto after_quarantine = replay_store(dir.string(), o, error);
|
|
PROPERTY(error.empty() && positives_in(after_quarantine) == 1); // late-detected via quarantine
|
|
|
|
append_label(dir.string(), LabelPolarity::retracted, "quarantine-remove", sha, "", "operator", "");
|
|
const auto after_remove = replay_store(dir.string(), o, error);
|
|
PROPERTY(error.empty() && positives_in(after_remove) == 0); // the quarantine reason is gone
|
|
fs::remove_all(dir);
|
|
std::cout << "R12 quarantine-remove retraction: PASSED" << std::endl;
|
|
}
|
|
|
|
void test_retraction_independent_positives() {
|
|
namespace fs = std::filesystem;
|
|
const std::string sha(64, 'c');
|
|
const auto dir = make_retract_store("retract_indep", sha, "/tmp/x/dropper", false);
|
|
std::string error;
|
|
ReplayOptions o = no_warmup();
|
|
o.collect_examples = true;
|
|
|
|
// Two independent positives on one hash: an operator quarantine and a
|
|
// retro-hunt rule. Removing the quarantine must not erase the rule reason.
|
|
append_label(dir.string(), LabelPolarity::positive, "quarantine", sha, "", "operator", "");
|
|
append_label(dir.string(), LabelPolarity::positive, "rules_update", sha, "", "rules_update", "BetaRule");
|
|
const auto before = replay_store(dir.string(), o, error);
|
|
PROPERTY(error.empty() && positives_in(before) == 1);
|
|
const auto* before_w = detected_rooted(before, "process_700_b_1");
|
|
PROPERTY(before_w != nullptr && before_w->late_rules.size() == 2);
|
|
|
|
append_label(dir.string(), LabelPolarity::retracted, "quarantine-remove", sha, "", "operator", "");
|
|
const auto after = replay_store(dir.string(), o, error);
|
|
PROPERTY(error.empty() && positives_in(after) == 1); // still late-detected via BetaRule
|
|
const auto* after_w = detected_rooted(after, "process_700_b_1");
|
|
PROPERTY(after_w != nullptr && after_w->late_rules == std::vector<std::string>{"BetaRule"});
|
|
fs::remove_all(dir);
|
|
std::cout << "R12 independent positives survive quarantine-remove: PASSED" << std::endl;
|
|
}
|
|
|
|
void test_retraction_multiple_entries() {
|
|
namespace fs = std::filesystem;
|
|
const std::string sha(64, 'e');
|
|
const auto dir = make_retract_store("retract_multi", sha, "/usr/bin/ss", true);
|
|
std::string error;
|
|
ReplayOptions o = no_warmup();
|
|
o.collect_examples = true;
|
|
|
|
append_label(dir.string(), LabelPolarity::negative, "allowlist-add", sha, "allowlist-e1", "operator", "");
|
|
append_label(dir.string(), LabelPolarity::negative, "allowlist-add", sha, "allowlist-e2", "operator", "");
|
|
const auto both = replay_store(dir.string(), o, error);
|
|
PROPERTY(error.empty() && positives_in(both) == 0);
|
|
|
|
append_label(dir.string(), LabelPolarity::retracted, "allowlist-remove", "", "allowlist-e1", "operator", "");
|
|
const auto one_left = replay_store(dir.string(), o, error);
|
|
PROPERTY(error.empty() && positives_in(one_left) == 0); // e2 still covers the hash
|
|
|
|
append_label(dir.string(), LabelPolarity::retracted, "allowlist-remove", "", "allowlist-e2", "operator", "");
|
|
const auto none = replay_store(dir.string(), o, error);
|
|
PROPERTY(error.empty() && positives_in(none) == 1); // both entries gone
|
|
fs::remove_all(dir);
|
|
std::cout << "R12 multiple allowlist entries resolve independently: PASSED" << std::endl;
|
|
}
|
|
|
|
void test_retraction_readd() {
|
|
namespace fs = std::filesystem;
|
|
const std::string sha(64, 'f');
|
|
const auto dir = make_retract_store("retract_readd", sha, "/usr/bin/ss", true);
|
|
std::string error;
|
|
ReplayOptions o = no_warmup();
|
|
o.collect_examples = true;
|
|
|
|
append_label(dir.string(), LabelPolarity::negative, "allowlist-add", sha, "allowlist-e1", "operator", "");
|
|
PROPERTY(error.empty() && positives_in(replay_store(dir.string(), o, error)) == 0);
|
|
append_label(dir.string(), LabelPolarity::retracted, "allowlist-remove", "", "allowlist-e1", "operator", "");
|
|
PROPERTY(error.empty() && positives_in(replay_store(dir.string(), o, error)) == 1);
|
|
append_label(dir.string(), LabelPolarity::negative, "allowlist-add", sha, "allowlist-e1", "operator", "");
|
|
PROPERTY(error.empty() && positives_in(replay_store(dir.string(), o, error)) == 0); // re-added
|
|
fs::remove_all(dir);
|
|
std::cout << "R12 add/remove/re-add resolves in stored order: PASSED" << std::endl;
|
|
}
|
|
|
|
// ---- positive per-entity cap (R13) ----
|
|
|
|
void test_positive_entity_cap() {
|
|
std::vector<TelemetryEvent> ev;
|
|
// One hot entity with many detected (positive) windows, plus many distinct
|
|
// negative entities to fill the sample.
|
|
for (int i = 0; i < 60; ++i) {
|
|
const auto ts = kT0 + i * 61 * kSec;
|
|
const auto node = "process_" + std::to_string(6000 + i) + "_b_1";
|
|
ev.push_back(exec(ts, 6000 + i, node, "", "/usr/bin/hot", "hot --x"));
|
|
ev.push_back(scan(ts, 6000 + i, node, {"Teacher"}));
|
|
}
|
|
for (int i = 0; i < 200; ++i) {
|
|
const auto ts = kT0 + 1000 * 61 * kSec + i * 61 * kSec;
|
|
const auto node = "process_" + std::to_string(8000 + i) + "_b_1";
|
|
const std::string entity =
|
|
std::string("/usr/bin/neg_") + static_cast<char>('a' + i / 26) + static_cast<char>('a' + i % 26);
|
|
ev.push_back(exec(ts, 8000 + i, node, "", entity, entity));
|
|
}
|
|
ReplayOptions o = no_warmup();
|
|
o.collect_examples = true;
|
|
o.max_examples = 50;
|
|
o.max_entity_share = 0.1; // combined cap = 5
|
|
o.sample_seed = kSeed;
|
|
const auto r = run(ev, o);
|
|
// Combined per-entity cap holds for every entity.
|
|
std::unordered_map<std::string, std::size_t> per_entity;
|
|
for (const auto& e : r.examples) ++per_entity[e.entity];
|
|
for (const auto& [entity, n] : per_entity) PROPERTY(n <= 5);
|
|
// The hot entity's 60 raw positives are capped to at most 5, and the
|
|
// rejection is counted (distinguishable from accidental starvation).
|
|
PROPERTY(per_entity["/usr/bin/hot"] <= 5);
|
|
// Extra candidates may refresh existing slots, so not every arrival
|
|
// beyond the cap is a rejection.
|
|
PROPERTY(r.example_positive_entity_rejections <= 55);
|
|
// The positive quota still bounds the retained positives.
|
|
PROPERTY(positives_in(r) <= o.max_examples / 5);
|
|
PROPERTY(r.examples.size() <= o.max_examples);
|
|
// Deterministic per seed.
|
|
const auto again = run(ev, o);
|
|
PROPERTY(again.examples.size() == r.examples.size());
|
|
for (std::size_t i = 0; i < r.examples.size(); ++i) {
|
|
PROPERTY(r.examples[i].start_ns == again.examples[i].start_ns);
|
|
PROPERTY(r.examples[i].entity == again.examples[i].entity);
|
|
PROPERTY(r.examples[i].label == again.examples[i].label);
|
|
}
|
|
std::cout << "R13 positive entity cap: PASSED (hot entity " << per_entity["/usr/bin/hot"]
|
|
<< " examples, " << r.example_positive_entity_rejections << " concentration rejections)" << std::endl;
|
|
}
|
|
|
|
void test_positive_not_starved_by_entity_negatives() {
|
|
std::vector<TelemetryEvent> ev;
|
|
// Entity "/usr/bin/mixed" first contributes many negatives (filling its
|
|
// combined cap), then one detected positive. The positive must displace a
|
|
// negative of the same entity rather than being starved by the negatives.
|
|
for (int i = 0; i < 10; ++i) {
|
|
const auto ts = kT0 + i * 61 * kSec;
|
|
const auto node = "process_" + std::to_string(7000 + i) + "_b_1";
|
|
ev.push_back(exec(ts, 7000 + i, node, "", "/usr/bin/mixed", "mixed"));
|
|
}
|
|
const int64_t tpos = kT0 + 100 * 61 * kSec;
|
|
ev.push_back(exec(tpos, 7999, "process_7999_b_1", "", "/usr/bin/mixed", "mixed"));
|
|
ev.push_back(scan(tpos, 7999, "process_7999_b_1", {"Teacher"}));
|
|
ReplayOptions o = no_warmup();
|
|
o.collect_examples = true;
|
|
o.max_examples = 8;
|
|
o.max_entity_share = 0.5; // combined cap = 4
|
|
o.sample_seed = kSeed;
|
|
const auto r = run(ev, o);
|
|
PROPERTY(positives_in(r) == 1); // the late positive is retained
|
|
std::size_t mixed = 0;
|
|
for (const auto& e : r.examples) mixed += e.entity == "/usr/bin/mixed";
|
|
PROPERTY(mixed <= 4); // combined cap held even though negatives preceded the positive
|
|
bool retained_positive = false;
|
|
for (const auto& e : r.examples) retained_positive |= e.label > 0.5f && e.entity == "/usr/bin/mixed";
|
|
PROPERTY(retained_positive);
|
|
std::cout << "R13 positive displaces same-entity negatives: PASSED" << std::endl;
|
|
}
|
|
|
|
void test_sampler_capacity_lifetime() {
|
|
// The debug sampler invariant checks ALLOCATED per-entity capacity, not
|
|
// just logical slot counts. Empty class vectors must release old storage.
|
|
auto o = no_warmup();
|
|
o.collect_examples = true;
|
|
o.max_examples = 1;
|
|
o.max_entity_share = 1;
|
|
o.sample_seed = 1;
|
|
auto events = std::vector<TelemetryEvent>{
|
|
exec(kT0, 9400, "process_9400_b_1", "", "/usr/bin/kept", "kept"),
|
|
exec(kT0 + 61 * kSec, 9401, "process_9401_b_1", "", "/usr/bin/kept", "kept"),
|
|
scan(kT0 + 61 * kSec, 9401, "process_9401_b_1", {"Teacher"})};
|
|
PROPERTY(positives_in(run(events, o)) == 1);
|
|
// Keep one negative of A while its positive reservoir grows and shrinks.
|
|
// Historic nonempty capacities must also shrink geometrically, otherwise
|
|
// many such retained entities can accumulate O(max_examples^2) storage.
|
|
o.max_examples = 128;
|
|
for (int i = 0; i < 532; ++i) {
|
|
const auto ts = kT0 + (i + 2) * 61 * kSec;
|
|
const auto node = "process_" + std::to_string(9500 + i) + "_b_1";
|
|
const auto entity = i < 20 ? "/usr/bin/kept" : "/usr/bin/other";
|
|
events.push_back(exec(ts, 9500 + i, node, "", entity, entity));
|
|
events.push_back(scan(ts, 9500 + i, node, {"Teacher"}));
|
|
}
|
|
const auto result = run(events, o);
|
|
PROPERTY(positives_in(result) == 25 && result.examples.size() == 26);
|
|
std::cout << "R15 allocated sampler capacity lifetime: PASSED" << std::endl;
|
|
}
|
|
|
|
void test_parent_audit_regressions() {
|
|
namespace fs = std::filesystem;
|
|
std::size_t failures = 0;
|
|
const auto verify = [&](bool ok, const char* reason) {
|
|
if (!ok) {
|
|
++failures;
|
|
std::cerr << "AUDIT REGRESSION FAILED: " << reason << std::endl;
|
|
}
|
|
};
|
|
ReplayOptions o = no_warmup();
|
|
o.collect_examples = true;
|
|
const std::string sha(64, 'a'), other(64, 'b');
|
|
std::string error;
|
|
auto dir = make_retract_store("audit_multi_hash", sha, "/usr/bin/ss", true);
|
|
// A path/rule entry can suppress several binaries. Removal must retire
|
|
// every binding, not just the last hash observed for that entry.
|
|
append_label(dir.string(), LabelPolarity::negative, "suppressed", sha, "path-entry", "allowlist", "");
|
|
append_label(dir.string(), LabelPolarity::negative, "suppressed", other, "path-entry", "allowlist", "");
|
|
append_label(dir.string(), LabelPolarity::retracted, "allowlist-remove", "", "path-entry", "operator", "");
|
|
verify(positives_in(replay_store(dir.string(), o, error)) == 1 && error.empty(), "remove all entry hash bindings");
|
|
fs::remove_all(dir);
|
|
|
|
dir = make_retract_store("audit_delayed_suppression", sha, "/usr/bin/ss", true);
|
|
append_label(dir.string(), LabelPolarity::negative, "allowlist-add", sha, "entry", "operator", "", kT0 + 100);
|
|
// Stored order, not wall time, controls policy judgments.
|
|
append_label(dir.string(), LabelPolarity::retracted, "allowlist-remove", "", "entry", "operator", "", kT0 + 50);
|
|
append_label(dir.string(), LabelPolarity::negative, "suppressed", sha, "entry", "allowlist", "");
|
|
verify(positives_in(replay_store(dir.string(), o, error)) == 1 && error.empty(), "late observation cannot resurrect removed entry");
|
|
// Path/rule adds carry no hash; they still explicitly start a new lifetime.
|
|
append_label(dir.string(), LabelPolarity::negative, "allowlist-add", "", "entry", "operator", "");
|
|
verify(positives_in(replay_store(dir.string(), o, error)) == 1 && error.empty(), "new lifetime has no old hash bindings");
|
|
append_label(dir.string(), LabelPolarity::negative, "suppressed", sha, "entry", "allowlist", "");
|
|
verify(positives_in(replay_store(dir.string(), o, error)) == 0 && error.empty(), "explicit re-add permits new observations");
|
|
fs::remove_all(dir);
|
|
|
|
dir = make_retract_store("audit_provenance", sha, "/tmp/x/dropper", false);
|
|
append_label(dir.string(), LabelPolarity::positive, "quarantine", sha, "", "operator", "");
|
|
append_label(dir.string(), LabelPolarity::retracted, "quarantine-remove", sha, "", "teacher", "");
|
|
verify(positives_in(replay_store(dir.string(), o, error)) == 1 && error.empty(), "unrelated source cannot retract operator judgment");
|
|
// Display strings are not identities: an independent reason with the
|
|
// same rendered text must survive an actual quarantine removal.
|
|
append_label(dir.string(), LabelPolarity::positive, "rules_update", sha, "", "teacher", "operator:quarantine");
|
|
append_label(dir.string(), LabelPolarity::retracted, "quarantine-remove", sha, "", "operator", "");
|
|
verify(positives_in(replay_store(dir.string(), o, error)) == 1 && error.empty(), "independent same-text positive survives");
|
|
fs::remove_all(dir);
|
|
|
|
dir = make_retract_store("audit_bad_label", sha, "/usr/bin/ss", true);
|
|
{
|
|
Database db;
|
|
PROPERTY(db.open(dir.string()));
|
|
PROPERTY(db.append_evidence(store::RecordType::label, std::string(1, static_cast<char>(0x80))));
|
|
}
|
|
(void) replay_store(dir.string(), o, error);
|
|
verify(!error.empty(), "malformed labels fail visibly instead of silently joining partial evidence");
|
|
fs::remove_all(dir);
|
|
|
|
// A capped entity must refresh its retained positives across time,
|
|
// rather than freezing the first cap examples forever.
|
|
std::vector<TelemetryEvent> events;
|
|
for (int i = 0; i < 100; ++i) {
|
|
const auto ts = kT0 + i * 61 * kSec;
|
|
const auto node = "process_" + std::to_string(9000 + i) + "_b_1";
|
|
events.push_back(exec(ts, 9000 + i, node, "", "/usr/bin/hot", "hot"));
|
|
events.push_back(scan(ts, 9000 + i, node, {"Teacher"}));
|
|
}
|
|
o.max_examples = 50;
|
|
o.max_entity_share = 0.1;
|
|
o.sample_seed = 1;
|
|
const auto capped = run(events, o);
|
|
verify(positives_in(capped) == 5, "achievable per-entity positive capacity is retained");
|
|
verify(std::any_of(capped.examples.begin(), capped.examples.end(), [](const auto& e) {
|
|
return e.start_ns >= kT0 + 5 * 61 * kSec;
|
|
}), "capped positives are sampled across time");
|
|
|
|
// With the positive quota full, an entity whose capacity is occupied
|
|
// by negatives must still have a chance to supply a teacher positive.
|
|
events.clear();
|
|
for (int i = 0; i < 4; ++i) {
|
|
events.push_back(exec(kT0 + i * 61 * kSec, 9100 + i,
|
|
"process_" + std::to_string(9100 + i) + "_b_1", "", "/usr/bin/mixed", "mixed"));
|
|
}
|
|
events.push_back(exec(kT0 + 300 * kSec, 9200, "process_9200_b_1", "", "/usr/bin/first", "first"));
|
|
events.push_back(scan(kT0 + 300 * kSec, 9200, "process_9200_b_1", {"Teacher"}));
|
|
events.push_back(exec(kT0 + 400 * kSec, 9201, "process_9201_b_1", "", "/usr/bin/mixed", "mixed"));
|
|
events.push_back(scan(kT0 + 400 * kSec, 9201, "process_9201_b_1", {"Teacher"}));
|
|
o.max_examples = 8;
|
|
o.max_entity_share = 0.5;
|
|
bool selected_mixed = false;
|
|
for (uint64_t seed = 1; seed <= 16; ++seed) {
|
|
o.sample_seed = seed;
|
|
const auto sample = run(events, o);
|
|
verify(positives_in(sample) == 1 && sample.examples.size() <= 8, "class quota survives cross-entity replacement");
|
|
std::unordered_map<std::string, std::size_t> counts;
|
|
for (const auto& e : sample.examples) {
|
|
++counts[e.entity];
|
|
selected_mixed |= e.label > 0.5f && e.entity == "/usr/bin/mixed";
|
|
}
|
|
for (const auto& [entity, count] : counts) verify(count <= 4, "combined cap survives cross-entity replacement");
|
|
}
|
|
verify(selected_mixed, "negative entity capacity cannot starve a positive at full global quota");
|
|
// Seeded mutation pressure exercises slot compaction, cross-class changes,
|
|
// entity eviction/re-entry and both full and partially occupied samples.
|
|
std::mt19937_64 rng(kSeed);
|
|
for (std::size_t trial = 0; trial < 48; ++trial) {
|
|
events.clear();
|
|
o.max_examples = 1 + trial % 24;
|
|
o.max_entity_share = trial % 3 == 0 ? 0.01 : trial % 3 == 1 ? 0.25 : 1.0;
|
|
o.sample_seed = kSeed + trial;
|
|
for (int i = 0; i < 72; ++i) {
|
|
const auto ts = kT0 + i * 61 * kSec;
|
|
const auto node = "process_" + std::to_string(9300 + i) + "_b_1";
|
|
const auto entity = std::string("/usr/bin/entity_") + static_cast<char>('a' + rng() % 7);
|
|
events.push_back(exec(ts, 9300 + i, node, "", entity, entity));
|
|
if (rng() % 3 == 0) events.push_back(scan(ts, 9300 + i, node, {"Teacher"}));
|
|
}
|
|
const auto sample = run(events, o);
|
|
const auto again = run(events, o);
|
|
const auto cap = std::max<std::size_t>(1, static_cast<std::size_t>(o.max_entity_share * o.max_examples));
|
|
verify(sample.examples.size() <= o.max_examples, "sample bound under mutation pressure");
|
|
verify(positives_in(sample) <= std::max<std::size_t>(1, o.max_examples / 5), "positive quota under mutation pressure");
|
|
verify(sample.example_positives == 0 || positives_in(sample) != 0, "positive cannot be lost to negatives");
|
|
PROPERTY(again.examples.size() == sample.examples.size());
|
|
std::unordered_map<std::string, std::size_t> counts;
|
|
for (std::size_t i = 0; i < sample.examples.size(); ++i) {
|
|
const auto& e = sample.examples[i];
|
|
verify(++counts[e.entity] <= cap, "combined entity cap under mutation pressure");
|
|
verify(e.start_ns == again.examples[i].start_ns && e.entity == again.examples[i].entity &&
|
|
e.label == again.examples[i].label, "same seed is deterministic");
|
|
}
|
|
}
|
|
PROPERTY(failures == 0);
|
|
std::cout << "R14 parent audit regressions: PASSED" << std::endl;
|
|
}
|
|
|
|
void no_partial_replay(const ReplayReport& r, const std::string& error, ReplayFailure failure) {
|
|
PROPERTY(!error.empty() && r.failure == failure);
|
|
PROPERTY(r.records == 0 && r.windows == 0 && r.examples.empty());
|
|
PROPERTY(r.top_windows.empty() && r.detected.empty() && !r.examples_collected && !r.snapshot.checked);
|
|
PROPERTY(replay_report_json(r).empty() && replay_training_summary_json(r).empty());
|
|
}
|
|
|
|
void append_test_tail(const std::filesystem::path& path) {
|
|
const int fd = io::open(path.c_str(), O_WRONLY | O_APPEND | O_CLOEXEC);
|
|
PROPERTY(fd >= 0);
|
|
const char byte = 'x';
|
|
PROPERTY(io::write(fd, &byte, 1) == 1);
|
|
PROPERTY(io::close(fd) == 0);
|
|
}
|
|
|
|
void test_replay_completeness() {
|
|
namespace fs = std::filesystem;
|
|
const std::string sha(64, 'a');
|
|
auto o = no_warmup();
|
|
o.collect_examples = true;
|
|
std::string error;
|
|
for (const auto stream : {store::Stream::evidence, store::Stream::telemetry}) {
|
|
const auto dir = make_retract_store("completeness_" + std::to_string(static_cast<int>(stream)), sha, "/tmp/private/a", true);
|
|
append_label(dir.string(), LabelPolarity::positive, "quarantine", sha, "", "operator", "private-rule");
|
|
const auto load = store::load_manifest(dir.string());
|
|
const auto it = std::find_if(load.manifest.segments.begin(), load.manifest.segments.end(),
|
|
[&](const auto& seg) { return seg.stream == stream; });
|
|
PROPERTY(it != load.manifest.segments.end());
|
|
const auto path = dir / (stream == store::Stream::evidence ? "evidence" : "telemetry") / it->file;
|
|
fs::rename(path, path.string() + ".saved");
|
|
const auto scan = store::scan_store_read_only(dir.string(), stream, [](uint32_t, std::string_view) {});
|
|
PROPERTY(scan.exists && scan.listed_segments != 0 && scan.skipped_segments == 1);
|
|
PROPERTY(scan.report.io_error.empty()); // ordinary readers still skip vanished segments
|
|
const auto missing = replay_store(dir.string(), o, error);
|
|
no_partial_replay(missing, error, ReplayFailure::unavailable);
|
|
fs::remove_all(dir);
|
|
}
|
|
const auto dir = make_retract_store("sealed_completeness", sha, "/tmp/private/a", true);
|
|
{
|
|
store::StoreOptions so;
|
|
so.dir = dir.string();
|
|
store::StoreOpenReport opened;
|
|
auto log = store::LogStore::open(so, opened, error);
|
|
PROPERTY(log && log->roll(store::Stream::telemetry));
|
|
const auto event = exec(kT0 + 1000 * kSec, 9900, "process_9900_b_1", "", "/usr/bin/new", "new");
|
|
PROPERTY(log->append(store::Stream::telemetry, event.type, event.payload));
|
|
PROPERTY(log->flush(store::Stream::telemetry, true));
|
|
}
|
|
auto load = store::load_manifest(dir.string());
|
|
auto sealed = std::find_if(load.manifest.segments.begin(), load.manifest.segments.end(),
|
|
[](const auto& seg) { return seg.stream == store::Stream::telemetry && seg.sealed; });
|
|
PROPERTY(sealed != load.manifest.segments.end());
|
|
const auto path = dir / "telemetry" / sealed->file;
|
|
// Count mismatches and absent sealed footers invalidate otherwise readable
|
|
// records. An ordinary active tail is tested separately below.
|
|
++sealed->records;
|
|
++load.manifest.generation;
|
|
PROPERTY(store::publish_manifest(dir.string(), load.manifest, error));
|
|
no_partial_replay(replay_store(dir.string(), o, error), error, ReplayFailure::invalid);
|
|
--sealed->records;
|
|
++load.manifest.generation;
|
|
PROPERTY(store::publish_manifest(dir.string(), load.manifest, error));
|
|
fs::resize_file(path, fs::file_size(path) - 1);
|
|
no_partial_replay(replay_store(dir.string(), o, error), error, ReplayFailure::invalid);
|
|
// Best-effort readers continue delivering a valid later segment even when
|
|
// the earlier manifest-listed segment cannot be opened.
|
|
fs::rename(path, path.string() + ".saved");
|
|
uint64_t delivered = 0;
|
|
const auto partial = store::scan_store_read_only(dir.string(), store::Stream::telemetry,
|
|
[&](uint32_t, std::string_view) { ++delivered; });
|
|
PROPERTY(delivered != 0 && partial.skipped_segments == 1 && partial.report.io_error.empty());
|
|
no_partial_replay(replay_store(dir.string(), o, error), error, ReplayFailure::unavailable);
|
|
fs::remove_all(dir);
|
|
std::cout << "R16 incomplete inputs export no partial replay: PASSED" << std::endl;
|
|
}
|
|
|
|
void test_snapshot_summary() {
|
|
namespace fs = std::filesystem;
|
|
const std::string sha(64, 'b');
|
|
std::string error;
|
|
// Fixed-bucket observations must not alter resolution, seeded sampling or
|
|
// leak host-controlled strings. Include zero/extreme clocks and overlap.
|
|
auto dir = make_retract_store("summary_labels", sha, "/tmp/host-secret/program", true);
|
|
append_label(dir.string(), LabelPolarity::positive, "quarantine", sha, "", "operator", "host-secret-rule");
|
|
append_label(dir.string(), LabelPolarity::negative, "host-secret-action", sha, "", "host-secret-source", "");
|
|
append_label(dir.string(), LabelPolarity::negative, "allowlist-add", "", "", "operator", "", 0);
|
|
append_label(dir.string(), LabelPolarity::retracted, "allowlist-remove", "", "", "host-secret-source", "", INT64_MIN);
|
|
append_label(dir.string(), LabelPolarity::positive, "host-secret-action", sha, "", "host-secret-source", "", INT64_MAX);
|
|
auto o = no_warmup();
|
|
o.collect_examples = true;
|
|
o.sample_seed = kSeed;
|
|
const auto ordinary = replay_store(dir.string(), o, error);
|
|
PROPERTY(error.empty());
|
|
o.training_summary = true;
|
|
const auto summary = replay_store(dir.string(), o, error);
|
|
PROPERTY(error.empty() && summary.snapshot.checked && summary.examples_collected);
|
|
PROPERTY(summary.examples.size() == ordinary.examples.size());
|
|
for (std::size_t i = 0; i < summary.examples.size(); ++i) {
|
|
PROPERTY(summary.examples[i].x == ordinary.examples[i].x && summary.examples[i].label == ordinary.examples[i].label);
|
|
PROPERTY(summary.examples[i].start_ns == ordinary.examples[i].start_ns && summary.examples[i].entity == ordinary.examples[i].entity);
|
|
}
|
|
PROPERTY(summary.labels.records == 5 && summary.labels.missing_hash == 2 && summary.labels.idless_allowlist == 2);
|
|
PROPERTY((summary.labels.polarity == std::array<uint64_t, 3>{2, 2, 1}));
|
|
PROPERTY((summary.labels.provenance == std::array<uint64_t, 3>{2, 0, 3}));
|
|
PROPERTY(summary.labels.timestamp_known == 3 && summary.labels.timestamp_unknown == 2);
|
|
PROPERTY(summary.labels.last_ns == INT64_MAX && summary.labels.resolved_positive_hashes == 1);
|
|
PROPERTY(summary.labels.resolved_negative_hashes == 1 && summary.labels.overlap_hashes == 1);
|
|
const auto json = replay_training_summary_json(summary);
|
|
PROPERTY(json.find("host-secret") == std::string::npos && json.find(sha) == std::string::npos);
|
|
PROPERTY(json == replay_training_summary_json(replay_store(dir.string(), o, error)) && error.empty());
|
|
fs::remove_all(dir);
|
|
|
|
for (const bool change_manifest : {false, true}) {
|
|
dir = make_retract_store(change_manifest ? "snapshot_generation" : "snapshot_extent", sha, "/tmp/private/a", false);
|
|
auto load = store::load_manifest(dir.string());
|
|
const auto telemetry = std::find_if(load.manifest.segments.begin(), load.manifest.segments.end(),
|
|
[](const auto& seg) { return seg.stream == store::Stream::telemetry; });
|
|
PROPERTY(telemetry != load.manifest.segments.end());
|
|
const auto path = dir / "telemetry" / telemetry->file;
|
|
bool changed = false;
|
|
// A trusted in-process observation seam, never exposed by the CLI,
|
|
// gives deterministic extent/generation changes without a racing writer.
|
|
o.before_snapshot_recheck = [&] {
|
|
if (changed) return;
|
|
changed = true;
|
|
if (change_manifest) {
|
|
++load.manifest.generation;
|
|
PROPERTY(store::publish_manifest(dir.string(), load.manifest, error));
|
|
} else append_test_tail(path);
|
|
};
|
|
const auto changed_report = replay_store(dir.string(), o, error);
|
|
PROPERTY(changed);
|
|
no_partial_replay(changed_report, error, ReplayFailure::changed);
|
|
fs::remove_all(dir);
|
|
}
|
|
o.before_snapshot_recheck = [] { throw 42; };
|
|
dir = make_retract_store("snapshot_observer_fault", sha, "/tmp/private/a", false);
|
|
no_partial_replay(replay_store(dir.string(), o, error), error, ReplayFailure::unavailable);
|
|
fs::remove_all(dir);
|
|
o.before_snapshot_recheck = {};
|
|
dir = make_retract_store("active_tail_summary", sha, "/tmp/private/a", false);
|
|
const auto load = store::load_manifest(dir.string());
|
|
const auto telemetry = std::find_if(load.manifest.segments.begin(), load.manifest.segments.end(),
|
|
[](const auto& seg) { return seg.stream == store::Stream::telemetry; });
|
|
PROPERTY(telemetry != load.manifest.segments.end());
|
|
append_test_tail(dir / "telemetry" / telemetry->file);
|
|
o.training_summary = false;
|
|
const auto active = replay_store(dir.string(), o, error);
|
|
PROPERTY(error.empty() && active.records != 0);
|
|
o.training_summary = true;
|
|
no_partial_replay(replay_store(dir.string(), o, error), error, ReplayFailure::unavailable);
|
|
fs::remove_all(dir);
|
|
|
|
dir = make_retract_store("fragment_summary", sha, "/tmp/private/a", false);
|
|
for (const char* name : {"MANIFEST", "MANIFEST.prev", "MANIFEST.tmp"}) {
|
|
const auto path = dir / name;
|
|
if (fs::exists(path)) fs::rename(path, path.string() + ".saved");
|
|
}
|
|
o.training_summary = false;
|
|
const auto fragment = replay_store(dir.string(), o, error);
|
|
PROPERTY(error.empty() && fragment.records != 0);
|
|
o.training_summary = true;
|
|
no_partial_replay(replay_store(dir.string(), o, error), error, ReplayFailure::unavailable);
|
|
fs::remove_all(dir);
|
|
std::cout << "R17 aggregate labels and strict cooperative snapshots: PASSED" << std::endl;
|
|
}
|
|
|
|
std::string read_whole_file(const std::filesystem::path& path) {
|
|
const int fd = io::open(path.c_str(), O_RDONLY | O_CLOEXEC);
|
|
PROPERTY(fd >= 0);
|
|
struct stat st {};
|
|
PROPERTY(io::fstat(fd, &st) == 0);
|
|
std::string out(static_cast<std::size_t>(st.st_size), '\0');
|
|
std::size_t got = 0;
|
|
while (got < out.size()) {
|
|
const ssize_t n = io::read(fd, out.data() + got, out.size() - got);
|
|
PROPERTY(n > 0);
|
|
got += static_cast<std::size_t>(n);
|
|
}
|
|
PROPERTY(io::close(fd) == 0);
|
|
return out;
|
|
}
|
|
|
|
void write_whole_file(const std::filesystem::path& path, std::string_view bytes) {
|
|
const int fd = io::open(path.c_str(), O_WRONLY | O_TRUNC | O_CLOEXEC);
|
|
PROPERTY(fd >= 0);
|
|
std::size_t done = 0;
|
|
while (done < bytes.size()) {
|
|
const ssize_t n = io::write(fd, bytes.data() + done, bytes.size() - done);
|
|
PROPERTY(n > 0);
|
|
done += static_cast<std::size_t>(n);
|
|
}
|
|
PROPERTY(io::close(fd) == 0);
|
|
}
|
|
|
|
uint32_t block_le32(std::string_view bytes, std::size_t at) {
|
|
uint32_t value = 0;
|
|
for (int i = 0; i < 4; ++i)
|
|
value |= static_cast<uint32_t>(static_cast<unsigned char>(bytes[at + i])) << (8 * i);
|
|
return value;
|
|
}
|
|
|
|
void block_put_le32(std::string& bytes, std::size_t at, uint32_t value) {
|
|
for (int i = 0; i < 4; ++i) bytes[at + i] = static_cast<char>((value >> (8 * i)) & 0xFFu);
|
|
}
|
|
|
|
// The first block starts right after the segment header.
|
|
constexpr std::size_t kFirstBlock = store::kSegmentHeaderBytes;
|
|
|
|
/// Makes the first block of `file` need a dictionary this build does not have
|
|
/// (the header stays self-consistent, so it is not corruption).
|
|
void require_missing_dictionary(std::string& file) {
|
|
PROPERTY(file.size() > kFirstBlock + store::kBlockHeaderBytes);
|
|
PROPERTY(block_le32(file, kFirstBlock) == store::kBlockMagic);
|
|
PROPERTY(file[kFirstBlock + 1 + 4] == static_cast<char>(store::BlockCodec::zstd));
|
|
block_put_le32(file, kFirstBlock + 16, 0x5EEDu);
|
|
block_put_le32(file, kFirstBlock + 24, store::crc32c(file.data() + kFirstBlock, 24));
|
|
}
|
|
|
|
/// Corrupts the magic of the first block: a complete header that is not valid.
|
|
void break_block_header(std::string& file) {
|
|
PROPERTY(file.size() > kFirstBlock + store::kBlockHeaderBytes);
|
|
file[kFirstBlock] = static_cast<char>(file[kFirstBlock] ^ 0x80);
|
|
}
|
|
|
|
std::filesystem::path stream_segment(const std::string& dir, const std::string& name, bool sealed) {
|
|
const auto load = store::load_manifest(dir);
|
|
const auto stream = name == "evidence" ? store::Stream::evidence : store::Stream::telemetry;
|
|
for (const auto& seg : load.manifest.segments) {
|
|
if (seg.stream == stream && seg.sealed == sealed) return std::filesystem::path(dir) / name / seg.file;
|
|
}
|
|
return {};
|
|
}
|
|
|
|
void test_decoder_availability() {
|
|
namespace fs = std::filesystem;
|
|
const std::string sha(64, 'd');
|
|
auto o = no_warmup();
|
|
o.collect_examples = true;
|
|
std::string error;
|
|
|
|
// A block needing a dictionary this build lacks is a missing decoder, not
|
|
// an empty stream and not corruption (ordinary replay and summary alike).
|
|
{
|
|
const auto dir = make_retract_store("decoder_active", sha, "/tmp/private/a", true);
|
|
const auto path = stream_segment(dir.string(), "telemetry", false);
|
|
PROPERTY(!path.empty());
|
|
auto file = read_whole_file(path);
|
|
require_missing_dictionary(file);
|
|
write_whole_file(path, file);
|
|
const auto scan = store::scan_store_read_only(dir.string(), store::Stream::telemetry,
|
|
[](uint32_t, std::string_view) {});
|
|
PROPERTY(scan.unavailable_segments == 1 && scan.malformed_tail_segments == 0);
|
|
PROPERTY(scan.invalid_segments == 0 && scan.report.torn_tail == false);
|
|
no_partial_replay(replay_store(dir.string(), o, error), error, ReplayFailure::unavailable);
|
|
o.training_summary = true;
|
|
no_partial_replay(replay_store(dir.string(), o, error), error, ReplayFailure::unavailable);
|
|
o.training_summary = false;
|
|
fs::remove_all(dir);
|
|
}
|
|
|
|
// The same block in a sealed segment must not be misread as bad data: the
|
|
// footer/manifest counts it explains are not comparable without the block.
|
|
{
|
|
const auto dir = make_retract_store("decoder_sealed", sha, "/tmp/private/a", true);
|
|
{
|
|
store::StoreOptions so;
|
|
so.dir = dir.string();
|
|
store::StoreOpenReport opened;
|
|
auto log = store::LogStore::open(so, opened, error);
|
|
PROPERTY(log && log->roll(store::Stream::telemetry));
|
|
const auto event = exec(kT0 + 1000 * kSec, 9900, "process_9900_b_1", "", "/usr/bin/new", "new");
|
|
PROPERTY(log->append(store::Stream::telemetry, event.type, event.payload));
|
|
PROPERTY(log->flush(store::Stream::telemetry, true));
|
|
}
|
|
const auto path = stream_segment(dir.string(), "telemetry", true);
|
|
PROPERTY(!path.empty());
|
|
auto file = read_whole_file(path);
|
|
require_missing_dictionary(file);
|
|
write_whole_file(path, file);
|
|
const auto scan = store::scan_store_read_only(dir.string(), store::Stream::telemetry,
|
|
[](uint32_t, std::string_view) {});
|
|
PROPERTY(scan.unavailable_segments == 1 && scan.malformed_tail_segments == 0);
|
|
PROPERTY(scan.invalid_segments == 0); // counts are not checkable here
|
|
no_partial_replay(replay_store(dir.string(), o, error), error, ReplayFailure::unavailable);
|
|
fs::remove_all(dir);
|
|
}
|
|
|
|
// A complete block header that fails validation is corrupt data: unlike a
|
|
// short in-flight tail it is not tolerated by ordinary replay.
|
|
{
|
|
const auto dir = make_retract_store("decoder_badheader", sha, "/tmp/private/a", true);
|
|
const auto path = stream_segment(dir.string(), "telemetry", false);
|
|
PROPERTY(!path.empty());
|
|
auto file = read_whole_file(path);
|
|
break_block_header(file);
|
|
write_whole_file(path, file);
|
|
const auto scan = store::scan_store_read_only(dir.string(), store::Stream::telemetry,
|
|
[](uint32_t, std::string_view) {});
|
|
PROPERTY(scan.malformed_tail_segments == 1 && scan.unavailable_segments == 0);
|
|
no_partial_replay(replay_store(dir.string(), o, error), error, ReplayFailure::invalid);
|
|
o.training_summary = true;
|
|
no_partial_replay(replay_store(dir.string(), o, error), error, ReplayFailure::invalid);
|
|
o.training_summary = false;
|
|
fs::remove_all(dir);
|
|
}
|
|
|
|
// An earlier unusable segment must not be masked by a later short tail
|
|
// (the aggregate tail_error keeps only the last one).
|
|
for (const bool missing_dictionary : {false, true}) {
|
|
const auto dir = make_retract_store(missing_dictionary ? "masked_decoder" : "masked_header", sha,
|
|
"/tmp/private/a", true);
|
|
{
|
|
store::StoreOptions so;
|
|
so.dir = dir.string();
|
|
store::StoreOpenReport opened;
|
|
auto log = store::LogStore::open(so, opened, error);
|
|
PROPERTY(log && log->roll(store::Stream::telemetry));
|
|
const auto event = exec(kT0 + 1000 * kSec, 9900, "process_9900_b_1", "", "/usr/bin/new", "new");
|
|
PROPERTY(log->append(store::Stream::telemetry, event.type, event.payload));
|
|
PROPERTY(log->flush(store::Stream::telemetry, true));
|
|
}
|
|
const auto sealed = stream_segment(dir.string(), "telemetry", true);
|
|
const auto active = stream_segment(dir.string(), "telemetry", false);
|
|
PROPERTY(!sealed.empty() && !active.empty());
|
|
auto file = read_whole_file(sealed);
|
|
if (missing_dictionary) require_missing_dictionary(file);
|
|
else break_block_header(file);
|
|
write_whole_file(sealed, file);
|
|
append_test_tail(active); // the later, tolerated short tail
|
|
const auto scan = store::scan_store_read_only(dir.string(), store::Stream::telemetry,
|
|
[](uint32_t, std::string_view) {});
|
|
PROPERTY(scan.report.tail_error == store::BlockError::truncated);
|
|
if (missing_dictionary) {
|
|
PROPERTY(scan.unavailable_segments == 1 && scan.malformed_tail_segments == 0);
|
|
no_partial_replay(replay_store(dir.string(), o, error), error, ReplayFailure::unavailable);
|
|
} else {
|
|
no_partial_replay(replay_store(dir.string(), o, error), error, ReplayFailure::invalid);
|
|
}
|
|
fs::remove_all(dir);
|
|
}
|
|
|
|
// Positive control: a genuine in-flight tail stays readable to ordinary
|
|
// replay (and is still refused for a summary snapshot).
|
|
{
|
|
const auto dir = make_retract_store("decoder_inflight", sha, "/tmp/private/a", true);
|
|
const auto path = stream_segment(dir.string(), "telemetry", false);
|
|
PROPERTY(!path.empty());
|
|
append_test_tail(path);
|
|
const auto scan = store::scan_store_read_only(dir.string(), store::Stream::telemetry,
|
|
[](uint32_t, std::string_view) {});
|
|
PROPERTY(scan.malformed_tail_segments == 0 && scan.unavailable_segments == 0);
|
|
PROPERTY(scan.report.torn_tail && scan.report.tail_error == store::BlockError::truncated);
|
|
const auto ordinary = replay_store(dir.string(), o, error);
|
|
PROPERTY(error.empty() && ordinary.records != 0 && ordinary.failure == ReplayFailure::none);
|
|
o.training_summary = true;
|
|
no_partial_replay(replay_store(dir.string(), o, error), error, ReplayFailure::unavailable);
|
|
o.training_summary = false;
|
|
fs::remove_all(dir);
|
|
}
|
|
std::cout << "R18 missing decoder and malformed tails are not empty inputs: PASSED" << std::endl;
|
|
}
|
|
|
|
// Stage 7a: LogGC must not change what replay detects. Replays a full store and the same store after
|
|
// reduce_telemetry(): dead benign subtrees are dropped (fewer execs), detections are identical.
|
|
void test_reduce_replay_equivalence() {
|
|
namespace fs = std::filesystem;
|
|
constexpr int64_t kDayNs = 86'400 * kSec;
|
|
auto ev = routine(20, kT0);
|
|
ev.push_back(exec(kT0 + 40 * 61 * kSec, 700, "process_700_b_1", "process_1_b_1", "/tmp/evil", "evil"));
|
|
ev.push_back(scan(kT0 + 40 * 61 * kSec + kSec, 700, "process_700_b_1", {"Evil_Rule"}));
|
|
for (int i = 0; i < 3; ++i) { // live (distinct nodes, days later): sets the horizon
|
|
const auto ts = kT0 + 5 * kDayNs + i * 61 * kSec;
|
|
const std::string sh = "process_" + std::to_string(5000 + i) + "_b_1";
|
|
ev.push_back(exec(ts, 5000 + i, sh, "process_1_b_1", "/usr/bin/zsh", "zsh"));
|
|
ev.push_back(exec(ts + kSec, 6000 + i, "process_" + std::to_string(6000 + i) + "_b_1", sh, "/usr/bin/cat", "cat x"));
|
|
}
|
|
|
|
const auto root = fs::temp_directory_path() / ("strawberry_reduce_replay_" + std::to_string(::getpid()));
|
|
std::error_code ec;
|
|
fs::remove_all(root, ec);
|
|
fs::create_directories(root, ec);
|
|
const std::string full_dir = (root / "full").string();
|
|
const std::string reduced_dir = (root / "reduced").string();
|
|
for (const auto& dir : {full_dir, reduced_dir}) {
|
|
Database db;
|
|
PROPERTY(db.open(dir));
|
|
auto copy = ev;
|
|
PROPERTY(db.append_telemetry(copy, true));
|
|
db.close();
|
|
}
|
|
store::ReduceReport reduce;
|
|
{
|
|
store::StoreOpenReport open_report;
|
|
std::string error;
|
|
auto log = store::LogStore::open(store::default_store_options(reduced_dir), open_report, error);
|
|
PROPERTY(log != nullptr);
|
|
PROPERTY(log->roll(store::Stream::telemetry)); // seal: only sealed telemetry is reduced
|
|
PROPERTY(log->reduce_telemetry(1, reduce));
|
|
}
|
|
PROPERTY(reduce.problems.empty() && reduce.subtrees_dropped > 0 && reduce.records_after < reduce.records_before);
|
|
|
|
std::string error;
|
|
const auto full = replay_store(full_dir, no_warmup(), error);
|
|
PROPERTY(error.empty());
|
|
const auto reduced = replay_store(reduced_dir, no_warmup(), error);
|
|
PROPERTY(error.empty());
|
|
PROPERTY(reduced.execs < full.execs); // the compute proxy went down
|
|
PROPERTY(reduced.detected_windows == full.detected_windows && full.detected_windows == 1);
|
|
PROPERTY(reduced.late_detected_windows == full.late_detected_windows);
|
|
PROPERTY(reduced.detected.size() == full.detected.size());
|
|
for (std::size_t i = 0; i < full.detected.size(); ++i) {
|
|
PROPERTY(reduced.detected[i].root_node == full.detected[i].root_node);
|
|
}
|
|
fs::remove_all(root, ec);
|
|
std::cout << "R-7a LogGC replay equivalence: PASSED" << std::endl;
|
|
}
|
|
|
|
int main() {
|
|
std::cout << "replay properties, seed " << kSeed << std::endl;
|
|
test_reduce_replay_equivalence();
|
|
test_sampler_capacity_lifetime();
|
|
test_parent_audit_regressions();
|
|
test_replay_completeness();
|
|
test_snapshot_summary();
|
|
test_decoder_availability();
|
|
test_chain_and_pid_reuse();
|
|
test_order_and_determinism();
|
|
test_ranking_and_detection();
|
|
test_store_round_trip();
|
|
test_invalid_utf8();
|
|
test_warmup_and_time_invariance();
|
|
test_injection_evaluation();
|
|
test_late_labels();
|
|
test_attribution();
|
|
test_allowlisted_window_is_negative();
|
|
test_retraction_allowlist_remove();
|
|
test_retraction_quarantine_remove();
|
|
test_retraction_independent_positives();
|
|
test_retraction_multiple_entries();
|
|
test_retraction_readd();
|
|
test_examples_and_model_ranking();
|
|
test_positive_sampling_under_pressure();
|
|
test_positive_entity_cap();
|
|
test_positive_not_starved_by_entity_negatives();
|
|
test_policy_evaluation();
|
|
test_extreme_timestamps();
|
|
return 0;
|
|
}
|