782 lines
34 KiB
C++
782 lines
34 KiB
C++
#include <algorithm>
|
|
#include <chrono>
|
|
#include <cstdio>
|
|
#include <cstdlib>
|
|
#include <unistd.h>
|
|
#include <exception>
|
|
#include <filesystem>
|
|
#include <random>
|
|
#include <string>
|
|
#include <utility>
|
|
#include <vector>
|
|
|
|
#include "strawberry/database.hpp"
|
|
#include "strawberry/fleet_export.hpp"
|
|
#include "strawberry/learn/rarity.hpp"
|
|
#include "strawberry/learn/replay.hpp"
|
|
#include "strawberry/proc_read.hpp"
|
|
#include <fstream>
|
|
#include "strawberry/telemetry.hpp"
|
|
|
|
using namespace strawberry;
|
|
using namespace strawberry::fleet;
|
|
|
|
#define CHECK(cond) \
|
|
do { \
|
|
if (!(cond)) { \
|
|
std::fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond); \
|
|
std::exit(1); \
|
|
} \
|
|
} while (0)
|
|
|
|
using Visited = std::vector<std::pair<uint32_t, std::string>>;
|
|
|
|
static std::string temp_path(const char* tag) {
|
|
static int n = 0;
|
|
auto p = std::filesystem::temp_directory_path() /
|
|
("sb_fleet_export_" + std::to_string(::getpid()) + "_" + tag + "_" + std::to_string(n++) + ".seg");
|
|
std::filesystem::remove(p);
|
|
return p.string();
|
|
}
|
|
|
|
static ExportManifest make_manifest() {
|
|
ExportManifest m;
|
|
m.node_uid = "abc123";
|
|
m.generation_low = 10;
|
|
m.generation_high = 20;
|
|
m.time_low_ns = 100;
|
|
m.time_high_ns = 900;
|
|
m.created_unix_ns = 500;
|
|
m.record_count = 3;
|
|
return m;
|
|
}
|
|
|
|
static bool read_back(const std::string& path, ExportManifest& m, Visited& v) {
|
|
return read_export_bundle(path, m, [&](uint32_t t, std::string_view p) { v.emplace_back(t, std::string(p)); });
|
|
}
|
|
|
|
int main() {
|
|
const auto W = static_cast<uint32_t>(store::RecordType::export_window);
|
|
|
|
// a. round-trip
|
|
{
|
|
const ExportManifest m = make_manifest();
|
|
std::vector<store::PendingRecord> recs;
|
|
for (int i = 0; i < 3; ++i) recs.push_back({store::RecordType::export_window, "payload-" + std::to_string(i)});
|
|
const std::string path = temp_path("a");
|
|
std::string err;
|
|
CHECK(write_export_bundle(path, m, recs, err));
|
|
ExportManifest out;
|
|
Visited v;
|
|
CHECK(read_back(path, out, v));
|
|
CHECK(out.created_unix_ns == m.created_unix_ns);
|
|
CHECK(out.node_uid == m.node_uid);
|
|
CHECK(out.generation_low == 10 && out.generation_high == 20);
|
|
CHECK(out.time_low_ns == 100 && out.time_high_ns == 900);
|
|
CHECK(out.record_count == 3);
|
|
CHECK(v.size() == 3);
|
|
for (size_t i = 0; i < v.size(); ++i) {
|
|
CHECK(v[i].first == W);
|
|
CHECK(v[i].second == "payload-" + std::to_string(i));
|
|
}
|
|
std::filesystem::remove(path);
|
|
}
|
|
|
|
// b. unknown record type skipped, not errored
|
|
{
|
|
std::vector<store::PendingRecord> recs;
|
|
recs.push_back({store::RecordType::export_window, "k0"});
|
|
recs.push_back({static_cast<store::RecordType>(9999), "x"});
|
|
recs.push_back({store::RecordType::export_window, "k1"});
|
|
const std::string path = temp_path("b");
|
|
std::string err;
|
|
CHECK(write_export_bundle(path, make_manifest(), recs, err));
|
|
ExportManifest out;
|
|
Visited v;
|
|
CHECK(read_back(path, out, v));
|
|
size_t known = 0;
|
|
for (const auto& e : v)
|
|
if (e.first == W) ++known;
|
|
CHECK(known == 2);
|
|
std::filesystem::remove(path);
|
|
}
|
|
|
|
// c. truncation
|
|
{
|
|
std::vector<store::PendingRecord> recs;
|
|
for (int i = 0; i < 3; ++i) recs.push_back({store::RecordType::export_window, "payload-" + std::to_string(i)});
|
|
const std::string path = temp_path("c");
|
|
std::string err;
|
|
CHECK(write_export_bundle(path, make_manifest(), recs, err));
|
|
const auto size = std::filesystem::file_size(path);
|
|
std::filesystem::resize_file(path, size / 2);
|
|
ExportManifest out;
|
|
Visited v;
|
|
bool ok = true;
|
|
try {
|
|
ok = read_back(path, out, v);
|
|
} catch (...) {
|
|
CHECK(false);
|
|
}
|
|
CHECK(!ok);
|
|
std::filesystem::remove(path);
|
|
}
|
|
|
|
// d. bad magic / empty
|
|
{
|
|
store::FieldWriter w;
|
|
w.u64(1, 500);
|
|
w.fixed32(2, kExportMagic ^ 0xFFu);
|
|
w.u64(3, kExportSchemaVersion);
|
|
w.bytes(4, "abc123");
|
|
ExportManifest out;
|
|
CHECK(!decode_export_manifest(w.data(), out));
|
|
CHECK(!decode_export_manifest("", out));
|
|
// sanity: the good encoding decodes
|
|
CHECK(decode_export_manifest(encode_export_manifest(make_manifest()), out));
|
|
// schema newer than supported is rejected
|
|
store::FieldWriter w2;
|
|
w2.fixed32(2, kExportMagic);
|
|
w2.u64(3, kExportSchemaVersion + 1);
|
|
CHECK(!decode_export_manifest(w2.data(), out));
|
|
}
|
|
|
|
// e. decoder robustness property
|
|
{
|
|
const uint64_t seed = std::random_device{}();
|
|
std::printf("seed=%llu\n", static_cast<unsigned long long>(seed));
|
|
std::mt19937_64 rng(seed);
|
|
for (int i = 0; i < 5000; ++i) {
|
|
std::string s(rng() % 129, '\0');
|
|
for (auto& c : s) c = static_cast<char>(rng() & 0xFF);
|
|
ExportManifest out;
|
|
try {
|
|
(void)decode_export_manifest(s, out);
|
|
} catch (...) {
|
|
CHECK(false);
|
|
}
|
|
}
|
|
}
|
|
|
|
// f. payload records round-trip
|
|
{
|
|
ExportWindow w;
|
|
w.window_id = "win-hash-01";
|
|
w.start_ns = -12345;
|
|
w.last_ns = 987654321;
|
|
w.execs = 1; w.connects = 2; w.persistence_writes = 3; w.drop_writes = 4; w.ptraces = 5;
|
|
w.module_loads = 6; w.memfd_execs = 7; w.deleted_execs = 8; w.uid0_execs = 9; w.max_depth = 10;
|
|
for (std::size_t i = 0; i < kExportLocationClasses; ++i) w.exec_locations[i] = static_cast<uint32_t>(i + 1);
|
|
w.public_connects = 11; w.private_connects = 12; w.loopback_connects = 13;
|
|
w.low_port_connects = 14; w.high_port_connects = 15;
|
|
w.scans_too_large = 16; w.scans_unavailable = 17; w.scan_errors = 18;
|
|
w.max_bits = {0.5f, 1.5f, 2.5f, 3.5f, 4.5f};
|
|
w.top3_bits = 5.5f;
|
|
w.governor_level = 3;
|
|
w.budget_left = 0.25f;
|
|
w.token_hashes = {1, 2, 3, 0xdeadbeefcafeULL};
|
|
ExportWindow wo;
|
|
CHECK(decode_export_window(encode_export_window(w), wo));
|
|
CHECK(wo.window_id == w.window_id);
|
|
CHECK(wo.start_ns == w.start_ns && wo.last_ns == w.last_ns);
|
|
CHECK(wo.execs == w.execs && wo.connects == w.connects);
|
|
CHECK(wo.persistence_writes == w.persistence_writes && wo.drop_writes == w.drop_writes);
|
|
CHECK(wo.ptraces == w.ptraces && wo.module_loads == w.module_loads);
|
|
CHECK(wo.memfd_execs == w.memfd_execs && wo.deleted_execs == w.deleted_execs);
|
|
CHECK(wo.uid0_execs == w.uid0_execs && wo.max_depth == w.max_depth);
|
|
for (std::size_t i = 0; i < kExportLocationClasses; ++i) CHECK(wo.exec_locations[i] == w.exec_locations[i]);
|
|
CHECK(wo.public_connects == w.public_connects && wo.private_connects == w.private_connects);
|
|
CHECK(wo.loopback_connects == w.loopback_connects);
|
|
CHECK(wo.low_port_connects == w.low_port_connects && wo.high_port_connects == w.high_port_connects);
|
|
CHECK(wo.scans_too_large == w.scans_too_large && wo.scans_unavailable == w.scans_unavailable);
|
|
CHECK(wo.scan_errors == w.scan_errors);
|
|
for (std::size_t i = 0; i < kExportTupleKinds; ++i) CHECK(wo.max_bits[i] == w.max_bits[i]);
|
|
CHECK(wo.top3_bits == w.top3_bits && wo.governor_level == w.governor_level);
|
|
CHECK(wo.budget_left == w.budget_left);
|
|
CHECK(wo.token_hashes == w.token_hashes);
|
|
CHECK(!decode_export_window("", wo));
|
|
|
|
ExportWindowFeature fe;
|
|
fe.window_id = "win-hash-01";
|
|
for (std::size_t i = 0; i < kExportFeatureDim; ++i) fe.features[i] = static_cast<float>(i) * 0.25f;
|
|
ExportWindowFeature fo;
|
|
CHECK(decode_export_window_feature(encode_export_window_feature(fe), fo));
|
|
CHECK(fo.window_id == fe.window_id);
|
|
for (std::size_t i = 0; i < kExportFeatureDim; ++i) CHECK(fo.features[i] == fe.features[i]);
|
|
CHECK(!decode_export_window_feature("", fo));
|
|
|
|
ExportLabel lb;
|
|
lb.window_id = "win-hash-01";
|
|
lb.timestamp_ns = 555666777;
|
|
lb.polarity = LabelPolarity::negative;
|
|
lb.source = ExportLabelSource::op;
|
|
lb.action = 7;
|
|
lb.uid = 1001;
|
|
lb.sha256 = "ff00ff00ff00ff00ff00ff00ff00ff00ff00ff00ff00ff00ff00ff00ff00ff00";
|
|
lb.origin_node_hash = 0xA1B2C3D4E5F60718ULL; // 3.5d attribution hashes round-trip
|
|
lb.origin_exe_hash = 0x1122334455667788ULL;
|
|
ExportLabel lo;
|
|
CHECK(decode_export_label(encode_export_label(lb), lo));
|
|
CHECK(lo.window_id == lb.window_id && lo.timestamp_ns == lb.timestamp_ns);
|
|
CHECK(lo.polarity == lb.polarity && lo.source == lb.source);
|
|
CHECK(lo.action == lb.action && lo.uid == lb.uid && lo.sha256 == lb.sha256);
|
|
CHECK(lo.origin_node_hash == lb.origin_node_hash && lo.origin_exe_hash == lb.origin_exe_hash);
|
|
CHECK(!decode_export_label("", lo));
|
|
// back-compat: a label with no origin set decodes with zero hashes (old bundles have no tags 8/9)
|
|
ExportLabel noorig;
|
|
noorig.window_id = "win-hash-02";
|
|
ExportLabel noout;
|
|
CHECK(decode_export_label(encode_export_label(noorig), noout));
|
|
CHECK(noout.origin_node_hash == 0 && noout.origin_exe_hash == 0);
|
|
}
|
|
|
|
// g. payload decoder robustness property
|
|
{
|
|
const uint64_t seed = std::random_device{}();
|
|
std::printf("payload seed=%llu\n", static_cast<unsigned long long>(seed));
|
|
std::mt19937_64 rng(seed);
|
|
for (int i = 0; i < 4000; ++i) {
|
|
std::string s(rng() % 161, '\0');
|
|
for (auto& c : s) c = static_cast<char>(rng() & 0xFF);
|
|
ExportWindow a;
|
|
ExportWindowFeature b;
|
|
ExportLabel c;
|
|
try {
|
|
(void)decode_export_window(s, a);
|
|
(void)decode_export_window_feature(s, b);
|
|
(void)decode_export_label(s, c);
|
|
} catch (...) {
|
|
CHECK(false);
|
|
}
|
|
}
|
|
}
|
|
|
|
// h. rarity delta record
|
|
{
|
|
ExportRaritySketch r;
|
|
r.kind = 3;
|
|
r.width_log2 = 4;
|
|
r.depth = 4;
|
|
r.total = 123.5;
|
|
r.cells.resize(64);
|
|
for (std::size_t i = 0; i < r.cells.size(); ++i) r.cells[i] = static_cast<float>(i) * 0.5f;
|
|
ExportRaritySketch ro;
|
|
CHECK(decode_export_rarity(encode_export_rarity(r), ro));
|
|
CHECK(ro.kind == r.kind && ro.width_log2 == r.width_log2 && ro.depth == r.depth);
|
|
CHECK(ro.total == 123.5);
|
|
CHECK(ro.cells == r.cells);
|
|
|
|
ExportRaritySketch bad = r;
|
|
bad.cells.resize(63);
|
|
CHECK(!decode_export_rarity(encode_export_rarity(bad), ro));
|
|
bad = r;
|
|
bad.width_log2 = static_cast<uint32_t>(kMaxRarityWidthLog2 + 1);
|
|
CHECK(!decode_export_rarity(encode_export_rarity(bad), ro));
|
|
bad = r;
|
|
bad.depth = 0;
|
|
CHECK(!decode_export_rarity(encode_export_rarity(bad), ro));
|
|
bad = r;
|
|
bad.kind = static_cast<uint8_t>(kExportTupleKinds);
|
|
CHECK(!decode_export_rarity(encode_export_rarity(bad), ro));
|
|
CHECK(!decode_export_rarity("", ro));
|
|
|
|
const uint64_t seed = std::random_device{}();
|
|
std::printf("rarity seed=%llu\n", static_cast<unsigned long long>(seed));
|
|
std::mt19937_64 rng(seed);
|
|
for (int i = 0; i < 3000; ++i) {
|
|
std::string s(rng() % 201, '\0');
|
|
for (auto& c : s) c = static_cast<char>(rng() & 0xFF);
|
|
ExportRaritySketch x;
|
|
try {
|
|
(void)decode_export_rarity(s, x);
|
|
} catch (...) {
|
|
CHECK(false);
|
|
}
|
|
}
|
|
}
|
|
|
|
{
|
|
// extraction: Replayer -> append_window_records -> bundle round-trip
|
|
constexpr int64_t kSec = 1'000'000'000;
|
|
constexpr int64_t kT0 = 1'800'000'000 * kSec;
|
|
using Ev = std::pair<store::RecordType, std::string>;
|
|
std::vector<Ev> evs;
|
|
auto add_exec = [&](int64_t ts, int pid, const std::string& node, const std::string& parent,
|
|
const std::string& exe, const std::string& cmd) {
|
|
ProcessExecRecord r;
|
|
r.timestamp_ns = ts;
|
|
r.pid = pid;
|
|
r.process_node_id = node;
|
|
r.parent_node_id = parent;
|
|
r.exe = exe;
|
|
r.cmdline = cmd;
|
|
r.source = "live";
|
|
evs.push_back({store::RecordType::process_exec, encode_process_exec(r)});
|
|
};
|
|
auto add_connect = [&](int64_t ts, int pid, const std::string& addr, int port) {
|
|
BehaviorEvent b;
|
|
b.type = BehaviorEvent::Type::connect;
|
|
b.timestamp_ns = static_cast<uint64_t>(ts);
|
|
b.pid = pid;
|
|
b.family = 2;
|
|
b.address = addr;
|
|
b.port = static_cast<uint16_t>(port);
|
|
evs.push_back({store::RecordType::connect, encode_behavior(b)});
|
|
};
|
|
auto add_scan = [&](int64_t ts, int pid, const std::string& node, std::vector<std::string> rules) {
|
|
ScanResultRecord sr;
|
|
sr.timestamp_ns = ts;
|
|
sr.pid = pid;
|
|
sr.process_node_id = node;
|
|
sr.rules = std::move(rules);
|
|
evs.push_back({store::RecordType::scan_result, encode_scan_result(sr)});
|
|
};
|
|
auto add_write = [&](int64_t ts, int pid, const std::string& path) {
|
|
FileWriteEvent fw;
|
|
fw.pid = pid;
|
|
fw.path = path;
|
|
fw.category = "drop";
|
|
fw.timestamp_ns = static_cast<uint64_t>(ts);
|
|
evs.push_back({store::RecordType::file_write, encode_file_write(fw)});
|
|
};
|
|
// detected window, NO in-set write for its flagged binary -> positive label has origin 0
|
|
add_exec(kT0, 100, "nodeA", "", "/tmp/evil", "/tmp/evil -x");
|
|
add_connect(kT0 + kSec, 100, "8.8.8.8", 443);
|
|
add_scan(kT0 + 2 * kSec, 100, "nodeA", {"Bad_Rule"});
|
|
// plain window
|
|
add_exec(kT0 + 10 * kSec, 200, "nodeB", "", "/usr/bin/ls", "ls -l");
|
|
// 3.5d attributed window: dropper nodeD writes /tmp/dropped, victim nodeC execs it and is
|
|
// detected -> nodeC's positive label carries a non-zero dropper origin hash.
|
|
add_exec(kT0 + 20 * kSec, 300, "nodeD", "", "/usr/bin/curl", "curl -o /tmp/dropped");
|
|
add_write(kT0 + 21 * kSec, 300, "/tmp/dropped");
|
|
add_exec(kT0 + 22 * kSec, 400, "nodeC", "", "/tmp/dropped", "/tmp/dropped");
|
|
add_scan(kT0 + 23 * kSec, 400, "nodeC", {"Bad_Rule"});
|
|
|
|
learn::ReplayOptions o;
|
|
o.live = true;
|
|
o.warmup_ns = 0;
|
|
std::vector<store::PendingRecord> recs;
|
|
o.on_window_closed = [&](const learn::ReplayWindow& w, const learn::WindowSummary& s) {
|
|
append_window_records(w, s, recs);
|
|
};
|
|
learn::Replayer r(o);
|
|
for (const auto& e : evs) r.feed(e.first, e.second);
|
|
r.finish();
|
|
|
|
std::size_t nwin = 0, nfeat = 0, nlabel = 0;
|
|
std::vector<std::string> win_ids, feat_ids;
|
|
bool saw_positive = false, saw_tokens = false, saw_origin = false;
|
|
for (const auto& pr : recs) {
|
|
if (pr.type == store::RecordType::export_window) {
|
|
ExportWindow ew;
|
|
CHECK(decode_export_window(pr.payload, ew));
|
|
++nwin;
|
|
win_ids.push_back(ew.window_id);
|
|
if (!ew.token_hashes.empty()) saw_tokens = true;
|
|
} else if (pr.type == store::RecordType::export_window_feature) {
|
|
ExportWindowFeature ef;
|
|
CHECK(decode_export_window_feature(pr.payload, ef));
|
|
CHECK(ef.features.size() == 256);
|
|
++nfeat;
|
|
feat_ids.push_back(ef.window_id);
|
|
} else if (pr.type == store::RecordType::export_label) {
|
|
ExportLabel el;
|
|
CHECK(decode_export_label(pr.payload, el));
|
|
++nlabel;
|
|
if (el.polarity == LabelPolarity::positive) saw_positive = true;
|
|
// 3.5d: a positive label with a non-zero origin hash is the attributed (dropper) window;
|
|
// a suppressed/negative label must never carry an origin.
|
|
if (el.polarity == LabelPolarity::positive && el.origin_node_hash != 0) saw_origin = true;
|
|
if (el.polarity == LabelPolarity::negative) CHECK(el.origin_node_hash == 0 && el.origin_exe_hash == 0);
|
|
}
|
|
}
|
|
CHECK(nwin >= 1);
|
|
CHECK(nwin == nfeat);
|
|
CHECK(win_ids == feat_ids);
|
|
CHECK(nlabel >= 1);
|
|
CHECK(saw_positive);
|
|
CHECK(saw_tokens);
|
|
CHECK(saw_origin); // 3.5d: the dropper->exec window's positive label carries an origin hash
|
|
|
|
ExportManifest man;
|
|
man.node_uid = "n1";
|
|
const std::string path = temp_path("extract");
|
|
std::string err;
|
|
CHECK(write_export_bundle(path, man, recs, err));
|
|
ExportManifest back;
|
|
std::size_t visited = 0;
|
|
CHECK(read_export_bundle(path, back, [&](uint32_t, std::string_view) {
|
|
++visited;
|
|
}));
|
|
CHECK(visited == recs.size());
|
|
CHECK(back.node_uid == "n1");
|
|
std::filesystem::remove(path);
|
|
}
|
|
|
|
{
|
|
// e2e: export_store on a real store dir built via Database (closes 1e deferred validation)
|
|
constexpr int64_t kSec = 1'000'000'000;
|
|
constexpr int64_t kT0 = 1'800'000'000 * kSec;
|
|
namespace fs = std::filesystem;
|
|
const auto dir = fs::temp_directory_path() / ("sb_fleet_export_e2e_" + std::to_string(::getpid()));
|
|
std::error_code ec;
|
|
fs::remove_all(dir, ec);
|
|
{
|
|
std::vector<TelemetryEvent> evs;
|
|
ProcessExecRecord a;
|
|
a.timestamp_ns = kT0;
|
|
a.pid = 100;
|
|
a.process_node_id = "nodeA";
|
|
a.exe = "/tmp/evil";
|
|
a.cmdline = "/tmp/evil -x";
|
|
a.source = "live";
|
|
evs.push_back({store::RecordType::process_exec, encode_process_exec(a)});
|
|
ScanResultRecord sr;
|
|
sr.timestamp_ns = kT0 + 2 * kSec;
|
|
sr.pid = 100;
|
|
sr.process_node_id = "nodeA";
|
|
sr.rules = {"Bad_Rule"};
|
|
evs.push_back({store::RecordType::scan_result, encode_scan_result(sr)});
|
|
ProcessExecRecord b;
|
|
b.timestamp_ns = kT0 + 10 * kSec;
|
|
b.pid = 200;
|
|
b.process_node_id = "nodeB";
|
|
b.exe = "/usr/bin/ls";
|
|
b.cmdline = "ls -l";
|
|
b.source = "live";
|
|
evs.push_back({store::RecordType::process_exec, encode_process_exec(b)});
|
|
Database db;
|
|
CHECK(db.open(dir.string()));
|
|
CHECK(db.append_telemetry(evs, true));
|
|
LabelRecord lbl;
|
|
lbl.timestamp_ns = kT0 + 300 * kSec;
|
|
lbl.polarity = LabelPolarity::positive;
|
|
lbl.source = "operator";
|
|
lbl.action = "quarantine";
|
|
lbl.sha256 = std::string(64, 'd');
|
|
lbl.rule = "Bad_Rule";
|
|
CHECK(db.append_evidence(store::RecordType::label, encode_label(lbl)));
|
|
db.close();
|
|
}
|
|
ExportManifest m;
|
|
m.node_uid = "e2e";
|
|
m.created_unix_ns = 1;
|
|
const std::string out_path = temp_path("e2e");
|
|
std::string err;
|
|
CHECK(export_store(dir.string(), m, out_path, err));
|
|
ExportManifest back;
|
|
std::size_t nwin = 0, nlabel = 0;
|
|
bool saw_positive = false;
|
|
CHECK(read_export_bundle(out_path, back, [&](uint32_t type, std::string_view payload) {
|
|
if (type == static_cast<uint32_t>(store::RecordType::export_window)) ++nwin;
|
|
if (type == static_cast<uint32_t>(store::RecordType::export_label)) {
|
|
ExportLabel el;
|
|
if (decode_export_label(payload, el)) {
|
|
++nlabel;
|
|
if (el.polarity == LabelPolarity::positive) saw_positive = true;
|
|
}
|
|
}
|
|
}));
|
|
CHECK(back.node_uid == "e2e");
|
|
CHECK(nwin >= 1);
|
|
CHECK(nlabel >= 1 && saw_positive);
|
|
std::filesystem::remove(out_path);
|
|
|
|
// integration: export_store with a rarity state attaches export_rarity_delta records
|
|
learn::RarityModel rmodel;
|
|
for (int i = 0; i < 50; ++i) rmodel.observe(learn::TupleKind::parent_child, "bash>ls");
|
|
rmodel.observe(learn::TupleKind::exe_connect, "curl|ext|443");
|
|
rmodel.observe(learn::TupleKind::exe_cmdline, "ls -l");
|
|
const std::string rpath = temp_path("rarity_e2e");
|
|
{
|
|
std::ofstream rf(rpath, std::ios::binary);
|
|
const std::string st = rmodel.encode_state(1);
|
|
rf.write(st.data(), static_cast<std::streamsize>(st.size()));
|
|
}
|
|
const std::string out2 = temp_path("e2e_rarity");
|
|
std::string err2;
|
|
CHECK(export_store(dir.string(), m, out2, err2, rpath));
|
|
ExportManifest back2;
|
|
std::size_t nwin2 = 0, nrar2 = 0;
|
|
CHECK(read_export_bundle(out2, back2, [&](uint32_t type, std::string_view) {
|
|
if (type == static_cast<uint32_t>(store::RecordType::export_window)) ++nwin2;
|
|
if (type == static_cast<uint32_t>(store::RecordType::export_rarity_delta)) ++nrar2;
|
|
}));
|
|
CHECK(nwin2 >= 1);
|
|
CHECK(nrar2 == rmodel.sketch_count());
|
|
std::filesystem::remove(rpath);
|
|
std::filesystem::remove(out2);
|
|
fs::remove_all(dir, ec);
|
|
}
|
|
|
|
{
|
|
// unit: append_rarity_records from a saved state file
|
|
learn::RarityModel model;
|
|
for (int i = 0; i < 50; ++i) model.observe(learn::TupleKind::parent_child, "bash>ls");
|
|
model.observe(learn::TupleKind::lineage3, "a>b>c");
|
|
model.observe(learn::TupleKind::exe_persist, "cron|/tmp/x");
|
|
const std::string path = temp_path("rarity_unit");
|
|
{
|
|
std::ofstream f(path, std::ios::binary);
|
|
const std::string st = model.encode_state(1);
|
|
f.write(st.data(), static_cast<std::streamsize>(st.size()));
|
|
}
|
|
std::vector<store::PendingRecord> recs;
|
|
std::string err;
|
|
CHECK(append_rarity_records(path, recs, err));
|
|
CHECK(recs.size() == model.sketch_count());
|
|
const auto RT = static_cast<uint32_t>(store::RecordType::export_rarity_delta);
|
|
std::size_t nrar = 0;
|
|
for (const auto& r : recs) {
|
|
if (static_cast<uint32_t>(r.type) == RT) ++nrar;
|
|
}
|
|
CHECK(nrar == model.sketch_count());
|
|
ExportRaritySketch r0;
|
|
CHECK(decode_export_rarity(recs[0].payload, r0));
|
|
const learn::CountMinSketch& ref = model.sketch(learn::TupleKind::parent_child);
|
|
CHECK(r0.kind == 0);
|
|
CHECK(r0.width_log2 == ref.width_log2());
|
|
CHECK(r0.depth == ref.depth());
|
|
CHECK(r0.total == ref.total());
|
|
CHECK(r0.cells == ref.cells());
|
|
std::filesystem::remove(path);
|
|
|
|
// negative: nonexistent path
|
|
std::vector<store::PendingRecord> r1;
|
|
std::string e1;
|
|
CHECK(!append_rarity_records(path, r1, e1));
|
|
CHECK(r1.empty());
|
|
|
|
// negative: garbage file
|
|
const std::string gpath = temp_path("rarity_garbage");
|
|
{
|
|
std::ofstream f(gpath, std::ios::binary);
|
|
f << "\x01\x02garbage-bytes-not-a-rarity-state";
|
|
}
|
|
std::vector<store::PendingRecord> r2;
|
|
std::string e2;
|
|
CHECK(!append_rarity_records(gpath, r2, e2));
|
|
std::filesystem::remove(gpath);
|
|
}
|
|
|
|
// E1: plan_export_prune (pure) + prune_export_dir (fs)
|
|
{
|
|
constexpr int64_t kHour = 3'600'000'000'000LL;
|
|
const int64_t now = 1000 * kHour;
|
|
const char *a = "a", *b = "b", *c = "c", *d = "d";
|
|
auto B = [&](const char* n, int64_t age_h, uint64_t bytes) {
|
|
return EgressBundleFile{n, now - age_h * kHour, bytes};
|
|
};
|
|
auto sorted = [](std::vector<std::string> v) {
|
|
std::sort(v.begin(), v.end());
|
|
return v;
|
|
};
|
|
using V = std::vector<std::string>;
|
|
const std::vector<EgressBundleFile> set = {B(a, 100, 10), B(b, 50, 10), B(c, 10, 10), B(d, 1, 10)};
|
|
|
|
// age-only
|
|
CHECK(sorted(plan_export_prune(set, 48, 0, now, "")) == (V{a, b}));
|
|
CHECK(plan_export_prune(set, 0, 0, now, "").empty()); // both off = nothing
|
|
CHECK(plan_export_prune(set, 200, 0, now, "").empty());
|
|
|
|
// size-only: oldest first, exactly enough (40 -> 25: drop a,b -> 20)
|
|
CHECK((plan_export_prune(set, 0, 25, now, "") == V{a, b}));
|
|
CHECK((plan_export_prune(set, 0, 30, now, "") == V{a}));
|
|
CHECK(plan_export_prune(set, 0, 40, now, "").empty()); // at the cap, not over
|
|
|
|
// keep is never returned, and not counted toward the size total
|
|
CHECK(sorted(plan_export_prune(set, 48, 0, now, a)) == (V{b}));
|
|
CHECK((plan_export_prune(set, 0, 10, now, d) == V{a, b})); // survivors a,b,c = 30 -> drop a,b
|
|
for (const auto& p : plan_export_prune(set, 1, 1, now, a)) CHECK(p != a);
|
|
|
|
// age-then-size: age removals count toward the survivors' total
|
|
CHECK((plan_export_prune(set, 48, 15, now, "") == V{a, b, c})); // a,b by age; c,d=20>15 -> c
|
|
CHECK((plan_export_prune(set, 48, 20, now, "") == V{a, b})); // c,d=20 fits
|
|
|
|
// order independence
|
|
std::mt19937 rng(7);
|
|
const auto want = plan_export_prune(set, 48, 15, now, d);
|
|
for (int i = 0; i < 20; ++i) {
|
|
auto shuffled = set;
|
|
std::shuffle(shuffled.begin(), shuffled.end(), rng);
|
|
CHECK(plan_export_prune(shuffled, 48, 15, now, d) == want);
|
|
}
|
|
|
|
// fs wrapper: only export-*.sbx is ever touched
|
|
namespace fs = std::filesystem;
|
|
const fs::path dir = fs::temp_directory_path() / ("sb_export_prune_" + std::to_string(::getpid()));
|
|
fs::remove_all(dir);
|
|
fs::create_directories(dir);
|
|
auto touch = [&](const char* name, size_t n) {
|
|
std::ofstream f(dir / name, std::ios::binary);
|
|
f << std::string(n, 'x');
|
|
};
|
|
touch("export-n-1.sbx", 1024);
|
|
touch("export-n-2.sbx", 1024);
|
|
touch("export-n-3.sbx", 1024);
|
|
touch("notes.txt", 1024);
|
|
touch("export-n-4.sbx.part", 1024);
|
|
touch("other.sbx", 1024);
|
|
const auto base = fs::file_time_type::clock::now();
|
|
fs::last_write_time(dir / "export-n-1.sbx", base - std::chrono::hours(100));
|
|
fs::last_write_time(dir / "export-n-2.sbx", base - std::chrono::hours(50));
|
|
fs::last_write_time(dir / "export-n-3.sbx", base);
|
|
const int64_t wall = std::chrono::duration_cast<std::chrono::nanoseconds>(
|
|
std::chrono::system_clock::now().time_since_epoch()).count();
|
|
ExportPruneReport rep;
|
|
std::string perr;
|
|
CHECK(prune_export_dir(dir.string(), 48, 0, (dir / "export-n-3.sbx").string(), wall, rep, perr));
|
|
CHECK(rep.pruned_files == 2 && rep.pruned_bytes == 2048 && rep.problems == 0);
|
|
CHECK(!fs::exists(dir / "export-n-1.sbx") && !fs::exists(dir / "export-n-2.sbx"));
|
|
CHECK(fs::exists(dir / "export-n-3.sbx"));
|
|
CHECK(fs::exists(dir / "notes.txt") && fs::exists(dir / "export-n-4.sbx.part") && fs::exists(dir / "other.sbx"));
|
|
ExportPruneReport rep2;
|
|
CHECK(!prune_export_dir((dir / "missing").string(), 1, 1, "", wall, rep2, perr));
|
|
fs::remove_all(dir);
|
|
}
|
|
|
|
{
|
|
// F1: export --since (E-since) + cursor
|
|
namespace fs = std::filesystem;
|
|
constexpr int64_t kSec = 1'000'000'000;
|
|
constexpr int64_t kT0 = 1'800'000'000 * kSec;
|
|
const auto dir = fs::temp_directory_path() / ("sb_fleet_since_" + std::to_string(::getpid()));
|
|
std::error_code ec;
|
|
fs::remove_all(dir, ec);
|
|
{
|
|
std::vector<TelemetryEvent> evs;
|
|
const struct {
|
|
int64_t off_s;
|
|
const char* node;
|
|
bool detected;
|
|
} specs[] = {{0, "nodeS0", false}, {100, "nodeS1", true}, {1000, "nodeS2", false}, {2000, "nodeS3", false}};
|
|
int pid = 100;
|
|
for (const auto& sp : specs) {
|
|
ProcessExecRecord a;
|
|
a.timestamp_ns = kT0 + sp.off_s * kSec;
|
|
a.pid = pid;
|
|
a.process_node_id = sp.node;
|
|
a.exe = "/usr/bin/ls";
|
|
a.cmdline = "ls -l";
|
|
a.source = "live";
|
|
evs.push_back({store::RecordType::process_exec, encode_process_exec(a)});
|
|
if (sp.detected) {
|
|
ScanResultRecord sr;
|
|
sr.timestamp_ns = a.timestamp_ns + 2 * kSec;
|
|
sr.pid = pid;
|
|
sr.process_node_id = sp.node;
|
|
sr.rules = {"Bad_Rule"};
|
|
evs.push_back({store::RecordType::scan_result, encode_scan_result(sr)});
|
|
}
|
|
++pid;
|
|
}
|
|
Database db;
|
|
CHECK(db.open(dir.string()));
|
|
CHECK(db.append_telemetry(evs, true));
|
|
db.close();
|
|
}
|
|
learn::RarityModel rmodel;
|
|
rmodel.observe(learn::TupleKind::parent_child, "bash>ls");
|
|
const std::string rpath = temp_path("since_rarity");
|
|
{
|
|
std::ofstream rf(rpath, std::ios::binary);
|
|
const std::string st = rmodel.encode_state(1);
|
|
rf.write(st.data(), static_cast<std::streamsize>(st.size()));
|
|
}
|
|
ExportManifest m;
|
|
m.node_uid = "since";
|
|
m.created_unix_ns = 1;
|
|
auto count = [&](const std::string& path, std::vector<int64_t>& starts, std::size_t& nrar) {
|
|
ExportManifest back;
|
|
starts.clear();
|
|
nrar = 0;
|
|
CHECK(read_export_bundle(path, back, [&](uint32_t type, std::string_view payload) {
|
|
if (type == static_cast<uint32_t>(store::RecordType::export_window)) {
|
|
ExportWindow ew;
|
|
CHECK(decode_export_window(payload, ew));
|
|
starts.push_back(ew.start_ns);
|
|
}
|
|
if (type == static_cast<uint32_t>(store::RecordType::export_rarity_delta)) ++nrar;
|
|
}));
|
|
};
|
|
std::string err;
|
|
const std::string full = temp_path("since_full");
|
|
ExportStats fst;
|
|
CHECK(export_store(dir.string(), m, full, err, rpath, {}, 0, &fst));
|
|
std::vector<int64_t> fstarts;
|
|
std::size_t frar = 0;
|
|
count(full, fstarts, frar);
|
|
CHECK(fstarts.size() == 4 && fst.windows == 4);
|
|
CHECK(frar == rmodel.sketch_count());
|
|
CHECK(fst.time_high_ns >= kT0 + 2000 * kSec);
|
|
|
|
// since_ns = 0 is byte-for-byte the legacy call
|
|
const std::string legacy = temp_path("since_legacy");
|
|
CHECK(export_store(dir.string(), m, legacy, err, rpath));
|
|
CHECK(read_small_file(legacy, 64u << 20) == read_small_file(full, 64u << 20));
|
|
|
|
const std::string inc = temp_path("since_inc");
|
|
ExportStats ist;
|
|
CHECK(export_store(dir.string(), m, inc, err, rpath, {}, kT0 + 500 * kSec, &ist));
|
|
std::vector<int64_t> istarts;
|
|
std::size_t irar = 0;
|
|
count(inc, istarts, irar);
|
|
// drops the old unlabeled window (T0); keeps the labeled old one (T0+100s) and the new ones
|
|
CHECK(istarts.size() == 3 && ist.windows == 3);
|
|
CHECK(std::find(istarts.begin(), istarts.end(), kT0) == istarts.end());
|
|
CHECK(std::find(istarts.begin(), istarts.end(), kT0 + 100 * kSec) != istarts.end());
|
|
CHECK(irar == frar); // rarity always present
|
|
|
|
// a since after every window start keeps only the labeled one
|
|
const std::string late = temp_path("since_late");
|
|
CHECK(export_store(dir.string(), m, late, err, rpath, {}, kT0 + 5000 * kSec));
|
|
std::vector<int64_t> lstarts;
|
|
std::size_t lrar = 0;
|
|
count(late, lstarts, lrar);
|
|
CHECK(lstarts.size() == 1 && lstarts[0] == kT0 + 100 * kSec && lrar == frar);
|
|
|
|
// since_from_cursor: 0 -> full, else minus the named overlap
|
|
CHECK(since_from_cursor(0) == 0 && since_from_cursor(-5) == 0);
|
|
CHECK(since_from_cursor(kT0) == kT0 - kExportSinceOverlapNs);
|
|
CHECK(kExportSinceOverlapNs == 120 * kSec);
|
|
|
|
// cursor file: missing -> not found; round trip; atomic (no tmp left); not matched by the prune glob
|
|
const auto egress = fs::temp_directory_path() / ("sb_fleet_cursor_" + std::to_string(::getpid()));
|
|
fs::remove_all(egress, ec);
|
|
fs::create_directories(egress);
|
|
int64_t cur = 77;
|
|
CHECK(!read_export_cursor(egress.string(), cur) && cur == 0);
|
|
CHECK(write_export_cursor(egress.string(), 123456789012345LL, err));
|
|
CHECK(read_export_cursor(egress.string(), cur) && cur == 123456789012345LL);
|
|
CHECK(write_export_cursor(egress.string(), 42, err));
|
|
CHECK(read_export_cursor(egress.string(), cur) && cur == 42);
|
|
CHECK(!fs::exists(egress / ".export-cursor.tmp"));
|
|
CHECK(std::string(kExportCursorName) == ".export-cursor");
|
|
{
|
|
std::ofstream f(egress / kExportCursorName, std::ios::binary | std::ios::trunc);
|
|
f << "not-a-number";
|
|
}
|
|
CHECK(!read_export_cursor(egress.string(), cur) && cur == 0);
|
|
CHECK(write_export_cursor(egress.string(), 42, err));
|
|
// prune with everything expired: the cursor must survive; a real bundle goes
|
|
{
|
|
std::ofstream f(egress / "export-n-1.sbx", std::ios::binary);
|
|
f << "x";
|
|
}
|
|
fs::last_write_time(egress / "export-n-1.sbx", fs::file_time_type::clock::now() - std::chrono::hours(1000));
|
|
const int64_t wall = std::chrono::duration_cast<std::chrono::nanoseconds>(
|
|
std::chrono::system_clock::now().time_since_epoch()).count();
|
|
ExportPruneReport prep;
|
|
CHECK(prune_export_dir(egress.string(), 1, 1, "", wall, prep, err));
|
|
CHECK(prep.pruned_files == 1);
|
|
CHECK(fs::exists(egress / kExportCursorName));
|
|
CHECK(!fs::exists(egress / "export-n-1.sbx"));
|
|
|
|
for (const auto& p : {full, legacy, inc, late, rpath}) fs::remove(p);
|
|
fs::remove_all(egress, ec);
|
|
fs::remove_all(dir, ec);
|
|
}
|
|
|
|
std::puts("ok");
|
|
return 0;
|
|
}
|