#include #include #include #include #include #include #include #include #include #include #include #include "strawberry/database.hpp" #include "strawberry/fleet_export.hpp" #include "strawberry/learn/rarity.hpp" #include "strawberry/learn/replay.hpp" #include "strawberry/proc_read.hpp" #include #include "strawberry/telemetry.hpp" using namespace strawberry; using namespace strawberry::fleet; #define CHECK(cond) \ do { \ if (!(cond)) { \ std::fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond); \ std::exit(1); \ } \ } while (0) using Visited = std::vector>; static std::string temp_path(const char* tag) { static int n = 0; auto p = std::filesystem::temp_directory_path() / ("sb_fleet_export_" + std::to_string(::getpid()) + "_" + tag + "_" + std::to_string(n++) + ".seg"); std::filesystem::remove(p); return p.string(); } static ExportManifest make_manifest() { ExportManifest m; m.node_uid = "abc123"; m.generation_low = 10; m.generation_high = 20; m.time_low_ns = 100; m.time_high_ns = 900; m.created_unix_ns = 500; m.record_count = 3; return m; } static bool read_back(const std::string& path, ExportManifest& m, Visited& v) { return read_export_bundle(path, m, [&](uint32_t t, std::string_view p) { v.emplace_back(t, std::string(p)); }); } int main() { const auto W = static_cast(store::RecordType::export_window); // a. round-trip { const ExportManifest m = make_manifest(); std::vector recs; for (int i = 0; i < 3; ++i) recs.push_back({store::RecordType::export_window, "payload-" + std::to_string(i)}); const std::string path = temp_path("a"); std::string err; CHECK(write_export_bundle(path, m, recs, err)); ExportManifest out; Visited v; CHECK(read_back(path, out, v)); CHECK(out.created_unix_ns == m.created_unix_ns); CHECK(out.node_uid == m.node_uid); CHECK(out.generation_low == 10 && out.generation_high == 20); CHECK(out.time_low_ns == 100 && out.time_high_ns == 900); CHECK(out.record_count == 3); CHECK(v.size() == 3); for (size_t i = 0; i < v.size(); ++i) { CHECK(v[i].first == W); CHECK(v[i].second == "payload-" + std::to_string(i)); } std::filesystem::remove(path); } // b. unknown record type skipped, not errored { std::vector recs; recs.push_back({store::RecordType::export_window, "k0"}); recs.push_back({static_cast(9999), "x"}); recs.push_back({store::RecordType::export_window, "k1"}); const std::string path = temp_path("b"); std::string err; CHECK(write_export_bundle(path, make_manifest(), recs, err)); ExportManifest out; Visited v; CHECK(read_back(path, out, v)); size_t known = 0; for (const auto& e : v) if (e.first == W) ++known; CHECK(known == 2); std::filesystem::remove(path); } // c. truncation { std::vector recs; for (int i = 0; i < 3; ++i) recs.push_back({store::RecordType::export_window, "payload-" + std::to_string(i)}); const std::string path = temp_path("c"); std::string err; CHECK(write_export_bundle(path, make_manifest(), recs, err)); const auto size = std::filesystem::file_size(path); std::filesystem::resize_file(path, size / 2); ExportManifest out; Visited v; bool ok = true; try { ok = read_back(path, out, v); } catch (...) { CHECK(false); } CHECK(!ok); std::filesystem::remove(path); } // d. bad magic / empty { store::FieldWriter w; w.u64(1, 500); w.fixed32(2, kExportMagic ^ 0xFFu); w.u64(3, kExportSchemaVersion); w.bytes(4, "abc123"); ExportManifest out; CHECK(!decode_export_manifest(w.data(), out)); CHECK(!decode_export_manifest("", out)); // sanity: the good encoding decodes CHECK(decode_export_manifest(encode_export_manifest(make_manifest()), out)); // schema newer than supported is rejected store::FieldWriter w2; w2.fixed32(2, kExportMagic); w2.u64(3, kExportSchemaVersion + 1); CHECK(!decode_export_manifest(w2.data(), out)); } // e. decoder robustness property { const uint64_t seed = std::random_device{}(); std::printf("seed=%llu\n", static_cast(seed)); std::mt19937_64 rng(seed); for (int i = 0; i < 5000; ++i) { std::string s(rng() % 129, '\0'); for (auto& c : s) c = static_cast(rng() & 0xFF); ExportManifest out; try { (void)decode_export_manifest(s, out); } catch (...) { CHECK(false); } } } // f. payload records round-trip { ExportWindow w; w.window_id = "win-hash-01"; w.start_ns = -12345; w.last_ns = 987654321; w.execs = 1; w.connects = 2; w.persistence_writes = 3; w.drop_writes = 4; w.ptraces = 5; w.module_loads = 6; w.memfd_execs = 7; w.deleted_execs = 8; w.uid0_execs = 9; w.max_depth = 10; for (std::size_t i = 0; i < kExportLocationClasses; ++i) w.exec_locations[i] = static_cast(i + 1); w.public_connects = 11; w.private_connects = 12; w.loopback_connects = 13; w.low_port_connects = 14; w.high_port_connects = 15; w.scans_too_large = 16; w.scans_unavailable = 17; w.scan_errors = 18; w.max_bits = {0.5f, 1.5f, 2.5f, 3.5f, 4.5f}; w.top3_bits = 5.5f; w.governor_level = 3; w.budget_left = 0.25f; w.token_hashes = {1, 2, 3, 0xdeadbeefcafeULL}; ExportWindow wo; CHECK(decode_export_window(encode_export_window(w), wo)); CHECK(wo.window_id == w.window_id); CHECK(wo.start_ns == w.start_ns && wo.last_ns == w.last_ns); CHECK(wo.execs == w.execs && wo.connects == w.connects); CHECK(wo.persistence_writes == w.persistence_writes && wo.drop_writes == w.drop_writes); CHECK(wo.ptraces == w.ptraces && wo.module_loads == w.module_loads); CHECK(wo.memfd_execs == w.memfd_execs && wo.deleted_execs == w.deleted_execs); CHECK(wo.uid0_execs == w.uid0_execs && wo.max_depth == w.max_depth); for (std::size_t i = 0; i < kExportLocationClasses; ++i) CHECK(wo.exec_locations[i] == w.exec_locations[i]); CHECK(wo.public_connects == w.public_connects && wo.private_connects == w.private_connects); CHECK(wo.loopback_connects == w.loopback_connects); CHECK(wo.low_port_connects == w.low_port_connects && wo.high_port_connects == w.high_port_connects); CHECK(wo.scans_too_large == w.scans_too_large && wo.scans_unavailable == w.scans_unavailable); CHECK(wo.scan_errors == w.scan_errors); for (std::size_t i = 0; i < kExportTupleKinds; ++i) CHECK(wo.max_bits[i] == w.max_bits[i]); CHECK(wo.top3_bits == w.top3_bits && wo.governor_level == w.governor_level); CHECK(wo.budget_left == w.budget_left); CHECK(wo.token_hashes == w.token_hashes); CHECK(!decode_export_window("", wo)); ExportWindowFeature fe; fe.window_id = "win-hash-01"; for (std::size_t i = 0; i < kExportFeatureDim; ++i) fe.features[i] = static_cast(i) * 0.25f; ExportWindowFeature fo; CHECK(decode_export_window_feature(encode_export_window_feature(fe), fo)); CHECK(fo.window_id == fe.window_id); for (std::size_t i = 0; i < kExportFeatureDim; ++i) CHECK(fo.features[i] == fe.features[i]); CHECK(!decode_export_window_feature("", fo)); ExportLabel lb; lb.window_id = "win-hash-01"; lb.timestamp_ns = 555666777; lb.polarity = LabelPolarity::negative; lb.source = ExportLabelSource::op; lb.action = 7; lb.uid = 1001; lb.sha256 = "ff00ff00ff00ff00ff00ff00ff00ff00ff00ff00ff00ff00ff00ff00ff00ff00"; lb.origin_node_hash = 0xA1B2C3D4E5F60718ULL; // 3.5d attribution hashes round-trip lb.origin_exe_hash = 0x1122334455667788ULL; ExportLabel lo; CHECK(decode_export_label(encode_export_label(lb), lo)); CHECK(lo.window_id == lb.window_id && lo.timestamp_ns == lb.timestamp_ns); CHECK(lo.polarity == lb.polarity && lo.source == lb.source); CHECK(lo.action == lb.action && lo.uid == lb.uid && lo.sha256 == lb.sha256); CHECK(lo.origin_node_hash == lb.origin_node_hash && lo.origin_exe_hash == lb.origin_exe_hash); CHECK(!decode_export_label("", lo)); // back-compat: a label with no origin set decodes with zero hashes (old bundles have no tags 8/9) ExportLabel noorig; noorig.window_id = "win-hash-02"; ExportLabel noout; CHECK(decode_export_label(encode_export_label(noorig), noout)); CHECK(noout.origin_node_hash == 0 && noout.origin_exe_hash == 0); } // g. payload decoder robustness property { const uint64_t seed = std::random_device{}(); std::printf("payload seed=%llu\n", static_cast(seed)); std::mt19937_64 rng(seed); for (int i = 0; i < 4000; ++i) { std::string s(rng() % 161, '\0'); for (auto& c : s) c = static_cast(rng() & 0xFF); ExportWindow a; ExportWindowFeature b; ExportLabel c; try { (void)decode_export_window(s, a); (void)decode_export_window_feature(s, b); (void)decode_export_label(s, c); } catch (...) { CHECK(false); } } } // h. rarity delta record { ExportRaritySketch r; r.kind = 3; r.width_log2 = 4; r.depth = 4; r.total = 123.5; r.cells.resize(64); for (std::size_t i = 0; i < r.cells.size(); ++i) r.cells[i] = static_cast(i) * 0.5f; ExportRaritySketch ro; CHECK(decode_export_rarity(encode_export_rarity(r), ro)); CHECK(ro.kind == r.kind && ro.width_log2 == r.width_log2 && ro.depth == r.depth); CHECK(ro.total == 123.5); CHECK(ro.cells == r.cells); ExportRaritySketch bad = r; bad.cells.resize(63); CHECK(!decode_export_rarity(encode_export_rarity(bad), ro)); bad = r; bad.width_log2 = static_cast(kMaxRarityWidthLog2 + 1); CHECK(!decode_export_rarity(encode_export_rarity(bad), ro)); bad = r; bad.depth = 0; CHECK(!decode_export_rarity(encode_export_rarity(bad), ro)); bad = r; bad.kind = static_cast(kExportTupleKinds); CHECK(!decode_export_rarity(encode_export_rarity(bad), ro)); CHECK(!decode_export_rarity("", ro)); const uint64_t seed = std::random_device{}(); std::printf("rarity seed=%llu\n", static_cast(seed)); std::mt19937_64 rng(seed); for (int i = 0; i < 3000; ++i) { std::string s(rng() % 201, '\0'); for (auto& c : s) c = static_cast(rng() & 0xFF); ExportRaritySketch x; try { (void)decode_export_rarity(s, x); } catch (...) { CHECK(false); } } } { // extraction: Replayer -> append_window_records -> bundle round-trip constexpr int64_t kSec = 1'000'000'000; constexpr int64_t kT0 = 1'800'000'000 * kSec; using Ev = std::pair; std::vector evs; auto add_exec = [&](int64_t ts, int pid, const std::string& node, const std::string& parent, const std::string& exe, const std::string& cmd) { ProcessExecRecord r; r.timestamp_ns = ts; r.pid = pid; r.process_node_id = node; r.parent_node_id = parent; r.exe = exe; r.cmdline = cmd; r.source = "live"; evs.push_back({store::RecordType::process_exec, encode_process_exec(r)}); }; auto add_connect = [&](int64_t ts, int pid, const std::string& addr, int port) { BehaviorEvent b; b.type = BehaviorEvent::Type::connect; b.timestamp_ns = static_cast(ts); b.pid = pid; b.family = 2; b.address = addr; b.port = static_cast(port); evs.push_back({store::RecordType::connect, encode_behavior(b)}); }; auto add_scan = [&](int64_t ts, int pid, const std::string& node, std::vector rules) { ScanResultRecord sr; sr.timestamp_ns = ts; sr.pid = pid; sr.process_node_id = node; sr.rules = std::move(rules); evs.push_back({store::RecordType::scan_result, encode_scan_result(sr)}); }; auto add_write = [&](int64_t ts, int pid, const std::string& path) { FileWriteEvent fw; fw.pid = pid; fw.path = path; fw.category = "drop"; fw.timestamp_ns = static_cast(ts); evs.push_back({store::RecordType::file_write, encode_file_write(fw)}); }; // detected window, NO in-set write for its flagged binary -> positive label has origin 0 add_exec(kT0, 100, "nodeA", "", "/tmp/evil", "/tmp/evil -x"); add_connect(kT0 + kSec, 100, "8.8.8.8", 443); add_scan(kT0 + 2 * kSec, 100, "nodeA", {"Bad_Rule"}); // plain window add_exec(kT0 + 10 * kSec, 200, "nodeB", "", "/usr/bin/ls", "ls -l"); // 3.5d attributed window: dropper nodeD writes /tmp/dropped, victim nodeC execs it and is // detected -> nodeC's positive label carries a non-zero dropper origin hash. add_exec(kT0 + 20 * kSec, 300, "nodeD", "", "/usr/bin/curl", "curl -o /tmp/dropped"); add_write(kT0 + 21 * kSec, 300, "/tmp/dropped"); add_exec(kT0 + 22 * kSec, 400, "nodeC", "", "/tmp/dropped", "/tmp/dropped"); add_scan(kT0 + 23 * kSec, 400, "nodeC", {"Bad_Rule"}); learn::ReplayOptions o; o.live = true; o.warmup_ns = 0; std::vector recs; o.on_window_closed = [&](const learn::ReplayWindow& w, const learn::WindowSummary& s) { append_window_records(w, s, recs); }; learn::Replayer r(o); for (const auto& e : evs) r.feed(e.first, e.second); r.finish(); std::size_t nwin = 0, nfeat = 0, nlabel = 0; std::vector win_ids, feat_ids; bool saw_positive = false, saw_tokens = false, saw_origin = false; for (const auto& pr : recs) { if (pr.type == store::RecordType::export_window) { ExportWindow ew; CHECK(decode_export_window(pr.payload, ew)); ++nwin; win_ids.push_back(ew.window_id); if (!ew.token_hashes.empty()) saw_tokens = true; } else if (pr.type == store::RecordType::export_window_feature) { ExportWindowFeature ef; CHECK(decode_export_window_feature(pr.payload, ef)); CHECK(ef.features.size() == 256); ++nfeat; feat_ids.push_back(ef.window_id); } else if (pr.type == store::RecordType::export_label) { ExportLabel el; CHECK(decode_export_label(pr.payload, el)); ++nlabel; if (el.polarity == LabelPolarity::positive) saw_positive = true; // 3.5d: a positive label with a non-zero origin hash is the attributed (dropper) window; // a suppressed/negative label must never carry an origin. if (el.polarity == LabelPolarity::positive && el.origin_node_hash != 0) saw_origin = true; if (el.polarity == LabelPolarity::negative) CHECK(el.origin_node_hash == 0 && el.origin_exe_hash == 0); } } CHECK(nwin >= 1); CHECK(nwin == nfeat); CHECK(win_ids == feat_ids); CHECK(nlabel >= 1); CHECK(saw_positive); CHECK(saw_tokens); CHECK(saw_origin); // 3.5d: the dropper->exec window's positive label carries an origin hash ExportManifest man; man.node_uid = "n1"; const std::string path = temp_path("extract"); std::string err; CHECK(write_export_bundle(path, man, recs, err)); ExportManifest back; std::size_t visited = 0; CHECK(read_export_bundle(path, back, [&](uint32_t, std::string_view) { ++visited; })); CHECK(visited == recs.size()); CHECK(back.node_uid == "n1"); std::filesystem::remove(path); } { // e2e: export_store on a real store dir built via Database (closes 1e deferred validation) constexpr int64_t kSec = 1'000'000'000; constexpr int64_t kT0 = 1'800'000'000 * kSec; namespace fs = std::filesystem; const auto dir = fs::temp_directory_path() / ("sb_fleet_export_e2e_" + std::to_string(::getpid())); std::error_code ec; fs::remove_all(dir, ec); { std::vector evs; ProcessExecRecord a; a.timestamp_ns = kT0; a.pid = 100; a.process_node_id = "nodeA"; a.exe = "/tmp/evil"; a.cmdline = "/tmp/evil -x"; a.source = "live"; evs.push_back({store::RecordType::process_exec, encode_process_exec(a)}); ScanResultRecord sr; sr.timestamp_ns = kT0 + 2 * kSec; sr.pid = 100; sr.process_node_id = "nodeA"; sr.rules = {"Bad_Rule"}; evs.push_back({store::RecordType::scan_result, encode_scan_result(sr)}); ProcessExecRecord b; b.timestamp_ns = kT0 + 10 * kSec; b.pid = 200; b.process_node_id = "nodeB"; b.exe = "/usr/bin/ls"; b.cmdline = "ls -l"; b.source = "live"; evs.push_back({store::RecordType::process_exec, encode_process_exec(b)}); Database db; CHECK(db.open(dir.string())); CHECK(db.append_telemetry(evs, true)); LabelRecord lbl; lbl.timestamp_ns = kT0 + 300 * kSec; lbl.polarity = LabelPolarity::positive; lbl.source = "operator"; lbl.action = "quarantine"; lbl.sha256 = std::string(64, 'd'); lbl.rule = "Bad_Rule"; CHECK(db.append_evidence(store::RecordType::label, encode_label(lbl))); db.close(); } ExportManifest m; m.node_uid = "e2e"; m.created_unix_ns = 1; const std::string out_path = temp_path("e2e"); std::string err; CHECK(export_store(dir.string(), m, out_path, err)); ExportManifest back; std::size_t nwin = 0, nlabel = 0; bool saw_positive = false; CHECK(read_export_bundle(out_path, back, [&](uint32_t type, std::string_view payload) { if (type == static_cast(store::RecordType::export_window)) ++nwin; if (type == static_cast(store::RecordType::export_label)) { ExportLabel el; if (decode_export_label(payload, el)) { ++nlabel; if (el.polarity == LabelPolarity::positive) saw_positive = true; } } })); CHECK(back.node_uid == "e2e"); CHECK(nwin >= 1); CHECK(nlabel >= 1 && saw_positive); std::filesystem::remove(out_path); // integration: export_store with a rarity state attaches export_rarity_delta records learn::RarityModel rmodel; for (int i = 0; i < 50; ++i) rmodel.observe(learn::TupleKind::parent_child, "bash>ls"); rmodel.observe(learn::TupleKind::exe_connect, "curl|ext|443"); rmodel.observe(learn::TupleKind::exe_cmdline, "ls -l"); const std::string rpath = temp_path("rarity_e2e"); { std::ofstream rf(rpath, std::ios::binary); const std::string st = rmodel.encode_state(1); rf.write(st.data(), static_cast(st.size())); } const std::string out2 = temp_path("e2e_rarity"); std::string err2; CHECK(export_store(dir.string(), m, out2, err2, rpath)); ExportManifest back2; std::size_t nwin2 = 0, nrar2 = 0; CHECK(read_export_bundle(out2, back2, [&](uint32_t type, std::string_view) { if (type == static_cast(store::RecordType::export_window)) ++nwin2; if (type == static_cast(store::RecordType::export_rarity_delta)) ++nrar2; })); CHECK(nwin2 >= 1); CHECK(nrar2 == rmodel.sketch_count()); std::filesystem::remove(rpath); std::filesystem::remove(out2); fs::remove_all(dir, ec); } { // unit: append_rarity_records from a saved state file learn::RarityModel model; for (int i = 0; i < 50; ++i) model.observe(learn::TupleKind::parent_child, "bash>ls"); model.observe(learn::TupleKind::lineage3, "a>b>c"); model.observe(learn::TupleKind::exe_persist, "cron|/tmp/x"); const std::string path = temp_path("rarity_unit"); { std::ofstream f(path, std::ios::binary); const std::string st = model.encode_state(1); f.write(st.data(), static_cast(st.size())); } std::vector recs; std::string err; CHECK(append_rarity_records(path, recs, err)); CHECK(recs.size() == model.sketch_count()); const auto RT = static_cast(store::RecordType::export_rarity_delta); std::size_t nrar = 0; for (const auto& r : recs) { if (static_cast(r.type) == RT) ++nrar; } CHECK(nrar == model.sketch_count()); ExportRaritySketch r0; CHECK(decode_export_rarity(recs[0].payload, r0)); const learn::CountMinSketch& ref = model.sketch(learn::TupleKind::parent_child); CHECK(r0.kind == 0); CHECK(r0.width_log2 == ref.width_log2()); CHECK(r0.depth == ref.depth()); CHECK(r0.total == ref.total()); CHECK(r0.cells == ref.cells()); std::filesystem::remove(path); // negative: nonexistent path std::vector r1; std::string e1; CHECK(!append_rarity_records(path, r1, e1)); CHECK(r1.empty()); // negative: garbage file const std::string gpath = temp_path("rarity_garbage"); { std::ofstream f(gpath, std::ios::binary); f << "\x01\x02garbage-bytes-not-a-rarity-state"; } std::vector r2; std::string e2; CHECK(!append_rarity_records(gpath, r2, e2)); std::filesystem::remove(gpath); } // E1: plan_export_prune (pure) + prune_export_dir (fs) { constexpr int64_t kHour = 3'600'000'000'000LL; const int64_t now = 1000 * kHour; const char *a = "a", *b = "b", *c = "c", *d = "d"; auto B = [&](const char* n, int64_t age_h, uint64_t bytes) { return EgressBundleFile{n, now - age_h * kHour, bytes}; }; auto sorted = [](std::vector v) { std::sort(v.begin(), v.end()); return v; }; using V = std::vector; const std::vector set = {B(a, 100, 10), B(b, 50, 10), B(c, 10, 10), B(d, 1, 10)}; // age-only CHECK(sorted(plan_export_prune(set, 48, 0, now, "")) == (V{a, b})); CHECK(plan_export_prune(set, 0, 0, now, "").empty()); // both off = nothing CHECK(plan_export_prune(set, 200, 0, now, "").empty()); // size-only: oldest first, exactly enough (40 -> 25: drop a,b -> 20) CHECK((plan_export_prune(set, 0, 25, now, "") == V{a, b})); CHECK((plan_export_prune(set, 0, 30, now, "") == V{a})); CHECK(plan_export_prune(set, 0, 40, now, "").empty()); // at the cap, not over // keep is never returned, and not counted toward the size total CHECK(sorted(plan_export_prune(set, 48, 0, now, a)) == (V{b})); CHECK((plan_export_prune(set, 0, 10, now, d) == V{a, b})); // survivors a,b,c = 30 -> drop a,b for (const auto& p : plan_export_prune(set, 1, 1, now, a)) CHECK(p != a); // age-then-size: age removals count toward the survivors' total CHECK((plan_export_prune(set, 48, 15, now, "") == V{a, b, c})); // a,b by age; c,d=20>15 -> c CHECK((plan_export_prune(set, 48, 20, now, "") == V{a, b})); // c,d=20 fits // order independence std::mt19937 rng(7); const auto want = plan_export_prune(set, 48, 15, now, d); for (int i = 0; i < 20; ++i) { auto shuffled = set; std::shuffle(shuffled.begin(), shuffled.end(), rng); CHECK(plan_export_prune(shuffled, 48, 15, now, d) == want); } // fs wrapper: only export-*.sbx is ever touched namespace fs = std::filesystem; const fs::path dir = fs::temp_directory_path() / ("sb_export_prune_" + std::to_string(::getpid())); fs::remove_all(dir); fs::create_directories(dir); auto touch = [&](const char* name, size_t n) { std::ofstream f(dir / name, std::ios::binary); f << std::string(n, 'x'); }; touch("export-n-1.sbx", 1024); touch("export-n-2.sbx", 1024); touch("export-n-3.sbx", 1024); touch("notes.txt", 1024); touch("export-n-4.sbx.part", 1024); touch("other.sbx", 1024); const auto base = fs::file_time_type::clock::now(); fs::last_write_time(dir / "export-n-1.sbx", base - std::chrono::hours(100)); fs::last_write_time(dir / "export-n-2.sbx", base - std::chrono::hours(50)); fs::last_write_time(dir / "export-n-3.sbx", base); const int64_t wall = std::chrono::duration_cast( std::chrono::system_clock::now().time_since_epoch()).count(); ExportPruneReport rep; std::string perr; CHECK(prune_export_dir(dir.string(), 48, 0, (dir / "export-n-3.sbx").string(), wall, rep, perr)); CHECK(rep.pruned_files == 2 && rep.pruned_bytes == 2048 && rep.problems == 0); CHECK(!fs::exists(dir / "export-n-1.sbx") && !fs::exists(dir / "export-n-2.sbx")); CHECK(fs::exists(dir / "export-n-3.sbx")); CHECK(fs::exists(dir / "notes.txt") && fs::exists(dir / "export-n-4.sbx.part") && fs::exists(dir / "other.sbx")); ExportPruneReport rep2; CHECK(!prune_export_dir((dir / "missing").string(), 1, 1, "", wall, rep2, perr)); fs::remove_all(dir); } { // F1: export --since (E-since) + cursor namespace fs = std::filesystem; constexpr int64_t kSec = 1'000'000'000; constexpr int64_t kT0 = 1'800'000'000 * kSec; const auto dir = fs::temp_directory_path() / ("sb_fleet_since_" + std::to_string(::getpid())); std::error_code ec; fs::remove_all(dir, ec); { std::vector evs; const struct { int64_t off_s; const char* node; bool detected; } specs[] = {{0, "nodeS0", false}, {100, "nodeS1", true}, {1000, "nodeS2", false}, {2000, "nodeS3", false}}; int pid = 100; for (const auto& sp : specs) { ProcessExecRecord a; a.timestamp_ns = kT0 + sp.off_s * kSec; a.pid = pid; a.process_node_id = sp.node; a.exe = "/usr/bin/ls"; a.cmdline = "ls -l"; a.source = "live"; evs.push_back({store::RecordType::process_exec, encode_process_exec(a)}); if (sp.detected) { ScanResultRecord sr; sr.timestamp_ns = a.timestamp_ns + 2 * kSec; sr.pid = pid; sr.process_node_id = sp.node; sr.rules = {"Bad_Rule"}; evs.push_back({store::RecordType::scan_result, encode_scan_result(sr)}); } ++pid; } Database db; CHECK(db.open(dir.string())); CHECK(db.append_telemetry(evs, true)); db.close(); } learn::RarityModel rmodel; rmodel.observe(learn::TupleKind::parent_child, "bash>ls"); const std::string rpath = temp_path("since_rarity"); { std::ofstream rf(rpath, std::ios::binary); const std::string st = rmodel.encode_state(1); rf.write(st.data(), static_cast(st.size())); } ExportManifest m; m.node_uid = "since"; m.created_unix_ns = 1; auto count = [&](const std::string& path, std::vector& starts, std::size_t& nrar) { ExportManifest back; starts.clear(); nrar = 0; CHECK(read_export_bundle(path, back, [&](uint32_t type, std::string_view payload) { if (type == static_cast(store::RecordType::export_window)) { ExportWindow ew; CHECK(decode_export_window(payload, ew)); starts.push_back(ew.start_ns); } if (type == static_cast(store::RecordType::export_rarity_delta)) ++nrar; })); }; std::string err; const std::string full = temp_path("since_full"); ExportStats fst; CHECK(export_store(dir.string(), m, full, err, rpath, {}, 0, &fst)); std::vector fstarts; std::size_t frar = 0; count(full, fstarts, frar); CHECK(fstarts.size() == 4 && fst.windows == 4); CHECK(frar == rmodel.sketch_count()); CHECK(fst.time_high_ns >= kT0 + 2000 * kSec); // since_ns = 0 is byte-for-byte the legacy call const std::string legacy = temp_path("since_legacy"); CHECK(export_store(dir.string(), m, legacy, err, rpath)); CHECK(read_small_file(legacy, 64u << 20) == read_small_file(full, 64u << 20)); const std::string inc = temp_path("since_inc"); ExportStats ist; CHECK(export_store(dir.string(), m, inc, err, rpath, {}, kT0 + 500 * kSec, &ist)); std::vector istarts; std::size_t irar = 0; count(inc, istarts, irar); // drops the old unlabeled window (T0); keeps the labeled old one (T0+100s) and the new ones CHECK(istarts.size() == 3 && ist.windows == 3); CHECK(std::find(istarts.begin(), istarts.end(), kT0) == istarts.end()); CHECK(std::find(istarts.begin(), istarts.end(), kT0 + 100 * kSec) != istarts.end()); CHECK(irar == frar); // rarity always present // a since after every window start keeps only the labeled one const std::string late = temp_path("since_late"); CHECK(export_store(dir.string(), m, late, err, rpath, {}, kT0 + 5000 * kSec)); std::vector lstarts; std::size_t lrar = 0; count(late, lstarts, lrar); CHECK(lstarts.size() == 1 && lstarts[0] == kT0 + 100 * kSec && lrar == frar); // since_from_cursor: 0 -> full, else minus the named overlap CHECK(since_from_cursor(0) == 0 && since_from_cursor(-5) == 0); CHECK(since_from_cursor(kT0) == kT0 - kExportSinceOverlapNs); CHECK(kExportSinceOverlapNs == 120 * kSec); // cursor file: missing -> not found; round trip; atomic (no tmp left); not matched by the prune glob const auto egress = fs::temp_directory_path() / ("sb_fleet_cursor_" + std::to_string(::getpid())); fs::remove_all(egress, ec); fs::create_directories(egress); int64_t cur = 77; CHECK(!read_export_cursor(egress.string(), cur) && cur == 0); CHECK(write_export_cursor(egress.string(), 123456789012345LL, err)); CHECK(read_export_cursor(egress.string(), cur) && cur == 123456789012345LL); CHECK(write_export_cursor(egress.string(), 42, err)); CHECK(read_export_cursor(egress.string(), cur) && cur == 42); CHECK(!fs::exists(egress / ".export-cursor.tmp")); CHECK(std::string(kExportCursorName) == ".export-cursor"); { std::ofstream f(egress / kExportCursorName, std::ios::binary | std::ios::trunc); f << "not-a-number"; } CHECK(!read_export_cursor(egress.string(), cur) && cur == 0); CHECK(write_export_cursor(egress.string(), 42, err)); // prune with everything expired: the cursor must survive; a real bundle goes { std::ofstream f(egress / "export-n-1.sbx", std::ios::binary); f << "x"; } fs::last_write_time(egress / "export-n-1.sbx", fs::file_time_type::clock::now() - std::chrono::hours(1000)); const int64_t wall = std::chrono::duration_cast( std::chrono::system_clock::now().time_since_epoch()).count(); ExportPruneReport prep; CHECK(prune_export_dir(egress.string(), 1, 1, "", wall, prep, err)); CHECK(prep.pruned_files == 1); CHECK(fs::exists(egress / kExportCursorName)); CHECK(!fs::exists(egress / "export-n-1.sbx")); for (const auto& p : {full, legacy, inc, late, rpath}) fs::remove(p); fs::remove_all(egress, ec); fs::remove_all(dir, ec); } std::puts("ok"); return 0; }