Some checks failed
Build and Test / build (debug, clang) (push) Has been cancelled
Build and Test / build (debug, gcc) (push) Has been cancelled
Build and Test / build (release, clang) (push) Has been cancelled
Build and Test / build (release, gcc) (push) Has been cancelled
Build and Test / docker (push) Has been cancelled
The main source of teacher labels over time: rules improve daily, while most malware runs once. - The pipeline remembers every binary that ran (image_seen evidence record: sha256, path, first/last seen, size; at most one write per hash a day). Deleted and memfd images are skipped: nothing to find again. - Each rules load computes a digest of the rule files and IOC hash lists (paths after resolving symlinks, sizes, mtimes). - RetroHunter (src/retro_hunt.cpp, own thread, idle priority, paced at retro_hunt_pace_ms, deferred by the governor as WorkClass::retro_hunt) re-scans when the digest differs from the last completed pass, most recently seen first. Only paths that still hold the same content are scanned (O_NOFOLLOW, regular files, hash compared); others count as changed or missing. A completed pass is recorded (retro_hunt record) so a restart does not repeat it; an interrupted one starts over. - New matches (rules not recorded before for that hash, not allowlisted) become indicators with scan_kind retro_hunt, a file/indicator graph link, a positive label and a notification. Correlation turns them into findings titled "YARA match on a binary that ran here (found after a rules update)". Nothing is blocked or contained from here. - Replay reads positive labels from the evidence stream and marks windows that ran a binary labelled bad later (LATE in the CLI). - Status: a retro_hunt block (API_CONTRACT.md); config: retro_hunt, retro_hunt_pace_ms. Tests: test_retro_hunt (remembered once; passes only on a digest change and survive a restart; a clean-at-exec binary matching a new rule becomes an indicator, label, finding and notification; a replaced binary is counted as changed, never scanned as the old one; a stop abandons a pass unrecorded); test_replay R9 (late labels, from a real store). fuzz_evidence_records covers the new records. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
184 lines
7.7 KiB
C++
184 lines
7.7 KiB
C++
// Retro-hunt: re-scan binaries seen before when the rules change.
|
|
//
|
|
// H1 every binary that ran is remembered once (per day), with its path
|
|
// H2 a pass runs only when the rules digest changed since the last
|
|
// completed pass, and a completed pass survives a restart
|
|
// H3 a binary clean at exec time but matching a new rule becomes an
|
|
// indicator (scan_kind retro_hunt), a positive label, a finding and a
|
|
// notification; matches already recorded are not repeated
|
|
// H4 a path that now holds different content is counted as changed and
|
|
// never scanned as the old binary
|
|
|
|
#include "strawberry/config.hpp"
|
|
#include "strawberry/database.hpp"
|
|
#include "strawberry/detection_pipeline.hpp"
|
|
#include "strawberry/retro_hunt.hpp"
|
|
#include "strawberry/store/log_store.hpp"
|
|
#include "strawberry/telemetry.hpp"
|
|
|
|
#include <sys/stat.h>
|
|
#include <unistd.h>
|
|
|
|
#include <cstdlib>
|
|
#include <filesystem>
|
|
#include <fstream>
|
|
#include <iostream>
|
|
#include <string>
|
|
#include <vector>
|
|
|
|
#ifndef STRAWBERRY_HAVE_YARA
|
|
#define STRAWBERRY_HAVE_YARA 0
|
|
#endif
|
|
|
|
namespace fs = std::filesystem;
|
|
using namespace strawberry;
|
|
|
|
namespace {
|
|
|
|
#define CHECK(cond) \
|
|
do { \
|
|
if (!(cond)) { \
|
|
std::cerr << "CHECK FAILED: " #cond " (" << __FILE__ << ":" << __LINE__ << ")" << std::endl; \
|
|
std::abort(); \
|
|
} \
|
|
} while (0)
|
|
|
|
void write_file(const fs::path& p, const std::string& content) {
|
|
fs::create_directories(p.parent_path());
|
|
std::ofstream out(p, std::ios::binary | std::ios::trunc);
|
|
out << content;
|
|
}
|
|
|
|
std::string identity_of(const fs::path& p) {
|
|
struct stat st {};
|
|
CHECK(::stat(p.c_str(), &st) == 0);
|
|
return std::to_string(st.st_dev) + ":" + std::to_string(st.st_ino);
|
|
}
|
|
|
|
ProcessInfo exec_of(int pid, const fs::path& image) {
|
|
ProcessInfo info;
|
|
info.pid = pid;
|
|
info.executable_path = image.string();
|
|
info.image_path = image.string();
|
|
info.image_identity = identity_of(image);
|
|
info.start_time_ticks = static_cast<uint64_t>(pid);
|
|
return info;
|
|
}
|
|
|
|
std::size_t count_records(const std::string& db, store::RecordType type) {
|
|
std::size_t n = 0;
|
|
store::scan_store_read_only(db, store::Stream::evidence, [&](uint32_t t, std::string_view) {
|
|
n += t == static_cast<uint32_t>(type) ? 1 : 0;
|
|
});
|
|
return n;
|
|
}
|
|
|
|
void test_retro_hunt() {
|
|
const auto root = fs::temp_directory_path() / ("strawberry_retro_hunt_" + std::to_string(::getpid()));
|
|
std::error_code ec;
|
|
fs::remove_all(root, ec);
|
|
const auto rules = root / "rules";
|
|
const auto tool = root / "bin" / "tool"; // clean at exec time
|
|
const auto other = root / "bin" / "other"; // detected at exec time
|
|
write_file(rules / "a.yar", "rule AlphaRule { meta: severity = \"high\" strings: $a = \"alpha-marker\" condition: $a }\n");
|
|
write_file(tool, std::string("\x7f\x45\x4c\x46") + " beta-marker ");
|
|
write_file(other, std::string("\x7f\x45\x4c\x46") + " alpha-marker ");
|
|
|
|
Config config;
|
|
config.db_path = (root / "db").string();
|
|
config.yara_rules_path = rules.string();
|
|
std::vector<DetectionEvent> notified;
|
|
{
|
|
DetectionPipeline pipeline;
|
|
CHECK(pipeline.initialize(config));
|
|
pipeline.set_detection_callback([&](const DetectionEvent& e) { notified.push_back(e); });
|
|
const auto first_digest = pipeline.rules_digest();
|
|
CHECK(first_digest.size() == 64);
|
|
|
|
pipeline.handle_process(exec_of(100, tool));
|
|
pipeline.handle_process(exec_of(101, other));
|
|
pipeline.handle_process(exec_of(102, tool)); // same binary again: remembered once (H1)
|
|
CHECK(pipeline.seen_images().size() == 2);
|
|
CHECK(count_records(config.db_path, store::RecordType::image_seen) == 2);
|
|
const auto exec_detections = pipeline.stats().detections;
|
|
notified.clear();
|
|
|
|
RetroHuntOptions options;
|
|
options.pace = std::chrono::milliseconds(0);
|
|
RetroHunter hunter(pipeline, nullptr, options);
|
|
const auto go = [] { return true; };
|
|
CHECK(hunter.run_once(go)); // first pass: nothing new (AlphaRule known)
|
|
CHECK(hunter.status().scanned == 2 && hunter.status().hits == 0);
|
|
CHECK(pipeline.stats().detections == exec_detections && notified.empty());
|
|
CHECK(!hunter.run_once(go)); // same digest: no work (H2)
|
|
CHECK(!pipeline.reload_rules() || !hunter.run_once(go)); // reload, unchanged rules: still none
|
|
|
|
// A new rule that matches the clean binary; the other binary is replaced.
|
|
write_file(rules / "b.yar", "rule BetaRule { meta: severity = \"critical\" strings: $b = \"beta-marker\" condition: $b }\n");
|
|
write_file(other, std::string("\x7f\x45\x4c\x46") + " replaced alpha-marker ");
|
|
CHECK(pipeline.reload_rules());
|
|
CHECK(pipeline.rules_digest() != first_digest);
|
|
CHECK(hunter.run_once(go));
|
|
const auto s = hunter.status();
|
|
CHECK(s.scanned == 1 && s.changed == 1 && s.hits == 1); // H3, H4
|
|
CHECK(notified.size() == 1 && notified[0].top_rule == "BetaRule" && notified[0].path == tool.string());
|
|
CHECK(!hunter.run_once(go));
|
|
|
|
CHECK(pipeline.run_pending_correlation());
|
|
bool retro_finding = false;
|
|
Database db;
|
|
CHECK(db.open_read_only(config.db_path));
|
|
for (const auto& f : db.list_findings()) {
|
|
retro_finding |= f.title.find("after a rules update") != std::string::npos;
|
|
}
|
|
const auto indicators = db.find_indicators_by_rule("BetaRule");
|
|
CHECK(indicators.size() == 1 && indicators[0].metadata.at("scan_kind") == "retro_hunt");
|
|
CHECK(indicators[0].metadata.at("process_path") == tool.string());
|
|
CHECK(retro_finding);
|
|
db.close();
|
|
pipeline.shutdown();
|
|
}
|
|
// Labels: one positive retro_hunt label for the new match.
|
|
std::size_t positives = 0;
|
|
store::scan_store_read_only(config.db_path, store::Stream::evidence, [&](uint32_t type, std::string_view payload) {
|
|
if (type != static_cast<uint32_t>(store::RecordType::label)) return;
|
|
LabelRecord label;
|
|
CHECK(decode_label(payload, label));
|
|
if (label.source == "retro_hunt") {
|
|
CHECK(label.polarity == LabelPolarity::positive && label.rule == "BetaRule");
|
|
++positives;
|
|
}
|
|
});
|
|
CHECK(positives == 1);
|
|
|
|
// H2 across a restart: the completed pass is remembered.
|
|
{
|
|
DetectionPipeline pipeline;
|
|
CHECK(pipeline.initialize(config));
|
|
CHECK(pipeline.seen_images().size() == 2);
|
|
CHECK(pipeline.last_retro_hunt().has_value());
|
|
RetroHuntOptions options;
|
|
options.pace = std::chrono::milliseconds(0);
|
|
RetroHunter hunter(pipeline, nullptr, options);
|
|
CHECK(!hunter.run_once([] { return true; }));
|
|
// A stop request abandons a pass without recording it.
|
|
write_file(rules / "c.yar", "rule GammaRule { strings: $c = \"gamma\" condition: $c }\n");
|
|
CHECK(pipeline.reload_rules());
|
|
CHECK(!hunter.run_once([] { return false; }));
|
|
CHECK(pipeline.last_retro_hunt()->rules_digest != pipeline.rules_digest());
|
|
pipeline.shutdown();
|
|
}
|
|
fs::remove_all(root, ec);
|
|
std::cout << "H1-H4 retro-hunt: PASSED" << std::endl;
|
|
}
|
|
|
|
} // namespace
|
|
|
|
int main() {
|
|
#if STRAWBERRY_HAVE_YARA
|
|
test_retro_hunt();
|
|
#else
|
|
std::cout << "retro-hunt: skipped (built without YARA)" << std::endl;
|
|
#endif
|
|
return 0;
|
|
}
|