strawberry/tests/test_shadow.cpp
trvon f76db466eb
Some checks failed
Build and Test / build (debug, clang) (push) Has been cancelled
Build and Test / build (debug, gcc) (push) Has been cancelled
Build and Test / build (release, clang) (push) Has been cancelled
Build and Test / build (release, gcc) (push) Has been cancelled
Build and Test / macos (push) Has been cancelled
Build and Test / docker (push) Has been cancelled
Stage 4: keep the live rarity model across daemon restarts
The shadow scorer's rarity model started empty at every daemon start, so
each restart cost an hour of warm-up with no decisions. The daemon now
saves it to model_dir/rarity.state (checksummed, atomic write, hourly and
at stop) and restores it at start. A restored state younger than the
7-day half-life skips the warm-up; decay catches up on the first event.

A file made with other rarity settings, a bad checksum, or a negative or
non-finite count is refused with a reason (status model.rarity_state,
doctor model.rarity_state) and the model starts empty. The file must be
owned by the daemon user and not group/world writable, like model files.

Also fixes status model.warmup_windows, which was never filled in.

Tests: R1/R2 in test_learn (round trip, decay clock, refusals), S8 in
test_shadow (warm-up skipped only when recent), daemon restart restores
the saved state, doctor warning; new fuzz_rarity_state target.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-27 23:25:01 -06:00

266 lines
11 KiB
C++

// Shadow mode. A failure prints the seed; re-run with STRAWBERRY_PROPERTY_SEED=<seed>.
//
// S1 without a model: windows are tracked, nothing is decided
// S2 a confident model asks for a memory scan, an unconfident one for
// nothing; propensities are 1 - eps (greedy) and eps / (A - 1) (explore)
// S3 the hourly budget is never exceeded; refusals are counted
// S4 decisions are deterministic for a seed
// S5 exploration happens at about the configured rate
// S6 inspection_action records round-trip; bad actions are rejected
// S8 a restored rarity state skips the warm-up, unless it is older than
// the half-life; a bad state is refused with a reason
#include "strawberry/learn/shadow.hpp"
#include "strawberry/telemetry.hpp"
#include <cmath>
#include <cstdlib>
#include <iostream>
#include <random>
using namespace strawberry;
using namespace strawberry::learn;
namespace {
uint64_t seed_from_env() {
if (const char* text = std::getenv("STRAWBERRY_PROPERTY_SEED")) return std::strtoull(text, nullptr, 10);
return std::random_device{}();
}
const uint64_t kSeed = seed_from_env();
#define PROPERTY(cond) \
do { \
if (!(cond)) { \
std::cerr << "PROPERTY FAILED: " #cond " (seed " << kSeed << ", " << __FILE__ << ":" \
<< __LINE__ << ")" << std::endl; \
std::abort(); \
} \
} while (0)
constexpr int64_t kSec = 1'000'000'000;
constexpr int64_t kT0 = 1'800'000'000 * kSec;
// `n` windows, one every 61 s, then enough time for all of them to close.
void feed_windows(ShadowScorer& s, int n) {
for (int i = 0; i < n; ++i) {
ProcessExecRecord r;
r.timestamp_ns = kT0 + i * 61 * kSec;
r.pid = 100 + i;
r.process_node_id = "process_" + std::to_string(100 + i) + "_b_1";
r.exe = "/usr/bin/tool" + std::to_string(i % 7);
r.cmdline = "tool";
r.source = "live";
s.feed(store::RecordType::process_exec, encode_process_exec(r));
}
// A late record moves time on: every window closes.
ProcessExecRecord tail;
tail.timestamp_ns = kT0 + (n + 10) * 61 * kSec;
tail.pid = 9;
tail.process_node_id = "process_9_b_1";
tail.exe = "/usr/bin/tail";
s.feed(store::RecordType::process_exec, encode_process_exec(tail));
ProcessExecRecord later = tail;
later.timestamp_ns += 600 * kSec;
later.pid = 10;
later.process_node_id = "process_10_b_1";
s.feed(store::RecordType::process_exec, encode_process_exec(later));
}
std::shared_ptr<Model> model_with_value_bias(float bias) {
auto m = std::make_shared<Model>(Model::initialized(ModelShape{}, 1));
std::fill(m->wv.begin(), m->wv.end(), 0.0f);
m->bv[0] = bias; // value = sigmoid(bias) for every window
m->generation = 7;
return m;
}
ShadowOptions no_warmup(double explore) {
ShadowOptions o;
o.warmup_ns = 0;
o.explore = explore;
o.seed = kSeed;
o.budget_seconds_per_hour = 1e9;
return o;
}
void test_basics() {
ShadowScorer none(no_warmup(0));
feed_windows(none, 20);
PROPERTY(none.take_decisions().empty() && none.stats().windows >= 20 && !none.stats().active); // S1
ShadowScorer confident(no_warmup(0));
confident.set_model(model_with_value_bias(8.0f));
feed_windows(confident, 20);
const auto d = confident.take_decisions();
PROPERTY(d.size() >= 20);
for (const auto& x : d) {
PROPERTY(x.action == InspectAction::memory_scan && x.reason == "model" && x.propensity == 1.0f);
PROPERTY(x.value > 0.99f && x.model_generation == 7 && x.shadow);
PROPERTY(x.root_exe.find('<') == std::string::npos); // actual image, not a path template
}
ShadowScorer unsure(no_warmup(0));
unsure.set_model(model_with_value_bias(-8.0f));
feed_windows(unsure, 20);
for (const auto& x : unsure.take_decisions()) {
PROPERTY(x.action == InspectAction::none && x.reason == "below_cost"); // S2
}
std::cout << "S1 no model, S2 greedy decisions: PASSED" << std::endl;
}
void test_budget_and_determinism() {
auto o = no_warmup(0);
o.budget_seconds_per_hour = 5; // memory scans cost 2 s: at most two per hour
ShadowScorer s(o);
s.set_model(model_with_value_bias(8.0f));
feed_windows(s, 120); // about two hours of windows
const auto d = s.take_decisions();
std::size_t scans = 0, refused = 0;
int64_t hour = 0;
double spent = 0;
for (const auto& x : d) {
if (hour == 0 || x.timestamp_ns - hour >= 3600 * kSec) {
hour = x.timestamp_ns;
spent = 0;
}
if (x.action == InspectAction::memory_scan) {
++scans;
spent += 2.0;
}
refused += x.reason == "budget";
PROPERTY(spent <= 5.0); // S3
}
PROPERTY(scans >= 2 && refused > 0 && s.stats().budget_refusals == refused);
ShadowScorer a(no_warmup(0.3)), b(no_warmup(0.3));
a.set_model(model_with_value_bias(0.5f));
b.set_model(model_with_value_bias(0.5f));
feed_windows(a, 60);
feed_windows(b, 60);
const auto da = a.take_decisions(), db = b.take_decisions();
PROPERTY(da.size() == db.size());
for (std::size_t i = 0; i < da.size(); ++i) {
PROPERTY(da[i].action == db[i].action && da[i].reason == db[i].reason); // S4
}
std::cout << "S3 budget, S4 deterministic: PASSED" << std::endl;
}
void test_exploration_and_codec() {
ShadowScorer s(no_warmup(0.2));
s.set_model(model_with_value_bias(8.0f));
feed_windows(s, 1000);
const auto d = s.take_decisions();
std::size_t explored = 0;
for (const auto& x : d) {
if (x.reason == "explore") {
++explored;
PROPERTY(x.action != InspectAction::memory_scan); // never the greedy action
PROPERTY(std::fabs(x.propensity - 0.2f / static_cast<float>(kActions - 1)) < 1e-6f);
} else {
PROPERTY(std::fabs(x.propensity - 0.8f) < 1e-6f);
}
}
const double rate = static_cast<double>(explored) / static_cast<double>(d.size());
PROPERTY(rate > 0.14 && rate < 0.26); // S5
PROPERTY(s.stats().explored == explored);
InspectionDecision x = d.front();
x.root_exe = "/usr/bin/\xff odd";
InspectionDecision y;
PROPERTY(decode_inspection_action(encode_inspection_action(x), y));
PROPERTY(y.action == x.action && y.propensity == x.propensity && y.value == x.value && y.root_node == x.root_node &&
y.benefit == x.benefit && y.reason == x.reason && y.model_generation == 7);
PROPERTY(!decode_inspection_action(std::string("\x20\x09", 2), y)); // action 9: rejected
std::cout << "S5 exploration rate " << rate << ", S6 codec: PASSED" << std::endl;
}
void test_promoted_actions() {
auto options = no_warmup(0);
options.budget_seconds_per_hour = 5;
ShadowScorer scorer(options);
scorer.set_model(model_with_value_bias(8.0f));
scorer.set_promoted(model_with_value_bias(8.0f));
feed_windows(scorer, 40);
const auto decisions = scorer.take_decisions();
const auto actions = scorer.take_actions();
std::size_t shadow_scans = 0, acting_scans = 0;
for (const auto& d : decisions) {
if (d.action == InspectAction::none) continue;
PROPERTY(kExecutableActions[static_cast<std::size_t>(d.action)]);
(d.shadow ? shadow_scans : acting_scans)++;
}
PROPERTY(shadow_scans == 2 && acting_scans == 2 && actions.size() == acting_scans);
for (const auto& d : actions) PROPERTY(!d.shadow && d.model_generation == 7);
const auto spent = scorer.stats().act_spend_seconds_this_hour;
scorer.set_promoted(nullptr);
PROPERTY(!scorer.stats().acting && scorer.take_actions().empty());
auto replacement = model_with_value_bias(8.0f);
replacement->generation = 8;
scorer.set_promoted(replacement);
PROPERTY(scorer.stats().act_spend_seconds_this_hour == spent);
// Exploration must never choose a non-executable action; even an
// undrained caller cannot accumulate an unbounded action queue.
auto exploring = no_warmup(1);
ShadowScorer bounded(exploring);
bounded.set_promoted(model_with_value_bias(8.0f));
feed_windows(bounded, 1000);
const auto queued = bounded.take_actions();
PROPERTY(queued.size() <= 64 && bounded.stats().action_queue_drops > 0);
for (const auto& d : queued) {
PROPERTY(!d.shadow && d.action != InspectAction::none);
PROPERTY(kExecutableActions[static_cast<std::size_t>(d.action)]);
}
ShadowScorer warmup(ShadowOptions{});
warmup.set_promoted(model_with_value_bias(8.0f));
feed_windows(warmup, 10);
PROPERTY(warmup.take_actions().empty());
std::cout << "S7 promoted budget, revocation, bounded executable actions and warmup: PASSED" << std::endl;
}
} // namespace
void test_rarity_restore() {
ShadowOptions o = no_warmup(0);
o.warmup_ns = 3600 * kSec;
ShadowScorer first(o);
first.set_model(model_with_value_bias(8.0f));
feed_windows(first, 20); // all inside the first hour: learned only
PROPERTY(first.take_decisions().empty() && first.stats().warmup_windows >= 20);
const int64_t saved = kT0 + 2 * 3600 * kSec;
const auto state = first.rarity_state(saved);
ShadowScorer restored(o);
std::string error;
PROPERTY(restored.restore_rarity(state, saved + 600 * kSec, error));
restored.set_model(model_with_value_bias(8.0f));
feed_windows(restored, 20);
PROPERTY(restored.take_decisions().size() >= 20); // S8
PROPERTY(restored.stats().rarity_restored && restored.stats().warmup_skipped &&
restored.stats().rarity_restored_saved_ns == saved);
ShadowScorer stale(o);
PROPERTY(stale.restore_rarity(state, saved + 8LL * 24 * 3600 * kSec, error));
stale.set_model(model_with_value_bias(8.0f));
feed_windows(stale, 20);
PROPERTY(stale.take_decisions().empty() && stale.stats().rarity_restored && !stale.stats().warmup_skipped);
ShadowScorer future(o); // saved "after" now: the clock went back
PROPERTY(future.restore_rarity(state, saved - kSec, error) && !future.stats().warmup_skipped);
ShadowScorer bad(o);
error.clear();
PROPERTY(!bad.restore_rarity("not a state file", saved, error) && !error.empty());
PROPERTY(!bad.stats().rarity_restored);
std::cout << "S8 rarity state restore: PASSED" << std::endl;
}
int main() {
std::cout << "shadow properties, seed " << kSeed << std::endl;
test_basics();
test_budget_and_determinism();
test_exploration_and_codec();
test_promoted_actions();
test_rarity_restore();
return 0;
}