Some checks failed
Build and Test / build (debug, clang) (push) Has been cancelled
Build and Test / build (debug, gcc) (push) Has been cancelled
Build and Test / build (release, clang) (push) Has been cancelled
Build and Test / build (release, gcc) (push) Has been cancelled
Build and Test / macos (push) Has been cancelled
Build and Test / docker (push) Has been cancelled
- Replay builds each window's WindowSummary as it goes (exec depth and locations, connections, writes, ptrace, module loads, scan outcomes, surprisal) and can collect training examples: features, a label (a detection or late label touched the window), time, root program and the rarity score. Sampled to at most 50k: positives up to a fifth, negatives by reservoir with at most 1% per root program. --inject-train traces become positive examples and are not evaluated. - learn/train: the value head (and hidden layers) learn P(teacher positive) with a class-weighted log loss and Adam; positive-unlabeled data, so unlabeled is "not known bad", documented. Split by time (newest 20% tested); reports test loss, top-k hits and ROC AUC for the model and for the rarity baseline. Refuses with fewer than 5 positives. Deterministic for a seed. The policy head waits for shadow-mode inspection outcomes. - CLI: `strawberry model train [store_dir] [--inject-train FILE] [--epochs N] [--seed N] [--out FILE] [--json]` writes the shadow model (generation after the newest existing one, a note saying what it was trained on); `strawberry model show [file]`; `strawberry replay --model FILE` ranks windows (and evaluates injections) by the model's value. Config model_dir (default /var/lib/strawberry/model). - doctor: the model check loads promoted.sbm and shadow.sbm; an invalid file is a problem (it is never used). Tests: test_train (gradients against finite differences, skipping steps that cross a ReLU kink; a planted pattern learned to high AUC; determinism; refusal without labels; roc_auc); test_replay R10 (examples, labels, per-program cap, training traces, model ranking); CLI and doctor cases. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
154 lines
6.1 KiB
C++
154 lines
6.1 KiB
C++
// Value-head training. A failure prints the seed; re-run with
|
|
// STRAWBERRY_PROPERTY_SEED=<seed>.
|
|
//
|
|
// T1 analytic gradients match finite differences
|
|
// T2 a planted pattern is learned: test AUC high, test loss falls
|
|
// T3 training is deterministic for a seed
|
|
// T4 too few teacher labels: refuses with a reason, model unchanged
|
|
// T5 roc_auc on known cases (ties count half)
|
|
|
|
#include "strawberry/learn/train.hpp"
|
|
|
|
#include <cmath>
|
|
#include <cstdlib>
|
|
#include <iostream>
|
|
#include <random>
|
|
|
|
using namespace strawberry::learn;
|
|
|
|
namespace {
|
|
|
|
uint64_t seed_from_env() {
|
|
if (const char* text = std::getenv("STRAWBERRY_PROPERTY_SEED")) return std::strtoull(text, nullptr, 10);
|
|
return std::random_device{}();
|
|
}
|
|
const uint64_t kSeed = seed_from_env();
|
|
|
|
#define PROPERTY(cond) \
|
|
do { \
|
|
if (!(cond)) { \
|
|
std::cerr << "PROPERTY FAILED: " #cond " (seed " << kSeed << ", " << __FILE__ << ":" \
|
|
<< __LINE__ << ")" << std::endl; \
|
|
std::abort(); \
|
|
} \
|
|
} while (0)
|
|
|
|
std::vector<float>* buffer_at(Model& m, std::size_t& index) {
|
|
for (auto* b : {&m.w1, &m.b1, &m.w2, &m.b2, &m.wp, &m.bp, &m.wv, &m.bv}) {
|
|
if (index < b->size()) return b;
|
|
index -= b->size();
|
|
}
|
|
return nullptr;
|
|
}
|
|
|
|
void test_gradients() {
|
|
std::mt19937_64 rng(kSeed);
|
|
auto m = Model::initialized(ModelShape{256, 8, 4, static_cast<uint32_t>(kActions)}, kSeed);
|
|
std::uniform_real_distribution<float> u(-1.0f, 1.0f);
|
|
for (auto* b : {&m.b1, &m.b2}) for (auto& f : *b) f = 0.1f; // keep units active
|
|
int checked = 0;
|
|
for (int iter = 0; iter < 200 && checked < 60; ++iter) {
|
|
TrainingExample e;
|
|
for (auto& f : e.x) f = u(rng);
|
|
e.label = static_cast<float>(rng() % 2);
|
|
const float w = 3.0f;
|
|
const std::size_t index = rng() % m.parameter_count();
|
|
const double analytic = value_gradient(m, e, w, index);
|
|
std::size_t local = index;
|
|
auto* buf = buffer_at(m, local);
|
|
const float saved = (*buf)[local];
|
|
const auto numeric_at = [&](float h) {
|
|
(*buf)[local] = saved + h;
|
|
const double up = value_loss(m, {e}, w);
|
|
(*buf)[local] = saved - h;
|
|
const double down = value_loss(m, {e}, w);
|
|
(*buf)[local] = saved;
|
|
return (up - down) / (2.0 * h);
|
|
};
|
|
const double n1 = numeric_at(1e-3f), n2 = numeric_at(5e-4f);
|
|
// A step that crosses a ReLU kink makes the two estimates disagree:
|
|
// skip that sample (the gradient there is one-sided).
|
|
if (std::fabs(n1 - n2) > 1e-3 + 1e-2 * std::fabs(n2)) continue;
|
|
++checked;
|
|
PROPERTY(std::fabs(analytic - n2) <= 2e-3 + 2e-2 * std::fabs(n2));
|
|
}
|
|
PROPERTY(checked >= 30);
|
|
std::cout << "T1 gradients: PASSED" << std::endl;
|
|
}
|
|
|
|
// Positives: memfd exec plus a public connection (x[7], x[19]); some noise
|
|
// windows have one of the two.
|
|
std::vector<TrainingExample> planted(std::mt19937_64& rng, std::size_t n) {
|
|
std::uniform_real_distribution<float> u(0.0f, 1.0f);
|
|
std::vector<TrainingExample> out;
|
|
for (std::size_t i = 0; i < n; ++i) {
|
|
TrainingExample e;
|
|
for (std::size_t k = 1; k < kDenseDims; ++k) e.x[k] = u(rng) < 0.2f ? u(rng) : 0.0f;
|
|
e.x[0] = 1.0f;
|
|
e.x[7] = 0.0f;
|
|
e.x[19] = 0.0f;
|
|
const bool positive = u(rng) < 0.06f;
|
|
if (positive) {
|
|
e.x[7] = 1.0f;
|
|
e.x[19] = 1.0f;
|
|
} else if (u(rng) < 0.3f) {
|
|
(u(rng) < 0.5f ? e.x[7] : e.x[19]) = 1.0f;
|
|
}
|
|
e.label = positive ? 1.0f : 0.0f;
|
|
e.start_ns = static_cast<int64_t>(i);
|
|
e.rarity_bits = u(rng) * 48.0f; // rarity knows nothing here
|
|
out.push_back(e);
|
|
}
|
|
return out;
|
|
}
|
|
|
|
void test_learning() {
|
|
std::mt19937_64 rng(kSeed);
|
|
const auto data = planted(rng, 3000);
|
|
auto m = Model::initialized(ModelShape{}, 5);
|
|
TrainOptions o;
|
|
o.epochs = 15;
|
|
o.seed = 9;
|
|
const auto r = train_value(m, data, o);
|
|
PROPERTY(r.error.empty() && r.test == 600 && r.test_positives > 0);
|
|
PROPERTY(r.test_loss < r.initial_test_loss);
|
|
PROPERTY(r.model_auc > 0.95);
|
|
PROPERTY(r.model_hits_at_k * 10 >= r.test_positives * 8); // most positives in the top k
|
|
std::cout << "T2 learns a planted pattern: PASSED (test AUC " << r.model_auc << ", rarity " << r.rarity_auc
|
|
<< ", loss " << r.initial_test_loss << " -> " << r.test_loss << ")" << std::endl;
|
|
|
|
auto a = Model::initialized(ModelShape{}, 5);
|
|
auto b = Model::initialized(ModelShape{}, 5);
|
|
(void) train_value(a, data, o);
|
|
(void) train_value(b, data, o);
|
|
PROPERTY(a.w1 == b.w1 && a.wv == b.wv && a.bv == b.bv); // T3
|
|
std::cout << "T3 deterministic: PASSED" << std::endl;
|
|
|
|
std::vector<TrainingExample> few = data;
|
|
for (auto& e : few) e.label = 0.0f;
|
|
few[3].label = 1.0f;
|
|
auto c = Model::initialized(ModelShape{}, 5);
|
|
const auto before = c.w1;
|
|
const auto refused = train_value(c, few, o);
|
|
PROPERTY(!refused.error.empty() && refused.error.find("not enough teacher labels") != std::string::npos);
|
|
PROPERTY(c.w1 == before); // T4
|
|
std::cout << "T4 refuses without labels: PASSED" << std::endl;
|
|
}
|
|
|
|
void test_auc() {
|
|
PROPERTY(roc_auc({0.1f, 0.4f, 0.35f, 0.8f}, {0, 0, 1, 1}) == 0.75);
|
|
PROPERTY(roc_auc({1, 2, 3}, {0, 1, 1}) == 1.0);
|
|
PROPERTY(roc_auc({0.5f, 0.5f}, {0, 1}) == 0.5); // tie
|
|
PROPERTY(roc_auc({1, 2}, {1, 1}) == 0.5); // one class
|
|
std::cout << "T5 roc_auc: PASSED" << std::endl;
|
|
}
|
|
|
|
} // namespace
|
|
|
|
int main() {
|
|
std::cout << "training properties, seed " << kSeed << std::endl;
|
|
test_auc();
|
|
test_gradients();
|
|
test_learning();
|
|
return 0;
|
|
}
|