strawberry/tests/test_model.cpp
trvon 7149a90893
gate: refuse in-sample and degenerate evidence, and identify the model
Four findings from a four-lane audit made the promotion gate unable to support
its own claims.

In-sample evaluation: training walks the whole post-warm-up store and the gate
evaluates the same windows, with nothing checking provenance - replay.cpp itself
listed point_in_time_splits_and_overlap_not_assessed. Models now record the
window range they learned from (file format v2; v1 still loads as unknown) and a
content fingerprint, evaluation reports its own range plus in_sample, and
promotion_gate fails with the reason: an overlapping range, no recorded range,
or no digest. Unprovable provenance never counts as out-of-sample.

Degenerate baseline: with rarity-greedy on 0 hits, hits_per_cpu_second() returns
0.0, so ">= 1.5x rarity" reduced to "model hit something" and the recall check
was vacuous. A cell whose baseline scored no hits or spent no CPU now fails,
naming the budget refusals that starved it.

Stale models: cells keyed on a bare filename, so a later cycle that refused to
train left an older cycle's dist/gate models to be judged, and nothing bound a
verdict to the file it judged. Cells and reports carry the model digest, and
fleet gate sets refused (non-zero exit) for a cell without one or for a
(model, period) whose digests disagree across nodes instead of pooling them.

3 seeds are not generalization: they vary initialisation and the per-node
shuffle over identical windows. The procedure document now says so and states
what would test generalization (disjoint periods, models trained on an earlier
cut, out-of-sample ranges).

Tests: in-sample flagged, disjoint range not flagged, unprovable provenance
fails, 0-hit and 0-CPU baselines fail, mixed digests refused, legacy report
refused, range round-trip and fingerprint, v1 compatibility.
2026-11-19 16:20:40 -07:00

239 lines
11 KiB
C++

// Inspection model: kernels, forward pass, model files, features. A failure
// prints the seed; re-run with STRAWBERRY_PROPERTY_SEED=<seed>.
//
// M1 every available kernel matches the scalar reference (any size)
// M2 forward: probabilities sum to 1, value in [0, 1]; a wrong input size
// or an inconsistent model gives the safe default (no inspection)
// M3 initialization is deterministic per seed
// M4 model files round-trip bit for bit; any flipped byte, truncation or
// non-finite weight is rejected with a reason; the directory is 0700
// F1 features are finite, bounded and deterministic; the dense part
// reflects the summary; tokens only touch the hashed part
#include "strawberry/hash_utils.hpp"
#include "strawberry/learn/features.hpp"
#include "strawberry/learn/mlp.hpp"
#include "strawberry/learn/model_file.hpp"
#include <sys/stat.h>
#include <unistd.h>
#include <cmath>
#include <cstdlib>
#include <cstring>
#include <filesystem>
#include <iostream>
#include <random>
#include <string>
#include <vector>
using namespace strawberry::learn;
namespace {
uint64_t seed_from_env() {
if (const char* text = std::getenv("STRAWBERRY_PROPERTY_SEED")) return std::strtoull(text, nullptr, 10);
return std::random_device{}();
}
const uint64_t kSeed = seed_from_env();
#define PROPERTY(cond) \
do { \
if (!(cond)) { \
std::cerr << "PROPERTY FAILED: " #cond " (seed " << kSeed << ", " << __FILE__ << ":" \
<< __LINE__ << ")" << std::endl; \
std::abort(); \
} \
} while (0)
void test_kernels() {
std::mt19937_64 rng(kSeed);
std::uniform_real_distribution<float> u(-1.0f, 1.0f);
std::vector<Kernel> kernels{Kernel::scalar, Kernel::avx2, Kernel::neon};
for (int iter = 0; iter < 300; ++iter) {
const std::size_t out = 1 + rng() % 70, in = 1 + rng() % 300; // not multiples of the vector width
std::vector<float> w(out * in), b(out), x(in), ref(out), got(out);
for (auto& f : w) f = u(rng);
for (auto& f : b) f = u(rng);
for (auto& f : x) f = u(rng);
const bool relu = rng() % 2;
dense(w.data(), b.data(), x.data(), ref.data(), out, in, relu, Kernel::scalar);
for (const auto k : kernels) {
dense(w.data(), b.data(), x.data(), got.data(), out, in, relu, k);
for (std::size_t o = 0; o < out; ++o) {
PROPERTY(std::fabs(got[o] - ref[o]) <= 1e-4f * (1.0f + std::fabs(ref[o])) + 1e-4f * std::sqrt(in));
}
}
}
std::cout << "M1 kernels match scalar (best here: " << kernel_name(best_kernel()) << "): PASSED" << std::endl;
}
void test_forward_and_init() {
const auto m = Model::initialized(ModelShape{}, 42);
PROPERTY(m.consistent() && m.parameter_count() == 256 * 128 + 128 + 128 * 64 + 64 + kActions * 64 + kActions + 64 + 1);
const auto again = Model::initialized(ModelShape{}, 42);
PROPERTY(again.w1 == m.w1 && again.wp == m.wp); // M3
PROPERTY(Model::initialized(ModelShape{}, 43).w1 != m.w1);
std::mt19937_64 rng(kSeed);
std::uniform_real_distribution<float> u(-2.0f, 2.0f);
for (int iter = 0; iter < 200; ++iter) {
FeatureVector x;
for (auto& f : x) f = u(rng);
const auto out = forward(m, x);
float sum = 0;
for (const auto p : out.policy) {
PROPERTY(p >= 0.0f && p <= 1.0f);
sum += p;
}
PROPERTY(std::fabs(sum - 1.0f) < 1e-4f && out.value >= 0.0f && out.value <= 1.0f);
const auto scalar = forward(m, x, Kernel::scalar);
for (std::size_t a = 0; a < kActions; ++a) PROPERTY(std::fabs(scalar.policy[a] - out.policy[a]) < 1e-3f);
}
std::vector<float> short_input(10, 1.0f);
const auto safe = forward(m, short_input);
PROPERTY(safe.policy[0] == 1.0f && safe.value == 0.0f);
Model broken = m;
broken.b2.pop_back();
const auto safe2 = forward(broken, FeatureVector{});
PROPERTY(safe2.policy[0] == 1.0f && safe2.value == 0.0f);
std::cout << "M2 forward, M3 init: PASSED" << std::endl;
}
void test_files() {
auto m = Model::initialized(ModelShape{}, 7);
m.generation = 3;
m.created_ns = 1'800'000'000'000'000'000;
m.note = "replay 2026-10-20..27, 1234 windows";
const auto bytes = encode_model(m);
Model back;
std::string error;
PROPERTY(decode_model(bytes, back, error));
PROPERTY(encode_model(back) == bytes && back.note == m.note && back.generation == 3);
// Gate integrity: the model records the window range it was trained on, and
// carries a content identity so a report can prove which bytes it judged.
m.training_first_ns = 1'793'000'000'000'000'000;
m.training_last_ns = 1'794'000'000'000'000'000;
const auto with_range = encode_model(m);
PROPERTY(decode_model(with_range, back, error));
PROPERTY(back.training_first_ns == m.training_first_ns && back.training_last_ns == m.training_last_ns);
PROPERTY(model_fingerprint(m) == model_fingerprint(back));
PROPERTY(model_fingerprint(m).size() == 16);
// An unchanged model keeps its identity; any change is a different model,
// even under the same file name.
Model other = m;
other.training_last_ns += 1;
PROPERTY(model_fingerprint(other) != model_fingerprint(m));
// A model without provenance (0/0) is encoded as such, not as an epoch range.
Model untrained = m;
untrained.training_first_ns = 0;
untrained.training_last_ns = 0;
PROPERTY(decode_model(encode_model(untrained), back, error));
PROPERTY(back.training_first_ns == 0 && back.training_last_ns == 0);
m.training_first_ns = 0;
m.training_last_ns = 0;
// 0/0 provenance is a stable encoding, not an epoch range.
PROPERTY(encode_model(untrained) == bytes);
// Version 1 files (no provenance field) still load; their range is unknown,
// which the gate must treat as unprovable rather than out-of-sample.
{
auto v1 = with_range; // v2 bytes
v1.erase(40, 16); // drop training_first/last
v1[4] = 1; // version = 1
v1[v1.size() - 64] = '\0'; // placeholder, recomputed below
auto body = v1.substr(0, v1.size() - 64);
const auto digest = strawberry::HashUtils::sha256_data(
reinterpret_cast<const unsigned char*>(body.data()), body.size());
v1.replace(v1.size() - 64, 64, digest);
PROPERTY(decode_model(v1, back, error));
PROPERTY(back.training_first_ns == 0 && back.training_last_ns == 0);
}
std::mt19937_64 rng(kSeed);
for (int iter = 0; iter < 200; ++iter) { // M4: any flipped byte is caught
auto bad = bytes;
bad[rng() % bad.size()] ^= static_cast<char>(1 + rng() % 255);
PROPERTY(!decode_model(bad, back, error) && !error.empty());
}
PROPERTY(!decode_model(bytes.substr(0, bytes.size() - 1), back, error));
PROPERTY(!decode_model("", back, error) && error == "file too short");
// A non-finite weight is rejected even with a valid checksum.
auto nan_model = m;
nan_model.w2[5] = std::nanf("");
PROPERTY(!decode_model(encode_model(nan_model), back, error) && error == "non-finite parameter");
namespace fs = std::filesystem;
const auto dir = fs::temp_directory_path() / ("strawberry_model_" + std::to_string(::getpid()));
std::error_code ec;
fs::remove_all(dir, ec);
PROPERTY(save_model(dir.string(), "shadow.sbm", m, error));
struct stat st {};
PROPERTY(::stat(dir.c_str(), &st) == 0 && (st.st_mode & 0777) == 0700);
PROPERTY(load_model((dir / "shadow.sbm").string(), back, error) && encode_model(back) == bytes);
PROPERTY(!fs::exists(dir / "shadow.sbm.tmp"));
PROPERTY(!save_model(dir.string(), "../escape.sbm", m, error));
Model inconsistent = m;
inconsistent.bv.clear();
PROPERTY(!save_model(dir.string(), "x.sbm", inconsistent, error));
fs::remove_all(dir, ec);
std::cout << "M4 model files: PASSED" << std::endl;
}
void test_features() {
WindowSummary empty;
const auto e = featurize(empty);
PROPERTY(e[0] == 1.0f);
for (std::size_t i = 1; i < kFeatureDim; ++i) PROPERTY(e[i] == 0.0f || i == 36); // 36: budget left
WindowSummary s;
note_exec(s, 100, "/memfd:x", "x", 0, true, false, 0);
note_exec(s, 200, "/home/u/.cache/tmp.Ab12Cd/run", "run -c 1", 1000, false, true, 2);
note_connect(s, 300, "203.0.113.9", 443);
note_file_write(s, 400, true, "systemd user unit");
note_surprisal(s, Surprisal{TupleKind::parent_child, "a > b", 16.0, 0});
note_surprisal(s, Surprisal{TupleKind::exe_connect, "c", 9.0, 3});
const auto f = featurize(s, DecisionContext{2, 0.25f});
PROPERTY(f[7] == 1.0f && f[8] == 1.0f && f[9] == 0.5f && f[19] == 1.0f && f[23] == 1.0f);
PROPERTY(f[25] == 1.0f && std::fabs(f[30] - 25.0f / 48.0f) < 1e-6f && std::fabs(f[35] - 2.0f / 3.0f) < 1e-6f);
PROPERTY(featurize(s, DecisionContext{2, 0.25f}) == f); // deterministic
for (const auto v : f) PROPERTY(std::isfinite(v) && v >= -2.0f && v <= 2.0f);
// Tokens touch only the hashed part.
WindowSummary t = s;
t.tokens.push_back("exe:/some/new/thing");
const auto g = featurize(t, DecisionContext{2, 0.25f});
for (std::size_t i = 0; i < kDenseDims; ++i) PROPERTY(g[i] == f[i]);
PROPERTY(g != f);
std::mt19937_64 rng(kSeed); // bounded under arbitrary input
for (int iter = 0; iter < 2000; ++iter) {
WindowSummary r;
for (int k = 0; k < 100; ++k) {
switch (rng() % 4) {
case 0: note_exec(r, static_cast<int64_t>(rng()), std::string(rng() % 40, 'a'), "x y z", rng() % 3, rng() % 2, rng() % 2, rng() % 50); break;
case 1: note_connect(r, static_cast<int64_t>(rng() % 1000), "10.0.0." + std::to_string(rng() % 300), static_cast<int>(rng() % 70000)); break;
case 2: note_file_write(r, 5, rng() % 2, std::to_string(rng())); break;
default: note_surprisal(r, Surprisal{static_cast<TupleKind>(rng() % kTupleKinds), "t", static_cast<double>(rng() % 100), 0}); break;
}
}
PROPERTY(r.tokens.size() <= kMaxSummaryTokens);
for (const auto v : featurize(r, DecisionContext{static_cast<uint8_t>(rng() % 9), 7.0f})) {
PROPERTY(std::isfinite(v) && v >= -2.0f && v <= 2.0f);
}
}
std::cout << "F1 features: PASSED" << std::endl;
}
} // namespace
int main() {
std::cout << "model properties, seed " << kSeed << std::endl;
test_kernels();
test_forward_and_init();
test_files();
test_features();
return 0;
}