Four findings from a four-lane audit made the promotion gate unable to support its own claims. In-sample evaluation: training walks the whole post-warm-up store and the gate evaluates the same windows, with nothing checking provenance - replay.cpp itself listed point_in_time_splits_and_overlap_not_assessed. Models now record the window range they learned from (file format v2; v1 still loads as unknown) and a content fingerprint, evaluation reports its own range plus in_sample, and promotion_gate fails with the reason: an overlapping range, no recorded range, or no digest. Unprovable provenance never counts as out-of-sample. Degenerate baseline: with rarity-greedy on 0 hits, hits_per_cpu_second() returns 0.0, so ">= 1.5x rarity" reduced to "model hit something" and the recall check was vacuous. A cell whose baseline scored no hits or spent no CPU now fails, naming the budget refusals that starved it. Stale models: cells keyed on a bare filename, so a later cycle that refused to train left an older cycle's dist/gate models to be judged, and nothing bound a verdict to the file it judged. Cells and reports carry the model digest, and fleet gate sets refused (non-zero exit) for a cell without one or for a (model, period) whose digests disagree across nodes instead of pooling them. 3 seeds are not generalization: they vary initialisation and the per-node shuffle over identical windows. The procedure document now says so and states what would test generalization (disjoint periods, models trained on an earlier cut, out-of-sample ranges). Tests: in-sample flagged, disjoint range not flagged, unprovable provenance fails, 0-hit and 0-CPU baselines fail, mixed digests refused, legacy report refused, range round-trip and fingerprint, v1 compatibility.
239 lines
11 KiB
C++
239 lines
11 KiB
C++
// Inspection model: kernels, forward pass, model files, features. A failure
|
|
// prints the seed; re-run with STRAWBERRY_PROPERTY_SEED=<seed>.
|
|
//
|
|
// M1 every available kernel matches the scalar reference (any size)
|
|
// M2 forward: probabilities sum to 1, value in [0, 1]; a wrong input size
|
|
// or an inconsistent model gives the safe default (no inspection)
|
|
// M3 initialization is deterministic per seed
|
|
// M4 model files round-trip bit for bit; any flipped byte, truncation or
|
|
// non-finite weight is rejected with a reason; the directory is 0700
|
|
// F1 features are finite, bounded and deterministic; the dense part
|
|
// reflects the summary; tokens only touch the hashed part
|
|
|
|
#include "strawberry/hash_utils.hpp"
|
|
#include "strawberry/learn/features.hpp"
|
|
#include "strawberry/learn/mlp.hpp"
|
|
#include "strawberry/learn/model_file.hpp"
|
|
|
|
#include <sys/stat.h>
|
|
#include <unistd.h>
|
|
|
|
#include <cmath>
|
|
#include <cstdlib>
|
|
#include <cstring>
|
|
#include <filesystem>
|
|
#include <iostream>
|
|
#include <random>
|
|
#include <string>
|
|
#include <vector>
|
|
|
|
using namespace strawberry::learn;
|
|
|
|
namespace {
|
|
|
|
uint64_t seed_from_env() {
|
|
if (const char* text = std::getenv("STRAWBERRY_PROPERTY_SEED")) return std::strtoull(text, nullptr, 10);
|
|
return std::random_device{}();
|
|
}
|
|
const uint64_t kSeed = seed_from_env();
|
|
|
|
#define PROPERTY(cond) \
|
|
do { \
|
|
if (!(cond)) { \
|
|
std::cerr << "PROPERTY FAILED: " #cond " (seed " << kSeed << ", " << __FILE__ << ":" \
|
|
<< __LINE__ << ")" << std::endl; \
|
|
std::abort(); \
|
|
} \
|
|
} while (0)
|
|
|
|
void test_kernels() {
|
|
std::mt19937_64 rng(kSeed);
|
|
std::uniform_real_distribution<float> u(-1.0f, 1.0f);
|
|
std::vector<Kernel> kernels{Kernel::scalar, Kernel::avx2, Kernel::neon};
|
|
for (int iter = 0; iter < 300; ++iter) {
|
|
const std::size_t out = 1 + rng() % 70, in = 1 + rng() % 300; // not multiples of the vector width
|
|
std::vector<float> w(out * in), b(out), x(in), ref(out), got(out);
|
|
for (auto& f : w) f = u(rng);
|
|
for (auto& f : b) f = u(rng);
|
|
for (auto& f : x) f = u(rng);
|
|
const bool relu = rng() % 2;
|
|
dense(w.data(), b.data(), x.data(), ref.data(), out, in, relu, Kernel::scalar);
|
|
for (const auto k : kernels) {
|
|
dense(w.data(), b.data(), x.data(), got.data(), out, in, relu, k);
|
|
for (std::size_t o = 0; o < out; ++o) {
|
|
PROPERTY(std::fabs(got[o] - ref[o]) <= 1e-4f * (1.0f + std::fabs(ref[o])) + 1e-4f * std::sqrt(in));
|
|
}
|
|
}
|
|
}
|
|
std::cout << "M1 kernels match scalar (best here: " << kernel_name(best_kernel()) << "): PASSED" << std::endl;
|
|
}
|
|
|
|
void test_forward_and_init() {
|
|
const auto m = Model::initialized(ModelShape{}, 42);
|
|
PROPERTY(m.consistent() && m.parameter_count() == 256 * 128 + 128 + 128 * 64 + 64 + kActions * 64 + kActions + 64 + 1);
|
|
const auto again = Model::initialized(ModelShape{}, 42);
|
|
PROPERTY(again.w1 == m.w1 && again.wp == m.wp); // M3
|
|
PROPERTY(Model::initialized(ModelShape{}, 43).w1 != m.w1);
|
|
|
|
std::mt19937_64 rng(kSeed);
|
|
std::uniform_real_distribution<float> u(-2.0f, 2.0f);
|
|
for (int iter = 0; iter < 200; ++iter) {
|
|
FeatureVector x;
|
|
for (auto& f : x) f = u(rng);
|
|
const auto out = forward(m, x);
|
|
float sum = 0;
|
|
for (const auto p : out.policy) {
|
|
PROPERTY(p >= 0.0f && p <= 1.0f);
|
|
sum += p;
|
|
}
|
|
PROPERTY(std::fabs(sum - 1.0f) < 1e-4f && out.value >= 0.0f && out.value <= 1.0f);
|
|
const auto scalar = forward(m, x, Kernel::scalar);
|
|
for (std::size_t a = 0; a < kActions; ++a) PROPERTY(std::fabs(scalar.policy[a] - out.policy[a]) < 1e-3f);
|
|
}
|
|
std::vector<float> short_input(10, 1.0f);
|
|
const auto safe = forward(m, short_input);
|
|
PROPERTY(safe.policy[0] == 1.0f && safe.value == 0.0f);
|
|
Model broken = m;
|
|
broken.b2.pop_back();
|
|
const auto safe2 = forward(broken, FeatureVector{});
|
|
PROPERTY(safe2.policy[0] == 1.0f && safe2.value == 0.0f);
|
|
std::cout << "M2 forward, M3 init: PASSED" << std::endl;
|
|
}
|
|
|
|
void test_files() {
|
|
auto m = Model::initialized(ModelShape{}, 7);
|
|
m.generation = 3;
|
|
m.created_ns = 1'800'000'000'000'000'000;
|
|
m.note = "replay 2026-10-20..27, 1234 windows";
|
|
const auto bytes = encode_model(m);
|
|
Model back;
|
|
std::string error;
|
|
PROPERTY(decode_model(bytes, back, error));
|
|
PROPERTY(encode_model(back) == bytes && back.note == m.note && back.generation == 3);
|
|
|
|
// Gate integrity: the model records the window range it was trained on, and
|
|
// carries a content identity so a report can prove which bytes it judged.
|
|
m.training_first_ns = 1'793'000'000'000'000'000;
|
|
m.training_last_ns = 1'794'000'000'000'000'000;
|
|
const auto with_range = encode_model(m);
|
|
PROPERTY(decode_model(with_range, back, error));
|
|
PROPERTY(back.training_first_ns == m.training_first_ns && back.training_last_ns == m.training_last_ns);
|
|
PROPERTY(model_fingerprint(m) == model_fingerprint(back));
|
|
PROPERTY(model_fingerprint(m).size() == 16);
|
|
// An unchanged model keeps its identity; any change is a different model,
|
|
// even under the same file name.
|
|
Model other = m;
|
|
other.training_last_ns += 1;
|
|
PROPERTY(model_fingerprint(other) != model_fingerprint(m));
|
|
// A model without provenance (0/0) is encoded as such, not as an epoch range.
|
|
Model untrained = m;
|
|
untrained.training_first_ns = 0;
|
|
untrained.training_last_ns = 0;
|
|
PROPERTY(decode_model(encode_model(untrained), back, error));
|
|
PROPERTY(back.training_first_ns == 0 && back.training_last_ns == 0);
|
|
m.training_first_ns = 0;
|
|
m.training_last_ns = 0;
|
|
// 0/0 provenance is a stable encoding, not an epoch range.
|
|
PROPERTY(encode_model(untrained) == bytes);
|
|
|
|
// Version 1 files (no provenance field) still load; their range is unknown,
|
|
// which the gate must treat as unprovable rather than out-of-sample.
|
|
{
|
|
auto v1 = with_range; // v2 bytes
|
|
v1.erase(40, 16); // drop training_first/last
|
|
v1[4] = 1; // version = 1
|
|
v1[v1.size() - 64] = '\0'; // placeholder, recomputed below
|
|
auto body = v1.substr(0, v1.size() - 64);
|
|
const auto digest = strawberry::HashUtils::sha256_data(
|
|
reinterpret_cast<const unsigned char*>(body.data()), body.size());
|
|
v1.replace(v1.size() - 64, 64, digest);
|
|
PROPERTY(decode_model(v1, back, error));
|
|
PROPERTY(back.training_first_ns == 0 && back.training_last_ns == 0);
|
|
}
|
|
|
|
std::mt19937_64 rng(kSeed);
|
|
for (int iter = 0; iter < 200; ++iter) { // M4: any flipped byte is caught
|
|
auto bad = bytes;
|
|
bad[rng() % bad.size()] ^= static_cast<char>(1 + rng() % 255);
|
|
PROPERTY(!decode_model(bad, back, error) && !error.empty());
|
|
}
|
|
PROPERTY(!decode_model(bytes.substr(0, bytes.size() - 1), back, error));
|
|
PROPERTY(!decode_model("", back, error) && error == "file too short");
|
|
|
|
// A non-finite weight is rejected even with a valid checksum.
|
|
auto nan_model = m;
|
|
nan_model.w2[5] = std::nanf("");
|
|
PROPERTY(!decode_model(encode_model(nan_model), back, error) && error == "non-finite parameter");
|
|
|
|
namespace fs = std::filesystem;
|
|
const auto dir = fs::temp_directory_path() / ("strawberry_model_" + std::to_string(::getpid()));
|
|
std::error_code ec;
|
|
fs::remove_all(dir, ec);
|
|
PROPERTY(save_model(dir.string(), "shadow.sbm", m, error));
|
|
struct stat st {};
|
|
PROPERTY(::stat(dir.c_str(), &st) == 0 && (st.st_mode & 0777) == 0700);
|
|
PROPERTY(load_model((dir / "shadow.sbm").string(), back, error) && encode_model(back) == bytes);
|
|
PROPERTY(!fs::exists(dir / "shadow.sbm.tmp"));
|
|
PROPERTY(!save_model(dir.string(), "../escape.sbm", m, error));
|
|
Model inconsistent = m;
|
|
inconsistent.bv.clear();
|
|
PROPERTY(!save_model(dir.string(), "x.sbm", inconsistent, error));
|
|
fs::remove_all(dir, ec);
|
|
std::cout << "M4 model files: PASSED" << std::endl;
|
|
}
|
|
|
|
void test_features() {
|
|
WindowSummary empty;
|
|
const auto e = featurize(empty);
|
|
PROPERTY(e[0] == 1.0f);
|
|
for (std::size_t i = 1; i < kFeatureDim; ++i) PROPERTY(e[i] == 0.0f || i == 36); // 36: budget left
|
|
|
|
WindowSummary s;
|
|
note_exec(s, 100, "/memfd:x", "x", 0, true, false, 0);
|
|
note_exec(s, 200, "/home/u/.cache/tmp.Ab12Cd/run", "run -c 1", 1000, false, true, 2);
|
|
note_connect(s, 300, "203.0.113.9", 443);
|
|
note_file_write(s, 400, true, "systemd user unit");
|
|
note_surprisal(s, Surprisal{TupleKind::parent_child, "a > b", 16.0, 0});
|
|
note_surprisal(s, Surprisal{TupleKind::exe_connect, "c", 9.0, 3});
|
|
const auto f = featurize(s, DecisionContext{2, 0.25f});
|
|
PROPERTY(f[7] == 1.0f && f[8] == 1.0f && f[9] == 0.5f && f[19] == 1.0f && f[23] == 1.0f);
|
|
PROPERTY(f[25] == 1.0f && std::fabs(f[30] - 25.0f / 48.0f) < 1e-6f && std::fabs(f[35] - 2.0f / 3.0f) < 1e-6f);
|
|
PROPERTY(featurize(s, DecisionContext{2, 0.25f}) == f); // deterministic
|
|
for (const auto v : f) PROPERTY(std::isfinite(v) && v >= -2.0f && v <= 2.0f);
|
|
|
|
// Tokens touch only the hashed part.
|
|
WindowSummary t = s;
|
|
t.tokens.push_back("exe:/some/new/thing");
|
|
const auto g = featurize(t, DecisionContext{2, 0.25f});
|
|
for (std::size_t i = 0; i < kDenseDims; ++i) PROPERTY(g[i] == f[i]);
|
|
PROPERTY(g != f);
|
|
|
|
std::mt19937_64 rng(kSeed); // bounded under arbitrary input
|
|
for (int iter = 0; iter < 2000; ++iter) {
|
|
WindowSummary r;
|
|
for (int k = 0; k < 100; ++k) {
|
|
switch (rng() % 4) {
|
|
case 0: note_exec(r, static_cast<int64_t>(rng()), std::string(rng() % 40, 'a'), "x y z", rng() % 3, rng() % 2, rng() % 2, rng() % 50); break;
|
|
case 1: note_connect(r, static_cast<int64_t>(rng() % 1000), "10.0.0." + std::to_string(rng() % 300), static_cast<int>(rng() % 70000)); break;
|
|
case 2: note_file_write(r, 5, rng() % 2, std::to_string(rng())); break;
|
|
default: note_surprisal(r, Surprisal{static_cast<TupleKind>(rng() % kTupleKinds), "t", static_cast<double>(rng() % 100), 0}); break;
|
|
}
|
|
}
|
|
PROPERTY(r.tokens.size() <= kMaxSummaryTokens);
|
|
for (const auto v : featurize(r, DecisionContext{static_cast<uint8_t>(rng() % 9), 7.0f})) {
|
|
PROPERTY(std::isfinite(v) && v >= -2.0f && v <= 2.0f);
|
|
}
|
|
}
|
|
std::cout << "F1 features: PASSED" << std::endl;
|
|
}
|
|
|
|
} // namespace
|
|
|
|
int main() {
|
|
std::cout << "model properties, seed " << kSeed << std::endl;
|
|
test_kernels();
|
|
test_forward_and_init();
|
|
test_files();
|
|
test_features();
|
|
return 0;
|
|
}
|