The rarity-greedy comparator acted on every evaluated window, so it spent each hour's budget on the first ~15 windows and scored 0 hits in every cell. Criteria 2 and 3 of the Stage 4 gate were void against it. rarity_calibrated keeps the same rarity value, chooser, costs and budget, and adds a per-node threshold frozen before the evaluated span: the m-th largest value over the trailing 24 h of label-free history between the warm-up and the --eval-after bound, with m set so expected spend matches the hourly budget. Without that history the comparator is uncalibrated and the gate fails by name; it never falls back to another rule. rarity_greedy (the old first-come rule) and rarity_hindsight (per-hour ordering, not causal) are reported as diagnostics and never judged. fleet gate pools counts, keeps thresholds per node, and fails a pooled cell when any node is uncalibrated or lacks the comparator. Contract change: in `fleet gate --json`, rarity_hits and rarity_cpu_seconds now describe the calibrated comparator; the old numbers are rarity_greedy_hits and rarity_greedy_cpu_seconds. Pre-registration: docs/evaluation/stage4-rarity-baseline-preregistration.md, sha256 4916c95cd2581e3749acfe46c3ccf827aa5ce98c06150b49b978fc70da013f95, indexed in YAMS at 2026-11-20T18:55:48Z, before the code. This commit is dated after the registration time in that document (2026-11-21T00:00Z); the YAMS record is what predates it. Checked: meson suite 58/58, scripts/fleet_e2e.sh, clang-tidy and the strict analyzer on the changed TUs. Not run: fold on the changed TUs, make fuzz, the live Docker tests and the perf KPI. The comparator has not been run on validation-range data. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
306 lines
18 KiB
C++
306 lines
18 KiB
C++
// Fleet promotion gate: node reports are pooled per (model, period) cell and the unchanged
|
|
// pre-registered gate decides each pooled cell.
|
|
//
|
|
// G1 two nodes' cells are summed field by field (windows, positives, hits, CPU-s, inspections)
|
|
// G2 pooling can pass a cell neither node passes alone (10 positives only together)
|
|
// G3 the fleet passes only if EVERY cell passes; one failing cell fails it
|
|
// G4 malformed / cell-less reports are skipped and reported, never counted
|
|
// G5 a cell must name the model bytes it judged (model sha256), mixed identities are refused
|
|
// G6 reports without a digest (an older replay) are refused, not pooled
|
|
// G7 an in-sample evaluation fails the pooled cell; a disjoint one passes it
|
|
// G8 a 0-hit rarity baseline fails the pooled cell (it used to pass free)
|
|
// G9 pooling may only weaken provenance: one node that cannot prove its training range (or
|
|
// reports no evaluated range) makes the pooled cell unknown, even if another node can
|
|
// G11 the comparator is the calibrated rarity policy; a legacy or uncalibrated node report
|
|
// fails the pooled cell and the first-come rarity_greedy is never judged
|
|
// G10 a one-seed, one-period, holed or per-node-inconsistent report set yields NO verdict:
|
|
// the pre-registered check is the full 3 seeds x 3 periods matrix
|
|
#include "strawberry/fleet_gate.hpp"
|
|
|
|
#include <cstdlib>
|
|
#include <iostream>
|
|
#include <string>
|
|
#include <vector>
|
|
|
|
using namespace strawberry;
|
|
|
|
namespace {
|
|
|
|
#define CHECK(cond) \
|
|
do { \
|
|
if (!(cond)) { \
|
|
std::cerr << "CHECK FAILED: " #cond " (" << __FILE__ << ":" << __LINE__ << ")" << std::endl; \
|
|
std::abort(); \
|
|
} \
|
|
} while (0)
|
|
|
|
// One node's cell for a (model, period), given as
|
|
// {model, period, positives, model_hits, model_cpu, rarity_hits, rarity_cpu, training_known}.
|
|
struct C {
|
|
const char* model;
|
|
int period;
|
|
int positives;
|
|
int mh;
|
|
double mc;
|
|
int rh;
|
|
double rc;
|
|
bool training_known = true;
|
|
};
|
|
|
|
const char* const kSeeds[] = {"seed1", "seed2", "seed3"};
|
|
|
|
// The pre-registered 3 seeds x 3 periods matrix, every cell alike.
|
|
std::vector<C> matrix(int positives = 6, int mh = 5, double mc = 10.0, int rh = 3, double rc = 20.0,
|
|
bool training_known = true) {
|
|
std::vector<C> v;
|
|
for (const char* m : kSeeds) {
|
|
for (int p = 0; p < 3; ++p) v.push_back(C{m, p, positives, mh, mc, rh, rc, training_known});
|
|
}
|
|
return v;
|
|
}
|
|
|
|
// The comparator a node reports: calibrated (the gate's), uncalibrated (it had no history to
|
|
// freeze a threshold from), or legacy (an older replay: only the first-come rarity_greedy).
|
|
enum class Baseline { calibrated, uncalibrated, legacy };
|
|
|
|
std::string node_report(const std::vector<C>& cells, const std::string& digest = "0123456789abcdef",
|
|
long training_last = 2000, long eval_first = 3000, long eval_last = 4000,
|
|
Baseline baseline = Baseline::calibrated) {
|
|
std::string out = R"({"policy_eval":{"windows":100,"extra_positives":0,"rule_detected":0,"policies":[],"cells":[)";
|
|
bool first = true;
|
|
for (const auto& c : cells) {
|
|
if (!first) out += ",";
|
|
first = false;
|
|
out += "{\"model\":\"" + std::string(c.model) + "\",\"period\":" + std::to_string(c.period) +
|
|
",\"windows\":50,\"extra_positives\":" + std::to_string(c.positives) + ",\"rule_detected\":1," +
|
|
"\"model_sha256\":\"" + digest + "\",\"training_range_known\":" +
|
|
(c.training_known ? "true" : "false") +
|
|
",\"training_first_ns\":1000,\"training_last_ns\":" + std::to_string(training_last) +
|
|
",\"eval_first_ns\":" + std::to_string(eval_first) +
|
|
",\"eval_last_ns\":" + std::to_string(eval_last) + ",\"in_sample\":false," +
|
|
"\"policies\":[{\"name\":\"model\",\"inspections\":" + std::to_string(c.mh + 2) +
|
|
",\"cpu_seconds\":" + std::to_string(c.mc) + ",\"hits\":" + std::to_string(c.mh) +
|
|
",\"weighted_hits\":" + std::to_string(c.mh) + ",\"budget_refusals\":0}," +
|
|
// The legacy first-come rule: a diagnostic that must never be judged. Here it
|
|
// looks unbeatable, so a gate that read it would fail every cell.
|
|
"{\"name\":\"rarity_greedy\",\"inspections\":999,\"cpu_seconds\":1.0,\"hits\":999," +
|
|
"\"weighted_hits\":999,\"budget_refusals\":3}";
|
|
if (baseline != Baseline::legacy) {
|
|
out += ",{\"name\":\"rarity_calibrated\",\"inspections\":" + std::to_string(c.rh + 5) +
|
|
",\"cpu_seconds\":" + std::to_string(c.rc) + ",\"hits\":" + std::to_string(c.rh) +
|
|
",\"weighted_hits\":" + std::to_string(c.rh) + ",\"budget_refusals\":3,\"threshold\":0.9," +
|
|
"\"calibration_windows\":" + (baseline == Baseline::calibrated ? "500" : "0") + "}," +
|
|
"{\"name\":\"rarity_hindsight\",\"inspections\":7,\"cpu_seconds\":14.0,\"hits\":4," +
|
|
"\"weighted_hits\":4,\"budget_refusals\":1}";
|
|
}
|
|
out += "]}";
|
|
}
|
|
return out + "]}}";
|
|
}
|
|
|
|
bool has_error(const fleet::FleetGateReport& r, const std::string& needle) {
|
|
for (const auto& e : r.errors) {
|
|
if (e.find(needle) != std::string::npos) return true;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
} // namespace
|
|
|
|
int main() {
|
|
// G1 + G2: 6 + 6 positives per cell: neither node alone has the 10 the gate needs.
|
|
const auto a = node_report(matrix());
|
|
const auto b = node_report(matrix());
|
|
auto alone = fleet::fleet_gate({a});
|
|
CHECK(alone.reports == 1 && alone.cells.size() == 9 && alone.complete && !alone.passed); // 6 < 10 positives
|
|
auto pooled = fleet::fleet_gate({a, b});
|
|
CHECK(pooled.reports == 2 && pooled.cells.size() == 9 && pooled.complete && pooled.expected_cells == 9);
|
|
const auto& c0 = pooled.cells[0];
|
|
CHECK(c0.model == "seed1" && c0.period == 0 && c0.nodes == 2);
|
|
CHECK(c0.pooled.windows == 100 && c0.pooled.extra_positives == 12 && c0.pooled.rule_detected == 2);
|
|
CHECK(c0.pooled.policies[0].name == "model" && c0.pooled.policies[0].hits == 10 && c0.pooled.policies[0].cpu_seconds == 20.0);
|
|
CHECK(c0.pooled.policies[0].inspections == 14);
|
|
CHECK(c0.pooled.policies[1].name == "rarity_calibrated" && c0.pooled.policies[1].hits == 6 &&
|
|
c0.pooled.policies[1].cpu_seconds == 40.0 && c0.pooled.policies[1].budget_refusals == 6);
|
|
CHECK(c0.pooled.policies[1].calibration_windows == 1000 && c0.baseline_uncalibrated_nodes == 0);
|
|
// The diagnostics pool too, but are never judged (rarity_greedy here would beat any model).
|
|
CHECK(c0.pooled.policies[2].name == "rarity_greedy" && c0.pooled.policies[2].hits == 1998);
|
|
CHECK(c0.pooled.policies[3].name == "rarity_hindsight" && c0.pooled.policies[3].hits == 8);
|
|
// model 10 hits / 20 s = 0.5/s vs rarity 6 / 40 s = 0.15/s (>= 1.5x), recall 10 >= 6: pass.
|
|
CHECK(c0.gate.passed && pooled.passed);
|
|
|
|
// G3: a third node makes period 1 fail (model recall drops below rarity-greedy).
|
|
auto bad_cells = matrix();
|
|
for (auto& c : bad_cells) {
|
|
if (c.period == 1) {
|
|
c.positives = 4;
|
|
c.mh = 0;
|
|
c.mc = 30.0;
|
|
c.rh = 9;
|
|
c.rc = 5.0;
|
|
}
|
|
}
|
|
auto mixed = fleet::fleet_gate({a, b, node_report(bad_cells)});
|
|
CHECK(mixed.cells.size() == 9 && mixed.complete);
|
|
// sorted by (model, period): seed1 p0..p2 = 0..2, so the failing period-1 cells are 1, 4, 7.
|
|
CHECK(mixed.cells[0].gate.passed && !mixed.cells[1].gate.passed && mixed.cells[2].gate.passed);
|
|
CHECK(mixed.cells[3].gate.passed && !mixed.cells[4].gate.passed && mixed.cells[5].gate.passed);
|
|
CHECK(mixed.cells[6].gate.passed && !mixed.cells[7].gate.passed && mixed.cells[8].gate.passed);
|
|
CHECK(!mixed.passed);
|
|
|
|
// G4: malformed and cell-less reports are skipped, not counted.
|
|
auto noisy = fleet::fleet_gate({a, b, "not json", R"({"policy_eval":{"windows":3,"policies":[]}})", ""});
|
|
CHECK(noisy.reports == 2 && noisy.errors.size() == 3 && noisy.passed && noisy.complete);
|
|
CHECK(!fleet::fleet_gate({}).passed); // no evidence is not a pass
|
|
// A report wrapped like the CLI's {"ok":..,"data":{..}} is read too.
|
|
auto wrapped = fleet::fleet_gate({"{\"ok\":true,\"data\":" + a + "}", b});
|
|
CHECK(wrapped.reports == 2 && wrapped.passed);
|
|
CHECK(fleet::fleet_gate_json(pooled).find("\"passed\": true") != std::string::npos);
|
|
|
|
// G5: a cell must name the model bytes it judged. A bare filename is not an
|
|
// identity: a stale dist/gate model from an older cycle can carry the same
|
|
// name as the one a later cycle trained.
|
|
const auto stale = node_report(matrix(), "fedcba9876543210");
|
|
auto mixed_models = fleet::fleet_gate({a, stale});
|
|
CHECK(mixed_models.refused && !mixed_models.passed);
|
|
CHECK(has_error(mixed_models, "stale or mixed report set"));
|
|
|
|
// G6: reports without a digest (an older replay) are refused, not pooled.
|
|
auto legacy = node_report(matrix(), "");
|
|
auto legacy_set = fleet::fleet_gate({a, legacy});
|
|
CHECK(legacy_set.refused && !legacy_set.passed);
|
|
CHECK(has_error(legacy_set, "no model digest"));
|
|
|
|
// G7: in-sample evaluation is refused by the pooled cell, with the reason.
|
|
const auto in_sample = node_report(matrix(), "0123456789abcdef",
|
|
/*training_last=*/5000, /*eval_first=*/3000, /*eval_last=*/4000);
|
|
auto same = node_report(matrix(), "0123456789abcdef",
|
|
/*training_last=*/5000, /*eval_first=*/3000, /*eval_last=*/4000);
|
|
auto in_sample_set = fleet::fleet_gate({in_sample, same});
|
|
CHECK(!in_sample_set.refused && in_sample_set.cells.size() == 9 && in_sample_set.cells[0].nodes == 2);
|
|
CHECK(in_sample_set.cells[0].pooled.provenance.in_sample && !in_sample_set.cells[0].gate.passed);
|
|
CHECK(in_sample_set.cells[0].gate.checks.back().find("in-sample evaluation") != std::string::npos);
|
|
CHECK(!in_sample_set.passed);
|
|
// The same reports with a disjoint training range pass that check.
|
|
const auto disjoint = node_report(matrix(), "0123456789abcdef",
|
|
/*training_last=*/2000, /*eval_first=*/3000, /*eval_last=*/4000);
|
|
auto disjoint_set = fleet::fleet_gate({disjoint, disjoint});
|
|
CHECK(!disjoint_set.refused && disjoint_set.cells.size() == 9 && disjoint_set.passed && disjoint_set.complete);
|
|
|
|
// G8: a 0-hit rarity baseline fails the pooled cell (it used to pass free).
|
|
const auto no_hits = node_report(matrix(6, 5, 10.0, 0, 0.0));
|
|
auto degenerate = fleet::fleet_gate({no_hits, no_hits});
|
|
CHECK(degenerate.cells.size() == 9 && !degenerate.cells[0].gate.passed && !degenerate.passed);
|
|
CHECK(degenerate.cells[0].gate.checks[1].find("degenerate baseline") != std::string::npos);
|
|
|
|
// G9a: pooling may not UPGRADE provenance. One node knows the training range, the
|
|
// other does not: every other criterion still passes (12 positives, 1.5x efficiency),
|
|
// so the only thing standing between this set and a PASS is the unknown provenance -
|
|
// which must win.
|
|
const auto known = node_report(matrix());
|
|
const auto unknown = node_report(matrix(6, 5, 10.0, 3, 20.0, /*training_known=*/false));
|
|
auto upgraded = fleet::fleet_gate({known, unknown});
|
|
CHECK(upgraded.complete && upgraded.cells.size() == 9);
|
|
CHECK(upgraded.cells[0].nodes == 2 && upgraded.cells[0].pooled.extra_positives == 12);
|
|
CHECK(upgraded.cells[0].provenance_incomplete && upgraded.cells[0].provenance_unknown_nodes == 1);
|
|
CHECK(!upgraded.cells[0].pooled.provenance.training_range_known);
|
|
CHECK(upgraded.cells[0].pooled.provenance.training_first_ns == 0 &&
|
|
upgraded.cells[0].pooled.provenance.training_last_ns == 0);
|
|
CHECK(!upgraded.cells[0].gate.passed);
|
|
CHECK(upgraded.cells[0].gate.checks.back().find("provenance incomplete") != std::string::npos);
|
|
CHECK(!upgraded.passed);
|
|
CHECK(has_error(upgraded, "no usable training range"));
|
|
|
|
// G9b: a "known" flag with an unusable range (inverted / missing end) is unknown too -
|
|
// the old code dropped it and kept the other node's range.
|
|
const auto bogus = node_report(matrix(), "0123456789abcdef", /*training_last=*/0);
|
|
auto bogus_set = fleet::fleet_gate({known, bogus});
|
|
CHECK(bogus_set.cells[0].provenance_incomplete && bogus_set.cells[0].provenance_unknown_nodes == 1);
|
|
CHECK(!bogus_set.passed);
|
|
|
|
// G9c: a node that reports no evaluated window range leaves the cell unable to show
|
|
// that its evaluation was disjoint from training - unknown, so it fails.
|
|
const auto no_eval_range = node_report(matrix(), "0123456789abcdef", 2000, /*eval_first=*/0, /*eval_last=*/0);
|
|
auto no_eval_set = fleet::fleet_gate({known, no_eval_range});
|
|
CHECK(no_eval_set.cells[0].provenance_incomplete && !no_eval_set.cells[0].gate.passed);
|
|
CHECK(!no_eval_set.cells[0].pooled.provenance.training_range_known);
|
|
CHECK(has_error(no_eval_set, "no recorded window range"));
|
|
CHECK(!no_eval_set.passed);
|
|
|
|
// G10a: a one-seed report set (3 periods) has a passing cell - 12 pooled positives -
|
|
// but it is not the pre-registered matrix, so it must not produce a verdict.
|
|
std::vector<C> one_seed;
|
|
for (int p = 0; p < 3; ++p) one_seed.push_back(C{"seed1", p, 6, 5, 10.0, 3, 20.0, true});
|
|
auto single_seed = fleet::fleet_gate({node_report(one_seed), node_report(one_seed)});
|
|
CHECK(single_seed.cells.size() == 3 && single_seed.cells[0].gate.passed);
|
|
CHECK(!single_seed.complete && !single_seed.passed);
|
|
CHECK(single_seed.expected_cells == 9);
|
|
CHECK(has_error(single_seed, "completeness:"));
|
|
CHECK(has_error(single_seed, "need 3 training seeds"));
|
|
CHECK(single_seed.reports == 2);
|
|
|
|
// G10b: a one-period report set (3 seeds) is equally incomplete.
|
|
std::vector<C> one_period;
|
|
for (const char* m : kSeeds) one_period.push_back(C{m, 0, 6, 5, 10.0, 3, 20.0, true});
|
|
auto single_period = fleet::fleet_gate({node_report(one_period), node_report(one_period)});
|
|
CHECK(single_period.cells.size() == 3 && single_period.cells[0].gate.passed);
|
|
CHECK(!single_period.complete && !single_period.passed);
|
|
CHECK(has_error(single_period, "need 3 replay periods"));
|
|
|
|
// G10c: a hole in the 3x3 matrix is incomplete, not a pass over the surviving cells.
|
|
auto holed = matrix();
|
|
holed.pop_back(); // seed3 period 2
|
|
auto holed_set = fleet::fleet_gate({node_report(holed), node_report(holed)});
|
|
CHECK(holed_set.cells.size() == 8 && !holed_set.complete && !holed_set.passed);
|
|
CHECK(has_error(holed_set, "cross product"));
|
|
CHECK(has_error(holed_set, "seed3 period 2"));
|
|
// One node with a hole the other fills still leaves the matrix whole, but the nodes did
|
|
// not run the same evaluation, so the set is incomplete too.
|
|
auto raggedy = fleet::fleet_gate({node_report(holed), node_report(matrix())});
|
|
CHECK(raggedy.cells.size() == 9 && !raggedy.complete && !raggedy.passed);
|
|
CHECK(has_error(raggedy, "(seed, period) cells"));
|
|
|
|
// G10d: nodes that disagree about --periods (2 vs 3) are not comparable - period 1 of
|
|
// a 2-way split is not period 1 of a 3-way split - so the set is incomplete even though
|
|
// the pooled union happens to be full.
|
|
std::vector<C> two_periods;
|
|
for (const char* m : kSeeds) {
|
|
for (int p = 0; p < 2; ++p) two_periods.push_back(C{m, p, 6, 5, 10.0, 3, 20.0, true});
|
|
}
|
|
auto inconsistent = fleet::fleet_gate({node_report(matrix()), node_report(two_periods)});
|
|
CHECK(inconsistent.cells.size() == 9 && !inconsistent.complete && !inconsistent.passed);
|
|
CHECK(has_error(inconsistent, "inconsistent --eval-model / --periods"));
|
|
|
|
// G10e: periods that do not start at 0 are not the replay's period indices.
|
|
std::vector<C> shifted;
|
|
for (const char* m : kSeeds) {
|
|
for (int p = 1; p < 4; ++p) shifted.push_back(C{m, p, 6, 5, 10.0, 3, 20.0, true});
|
|
}
|
|
auto shifted_set = fleet::fleet_gate({node_report(shifted), node_report(shifted)});
|
|
CHECK(!shifted_set.complete && !shifted_set.passed);
|
|
CHECK(has_error(shifted_set, "period indices are not 0.."));
|
|
|
|
// G11: the comparator is rarity_calibrated, per node. A legacy report (only the first-come
|
|
// rarity_greedy) is never judged against it, and one uncalibrated node makes the pooled
|
|
// cell uncalibrated even when the other node is calibrated: pooling may only weaken.
|
|
const auto first_come = node_report(matrix(), "0123456789abcdef", 2000, 3000, 4000, Baseline::legacy);
|
|
auto first_come_set = fleet::fleet_gate({first_come, first_come});
|
|
CHECK(first_come_set.complete && !first_come_set.passed);
|
|
CHECK(has_error(first_come_set, "no calibrated rarity comparator"));
|
|
for (const auto& c : first_come_set.cells) {
|
|
CHECK(!c.gate.passed && c.baseline_uncalibrated_nodes == 2);
|
|
CHECK(c.gate.checks[1].rfind("FAIL: baseline uncalibrated", 0) == 0);
|
|
}
|
|
const auto cold = node_report(matrix(), "0123456789abcdef", 2000, 3000, 4000, Baseline::uncalibrated);
|
|
auto half = fleet::fleet_gate({node_report(matrix()), cold});
|
|
CHECK(half.complete && !half.passed);
|
|
for (const auto& c : half.cells) {
|
|
CHECK(!c.gate.passed && c.baseline_uncalibrated_nodes == 1 && c.pooled.policies[1].calibration_windows == 0);
|
|
}
|
|
const auto gate_json = fleet::fleet_gate_json(pooled);
|
|
CHECK(gate_json.find("\"baseline\": \"rarity_calibrated\"") != std::string::npos);
|
|
CHECK(gate_json.find("\"rarity_hindsight_hits\"") != std::string::npos);
|
|
|
|
std::cout << "fleet gate: PASSED" << std::endl;
|
|
return 0;
|
|
}
|