// Fleet promotion gate: node reports are pooled per (model, period) cell and the unchanged // pre-registered gate decides each pooled cell. // // G1 two nodes' cells are summed field by field (windows, positives, hits, CPU-s, inspections) // G2 pooling can pass a cell neither node passes alone (10 positives only together) // G3 the fleet passes only if EVERY cell passes; one failing cell fails it // G4 malformed / cell-less reports are skipped and reported, never counted // G5 a cell must name the model bytes it judged (model sha256), mixed identities are refused // G6 reports without a digest (an older replay) are refused, not pooled // G7 an in-sample evaluation fails the pooled cell; a disjoint one passes it // G8 a 0-hit rarity baseline fails the pooled cell (it used to pass free) // G9 pooling may only weaken provenance: one node that cannot prove its training range (or // reports no evaluated range) makes the pooled cell unknown, even if another node can // G11 the comparator is the calibrated rarity policy; a legacy or uncalibrated node report // fails the pooled cell and the first-come rarity_greedy is never judged // G10 a one-seed, one-period, holed or per-node-inconsistent report set yields NO verdict: // the pre-registered check is the full 3 seeds x 3 periods matrix #include "strawberry/fleet_gate.hpp" #include #include #include #include using namespace strawberry; namespace { #define CHECK(cond) \ do { \ if (!(cond)) { \ std::cerr << "CHECK FAILED: " #cond " (" << __FILE__ << ":" << __LINE__ << ")" << std::endl; \ std::abort(); \ } \ } while (0) // One node's cell for a (model, period), given as // {model, period, positives, model_hits, model_cpu, rarity_hits, rarity_cpu, training_known}. struct C { const char* model; int period; int positives; int mh; double mc; int rh; double rc; bool training_known = true; }; const char* const kSeeds[] = {"seed1", "seed2", "seed3"}; // The pre-registered 3 seeds x 3 periods matrix, every cell alike. std::vector matrix(int positives = 6, int mh = 5, double mc = 10.0, int rh = 3, double rc = 20.0, bool training_known = true) { std::vector v; for (const char* m : kSeeds) { for (int p = 0; p < 3; ++p) v.push_back(C{m, p, positives, mh, mc, rh, rc, training_known}); } return v; } // The comparator a node reports: calibrated (the gate's), uncalibrated (it had no history to // freeze a threshold from), or legacy (an older replay: only the first-come rarity_greedy). enum class Baseline { calibrated, uncalibrated, legacy }; std::string node_report(const std::vector& cells, const std::string& digest = "0123456789abcdef", long training_last = 2000, long eval_first = 3000, long eval_last = 4000, Baseline baseline = Baseline::calibrated) { std::string out = R"({"policy_eval":{"windows":100,"extra_positives":0,"rule_detected":0,"policies":[],"cells":[)"; bool first = true; for (const auto& c : cells) { if (!first) out += ","; first = false; out += "{\"model\":\"" + std::string(c.model) + "\",\"period\":" + std::to_string(c.period) + ",\"windows\":50,\"extra_positives\":" + std::to_string(c.positives) + ",\"rule_detected\":1," + "\"model_sha256\":\"" + digest + "\",\"training_range_known\":" + (c.training_known ? "true" : "false") + ",\"training_first_ns\":1000,\"training_last_ns\":" + std::to_string(training_last) + ",\"eval_first_ns\":" + std::to_string(eval_first) + ",\"eval_last_ns\":" + std::to_string(eval_last) + ",\"in_sample\":false," + "\"policies\":[{\"name\":\"model\",\"inspections\":" + std::to_string(c.mh + 2) + ",\"cpu_seconds\":" + std::to_string(c.mc) + ",\"hits\":" + std::to_string(c.mh) + ",\"weighted_hits\":" + std::to_string(c.mh) + ",\"budget_refusals\":0}," + // The legacy first-come rule: a diagnostic that must never be judged. Here it // looks unbeatable, so a gate that read it would fail every cell. "{\"name\":\"rarity_greedy\",\"inspections\":999,\"cpu_seconds\":1.0,\"hits\":999," + "\"weighted_hits\":999,\"budget_refusals\":3}"; if (baseline != Baseline::legacy) { out += ",{\"name\":\"rarity_calibrated\",\"inspections\":" + std::to_string(c.rh + 5) + ",\"cpu_seconds\":" + std::to_string(c.rc) + ",\"hits\":" + std::to_string(c.rh) + ",\"weighted_hits\":" + std::to_string(c.rh) + ",\"budget_refusals\":3,\"threshold\":0.9," + "\"calibration_windows\":" + (baseline == Baseline::calibrated ? "500" : "0") + "}," + "{\"name\":\"rarity_hindsight\",\"inspections\":7,\"cpu_seconds\":14.0,\"hits\":4," + "\"weighted_hits\":4,\"budget_refusals\":1}"; } out += "]}"; } return out + "]}}"; } bool has_error(const fleet::FleetGateReport& r, const std::string& needle) { for (const auto& e : r.errors) { if (e.find(needle) != std::string::npos) return true; } return false; } } // namespace int main() { // G1 + G2: 6 + 6 positives per cell: neither node alone has the 10 the gate needs. const auto a = node_report(matrix()); const auto b = node_report(matrix()); auto alone = fleet::fleet_gate({a}); CHECK(alone.reports == 1 && alone.cells.size() == 9 && alone.complete && !alone.passed); // 6 < 10 positives auto pooled = fleet::fleet_gate({a, b}); CHECK(pooled.reports == 2 && pooled.cells.size() == 9 && pooled.complete && pooled.expected_cells == 9); const auto& c0 = pooled.cells[0]; CHECK(c0.model == "seed1" && c0.period == 0 && c0.nodes == 2); CHECK(c0.pooled.windows == 100 && c0.pooled.extra_positives == 12 && c0.pooled.rule_detected == 2); CHECK(c0.pooled.policies[0].name == "model" && c0.pooled.policies[0].hits == 10 && c0.pooled.policies[0].cpu_seconds == 20.0); CHECK(c0.pooled.policies[0].inspections == 14); CHECK(c0.pooled.policies[1].name == "rarity_calibrated" && c0.pooled.policies[1].hits == 6 && c0.pooled.policies[1].cpu_seconds == 40.0 && c0.pooled.policies[1].budget_refusals == 6); CHECK(c0.pooled.policies[1].calibration_windows == 1000 && c0.baseline_uncalibrated_nodes == 0); // The diagnostics pool too, but are never judged (rarity_greedy here would beat any model). CHECK(c0.pooled.policies[2].name == "rarity_greedy" && c0.pooled.policies[2].hits == 1998); CHECK(c0.pooled.policies[3].name == "rarity_hindsight" && c0.pooled.policies[3].hits == 8); // model 10 hits / 20 s = 0.5/s vs rarity 6 / 40 s = 0.15/s (>= 1.5x), recall 10 >= 6: pass. CHECK(c0.gate.passed && pooled.passed); // G3: a third node makes period 1 fail (model recall drops below rarity-greedy). auto bad_cells = matrix(); for (auto& c : bad_cells) { if (c.period == 1) { c.positives = 4; c.mh = 0; c.mc = 30.0; c.rh = 9; c.rc = 5.0; } } auto mixed = fleet::fleet_gate({a, b, node_report(bad_cells)}); CHECK(mixed.cells.size() == 9 && mixed.complete); // sorted by (model, period): seed1 p0..p2 = 0..2, so the failing period-1 cells are 1, 4, 7. CHECK(mixed.cells[0].gate.passed && !mixed.cells[1].gate.passed && mixed.cells[2].gate.passed); CHECK(mixed.cells[3].gate.passed && !mixed.cells[4].gate.passed && mixed.cells[5].gate.passed); CHECK(mixed.cells[6].gate.passed && !mixed.cells[7].gate.passed && mixed.cells[8].gate.passed); CHECK(!mixed.passed); // G4: malformed and cell-less reports are skipped, not counted. auto noisy = fleet::fleet_gate({a, b, "not json", R"({"policy_eval":{"windows":3,"policies":[]}})", ""}); CHECK(noisy.reports == 2 && noisy.errors.size() == 3 && noisy.passed && noisy.complete); CHECK(!fleet::fleet_gate({}).passed); // no evidence is not a pass // A report wrapped like the CLI's {"ok":..,"data":{..}} is read too. auto wrapped = fleet::fleet_gate({"{\"ok\":true,\"data\":" + a + "}", b}); CHECK(wrapped.reports == 2 && wrapped.passed); CHECK(fleet::fleet_gate_json(pooled).find("\"passed\": true") != std::string::npos); // G5: a cell must name the model bytes it judged. A bare filename is not an // identity: a stale dist/gate model from an older cycle can carry the same // name as the one a later cycle trained. const auto stale = node_report(matrix(), "fedcba9876543210"); auto mixed_models = fleet::fleet_gate({a, stale}); CHECK(mixed_models.refused && !mixed_models.passed); CHECK(has_error(mixed_models, "stale or mixed report set")); // G6: reports without a digest (an older replay) are refused, not pooled. auto legacy = node_report(matrix(), ""); auto legacy_set = fleet::fleet_gate({a, legacy}); CHECK(legacy_set.refused && !legacy_set.passed); CHECK(has_error(legacy_set, "no model digest")); // G7: in-sample evaluation is refused by the pooled cell, with the reason. const auto in_sample = node_report(matrix(), "0123456789abcdef", /*training_last=*/5000, /*eval_first=*/3000, /*eval_last=*/4000); auto same = node_report(matrix(), "0123456789abcdef", /*training_last=*/5000, /*eval_first=*/3000, /*eval_last=*/4000); auto in_sample_set = fleet::fleet_gate({in_sample, same}); CHECK(!in_sample_set.refused && in_sample_set.cells.size() == 9 && in_sample_set.cells[0].nodes == 2); CHECK(in_sample_set.cells[0].pooled.provenance.in_sample && !in_sample_set.cells[0].gate.passed); CHECK(in_sample_set.cells[0].gate.checks.back().find("in-sample evaluation") != std::string::npos); CHECK(!in_sample_set.passed); // The same reports with a disjoint training range pass that check. const auto disjoint = node_report(matrix(), "0123456789abcdef", /*training_last=*/2000, /*eval_first=*/3000, /*eval_last=*/4000); auto disjoint_set = fleet::fleet_gate({disjoint, disjoint}); CHECK(!disjoint_set.refused && disjoint_set.cells.size() == 9 && disjoint_set.passed && disjoint_set.complete); // G8: a 0-hit rarity baseline fails the pooled cell (it used to pass free). const auto no_hits = node_report(matrix(6, 5, 10.0, 0, 0.0)); auto degenerate = fleet::fleet_gate({no_hits, no_hits}); CHECK(degenerate.cells.size() == 9 && !degenerate.cells[0].gate.passed && !degenerate.passed); CHECK(degenerate.cells[0].gate.checks[1].find("degenerate baseline") != std::string::npos); // G9a: pooling may not UPGRADE provenance. One node knows the training range, the // other does not: every other criterion still passes (12 positives, 1.5x efficiency), // so the only thing standing between this set and a PASS is the unknown provenance - // which must win. const auto known = node_report(matrix()); const auto unknown = node_report(matrix(6, 5, 10.0, 3, 20.0, /*training_known=*/false)); auto upgraded = fleet::fleet_gate({known, unknown}); CHECK(upgraded.complete && upgraded.cells.size() == 9); CHECK(upgraded.cells[0].nodes == 2 && upgraded.cells[0].pooled.extra_positives == 12); CHECK(upgraded.cells[0].provenance_incomplete && upgraded.cells[0].provenance_unknown_nodes == 1); CHECK(!upgraded.cells[0].pooled.provenance.training_range_known); CHECK(upgraded.cells[0].pooled.provenance.training_first_ns == 0 && upgraded.cells[0].pooled.provenance.training_last_ns == 0); CHECK(!upgraded.cells[0].gate.passed); CHECK(upgraded.cells[0].gate.checks.back().find("provenance incomplete") != std::string::npos); CHECK(!upgraded.passed); CHECK(has_error(upgraded, "no usable training range")); // G9b: a "known" flag with an unusable range (inverted / missing end) is unknown too - // the old code dropped it and kept the other node's range. const auto bogus = node_report(matrix(), "0123456789abcdef", /*training_last=*/0); auto bogus_set = fleet::fleet_gate({known, bogus}); CHECK(bogus_set.cells[0].provenance_incomplete && bogus_set.cells[0].provenance_unknown_nodes == 1); CHECK(!bogus_set.passed); // G9c: a node that reports no evaluated window range leaves the cell unable to show // that its evaluation was disjoint from training - unknown, so it fails. const auto no_eval_range = node_report(matrix(), "0123456789abcdef", 2000, /*eval_first=*/0, /*eval_last=*/0); auto no_eval_set = fleet::fleet_gate({known, no_eval_range}); CHECK(no_eval_set.cells[0].provenance_incomplete && !no_eval_set.cells[0].gate.passed); CHECK(!no_eval_set.cells[0].pooled.provenance.training_range_known); CHECK(has_error(no_eval_set, "no recorded window range")); CHECK(!no_eval_set.passed); // G10a: a one-seed report set (3 periods) has a passing cell - 12 pooled positives - // but it is not the pre-registered matrix, so it must not produce a verdict. std::vector one_seed; for (int p = 0; p < 3; ++p) one_seed.push_back(C{"seed1", p, 6, 5, 10.0, 3, 20.0, true}); auto single_seed = fleet::fleet_gate({node_report(one_seed), node_report(one_seed)}); CHECK(single_seed.cells.size() == 3 && single_seed.cells[0].gate.passed); CHECK(!single_seed.complete && !single_seed.passed); CHECK(single_seed.expected_cells == 9); CHECK(has_error(single_seed, "completeness:")); CHECK(has_error(single_seed, "need 3 training seeds")); CHECK(single_seed.reports == 2); // G10b: a one-period report set (3 seeds) is equally incomplete. std::vector one_period; for (const char* m : kSeeds) one_period.push_back(C{m, 0, 6, 5, 10.0, 3, 20.0, true}); auto single_period = fleet::fleet_gate({node_report(one_period), node_report(one_period)}); CHECK(single_period.cells.size() == 3 && single_period.cells[0].gate.passed); CHECK(!single_period.complete && !single_period.passed); CHECK(has_error(single_period, "need 3 replay periods")); // G10c: a hole in the 3x3 matrix is incomplete, not a pass over the surviving cells. auto holed = matrix(); holed.pop_back(); // seed3 period 2 auto holed_set = fleet::fleet_gate({node_report(holed), node_report(holed)}); CHECK(holed_set.cells.size() == 8 && !holed_set.complete && !holed_set.passed); CHECK(has_error(holed_set, "cross product")); CHECK(has_error(holed_set, "seed3 period 2")); // One node with a hole the other fills still leaves the matrix whole, but the nodes did // not run the same evaluation, so the set is incomplete too. auto raggedy = fleet::fleet_gate({node_report(holed), node_report(matrix())}); CHECK(raggedy.cells.size() == 9 && !raggedy.complete && !raggedy.passed); CHECK(has_error(raggedy, "(seed, period) cells")); // G10d: nodes that disagree about --periods (2 vs 3) are not comparable - period 1 of // a 2-way split is not period 1 of a 3-way split - so the set is incomplete even though // the pooled union happens to be full. std::vector two_periods; for (const char* m : kSeeds) { for (int p = 0; p < 2; ++p) two_periods.push_back(C{m, p, 6, 5, 10.0, 3, 20.0, true}); } auto inconsistent = fleet::fleet_gate({node_report(matrix()), node_report(two_periods)}); CHECK(inconsistent.cells.size() == 9 && !inconsistent.complete && !inconsistent.passed); CHECK(has_error(inconsistent, "inconsistent --eval-model / --periods")); // G10e: periods that do not start at 0 are not the replay's period indices. std::vector shifted; for (const char* m : kSeeds) { for (int p = 1; p < 4; ++p) shifted.push_back(C{m, p, 6, 5, 10.0, 3, 20.0, true}); } auto shifted_set = fleet::fleet_gate({node_report(shifted), node_report(shifted)}); CHECK(!shifted_set.complete && !shifted_set.passed); CHECK(has_error(shifted_set, "period indices are not 0..")); // G11: the comparator is rarity_calibrated, per node. A legacy report (only the first-come // rarity_greedy) is never judged against it, and one uncalibrated node makes the pooled // cell uncalibrated even when the other node is calibrated: pooling may only weaken. const auto first_come = node_report(matrix(), "0123456789abcdef", 2000, 3000, 4000, Baseline::legacy); auto first_come_set = fleet::fleet_gate({first_come, first_come}); CHECK(first_come_set.complete && !first_come_set.passed); CHECK(has_error(first_come_set, "no calibrated rarity comparator")); for (const auto& c : first_come_set.cells) { CHECK(!c.gate.passed && c.baseline_uncalibrated_nodes == 2); CHECK(c.gate.checks[1].rfind("FAIL: baseline uncalibrated", 0) == 0); } const auto cold = node_report(matrix(), "0123456789abcdef", 2000, 3000, 4000, Baseline::uncalibrated); auto half = fleet::fleet_gate({node_report(matrix()), cold}); CHECK(half.complete && !half.passed); for (const auto& c : half.cells) { CHECK(!c.gate.passed && c.baseline_uncalibrated_nodes == 1 && c.pooled.policies[1].calibration_windows == 0); } const auto gate_json = fleet::fleet_gate_json(pooled); CHECK(gate_json.find("\"baseline\": \"rarity_calibrated\"") != std::string::npos); CHECK(gate_json.find("\"rarity_hindsight_hits\"") != std::string::npos); std::cout << "fleet gate: PASSED" << std::endl; return 0; }