strawberry/tests/test_fleet_gate.cpp
trvon 6618328059
fleet gate: refuse incomplete evidence and unprovable pooled provenance
Two ways a verdict could be built on evidence the tool did not actually have.

Pooling only ever upgraded provenance: one node that knew its training range set
the pooled flag, so a cell whose other nodes could not prove a range was still
reported as evaluated, and a cell could pass with no evaluated range at all. A
node with an unknown or unusable range now marks the pooled cell incomplete,
zeroes the displayed range (no range is shown that only some nodes could prove),
and appends "provenance incomplete: N of M contributing node report(s)". A known
range can no longer upgrade an unknown cell.

The documented "passes only if all 9 cells pass" was not enforced: passed was
just "some cells and not refused", so a one-seed, one-period report set exited 0.
A completeness gate now requires 3 seeds, 3 periods contiguous from 0, the full
seed x period cross product, and per-report consistency, and names every reason
("need 3 training seeds...", "missing: seed3 period 2", "inconsistent
--eval-model / --periods across node reports") before any verdict is computed.
2026-11-19 17:28:46 -07:00

264 lines
15 KiB
C++

// Fleet promotion gate: node reports are pooled per (model, period) cell and the unchanged
// pre-registered gate decides each pooled cell.
//
// G1 two nodes' cells are summed field by field (windows, positives, hits, CPU-s, inspections)
// G2 pooling can pass a cell neither node passes alone (10 positives only together)
// G3 the fleet passes only if EVERY cell passes; one failing cell fails it
// G4 malformed / cell-less reports are skipped and reported, never counted
// G5 a cell must name the model bytes it judged (model sha256), mixed identities are refused
// G6 reports without a digest (an older replay) are refused, not pooled
// G7 an in-sample evaluation fails the pooled cell; a disjoint one passes it
// G8 a 0-hit rarity baseline fails the pooled cell (it used to pass free)
// G9 pooling may only weaken provenance: one node that cannot prove its training range (or
// reports no evaluated range) makes the pooled cell unknown, even if another node can
// G10 a one-seed, one-period, holed or per-node-inconsistent report set yields NO verdict:
// the pre-registered check is the full 3 seeds x 3 periods matrix
#include "strawberry/fleet_gate.hpp"
#include <cstdlib>
#include <iostream>
#include <string>
#include <vector>
using namespace strawberry;
namespace {
#define CHECK(cond) \
do { \
if (!(cond)) { \
std::cerr << "CHECK FAILED: " #cond " (" << __FILE__ << ":" << __LINE__ << ")" << std::endl; \
std::abort(); \
} \
} while (0)
// One node's cell for a (model, period), given as
// {model, period, positives, model_hits, model_cpu, rarity_hits, rarity_cpu, training_known}.
struct C {
const char* model;
int period;
int positives;
int mh;
double mc;
int rh;
double rc;
bool training_known = true;
};
const char* const kSeeds[] = {"seed1", "seed2", "seed3"};
// The pre-registered 3 seeds x 3 periods matrix, every cell alike.
std::vector<C> matrix(int positives = 6, int mh = 5, double mc = 10.0, int rh = 3, double rc = 20.0,
bool training_known = true) {
std::vector<C> v;
for (const char* m : kSeeds) {
for (int p = 0; p < 3; ++p) v.push_back(C{m, p, positives, mh, mc, rh, rc, training_known});
}
return v;
}
std::string node_report(const std::vector<C>& cells, const std::string& digest = "0123456789abcdef",
long training_last = 2000, long eval_first = 3000, long eval_last = 4000) {
std::string out = R"({"policy_eval":{"windows":100,"extra_positives":0,"rule_detected":0,"policies":[],"cells":[)";
bool first = true;
for (const auto& c : cells) {
if (!first) out += ",";
first = false;
out += "{\"model\":\"" + std::string(c.model) + "\",\"period\":" + std::to_string(c.period) +
",\"windows\":50,\"extra_positives\":" + std::to_string(c.positives) + ",\"rule_detected\":1," +
"\"model_sha256\":\"" + digest + "\",\"training_range_known\":" +
(c.training_known ? "true" : "false") +
",\"training_first_ns\":1000,\"training_last_ns\":" + std::to_string(training_last) +
",\"eval_first_ns\":" + std::to_string(eval_first) +
",\"eval_last_ns\":" + std::to_string(eval_last) + ",\"in_sample\":false," +
"\"policies\":[{\"name\":\"model\",\"inspections\":" + std::to_string(c.mh + 2) +
",\"cpu_seconds\":" + std::to_string(c.mc) + ",\"hits\":" + std::to_string(c.mh) +
",\"weighted_hits\":" + std::to_string(c.mh) + ",\"budget_refusals\":0}," +
"{\"name\":\"rarity_greedy\",\"inspections\":" + std::to_string(c.rh + 5) +
",\"cpu_seconds\":" + std::to_string(c.rc) + ",\"hits\":" + std::to_string(c.rh) +
",\"weighted_hits\":" + std::to_string(c.rh) + ",\"budget_refusals\":3}]}";
}
return out + "]}}";
}
bool has_error(const fleet::FleetGateReport& r, const std::string& needle) {
for (const auto& e : r.errors) {
if (e.find(needle) != std::string::npos) return true;
}
return false;
}
} // namespace
int main() {
// G1 + G2: 6 + 6 positives per cell: neither node alone has the 10 the gate needs.
const auto a = node_report(matrix());
const auto b = node_report(matrix());
auto alone = fleet::fleet_gate({a});
CHECK(alone.reports == 1 && alone.cells.size() == 9 && alone.complete && !alone.passed); // 6 < 10 positives
auto pooled = fleet::fleet_gate({a, b});
CHECK(pooled.reports == 2 && pooled.cells.size() == 9 && pooled.complete && pooled.expected_cells == 9);
const auto& c0 = pooled.cells[0];
CHECK(c0.model == "seed1" && c0.period == 0 && c0.nodes == 2);
CHECK(c0.pooled.windows == 100 && c0.pooled.extra_positives == 12 && c0.pooled.rule_detected == 2);
CHECK(c0.pooled.policies[0].name == "model" && c0.pooled.policies[0].hits == 10 && c0.pooled.policies[0].cpu_seconds == 20.0);
CHECK(c0.pooled.policies[0].inspections == 14);
CHECK(c0.pooled.policies[1].name == "rarity_greedy" && c0.pooled.policies[1].hits == 6 &&
c0.pooled.policies[1].cpu_seconds == 40.0 && c0.pooled.policies[1].budget_refusals == 6);
// model 10 hits / 20 s = 0.5/s vs rarity 6 / 40 s = 0.15/s (>= 1.5x), recall 10 >= 6: pass.
CHECK(c0.gate.passed && pooled.passed);
// G3: a third node makes period 1 fail (model recall drops below rarity-greedy).
auto bad_cells = matrix();
for (auto& c : bad_cells) {
if (c.period == 1) {
c.positives = 4;
c.mh = 0;
c.mc = 30.0;
c.rh = 9;
c.rc = 5.0;
}
}
auto mixed = fleet::fleet_gate({a, b, node_report(bad_cells)});
CHECK(mixed.cells.size() == 9 && mixed.complete);
// sorted by (model, period): seed1 p0..p2 = 0..2, so the failing period-1 cells are 1, 4, 7.
CHECK(mixed.cells[0].gate.passed && !mixed.cells[1].gate.passed && mixed.cells[2].gate.passed);
CHECK(mixed.cells[3].gate.passed && !mixed.cells[4].gate.passed && mixed.cells[5].gate.passed);
CHECK(mixed.cells[6].gate.passed && !mixed.cells[7].gate.passed && mixed.cells[8].gate.passed);
CHECK(!mixed.passed);
// G4: malformed and cell-less reports are skipped, not counted.
auto noisy = fleet::fleet_gate({a, b, "not json", R"({"policy_eval":{"windows":3,"policies":[]}})", ""});
CHECK(noisy.reports == 2 && noisy.errors.size() == 3 && noisy.passed && noisy.complete);
CHECK(!fleet::fleet_gate({}).passed); // no evidence is not a pass
// A report wrapped like the CLI's {"ok":..,"data":{..}} is read too.
auto wrapped = fleet::fleet_gate({"{\"ok\":true,\"data\":" + a + "}", b});
CHECK(wrapped.reports == 2 && wrapped.passed);
CHECK(fleet::fleet_gate_json(pooled).find("\"passed\": true") != std::string::npos);
// G5: a cell must name the model bytes it judged. A bare filename is not an
// identity: a stale dist/gate model from an older cycle can carry the same
// name as the one a later cycle trained.
const auto stale = node_report(matrix(), "fedcba9876543210");
auto mixed_models = fleet::fleet_gate({a, stale});
CHECK(mixed_models.refused && !mixed_models.passed);
CHECK(has_error(mixed_models, "stale or mixed report set"));
// G6: reports without a digest (an older replay) are refused, not pooled.
auto legacy = node_report(matrix(), "");
auto legacy_set = fleet::fleet_gate({a, legacy});
CHECK(legacy_set.refused && !legacy_set.passed);
CHECK(has_error(legacy_set, "no model digest"));
// G7: in-sample evaluation is refused by the pooled cell, with the reason.
const auto in_sample = node_report(matrix(), "0123456789abcdef",
/*training_last=*/5000, /*eval_first=*/3000, /*eval_last=*/4000);
auto same = node_report(matrix(), "0123456789abcdef",
/*training_last=*/5000, /*eval_first=*/3000, /*eval_last=*/4000);
auto in_sample_set = fleet::fleet_gate({in_sample, same});
CHECK(!in_sample_set.refused && in_sample_set.cells.size() == 9 && in_sample_set.cells[0].nodes == 2);
CHECK(in_sample_set.cells[0].pooled.provenance.in_sample && !in_sample_set.cells[0].gate.passed);
CHECK(in_sample_set.cells[0].gate.checks.back().find("in-sample evaluation") != std::string::npos);
CHECK(!in_sample_set.passed);
// The same reports with a disjoint training range pass that check.
const auto disjoint = node_report(matrix(), "0123456789abcdef",
/*training_last=*/2000, /*eval_first=*/3000, /*eval_last=*/4000);
auto disjoint_set = fleet::fleet_gate({disjoint, disjoint});
CHECK(!disjoint_set.refused && disjoint_set.cells.size() == 9 && disjoint_set.passed && disjoint_set.complete);
// G8: a 0-hit rarity baseline fails the pooled cell (it used to pass free).
const auto no_hits = node_report(matrix(6, 5, 10.0, 0, 0.0));
auto degenerate = fleet::fleet_gate({no_hits, no_hits});
CHECK(degenerate.cells.size() == 9 && !degenerate.cells[0].gate.passed && !degenerate.passed);
CHECK(degenerate.cells[0].gate.checks[1].find("degenerate baseline") != std::string::npos);
// G9a: pooling may not UPGRADE provenance. One node knows the training range, the
// other does not: every other criterion still passes (12 positives, 1.5x efficiency),
// so the only thing standing between this set and a PASS is the unknown provenance -
// which must win.
const auto known = node_report(matrix());
const auto unknown = node_report(matrix(6, 5, 10.0, 3, 20.0, /*training_known=*/false));
auto upgraded = fleet::fleet_gate({known, unknown});
CHECK(upgraded.complete && upgraded.cells.size() == 9);
CHECK(upgraded.cells[0].nodes == 2 && upgraded.cells[0].pooled.extra_positives == 12);
CHECK(upgraded.cells[0].provenance_incomplete && upgraded.cells[0].provenance_unknown_nodes == 1);
CHECK(!upgraded.cells[0].pooled.provenance.training_range_known);
CHECK(upgraded.cells[0].pooled.provenance.training_first_ns == 0 &&
upgraded.cells[0].pooled.provenance.training_last_ns == 0);
CHECK(!upgraded.cells[0].gate.passed);
CHECK(upgraded.cells[0].gate.checks.back().find("provenance incomplete") != std::string::npos);
CHECK(!upgraded.passed);
CHECK(has_error(upgraded, "no usable training range"));
// G9b: a "known" flag with an unusable range (inverted / missing end) is unknown too -
// the old code dropped it and kept the other node's range.
const auto bogus = node_report(matrix(), "0123456789abcdef", /*training_last=*/0);
auto bogus_set = fleet::fleet_gate({known, bogus});
CHECK(bogus_set.cells[0].provenance_incomplete && bogus_set.cells[0].provenance_unknown_nodes == 1);
CHECK(!bogus_set.passed);
// G9c: a node that reports no evaluated window range leaves the cell unable to show
// that its evaluation was disjoint from training - unknown, so it fails.
const auto no_eval_range = node_report(matrix(), "0123456789abcdef", 2000, /*eval_first=*/0, /*eval_last=*/0);
auto no_eval_set = fleet::fleet_gate({known, no_eval_range});
CHECK(no_eval_set.cells[0].provenance_incomplete && !no_eval_set.cells[0].gate.passed);
CHECK(!no_eval_set.cells[0].pooled.provenance.training_range_known);
CHECK(has_error(no_eval_set, "no recorded window range"));
CHECK(!no_eval_set.passed);
// G10a: a one-seed report set (3 periods) has a passing cell - 12 pooled positives -
// but it is not the pre-registered matrix, so it must not produce a verdict.
std::vector<C> one_seed;
for (int p = 0; p < 3; ++p) one_seed.push_back(C{"seed1", p, 6, 5, 10.0, 3, 20.0, true});
auto single_seed = fleet::fleet_gate({node_report(one_seed), node_report(one_seed)});
CHECK(single_seed.cells.size() == 3 && single_seed.cells[0].gate.passed);
CHECK(!single_seed.complete && !single_seed.passed);
CHECK(single_seed.expected_cells == 9);
CHECK(has_error(single_seed, "completeness:"));
CHECK(has_error(single_seed, "need 3 training seeds"));
CHECK(single_seed.reports == 2);
// G10b: a one-period report set (3 seeds) is equally incomplete.
std::vector<C> one_period;
for (const char* m : kSeeds) one_period.push_back(C{m, 0, 6, 5, 10.0, 3, 20.0, true});
auto single_period = fleet::fleet_gate({node_report(one_period), node_report(one_period)});
CHECK(single_period.cells.size() == 3 && single_period.cells[0].gate.passed);
CHECK(!single_period.complete && !single_period.passed);
CHECK(has_error(single_period, "need 3 replay periods"));
// G10c: a hole in the 3x3 matrix is incomplete, not a pass over the surviving cells.
auto holed = matrix();
holed.pop_back(); // seed3 period 2
auto holed_set = fleet::fleet_gate({node_report(holed), node_report(holed)});
CHECK(holed_set.cells.size() == 8 && !holed_set.complete && !holed_set.passed);
CHECK(has_error(holed_set, "cross product"));
CHECK(has_error(holed_set, "seed3 period 2"));
// One node with a hole the other fills still leaves the matrix whole, but the nodes did
// not run the same evaluation, so the set is incomplete too.
auto raggedy = fleet::fleet_gate({node_report(holed), node_report(matrix())});
CHECK(raggedy.cells.size() == 9 && !raggedy.complete && !raggedy.passed);
CHECK(has_error(raggedy, "(seed, period) cells"));
// G10d: nodes that disagree about --periods (2 vs 3) are not comparable - period 1 of
// a 2-way split is not period 1 of a 3-way split - so the set is incomplete even though
// the pooled union happens to be full.
std::vector<C> two_periods;
for (const char* m : kSeeds) {
for (int p = 0; p < 2; ++p) two_periods.push_back(C{m, p, 6, 5, 10.0, 3, 20.0, true});
}
auto inconsistent = fleet::fleet_gate({node_report(matrix()), node_report(two_periods)});
CHECK(inconsistent.cells.size() == 9 && !inconsistent.complete && !inconsistent.passed);
CHECK(has_error(inconsistent, "inconsistent --eval-model / --periods"));
// G10e: periods that do not start at 0 are not the replay's period indices.
std::vector<C> shifted;
for (const char* m : kSeeds) {
for (int p = 1; p < 4; ++p) shifted.push_back(C{m, p, 6, 5, 10.0, 3, 20.0, true});
}
auto shifted_set = fleet::fleet_gate({node_report(shifted), node_report(shifted)});
CHECK(!shifted_set.complete && !shifted_set.passed);
CHECK(has_error(shifted_set, "period indices are not 0.."));
std::cout << "fleet gate: PASSED" << std::endl;
return 0;
}