The pre-registered gate (docs/evaluation/stage4-promotion-gate.md) asks for the criteria to hold on 3 non-overlapping periods x 3 training seeds; until now the tool reported a single run and the operator recorded the rest. This produces the 9 results for a fleet, with the criteria unchanged: - replay --policy-eval --eval-model FILE (repeatable) --periods N: one pass evaluates every model on every equal time slice of the ranked history; each (model, period) cell simulates the model and rarity-greedy with their own hourly budgets exactly as policy_eval does (policy_eval.cells). With one model equal to --model and one period the cell equals the single-model evaluation (test R11b); periods partition windows and positives. - strawberry fleet gate DIR: pools the cells of every node report (windows, positives, inspections, CPU-s, hits, refusals) per (model, period) and applies learn::promotion_gate unchanged to each; passes only if all pass. Malformed / cell-less reports are skipped and listed. - Evaluation stays on the node (raw telemetry never leaves it): the central cycle now also publishes all three seed models to dist/gate/; strawberry.gate (sensors) evaluates them with the pre-registered evaluation traces; the operator-run orch.strawberry_gate pushes the reports back and pools them; evidence under central.work_dir/gate-<stamp>/. Nothing promotes. - docs/evaluation/stage4-fleet-gate.md: the procedure (new dated document; criteria and numbers unchanged). Validated: test_fleet_gate (pooling sums, pooling reaching 10 positives neither node has alone, one failing cell fails the fleet, malformed reports skipped), R11b, Salt render checks (no models -> refusal; 3 models x 3 periods; orchestration pools), and scripts/fleet_e2e.sh end to end: 3 nodes -> 9 pooled cells with 28-50 positives each; the untrained synthetic models correctly do not pass (exit code matches the verdict). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
137 lines
7.2 KiB
Bash
Executable file
137 lines
7.2 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
# Offline render check for the Salt states under deploy/salt/. No root, no network,
|
|
# never touches /etc/salt: salt-call runs --local with a scratch config dir.
|
|
#
|
|
# scripts/test_salt_states.sh
|
|
set -euo pipefail
|
|
|
|
cd "$(dirname "$0")/.."
|
|
repo=$PWD
|
|
command -v salt-call >/dev/null || { echo "salt-call not found" >&2; exit 2; }
|
|
|
|
tmp=$(mktemp -d "${TMPDIR:-/tmp}/salt-states-XXXXXX")
|
|
trap '[ -n "${KEEP_TMP:-}" ] || find "$tmp" -delete' EXIT
|
|
conf=$tmp/conf
|
|
mkdir -p "$conf" "$tmp/cache" "$tmp/log" "$tmp/pki" "$tmp/dist-root/strawberry/dist"
|
|
|
|
cat >"$conf/minion" <<EOF
|
|
file_client: local
|
|
root_dir: $tmp
|
|
cachedir: $tmp/cache
|
|
log_file: $tmp/log/minion
|
|
pki_dir: $tmp/pki
|
|
sock_dir: $tmp/sock
|
|
pidfile: $tmp/minion.pid
|
|
file_roots:
|
|
base:
|
|
- $repo/deploy/salt/salt
|
|
- $repo/deploy/salt
|
|
- $tmp/dist-root
|
|
pillar_roots:
|
|
base:
|
|
- $repo/deploy/salt/pillar
|
|
EOF
|
|
|
|
pins='{"strawberry":{"release":{"version":"9.9.9","artifact_sha256":{"trixie":"'"$(printf 'a%.0s' $(seq 64))"'","noble":"'"$(printf 'b%.0s' $(seq 64))"'"}}}}'
|
|
fail=0
|
|
check() { # description, command...
|
|
local d=$1
|
|
shift
|
|
if "$@"; then echo "ok $d"; else echo "FAIL $d"; fail=1; fi
|
|
}
|
|
|
|
# Fixed grains so distro/arch selection is deterministic.
|
|
cat >>"$conf/minion" <<EOF
|
|
grains:
|
|
oscodename: trixie
|
|
osarch: amd64
|
|
EOF
|
|
render() {
|
|
local id=$1 sls=$2 pil=${3:-$pins}
|
|
local out
|
|
if ! out=$(salt-call -c "$conf" --local --id "$id" --out=json --log-level=error \
|
|
state.show_sls "$sls" pillar="$pil" 2>"$tmp/err"); then
|
|
echo "render of $sls as $id failed:" >&2
|
|
cat "$tmp/err" >&2
|
|
return 1
|
|
fi
|
|
# show_sls reports render errors as a list of strings, not a dict of states.
|
|
if ! printf '%s' "$out" | grep -q '"__sls__"'; then
|
|
echo "render of $sls as $id returned no states:" >&2
|
|
printf '%s\n' "$out" >&2
|
|
return 1
|
|
fi
|
|
printf '%s\n' "$out"
|
|
}
|
|
|
|
render sensor-1 strawberry >"$tmp/sensor.json" || fail=1
|
|
render strawberry-central strawberry >"$tmp/central.json" || fail=1
|
|
render strawberry-central orch.strawberry_cycle >"$tmp/orch.json" || fail=1
|
|
render sensor-1 orch.strawberry_cycle \
|
|
'{"strawberry":{"distribute":{"canary_minions":["sensor-1","sensor-2"],"canary_ok":true}}}' \
|
|
>"$tmp/orch-canary.json" || fail=1
|
|
# distribute serves only what the central cycle published: overlay-only first (training refused), then both.
|
|
dist=$tmp/dist-root/strawberry/dist
|
|
echo overlay >"$dist/fleet-overlay.sbo"; (cd "$dist" && sha256sum fleet-overlay.sbo >fleet-overlay.sbo.sha256)
|
|
render sensor-1 strawberry.distribute >"$tmp/dist-overlay-only.json" || fail=1
|
|
check "overlay-only publish: distribute ships the overlay and no model" \
|
|
bash -c "grep -q strawberry-dist-overlay '$tmp/dist-overlay-only.json' && ! grep -q strawberry-dist-shadow '$tmp/dist-overlay-only.json'"
|
|
echo model >"$dist/shadow.sbm"; (cd "$dist" && sha256sum shadow.sbm >shadow.sbm.sha256)
|
|
render sensor-1 strawberry.distribute >"$tmp/dist.json" || fail=1
|
|
|
|
check "sensor render has install, config, export and NO distribute (orchestration owns it)" \
|
|
bash -c "grep -q strawberry-release-tarball '$tmp/sensor.json' && grep -q strawberry-config '$tmp/sensor.json' && grep -q strawberry-export-timer '$tmp/sensor.json' && ! grep -q strawberry-dist-shadow '$tmp/sensor.json'"
|
|
check "central render has cycle timer and no export timer" \
|
|
bash -c "grep -q strawberry-central-cycle-timer '$tmp/central.json' && ! grep -q strawberry-export-timer '$tmp/central.json'"
|
|
check "install uses source_hash sha256=<pin> (distro-selected)" \
|
|
bash -c "grep -q 'sha256=a\{64\}' '$tmp/sensor.json'"
|
|
check "capture_redact_cmdline is true" \
|
|
bash -c "tr -d ' \n' <'$tmp/sensor.json' | grep -q '\"capture_redact_cmdline\":\"true\"'"
|
|
check "rule_auto_fetch is false on fleet nodes" \
|
|
bash -c "tr -d ' \n' <'$tmp/sensor.json' | grep -q '\"rule_auto_fetch\":\"false\"'"
|
|
check "distribute pulls shadow + overlay, hashes from sidecars" \
|
|
bash -c "grep -q 'dist/shadow.sbm.sha256' '$tmp/dist.json' && grep -q 'dist/fleet-overlay.sbo.sha256' '$tmp/dist.json'"
|
|
check "orch canary step present when canary_minions set" \
|
|
bash -c "grep -q strawberry-distribute-canary '$tmp/orch-canary.json' && grep -q strawberry-distribute-rest '$tmp/orch-canary.json' && ! grep -q strawberry-distribute-rest-held '$tmp/orch-canary.json'"
|
|
check "orch holds the fleet when canary_ok is false" \
|
|
bash -c "grep -q strawberry-distribute-rest-held '$tmp/orch.json' && ! grep -q '\"strawberry-distribute-rest\"' '$tmp/orch.json'"
|
|
|
|
# The operator-only model gate must not appear anywhere a state, orch or script could run it.
|
|
# Fleet gate: without published gate models the state fails clearly; with them it evaluates all three.
|
|
render sensor-1 strawberry.gate >"$tmp/gate-none.json" || fail=1
|
|
check "gate state refuses without published gate models" \
|
|
bash -c "grep -q strawberry-gate-no-models '$tmp/gate-none.json' && ! grep -q strawberry-gate-evaluate '$tmp/gate-none.json'"
|
|
mkdir -p "$dist/gate"
|
|
for seed in 1 2 3; do
|
|
echo "m$seed" >"$dist/gate/model-seed$seed.sbm"
|
|
(cd "$dist/gate" && sha256sum "model-seed$seed.sbm" >"model-seed$seed.sbm.sha256")
|
|
done
|
|
render sensor-1 strawberry.gate >"$tmp/gate.json" || fail=1
|
|
check "gate state evaluates 3 seed models x 3 periods with the evaluation traces" \
|
|
bash -c "[ \$(grep -o -- '--eval-model' '$tmp/gate.json' | wc -l) -eq 3 ] && grep -q -- '--periods 3' '$tmp/gate.json' && grep -q -- '--inject /usr/local/share/strawberry/traces/standard.trace' '$tmp/gate.json' && grep -q -- '--inject-copies 30' '$tmp/gate.json'"
|
|
render strawberry-central orch.strawberry_gate >"$tmp/orch-gate.json" || fail=1
|
|
check "gate orchestration evaluates on sensors, pushes reports and pools them with fleet gate" \
|
|
bash -c "grep -q strawberry-gate-evaluate '$tmp/orch-gate.json' && grep -q strawberry-gate-push '$tmp/orch-gate.json' && grep -q 'fleet gate' '$tmp/orch-gate.json'"
|
|
|
|
for f in "$tmp"/sensor.json "$tmp"/central.json "$tmp"/orch.json "$tmp"/orch-canary.json "$tmp"/dist.json "$tmp"/dist-overlay-only.json "$tmp"/gate.json "$tmp"/orch-gate.json; do
|
|
check "no 'promote' in $(basename "$f")" bash -c "! grep -qi promote '$f'"
|
|
done
|
|
check "no 'promote' in states, orch, files or cycle script" \
|
|
bash -c "! grep -rliE 'promote' '$repo/deploy/salt/salt' '$repo/deploy/salt/files'"
|
|
|
|
# Fail-closed cases: missing pin, and redaction off.
|
|
render sensor-1 strawberry.install '{"strawberry":{"release":{"version":"","artifact_sha256":""}}}' >"$tmp/nopin.json" || fail=1
|
|
check "empty pin renders the fail state, not a download" \
|
|
bash -c "grep -q strawberry-release-pin-missing '$tmp/nopin.json' && ! grep -q source_hash '$tmp/nopin.json'"
|
|
render sensor-1 strawberry.config "$(printf '%s' '{"strawberry":{"config":{"capture_redact_cmdline":false}}}')" >"$tmp/noredact.json" || fail=1
|
|
check "redaction=false renders the fail state, not a config edit" \
|
|
bash -c "grep -q strawberry-redaction-required '$tmp/noredact.json' && ! grep -q 'key_values' '$tmp/noredact.json'"
|
|
|
|
check "cycle script: bash -n" bash -n deploy/salt/files/strawberry-central-cycle.sh
|
|
check "cycle script is executable" test -x deploy/salt/files/strawberry-central-cycle.sh
|
|
|
|
if [ "$fail" -ne 0 ]; then
|
|
echo "salt state render checks FAILED" >&2
|
|
exit 1
|
|
fi
|
|
echo "salt state render checks passed"
|