176 lines
8.5 KiB
Bash
Executable file
176 lines
8.5 KiB
Bash
Executable file
#!/bin/sh
|
|
# Validate the authentic evaluated-prefix BK64/BK128 full-graph gate.
|
|
set -eu
|
|
|
|
[ "$#" -eq 1 ] || [ "$#" -eq 2 ] || {
|
|
echo "usage: $0 GATE_OUTPUT_DIR [OUTPUT_JSON]" >&2
|
|
exit 2
|
|
}
|
|
directory=$1
|
|
output=${2:-}
|
|
|
|
for file in gate-config.json operator-decision.json print-config.json result.json \
|
|
host-preflight.json host-postflight.json host-monitor.ndjson \
|
|
sha256.txt native-linkage.txt macos-version.txt run-start-utc.txt \
|
|
model-sha256.txt; do
|
|
[ -s "$directory/$file" ] || {
|
|
echo "gem: missing BK128 graph-gate input: $directory/$file" >&2
|
|
exit 2
|
|
}
|
|
done
|
|
[ ! -e "$directory/host-monitor.failure.txt" ] || {
|
|
echo "gem: BK128 graph gate recorded a host-monitor failure" >&2
|
|
exit 1
|
|
}
|
|
|
|
analyze()
|
|
{
|
|
jq -n \
|
|
--slurpfile cfg "$directory/gate-config.json" \
|
|
--slurpfile operator "$directory/operator-decision.json" \
|
|
--slurpfile config "$directory/print-config.json" \
|
|
--slurpfile result "$directory/result.json" \
|
|
--slurpfile pre "$directory/host-preflight.json" \
|
|
--slurpfile post "$directory/host-postflight.json" \
|
|
--slurpfile monitor "$directory/host-monitor.ndjson" \
|
|
--rawfile sha "$directory/sha256.txt" \
|
|
--rawfile linkage "$directory/native-linkage.txt" \
|
|
--rawfile macos "$directory/macos-version.txt" \
|
|
--rawfile started "$directory/run-start-utc.txt" \
|
|
--rawfile model_sha "$directory/model-sha256.txt" '
|
|
def mean($xs): $xs | add / length;
|
|
def sample_stddev($xs): mean($xs) as $m |
|
|
(($xs | map((. - $m) * (. - $m)) | add) / (($xs | length) - 1)) | sqrt;
|
|
def cv_pct($xs): 100 * sample_stddev($xs) / mean($xs);
|
|
def close($a; $b): (($a - $b) | fabs) <= 1e-6 * ([1, ($a | fabs), ($b | fabs)] | max);
|
|
def host_ok($h):
|
|
$h.schema_version == 2 and $h.status == "pass" and
|
|
$h.logical_cpus > 0 and $h.load_per_logical_cpu >= 0 and
|
|
$h.load_per_logical_cpu <= $h.max_load_per_logical_cpu and
|
|
$h.short_window_busy_cpu_percent >= 0 and
|
|
$h.short_window_busy_cpu_percent <= $h.max_short_window_busy_cpu_percent and
|
|
$h.ac_power == true and $h.no_thermal_warning == true and
|
|
($h.power_state | contains("AC Power"));
|
|
def monitor_ok($h):
|
|
$h.schema_version == 2 and $h.status == "pass" and
|
|
$h.overloaded == false and host_ok($h);
|
|
|
|
$cfg[0] as $g | $operator[0] as $o | $config[0] as $pc |
|
|
$result[0] as $r | $pre[0] as $hp | $post[0] as $hq |
|
|
$r.baseline.samples_seconds as $b | $r.candidate.samples_seconds as $c |
|
|
[range(0; $b | length) as $i | 100 * (1 - $c[$i] / $b[$i])] as $paired |
|
|
mean($b) as $bm | mean($c) as $cm |
|
|
cv_pct($b) as $bcv | cv_pct($c) as $ccv |
|
|
mean($paired) as $pm | sample_stddev($paired) as $psd |
|
|
(100 * (1 - $cm / $bm)) as $gain |
|
|
($pm - 4.303 * $psd / (($paired | length) | sqrt)) as $paired_lower |
|
|
|
|
if (($sha | contains("build/gem-long-prefix-ab")) and
|
|
($sha | contains("src/long_prefix_ab.c")) and
|
|
($sha | contains("src/metal.m")) and
|
|
($sha | contains("src/metal_layer.c")) and
|
|
($sha | contains("src/inference.c")) and
|
|
($sha | contains("scripts/run-bq8-bk128-graph-gate.sh")) and
|
|
($sha | contains("scripts/analyze-bq8-bk128-graph-gate.sh")) and
|
|
($linkage | contains("gem-long-prefix-ab")) and
|
|
(($linkage | test("lib(llama|ggml)"; "i")) | not) and
|
|
($macos | length) > 1 and
|
|
($started | test("^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z")) and
|
|
(($model_sha | rtrimstr("\n")) | test("^[0-9a-f]{64} .+")) and
|
|
$g.schema_version == 1 and $g.engine == "gem-native-c-metal" and
|
|
$g.axis == {name: "global_prefill_cache_tile_width", baseline: 64, candidate: 128} and
|
|
$g.operator_gate_required == true and
|
|
$g.prefix_tokens == 65536 and $g.extension_tokens == 32 and
|
|
$g.repetitions == 3 and $g.kv_type == "f32" and
|
|
$g.prefix_mode == "evaluated-prompt" and
|
|
$g.baseline_selector == 19 and $g.candidate_selector == 23 and
|
|
$g.minimum_improvement_pct == 3 and
|
|
$g.minimum_logit_correlation == 0.9999 and
|
|
$g.maximum_logit_nmse == 0.0001 and
|
|
$g.paired_confidence_level_pct == 95 and
|
|
$g.paired_t_critical_df2 == 4.303 and
|
|
$g.host_policy.max_load_per_logical_cpu > 0 and
|
|
$g.host_policy.max_short_window_busy_cpu_percent > 0 and
|
|
$g.host_policy.requires_ac_power == true and
|
|
$g.host_policy.requires_no_thermal_warning == true and
|
|
$g.host_policy.monitor_interval_seconds > 0 and
|
|
$g.host_policy.abort_after_consecutive_failures == 1 and
|
|
$g.host_policy.sleep_assertion == "caffeinate -i -w benchmark_pid" and
|
|
$o.schema_version == 2 and
|
|
$o.screening.advance_to_evaluated_graph_gate == true and
|
|
$o.screening.promotion_eligible == false and
|
|
$o.screening.decision == "advance_to_evaluated_graph_gate" and
|
|
$pc.engine == "gem-native-c-metal" and $pc.model_size == 31 and
|
|
$pc.layers == 60 and $pc.context == 65568 and $pc.kv_type == "f32" and
|
|
$pc.matrix_type == "Q4_0" and $pc.prefill_batch == 32 and
|
|
$pc.attention_prefill_rows == 32 and
|
|
($pc.attention_kernel | contains("bq8-bk64")) and
|
|
$r.schema_version == 1 and $r.engine == "gem-native-c-metal" and
|
|
$r.baseline_route == "selector19-bq8-bk64-matrix-global" and
|
|
$r.candidate_route == "selector23-bq8-bk128-matrix-global" and
|
|
$r.shared_prefix == true and $r.prefix_mode == "evaluated-prompt" and
|
|
$r.prefix_builds == 1 and $r.repeated_logits_bit_identical == true and
|
|
$r.prefix_tokens == 65536 and $r.extension_tokens == 32 and
|
|
$r.repetitions == 3 and ($b | length) == 3 and ($c | length) == 3 and
|
|
($b | all(. > 0)) and ($c | all(. > 0)) and
|
|
close($r.baseline.mean_seconds; $bm) and
|
|
close($r.candidate.mean_seconds; $cm) and
|
|
$r.logits.rowwise_argmax == $r.logits.candidate_argmax and
|
|
$r.logits.same_top10_set == true and
|
|
$r.logits.correlation >= $g.minimum_logit_correlation and
|
|
$r.logits.nmse <= $g.maximum_logit_nmse and
|
|
host_ok($hp) and host_ok($hq) and
|
|
($monitor | length) > 0 and ($monitor | all(monitor_ok(.)))) | not
|
|
then error("BK128 evaluated-graph inputs violate frozen controls")
|
|
else
|
|
([ $bcv, $ccv ] | max) as $noise |
|
|
($gain >= $g.minimum_improvement_pct and $gain > $noise and
|
|
$paired_lower > 0) as $performance |
|
|
{
|
|
schema_version: 1,
|
|
scope: "authentic evaluated-prefix 31B/F32 full-graph BK64/BK128 gate",
|
|
axis: $g.axis,
|
|
provenance: {
|
|
run_started_utc: ($started | rtrimstr("\n")),
|
|
macos_version: ($macos | rtrimstr("\n")),
|
|
model_sha256_receipt: ($model_sha | rtrimstr("\n")),
|
|
sha256_receipt: ($sha | rtrimstr("\n")),
|
|
native_linkage_receipt: ($linkage | rtrimstr("\n"))
|
|
},
|
|
host: {preflight: $hp, monitor_samples: $monitor, postflight: $hq},
|
|
correctness: {
|
|
repeated_logits_bit_identical: true,
|
|
same_argmax: true,
|
|
same_top10_set: true,
|
|
correlation: $r.logits.correlation,
|
|
nmse: $r.logits.nmse,
|
|
passed: true
|
|
},
|
|
performance: {
|
|
baseline_mean_seconds: $bm,
|
|
candidate_mean_seconds: $cm,
|
|
improvement_pct: $gain,
|
|
baseline_cv_pct: $bcv,
|
|
candidate_cv_pct: $ccv,
|
|
observed_noise_floor_pct: $noise,
|
|
paired_improvement_mean_pct: $pm,
|
|
paired_improvement_stddev_pct: $psd,
|
|
paired_improvement_95ci_lower_pct: $paired_lower,
|
|
minimum_improvement_pct: $g.minimum_improvement_pct,
|
|
passed: $performance
|
|
},
|
|
decision: {
|
|
selector19_internal_bk128_change_permitted: $performance,
|
|
production_changed: false,
|
|
value: (if $performance then "permit_selector19_internal_bk128_change"
|
|
else "keep_bk64_and_remove_bk128" end),
|
|
reason: (if $performance
|
|
then "operator and authentic evaluated-graph gates passed; implementation, correctness, oracle, and server gates remain"
|
|
else "authentic evaluated-graph gain did not exceed 3%, observed variation, and paired uncertainty"
|
|
end)
|
|
}
|
|
}
|
|
end'
|
|
}
|
|
|
|
if [ -n "$output" ]; then analyze > "$output"; else analyze; fi
|