143 lines
6 KiB
Bash
Executable file
143 lines
6 KiB
Bash
Executable file
#!/bin/sh
|
|
set -eu
|
|
|
|
[ "$#" -eq 1 ] || { echo "usage: $0 ANALYZER" >&2; exit 2; }
|
|
analyzer=$1
|
|
fixture=$(mktemp -d /private/tmp/gem-bk128-graph-analysis.XXXXXX)
|
|
trap 'rm -rf "$fixture"' EXIT INT TERM HUP
|
|
gate="$fixture/gate"
|
|
mkdir -p "$gate"
|
|
|
|
printf '%s\n' \
|
|
'hash build/gem-long-prefix-ab' \
|
|
'hash build/gem' \
|
|
'hash src/long_prefix_ab.c' \
|
|
'hash src/metal.m' \
|
|
'hash src/metal_layer.c' \
|
|
'hash src/inference.c' \
|
|
'hash scripts/run-bq8-bk128-graph-gate.sh' \
|
|
'hash scripts/analyze-bq8-bk128-graph-gate.sh' > "$gate/sha256.txt"
|
|
printf '%s\n' \
|
|
'build/gem-long-prefix-ab:' \
|
|
' /System/Library/Frameworks/Metal.framework/Metal' \
|
|
'build/gem:' > "$gate/native-linkage.txt"
|
|
printf '%s\n' 'ProductName: macOS' 'ProductVersion: 15.6' > "$gate/macos-version.txt"
|
|
printf '%s\n' '2026-08-26T22:00:00Z' > "$gate/run-start-utc.txt"
|
|
printf '%s %s\n' \
|
|
'0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef' \
|
|
'/models/gemma-4-31B-it-Q4_0.gguf' > "$gate/model-sha256.txt"
|
|
|
|
jq -n '{schema_version: 1, engine: "gem-native-c-metal",
|
|
model: "/models/gemma-4-31B-it-Q4_0.gguf",
|
|
model_sha256: "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
|
|
axis: {name: "global_prefill_cache_tile_width", baseline: 64, candidate: 128},
|
|
operator_gate_required: true, prefix_tokens: 65536, extension_tokens: 32,
|
|
repetitions: 3, kv_type: "f32", prefix_mode: "evaluated-prompt",
|
|
baseline_selector: 19, candidate_selector: 23,
|
|
minimum_improvement_pct: 3, minimum_logit_correlation: 0.9999,
|
|
maximum_logit_nmse: 0.0001, paired_confidence_level_pct: 95,
|
|
paired_t_critical_df2: 4.303,
|
|
host_policy: {max_load_per_logical_cpu: 1,
|
|
max_short_window_busy_cpu_percent: 10,
|
|
requires_ac_power: true, requires_no_thermal_warning: true,
|
|
monitor_interval_seconds: 30, abort_after_consecutive_failures: 1,
|
|
sleep_assertion: "caffeinate -i -w benchmark_pid"}}
|
|
' > "$gate/gate-config.json"
|
|
jq -n '{schema_version: 2,
|
|
screening: {advance_to_evaluated_graph_gate: true, promotion_eligible: false,
|
|
decision: "advance_to_evaluated_graph_gate"}}
|
|
' > "$gate/operator-decision.json"
|
|
jq -n '{engine: "gem-native-c-metal", model_size: 31, layers: 60,
|
|
context: 65568, kv_type: "f32", matrix_type: "Q4_0", prefill_batch: 32,
|
|
attention_prefill_rows: 32,
|
|
attention_kernel: "rowwise-gqa4096+31b-full-online-bq8-bk64-matrix+decode-split16"}
|
|
' > "$gate/print-config.json"
|
|
|
|
write_host()
|
|
{
|
|
phase=$1
|
|
jq -cn --arg phase "$phase" '
|
|
{schema_version: 2, phase: $phase, status: "pass", overloaded: false,
|
|
logical_cpus: 16, load_per_logical_cpu: 0.25,
|
|
max_load_per_logical_cpu: 1,
|
|
recent_aggregate_process_cpu_percent: 100,
|
|
short_window_busy_cpu_percent: 5,
|
|
max_short_window_busy_cpu_percent: 10,
|
|
power_state: "Now drawing from AC Power", ac_power: true,
|
|
thermal_state: "no warnings", no_thermal_warning: true}'
|
|
}
|
|
write_host preflight > "$gate/host-preflight.json"
|
|
write_host postflight > "$gate/host-postflight.json"
|
|
write_host monitor > "$gate/host-monitor.ndjson"
|
|
|
|
write_result()
|
|
{
|
|
candidate=$1
|
|
jq -n --argjson candidate "$candidate" '
|
|
{schema_version: 1, engine: "gem-native-c-metal",
|
|
baseline_route: "selector19-bq8-bk64-matrix-global",
|
|
candidate_route: "selector23-bq8-bk128-matrix-global",
|
|
shared_prefix: true, prefix_mode: "evaluated-prompt", prefix_builds: 1,
|
|
repeated_logits_bit_identical: true, prefix_tokens: 65536,
|
|
extension_tokens: 32, repetitions: 3,
|
|
baseline: {mean_seconds: 1, samples_seconds: [1,1,1]},
|
|
candidate: {mean_seconds: $candidate,
|
|
samples_seconds: [$candidate,$candidate,$candidate]},
|
|
delta: {wall_percent: (100 * ($candidate - 1)), speedup: (1 / $candidate)},
|
|
logits: {rowwise_argmax: 42, candidate_argmax: 42, same_top10_set: true,
|
|
correlation: 0.99999, nmse: 0.00001}}
|
|
' > "$gate/result.json"
|
|
}
|
|
|
|
write_result 0.95
|
|
"$analyzer" "$gate" "$fixture/pass.json"
|
|
jq -e '.correctness.passed == true and .performance.passed == true and
|
|
(.performance.improvement_pct > 4.999 and
|
|
.performance.improvement_pct < 5.001) and
|
|
.performance.paired_improvement_95ci_lower_pct > 0 and
|
|
.decision.selector19_internal_bk128_change_permitted == true and
|
|
.decision.production_changed == false and
|
|
.decision.value == "permit_selector19_internal_bk128_change"' \
|
|
"$fixture/pass.json" >/dev/null
|
|
|
|
write_result 0.98
|
|
"$analyzer" "$gate" "$fixture/slow.json"
|
|
jq -e '.correctness.passed == true and .performance.passed == false and
|
|
.decision.selector19_internal_bk128_change_permitted == false and
|
|
.decision.value == "keep_bk64_and_remove_bk128"' "$fixture/slow.json" >/dev/null
|
|
write_result 0.95
|
|
|
|
jq '.screening.advance_to_evaluated_graph_gate = false' \
|
|
"$gate/operator-decision.json" > "$gate/operator-decision.tmp"
|
|
mv "$gate/operator-decision.tmp" "$gate/operator-decision.json"
|
|
if "$analyzer" "$gate" >/dev/null 2>&1; then
|
|
echo "BK128 graph analyzer accepted a non-advancing operator decision" >&2
|
|
exit 1
|
|
fi
|
|
jq '.screening.advance_to_evaluated_graph_gate = true' \
|
|
"$gate/operator-decision.json" > "$gate/operator-decision.tmp"
|
|
mv "$gate/operator-decision.tmp" "$gate/operator-decision.json"
|
|
|
|
jq '.logits.correlation = 0.99' "$gate/result.json" > "$gate/result.tmp"
|
|
mv "$gate/result.tmp" "$gate/result.json"
|
|
if "$analyzer" "$gate" >/dev/null 2>&1; then
|
|
echo "BK128 graph analyzer accepted bad logit correlation" >&2
|
|
exit 1
|
|
fi
|
|
jq '.logits.correlation = 0.99999' "$gate/result.json" > "$gate/result.tmp"
|
|
mv "$gate/result.tmp" "$gate/result.json"
|
|
|
|
jq '.ac_power = false | .status = "fail" | .overloaded = true' \
|
|
"$gate/host-monitor.ndjson" > "$gate/host-monitor.tmp"
|
|
mv "$gate/host-monitor.tmp" "$gate/host-monitor.ndjson"
|
|
if "$analyzer" "$gate" >/dev/null 2>&1; then
|
|
echo "BK128 graph analyzer accepted a failed host monitor" >&2
|
|
exit 1
|
|
fi
|
|
write_host monitor > "$gate/host-monitor.ndjson"
|
|
|
|
printf '%s\n' 'build/gem-long-prefix-ab: libllama.dylib' > "$gate/native-linkage.txt"
|
|
if "$analyzer" "$gate" >/dev/null 2>&1; then
|
|
echo "BK128 graph analyzer accepted llama linkage" >&2
|
|
exit 1
|
|
fi
|