150 lines
7.2 KiB
Bash
Executable file
150 lines
7.2 KiB
Bash
Executable file
#!/bin/sh
|
|
set -eu
|
|
|
|
[ "$#" -eq 1 ] || { echo "usage: $0 ANALYZER" >&2; exit 2; }
|
|
analyzer=$1
|
|
fixture=$(mktemp -d /private/tmp/gem-bq8-bk128-analysis.XXXXXX)
|
|
trap 'rm -rf "$fixture"' EXIT INT TERM HUP
|
|
|
|
write_run()
|
|
{
|
|
directory=$1
|
|
tokens=$2
|
|
candidate=$3
|
|
position=$((tokens - 32))
|
|
jq -n --argjson position "$position" --argjson capacity "$tokens" \
|
|
--argjson candidate "$candidate" '
|
|
{schema_version: 1, engine: "gem-native-c-metal", shape: "31b-full",
|
|
kv_type: "f32", position: $position, rows: 32, cache_capacity: $capacity,
|
|
repetitions: 10, warmup: 10, seed_prefix: true,
|
|
bq8_pipeline_resources: {thread_execution_width: 32,
|
|
max_total_threads_per_threadgroup: 1024, static_threadgroup_memory_bytes: 0,
|
|
dynamic_threadgroup_memory_bytes: 18496, threads_per_threadgroup: 256},
|
|
bq8_bk128_pipeline_resources: {thread_execution_width: 32,
|
|
max_total_threads_per_threadgroup: 1024, static_threadgroup_memory_bytes: 0,
|
|
dynamic_threadgroup_memory_bytes: 20544, threads_per_threadgroup: 256},
|
|
variants: {
|
|
"online-bq8": {samples: [1,1,1,1,1,1,1,1,1,1]},
|
|
"online-bq8-bk128": {samples: [$candidate,$candidate,$candidate,$candidate,$candidate,
|
|
$candidate,$candidate,$candidate,$candidate,$candidate]}
|
|
}}' > "$directory/seeded-$tokens.json"
|
|
for phase in preflight postflight; do
|
|
jq -n --arg phase "$phase" \
|
|
'{schema_version: 2, phase: $phase, status: "pass", logical_cpus: 16,
|
|
load_per_logical_cpu: 0.25, max_load_per_logical_cpu: 1,
|
|
recent_aggregate_process_cpu_percent: 250,
|
|
short_window_busy_cpu_percent: 5,
|
|
max_short_window_busy_cpu_percent: 10,
|
|
power_state: "Now drawing from '\''AC Power'\''", ac_power: true,
|
|
thermal_state: "Note: No thermal warning level has been recorded\nNote: No performance warning level has been recorded",
|
|
no_thermal_warning: true}' \
|
|
> "$directory/host-$phase-$tokens.json"
|
|
done
|
|
}
|
|
|
|
mkdir -p "$fixture/pass" "$fixture/fail"
|
|
for directory in "$fixture/pass" "$fixture/fail"; do
|
|
printf '%s\n' \
|
|
'hash build/gem-long-attention-profile' \
|
|
'hash build/gem-metal-counter-info' \
|
|
'hash src/metal.m' \
|
|
'hash src/long_attention_profile.c' \
|
|
'hash scripts/run-bq8-bk128-gate.sh' \
|
|
'hash scripts/analyze-bq8-bk128-gate.sh' > "$directory/sha256.txt"
|
|
printf '%s\n' \
|
|
'build/gem-long-attention-profile:' \
|
|
' /System/Library/Frameworks/Metal.framework/Metal' \
|
|
'build/gem-metal-counter-info:' > "$directory/native-linkage.txt"
|
|
printf '%s\n' 'ProductName: macOS' 'ProductVersion: 15.6' \
|
|
> "$directory/macos-version.txt"
|
|
printf '%s\n' '2026-08-26T20:00:00Z' > "$directory/run-start-utc.txt"
|
|
jq -n '{schema_version: 1, workload: "metal-counter-info",
|
|
device: {name: "Apple M4 Max", has_unified_memory: true,
|
|
gpu_families: {apple9: true}, max_threadgroup_memory_bytes: 32768}}' \
|
|
> "$directory/metal-device.json"
|
|
jq -n '{schema_version: 2, engine: "gem-native-c-metal",
|
|
axis: {name: "cache_tile_width", baseline: 64, candidate: 128},
|
|
positions: [4064, 65504, 131040], rows: 32, kv_type: "f32",
|
|
repetitions: 10, warmup: 10, seed_prefix: true,
|
|
max_load_per_logical_cpu: 1, max_short_window_busy_cpu_percent: 10,
|
|
requires_ac_power: true, requires_no_thermal_warning: true}' \
|
|
> "$directory/gate-config.json"
|
|
done
|
|
write_run "$fixture/pass" 4096 0.99
|
|
write_run "$fixture/pass" 65536 0.95
|
|
write_run "$fixture/pass" 131072 0.94
|
|
"$analyzer" "$fixture/pass" "$fixture/pass.json"
|
|
jq -e '.screening.advance_to_evaluated_graph_gate == true and
|
|
.screening.promotion_eligible == false and
|
|
.threadgroup_memory.baseline_limit_utilization_pct == 56.4453125 and
|
|
.threadgroup_memory.candidate_limit_utilization_pct == 62.6953125 and
|
|
.contexts.at_65536.paired_improvement_95ci_lower_pct > 0 and
|
|
(.provenance.sha256_receipt | contains("src/metal.m")) and
|
|
.screening.decision == "advance_to_evaluated_graph_gate"' \
|
|
"$fixture/pass.json" >/dev/null
|
|
|
|
# Aggregate means and per-arm CVs can look acceptable while paired passes disagree.
|
|
# The paired 95% lower confidence bound must independently reject that case.
|
|
jq '.variants["online-bq8"].samples =
|
|
[0.972,1.028,0.972,1.028,0.972,1.028,0.972,1.028,0.972,1.028] |
|
|
.variants["online-bq8-bk128"].samples =
|
|
[0.986,0.934,0.986,0.934,0.986,0.934,0.986,0.934,0.986,0.934]' \
|
|
"$fixture/pass/seeded-65536.json" > "$fixture/pass/seeded-65536.json.tmp"
|
|
mv "$fixture/pass/seeded-65536.json.tmp" "$fixture/pass/seeded-65536.json"
|
|
"$analyzer" "$fixture/pass" "$fixture/paired-noise.json"
|
|
jq -e '.contexts.at_65536.improvement_pct >= 3 and
|
|
.contexts.at_65536.stable == true and
|
|
.contexts.at_65536.paired_improvement_95ci_lower_pct <= 0 and
|
|
.screening.advance_to_evaluated_graph_gate == false' \
|
|
"$fixture/paired-noise.json" >/dev/null
|
|
write_run "$fixture/pass" 65536 0.95
|
|
|
|
write_run "$fixture/fail" 4096 0.99
|
|
write_run "$fixture/fail" 65536 1.04
|
|
write_run "$fixture/fail" 131072 0.94
|
|
"$analyzer" "$fixture/fail" "$fixture/fail.json"
|
|
jq -e '.screening.advance_to_evaluated_graph_gate == false and
|
|
.screening.decision == "keep_bk64_and_remove_bk128"' \
|
|
"$fixture/fail.json" >/dev/null
|
|
|
|
jq '.bq8_bk128_pipeline_resources.max_total_threads_per_threadgroup = 512' \
|
|
"$fixture/pass/seeded-65536.json" > "$fixture/pass/seeded-65536.json.tmp"
|
|
mv "$fixture/pass/seeded-65536.json.tmp" "$fixture/pass/seeded-65536.json"
|
|
if "$analyzer" "$fixture/pass" >/dev/null 2>&1; then
|
|
echo "BK128 analyzer accepted a changed pipeline-resource control" >&2
|
|
exit 1
|
|
fi
|
|
|
|
jq '.status = "pass" | .ac_power = false |
|
|
.power_state = "Now drawing from '\''Battery Power'\''"' \
|
|
"$fixture/pass/host-postflight-131072.json" \
|
|
> "$fixture/pass/host-postflight-131072.json.tmp"
|
|
mv "$fixture/pass/host-postflight-131072.json.tmp" \
|
|
"$fixture/pass/host-postflight-131072.json"
|
|
if "$analyzer" "$fixture/pass" >/dev/null 2>&1; then
|
|
echo "BK128 analyzer accepted a battery-power postflight receipt" >&2
|
|
exit 1
|
|
fi
|
|
|
|
jq '.ac_power = true | .power_state = "Now drawing from '\''AC Power'\''" |
|
|
.no_thermal_warning = false' \
|
|
"$fixture/pass/host-postflight-131072.json" \
|
|
> "$fixture/pass/host-postflight-131072.json.tmp"
|
|
mv "$fixture/pass/host-postflight-131072.json.tmp" \
|
|
"$fixture/pass/host-postflight-131072.json"
|
|
if "$analyzer" "$fixture/pass" >/dev/null 2>&1; then
|
|
echo "BK128 analyzer accepted a thermal-warning postflight receipt" >&2
|
|
exit 1
|
|
fi
|
|
jq '.bq8_bk128_pipeline_resources.max_total_threads_per_threadgroup = 1024' \
|
|
"$fixture/pass/seeded-65536.json" > "$fixture/pass/seeded-65536.json.tmp"
|
|
mv "$fixture/pass/seeded-65536.json.tmp" "$fixture/pass/seeded-65536.json"
|
|
|
|
jq '.status = "fail"' "$fixture/pass/host-postflight-131072.json" \
|
|
> "$fixture/pass/host-postflight-131072.json.tmp"
|
|
mv "$fixture/pass/host-postflight-131072.json.tmp" \
|
|
"$fixture/pass/host-postflight-131072.json"
|
|
if "$analyzer" "$fixture/pass" >/dev/null 2>&1; then
|
|
echo "BK128 analyzer accepted a failed postflight receipt" >&2
|
|
exit 1
|
|
fi
|