105 lines
4.7 KiB
Bash
Executable file
105 lines
4.7 KiB
Bash
Executable file
#!/bin/sh
|
|
set -eu
|
|
|
|
if [ "$#" -ne 1 ]; then
|
|
echo "usage: $0 ANALYZER" >&2
|
|
exit 2
|
|
fi
|
|
analyzer=$1
|
|
temporary=$(mktemp -d /private/tmp/gem-score-tile8-decision.XXXXXX)
|
|
trap 'rm -rf "$temporary"' EXIT INT TERM HUP
|
|
|
|
make_comparison() {
|
|
tokens=$1
|
|
baseline=$2
|
|
candidate=$3
|
|
destination=$4
|
|
jq -n --argjson tokens "$tokens" --argjson baseline "$baseline" \
|
|
--argjson candidate "$candidate" '
|
|
{axis: {name: "score_token_tile", baseline: 1, candidate: 8},
|
|
controls: {tokens: $tokens},
|
|
score_stage: {baseline_mean_seconds: $baseline,
|
|
candidate_mean_seconds: $candidate},
|
|
screening: {threshold_passed: true, promotion_eligible: false},
|
|
bracket: {total_gpu: {every_candidate_sample_faster: true}}}
|
|
' > "$destination"
|
|
}
|
|
|
|
make_comparison 65536 0.00311742207 0.00249393766 "$temporary/c64.json"
|
|
make_comparison 131072 0.00671256673 0.00503442505 "$temporary/c128.json"
|
|
jq -n '{selected_next_axis: "full-attention-head-dim512",
|
|
screening: {promotion_eligible: false}}' > "$temporary/prefill.json"
|
|
jq -n '{passed: 4793, total: 4793}' > "$temporary/oracle.json"
|
|
jq -n '{schema_version: 2, engine: "gem-native-c-metal",
|
|
axis: {name: "score_token_tile", baseline: 1, candidate: 8},
|
|
contexts: [65536, 131072], repetitions_per_arm: 3, iterations: 5, warmup: 3,
|
|
abba_order: ["baseline-a1", "candidate-b1", "candidate-b2", "baseline-a2"],
|
|
requires_ac_power: true, requires_no_thermal_warning: true,
|
|
max_load_per_logical_cpu: 1, max_short_window_busy_cpu_percent: 10,
|
|
correctness_after_timing: true}' > "$temporary/gate.json"
|
|
write_host() {
|
|
jq -n '{schema_version: 2, status: "pass", logical_cpus: 16,
|
|
load_per_logical_cpu: 0.25, max_load_per_logical_cpu: 1,
|
|
short_window_busy_cpu_percent: 5, max_short_window_busy_cpu_percent: 10,
|
|
power_state: "Now drawing from AC Power", ac_power: true,
|
|
thermal_state: "no warnings", no_thermal_warning: true}' > "$1"
|
|
}
|
|
write_host "$temporary/h64pre.json"
|
|
write_host "$temporary/h64post.json"
|
|
write_host "$temporary/h128pre.json"
|
|
write_host "$temporary/h128post.json"
|
|
sha=0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
|
|
|
|
"$analyzer" "$temporary/c64.json" "$temporary/c128.json" \
|
|
"$temporary/prefill.json" "$temporary/oracle.json" fixture.gguf "$sha" \
|
|
"$temporary/gate.json" "$temporary/h64pre.json" "$temporary/h64post.json" \
|
|
"$temporary/h128pre.json" "$temporary/h128post.json" \
|
|
"$temporary/decision.json"
|
|
jq -e '
|
|
.operator_screen_passed == true and .promotion_eligible == false and
|
|
.projected_populated_model.projected_whole_token_improvement_pct_64k > 3.5 and
|
|
.projected_populated_model.projected_whole_token_improvement_pct_128k > 5.4
|
|
' "$temporary/decision.json" >/dev/null
|
|
|
|
make_comparison 65536 0.00311742207 0.00280567986 "$temporary/c64.json"
|
|
"$analyzer" "$temporary/c64.json" "$temporary/c128.json" \
|
|
"$temporary/prefill.json" "$temporary/oracle.json" fixture.gguf "$sha" \
|
|
"$temporary/gate.json" "$temporary/h64pre.json" "$temporary/h64post.json" \
|
|
"$temporary/h128pre.json" "$temporary/h128post.json" \
|
|
"$temporary/decision.json"
|
|
jq -e '
|
|
.projected_populated_model.projected_whole_token_improvement_pct_64k < 2 and
|
|
.operator_screen_passed == false
|
|
' "$temporary/decision.json" >/dev/null
|
|
|
|
jq '.total = 4794' "$temporary/oracle.json" > "$temporary/invalid-oracle.json"
|
|
if "$analyzer" "$temporary/c64.json" "$temporary/c128.json" \
|
|
"$temporary/prefill.json" "$temporary/invalid-oracle.json" fixture.gguf "$sha" \
|
|
"$temporary/gate.json" "$temporary/h64pre.json" "$temporary/h64post.json" \
|
|
"$temporary/h128pre.json" "$temporary/h128post.json" \
|
|
>/dev/null 2>&1; then
|
|
echo "failed adjacent oracle was accepted" >&2
|
|
exit 1
|
|
fi
|
|
|
|
jq '.ac_power = false | .power_state = "Now drawing from Battery Power"' \
|
|
"$temporary/h128post.json" > "$temporary/h128post-battery.json"
|
|
if "$analyzer" "$temporary/c64.json" "$temporary/c128.json" \
|
|
"$temporary/prefill.json" "$temporary/oracle.json" fixture.gguf "$sha" \
|
|
"$temporary/gate.json" "$temporary/h64pre.json" "$temporary/h64post.json" \
|
|
"$temporary/h128pre.json" "$temporary/h128post-battery.json" \
|
|
>/dev/null 2>&1; then
|
|
echo "battery-power score-tile8 receipt was accepted" >&2
|
|
exit 1
|
|
fi
|
|
|
|
jq '.no_thermal_warning = false' "$temporary/h64pre.json" \
|
|
> "$temporary/h64pre-thermal.json"
|
|
if "$analyzer" "$temporary/c64.json" "$temporary/c128.json" \
|
|
"$temporary/prefill.json" "$temporary/oracle.json" fixture.gguf "$sha" \
|
|
"$temporary/gate.json" "$temporary/h64pre-thermal.json" "$temporary/h64post.json" \
|
|
"$temporary/h128pre.json" "$temporary/h128post.json" \
|
|
>/dev/null 2>&1; then
|
|
echo "thermal-warning score-tile8 receipt was accepted" >&2
|
|
exit 1
|
|
fi
|