#!/bin/sh set -eu if [ "$#" -ne 1 ]; then echo "usage: $0 ANALYZER" >&2 exit 2 fi analyzer=$1 temporary=$(mktemp -d /private/tmp/gem-score-tile8-decision.XXXXXX) trap 'rm -rf "$temporary"' EXIT INT TERM HUP make_comparison() { tokens=$1 baseline=$2 candidate=$3 destination=$4 jq -n --argjson tokens "$tokens" --argjson baseline "$baseline" \ --argjson candidate "$candidate" ' {axis: {name: "score_token_tile", baseline: 1, candidate: 8}, controls: {tokens: $tokens}, score_stage: {baseline_mean_seconds: $baseline, candidate_mean_seconds: $candidate}, screening: {threshold_passed: true, promotion_eligible: false}, bracket: {total_gpu: {every_candidate_sample_faster: true}}} ' > "$destination" } make_comparison 65536 0.00311742207 0.00249393766 "$temporary/c64.json" make_comparison 131072 0.00671256673 0.00503442505 "$temporary/c128.json" jq -n '{selected_next_axis: "full-attention-head-dim512", screening: {promotion_eligible: false}}' > "$temporary/prefill.json" jq -n '{passed: 4793, total: 4793}' > "$temporary/oracle.json" jq -n '{schema_version: 2, engine: "gem-native-c-metal", axis: {name: "score_token_tile", baseline: 1, candidate: 8}, contexts: [65536, 131072], repetitions_per_arm: 3, iterations: 5, warmup: 3, abba_order: ["baseline-a1", "candidate-b1", "candidate-b2", "baseline-a2"], requires_ac_power: true, requires_no_thermal_warning: true, max_load_per_logical_cpu: 1, max_short_window_busy_cpu_percent: 10, correctness_after_timing: true}' > "$temporary/gate.json" write_host() { jq -n '{schema_version: 2, status: "pass", logical_cpus: 16, load_per_logical_cpu: 0.25, max_load_per_logical_cpu: 1, short_window_busy_cpu_percent: 5, max_short_window_busy_cpu_percent: 10, power_state: "Now drawing from AC Power", ac_power: true, thermal_state: "no warnings", no_thermal_warning: true}' > "$1" } write_host "$temporary/h64pre.json" write_host "$temporary/h64post.json" write_host "$temporary/h128pre.json" write_host "$temporary/h128post.json" sha=0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef "$analyzer" "$temporary/c64.json" "$temporary/c128.json" \ "$temporary/prefill.json" "$temporary/oracle.json" fixture.gguf "$sha" \ "$temporary/gate.json" "$temporary/h64pre.json" "$temporary/h64post.json" \ "$temporary/h128pre.json" "$temporary/h128post.json" \ "$temporary/decision.json" jq -e ' .operator_screen_passed == true and .promotion_eligible == false and .projected_populated_model.projected_whole_token_improvement_pct_64k > 3.5 and .projected_populated_model.projected_whole_token_improvement_pct_128k > 5.4 ' "$temporary/decision.json" >/dev/null make_comparison 65536 0.00311742207 0.00280567986 "$temporary/c64.json" "$analyzer" "$temporary/c64.json" "$temporary/c128.json" \ "$temporary/prefill.json" "$temporary/oracle.json" fixture.gguf "$sha" \ "$temporary/gate.json" "$temporary/h64pre.json" "$temporary/h64post.json" \ "$temporary/h128pre.json" "$temporary/h128post.json" \ "$temporary/decision.json" jq -e ' .projected_populated_model.projected_whole_token_improvement_pct_64k < 2 and .operator_screen_passed == false ' "$temporary/decision.json" >/dev/null jq '.total = 4794' "$temporary/oracle.json" > "$temporary/invalid-oracle.json" if "$analyzer" "$temporary/c64.json" "$temporary/c128.json" \ "$temporary/prefill.json" "$temporary/invalid-oracle.json" fixture.gguf "$sha" \ "$temporary/gate.json" "$temporary/h64pre.json" "$temporary/h64post.json" \ "$temporary/h128pre.json" "$temporary/h128post.json" \ >/dev/null 2>&1; then echo "failed adjacent oracle was accepted" >&2 exit 1 fi jq '.ac_power = false | .power_state = "Now drawing from Battery Power"' \ "$temporary/h128post.json" > "$temporary/h128post-battery.json" if "$analyzer" "$temporary/c64.json" "$temporary/c128.json" \ "$temporary/prefill.json" "$temporary/oracle.json" fixture.gguf "$sha" \ "$temporary/gate.json" "$temporary/h64pre.json" "$temporary/h64post.json" \ "$temporary/h128pre.json" "$temporary/h128post-battery.json" \ >/dev/null 2>&1; then echo "battery-power score-tile8 receipt was accepted" >&2 exit 1 fi jq '.no_thermal_warning = false' "$temporary/h64pre.json" \ > "$temporary/h64pre-thermal.json" if "$analyzer" "$temporary/c64.json" "$temporary/c128.json" \ "$temporary/prefill.json" "$temporary/oracle.json" fixture.gguf "$sha" \ "$temporary/gate.json" "$temporary/h64pre-thermal.json" "$temporary/h64post.json" \ "$temporary/h128pre.json" "$temporary/h128post.json" \ >/dev/null 2>&1; then echo "thermal-warning score-tile8 receipt was accepted" >&2 exit 1 fi