#!/bin/sh
set -eu

if [ "$#" -ne 2 ]; then
    echo "usage: $0 ANALYZER FIXTURE_DIR" >&2
    exit 1
fi

analyzer=$1
fixtures=$2
report=$(mktemp /private/tmp/gem-benchmark-report.XXXXXX)
noisy=$(mktemp /private/tmp/gem-benchmark-noisy.XXXXXX)
mismatch=$(mktemp /private/tmp/gem-benchmark-mismatch.XXXXXX)
kv=$(mktemp /private/tmp/gem-benchmark-kv.XXXXXX)
attention=$(mktemp /private/tmp/gem-benchmark-attention.XXXXXX)
trap 'rm -f "$report" "$noisy" "$mismatch" "$kv" "$attention"' EXIT

"$analyzer" "$fixtures/benchmark-baseline.json" \
    "$fixtures/benchmark-candidate.json" "$report"
jq -e '.comparability.comparable == true' "$report" >/dev/null
jq -e '.promotion.eligible == true' "$report" >/dev/null
jq -e '.metrics.tg.delta_pct > 4.9 and .metrics.tg.delta_pct < 5.1' "$report" >/dev/null

jq 'map(if .n_gen > 0 then .stddev_ts = 3.0 else . end)' \
    "$fixtures/benchmark-candidate.json" > "$noisy"
"$analyzer" "$fixtures/benchmark-baseline.json" "$noisy" "$report"
jq -e '.promotion.eligible == false' "$report" >/dev/null
jq -e '.promotion.primary_threshold_passed == false' "$report" >/dev/null

jq 'map(.model_filename = "other.gguf")' \
    "$fixtures/benchmark-candidate.json" > "$mismatch"
"$analyzer" "$fixtures/benchmark-baseline.json" "$mismatch" "$report"
jq -e '.comparability.comparable == false' "$report" >/dev/null
"$analyzer" --allow-model-change "$fixtures/benchmark-baseline.json" "$mismatch" "$report"
jq -e '.comparability.comparable == true' "$report" >/dev/null

jq 'map(.type_k = "f32" | .type_v = "f32")' \
    "$fixtures/benchmark-baseline.json" > "$kv"
"$analyzer" "$fixtures/benchmark-baseline.json" "$kv" "$report"
jq -e '.comparability.comparable == true' "$report" >/dev/null
jq -e '.comparability.tuning_axis_differences ==
       [{"field":"kv_type","baseline":"f16","candidate":"f32"}]' "$report" >/dev/null

jq 'map(.attention_kernel = "blocked")' \
    "$fixtures/benchmark-baseline.json" > "$attention"
"$analyzer" "$fixtures/benchmark-baseline.json" "$attention" "$report"
jq -e '.comparability.comparable == true' "$report" >/dev/null
jq -e '.comparability.tuning_axis_differences ==
       [{"field":"attention_kernel","baseline":"parallel","candidate":"blocked"}]' \
    "$report" >/dev/null