50 lines
2.2 KiB
Bash
Executable file
50 lines
2.2 KiB
Bash
Executable file
#!/bin/sh
|
|
set -eu
|
|
|
|
if [ "$#" -ne 2 ]; then
|
|
echo "usage: $0 ANALYZER FIXTURE_DIR" >&2
|
|
exit 1
|
|
fi
|
|
|
|
analyzer=$1
|
|
fixtures=$2
|
|
report=$(mktemp /private/tmp/gem-benchmark-report.XXXXXX)
|
|
noisy=$(mktemp /private/tmp/gem-benchmark-noisy.XXXXXX)
|
|
mismatch=$(mktemp /private/tmp/gem-benchmark-mismatch.XXXXXX)
|
|
kv=$(mktemp /private/tmp/gem-benchmark-kv.XXXXXX)
|
|
attention=$(mktemp /private/tmp/gem-benchmark-attention.XXXXXX)
|
|
trap 'rm -f "$report" "$noisy" "$mismatch" "$kv" "$attention"' EXIT
|
|
|
|
"$analyzer" "$fixtures/benchmark-baseline.json" \
|
|
"$fixtures/benchmark-candidate.json" "$report"
|
|
jq -e '.comparability.comparable == true' "$report" >/dev/null
|
|
jq -e '.promotion.eligible == true' "$report" >/dev/null
|
|
jq -e '.metrics.tg.delta_pct > 4.9 and .metrics.tg.delta_pct < 5.1' "$report" >/dev/null
|
|
|
|
jq 'map(if .n_gen > 0 then .stddev_ts = 3.0 else . end)' \
|
|
"$fixtures/benchmark-candidate.json" > "$noisy"
|
|
"$analyzer" "$fixtures/benchmark-baseline.json" "$noisy" "$report"
|
|
jq -e '.promotion.eligible == false' "$report" >/dev/null
|
|
jq -e '.promotion.primary_threshold_passed == false' "$report" >/dev/null
|
|
|
|
jq 'map(.model_filename = "other.gguf")' \
|
|
"$fixtures/benchmark-candidate.json" > "$mismatch"
|
|
"$analyzer" "$fixtures/benchmark-baseline.json" "$mismatch" "$report"
|
|
jq -e '.comparability.comparable == false' "$report" >/dev/null
|
|
"$analyzer" --allow-model-change "$fixtures/benchmark-baseline.json" "$mismatch" "$report"
|
|
jq -e '.comparability.comparable == true' "$report" >/dev/null
|
|
|
|
jq 'map(.type_k = "f32" | .type_v = "f32")' \
|
|
"$fixtures/benchmark-baseline.json" > "$kv"
|
|
"$analyzer" "$fixtures/benchmark-baseline.json" "$kv" "$report"
|
|
jq -e '.comparability.comparable == true' "$report" >/dev/null
|
|
jq -e '.comparability.tuning_axis_differences ==
|
|
[{"field":"kv_type","baseline":"f16","candidate":"f32"}]' "$report" >/dev/null
|
|
|
|
jq 'map(.attention_kernel = "blocked")' \
|
|
"$fixtures/benchmark-baseline.json" > "$attention"
|
|
"$analyzer" "$fixtures/benchmark-baseline.json" "$attention" "$report"
|
|
jq -e '.comparability.comparable == true' "$report" >/dev/null
|
|
jq -e '.comparability.tuning_axis_differences ==
|
|
[{"field":"attention_kernel","baseline":"parallel","candidate":"blocked"}]' \
|
|
"$report" >/dev/null
|