44 lines
1.4 KiB
Bash
Executable file
44 lines
1.4 KiB
Bash
Executable file
#!/bin/sh
|
|
set -eu
|
|
|
|
if [ "$#" -ne 2 ]; then
|
|
echo "usage: $0 ANALYZER PROFILE_FIXTURE" >&2
|
|
exit 2
|
|
fi
|
|
|
|
analyzer=$1
|
|
fixture=$2
|
|
baseline=$(mktemp /private/tmp/gem-decode-attention-baseline.XXXXXX)
|
|
candidate=$(mktemp /private/tmp/gem-decode-attention-candidate.XXXXXX)
|
|
invalid=$(mktemp /private/tmp/gem-decode-attention-invalid.XXXXXX)
|
|
report=$(mktemp /private/tmp/gem-decode-attention-report.XXXXXX)
|
|
trap 'rm -f "$baseline" "$candidate" "$invalid" "$report"' EXIT INT TERM HUP
|
|
|
|
jq '.score_token_tile = 1' "$fixture" > "$baseline"
|
|
jq '
|
|
.score_token_tile = 8 |
|
|
.gpu_seconds_samples |= map(. * 0.9) |
|
|
.stages.score.samples_seconds |= map(. * 0.8)
|
|
' "$fixture" > "$candidate"
|
|
|
|
"$analyzer" "$baseline" "$candidate" "$report"
|
|
jq -e '
|
|
.axis.name == "score_token_tile" and
|
|
.axis.baseline == 1 and .axis.candidate == 8 and
|
|
.total_gpu.improvement_pct > 9.99 and
|
|
.score_stage.improvement_pct > 19.99 and
|
|
.screening.threshold_passed == true and
|
|
.screening.promotion_eligible == false
|
|
' "$report" >/dev/null
|
|
|
|
jq '.score_token_tile = 8 | .tokens -= 1' "$fixture" > "$invalid"
|
|
if "$analyzer" "$baseline" "$invalid" "$report" >/dev/null 2>&1; then
|
|
echo "mismatched decode-attention control was accepted" >&2
|
|
exit 1
|
|
fi
|
|
|
|
jq '.score_token_tile = 1' "$fixture" > "$invalid"
|
|
if "$analyzer" "$baseline" "$invalid" "$report" >/dev/null 2>&1; then
|
|
echo "unchanged score-token tile was accepted" >&2
|
|
exit 1
|
|
fi
|