85 lines
3.7 KiB
Bash
Executable file
85 lines
3.7 KiB
Bash
Executable file
#!/bin/sh
|
|
set -eu
|
|
|
|
[ "$#" -eq 2 ] || [ "$#" -eq 3 ] || {
|
|
echo "usage: $0 BASELINE_JSON CANDIDATE_JSON [OUTPUT_JSON]" >&2
|
|
exit 2
|
|
}
|
|
baseline=$1
|
|
candidate=$2
|
|
output=${3:-}
|
|
|
|
analyze() {
|
|
jq -n --slurpfile baseline "$baseline" --slurpfile candidate "$candidate" '
|
|
def mean($xs): $xs | add / length;
|
|
def sample_stddev($xs):
|
|
mean($xs) as $m |
|
|
(($xs | map((. - $m) * (. - $m)) | add) / (($xs | length) - 1)) | sqrt;
|
|
def cv_pct($xs): 100 * sample_stddev($xs) / mean($xs);
|
|
def improvement_pct($before; $after): 100 * (1 - mean($after) / mean($before));
|
|
|
|
$baseline[0] as $b | $candidate[0] as $c |
|
|
[
|
|
"schema_version", "engine", "workload", "shape", "query_heads", "kv_heads",
|
|
"head_dimension", "tokens", "live_tokens", "cache_capacity", "window",
|
|
"kv_cache_bytes", "score_scratch_bytes", "value_kernel",
|
|
"value_dimension_tile", "value_threadgroups", "value_partition_scratch_bytes",
|
|
"value_partitions", "iterations", "repetitions", "warmup", "dispatches"
|
|
] as $controls |
|
|
($controls | map(select($b[.] != $c[.]))) as $changed_controls |
|
|
if ($b.schema_version != 1 or $c.schema_version != 1 or
|
|
$b.workload != "synthetic-exact-shape-decode-attention-stage-profile" or
|
|
$c.workload != $b.workload or
|
|
$changed_controls != [] or
|
|
($b.score_token_tile | type) != "number" or
|
|
($c.score_token_tile | type) != "number" or
|
|
$b.score_token_tile < 1 or $c.score_token_tile < 1 or
|
|
$b.score_token_tile == $c.score_token_tile or
|
|
($b.gpu_seconds_samples | length) != $b.repetitions or
|
|
($c.gpu_seconds_samples | length) != $c.repetitions or
|
|
($b.stages.score.samples_seconds | length) != $b.repetitions or
|
|
($c.stages.score.samples_seconds | length) != $c.repetitions or
|
|
$b.repetitions < 3 or $c.repetitions < 3)
|
|
then error("decode-attention inputs are not a valid one-axis score-token-tile comparison")
|
|
else
|
|
improvement_pct($b.gpu_seconds_samples; $c.gpu_seconds_samples) as $total_gain |
|
|
improvement_pct($b.stages.score.samples_seconds; $c.stages.score.samples_seconds) as $score_gain |
|
|
cv_pct($b.gpu_seconds_samples) as $bt_cv |
|
|
cv_pct($c.gpu_seconds_samples) as $ct_cv |
|
|
cv_pct($b.stages.score.samples_seconds) as $bs_cv |
|
|
cv_pct($c.stages.score.samples_seconds) as $cs_cv |
|
|
([$bt_cv, $ct_cv, $bs_cv, $cs_cv] | max) as $noise |
|
|
{
|
|
schema_version: 1,
|
|
scope: "synthetic exact-shape decode-attention microbenchmark",
|
|
axis: {
|
|
name: "score_token_tile",
|
|
baseline: $b.score_token_tile,
|
|
candidate: $c.score_token_tile
|
|
},
|
|
controls: ($controls | map({key: ., value: $b[.]} ) | from_entries),
|
|
total_gpu: {
|
|
baseline_mean_seconds: mean($b.gpu_seconds_samples),
|
|
candidate_mean_seconds: mean($c.gpu_seconds_samples),
|
|
improvement_pct: $total_gain,
|
|
baseline_cv_pct: $bt_cv,
|
|
candidate_cv_pct: $ct_cv
|
|
},
|
|
score_stage: {
|
|
baseline_mean_seconds: mean($b.stages.score.samples_seconds),
|
|
candidate_mean_seconds: mean($c.stages.score.samples_seconds),
|
|
improvement_pct: $score_gain,
|
|
baseline_cv_pct: $bs_cv,
|
|
candidate_cv_pct: $cs_cv
|
|
},
|
|
observed_noise_floor_pct: $noise,
|
|
screening: {
|
|
threshold_passed: ($total_gain >= 3 and $total_gain > $noise),
|
|
promotion_eligible: false,
|
|
reason: "microbenchmark only; native correctness and authentic populated-model gates remain required"
|
|
}
|
|
}
|
|
end'
|
|
}
|
|
|
|
if [ -n "$output" ]; then analyze > "$output"; else analyze; fi
|