#!/bin/sh set -eu [ "$#" -eq 2 ] || [ "$#" -eq 3 ] || { echo "usage: $0 BASELINE_JSON CANDIDATE_JSON [OUTPUT_JSON]" >&2 exit 2 } baseline=$1 candidate=$2 output=${3:-} analyze() { jq -n --slurpfile baseline "$baseline" --slurpfile candidate "$candidate" ' def mean($xs): $xs | add / length; def sample_stddev($xs): mean($xs) as $m | (($xs | map((. - $m) * (. - $m)) | add) / (($xs | length) - 1)) | sqrt; def cv_pct($xs): 100 * sample_stddev($xs) / mean($xs); def improvement_pct($before; $after): 100 * (1 - mean($after) / mean($before)); $baseline[0] as $b | $candidate[0] as $c | [ "schema_version", "engine", "workload", "shape", "query_heads", "kv_heads", "head_dimension", "tokens", "live_tokens", "cache_capacity", "window", "kv_cache_bytes", "score_scratch_bytes", "value_kernel", "value_dimension_tile", "value_threadgroups", "value_partition_scratch_bytes", "value_partitions", "iterations", "repetitions", "warmup", "dispatches" ] as $controls | ($controls | map(select($b[.] != $c[.]))) as $changed_controls | if ($b.schema_version != 1 or $c.schema_version != 1 or $b.workload != "synthetic-exact-shape-decode-attention-stage-profile" or $c.workload != $b.workload or $changed_controls != [] or ($b.score_token_tile | type) != "number" or ($c.score_token_tile | type) != "number" or $b.score_token_tile < 1 or $c.score_token_tile < 1 or $b.score_token_tile == $c.score_token_tile or ($b.gpu_seconds_samples | length) != $b.repetitions or ($c.gpu_seconds_samples | length) != $c.repetitions or ($b.stages.score.samples_seconds | length) != $b.repetitions or ($c.stages.score.samples_seconds | length) != $c.repetitions or $b.repetitions < 3 or $c.repetitions < 3) then error("decode-attention inputs are not a valid one-axis score-token-tile comparison") else improvement_pct($b.gpu_seconds_samples; $c.gpu_seconds_samples) as $total_gain | improvement_pct($b.stages.score.samples_seconds; $c.stages.score.samples_seconds) as $score_gain | cv_pct($b.gpu_seconds_samples) as $bt_cv | cv_pct($c.gpu_seconds_samples) as $ct_cv | cv_pct($b.stages.score.samples_seconds) as $bs_cv | cv_pct($c.stages.score.samples_seconds) as $cs_cv | ([$bt_cv, $ct_cv, $bs_cv, $cs_cv] | max) as $noise | { schema_version: 1, scope: "synthetic exact-shape decode-attention microbenchmark", axis: { name: "score_token_tile", baseline: $b.score_token_tile, candidate: $c.score_token_tile }, controls: ($controls | map({key: ., value: $b[.]} ) | from_entries), total_gpu: { baseline_mean_seconds: mean($b.gpu_seconds_samples), candidate_mean_seconds: mean($c.gpu_seconds_samples), improvement_pct: $total_gain, baseline_cv_pct: $bt_cv, candidate_cv_pct: $ct_cv }, score_stage: { baseline_mean_seconds: mean($b.stages.score.samples_seconds), candidate_mean_seconds: mean($c.stages.score.samples_seconds), improvement_pct: $score_gain, baseline_cv_pct: $bs_cv, candidate_cv_pct: $cs_cv }, observed_noise_floor_pct: $noise, screening: { threshold_passed: ($total_gain >= 3 and $total_gain > $noise), promotion_eligible: false, reason: "microbenchmark only; native correctness and authentic populated-model gates remain required" } } end' } if [ -n "$output" ]; then analyze > "$output"; else analyze; fi