305 lines
11 KiB
YAML
305 lines
11 KiB
YAML
name: Weekly Benchmarks
|
|
|
|
on:
|
|
schedule:
|
|
- cron: '0 7 * * 0' # 07:00 UTC every Sunday
|
|
workflow_dispatch:
|
|
|
|
permissions:
|
|
contents: read
|
|
|
|
jobs:
|
|
check-commits:
|
|
name: Check for new commits
|
|
runs-on: ubuntu-latest
|
|
outputs:
|
|
has_new_commits: ${{ steps.check.outputs.has_new_commits }}
|
|
steps:
|
|
- name: Check if there are new commits in last 7 days
|
|
id: check
|
|
uses: actions/github-script@v7
|
|
with:
|
|
script: |
|
|
const weekMs = 7 * 24 * 60 * 60 * 1000;
|
|
const since = new Date(Date.now() - weekMs).toISOString();
|
|
const { data } = await github.rest.repos.listCommits({
|
|
owner: context.repo.owner,
|
|
repo: context.repo.repo,
|
|
sha: 'main',
|
|
since,
|
|
per_page: 1,
|
|
});
|
|
const has = data.length > 0;
|
|
core.setOutput('has_new_commits', has ? 'true' : 'false');
|
|
core.info(
|
|
has
|
|
? `Found ${data.length} new commit(s) since ${since}`
|
|
: 'No new commits in last 7 days, skipping benchmarks'
|
|
);
|
|
|
|
nightly-linux:
|
|
needs: check-commits
|
|
if: needs.check-commits.outputs.has_new_commits == 'true' || github.event_name == 'workflow_dispatch'
|
|
name: Benchmarks (Linux x86_64)
|
|
runs-on: ubuntu-24.04
|
|
timeout-minutes: 135
|
|
env:
|
|
BUILD_DIR: builddir
|
|
YAMS_DISABLE_FAISS: 1
|
|
# Opt-in switch to run the daemon warm-latency bench (off by default)
|
|
RUN_DAEMON_WARM_BENCH: 'false'
|
|
steps:
|
|
- name: Checkout
|
|
uses: actions/checkout@v5
|
|
with:
|
|
fetch-depth: 0
|
|
submodules: false
|
|
|
|
- name: Initialize required submodules
|
|
shell: bash
|
|
run: |
|
|
# Force HTTPS for GitHub to handle SSH submodule URLs in CI
|
|
git config --global url."https://github.com/".insteadOf "git@github.com:"
|
|
git submodule update --init --depth 1 third_party/sqlite-vec-cpp
|
|
git submodule update --init --depth 1 third_party/simeon
|
|
git submodule update --init --depth 1 third_party/symspell
|
|
|
|
- name: Install system dependencies
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
sudo apt-get update
|
|
sudo apt-get install -y --no-install-recommends pkg-config meson ninja-build cmake python3-pip libboost-all-dev libomp-18-dev ccache
|
|
|
|
- name: Install Conan
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
PYTOOLS_DIR="${RUNNER_TEMP}/yams-python-tools"
|
|
python3 -m venv "$PYTOOLS_DIR"
|
|
"$PYTOOLS_DIR/bin/python" -m pip install --upgrade pip
|
|
"$PYTOOLS_DIR/bin/python" -m pip install --upgrade conan
|
|
echo "$PYTOOLS_DIR/bin" >> "$GITHUB_PATH"
|
|
export PATH="$PYTOOLS_DIR/bin:$PATH"
|
|
conan --version
|
|
|
|
# Detect Conan profile (required for setup.sh)
|
|
conan profile detect --force || true
|
|
|
|
- name: Cache Conan
|
|
uses: actions/cache@v4
|
|
with:
|
|
path: |
|
|
~/.conan2/p
|
|
~/.conan2/r
|
|
~/.conan2/recipes
|
|
~/.conan2/metadata
|
|
~/.conan2/s
|
|
key: nightly-conan2-linux-${{ hashFiles('conanfile.py', 'meson.build', 'meson_options.txt') }}
|
|
restore-keys: |
|
|
nightly-conan2-linux-
|
|
|
|
- name: Cache ccache
|
|
uses: actions/cache@v4
|
|
with:
|
|
path: ~/.cache/ccache
|
|
key: nightly-ccache-linux-${{ hashFiles('conanfile.py', 'meson.build', 'meson_options.txt') }}
|
|
restore-keys: |
|
|
nightly-ccache-linux-
|
|
|
|
- name: Setup ccache env
|
|
shell: bash
|
|
run: |
|
|
echo "CCACHE_DIR=$HOME/.cache/ccache" >> $GITHUB_ENV
|
|
echo "CCACHE_BASEDIR=$GITHUB_WORKSPACE" >> $GITHUB_ENV
|
|
echo "CCACHE_COMPRESS=1" >> $GITHUB_ENV
|
|
echo "CCACHE_MAXSIZE=5G" >> $GITHUB_ENV
|
|
echo "CCACHE_SLOPPINESS=time_macros,include_file_mtime,include_file_ctime,file_macro" >> $GITHUB_ENV
|
|
ccache -z >/dev/null 2>&1 || true
|
|
|
|
- name: Configure (Release) and Build
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
rm -rf "$BUILD_DIR"
|
|
|
|
# Keep AF_UNIX socket path short
|
|
export XDG_RUNTIME_DIR="/run/user/$(id -u)"
|
|
mkdir -p "$XDG_RUNTIME_DIR" || true
|
|
|
|
# Use setup.sh with CI environment variables for consistent configuration
|
|
export YAMS_CONAN_HOST_PROFILE="./conan/profiles/host-linux-clang"
|
|
export YAMS_CONAN_ARCH="x86_64"
|
|
# Ensure Conan and Meson both opt into benchmark targets.
|
|
export YAMS_CONAN_EXTRA_OPTIONS="-o build_tests=True -o build_benchmarks=True"
|
|
export YAMS_EXTRA_MESON_FLAGS="-Dbuild-tests=true -Dbuild-benchmarks=true -Denable-onnx=disabled"
|
|
export YAMS_ENABLE_MOBILE_BINDINGS=false
|
|
|
|
./setup.sh Debug
|
|
CONAN_BUILD_ENV=""
|
|
for candidate in \
|
|
"$BUILD_DIR/build-debug/conan/conanbuild.sh" \
|
|
"$BUILD_DIR/conan/conanbuild.sh"
|
|
do
|
|
if [ -f "$candidate" ]; then
|
|
CONAN_BUILD_ENV="$candidate"
|
|
break
|
|
fi
|
|
done
|
|
if [ -n "$CONAN_BUILD_ENV" ]; then
|
|
set +u
|
|
source "$CONAN_BUILD_ENV"
|
|
set -u
|
|
fi
|
|
meson compile -C "$BUILD_DIR"
|
|
|
|
- name: Run Bench Test Suite
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
meson test -C "$BUILD_DIR" --suite bench -t 7 --no-rebuild
|
|
|
|
- name: Capture failing test diagnostics
|
|
if: failure()
|
|
shell: bash
|
|
run: |
|
|
echo "### Failing Bench Diagnostics" >> "$GITHUB_STEP_SUMMARY"
|
|
LOG="$BUILD_DIR/meson-logs/testlog.txt"
|
|
if [ -f "$LOG" ]; then
|
|
echo '```' >> "$GITHUB_STEP_SUMMARY"
|
|
grep -E 'FAIL|killed|SIGABRT|Signal|abort|Error|error:' "$LOG" | head -80 >> "$GITHUB_STEP_SUMMARY" || true
|
|
echo '```' >> "$GITHUB_STEP_SUMMARY"
|
|
fi
|
|
echo "### ccache stats" >> "$GITHUB_STEP_SUMMARY"
|
|
echo '```' >> "$GITHUB_STEP_SUMMARY"
|
|
ccache -s 2>/dev/null | head -10 >> "$GITHUB_STEP_SUMMARY" || true
|
|
echo '```' >> "$GITHUB_STEP_SUMMARY"
|
|
|
|
- name: Run Daemon Warm-Latency Bench (opt-in)
|
|
if: env.RUN_DAEMON_WARM_BENCH == 'true'
|
|
shell: bash
|
|
env:
|
|
YAMS_ENABLE_DAEMON_BENCH: '1'
|
|
run: |
|
|
set -euo pipefail
|
|
echo "Running bench_daemon_warm_latency (opt-in)..."
|
|
meson test -C "$BUILD_DIR" bench_daemon_warm_latency --suite bench --no-rebuild -t 5
|
|
|
|
- name: Run Bus Micro-benchmark
|
|
id: bench
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
mkdir -p "$BUILD_DIR/bench_results"
|
|
echo "Running yams_bus_bench (SPSC default)..."
|
|
RAW=$("$BUILD_DIR"/src/benchmarks/yams_bus_bench)
|
|
echo "$RAW"
|
|
# Convert key=value pairs to JSON using bash
|
|
ts=$(date -Is)
|
|
json='{"timestamp":"'"$ts"'","bench":{'
|
|
first=1
|
|
for kv in $RAW; do
|
|
case "$kv" in
|
|
*=*)
|
|
key="${kv%%=*}"
|
|
val="${kv#*=}"
|
|
if [ $first -eq 0 ]; then json="$json,"; else first=0; fi
|
|
json="$json\"$key\":\"$val\""
|
|
;;
|
|
esac
|
|
done
|
|
json="$json}}"
|
|
printf '%s\n' "$json" > "$BUILD_DIR/bench_results/bus_bench.json"
|
|
echo "Wrote bench_results/bus_bench.json"
|
|
|
|
- name: Run API Benchmarks
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
echo "Running yams_api_benchmarks..."
|
|
( cd "$BUILD_DIR" && ./tests/benchmarks/yams_api_benchmarks ) || exit 1
|
|
|
|
- name: Run Search Benchmarks
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
echo "Running yams_search_benchmarks..."
|
|
( cd "$BUILD_DIR" && ./tests/benchmarks/yams_search_benchmarks ) || exit 1
|
|
|
|
- name: Run Tree Diff Benchmarks (PBI-043)
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
echo "Running tree_diff_benchmarks..."
|
|
( cd "$BUILD_DIR" && ./tests/benchmarks/tree_diff_benchmarks ) || exit 1
|
|
|
|
- name: Regression Check (API)
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
python3 tests/scripts/check_regression.py \
|
|
"$BUILD_DIR/bench_results/api_benchmarks.json" \
|
|
tests/benchmarks/baseline/api_benchmarks.baseline.json \
|
|
--threshold 0.5 \
|
|
--output "$BUILD_DIR/bench_results/api_benchmarks_regression.json" \
|
|
--fail-on-regression
|
|
|
|
- name: Regression Check (Search)
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
python3 tests/scripts/check_regression.py \
|
|
"$BUILD_DIR/bench_results/search_benchmarks.json" \
|
|
tests/benchmarks/baseline/search_benchmarks.baseline.json \
|
|
--threshold 0.5 \
|
|
--output "$BUILD_DIR/bench_results/search_benchmarks_regression.json" \
|
|
--fail-on-regression
|
|
|
|
- name: Regression Check (Tree Diff - PBI-043)
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
python3 tests/scripts/check_regression.py \
|
|
"$BUILD_DIR/bench_results/tree_diff_benchmark_results.json" \
|
|
tests/benchmarks/baseline/tree_diff_benchmarks.baseline.json \
|
|
--threshold 0.5 \
|
|
--output "$BUILD_DIR/bench_results/tree_diff_benchmarks_regression.json" \
|
|
--fail-on-regression
|
|
|
|
- name: Summarize Benchmarks in Job Summary
|
|
if: always()
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
SUMMARY_FILE="$GITHUB_STEP_SUMMARY"
|
|
echo "### Bench Results" >> "$SUMMARY_FILE"
|
|
if [ -d "$BUILD_DIR/bench_results" ] && ls "$BUILD_DIR/bench_results"/*.json >/dev/null 2>&1; then
|
|
for json in "$BUILD_DIR/bench_results"/*.json; do
|
|
echo >> "$SUMMARY_FILE"
|
|
echo "#### $(basename "$json")" >> "$SUMMARY_FILE"
|
|
echo '```json' >> "$SUMMARY_FILE"
|
|
sed -e 's/[[:cntrl:]]//g' "$json" | head -c 120000 >> "$SUMMARY_FILE" || true
|
|
echo >> "$SUMMARY_FILE"
|
|
echo '```' >> "$SUMMARY_FILE"
|
|
done
|
|
else
|
|
echo "_No bench_results found._" >> "$SUMMARY_FILE"
|
|
fi
|
|
|
|
- name: Upload Benchmark Artifacts
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: nightly-benchmarks-${{ github.run_number }}
|
|
path: |
|
|
${{ env.BUILD_DIR }}/bench_results/bus_bench.json
|
|
${{ env.BUILD_DIR }}/bench_results/api_benchmarks.json
|
|
${{ env.BUILD_DIR }}/bench_results/api_benchmark_report.json
|
|
${{ env.BUILD_DIR }}/bench_results/api_benchmark_report.md
|
|
${{ env.BUILD_DIR }}/bench_results/search_benchmarks.json
|
|
${{ env.BUILD_DIR }}/bench_results/search_benchmark_report.json
|
|
${{ env.BUILD_DIR }}/bench_results/search_benchmark_report.md
|
|
${{ env.BUILD_DIR }}/bench_results/api_benchmarks_regression.json
|
|
${{ env.BUILD_DIR }}/bench_results/search_benchmarks_regression.json
|
|
${{ env.BUILD_DIR }}/bench_results/tree_diff_benchmark_results.json
|
|
${{ env.BUILD_DIR }}/bench_results/tree_diff_benchmarks_regression.json
|
|
retention-days: 7
|