d398556ee5* test(search,daemon,storage): assert behaviour instead of frozen literals Several suites encoded exact internal values that legitimate tuning and ranking changes are allowed to move: SCIENTIFIC-era profile weights and rrfK values, corpus-statistic figures, and hardcoded timeout/cancellation numbers. They failed on every platform after those changes landed, so the real behaviour regressions they should catch were hidden behind noise. Replace the frozen literals with contract assertions: weights are finite, non-negative and normalize, state selection still holds, capability-gated sources stay zeroed, and hybrid results keep matching canonical hash/path sets. Deadlines are not widened and the daemon paths they cover are unchanged. Also add independent corpus and adapter oracles, deterministic collector cancellation and late-result gates, and synchronize ServiceManager shutdown observation, so the tests fail for the behaviour they name. The portability allowlist keeps its two reviewed raw test-env reads; only their line numbers moved. Signed-off-by: trvon <git@trevon.dev> * ci(tests): summarise Meson failures without flooding the log A failing test run printed the whole Meson suite output, so the actual failing test names and their assertion messages were buried and the uploaded logs were dominated by passing-test chatter. Add a bounded reporter that extracts the failing test names plus their surrounding diagnostics, keeps an overall cap on emitted lines, and is unit-tested against representative Meson output. Wire it into the Tests workflow so failures stay readable. Signed-off-by: trvon <git@trevon.dev> * build(deb): validate candidate identity, mapping and upgrade ordering The Debian packaging scripts took filenames and versions on trust, so a misnamed candidate or a wrong package ordering could pass validation while the real install path was never exercised. Fail closed on candidate identity, version and architecture, verify the package-to-source mapping, seed the legacy-only layout, wait on the actual services, and assert install/upgrade/remove ordering for both Debian and RPM in an offline container. Cases that cannot be proven on a host are reported as skipped rather than silently passing. Signed-off-by: trvon <git@trevon.dev> --------- Signed-off-by: trvon <git@trevon.dev>
861 lines
36 KiB
YAML
861 lines
36 KiB
YAML
name: Tests
|
|
|
|
on:
|
|
push:
|
|
branches: [main, master, experimental]
|
|
pull_request:
|
|
branches: [main, experimental]
|
|
|
|
permissions:
|
|
contents: read
|
|
|
|
concurrency:
|
|
group: ${{ github.workflow }}-${{ github.ref }}
|
|
# Keep main-branch history intact so slower matrix lanes still publish evidence.
|
|
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
|
|
|
jobs:
|
|
warm:
|
|
uses: ./.github/workflows/matrix-warm.yml
|
|
with:
|
|
build_type: Debug
|
|
enable_onnx: false
|
|
|
|
quality:
|
|
name: Static Analysis
|
|
runs-on: ubuntu-24.04
|
|
timeout-minutes: 20
|
|
permissions:
|
|
contents: read
|
|
steps:
|
|
- name: Checkout
|
|
uses: actions/checkout@v4
|
|
with:
|
|
fetch-depth: 0
|
|
submodules: false
|
|
|
|
- name: Initialize required submodules
|
|
shell: bash
|
|
run: |
|
|
git submodule update --init --depth 1 third_party/sqlite-vec-cpp || true
|
|
git submodule update --init --depth 1 third_party/simeon || true
|
|
git submodule update --init --depth 1 third_party/symspell || true
|
|
|
|
- name: Test package version and upgrade validation scripts
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
python3 tests/scripts/test_package_upgrade_validation.py -v
|
|
bash -n scripts/build-deb.sh \
|
|
scripts/local-ci/package-validate-checks.sh \
|
|
scripts/local-ci/package-validate.sh
|
|
|
|
- name: Install static analysis tools
|
|
shell: bash
|
|
run: |
|
|
bash .github/scripts/apt-get-with-retry.sh update
|
|
bash .github/scripts/apt-get-with-retry.sh \
|
|
install -y --no-install-recommends cppcheck clang-tidy
|
|
|
|
- name: Run cppcheck (CI profile, changed lines)
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
BASE=""
|
|
if [[ "${{ github.event_name }}" == "pull_request" ]]; then
|
|
BASE="${{ github.event.pull_request.base.sha }}"
|
|
elif [[ -n "${{ github.event.before }}" && "${{ github.event.before }}" != "0000000000000000000000000000000000000000" ]]; then
|
|
BASE="${{ github.event.before }}"
|
|
elif git rev-parse HEAD^ >/dev/null 2>&1; then
|
|
BASE="HEAD^"
|
|
fi
|
|
|
|
# Push events can report a `before` SHA that is no longer reachable
|
|
# after a force-push/rewrite. In that case, fall back to the checked
|
|
# out parent instead of failing before cppcheck runs.
|
|
if [[ -n "$BASE" ]] && ! git rev-parse --verify "${BASE}^{commit}" >/dev/null 2>&1; then
|
|
echo "Diff base '$BASE' is not available in this checkout; falling back to HEAD^."
|
|
if git rev-parse HEAD^ >/dev/null 2>&1; then
|
|
BASE="HEAD^"
|
|
else
|
|
BASE=""
|
|
fi
|
|
fi
|
|
|
|
if [[ -n "$BASE" ]]; then
|
|
scripts/lint.sh cppcheck --diff "$BASE"
|
|
else
|
|
scripts/lint.sh cppcheck
|
|
fi
|
|
|
|
- name: Run clang-tidy on changed files (PR only)
|
|
if: github.event_name == 'pull_request'
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
COMPILE_DB_DIR=""
|
|
if [ -f build/debug/compile_commands.json ]; then
|
|
COMPILE_DB_DIR="build/debug"
|
|
elif [ -f compile_commands.json ]; then
|
|
COMPILE_DB_DIR="."
|
|
else
|
|
echo "No compile_commands.json found; skipping clang-tidy." | tee clang-tidy-pr.txt
|
|
exit 0
|
|
fi
|
|
|
|
mapfile -t CHANGED_FILES < <(git diff --name-only "origin/${{ github.base_ref }}...HEAD" \
|
|
| grep -E '\.(cpp|h|hpp|cc|cxx)$' \
|
|
| grep -v -E '(third_party|external|subprojects|\.pb\.(h|cc))' \
|
|
|| true)
|
|
if [ "${#CHANGED_FILES[@]}" -eq 0 ]; then
|
|
echo "No C++ files changed in this PR."
|
|
exit 0
|
|
fi
|
|
printf 'Running clang-tidy on:\n'
|
|
printf ' %s\n' "${CHANGED_FILES[@]}"
|
|
clang-tidy \
|
|
-p "$COMPILE_DB_DIR" \
|
|
--warnings-as-errors='bugprone-*,clang-analyzer-*,concurrency-*' \
|
|
"${CHANGED_FILES[@]}" \
|
|
2>&1 | tee clang-tidy-pr.txt
|
|
|
|
- name: Upload clang-tidy results
|
|
if: github.event_name == 'pull_request' && always()
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: clang-tidy-pr-${{ github.run_number }}
|
|
path: clang-tidy-pr.txt
|
|
if-no-files-found: ignore
|
|
retention-days: 7
|
|
|
|
tests:
|
|
needs: warm
|
|
if: ${{ success() }}
|
|
name: Tests (${{ matrix.suffix }})
|
|
env:
|
|
BUILD_DIR: build/debug
|
|
YAMS_CONAN_COMPILER_VERSION: ${{ matrix.conan_compiler_version || '' }}
|
|
YAMS_MSBUILD_VS_VERSION: ${{ matrix.msbuild_vs_version || '' }}
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
include:
|
|
- suffix: linux-x86_64
|
|
runs_on: ubuntu-24.04
|
|
conan_host_profile: ./conan/profiles/host-linux-clang
|
|
conan_arch: x86_64
|
|
needs_rosetta: false
|
|
build_type: Debug
|
|
extra_meson_flags: "--buildtype=debug -Dbuild-tests=true -Denable-onnx=disabled -Dtest-timeout-scale=2 -Denable-bench-tests=false -Dbuild-benchmarks=false -Denable-dcheck=true"
|
|
- suffix: linux-arm64
|
|
runs_on: ubuntu-24.04-arm
|
|
conan_host_profile: ./conan/profiles/host-linux-clang
|
|
conan_arch: armv8
|
|
needs_rosetta: false
|
|
build_type: Debug
|
|
extra_meson_flags: "--buildtype=debug -Dbuild-tests=true -Denable-onnx=disabled -Dtest-timeout-scale=2 -Denable-bench-tests=false -Dbuild-benchmarks=false -Denable-dcheck=true"
|
|
- suffix: macos-arm64
|
|
runs_on: macos-15
|
|
conan_host_profile: ./conan/profiles/host-macos-apple-clang
|
|
conan_arch: armv8
|
|
needs_rosetta: false
|
|
build_type: Debug
|
|
extra_meson_flags: "--buildtype=debug -Dbuild-tests=true -Denable-onnx=disabled -Dtest-timeout-scale=2 -Denable-dcheck=true"
|
|
- suffix: windows-x86_64
|
|
runs_on: windows-2025-vs2026
|
|
conan_host_profile: ./conan/profiles/host-windows-msvc
|
|
conan_arch: x86_64
|
|
needs_rosetta: false
|
|
conan_compiler_version: "195"
|
|
msbuild_vs_version: "18"
|
|
build_type: Debug
|
|
extra_meson_flags: "--buildtype=debug -Dbuild-tests=true -Denable-onnx=disabled -Dtest-timeout-scale=2 -Denable-bench-tests=false -Dbuild-benchmarks=false -Denable-dcheck=true"
|
|
- suffix: linux-x86_64-asan
|
|
runs_on: ubuntu-24.04
|
|
conan_host_profile: ./conan/profiles/host-linux-clang
|
|
conan_arch: x86_64
|
|
needs_rosetta: false
|
|
build_type: Debug
|
|
extra_meson_flags: "--buildtype=debug -Dbuild-tests=true -Denable-onnx=disabled -Denable-asan=true -Db_sanitize=address -Db_lundef=false -Dtest-timeout-scale=2 -Denable-bench-tests=false -Dbuild-benchmarks=false -Denable-dcheck=true"
|
|
- suffix: linux-x86_64-ubsan
|
|
runs_on: ubuntu-24.04
|
|
conan_host_profile: ./conan/profiles/host-linux-clang
|
|
conan_arch: x86_64
|
|
needs_rosetta: false
|
|
build_type: Debug
|
|
extra_meson_flags: "--buildtype=debug -Dbuild-tests=true -Denable-onnx=disabled -Db_sanitize=undefined -Db_lundef=false -Dtest-timeout-scale=2 -Denable-dcheck=true"
|
|
# Host-only mobile SDK lane: builds libyams_mobile plus its Catch2 ABI tests and
|
|
# checks the exported C surface. It proves the Linux host build, not that an
|
|
# Android .so or iOS XCFramework loads on a device.
|
|
- suffix: linux-x86_64-mobile
|
|
runs_on: ubuntu-24.04
|
|
conan_host_profile: ./conan/profiles/host-linux-clang
|
|
conan_arch: x86_64
|
|
needs_rosetta: false
|
|
build_type: Debug
|
|
mobile_bindings: true
|
|
extra_meson_flags: "--buildtype=debug -Dbuild-tests=true -Denable-onnx=disabled -Denable-mobile-tests=true -Dtest-timeout-scale=2 -Denable-bench-tests=false -Dbuild-benchmarks=false -Denable-dcheck=true"
|
|
# TSAN intentionally excluded from CI: boost::asio strand destruction triggers
|
|
# a heap-use-after-free that TSAN detects but cannot be suppressed via tsan.suppressions.
|
|
# Run TSAN locally with: meson setup build/debug-tsan -Db_sanitize=thread
|
|
runs-on: ${{ matrix.runs_on }}
|
|
# Windows debug builds can consume most of the previous 100-minute budget before tests start.
|
|
timeout-minutes: 180
|
|
steps:
|
|
- name: Checkout
|
|
uses: actions/checkout@v4
|
|
with:
|
|
fetch-depth: 0
|
|
submodules: false # Manual submodule init below (some use SSH URLs)
|
|
|
|
- name: Skip non-linux-x86_64 matrix entries under act
|
|
if: github.actor == 'nektos/act' && matrix.suffix != 'linux-x86_64'
|
|
shell: bash
|
|
run: |
|
|
echo "Skipping matrix entry '${{ matrix.suffix }}' under act."
|
|
echo "Act provides Linux container execution only; keep local runs deterministic on linux-x86_64."
|
|
exit 0
|
|
|
|
- name: Free disk space (Linux)
|
|
if: runner.os == 'Linux'
|
|
uses: jlumbroso/free-disk-space@v1.3.1
|
|
with:
|
|
tool-cache: false # Keep tools we might need
|
|
android: true # ~14GB savings
|
|
dotnet: true # ~2.7GB savings
|
|
haskell: true # Haskell runtime
|
|
large-packages: false # Keep - might need some of these
|
|
docker-images: true # Cached docker images
|
|
swap-storage: false # Keep swap for large links
|
|
|
|
- name: Free disk space (macOS)
|
|
if: runner.os == 'macOS'
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
df -h /
|
|
# Observed on this lane: "zstd: error 70 ... No space left on device" followed by a
|
|
# linker failure. Drop multi-GB toolchains this build never uses before building.
|
|
# Deliberately keep Xcode/clang, Homebrew, cmake, ccache, zig, boost and the Rust
|
|
# toolchain (libSQL). Removing /Applications/Xcode_*.app would reclaim the most space
|
|
# but risks the selected toolchain, so it is left alone.
|
|
sudo rm -rf /Users/runner/Library/Android/sdk || true
|
|
sudo rm -rf /usr/local/share/dotnet /Users/runner/.dotnet || true
|
|
# Do NOT remove ~/.cargo or ~/.rustup: meson.build builds the libSQL subproject only when
|
|
# `cargo` is found, and emits warnings before falling back to plain sqlite3 without failing
|
|
# (meson.build:555-572). Deleting them would switch this lane's database backend without failing.
|
|
sudo rm -rf /usr/local/share/powershell || true
|
|
sudo rm -rf /Library/Frameworks/Mono.framework /usr/local/lib/mono || true
|
|
df -h /
|
|
|
|
- name: Initialize required submodules
|
|
shell: bash
|
|
run: |
|
|
retry_cmd() {
|
|
local attempts="$1"
|
|
shift
|
|
local attempt=1
|
|
until "$@"; do
|
|
if [ "$attempt" -ge "$attempts" ]; then
|
|
return 1
|
|
fi
|
|
echo "Command failed (attempt ${attempt}/${attempts}): $*" >&2
|
|
sleep $((attempt * 5))
|
|
attempt=$((attempt + 1))
|
|
done
|
|
}
|
|
# Force HTTPS for GitHub to handle SSH submodule URLs in CI
|
|
git config --global url."https://github.com/".insteadOf "git@github.com:"
|
|
# Only init submodules required for the build
|
|
retry_cmd 4 git submodule update --init --depth 1 third_party/sqlite-vec-cpp
|
|
retry_cmd 4 git submodule update --init --depth 1 third_party/simeon
|
|
retry_cmd 4 git submodule update --init --depth 1 third_party/symspell
|
|
- name: Install system dependencies (Linux)
|
|
if: runner.os == 'Linux'
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
bash .github/scripts/apt-get-with-retry.sh update
|
|
# Provide system Boost as a fallback per docs/BUILD-GCC.md when CMakeDeps resolution
|
|
# is not detected by CMake (module mode will pick this up).
|
|
# Install libbenchmark-dev for Google Benchmark support
|
|
bash .github/scripts/apt-get-with-retry.sh \
|
|
install -y --fix-missing --no-install-recommends \
|
|
cmake ninja-build clang clang-tools lld ccache \
|
|
libboost-all-dev pkg-config libbenchmark-dev xz-utils libomp-18-dev
|
|
PYTOOLS_DIR="${RUNNER_TEMP}/yams-python-tools"
|
|
python3 -m venv "$PYTOOLS_DIR"
|
|
"$PYTOOLS_DIR/bin/python" -m pip install --upgrade pip
|
|
"$PYTOOLS_DIR/bin/python" -m pip install --upgrade 'meson==1.4.*' 'ninja==1.11.*'
|
|
echo "$PYTOOLS_DIR/bin" >> "$GITHUB_PATH"
|
|
export PATH="$PYTOOLS_DIR/bin:$PATH"
|
|
echo "meson version: $(meson --version || echo 'missing')"
|
|
echo "ninja version: $(ninja --version || echo 'missing')"
|
|
echo "Installed system Boost headers/libs for module-mode fallback"
|
|
|
|
- name: Install Zig (Linux)
|
|
if: runner.os == 'Linux'
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
ARCH=$(uname -m)
|
|
case "$ARCH" in
|
|
x86_64) ZIG_ARCH="x86_64" ;;
|
|
aarch64) ZIG_ARCH="aarch64" ;;
|
|
*) echo "Unsupported architecture: $ARCH"; exit 1 ;;
|
|
esac
|
|
ZIG_VERSION="0.15.2"
|
|
curl -fsSL "https://ziglang.org/download/${ZIG_VERSION}/zig-${ZIG_ARCH}-linux-${ZIG_VERSION}.tar.xz" -o /tmp/zig.tar.xz
|
|
sudo tar -xJf /tmp/zig.tar.xz -C /opt
|
|
sudo mv /opt/zig-${ZIG_ARCH}-linux-${ZIG_VERSION} /opt/zig
|
|
sudo ln -sf /opt/zig/zig /usr/local/bin/zig
|
|
rm /tmp/zig.tar.xz
|
|
zig version
|
|
|
|
- name: Ensure Node for act cache actions (Linux)
|
|
if: runner.os == 'Linux' && github.actor == 'nektos/act'
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
if ! command -v node >/dev/null 2>&1; then
|
|
bash .github/scripts/apt-get-with-retry.sh update
|
|
bash .github/scripts/apt-get-with-retry.sh \
|
|
install -y --no-install-recommends nodejs
|
|
fi
|
|
sudo ln -sf "$(command -v node)" /usr/local/bin/node
|
|
node --version
|
|
- name: Install system dependencies (macOS)
|
|
if: runner.os == 'macOS'
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
for attempt in 1 2 3; do
|
|
if brew update; then
|
|
break
|
|
fi
|
|
if [ "$attempt" -eq 3 ]; then
|
|
echo "brew update failed after ${attempt} attempts" >&2
|
|
exit 1
|
|
fi
|
|
sleep $((attempt * 10))
|
|
done
|
|
# Align with release workflow to ensure Meson builds have required toolchain bits.
|
|
# Install google-benchmark for benchmark tests, zig for zyp PDF plugin
|
|
brew install pkg-config cmake ccache boost google-benchmark zig || true
|
|
PYTOOLS_DIR="${RUNNER_TEMP}/yams-python-tools"
|
|
python3 -m venv "$PYTOOLS_DIR"
|
|
"$PYTOOLS_DIR/bin/python" -m pip install --upgrade pip
|
|
"$PYTOOLS_DIR/bin/python" -m pip install --upgrade 'meson==1.4.*' 'ninja==1.11.*'
|
|
echo "$PYTOOLS_DIR/bin" >> "$GITHUB_PATH"
|
|
export PATH="$PYTOOLS_DIR/bin:$PATH"
|
|
echo "meson version: $(meson --version || echo 'missing')"
|
|
echo "ninja version: $(ninja --version || echo 'missing')"
|
|
echo "zig version: $(zig version || echo 'missing')"
|
|
- name: Install system dependencies (Windows)
|
|
if: runner.os == 'Windows'
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
PYTOOLS_DIR="${RUNNER_TEMP}/yams-python-tools"
|
|
python -m venv "$PYTOOLS_DIR"
|
|
"$PYTOOLS_DIR/Scripts/python.exe" -m pip install --upgrade pip
|
|
"$PYTOOLS_DIR/Scripts/python.exe" -m pip install --upgrade 'meson==1.4.*' 'ninja==1.11.*'
|
|
echo "$PYTOOLS_DIR/Scripts" >> "$GITHUB_PATH"
|
|
export PATH="$PYTOOLS_DIR/Scripts:$PATH"
|
|
echo "meson version: $(meson --version || echo 'missing')"
|
|
echo "ninja version: $(ninja --version || echo 'missing')"
|
|
# Clean stale build dir to avoid Meson version incompatibility
|
|
if [ -d "${{ env.BUILD_DIR }}" ]; then
|
|
echo "Removing stale builddir to ensure clean Meson configuration"
|
|
rm -rf "${{ env.BUILD_DIR }}"
|
|
fi
|
|
|
|
- name: Install Zig (Windows)
|
|
if: runner.os == 'Windows'
|
|
shell: pwsh
|
|
run: |
|
|
$zigVersion = "0.15.2"
|
|
$zigUrl = "https://ziglang.org/download/${zigVersion}/zig-x86_64-windows-${zigVersion}.zip"
|
|
$zigZip = "$env:TEMP\zig.zip"
|
|
$zigDir = "C:\zig"
|
|
Invoke-WebRequest -Uri $zigUrl -OutFile $zigZip
|
|
Expand-Archive -Path $zigZip -DestinationPath $zigDir -Force
|
|
Move-Item "$zigDir\zig-x86_64-windows-${zigVersion}\*" $zigDir -Force
|
|
Remove-Item "$zigDir\zig-x86_64-windows-${zigVersion}" -Recurse -ErrorAction SilentlyContinue
|
|
$env:PATH = "$zigDir;$env:PATH"
|
|
"$zigDir" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
|
|
zig version
|
|
- name: Cache Conan packages
|
|
id: cache_conan
|
|
uses: actions/cache@v4
|
|
with:
|
|
path: |
|
|
~/.conan2/p
|
|
~/.conan2/r
|
|
~/.conan2/recipes
|
|
~/.conan2/metadata
|
|
~/.conan2/s
|
|
key: conan2-${{ runner.os }}-${{ matrix.suffix }}-${{ hashFiles('conanfile.py', 'conan/profiles/**') }}
|
|
restore-keys: |
|
|
conan2-${{ runner.os }}-${{ matrix.suffix }}-
|
|
conan2-${{ runner.os }}-linux-x86_64-
|
|
conan2-${{ runner.os }}-
|
|
- name: Conan cache diagnostics
|
|
shell: bash
|
|
run: |
|
|
echo "Conan cache hit: ${{ steps.cache_conan.outputs.cache-hit }}"
|
|
du -sh ~/.conan2/p 2>/dev/null || true
|
|
find ~/.conan2/r -maxdepth 1 -mindepth 1 -print 2>/dev/null | head || true
|
|
- name: Restore ccache
|
|
if: matrix.suffix != 'linux-arm64'
|
|
uses: actions/cache/restore@v4
|
|
continue-on-error: true
|
|
with:
|
|
path: |
|
|
~/.cache/ccache
|
|
~/.ccache
|
|
key: ccache-${{ runner.os }}-${{ matrix.suffix }}-${{ hashFiles('conanfile.py', 'meson.build', 'meson_options.txt') }}-${{ github.run_id }}-${{ github.run_attempt }}
|
|
restore-keys: |
|
|
ccache-${{ runner.os }}-${{ matrix.suffix }}-
|
|
ccache-${{ runner.os }}-linux-x86_64-
|
|
ccache-${{ runner.os }}-
|
|
- name: Install Conan (latest)
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
PYTOOLS_DIR="${RUNNER_TEMP}/yams-python-tools"
|
|
if [ "$RUNNER_OS" = "Windows" ]; then
|
|
VENV_PYTHON="$PYTOOLS_DIR/Scripts/python.exe"
|
|
TOOL_BIN="$PYTOOLS_DIR/Scripts"
|
|
if [ ! -f "$VENV_PYTHON" ]; then
|
|
python -m venv "$PYTOOLS_DIR"
|
|
fi
|
|
else
|
|
VENV_PYTHON="$PYTOOLS_DIR/bin/python"
|
|
TOOL_BIN="$PYTOOLS_DIR/bin"
|
|
if [ ! -x "$VENV_PYTHON" ]; then
|
|
python3 -m venv "$PYTOOLS_DIR"
|
|
fi
|
|
fi
|
|
|
|
"$VENV_PYTHON" -m pip install --upgrade pip
|
|
"$VENV_PYTHON" -m pip install --upgrade 'conan==2.21.*' 'meson==1.4.*' 'ninja==1.11.*'
|
|
echo "$TOOL_BIN" >> "$GITHUB_PATH"
|
|
export PATH="$TOOL_BIN:$PATH"
|
|
conan --version
|
|
meson --version
|
|
ninja --version
|
|
mkdir -p ~/.conan2
|
|
cp .conan/global.conf ~/.conan2/global.conf 2>/dev/null || true
|
|
|
|
# Continue directly with build/test steps (single job path)
|
|
- name: Install optional system dependencies
|
|
if: runner.os == 'Linux'
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
extra_packages=(liburing-dev libarchive-dev libtag1-dev)
|
|
if [ "${{ matrix.suffix }}" != "linux-arm64" ]; then
|
|
extra_packages+=(ccache)
|
|
fi
|
|
sudo apt-get install -y --no-install-recommends "${extra_packages[@]}" || true
|
|
echo "Optional packages installed (best-effort)."
|
|
|
|
- name: Setup ccache env
|
|
if: matrix.suffix != 'linux-arm64'
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
{
|
|
echo "CCACHE_DIR=$HOME/.cache/ccache"
|
|
echo "CCACHE_BASEDIR=$GITHUB_WORKSPACE"
|
|
echo "CCACHE_COMPRESS=1"
|
|
echo "CCACHE_MAXSIZE=5G"
|
|
echo "CCACHE_SLOPPINESS=time_macros,include_file_mtime,include_file_ctime,file_macro"
|
|
} >> "$GITHUB_ENV"
|
|
ccache -p 2>/dev/null || true
|
|
ccache -z 2>/dev/null || true
|
|
|
|
- name: Reclaim disk for linux-arm64 builds
|
|
if: matrix.suffix == 'linux-arm64'
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
echo "YAMS_SKIP_CCACHE=1" >> "$GITHUB_ENV"
|
|
rm -rf ~/.cache/ccache ~/.ccache || true
|
|
python3 -m pip cache purge || true
|
|
sudo apt-get clean || true
|
|
df -h
|
|
|
|
# Conan already installed above
|
|
|
|
- name: Setup Conan Profile
|
|
shell: bash
|
|
run: |
|
|
conan profile detect --force
|
|
sed -i 's/compiler.cppstd=.*/compiler.cppstd=20/' ~/.conan2/profiles/default || true
|
|
echo "Conan profile configured:" >&2
|
|
conan profile show >&2
|
|
|
|
- name: Detect clang version (log only)
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
if command -v clang++ >/dev/null 2>&1; then
|
|
RAW_VER=$(clang++ --version | sed -n 's/.*version \([0-9][0-9]*\).*/\1/p' | head -1 || true)
|
|
elif command -v clang >/dev/null 2>&1; then
|
|
RAW_VER=$(clang --version | sed -n 's/.*version \([0-9][0-9]*\).*/\1/p' | head -1 || true)
|
|
else
|
|
RAW_VER=""
|
|
fi
|
|
if [ -n "$RAW_VER" ]; then
|
|
echo "Detected clang major: $RAW_VER"
|
|
else
|
|
echo "WARNING: clang not found; default profile may resolve gcc instead" >&2
|
|
fi
|
|
|
|
- name: Clean corrupted Conan cache if needed
|
|
shell: bash
|
|
run: |
|
|
if ! conan list "*" 2>/dev/null;
|
|
then
|
|
echo "Conan cache appears corrupted, cleaning locks and temp files..."
|
|
conan cache clean --locks --temp 2>/dev/null || true
|
|
echo "Conan cache locks/temp cleaned"
|
|
fi
|
|
|
|
- name: Tune Conan client (parallel downloads)
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
CONF_FILE="$HOME/.conan2/global.conf"
|
|
DOWNLOAD_PARALLEL=8
|
|
BUILD_JOBS=""
|
|
if [ "${{ matrix.suffix }}" = "linux-arm64" ]; then
|
|
DOWNLOAD_PARALLEL=2
|
|
BUILD_JOBS=2
|
|
fi
|
|
LINE="core.download:parallel=${DOWNLOAD_PARALLEL}"
|
|
mkdir -p "$(dirname "$CONF_FILE")"
|
|
if [ -f "$CONF_FILE" ]; then
|
|
TMP_FILE="$(mktemp)"
|
|
awk -v line="$LINE" '
|
|
BEGIN { replaced = 0 }
|
|
/^core.download:parallel=/ { if (!replaced) { print line; replaced = 1 } ; next }
|
|
{ print }
|
|
END { if (!replaced) print line }
|
|
' "$CONF_FILE" > "$TMP_FILE"
|
|
mv "$TMP_FILE" "$CONF_FILE"
|
|
else
|
|
printf '%s\n' "$LINE" > "$CONF_FILE"
|
|
fi
|
|
if [ -n "$BUILD_JOBS" ]; then
|
|
if grep -q '^tools.build:jobs=' "$CONF_FILE"; then
|
|
TMP_FILE="$(mktemp)"
|
|
awk -v line="tools.build:jobs=${BUILD_JOBS}" '
|
|
BEGIN { replaced = 0 }
|
|
/^tools\.build:jobs=/ { if (!replaced) { print line; replaced = 1 } ; next }
|
|
{ print }
|
|
END { if (!replaced) print line }
|
|
' "$CONF_FILE" > "$TMP_FILE"
|
|
mv "$TMP_FILE" "$CONF_FILE"
|
|
else
|
|
printf '%s\n' "tools.build:jobs=${BUILD_JOBS}" >> "$CONF_FILE"
|
|
fi
|
|
fi
|
|
echo "Configured $LINE in $CONF_FILE"
|
|
grep -n '^core.download:parallel=' "$CONF_FILE" || true
|
|
grep -n '^tools.build:jobs=' "$CONF_FILE" || true
|
|
conan config show core.download:parallel 2>/dev/null || true
|
|
conan config show tools.build:jobs 2>/dev/null || true
|
|
conan --version
|
|
|
|
- name: Configure and Build with Meson (Unix)
|
|
id: build_unix
|
|
if: runner.os != 'Windows'
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
ARCH_PREFIX=""
|
|
if [ "${{ matrix.needs_rosetta }}" = "true" ]; then
|
|
softwareupdate --install-rosetta --agree-to-license || true
|
|
ARCH_PREFIX="arch -x86_64"
|
|
fi
|
|
# Keep AF_UNIX socket path short on Linux to avoid sun_path length limits
|
|
if [ "$RUNNER_OS" = "Linux" ]; then
|
|
XDG_RUNTIME_DIR="/run/user/$(id -u)"
|
|
mkdir -p "$XDG_RUNTIME_DIR" || true
|
|
echo "XDG_RUNTIME_DIR=$XDG_RUNTIME_DIR" >> "$GITHUB_ENV"
|
|
fi
|
|
# Use setup.sh with CI environment variables for consistent build configuration
|
|
export YAMS_CONAN_HOST_PROFILE="${{ matrix.conan_host_profile }}"
|
|
export YAMS_CONAN_ARCH="${{ matrix.conan_arch }}"
|
|
export YAMS_EXTRA_MESON_FLAGS="${{ matrix.extra_meson_flags }}"
|
|
export YAMS_ENABLE_MOBILE_BINDINGS="${{ matrix.mobile_bindings || 'false' }}"
|
|
# Cap ninja jobs at 4 by default so push-triggered builds do not saturate
|
|
# every runner core; larger/self-hosted runners stay bounded.
|
|
MESON_COMPILE_JOBS=4
|
|
if [ "${{ matrix.suffix }}" = "linux-arm64" ]; then
|
|
# Hosted arm64 runners are more memory-sensitive; unbounded Conan/Ninja
|
|
# parallelism can starve the runner until GitHub loses communication.
|
|
MESON_COMPILE_JOBS=2
|
|
export CONAN_CPU_COUNT=2
|
|
export CMAKE_BUILD_PARALLEL_LEVEL=2
|
|
export MAKEFLAGS="-j2"
|
|
export YAMS_CONAN_EXTRA_OPTIONS="-c tools.build:jobs=2 ${YAMS_CONAN_EXTRA_OPTIONS:-}"
|
|
fi
|
|
# Disable TSAN in CI - boost::asio strand destruction causes heap-use-after-free
|
|
# that TSAN detects but cannot be suppressed. Run TSAN locally for development.
|
|
export ENABLE_TSAN=false
|
|
|
|
chmod +x setup.sh
|
|
$ARCH_PREFIX ./setup.sh Debug
|
|
# Source Conan build env so meson compile uses the same meson version
|
|
# that setup.sh used for meson setup (Conan provides meson 1.10.x which
|
|
# creates build.dat incompatible with older system meson).
|
|
CONAN_BUILD_ENV="build/debug/build-debug/conan/conanbuild.sh"
|
|
# shellcheck source=/dev/null
|
|
if [ -f "${CONAN_BUILD_ENV}" ]; then set +u; . "${CONAN_BUILD_ENV}"; set -u; fi
|
|
if [ "${{ matrix.suffix }}" = "linux-x86_64-asan" ] || \
|
|
[ "${{ matrix.suffix }}" = "linux-x86_64-ubsan" ]; then
|
|
$ARCH_PREFIX meson compile -C build/debug \
|
|
catch2_daemon_metrics_status \
|
|
catch2_post_ingest_queue_unit \
|
|
catch2_storage_submodule \
|
|
catch2_work_coordinator
|
|
elif [ "${{ matrix.suffix }}" = "linux-x86_64-mobile" ]; then
|
|
$ARCH_PREFIX meson compile -C build/debug -j "$MESON_COMPILE_JOBS" \
|
|
yams_mobile \
|
|
catch2_mobile_abi
|
|
else
|
|
if [ -n "$MESON_COMPILE_JOBS" ]; then
|
|
$ARCH_PREFIX meson compile -C build/debug -j "$MESON_COMPILE_JOBS"
|
|
else
|
|
$ARCH_PREFIX meson compile -C build/debug
|
|
fi
|
|
fi
|
|
|
|
- name: Configure and Build with Meson (Windows)
|
|
id: build_windows
|
|
if: runner.os == 'Windows'
|
|
shell: pwsh
|
|
env:
|
|
YAMS_CONAN_HOST_PROFILE: ${{ matrix.conan_host_profile }}
|
|
YAMS_CONAN_ARCH: ${{ matrix.conan_arch }}
|
|
YAMS_EXTRA_MESON_FLAGS: ${{ matrix.extra_meson_flags }}
|
|
YAMS_ENABLE_MOBILE_BINDINGS: "false"
|
|
run: |
|
|
.\setup.ps1 Debug
|
|
|
|
# Diagnostic for Windows Boost lookup. Reads BOOST_* env vars set by
|
|
# setup.ps1 (persisted to GITHUB_ENV) and lists the actual Conan boost
|
|
# lib filenames so we can pattern-match meson's expected naming if the
|
|
# build above fails. continue-on-error + always() so it never blocks.
|
|
- name: (diag) Boost env + Conan lib layout (Windows)
|
|
if: runner.os == 'Windows' && always()
|
|
shell: pwsh
|
|
continue-on-error: true
|
|
run: |
|
|
Write-Host "BOOST_ROOT=$env:BOOST_ROOT"
|
|
Write-Host "BOOST_INCLUDEDIR=$env:BOOST_INCLUDEDIR"
|
|
Write-Host "BOOST_LIBRARYDIR=$env:BOOST_LIBRARYDIR"
|
|
if ($env:BOOST_LIBRARYDIR -and (Test-Path $env:BOOST_LIBRARYDIR)) {
|
|
Write-Host "--- $env:BOOST_LIBRARYDIR contents (boost_*) ---"
|
|
Get-ChildItem $env:BOOST_LIBRARYDIR -Filter 'boost_*' | Select Name | Format-Table -AutoSize | Out-String
|
|
Write-Host "--- $env:BOOST_LIBRARYDIR contents (libboost_*) ---"
|
|
Get-ChildItem $env:BOOST_LIBRARYDIR -Filter 'libboost_*' -ErrorAction SilentlyContinue | Select Name | Format-Table -AutoSize | Out-String
|
|
}
|
|
|
|
- name: ccache stats (post-build)
|
|
if: always() && runner.os != 'Windows'
|
|
shell: bash
|
|
run: |
|
|
set +e
|
|
ccache -s 2>/dev/null || true
|
|
du -sh "${CCACHE_DIR:-$HOME/.cache/ccache}" 2>/dev/null || true
|
|
|
|
- name: Save ccache
|
|
if: always() && matrix.suffix != 'linux-arm64' && github.actor != 'nektos/act'
|
|
uses: actions/cache/save@v4
|
|
continue-on-error: true
|
|
with:
|
|
path: |
|
|
~/.cache/ccache
|
|
~/.ccache
|
|
key: ccache-${{ runner.os }}-${{ matrix.suffix }}-${{ hashFiles('conanfile.py', 'meson.build', 'meson_options.txt') }}-${{ github.run_id }}-${{ github.run_attempt }}
|
|
|
|
- name: Check mobile ABI surface
|
|
if: matrix.mobile_bindings
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
LIB_PATH=$(find "${{ env.BUILD_DIR }}/src/mobile" -type f \( -name 'libyams_mobile*.so' -o -name 'libyams_mobile*.dylib' \) | head -n1)
|
|
if [ -z "$LIB_PATH" ]; then
|
|
echo "libyams_mobile shared library not found under ${{ env.BUILD_DIR }}/src/mobile" >&2
|
|
exit 1
|
|
fi
|
|
file "$LIB_PATH"
|
|
scripts/ci/check_mobile_abi.sh "$LIB_PATH"
|
|
|
|
- name: Run Tests with Meson (Unix)
|
|
id: meson_tests_unix
|
|
if: ${{ !matrix.needs_rosetta && runner.os != 'Windows' }}
|
|
shell: bash
|
|
continue-on-error: true
|
|
env:
|
|
YAMS_TEST_SAFE_SINGLE_INSTANCE: "1"
|
|
YAMS_SQLITE_MINIMAL_PRAGMAS: "1"
|
|
YAMS_SQLITE_VEC_INIT_TIMEOUT_MS: "1500"
|
|
YAMS_SQLITE_BUSY_TIMEOUT_MS: "1000"
|
|
YAMS_VDB_IN_MEMORY: "1"
|
|
YAMS_SQLITE_VEC_SKIP_INIT: "1"
|
|
YAMS_DISABLE_VECTORS: "1"
|
|
YAMS_TESTING: "1"
|
|
run: |
|
|
set -euo pipefail
|
|
if [ "$RUNNER_OS" = "Linux" ]; then
|
|
mkdir -p "${XDG_RUNTIME_DIR:-/run/user/$(id -u)}" || true
|
|
echo "Using XDG_RUNTIME_DIR=${XDG_RUNTIME_DIR:-/run/user/$(id -u)}"
|
|
fi
|
|
CONAN_BUILD_ENV=""
|
|
for candidate in \
|
|
"${{ env.BUILD_DIR }}/build-debug/conan/conanbuild.sh" \
|
|
"${{ env.BUILD_DIR }}/conan/conanbuild.sh"
|
|
do
|
|
if [ -f "$candidate" ]; then
|
|
CONAN_BUILD_ENV="$candidate"
|
|
break
|
|
fi
|
|
done
|
|
if [ -n "$CONAN_BUILD_ENV" ]; then
|
|
set +u
|
|
# shellcheck source=/dev/null
|
|
. "$CONAN_BUILD_ENV"
|
|
set -u
|
|
fi
|
|
# Run unit and integration tests (TSAN disabled in CI). ASAN is a sanitizer smoke lane;
|
|
# build/test selected targets only so hosted runners do not get recycled.
|
|
if [ "${{ matrix.suffix }}" = "linux-x86_64-asan" ] || \
|
|
[ "${{ matrix.suffix }}" = "linux-x86_64-ubsan" ]; then
|
|
meson test -C ${{ env.BUILD_DIR }} --no-rebuild --print-errorlogs --timeout-multiplier 2 \
|
|
daemon_metrics_status \
|
|
post_ingest_queue_unit \
|
|
storage_submodule \
|
|
work_coordinator
|
|
elif [ "${{ matrix.suffix }}" = "linux-x86_64-mobile" ]; then
|
|
meson test -C ${{ env.BUILD_DIR }} --no-rebuild --print-errorlogs --timeout-multiplier 2 \
|
|
--suite mobile
|
|
else
|
|
meson test -C ${{ env.BUILD_DIR }} --suite unit --suite integration --print-errorlogs --timeout-multiplier 2
|
|
fi
|
|
|
|
- name: Run Tests with Meson (Windows)
|
|
id: meson_tests_windows
|
|
if: runner.os == 'Windows'
|
|
shell: pwsh
|
|
continue-on-error: true
|
|
env:
|
|
YAMS_TEST_SAFE_SINGLE_INSTANCE: "1"
|
|
YAMS_SQLITE_MINIMAL_PRAGMAS: "1"
|
|
YAMS_SQLITE_VEC_INIT_TIMEOUT_MS: "1500"
|
|
YAMS_SQLITE_BUSY_TIMEOUT_MS: "1000"
|
|
YAMS_VDB_IN_MEMORY: "1"
|
|
YAMS_SQLITE_VEC_SKIP_INIT: "1"
|
|
YAMS_DISABLE_VECTORS: "1"
|
|
YAMS_TESTING: "1"
|
|
run: |
|
|
$ErrorActionPreference = "Stop"
|
|
$conanBuildPs1 = $null
|
|
$candidates = @(
|
|
(Join-Path "${{ env.BUILD_DIR }}" "build-debug/conan/conanbuild.ps1"),
|
|
(Join-Path "${{ env.BUILD_DIR }}" "conan/conanbuild.ps1")
|
|
)
|
|
foreach ($candidate in $candidates) {
|
|
if (Test-Path $candidate) {
|
|
$conanBuildPs1 = $candidate
|
|
break
|
|
}
|
|
}
|
|
if ($conanBuildPs1) {
|
|
. $conanBuildPs1
|
|
}
|
|
# Run only unit and integration tests (exclude bench, stress, daemon-long, etc.)
|
|
meson test -C ${{ env.BUILD_DIR }} --suite unit --suite integration --print-errorlogs --timeout-multiplier 2
|
|
|
|
- name: Run Tests with Meson (Rosetta x86_64)
|
|
id: meson_tests_rosetta
|
|
if: ${{ matrix.needs_rosetta }}
|
|
shell: bash
|
|
continue-on-error: true
|
|
env:
|
|
YAMS_TEST_SAFE_SINGLE_INSTANCE: "1"
|
|
YAMS_SQLITE_MINIMAL_PRAGMAS: "1"
|
|
YAMS_SQLITE_VEC_INIT_TIMEOUT_MS: "1500"
|
|
YAMS_SQLITE_BUSY_TIMEOUT_MS: "1000"
|
|
YAMS_VDB_IN_MEMORY: "1"
|
|
YAMS_SQLITE_VEC_SKIP_INIT: "1"
|
|
YAMS_DISABLE_VECTORS: "1"
|
|
YAMS_TESTING: "1"
|
|
run: |
|
|
set -euo pipefail
|
|
# Ensure Rosetta is present (idempotent)
|
|
softwareupdate --install-rosetta --agree-to-license || true
|
|
CONAN_BUILD_ENV=""
|
|
for candidate in \
|
|
"${{ env.BUILD_DIR }}/build-debug/conan/conanbuild.sh" \
|
|
"${{ env.BUILD_DIR }}/conan/conanbuild.sh"
|
|
do
|
|
if [ -f "$candidate" ]; then
|
|
CONAN_BUILD_ENV="$candidate"
|
|
break
|
|
fi
|
|
done
|
|
if [ -n "$CONAN_BUILD_ENV" ]; then
|
|
set +u
|
|
# shellcheck source=/dev/null
|
|
. "$CONAN_BUILD_ENV"
|
|
set -u
|
|
fi
|
|
# Run only unit and integration tests (exclude bench, stress, daemon-long, etc.)
|
|
arch -x86_64 meson test -C ${{ env.BUILD_DIR }} --suite unit --suite integration --print-errorlogs --timeout-multiplier 2
|
|
|
|
- name: Upload build and test logs (artifacts)
|
|
if: ${{ always() && (steps.build_unix.outcome == 'failure' || steps.build_windows.outcome == 'failure' || steps.meson_tests_unix.outcome == 'failure' || steps.meson_tests_windows.outcome == 'failure' || steps.meson_tests_rosetta.outcome == 'failure') }}
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: logs-${{ matrix.suffix }}-${{ github.run_number }}
|
|
path: |
|
|
${{ env.BUILD_DIR }}/meson-logs/
|
|
${{ env.BUILD_DIR }}/meson-info/
|
|
${{ env.BUILD_DIR }}/meson-logs/testlog.txt
|
|
${{ env.BUILD_DIR }}/**/conan-install/**/*.log
|
|
${{ env.BUILD_DIR }}/**/conan*.log
|
|
if-no-files-found: ignore
|
|
retention-days: 14
|
|
|
|
- name: Job Summary (artifacts link)
|
|
if: always()
|
|
shell: bash
|
|
run: |
|
|
echo "### Artifacts" >> "$GITHUB_STEP_SUMMARY"
|
|
echo "[View artifacts for this run]($GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID)" >> "$GITHUB_STEP_SUMMARY"
|
|
|
|
- name: Enforce test failure
|
|
if: ${{ always() && (steps.meson_tests_unix.outcome == 'failure' || steps.meson_tests_windows.outcome == 'failure' || steps.meson_tests_rosetta.outcome == 'failure') }}
|
|
shell: bash
|
|
run: |
|
|
python tests/scripts/report_meson_failures.py "${{ env.BUILD_DIR }}" || \
|
|
echo "Meson failure reporter encountered an error; continuing gate enforcement." >&2
|
|
echo "Meson tests reported failures; enforcing job failure." >&2
|
|
exit 1
|
|
|
|
- name: Upload Benchmark Results
|
|
if: github.actor != 'nektos/act' && matrix.conan_arch == 'x86_64'
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: benchmark-results-linux-hosted-yams
|
|
path: |
|
|
${{ env.BUILD_DIR }}/bench_results/**/*.json
|
|
${{ env.BUILD_DIR }}/bench_results/*.json
|
|
if-no-files-found: ignore
|
|
retention-days: 5
|