name: Matrix Warm permissions: contents: read on: workflow_call: inputs: conan_profile_linux: description: Path to Linux Conan host profile required: false type: string default: ./conan/profiles/host-linux-clang conan_profile_macos: description: Path to macOS Conan host profile required: false type: string default: ./conan/profiles/host-macos-apple-clang conan_profile_macos_x86: description: Path to macOS x86 Conan host profile required: false type: string default: ./conan/profiles/host-macos-apple-clang-x86 conan_profile_windows: description: Path to Windows Conan host profile required: false type: string default: ./conan/profiles/host-windows-msvc build_type: description: Conan build type for warming (Release/Debug) required: false type: string default: Release enable_onnx: description: Enable ONNX deps during warm required: false type: boolean default: false checkout_ref: description: Immutable source SHA to warm required: false type: string default: "" jobs: warm: name: Warm Conan & ccache (${{ matrix.suffix }}) strategy: fail-fast: false matrix: include: - suffix: linux-x86_64 runs_on: ubuntu-24.04 conan_host_profile: ${{ inputs.conan_profile_linux }} conan_arch: x86_64 needs_rosetta: false timeout: 60 - suffix: linux-arm64 runs_on: ubuntu-24.04-arm conan_host_profile: ${{ inputs.conan_profile_linux }} conan_arch: armv8 needs_rosetta: false timeout: 60 - suffix: macos-arm64 runs_on: macos-15 conan_host_profile: ${{ inputs.conan_profile_macos }} conan_arch: armv8 needs_rosetta: false - suffix: macos-x86_64 runs_on: macos-15 conan_host_profile: ${{ inputs.conan_profile_macos_x86 }} conan_arch: x86_64 needs_rosetta: true - suffix: windows-x86_64 runs_on: windows-2025-vs2026 conan_host_profile: ${{ inputs.conan_profile_windows }} conan_arch: x86_64 needs_rosetta: false conan_compiler_version: "195" msbuild_vs_version: "18" timeout: 60 runs-on: ${{ matrix.runs_on }} timeout-minutes: ${{ matrix.timeout || 30 }} env: YAMS_CONAN_COMPILER_VERSION: ${{ matrix.conan_compiler_version || '' }} YAMS_MSBUILD_VS_VERSION: ${{ matrix.msbuild_vs_version || '' }} steps: - name: Checkout uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5 with: fetch-depth: 0 submodules: false # Manual submodule init below (some use SSH URLs) ref: ${{ inputs.checkout_ref || github.sha }} - name: Initialize required submodules shell: bash run: | retry_cmd() { local attempts="$1" shift local attempt=1 until "$@"; do if [ "$attempt" -ge "$attempts" ]; then return 1 fi echo "Command failed (attempt ${attempt}/${attempts}): $*" >&2 sleep $((attempt * 5)) attempt=$((attempt + 1)) done } # Force HTTPS for GitHub to handle SSH submodule URLs in CI git config --global url."https://github.com/".insteadOf "git@github.com:" # Only init the build-required third_party submodules # Skip other submodules that use SSH URLs requiring authentication retry_cmd 4 git submodule update --init --depth 1 third_party/sqlite-vec-cpp retry_cmd 4 git submodule update --init --depth 1 third_party/simeon retry_cmd 4 git submodule update --init --depth 1 third_party/symspell - name: Cache Conan packages (warm) if: github.actor != 'nektos/act' id: cache_conan_warm uses: actions/cache@v4 continue-on-error: true with: path: | ~/.conan2/p ~/.conan2/r ~/.conan2/recipes ~/.conan2/metadata ~/.conan2/s key: conan2-${{ runner.os }}-${{ matrix.suffix }}-${{ hashFiles('conanfile.py', 'conan/profiles/**') }} restore-keys: | conan2-${{ runner.os }}-${{ matrix.suffix }}- conan2-${{ runner.os }}-linux-x86_64- conan2-${{ runner.os }}- - name: Restore ccache (warm) if: github.actor != 'nektos/act' id: cache_ccache_warm uses: actions/cache/restore@v4 continue-on-error: true with: path: | ~/.cache/ccache ~/.ccache key: ccache-${{ runner.os }}-${{ matrix.suffix }}-${{ hashFiles('conanfile.py', 'meson.build', 'meson_options.txt') }}-${{ github.run_id }}-${{ github.run_attempt }} restore-keys: | ccache-${{ runner.os }}-${{ matrix.suffix }}- ccache-${{ runner.os }}-linux-x86_64- ccache-${{ runner.os }}- - name: Install Conan and Meson shell: bash run: | set -euo pipefail PYTOOLS_DIR="${RUNNER_TEMP}/yams-python-tools" if [ "$RUNNER_OS" = "Windows" ]; then VENV_PYTHON="$PYTOOLS_DIR/Scripts/python.exe" TOOL_BIN="$PYTOOLS_DIR/Scripts" if [ ! -f "$VENV_PYTHON" ]; then python -m venv "$PYTOOLS_DIR" fi else VENV_PYTHON="$PYTOOLS_DIR/bin/python" TOOL_BIN="$PYTOOLS_DIR/bin" if [ ! -x "$VENV_PYTHON" ]; then python3 -m venv "$PYTOOLS_DIR" fi fi "$VENV_PYTHON" -m pip install --upgrade pip "$VENV_PYTHON" -m pip install --upgrade 'conan==2.21.*' 'meson==1.4.*' 'ninja==1.11.*' echo "$TOOL_BIN" >> "$GITHUB_PATH" export PATH="$TOOL_BIN:$PATH" conan --version meson --version ninja --version mkdir -p ~/.conan2 cp .conan/global.conf ~/.conan2/global.conf 2>/dev/null || true - name: Prime Conan profiles shell: bash run: | set -euo pipefail conan profile detect --force # Use C++20 instead of C++23 to avoid pdfium compilation issues if [ "$RUNNER_OS" = "macOS" ]; then sed -i '' 's/compiler.cppstd=.*/compiler.cppstd=20/' ~/.conan2/profiles/default || true else sed -i 's/compiler.cppstd=.*/compiler.cppstd=20/' ~/.conan2/profiles/default || true fi conan profile show - name: Pin conancenter to v2 index shell: bash run: | set -euo pipefail if conan remote list | grep -q 'conancenter'; then conan remote update conancenter --url https://center2.conan.io || true else conan remote add conancenter https://center2.conan.io fi conan remote list - name: Install system dependencies (Linux) if: runner.os == 'Linux' shell: bash run: | set -euo pipefail bash .github/scripts/apt-get-with-retry.sh update bash .github/scripts/apt-get-with-retry.sh \ install -y --fix-missing --no-install-recommends \ cmake ninja-build pkg-config clang clang-tools lld libomp-18-dev ccache - name: Warm dependencies (${{ inputs.build_type }}) # Always run on Linux to populate ccache (cache-hit can be true on a tiny # restored cache). On macOS/Windows, skip if Conan deps are already warm. if: runner.os == 'Linux' || steps.cache_conan_warm.outputs.cache-hit != 'true' shell: bash run: | set -euo pipefail if command -v nproc >/dev/null 2>&1; then CONAN_CPU_COUNT=$(nproc) elif command -v sysctl >/dev/null 2>&1; then CONAN_CPU_COUNT=$(sysctl -n hw.ncpu) elif [ "$RUNNER_OS" = "Windows" ]; then CONAN_CPU_COUNT=$NUMBER_OF_PROCESSORS elif command -v getconf >/dev/null 2>&1; then CONAN_CPU_COUNT=$(getconf _NPROCESSORS_ONLN) else CONAN_CPU_COUNT=4 fi export CONAN_CPU_COUNT if [ "${{ inputs.enable_onnx }}" = 'true' ]; then ONNX=True else ONNX=False fi conan install . \ -of build/warm-${{ inputs.build_type }} \ -pr:b=default \ -pr:h="${{ matrix.conan_host_profile }}" \ -s build_type=${{ inputs.build_type }} \ -s:h arch=${{ matrix.conan_arch }} \ -o "sqlite3/*:fts5=True" \ -o "yams/*:enable_onnx=${ONNX}" \ --build=missing - name: Populate ccache with source compilation if: runner.os == 'Linux' continue-on-error: true shell: bash env: CCACHE_DIR: ~/.cache/ccache CCACHE_BASEDIR: ${{ github.workspace }} CCACHE_COMPRESS: "1" CCACHE_MAXSIZE: 5G CCACHE_SLOPPINESS: time_macros,include_file_mtime,include_file_ctime,file_macro run: | set -euo pipefail CACHE_BYTES=$(du -sb ~/.cache/ccache 2>/dev/null | cut -f1 || echo 0) CACHE_MB=$((CACHE_BYTES / 1048576)) if [ "$CACHE_MB" -gt 100 ]; then echo "ccache already populated: ${CACHE_MB} MB; skipping source compile" exit 0 fi echo "ccache=${CACHE_MB} MB (< 100 MB); compiling source to populate cache" if ! command -v ccache >/dev/null 2>&1 || ! command -v clang >/dev/null 2>&1 || ! command -v clang++ >/dev/null 2>&1; then echo "ccache/clang toolchain unavailable; skipping source compile" exit 0 fi ccache -z >/dev/null 2>&1 || true BUILD_DIR="build/warm-${{ inputs.build_type }}" NATIVE_FILE=$(find "$BUILD_DIR" -name conan_meson_native.ini -type f 2>/dev/null | head -1) if [ -z "$NATIVE_FILE" ]; then echo "No Meson native file found in $BUILD_DIR; skipping source compile" exit 0 fi echo "Using Conan native file: $NATIVE_FILE" CCACHE_NATIVE_FILE="$BUILD_DIR/ccache-native.ini" cat > "$CCACHE_NATIVE_FILE" <<'EOF' [binaries] c = ['ccache', 'clang'] cpp = ['ccache', 'clang++'] EOF echo "Using ccache Meson override: $CCACHE_NATIVE_FILE" if [ "${{ inputs.build_type }}" = "Debug" ]; then MESON_BUILDTYPE="--buildtype=debug" else MESON_BUILDTYPE="--buildtype=release" fi if ! meson setup "$BUILD_DIR" --native-file="$NATIVE_FILE" --native-file="$CCACHE_NATIVE_FILE" \ "$MESON_BUILDTYPE" --reconfigure 2>&1; then meson setup "$BUILD_DIR" --native-file="$NATIVE_FILE" --native-file="$CCACHE_NATIVE_FILE" \ "$MESON_BUILDTYPE" 2>&1 fi JOBS=$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 4) meson compile -C "$BUILD_DIR" -j"$JOBS" echo echo "=== ccache stats after warm compilation ===" ccache -s | head -8 - name: Save ccache (warm) if: always() && github.actor != 'nektos/act' uses: actions/cache/save@v4 continue-on-error: true with: path: | ~/.cache/ccache ~/.ccache key: ccache-${{ runner.os }}-${{ matrix.suffix }}-${{ hashFiles('conanfile.py', 'meson.build', 'meson_options.txt') }}-${{ github.run_id }}-${{ github.run_attempt }} - name: Warm summary shell: bash run: | echo "Conan warm cache hit: ${{ steps.cache_conan_warm.outputs.cache-hit }}" || true du -sh ~/.conan2/p 2>/dev/null || true du -sh ~/.cache/ccache 2>/dev/null || true