yams/.github/workflows/matrix-warm.yml

name: Matrix Warm

permissions:
  contents: read

on:
  workflow_call:
    inputs:
      conan_profile_linux:
        description: Path to Linux Conan host profile
        required: false
        type: string
        default: ./conan/profiles/host-linux-clang
      conan_profile_macos:
        description: Path to macOS Conan host profile
        required: false
        type: string
        default: ./conan/profiles/host-macos-apple-clang
      conan_profile_macos_x86:
        description: Path to macOS x86 Conan host profile
        required: false
        type: string
        default: ./conan/profiles/host-macos-apple-clang-x86
      conan_profile_windows:
        description: Path to Windows Conan host profile
        required: false
        type: string
        default: ./conan/profiles/host-windows-msvc
      build_type:
        description: Conan build type for warming (Release/Debug)
        required: false
        type: string
        default: Release
      enable_onnx:
        description: Enable ONNX deps during warm
        required: false
        type: boolean
        default: false

jobs:
  warm:
    name: Warm Conan & ccache (${{ matrix.suffix }})
    strategy:
      fail-fast: false
      matrix:
        include:
          - suffix: linux-x86_64
            runs_on: ubuntu-24.04
            conan_host_profile: ${{ inputs.conan_profile_linux }}
            conan_arch: x86_64
            needs_rosetta: false
            timeout: 60
          - suffix: linux-arm64
            runs_on: ubuntu-24.04-arm
            conan_host_profile: ${{ inputs.conan_profile_linux }}
            conan_arch: armv8
            needs_rosetta: false
            timeout: 60
          - suffix: macos-arm64
            runs_on: macos-15
            conan_host_profile: ${{ inputs.conan_profile_macos }}
            conan_arch: armv8
            needs_rosetta: false
          - suffix: macos-x86_64
            runs_on: macos-15
            conan_host_profile: ${{ inputs.conan_profile_macos_x86 }}
            conan_arch: x86_64
            needs_rosetta: true
          - suffix: windows-x86_64
            runs_on: windows-2025-vs2026
            conan_host_profile: ${{ inputs.conan_profile_windows }}
            conan_arch: x86_64
            needs_rosetta: false
            conan_compiler_version: "195"
            msbuild_vs_version: "18"
            timeout: 60
    runs-on: ${{ matrix.runs_on }}
    timeout-minutes: ${{ matrix.timeout || 30 }}
    env:
      YAMS_CONAN_COMPILER_VERSION: ${{ matrix.conan_compiler_version || '' }}
      YAMS_MSBUILD_VS_VERSION: ${{ matrix.msbuild_vs_version || '' }}
    steps:
      - name: Checkout
        uses: actions/checkout@v5
        with:
          fetch-depth: 0
          submodules: false # Manual submodule init below (some use SSH URLs)

      - name: Initialize required submodules
        shell: bash
        run: |
          retry_cmd() {
            local attempts="$1"
            shift
            local attempt=1
            until "$@"; do
              if [ "$attempt" -ge "$attempts" ]; then
                return 1
              fi
              echo "Command failed (attempt ${attempt}/${attempts}): $*" >&2
              sleep $((attempt * 5))
              attempt=$((attempt + 1))
            done
          }
          # Force HTTPS for GitHub to handle SSH submodule URLs in CI
          git config --global url."https://github.com/".insteadOf "git@github.com:"
          # Only init the build-required third_party submodules
          # Skip other submodules that use SSH URLs requiring authentication
          retry_cmd 4 git submodule update --init --depth 1 third_party/sqlite-vec-cpp
          retry_cmd 4 git submodule update --init --depth 1 third_party/simeon
          retry_cmd 4 git submodule update --init --depth 1 third_party/symspell

      - name: Cache Conan packages (warm)
        if: github.actor != 'nektos/act'
        id: cache_conan_warm
        uses: actions/cache@v4
        continue-on-error: true
        with:
          path: |
            ~/.conan2/p
            ~/.conan2/r
            ~/.conan2/recipes
            ~/.conan2/metadata
            ~/.conan2/s
          key: conan2-${{ runner.os }}-${{ matrix.suffix }}-${{ hashFiles('conanfile.py', 'conan/profiles/**') }}
          restore-keys: |
            conan2-${{ runner.os }}-${{ matrix.suffix }}-
            conan2-${{ runner.os }}-linux-x86_64-
            conan2-${{ runner.os }}-

      - name: Restore ccache (warm)
        if: github.actor != 'nektos/act'
        id: cache_ccache_warm
        uses: actions/cache/restore@v4
        continue-on-error: true
        with:
          path: |
            ~/.cache/ccache
            ~/.ccache
          key: ccache-${{ runner.os }}-${{ matrix.suffix }}-${{ hashFiles('conanfile.py', 'meson.build', 'meson_options.txt') }}-${{ github.run_id }}-${{ github.run_attempt }}
          restore-keys: |
            ccache-${{ runner.os }}-${{ matrix.suffix }}-
            ccache-${{ runner.os }}-linux-x86_64-
            ccache-${{ runner.os }}-

      - name: Install Conan and Meson
        shell: bash
        run: |
          set -euo pipefail
          PYTOOLS_DIR="${RUNNER_TEMP}/yams-python-tools"
          if [ "$RUNNER_OS" = "Windows" ]; then
            VENV_PYTHON="$PYTOOLS_DIR/Scripts/python.exe"
            TOOL_BIN="$PYTOOLS_DIR/Scripts"
            if [ ! -f "$VENV_PYTHON" ]; then
              python -m venv "$PYTOOLS_DIR"
            fi
          else
            VENV_PYTHON="$PYTOOLS_DIR/bin/python"
            TOOL_BIN="$PYTOOLS_DIR/bin"
            if [ ! -x "$VENV_PYTHON" ]; then
              python3 -m venv "$PYTOOLS_DIR"
            fi
          fi
          "$VENV_PYTHON" -m pip install --upgrade pip
          "$VENV_PYTHON" -m pip install --upgrade 'conan==2.21.*' 'meson==1.4.*' 'ninja==1.11.*'
          echo "$TOOL_BIN" >> "$GITHUB_PATH"
          export PATH="$TOOL_BIN:$PATH"
          conan --version
          meson --version
          ninja --version
          mkdir -p ~/.conan2
          cp .conan/global.conf ~/.conan2/global.conf 2>/dev/null || true

      - name: Prime Conan profiles
        shell: bash
        run: |
          set -euo pipefail
          conan profile detect --force
          # Use C++20 instead of C++23 to avoid pdfium compilation issues
          if [ "$RUNNER_OS" = "macOS" ]; then
            sed -i '' 's/compiler.cppstd=.*/compiler.cppstd=20/' ~/.conan2/profiles/default || true
          else
            sed -i 's/compiler.cppstd=.*/compiler.cppstd=20/' ~/.conan2/profiles/default || true
          fi
          conan profile show

      - name: Pin conancenter to v2 index
        shell: bash
        run: |
          set -euo pipefail
          if conan remote list | grep -q 'conancenter'; then
            conan remote update conancenter --url https://center2.conan.io || true
          else
            conan remote add conancenter https://center2.conan.io
          fi
          conan remote list

      - name: Install system dependencies (Linux)
        if: runner.os == 'Linux'
        shell: bash
        run: |
          set -euo pipefail
          retry_cmd() {
            local attempts="$1"
            shift
            local attempt=1
            until "$@"; do
              if [ "$attempt" -ge "$attempts" ]; then
                return 1
              fi
              echo "Command failed (attempt ${attempt}/${attempts}): $*" >&2
              sleep $((attempt * 10))
              attempt=$((attempt + 1))
            done
          }
          if [ "$(dpkg --print-architecture)" = "arm64" ]; then
            source_files="$(sudo grep -RIl '://ports.ubuntu.com/ubuntu-ports' \
              /etc/apt/sources.list /etc/apt/sources.list.d 2>/dev/null || true)"
            if [ -n "$source_files" ]; then
              printf '%s\n' "$source_files" | xargs sudo sed -i \
                's|http://ports.ubuntu.com/ubuntu-ports|https://azure.ports.ubuntu.com/ubuntu-ports|g; s|https://ports.ubuntu.com/ubuntu-ports|https://azure.ports.ubuntu.com/ubuntu-ports|g'
            fi
          fi
          retry_cmd 4 sudo apt-get -o Acquire::Retries=5 -o Acquire::ForceIPv4=true update
          retry_cmd 4 sudo apt-get -o Acquire::Retries=5 -o Acquire::ForceIPv4=true \
            install -y --fix-missing --no-install-recommends \
            cmake ninja-build pkg-config clang clang-tools lld libomp-18-dev ccache

      - name: Warm dependencies (${{ inputs.build_type }})
        # Always run on Linux to populate ccache (cache-hit can be true on a tiny
        # restored cache). On macOS/Windows, skip if Conan deps are already warm.
        if: runner.os == 'Linux' || steps.cache_conan_warm.outputs.cache-hit != 'true'
        shell: bash
        run: |
          set -euo pipefail
          if command -v nproc >/dev/null 2>&1; then
            CONAN_CPU_COUNT=$(nproc)
          elif command -v sysctl >/dev/null 2>&1; then
            CONAN_CPU_COUNT=$(sysctl -n hw.ncpu)
          elif [ "$RUNNER_OS" = "Windows" ]; then
            CONAN_CPU_COUNT=$NUMBER_OF_PROCESSORS
          elif command -v getconf >/dev/null 2>&1; then
            CONAN_CPU_COUNT=$(getconf _NPROCESSORS_ONLN)
          else
            CONAN_CPU_COUNT=4
          fi
          export CONAN_CPU_COUNT
          if [ "${{ inputs.enable_onnx }}" = 'true' ]; then
            ONNX=True
          else
            ONNX=False
          fi
          conan install . \
            -of build/warm-${{ inputs.build_type }} \
            -pr:b=default \
            -pr:h="${{ matrix.conan_host_profile }}" \
            -s build_type=${{ inputs.build_type }} \
            -s:h arch=${{ matrix.conan_arch }} \
            -o "sqlite3/*:fts5=True" \
            -o "yams/*:enable_onnx=${ONNX}" \
            --build=missing

      - name: Populate ccache with source compilation
        if: runner.os == 'Linux'
        continue-on-error: true
        shell: bash
        env:
          CCACHE_DIR: ~/.cache/ccache
          CCACHE_BASEDIR: ${{ github.workspace }}
          CCACHE_COMPRESS: "1"
          CCACHE_MAXSIZE: 5G
          CCACHE_SLOPPINESS: time_macros,include_file_mtime,include_file_ctime,file_macro
        run: |
          set -euo pipefail
          CACHE_BYTES=$(du -sb ~/.cache/ccache 2>/dev/null | cut -f1 || echo 0)
          CACHE_MB=$((CACHE_BYTES / 1048576))
          if [ "$CACHE_MB" -gt 100 ]; then
            echo "ccache already populated: ${CACHE_MB} MB; skipping source compile"
            exit 0
          fi
          echo "ccache=${CACHE_MB} MB (< 100 MB); compiling source to populate cache"
          if ! command -v ccache >/dev/null 2>&1 || ! command -v clang >/dev/null 2>&1 || ! command -v clang++ >/dev/null 2>&1; then
            echo "ccache/clang toolchain unavailable; skipping source compile"
            exit 0
          fi
          ccache -z >/dev/null 2>&1 || true
          BUILD_DIR="build/warm-${{ inputs.build_type }}"
          NATIVE_FILE=$(find "$BUILD_DIR" -name conan_meson_native.ini -type f 2>/dev/null | head -1)
          if [ -z "$NATIVE_FILE" ]; then
            echo "No Meson native file found in $BUILD_DIR; skipping source compile"
            exit 0
          fi
          echo "Using Conan native file: $NATIVE_FILE"
          CCACHE_NATIVE_FILE="$BUILD_DIR/ccache-native.ini"
          cat > "$CCACHE_NATIVE_FILE" <<'EOF'
          [binaries]
          c = ['ccache', 'clang']
          cpp = ['ccache', 'clang++']
          EOF
          echo "Using ccache Meson override: $CCACHE_NATIVE_FILE"
          if [ "${{ inputs.build_type }}" = "Debug" ]; then
            MESON_BUILDTYPE="--buildtype=debug"
          else
            MESON_BUILDTYPE="--buildtype=release"
          fi
          if ! meson setup "$BUILD_DIR" --native-file="$NATIVE_FILE" --native-file="$CCACHE_NATIVE_FILE" \
               "$MESON_BUILDTYPE" --reconfigure 2>&1; then
            meson setup "$BUILD_DIR" --native-file="$NATIVE_FILE" --native-file="$CCACHE_NATIVE_FILE" \
              "$MESON_BUILDTYPE" 2>&1
          fi
          JOBS=$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 4)
          meson compile -C "$BUILD_DIR" -j"$JOBS"
          echo
          echo "=== ccache stats after warm compilation ==="
          ccache -s | head -8

      - name: Save ccache (warm)
        if: always() && github.actor != 'nektos/act'
        uses: actions/cache/save@v4
        continue-on-error: true
        with:
          path: |
            ~/.cache/ccache
            ~/.ccache
          key: ccache-${{ runner.os }}-${{ matrix.suffix }}-${{ hashFiles('conanfile.py', 'meson.build', 'meson_options.txt') }}-${{ github.run_id }}-${{ github.run_attempt }}

      - name: Warm summary
        shell: bash
        run: |
          echo "Conan warm cache hit: ${{ steps.cache_conan_warm.outputs.cache-hit }}" || true
          du -sh ~/.conan2/p 2>/dev/null || true
          du -sh ~/.cache/ccache 2>/dev/null || true