296 lines
11 KiB
YAML
296 lines
11 KiB
YAML
name: Matrix Warm
|
|
|
|
permissions:
|
|
contents: read
|
|
|
|
on:
|
|
workflow_call:
|
|
inputs:
|
|
conan_profile_linux:
|
|
description: Path to Linux Conan host profile
|
|
required: false
|
|
type: string
|
|
default: ./conan/profiles/host-linux-clang
|
|
conan_profile_macos:
|
|
description: Path to macOS Conan host profile
|
|
required: false
|
|
type: string
|
|
default: ./conan/profiles/host-macos-apple-clang
|
|
conan_profile_macos_x86:
|
|
description: Path to macOS x86 Conan host profile
|
|
required: false
|
|
type: string
|
|
default: ./conan/profiles/host-macos-apple-clang-x86
|
|
conan_profile_windows:
|
|
description: Path to Windows Conan host profile
|
|
required: false
|
|
type: string
|
|
default: ./conan/profiles/host-windows-msvc
|
|
build_type:
|
|
description: Conan build type for warming (Release/Debug)
|
|
required: false
|
|
type: string
|
|
default: Release
|
|
enable_onnx:
|
|
description: Enable ONNX deps during warm
|
|
required: false
|
|
type: boolean
|
|
default: false
|
|
|
|
jobs:
|
|
warm:
|
|
name: Warm Conan & ccache (${{ matrix.suffix }})
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
include:
|
|
- suffix: linux-x86_64
|
|
runs_on: ubuntu-24.04
|
|
conan_host_profile: ${{ inputs.conan_profile_linux }}
|
|
conan_arch: x86_64
|
|
needs_rosetta: false
|
|
timeout: 60
|
|
- suffix: linux-arm64
|
|
runs_on: ubuntu-24.04-arm
|
|
conan_host_profile: ${{ inputs.conan_profile_linux }}
|
|
conan_arch: armv8
|
|
needs_rosetta: false
|
|
timeout: 60
|
|
- suffix: macos-arm64
|
|
runs_on: macos-15
|
|
conan_host_profile: ${{ inputs.conan_profile_macos }}
|
|
conan_arch: armv8
|
|
needs_rosetta: false
|
|
- suffix: macos-x86_64
|
|
runs_on: macos-15
|
|
conan_host_profile: ${{ inputs.conan_profile_macos_x86 }}
|
|
conan_arch: x86_64
|
|
needs_rosetta: true
|
|
- suffix: windows-x86_64
|
|
runs_on: windows-latest
|
|
conan_host_profile: ${{ inputs.conan_profile_windows }}
|
|
conan_arch: x86_64
|
|
needs_rosetta: false
|
|
timeout: 60
|
|
runs-on: ${{ matrix.runs_on }}
|
|
timeout-minutes: ${{ matrix.timeout || 30 }}
|
|
env:
|
|
YAMS_DISABLE_FAISS: 1
|
|
steps:
|
|
- name: Checkout
|
|
uses: actions/checkout@v5
|
|
with:
|
|
fetch-depth: 0
|
|
submodules: false # Manual submodule init below (some use SSH URLs)
|
|
|
|
- name: Initialize required submodules
|
|
shell: bash
|
|
run: |
|
|
retry_cmd() {
|
|
local attempts="$1"
|
|
shift
|
|
local attempt=1
|
|
until "$@"; do
|
|
if [ "$attempt" -ge "$attempts" ]; then
|
|
return 1
|
|
fi
|
|
echo "Command failed (attempt ${attempt}/${attempts}): $*" >&2
|
|
sleep $((attempt * 5))
|
|
attempt=$((attempt + 1))
|
|
done
|
|
}
|
|
# Force HTTPS for GitHub to handle SSH submodule URLs in CI
|
|
git config --global url."https://github.com/".insteadOf "git@github.com:"
|
|
# Only init the build-required third_party submodules
|
|
# Skip other submodules that use SSH URLs requiring authentication
|
|
retry_cmd 4 git submodule update --init --depth 1 third_party/sqlite-vec-cpp
|
|
retry_cmd 4 git submodule update --init --depth 1 third_party/simeon
|
|
retry_cmd 4 git submodule update --init --depth 1 third_party/symspell
|
|
|
|
- name: Cache Conan packages (warm)
|
|
if: github.actor != 'nektos/act'
|
|
id: cache_conan_warm
|
|
uses: actions/cache@v4
|
|
continue-on-error: true
|
|
with:
|
|
path: |
|
|
~/.conan2/p
|
|
~/.conan2/r
|
|
~/.conan2/recipes
|
|
~/.conan2/metadata
|
|
~/.conan2/s
|
|
key: conan2-${{ runner.os }}-${{ matrix.suffix }}-${{ hashFiles('conanfile.py', 'conan/profiles/**') }}
|
|
restore-keys: |
|
|
conan2-${{ runner.os }}-${{ matrix.suffix }}-
|
|
conan2-${{ runner.os }}-linux-x86_64-
|
|
conan2-${{ runner.os }}-
|
|
|
|
- name: Cache ccache (warm)
|
|
if: github.actor != 'nektos/act'
|
|
id: cache_ccache_warm
|
|
uses: actions/cache@v4
|
|
continue-on-error: true
|
|
with:
|
|
path: |
|
|
~/.cache/ccache
|
|
~/.ccache
|
|
key: ccache-${{ runner.os }}-${{ matrix.suffix }}-${{ hashFiles('conanfile.py', 'meson.build', 'meson_options.txt') }}
|
|
restore-keys: |
|
|
ccache-${{ runner.os }}-${{ matrix.suffix }}-
|
|
ccache-${{ runner.os }}-linux-x86_64-
|
|
ccache-${{ runner.os }}-
|
|
|
|
- name: Install Conan and Meson
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
PYTOOLS_DIR="${RUNNER_TEMP}/yams-python-tools"
|
|
if [ "$RUNNER_OS" = "Windows" ]; then
|
|
VENV_PYTHON="$PYTOOLS_DIR/Scripts/python.exe"
|
|
TOOL_BIN="$PYTOOLS_DIR/Scripts"
|
|
if [ ! -f "$VENV_PYTHON" ]; then
|
|
python -m venv "$PYTOOLS_DIR"
|
|
fi
|
|
else
|
|
VENV_PYTHON="$PYTOOLS_DIR/bin/python"
|
|
TOOL_BIN="$PYTOOLS_DIR/bin"
|
|
if [ ! -x "$VENV_PYTHON" ]; then
|
|
python3 -m venv "$PYTOOLS_DIR"
|
|
fi
|
|
fi
|
|
"$VENV_PYTHON" -m pip install --upgrade pip
|
|
"$VENV_PYTHON" -m pip install --upgrade 'conan==2.21.*' 'meson==1.4.*' 'ninja==1.11.*'
|
|
echo "$TOOL_BIN" >> "$GITHUB_PATH"
|
|
export PATH="$TOOL_BIN:$PATH"
|
|
conan --version
|
|
meson --version
|
|
ninja --version
|
|
mkdir -p ~/.conan2
|
|
cp .conan/global.conf ~/.conan2/global.conf 2>/dev/null || true
|
|
|
|
- name: Prime Conan profiles
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
conan profile detect --force
|
|
# Use C++20 instead of C++23 to avoid pdfium compilation issues
|
|
if [ "$RUNNER_OS" = "macOS" ]; then
|
|
sed -i '' 's/compiler.cppstd=.*/compiler.cppstd=20/' ~/.conan2/profiles/default || true
|
|
else
|
|
sed -i 's/compiler.cppstd=.*/compiler.cppstd=20/' ~/.conan2/profiles/default || true
|
|
fi
|
|
conan profile show
|
|
|
|
- name: Pin conancenter to v2 index
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
if conan remote list | grep -q 'conancenter'; then
|
|
conan remote update conancenter --url https://center2.conan.io || true
|
|
else
|
|
conan remote add conancenter https://center2.conan.io
|
|
fi
|
|
conan remote list
|
|
|
|
- name: Install system dependencies (Linux)
|
|
if: runner.os == 'Linux'
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
retry_cmd() {
|
|
local attempts="$1"
|
|
shift
|
|
local attempt=1
|
|
until "$@"; do
|
|
if [ "$attempt" -ge "$attempts" ]; then
|
|
return 1
|
|
fi
|
|
echo "Command failed (attempt ${attempt}/${attempts}): $*" >&2
|
|
sleep $((attempt * 10))
|
|
attempt=$((attempt + 1))
|
|
done
|
|
}
|
|
if [ "$(dpkg --print-architecture)" = "arm64" ]; then
|
|
source_files="$(sudo grep -RIl '://ports.ubuntu.com/ubuntu-ports' \
|
|
/etc/apt/sources.list /etc/apt/sources.list.d 2>/dev/null || true)"
|
|
if [ -n "$source_files" ]; then
|
|
printf '%s\n' "$source_files" | xargs sudo sed -i \
|
|
's|http://ports.ubuntu.com/ubuntu-ports|https://azure.ports.ubuntu.com/ubuntu-ports|g; s|https://ports.ubuntu.com/ubuntu-ports|https://azure.ports.ubuntu.com/ubuntu-ports|g'
|
|
fi
|
|
fi
|
|
retry_cmd 4 sudo apt-get -o Acquire::Retries=5 -o Acquire::ForceIPv4=true update
|
|
retry_cmd 4 sudo apt-get -o Acquire::Retries=5 -o Acquire::ForceIPv4=true \
|
|
install -y --fix-missing --no-install-recommends libomp-18-dev ccache
|
|
|
|
- name: Warm dependencies (${{ inputs.build_type }})
|
|
# Always run on Linux to populate ccache (cache-hit can be true on a tiny
|
|
# restored cache). On macOS/Windows, skip if both caches are already warm.
|
|
if: runner.os == 'Linux' || steps.cache_conan_warm.outputs.cache-hit != 'true' || steps.cache_ccache_warm.outputs.cache-hit != 'true'
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
if command -v nproc >/dev/null 2>&1; then
|
|
export CONAN_CPU_COUNT=$(nproc)
|
|
elif command -v sysctl >/dev/null 2>&1; then
|
|
export CONAN_CPU_COUNT=$(sysctl -n hw.ncpu)
|
|
elif [ "$RUNNER_OS" = "Windows" ]; then
|
|
export CONAN_CPU_COUNT=$NUMBER_OF_PROCESSORS
|
|
elif command -v getconf >/dev/null 2>&1; then
|
|
export CONAN_CPU_COUNT=$(getconf _NPROCESSORS_ONLN)
|
|
else
|
|
export CONAN_CPU_COUNT=4
|
|
fi
|
|
ONNX=$([ "${{ inputs.enable_onnx }}" = 'true' ] && echo True || echo False)
|
|
conan install . \
|
|
-of build/warm-${{ inputs.build_type }} \
|
|
-pr:b=default \
|
|
-pr:h=${{ matrix.conan_host_profile }} \
|
|
-s build_type=${{ inputs.build_type }} \
|
|
-s:h arch=${{ matrix.conan_arch }} \
|
|
-o "sqlite3/*:fts5=True" \
|
|
-o "yams/*:enable_onnx=${ONNX}" \
|
|
--build=missing
|
|
|
|
- name: Populate ccache with source compilation
|
|
if: runner.os == 'Linux'
|
|
continue-on-error: true
|
|
shell: bash
|
|
env:
|
|
CCACHE_DIR: ~/.cache/ccache
|
|
CCACHE_BASEDIR: ${{ github.workspace }}
|
|
CCACHE_COMPRESS: '1'
|
|
CCACHE_MAXSIZE: 5G
|
|
CCACHE_SLOPPINESS: time_macros,include_file_mtime,include_file_ctime,file_macro
|
|
run: |
|
|
set -euo pipefail
|
|
CACHE_BYTES=$(du -sb ~/.cache/ccache 2>/dev/null | cut -f1 || echo 0)
|
|
CACHE_MB=$((CACHE_BYTES / 1048576))
|
|
if [ "$CACHE_MB" -gt 100 ]; then
|
|
echo "ccache already populated: ${CACHE_MB} MB; skipping source compile"
|
|
exit 0
|
|
fi
|
|
echo "ccache=${CACHE_MB} MB (< 100 MB); compiling source to populate cache"
|
|
ccache -z >/dev/null 2>&1 || true
|
|
BUILD_DIR="build/warm-${{ inputs.build_type }}"
|
|
NATIVE_FILE=$(find "$BUILD_DIR" -name conan_meson_native.ini -type f 2>/dev/null | head -1)
|
|
if [ -z "$NATIVE_FILE" ]; then
|
|
echo "No Meson native file found in $BUILD_DIR; skipping source compile"
|
|
exit 0
|
|
fi
|
|
echo "Using Conan native file: $NATIVE_FILE"
|
|
MESON_BUILDTYPE=$( [ "${{ inputs.build_type }}" = "Debug" ] && echo "--buildtype=debug" || echo "--buildtype=release" )
|
|
if ! meson setup "$BUILD_DIR" --native-file="$NATIVE_FILE" \
|
|
$MESON_BUILDTYPE --reconfigure 2>&1; then
|
|
meson setup "$BUILD_DIR" --native-file="$NATIVE_FILE" $MESON_BUILDTYPE 2>&1
|
|
fi
|
|
meson compile -C "$BUILD_DIR" -j$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 4)
|
|
echo
|
|
echo "=== ccache stats after warm compilation ==="
|
|
ccache -s | head -8
|
|
|
|
- name: Warm summary
|
|
shell: bash
|
|
run: |
|
|
echo "Conan warm cache hit: ${{ steps.cache_conan_warm.outputs.cache-hit }}" || true
|
|
du -sh ~/.conan2/p 2>/dev/null || true
|
|
du -sh ~/.cache/ccache 2>/dev/null || true
|