# yamllint disable rule:line-length name: Release 'on': push: tags: - "v*" - "yams-v*" schedule: - cron: '0 6 * * *' # 06:00 UTC daily workflow_dispatch: inputs: channel: description: "Release channel (stable|weekly|nightly)" required: false default: "nightly" fast_mode: description: "Skip packaging & verification steps (speed up local / act runs)" required: false default: "false" macos_only: description: "Limit matrix to macOS variants only (skip linux-hosted)" required: false default: "false" permissions: contents: write jobs: check-commits: name: Check for new commits (nightly only) runs-on: ubuntu-latest if: github.event_name == 'schedule' && github.actor != 'nektos/act' outputs: has_new_commits: ${{ steps.check.outputs.has_new_commits }} last_release_sha: ${{ steps.check.outputs.last_release_sha }} steps: - name: Check if there are new commits since last nightly release id: check uses: actions/github-script@v7 with: script: | const owner = context.repo.owner; const repo = context.repo.repo; const releases = await github.paginate(github.rest.repos.listReleases, { owner, repo, per_page: 100, }); const latest = releases.find((r) => r.tag_name?.startsWith('nightly-')); if (!latest) { core.info('No previous nightly release found, will create one'); core.setOutput('has_new_commits', 'true'); core.setOutput('last_release_sha', 'none'); return; } const tagName = latest.tag_name; const tagRef = await github.rest.git.getRef({ owner, repo, ref: `tags/${tagName}`, }); let lastSha = tagRef.data.object.sha; if (tagRef.data.object.type === 'tag') { const tagObj = await github.rest.git.getTag({ owner, repo, tag_sha: lastSha, }); lastSha = tagObj.data.object.sha; } core.setOutput('last_release_sha', lastSha); const headRef = await github.rest.git.getRef({ owner, repo, ref: 'heads/main', }); const currentSha = headRef.data.object.sha; if (lastSha !== currentSha) { const compare = await github.rest.repos.compareCommits({ owner, repo, base: lastSha, head: currentSha, }); const total = compare.data.total_commits ?? 1; core.setOutput('has_new_commits', 'true'); core.info(`Found ${total} new commit(s) since last nightly (${tagName})`); } else { core.setOutput('has_new_commits', 'false'); core.info(`No new commits since last nightly release (${tagName}), skipping`); } warm: needs: [check-commits] if: | always() && github.actor != 'nektos/act' && (needs.check-commits.result == 'skipped' || needs.check-commits.outputs.has_new_commits == 'true' || github.event_name == 'push' || github.event_name == 'workflow_dispatch') uses: ./.github/workflows/matrix-warm.yml with: build_type: Release enable_onnx: false build-release: name: Build and Package (matrix) needs: warm if: | always() && (needs.warm.result == 'success' || needs.warm.result == 'skipped') strategy: fail-fast: false matrix: include: - os: linux-hosted runs_on: ubuntu-24.04 suffix: linux-x86_64 packaging: tar build_packages: true meson_args: "" hosted: true - os: linux-hosted-arm64 runs_on: ubuntu-24.04-arm suffix: linux-arm64 packaging: tar build_packages: true meson_args: "" hosted: true - os: macos-hosted-arm64 runs_on: macos-15 suffix: macos-arm64 packaging: zip build_packages: false meson_args: "" # arch via profile edit hosted: true - os: macos-hosted-x64 runs_on: macos-15 suffix: macos-x86_64 packaging: zip build_packages: false meson_args: "" hosted: true - os: windows-hosted runs_on: windows-latest suffix: windows-x86_64 packaging: msi build_packages: true meson_args: "" hosted: true runs-on: ${{ matrix.runs_on }} env: STAGE_DIR: stage steps: - name: Checkout uses: actions/checkout@v4 with: fetch-depth: 0 submodules: false # Manual submodule init below (some use SSH URLs) - name: Initialize required submodules shell: bash run: | retry_cmd() { local attempts="$1" shift local attempt=1 until "$@"; do if [ "$attempt" -ge "$attempts" ]; then return 1 fi echo "Command failed (attempt ${attempt}/${attempts}): $*" >&2 sleep $((attempt * 5)) attempt=$((attempt + 1)) done } # Force HTTPS for GitHub to handle SSH submodule URLs in CI git config --global url."https://github.com/".insteadOf "git@github.com:" # Init submodules required for the build # Skip other submodules that use SSH URLs requiring authentication retry_cmd 4 git submodule update --init --depth 1 third_party/sqlite-vec-cpp retry_cmd 4 git submodule update --init --depth 1 third_party/simeon retry_cmd 4 git submodule update --init --depth 1 third_party/symspell - name: Skip non-linux entries under act (build) if: github.actor == 'nektos/act' && !startsWith(matrix.os, 'linux-hosted') shell: bash run: | echo "Skipping build-release job steps for ${{ matrix.os }} under act" exit 0 - name: Skip linux-hosted when macos_only=true if: inputs.macos_only == 'true' && matrix.os == 'linux-hosted' shell: bash run: | echo "macos_only=true: skipping linux-hosted build-release job entry" exit 0 - name: Verify runner architecture (act diagnostics) if: github.actor == 'nektos/act' && matrix.os == 'linux-hosted' shell: bash run: | set -euo pipefail ACT_UNAME="$(uname -m || true)" echo "Detected uname -m: ${ACT_UNAME}" if [ "${ACT_UNAME}" != "x86_64" ]; then echo "ERROR: Non-x86_64 architecture detected under act: ${ACT_UNAME}" >&2 echo "cpuinfo (and other x86-specific code) will not build on this architecture." >&2 echo "Re-run forcing amd64 containers, e.g.:" >&2 echo " act --container-architecture linux/amd64 --job build-release --input fast_mode=true" >&2 exit 1 fi - name: Cache Conan packages if: github.actor != 'nektos/act' id: cache_conan uses: actions/cache@v4 continue-on-error: true with: path: | ~/.conan2/p ~/.conan2/r ~/.conan2/recipes ~/.conan2/metadata ~/.conan2/s key: conan2-${{ runner.os }}-${{ matrix.suffix }}-${{ hashFiles('conanfile.py', 'meson.build', 'meson_options.txt') }} restore-keys: | conan2-${{ runner.os }}-${{ matrix.suffix }}- conan2-${{ runner.os }}- - name: Conan cache diagnostics if: always() shell: bash run: | echo "Cache hit: ${{ steps.cache_conan.outputs.cache-hit }}" for d in p r recipes metadata s; do if [ -d "$HOME/.conan2/$d" ]; then echo "Directory $d present: $(find $HOME/.conan2/$d -mindepth 1 -maxdepth 1 | wc -l | tr -d ' ')" entries || true du -sh $HOME/.conan2/$d 2>/dev/null || true else echo "Directory $d missing" fi done - name: Cache macOS build directory if: runner.os == 'macOS' && github.actor != 'nektos/act' uses: actions/cache@v4 with: path: build/release key: macos-build-${{ matrix.suffix }}-${{ hashFiles('meson.build', 'meson_options.txt') }}-${{ hashFiles('src/**', 'include/**') }} restore-keys: | macos-build-${{ matrix.suffix }}- - name: Cache ccache if: github.actor != 'nektos/act' uses: actions/cache@v4 continue-on-error: true with: path: | ~/.cache/ccache ~/.ccache key: ccache-${{ runner.os }}-${{ matrix.suffix }}-${{ hashFiles('conanfile.py', 'meson.build', 'meson_options.txt') }} restore-keys: | ccache-${{ runner.os }}-${{ matrix.suffix }}- ccache-${{ runner.os }}- - name: Install optional system dependencies (Linux only) if: runner.os == 'Linux' continue-on-error: true shell: bash run: | set -euo pipefail sudo apt-get update -o Acquire::Retries=3 || sudo apt-get update -o Acquire::Retries=3 || true # Add rpm / fakeroot / dpkg-dev so CPack can emit .deb and .rpm packages sudo apt-get install -y --no-install-recommends \ liburing-dev \ libarchive-dev \ libtag1-dev \ libboost-all-dev \ pkg-config \ ccache \ rpm \ fakeroot \ dpkg-dev \ meson \ cmake \ ninja-build \ clang \ clang-tools \ xz-utils \ lld || true echo "Installed optional packages: liburing-dev libarchive-dev libtag1-dev libboost-all-dev ccache rpm fakeroot dpkg-dev meson cmake ninja-build clang clang-tools lld" # Ensure compilers are preferred echo "CC=clang" >> $GITHUB_ENV echo "CXX=clang++" >> $GITHUB_ENV - name: Install Zig (Linux) if: runner.os == 'Linux' shell: bash run: | set -euo pipefail ARCH=$(uname -m) case "$ARCH" in x86_64) ZIG_ARCH="x86_64" ;; aarch64) ZIG_ARCH="aarch64" ;; *) echo "Unsupported architecture: $ARCH"; exit 1 ;; esac ZIG_VERSION="0.15.2" curl -fsSL "https://ziglang.org/download/${ZIG_VERSION}/zig-${ZIG_ARCH}-linux-${ZIG_VERSION}.tar.xz" -o /tmp/zig.tar.xz sudo tar -xJf /tmp/zig.tar.xz -C /opt sudo mv /opt/zig-${ZIG_ARCH}-linux-${ZIG_VERSION} /opt/zig sudo ln -sf /opt/zig/zig /usr/local/bin/zig rm /tmp/zig.tar.xz zig version - name: Sanitize stdlib flags (macOS only) if: runner.os == 'macOS' shell: bash run: | set -euo pipefail # Remove any accidentally injected libstdc++ selections and force libc++ for var in CXXFLAGS CFLAGS LDFLAGS; do current="${!var:-}" sanitized="$(echo "$current" | sed -E 's/-stdlib=libstdc\+\+//g') -stdlib=libc++" # Collapse whitespace sanitized="$(echo "$sanitized" | tr -s ' ')" echo "$var=$sanitized" >> "$GITHUB_ENV" printf '%s\n' "$var=$sanitized" done echo "Stdlib flags sanitized for macOS (forcing -stdlib=libc++)." - name: Determine release channel and version/tag id: meta shell: bash run: | set -euo pipefail EVENT="${{ github.event_name }}" SCHEDULE_CRON="${{ github.event.schedule || '' }}" INPUT_CHANNEL="${{ inputs.channel || '' }}" SHORT_HASH=$(git rev-parse --short HEAD) channel="" if [ "$EVENT" = "push" ]; then channel="stable" elif [ "$EVENT" = "workflow_dispatch" ] && [ -n "$INPUT_CHANNEL" ]; then case "$INPUT_CHANNEL" in stable|weekly|nightly) channel="$INPUT_CHANNEL" ;; *) channel="nightly" ;; esac elif [ "$EVENT" = "schedule" ]; then channel="nightly" else channel="stable" fi # Derive version and tag if [ "$channel" = "stable" ]; then TAG="${GITHUB_REF_NAME}" VERSION="${TAG#yams-v}" if [ "$VERSION" = "$TAG" ]; then VERSION="${TAG#v}" fi TAG_OUT="$TAG" RELEASE_NAME="YAMS ${VERSION}" elif [ "$channel" = "nightly" ]; then DATE_ISO=$(date -u +%Y-%m-%d) DATE_COMPACT=$(date -u +%Y%m%d) VERSION="nightly-${DATE_COMPACT}-${SHORT_HASH}" TAG_OUT="nightly-${DATE_COMPACT}-${SHORT_HASH}" RELEASE_NAME="YAMS Nightly ${DATE_ISO} (${SHORT_HASH})" else # weekly YEAR=$(date -u +%G) # ISO week-numbering year WEEK=$(date -u +%V) # ISO week number 01-53 VERSION="weekly-${YEAR}w${WEEK}-${SHORT_HASH}" TAG_OUT="$VERSION" RELEASE_NAME="YAMS Weekly ${YEAR}-w${WEEK} (${SHORT_HASH})" fi BASE_NUMERIC_VERSION="$(git describe --tags --abbrev=0 --match='v*' 2>/dev/null || echo 0.0.0)" BASE_NUMERIC_VERSION="${BASE_NUMERIC_VERSION#v}" echo "channel=$channel" >> "$GITHUB_OUTPUT" echo "version=$VERSION" >> "$GITHUB_OUTPUT" echo "tag=$TAG_OUT" >> "$GITHUB_OUTPUT" echo "release_name=$RELEASE_NAME" >> "$GITHUB_OUTPUT" echo "base_version=$BASE_NUMERIC_VERSION" >> "$GITHUB_OUTPUT" if [ "$channel" = "stable" ]; then echo "prerelease=false" >> "$GITHUB_OUTPUT" else echo "prerelease=true" >> "$GITHUB_OUTPUT" fi echo "Detected channel=$channel, version=$VERSION, tag=$TAG_OUT, name=$RELEASE_NAME" - name: Download CI benchmark artifacts (-yams, if available) if: github.event_name == 'workflow_run' && matrix.os == 'linux-hosted' && github.actor != 'nektos/act' uses: dawidd6/action-download-artifact@v6 continue-on-error: true with: run_id: ${{ github.event.workflow_run.id }} workflow_conclusion: completed name: benchmark-results-linux-hosted-yams path: ci-artifacts/benchmarks if_no_artifact_found: warn - name: Download CI benchmark artifacts (legacy -conan, fallback) if: github.event_name == 'workflow_run' && matrix.os == 'linux-hosted' && github.actor != 'nektos/act' uses: dawidd6/action-download-artifact@v6 continue-on-error: true with: run_id: ${{ github.event.workflow_run.id }} workflow_conclusion: completed name: benchmark-results-linux-hosted-conan path: ci-artifacts/benchmarks if_no_artifact_found: warn - name: Download CI coverage (if available) if: github.event_name == 'workflow_run' && matrix.os == 'linux-hosted' && github.actor != 'nektos/act' uses: dawidd6/action-download-artifact@v6 continue-on-error: true with: run_id: ${{ github.event.workflow_run.id }} workflow_conclusion: completed name: coverage-report path: ci-artifacts/coverage if_no_artifact_found: warn # Hosted runners: install and configure Conan (self-hosted must already be configured) - name: Install Conan (hosted only) if: ${{ matrix.hosted }} shell: bash run: | set -euo pipefail PYTOOLS_DIR="${RUNNER_TEMP}/yams-python-tools" if [ "$RUNNER_OS" = "Windows" ]; then VENV_PYTHON="$PYTOOLS_DIR/Scripts/python.exe" TOOL_BIN="$PYTOOLS_DIR/Scripts" if [ ! -f "$VENV_PYTHON" ]; then python -m venv "$PYTOOLS_DIR" fi else VENV_PYTHON="$PYTOOLS_DIR/bin/python" TOOL_BIN="$PYTOOLS_DIR/bin" if [ ! -x "$VENV_PYTHON" ]; then python3 -m venv "$PYTOOLS_DIR" fi fi "$VENV_PYTHON" -m pip install --upgrade pip "$VENV_PYTHON" -m pip install --upgrade conan meson 'ninja>=1.10' echo "$TOOL_BIN" >> "$GITHUB_PATH" export PATH="$TOOL_BIN:$PATH" conan --version meson --version ninja --version mkdir -p ~/.conan2 # Best-effort: global.conf may not exist in repo for hosted macOS cp .conan/global.conf ~/.conan2/global.conf 2>/dev/null || true - name: Ensure build tools (meson, ninja) for local runners/act if: ${{ matrix.hosted && runner.os != 'Windows' }} shell: bash run: | set -euo pipefail if [ "$RUNNER_OS" = "macOS" ]; then brew update || true brew install meson ninja ccache || true elif [ "$RUNNER_OS" = "Linux" ]; then if command -v meson >/dev/null 2>&1 && command -v ninja >/dev/null 2>&1; then echo "meson/ninja already available on PATH" else sudo apt-get update || true sudo apt-get install -y --no-install-recommends meson ninja-build || true fi if ! command -v meson >/dev/null 2>&1 || ! command -v ninja >/dev/null 2>&1; then python3 -m pip install --user --upgrade --break-system-packages meson 'ninja>=1.10' || \ python -m pip install --user --upgrade --break-system-packages meson 'ninja>=1.10' || true fi if [ -n "${GITHUB_PATH:-}" ]; then echo "$HOME/.local/bin" >> "$GITHUB_PATH" else export PATH="$HOME/.local/bin:$PATH" fi fi echo "meson version: $(meson --version || true)" echo "ninja version: $(ninja --version || true)" - name: Ensure meson and ninja (macOS) if: runner.os == 'macOS' shell: bash run: | set -euo pipefail # If meson or ninja are not on PATH, install via Homebrew for robustness NEED_INSTALL=0 command -v meson >/dev/null 2>&1 || NEED_INSTALL=1 command -v ninja >/dev/null 2>&1 || NEED_INSTALL=1 if [ "$NEED_INSTALL" -eq 1 ]; then brew update || true brew install meson ninja || true fi echo "meson version (post-check): $(meson --version || echo 'missing')" echo "ninja version (post-check): $(ninja --version || echo 'missing')" - name: Setup ccache env shell: bash run: | echo "CCACHE_DIR=$HOME/.cache/ccache" >> $GITHUB_ENV echo "CCACHE_BASEDIR=$GITHUB_WORKSPACE" >> $GITHUB_ENV echo "CCACHE_COMPRESS=1" >> $GITHUB_ENV echo "CCACHE_MAXSIZE=700M" >> $GITHUB_ENV ccache -z 2>/dev/null || true - name: Setup Conan (hosted Linux/macOS) if: ${{ matrix.hosted }} shell: bash run: | set -e conan profile detect --force echo "Conan detected default profile. A platform-specific profile (host-macos-apple-clang or host-linux-clang) will be applied explicitly later." - name: Detect clang version (log only) shell: bash run: | set -euo pipefail # Prefer clang++, fallback to clang if command -v clang++ >/dev/null 2>&1; then RAW_VER=$(clang++ --version | sed -E -n 's/.*version ([0-9]+).*/\1/p' | head -1 || true) elif command -v clang >/dev/null 2>&1; then RAW_VER=$(clang --version | sed -E -n 's/.*version ([0-9]+).*/\1/p' | head -1 || true) else RAW_VER="" fi if [ -z "$RAW_VER" ]; then echo "Could not auto-detect clang version; using profile defaults" >&2 else echo "Detected clang major: $RAW_VER" fi - name: Compare profile vs detected clang (advice) shell: bash run: | set -euo pipefail PROFILE_FILE="./conan/profiles/host-linux-clang" if [ -f "$PROFILE_FILE" ]; then PROFILE_VER=$(grep -E '^compiler.version=' "$PROFILE_FILE" | sed -E 's/.*=//') else PROFILE_VER="(missing profile)" fi if command -v clang++ >/dev/null 2>&1; then SYS_VER=$(clang++ --version | sed -E -n 's/.*version ([0-9]+).*/\1/p' | head -1 || true) elif command -v clang >/dev/null 2>&1; then SYS_VER=$(clang --version | sed -E -n 's/.*version ([0-9]+).*/\1/p' | head -1 || true) else SYS_VER="unknown" fi echo "Profile declares clang version: $PROFILE_VER" echo "System detected clang version: $SYS_VER" if [ -n "$SYS_VER" ] && [ "$SYS_VER" != "unknown" ] && [ "$PROFILE_VER" != "$SYS_VER" ]; then echo "NOTE: Mismatch between profile (clang $PROFILE_VER) and system (clang $SYS_VER)." >&2 echo " Consider either installing clang-$PROFILE_VER or adjusting profile to $SYS_VER for local/act runs to avoid subtle ABI or flag differences." >&2 fi - name: Check Conan (self-hosted sanity) if: ${{ !matrix.hosted }} shell: bash run: conan --version || echo "Conan is expected to be pre-installed and configured on self-hosted runners" # Hosted runners: clean corrupted cache if needed - name: Clean corrupted Conan cache if needed (hosted only) if: ${{ matrix.hosted }} shell: bash run: | if ! conan list "*" 2>/dev/null; then echo "Conan cache appears corrupted, cleaning locks and temp files..." conan cache clean --locks --temp 2>/dev/null || true echo "Conan cache locks/temp cleaned" fi - name: Install system dependencies (macOS) if: runner.os == 'macOS' shell: bash run: | set -euo pipefail for attempt in 1 2 3; do if brew update; then break fi if [ "$attempt" -eq 3 ]; then echo "brew update failed after ${attempt} attempts" >&2 exit 1 fi sleep $((attempt * 10)) done brew install pkg-config cmake ninja meson ccache boost zig || true # Ensure Meson/Ninja are present even if Homebrew failed to link binaries. command -v meson >/dev/null 2>&1 || python3 -m pip install --user --upgrade meson command -v ninja >/dev/null 2>&1 || python3 -m pip install --user --upgrade 'ninja>=1.10' echo "meson version: $(meson --version || echo 'missing')" echo "ninja version: $(ninja --version || echo 'missing')" echo "zig version: $(zig version || echo 'missing')" - name: Tune Conan client (parallel downloads) shell: bash run: | set -euo pipefail CONF_FILE="$HOME/.conan2/global.conf" LINE="core.download:parallel=8" mkdir -p "$(dirname "$CONF_FILE")" if [ -f "$CONF_FILE" ]; then TMP_FILE="$(mktemp)" awk -v line="$LINE" ' BEGIN { replaced = 0 } /^core.download:parallel=/ { if (!replaced) { print line; replaced = 1 } ; next } { print } END { if (!replaced) print line } ' "$CONF_FILE" > "$TMP_FILE" mv "$TMP_FILE" "$CONF_FILE" else printf '%s\n' "$LINE" > "$CONF_FILE" fi echo "Configured $LINE in $CONF_FILE" grep -n '^core.download:parallel=' "$CONF_FILE" || true conan config show core.download:parallel 2>/dev/null || true conan --version - name: Warm key Conan packages (best-effort) if: ${{ matrix.hosted }} shell: bash run: | set +e # Try to prefetch heavy packages to minimize build-time compiles for ref in \ openjpeg/2.5.3 \ pdfium/95.0.4629 \ onnxruntime/1.18.1 \ icu/74.1 \ libarchive/3.7.6; do echo "Prefetching $ref ..." || true conan download "$ref" -r=conancenter || true done set -e # Windows: Install MSVC and build tools - name: Setup MSVC (Windows only) if: runner.os == 'Windows' uses: ilammy/msvc-dev-cmd@v1 with: arch: x64 - name: Install build tools (Windows only) if: runner.os == 'Windows' shell: pwsh run: | $pyToolsScripts = Join-Path $env:RUNNER_TEMP "yams-python-tools\Scripts" "$pyToolsScripts" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append $env:PATH = "$pyToolsScripts;$env:PATH" # Install WiX for MSI packaging (pin versions to avoid prerelease incompatibility) dotnet tool install --global wix --version 6.0.2 wix extension add -g WixToolset.UI.wixext/6.0.2 # Install Zig for zyp PDF plugin $zigVersion = "0.15.2" $zigUrl = "https://ziglang.org/download/${zigVersion}/zig-x86_64-windows-${zigVersion}.zip" $zigZip = "$env:TEMP\zig.zip" $zigDir = "C:\zig" Invoke-WebRequest -Uri $zigUrl -OutFile $zigZip Expand-Archive -Path $zigZip -DestinationPath $zigDir -Force Move-Item "$zigDir\zig-x86_64-windows-${zigVersion}\*" $zigDir -Force Remove-Item "$zigDir\zig-x86_64-windows-${zigVersion}" -Recurse -ErrorAction SilentlyContinue $env:PATH = "$zigDir;$env:PATH" [Environment]::SetEnvironmentVariable("PATH", "$zigDir;$env:PATH", "Process") "$zigDir" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append # Verify tools conan --version meson --version ninja --version wix --version zig version - name: Configure and Build Release (Windows) if: runner.os == 'Windows' shell: pwsh run: | $ErrorActionPreference = "Stop" Write-Host "Building release for Windows x86_64" # Set version $env:YAMS_EXTRA_MESON_FLAGS = "-Dyams-version=${{ steps.meta.outputs.version }}" # Run the build via setup.ps1 .\setup.ps1 -BuildType Release # Set BUILD_DIR for subsequent steps "BUILD_DIR=build/release" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append # Release build for Linux/macOS targets - name: Configure and Build Release (Optimized) if: runner.os != 'Windows' shell: bash run: | echo "Optimized release build for ${{ matrix.suffix }}" # Align build directory with README (build/release) rm -rf build/release # Determine host profile and architecture based on platform if [ "$RUNNER_OS" = "macOS" ]; then case "${{ matrix.suffix }}" in macos-arm64) HOST_PROFILE="./conan/profiles/host-macos-apple-clang" ARCH="armv8" ;; macos-x86_64) HOST_PROFILE="./conan/profiles/host-macos-apple-clang-x86" ARCH="x86_64" ;; esac elif [ "$RUNNER_OS" = "Linux" ]; then HOST_PROFILE="./conan/profiles/host-linux-clang" case "${{ matrix.suffix }}" in linux-arm64) ARCH="armv8" ;; linux-x86_64) ARCH="x86_64" ;; esac fi # Use setup.sh with CI environment variables for consistent configuration export YAMS_CONAN_HOST_PROFILE="$HOST_PROFILE" export YAMS_CONAN_ARCH="$ARCH" export YAMS_ENABLE_MOBILE_BINDINGS=false export YAMS_EXTRA_MESON_FLAGS="-Dyams-version=${{ steps.meta.outputs.version }} -Dwerror=false -Dwarning_level=2 -Denable-lzma=auto ${{ matrix.meson_args }}" # Set install prefix: /usr for Linux packages, /usr/local for macOS if [ "$RUNNER_OS" = "Linux" ]; then export YAMS_INSTALL_PREFIX="/usr" else if [ "${{ matrix.suffix }}" = "macos-arm64" ]; then export YAMS_INSTALL_PREFIX="/opt/homebrew" else export YAMS_INSTALL_PREFIX="/usr/local" fi # Enable CoreML GPU acceleration for macOS builds export YAMS_ONNX_GPU="coreml" fi # Persist prefix for later steps echo "YAMS_INSTALL_PREFIX=$YAMS_INSTALL_PREFIX" >> "$GITHUB_ENV" # For cross-compilation, ensure pkg-config is available export PKG_CONFIG="$(command -v pkg-config)" echo "Using pkg-config: $PKG_CONFIG" # Version is already included in YAMS_EXTRA_MESON_FLAGS above. Keep this # echo for diagnostics, but do not overwrite feature flags. VERSION="${{ steps.meta.outputs.version }}" echo "Setting version: ${VERSION}" ./setup.sh Release # Source Conan build env so meson compile uses the same Meson version # that setup.sh used for meson setup (Conan provides Meson 1.10.x which # creates build.dat incompatible with older system Meson). CONAN_BUILD_ENV="" for candidate in \ "build/release/build-release/conan/conanbuild.sh" \ "build/release/conan/conanbuild.sh" do if [ -f "$candidate" ]; then CONAN_BUILD_ENV="$candidate" break fi done if [ -n "$CONAN_BUILD_ENV" ]; then set +u source "$CONAN_BUILD_ENV" set -u fi meson --version meson compile -C build/release echo "BUILD_DIR=build/release" >> $GITHUB_ENV - name: Upload build logs (artifacts) if: failure() uses: actions/upload-artifact@v4 with: name: logs-${{ matrix.suffix }}-${{ github.run_number }} path: | build/release/meson-logs/ build/release/meson-info/ build/release/meson-logs/meson-log.txt build/release/build-release/conan/**/*.log if-no-files-found: ignore retention-days: 14 - name: ccache stats (post-build) shell: bash run: | ccache -s || echo "ccache stats unavailable" - name: Collect benchmark results shell: bash run: | mkdir -p "$BUILD_DIR/bench_results" # Collect from validation build (if Linux hosted ran validation) if [ -f "build/release/bench_results.json" ]; then cp build/release/bench_results.json "$BUILD_DIR/bench_results/" echo "Collected benchmark results from validation build" fi # Also check CI artifacts if available if [ -d "ci-artifacts/benchmarks" ] && [ "$(ls -A ci-artifacts/benchmarks/*.json 2>/dev/null)" ]; then cp ci-artifacts/benchmarks/*.json "$BUILD_DIR/bench_results/" || true fi - name: Install (Linux/macOS) if: runner.os != 'Windows' shell: bash run: | # Ensure BUILD_DIR is properly set if [ ! -d "$BUILD_DIR" ]; then echo "BUILD_DIR not found at $BUILD_DIR, attempting fallback to build/release (legacy path build/yams-release no longer used)." >&2 exit 1 fi # Install to an absolute stage path under the build dir to avoid path duplication BD="$(cd "$BUILD_DIR" && pwd)" CONAN_BUILD_ENV="" for candidate in \ "$BD/build-release/conan/conanbuild.sh" \ "$BD/conan/conanbuild.sh" do if [ -f "$candidate" ]; then CONAN_BUILD_ENV="$candidate" break fi done if [ -n "$CONAN_BUILD_ENV" ]; then set +u source "$CONAN_BUILD_ENV" set -u fi meson --version # --no-rebuild: setup.sh + a prior `meson compile` already built; # this step only stages files, never re-triggers the compiler. meson install -C "$BD" --destdir "$BD/stage" --no-rebuild if [ ! -d "$BD/stage" ]; then echo "Stage directory missing after meson install" >&2 ls -al "$BD" >&2 || true exit 1 fi echo "STAGE_DIR=$BD/stage" >> "$GITHUB_ENV" - name: Prune staged runtime payload (Linux/macOS) if: runner.os != 'Windows' shell: bash run: | set -euo pipefail : "${STAGE_DIR:?STAGE_DIR not set}" PREFIX="${YAMS_INSTALL_PREFIX:-}" if [ -z "$PREFIX" ]; then if [ "$RUNNER_OS" = "Linux" ]; then PREFIX="/usr" else PREFIX="/usr/local" fi fi INSTALL_ROOT="$STAGE_DIR$PREFIX" if [ ! -d "$INSTALL_ROOT" ]; then echo "Missing staged install root: $INSTALL_ROOT" >&2 exit 1 fi bash scripts/prune-runtime-install.sh "$INSTALL_ROOT" - name: Verify staged runtime assets (Linux/macOS) if: runner.os != 'Windows' shell: bash run: | set -euo pipefail : "${STAGE_DIR:?STAGE_DIR not set}" PREFIX="${YAMS_INSTALL_PREFIX:-}" if [ -z "$PREFIX" ]; then if [ "$RUNNER_OS" = "Linux" ]; then PREFIX="/usr" else PREFIX="/usr/local" fi fi if [ ! -f "$STAGE_DIR$PREFIX/share/yams/sql/reference_schema.sql" ]; then echo "Missing staged reference schema: $STAGE_DIR$PREFIX/share/yams/sql/reference_schema.sql" >&2 echo "Stage tree (top):" >&2 ls -al "$STAGE_DIR" >&2 || true exit 1 fi # Plugins are expected for full releases; fail loudly if absent/empty. if [ ! -d "$STAGE_DIR$PREFIX/lib/yams/plugins" ]; then echo "Missing staged plugins dir: $STAGE_DIR$PREFIX/lib/yams/plugins" >&2 exit 1 fi shopt -s nullglob plugin_bins=( "$STAGE_DIR$PREFIX/lib/yams/plugins"/*.so "$STAGE_DIR$PREFIX/lib/yams/plugins"/*.dylib ) if [ ${#plugin_bins[@]} -eq 0 ]; then echo "No plugin binaries staged under: $STAGE_DIR$PREFIX/lib/yams/plugins" >&2 ls -al "$STAGE_DIR$PREFIX/lib/yams/plugins" >&2 || true exit 1 fi - name: Install and Package MSI (Windows) if: runner.os == 'Windows' && inputs.fast_mode != 'true' shell: pwsh run: | $ErrorActionPreference = "Stop" $buildDir = "build\release" $stageDir = Join-Path (Resolve-Path $buildDir).Path "stage" $version = "${{ steps.meta.outputs.version }}" Write-Host "Staging install to: $stageDir" # Remove old stage if exists if (Test-Path $stageDir) { Remove-Item -Recurse -Force $stageDir } New-Item -ItemType Directory -Path $stageDir -Force | Out-Null # Reconfigure if needed (handles Meson version mismatches from cached build dirs) $mesonInfo = Join-Path $buildDir "meson-private\coredata.dat" if (Test-Path $mesonInfo) { Write-Host "Reconfiguring build directory to ensure Meson version compatibility..." meson setup --reconfigure $buildDir if ($LASTEXITCODE -ne 0) { Write-Warning "Reconfigure failed, attempting fresh setup..." Remove-Item -Recurse -Force $buildDir -ErrorAction SilentlyContinue meson setup $buildDir --buildtype=release } } # Install to stage directory. # --no-rebuild: setup.ps1's release build already compiled; this # step only stages files, never re-triggers the compiler. meson install -C $buildDir --destdir $stageDir --no-rebuild if ($LASTEXITCODE -ne 0) { Write-Error "Meson install failed" exit 1 } # Build MSI using the packaging script $msiScript = "packaging\windows\build-msi.ps1" & $msiScript -StageDir $stageDir -Version $version -OutputDir $buildDir if ($LASTEXITCODE -ne 0) { Write-Error "MSI build failed" exit 1 } # Move MSI to workspace root for artifact upload Get-ChildItem $buildDir -Filter "*.msi" | ForEach-Object { Copy-Item $_.FullName -Destination $env:GITHUB_WORKSPACE Write-Host "Created MSI: $($_.Name)" } # Set environment variables for subsequent steps "STAGE_DIR=$stageDir" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append "BUILD_DIR=$buildDir" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append - name: Generate winget manifest (stable, windows only) if: runner.os == 'Windows' && steps.meta.outputs.channel == 'stable' && inputs.fast_mode != 'true' shell: pwsh run: | $ErrorActionPreference = "Stop" $version = "${{ steps.meta.outputs.version }}" $tag = "${{ steps.meta.outputs.tag }}" # Find the Windows MSI produced in this job $msiFile = Get-ChildItem -Path $env:GITHUB_WORKSPACE -Filter "*.msi" | Select-Object -First 1 if (-not $msiFile) { Write-Error "No MSI file found in workspace; cannot generate winget manifest" exit 1 } Write-Host "Generating winget manifest for version $version" Write-Host "MSI file: $($msiFile.FullName)" & ./packaging/windows/generate-winget-manifest.ps1 ` -Version $version ` -Tag $tag ` -MsiPath $msiFile.FullName ` -GitHubRepo "${{ github.repository }}" ` -OutputDir "winget-manifests" if ($LASTEXITCODE -ne 0) { Write-Error "Winget manifest generation failed" exit 1 } - name: Generate release summary shell: bash run: | echo "### Artifacts" >> "$GITHUB_STEP_SUMMARY" echo "[View artifacts for this run]($GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID)" >> "$GITHUB_STEP_SUMMARY" SUMMARY_FILE="$BUILD_DIR/release_summary.md" echo "## Release Build Information (${{ steps.meta.outputs.version }})" > "$SUMMARY_FILE" echo "" >> "$SUMMARY_FILE" echo "**Platform:** ${{ matrix.suffix }}" >> "$SUMMARY_FILE" echo "**Build Type:** Release" >> "$SUMMARY_FILE" echo "**Channel:** ${{ steps.meta.outputs.channel }}" >> "$SUMMARY_FILE" # Only mention CI validation if we actually have CI artifacts if [ -d "ci-artifacts" ] && [ "$(ls -A ci-artifacts/ 2>/dev/null)" ]; then echo "**CI Status:** Tests and benchmarks from CI workflow" >> "$SUMMARY_FILE" fi echo "" >> "$SUMMARY_FILE" # Add coverage information if available (no python dependency) if [ -f "ci-artifacts/coverage/coverage.xml" ]; then echo "### Test Coverage" >> "$SUMMARY_FILE" echo "" >> "$SUMMARY_FILE" # Extract line-rate="0.8234" attribute and convert to percent with one decimal raw_rate=$(grep -Eo 'line-rate="[0-9.]+"' ci-artifacts/coverage/coverage.xml | head -1 | sed -E 's/.*="([0-9.]+)"/\1/') || raw_rate="" if [ -n "$raw_rate" ]; then # Use awk for safe float multiplication & formatting coverage_pct=$(awk -v r="$raw_rate" 'BEGIN{ printf("%.1f%%", r*100) }') else coverage_pct="N/A" fi echo "**Line Coverage:** $coverage_pct" >> "$SUMMARY_FILE" echo "" >> "$SUMMARY_FILE" fi # Add benchmark results if available from CI or validation (jq preferred, fallback to grep) if [ -d "$BUILD_DIR/bench_results" ] && ls "$BUILD_DIR/bench_results"/*.json >/dev/null 2>&1; then echo "### Performance Metrics" >> "$SUMMARY_FILE" echo "" >> "$SUMMARY_FILE" echo "| Benchmark | Time (ns/op) | CPU (ns/op) | Memory |" >> "$SUMMARY_FILE" echo "|-----------|--------------|-------------|--------|" >> "$SUMMARY_FILE" if command -v jq >/dev/null 2>&1; then for json_file in "$BUILD_DIR/bench_results"/*.json; do jq -r '.benchmarks[]? | "| ".name // "Unknown" | ".real_time // 0 | floor) | ".cpu_time // 0 | floor) | "' "$json_file" 2>/dev/null | while IFS= read -r line; do name=$(echo "$line" | cut -d'|' -f2 | sed 's/^ *//;s/ *$//') rtime=$(echo "$line" | cut -d'|' -f3 | tr -d ' ') ctime=$(echo "$line" | cut -d'|' -f4 | tr -d ' ') # Derive memory throughput if bytes_per_second present mem=$(jq -r --arg n "$name" '.benchmarks[]? | select(.name==$n) | if (.bytes_per_second // 0) > 0 then ((.bytes_per_second/1024/1024)|tostring)+" MB/s" else (if (.items_per_second // 0) > 0 then ((.items_per_second)|tostring)+" ops/s" else "N/A" end) end' "$json_file" 2>/dev/null) [ -z "$mem" ] && mem="N/A" echo "| $name | $rtime | $ctime | $mem |" >> "$SUMMARY_FILE" done done else # Minimal fallback: grep fields (less robust) for json_file in "$BUILD_DIR/bench_results"/*.json; do grep '"name"' "$json_file" | sed -E 's/.*"name": *"([^"]+)".*/| \1 | ? | ? | N/A |/' >> "$SUMMARY_FILE" || true done fi echo "" >> "$SUMMARY_FILE" fi echo "Generated release summary at: $SUMMARY_FILE" cat "$SUMMARY_FILE" - name: Package if: ${{ inputs.fast_mode != 'true' && github.actor != 'nektos/act' && matrix.packaging != 'msi' }} shell: bash run: | STAGE_PATH="${{ env.STAGE_DIR }}" if [ ! -d "$STAGE_PATH" ]; then echo "Stage directory not found at $STAGE_PATH" >&2 ls -al "$BUILD_DIR" >&2 || true exit 1 fi cd "$STAGE_PATH" VERSION="${{ steps.meta.outputs.version }}" SUFFIX="${{ matrix.suffix }}" OUTDIR="$GITHUB_WORKSPACE" if [ "${{ matrix.packaging }}" = "zip" ]; then ASSET="yams-${VERSION}-${SUFFIX}.zip" zip -r "$OUTDIR/$ASSET" . else ASSET="yams-${VERSION}-${SUFFIX}.tar.gz" tar czf "$OUTDIR/$ASSET" . fi echo "ASSET_PATH=$OUTDIR/$ASSET" >> "$GITHUB_ENV" echo "Created asset: $OUTDIR/$ASSET" - name: Build Linux packages (DEB/RPM) if: ${{ matrix.build_packages == true && runner.os == 'Linux' && inputs.fast_mode != 'true' && github.actor != 'nektos/act' }} env: BASE_VERSION: ${{ steps.meta.outputs.base_version }} shell: bash run: | # Package-only: reuse staged install without triggering a rebuild # Use BASE_VERSION to generate Debian/RPM policy-compliant versions for nightly/weekly chmod +x scripts/build-deb.sh scripts/build-deb.sh package_only "${{ steps.meta.outputs.version }}" "${{ env.BUILD_DIR }}" "stage" # Move packages to root directory for artifact upload if ls "${{ env.BUILD_DIR }}"/yams-*.deb 1> /dev/null 2>&1; then mv "${{ env.BUILD_DIR }}"/yams-*.deb ./ || true fi if ls "${{ env.BUILD_DIR }}"/yams-*.rpm 1> /dev/null 2>&1; then mv "${{ env.BUILD_DIR }}"/yams-*.rpm ./ || true fi - name: Smoke-test ONNX plugin load (Linux) if: ${{ runner.os == 'Linux' && inputs.fast_mode != 'true' && github.actor != 'nektos/act' }} shell: bash run: | set -euo pipefail : "${STAGE_DIR:?STAGE_DIR not set}" PREFIX="${YAMS_INSTALL_PREFIX:-/usr}" YAMS_BIN="$STAGE_DIR$PREFIX/bin/yams" PLUGIN_SO="$STAGE_DIR$PREFIX/lib/yams/plugins/libyams_onnx_plugin.so" if [ ! -x "$YAMS_BIN" ]; then echo "ERROR: Missing staged yams binary: $YAMS_BIN" >&2 exit 1 fi if [ ! -f "$PLUGIN_SO" ]; then echo "ERROR: Missing staged ONNX plugin: $PLUGIN_SO" >&2 exit 1 fi echo "Checking ONNX plugin relocations" if ldd -r "$PLUGIN_SO" 2>&1 | tee /tmp/yams-onnx-ldd.txt | grep -q "undefined symbol"; then echo "ERROR: unresolved symbols detected in ONNX plugin" >&2 cat /tmp/yams-onnx-ldd.txt >&2 exit 1 fi tmp_data="$(mktemp -d)" export YAMS_DATA_DIR="$tmp_data" export YAMS_PLUGIN_DIR="$STAGE_DIR$PREFIX/lib/yams/plugins" export LD_LIBRARY_PATH="$STAGE_DIR$PREFIX/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" echo "Using LD_LIBRARY_PATH=$LD_LIBRARY_PATH" echo "Loading ONNX plugin via staged CLI" "$YAMS_BIN" plugin load "$PLUGIN_SO" "$YAMS_BIN" daemon stop >/dev/null 2>&1 || true # Release notes are sourced directly from CHANGELOG.md during release creation - name: Publish matrix.json (expected platform count) if: matrix.os == 'linux-hosted' shell: bash run: | # Only generate once (linux-hosted) to avoid artifact name collision across matrix entries. # Update to reflect current matrix total (5 platforms: linux-x86_64, linux-arm64, macos-arm64, macos-x86_64, windows-x86_64) echo '{"total": 5}' > matrix.json - name: Upload matrix metadata if: matrix.os == 'linux-hosted' && github.actor != 'nektos/act' uses: actions/upload-artifact@v4 with: name: matrix-meta path: matrix.json retention-days: 1 - name: Upload Release Artifacts if: github.actor != 'nektos/act' uses: actions/upload-artifact@v4 with: name: release-${{ matrix.suffix }}-files${{ github.run_attempt > 1 && format('-a{0}', github.run_attempt) || '' }} path: | yams-*.tar.gz yams-*.zip yams-*.msi yams-*.msi.sha256 ${{ env.BUILD_DIR }}/release_summary.md ${{ env.BUILD_DIR }}/bench_results/*.json yams-*.deb yams-*.rpm yams-*.AppImage yams-*.pkg winget-manifests/** retention-days: 1 - name: Verify Linux package artifacts if: ${{ matrix.build_packages == true && runner.os == 'Linux' && github.actor != 'nektos/act' && inputs.fast_mode != 'true' }} shell: bash run: | set -euo pipefail case "${{ matrix.suffix }}" in linux-x86_64) expected_arch="x86_64" ;; linux-arm64) expected_arch="aarch64" ;; *) echo "Skipping Linux package verification for unexpected suffix: ${{ matrix.suffix }}" >&2; exit 0 ;; esac echo "Verifying expected Linux package artifacts (DEB/RPM) for ${expected_arch}" ls -1 yams-*linux-${expected_arch}.deb 2>/dev/null || { echo "ERROR: Missing .deb package for ${expected_arch}" >&2; exit 1; } if command -v rpmbuild >/dev/null 2>&1; then ls -1 yams-*linux-${expected_arch}.rpm 2>/dev/null || { echo "ERROR: Missing .rpm package for ${expected_arch}" >&2; exit 1; } else echo "rpmbuild unavailable; RPM check skipped" >&2 fi echo "Linux package verification complete" create-release: name: Create GitHub Release needs: [check-commits, build-release] runs-on: ubuntu-latest # Skip release creation when: running under act, OR check-commits skipped release if: | always() && !cancelled() && github.actor != 'nektos/act' && (needs.check-commits.result == 'skipped' || needs.check-commits.outputs.has_new_commits == 'true' || github.event_name == 'push' || github.event_name == 'workflow_dispatch') && needs.build-release.result == 'success' permissions: contents: write actions: read outputs: channel: ${{ steps.meta.outputs.channel }} version: ${{ steps.meta.outputs.version }} tag: ${{ steps.meta.outputs.tag }} steps: - name: Checkout uses: actions/checkout@v4 with: fetch-depth: 0 submodules: false repository: ${{ github.event.workflow_run.head_repository.full_name || github.repository }} ref: ${{ github.event.workflow_run.head_sha || github.ref }} - name: Determine release channel and version/tag id: meta shell: bash run: | set -euo pipefail EVENT="${{ github.event_name }}" SCHEDULE_CRON="${{ github.event.schedule || '' }}" INPUT_CHANNEL="${{ inputs.channel || '' }}" SHORT_HASH=$(git rev-parse --short HEAD) channel="" if [ "$EVENT" = "push" ]; then channel="stable" elif [ "$EVENT" = "workflow_dispatch" ] && [ -n "$INPUT_CHANNEL" ]; then case "$INPUT_CHANNEL" in stable|weekly|nightly) channel="$INPUT_CHANNEL" ;; *) channel="nightly" ;; esac elif [ "$EVENT" = "schedule" ]; then channel="nightly" else channel="stable" fi # Derive version and tag if [ "$channel" = "stable" ]; then TAG="${GITHUB_REF_NAME}" VERSION="${TAG#yams-v}" if [ "$VERSION" = "$TAG" ]; then VERSION="${TAG#v}" fi TAG_OUT="$TAG" RELEASE_NAME="YAMS ${VERSION}" elif [ "$channel" = "nightly" ]; then DATE_ISO=$(date -u +%Y-%m-%d) DATE_COMPACT=$(date -u +%Y%m%d) VERSION="nightly-${DATE_COMPACT}-${SHORT_HASH}" TAG_OUT="nightly-${DATE_COMPACT}-${SHORT_HASH}" RELEASE_NAME="YAMS Nightly ${DATE_ISO} (${SHORT_HASH})" else # weekly YEAR=$(date -u +%G) # ISO week-numbering year WEEK=$(date -u +%V) # ISO week number 01-53 VERSION="weekly-${YEAR}w${WEEK}-${SHORT_HASH}" TAG_OUT="$VERSION" RELEASE_NAME="YAMS Weekly ${YEAR}-w${WEEK} (${SHORT_HASH})" fi BASE_NUMERIC_VERSION="$(git describe --tags --abbrev=0 --match='v*' 2>/dev/null || echo 0.0.0)" BASE_NUMERIC_VERSION="${BASE_NUMERIC_VERSION#v}" echo "channel=$channel" >> "$GITHUB_OUTPUT" echo "version=$VERSION" >> "$GITHUB_OUTPUT" echo "tag=$TAG_OUT" >> "$GITHUB_OUTPUT" echo "release_name=$RELEASE_NAME" >> "$GITHUB_OUTPUT" echo "base_version=$BASE_NUMERIC_VERSION" >> "$GITHUB_OUTPUT" if [ "$channel" = "stable" ]; then echo "prerelease=false" >> "$GITHUB_OUTPUT" else echo "prerelease=true" >> "$GITHUB_OUTPUT" fi echo "Detected channel=$channel, version=$VERSION, tag=$TAG_OUT, name=$RELEASE_NAME" - name: Checkout repository uses: actions/checkout@v4 with: fetch-depth: 0 - name: Download all artifacts uses: actions/download-artifact@v4 with: path: release-artifacts pattern: "*" - name: List downloaded artifacts (debug) shell: bash run: | echo "Downloaded artifact directories:" >&2 find release-artifacts -maxdepth 2 -type f -printf '%P\n' 2>/dev/null || true - name: Collect assets into staging directory shell: bash run: | set -euo pipefail rm -rf assets || true mkdir -p assets # Collect all per-platform release artifacts for dir in release-artifacts/release-*; do [ -d "$dir" ] || continue # Archives cp -v $dir/*.tar.gz assets/ 2>/dev/null || true cp -v $dir/*.zip assets/ 2>/dev/null || true # Packages cp -v $dir/*.deb assets/ 2>/dev/null || true cp -v $dir/*.rpm assets/ 2>/dev/null || true cp -v $dir/*.AppImage assets/ 2>/dev/null || true # Windows MSI packages cp -v $dir/*.msi assets/ 2>/dev/null || true # Summaries & bench json if [ -d "$dir/bench_results" ]; then cp -v $dir/bench_results/*.json assets/ 2>/dev/null || true fi if [ -f "$dir/release_summary.md" ]; then cp -v "$dir/release_summary.md" "assets/release_summary-$(basename "$dir").md" 2>/dev/null || true fi done echo "Final assets content:" >&2 ls -al assets >&2 || true - name: Generate checksums shell: bash run: | set -euo pipefail cd assets echo "Generating SHA256SUMS for release assets" >&2 # Exclude bench JSON and summaries from checksum manifest (only distributable binaries) # Check if any distributable files exist before creating SHA256SUMS if ls -1 | grep -qE '\.(tar.gz|zip|deb|rpm|AppImage|pkg|msi)$' 2>/dev/null; then ls -1 | grep -E '\.(tar.gz|zip|deb|rpm|AppImage|pkg|msi)$' | while read -r f; do sha256sum "$f" >> SHA256SUMS done sort -o SHA256SUMS SHA256SUMS echo "Created checksum manifest:" >&2 cat SHA256SUMS >&2 else echo "No distributable assets found; creating empty SHA256SUMS" >&2 touch SHA256SUMS fi - name: (Optional) Import GPG key and sign checksums if: env.GPG_PRIVATE_KEY && env.GPG_PRIVATE_KEY != '' env: GPG_PRIVATE_KEY: ${{ secrets.GPG_PRIVATE_KEY || '' }} GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE || '' }} shell: bash run: | set -euo pipefail cd assets echo "GPG key present: generating detached signature for SHA256SUMS" >&2 echo "$GPG_PRIVATE_KEY" | base64 -d | gpg --batch --yes --pinentry-mode loopback --passphrase "$GPG_PASSPHRASE" --import gpg --batch --yes --pinentry-mode loopback --passphrase "$GPG_PASSPHRASE" --detach-sign --armor -o SHA256SUMS.asc SHA256SUMS echo "Signature generated (SHA256SUMS.asc)" >&2 - name: Verify asset set id: asset_check shell: bash run: | COUNT=$(find assets -type f | wc -l | tr -d ' ') echo "Discovered $COUNT asset files" >&2 echo "file_count=$COUNT" >> $GITHUB_OUTPUT if [ "$COUNT" -eq 0 ]; then echo "ERROR: No asset files assembled for release." >&2 exit 1 fi - name: Generate Homebrew formula (stable only) if: steps.meta.outputs.channel == 'stable' shell: bash run: | set -euo pipefail TAG="${{ steps.meta.outputs.tag }}" VERSION="${{ steps.meta.outputs.version }}" FORMULA_DIR="homebrew" mkdir -p "$FORMULA_DIR" # Locate macOS archives from local assets (supports legacy/new naming) ARM64_FILE="" X86_64_FILE="" for candidate in \ "assets/yams-${VERSION}-macos-arm64.zip" \ "assets/yams-v${VERSION}-macos-arm64.zip" \ "assets/yams-${TAG}-macos-arm64.zip" \ "assets/yams-yams-v${VERSION}-macos-arm64.zip"; do if [ -f "$candidate" ]; then ARM64_FILE="$candidate" break fi done if [ -z "$ARM64_FILE" ]; then ARM64_FILE="$(ls -1 assets/*-macos-arm64.zip 2>/dev/null | head -n 1 || true)" fi for candidate in \ "assets/yams-${VERSION}-macos-x86_64.zip" \ "assets/yams-v${VERSION}-macos-x86_64.zip" \ "assets/yams-${TAG}-macos-x86_64.zip" \ "assets/yams-yams-v${VERSION}-macos-x86_64.zip"; do if [ -f "$candidate" ]; then X86_64_FILE="$candidate" break fi done if [ -z "$X86_64_FILE" ]; then X86_64_FILE="$(ls -1 assets/*-macos-x86_64.zip 2>/dev/null | head -n 1 || true)" fi ASSET_ARM64="$(basename "${ARM64_FILE:-yams-${VERSION}-macos-arm64.zip}")" ASSET_X86_64="$(basename "${X86_64_FILE:-yams-${VERSION}-macos-x86_64.zip}")" if [ -f "$ARM64_FILE" ]; then SHA256_ARM64=$(shasum -a 256 "$ARM64_FILE" | awk '{print $1}') echo "Computed ARM64 sha256: $SHA256_ARM64" >&2 else echo "WARNING: ARM64 zip not found at $ARM64_FILE, formula may be incomplete" >&2 SHA256_ARM64="PLACEHOLDER_ARM64_SHA256" fi if [ -f "$X86_64_FILE" ]; then SHA256_X86_64=$(shasum -a 256 "$X86_64_FILE" | awk '{print $1}') echo "Computed x86_64 sha256: $SHA256_X86_64" >&2 else echo "WARNING: x86_64 zip not found at $X86_64_FILE, formula may be incomplete" >&2 SHA256_X86_64="PLACEHOLDER_X86_64_SHA256" fi validate_homebrew_archive() { local archive="$1" local label="$2" if [ ! -f "$archive" ]; then echo "Skipping ${label} Homebrew archive validation; file not found: $archive" >&2 return 0 fi python_script=$'import sys\nimport zipfile\n\narchive, label = sys.argv[1], sys.argv[2]\n\nwith zipfile.ZipFile(archive) as zf:\n names = zf.namelist()\n\ndef has_suffix(suffix: str) -> bool:\n suffix = suffix.rstrip("/")\n return any(name.rstrip("/").endswith(suffix) for name in names)\n\nhas_plugin_payload = any(\n "lib/yams/plugins/" in name and not name.endswith("/")\n for name in names\n)\n\nmissing = []\nif not has_suffix("bin/yams"):\n missing.append("bin/yams")\nif not has_suffix("lib/yams/plugins/libyams_onnx_plugin.dylib"):\n missing.append("lib/yams/plugins/libyams_onnx_plugin.dylib")\nif not has_plugin_payload:\n missing.append("lib/yams/plugins/*")\nif not has_suffix("share/yams/sql/reference_schema.sql"):\n missing.append("share/yams/sql/reference_schema.sql")\n\nif missing:\n joined = ", ".join(missing)\n raise SystemExit(f"{label} archive {archive} is missing required Homebrew paths: {joined}")\n\nprint(f"Validated {label} Homebrew archive layout: {archive}")' python3 -c "$python_script" "$archive" "$label" } validate_homebrew_archive "$ARM64_FILE" "arm64" validate_homebrew_archive "$X86_64_FILE" "x86_64" # Release assets are published later in this job; defer URL checks until post-publish. echo "Deferring Homebrew URL validation until after release publish" >&2 TEMPLATE_PATH="packaging/homebrew/yams.rb.template" if [ ! -f "$TEMPLATE_PATH" ]; then echo "ERROR: Homebrew template not found at $TEMPLATE_PATH" >&2 exit 1 fi OUT_FORMULA="$FORMULA_DIR/yams.rb" # Replace template placeholders with release metadata sed \ -e "s|__VERSION__|$VERSION|g" \ -e "s|__ASSET_ARM64__|$ASSET_ARM64|g" \ -e "s|__ASSET_X86_64__|$ASSET_X86_64|g" \ -e "s|__SHA256_ARM64__|$SHA256_ARM64|g" \ -e "s|__SHA256_X86_64__|$SHA256_X86_64|g" \ "$TEMPLATE_PATH" > "$OUT_FORMULA" echo "Updated formula at $OUT_FORMULA:" >&2 grep -E 'version\s+"|sha256 "' "$OUT_FORMULA" >&2 || true # Create quick install instructions file cat > "$FORMULA_DIR/README.txt" </homebrew-yams 2. Add this formula as Formula/yams.rb 3. Users run: brew install /yams/yams EOF - name: Update Homebrew tap submodule (stable only) if: steps.meta.outputs.channel == 'stable' env: HOMEBREW_TAP_TOKEN: ${{ secrets.HOMEBREW_TAP_TOKEN || '' }} shell: bash run: | set -euo pipefail VERSION="${{ steps.meta.outputs.version }}" TAP_PATH="homebrew-yams" # Check if generated formula exists first if [ ! -f "homebrew/yams.rb" ]; then echo "homebrew/yams.rb not found; skipping tap update" >&2 exit 0 fi # Initialize the homebrew-yams submodule (must happen before directory check) git submodule update --init homebrew-yams || { echo "Failed to initialize homebrew-yams submodule; skipping tap update" >&2 exit 0 } if [ ! -d "$TAP_PATH" ]; then echo "Tap submodule not found at $TAP_PATH after init; skipping tap update" >&2 exit 0 fi pushd "$TAP_PATH" >/dev/null # Fetch latest and reset to main BEFORE copying formula (avoids checkout conflicts) if [ -n "${HOMEBREW_TAP_TOKEN:-}" ]; then REMOTE="https://x-access-token:${HOMEBREW_TAP_TOKEN}@github.com/${{ github.repository_owner }}/homebrew-yams.git" git remote add tap-origin "$REMOTE" 2>/dev/null || git remote set-url tap-origin "$REMOTE" git fetch tap-origin main git checkout main || git checkout -b main tap-origin/main git reset --hard tap-origin/main else git checkout main || git checkout -b main git pull --ff-only 2>/dev/null || true fi popd >/dev/null # Now copy the formula after we're on the correct branch mkdir -p "$TAP_PATH/Formula" cp homebrew/yams.rb "$TAP_PATH/Formula/yams.rb" pushd "$TAP_PATH" >/dev/null git config user.name "github-actions[bot]" git config user.email "github-actions[bot]@users.noreply.github.com" if ! grep -Fq "version \"${VERSION}\"" Formula/yams.rb; then echo "ERROR: Generated stable formula does not contain expected version ${VERSION}" >&2 exit 1 fi if grep -Eq '__[A-Z0-9_]+__' Formula/yams.rb; then echo "ERROR: Generated stable formula still contains unresolved placeholders" >&2 exit 1 fi git add -A Formula/yams.rb if git diff --cached --quiet; then echo "Tap formula already at expected version ${VERSION}; skipping commit" >&2 else git commit -m "chore(homebrew): update formula to v${VERSION}" if [ -n "${HOMEBREW_TAP_TOKEN:-}" ]; then for attempt in 1 2 3; do if git push tap-origin main; then break fi if [ "$attempt" -eq 3 ]; then echo "Failed to push updated Homebrew formula after ${attempt} attempts" >&2 exit 1 fi sleep $((attempt * 10)) done else echo "HOMEBREW_TAP_TOKEN not set; manual push required" >&2 fi fi if [ -n "${HOMEBREW_TAP_TOKEN:-}" ]; then for attempt in 1 2 3 4 5; do remote_formula="$(curl -fsSL "https://raw.githubusercontent.com/${{ github.repository_owner }}/homebrew-yams/main/Formula/yams.rb" || true)" if [ -n "$remote_formula" ] && printf '%s\n' "$remote_formula" | grep -Fq "version \"${VERSION}\""; then echo "Confirmed remote stable formula is version ${VERSION}" >&2 break fi if [ "$attempt" -eq 5 ]; then echo "ERROR: Remote stable formula did not update to version ${VERSION}" >&2 exit 1 fi sleep $((attempt * 10)) done fi popd >/dev/null - name: Update Homebrew nightly formula (nightly/weekly only) if: steps.meta.outputs.channel != 'stable' env: HOMEBREW_TAP_TOKEN: ${{ secrets.HOMEBREW_TAP_TOKEN || '' }} shell: bash run: | set -euo pipefail VERSION="${{ steps.meta.outputs.version }}" TAG="${{ steps.meta.outputs.tag }}" TAP_PATH="homebrew-yams" # Extract date and hash from version (e.g., nightly-20251109-ebd0701) DATE=$(echo "$VERSION" | sed -E 's/.*-([0-9]{8})-.*/\1/') HASH=$(echo "$VERSION" | sed -E 's/.*-([a-f0-9]{7})$/\1/') # Initialize submodule before checking path (checkout has submodules disabled by default) git submodule update --init homebrew-yams if [ ! -d "$TAP_PATH" ]; then echo "Tap submodule not found at $TAP_PATH after init" >&2 exit 1 fi # Checkout main BEFORE modifying any files (avoids checkout conflicts) pushd "$TAP_PATH" >/dev/null if [ -n "${HOMEBREW_TAP_TOKEN:-}" ]; then REMOTE="https://x-access-token:${HOMEBREW_TAP_TOKEN}@github.com/${{ github.repository_owner }}/homebrew-yams.git" git remote add tap-origin "$REMOTE" 2>/dev/null || git remote set-url tap-origin "$REMOTE" git fetch tap-origin main 2>/dev/null || true git checkout main || git checkout -b main tap-origin/main 2>/dev/null || git checkout -b main git reset --hard tap-origin/main 2>/dev/null || true else git checkout main || git checkout -b main fi popd >/dev/null # Locate macOS archives from local assets (supports naming changes) ARM64_FILE="" X86_64_FILE="" for candidate in \ "assets/yams-${VERSION}-macos-arm64.zip" \ "assets/yams-${TAG}-macos-arm64.zip" \ "assets/yams-nightly-${DATE}-${HASH}-macos-arm64.zip"; do if [ -f "$candidate" ]; then ARM64_FILE="$candidate" break fi done if [ -z "$ARM64_FILE" ]; then ARM64_FILE="$(ls -1 assets/*-macos-arm64.zip 2>/dev/null | head -n 1 || true)" fi for candidate in \ "assets/yams-${VERSION}-macos-x86_64.zip" \ "assets/yams-${TAG}-macos-x86_64.zip" \ "assets/yams-nightly-${DATE}-${HASH}-macos-x86_64.zip"; do if [ -f "$candidate" ]; then X86_64_FILE="$candidate" break fi done if [ -z "$X86_64_FILE" ]; then X86_64_FILE="$(ls -1 assets/*-macos-x86_64.zip 2>/dev/null | head -n 1 || true)" fi ASSET_ARM64="$(basename "${ARM64_FILE:-yams-${VERSION}-macos-arm64.zip}")" ASSET_X86_64="$(basename "${X86_64_FILE:-yams-${VERSION}-macos-x86_64.zip}")" if [ -f "$ARM64_FILE" ]; then SHA256_ARM64=$(shasum -a 256 "$ARM64_FILE" | awk '{print $1}') echo "Computed ARM64 sha256: $SHA256_ARM64" >&2 else echo "WARNING: ARM64 zip not found at $ARM64_FILE, formula may be incomplete" >&2 SHA256_ARM64="PLACEHOLDER_ARM64_SHA256" fi if [ -f "$X86_64_FILE" ]; then SHA256_X86_64=$(shasum -a 256 "$X86_64_FILE" | awk '{print $1}') echo "Computed x86_64 sha256: $SHA256_X86_64" >&2 else echo "WARNING: x86_64 zip not found at $X86_64_FILE, formula may be incomplete" >&2 SHA256_X86_64="PLACEHOLDER_X86_64_SHA256" fi validate_homebrew_archive() { local archive="$1" local label="$2" if [ ! -f "$archive" ]; then echo "Skipping ${label} Homebrew archive validation; file not found: $archive" >&2 return 0 fi python_script=$'import sys\nimport zipfile\n\narchive, label = sys.argv[1], sys.argv[2]\n\nwith zipfile.ZipFile(archive) as zf:\n names = zf.namelist()\n\ndef has_suffix(suffix: str) -> bool:\n suffix = suffix.rstrip("/")\n return any(name.rstrip("/").endswith(suffix) for name in names)\n\nhas_plugin_payload = any(\n "lib/yams/plugins/" in name and not name.endswith("/")\n for name in names\n)\n\nmissing = []\nif not has_suffix("bin/yams"):\n missing.append("bin/yams")\nif not has_suffix("lib/yams/plugins/libyams_onnx_plugin.dylib"):\n missing.append("lib/yams/plugins/libyams_onnx_plugin.dylib")\nif not has_plugin_payload:\n missing.append("lib/yams/plugins/*")\nif not has_suffix("share/yams/sql/reference_schema.sql"):\n missing.append("share/yams/sql/reference_schema.sql")\n\nif missing:\n joined = ", ".join(missing)\n raise SystemExit(f"{label} archive {archive} is missing required Homebrew paths: {joined}")\n\nprint(f"Validated {label} Homebrew archive layout: {archive}")' python3 -c "$python_script" "$archive" "$label" } validate_homebrew_archive "$ARM64_FILE" "arm64" validate_homebrew_archive "$X86_64_FILE" "x86_64" # Release assets are published later in this job; defer URL checks until post-publish. echo "Deferring Homebrew URL validation until after release publish" >&2 # Update nightly formula (now safe to modify after checkout) TEMPLATE_PATH="packaging/homebrew/yams-nightly.rb.template" if [ ! -f "$TEMPLATE_PATH" ]; then echo "Nightly Homebrew template not found at $TEMPLATE_PATH" >&2 exit 1 fi NIGHTLY_FORMULA="$TAP_PATH/Formula/yams-nightly.rb" mkdir -p "$TAP_PATH/Formula" cp "$TEMPLATE_PATH" "$NIGHTLY_FORMULA" # Remove deprecated nightly formula names now that yams-nightly is canonical rm -f "$TAP_PATH/Formula/yams@nightly.rb" rm -f "$TAP_PATH/Aliases/yams@nightly" # Replace placeholders sed -i.bak \ -e "s/__VERSION__/${VERSION}/g" \ -e "s/__TAG__/${TAG}/g" \ -e "s/__DATE__/${DATE}/g" \ -e "s/__HASH__/${HASH}/g" \ -e "s/__ASSET_ARM64__/${ASSET_ARM64}/g" \ -e "s/__ASSET_X86_64__/${ASSET_X86_64}/g" \ -e "s/__SHA256_ARM64__/${SHA256_ARM64}/g" \ -e "s/__SHA256_X86_64__/${SHA256_X86_64}/g" \ "$NIGHTLY_FORMULA" rm -f "${NIGHTLY_FORMULA}.bak" pushd "$TAP_PATH" >/dev/null git config user.name "github-actions[bot]" git config user.email "github-actions[bot]@users.noreply.github.com" if ! grep -Fq "version \"${VERSION}\"" Formula/yams-nightly.rb; then echo "ERROR: Generated nightly formula does not contain expected version ${VERSION}" >&2 exit 1 fi if ! grep -Fq "/releases/download/${TAG}/" Formula/yams-nightly.rb; then echo "ERROR: Generated nightly formula does not reference expected tag ${TAG}" >&2 exit 1 fi if grep -Eq '__[A-Z0-9_]+__' Formula/yams-nightly.rb; then echo "ERROR: Generated nightly formula still contains unresolved placeholders" >&2 exit 1 fi git add -A Formula if [ -d Aliases ]; then git add -A Aliases fi if git diff --cached --quiet; then echo "Nightly formulas already at expected version ${VERSION}; skipping commit" >&2 else git commit -m "chore(homebrew): update nightly formula to ${VERSION}" if [ -n "${HOMEBREW_TAP_TOKEN:-}" ]; then REMOTE="https://x-access-token:${HOMEBREW_TAP_TOKEN}@github.com/${{ github.repository_owner }}/homebrew-yams.git" for attempt in 1 2 3; do if git push "$REMOTE" main; then break fi if [ "$attempt" -eq 3 ]; then echo "Failed to push updated nightly Homebrew formula after ${attempt} attempts" >&2 exit 1 fi sleep $((attempt * 10)) done else echo "HOMEBREW_TAP_TOKEN not set; manual push required" >&2 fi fi if [ -n "${HOMEBREW_TAP_TOKEN:-}" ]; then for attempt in 1 2 3 4 5; do remote_formula="$(curl -fsSL "https://raw.githubusercontent.com/${{ github.repository_owner }}/homebrew-yams/main/Formula/yams-nightly.rb" || true)" if [ -n "$remote_formula" ] && printf '%s\n' "$remote_formula" | grep -Fq "version \"${VERSION}\"" && printf '%s\n' "$remote_formula" | grep -Fq "/releases/download/${TAG}/"; then echo "Confirmed remote nightly formula references ${VERSION} (${TAG})" >&2 break fi if [ "$attempt" -eq 5 ]; then echo "ERROR: Remote nightly formula did not update to ${VERSION} (${TAG})" >&2 exit 1 fi sleep $((attempt * 10)) done fi popd >/dev/null - name: Build APT repository metadata (stable only, linux .deb present) if: steps.meta.outputs.channel == 'stable' env: GPG_PRIVATE_KEY: ${{ secrets.GPG_PRIVATE_KEY || '' }} GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE || '' }} shell: bash run: | set -euo pipefail # Require at least one .deb in assets if ! ls assets/*.deb >/dev/null 2>&1; then echo "No .deb found; skipping APT repo generation" >&2 exit 0 fi BASE_VERSION="${{ steps.meta.outputs.base_version }}" REPO_ROOT="aptrepo" POOL_DIR="$REPO_ROOT/pool/main/y/yams" DIST=stable COMPONENT=main mkdir -p "$POOL_DIR" # Copy and normalize Debian filenames by architecture. # Source naming is expected as yams--linux-x86_64.deb or yams--linux-aarch64.deb. seen_amd64=0 seen_arm64=0 for deb in assets/*.deb; do deb_name="$(basename "$deb")" case "$deb_name" in *-linux-x86_64.deb) target_name="yams_${BASE_VERSION}_amd64.deb" seen_amd64=1 ;; *-linux-aarch64.deb|*-linux-arm64.deb) target_name="yams_${BASE_VERSION}_arm64.deb" seen_arm64=1 ;; *) echo "Skipping unrecognized .deb naming pattern: $deb_name" >&2 continue ;; esac cp "$deb" "$POOL_DIR/$target_name" done ARCHES=() [ "$seen_amd64" -eq 1 ] && ARCHES+=(amd64) [ "$seen_arm64" -eq 1 ] && ARCHES+=(arm64) if [ ${#ARCHES[@]} -eq 0 ]; then echo "No recognized .deb package architectures found in assets; skipping APT metadata generation" >&2 exit 0 fi # Install dpkg-dev if missing (for dpkg-scanpackages) if ! command -v dpkg-scanpackages >/dev/null 2>&1; then sudo apt-get update sudo apt-get install -y dpkg-dev fi pushd "$REPO_ROOT" >/dev/null mkdir -p "dists/$DIST/$COMPONENT" # Scan pool once, then split package stanzas per architecture. dpkg-scanpackages --multiversion pool > dists/$DIST/$COMPONENT/Packages.all for ARCH in "${ARCHES[@]}"; do mkdir -p "dists/$DIST/$COMPONENT/binary-$ARCH" awk -v arch="$ARCH" ' BEGIN { RS=""; ORS="\n\n" } $0 ~ ("(^|\\n)Architecture: " arch "(\\n|$)") { print $0 } ' "dists/$DIST/$COMPONENT/Packages.all" > "dists/$DIST/$COMPONENT/binary-$ARCH/Packages" gzip -c "dists/$DIST/$COMPONENT/binary-$ARCH/Packages" > "dists/$DIST/$COMPONENT/binary-$ARCH/Packages.gz" done rm -f "dists/$DIST/$COMPONENT/Packages.all" ARCH_LIST="${ARCHES[*]}" # Generate Release file with proper Date and SHA256 hash entries { echo "Origin: YAMS" echo "Label: YAMS" echo "Suite: $DIST" echo "Codename: $DIST" echo "Architectures: $ARCH_LIST" echo "Components: $COMPONENT" echo "Date: $(date -Ru)" echo "Description: YAMS APT repository" echo "SHA256:" for ARCH in "${ARCHES[@]}"; do for f in "$COMPONENT/binary-$ARCH/Packages" "$COMPONENT/binary-$ARCH/Packages.gz"; do if [ -f "dists/$DIST/$f" ]; then hash=$(sha256sum "dists/$DIST/$f" | awk '{print $1}') size=$(stat --format='%s' "dists/$DIST/$f" 2>/dev/null || stat -f '%z' "dists/$DIST/$f") printf " %s %16d %s\n" "$hash" "$size" "$f" fi done done } > dists/$DIST/Release if [ -z "${GPG_PRIVATE_KEY:-}" ]; then echo "ERROR: Missing GPG_PRIVATE_KEY; refusing to publish unsigned stable APT metadata" >&2 exit 1 fi GPG_ARGS=(--batch --yes --pinentry-mode loopback) if [ -n "${GPG_PASSPHRASE:-}" ]; then GPG_ARGS+=(--passphrase "$GPG_PASSPHRASE") fi echo "$GPG_PRIVATE_KEY" | base64 -d | gpg "${GPG_ARGS[@]}" --import gpg "${GPG_ARGS[@]}" -abs -o dists/$DIST/Release.gpg dists/$DIST/Release gpg "${GPG_ARGS[@]}" --clearsign -o dists/$DIST/InRelease dists/$DIST/Release popd >/dev/null tree "$REPO_ROOT" || find "$REPO_ROOT" -maxdepth 4 -type f -print printf '%s\n' \ 'Add this APT repo (signed):' \ ' curl -fsSL https://repo.yamsmemory.ai/gpg.key | sudo gpg --dearmor -o /usr/share/keyrings/yams.gpg' \ ' echo "deb [arch=amd64,arm64 signed-by=/usr/share/keyrings/yams.gpg] https://repo.yamsmemory.ai/aptrepo stable main" | sudo tee /etc/apt/sources.list.d/yams.list' \ ' sudo apt-get update && sudo apt-get install yams' \ '' \ 'If unsigned (temporary/testing):' \ ' echo "deb [arch=amd64,arm64 trusted=yes] https://repo.yamsmemory.ai/aptrepo stable main" | sudo tee /etc/apt/sources.list.d/yams.list' \ ' sudo apt-get update && sudo apt-get install yams' \ > "$REPO_ROOT/USAGE.txt" - name: Upload APT repo artifact if: steps.meta.outputs.channel == 'stable' && always() uses: actions/upload-artifact@v4 with: name: apt-repo path: aptrepo/ retention-days: 7 - name: Build YUM repository metadata (stable only, rpm present) if: steps.meta.outputs.channel == 'stable' env: GPG_PRIVATE_KEY: ${{ secrets.GPG_PRIVATE_KEY || '' }} GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE || '' }} shell: bash run: | set -euo pipefail if ! ls assets/*.rpm >/dev/null 2>&1; then echo "No .rpm found; skipping YUM repo generation" >&2 exit 0 fi YUM_ROOT="yumrepo" mkdir -p "$YUM_ROOT" cp assets/*.rpm "$YUM_ROOT/" 2>/dev/null || true # Install createrepo_c if needed if ! command -v createrepo_c >/dev/null 2>&1; then sudo apt-get update sudo apt-get install -y createrepo-c fi createrepo_c "$YUM_ROOT" if [ -n "${GPG_PRIVATE_KEY:-}" ]; then GPG_ARGS=(--batch --yes) if [ -n "${GPG_PASSPHRASE:-}" ]; then GPG_ARGS+=(--pinentry-mode loopback --passphrase "$GPG_PASSPHRASE") fi echo "$GPG_PRIVATE_KEY" | base64 -d | gpg "${GPG_ARGS[@]}" --import gpg "${GPG_ARGS[@]}" --detach-sign --armor -o "$YUM_ROOT/repodata/repomd.xml.asc" "$YUM_ROOT/repodata/repomd.xml" fi printf '%s\n' \ 'Add YUM/DNF repo (unsigned example):' \ ' sudo tee /etc/yum.repos.d/yams.repo < "$YUM_ROOT/USAGE.txt" - name: Upload YUM repo artifact if: steps.meta.outputs.channel == 'stable' && always() uses: actions/upload-artifact@v4 with: name: yum-repo path: yumrepo/ retention-days: 7 - name: Generate latest manifest (latest.json) shell: bash run: | set -euo pipefail VERSION="${{ steps.meta.outputs.version }}" TAG="${{ steps.meta.outputs.tag }}" CHANNEL="${{ steps.meta.outputs.channel }}" DATE=$(date -u +%Y-%m-%dT%H:%M:%SZ) mkdir -p assets cd assets if [ ! -f SHA256SUMS ]; then echo "Missing SHA256SUMS file in assets/; cannot build manifest" >&2 ls -al >&2 || true exit 1 fi # Build assets array JSON using awk (avoid jq dependency); trim trailing comma ASSETS_JSON=$(awk '{printf "{\"name\":\"%s\",\"sha256\":\"%s\"},", $2, $1}' SHA256SUMS | sed 's/,$//') cat > latest.json <&2 sed 's/^/ /' latest.json >&2 - name: Check existing release (stable only) id: existing_release shell: bash run: | TAG="${{ steps.meta.outputs.tag }}" echo "Checking if release $TAG already exists" >&2 if command -v gh >/dev/null 2>&1; then if release_json="$(gh release view "$TAG" --json isDraft,url 2>/dev/null)"; then echo "exists=true" >> $GITHUB_OUTPUT draft="$(printf '%s' "$release_json" | sed -n 's/.*"isDraft":[[:space:]]*\(true\|false\).*/\1/p')" url="$(printf '%s' "$release_json" | sed -n 's/.*"url":[[:space:]]*"\([^"]*\)".*/\1/p')" echo "draft=${draft:-unknown}" >> $GITHUB_OUTPUT echo "url=${url:-}" >> $GITHUB_OUTPUT else echo "exists=false" >> $GITHUB_OUTPUT echo "draft=false" >> $GITHUB_OUTPUT echo "url=" >> $GITHUB_OUTPUT fi else echo "gh CLI not installed; skipping existence check" >&2 echo "exists=unknown" >> $GITHUB_OUTPUT echo "draft=unknown" >> $GITHUB_OUTPUT echo "url=" >> $GITHUB_OUTPUT fi env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Refuse immutable release asset overwrite if: ${{ github.actor != 'nektos/act' && steps.existing_release.outputs.exists == 'true' && steps.existing_release.outputs.draft != 'true' && steps.asset_check.outputs.file_count != '0' }} shell: bash run: | cat >&2 <<'EOF' ERROR: The GitHub release for this tag is already published. This repository uses immutable releases, so published release assets cannot be replaced. The release upload action overwrites by deleting matching assets first, which GitHub rejects for immutable releases. Create a new patch release from the fixed commit, or delete/recreate the release and tag only if the published release is intentionally being withdrawn. EOF if [ -n "${{ steps.existing_release.outputs.url }}" ]; then echo "Existing release: ${{ steps.existing_release.outputs.url }}" >&2 fi exit 1 - name: Check job completion status id: check-status shell: bash run: | # Discover expected platforms from matrix job names in this run JOBS_JSON=$(curl -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ -H "Accept: application/vnd.github.v3+json" \ -s "https://api.github.com/repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/jobs?per_page=100") # Extract unique platform identifiers from "Build and Release ()" job names EXPECTED_PLATFORMS=$(echo "$JOBS_JSON" | sed -n 's/.*"name"[[:space:]]*:[[:space:]]*"\(Build and Release ([^\"]*)\)".*/\1/p' | sed -n 's/Build and Release (\(.*\))/\1/p' | sort -u) TOTAL_EXPECTED=$(echo "$EXPECTED_PLATFORMS" | grep -c . || true) # Count successful artifacts from downloaded files (release-*) if [ -d "release-artifacts" ]; then # Count per-platform artifact directories (robust against file/dir listing quirks) SUCCESS_COUNT=$(find "release-artifacts" -mindepth 1 -maxdepth 1 -type d -name 'release-*' 2>/dev/null | grep -c . || true) else SUCCESS_COUNT=0 fi # Determine expected builds: # 1) Prefer matrix.json (if provided) # 2) Otherwise, fall back to number of artifact groups downloaded if [ -f "release-artifacts/matrix.json" ]; then TOTAL_EXPECTED=$(cat "release-artifacts/matrix.json" | tr -d '\r' | grep -Eo '"total"[[:space:]]*:[[:space:]]*[0-9]+' | grep -Eo '[0-9]+' | head -1) fi if [ -z "$TOTAL_EXPECTED" ] || [ "$TOTAL_EXPECTED" -eq 0 ]; then TOTAL_EXPECTED="$SUCCESS_COUNT" fi # Compute dynamic threshold = ceil(TOTAL_EXPECTED / 2) THRESHOLD=$(( (TOTAL_EXPECTED + 1) / 2 )) echo "Successful builds: $SUCCESS_COUNT out of $TOTAL_EXPECTED (threshold: $THRESHOLD)" if [ "$SUCCESS_COUNT" -lt "$THRESHOLD" ]; then echo "ERROR: Need at least $THRESHOLD successful builds to create release (out of $TOTAL_EXPECTED expected)" exit 1 fi echo "success_count=$SUCCESS_COUNT" >> $GITHUB_OUTPUT echo "total_expected=$TOTAL_EXPECTED" >> $GITHUB_OUTPUT echo "threshold=$THRESHOLD" >> $GITHUB_OUTPUT - name: Extract release notes from CHANGELOG.md (stable only) id: changelog if: steps.meta.outputs.channel == 'stable' uses: ffurrer2/extract-release-notes@v2 with: changelog_file: CHANGELOG.md release_notes_file: RELEASE_NOTES_EXTRACTED.md - name: Prepare release body id: release-body shell: bash env: CHANNEL: ${{ steps.meta.outputs.channel }} RELEASE_NAME: ${{ steps.meta.outputs.release_name }} REPO_URL: ${{ github.server_url }}/${{ github.repository }} TAG: ${{ steps.meta.outputs.tag }} run: | set -euo pipefail BODY_FILE="RELEASE_BODY.md" if [ "$CHANNEL" = "stable" ]; then if [ ! -s "RELEASE_NOTES_EXTRACTED.md" ]; then echo "Expected extracted stable release notes in RELEASE_NOTES_EXTRACTED.md" >&2 exit 1 fi { echo "## Release Notes" echo cat RELEASE_NOTES_EXTRACTED.md echo echo "Full changelog: [CHANGELOG.md]($REPO_URL/blob/$TAG/CHANGELOG.md)" } > "$BODY_FILE" else FULL_SHA="$(git rev-parse HEAD)" SHORT_SHA="$(git rev-parse --short HEAD)" { echo "## ${RELEASE_NAME}" echo printf -- '- Channel: `%s`\n' "$CHANNEL" printf -- '- Commit: [`%s`](%s/commit/%s)\n' "$SHORT_SHA" "$REPO_URL" "$FULL_SHA" echo "- Changelog: [CHANGELOG.md]($REPO_URL/blob/main/CHANGELOG.md)" } > "$BODY_FILE" fi echo "body_file=$BODY_FILE" >> "$GITHUB_OUTPUT" - name: Prepare release assets shell: bash run: | mkdir -p assets # Collect all release files for dir in release-artifacts/release-*; do if [ -d "$dir" ]; then # Move main archive mv $dir/*.{tar.gz,zip} assets/ 2>/dev/null || true # Move Linux packages mv $dir/*.deb assets/ 2>/dev/null || true mv $dir/*.rpm assets/ 2>/dev/null || true mv $dir/*.AppImage assets/ 2>/dev/null || true # Move macOS pkg packages mv $dir/*.pkg assets/ 2>/dev/null || true # Collect benchmark results cp $dir/bench_results/*.json assets/ 2>/dev/null || true fi done echo "Collected assets:" ls -la assets/ - name: "Dry-run: print release notes (act only)" if: ${{ github.actor == 'nektos/act' }} shell: bash run: | echo "--- RELEASE_BODY.md (dry-run) ---" cat "${{ steps.release-body.outputs.body_file }}" - name: Create or update GitHub Release (draft, upload assets first) if: ${{ github.actor != 'nektos/act' && steps.asset_check.outputs.file_count != '0' && (steps.existing_release.outputs.exists != 'true' || steps.existing_release.outputs.draft == 'true') }} uses: softprops/action-gh-release@v2 with: tag_name: ${{ steps.meta.outputs.tag }} name: ${{ steps.meta.outputs.release_name }} body_path: ${{ steps.release-body.outputs.body_file }} files: assets/* draft: true prerelease: ${{ steps.meta.outputs.prerelease }} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Publish GitHub Release (flip draft=false) if: ${{ github.actor != 'nektos/act' && steps.asset_check.outputs.file_count != '0' }} shell: bash env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | if ! command -v gh >/dev/null 2>&1; then echo "Installing gh CLI to publish release" >&2 type -p curl >/dev/null || sudo apt-get update sudo apt-get install -y gh >/dev/null 2>&1 || true fi TAG="${{ steps.meta.outputs.tag }}" # publish the release after assets are uploaded gh release edit "$TAG" --draft=false --title "${{ steps.meta.outputs.release_name }}" --notes-file "${{ steps.release-body.outputs.body_file }}" \ ${{ steps.meta.outputs.prerelease == 'true' && '--prerelease' || '--latest' }} - name: Validate Homebrew asset URLs (post-publish) if: ${{ github.actor != 'nektos/act' && steps.asset_check.outputs.file_count != '0' }} shell: bash run: | set -euo pipefail TAG="${{ steps.meta.outputs.tag }}" REPO="${{ github.repository }}" shopt -s nullglob arm64_candidates=(assets/*-macos-arm64.zip) x86_64_candidates=(assets/*-macos-x86_64.zip) if [ ${#arm64_candidates[@]} -eq 0 ]; then echo "ERROR: Missing macOS arm64 archive in assets/ for Homebrew validation" >&2 exit 1 fi if [ ${#x86_64_candidates[@]} -eq 0 ]; then echo "ERROR: Missing macOS x86_64 archive in assets/ for Homebrew validation" >&2 exit 1 fi verify_asset_url() { local asset_name="$1" local url="https://github.com/${REPO}/releases/download/${TAG}/${asset_name}" local attempts=12 for attempt in $(seq 1 "$attempts"); do if curl -fsIL "$url" >/dev/null; then echo "Verified Homebrew asset URL: $url" return 0 fi if [ "$attempt" -eq "$attempts" ]; then echo "ERROR: Homebrew asset URL not reachable after publish: $url" >&2 return 1 fi echo "Homebrew asset URL not reachable yet (attempt ${attempt}/${attempts}); waiting 10s: $url" >&2 sleep 10 done } verify_asset_url "$(basename "${arm64_candidates[0]}")" verify_asset_url "$(basename "${x86_64_candidates[0]}")" - name: Publish APT/YUM/latest.json to R2 (stable only) if: ${{ steps.meta.outputs.channel == 'stable' && github.actor != 'nektos/act' }} env: CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN || secrets.R2_SECRET_ACCESS_KEY }} CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} GPG_PUBLIC_KEY_B64: ${{ secrets.GPG_PUBLIC_KEY_B64 || '' }} shell: bash run: | set -euo pipefail if [ -z "${CLOUDFLARE_API_TOKEN:-}" ] || [ -z "${CLOUDFLARE_ACCOUNT_ID:-}" ]; then echo "Cloudflare API token/account missing; aborting R2 publish." >&2 exit 1 fi WRANGLER_ROOT="${RUNNER_TEMP:-/tmp}/wrangler-cli" WRANGLER_BIN="$WRANGLER_ROOT/node_modules/.bin/wrangler" if [ ! -x "$WRANGLER_BIN" ]; then npm install --prefix "$WRANGLER_ROOT" wrangler@4.69.0 >/dev/null fi "$WRANGLER_BIN" --version >&2 upload_file() { local src="$1"; shift local key="$1"; shift local ct="application/octet-stream" case "$key" in *.json) ct="application/json" ;; *.key) ct="text/plain" ;; *.txt) ct="text/plain" ;; *.xml) ct="application/xml" ;; *.gz) ct="application/gzip" ;; *.deb) ct="application/vnd.debian.binary-package" ;; *.rpm) ct="application/x-rpm" ;; *.asc) ct="application/pgp-signature" ;; esac echo "-- Upload $key" >&2 "$WRANGLER_BIN" r2 object put "yams-repository/$key" \ --remote \ --file "$src" \ --content-type "$ct" } if [ -n "${GPG_PUBLIC_KEY_B64:-}" ]; then echo "$GPG_PUBLIC_KEY_B64" | base64 -d > gpg.key upload_file gpg.key gpg.key fi if [ -d aptrepo ]; then find aptrepo -type f -print0 | while IFS= read -r -d '' f; do rel="${f#aptrepo/}"; upload_file "$f" "aptrepo/$rel"; done fi if [ -d yumrepo ]; then find yumrepo -type f -print0 | while IFS= read -r -d '' f; do rel="${f#yumrepo/}"; upload_file "$f" "yumrepo/$rel"; done fi if [ -f assets/latest.json ]; then upload_file assets/latest.json latest.json fi echo "R2 publish complete." >&2 - name: List release assets (post-upload) if: ${{ github.actor != 'nektos/act' }} shell: bash run: | if ! command -v gh >/dev/null 2>&1; then echo "Installing gh CLI for asset listing" >&2 type -p curl >/dev/null || sudo apt-get update sudo apt-get install -y gh >/dev/null 2>&1 || true fi TAG="${{ steps.meta.outputs.tag }}" echo "Release assets for $TAG:" >&2 gh release view "$TAG" --json assets --jq '.assets[].name' || echo "(Could not list assets)" env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}