trevon/yams
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 2
yams/.github/workflows/release.yml

2304 lines
95 KiB
YAML
Raw Permalink Blame History

# yamllint disable rule:line-length
name: Release
'on':
push:
tags:
- "v*"
- "yams-v*"
schedule:
- cron: '0 6 * * *' # 06:00 UTC daily
workflow_dispatch:
inputs:
channel:
description: "Release channel (stable|weekly|nightly)"
required: false
default: "nightly"
fast_mode:
description: "Skip packaging & verification steps (speed up local / act runs)"
required: false
default: "false"
macos_only:
description: "Limit matrix to macOS variants only (skip linux-hosted)"
required: false
default: "false"
permissions:
contents: write
jobs:
check-commits:
name: Check for new commits (nightly only)
runs-on: ubuntu-latest
if: github.event_name == 'schedule' && github.actor != 'nektos/act'
outputs:
has_new_commits: ${{ steps.check.outputs.has_new_commits }}
last_release_sha: ${{ steps.check.outputs.last_release_sha }}
steps:
- name: Check if there are new commits since last nightly release
id: check
uses: actions/github-script@v7
with:
script: |
const owner = context.repo.owner;
const repo = context.repo.repo;
const releases = await github.paginate(github.rest.repos.listReleases, {
owner,
repo,
per_page: 100,
});
const latest = releases.find((r) => r.tag_name?.startsWith('nightly-'));
if (!latest) {
core.info('No previous nightly release found, will create one');
core.setOutput('has_new_commits', 'true');
core.setOutput('last_release_sha', 'none');
return;
}
const tagName = latest.tag_name;
const tagRef = await github.rest.git.getRef({
owner,
repo,
ref: `tags/${tagName}`,
});
let lastSha = tagRef.data.object.sha;
if (tagRef.data.object.type === 'tag') {
const tagObj = await github.rest.git.getTag({
owner,
repo,
tag_sha: lastSha,
});
lastSha = tagObj.data.object.sha;
}
core.setOutput('last_release_sha', lastSha);
const headRef = await github.rest.git.getRef({
owner,
repo,
ref: 'heads/main',
});
const currentSha = headRef.data.object.sha;
if (lastSha !== currentSha) {
const compare = await github.rest.repos.compareCommits({
owner,
repo,
base: lastSha,
head: currentSha,
});
const total = compare.data.total_commits ?? 1;
core.setOutput('has_new_commits', 'true');
core.info(`Found ${total} new commit(s) since last nightly (${tagName})`);
} else {
core.setOutput('has_new_commits', 'false');
core.info(`No new commits since last nightly release (${tagName}), skipping`);
}
warm:
needs: [check-commits]
if: |
always() &&
github.actor != 'nektos/act' &&
(needs.check-commits.result == 'skipped' ||
needs.check-commits.outputs.has_new_commits == 'true' ||
github.event_name == 'push' ||
github.event_name == 'workflow_dispatch')
uses: ./.github/workflows/matrix-warm.yml
with:
build_type: Release
enable_onnx: false
build-release:
name: Build and Package (matrix)
needs: warm
if: |
always() &&
(needs.warm.result == 'success' || needs.warm.result == 'skipped')
strategy:
fail-fast: false
matrix:
include:
- os: linux-hosted
runs_on: ubuntu-24.04
suffix: linux-x86_64
packaging: tar
build_packages: true
meson_args: ""
hosted: true
- os: linux-hosted-arm64
runs_on: ubuntu-24.04-arm
suffix: linux-arm64
packaging: tar
build_packages: true
meson_args: ""
hosted: true
- os: macos-hosted-arm64
runs_on: macos-15
suffix: macos-arm64
packaging: zip
build_packages: false
meson_args: "" # arch via profile edit
hosted: true
- os: macos-hosted-x64
runs_on: macos-15
suffix: macos-x86_64
packaging: zip
build_packages: false
meson_args: ""
hosted: true
- os: windows-hosted
runs_on: windows-latest
suffix: windows-x86_64
packaging: msi
build_packages: true
meson_args: ""
hosted: true
runs-on: ${{ matrix.runs_on }}
env:
STAGE_DIR: stage
steps:
- name: Checkout
uses: actions/checkout@v4
with:
fetch-depth: 0
submodules: false # Manual submodule init below (some use SSH URLs)
- name: Initialize required submodules
shell: bash
run: |
retry_cmd() {
local attempts="$1"
shift
local attempt=1
until "$@"; do
if [ "$attempt" -ge "$attempts" ]; then
return 1
fi
echo "Command failed (attempt ${attempt}/${attempts}): $*" >&2
sleep $((attempt * 5))
attempt=$((attempt + 1))
done
}
# Force HTTPS for GitHub to handle SSH submodule URLs in CI
git config --global url."https://github.com/".insteadOf "git@github.com:"
# Init submodules required for the build
# Skip other submodules that use SSH URLs requiring authentication
retry_cmd 4 git submodule update --init --depth 1 third_party/sqlite-vec-cpp
retry_cmd 4 git submodule update --init --depth 1 third_party/simeon
retry_cmd 4 git submodule update --init --depth 1 third_party/symspell
- name: Skip non-linux entries under act (build)
if: github.actor == 'nektos/act' && !startsWith(matrix.os, 'linux-hosted')
shell: bash
run: |
echo "Skipping build-release job steps for ${{ matrix.os }} under act"
exit 0
- name: Skip linux-hosted when macos_only=true
if: inputs.macos_only == 'true' && matrix.os == 'linux-hosted'
shell: bash
run: |
echo "macos_only=true: skipping linux-hosted build-release job entry"
exit 0
- name: Verify runner architecture (act diagnostics)
if: github.actor == 'nektos/act' && matrix.os == 'linux-hosted'
shell: bash
run: |
set -euo pipefail
ACT_UNAME="$(uname -m || true)"
echo "Detected uname -m: ${ACT_UNAME}"
if [ "${ACT_UNAME}" != "x86_64" ]; then
echo "ERROR: Non-x86_64 architecture detected under act: ${ACT_UNAME}" >&2
echo "cpuinfo (and other x86-specific code) will not build on this architecture." >&2
echo "Re-run forcing amd64 containers, e.g.:" >&2
echo " act --container-architecture linux/amd64 --job build-release --input fast_mode=true" >&2
exit 1
fi
- name: Cache Conan packages
if: github.actor != 'nektos/act'
id: cache_conan
uses: actions/cache@v4
continue-on-error: true
with:
path: |
~/.conan2/p
~/.conan2/r
~/.conan2/recipes
~/.conan2/metadata
~/.conan2/s
key: conan2-${{ runner.os }}-${{ matrix.suffix }}-${{ hashFiles('conanfile.py', 'meson.build', 'meson_options.txt') }}
restore-keys: |
conan2-${{ runner.os }}-${{ matrix.suffix }}-
conan2-${{ runner.os }}-
- name: Conan cache diagnostics
if: always()
shell: bash
run: |
echo "Cache hit: ${{ steps.cache_conan.outputs.cache-hit }}"
for d in p r recipes metadata s; do
if [ -d "$HOME/.conan2/$d" ]; then
echo "Directory $d present: $(find $HOME/.conan2/$d -mindepth 1 -maxdepth 1 | wc -l | tr -d ' ')" entries || true
du -sh $HOME/.conan2/$d 2>/dev/null || true
else
echo "Directory $d missing"
fi
done
- name: Cache macOS build directory
if: runner.os == 'macOS' && github.actor != 'nektos/act'
uses: actions/cache@v4
with:
path: build/release
key: macos-build-${{ matrix.suffix }}-${{ hashFiles('meson.build', 'meson_options.txt') }}-${{ hashFiles('src/**', 'include/**') }}
restore-keys: |
macos-build-${{ matrix.suffix }}-
- name: Cache ccache
if: github.actor != 'nektos/act'
uses: actions/cache@v4
continue-on-error: true
with:
path: |
~/.cache/ccache
~/.ccache
key: ccache-${{ runner.os }}-${{ matrix.suffix }}-${{ hashFiles('conanfile.py', 'meson.build', 'meson_options.txt') }}
restore-keys: |
ccache-${{ runner.os }}-${{ matrix.suffix }}-
ccache-${{ runner.os }}-
- name: Install optional system dependencies (Linux only)
if: runner.os == 'Linux'
continue-on-error: true
shell: bash
run: |
set -euo pipefail
sudo apt-get update -o Acquire::Retries=3 || sudo apt-get update -o Acquire::Retries=3 || true
# Add rpm / fakeroot / dpkg-dev so CPack can emit .deb and .rpm packages
sudo apt-get install -y --no-install-recommends \
liburing-dev \
libarchive-dev \
libtag1-dev \
libboost-all-dev \
pkg-config \
ccache \
rpm \
fakeroot \
dpkg-dev \
meson \
cmake \
ninja-build \
clang \
clang-tools \
xz-utils \
lld || true
echo "Installed optional packages: liburing-dev libarchive-dev libtag1-dev libboost-all-dev ccache rpm fakeroot dpkg-dev meson cmake ninja-build clang clang-tools lld"
# Ensure compilers are preferred
echo "CC=clang" >> $GITHUB_ENV
echo "CXX=clang++" >> $GITHUB_ENV
- name: Install Zig (Linux)
if: runner.os == 'Linux'
shell: bash
run: |
set -euo pipefail
ARCH=$(uname -m)
case "$ARCH" in
x86_64) ZIG_ARCH="x86_64" ;;
aarch64) ZIG_ARCH="aarch64" ;;
*) echo "Unsupported architecture: $ARCH"; exit 1 ;;
esac
ZIG_VERSION="0.15.2"
curl -fsSL "https://ziglang.org/download/${ZIG_VERSION}/zig-${ZIG_ARCH}-linux-${ZIG_VERSION}.tar.xz" -o /tmp/zig.tar.xz
sudo tar -xJf /tmp/zig.tar.xz -C /opt
sudo mv /opt/zig-${ZIG_ARCH}-linux-${ZIG_VERSION} /opt/zig
sudo ln -sf /opt/zig/zig /usr/local/bin/zig
rm /tmp/zig.tar.xz
zig version
- name: Sanitize stdlib flags (macOS only)
if: runner.os == 'macOS'
shell: bash
run: |
set -euo pipefail
# Remove any accidentally injected libstdc++ selections and force libc++
for var in CXXFLAGS CFLAGS LDFLAGS; do
current="${!var:-}"
sanitized="$(echo "$current" | sed -E 's/-stdlib=libstdc\+\+//g') -stdlib=libc++"
# Collapse whitespace
sanitized="$(echo "$sanitized" | tr -s ' ')"
echo "$var=$sanitized" >> "$GITHUB_ENV"
printf '%s\n' "$var=$sanitized"
done
echo "Stdlib flags sanitized for macOS (forcing -stdlib=libc++)."
- name: Determine release channel and version/tag
id: meta
shell: bash
run: |
set -euo pipefail
EVENT="${{ github.event_name }}"
SCHEDULE_CRON="${{ github.event.schedule || '' }}"
INPUT_CHANNEL="${{ inputs.channel || '' }}"
SHORT_HASH=$(git rev-parse --short HEAD)
channel=""
if [ "$EVENT" = "push" ]; then
channel="stable"
elif [ "$EVENT" = "workflow_dispatch" ] && [ -n "$INPUT_CHANNEL" ]; then
case "$INPUT_CHANNEL" in
stable|weekly|nightly) channel="$INPUT_CHANNEL" ;;
*) channel="nightly" ;;
esac
elif [ "$EVENT" = "schedule" ]; then
channel="nightly"
else
channel="stable"
fi
# Derive version and tag
if [ "$channel" = "stable" ]; then
TAG="${GITHUB_REF_NAME}"
VERSION="${TAG#yams-v}"
if [ "$VERSION" = "$TAG" ]; then
VERSION="${TAG#v}"
fi
TAG_OUT="$TAG"
RELEASE_NAME="YAMS ${VERSION}"
elif [ "$channel" = "nightly" ]; then
DATE_ISO=$(date -u +%Y-%m-%d)
DATE_COMPACT=$(date -u +%Y%m%d)
VERSION="nightly-${DATE_COMPACT}-${SHORT_HASH}"
TAG_OUT="nightly-${DATE_COMPACT}-${SHORT_HASH}"
RELEASE_NAME="YAMS Nightly ${DATE_ISO} (${SHORT_HASH})"
else # weekly
YEAR=$(date -u +%G) # ISO week-numbering year
WEEK=$(date -u +%V) # ISO week number 01-53
VERSION="weekly-${YEAR}w${WEEK}-${SHORT_HASH}"
TAG_OUT="$VERSION"
RELEASE_NAME="YAMS Weekly ${YEAR}-w${WEEK} (${SHORT_HASH})"
fi
BASE_NUMERIC_VERSION="$(git describe --tags --abbrev=0 --match='v*' 2>/dev/null || echo 0.0.0)"
BASE_NUMERIC_VERSION="${BASE_NUMERIC_VERSION#v}"
echo "channel=$channel" >> "$GITHUB_OUTPUT"
echo "version=$VERSION" >> "$GITHUB_OUTPUT"
echo "tag=$TAG_OUT" >> "$GITHUB_OUTPUT"
echo "release_name=$RELEASE_NAME" >> "$GITHUB_OUTPUT"
echo "base_version=$BASE_NUMERIC_VERSION" >> "$GITHUB_OUTPUT"
if [ "$channel" = "stable" ]; then
echo "prerelease=false" >> "$GITHUB_OUTPUT"
else
echo "prerelease=true" >> "$GITHUB_OUTPUT"
fi
echo "Detected channel=$channel, version=$VERSION, tag=$TAG_OUT, name=$RELEASE_NAME"
- name: Download CI benchmark artifacts (-yams, if available)
if: github.event_name == 'workflow_run' && matrix.os == 'linux-hosted' && github.actor != 'nektos/act'
uses: dawidd6/action-download-artifact@v6
continue-on-error: true
with:
run_id: ${{ github.event.workflow_run.id }}
workflow_conclusion: completed
name: benchmark-results-linux-hosted-yams
path: ci-artifacts/benchmarks
if_no_artifact_found: warn
- name: Download CI benchmark artifacts (legacy -conan, fallback)
if: github.event_name == 'workflow_run' && matrix.os == 'linux-hosted' && github.actor != 'nektos/act'
uses: dawidd6/action-download-artifact@v6
continue-on-error: true
with:
run_id: ${{ github.event.workflow_run.id }}
workflow_conclusion: completed
name: benchmark-results-linux-hosted-conan
path: ci-artifacts/benchmarks
if_no_artifact_found: warn
- name: Download CI coverage (if available)
if: github.event_name == 'workflow_run' && matrix.os == 'linux-hosted' && github.actor != 'nektos/act'
uses: dawidd6/action-download-artifact@v6
continue-on-error: true
with:
run_id: ${{ github.event.workflow_run.id }}
workflow_conclusion: completed
name: coverage-report
path: ci-artifacts/coverage
if_no_artifact_found: warn
# Hosted runners: install and configure Conan (self-hosted must already be configured)
- name: Install Conan (hosted only)
if: ${{ matrix.hosted }}
shell: bash
run: |
set -euo pipefail
PYTOOLS_DIR="${RUNNER_TEMP}/yams-python-tools"
if [ "$RUNNER_OS" = "Windows" ]; then
VENV_PYTHON="$PYTOOLS_DIR/Scripts/python.exe"
TOOL_BIN="$PYTOOLS_DIR/Scripts"
if [ ! -f "$VENV_PYTHON" ]; then
python -m venv "$PYTOOLS_DIR"
fi
else
VENV_PYTHON="$PYTOOLS_DIR/bin/python"
TOOL_BIN="$PYTOOLS_DIR/bin"
if [ ! -x "$VENV_PYTHON" ]; then
python3 -m venv "$PYTOOLS_DIR"
fi
fi
"$VENV_PYTHON" -m pip install --upgrade pip
"$VENV_PYTHON" -m pip install --upgrade conan meson 'ninja>=1.10'
echo "$TOOL_BIN" >> "$GITHUB_PATH"
export PATH="$TOOL_BIN:$PATH"
conan --version
meson --version
ninja --version
mkdir -p ~/.conan2
# Best-effort: global.conf may not exist in repo for hosted macOS
cp .conan/global.conf ~/.conan2/global.conf 2>/dev/null || true
- name: Ensure build tools (meson, ninja) for local runners/act
if: ${{ matrix.hosted && runner.os != 'Windows' }}
shell: bash
run: |
set -euo pipefail
if [ "$RUNNER_OS" = "macOS" ]; then
brew update || true
brew install meson ninja ccache || true
elif [ "$RUNNER_OS" = "Linux" ]; then
if command -v meson >/dev/null 2>&1 && command -v ninja >/dev/null 2>&1; then
echo "meson/ninja already available on PATH"
else
sudo apt-get update || true
sudo apt-get install -y --no-install-recommends meson ninja-build || true
fi
if ! command -v meson >/dev/null 2>&1 || ! command -v ninja >/dev/null 2>&1; then
python3 -m pip install --user --upgrade --break-system-packages meson 'ninja>=1.10' || \
python -m pip install --user --upgrade --break-system-packages meson 'ninja>=1.10' || true
fi
if [ -n "${GITHUB_PATH:-}" ]; then
echo "$HOME/.local/bin" >> "$GITHUB_PATH"
else
export PATH="$HOME/.local/bin:$PATH"
fi
fi
echo "meson version: $(meson --version || true)"
echo "ninja version: $(ninja --version || true)"
- name: Ensure meson and ninja (macOS)
if: runner.os == 'macOS'
shell: bash
run: |
set -euo pipefail
# If meson or ninja are not on PATH, install via Homebrew for robustness
NEED_INSTALL=0
command -v meson >/dev/null 2>&1 || NEED_INSTALL=1
command -v ninja >/dev/null 2>&1 || NEED_INSTALL=1
if [ "$NEED_INSTALL" -eq 1 ]; then
brew update || true
brew install meson ninja || true
fi
echo "meson version (post-check): $(meson --version || echo 'missing')"
echo "ninja version (post-check): $(ninja --version || echo 'missing')"
- name: Setup ccache env
shell: bash
run: |
echo "CCACHE_DIR=$HOME/.cache/ccache" >> $GITHUB_ENV
echo "CCACHE_BASEDIR=$GITHUB_WORKSPACE" >> $GITHUB_ENV
echo "CCACHE_COMPRESS=1" >> $GITHUB_ENV
echo "CCACHE_MAXSIZE=700M" >> $GITHUB_ENV
ccache -z 2>/dev/null || true
- name: Setup Conan (hosted Linux/macOS)
if: ${{ matrix.hosted }}
shell: bash
run: |
set -e
conan profile detect --force
echo "Conan detected default profile. A platform-specific profile (host-macos-apple-clang or host-linux-clang) will be applied explicitly later."
- name: Detect clang version (log only)
shell: bash
run: |
set -euo pipefail
# Prefer clang++, fallback to clang
if command -v clang++ >/dev/null 2>&1; then
RAW_VER=$(clang++ --version | sed -E -n 's/.*version ([0-9]+).*/\1/p' | head -1 || true)
elif command -v clang >/dev/null 2>&1; then
RAW_VER=$(clang --version | sed -E -n 's/.*version ([0-9]+).*/\1/p' | head -1 || true)
else
RAW_VER=""
fi
if [ -z "$RAW_VER" ]; then
echo "Could not auto-detect clang version; using profile defaults" >&2
else
echo "Detected clang major: $RAW_VER"
fi
- name: Compare profile vs detected clang (advice)
shell: bash
run: |
set -euo pipefail
PROFILE_FILE="./conan/profiles/host-linux-clang"
if [ -f "$PROFILE_FILE" ]; then
PROFILE_VER=$(grep -E '^compiler.version=' "$PROFILE_FILE" | sed -E 's/.*=//')
else
PROFILE_VER="(missing profile)"
fi
if command -v clang++ >/dev/null 2>&1; then
SYS_VER=$(clang++ --version | sed -E -n 's/.*version ([0-9]+).*/\1/p' | head -1 || true)
elif command -v clang >/dev/null 2>&1; then
SYS_VER=$(clang --version | sed -E -n 's/.*version ([0-9]+).*/\1/p' | head -1 || true)
else
SYS_VER="unknown"
fi
echo "Profile declares clang version: $PROFILE_VER"
echo "System detected clang version: $SYS_VER"
if [ -n "$SYS_VER" ] && [ "$SYS_VER" != "unknown" ] && [ "$PROFILE_VER" != "$SYS_VER" ]; then
echo "NOTE: Mismatch between profile (clang $PROFILE_VER) and system (clang $SYS_VER)." >&2
echo " Consider either installing clang-$PROFILE_VER or adjusting profile to $SYS_VER for local/act runs to avoid subtle ABI or flag differences." >&2
fi
- name: Check Conan (self-hosted sanity)
if: ${{ !matrix.hosted }}
shell: bash
run: conan --version || echo "Conan is expected to be pre-installed and configured on self-hosted runners"
# Hosted runners: clean corrupted cache if needed
- name: Clean corrupted Conan cache if needed (hosted only)
if: ${{ matrix.hosted }}
shell: bash
run: |
if ! conan list "*" 2>/dev/null;
then
echo "Conan cache appears corrupted, cleaning locks and temp files..."
conan cache clean --locks --temp 2>/dev/null || true
echo "Conan cache locks/temp cleaned"
fi
- name: Install system dependencies (macOS)
if: runner.os == 'macOS'
shell: bash
run: |
set -euo pipefail
for attempt in 1 2 3; do
if brew update; then
break
fi
if [ "$attempt" -eq 3 ]; then
echo "brew update failed after ${attempt} attempts" >&2
exit 1
fi
sleep $((attempt * 10))
done
brew install pkg-config cmake ninja meson ccache boost zig || true
# Ensure Meson/Ninja are present even if Homebrew failed to link binaries.
command -v meson >/dev/null 2>&1 || python3 -m pip install --user --upgrade meson
command -v ninja >/dev/null 2>&1 || python3 -m pip install --user --upgrade 'ninja>=1.10'
echo "meson version: $(meson --version || echo 'missing')"
echo "ninja version: $(ninja --version || echo 'missing')"
echo "zig version: $(zig version || echo 'missing')"
- name: Tune Conan client (parallel downloads)
shell: bash
run: |
set -euo pipefail
CONF_FILE="$HOME/.conan2/global.conf"
LINE="core.download:parallel=8"
mkdir -p "$(dirname "$CONF_FILE")"
if [ -f "$CONF_FILE" ]; then
TMP_FILE="$(mktemp)"
awk -v line="$LINE" '
BEGIN { replaced = 0 }
/^core.download:parallel=/ { if (!replaced) { print line; replaced = 1 } ; next }
{ print }
END { if (!replaced) print line }
' "$CONF_FILE" > "$TMP_FILE"
mv "$TMP_FILE" "$CONF_FILE"
else
printf '%s\n' "$LINE" > "$CONF_FILE"
fi
echo "Configured $LINE in $CONF_FILE"
grep -n '^core.download:parallel=' "$CONF_FILE" || true
conan config show core.download:parallel 2>/dev/null || true
conan --version
- name: Warm key Conan packages (best-effort)
if: ${{ matrix.hosted }}
shell: bash
run: |
set +e
# Try to prefetch heavy packages to minimize build-time compiles
for ref in \
openjpeg/2.5.3 \
pdfium/95.0.4629 \
onnxruntime/1.18.1 \
icu/74.1 \
libarchive/3.7.6; do
echo "Prefetching $ref ..." || true
conan download "$ref" -r=conancenter || true
done
set -e
# Windows: Install MSVC and build tools
- name: Setup MSVC (Windows only)
if: runner.os == 'Windows'
uses: ilammy/msvc-dev-cmd@v1
with:
arch: x64
- name: Install build tools (Windows only)
if: runner.os == 'Windows'
shell: pwsh
run: |
$pyToolsScripts = Join-Path $env:RUNNER_TEMP "yams-python-tools\Scripts"
"$pyToolsScripts" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
$env:PATH = "$pyToolsScripts;$env:PATH"
# Install WiX for MSI packaging (pin versions to avoid prerelease incompatibility)
dotnet tool install --global wix --version 6.0.2
wix extension add -g WixToolset.UI.wixext/6.0.2
# Install Zig for zyp PDF plugin
$zigVersion = "0.15.2"
$zigUrl = "https://ziglang.org/download/${zigVersion}/zig-x86_64-windows-${zigVersion}.zip"
$zigZip = "$env:TEMP\zig.zip"
$zigDir = "C:\zig"
Invoke-WebRequest -Uri $zigUrl -OutFile $zigZip
Expand-Archive -Path $zigZip -DestinationPath $zigDir -Force
Move-Item "$zigDir\zig-x86_64-windows-${zigVersion}\*" $zigDir -Force
Remove-Item "$zigDir\zig-x86_64-windows-${zigVersion}" -Recurse -ErrorAction SilentlyContinue
$env:PATH = "$zigDir;$env:PATH"
[Environment]::SetEnvironmentVariable("PATH", "$zigDir;$env:PATH", "Process")
"$zigDir" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
# Verify tools
conan --version
meson --version
ninja --version
wix --version
zig version
- name: Configure and Build Release (Windows)
if: runner.os == 'Windows'
shell: pwsh
run: |
$ErrorActionPreference = "Stop"
Write-Host "Building release for Windows x86_64"
# Set version
$env:YAMS_EXTRA_MESON_FLAGS = "-Dyams-version=${{ steps.meta.outputs.version }}"
# Run the build via setup.ps1
.\setup.ps1 -BuildType Release
# Set BUILD_DIR for subsequent steps
"BUILD_DIR=build/release" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
# Release build for Linux/macOS targets
- name: Configure and Build Release (Optimized)
if: runner.os != 'Windows'
shell: bash
run: |
echo "Optimized release build for ${{ matrix.suffix }}"
# Align build directory with README (build/release)
rm -rf build/release
# Determine host profile and architecture based on platform
if [ "$RUNNER_OS" = "macOS" ]; then
case "${{ matrix.suffix }}" in
macos-arm64)
HOST_PROFILE="./conan/profiles/host-macos-apple-clang"
ARCH="armv8"
;;
macos-x86_64)
HOST_PROFILE="./conan/profiles/host-macos-apple-clang-x86"
ARCH="x86_64"
;;
esac
elif [ "$RUNNER_OS" = "Linux" ]; then
HOST_PROFILE="./conan/profiles/host-linux-clang"
case "${{ matrix.suffix }}" in
linux-arm64) ARCH="armv8" ;;
linux-x86_64) ARCH="x86_64" ;;
esac
fi
# Use setup.sh with CI environment variables for consistent configuration
export YAMS_CONAN_HOST_PROFILE="$HOST_PROFILE"
export YAMS_CONAN_ARCH="$ARCH"
export YAMS_ENABLE_MOBILE_BINDINGS=false
export YAMS_EXTRA_MESON_FLAGS="-Dyams-version=${{ steps.meta.outputs.version }} -Dwerror=false -Dwarning_level=2 -Denable-lzma=auto ${{ matrix.meson_args }}"
# Set install prefix: /usr for Linux packages, /usr/local for macOS
if [ "$RUNNER_OS" = "Linux" ]; then
export YAMS_INSTALL_PREFIX="/usr"
else
if [ "${{ matrix.suffix }}" = "macos-arm64" ]; then
export YAMS_INSTALL_PREFIX="/opt/homebrew"
else
export YAMS_INSTALL_PREFIX="/usr/local"
fi
# Enable CoreML GPU acceleration for macOS builds
export YAMS_ONNX_GPU="coreml"
fi
# Persist prefix for later steps
echo "YAMS_INSTALL_PREFIX=$YAMS_INSTALL_PREFIX" >> "$GITHUB_ENV"
# For cross-compilation, ensure pkg-config is available
export PKG_CONFIG="$(command -v pkg-config)"
echo "Using pkg-config: $PKG_CONFIG"
# Version is already included in YAMS_EXTRA_MESON_FLAGS above. Keep this
# echo for diagnostics, but do not overwrite feature flags.
VERSION="${{ steps.meta.outputs.version }}"
echo "Setting version: ${VERSION}"
./setup.sh Release
# Source Conan build env so meson compile uses the same Meson version
# that setup.sh used for meson setup (Conan provides Meson 1.10.x which
# creates build.dat incompatible with older system Meson).
CONAN_BUILD_ENV=""
for candidate in \
"build/release/build-release/conan/conanbuild.sh" \
"build/release/conan/conanbuild.sh"
do
if [ -f "$candidate" ]; then
CONAN_BUILD_ENV="$candidate"
break
fi
done
if [ -n "$CONAN_BUILD_ENV" ]; then
set +u
source "$CONAN_BUILD_ENV"
set -u
fi
meson --version
meson compile -C build/release
echo "BUILD_DIR=build/release" >> $GITHUB_ENV
- name: Upload build logs (artifacts)
if: failure()
uses: actions/upload-artifact@v4
with:
name: logs-${{ matrix.suffix }}-${{ github.run_number }}
path: |
build/release/meson-logs/
build/release/meson-info/
build/release/meson-logs/meson-log.txt
build/release/build-release/conan/**/*.log
if-no-files-found: ignore
retention-days: 14
- name: ccache stats (post-build)
shell: bash
run: |
ccache -s || echo "ccache stats unavailable"
- name: Collect benchmark results
shell: bash
run: |
mkdir -p "$BUILD_DIR/bench_results"
# Collect from validation build (if Linux hosted ran validation)
if [ -f "build/release/bench_results.json" ]; then
cp build/release/bench_results.json "$BUILD_DIR/bench_results/"
echo "Collected benchmark results from validation build"
fi
# Also check CI artifacts if available
if [ -d "ci-artifacts/benchmarks" ] && [ "$(ls -A ci-artifacts/benchmarks/*.json 2>/dev/null)" ]; then
cp ci-artifacts/benchmarks/*.json "$BUILD_DIR/bench_results/" || true
fi
- name: Install (Linux/macOS)
if: runner.os != 'Windows'
shell: bash
run: |
# Ensure BUILD_DIR is properly set
if [ ! -d "$BUILD_DIR" ]; then
echo "BUILD_DIR not found at $BUILD_DIR, attempting fallback to build/release (legacy path build/yams-release no longer used)." >&2
exit 1
fi
# Install to an absolute stage path under the build dir to avoid path duplication
BD="$(cd "$BUILD_DIR" && pwd)"
CONAN_BUILD_ENV=""
for candidate in \
"$BD/build-release/conan/conanbuild.sh" \
"$BD/conan/conanbuild.sh"
do
if [ -f "$candidate" ]; then
CONAN_BUILD_ENV="$candidate"
break
fi
done
if [ -n "$CONAN_BUILD_ENV" ]; then
set +u
source "$CONAN_BUILD_ENV"
set -u
fi
meson --version
# --no-rebuild: setup.sh + a prior `meson compile` already built;
# this step only stages files, never re-triggers the compiler.
meson install -C "$BD" --destdir "$BD/stage" --no-rebuild
if [ ! -d "$BD/stage" ]; then
echo "Stage directory missing after meson install" >&2
ls -al "$BD" >&2 || true
exit 1
fi
echo "STAGE_DIR=$BD/stage" >> "$GITHUB_ENV"
- name: Prune staged runtime payload (Linux/macOS)
if: runner.os != 'Windows'
shell: bash
run: |
set -euo pipefail
: "${STAGE_DIR:?STAGE_DIR not set}"
PREFIX="${YAMS_INSTALL_PREFIX:-}"
if [ -z "$PREFIX" ]; then
if [ "$RUNNER_OS" = "Linux" ]; then
PREFIX="/usr"
else
PREFIX="/usr/local"
fi
fi
INSTALL_ROOT="$STAGE_DIR$PREFIX"
if [ ! -d "$INSTALL_ROOT" ]; then
echo "Missing staged install root: $INSTALL_ROOT" >&2
exit 1
fi
bash scripts/prune-runtime-install.sh "$INSTALL_ROOT"
- name: Verify staged runtime assets (Linux/macOS)
if: runner.os != 'Windows'
shell: bash
run: |
set -euo pipefail
: "${STAGE_DIR:?STAGE_DIR not set}"
PREFIX="${YAMS_INSTALL_PREFIX:-}"
if [ -z "$PREFIX" ]; then
if [ "$RUNNER_OS" = "Linux" ]; then
PREFIX="/usr"
else
PREFIX="/usr/local"
fi
fi
if [ ! -f "$STAGE_DIR$PREFIX/share/yams/sql/reference_schema.sql" ]; then
echo "Missing staged reference schema: $STAGE_DIR$PREFIX/share/yams/sql/reference_schema.sql" >&2
echo "Stage tree (top):" >&2
ls -al "$STAGE_DIR" >&2 || true
exit 1
fi
# Plugins are expected for full releases; fail loudly if absent/empty.
if [ ! -d "$STAGE_DIR$PREFIX/lib/yams/plugins" ]; then
echo "Missing staged plugins dir: $STAGE_DIR$PREFIX/lib/yams/plugins" >&2
exit 1
fi
shopt -s nullglob
plugin_bins=(
"$STAGE_DIR$PREFIX/lib/yams/plugins"/*.so
"$STAGE_DIR$PREFIX/lib/yams/plugins"/*.dylib
)
if [ ${#plugin_bins[@]} -eq 0 ]; then
echo "No plugin binaries staged under: $STAGE_DIR$PREFIX/lib/yams/plugins" >&2
ls -al "$STAGE_DIR$PREFIX/lib/yams/plugins" >&2 || true
exit 1
fi
- name: Install and Package MSI (Windows)
if: runner.os == 'Windows' && inputs.fast_mode != 'true'
shell: pwsh
run: |
$ErrorActionPreference = "Stop"
$buildDir = "build\release"
$stageDir = Join-Path (Resolve-Path $buildDir).Path "stage"
$version = "${{ steps.meta.outputs.version }}"
Write-Host "Staging install to: $stageDir"
# Remove old stage if exists
if (Test-Path $stageDir) {
Remove-Item -Recurse -Force $stageDir
}
New-Item -ItemType Directory -Path $stageDir -Force | Out-Null
# Reconfigure if needed (handles Meson version mismatches from cached build dirs)
$mesonInfo = Join-Path $buildDir "meson-private\coredata.dat"
if (Test-Path $mesonInfo) {
Write-Host "Reconfiguring build directory to ensure Meson version compatibility..."
meson setup --reconfigure $buildDir
if ($LASTEXITCODE -ne 0) {
Write-Warning "Reconfigure failed, attempting fresh setup..."
Remove-Item -Recurse -Force $buildDir -ErrorAction SilentlyContinue
meson setup $buildDir --buildtype=release
}
}
# Install to stage directory.
# --no-rebuild: setup.ps1's release build already compiled; this
# step only stages files, never re-triggers the compiler.
meson install -C $buildDir --destdir $stageDir --no-rebuild
if ($LASTEXITCODE -ne 0) {
Write-Error "Meson install failed"
exit 1
}
# Build MSI using the packaging script
$msiScript = "packaging\windows\build-msi.ps1"
& $msiScript -StageDir $stageDir -Version $version -OutputDir $buildDir
if ($LASTEXITCODE -ne 0) {
Write-Error "MSI build failed"
exit 1
}
# Move MSI to workspace root for artifact upload
Get-ChildItem $buildDir -Filter "*.msi" | ForEach-Object {
Copy-Item $_.FullName -Destination $env:GITHUB_WORKSPACE
Write-Host "Created MSI: $($_.Name)"
}
# Set environment variables for subsequent steps
"STAGE_DIR=$stageDir" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
"BUILD_DIR=$buildDir" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
- name: Generate winget manifest (stable, windows only)
if: runner.os == 'Windows' && steps.meta.outputs.channel == 'stable' && inputs.fast_mode != 'true'
shell: pwsh
run: |
$ErrorActionPreference = "Stop"
$version = "${{ steps.meta.outputs.version }}"
$tag = "${{ steps.meta.outputs.tag }}"
# Find the Windows MSI produced in this job
$msiFile = Get-ChildItem -Path $env:GITHUB_WORKSPACE -Filter "*.msi" | Select-Object -First 1
if (-not $msiFile) {
Write-Error "No MSI file found in workspace; cannot generate winget manifest"
exit 1
}
Write-Host "Generating winget manifest for version $version"
Write-Host "MSI file: $($msiFile.FullName)"
& ./packaging/windows/generate-winget-manifest.ps1 `
-Version $version `
-Tag $tag `
-MsiPath $msiFile.FullName `
-GitHubRepo "${{ github.repository }}" `
-OutputDir "winget-manifests"
if ($LASTEXITCODE -ne 0) {
Write-Error "Winget manifest generation failed"
exit 1
}
- name: Generate release summary
shell: bash
run: |
echo "### Artifacts" >> "$GITHUB_STEP_SUMMARY"
echo "[View artifacts for this run]($GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID)" >> "$GITHUB_STEP_SUMMARY"
SUMMARY_FILE="$BUILD_DIR/release_summary.md"
echo "## Release Build Information (${{ steps.meta.outputs.version }})" > "$SUMMARY_FILE"
echo "" >> "$SUMMARY_FILE"
echo "**Platform:** ${{ matrix.suffix }}" >> "$SUMMARY_FILE"
echo "**Build Type:** Release" >> "$SUMMARY_FILE"
echo "**Channel:** ${{ steps.meta.outputs.channel }}" >> "$SUMMARY_FILE"
# Only mention CI validation if we actually have CI artifacts
if [ -d "ci-artifacts" ] && [ "$(ls -A ci-artifacts/ 2>/dev/null)" ]; then
echo "**CI Status:** Tests and benchmarks from CI workflow" >> "$SUMMARY_FILE"
fi
echo "" >> "$SUMMARY_FILE"
# Add coverage information if available (no python dependency)
if [ -f "ci-artifacts/coverage/coverage.xml" ]; then
echo "### Test Coverage" >> "$SUMMARY_FILE"
echo "" >> "$SUMMARY_FILE"
# Extract line-rate="0.8234" attribute and convert to percent with one decimal
raw_rate=$(grep -Eo 'line-rate="[0-9.]+"' ci-artifacts/coverage/coverage.xml | head -1 | sed -E 's/.*="([0-9.]+)"/\1/') || raw_rate=""
if [ -n "$raw_rate" ]; then
# Use awk for safe float multiplication & formatting
coverage_pct=$(awk -v r="$raw_rate" 'BEGIN{ printf("%.1f%%", r*100) }')
else
coverage_pct="N/A"
fi
echo "**Line Coverage:** $coverage_pct" >> "$SUMMARY_FILE"
echo "" >> "$SUMMARY_FILE"
fi
# Add benchmark results if available from CI or validation (jq preferred, fallback to grep)
if [ -d "$BUILD_DIR/bench_results" ] && ls "$BUILD_DIR/bench_results"/*.json >/dev/null 2>&1; then
echo "### Performance Metrics" >> "$SUMMARY_FILE"
echo "" >> "$SUMMARY_FILE"
echo "| Benchmark | Time (ns/op) | CPU (ns/op) | Memory |" >> "$SUMMARY_FILE"
echo "|-----------|--------------|-------------|--------|" >> "$SUMMARY_FILE"
if command -v jq >/dev/null 2>&1; then
for json_file in "$BUILD_DIR/bench_results"/*.json; do
jq -r '.benchmarks[]? | "| ".name // "Unknown" | ".real_time // 0 | floor) | ".cpu_time // 0 | floor) | "' "$json_file" 2>/dev/null | while IFS= read -r line; do
name=$(echo "$line" | cut -d'|' -f2 | sed 's/^ *//;s/ *$//')
rtime=$(echo "$line" | cut -d'|' -f3 | tr -d ' ')
ctime=$(echo "$line" | cut -d'|' -f4 | tr -d ' ')
# Derive memory throughput if bytes_per_second present
mem=$(jq -r --arg n "$name" '.benchmarks[]? | select(.name==$n) | if (.bytes_per_second // 0) > 0 then ((.bytes_per_second/1024/1024)|tostring)+" MB/s" else (if (.items_per_second // 0) > 0 then ((.items_per_second)|tostring)+" ops/s" else "N/A" end) end' "$json_file" 2>/dev/null)
[ -z "$mem" ] && mem="N/A"
echo "| $name | $rtime | $ctime | $mem |" >> "$SUMMARY_FILE"
done
done
else
# Minimal fallback: grep fields (less robust)
for json_file in "$BUILD_DIR/bench_results"/*.json; do
grep '"name"' "$json_file" | sed -E 's/.*"name": *"([^"]+)".*/| \1 | ? | ? | N/A |/' >> "$SUMMARY_FILE" || true
done
fi
echo "" >> "$SUMMARY_FILE"
fi
echo "Generated release summary at: $SUMMARY_FILE"
cat "$SUMMARY_FILE"
- name: Package
if: ${{ inputs.fast_mode != 'true' && github.actor != 'nektos/act' && matrix.packaging != 'msi' }}
shell: bash
run: |
STAGE_PATH="${{ env.STAGE_DIR }}"
if [ ! -d "$STAGE_PATH" ]; then
echo "Stage directory not found at $STAGE_PATH" >&2
ls -al "$BUILD_DIR" >&2 || true
exit 1
fi
cd "$STAGE_PATH"
VERSION="${{ steps.meta.outputs.version }}"
SUFFIX="${{ matrix.suffix }}"
OUTDIR="$GITHUB_WORKSPACE"
if [ "${{ matrix.packaging }}" = "zip" ]; then
ASSET="yams-${VERSION}-${SUFFIX}.zip"
zip -r "$OUTDIR/$ASSET" .
else
ASSET="yams-${VERSION}-${SUFFIX}.tar.gz"
tar czf "$OUTDIR/$ASSET" .
fi
echo "ASSET_PATH=$OUTDIR/$ASSET" >> "$GITHUB_ENV"
echo "Created asset: $OUTDIR/$ASSET"
- name: Build Linux packages (DEB/RPM)
if: ${{ matrix.build_packages == true && runner.os == 'Linux' && inputs.fast_mode != 'true' && github.actor != 'nektos/act' }}
env:
BASE_VERSION: ${{ steps.meta.outputs.base_version }}
shell: bash
run: |
# Package-only: reuse staged install without triggering a rebuild
# Use BASE_VERSION to generate Debian/RPM policy-compliant versions for nightly/weekly
chmod +x scripts/build-deb.sh
scripts/build-deb.sh package_only "${{ steps.meta.outputs.version }}" "${{ env.BUILD_DIR }}" "stage"
# Move packages to root directory for artifact upload
if ls "${{ env.BUILD_DIR }}"/yams-*.deb 1> /dev/null 2>&1; then
mv "${{ env.BUILD_DIR }}"/yams-*.deb ./ || true
fi
if ls "${{ env.BUILD_DIR }}"/yams-*.rpm 1> /dev/null 2>&1; then
mv "${{ env.BUILD_DIR }}"/yams-*.rpm ./ || true
fi
- name: Smoke-test ONNX plugin load (Linux)
if: ${{ runner.os == 'Linux' && inputs.fast_mode != 'true' && github.actor != 'nektos/act' }}
shell: bash
run: |
set -euo pipefail
: "${STAGE_DIR:?STAGE_DIR not set}"
PREFIX="${YAMS_INSTALL_PREFIX:-/usr}"
YAMS_BIN="$STAGE_DIR$PREFIX/bin/yams"
PLUGIN_SO="$STAGE_DIR$PREFIX/lib/yams/plugins/libyams_onnx_plugin.so"
if [ ! -x "$YAMS_BIN" ]; then
echo "ERROR: Missing staged yams binary: $YAMS_BIN" >&2
exit 1
fi
if [ ! -f "$PLUGIN_SO" ]; then
echo "ERROR: Missing staged ONNX plugin: $PLUGIN_SO" >&2
exit 1
fi
echo "Checking ONNX plugin relocations"
if ldd -r "$PLUGIN_SO" 2>&1 | tee /tmp/yams-onnx-ldd.txt | grep -q "undefined symbol"; then
echo "ERROR: unresolved symbols detected in ONNX plugin" >&2
cat /tmp/yams-onnx-ldd.txt >&2
exit 1
fi
tmp_data="$(mktemp -d)"
export YAMS_DATA_DIR="$tmp_data"
export YAMS_PLUGIN_DIR="$STAGE_DIR$PREFIX/lib/yams/plugins"
export LD_LIBRARY_PATH="$STAGE_DIR$PREFIX/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"
echo "Using LD_LIBRARY_PATH=$LD_LIBRARY_PATH"
echo "Loading ONNX plugin via staged CLI"
"$YAMS_BIN" plugin load "$PLUGIN_SO"
"$YAMS_BIN" daemon stop >/dev/null 2>&1 || true
# Release notes are sourced directly from CHANGELOG.md during release creation
- name: Publish matrix.json (expected platform count)
if: matrix.os == 'linux-hosted'
shell: bash
run: |
# Only generate once (linux-hosted) to avoid artifact name collision across matrix entries.
# Update to reflect current matrix total (5 platforms: linux-x86_64, linux-arm64, macos-arm64, macos-x86_64, windows-x86_64)
echo '{"total": 5}' > matrix.json
- name: Upload matrix metadata
if: matrix.os == 'linux-hosted' && github.actor != 'nektos/act'
uses: actions/upload-artifact@v4
with:
name: matrix-meta
path: matrix.json
retention-days: 1
- name: Upload Release Artifacts
if: github.actor != 'nektos/act'
uses: actions/upload-artifact@v4
with:
name: release-${{ matrix.suffix }}-files${{ github.run_attempt > 1 && format('-a{0}', github.run_attempt) || '' }}
path: |
yams-*.tar.gz
yams-*.zip
yams-*.msi
yams-*.msi.sha256
${{ env.BUILD_DIR }}/release_summary.md
${{ env.BUILD_DIR }}/bench_results/*.json
yams-*.deb
yams-*.rpm
yams-*.AppImage
yams-*.pkg
winget-manifests/**
retention-days: 1
- name: Verify Linux package artifacts
if: ${{ matrix.build_packages == true && runner.os == 'Linux' && github.actor != 'nektos/act' && inputs.fast_mode != 'true' }}
shell: bash
run: |
set -euo pipefail
case "${{ matrix.suffix }}" in
linux-x86_64) expected_arch="x86_64" ;;
linux-arm64) expected_arch="aarch64" ;;
*) echo "Skipping Linux package verification for unexpected suffix: ${{ matrix.suffix }}" >&2; exit 0 ;;
esac
echo "Verifying expected Linux package artifacts (DEB/RPM) for ${expected_arch}"
ls -1 yams-*linux-${expected_arch}.deb 2>/dev/null || { echo "ERROR: Missing .deb package for ${expected_arch}" >&2; exit 1; }
if command -v rpmbuild >/dev/null 2>&1; then
ls -1 yams-*linux-${expected_arch}.rpm 2>/dev/null || { echo "ERROR: Missing .rpm package for ${expected_arch}" >&2; exit 1; }
else
echo "rpmbuild unavailable; RPM check skipped" >&2
fi
echo "Linux package verification complete"
create-release:
name: Create GitHub Release
needs: [check-commits, build-release]
runs-on: ubuntu-latest
# Skip release creation when: running under act, OR check-commits skipped release
if: |
always() && !cancelled() &&
github.actor != 'nektos/act' &&
(needs.check-commits.result == 'skipped' || needs.check-commits.outputs.has_new_commits == 'true' || github.event_name == 'push' || github.event_name == 'workflow_dispatch') &&
needs.build-release.result == 'success'
permissions:
contents: write
actions: read
outputs:
channel: ${{ steps.meta.outputs.channel }}
version: ${{ steps.meta.outputs.version }}
tag: ${{ steps.meta.outputs.tag }}
steps:
- name: Checkout
uses: actions/checkout@v4
with:
fetch-depth: 0
submodules: false
repository: ${{ github.event.workflow_run.head_repository.full_name || github.repository }}
ref: ${{ github.event.workflow_run.head_sha || github.ref }}
- name: Determine release channel and version/tag
id: meta
shell: bash
run: |
set -euo pipefail
EVENT="${{ github.event_name }}"
SCHEDULE_CRON="${{ github.event.schedule || '' }}"
INPUT_CHANNEL="${{ inputs.channel || '' }}"
SHORT_HASH=$(git rev-parse --short HEAD)
channel=""
if [ "$EVENT" = "push" ]; then
channel="stable"
elif [ "$EVENT" = "workflow_dispatch" ] && [ -n "$INPUT_CHANNEL" ]; then
case "$INPUT_CHANNEL" in
stable|weekly|nightly) channel="$INPUT_CHANNEL" ;;
*) channel="nightly" ;;
esac
elif [ "$EVENT" = "schedule" ]; then
channel="nightly"
else
channel="stable"
fi
# Derive version and tag
if [ "$channel" = "stable" ]; then
TAG="${GITHUB_REF_NAME}"
VERSION="${TAG#yams-v}"
if [ "$VERSION" = "$TAG" ]; then
VERSION="${TAG#v}"
fi
TAG_OUT="$TAG"
RELEASE_NAME="YAMS ${VERSION}"
elif [ "$channel" = "nightly" ]; then
DATE_ISO=$(date -u +%Y-%m-%d)
DATE_COMPACT=$(date -u +%Y%m%d)
VERSION="nightly-${DATE_COMPACT}-${SHORT_HASH}"
TAG_OUT="nightly-${DATE_COMPACT}-${SHORT_HASH}"
RELEASE_NAME="YAMS Nightly ${DATE_ISO} (${SHORT_HASH})"
else # weekly
YEAR=$(date -u +%G) # ISO week-numbering year
WEEK=$(date -u +%V) # ISO week number 01-53
VERSION="weekly-${YEAR}w${WEEK}-${SHORT_HASH}"
TAG_OUT="$VERSION"
RELEASE_NAME="YAMS Weekly ${YEAR}-w${WEEK} (${SHORT_HASH})"
fi
BASE_NUMERIC_VERSION="$(git describe --tags --abbrev=0 --match='v*' 2>/dev/null || echo 0.0.0)"
BASE_NUMERIC_VERSION="${BASE_NUMERIC_VERSION#v}"
echo "channel=$channel" >> "$GITHUB_OUTPUT"
echo "version=$VERSION" >> "$GITHUB_OUTPUT"
echo "tag=$TAG_OUT" >> "$GITHUB_OUTPUT"
echo "release_name=$RELEASE_NAME" >> "$GITHUB_OUTPUT"
echo "base_version=$BASE_NUMERIC_VERSION" >> "$GITHUB_OUTPUT"
if [ "$channel" = "stable" ]; then
echo "prerelease=false" >> "$GITHUB_OUTPUT"
else
echo "prerelease=true" >> "$GITHUB_OUTPUT"
fi
echo "Detected channel=$channel, version=$VERSION, tag=$TAG_OUT, name=$RELEASE_NAME"
- name: Checkout repository
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Download all artifacts
uses: actions/download-artifact@v4
with:
path: release-artifacts
pattern: "*"
- name: List downloaded artifacts (debug)
shell: bash
run: |
echo "Downloaded artifact directories:" >&2
find release-artifacts -maxdepth 2 -type f -printf '%P\n' 2>/dev/null || true
- name: Collect assets into staging directory
shell: bash
run: |
set -euo pipefail
rm -rf assets || true
mkdir -p assets
# Collect all per-platform release artifacts
for dir in release-artifacts/release-*; do
[ -d "$dir" ] || continue
# Archives
cp -v $dir/*.tar.gz assets/ 2>/dev/null || true
cp -v $dir/*.zip assets/ 2>/dev/null || true
# Packages
cp -v $dir/*.deb assets/ 2>/dev/null || true
cp -v $dir/*.rpm assets/ 2>/dev/null || true
cp -v $dir/*.AppImage assets/ 2>/dev/null || true
# Windows MSI packages
cp -v $dir/*.msi assets/ 2>/dev/null || true
# Summaries & bench json
if [ -d "$dir/bench_results" ]; then
cp -v $dir/bench_results/*.json assets/ 2>/dev/null || true
fi
if [ -f "$dir/release_summary.md" ]; then
cp -v "$dir/release_summary.md" "assets/release_summary-$(basename "$dir").md" 2>/dev/null || true
fi
done
echo "Final assets content:" >&2
ls -al assets >&2 || true
- name: Generate checksums
shell: bash
run: |
set -euo pipefail
cd assets
echo "Generating SHA256SUMS for release assets" >&2
# Exclude bench JSON and summaries from checksum manifest (only distributable binaries)
# Check if any distributable files exist before creating SHA256SUMS
if ls -1 | grep -qE '\.(tar.gz|zip|deb|rpm|AppImage|pkg|msi)$' 2>/dev/null; then
ls -1 | grep -E '\.(tar.gz|zip|deb|rpm|AppImage|pkg|msi)$' | while read -r f; do
sha256sum "$f" >> SHA256SUMS
done
sort -o SHA256SUMS SHA256SUMS
echo "Created checksum manifest:" >&2
cat SHA256SUMS >&2
else
echo "No distributable assets found; creating empty SHA256SUMS" >&2
touch SHA256SUMS
fi
- name: (Optional) Import GPG key and sign checksums
if: env.GPG_PRIVATE_KEY && env.GPG_PRIVATE_KEY != ''
env:
GPG_PRIVATE_KEY: ${{ secrets.GPG_PRIVATE_KEY || '' }}
GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE || '' }}
shell: bash
run: |
set -euo pipefail
cd assets
echo "GPG key present: generating detached signature for SHA256SUMS" >&2
echo "$GPG_PRIVATE_KEY" | base64 -d | gpg --batch --yes --pinentry-mode loopback --passphrase "$GPG_PASSPHRASE" --import
gpg --batch --yes --pinentry-mode loopback --passphrase "$GPG_PASSPHRASE" --detach-sign --armor -o SHA256SUMS.asc SHA256SUMS
echo "Signature generated (SHA256SUMS.asc)" >&2
- name: Verify asset set
id: asset_check
shell: bash
run: |
COUNT=$(find assets -type f | wc -l | tr -d ' ')
echo "Discovered $COUNT asset files" >&2
echo "file_count=$COUNT" >> $GITHUB_OUTPUT
if [ "$COUNT" -eq 0 ]; then
echo "ERROR: No asset files assembled for release." >&2
exit 1
fi
- name: Generate Homebrew formula (stable only)
if: steps.meta.outputs.channel == 'stable'
shell: bash
run: |
set -euo pipefail
TAG="${{ steps.meta.outputs.tag }}"
VERSION="${{ steps.meta.outputs.version }}"
FORMULA_DIR="homebrew"
mkdir -p "$FORMULA_DIR"
# Locate macOS archives from local assets (supports legacy/new naming)
ARM64_FILE=""
X86_64_FILE=""
for candidate in \
"assets/yams-${VERSION}-macos-arm64.zip" \
"assets/yams-v${VERSION}-macos-arm64.zip" \
"assets/yams-${TAG}-macos-arm64.zip" \
"assets/yams-yams-v${VERSION}-macos-arm64.zip"; do
if [ -f "$candidate" ]; then
ARM64_FILE="$candidate"
break
fi
done
if [ -z "$ARM64_FILE" ]; then
ARM64_FILE="$(ls -1 assets/*-macos-arm64.zip 2>/dev/null | head -n 1 || true)"
fi
for candidate in \
"assets/yams-${VERSION}-macos-x86_64.zip" \
"assets/yams-v${VERSION}-macos-x86_64.zip" \
"assets/yams-${TAG}-macos-x86_64.zip" \
"assets/yams-yams-v${VERSION}-macos-x86_64.zip"; do
if [ -f "$candidate" ]; then
X86_64_FILE="$candidate"
break
fi
done
if [ -z "$X86_64_FILE" ]; then
X86_64_FILE="$(ls -1 assets/*-macos-x86_64.zip 2>/dev/null | head -n 1 || true)"
fi
ASSET_ARM64="$(basename "${ARM64_FILE:-yams-${VERSION}-macos-arm64.zip}")"
ASSET_X86_64="$(basename "${X86_64_FILE:-yams-${VERSION}-macos-x86_64.zip}")"
if [ -f "$ARM64_FILE" ]; then
SHA256_ARM64=$(shasum -a 256 "$ARM64_FILE" | awk '{print $1}')
echo "Computed ARM64 sha256: $SHA256_ARM64" >&2
else
echo "WARNING: ARM64 zip not found at $ARM64_FILE, formula may be incomplete" >&2
SHA256_ARM64="PLACEHOLDER_ARM64_SHA256"
fi
if [ -f "$X86_64_FILE" ]; then
SHA256_X86_64=$(shasum -a 256 "$X86_64_FILE" | awk '{print $1}')
echo "Computed x86_64 sha256: $SHA256_X86_64" >&2
else
echo "WARNING: x86_64 zip not found at $X86_64_FILE, formula may be incomplete" >&2
SHA256_X86_64="PLACEHOLDER_X86_64_SHA256"
fi
validate_homebrew_archive() {
local archive="$1"
local label="$2"
if [ ! -f "$archive" ]; then
echo "Skipping ${label} Homebrew archive validation; file not found: $archive" >&2
return 0
fi
python_script=$'import sys\nimport zipfile\n\narchive, label = sys.argv[1], sys.argv[2]\n\nwith zipfile.ZipFile(archive) as zf:\n names = zf.namelist()\n\ndef has_suffix(suffix: str) -> bool:\n suffix = suffix.rstrip("/")\n return any(name.rstrip("/").endswith(suffix) for name in names)\n\nhas_plugin_payload = any(\n "lib/yams/plugins/" in name and not name.endswith("/")\n for name in names\n)\n\nmissing = []\nif not has_suffix("bin/yams"):\n missing.append("bin/yams")\nif not has_suffix("lib/yams/plugins/libyams_onnx_plugin.dylib"):\n missing.append("lib/yams/plugins/libyams_onnx_plugin.dylib")\nif not has_plugin_payload:\n missing.append("lib/yams/plugins/*")\nif not has_suffix("share/yams/sql/reference_schema.sql"):\n missing.append("share/yams/sql/reference_schema.sql")\n\nif missing:\n joined = ", ".join(missing)\n raise SystemExit(f"{label} archive {archive} is missing required Homebrew paths: {joined}")\n\nprint(f"Validated {label} Homebrew archive layout: {archive}")'
python3 -c "$python_script" "$archive" "$label"
}
validate_homebrew_archive "$ARM64_FILE" "arm64"
validate_homebrew_archive "$X86_64_FILE" "x86_64"
# Release assets are published later in this job; defer URL checks until post-publish.
echo "Deferring Homebrew URL validation until after release publish" >&2
TEMPLATE_PATH="packaging/homebrew/yams.rb.template"
if [ ! -f "$TEMPLATE_PATH" ]; then
echo "ERROR: Homebrew template not found at $TEMPLATE_PATH" >&2
exit 1
fi
OUT_FORMULA="$FORMULA_DIR/yams.rb"
# Replace template placeholders with release metadata
sed \
-e "s|__VERSION__|$VERSION|g" \
-e "s|__ASSET_ARM64__|$ASSET_ARM64|g" \
-e "s|__ASSET_X86_64__|$ASSET_X86_64|g" \
-e "s|__SHA256_ARM64__|$SHA256_ARM64|g" \
-e "s|__SHA256_X86_64__|$SHA256_X86_64|g" \
"$TEMPLATE_PATH" > "$OUT_FORMULA"
echo "Updated formula at $OUT_FORMULA:" >&2
grep -E 'version\s+"|sha256 "' "$OUT_FORMULA" >&2 || true
# Create quick install instructions file
cat > "$FORMULA_DIR/README.txt" <<EOF
Homebrew formula generated for YAMS $VERSION
To use without a tap:
brew install --build-from-source $OUT_FORMULA
Recommended (create a tap repository and place yams.rb there), e.g.:
1. Create repo: github.com/<you>/homebrew-yams
2. Add this formula as Formula/yams.rb
3. Users run: brew install <you>/yams/yams
EOF
- name: Update Homebrew tap submodule (stable only)
if: steps.meta.outputs.channel == 'stable'
env:
HOMEBREW_TAP_TOKEN: ${{ secrets.HOMEBREW_TAP_TOKEN || '' }}
shell: bash
run: |
set -euo pipefail
VERSION="${{ steps.meta.outputs.version }}"
TAP_PATH="homebrew-yams"
# Check if generated formula exists first
if [ ! -f "homebrew/yams.rb" ]; then
echo "homebrew/yams.rb not found; skipping tap update" >&2
exit 0
fi
# Initialize the homebrew-yams submodule (must happen before directory check)
git submodule update --init homebrew-yams || {
echo "Failed to initialize homebrew-yams submodule; skipping tap update" >&2
exit 0
}
if [ ! -d "$TAP_PATH" ]; then
echo "Tap submodule not found at $TAP_PATH after init; skipping tap update" >&2
exit 0
fi
pushd "$TAP_PATH" >/dev/null
# Fetch latest and reset to main BEFORE copying formula (avoids checkout conflicts)
if [ -n "${HOMEBREW_TAP_TOKEN:-}" ]; then
REMOTE="https://x-access-token:${HOMEBREW_TAP_TOKEN}@github.com/${{ github.repository_owner }}/homebrew-yams.git"
git remote add tap-origin "$REMOTE" 2>/dev/null || git remote set-url tap-origin "$REMOTE"
git fetch tap-origin main
git checkout main || git checkout -b main tap-origin/main
git reset --hard tap-origin/main
else
git checkout main || git checkout -b main
git pull --ff-only 2>/dev/null || true
fi
popd >/dev/null
# Now copy the formula after we're on the correct branch
mkdir -p "$TAP_PATH/Formula"
cp homebrew/yams.rb "$TAP_PATH/Formula/yams.rb"
pushd "$TAP_PATH" >/dev/null
git config user.name "github-actions[bot]"
git config user.email "github-actions[bot]@users.noreply.github.com"
if ! grep -Fq "version \"${VERSION}\"" Formula/yams.rb; then
echo "ERROR: Generated stable formula does not contain expected version ${VERSION}" >&2
exit 1
fi
if grep -Eq '__[A-Z0-9_]+__' Formula/yams.rb; then
echo "ERROR: Generated stable formula still contains unresolved placeholders" >&2
exit 1
fi
git add -A Formula/yams.rb
if git diff --cached --quiet; then
echo "Tap formula already at expected version ${VERSION}; skipping commit" >&2
else
git commit -m "chore(homebrew): update formula to v${VERSION}"
if [ -n "${HOMEBREW_TAP_TOKEN:-}" ]; then
for attempt in 1 2 3; do
if git push tap-origin main; then
break
fi
if [ "$attempt" -eq 3 ]; then
echo "Failed to push updated Homebrew formula after ${attempt} attempts" >&2
exit 1
fi
sleep $((attempt * 10))
done
else
echo "HOMEBREW_TAP_TOKEN not set; manual push required" >&2
fi
fi
if [ -n "${HOMEBREW_TAP_TOKEN:-}" ]; then
for attempt in 1 2 3 4 5; do
remote_formula="$(curl -fsSL "https://raw.githubusercontent.com/${{ github.repository_owner }}/homebrew-yams/main/Formula/yams.rb" || true)"
if [ -n "$remote_formula" ] && printf '%s\n' "$remote_formula" | grep -Fq "version \"${VERSION}\""; then
echo "Confirmed remote stable formula is version ${VERSION}" >&2
break
fi
if [ "$attempt" -eq 5 ]; then
echo "ERROR: Remote stable formula did not update to version ${VERSION}" >&2
exit 1
fi
sleep $((attempt * 10))
done
fi
popd >/dev/null
- name: Update Homebrew nightly formula (nightly/weekly only)
if: steps.meta.outputs.channel != 'stable'
env:
HOMEBREW_TAP_TOKEN: ${{ secrets.HOMEBREW_TAP_TOKEN || '' }}
shell: bash
run: |
set -euo pipefail
VERSION="${{ steps.meta.outputs.version }}"
TAG="${{ steps.meta.outputs.tag }}"
TAP_PATH="homebrew-yams"
# Extract date and hash from version (e.g., nightly-20251109-ebd0701)
DATE=$(echo "$VERSION" | sed -E 's/.*-([0-9]{8})-.*/\1/')
HASH=$(echo "$VERSION" | sed -E 's/.*-([a-f0-9]{7})$/\1/')
# Initialize submodule before checking path (checkout has submodules disabled by default)
git submodule update --init homebrew-yams
if [ ! -d "$TAP_PATH" ]; then
echo "Tap submodule not found at $TAP_PATH after init" >&2
exit 1
fi
# Checkout main BEFORE modifying any files (avoids checkout conflicts)
pushd "$TAP_PATH" >/dev/null
if [ -n "${HOMEBREW_TAP_TOKEN:-}" ]; then
REMOTE="https://x-access-token:${HOMEBREW_TAP_TOKEN}@github.com/${{ github.repository_owner }}/homebrew-yams.git"
git remote add tap-origin "$REMOTE" 2>/dev/null || git remote set-url tap-origin "$REMOTE"
git fetch tap-origin main 2>/dev/null || true
git checkout main || git checkout -b main tap-origin/main 2>/dev/null || git checkout -b main
git reset --hard tap-origin/main 2>/dev/null || true
else
git checkout main || git checkout -b main
fi
popd >/dev/null
# Locate macOS archives from local assets (supports naming changes)
ARM64_FILE=""
X86_64_FILE=""
for candidate in \
"assets/yams-${VERSION}-macos-arm64.zip" \
"assets/yams-${TAG}-macos-arm64.zip" \
"assets/yams-nightly-${DATE}-${HASH}-macos-arm64.zip"; do
if [ -f "$candidate" ]; then
ARM64_FILE="$candidate"
break
fi
done
if [ -z "$ARM64_FILE" ]; then
ARM64_FILE="$(ls -1 assets/*-macos-arm64.zip 2>/dev/null | head -n 1 || true)"
fi
for candidate in \
"assets/yams-${VERSION}-macos-x86_64.zip" \
"assets/yams-${TAG}-macos-x86_64.zip" \
"assets/yams-nightly-${DATE}-${HASH}-macos-x86_64.zip"; do
if [ -f "$candidate" ]; then
X86_64_FILE="$candidate"
break
fi
done
if [ -z "$X86_64_FILE" ]; then
X86_64_FILE="$(ls -1 assets/*-macos-x86_64.zip 2>/dev/null | head -n 1 || true)"
fi
ASSET_ARM64="$(basename "${ARM64_FILE:-yams-${VERSION}-macos-arm64.zip}")"
ASSET_X86_64="$(basename "${X86_64_FILE:-yams-${VERSION}-macos-x86_64.zip}")"
if [ -f "$ARM64_FILE" ]; then
SHA256_ARM64=$(shasum -a 256 "$ARM64_FILE" | awk '{print $1}')
echo "Computed ARM64 sha256: $SHA256_ARM64" >&2
else
echo "WARNING: ARM64 zip not found at $ARM64_FILE, formula may be incomplete" >&2
SHA256_ARM64="PLACEHOLDER_ARM64_SHA256"
fi
if [ -f "$X86_64_FILE" ]; then
SHA256_X86_64=$(shasum -a 256 "$X86_64_FILE" | awk '{print $1}')
echo "Computed x86_64 sha256: $SHA256_X86_64" >&2
else
echo "WARNING: x86_64 zip not found at $X86_64_FILE, formula may be incomplete" >&2
SHA256_X86_64="PLACEHOLDER_X86_64_SHA256"
fi
validate_homebrew_archive() {
local archive="$1"
local label="$2"
if [ ! -f "$archive" ]; then
echo "Skipping ${label} Homebrew archive validation; file not found: $archive" >&2
return 0
fi
python_script=$'import sys\nimport zipfile\n\narchive, label = sys.argv[1], sys.argv[2]\n\nwith zipfile.ZipFile(archive) as zf:\n names = zf.namelist()\n\ndef has_suffix(suffix: str) -> bool:\n suffix = suffix.rstrip("/")\n return any(name.rstrip("/").endswith(suffix) for name in names)\n\nhas_plugin_payload = any(\n "lib/yams/plugins/" in name and not name.endswith("/")\n for name in names\n)\n\nmissing = []\nif not has_suffix("bin/yams"):\n missing.append("bin/yams")\nif not has_suffix("lib/yams/plugins/libyams_onnx_plugin.dylib"):\n missing.append("lib/yams/plugins/libyams_onnx_plugin.dylib")\nif not has_plugin_payload:\n missing.append("lib/yams/plugins/*")\nif not has_suffix("share/yams/sql/reference_schema.sql"):\n missing.append("share/yams/sql/reference_schema.sql")\n\nif missing:\n joined = ", ".join(missing)\n raise SystemExit(f"{label} archive {archive} is missing required Homebrew paths: {joined}")\n\nprint(f"Validated {label} Homebrew archive layout: {archive}")'
python3 -c "$python_script" "$archive" "$label"
}
validate_homebrew_archive "$ARM64_FILE" "arm64"
validate_homebrew_archive "$X86_64_FILE" "x86_64"
# Release assets are published later in this job; defer URL checks until post-publish.
echo "Deferring Homebrew URL validation until after release publish" >&2
# Update nightly formula (now safe to modify after checkout)
TEMPLATE_PATH="packaging/homebrew/yams-nightly.rb.template"
if [ ! -f "$TEMPLATE_PATH" ]; then
echo "Nightly Homebrew template not found at $TEMPLATE_PATH" >&2
exit 1
fi
NIGHTLY_FORMULA="$TAP_PATH/Formula/yams-nightly.rb"
mkdir -p "$TAP_PATH/Formula"
cp "$TEMPLATE_PATH" "$NIGHTLY_FORMULA"
# Remove deprecated nightly formula names now that yams-nightly is canonical
rm -f "$TAP_PATH/Formula/yams@nightly.rb"
rm -f "$TAP_PATH/Aliases/yams@nightly"
# Replace placeholders
sed -i.bak \
-e "s/__VERSION__/${VERSION}/g" \
-e "s/__TAG__/${TAG}/g" \
-e "s/__DATE__/${DATE}/g" \
-e "s/__HASH__/${HASH}/g" \
-e "s/__ASSET_ARM64__/${ASSET_ARM64}/g" \
-e "s/__ASSET_X86_64__/${ASSET_X86_64}/g" \
-e "s/__SHA256_ARM64__/${SHA256_ARM64}/g" \
-e "s/__SHA256_X86_64__/${SHA256_X86_64}/g" \
"$NIGHTLY_FORMULA"
rm -f "${NIGHTLY_FORMULA}.bak"
pushd "$TAP_PATH" >/dev/null
git config user.name "github-actions[bot]"
git config user.email "github-actions[bot]@users.noreply.github.com"
if ! grep -Fq "version \"${VERSION}\"" Formula/yams-nightly.rb; then
echo "ERROR: Generated nightly formula does not contain expected version ${VERSION}" >&2
exit 1
fi
if ! grep -Fq "/releases/download/${TAG}/" Formula/yams-nightly.rb; then
echo "ERROR: Generated nightly formula does not reference expected tag ${TAG}" >&2
exit 1
fi
if grep -Eq '__[A-Z0-9_]+__' Formula/yams-nightly.rb; then
echo "ERROR: Generated nightly formula still contains unresolved placeholders" >&2
exit 1
fi
git add -A Formula
if [ -d Aliases ]; then
git add -A Aliases
fi
if git diff --cached --quiet; then
echo "Nightly formulas already at expected version ${VERSION}; skipping commit" >&2
else
git commit -m "chore(homebrew): update nightly formula to ${VERSION}"
if [ -n "${HOMEBREW_TAP_TOKEN:-}" ]; then
REMOTE="https://x-access-token:${HOMEBREW_TAP_TOKEN}@github.com/${{ github.repository_owner }}/homebrew-yams.git"
for attempt in 1 2 3; do
if git push "$REMOTE" main; then
break
fi
if [ "$attempt" -eq 3 ]; then
echo "Failed to push updated nightly Homebrew formula after ${attempt} attempts" >&2
exit 1
fi
sleep $((attempt * 10))
done
else
echo "HOMEBREW_TAP_TOKEN not set; manual push required" >&2
fi
fi
if [ -n "${HOMEBREW_TAP_TOKEN:-}" ]; then
for attempt in 1 2 3 4 5; do
remote_formula="$(curl -fsSL "https://raw.githubusercontent.com/${{ github.repository_owner }}/homebrew-yams/main/Formula/yams-nightly.rb" || true)"
if [ -n "$remote_formula" ] &&
printf '%s\n' "$remote_formula" | grep -Fq "version \"${VERSION}\"" &&
printf '%s\n' "$remote_formula" | grep -Fq "/releases/download/${TAG}/"; then
echo "Confirmed remote nightly formula references ${VERSION} (${TAG})" >&2
break
fi
if [ "$attempt" -eq 5 ]; then
echo "ERROR: Remote nightly formula did not update to ${VERSION} (${TAG})" >&2
exit 1
fi
sleep $((attempt * 10))
done
fi
popd >/dev/null
- name: Build APT repository metadata (stable only, linux .deb present)
if: steps.meta.outputs.channel == 'stable'
env:
GPG_PRIVATE_KEY: ${{ secrets.GPG_PRIVATE_KEY || '' }}
GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE || '' }}
shell: bash
run: |
set -euo pipefail
# Require at least one .deb in assets
if ! ls assets/*.deb >/dev/null 2>&1; then
echo "No .deb found; skipping APT repo generation" >&2
exit 0
fi
BASE_VERSION="${{ steps.meta.outputs.base_version }}"
REPO_ROOT="aptrepo"
POOL_DIR="$REPO_ROOT/pool/main/y/yams"
DIST=stable
COMPONENT=main
mkdir -p "$POOL_DIR"
# Copy and normalize Debian filenames by architecture.
# Source naming is expected as yams-<version>-linux-x86_64.deb or yams-<version>-linux-aarch64.deb.
seen_amd64=0
seen_arm64=0
for deb in assets/*.deb; do
deb_name="$(basename "$deb")"
case "$deb_name" in
*-linux-x86_64.deb)
target_name="yams_${BASE_VERSION}_amd64.deb"
seen_amd64=1
;;
*-linux-aarch64.deb|*-linux-arm64.deb)
target_name="yams_${BASE_VERSION}_arm64.deb"
seen_arm64=1
;;
*)
echo "Skipping unrecognized .deb naming pattern: $deb_name" >&2
continue
;;
esac
cp "$deb" "$POOL_DIR/$target_name"
done
ARCHES=()
[ "$seen_amd64" -eq 1 ] && ARCHES+=(amd64)
[ "$seen_arm64" -eq 1 ] && ARCHES+=(arm64)
if [ ${#ARCHES[@]} -eq 0 ]; then
echo "No recognized .deb package architectures found in assets; skipping APT metadata generation" >&2
exit 0
fi
# Install dpkg-dev if missing (for dpkg-scanpackages)
if ! command -v dpkg-scanpackages >/dev/null 2>&1; then
sudo apt-get update
sudo apt-get install -y dpkg-dev
fi
pushd "$REPO_ROOT" >/dev/null
mkdir -p "dists/$DIST/$COMPONENT"
# Scan pool once, then split package stanzas per architecture.
dpkg-scanpackages --multiversion pool > dists/$DIST/$COMPONENT/Packages.all
for ARCH in "${ARCHES[@]}"; do
mkdir -p "dists/$DIST/$COMPONENT/binary-$ARCH"
awk -v arch="$ARCH" '
BEGIN { RS=""; ORS="\n\n" }
$0 ~ ("(^|\\n)Architecture: " arch "(\\n|$)") { print $0 }
' "dists/$DIST/$COMPONENT/Packages.all" > "dists/$DIST/$COMPONENT/binary-$ARCH/Packages"
gzip -c "dists/$DIST/$COMPONENT/binary-$ARCH/Packages" > "dists/$DIST/$COMPONENT/binary-$ARCH/Packages.gz"
done
rm -f "dists/$DIST/$COMPONENT/Packages.all"
ARCH_LIST="${ARCHES[*]}"
# Generate Release file with proper Date and SHA256 hash entries
{
echo "Origin: YAMS"
echo "Label: YAMS"
echo "Suite: $DIST"
echo "Codename: $DIST"
echo "Architectures: $ARCH_LIST"
echo "Components: $COMPONENT"
echo "Date: $(date -Ru)"
echo "Description: YAMS APT repository"
echo "SHA256:"
for ARCH in "${ARCHES[@]}"; do
for f in "$COMPONENT/binary-$ARCH/Packages" "$COMPONENT/binary-$ARCH/Packages.gz"; do
if [ -f "dists/$DIST/$f" ]; then
hash=$(sha256sum "dists/$DIST/$f" | awk '{print $1}')
size=$(stat --format='%s' "dists/$DIST/$f" 2>/dev/null || stat -f '%z' "dists/$DIST/$f")
printf " %s %16d %s\n" "$hash" "$size" "$f"
fi
done
done
} > dists/$DIST/Release
if [ -z "${GPG_PRIVATE_KEY:-}" ]; then
echo "ERROR: Missing GPG_PRIVATE_KEY; refusing to publish unsigned stable APT metadata" >&2
exit 1
fi
GPG_ARGS=(--batch --yes --pinentry-mode loopback)
if [ -n "${GPG_PASSPHRASE:-}" ]; then
GPG_ARGS+=(--passphrase "$GPG_PASSPHRASE")
fi
echo "$GPG_PRIVATE_KEY" | base64 -d | gpg "${GPG_ARGS[@]}" --import
gpg "${GPG_ARGS[@]}" -abs -o dists/$DIST/Release.gpg dists/$DIST/Release
gpg "${GPG_ARGS[@]}" --clearsign -o dists/$DIST/InRelease dists/$DIST/Release
popd >/dev/null
tree "$REPO_ROOT" || find "$REPO_ROOT" -maxdepth 4 -type f -print
printf '%s\n' \
'Add this APT repo (signed):' \
' curl -fsSL https://repo.yamsmemory.ai/gpg.key | sudo gpg --dearmor -o /usr/share/keyrings/yams.gpg' \
' echo "deb [arch=amd64,arm64 signed-by=/usr/share/keyrings/yams.gpg] https://repo.yamsmemory.ai/aptrepo stable main" | sudo tee /etc/apt/sources.list.d/yams.list' \
' sudo apt-get update && sudo apt-get install yams' \
'' \
'If unsigned (temporary/testing):' \
' echo "deb [arch=amd64,arm64 trusted=yes] https://repo.yamsmemory.ai/aptrepo stable main" | sudo tee /etc/apt/sources.list.d/yams.list' \
' sudo apt-get update && sudo apt-get install yams' \
> "$REPO_ROOT/USAGE.txt"
- name: Upload APT repo artifact
if: steps.meta.outputs.channel == 'stable' && always()
uses: actions/upload-artifact@v4
with:
name: apt-repo
path: aptrepo/
retention-days: 7
- name: Build YUM repository metadata (stable only, rpm present)
if: steps.meta.outputs.channel == 'stable'
env:
GPG_PRIVATE_KEY: ${{ secrets.GPG_PRIVATE_KEY || '' }}
GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE || '' }}
shell: bash
run: |
set -euo pipefail
if ! ls assets/*.rpm >/dev/null 2>&1; then
echo "No .rpm found; skipping YUM repo generation" >&2
exit 0
fi
YUM_ROOT="yumrepo"
mkdir -p "$YUM_ROOT"
cp assets/*.rpm "$YUM_ROOT/" 2>/dev/null || true
# Install createrepo_c if needed
if ! command -v createrepo_c >/dev/null 2>&1; then
sudo apt-get update
sudo apt-get install -y createrepo-c
fi
createrepo_c "$YUM_ROOT"
if [ -n "${GPG_PRIVATE_KEY:-}" ]; then
GPG_ARGS=(--batch --yes)
if [ -n "${GPG_PASSPHRASE:-}" ]; then
GPG_ARGS+=(--pinentry-mode loopback --passphrase "$GPG_PASSPHRASE")
fi
echo "$GPG_PRIVATE_KEY" | base64 -d | gpg "${GPG_ARGS[@]}" --import
gpg "${GPG_ARGS[@]}" --detach-sign --armor -o "$YUM_ROOT/repodata/repomd.xml.asc" "$YUM_ROOT/repodata/repomd.xml"
fi
printf '%s\n' \
'Add YUM/DNF repo (unsigned example):' \
' sudo tee /etc/yum.repos.d/yams.repo <<REPO' \
' [yams]' \
' name=YAMS Repository' \
' baseurl=https://repo.yamsmemory.ai/yumrepo/' \
' enabled=1' \
' gpgcheck=0' \
' repo_gpgcheck=0' \
' REPO' \
'' \
'Then install:' \
' sudo dnf makecache || sudo yum makecache' \
' sudo dnf install yams || sudo yum install yams' \
'' \
'(Enable gpgcheck once you publish a key and sign RPMs/repodata.)' \
> "$YUM_ROOT/USAGE.txt"
- name: Upload YUM repo artifact
if: steps.meta.outputs.channel == 'stable' && always()
uses: actions/upload-artifact@v4
with:
name: yum-repo
path: yumrepo/
retention-days: 7
- name: Generate latest manifest (latest.json)
shell: bash
run: |
set -euo pipefail
VERSION="${{ steps.meta.outputs.version }}"
TAG="${{ steps.meta.outputs.tag }}"
CHANNEL="${{ steps.meta.outputs.channel }}"
DATE=$(date -u +%Y-%m-%dT%H:%M:%SZ)
mkdir -p assets
cd assets
if [ ! -f SHA256SUMS ]; then
echo "Missing SHA256SUMS file in assets/; cannot build manifest" >&2
ls -al >&2 || true
exit 1
fi
# Build assets array JSON using awk (avoid jq dependency); trim trailing comma
ASSETS_JSON=$(awk '{printf "{\"name\":\"%s\",\"sha256\":\"%s\"},", $2, $1}' SHA256SUMS | sed 's/,$//')
cat > latest.json <<LJSON
{
"version": "${VERSION}",
"tag": "${TAG}",
"channel": "${CHANNEL}",
"published_at": "${DATE}",
"assets": [${ASSETS_JSON}]
}
LJSON
echo "latest.json manifest (assets/latest.json):" >&2
sed 's/^/ /' latest.json >&2
- name: Check existing release (stable only)
id: existing_release
shell: bash
run: |
TAG="${{ steps.meta.outputs.tag }}"
echo "Checking if release $TAG already exists" >&2
if command -v gh >/dev/null 2>&1; then
if release_json="$(gh release view "$TAG" --json isDraft,url 2>/dev/null)"; then
echo "exists=true" >> $GITHUB_OUTPUT
draft="$(printf '%s' "$release_json" | sed -n 's/.*"isDraft":[[:space:]]*\(true\|false\).*/\1/p')"
url="$(printf '%s' "$release_json" | sed -n 's/.*"url":[[:space:]]*"\([^"]*\)".*/\1/p')"
echo "draft=${draft:-unknown}" >> $GITHUB_OUTPUT
echo "url=${url:-}" >> $GITHUB_OUTPUT
else
echo "exists=false" >> $GITHUB_OUTPUT
echo "draft=false" >> $GITHUB_OUTPUT
echo "url=" >> $GITHUB_OUTPUT
fi
else
echo "gh CLI not installed; skipping existence check" >&2
echo "exists=unknown" >> $GITHUB_OUTPUT
echo "draft=unknown" >> $GITHUB_OUTPUT
echo "url=" >> $GITHUB_OUTPUT
fi
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Refuse immutable release asset overwrite
if: ${{ github.actor != 'nektos/act' && steps.existing_release.outputs.exists == 'true' && steps.existing_release.outputs.draft != 'true' && steps.asset_check.outputs.file_count != '0' }}
shell: bash
run: |
cat >&2 <<'EOF'
ERROR: The GitHub release for this tag is already published.
This repository uses immutable releases, so published release assets
cannot be replaced. The release upload action overwrites by deleting
matching assets first, which GitHub rejects for immutable releases.
Create a new patch release from the fixed commit, or delete/recreate
the release and tag only if the published release is intentionally being
withdrawn.
EOF
if [ -n "${{ steps.existing_release.outputs.url }}" ]; then
echo "Existing release: ${{ steps.existing_release.outputs.url }}" >&2
fi
exit 1
- name: Check job completion status
id: check-status
shell: bash
run: |
# Discover expected platforms from matrix job names in this run
JOBS_JSON=$(curl -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
-H "Accept: application/vnd.github.v3+json" \
-s "https://api.github.com/repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/jobs?per_page=100")
# Extract unique platform identifiers from "Build and Release (<platform>)" job names
EXPECTED_PLATFORMS=$(echo "$JOBS_JSON" | sed -n 's/.*"name"[[:space:]]*:[[:space:]]*"\(Build and Release ([^\"]*)\)".*/\1/p' | sed -n 's/Build and Release (\(.*\))/\1/p' | sort -u)
TOTAL_EXPECTED=$(echo "$EXPECTED_PLATFORMS" | grep -c . || true)
# Count successful artifacts from downloaded files (release-*)
if [ -d "release-artifacts" ]; then
# Count per-platform artifact directories (robust against file/dir listing quirks)
SUCCESS_COUNT=$(find "release-artifacts" -mindepth 1 -maxdepth 1 -type d -name 'release-*' 2>/dev/null | grep -c . || true)
else
SUCCESS_COUNT=0
fi
# Determine expected builds:
# 1) Prefer matrix.json (if provided)
# 2) Otherwise, fall back to number of artifact groups downloaded
if [ -f "release-artifacts/matrix.json" ]; then
TOTAL_EXPECTED=$(cat "release-artifacts/matrix.json" | tr -d '\r' | grep -Eo '"total"[[:space:]]*:[[:space:]]*[0-9]+' | grep -Eo '[0-9]+' | head -1)
fi
if [ -z "$TOTAL_EXPECTED" ] || [ "$TOTAL_EXPECTED" -eq 0 ]; then
TOTAL_EXPECTED="$SUCCESS_COUNT"
fi
# Compute dynamic threshold = ceil(TOTAL_EXPECTED / 2)
THRESHOLD=$(( (TOTAL_EXPECTED + 1) / 2 ))
echo "Successful builds: $SUCCESS_COUNT out of $TOTAL_EXPECTED (threshold: $THRESHOLD)"
if [ "$SUCCESS_COUNT" -lt "$THRESHOLD" ]; then
echo "ERROR: Need at least $THRESHOLD successful builds to create release (out of $TOTAL_EXPECTED expected)"
exit 1
fi
echo "success_count=$SUCCESS_COUNT" >> $GITHUB_OUTPUT
echo "total_expected=$TOTAL_EXPECTED" >> $GITHUB_OUTPUT
echo "threshold=$THRESHOLD" >> $GITHUB_OUTPUT
- name: Extract release notes from CHANGELOG.md (stable only)
id: changelog
if: steps.meta.outputs.channel == 'stable'
uses: ffurrer2/extract-release-notes@v2
with:
changelog_file: CHANGELOG.md
release_notes_file: RELEASE_NOTES_EXTRACTED.md
- name: Prepare release body
id: release-body
shell: bash
env:
CHANNEL: ${{ steps.meta.outputs.channel }}
RELEASE_NAME: ${{ steps.meta.outputs.release_name }}
REPO_URL: ${{ github.server_url }}/${{ github.repository }}
TAG: ${{ steps.meta.outputs.tag }}
run: |
set -euo pipefail
BODY_FILE="RELEASE_BODY.md"
if [ "$CHANNEL" = "stable" ]; then
if [ ! -s "RELEASE_NOTES_EXTRACTED.md" ]; then
echo "Expected extracted stable release notes in RELEASE_NOTES_EXTRACTED.md" >&2
exit 1
fi
{
echo "## Release Notes"
echo
cat RELEASE_NOTES_EXTRACTED.md
echo
echo "Full changelog: [CHANGELOG.md]($REPO_URL/blob/$TAG/CHANGELOG.md)"
} > "$BODY_FILE"
else
FULL_SHA="$(git rev-parse HEAD)"
SHORT_SHA="$(git rev-parse --short HEAD)"
{
echo "## ${RELEASE_NAME}"
echo
printf -- '- Channel: `%s`\n' "$CHANNEL"
printf -- '- Commit: [`%s`](%s/commit/%s)\n' "$SHORT_SHA" "$REPO_URL" "$FULL_SHA"
echo "- Changelog: [CHANGELOG.md]($REPO_URL/blob/main/CHANGELOG.md)"
} > "$BODY_FILE"
fi
echo "body_file=$BODY_FILE" >> "$GITHUB_OUTPUT"
- name: Prepare release assets
shell: bash
run: |
mkdir -p assets
# Collect all release files
for dir in release-artifacts/release-*; do
if [ -d "$dir" ]; then
# Move main archive
mv $dir/*.{tar.gz,zip} assets/ 2>/dev/null || true
# Move Linux packages
mv $dir/*.deb assets/ 2>/dev/null || true
mv $dir/*.rpm assets/ 2>/dev/null || true
mv $dir/*.AppImage assets/ 2>/dev/null || true
# Move macOS pkg packages
mv $dir/*.pkg assets/ 2>/dev/null || true
# Collect benchmark results
cp $dir/bench_results/*.json assets/ 2>/dev/null || true
fi
done
echo "Collected assets:"
ls -la assets/
- name: "Dry-run: print release notes (act only)"
if: ${{ github.actor == 'nektos/act' }}
shell: bash
run: |
echo "--- RELEASE_BODY.md (dry-run) ---"
cat "${{ steps.release-body.outputs.body_file }}"
- name: Create or update GitHub Release (draft, upload assets first)
if: ${{ github.actor != 'nektos/act' && steps.asset_check.outputs.file_count != '0' && (steps.existing_release.outputs.exists != 'true' || steps.existing_release.outputs.draft == 'true') }}
uses: softprops/action-gh-release@v2
with:
tag_name: ${{ steps.meta.outputs.tag }}
name: ${{ steps.meta.outputs.release_name }}
body_path: ${{ steps.release-body.outputs.body_file }}
files: assets/*
draft: true
prerelease: ${{ steps.meta.outputs.prerelease }}
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Publish GitHub Release (flip draft=false)
if: ${{ github.actor != 'nektos/act' && steps.asset_check.outputs.file_count != '0' }}
shell: bash
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
if ! command -v gh >/dev/null 2>&1; then
echo "Installing gh CLI to publish release" >&2
type -p curl >/dev/null || sudo apt-get update
sudo apt-get install -y gh >/dev/null 2>&1 || true
fi
TAG="${{ steps.meta.outputs.tag }}"
# publish the release after assets are uploaded
gh release edit "$TAG" --draft=false --title "${{ steps.meta.outputs.release_name }}" --notes-file "${{ steps.release-body.outputs.body_file }}" \
${{ steps.meta.outputs.prerelease == 'true' && '--prerelease' || '--latest' }}
- name: Validate Homebrew asset URLs (post-publish)
if: ${{ github.actor != 'nektos/act' && steps.asset_check.outputs.file_count != '0' }}
shell: bash
run: |
set -euo pipefail
TAG="${{ steps.meta.outputs.tag }}"
REPO="${{ github.repository }}"
shopt -s nullglob
arm64_candidates=(assets/*-macos-arm64.zip)
x86_64_candidates=(assets/*-macos-x86_64.zip)
if [ ${#arm64_candidates[@]} -eq 0 ]; then
echo "ERROR: Missing macOS arm64 archive in assets/ for Homebrew validation" >&2
exit 1
fi
if [ ${#x86_64_candidates[@]} -eq 0 ]; then
echo "ERROR: Missing macOS x86_64 archive in assets/ for Homebrew validation" >&2
exit 1
fi
verify_asset_url() {
local asset_name="$1"
local url="https://github.com/${REPO}/releases/download/${TAG}/${asset_name}"
local attempts=12
for attempt in $(seq 1 "$attempts"); do
if curl -fsIL "$url" >/dev/null; then
echo "Verified Homebrew asset URL: $url"
return 0
fi
if [ "$attempt" -eq "$attempts" ]; then
echo "ERROR: Homebrew asset URL not reachable after publish: $url" >&2
return 1
fi
echo "Homebrew asset URL not reachable yet (attempt ${attempt}/${attempts}); waiting 10s: $url" >&2
sleep 10
done
}
verify_asset_url "$(basename "${arm64_candidates[0]}")"
verify_asset_url "$(basename "${x86_64_candidates[0]}")"
- name: Publish APT/YUM/latest.json to R2 (stable only)
if: ${{ steps.meta.outputs.channel == 'stable' && github.actor != 'nektos/act' }}
env:
CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN || secrets.R2_SECRET_ACCESS_KEY }}
CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
GPG_PUBLIC_KEY_B64: ${{ secrets.GPG_PUBLIC_KEY_B64 || '' }}
shell: bash
run: |
set -euo pipefail
if [ -z "${CLOUDFLARE_API_TOKEN:-}" ] || [ -z "${CLOUDFLARE_ACCOUNT_ID:-}" ]; then
echo "Cloudflare API token/account missing; aborting R2 publish." >&2
exit 1
fi
WRANGLER_ROOT="${RUNNER_TEMP:-/tmp}/wrangler-cli"
WRANGLER_BIN="$WRANGLER_ROOT/node_modules/.bin/wrangler"
if [ ! -x "$WRANGLER_BIN" ]; then
npm install --prefix "$WRANGLER_ROOT" wrangler@4.69.0 >/dev/null
fi
"$WRANGLER_BIN" --version >&2
upload_file() {
local src="$1"; shift
local key="$1"; shift
local ct="application/octet-stream"
case "$key" in
*.json) ct="application/json" ;;
*.key) ct="text/plain" ;;
*.txt) ct="text/plain" ;;
*.xml) ct="application/xml" ;;
*.gz) ct="application/gzip" ;;
*.deb) ct="application/vnd.debian.binary-package" ;;
*.rpm) ct="application/x-rpm" ;;
*.asc) ct="application/pgp-signature" ;;
esac
echo "-- Upload $key" >&2
"$WRANGLER_BIN" r2 object put "yams-repository/$key" \
--remote \
--file "$src" \
--content-type "$ct"
}
if [ -n "${GPG_PUBLIC_KEY_B64:-}" ]; then
echo "$GPG_PUBLIC_KEY_B64" | base64 -d > gpg.key
upload_file gpg.key gpg.key
fi
if [ -d aptrepo ]; then
find aptrepo -type f -print0 | while IFS= read -r -d '' f; do
rel="${f#aptrepo/}"; upload_file "$f" "aptrepo/$rel";
done
fi
if [ -d yumrepo ]; then
find yumrepo -type f -print0 | while IFS= read -r -d '' f; do
rel="${f#yumrepo/}"; upload_file "$f" "yumrepo/$rel";
done
fi
if [ -f assets/latest.json ]; then
upload_file assets/latest.json latest.json
fi
echo "R2 publish complete." >&2
- name: List release assets (post-upload)
if: ${{ github.actor != 'nektos/act' }}
shell: bash
run: |
if ! command -v gh >/dev/null 2>&1; then
echo "Installing gh CLI for asset listing" >&2
type -p curl >/dev/null || sudo apt-get update
sudo apt-get install -y gh >/dev/null 2>&1 || true
fi
TAG="${{ steps.meta.outputs.tag }}"
echo "Release assets for $TAG:" >&2
gh release view "$TAG" --json assets --jq '.assets[].name' || echo "(Could not list assets)"
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
Reference in a new issue View git blame Copy permalink