# yamllint disable rule:line-length name: Release "on": push: tags: - "v*" - "yams-v*" schedule: - cron: "0 6 * * *" # 06:00 UTC daily workflow_dispatch: inputs: channel: description: "Release channel (stable|weekly|nightly)" required: false default: "nightly" fast_mode: description: "Skip packaging & verification steps (speed up local / act runs)" required: false default: "false" macos_only: description: "Limit matrix to macOS variants only (skip linux-hosted)" required: false default: "false" permissions: contents: write jobs: resolve-release: name: Resolve immutable release source runs-on: ubuntu-latest outputs: source_sha: ${{ steps.source.outputs.source_sha }} source_ref: ${{ steps.source.outputs.source_ref }} channel: ${{ steps.meta.outputs.channel }} version: ${{ steps.meta.outputs.version }} tag: ${{ steps.meta.outputs.tag }} release_name: ${{ steps.meta.outputs.release_name }} base_version: ${{ steps.meta.outputs.base_version }} prerelease: ${{ steps.meta.outputs.prerelease }} tests_run_id: ${{ steps.tests.outputs.tests_run_id }} tests_run_attempt: ${{ steps.tests.outputs.tests_run_attempt }} tests_run_url: ${{ steps.tests.outputs.tests_run_url }} permissions: contents: read actions: read steps: - name: Resolve canonical source once id: source uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7 env: REQUESTED_CHANNEL: ${{ inputs.channel || 'nightly' }} with: script: | const requestedChannel = process.env.REQUESTED_CHANNEL; let channel = 'stable'; if (context.eventName === 'schedule') { channel = 'nightly'; } else if (context.eventName === 'workflow_dispatch') { if (!['stable', 'weekly', 'nightly'].includes(requestedChannel)) { core.setFailed(`Unsupported release channel: ${requestedChannel}`); return; } channel = requestedChannel; } core.setOutput('channel', channel); let sourceSha; let sourceRef; if (channel !== 'stable') { const ref = await github.rest.git.getRef({ owner: context.repo.owner, repo: context.repo.repo, ref: 'heads/experimental', }); sourceSha = ref.data.object.sha; sourceRef = 'refs/heads/experimental'; } else { sourceSha = context.sha; sourceRef = context.ref; } if (!/^[0-9a-f]{40}$/.test(sourceSha)) { core.setFailed(`Resolved source is not a full commit SHA: ${sourceSha}`); return; } core.setOutput('source_sha', sourceSha); core.setOutput('source_ref', sourceRef); - name: Require successful Tests run for exact experimental source id: tests if: steps.source.outputs.channel != 'stable' uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7 env: SOURCE_SHA: ${{ steps.source.outputs.source_sha }} with: script: | const sourceSha = process.env.SOURCE_SHA; if (!/^[0-9a-f]{40}$/.test(sourceSha)) { core.setFailed(`Refusing Tests lookup for invalid source SHA: ${sourceSha}`); return; } const runs = await github.paginate(github.rest.actions.listWorkflowRuns, { owner: context.repo.owner, repo: context.repo.repo, workflow_id: 'tests.yml', head_sha: sourceSha, status: 'completed', per_page: 100, }); const successful = runs .filter((run) => run.name === 'Tests' && run.status === 'completed' && run.conclusion === 'success' && run.head_sha === sourceSha) .sort((left, right) => right.id - left.id); if (successful.length === 0) { // Name the red streak instead of refusing silently: the newest failing Tests run // on experimental, its SHA and failing jobs, and when experimental was last green. // Diagnostics only; the refusal below stands whatever they find. const lines = [ `No successful completed Tests workflow run has exact head_sha ${sourceSha}.`, ]; try { const sameSha = runs.filter((run) => run.head_sha === sourceSha); if (sameSha.length > 0) { const latest = sameSha.sort((left, right) => right.id - left.id)[0]; lines.push(`Tests on this SHA concluded ${latest.conclusion}: ${latest.html_url}`); } const recent = (await github.rest.actions.listWorkflowRuns({ owner: context.repo.owner, repo: context.repo.repo, workflow_id: 'tests.yml', branch: 'experimental', event: 'push', status: 'completed', per_page: 50, })).data.workflow_runs.filter((run) => run.conclusion !== 'cancelled'); const failing = recent.find((run) => run.conclusion !== 'success'); const lastGreen = recent.find((run) => run.conclusion === 'success'); if (failing) { const jobs = await github.paginate(github.rest.actions.listJobsForWorkflowRun, { owner: context.repo.owner, repo: context.repo.repo, run_id: failing.id, per_page: 100, }); const failedJobs = jobs .filter((job) => job.conclusion === 'failure' || job.conclusion === 'timed_out') .map((job) => job.name); lines.push( `Newest failing Tests run on experimental: ${failing.html_url} ` + `(sha ${failing.head_sha.slice(0, 12)}, ${failing.created_at}, ${failing.conclusion}).`, `Failing jobs: ${failedJobs.length > 0 ? failedJobs.join(', ') : '(none reported)'}.`, ); } const streak = lastGreen ? recent.indexOf(lastGreen) : recent.length; lines.push(lastGreen ? `Last green Tests run on experimental: ${lastGreen.html_url} ` + `(${lastGreen.created_at}); ${streak} completed push run(s) since are red.` : `No green Tests run among the last ${recent.length} completed experimental push runs.`); } catch (error) { lines.push(`Could not summarize recent Tests runs: ${error.message}`); } for (const line of lines.slice(1)) { core.error(line); } await core.summary .addHeading('Nightly refused: experimental Tests are not green', 3) .addList(lines) .write(); core.setFailed(lines[0]); return; } const run = successful[0]; if (!run.html_url || !run.html_url.endsWith(`/actions/runs/${run.id}`) || !Number.isInteger(run.run_attempt) || run.run_attempt < 1) { core.setFailed(`Tests run ${run.id} returned incomplete immutable identity`); return; } const attemptUrl = `${run.html_url}/attempts/${run.run_attempt}`; core.info(`Using exact-sha Tests run ${run.id} attempt ${run.run_attempt}: ${attemptUrl}`); core.setOutput('tests_run_id', String(run.id)); core.setOutput('tests_run_attempt', String(run.run_attempt)); core.setOutput('tests_run_url', attemptUrl); - name: Checkout immutable source uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 with: fetch-depth: 0 submodules: false ref: ${{ steps.source.outputs.source_sha }} - name: Resolve release metadata once id: meta env: RESOLVED_CHANNEL: ${{ steps.source.outputs.channel }} shell: bash run: | set -euo pipefail SHORT_HASH=$(git rev-parse --short HEAD) SOURCE_SHA=$(git rev-parse HEAD) channel="$RESOLVED_CHANNEL" if ! [[ "$SOURCE_SHA" =~ ^[0-9a-f]{40}$ ]]; then echo "Resolved source is not a full commit SHA: $SOURCE_SHA" >&2 exit 1 fi if [ "$channel" = "stable" ]; then # A stable release must come from a version tag; dispatching from a branch would # otherwise publish a release (and tag) named after the branch, e.g. "main". case "${GITHUB_REF}" in refs/tags/v[0-9]*|refs/tags/yams-v[0-9]*) ;; *) echo "Stable releases must run from a refs/tags/v* or refs/tags/yams-v* ref, got: ${GITHUB_REF}" >&2 exit 1 ;; esac TAG="${GITHUB_REF_NAME}" VERSION="${TAG#yams-v}" if [ "$VERSION" = "$TAG" ]; then VERSION="${TAG#v}" fi TAG_OUT="$TAG" RELEASE_NAME="YAMS ${VERSION}" elif [ "$channel" = "nightly" ]; then DATE_ISO=$(date -u +%Y-%m-%d) DATE_COMPACT=$(date -u +%Y%m%d) VERSION="nightly-${DATE_COMPACT}-${SHORT_HASH}" TAG_OUT="experimental-nightly-${DATE_COMPACT}-${SOURCE_SHA}" RELEASE_NAME="YAMS Nightly ${DATE_ISO} (${SHORT_HASH})" else YEAR=$(date -u +%G) WEEK=$(date -u +%V) VERSION="weekly-${YEAR}w${WEEK}-${SHORT_HASH}" TAG_OUT="$VERSION" RELEASE_NAME="YAMS Weekly ${YEAR}-w${WEEK} (${SHORT_HASH})" fi BASE_NUMERIC_VERSION="$(git describe --tags --abbrev=0 --match='v*' 2>/dev/null || echo 0.0.0)" BASE_NUMERIC_VERSION="${BASE_NUMERIC_VERSION#v}" { echo "channel=$channel" echo "version=$VERSION" echo "tag=$TAG_OUT" echo "release_name=$RELEASE_NAME" echo "base_version=$BASE_NUMERIC_VERSION" if [ "$channel" = "stable" ]; then echo "prerelease=false" else echo "prerelease=true" fi } >> "$GITHUB_OUTPUT" check-commits: name: Check for new commits (nightly only) needs: resolve-release runs-on: ubuntu-latest if: github.event_name == 'schedule' && github.actor != 'nektos/act' outputs: has_new_commits: ${{ steps.check.outputs.has_new_commits }} last_release_sha: ${{ steps.check.outputs.last_release_sha }} steps: - name: Check if there are new commits since last nightly release id: check uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7 env: SOURCE_SHA: ${{ needs.resolve-release.outputs.source_sha }} with: script: | const owner = context.repo.owner; const repo = context.repo.repo; const currentSha = process.env.SOURCE_SHA; if (!/^[0-9a-f]{40}$/.test(currentSha)) { core.setFailed(`Invalid resolved source SHA: ${currentSha}`); return; } const releases = await github.paginate(github.rest.repos.listReleases, { owner, repo, per_page: 100, }); const immutableNightly = /^experimental-nightly-[0-9]{8}-[0-9a-f]{40}$/; const legacyNightly = /^nightly-[0-9]{8}-[0-9a-f]{7,40}$/; const latest = releases.find((release) => immutableNightly.test(release.tag_name ?? '') || legacyNightly.test(release.tag_name ?? '')); if (!latest) { core.info('No previous valid nightly release found, will create one'); core.setOutput('has_new_commits', 'true'); core.setOutput('last_release_sha', 'none'); return; } const tagName = latest.tag_name; let object = (await github.rest.git.getRef({ owner, repo, ref: `tags/${tagName}`, })).data.object; for (let depth = 0; object.type === 'tag' && depth < 5; depth += 1) { object = (await github.rest.git.getTag({ owner, repo, tag_sha: object.sha, })).data.object; } if (object.type !== 'commit' || !/^[0-9a-f]{40}$/.test(object.sha)) { core.setFailed(`Nightly tag ${tagName} did not resolve safely to a commit`); return; } const lastSha = object.sha; core.setOutput('last_release_sha', lastSha); if (lastSha !== currentSha) { core.setOutput('has_new_commits', 'true'); try { const compare = await github.rest.repos.compareCommits({ owner, repo, base: lastSha, head: currentSha, }); const total = compare.data.total_commits ?? 1; core.info(`Found ${total} new commit(s) since last nightly (${tagName})`); } catch (error) { core.info(`Release SHAs differ; comparison was unavailable: ${error.message}`); } } else { core.setOutput('has_new_commits', 'false'); core.info(`No new commits since last nightly release (${tagName}), skipping`); } warm: needs: [resolve-release, check-commits] if: | always() && needs.resolve-release.result == 'success' && github.actor != 'nektos/act' && (needs.check-commits.result == 'skipped' || needs.check-commits.outputs.has_new_commits == 'true' || github.event_name == 'push' || github.event_name == 'workflow_dispatch') uses: ./.github/workflows/matrix-warm.yml with: build_type: Release enable_onnx: false checkout_ref: ${{ needs.resolve-release.outputs.source_sha }} build-release: name: Build and Package (matrix) needs: [resolve-release, warm] if: | always() && needs.resolve-release.result == 'success' && (needs.warm.result == 'success' || needs.warm.result == 'skipped') strategy: fail-fast: false matrix: include: - os: linux-hosted runs_on: ubuntu-24.04 suffix: linux-x86_64 packaging: tar build_packages: true meson_args: "" hosted: true - os: linux-hosted-arm64 runs_on: ubuntu-24.04-arm suffix: linux-arm64 packaging: tar build_packages: true meson_args: "" hosted: true - os: macos-hosted-arm64 runs_on: macos-15 suffix: macos-arm64 packaging: zip build_packages: false meson_args: "" # arch via profile edit hosted: true - os: macos-hosted-x64 runs_on: macos-15 suffix: macos-x86_64 packaging: zip build_packages: false meson_args: "" hosted: true - os: windows-hosted runs_on: windows-latest suffix: windows-x86_64 packaging: msi build_packages: true meson_args: "" hosted: true - os: arch-linux-hosted-x86_64 runs_on: ubuntu-24.04 suffix: arch-x86_64 packaging: pkg.tar.zst build_packages: true meson_args: "" hosted: false arch_container: true arch_name: x86_64 expected_host_arch: x86_64 docker_platform: linux/amd64 arch_base_image: archlinux/archlinux:latest - os: arch-linux-hosted-aarch64 runs_on: ubuntu-24.04-arm suffix: arch-aarch64 packaging: pkg.tar.zst build_packages: true meson_args: "" hosted: false arch_container: true arch_name: aarch64 expected_host_arch: aarch64 docker_platform: linux/arm64 arch_base_image: ghcr.io/menci/archlinuxarm@sha256:95491e07ec919efcbb134e3b228b7e5869938448985becaeaa4dc0e728a6499a runs-on: ${{ matrix.runs_on }} env: BUILD_DIR: build/release STAGE_DIR: stage steps: - name: Checkout uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 with: fetch-depth: 0 submodules: false # Manual submodule init below (some use SSH URLs) ref: ${{ needs.resolve-release.outputs.source_sha }} - name: Initialize required submodules shell: bash run: | retry_cmd() { local attempts="$1" shift local attempt=1 until "$@"; do if [ "$attempt" -ge "$attempts" ]; then return 1 fi echo "Command failed (attempt ${attempt}/${attempts}): $*" >&2 sleep $((attempt * 5)) attempt=$((attempt + 1)) done } # Force HTTPS for GitHub to handle SSH submodule URLs in CI git config --global url."https://github.com/".insteadOf "git@github.com:" # Init submodules required for the build # Skip other submodules that use SSH URLs requiring authentication retry_cmd 4 git submodule update --init --depth 1 third_party/sqlite-vec-cpp retry_cmd 4 git submodule update --init --depth 1 third_party/simeon retry_cmd 4 git submodule update --init --depth 1 third_party/symspell - name: Skip non-linux entries under act (build) if: github.actor == 'nektos/act' && !startsWith(matrix.os, 'linux-hosted') shell: bash run: | echo "Skipping build-release job steps for ${{ matrix.os }} under act" exit 0 - name: Skip linux-hosted when macos_only=true if: inputs.macos_only == 'true' && matrix.os == 'linux-hosted' shell: bash run: | echo "macos_only=true: skipping linux-hosted build-release job entry" exit 0 - name: Build Arch Linux (native container platform) if: matrix.arch_container == true shell: bash run: | set -euo pipefail arch_name="${{ matrix.arch_name }}" expected_host_arch="${{ matrix.expected_host_arch }}" docker_platform="${{ matrix.docker_platform }}" host_arch="$(uname -m)" if [ "$host_arch" != "$expected_host_arch" ]; then echo "ERROR: Arch lane requires native host ${expected_host_arch}, got ${host_arch}" >&2 exit 1 fi docker_arch="$(docker info --format '{{.Architecture}}')" case "$docker_arch" in amd64 | x86_64) docker_arch="x86_64" ;; arm64 | aarch64) docker_arch="aarch64" ;; *) echo "ERROR: unsupported Docker host architecture: $docker_arch" >&2; exit 1 ;; esac if [ "$docker_arch" != "$expected_host_arch" ]; then echo "ERROR: Docker host ${docker_arch} is not native ${expected_host_arch}" >&2 exit 1 fi build_dir="build/pkg/arch-${arch_name}" meson_build_dir="${build_dir}/meson-${arch_name}" channel="${{ needs.resolve-release.outputs.channel }}" version="${{ needs.resolve-release.outputs.version }}" if [ "$channel" = "stable" ]; then git_ref_value="${{ needs.resolve-release.outputs.source_ref }}" else git_ref_value="refs/tags/v${version}" fi echo "Arch release channel=${channel} version=${version} host=${host_arch} platform=${docker_platform} build_dir=${build_dir} meson_build_dir=${meson_build_dir}" docker build --platform "${docker_platform}" \ --build-arg "ARCH_BASE_IMAGE=${{ matrix.arch_base_image }}" \ -t yams/ci-arch:latest -f docker/local-ci/arch-package.Dockerfile . mkdir -p "${build_dir}" docker run --rm \ --platform "${docker_platform}" \ -v "${GITHUB_WORKSPACE}:/workspace/yams" \ -v "yams-arch-conan-cache-${arch_name}:/root/.conan2" \ -w /workspace \ -e RELEASE_CHANNEL="${channel}" \ -e GIT_REF="${git_ref_value}" \ -e BUILD_DIR="${build_dir}" \ -e YAMS_BUILD_DIR="${meson_build_dir}" \ -e HOST_UID=0 \ -e HOST_GID=0 \ yams/ci-arch:latest \ bash -lc '/workspace/yams/scripts/build-arch-pkg.sh build' shopt -s nullglob packages=("${build_dir}"/yams-*-"${arch_name}".pkg.tar.zst) shopt -u nullglob if [ "${#packages[@]}" -ne 1 ]; then echo "ERROR: expected exactly one ${arch_name} Arch package, found ${#packages[@]}" >&2 exit 1 fi cp -v "${packages[0]}" "${GITHUB_WORKSPACE}/" package_path="${GITHUB_WORKSPACE}/$(basename "${packages[0]}")" echo "Arch build complete. Artifacts in ${build_dir}:" find "${build_dir}" -maxdepth 4 -type f \( -name 'yams-*.pkg.tar.zst' -o -name 'yams.db' -o -name 'yams.files' \) -print { echo "BUILD_DIR=${build_dir}" echo "ARCH_PKG_PATH=${package_path}" } >> "${GITHUB_ENV}" - name: Inspect and install-validate Arch package natively if: matrix.arch_container == true shell: bash env: ARCH_DOCKER_PLATFORM: ${{ matrix.docker_platform }} ARCH_BASE_IMAGE: ${{ matrix.arch_base_image }} GH_TOKEN: ${{ github.token }} run: | set -euo pipefail : "${ARCH_PKG_PATH:?ARCH_PKG_PATH not set}" expected_arch="${{ matrix.arch_name }}" expected_host_arch="${{ matrix.expected_host_arch }}" if [ "$(uname -m)" != "$expected_host_arch" ]; then echo "ERROR: refusing non-native Arch validation for ${expected_host_arch}" >&2 exit 1 fi package_name="$(basename "$ARCH_PKG_PATH")" package_info="$(docker run --rm \ --platform "${ARCH_DOCKER_PLATFORM}" \ -v "${GITHUB_WORKSPACE}:/workspace/yams:ro" \ -w /workspace/yams \ yams/ci-arch:latest \ bsdtar -xOf "$package_name" .PKGINFO)" metadata_arch="$(printf '%s\n' "$package_info" | sed -n 's/^arch = //p')" if [ "$metadata_arch" != "$expected_arch" ]; then echo "ERROR: package metadata architecture ${metadata_arch:-missing} != ${expected_arch}" >&2 exit 1 fi printf '%s\n' "$package_info" | grep -F 'pkgname = yams' >/dev/null echo "Validated package metadata architecture: ${metadata_arch}" # Publish gate: install, layout, service, CLI, reboot, removal and the # upgrade from the latest published release (when it has an Arch package). upgrade_args=() prev_dir="$(mktemp -d)" prev_tag="$(gh release list -R "$GITHUB_REPOSITORY" --exclude-drafts --exclude-pre-releases --limit 1 --json tagName -q '.[0].tagName' 2>/dev/null || true)" if [ -n "$prev_tag" ] && gh release download "$prev_tag" -R "$GITHUB_REPOSITORY" -D "$prev_dir" -p "yams-*-${expected_arch}.pkg.tar.zst" 2>/dev/null; then prev_pkg="$(find "$prev_dir" -name 'yams-*.pkg.tar.zst' -print -quit)" if [ -n "$prev_pkg" ]; then upgrade_args=(--upgrade-from-arch "$prev_pkg"); fi fi echo "Upgrade source: ${prev_tag:-none} ${upgrade_args[*]:-}" chmod +x scripts/local-ci/package-validate.sh bash scripts/local-ci/package-validate.sh --only arch --arch "$ARCH_PKG_PATH" "${upgrade_args[@]}" - name: Skip remaining steps for Arch container builds if: matrix.arch_container == true shell: bash run: | echo "Arch container build completed — skipping host-level build steps" # Signal success; subsequent packaging/upload steps handle arch artifacts - name: Verify runner architecture (act diagnostics) if: github.actor == 'nektos/act' && matrix.os == 'linux-hosted' && matrix.arch_container != true shell: bash run: | set -euo pipefail ACT_UNAME="$(uname -m || true)" echo "Detected uname -m: ${ACT_UNAME}" if [ "${ACT_UNAME}" != "x86_64" ]; then echo "ERROR: Non-x86_64 architecture detected under act: ${ACT_UNAME}" >&2 echo "cpuinfo (and other x86-specific code) will not build on this architecture." >&2 echo "Re-run forcing amd64 containers, e.g.:" >&2 echo " act --container-architecture linux/amd64 --job build-release --input fast_mode=true" >&2 exit 1 fi - name: Cache Conan packages if: github.actor != 'nektos/act' && matrix.arch_container != true id: cache_conan uses: actions/cache@v4 continue-on-error: true with: path: | ~/.conan2/p ~/.conan2/r ~/.conan2/recipes ~/.conan2/metadata ~/.conan2/s key: conan2-${{ runner.os }}-${{ matrix.suffix }}-${{ hashFiles('conanfile.py', 'conan/profiles/**') }} restore-keys: | conan2-${{ runner.os }}-${{ matrix.suffix }}- conan2-${{ runner.os }}-linux-x86_64- conan2-${{ runner.os }}- - name: Conan cache diagnostics if: always() && matrix.arch_container != true shell: bash run: | echo "Cache hit: ${{ steps.cache_conan.outputs.cache-hit }}" for d in p r recipes metadata s; do if [ -d "$HOME/.conan2/$d" ]; then echo "Directory $d present: $(find "$HOME/.conan2/$d" -mindepth 1 -maxdepth 1 | wc -l | tr -d ' ')" entries || true du -sh "$HOME/.conan2/$d" 2>/dev/null || true else echo "Directory $d missing" fi done - name: Restore ccache if: github.actor != 'nektos/act' && matrix.arch_container != true uses: actions/cache/restore@v4 continue-on-error: true with: path: | ~/.cache/ccache ~/.ccache key: ccache-${{ runner.os }}-${{ matrix.suffix }}-${{ hashFiles('conanfile.py', 'meson.build', 'meson_options.txt') }}-${{ github.run_id }}-${{ github.run_attempt }} restore-keys: | ccache-${{ runner.os }}-${{ matrix.suffix }}- ccache-${{ runner.os }}- - name: Install optional system dependencies (Linux only) if: runner.os == 'Linux' && matrix.arch_container != true continue-on-error: true shell: bash run: | set -euo pipefail sudo apt-get update -o Acquire::Retries=3 || sudo apt-get update -o Acquire::Retries=3 || true # Add rpm / fakeroot / dpkg-dev so CPack can emit .deb and .rpm packages sudo apt-get install -y --no-install-recommends \ liburing-dev \ libarchive-dev \ libtag1-dev \ libboost-all-dev \ pkg-config \ ccache \ rpm \ fakeroot \ dpkg-dev \ meson \ cmake \ ninja-build \ clang \ clang-tools \ xz-utils \ lld || true echo "Installed optional packages: liburing-dev libarchive-dev libtag1-dev libboost-all-dev ccache rpm fakeroot dpkg-dev meson cmake ninja-build clang clang-tools lld" # Ensure compilers are preferred echo "CC=clang" >> "$GITHUB_ENV" echo "CXX=clang++" >> "$GITHUB_ENV" - name: Install Zig (Linux) if: runner.os == 'Linux' && matrix.arch_container != true shell: bash run: | set -euo pipefail ARCH=$(uname -m) case "$ARCH" in x86_64) ZIG_ARCH="x86_64" ;; aarch64) ZIG_ARCH="aarch64" ;; *) echo "Unsupported architecture: $ARCH"; exit 1 ;; esac ZIG_VERSION="0.15.2" curl -fsSL "https://ziglang.org/download/${ZIG_VERSION}/zig-${ZIG_ARCH}-linux-${ZIG_VERSION}.tar.xz" -o /tmp/zig.tar.xz sudo tar -xJf /tmp/zig.tar.xz -C /opt sudo mv /opt/zig-${ZIG_ARCH}-linux-${ZIG_VERSION} /opt/zig sudo ln -sf /opt/zig/zig /usr/local/bin/zig rm /tmp/zig.tar.xz zig version - name: Sanitize stdlib flags (macOS only) if: runner.os == 'macOS' shell: bash run: | set -euo pipefail # Remove any accidentally injected libstdc++ selections and force libc++ for var in CXXFLAGS CFLAGS LDFLAGS; do current="${!var:-}" sanitized="$(echo "$current" | sed -E 's/-stdlib=libstdc\+\+//g') -stdlib=libc++" # Collapse whitespace sanitized="$(echo "$sanitized" | tr -s ' ')" echo "$var=$sanitized" >> "$GITHUB_ENV" printf '%s\n' "$var=$sanitized" done echo "Stdlib flags sanitized for macOS (forcing -stdlib=libc++)." - name: Load resolved release metadata if: matrix.arch_container != true id: meta shell: bash run: | { echo "channel=${{ needs.resolve-release.outputs.channel }}" echo "version=${{ needs.resolve-release.outputs.version }}" echo "tag=${{ needs.resolve-release.outputs.tag }}" echo "release_name=${{ needs.resolve-release.outputs.release_name }}" echo "base_version=${{ needs.resolve-release.outputs.base_version }}" echo "prerelease=${{ needs.resolve-release.outputs.prerelease }}" } >> "$GITHUB_OUTPUT" - name: Download CI benchmark artifacts (-yams, if available) if: github.event_name == 'workflow_run' && matrix.os == 'linux-hosted' && github.actor != 'nektos/act' uses: dawidd6/action-download-artifact@v6 continue-on-error: true with: run_id: ${{ github.event.workflow_run.id }} workflow_conclusion: completed name: benchmark-results-linux-hosted-yams path: ci-artifacts/benchmarks if_no_artifact_found: warn - name: Download CI benchmark artifacts (legacy -conan, fallback) if: github.event_name == 'workflow_run' && matrix.os == 'linux-hosted' && github.actor != 'nektos/act' uses: dawidd6/action-download-artifact@v6 continue-on-error: true with: run_id: ${{ github.event.workflow_run.id }} workflow_conclusion: completed name: benchmark-results-linux-hosted-conan path: ci-artifacts/benchmarks if_no_artifact_found: warn # Hosted runners: install and configure Conan (self-hosted must already be configured) - name: Install Conan (hosted only) if: ${{ matrix.hosted }} shell: bash run: | set -euo pipefail PYTOOLS_DIR="${RUNNER_TEMP}/yams-python-tools" if [ "$RUNNER_OS" = "Windows" ]; then VENV_PYTHON="$PYTOOLS_DIR/Scripts/python.exe" TOOL_BIN="$PYTOOLS_DIR/Scripts" if [ ! -f "$VENV_PYTHON" ]; then python -m venv "$PYTOOLS_DIR" fi else VENV_PYTHON="$PYTOOLS_DIR/bin/python" TOOL_BIN="$PYTOOLS_DIR/bin" if [ ! -x "$VENV_PYTHON" ]; then python3 -m venv "$PYTOOLS_DIR" fi fi "$VENV_PYTHON" -m pip install --upgrade pip "$VENV_PYTHON" -m pip install --upgrade conan meson 'ninja>=1.10' echo "$TOOL_BIN" >> "$GITHUB_PATH" export PATH="$TOOL_BIN:$PATH" conan --version meson --version ninja --version mkdir -p ~/.conan2 # Best-effort: global.conf may not exist in repo for hosted macOS cp .conan/global.conf ~/.conan2/global.conf 2>/dev/null || true - name: Ensure build tools (meson, ninja) for local runners/act if: ${{ matrix.hosted && runner.os != 'Windows' }} shell: bash run: | set -euo pipefail if [ "$RUNNER_OS" = "macOS" ]; then brew update || true brew install meson ninja ccache || true elif [ "$RUNNER_OS" = "Linux" ]; then if command -v meson >/dev/null 2>&1 && command -v ninja >/dev/null 2>&1; then echo "meson/ninja already available on PATH" else sudo apt-get update || true sudo apt-get install -y --no-install-recommends meson ninja-build || true fi if ! command -v meson >/dev/null 2>&1 || ! command -v ninja >/dev/null 2>&1; then python3 -m pip install --user --upgrade --break-system-packages meson 'ninja>=1.10' || \ python -m pip install --user --upgrade --break-system-packages meson 'ninja>=1.10' || true fi if [ -n "${GITHUB_PATH:-}" ]; then echo "$HOME/.local/bin" >> "$GITHUB_PATH" else export PATH="$HOME/.local/bin:$PATH" fi fi echo "meson version: $(meson --version || true)" echo "ninja version: $(ninja --version || true)" - name: Ensure meson and ninja (macOS) if: runner.os == 'macOS' shell: bash run: | set -euo pipefail # If meson or ninja are not on PATH, install via Homebrew for robustness NEED_INSTALL=0 command -v meson >/dev/null 2>&1 || NEED_INSTALL=1 command -v ninja >/dev/null 2>&1 || NEED_INSTALL=1 if [ "$NEED_INSTALL" -eq 1 ]; then brew update || true brew install meson ninja || true fi echo "meson version (post-check): $(meson --version || echo 'missing')" echo "ninja version (post-check): $(ninja --version || echo 'missing')" - name: Setup ccache env if: matrix.arch_container != true shell: bash run: | { echo "CCACHE_DIR=$HOME/.cache/ccache" echo "CCACHE_BASEDIR=$GITHUB_WORKSPACE" echo "CCACHE_COMPRESS=1" echo "CCACHE_MAXSIZE=700M" } >> "$GITHUB_ENV" ccache -z 2>/dev/null || true - name: Setup Conan (hosted Linux/macOS) if: ${{ matrix.hosted }} shell: bash run: | set -e conan profile detect --force echo "Conan detected default profile. A platform-specific profile (host-macos-apple-clang or host-linux-clang) will be applied explicitly later." - name: Detect clang version (log only) if: matrix.arch_container != true shell: bash run: | set -euo pipefail # Prefer clang++, fallback to clang if command -v clang++ >/dev/null 2>&1; then RAW_VER=$(clang++ --version | sed -E -n 's/.*version ([0-9]+).*/\1/p' | head -1 || true) elif command -v clang >/dev/null 2>&1; then RAW_VER=$(clang --version | sed -E -n 's/.*version ([0-9]+).*/\1/p' | head -1 || true) else RAW_VER="" fi if [ -z "$RAW_VER" ]; then echo "Could not auto-detect clang version; using profile defaults" >&2 else echo "Detected clang major: $RAW_VER" fi - name: Compare profile vs detected clang (advice) if: matrix.arch_container != true shell: bash run: | set -euo pipefail PROFILE_FILE="./conan/profiles/host-linux-clang" if [ -f "$PROFILE_FILE" ]; then PROFILE_VER=$(grep -E '^compiler.version=' "$PROFILE_FILE" | sed -E 's/.*=//') else PROFILE_VER="(missing profile)" fi if command -v clang++ >/dev/null 2>&1; then SYS_VER=$(clang++ --version | sed -E -n 's/.*version ([0-9]+).*/\1/p' | head -1 || true) elif command -v clang >/dev/null 2>&1; then SYS_VER=$(clang --version | sed -E -n 's/.*version ([0-9]+).*/\1/p' | head -1 || true) else SYS_VER="unknown" fi echo "Profile declares clang version: $PROFILE_VER" echo "System detected clang version: $SYS_VER" if [ -n "$SYS_VER" ] && [ "$SYS_VER" != "unknown" ] && [ "$PROFILE_VER" != "$SYS_VER" ]; then echo "NOTE: Mismatch between profile (clang $PROFILE_VER) and system (clang $SYS_VER)." >&2 echo " Consider either installing clang-$PROFILE_VER or adjusting profile to $SYS_VER for local/act runs to avoid subtle ABI or flag differences." >&2 fi - name: Check Conan (self-hosted sanity) if: ${{ !matrix.hosted && matrix.arch_container != true }} shell: bash run: conan --version || echo "Conan is expected to be pre-installed and configured on self-hosted runners" # Hosted runners: clean corrupted cache if needed - name: Clean corrupted Conan cache if needed (hosted only) if: ${{ matrix.hosted }} shell: bash run: | if ! conan list "*" 2>/dev/null; then echo "Conan cache appears corrupted, cleaning locks and temp files..." conan cache clean --locks --temp 2>/dev/null || true echo "Conan cache locks/temp cleaned" fi - name: Install system dependencies (macOS) if: runner.os == 'macOS' shell: bash run: | set -euo pipefail for attempt in 1 2 3; do if brew update; then break fi if [ "$attempt" -eq 3 ]; then echo "brew update failed after ${attempt} attempts" >&2 exit 1 fi sleep $((attempt * 10)) done brew install pkg-config cmake ninja meson ccache boost zig || true # Ensure Meson/Ninja are present even if Homebrew failed to link binaries. command -v meson >/dev/null 2>&1 || python3 -m pip install --user --upgrade meson command -v ninja >/dev/null 2>&1 || python3 -m pip install --user --upgrade 'ninja>=1.10' echo "meson version: $(meson --version || echo 'missing')" echo "ninja version: $(ninja --version || echo 'missing')" echo "zig version: $(zig version || echo 'missing')" - name: Tune Conan client (parallel downloads) if: matrix.arch_container != true shell: bash run: | set -euo pipefail CONF_FILE="$HOME/.conan2/global.conf" LINE="core.download:parallel=8" mkdir -p "$(dirname "$CONF_FILE")" if [ -f "$CONF_FILE" ]; then TMP_FILE="$(mktemp)" awk -v line="$LINE" ' BEGIN { replaced = 0 } /^core.download:parallel=/ { if (!replaced) { print line; replaced = 1 } ; next } { print } END { if (!replaced) print line } ' "$CONF_FILE" > "$TMP_FILE" mv "$TMP_FILE" "$CONF_FILE" else printf '%s\n' "$LINE" > "$CONF_FILE" fi echo "Configured $LINE in $CONF_FILE" grep -n '^core.download:parallel=' "$CONF_FILE" || true conan config show core.download:parallel 2>/dev/null || true conan --version - name: Warm key Conan packages (best-effort) if: ${{ matrix.hosted }} shell: bash run: | set +e # Try to prefetch heavy packages to minimize build-time compiles for ref in \ openjpeg/2.5.3 \ pdfium/95.0.4629 \ onnxruntime/1.18.1 \ icu/74.1 \ libarchive/3.7.6; do echo "Prefetching $ref ..." || true conan download "$ref" -r=conancenter || true done set -e # Windows: Install MSVC and build tools - name: Setup MSVC (Windows only) if: runner.os == 'Windows' uses: ilammy/msvc-dev-cmd@v1 with: arch: x64 - name: Install build tools (Windows only) if: runner.os == 'Windows' shell: pwsh run: | $pyToolsScripts = Join-Path $env:RUNNER_TEMP "yams-python-tools\Scripts" "$pyToolsScripts" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append $env:PATH = "$pyToolsScripts;$env:PATH" # Install WiX for MSI packaging (pin versions to avoid prerelease incompatibility) dotnet tool install --global wix --version 6.0.2 wix extension add -g WixToolset.UI.wixext/6.0.2 # Install Zig for zyp PDF plugin $zigVersion = "0.15.2" $zigUrl = "https://ziglang.org/download/${zigVersion}/zig-x86_64-windows-${zigVersion}.zip" $zigZip = "$env:TEMP\zig.zip" $zigDir = "C:\zig" Invoke-WebRequest -Uri $zigUrl -OutFile $zigZip Expand-Archive -Path $zigZip -DestinationPath $zigDir -Force Move-Item "$zigDir\zig-x86_64-windows-${zigVersion}\*" $zigDir -Force Remove-Item "$zigDir\zig-x86_64-windows-${zigVersion}" -Recurse -ErrorAction SilentlyContinue $env:PATH = "$zigDir;$env:PATH" [Environment]::SetEnvironmentVariable("PATH", "$zigDir;$env:PATH", "Process") "$zigDir" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append # Verify tools conan --version meson --version ninja --version wix --version zig version - name: Configure and Build Release (Windows) if: runner.os == 'Windows' shell: pwsh run: | $ErrorActionPreference = "Stop" Write-Host "Building release for Windows x86_64" # Set version $env:YAMS_EXTRA_MESON_FLAGS = "-Dyams-version=${{ steps.meta.outputs.version }}" # Windows hosted runners can lose communication under unbounded Ninja parallelism. $env:YAMS_NINJA_JOBS = "2" # Run the build via setup.ps1 # ONNX Runtime is deprecated and opt-in; release artifacts still ship it for now. .\setup.ps1 -BuildType Release -WithOnnx # Set BUILD_DIR for subsequent steps "BUILD_DIR=build/release" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append # Release build for Linux/macOS targets (not Arch container builds) - name: Configure and Build Release (Optimized) if: runner.os != 'Windows' && matrix.arch_container != true shell: bash run: | echo "Optimized release build for ${{ matrix.suffix }}" # Align build directory with README (build/release) rm -rf build/release # Determine host profile and architecture based on platform if [ "$RUNNER_OS" = "macOS" ]; then case "${{ matrix.suffix }}" in macos-arm64) HOST_PROFILE="./conan/profiles/host-macos-apple-clang" ARCH="armv8" ;; macos-x86_64) HOST_PROFILE="./conan/profiles/host-macos-apple-clang-x86" ARCH="x86_64" ;; esac elif [ "$RUNNER_OS" = "Linux" ]; then HOST_PROFILE="./conan/profiles/host-linux-clang" case "${{ matrix.suffix }}" in linux-arm64) ARCH="armv8" ;; linux-x86_64) ARCH="x86_64" ;; esac fi # Hosted runners already provide CMake. Model it as a Conan platform # tool so cold ARM64 caches do not download another CMake from cmake.org. if [ "${{ matrix.hosted }}" = "true" ]; then SYSTEM_CMAKE_VERSION="$(cmake --version | sed -n -E '1s/^cmake version ([0-9]+\.[0-9]+\.[0-9]+).*$/\1/p')" if [ -z "$SYSTEM_CMAKE_VERSION" ]; then echo "Unable to detect the hosted CMake version" >&2 exit 1 fi CI_HOST_PROFILE="${RUNNER_TEMP}/yams-conan-host-${{ matrix.suffix }}" cp "$HOST_PROFILE" "$CI_HOST_PROFILE" printf '\n[platform_tool_requires]\ncmake/%s\n' "$SYSTEM_CMAKE_VERSION" >> "$CI_HOST_PROFILE" HOST_PROFILE="$CI_HOST_PROFILE" echo "Using platform CMake ${SYSTEM_CMAKE_VERSION} via ${HOST_PROFILE}" fi # Use setup.sh with CI environment variables for consistent configuration export YAMS_CONAN_HOST_PROFILE="$HOST_PROFILE" export YAMS_CONAN_ARCH="$ARCH" export YAMS_ENABLE_MOBILE_BINDINGS=false export YAMS_EXTRA_MESON_FLAGS="-Dyams-version=${{ steps.meta.outputs.version }} -Dwerror=false -Dwarning_level=2 ${{ matrix.meson_args }}" # Set install prefix: /usr for Linux packages, /usr/local for macOS if [ "$RUNNER_OS" = "Linux" ]; then export YAMS_INSTALL_PREFIX="/usr" else if [ "${{ matrix.suffix }}" = "macos-arm64" ]; then export YAMS_INSTALL_PREFIX="/opt/homebrew" else export YAMS_INSTALL_PREFIX="/usr/local" fi # Enable CoreML GPU acceleration for macOS builds export YAMS_ONNX_GPU="coreml" fi # Persist prefix for later steps echo "YAMS_INSTALL_PREFIX=$YAMS_INSTALL_PREFIX" >> "$GITHUB_ENV" # For cross-compilation, ensure pkg-config is available PKG_CONFIG="$(command -v pkg-config)" export PKG_CONFIG echo "Using pkg-config: $PKG_CONFIG" # Version is already included in YAMS_EXTRA_MESON_FLAGS above. Keep this # echo for diagnostics, but do not overwrite feature flags. VERSION="${{ steps.meta.outputs.version }}" echo "Setting version: ${VERSION}" # ONNX Runtime is deprecated and opt-in; release artifacts still ship it for now. ./setup.sh Release --with-onnx # Source Conan build env so meson compile uses the same Meson version # that setup.sh used for meson setup (Conan provides Meson 1.10.x which # creates build.dat incompatible with older system Meson). CONAN_BUILD_ENV="" for candidate in \ "build/release/build-release/conan/conanbuild.sh" \ "build/release/conan/conanbuild.sh" do if [ -f "$candidate" ]; then CONAN_BUILD_ENV="$candidate" break fi done if [ -n "$CONAN_BUILD_ENV" ]; then set +u # shellcheck disable=SC1090 source "$CONAN_BUILD_ENV" set -u fi meson --version meson compile -C build/release echo "BUILD_DIR=build/release" >> "$GITHUB_ENV" - name: Upload build logs (artifacts) if: failure() uses: actions/upload-artifact@v4 with: name: logs-${{ matrix.suffix }}-${{ github.run_number }} path: | build/release/meson-logs/ build/release/meson-info/ build/release/meson-logs/meson-log.txt build/release/build-release/conan/**/*.log if-no-files-found: ignore retention-days: 14 - name: ccache stats (post-build) shell: bash run: | ccache -s || echo "ccache stats unavailable" - name: Save ccache if: always() && github.actor != 'nektos/act' uses: actions/cache/save@v4 continue-on-error: true with: path: | ~/.cache/ccache ~/.ccache key: ccache-${{ runner.os }}-${{ matrix.suffix }}-${{ hashFiles('conanfile.py', 'meson.build', 'meson_options.txt') }}-${{ github.run_id }}-${{ github.run_attempt }} - name: Collect benchmark results if: always() && matrix.arch_container != true shell: bash run: | mkdir -p "$BUILD_DIR/bench_results" # Collect from validation build (if Linux hosted ran validation) if [ -f "build/release/bench_results.json" ]; then cp build/release/bench_results.json "$BUILD_DIR/bench_results/" echo "Collected benchmark results from validation build" fi # Also check CI artifacts if available if [ -d "ci-artifacts/benchmarks" ] && [ "$(ls -A ci-artifacts/benchmarks/*.json 2>/dev/null)" ]; then cp ci-artifacts/benchmarks/*.json "$BUILD_DIR/bench_results/" || true fi - name: Install (Linux/macOS) if: runner.os != 'Windows' && matrix.arch_container != true shell: bash run: | # Ensure BUILD_DIR is properly set if [ ! -d "$BUILD_DIR" ]; then echo "BUILD_DIR not found at $BUILD_DIR, attempting fallback to build/release (legacy path build/yams-release no longer used)." >&2 exit 1 fi # Install to an absolute stage path under the build dir to avoid path duplication BD="$(cd "$BUILD_DIR" && pwd)" CONAN_BUILD_ENV="" for candidate in \ "$BD/build-release/conan/conanbuild.sh" \ "$BD/conan/conanbuild.sh" do if [ -f "$candidate" ]; then CONAN_BUILD_ENV="$candidate" break fi done if [ -n "$CONAN_BUILD_ENV" ]; then set +u # shellcheck disable=SC1090 source "$CONAN_BUILD_ENV" set -u fi meson --version # --no-rebuild: setup.sh + a prior `meson compile` already built; # this step only stages files, never re-triggers the compiler. meson install -C "$BD" --destdir "$BD/stage" --no-rebuild if [ ! -d "$BD/stage" ]; then echo "Stage directory missing after meson install" >&2 ls -al "$BD" >&2 || true exit 1 fi echo "STAGE_DIR=$BD/stage" >> "$GITHUB_ENV" - name: Split debug symbols from staged payload (Linux/macOS) if: runner.os != 'Windows' && matrix.arch_container != true shell: bash run: | set -euo pipefail : "${STAGE_DIR:?STAGE_DIR not set}" PREFIX="${YAMS_INSTALL_PREFIX:-}" if [ -z "$PREFIX" ]; then if [ "$RUNNER_OS" = "Linux" ]; then PREFIX="/usr" else PREFIX="/usr/local" fi fi INSTALL_ROOT="$STAGE_DIR$PREFIX" if [ ! -d "$INSTALL_ROOT" ]; then echo "Missing staged install root: $INSTALL_ROOT" >&2 exit 1 fi # Release builds keep -g; ship stripped binaries and keep the symbols as # a separate workflow artifact (ELF .debug + .build-id links, macOS dSYM). bash scripts/split-debug-symbols.sh "$INSTALL_ROOT" "$BUILD_DIR/debug-symbols" - name: Verify staged runtime assets (Linux/macOS) if: runner.os != 'Windows' && matrix.arch_container != true shell: bash run: | set -euo pipefail : "${STAGE_DIR:?STAGE_DIR not set}" PREFIX="${YAMS_INSTALL_PREFIX:-}" if [ -z "$PREFIX" ]; then if [ "$RUNNER_OS" = "Linux" ]; then PREFIX="/usr" else PREFIX="/usr/local" fi fi if [ ! -f "$STAGE_DIR$PREFIX/share/yams/sql/reference_schema.sql" ]; then echo "Missing staged reference schema: $STAGE_DIR$PREFIX/share/yams/sql/reference_schema.sql" >&2 echo "Stage tree (top):" >&2 ls -al "$STAGE_DIR" >&2 || true exit 1 fi # Plugins are expected for full releases; fail loudly if absent/empty. if [ ! -d "$STAGE_DIR$PREFIX/lib/yams/plugins" ]; then echo "Missing staged plugins dir: $STAGE_DIR$PREFIX/lib/yams/plugins" >&2 exit 1 fi shopt -s nullglob plugin_bins=( "$STAGE_DIR$PREFIX/lib/yams/plugins"/*.so "$STAGE_DIR$PREFIX/lib/yams/plugins"/*.dylib ) if [ ${#plugin_bins[@]} -eq 0 ]; then echo "No plugin binaries staged under: $STAGE_DIR$PREFIX/lib/yams/plugins" >&2 ls -al "$STAGE_DIR$PREFIX/lib/yams/plugins" >&2 || true exit 1 fi - name: Check install surface (Linux/macOS) if: runner.os != 'Windows' && matrix.arch_container != true shell: bash run: | set -euo pipefail : "${STAGE_DIR:?STAGE_DIR not set}" PREFIX="${YAMS_INSTALL_PREFIX:-}" if [ -z "$PREFIX" ]; then if [ "$RUNNER_OS" = "Linux" ]; then PREFIX="/usr"; else PREFIX="/usr/local"; fi fi # Paths, size/export budgets, forbidden third-party exports, RUNPATH, # hardening and strip state of the exact payload that gets packaged. rc=0 python3 scripts/ci/check_install_surface.py \ --stage "$STAGE_DIR" --prefix "$PREFIX" --libdir lib \ --json "$BUILD_DIR/install-surface.json" > "$BUILD_DIR/install-surface.txt" || rc=$? cat "$BUILD_DIR/install-surface.txt" { echo '```'; cat "$BUILD_DIR/install-surface.txt"; echo '```'; } >> "$GITHUB_STEP_SUMMARY" exit "$rc" - name: Upload debug symbols (Linux/macOS) if: runner.os != 'Windows' && matrix.arch_container != true && github.actor != 'nektos/act' uses: actions/upload-artifact@v4 with: name: debug-symbols-${{ matrix.suffix }}${{ github.run_attempt > 1 && format('-a{0}', github.run_attempt) || '' }} path: ${{ env.BUILD_DIR }}/debug-symbols retention-days: 30 if-no-files-found: warn - name: Install and Package MSI (Windows) if: runner.os == 'Windows' && inputs.fast_mode != 'true' shell: pwsh run: | $ErrorActionPreference = "Stop" $buildDir = "build\release" $stageDir = Join-Path (Resolve-Path $buildDir).Path "stage" $version = "${{ steps.meta.outputs.version }}" Write-Host "Staging install to: $stageDir" # Remove old stage if exists if (Test-Path $stageDir) { Remove-Item -Recurse -Force $stageDir } New-Item -ItemType Directory -Path $stageDir -Force | Out-Null # Reconfigure if needed (handles Meson version mismatches from cached build dirs) $mesonInfo = Join-Path $buildDir "meson-private\coredata.dat" if (Test-Path $mesonInfo) { Write-Host "Reconfiguring build directory to ensure Meson version compatibility..." meson setup --reconfigure $buildDir if ($LASTEXITCODE -ne 0) { Write-Warning "Reconfigure failed, attempting fresh setup..." Remove-Item -Recurse -Force $buildDir -ErrorAction SilentlyContinue meson setup $buildDir --buildtype=release } } # Install to stage directory. # --no-rebuild: setup.ps1's release build already compiled; this # step only stages files, never re-triggers the compiler. meson install -C $buildDir --destdir $stageDir --no-rebuild if ($LASTEXITCODE -ne 0) { Write-Error "Meson install failed" exit 1 } # Smoke-test the staged CLI before packaging. A missing load-time DLL # (STATUS_DLL_NOT_FOUND, 0xC0000135, or 0xC00004BC) fails here # instead of shipping an MSI whose yams.exe cannot start (v0.20.3). $yamsExe = Get-ChildItem -Path $stageDir -Recurse -Filter "yams.exe" | Select-Object -First 1 if (-not $yamsExe) { Write-Error "Staged yams.exe not found under $stageDir" exit 1 } $proc = Start-Process -FilePath $yamsExe.FullName -ArgumentList "--version" ` -NoNewWindow -PassThru -RedirectStandardOutput "$buildDir\yams-version.txt" $null = $proc.Handle # cache the handle so ExitCode is populated if (-not $proc.WaitForExit(60000)) { $proc.Kill() Write-Error "Staged yams.exe --version did not exit within 60s" exit 1 } if ($proc.ExitCode -ne 0) { Write-Error ("Staged yams.exe --version exited 0x{0:X8}" -f $proc.ExitCode) exit 1 } Write-Host "Staged CLI: $(Get-Content "$buildDir\yams-version.txt" -Raw)" # Build MSI using the packaging script $msiScript = "packaging\windows\build-msi.ps1" & $msiScript -StageDir $stageDir -Version $version -OutputDir $buildDir if ($LASTEXITCODE -ne 0) { Write-Error "MSI build failed" exit 1 } # Move MSI to workspace root for artifact upload Get-ChildItem $buildDir -Filter "*.msi" | ForEach-Object { Copy-Item $_.FullName -Destination $env:GITHUB_WORKSPACE Write-Host "Created MSI: $($_.Name)" } # Set environment variables for subsequent steps "STAGE_DIR=$stageDir" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append "BUILD_DIR=$buildDir" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append - name: Generate winget manifest (stable, windows only) if: runner.os == 'Windows' && steps.meta.outputs.channel == 'stable' && inputs.fast_mode != 'true' shell: pwsh run: | $ErrorActionPreference = "Stop" $version = "${{ steps.meta.outputs.version }}" $tag = "${{ steps.meta.outputs.tag }}" # Find the Windows MSI produced in this job $msiFile = Get-ChildItem -Path $env:GITHUB_WORKSPACE -Filter "*.msi" | Select-Object -First 1 if (-not $msiFile) { Write-Error "No MSI file found in workspace; cannot generate winget manifest" exit 1 } Write-Host "Generating winget manifest for version $version" Write-Host "MSI file: $($msiFile.FullName)" & ./packaging/windows/generate-winget-manifest.ps1 ` -Version $version ` -Tag $tag ` -MsiPath $msiFile.FullName ` -GitHubRepo "${{ github.repository }}" ` -OutputDir "winget-manifests" if ($LASTEXITCODE -ne 0) { Write-Error "Winget manifest generation failed" exit 1 } - name: Generate release summary if: matrix.arch_container != true shell: bash run: | { echo "### Artifacts" echo "[View artifacts for this run]($GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID)" } >> "$GITHUB_STEP_SUMMARY" SUMMARY_FILE="$BUILD_DIR/release_summary.md" { echo "## Release Build Information (${{ steps.meta.outputs.version }})" echo echo "**Platform:** ${{ matrix.suffix }}" echo "**Build Type:** Release" echo "**Channel:** ${{ steps.meta.outputs.channel }}" } > "$SUMMARY_FILE" { # Only mention CI validation if we actually have CI artifacts if find "ci-artifacts" -mindepth 1 -print -quit >/dev/null 2>&1; then echo "**CI Status:** Tests and benchmarks from CI workflow" fi echo } >> "$SUMMARY_FILE" # Add benchmark results if available from CI or validation (jq preferred, fallback to grep) benchmark_json_files=() while IFS= read -r json_file; do benchmark_json_files+=("$json_file") done < <(find "$BUILD_DIR/bench_results" -maxdepth 1 -type f -name '*.json' -print 2>/dev/null | sort) if ((${#benchmark_json_files[@]} > 0)); then { echo "### Performance Metrics" echo echo "| Benchmark | Time (ns/op) | CPU (ns/op) | Memory |" echo "|-----------|--------------|-------------|--------|" } >> "$SUMMARY_FILE" if command -v jq >/dev/null 2>&1; then for json_file in "${benchmark_json_files[@]}"; do jq -r ' .benchmarks[]? | [ (.name // "Unknown"), ((.real_time // 0) | floor | tostring), ((.cpu_time // 0) | floor | tostring), (if ((.bytes_per_second // 0) > 0) then (((.bytes_per_second / 1024 / 1024) | floor | tostring) + " MB/s") elif ((.items_per_second // 0) > 0) then (((.items_per_second) | floor | tostring) + " ops/s") else "N/A" end) ] | "| \(.[0]) | \(.[1]) | \(.[2]) | \(.[3]) |" ' "$json_file" >> "$SUMMARY_FILE" done else # Minimal fallback: grep fields (less robust) for json_file in "${benchmark_json_files[@]}"; do grep '"name"' "$json_file" | sed -E 's/.*"name": *"([^"]+)".*/| \1 | ? | ? | N/A |/' >> "$SUMMARY_FILE" || true done fi echo "" >> "$SUMMARY_FILE" fi echo "Generated release summary at: $SUMMARY_FILE" cat "$SUMMARY_FILE" - name: Package if: ${{ inputs.fast_mode != 'true' && github.actor != 'nektos/act' && matrix.packaging != 'msi' && matrix.packaging != 'pkg.tar.zst' }} shell: bash run: | STAGE_PATH="${{ env.STAGE_DIR }}" if [ ! -d "$STAGE_PATH" ]; then echo "Stage directory not found at $STAGE_PATH" >&2 ls -al "$BUILD_DIR" >&2 || true exit 1 fi cd "$STAGE_PATH" VERSION="${{ steps.meta.outputs.version }}" SUFFIX="${{ matrix.suffix }}" OUTDIR="$GITHUB_WORKSPACE" if [ "${{ matrix.packaging }}" = "zip" ]; then ASSET="yams-${VERSION}-${SUFFIX}.zip" # -y stores symlinks (bin/yams -> yams-cli, versioned dylib links) as links # instead of a second full copy of the target. zip -ry "$OUTDIR/$ASSET" . else ASSET="yams-${VERSION}-${SUFFIX}.tar.gz" tar czf "$OUTDIR/$ASSET" . fi echo "ASSET_PATH=$OUTDIR/$ASSET" >> "$GITHUB_ENV" echo "Created asset: $OUTDIR/$ASSET" - name: Build Linux packages (DEB/RPM) if: ${{ matrix.build_packages == true && runner.os == 'Linux' && inputs.fast_mode != 'true' && github.actor != 'nektos/act' && matrix.packaging != 'pkg.tar.zst' }} env: BASE_VERSION: ${{ steps.meta.outputs.base_version }} shell: bash run: | # Package-only: reuse staged install without triggering a rebuild # Use BASE_VERSION to generate Debian/RPM policy-compliant versions for nightly/weekly chmod +x scripts/build-deb.sh scripts/build-deb.sh package_only "${{ steps.meta.outputs.version }}" "${{ env.BUILD_DIR }}" "stage" # Move packages to root directory for artifact upload if ls "${{ env.BUILD_DIR }}"/yams-*.deb 1> /dev/null 2>&1; then mv "${{ env.BUILD_DIR }}"/yams-*.deb ./ || true fi if ls "${{ env.BUILD_DIR }}"/yams-*.rpm 1> /dev/null 2>&1; then mv "${{ env.BUILD_DIR }}"/yams-*.rpm ./ || true fi - name: Smoke-test ONNX plugin load (Linux) if: ${{ runner.os == 'Linux' && inputs.fast_mode != 'true' && github.actor != 'nektos/act' && matrix.arch_container != true }} shell: bash run: | set -euo pipefail : "${STAGE_DIR:?STAGE_DIR not set}" PREFIX="${YAMS_INSTALL_PREFIX:-/usr}" YAMS_BIN="$STAGE_DIR$PREFIX/bin/yams" PLUGIN_SO="$STAGE_DIR$PREFIX/lib/yams/plugins/libyams_onnx_plugin.so" if [ ! -x "$YAMS_BIN" ]; then echo "ERROR: Missing staged yams binary: $YAMS_BIN" >&2 exit 1 fi if [ ! -f "$PLUGIN_SO" ]; then echo "ERROR: Missing staged ONNX plugin: $PLUGIN_SO" >&2 exit 1 fi echo "Checking ONNX plugin relocations" if ldd -r "$PLUGIN_SO" 2>&1 | tee /tmp/yams-onnx-ldd.txt | grep -q "undefined symbol"; then echo "ERROR: unresolved symbols detected in ONNX plugin" >&2 cat /tmp/yams-onnx-ldd.txt >&2 exit 1 fi tmp_data="$(mktemp -d)" export YAMS_DATA_DIR="$tmp_data" export YAMS_PLUGIN_DIR="$STAGE_DIR$PREFIX/lib/yams/plugins" export LD_LIBRARY_PATH="$STAGE_DIR$PREFIX/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" echo "Using LD_LIBRARY_PATH=$LD_LIBRARY_PATH" echo "Loading ONNX plugin via staged CLI" "$YAMS_BIN" plugin load "$PLUGIN_SO" "$YAMS_BIN" daemon stop >/dev/null 2>&1 || true # Release notes are sourced directly from CHANGELOG.md during release creation - name: Publish matrix.json (expected platform count) if: matrix.os == 'linux-hosted' && matrix.suffix == 'linux-x86_64' && matrix.arch_container != true shell: bash run: | # Only generate once to avoid artifact name collision across matrix entries. # Current matrix total: 7 artifacts (5 native platform builds + 2 native Arch lanes). echo '{"total": 7}' > matrix.json - name: Upload matrix metadata if: matrix.os == 'linux-hosted' && matrix.suffix == 'linux-x86_64' && matrix.arch_container != true && github.actor != 'nektos/act' uses: actions/upload-artifact@v4 with: name: matrix-meta path: matrix.json retention-days: 1 - name: Verify Linux package artifacts if: ${{ matrix.build_packages == true && runner.os == 'Linux' && github.actor != 'nektos/act' && inputs.fast_mode != 'true' && matrix.arch_container != true }} shell: bash run: | set -euo pipefail case "${{ matrix.suffix }}" in linux-x86_64) expected_arch="x86_64" ;; linux-arm64) expected_arch="aarch64" ;; *) echo "Skipping Linux package verification for unexpected suffix: ${{ matrix.suffix }}" >&2; exit 0 ;; esac echo "Verifying expected Linux package artifacts (DEB/RPM) for ${expected_arch}" ls -1 yams-*linux-${expected_arch}.deb 2>/dev/null || { echo "ERROR: Missing .deb package for ${expected_arch}" >&2; exit 1; } if command -v rpmbuild >/dev/null 2>&1; then ls -1 yams-*linux-${expected_arch}.rpm 2>/dev/null || { echo "ERROR: Missing .rpm package for ${expected_arch}" >&2; exit 1; } else echo "rpmbuild unavailable; RPM check skipped" >&2 fi echo "Linux package verification complete" - name: Validate Linux package install (systemd lanes) if: ${{ matrix.build_packages == true && runner.os == 'Linux' && github.actor != 'nektos/act' && inputs.fast_mode != 'true' && matrix.arch_container != true }} shell: bash env: GH_TOKEN: ${{ github.token }} run: | set -euo pipefail case "${{ matrix.suffix }}" in linux-x86_64) expected_arch="x86_64" ;; linux-arm64) expected_arch="aarch64" ;; *) echo "Skipping package install validation for suffix: ${{ matrix.suffix }}" >&2; exit 0 ;; esac host_arch="$(uname -m)" if [ "${host_arch}" != "${expected_arch}" ]; then echo "ERROR: runner arch (${host_arch}) != package arch (${expected_arch})" >&2 exit 1 fi deb="$(find . -maxdepth 1 -type f -name "yams-*linux-${expected_arch}.deb" -print -quit | sed 's#^\./##')" rpm="$(find . -maxdepth 1 -type f -name "yams-*linux-${expected_arch}.rpm" -print -quit | sed 's#^\./##')" args=() [ -n "${deb}" ] && args+=(--deb "${deb}") [ -n "${rpm}" ] && args+=(--rpm "${rpm}") if [ -z "${deb}" ] && [ -z "${rpm}" ]; then echo "ERROR: no Linux packages found to validate" >&2 exit 1 fi # Upgrade path: the same lanes first install the latest published # release's packages, store a document, then upgrade to these. prev_dir="$(mktemp -d)" prev_tag="$(gh release list -R "$GITHUB_REPOSITORY" --exclude-drafts --exclude-pre-releases --limit 1 --json tagName -q '.[0].tagName' 2>/dev/null || true)" if [ -n "$prev_tag" ]; then gh release download "$prev_tag" -R "$GITHUB_REPOSITORY" -D "$prev_dir" \ -p "yams-*linux-${expected_arch}.deb" -p "yams-*linux-${expected_arch}.rpm" 2>/dev/null || true prev_deb="$(find "$prev_dir" -name 'yams-*.deb' -print -quit)" prev_rpm="$(find "$prev_dir" -name 'yams-*.rpm' -print -quit)" if [ -n "${prev_deb}" ] && [ -n "${deb}" ]; then args+=(--upgrade-from-deb "${prev_deb}"); fi if [ -n "${prev_rpm}" ] && [ -n "${rpm}" ]; then args+=(--upgrade-from-rpm "${prev_rpm}"); fi fi echo "Upgrade source: ${prev_tag:-none}" # Publish gate: Debian, Ubuntu and Fedora lanes (see the script header). chmod +x scripts/local-ci/package-validate.sh bash scripts/local-ci/package-validate.sh --only linux "${args[@]}" - name: Upload validated release artifacts if: github.actor != 'nektos/act' uses: actions/upload-artifact@v4 with: name: release-${{ matrix.os }}-${{ matrix.suffix }}-files${{ github.run_attempt > 1 && format('-a{0}', github.run_attempt) || '' }} path: | yams-*.tar.gz yams-*.zip yams-*.msi yams-*.msi.sha256 ${{ env.BUILD_DIR }}/release_summary.md ${{ env.BUILD_DIR }}/bench_results/*.json yams-*.deb yams-*.rpm yams-*.pkg yams-*.pkg.tar.zst winget-manifests/** retention-days: 1 if-no-files-found: error create-release: name: Create GitHub Release needs: [resolve-release, check-commits, build-release] runs-on: ubuntu-latest concurrency: group: release-publish-${{ needs.resolve-release.outputs.channel }}-${{ needs.resolve-release.outputs.source_sha }} cancel-in-progress: false # Skip release creation when: running under act, OR check-commits skipped release if: | always() && !cancelled() && needs.resolve-release.result == 'success' && github.actor != 'nektos/act' && (needs.check-commits.result == 'skipped' || needs.check-commits.outputs.has_new_commits == 'true' || github.event_name == 'push' || github.event_name == 'workflow_dispatch') && needs.build-release.result == 'success' permissions: contents: write actions: read id-token: write attestations: write outputs: channel: ${{ steps.meta.outputs.channel }} version: ${{ steps.meta.outputs.version }} tag: ${{ steps.meta.outputs.tag }} steps: - name: Checkout immutable release source uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 with: fetch-depth: 0 submodules: false ref: ${{ needs.resolve-release.outputs.source_sha }} - name: Load resolved release metadata id: meta shell: bash run: | { echo "channel=${{ needs.resolve-release.outputs.channel }}" echo "version=${{ needs.resolve-release.outputs.version }}" echo "tag=${{ needs.resolve-release.outputs.tag }}" echo "release_name=${{ needs.resolve-release.outputs.release_name }}" echo "base_version=${{ needs.resolve-release.outputs.base_version }}" echo "prerelease=${{ needs.resolve-release.outputs.prerelease }}" } >> "$GITHUB_OUTPUT" - name: Download all artifacts uses: actions/download-artifact@v4 with: path: release-artifacts pattern: "*" - name: List downloaded artifacts (debug) shell: bash run: | echo "Downloaded artifact directories:" >&2 find release-artifacts -maxdepth 2 -type f -printf '%P\n' 2>/dev/null || true - name: Collect assets into staging directory shell: bash run: | set -euo pipefail shopt -s nullglob rm -rf assets || true mkdir -p assets copy_unique_asset() { local source="$1" local destination destination="assets/$(basename "$source")" if [ -e "$destination" ]; then echo "ERROR: duplicate release asset name: $(basename "$source")" >&2 exit 1 fi cp -v "$source" "$destination" } # Collect all per-platform release artifacts without silently overwriting names. for dir in release-artifacts/release-*; do [ -d "$dir" ] || continue # Archives for file in "$dir"/*.tar.gz "$dir"/*.zip; do [ -f "$file" ] && copy_unique_asset "$file" done # Packages for file in "$dir"/*.deb "$dir"/*.rpm "$dir"/*.pkg.tar.zst "$dir"/*.msi; do [ -f "$file" ] && copy_unique_asset "$file" done # Arch repo metadata if [ -d "$dir/archrepo" ]; then cp -rv "$dir/archrepo" assets/archrepo 2>/dev/null || true fi # Summaries & bench json if [ -d "$dir/bench_results" ]; then for file in "$dir"/bench_results/*.json; do cp -v "$file" assets/ 2>/dev/null || true done fi if [ -f "$dir/release_summary.md" ]; then cp -v "$dir/release_summary.md" "assets/release_summary-$(basename "$dir").md" 2>/dev/null || true fi done shopt -u nullglob echo "Final assets content:" >&2 ls -al assets >&2 || true - name: Generate checksums shell: bash run: | set -euo pipefail cd assets echo "Generating SHA256SUMS for release assets" >&2 # Exclude bench JSON and summaries from checksum manifest (only distributable binaries) distributables=() while IFS= read -r f; do distributables+=("$f") done < <(find . -maxdepth 1 -type f \( \ -name '*.tar.gz' -o -name '*.zip' -o -name '*.deb' -o -name '*.rpm' -o \ -name '*.pkg' -o -name '*.msi' -o -name '*.pkg.tar.zst' \ \) -printf '%f\n' | sort) if [ ${#distributables[@]} -gt 0 ]; then : > SHA256SUMS for f in "${distributables[@]}"; do sha256sum "$f" >> SHA256SUMS done sort -o SHA256SUMS SHA256SUMS echo "Created checksum manifest:" >&2 cat SHA256SUMS >&2 else echo "No distributable assets found; creating empty SHA256SUMS" >&2 touch SHA256SUMS fi - name: (Optional) Import channel GPG key and sign checksums env: STABLE_GPG_PRIVATE_KEY: ${{ secrets.GPG_PRIVATE_KEY || '' }} STABLE_GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE || '' }} EXPERIMENTAL_GPG_PRIVATE_KEY: ${{ secrets.EXPERIMENTAL_GPG_PRIVATE_KEY || secrets.GPG_PRIVATE_KEY || '' }} EXPERIMENTAL_GPG_PASSPHRASE: ${{ secrets.EXPERIMENTAL_GPG_PASSPHRASE || secrets.GPG_PASSPHRASE || '' }} shell: bash run: | set -euo pipefail if [ "${{ steps.meta.outputs.channel }}" = stable ]; then GPG_PRIVATE_KEY="$STABLE_GPG_PRIVATE_KEY" GPG_PASSPHRASE="$STABLE_GPG_PASSPHRASE" else GPG_PRIVATE_KEY="$EXPERIMENTAL_GPG_PRIVATE_KEY" GPG_PASSPHRASE="$EXPERIMENTAL_GPG_PASSPHRASE" fi unset STABLE_GPG_PRIVATE_KEY STABLE_GPG_PASSPHRASE EXPERIMENTAL_GPG_PRIVATE_KEY EXPERIMENTAL_GPG_PASSPHRASE if [ -z "$GPG_PRIVATE_KEY" ]; then echo "No channel signing key configured; leaving SHA256SUMS unsigned" >&2 exit 0 fi cd assets echo "GPG key present: generating detached signature for SHA256SUMS" >&2 echo "$GPG_PRIVATE_KEY" | base64 -d | gpg --batch --yes --pinentry-mode loopback --passphrase "$GPG_PASSPHRASE" --import gpg --batch --yes --pinentry-mode loopback --passphrase "$GPG_PASSPHRASE" --detach-sign --armor -o SHA256SUMS.asc SHA256SUMS echo "Signature generated (SHA256SUMS.asc)" >&2 - name: Verify asset set id: asset_check shell: bash run: | COUNT=$(find assets -type f | wc -l | tr -d ' ') echo "Discovered $COUNT asset files" >&2 echo "file_count=$COUNT" >> "$GITHUB_OUTPUT" if [ "$COUNT" -eq 0 ]; then echo "ERROR: No asset files assembled for release." >&2 exit 1 fi - name: Generate Homebrew formula (stable only) if: steps.meta.outputs.channel == 'stable' shell: bash run: | set -euo pipefail TAG="${{ steps.meta.outputs.tag }}" VERSION="${{ steps.meta.outputs.version }}" FORMULA_DIR="homebrew" mkdir -p "$FORMULA_DIR" # Locate macOS archives from local assets (supports legacy/new naming) ARM64_FILE="" X86_64_FILE="" for candidate in \ "assets/yams-${VERSION}-macos-arm64.zip" \ "assets/yams-v${VERSION}-macos-arm64.zip" \ "assets/yams-${TAG}-macos-arm64.zip" \ "assets/yams-yams-v${VERSION}-macos-arm64.zip"; do if [ -f "$candidate" ]; then ARM64_FILE="$candidate" break fi done if [ -z "$ARM64_FILE" ]; then shopt -s nullglob for candidate in assets/*-macos-arm64.zip; do ARM64_FILE="$candidate" break done shopt -u nullglob fi for candidate in \ "assets/yams-${VERSION}-macos-x86_64.zip" \ "assets/yams-v${VERSION}-macos-x86_64.zip" \ "assets/yams-${TAG}-macos-x86_64.zip" \ "assets/yams-yams-v${VERSION}-macos-x86_64.zip"; do if [ -f "$candidate" ]; then X86_64_FILE="$candidate" break fi done if [ -z "$X86_64_FILE" ]; then shopt -s nullglob for candidate in assets/*-macos-x86_64.zip; do X86_64_FILE="$candidate" break done shopt -u nullglob fi ASSET_ARM64="$(basename "${ARM64_FILE:-yams-${VERSION}-macos-arm64.zip}")" ASSET_X86_64="$(basename "${X86_64_FILE:-yams-${VERSION}-macos-x86_64.zip}")" if [ -f "$ARM64_FILE" ]; then SHA256_ARM64=$(shasum -a 256 "$ARM64_FILE" | awk '{print $1}') echo "Computed ARM64 sha256: $SHA256_ARM64" >&2 else echo "WARNING: ARM64 zip not found at $ARM64_FILE, formula may be incomplete" >&2 SHA256_ARM64="PLACEHOLDER_ARM64_SHA256" fi if [ -f "$X86_64_FILE" ]; then SHA256_X86_64=$(shasum -a 256 "$X86_64_FILE" | awk '{print $1}') echo "Computed x86_64 sha256: $SHA256_X86_64" >&2 else echo "WARNING: x86_64 zip not found at $X86_64_FILE, formula may be incomplete" >&2 SHA256_X86_64="PLACEHOLDER_X86_64_SHA256" fi validate_homebrew_archive() { local archive="$1" local label="$2" if [ ! -f "$archive" ]; then echo "Skipping ${label} Homebrew archive validation; file not found: $archive" >&2 return 0 fi python_script=$'import sys\nimport zipfile\n\narchive, label = sys.argv[1], sys.argv[2]\n\nwith zipfile.ZipFile(archive) as zf:\n names = zf.namelist()\n\ndef has_suffix(suffix: str) -> bool:\n suffix = suffix.rstrip("/")\n return any(name.rstrip("/").endswith(suffix) for name in names)\n\nhas_plugin_payload = any(\n "lib/yams/plugins/" in name and not name.endswith("/")\n for name in names\n)\n\nmissing = []\nif not has_suffix("bin/yams"):\n missing.append("bin/yams")\nif not has_suffix("lib/yams/plugins/libyams_onnx_plugin.dylib"):\n missing.append("lib/yams/plugins/libyams_onnx_plugin.dylib")\nif not has_plugin_payload:\n missing.append("lib/yams/plugins/*")\nif not has_suffix("share/yams/sql/reference_schema.sql"):\n missing.append("share/yams/sql/reference_schema.sql")\n\nif missing:\n joined = ", ".join(missing)\n raise SystemExit(f"{label} archive {archive} is missing required Homebrew paths: {joined}")\n\nprint(f"Validated {label} Homebrew archive layout: {archive}")' python3 -c "$python_script" "$archive" "$label" } validate_homebrew_archive "$ARM64_FILE" "arm64" validate_homebrew_archive "$X86_64_FILE" "x86_64" # Release assets are published later in this job; defer URL checks until post-publish. echo "Deferring Homebrew URL validation until after release publish" >&2 TEMPLATE_PATH="packaging/homebrew/yams.rb.template" if [ ! -f "$TEMPLATE_PATH" ]; then echo "ERROR: Homebrew template not found at $TEMPLATE_PATH" >&2 exit 1 fi OUT_FORMULA="$FORMULA_DIR/yams.rb" # Replace template placeholders with release metadata sed \ -e "s|__VERSION__|$VERSION|g" \ -e "s|__ASSET_ARM64__|$ASSET_ARM64|g" \ -e "s|__ASSET_X86_64__|$ASSET_X86_64|g" \ -e "s|__SHA256_ARM64__|$SHA256_ARM64|g" \ -e "s|__SHA256_X86_64__|$SHA256_X86_64|g" \ "$TEMPLATE_PATH" > "$OUT_FORMULA" echo "Updated formula at $OUT_FORMULA:" >&2 grep -E 'version\s+"|sha256 "' "$OUT_FORMULA" >&2 || true # Create quick install instructions file cat > "$FORMULA_DIR/README.txt" </homebrew-yams 2. Add this formula as Formula/yams.rb 3. Users run: brew install /yams/yams EOF - name: Update Homebrew tap submodule (stable only) if: steps.meta.outputs.channel == 'stable' env: HOMEBREW_TAP_TOKEN: ${{ secrets.HOMEBREW_TAP_TOKEN || '' }} shell: bash run: | set -euo pipefail VERSION="${{ steps.meta.outputs.version }}" TAP_PATH="homebrew-yams" # Check if generated formula exists first if [ ! -f "homebrew/yams.rb" ]; then echo "homebrew/yams.rb not found; skipping tap update" >&2 exit 0 fi # Initialize the homebrew-yams submodule (must happen before directory check) git submodule update --init homebrew-yams || { echo "Failed to initialize homebrew-yams submodule; skipping tap update" >&2 exit 0 } if [ ! -d "$TAP_PATH" ]; then echo "Tap submodule not found at $TAP_PATH after init; skipping tap update" >&2 exit 0 fi pushd "$TAP_PATH" >/dev/null # Fetch latest and reset to main BEFORE copying formula (avoids checkout conflicts) if [ -n "${HOMEBREW_TAP_TOKEN:-}" ]; then REMOTE="https://x-access-token:${HOMEBREW_TAP_TOKEN}@github.com/${{ github.repository_owner }}/homebrew-yams.git" git remote add tap-origin "$REMOTE" 2>/dev/null || git remote set-url tap-origin "$REMOTE" git fetch tap-origin main git checkout main || git checkout -b main tap-origin/main git reset --hard tap-origin/main else git checkout main || git checkout -b main git pull --ff-only 2>/dev/null || true fi popd >/dev/null # Now copy the formula after we're on the correct branch mkdir -p "$TAP_PATH/Formula" cp homebrew/yams.rb "$TAP_PATH/Formula/yams.rb" pushd "$TAP_PATH" >/dev/null git config user.name "github-actions[bot]" git config user.email "github-actions[bot]@users.noreply.github.com" if ! grep -Fq "version \"${VERSION}\"" Formula/yams.rb; then echo "ERROR: Generated stable formula does not contain expected version ${VERSION}" >&2 exit 1 fi if grep -Eq '__[A-Z0-9_]+__' Formula/yams.rb; then echo "ERROR: Generated stable formula still contains unresolved placeholders" >&2 exit 1 fi git add -A Formula/yams.rb if git diff --cached --quiet; then echo "Tap formula already at expected version ${VERSION}; skipping commit" >&2 else git commit -m "chore(homebrew): update formula to v${VERSION}" if [ -n "${HOMEBREW_TAP_TOKEN:-}" ]; then for attempt in 1 2 3; do if git push tap-origin main; then break fi if [ "$attempt" -eq 3 ]; then echo "Failed to push updated Homebrew formula after ${attempt} attempts" >&2 exit 1 fi sleep $((attempt * 10)) done else echo "HOMEBREW_TAP_TOKEN not set; manual push required" >&2 fi fi if [ -n "${HOMEBREW_TAP_TOKEN:-}" ]; then for attempt in 1 2 3 4 5; do remote_formula="$(curl -fsSL "https://raw.githubusercontent.com/${{ github.repository_owner }}/homebrew-yams/main/Formula/yams.rb" || true)" if [ -n "$remote_formula" ] && printf '%s\n' "$remote_formula" | grep -Fq "version \"${VERSION}\""; then echo "Confirmed remote stable formula is version ${VERSION}" >&2 break fi if [ "$attempt" -eq 5 ]; then echo "ERROR: Remote stable formula did not update to version ${VERSION}" >&2 exit 1 fi sleep $((attempt * 10)) done fi popd >/dev/null - name: Update Homebrew nightly formula (nightly/weekly only) if: steps.meta.outputs.channel != 'stable' env: HOMEBREW_TAP_TOKEN: ${{ secrets.HOMEBREW_TAP_TOKEN || '' }} shell: bash run: | set -euo pipefail VERSION="${{ steps.meta.outputs.version }}" TAG="${{ steps.meta.outputs.tag }}" TAP_PATH="homebrew-yams" # Extract date and hash from version (e.g., nightly-20251109-ebd0701) DATE=$(echo "$VERSION" | sed -E 's/.*-([0-9]{8})-.*/\1/') HASH=$(echo "$VERSION" | sed -E 's/.*-([a-f0-9]{7})$/\1/') # Initialize submodule before checking path (checkout has submodules disabled by default) git submodule update --init homebrew-yams if [ ! -d "$TAP_PATH" ]; then echo "Tap submodule not found at $TAP_PATH after init" >&2 exit 1 fi # Checkout main BEFORE modifying any files (avoids checkout conflicts) pushd "$TAP_PATH" >/dev/null if [ -n "${HOMEBREW_TAP_TOKEN:-}" ]; then REMOTE="https://x-access-token:${HOMEBREW_TAP_TOKEN}@github.com/${{ github.repository_owner }}/homebrew-yams.git" git remote add tap-origin "$REMOTE" 2>/dev/null || git remote set-url tap-origin "$REMOTE" git fetch tap-origin main 2>/dev/null || true git checkout main || git checkout -b main tap-origin/main 2>/dev/null || git checkout -b main git reset --hard tap-origin/main 2>/dev/null || true else git checkout main || git checkout -b main fi popd >/dev/null # Locate macOS archives from local assets (supports naming changes) ARM64_FILE="" X86_64_FILE="" for candidate in \ "assets/yams-${VERSION}-macos-arm64.zip" \ "assets/yams-${TAG}-macos-arm64.zip" \ "assets/yams-nightly-${DATE}-${HASH}-macos-arm64.zip"; do if [ -f "$candidate" ]; then ARM64_FILE="$candidate" break fi done if [ -z "$ARM64_FILE" ]; then shopt -s nullglob for candidate in assets/*-macos-arm64.zip; do ARM64_FILE="$candidate" break done shopt -u nullglob fi for candidate in \ "assets/yams-${VERSION}-macos-x86_64.zip" \ "assets/yams-${TAG}-macos-x86_64.zip" \ "assets/yams-nightly-${DATE}-${HASH}-macos-x86_64.zip"; do if [ -f "$candidate" ]; then X86_64_FILE="$candidate" break fi done if [ -z "$X86_64_FILE" ]; then shopt -s nullglob for candidate in assets/*-macos-x86_64.zip; do X86_64_FILE="$candidate" break done shopt -u nullglob fi ASSET_ARM64="$(basename "${ARM64_FILE:-yams-${VERSION}-macos-arm64.zip}")" ASSET_X86_64="$(basename "${X86_64_FILE:-yams-${VERSION}-macos-x86_64.zip}")" if [ -f "$ARM64_FILE" ]; then SHA256_ARM64=$(shasum -a 256 "$ARM64_FILE" | awk '{print $1}') echo "Computed ARM64 sha256: $SHA256_ARM64" >&2 else echo "WARNING: ARM64 zip not found at $ARM64_FILE, formula may be incomplete" >&2 SHA256_ARM64="PLACEHOLDER_ARM64_SHA256" fi if [ -f "$X86_64_FILE" ]; then SHA256_X86_64=$(shasum -a 256 "$X86_64_FILE" | awk '{print $1}') echo "Computed x86_64 sha256: $SHA256_X86_64" >&2 else echo "WARNING: x86_64 zip not found at $X86_64_FILE, formula may be incomplete" >&2 SHA256_X86_64="PLACEHOLDER_X86_64_SHA256" fi validate_homebrew_archive() { local archive="$1" local label="$2" if [ ! -f "$archive" ]; then echo "Skipping ${label} Homebrew archive validation; file not found: $archive" >&2 return 0 fi python_script=$'import sys\nimport zipfile\n\narchive, label = sys.argv[1], sys.argv[2]\n\nwith zipfile.ZipFile(archive) as zf:\n names = zf.namelist()\n\ndef has_suffix(suffix: str) -> bool:\n suffix = suffix.rstrip("/")\n return any(name.rstrip("/").endswith(suffix) for name in names)\n\nhas_plugin_payload = any(\n "lib/yams/plugins/" in name and not name.endswith("/")\n for name in names\n)\n\nmissing = []\nif not has_suffix("bin/yams"):\n missing.append("bin/yams")\nif not has_suffix("lib/yams/plugins/libyams_onnx_plugin.dylib"):\n missing.append("lib/yams/plugins/libyams_onnx_plugin.dylib")\nif not has_plugin_payload:\n missing.append("lib/yams/plugins/*")\nif not has_suffix("share/yams/sql/reference_schema.sql"):\n missing.append("share/yams/sql/reference_schema.sql")\n\nif missing:\n joined = ", ".join(missing)\n raise SystemExit(f"{label} archive {archive} is missing required Homebrew paths: {joined}")\n\nprint(f"Validated {label} Homebrew archive layout: {archive}")' python3 -c "$python_script" "$archive" "$label" } validate_homebrew_archive "$ARM64_FILE" "arm64" validate_homebrew_archive "$X86_64_FILE" "x86_64" # Release assets are published later in this job; defer URL checks until post-publish. echo "Deferring Homebrew URL validation until after release publish" >&2 # Update nightly formula (now safe to modify after checkout) TEMPLATE_PATH="packaging/homebrew/yams-nightly.rb.template" if [ ! -f "$TEMPLATE_PATH" ]; then echo "Nightly Homebrew template not found at $TEMPLATE_PATH" >&2 exit 1 fi NIGHTLY_FORMULA="$TAP_PATH/Formula/yams-nightly.rb" mkdir -p "$TAP_PATH/Formula" cp "$TEMPLATE_PATH" "$NIGHTLY_FORMULA" # Remove deprecated nightly formula names now that yams-nightly is canonical rm -f "$TAP_PATH/Formula/yams@nightly.rb" rm -f "$TAP_PATH/Aliases/yams@nightly" # Replace placeholders sed -i.bak \ -e "s/__VERSION__/${VERSION}/g" \ -e "s/__TAG__/${TAG}/g" \ -e "s/__DATE__/${DATE}/g" \ -e "s/__HASH__/${HASH}/g" \ -e "s/__ASSET_ARM64__/${ASSET_ARM64}/g" \ -e "s/__ASSET_X86_64__/${ASSET_X86_64}/g" \ -e "s/__SHA256_ARM64__/${SHA256_ARM64}/g" \ -e "s/__SHA256_X86_64__/${SHA256_X86_64}/g" \ "$NIGHTLY_FORMULA" rm -f "${NIGHTLY_FORMULA}.bak" pushd "$TAP_PATH" >/dev/null git config user.name "github-actions[bot]" git config user.email "github-actions[bot]@users.noreply.github.com" if ! grep -Fq "version \"${VERSION}\"" Formula/yams-nightly.rb; then echo "ERROR: Generated nightly formula does not contain expected version ${VERSION}" >&2 exit 1 fi if ! grep -Fq "/releases/download/${TAG}/" Formula/yams-nightly.rb; then echo "ERROR: Generated nightly formula does not reference expected tag ${TAG}" >&2 exit 1 fi if grep -Eq '__[A-Z0-9_]+__' Formula/yams-nightly.rb; then echo "ERROR: Generated nightly formula still contains unresolved placeholders" >&2 exit 1 fi git add -A Formula if [ -d Aliases ]; then git add -A Aliases fi if git diff --cached --quiet; then echo "Nightly formulas already at expected version ${VERSION}; skipping commit" >&2 else git commit -m "chore(homebrew): update nightly formula to ${VERSION}" if [ -n "${HOMEBREW_TAP_TOKEN:-}" ]; then REMOTE="https://x-access-token:${HOMEBREW_TAP_TOKEN}@github.com/${{ github.repository_owner }}/homebrew-yams.git" for attempt in 1 2 3; do if git push "$REMOTE" main; then break fi if [ "$attempt" -eq 3 ]; then echo "Failed to push updated nightly Homebrew formula after ${attempt} attempts" >&2 exit 1 fi sleep $((attempt * 10)) done else echo "HOMEBREW_TAP_TOKEN not set; manual push required" >&2 fi fi if [ -n "${HOMEBREW_TAP_TOKEN:-}" ]; then for attempt in 1 2 3 4 5; do remote_formula="$(curl -fsSL "https://raw.githubusercontent.com/${{ github.repository_owner }}/homebrew-yams/main/Formula/yams-nightly.rb" || true)" if [ -n "$remote_formula" ] && printf '%s\n' "$remote_formula" | grep -Fq "version \"${VERSION}\"" && printf '%s\n' "$remote_formula" | grep -Fq "/releases/download/${TAG}/"; then echo "Confirmed remote nightly formula references ${VERSION} (${TAG})" >&2 break fi if [ "$attempt" -eq 5 ]; then echo "ERROR: Remote nightly formula did not update to ${VERSION} (${TAG})" >&2 exit 1 fi sleep $((attempt * 10)) done fi popd >/dev/null - name: Build APT repository metadata (linux .deb present) env: STABLE_GPG_PRIVATE_KEY: ${{ secrets.GPG_PRIVATE_KEY || '' }} STABLE_GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE || '' }} EXPERIMENTAL_GPG_PRIVATE_KEY: ${{ secrets.EXPERIMENTAL_GPG_PRIVATE_KEY || secrets.GPG_PRIVATE_KEY || '' }} EXPERIMENTAL_GPG_PASSPHRASE: ${{ secrets.EXPERIMENTAL_GPG_PASSPHRASE || secrets.GPG_PASSPHRASE || '' }} REPOSITORY_CHANNEL: ${{ steps.meta.outputs.channel == 'stable' && 'stable' || 'experimental' }} REPOSITORY_VERSION: ${{ steps.meta.outputs.version }} shell: bash run: | set -euo pipefail if [ "$REPOSITORY_CHANNEL" = stable ]; then GPG_PRIVATE_KEY="$STABLE_GPG_PRIVATE_KEY" GPG_PASSPHRASE="$STABLE_GPG_PASSPHRASE" else GPG_PRIVATE_KEY="$EXPERIMENTAL_GPG_PRIVATE_KEY" GPG_PASSPHRASE="$EXPERIMENTAL_GPG_PASSPHRASE" fi unset STABLE_GPG_PRIVATE_KEY STABLE_GPG_PASSPHRASE EXPERIMENTAL_GPG_PRIVATE_KEY EXPERIMENTAL_GPG_PASSPHRASE # Require at least one .deb in assets if ! ls assets/*.deb >/dev/null 2>&1; then echo "No .deb found; skipping APT repo generation" >&2 exit 0 fi BASE_VERSION="${{ steps.meta.outputs.base_version }}" PACKAGE_FILENAME_VERSION="$BASE_VERSION" if [ "$REPOSITORY_CHANNEL" = experimental ]; then PACKAGE_FILENAME_VERSION="$REPOSITORY_VERSION" fi REPO_ROOT="aptrepo" POOL_DIR="$REPO_ROOT/pool/main/y/yams" DIST="$REPOSITORY_CHANNEL" COMPONENT=main mkdir -p "$POOL_DIR" # Copy and normalize Debian filenames by architecture. # Source naming is expected as yams--linux-x86_64.deb or yams--linux-aarch64.deb. seen_amd64=0 seen_arm64=0 for deb in assets/*.deb; do deb_name="$(basename "$deb")" case "$deb_name" in *-linux-x86_64.deb) target_name="yams_${PACKAGE_FILENAME_VERSION}_amd64.deb" seen_amd64=1 ;; *-linux-aarch64.deb|*-linux-arm64.deb) target_name="yams_${PACKAGE_FILENAME_VERSION}_arm64.deb" seen_arm64=1 ;; *) echo "Skipping unrecognized .deb naming pattern: $deb_name" >&2 continue ;; esac cp "$deb" "$POOL_DIR/$target_name" done ARCHES=() [ "$seen_amd64" -eq 1 ] && ARCHES+=(amd64) [ "$seen_arm64" -eq 1 ] && ARCHES+=(arm64) if [ "${#ARCHES[@]}" -eq 0 ]; then echo "No recognized .deb package architectures found in assets; skipping APT metadata generation" >&2 exit 0 fi # Install dpkg-dev if missing (for dpkg-scanpackages) if ! command -v dpkg-scanpackages >/dev/null 2>&1; then sudo apt-get update sudo apt-get install -y dpkg-dev fi pushd "$REPO_ROOT" >/dev/null mkdir -p "dists/$DIST/$COMPONENT" # Scan pool once, then split package stanzas per architecture. dpkg-scanpackages --multiversion pool > "dists/$DIST/$COMPONENT/Packages.all" for ARCH in "${ARCHES[@]}"; do mkdir -p "dists/$DIST/$COMPONENT/binary-$ARCH" awk -v arch="$ARCH" ' BEGIN { RS=""; ORS="\n\n" } $0 ~ ("(^|\\n)Architecture: " arch "(\\n|$)") { print $0 } ' "dists/$DIST/$COMPONENT/Packages.all" > "dists/$DIST/$COMPONENT/binary-$ARCH/Packages" gzip -c "dists/$DIST/$COMPONENT/binary-$ARCH/Packages" > "dists/$DIST/$COMPONENT/binary-$ARCH/Packages.gz" done rm -f "dists/$DIST/$COMPONENT/Packages.all" ARCH_LIST="${ARCHES[*]}" # Generate Release file with proper Date and SHA256 hash entries { echo "Origin: YAMS" echo "Label: YAMS" echo "Suite: $DIST" echo "Codename: $DIST" echo "Architectures: $ARCH_LIST" echo "Components: $COMPONENT" echo "Date: $(date -Ru)" echo "Description: YAMS APT repository" echo "SHA256:" for ARCH in "${ARCHES[@]}"; do for f in "$COMPONENT/binary-$ARCH/Packages" "$COMPONENT/binary-$ARCH/Packages.gz"; do if [ -f "dists/$DIST/$f" ]; then hash="$(sha256sum "dists/$DIST/$f" | awk '{print $1}')" size="$(stat --format='%s' "dists/$DIST/$f" 2>/dev/null || stat -f '%z' "dists/$DIST/$f")" printf " %s %16d %s\n" "$hash" "$size" "$f" fi done done } > "dists/$DIST/Release" if [ -z "${GPG_PRIVATE_KEY:-}" ]; then echo "ERROR: Missing channel-specific GPG_PRIVATE_KEY; refusing unsigned APT metadata" >&2 exit 1 fi GPG_ARGS=(--batch --yes --pinentry-mode loopback) if [ -n "${GPG_PASSPHRASE:-}" ]; then GPG_ARGS+=(--passphrase "$GPG_PASSPHRASE") fi echo "$GPG_PRIVATE_KEY" | base64 -d | gpg "${GPG_ARGS[@]}" --import gpg "${GPG_ARGS[@]}" -abs -o "dists/$DIST/Release.gpg" "dists/$DIST/Release" gpg "${GPG_ARGS[@]}" --clearsign -o "dists/$DIST/InRelease" "dists/$DIST/Release" popd >/dev/null tree "$REPO_ROOT" || find "$REPO_ROOT" -maxdepth 4 -type f -print if [ "$REPOSITORY_CHANNEL" = stable ]; then repository_url="https://repo.yamsmemory.ai/aptrepo" public_key_url="https://repo.yamsmemory.ai/gpg.key" else repository_url="https://repo.yamsmemory.ai/experimental/aptrepo" public_key_url="https://repo.yamsmemory.ai/experimental/aptrepo/gpg.key" fi printf '%s\n' \ 'Add this APT repo (signed):' \ " curl -fsSL $public_key_url | sudo gpg --dearmor -o /usr/share/keyrings/yams-$REPOSITORY_CHANNEL.gpg" \ " echo \"deb [arch=amd64,arm64 signed-by=/usr/share/keyrings/yams-$REPOSITORY_CHANNEL.gpg] $repository_url $REPOSITORY_CHANNEL main\" | sudo tee /etc/apt/sources.list.d/yams-$REPOSITORY_CHANNEL.list" \ ' sudo apt-get update && sudo apt-get install yams' \ > "$REPO_ROOT/USAGE.txt" - name: Upload APT repo artifact if: always() uses: actions/upload-artifact@v4 with: name: apt-repo path: aptrepo/ retention-days: 7 - name: Build YUM repository metadata (rpm present) env: STABLE_GPG_PRIVATE_KEY: ${{ secrets.GPG_PRIVATE_KEY || '' }} STABLE_GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE || '' }} EXPERIMENTAL_GPG_PRIVATE_KEY: ${{ secrets.EXPERIMENTAL_GPG_PRIVATE_KEY || secrets.GPG_PRIVATE_KEY || '' }} EXPERIMENTAL_GPG_PASSPHRASE: ${{ secrets.EXPERIMENTAL_GPG_PASSPHRASE || secrets.GPG_PASSPHRASE || '' }} REPOSITORY_PREFIX: ${{ steps.meta.outputs.channel != 'stable' && 'experimental/' || '' }} shell: bash run: | set -euo pipefail if [ "${{ steps.meta.outputs.channel }}" = stable ]; then GPG_PRIVATE_KEY="$STABLE_GPG_PRIVATE_KEY" GPG_PASSPHRASE="$STABLE_GPG_PASSPHRASE" else GPG_PRIVATE_KEY="$EXPERIMENTAL_GPG_PRIVATE_KEY" GPG_PASSPHRASE="$EXPERIMENTAL_GPG_PASSPHRASE" fi unset STABLE_GPG_PRIVATE_KEY STABLE_GPG_PASSPHRASE EXPERIMENTAL_GPG_PRIVATE_KEY EXPERIMENTAL_GPG_PASSPHRASE if ! ls assets/*.rpm >/dev/null 2>&1; then echo "No .rpm found; skipping YUM repo generation" >&2 exit 0 fi YUM_ROOT="yumrepo" mkdir -p "$YUM_ROOT" cp assets/*.rpm "$YUM_ROOT/" 2>/dev/null || true # Install createrepo_c if needed if ! command -v createrepo_c >/dev/null 2>&1; then sudo apt-get update sudo apt-get install -y createrepo-c fi createrepo_c "$YUM_ROOT" if [ -n "${GPG_PRIVATE_KEY:-}" ]; then GPG_ARGS=(--batch --yes) if [ -n "${GPG_PASSPHRASE:-}" ]; then GPG_ARGS+=(--pinentry-mode loopback --passphrase "$GPG_PASSPHRASE") fi echo "$GPG_PRIVATE_KEY" | base64 -d | gpg "${GPG_ARGS[@]}" --import gpg "${GPG_ARGS[@]}" --detach-sign --armor -o "$YUM_ROOT/repodata/repomd.xml.asc" "$YUM_ROOT/repodata/repomd.xml" fi # Repository metadata is signed when a channel key is configured; the RPMs are not. if [ -z "$REPOSITORY_PREFIX" ]; then public_key_url="https://repo.yamsmemory.ai/gpg.key" else public_key_url="https://repo.yamsmemory.ai/experimental/aptrepo/gpg.key" fi if [ -f "$YUM_ROOT/repodata/repomd.xml.asc" ]; then usage_title='Add YUM/DNF repo (signed repository metadata; RPMs unsigned):' repo_gpg_lines=('repo_gpgcheck=1' "gpgkey=$public_key_url") else usage_title='Add YUM/DNF repo (unsigned):' repo_gpg_lines=('repo_gpgcheck=0') fi printf '%s\n' \ "$usage_title" \ '' \ "sudo tee /etc/yum.repos.d/yams.repo <<'REPO'" \ '[yams]' \ 'name=YAMS Repository' \ "baseurl=https://repo.yamsmemory.ai/${REPOSITORY_PREFIX}yumrepo/" \ 'enabled=1' \ 'gpgcheck=0' \ "${repo_gpg_lines[@]}" \ 'REPO' \ '' \ 'Then install:' \ '' \ 'sudo dnf makecache && sudo dnf install yams' \ > "$YUM_ROOT/USAGE.txt" - name: Upload YUM repo artifact if: always() uses: actions/upload-artifact@v4 with: name: yum-repo path: yumrepo/ retention-days: 7 - name: Build Arch repository metadata (arch packages present) env: REPOSITORY_PREFIX: ${{ steps.meta.outputs.channel != 'stable' && 'experimental/' || '' }} shell: bash run: | set -euo pipefail shopt -s nullglob arch_packages=(assets/*.pkg.tar.zst) shopt -u nullglob if [ ${#arch_packages[@]} -eq 0 ]; then echo "No .pkg.tar.zst found; skipping Arch repo generation" >&2 exit 0 fi ARCH_ROOT="archrepo" mkdir -p "${ARCH_ROOT}/os/x86_64" "${ARCH_ROOT}/os/aarch64" for pkg in "${arch_packages[@]}"; do pkg_name="$(basename "$pkg")" case "$pkg_name" in *-x86_64*) cp "$pkg" "${ARCH_ROOT}/os/x86_64/" ;; *-aarch64*) cp "$pkg" "${ARCH_ROOT}/os/aarch64/" ;; *) cp "$pkg" "${ARCH_ROOT}/" ;; esac done if ! command -v repo-add >/dev/null 2>&1; then echo "repo-add not available; installing pacman tools" >&2 sudo apt-get update -qq sudo apt-get install -y -qq libarchive-tools zstd pacman-package-manager fi if ! command -v repo-add >/dev/null 2>&1; then echo "ERROR: repo-add unavailable; cannot build Arch repository database" >&2 exit 1 fi for arch_dir in "${ARCH_ROOT}/os/"*; do [ -d "$arch_dir" ] || continue if find "$arch_dir" -maxdepth 1 -type f -name '*.pkg.tar.zst' -print -quit | grep -q .; then ( cd "$arch_dir" rm -f yams.db yams.db.tar.gz yams.files yams.files.tar.gz repo-add -q yams.db.tar.gz ./*.pkg.tar.zst ln -sf yams.db.tar.gz yams.db [ -f yams.files.tar.gz ] && ln -sf yams.files.tar.gz yams.files ) fi done printf '%s\n' \ 'Add Arch repo (unsigned):' \ '' \ "sudo tee -a /etc/pacman.conf <<'REPO'" \ '' \ '[yams]' \ 'SigLevel = Optional TrustAll' \ "Server = https://repo.yamsmemory.ai/${REPOSITORY_PREFIX}archrepo/os/\$arch" \ 'REPO' \ '' \ 'Then install:' \ '' \ 'sudo pacman -Sy yams' \ > "${ARCH_ROOT}/USAGE.txt" tree "${ARCH_ROOT}" 2>/dev/null || find "${ARCH_ROOT}" -maxdepth 3 -type f -print - name: Upload Arch repo artifact if: always() uses: actions/upload-artifact@v4 with: name: arch-repo path: archrepo/ retention-days: 7 - name: Generate and validate release manifest (latest.json) shell: bash run: | set -euo pipefail published_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)" release_run_url="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }}" python3 scripts/ci/generate_release_manifest.py \ --assets-dir assets \ --checksums assets/SHA256SUMS \ --output assets/latest.json \ --channel "${{ steps.meta.outputs.channel }}" \ --version "${{ steps.meta.outputs.version }}" \ --tag "${{ steps.meta.outputs.tag }}" \ --source-sha "${{ needs.resolve-release.outputs.source_sha }}" \ --source-ref "${{ needs.resolve-release.outputs.source_ref }}" \ --repository "${{ github.repository }}" \ --github-server-url "${{ github.server_url }}" \ --release-run-id "${{ github.run_id }}" \ --release-run-attempt "${{ github.run_attempt }}" \ --release-run-url "$release_run_url" \ --tests-run-id "${{ needs.resolve-release.outputs.tests_run_id }}" \ --tests-run-attempt "${{ needs.resolve-release.outputs.tests_run_attempt }}" \ --tests-run-url "${{ needs.resolve-release.outputs.tests_run_url }}" \ --ipc-protocol-header include/yams/daemon/ipc/ipc_protocol_envelope.h \ --p2p-protocol-header include/yams/daemon/p2p/p2p_protocol.h \ --published-at "$published_at" python3 -m json.tool assets/latest.json >/dev/null echo "latest.json manifest (assets/latest.json):" >&2 sed 's/^/ /' assets/latest.json >&2 - name: Upload trusted release manifest artifact uses: actions/upload-artifact@v4 with: name: release-manifest path: assets/latest.json if-no-files-found: error retention-days: 30 - name: Create exact-source provenance predicate if: github.actor != 'nektos/act' env: SOURCE_SHA: ${{ needs.resolve-release.outputs.source_sha }} SOURCE_REF: ${{ needs.resolve-release.outputs.source_ref }} TESTS_RUN_ID: ${{ needs.resolve-release.outputs.tests_run_id }} TESTS_RUN_ATTEMPT: ${{ needs.resolve-release.outputs.tests_run_attempt }} TESTS_RUN_URL: ${{ needs.resolve-release.outputs.tests_run_url }} shell: bash run: | jq -n \ --arg source_sha "$SOURCE_SHA" \ --arg source_ref "$SOURCE_REF" \ --arg release_run_id "$GITHUB_RUN_ID" \ --arg release_run_attempt "$GITHUB_RUN_ATTEMPT" \ --arg tests_run_id "$TESTS_RUN_ID" \ --arg tests_run_attempt "$TESTS_RUN_ATTEMPT" \ --arg tests_run_url "$TESTS_RUN_URL" \ '{source_sha: $source_sha, source_ref: $source_ref, release_workflow: {run_id: $release_run_id, attempt: $release_run_attempt}, tests_workflow: {run_id: $tests_run_id, attempt: $tests_run_attempt, url: $tests_run_url}}' \ > release-provenance.json - name: Attest validated release distributables to exact source if: github.actor != 'nektos/act' uses: actions/attest@11bbd243972067817e9ed160cb123cab3601f436 # v2 with: subject-checksums: assets/SHA256SUMS predicate-type: https://yamsmemory.ai/attestations/release-source/v1 predicate-path: release-provenance.json - name: Check existing release (stable only) id: existing_release shell: bash run: | TAG="${{ steps.meta.outputs.tag }}" echo "Checking if release $TAG already exists" >&2 if command -v gh >/dev/null 2>&1; then if release_json="$(gh release view "$TAG" --json isDraft,url 2>/dev/null)"; then { echo "exists=true" draft="$(printf '%s' "$release_json" | sed -n 's/.*"isDraft":[[:space:]]*\(true\|false\).*/\1/p')" url="$(printf '%s' "$release_json" | sed -n 's/.*"url":[[:space:]]*"\([^"]*\)".*/\1/p')" echo "draft=${draft:-unknown}" echo "url=${url:-}" } >> "$GITHUB_OUTPUT" else { echo "exists=false" echo "draft=false" echo "url=" } >> "$GITHUB_OUTPUT" fi else echo "gh CLI not installed; skipping existence check" >&2 { echo "exists=unknown" echo "draft=unknown" echo "url=" } >> "$GITHUB_OUTPUT" fi env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Refuse immutable release asset overwrite if: ${{ github.actor != 'nektos/act' && steps.existing_release.outputs.exists == 'true' && steps.existing_release.outputs.draft != 'true' && steps.asset_check.outputs.file_count != '0' }} shell: bash run: | cat >&2 <<'EOF' ERROR: The GitHub release for this tag is already published. This repository uses immutable releases, so published release assets cannot be replaced. The release upload action overwrites by deleting matching assets first, which GitHub rejects for immutable releases. Create a new patch release from the fixed commit, or delete/recreate the release and tag only if the published release is intentionally being withdrawn. EOF if [ -n "${{ steps.existing_release.outputs.url }}" ]; then echo "Existing release: ${{ steps.existing_release.outputs.url }}" >&2 fi exit 1 - name: Check job completion status id: check-status shell: bash run: | # Discover expected platforms from matrix job names in this run JOBS_JSON=$(curl -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ -H "Accept: application/vnd.github.v3+json" \ -s "https://api.github.com/repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/jobs?per_page=100") # Extract unique platform identifiers from "Build and Release ()" job names EXPECTED_PLATFORMS=$(echo "$JOBS_JSON" | sed -n 's/.*"name"[[:space:]]*:[[:space:]]*"\(Build and Release ([^\"]*)\)".*/\1/p' | sed -n 's/Build and Release (\(.*\))/\1/p' | sort -u) TOTAL_EXPECTED=$(echo "$EXPECTED_PLATFORMS" | grep -c . || true) # Count successful artifacts from downloaded files (release-*) if [ -d "release-artifacts" ]; then # Count per-platform artifact directories (robust against file/dir listing quirks) SUCCESS_COUNT=$(find "release-artifacts" -mindepth 1 -maxdepth 1 -type d -name 'release-*' 2>/dev/null | grep -c . || true) else SUCCESS_COUNT=0 fi # Determine expected builds: # 1) Prefer matrix.json (if provided) # 2) Otherwise, fall back to number of artifact groups downloaded MATRIX_JSON="" if [ -f "release-artifacts/matrix-meta/matrix.json" ]; then MATRIX_JSON="release-artifacts/matrix-meta/matrix.json" elif [ -f "release-artifacts/matrix.json" ]; then MATRIX_JSON="release-artifacts/matrix.json" fi if [ -n "$MATRIX_JSON" ]; then TOTAL_EXPECTED=$(tr -d '\r' < "$MATRIX_JSON" | grep -Eo '"total"[[:space:]]*:[[:space:]]*[0-9]+' | grep -Eo '[0-9]+' | head -1) fi if [ -z "$TOTAL_EXPECTED" ] || [ "$TOTAL_EXPECTED" -eq 0 ]; then TOTAL_EXPECTED="$SUCCESS_COUNT" fi # Compute dynamic threshold = ceil(TOTAL_EXPECTED / 2) THRESHOLD=$(( (TOTAL_EXPECTED + 1) / 2 )) echo "Successful builds: $SUCCESS_COUNT out of $TOTAL_EXPECTED (threshold: $THRESHOLD)" if [ "$SUCCESS_COUNT" -lt "$THRESHOLD" ]; then echo "ERROR: Need at least $THRESHOLD successful builds to create release (out of $TOTAL_EXPECTED expected)" exit 1 fi { echo "success_count=$SUCCESS_COUNT" echo "total_expected=$TOTAL_EXPECTED" echo "threshold=$THRESHOLD" } >> "$GITHUB_OUTPUT" - name: Extract release notes from CHANGELOG.md (stable only) id: changelog if: steps.meta.outputs.channel == 'stable' uses: ffurrer2/extract-release-notes@v2 with: changelog_file: CHANGELOG.md release_notes_file: RELEASE_NOTES_EXTRACTED.md - name: Prepare release body id: release-body shell: bash env: CHANNEL: ${{ steps.meta.outputs.channel }} RELEASE_NAME: ${{ steps.meta.outputs.release_name }} REPO_URL: ${{ github.server_url }}/${{ github.repository }} TAG: ${{ steps.meta.outputs.tag }} run: | set -euo pipefail BODY_FILE="RELEASE_BODY.md" if [ "$CHANNEL" = "stable" ]; then if [ ! -s "RELEASE_NOTES_EXTRACTED.md" ]; then echo "Expected extracted stable release notes in RELEASE_NOTES_EXTRACTED.md" >&2 exit 1 fi { echo "## Release Notes" echo cat RELEASE_NOTES_EXTRACTED.md echo echo "Full changelog: [CHANGELOG.md]($REPO_URL/blob/$TAG/CHANGELOG.md)" } > "$BODY_FILE" else FULL_SHA="$(git rev-parse HEAD)" SHORT_SHA="$(git rev-parse --short HEAD)" { echo "## ${RELEASE_NAME}" echo printf -- "- Channel: \`%s\`\n" "$CHANNEL" printf -- "- Commit: [\`%s\`](%s/commit/%s)\n" "$SHORT_SHA" "$REPO_URL" "$FULL_SHA" echo "- Changelog: [CHANGELOG.md]($REPO_URL/blob/main/CHANGELOG.md)" } > "$BODY_FILE" fi echo "body_file=$BODY_FILE" >> "$GITHUB_OUTPUT" - name: Prepare release assets shell: bash run: | mkdir -p assets shopt -s nullglob # Collect all release files for dir in release-artifacts/release-*; do if [ -d "$dir" ]; then # Move main archive for file in "$dir"/*.tar.gz "$dir"/*.zip; do mv "$file" assets/ 2>/dev/null || true done # Move Linux packages for file in "$dir"/*.deb "$dir"/*.rpm "$dir"/*.pkg; do mv "$file" assets/ 2>/dev/null || true done # Collect benchmark results for file in "$dir"/bench_results/*.json; do cp "$file" assets/ 2>/dev/null || true done fi done shopt -u nullglob echo "Collected assets:" ls -la assets/ - name: "Dry-run: print release notes (act only)" if: ${{ github.actor == 'nektos/act' }} shell: bash run: | echo "--- RELEASE_BODY.md (dry-run) ---" cat "${{ steps.release-body.outputs.body_file }}" - name: Create or update GitHub Release (draft, upload assets first) if: ${{ github.actor != 'nektos/act' && steps.asset_check.outputs.file_count != '0' && (steps.existing_release.outputs.exists != 'true' || steps.existing_release.outputs.draft == 'true') }} uses: softprops/action-gh-release@v2 with: tag_name: ${{ steps.meta.outputs.tag }} name: ${{ steps.meta.outputs.release_name }} body_path: ${{ steps.release-body.outputs.body_file }} files: assets/* draft: true prerelease: ${{ steps.meta.outputs.prerelease }} target_commitish: ${{ needs.resolve-release.outputs.source_sha }} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Publish GitHub Release (flip draft=false) if: ${{ github.actor != 'nektos/act' && steps.asset_check.outputs.file_count != '0' }} shell: bash env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | if ! command -v gh >/dev/null 2>&1; then echo "Installing gh CLI to publish release" >&2 type -p curl >/dev/null || sudo apt-get update sudo apt-get install -y gh >/dev/null 2>&1 || true fi TAG="${{ steps.meta.outputs.tag }}" # publish the release after assets are uploaded gh release edit "$TAG" --draft=false --title "${{ steps.meta.outputs.release_name }}" --notes-file "${{ steps.release-body.outputs.body_file }}" \ ${{ steps.meta.outputs.prerelease == 'true' && '--prerelease' || '--latest' }} - name: Validate Homebrew asset URLs (post-publish) if: ${{ github.actor != 'nektos/act' && steps.asset_check.outputs.file_count != '0' }} shell: bash run: | set -euo pipefail TAG="${{ steps.meta.outputs.tag }}" REPO="${{ github.repository }}" shopt -s nullglob arm64_candidates=(assets/*-macos-arm64.zip) x86_64_candidates=(assets/*-macos-x86_64.zip) if [ ${#arm64_candidates[@]} -eq 0 ]; then echo "ERROR: Missing macOS arm64 archive in assets/ for Homebrew validation" >&2 exit 1 fi if [ ${#x86_64_candidates[@]} -eq 0 ]; then echo "ERROR: Missing macOS x86_64 archive in assets/ for Homebrew validation" >&2 exit 1 fi verify_asset_url() { local asset_name="$1" local url="https://github.com/${REPO}/releases/download/${TAG}/${asset_name}" local attempts=12 for attempt in $(seq 1 "$attempts"); do if curl -fsIL "$url" >/dev/null; then echo "Verified Homebrew asset URL: $url" return 0 fi if [ "$attempt" -eq "$attempts" ]; then echo "ERROR: Homebrew asset URL not reachable after publish: $url" >&2 return 1 fi echo "Homebrew asset URL not reachable yet (attempt ${attempt}/${attempts}); waiting 10s: $url" >&2 sleep 10 done } verify_asset_url "$(basename "${arm64_candidates[0]}")" verify_asset_url "$(basename "${x86_64_candidates[0]}")" - name: Publish channel package repositories to R2 if: ${{ github.actor != 'nektos/act' }} env: STABLE_CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN || secrets.R2_SECRET_ACCESS_KEY }} STABLE_CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} EXPERIMENTAL_CLOUDFLARE_API_TOKEN: ${{ secrets.EXPERIMENTAL_CLOUDFLARE_API_TOKEN || secrets.CLOUDFLARE_API_TOKEN || secrets.R2_SECRET_ACCESS_KEY }} EXPERIMENTAL_CLOUDFLARE_ACCOUNT_ID: ${{ secrets.EXPERIMENTAL_CLOUDFLARE_ACCOUNT_ID || secrets.CLOUDFLARE_ACCOUNT_ID }} STABLE_GPG_PUBLIC_KEY_B64: ${{ secrets.GPG_PUBLIC_KEY_B64 || '' }} EXPERIMENTAL_GPG_PUBLIC_KEY_B64: ${{ secrets.EXPERIMENTAL_GPG_PUBLIC_KEY_B64 || secrets.GPG_PUBLIC_KEY_B64 || '' }} shell: bash run: | set -euo pipefail if [ "${{ steps.meta.outputs.channel }}" = stable ]; then PUBLICATION_CHANNEL=stable CLOUDFLARE_API_TOKEN="$STABLE_CLOUDFLARE_API_TOKEN" CLOUDFLARE_ACCOUNT_ID="$STABLE_CLOUDFLARE_ACCOUNT_ID" GPG_PUBLIC_KEY_B64="$STABLE_GPG_PUBLIC_KEY_B64" else PUBLICATION_CHANNEL=experimental CLOUDFLARE_API_TOKEN="$EXPERIMENTAL_CLOUDFLARE_API_TOKEN" CLOUDFLARE_ACCOUNT_ID="$EXPERIMENTAL_CLOUDFLARE_ACCOUNT_ID" GPG_PUBLIC_KEY_B64="$EXPERIMENTAL_GPG_PUBLIC_KEY_B64" fi export CLOUDFLARE_API_TOKEN CLOUDFLARE_ACCOUNT_ID unset STABLE_CLOUDFLARE_API_TOKEN STABLE_CLOUDFLARE_ACCOUNT_ID unset EXPERIMENTAL_CLOUDFLARE_API_TOKEN EXPERIMENTAL_CLOUDFLARE_ACCOUNT_ID unset STABLE_GPG_PUBLIC_KEY_B64 EXPERIMENTAL_GPG_PUBLIC_KEY_B64 if [ -z "$CLOUDFLARE_API_TOKEN" ] || [ -z "$CLOUDFLARE_ACCOUNT_ID" ]; then echo "Missing channel-specific Cloudflare R2 authority" >&2 exit 1 fi WRANGLER_ROOT="${RUNNER_TEMP:-/tmp}/wrangler-cli" WRANGLER_BIN="$WRANGLER_ROOT/node_modules/.bin/wrangler" if [ ! -x "$WRANGLER_BIN" ]; then npm install --prefix "$WRANGLER_ROOT" wrangler@4.69.0 >/dev/null fi "$WRANGLER_BIN" --version >&2 public_key_args=() if [ -n "$GPG_PUBLIC_KEY_B64" ]; then echo "$GPG_PUBLIC_KEY_B64" | base64 -d > channel-gpg.key public_key_args=(--public-key channel-gpg.key) fi python3 scripts/ci/publish_repository.py \ --repo-root . \ --manifest assets/latest.json \ --expected-channel "$PUBLICATION_CHANNEL" \ --selected-run-id "${{ github.run_id }}" \ --selected-run-attempt "${{ github.run_attempt }}" \ --bucket yams-repository \ --wrangler "$WRANGLER_BIN" \ "${public_key_args[@]}" echo "R2 $PUBLICATION_CHANNEL publication complete; no objects were pruned." >&2 - name: List release assets (post-upload) if: ${{ github.actor != 'nektos/act' }} shell: bash run: | if ! command -v gh >/dev/null 2>&1; then echo "Installing gh CLI for asset listing" >&2 type -p curl >/dev/null || sudo apt-get update sudo apt-get install -y gh >/dev/null 2>&1 || true fi TAG="${{ steps.meta.outputs.tag }}" echo "Release assets for $TAG:" >&2 gh release view "$TAG" --json assets --jq '.assets[].name' || echo "(Could not list assets)" env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}