yams/scripts/local-ci/smolvm-lane.sh

#!/usr/bin/env bash
# yams/scripts/local-ci/smolvm-lane.sh
#
# smolvm-backed local validation lanes.
#
# Static profiles are fast script/workflow sanity checks. linux-ci is the
# pre-push Linux build+test gate: it runs the repo's CI-style Debug build and
# unit+integration Meson suites inside a clean Linux microVM.
#
# Examples:
#   bash scripts/local-ci/smolvm-lane.sh                         # static profile
#   bash scripts/local-ci/smolvm-lane.sh --profile linux-ci      # Linux build+tests
#   bash scripts/local-ci/smolvm-lane.sh --profile smoke         # mount/boot only
#   bash scripts/local-ci/smolvm-lane.sh --dry-run

set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"
LOG_ROOT_DEFAULT="${REPO_ROOT}/build/local-ci/smolvm-lane"

PROFILE="static"
LOG_ROOT="${LOG_ROOT_DEFAULT}"
DRY_RUN=0
SMOLVM_BIN="${SMOLVM_BIN:-}"
IMAGE="${SMOLVM_IMAGE:-}"
ACTIONLINT_VERSION="${ACTIONLINT_VERSION:-v1.7.7}"
RUN_TAG="$(date -u +%Y%m%dT%H%M%SZ)-$$"

usage() {
	cat <<'USAGE'
Usage: smolvm-lane.sh [options]

Profiles:
  smoke     Boot Linux under smolvm and verify the repo mount.
  syntax    smoke + bash -n for local CI/package scripts.
  static    syntax + shellcheck and actionlint inside the VM (default).
  linux-ci  CI-style Linux Debug build + unit/integration tests inside smolvm.

Options:
  --profile NAME   smoke|syntax|static|linux-ci (default: static)
  --image IMAGE    smolvm image to run (default: alpine; linux-ci: ubuntu:24.04)
  --log-root DIR   Log directory (default: build/local-ci/smolvm-lane)
  --dry-run        Print the smolvm command without executing it
  -h, --help       Show this help

Environment for linux-ci:
  YAMS_SMOLVM_BUILD_DIR       Build dir (default: build/smolvm-linux)
  YAMS_SMOLVM_CLEAN_BUILD     Clean build dir before linux-ci (default: 1)
  YAMS_SMOLVM_CONAN_JOBS      Conan/dependency build jobs (default: 1)
  YAMS_SMOLVM_NINJA_JOBS      Meson/Ninja compile jobs (default: 1)
  YAMS_SMOLVM_COMPILE_TARGETS Optional space-separated Meson compile targets
  YAMS_SMOLVM_TEST_JOBS       Meson test jobs when YAMS_SMOLVM_TEST_ARGS unset (default: 1)
  YAMS_SMOLVM_TEST_ARGS       Meson test args (default: --suite unit --suite integration ...)
  YAMS_SMOLVM_SWAP_SIZE       Guest swap file size for link-heavy Debug builds (default: 2G, 0 disables)
  YAMS_SMOLVM_USE_LLD         Add clang lld link args for lower link memory (default: 1)
USAGE
}

log() { printf '\033[1;34m[smolvm-lane]\033[0m %s\n' "$*"; }
ok() { printf '\033[1;32m[ ok ]\033[0m %s\n' "$*"; }
fail() { printf '\033[1;31m[fail]\033[0m %s\n' "$*" >&2; }

while [ "$#" -gt 0 ]; do
	case "$1" in
	--profile)
		PROFILE="${2:-}"
		shift 2
		;;
	--image)
		IMAGE="${2:-}"
		shift 2
		;;
	--log-root)
		LOG_ROOT="${2:-}"
		shift 2
		;;
	--dry-run)
		DRY_RUN=1
		shift
		;;
	-h | --help)
		usage
		exit 0
		;;
	*)
		fail "unknown argument: $1"
		usage
		exit 2
		;;
	esac
done

case "${PROFILE}" in
smoke | syntax | static | linux-ci) ;;
*)
	fail "invalid profile: ${PROFILE}"
	echo "valid profiles: smoke, syntax, static, linux-ci" >&2
	exit 2
	;;
esac

if [ -z "${IMAGE}" ]; then
	case "${PROFILE}" in
	linux-ci) IMAGE="ubuntu:24.04" ;;
	*) IMAGE="alpine" ;;
	esac
fi

find_smolvm() {
	if [ -n "${SMOLVM_BIN}" ]; then
		[ -x "${SMOLVM_BIN}" ] || command -v "${SMOLVM_BIN}" >/dev/null 2>&1 || return 1
		return 0
	fi
	if command -v smolvm >/dev/null 2>&1; then
		SMOLVM_BIN="smolvm"
		return 0
	fi
	if [ -x "${HOME}/.local/bin/smolvm" ]; then
		SMOLVM_BIN="${HOME}/.local/bin/smolvm"
		return 0
	fi
	return 1
}

if ! find_smolvm; then
	fail "smolvm not found on PATH (or at ~/.local/bin/smolvm). Set SMOLVM_BIN."
	exit 2
fi

mkdir -p "${LOG_ROOT}"
SUMMARY="${LOG_ROOT}/summary-${RUN_TAG}.md"
LOG_FILE="${LOG_ROOT}/${RUN_TAG}-${PROFILE}.log"

static_guest_script() {
	cat <<'GUEST_SCRIPT'
set -eu
printf '[guest] pwd=%s\n' "$(pwd)"
printf '[guest] uname=%s\n' "$(uname -m)"
test -f meson.build
test -d scripts/local-ci

case "${YAMS_SMOLVM_PROFILE}" in
smoke)
	echo '[guest] smoke profile complete'
	;;
syntax | static)
	apk add --no-cache bash >/dev/null
	bash -n \
		scripts/local-ci/push-lane.sh \
		scripts/local-ci/package-lane.sh \
		scripts/local-ci/package-validate.sh \
		scripts/local-ci/pre-push-ci-gate.sh \
		scripts/local-ci/native-macos-lane.sh \
		scripts/local-ci/smolvm-lane.sh \
		scripts/build-deb.sh \
		scripts/build-arch-pkg.sh
	if [ "${YAMS_SMOLVM_PROFILE}" = "syntax" ]; then
		echo '[guest] syntax profile complete'
		exit 0
	fi

	apk add --no-cache curl tar shellcheck >/dev/null
	guest_arch="$(uname -m)"
	case "${guest_arch}" in
	x86_64) actionlint_arch="amd64" ;;
	aarch64 | arm64) actionlint_arch="arm64" ;;
	*) echo "unsupported actionlint arch: ${guest_arch}" >&2; exit 2 ;;
	esac
	actionlint_url="https://github.com/rhysd/actionlint/releases/download/${ACTIONLINT_VERSION}/actionlint_${ACTIONLINT_VERSION#v}_linux_${actionlint_arch}.tar.gz"
	curl -fsSL "${actionlint_url}" | tar -xz -C /usr/local/bin actionlint
	shellcheck \
		scripts/local-ci/push-lane.sh \
		scripts/local-ci/package-lane.sh \
		scripts/local-ci/package-validate.sh \
		scripts/local-ci/pre-push-ci-gate.sh \
		scripts/local-ci/native-macos-lane.sh \
		scripts/local-ci/smolvm-lane.sh
	actionlint -shellcheck= .github/workflows/act-sanity.yml .github/workflows/tests.yml .github/workflows/release.yml
	echo '[guest] static profile complete'
	;;
*)
	echo "unknown YAMS_SMOLVM_PROFILE=${YAMS_SMOLVM_PROFILE}" >&2
	exit 2
	;;
esac
GUEST_SCRIPT
}

linux_ci_guest_script() {
	cat <<'GUEST_SCRIPT'
set -eu
export DEBIAN_FRONTEND=noninteractive
printf '[guest] pwd=%s\n' "$(pwd)"
printf '[guest] uname=%s\n' "$(uname -m)"
test -f meson.build

git config --global --add safe.directory /workspace || true
apt-get -o Acquire::Retries=5 -o Acquire::ForceIPv4=true update
apt-get -o Acquire::Retries=5 -o Acquire::ForceIPv4=true install -y --no-install-recommends \
	build-essential ca-certificates ccache clang clang-tools cmake curl git lld \
	ninja-build pkg-config python3 python3-pip python3-venv xz-utils \
	libbenchmark-dev libboost-all-dev libomp-dev

export YAMS_SMOLVM_SWAP_SIZE="${YAMS_SMOLVM_SWAP_SIZE:-2G}"
if [ "${YAMS_SMOLVM_SWAP_SIZE}" != "0" ] && command -v swapon >/dev/null 2>&1; then
	SWAP_FILE=/tmp/yams-smolvm.swap
	echo "[guest] provisioning ${YAMS_SMOLVM_SWAP_SIZE} swap for link-heavy debug targets"
	rm -f -- "${SWAP_FILE}"
	if command -v fallocate >/dev/null 2>&1; then
		fallocate -l "${YAMS_SMOLVM_SWAP_SIZE}" "${SWAP_FILE}"
	else
		swap_size_to_mib() {
			size="$1"
			case "${size}" in
			*[Gg]) value="${size%?}"; multiplier=1024 ;;
			*[Mm]) value="${size%?}"; multiplier=1 ;;
			*[Kk])
				value="${size%?}"
				case "${value}" in '' | *[!0-9]*) return 1 ;; esac
				echo $(((value + 1023) / 1024))
				return 0
				;;
			*[0-9]) value="${size}"; multiplier=1 ;;
			*) return 1 ;;
			esac
			case "${value}" in '' | *[!0-9]*) return 1 ;; esac
			echo $((value * multiplier))
		}
		SWAP_MIB="$(swap_size_to_mib "${YAMS_SMOLVM_SWAP_SIZE}")" || {
			echo "unsupported YAMS_SMOLVM_SWAP_SIZE='${YAMS_SMOLVM_SWAP_SIZE}' without fallocate; use integer M/G/K units" >&2
			exit 2
		}
		dd if=/dev/zero of="${SWAP_FILE}" bs=1M count="${SWAP_MIB}" status=none
	fi
	chmod 600 "${SWAP_FILE}"
	mkswap "${SWAP_FILE}" >/dev/null
	swapon "${SWAP_FILE}" || echo "[guest] warning: swapon failed; continuing without extra swap" >&2
fi

PYTOOLS_DIR=/tmp/yams-python-tools
python3 -m venv "${PYTOOLS_DIR}"
export PIP_NO_INPUT=1
export PIP_DISABLE_PIP_VERSION_CHECK=1
export PIP_DEFAULT_TIMEOUT="${PIP_DEFAULT_TIMEOUT:-60}"
"${PYTOOLS_DIR}/bin/python" -m pip install --retries 5 --timeout "${PIP_DEFAULT_TIMEOUT}" --upgrade pip
"${PYTOOLS_DIR}/bin/python" -m pip install --retries 5 --timeout "${PIP_DEFAULT_TIMEOUT}" --upgrade 'conan==2.21.*' 'meson==1.4.*' 'ninja==1.11.*'
export PATH="${PYTOOLS_DIR}/bin:${PATH}"
conan profile detect --force

# The repo is a host bind mount. Do not run `git submodule update` inside
# the guest because host .git paths can be absolute macOS paths that are invalid
# in the Linux VM. Require host-initialized submodules instead.
for submodule_dir in third_party/sqlite-vec-cpp third_party/simeon third_party/symspell; do
	if [ ! -d "${submodule_dir}" ] || ! find "${submodule_dir}" -mindepth 1 -maxdepth 2 -print -quit | grep -q .; then
		echo "required submodule missing or empty: ${submodule_dir}" >&2
		echo "run on host: git submodule update --init --depth 1 ${submodule_dir}" >&2
		exit 128
	fi
done

case "$(uname -m)" in
x86_64) conan_arch="x86_64" ;;
aarch64 | arm64) conan_arch="armv8" ;;
*) echo "unsupported linux-ci arch: $(uname -m)" >&2; exit 2 ;;
esac

export CI=true
export RUNNER_OS=Linux
export ENABLE_TSAN=false
export YAMS_ENABLE_MOBILE_BINDINGS=false
export YAMS_BUILD_DIR="${YAMS_SMOLVM_BUILD_DIR:-build/smolvm-linux}"
export YAMS_CONAN_HOST_PROFILE="${YAMS_CONAN_HOST_PROFILE:-./conan/profiles/host-linux-clang}"
export YAMS_CONAN_ARCH="${YAMS_CONAN_ARCH:-${conan_arch}}"
export YAMS_EXTRA_MESON_FLAGS="${YAMS_EXTRA_MESON_FLAGS:---buildtype=debug -Dbuild-tests=true -Denable-onnx=disabled -Dtest-timeout-scale=2 -Denable-bench-tests=false -Dbuild-benchmarks=false -Denable-dcheck=true}"
export YAMS_SMOLVM_USE_LLD="${YAMS_SMOLVM_USE_LLD:-1}"
if [ "${YAMS_SMOLVM_USE_LLD}" != "0" ] && command -v ld.lld >/dev/null 2>&1; then
	case " ${YAMS_EXTRA_MESON_FLAGS} " in
	*"-Dcpp_link_args="*) ;;
	*) YAMS_EXTRA_MESON_FLAGS="${YAMS_EXTRA_MESON_FLAGS} -Dcpp_link_args=-fuse-ld=lld -Dc_link_args=-fuse-ld=lld" ;;
	esac
fi
export YAMS_SMOLVM_CLEAN_BUILD="${YAMS_SMOLVM_CLEAN_BUILD:-1}"
export YAMS_SMOLVM_CONAN_JOBS="${YAMS_SMOLVM_CONAN_JOBS:-1}"
export YAMS_CONAN_EXTRA_OPTIONS="-c tools.build:jobs=${YAMS_SMOLVM_CONAN_JOBS} ${YAMS_CONAN_EXTRA_OPTIONS:-}"
export CONAN_CPU_COUNT="${YAMS_SMOLVM_CONAN_JOBS}"
export CMAKE_BUILD_PARALLEL_LEVEL="${YAMS_SMOLVM_CONAN_JOBS}"
export MAKEFLAGS="-j${YAMS_SMOLVM_CONAN_JOBS}"

if [ "${YAMS_SMOLVM_CLEAN_BUILD}" != "0" ]; then
	case "${YAMS_BUILD_DIR}" in
	build/* | ./build/*)
		echo "[guest] cleaning smolvm build dir: ${YAMS_BUILD_DIR}"
		rm -rf -- "${YAMS_BUILD_DIR}"
		;;
	*)
		echo "refusing to clean non-build YAMS_BUILD_DIR=${YAMS_BUILD_DIR}" >&2
		exit 2
		;;
	esac
fi

./setup.sh Debug

CONAN_BUILD_ENV=""
for candidate in \
	"${YAMS_BUILD_DIR}/build-debug/conan/conanbuild.sh" \
	"${YAMS_BUILD_DIR}/conan/conanbuild.sh"
do
	if [ -f "${candidate}" ]; then
		CONAN_BUILD_ENV="${candidate}"
		break
	fi
done
if [ -n "${CONAN_BUILD_ENV}" ]; then
	set +u
	# shellcheck source=/dev/null
	. "${CONAN_BUILD_ENV}"
	set -u
fi

export YAMS_TEST_SAFE_SINGLE_INSTANCE=1
export YAMS_SQLITE_MINIMAL_PRAGMAS=1
export YAMS_SQLITE_VEC_INIT_TIMEOUT_MS=1500
export YAMS_SQLITE_BUSY_TIMEOUT_MS=1000
export YAMS_VDB_IN_MEMORY=1
export YAMS_SQLITE_VEC_SKIP_INIT=1
export YAMS_DISABLE_VECTORS=1
export YAMS_TESTING=1
export YAMS_SMOLVM_NINJA_JOBS="${YAMS_SMOLVM_NINJA_JOBS:-1}"
export YAMS_SMOLVM_TEST_JOBS="${YAMS_SMOLVM_TEST_JOBS:-1}"

printf '[guest] compile jobs=%s test jobs=%s\n' "${YAMS_SMOLVM_NINJA_JOBS}" "${YAMS_SMOLVM_TEST_JOBS}"
if [ -n "${YAMS_SMOLVM_COMPILE_TARGETS:-}" ]; then
	# shellcheck disable=SC2086
	meson compile -C "${YAMS_BUILD_DIR}" -j "${YAMS_SMOLVM_NINJA_JOBS}" ${YAMS_SMOLVM_COMPILE_TARGETS}
else
	meson compile -C "${YAMS_BUILD_DIR}" -j "${YAMS_SMOLVM_NINJA_JOBS}"
fi

if [ -n "${YAMS_SMOLVM_TEST_ARGS:-}" ]; then
	# shellcheck disable=SC2086
	meson test -C "${YAMS_BUILD_DIR}" ${YAMS_SMOLVM_TEST_ARGS}
else
	meson test -C "${YAMS_BUILD_DIR}" \
		--suite unit \
		--suite integration \
		--print-errorlogs \
		--timeout-multiplier 2 \
		--num-processes "${YAMS_SMOLVM_TEST_JOBS}"
fi
echo '[guest] linux-ci profile complete'
GUEST_SCRIPT
}

case "${PROFILE}" in
linux-ci)
	GUEST_SCRIPT_CONTENT="$(linux_ci_guest_script)"
	VOLUME_SPEC="${REPO_ROOT}:/workspace"
	if [ "${YAMS_SMOLVM_CLEAN_BUILD:-1}" != "0" ]; then
		HOST_BUILD_DIR="${YAMS_SMOLVM_BUILD_DIR:-build/smolvm-linux}"
		case "${HOST_BUILD_DIR}" in
		build/* | ./build/*)
			HOST_CLEAN_PATH="${REPO_ROOT}/${HOST_BUILD_DIR#./}"
			log "host cleaning smolvm build dir: ${HOST_BUILD_DIR}"
			rm -rf -- "${HOST_CLEAN_PATH:?}"
			;;
		*)
			fail "refusing to host-clean non-build YAMS_SMOLVM_BUILD_DIR=${HOST_BUILD_DIR}"
			exit 2
			;;
		esac
	fi
	;;
*)
	GUEST_SCRIPT_CONTENT="$(static_guest_script)"
	VOLUME_SPEC="${REPO_ROOT}:/workspace:ro"
	;;
esac

CMD=(
	"${SMOLVM_BIN}" machine run
	--net
	--image "${IMAGE}"
	-v "${VOLUME_SPEC}"
	-w /workspace
	-e "YAMS_SMOLVM_PROFILE=${PROFILE}"
	-e "ACTIONLINT_VERSION=${ACTIONLINT_VERSION}"
	-e "YAMS_SMOLVM_BUILD_DIR=${YAMS_SMOLVM_BUILD_DIR:-build/smolvm-linux}"
	-e "YAMS_SMOLVM_CLEAN_BUILD=${YAMS_SMOLVM_CLEAN_BUILD:-1}"
	-e "YAMS_SMOLVM_CONAN_JOBS=${YAMS_SMOLVM_CONAN_JOBS:-1}"
	-e "YAMS_SMOLVM_NINJA_JOBS=${YAMS_SMOLVM_NINJA_JOBS:-1}"
	-e "YAMS_SMOLVM_COMPILE_TARGETS=${YAMS_SMOLVM_COMPILE_TARGETS:-}"
	-e "YAMS_SMOLVM_TEST_JOBS=${YAMS_SMOLVM_TEST_JOBS:-1}"
	-e "YAMS_SMOLVM_TEST_ARGS=${YAMS_SMOLVM_TEST_ARGS:-}"
	-- sh -lc "${GUEST_SCRIPT_CONTENT}"
)

{
	echo "# YAMS smolvm lane (${PROFILE})"
	echo
	echo "- Started: ${RUN_TAG}"
	echo "- Image: ${IMAGE}"
	echo "- smolvm: ${SMOLVM_BIN}"
	echo
	echo '```sh'
	printf '%q ' "${CMD[@]}"
	echo
	echo '```'
	echo
} >"${SUMMARY}"

log "profile=${PROFILE} image=${IMAGE}"
if [ "${DRY_RUN}" -eq 1 ]; then
	printf '%q ' "${CMD[@]}" | tee "${LOG_FILE}"
	echo | tee -a "${LOG_FILE}"
	echo "- Result: dry-run" >>"${SUMMARY}"
	ok "summary: ${SUMMARY#"${REPO_ROOT}"/}"
	exit 0
fi

set +e
"${CMD[@]}" > >(tee "${LOG_FILE}") 2> >(tee -a "${LOG_FILE}" >&2)
rc=$?
set -e

if [ "${rc}" -eq 0 ]; then
	echo "- Result: pass" >>"${SUMMARY}"
	echo "- Log: ${LOG_FILE#"${REPO_ROOT}"/}" >>"${SUMMARY}"
	ok "summary: ${SUMMARY#"${REPO_ROOT}"/}"
else
	echo "- Result: fail (${rc})" >>"${SUMMARY}"
	echo "- Log: ${LOG_FILE#"${REPO_ROOT}"/}" >>"${SUMMARY}"
	fail "smolvm lane failed (${rc}); log: ${LOG_FILE}"
	exit "${rc}"
fi