* test(search,daemon,storage): assert behaviour instead of frozen literals Several suites encoded exact internal values that legitimate tuning and ranking changes are allowed to move: SCIENTIFIC-era profile weights and rrfK values, corpus-statistic figures, and hardcoded timeout/cancellation numbers. They failed on every platform after those changes landed, so the real behaviour regressions they should catch were hidden behind noise. Replace the frozen literals with contract assertions: weights are finite, non-negative and normalize, state selection still holds, capability-gated sources stay zeroed, and hybrid results keep matching canonical hash/path sets. Deadlines are not widened and the daemon paths they cover are unchanged. Also add independent corpus and adapter oracles, deterministic collector cancellation and late-result gates, and synchronize ServiceManager shutdown observation, so the tests fail for the behaviour they name. The portability allowlist keeps its two reviewed raw test-env reads; only their line numbers moved. Signed-off-by: trvon <git@trevon.dev> * ci(tests): summarise Meson failures without flooding the log A failing test run printed the whole Meson suite output, so the actual failing test names and their assertion messages were buried and the uploaded logs were dominated by passing-test chatter. Add a bounded reporter that extracts the failing test names plus their surrounding diagnostics, keeps an overall cap on emitted lines, and is unit-tested against representative Meson output. Wire it into the Tests workflow so failures stay readable. Signed-off-by: trvon <git@trevon.dev> * build(deb): validate candidate identity, mapping and upgrade ordering The Debian packaging scripts took filenames and versions on trust, so a misnamed candidate or a wrong package ordering could pass validation while the real install path was never exercised. Fail closed on candidate identity, version and architecture, verify the package-to-source mapping, seed the legacy-only layout, wait on the actual services, and assert install/upgrade/remove ordering for both Debian and RPM in an offline container. Cases that cannot be proven on a host are reported as skipped rather than silently passing. Signed-off-by: trvon <git@trevon.dev> --------- Signed-off-by: trvon <git@trevon.dev>
871 lines
28 KiB
Bash
Executable file
871 lines
28 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
# Debian package builder for YAMS (trixie)
|
|
# - Mirrors the release CI tasks using Meson + Conan toolchain
|
|
# - Produces tarball, .deb, and .rpm artifacts for tagged builds
|
|
# - Intended to run inside a Debian trixie container with your repo mounted at /workspace/yams
|
|
#
|
|
# Example:
|
|
# docker run --rm -it \
|
|
# -e GIT_REF=refs/tags/v1.2.3 \
|
|
# -e CONAN_CPU_COUNT=2 \
|
|
# -e MESON_EXTRA_ARGS="-Denable-onnx=disabled" \
|
|
# -v "$PWD":/workspace/yams \
|
|
# -w /workspace \
|
|
# debian:trixie bash -lc '/workspace/yams/scripts/srht-debian.sh'
|
|
#
|
|
# Artifacts:
|
|
# - /workspace/yams/build/release/yams-*.tar.gz
|
|
# - /workspace/yams/build/release/yams-*.deb
|
|
# - /workspace/yams/build/release/yams-*.rpm
|
|
set -euo pipefail
|
|
set -x
|
|
|
|
export DEBIAN_FRONTEND=noninteractive
|
|
export PATH="${HOME}/.local/bin:${PATH}"
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
if [ -d /workspace/yams ]; then
|
|
REPO_ROOT="/workspace/yams"
|
|
else
|
|
REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
|
|
fi
|
|
|
|
allowed_deb_ver_chars='A-Za-z0-9.+:~'
|
|
allowed_rpm_ver_chars='A-Za-z0-9._+'
|
|
|
|
normalize_arch_label() {
|
|
local arch="$1"
|
|
case "${arch}" in
|
|
amd64|x86_64) echo "x86_64" ;;
|
|
arm64|aarch64) echo "aarch64" ;;
|
|
*) echo "${arch}" ;;
|
|
esac
|
|
}
|
|
|
|
append_buildenv() {
|
|
echo "$1" >> "${HOME}/.buildenv"
|
|
}
|
|
|
|
persist_env_defaults() {
|
|
export BUILD_DIR="${BUILD_DIR:-build/release}"
|
|
export STAGE_DIR="${STAGE_DIR:-stage}"
|
|
local default_jobs
|
|
if command -v nproc >/dev/null 2>&1; then
|
|
default_jobs="$(nproc)"
|
|
else
|
|
default_jobs="$(getconf _NPROCESSORS_ONLN 2>/dev/null || echo 2)"
|
|
fi
|
|
export CONAN_CPU_COUNT="${CONAN_CPU_COUNT:-${default_jobs}}"
|
|
: > "${HOME}/.buildenv"
|
|
append_buildenv "export BUILD_DIR=${BUILD_DIR}"
|
|
append_buildenv "export STAGE_DIR=${STAGE_DIR}"
|
|
append_buildenv "export CONAN_CPU_COUNT=${CONAN_CPU_COUNT}"
|
|
append_buildenv "export PATH=${HOME}/.local/bin:\\$PATH"
|
|
}
|
|
|
|
ensure_base_packages() {
|
|
if ! command -v apt-get >/dev/null 2>&1; then
|
|
echo "apt-get not available; skipping base package install" >&2
|
|
return
|
|
fi
|
|
|
|
local -a apt_runner
|
|
if [ "$(id -u)" -eq 0 ]; then
|
|
apt_runner=(apt-get)
|
|
elif command -v sudo >/dev/null 2>&1; then
|
|
apt_runner=(sudo apt-get)
|
|
else
|
|
echo "Skipping apt-get install (requires root privileges); assuming base image already provides build dependencies" >&2
|
|
return
|
|
fi
|
|
|
|
"${apt_runner[@]}" update -y
|
|
"${apt_runner[@]}" install -y --no-install-recommends \
|
|
build-essential \
|
|
cmake \
|
|
meson \
|
|
ninja-build \
|
|
zip \
|
|
ccache \
|
|
dpkg-dev \
|
|
fakeroot \
|
|
rpm \
|
|
lld \
|
|
clang \
|
|
llvm \
|
|
python3 \
|
|
python3-venv \
|
|
python3-pip \
|
|
python3-numpy \
|
|
pipx \
|
|
git \
|
|
ca-certificates \
|
|
pkg-config \
|
|
liburing-dev \
|
|
libarchive-dev \
|
|
libtag1-dev \
|
|
libsqlite3-dev \
|
|
libssl-dev \
|
|
libcurl4-gnutls-dev \
|
|
protobuf-compiler \
|
|
libprotobuf-dev \
|
|
libmediainfo-dev \
|
|
mediainfo
|
|
}
|
|
|
|
install_conan() {
|
|
if ! command -v conan >/dev/null 2>&1; then
|
|
if command -v pipx >/dev/null 2>&1; then
|
|
pipx install 'conan<3' --pip-args='--upgrade' || true
|
|
fi
|
|
fi
|
|
if ! command -v conan >/dev/null 2>&1; then
|
|
python3 -m pip install --user --upgrade 'conan<3' || true
|
|
hash -r || true
|
|
fi
|
|
if ! command -v conan >/dev/null 2>&1; then
|
|
python3 -m venv /opt/conan
|
|
. /opt/conan/bin/activate
|
|
python -m pip install --upgrade pip setuptools wheel
|
|
python -m pip install 'conan<3'
|
|
export PATH="/opt/conan/bin:${PATH}"
|
|
hash -r
|
|
fi
|
|
if command -v conan >/dev/null 2>&1; then
|
|
append_buildenv "export PATH=$(dirname "$(command -v conan)"):\\$PATH"
|
|
else
|
|
echo "Conan installation failed" >&2
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
export_custom_conan_recipes() {
|
|
cd "${REPO_ROOT}"
|
|
local recipe_dir="conan/onnxruntime"
|
|
if [ ! -f "${recipe_dir}/conanfile.py" ]; then
|
|
echo "Missing required Conan recipe at ${REPO_ROOT}/${recipe_dir}" >&2
|
|
exit 1
|
|
fi
|
|
|
|
echo "Exporting custom Conan recipe from ${recipe_dir}" >&2
|
|
conan export "${recipe_dir}"
|
|
}
|
|
|
|
compute_version() {
|
|
cd "${REPO_ROOT}"
|
|
local raw_tag
|
|
if [[ "${GIT_REF:-}" =~ ^refs/tags/ ]]; then
|
|
raw_tag="${GIT_REF#refs/tags/}"
|
|
else
|
|
raw_tag="$(git describe --tags --abbrev=0 2>/dev/null || echo 0.0.0-dev)"
|
|
fi
|
|
CLEAN_VERSION="${raw_tag#v}"
|
|
export YAMS_VERSION="${CLEAN_VERSION}"
|
|
append_buildenv "export YAMS_VERSION=${CLEAN_VERSION}"
|
|
echo "Computed YAMS_VERSION=${CLEAN_VERSION} (GIT_REF=${GIT_REF:-n/a})"
|
|
}
|
|
|
|
configure_conan_profile() {
|
|
conan --version
|
|
conan profile detect --force || true
|
|
conan profile show -pr default || conan profile show || true
|
|
}
|
|
|
|
run_conan_install() {
|
|
cd "${REPO_ROOT}"
|
|
rm -rf "${BUILD_DIR}"
|
|
# Select the Conan profile by host architecture so native builds work on both
|
|
# x86_64 and arm64 (the x86_64 profile pins arch=x86_64 and emits -m64, which
|
|
# a native arm64 gcc rejects).
|
|
local conan_profile="conan/profiles/host-linux-gcc"
|
|
case "$(uname -m)" in
|
|
aarch64|arm64) conan_profile="conan/profiles/host-linux-gcc-arm" ;;
|
|
esac
|
|
echo "Using Conan profile: ${conan_profile} (host $(uname -m))"
|
|
local -a conan_extra=()
|
|
if [ -n "${CONAN_EXTRA_OPTS:-}" ]; then
|
|
# shellcheck disable=SC2206
|
|
conan_extra=(${CONAN_EXTRA_OPTS})
|
|
fi
|
|
conan install . \
|
|
-of "${BUILD_DIR}" \
|
|
-pr:h="./${conan_profile}" \
|
|
-pr:b="./${conan_profile}" \
|
|
-s build_type=Release \
|
|
-c tools.build:jobs="${CONAN_CPU_COUNT}" \
|
|
-o "sqlite3/*:fts5=True" \
|
|
"${conan_extra[@]}" \
|
|
--build=missing
|
|
}
|
|
|
|
run_meson_build() {
|
|
cd "${REPO_ROOT}"
|
|
export YAMS_DISABLE_MODEL_PRELOAD=1
|
|
local native_file="${BUILD_DIR}/build-release/conan/conan_meson_native.ini"
|
|
local meson_args=()
|
|
if [ -n "${MESON_EXTRA_ARGS:-}" ]; then
|
|
# shellcheck disable=SC2206
|
|
meson_args=(${MESON_EXTRA_ARGS})
|
|
fi
|
|
# Check if meson is already configured; use --reconfigure instead of --wipe
|
|
# to preserve Conan-generated files in the build directory
|
|
local setup_mode=""
|
|
if [ -f "${BUILD_DIR}/meson-private/coredata.dat" ]; then
|
|
setup_mode="--reconfigure"
|
|
fi
|
|
meson setup "${BUILD_DIR}" \
|
|
--prefix=/usr \
|
|
--buildtype=release \
|
|
--native-file "${native_file}" \
|
|
${setup_mode} \
|
|
"${meson_args[@]}"
|
|
meson compile -C "${BUILD_DIR}"
|
|
meson install -C "${BUILD_DIR}" --destdir "${STAGE_DIR}"
|
|
}
|
|
|
|
# --- Version sanitization helpers -------------------------------------------------
|
|
|
|
get_base_version() {
|
|
# Prefer explicit env from CI; otherwise attempt to derive from git; fallback 0.0.0
|
|
if [ -n "${BASE_VERSION:-}" ]; then
|
|
echo "${BASE_VERSION}"
|
|
return
|
|
fi
|
|
if git -C "${REPO_ROOT}" describe --tags --abbrev=0 --match='v*' >/dev/null 2>&1; then
|
|
git -C "${REPO_ROOT}" describe --tags --abbrev=0 --match='v*' 2>/dev/null | sed 's/^v//'
|
|
return
|
|
fi
|
|
echo "0.0.0"
|
|
}
|
|
|
|
sanitize_for_deb_upstream() {
|
|
# Map arbitrary string to Debian upstream version-safe token (no hyphens)
|
|
# Allowed: ${allowed_deb_ver_chars}; convert others to '.' and collapse repeats
|
|
local s="$1"
|
|
s="${s//-/.}"
|
|
s="${s//_/.}"
|
|
s="$(echo "$s" | sed -E "s/[^${allowed_deb_ver_chars}]+/./g" | sed -E 's/\.+/./g' | sed -E 's/^\.|\.$//g')"
|
|
echo "$s"
|
|
}
|
|
|
|
compute_deb_version() {
|
|
local in_ver="$1"
|
|
local base_ver
|
|
base_ver="$(get_base_version)"
|
|
# Stable numeric-ish versions without hyphens are acceptable as-is
|
|
if [[ "$in_ver" =~ ^[0-9][0-9A-Za-z.+:~]*$ ]] && [[ "$in_ver" != *-* ]]; then
|
|
echo "$in_ver"
|
|
return
|
|
fi
|
|
# Nightly: nightly-YYYYMMDD-<hash>
|
|
if [[ "$in_ver" =~ ^nightly-([0-9]{8})-([0-9a-fA-F]{7,40})$ ]]; then
|
|
# Nightlies follow the base stable release, but precede the next numeric release.
|
|
echo "${base_ver}+nightly.${BASH_REMATCH[1]}+g${BASH_REMATCH[2],,}"
|
|
return
|
|
fi
|
|
# Weekly: weekly-YYYYwWW-<hash>
|
|
if [[ "$in_ver" =~ ^weekly-([0-9]{4}w[0-9]{2})-([0-9a-fA-F]{7,40})$ ]]; then
|
|
echo "${base_ver}+weekly.${BASH_REMATCH[1]}+g${BASH_REMATCH[2],,}"
|
|
return
|
|
fi
|
|
# Fallback: general sanitize
|
|
local token
|
|
token="$(sanitize_for_deb_upstream "$in_ver")"
|
|
echo "${base_ver}~${token}"
|
|
}
|
|
|
|
sanitize_for_rpm_token() {
|
|
# Allowed in rpm Version/Release: ${allowed_rpm_ver_chars} and '~' (for pre-release)
|
|
local s="$1"
|
|
s="${s//-/.}"
|
|
s="${s//_/.}"
|
|
s="$(echo "$s" | sed -E "s/[^${allowed_rpm_ver_chars}~]+/./g" | sed -E 's/\.+/./g' | sed -E 's/^\.|\.$//g')"
|
|
echo "$s"
|
|
}
|
|
|
|
compute_rpm_nv() {
|
|
# Outputs two values via echoes: rpm_version rpm_release
|
|
local in_ver="$1"
|
|
local base_ver
|
|
base_ver="$(get_base_version)"
|
|
# Stable numeric-ish with no hyphen: put all in Version and keep Release=1
|
|
if [[ "$in_ver" =~ ^[0-9][0-9A-Za-z._+]*$ ]]; then
|
|
echo "$in_ver"
|
|
echo "1"
|
|
return
|
|
fi
|
|
# Nightly pattern
|
|
if [[ "$in_ver" =~ ^nightly-([0-9]{8})-([0-9a-fA-F]{7,40})$ ]]; then
|
|
echo "$base_ver"
|
|
# Release 1.1.* follows stable Release 1 at the same base Version.
|
|
echo "1.1.nightly.${BASH_REMATCH[1]}.g${BASH_REMATCH[2],,}"
|
|
return
|
|
fi
|
|
# Weekly pattern
|
|
if [[ "$in_ver" =~ ^weekly-([0-9]{4}w[0-9]{2})-([0-9a-fA-F]{7,40})$ ]]; then
|
|
echo "$base_ver"
|
|
echo "1.1.weekly.${BASH_REMATCH[1]}.g${BASH_REMATCH[2],,}"
|
|
return
|
|
fi
|
|
# Fallback: generic sanitize in Release, keep Version as base
|
|
local token
|
|
token="$(sanitize_for_rpm_token "$in_ver")"
|
|
echo "$base_ver"
|
|
echo "0.1~${token}"
|
|
}
|
|
|
|
prepare_stage_docs() {
|
|
local stage_root="$1"
|
|
install -Dm644 "${REPO_ROOT}/LICENSE" "${stage_root}/usr/share/doc/yams/LICENSE"
|
|
if [ -f "${REPO_ROOT}/README.md" ]; then
|
|
install -Dm644 "${REPO_ROOT}/README.md" "${stage_root}/usr/share/doc/yams/README.md"
|
|
fi
|
|
}
|
|
|
|
package_tarball() {
|
|
local version="$1"
|
|
local build_dir="$2"
|
|
local stage_root="$3"
|
|
local arch_label
|
|
arch_label="$(normalize_arch_label "$(uname -m)")"
|
|
local tar_name="yams-${version}-linux-${arch_label}.tar.gz"
|
|
tar -C "${stage_root}" -czf "${build_dir}/${tar_name}" .
|
|
echo "Created tarball: ${build_dir}/${tar_name}"
|
|
}
|
|
|
|
package_deb() {
|
|
if ! command -v dpkg-deb >/dev/null 2>&1; then
|
|
echo "dpkg-deb not available; skipping .deb packaging" >&2
|
|
return
|
|
fi
|
|
local version="$1"
|
|
local build_dir="$2"
|
|
local stage_root="$3"
|
|
local deb_arch
|
|
deb_arch="$(dpkg --print-architecture 2>/dev/null || echo amd64)"
|
|
local work_dir="${build_dir}/deb-work"
|
|
rm -rf "${work_dir}"
|
|
mkdir -p "${work_dir}"
|
|
cp -a "${stage_root}/." "${work_dir}/"
|
|
mkdir -p "${work_dir}/DEBIAN"
|
|
local control="${work_dir}/DEBIAN/control"
|
|
cat > "${control}" <<'__DEB_CONTROL__'
|
|
Package: yams
|
|
Version: __VERSION__
|
|
Section: utils
|
|
Priority: optional
|
|
Architecture: __ARCH__
|
|
Maintainer: YAMS Contributors <git@trevon.dev>
|
|
Homepage: https://git.trevon.dev/trevon/yams
|
|
Depends: @DEPENDENCIES@
|
|
Description: Yet Another Memory System - persistent memory for LLMs
|
|
YAMS provides content-addressed storage, deduplication, and search for long-term LLM memory.
|
|
__DEB_CONTROL__
|
|
local deb_version
|
|
deb_version="$(compute_deb_version "$version")"
|
|
sed -i "s|__VERSION__|${deb_version}|" "${control}"
|
|
sed -i "s|__ARCH__|${deb_arch}|" "${control}"
|
|
|
|
local -a binaries=()
|
|
mapfile -d '' binaries < <(find "${work_dir}" -type f \( -perm -111 -o -name '*.so*' \) -print0) || true
|
|
local depends_fallback="libc6 (>= 2.31), libstdc++6 (>= 12)"
|
|
if command -v dpkg-shlibdeps >/dev/null 2>&1 && ((${#binaries[@]} > 0)); then
|
|
local shlibdeps_root="${build_dir}/deb-shlibdeps"
|
|
rm -rf "${shlibdeps_root}"
|
|
mkdir -p "${shlibdeps_root}/debian"
|
|
ln -s "${work_dir}" "${shlibdeps_root}/debian/yams"
|
|
cat > "${shlibdeps_root}/debian/control" <<'__DEBIAN_CONTROL__'
|
|
Source: yams
|
|
Section: utils
|
|
Priority: optional
|
|
Maintainer: YAMS Contributors <git@trevon.dev>
|
|
Standards-Version: 4.7.0
|
|
|
|
Package: yams
|
|
Architecture: any
|
|
Depends: ${shlibs:Depends}, ${misc:Depends}
|
|
Description: Yet Another Memory System - persistent memory for LLMs
|
|
YAMS provides content-addressed storage, deduplication, and search for long-term LLM memory.
|
|
__DEBIAN_CONTROL__
|
|
: > "${shlibdeps_root}/debian/substvars"
|
|
local -a relative_binaries=()
|
|
for bin in "${binaries[@]}"; do
|
|
relative_binaries+=("debian/yams/${bin#"${work_dir}/"}")
|
|
done
|
|
local had_xtrace=0
|
|
case "$-" in
|
|
*x*)
|
|
had_xtrace=1
|
|
set +x
|
|
;;
|
|
esac
|
|
set +e
|
|
local shlib_output
|
|
shlib_output="$(cd "${shlibdeps_root}" && dpkg-shlibdeps -O -Tdebian/substvars -Sdebian/yams -ldebian/yams/usr/lib -ldebian/yams/usr/lib/yams/plugins "${relative_binaries[@]}" 2>&1)"
|
|
local shlib_status=$?
|
|
set -e
|
|
if [ "${shlib_status}" -eq 0 ] && [[ "${shlib_output}" =~ shlibs:Depends=(.*) ]]; then
|
|
depends_fallback="${BASH_REMATCH[1]}"
|
|
else
|
|
echo "${shlib_output}" >&2
|
|
fi
|
|
rm -rf "${shlibdeps_root}"
|
|
if [ "${had_xtrace}" -eq 1 ]; then
|
|
set -x
|
|
fi
|
|
fi
|
|
# The maintainer scripts only call systemctl/runuser when they exist, so the
|
|
# package needs nothing beyond its shared-library dependencies.
|
|
sed -i "s|@DEPENDENCIES@|${depends_fallback}|" "${control}"
|
|
|
|
find "${work_dir}" -type d -exec chmod 0755 {} +
|
|
find "${work_dir}" -type f -exec chmod 0644 {} +
|
|
if ((${#binaries[@]} > 0)); then
|
|
for bin in "${binaries[@]}"; do
|
|
chmod 0755 "${bin}"
|
|
done
|
|
fi
|
|
|
|
# Maintainer scripts. The package ships a systemd *user* unit
|
|
# (/usr/lib/systemd/user/yams-daemon.service) and a user preset that enables
|
|
# it; it starts with each user's next session. Releases up to 0.20.x shipped a
|
|
# *system* unit, which preinst stops and disables while its file still exists.
|
|
cat > "${work_dir}/DEBIAN/preinst" <<'__DEB_PREINST__'
|
|
#!/bin/sh
|
|
set -e
|
|
|
|
UNIT=yams-daemon.service
|
|
|
|
if [ "$1" = "install" ] || [ "$1" = "upgrade" ]; then
|
|
if [ -f "/usr/lib/systemd/system/$UNIT" ] || [ -f "/lib/systemd/system/$UNIT" ]; then
|
|
if command -v systemctl >/dev/null 2>&1; then
|
|
if [ -d /run/systemd/system ]; then
|
|
systemctl --system stop "$UNIT" >/dev/null 2>&1 || true
|
|
fi
|
|
systemctl --system disable "$UNIT" >/dev/null 2>&1 || true
|
|
fi
|
|
if command -v deb-systemd-helper >/dev/null 2>&1; then
|
|
deb-systemd-helper purge "$UNIT" >/dev/null 2>&1 || true
|
|
deb-systemd-helper unmask "$UNIT" >/dev/null 2>&1 || true
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
exit 0
|
|
__DEB_PREINST__
|
|
|
|
cat > "${work_dir}/DEBIAN/postinst" <<'__DEB_POSTINST__'
|
|
#!/bin/sh
|
|
set -e
|
|
|
|
UNIT=yams-daemon.service
|
|
|
|
# Run `systemctl --user <args>` in every running user manager (no-op without
|
|
# systemd, runuser or logged-in users). Used for daemon-reload, try-restart and
|
|
# stop; the scripts never start per-user instances.
|
|
yams_user_managers() {
|
|
command -v runuser >/dev/null 2>&1 || return 0
|
|
command -v systemctl >/dev/null 2>&1 || return 0
|
|
for d in /run/user/*; do
|
|
# A running user manager always has its private socket; the D-Bus user
|
|
# bus is optional (Debian/Ubuntu without dbus-user-session).
|
|
[ -S "$d/systemd/private" ] || continue
|
|
uid="${d##*/}"
|
|
user="$(getent passwd "$uid" | cut -d: -f1)"
|
|
[ -n "$user" ] || continue
|
|
runuser -u "$user" -- env XDG_RUNTIME_DIR="$d" \
|
|
systemctl --user "$@" >/dev/null 2>&1 || true
|
|
done
|
|
}
|
|
|
|
if [ "$1" = "configure" ]; then
|
|
if command -v systemctl >/dev/null 2>&1; then
|
|
# First install, or upgrade from a release with the system unit: enable
|
|
# the user unit for all users per the user preset. Later upgrades keep
|
|
# an administrator's `systemctl --global disable`.
|
|
if [ -z "$2" ] || dpkg --compare-versions "$2" lt "0.20.4~"; then
|
|
systemctl --global preset "$UNIT" >/dev/null 2>&1 || true
|
|
fi
|
|
if [ -d /run/systemd/system ]; then
|
|
systemctl --system daemon-reload >/dev/null 2>&1 || true
|
|
fi
|
|
fi
|
|
yams_user_managers daemon-reload
|
|
if [ -n "$2" ]; then
|
|
yams_user_managers try-restart "$UNIT"
|
|
else
|
|
echo "yams: the per-user daemon starts with your next login session; to start it now run"
|
|
echo "yams: systemctl --user enable --now $UNIT"
|
|
fi
|
|
fi
|
|
|
|
exit 0
|
|
__DEB_POSTINST__
|
|
|
|
cat > "${work_dir}/DEBIAN/prerm" <<'__DEB_PRERM__'
|
|
#!/bin/sh
|
|
set -e
|
|
|
|
UNIT=yams-daemon.service
|
|
|
|
# Run `systemctl --user <args>` in every running user manager (no-op without
|
|
# systemd, runuser or logged-in users). Used for daemon-reload, try-restart and
|
|
# stop; the scripts never start per-user instances.
|
|
yams_user_managers() {
|
|
command -v runuser >/dev/null 2>&1 || return 0
|
|
command -v systemctl >/dev/null 2>&1 || return 0
|
|
for d in /run/user/*; do
|
|
# A running user manager always has its private socket; the D-Bus user
|
|
# bus is optional (Debian/Ubuntu without dbus-user-session).
|
|
[ -S "$d/systemd/private" ] || continue
|
|
uid="${d##*/}"
|
|
user="$(getent passwd "$uid" | cut -d: -f1)"
|
|
[ -n "$user" ] || continue
|
|
runuser -u "$user" -- env XDG_RUNTIME_DIR="$d" \
|
|
systemctl --user "$@" >/dev/null 2>&1 || true
|
|
done
|
|
}
|
|
|
|
if [ "$1" = "remove" ] || [ "$1" = "deconfigure" ]; then
|
|
yams_user_managers stop "$UNIT"
|
|
if command -v systemctl >/dev/null 2>&1; then
|
|
systemctl --global disable "$UNIT" >/dev/null 2>&1 || true
|
|
fi
|
|
fi
|
|
|
|
exit 0
|
|
__DEB_PRERM__
|
|
|
|
cat > "${work_dir}/DEBIAN/postrm" <<'__DEB_POSTRM__'
|
|
#!/bin/sh
|
|
set -e
|
|
|
|
# Run `systemctl --user <args>` in every running user manager (no-op without
|
|
# systemd, runuser or logged-in users). Used for daemon-reload, try-restart and
|
|
# stop; the scripts never start per-user instances.
|
|
yams_user_managers() {
|
|
command -v runuser >/dev/null 2>&1 || return 0
|
|
command -v systemctl >/dev/null 2>&1 || return 0
|
|
for d in /run/user/*; do
|
|
# A running user manager always has its private socket; the D-Bus user
|
|
# bus is optional (Debian/Ubuntu without dbus-user-session).
|
|
[ -S "$d/systemd/private" ] || continue
|
|
uid="${d##*/}"
|
|
user="$(getent passwd "$uid" | cut -d: -f1)"
|
|
[ -n "$user" ] || continue
|
|
runuser -u "$user" -- env XDG_RUNTIME_DIR="$d" \
|
|
systemctl --user "$@" >/dev/null 2>&1 || true
|
|
done
|
|
}
|
|
|
|
if [ "$1" = "remove" ] || [ "$1" = "purge" ]; then
|
|
yams_user_managers daemon-reload
|
|
fi
|
|
# Users' corpora (~/.local/share/yams) are never touched. /var/lib/yams is the
|
|
# unused corpus of the old system service (0.20.x); remove it by hand.
|
|
|
|
exit 0
|
|
__DEB_POSTRM__
|
|
|
|
chmod 0755 "${work_dir}/DEBIAN/preinst" "${work_dir}/DEBIAN/postinst" "${work_dir}/DEBIAN/prerm" "${work_dir}/DEBIAN/postrm"
|
|
|
|
local deb_name="yams-${version}-linux-$(normalize_arch_label "${deb_arch}")".deb
|
|
if command -v fakeroot >/dev/null 2>&1; then
|
|
fakeroot dpkg-deb --build "${work_dir}" "${build_dir}/${deb_name}"
|
|
else
|
|
dpkg-deb --build "${work_dir}" "${build_dir}/${deb_name}"
|
|
fi
|
|
rm -rf "${work_dir}"
|
|
echo "Created Debian package: ${build_dir}/${deb_name}"
|
|
}
|
|
|
|
package_rpm() {
|
|
if ! command -v rpmbuild >/dev/null 2>&1; then
|
|
echo "rpmbuild not available; skipping .rpm packaging" >&2
|
|
return
|
|
fi
|
|
local version="$1"
|
|
local build_dir="$2"
|
|
local stage_root="$3"
|
|
local rpm_arch
|
|
rpm_arch="$(rpm --eval '%{_arch}' 2>/dev/null || uname -m)"
|
|
local rpm_root="${build_dir}/rpmbuild"
|
|
rm -rf "${rpm_root}"
|
|
mkdir -p "${rpm_root}/"{BUILD,RPMS,SOURCES,SPECS,SRPMS}
|
|
|
|
# Compute rpm_version early to use for tarball name
|
|
local rpm_version rpm_release
|
|
local rpm_output
|
|
rpm_output="$(compute_rpm_nv "$version")"
|
|
rpm_version="$(echo "$rpm_output" | sed -n '1p')"
|
|
rpm_release="$(echo "$rpm_output" | sed -n '2p')"
|
|
rpm_release="${rpm_release:-1}"
|
|
|
|
local source_dir="${rpm_root}/SOURCES/yams-${rpm_version}"
|
|
mkdir -p "${source_dir}"
|
|
cp -a "${stage_root}/." "${source_dir}/"
|
|
tar -C "${rpm_root}/SOURCES" -czf "${rpm_root}/SOURCES/yams-${rpm_version}.tar.gz" "yams-${rpm_version}"
|
|
rm -rf "${source_dir}"
|
|
|
|
local rpm_filelist="${rpm_root}/SOURCES/filelist"
|
|
# System directories owned by the filesystem package — never claim these.
|
|
local -A system_dirs=(
|
|
[usr]=1 [usr/bin]=1 [usr/sbin]=1
|
|
[usr/lib]=1 [usr/lib64]=1
|
|
[usr/share]=1 [usr/share/doc]=1 [usr/share/man]=1
|
|
[usr/share/man/man1]=1 [usr/share/man/man5]=1 [usr/share/man/man8]=1
|
|
[usr/share/bash-completion]=1 [usr/share/bash-completion/completions]=1
|
|
[usr/share/zsh]=1 [usr/share/zsh/site-functions]=1
|
|
[usr/share/fish]=1 [usr/share/fish/vendor_completions.d]=1
|
|
[usr/share/licenses]=1
|
|
[usr/lib/systemd]=1 [usr/lib/systemd/system]=1 [usr/lib/systemd/system-preset]=1
|
|
[usr/lib/systemd/user]=1 [usr/lib/systemd/user-preset]=1
|
|
[etc]=1 [var]=1
|
|
)
|
|
(
|
|
cd "${stage_root}"
|
|
find . -type d -printf "%P\n" | LC_ALL=C sort | while read -r dir; do
|
|
[ -z "${dir}" ] && continue
|
|
[[ -n "${system_dirs[${dir}]+_}" ]] && continue
|
|
echo "%dir /${dir}"
|
|
done
|
|
find . \( -type f -o -type l \) -printf "%P\n" | LC_ALL=C sort | while read -r file; do
|
|
[ -z "${file}" ] && continue
|
|
echo "/${file}"
|
|
done
|
|
) > "${rpm_filelist}"
|
|
|
|
local spec="${rpm_root}/SPECS/yams.spec"
|
|
cat > "${spec}" <<'__RPM_SPEC__'
|
|
%global debug_package %{nil}
|
|
%global __strip /bin/true
|
|
# zstd payload (level 19) instead of rpm's gzip-class default; rpm >= 4.14.
|
|
%define _binary_payload w19.zstdio
|
|
# Libraries under /usr/lib/yams are private (plugins, libyams_onnx_resource, the
|
|
# bundled ONNX Runtime): do not advertise them as system sonames, and do not require
|
|
# them from elsewhere since this package ships them.
|
|
%global __provides_exclude_from ^/usr/lib(64)?/yams/.*$
|
|
%global __requires_exclude ^(libyams_onnx_resource|libzpdf|libonnxruntime)\\.so.*$
|
|
Name: yams
|
|
Version: __VERSION__
|
|
Release: __RELEASE__%{?dist}
|
|
Summary: Yet Another Memory System
|
|
License: GPL-3.0-or-later
|
|
URL: https://git.trevon.dev/trevon/yams
|
|
Source0: %{name}-%{version}.tar.gz
|
|
BuildArch: __RPM_ARCH__
|
|
Requires(post): systemd
|
|
Requires(preun): systemd
|
|
Requires(postun): systemd
|
|
Requires(post): util-linux
|
|
|
|
%description
|
|
Yet Another Memory System (YAMS) provides content-addressed storage with deduplication and search designed for long-term large language model memory.
|
|
|
|
%prep
|
|
%setup -q
|
|
|
|
%build
|
|
# Upstream build executed via Meson prior to packaging.
|
|
|
|
%install
|
|
rm -rf %{buildroot}
|
|
mkdir -p %{buildroot}
|
|
cp -a %{_builddir}/%{name}-%{version}/. %{buildroot}/
|
|
|
|
# Scriptlets for a systemd *user* unit, hand-written so the rpm can be built on
|
|
# hosts without systemd-rpm-macros (macro names in comments are escaped, e.g.
|
|
# %%systemd_user_post, because rpm expands macros even here). The user preset
|
|
# enables the unit for all users; instances start with each user's session.
|
|
%pre
|
|
# Releases up to 0.20.x shipped a *system* unit: stop and disable it while its
|
|
# file is still installed, and remember to apply the user preset in %%post.
|
|
if [ $1 -ge 2 ] && [ -f /usr/lib/systemd/system/yams-daemon.service ]; then
|
|
systemctl --no-reload disable --now yams-daemon.service >/dev/null 2>&1 || :
|
|
mkdir -p /var/lib/rpm-state/yams && touch /var/lib/rpm-state/yams/migrate-user-unit || :
|
|
fi
|
|
|
|
%post
|
|
# Run `systemctl --user <args>` in every running user manager (no-op without
|
|
# systemd, runuser or logged-in users). Used for daemon-reload, try-restart and
|
|
# stop; the scripts never start per-user instances.
|
|
yams_user_managers() {
|
|
command -v runuser >/dev/null 2>&1 || return 0
|
|
command -v systemctl >/dev/null 2>&1 || return 0
|
|
for d in /run/user/*; do
|
|
# A running user manager always has its private socket; the D-Bus user
|
|
# bus is optional (Debian/Ubuntu without dbus-user-session).
|
|
[ -S "$d/systemd/private" ] || continue
|
|
uid="${d##*/}"
|
|
user="$(getent passwd "$uid" | cut -d: -f1)"
|
|
[ -n "$user" ] || continue
|
|
runuser -u "$user" -- env XDG_RUNTIME_DIR="$d" \
|
|
systemctl --user "$@" >/dev/null 2>&1 || true
|
|
done
|
|
}
|
|
if [ $1 -eq 1 ] || [ -e /var/lib/rpm-state/yams/migrate-user-unit ]; then
|
|
systemctl --global preset yams-daemon.service >/dev/null 2>&1 || :
|
|
rm -rf /var/lib/rpm-state/yams
|
|
fi
|
|
yams_user_managers daemon-reload
|
|
if [ $1 -ge 2 ]; then
|
|
yams_user_managers try-restart yams-daemon.service
|
|
fi
|
|
|
|
%preun
|
|
# Run `systemctl --user <args>` in every running user manager (no-op without
|
|
# systemd, runuser or logged-in users). Used for daemon-reload, try-restart and
|
|
# stop; the scripts never start per-user instances.
|
|
yams_user_managers() {
|
|
command -v runuser >/dev/null 2>&1 || return 0
|
|
command -v systemctl >/dev/null 2>&1 || return 0
|
|
for d in /run/user/*; do
|
|
# A running user manager always has its private socket; the D-Bus user
|
|
# bus is optional (Debian/Ubuntu without dbus-user-session).
|
|
[ -S "$d/systemd/private" ] || continue
|
|
uid="${d##*/}"
|
|
user="$(getent passwd "$uid" | cut -d: -f1)"
|
|
[ -n "$user" ] || continue
|
|
runuser -u "$user" -- env XDG_RUNTIME_DIR="$d" \
|
|
systemctl --user "$@" >/dev/null 2>&1 || true
|
|
done
|
|
}
|
|
if [ $1 -eq 0 ]; then
|
|
yams_user_managers stop yams-daemon.service
|
|
systemctl --global disable yams-daemon.service >/dev/null 2>&1 || :
|
|
fi
|
|
|
|
%postun
|
|
# Run `systemctl --user <args>` in every running user manager (no-op without
|
|
# systemd, runuser or logged-in users). Used for daemon-reload, try-restart and
|
|
# stop; the scripts never start per-user instances.
|
|
yams_user_managers() {
|
|
command -v runuser >/dev/null 2>&1 || return 0
|
|
command -v systemctl >/dev/null 2>&1 || return 0
|
|
for d in /run/user/*; do
|
|
# A running user manager always has its private socket; the D-Bus user
|
|
# bus is optional (Debian/Ubuntu without dbus-user-session).
|
|
[ -S "$d/systemd/private" ] || continue
|
|
uid="${d##*/}"
|
|
user="$(getent passwd "$uid" | cut -d: -f1)"
|
|
[ -n "$user" ] || continue
|
|
runuser -u "$user" -- env XDG_RUNTIME_DIR="$d" \
|
|
systemctl --user "$@" >/dev/null 2>&1 || true
|
|
done
|
|
}
|
|
systemctl daemon-reload >/dev/null 2>&1 || :
|
|
yams_user_managers daemon-reload
|
|
|
|
%files -f %{_sourcedir}/filelist
|
|
%defattr(-,root,root,-)
|
|
__RPM_SPEC__
|
|
echo "DEBUG: version=$version, rpm_version=$rpm_version, rpm_release=$rpm_release" >&2
|
|
sed -i "s|__VERSION__|${rpm_version}|" "${spec}"
|
|
sed -i "s|__RELEASE__|${rpm_release}|" "${spec}"
|
|
sed -i "s|__RPM_ARCH__|${rpm_arch}|" "${spec}"
|
|
|
|
rpmbuild --define "_topdir ${rpm_root}" -bb "${spec}"
|
|
local built_rpm
|
|
built_rpm="$(find "${rpm_root}/RPMS" -type f -name 'yams-*.rpm' -print -quit)"
|
|
if [ -n "${built_rpm}" ]; then
|
|
local rpm_name="yams-${version}-linux-$(normalize_arch_label "${rpm_arch}")".rpm
|
|
cp -f "${built_rpm}" "${build_dir}/${rpm_name}"
|
|
echo "Created RPM package: ${build_dir}/${rpm_name}"
|
|
else
|
|
echo "rpmbuild completed but RPM artifact not found" >&2
|
|
fi
|
|
rm -rf "${rpm_root}"
|
|
}
|
|
|
|
package_all() {
|
|
local version="$1"
|
|
local build_dir="$2"
|
|
local stage_dir="$3"
|
|
local force="${4:-0}"
|
|
|
|
# Package if: force=1, or tag build, or nightly/weekly channel
|
|
if [[ "${force}" != "1" && "${GIT_REF:-}" != refs/tags/* ]]; then
|
|
if [[ "${RELEASE_CHANNEL:-}" != "nightly" && "${RELEASE_CHANNEL:-}" != "weekly" ]]; then
|
|
echo "Skipping packaging (not a tag build and no release channel): GIT_REF=${GIT_REF:-n/a} RELEASE_CHANNEL=${RELEASE_CHANNEL:-n/a}"
|
|
return
|
|
fi
|
|
fi
|
|
|
|
if [[ "${build_dir}" != /* ]]; then
|
|
build_dir="${REPO_ROOT}/${build_dir}"
|
|
fi
|
|
build_dir="$(readlink -f "${build_dir}")"
|
|
local stage_root="${build_dir}/${stage_dir}"
|
|
if [ ! -d "${stage_root}" ]; then
|
|
echo "Stage directory ${stage_root} not found" >&2
|
|
exit 1
|
|
fi
|
|
|
|
if [ -d "${stage_root}/usr" ]; then
|
|
bash "${REPO_ROOT}/scripts/split-debug-symbols.sh" "${stage_root}/usr"
|
|
fi
|
|
|
|
prepare_stage_docs "${stage_root}"
|
|
package_tarball "${version}" "${build_dir}" "${stage_root}"
|
|
package_deb "${version}" "${build_dir}" "${stage_root}"
|
|
package_rpm "${version}" "${build_dir}" "${stage_root}"
|
|
}
|
|
|
|
cleanup_apt() {
|
|
# Use sudo if not running as root
|
|
if [ "$(id -u)" -eq 0 ]; then
|
|
apt-get clean || true
|
|
rm -rf /var/lib/apt/lists/* || true
|
|
elif command -v sudo >/dev/null 2>&1; then
|
|
sudo apt-get clean || true
|
|
sudo rm -rf /var/lib/apt/lists/* 2>/dev/null || true
|
|
else
|
|
echo "Skipping apt-get clean (requires root privileges)" >&2
|
|
fi
|
|
}
|
|
|
|
package_only_main() {
|
|
shift
|
|
if [ "$#" -lt 2 ] || [ "$#" -gt 3 ]; then
|
|
echo "Usage: $0 package_only <version> <build_dir> [stage_dir]" >&2
|
|
exit 1
|
|
fi
|
|
local version="$1"
|
|
local build_dir="$2"
|
|
local stage_dir="${3:-stage}"
|
|
package_all "${version}" "${build_dir}" "${stage_dir}" 1
|
|
exit 0
|
|
}
|
|
|
|
provision_main() {
|
|
# Install only the build toolchain + Conan, no build. Used to pre-bake a cached
|
|
# Docker builder image so the package list stays single-sourced in this script.
|
|
persist_env_defaults
|
|
ensure_base_packages
|
|
install_conan
|
|
echo "Provisioning complete."
|
|
exit 0
|
|
}
|
|
|
|
main() {
|
|
if [ "${1:-}" = "package_only" ]; then
|
|
package_only_main "$@"
|
|
fi
|
|
if [ "${1:-}" = "provision" ]; then
|
|
provision_main
|
|
fi
|
|
|
|
ensure_base_packages
|
|
persist_env_defaults
|
|
install_conan
|
|
export_custom_conan_recipes
|
|
compute_version
|
|
configure_conan_profile
|
|
run_conan_install
|
|
run_meson_build
|
|
package_all "${CLEAN_VERSION}" "${BUILD_DIR}" "${STAGE_DIR}" 0
|
|
cleanup_apt
|
|
echo "All tasks finished."
|
|
}
|
|
|
|
main "$@"
|