5eeb3b8239
The #206 revert restored the pre-September upload code, which passed --force, while the workflow still pins wrangler 4.69.0. That version's 'r2 object put' has no --force option (it overwrites without prompting) and exits with 'Unknown argument: force', so the v0.20.3 release failed its first R2 upload and no apt, yum or Arch repository was published. Immutable objects are still checked with 'get' before any write. Signed-off-by: trvon <me@trevon.dev>
390 lines
13 KiB
Python
Executable file
390 lines
13 KiB
Python
Executable file
#!/usr/bin/env python3
|
|
"""Publish one validated package-repository channel to R2 in fail-closed order."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
from typing import Protocol
|
|
|
|
RELEASE_CHANNELS = frozenset({"stable", "nightly", "weekly"})
|
|
PACKAGE_SUFFIXES = (".deb", ".rpm", ".pkg.tar.zst")
|
|
REPOSITORY_DIRS = ("aptrepo", "yumrepo", "archrepo")
|
|
MISSING_OBJECT_RE = re.compile(
|
|
r"(?:404|object[^\n]*not found|key[^\n]*does not exist|specified key does not exist)",
|
|
re.IGNORECASE,
|
|
)
|
|
|
|
|
|
class PublicationError(RuntimeError):
|
|
"""A publication precondition or storage operation failed."""
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class PublicationObject:
|
|
source: Path
|
|
key: str
|
|
immutable: bool
|
|
phase: str
|
|
content_type: str
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class ValidatedManifest:
|
|
release_channel: str
|
|
publication_channel: str
|
|
release_run_id: int
|
|
release_run_attempt: int
|
|
|
|
|
|
class ObjectStore(Protocol):
|
|
def get(self, key: str) -> bytes | None:
|
|
raise NotImplementedError
|
|
|
|
def put(self, key: str, source: Path, content_type: str) -> None:
|
|
raise NotImplementedError
|
|
|
|
|
|
def sha256_bytes(data: bytes) -> str:
|
|
return hashlib.sha256(data).hexdigest()
|
|
|
|
|
|
def sha256_file(path: Path) -> str:
|
|
digest = hashlib.sha256()
|
|
with path.open("rb") as stream:
|
|
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
|
|
digest.update(chunk)
|
|
return digest.hexdigest()
|
|
|
|
|
|
def content_type(key: str) -> str:
|
|
suffixes = (
|
|
(".json", "application/json"),
|
|
(".key", "application/pgp-keys"),
|
|
(".txt", "text/plain"),
|
|
(".xml", "application/xml"),
|
|
(".gz", "application/gzip"),
|
|
(".deb", "application/vnd.debian.binary-package"),
|
|
(".rpm", "application/x-rpm"),
|
|
(".pkg.tar.zst", "application/zstd"),
|
|
(".db", "application/gzip"),
|
|
(".files", "application/gzip"),
|
|
(".asc", "application/pgp-signature"),
|
|
)
|
|
for suffix, value in suffixes:
|
|
if key.endswith(suffix):
|
|
return value
|
|
return "application/octet-stream"
|
|
|
|
|
|
def validate_manifest(
|
|
manifest_path: Path,
|
|
expected_channel: str,
|
|
selected_run_id: int,
|
|
selected_run_attempt: int,
|
|
) -> ValidatedManifest:
|
|
if expected_channel not in {"stable", "experimental"}:
|
|
raise PublicationError("expected channel must be stable or experimental")
|
|
try:
|
|
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
|
except FileNotFoundError as error:
|
|
raise PublicationError(f"missing release manifest: {manifest_path}") from error
|
|
except json.JSONDecodeError as error:
|
|
raise PublicationError(f"invalid release manifest JSON: {error}") from error
|
|
if not isinstance(manifest, dict):
|
|
raise PublicationError("release manifest must be a JSON object")
|
|
|
|
release_channel = manifest.get("channel")
|
|
if release_channel not in RELEASE_CHANNELS:
|
|
raise PublicationError("release manifest has missing or invalid channel")
|
|
publication_channel = "stable" if release_channel == "stable" else "experimental"
|
|
if publication_channel != expected_channel:
|
|
raise PublicationError(
|
|
f"release manifest channel {release_channel!r} does not match "
|
|
f"requested {expected_channel!r} publication"
|
|
)
|
|
|
|
run_id = manifest.get("release_workflow_run_id")
|
|
if isinstance(run_id, bool) or not isinstance(run_id, int) or run_id <= 0:
|
|
raise PublicationError(
|
|
"release manifest has missing or invalid workflow run ID"
|
|
)
|
|
if run_id != selected_run_id:
|
|
raise PublicationError(
|
|
f"release manifest run ID {run_id} does not match selected run {selected_run_id}"
|
|
)
|
|
run_attempt = manifest.get("release_workflow_run_attempt")
|
|
if (
|
|
isinstance(run_attempt, bool)
|
|
or not isinstance(run_attempt, int)
|
|
or run_attempt <= 0
|
|
):
|
|
raise PublicationError(
|
|
"release manifest has missing or invalid workflow run attempt"
|
|
)
|
|
if run_attempt != selected_run_attempt:
|
|
raise PublicationError(
|
|
f"release manifest run attempt {run_attempt} does not match selected "
|
|
f"attempt {selected_run_attempt}"
|
|
)
|
|
|
|
source_sha = manifest.get("source_sha")
|
|
source_ref = manifest.get("source_ref")
|
|
if (
|
|
not isinstance(source_sha, str)
|
|
or re.fullmatch(r"[0-9a-f]{40}", source_sha) is None
|
|
):
|
|
raise PublicationError("release manifest has invalid source SHA")
|
|
if not isinstance(source_ref, str):
|
|
raise PublicationError("release manifest has missing source ref")
|
|
if (
|
|
publication_channel == "experimental"
|
|
and source_ref != "refs/heads/experimental"
|
|
):
|
|
raise PublicationError(
|
|
"experimental manifest must come from refs/heads/experimental"
|
|
)
|
|
if publication_channel == "stable" and not source_ref.startswith("refs/tags/"):
|
|
raise PublicationError("stable manifest must come from a tag ref")
|
|
|
|
return ValidatedManifest(release_channel, publication_channel, run_id, run_attempt)
|
|
|
|
|
|
def _safe_relative_files(root: Path) -> list[Path]:
|
|
files: list[Path] = []
|
|
if not root.is_dir():
|
|
return files
|
|
resolved_root = root.resolve()
|
|
for path in root.rglob("*"):
|
|
if path.is_symlink():
|
|
resolved = path.resolve()
|
|
if resolved_root not in resolved.parents or not resolved.is_file():
|
|
raise PublicationError(
|
|
f"refusing unsafe symlink in repository artifact: {path}"
|
|
)
|
|
if path.is_file():
|
|
files.append(path)
|
|
return sorted(files)
|
|
|
|
|
|
def build_publication_plan(
|
|
repo_root: Path,
|
|
manifest_path: Path,
|
|
expected_channel: str,
|
|
selected_run_id: int,
|
|
selected_run_attempt: int,
|
|
public_key_path: Path | None = None,
|
|
) -> list[PublicationObject]:
|
|
validated = validate_manifest(
|
|
manifest_path, expected_channel, selected_run_id, selected_run_attempt
|
|
)
|
|
prefix = "" if validated.publication_channel == "stable" else "experimental/"
|
|
objects: list[PublicationObject] = []
|
|
|
|
for directory in REPOSITORY_DIRS:
|
|
root = repo_root / directory
|
|
for path in _safe_relative_files(root):
|
|
relative = path.relative_to(root).as_posix()
|
|
key = f"{prefix}{directory}/{relative}"
|
|
is_package = key.endswith(PACKAGE_SUFFIXES)
|
|
objects.append(
|
|
PublicationObject(
|
|
source=path,
|
|
key=key,
|
|
immutable=is_package,
|
|
phase="payload" if is_package else "metadata",
|
|
content_type=content_type(key),
|
|
)
|
|
)
|
|
|
|
if not objects:
|
|
raise PublicationError("no package repository objects were found")
|
|
|
|
if public_key_path is not None:
|
|
if not public_key_path.is_file():
|
|
raise PublicationError(f"missing public signing key: {public_key_path}")
|
|
key = (
|
|
"gpg.key"
|
|
if validated.publication_channel == "stable"
|
|
else ("experimental/aptrepo/gpg.key")
|
|
)
|
|
objects.append(
|
|
PublicationObject(
|
|
source=public_key_path,
|
|
key=key,
|
|
immutable=False,
|
|
phase="metadata",
|
|
content_type=content_type(key),
|
|
)
|
|
)
|
|
|
|
latest_key = (
|
|
"latest.json"
|
|
if validated.publication_channel == "stable"
|
|
else ("experimental/latest.json")
|
|
)
|
|
objects.append(
|
|
PublicationObject(
|
|
source=manifest_path,
|
|
key=latest_key,
|
|
immutable=False,
|
|
phase="latest",
|
|
content_type="application/json",
|
|
)
|
|
)
|
|
|
|
phase_order = {"payload": 0, "metadata": 1, "latest": 2}
|
|
return sorted(objects, key=lambda item: (phase_order[item.phase], item.key))
|
|
|
|
|
|
def publish(plan: list[PublicationObject], store: ObjectStore) -> None:
|
|
latest_seen = False
|
|
for item in plan:
|
|
if latest_seen:
|
|
raise PublicationError(
|
|
"publication plan contains objects after latest.json"
|
|
)
|
|
if item.phase == "latest":
|
|
latest_seen = True
|
|
|
|
if item.immutable:
|
|
existing = store.get(item.key)
|
|
if existing is not None:
|
|
existing_digest = sha256_bytes(existing)
|
|
source_digest = sha256_file(item.source)
|
|
if existing_digest == source_digest:
|
|
print(f"immutable object already matches; skipping {item.key}")
|
|
continue
|
|
raise PublicationError(
|
|
f"immutable object conflict for {item.key}: "
|
|
f"existing sha256={existing_digest}, source sha256={source_digest}"
|
|
)
|
|
store.put(item.key, item.source, item.content_type)
|
|
|
|
if not latest_seen:
|
|
raise PublicationError("publication plan did not contain latest.json")
|
|
|
|
|
|
class WranglerObjectStore:
|
|
def __init__(self, wrangler: Path, bucket: str) -> None:
|
|
self.wrangler = wrangler
|
|
self.bucket = bucket
|
|
|
|
def _target(self, key: str) -> str:
|
|
return f"{self.bucket}/{key}"
|
|
|
|
def get(self, key: str) -> bytes | None:
|
|
with tempfile.NamedTemporaryFile(delete=False) as temporary:
|
|
output = Path(temporary.name)
|
|
try:
|
|
result = subprocess.run(
|
|
[
|
|
str(self.wrangler),
|
|
"r2",
|
|
"object",
|
|
"get",
|
|
self._target(key),
|
|
"--remote",
|
|
"--file",
|
|
str(output),
|
|
],
|
|
check=False,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
if result.returncode == 0:
|
|
return output.read_bytes()
|
|
diagnostic = f"{result.stdout}\n{result.stderr}".strip()
|
|
if MISSING_OBJECT_RE.search(diagnostic):
|
|
return None
|
|
raise PublicationError(
|
|
f"failed to check immutable object {key!r}; refusing overwrite: {diagnostic}"
|
|
)
|
|
finally:
|
|
output.unlink(missing_ok=True)
|
|
|
|
def put(self, key: str, source: Path, content_type: str) -> None:
|
|
print(f"uploading {key}", file=sys.stderr)
|
|
result = subprocess.run(
|
|
[
|
|
str(self.wrangler),
|
|
"r2",
|
|
"object",
|
|
"put",
|
|
self._target(key),
|
|
"--remote",
|
|
"--file",
|
|
str(source),
|
|
"--content-type",
|
|
content_type,
|
|
],
|
|
check=False,
|
|
)
|
|
if result.returncode != 0:
|
|
raise PublicationError(f"R2 upload failed for {key}")
|
|
|
|
|
|
def parse_args() -> argparse.Namespace:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--repo-root", type=Path, required=True)
|
|
parser.add_argument("--manifest", type=Path, required=True)
|
|
parser.add_argument(
|
|
"--expected-channel", choices=("stable", "experimental"), required=True
|
|
)
|
|
parser.add_argument("--selected-run-id", type=int, required=True)
|
|
parser.add_argument("--selected-run-attempt", type=int, required=True)
|
|
parser.add_argument("--bucket", default="yams-repository")
|
|
parser.add_argument("--wrangler", type=Path, required=True)
|
|
parser.add_argument("--public-key", type=Path)
|
|
parser.add_argument("--plan-only", action="store_true")
|
|
return parser.parse_args()
|
|
|
|
|
|
def main() -> int:
|
|
args = parse_args()
|
|
try:
|
|
plan = build_publication_plan(
|
|
args.repo_root,
|
|
args.manifest,
|
|
args.expected_channel,
|
|
args.selected_run_id,
|
|
args.selected_run_attempt,
|
|
args.public_key,
|
|
)
|
|
if args.plan_only:
|
|
print(
|
|
json.dumps(
|
|
[
|
|
{
|
|
"key": item.key,
|
|
"immutable": item.immutable,
|
|
"phase": item.phase,
|
|
"content_type": item.content_type,
|
|
}
|
|
for item in plan
|
|
],
|
|
indent=2,
|
|
)
|
|
)
|
|
return 0
|
|
if not os.environ.get("CLOUDFLARE_API_TOKEN") or not os.environ.get(
|
|
"CLOUDFLARE_ACCOUNT_ID"
|
|
):
|
|
raise PublicationError("Cloudflare API token/account missing")
|
|
publish(plan, WranglerObjectStore(args.wrangler, args.bucket))
|
|
except (OSError, PublicationError) as error:
|
|
print(f"repository publication error: {error}", file=sys.stderr)
|
|
return 2
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|