- fail closed on MCP input and compile database policy - add shared scan catalogs, finding queries, and tiered gates
828 lines
30 KiB
Python
828 lines
30 KiB
Python
#!/usr/bin/env python3
|
|
"""Run Fold's local code-quality gates and write one machine-readable report."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import importlib
|
|
import json
|
|
import os
|
|
import shlex
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from pathlib import Path
|
|
from typing import Protocol, cast
|
|
|
|
from benchmarks.fold_eval.token_efficiency import estimate_text_tokens
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
DEFAULT_MANIFEST = ROOT / "tests" / "benchmarks" / "projects" / "fold_source.json"
|
|
DEFAULT_TEST_IMPACT_MANIFEST = ROOT / "tests" / "quality" / "impacted_tests.json"
|
|
SOURCE_SUFFIXES = {".c", ".cc", ".cpp", ".cxx"}
|
|
HEADER_SUFFIXES = {".h", ".hh", ".hpp", ".hxx"}
|
|
MAX_EVIDENCE_CHARS = 1000
|
|
MAX_EVIDENCE_LINES = 5
|
|
MAX_FINDING_DETAILS = 20
|
|
EXCLUDED_PATH_PARTS = {
|
|
"build",
|
|
"generated",
|
|
"subprojects",
|
|
"third_party",
|
|
"vendor",
|
|
"vendored",
|
|
"external",
|
|
}
|
|
|
|
|
|
def positive_integer(value: str) -> int:
|
|
try:
|
|
parsed = int(value)
|
|
except ValueError as error:
|
|
raise argparse.ArgumentTypeError("must be an integer") from error
|
|
if parsed < 1:
|
|
raise argparse.ArgumentTypeError("must be at least 1")
|
|
return parsed
|
|
|
|
|
|
class FlakyEvidenceModule(Protocol):
|
|
def parse_meson_testlog(
|
|
self, path: Path, attempt: int, exit_code: int
|
|
) -> dict[str, object]: ...
|
|
|
|
def summarize_attempts(
|
|
self,
|
|
attempts: list[dict[str, object]],
|
|
*,
|
|
expected_tests: list[str] | None = None,
|
|
) -> dict[str, object]: ...
|
|
|
|
|
|
def flaky_evidence_module() -> FlakyEvidenceModule:
|
|
module = importlib.import_module("benchmarks.fold_eval.flaky_evidence")
|
|
return cast(FlakyEvidenceModule, module)
|
|
|
|
|
|
def parse_test_attempt(path: Path, attempt: int, exit_code: int) -> dict[str, object]:
|
|
parser = flaky_evidence_module().parse_meson_testlog
|
|
result = parser(path, attempt, exit_code)
|
|
if not isinstance(result, dict):
|
|
raise ValueError("test evidence parser returned a non-object result")
|
|
return result
|
|
|
|
|
|
def summarize_test_attempts(
|
|
attempts: list[dict[str, object]], expected_tests: list[str] | None
|
|
) -> dict[str, object]:
|
|
summarizer = flaky_evidence_module().summarize_attempts
|
|
result = summarizer(attempts, expected_tests=expected_tests)
|
|
if not isinstance(result, dict):
|
|
raise ValueError("test evidence summarizer returned a non-object result")
|
|
return result
|
|
|
|
|
|
def select_impacted_tests(changed: list[str], manifest: Path) -> dict[str, object]:
|
|
module = importlib.import_module("benchmarks.fold_eval.test_selection")
|
|
selector = module.select_tests
|
|
result = selector(changed, manifest)
|
|
if not isinstance(result, dict):
|
|
raise ValueError("test selector returned a non-object result")
|
|
return result
|
|
|
|
|
|
def apply_full_test_override(selection: dict[str, object], force_full: bool) -> None:
|
|
if not force_full:
|
|
return
|
|
selection["full_suite"] = True
|
|
reasons_value = selection.get("fallback_reasons", [])
|
|
reasons = reasons_value if isinstance(reasons_value, list) else []
|
|
reasons.append("complete suite requested explicitly")
|
|
selection["fallback_reasons"] = reasons
|
|
|
|
|
|
def compact_evidence(output: str) -> list[str]:
|
|
lines = [line for line in output.splitlines() if line.strip()]
|
|
selected: list[str] = []
|
|
remaining = MAX_EVIDENCE_CHARS
|
|
for line in reversed(lines[-MAX_EVIDENCE_LINES:]):
|
|
if remaining <= 0:
|
|
break
|
|
value = line[-remaining:]
|
|
selected.append(value)
|
|
remaining -= len(value) + (1 if selected else 0)
|
|
selected.reverse()
|
|
return selected
|
|
|
|
|
|
def text_artifact(path: Path, root: Path, kind: str) -> dict[str, object]:
|
|
text = path.read_text(encoding="utf-8", errors="replace")
|
|
return {
|
|
"kind": kind,
|
|
"path": path.resolve().relative_to(root.resolve()).as_posix(),
|
|
"bytes": len(text.encode("utf-8")),
|
|
"estimated_tokens": estimate_text_tokens(text),
|
|
}
|
|
|
|
|
|
def run_step(
|
|
name: str,
|
|
command: list[str],
|
|
*,
|
|
artifact_dir: Path,
|
|
acceptable: set[int] | None = None,
|
|
) -> dict[str, object]:
|
|
started = time.monotonic()
|
|
proc = subprocess.run(command, cwd=ROOT, text=True, capture_output=True)
|
|
accepted = {0} if acceptable is None else acceptable
|
|
output = proc.stdout + proc.stderr
|
|
log_path = artifact_dir / "logs" / f"{name}.log"
|
|
log_path.parent.mkdir(parents=True, exist_ok=True)
|
|
log_path.write_text(output, encoding="utf-8")
|
|
return {
|
|
"name": name,
|
|
"command": command,
|
|
"exit_code": proc.returncode,
|
|
"passed": proc.returncode in accepted,
|
|
"duration_seconds": round(time.monotonic() - started, 3),
|
|
"evidence": compact_evidence(output),
|
|
"artifacts": [text_artifact(log_path, artifact_dir, "log")],
|
|
}
|
|
|
|
|
|
def summarize_policy_report(summary_path: Path, output_root: Path) -> dict[str, object]:
|
|
try:
|
|
summary = json.loads(summary_path.read_text(encoding="utf-8"))
|
|
except (OSError, json.JSONDecodeError) as error:
|
|
return {"policy_summary_error": str(error), "artifacts": []}
|
|
change_time = summary.get("change_time", {})
|
|
if not isinstance(change_time, dict):
|
|
change_time = {}
|
|
counts = change_time.get("counts", {})
|
|
if not isinstance(counts, dict):
|
|
counts = {}
|
|
new = change_time.get("new", [])
|
|
resolved = change_time.get("resolved", [])
|
|
new = new if isinstance(new, list) else []
|
|
resolved = resolved if isinstance(resolved, list) else []
|
|
artifacts = []
|
|
candidates = [summary_path, summary_path.with_suffix(".md")]
|
|
for value in summary.get("run_artifacts", []):
|
|
if isinstance(value, str):
|
|
candidates.append(Path(value))
|
|
for path in candidates:
|
|
if path.is_file():
|
|
try:
|
|
artifacts.append(text_artifact(path, output_root, "policy-report"))
|
|
except ValueError:
|
|
continue
|
|
failures = summary.get("reliability", {}).get("failures", [])
|
|
failures = failures if isinstance(failures, list) else []
|
|
return {
|
|
"policy_status": str(summary.get("status", "unknown")),
|
|
"finding_delta": {
|
|
"counts": {
|
|
"new": counts.get("new", len(new)),
|
|
"resolved": counts.get("resolved", len(resolved)),
|
|
},
|
|
"new": new[:MAX_FINDING_DETAILS],
|
|
"resolved": resolved[:MAX_FINDING_DETAILS],
|
|
"truncated": {
|
|
"new": max(0, len(new) - MAX_FINDING_DETAILS),
|
|
"resolved": max(0, len(resolved) - MAX_FINDING_DETAILS),
|
|
},
|
|
},
|
|
"failures": [str(value) for value in failures[:MAX_EVIDENCE_LINES]],
|
|
"artifacts": artifacts,
|
|
}
|
|
|
|
|
|
def finalize_report(steps: list[dict[str, object]]) -> dict[str, object]:
|
|
artifacts: list[dict[str, object]] = []
|
|
for step in steps:
|
|
value = step.get("artifacts", [])
|
|
if isinstance(value, list):
|
|
artifacts.extend(item for item in value if isinstance(item, dict))
|
|
report: dict[str, object] = {
|
|
"schema_version": 2,
|
|
"status": "ok" if all(step.get("passed", True) for step in steps) else "failed",
|
|
"steps": steps,
|
|
"artifacts": sorted(artifacts, key=lambda item: str(item.get("path", ""))),
|
|
}
|
|
compact_payload = json.dumps(report, indent=2, sort_keys=True) + "\n"
|
|
artifact_bytes = 0
|
|
artifact_tokens = 0
|
|
for item in artifacts:
|
|
byte_count = item.get("bytes")
|
|
token_count = item.get("estimated_tokens")
|
|
if isinstance(byte_count, int) and not isinstance(byte_count, bool):
|
|
artifact_bytes += byte_count
|
|
if isinstance(token_count, int) and not isinstance(token_count, bool):
|
|
artifact_tokens += token_count
|
|
report["token_efficiency"] = {
|
|
"estimator": "ceil(UTF-8 bytes / 4)",
|
|
"compact_payload_bytes": len(compact_payload.encode("utf-8")),
|
|
"compact_payload_estimated_tokens": estimate_text_tokens(compact_payload),
|
|
"artifact_bytes": artifact_bytes,
|
|
"artifact_estimated_tokens": artifact_tokens,
|
|
}
|
|
return report
|
|
|
|
|
|
def git_lines(root: Path, args: list[str]) -> list[str]:
|
|
proc = subprocess.run(
|
|
["git", *args],
|
|
cwd=root,
|
|
text=True,
|
|
capture_output=True,
|
|
)
|
|
if proc.returncode != 0:
|
|
detail = proc.stderr.strip() or proc.stdout.strip() or f"exit {proc.returncode}"
|
|
raise ValueError(f"git {' '.join(args)} failed: {detail}")
|
|
return proc.stdout.splitlines()
|
|
|
|
|
|
def git_paths(root: Path, args: list[str]) -> list[str]:
|
|
proc = subprocess.run(
|
|
["git", *args],
|
|
cwd=root,
|
|
capture_output=True,
|
|
)
|
|
if proc.returncode != 0:
|
|
stderr = os.fsdecode(proc.stderr).strip()
|
|
stdout = os.fsdecode(proc.stdout).strip()
|
|
detail = stderr or stdout or f"exit {proc.returncode}"
|
|
raise ValueError(f"git {' '.join(args)} failed: {detail}")
|
|
return [os.fsdecode(path) for path in proc.stdout.split(b"\0") if path]
|
|
|
|
|
|
def changed_paths(
|
|
*, root: Path = ROOT, base_ref: str | None = None, head_ref: str | None = None
|
|
) -> list[str]:
|
|
if bool(base_ref) != bool(head_ref):
|
|
raise ValueError("base_ref and head_ref must be provided together")
|
|
|
|
if base_ref and head_ref:
|
|
merge_base = git_lines(root, ["merge-base", base_ref, head_ref])
|
|
if len(merge_base) != 1 or not merge_base[0]:
|
|
raise ValueError(
|
|
f"git merge-base returned no revision for {base_ref} and {head_ref}"
|
|
)
|
|
changed = git_paths(
|
|
root,
|
|
[
|
|
"diff",
|
|
"--name-only",
|
|
"-z",
|
|
"--diff-filter=ACMR",
|
|
merge_base[0],
|
|
head_ref,
|
|
],
|
|
)
|
|
else:
|
|
changed = git_paths(
|
|
root, ["diff", "--name-only", "-z", "--diff-filter=ACMR", "HEAD"]
|
|
)
|
|
changed.extend(
|
|
git_paths(root, ["ls-files", "--others", "--exclude-standard", "-z"])
|
|
)
|
|
return sorted(set(changed))
|
|
|
|
|
|
def changed_test_paths(
|
|
*, root: Path = ROOT, base_ref: str | None = None, head_ref: str | None = None
|
|
) -> list[str]:
|
|
if bool(base_ref) != bool(head_ref):
|
|
raise ValueError("base_ref and head_ref must be provided together")
|
|
if base_ref and head_ref:
|
|
merge_base = git_lines(root, ["merge-base", base_ref, head_ref])
|
|
if len(merge_base) != 1 or not merge_base[0]:
|
|
raise ValueError(
|
|
f"git merge-base returned no revision for {base_ref} and {head_ref}"
|
|
)
|
|
changed = git_lines(
|
|
root,
|
|
["diff", "--name-only", "--diff-filter=ACMRD", merge_base[0], head_ref],
|
|
)
|
|
else:
|
|
changed = git_lines(
|
|
root, ["diff", "--name-only", "--diff-filter=ACMRD", "HEAD"]
|
|
)
|
|
changed.extend(git_lines(root, ["ls-files", "--others", "--exclude-standard"]))
|
|
return sorted(set(changed))
|
|
|
|
|
|
def excluded_project_path(path: Path) -> bool:
|
|
return any(part in EXCLUDED_PATH_PARTS for part in path.parts)
|
|
|
|
|
|
def compile_db_file(path: Path, root: Path = ROOT) -> Path:
|
|
resolved = path if path.is_absolute() else root / path
|
|
return resolved / "compile_commands.json" if resolved.is_dir() else resolved
|
|
|
|
|
|
def compile_db_exists(path: Path, root: Path = ROOT) -> bool:
|
|
return compile_db_file(path, root).is_file()
|
|
|
|
|
|
def load_compile_commands(path: Path, root: Path) -> list[dict[str, object]]:
|
|
database = compile_db_file(path, root)
|
|
try:
|
|
payload = json.loads(database.read_text(encoding="utf-8"))
|
|
except (OSError, json.JSONDecodeError) as error:
|
|
raise ValueError(
|
|
f"failed to read compile database {database}: {error}"
|
|
) from error
|
|
if not isinstance(payload, list):
|
|
raise ValueError(f"compile database must contain a JSON array: {database}")
|
|
return [entry for entry in payload if isinstance(entry, dict)]
|
|
|
|
|
|
def entry_arguments(entry: dict[str, object]) -> list[str]:
|
|
arguments = entry.get("arguments")
|
|
if isinstance(arguments, list) and all(
|
|
isinstance(value, str) for value in arguments
|
|
):
|
|
return arguments
|
|
command = entry.get("command")
|
|
if not isinstance(command, str):
|
|
return []
|
|
try:
|
|
return shlex.split(command)
|
|
except ValueError:
|
|
return []
|
|
|
|
|
|
def entry_depfile(entry: dict[str, object], directory: Path) -> Path | None:
|
|
arguments = entry_arguments(entry)
|
|
for index, argument in enumerate(arguments):
|
|
value: str | None = None
|
|
if argument == "-MF" and index + 1 < len(arguments):
|
|
value = arguments[index + 1]
|
|
elif argument.startswith("-MF") and len(argument) > 3:
|
|
value = argument[3:]
|
|
if value:
|
|
path = Path(value)
|
|
return path if path.is_absolute() else directory / path
|
|
return None
|
|
|
|
|
|
def depfile_dependencies(path: Path, directory: Path) -> set[Path] | None:
|
|
try:
|
|
content = path.read_text(encoding="utf-8").replace("\\\n", " ")
|
|
except OSError:
|
|
return None
|
|
_, separator, dependencies = content.partition(":")
|
|
if not separator:
|
|
return None
|
|
try:
|
|
values = shlex.split(dependencies)
|
|
except ValueError:
|
|
return None
|
|
return {
|
|
(candidate if candidate.is_absolute() else directory / candidate).resolve()
|
|
for value in values
|
|
if (candidate := Path(value))
|
|
}
|
|
|
|
|
|
def parse_ninja_dependencies(output: str, build_dir: Path) -> dict[Path, set[Path]]:
|
|
dependencies: dict[Path, set[Path]] = {}
|
|
target: Path | None = None
|
|
for line in output.splitlines():
|
|
if line and not line[0].isspace():
|
|
name, separator, _ = line.partition(": #deps ")
|
|
target = (build_dir / name).resolve() if separator else None
|
|
if target is not None:
|
|
dependencies[target] = set()
|
|
elif target is not None and line.strip():
|
|
dependency = Path(line.strip())
|
|
resolved = (
|
|
dependency if dependency.is_absolute() else build_dir / dependency
|
|
).resolve()
|
|
dependencies[target].add(resolved)
|
|
return dependencies
|
|
|
|
|
|
def load_ninja_dependencies(build_dir: Path) -> dict[Path, set[Path]]:
|
|
if not (build_dir / "build.ninja").is_file():
|
|
return {}
|
|
try:
|
|
proc = subprocess.run(
|
|
["ninja", "-C", str(build_dir), "-t", "deps"],
|
|
text=True,
|
|
capture_output=True,
|
|
)
|
|
except OSError:
|
|
return {}
|
|
if proc.returncode != 0:
|
|
return {}
|
|
return parse_ninja_dependencies(proc.stdout, build_dir)
|
|
|
|
|
|
def entry_output(entry: dict[str, object], directory: Path) -> Path | None:
|
|
value = entry.get("output")
|
|
if not isinstance(value, str):
|
|
return None
|
|
path = Path(value)
|
|
return (path if path.is_absolute() else directory / path).resolve()
|
|
|
|
|
|
def impacted_translation_units(
|
|
headers: list[str], compile_db: Path, root: Path
|
|
) -> list[str]:
|
|
changed_headers = {(root / header).resolve() for header in headers}
|
|
impacted: set[str] = set()
|
|
root_resolved = root.resolve()
|
|
database = compile_db_file(compile_db, root)
|
|
ninja_dependencies: dict[Path, set[Path]] | None = None
|
|
for entry in load_compile_commands(database, root):
|
|
directory_value = entry.get("directory")
|
|
file_value = entry.get("file")
|
|
if not isinstance(directory_value, str) or not isinstance(file_value, str):
|
|
continue
|
|
directory = Path(directory_value)
|
|
source = Path(file_value)
|
|
source = source if source.is_absolute() else directory / source
|
|
try:
|
|
relative_source = source.resolve().relative_to(root_resolved)
|
|
except ValueError:
|
|
continue
|
|
if (
|
|
source.suffix not in SOURCE_SUFFIXES
|
|
or excluded_project_path(relative_source)
|
|
or not source.is_file()
|
|
):
|
|
continue
|
|
depfile = entry_depfile(entry, directory)
|
|
dependencies = (
|
|
depfile_dependencies(depfile, directory) if depfile is not None else None
|
|
)
|
|
if dependencies is None:
|
|
if ninja_dependencies is None:
|
|
ninja_dependencies = load_ninja_dependencies(database.parent)
|
|
output = entry_output(entry, directory)
|
|
dependencies = (
|
|
ninja_dependencies.get(output) if output is not None else None
|
|
)
|
|
if dependencies is None or changed_headers.intersection(dependencies):
|
|
impacted.add(relative_source.as_posix())
|
|
return sorted(impacted)
|
|
|
|
|
|
def changed_source_files(
|
|
*,
|
|
root: Path = ROOT,
|
|
base_ref: str | None = None,
|
|
head_ref: str | None = None,
|
|
compile_db: Path | None = None,
|
|
) -> list[str]:
|
|
changed = changed_paths(root=root, base_ref=base_ref, head_ref=head_ref)
|
|
sources = {
|
|
path
|
|
for path in changed
|
|
if Path(path).suffix in SOURCE_SUFFIXES
|
|
and not excluded_project_path(Path(path))
|
|
and (root / path).is_file()
|
|
}
|
|
headers = [
|
|
path
|
|
for path in changed
|
|
if Path(path).suffix in HEADER_SUFFIXES
|
|
and not excluded_project_path(Path(path))
|
|
and (root / path).is_file()
|
|
]
|
|
if headers and compile_db is not None:
|
|
sources.update(impacted_translation_units(headers, compile_db, root))
|
|
return sorted(sources)
|
|
|
|
|
|
def diff_scan_precondition_errors(
|
|
*,
|
|
ci: bool,
|
|
skip_diff_scan: bool,
|
|
base_ref: str | None,
|
|
head_ref: str | None,
|
|
compile_db: Path | None,
|
|
) -> list[str]:
|
|
if skip_diff_scan:
|
|
return []
|
|
errors: list[str] = []
|
|
if bool(base_ref) != bool(head_ref):
|
|
errors.append("--base-ref and --head-ref must be provided together")
|
|
elif ci and not (base_ref and head_ref):
|
|
errors.append("CI differential scan requires base/head refs")
|
|
if ci and compile_db is None:
|
|
errors.append("CI differential scan requires a compile database")
|
|
elif compile_db is not None and not compile_db_exists(compile_db):
|
|
errors.append(f"compile database not found: {compile_db}")
|
|
return errors
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--fold", type=Path, required=True)
|
|
parser.add_argument("--out-dir", type=Path, default=ROOT / "build" / "quality")
|
|
parser.add_argument("--manifest", type=Path, default=DEFAULT_MANIFEST)
|
|
parser.add_argument("--compile-db", type=Path)
|
|
parser.add_argument(
|
|
"--test-impact-manifest", type=Path, default=DEFAULT_TEST_IMPACT_MANIFEST
|
|
)
|
|
parser.add_argument(
|
|
"--test-repetitions",
|
|
type=positive_integer,
|
|
default=1,
|
|
help="run each selected test set N times and retain flake evidence",
|
|
)
|
|
parser.add_argument(
|
|
"--base-ref", help="base revision used to compute the PR merge base"
|
|
)
|
|
parser.add_argument("--head-ref", help="head revision used for the branch diff")
|
|
parser.add_argument("--skip-format", action="store_true")
|
|
parser.add_argument("--skip-tests", action="store_true")
|
|
parser.add_argument(
|
|
"--full-tests",
|
|
action="store_true",
|
|
help="run the complete Meson suite instead of impacted tests",
|
|
)
|
|
parser.add_argument("--skip-policy", action="store_true")
|
|
parser.add_argument("--skip-diff-scan", action="store_true")
|
|
parser.add_argument("--allow-findings", action="store_true")
|
|
args = parser.parse_args()
|
|
|
|
fold = args.fold.expanduser().resolve()
|
|
if not fold.is_file():
|
|
print(f"fold binary not found: {fold}", file=sys.stderr)
|
|
return 2
|
|
out_dir = args.out_dir.expanduser().resolve()
|
|
out_dir.mkdir(parents=True, exist_ok=True)
|
|
steps: list[dict[str, object]] = []
|
|
if not args.skip_format:
|
|
steps.append(
|
|
run_step(
|
|
"format",
|
|
["./scripts/format-code.sh", "--check"],
|
|
artifact_dir=out_dir,
|
|
)
|
|
)
|
|
steps.append(
|
|
run_step(
|
|
"diff-hygiene",
|
|
["git", "diff", "--check"],
|
|
artifact_dir=out_dir,
|
|
)
|
|
)
|
|
if not args.skip_tests:
|
|
try:
|
|
test_changed_paths = changed_test_paths(
|
|
base_ref=args.base_ref, head_ref=args.head_ref
|
|
)
|
|
test_selection = select_impacted_tests(
|
|
test_changed_paths, args.test_impact_manifest
|
|
)
|
|
except ValueError as error:
|
|
test_selection = {
|
|
"schema_version": 1,
|
|
"manifest": args.test_impact_manifest.as_posix(),
|
|
"changed_paths": [],
|
|
"full_suite": True,
|
|
"fallback_reasons": [f"test selection failed closed: {error}"],
|
|
"unmapped_paths": [],
|
|
"matched_rules": [],
|
|
"selected_tests": [],
|
|
}
|
|
apply_full_test_override(test_selection, args.full_tests)
|
|
selection_path = out_dir / "test-selection.json"
|
|
selection_path.write_text(
|
|
json.dumps(test_selection, indent=2, sort_keys=True) + "\n",
|
|
encoding="utf-8",
|
|
)
|
|
full_suite_value = test_selection.get("full_suite")
|
|
full_suite = full_suite_value if isinstance(full_suite_value, bool) else True
|
|
selected_value = test_selection.get("selected_tests", [])
|
|
selected = selected_value if isinstance(selected_value, list) else []
|
|
unmapped_value = test_selection.get("unmapped_paths", [])
|
|
unmapped_paths = unmapped_value if isinstance(unmapped_value, list) else []
|
|
matched_value = test_selection.get("matched_rules", [])
|
|
matched_rules = matched_value if isinstance(matched_value, list) else []
|
|
fallback_value = test_selection.get("fallback_reasons", [])
|
|
fallback_reasons = fallback_value if isinstance(fallback_value, list) else []
|
|
selected_names = [
|
|
str(item["name"])
|
|
for item in selected
|
|
if isinstance(item, dict) and isinstance(item.get("name"), str)
|
|
]
|
|
compact_matched_rules = []
|
|
for item in matched_rules:
|
|
if not isinstance(item, dict):
|
|
continue
|
|
paths = item.get("paths", [])
|
|
compact_matched_rules.append(
|
|
{
|
|
"id": str(item.get("id", "unknown")),
|
|
"path_count": len(paths) if isinstance(paths, list) else 0,
|
|
}
|
|
)
|
|
attempt_records: list[dict[str, object]] = []
|
|
evidence_dir = out_dir / "test-evidence"
|
|
evidence_dir.mkdir(parents=True, exist_ok=True)
|
|
for attempt in range(1, args.test_repetitions + 1):
|
|
logbase = f"fold-quality-attempt-{attempt}"
|
|
meson_log = ROOT / "build" / "release" / "meson-logs" / f"{logbase}.json"
|
|
meson_log.unlink(missing_ok=True)
|
|
test_command = [
|
|
"meson",
|
|
"test",
|
|
"-C",
|
|
"build/release",
|
|
"--print-errorlogs",
|
|
"--logbase",
|
|
logbase,
|
|
]
|
|
if not full_suite:
|
|
test_command.extend(selected_names)
|
|
test_step = run_step(
|
|
f"impacted-tests-attempt-{attempt}",
|
|
test_command,
|
|
artifact_dir=out_dir,
|
|
)
|
|
exit_code = test_step.get("exit_code")
|
|
parsed_attempt = parse_test_attempt(
|
|
meson_log,
|
|
attempt,
|
|
exit_code if isinstance(exit_code, int) else 2,
|
|
)
|
|
attempt_path = evidence_dir / f"attempt-{attempt}.json"
|
|
attempt_path.write_text(
|
|
json.dumps(parsed_attempt, indent=2, sort_keys=True) + "\n",
|
|
encoding="utf-8",
|
|
)
|
|
artifacts = test_step.get("artifacts", [])
|
|
if isinstance(artifacts, list):
|
|
artifacts.append(
|
|
text_artifact(attempt_path, out_dir, "test-attempt-evidence")
|
|
)
|
|
parsed_tests_value = parsed_attempt.get("tests", [])
|
|
parsed_tests = (
|
|
parsed_tests_value if isinstance(parsed_tests_value, list) else []
|
|
)
|
|
test_step["test_attempt"] = {
|
|
"attempt": attempt,
|
|
"result_count": len(parsed_tests),
|
|
"parse_error": parsed_attempt.get("parse_error"),
|
|
}
|
|
steps.append(test_step)
|
|
attempt_records.append(parsed_attempt)
|
|
|
|
flaky_summary = summarize_test_attempts(
|
|
attempt_records,
|
|
selected_names if not full_suite else None,
|
|
)
|
|
flaky_path = out_dir / "flaky-test-evidence.json"
|
|
flaky_path.write_text(
|
|
json.dumps(flaky_summary, indent=2, sort_keys=True) + "\n",
|
|
encoding="utf-8",
|
|
)
|
|
passed_value = flaky_summary.get("all_attempts_passed")
|
|
classification_value = flaky_summary.get("counts", {})
|
|
classification_counts = (
|
|
classification_value if isinstance(classification_value, dict) else {}
|
|
)
|
|
flaky_value = flaky_summary.get("flaky_tests", [])
|
|
flaky_tests = flaky_value if isinstance(flaky_value, list) else []
|
|
parse_errors_value = flaky_summary.get("parse_errors", [])
|
|
parse_errors = (
|
|
parse_errors_value if isinstance(parse_errors_value, list) else []
|
|
)
|
|
steps.append(
|
|
{
|
|
"name": "flaky-test-evidence",
|
|
"passed": passed_value if isinstance(passed_value, bool) else False,
|
|
"test_repetitions": args.test_repetitions,
|
|
"classification_counts": classification_counts,
|
|
"flaky_tests": flaky_tests[:MAX_FINDING_DETAILS],
|
|
"parse_error_count": len(parse_errors),
|
|
"test_selection": {
|
|
"full_suite": full_suite,
|
|
"selected_tests": selected,
|
|
"matched_rules": compact_matched_rules,
|
|
"fallback_reasons": fallback_reasons,
|
|
"unmapped_path_count": len(unmapped_paths),
|
|
},
|
|
"artifacts": [
|
|
text_artifact(selection_path, out_dir, "test-selection"),
|
|
text_artifact(flaky_path, out_dir, "flaky-test-evidence"),
|
|
],
|
|
}
|
|
)
|
|
if not args.skip_policy:
|
|
policy_step = run_step(
|
|
"policy",
|
|
[
|
|
sys.executable,
|
|
"scripts/benchmarks/run_project_eval.py",
|
|
"--manifest",
|
|
str(args.manifest),
|
|
"--fold",
|
|
str(fold),
|
|
"--out-dir",
|
|
str(out_dir / "policy"),
|
|
],
|
|
artifact_dir=out_dir,
|
|
)
|
|
policy_details = summarize_policy_report(
|
|
out_dir / "policy" / "summary.json", out_dir
|
|
)
|
|
policy_artifacts = policy_details.pop("artifacts", [])
|
|
if isinstance(policy_artifacts, list):
|
|
artifacts = policy_step.get("artifacts", [])
|
|
if isinstance(artifacts, list):
|
|
artifacts.extend(policy_artifacts)
|
|
policy_step.update(policy_details)
|
|
steps.append(policy_step)
|
|
ci = os.environ.get("CI", "").lower() in {"1", "true", "yes"}
|
|
precondition_errors = diff_scan_precondition_errors(
|
|
ci=ci,
|
|
skip_diff_scan=args.skip_diff_scan,
|
|
base_ref=args.base_ref,
|
|
head_ref=args.head_ref,
|
|
compile_db=args.compile_db,
|
|
)
|
|
changed: list[str] = []
|
|
if precondition_errors:
|
|
steps.append(
|
|
{
|
|
"name": "diff-scan",
|
|
"passed": False,
|
|
"errors": precondition_errors,
|
|
"changed_files": changed,
|
|
}
|
|
)
|
|
elif not args.skip_diff_scan:
|
|
try:
|
|
changed = changed_source_files(
|
|
base_ref=args.base_ref,
|
|
head_ref=args.head_ref,
|
|
compile_db=args.compile_db,
|
|
)
|
|
except ValueError as error:
|
|
steps.append(
|
|
{
|
|
"name": "diff-scan",
|
|
"passed": False,
|
|
"errors": [str(error)],
|
|
"changed_files": changed,
|
|
}
|
|
)
|
|
else:
|
|
if changed and args.compile_db:
|
|
result = run_step(
|
|
"diff-scan",
|
|
[
|
|
str(fold),
|
|
"scan",
|
|
"--mode",
|
|
"source",
|
|
"--compile-db",
|
|
str(args.compile_db),
|
|
"--format",
|
|
"sarif",
|
|
"--output",
|
|
str(out_dir / "diff-scan.sarif.json"),
|
|
*changed,
|
|
],
|
|
artifact_dir=out_dir,
|
|
acceptable={0, 1} if args.allow_findings else {0},
|
|
)
|
|
result["changed_files"] = changed
|
|
scan_report = out_dir / "diff-scan.sarif.json"
|
|
if scan_report.is_file():
|
|
artifacts = result.get("artifacts", [])
|
|
if isinstance(artifacts, list):
|
|
artifacts.append(
|
|
text_artifact(scan_report, out_dir, "scan-report")
|
|
)
|
|
steps.append(result)
|
|
else:
|
|
reason = (
|
|
"no changed C/C++ files"
|
|
if not changed
|
|
else "compile database omitted for local differential scan"
|
|
)
|
|
steps.append(
|
|
{
|
|
"name": "diff-scan",
|
|
"skipped": True,
|
|
"changed_files": changed,
|
|
"reason": reason,
|
|
}
|
|
)
|
|
|
|
report = finalize_report(steps)
|
|
report_path = out_dir / "summary.json"
|
|
report_path.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8")
|
|
print(json.dumps(report, indent=2))
|
|
return 0 if report["status"] == "ok" else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|