bench: fingerprint benchmark working trees

tests/benchmarks/README.md

@@ -19,6 +19,10 @@ Brick's explicit `--exclude` option. Validate each resulting graph with YAMS's
 `tests/benchmarks/brick/contract.py`; the plan's broad count bounds are coverage
 guards, not precision or architecture grades.
 
+`run_manifest.json` records Git provenance for both Brick and the corpus. Dirty
+trees include a SHA-256 fingerprint over status, tracked diffs, and untracked
+file contents so retained results identify the exact measured working state.
+
 Before changing an extractor or score, add the expected positive and
 clean-negative cases, name one owner, and state the expected metric/identity
 delta. Use at least three repeats for performance claims.

tests/benchmarks/runner.py

@@ -5,7 +5,9 @@ from __future__ import annotations
 
 import argparse
 import datetime as dt
+import hashlib
 import json
+import os
 import pathlib
 import platform
 import statistics
@@ -17,6 +19,89 @@ ROOT = pathlib.Path(__file__).resolve().parents[2]
 PLANS = ROOT / "tests" / "benchmarks" / "plans"
 
 
+def git_provenance(path: pathlib.Path) -> dict:
+    """Describe a Git worktree and fingerprint all tracked and untracked changes."""
+
+    discovered = subprocess.run(
+        ["git", "rev-parse", "--show-toplevel"],
+        cwd=path,
+        text=True,
+        capture_output=True,
+        check=False,
+    )
+    if discovered.returncode != 0:
+        return {
+            "root": str(path.resolve()),
+            "revision": None,
+            "dirty": None,
+            "working_tree_fingerprint": None,
+        }
+
+    repository = pathlib.Path(discovered.stdout.strip()).resolve()
+    revision = subprocess.run(
+        ["git", "rev-parse", "HEAD"],
+        cwd=repository,
+        text=True,
+        capture_output=True,
+        check=False,
+    )
+    status = subprocess.run(
+        ["git", "status", "--porcelain=v1", "--untracked-files=all", "-z"],
+        cwd=repository,
+        capture_output=True,
+        check=False,
+    )
+    if revision.returncode != 0 or status.returncode != 0:
+        return {
+            "root": str(repository),
+            "revision": None,
+            "dirty": None,
+            "working_tree_fingerprint": None,
+        }
+
+    dirty = bool(status.stdout)
+    fingerprint = None
+    if dirty:
+        diff = subprocess.run(
+            ["git", "diff", "--binary", "HEAD"],
+            cwd=repository,
+            capture_output=True,
+            check=False,
+        )
+        untracked = subprocess.run(
+            ["git", "ls-files", "--others", "--exclude-standard", "-z"],
+            cwd=repository,
+            capture_output=True,
+            check=False,
+        )
+        if diff.returncode == 0 and untracked.returncode == 0:
+            digest = hashlib.sha256()
+            digest.update(b"status\0")
+            digest.update(status.stdout)
+            digest.update(b"diff\0")
+            digest.update(diff.stdout)
+            for encoded_path in filter(None, untracked.stdout.split(b"\0")):
+                entry = repository / os.fsdecode(encoded_path)
+                if entry.is_symlink():
+                    contents = os.fsencode(os.readlink(entry))
+                elif entry.is_file():
+                    contents = entry.read_bytes()
+                else:
+                    contents = b""
+                digest.update(b"untracked\0")
+                digest.update(encoded_path)
+                digest.update(b"\0")
+                digest.update(contents)
+            fingerprint = digest.hexdigest()
+
+    return {
+        "root": str(repository),
+        "revision": revision.stdout.strip(),
+        "dirty": dirty,
+        "working_tree_fingerprint": fingerprint,
+    }
+
+
 def load_plan(name: str) -> dict:
     path = PLANS / f"{name}.json"
     with path.open(encoding="utf-8") as handle:
@@ -71,20 +156,19 @@ def run_plan(
     stamp = dt.datetime.now(dt.UTC).strftime("%Y%m%dT%H%M%SZ")
     artifact = out_dir or ROOT / "build" / "benchmarks" / name / stamp
     artifact.mkdir(parents=True, exist_ok=False)
-    revision = subprocess.run(
+    brick_provenance = git_provenance(ROOT)
-        ["git", "rev-parse", "HEAD"],
+    corpus_provenance = git_provenance(corpus)
-        cwd=ROOT,
-        text=True,
-        capture_output=True,
-        check=False,
-    )
     manifest = {
         "schema_version": 1,
         "started_at": stamp,
         "plan": name,
         "corpus": str(corpus),
         "root": str(ROOT),
-        "revision": revision.stdout.strip() if revision.returncode == 0 else None,
+        "revision": brick_provenance["revision"],
+        "provenance": {
+            "brick": brick_provenance,
+            "corpus": corpus_provenance,
+        },
         "host": {
             "platform": platform.platform(),
             "python": platform.python_version(),

tests/benchmarks/test_runner.py

@@ -5,6 +5,8 @@ from __future__ import annotations
 
 import importlib.util
 import pathlib
+import subprocess
+import tempfile
 import unittest
 
 HERE = pathlib.Path(__file__).resolve().parent
@@ -15,6 +17,45 @@ SPEC.loader.exec_module(runner)
 
 
 class RunnerTests(unittest.TestCase):
+    def test_git_provenance_fingerprints_dirty_corpus_state(self) -> None:
+        with tempfile.TemporaryDirectory() as directory:
+            repository = pathlib.Path(directory)
+            subprocess.run(["git", "init", "--quiet"], cwd=repository, check=True)
+            subprocess.run(
+                ["git", "config", "user.email", "brick-tests@example.invalid"],
+                cwd=repository,
+                check=True,
+            )
+            subprocess.run(
+                ["git", "config", "user.name", "Brick Tests"], cwd=repository, check=True
+            )
+            subprocess.run(
+                ["git", "config", "commit.gpgsign", "false"], cwd=repository, check=True
+            )
+            tracked = repository / "tracked.cpp"
+            tracked.write_text("int value = 1;\n", encoding="utf-8")
+            subprocess.run(["git", "add", "tracked.cpp"], cwd=repository, check=True)
+            subprocess.run(
+                ["git", "commit", "--quiet", "-m", "fixture"], cwd=repository, check=True
+            )
+
+            clean = runner.git_provenance(repository)
+            self.assertFalse(clean["dirty"])
+            self.assertIsNone(clean["working_tree_fingerprint"])
+
+            tracked.write_text("int value = 2;\n", encoding="utf-8")
+            (repository / "untracked.dart").write_text("void main() {}\n", encoding="utf-8")
+            dirty = runner.git_provenance(repository)
+            self.assertTrue(dirty["dirty"])
+            self.assertEqual(dirty["revision"], clean["revision"])
+            self.assertRegex(dirty["working_tree_fingerprint"], r"^[0-9a-f]{64}$")
+
+            (repository / "untracked.dart").write_text("void changed() {}\n", encoding="utf-8")
+            changed = runner.git_provenance(repository)
+            self.assertNotEqual(
+                changed["working_tree_fingerprint"], dirty["working_tree_fingerprint"]
+            )
+
     def test_external_corpus_and_build_arguments_are_explicit(self) -> None:
         corpus = pathlib.Path("/tmp/yams")
         plan = {"corpus": "tests/fixtures/mixed", "build_args": ["--exclude", "site"]}