bench: harden architecture scan evidence #1

Merged
trevon merged 4 commits from bench/yams-architecture-reproducibility into main 2026-08-30 22:09:35 +00:00
8 changed files with 231 additions and 10 deletions

3
.gitignore vendored
View file

@ -2,4 +2,5 @@
/build/ /build/
/brick.json /brick.json
/brick.html /brick.html
__pycache__/
*.py[cod]

View file

@ -15,6 +15,11 @@ location so consumers can distinguish it from imported symbolic evidence.
cargo run -- build ../fold --json fold.brick.json --html fold.brick.html cargo run -- build ../fold --json fold.brick.json --html fold.brick.html
Exclude generated or checked-in output by exact directory component (repeatable):
cargo run -- build ../yams --exclude site --exclude build \
--json yams.brick.json --html yams.brick.html
Layer Fold findings onto the building: Layer Fold findings onto the building:
cargo run -- build ../fold \ cargo run -- build ../fold \
@ -22,6 +27,11 @@ Layer Fold findings onto the building:
--json fold.brick.json \ --json fold.brick.json \
--html fold.brick.html --html fold.brick.html
Fold artifact paths are normalized lexically across `/`, `\\`, `.`, and `..`
components before repository-relative matching. Findings for excluded,
generated, or genuinely missing files remain diagnostics instead of being
attached to a neighboring source file.
Render an existing model or enforce an explicit experimental gate: Render an existing model or enforce an explicit experimental gate:
cargo run -- render fold.brick.json --html fold.brick.html cargo run -- render fold.brick.json --html fold.brick.html
@ -53,6 +63,13 @@ multiline signatures. Its score is not proof of correctness, maintainability,
or security. Use the component metrics and evidence records; do not rank teams or security. Use the component metrics and evidence records; do not rank teams
or repositories by the composite alone. or repositories by the composite alone.
Unqualified calls such as `lock`, `size`, and `empty` may merge unrelated C++
targets or remain external; recursion and overloads may also appear as lexical
self-cycles. File fan-out is therefore a decomposition signal, not proof of
semantic coupling. Scope architecture gates to production paths and verify each
candidate against source and tests. Cross-translation-unit and namespaced C++
resolution remain priority adapter work.
The next serious analyzer milestone should consume compiler/YAMS/Fold graph The next serious analyzer milestone should consume compiler/YAMS/Fold graph
facts through adapters while retaining the standalone lexical lane as a fast facts through adapters while retaining the standalone lexical lane as a fast
preview. See docs/architecture/graph-model.md and docs/ROADMAP.md. preview. See docs/architecture/graph-model.md and docs/ROADMAP.md.

View file

@ -207,19 +207,45 @@ fn native_backend(value: &Value) -> String {
} }
fn find_file(building: &Building, path: &str) -> Option<String> { fn find_file(building: &Building, path: &str) -> Option<String> {
let normalized = path.replace('\\', "/"); let normalized = normalize_lexical_path(path);
building building
.nodes .nodes
.iter() .iter()
.filter(|node| node.kind == NodeKind::File) .filter(|node| node.kind == NodeKind::File)
.find(|node| { .find(|node| {
node.path.as_deref().is_some_and(|candidate| { node.path.as_deref().is_some_and(|candidate| {
let candidate = normalize_lexical_path(candidate);
normalized == candidate || normalized.ends_with(&format!("/{candidate}")) normalized == candidate || normalized.ends_with(&format!("/{candidate}"))
}) })
}) })
.map(|node| node.id.clone()) .map(|node| node.id.clone())
} }
fn normalize_lexical_path(path: &str) -> String {
let portable = path.replace('\\', "/");
let absolute = portable.starts_with('/');
let mut components = Vec::new();
for component in portable.split('/') {
match component {
"" | "." => {}
".." => {
if components.last().is_some_and(|previous| *previous != "..") {
components.pop();
} else if !absolute {
components.push(component);
}
}
_ => components.push(component),
}
}
let normalized = components.join("/");
if absolute {
format!("/{normalized}")
} else {
normalized
}
}
fn confidence_value(confidence: &str) -> f64 { fn confidence_value(confidence: &str) -> f64 {
match confidence { match confidence {
"concrete-witness" => 1.0, "concrete-witness" => 1.0,
@ -258,4 +284,52 @@ mod tests {
assert_eq!(finding.attributes["confidence"], "proved"); assert_eq!(finding.attributes["confidence"], "proved");
assert_eq!(finding.attributes["backend"], "source"); assert_eq!(finding.attributes["backend"], "source");
} }
#[test]
fn normalizes_fold_paths_without_matching_a_missing_neighbor() {
let dir = tempfile::tempdir().unwrap();
let header = dir.path().join("include/yams/core/types.h");
fs::create_dir_all(header.parent().unwrap()).unwrap();
fs::write(&header, "struct Type {};\n").unwrap();
let report = dir.path().join("fold.sarif");
fs::write(
&report,
r#"{
"runs": [{
"results": [
{
"ruleId": "fold.valid",
"message": {"text": "valid"},
"locations": [{"physicalLocation": {
"artifactLocation": {"uri": "../../include/yams/daemon/components/../../core/types.h"},
"region": {"startLine": 1}
}}]
},
{
"ruleId": "fold.missing",
"message": {"text": "missing"},
"locations": [{"physicalLocation": {
"artifactLocation": {"uri": "../../include/yams/daemon/components/../../core/missing.h"},
"region": {"startLine": 1}
}}]
}
]
}]
}"#,
)
.unwrap();
let mut building = scan(dir.path()).unwrap();
assert_eq!(import_findings(&mut building, &report).unwrap(), 1);
assert_eq!(
building
.nodes
.iter()
.filter(|node| node.kind == NodeKind::Finding)
.count(),
1
);
assert_eq!(building.diagnostics.len(), 1);
assert!(building.diagnostics[0].message.contains("missing.h"));
}
} }

View file

@ -19,6 +19,10 @@ Brick's explicit `--exclude` option. Validate each resulting graph with YAMS's
`tests/benchmarks/brick/contract.py`; the plan's broad count bounds are coverage `tests/benchmarks/brick/contract.py`; the plan's broad count bounds are coverage
guards, not precision or architecture grades. guards, not precision or architecture grades.
`run_manifest.json` records Git provenance for both Brick and the corpus. Dirty
trees include a SHA-256 fingerprint over status, tracked diffs, and untracked
file contents so retained results identify the exact measured working state.
Before changing an extractor or score, add the expected positive and Before changing an extractor or score, add the expected positive and
clean-negative cases, name one owner, and state the expected metric/identity clean-negative cases, name one owner, and state the expected metric/identity
delta. Use at least three repeats for performance claims. delta. Use at least three repeats for performance claims.

View file

@ -5,7 +5,9 @@ from __future__ import annotations
import argparse import argparse
import datetime as dt import datetime as dt
import hashlib
import json import json
import os
import pathlib import pathlib
import platform import platform
import statistics import statistics
@ -17,6 +19,89 @@ ROOT = pathlib.Path(__file__).resolve().parents[2]
PLANS = ROOT / "tests" / "benchmarks" / "plans" PLANS = ROOT / "tests" / "benchmarks" / "plans"
def git_provenance(path: pathlib.Path) -> dict:
"""Describe a Git worktree and fingerprint all tracked and untracked changes."""
discovered = subprocess.run(
["git", "rev-parse", "--show-toplevel"],
cwd=path,
text=True,
capture_output=True,
check=False,
)
if discovered.returncode != 0:
return {
"root": str(path.resolve()),
"revision": None,
"dirty": None,
"working_tree_fingerprint": None,
}
repository = pathlib.Path(discovered.stdout.strip()).resolve()
revision = subprocess.run(
["git", "rev-parse", "HEAD"],
cwd=repository,
text=True,
capture_output=True,
check=False,
)
status = subprocess.run(
["git", "status", "--porcelain=v1", "--untracked-files=all", "-z"],
cwd=repository,
capture_output=True,
check=False,
)
if revision.returncode != 0 or status.returncode != 0:
return {
"root": str(repository),
"revision": None,
"dirty": None,
"working_tree_fingerprint": None,
}
dirty = bool(status.stdout)
fingerprint = None
if dirty:
diff = subprocess.run(
["git", "diff", "--binary", "HEAD"],
cwd=repository,
capture_output=True,
check=False,
)
untracked = subprocess.run(
["git", "ls-files", "--others", "--exclude-standard", "-z"],
cwd=repository,
capture_output=True,
check=False,
)
if diff.returncode == 0 and untracked.returncode == 0:
digest = hashlib.sha256()
digest.update(b"status\0")
digest.update(status.stdout)
digest.update(b"diff\0")
digest.update(diff.stdout)
for encoded_path in filter(None, untracked.stdout.split(b"\0")):
entry = repository / os.fsdecode(encoded_path)
if entry.is_symlink():
contents = os.fsencode(os.readlink(entry))
elif entry.is_file():
contents = entry.read_bytes()
else:
contents = b""
digest.update(b"untracked\0")
digest.update(encoded_path)
digest.update(b"\0")
digest.update(contents)
fingerprint = digest.hexdigest()
return {
"root": str(repository),
"revision": revision.stdout.strip(),
"dirty": dirty,
"working_tree_fingerprint": fingerprint,
}
def load_plan(name: str) -> dict: def load_plan(name: str) -> dict:
path = PLANS / f"{name}.json" path = PLANS / f"{name}.json"
with path.open(encoding="utf-8") as handle: with path.open(encoding="utf-8") as handle:
@ -71,20 +156,19 @@ def run_plan(
stamp = dt.datetime.now(dt.UTC).strftime("%Y%m%dT%H%M%SZ") stamp = dt.datetime.now(dt.UTC).strftime("%Y%m%dT%H%M%SZ")
artifact = out_dir or ROOT / "build" / "benchmarks" / name / stamp artifact = out_dir or ROOT / "build" / "benchmarks" / name / stamp
artifact.mkdir(parents=True, exist_ok=False) artifact.mkdir(parents=True, exist_ok=False)
revision = subprocess.run( brick_provenance = git_provenance(ROOT)
["git", "rev-parse", "HEAD"], corpus_provenance = git_provenance(corpus)
cwd=ROOT,
text=True,
capture_output=True,
check=False,
)
manifest = { manifest = {
"schema_version": 1, "schema_version": 1,
"started_at": stamp, "started_at": stamp,
"plan": name, "plan": name,
"corpus": str(corpus), "corpus": str(corpus),
"root": str(ROOT), "root": str(ROOT),
"revision": revision.stdout.strip() if revision.returncode == 0 else None, "revision": brick_provenance["revision"],
"provenance": {
"brick": brick_provenance,
"corpus": corpus_provenance,
},
"host": { "host": {
"platform": platform.platform(), "platform": platform.platform(),
"python": platform.python_version(), "python": platform.python_version(),

View file

@ -5,6 +5,8 @@ from __future__ import annotations
import importlib.util import importlib.util
import pathlib import pathlib
import subprocess
import tempfile
import unittest import unittest
HERE = pathlib.Path(__file__).resolve().parent HERE = pathlib.Path(__file__).resolve().parent
@ -15,6 +17,45 @@ SPEC.loader.exec_module(runner)
class RunnerTests(unittest.TestCase): class RunnerTests(unittest.TestCase):
def test_git_provenance_fingerprints_dirty_corpus_state(self) -> None:
with tempfile.TemporaryDirectory() as directory:
repository = pathlib.Path(directory)
subprocess.run(["git", "init", "--quiet"], cwd=repository, check=True)
subprocess.run(
["git", "config", "user.email", "brick-tests@example.invalid"],
cwd=repository,
check=True,
)
subprocess.run(
["git", "config", "user.name", "Brick Tests"], cwd=repository, check=True
)
subprocess.run(
["git", "config", "commit.gpgsign", "false"], cwd=repository, check=True
)
tracked = repository / "tracked.cpp"
tracked.write_text("int value = 1;\n", encoding="utf-8")
subprocess.run(["git", "add", "tracked.cpp"], cwd=repository, check=True)
subprocess.run(
["git", "commit", "--quiet", "-m", "fixture"], cwd=repository, check=True
)
clean = runner.git_provenance(repository)
self.assertFalse(clean["dirty"])
self.assertIsNone(clean["working_tree_fingerprint"])
tracked.write_text("int value = 2;\n", encoding="utf-8")
(repository / "untracked.dart").write_text("void main() {}\n", encoding="utf-8")
dirty = runner.git_provenance(repository)
self.assertTrue(dirty["dirty"])
self.assertEqual(dirty["revision"], clean["revision"])
self.assertRegex(dirty["working_tree_fingerprint"], r"^[0-9a-f]{64}$")
(repository / "untracked.dart").write_text("void changed() {}\n", encoding="utf-8")
changed = runner.git_provenance(repository)
self.assertNotEqual(
changed["working_tree_fingerprint"], dirty["working_tree_fingerprint"]
)
def test_external_corpus_and_build_arguments_are_explicit(self) -> None: def test_external_corpus_and_build_arguments_are_explicit(self) -> None:
corpus = pathlib.Path("/tmp/yams") corpus = pathlib.Path("/tmp/yams")
plan = {"corpus": "tests/fixtures/mixed", "build_args": ["--exclude", "site"]} plan = {"corpus": "tests/fixtures/mixed", "build_args": ["--exclude", "site"]}