bench: harden architecture scan evidence #1

Merged
trevon merged 4 commits from bench/yams-architecture-reproducibility into main 2026-08-30 22:09:35 +00:00
8 changed files with 231 additions and 10 deletions

3
.gitignore vendored
View file

@ -2,4 +2,5 @@
/build/
/brick.json
/brick.html
__pycache__/
*.py[cod]

View file

@ -15,6 +15,11 @@ location so consumers can distinguish it from imported symbolic evidence.
cargo run -- build ../fold --json fold.brick.json --html fold.brick.html
Exclude generated or checked-in output by exact directory component (repeatable):
cargo run -- build ../yams --exclude site --exclude build \
--json yams.brick.json --html yams.brick.html
Layer Fold findings onto the building:
cargo run -- build ../fold \
@ -22,6 +27,11 @@ Layer Fold findings onto the building:
--json fold.brick.json \
--html fold.brick.html
Fold artifact paths are normalized lexically across `/`, `\\`, `.`, and `..`
components before repository-relative matching. Findings for excluded,
generated, or genuinely missing files remain diagnostics instead of being
attached to a neighboring source file.
Render an existing model or enforce an explicit experimental gate:
cargo run -- render fold.brick.json --html fold.brick.html
@ -53,6 +63,13 @@ multiline signatures. Its score is not proof of correctness, maintainability,
or security. Use the component metrics and evidence records; do not rank teams
or repositories by the composite alone.
Unqualified calls such as `lock`, `size`, and `empty` may merge unrelated C++
targets or remain external; recursion and overloads may also appear as lexical
self-cycles. File fan-out is therefore a decomposition signal, not proof of
semantic coupling. Scope architecture gates to production paths and verify each
candidate against source and tests. Cross-translation-unit and namespaced C++
resolution remain priority adapter work.
The next serious analyzer milestone should consume compiler/YAMS/Fold graph
facts through adapters while retaining the standalone lexical lane as a fast
preview. See docs/architecture/graph-model.md and docs/ROADMAP.md.

View file

@ -207,19 +207,45 @@ fn native_backend(value: &Value) -> String {
}
fn find_file(building: &Building, path: &str) -> Option<String> {
let normalized = path.replace('\\', "/");
let normalized = normalize_lexical_path(path);
building
.nodes
.iter()
.filter(|node| node.kind == NodeKind::File)
.find(|node| {
node.path.as_deref().is_some_and(|candidate| {
let candidate = normalize_lexical_path(candidate);
normalized == candidate || normalized.ends_with(&format!("/{candidate}"))
})
})
.map(|node| node.id.clone())
}
fn normalize_lexical_path(path: &str) -> String {
let portable = path.replace('\\', "/");
let absolute = portable.starts_with('/');
let mut components = Vec::new();
for component in portable.split('/') {
match component {
"" | "." => {}
".." => {
if components.last().is_some_and(|previous| *previous != "..") {
components.pop();
} else if !absolute {
components.push(component);
}
}
_ => components.push(component),
}
}
let normalized = components.join("/");
if absolute {
format!("/{normalized}")
} else {
normalized
}
}
fn confidence_value(confidence: &str) -> f64 {
match confidence {
"concrete-witness" => 1.0,
@ -258,4 +284,52 @@ mod tests {
assert_eq!(finding.attributes["confidence"], "proved");
assert_eq!(finding.attributes["backend"], "source");
}
#[test]
fn normalizes_fold_paths_without_matching_a_missing_neighbor() {
let dir = tempfile::tempdir().unwrap();
let header = dir.path().join("include/yams/core/types.h");
fs::create_dir_all(header.parent().unwrap()).unwrap();
fs::write(&header, "struct Type {};\n").unwrap();
let report = dir.path().join("fold.sarif");
fs::write(
&report,
r#"{
"runs": [{
"results": [
{
"ruleId": "fold.valid",
"message": {"text": "valid"},
"locations": [{"physicalLocation": {
"artifactLocation": {"uri": "../../include/yams/daemon/components/../../core/types.h"},
"region": {"startLine": 1}
}}]
},
{
"ruleId": "fold.missing",
"message": {"text": "missing"},
"locations": [{"physicalLocation": {
"artifactLocation": {"uri": "../../include/yams/daemon/components/../../core/missing.h"},
"region": {"startLine": 1}
}}]
}
]
}]
}"#,
)
.unwrap();
let mut building = scan(dir.path()).unwrap();
assert_eq!(import_findings(&mut building, &report).unwrap(), 1);
assert_eq!(
building
.nodes
.iter()
.filter(|node| node.kind == NodeKind::Finding)
.count(),
1
);
assert_eq!(building.diagnostics.len(), 1);
assert!(building.diagnostics[0].message.contains("missing.h"));
}
}

View file

@ -19,6 +19,10 @@ Brick's explicit `--exclude` option. Validate each resulting graph with YAMS's
`tests/benchmarks/brick/contract.py`; the plan's broad count bounds are coverage
guards, not precision or architecture grades.
`run_manifest.json` records Git provenance for both Brick and the corpus. Dirty
trees include a SHA-256 fingerprint over status, tracked diffs, and untracked
file contents so retained results identify the exact measured working state.
Before changing an extractor or score, add the expected positive and
clean-negative cases, name one owner, and state the expected metric/identity
delta. Use at least three repeats for performance claims.

View file

@ -5,7 +5,9 @@ from __future__ import annotations
import argparse
import datetime as dt
import hashlib
import json
import os
import pathlib
import platform
import statistics
@ -17,6 +19,89 @@ ROOT = pathlib.Path(__file__).resolve().parents[2]
PLANS = ROOT / "tests" / "benchmarks" / "plans"
def git_provenance(path: pathlib.Path) -> dict:
"""Describe a Git worktree and fingerprint all tracked and untracked changes."""
discovered = subprocess.run(
["git", "rev-parse", "--show-toplevel"],
cwd=path,
text=True,
capture_output=True,
check=False,
)
if discovered.returncode != 0:
return {
"root": str(path.resolve()),
"revision": None,
"dirty": None,
"working_tree_fingerprint": None,
}
repository = pathlib.Path(discovered.stdout.strip()).resolve()
revision = subprocess.run(
["git", "rev-parse", "HEAD"],
cwd=repository,
text=True,
capture_output=True,
check=False,
)
status = subprocess.run(
["git", "status", "--porcelain=v1", "--untracked-files=all", "-z"],
cwd=repository,
capture_output=True,
check=False,
)
if revision.returncode != 0 or status.returncode != 0:
return {
"root": str(repository),
"revision": None,
"dirty": None,
"working_tree_fingerprint": None,
}
dirty = bool(status.stdout)
fingerprint = None
if dirty:
diff = subprocess.run(
["git", "diff", "--binary", "HEAD"],
cwd=repository,
capture_output=True,
check=False,
)
untracked = subprocess.run(
["git", "ls-files", "--others", "--exclude-standard", "-z"],
cwd=repository,
capture_output=True,
check=False,
)
if diff.returncode == 0 and untracked.returncode == 0:
digest = hashlib.sha256()
digest.update(b"status\0")
digest.update(status.stdout)
digest.update(b"diff\0")
digest.update(diff.stdout)
for encoded_path in filter(None, untracked.stdout.split(b"\0")):
entry = repository / os.fsdecode(encoded_path)
if entry.is_symlink():
contents = os.fsencode(os.readlink(entry))
elif entry.is_file():
contents = entry.read_bytes()
else:
contents = b""
digest.update(b"untracked\0")
digest.update(encoded_path)
digest.update(b"\0")
digest.update(contents)
fingerprint = digest.hexdigest()
return {
"root": str(repository),
"revision": revision.stdout.strip(),
"dirty": dirty,
"working_tree_fingerprint": fingerprint,
}
def load_plan(name: str) -> dict:
path = PLANS / f"{name}.json"
with path.open(encoding="utf-8") as handle:
@ -71,20 +156,19 @@ def run_plan(
stamp = dt.datetime.now(dt.UTC).strftime("%Y%m%dT%H%M%SZ")
artifact = out_dir or ROOT / "build" / "benchmarks" / name / stamp
artifact.mkdir(parents=True, exist_ok=False)
revision = subprocess.run(
["git", "rev-parse", "HEAD"],
cwd=ROOT,
text=True,
capture_output=True,
check=False,
)
brick_provenance = git_provenance(ROOT)
corpus_provenance = git_provenance(corpus)
manifest = {
"schema_version": 1,
"started_at": stamp,
"plan": name,
"corpus": str(corpus),
"root": str(ROOT),
"revision": revision.stdout.strip() if revision.returncode == 0 else None,
"revision": brick_provenance["revision"],
"provenance": {
"brick": brick_provenance,
"corpus": corpus_provenance,
},
"host": {
"platform": platform.platform(),
"python": platform.python_version(),

View file

@ -5,6 +5,8 @@ from __future__ import annotations
import importlib.util
import pathlib
import subprocess
import tempfile
import unittest
HERE = pathlib.Path(__file__).resolve().parent
@ -15,6 +17,45 @@ SPEC.loader.exec_module(runner)
class RunnerTests(unittest.TestCase):
def test_git_provenance_fingerprints_dirty_corpus_state(self) -> None:
with tempfile.TemporaryDirectory() as directory:
repository = pathlib.Path(directory)
subprocess.run(["git", "init", "--quiet"], cwd=repository, check=True)
subprocess.run(
["git", "config", "user.email", "brick-tests@example.invalid"],
cwd=repository,
check=True,
)
subprocess.run(
["git", "config", "user.name", "Brick Tests"], cwd=repository, check=True
)
subprocess.run(
["git", "config", "commit.gpgsign", "false"], cwd=repository, check=True
)
tracked = repository / "tracked.cpp"
tracked.write_text("int value = 1;\n", encoding="utf-8")
subprocess.run(["git", "add", "tracked.cpp"], cwd=repository, check=True)
subprocess.run(
["git", "commit", "--quiet", "-m", "fixture"], cwd=repository, check=True
)
clean = runner.git_provenance(repository)
self.assertFalse(clean["dirty"])
self.assertIsNone(clean["working_tree_fingerprint"])
tracked.write_text("int value = 2;\n", encoding="utf-8")
(repository / "untracked.dart").write_text("void main() {}\n", encoding="utf-8")
dirty = runner.git_provenance(repository)
self.assertTrue(dirty["dirty"])
self.assertEqual(dirty["revision"], clean["revision"])
self.assertRegex(dirty["working_tree_fingerprint"], r"^[0-9a-f]{64}$")
(repository / "untracked.dart").write_text("void changed() {}\n", encoding="utf-8")
changed = runner.git_provenance(repository)
self.assertNotEqual(
changed["working_tree_fingerprint"], dirty["working_tree_fingerprint"]
)
def test_external_corpus_and_build_arguments_are_explicit(self) -> None:
corpus = pathlib.Path("/tmp/yams")
plan = {"corpus": "tests/fixtures/mixed", "build_args": ["--exclude", "site"]}