bench: harden architecture scan evidence #1
8 changed files with 231 additions and 10 deletions
3
.gitignore
vendored
3
.gitignore
vendored
|
|
@ -2,4 +2,5 @@
|
||||||
/build/
|
/build/
|
||||||
/brick.json
|
/brick.json
|
||||||
/brick.html
|
/brick.html
|
||||||
|
__pycache__/
|
||||||
|
*.py[cod]
|
||||||
|
|
|
||||||
17
README.md
17
README.md
|
|
@ -15,6 +15,11 @@ location so consumers can distinguish it from imported symbolic evidence.
|
||||||
|
|
||||||
cargo run -- build ../fold --json fold.brick.json --html fold.brick.html
|
cargo run -- build ../fold --json fold.brick.json --html fold.brick.html
|
||||||
|
|
||||||
|
Exclude generated or checked-in output by exact directory component (repeatable):
|
||||||
|
|
||||||
|
cargo run -- build ../yams --exclude site --exclude build \
|
||||||
|
--json yams.brick.json --html yams.brick.html
|
||||||
|
|
||||||
Layer Fold findings onto the building:
|
Layer Fold findings onto the building:
|
||||||
|
|
||||||
cargo run -- build ../fold \
|
cargo run -- build ../fold \
|
||||||
|
|
@ -22,6 +27,11 @@ Layer Fold findings onto the building:
|
||||||
--json fold.brick.json \
|
--json fold.brick.json \
|
||||||
--html fold.brick.html
|
--html fold.brick.html
|
||||||
|
|
||||||
|
Fold artifact paths are normalized lexically across `/`, `\\`, `.`, and `..`
|
||||||
|
components before repository-relative matching. Findings for excluded,
|
||||||
|
generated, or genuinely missing files remain diagnostics instead of being
|
||||||
|
attached to a neighboring source file.
|
||||||
|
|
||||||
Render an existing model or enforce an explicit experimental gate:
|
Render an existing model or enforce an explicit experimental gate:
|
||||||
|
|
||||||
cargo run -- render fold.brick.json --html fold.brick.html
|
cargo run -- render fold.brick.json --html fold.brick.html
|
||||||
|
|
@ -53,6 +63,13 @@ multiline signatures. Its score is not proof of correctness, maintainability,
|
||||||
or security. Use the component metrics and evidence records; do not rank teams
|
or security. Use the component metrics and evidence records; do not rank teams
|
||||||
or repositories by the composite alone.
|
or repositories by the composite alone.
|
||||||
|
|
||||||
|
Unqualified calls such as `lock`, `size`, and `empty` may merge unrelated C++
|
||||||
|
targets or remain external; recursion and overloads may also appear as lexical
|
||||||
|
self-cycles. File fan-out is therefore a decomposition signal, not proof of
|
||||||
|
semantic coupling. Scope architecture gates to production paths and verify each
|
||||||
|
candidate against source and tests. Cross-translation-unit and namespaced C++
|
||||||
|
resolution remain priority adapter work.
|
||||||
|
|
||||||
The next serious analyzer milestone should consume compiler/YAMS/Fold graph
|
The next serious analyzer milestone should consume compiler/YAMS/Fold graph
|
||||||
facts through adapters while retaining the standalone lexical lane as a fast
|
facts through adapters while retaining the standalone lexical lane as a fast
|
||||||
preview. See docs/architecture/graph-model.md and docs/ROADMAP.md.
|
preview. See docs/architecture/graph-model.md and docs/ROADMAP.md.
|
||||||
|
|
|
||||||
Binary file not shown.
76
src/fold.rs
76
src/fold.rs
|
|
@ -207,19 +207,45 @@ fn native_backend(value: &Value) -> String {
|
||||||
}
|
}
|
||||||
|
|
||||||
fn find_file(building: &Building, path: &str) -> Option<String> {
|
fn find_file(building: &Building, path: &str) -> Option<String> {
|
||||||
let normalized = path.replace('\\', "/");
|
let normalized = normalize_lexical_path(path);
|
||||||
building
|
building
|
||||||
.nodes
|
.nodes
|
||||||
.iter()
|
.iter()
|
||||||
.filter(|node| node.kind == NodeKind::File)
|
.filter(|node| node.kind == NodeKind::File)
|
||||||
.find(|node| {
|
.find(|node| {
|
||||||
node.path.as_deref().is_some_and(|candidate| {
|
node.path.as_deref().is_some_and(|candidate| {
|
||||||
|
let candidate = normalize_lexical_path(candidate);
|
||||||
normalized == candidate || normalized.ends_with(&format!("/{candidate}"))
|
normalized == candidate || normalized.ends_with(&format!("/{candidate}"))
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
.map(|node| node.id.clone())
|
.map(|node| node.id.clone())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn normalize_lexical_path(path: &str) -> String {
|
||||||
|
let portable = path.replace('\\', "/");
|
||||||
|
let absolute = portable.starts_with('/');
|
||||||
|
let mut components = Vec::new();
|
||||||
|
for component in portable.split('/') {
|
||||||
|
match component {
|
||||||
|
"" | "." => {}
|
||||||
|
".." => {
|
||||||
|
if components.last().is_some_and(|previous| *previous != "..") {
|
||||||
|
components.pop();
|
||||||
|
} else if !absolute {
|
||||||
|
components.push(component);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
_ => components.push(component),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let normalized = components.join("/");
|
||||||
|
if absolute {
|
||||||
|
format!("/{normalized}")
|
||||||
|
} else {
|
||||||
|
normalized
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn confidence_value(confidence: &str) -> f64 {
|
fn confidence_value(confidence: &str) -> f64 {
|
||||||
match confidence {
|
match confidence {
|
||||||
"concrete-witness" => 1.0,
|
"concrete-witness" => 1.0,
|
||||||
|
|
@ -258,4 +284,52 @@ mod tests {
|
||||||
assert_eq!(finding.attributes["confidence"], "proved");
|
assert_eq!(finding.attributes["confidence"], "proved");
|
||||||
assert_eq!(finding.attributes["backend"], "source");
|
assert_eq!(finding.attributes["backend"], "source");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn normalizes_fold_paths_without_matching_a_missing_neighbor() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let header = dir.path().join("include/yams/core/types.h");
|
||||||
|
fs::create_dir_all(header.parent().unwrap()).unwrap();
|
||||||
|
fs::write(&header, "struct Type {};\n").unwrap();
|
||||||
|
let report = dir.path().join("fold.sarif");
|
||||||
|
fs::write(
|
||||||
|
&report,
|
||||||
|
r#"{
|
||||||
|
"runs": [{
|
||||||
|
"results": [
|
||||||
|
{
|
||||||
|
"ruleId": "fold.valid",
|
||||||
|
"message": {"text": "valid"},
|
||||||
|
"locations": [{"physicalLocation": {
|
||||||
|
"artifactLocation": {"uri": "../../include/yams/daemon/components/../../core/types.h"},
|
||||||
|
"region": {"startLine": 1}
|
||||||
|
}}]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ruleId": "fold.missing",
|
||||||
|
"message": {"text": "missing"},
|
||||||
|
"locations": [{"physicalLocation": {
|
||||||
|
"artifactLocation": {"uri": "../../include/yams/daemon/components/../../core/missing.h"},
|
||||||
|
"region": {"startLine": 1}
|
||||||
|
}}]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}]
|
||||||
|
}"#,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let mut building = scan(dir.path()).unwrap();
|
||||||
|
assert_eq!(import_findings(&mut building, &report).unwrap(), 1);
|
||||||
|
assert_eq!(
|
||||||
|
building
|
||||||
|
.nodes
|
||||||
|
.iter()
|
||||||
|
.filter(|node| node.kind == NodeKind::Finding)
|
||||||
|
.count(),
|
||||||
|
1
|
||||||
|
);
|
||||||
|
assert_eq!(building.diagnostics.len(), 1);
|
||||||
|
assert!(building.diagnostics[0].message.contains("missing.h"));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -19,6 +19,10 @@ Brick's explicit `--exclude` option. Validate each resulting graph with YAMS's
|
||||||
`tests/benchmarks/brick/contract.py`; the plan's broad count bounds are coverage
|
`tests/benchmarks/brick/contract.py`; the plan's broad count bounds are coverage
|
||||||
guards, not precision or architecture grades.
|
guards, not precision or architecture grades.
|
||||||
|
|
||||||
|
`run_manifest.json` records Git provenance for both Brick and the corpus. Dirty
|
||||||
|
trees include a SHA-256 fingerprint over status, tracked diffs, and untracked
|
||||||
|
file contents so retained results identify the exact measured working state.
|
||||||
|
|
||||||
Before changing an extractor or score, add the expected positive and
|
Before changing an extractor or score, add the expected positive and
|
||||||
clean-negative cases, name one owner, and state the expected metric/identity
|
clean-negative cases, name one owner, and state the expected metric/identity
|
||||||
delta. Use at least three repeats for performance claims.
|
delta. Use at least three repeats for performance claims.
|
||||||
|
|
|
||||||
Binary file not shown.
|
|
@ -5,7 +5,9 @@ from __future__ import annotations
|
||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
import datetime as dt
|
import datetime as dt
|
||||||
|
import hashlib
|
||||||
import json
|
import json
|
||||||
|
import os
|
||||||
import pathlib
|
import pathlib
|
||||||
import platform
|
import platform
|
||||||
import statistics
|
import statistics
|
||||||
|
|
@ -17,6 +19,89 @@ ROOT = pathlib.Path(__file__).resolve().parents[2]
|
||||||
PLANS = ROOT / "tests" / "benchmarks" / "plans"
|
PLANS = ROOT / "tests" / "benchmarks" / "plans"
|
||||||
|
|
||||||
|
|
||||||
|
def git_provenance(path: pathlib.Path) -> dict:
|
||||||
|
"""Describe a Git worktree and fingerprint all tracked and untracked changes."""
|
||||||
|
|
||||||
|
discovered = subprocess.run(
|
||||||
|
["git", "rev-parse", "--show-toplevel"],
|
||||||
|
cwd=path,
|
||||||
|
text=True,
|
||||||
|
capture_output=True,
|
||||||
|
check=False,
|
||||||
|
)
|
||||||
|
if discovered.returncode != 0:
|
||||||
|
return {
|
||||||
|
"root": str(path.resolve()),
|
||||||
|
"revision": None,
|
||||||
|
"dirty": None,
|
||||||
|
"working_tree_fingerprint": None,
|
||||||
|
}
|
||||||
|
|
||||||
|
repository = pathlib.Path(discovered.stdout.strip()).resolve()
|
||||||
|
revision = subprocess.run(
|
||||||
|
["git", "rev-parse", "HEAD"],
|
||||||
|
cwd=repository,
|
||||||
|
text=True,
|
||||||
|
capture_output=True,
|
||||||
|
check=False,
|
||||||
|
)
|
||||||
|
status = subprocess.run(
|
||||||
|
["git", "status", "--porcelain=v1", "--untracked-files=all", "-z"],
|
||||||
|
cwd=repository,
|
||||||
|
capture_output=True,
|
||||||
|
check=False,
|
||||||
|
)
|
||||||
|
if revision.returncode != 0 or status.returncode != 0:
|
||||||
|
return {
|
||||||
|
"root": str(repository),
|
||||||
|
"revision": None,
|
||||||
|
"dirty": None,
|
||||||
|
"working_tree_fingerprint": None,
|
||||||
|
}
|
||||||
|
|
||||||
|
dirty = bool(status.stdout)
|
||||||
|
fingerprint = None
|
||||||
|
if dirty:
|
||||||
|
diff = subprocess.run(
|
||||||
|
["git", "diff", "--binary", "HEAD"],
|
||||||
|
cwd=repository,
|
||||||
|
capture_output=True,
|
||||||
|
check=False,
|
||||||
|
)
|
||||||
|
untracked = subprocess.run(
|
||||||
|
["git", "ls-files", "--others", "--exclude-standard", "-z"],
|
||||||
|
cwd=repository,
|
||||||
|
capture_output=True,
|
||||||
|
check=False,
|
||||||
|
)
|
||||||
|
if diff.returncode == 0 and untracked.returncode == 0:
|
||||||
|
digest = hashlib.sha256()
|
||||||
|
digest.update(b"status\0")
|
||||||
|
digest.update(status.stdout)
|
||||||
|
digest.update(b"diff\0")
|
||||||
|
digest.update(diff.stdout)
|
||||||
|
for encoded_path in filter(None, untracked.stdout.split(b"\0")):
|
||||||
|
entry = repository / os.fsdecode(encoded_path)
|
||||||
|
if entry.is_symlink():
|
||||||
|
contents = os.fsencode(os.readlink(entry))
|
||||||
|
elif entry.is_file():
|
||||||
|
contents = entry.read_bytes()
|
||||||
|
else:
|
||||||
|
contents = b""
|
||||||
|
digest.update(b"untracked\0")
|
||||||
|
digest.update(encoded_path)
|
||||||
|
digest.update(b"\0")
|
||||||
|
digest.update(contents)
|
||||||
|
fingerprint = digest.hexdigest()
|
||||||
|
|
||||||
|
return {
|
||||||
|
"root": str(repository),
|
||||||
|
"revision": revision.stdout.strip(),
|
||||||
|
"dirty": dirty,
|
||||||
|
"working_tree_fingerprint": fingerprint,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
def load_plan(name: str) -> dict:
|
def load_plan(name: str) -> dict:
|
||||||
path = PLANS / f"{name}.json"
|
path = PLANS / f"{name}.json"
|
||||||
with path.open(encoding="utf-8") as handle:
|
with path.open(encoding="utf-8") as handle:
|
||||||
|
|
@ -71,20 +156,19 @@ def run_plan(
|
||||||
stamp = dt.datetime.now(dt.UTC).strftime("%Y%m%dT%H%M%SZ")
|
stamp = dt.datetime.now(dt.UTC).strftime("%Y%m%dT%H%M%SZ")
|
||||||
artifact = out_dir or ROOT / "build" / "benchmarks" / name / stamp
|
artifact = out_dir or ROOT / "build" / "benchmarks" / name / stamp
|
||||||
artifact.mkdir(parents=True, exist_ok=False)
|
artifact.mkdir(parents=True, exist_ok=False)
|
||||||
revision = subprocess.run(
|
brick_provenance = git_provenance(ROOT)
|
||||||
["git", "rev-parse", "HEAD"],
|
corpus_provenance = git_provenance(corpus)
|
||||||
cwd=ROOT,
|
|
||||||
text=True,
|
|
||||||
capture_output=True,
|
|
||||||
check=False,
|
|
||||||
)
|
|
||||||
manifest = {
|
manifest = {
|
||||||
"schema_version": 1,
|
"schema_version": 1,
|
||||||
"started_at": stamp,
|
"started_at": stamp,
|
||||||
"plan": name,
|
"plan": name,
|
||||||
"corpus": str(corpus),
|
"corpus": str(corpus),
|
||||||
"root": str(ROOT),
|
"root": str(ROOT),
|
||||||
"revision": revision.stdout.strip() if revision.returncode == 0 else None,
|
"revision": brick_provenance["revision"],
|
||||||
|
"provenance": {
|
||||||
|
"brick": brick_provenance,
|
||||||
|
"corpus": corpus_provenance,
|
||||||
|
},
|
||||||
"host": {
|
"host": {
|
||||||
"platform": platform.platform(),
|
"platform": platform.platform(),
|
||||||
"python": platform.python_version(),
|
"python": platform.python_version(),
|
||||||
|
|
|
||||||
|
|
@ -5,6 +5,8 @@ from __future__ import annotations
|
||||||
|
|
||||||
import importlib.util
|
import importlib.util
|
||||||
import pathlib
|
import pathlib
|
||||||
|
import subprocess
|
||||||
|
import tempfile
|
||||||
import unittest
|
import unittest
|
||||||
|
|
||||||
HERE = pathlib.Path(__file__).resolve().parent
|
HERE = pathlib.Path(__file__).resolve().parent
|
||||||
|
|
@ -15,6 +17,45 @@ SPEC.loader.exec_module(runner)
|
||||||
|
|
||||||
|
|
||||||
class RunnerTests(unittest.TestCase):
|
class RunnerTests(unittest.TestCase):
|
||||||
|
def test_git_provenance_fingerprints_dirty_corpus_state(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
|
repository = pathlib.Path(directory)
|
||||||
|
subprocess.run(["git", "init", "--quiet"], cwd=repository, check=True)
|
||||||
|
subprocess.run(
|
||||||
|
["git", "config", "user.email", "brick-tests@example.invalid"],
|
||||||
|
cwd=repository,
|
||||||
|
check=True,
|
||||||
|
)
|
||||||
|
subprocess.run(
|
||||||
|
["git", "config", "user.name", "Brick Tests"], cwd=repository, check=True
|
||||||
|
)
|
||||||
|
subprocess.run(
|
||||||
|
["git", "config", "commit.gpgsign", "false"], cwd=repository, check=True
|
||||||
|
)
|
||||||
|
tracked = repository / "tracked.cpp"
|
||||||
|
tracked.write_text("int value = 1;\n", encoding="utf-8")
|
||||||
|
subprocess.run(["git", "add", "tracked.cpp"], cwd=repository, check=True)
|
||||||
|
subprocess.run(
|
||||||
|
["git", "commit", "--quiet", "-m", "fixture"], cwd=repository, check=True
|
||||||
|
)
|
||||||
|
|
||||||
|
clean = runner.git_provenance(repository)
|
||||||
|
self.assertFalse(clean["dirty"])
|
||||||
|
self.assertIsNone(clean["working_tree_fingerprint"])
|
||||||
|
|
||||||
|
tracked.write_text("int value = 2;\n", encoding="utf-8")
|
||||||
|
(repository / "untracked.dart").write_text("void main() {}\n", encoding="utf-8")
|
||||||
|
dirty = runner.git_provenance(repository)
|
||||||
|
self.assertTrue(dirty["dirty"])
|
||||||
|
self.assertEqual(dirty["revision"], clean["revision"])
|
||||||
|
self.assertRegex(dirty["working_tree_fingerprint"], r"^[0-9a-f]{64}$")
|
||||||
|
|
||||||
|
(repository / "untracked.dart").write_text("void changed() {}\n", encoding="utf-8")
|
||||||
|
changed = runner.git_provenance(repository)
|
||||||
|
self.assertNotEqual(
|
||||||
|
changed["working_tree_fingerprint"], dirty["working_tree_fingerprint"]
|
||||||
|
)
|
||||||
|
|
||||||
def test_external_corpus_and_build_arguments_are_explicit(self) -> None:
|
def test_external_corpus_and_build_arguments_are_explicit(self) -> None:
|
||||||
corpus = pathlib.Path("/tmp/yams")
|
corpus = pathlib.Path("/tmp/yams")
|
||||||
plan = {"corpus": "tests/fixtures/mixed", "build_args": ["--exclude", "site"]}
|
plan = {"corpus": "tests/fixtures/mixed", "build_args": ["--exclude", "site"]}
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue