249 lines
9.2 KiB
Python
249 lines
9.2 KiB
Python
import json
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
SCRIPT = (
|
|
Path(__file__).parents[1]
|
|
/ "skills"
|
|
/ "brick-sast"
|
|
/ "scripts"
|
|
/ "architecture_summary.py"
|
|
)
|
|
P4_METRICS = {
|
|
"parser_state_count": 0,
|
|
"terminal_state_count": 0,
|
|
"parser_transition_count": 0,
|
|
"parser_cycle_count": 0,
|
|
"cyclic_parser_state_count": 0,
|
|
"unreachable_state_count": 0,
|
|
"control_apply_node_count": 0,
|
|
"control_apply_relationship_count": 0,
|
|
"control_apply_cycle_count": 0,
|
|
"cyclic_control_apply_node_count": 0,
|
|
"unresolved_dependency_count": 0,
|
|
}
|
|
|
|
|
|
class ArchitectureSummaryTests(unittest.TestCase):
|
|
def run_summary(self, graph: dict[str, object]) -> subprocess.CompletedProcess[str]:
|
|
return self.run_summary_raw(json.dumps(graph))
|
|
|
|
def run_summary_raw(self, graph: str) -> subprocess.CompletedProcess[str]:
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
graph_path = Path(directory) / "graph.json"
|
|
graph_path.write_text(graph, encoding="utf-8")
|
|
return subprocess.run(
|
|
[sys.executable, str(SCRIPT), str(graph_path)],
|
|
check=False,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
|
|
def test_counts_every_schema_v4_evidence_record(self) -> None:
|
|
graph = {
|
|
"schema_version": 4,
|
|
"generated_by": "test",
|
|
"root": str(Path.cwd()),
|
|
"scope": {"path": ".", "p4_frontend": "lexical"},
|
|
"nodes": [
|
|
{
|
|
"id": "repository:r",
|
|
"kind": "repository",
|
|
"name": "repository",
|
|
"path": ".",
|
|
"lines_of_code": 0,
|
|
},
|
|
{
|
|
"id": "function:a",
|
|
"kind": "function",
|
|
"name": "a",
|
|
"lines_of_code": 1,
|
|
},
|
|
{
|
|
"id": "function:b",
|
|
"kind": "function",
|
|
"name": "b",
|
|
"lines_of_code": 1,
|
|
},
|
|
],
|
|
"edges": [
|
|
{
|
|
"id": "edge:1",
|
|
"source": "function:a",
|
|
"target": "function:b",
|
|
"kind": "calls",
|
|
"evidence": [
|
|
{
|
|
"lane": "syntactic",
|
|
"backend": "tree-sitter-p4",
|
|
"confidence": "grammar-match",
|
|
"provenance": "main.p4:2",
|
|
},
|
|
{
|
|
"lane": "heuristic",
|
|
"backend": "brick-lexical",
|
|
"confidence": 0.65,
|
|
"provenance": "main.p4:2",
|
|
},
|
|
],
|
|
}
|
|
],
|
|
"metrics": {"p4": P4_METRICS},
|
|
}
|
|
result = self.run_summary(graph)
|
|
self.assertEqual(result.returncode, 0, result.stderr)
|
|
summary = json.loads(result.stdout)
|
|
self.assertEqual(summary["evidence"]["lanes"], {"heuristic": 1, "syntactic": 1})
|
|
self.assertEqual(summary["evidence"]["backends"]["brick-lexical"], 1)
|
|
self.assertEqual(summary["evidence"]["backends"]["tree-sitter-p4"], 1)
|
|
|
|
def test_rejects_malformed_schema_v4_graph_records_and_scope(self) -> None:
|
|
base = {
|
|
"schema_version": 4,
|
|
"generated_by": "test",
|
|
"root": str(Path.cwd()),
|
|
"scope": {"path": ".", "p4_frontend": "lexical"},
|
|
"nodes": [
|
|
{"id": "repository:r", "kind": "repository", "name": "r"},
|
|
{"id": "function:a", "kind": "function", "name": "a"},
|
|
],
|
|
"edges": [
|
|
{
|
|
"id": "edge:1",
|
|
"source": "repository:r",
|
|
"target": "function:a",
|
|
"kind": "defines",
|
|
"evidence": [
|
|
{
|
|
"lane": "syntactic",
|
|
"backend": "test",
|
|
"confidence": None,
|
|
"provenance": "fixture",
|
|
}
|
|
],
|
|
}
|
|
],
|
|
"metrics": {"p4": P4_METRICS},
|
|
}
|
|
cases = []
|
|
missing_repository = json.loads(json.dumps(base))
|
|
missing_repository["nodes"] = missing_repository["nodes"][1:]
|
|
cases.append((missing_repository, "exactly one repository"))
|
|
duplicate = json.loads(json.dumps(base))
|
|
duplicate["nodes"].append(dict(duplicate["nodes"][1]))
|
|
cases.append((duplicate, "duplicate node ID"))
|
|
dangling = json.loads(json.dumps(base))
|
|
dangling["edges"][0]["target"] = "function:missing"
|
|
cases.append((dangling, "unknown endpoint"))
|
|
empty_evidence = json.loads(json.dumps(base))
|
|
empty_evidence["edges"][0]["evidence"] = []
|
|
cases.append((empty_evidence, "non-empty evidence"))
|
|
malformed_evidence = json.loads(json.dumps(base))
|
|
malformed_evidence["edges"][0]["evidence"][0]["lane"] = "future"
|
|
cases.append((malformed_evidence, "unknown evidence lane"))
|
|
traversing_scope = json.loads(json.dumps(base))
|
|
traversing_scope["scope"]["path"] = "../src"
|
|
cases.append((traversing_scope, "scope.path"))
|
|
duplicate_exclusions = json.loads(json.dumps(base))
|
|
duplicate_exclusions["scope"]["excluded_directories"] = ["target", "target"]
|
|
cases.append((duplicate_exclusions, "sorted unique path components"))
|
|
for root in [
|
|
"",
|
|
"repository",
|
|
"../repository",
|
|
"C:repository",
|
|
"/repo/../other",
|
|
"//?/C:/repo",
|
|
"//?/UNC/server/share",
|
|
]:
|
|
invalid_root = json.loads(json.dumps(base))
|
|
invalid_root["root"] = root
|
|
cases.append((invalid_root, "normalized portable absolute path"))
|
|
|
|
for graph, message in cases:
|
|
with self.subTest(message=message):
|
|
result = self.run_summary(graph)
|
|
self.assertEqual(result.returncode, 2)
|
|
self.assertIn(message, result.stderr)
|
|
|
|
def test_rejects_nonfinite_evidence_confidence(self) -> None:
|
|
graph = {
|
|
"schema_version": 4,
|
|
"generated_by": "test",
|
|
"root": str(Path.cwd()),
|
|
"scope": {"path": ".", "p4_frontend": "lexical"},
|
|
"nodes": [
|
|
{"id": "repository:r", "kind": "repository", "name": "r"},
|
|
{"id": "function:a", "kind": "function", "name": "a"},
|
|
],
|
|
"edges": [
|
|
{
|
|
"id": "edge:1",
|
|
"source": "repository:r",
|
|
"target": "function:a",
|
|
"kind": "defines",
|
|
"evidence": [
|
|
{
|
|
"lane": "syntactic",
|
|
"backend": "test",
|
|
"confidence": None,
|
|
"provenance": "fixture",
|
|
}
|
|
],
|
|
}
|
|
],
|
|
"metrics": {"p4": P4_METRICS},
|
|
}
|
|
raw = json.dumps(graph)
|
|
for constant in ["NaN", "Infinity", "-Infinity", "1e999", "9" * 400]:
|
|
with self.subTest(constant=constant):
|
|
result = self.run_summary_raw(raw.replace("null", constant, 1))
|
|
self.assertEqual(result.returncode, 2)
|
|
self.assertTrue(
|
|
"non-finite JSON number" in result.stderr
|
|
or "confidence has an invalid type" in result.stderr
|
|
)
|
|
|
|
def test_rejects_schema_v1_instead_of_guessing(self) -> None:
|
|
result = self.run_summary(
|
|
{"schema_version": 1, "nodes": [], "edges": [], "metrics": {}}
|
|
)
|
|
self.assertEqual(result.returncode, 2)
|
|
self.assertIn("unsupported schema_version 1; expected 4", result.stderr)
|
|
|
|
def test_rejects_schema_v4_without_complete_p4_metrics(self) -> None:
|
|
result = self.run_summary(
|
|
{
|
|
"schema_version": 4,
|
|
"generated_by": "test",
|
|
"root": str(Path.cwd()),
|
|
"scope": {"path": ".", "p4_frontend": "lexical"},
|
|
"nodes": [],
|
|
"edges": [],
|
|
"metrics": {},
|
|
}
|
|
)
|
|
self.assertEqual(result.returncode, 2)
|
|
self.assertIn("complete metrics.p4 contract", result.stderr)
|
|
|
|
def test_rejects_unknown_p4_frontend_in_scope(self) -> None:
|
|
result = self.run_summary(
|
|
{
|
|
"schema_version": 4,
|
|
"generated_by": "test",
|
|
"root": str(Path.cwd()),
|
|
"scope": {"path": ".", "p4_frontend": "future-parser"},
|
|
"nodes": [],
|
|
"edges": [],
|
|
"metrics": {"p4": P4_METRICS},
|
|
}
|
|
)
|
|
self.assertEqual(result.returncode, 2)
|
|
self.assertIn("unsupported P4 frontend", result.stderr)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|