feat: establish first-class lexical language parity #2

Merged
trevon merged 14 commits from feat/language-parity-foundation into bench/yams-architecture-reproducibility 2026-08-30 22:09:40 +00:00
11 changed files with 177 additions and 34 deletions
Showing only changes of commit 02ac22b8de - Show all commits

feat(scan): promote JavaScript and TypeScript forms

trvon 2026-08-30 15:51:59 -06:00
Signed by: trevon
GPG key ID: 38686EA797D7A8A7

View file

@ -22,8 +22,8 @@ frontend; Brick does not guess that ownership.
| Rust | Structs, enums, traits, ordinary/async/const/unsafe/extern functions, simple inherent/trait impl ownership, ordinary/turbofish/macro calls, braced `use` resolution, and raw-string masking | module ownership, unions/type aliases, nested generic impl headers, glob imports, and macro definitions | first-class target | | Rust | Structs, enums, traits, ordinary/async/const/unsafe/extern functions, simple inherent/trait impl ownership, ordinary/turbofish/macro calls, braced `use` resolution, and raw-string masking | module ownership, unions/type aliases, nested generic impl headers, glob imports, and macro definitions | first-class target |
| C | Distinct `.c` identity, includes, structs/unions/enums, narrow function forms, and ordinary calls | anonymous typedef aggregates, function pointers, C23 attributes, macros, and multiline declarations | first-class target | | C | Distinct `.c` identity, includes, structs/unions/enums, narrow function forms, and ordinary calls | anonymous typedef aggregates, function pointers, C23 attributes, macros, and multiline declarations | first-class target |
| C++ | Classes/structs/unions/enums, `final`, constructors/destructors/operators, override-qualified methods, multiple inheritance, ordinary calls, and local include resolution | namespaces, overload disambiguation, lambdas, concepts/requires, complex template bases, and multiline declarations | first-class target | | C++ | Classes/structs/unions/enums, `final`, constructors/destructors/operators, override-qualified methods, multiple inheritance, ordinary calls, and local include resolution | namespaces, overload disambiguation, lambdas, concepts/requires, complex template bases, and multiline declarations | first-class target |
| JavaScript | Named functions, simple classes/extends, ES imports, and call-shaped expressions | methods, arrows/function expressions, CommonJS, dynamic imports, re-exports, decorators, and template literals | first-class target | | JavaScript | Default/exported classes, inheritance, owned methods/constructors, named functions, arrows, ES imports/re-exports, CommonJS `require`, dynamic imports, ordinary calls, and multiline template masking | function expressions, computed methods, decorators, JSX structure, package exports, and type-aware member calls | first-class target |
| TypeScript | JavaScript basics plus TypeScript file extensions and simple imports | interfaces, type aliases, enums/namespaces, generic/abstract declarations, overloads, decorators, and path mappings | first-class target | | TypeScript | JavaScript forms plus interfaces/enums/namespaces, generic and abstract classes/functions/methods, `implements`, type imports/re-exports, and typed arrows | type aliases, overload declarations, decorators, declaration merging, TSX structure, and path mappings | first-class target |
| Python | Basic classes, functions/async functions, imports, calls, and indentation scopes | multiple bases, qualified ownership, decorators, multiline strings, relative/dynamic imports, and complete multi-import handling | secondary | | Python | Basic classes, functions/async functions, imports, calls, and indentation scopes | multiple bases, qualified ownership, decorators, multiline strings, relative/dynamic imports, and complete multi-import handling | secondary |
| Swift | File recognition and incidental matching of some class/function syntax | no dedicated patterns; imports, protocols, actors, enums, extensions, async/generic forms, and ownership are incomplete | secondary | | Swift | File recognition and incidental matching of some class/function syntax | no dedicated patterns; imports, protocols, actors, enums, extensions, async/generic forms, and ownership are incomplete | secondary |

View file

@ -242,6 +242,7 @@ impl ScanState {
let mut scopes: Vec<Scope> = Vec::new(); let mut scopes: Vec<Scope> = Vec::new();
let mut brace_depth = 0_i32; let mut brace_depth = 0_i32;
let mut rust_raw_end: Option<String> = None; let mut rust_raw_end: Option<String> = None;
let mut javascript_template_open = false;
for (index, raw_line) in content.lines().enumerate() { for (index, raw_line) in content.lines().enumerate() {
let line_number = index + 1; let line_number = index + 1;
@ -251,7 +252,12 @@ impl ScanState {
} }
continue; continue;
} }
let code_line = if language == "rust" { let template_masked;
let code_line = if matches!(language, "javascript" | "typescript") {
template_masked =
mask_javascript_templates(raw_line, &mut javascript_template_open);
template_masked.as_str()
} else if language == "rust" {
if let Some((start, content_start, delimiter)) = rust_raw_string_start(raw_line) { if let Some((start, content_start, delimiter)) = rust_raw_string_start(raw_line) {
if !raw_line[content_start..].contains(&delimiter) { if !raw_line[content_start..].contains(&delimiter) {
rust_raw_end = Some(delimiter); rust_raw_end = Some(delimiter);
@ -378,7 +384,11 @@ impl ScanState {
.map(|scope| scope.id.clone()) .map(|scope| scope.id.clone())
.unwrap_or_else(|| file_id.into()); .unwrap_or_else(|| file_id.into());
for captures in patterns.import.captures_iter(line) { for captures in patterns
.import
.iter()
.flat_map(|pattern| pattern.captures_iter(line))
{
if let Some(target) = captures.name("name").or_else(|| captures.name("name2")) { if let Some(target) = captures.name("name").or_else(|| captures.name("name2")) {
self.pending.push(PendingRelationship { self.pending.push(PendingRelationship {
source: file_id.into(), source: file_id.into(),
@ -623,7 +633,7 @@ struct Scope {
struct Patterns { struct Patterns {
class: Regex, class: Regex,
function: Vec<Regex>, function: Vec<Regex>,
import: Regex, import: Vec<Regex>,
call: Vec<Regex>, call: Vec<Regex>,
relationships: Option<Regex>, relationships: Option<Regex>,
enclosing_scope: Option<Regex>, enclosing_scope: Option<Regex>,
@ -631,11 +641,13 @@ struct Patterns {
impl Patterns { impl Patterns {
fn new(language: &str) -> Self { fn new(language: &str) -> Self {
let (class, functions, import, relationships) = match language { let (class, functions, imports, relationships) = match language {
"python" => ( "python" => (
r"^\s*class\s+(?P<name>[A-Za-z_]\w*)(?:\((?P<base>[A-Za-z_][\w.]*)\))?", r"^\s*class\s+(?P<name>[A-Za-z_]\w*)(?:\((?P<base>[A-Za-z_][\w.]*)\))?",
vec![r"^\s*(?:async\s+)?def\s+(?P<name>[A-Za-z_]\w*)\s*\("], vec![r"^\s*(?:async\s+)?def\s+(?P<name>[A-Za-z_]\w*)\s*\("],
r"^\s*(?:from\s+(?P<name>[A-Za-z_][\w.]*)\s+import|import\s+(?P<name2>[A-Za-z_][\w.]*))", vec![
r"^\s*(?:from\s+(?P<name>[A-Za-z_][\w.]*)\s+import|import\s+(?P<name2>[A-Za-z_][\w.]*))",
],
None, None,
), ),
"rust" => ( "rust" => (
@ -643,13 +655,13 @@ impl Patterns {
vec![ vec![
r#"^\s*(?:pub(?:\([^)]*\))?\s+)?(?:(?:const|async|unsafe)\s+)*(?:extern(?:\s+"[^"]+")?\s+)?fn\s+(?P<name>[A-Za-z_]\w*)\s*(?:<[^>]*>)?\s*\("#, r#"^\s*(?:pub(?:\([^)]*\))?\s+)?(?:(?:const|async|unsafe)\s+)*(?:extern(?:\s+"[^"]+")?\s+)?fn\s+(?P<name>[A-Za-z_]\w*)\s*(?:<[^>]*>)?\s*\("#,
], ],
r"^\s*use\s+(?P<name>[A-Za-z_][\w:]*)", vec![r"^\s*use\s+(?P<name>[A-Za-z_][\w:]*)"],
None, None,
), ),
"c" => ( "c" => (
r"^\s*(?:typedef\s+)?(?:struct|union|enum)\s+(?P<name>[A-Za-z_]\w*)", r"^\s*(?:typedef\s+)?(?:struct|union|enum)\s+(?P<name>[A-Za-z_]\w*)",
vec![r"^\s*(?:[\w*&]+\s+)+(?P<name>[A-Za-z_]\w*)\s*\([^;{}]*\)\s*\{?\s*$"], vec![r"^\s*(?:[\w*&]+\s+)+(?P<name>[A-Za-z_]\w*)\s*\([^;{}]*\)\s*\{?\s*$"],
r#"^\s*#\s*include\s*[<\"](?P<name>[^>\"]+)[>\"]"#, vec![r#"^\s*#\s*include\s*[<\"](?P<name>[^>\"]+)[>\"]"#],
None, None,
), ),
"cpp" => ( "cpp" => (
@ -659,14 +671,30 @@ impl Patterns {
r"^\s*(?P<name>~?[A-Za-z_]\w*)\s*\([^;{}]*\)\s*(?::[^{}]*)?\s*(?:noexcept\s*)?(?:\{\s*\}|\{|=\s*(?:default|delete)\s*;)\s*$", r"^\s*(?P<name>~?[A-Za-z_]\w*)\s*\([^;{}]*\)\s*(?::[^{}]*)?\s*(?:noexcept\s*)?(?:\{\s*\}|\{|=\s*(?:default|delete)\s*;)\s*$",
r"^\s*(?:[\w:<>,*&]+\s+)+(?P<name>operator\s*(?:[^\s(]+|\(\)))\s*\([^;{}]*\)\s*(?:const\s*)?(?:noexcept\s*)?(?:\{|=\s*(?:default|delete)\s*;)\s*$", r"^\s*(?:[\w:<>,*&]+\s+)+(?P<name>operator\s*(?:[^\s(]+|\(\)))\s*\([^;{}]*\)\s*(?:const\s*)?(?:noexcept\s*)?(?:\{|=\s*(?:default|delete)\s*;)\s*$",
], ],
r#"^\s*#\s*include\s*[<\"](?P<name>[^>\"]+)[>\"]"#, vec![r#"^\s*#\s*include\s*[<\"](?P<name>[^>\"]+)[>\"]"#],
Some(r":\s*(?P<names>[^\{]+)"), Some(r":\s*(?P<names>[^\{]+)"),
), ),
"javascript" | "typescript" => ( "javascript" => (
r"^\s*(?:export\s+)?class\s+(?P<name>[A-Za-z_$][\w$]*)(?:\s+extends\s+(?P<base>[A-Za-z_$][\w$]*))?", r"^\s*(?:export\s+)?(?:default\s+)?class\s+(?P<name>[A-Za-z_$][\w$]*)(?:\s+extends\s+(?P<base>[A-Za-z_$][\w$]*))?",
vec![r"^\s*(?:export\s+)?(?:async\s+)?function\s+(?P<name>[A-Za-z_$][\w$]*)\s*\("], vec![
r#"^\s*import(?:[^'\"]*from\s*)?['\"](?P<name>[^'\"]+)['\"]"#, r"^\s*(?:export\s+)?(?:default\s+)?(?:async\s+)?function\s+(?P<name>[A-Za-z_$][\w$]*)\s*\(",
None, r"^\s*(?:(?:static|async|get|set)\s+)*(?P<name>constructor|[A-Za-z_$][\w$]*)\s*\([^)]*\)\s*\{",
r"^\s*(?:export\s+)?(?:const|let|var)\s+(?P<name>[A-Za-z_$][\w$]*)\s*=\s*(?:async\s*)?(?:\([^)]*\)|[A-Za-z_$][\w$]*)\s*=>",
],
javascript_import_patterns(),
Some(r"\bextends\s+(?P<names>[A-Za-z_$][\w$]*)"),
),
"typescript" => (
r"^\s*(?:export\s+)?(?:default\s+)?(?:(?:abstract|declare)\s+)?(?:class|interface|enum|namespace)\s+(?P<name>[A-Za-z_$][\w$]*)(?:\s*<[^>{}]+>)?",
vec![
r"^\s*(?:export\s+)?(?:default\s+)?(?:async\s+)?function\s+(?P<name>[A-Za-z_$][\w$]*)\s*(?:<[^>{}]+>)?\s*\(",
r"^\s*(?:(?:public|protected|private|static|abstract|async|get|set|readonly|override)\s+)*(?P<name>constructor|[A-Za-z_$][\w$]*)\s*(?:<[^>{}]+>)?\s*\([^)]*\)\s*(?::[^;{]+)?\s*\{",
r"^\s*(?:export\s+)?(?:const|let|var)\s+(?P<name>[A-Za-z_$][\w$]*)\s*(?::[^=]+)?=\s*(?:async\s*)?(?:<[^>{}]+>\s*)?(?:\([^)]*\)|[A-Za-z_$][\w$]*)\s*(?::[^=]+)?=>",
],
javascript_import_patterns(),
Some(
r"\b(?:extends|implements)\s+(?P<names>[A-Za-z_$][\w$]*(?:<[^>{}]+>)?(?:\s*,\s*[A-Za-z_$][\w$]*(?:<[^>{}]+>)?)*)",
),
), ),
"dart" => ( "dart" => (
r"^\s*(?:(?:abstract|base|final|interface|sealed)\s+)?(?:class|mixin|enum|extension(?:\s+type)?)\s+(?P<name>[A-Za-z_]\w*)(?:\s+extends\s+(?P<base>[A-Za-z_]\w*))?", r"^\s*(?:(?:abstract|base|final|interface|sealed)\s+)?(?:class|mixin|enum|extension(?:\s+type)?)\s+(?P<name>[A-Za-z_]\w*)(?:\s+extends\s+(?P<base>[A-Za-z_]\w*))?",
@ -676,7 +704,7 @@ impl Patterns {
r"^\s*(?:(?:external|static|abstract|covariant|late)\s+)*(?:(?:[A-Za-z_]\w*(?:<[^>{};()]+>)?[?!]?)\s+)?get\s+(?P<name>[A-Za-z_]\w*)\s*(?:=>|\{)", r"^\s*(?:(?:external|static|abstract|covariant|late)\s+)*(?:(?:[A-Za-z_]\w*(?:<[^>{};()]+>)?[?!]?)\s+)?get\s+(?P<name>[A-Za-z_]\w*)\s*(?:=>|\{)",
r"^\s*(?:(?:external|static|abstract|covariant|late)\s+)*set\s+(?P<name>[A-Za-z_]\w*)\s*\([^;{}]*\)\s*(?:=>|\{|;)", r"^\s*(?:(?:external|static|abstract|covariant|late)\s+)*set\s+(?P<name>[A-Za-z_]\w*)\s*\([^;{}]*\)\s*(?:=>|\{|;)",
], ],
r#"^\s*(?:import|export|part)\s+['\"](?P<name>[^'\"]+)['\"]"#, vec![r#"^\s*(?:import|export|part)\s+['\"](?P<name>[^'\"]+)['\"]"#],
Some( Some(
r"\b(?:extends|with|implements|on)\s+(?P<names>[A-Za-z_]\w*(?:\s*,\s*[A-Za-z_]\w*)*)", r"\b(?:extends|with|implements|on)\s+(?P<names>[A-Za-z_]\w*(?:\s*,\s*[A-Za-z_]\w*)*)",
), ),
@ -686,7 +714,7 @@ impl Patterns {
vec![ vec![
r"^\s*(?:[\w:<>,*&]+\s+)+(?P<name>[A-Za-z_]\w*(?:::\w+)*)\s*\([^;{}]*\)\s*(?:const\s*)?(?:noexcept\s*)?\{?\s*$", r"^\s*(?:[\w:<>,*&]+\s+)+(?P<name>[A-Za-z_]\w*(?:::\w+)*)\s*\([^;{}]*\)\s*(?:const\s*)?(?:noexcept\s*)?\{?\s*$",
], ],
r#"^\s*#\s*include\s*[<\"](?P<name>[^>\"]+)[>\"]"#, vec![r#"^\s*#\s*include\s*[<\"](?P<name>[^>\"]+)[>\"]"#],
None, None,
), ),
}; };
@ -703,7 +731,10 @@ impl Patterns {
.into_iter() .into_iter()
.map(|pattern| Regex::new(pattern).expect("function regex is static")) .map(|pattern| Regex::new(pattern).expect("function regex is static"))
.collect(), .collect(),
import: Regex::new(import).expect("import regex is static"), import: imports
.into_iter()
.map(|pattern| Regex::new(pattern).expect("import regex is static"))
.collect(),
call: calls call: calls
.into_iter() .into_iter()
.map(|pattern| Regex::new(pattern).expect("call regex is static")) .map(|pattern| Regex::new(pattern).expect("call regex is static"))
@ -720,6 +751,15 @@ impl Patterns {
} }
} }
fn javascript_import_patterns() -> Vec<&'static str> {
vec![
r#"^\s*import(?:\s+type)?(?:[^'\"]*from\s*)?['\"](?P<name>[^'\"]+)['\"]"#,
r#"^\s*export(?:\s+type)?\s+[^'\"]*from\s*['\"](?P<name>[^'\"]+)['\"]"#,
r#"\brequire\s*\(\s*['\"](?P<name>[^'\"]+)['\"]\s*\)"#,
r#"\bimport\s*\(\s*['\"](?P<name>[^'\"]+)['\"]\s*\)"#,
]
}
fn language_for(path: &Path) -> Option<&'static str> { fn language_for(path: &Path) -> Option<&'static str> {
match path.extension()?.to_str()? { match path.extension()?.to_str()? {
"rs" => Some("rust"), "rs" => Some("rust"),
@ -775,6 +815,29 @@ fn mask_quoted_strings(line: &str) -> String {
result result
} }
fn mask_javascript_templates(line: &str, open: &mut bool) -> String {
let mut result = String::with_capacity(line.len());
let mut escaped = false;
for ch in line.chars() {
if *open {
if escaped {
escaped = false;
} else if ch == '\\' {
escaped = true;
} else if ch == '`' {
*open = false;
}
result.push(' ');
} else if ch == '`' {
*open = true;
result.push(' ');
} else {
result.push(ch);
}
}
result
}
fn rust_raw_string_start(line: &str) -> Option<(usize, usize, String)> { fn rust_raw_string_start(line: &str) -> Option<(usize, usize, String)> {
let bytes = line.as_bytes(); let bytes = line.as_bytes();
for start in 0..bytes.len() { for start in 0..bytes.len() {
@ -820,6 +883,8 @@ fn clean_relationship_target(value: &str, language: &str) -> String {
.filter(|part| !matches!(*part, "public" | "protected" | "private" | "virtual")) .filter(|part| !matches!(*part, "public" | "protected" | "private" | "virtual"))
.collect::<Vec<_>>() .collect::<Vec<_>>()
.join(" ") .join(" ")
} else if matches!(language, "javascript" | "typescript") {
target.split('<').next().unwrap_or(&target).trim().into()
} else { } else {
target target
} }
@ -885,6 +950,8 @@ fn is_keyword(name: &str) -> bool {
| "assert" | "assert"
| "assert_eq" | "assert_eq"
| "println" | "println"
| "import"
| "require"
) )
} }

View file

@ -0,0 +1 @@
export function api() {}

View file

@ -0,0 +1 @@
export class Base {}

View file

@ -1,9 +1,22 @@
import { Base } from './base.js'; import { Base } from './base.js';
export { api } from './api.js';
class Worker extends Base {} const legacy = require('./legacy.cjs');
const lazy = import('./lazy.js');
function start() { export default class Worker extends Base {
helper(); constructor() {}
start() {
helper();
}
} }
const arrow = () => helper();
function helper() {} function helper() {}
const ignored = `
class Phantom extends Ghost {}
function fake() { phantomCall(); }
`;

View file

@ -0,0 +1 @@
export function lazy() {}

View file

@ -0,0 +1 @@
module.exports = {};

View file

@ -0,0 +1 @@
export class Base<T> {}

View file

@ -1,9 +1,27 @@
import { Base } from './base.js'; import type { Base } from './base.js';
export type { Contract } from './contract.js';
class Worker extends Base {} interface Runnable {
run<T>(value: T): void;
}
function start(): void { export abstract class Worker<T> extends Base<T> implements Runnable {
helper(); constructor() {}
run<U>(value: U): void {
helper();
}
}
const arrow = <T>(value: T): T => identity(value);
function identity<T>(value: T): T {
return value;
} }
function helper(): void {} function helper(): void {}
const ignored = `
interface Phantom {}
function fake(): void { phantomCall(); }
`;

View file

@ -0,0 +1 @@
export interface Contract {}

View file

@ -61,20 +61,20 @@ fn first_class_language_fixtures_have_stable_baseline_evidence() {
FixtureCase { FixtureCase {
directory: "javascript", directory: "javascript",
language: "javascript", language: "javascript",
classes: &["Worker"], classes: &["Base", "Worker"],
functions: &["helper", "start"], functions: &["api", "arrow", "constructor", "helper", "lazy", "start"],
imports: 1, imports: 4,
inherits: 1, inherits: 1,
calls: 1, calls: 2,
}, },
FixtureCase { FixtureCase {
directory: "typescript", directory: "typescript",
language: "typescript", language: "typescript",
classes: &["Worker"], classes: &["Base", "Contract", "Runnable", "Worker"],
functions: &["helper", "start"], functions: &["arrow", "constructor", "helper", "identity", "run"],
imports: 1, imports: 2,
inherits: 1, inherits: 2,
calls: 1, calls: 2,
}, },
]; ];
@ -141,6 +141,45 @@ fn cpp_fixture_models_member_ownership_without_declaration_calls() {
})); }));
} }
#[test]
fn javascript_and_typescript_model_members_without_template_phantoms() {
for directory in ["javascript", "typescript"] {
let building = scan(&fixture_root(directory)).unwrap();
let worker = building
.nodes
.iter()
.find(|node| node.kind == NodeKind::Class && node.name == "Worker")
.unwrap();
let member_names: Vec<_> = building
.nodes
.iter()
.filter(|node| {
node.kind == NodeKind::Function && node.parent.as_ref() == Some(&worker.id)
})
.map(|node| node.name.as_str())
.collect();
assert_eq!(member_names.len(), 2, "{directory} member ownership");
assert!(member_names.contains(&"constructor"));
assert!(
!building
.nodes
.iter()
.any(|node| { matches!(node.name.as_str(), "Phantom" | "fake" | "phantomCall") })
);
assert!(
building
.edges
.iter()
.filter(|edge| edge.kind == EdgeKind::Imports)
.all(|edge| building
.nodes
.iter()
.any(|node| { node.id == edge.target && node.kind == NodeKind::File }))
);
}
}
fn assert_fixture(case: FixtureCase) { fn assert_fixture(case: FixtureCase) {
let root = fixture_root(case.directory); let root = fixture_root(case.directory);
let building = scan(&root).unwrap(); let building = scan(&root).unwrap();