feat: establish first-class lexical language parity #2

Merged
trevon merged 14 commits from feat/language-parity-foundation into bench/yams-architecture-reproducibility 2026-08-30 22:09:40 +00:00
8 changed files with 156 additions and 14 deletions
Showing only changes of commit 543bafcfd4 - Show all commits

fix(scan): mask multiline block comments

trvon 2026-08-30 15:57:50 -06:00
Signed by: trevon
GPG key ID: 38686EA797D7A8A7

View file

@ -243,6 +243,7 @@ impl ScanState {
let mut brace_depth = 0_i32; let mut brace_depth = 0_i32;
let mut rust_raw_end: Option<String> = None; let mut rust_raw_end: Option<String> = None;
let mut javascript_template_open = false; let mut javascript_template_open = false;
let mut block_comment_depth = 0_usize;
for (index, raw_line) in content.lines().enumerate() { for (index, raw_line) in content.lines().enumerate() {
let line_number = index + 1; let line_number = index + 1;
@ -269,8 +270,9 @@ impl ScanState {
} else { } else {
raw_line raw_line
}; };
let line = strip_line_comment(code_line, language); let block_masked = mask_block_comments(code_line, &mut block_comment_depth);
let call_line = mask_quoted_strings(line); let line = strip_line_comment(&block_masked, language);
let call_line = mask_quoted_strings(&line);
let indent = raw_line.len() - raw_line.trim_start().len(); let indent = raw_line.len() - raw_line.trim_start().len();
if language == "python" && !line.trim().is_empty() { if language == "python" && !line.trim().is_empty() {
while scopes.last().is_some_and(|scope| scope.indent >= indent) { while scopes.last().is_some_and(|scope| scope.indent >= indent) {
@ -287,7 +289,7 @@ impl ScanState {
if let Some(pattern) = patterns.enclosing_scope.as_ref() if let Some(pattern) = patterns.enclosing_scope.as_ref()
&& let Some(name) = pattern && let Some(name) = pattern
.captures(line) .captures(&line)
.and_then(|captures| captures.name("name")) .and_then(|captures| captures.name("name"))
&& let Some(id) = self && let Some(id) = self
.symbols_by_name .symbols_by_name
@ -303,7 +305,7 @@ impl ScanState {
} }
let mut type_definition_name = None; let mut type_definition_name = None;
if let Some(captures) = patterns.class.captures(line) if let Some(captures) = patterns.class.captures(&line)
&& let Some(name) = captures.name("name") && let Some(name) = captures.name("name")
{ {
type_definition_name = Some(name.as_str().to_owned()); type_definition_name = Some(name.as_str().to_owned());
@ -319,7 +321,7 @@ impl ScanState {
.relationships .relationships
.as_ref() .as_ref()
.into_iter() .into_iter()
.flat_map(|pattern| pattern.captures_iter(line)) .flat_map(|pattern| pattern.captures_iter(&line))
.filter_map(|captures| captures.name("names")) .filter_map(|captures| captures.name("names"))
.flat_map(|names| names.as_str().split(',')) .flat_map(|names| names.as_str().split(','))
.map(str::trim) .map(str::trim)
@ -353,7 +355,7 @@ impl ScanState {
let mut definition_name = None; let mut definition_name = None;
if let Some(name) = patterns.function.iter().find_map(|pattern| { if let Some(name) = patterns.function.iter().find_map(|pattern| {
pattern pattern
.captures(line) .captures(&line)
.and_then(|captures| captures.name("name").map(|name| name.as_str().to_owned())) .and_then(|captures| captures.name("name").map(|name| name.as_str().to_owned()))
}) && !is_keyword(&name) }) && !is_keyword(&name)
{ {
@ -389,7 +391,7 @@ impl ScanState {
for captures in patterns for captures in patterns
.import .import
.iter() .iter()
.flat_map(|pattern| pattern.captures_iter(line)) .flat_map(|pattern| pattern.captures_iter(&line))
{ {
if let Some(target) = captures.name("name").or_else(|| captures.name("name2")) { if let Some(target) = captures.name("name").or_else(|| captures.name("name2")) {
self.pending.push(PendingRelationship { self.pending.push(PendingRelationship {
@ -779,13 +781,93 @@ fn language_for(path: &Path) -> Option<&'static str> {
} }
} }
fn strip_line_comment<'a>(line: &'a str, language: &str) -> &'a str { fn strip_line_comment(line: &str, language: &str) -> String {
let marker = if language == "python" { '#' } else { '/' }; let chars: Vec<_> = line.chars().collect();
if marker == '/' { let mut quote = None;
line.split_once("//").map_or(line, |(before, _)| before) let mut escaped = false;
} else { let mut index = 0;
line.split_once('#').map_or(line, |(before, _)| before) while index < chars.len() {
let ch = chars[index];
if let Some(active_quote) = quote {
if escaped {
escaped = false;
} else if ch == '\\' {
escaped = true;
} else if ch == active_quote {
quote = None;
}
} else if matches!(ch, '\'' | '"') && has_closing_quote(&chars, index + 1, ch) {
quote = Some(ch);
} else if language == "python" && ch == '#'
|| language != "python" && ch == '/' && chars.get(index + 1) == Some(&'/')
{
return chars[..index].iter().collect();
}
index += 1;
} }
line.into()
}
fn mask_block_comments(line: &str, depth: &mut usize) -> String {
let chars: Vec<_> = line.chars().collect();
let mut result = String::with_capacity(line.len());
let mut quote = None;
let mut escaped = false;
let mut index = 0;
while index < chars.len() {
let ch = chars[index];
let next = chars.get(index + 1).copied();
if *depth > 0 {
if ch == '/' && next == Some('*') {
*depth += 1;
result.push_str(" ");
index += 2;
} else if ch == '*' && next == Some('/') {
*depth -= 1;
result.push_str(" ");
index += 2;
} else {
result.push(' ');
index += 1;
}
} else if let Some(active_quote) = quote {
result.push(ch);
if escaped {
escaped = false;
} else if ch == '\\' {
escaped = true;
} else if ch == active_quote {
quote = None;
}
index += 1;
} else if matches!(ch, '\'' | '"') && has_closing_quote(&chars, index + 1, ch) {
quote = Some(ch);
result.push(ch);
index += 1;
} else if ch == '/' && next == Some('*') {
*depth = 1;
result.push_str(" ");
index += 2;
} else {
result.push(ch);
index += 1;
}
}
result
}
fn has_closing_quote(chars: &[char], start: usize, quote: char) -> bool {
let mut escaped = false;
for ch in &chars[start..] {
if escaped {
escaped = false;
} else if *ch == '\\' {
escaped = true;
} else if *ch == quote {
return true;
}
}
false
} }
fn brace_delta(line: &str) -> i32 { fn brace_delta(line: &str) -> i32 {
@ -962,10 +1044,32 @@ fn is_keyword(name: &str) -> bool {
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::{ScanOptions, scan, scan_with_options}; use super::{ScanOptions, mask_block_comments, scan, scan_with_options, strip_line_comment};
use crate::model::{EdgeKind, NodeKind}; use crate::model::{EdgeKind, NodeKind};
use std::fs; use std::fs;
#[test]
fn masks_nested_block_comments_without_truncating_quoted_urls() {
let mut depth = 0;
assert_eq!(
strip_line_comment(
r#"import value from "https://example.invalid/value.js"; // trailing"#,
"javascript"
),
r#"import value from "https://example.invalid/value.js"; "#
);
assert_eq!(
mask_block_comments("code /* outer", &mut depth),
"code "
);
assert_eq!(depth, 1);
let nested = "/* inner */ end */";
let masked = mask_block_comments(nested, &mut depth);
assert_eq!(masked.len(), nested.len());
assert!(masked.trim().is_empty());
assert_eq!(depth, 0);
}
#[test] #[test]
fn scans_functions_calls_and_cycles() { fn scans_functions_calls_and_cycles() {
let dir = tempfile::tempdir().unwrap(); let dir = tempfile::tempdir().unwrap();

View file

@ -20,3 +20,8 @@ int start() {
int helper() { int helper() {
return 0; return 0;
} }
/*
struct CommentGhost {};
int comment_call() { return phantom_call(); }
*/

View file

@ -19,3 +19,8 @@ public:
int helper() { int helper() {
return 0; return 0;
} }
/*
class CommentGhost {};
int comment_call() { return phantom_call(); }
*/

View file

@ -17,3 +17,8 @@ abstract class Controller extends Base with Logging implements Runnable {
void helper() {} void helper() {}
} }
/*
class CommentGhost {}
void commentCall() { phantomCall(); }
*/

View file

@ -20,3 +20,8 @@ const ignored = `
class Phantom extends Ghost {} class Phantom extends Ghost {}
function fake() { phantomCall(); } function fake() { phantomCall(); }
`; `;
/*
class CommentGhost {}
function commentCall() { phantomCall(); }
*/

View file

@ -21,3 +21,8 @@ pub const fn constant() -> usize {
pub unsafe fn unsafe_op() {} pub unsafe fn unsafe_op() {}
pub extern "C" fn ffi_entry() {} pub extern "C" fn ffi_entry() {}
/*
struct CommentGhost;
fn comment_call() { phantom_call(); }
*/

View file

@ -25,3 +25,8 @@ const ignored = `
interface Phantom {} interface Phantom {}
function fake(): void { phantomCall(); } function fake(): void { phantomCall(); }
`; `;
/*
interface CommentGhost {}
function commentCall(): void { phantomCall(); }
*/

View file

@ -266,6 +266,14 @@ fn assert_fixture(case: FixtureCase) {
"{} call baseline", "{} call baseline",
case.directory case.directory
); );
assert!(
!building.nodes.iter().any(|node| matches!(
node.name.as_str(),
"CommentGhost" | "commentCall" | "comment_call" | "phantom_call"
)),
"{} block-comment clean negative",
case.directory
);
} }
fn fixture_root(directory: &str) -> PathBuf { fn fixture_root(directory: &str) -> PathBuf {