feat: establish first-class lexical language parity #2

Merged
trevon merged 14 commits from feat/language-parity-foundation into bench/yams-architecture-reproducibility 2026-08-30 22:09:40 +00:00
8 changed files with 156 additions and 14 deletions
Showing only changes of commit 543bafcfd4 - Show all commits

fix(scan): mask multiline block comments

trvon 2026-08-30 15:57:50 -06:00
Signed by: trevon
GPG key ID: 38686EA797D7A8A7

View file

@ -243,6 +243,7 @@ impl ScanState {
let mut brace_depth = 0_i32;
let mut rust_raw_end: Option<String> = None;
let mut javascript_template_open = false;
let mut block_comment_depth = 0_usize;
for (index, raw_line) in content.lines().enumerate() {
let line_number = index + 1;
@ -269,8 +270,9 @@ impl ScanState {
} else {
raw_line
};
let line = strip_line_comment(code_line, language);
let call_line = mask_quoted_strings(line);
let block_masked = mask_block_comments(code_line, &mut block_comment_depth);
let line = strip_line_comment(&block_masked, language);
let call_line = mask_quoted_strings(&line);
let indent = raw_line.len() - raw_line.trim_start().len();
if language == "python" && !line.trim().is_empty() {
while scopes.last().is_some_and(|scope| scope.indent >= indent) {
@ -287,7 +289,7 @@ impl ScanState {
if let Some(pattern) = patterns.enclosing_scope.as_ref()
&& let Some(name) = pattern
.captures(line)
.captures(&line)
.and_then(|captures| captures.name("name"))
&& let Some(id) = self
.symbols_by_name
@ -303,7 +305,7 @@ impl ScanState {
}
let mut type_definition_name = None;
if let Some(captures) = patterns.class.captures(line)
if let Some(captures) = patterns.class.captures(&line)
&& let Some(name) = captures.name("name")
{
type_definition_name = Some(name.as_str().to_owned());
@ -319,7 +321,7 @@ impl ScanState {
.relationships
.as_ref()
.into_iter()
.flat_map(|pattern| pattern.captures_iter(line))
.flat_map(|pattern| pattern.captures_iter(&line))
.filter_map(|captures| captures.name("names"))
.flat_map(|names| names.as_str().split(','))
.map(str::trim)
@ -353,7 +355,7 @@ impl ScanState {
let mut definition_name = None;
if let Some(name) = patterns.function.iter().find_map(|pattern| {
pattern
.captures(line)
.captures(&line)
.and_then(|captures| captures.name("name").map(|name| name.as_str().to_owned()))
}) && !is_keyword(&name)
{
@ -389,7 +391,7 @@ impl ScanState {
for captures in patterns
.import
.iter()
.flat_map(|pattern| pattern.captures_iter(line))
.flat_map(|pattern| pattern.captures_iter(&line))
{
if let Some(target) = captures.name("name").or_else(|| captures.name("name2")) {
self.pending.push(PendingRelationship {
@ -779,13 +781,93 @@ fn language_for(path: &Path) -> Option<&'static str> {
}
}
fn strip_line_comment<'a>(line: &'a str, language: &str) -> &'a str {
let marker = if language == "python" { '#' } else { '/' };
if marker == '/' {
line.split_once("//").map_or(line, |(before, _)| before)
} else {
line.split_once('#').map_or(line, |(before, _)| before)
fn strip_line_comment(line: &str, language: &str) -> String {
let chars: Vec<_> = line.chars().collect();
let mut quote = None;
let mut escaped = false;
let mut index = 0;
while index < chars.len() {
let ch = chars[index];
if let Some(active_quote) = quote {
if escaped {
escaped = false;
} else if ch == '\\' {
escaped = true;
} else if ch == active_quote {
quote = None;
}
} else if matches!(ch, '\'' | '"') && has_closing_quote(&chars, index + 1, ch) {
quote = Some(ch);
} else if language == "python" && ch == '#'
|| language != "python" && ch == '/' && chars.get(index + 1) == Some(&'/')
{
return chars[..index].iter().collect();
}
index += 1;
}
line.into()
}
fn mask_block_comments(line: &str, depth: &mut usize) -> String {
let chars: Vec<_> = line.chars().collect();
let mut result = String::with_capacity(line.len());
let mut quote = None;
let mut escaped = false;
let mut index = 0;
while index < chars.len() {
let ch = chars[index];
let next = chars.get(index + 1).copied();
if *depth > 0 {
if ch == '/' && next == Some('*') {
*depth += 1;
result.push_str(" ");
index += 2;
} else if ch == '*' && next == Some('/') {
*depth -= 1;
result.push_str(" ");
index += 2;
} else {
result.push(' ');
index += 1;
}
} else if let Some(active_quote) = quote {
result.push(ch);
if escaped {
escaped = false;
} else if ch == '\\' {
escaped = true;
} else if ch == active_quote {
quote = None;
}
index += 1;
} else if matches!(ch, '\'' | '"') && has_closing_quote(&chars, index + 1, ch) {
quote = Some(ch);
result.push(ch);
index += 1;
} else if ch == '/' && next == Some('*') {
*depth = 1;
result.push_str(" ");
index += 2;
} else {
result.push(ch);
index += 1;
}
}
result
}
fn has_closing_quote(chars: &[char], start: usize, quote: char) -> bool {
let mut escaped = false;
for ch in &chars[start..] {
if escaped {
escaped = false;
} else if *ch == '\\' {
escaped = true;
} else if *ch == quote {
return true;
}
}
false
}
fn brace_delta(line: &str) -> i32 {
@ -962,10 +1044,32 @@ fn is_keyword(name: &str) -> bool {
#[cfg(test)]
mod tests {
use super::{ScanOptions, scan, scan_with_options};
use super::{ScanOptions, mask_block_comments, scan, scan_with_options, strip_line_comment};
use crate::model::{EdgeKind, NodeKind};
use std::fs;
#[test]
fn masks_nested_block_comments_without_truncating_quoted_urls() {
let mut depth = 0;
assert_eq!(
strip_line_comment(
r#"import value from "https://example.invalid/value.js"; // trailing"#,
"javascript"
),
r#"import value from "https://example.invalid/value.js"; "#
);
assert_eq!(
mask_block_comments("code /* outer", &mut depth),
"code "
);
assert_eq!(depth, 1);
let nested = "/* inner */ end */";
let masked = mask_block_comments(nested, &mut depth);
assert_eq!(masked.len(), nested.len());
assert!(masked.trim().is_empty());
assert_eq!(depth, 0);
}
#[test]
fn scans_functions_calls_and_cycles() {
let dir = tempfile::tempdir().unwrap();

View file

@ -20,3 +20,8 @@ int start() {
int helper() {
return 0;
}
/*
struct CommentGhost {};
int comment_call() { return phantom_call(); }
*/

View file

@ -19,3 +19,8 @@ public:
int helper() {
return 0;
}
/*
class CommentGhost {};
int comment_call() { return phantom_call(); }
*/

View file

@ -17,3 +17,8 @@ abstract class Controller extends Base with Logging implements Runnable {
void helper() {}
}
/*
class CommentGhost {}
void commentCall() { phantomCall(); }
*/

View file

@ -20,3 +20,8 @@ const ignored = `
class Phantom extends Ghost {}
function fake() { phantomCall(); }
`;
/*
class CommentGhost {}
function commentCall() { phantomCall(); }
*/

View file

@ -21,3 +21,8 @@ pub const fn constant() -> usize {
pub unsafe fn unsafe_op() {}
pub extern "C" fn ffi_entry() {}
/*
struct CommentGhost;
fn comment_call() { phantom_call(); }
*/

View file

@ -25,3 +25,8 @@ const ignored = `
interface Phantom {}
function fake(): void { phantomCall(); }
`;
/*
interface CommentGhost {}
function commentCall(): void { phantomCall(); }
*/

View file

@ -266,6 +266,14 @@ fn assert_fixture(case: FixtureCase) {
"{} call baseline",
case.directory
);
assert!(
!building.nodes.iter().any(|node| matches!(
node.name.as_str(),
"CommentGhost" | "commentCall" | "comment_call" | "phantom_call"
)),
"{} block-comment clean negative",
case.directory
);
}
fn fixture_root(directory: &str) -> PathBuf {