feat: establish first-class lexical language parity #2
8 changed files with 156 additions and 14 deletions
fix(scan): mask multiline block comments
commit
543bafcfd4
132
src/scan.rs
132
src/scan.rs
|
|
@ -243,6 +243,7 @@ impl ScanState {
|
|||
let mut brace_depth = 0_i32;
|
||||
let mut rust_raw_end: Option<String> = None;
|
||||
let mut javascript_template_open = false;
|
||||
let mut block_comment_depth = 0_usize;
|
||||
|
||||
for (index, raw_line) in content.lines().enumerate() {
|
||||
let line_number = index + 1;
|
||||
|
|
@ -269,8 +270,9 @@ impl ScanState {
|
|||
} else {
|
||||
raw_line
|
||||
};
|
||||
let line = strip_line_comment(code_line, language);
|
||||
let call_line = mask_quoted_strings(line);
|
||||
let block_masked = mask_block_comments(code_line, &mut block_comment_depth);
|
||||
let line = strip_line_comment(&block_masked, language);
|
||||
let call_line = mask_quoted_strings(&line);
|
||||
let indent = raw_line.len() - raw_line.trim_start().len();
|
||||
if language == "python" && !line.trim().is_empty() {
|
||||
while scopes.last().is_some_and(|scope| scope.indent >= indent) {
|
||||
|
|
@ -287,7 +289,7 @@ impl ScanState {
|
|||
|
||||
if let Some(pattern) = patterns.enclosing_scope.as_ref()
|
||||
&& let Some(name) = pattern
|
||||
.captures(line)
|
||||
.captures(&line)
|
||||
.and_then(|captures| captures.name("name"))
|
||||
&& let Some(id) = self
|
||||
.symbols_by_name
|
||||
|
|
@ -303,7 +305,7 @@ impl ScanState {
|
|||
}
|
||||
|
||||
let mut type_definition_name = None;
|
||||
if let Some(captures) = patterns.class.captures(line)
|
||||
if let Some(captures) = patterns.class.captures(&line)
|
||||
&& let Some(name) = captures.name("name")
|
||||
{
|
||||
type_definition_name = Some(name.as_str().to_owned());
|
||||
|
|
@ -319,7 +321,7 @@ impl ScanState {
|
|||
.relationships
|
||||
.as_ref()
|
||||
.into_iter()
|
||||
.flat_map(|pattern| pattern.captures_iter(line))
|
||||
.flat_map(|pattern| pattern.captures_iter(&line))
|
||||
.filter_map(|captures| captures.name("names"))
|
||||
.flat_map(|names| names.as_str().split(','))
|
||||
.map(str::trim)
|
||||
|
|
@ -353,7 +355,7 @@ impl ScanState {
|
|||
let mut definition_name = None;
|
||||
if let Some(name) = patterns.function.iter().find_map(|pattern| {
|
||||
pattern
|
||||
.captures(line)
|
||||
.captures(&line)
|
||||
.and_then(|captures| captures.name("name").map(|name| name.as_str().to_owned()))
|
||||
}) && !is_keyword(&name)
|
||||
{
|
||||
|
|
@ -389,7 +391,7 @@ impl ScanState {
|
|||
for captures in patterns
|
||||
.import
|
||||
.iter()
|
||||
.flat_map(|pattern| pattern.captures_iter(line))
|
||||
.flat_map(|pattern| pattern.captures_iter(&line))
|
||||
{
|
||||
if let Some(target) = captures.name("name").or_else(|| captures.name("name2")) {
|
||||
self.pending.push(PendingRelationship {
|
||||
|
|
@ -779,13 +781,93 @@ fn language_for(path: &Path) -> Option<&'static str> {
|
|||
}
|
||||
}
|
||||
|
||||
fn strip_line_comment<'a>(line: &'a str, language: &str) -> &'a str {
|
||||
let marker = if language == "python" { '#' } else { '/' };
|
||||
if marker == '/' {
|
||||
line.split_once("//").map_or(line, |(before, _)| before)
|
||||
} else {
|
||||
line.split_once('#').map_or(line, |(before, _)| before)
|
||||
fn strip_line_comment(line: &str, language: &str) -> String {
|
||||
let chars: Vec<_> = line.chars().collect();
|
||||
let mut quote = None;
|
||||
let mut escaped = false;
|
||||
let mut index = 0;
|
||||
while index < chars.len() {
|
||||
let ch = chars[index];
|
||||
if let Some(active_quote) = quote {
|
||||
if escaped {
|
||||
escaped = false;
|
||||
} else if ch == '\\' {
|
||||
escaped = true;
|
||||
} else if ch == active_quote {
|
||||
quote = None;
|
||||
}
|
||||
} else if matches!(ch, '\'' | '"') && has_closing_quote(&chars, index + 1, ch) {
|
||||
quote = Some(ch);
|
||||
} else if language == "python" && ch == '#'
|
||||
|| language != "python" && ch == '/' && chars.get(index + 1) == Some(&'/')
|
||||
{
|
||||
return chars[..index].iter().collect();
|
||||
}
|
||||
index += 1;
|
||||
}
|
||||
line.into()
|
||||
}
|
||||
|
||||
fn mask_block_comments(line: &str, depth: &mut usize) -> String {
|
||||
let chars: Vec<_> = line.chars().collect();
|
||||
let mut result = String::with_capacity(line.len());
|
||||
let mut quote = None;
|
||||
let mut escaped = false;
|
||||
let mut index = 0;
|
||||
while index < chars.len() {
|
||||
let ch = chars[index];
|
||||
let next = chars.get(index + 1).copied();
|
||||
if *depth > 0 {
|
||||
if ch == '/' && next == Some('*') {
|
||||
*depth += 1;
|
||||
result.push_str(" ");
|
||||
index += 2;
|
||||
} else if ch == '*' && next == Some('/') {
|
||||
*depth -= 1;
|
||||
result.push_str(" ");
|
||||
index += 2;
|
||||
} else {
|
||||
result.push(' ');
|
||||
index += 1;
|
||||
}
|
||||
} else if let Some(active_quote) = quote {
|
||||
result.push(ch);
|
||||
if escaped {
|
||||
escaped = false;
|
||||
} else if ch == '\\' {
|
||||
escaped = true;
|
||||
} else if ch == active_quote {
|
||||
quote = None;
|
||||
}
|
||||
index += 1;
|
||||
} else if matches!(ch, '\'' | '"') && has_closing_quote(&chars, index + 1, ch) {
|
||||
quote = Some(ch);
|
||||
result.push(ch);
|
||||
index += 1;
|
||||
} else if ch == '/' && next == Some('*') {
|
||||
*depth = 1;
|
||||
result.push_str(" ");
|
||||
index += 2;
|
||||
} else {
|
||||
result.push(ch);
|
||||
index += 1;
|
||||
}
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
fn has_closing_quote(chars: &[char], start: usize, quote: char) -> bool {
|
||||
let mut escaped = false;
|
||||
for ch in &chars[start..] {
|
||||
if escaped {
|
||||
escaped = false;
|
||||
} else if *ch == '\\' {
|
||||
escaped = true;
|
||||
} else if *ch == quote {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
fn brace_delta(line: &str) -> i32 {
|
||||
|
|
@ -962,10 +1044,32 @@ fn is_keyword(name: &str) -> bool {
|
|||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{ScanOptions, scan, scan_with_options};
|
||||
use super::{ScanOptions, mask_block_comments, scan, scan_with_options, strip_line_comment};
|
||||
use crate::model::{EdgeKind, NodeKind};
|
||||
use std::fs;
|
||||
|
||||
#[test]
|
||||
fn masks_nested_block_comments_without_truncating_quoted_urls() {
|
||||
let mut depth = 0;
|
||||
assert_eq!(
|
||||
strip_line_comment(
|
||||
r#"import value from "https://example.invalid/value.js"; // trailing"#,
|
||||
"javascript"
|
||||
),
|
||||
r#"import value from "https://example.invalid/value.js"; "#
|
||||
);
|
||||
assert_eq!(
|
||||
mask_block_comments("code /* outer", &mut depth),
|
||||
"code "
|
||||
);
|
||||
assert_eq!(depth, 1);
|
||||
let nested = "/* inner */ end */";
|
||||
let masked = mask_block_comments(nested, &mut depth);
|
||||
assert_eq!(masked.len(), nested.len());
|
||||
assert!(masked.trim().is_empty());
|
||||
assert_eq!(depth, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scans_functions_calls_and_cycles() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
|
|
|
|||
5
tests/fixtures/languages/c/basic.c
vendored
5
tests/fixtures/languages/c/basic.c
vendored
|
|
@ -20,3 +20,8 @@ int start() {
|
|||
int helper() {
|
||||
return 0;
|
||||
}
|
||||
|
||||
/*
|
||||
struct CommentGhost {};
|
||||
int comment_call() { return phantom_call(); }
|
||||
*/
|
||||
|
|
|
|||
5
tests/fixtures/languages/cpp/basic.cpp
vendored
5
tests/fixtures/languages/cpp/basic.cpp
vendored
|
|
@ -19,3 +19,8 @@ public:
|
|||
int helper() {
|
||||
return 0;
|
||||
}
|
||||
|
||||
/*
|
||||
class CommentGhost {};
|
||||
int comment_call() { return phantom_call(); }
|
||||
*/
|
||||
|
|
|
|||
5
tests/fixtures/languages/dart/basic.dart
vendored
5
tests/fixtures/languages/dart/basic.dart
vendored
|
|
@ -17,3 +17,8 @@ abstract class Controller extends Base with Logging implements Runnable {
|
|||
|
||||
void helper() {}
|
||||
}
|
||||
|
||||
/*
|
||||
class CommentGhost {}
|
||||
void commentCall() { phantomCall(); }
|
||||
*/
|
||||
|
|
|
|||
5
tests/fixtures/languages/javascript/basic.js
vendored
5
tests/fixtures/languages/javascript/basic.js
vendored
|
|
@ -20,3 +20,8 @@ const ignored = `
|
|||
class Phantom extends Ghost {}
|
||||
function fake() { phantomCall(); }
|
||||
`;
|
||||
|
||||
/*
|
||||
class CommentGhost {}
|
||||
function commentCall() { phantomCall(); }
|
||||
*/
|
||||
|
|
|
|||
5
tests/fixtures/languages/rust/basic.rs
vendored
5
tests/fixtures/languages/rust/basic.rs
vendored
|
|
@ -21,3 +21,8 @@ pub const fn constant() -> usize {
|
|||
pub unsafe fn unsafe_op() {}
|
||||
|
||||
pub extern "C" fn ffi_entry() {}
|
||||
|
||||
/*
|
||||
struct CommentGhost;
|
||||
fn comment_call() { phantom_call(); }
|
||||
*/
|
||||
|
|
|
|||
5
tests/fixtures/languages/typescript/basic.ts
vendored
5
tests/fixtures/languages/typescript/basic.ts
vendored
|
|
@ -25,3 +25,8 @@ const ignored = `
|
|||
interface Phantom {}
|
||||
function fake(): void { phantomCall(); }
|
||||
`;
|
||||
|
||||
/*
|
||||
interface CommentGhost {}
|
||||
function commentCall(): void { phantomCall(); }
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -266,6 +266,14 @@ fn assert_fixture(case: FixtureCase) {
|
|||
"{} call baseline",
|
||||
case.directory
|
||||
);
|
||||
assert!(
|
||||
!building.nodes.iter().any(|node| matches!(
|
||||
node.name.as_str(),
|
||||
"CommentGhost" | "commentCall" | "comment_call" | "phantom_call"
|
||||
)),
|
||||
"{} block-comment clean negative",
|
||||
case.directory
|
||||
);
|
||||
}
|
||||
|
||||
fn fixture_root(directory: &str) -> PathBuf {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue