feat: establish first-class lexical language parity #2
8 changed files with 156 additions and 14 deletions
fix(scan): mask multiline block comments
commit
543bafcfd4
132
src/scan.rs
132
src/scan.rs
|
|
@ -243,6 +243,7 @@ impl ScanState {
|
||||||
let mut brace_depth = 0_i32;
|
let mut brace_depth = 0_i32;
|
||||||
let mut rust_raw_end: Option<String> = None;
|
let mut rust_raw_end: Option<String> = None;
|
||||||
let mut javascript_template_open = false;
|
let mut javascript_template_open = false;
|
||||||
|
let mut block_comment_depth = 0_usize;
|
||||||
|
|
||||||
for (index, raw_line) in content.lines().enumerate() {
|
for (index, raw_line) in content.lines().enumerate() {
|
||||||
let line_number = index + 1;
|
let line_number = index + 1;
|
||||||
|
|
@ -269,8 +270,9 @@ impl ScanState {
|
||||||
} else {
|
} else {
|
||||||
raw_line
|
raw_line
|
||||||
};
|
};
|
||||||
let line = strip_line_comment(code_line, language);
|
let block_masked = mask_block_comments(code_line, &mut block_comment_depth);
|
||||||
let call_line = mask_quoted_strings(line);
|
let line = strip_line_comment(&block_masked, language);
|
||||||
|
let call_line = mask_quoted_strings(&line);
|
||||||
let indent = raw_line.len() - raw_line.trim_start().len();
|
let indent = raw_line.len() - raw_line.trim_start().len();
|
||||||
if language == "python" && !line.trim().is_empty() {
|
if language == "python" && !line.trim().is_empty() {
|
||||||
while scopes.last().is_some_and(|scope| scope.indent >= indent) {
|
while scopes.last().is_some_and(|scope| scope.indent >= indent) {
|
||||||
|
|
@ -287,7 +289,7 @@ impl ScanState {
|
||||||
|
|
||||||
if let Some(pattern) = patterns.enclosing_scope.as_ref()
|
if let Some(pattern) = patterns.enclosing_scope.as_ref()
|
||||||
&& let Some(name) = pattern
|
&& let Some(name) = pattern
|
||||||
.captures(line)
|
.captures(&line)
|
||||||
.and_then(|captures| captures.name("name"))
|
.and_then(|captures| captures.name("name"))
|
||||||
&& let Some(id) = self
|
&& let Some(id) = self
|
||||||
.symbols_by_name
|
.symbols_by_name
|
||||||
|
|
@ -303,7 +305,7 @@ impl ScanState {
|
||||||
}
|
}
|
||||||
|
|
||||||
let mut type_definition_name = None;
|
let mut type_definition_name = None;
|
||||||
if let Some(captures) = patterns.class.captures(line)
|
if let Some(captures) = patterns.class.captures(&line)
|
||||||
&& let Some(name) = captures.name("name")
|
&& let Some(name) = captures.name("name")
|
||||||
{
|
{
|
||||||
type_definition_name = Some(name.as_str().to_owned());
|
type_definition_name = Some(name.as_str().to_owned());
|
||||||
|
|
@ -319,7 +321,7 @@ impl ScanState {
|
||||||
.relationships
|
.relationships
|
||||||
.as_ref()
|
.as_ref()
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.flat_map(|pattern| pattern.captures_iter(line))
|
.flat_map(|pattern| pattern.captures_iter(&line))
|
||||||
.filter_map(|captures| captures.name("names"))
|
.filter_map(|captures| captures.name("names"))
|
||||||
.flat_map(|names| names.as_str().split(','))
|
.flat_map(|names| names.as_str().split(','))
|
||||||
.map(str::trim)
|
.map(str::trim)
|
||||||
|
|
@ -353,7 +355,7 @@ impl ScanState {
|
||||||
let mut definition_name = None;
|
let mut definition_name = None;
|
||||||
if let Some(name) = patterns.function.iter().find_map(|pattern| {
|
if let Some(name) = patterns.function.iter().find_map(|pattern| {
|
||||||
pattern
|
pattern
|
||||||
.captures(line)
|
.captures(&line)
|
||||||
.and_then(|captures| captures.name("name").map(|name| name.as_str().to_owned()))
|
.and_then(|captures| captures.name("name").map(|name| name.as_str().to_owned()))
|
||||||
}) && !is_keyword(&name)
|
}) && !is_keyword(&name)
|
||||||
{
|
{
|
||||||
|
|
@ -389,7 +391,7 @@ impl ScanState {
|
||||||
for captures in patterns
|
for captures in patterns
|
||||||
.import
|
.import
|
||||||
.iter()
|
.iter()
|
||||||
.flat_map(|pattern| pattern.captures_iter(line))
|
.flat_map(|pattern| pattern.captures_iter(&line))
|
||||||
{
|
{
|
||||||
if let Some(target) = captures.name("name").or_else(|| captures.name("name2")) {
|
if let Some(target) = captures.name("name").or_else(|| captures.name("name2")) {
|
||||||
self.pending.push(PendingRelationship {
|
self.pending.push(PendingRelationship {
|
||||||
|
|
@ -779,13 +781,93 @@ fn language_for(path: &Path) -> Option<&'static str> {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn strip_line_comment<'a>(line: &'a str, language: &str) -> &'a str {
|
fn strip_line_comment(line: &str, language: &str) -> String {
|
||||||
let marker = if language == "python" { '#' } else { '/' };
|
let chars: Vec<_> = line.chars().collect();
|
||||||
if marker == '/' {
|
let mut quote = None;
|
||||||
line.split_once("//").map_or(line, |(before, _)| before)
|
let mut escaped = false;
|
||||||
} else {
|
let mut index = 0;
|
||||||
line.split_once('#').map_or(line, |(before, _)| before)
|
while index < chars.len() {
|
||||||
|
let ch = chars[index];
|
||||||
|
if let Some(active_quote) = quote {
|
||||||
|
if escaped {
|
||||||
|
escaped = false;
|
||||||
|
} else if ch == '\\' {
|
||||||
|
escaped = true;
|
||||||
|
} else if ch == active_quote {
|
||||||
|
quote = None;
|
||||||
}
|
}
|
||||||
|
} else if matches!(ch, '\'' | '"') && has_closing_quote(&chars, index + 1, ch) {
|
||||||
|
quote = Some(ch);
|
||||||
|
} else if language == "python" && ch == '#'
|
||||||
|
|| language != "python" && ch == '/' && chars.get(index + 1) == Some(&'/')
|
||||||
|
{
|
||||||
|
return chars[..index].iter().collect();
|
||||||
|
}
|
||||||
|
index += 1;
|
||||||
|
}
|
||||||
|
line.into()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn mask_block_comments(line: &str, depth: &mut usize) -> String {
|
||||||
|
let chars: Vec<_> = line.chars().collect();
|
||||||
|
let mut result = String::with_capacity(line.len());
|
||||||
|
let mut quote = None;
|
||||||
|
let mut escaped = false;
|
||||||
|
let mut index = 0;
|
||||||
|
while index < chars.len() {
|
||||||
|
let ch = chars[index];
|
||||||
|
let next = chars.get(index + 1).copied();
|
||||||
|
if *depth > 0 {
|
||||||
|
if ch == '/' && next == Some('*') {
|
||||||
|
*depth += 1;
|
||||||
|
result.push_str(" ");
|
||||||
|
index += 2;
|
||||||
|
} else if ch == '*' && next == Some('/') {
|
||||||
|
*depth -= 1;
|
||||||
|
result.push_str(" ");
|
||||||
|
index += 2;
|
||||||
|
} else {
|
||||||
|
result.push(' ');
|
||||||
|
index += 1;
|
||||||
|
}
|
||||||
|
} else if let Some(active_quote) = quote {
|
||||||
|
result.push(ch);
|
||||||
|
if escaped {
|
||||||
|
escaped = false;
|
||||||
|
} else if ch == '\\' {
|
||||||
|
escaped = true;
|
||||||
|
} else if ch == active_quote {
|
||||||
|
quote = None;
|
||||||
|
}
|
||||||
|
index += 1;
|
||||||
|
} else if matches!(ch, '\'' | '"') && has_closing_quote(&chars, index + 1, ch) {
|
||||||
|
quote = Some(ch);
|
||||||
|
result.push(ch);
|
||||||
|
index += 1;
|
||||||
|
} else if ch == '/' && next == Some('*') {
|
||||||
|
*depth = 1;
|
||||||
|
result.push_str(" ");
|
||||||
|
index += 2;
|
||||||
|
} else {
|
||||||
|
result.push(ch);
|
||||||
|
index += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
result
|
||||||
|
}
|
||||||
|
|
||||||
|
fn has_closing_quote(chars: &[char], start: usize, quote: char) -> bool {
|
||||||
|
let mut escaped = false;
|
||||||
|
for ch in &chars[start..] {
|
||||||
|
if escaped {
|
||||||
|
escaped = false;
|
||||||
|
} else if *ch == '\\' {
|
||||||
|
escaped = true;
|
||||||
|
} else if *ch == quote {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
false
|
||||||
}
|
}
|
||||||
|
|
||||||
fn brace_delta(line: &str) -> i32 {
|
fn brace_delta(line: &str) -> i32 {
|
||||||
|
|
@ -962,10 +1044,32 @@ fn is_keyword(name: &str) -> bool {
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::{ScanOptions, scan, scan_with_options};
|
use super::{ScanOptions, mask_block_comments, scan, scan_with_options, strip_line_comment};
|
||||||
use crate::model::{EdgeKind, NodeKind};
|
use crate::model::{EdgeKind, NodeKind};
|
||||||
use std::fs;
|
use std::fs;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn masks_nested_block_comments_without_truncating_quoted_urls() {
|
||||||
|
let mut depth = 0;
|
||||||
|
assert_eq!(
|
||||||
|
strip_line_comment(
|
||||||
|
r#"import value from "https://example.invalid/value.js"; // trailing"#,
|
||||||
|
"javascript"
|
||||||
|
),
|
||||||
|
r#"import value from "https://example.invalid/value.js"; "#
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
mask_block_comments("code /* outer", &mut depth),
|
||||||
|
"code "
|
||||||
|
);
|
||||||
|
assert_eq!(depth, 1);
|
||||||
|
let nested = "/* inner */ end */";
|
||||||
|
let masked = mask_block_comments(nested, &mut depth);
|
||||||
|
assert_eq!(masked.len(), nested.len());
|
||||||
|
assert!(masked.trim().is_empty());
|
||||||
|
assert_eq!(depth, 0);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn scans_functions_calls_and_cycles() {
|
fn scans_functions_calls_and_cycles() {
|
||||||
let dir = tempfile::tempdir().unwrap();
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
|
|
||||||
5
tests/fixtures/languages/c/basic.c
vendored
5
tests/fixtures/languages/c/basic.c
vendored
|
|
@ -20,3 +20,8 @@ int start() {
|
||||||
int helper() {
|
int helper() {
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
struct CommentGhost {};
|
||||||
|
int comment_call() { return phantom_call(); }
|
||||||
|
*/
|
||||||
|
|
|
||||||
5
tests/fixtures/languages/cpp/basic.cpp
vendored
5
tests/fixtures/languages/cpp/basic.cpp
vendored
|
|
@ -19,3 +19,8 @@ public:
|
||||||
int helper() {
|
int helper() {
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
class CommentGhost {};
|
||||||
|
int comment_call() { return phantom_call(); }
|
||||||
|
*/
|
||||||
|
|
|
||||||
5
tests/fixtures/languages/dart/basic.dart
vendored
5
tests/fixtures/languages/dart/basic.dart
vendored
|
|
@ -17,3 +17,8 @@ abstract class Controller extends Base with Logging implements Runnable {
|
||||||
|
|
||||||
void helper() {}
|
void helper() {}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
class CommentGhost {}
|
||||||
|
void commentCall() { phantomCall(); }
|
||||||
|
*/
|
||||||
|
|
|
||||||
5
tests/fixtures/languages/javascript/basic.js
vendored
5
tests/fixtures/languages/javascript/basic.js
vendored
|
|
@ -20,3 +20,8 @@ const ignored = `
|
||||||
class Phantom extends Ghost {}
|
class Phantom extends Ghost {}
|
||||||
function fake() { phantomCall(); }
|
function fake() { phantomCall(); }
|
||||||
`;
|
`;
|
||||||
|
|
||||||
|
/*
|
||||||
|
class CommentGhost {}
|
||||||
|
function commentCall() { phantomCall(); }
|
||||||
|
*/
|
||||||
|
|
|
||||||
5
tests/fixtures/languages/rust/basic.rs
vendored
5
tests/fixtures/languages/rust/basic.rs
vendored
|
|
@ -21,3 +21,8 @@ pub const fn constant() -> usize {
|
||||||
pub unsafe fn unsafe_op() {}
|
pub unsafe fn unsafe_op() {}
|
||||||
|
|
||||||
pub extern "C" fn ffi_entry() {}
|
pub extern "C" fn ffi_entry() {}
|
||||||
|
|
||||||
|
/*
|
||||||
|
struct CommentGhost;
|
||||||
|
fn comment_call() { phantom_call(); }
|
||||||
|
*/
|
||||||
|
|
|
||||||
5
tests/fixtures/languages/typescript/basic.ts
vendored
5
tests/fixtures/languages/typescript/basic.ts
vendored
|
|
@ -25,3 +25,8 @@ const ignored = `
|
||||||
interface Phantom {}
|
interface Phantom {}
|
||||||
function fake(): void { phantomCall(); }
|
function fake(): void { phantomCall(); }
|
||||||
`;
|
`;
|
||||||
|
|
||||||
|
/*
|
||||||
|
interface CommentGhost {}
|
||||||
|
function commentCall(): void { phantomCall(); }
|
||||||
|
*/
|
||||||
|
|
|
||||||
|
|
@ -266,6 +266,14 @@ fn assert_fixture(case: FixtureCase) {
|
||||||
"{} call baseline",
|
"{} call baseline",
|
||||||
case.directory
|
case.directory
|
||||||
);
|
);
|
||||||
|
assert!(
|
||||||
|
!building.nodes.iter().any(|node| matches!(
|
||||||
|
node.name.as_str(),
|
||||||
|
"CommentGhost" | "commentCall" | "comment_call" | "phantom_call"
|
||||||
|
)),
|
||||||
|
"{} block-comment clean negative",
|
||||||
|
case.directory
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
fn fixture_root(directory: &str) -> PathBuf {
|
fn fixture_root(directory: &str) -> PathBuf {
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue