fix(scan): decode non-UTF-8 source lossily instead of dropping the file

A single invalid byte, such as a Latin-1 `é` in a comment, made
`read_to_string` fail and the whole file and its symbols were skipped.

Files are now read as bytes: a NUL byte still marks a binary file,
which is skipped with a diagnostic; otherwise invalid UTF-8 is decoded
lossily (U+FFFD) and a warning diagnostic records it, so the file keeps
its symbols and calls.

Test: languages::a_non_utf8_byte_does_not_drop_the_whole_file fails
before this change (latin1.rs has no symbols).

src/scan.rs

@@ -368,17 +368,39 @@ impl ScanState {
                 return;
             }
         };
-        let content = match fs::read_to_string(path) {
+        let bytes = match fs::read(path) {
             Ok(value) => value,
             Err(error) => {
                 self.diagnostics.push(Diagnostic {
                     level: "warning".into(),
-                    message: format!("skipped non-UTF-8 or unreadable source: {error}"),
+                    message: format!("skipped unreadable source: {error}"),
                     path: Some(relative),
                 });
                 return;
             }
         };
+        // NUL bytes mean a binary file, not source. Any other invalid UTF-8 (a
+        // Latin-1 byte in a comment) is decoded lossily: dropping the whole file hid
+        // every symbol in it.
+        if bytes.contains(&0) {
+            self.diagnostics.push(Diagnostic {
+                level: "warning".into(),
+                message: "skipped binary file with a source extension".into(),
+                path: Some(relative),
+            });
+            return;
+        }
+        let content = match String::from_utf8(bytes) {
+            Ok(value) => value,
+            Err(error) => {
+                self.diagnostics.push(Diagnostic {
+                    level: "warning".into(),
+                    message: "decoded non-UTF-8 bytes lossily (U+FFFD)".into(),
+                    path: Some(relative.clone()),
+                });
+                String::from_utf8_lossy(error.as_bytes()).into_owned()
+            }
+        };
         let file_id = stable_id("file", &relative);
         let parent = path
             .parent()

tests/languages.rs

@@ -702,3 +702,30 @@ fn dart_calls_resolve_through_imported_libraries() {
         [("helper".into(), String::new(), "not-visible".into())]
     );
 }
+
+#[test]
+fn a_non_utf8_byte_does_not_drop_the_whole_file() {
+    // One Latin-1 `é` in a comment made the whole file (and its functions) vanish.
+    let dir = tempfile::tempdir().unwrap();
+    let mut source = b"// caf".to_vec();
+    source.push(0xE9);
+    source.extend_from_slice(b"\nfn kept() {}\nfn caller() { kept(); }\n");
+    fs::write(dir.path().join("latin1.rs"), source).unwrap();
+    fs::write(dir.path().join("blob.rs"), [0u8, 159, 146, 150]).unwrap();
+    let model = scan::scan(dir.path()).unwrap();
+    let caller = symbol(&model, "latin1.rs", "caller");
+    assert!(calls(&model, caller, symbol(&model, "latin1.rs", "kept")));
+    assert!(
+        model
+            .diagnostics
+            .iter()
+            .any(|d| d.path.as_deref() == Some("latin1.rs") && d.message.contains("non-UTF-8"))
+    );
+    // A file with NUL bytes is binary, not source: still skipped with a diagnostic.
+    assert!(
+        !model
+            .nodes
+            .iter()
+            .any(|n| n.path.as_deref() == Some("blob.rs"))
+    );
+}