#include "grammar_loader.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include extern "C" { #include } namespace yams::plugins::treesitter { namespace { bool symbolExtractorDebugEnabled() { static const bool enabled = [] { const char* raw = std::getenv("YAMS_SYMBOL_EXTRACTOR_DEBUG"); if (!raw || !*raw) { return false; } std::string_view value(raw); return value != "0" && value != "false" && value != "FALSE"; }(); return enabled; } void debugLog(const char* fmt, ...) { if (!symbolExtractorDebugEnabled()) { return; } va_list args; va_start(args, fmt); std::vfprintf(stderr, fmt, args); va_end(args); } } // namespace GrammarLoader::GrammarHandle::GrammarHandle(void* handle, TSLanguage* lang) noexcept : library_handle(handle), language(lang) {} GrammarLoader::GrammarHandle::GrammarHandle(GrammarLoader::GrammarHandle&& other) noexcept : library_handle(other.library_handle), language(other.language) { other.library_handle = nullptr; other.language = nullptr; } GrammarLoader::GrammarHandle& GrammarLoader::GrammarHandle::operator=(GrammarLoader::GrammarHandle&& other) noexcept { if (this != &other) { if (library_handle) { dlclose(library_handle); } library_handle = other.library_handle; language = other.language; other.library_handle = nullptr; other.language = nullptr; } return *this; } GrammarLoader::GrammarHandle::~GrammarHandle() { if (library_handle) { dlclose(library_handle); } } // GrammarLoader Implementation GrammarLoader::GrammarLoader() { // Initialize default search paths auto paths = getGrammarSearchPaths(); cached_search_paths_ = std::move(paths); search_paths_cached_ = true; } void GrammarLoader::addGrammarPath(std::string_view language, std::string_view path) { grammar_paths_[std::string(language)] = std::string(path); cached_search_paths_.clear(); search_paths_cached_ = false; } const GrammarLoader::GrammarSpec* GrammarLoader::findSpec(std::string_view language) const { std::string lang_lower(language); std::transform(lang_lower.begin(), lang_lower.end(), lang_lower.begin(), [](unsigned char c) { return std::tolower(c); }); for (const auto& spec : kSpecs) { if (lang_lower == spec.key) { return &spec; } } return nullptr; } std::vector GrammarLoader::getGrammarSearchPaths() const { if (search_paths_cached_) { return cached_search_paths_; } std::vector paths; // Add config override paths first (highest priority) for (const auto& [lang, path] : grammar_paths_) { paths.emplace_back(path); } // Environment variable overrides for specific grammars for (const auto& spec : kSpecs) { if (const char* env_path = std::getenv(spec.env_var.data())) { if (*env_path) { paths.emplace_back(env_path); } } } // Use config helper for platform-aware data directory resolution // This handles YAMS_DATA_DIR, YAMS_STORAGE env vars and XDG/Windows paths paths.emplace_back(yams::config::get_data_dir() / "grammars"); // System-wide locations (lower priority) #ifdef _WIN32 // Windows: Program installation paths if (const char* localappdata = std::getenv("LOCALAPPDATA")) { if (*localappdata) { paths.emplace_back(std::filesystem::path(localappdata) / "Programs" / "yams" / "lib" / "yams" / "grammars"); } } #else // Unix/Linux/macOS: System-wide locations paths.emplace_back("/usr/local/share/yams/grammars"); paths.emplace_back("/usr/share/yams/grammars"); #endif cached_search_paths_ = paths; search_paths_cached_ = true; return cached_search_paths_; } std::vector GrammarLoader::getLibraryCandidates(std::string_view language) const { const auto* spec = findSpec(language); if (!spec) { return {}; } std::vector candidates; auto search_paths = getGrammarSearchPaths(); // Extract the base name from spec (e.g., "tree-sitter-cpp" from "libtree-sitter-cpp.so") std::string spec_name(spec->default_so); // Remove extension auto dot_pos = spec_name.rfind('.'); if (dot_pos != std::string::npos) { spec_name = spec_name.substr(0, dot_pos); } // Remove "lib" prefix if present to get the core name std::string core_name = spec_name; if (core_name.rfind("lib", 0) == 0) { core_name = core_name.substr(3); } // Generate all library name variants for this platform std::vector lib_names; #ifdef _WIN32 // Windows: typically no "lib" prefix, .dll extension // Primary: tree-sitter-cpp.dll lib_names.push_back(core_name + ".dll"); // Also try with lib prefix: libtree-sitter-cpp.dll lib_names.push_back("lib" + core_name + ".dll"); // Underscore variants std::string underscore_name = core_name; std::replace(underscore_name.begin(), underscore_name.end(), '-', '_'); lib_names.push_back(underscore_name + ".dll"); lib_names.push_back("lib" + underscore_name + ".dll"); #elif defined(__APPLE__) // macOS: typically "lib" prefix, .dylib extension // Primary: libtree-sitter-cpp.dylib lib_names.push_back("lib" + core_name + ".dylib"); // Also try without lib prefix lib_names.push_back(core_name + ".dylib"); // Try .so extension (some builds use this) lib_names.push_back("lib" + core_name + ".so"); lib_names.push_back(core_name + ".so"); // Underscore variants std::string underscore_name = core_name; std::replace(underscore_name.begin(), underscore_name.end(), '-', '_'); lib_names.push_back("lib" + underscore_name + ".dylib"); lib_names.push_back(underscore_name + ".dylib"); #else // Linux: typically "lib" prefix, .so extension // Primary: libtree-sitter-cpp.so lib_names.push_back("lib" + core_name + ".so"); // Also try without lib prefix lib_names.push_back(core_name + ".so"); // Underscore variants std::string underscore_name = core_name; std::replace(underscore_name.begin(), underscore_name.end(), '-', '_'); lib_names.push_back("lib" + underscore_name + ".so"); lib_names.push_back(underscore_name + ".so"); #endif candidates.reserve(search_paths.size() * lib_names.size()); // For each search path, add all library name variants for (const auto& base_path : search_paths) { if (!std::filesystem::exists(base_path)) continue; for (const auto& lib_name : lib_names) { candidates.push_back((base_path / lib_name).string()); } } return candidates; } tl::expected GrammarLoader::loadGrammar(std::string_view language) { const auto* spec = findSpec(language); if (!spec) { return tl::unexpected(GrammarLoadError{ GrammarLoadError::NOT_FOUND, fmt::format("Language '{}' not supported", language)}); } auto candidates = getLibraryCandidates(language); if (candidates.empty()) { return tl::unexpected( GrammarLoadError{GrammarLoadError::NOT_FOUND, fmt::format("No library candidates for language '{}'", language)}); } // Try each candidate std::string tried_join; for (const auto& candidate : candidates) { if (!tried_join.empty()) { tried_join += ", "; } tried_join += candidate; debugLog("[yams] trying grammar candidate: %s\n", candidate.c_str()); void* handle = dlopen(candidate.c_str(), RTLD_LAZY | RTLD_LOCAL); if (!handle) { debugLog("[yams] dlopen failed: %s\n", dlerror()); continue; // Try next candidate } // Try to get the language factory function auto* factory_fn = reinterpret_cast(dlsym(handle, spec->symbol.data())); if (factory_fn) { // Success! Create language and return TSLanguage* lang = factory_fn(); if (lang) { return GrammarHandle(handle, lang); } } // Factory function failed, close handle and continue dlclose(handle); } // All candidates failed return tl::unexpected(GrammarLoadError{ GrammarLoadError::LOAD_FAILED, fmt::format("Failed to load grammar for '{}'. Tried: {}", language, tried_join)}); } bool GrammarLoader::grammarExists(std::string_view language) const { const auto* spec = findSpec(language); if (!spec) return false; auto candidates = getLibraryCandidates(language); for (const auto& candidate : candidates) { if (std::filesystem::exists(candidate)) { // Quick check if it's loadable void* handle = dlopen(candidate.c_str(), RTLD_LAZY | RTLD_LOCAL); if (handle) { auto* factory_fn = reinterpret_cast(dlsym(handle, spec->symbol.data())); bool ok = (factory_fn != nullptr); dlclose(handle); return ok; } } } return false; } // GrammarDownloader Implementation tl::expected GrammarDownloader::downloadGrammar(std::string_view language) { if (!canAutoDownload()) { return tl::unexpected(std::string{"Auto-download tools not available (git, gcc required)"}); } // Find grammar repository auto* repo_info = std::find_if(std::begin(kGrammarRepos), std::end(kGrammarRepos), [language](const auto& repo) { return repo.language == language; }); if (repo_info == std::end(kGrammarRepos)) { return tl::unexpected(fmt::format("Language '{}' auto-download not supported", language)); } auto grammar_paths = GrammarLoader{}.getGrammarSearchPaths(); const auto& grammar_path = grammar_paths[0]; // Use primary user path (XDG or ~/.local/share) std::filesystem::create_directories(grammar_path); // Create temp directory for build auto temp_dir = std::filesystem::temp_directory_path() / fmt::format("yams-grammar-build-{}", language); std::filesystem::create_directories(temp_dir); // Cleanup guard auto cleanup = [&temp_dir]() { try { std::filesystem::remove_all(temp_dir); } catch (...) { } }; try { // Clone repository std::string clone_cmd = fmt::format("cd {} && git clone --depth 1 https://github.com/{} tree-sitter-{}", temp_dir.string(), repo_info->repo, language); debugLog("[yams] cloning grammar repository for '%.*s'\n", static_cast(language.size()), language.data()); if (std::system(clone_cmd.c_str()) != 0) { cleanup(); return tl::unexpected(std::string{"Failed to clone grammar repository"}); } // Build grammar (portable) auto build_dir = temp_dir / fmt::format("tree-sitter-{}", language); // Special handling for TypeScript which has subdirectories if (language == "typescript") { build_dir = build_dir / "typescript"; } auto have = [](const char* tool) { #ifdef _WIN32 std::string cmd = std::string("where ") + tool + " > NUL 2>&1"; #else std::string cmd = std::string("which ") + tool + " > /dev/null 2>&1"; #endif return std::system(cmd.c_str()) == 0; }; std::string cxx = have("g++") ? "g++" : (have("c++") ? "c++" : (have("clang++") ? "clang++" : "")); std::string cc = have("gcc") ? "gcc" : (have("cc") ? "cc" : (have("clang") ? "clang" : "")); auto parser_c = build_dir / "src" / "parser.c"; auto scanner_c = build_dir / "src" / "scanner.c"; auto scanner_cc = build_dir / "src" / "scanner.cc"; bool use_cxx = std::filesystem::exists(scanner_cc); std::string compiler = use_cxx ? (cxx.empty() ? cc : cxx) : (cc.empty() ? cxx : cc); if (compiler.empty()) { cleanup(); return tl::unexpected(std::string{"No suitable compiler found"}); } #ifdef __APPLE__ std::string lib_name = fmt::format("libtree-sitter-{}.dylib", language); std::string flags = "-dynamiclib"; #elif defined(_WIN32) std::string lib_name = fmt::format("tree-sitter-{}.dll", language); std::string flags = "-shared"; #else std::string lib_name = fmt::format("libtree-sitter-{}.so", language); std::string flags = "-shared -fPIC"; #endif // Find tree-sitter include directory std::vector include_search; #ifdef _WIN32 // Windows: check vcpkg, conan, and common install locations if (const char* vcpkg_root = std::getenv("VCPKG_ROOT")) { include_search.push_back(std::string(vcpkg_root) + "/installed/x64-windows/include"); } if (const char* localappdata = std::getenv("LOCALAPPDATA")) { include_search.push_back(std::string(localappdata) + "/yams/include"); } #else include_search = {"/usr/local/include", "/usr/include", "/opt/homebrew/include"}; #endif // Add conan tree-sitter paths (works on all platforms) #ifdef _WIN32 const char* home = std::getenv("USERPROFILE"); #else const char* home = std::getenv("HOME"); #endif if (home) { std::string conan_base = std::string(home) + "/.conan2/p/b"; if (std::filesystem::exists(conan_base)) { try { for (const auto& entry : std::filesystem::directory_iterator(conan_base)) { if (entry.is_directory()) { std::string dirname = entry.path().filename().string(); if (dirname.find("tree-") == 0) { // starts_with for C++20 include_search.push_back(entry.path().string() + "/p/include"); } } } } catch (...) { } } } std::string ts_include; for (const auto& path : include_search) { auto ts_header = std::filesystem::path(path) / "tree_sitter" / "api.h"; if (std::filesystem::exists(ts_header)) { ts_include = " -I" + path; debugLog("[yams] found tree-sitter headers at: %s\n", path.c_str()); break; } } if (ts_include.empty()) { debugLog("[yams] tree-sitter headers not found, build may fail\n"); } std::ostringstream cmd; cmd << "cd " << build_dir.string() << " && " << compiler << ' ' << flags << " -I." << ts_include << " " << parser_c.string() << ' '; if (std::filesystem::exists(scanner_c)) { cmd << scanner_c.string() << ' '; } else if (std::filesystem::exists(scanner_cc)) { cmd << scanner_cc.string() << ' '; } cmd << "-o " << lib_name; debugLog("[yams] build command: %s\n", cmd.str().c_str()); debugLog("[yams] building grammar for '%.*s'\n", static_cast(language.size()), language.data()); if (std::system(cmd.str().c_str()) != 0) { cleanup(); return tl::unexpected(std::string{"Failed to build grammar"}); } // Find built library #ifdef __APPLE__ auto built_lib = build_dir / fmt::format("libtree-sitter-{}.dylib", language); #elif defined(_WIN32) auto built_lib = build_dir / fmt::format("tree-sitter-{}.dll", language); #else auto built_lib = build_dir / fmt::format("libtree-sitter-{}.so", language); #endif if (!std::filesystem::exists(built_lib)) { cleanup(); return tl::unexpected(std::string{"Built library not found"}); } // Copy to final location auto final_path = grammar_path / lib_name; std::filesystem::copy_file(built_lib, final_path, std::filesystem::copy_options::overwrite_existing); cleanup(); debugLog("[yams] grammar downloaded successfully to: %s\n", final_path.c_str()); return final_path; } catch (const std::exception& e) { cleanup(); return tl::unexpected(std::string{"Download failed: "} + e.what()); } } bool GrammarDownloader::canAutoDownload() { // Check for required tools: git and at least one compiler auto have = [](const char* tool) { #ifdef _WIN32 std::string cmd = std::string("where ") + tool + " > NUL 2>&1"; #else std::string cmd = std::string("which ") + tool + " > /dev/null 2>&1"; #endif return std::system(cmd.c_str()) == 0; }; bool has_git = have("git"); bool has_cc = have("g++") || have("gcc") || have("clang++") || have("clang") || have("c++") || have("cc"); return has_git && has_cc; } } // namespace yams::plugins::treesitter