830 lines
31 KiB
C++
830 lines
31 KiB
C++
#include "strawberry/rule_updater.hpp"
|
|
#include "strawberry/parse_num.hpp"
|
|
#include "strawberry/ioc_set.hpp"
|
|
|
|
#include "strawberry/control_server.hpp"
|
|
#include "strawberry/hash_utils.hpp"
|
|
#include "strawberry/yara_scanner.hpp"
|
|
|
|
#include <nlohmann/json.hpp>
|
|
|
|
#include <algorithm>
|
|
#include <chrono>
|
|
#include <filesystem>
|
|
#include <fstream>
|
|
#include <iostream>
|
|
#include <map>
|
|
#include <regex>
|
|
#include <set>
|
|
#include <sstream>
|
|
|
|
#include <arpa/inet.h>
|
|
#include <netinet/in.h>
|
|
#include <stdlib.h>
|
|
#include <unistd.h>
|
|
|
|
#ifndef STRAWBERRY_HAVE_UPDATER
|
|
#define STRAWBERRY_HAVE_UPDATER 0
|
|
#endif
|
|
|
|
#if STRAWBERRY_HAVE_UPDATER
|
|
#include <archive.h>
|
|
#include <archive_entry.h>
|
|
#include <curl/curl.h>
|
|
#endif
|
|
|
|
namespace strawberry {
|
|
|
|
namespace {
|
|
|
|
using json = nlohmann::json;
|
|
namespace fs = std::filesystem;
|
|
using namespace std::chrono_literals;
|
|
|
|
int64_t now_ns() {
|
|
return std::chrono::duration_cast<std::chrono::nanoseconds>(
|
|
std::chrono::system_clock::now().time_since_epoch()).count();
|
|
}
|
|
|
|
std::string sha256_hex(const std::string& data) {
|
|
return HashUtils::sha256_data(reinterpret_cast<const unsigned char*>(data.data()), data.size());
|
|
}
|
|
|
|
bool write_file_atomic(const fs::path& path, const std::string& content, std::string* error) {
|
|
const auto temp = path.string() + ".tmp";
|
|
{
|
|
std::ofstream out(temp, std::ios::binary | std::ios::trunc);
|
|
out.write(content.data(), static_cast<std::streamsize>(content.size()));
|
|
out.flush();
|
|
if (!out) {
|
|
if (error) *error = "cannot write " + temp;
|
|
return false;
|
|
}
|
|
}
|
|
std::error_code ec;
|
|
fs::rename(temp, path, ec);
|
|
if (ec) {
|
|
if (error) *error = "cannot replace " + path.string() + ": " + ec.message();
|
|
return false;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
std::string version_dir_name(const std::string& version, const std::string& sha256) {
|
|
if (std::regex_match(version, std::regex("[A-Za-z0-9._-]{1,64}")) && version != "." && version != ".."
|
|
&& version != "current") {
|
|
return version;
|
|
}
|
|
return "sha256-" + sha256.substr(0, 16);
|
|
}
|
|
|
|
struct RuleFileBlob {
|
|
std::string name;
|
|
std::string content;
|
|
};
|
|
|
|
bool is_rule_name(const std::string& name) {
|
|
const auto lower = [](std::string s) {
|
|
std::transform(s.begin(), s.end(), s.begin(), [](unsigned char c) { return static_cast<char>(std::tolower(c)); });
|
|
return s;
|
|
}(name);
|
|
return lower.ends_with(".yar") || lower.ends_with(".yara");
|
|
}
|
|
|
|
// Extract regular *.yar/*.yara entries from an archive in memory. Paths are
|
|
// flattened to sanitized basenames; links, devices and traversal are ignored.
|
|
bool extract_rules(const std::string& body, std::size_t max_bytes, std::vector<RuleFileBlob>& out, std::string& error) {
|
|
#if !STRAWBERRY_HAVE_UPDATER
|
|
(void) body;
|
|
(void) max_bytes;
|
|
(void) out;
|
|
error = "built without libarchive";
|
|
return false;
|
|
#else
|
|
archive* a = archive_read_new();
|
|
archive_read_support_format_zip(a);
|
|
archive_read_support_format_tar(a);
|
|
archive_read_support_filter_gzip(a);
|
|
if (archive_read_open_memory(a, body.data(), body.size()) != ARCHIVE_OK) {
|
|
error = std::string("cannot open archive: ") + archive_error_string(a);
|
|
archive_read_free(a);
|
|
return false;
|
|
}
|
|
|
|
std::size_t total = 0;
|
|
std::set<std::string> used;
|
|
archive_entry* entry = nullptr;
|
|
int rc = ARCHIVE_OK;
|
|
while ((rc = archive_read_next_header(a, &entry)) == ARCHIVE_OK) {
|
|
const char* raw_name = archive_entry_pathname(entry);
|
|
if (archive_entry_filetype(entry) != AE_IFREG || raw_name == nullptr || !is_rule_name(raw_name)) {
|
|
archive_read_data_skip(a);
|
|
continue;
|
|
}
|
|
std::string name = sanitize_rule_file_name(fs::path(raw_name).filename().string());
|
|
for (int i = 2; used.contains(name); ++i) {
|
|
name = sanitize_rule_file_name(fs::path(raw_name).stem().string() + "-" + std::to_string(i) + ".yar");
|
|
}
|
|
used.insert(name);
|
|
|
|
std::string content;
|
|
char buffer[65536];
|
|
la_ssize_t n = 0;
|
|
while ((n = archive_read_data(a, buffer, sizeof(buffer))) > 0) {
|
|
total += static_cast<std::size_t>(n);
|
|
if (total > max_bytes) {
|
|
error = "archive expands beyond the " + std::to_string(max_bytes) + " byte limit";
|
|
archive_read_free(a);
|
|
return false;
|
|
}
|
|
content.append(buffer, static_cast<std::size_t>(n));
|
|
}
|
|
if (n < 0) {
|
|
error = std::string("archive read error: ") + archive_error_string(a);
|
|
archive_read_free(a);
|
|
return false;
|
|
}
|
|
out.push_back({name, std::move(content)});
|
|
}
|
|
if (rc != ARCHIVE_EOF) {
|
|
error = std::string("archive error: ") + archive_error_string(a);
|
|
archive_read_free(a);
|
|
return false;
|
|
}
|
|
archive_read_free(a);
|
|
if (out.empty()) {
|
|
error = "archive contains no .yar/.yara files";
|
|
return false;
|
|
}
|
|
return true;
|
|
#endif
|
|
}
|
|
|
|
bool looks_like_archive(const std::string& body) {
|
|
return body.starts_with(std::string("PK\x03\x04", 4)) || body.starts_with(std::string("\x1f\x8b", 2))
|
|
|| (body.size() > 262 && body.compare(257, 5, "ustar") == 0);
|
|
}
|
|
|
|
std::size_t max_list_entries(const std::string& kind) {
|
|
if (kind == "hashes") return 2'000'000;
|
|
if (kind == "domains") return 2'000'000;
|
|
return 1'000'000;
|
|
}
|
|
|
|
// Extract every regular file (any name) from an archive, for zipped lists.
|
|
bool extract_text_files(const std::string& body, std::size_t max_bytes, std::vector<RuleFileBlob>& out, std::string& error) {
|
|
#if !STRAWBERRY_HAVE_UPDATER
|
|
(void) body;
|
|
(void) max_bytes;
|
|
(void) out;
|
|
error = "built without libarchive";
|
|
return false;
|
|
#else
|
|
archive* a = archive_read_new();
|
|
archive_read_support_format_zip(a);
|
|
archive_read_support_format_tar(a);
|
|
archive_read_support_filter_gzip(a);
|
|
archive_read_support_format_raw(a);
|
|
if (archive_read_open_memory(a, body.data(), body.size()) != ARCHIVE_OK) {
|
|
error = std::string("cannot open archive: ") + archive_error_string(a);
|
|
archive_read_free(a);
|
|
return false;
|
|
}
|
|
std::size_t total = 0;
|
|
archive_entry* entry = nullptr;
|
|
while (archive_read_next_header(a, &entry) == ARCHIVE_OK) {
|
|
if (archive_entry_filetype(entry) != AE_IFREG && archive_entry_filetype(entry) != 0) {
|
|
archive_read_data_skip(a);
|
|
continue;
|
|
}
|
|
std::string content;
|
|
char buffer[65536];
|
|
la_ssize_t n = 0;
|
|
while ((n = archive_read_data(a, buffer, sizeof(buffer))) > 0) {
|
|
total += static_cast<std::size_t>(n);
|
|
if (total > max_bytes) {
|
|
error = "archive expands beyond the size limit";
|
|
archive_read_free(a);
|
|
return false;
|
|
}
|
|
content.append(buffer, static_cast<std::size_t>(n));
|
|
}
|
|
out.push_back({"part", std::move(content)});
|
|
}
|
|
archive_read_free(a);
|
|
return true;
|
|
#endif
|
|
}
|
|
|
|
struct Resolved {
|
|
std::vector<std::string> download_urls;
|
|
std::string version;
|
|
std::string sha256; // expected digest of a single download (empty = not pinned)
|
|
std::size_t size = 0; // advertised size (0 = unknown)
|
|
std::string file_name;
|
|
};
|
|
|
|
// Optional "Name: value" header from a root-only file (first line).
|
|
bool read_auth_header(const FeedSpec& feed, std::string& header, std::string& error) {
|
|
header.clear();
|
|
if (feed.auth_header_file.empty()) {
|
|
return true;
|
|
}
|
|
std::ifstream input(feed.auth_header_file); // invariant-allow(ifstream): root-only header file, single getline
|
|
if (!input) {
|
|
error = "cannot read auth_header_file " + feed.auth_header_file;
|
|
return false;
|
|
}
|
|
std::getline(input, header);
|
|
while (!header.empty() && (header.back() == '\r' || header.back() == ' ')) {
|
|
header.pop_back();
|
|
}
|
|
if (header.find(':') == std::string::npos || header.find('\n') != std::string::npos) {
|
|
error = "auth_header_file must contain one \"Name: value\" line";
|
|
return false;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
bool finish_github_asset(const FeedSpec& feed, const UpdaterOptions& options, const std::string& download_url,
|
|
Resolved& out, std::string& error) {
|
|
if (download_url.empty()) {
|
|
error = "release " + out.version + " has no asset named " + feed.asset;
|
|
return false;
|
|
}
|
|
if (!download_url.starts_with("https://")) {
|
|
error = "asset URL is not https";
|
|
return false;
|
|
}
|
|
if (out.sha256.size() != 64) {
|
|
error = "release asset has no published sha256 digest; refusing unverified download";
|
|
return false;
|
|
}
|
|
if (out.size > options.max_download_bytes) {
|
|
error = "asset is larger than the download limit";
|
|
return false;
|
|
}
|
|
out.file_name = feed.asset;
|
|
out.download_urls = {download_url};
|
|
return true;
|
|
}
|
|
|
|
// REST API path: GET /repos/<repo>/releases/latest (60 requests/hour anonymously).
|
|
bool resolve_github_api(const FeedSpec& feed, Fetcher& fetcher, const UpdaterOptions& options, Resolved& out,
|
|
std::string& error, bool& rate_limited) {
|
|
rate_limited = false;
|
|
const auto api = "https://api.github.com/repos/" + feed.repo + "/releases/latest";
|
|
std::vector<std::string> headers{"Accept: application/vnd.github+json", "X-GitHub-Api-Version: 2022-11-28"};
|
|
if (!options.github_token.empty()) {
|
|
headers.push_back("Authorization: Bearer " + options.github_token);
|
|
}
|
|
const auto response = fetcher.get(api, 4 * 1024 * 1024, headers);
|
|
if (!response.ok) {
|
|
rate_limited = (response.http_status == 403 || response.http_status == 429)
|
|
&& response.body.find("rate limit") != std::string::npos;
|
|
error = rate_limited ? std::string("GitHub API rate limit reached") : "release lookup failed: " + response.error;
|
|
return false;
|
|
}
|
|
const auto release = json::parse(response.body, nullptr, false);
|
|
if (release.is_discarded() || !release.is_object()) {
|
|
error = "release lookup returned invalid JSON";
|
|
return false;
|
|
}
|
|
out.version = release.value("tag_name", "");
|
|
std::string download_url;
|
|
for (const auto& asset : release.value("assets", json::array())) {
|
|
if (asset.value("name", "") != feed.asset) {
|
|
continue;
|
|
}
|
|
download_url = asset.value("browser_download_url", "");
|
|
out.size = asset.value("size", std::size_t{0});
|
|
const auto digest = asset.value("digest", std::string{});
|
|
if (digest.rfind("sha256:", 0) == 0) {
|
|
out.sha256 = digest.substr(7);
|
|
}
|
|
}
|
|
return finish_github_asset(feed, options, download_url, out, error);
|
|
}
|
|
|
|
// Web path (no API quota): /releases/latest redirects to /releases/tag/<tag>;
|
|
// /releases/expanded_assets/<tag> lists each asset with its sha256 digest.
|
|
bool resolve_github_web(const FeedSpec& feed, Fetcher& fetcher, const UpdaterOptions& options, Resolved& out,
|
|
std::string& error) {
|
|
const std::string base = "https://github.com/" + feed.repo + "/releases";
|
|
const auto latest = fetcher.get(base + "/latest", 4 * 1024 * 1024, {});
|
|
const std::string marker = "/releases/tag/";
|
|
const auto tag_pos = latest.effective_url.find(marker);
|
|
if (!latest.ok || tag_pos == std::string::npos) {
|
|
error = "cannot resolve latest release tag: " + (latest.ok ? std::string("no redirect to a tag") : latest.error);
|
|
return false;
|
|
}
|
|
out.version = latest.effective_url.substr(tag_pos + marker.size());
|
|
if (!std::regex_match(out.version, std::regex("[A-Za-z0-9._-]{1,64}"))) {
|
|
error = "unexpected release tag";
|
|
return false;
|
|
}
|
|
const auto assets = fetcher.get(base + "/expanded_assets/" + out.version, 4 * 1024 * 1024, {});
|
|
if (!assets.ok) {
|
|
error = "cannot read release assets: " + assets.error;
|
|
return false;
|
|
}
|
|
const std::string link = "/" + feed.repo + "/releases/download/" + out.version + "/" + feed.asset + "\"";
|
|
const auto at = assets.body.find(link);
|
|
if (at == std::string::npos) {
|
|
error = "release " + out.version + " has no asset named " + feed.asset;
|
|
return false;
|
|
}
|
|
// The digest belongs to this asset if it appears before the next asset link.
|
|
const auto next = assets.body.find("/releases/download/", at + link.size());
|
|
const auto segment = assets.body.substr(at, (next == std::string::npos ? assets.body.size() : next) - at);
|
|
std::smatch digest;
|
|
if (std::regex_search(segment, digest, std::regex("sha256:([0-9a-f]{64})"))) {
|
|
out.sha256 = digest[1].str();
|
|
}
|
|
return finish_github_asset(feed, options, "https://github.com" + link.substr(0, link.size() - 1), out, error);
|
|
}
|
|
|
|
bool resolve_feed(const FeedSpec& feed, Fetcher& fetcher, const UpdaterOptions& options, Resolved& out, std::string& error) {
|
|
if (feed.type == "github-release") {
|
|
bool rate_limited = false;
|
|
if (resolve_github_api(feed, fetcher, options, out, error, rate_limited)) {
|
|
return true;
|
|
}
|
|
// API unavailable (often the shared anonymous quota): use the web pages.
|
|
Resolved web;
|
|
std::string web_error;
|
|
if (resolve_github_web(feed, fetcher, options, web, web_error)) {
|
|
std::cout << "[RuleUpdater] " << feed.id << ": GitHub API unavailable (" << error
|
|
<< "); resolved via release page instead" << std::endl;
|
|
out = web;
|
|
return true;
|
|
}
|
|
error = (rate_limited ? "GitHub API rate limit reached and release page lookup failed (" + web_error + ")"
|
|
: error + "; release page lookup also failed: " + web_error)
|
|
+ "; keeping current rules";
|
|
return false;
|
|
}
|
|
if (feed.type == "url") {
|
|
if (feed.urls.empty()) {
|
|
error = "feed has no urls";
|
|
return false;
|
|
}
|
|
out.download_urls = feed.urls;
|
|
out.sha256 = feed.urls.size() == 1 ? feed.sha256 : std::string{};
|
|
const auto& first = feed.urls.front();
|
|
out.file_name = fs::path(first.substr(0, first.find('?'))).filename().string();
|
|
return true;
|
|
}
|
|
error = "unknown feed type " + feed.type;
|
|
return false;
|
|
}
|
|
|
|
std::vector<YaraScanner::RuleSource> compile_sources(const std::string& local_rules,
|
|
const std::vector<std::pair<std::string, std::string>>& roots,
|
|
const std::string& skip_id) {
|
|
std::vector<YaraScanner::RuleSource> sources;
|
|
if (!local_rules.empty()) {
|
|
sources.push_back({local_rules, ""});
|
|
}
|
|
for (const auto& [id, path] : roots) {
|
|
if (id != skip_id) {
|
|
sources.push_back({path, id});
|
|
}
|
|
}
|
|
return sources;
|
|
}
|
|
|
|
FeedState process_feed(const FeedSpec& feed, const FeedState& previous, const UpdaterOptions& options,
|
|
const std::vector<std::pair<std::string, std::string>>& installed, Fetcher& fetcher) {
|
|
FeedState state = previous;
|
|
state.id = feed.id;
|
|
state.changed = false;
|
|
state.checked_at = now_ns();
|
|
state.error.clear();
|
|
|
|
const fs::path feed_dir = fs::path(options.feeds_dir) / feed.id;
|
|
const fs::path current = feed_dir / "current";
|
|
std::error_code ec;
|
|
const bool have_current = fs::exists(current, ec);
|
|
|
|
Resolved resolved;
|
|
std::string error;
|
|
if (!resolve_feed(feed, fetcher, options, resolved, error)) {
|
|
state.error = error;
|
|
state.ok = have_current; // previous version stays active
|
|
return state;
|
|
}
|
|
|
|
// Pinned and unchanged: nothing to download.
|
|
if (have_current && !resolved.sha256.empty() && resolved.sha256 == previous.sha256
|
|
&& (resolved.version.empty() || resolved.version == previous.version)) {
|
|
state.ok = true;
|
|
return state;
|
|
}
|
|
|
|
std::string auth_header;
|
|
if (!read_auth_header(feed, auth_header, error)) {
|
|
state.error = error;
|
|
state.ok = have_current;
|
|
return state;
|
|
}
|
|
std::vector<std::string> headers;
|
|
if (!auth_header.empty()) {
|
|
headers.push_back(auth_header);
|
|
}
|
|
|
|
// Download every URL; the change digest covers all of them in order.
|
|
std::vector<std::string> bodies;
|
|
std::string combined_digest_input;
|
|
std::size_t total = 0;
|
|
for (const auto& url : resolved.download_urls) {
|
|
const auto response = fetcher.get(url, options.max_download_bytes, headers);
|
|
if (!response.ok) {
|
|
state.error = "download failed (" + url + "): " + response.error;
|
|
state.ok = have_current;
|
|
return state;
|
|
}
|
|
total += response.body.size();
|
|
if (total > options.max_download_bytes) {
|
|
state.error = "feed downloads exceed the size limit";
|
|
state.ok = have_current;
|
|
return state;
|
|
}
|
|
if (resolved.download_urls.size() == 1 && !resolved.sha256.empty()) {
|
|
const auto digest = sha256_hex(response.body);
|
|
if (digest != resolved.sha256) {
|
|
state.error = "sha256 mismatch: expected " + resolved.sha256 + ", got " + digest;
|
|
state.ok = have_current;
|
|
return state;
|
|
}
|
|
}
|
|
combined_digest_input += sha256_hex(response.body);
|
|
bodies.push_back(response.body);
|
|
}
|
|
const auto digest = bodies.size() == 1 ? sha256_hex(bodies.front()) : sha256_hex(combined_digest_input);
|
|
if (have_current && digest == previous.sha256) {
|
|
state.ok = true;
|
|
return state;
|
|
}
|
|
if (resolved.version.empty()) {
|
|
resolved.version = "sha256-" + digest.substr(0, 16);
|
|
}
|
|
|
|
std::vector<RuleFileBlob> files;
|
|
std::size_t entry_count = 0;
|
|
if (is_list_kind(feed.kind)) {
|
|
// Threat-intel list: validate and normalize every entry.
|
|
std::vector<std::string> entries;
|
|
for (const auto& body : bodies) {
|
|
std::vector<RuleFileBlob> parts;
|
|
if (looks_like_archive(body)) {
|
|
if (!extract_text_files(body, options.max_extract_bytes, parts, error)) {
|
|
state.error = error;
|
|
state.ok = have_current;
|
|
return state;
|
|
}
|
|
} else {
|
|
parts.push_back({"list.txt", body});
|
|
}
|
|
for (const auto& part : parts) {
|
|
parse_ioc_list(feed.kind, part.content, max_list_entries(feed.kind), entries);
|
|
}
|
|
}
|
|
std::sort(entries.begin(), entries.end());
|
|
entries.erase(std::unique(entries.begin(), entries.end()), entries.end());
|
|
if (entries.empty()) {
|
|
state.error = "feed contained no valid " + feed.kind + " entries";
|
|
state.ok = have_current;
|
|
return state;
|
|
}
|
|
// A sudden collapse usually means a truncated or error-page download.
|
|
if (have_current && previous.rules >= 100 && entries.size() * 10 < previous.rules) {
|
|
state.error = "new list has " + std::to_string(entries.size()) + " entries vs "
|
|
+ std::to_string(previous.rules) + " before; refusing suspected truncated download";
|
|
state.ok = true;
|
|
return state;
|
|
}
|
|
std::string text;
|
|
for (const auto& entry : entries) {
|
|
text += entry;
|
|
text += '\n';
|
|
}
|
|
files.push_back({"list.txt", std::move(text)});
|
|
entry_count = entries.size();
|
|
} else {
|
|
for (const auto& body : bodies) {
|
|
if (looks_like_archive(body)) {
|
|
std::vector<RuleFileBlob> extracted;
|
|
if (!extract_rules(body, options.max_extract_bytes, extracted, error)) {
|
|
state.error = error;
|
|
state.ok = have_current;
|
|
return state;
|
|
}
|
|
for (auto& file : extracted) {
|
|
files.push_back(std::move(file));
|
|
}
|
|
} else {
|
|
const auto name = is_rule_name(resolved.file_name) ? resolved.file_name : std::string("rules.yar");
|
|
const auto unique = bodies.size() == 1 ? name : std::to_string(files.size()) + "-" + name;
|
|
files.push_back({sanitize_rule_file_name(unique), body});
|
|
}
|
|
}
|
|
}
|
|
|
|
// Stage, compile-check together with everything else that will be active,
|
|
// then switch `current` atomically.
|
|
fs::create_directories(feed_dir, ec);
|
|
std::string staging_template = (feed_dir / ".staging-XXXXXX").string();
|
|
if (::mkdtemp(staging_template.data()) == nullptr) {
|
|
state.error = "cannot create staging directory in " + feed_dir.string();
|
|
state.ok = have_current;
|
|
return state;
|
|
}
|
|
const fs::path staging(staging_template);
|
|
std::size_t rule_decls = entry_count;
|
|
for (const auto& file : files) {
|
|
std::ofstream out(staging / file.name, std::ios::binary);
|
|
out.write(file.content.data(), static_cast<std::streamsize>(file.content.size()));
|
|
if (feed.kind == "yara") {
|
|
rule_decls += count_rule_declarations(file.content);
|
|
}
|
|
}
|
|
|
|
if (feed.kind == "yara") {
|
|
auto sources = compile_sources(options.local_rules_path, installed, feed.id);
|
|
sources.push_back({staging.string(), feed.id});
|
|
std::size_t compiled_rules = 0;
|
|
if (!YaraScanner::check_rules(sources, &error, &compiled_rules)) {
|
|
fs::remove_all(staging, ec);
|
|
state.error = "new rules failed to compile (previous version kept): " + error;
|
|
state.ok = have_current;
|
|
return state;
|
|
}
|
|
}
|
|
|
|
const auto version_name = version_dir_name(resolved.version, digest);
|
|
const fs::path target = feed_dir / version_name;
|
|
std::string previous_target;
|
|
if (fs::is_symlink(current, ec)) {
|
|
previous_target = fs::read_symlink(current, ec).string();
|
|
}
|
|
if (fs::exists(target, ec)) {
|
|
fs::remove_all(target, ec);
|
|
}
|
|
fs::rename(staging, target, ec);
|
|
if (!ec) {
|
|
fs::permissions(target, fs::perms::owner_all | fs::perms::group_read | fs::perms::group_exec, ec);
|
|
ec.clear();
|
|
}
|
|
if (ec) {
|
|
fs::remove_all(staging, ec);
|
|
state.error = "cannot install version directory: " + ec.message();
|
|
state.ok = have_current;
|
|
return state;
|
|
}
|
|
const fs::path link_temp = feed_dir / ".current.tmp";
|
|
fs::remove(link_temp, ec);
|
|
fs::create_directory_symlink(version_name, link_temp, ec);
|
|
if (!ec) {
|
|
fs::rename(link_temp, current, ec);
|
|
}
|
|
if (ec) {
|
|
state.error = "cannot switch current version: " + ec.message();
|
|
state.ok = have_current;
|
|
return state;
|
|
}
|
|
|
|
// Keep the new version and the one it replaced; drop older ones.
|
|
for (const auto& entry : fs::directory_iterator(feed_dir, ec)) {
|
|
const auto name = entry.path().filename().string();
|
|
if (name == "current" || name == version_name || name == previous_target || name.starts_with(".")) {
|
|
continue;
|
|
}
|
|
if (entry.is_directory(ec) && !entry.is_symlink(ec)) {
|
|
fs::remove_all(entry.path(), ec);
|
|
}
|
|
}
|
|
|
|
state.ok = true;
|
|
state.changed = true;
|
|
state.version = resolved.version;
|
|
state.sha256 = digest;
|
|
state.rule_files = files.size();
|
|
state.rules = rule_decls;
|
|
state.updated_at = now_ns();
|
|
return state;
|
|
}
|
|
|
|
std::map<std::string, FeedState> load_states(const std::string& feeds_dir) {
|
|
std::map<std::string, FeedState> states;
|
|
const auto parsed = json::parse(read_feed_state(feeds_dir), nullptr, false);
|
|
if (parsed.is_discarded() || !parsed.is_object()) {
|
|
return states;
|
|
}
|
|
for (const auto& f : parsed.value("feeds", json::array())) {
|
|
FeedState s;
|
|
s.id = f.value("id", "");
|
|
s.ok = f.value("ok", false);
|
|
s.version = f.value("version", "");
|
|
s.sha256 = f.value("sha256", "");
|
|
s.rule_files = f.value("rule_files", std::size_t{0});
|
|
s.rules = f.value("rules", std::size_t{0});
|
|
s.checked_at = f.value("checked_at", int64_t{0});
|
|
s.updated_at = f.value("updated_at", int64_t{0});
|
|
s.error = f.value("error", "");
|
|
if (!s.id.empty()) {
|
|
states[s.id] = s;
|
|
}
|
|
}
|
|
return states;
|
|
}
|
|
|
|
} // namespace
|
|
|
|
|
|
std::string sanitize_rule_file_name(const std::string& name) {
|
|
std::string out;
|
|
for (const char c : name) {
|
|
const bool safe = std::isalnum(static_cast<unsigned char>(c)) || c == '.' || c == '_' || c == '-';
|
|
out.push_back(safe ? c : '_');
|
|
}
|
|
while (!out.empty() && out.front() == '.') {
|
|
out.erase(out.begin());
|
|
}
|
|
if (out.empty() || !is_rule_name(out)) {
|
|
out += ".yar";
|
|
}
|
|
return out.substr(0, 128);
|
|
}
|
|
|
|
std::size_t count_rule_declarations(const std::string& text) {
|
|
static const std::regex declaration(R"((^|\n)[ \t]*((private|global)[ \t]+)*rule[ \t]+[A-Za-z_][A-Za-z0-9_]*)");
|
|
return static_cast<std::size_t>(std::distance(
|
|
std::sregex_iterator(text.begin(), text.end(), declaration), std::sregex_iterator()));
|
|
}
|
|
|
|
UpdateReport update_feeds(const std::vector<FeedSpec>& feeds, const UpdaterOptions& options, Fetcher& fetcher) {
|
|
UpdateReport report;
|
|
std::error_code ec;
|
|
fs::create_directories(options.feeds_dir, ec);
|
|
fs::remove(fs::path(options.feeds_dir) / "update-requested", ec);
|
|
|
|
auto previous = load_states(options.feeds_dir);
|
|
for (const auto& feed : feeds) {
|
|
if (!feed.enabled) {
|
|
FeedState state = previous.contains(feed.id) ? previous[feed.id] : FeedState{};
|
|
state.id = feed.id;
|
|
state.error = "disabled";
|
|
report.feeds.push_back(state);
|
|
continue;
|
|
}
|
|
const auto installed = installed_feed_roots(feeds, options.feeds_dir);
|
|
const auto state = process_feed(feed, previous.contains(feed.id) ? previous[feed.id] : FeedState{},
|
|
options, installed, fetcher);
|
|
std::cout << "[RuleUpdater] " << feed.id << ": "
|
|
<< (state.changed ? "installed " + state.version
|
|
: (state.error.empty() ? "up to date (" + state.version + ")" : "error: " + state.error))
|
|
<< std::endl;
|
|
report.any_changed |= state.changed;
|
|
report.feeds.push_back(state);
|
|
}
|
|
|
|
// Remove directories of feeds that are no longer in the policy.
|
|
std::set<std::string> known;
|
|
for (const auto& feed : feeds) {
|
|
known.insert(feed.id);
|
|
}
|
|
for (const auto& entry : fs::directory_iterator(options.feeds_dir, ec)) {
|
|
const auto name = entry.path().filename().string();
|
|
if (entry.is_directory(ec) && !entry.is_symlink(ec) && !known.contains(name)
|
|
&& std::regex_match(name, std::regex("[a-z0-9-]{1,32}"))) {
|
|
fs::remove_all(entry.path(), ec);
|
|
report.any_changed = true;
|
|
}
|
|
}
|
|
|
|
json state = {{"updated_at", now_ns()}, {"feeds", json::array()}};
|
|
for (const auto& s : report.feeds) {
|
|
state["feeds"].push_back({
|
|
{"id", s.id}, {"ok", s.ok}, {"version", s.version}, {"sha256", s.sha256},
|
|
{"rule_files", s.rule_files}, {"rules", s.rules}, {"checked_at", s.checked_at},
|
|
{"updated_at", s.updated_at}, {"error", s.error},
|
|
});
|
|
}
|
|
std::string error;
|
|
if (!write_file_atomic(fs::path(options.feeds_dir) / "state.json",
|
|
state.dump(2, ' ', false, json::error_handler_t::replace), &error)) {
|
|
std::cerr << "[RuleUpdater] " << error << std::endl;
|
|
}
|
|
|
|
if (report.any_changed && !options.socket_path.empty()) {
|
|
std::string reload_error;
|
|
const auto reply = control_request(options.socket_path, "reload-rules", 60s, &reload_error);
|
|
if (reply && reply->find("\"reloaded\":true") != std::string::npos) {
|
|
report.reloaded = true;
|
|
} else {
|
|
report.reload_error = reply ? *reply : reload_error;
|
|
}
|
|
}
|
|
return report;
|
|
}
|
|
|
|
bool rule_fetch_enabled(bool configured, bool force) {
|
|
return force || configured;
|
|
}
|
|
|
|
#if STRAWBERRY_HAVE_UPDATER
|
|
namespace {
|
|
|
|
class CurlFetcher : public Fetcher {
|
|
public:
|
|
CurlFetcher() { curl_global_init(CURL_GLOBAL_DEFAULT); }
|
|
~CurlFetcher() override { curl_global_cleanup(); }
|
|
|
|
FetchResult get(const std::string& url, std::size_t max_bytes, const std::vector<std::string>& headers) override {
|
|
FetchResult result;
|
|
if (!url.starts_with("https://")) {
|
|
result.error = "only https URLs are allowed";
|
|
return result;
|
|
}
|
|
CURL* curl = curl_easy_init();
|
|
if (curl == nullptr) {
|
|
result.error = "curl_easy_init failed";
|
|
return result;
|
|
}
|
|
struct Sink {
|
|
std::string* body;
|
|
std::size_t max;
|
|
bool overflow = false;
|
|
} sink{&result.body, max_bytes};
|
|
|
|
curl_slist* header_list = nullptr;
|
|
for (const auto& header : headers) {
|
|
header_list = curl_slist_append(header_list, header.c_str());
|
|
}
|
|
|
|
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
|
|
curl_easy_setopt(curl, CURLOPT_PROTOCOLS_STR, "https");
|
|
curl_easy_setopt(curl, CURLOPT_REDIR_PROTOCOLS_STR, "https");
|
|
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
|
|
curl_easy_setopt(curl, CURLOPT_MAXREDIRS, 3L);
|
|
curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 1L);
|
|
curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 2L);
|
|
curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, 20L);
|
|
curl_easy_setopt(curl, CURLOPT_TIMEOUT, 300L);
|
|
curl_easy_setopt(curl, CURLOPT_USERAGENT, "strawberry-rule-updater/0.1");
|
|
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, header_list);
|
|
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &sink);
|
|
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, +[](char* data, size_t size, size_t count, void* user) noexcept -> size_t {
|
|
// Runs inside libcurl's C frames: an escaping exception (bad_alloc
|
|
// from append) would terminate the daemon. Returning 0 aborts.
|
|
auto* s = static_cast<Sink*>(user);
|
|
const size_t bytes = size * count;
|
|
if (s->body->size() + bytes > s->max) {
|
|
s->overflow = true;
|
|
return 0; // abort transfer
|
|
}
|
|
try {
|
|
s->body->append(data, bytes);
|
|
} catch (...) {
|
|
s->overflow = true;
|
|
return 0;
|
|
}
|
|
return bytes;
|
|
});
|
|
|
|
const CURLcode rc = curl_easy_perform(curl);
|
|
curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &result.http_status);
|
|
char* effective = nullptr;
|
|
if (curl_easy_getinfo(curl, CURLINFO_EFFECTIVE_URL, &effective) == CURLE_OK && effective != nullptr) {
|
|
result.effective_url = effective;
|
|
}
|
|
curl_slist_free_all(header_list);
|
|
curl_easy_cleanup(curl);
|
|
|
|
if (sink.overflow) {
|
|
result.error = "response exceeds " + std::to_string(max_bytes) + " bytes";
|
|
result.body.clear();
|
|
} else if (rc != CURLE_OK) {
|
|
result.error = curl_easy_strerror(rc);
|
|
} else if (result.http_status != 200) {
|
|
result.error = "HTTP " + std::to_string(result.http_status);
|
|
} else {
|
|
result.ok = true;
|
|
}
|
|
return result;
|
|
}
|
|
};
|
|
|
|
} // namespace
|
|
|
|
std::unique_ptr<Fetcher> make_https_fetcher() {
|
|
return std::make_unique<CurlFetcher>();
|
|
}
|
|
#else
|
|
std::unique_ptr<Fetcher> make_https_fetcher() {
|
|
return nullptr;
|
|
}
|
|
#endif
|
|
|
|
} // namespace strawberry
|