Some checks failed
Build and Test / build (debug, clang) (push) Has been cancelled
Build and Test / build (debug, gcc) (push) Has been cancelled
Build and Test / build (release, clang) (push) Has been cancelled
Build and Test / build (release, gcc) (push) Has been cancelled
Build and Test / docker (push) Has been cancelled
Nothing shipped used the library after the event store landed. A small reader/writer for its file format (src/store/legacy_kv) replaces it for the legacy code, and the submodule, subproject and library test are removed. A writer open now imports a legacy database.db file into the event store: indicators are replayed in an order consistent with the old hash/rule indexes, the result is verified against the old file, the old file is kept as database.db.migrated-<ts>, and an import interrupted between its final renames is finished on the next open. Read-only handles read a legacy file directly and switch to the store once it is imported. Ids re-stored under another hash are reported (their old index entries cannot be carried over). Tests: legacy import properties I1-I6 (locked and corrupt files are left untouched), the differential test now shares its comparisons, and fuzz_legacy_kv covers the file parser. A copy of a real 137 KB legacy database imported cleanly into 7 KB. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
259 lines
9.9 KiB
C++
259 lines
9.9 KiB
C++
// Legacy (unqlite-format) database import. A failure prints the seed; re-run
|
|
// with STRAWBERRY_PROPERTY_SEED=<seed>.
|
|
//
|
|
// I1 a writer open imports the file: every query answers as the old file did,
|
|
// the old file is kept as <path>.migrated-<ts>, reopening is a no-op
|
|
// I2 a read-only handle reads the legacy file directly, and refresh()
|
|
// switches to the store once the daemon has imported it
|
|
// I3 a crash between the final renames is finished on the next open
|
|
// I4 a stale .importing directory is moved aside (never deleted) and the
|
|
// import redone
|
|
// I5 a locked or corrupt legacy file is left untouched and the open fails
|
|
// I6 an id re-stored under another hash is reported (its old index entry
|
|
// cannot be carried over); everything else still matches
|
|
|
|
#include "strawberry/database.hpp"
|
|
|
|
#include "../src/store/legacy_import.hpp"
|
|
#include "../src/store/legacy_unqlite.hpp"
|
|
|
|
#include <cstdint>
|
|
#include <cstdlib>
|
|
#include <filesystem>
|
|
#include <fstream>
|
|
#include <iostream>
|
|
#include <random>
|
|
#include <string>
|
|
|
|
#include <fcntl.h>
|
|
#include <sys/file.h>
|
|
#include <unistd.h>
|
|
|
|
using namespace strawberry;
|
|
namespace fs = std::filesystem;
|
|
|
|
namespace {
|
|
|
|
uint64_t seed_from_env() {
|
|
if (const char* text = std::getenv("STRAWBERRY_PROPERTY_SEED")) {
|
|
return std::strtoull(text, nullptr, 10);
|
|
}
|
|
return std::random_device{}();
|
|
}
|
|
|
|
const uint64_t kSeed = seed_from_env();
|
|
|
|
#define PROPERTY(cond) \
|
|
do { \
|
|
if (!(cond)) { \
|
|
std::cerr << "PROPERTY FAILED: " #cond " (seed " << kSeed << ", " << __FILE__ << ":" \
|
|
<< __LINE__ << ")" << std::endl; \
|
|
std::abort(); \
|
|
} \
|
|
} while (0)
|
|
|
|
} // namespace
|
|
|
|
#include "store_compare.hpp"
|
|
|
|
namespace {
|
|
|
|
using namespace store_compare;
|
|
|
|
Pools pools() {
|
|
Pools p;
|
|
for (int i = 0; i < 8; ++i) p.ids.push_back("ind-" + std::to_string(i));
|
|
for (int i = 0; i < 4; ++i) p.hashes.push_back(std::string(8, static_cast<char>('a' + i)));
|
|
for (int i = 0; i < 3; ++i) p.rules.push_back("Rule_" + std::to_string(i));
|
|
for (int i = 0; i < 8; ++i) p.nodes.push_back((i % 2 ? "process_" : "file_") + std::to_string(i));
|
|
p.types = {"process", "file", "indicator"};
|
|
p.rels = {"spawned", "executed", "matched"};
|
|
p.words = {"", "high", "low", "multi word"};
|
|
return p;
|
|
}
|
|
|
|
/// Writes a random legacy database at `path` (with the old store's code).
|
|
void make_legacy(const std::string& path, std::mt19937_64& rng, const Pools& p) {
|
|
LegacyUnqliteDatabase db;
|
|
PROPERTY(db.open(path));
|
|
const int n = 20 + static_cast<int>(rng() % 120);
|
|
for (int i = 0; i < n; ++i) {
|
|
switch (rng() % 4) {
|
|
case 0: {
|
|
IndicatorEntry e;
|
|
const std::size_t id = rng() % p.ids.size();
|
|
e.id = p.ids[id];
|
|
// Real detection ids never change hash or rule (they embed
|
|
// pid, time and match index); I6 covers the case where one did.
|
|
e.hash = p.hashes[id % p.hashes.size()];
|
|
e.rule_name = p.rules[id % p.rules.size()];
|
|
e.severity = p.words[rng() % p.words.size()];
|
|
e.description = "d";
|
|
e.timestamp = static_cast<int64_t>(rng() % 50);
|
|
e.metadata["k"] = std::to_string(rng() % 3);
|
|
PROPERTY(db.store_indicator(e));
|
|
break;
|
|
}
|
|
case 1: {
|
|
GraphNode node{p.nodes[rng() % p.nodes.size()], p.types[rng() % p.types.size()], {{"x", std::to_string(rng() % 3)}}};
|
|
PROPERTY(db.add_graph_node(node));
|
|
break;
|
|
}
|
|
case 2: {
|
|
// Endpoints may be nodes that are never stored, as in real data.
|
|
GraphEdge e{p.nodes[rng() % p.nodes.size()], p.nodes[rng() % p.nodes.size()], p.rels[rng() % p.rels.size()],
|
|
static_cast<int64_t>(rng() % 12)};
|
|
PROPERTY(db.add_graph_edge(e));
|
|
break;
|
|
}
|
|
default: {
|
|
FindingRecord f;
|
|
f.id = "finding-" + std::to_string(rng() % 5);
|
|
f.title = "t";
|
|
f.severity = "high";
|
|
f.score = static_cast<int>(rng() % 5) * 10;
|
|
f.related_node_ids = {p.nodes[rng() % p.nodes.size()]};
|
|
f.first_seen = 1;
|
|
f.last_seen = static_cast<int64_t>(rng() % 9);
|
|
PROPERTY(db.store_finding(f));
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
std::string backup_of(const fs::path& dir, const std::string& name) {
|
|
for (const auto& e : fs::directory_iterator(dir)) {
|
|
if (e.path().filename().string().starts_with(name + ".migrated-")) return e.path().string();
|
|
}
|
|
return {};
|
|
}
|
|
|
|
uint64_t g_checks = 0;
|
|
|
|
void run(std::mt19937_64& rng, const fs::path& root, int iteration) {
|
|
const Pools p = pools();
|
|
const fs::path dir = root / ("i" + std::to_string(iteration));
|
|
fs::create_directories(dir);
|
|
const std::string path = (dir / "database.db").string();
|
|
make_legacy(path, rng, p);
|
|
const std::string reference = (dir / "reference.db").string();
|
|
fs::copy_file(path, reference);
|
|
LegacyUnqliteDatabase ref;
|
|
PROPERTY(ref.open_read_only(reference));
|
|
|
|
// I2: read-only on the legacy file, before any import.
|
|
Database reader;
|
|
PROPERTY(reader.open_read_only(path));
|
|
compare(ref, reader, p, g_checks);
|
|
PROPERTY(!reader.refresh());
|
|
|
|
const int scenario = iteration % 4;
|
|
if (scenario == 1) {
|
|
// I4: a stale partial import from an earlier crash.
|
|
fs::create_directories(path + ".importing/evidence");
|
|
std::ofstream(path + ".importing/evidence/seg-1.log") << "partial";
|
|
}
|
|
if (scenario == 2) {
|
|
// I5: another writer holds the old lock: refuse, change nothing.
|
|
const int fd = ::open((path + ".lock").c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600);
|
|
PROPERTY(fd >= 0 && ::flock(fd, LOCK_EX | LOCK_NB) == 0);
|
|
Database blocked;
|
|
PROPERTY(!blocked.open(path));
|
|
PROPERTY(fs::is_regular_file(path) && backup_of(dir, "database.db").empty());
|
|
::close(fd);
|
|
}
|
|
|
|
// I1: the writer open imports.
|
|
{
|
|
Database writer;
|
|
PROPERTY(writer.open(path));
|
|
compare(ref, writer, p, g_checks);
|
|
}
|
|
PROPERTY(fs::is_directory(path));
|
|
const std::string backup = backup_of(dir, "database.db");
|
|
PROPERTY(!backup.empty());
|
|
PROPERTY(fs::file_size(backup) == fs::file_size(reference)); // the old file, unchanged
|
|
if (scenario == 1) {
|
|
bool moved_aside = false;
|
|
for (const auto& e : fs::directory_iterator(dir)) {
|
|
moved_aside |= e.path().filename().string().starts_with("database.db.importing.failed-");
|
|
}
|
|
PROPERTY(moved_aside);
|
|
}
|
|
|
|
// I2: the reader notices the import and switches to the store.
|
|
PROPERTY(reader.refresh());
|
|
compare(ref, reader, p, g_checks);
|
|
reader.close();
|
|
|
|
if (scenario == 3) {
|
|
// I3: crash between the two renames: old file already moved to its
|
|
// backup name, the new store still under .importing.
|
|
fs::rename(path, path + ".importing");
|
|
PROPERTY(!fs::exists(path));
|
|
}
|
|
Database again;
|
|
PROPERTY(again.open(path)); // reopen: no second import (or finish I3)
|
|
compare(ref, again, p, g_checks);
|
|
PROPERTY(backup_of(dir, "database.db") == backup);
|
|
again.close();
|
|
}
|
|
|
|
void corrupt_file_is_left_alone(const fs::path& root) { // I5
|
|
const std::string path = (root / "corrupt.db").string();
|
|
{
|
|
std::ofstream f(path, std::ios::binary);
|
|
f.write("LQNU\x01\0\0\0\x05\0\0\0\0\0\0\0\x03\0\0\0\0\0\0\0abc", 27); // 5 entries promised, 0.5 present
|
|
}
|
|
const auto size = fs::file_size(path);
|
|
Database db;
|
|
PROPERTY(!db.open(path));
|
|
PROPERTY(fs::is_regular_file(path) && fs::file_size(path) == size);
|
|
PROPERTY(!fs::exists(path + ".importing") || fs::is_directory(path + ".importing"));
|
|
Database ro;
|
|
PROPERTY(!ro.open_read_only(path));
|
|
}
|
|
|
|
void restored_under_other_hash_is_reported(const fs::path& root) { // I6
|
|
const std::string path = (root / "restored.db").string();
|
|
{
|
|
LegacyUnqliteDatabase db;
|
|
PROPERTY(db.open(path));
|
|
IndicatorEntry e;
|
|
e.id = "ind-x";
|
|
e.hash = "aaaa";
|
|
e.rule_name = "R";
|
|
e.timestamp = 1;
|
|
PROPERTY(db.store_indicator(e));
|
|
e.hash = "bbbb";
|
|
PROPERTY(db.store_indicator(e));
|
|
}
|
|
store::LegacyImportReport report;
|
|
std::string error;
|
|
PROPERTY(store::import_legacy_database(path, report, error));
|
|
bool warned = false;
|
|
for (const auto& w : report.warnings) warned |= w.find("re-stored") != std::string::npos;
|
|
PROPERTY(warned && report.indicators == 1);
|
|
Database db;
|
|
PROPERTY(db.open(path));
|
|
PROPERTY(db.find_indicators_by_hash("bbbb").size() == 1 && db.list_indicators().size() == 1);
|
|
}
|
|
|
|
} // namespace
|
|
|
|
int main(int argc, char** argv) {
|
|
const int iterations = argc > 1 ? std::atoi(argv[1]) : 24;
|
|
std::cout << "legacy import properties, seed " << kSeed << std::endl;
|
|
std::mt19937_64 rng(kSeed);
|
|
const fs::path root = fs::temp_directory_path() / ("sb-import-" + std::to_string(::getpid()));
|
|
fs::remove_all(root);
|
|
fs::create_directories(root);
|
|
for (int i = 0; i < iterations; ++i) run(rng, root, i);
|
|
corrupt_file_is_left_alone(root);
|
|
restored_under_other_hash_is_reported(root);
|
|
fs::remove_all(root);
|
|
std::cout << "legacy import: all properties hold (" << iterations << " imports, " << g_checks
|
|
<< " full comparisons)" << std::endl;
|
|
return 0;
|
|
}
|