strawberry/tests/test_store_import.cpp
trvon dc50f06f8d
Some checks failed
Build and Test / build (debug, clang) (push) Has been cancelled
Build and Test / build (debug, gcc) (push) Has been cancelled
Build and Test / build (release, clang) (push) Has been cancelled
Build and Test / build (release, gcc) (push) Has been cancelled
Build and Test / docker (push) Has been cancelled
Drop unqlite-cpp and import legacy databases
Nothing shipped used the library after the event store landed. A small
reader/writer for its file format (src/store/legacy_kv) replaces it for
the legacy code, and the submodule, subproject and library test are
removed.

A writer open now imports a legacy database.db file into the event
store: indicators are replayed in an order consistent with the old
hash/rule indexes, the result is verified against the old file, the
old file is kept as database.db.migrated-<ts>, and an import
interrupted between its final renames is finished on the next open.
Read-only handles read a legacy file directly and switch to the store
once it is imported. Ids re-stored under another hash are reported
(their old index entries cannot be carried over).

Tests: legacy import properties I1-I6 (locked and corrupt files are
left untouched), the differential test now shares its comparisons, and
fuzz_legacy_kv covers the file parser. A copy of a real 137 KB legacy
database imported cleanly into 7 KB.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-25 18:46:21 -06:00

259 lines
9.9 KiB
C++

// Legacy (unqlite-format) database import. A failure prints the seed; re-run
// with STRAWBERRY_PROPERTY_SEED=<seed>.
//
// I1 a writer open imports the file: every query answers as the old file did,
// the old file is kept as <path>.migrated-<ts>, reopening is a no-op
// I2 a read-only handle reads the legacy file directly, and refresh()
// switches to the store once the daemon has imported it
// I3 a crash between the final renames is finished on the next open
// I4 a stale .importing directory is moved aside (never deleted) and the
// import redone
// I5 a locked or corrupt legacy file is left untouched and the open fails
// I6 an id re-stored under another hash is reported (its old index entry
// cannot be carried over); everything else still matches
#include "strawberry/database.hpp"
#include "../src/store/legacy_import.hpp"
#include "../src/store/legacy_unqlite.hpp"
#include <cstdint>
#include <cstdlib>
#include <filesystem>
#include <fstream>
#include <iostream>
#include <random>
#include <string>
#include <fcntl.h>
#include <sys/file.h>
#include <unistd.h>
using namespace strawberry;
namespace fs = std::filesystem;
namespace {
uint64_t seed_from_env() {
if (const char* text = std::getenv("STRAWBERRY_PROPERTY_SEED")) {
return std::strtoull(text, nullptr, 10);
}
return std::random_device{}();
}
const uint64_t kSeed = seed_from_env();
#define PROPERTY(cond) \
do { \
if (!(cond)) { \
std::cerr << "PROPERTY FAILED: " #cond " (seed " << kSeed << ", " << __FILE__ << ":" \
<< __LINE__ << ")" << std::endl; \
std::abort(); \
} \
} while (0)
} // namespace
#include "store_compare.hpp"
namespace {
using namespace store_compare;
Pools pools() {
Pools p;
for (int i = 0; i < 8; ++i) p.ids.push_back("ind-" + std::to_string(i));
for (int i = 0; i < 4; ++i) p.hashes.push_back(std::string(8, static_cast<char>('a' + i)));
for (int i = 0; i < 3; ++i) p.rules.push_back("Rule_" + std::to_string(i));
for (int i = 0; i < 8; ++i) p.nodes.push_back((i % 2 ? "process_" : "file_") + std::to_string(i));
p.types = {"process", "file", "indicator"};
p.rels = {"spawned", "executed", "matched"};
p.words = {"", "high", "low", "multi word"};
return p;
}
/// Writes a random legacy database at `path` (with the old store's code).
void make_legacy(const std::string& path, std::mt19937_64& rng, const Pools& p) {
LegacyUnqliteDatabase db;
PROPERTY(db.open(path));
const int n = 20 + static_cast<int>(rng() % 120);
for (int i = 0; i < n; ++i) {
switch (rng() % 4) {
case 0: {
IndicatorEntry e;
const std::size_t id = rng() % p.ids.size();
e.id = p.ids[id];
// Real detection ids never change hash or rule (they embed
// pid, time and match index); I6 covers the case where one did.
e.hash = p.hashes[id % p.hashes.size()];
e.rule_name = p.rules[id % p.rules.size()];
e.severity = p.words[rng() % p.words.size()];
e.description = "d";
e.timestamp = static_cast<int64_t>(rng() % 50);
e.metadata["k"] = std::to_string(rng() % 3);
PROPERTY(db.store_indicator(e));
break;
}
case 1: {
GraphNode node{p.nodes[rng() % p.nodes.size()], p.types[rng() % p.types.size()], {{"x", std::to_string(rng() % 3)}}};
PROPERTY(db.add_graph_node(node));
break;
}
case 2: {
// Endpoints may be nodes that are never stored, as in real data.
GraphEdge e{p.nodes[rng() % p.nodes.size()], p.nodes[rng() % p.nodes.size()], p.rels[rng() % p.rels.size()],
static_cast<int64_t>(rng() % 12)};
PROPERTY(db.add_graph_edge(e));
break;
}
default: {
FindingRecord f;
f.id = "finding-" + std::to_string(rng() % 5);
f.title = "t";
f.severity = "high";
f.score = static_cast<int>(rng() % 5) * 10;
f.related_node_ids = {p.nodes[rng() % p.nodes.size()]};
f.first_seen = 1;
f.last_seen = static_cast<int64_t>(rng() % 9);
PROPERTY(db.store_finding(f));
break;
}
}
}
}
std::string backup_of(const fs::path& dir, const std::string& name) {
for (const auto& e : fs::directory_iterator(dir)) {
if (e.path().filename().string().starts_with(name + ".migrated-")) return e.path().string();
}
return {};
}
uint64_t g_checks = 0;
void run(std::mt19937_64& rng, const fs::path& root, int iteration) {
const Pools p = pools();
const fs::path dir = root / ("i" + std::to_string(iteration));
fs::create_directories(dir);
const std::string path = (dir / "database.db").string();
make_legacy(path, rng, p);
const std::string reference = (dir / "reference.db").string();
fs::copy_file(path, reference);
LegacyUnqliteDatabase ref;
PROPERTY(ref.open_read_only(reference));
// I2: read-only on the legacy file, before any import.
Database reader;
PROPERTY(reader.open_read_only(path));
compare(ref, reader, p, g_checks);
PROPERTY(!reader.refresh());
const int scenario = iteration % 4;
if (scenario == 1) {
// I4: a stale partial import from an earlier crash.
fs::create_directories(path + ".importing/evidence");
std::ofstream(path + ".importing/evidence/seg-1.log") << "partial";
}
if (scenario == 2) {
// I5: another writer holds the old lock: refuse, change nothing.
const int fd = ::open((path + ".lock").c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600);
PROPERTY(fd >= 0 && ::flock(fd, LOCK_EX | LOCK_NB) == 0);
Database blocked;
PROPERTY(!blocked.open(path));
PROPERTY(fs::is_regular_file(path) && backup_of(dir, "database.db").empty());
::close(fd);
}
// I1: the writer open imports.
{
Database writer;
PROPERTY(writer.open(path));
compare(ref, writer, p, g_checks);
}
PROPERTY(fs::is_directory(path));
const std::string backup = backup_of(dir, "database.db");
PROPERTY(!backup.empty());
PROPERTY(fs::file_size(backup) == fs::file_size(reference)); // the old file, unchanged
if (scenario == 1) {
bool moved_aside = false;
for (const auto& e : fs::directory_iterator(dir)) {
moved_aside |= e.path().filename().string().starts_with("database.db.importing.failed-");
}
PROPERTY(moved_aside);
}
// I2: the reader notices the import and switches to the store.
PROPERTY(reader.refresh());
compare(ref, reader, p, g_checks);
reader.close();
if (scenario == 3) {
// I3: crash between the two renames: old file already moved to its
// backup name, the new store still under .importing.
fs::rename(path, path + ".importing");
PROPERTY(!fs::exists(path));
}
Database again;
PROPERTY(again.open(path)); // reopen: no second import (or finish I3)
compare(ref, again, p, g_checks);
PROPERTY(backup_of(dir, "database.db") == backup);
again.close();
}
void corrupt_file_is_left_alone(const fs::path& root) { // I5
const std::string path = (root / "corrupt.db").string();
{
std::ofstream f(path, std::ios::binary);
f.write("LQNU\x01\0\0\0\x05\0\0\0\0\0\0\0\x03\0\0\0\0\0\0\0abc", 27); // 5 entries promised, 0.5 present
}
const auto size = fs::file_size(path);
Database db;
PROPERTY(!db.open(path));
PROPERTY(fs::is_regular_file(path) && fs::file_size(path) == size);
PROPERTY(!fs::exists(path + ".importing") || fs::is_directory(path + ".importing"));
Database ro;
PROPERTY(!ro.open_read_only(path));
}
void restored_under_other_hash_is_reported(const fs::path& root) { // I6
const std::string path = (root / "restored.db").string();
{
LegacyUnqliteDatabase db;
PROPERTY(db.open(path));
IndicatorEntry e;
e.id = "ind-x";
e.hash = "aaaa";
e.rule_name = "R";
e.timestamp = 1;
PROPERTY(db.store_indicator(e));
e.hash = "bbbb";
PROPERTY(db.store_indicator(e));
}
store::LegacyImportReport report;
std::string error;
PROPERTY(store::import_legacy_database(path, report, error));
bool warned = false;
for (const auto& w : report.warnings) warned |= w.find("re-stored") != std::string::npos;
PROPERTY(warned && report.indicators == 1);
Database db;
PROPERTY(db.open(path));
PROPERTY(db.find_indicators_by_hash("bbbb").size() == 1 && db.list_indicators().size() == 1);
}
} // namespace
int main(int argc, char** argv) {
const int iterations = argc > 1 ? std::atoi(argv[1]) : 24;
std::cout << "legacy import properties, seed " << kSeed << std::endl;
std::mt19937_64 rng(kSeed);
const fs::path root = fs::temp_directory_path() / ("sb-import-" + std::to_string(::getpid()));
fs::remove_all(root);
fs::create_directories(root);
for (int i = 0; i < iterations; ++i) run(rng, root, i);
corrupt_file_is_left_alone(root);
restored_under_other_hash_is_reported(root);
fs::remove_all(root);
std::cout << "legacy import: all properties hold (" << iterations << " imports, " << g_checks
<< " full comparisons)" << std::endl;
return 0;
}