// Legacy (unqlite-format) database import. A failure prints the seed; re-run // with STRAWBERRY_PROPERTY_SEED=. // // I1 a writer open imports the file: every query answers as the old file did, // the old file is kept as .migrated-, reopening is a no-op // I2 a read-only handle reads the legacy file directly, and refresh() // switches to the store once the daemon has imported it // I3 a crash between the final renames is finished on the next open // I4 a stale .importing directory is moved aside (never deleted) and the // import redone // I5 a locked or corrupt legacy file is left untouched and the open fails // I6 an id re-stored under another hash is reported (its old index entry // cannot be carried over); everything else still matches #include "strawberry/database.hpp" #include "../src/store/legacy_import.hpp" #include "../src/store/legacy_unqlite.hpp" #include #include #include #include #include #include #include #include #include #include using namespace strawberry; namespace fs = std::filesystem; namespace { uint64_t seed_from_env() { if (const char* text = std::getenv("STRAWBERRY_PROPERTY_SEED")) { return std::strtoull(text, nullptr, 10); } return std::random_device{}(); } const uint64_t kSeed = seed_from_env(); #define PROPERTY(cond) \ do { \ if (!(cond)) { \ std::cerr << "PROPERTY FAILED: " #cond " (seed " << kSeed << ", " << __FILE__ << ":" \ << __LINE__ << ")" << std::endl; \ std::abort(); \ } \ } while (0) } // namespace #include "store_compare.hpp" namespace { using namespace store_compare; Pools pools() { Pools p; for (int i = 0; i < 8; ++i) p.ids.push_back("ind-" + std::to_string(i)); for (int i = 0; i < 4; ++i) p.hashes.push_back(std::string(8, static_cast('a' + i))); for (int i = 0; i < 3; ++i) p.rules.push_back("Rule_" + std::to_string(i)); for (int i = 0; i < 8; ++i) p.nodes.push_back((i % 2 ? "process_" : "file_") + std::to_string(i)); p.types = {"process", "file", "indicator"}; p.rels = {"spawned", "executed", "matched"}; p.words = {"", "high", "low", "multi word"}; return p; } /// Writes a random legacy database at `path` (with the old store's code). void make_legacy(const std::string& path, std::mt19937_64& rng, const Pools& p) { LegacyUnqliteDatabase db; PROPERTY(db.open(path)); const int n = 20 + static_cast(rng() % 120); for (int i = 0; i < n; ++i) { switch (rng() % 4) { case 0: { IndicatorEntry e; const std::size_t id = rng() % p.ids.size(); e.id = p.ids[id]; // Real detection ids never change hash or rule (they embed // pid, time and match index); I6 covers the case where one did. e.hash = p.hashes[id % p.hashes.size()]; e.rule_name = p.rules[id % p.rules.size()]; e.severity = p.words[rng() % p.words.size()]; e.description = "d"; e.timestamp = static_cast(rng() % 50); e.metadata["k"] = std::to_string(rng() % 3); PROPERTY(db.store_indicator(e)); break; } case 1: { GraphNode node{p.nodes[rng() % p.nodes.size()], p.types[rng() % p.types.size()], {{"x", std::to_string(rng() % 3)}}}; PROPERTY(db.add_graph_node(node)); break; } case 2: { // Endpoints may be nodes that are never stored, as in real data. GraphEdge e{p.nodes[rng() % p.nodes.size()], p.nodes[rng() % p.nodes.size()], p.rels[rng() % p.rels.size()], static_cast(rng() % 12)}; PROPERTY(db.add_graph_edge(e)); break; } default: { FindingRecord f; f.id = "finding-" + std::to_string(rng() % 5); f.title = "t"; f.severity = "high"; f.score = static_cast(rng() % 5) * 10; f.related_node_ids = {p.nodes[rng() % p.nodes.size()]}; f.first_seen = 1; f.last_seen = static_cast(rng() % 9); PROPERTY(db.store_finding(f)); break; } } } } std::string backup_of(const fs::path& dir, const std::string& name) { for (const auto& e : fs::directory_iterator(dir)) { if (e.path().filename().string().starts_with(name + ".migrated-")) return e.path().string(); } return {}; } uint64_t g_checks = 0; void run(std::mt19937_64& rng, const fs::path& root, int iteration) { const Pools p = pools(); const fs::path dir = root / ("i" + std::to_string(iteration)); fs::create_directories(dir); const std::string path = (dir / "database.db").string(); make_legacy(path, rng, p); const std::string reference = (dir / "reference.db").string(); fs::copy_file(path, reference); LegacyUnqliteDatabase ref; PROPERTY(ref.open_read_only(reference)); // I2: read-only on the legacy file, before any import. Database reader; PROPERTY(reader.open_read_only(path)); compare(ref, reader, p, g_checks); PROPERTY(!reader.refresh()); const int scenario = iteration % 4; if (scenario == 1) { // I4: a stale partial import from an earlier crash. fs::create_directories(path + ".importing/evidence"); std::ofstream(path + ".importing/evidence/seg-1.log") << "partial"; } if (scenario == 2) { // I5: another writer holds the old lock: refuse, change nothing. const int fd = ::open((path + ".lock").c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600); PROPERTY(fd >= 0 && ::flock(fd, LOCK_EX | LOCK_NB) == 0); Database blocked; PROPERTY(!blocked.open(path)); PROPERTY(fs::is_regular_file(path) && backup_of(dir, "database.db").empty()); ::close(fd); } // I1: the writer open imports. { Database writer; PROPERTY(writer.open(path)); compare(ref, writer, p, g_checks); } PROPERTY(fs::is_directory(path)); const std::string backup = backup_of(dir, "database.db"); PROPERTY(!backup.empty()); PROPERTY(fs::file_size(backup) == fs::file_size(reference)); // the old file, unchanged if (scenario == 1) { bool moved_aside = false; for (const auto& e : fs::directory_iterator(dir)) { moved_aside |= e.path().filename().string().starts_with("database.db.importing.failed-"); } PROPERTY(moved_aside); } // I2: the reader notices the import and switches to the store. PROPERTY(reader.refresh()); compare(ref, reader, p, g_checks); reader.close(); if (scenario == 3) { // I3: crash between the two renames: old file already moved to its // backup name, the new store still under .importing. fs::rename(path, path + ".importing"); PROPERTY(!fs::exists(path)); } Database again; PROPERTY(again.open(path)); // reopen: no second import (or finish I3) compare(ref, again, p, g_checks); PROPERTY(backup_of(dir, "database.db") == backup); again.close(); } void corrupt_file_is_left_alone(const fs::path& root) { // I5 const std::string path = (root / "corrupt.db").string(); { std::ofstream f(path, std::ios::binary); f.write("LQNU\x01\0\0\0\x05\0\0\0\0\0\0\0\x03\0\0\0\0\0\0\0abc", 27); // 5 entries promised, 0.5 present } const auto size = fs::file_size(path); Database db; PROPERTY(!db.open(path)); PROPERTY(fs::is_regular_file(path) && fs::file_size(path) == size); PROPERTY(!fs::exists(path + ".importing") || fs::is_directory(path + ".importing")); Database ro; PROPERTY(!ro.open_read_only(path)); } void restored_under_other_hash_is_reported(const fs::path& root) { // I6 const std::string path = (root / "restored.db").string(); { LegacyUnqliteDatabase db; PROPERTY(db.open(path)); IndicatorEntry e; e.id = "ind-x"; e.hash = "aaaa"; e.rule_name = "R"; e.timestamp = 1; PROPERTY(db.store_indicator(e)); e.hash = "bbbb"; PROPERTY(db.store_indicator(e)); } store::LegacyImportReport report; std::string error; PROPERTY(store::import_legacy_database(path, report, error)); bool warned = false; for (const auto& w : report.warnings) warned |= w.find("re-stored") != std::string::npos; PROPERTY(warned && report.indicators == 1); Database db; PROPERTY(db.open(path)); PROPERTY(db.find_indicators_by_hash("bbbb").size() == 1 && db.list_indicators().size() == 1); } } // namespace int main(int argc, char** argv) { const int iterations = argc > 1 ? std::atoi(argv[1]) : 24; std::cout << "legacy import properties, seed " << kSeed << std::endl; std::mt19937_64 rng(kSeed); const fs::path root = fs::temp_directory_path() / ("sb-import-" + std::to_string(::getpid())); fs::remove_all(root); fs::create_directories(root); for (int i = 0; i < iterations; ++i) run(rng, root, i); corrupt_file_is_left_alone(root); restored_under_other_hash_is_reported(root); fs::remove_all(root); std::cout << "legacy import: all properties hold (" << iterations << " imports, " << g_checks << " full comparisons)" << std::endl; return 0; }