9ca69d42a1
ProgressStage declares Resolving and Verifying, and the init, model and download commands render both, but DownloadManager never emitted either: the server probe and the integrity finalize / checksum check ran with no progress event, so those CLI branches were unreachable and a long hash of a large file looked like a stall. The manager now reports Resolving before the probe and Verifying before the digest is finalized. A new test records every stage the manager emits for a checksummed download and requires Resolving < Connecting < Verifying < Finalizing; it fails before this change. Found by pipe's experimental C++ front end: both stages were matched in CLI switches but never produced anywhere in src/.
1289 lines
48 KiB
C++
1289 lines
48 KiB
C++
// Catch2 migration of test_downloader_basic.cpp with expanded test coverage
|
|
// Epic: yams-3s4 | Migration: downloader tests
|
|
|
|
#include <catch2/catch_test_macros.hpp>
|
|
#include <catch2/matchers/catch_matchers_string.hpp>
|
|
#include <catch2/matchers/catch_matchers_vector.hpp>
|
|
|
|
#include <nlohmann/json.hpp>
|
|
#include <yams/downloader/downloader.hpp>
|
|
|
|
#include <algorithm>
|
|
#include <cctype>
|
|
#include <chrono>
|
|
#include <cstddef>
|
|
#include <cstring>
|
|
#include <filesystem>
|
|
#include <fstream>
|
|
#include <iterator>
|
|
#include <memory>
|
|
#include <optional>
|
|
#include <random>
|
|
#include <string>
|
|
#include <string_view>
|
|
#include <vector>
|
|
|
|
namespace fs = std::filesystem;
|
|
using json = nlohmann::json;
|
|
using namespace yams::downloader;
|
|
|
|
namespace yams::downloader {
|
|
std::unique_ptr<IDiskWriter> makeDiskWriter();
|
|
std::unique_ptr<IIntegrityVerifier> makeIntegrityVerifierSha256Only();
|
|
std::unique_ptr<IResumeStore> makeInMemoryResumeStore();
|
|
std::unique_ptr<IRateLimiter> makeRateLimiter();
|
|
std::unique_ptr<IHttpAdapter> makeCurlHttpAdapter();
|
|
} // namespace yams::downloader
|
|
|
|
namespace {
|
|
|
|
// Catch2 assertion macros decompose comparisons for diagnostics, which triggers
|
|
// clang-tidy chained-comparison false positives. This migrated test also has a
|
|
// few harmless integer-size fixture constants.
|
|
// NOLINTBEGIN(bugprone-chained-comparison, bugprone-implicit-widening-of-multiplication-result)
|
|
|
|
// Helper to generate a unique temporary directory for tests
|
|
fs::path make_temp_dir(const std::string& prefix = "yams-dl-test-") {
|
|
auto base = fs::temp_directory_path();
|
|
std::random_device rd;
|
|
std::mt19937_64 gen(rd());
|
|
std::uniform_int_distribution<unsigned long long> dist;
|
|
fs::path dir;
|
|
for (int i = 0; i < 5; ++i) {
|
|
dir = base / (prefix + std::to_string(dist(gen)));
|
|
if (!fs::exists(dir)) {
|
|
fs::create_directories(dir);
|
|
break;
|
|
}
|
|
}
|
|
return dir;
|
|
}
|
|
|
|
// HTTP metadata parser helper
|
|
struct ParsedHttpMeta {
|
|
std::optional<std::string> etag;
|
|
std::optional<std::string> lastModified;
|
|
};
|
|
|
|
static inline std::string trim(std::string_view sv) {
|
|
size_t b = 0, e = sv.size();
|
|
while (b < e && std::isspace(static_cast<unsigned char>(sv[b])))
|
|
++b;
|
|
while (e > b && std::isspace(static_cast<unsigned char>(sv[e - 1])))
|
|
--e;
|
|
return std::string(sv.substr(b, e - b));
|
|
}
|
|
|
|
ParsedHttpMeta parse_http_headers_for_meta(const std::vector<std::string>& headers) {
|
|
ParsedHttpMeta out;
|
|
for (const auto& raw : headers) {
|
|
auto pos = raw.find(':');
|
|
if (pos == std::string::npos)
|
|
continue;
|
|
std::string key = trim(std::string_view(raw.data(), pos));
|
|
std::string val = trim(std::string_view(raw.data() + pos + 1, raw.size() - pos - 1));
|
|
std::transform(key.begin(), key.end(), key.begin(),
|
|
[](unsigned char c) { return static_cast<char>(std::tolower(c)); });
|
|
|
|
if (key == "etag" && !out.etag.has_value()) {
|
|
if (val.size() >= 2 && ((val.front() == '"' && val.back() == '"') ||
|
|
(val.front() == '\'' && val.back() == '\''))) {
|
|
out.etag = val.substr(1, val.size() - 2);
|
|
} else {
|
|
out.etag = val;
|
|
}
|
|
} else if (key == "last-modified" && !out.lastModified.has_value()) {
|
|
out.lastModified = val;
|
|
}
|
|
}
|
|
return out;
|
|
}
|
|
|
|
// Persistent JSON-based ResumeStore (test implementation)
|
|
class PersistentResumeStore final : public yams::downloader::IResumeStore {
|
|
public:
|
|
explicit PersistentResumeStore(fs::path json_path) : path_(std::move(json_path)) {
|
|
(void)loadFile();
|
|
}
|
|
|
|
yams::downloader::Expected<std::optional<State>> load(std::string_view url) override {
|
|
auto j = loadFile();
|
|
if (!j.ok())
|
|
return j.error();
|
|
auto& root = j.value();
|
|
const auto it = root.find(std::string(url));
|
|
if (it == root.end())
|
|
return std::optional<State>{std::nullopt};
|
|
|
|
State st;
|
|
const auto& entry = it.value();
|
|
if (entry.contains("etag") && entry["etag"].is_string()) {
|
|
st.etag = entry["etag"].get<std::string>();
|
|
}
|
|
if (entry.contains("last_modified") && entry["last_modified"].is_string()) {
|
|
st.lastModified = entry["last_modified"].get<std::string>();
|
|
}
|
|
if (entry.contains("total_bytes") && entry["total_bytes"].is_number_unsigned()) {
|
|
st.totalBytes = entry["total_bytes"].get<std::uint64_t>();
|
|
}
|
|
if (entry.contains("completed_ranges") && entry["completed_ranges"].is_array()) {
|
|
for (const auto& r : entry["completed_ranges"]) {
|
|
if (r.is_array() && r.size() == 2 && r[0].is_number_unsigned() &&
|
|
r[1].is_number_unsigned()) {
|
|
st.completedRanges.emplace_back(r[0].get<std::uint64_t>(),
|
|
r[1].get<std::uint64_t>());
|
|
}
|
|
}
|
|
}
|
|
return std::optional<State>{st};
|
|
}
|
|
|
|
yams::downloader::Expected<void> save(std::string_view url, const State& state) override {
|
|
auto j = loadFile();
|
|
if (!j.ok())
|
|
return j.error();
|
|
auto& root = j.value();
|
|
|
|
json entry;
|
|
if (state.etag)
|
|
entry["etag"] = *state.etag;
|
|
if (state.lastModified)
|
|
entry["last_modified"] = *state.lastModified;
|
|
entry["total_bytes"] = state.totalBytes;
|
|
entry["completed_ranges"] = json::array();
|
|
for (const auto& [off, len] : state.completedRanges) {
|
|
entry["completed_ranges"].push_back(json::array({off, len}));
|
|
}
|
|
|
|
root[std::string(url)] = entry;
|
|
return writeFile(root);
|
|
}
|
|
|
|
void remove(std::string_view url) noexcept override {
|
|
auto j = loadFile();
|
|
if (!j.ok())
|
|
return;
|
|
auto& root = j.value();
|
|
root.erase(std::string(url));
|
|
(void)writeFile(root);
|
|
}
|
|
|
|
private:
|
|
yams::downloader::Expected<json> loadFile() const {
|
|
if (!fs::exists(path_)) {
|
|
return json::object();
|
|
}
|
|
std::ifstream in(path_);
|
|
if (!in) {
|
|
return yams::downloader::Error{yams::downloader::ErrorCode::IoError,
|
|
"Failed to open resume JSON for read"};
|
|
}
|
|
try {
|
|
json root;
|
|
in >> root;
|
|
if (!root.is_object())
|
|
root = json::object();
|
|
return root;
|
|
} catch (const std::exception&) {
|
|
return json::object();
|
|
}
|
|
}
|
|
|
|
yams::downloader::Expected<void> writeFile(const json& root) const {
|
|
std::ofstream out(path_, std::ios::trunc);
|
|
if (!out) {
|
|
return yams::downloader::Error{yams::downloader::ErrorCode::IoError,
|
|
"Failed to open resume JSON for write"};
|
|
}
|
|
out << root.dump(2);
|
|
return {};
|
|
}
|
|
|
|
fs::path path_;
|
|
};
|
|
|
|
// Tracking resume store for test verification
|
|
class TrackingResumeStore final : public IResumeStore {
|
|
public:
|
|
std::optional<State> initialState;
|
|
std::vector<State> savedStates;
|
|
std::vector<std::string> savedUrls;
|
|
std::vector<std::string> loadUrls;
|
|
std::optional<std::string> removedUrl;
|
|
|
|
Expected<std::optional<State>> load(std::string_view url) override {
|
|
loadUrls.emplace_back(url);
|
|
return initialState;
|
|
}
|
|
|
|
Expected<void> save(std::string_view url, const State& state) override {
|
|
savedUrls.emplace_back(url);
|
|
savedStates.push_back(state);
|
|
initialState = state;
|
|
return {};
|
|
}
|
|
|
|
void remove(std::string_view url) noexcept override {
|
|
removedUrl = std::string(url);
|
|
initialState.reset();
|
|
}
|
|
};
|
|
|
|
// Test HTTP adapter
|
|
class TestHttpAdapter final : public IHttpAdapter {
|
|
public:
|
|
bool resumeSupported{true};
|
|
std::optional<std::uint64_t> contentLength{};
|
|
std::optional<std::string> etag{};
|
|
std::optional<std::string> lastModified{};
|
|
std::vector<std::uint64_t> requestedOffsets;
|
|
std::vector<std::uint64_t> requestedSizes;
|
|
std::vector<std::byte> payload;
|
|
bool checkCancelBeforeSink{false};
|
|
bool cancelObserved{false};
|
|
bool lastProbeFollowRedirects{true};
|
|
int probeCallCount{0};
|
|
int fetchCallCount{0};
|
|
|
|
Expected<void> probe(std::string_view, const std::vector<Header>&, bool& resumeSupportedOut,
|
|
std::optional<std::uint64_t>& contentLengthOut,
|
|
std::optional<std::string>& etagOut,
|
|
std::optional<std::string>& lastModifiedOut,
|
|
std::optional<std::string>& contentTypeOut,
|
|
std::optional<std::string>& suggestedFilenameOut, const TlsConfig&,
|
|
const std::optional<std::string>&, bool followRedirects,
|
|
std::chrono::milliseconds) override {
|
|
probeCallCount++;
|
|
lastProbeFollowRedirects = followRedirects;
|
|
resumeSupportedOut = resumeSupported;
|
|
contentLengthOut = contentLength;
|
|
etagOut = etag;
|
|
lastModifiedOut = lastModified;
|
|
contentTypeOut.reset();
|
|
suggestedFilenameOut.reset();
|
|
return {};
|
|
}
|
|
|
|
Expected<void> fetchRange(std::string_view, const std::vector<Header>&, std::uint64_t offset,
|
|
std::uint64_t size, const TlsConfig&,
|
|
const std::optional<std::string>&, std::chrono::milliseconds,
|
|
const std::function<Expected<void>(std::span<const std::byte>)>& sink,
|
|
const ShouldCancel& shouldCancel,
|
|
const ProgressCallback& onProgress) override {
|
|
fetchCallCount++;
|
|
requestedOffsets.push_back(offset);
|
|
requestedSizes.push_back(size);
|
|
if (onProgress) {
|
|
ProgressEvent ev;
|
|
ev.downloadedBytes = 0;
|
|
ev.stage = ProgressStage::Downloading;
|
|
onProgress(ev);
|
|
}
|
|
if (checkCancelBeforeSink && shouldCancel && shouldCancel()) {
|
|
cancelObserved = true;
|
|
return Error{ErrorCode::OperationCancelled, "Operation cancelled"};
|
|
}
|
|
if (!payload.empty()) {
|
|
auto res = sink(std::span<const std::byte>(payload.data(), payload.size()));
|
|
if (!res.ok())
|
|
return res;
|
|
}
|
|
return {};
|
|
}
|
|
};
|
|
|
|
// Fake disk writer for testing
|
|
class FakeDiskWriter final : public IDiskWriter {
|
|
public:
|
|
explicit FakeDiskWriter(StorageConfig storage) : storage_(std::move(storage)) {}
|
|
|
|
void setInitialData(const std::vector<std::byte>& data) { initialData_ = data; }
|
|
|
|
const std::vector<std::byte>& stagedData() const { return data_; }
|
|
|
|
const fs::path& lastFinalPath() const { return lastFinalPath_; }
|
|
|
|
const std::string& lastSessionId() const { return lastSessionId_; }
|
|
|
|
Expected<fs::path> createStagingFile(const StorageConfig&, std::string_view sessionId,
|
|
std::string_view tempExtension) override {
|
|
lastSessionId_ = std::string(sessionId);
|
|
currentPath_ = stagingPath(sessionId, tempExtension);
|
|
data_.clear();
|
|
writeToFile();
|
|
return currentPath_;
|
|
}
|
|
|
|
Expected<fs::path> createOrOpenStagingFile(const StorageConfig&, std::string_view sessionId,
|
|
std::string_view tempExtension,
|
|
std::optional<std::uint64_t> expectedSize,
|
|
std::uint64_t& currentSize) override {
|
|
lastSessionId_ = std::string(sessionId);
|
|
currentPath_ = stagingPath(sessionId, tempExtension);
|
|
data_ = initialData_;
|
|
currentSize = data_.size();
|
|
writeToFile();
|
|
if (expectedSize && currentSize == 0) {
|
|
std::error_code ec;
|
|
std::filesystem::resize_file(currentPath_, static_cast<std::uint64_t>(*expectedSize),
|
|
ec);
|
|
if (!ec) {
|
|
data_.resize(static_cast<std::size_t>(*expectedSize));
|
|
writeToFile();
|
|
}
|
|
}
|
|
return currentPath_;
|
|
}
|
|
|
|
Expected<void> writeAt(const fs::path&, std::uint64_t offset,
|
|
std::span<const std::byte> data) override {
|
|
const auto end = offset + static_cast<std::uint64_t>(data.size());
|
|
if (end > data_.size()) {
|
|
data_.resize(static_cast<std::size_t>(end));
|
|
}
|
|
std::memcpy(data_.data() + offset, data.data(), data.size());
|
|
writeToFile();
|
|
return {};
|
|
}
|
|
|
|
Expected<void> sync(const fs::path&, const StorageConfig&) override { return {}; }
|
|
|
|
Expected<fs::path> finalizeToCas(const fs::path&, std::string_view hashHex,
|
|
const StorageConfig&) override {
|
|
lastFinalPath_ = casPathForSha256(storage_.objectsDir, hashHex);
|
|
std::error_code ec;
|
|
std::filesystem::create_directories(lastFinalPath_.parent_path(), ec);
|
|
std::ofstream out(lastFinalPath_, std::ios::binary | std::ios::trunc);
|
|
if (!out) {
|
|
return Error{ErrorCode::IoError,
|
|
"Failed to write CAS object: " + lastFinalPath_.string()};
|
|
}
|
|
if (!data_.empty()) {
|
|
out.write(reinterpret_cast<const char*>(data_.data()),
|
|
static_cast<std::streamsize>(data_.size()));
|
|
}
|
|
return lastFinalPath_;
|
|
}
|
|
|
|
void cleanup(const fs::path& stagingFile) noexcept override {
|
|
std::error_code ec;
|
|
std::filesystem::remove(stagingFile, ec);
|
|
}
|
|
|
|
private:
|
|
fs::path stagingPath(std::string_view sessionId, std::string_view ext) const {
|
|
fs::path dir = storage_.stagingDir / "downloader";
|
|
std::filesystem::create_directories(dir);
|
|
std::string fn(sessionId);
|
|
if (!ext.empty()) {
|
|
if (ext.front() == '.')
|
|
fn.append(ext);
|
|
else {
|
|
fn.push_back('.');
|
|
fn.append(ext);
|
|
}
|
|
} else {
|
|
fn.append(".part");
|
|
}
|
|
return dir / fn;
|
|
}
|
|
|
|
void writeToFile() {
|
|
if (currentPath_.empty())
|
|
return;
|
|
std::error_code ec;
|
|
std::filesystem::create_directories(currentPath_.parent_path(), ec);
|
|
std::ofstream out(currentPath_, std::ios::binary | std::ios::trunc);
|
|
if (!out)
|
|
return;
|
|
if (!data_.empty()) {
|
|
out.write(reinterpret_cast<const char*>(data_.data()),
|
|
static_cast<std::streamsize>(data_.size()));
|
|
}
|
|
}
|
|
|
|
StorageConfig storage_;
|
|
std::vector<std::byte> initialData_{};
|
|
std::vector<std::byte> data_{};
|
|
fs::path currentPath_{};
|
|
fs::path lastFinalPath_{};
|
|
std::string lastSessionId_{};
|
|
};
|
|
|
|
std::vector<std::byte> to_bytes(std::string_view s) {
|
|
std::vector<std::byte> out(s.size());
|
|
std::memcpy(out.data(), s.data(), s.size());
|
|
return out;
|
|
}
|
|
|
|
constexpr const char kExportMetaSuffix[] = ".yams.meta.json";
|
|
|
|
fs::path meta_path_for(const fs::path& exportPath) {
|
|
return fs::path(exportPath.string() + kExportMetaSuffix);
|
|
}
|
|
|
|
} // namespace
|
|
|
|
// =============================================================================
|
|
// HTTP Header Parsing Tests
|
|
// =============================================================================
|
|
|
|
TEST_CASE("HTTP Header Parsing: Basic extraction", "[downloader][http][headers]") {
|
|
SECTION("Extracts ETag and Last-Modified") {
|
|
std::vector<std::string> headers = {"Date: Tue, 19 Aug 2025 10:11:12 GMT",
|
|
"Server: test-http",
|
|
"ETag: \"abc123-xyz\"",
|
|
"Last-Modified: Tue, 19 Aug 2025 09:00:00 GMT",
|
|
"Accept-Ranges: bytes",
|
|
"Content-Length: 1048576"};
|
|
|
|
auto parsed = parse_http_headers_for_meta(headers);
|
|
REQUIRE(parsed.etag.has_value());
|
|
REQUIRE(parsed.lastModified.has_value());
|
|
CHECK((*parsed.etag == "abc123-xyz"));
|
|
CHECK((*parsed.lastModified == "Tue, 19 Aug 2025 09:00:00 GMT"));
|
|
}
|
|
|
|
SECTION("Handles missing values") {
|
|
std::vector<std::string> headers = {"date: Tue, 19 Aug 2025 10:11:12 GMT",
|
|
"server: test-http", "content-length: 512"};
|
|
|
|
auto parsed = parse_http_headers_for_meta(headers);
|
|
CHECK_FALSE(parsed.etag.has_value());
|
|
CHECK_FALSE(parsed.lastModified.has_value());
|
|
}
|
|
|
|
SECTION("Case insensitive header names") {
|
|
std::vector<std::string> headers = {"eTaG: 'W/\"weak-etag-value\"'",
|
|
"LAST-MODIFIED: Wed, 20 Aug 2025 00:00:00 GMT "};
|
|
|
|
auto parsed = parse_http_headers_for_meta(headers);
|
|
REQUIRE(parsed.etag.has_value());
|
|
REQUIRE(parsed.lastModified.has_value());
|
|
CHECK((*parsed.etag == "W/\"weak-etag-value\""));
|
|
CHECK((*parsed.lastModified == "Wed, 20 Aug 2025 00:00:00 GMT"));
|
|
}
|
|
|
|
SECTION("First occurrence wins") {
|
|
std::vector<std::string> headers = {"ETag: first-etag", "ETag: second-etag"};
|
|
|
|
auto parsed = parse_http_headers_for_meta(headers);
|
|
REQUIRE(parsed.etag.has_value());
|
|
CHECK((*parsed.etag == "first-etag"));
|
|
}
|
|
|
|
SECTION("Strips quotes from ETag") {
|
|
std::vector<std::string> headers = {"ETag: \"quoted-etag\""};
|
|
auto parsed = parse_http_headers_for_meta(headers);
|
|
REQUIRE(parsed.etag.has_value());
|
|
CHECK((*parsed.etag == "quoted-etag"));
|
|
}
|
|
|
|
SECTION("Handles unquoted ETag") {
|
|
std::vector<std::string> headers = {"ETag: unquoted-etag"};
|
|
auto parsed = parse_http_headers_for_meta(headers);
|
|
REQUIRE(parsed.etag.has_value());
|
|
CHECK((*parsed.etag == "unquoted-etag"));
|
|
}
|
|
}
|
|
|
|
// =============================================================================
|
|
// Persistent Resume Store Tests
|
|
// =============================================================================
|
|
|
|
TEST_CASE("PersistentResumeStore: Round-trip", "[downloader][resume]") {
|
|
auto dir = make_temp_dir();
|
|
auto jsonPath = dir / "resume.json";
|
|
|
|
SECTION("Save and load single URL") {
|
|
PersistentResumeStore store(jsonPath);
|
|
|
|
yams::downloader::IResumeStore::State state;
|
|
state.etag = std::string("abc123etag");
|
|
state.lastModified = std::string("Tue, 19 Aug 2025 09:00:00 GMT");
|
|
state.totalBytes = 1024ull * 1024ull * 5ull;
|
|
state.completedRanges = {{0, 256ull * 1024ull}, {256ull * 1024ull, 512ull * 1024ull}};
|
|
|
|
auto saveRes = store.save("https://example.com/file.bin", state);
|
|
REQUIRE(saveRes.ok());
|
|
|
|
PersistentResumeStore store2(jsonPath);
|
|
auto loadRes = store2.load("https://example.com/file.bin");
|
|
REQUIRE(loadRes.ok());
|
|
REQUIRE(loadRes.value().has_value());
|
|
|
|
const auto& got = *loadRes.value();
|
|
REQUIRE(got.etag.has_value());
|
|
REQUIRE(got.lastModified.has_value());
|
|
CHECK((*got.etag == *state.etag));
|
|
CHECK((*got.lastModified == *state.lastModified));
|
|
CHECK((got.totalBytes == state.totalBytes));
|
|
REQUIRE((got.completedRanges.size() == state.completedRanges.size()));
|
|
}
|
|
|
|
SECTION("Remove entry") {
|
|
PersistentResumeStore store(jsonPath);
|
|
|
|
yams::downloader::IResumeStore::State state;
|
|
state.etag = std::string("to-be-removed");
|
|
state.totalBytes = 42;
|
|
|
|
REQUIRE(store.save("http://example.com/a", state).ok());
|
|
REQUIRE(store.save("http://example.com/b", state).ok());
|
|
|
|
store.remove("http://example.com/a");
|
|
|
|
PersistentResumeStore store2(jsonPath);
|
|
auto la = store2.load("http://example.com/a");
|
|
auto lb = store2.load("http://example.com/b");
|
|
|
|
REQUIRE(la.ok());
|
|
REQUIRE(lb.ok());
|
|
CHECK_FALSE(la.value().has_value());
|
|
REQUIRE(lb.value().has_value());
|
|
CHECK(lb.value()->etag.has_value());
|
|
CHECK((*lb.value()->etag == "to-be-removed"));
|
|
}
|
|
|
|
SECTION("Multiple URLs") {
|
|
PersistentResumeStore store(jsonPath);
|
|
|
|
for (int i = 0; i < 5; ++i) {
|
|
yams::downloader::IResumeStore::State state;
|
|
state.etag = "etag-" + std::to_string(i);
|
|
state.totalBytes = 1000L * (i + 1);
|
|
REQUIRE(store.save("http://example.com/file" + std::to_string(i), state).ok());
|
|
}
|
|
|
|
PersistentResumeStore store2(jsonPath);
|
|
for (int i = 0; i < 5; ++i) {
|
|
auto result = store2.load("http://example.com/file" + std::to_string(i));
|
|
REQUIRE(result.ok());
|
|
REQUIRE(result.value().has_value());
|
|
CHECK((*result.value()->etag == "etag-" + std::to_string(i)));
|
|
}
|
|
}
|
|
|
|
SECTION("Load non-existent URL") {
|
|
PersistentResumeStore store(jsonPath);
|
|
auto result = store.load("http://nonexistent.com/file");
|
|
REQUIRE(result.ok());
|
|
CHECK_FALSE(result.value().has_value());
|
|
}
|
|
|
|
// Cleanup
|
|
std::error_code ec;
|
|
fs::remove_all(dir, ec);
|
|
}
|
|
|
|
// =============================================================================
|
|
// CAS Helpers Tests
|
|
// =============================================================================
|
|
|
|
TEST_CASE("CAS Helpers: Path and ID formatting", "[downloader][cas]") {
|
|
fs::path objectsDir = "/tmp/yams-objects";
|
|
const std::string digest = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
|
|
|
|
SECTION("CAS path includes shard directories") {
|
|
auto path = yams::downloader::casPathForSha256(objectsDir, digest);
|
|
CHECK((path.generic_string().find("sha256/01/23/0123456789abcdef") != std::string::npos));
|
|
}
|
|
|
|
SECTION("SHA256 ID format") {
|
|
auto id = yams::downloader::makeSha256Id(digest);
|
|
CHECK((id == "sha256:" + digest));
|
|
}
|
|
|
|
SECTION("Empty digest") {
|
|
auto id = yams::downloader::makeSha256Id("");
|
|
CHECK((id == "sha256:"));
|
|
}
|
|
}
|
|
|
|
// =============================================================================
|
|
// Download Manager Resume Tests
|
|
// =============================================================================
|
|
|
|
TEST_CASE("DownloadManager: Resume functionality", "[downloader][resume]") {
|
|
auto tempDir = make_temp_dir();
|
|
StorageConfig storage{tempDir / "objects", tempDir / "staging"};
|
|
DownloaderConfig cfg;
|
|
cfg.defaultChunkSizeBytes = 4;
|
|
|
|
SECTION("Persists and cleans ranges") {
|
|
auto http = std::make_unique<TestHttpAdapter>();
|
|
auto* httpPtr = http.get();
|
|
httpPtr->resumeSupported = true;
|
|
httpPtr->contentLength = 10;
|
|
httpPtr->etag = "etag-123";
|
|
httpPtr->payload = to_bytes("WORLD");
|
|
|
|
auto disk = std::make_unique<FakeDiskWriter>(storage);
|
|
auto* diskPtr = disk.get();
|
|
diskPtr->setInitialData(to_bytes("HELLO"));
|
|
|
|
auto resume = std::make_unique<TrackingResumeStore>();
|
|
auto* resumePtr = resume.get();
|
|
IResumeStore::State initial;
|
|
initial.totalBytes = 10;
|
|
initial.completedRanges.emplace_back(0, 5);
|
|
initial.etag = "etag-123";
|
|
resumePtr->initialState = initial;
|
|
|
|
auto dm = makeDownloadManagerWithDependencies(
|
|
storage, cfg, std::move(http), std::move(disk), nullptr, std::move(resume), nullptr);
|
|
|
|
DownloadRequest req;
|
|
req.url = "https://example.com/file.bin";
|
|
req.resume = true;
|
|
req.chunkSizeBytes = 5;
|
|
req.checksum = Checksum{HashAlgo::Sha256,
|
|
"0b21b7db59cd154904fac6336fa7d2be1bab38d632794f281549584068cdcb74"};
|
|
|
|
auto result = dm->download(req);
|
|
REQUIRE(result.ok());
|
|
auto final = result.value();
|
|
|
|
REQUIRE((httpPtr->requestedOffsets.size() == 1));
|
|
CHECK((httpPtr->requestedOffsets.front() == 5));
|
|
|
|
REQUIRE_FALSE(resumePtr->savedStates.empty());
|
|
const auto& lastState = resumePtr->savedStates.back();
|
|
REQUIRE((lastState.completedRanges.size() == 1));
|
|
CHECK((lastState.completedRanges.front().first == 0));
|
|
CHECK((lastState.completedRanges.front().second == 10));
|
|
|
|
REQUIRE(resumePtr->removedUrl.has_value());
|
|
CHECK((*resumePtr->removedUrl == req.url));
|
|
|
|
REQUIRE_FALSE(diskPtr->lastFinalPath().empty());
|
|
std::ifstream finalFile(diskPtr->lastFinalPath(), std::ios::binary);
|
|
REQUIRE(finalFile.good());
|
|
std::string contents((std::istreambuf_iterator<char>(finalFile)),
|
|
std::istreambuf_iterator<char>());
|
|
CHECK((contents == "HELLOWORLD"));
|
|
|
|
CHECK(final.hash ==
|
|
"sha256:0b21b7db59cd154904fac6336fa7d2be1bab38d632794f281549584068cdcb74");
|
|
CHECK((final.sizeBytes == 10));
|
|
}
|
|
|
|
SECTION("Uses a stable resume session id for the same URL") {
|
|
auto makeResumingManager = [&](FakeDiskWriter*& diskPtr) {
|
|
auto http = std::make_unique<TestHttpAdapter>();
|
|
auto* httpPtr = http.get();
|
|
httpPtr->resumeSupported = true;
|
|
httpPtr->contentLength = 10;
|
|
httpPtr->etag = "etag-123";
|
|
httpPtr->payload = to_bytes("WORLD");
|
|
|
|
auto disk = std::make_unique<FakeDiskWriter>(storage);
|
|
diskPtr = disk.get();
|
|
diskPtr->setInitialData(to_bytes("HELLO"));
|
|
|
|
auto resume = std::make_unique<TrackingResumeStore>();
|
|
IResumeStore::State initial;
|
|
initial.totalBytes = 10;
|
|
initial.completedRanges.emplace_back(0, 5);
|
|
initial.etag = "etag-123";
|
|
resume->initialState = initial;
|
|
|
|
return makeDownloadManagerWithDependencies(storage, cfg, std::move(http),
|
|
std::move(disk), nullptr, std::move(resume),
|
|
nullptr);
|
|
};
|
|
|
|
FakeDiskWriter* firstDisk = nullptr;
|
|
auto firstManager = makeResumingManager(firstDisk);
|
|
|
|
DownloadRequest req;
|
|
req.url = "https://example.com/file.bin";
|
|
req.resume = true;
|
|
|
|
auto firstResult = firstManager->download(req);
|
|
REQUIRE(firstResult.ok());
|
|
REQUIRE_FALSE(firstDisk->lastSessionId().empty());
|
|
|
|
FakeDiskWriter* secondDisk = nullptr;
|
|
auto secondManager = makeResumingManager(secondDisk);
|
|
|
|
auto secondResult = secondManager->download(req);
|
|
REQUIRE(secondResult.ok());
|
|
REQUIRE_FALSE(secondDisk->lastSessionId().empty());
|
|
CHECK((secondDisk->lastSessionId() == firstDisk->lastSessionId()));
|
|
}
|
|
|
|
// Cleanup
|
|
std::error_code ec;
|
|
fs::remove_all(tempDir, ec);
|
|
}
|
|
|
|
TEST_CASE("DownloadManager: probe forwards follow_redirects policy", "[downloader][redirect]") {
|
|
auto tempDir = make_temp_dir();
|
|
StorageConfig storage{tempDir / "objects", tempDir / "staging"};
|
|
DownloaderConfig cfg;
|
|
|
|
auto http = std::make_unique<TestHttpAdapter>();
|
|
auto* httpPtr = http.get();
|
|
httpPtr->resumeSupported = false;
|
|
httpPtr->contentLength = 5;
|
|
httpPtr->payload = to_bytes("ABCDE");
|
|
|
|
auto disk = std::make_unique<FakeDiskWriter>(storage);
|
|
auto dm = makeDownloadManagerWithDependencies(storage, cfg, std::move(http), std::move(disk),
|
|
nullptr, nullptr, nullptr);
|
|
|
|
DownloadRequest req;
|
|
req.url = "https://example.com/no-redirects.bin";
|
|
req.followRedirects = false;
|
|
|
|
auto result = dm->download(req);
|
|
REQUIRE(result.ok());
|
|
CHECK_FALSE(httpPtr->lastProbeFollowRedirects);
|
|
|
|
std::error_code ec;
|
|
fs::remove_all(tempDir, ec);
|
|
}
|
|
|
|
TEST_CASE("DownloadManager: verifies configured checksum algorithms",
|
|
"[downloader][checksum][algorithms]") {
|
|
const auto sha256 =
|
|
std::string{"f0393febe8baaa55e32f7be2a7cc180bf34e52137d99e056c817a9c07b8f239a"};
|
|
const auto sha512 =
|
|
std::string{"9989a8fcbc29044b5883a0a36c146fe7415b1439e995b4d806ea0af7da9ca4390eb92a604b3ecf"
|
|
"a3d75f9911c768fbe2aecc59eff1e48dcaeca1957bdde01dfb"};
|
|
const auto md5 = std::string{"2ecdde3959051d913f61b14579ea136d"};
|
|
|
|
auto runCase = [&](HashAlgo algo, const std::string& expectedHex) {
|
|
auto tempDir = make_temp_dir();
|
|
StorageConfig storage{tempDir / "objects", tempDir / "staging"};
|
|
DownloaderConfig cfg;
|
|
|
|
auto http = std::make_unique<TestHttpAdapter>();
|
|
http->resumeSupported = false;
|
|
http->contentLength = 5;
|
|
http->payload = to_bytes("ABCDE");
|
|
|
|
auto disk = std::make_unique<FakeDiskWriter>(storage);
|
|
auto dm = makeDownloadManagerWithDependencies(storage, cfg, std::move(http),
|
|
std::move(disk), nullptr, nullptr, nullptr);
|
|
|
|
DownloadRequest req;
|
|
req.url = "https://example.com/checksum.bin";
|
|
req.checksum = Checksum{algo, expectedHex};
|
|
|
|
auto result = dm->download(req);
|
|
REQUIRE(result.ok());
|
|
CHECK(result.value().success);
|
|
|
|
std::error_code ec;
|
|
fs::remove_all(tempDir, ec);
|
|
};
|
|
|
|
SECTION("sha256") {
|
|
runCase(HashAlgo::Sha256, sha256);
|
|
}
|
|
SECTION("sha512") {
|
|
runCase(HashAlgo::Sha512, sha512);
|
|
}
|
|
SECTION("md5") {
|
|
runCase(HashAlgo::Md5, md5);
|
|
}
|
|
}
|
|
|
|
TEST_CASE("DownloadManager: Export with ETag matching", "[downloader][export]") {
|
|
auto tempDir = make_temp_dir();
|
|
StorageConfig storage{tempDir / "objects", tempDir / "staging"};
|
|
DownloaderConfig cfg;
|
|
|
|
SECTION("Skips copy when ETag matches") {
|
|
auto http = std::make_unique<TestHttpAdapter>();
|
|
auto* httpPtr = http.get();
|
|
httpPtr->resumeSupported = false;
|
|
httpPtr->contentLength = 5;
|
|
httpPtr->etag = "tag-777";
|
|
httpPtr->payload = to_bytes("ABCDE");
|
|
|
|
auto disk = std::make_unique<FakeDiskWriter>(storage);
|
|
auto* diskPtr = disk.get();
|
|
|
|
auto resume = std::make_unique<TrackingResumeStore>();
|
|
auto* resumePtr = resume.get();
|
|
|
|
auto dm = makeDownloadManagerWithDependencies(
|
|
storage, cfg, std::move(http), std::move(disk), nullptr, std::move(resume), nullptr);
|
|
|
|
fs::path exportDir = tempDir / "exports";
|
|
fs::create_directories(exportDir);
|
|
fs::path exportPath = exportDir / "file.bin";
|
|
{
|
|
std::ofstream out(exportPath, std::ios::binary | std::ios::trunc);
|
|
out << "OLD";
|
|
}
|
|
auto metaPath = meta_path_for(exportPath);
|
|
{
|
|
json meta;
|
|
meta["etag"] = "tag-777";
|
|
meta["hash"] = "placeholder";
|
|
std::ofstream out(metaPath, std::ios::binary | std::ios::trunc);
|
|
out << meta.dump(2);
|
|
}
|
|
|
|
DownloadRequest req;
|
|
req.url = "https://example.com/export.bin";
|
|
req.exportPath = exportPath;
|
|
req.overwrite = OverwritePolicy::IfDifferentEtag;
|
|
|
|
auto result = dm->download(req);
|
|
REQUIRE(result.ok());
|
|
auto final = result.value();
|
|
|
|
std::string exported;
|
|
{
|
|
std::ifstream in(exportPath, std::ios::binary);
|
|
exported.assign((std::istreambuf_iterator<char>(in)), std::istreambuf_iterator<char>());
|
|
}
|
|
CHECK((exported == "OLD"));
|
|
|
|
json metaUpdated;
|
|
{
|
|
std::ifstream in(metaPath);
|
|
REQUIRE(in.good());
|
|
in >> metaUpdated;
|
|
}
|
|
REQUIRE(metaUpdated.contains("hash"));
|
|
REQUIRE(metaUpdated.contains("etag"));
|
|
CHECK((metaUpdated["etag"].get<std::string>() == "tag-777"));
|
|
|
|
REQUIRE_FALSE(diskPtr->lastFinalPath().empty());
|
|
std::ifstream casFile(diskPtr->lastFinalPath(), std::ios::binary);
|
|
std::string casData((std::istreambuf_iterator<char>(casFile)),
|
|
std::istreambuf_iterator<char>());
|
|
CHECK((casData == "ABCDE"));
|
|
|
|
CHECK(resumePtr->savedStates.empty());
|
|
CHECK((httpPtr->requestedOffsets.size() == 1));
|
|
}
|
|
|
|
// Cleanup
|
|
std::error_code ec;
|
|
fs::remove_all(tempDir, ec);
|
|
}
|
|
|
|
TEST_CASE("DownloadManager: Cooperative cancellation stops fetch", "[downloader][cancel][catch2]") {
|
|
auto tempDir = make_temp_dir();
|
|
StorageConfig storage{tempDir / "objects", tempDir / "staging"};
|
|
DownloaderConfig cfg;
|
|
|
|
auto http = std::make_unique<TestHttpAdapter>();
|
|
auto* httpPtr = http.get();
|
|
httpPtr->resumeSupported = false;
|
|
httpPtr->contentLength = 5;
|
|
httpPtr->payload = to_bytes("ABCDE");
|
|
httpPtr->checkCancelBeforeSink = true;
|
|
|
|
auto disk = std::make_unique<FakeDiskWriter>(storage);
|
|
|
|
auto dm = makeDownloadManagerWithDependencies(storage, cfg, std::move(http), std::move(disk),
|
|
nullptr, nullptr, nullptr);
|
|
|
|
DownloadRequest req;
|
|
req.url = "https://example.com/cancel.bin";
|
|
|
|
bool canceled = true;
|
|
auto result = dm->download(req, {}, [&]() { return canceled; });
|
|
REQUIRE_FALSE(result.ok());
|
|
CHECK((result.error().code == ErrorCode::OperationCancelled));
|
|
CHECK(httpPtr->cancelObserved);
|
|
|
|
std::error_code ec;
|
|
fs::remove_all(tempDir, ec);
|
|
}
|
|
|
|
TEST_CASE("DownloadManager: reports every progress stage in order",
|
|
"[downloader][progress][catch2]") {
|
|
// ProgressStage::Resolving and ::Verifying were declared, and the init, model and download
|
|
// commands render them, but the manager never emitted either: the probe and the checksum
|
|
// check ran silently. Only the manager's own events are checked; the fake adapter emits its
|
|
// own Downloading event from inside fetchRange.
|
|
auto tempDir = make_temp_dir("yams-dl-stages-");
|
|
StorageConfig storage{tempDir / "objects", tempDir / "staging"};
|
|
DownloaderConfig cfg;
|
|
|
|
auto http = std::make_unique<TestHttpAdapter>();
|
|
http->resumeSupported = false;
|
|
http->contentLength = 5;
|
|
http->payload = to_bytes("ABCDE");
|
|
auto disk = std::make_unique<FakeDiskWriter>(storage);
|
|
auto dm = makeDownloadManagerWithDependencies(storage, cfg, std::move(http), std::move(disk),
|
|
nullptr, nullptr, nullptr);
|
|
|
|
DownloadRequest req;
|
|
req.url = "https://example.com/stages.bin";
|
|
req.checksum = Checksum{HashAlgo::Sha256,
|
|
"f0393febe8baaa55e32f7be2a7cc180bf34e52137d99e056c817a9c07b8f239a"};
|
|
|
|
std::vector<ProgressStage> stages;
|
|
auto result = dm->download(req, [&](const ProgressEvent& ev) { stages.push_back(ev.stage); });
|
|
REQUIRE(result.ok());
|
|
|
|
auto indexOf = [&](ProgressStage stage) {
|
|
return std::find(stages.begin(), stages.end(), stage) - stages.begin();
|
|
};
|
|
const auto end = static_cast<std::ptrdiff_t>(stages.size());
|
|
REQUIRE(indexOf(ProgressStage::Resolving) < end);
|
|
REQUIRE(indexOf(ProgressStage::Connecting) < end);
|
|
REQUIRE(indexOf(ProgressStage::Verifying) < end);
|
|
REQUIRE(indexOf(ProgressStage::Finalizing) < end);
|
|
CHECK(indexOf(ProgressStage::Resolving) < indexOf(ProgressStage::Connecting));
|
|
CHECK(indexOf(ProgressStage::Connecting) < indexOf(ProgressStage::Verifying));
|
|
CHECK(indexOf(ProgressStage::Verifying) < indexOf(ProgressStage::Finalizing));
|
|
|
|
std::error_code ec;
|
|
fs::remove_all(tempDir, ec);
|
|
}
|
|
|
|
TEST_CASE("CurlHttpAdapter: a user cancel is reported as OperationCancelled",
|
|
"[downloader][cancel][curl][catch2]") {
|
|
// The fake adapter above returns OperationCancelled on its own, so the manager-level
|
|
// cancellation test never exercised the real adapter, which reported a user cancel as
|
|
// PolicyViolation. DownloadService only recognizes OperationCancelled, so a cancel reached
|
|
// callers as a network error. A file:// transfer drives the real curl write callback
|
|
// without a network.
|
|
auto tempDir = make_temp_dir("yams-dl-curl-cancel-");
|
|
const auto payloadPath = tempDir / "payload.bin";
|
|
{
|
|
std::ofstream out(payloadPath, std::ios::binary);
|
|
const std::string chunk(64 * 1024, 'x');
|
|
out.write(chunk.data(), static_cast<std::streamsize>(chunk.size()));
|
|
}
|
|
|
|
auto http = makeCurlHttpAdapter();
|
|
REQUIRE(http);
|
|
const std::string url = "file://" + payloadPath.string();
|
|
auto result = http->fetchRange(
|
|
url, {}, 0, 0, TlsConfig{}, std::nullopt, std::chrono::milliseconds{5000},
|
|
[](std::span<const std::byte>) -> Expected<void> { return {}; }, [] { return true; }, {});
|
|
|
|
REQUIRE_FALSE(result.ok());
|
|
CHECK((result.error().code == ErrorCode::OperationCancelled));
|
|
|
|
std::error_code ec;
|
|
fs::remove_all(tempDir, ec);
|
|
}
|
|
|
|
// =============================================================================
|
|
// Test HTTP Adapter Tests
|
|
// =============================================================================
|
|
|
|
TEST_CASE("TestHttpAdapter: Behavior verification", "[downloader][adapter]") {
|
|
SECTION("Tracks probe calls") {
|
|
TestHttpAdapter adapter;
|
|
adapter.contentLength = 1000;
|
|
|
|
bool resumeSupported = false;
|
|
std::optional<std::uint64_t> contentLength;
|
|
std::optional<std::string> etag, lastModified, contentType, filename;
|
|
|
|
adapter.probe("http://test.com", {}, resumeSupported, contentLength, etag, lastModified,
|
|
contentType, filename, TlsConfig{}, std::nullopt, true,
|
|
std::chrono::milliseconds{1000});
|
|
|
|
CHECK((adapter.probeCallCount == 1));
|
|
CHECK((resumeSupported == true));
|
|
CHECK(contentLength.has_value());
|
|
CHECK((*contentLength == 1000));
|
|
}
|
|
|
|
SECTION("Tracks fetch calls") {
|
|
TestHttpAdapter adapter;
|
|
adapter.payload = to_bytes("test data");
|
|
|
|
std::vector<std::byte> received;
|
|
auto sink = [&received](std::span<const std::byte> data) -> Expected<void> {
|
|
received.insert(received.end(), data.begin(), data.end());
|
|
return {};
|
|
};
|
|
|
|
adapter.fetchRange("http://test.com", {}, 0, 9, TlsConfig{}, std::nullopt,
|
|
std::chrono::milliseconds{1000}, sink, nullptr, nullptr);
|
|
|
|
CHECK((adapter.fetchCallCount == 1));
|
|
CHECK((adapter.requestedOffsets.size() == 1));
|
|
CHECK((adapter.requestedOffsets[0] == 0));
|
|
CHECK((adapter.requestedSizes[0] == 9));
|
|
CHECK((received.size() == 9));
|
|
}
|
|
}
|
|
|
|
// =============================================================================
|
|
// Tracking Resume Store Tests
|
|
// =============================================================================
|
|
|
|
TEST_CASE("TrackingResumeStore: Tracking verification", "[downloader][resume][tracking]") {
|
|
TrackingResumeStore store;
|
|
|
|
SECTION("Tracks load calls") {
|
|
store.load("http://url1.com");
|
|
store.load("http://url2.com");
|
|
store.load("http://url1.com");
|
|
|
|
CHECK((store.loadUrls.size() == 3));
|
|
CHECK((store.loadUrls[0] == "http://url1.com"));
|
|
CHECK((store.loadUrls[1] == "http://url2.com"));
|
|
}
|
|
|
|
SECTION("Tracks save calls") {
|
|
IResumeStore::State state;
|
|
state.totalBytes = 100;
|
|
|
|
store.save("http://url1.com", state);
|
|
state.totalBytes = 200;
|
|
store.save("http://url2.com", state);
|
|
|
|
CHECK((store.savedUrls.size() == 2));
|
|
CHECK((store.savedStates.size() == 2));
|
|
CHECK((store.savedStates[0].totalBytes == 100));
|
|
CHECK((store.savedStates[1].totalBytes == 200));
|
|
}
|
|
|
|
SECTION("Tracks remove calls") {
|
|
store.remove("http://removed.com");
|
|
REQUIRE(store.removedUrl.has_value());
|
|
CHECK((*store.removedUrl == "http://removed.com"));
|
|
}
|
|
|
|
SECTION("Initial state returned on load") {
|
|
IResumeStore::State initial;
|
|
initial.totalBytes = 500;
|
|
initial.etag = "initial-etag";
|
|
store.initialState = initial;
|
|
|
|
auto result = store.load("http://any.com");
|
|
REQUIRE(result.ok());
|
|
REQUIRE(result.value().has_value());
|
|
CHECK((result.value()->totalBytes == 500));
|
|
CHECK((*result.value()->etag == "initial-etag"));
|
|
}
|
|
}
|
|
|
|
// =============================================================================
|
|
// Concrete downloader component tests
|
|
// =============================================================================
|
|
|
|
TEST_CASE("Concrete downloader components cover storage resume integrity and rate limits",
|
|
"[downloader][concrete]") {
|
|
SECTION("InMemoryResumeStore rejects empty keys and round-trips state") {
|
|
auto store = makeInMemoryResumeStore();
|
|
REQUIRE((store != nullptr));
|
|
|
|
auto emptyLoad = store->load("");
|
|
CHECK_FALSE(emptyLoad.ok());
|
|
CHECK((emptyLoad.error().code == ErrorCode::InvalidArgument));
|
|
|
|
IResumeStore::State state;
|
|
state.etag = "etag-1";
|
|
state.lastModified = "Mon, 01 Jan 2024 00:00:00 GMT";
|
|
state.totalBytes = 42;
|
|
state.completedRanges = {{0, 10}, {10, 32}};
|
|
REQUIRE(store->save("https://example.test/file", state).ok());
|
|
|
|
auto loaded = store->load("https://example.test/file");
|
|
REQUIRE(loaded.ok());
|
|
REQUIRE(loaded.value().has_value());
|
|
CHECK((loaded.value()->etag == state.etag));
|
|
CHECK((loaded.value()->lastModified == state.lastModified));
|
|
CHECK((loaded.value()->totalBytes == 42));
|
|
CHECK((loaded.value()->completedRanges == state.completedRanges));
|
|
|
|
store->remove("https://example.test/file");
|
|
auto removed = store->load("https://example.test/file");
|
|
REQUIRE(removed.ok());
|
|
CHECK_FALSE(removed.value().has_value());
|
|
}
|
|
|
|
SECTION("OpenSSL integrity verifier hashes bytes and resets after finalize") {
|
|
auto verifier = makeIntegrityVerifierSha256Only();
|
|
REQUIRE((verifier != nullptr));
|
|
verifier->reset(HashAlgo::Sha256);
|
|
auto data = to_bytes("abc");
|
|
verifier->update(data);
|
|
auto sum = verifier->finalize();
|
|
CHECK((sum.algo == HashAlgo::Sha256));
|
|
CHECK((sum.hex == "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"));
|
|
|
|
verifier->update(data);
|
|
auto sum2 = verifier->finalize();
|
|
CHECK((sum2.hex == sum.hex));
|
|
}
|
|
|
|
SECTION("RateLimiter supports unlimited and cooperative cancellation fast paths") {
|
|
auto limiter = makeRateLimiter();
|
|
REQUIRE((limiter != nullptr));
|
|
limiter->setLimits(RateLimit{});
|
|
const auto start = std::chrono::steady_clock::now();
|
|
limiter->acquire(1024, [] { return false; });
|
|
CHECK((std::chrono::steady_clock::now() - start < std::chrono::milliseconds(100)));
|
|
|
|
RateLimit limited;
|
|
limited.globalBps = 1;
|
|
limiter->setLimits(limited);
|
|
bool cancelled = false;
|
|
limiter->acquire(1024, [&] {
|
|
cancelled = true;
|
|
return true;
|
|
});
|
|
CHECK(cancelled);
|
|
}
|
|
|
|
SECTION("DiskWriter stages writes and finalizes into SHA-256 CAS layout") {
|
|
auto tempDir = make_temp_dir("yams-dl-concrete-");
|
|
StorageConfig storage{tempDir / "objects", tempDir / "staging"};
|
|
auto writer = makeDiskWriter();
|
|
REQUIRE((writer != nullptr));
|
|
|
|
auto staging = writer->createStagingFile(storage, "session-abc", "part");
|
|
REQUIRE(staging.ok());
|
|
CHECK((staging.value().extension() == ".part"));
|
|
CHECK(fs::exists(staging.value()));
|
|
|
|
auto data = to_bytes("abc");
|
|
REQUIRE(writer->writeAt(staging.value(), 0, data).ok());
|
|
REQUIRE(writer->sync(staging.value(), storage).ok());
|
|
|
|
const std::string digest =
|
|
"ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad";
|
|
auto finalPath = writer->finalizeToCas(staging.value(), digest, storage);
|
|
REQUIRE(finalPath.ok());
|
|
CHECK((finalPath.value() == casPathForSha256(storage.objectsDir, digest)));
|
|
CHECK(fs::exists(finalPath.value()));
|
|
CHECK_FALSE(fs::exists(staging.value()));
|
|
|
|
std::ifstream in(finalPath.value(), std::ios::binary);
|
|
std::string contents((std::istreambuf_iterator<char>(in)),
|
|
std::istreambuf_iterator<char>());
|
|
CHECK((contents == "abc"));
|
|
|
|
std::uint64_t currentSize = 0;
|
|
auto resumed =
|
|
writer->createOrOpenStagingFile(storage, "session-resume", ".tmp", 4096, currentSize);
|
|
REQUIRE(resumed.ok());
|
|
CHECK((currentSize == 0));
|
|
REQUIRE(writer->writeAt(resumed.value(), 4, data).ok());
|
|
auto reopened = writer->createOrOpenStagingFile(storage, "session-resume", ".tmp",
|
|
std::nullopt, currentSize);
|
|
REQUIRE(reopened.ok());
|
|
CHECK((reopened.value() == resumed.value()));
|
|
CHECK((currentSize >= 7));
|
|
writer->cleanup(reopened.value());
|
|
CHECK_FALSE(fs::exists(reopened.value()));
|
|
|
|
std::error_code ec;
|
|
fs::remove_all(tempDir, ec);
|
|
}
|
|
}
|
|
|
|
// =============================================================================
|
|
// FakeDiskWriter Tests
|
|
// =============================================================================
|
|
|
|
TEST_CASE("FakeDiskWriter: Operations", "[downloader][disk]") {
|
|
auto tempDir = make_temp_dir();
|
|
StorageConfig storage{tempDir / "objects", tempDir / "staging"};
|
|
FakeDiskWriter writer(storage);
|
|
|
|
SECTION("Creates staging file") {
|
|
auto result = writer.createStagingFile(storage, "session-123", ".part");
|
|
REQUIRE(result.ok());
|
|
CHECK(fs::exists(result.value()));
|
|
}
|
|
|
|
SECTION("Writes at offset") {
|
|
auto staging = writer.createStagingFile(storage, "session-456", ".part");
|
|
REQUIRE(staging.ok());
|
|
|
|
auto data = to_bytes("hello");
|
|
auto writeResult = writer.writeAt(staging.value(), 0, data);
|
|
REQUIRE(writeResult.ok());
|
|
|
|
auto data2 = to_bytes("world");
|
|
writeResult = writer.writeAt(staging.value(), 5, data2);
|
|
REQUIRE(writeResult.ok());
|
|
|
|
CHECK((writer.stagedData().size() == 10));
|
|
}
|
|
|
|
SECTION("Handles initial data") {
|
|
writer.setInitialData(to_bytes("PREFIX"));
|
|
std::uint64_t currentSize = 0;
|
|
auto result =
|
|
writer.createOrOpenStagingFile(storage, "session-789", ".part", 100, currentSize);
|
|
REQUIRE(result.ok());
|
|
CHECK((currentSize == 6));
|
|
}
|
|
|
|
// Cleanup
|
|
std::error_code ec;
|
|
fs::remove_all(tempDir, ec);
|
|
}
|
|
|
|
// =============================================================================
|
|
// Edge Cases and Error Handling
|
|
// =============================================================================
|
|
|
|
TEST_CASE("Downloader: Edge cases", "[downloader][edge]") {
|
|
SECTION("Empty URL handling") {
|
|
TrackingResumeStore store;
|
|
auto result = store.load("");
|
|
CHECK(result.ok());
|
|
}
|
|
|
|
SECTION("Very long URL") {
|
|
std::string longUrl = "http://example.com/" + std::string(1000, 'x');
|
|
TrackingResumeStore store;
|
|
auto result = store.load(longUrl);
|
|
CHECK(result.ok());
|
|
CHECK((store.loadUrls[0] == longUrl));
|
|
}
|
|
|
|
SECTION("Unicode in URL") {
|
|
std::string unicodeUrl = "http://example.com/文件/файл.bin";
|
|
TrackingResumeStore store;
|
|
IResumeStore::State state;
|
|
state.totalBytes = 100;
|
|
auto saveResult = store.save(unicodeUrl, state);
|
|
CHECK(saveResult.ok());
|
|
CHECK((store.savedUrls[0] == unicodeUrl));
|
|
}
|
|
}
|
|
|
|
TEST_CASE("HTTP header parsing edge cases", "[downloader][http][edge]") {
|
|
SECTION("Empty headers list") {
|
|
std::vector<std::string> headers;
|
|
auto parsed = parse_http_headers_for_meta(headers);
|
|
CHECK_FALSE(parsed.etag.has_value());
|
|
CHECK_FALSE(parsed.lastModified.has_value());
|
|
}
|
|
|
|
SECTION("Malformed headers (no colon)") {
|
|
std::vector<std::string> headers = {"InvalidHeaderNoColon", "ETag: valid-etag"};
|
|
auto parsed = parse_http_headers_for_meta(headers);
|
|
REQUIRE(parsed.etag.has_value());
|
|
CHECK((*parsed.etag == "valid-etag"));
|
|
}
|
|
|
|
SECTION("Empty header value") {
|
|
std::vector<std::string> headers = {"ETag: ", "Last-Modified: "};
|
|
auto parsed = parse_http_headers_for_meta(headers);
|
|
// Empty values should still be captured
|
|
REQUIRE(parsed.etag.has_value());
|
|
CHECK(parsed.etag->empty());
|
|
}
|
|
|
|
SECTION("Whitespace only value") {
|
|
std::vector<std::string> headers = {"ETag: "};
|
|
auto parsed = parse_http_headers_for_meta(headers);
|
|
REQUIRE(parsed.etag.has_value());
|
|
CHECK(parsed.etag->empty());
|
|
}
|
|
}
|