LibHTTP: Store associated data with disk cache entries
Add disk cache helpers that store and retrieve sidecar payloads with the same cache key and vary key as the HTTP response entry. The first consumer is JavaScript bytecode. Delete sidecars when the owning entry is removed, evicted, or replaced by a fresh response. Count their on-disk size toward the cache budget so bytecode data cannot grow outside eviction accounting. Disk cache tests cover sidecar round-trips, replacement cleanup, and eviction size accounting.
This commit is contained in:
Andreas Kling
Andreas Kling
parent
54fbf3ff30
commit
f70f485e48
12 changed files with 312 additions and 11 deletions
|
|
@ -159,6 +159,15 @@ ErrorOr<void> mkdir(StringView path, mode_t)
|
|||
return {};
|
||||
}
|
||||
|
||||
ErrorOr<void> rename(StringView old_path, StringView new_path)
|
||||
{
|
||||
ByteString old_path_string = old_path;
|
||||
ByteString new_path_string = new_path;
|
||||
if (!MoveFileExA(old_path_string.characters(), new_path_string.characters(), MOVEFILE_REPLACE_EXISTING))
|
||||
return Error::from_windows_error();
|
||||
return {};
|
||||
}
|
||||
|
||||
ErrorOr<int> openat(int, StringView, int, mode_t)
|
||||
{
|
||||
dbgln("Core::System::openat() is not implemented");
|
||||
|
|
|
|||
|
|
@ -89,6 +89,8 @@ void CacheEntry::remove()
|
|||
return;
|
||||
|
||||
(void)FileSystem::remove(m_path->string(), FileSystem::RecursionMode::Disallowed);
|
||||
for (auto associated_data : CACHE_ENTRY_ASSOCIATED_DATA_TYPES)
|
||||
(void)FileSystem::remove(path_for_cache_entry_associated_data(m_disk_cache.cache_directory(), m_cache_key, m_vary_key, associated_data).string(), FileSystem::RecursionMode::Disallowed);
|
||||
m_index.remove_entry(m_cache_key, m_vary_key);
|
||||
}
|
||||
|
||||
|
|
@ -203,6 +205,11 @@ ErrorOr<void> CacheEntryWriter::flush(NonnullRefPtr<HeaderList> request_headers,
|
|||
return result.release_error();
|
||||
}
|
||||
|
||||
// Drop any sidecars left over from an older entry at the same (cache_key, vary_key). They are tied to the previous
|
||||
// response body, so reusing them with the freshly written one would mismatch the source they were generated for.
|
||||
for (auto associated_data : CACHE_ENTRY_ASSOCIATED_DATA_TYPES)
|
||||
(void)FileSystem::remove(path_for_cache_entry_associated_data(m_disk_cache.cache_directory(), m_cache_key, m_vary_key, associated_data).string(), FileSystem::RecursionMode::Disallowed);
|
||||
|
||||
if (auto result = m_index.create_entry(m_cache_key, m_vary_key, m_url, move(request_headers), move(response_headers), m_cache_footer.data_size, m_request_time, m_response_time); result.is_error()) {
|
||||
dbgln_if(HTTP_DISK_CACHE_DEBUG, "\033[36m[disk]\033[0m \033[31;1mUnable to flush cache entry for\033[0m {} ({} bytes): {}", m_url, m_cache_footer.data_size, result.error());
|
||||
remove();
|
||||
|
|
|
|||
|
|
@ -88,6 +88,7 @@ ErrorOr<CacheIndex> CacheIndex::create(Database::Database& database, LexicalPath
|
|||
request_headers BLOB,
|
||||
response_headers BLOB,
|
||||
data_size INTEGER,
|
||||
associated_data_size INTEGER,
|
||||
request_time INTEGER,
|
||||
response_time INTEGER,
|
||||
last_access_time INTEGER,
|
||||
|
|
@ -97,19 +98,20 @@ ErrorOr<CacheIndex> CacheIndex::create(Database::Database& database, LexicalPath
|
|||
database.execute_statement(create_cache_index_table, {});
|
||||
|
||||
Statements statements {};
|
||||
statements.insert_entry = TRY(database.prepare_statement("INSERT OR REPLACE INTO CacheIndex VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?);"sv));
|
||||
statements.insert_entry = TRY(database.prepare_statement("INSERT OR REPLACE INTO CacheIndex VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?);"sv));
|
||||
statements.remove_entry = TRY(database.prepare_statement(R"#(
|
||||
DELETE FROM CacheIndex
|
||||
WHERE cache_key = ? AND vary_key = ?
|
||||
RETURNING data_size + OCTET_LENGTH(request_headers) + OCTET_LENGTH(response_headers);
|
||||
RETURNING data_size + associated_data_size + OCTET_LENGTH(request_headers) + OCTET_LENGTH(response_headers);
|
||||
)#"sv));
|
||||
statements.remove_entries_accessed_since = TRY(database.prepare_statement(R"#(
|
||||
DELETE FROM CacheIndex
|
||||
WHERE last_access_time >= ?
|
||||
RETURNING cache_key, vary_key, data_size + OCTET_LENGTH(request_headers) + OCTET_LENGTH(response_headers);
|
||||
RETURNING cache_key, vary_key, data_size + associated_data_size + OCTET_LENGTH(request_headers) + OCTET_LENGTH(response_headers);
|
||||
)#"sv));
|
||||
statements.select_entries = TRY(database.prepare_statement("SELECT * FROM CacheIndex WHERE cache_key = ?;"sv));
|
||||
statements.update_response_headers = TRY(database.prepare_statement("UPDATE CacheIndex SET response_headers = ? WHERE cache_key = ? AND vary_key = ?;"sv));
|
||||
statements.update_associated_data_size = TRY(database.prepare_statement("UPDATE CacheIndex SET associated_data_size = ? WHERE cache_key = ? AND vary_key = ?;"sv));
|
||||
statements.update_last_access_time = TRY(database.prepare_statement("UPDATE CacheIndex SET last_access_time = ? WHERE cache_key = ? AND vary_key = ?;"sv));
|
||||
|
||||
statements.remove_entries_exceeding_cache_limit = TRY(database.prepare_statement(R"#(
|
||||
|
|
@ -117,7 +119,7 @@ ErrorOr<CacheIndex> CacheIndex::create(Database::Database& database, LexicalPath
|
|||
SELECT
|
||||
cache_key,
|
||||
vary_key,
|
||||
SUM(data_size + OCTET_LENGTH(request_headers) + OCTET_LENGTH(response_headers))
|
||||
SUM(data_size + associated_data_size + OCTET_LENGTH(request_headers) + OCTET_LENGTH(response_headers))
|
||||
OVER (ORDER BY last_access_time DESC)
|
||||
AS cumulative_estimated_size
|
||||
FROM CacheIndex
|
||||
|
|
@ -128,17 +130,17 @@ ErrorOr<CacheIndex> CacheIndex::create(Database::Database& database, LexicalPath
|
|||
FROM RankedCacheIndex
|
||||
WHERE cumulative_estimated_size > ?
|
||||
)
|
||||
RETURNING cache_key, vary_key, data_size + OCTET_LENGTH(request_headers) + OCTET_LENGTH(response_headers);
|
||||
RETURNING cache_key, vary_key, data_size + associated_data_size + OCTET_LENGTH(request_headers) + OCTET_LENGTH(response_headers);
|
||||
)#"sv));
|
||||
|
||||
statements.estimate_cache_size_accessed_since = TRY(database.prepare_statement(R"#(
|
||||
SELECT COALESCE(SUM(data_size + OCTET_LENGTH(request_headers) + OCTET_LENGTH(response_headers)), 0)
|
||||
SELECT COALESCE(SUM(data_size + associated_data_size + OCTET_LENGTH(request_headers) + OCTET_LENGTH(response_headers)), 0)
|
||||
FROM CacheIndex
|
||||
WHERE last_access_time >= ?;
|
||||
)#"sv));
|
||||
|
||||
statements.select_total_estimated_size = TRY(database.prepare_statement(R"#(
|
||||
SELECT COALESCE(SUM(data_size + OCTET_LENGTH(request_headers) + OCTET_LENGTH(response_headers)), 0)
|
||||
SELECT COALESCE(SUM(data_size + associated_data_size + OCTET_LENGTH(request_headers) + OCTET_LENGTH(response_headers)), 0)
|
||||
FROM CacheIndex;
|
||||
)#"sv));
|
||||
|
||||
|
|
@ -189,6 +191,7 @@ ErrorOr<void> CacheIndex::create_entry(u64 cache_key, u64 vary_key, String url,
|
|||
.request_headers = move(request_headers),
|
||||
.response_headers = move(response_headers),
|
||||
.data_size = data_size,
|
||||
.associated_data_size = 0,
|
||||
.serialized_request_headers_size = static_cast<u64>(serialized_request_headers.length()),
|
||||
.serialized_response_headers_size = static_cast<u64>(serialized_response_headers.length()),
|
||||
.request_time = request_time,
|
||||
|
|
@ -214,7 +217,7 @@ ErrorOr<void> CacheIndex::create_entry(u64 cache_key, u64 vary_key, String url,
|
|||
return existing_entry.vary_key == vary_key;
|
||||
});
|
||||
|
||||
m_database->execute_statement(m_statements.insert_entry, {}, cache_key, vary_key, entry.url, serialized_request_headers, serialized_response_headers, entry.data_size, entry.request_time, entry.response_time, entry.last_access_time);
|
||||
m_database->execute_statement(m_statements.insert_entry, {}, cache_key, vary_key, entry.url, serialized_request_headers, serialized_response_headers, entry.data_size, entry.associated_data_size, entry.request_time, entry.response_time, entry.last_access_time);
|
||||
|
||||
if (existing_entry_index.has_value())
|
||||
entries[*existing_entry_index] = move(entry);
|
||||
|
|
@ -294,6 +297,19 @@ void CacheIndex::update_response_headers(u64 cache_key, u64 vary_key, NonnullRef
|
|||
entry->serialized_response_headers_size = serialized_response_headers_size;
|
||||
}
|
||||
|
||||
void CacheIndex::update_associated_data_size(u64 cache_key, u64 vary_key, u64 associated_data_size)
|
||||
{
|
||||
auto entry = get_entry(cache_key, vary_key);
|
||||
if (!entry.has_value())
|
||||
return;
|
||||
|
||||
m_database->execute_statement(m_statements.update_associated_data_size, {}, associated_data_size, cache_key, vary_key);
|
||||
|
||||
m_total_estimated_size -= entry->associated_data_size;
|
||||
m_total_estimated_size += associated_data_size;
|
||||
entry->associated_data_size = associated_data_size;
|
||||
}
|
||||
|
||||
void CacheIndex::update_last_access_time(u64 cache_key, u64 vary_key)
|
||||
{
|
||||
auto entry = get_entry(cache_key, vary_key);
|
||||
|
|
@ -321,11 +337,12 @@ Optional<CacheIndex::Entry const&> CacheIndex::find_entry(u64 cache_key, HeaderL
|
|||
auto request_headers = m_database->result_column<ByteString>(statement_id, column++);
|
||||
auto response_headers = m_database->result_column<ByteString>(statement_id, column++);
|
||||
auto data_size = m_database->result_column<u64>(statement_id, column++);
|
||||
auto associated_data_size = m_database->result_column<u64>(statement_id, column++);
|
||||
auto request_time = m_database->result_column<UnixDateTime>(statement_id, column++);
|
||||
auto response_time = m_database->result_column<UnixDateTime>(statement_id, column++);
|
||||
auto last_access_time = m_database->result_column<UnixDateTime>(statement_id, column++);
|
||||
|
||||
entries.empend(vary_key, move(url), deserialize_headers(request_headers), deserialize_headers(response_headers), data_size, request_headers.length(), response_headers.length(), request_time, response_time, last_access_time);
|
||||
entries.empend(vary_key, move(url), deserialize_headers(request_headers), deserialize_headers(response_headers), data_size, associated_data_size, request_headers.length(), response_headers.length(), request_time, response_time, last_access_time);
|
||||
},
|
||||
cache_key);
|
||||
|
||||
|
|
|
|||
|
|
@ -27,6 +27,7 @@ class CacheIndex {
|
|||
NonnullRefPtr<HeaderList> request_headers;
|
||||
NonnullRefPtr<HeaderList> response_headers;
|
||||
u64 data_size { 0 };
|
||||
u64 associated_data_size { 0 };
|
||||
u64 serialized_request_headers_size { 0 };
|
||||
u64 serialized_response_headers_size { 0 };
|
||||
|
||||
|
|
@ -36,7 +37,7 @@ class CacheIndex {
|
|||
|
||||
u64 estimated_size() const
|
||||
{
|
||||
return data_size + serialized_request_headers_size + serialized_response_headers_size;
|
||||
return data_size + associated_data_size + serialized_request_headers_size + serialized_response_headers_size;
|
||||
}
|
||||
};
|
||||
|
||||
|
|
@ -51,6 +52,7 @@ public:
|
|||
Optional<Entry const&> find_entry(u64 cache_key, HeaderList const& request_headers);
|
||||
|
||||
void update_response_headers(u64 cache_key, u64 vary_key, NonnullRefPtr<HeaderList>);
|
||||
void update_associated_data_size(u64 cache_key, u64 vary_key, u64 associated_data_size);
|
||||
void update_last_access_time(u64 cache_key, u64 vary_key);
|
||||
|
||||
Requests::CacheSizes estimate_cache_size_accessed_since(UnixDateTime since);
|
||||
|
|
@ -65,6 +67,7 @@ private:
|
|||
Database::StatementID remove_entries_accessed_since { 0 };
|
||||
Database::StatementID select_entries { 0 };
|
||||
Database::StatementID update_response_headers { 0 };
|
||||
Database::StatementID update_associated_data_size { 0 };
|
||||
Database::StatementID update_last_access_time { 0 };
|
||||
Database::StatementID estimate_cache_size_accessed_since { 0 };
|
||||
Database::StatementID select_total_estimated_size { 0 };
|
||||
|
|
|
|||
|
|
@ -5,8 +5,10 @@
|
|||
*/
|
||||
|
||||
#include <AK/Debug.h>
|
||||
#include <AK/ScopeGuard.h>
|
||||
#include <LibCore/Directory.h>
|
||||
#include <LibCore/EventLoop.h>
|
||||
#include <LibCore/File.h>
|
||||
#include <LibCore/StandardPaths.h>
|
||||
#include <LibCore/System.h>
|
||||
#include <LibFileSystem/FileSystem.h>
|
||||
|
|
@ -19,6 +21,25 @@ namespace HTTP {
|
|||
|
||||
static constexpr auto INDEX_DATABASE = "INDEX"sv;
|
||||
|
||||
static ErrorOr<u64> compute_associated_data_size(LexicalPath const& cache_directory, u64 cache_key, u64 vary_key)
|
||||
{
|
||||
u64 associated_data_size = 0;
|
||||
for (auto associated_data : CACHE_ENTRY_ASSOCIATED_DATA_TYPES) {
|
||||
auto path = path_for_cache_entry_associated_data(cache_directory, cache_key, vary_key, associated_data);
|
||||
auto size = FileSystem::size_from_stat(path.string());
|
||||
if (size.is_error()) {
|
||||
if (size.error().is_errno() && size.error().code() == ENOENT)
|
||||
continue;
|
||||
return size.release_error();
|
||||
}
|
||||
|
||||
if (size.value() < 0)
|
||||
return Error::from_errno(EINVAL);
|
||||
associated_data_size += static_cast<u64>(size.value());
|
||||
}
|
||||
return associated_data_size;
|
||||
}
|
||||
|
||||
static constexpr StringView cache_directory_for_mode(DiskCache::Mode mode)
|
||||
{
|
||||
switch (mode) {
|
||||
|
|
@ -202,6 +223,57 @@ Variant<Optional<CacheEntryReader&>, DiskCache::CacheHasOpenEntry> DiskCache::op
|
|||
return Optional<CacheEntryReader&> { *cache_entry_pointer };
|
||||
}
|
||||
|
||||
ErrorOr<bool> DiskCache::store_associated_data(URL::URL const& url, StringView method, HeaderList const& request_headers, CacheEntryAssociatedData associated_data, ReadonlyBytes data)
|
||||
{
|
||||
if (!is_cacheable(method, request_headers))
|
||||
return false;
|
||||
|
||||
auto serialized_url = serialize_url_for_cache_storage(url);
|
||||
auto cache_key = create_cache_key(serialized_url, method, m_partitioned_cache_key);
|
||||
auto index_entry = m_index.find_entry(cache_key, request_headers);
|
||||
if (!index_entry.has_value())
|
||||
return false;
|
||||
|
||||
auto path = path_for_cache_entry_associated_data(m_cache_directory, cache_key, index_entry->vary_key, associated_data);
|
||||
auto temporary_path = LexicalPath::join(m_cache_directory.string(), ByteString::formatted("{}.tmp", path.basename()));
|
||||
ArmedScopeGuard remove_temporary_file = [&]() {
|
||||
(void)FileSystem::remove(temporary_path.string(), FileSystem::RecursionMode::Disallowed);
|
||||
};
|
||||
|
||||
{
|
||||
auto file = TRY(Core::File::open(temporary_path.string(), Core::File::OpenMode::Write));
|
||||
TRY(file->write_until_depleted(data));
|
||||
}
|
||||
|
||||
TRY(Core::System::rename(temporary_path.string(), path.string()));
|
||||
remove_temporary_file.disarm();
|
||||
m_index.update_associated_data_size(cache_key, index_entry->vary_key, TRY(compute_associated_data_size(m_cache_directory, cache_key, index_entry->vary_key)));
|
||||
remove_entries_exceeding_cache_limit();
|
||||
return m_index.find_entry(cache_key, request_headers).has_value();
|
||||
}
|
||||
|
||||
ErrorOr<Optional<ByteBuffer>> DiskCache::retrieve_associated_data(URL::URL const& url, StringView method, HeaderList const& request_headers, CacheEntryAssociatedData associated_data)
|
||||
{
|
||||
if (!is_cacheable(method, request_headers))
|
||||
return Optional<ByteBuffer> {};
|
||||
|
||||
auto serialized_url = serialize_url_for_cache_storage(url);
|
||||
auto cache_key = create_cache_key(serialized_url, method, m_partitioned_cache_key);
|
||||
auto index_entry = m_index.find_entry(cache_key, request_headers);
|
||||
if (!index_entry.has_value())
|
||||
return Optional<ByteBuffer> {};
|
||||
|
||||
auto path = path_for_cache_entry_associated_data(m_cache_directory, cache_key, index_entry->vary_key, associated_data);
|
||||
auto file = Core::File::open(path.string(), Core::File::OpenMode::Read);
|
||||
if (file.is_error()) {
|
||||
if (file.error().is_errno() && file.error().code() == ENOENT)
|
||||
return Optional<ByteBuffer> {};
|
||||
return file.release_error();
|
||||
}
|
||||
|
||||
return TRY(file.value()->read_until_eof());
|
||||
}
|
||||
|
||||
bool DiskCache::check_if_cache_has_open_entry(CacheRequest& request, u64 cache_key, URL::URL const& url, CheckReaderEntries check_reader_entries)
|
||||
{
|
||||
// FIXME: We purposefully do not use the vary key here, as we do not yet have it when creating a CacheEntryWriter
|
||||
|
|
@ -294,6 +366,8 @@ void DiskCache::delete_entry(u64 cache_key, u64 vary_key)
|
|||
|
||||
auto cache_path = path_for_cache_entry(m_cache_directory, cache_key, vary_key);
|
||||
(void)FileSystem::remove(cache_path.string(), FileSystem::RecursionMode::Disallowed);
|
||||
for (auto associated_data : CACHE_ENTRY_ASSOCIATED_DATA_TYPES)
|
||||
(void)FileSystem::remove(path_for_cache_entry_associated_data(m_cache_directory, cache_key, vary_key, associated_data).string(), FileSystem::RecursionMode::Disallowed);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@
|
|||
|
||||
#pragma once
|
||||
|
||||
#include <AK/ByteBuffer.h>
|
||||
#include <AK/Error.h>
|
||||
#include <AK/LexicalPath.h>
|
||||
#include <AK/Optional.h>
|
||||
|
|
@ -17,6 +18,7 @@
|
|||
#include <LibHTTP/Cache/CacheEntry.h>
|
||||
#include <LibHTTP/Cache/CacheIndex.h>
|
||||
#include <LibHTTP/Cache/CacheMode.h>
|
||||
#include <LibHTTP/Cache/Utilities.h>
|
||||
#include <LibURL/Forward.h>
|
||||
|
||||
namespace HTTP {
|
||||
|
|
@ -52,6 +54,9 @@ public:
|
|||
};
|
||||
Variant<Optional<CacheEntryReader&>, CacheHasOpenEntry> open_entry(CacheRequest&, URL::URL const&, StringView method, HeaderList const& request_headers, CacheMode, OpenMode);
|
||||
|
||||
ErrorOr<bool> store_associated_data(URL::URL const&, StringView method, HeaderList const& request_headers, CacheEntryAssociatedData, ReadonlyBytes);
|
||||
ErrorOr<Optional<ByteBuffer>> retrieve_associated_data(URL::URL const&, StringView method, HeaderList const& request_headers, CacheEntryAssociatedData);
|
||||
|
||||
void remove_entries_exceeding_cache_limit();
|
||||
void set_maximum_disk_cache_size(u64 maximum_disk_cache_size);
|
||||
|
||||
|
|
|
|||
|
|
@ -111,6 +111,24 @@ LexicalPath path_for_cache_entry(LexicalPath const& cache_directory, u64 cache_k
|
|||
return cache_directory.append(file);
|
||||
}
|
||||
|
||||
static StringView cache_entry_associated_data_suffix(CacheEntryAssociatedData associated_data)
|
||||
{
|
||||
switch (associated_data) {
|
||||
case CacheEntryAssociatedData::JavaScriptBytecode:
|
||||
return "jsbc"sv;
|
||||
}
|
||||
VERIFY_NOT_REACHED();
|
||||
}
|
||||
|
||||
LexicalPath path_for_cache_entry_associated_data(LexicalPath const& cache_directory, u64 cache_key, u64 vary_key, CacheEntryAssociatedData associated_data)
|
||||
{
|
||||
auto file = vary_key == 0
|
||||
? ByteString::formatted("{:016x}.{}", cache_key, cache_entry_associated_data_suffix(associated_data))
|
||||
: ByteString::formatted("{:016x}_{:016x}.{}", cache_key, vary_key, cache_entry_associated_data_suffix(associated_data));
|
||||
|
||||
return cache_directory.append(file);
|
||||
}
|
||||
|
||||
// https://httpwg.org/specs/rfc9111.html#response.cacheability
|
||||
bool is_cacheable(StringView method, HTTP::HeaderList const& request_headers)
|
||||
{
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@
|
|||
|
||||
#pragma once
|
||||
|
||||
#include <AK/Array.h>
|
||||
#include <AK/LexicalPath.h>
|
||||
#include <AK/StringView.h>
|
||||
#include <AK/Time.h>
|
||||
|
|
@ -23,6 +24,11 @@ constexpr inline auto TEST_CACHE_REQUEST_TIME_OFFSET = "X-Ladybird-Request-Time-
|
|||
|
||||
constexpr inline u64 DEFAULT_MAXIMUM_DISK_CACHE_SIZE = 5 * GiB;
|
||||
|
||||
enum class CacheEntryAssociatedData {
|
||||
JavaScriptBytecode,
|
||||
};
|
||||
constexpr inline Array CACHE_ENTRY_ASSOCIATED_DATA_TYPES { CacheEntryAssociatedData::JavaScriptBytecode };
|
||||
|
||||
u64 compute_maximum_disk_cache_size(u64 free_bytes, u64 limit_maximum_disk_cache_size = DEFAULT_MAXIMUM_DISK_CACHE_SIZE);
|
||||
u64 compute_maximum_disk_cache_entry_size(u64 maximum_disk_cache_size);
|
||||
|
||||
|
|
@ -30,6 +36,7 @@ String serialize_url_for_cache_storage(URL::URL const&);
|
|||
u64 create_cache_key(StringView url, StringView method, Optional<String const&> extra_cache_key = {});
|
||||
u64 create_vary_key(HeaderList const& request_headers, HeaderList const& response_headers);
|
||||
LexicalPath path_for_cache_entry(LexicalPath const& cache_directory, u64 cache_key, u64 vary_key);
|
||||
LexicalPath path_for_cache_entry_associated_data(LexicalPath const& cache_directory, u64 cache_key, u64 vary_key, CacheEntryAssociatedData);
|
||||
|
||||
bool is_cacheable(StringView method, HeaderList const&);
|
||||
bool is_cacheable(u32 status_code, HeaderList const&);
|
||||
|
|
|
|||
|
|
@ -11,6 +11,6 @@
|
|||
namespace HTTP {
|
||||
|
||||
// Increment this version when a breaking change is made to the cache index or cache entry formats.
|
||||
static constexpr inline u32 CACHE_VERSION = 6u;
|
||||
static constexpr inline u32 CACHE_VERSION = 7u;
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,3 +8,4 @@ foreach(source IN LISTS TEST_SOURCES)
|
|||
endforeach()
|
||||
|
||||
ladybird_test("TestCacheIndex.cpp" LibWeb LIBS LibHTTP LibDatabase)
|
||||
ladybird_test("TestDiskCache.cpp" LibWeb LIBS LibHTTP LibURL)
|
||||
|
|
|
|||
|
|
@ -112,3 +112,29 @@ TEST_CASE(remove_entries_exceeding_cache_limit_tolerates_replaced_unloaded_entri
|
|||
EXPECT_EQ(removed_entries.size(), 0u);
|
||||
EXPECT_EQ(reloaded_index.estimate_cache_size_accessed_since(UnixDateTime::earliest()).total, 80u);
|
||||
}
|
||||
|
||||
TEST_CASE(associated_data_counts_toward_cache_size)
|
||||
{
|
||||
auto state = create_cache_index();
|
||||
|
||||
auto request_headers = HTTP::HeaderList::create();
|
||||
auto response_headers = HTTP::HeaderList::create();
|
||||
auto vary_key = HTTP::create_vary_key(*request_headers, *response_headers);
|
||||
auto now = UnixDateTime::now();
|
||||
|
||||
state.index.set_maximum_disk_cache_size(80);
|
||||
|
||||
for (u64 cache_key = 1; cache_key <= 5; ++cache_key)
|
||||
TRY_OR_FAIL(state.index.create_entry(cache_key, vary_key, "https://example.com/script.js"_string, request_headers, response_headers, 10, now, now));
|
||||
state.index.update_associated_data_size(1, vary_key, 50);
|
||||
|
||||
EXPECT_EQ(state.index.estimate_cache_size_accessed_since(UnixDateTime::earliest()).total, 100u);
|
||||
|
||||
Vector<u64> removed_entries;
|
||||
state.index.remove_entries_exceeding_cache_limit([&](auto removed_cache_key, auto) {
|
||||
removed_entries.append(removed_cache_key);
|
||||
});
|
||||
|
||||
EXPECT(removed_entries.size() > 0);
|
||||
EXPECT(state.index.estimate_cache_size_accessed_since(UnixDateTime::earliest()).total <= 80u);
|
||||
}
|
||||
|
|
|
|||
134
Tests/LibHTTP/TestDiskCache.cpp
Normal file
134
Tests/LibHTTP/TestDiskCache.cpp
Normal file
|
|
@ -0,0 +1,134 @@
|
|||
/*
|
||||
* Copyright (c) 2026-present, the Ladybird developers.
|
||||
*
|
||||
* SPDX-License-Identifier: BSD-2-Clause
|
||||
*/
|
||||
|
||||
#include <AK/ByteBuffer.h>
|
||||
#include <LibHTTP/Cache/CacheRequest.h>
|
||||
#include <LibHTTP/Cache/DiskCache.h>
|
||||
#include <LibHTTP/Cache/Utilities.h>
|
||||
#include <LibHTTP/HeaderList.h>
|
||||
#include <LibTest/TestCase.h>
|
||||
#include <LibURL/Parser.h>
|
||||
|
||||
struct TestCacheRequest final : public HTTP::CacheRequest {
|
||||
virtual bool is_revalidation_request() const override { return false; }
|
||||
virtual void notify_request_unblocked(Badge<HTTP::DiskCache>) override { }
|
||||
};
|
||||
|
||||
static URL::URL parse_url(StringView url)
|
||||
{
|
||||
return URL::Parser::basic_parse(url).release_value();
|
||||
}
|
||||
|
||||
static NonnullRefPtr<HTTP::HeaderList> create_cacheable_request_headers()
|
||||
{
|
||||
return HTTP::HeaderList::create({
|
||||
{ HTTP::TEST_CACHE_ENABLED_HEADER, "1"sv },
|
||||
});
|
||||
}
|
||||
|
||||
static NonnullRefPtr<HTTP::HeaderList> create_cacheable_response_headers()
|
||||
{
|
||||
return HTTP::HeaderList::create({
|
||||
{ "Cache-Control"sv, "max-age=60"sv },
|
||||
});
|
||||
}
|
||||
|
||||
static HTTP::CacheEntryWriter& create_cache_entry(HTTP::DiskCache& disk_cache, TestCacheRequest& request, URL::URL const& url, HTTP::HeaderList const& request_headers)
|
||||
{
|
||||
Optional<HTTP::CacheEntryWriter&> writer;
|
||||
|
||||
disk_cache.create_entry(request, url, "GET"sv, request_headers, UnixDateTime::now())
|
||||
.visit(
|
||||
[&](Optional<HTTP::CacheEntryWriter&> cache_entry_writer) {
|
||||
writer = cache_entry_writer;
|
||||
},
|
||||
[](HTTP::DiskCache::CacheHasOpenEntry) {
|
||||
FAIL("Cache entry was unexpectedly open");
|
||||
});
|
||||
|
||||
VERIFY(writer.has_value());
|
||||
return *writer;
|
||||
}
|
||||
|
||||
TEST_CASE(associated_data_round_trips_with_cache_entry)
|
||||
{
|
||||
auto disk_cache = MUST(HTTP::DiskCache::create(HTTP::DiskCache::Mode::Testing));
|
||||
TestCacheRequest request;
|
||||
|
||||
auto url = parse_url("https://example.com/script.js"sv);
|
||||
auto request_headers = create_cacheable_request_headers();
|
||||
auto response_headers = create_cacheable_response_headers();
|
||||
|
||||
auto& writer = create_cache_entry(disk_cache, request, url, *request_headers);
|
||||
TRY_OR_FAIL(writer.write_status_and_reason(200, "OK"_string, *request_headers, *response_headers));
|
||||
TRY_OR_FAIL(writer.write_data("console.log('hello');"sv.bytes()));
|
||||
TRY_OR_FAIL(writer.flush(request_headers, response_headers));
|
||||
|
||||
auto bytecode = TRY_OR_FAIL(ByteBuffer::copy("bytecode"sv.bytes()));
|
||||
EXPECT(TRY_OR_FAIL(disk_cache.store_associated_data(url, "GET"sv, *request_headers, HTTP::CacheEntryAssociatedData::JavaScriptBytecode, bytecode.bytes())));
|
||||
|
||||
auto retrieved_bytecode = TRY_OR_FAIL(disk_cache.retrieve_associated_data(url, "GET"sv, *request_headers, HTTP::CacheEntryAssociatedData::JavaScriptBytecode));
|
||||
VERIFY(retrieved_bytecode.has_value());
|
||||
EXPECT_EQ(retrieved_bytecode->bytes(), bytecode.bytes());
|
||||
|
||||
disk_cache.remove_entries_accessed_since(UnixDateTime::earliest());
|
||||
|
||||
retrieved_bytecode = TRY_OR_FAIL(disk_cache.retrieve_associated_data(url, "GET"sv, *request_headers, HTTP::CacheEntryAssociatedData::JavaScriptBytecode));
|
||||
EXPECT(!retrieved_bytecode.has_value());
|
||||
}
|
||||
|
||||
TEST_CASE(replacing_cache_entry_removes_associated_data)
|
||||
{
|
||||
auto disk_cache = MUST(HTTP::DiskCache::create(HTTP::DiskCache::Mode::Testing));
|
||||
TestCacheRequest request;
|
||||
|
||||
auto url = parse_url("https://example.com/script.js"sv);
|
||||
auto request_headers = create_cacheable_request_headers();
|
||||
auto response_headers = create_cacheable_response_headers();
|
||||
|
||||
auto& writer = create_cache_entry(disk_cache, request, url, *request_headers);
|
||||
TRY_OR_FAIL(writer.write_status_and_reason(200, "OK"_string, *request_headers, *response_headers));
|
||||
TRY_OR_FAIL(writer.write_data("console.log('old');"sv.bytes()));
|
||||
TRY_OR_FAIL(writer.flush(request_headers, response_headers));
|
||||
|
||||
auto bytecode = TRY_OR_FAIL(ByteBuffer::copy("bytecode"sv.bytes()));
|
||||
EXPECT(TRY_OR_FAIL(disk_cache.store_associated_data(url, "GET"sv, *request_headers, HTTP::CacheEntryAssociatedData::JavaScriptBytecode, bytecode.bytes())));
|
||||
|
||||
auto retrieved_bytecode = TRY_OR_FAIL(disk_cache.retrieve_associated_data(url, "GET"sv, *request_headers, HTTP::CacheEntryAssociatedData::JavaScriptBytecode));
|
||||
VERIFY(retrieved_bytecode.has_value());
|
||||
|
||||
auto replacement_request_headers = create_cacheable_request_headers();
|
||||
auto replacement_response_headers = create_cacheable_response_headers();
|
||||
auto& replacement_writer = create_cache_entry(disk_cache, request, url, *replacement_request_headers);
|
||||
TRY_OR_FAIL(replacement_writer.write_status_and_reason(200, "OK"_string, *replacement_request_headers, *replacement_response_headers));
|
||||
TRY_OR_FAIL(replacement_writer.write_data("console.log('new');"sv.bytes()));
|
||||
TRY_OR_FAIL(replacement_writer.flush(replacement_request_headers, replacement_response_headers));
|
||||
|
||||
retrieved_bytecode = TRY_OR_FAIL(disk_cache.retrieve_associated_data(url, "GET"sv, *request_headers, HTTP::CacheEntryAssociatedData::JavaScriptBytecode));
|
||||
EXPECT(!retrieved_bytecode.has_value());
|
||||
}
|
||||
|
||||
TEST_CASE(associated_data_participates_in_cache_eviction)
|
||||
{
|
||||
auto disk_cache = MUST(HTTP::DiskCache::create(HTTP::DiskCache::Mode::Testing));
|
||||
TestCacheRequest request;
|
||||
|
||||
auto url = parse_url("https://example.com/script.js"sv);
|
||||
auto request_headers = create_cacheable_request_headers();
|
||||
auto response_headers = create_cacheable_response_headers();
|
||||
|
||||
auto& writer = create_cache_entry(disk_cache, request, url, *request_headers);
|
||||
TRY_OR_FAIL(writer.write_status_and_reason(200, "OK"_string, *request_headers, *response_headers));
|
||||
TRY_OR_FAIL(writer.write_data("console.log('hello');"sv.bytes()));
|
||||
TRY_OR_FAIL(writer.flush(request_headers, response_headers));
|
||||
|
||||
disk_cache.set_maximum_disk_cache_size(80);
|
||||
auto bytecode = TRY_OR_FAIL(ByteBuffer::create_zeroed(100));
|
||||
EXPECT(!TRY_OR_FAIL(disk_cache.store_associated_data(url, "GET"sv, *request_headers, HTTP::CacheEntryAssociatedData::JavaScriptBytecode, bytecode.bytes())));
|
||||
|
||||
auto retrieved_bytecode = TRY_OR_FAIL(disk_cache.retrieve_associated_data(url, "GET"sv, *request_headers, HTTP::CacheEntryAssociatedData::JavaScriptBytecode));
|
||||
EXPECT(!retrieved_bytecode.has_value());
|
||||
}
|
||||
Loading…
Reference in a new issue