From ba49942b6d1a6870065683cff12496825c043229 Mon Sep 17 00:00:00 2001 From: Timothy Flynn Date: Sun, 2 Nov 2025 13:10:27 -0500 Subject: [PATCH] LibRequests+RequestServer: Add a method to estimate disk cache size This allows estimating the cache size stored on disk since a provided time stamp, and in total. --- Libraries/LibRequests/CMakeLists.txt | 1 + Libraries/LibRequests/CacheSizes.cpp | 31 +++++++++++++++++++ Libraries/LibRequests/CacheSizes.h | 29 +++++++++++++++++ Libraries/LibRequests/RequestClient.cpp | 23 ++++++++++++++ Libraries/LibRequests/RequestClient.h | 12 +++++-- Services/RequestServer/CMakeLists.txt | 2 +- Services/RequestServer/Cache/CacheIndex.cpp | 18 +++++++++++ Services/RequestServer/Cache/CacheIndex.h | 4 +++ Services/RequestServer/Cache/DiskCache.cpp | 5 +++ Services/RequestServer/Cache/DiskCache.h | 1 + .../RequestServer/ConnectionFromClient.cpp | 10 ++++++ Services/RequestServer/ConnectionFromClient.h | 1 + Services/RequestServer/RequestClient.ipc | 3 ++ Services/RequestServer/RequestServer.ipc | 1 + 14 files changed, 138 insertions(+), 3 deletions(-) create mode 100644 Libraries/LibRequests/CacheSizes.cpp create mode 100644 Libraries/LibRequests/CacheSizes.h diff --git a/Libraries/LibRequests/CMakeLists.txt b/Libraries/LibRequests/CMakeLists.txt index 927d94862f..48b9d3a3f2 100644 --- a/Libraries/LibRequests/CMakeLists.txt +++ b/Libraries/LibRequests/CMakeLists.txt @@ -1,4 +1,5 @@ set(SOURCES + CacheSizes.cpp NetworkError.h Request.cpp RequestClient.cpp diff --git a/Libraries/LibRequests/CacheSizes.cpp b/Libraries/LibRequests/CacheSizes.cpp new file mode 100644 index 0000000000..9049b2155c --- /dev/null +++ b/Libraries/LibRequests/CacheSizes.cpp @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2025, Tim Flynn + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include +#include +#include + +namespace IPC { + +template<> +ErrorOr encode(Encoder& encoder, Requests::CacheSizes const& sizes) +{ + TRY(encoder.encode(sizes.since_requested_time)); + TRY(encoder.encode(sizes.total)); + + return {}; +} + +template<> +ErrorOr decode(Decoder& decoder) +{ + auto since_requested_time = TRY(decoder.decode()); + auto total = TRY(decoder.decode()); + + return Requests::CacheSizes { since_requested_time, total }; +} + +} diff --git a/Libraries/LibRequests/CacheSizes.h b/Libraries/LibRequests/CacheSizes.h new file mode 100644 index 0000000000..8ed45418a8 --- /dev/null +++ b/Libraries/LibRequests/CacheSizes.h @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2025, Tim Flynn + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#pragma once + +#include +#include + +namespace Requests { + +struct CacheSizes { + u64 since_requested_time { 0 }; + u64 total { 0 }; +}; + +} + +namespace IPC { + +template<> +ErrorOr encode(Encoder&, Requests::CacheSizes const&); + +template<> +ErrorOr decode(Decoder&); + +} diff --git a/Libraries/LibRequests/RequestClient.cpp b/Libraries/LibRequests/RequestClient.cpp index c5770f5874..693be48381 100644 --- a/Libraries/LibRequests/RequestClient.cpp +++ b/Libraries/LibRequests/RequestClient.cpp @@ -4,6 +4,7 @@ * SPDX-License-Identifier: BSD-2-Clause */ +#include #include #include #include @@ -24,7 +25,11 @@ void RequestClient::die() request->did_finish({}, {}, {}, NetworkError::RequestServerDied); } + for (auto& [id, promise] : m_pending_cache_size_estimations) + promise->reject(Error::from_string_literal("RequestServer process died")); + m_requests.clear(); + m_pending_cache_size_estimations.clear(); } void RequestClient::ensure_connection(URL::URL const& url, ::RequestServer::CacheLevel cache_level) @@ -73,6 +78,24 @@ bool RequestClient::set_certificate(Badge, Request& request, ByteString return IPCProxy::set_certificate(request.id(), move(certificate), move(key)); } +NonnullRefPtr> RequestClient::estimate_cache_size_accessed_since(UnixDateTime since) +{ + auto promise = Core::Promise::construct(); + + auto cache_size_estimation_id = m_next_cache_size_estimation_id++; + m_pending_cache_size_estimations.set(cache_size_estimation_id, promise); + + async_estimate_cache_size_accessed_since(cache_size_estimation_id, since); + + return promise; +} + +void RequestClient::estimated_cache_size(u64 cache_size_estimation_id, CacheSizes sizes) +{ + if (auto promise = m_pending_cache_size_estimations.take(cache_size_estimation_id); promise.has_value()) + (*promise)->resolve(sizes); +} + void RequestClient::request_finished(i32 request_id, u64 total_size, RequestTimingInfo timing_info, Optional network_error) { RefPtr request; diff --git a/Libraries/LibRequests/RequestClient.h b/Libraries/LibRequests/RequestClient.h index 2d338f3f73..e57b10ee70 100644 --- a/Libraries/LibRequests/RequestClient.h +++ b/Libraries/LibRequests/RequestClient.h @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -39,6 +40,8 @@ public: bool stop_request(Badge, Request&); bool set_certificate(Badge, Request&, ByteString, ByteString); + NonnullRefPtr> estimate_cache_size_accessed_since(UnixDateTime since); + Function on_request_server_died; private: @@ -57,10 +60,15 @@ private: virtual void websocket_subprotocol(i64 websocket_id, ByteString subprotocol) override; virtual void websocket_certificate_requested(i64 websocket_id) override; - HashMap> m_requests; - HashMap> m_websockets; + virtual void estimated_cache_size(u64 cache_size_estimation_id, CacheSizes sizes) override; + HashMap> m_requests; + + HashMap> m_websockets; i64 m_next_websocket_id { 0 }; + + HashMap>> m_pending_cache_size_estimations; + u64 m_next_cache_size_estimation_id { 0 }; }; } diff --git a/Services/RequestServer/CMakeLists.txt b/Services/RequestServer/CMakeLists.txt index afabf89a72..9967917b1b 100644 --- a/Services/RequestServer/CMakeLists.txt +++ b/Services/RequestServer/CMakeLists.txt @@ -41,7 +41,7 @@ target_include_directories(requestserverservice PRIVATE ${CMAKE_CURRENT_BINARY_D target_include_directories(requestserverservice PRIVATE ${LADYBIRD_SOURCE_DIR}/Services/) target_link_libraries(RequestServer PRIVATE requestserverservice) -target_link_libraries(requestserverservice PUBLIC LibCore LibDatabase LibDNS LibCrypto LibFileSystem LibIPC LibMain LibTLS LibWebSocket LibURL LibTextCodec LibThreading CURL::libcurl) +target_link_libraries(requestserverservice PUBLIC LibCore LibDatabase LibDNS LibCrypto LibFileSystem LibIPC LibMain LibRequests LibTLS LibWebSocket LibURL LibTextCodec LibThreading CURL::libcurl) target_link_libraries(requestserverservice PRIVATE OpenSSL::Crypto OpenSSL::SSL) if (WIN32) diff --git a/Services/RequestServer/Cache/CacheIndex.cpp b/Services/RequestServer/Cache/CacheIndex.cpp index bd1639d015..feb7e21042 100644 --- a/Services/RequestServer/Cache/CacheIndex.cpp +++ b/Services/RequestServer/Cache/CacheIndex.cpp @@ -101,6 +101,7 @@ ErrorOr CacheIndex::create(Database::Database& database) statements.select_entry = TRY(database.prepare_statement("SELECT * FROM CacheIndex WHERE cache_key = ?;"sv)); statements.update_response_headers = TRY(database.prepare_statement("UPDATE CacheIndex SET response_headers = ? WHERE cache_key = ?;"sv)); statements.update_last_access_time = TRY(database.prepare_statement("UPDATE CacheIndex SET last_access_time = ? WHERE cache_key = ?;"sv)); + statements.estimate_cache_size_accessed_since = TRY(database.prepare_statement("SELECT SUM(data_size) + SUM(OCTET_LENGTH(response_headers)) FROM CacheIndex WHERE last_access_time >= ?;"sv)); return CacheIndex { database, statements }; } @@ -188,4 +189,21 @@ Optional CacheIndex::find_entry(u64 cache_key) return m_entries.get(cache_key); } +Requests::CacheSizes CacheIndex::estimate_cache_size_accessed_since(UnixDateTime since) const +{ + Requests::CacheSizes sizes; + + m_database.execute_statement( + m_statements.estimate_cache_size_accessed_since, + [&](auto statement_id) { sizes.since_requested_time = m_database.result_column(statement_id, 0); }, + since); + + m_database.execute_statement( + m_statements.estimate_cache_size_accessed_since, + [&](auto statement_id) { sizes.total = m_database.result_column(statement_id, 0); }, + UnixDateTime::earliest()); + + return sizes; +} + } diff --git a/Services/RequestServer/Cache/CacheIndex.h b/Services/RequestServer/Cache/CacheIndex.h index 58fcd20bbd..571625dbd5 100644 --- a/Services/RequestServer/Cache/CacheIndex.h +++ b/Services/RequestServer/Cache/CacheIndex.h @@ -12,6 +12,7 @@ #include #include #include +#include namespace RequestServer { @@ -42,6 +43,8 @@ public: void update_response_headers(u64 cache_key, HTTP::HeaderMap); void update_last_access_time(u64 cache_key); + Requests::CacheSizes estimate_cache_size_accessed_since(UnixDateTime since) const; + private: struct Statements { Database::StatementID insert_entry { 0 }; @@ -50,6 +53,7 @@ private: Database::StatementID select_entry { 0 }; Database::StatementID update_response_headers { 0 }; Database::StatementID update_last_access_time { 0 }; + Database::StatementID estimate_cache_size_accessed_since { 0 }; }; CacheIndex(Database::Database&, Statements); diff --git a/Services/RequestServer/Cache/DiskCache.cpp b/Services/RequestServer/Cache/DiskCache.cpp index dae16d55de..384cf1ffaf 100644 --- a/Services/RequestServer/Cache/DiskCache.cpp +++ b/Services/RequestServer/Cache/DiskCache.cpp @@ -139,6 +139,11 @@ bool DiskCache::check_if_cache_has_open_entry(Request& request, u64 cache_key, C return false; } +Requests::CacheSizes DiskCache::estimate_cache_size_accessed_since(UnixDateTime since) const +{ + return m_index.estimate_cache_size_accessed_since(since); +} + void DiskCache::clear_cache() { for (auto const& [_, open_entries] : m_open_cache_entries) { diff --git a/Services/RequestServer/Cache/DiskCache.h b/Services/RequestServer/Cache/DiskCache.h index 0c361e9316..55bc0fcbab 100644 --- a/Services/RequestServer/Cache/DiskCache.h +++ b/Services/RequestServer/Cache/DiskCache.h @@ -27,6 +27,7 @@ public: Variant, CacheHasOpenEntry> create_entry(Request&); Variant, CacheHasOpenEntry> open_entry(Request&); + Requests::CacheSizes estimate_cache_size_accessed_since(UnixDateTime since) const; void clear_cache(); LexicalPath const& cache_directory() { return m_cache_directory; } diff --git a/Services/RequestServer/ConnectionFromClient.cpp b/Services/RequestServer/ConnectionFromClient.cpp index 9422839dc9..9b21d497ec 100644 --- a/Services/RequestServer/ConnectionFromClient.cpp +++ b/Services/RequestServer/ConnectionFromClient.cpp @@ -302,6 +302,16 @@ void ConnectionFromClient::ensure_connection(URL::URL url, ::RequestServer::Cach m_active_requests.set(connect_only_request_id, move(request)); } +void ConnectionFromClient::estimate_cache_size_accessed_since(u64 cache_size_estimation_id, UnixDateTime since) +{ + Requests::CacheSizes sizes; + + if (g_disk_cache.has_value()) + sizes = g_disk_cache->estimate_cache_size_accessed_since(since); + + async_estimated_cache_size(cache_size_estimation_id, sizes); +} + void ConnectionFromClient::clear_cache() { if (g_disk_cache.has_value()) diff --git a/Services/RequestServer/ConnectionFromClient.h b/Services/RequestServer/ConnectionFromClient.h index 8c11b5e1f3..048f30dbc9 100644 --- a/Services/RequestServer/ConnectionFromClient.h +++ b/Services/RequestServer/ConnectionFromClient.h @@ -42,6 +42,7 @@ private: virtual Messages::RequestServer::SetCertificateResponse set_certificate(i32, ByteString, ByteString) override; virtual void ensure_connection(URL::URL url, ::RequestServer::CacheLevel cache_level) override; + virtual void estimate_cache_size_accessed_since(u64 cache_size_estimation_id, UnixDateTime since) override; virtual void clear_cache() override; virtual void websocket_connect(i64 websocket_id, URL::URL, ByteString, Vector, Vector, HTTP::HeaderMap) override; diff --git a/Services/RequestServer/RequestClient.ipc b/Services/RequestServer/RequestClient.ipc index 12d39bd055..0661887efd 100644 --- a/Services/RequestServer/RequestClient.ipc +++ b/Services/RequestServer/RequestClient.ipc @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -21,4 +22,6 @@ endpoint RequestClient // Certificate requests certificate_requested(i32 request_id) =| + + estimated_cache_size(u64 cache_size_estimation_id, Requests::CacheSizes sizes) =| } diff --git a/Services/RequestServer/RequestServer.ipc b/Services/RequestServer/RequestServer.ipc index 848efc91a9..0148fb626b 100644 --- a/Services/RequestServer/RequestServer.ipc +++ b/Services/RequestServer/RequestServer.ipc @@ -22,6 +22,7 @@ endpoint RequestServer ensure_connection(URL::URL url, ::RequestServer::CacheLevel cache_level) =| + estimate_cache_size_accessed_since(u64 cache_size_estimation_id, UnixDateTime since) =| clear_cache() =| // Websocket Connection API