LibHTTP+LibWebView+RequestServer: Allow using the disk cache during WPT

We currently disable the disk cache because the WPT runner will run more
than one RequestServer process at a time. The SQLite database does not
handle this concurrent read/write access well.

We will now enable the disk cache with a per-process database. This is
needed to ensure that WPT Fetch cache tests are sufficiently handled by
RequestServer.

Libraries/LibHTTP/Cache/DiskCache.cpp

@@ -7,6 +7,7 @@
 #include <AK/Debug.h>
 #include <LibCore/EventLoop.h>
 #include <LibCore/StandardPaths.h>
+#include <LibCore/System.h>
 #include <LibFileSystem/FileSystem.h>
 #include <LibHTTP/Cache/CacheRequest.h>
 #include <LibHTTP/Cache/DiskCache.h>
@@ -17,10 +18,25 @@ namespace HTTP {
 
 static constexpr auto INDEX_DATABASE = "INDEX"sv;
 
+static ByteString cache_directory_for_mode(DiskCache::Mode mode)
+{
+    switch (mode) {
+    case DiskCache::Mode::Normal:
+        return "Cache"sv;
+    case DiskCache::Mode::Partitioned:
+        // FIXME: Ideally, we could support multiple RequestServer processes using the same database by enabling the
+        //        WAL and setting a reasonable busy timeout. We would also have to prevent multiple processes writing
+        //        to the same cache entry file at the same time with some locking mechanism.
+        return ByteString::formatted("PartitionedCache-{}", Core::System::getpid());
+    case DiskCache::Mode::Testing:
+        return "TestCache"sv;
+    }
+    VERIFY_NOT_REACHED();
+}
+
 ErrorOr<DiskCache> DiskCache::create(Mode mode)
 {
-    auto cache_name = mode == Mode::Normal ? "Cache"sv : "TestCache"sv;
-    auto cache_directory = LexicalPath::join(Core::StandardPaths::cache_directory(), "Ladybird"sv, cache_name);
+    auto cache_directory = LexicalPath::join(Core::StandardPaths::cache_directory(), "Ladybird"sv, cache_directory_for_mode(mode));
 
     auto database = TRY(Database::Database::create(cache_directory.string(), INDEX_DATABASE));
     auto index = TRY(CacheIndex::create(database));
@@ -34,15 +50,23 @@ DiskCache::DiskCache(Mode mode, NonnullRefPtr<Database::Database> database, Lexi
     , m_cache_directory(move(cache_directory))
     , m_index(move(index))
 {
-    // Start with a clean slate in test mode.
-    if (m_mode == Mode::Testing)
+    // Start with a clean slate in non-normal modes.
+    if (m_mode != Mode::Normal)
         remove_entries_accessed_since(UnixDateTime::earliest());
 }
 
 DiskCache::DiskCache(DiskCache&&) = default;
 DiskCache& DiskCache::operator=(DiskCache&&) = default;
 
-DiskCache::~DiskCache() = default;
+DiskCache::~DiskCache()
+{
+    if (m_mode != Mode::Partitioned)
+        return;
+
+    // Clean up partitioned cache directories to prevent endless growth of disk usage.
+    if (auto const& cache_directory = m_cache_directory.string(); !cache_directory.is_empty())
+        (void)FileSystem::remove(cache_directory, FileSystem::RecursionMode::Allowed);
+}
 
 Variant<Optional<CacheEntryWriter&>, DiskCache::CacheHasOpenEntry> DiskCache::create_entry(CacheRequest& request, URL::URL const& url, StringView method, HeaderList const& request_headers, UnixDateTime request_start_time)
 {

Libraries/LibHTTP/Cache/DiskCache.h

@@ -25,6 +25,10 @@ public:
     enum class Mode {
         Normal,
 
+        // In partitioned mode, the cache is enabled as normal, but each RequestServer process operates with a unique
+        // disk cache database.
+        Partitioned,
+
         // In test mode, we only enable caching of responses on a per-request basis, signified by a request header. The
         // response headers will include some status on how the request was handled.
         Testing,

Libraries/LibWebView/Application.cpp

@@ -216,10 +216,8 @@ ErrorOr<void> Application::initialize(Main::Arguments const& arguments)
 
     // Our persisted SQL storage assumes it runs in a singleton process. If we have multiple UI processes accessing
     // the same underlying database, one of them is likely to fail.
-    if (force_new_process) {
+    if (force_new_process)
         disable_sql_database = true;
-        disable_http_disk_cache = true;
-    }
 
     if (!dns_server_port.has_value())
         dns_server_port = use_dns_over_tls ? 853 : 53;
@@ -264,9 +262,15 @@ ErrorOr<void> Application::initialize(Main::Arguments const& arguments)
     if (webdriver_content_ipc_path.has_value())
         m_browser_options.webdriver_content_ipc_path = *webdriver_content_ipc_path;
 
+    auto http_disk_cache_mode = HTTPDiskCacheMode::Enabled;
+    if (disable_http_disk_cache)
+        http_disk_cache_mode = HTTPDiskCacheMode::Disabled;
+    else if (force_new_process)
+        http_disk_cache_mode = HTTPDiskCacheMode::Partitioned;
+
     m_request_server_options = {
         .certificates = move(certificates),
-        .http_disk_cache_mode = disable_http_disk_cache ? HTTPDiskCacheMode::Disabled : HTTPDiskCacheMode::Enabled,
+        .http_disk_cache_mode = http_disk_cache_mode,
         .resource_substitution_map_path = resource_substitution_map_path.has_value() ? Optional<ByteString> { *resource_substitution_map_path } : OptionalNone {},
     };
 

Libraries/LibWebView/HelperProcess.cpp

@@ -224,6 +224,9 @@ ErrorOr<NonnullRefPtr<Requests::RequestClient>> launch_request_server_process()
     case HTTPDiskCacheMode::Enabled:
         arguments.append("enabled"sv);
         break;
+    case HTTPDiskCacheMode::Partitioned:
+        arguments.append("partitioned"sv);
+        break;
     case HTTPDiskCacheMode::Testing:
         arguments.append("testing"sv);
         break;

Libraries/LibWebView/Options.h

@@ -96,6 +96,7 @@ struct BrowserOptions {
 enum class HTTPDiskCacheMode {
     Disabled,
     Enabled,
+    Partitioned,
     Testing,
 };
 

Services/RequestServer/main.cpp

@@ -12,6 +12,7 @@
 #include <LibCore/ArgsParser.h>
 #include <LibCore/EventLoop.h>
 #include <LibCore/Process.h>
+#include <LibCore/System.h>
 #include <LibHTTP/Cache/DiskCache.h>
 #include <LibIPC/SingleServer.h>
 #include <LibMain/Main.h>
@@ -30,6 +31,12 @@ OwnPtr<ResourceSubstitutionMap> g_resource_substitution_map;
 
 }
 
+static void handle_signal(int signal)
+{
+    VERIFY(signal == SIGINT || signal == SIGTERM);
+    Core::EventLoop::current().quit(0);
+}
+
 ErrorOr<int> ladybird_main(Main::Arguments arguments)
 {
     AK::set_rich_debug_enabled(true);
@@ -64,16 +71,24 @@ ErrorOr<int> ladybird_main(Main::Arguments arguments)
     }
 
     Core::EventLoop event_loop;
+    Core::EventLoop::register_signal(SIGINT, handle_signal);
+    Core::EventLoop::register_signal(SIGTERM, handle_signal);
 
 #if defined(AK_OS_MACOS)
     if (!mach_server_name.is_empty())
         Core::Platform::register_with_mach_server(mach_server_name);
 #endif
 
-    if (http_disk_cache_mode.is_one_of("enabled"sv, "testing"sv)) {
-        auto mode = http_disk_cache_mode == "enabled"sv
-            ? HTTP::DiskCache::Mode::Normal
-            : HTTP::DiskCache::Mode::Testing;
+    if (http_disk_cache_mode != "disabled"sv) {
+        auto mode = TRY([&]() -> ErrorOr<HTTP::DiskCache::Mode> {
+            if (http_disk_cache_mode == "enabled"sv)
+                return HTTP::DiskCache::Mode::Normal;
+            if (http_disk_cache_mode == "partitioned"sv)
+                return HTTP::DiskCache::Mode::Partitioned;
+            if (http_disk_cache_mode == "testing"sv)
+                return HTTP::DiskCache::Mode::Testing;
+            return Error::from_string_literal("Unrecognized disk cache mode");
+        }());
 
         if (auto cache = HTTP::DiskCache::create(mode); cache.is_error())
             warnln("Unable to create disk cache: {}", cache.error());