RequestServer: Log orphaned cache files when HTTP_DISK_CACHE_DEBUG is on

When enabled, this will log all cache entry files which no longer appear
in the cache index.

Libraries/LibHTTP/Cache/CacheIndex.cpp

@@ -6,6 +6,7 @@
 
 #include <AK/Debug.h>
 #include <AK/StringBuilder.h>
+#include <LibCore/Directory.h>
 #include <LibFileSystem/FileSystem.h>
 #include <LibHTTP/Cache/CacheIndex.h>
 #include <LibHTTP/Cache/Utilities.h>
@@ -49,6 +50,54 @@ static NonnullRefPtr<HeaderList> deserialize_headers(StringView serialized_heade
     return headers;
 }
 
+#if HTTP_DISK_CACHE_DEBUG
+
+template<typename Callback>
+static void for_each_cache_entry_file(Database::Database& database, Callback&& callback)
+{
+    auto const& index_path = database.database_path();
+    if (!index_path.has_value())
+        return;
+
+    (void)Core::Directory::for_each_entry(
+        index_path->parent().string(),
+        static_cast<Core::DirIterator::Flags>(Core::DirIterator::SkipDots | Core::DirIterator::NoStat),
+        [&](Core::DirectoryEntry const& entry, Core::Directory const& parent) -> ErrorOr<IterationDecision> {
+            if (entry.type != Core::DirectoryEntry::Type::File)
+                return IterationDecision::Continue;
+
+            // Ignore INDEX.db and related files (e.g. the WAL file, INDEX.db-wal).
+            if (entry.name.starts_with(index_path->basename()))
+                return IterationDecision::Continue;
+
+            callback(parent.path().append(entry.name));
+            return IterationDecision::Continue;
+        });
+}
+
+static void log_orphaned_disk_cache_entries(Database::Database& database)
+{
+    auto check_entry = MUST(database.prepare_statement(R"#(
+        SELECT 1 FROM CacheIndex WHERE cache_key = ? AND vary_key = ? LIMIT 1;
+    )#"sv));
+
+    for_each_cache_entry_file(database, [&](LexicalPath const& cache_entry) {
+        auto cache_entry_data = cache_entry_data_for_file(cache_entry);
+        if (!cache_entry_data.has_value()) {
+            dbgln("Unrecognized cache file: {}", cache_entry);
+            return;
+        }
+
+        bool has_entry = false;
+        database.execute_statement(check_entry, [&](auto) { has_entry = true; }, cache_entry_data->cache_key, cache_entry_data->vary_key);
+
+        if (!has_entry)
+            dbgln("Cache file missing from cache index: {}", cache_entry);
+    });
+}
+
+#endif
+
 ErrorOr<CacheIndex> CacheIndex::create(Database::Database& database, LexicalPath const& cache_directory)
 {
     auto create_cache_metadata_table = TRY(database.prepare_statement(R"#(
@@ -97,6 +146,10 @@ ErrorOr<CacheIndex> CacheIndex::create(Database::Database& database, LexicalPath
     )#"sv));
     database.execute_statement(create_cache_index_table, {});
 
+#if HTTP_DISK_CACHE_DEBUG
+    log_orphaned_disk_cache_entries(database);
+#endif
+
     Statements statements {};
     statements.insert_entry = TRY(database.prepare_statement("INSERT OR REPLACE INTO CacheIndex VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?);"sv));
     statements.remove_entry = TRY(database.prepare_statement(R"#(

Libraries/LibHTTP/Cache/Utilities.cpp

@@ -7,6 +7,7 @@
 #include <AK/GenericLexer.h>
 #include <AK/QuickSort.h>
 #include <AK/StringBuilder.h>
+#include <AK/StringConversions.h>
 #include <LibCrypto/Hash/SHA1.h>
 #include <LibHTTP/Cache/DiskCache.h>
 #include <LibHTTP/Cache/Utilities.h>
@@ -111,7 +112,7 @@ LexicalPath path_for_cache_entry(LexicalPath const& cache_directory, u64 cache_k
     return cache_directory.append(file);
 }
 
-static StringView cache_entry_associated_data_suffix(CacheEntryAssociatedData associated_data)
+static constexpr StringView cache_entry_associated_data_suffix(CacheEntryAssociatedData associated_data)
 {
     switch (associated_data) {
     case CacheEntryAssociatedData::JavaScriptBytecode:
@@ -120,6 +121,13 @@ static StringView cache_entry_associated_data_suffix(CacheEntryAssociatedData as
     VERIFY_NOT_REACHED();
 }
 
+static constexpr Optional<CacheEntryAssociatedData> cache_entry_associated_data_from_suffix(StringView suffix)
+{
+    if (suffix == "jsbc"sv)
+        return CacheEntryAssociatedData::JavaScriptBytecode;
+    return {};
+}
+
 LexicalPath path_for_cache_entry_associated_data(LexicalPath const& cache_directory, u64 cache_key, u64 vary_key, CacheEntryAssociatedData associated_data)
 {
     auto file = vary_key == 0
@@ -129,6 +137,39 @@ LexicalPath path_for_cache_entry_associated_data(LexicalPath const& cache_direct
     return cache_directory.append(file);
 }
 
+Optional<CacheEntryData> cache_entry_data_for_file(LexicalPath const& cache_file)
+{
+    CacheEntryData result;
+
+    if (auto file_name = cache_file.basename(LexicalPath::StripExtension::Yes); file_name.contains('_')) {
+        auto parts = file_name.split_view('_', SplitBehavior::KeepEmpty);
+        if (parts.size() != 2)
+            return {};
+
+        auto cache_key = AK::parse_number<u64>(parts[0], TrimWhitespace::No, 16);
+        auto vary_key = AK::parse_number<u64>(parts[1], TrimWhitespace::No, 16);
+        if (!cache_key.has_value() || !vary_key.has_value())
+            return {};
+
+        result.cache_key = *cache_key;
+        result.vary_key = *vary_key;
+    } else {
+        auto cache_key = AK::parse_number<u64>(file_name, TrimWhitespace::No, 16);
+        if (!cache_key.has_value())
+            return {};
+
+        result.cache_key = *cache_key;
+    }
+
+    if (auto extension = cache_file.extension(); !extension.is_empty()) {
+        result.associated_data = cache_entry_associated_data_from_suffix(extension);
+        if (!result.associated_data.has_value())
+            return {};
+    }
+
+    return result;
+}
+
 // https://httpwg.org/specs/rfc9111.html#response.cacheability
 bool is_cacheable(StringView method, HTTP::HeaderList const& request_headers)
 {

Libraries/LibHTTP/Cache/Utilities.h

@@ -38,6 +38,13 @@ u64 create_vary_key(HeaderList const& request_headers, HeaderList const& respons
 LexicalPath path_for_cache_entry(LexicalPath const& cache_directory, u64 cache_key, u64 vary_key);
 LexicalPath path_for_cache_entry_associated_data(LexicalPath const& cache_directory, u64 cache_key, u64 vary_key, CacheEntryAssociatedData);
 
+struct CacheEntryData {
+    u64 cache_key { 0 };
+    u64 vary_key { 0 };
+    Optional<CacheEntryAssociatedData> associated_data;
+};
+Optional<CacheEntryData> cache_entry_data_for_file(LexicalPath const&);
+
 bool is_cacheable(StringView method, HeaderList const&);
 bool is_cacheable(u32 status_code, HeaderList const&);
 bool is_header_exempted_from_storage(StringView name);