LibWeb: Support MIME type sniffing for streaming HTTP responses

Previously, when loading a document, we would try to sniff the MIME
type by reading from the response body's source. However, for streaming
HTTP responses, the body source is Empty (the data comes through the
stream instead), so we had no bytes to sniff.

This caused pages like hypr.land (which sends no Content-Type header)
to be misidentified as plain text instead of HTML, since the MIME
sniffing algorithm would receive zero bytes and fall back to the
default type.

The fix captures the first bytes of the response body during fetch,
storing them on the Body object. These bytes are the "resource header"
defined by the MIME Sniffing spec - up to 1445 bytes, which is enough
to identify any MIME type the spec can detect.

Since bytes may arrive asynchronously during streaming, we use a
callback mechanism: if bytes aren't ready yet when load_document()
needs them, it registers a callback that fires once enough bytes have
been captured (or the stream ends).

The flow is:
1. FetchedDataReceiver receives network bytes, buffers them
2. When Body is created, buffered bytes are flushed to Body's sniff
   buffer, and subsequent bytes are appended as they arrive
3. Before calling load_document(), Navigable waits for sniff bytes
4. load_document() passes the bytes to MimeSniff::Resource::sniff()

Libraries/LibWeb/DOM/DocumentLoading.cpp

@@ -412,7 +412,7 @@ bool can_load_document_with_type(MimeSniff::MimeType const& type)
 }
 
 // https://html.spec.whatwg.org/multipage/browsing-the-web.html#loading-a-document
-GC::Ptr<DOM::Document> load_document(HTML::NavigationParams const& navigation_params, NonnullRefPtr<Core::Promise<Empty>> signal_to_continue_session_history_processing)
+GC::Ptr<DOM::Document> load_document(HTML::NavigationParams const& navigation_params, NonnullRefPtr<Core::Promise<Empty>> signal_to_continue_session_history_processing, ReadonlyBytes sniff_bytes)
 {
     // To load a document given navigation params navigationParams, source snapshot params sourceSnapshotParams,
     // and origin initiatorOrigin, perform the following steps. They return a Document or null.
@@ -422,10 +422,7 @@ GC::Ptr<DOM::Document> load_document(HTML::NavigationParams const& navigation_pa
     // 1. Let type be the computed type of navigationParams's response.
     auto supplied_type = Fetch::Infrastructure::extract_mime_type(navigation_params.response->header_list());
     auto type = MimeSniff::Resource::sniff(
-        navigation_params.response->body()->source().visit(
-            [](Empty) { return ReadonlyBytes {}; },
-            [](ByteBuffer const& buffer) { return ReadonlyBytes { buffer }; },
-            [](GC::Root<FileAPI::Blob> const& blob) { return blob->raw_bytes(); }),
+        sniff_bytes,
         MimeSniff::SniffingConfiguration {
             .sniffing_context = MimeSniff::SniffingContext::Browsing,
             .supplied_type = move(supplied_type) });

Libraries/LibWeb/DOM/DocumentLoading.h

@@ -8,14 +8,13 @@
 #pragma once
 
 #include <LibWeb/DOM/Document.h>
-#include <LibWeb/Fetch/Infrastructure/HTTP/Responses.h>
 #include <LibWeb/HTML/Navigable.h>
 #include <LibWeb/HTML/UserNavigationInvolvement.h>
 
 namespace Web {
 
 bool build_xml_document(DOM::Document& document, ByteBuffer const& data, Optional<String> content_encoding);
-GC::Ptr<DOM::Document> load_document(HTML::NavigationParams const& navigation_params, NonnullRefPtr<Core::Promise<Empty>> signal_to_continue_session_history_processing);
+GC::Ptr<DOM::Document> load_document(HTML::NavigationParams const& navigation_params, NonnullRefPtr<Core::Promise<Empty>> signal_to_continue_session_history_processing, ReadonlyBytes sniff_bytes);
 bool can_load_document_with_type(MimeSniff::MimeType const&);
 
 // https://html.spec.whatwg.org/multipage/document-lifecycle.html#read-ua-inline

Libraries/LibWeb/Fetch/Fetching/FetchedDataReceiver.cpp

@@ -10,6 +10,7 @@
 #include <LibWeb/Bindings/ExceptionOrUtils.h>
 #include <LibWeb/Fetch/Fetching/FetchedDataReceiver.h>
 #include <LibWeb/Fetch/Infrastructure/FetchParams.h>
+#include <LibWeb/Fetch/Infrastructure/HTTP/Bodies.h>
 #include <LibWeb/Fetch/Infrastructure/HTTP/Responses.h>
 #include <LibWeb/Fetch/Infrastructure/Task.h>
 #include <LibWeb/HTML/Scripting/ExceptionReporter.h>
@@ -30,11 +31,20 @@ FetchedDataReceiver::FetchedDataReceiver(GC::Ref<Infrastructure::FetchParams con
 
 FetchedDataReceiver::~FetchedDataReceiver() = default;
 
+void FetchedDataReceiver::set_body(GC::Ref<Fetch::Infrastructure::Body> body)
+{
+    m_body = body;
+    // Flush any bytes that were buffered before the body was set
+    if (!m_buffer.is_empty())
+        m_body->append_sniff_bytes(m_buffer);
+}
+
 void FetchedDataReceiver::visit_edges(Visitor& visitor)
 {
     Base::visit_edges(visitor);
     visitor.visit(m_fetch_params);
     visitor.visit(m_response);
+    visitor.visit(m_body);
     visitor.visit(m_stream);
     visitor.visit(m_pending_promise);
 }
@@ -61,10 +71,17 @@ void FetchedDataReceiver::handle_network_bytes(ReadonlyBytes bytes, NetworkState
     if (state == NetworkState::Complete) {
         VERIFY(bytes.is_empty());
         m_lifecycle_state = LifecycleState::CompletePending;
+        // Mark sniff bytes as complete when the stream ends
+        if (m_body)
+            m_body->set_sniff_bytes_complete();
     }
 
-    if (state == NetworkState::Ongoing)
+    if (state == NetworkState::Ongoing) {
         m_buffer.append(bytes);
+        // Capture bytes for MIME sniffing
+        if (m_body)
+            m_body->append_sniff_bytes(bytes);
+    }
 
     if (!m_pending_promise) {
         if (m_lifecycle_state == LifecycleState::CompletePending && buffer_is_eof() && !m_has_unfulfilled_promise)

Libraries/LibWeb/Fetch/Fetching/FetchedDataReceiver.h

@@ -25,6 +25,7 @@ public:
     void set_pending_promise(GC::Ref<WebIDL::Promise>);
 
     void set_response(GC::Ref<Fetch::Infrastructure::Response const> response) { m_response = response; }
+    void set_body(GC::Ref<Fetch::Infrastructure::Body> body);
 
     enum class NetworkState {
         Ongoing,
@@ -46,6 +47,7 @@ private:
 
     GC::Ref<Infrastructure::FetchParams const> m_fetch_params;
     GC::Ptr<Fetch::Infrastructure::Response const> m_response;
+    GC::Ptr<Fetch::Infrastructure::Body> m_body;
 
     GC::Ref<Streams::ReadableStream> m_stream;
     GC::Ptr<WebIDL::Promise> m_pending_promise;

Libraries/LibWeb/Fetch/Fetching/Fetching.cpp

@@ -2131,7 +2131,9 @@ GC::Ref<PendingResponse> nonstandard_resource_loader_file_or_http_network_fetch(
         fetched_data_receiver->set_response(response);
 
         // 14. Set response’s body to a new body whose stream is stream.
-        response->set_body(Infrastructure::Body::create(vm, stream));
+        auto body = Infrastructure::Body::create(vm, stream);
+        response->set_body(body);
+        fetched_data_receiver->set_body(body);
 
         // 17. Return response.
         // NOTE: Typically response’s body’s stream is still being enqueued to after returning.

Libraries/LibWeb/Fetch/Infrastructure/HTTP/Bodies.cpp

@@ -19,6 +19,12 @@ namespace Web::Fetch::Infrastructure {
 
 GC_DEFINE_ALLOCATOR(Body);
 
+// https://mimesniff.spec.whatwg.org/#reading-the-resource-header
+// To read the resource header, a user agent MUST read bytes of the resource until one of the following conditions is met:
+// - the end of the resource is reached
+// - 1445 or more bytes have been read
+static constexpr size_t MAX_SNIFF_BYTES = 1445;
+
 GC::Ref<Body> Body::create(JS::VM& vm, GC::Ref<Streams::ReadableStream> stream)
 {
     return vm.heap().allocate<Body>(stream);
@@ -45,6 +51,68 @@ void Body::visit_edges(Cell::Visitor& visitor)
 {
     Base::visit_edges(visitor);
     visitor.visit(m_stream);
+    visitor.visit(m_sniff_bytes_callback);
+}
+
+void Body::append_sniff_bytes(ReadonlyBytes bytes)
+{
+    if (m_sniff_bytes_complete)
+        return;
+
+    size_t space_remaining = MAX_SNIFF_BYTES - m_sniff_bytes.size();
+    if (space_remaining == 0) {
+        set_sniff_bytes_complete();
+        return;
+    }
+
+    size_t to_append = min(bytes.size(), space_remaining);
+    m_sniff_bytes.append(bytes.slice(0, to_append));
+
+    if (m_sniff_bytes.size() >= MAX_SNIFF_BYTES)
+        set_sniff_bytes_complete();
+}
+
+void Body::set_sniff_bytes_complete()
+{
+    if (m_sniff_bytes_complete)
+        return;
+    m_sniff_bytes_complete = true;
+    if (m_sniff_bytes_callback) {
+        auto callback = exchange(m_sniff_bytes_callback, nullptr);
+        callback->function()(m_sniff_bytes);
+    }
+}
+
+Optional<ReadonlyBytes> Body::sniff_bytes_if_available() const
+{
+    // Non-streaming body: source has bytes
+    if (m_source.has<ByteBuffer>()) {
+        auto const& buffer = m_source.get<ByteBuffer>();
+        return buffer.bytes().slice(0, min(buffer.size(), MAX_SNIFF_BYTES));
+    }
+
+    if (m_source.has<GC::Root<FileAPI::Blob>>()) {
+        auto raw = m_source.get<GC::Root<FileAPI::Blob>>()->raw_bytes();
+        return raw.slice(0, min(raw.size(), MAX_SNIFF_BYTES));
+    }
+
+    // Streaming body: bytes captured during fetch
+    if (m_sniff_bytes_complete)
+        return m_sniff_bytes;
+
+    // Still waiting for bytes
+    return {};
+}
+
+void Body::wait_for_sniff_bytes(SniffBytesCallback on_ready)
+{
+    if (auto bytes = sniff_bytes_if_available(); bytes.has_value()) {
+        on_ready->function()(bytes.value());
+        return;
+    }
+
+    // Wait for bytes to arrive
+    m_sniff_bytes_callback = on_ready;
 }
 
 // https://fetch.spec.whatwg.org/#concept-body-clone

Libraries/LibWeb/Fetch/Infrastructure/HTTP/Bodies.h

@@ -45,6 +45,20 @@ public:
     [[nodiscard]] SourceType const& source() const { return m_source; }
     [[nodiscard]] Optional<u64> const& length() const { return m_length; }
 
+    // https://mimesniff.spec.whatwg.org/#reading-the-resource-header
+    // Non-standard infrastructure to obtain the "resource header" for MIME type sniffing.
+    // The spec defines resource header as the byte sequence to sniff, obtained by reading
+    // "until [...] 1445 or more bytes have been read" or end of resource is reached.
+    // For non-streaming bodies (ByteBuffer/Blob source), bytes are available immediately.
+    // For streaming bodies, bytes are captured during fetch and delivered via callback.
+    using SniffBytesCallback = GC::Ref<GC::Function<void(ReadonlyBytes)>>;
+    Optional<ReadonlyBytes> sniff_bytes_if_available() const;
+    void wait_for_sniff_bytes(SniffBytesCallback on_ready);
+
+    // Called by FetchedDataReceiver to provide sniff bytes during streaming fetch.
+    void append_sniff_bytes(ReadonlyBytes bytes);
+    void set_sniff_bytes_complete();
+
     [[nodiscard]] GC::Ref<Body> clone(JS::Realm&);
 
     void fully_read(JS::Realm&, ProcessBodyCallback process_body, ProcessBodyErrorCallback process_body_error, TaskDestination) const;
@@ -68,6 +82,12 @@ private:
     // https://fetch.spec.whatwg.org/#concept-body-total-bytes
     // A length (null or an integer), initially null.
     Optional<u64> m_length;
+
+    // https://mimesniff.spec.whatwg.org/#reading-the-resource-header
+    // Non-standard: Captured "resource header" bytes for MIME type sniffing.
+    ByteBuffer m_sniff_bytes;
+    bool m_sniff_bytes_complete { false };
+    GC::Ptr<GC::Function<void(ReadonlyBytes)>> m_sniff_bytes_callback;
 };
 
 // https://fetch.spec.whatwg.org/#body-with-type

Libraries/LibWeb/HTML/Navigable.cpp

@@ -1326,6 +1326,52 @@ static void create_navigation_params_by_fetching(GC::Ptr<SessionHistoryEntry> en
     }));
 }
 
+// Helper for populate_session_history_entry_document: runs steps 7 and 8
+static void finalize_session_history_entry(
+    GC::Ptr<SessionHistoryEntry> entry,
+    Navigable::NavigationParamsVariant const& received_navigation_params,
+    bool saveExtraDocumentState,
+    GC::Ptr<GC::Function<void()>> completion_steps)
+{
+    // 7. If entry's document state's document is not null, then:
+    if (entry->document()) {
+        // 1. Set entry's document state's ever populated to true.
+        entry->document_state()->set_ever_populated(true);
+
+        // 2. If saveExtraDocumentState is true:
+        if (saveExtraDocumentState) {
+            // 1. Let document be entry's document state's document.
+            auto document = entry->document();
+
+            // 2. Set entry's document state's origin to document's origin.
+            entry->document_state()->set_origin(document->origin());
+
+            // 3. If document's URL requires storing the policy container in history, then:
+            if (url_requires_storing_the_policy_container_in_history(document->url())) {
+                // 1. Assert: navigationParams is a navigation params (i.e., neither null nor a non-fetch scheme navigation params).
+                VERIFY(received_navigation_params.has<GC::Ref<NavigationParams>>());
+
+                // 2. Set entry's document state's history policy container to navigationParams's policy container.
+                entry->document_state()->set_history_policy_container(GC::Ref { *received_navigation_params.get<GC::Ref<NavigationParams>>()->policy_container });
+            }
+        }
+
+        // 3. If entry's document state's request referrer is "client", and navigationParams is a navigation params (i.e., neither null nor a non-fetch scheme navigation params), then:
+        if (entry->document_state()->request_referrer() == Fetch::Infrastructure::Request::Referrer::Client
+            && (!received_navigation_params.has<Navigable::NullOrError>() && received_navigation_params.has<GC::Ref<NonFetchSchemeNavigationParams>>())) {
+            // 1. Assert: navigationParams's request is not null.
+            VERIFY(received_navigation_params.has<GC::Ref<NavigationParams>>() && received_navigation_params.get<GC::Ref<NavigationParams>>()->request);
+
+            // 2. Set entry's document state's request referrer to navigationParams's request's referrer.
+            entry->document_state()->set_request_referrer(received_navigation_params.get<GC::Ref<NavigationParams>>()->request->referrer());
+        }
+    }
+
+    // 8. Run completionSteps.
+    if (completion_steps)
+        completion_steps->function()();
+}
+
 // https://html.spec.whatwg.org/multipage/browsing-the-web.html#populating-a-session-history-entry
 void Navigable::populate_session_history_entry_document(
     GC::Ptr<SessionHistoryEntry> entry,
@@ -1453,47 +1499,30 @@ void Navigable::populate_session_history_entry_document(
             // 6. Otherwise, if navigationParams's response's status is not 204 and is not 205, then set entry's document state's document to the result of
             //    loading a document given navigationParams, sourceSnapshotParams, and entry's document state's initiator origin.
             else if (auto const& response = received_navigation_params.get<GC::Ref<NavigationParams>>()->response; response->status() != 204 && response->status() != 205) {
-                auto document = load_document(received_navigation_params.get<GC::Ref<NavigationParams>>(), signal_to_continue_session_history_processing);
+                auto navigation_params = received_navigation_params.get<GC::Ref<NavigationParams>>();
+                auto body = navigation_params->response->body();
+
+                // Get sniff bytes for MIME type detection. For streaming responses where bytes
+                // haven't arrived yet, we must wait asynchronously.
+                auto sniff_bytes = body ? body->sniff_bytes_if_available() : Optional<ReadonlyBytes> { ReadonlyBytes {} };
+                if (!sniff_bytes.has_value()) {
+                    // Async path: bytes not yet available, wait for them
+                    body->wait_for_sniff_bytes(GC::create_function(heap(),
+                        [entry, navigation_params, signal_to_continue_session_history_processing,
+                            received_navigation_params, saveExtraDocumentState, completion_steps](ReadonlyBytes sniff_bytes) {
+                            auto document = load_document(navigation_params, signal_to_continue_session_history_processing, sniff_bytes);
+                            entry->document_state()->set_document(document);
+                            finalize_session_history_entry(entry, received_navigation_params, saveExtraDocumentState, completion_steps);
+                        }));
+                    return;
+                }
+
+                // Sync path: bytes available immediately
+                auto document = load_document(navigation_params, signal_to_continue_session_history_processing, sniff_bytes.value());
                 entry->document_state()->set_document(document);
             }
 
-            // 7. If entry's document state's document is not null, then:
-            if (entry->document()) {
-                // 1. Set entry's document state's ever populated to true.
-                entry->document_state()->set_ever_populated(true);
-
-                // 2. If saveExtraDocumentState is true:
-                if (saveExtraDocumentState) {
-                    // 1. Let document be entry's document state's document.
-                    auto document = entry->document();
-
-                    // 2. Set entry's document state's origin to document's origin.
-                    entry->document_state()->set_origin(document->origin());
-
-                    // 3. If document's URL requires storing the policy container in history, then:
-                    if (url_requires_storing_the_policy_container_in_history(document->url())) {
-                        // 1. Assert: navigationParams is a navigation params (i.e., neither null nor a non-fetch scheme navigation params).
-                        VERIFY(received_navigation_params.has<GC::Ref<NavigationParams>>());
-
-                        // 2. Set entry's document state's history policy container to navigationParams's policy container.
-                        entry->document_state()->set_history_policy_container(GC::Ref { *received_navigation_params.get<GC::Ref<NavigationParams>>()->policy_container });
-                    }
-                }
-
-                // 3. If entry's document state's request referrer is "client", and navigationParams is a navigation params (i.e., neither null nor a non-fetch scheme navigation params), then:
-                if (entry->document_state()->request_referrer() == Fetch::Infrastructure::Request::Referrer::Client
-                    && (!received_navigation_params.has<NullOrError>() && received_navigation_params.has<GC::Ref<NonFetchSchemeNavigationParams>>())) {
-                    // 1. Assert: navigationParams's request is not null.
-                    VERIFY(received_navigation_params.has<GC::Ref<NavigationParams>>() && received_navigation_params.get<GC::Ref<NavigationParams>>()->request);
-
-                    // 2. Set entry's document state's request referrer to navigationParams's request's referrer.
-                    entry->document_state()->set_request_referrer(received_navigation_params.get<GC::Ref<NavigationParams>>()->request->referrer());
-                }
-            }
-
-            // 8. Run completionSteps.
-            if (completion_steps)
-                completion_steps->function()();
+            finalize_session_history_entry(entry, received_navigation_params, saveExtraDocumentState, completion_steps);
         }));
     });
 
@@ -2104,7 +2133,8 @@ GC::Ptr<DOM::Document> Navigable::evaluate_javascript_url(URL::URL const& url, U
         user_involvement);
 
     // 17. Return the result of loading an HTML document given navigationParams.
-    return load_document(navigation_params, Core::Promise<Empty>::construct());
+    // NB: The response body is a known byte sequence, so we can pass it directly for sniffing.
+    return load_document(navigation_params, Core::Promise<Empty>::construct(), result.bytes());
 }
 
 // https://html.spec.whatwg.org/multipage/browsing-the-web.html#navigate-to-a-javascript:-url