LibWeb: Normalize decoded HTML string parsing

Preserve leading BOMs when parsing already-decoded HTML strings, since
those strings do not go through the encoded byte decoder path.

Decoded markup from JS strings can also contain WTF-8 for lone surrogate
code units. Keep the common scalar UTF-8 path to a single validation and
copy, but replace surrogates before handing bytes to the Rust tokenizer.

Add text coverage for DOMParser and innerHTML string parsing, including
leading BOMs, text and attributes, lone high and low surrogates, and a
valid surrogate pair.

Libraries/LibWeb/DOM/Document.cpp

@@ -8013,7 +8013,7 @@ void Document::parse_html_from_a_string(StringView html)
     // 3. Place html into the input stream for parser. The encoding confidence is irrelevant.
     // FIXME: We don't have the concept of encoding confidence yet.
     auto scripting_mode = is_scripting_enabled() ? HTML::ParserScriptingMode::Normal : HTML::ParserScriptingMode::Disabled;
-    auto parser = HTML::HTMLParser::create(*this, html, scripting_mode, "UTF-8"sv);
+    auto parser = HTML::HTMLParser::create_for_decoded_string(*this, html, scripting_mode, "UTF-8"sv);
 
     // 4. Start parser and let it run until it has consumed all the characters just inserted into the input stream.
     parser->run(as<HTML::Window>(HTML::relevant_global_object(*this)).associated_document().url());

Libraries/LibWeb/HTML/Parser/HTMLParser.cpp

@@ -90,8 +90,8 @@ extern "C" size_t ladybird_html_parser_attach_declarative_shadow_root(size_t, Ru
 extern "C" void ladybird_html_parser_set_template_content(size_t, size_t);
 extern "C" bool ladybird_html_parser_allows_declarative_shadow_roots(size_t);
 
-HTMLParser::HTMLParser(DOM::Document& document, ParserScriptingMode scripting_mode, StringView input, StringView encoding)
-    : m_tokenizer(input, encoding)
+HTMLParser::HTMLParser(DOM::Document& document, ParserScriptingMode scripting_mode, StringView input, StringView encoding, HTMLTokenizer::InputType input_type)
+    : m_tokenizer(input, encoding, input_type)
     , m_scripting_mode(scripting_mode)
     , m_document(document)
 {
@@ -923,7 +923,7 @@ WebIDL::ExceptionOr<Vector<GC::Root<DOM::Node>>> HTMLParser::parse_html_fragment
     if (context_element.document().is_scripting_disabled())
         scripting_mode = HTML::ParserScriptingMode::Disabled;
 
-    auto parser = HTMLParser::create(*temp_document, markup, scripting_mode, "utf-8"sv);
+    auto parser = HTMLParser::create_for_decoded_string(*temp_document, markup, scripting_mode, "utf-8"sv);
     parser->m_context_element = context_element;
     parser->m_parsing_fragment = true;
 
@@ -1061,6 +1061,11 @@ GC::Ref<HTMLParser> HTMLParser::create(DOM::Document& document, StringView input
     return document.realm().create<HTMLParser>(document, scripting_mode, input, encoding);
 }
 
+GC::Ref<HTMLParser> HTMLParser::create_for_decoded_string(DOM::Document& document, StringView input, ParserScriptingMode scripting_mode, StringView encoding)
+{
+    return document.realm().create<HTMLParser>(document, scripting_mode, input, encoding, HTMLTokenizer::InputType::DecodedString);
+}
+
 enum class AttributeMode {
     No,
     Yes,

Libraries/LibWeb/HTML/Parser/HTMLParser.h

@@ -41,6 +41,7 @@ public:
     static GC::Ref<HTMLParser> create_with_open_input_stream(DOM::Document&);
     static GC::Ref<HTMLParser> create_with_uncertain_encoding(DOM::Document&, ByteBuffer const& input, Optional<MimeSniff::MimeType> maybe_mime_type = {});
     static GC::Ref<HTMLParser> create(DOM::Document&, StringView input, ParserScriptingMode, StringView encoding);
+    static GC::Ref<HTMLParser> create_for_decoded_string(DOM::Document&, StringView input, ParserScriptingMode, StringView encoding);
 
     void run(HTMLTokenizer::StopAtInsertionPoint = HTMLTokenizer::StopAtInsertionPoint::No);
     void run(URL::URL const&, HTMLTokenizer::StopAtInsertionPoint = HTMLTokenizer::StopAtInsertionPoint::No);
@@ -93,7 +94,7 @@ private:
         Yes,
     };
 
-    HTMLParser(DOM::Document&, ParserScriptingMode, StringView input, StringView encoding);
+    HTMLParser(DOM::Document&, ParserScriptingMode, StringView input, StringView encoding, HTMLTokenizer::InputType = HTMLTokenizer::InputType::EncodedBytes);
     HTMLParser(DOM::Document&, ParserScriptingMode, ScriptCreatedParser);
 
     virtual void visit_edges(Cell::Visitor&) override;

Libraries/LibWeb/HTML/Parser/HTMLTokenizer.cpp

@@ -7,6 +7,8 @@
 
 #include <AK/Debug.h>
 #include <AK/FlyString.h>
+#include <AK/StringBuilder.h>
+#include <AK/Utf8View.h>
 #include <AK/Vector.h>
 #include <LibTextCodec/Decoder.h>
 #include <LibWeb/HTML/Parser/HTMLToken.h>
@@ -39,6 +41,22 @@ static RustFfiTokenizerHandle* create_tokenizer_from_utf8(StringView utf8_bytes)
     return rust_html_tokenizer_create_from_utf8(bytes, utf8_bytes.length());
 }
 
+static String decoded_string_for_utf8_tokenizer(StringView input)
+{
+    Utf8View utf8_view { input };
+    if (utf8_view.validate(AllowLonelySurrogates::No))
+        return String::from_utf8_without_validation(input.bytes());
+
+    // Decoded strings may come from WTF-16 JS strings. Rust's UTF-8 path
+    // requires scalar-value UTF-8, so replace lone surrogates but keep BOMs.
+    VERIFY(utf8_view.validate());
+
+    StringBuilder builder(input.length());
+    for (auto code_point : utf8_view)
+        builder.append_code_point(is_unicode_surrogate(code_point) ? AK::UnicodeUtils::REPLACEMENT_CODE_POINT : code_point);
+    return builder.to_string_without_validation();
+}
+
 static Vector<FlyString> build_interned_name_table(size_t count, void (*fetch)(uint16_t, uint8_t const**, size_t*))
 {
     Vector<FlyString> table;
@@ -90,11 +108,15 @@ HTMLTokenizer::~HTMLTokenizer()
         rust_html_tokenizer_destroy(m_tokenizer);
 }
 
-HTMLTokenizer::HTMLTokenizer(StringView input, ByteString const& encoding)
+HTMLTokenizer::HTMLTokenizer(StringView input, ByteString const& encoding, InputType input_type)
 {
-    auto decoder = TextCodec::decoder_for(encoding);
-    VERIFY(decoder.has_value());
-    m_source = MUST(decoder->to_utf8(input));
+    if (input_type == InputType::EncodedBytes) {
+        auto decoder = TextCodec::decoder_for(encoding);
+        VERIFY(decoder.has_value());
+        m_source = MUST(decoder->to_utf8(input));
+    } else {
+        m_source = decoded_string_for_utf8_tokenizer(input);
+    }
     m_input_stream_closed = true;
     m_tokenizer = create_tokenizer_from_utf8(m_source.bytes_as_string_view());
 }

Libraries/LibWeb/HTML/Parser/HTMLTokenizer.h

@@ -103,8 +103,13 @@ class HTMLParser;
 
 class WEB_API HTMLTokenizer {
 public:
+    enum class InputType {
+        EncodedBytes,
+        DecodedString,
+    };
+
     explicit HTMLTokenizer();
-    explicit HTMLTokenizer(StringView input, ByteString const& encoding);
+    explicit HTMLTokenizer(StringView input, ByteString const& encoding, InputType = InputType::EncodedBytes);
     ~HTMLTokenizer();
 
     enum class State {

Tests/LibWeb/Text/expected/HTML/preserve-leading-bom-in-string-parsing.txt

@@ -0,0 +1,2 @@
+DOMParser: length=4, codePoints=feff,54,68,65
+innerHTML: length=4, codePoints=feff,54,68,65

Tests/LibWeb/Text/expected/HTML/replace-lone-surrogates-in-string-parsing.txt

@@ -0,0 +1,4 @@
+DOMParser text: length=7, codePoints=feff,fffd,41,fffd,42,1f600
+innerHTML text: length=7, codePoints=feff,fffd,41,fffd,42,1f600
+DOMParser attribute: length=7, codePoints=feff,fffd,41,fffd,42,1f600
+innerHTML attribute: length=7, codePoints=feff,fffd,41,fffd,42,1f600

Tests/LibWeb/Text/input/HTML/preserve-leading-bom-in-string-parsing.html

@@ -0,0 +1,17 @@
+<!DOCTYPE html>
+<script src="../include.js"></script>
+<script>
+    test(() => {
+        function dumpText(label, text) {
+            const codePoints = Array.from(text, character => character.codePointAt(0).toString(16)).join(",");
+            println(`${label}: length=${text.length}, codePoints=${codePoints}`);
+        }
+
+        const parsedDocument = new DOMParser().parseFromString("\uFEFFThe", "text/html");
+        dumpText("DOMParser", parsedDocument.body.textContent);
+
+        const div = document.createElement("div");
+        div.innerHTML = "\uFEFFThe";
+        dumpText("innerHTML", div.textContent);
+    });
+</script>

Tests/LibWeb/Text/input/HTML/replace-lone-surrogates-in-string-parsing.html

@@ -0,0 +1,27 @@
+<!DOCTYPE html>
+<script src="../include.js"></script>
+<script>
+    test(() => {
+        function dumpText(label, text) {
+            const codePoints = Array.from(text, character => character.codePointAt(0).toString(16)).join(",");
+            println(`${label}: length=${text.length}, codePoints=${codePoints}`);
+        }
+
+        const textMarkup = "\uFEFF\uD800A\uDC00B\uD83D\uDE00";
+
+        const parsedDocument = new DOMParser().parseFromString(textMarkup, "text/html");
+        dumpText("DOMParser text", parsedDocument.body.textContent);
+
+        const div = document.createElement("div");
+        div.innerHTML = textMarkup;
+        dumpText("innerHTML text", div.textContent);
+
+        const attributeMarkup = `<span title="${textMarkup}"></span>`;
+
+        const parsedAttributeDocument = new DOMParser().parseFromString(attributeMarkup, "text/html");
+        dumpText("DOMParser attribute", parsedAttributeDocument.body.firstChild.getAttribute("title"));
+
+        div.innerHTML = attributeMarkup;
+        dumpText("innerHTML attribute", div.firstChild.getAttribute("title"));
+    });
+</script>