LibWeb: Honor requested CSS tokenizer encoding

Keep decoded CSS text separate from tokenizer byte input. CSSOM and
already-decoded stylesheet text preserve code point preprocessing, so a
lone surrogate maps to one replacement character instead of being
re-decoded as malformed UTF-8 bytes.

Decode tokenizer byte input with the requested encoding unless that
encoding is UTF-8 and the byte stream is strictly valid UTF-8. Keep the
fast path by constructing the decoded string without validating twice
after strict validation succeeds.

Preserve UTF-8 decoder behavior on the byte fast path by stripping an
initial UTF-8 BOM and rejecting encoded surrogate bytes. Invalid UTF-8
still goes through the decoder. Add tokenizer coverage for both the C++
and Rust backends across decoded text, UTF-8 aliases, BOM-prefixed
input, invalid UTF-8, and non-UTF requested encodings.

Libraries/LibWeb/CSS/Parser/RustTokenizer.cpp

@@ -5,6 +5,7 @@
  */
 
 #include <AK/StringBuilder.h>
+#include <AK/Utf8View.h>
 #include <LibTextCodec/Decoder.h>
 #include <LibWeb/CSS/CharacterTypes.h>
 #include <LibWeb/CSS/Number.h>
@@ -16,13 +17,26 @@ namespace Web::CSS::Parser {
 // U+FFFD REPLACEMENT CHARACTER (<28>)
 static constexpr u32 REPLACEMENT_CHARACTER = 0xFFFD;
 
-static String decode_and_filter_code_points(StringView input, StringView encoding)
+static String decode_and_filter_code_points(StringView input, StringView encoding, TokenizerInput tokenizer_input)
 {
     // https://www.w3.org/TR/css-syntax-3/#css-filter-code-points
+    auto standardized_encoding = TextCodec::get_standardized_encoding(encoding);
+    VERIFY(standardized_encoding.has_value());
     auto decoder = TextCodec::decoder_for(encoding);
     VERIFY(decoder.has_value());
 
-    auto decoded_input = MUST(decoder->to_utf8(input));
+    auto decoded_input = [&] {
+        if (tokenizer_input == TokenizerInput::DecodedText) {
+            VERIFY(Utf8View { input }.validate());
+            return String::from_utf8_without_validation(input.bytes());
+        }
+        if (standardized_encoding->equals_ignoring_ascii_case("utf-8"sv) && Utf8View { input }.validate(AllowLonelySurrogates::No)) {
+            if (input.bytes().starts_with({ { 0xef, 0xbb, 0xbf } }))
+                input = input.substring_view(3);
+            return String::from_utf8_without_validation(input.bytes());
+        }
+        return MUST(decoder->to_utf8(input));
+    }();
 
     // OPTIMIZATION: If the input doesn't contain any filterable characters, we can skip the filtering
     bool const contains_filterable = [&] {
@@ -231,13 +245,13 @@ static_assert(static_cast<u8>(FFI::CssNumberType::Number) == static_cast<u8>(Num
 static_assert(static_cast<u8>(FFI::CssNumberType::IntegerWithExplicitSign) == static_cast<u8>(Number::Type::IntegerWithExplicitSign));
 static_assert(static_cast<u8>(FFI::CssNumberType::Integer) == static_cast<u8>(Number::Type::Integer));
 
-Vector<Token> RustTokenizer::tokenize(StringView input, StringView encoding)
+Vector<Token> RustTokenizer::tokenize(StringView input, StringView encoding, TokenizerInput tokenizer_input)
 {
     struct CallbackContext {
         Vector<Token> tokens;
     };
 
-    auto filtered_input = decode_and_filter_code_points(input, encoding);
+    auto filtered_input = decode_and_filter_code_points(input, encoding, tokenizer_input);
     auto filtered_input_bytes = filtered_input.bytes();
     CallbackContext context;
     context.tokens.ensure_capacity((filtered_input_bytes.size() / 2) + 1);

Libraries/LibWeb/CSS/Parser/RustTokenizer.h

@@ -9,6 +9,7 @@
 #include <AK/StringView.h>
 #include <AK/Vector.h>
 #include <LibWeb/CSS/Parser/Token.h>
+#include <LibWeb/CSS/Parser/Tokenizer.h>
 #include <LibWeb/Export.h>
 
 namespace Web::CSS::Parser::FFI {
@@ -21,7 +22,7 @@ namespace Web::CSS::Parser {
 
 class WEB_API RustTokenizer {
 public:
-    static Vector<Token> tokenize(StringView input, StringView encoding);
+    static Vector<Token> tokenize(StringView input, StringView encoding, TokenizerInput = TokenizerInput::DecodedText);
 
 private:
     static Token token_from_ffi(FFI::CssToken const&);

Libraries/LibWeb/CSS/Parser/Tokenizer.cpp

@@ -8,6 +8,7 @@
 #include <AK/Debug.h>
 #include <AK/SourceLocation.h>
 #include <AK/StringConversions.h>
+#include <AK/Utf8View.h>
 #include <AK/Vector.h>
 #include <LibTextCodec/Decoder.h>
 #include <LibWeb/CSS/CharacterTypes.h>
@@ -155,14 +156,27 @@ static inline bool is_E(u32 code_point)
     return code_point == 0x45;
 }
 
-Vector<Token> Tokenizer::tokenize(StringView input, StringView encoding)
+Vector<Token> Tokenizer::tokenize(StringView input, StringView encoding, TokenizerInput tokenizer_input)
 {
     // https://www.w3.org/TR/css-syntax-3/#css-filter-code-points
-    auto filter_code_points = [](StringView input, auto encoding) -> String {
+    auto filter_code_points = [](StringView input, auto encoding, TokenizerInput tokenizer_input) -> String {
+        auto standardized_encoding = TextCodec::get_standardized_encoding(encoding);
+        VERIFY(standardized_encoding.has_value());
         auto decoder = TextCodec::decoder_for(encoding);
         VERIFY(decoder.has_value());
 
-        auto decoded_input = MUST(decoder->to_utf8(input));
+        auto decoded_input = [&] {
+            if (tokenizer_input == TokenizerInput::DecodedText) {
+                VERIFY(Utf8View { input }.validate());
+                return String::from_utf8_without_validation(input.bytes());
+            }
+            if (standardized_encoding->equals_ignoring_ascii_case("utf-8"sv) && Utf8View { input }.validate(AllowLonelySurrogates::No)) {
+                if (input.bytes().starts_with({ { 0xef, 0xbb, 0xbf } }))
+                    input = input.substring_view(3);
+                return String::from_utf8_without_validation(input.bytes());
+            }
+            return MUST(decoder->to_utf8(input));
+        }();
 
         // OPTIMIZATION: If the input doesn't contain any filterable characters, we can skip the filtering
         bool const contains_filterable = [&] {
@@ -214,7 +228,7 @@ Vector<Token> Tokenizer::tokenize(StringView input, StringView encoding)
         return builder.to_string_without_validation();
     };
 
-    Tokenizer tokenizer { filter_code_points(input, encoding) };
+    Tokenizer tokenizer { filter_code_points(input, encoding, tokenizer_input) };
     return tokenizer.tokenize();
 }
 

Libraries/LibWeb/CSS/Parser/Tokenizer.h

@@ -17,6 +17,11 @@
 
 namespace Web::CSS::Parser {
 
+enum class TokenizerInput {
+    DecodedText,
+    EncodedBytes,
+};
+
 class U32Twin {
 public:
     void set(size_t index, u32 value)
@@ -60,7 +65,7 @@ public:
 
 class WEB_API Tokenizer {
 public:
-    static Vector<Token> tokenize(StringView input, StringView encoding);
+    static Vector<Token> tokenize(StringView input, StringView encoding, TokenizerInput = TokenizerInput::DecodedText);
 
     [[nodiscard]] static Token create_eof_token();
 

Tests/LibWeb/CMakeLists.txt

@@ -6,6 +6,7 @@ set(TEST_SOURCES
     TestCSSPixels.cpp
     TestCSSStyleSheetInvalidation.cpp
     TestCSSSyntaxParser.cpp
+    TestCSSTokenizer.cpp
     TestCSSTokenStream.cpp
     TestFetchURL.cpp
     TestHTMLTokenizer.cpp

Tests/LibWeb/TestCSSTokenizer.cpp

@@ -0,0 +1,110 @@
+/*
+ * Copyright (c) 2026-present, the Ladybird developers.
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include <AK/Vector.h>
+#include <LibTest/TestCase.h>
+#include <LibWeb/CSS/Parser/RustTokenizer.h>
+#include <LibWeb/CSS/Parser/Tokenizer.h>
+
+namespace Web::CSS::Parser {
+
+static ReadonlyBytes bytes(Vector<u8> const& data)
+{
+    return data.span();
+}
+
+static void expect_first_token_is_ident(Vector<Token> const& tokens, StringView expected_ident, StringView expected_source)
+{
+    EXPECT(!tokens.is_empty());
+    auto const& token = tokens.first();
+    EXPECT(token.is(Token::Type::Ident));
+    EXPECT_EQ(token.ident(), FlyString::from_utf8_without_validation(expected_ident.bytes()));
+    EXPECT_EQ(token.original_source_text(), expected_source);
+}
+
+static void expect_first_token_is_ident_for_both_tokenizers(StringView input, StringView encoding, StringView expected_ident, StringView expected_source, TokenizerInput tokenizer_input = TokenizerInput::EncodedBytes)
+{
+    expect_first_token_is_ident(Tokenizer::tokenize(input, encoding, tokenizer_input), expected_ident, expected_source);
+    expect_first_token_is_ident(RustTokenizer::tokenize(input, encoding, tokenizer_input), expected_ident, expected_source);
+}
+
+TEST_CASE(tokenizer_decodes_valid_utf8_bytes_with_requested_single_byte_encoding)
+{
+    auto input = Vector<u8> { 0xc3, 0xa9, ' ', '{', '}' };
+    expect_first_token_is_ident_for_both_tokenizers(StringView(bytes(input)), "windows-1252"sv, "é"sv, "é"sv);
+}
+
+TEST_CASE(tokenizer_decodes_valid_utf8_bytes_with_requested_utf16_encoding)
+{
+    auto input = Vector<u8> { 'a', 0x00, ' ', 0x00, '{', 0x00, '}', 0x00 };
+    expect_first_token_is_ident_for_both_tokenizers(StringView(bytes(input)), "utf-16le"sv, "a"sv, "a"sv);
+}
+
+TEST_CASE(tokenizer_keeps_utf8_fast_path_for_utf8_encoding)
+{
+    expect_first_token_is_ident_for_both_tokenizers("é {}"sv, "utf-8"sv, "é"sv, "é"sv);
+}
+
+TEST_CASE(tokenizer_strips_bom_in_utf8_fast_path)
+{
+    auto input = Vector<u8> { 0xef, 0xbb, 0xbf, 'b', 'o', 'd', 'y', ' ', '{', '}' };
+    expect_first_token_is_ident_for_both_tokenizers(StringView(bytes(input)), "utf-8"sv, "body"sv, "body"sv);
+}
+
+TEST_CASE(tokenizer_strips_bom_for_utf8_encoding_alias)
+{
+    auto input = Vector<u8> { 0xef, 0xbb, 0xbf, 'b', 'o', 'd', 'y', ' ', '{', '}' };
+    expect_first_token_is_ident_for_both_tokenizers(StringView(bytes(input)), "utf8"sv, "body"sv, "body"sv);
+}
+
+TEST_CASE(tokenizer_does_not_strip_bom_bytes_for_non_utf8_encoding)
+{
+    auto input = Vector<u8> { 0xef, 0xbb, 0xbf, 'b', 'o', 'd', 'y', ' ', '{', '}' };
+    expect_first_token_is_ident_for_both_tokenizers(StringView(bytes(input)), "windows-1252"sv, "body"sv, "body"sv);
+}
+
+TEST_CASE(tokenizer_decodes_utf8_surrogate_bytes_as_three_replacements)
+{
+    auto input = Vector<u8> { 0xed, 0xa0, 0x80, 'b', 'o', 'd', 'y', ' ', '{', '}' };
+    expect_first_token_is_ident_for_both_tokenizers(StringView(bytes(input)), "utf-8"sv, "<EFBFBD><EFBFBD><EFBFBD>body"sv, "<EFBFBD><EFBFBD><EFBFBD>body"sv);
+}
+
+TEST_CASE(tokenizer_decodes_utf8_surrogate_bytes_as_three_replacements_for_utf8_alias)
+{
+    auto input = Vector<u8> { 0xed, 0xa0, 0x80, 'b', 'o', 'd', 'y', ' ', '{', '}' };
+    expect_first_token_is_ident_for_both_tokenizers(StringView(bytes(input)), "utf8"sv, "<EFBFBD><EFBFBD><EFBFBD>body"sv, "<EFBFBD><EFBFBD><EFBFBD>body"sv);
+}
+
+TEST_CASE(tokenizer_strips_utf8_bom_before_decoding_surrogate_bytes)
+{
+    auto input = Vector<u8> { 0xef, 0xbb, 0xbf, 0xed, 0xa0, 0x80, 'b', 'o', 'd', 'y', ' ', '{', '}' };
+    expect_first_token_is_ident_for_both_tokenizers(StringView(bytes(input)), "utf-8"sv, "<EFBFBD><EFBFBD><EFBFBD>body"sv, "<EFBFBD><EFBFBD><EFBFBD>body"sv);
+}
+
+TEST_CASE(tokenizer_decodes_invalid_utf8_second_byte_tail_as_replacements)
+{
+    auto input = Vector<u8> { 0xe0, 0x80, 'b', 'o', 'd', 'y', ' ', '{', '}' };
+    expect_first_token_is_ident_for_both_tokenizers(StringView(bytes(input)), "utf-8"sv, "<EFBFBD><EFBFBD>body"sv, "<EFBFBD><EFBFBD>body"sv);
+}
+
+TEST_CASE(tokenizer_decodes_truncated_utf8_tail_as_single_replacement)
+{
+    auto input = Vector<u8> { 0xf0, 0x9f, 0x98 };
+    expect_first_token_is_ident_for_both_tokenizers(StringView(bytes(input)), "utf-8"sv, "<EFBFBD>"sv, "<EFBFBD>"sv);
+}
+
+TEST_CASE(tokenizer_decodes_surrogate_shaped_bytes_with_requested_single_byte_encoding)
+{
+    auto input = Vector<u8> { 0xed, 0xa0, 0x80, 'b', 'o', 'd', 'y', ' ', '{', '}' };
+    expect_first_token_is_ident_for_both_tokenizers(StringView(bytes(input)), "windows-1252"sv, "í €body"sv, "í €body"sv);
+}
+
+TEST_CASE(tokenizer_filters_decoded_surrogate_code_points_as_single_replacements)
+{
+    expect_first_token_is_ident_for_both_tokenizers("foo\xed\xa0\x80"sv, "utf-8"sv, "foo<EFBFBD>"sv, "foo<EFBFBD>"sv, TokenizerInput::DecodedText);
+}
+
+}

Tests/LibWeb/css-tokenizer.cpp

@@ -33,8 +33,8 @@ ErrorOr<int> ladybird_main(Main::Arguments arguments)
     auto file = TRY(Core::File::open(input_path, Core::File::OpenMode::Read));
     auto input = TRY(file->read_until_eof());
     auto tokens = backend == "rust"sv
-        ? Web::CSS::Parser::RustTokenizer::tokenize(input, encoding)
-        : Web::CSS::Parser::Tokenizer::tokenize(input, encoding);
+        ? Web::CSS::Parser::RustTokenizer::tokenize(input, encoding, Web::CSS::Parser::TokenizerInput::EncodedBytes)
+        : Web::CSS::Parser::Tokenizer::tokenize(input, encoding, Web::CSS::Parser::TokenizerInput::EncodedBytes);
 
     if (!silent) {
         for (auto const& token : tokens)