diff --git a/Libraries/LibWeb/CSS/Parser/RustTokenizer.cpp b/Libraries/LibWeb/CSS/Parser/RustTokenizer.cpp index 5a8991c4e2..8a2a57fc2f 100644 --- a/Libraries/LibWeb/CSS/Parser/RustTokenizer.cpp +++ b/Libraries/LibWeb/CSS/Parser/RustTokenizer.cpp @@ -5,6 +5,7 @@ */ #include +#include #include #include #include @@ -16,13 +17,26 @@ namespace Web::CSS::Parser { // U+FFFD REPLACEMENT CHARACTER (�) static constexpr u32 REPLACEMENT_CHARACTER = 0xFFFD; -static String decode_and_filter_code_points(StringView input, StringView encoding) +static String decode_and_filter_code_points(StringView input, StringView encoding, TokenizerInput tokenizer_input) { // https://www.w3.org/TR/css-syntax-3/#css-filter-code-points + auto standardized_encoding = TextCodec::get_standardized_encoding(encoding); + VERIFY(standardized_encoding.has_value()); auto decoder = TextCodec::decoder_for(encoding); VERIFY(decoder.has_value()); - auto decoded_input = MUST(decoder->to_utf8(input)); + auto decoded_input = [&] { + if (tokenizer_input == TokenizerInput::DecodedText) { + VERIFY(Utf8View { input }.validate()); + return String::from_utf8_without_validation(input.bytes()); + } + if (standardized_encoding->equals_ignoring_ascii_case("utf-8"sv) && Utf8View { input }.validate(AllowLonelySurrogates::No)) { + if (input.bytes().starts_with({ { 0xef, 0xbb, 0xbf } })) + input = input.substring_view(3); + return String::from_utf8_without_validation(input.bytes()); + } + return MUST(decoder->to_utf8(input)); + }(); // OPTIMIZATION: If the input doesn't contain any filterable characters, we can skip the filtering bool const contains_filterable = [&] { @@ -231,13 +245,13 @@ static_assert(static_cast(FFI::CssNumberType::Number) == static_cast(Num static_assert(static_cast(FFI::CssNumberType::IntegerWithExplicitSign) == static_cast(Number::Type::IntegerWithExplicitSign)); static_assert(static_cast(FFI::CssNumberType::Integer) == static_cast(Number::Type::Integer)); -Vector RustTokenizer::tokenize(StringView input, StringView encoding) +Vector RustTokenizer::tokenize(StringView input, StringView encoding, TokenizerInput tokenizer_input) { struct CallbackContext { Vector tokens; }; - auto filtered_input = decode_and_filter_code_points(input, encoding); + auto filtered_input = decode_and_filter_code_points(input, encoding, tokenizer_input); auto filtered_input_bytes = filtered_input.bytes(); CallbackContext context; context.tokens.ensure_capacity((filtered_input_bytes.size() / 2) + 1); diff --git a/Libraries/LibWeb/CSS/Parser/RustTokenizer.h b/Libraries/LibWeb/CSS/Parser/RustTokenizer.h index efa55e1338..019c1075bb 100644 --- a/Libraries/LibWeb/CSS/Parser/RustTokenizer.h +++ b/Libraries/LibWeb/CSS/Parser/RustTokenizer.h @@ -9,6 +9,7 @@ #include #include #include +#include #include namespace Web::CSS::Parser::FFI { @@ -21,7 +22,7 @@ namespace Web::CSS::Parser { class WEB_API RustTokenizer { public: - static Vector tokenize(StringView input, StringView encoding); + static Vector tokenize(StringView input, StringView encoding, TokenizerInput = TokenizerInput::DecodedText); private: static Token token_from_ffi(FFI::CssToken const&); diff --git a/Libraries/LibWeb/CSS/Parser/Tokenizer.cpp b/Libraries/LibWeb/CSS/Parser/Tokenizer.cpp index 0d20b274bb..569e14b9cc 100644 --- a/Libraries/LibWeb/CSS/Parser/Tokenizer.cpp +++ b/Libraries/LibWeb/CSS/Parser/Tokenizer.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -155,14 +156,27 @@ static inline bool is_E(u32 code_point) return code_point == 0x45; } -Vector Tokenizer::tokenize(StringView input, StringView encoding) +Vector Tokenizer::tokenize(StringView input, StringView encoding, TokenizerInput tokenizer_input) { // https://www.w3.org/TR/css-syntax-3/#css-filter-code-points - auto filter_code_points = [](StringView input, auto encoding) -> String { + auto filter_code_points = [](StringView input, auto encoding, TokenizerInput tokenizer_input) -> String { + auto standardized_encoding = TextCodec::get_standardized_encoding(encoding); + VERIFY(standardized_encoding.has_value()); auto decoder = TextCodec::decoder_for(encoding); VERIFY(decoder.has_value()); - auto decoded_input = MUST(decoder->to_utf8(input)); + auto decoded_input = [&] { + if (tokenizer_input == TokenizerInput::DecodedText) { + VERIFY(Utf8View { input }.validate()); + return String::from_utf8_without_validation(input.bytes()); + } + if (standardized_encoding->equals_ignoring_ascii_case("utf-8"sv) && Utf8View { input }.validate(AllowLonelySurrogates::No)) { + if (input.bytes().starts_with({ { 0xef, 0xbb, 0xbf } })) + input = input.substring_view(3); + return String::from_utf8_without_validation(input.bytes()); + } + return MUST(decoder->to_utf8(input)); + }(); // OPTIMIZATION: If the input doesn't contain any filterable characters, we can skip the filtering bool const contains_filterable = [&] { @@ -214,7 +228,7 @@ Vector Tokenizer::tokenize(StringView input, StringView encoding) return builder.to_string_without_validation(); }; - Tokenizer tokenizer { filter_code_points(input, encoding) }; + Tokenizer tokenizer { filter_code_points(input, encoding, tokenizer_input) }; return tokenizer.tokenize(); } diff --git a/Libraries/LibWeb/CSS/Parser/Tokenizer.h b/Libraries/LibWeb/CSS/Parser/Tokenizer.h index 16eb8fbb2a..de206c5a49 100644 --- a/Libraries/LibWeb/CSS/Parser/Tokenizer.h +++ b/Libraries/LibWeb/CSS/Parser/Tokenizer.h @@ -17,6 +17,11 @@ namespace Web::CSS::Parser { +enum class TokenizerInput { + DecodedText, + EncodedBytes, +}; + class U32Twin { public: void set(size_t index, u32 value) @@ -60,7 +65,7 @@ public: class WEB_API Tokenizer { public: - static Vector tokenize(StringView input, StringView encoding); + static Vector tokenize(StringView input, StringView encoding, TokenizerInput = TokenizerInput::DecodedText); [[nodiscard]] static Token create_eof_token(); diff --git a/Tests/LibWeb/CMakeLists.txt b/Tests/LibWeb/CMakeLists.txt index 41d518f767..c57cb9611a 100644 --- a/Tests/LibWeb/CMakeLists.txt +++ b/Tests/LibWeb/CMakeLists.txt @@ -6,6 +6,7 @@ set(TEST_SOURCES TestCSSPixels.cpp TestCSSStyleSheetInvalidation.cpp TestCSSSyntaxParser.cpp + TestCSSTokenizer.cpp TestCSSTokenStream.cpp TestFetchURL.cpp TestHTMLTokenizer.cpp diff --git a/Tests/LibWeb/TestCSSTokenizer.cpp b/Tests/LibWeb/TestCSSTokenizer.cpp new file mode 100644 index 0000000000..6d6eaeaf23 --- /dev/null +++ b/Tests/LibWeb/TestCSSTokenizer.cpp @@ -0,0 +1,110 @@ +/* + * Copyright (c) 2026-present, the Ladybird developers. + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include +#include +#include +#include + +namespace Web::CSS::Parser { + +static ReadonlyBytes bytes(Vector const& data) +{ + return data.span(); +} + +static void expect_first_token_is_ident(Vector const& tokens, StringView expected_ident, StringView expected_source) +{ + EXPECT(!tokens.is_empty()); + auto const& token = tokens.first(); + EXPECT(token.is(Token::Type::Ident)); + EXPECT_EQ(token.ident(), FlyString::from_utf8_without_validation(expected_ident.bytes())); + EXPECT_EQ(token.original_source_text(), expected_source); +} + +static void expect_first_token_is_ident_for_both_tokenizers(StringView input, StringView encoding, StringView expected_ident, StringView expected_source, TokenizerInput tokenizer_input = TokenizerInput::EncodedBytes) +{ + expect_first_token_is_ident(Tokenizer::tokenize(input, encoding, tokenizer_input), expected_ident, expected_source); + expect_first_token_is_ident(RustTokenizer::tokenize(input, encoding, tokenizer_input), expected_ident, expected_source); +} + +TEST_CASE(tokenizer_decodes_valid_utf8_bytes_with_requested_single_byte_encoding) +{ + auto input = Vector { 0xc3, 0xa9, ' ', '{', '}' }; + expect_first_token_is_ident_for_both_tokenizers(StringView(bytes(input)), "windows-1252"sv, "é"sv, "é"sv); +} + +TEST_CASE(tokenizer_decodes_valid_utf8_bytes_with_requested_utf16_encoding) +{ + auto input = Vector { 'a', 0x00, ' ', 0x00, '{', 0x00, '}', 0x00 }; + expect_first_token_is_ident_for_both_tokenizers(StringView(bytes(input)), "utf-16le"sv, "a"sv, "a"sv); +} + +TEST_CASE(tokenizer_keeps_utf8_fast_path_for_utf8_encoding) +{ + expect_first_token_is_ident_for_both_tokenizers("é {}"sv, "utf-8"sv, "é"sv, "é"sv); +} + +TEST_CASE(tokenizer_strips_bom_in_utf8_fast_path) +{ + auto input = Vector { 0xef, 0xbb, 0xbf, 'b', 'o', 'd', 'y', ' ', '{', '}' }; + expect_first_token_is_ident_for_both_tokenizers(StringView(bytes(input)), "utf-8"sv, "body"sv, "body"sv); +} + +TEST_CASE(tokenizer_strips_bom_for_utf8_encoding_alias) +{ + auto input = Vector { 0xef, 0xbb, 0xbf, 'b', 'o', 'd', 'y', ' ', '{', '}' }; + expect_first_token_is_ident_for_both_tokenizers(StringView(bytes(input)), "utf8"sv, "body"sv, "body"sv); +} + +TEST_CASE(tokenizer_does_not_strip_bom_bytes_for_non_utf8_encoding) +{ + auto input = Vector { 0xef, 0xbb, 0xbf, 'b', 'o', 'd', 'y', ' ', '{', '}' }; + expect_first_token_is_ident_for_both_tokenizers(StringView(bytes(input)), "windows-1252"sv, "body"sv, "body"sv); +} + +TEST_CASE(tokenizer_decodes_utf8_surrogate_bytes_as_three_replacements) +{ + auto input = Vector { 0xed, 0xa0, 0x80, 'b', 'o', 'd', 'y', ' ', '{', '}' }; + expect_first_token_is_ident_for_both_tokenizers(StringView(bytes(input)), "utf-8"sv, "���body"sv, "���body"sv); +} + +TEST_CASE(tokenizer_decodes_utf8_surrogate_bytes_as_three_replacements_for_utf8_alias) +{ + auto input = Vector { 0xed, 0xa0, 0x80, 'b', 'o', 'd', 'y', ' ', '{', '}' }; + expect_first_token_is_ident_for_both_tokenizers(StringView(bytes(input)), "utf8"sv, "���body"sv, "���body"sv); +} + +TEST_CASE(tokenizer_strips_utf8_bom_before_decoding_surrogate_bytes) +{ + auto input = Vector { 0xef, 0xbb, 0xbf, 0xed, 0xa0, 0x80, 'b', 'o', 'd', 'y', ' ', '{', '}' }; + expect_first_token_is_ident_for_both_tokenizers(StringView(bytes(input)), "utf-8"sv, "���body"sv, "���body"sv); +} + +TEST_CASE(tokenizer_decodes_invalid_utf8_second_byte_tail_as_replacements) +{ + auto input = Vector { 0xe0, 0x80, 'b', 'o', 'd', 'y', ' ', '{', '}' }; + expect_first_token_is_ident_for_both_tokenizers(StringView(bytes(input)), "utf-8"sv, "��body"sv, "��body"sv); +} + +TEST_CASE(tokenizer_decodes_truncated_utf8_tail_as_single_replacement) +{ + auto input = Vector { 0xf0, 0x9f, 0x98 }; + expect_first_token_is_ident_for_both_tokenizers(StringView(bytes(input)), "utf-8"sv, "�"sv, "�"sv); +} + +TEST_CASE(tokenizer_decodes_surrogate_shaped_bytes_with_requested_single_byte_encoding) +{ + auto input = Vector { 0xed, 0xa0, 0x80, 'b', 'o', 'd', 'y', ' ', '{', '}' }; + expect_first_token_is_ident_for_both_tokenizers(StringView(bytes(input)), "windows-1252"sv, "í €body"sv, "í €body"sv); +} + +TEST_CASE(tokenizer_filters_decoded_surrogate_code_points_as_single_replacements) +{ + expect_first_token_is_ident_for_both_tokenizers("foo\xed\xa0\x80"sv, "utf-8"sv, "foo�"sv, "foo�"sv, TokenizerInput::DecodedText); +} + +} diff --git a/Tests/LibWeb/css-tokenizer.cpp b/Tests/LibWeb/css-tokenizer.cpp index 0ab2f6c4e7..dae07ec840 100644 --- a/Tests/LibWeb/css-tokenizer.cpp +++ b/Tests/LibWeb/css-tokenizer.cpp @@ -33,8 +33,8 @@ ErrorOr ladybird_main(Main::Arguments arguments) auto file = TRY(Core::File::open(input_path, Core::File::OpenMode::Read)); auto input = TRY(file->read_until_eof()); auto tokens = backend == "rust"sv - ? Web::CSS::Parser::RustTokenizer::tokenize(input, encoding) - : Web::CSS::Parser::Tokenizer::tokenize(input, encoding); + ? Web::CSS::Parser::RustTokenizer::tokenize(input, encoding, Web::CSS::Parser::TokenizerInput::EncodedBytes) + : Web::CSS::Parser::Tokenizer::tokenize(input, encoding, Web::CSS::Parser::TokenizerInput::EncodedBytes); if (!silent) { for (auto const& token : tokens)