f4960d9d7d
Keep decoded CSS text separate from tokenizer byte input. CSSOM and already-decoded stylesheet text preserve code point preprocessing, so a lone surrogate maps to one replacement character instead of being re-decoded as malformed UTF-8 bytes. Decode tokenizer byte input with the requested encoding unless that encoding is UTF-8 and the byte stream is strictly valid UTF-8. Keep the fast path by constructing the decoded string without validating twice after strict validation succeeds. Preserve UTF-8 decoder behavior on the byte fast path by stripping an initial UTF-8 BOM and rejecting encoded surrogate bytes. Invalid UTF-8 still goes through the decoder. Add tokenizer coverage for both the C++ and Rust backends across decoded text, UTF-8 aliases, BOM-prefixed input, invalid UTF-8, and non-UTF requested encodings.
110 lines
4.5 KiB
C++
110 lines
4.5 KiB
C++
/*
|
||
* Copyright (c) 2026-present, the Ladybird developers.
|
||
*
|
||
* SPDX-License-Identifier: BSD-2-Clause
|
||
*/
|
||
|
||
#include <AK/Vector.h>
|
||
#include <LibTest/TestCase.h>
|
||
#include <LibWeb/CSS/Parser/RustTokenizer.h>
|
||
#include <LibWeb/CSS/Parser/Tokenizer.h>
|
||
|
||
namespace Web::CSS::Parser {
|
||
|
||
static ReadonlyBytes bytes(Vector<u8> const& data)
|
||
{
|
||
return data.span();
|
||
}
|
||
|
||
static void expect_first_token_is_ident(Vector<Token> const& tokens, StringView expected_ident, StringView expected_source)
|
||
{
|
||
EXPECT(!tokens.is_empty());
|
||
auto const& token = tokens.first();
|
||
EXPECT(token.is(Token::Type::Ident));
|
||
EXPECT_EQ(token.ident(), FlyString::from_utf8_without_validation(expected_ident.bytes()));
|
||
EXPECT_EQ(token.original_source_text(), expected_source);
|
||
}
|
||
|
||
static void expect_first_token_is_ident_for_both_tokenizers(StringView input, StringView encoding, StringView expected_ident, StringView expected_source, TokenizerInput tokenizer_input = TokenizerInput::EncodedBytes)
|
||
{
|
||
expect_first_token_is_ident(Tokenizer::tokenize(input, encoding, tokenizer_input), expected_ident, expected_source);
|
||
expect_first_token_is_ident(RustTokenizer::tokenize(input, encoding, tokenizer_input), expected_ident, expected_source);
|
||
}
|
||
|
||
TEST_CASE(tokenizer_decodes_valid_utf8_bytes_with_requested_single_byte_encoding)
|
||
{
|
||
auto input = Vector<u8> { 0xc3, 0xa9, ' ', '{', '}' };
|
||
expect_first_token_is_ident_for_both_tokenizers(StringView(bytes(input)), "windows-1252"sv, "é"sv, "é"sv);
|
||
}
|
||
|
||
TEST_CASE(tokenizer_decodes_valid_utf8_bytes_with_requested_utf16_encoding)
|
||
{
|
||
auto input = Vector<u8> { 'a', 0x00, ' ', 0x00, '{', 0x00, '}', 0x00 };
|
||
expect_first_token_is_ident_for_both_tokenizers(StringView(bytes(input)), "utf-16le"sv, "a"sv, "a"sv);
|
||
}
|
||
|
||
TEST_CASE(tokenizer_keeps_utf8_fast_path_for_utf8_encoding)
|
||
{
|
||
expect_first_token_is_ident_for_both_tokenizers("é {}"sv, "utf-8"sv, "é"sv, "é"sv);
|
||
}
|
||
|
||
TEST_CASE(tokenizer_strips_bom_in_utf8_fast_path)
|
||
{
|
||
auto input = Vector<u8> { 0xef, 0xbb, 0xbf, 'b', 'o', 'd', 'y', ' ', '{', '}' };
|
||
expect_first_token_is_ident_for_both_tokenizers(StringView(bytes(input)), "utf-8"sv, "body"sv, "body"sv);
|
||
}
|
||
|
||
TEST_CASE(tokenizer_strips_bom_for_utf8_encoding_alias)
|
||
{
|
||
auto input = Vector<u8> { 0xef, 0xbb, 0xbf, 'b', 'o', 'd', 'y', ' ', '{', '}' };
|
||
expect_first_token_is_ident_for_both_tokenizers(StringView(bytes(input)), "utf8"sv, "body"sv, "body"sv);
|
||
}
|
||
|
||
TEST_CASE(tokenizer_does_not_strip_bom_bytes_for_non_utf8_encoding)
|
||
{
|
||
auto input = Vector<u8> { 0xef, 0xbb, 0xbf, 'b', 'o', 'd', 'y', ' ', '{', '}' };
|
||
expect_first_token_is_ident_for_both_tokenizers(StringView(bytes(input)), "windows-1252"sv, "body"sv, "body"sv);
|
||
}
|
||
|
||
TEST_CASE(tokenizer_decodes_utf8_surrogate_bytes_as_three_replacements)
|
||
{
|
||
auto input = Vector<u8> { 0xed, 0xa0, 0x80, 'b', 'o', 'd', 'y', ' ', '{', '}' };
|
||
expect_first_token_is_ident_for_both_tokenizers(StringView(bytes(input)), "utf-8"sv, "<EFBFBD><EFBFBD><EFBFBD>body"sv, "<EFBFBD><EFBFBD><EFBFBD>body"sv);
|
||
}
|
||
|
||
TEST_CASE(tokenizer_decodes_utf8_surrogate_bytes_as_three_replacements_for_utf8_alias)
|
||
{
|
||
auto input = Vector<u8> { 0xed, 0xa0, 0x80, 'b', 'o', 'd', 'y', ' ', '{', '}' };
|
||
expect_first_token_is_ident_for_both_tokenizers(StringView(bytes(input)), "utf8"sv, "<EFBFBD><EFBFBD><EFBFBD>body"sv, "<EFBFBD><EFBFBD><EFBFBD>body"sv);
|
||
}
|
||
|
||
TEST_CASE(tokenizer_strips_utf8_bom_before_decoding_surrogate_bytes)
|
||
{
|
||
auto input = Vector<u8> { 0xef, 0xbb, 0xbf, 0xed, 0xa0, 0x80, 'b', 'o', 'd', 'y', ' ', '{', '}' };
|
||
expect_first_token_is_ident_for_both_tokenizers(StringView(bytes(input)), "utf-8"sv, "<EFBFBD><EFBFBD><EFBFBD>body"sv, "<EFBFBD><EFBFBD><EFBFBD>body"sv);
|
||
}
|
||
|
||
TEST_CASE(tokenizer_decodes_invalid_utf8_second_byte_tail_as_replacements)
|
||
{
|
||
auto input = Vector<u8> { 0xe0, 0x80, 'b', 'o', 'd', 'y', ' ', '{', '}' };
|
||
expect_first_token_is_ident_for_both_tokenizers(StringView(bytes(input)), "utf-8"sv, "<EFBFBD><EFBFBD>body"sv, "<EFBFBD><EFBFBD>body"sv);
|
||
}
|
||
|
||
TEST_CASE(tokenizer_decodes_truncated_utf8_tail_as_single_replacement)
|
||
{
|
||
auto input = Vector<u8> { 0xf0, 0x9f, 0x98 };
|
||
expect_first_token_is_ident_for_both_tokenizers(StringView(bytes(input)), "utf-8"sv, "<EFBFBD>"sv, "<EFBFBD>"sv);
|
||
}
|
||
|
||
TEST_CASE(tokenizer_decodes_surrogate_shaped_bytes_with_requested_single_byte_encoding)
|
||
{
|
||
auto input = Vector<u8> { 0xed, 0xa0, 0x80, 'b', 'o', 'd', 'y', ' ', '{', '}' };
|
||
expect_first_token_is_ident_for_both_tokenizers(StringView(bytes(input)), "windows-1252"sv, "í €body"sv, "í €body"sv);
|
||
}
|
||
|
||
TEST_CASE(tokenizer_filters_decoded_surrogate_code_points_as_single_replacements)
|
||
{
|
||
expect_first_token_is_ident_for_both_tokenizers("foo\xed\xa0\x80"sv, "utf-8"sv, "foo<EFBFBD>"sv, "foo<EFBFBD>"sv, TokenizerInput::DecodedText);
|
||
}
|
||
|
||
}
|