dmaax/ladybird
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions
ladybird/Tests/LibWeb/TestHTMLTokenizer.cpp
Andreas Kling 171e3adf01 LibWeb: Replace the HTML tokenizer with Rust 
Replace the C++ HTML tokenizer with a Rust implementation behind the
existing HTMLTokenizer API.

Keep the parser-facing integration points for streaming input,
insertion points, document.write(), EOF insertion, parser aborts,
speculative parser input, and last start tag tracking. The generated
FFI handle stays an implementation detail of HTMLTokenizer, so callers
keep a single tokenizer class.

Preserve duplicate attributes through FFI so C++ token normalization can
record the duplicate-attribute signal used by CSP nonce checks. Keep
bulk tag-name and attribute scans capped at the active insertion point
so streamed parser input is spliced at the right offset.

Use generated DAFSA tables for named character references and intern
common tag and attribute names to reduce FFI marshalling overhead. This
also fixes attribute name source positions, nested old insertion points,
and aborted fast-path handling.

TestHTMLTokenizer covers duplicate attributes and insertion points in
fast tag-name, attribute-name, and quoted-value scans. A CSP text test
covers duplicate nonce attributes on parser-created script elements.
The tokenizer dump fixtures still match, TestHTMLTokenizer passes, and
the full release test-web run passes with 6981 tests and 226 skipped.
2026-05-15 21:01:40 +02:00

360 lines
12 KiB
C++
Raw Blame History

/*
* Copyright (c) 2021, Max Wipfli <max.wipfli@serenityos.org>
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#include <LibTest/TestCase.h>
#include <LibCore/File.h>
#include <LibWeb/HTML/Parser/HTMLTokenizer.h>
using Tokenizer = Web::HTML::HTMLTokenizer;
using Token = Web::HTML::HTMLToken;
#define BEGIN_ENUMERATION(tokens) \
auto current_token = (tokens).begin(); \
[[maybe_unused]] Token* last_token;
#define END_ENUMERATION() \
EXPECT(current_token.is_end());
#define NEXT_TOKEN() \
last_token = &*current_token; \
++current_token;
#define EXPECT_START_TAG_TOKEN(_tag_name, start_column, end_column) \
EXPECT_EQ(current_token->type(), Token::Type::StartTag); \
EXPECT_EQ(current_token->tag_name(), #_tag_name); \
EXPECT_EQ(current_token->start_position().column, start_column); \
EXPECT_EQ(current_token->end_position().column, end_column); \
NEXT_TOKEN();
#define EXPECT_END_TAG_TOKEN(_tag_name, start_column, end_column) \
EXPECT_EQ(current_token->type(), Token::Type::EndTag); \
EXPECT_EQ(current_token->tag_name(), #_tag_name); \
EXPECT_EQ(current_token->start_position().column, start_column); \
EXPECT_EQ(current_token->end_position().column, end_column); \
NEXT_TOKEN();
#define EXPECT_END_OF_FILE_TOKEN() \
EXPECT_EQ(current_token->type(), Token::Type::EndOfFile); \
NEXT_TOKEN();
#define EXPECT_CHARACTER_TOKEN(character) \
EXPECT_EQ(current_token->type(), Token::Type::Character); \
EXPECT_EQ(current_token->code_point(), (u32)(character)); \
NEXT_TOKEN();
#define EXPECT_CHARACTER_TOKENS(string) \
for (auto c : #string##sv) { \
EXPECT_CHARACTER_TOKEN(c); \
}
#define EXPECT_COMMENT_TOKEN() \
EXPECT_EQ(current_token->type(), Token::Type::Comment); \
NEXT_TOKEN();
#define EXPECT_DOCTYPE_TOKEN() \
EXPECT_EQ(current_token->type(), Token::Type::DOCTYPE); \
NEXT_TOKEN();
#define EXPECT_TAG_TOKEN_ATTRIBUTE(name, attribute_value, name_start_column, name_end_column, value_start_column, value_end_column) \
VERIFY(last_token); \
auto name##_attr = last_token->raw_attribute(#name##_fly_string); \
VERIFY(name##_attr.has_value()); \
EXPECT_EQ(name##_attr->value, attribute_value); \
EXPECT_EQ(name##_attr->name_start_position.column, name_start_column); \
EXPECT_EQ(name##_attr->name_end_position.column, name_end_column); \
EXPECT_EQ(name##_attr->value_start_position.column, value_start_column); \
EXPECT_EQ(name##_attr->value_end_position.column, value_end_column);
#define EXPECT_TAG_TOKEN_ATTRIBUTE_COUNT(count) \
VERIFY(last_token); \
EXPECT_EQ(last_token->attribute_count(), (size_t)(count));
static Vector<Token> run_tokenizer(StringView input)
{
Vector<Token> tokens;
Tokenizer tokenizer { input, "UTF-8"sv };
while (true) {
auto maybe_token = tokenizer.next_token();
if (!maybe_token.has_value())
break;
tokens.append(maybe_token.release_value());
}
return tokens;
}
// FIXME: It's not very nice to rely on the format of HTMLToken::to_string() to stay the same.
static u32 hash_tokens(Vector<Token> const& tokens)
{
StringBuilder builder;
for (auto& token : tokens)
builder.append(token.to_string());
return (u32)builder.string_view().hash();
}
TEST_CASE(empty)
{
auto tokens = run_tokenizer(""sv);
BEGIN_ENUMERATION(tokens);
EXPECT_END_OF_FILE_TOKEN();
END_ENUMERATION();
}
TEST_CASE(basic)
{
auto tokens = run_tokenizer("<html><head></head><body></body></html>"sv);
BEGIN_ENUMERATION(tokens);
EXPECT_START_TAG_TOKEN(html, 1u, 5u);
EXPECT_START_TAG_TOKEN(head, 7u, 11u);
EXPECT_END_TAG_TOKEN(head, 14u, 18u);
EXPECT_START_TAG_TOKEN(body, 20u, 24u);
EXPECT_END_TAG_TOKEN(body, 27u, 31u);
EXPECT_END_TAG_TOKEN(html, 34u, 38u);
EXPECT_END_OF_FILE_TOKEN();
END_ENUMERATION();
}
TEST_CASE(basic_with_text)
{
auto tokens = run_tokenizer("<p>This is some text.</p>"sv);
BEGIN_ENUMERATION(tokens);
EXPECT_START_TAG_TOKEN(p, 1u, 2u);
EXPECT_CHARACTER_TOKENS(This is some text.);
EXPECT_END_TAG_TOKEN(p, 23u, 24u);
EXPECT_END_OF_FILE_TOKEN();
END_ENUMERATION();
}
TEST_CASE(unquoted_attributes)
{
auto tokens = run_tokenizer("<p foo=bar>"sv);
BEGIN_ENUMERATION(tokens);
EXPECT_START_TAG_TOKEN(p, 1u, 10u);
EXPECT_TAG_TOKEN_ATTRIBUTE_COUNT(1);
EXPECT_TAG_TOKEN_ATTRIBUTE(foo, "bar", 3u, 6u, 7u, 10u);
EXPECT_END_OF_FILE_TOKEN();
END_ENUMERATION();
}
TEST_CASE(single_quoted_attributes)
{
auto tokens = run_tokenizer("<p foo='bar'>"sv);
BEGIN_ENUMERATION(tokens);
EXPECT_START_TAG_TOKEN(p, 1u, 12u);
EXPECT_TAG_TOKEN_ATTRIBUTE_COUNT(1);
EXPECT_TAG_TOKEN_ATTRIBUTE(foo, "bar", 3u, 6u, 7u, 12u);
EXPECT_END_OF_FILE_TOKEN();
END_ENUMERATION();
}
TEST_CASE(double_quoted_attributes)
{
auto tokens = run_tokenizer("<p foo=\"bar\">"sv);
BEGIN_ENUMERATION(tokens);
EXPECT_START_TAG_TOKEN(p, 1u, 12u);
EXPECT_TAG_TOKEN_ATTRIBUTE_COUNT(1);
EXPECT_TAG_TOKEN_ATTRIBUTE(foo, "bar", 3u, 6u, 7u, 12u);
EXPECT_END_OF_FILE_TOKEN();
END_ENUMERATION();
}
TEST_CASE(valueless_attribute)
{
auto tokens = run_tokenizer("<p foo>"sv);
BEGIN_ENUMERATION(tokens);
EXPECT_START_TAG_TOKEN(p, 1u, 6u);
EXPECT_TAG_TOKEN_ATTRIBUTE_COUNT(1);
EXPECT_TAG_TOKEN_ATTRIBUTE(foo, "", 3u, 6u, 0u, 0u);
EXPECT_END_OF_FILE_TOKEN();
END_ENUMERATION();
}
TEST_CASE(multiple_attributes)
{
auto tokens = run_tokenizer("<p foo=\"bar\" baz=foobar biz foo2=\"bar2\">"sv);
BEGIN_ENUMERATION(tokens);
EXPECT_START_TAG_TOKEN(p, 1u, 39u);
EXPECT_TAG_TOKEN_ATTRIBUTE_COUNT(4);
EXPECT_TAG_TOKEN_ATTRIBUTE(foo, "bar", 3u, 6u, 7u, 12u);
EXPECT_TAG_TOKEN_ATTRIBUTE(baz, "foobar", 13u, 16u, 17u, 23u);
EXPECT_TAG_TOKEN_ATTRIBUTE(biz, "", 24u, 27u, 0u, 0u);
EXPECT_TAG_TOKEN_ATTRIBUTE(foo2, "bar2", 28u, 32u, 33u, 39u);
EXPECT_END_OF_FILE_TOKEN();
END_ENUMERATION();
}
TEST_CASE(character_reference_in_attribute)
{
auto tokens = run_tokenizer("<p foo=a&amp;b bar='a&#38;b' baz=\"a&#x26;b\">"sv);
BEGIN_ENUMERATION(tokens);
EXPECT_START_TAG_TOKEN(p, 1u, 43u);
EXPECT_TAG_TOKEN_ATTRIBUTE_COUNT(3);
EXPECT_TAG_TOKEN_ATTRIBUTE(foo, "a&b", 3u, 6u, 7u, 14u);
EXPECT_TAG_TOKEN_ATTRIBUTE(bar, "a&b", 15u, 18u, 19u, 28u);
EXPECT_TAG_TOKEN_ATTRIBUTE(baz, "a&b", 29u, 32u, 33u, 43u);
EXPECT_END_OF_FILE_TOKEN();
END_ENUMERATION();
}
TEST_CASE(duplicate_attributes_are_reported)
{
auto tokens = run_tokenizer("<script nonce=x nonce=y></script>"sv);
auto& token = tokens.first();
EXPECT_EQ(token.type(), Token::Type::StartTag);
EXPECT(token.had_duplicate_attribute());
EXPECT_EQ(token.attribute_count(), 1u);
auto nonce = token.raw_attribute("nonce"_fly_string);
VERIFY(nonce.has_value());
EXPECT_EQ(nonce->value, "x");
}
TEST_CASE(named_character_reference)
{
auto tokens = run_tokenizer("&notinvc;&notit;&cz"sv);
BEGIN_ENUMERATION(tokens);
EXPECT_CHARACTER_TOKEN(0x22F6); // &notinvc;
EXPECT_CHARACTER_TOKEN(0xAC); // &not (backtracked from &notit)
EXPECT_CHARACTER_TOKENS(it);
EXPECT_CHARACTER_TOKEN(';');
EXPECT_CHARACTER_TOKENS(&cz); // invalid
EXPECT_END_OF_FILE_TOKEN();
END_ENUMERATION();
}
TEST_CASE(numeric_character_reference)
{
auto tokens = run_tokenizer("&#1111"sv);
BEGIN_ENUMERATION(tokens);
EXPECT_CHARACTER_TOKEN(1111);
EXPECT_END_OF_FILE_TOKEN();
END_ENUMERATION();
}
TEST_CASE(hex_character_reference)
{
auto tokens = run_tokenizer("&#xA12bZ"sv);
BEGIN_ENUMERATION(tokens);
EXPECT_CHARACTER_TOKEN(0xA12B);
EXPECT_CHARACTER_TOKEN('Z');
EXPECT_END_OF_FILE_TOKEN();
END_ENUMERATION();
}
TEST_CASE(comment)
{
auto tokens = run_tokenizer("<p><!-- This is a comment --></p>"sv);
BEGIN_ENUMERATION(tokens);
EXPECT_START_TAG_TOKEN(p, 1u, 2u);
EXPECT_COMMENT_TOKEN();
EXPECT_END_TAG_TOKEN(p, 31u, 32u);
EXPECT_END_OF_FILE_TOKEN();
END_ENUMERATION();
}
TEST_CASE(doctype)
{
auto tokens = run_tokenizer("<!DOCTYPE html><html></html>"sv);
BEGIN_ENUMERATION(tokens);
EXPECT_DOCTYPE_TOKEN();
EXPECT_START_TAG_TOKEN(html, 16u, 20u);
EXPECT_END_TAG_TOKEN(html, 23u, 27u);
}
// NOTE: This relies on the format of HTMLToken::to_string() staying the same.
// If that changes, or something is added to the test HTML, the hash needs to be adjusted.
TEST_CASE(regression)
{
StringView path = "tokenizer-test.html"sv;
auto file = MUST(Core::File::open(path, Core::File::OpenMode::Read));
auto file_size = MUST(file->size());
auto content = MUST(ByteBuffer::create_uninitialized(file_size));
MUST(file->read_until_filled(content.bytes()));
ByteString file_contents { content.bytes() };
auto tokens = run_tokenizer(file_contents);
u32 hash = hash_tokens(tokens);
EXPECT_EQ(hash, 3657343287u);
}
TEST_CASE(ambiguous_ampersand_offset)
{
auto tokens = run_tokenizer("&a"sv);
auto& token = tokens.first();
EXPECT_EQ(token.type(), Token::Type::Character);
EXPECT_EQ(token.start_position().line, 0u);
EXPECT_EQ(token.start_position().column, 1u);
}
TEST_CASE(insertion_point_inside_fast_tag_name)
{
Tokenizer tokenizer;
tokenizer.update_insertion_point();
tokenizer.insert_input_at_insertion_point("<abc"sv);
tokenizer.append_to_input_stream("def>"sv);
EXPECT(!tokenizer.next_token(Tokenizer::StopAtInsertionPoint::Yes).has_value());
EXPECT(tokenizer.is_insertion_point_reached());
EXPECT_EQ(tokenizer.unparsed_input(), "def>"sv);
tokenizer.insert_input_at_insertion_point("x"sv);
tokenizer.undefine_insertion_point();
tokenizer.close_input_stream();
auto token = tokenizer.next_token();
VERIFY(token.has_value());
EXPECT_EQ(token->type(), Token::Type::StartTag);
EXPECT_EQ(token->tag_name(), "abcxdef");
}
TEST_CASE(insertion_point_inside_fast_attribute_name)
{
Tokenizer tokenizer;
tokenizer.update_insertion_point();
tokenizer.insert_input_at_insertion_point("<p abc"sv);
tokenizer.append_to_input_stream("def=value>"sv);
EXPECT(!tokenizer.next_token(Tokenizer::StopAtInsertionPoint::Yes).has_value());
EXPECT(tokenizer.is_insertion_point_reached());
EXPECT_EQ(tokenizer.unparsed_input(), "def=value>"sv);
tokenizer.insert_input_at_insertion_point("x"sv);
tokenizer.undefine_insertion_point();
tokenizer.close_input_stream();
auto token = tokenizer.next_token();
VERIFY(token.has_value());
EXPECT_EQ(token->type(), Token::Type::StartTag);
EXPECT_EQ(token->attribute_count(), 1u);
auto attribute = token->raw_attribute("abcxdef"_fly_string);
VERIFY(attribute.has_value());
EXPECT_EQ(attribute->value, "value");
}
TEST_CASE(insertion_point_inside_fast_quoted_attribute_value)
{
Tokenizer tokenizer;
tokenizer.update_insertion_point();
tokenizer.insert_input_at_insertion_point("<p a=\"abc"sv);
tokenizer.append_to_input_stream("def\">"sv);
EXPECT(!tokenizer.next_token(Tokenizer::StopAtInsertionPoint::Yes).has_value());
EXPECT(tokenizer.is_insertion_point_reached());
EXPECT_EQ(tokenizer.unparsed_input(), "def\">"sv);
tokenizer.insert_input_at_insertion_point("x"sv);
tokenizer.undefine_insertion_point();
tokenizer.close_input_stream();
auto token = tokenizer.next_token();
VERIFY(token.has_value());
EXPECT_EQ(token->type(), Token::Type::StartTag);
auto attribute = token->raw_attribute("a"_fly_string);
VERIFY(attribute.has_value());
EXPECT_EQ(attribute->value, "abcxdef");
}
Reference in a new issue View git blame Copy permalink