dmaax/ladybird
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

LibWeb: Add HTML tokenizer dump tests

Browse source 
Add a dump-html-tokens utility that prints HTML tokenizer output in a
stable text format, along with focused tokenizer input and expectation
files.

Include silent and repeated-iteration modes so the utility can also be
used for local tokenizer throughput checks without changing the dumped
token format.
This commit is contained in:
Andreas Kling 2026-05-15 14:50:17 +02:00 committed by Alexander Kalenik
parent d595369ae4
commit 54a172f3b5
22 changed files with 260 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
Tests/LibWeb/Tokenizer/expected/attributes.txt Normal file
View file

@ -0,0 +1,4 @@
StartTag <div> [id="main" class="container" data-value="foo" hidden=""] @0:1-0:54
EndTag </div> @0:57-0:60
Character U+000A @1:0
EndOfFile @1:0

8
Tests/LibWeb/Tokenizer/expected/basic-tags.txt Normal file
View file

@ -0,0 +1,8 @@
StartTag <html> [] @0:1-0:5
StartTag <head> [] @0:7-0:11
EndTag </head> @0:14-0:18
StartTag <body> [] @0:20-0:24
EndTag </body> @0:27-0:31
EndTag </html> @0:34-0:38
Character U+000A @1:0
EndOfFile @1:0

3
Tests/LibWeb/Tokenizer/expected/cdata.txt Normal file
View file

@ -0,0 +1,3 @@
Comment "[CDATA[some cdata content]]" @0:9-0:30
Character U+000A @1:0
EndOfFile @1:0

11
Tests/LibWeb/Tokenizer/expected/character-references.txt Normal file
View file

@ -0,0 +1,11 @@
Character U+0026 '&' @0:5
Character U+003C '<' @0:9
Character U+003E '>' @0:13
Character U+0022 '"' @0:19
Character U+0027 ''' @0:25
Character U+0041 'A' @0:30
Character U+0041 'A' @0:36
Character U+22F6 @0:45
Character U+00AC @0:50
Character U+000A @1:0
EndOfFile @1:0

10
Tests/LibWeb/Tokenizer/expected/comments.txt Normal file
View file

@ -0,0 +1,10 @@
Comment " simple comment " @0:0-0:23
Character U+000A @1:0
Comment "" @1:0-1:7
Character U+000A @2:0
Comment "-" @2:0-2:8
Character U+000A @3:0
Comment " multi
line comment " @3:0-4:16
Character U+000A @5:0
EndOfFile @5:0

5
Tests/LibWeb/Tokenizer/expected/doctype.txt Normal file
View file

@ -0,0 +1,5 @@
DOCTYPE name="html" @0:11-0:15
Character U+000A @1:0
DOCTYPE name="html" public_id="-//W3C//DTD HTML 4.01//EN" system_id="http://www.w3.org/TR/html4/strict.dtd" @1:11-1:90
Character U+000A @2:0
EndOfFile @2:0

14
Tests/LibWeb/Tokenizer/expected/nested-tags.txt Normal file
View file

@ -0,0 +1,14 @@
StartTag <div> [] @0:1-0:4
StartTag <p> [] @0:6-0:7
Character U+006F 'o' @0:9
Character U+006E 'n' @0:10
Character U+0065 'e' @0:11
EndTag </p> @0:13-0:14
StartTag <p> [] @0:16-0:17
Character U+0074 't' @0:19
Character U+0077 'w' @0:20
Character U+006F 'o' @0:21
EndTag </p> @0:23-0:24
EndTag </div> @0:27-0:30
Character U+000A @1:0
EndOfFile @1:0

12
Tests/LibWeb/Tokenizer/expected/numeric-char-refs.txt Normal file
View file

@ -0,0 +1,12 @@
Character U+0009 @0:4
Character U+000A @0:9
Character U+000D @0:14
Character U+0020 ' ' @0:19
Character U+007F @0:25
Character U+20AC @0:31
Character U+0178 @0:37
Character U+FFFD @0:41
Character U+10FFFF @0:51
Character U+FFFD @0:61
Character U+000A @1:0
EndOfFile @1:0

5
Tests/LibWeb/Tokenizer/expected/self-closing.txt Normal file
View file

@ -0,0 +1,5 @@
StartTag <br/> [] @0:1-0:4
StartTag <hr/> [] @0:6-0:10
StartTag <img/> [src="x.png"] @0:12-0:28
Character U+000A @1:0
EndOfFile @1:0

17
Tests/LibWeb/Tokenizer/expected/text-content.txt Normal file
View file

@ -0,0 +1,17 @@
StartTag <p> [] @0:1-0:2
Character U+0048 'H' @0:4
Character U+0065 'e' @0:5
Character U+006C 'l' @0:6
Character U+006C 'l' @0:7
Character U+006F 'o' @0:8
Character U+002C ',' @0:9
Character U+0020 ' ' @0:10
Character U+0077 'w' @0:11
Character U+006F 'o' @0:12
Character U+0072 'r' @0:13
Character U+006C 'l' @0:14
Character U+0064 'd' @0:15
Character U+0021 '!' @0:16
EndTag </p> @0:18-0:19
Character U+000A @1:0
EndOfFile @1:0

1
Tests/LibWeb/Tokenizer/input/attributes.html Normal file
View file

@ -0,0 +1 @@
<div id="main" class='container' data-value=foo hidden></div>

1
Tests/LibWeb/Tokenizer/input/basic-tags.html Normal file
View file

@ -0,0 +1 @@
<html><head></head><body></body></html>

1
Tests/LibWeb/Tokenizer/input/cdata.html Normal file
View file

@ -0,0 +1 @@
<![CDATA[some cdata content]]>

1
Tests/LibWeb/Tokenizer/input/character-references.html Normal file
View file

@ -0,0 +1 @@
&amp;&lt;&gt;&quot;&apos;&#65;&#x41;&notinvc;&not;

5
Tests/LibWeb/Tokenizer/input/comments.html Normal file
View file

@ -0,0 +1,5 @@
<!-- simple comment -->
<!---->
<!----->
<!-- multi
line comment -->

2
Tests/LibWeb/Tokenizer/input/doctype.html Normal file
View file

@ -0,0 +1,2 @@
<!DOCTYPE html>
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd">

1
Tests/LibWeb/Tokenizer/input/nested-tags.html Normal file
View file

@ -0,0 +1 @@
<div><p>one</p><p>two</p></div>

1
Tests/LibWeb/Tokenizer/input/numeric-char-refs.html Normal file
View file

@ -0,0 +1 @@
&#9;&#10;&#13;&#32;&#127;&#128;&#159;&#0;&#x10FFFF;&#x110000;

1
Tests/LibWeb/Tokenizer/input/self-closing.html Normal file
View file

@ -0,0 +1 @@
<br/><hr /><img src="x.png"/>

1
Tests/LibWeb/Tokenizer/input/text-content.html Normal file
View file

@ -0,0 +1 @@
<p>Hello, world!</p>

1
Utilities/CMakeLists.txt
View file

@ -20,6 +20,7 @@ if (WIN32)
endif()
ladybird_utility(test262-runner SOURCES test262-runner.cpp LIBS LibJS LibFileSystem LibGC)
ladybird_utility(dump-html-tokens SOURCES dump-html-tokens.cpp LIBS LibMain LibWeb)
if (CMAKE_SYSTEM_NAME STREQUAL "Linux")
include(CheckCSourceCompiles)

155
Utilities/dump-html-tokens.cpp Normal file
View file

@ -0,0 +1,155 @@
/*
* Copyright (c) 2026-present, the Ladybird developers.
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#include <AK/ByteString.h>
#include <LibCore/ArgsParser.h>
#include <LibCore/ElapsedTimer.h>
#include <LibCore/File.h>
#include <LibMain/Main.h>
#include <LibWeb/HTML/Parser/HTMLToken.h>
#include <LibWeb/HTML/Parser/HTMLTokenizer.h>
using Web::HTML::HTMLToken;
using Web::HTML::HTMLTokenizer;
static String format_position(HTMLToken::Position const& pos)
{
return MUST(String::formatted("{}:{}", pos.line, pos.column));
}
static void dump_token(HTMLToken const& token)
{
switch (token.type()) {
case HTMLToken::Type::DOCTYPE: {
auto const& doctype = token.doctype_data();
out("DOCTYPE");
if (!doctype.missing_name)
out(" name=\"{}\"", doctype.name);
if (!doctype.missing_public_identifier)
out(" public_id=\"{}\"", doctype.public_identifier);
if (!doctype.missing_system_identifier)
out(" system_id=\"{}\"", doctype.system_identifier);
if (doctype.force_quirks)
out(" force_quirks");
outln(" @{}-{}", format_position(token.start_position()), format_position(token.end_position()));
break;
}
case HTMLToken::Type::StartTag: {
if (token.is_self_closing())
out("StartTag <{}/> [", token.tag_name());
else
out("StartTag <{}> [", token.tag_name());
bool first = true;
token.for_each_attribute([&](auto& attribute) {
if (!first)
out(" ");
first = false;
out("{}=\"{}\"", attribute.local_name, attribute.value);
return IterationDecision::Continue;
});
outln("] @{}-{}", format_position(token.start_position()), format_position(token.end_position()));
break;
}
case HTMLToken::Type::EndTag:
outln("EndTag </{}> @{}-{}", token.tag_name(), format_position(token.start_position()), format_position(token.end_position()));
break;
case HTMLToken::Type::Comment:
outln("Comment \"{}\" @{}-{}", token.comment(), format_position(token.start_position()), format_position(token.end_position()));
break;
case HTMLToken::Type::Character: {
auto code_point = token.code_point();
if (code_point >= 0x20 && code_point < 0x7F)
outln("Character U+{:04X} '{:c}' @{}", code_point, static_cast<char>(code_point), format_position(token.start_position()));
else
outln("Character U+{:04X} @{}", code_point, format_position(token.start_position()));
break;
}
case HTMLToken::Type::EndOfFile:
outln("EndOfFile @{}", format_position(token.start_position()));
break;
case HTMLToken::Type::Invalid:
VERIFY_NOT_REACHED();
}
}
ErrorOr<int> ladybird_main(Main::Arguments arguments)
{
StringView file_path;
StringView initial_state_name;
bool silent = false;
int iterations = 1;
Core::ArgsParser args_parser;
args_parser.set_general_help(
"Tokenize HTML and dump tokens in a canonical format. "
"Use --silent and --iterations for perf work on the tokenizer itself.");
args_parser.add_positional_argument(file_path, "Path to HTML file (or - for stdin)", "file", Core::ArgsParser::Required::No);
args_parser.add_option(initial_state_name, "Initial tokenizer state", "initial-state", 's', "state");
args_parser.add_option(silent, "Don't print tokens (for benchmarking)", "silent", 'q');
args_parser.add_option(iterations, "Run the tokenizer N times and report timing", "iterations", 'n', "count");
args_parser.parse(arguments);
if (iterations < 1) {
warnln("--iterations must be at least 1");
return 1;
}
Optional<HTMLTokenizer::State> initial_state;
if (!initial_state_name.is_empty()) {
#define __ENUMERATE_TOKENIZER_STATE(s) \
if (initial_state_name == #s##sv) \
initial_state = HTMLTokenizer::State::s;
ENUMERATE_TOKENIZER_STATES
#undef __ENUMERATE_TOKENIZER_STATE
if (!initial_state.has_value()) {
warnln("Unknown tokenizer state: '{}'", initial_state_name);
return 1;
}
}
ByteBuffer input_data;
if (file_path.is_empty() || file_path == "-"sv) {
auto stdin_file = TRY(Core::File::standard_input());
input_data = TRY(stdin_file->read_until_eof());
} else {
auto file = TRY(Core::File::open(file_path, Core::File::OpenMode::Read));
input_data = TRY(file->read_until_eof());
}
StringView input { input_data };
auto timer = Core::ElapsedTimer::start_new();
u64 total_tokens = 0;
for (int i = 0; i < iterations; i++) {
HTMLTokenizer tokenizer { input, "UTF-8"sv };
if (initial_state.has_value())
tokenizer.switch_to(initial_state.value());
while (true) {
auto maybe_token = tokenizer.next_token();
if (!maybe_token.has_value())
break;
++total_tokens;
if (!silent && i == 0)
dump_token(maybe_token.value());
if (maybe_token->is_end_of_file())
break;
}
}
if (iterations > 1 || silent) {
auto elapsed_ms = timer.elapsed_milliseconds();
warnln("input={}B iterations={} tokens/iter={} total={}ms ({:.3f}ms/iter)",
input_data.size(),
iterations,
total_tokens / iterations,
elapsed_ms,
static_cast<double>(elapsed_ms) / iterations);
}
return 0;
}