diff --git a/Tests/LibWeb/Tokenizer/expected/attributes.txt b/Tests/LibWeb/Tokenizer/expected/attributes.txt new file mode 100644 index 0000000000..131fc9114a --- /dev/null +++ b/Tests/LibWeb/Tokenizer/expected/attributes.txt @@ -0,0 +1,4 @@ +StartTag
[id="main" class="container" data-value="foo" hidden=""] @0:1-0:54 +EndTag
@0:57-0:60 +Character U+000A @1:0 +EndOfFile @1:0 diff --git a/Tests/LibWeb/Tokenizer/expected/basic-tags.txt b/Tests/LibWeb/Tokenizer/expected/basic-tags.txt new file mode 100644 index 0000000000..7d4b063aba --- /dev/null +++ b/Tests/LibWeb/Tokenizer/expected/basic-tags.txt @@ -0,0 +1,8 @@ +StartTag [] @0:1-0:5 +StartTag [] @0:7-0:11 +EndTag @0:14-0:18 +StartTag [] @0:20-0:24 +EndTag @0:27-0:31 +EndTag @0:34-0:38 +Character U+000A @1:0 +EndOfFile @1:0 diff --git a/Tests/LibWeb/Tokenizer/expected/cdata.txt b/Tests/LibWeb/Tokenizer/expected/cdata.txt new file mode 100644 index 0000000000..8cf3392373 --- /dev/null +++ b/Tests/LibWeb/Tokenizer/expected/cdata.txt @@ -0,0 +1,3 @@ +Comment "[CDATA[some cdata content]]" @0:9-0:30 +Character U+000A @1:0 +EndOfFile @1:0 diff --git a/Tests/LibWeb/Tokenizer/expected/character-references.txt b/Tests/LibWeb/Tokenizer/expected/character-references.txt new file mode 100644 index 0000000000..b58e5d2f41 --- /dev/null +++ b/Tests/LibWeb/Tokenizer/expected/character-references.txt @@ -0,0 +1,11 @@ +Character U+0026 '&' @0:5 +Character U+003C '<' @0:9 +Character U+003E '>' @0:13 +Character U+0022 '"' @0:19 +Character U+0027 ''' @0:25 +Character U+0041 'A' @0:30 +Character U+0041 'A' @0:36 +Character U+22F6 @0:45 +Character U+00AC @0:50 +Character U+000A @1:0 +EndOfFile @1:0 diff --git a/Tests/LibWeb/Tokenizer/expected/comments.txt b/Tests/LibWeb/Tokenizer/expected/comments.txt new file mode 100644 index 0000000000..311483d8a4 --- /dev/null +++ b/Tests/LibWeb/Tokenizer/expected/comments.txt @@ -0,0 +1,10 @@ +Comment " simple comment " @0:0-0:23 +Character U+000A @1:0 +Comment "" @1:0-1:7 +Character U+000A @2:0 +Comment "-" @2:0-2:8 +Character U+000A @3:0 +Comment " multi +line comment " @3:0-4:16 +Character U+000A @5:0 +EndOfFile @5:0 diff --git a/Tests/LibWeb/Tokenizer/expected/doctype.txt b/Tests/LibWeb/Tokenizer/expected/doctype.txt new file mode 100644 index 0000000000..06e5d95d17 --- /dev/null +++ b/Tests/LibWeb/Tokenizer/expected/doctype.txt @@ -0,0 +1,5 @@ +DOCTYPE name="html" @0:11-0:15 +Character U+000A @1:0 +DOCTYPE name="html" public_id="-//W3C//DTD HTML 4.01//EN" system_id="http://www.w3.org/TR/html4/strict.dtd" @1:11-1:90 +Character U+000A @2:0 +EndOfFile @2:0 diff --git a/Tests/LibWeb/Tokenizer/expected/nested-tags.txt b/Tests/LibWeb/Tokenizer/expected/nested-tags.txt new file mode 100644 index 0000000000..e04b4b8e8e --- /dev/null +++ b/Tests/LibWeb/Tokenizer/expected/nested-tags.txt @@ -0,0 +1,14 @@ +StartTag
[] @0:1-0:4 +StartTag

[] @0:6-0:7 +Character U+006F 'o' @0:9 +Character U+006E 'n' @0:10 +Character U+0065 'e' @0:11 +EndTag

@0:13-0:14 +StartTag

[] @0:16-0:17 +Character U+0074 't' @0:19 +Character U+0077 'w' @0:20 +Character U+006F 'o' @0:21 +EndTag

@0:23-0:24 +EndTag
@0:27-0:30 +Character U+000A @1:0 +EndOfFile @1:0 diff --git a/Tests/LibWeb/Tokenizer/expected/numeric-char-refs.txt b/Tests/LibWeb/Tokenizer/expected/numeric-char-refs.txt new file mode 100644 index 0000000000..78dab39cdb --- /dev/null +++ b/Tests/LibWeb/Tokenizer/expected/numeric-char-refs.txt @@ -0,0 +1,12 @@ +Character U+0009 @0:4 +Character U+000A @0:9 +Character U+000D @0:14 +Character U+0020 ' ' @0:19 +Character U+007F @0:25 +Character U+20AC @0:31 +Character U+0178 @0:37 +Character U+FFFD @0:41 +Character U+10FFFF @0:51 +Character U+FFFD @0:61 +Character U+000A @1:0 +EndOfFile @1:0 diff --git a/Tests/LibWeb/Tokenizer/expected/self-closing.txt b/Tests/LibWeb/Tokenizer/expected/self-closing.txt new file mode 100644 index 0000000000..125601644b --- /dev/null +++ b/Tests/LibWeb/Tokenizer/expected/self-closing.txt @@ -0,0 +1,5 @@ +StartTag
[] @0:1-0:4 +StartTag
[] @0:6-0:10 +StartTag [src="x.png"] @0:12-0:28 +Character U+000A @1:0 +EndOfFile @1:0 diff --git a/Tests/LibWeb/Tokenizer/expected/text-content.txt b/Tests/LibWeb/Tokenizer/expected/text-content.txt new file mode 100644 index 0000000000..b75c9ce2e9 --- /dev/null +++ b/Tests/LibWeb/Tokenizer/expected/text-content.txt @@ -0,0 +1,17 @@ +StartTag

[] @0:1-0:2 +Character U+0048 'H' @0:4 +Character U+0065 'e' @0:5 +Character U+006C 'l' @0:6 +Character U+006C 'l' @0:7 +Character U+006F 'o' @0:8 +Character U+002C ',' @0:9 +Character U+0020 ' ' @0:10 +Character U+0077 'w' @0:11 +Character U+006F 'o' @0:12 +Character U+0072 'r' @0:13 +Character U+006C 'l' @0:14 +Character U+0064 'd' @0:15 +Character U+0021 '!' @0:16 +EndTag

@0:18-0:19 +Character U+000A @1:0 +EndOfFile @1:0 diff --git a/Tests/LibWeb/Tokenizer/input/attributes.html b/Tests/LibWeb/Tokenizer/input/attributes.html new file mode 100644 index 0000000000..cb42a95b60 --- /dev/null +++ b/Tests/LibWeb/Tokenizer/input/attributes.html @@ -0,0 +1 @@ + diff --git a/Tests/LibWeb/Tokenizer/input/basic-tags.html b/Tests/LibWeb/Tokenizer/input/basic-tags.html new file mode 100644 index 0000000000..30e84fdd6e --- /dev/null +++ b/Tests/LibWeb/Tokenizer/input/basic-tags.html @@ -0,0 +1 @@ + diff --git a/Tests/LibWeb/Tokenizer/input/cdata.html b/Tests/LibWeb/Tokenizer/input/cdata.html new file mode 100644 index 0000000000..6eab34b634 --- /dev/null +++ b/Tests/LibWeb/Tokenizer/input/cdata.html @@ -0,0 +1 @@ + diff --git a/Tests/LibWeb/Tokenizer/input/character-references.html b/Tests/LibWeb/Tokenizer/input/character-references.html new file mode 100644 index 0000000000..d977a81305 --- /dev/null +++ b/Tests/LibWeb/Tokenizer/input/character-references.html @@ -0,0 +1 @@ +&<>"'AA⋶¬ diff --git a/Tests/LibWeb/Tokenizer/input/comments.html b/Tests/LibWeb/Tokenizer/input/comments.html new file mode 100644 index 0000000000..6bba73561a --- /dev/null +++ b/Tests/LibWeb/Tokenizer/input/comments.html @@ -0,0 +1,5 @@ + + + + diff --git a/Tests/LibWeb/Tokenizer/input/doctype.html b/Tests/LibWeb/Tokenizer/input/doctype.html new file mode 100644 index 0000000000..4d258d505c --- /dev/null +++ b/Tests/LibWeb/Tokenizer/input/doctype.html @@ -0,0 +1,2 @@ + + diff --git a/Tests/LibWeb/Tokenizer/input/nested-tags.html b/Tests/LibWeb/Tokenizer/input/nested-tags.html new file mode 100644 index 0000000000..45a403cb53 --- /dev/null +++ b/Tests/LibWeb/Tokenizer/input/nested-tags.html @@ -0,0 +1 @@ +

one

two

diff --git a/Tests/LibWeb/Tokenizer/input/numeric-char-refs.html b/Tests/LibWeb/Tokenizer/input/numeric-char-refs.html new file mode 100644 index 0000000000..8804713f5b --- /dev/null +++ b/Tests/LibWeb/Tokenizer/input/numeric-char-refs.html @@ -0,0 +1 @@ + €Ÿ�􏿿� diff --git a/Tests/LibWeb/Tokenizer/input/self-closing.html b/Tests/LibWeb/Tokenizer/input/self-closing.html new file mode 100644 index 0000000000..5a4101f0a8 --- /dev/null +++ b/Tests/LibWeb/Tokenizer/input/self-closing.html @@ -0,0 +1 @@ +
diff --git a/Tests/LibWeb/Tokenizer/input/text-content.html b/Tests/LibWeb/Tokenizer/input/text-content.html new file mode 100644 index 0000000000..7ce535433d --- /dev/null +++ b/Tests/LibWeb/Tokenizer/input/text-content.html @@ -0,0 +1 @@ +

Hello, world!

diff --git a/Utilities/CMakeLists.txt b/Utilities/CMakeLists.txt index a600054758..b41553d48c 100644 --- a/Utilities/CMakeLists.txt +++ b/Utilities/CMakeLists.txt @@ -20,6 +20,7 @@ if (WIN32) endif() ladybird_utility(test262-runner SOURCES test262-runner.cpp LIBS LibJS LibFileSystem LibGC) +ladybird_utility(dump-html-tokens SOURCES dump-html-tokens.cpp LIBS LibMain LibWeb) if (CMAKE_SYSTEM_NAME STREQUAL "Linux") include(CheckCSourceCompiles) diff --git a/Utilities/dump-html-tokens.cpp b/Utilities/dump-html-tokens.cpp new file mode 100644 index 0000000000..6f5a5ef055 --- /dev/null +++ b/Utilities/dump-html-tokens.cpp @@ -0,0 +1,155 @@ +/* + * Copyright (c) 2026-present, the Ladybird developers. + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include +#include +#include +#include +#include +#include +#include + +using Web::HTML::HTMLToken; +using Web::HTML::HTMLTokenizer; + +static String format_position(HTMLToken::Position const& pos) +{ + return MUST(String::formatted("{}:{}", pos.line, pos.column)); +} + +static void dump_token(HTMLToken const& token) +{ + switch (token.type()) { + case HTMLToken::Type::DOCTYPE: { + auto const& doctype = token.doctype_data(); + out("DOCTYPE"); + if (!doctype.missing_name) + out(" name=\"{}\"", doctype.name); + if (!doctype.missing_public_identifier) + out(" public_id=\"{}\"", doctype.public_identifier); + if (!doctype.missing_system_identifier) + out(" system_id=\"{}\"", doctype.system_identifier); + if (doctype.force_quirks) + out(" force_quirks"); + outln(" @{}-{}", format_position(token.start_position()), format_position(token.end_position())); + break; + } + case HTMLToken::Type::StartTag: { + if (token.is_self_closing()) + out("StartTag <{}/> [", token.tag_name()); + else + out("StartTag <{}> [", token.tag_name()); + bool first = true; + token.for_each_attribute([&](auto& attribute) { + if (!first) + out(" "); + first = false; + out("{}=\"{}\"", attribute.local_name, attribute.value); + return IterationDecision::Continue; + }); + outln("] @{}-{}", format_position(token.start_position()), format_position(token.end_position())); + break; + } + case HTMLToken::Type::EndTag: + outln("EndTag @{}-{}", token.tag_name(), format_position(token.start_position()), format_position(token.end_position())); + break; + case HTMLToken::Type::Comment: + outln("Comment \"{}\" @{}-{}", token.comment(), format_position(token.start_position()), format_position(token.end_position())); + break; + case HTMLToken::Type::Character: { + auto code_point = token.code_point(); + if (code_point >= 0x20 && code_point < 0x7F) + outln("Character U+{:04X} '{:c}' @{}", code_point, static_cast(code_point), format_position(token.start_position())); + else + outln("Character U+{:04X} @{}", code_point, format_position(token.start_position())); + break; + } + case HTMLToken::Type::EndOfFile: + outln("EndOfFile @{}", format_position(token.start_position())); + break; + case HTMLToken::Type::Invalid: + VERIFY_NOT_REACHED(); + } +} + +ErrorOr ladybird_main(Main::Arguments arguments) +{ + StringView file_path; + StringView initial_state_name; + bool silent = false; + int iterations = 1; + + Core::ArgsParser args_parser; + args_parser.set_general_help( + "Tokenize HTML and dump tokens in a canonical format. " + "Use --silent and --iterations for perf work on the tokenizer itself."); + args_parser.add_positional_argument(file_path, "Path to HTML file (or - for stdin)", "file", Core::ArgsParser::Required::No); + args_parser.add_option(initial_state_name, "Initial tokenizer state", "initial-state", 's', "state"); + args_parser.add_option(silent, "Don't print tokens (for benchmarking)", "silent", 'q'); + args_parser.add_option(iterations, "Run the tokenizer N times and report timing", "iterations", 'n', "count"); + args_parser.parse(arguments); + + if (iterations < 1) { + warnln("--iterations must be at least 1"); + return 1; + } + + Optional initial_state; + if (!initial_state_name.is_empty()) { +#define __ENUMERATE_TOKENIZER_STATE(s) \ + if (initial_state_name == #s##sv) \ + initial_state = HTMLTokenizer::State::s; + ENUMERATE_TOKENIZER_STATES +#undef __ENUMERATE_TOKENIZER_STATE + if (!initial_state.has_value()) { + warnln("Unknown tokenizer state: '{}'", initial_state_name); + return 1; + } + } + + ByteBuffer input_data; + if (file_path.is_empty() || file_path == "-"sv) { + auto stdin_file = TRY(Core::File::standard_input()); + input_data = TRY(stdin_file->read_until_eof()); + } else { + auto file = TRY(Core::File::open(file_path, Core::File::OpenMode::Read)); + input_data = TRY(file->read_until_eof()); + } + + StringView input { input_data }; + + auto timer = Core::ElapsedTimer::start_new(); + u64 total_tokens = 0; + + for (int i = 0; i < iterations; i++) { + HTMLTokenizer tokenizer { input, "UTF-8"sv }; + if (initial_state.has_value()) + tokenizer.switch_to(initial_state.value()); + + while (true) { + auto maybe_token = tokenizer.next_token(); + if (!maybe_token.has_value()) + break; + ++total_tokens; + if (!silent && i == 0) + dump_token(maybe_token.value()); + if (maybe_token->is_end_of_file()) + break; + } + } + + if (iterations > 1 || silent) { + auto elapsed_ms = timer.elapsed_milliseconds(); + warnln("input={}B iterations={} tokens/iter={} total={}ms ({:.3f}ms/iter)", + input_data.size(), + iterations, + total_tokens / iterations, + elapsed_ms, + static_cast(elapsed_ms) / iterations); + } + + return 0; +}