From 171e3adf0171cb040687c3c122be3377936f5b37 Mon Sep 17 00:00:00 2001 From: Andreas Kling Date: Fri, 15 May 2026 15:13:43 +0200 Subject: [PATCH] LibWeb: Replace the HTML tokenizer with Rust Replace the C++ HTML tokenizer with a Rust implementation behind the existing HTMLTokenizer API. Keep the parser-facing integration points for streaming input, insertion points, document.write(), EOF insertion, parser aborts, speculative parser input, and last start tag tracking. The generated FFI handle stays an implementation detail of HTMLTokenizer, so callers keep a single tokenizer class. Preserve duplicate attributes through FFI so C++ token normalization can record the duplicate-attribute signal used by CSP nonce checks. Keep bulk tag-name and attribute scans capped at the active insertion point so streamed parser input is spliced at the right offset. Use generated DAFSA tables for named character references and intern common tag and attribute names to reduce FFI marshalling overhead. This also fixes attribute name source positions, nested old insertion points, and aborted fast-path handling. TestHTMLTokenizer covers duplicate attributes and insertion points in fast tag-name, attribute-name, and quoted-value scans. A CSP text test covers duplicate nonce attributes on parser-created script elements. The tokenizer dump fixtures still match, TestHTMLTokenizer passes, and the full release test-web run passes with 6981 tests and 226 skipped. --- Cargo.lock | 8 + Cargo.toml | 1 + Libraries/LibWeb/CMakeLists.txt | 3 +- .../LibWeb/HTML/Parser/HTMLTokenizer.cpp | 3197 ++--------------- Libraries/LibWeb/HTML/Parser/HTMLTokenizer.h | 91 +- Libraries/LibWeb/HTML/Parser/Rust/Cargo.toml | 10 + Libraries/LibWeb/HTML/Parser/Rust/build.rs | 821 +++++ .../LibWeb/HTML/Parser/Rust/cbindgen.toml | 19 + .../LibWeb/HTML/Parser/Rust/src/entities.rs | 151 + .../HTML/Parser/Rust/src/interned_names.rs | 61 + Libraries/LibWeb/HTML/Parser/Rust/src/lib.rs | 702 ++++ .../LibWeb/HTML/Parser/Rust/src/token.rs | 202 ++ .../LibWeb/HTML/Parser/Rust/src/tokenizer.rs | 3152 ++++++++++++++++ Libraries/LibWeb/Rust/Cargo.toml | 1 + Libraries/LibWeb/Rust/src/lib.rs | 2 + Tests/LibWeb/TestHTMLTokenizer.cpp | 83 + .../script-duplicate-nonce-attribute.txt | 2 + .../script-duplicate-nonce-attribute.html | 15 + 18 files changed, 5465 insertions(+), 3056 deletions(-) create mode 100644 Libraries/LibWeb/HTML/Parser/Rust/Cargo.toml create mode 100644 Libraries/LibWeb/HTML/Parser/Rust/build.rs create mode 100644 Libraries/LibWeb/HTML/Parser/Rust/cbindgen.toml create mode 100644 Libraries/LibWeb/HTML/Parser/Rust/src/entities.rs create mode 100644 Libraries/LibWeb/HTML/Parser/Rust/src/interned_names.rs create mode 100644 Libraries/LibWeb/HTML/Parser/Rust/src/lib.rs create mode 100644 Libraries/LibWeb/HTML/Parser/Rust/src/token.rs create mode 100644 Libraries/LibWeb/HTML/Parser/Rust/src/tokenizer.rs create mode 100644 Tests/LibWeb/Text/expected/ContentSecurityPolicy/script-duplicate-nonce-attribute.txt create mode 100644 Tests/LibWeb/Text/input/ContentSecurityPolicy/script-duplicate-nonce-attribute.html diff --git a/Cargo.lock b/Cargo.lock index 58bf1506fc..14dca35439 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -585,11 +585,19 @@ dependencies = [ "target-lexicon", ] +[[package]] +name = "libweb_html_tokenizer" +version = "0.1.0" +dependencies = [ + "cbindgen", +] + [[package]] name = "libweb_rust" version = "0.1.0" dependencies = [ "cbindgen", + "libweb_html_tokenizer", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 0140f68dad..1640bcabbe 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,6 +6,7 @@ members = [ "Libraries/LibUnicode/Rust", "Libraries/LibWasm/Rust", "Libraries/LibWeb/Rust", + "Libraries/LibWeb/HTML/Parser/Rust", ] exclude = [ "Libraries/LibJS/AsmIntGen", diff --git a/Libraries/LibWeb/CMakeLists.txt b/Libraries/LibWeb/CMakeLists.txt index 2b05d1c312..87adff4380 100644 --- a/Libraries/LibWeb/CMakeLists.txt +++ b/Libraries/LibWeb/CMakeLists.txt @@ -1253,6 +1253,8 @@ target_link_libraries(LibWeb PRIVATE LibCore LibCompress LibCrypto LibJS LibHTTP import_rust_crate(MANIFEST_PATH Rust/Cargo.toml CRATE_NAME libweb_rust FFI_HEADER RustFFI.h) target_link_libraries(LibWeb PRIVATE libweb_rust) +get_target_property(_libweb_rust_lib libweb_rust IMPORTED_LOCATION) +set_property(SOURCE HTML/Parser/HTMLTokenizer.cpp APPEND PROPERTY OBJECT_DEPENDS ${_libweb_rust_lib}) if ((LINUX OR BSD) AND NOT BUILD_SHARED_LIBS) target_link_options(LibWeb INTERFACE LINKER:--allow-multiple-definition) endif() @@ -1273,7 +1275,6 @@ if(NOT BUILD_SHARED_LIBS) # stale Rust objects. Force the C++ FFI bridge file to depend on the # Rust archive: when it changes, RustTokenizer.cpp recompiles, LibWeb # re-archives, and POST_BUILD re-merges the fresh Rust objects. - get_target_property(_libweb_rust_lib libweb_rust IMPORTED_LOCATION) set_property(SOURCE CSS/Parser/RustTokenizer.cpp APPEND PROPERTY OBJECT_DEPENDS ${_libweb_rust_lib}) endif() diff --git a/Libraries/LibWeb/HTML/Parser/HTMLTokenizer.cpp b/Libraries/LibWeb/HTML/Parser/HTMLTokenizer.cpp index 37753c7319..7f294062bd 100644 --- a/Libraries/LibWeb/HTML/Parser/HTMLTokenizer.cpp +++ b/Libraries/LibWeb/HTML/Parser/HTMLTokenizer.cpp @@ -5,2918 +5,92 @@ * SPDX-License-Identifier: BSD-2-Clause */ -#include #include -#include -#include -#include -#include +#include +#include #include -#include +#include #include #include #include +#include #include -#include namespace Web::HTML { -#pragma GCC diagnostic ignored "-Wunused-label" - -#define CONSUME_NEXT_INPUT_CHARACTER \ - current_input_character = next_code_point(stop_at_insertion_point); - -#define SWITCH_TO(new_state) \ - do { \ - VERIFY(m_current_builder.is_empty()); \ - SWITCH_TO_WITH_UNCLEAN_BUILDER(new_state); \ - } while (0) - -#define SWITCH_TO_WITH_UNCLEAN_BUILDER(new_state) \ - do { \ - will_switch_to(State::new_state); \ - m_state = State::new_state; \ - if (should_pause_before_next_input_character(stop_at_insertion_point)) \ - return {}; \ - CONSUME_NEXT_INPUT_CHARACTER; \ - goto new_state; \ - } while (0) - -#define RECONSUME_IN(new_state) \ - do { \ - will_reconsume_in(State::new_state); \ - m_state = State::new_state; \ - goto new_state; \ - } while (0) - -#define SWITCH_TO_RETURN_STATE \ - do { \ - will_switch_to(m_return_state); \ - m_state = m_return_state; \ - goto _StartOfFunction; \ - } while (0) - -#define RECONSUME_IN_RETURN_STATE \ - do { \ - will_reconsume_in(m_return_state); \ - m_state = m_return_state; \ - if (current_input_character.has_value()) \ - restore_to(m_prev_offset); \ - goto _StartOfFunction; \ - } while (0) - -#define SWITCH_TO_AND_EMIT_CURRENT_TOKEN(new_state) \ - do { \ - VERIFY(m_current_builder.is_empty()); \ - will_switch_to(State::new_state); \ - m_state = State::new_state; \ - will_emit(m_current_token); \ - m_queued_tokens.enqueue(move(m_current_token)); \ - return m_queued_tokens.dequeue(); \ - } while (0) - -#define EMIT_CHARACTER_AND_RECONSUME_IN(code_point, new_state) \ - do { \ - m_queued_tokens.enqueue(HTMLToken::make_character(code_point)); \ - will_reconsume_in(State::new_state); \ - m_state = State::new_state; \ - goto new_state; \ - } while (0) - -#define FLUSH_CODEPOINTS_CONSUMED_AS_A_CHARACTER_REFERENCE \ - do { \ - for (auto code_point : m_temporary_buffer) { \ - if (consumed_as_part_of_an_attribute()) { \ - m_current_builder.append_code_point(code_point); \ - } else { \ - create_new_token(HTMLToken::Type::Character); \ - m_current_token.set_code_point(code_point); \ - m_queued_tokens.enqueue(move(m_current_token)); \ - } \ - } \ - } while (0) - -#define DONT_CONSUME_NEXT_INPUT_CHARACTER \ - do { \ - if (current_input_character.has_value()) \ - restore_to(m_prev_offset); \ - } while (0) - -#define ON(code_point) \ - if (current_input_character.has_value() && current_input_character.value() == code_point) - -#define ON_EOF \ - if (!current_input_character.has_value()) - -#define ON_ASCII_ALPHA \ - if (current_input_character.has_value() && is_ascii_alpha(current_input_character.value())) - -#define ON_ASCII_ALPHANUMERIC \ - if (current_input_character.has_value() && is_ascii_alphanumeric(current_input_character.value())) - -#define ON_ASCII_UPPER_ALPHA \ - if (current_input_character.has_value() && is_ascii_upper_alpha(current_input_character.value())) - -#define ON_ASCII_LOWER_ALPHA \ - if (current_input_character.has_value() && is_ascii_lower_alpha(current_input_character.value())) - -#define ON_ASCII_DIGIT \ - if (current_input_character.has_value() && is_ascii_digit(current_input_character.value())) - -#define ON_ASCII_HEX_DIGIT \ - if (current_input_character.has_value() && is_ascii_hex_digit(current_input_character.value())) - -#define ON_WHITESPACE \ - if (current_input_character.has_value() && is_ascii(*current_input_character) && first_is_one_of(static_cast(*current_input_character), '\t', '\n', '\f', ' ')) - -#define ANYTHING_ELSE if (1) - -#define EMIT_EOF \ - do { \ - if (m_has_emitted_eof) \ - return {}; \ - m_has_emitted_eof = true; \ - create_new_token(HTMLToken::Type::EndOfFile); \ - will_emit(m_current_token); \ - m_queued_tokens.enqueue(move(m_current_token)); \ - return m_queued_tokens.dequeue(); \ - } while (0) - -#define EMIT_CURRENT_TOKEN_FOLLOWED_BY_EOF \ - do { \ - VERIFY(m_current_builder.is_empty()); \ - will_emit(m_current_token); \ - m_queued_tokens.enqueue(move(m_current_token)); \ - \ - m_has_emitted_eof = true; \ - create_new_token(HTMLToken::Type::EndOfFile); \ - will_emit(m_current_token); \ - m_queued_tokens.enqueue(move(m_current_token)); \ - \ - return m_queued_tokens.dequeue(); \ - } while (0) - -#define EMIT_CHARACTER(code_point) \ - do { \ - create_new_token(HTMLToken::Type::Character); \ - m_current_token.set_code_point(code_point); \ - m_queued_tokens.enqueue(move(m_current_token)); \ - return m_queued_tokens.dequeue(); \ - } while (0) - -#define EMIT_CURRENT_CHARACTER \ - EMIT_CHARACTER(current_input_character.value()); - -#define SWITCH_TO_AND_EMIT_CHARACTER(code_point, new_state) \ - do { \ - will_switch_to(State::new_state); \ - m_state = State::new_state; \ - EMIT_CHARACTER(code_point); \ - } while (0) - -#define SWITCH_TO_AND_EMIT_CURRENT_CHARACTER(new_state) \ - SWITCH_TO_AND_EMIT_CHARACTER(current_input_character.value(), new_state) - -#define BEGIN_STATE(state) \ - state: \ - case State::state: { \ - { \ - { - -#define END_STATE \ - VERIFY_NOT_REACHED(); \ - break; \ - } \ - } \ - } - -static inline void log_parse_error(SourceLocation const& location = SourceLocation::current()) +static Vector code_points_from_string(String const& string) { - dbgln_if(TOKENIZER_TRACE_DEBUG, "Parse error (tokenization) {}", location); + Vector code_points; + code_points.ensure_capacity(string.bytes().size()); + for (auto code_point : string.code_points()) + code_points.append(code_point); + return code_points; } -bool HTMLTokenizer::should_pause_before_next_input_character(StopAtInsertionPoint stop_at_insertion_point) const +static StringView ffi_string_view(u8 const* ptr, size_t len) { - if (stop_at_insertion_point == StopAtInsertionPoint::Yes && m_insertion_point.has_value()) { - if (m_current_offset >= *m_insertion_point) - return true; - - if (m_current_offset < static_cast(m_decoded_input.size()) - && m_decoded_input[m_current_offset] == '\r' - && m_current_offset + 1 == *m_insertion_point) { - // Newline normalization needs one code point of lookahead for CRLF. If document.write() - // stops the tokenizer immediately after a CR, wait for more input before deciding whether - // it is a CRLF pair or a standalone CR. - return true; - } - } - - if (m_input_stream_closed) - return false; - - if (m_current_offset >= static_cast(m_decoded_input.size())) - return true; - - // Newline normalization needs one code point of lookahead for CRLF. If a network chunk ends in - // CR, wait for either another code point or the input stream close before deciding whether it is - // a CRLF pair or a standalone CR. - return m_decoded_input[m_current_offset] == '\r' && m_current_offset + 1 >= static_cast(m_decoded_input.size()); -} - -bool HTMLTokenizer::can_run_out_of_characters(StopAtInsertionPoint stop_at_insertion_point) const -{ - return (stop_at_insertion_point == StopAtInsertionPoint::Yes && m_insertion_point.has_value()) - || !m_input_stream_closed; -} - -Optional HTMLTokenizer::next_code_point(StopAtInsertionPoint stop_at_insertion_point) -{ - if (m_current_offset >= static_cast(m_decoded_input.size())) + if (ptr == nullptr || len == 0) return {}; - - u32 code_point; - // https://html.spec.whatwg.org/multipage/parsing.html#preprocessing-the-input-stream:tokenization - // https://infra.spec.whatwg.org/#normalize-newlines - if (peek_code_point(0, stop_at_insertion_point).value_or(0) == '\r' && peek_code_point(1, stop_at_insertion_point).value_or(0) == '\n') { - // replace every U+000D CR U+000A LF code point pair with a single U+000A LF code point, - skip(2); - code_point = '\n'; - } else if (peek_code_point(0, stop_at_insertion_point).value_or(0) == '\r') { - // replace every remaining U+000D CR code point with a U+000A LF code point. - skip(1); - code_point = '\n'; - } else { - skip(1); - code_point = m_decoded_input[m_prev_offset]; - } - - dbgln_if(TOKENIZER_TRACE_DEBUG, "(Tokenizer) Next code_point: {}", code_point); - return code_point; + return { ptr, len }; } -void HTMLTokenizer::skip(size_t count) +static RustFfiTokenizerHandle* create_tokenizer_from_utf8(StringView utf8_bytes) { - VERIFY(count > 0); - if (!m_source_positions.is_empty()) - m_source_positions.append(m_source_positions.last()); + auto* bytes = reinterpret_cast(utf8_bytes.characters_without_null_termination()); + if (bytes == nullptr) + bytes = reinterpret_cast(""); + return rust_html_tokenizer_create_from_utf8(bytes, utf8_bytes.length()); +} + +static Vector build_interned_name_table(size_t count, void (*fetch)(uint16_t, uint8_t const**, size_t*)) +{ + Vector table; + // Slot 0 is unused (id 0 means "not interned"); store an empty FlyString there. + table.append(FlyString {}); + table.ensure_capacity(count + 1); for (size_t i = 0; i < count; ++i) { - m_prev_offset = m_current_offset; - auto code_point = m_decoded_input[m_current_offset]; - if (!m_source_positions.is_empty()) { - if (code_point == '\n') { - m_source_positions.last().column = 0; - m_source_positions.last().line++; - } else { - m_source_positions.last().column++; - } + uint8_t const* ptr = nullptr; + size_t len = 0; + fetch(static_cast(i + 1), &ptr, &len); + if (ptr == nullptr || len == 0) { + table.append(FlyString {}); + continue; } - ++m_current_offset; + table.append(MUST(FlyString::from_utf8(StringView { ptr, len }))); } + return table; } -Optional HTMLTokenizer::peek_code_point(ssize_t offset, StopAtInsertionPoint stop_at_insertion_point) const +static FlyString const& interned_rust_tag_name(uint16_t id) { - auto it = m_current_offset + offset; - if (it >= static_cast(m_decoded_input.size())) - return {}; - if (stop_at_insertion_point == StopAtInsertionPoint::Yes - && m_insertion_point.has_value() - && it >= *m_insertion_point) { - return {}; - } - return m_decoded_input[it]; + static Vector const s_table = build_interned_name_table( + rust_html_tokenizer_interned_tag_name_count(), + rust_html_tokenizer_interned_tag_name); + if (id == 0 || id >= s_table.size()) + return s_table[0]; + return s_table[id]; } -HTMLToken::Position HTMLTokenizer::nth_last_position(size_t n) +static FlyString const& interned_rust_attr_name(uint16_t id) { - if (n + 1 > m_source_positions.size()) { - dbgln_if(TOKENIZER_TRACE_DEBUG, "(Tokenizer::nth_last_position) Invalid position requested: {}th-last of {}. Returning (0-0).", n, m_source_positions.size()); - return HTMLToken::Position { 0, 0 }; - }; - return m_source_positions.at(m_source_positions.size() - 1 - n); -} - -Optional HTMLTokenizer::next_token(StopAtInsertionPoint stop_at_insertion_point) -{ - if (!m_source_positions.is_empty()) { - auto last_position = m_source_positions.last(); - m_source_positions.clear_with_capacity(); - m_source_positions.append(move(last_position)); - } -_StartOfFunction: - if (!m_queued_tokens.is_empty()) - return m_queued_tokens.dequeue(); - - if (m_aborted) - return {}; - - for (;;) { - if (should_pause_before_next_input_character(stop_at_insertion_point)) - return {}; - - auto current_input_character = next_code_point(stop_at_insertion_point); - switch (m_state) { - // 13.2.5.1 Data state, https://html.spec.whatwg.org/multipage/parsing.html#data-state - BEGIN_STATE(Data) - { - ON('&') - { - m_return_state = State::Data; - SWITCH_TO(CharacterReference); - } - ON('<') - { - SWITCH_TO(TagOpen); - } - ON(0) - { - log_parse_error(); - EMIT_CURRENT_CHARACTER; - } - ON_EOF - { - EMIT_EOF; - } - ANYTHING_ELSE - { - EMIT_CURRENT_CHARACTER; - } - } - END_STATE - - // 13.2.5.6 Tag open state, https://html.spec.whatwg.org/multipage/parsing.html#tag-open-state - BEGIN_STATE(TagOpen) - { - ON('!') - { - SWITCH_TO(MarkupDeclarationOpen); - } - ON('/') - { - SWITCH_TO(EndTagOpen); - } - ON_ASCII_ALPHA - { - create_new_token(HTMLToken::Type::StartTag); - RECONSUME_IN(TagName); - } - ON('?') - { - log_parse_error(); - create_new_token(HTMLToken::Type::Comment); - m_current_token.set_start_position({}, nth_last_position(2)); - RECONSUME_IN(BogusComment); - } - ON_EOF - { - log_parse_error(); - m_queued_tokens.enqueue(HTMLToken::make_character('<')); - EMIT_EOF; - } - ANYTHING_ELSE - { - log_parse_error(); - EMIT_CHARACTER_AND_RECONSUME_IN('<', Data); - } - } - END_STATE - - // 13.2.5.8 Tag name state, https://html.spec.whatwg.org/multipage/parsing.html#tag-name-state - BEGIN_STATE(TagName) - { - ON_WHITESPACE - { - m_current_token.set_tag_name(consume_current_builder()); - m_current_token.set_end_position({}, nth_last_position(1)); - SWITCH_TO(BeforeAttributeName); - } - ON('/') - { - m_current_token.set_tag_name(consume_current_builder()); - m_current_token.set_end_position({}, nth_last_position(0)); - SWITCH_TO(SelfClosingStartTag); - } - ON('>') - { - m_current_token.set_tag_name(consume_current_builder()); - SWITCH_TO_AND_EMIT_CURRENT_TOKEN(Data); - } - ON_ASCII_UPPER_ALPHA - { - m_current_builder.append_code_point(to_ascii_lowercase(current_input_character.value())); - m_current_token.set_end_position({}, nth_last_position(0)); - continue; - } - ON(0) - { - log_parse_error(); - m_current_builder.append_code_point(0xFFFD); - m_current_token.set_end_position({}, nth_last_position(0)); - continue; - } - ON_EOF - { - log_parse_error(); - EMIT_EOF; - } - ANYTHING_ELSE - { - m_current_builder.append_code_point(current_input_character.value()); - m_current_token.set_end_position({}, nth_last_position(0)); - continue; - } - } - END_STATE - - // 13.2.5.7 End tag open state, https://html.spec.whatwg.org/multipage/parsing.html#end-tag-open-state - BEGIN_STATE(EndTagOpen) - { - ON_ASCII_ALPHA - { - create_new_token(HTMLToken::Type::EndTag); - RECONSUME_IN(TagName); - } - ON('>') - { - log_parse_error(); - SWITCH_TO(Data); - } - ON_EOF - { - log_parse_error(); - m_queued_tokens.enqueue(HTMLToken::make_character('<')); - m_queued_tokens.enqueue(HTMLToken::make_character('/')); - EMIT_EOF; - } - ANYTHING_ELSE - { - log_parse_error(); - create_new_token(HTMLToken::Type::Comment); - RECONSUME_IN(BogusComment); - } - } - END_STATE - - // 13.2.5.42 Markup declaration open state, https://html.spec.whatwg.org/multipage/parsing.html#markup-declaration-open-state - BEGIN_STATE(MarkupDeclarationOpen) - { - DONT_CONSUME_NEXT_INPUT_CHARACTER; - - switch (consume_next_if_match("--"sv, stop_at_insertion_point)) { - case ConsumeNextResult::Consumed: - create_new_token(HTMLToken::Type::Comment); - m_current_token.set_start_position({}, nth_last_position(3)); - SWITCH_TO(CommentStart); - break; - case ConsumeNextResult::NotConsumed: - break; - case ConsumeNextResult::RanOutOfCharacters: - return {}; - } - - switch (consume_next_if_match("DOCTYPE"sv, stop_at_insertion_point, CaseSensitivity::CaseInsensitive)) { - case ConsumeNextResult::Consumed: - SWITCH_TO(DOCTYPE); - break; - case ConsumeNextResult::NotConsumed: - break; - case ConsumeNextResult::RanOutOfCharacters: - return {}; - } - - switch (consume_next_if_match("[CDATA["sv, stop_at_insertion_point)) { - case ConsumeNextResult::Consumed: - // We keep the parser optional so that syntax highlighting can be lexer-only. - // The parser registers itself with the lexer it creates. - if (m_parser != nullptr - && m_parser->adjusted_current_node() - && m_parser->adjusted_current_node()->namespace_uri() != Namespace::HTML) { - SWITCH_TO(CDATASection); - } else { - create_new_token(HTMLToken::Type::Comment); - m_current_builder.append("[CDATA["sv); - SWITCH_TO_WITH_UNCLEAN_BUILDER(BogusComment); - } - break; - case ConsumeNextResult::NotConsumed: - break; - case ConsumeNextResult::RanOutOfCharacters: - return {}; - } - ANYTHING_ELSE - { - log_parse_error(); - create_new_token(HTMLToken::Type::Comment); - SWITCH_TO(BogusComment); - } - } - END_STATE - - // 13.2.5.41 Bogus comment state, https://html.spec.whatwg.org/multipage/parsing.html#bogus-comment-state - BEGIN_STATE(BogusComment) - { - ON('>') - { - m_current_token.set_comment(consume_current_builder()); - SWITCH_TO_AND_EMIT_CURRENT_TOKEN(Data); - } - ON_EOF - { - m_current_token.set_comment(consume_current_builder()); - EMIT_CURRENT_TOKEN_FOLLOWED_BY_EOF; - } - ON(0) - { - log_parse_error(); - m_current_builder.append_code_point(0xFFFD); - continue; - } - ANYTHING_ELSE - { - m_current_builder.append_code_point(current_input_character.value()); - continue; - } - } - END_STATE - - // 13.2.5.53 DOCTYPE state, https://html.spec.whatwg.org/multipage/parsing.html#doctype-state - BEGIN_STATE(DOCTYPE) - { - ON_WHITESPACE - { - SWITCH_TO(BeforeDOCTYPEName); - } - ON('>') - { - RECONSUME_IN(BeforeDOCTYPEName); - } - ON_EOF - { - log_parse_error(); - create_new_token(HTMLToken::Type::DOCTYPE); - m_current_token.ensure_doctype_data().force_quirks = true; - m_queued_tokens.enqueue(move(m_current_token)); - EMIT_EOF; - } - ANYTHING_ELSE - { - log_parse_error(); - RECONSUME_IN(BeforeDOCTYPEName); - } - } - END_STATE - - // 13.2.5.54 Before DOCTYPE name state, https://html.spec.whatwg.org/multipage/parsing.html#before-doctype-name-state - BEGIN_STATE(BeforeDOCTYPEName) - { - ON_WHITESPACE - { - continue; - } - ON_ASCII_UPPER_ALPHA - { - create_new_token(HTMLToken::Type::DOCTYPE); - m_current_builder.append_code_point(to_ascii_lowercase(current_input_character.value())); - m_current_token.ensure_doctype_data().missing_name = false; - SWITCH_TO_WITH_UNCLEAN_BUILDER(DOCTYPEName); - } - ON(0) - { - log_parse_error(); - create_new_token(HTMLToken::Type::DOCTYPE); - m_current_builder.append_code_point(0xFFFD); - m_current_token.ensure_doctype_data().missing_name = false; - SWITCH_TO_WITH_UNCLEAN_BUILDER(DOCTYPEName); - } - ON('>') - { - log_parse_error(); - create_new_token(HTMLToken::Type::DOCTYPE); - m_current_token.ensure_doctype_data().force_quirks = true; - SWITCH_TO_AND_EMIT_CURRENT_TOKEN(Data); - } - ON_EOF - { - log_parse_error(); - create_new_token(HTMLToken::Type::DOCTYPE); - m_current_token.ensure_doctype_data().force_quirks = true; - m_queued_tokens.enqueue(move(m_current_token)); - EMIT_EOF; - } - ANYTHING_ELSE - { - create_new_token(HTMLToken::Type::DOCTYPE); - m_current_builder.append_code_point(current_input_character.value()); - m_current_token.ensure_doctype_data().missing_name = false; - SWITCH_TO_WITH_UNCLEAN_BUILDER(DOCTYPEName); - } - } - END_STATE - - // 13.2.5.55 DOCTYPE name state, https://html.spec.whatwg.org/multipage/parsing.html#doctype-name-state - BEGIN_STATE(DOCTYPEName) - { - ON_WHITESPACE - { - m_current_token.ensure_doctype_data().name = consume_current_builder(); - SWITCH_TO(AfterDOCTYPEName); - } - ON('>') - { - m_current_token.ensure_doctype_data().name = consume_current_builder(); - SWITCH_TO_AND_EMIT_CURRENT_TOKEN(Data); - } - ON_ASCII_UPPER_ALPHA - { - m_current_builder.append_code_point(to_ascii_lowercase(current_input_character.value())); - continue; - } - ON(0) - { - log_parse_error(); - m_current_builder.append_code_point(0xFFFD); - continue; - } - ON_EOF - { - log_parse_error(); - m_current_token.ensure_doctype_data().force_quirks = true; - m_queued_tokens.enqueue(move(m_current_token)); - EMIT_EOF; - } - ANYTHING_ELSE - { - m_current_builder.append_code_point(current_input_character.value()); - continue; - } - } - END_STATE - - // 13.2.5.56 After DOCTYPE name state, https://html.spec.whatwg.org/multipage/parsing.html#after-doctype-name-state - BEGIN_STATE(AfterDOCTYPEName) - { - ON_WHITESPACE - { - continue; - } - ON('>') - { - SWITCH_TO_AND_EMIT_CURRENT_TOKEN(Data); - } - ON_EOF - { - log_parse_error(); - m_current_token.ensure_doctype_data().force_quirks = true; - m_queued_tokens.enqueue(move(m_current_token)); - EMIT_EOF; - } - ANYTHING_ELSE - { - if (to_ascii_uppercase(current_input_character.value()) == 'P') { - switch (consume_next_if_match("UBLIC"sv, stop_at_insertion_point, CaseSensitivity::CaseInsensitive)) { - case ConsumeNextResult::Consumed: - SWITCH_TO(AfterDOCTYPEPublicKeyword); - break; - case ConsumeNextResult::NotConsumed: - break; - case ConsumeNextResult::RanOutOfCharacters: - DONT_CONSUME_NEXT_INPUT_CHARACTER; - return {}; - } - } - if (to_ascii_uppercase(current_input_character.value()) == 'S') { - switch (consume_next_if_match("YSTEM"sv, stop_at_insertion_point, CaseSensitivity::CaseInsensitive)) { - case ConsumeNextResult::Consumed: - SWITCH_TO(AfterDOCTYPESystemKeyword); - break; - case ConsumeNextResult::NotConsumed: - break; - case ConsumeNextResult::RanOutOfCharacters: - DONT_CONSUME_NEXT_INPUT_CHARACTER; - return {}; - } - } - log_parse_error(); - m_current_token.ensure_doctype_data().force_quirks = true; - RECONSUME_IN(BogusDOCTYPE); - } - } - END_STATE - - // 13.2.5.57 After DOCTYPE public keyword state, https://html.spec.whatwg.org/multipage/parsing.html#after-doctype-public-keyword-state - BEGIN_STATE(AfterDOCTYPEPublicKeyword) - { - ON_WHITESPACE - { - SWITCH_TO(BeforeDOCTYPEPublicIdentifier); - } - ON('"') - { - log_parse_error(); - m_current_token.ensure_doctype_data().missing_public_identifier = false; - SWITCH_TO(DOCTYPEPublicIdentifierDoubleQuoted); - } - ON('\'') - { - log_parse_error(); - m_current_token.ensure_doctype_data().missing_public_identifier = false; - SWITCH_TO(DOCTYPEPublicIdentifierSingleQuoted); - } - ON('>') - { - log_parse_error(); - m_current_token.ensure_doctype_data().force_quirks = true; - SWITCH_TO_AND_EMIT_CURRENT_TOKEN(Data); - } - ON_EOF - { - log_parse_error(); - m_current_token.ensure_doctype_data().force_quirks = true; - m_queued_tokens.enqueue(move(m_current_token)); - EMIT_EOF; - } - ANYTHING_ELSE - { - log_parse_error(); - m_current_token.ensure_doctype_data().force_quirks = true; - RECONSUME_IN(BogusDOCTYPE); - } - } - END_STATE - - // 13.2.5.63 After DOCTYPE system keyword state, https://html.spec.whatwg.org/multipage/parsing.html#after-doctype-system-keyword-state - BEGIN_STATE(AfterDOCTYPESystemKeyword) - { - ON_WHITESPACE - { - SWITCH_TO(BeforeDOCTYPESystemIdentifier); - } - ON('"') - { - log_parse_error(); - m_current_token.ensure_doctype_data().system_identifier = {}; - m_current_token.ensure_doctype_data().missing_system_identifier = false; - SWITCH_TO(DOCTYPESystemIdentifierDoubleQuoted); - } - ON('\'') - { - log_parse_error(); - m_current_token.ensure_doctype_data().system_identifier = {}; - m_current_token.ensure_doctype_data().missing_system_identifier = false; - SWITCH_TO(DOCTYPESystemIdentifierSingleQuoted); - } - ON('>') - { - log_parse_error(); - m_current_token.ensure_doctype_data().force_quirks = true; - SWITCH_TO_AND_EMIT_CURRENT_TOKEN(Data); - } - ON_EOF - { - log_parse_error(); - m_current_token.ensure_doctype_data().force_quirks = true; - m_queued_tokens.enqueue(move(m_current_token)); - EMIT_EOF; - } - ANYTHING_ELSE - { - log_parse_error(); - m_current_token.ensure_doctype_data().force_quirks = true; - RECONSUME_IN(BogusDOCTYPE); - } - } - END_STATE - - // 13.2.5.58 Before DOCTYPE public identifier state, https://html.spec.whatwg.org/multipage/parsing.html#before-doctype-public-identifier-state - BEGIN_STATE(BeforeDOCTYPEPublicIdentifier) - { - ON_WHITESPACE - { - continue; - } - ON('"') - { - m_current_token.ensure_doctype_data().missing_public_identifier = false; - SWITCH_TO(DOCTYPEPublicIdentifierDoubleQuoted); - } - ON('\'') - { - m_current_token.ensure_doctype_data().missing_public_identifier = false; - SWITCH_TO(DOCTYPEPublicIdentifierSingleQuoted); - } - ON('>') - { - log_parse_error(); - m_current_token.ensure_doctype_data().force_quirks = true; - SWITCH_TO_AND_EMIT_CURRENT_TOKEN(Data); - } - ON_EOF - { - log_parse_error(); - m_current_token.ensure_doctype_data().force_quirks = true; - m_queued_tokens.enqueue(move(m_current_token)); - EMIT_EOF; - } - ANYTHING_ELSE - { - log_parse_error(); - m_current_token.ensure_doctype_data().force_quirks = true; - RECONSUME_IN(BogusDOCTYPE); - } - } - END_STATE - - // 13.2.5.64 Before DOCTYPE system identifier state, https://html.spec.whatwg.org/multipage/parsing.html#before-doctype-system-identifier-state - BEGIN_STATE(BeforeDOCTYPESystemIdentifier) - { - ON_WHITESPACE - { - continue; - } - ON('"') - { - m_current_token.ensure_doctype_data().missing_system_identifier = false; - SWITCH_TO(DOCTYPESystemIdentifierDoubleQuoted); - } - ON('\'') - { - m_current_token.ensure_doctype_data().missing_system_identifier = false; - SWITCH_TO(DOCTYPESystemIdentifierSingleQuoted); - } - ON('>') - { - log_parse_error(); - m_current_token.ensure_doctype_data().force_quirks = true; - SWITCH_TO_AND_EMIT_CURRENT_TOKEN(Data); - } - ON_EOF - { - log_parse_error(); - m_current_token.ensure_doctype_data().force_quirks = true; - m_queued_tokens.enqueue(move(m_current_token)); - EMIT_EOF; - } - ANYTHING_ELSE - { - log_parse_error(); - m_current_token.ensure_doctype_data().force_quirks = true; - RECONSUME_IN(BogusDOCTYPE); - } - } - END_STATE - - // 13.2.5.59 DOCTYPE public identifier (double-quoted) state, https://html.spec.whatwg.org/multipage/parsing.html#doctype-public-identifier-(double-quoted)-state - BEGIN_STATE(DOCTYPEPublicIdentifierDoubleQuoted) - { - ON('"') - { - m_current_token.ensure_doctype_data().public_identifier = consume_current_builder(); - SWITCH_TO(AfterDOCTYPEPublicIdentifier); - } - ON(0) - { - log_parse_error(); - m_current_builder.append_code_point(0xFFFD); - continue; - } - ON('>') - { - log_parse_error(); - m_current_token.ensure_doctype_data().public_identifier = consume_current_builder(); - m_current_token.ensure_doctype_data().force_quirks = true; - SWITCH_TO_AND_EMIT_CURRENT_TOKEN(Data); - } - ON_EOF - { - log_parse_error(); - m_current_token.ensure_doctype_data().force_quirks = true; - m_queued_tokens.enqueue(move(m_current_token)); - EMIT_EOF; - } - ANYTHING_ELSE - { - m_current_builder.append_code_point(current_input_character.value()); - continue; - } - } - END_STATE - - // 13.2.5.60 DOCTYPE public identifier (single-quoted) state, https://html.spec.whatwg.org/multipage/parsing.html#doctype-public-identifier-(single-quoted)-state - BEGIN_STATE(DOCTYPEPublicIdentifierSingleQuoted) - { - ON('\'') - { - m_current_token.ensure_doctype_data().public_identifier = consume_current_builder(); - SWITCH_TO(AfterDOCTYPEPublicIdentifier); - } - ON(0) - { - log_parse_error(); - m_current_builder.append_code_point(0xFFFD); - continue; - } - ON('>') - { - log_parse_error(); - m_current_token.ensure_doctype_data().public_identifier = consume_current_builder(); - m_current_token.ensure_doctype_data().force_quirks = true; - SWITCH_TO_AND_EMIT_CURRENT_TOKEN(Data); - } - ON_EOF - { - log_parse_error(); - m_current_token.ensure_doctype_data().force_quirks = true; - m_queued_tokens.enqueue(move(m_current_token)); - EMIT_EOF; - } - ANYTHING_ELSE - { - m_current_builder.append_code_point(current_input_character.value()); - continue; - } - } - END_STATE - - // 13.2.5.65 DOCTYPE system identifier (double-quoted) state, https://html.spec.whatwg.org/multipage/parsing.html#doctype-system-identifier-(double-quoted)-state - BEGIN_STATE(DOCTYPESystemIdentifierDoubleQuoted) - { - ON('"') - { - m_current_token.ensure_doctype_data().system_identifier = consume_current_builder(); - SWITCH_TO(AfterDOCTYPESystemIdentifier); - } - ON(0) - { - log_parse_error(); - m_current_builder.append_code_point(0xFFFD); - continue; - } - ON('>') - { - log_parse_error(); - m_current_token.ensure_doctype_data().system_identifier = consume_current_builder(); - m_current_token.ensure_doctype_data().force_quirks = true; - SWITCH_TO_AND_EMIT_CURRENT_TOKEN(Data); - } - ON_EOF - { - log_parse_error(); - m_current_token.ensure_doctype_data().force_quirks = true; - m_queued_tokens.enqueue(move(m_current_token)); - EMIT_EOF; - } - ANYTHING_ELSE - { - m_current_builder.append_code_point(current_input_character.value()); - continue; - } - } - END_STATE - - // 13.2.5.66 DOCTYPE system identifier (single-quoted) state, https://html.spec.whatwg.org/multipage/parsing.html#doctype-system-identifier-(single-quoted)-state - BEGIN_STATE(DOCTYPESystemIdentifierSingleQuoted) - { - ON('\'') - { - m_current_token.ensure_doctype_data().system_identifier = consume_current_builder(); - SWITCH_TO(AfterDOCTYPESystemIdentifier); - } - ON(0) - { - log_parse_error(); - m_current_builder.append_code_point(0xFFFD); - continue; - } - ON('>') - { - log_parse_error(); - m_current_token.ensure_doctype_data().system_identifier = consume_current_builder(); - m_current_token.ensure_doctype_data().force_quirks = true; - SWITCH_TO_AND_EMIT_CURRENT_TOKEN(Data); - } - ON_EOF - { - log_parse_error(); - m_current_token.ensure_doctype_data().force_quirks = true; - m_queued_tokens.enqueue(move(m_current_token)); - EMIT_EOF; - } - ANYTHING_ELSE - { - m_current_builder.append_code_point(current_input_character.value()); - continue; - } - } - END_STATE - - // 13.2.5.61 After DOCTYPE public identifier state, https://html.spec.whatwg.org/multipage/parsing.html#after-doctype-public-identifier-state - BEGIN_STATE(AfterDOCTYPEPublicIdentifier) - { - ON_WHITESPACE - { - SWITCH_TO(BetweenDOCTYPEPublicAndSystemIdentifiers); - } - ON('>') - { - SWITCH_TO_AND_EMIT_CURRENT_TOKEN(Data); - } - ON('"') - { - log_parse_error(); - m_current_token.ensure_doctype_data().missing_system_identifier = false; - SWITCH_TO(DOCTYPESystemIdentifierDoubleQuoted); - } - ON('\'') - { - log_parse_error(); - m_current_token.ensure_doctype_data().missing_system_identifier = false; - SWITCH_TO(DOCTYPESystemIdentifierSingleQuoted); - } - ON_EOF - { - log_parse_error(); - m_current_token.ensure_doctype_data().force_quirks = true; - m_queued_tokens.enqueue(move(m_current_token)); - EMIT_EOF; - } - ANYTHING_ELSE - { - log_parse_error(); - m_current_token.ensure_doctype_data().force_quirks = true; - RECONSUME_IN(BogusDOCTYPE); - } - } - END_STATE - - // 13.2.5.62 Between DOCTYPE public and system identifiers state, https://html.spec.whatwg.org/multipage/parsing.html#between-doctype-public-and-system-identifiers-state - BEGIN_STATE(BetweenDOCTYPEPublicAndSystemIdentifiers) - { - ON_WHITESPACE - { - continue; - } - ON('>') - { - SWITCH_TO_AND_EMIT_CURRENT_TOKEN(Data); - } - ON('"') - { - m_current_token.ensure_doctype_data().missing_system_identifier = false; - SWITCH_TO(DOCTYPESystemIdentifierDoubleQuoted); - } - ON('\'') - { - m_current_token.ensure_doctype_data().missing_system_identifier = false; - SWITCH_TO(DOCTYPESystemIdentifierSingleQuoted); - } - ON_EOF - { - log_parse_error(); - m_current_token.ensure_doctype_data().force_quirks = true; - m_queued_tokens.enqueue(move(m_current_token)); - EMIT_EOF; - } - ANYTHING_ELSE - { - log_parse_error(); - m_current_token.ensure_doctype_data().force_quirks = true; - RECONSUME_IN(BogusDOCTYPE); - } - } - END_STATE - - // 13.2.5.67 After DOCTYPE system identifier state, https://html.spec.whatwg.org/multipage/parsing.html#after-doctype-system-identifier-state - BEGIN_STATE(AfterDOCTYPESystemIdentifier) - { - ON_WHITESPACE - { - continue; - } - ON('>') - { - SWITCH_TO_AND_EMIT_CURRENT_TOKEN(Data); - } - ON_EOF - { - log_parse_error(); - m_current_token.ensure_doctype_data().force_quirks = true; - m_queued_tokens.enqueue(move(m_current_token)); - EMIT_EOF; - } - ANYTHING_ELSE - { - log_parse_error(); - RECONSUME_IN(BogusDOCTYPE); - } - } - END_STATE - - // 13.2.5.68 Bogus DOCTYPE state, https://html.spec.whatwg.org/multipage/parsing.html#bogus-doctype-state - BEGIN_STATE(BogusDOCTYPE) - { - ON('>') - { - SWITCH_TO_AND_EMIT_CURRENT_TOKEN(Data); - } - ON(0) - { - log_parse_error(); - continue; - } - ON_EOF - { - m_queued_tokens.enqueue(move(m_current_token)); - EMIT_EOF; - } - ANYTHING_ELSE - { - continue; - } - } - END_STATE - - // 13.2.5.32 Before attribute name state, https://html.spec.whatwg.org/multipage/parsing.html#before-attribute-name-state - BEGIN_STATE(BeforeAttributeName) - { - ON_WHITESPACE - { - continue; - } - ON('/') - { - RECONSUME_IN(AfterAttributeName); - } - ON('>') - { - RECONSUME_IN(AfterAttributeName); - } - ON_EOF - { - RECONSUME_IN(AfterAttributeName); - } - ON('=') - { - log_parse_error(); - HTMLToken::Attribute new_attribute; - new_attribute.name_start_position = nth_last_position(1); - m_current_builder.append_code_point(current_input_character.value()); - m_current_token.add_attribute(move(new_attribute)); - SWITCH_TO_WITH_UNCLEAN_BUILDER(AttributeName); - } - ANYTHING_ELSE - { - HTMLToken::Attribute new_attribute; - new_attribute.name_start_position = nth_last_position(1); - m_current_token.add_attribute(move(new_attribute)); - RECONSUME_IN(AttributeName); - } - } - END_STATE - - // 13.2.5.40 Self-closing start tag state, https://html.spec.whatwg.org/multipage/parsing.html#self-closing-start-tag-state - BEGIN_STATE(SelfClosingStartTag) - { - ON('>') - { - m_current_token.set_self_closing(true); - SWITCH_TO_AND_EMIT_CURRENT_TOKEN(Data); - } - ON_EOF - { - log_parse_error(); - EMIT_EOF; - } - ANYTHING_ELSE - { - log_parse_error(); - RECONSUME_IN(BeforeAttributeName); - } - } - END_STATE - - // 13.2.5.33 Attribute name state, https://html.spec.whatwg.org/multipage/parsing.html#attribute-name-state - BEGIN_STATE(AttributeName) - { - ON_WHITESPACE - { - m_current_token.last_attribute().name_end_position = nth_last_position(1); - m_current_token.last_attribute().local_name = consume_current_builder(); - RECONSUME_IN(AfterAttributeName); - } - ON('/') - { - m_current_token.last_attribute().name_end_position = nth_last_position(1); - m_current_token.last_attribute().local_name = consume_current_builder(); - RECONSUME_IN(AfterAttributeName); - } - ON('>') - { - m_current_token.last_attribute().name_end_position = nth_last_position(1); - m_current_token.last_attribute().local_name = consume_current_builder(); - RECONSUME_IN(AfterAttributeName); - } - ON_EOF - { - m_current_token.last_attribute().name_end_position = nth_last_position(1); - m_current_token.last_attribute().local_name = consume_current_builder(); - RECONSUME_IN(AfterAttributeName); - } - ON('=') - { - m_current_token.last_attribute().name_end_position = nth_last_position(1); - m_current_token.last_attribute().local_name = consume_current_builder(); - SWITCH_TO(BeforeAttributeValue); - } - ON_ASCII_UPPER_ALPHA - { - m_current_builder.append_code_point(to_ascii_lowercase(current_input_character.value())); - continue; - } - ON(0) - { - log_parse_error(); - m_current_builder.append_code_point(0xFFFD); - continue; - } - ON('"') - { - log_parse_error(); - goto AnythingElseAttributeName; - } - ON('\'') - { - log_parse_error(); - goto AnythingElseAttributeName; - } - ON('<') - { - log_parse_error(); - goto AnythingElseAttributeName; - } - ANYTHING_ELSE - { - AnythingElseAttributeName: - m_current_builder.append_code_point(current_input_character.value()); - continue; - } - } - END_STATE - - // 13.2.5.34 After attribute name state, https://html.spec.whatwg.org/multipage/parsing.html#after-attribute-name-state - BEGIN_STATE(AfterAttributeName) - { - ON_WHITESPACE - { - continue; - } - ON('/') - { - SWITCH_TO(SelfClosingStartTag); - } - ON('=') - { - m_current_token.last_attribute().name_end_position = nth_last_position(1); - SWITCH_TO(BeforeAttributeValue); - } - ON('>') - { - SWITCH_TO_AND_EMIT_CURRENT_TOKEN(Data); - } - ON_EOF - { - log_parse_error(); - EMIT_EOF; - } - ANYTHING_ELSE - { - m_current_token.add_attribute({}); - if (!m_source_positions.is_empty()) - m_current_token.last_attribute().name_start_position = nth_last_position(1); - RECONSUME_IN(AttributeName); - } - } - END_STATE - - // 13.2.5.35 Before attribute value state, https://html.spec.whatwg.org/multipage/parsing.html#before-attribute-value-state - BEGIN_STATE(BeforeAttributeValue) - { - m_current_token.last_attribute().value_start_position = nth_last_position(1); - ON_WHITESPACE - { - continue; - } - ON('"') - { - SWITCH_TO(AttributeValueDoubleQuoted); - } - ON('\'') - { - SWITCH_TO(AttributeValueSingleQuoted); - } - ON('>') - { - log_parse_error(); - SWITCH_TO_AND_EMIT_CURRENT_TOKEN(Data); - } - ANYTHING_ELSE - { - RECONSUME_IN(AttributeValueUnquoted); - } - } - END_STATE - - // 13.2.5.36 Attribute value (double-quoted) state, https://html.spec.whatwg.org/multipage/parsing.html#attribute-value-(double-quoted)-state - BEGIN_STATE(AttributeValueDoubleQuoted) - { - ON('"') - { - m_current_token.last_attribute().value = consume_current_builder(); - SWITCH_TO(AfterAttributeValueQuoted); - } - ON('&') - { - m_return_state = State::AttributeValueDoubleQuoted; - SWITCH_TO_WITH_UNCLEAN_BUILDER(CharacterReference); - } - ON(0) - { - log_parse_error(); - m_current_builder.append_code_point(0xFFFD); - continue; - } - ON_EOF - { - log_parse_error(); - EMIT_EOF; - } - ANYTHING_ELSE - { - m_current_builder.append_code_point(current_input_character.value()); - continue; - } - } - END_STATE - - // 13.2.5.37 Attribute value (single-quoted) state, https://html.spec.whatwg.org/multipage/parsing.html#attribute-value-(single-quoted)-state - BEGIN_STATE(AttributeValueSingleQuoted) - { - ON('\'') - { - m_current_token.last_attribute().value = consume_current_builder(); - SWITCH_TO(AfterAttributeValueQuoted); - } - ON('&') - { - m_return_state = State::AttributeValueSingleQuoted; - SWITCH_TO_WITH_UNCLEAN_BUILDER(CharacterReference); - } - ON(0) - { - log_parse_error(); - m_current_builder.append_code_point(0xFFFD); - continue; - } - ON_EOF - { - log_parse_error(); - EMIT_EOF; - } - ANYTHING_ELSE - { - m_current_builder.append_code_point(current_input_character.value()); - continue; - } - } - END_STATE - - // 13.2.5.38 Attribute value (unquoted) state, https://html.spec.whatwg.org/multipage/parsing.html#attribute-value-(single-quoted)-state - BEGIN_STATE(AttributeValueUnquoted) - { - ON_WHITESPACE - { - m_current_token.last_attribute().value = consume_current_builder(); - m_current_token.last_attribute().value_end_position = nth_last_position(1); - SWITCH_TO(BeforeAttributeName); - } - ON('&') - { - m_return_state = State::AttributeValueUnquoted; - SWITCH_TO_WITH_UNCLEAN_BUILDER(CharacterReference); - } - ON('>') - { - m_current_token.last_attribute().value = consume_current_builder(); - m_current_token.last_attribute().value_end_position = nth_last_position(1); - SWITCH_TO_AND_EMIT_CURRENT_TOKEN(Data); - } - ON(0) - { - log_parse_error(); - m_current_builder.append_code_point(0xFFFD); - continue; - } - ON('"') - { - log_parse_error(); - goto AnythingElseAttributeValueUnquoted; - } - ON('\'') - { - log_parse_error(); - goto AnythingElseAttributeValueUnquoted; - } - ON('<') - { - log_parse_error(); - goto AnythingElseAttributeValueUnquoted; - } - ON('=') - { - log_parse_error(); - goto AnythingElseAttributeValueUnquoted; - } - ON('`') - { - log_parse_error(); - goto AnythingElseAttributeValueUnquoted; - } - ON_EOF - { - log_parse_error(); - EMIT_EOF; - } - ANYTHING_ELSE - { - AnythingElseAttributeValueUnquoted: - m_current_builder.append_code_point(current_input_character.value()); - continue; - } - } - END_STATE - - // 13.2.5.39 After attribute value (quoted) state, https://html.spec.whatwg.org/multipage/parsing.html#after-attribute-value-(quoted)-state - BEGIN_STATE(AfterAttributeValueQuoted) - { - m_current_token.last_attribute().value_end_position = nth_last_position(1); - ON_WHITESPACE - { - SWITCH_TO(BeforeAttributeName); - } - ON('/') - { - SWITCH_TO(SelfClosingStartTag); - } - ON('>') - { - SWITCH_TO_AND_EMIT_CURRENT_TOKEN(Data); - } - ON_EOF - { - log_parse_error(); - EMIT_EOF; - } - ANYTHING_ELSE - { - log_parse_error(); - RECONSUME_IN(BeforeAttributeName); - } - } - END_STATE - - // 13.2.5.43 Comment start state, https://html.spec.whatwg.org/multipage/parsing.html#comment-start-state - BEGIN_STATE(CommentStart) - { - ON('-') - { - SWITCH_TO(CommentStartDash); - } - ON('>') - { - log_parse_error(); - SWITCH_TO_AND_EMIT_CURRENT_TOKEN(Data); - } - ANYTHING_ELSE - { - RECONSUME_IN(Comment); - } - } - END_STATE - - // 13.2.5.44 Comment start dash state, https://html.spec.whatwg.org/multipage/parsing.html#comment-start-dash-state - BEGIN_STATE(CommentStartDash) - { - ON('-') - { - SWITCH_TO(CommentEnd); - } - ON('>') - { - log_parse_error(); - SWITCH_TO_AND_EMIT_CURRENT_TOKEN(Data); - } - ON_EOF - { - log_parse_error(); - EMIT_CURRENT_TOKEN_FOLLOWED_BY_EOF; - } - ANYTHING_ELSE - { - m_current_builder.append('-'); - RECONSUME_IN(Comment); - } - } - END_STATE - - // 13.2.5.45 Comment state, https://html.spec.whatwg.org/multipage/parsing.html#comment-state - BEGIN_STATE(Comment) - { - ON('<') - { - m_current_builder.append_code_point(current_input_character.value()); - SWITCH_TO_WITH_UNCLEAN_BUILDER(CommentLessThanSign); - } - ON('-') - { - SWITCH_TO_WITH_UNCLEAN_BUILDER(CommentEndDash); - } - ON(0) - { - log_parse_error(); - m_current_builder.append_code_point(0xFFFD); - continue; - } - ON_EOF - { - log_parse_error(); - m_current_token.set_comment(consume_current_builder()); - EMIT_CURRENT_TOKEN_FOLLOWED_BY_EOF; - } - ANYTHING_ELSE - { - m_current_builder.append_code_point(current_input_character.value()); - continue; - } - } - END_STATE - - // 13.2.5.51 Comment end state, https://html.spec.whatwg.org/multipage/parsing.html#comment-end-state - BEGIN_STATE(CommentEnd) - { - ON('>') - { - m_current_token.set_comment(consume_current_builder()); - SWITCH_TO_AND_EMIT_CURRENT_TOKEN(Data); - } - ON('!') - { - SWITCH_TO_WITH_UNCLEAN_BUILDER(CommentEndBang); - } - ON('-') - { - m_current_builder.append('-'); - continue; - } - ON_EOF - { - log_parse_error(); - m_current_token.set_comment(consume_current_builder()); - EMIT_CURRENT_TOKEN_FOLLOWED_BY_EOF; - } - ANYTHING_ELSE - { - m_current_builder.append("--"sv); - RECONSUME_IN(Comment); - } - } - END_STATE - - // 13.2.5.52 Comment end bang state, https://html.spec.whatwg.org/multipage/parsing.html#comment-end-bang-state - BEGIN_STATE(CommentEndBang) - { - ON('-') - { - m_current_builder.append("--!"sv); - SWITCH_TO_WITH_UNCLEAN_BUILDER(CommentEndDash); - } - ON('>') - { - log_parse_error(); - m_current_token.set_comment(consume_current_builder()); - SWITCH_TO_AND_EMIT_CURRENT_TOKEN(Data); - } - ON_EOF - { - log_parse_error(); - m_current_token.set_comment(consume_current_builder()); - EMIT_CURRENT_TOKEN_FOLLOWED_BY_EOF; - } - ANYTHING_ELSE - { - m_current_builder.append("--!"sv); - RECONSUME_IN(Comment); - } - } - END_STATE - - // 13.2.5.50 Comment end dash state, https://html.spec.whatwg.org/multipage/parsing.html#comment-end-dash-state - BEGIN_STATE(CommentEndDash) - { - ON('-') - { - SWITCH_TO_WITH_UNCLEAN_BUILDER(CommentEnd); - } - ON_EOF - { - log_parse_error(); - m_current_token.set_comment(consume_current_builder()); - EMIT_CURRENT_TOKEN_FOLLOWED_BY_EOF; - } - ANYTHING_ELSE - { - m_current_builder.append('-'); - RECONSUME_IN(Comment); - } - } - END_STATE - - // 13.2.5.46 Comment less-than sign state, https://html.spec.whatwg.org/multipage/parsing.html#comment-less-than-sign-state - BEGIN_STATE(CommentLessThanSign) - { - ON('!') - { - m_current_builder.append_code_point(current_input_character.value()); - SWITCH_TO_WITH_UNCLEAN_BUILDER(CommentLessThanSignBang); - } - ON('<') - { - m_current_builder.append_code_point(current_input_character.value()); - continue; - } - ANYTHING_ELSE - { - RECONSUME_IN(Comment); - } - } - END_STATE - - // 13.2.5.47 Comment less-than sign bang state, https://html.spec.whatwg.org/multipage/parsing.html#comment-less-than-sign-bang-state - BEGIN_STATE(CommentLessThanSignBang) - { - ON('-') - { - SWITCH_TO_WITH_UNCLEAN_BUILDER(CommentLessThanSignBangDash); - } - ANYTHING_ELSE - { - RECONSUME_IN(Comment); - } - } - END_STATE - - // 13.2.5.48 Comment less-than sign bang dash state, https://html.spec.whatwg.org/multipage/parsing.html#comment-less-than-sign-bang-dash-state - BEGIN_STATE(CommentLessThanSignBangDash) - { - ON('-') - { - SWITCH_TO_WITH_UNCLEAN_BUILDER(CommentLessThanSignBangDashDash); - } - ANYTHING_ELSE - { - RECONSUME_IN(CommentEndDash); - } - } - END_STATE - - // 13.2.5.49 Comment less-than sign bang dash dash state, https://html.spec.whatwg.org/multipage/parsing.html#comment-less-than-sign-bang-dash-dash-state - BEGIN_STATE(CommentLessThanSignBangDashDash) - { - ON('>') - { - RECONSUME_IN(CommentEnd); - } - ON_EOF - { - RECONSUME_IN(CommentEnd); - } - ANYTHING_ELSE - { - log_parse_error(); - RECONSUME_IN(CommentEnd); - } - } - END_STATE - - // 13.2.5.72 Character reference state, https://html.spec.whatwg.org/multipage/parsing.html#character-reference-state - BEGIN_STATE(CharacterReference) - { - m_temporary_buffer.clear(); - m_temporary_buffer.append('&'); - - ON_ASCII_ALPHANUMERIC - { - m_named_character_reference_matcher = {}; - RECONSUME_IN(NamedCharacterReference); - } - ON('#') - { - m_temporary_buffer.append(current_input_character.value()); - SWITCH_TO_WITH_UNCLEAN_BUILDER(NumericCharacterReference); - } - ANYTHING_ELSE - { - FLUSH_CODEPOINTS_CONSUMED_AS_A_CHARACTER_REFERENCE; - RECONSUME_IN_RETURN_STATE; - } - } - END_STATE - - // 13.2.5.73 Named character reference state, https://html.spec.whatwg.org/multipage/parsing.html#named-character-reference-state - BEGIN_STATE(NamedCharacterReference) - { - if (can_run_out_of_characters(stop_at_insertion_point)) { - // If there is an insertion point, match code-point-by-code-point to handle the possibility of - // document.write being used to insert a named character reference one-code-point-at-a-time. Do - // the same for an open input stream, since more network input can extend the reference later. - if (current_input_character.has_value()) { - if (m_named_character_reference_matcher.try_consume_code_point(current_input_character.value())) { - m_temporary_buffer.append(current_input_character.value()); - continue; - } else { - DONT_CONSUME_NEXT_INPUT_CHARACTER; - } - } - } else { - // If there's no insertion point (this is the common case), it is safe to look ahead at the rest - // of the input and try to match a named character reference all-at-once. This is worthwhile - // because matching all-at-once ends up being more efficient. - auto starting_consumed_count = m_temporary_buffer.size(); - auto remaining_source = m_decoded_input.span().slice(m_prev_offset); - - for (auto const code_point : remaining_source) { - if (m_named_character_reference_matcher.try_consume_code_point(code_point)) { - m_temporary_buffer.append(code_point); - } else { - break; - } - } - - auto num_consumed = m_temporary_buffer.size() - starting_consumed_count; - if (num_consumed == 0) { - DONT_CONSUME_NEXT_INPUT_CHARACTER; - } else if (num_consumed > 1) { - skip(num_consumed - 1); - } - } - - // Only consume the characters within the longest match. It's possible that we've overconsumed code points, - // though, so we want to backtrack to the longest match found. For example, `¬indo` (which could still - // have lead to `⋵̸`) would need to backtrack back to `¬`. - auto overconsumed_code_points = m_named_character_reference_matcher.overconsumed_code_points(); - if (overconsumed_code_points > 0) { - restore_to(m_current_offset - overconsumed_code_points); - m_temporary_buffer.resize_and_keep_capacity(m_temporary_buffer.size() - overconsumed_code_points); - } - - auto mapped_codepoints = m_named_character_reference_matcher.code_points(); - // If there is a match - if (mapped_codepoints.has_value()) { - if (consumed_as_part_of_an_attribute() && !m_named_character_reference_matcher.last_match_ends_with_semicolon()) { - auto next_code_point = peek_code_point(0, stop_at_insertion_point); - if (next_code_point.has_value() && (next_code_point.value() == '=' || is_ascii_alphanumeric(next_code_point.value()))) { - FLUSH_CODEPOINTS_CONSUMED_AS_A_CHARACTER_REFERENCE; - SWITCH_TO_RETURN_STATE; - } - } - - if (!m_named_character_reference_matcher.last_match_ends_with_semicolon()) { - log_parse_error(); - } - - m_temporary_buffer.clear_with_capacity(); - m_temporary_buffer.append(mapped_codepoints.value().first); - auto second_codepoint = named_character_reference_second_codepoint_value(mapped_codepoints.value().second); - if (second_codepoint.has_value()) { - m_temporary_buffer.append(second_codepoint.value()); - } - - FLUSH_CODEPOINTS_CONSUMED_AS_A_CHARACTER_REFERENCE; - SWITCH_TO_RETURN_STATE; - } else { - FLUSH_CODEPOINTS_CONSUMED_AS_A_CHARACTER_REFERENCE; - SWITCH_TO_WITH_UNCLEAN_BUILDER(AmbiguousAmpersand); - } - } - END_STATE - - // 13.2.5.74 Ambiguous ampersand state, https://html.spec.whatwg.org/multipage/parsing.html#ambiguous-ampersand-state - BEGIN_STATE(AmbiguousAmpersand) - { - ON_ASCII_ALPHANUMERIC - { - if (consumed_as_part_of_an_attribute()) { - m_current_builder.append_code_point(current_input_character.value()); - continue; - } else { - EMIT_CURRENT_CHARACTER; - } - } - ON(';') - { - log_parse_error(); - RECONSUME_IN_RETURN_STATE; - } - ANYTHING_ELSE - { - RECONSUME_IN_RETURN_STATE; - } - } - END_STATE - - // 13.2.5.75 Numeric character reference state, https://html.spec.whatwg.org/multipage/parsing.html#numeric-character-reference-state - BEGIN_STATE(NumericCharacterReference) - { - m_character_reference_code = 0; - - ON('X') - { - m_temporary_buffer.append(current_input_character.value()); - SWITCH_TO_WITH_UNCLEAN_BUILDER(HexadecimalCharacterReferenceStart); - } - ON('x') - { - m_temporary_buffer.append(current_input_character.value()); - SWITCH_TO_WITH_UNCLEAN_BUILDER(HexadecimalCharacterReferenceStart); - } - ANYTHING_ELSE - { - RECONSUME_IN(DecimalCharacterReferenceStart); - } - } - END_STATE - - // 13.2.5.76 Hexadecimal character reference start state, https://html.spec.whatwg.org/multipage/parsing.html#hexadecimal-character-reference-start-state - BEGIN_STATE(HexadecimalCharacterReferenceStart) - { - ON_ASCII_HEX_DIGIT - { - RECONSUME_IN(HexadecimalCharacterReference); - } - ANYTHING_ELSE - { - log_parse_error(); - FLUSH_CODEPOINTS_CONSUMED_AS_A_CHARACTER_REFERENCE; - RECONSUME_IN_RETURN_STATE; - } - } - END_STATE - - // 13.2.5.77 Decimal character reference start state, https://html.spec.whatwg.org/multipage/parsing.html#decimal-character-reference-start-state - BEGIN_STATE(DecimalCharacterReferenceStart) - { - ON_ASCII_DIGIT - { - RECONSUME_IN(DecimalCharacterReference); - } - ANYTHING_ELSE - { - log_parse_error(); - FLUSH_CODEPOINTS_CONSUMED_AS_A_CHARACTER_REFERENCE; - RECONSUME_IN_RETURN_STATE; - } - } - END_STATE - - // 13.2.5.78 Hexadecimal character reference state, https://html.spec.whatwg.org/multipage/parsing.html#decimal-character-reference-start-state - BEGIN_STATE(HexadecimalCharacterReference) - { - ON_ASCII_DIGIT - { - m_character_reference_code *= 16; - m_character_reference_code += current_input_character.value() - 0x30; - continue; - } - ON_ASCII_HEX_DIGIT - { - m_character_reference_code *= 16; - auto hex_digit_min_ascii_value = is_ascii_upper_alpha(current_input_character.value()) ? 0x37 : 0x57; - m_character_reference_code += current_input_character.value() - hex_digit_min_ascii_value; - continue; - } - ON(';') - { - SWITCH_TO_WITH_UNCLEAN_BUILDER(NumericCharacterReferenceEnd); - } - ANYTHING_ELSE - { - log_parse_error(); - RECONSUME_IN(NumericCharacterReferenceEnd); - } - } - END_STATE - - // 13.2.5.79 Decimal character reference state, https://html.spec.whatwg.org/multipage/parsing.html#decimal-character-reference-state - BEGIN_STATE(DecimalCharacterReference) - { - ON_ASCII_DIGIT - { - m_character_reference_code *= 10; - m_character_reference_code += current_input_character.value() - 0x30; - continue; - } - ON(';') - { - SWITCH_TO_WITH_UNCLEAN_BUILDER(NumericCharacterReferenceEnd); - } - ANYTHING_ELSE - { - log_parse_error(); - RECONSUME_IN(NumericCharacterReferenceEnd); - } - } - END_STATE - - // 13.2.5.80 Numeric character reference end state, https://html.spec.whatwg.org/multipage/parsing.html#numeric-character-reference-end-state - BEGIN_STATE(NumericCharacterReferenceEnd) - { - DONT_CONSUME_NEXT_INPUT_CHARACTER; - - if (m_character_reference_code == 0) { - log_parse_error(); - m_character_reference_code = 0xFFFD; - } - if (m_character_reference_code > 0x10ffff) { - log_parse_error(); - m_character_reference_code = 0xFFFD; - } - if (is_unicode_surrogate(m_character_reference_code)) { - log_parse_error(); - m_character_reference_code = 0xFFFD; - } - if (is_unicode_noncharacter(m_character_reference_code)) { - log_parse_error(); - } - if (m_character_reference_code == 0xd || (is_unicode_control(m_character_reference_code) && !is_ascii_space(m_character_reference_code))) { - log_parse_error(); - constexpr struct { - u32 number; - u32 code_point; - } conversion_table[] = { - { 0x80, 0x20AC }, - { 0x82, 0x201A }, - { 0x83, 0x0192 }, - { 0x84, 0x201E }, - { 0x85, 0x2026 }, - { 0x86, 0x2020 }, - { 0x87, 0x2021 }, - { 0x88, 0x02C6 }, - { 0x89, 0x2030 }, - { 0x8A, 0x0160 }, - { 0x8B, 0x2039 }, - { 0x8C, 0x0152 }, - { 0x8E, 0x017D }, - { 0x91, 0x2018 }, - { 0x92, 0x2019 }, - { 0x93, 0x201C }, - { 0x94, 0x201D }, - { 0x95, 0x2022 }, - { 0x96, 0x2013 }, - { 0x97, 0x2014 }, - { 0x98, 0x02DC }, - { 0x99, 0x2122 }, - { 0x9A, 0x0161 }, - { 0x9B, 0x203A }, - { 0x9C, 0x0153 }, - { 0x9E, 0x017E }, - { 0x9F, 0x0178 }, - }; - for (auto& entry : conversion_table) { - if (m_character_reference_code == entry.number) { - m_character_reference_code = entry.code_point; - break; - } - } - } - - m_temporary_buffer.clear(); - m_temporary_buffer.append(m_character_reference_code); - FLUSH_CODEPOINTS_CONSUMED_AS_A_CHARACTER_REFERENCE; - SWITCH_TO_RETURN_STATE; - } - END_STATE - - // 13.2.5.2 RCDATA state, https://html.spec.whatwg.org/multipage/parsing.html#rcdata-state - BEGIN_STATE(RCDATA) - { - ON('&') - { - m_return_state = State::RCDATA; - SWITCH_TO(CharacterReference); - } - ON('<') - { - SWITCH_TO(RCDATALessThanSign); - } - ON(0) - { - log_parse_error(); - EMIT_CHARACTER(0xFFFD); - } - ON_EOF - { - EMIT_EOF; - } - ANYTHING_ELSE - { - EMIT_CURRENT_CHARACTER; - } - } - END_STATE - - // 13.2.5.9 RCDATA less-than sign state, https://html.spec.whatwg.org/multipage/parsing.html#rcdata-less-than-sign-state - BEGIN_STATE(RCDATALessThanSign) - { - ON('/') - { - m_temporary_buffer.clear(); - SWITCH_TO(RCDATAEndTagOpen); - } - ANYTHING_ELSE - { - EMIT_CHARACTER_AND_RECONSUME_IN('<', RCDATA); - } - } - END_STATE - - // 13.2.5.10 RCDATA end tag open state, https://html.spec.whatwg.org/multipage/parsing.html#rcdata-end-tag-open-state - BEGIN_STATE(RCDATAEndTagOpen) - { - ON_ASCII_ALPHA - { - create_new_token(HTMLToken::Type::EndTag); - RECONSUME_IN(RCDATAEndTagName); - } - ANYTHING_ELSE - { - m_queued_tokens.enqueue(HTMLToken::make_character('<')); - m_queued_tokens.enqueue(HTMLToken::make_character('/')); - RECONSUME_IN(RCDATA); - } - } - END_STATE - - // 13.2.5.11 RCDATA end tag name state, https://html.spec.whatwg.org/multipage/parsing.html#rcdata-end-tag-name-state - BEGIN_STATE(RCDATAEndTagName) - { - ON_WHITESPACE - { - m_current_token.set_tag_name(consume_current_builder()); - if (!current_end_tag_token_is_appropriate()) { - m_queued_tokens.enqueue(HTMLToken::make_character('<')); - m_queued_tokens.enqueue(HTMLToken::make_character('/')); - for (auto code_point : m_temporary_buffer) - m_queued_tokens.enqueue(HTMLToken::make_character(code_point)); - RECONSUME_IN(RCDATA); - } - SWITCH_TO(BeforeAttributeName); - } - ON('/') - { - m_current_token.set_tag_name(consume_current_builder()); - if (!current_end_tag_token_is_appropriate()) { - m_queued_tokens.enqueue(HTMLToken::make_character('<')); - m_queued_tokens.enqueue(HTMLToken::make_character('/')); - for (auto code_point : m_temporary_buffer) - m_queued_tokens.enqueue(HTMLToken::make_character(code_point)); - RECONSUME_IN(RCDATA); - } - SWITCH_TO(SelfClosingStartTag); - } - ON('>') - { - m_current_token.set_tag_name(consume_current_builder()); - if (!current_end_tag_token_is_appropriate()) { - m_queued_tokens.enqueue(HTMLToken::make_character('<')); - m_queued_tokens.enqueue(HTMLToken::make_character('/')); - for (auto code_point : m_temporary_buffer) - m_queued_tokens.enqueue(HTMLToken::make_character(code_point)); - RECONSUME_IN(RCDATA); - } - SWITCH_TO_AND_EMIT_CURRENT_TOKEN(Data); - } - ON_ASCII_UPPER_ALPHA - { - m_current_builder.append_code_point(to_ascii_lowercase(current_input_character.value())); - m_temporary_buffer.append(current_input_character.value()); - continue; - } - ON_ASCII_LOWER_ALPHA - { - m_current_builder.append_code_point(current_input_character.value()); - m_temporary_buffer.append(current_input_character.value()); - continue; - } - ANYTHING_ELSE - { - m_queued_tokens.enqueue(HTMLToken::make_character('<')); - m_queued_tokens.enqueue(HTMLToken::make_character('/')); - // NOTE: The spec doesn't mention this, but it seems that m_current_token (an end tag) is just dropped in this case. - m_current_builder.clear(); - for (auto code_point : m_temporary_buffer) - m_queued_tokens.enqueue(HTMLToken::make_character(code_point)); - RECONSUME_IN(RCDATA); - } - } - END_STATE - - // 13.2.5.3 RAWTEXT state, https://html.spec.whatwg.org/multipage/parsing.html#rawtext-state - BEGIN_STATE(RAWTEXT) - { - ON('<') - { - SWITCH_TO(RAWTEXTLessThanSign); - } - ON(0) - { - log_parse_error(); - EMIT_CHARACTER(0xFFFD); - } - ON_EOF - { - EMIT_EOF; - } - ANYTHING_ELSE - { - EMIT_CURRENT_CHARACTER; - } - } - END_STATE - - // 13.2.5.12 RAWTEXT less-than sign state, https://html.spec.whatwg.org/multipage/parsing.html#rawtext-less-than-sign-state - BEGIN_STATE(RAWTEXTLessThanSign) - { - ON('/') - { - m_temporary_buffer.clear(); - SWITCH_TO(RAWTEXTEndTagOpen); - } - ANYTHING_ELSE - { - EMIT_CHARACTER_AND_RECONSUME_IN('<', RAWTEXT); - } - } - END_STATE - - // 13.2.5.13 RAWTEXT end tag open state, https://html.spec.whatwg.org/multipage/parsing.html#rawtext-end-tag-open-state - BEGIN_STATE(RAWTEXTEndTagOpen) - { - ON_ASCII_ALPHA - { - create_new_token(HTMLToken::Type::EndTag); - RECONSUME_IN(RAWTEXTEndTagName); - } - ANYTHING_ELSE - { - m_queued_tokens.enqueue(HTMLToken::make_character('<')); - m_queued_tokens.enqueue(HTMLToken::make_character('/')); - RECONSUME_IN(RAWTEXT); - } - } - END_STATE - - // 13.2.5.14 RAWTEXT end tag name state, https://html.spec.whatwg.org/multipage/parsing.html#rawtext-end-tag-name-state - BEGIN_STATE(RAWTEXTEndTagName) - { - ON_WHITESPACE - { - m_current_token.set_tag_name(consume_current_builder()); - if (!current_end_tag_token_is_appropriate()) { - m_queued_tokens.enqueue(HTMLToken::make_character('<')); - m_queued_tokens.enqueue(HTMLToken::make_character('/')); - for (auto code_point : m_temporary_buffer) - m_queued_tokens.enqueue(HTMLToken::make_character(code_point)); - RECONSUME_IN(RAWTEXT); - } - SWITCH_TO(BeforeAttributeName); - } - ON('/') - { - m_current_token.set_tag_name(consume_current_builder()); - if (!current_end_tag_token_is_appropriate()) { - m_queued_tokens.enqueue(HTMLToken::make_character('<')); - m_queued_tokens.enqueue(HTMLToken::make_character('/')); - for (auto code_point : m_temporary_buffer) - m_queued_tokens.enqueue(HTMLToken::make_character(code_point)); - RECONSUME_IN(RAWTEXT); - } - SWITCH_TO(SelfClosingStartTag); - } - ON('>') - { - m_current_token.set_tag_name(consume_current_builder()); - if (!current_end_tag_token_is_appropriate()) { - m_queued_tokens.enqueue(HTMLToken::make_character('<')); - m_queued_tokens.enqueue(HTMLToken::make_character('/')); - for (auto code_point : m_temporary_buffer) - m_queued_tokens.enqueue(HTMLToken::make_character(code_point)); - RECONSUME_IN(RAWTEXT); - } - SWITCH_TO_AND_EMIT_CURRENT_TOKEN(Data); - } - ON_ASCII_UPPER_ALPHA - { - m_current_builder.append_code_point(to_ascii_lowercase(current_input_character.value())); - m_temporary_buffer.append(current_input_character.value()); - continue; - } - ON_ASCII_LOWER_ALPHA - { - m_current_builder.append(current_input_character.value()); - m_temporary_buffer.append(current_input_character.value()); - continue; - } - ANYTHING_ELSE - { - m_queued_tokens.enqueue(HTMLToken::make_character('<')); - m_queued_tokens.enqueue(HTMLToken::make_character('/')); - // NOTE: The spec doesn't mention this, but it seems that m_current_token (an end tag) is just dropped in this case. - m_current_builder.clear(); - for (auto code_point : m_temporary_buffer) - m_queued_tokens.enqueue(HTMLToken::make_character(code_point)); - RECONSUME_IN(RAWTEXT); - } - } - END_STATE - - // 13.2.5.4 Script data state, https://html.spec.whatwg.org/multipage/parsing.html#script-data-state - BEGIN_STATE(ScriptData) - { - ON('<') - { - SWITCH_TO(ScriptDataLessThanSign); - } - ON(0) - { - log_parse_error(); - EMIT_CHARACTER(0xFFFD); - } - ON_EOF - { - EMIT_EOF; - } - ANYTHING_ELSE - { - EMIT_CURRENT_CHARACTER; - } - } - END_STATE - - // 13.2.5.5 PLAINTEXT state, https://html.spec.whatwg.org/multipage/parsing.html#plaintext-state - BEGIN_STATE(PLAINTEXT) - { - ON(0) - { - log_parse_error(); - EMIT_CHARACTER(0xFFFD); - } - ON_EOF - { - EMIT_EOF; - } - ANYTHING_ELSE - { - EMIT_CURRENT_CHARACTER; - } - } - END_STATE - - // 13.2.5.15 Script data less-than sign state, https://html.spec.whatwg.org/multipage/parsing.html#script-data-less-than-sign-state - BEGIN_STATE(ScriptDataLessThanSign) - { - ON('/') - { - m_temporary_buffer.clear(); - SWITCH_TO(ScriptDataEndTagOpen); - } - ON('!') - { - m_queued_tokens.enqueue(HTMLToken::make_character('<')); - m_queued_tokens.enqueue(HTMLToken::make_character('!')); - SWITCH_TO(ScriptDataEscapeStart); - } - ANYTHING_ELSE - { - EMIT_CHARACTER_AND_RECONSUME_IN('<', ScriptData); - } - } - END_STATE - - // 13.2.5.18 Script data escape start state, https://html.spec.whatwg.org/multipage/parsing.html#script-data-escape-start-state - BEGIN_STATE(ScriptDataEscapeStart) - { - ON('-') - { - SWITCH_TO_AND_EMIT_CHARACTER('-', ScriptDataEscapeStartDash); - } - ANYTHING_ELSE - { - RECONSUME_IN(ScriptData); - } - } - END_STATE - - // 13.2.5.19 Script data escape start dash state, https://html.spec.whatwg.org/multipage/parsing.html#script-data-escape-start-dash-state - BEGIN_STATE(ScriptDataEscapeStartDash) - { - ON('-') - { - SWITCH_TO_AND_EMIT_CHARACTER('-', ScriptDataEscapedDashDash); - } - ANYTHING_ELSE - { - RECONSUME_IN(ScriptData); - } - } - END_STATE - - // 13.2.5.22 Script data escaped dash dash state, https://html.spec.whatwg.org/multipage/parsing.html#script-data-escaped-dash-dash-state - BEGIN_STATE(ScriptDataEscapedDashDash) - { - ON('-') - { - EMIT_CHARACTER('-'); - } - ON('<') - { - SWITCH_TO(ScriptDataEscapedLessThanSign); - } - ON('>') - { - SWITCH_TO_AND_EMIT_CHARACTER('>', ScriptData); - } - ON(0) - { - log_parse_error(); - SWITCH_TO_AND_EMIT_CHARACTER(0xFFFD, ScriptDataEscaped); - } - ON_EOF - { - log_parse_error(); - EMIT_EOF; - } - ANYTHING_ELSE - { - SWITCH_TO_AND_EMIT_CURRENT_CHARACTER(ScriptDataEscaped); - } - } - END_STATE - - // 13.2.5.23 Script data escaped less-than sign state, https://html.spec.whatwg.org/multipage/parsing.html#script-data-escaped-less-than-sign-state - BEGIN_STATE(ScriptDataEscapedLessThanSign) - { - ON('/') - { - m_temporary_buffer.clear(); - SWITCH_TO(ScriptDataEscapedEndTagOpen); - } - ON_ASCII_ALPHA - { - m_temporary_buffer.clear(); - EMIT_CHARACTER_AND_RECONSUME_IN('<', ScriptDataDoubleEscapeStart); - } - ANYTHING_ELSE - { - EMIT_CHARACTER_AND_RECONSUME_IN('<', ScriptDataEscaped); - } - } - END_STATE - - // 13.2.5.24 Script data escaped end tag open state, https://html.spec.whatwg.org/multipage/parsing.html#script-data-escaped-end-tag-open-state - BEGIN_STATE(ScriptDataEscapedEndTagOpen) - { - ON_ASCII_ALPHA - { - create_new_token(HTMLToken::Type::EndTag); - RECONSUME_IN(ScriptDataEscapedEndTagName); - } - ANYTHING_ELSE - { - m_queued_tokens.enqueue(HTMLToken::make_character('<')); - m_queued_tokens.enqueue(HTMLToken::make_character('/')); - RECONSUME_IN(ScriptDataEscaped); - } - } - END_STATE - - // 13.2.5.25 Script data escaped end tag name state, https://html.spec.whatwg.org/multipage/parsing.html#script-data-escaped-end-tag-name-state - BEGIN_STATE(ScriptDataEscapedEndTagName) - { - ON_WHITESPACE - { - m_current_token.set_tag_name(consume_current_builder()); - if (current_end_tag_token_is_appropriate()) - SWITCH_TO(BeforeAttributeName); - - m_queued_tokens.enqueue(HTMLToken::make_character('<')); - m_queued_tokens.enqueue(HTMLToken::make_character('/')); - // NOTE: The spec doesn't mention this, but it seems that m_current_token (an end tag) is just dropped in this case. - m_current_builder.clear(); - for (auto code_point : m_temporary_buffer) { - m_queued_tokens.enqueue(HTMLToken::make_character(code_point)); - } - RECONSUME_IN(ScriptDataEscaped); - } - ON('/') - { - m_current_token.set_tag_name(consume_current_builder()); - if (current_end_tag_token_is_appropriate()) - SWITCH_TO(SelfClosingStartTag); - - m_queued_tokens.enqueue(HTMLToken::make_character('<')); - m_queued_tokens.enqueue(HTMLToken::make_character('/')); - // NOTE: The spec doesn't mention this, but it seems that m_current_token (an end tag) is just dropped in this case. - m_current_builder.clear(); - for (auto code_point : m_temporary_buffer) { - m_queued_tokens.enqueue(HTMLToken::make_character(code_point)); - } - RECONSUME_IN(ScriptDataEscaped); - } - ON('>') - { - m_current_token.set_tag_name(consume_current_builder()); - if (current_end_tag_token_is_appropriate()) - SWITCH_TO_AND_EMIT_CURRENT_TOKEN(Data); - - m_queued_tokens.enqueue(HTMLToken::make_character('<')); - m_queued_tokens.enqueue(HTMLToken::make_character('/')); - // NOTE: The spec doesn't mention this, but it seems that m_current_token (an end tag) is just dropped in this case. - m_current_builder.clear(); - for (auto code_point : m_temporary_buffer) { - m_queued_tokens.enqueue(HTMLToken::make_character(code_point)); - } - RECONSUME_IN(ScriptDataEscaped); - } - ON_ASCII_UPPER_ALPHA - { - m_current_builder.append_code_point(to_ascii_lowercase(current_input_character.value())); - m_temporary_buffer.append(current_input_character.value()); - continue; - } - ON_ASCII_LOWER_ALPHA - { - m_current_builder.append(current_input_character.value()); - m_temporary_buffer.append(current_input_character.value()); - continue; - } - ANYTHING_ELSE - { - m_queued_tokens.enqueue(HTMLToken::make_character('<')); - m_queued_tokens.enqueue(HTMLToken::make_character('/')); - // NOTE: The spec doesn't mention this, but it seems that m_current_token (an end tag) is just dropped in this case. - m_current_builder.clear(); - for (auto code_point : m_temporary_buffer) { - m_queued_tokens.enqueue(HTMLToken::make_character(code_point)); - } - RECONSUME_IN(ScriptDataEscaped); - } - } - END_STATE - - // 13.2.5.26 Script data double escape start state, https://html.spec.whatwg.org/multipage/parsing.html#script-data-double-escape-start-state - BEGIN_STATE(ScriptDataDoubleEscapeStart) - { - auto temporary_buffer_equal_to_script = [this]() -> bool { - if (m_temporary_buffer.size() != 6) - return false; - - // FIXME: Is there a better way of doing this? - return m_temporary_buffer[0] == 's' && m_temporary_buffer[1] == 'c' && m_temporary_buffer[2] == 'r' && m_temporary_buffer[3] == 'i' && m_temporary_buffer[4] == 'p' && m_temporary_buffer[5] == 't'; - }; - ON_WHITESPACE - { - if (temporary_buffer_equal_to_script()) - SWITCH_TO_AND_EMIT_CURRENT_CHARACTER(ScriptDataDoubleEscaped); - else - SWITCH_TO_AND_EMIT_CURRENT_CHARACTER(ScriptDataEscaped); - } - ON('/') - { - if (temporary_buffer_equal_to_script()) - SWITCH_TO_AND_EMIT_CURRENT_CHARACTER(ScriptDataDoubleEscaped); - else - SWITCH_TO_AND_EMIT_CURRENT_CHARACTER(ScriptDataEscaped); - } - ON('>') - { - if (temporary_buffer_equal_to_script()) - SWITCH_TO_AND_EMIT_CURRENT_CHARACTER(ScriptDataDoubleEscaped); - else - SWITCH_TO_AND_EMIT_CURRENT_CHARACTER(ScriptDataEscaped); - } - ON_ASCII_UPPER_ALPHA - { - m_temporary_buffer.append(to_ascii_lowercase(current_input_character.value())); - EMIT_CURRENT_CHARACTER; - } - ON_ASCII_LOWER_ALPHA - { - m_temporary_buffer.append(current_input_character.value()); - EMIT_CURRENT_CHARACTER; - } - ANYTHING_ELSE - { - RECONSUME_IN(ScriptDataEscaped); - } - } - END_STATE - - // 13.2.5.27 Script data double escaped state, https://html.spec.whatwg.org/multipage/parsing.html#script-data-double-escaped-state - BEGIN_STATE(ScriptDataDoubleEscaped) - { - ON('-') - { - SWITCH_TO_AND_EMIT_CHARACTER('-', ScriptDataDoubleEscapedDash); - } - ON('<') - { - SWITCH_TO_AND_EMIT_CHARACTER('<', ScriptDataDoubleEscapedLessThanSign); - } - ON(0) - { - log_parse_error(); - EMIT_CHARACTER(0xFFFD); - } - ON_EOF - { - log_parse_error(); - EMIT_EOF; - } - ANYTHING_ELSE - { - EMIT_CURRENT_CHARACTER; - } - } - END_STATE - - // 13.2.5.28 Script data double escaped dash state, https://html.spec.whatwg.org/multipage/parsing.html#script-data-double-escaped-dash-state - BEGIN_STATE(ScriptDataDoubleEscapedDash) - { - ON('-') - { - SWITCH_TO_AND_EMIT_CHARACTER('-', ScriptDataDoubleEscapedDashDash); - } - ON('<') - { - SWITCH_TO_AND_EMIT_CHARACTER('<', ScriptDataDoubleEscapedLessThanSign); - } - ON(0) - { - log_parse_error(); - SWITCH_TO_AND_EMIT_CHARACTER(0xFFFD, ScriptDataDoubleEscaped); - } - ON_EOF - { - log_parse_error(); - EMIT_EOF; - } - ANYTHING_ELSE - { - SWITCH_TO_AND_EMIT_CURRENT_CHARACTER(ScriptDataDoubleEscaped); - } - } - END_STATE - - // 13.2.5.29 Script data double escaped dash dash state, https://html.spec.whatwg.org/multipage/parsing.html#script-data-double-escaped-dash-dash-state - BEGIN_STATE(ScriptDataDoubleEscapedDashDash) - { - ON('-') - { - EMIT_CHARACTER('-'); - } - ON('<') - { - SWITCH_TO_AND_EMIT_CHARACTER('<', ScriptDataDoubleEscapedLessThanSign); - } - ON('>') - { - SWITCH_TO_AND_EMIT_CHARACTER('>', ScriptData); - } - ON(0) - { - log_parse_error(); - SWITCH_TO_AND_EMIT_CHARACTER(0xFFFD, ScriptDataDoubleEscaped); - } - ON_EOF - { - log_parse_error(); - EMIT_EOF; - } - ANYTHING_ELSE - { - SWITCH_TO_AND_EMIT_CURRENT_CHARACTER(ScriptDataDoubleEscaped); - } - } - END_STATE - - // 13.2.5.30 Script data double escaped less-than sign state, https://html.spec.whatwg.org/multipage/parsing.html#script-data-double-escaped-less-than-sign-state - BEGIN_STATE(ScriptDataDoubleEscapedLessThanSign) - { - ON('/') - { - m_temporary_buffer.clear(); - SWITCH_TO_AND_EMIT_CHARACTER('/', ScriptDataDoubleEscapeEnd); - } - ANYTHING_ELSE - { - RECONSUME_IN(ScriptDataDoubleEscaped); - } - } - END_STATE - - // 13.2.5.31 Script data double escape end state, https://html.spec.whatwg.org/multipage/parsing.html#script-data-double-escape-end-state - BEGIN_STATE(ScriptDataDoubleEscapeEnd) - { - auto temporary_buffer_equal_to_script = [this]() -> bool { - if (m_temporary_buffer.size() != 6) - return false; - - // FIXME: Is there a better way of doing this? - return m_temporary_buffer[0] == 's' && m_temporary_buffer[1] == 'c' && m_temporary_buffer[2] == 'r' && m_temporary_buffer[3] == 'i' && m_temporary_buffer[4] == 'p' && m_temporary_buffer[5] == 't'; - }; - ON_WHITESPACE - { - if (temporary_buffer_equal_to_script()) - SWITCH_TO_AND_EMIT_CURRENT_CHARACTER(ScriptDataEscaped); - else - SWITCH_TO_AND_EMIT_CURRENT_CHARACTER(ScriptDataDoubleEscaped); - } - ON('/') - { - if (temporary_buffer_equal_to_script()) - SWITCH_TO_AND_EMIT_CURRENT_CHARACTER(ScriptDataEscaped); - else - SWITCH_TO_AND_EMIT_CURRENT_CHARACTER(ScriptDataDoubleEscaped); - } - ON('>') - { - if (temporary_buffer_equal_to_script()) - SWITCH_TO_AND_EMIT_CURRENT_CHARACTER(ScriptDataEscaped); - else - SWITCH_TO_AND_EMIT_CURRENT_CHARACTER(ScriptDataDoubleEscaped); - } - ON_ASCII_UPPER_ALPHA - { - m_temporary_buffer.append(to_ascii_lowercase(current_input_character.value())); - EMIT_CURRENT_CHARACTER; - } - ON_ASCII_LOWER_ALPHA - { - m_temporary_buffer.append(current_input_character.value()); - EMIT_CURRENT_CHARACTER; - } - ANYTHING_ELSE - { - RECONSUME_IN(ScriptDataDoubleEscaped); - } - } - END_STATE - - // 13.2.5.21 Script data escaped dash state, https://html.spec.whatwg.org/multipage/parsing.html#script-data-escaped-dash-state - BEGIN_STATE(ScriptDataEscapedDash) - { - ON('-') - { - SWITCH_TO_AND_EMIT_CHARACTER('-', ScriptDataEscapedDashDash); - } - ON('<') - { - SWITCH_TO(ScriptDataEscapedLessThanSign); - } - ON(0) - { - log_parse_error(); - SWITCH_TO_AND_EMIT_CHARACTER(0xFFFD, ScriptDataEscaped); - } - ON_EOF - { - log_parse_error(); - EMIT_EOF; - } - ANYTHING_ELSE - { - SWITCH_TO_AND_EMIT_CURRENT_CHARACTER(ScriptDataEscaped); - } - } - END_STATE - - // 13.2.5.20 Script data escaped state, https://html.spec.whatwg.org/multipage/parsing.html#script-data-escaped-state - BEGIN_STATE(ScriptDataEscaped) - { - ON('-') - { - SWITCH_TO_AND_EMIT_CHARACTER('-', ScriptDataEscapedDash); - } - ON('<') - { - SWITCH_TO(ScriptDataEscapedLessThanSign); - } - ON(0) - { - log_parse_error(); - EMIT_CHARACTER(0xFFFD); - } - ON_EOF - { - log_parse_error(); - EMIT_EOF; - } - ANYTHING_ELSE - { - EMIT_CURRENT_CHARACTER; - } - } - END_STATE - - // 13.2.5.16 Script data end tag open state, https://html.spec.whatwg.org/multipage/parsing.html#script-data-end-tag-open-state - BEGIN_STATE(ScriptDataEndTagOpen) - { - ON_ASCII_ALPHA - { - create_new_token(HTMLToken::Type::EndTag); - RECONSUME_IN(ScriptDataEndTagName); - } - ANYTHING_ELSE - { - m_queued_tokens.enqueue(HTMLToken::make_character('<')); - m_queued_tokens.enqueue(HTMLToken::make_character('/')); - RECONSUME_IN(ScriptData); - } - } - END_STATE - - // 13.2.5.17 Script data end tag name state, https://html.spec.whatwg.org/multipage/parsing.html#script-data-end-tag-name-state - BEGIN_STATE(ScriptDataEndTagName) - { - ON_WHITESPACE - { - m_current_token.set_tag_name(consume_current_builder()); - if (current_end_tag_token_is_appropriate()) - SWITCH_TO(BeforeAttributeName); - m_queued_tokens.enqueue(HTMLToken::make_character('<')); - m_queued_tokens.enqueue(HTMLToken::make_character('/')); - // NOTE: The spec doesn't mention this, but it seems that m_current_token (an end tag) is just dropped in this case. - m_current_builder.clear(); - for (auto code_point : m_temporary_buffer) - m_queued_tokens.enqueue(HTMLToken::make_character(code_point)); - RECONSUME_IN(ScriptData); - } - ON('/') - { - m_current_token.set_tag_name(consume_current_builder()); - if (current_end_tag_token_is_appropriate()) - SWITCH_TO(SelfClosingStartTag); - m_queued_tokens.enqueue(HTMLToken::make_character('<')); - m_queued_tokens.enqueue(HTMLToken::make_character('/')); - // NOTE: The spec doesn't mention this, but it seems that m_current_token (an end tag) is just dropped in this case. - m_current_builder.clear(); - for (auto code_point : m_temporary_buffer) - m_queued_tokens.enqueue(HTMLToken::make_character(code_point)); - RECONSUME_IN(ScriptData); - } - ON('>') - { - m_current_token.set_tag_name(consume_current_builder()); - if (current_end_tag_token_is_appropriate()) - SWITCH_TO_AND_EMIT_CURRENT_TOKEN(Data); - m_queued_tokens.enqueue(HTMLToken::make_character('<')); - m_queued_tokens.enqueue(HTMLToken::make_character('/')); - // NOTE: The spec doesn't mention this, but it seems that m_current_token (an end tag) is just dropped in this case. - m_current_builder.clear(); - for (auto code_point : m_temporary_buffer) - m_queued_tokens.enqueue(HTMLToken::make_character(code_point)); - RECONSUME_IN(ScriptData); - } - ON_ASCII_UPPER_ALPHA - { - m_current_builder.append_code_point(to_ascii_lowercase(current_input_character.value())); - m_temporary_buffer.append(current_input_character.value()); - continue; - } - ON_ASCII_LOWER_ALPHA - { - m_current_builder.append(current_input_character.value()); - m_temporary_buffer.append(current_input_character.value()); - continue; - } - ANYTHING_ELSE - { - m_queued_tokens.enqueue(HTMLToken::make_character('<')); - m_queued_tokens.enqueue(HTMLToken::make_character('/')); - // NOTE: The spec doesn't mention this, but it seems that m_current_token (an end tag) is just dropped in this case. - m_current_builder.clear(); - for (auto code_point : m_temporary_buffer) - m_queued_tokens.enqueue(HTMLToken::make_character(code_point)); - RECONSUME_IN(ScriptData); - } - } - END_STATE - - // 13.2.5.69 CDATA section state, https://html.spec.whatwg.org/multipage/parsing.html#cdata-section-state - BEGIN_STATE(CDATASection) - { - ON(']') - { - SWITCH_TO(CDATASectionBracket); - } - ON_EOF - { - log_parse_error(); - EMIT_EOF; - } - ANYTHING_ELSE - { - EMIT_CURRENT_CHARACTER; - } - } - END_STATE - - // 13.2.5.70 CDATA section bracket state, https://html.spec.whatwg.org/multipage/parsing.html#cdata-section-bracket-state - BEGIN_STATE(CDATASectionBracket) - { - ON(']') - { - SWITCH_TO(CDATASectionEnd); - } - ANYTHING_ELSE - { - EMIT_CHARACTER_AND_RECONSUME_IN(']', CDATASection); - } - } - END_STATE - - // 13.2.5.71 CDATA section end state, https://html.spec.whatwg.org/multipage/parsing.html#cdata-section-end-state - BEGIN_STATE(CDATASectionEnd) - { - ON(']') - { - EMIT_CHARACTER(']'); - } - ON('>') - { - SWITCH_TO(Data); - } - ANYTHING_ELSE - { - m_queued_tokens.enqueue(HTMLToken::make_character(']')); - m_queued_tokens.enqueue(HTMLToken::make_character(']')); - RECONSUME_IN(CDATASection); - } - } - END_STATE - - default: - TODO(); - } - } -} - -HTMLTokenizer::ConsumeNextResult HTMLTokenizer::consume_next_if_match(StringView string, StopAtInsertionPoint stop_at_insertion_point, CaseSensitivity case_sensitivity) -{ - for (size_t i = 0; i < string.length(); ++i) { - auto code_point = peek_code_point(i, stop_at_insertion_point); - if (!code_point.has_value()) { - if (can_run_out_of_characters(stop_at_insertion_point)) - return ConsumeNextResult::RanOutOfCharacters; - return ConsumeNextResult::NotConsumed; - } - // FIXME: This should be more Unicode-aware. - if (case_sensitivity == CaseSensitivity::CaseInsensitive) { - if (code_point.value() < 0x80) { - if (to_ascii_lowercase(code_point.value()) != to_ascii_lowercase(string[i])) - return ConsumeNextResult::NotConsumed; - continue; - } - } - if (code_point.value() != (u32)string[i]) - return ConsumeNextResult::NotConsumed; - } - skip(string.length()); - return ConsumeNextResult::Consumed; -} - -void HTMLTokenizer::create_new_token(HTMLToken::Type type) -{ - m_current_token = { type }; - - auto is_start_or_end_tag = type == HTMLToken::Type::StartTag || type == HTMLToken::Type::EndTag; - m_current_token.set_start_position({}, nth_last_position(is_start_or_end_tag ? 1 : 0)); + static Vector const s_table = build_interned_name_table( + rust_html_tokenizer_interned_attr_name_count(), + rust_html_tokenizer_interned_attr_name); + if (id == 0 || id >= s_table.size()) + return s_table[0]; + return s_table[id]; } HTMLTokenizer::HTMLTokenizer() { - m_decoded_input = {}; - m_current_offset = 0; - m_prev_offset = 0; - m_source_positions.empend(0u, 0u); + m_tokenizer = create_tokenizer_from_utf8({}); + rust_html_tokenizer_set_input_stream_closed(m_tokenizer, false); +} + +HTMLTokenizer::~HTMLTokenizer() +{ + if (m_tokenizer) + rust_html_tokenizer_destroy(m_tokenizer); } HTMLTokenizer::HTMLTokenizer(StringView input, ByteString const& encoding) @@ -2924,148 +98,209 @@ HTMLTokenizer::HTMLTokenizer(StringView input, ByteString const& encoding) auto decoder = TextCodec::decoder_for(encoding); VERIFY(decoder.has_value()); m_source = MUST(decoder->to_utf8(input)); - m_decoded_input.ensure_capacity(m_source.bytes().size()); - for (auto code_point : m_source.code_points()) - m_decoded_input.append(code_point); - m_current_offset = 0; - m_prev_offset = 0; - m_source_positions.empend(0u, 0u); m_input_stream_closed = true; + m_tokenizer = create_tokenizer_from_utf8(m_source.bytes_as_string_view()); +} + +Optional HTMLTokenizer::next_token(StopAtInsertionPoint stop_at_insertion_point) +{ + RustFfiToken ffi; + bool stop = stop_at_insertion_point == StopAtInsertionPoint::Yes; + bool cdata_allowed = m_parser != nullptr + && m_parser->adjusted_current_node() + && m_parser->adjusted_current_node()->namespace_uri() != Namespace::HTML; + if (!rust_html_tokenizer_next_token(m_tokenizer, &ffi, stop, cdata_allowed)) + return {}; + + HTMLToken::Type type; + switch (ffi.token_type) { + case 1: + type = HTMLToken::Type::DOCTYPE; + break; + case 2: + type = HTMLToken::Type::StartTag; + break; + case 3: + type = HTMLToken::Type::EndTag; + break; + case 4: + type = HTMLToken::Type::Comment; + break; + case 5: + type = HTMLToken::Type::Character; + break; + case 6: + type = HTMLToken::Type::EndOfFile; + break; + default: + VERIFY_NOT_REACHED(); + } + + HTMLToken token { type }; + token.set_start_position({}, { ffi.start_line, ffi.start_column }); + token.set_end_position({}, { ffi.end_line, ffi.end_column }); + + switch (type) { + case HTMLToken::Type::Character: + token.set_code_point(ffi.code_point); + break; + case HTMLToken::Type::StartTag: + case HTMLToken::Type::EndTag: { + if (ffi.tag_name_id != 0) + token.set_tag_name(interned_rust_tag_name(ffi.tag_name_id)); + else + token.set_tag_name(MUST(FlyString::from_utf8(ffi_string_view(ffi.tag_name_ptr, ffi.tag_name_len)))); + + token.set_self_closing(ffi.self_closing); + for (size_t i = 0; i < ffi.attributes_len; ++i) { + auto const& ffi_attribute = ffi.attributes_ptr[i]; + HTMLToken::Attribute attribute; + if (ffi_attribute.name_id != 0) + attribute.local_name = interned_rust_attr_name(ffi_attribute.name_id); + else + attribute.local_name = MUST(FlyString::from_utf8(ffi_string_view(ffi_attribute.name_ptr, ffi_attribute.name_len))); + attribute.value = MUST(String::from_utf8(ffi_string_view(ffi_attribute.value_ptr, ffi_attribute.value_len))); + attribute.name_start_position = { ffi_attribute.name_start_line, ffi_attribute.name_start_column }; + attribute.name_end_position = { ffi_attribute.name_end_line, ffi_attribute.name_end_column }; + attribute.value_start_position = { ffi_attribute.value_start_line, ffi_attribute.value_start_column }; + attribute.value_end_position = { ffi_attribute.value_end_line, ffi_attribute.value_end_column }; + token.add_attribute(move(attribute)); + } + token.normalize_attributes(); + break; + } + case HTMLToken::Type::Comment: + token.set_comment(MUST(String::from_utf8(ffi_string_view(ffi.comment_ptr, ffi.comment_len)))); + break; + case HTMLToken::Type::DOCTYPE: { + auto& doctype = token.ensure_doctype_data(); + if (!ffi.missing_name) { + doctype.name = MUST(String::from_utf8(ffi_string_view(ffi.doctype_name_ptr, ffi.doctype_name_len))); + doctype.missing_name = false; + } + if (!ffi.missing_public_id) { + doctype.public_identifier = MUST(String::from_utf8(ffi_string_view(ffi.public_id_ptr, ffi.public_id_len))); + doctype.missing_public_identifier = false; + } + if (!ffi.missing_system_id) { + doctype.system_identifier = MUST(String::from_utf8(ffi_string_view(ffi.system_id_ptr, ffi.system_id_len))); + doctype.missing_system_identifier = false; + } + doctype.force_quirks = ffi.force_quirks; + break; + } + case HTMLToken::Type::EndOfFile: + break; + case HTMLToken::Type::Invalid: + VERIFY_NOT_REACHED(); + } + + return token; } void HTMLTokenizer::parser_did_run(Badge) { - // OPTIMIZATION: If we've consumed all input and the insertion point is at the start, - // we can throw away the decoded input buffer to save memory. - if (m_current_offset > 0 - && static_cast(m_current_offset) == m_decoded_input.size() - && (!m_insertion_point.has_value() || *m_insertion_point == 0) - && m_old_insertion_points.is_empty()) { - m_decoded_input.clear(); - m_current_offset = 0; - m_prev_offset = 0; - } + rust_html_tokenizer_parser_did_run(m_tokenizer); } String HTMLTokenizer::unparsed_input() const { - if (m_current_offset < 0 || static_cast(m_current_offset) >= m_decoded_input.size()) - return {}; - StringBuilder builder; - builder.append(Utf32View { m_decoded_input.span().slice(m_current_offset) }); - return MUST(builder.to_string()); + uint8_t const* ptr = nullptr; + size_t len = 0; + rust_html_tokenizer_unparsed_input(m_tokenizer, &ptr, &len); + return MUST(String::from_utf8(ffi_string_view(ptr, len))); } void HTMLTokenizer::append_to_input_stream(StringView input) { - for (auto code_point : Utf8View(input)) - m_decoded_input.append(code_point); + if (input.is_empty()) + return; + + auto utf8_input = MUST(String::from_utf8(input)); + auto code_points = code_points_from_string(utf8_input); + rust_html_tokenizer_append_input(m_tokenizer, code_points.data(), code_points.size()); } void HTMLTokenizer::close_input_stream() { m_input_stream_closed = true; + rust_html_tokenizer_set_input_stream_closed(m_tokenizer, true); } void HTMLTokenizer::insert_input_at_insertion_point(StringView input) { - Vector new_decoded_input; - new_decoded_input.ensure_capacity(m_decoded_input.size() + input.length()); - - auto insertion_point = *m_insertion_point; - auto before = m_decoded_input.span().slice(0, insertion_point); - new_decoded_input.append(before.data(), before.size()); - - auto utf8_to_insert = MUST(String::from_utf8(input)); - ssize_t code_points_inserted = 0; - for (auto code_point : utf8_to_insert.code_points()) { - new_decoded_input.append(code_point); - ++code_points_inserted; - } - - auto after = m_decoded_input.span().slice(insertion_point); - new_decoded_input.append(after.data(), after.size()); - m_decoded_input = move(new_decoded_input); - - m_insertion_point.value() += code_points_inserted; - for (auto& old_insertion_point : m_old_insertion_points) { - if (old_insertion_point.has_value() && insertion_point <= *old_insertion_point) - old_insertion_point.value() += code_points_inserted; - } + auto utf8_input = MUST(String::from_utf8(input)); + auto code_points = code_points_from_string(utf8_input); + rust_html_tokenizer_insert_input(m_tokenizer, code_points.data(), code_points.size()); } void HTMLTokenizer::insert_eof() { close_input_stream(); - m_explicit_eof_inserted = true; + rust_html_tokenizer_insert_eof(m_tokenizer); } bool HTMLTokenizer::is_eof_inserted() { - return m_explicit_eof_inserted; + return rust_html_tokenizer_is_eof_inserted(m_tokenizer); } -void HTMLTokenizer::will_switch_to([[maybe_unused]] State new_state) +void HTMLTokenizer::set_blocked(bool blocked) { - dbgln_if(TOKENIZER_TRACE_DEBUG, "[{}] Switch to {}", state_name(m_state), state_name(new_state)); + rust_html_tokenizer_set_blocked(m_tokenizer, blocked); } -void HTMLTokenizer::will_reconsume_in([[maybe_unused]] State new_state) +bool HTMLTokenizer::is_blocked() const { - dbgln_if(TOKENIZER_TRACE_DEBUG, "[{}] Reconsume in {}", state_name(m_state), state_name(new_state)); + return rust_html_tokenizer_is_blocked(m_tokenizer); +} + +bool HTMLTokenizer::is_insertion_point_defined() const +{ + return rust_html_tokenizer_is_insertion_point_defined(m_tokenizer); +} + +bool HTMLTokenizer::is_insertion_point_reached() +{ + return rust_html_tokenizer_is_insertion_point_reached(m_tokenizer); +} + +void HTMLTokenizer::undefine_insertion_point() +{ + rust_html_tokenizer_undefine_insertion_point(m_tokenizer); +} + +void HTMLTokenizer::store_insertion_point() +{ + rust_html_tokenizer_store_insertion_point(m_tokenizer); +} + +void HTMLTokenizer::restore_insertion_point() +{ + rust_html_tokenizer_restore_insertion_point(m_tokenizer); +} + +void HTMLTokenizer::update_insertion_point() +{ + rust_html_tokenizer_update_insertion_point(m_tokenizer); +} + +void HTMLTokenizer::abort() +{ + rust_html_tokenizer_abort(m_tokenizer); } void HTMLTokenizer::switch_to(Badge, State new_state) { dbgln_if(TOKENIZER_TRACE_DEBUG, "[{}] Parser switches tokenizer state to {}", state_name(m_state), state_name(new_state)); + switch_to(new_state); +} + +void HTMLTokenizer::switch_to(State new_state) +{ + dbgln_if(TOKENIZER_TRACE_DEBUG, "[{}] Switch to {}", state_name(m_state), state_name(new_state)); m_state = new_state; -} - -void HTMLTokenizer::will_emit(HTMLToken& token) -{ - if (token.is_start_tag()) - m_last_emitted_start_tag_name = token.tag_name(); - - auto is_start_or_end_tag = token.type() == HTMLToken::Type::StartTag || token.type() == HTMLToken::Type::EndTag; - token.set_end_position({}, nth_last_position(is_start_or_end_tag ? 1 : 0)); - - if (is_start_or_end_tag) - token.normalize_attributes(); -} - -bool HTMLTokenizer::current_end_tag_token_is_appropriate() const -{ - VERIFY(m_current_token.is_end_tag()); - if (!m_last_emitted_start_tag_name.has_value()) - return false; - return m_current_token.tag_name() == m_last_emitted_start_tag_name.value(); -} - -bool HTMLTokenizer::consumed_as_part_of_an_attribute() const -{ - return m_return_state == State::AttributeValueUnquoted || m_return_state == State::AttributeValueSingleQuoted || m_return_state == State::AttributeValueDoubleQuoted; -} - -void HTMLTokenizer::restore_to(ssize_t new_iterator) -{ - auto diff = m_current_offset - new_iterator; - if (diff > 0) { - for (ssize_t i = 0; i < diff; ++i) { - if (!m_source_positions.is_empty()) - m_source_positions.take_last(); - } - } else { - // Going forwards...? - TODO(); - } - m_current_offset = new_iterator; -} - -String HTMLTokenizer::consume_current_builder() -{ - auto string = m_current_builder.to_string_without_validation(); - m_current_builder.clear(); - return string; + rust_html_tokenizer_switch_state(m_tokenizer, static_cast(new_state)); } void HTMLTokenizer::visit_edges(GC::Cell::Visitor& visitor) diff --git a/Libraries/LibWeb/HTML/Parser/HTMLTokenizer.h b/Libraries/LibWeb/HTML/Parser/HTMLTokenizer.h index 2f99818f32..dac5355f00 100644 --- a/Libraries/LibWeb/HTML/Parser/HTMLTokenizer.h +++ b/Libraries/LibWeb/HTML/Parser/HTMLTokenizer.h @@ -7,18 +7,17 @@ #pragma once -#include -#include +#include #include #include -#include #include #include #include #include -#include #include +struct RustFfiTokenizerHandle; + namespace Web::HTML { #define ENUMERATE_TOKENIZER_STATES \ @@ -107,6 +106,7 @@ class WEB_API HTMLTokenizer { public: explicit HTMLTokenizer(); explicit HTMLTokenizer(StringView input, ByteString const& encoding); + ~HTMLTokenizer(); enum class State { #define __ENUMERATE_TOKENIZER_STATE(state) state, @@ -123,13 +123,10 @@ public: void set_parser(Badge, HTMLParser& parser) { m_parser = &parser; } void switch_to(Badge, State new_state); - void switch_to(State new_state) - { - m_state = new_state; - } + void switch_to(State new_state); - void set_blocked(bool b) { m_blocked = b; } - bool is_blocked() const { return m_blocked; } + void set_blocked(bool b); + bool is_blocked() const; auto const& source() const { return m_source; } @@ -142,38 +139,23 @@ public: void insert_eof(); bool is_eof_inserted(); - bool is_insertion_point_defined() const { return m_insertion_point.has_value(); } - bool is_insertion_point_reached() { return m_insertion_point.has_value() && m_current_offset >= *m_insertion_point; } - void undefine_insertion_point() { m_insertion_point = {}; } - void store_old_insertion_point() { m_old_insertion_points.append(m_insertion_point); } - void restore_old_insertion_point() { m_insertion_point = m_old_insertion_points.take_last(); } - void update_insertion_point() { m_insertion_point = m_current_offset; } + bool is_insertion_point_defined() const; + bool is_insertion_point_reached(); + void undefine_insertion_point(); + void store_insertion_point(); + void restore_insertion_point(); + void store_old_insertion_point() { store_insertion_point(); } + void restore_old_insertion_point() { restore_insertion_point(); } + void update_insertion_point(); // This permanently cuts off the tokenizer input stream. - void abort() { m_aborted = true; } + void abort(); void parser_did_run(Badge); void visit_edges(GC::Cell::Visitor&); private: - void skip(size_t count); - Optional next_code_point(StopAtInsertionPoint); - Optional peek_code_point(ssize_t offset, StopAtInsertionPoint) const; - - enum class ConsumeNextResult { - Consumed, - NotConsumed, - RanOutOfCharacters, - }; - [[nodiscard]] ConsumeNextResult consume_next_if_match(StringView, StopAtInsertionPoint, CaseSensitivity = CaseSensitivity::CaseSensitive); - bool should_pause_before_next_input_character(StopAtInsertionPoint) const; - bool can_run_out_of_characters(StopAtInsertionPoint) const; - - void create_new_token(HTMLToken::Type); - bool current_end_tag_token_is_appropriate() const; - String consume_current_builder(); - static char const* state_name(State state) { switch (state) { @@ -186,52 +168,13 @@ private: VERIFY_NOT_REACHED(); } - void will_emit(HTMLToken&); - void will_switch_to(State); - void will_reconsume_in(State); - - bool consumed_as_part_of_an_attribute() const; - - void restore_to(ssize_t new_iterator); - HTMLToken::Position nth_last_position(size_t n = 0); - GC::Ptr m_parser; State m_state { State::Data }; - State m_return_state { State::Data }; - - Vector m_temporary_buffer; - String m_source; - Vector m_decoded_input; - - Optional m_insertion_point; - // Spec algorithms have an "old insertion point" local; reentrant script execution can nest those locals. - Vector> m_old_insertion_points; - - ssize_t m_current_offset { 0 }; - ssize_t m_prev_offset { 0 }; - - HTMLToken m_current_token; - StringBuilder m_current_builder; - - NamedCharacterReferenceMatcher m_named_character_reference_matcher; - - Optional m_last_emitted_start_tag_name; - - bool m_explicit_eof_inserted { false }; bool m_input_stream_closed { false }; - bool m_has_emitted_eof { false }; - Queue m_queued_tokens; - - u32 m_character_reference_code { 0 }; - - bool m_blocked { false }; - - bool m_aborted { false }; - - Vector m_source_positions; + RustFfiTokenizerHandle* m_tokenizer { nullptr }; }; } diff --git a/Libraries/LibWeb/HTML/Parser/Rust/Cargo.toml b/Libraries/LibWeb/HTML/Parser/Rust/Cargo.toml new file mode 100644 index 0000000000..25cb66ec95 --- /dev/null +++ b/Libraries/LibWeb/HTML/Parser/Rust/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "libweb_html_tokenizer" +version = "0.1.0" +edition = "2024" + +[lib] +crate-type = ["rlib"] + +[build-dependencies] +cbindgen = "0.29" diff --git a/Libraries/LibWeb/HTML/Parser/Rust/build.rs b/Libraries/LibWeb/HTML/Parser/Rust/build.rs new file mode 100644 index 0000000000..9d7a5874f6 --- /dev/null +++ b/Libraries/LibWeb/HTML/Parser/Rust/build.rs @@ -0,0 +1,821 @@ +/* + * Copyright (c) 2026-present, the Ladybird developers. + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +//! Build script that generates a DAFSA (Deterministic Acyclic Finite State Automaton) +//! for named character reference matching. This is a Rust port of the C++ generator at +//! Meta/Lagom/Tools/CodeGenerators/LibWeb/GenerateNamedCharacterReferences.cpp. + +use std::cell::RefCell; +use std::collections::HashMap; +use std::env; +use std::fs; +use std::path::{Path, PathBuf}; +use std::rc::Rc; + +const FFI_HEADER: &str = "HTMLTokenizerRustFFI.h"; + +fn main() { + let manifest_dir = env::var("CARGO_MANIFEST_DIR").unwrap(); + let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap()); + + println!("cargo:rerun-if-changed=build.rs"); + println!("cargo:rerun-if-changed=cbindgen.toml"); + println!("cargo:rerun-if-changed=src"); + println!("cargo:rerun-if-env-changed=FFI_OUTPUT_DIR"); + + let ffi_out_dir = env::var("FFI_OUTPUT_DIR") + .map(PathBuf::from) + .unwrap_or_else(|_| out_dir.clone()); + + cbindgen::generate(&manifest_dir).map_or_else( + |error| match error { + cbindgen::Error::ParseSyntaxError { .. } => {} + e => panic!("{e:?}"), + }, + |bindings| { + bindings.write_to_file(out_dir.join(FFI_HEADER)); + if ffi_out_dir != out_dir { + bindings.write_to_file(ffi_out_dir.join(FFI_HEADER)); + } + }, + ); + + // Generate interned name tables from the existing C++ headers. + let tag_names_header = Path::new(&manifest_dir).join("../../TagNames.h"); + let attr_names_header = Path::new(&manifest_dir).join("../../AttributeNames.h"); + println!("cargo:rerun-if-changed={}", tag_names_header.display()); + println!("cargo:rerun-if-changed={}", attr_names_header.display()); + let tag_names = parse_enumerate_macro( + &fs::read_to_string(&tag_names_header).expect("Failed to read TagNames.h"), + "__ENUMERATE_HTML_TAG", + ); + let attr_names = parse_enumerate_macro( + &fs::read_to_string(&attr_names_header).expect("Failed to read AttributeNames.h"), + "__ENUMERATE_HTML_ATTRIBUTE", + ); + emit_interned_names(&out_dir.join("interned_names_generated.rs"), &tag_names, &attr_names); + + let json_path = Path::new(&manifest_dir).join("../Entities.json"); + println!("cargo:rerun-if-changed={}", json_path.display()); + + let json_str = fs::read_to_string(&json_path).expect("Failed to read Entities.json"); + let entities = parse_entities_json(&json_str); + + // Build DAFSA. + let mut builder = DafsaBuilder::new(); + for (name, _, _) in &entities { + builder.insert(name); + } + builder.minimize(0); + builder.calc_numbers(); + + // Verify minimal perfect hashing (no collisions). + let mut seen: Vec = vec![false; entities.len() + 1]; + for (name, _, _) in &entities { + let idx = builder.get_unique_index(name).unwrap(); + assert!(!seen[idx], "Hash collision at index {idx} for '{name}'"); + seen[idx] = true; + } + + // Build codepoints lookup table indexed by unique_index. + let mut index_to_codepoints = vec![(0u32, 0u32); entities.len()]; + for (name, first, second) in &entities { + let idx = builder.get_unique_index(name).unwrap(); + index_to_codepoints[idx - 1] = (*first, *second); + } + + // Extract DAFSA layers. + let root = &builder.root; + let root_ref = root.borrow(); + + let mut first_layer: Vec = Vec::new(); + let mut first_to_second_layer: Vec<(u64, u16)> = Vec::new(); + + let mut first_layer_tally: u16 = 0; + let mut second_layer_offset: u16 = 0; + + for c in 0u8..128 { + if root_ref.children[c as usize].is_none() { + continue; + } + assert!(c.is_ascii_alphabetic()); + let child = root_ref.children[c as usize].as_ref().unwrap(); + let child_ref = child.borrow(); + + first_layer.push(first_layer_tally); + first_layer_tally += child_ref.number; + + let mask = child_ref.get_ascii_alphabetic_bit_mask(); + first_to_second_layer.push((mask, second_layer_offset)); + second_layer_offset += child_ref.num_direct_children() as u16; + } + assert_eq!(first_layer.len(), 52); + + // BFS to build DAFSA node array. + // Following the C++ write_node_data three-phase approach: + type NodePtr = Rc>; + let mut queue: Vec = Vec::new(); + let mut child_indexes: HashMap<*const RefCell, u16> = HashMap::new(); + + // Phase 1: Queue root's children (first-layer nodes = 52 A-Z/a-z). + // This assigns temporary child_indexes for first-layer children. + queue_children(root, &mut queue, &mut child_indexes, 1); + + // Phase 2: Clear indexes and re-process. For each first-layer child, + // queue ITS children (second-layer nodes) and assign their child_indexes. + child_indexes.clear(); + let mut first_available_index: u16 = 1; // 0 is reserved (dummy node) + let first_layer_count = queue.len(); + for i in 0..first_layer_count { + let node = Rc::clone(&queue[i]); + first_available_index = queue_children(&node, &mut queue, &mut child_indexes, first_available_index); + } + // Remove first-layer nodes from queue, keep only second-layer+ nodes. + let second_layer_nodes: Vec = queue.drain(first_layer_count..).collect(); + queue.clear(); + queue.extend(second_layer_nodes); + + // Phase 3: BFS remaining nodes, writing node data. + let mut node_data: Vec = Vec::new(); + let mut qi = 0; + #[allow(unused_assignments)] + while qi < queue.len() { + let node = Rc::clone(&queue[qi]); + qi += 1; + first_available_index = write_children_data( + &node, + &mut node_data, + &mut queue, + &mut child_indexes, + first_available_index, + ); + } + + // Build second_layer entries with child_indexes from phase 2. + let mut second_layer: Vec = Vec::new(); + for c in 0u8..128 { + if root_ref.children[c as usize].is_none() { + continue; + } + let first_child = root_ref.children[c as usize].as_ref().unwrap(); + let first_child_ref = first_child.borrow(); + let mut tally: u8 = 0; + for cc in 0u8..128 { + if first_child_ref.children[cc as usize].is_none() { + continue; + } + let second_child = first_child_ref.children[cc as usize].as_ref().unwrap(); + let second_child_ref = second_child.borrow(); + let key = Rc::as_ptr(second_child); + let ci = child_indexes.get(&key).copied().unwrap_or(0); + let children_len = second_child_ref.num_direct_children(); + second_layer.push(SecondLayerEntry { + child_index: ci, + number: tally, + children_len, + end_of_word: second_child_ref.is_terminal, + }); + tally = tally.wrapping_add(second_child_ref.number as u8); + } + } + drop(root_ref); + + // Generate output file. + let out_dir = env::var("OUT_DIR").unwrap(); + let out_path = Path::new(&out_dir).join("named_character_references.rs"); + let mut out = String::new(); + + out.push_str("// Auto-generated by build.rs -- do not edit!\n\n"); + + // Second codepoint enum. + out.push_str("#[derive(Clone, Copy, PartialEq, Eq)]\n"); + out.push_str("#[repr(u8)]\n"); + out.push_str("pub enum SecondCodepoint {\n"); + out.push_str(" None = 0,\n"); + out.push_str(" CombiningLongSolidusOverlay = 1,\n"); + out.push_str(" CombiningLongVerticalLineOverlay = 2,\n"); + out.push_str(" HairSpace = 3,\n"); + out.push_str(" CombiningDoubleLowLine = 4,\n"); + out.push_str(" CombiningReverseSolidusOverlay = 5,\n"); + out.push_str(" VariationSelector1 = 6,\n"); + out.push_str(" LatinSmallLetterJ = 7,\n"); + out.push_str(" CombiningMacronBelow = 8,\n"); + out.push_str("}\n\n"); + + out.push_str("impl SecondCodepoint {\n"); + out.push_str(" pub fn value(self) -> u32 {\n"); + out.push_str(" match self {\n"); + out.push_str(" SecondCodepoint::None => 0,\n"); + out.push_str(" SecondCodepoint::CombiningLongSolidusOverlay => 0x0338,\n"); + out.push_str(" SecondCodepoint::CombiningLongVerticalLineOverlay => 0x20D2,\n"); + out.push_str(" SecondCodepoint::HairSpace => 0x200A,\n"); + out.push_str(" SecondCodepoint::CombiningDoubleLowLine => 0x0333,\n"); + out.push_str(" SecondCodepoint::CombiningReverseSolidusOverlay => 0x20E5,\n"); + out.push_str(" SecondCodepoint::VariationSelector1 => 0xFE00,\n"); + out.push_str(" SecondCodepoint::LatinSmallLetterJ => 0x006A,\n"); + out.push_str(" SecondCodepoint::CombiningMacronBelow => 0x0331,\n"); + out.push_str(" }\n"); + out.push_str(" }\n"); + out.push_str("}\n\n"); + + // Struct definitions. + out.push_str("#[derive(Clone, Copy)]\n"); + out.push_str("pub struct DafsaNode {\n"); + out.push_str(" pub character: u8,\n"); + out.push_str(" pub number: u8,\n"); + out.push_str(" pub end_of_word: bool,\n"); + out.push_str(" pub child_index: u16,\n"); + out.push_str(" pub children_len: u8,\n"); + out.push_str("}\n\n"); + + out.push_str("#[derive(Clone, Copy)]\n"); + out.push_str("pub struct SecondLayerNode {\n"); + out.push_str(" pub child_index: u16,\n"); + out.push_str(" pub number: u8,\n"); + out.push_str(" pub children_len: u8,\n"); + out.push_str(" pub end_of_word: bool,\n"); + out.push_str("}\n\n"); + + // Codepoints lookup table. + out.push_str(&format!( + "pub static CODEPOINTS_LOOKUP: [(u32, SecondCodepoint); {}] = [\n", + index_to_codepoints.len() + )); + for (first, second) in &index_to_codepoints { + let variant = second_codepoint_variant(*second); + out.push_str(&format!(" ({first:#06X}, SecondCodepoint::{variant}),\n")); + } + out.push_str("];\n\n"); + + // DAFSA nodes array (with dummy node at index 0). + out.push_str(&format!( + "pub static DAFSA_NODES: [DafsaNode; {}] = [\n", + node_data.len() + 1 + )); + out.push_str(" DafsaNode { character: 0, number: 0, end_of_word: false, child_index: 0, children_len: 0 },\n"); + for nd in &node_data { + out.push_str(&format!( + " DafsaNode {{ character: b'{}', number: {}, end_of_word: {}, child_index: {}, children_len: {} }},\n", + escape_byte(nd.character), + nd.number, + nd.end_of_word, + nd.child_index, + nd.children_len + )); + } + out.push_str("];\n\n"); + + // First layer. + out.push_str(&format!("pub static FIRST_LAYER: [u16; {}] = [\n", first_layer.len())); + for n in &first_layer { + out.push_str(&format!(" {n},\n")); + } + out.push_str("];\n\n"); + + // First-to-second layer links. + out.push_str(&format!( + "pub static FIRST_TO_SECOND_LAYER: [(u64, u16); {}] = [\n", + first_to_second_layer.len() + )); + for (mask, offset) in &first_to_second_layer { + out.push_str(&format!(" ({mask:#018X}, {offset}),\n")); + } + out.push_str("];\n\n"); + + // Second layer nodes. + out.push_str(&format!( + "pub static SECOND_LAYER: [SecondLayerNode; {}] = [\n", + second_layer.len() + )); + for sl in &second_layer { + out.push_str(&format!( + " SecondLayerNode {{ child_index: {}, number: {}, children_len: {}, end_of_word: {} }},\n", + sl.child_index, sl.number, sl.children_len, sl.end_of_word + )); + } + out.push_str("];\n\n"); + + // Total entity count. + out.push_str(&format!("pub const ENTITY_COUNT: usize = {};\n", entities.len())); + + fs::write(&out_path, &out).expect("Failed to write generated file"); +} + +/// Extract the string literal from `__ENUMERATE_FOO(ident, "string")` macro +/// invocations in a C++ header. +fn parse_enumerate_macro(source: &str, macro_name: &str) -> Vec { + let needle = format!("{macro_name}("); + let mut out = Vec::new(); + for line in source.lines() { + let Some(idx) = line.find(&needle) else { + continue; + }; + let rest = &line[idx + needle.len()..]; + // Take the second argument, which is the quoted string literal. + let Some(first_quote) = rest.find('"') else { + continue; + }; + let after = &rest[first_quote + 1..]; + let Some(end_quote) = after.find('"') else { + continue; + }; + out.push(after[..end_quote].to_string()); + } + out +} + +/// Emit a Rust source file with two const byte-slice arrays and two lookup +/// functions that dispatch on length and then on the exact bytes. rustc +/// compiles this pattern to a jump table + direct memcmp, which beats a +/// HashMap lookup with a cryptographic default hasher by a wide margin for +/// the small, fixed set of HTML names. +fn emit_interned_names(out_path: &Path, tag_names: &[String], attr_names: &[String]) { + let mut out = String::new(); + out.push_str("// Auto-generated by build.rs from TagNames.h / AttributeNames.h.\n"); + out.push_str("// Do not edit by hand.\n\n"); + + out.push_str("pub const INTERNED_TAG_NAMES: &[&[u8]] = &[\n"); + for name in tag_names { + out.push_str(&format!(" b\"{}\",\n", name)); + } + out.push_str("];\n\n"); + + out.push_str("pub const INTERNED_ATTR_NAMES: &[&[u8]] = &[\n"); + for name in attr_names { + out.push_str(&format!(" b\"{}\",\n", name)); + } + out.push_str("];\n\n"); + + emit_lookup_fn(&mut out, "lookup_tag_name_generated", tag_names); + emit_lookup_fn(&mut out, "lookup_attr_name_generated", attr_names); + + fs::write(out_path, out).expect("Failed to write interned_names_generated.rs"); +} + +fn emit_lookup_fn(out: &mut String, fn_name: &str, names: &[String]) { + // Group names by byte length so the outer dispatch can be a single match. + let mut by_length: std::collections::BTreeMap> = std::collections::BTreeMap::new(); + for (i, name) in names.iter().enumerate() { + by_length.entry(name.len()).or_default().push((i, name)); + } + + out.push_str(&format!("#[inline]\npub fn {fn_name}(bytes: &[u8]) -> u16 {{\n")); + out.push_str(" match bytes.len() {\n"); + for (length, entries) in &by_length { + out.push_str(&format!(" {length} => match bytes {{\n")); + for (index, name) in entries { + // id is 1-based. + let id = index + 1; + out.push_str(&format!(" b\"{name}\" => {id},\n")); + } + out.push_str(" _ => 0,\n"); + out.push_str(" },\n"); + } + out.push_str(" _ => 0,\n"); + out.push_str(" }\n"); + out.push_str("}\n\n"); +} + +fn escape_byte(b: u8) -> String { + if b == b'\'' { + "\\'".to_string() + } else if b == b'\\' { + "\\\\".to_string() + } else if b.is_ascii_graphic() || b == b' ' { + String::from(b as char) + } else { + format!("\\x{b:02X}") + } +} + +fn second_codepoint_variant(cp: u32) -> &'static str { + match cp { + 0 => "None", + 0x0338 => "CombiningLongSolidusOverlay", + 0x20D2 => "CombiningLongVerticalLineOverlay", + 0x200A => "HairSpace", + 0x0333 => "CombiningDoubleLowLine", + 0x20E5 => "CombiningReverseSolidusOverlay", + 0xFE00 => "VariationSelector1", + 0x006A => "LatinSmallLetterJ", + 0x0331 => "CombiningMacronBelow", + _ => panic!("Unknown second codepoint: {cp:#X}"), + } +} + +// Minimal JSON parser for Entities.json. +fn parse_entities_json(json: &str) -> Vec<(String, u32, u32)> { + let mut entities = Vec::new(); + let bytes = json.as_bytes(); + let len = bytes.len(); + let mut i = 0; + + while i < len && bytes[i] != b'{' { + i += 1; + } + i += 1; + + loop { + while i < len && bytes[i].is_ascii_whitespace() { + i += 1; + } + if i >= len || bytes[i] == b'}' { + break; + } + if bytes[i] == b',' { + i += 1; + continue; + } + + // Parse key. + assert_eq!(bytes[i], b'"'); + i += 1; + let key_start = i; + while i < len && bytes[i] != b'"' { + if bytes[i] == b'\\' { + i += 1; + } + i += 1; + } + let key = std::str::from_utf8(&bytes[key_start..i]).unwrap().to_string(); + i += 1; + + // Skip ':'. + while i < len && bytes[i].is_ascii_whitespace() { + i += 1; + } + assert_eq!(bytes[i], b':'); + i += 1; + + // Skip to inner '{'. + while i < len && bytes[i] != b'{' { + i += 1; + } + i += 1; + + // Parse inner object for "codepoints". + let mut codepoints: Vec = Vec::new(); + while i < len && bytes[i] != b'}' { + if bytes[i] == b'"' { + i += 1; + let field_start = i; + while i < len && bytes[i] != b'"' { + i += 1; + } + let field_name = std::str::from_utf8(&bytes[field_start..i]).unwrap(); + i += 1; + + while i < len && bytes[i].is_ascii_whitespace() { + i += 1; + } + assert_eq!(bytes[i], b':'); + i += 1; + while i < len && bytes[i].is_ascii_whitespace() { + i += 1; + } + + if field_name == "codepoints" { + assert_eq!(bytes[i], b'['); + i += 1; + loop { + while i < len && (bytes[i].is_ascii_whitespace() || bytes[i] == b',') { + i += 1; + } + if i >= len || bytes[i] == b']' { + i += 1; + break; + } + let num_start = i; + while i < len && bytes[i].is_ascii_digit() { + i += 1; + } + let num_str = std::str::from_utf8(&bytes[num_start..i]).unwrap(); + codepoints.push(num_str.parse().unwrap()); + } + } else { + // Skip value. + if bytes[i] == b'"' { + i += 1; + while i < len && bytes[i] != b'"' { + if bytes[i] == b'\\' { + i += 1; + } + i += 1; + } + i += 1; + } else if bytes[i] == b'[' { + let mut depth = 1; + i += 1; + while i < len && depth > 0 { + if bytes[i] == b'[' { + depth += 1; + } else if bytes[i] == b']' { + depth -= 1; + } + i += 1; + } + } + } + } else { + i += 1; + } + } + i += 1; + + let name = key.strip_prefix('&').unwrap_or(&key).to_string(); + let first = codepoints.first().copied().unwrap_or(0); + let second = if codepoints.len() > 1 { codepoints[1] } else { 0 }; + entities.push((name, first, second)); + } + + entities.sort_by(|a, b| a.0.cmp(&b.0)); + entities +} + +// DAFSA builder using Rc> for shared ownership. + +type NodeRc = Rc>; + +struct Node { + children: Vec>, // 128 slots + is_terminal: bool, + number: u16, +} + +struct SecondLayerEntry { + child_index: u16, + number: u8, + children_len: u8, + end_of_word: bool, +} + +struct NodeData { + character: u8, + number: u8, + end_of_word: bool, + child_index: u16, + children_len: u8, +} + +impl Node { + fn new_rc() -> NodeRc { + Rc::new(RefCell::new(Node { + children: (0..128).map(|_| Option::None).collect(), + is_terminal: false, + number: 0, + })) + } + + fn calc_numbers(&mut self) { + self.number = if self.is_terminal { 1 } else { 0 }; + for child in self.children.iter().flatten() { + child.borrow_mut().calc_numbers(); + self.number += child.borrow().number; + } + } + + fn num_direct_children(&self) -> u8 { + let mut n = 0u8; + for c in &self.children { + if c.is_some() { + n += 1; + } + } + n + } + + fn get_ascii_alphabetic_bit_mask(&self) -> u64 { + let mut mask = 0u64; + for i in 0..128u8 { + if self.children[i as usize].is_some() { + mask |= 1u64 << ascii_alphabetic_to_index(i); + } + } + mask + } + + /// Hash based on child identities (Rc pointer) and terminal status. + fn structure_hash(&self) -> u64 { + let mut h: u64 = if self.is_terminal { 1 } else { 0 }; + for (i, child) in self.children.iter().enumerate() { + if let Some(c) = child { + h = h.wrapping_mul(31).wrapping_add(i as u64); + h = h.wrapping_mul(31).wrapping_add(Rc::as_ptr(c) as u64); + } + } + h + } + + /// Check structural equality via Rc pointer identity. + fn structure_eq(&self, other: &Node) -> bool { + if self.is_terminal != other.is_terminal { + return false; + } + for i in 0..128 { + match (&self.children[i], &other.children[i]) { + (None, None) => {} + (Some(a), Some(b)) => { + if !Rc::ptr_eq(a, b) { + return false; + } + } + _ => return false, + } + } + true + } +} + +fn ascii_alphabetic_to_index(c: u8) -> u8 { + if c <= b'Z' { c - b'A' } else { c - b'a' + 26 } +} + +struct UncheckedNode { + parent: NodeRc, + character: u8, +} + +struct DafsaBuilder { + root: NodeRc, + minimized_nodes: HashMap>, + unchecked_nodes: Vec, + previous_word: String, +} + +impl DafsaBuilder { + fn new() -> Self { + DafsaBuilder { + root: Node::new_rc(), + minimized_nodes: HashMap::new(), + unchecked_nodes: Vec::new(), + previous_word: String::new(), + } + } + + fn insert(&mut self, word: &str) { + assert!( + word > self.previous_word.as_str(), + "Words must be inserted in sorted order: '{word}' <= '{}'", + self.previous_word + ); + + let common_prefix_len = word + .bytes() + .zip(self.previous_word.bytes()) + .take_while(|(a, b)| a == b) + .count(); + + self.minimize(common_prefix_len); + + let node: NodeRc = if self.unchecked_nodes.is_empty() { + Rc::clone(&self.root) + } else { + let last = &self.unchecked_nodes[self.unchecked_nodes.len() - 1]; + let parent = last.parent.borrow(); + Rc::clone(parent.children[last.character as usize].as_ref().unwrap()) + }; + + let remaining = &word[common_prefix_len..]; + let mut current = node; + for c in remaining.bytes() { + let new_child = Node::new_rc(); + { + let mut current_ref = current.borrow_mut(); + assert!(current_ref.children[c as usize].is_none()); + current_ref.children[c as usize] = Some(Rc::clone(&new_child)); + } + self.unchecked_nodes.push(UncheckedNode { + parent: Rc::clone(¤t), + character: c, + }); + current = new_child; + } + current.borrow_mut().is_terminal = true; + + self.previous_word = word.to_string(); + } + + fn minimize(&mut self, down_to: usize) { + while self.unchecked_nodes.len() > down_to { + let unchecked = self.unchecked_nodes.pop().unwrap(); + let parent = &unchecked.parent; + let child = { + let parent_ref = parent.borrow(); + Rc::clone(parent_ref.children[unchecked.character as usize].as_ref().unwrap()) + }; + + let hash = child.borrow().structure_hash(); + let mut found_replacement: Option = Option::None; + + if let Some(bucket) = self.minimized_nodes.get(&hash) { + for existing in bucket { + if child.borrow().structure_eq(&existing.borrow()) { + found_replacement = Some(Rc::clone(existing)); + break; + } + } + } + + if let Some(replacement) = found_replacement { + parent.borrow_mut().children[unchecked.character as usize] = Some(replacement); + } else { + self.minimized_nodes.entry(hash).or_default().push(Rc::clone(&child)); + } + } + } + + fn calc_numbers(&mut self) { + self.root.borrow_mut().calc_numbers(); + } + + fn get_unique_index(&self, word: &str) -> Option { + let mut index: usize = 0; + let mut current = Rc::clone(&self.root); + + for c in word.bytes() { + let next = { + let node = current.borrow(); + let child = node.children[c as usize].as_ref()?; + for sibling_c in 0u8..128 { + if let Some(sibling) = &node.children[sibling_c as usize] + && sibling_c < c + { + index += sibling.borrow().number as usize; + } + } + Rc::clone(child) + }; + if next.borrow().is_terminal { + index += 1; + } + current = next; + } + + Some(index) + } +} + +fn queue_children( + node: &NodeRc, + queue: &mut Vec, + child_indexes: &mut HashMap<*const RefCell, u16>, + first_available_index: u16, +) -> u16 { + let mut current = first_available_index; + let node_ref = node.borrow(); + for c in 0..128u8 { + if let Some(child) = &node_ref.children[c as usize] { + let key = Rc::as_ptr(child); + if let std::collections::hash_map::Entry::Vacant(entry) = child_indexes.entry(key) { + let num_children = child.borrow().num_direct_children(); + if num_children > 0 { + entry.insert(current); + current += num_children as u16; + } + queue.push(Rc::clone(child)); + } + } + } + current +} + +fn write_children_data( + node: &NodeRc, + node_data: &mut Vec, + queue: &mut Vec, + child_indexes: &mut HashMap<*const RefCell, u16>, + first_available_index: u16, +) -> u16 { + let mut current = first_available_index; + let mut unique_index_tally: u8 = 0; + let node_ref = node.borrow(); + for c in 0..128u8 { + if let Some(child) = &node_ref.children[c as usize] { + let key = Rc::as_ptr(child); + let child_ref = child.borrow(); + let num_children = child_ref.num_direct_children(); + + if let std::collections::hash_map::Entry::Vacant(entry) = child_indexes.entry(key) { + if num_children > 0 { + entry.insert(current); + current += num_children as u16; + } + queue.push(Rc::clone(child)); + } + + node_data.push(NodeData { + character: c, + number: unique_index_tally, + end_of_word: child_ref.is_terminal, + child_index: child_indexes.get(&key).copied().unwrap_or(0), + children_len: num_children, + }); + + unique_index_tally = unique_index_tally.wrapping_add(child_ref.number as u8); + } + } + current +} diff --git a/Libraries/LibWeb/HTML/Parser/Rust/cbindgen.toml b/Libraries/LibWeb/HTML/Parser/Rust/cbindgen.toml new file mode 100644 index 0000000000..6b2b44e957 --- /dev/null +++ b/Libraries/LibWeb/HTML/Parser/Rust/cbindgen.toml @@ -0,0 +1,19 @@ +language = "C++" +header = """/* + * Copyright (c) 2026-present, the Ladybird developers. + * + * SPDX-License-Identifier: BSD-2-Clause + */""" +pragma_once = true +include_version = true +line_length = 120 +tab_width = 4 +no_includes = true +sys_includes = ["stdint.h", "stddef.h"] +usize_is_size_t = true + +[parse] +parse_deps = false + +[parse.expand] +all_features = false diff --git a/Libraries/LibWeb/HTML/Parser/Rust/src/entities.rs b/Libraries/LibWeb/HTML/Parser/Rust/src/entities.rs new file mode 100644 index 0000000000..591e1369bd --- /dev/null +++ b/Libraries/LibWeb/HTML/Parser/Rust/src/entities.rs @@ -0,0 +1,151 @@ +/* + * Copyright (c) 2026-present, the Ladybird developers. + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +//! Named character reference matching using a DAFSA (Deterministic Acyclic Finite +//! State Automaton) with minimal perfect hashing. The DAFSA data is generated at +//! build time by build.rs from Entities.json. + +include!(concat!(env!("OUT_DIR"), "/named_character_references.rs")); + +fn ascii_alphabetic_to_index(c: u8) -> u8 { + if c <= b'Z' { c - b'A' } else { c - b'a' + 26 } +} + +#[derive(Clone, Copy)] +enum SearchState { + Init, + FirstToSecondLayer { mask: u64, offset: u16 }, + DafsaChildren { start_index: u16, len: u8 }, +} + +/// Incremental matcher for named character references using the DAFSA. +/// +/// Feed characters one at a time via `try_consume_code_point()`. After each +/// rejected character (returns false), call `code_points()` to get the longest +/// match found so far, and `overconsumed_code_points()` to know how many +/// characters were consumed past the longest match. +pub struct NamedCharacterReferenceMatcher { + search_state: SearchState, + last_matched_unique_index: u16, + pending_unique_index: u16, + overconsumed_code_points: u8, + ends_with_semicolon: bool, +} + +impl NamedCharacterReferenceMatcher { + pub fn new() -> Self { + Self { + search_state: SearchState::Init, + last_matched_unique_index: 0, + pending_unique_index: 0, + overconsumed_code_points: 0, + ends_with_semicolon: false, + } + } + + /// Feed one code point to the matcher. Returns true if the character was + /// consumed (there may still be longer matches), false if no further + /// matches are possible. + pub fn try_consume_code_point(&mut self, c: u32) -> bool { + if c > 0x7F { + return false; + } + self.try_consume_ascii_char(c as u8) + } + + fn try_consume_ascii_char(&mut self, c: u8) -> bool { + match self.search_state { + SearchState::Init => { + if !c.is_ascii_alphabetic() { + return false; + } + let index = ascii_alphabetic_to_index(c) as usize; + self.search_state = SearchState::FirstToSecondLayer { + mask: FIRST_TO_SECOND_LAYER[index].0, + offset: FIRST_TO_SECOND_LAYER[index].1, + }; + self.pending_unique_index = FIRST_LAYER[index]; + self.overconsumed_code_points += 1; + true + } + SearchState::FirstToSecondLayer { mask, offset } => { + if !c.is_ascii_alphabetic() { + return false; + } + let bit_index = ascii_alphabetic_to_index(c); + if ((1u64 << bit_index) & mask) == 0 { + return false; + } + + // Count set bits below bit_index to find the node's position. + let lower_mask = (1u64 << bit_index) - 1; + let char_index = (mask & lower_mask).count_ones() as u16; + let node = &SECOND_LAYER[(offset + char_index) as usize]; + + self.pending_unique_index += node.number as u16; + self.overconsumed_code_points += 1; + if node.end_of_word { + self.pending_unique_index += 1; + self.last_matched_unique_index = self.pending_unique_index; + self.ends_with_semicolon = c == b';'; + self.overconsumed_code_points = 0; + } + self.search_state = SearchState::DafsaChildren { + start_index: node.child_index, + len: node.children_len, + }; + true + } + SearchState::DafsaChildren { start_index, len } => { + for i in 0..len as u16 { + let node = &DAFSA_NODES[(start_index + i) as usize]; + if node.character == c { + self.pending_unique_index += node.number as u16; + self.overconsumed_code_points += 1; + if node.end_of_word { + self.pending_unique_index += 1; + self.last_matched_unique_index = self.pending_unique_index; + self.ends_with_semicolon = c == b';'; + self.overconsumed_code_points = 0; + } + self.search_state = SearchState::DafsaChildren { + start_index: node.child_index, + len: node.children_len, + }; + return true; + } + } + false + } + } + } + + /// Returns the codepoints for the longest match found, or None if no match. + /// Returns (first_codepoint, second_codepoint). second is 0 if there is no second codepoint. + pub fn code_points(&self) -> Option<(u32, u32)> { + if self.last_matched_unique_index == 0 { + return None; + } + let entry = &CODEPOINTS_LOOKUP[(self.last_matched_unique_index - 1) as usize]; + Some((entry.0, entry.1.value())) + } + + /// Number of characters consumed past the longest match. + pub fn overconsumed_code_points(&self) -> u8 { + self.overconsumed_code_points + } + + /// Whether the longest match ended with a semicolon. + pub fn last_match_ends_with_semicolon(&self) -> bool { + self.ends_with_semicolon + } +} + +impl Default for NamedCharacterReferenceMatcher { + fn default() -> Self { + Self::new() + } +} diff --git a/Libraries/LibWeb/HTML/Parser/Rust/src/interned_names.rs b/Libraries/LibWeb/HTML/Parser/Rust/src/interned_names.rs new file mode 100644 index 0000000000..ed7af9fb51 --- /dev/null +++ b/Libraries/LibWeb/HTML/Parser/Rust/src/interned_names.rs @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2026-present, the Ladybird developers. + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +//! Lookup tables for common HTML tag and attribute names. +//! +//! The tables are generated at build time from LibWeb's TagNames.h and +//! AttributeNames.h headers so they stay in sync with the C++ side. Each +//! name gets a 1-based u16 id; id 0 is reserved for "not interned" so the +//! FFI layer can carry either an id or raw bytes in the same slot. +//! +//! The lookup functions themselves are generated as a `match` on byte +//! length and exact byte sequence, which rustc compiles to a jump table +//! plus direct memcmp per length bucket. This is dramatically faster than +//! a HashMap lookup with the default cryptographic hasher. + +include!(concat!(env!("OUT_DIR"), "/interned_names_generated.rs")); + +/// Look up a tag name. Returns 0 if not interned. +#[inline] +pub fn lookup_tag_name(bytes: &[u8]) -> u16 { + lookup_tag_name_generated(bytes) +} + +/// Look up an attribute name. Returns 0 if not interned. +#[inline] +pub fn lookup_attr_name(bytes: &[u8]) -> u16 { + lookup_attr_name_generated(bytes) +} + +/// Number of interned tag names (for C++ table sizing at startup). +#[inline] +pub fn tag_name_count() -> usize { + INTERNED_TAG_NAMES.len() +} + +/// Number of interned attribute names. +#[inline] +pub fn attr_name_count() -> usize { + INTERNED_ATTR_NAMES.len() +} + +/// Fetch a tag name by id (1-based). Returns None for id 0 or out of range. +#[inline] +pub fn tag_name_by_id(id: u16) -> Option<&'static [u8]> { + if id == 0 { + return None; + } + INTERNED_TAG_NAMES.get((id - 1) as usize).copied() +} + +/// Fetch an attribute name by id (1-based). Returns None for id 0 or out of range. +#[inline] +pub fn attr_name_by_id(id: u16) -> Option<&'static [u8]> { + if id == 0 { + return None; + } + INTERNED_ATTR_NAMES.get((id - 1) as usize).copied() +} diff --git a/Libraries/LibWeb/HTML/Parser/Rust/src/lib.rs b/Libraries/LibWeb/HTML/Parser/Rust/src/lib.rs new file mode 100644 index 0000000000..262f7efc17 --- /dev/null +++ b/Libraries/LibWeb/HTML/Parser/Rust/src/lib.rs @@ -0,0 +1,702 @@ +/* + * Copyright (c) 2026-present, the Ladybird developers. + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +pub mod entities; +pub mod interned_names; +pub mod token; +pub mod tokenizer; + +use std::ptr; +use token::{Attribute, Position, TokenPayload, TokenType}; +use tokenizer::{HtmlTokenizer, State}; + +/// Opaque handle for the Rust tokenizer, passed across the FFI boundary. +pub struct RustFfiTokenizerHandle { + tokenizer: HtmlTokenizer, + /// Temporary storage for the last token's string data, kept alive + /// so that pointers in RustFfiToken remain valid until the next call. + last_tag_name: Vec, + last_comment: Vec, + last_doctype_name: Vec, + last_public_id: Vec, + last_system_id: Vec, + last_attributes: Vec, + last_attr_names: Vec>, + last_attr_values: Vec>, + last_unparsed_input: Vec, +} + +/// C-compatible token representation. +/// +/// Pointer fields (`tag_name_ptr`, `comment_ptr`, doctype/attribute pointers) +/// borrow into buffers owned by the `RustFfiTokenizerHandle` that produced this +/// token. They remain valid only until the next call into the tokenizer for +/// that handle (any of `next_token`, `insert_input`, `destroy`, etc.). +/// Callers must consume the data before making the next call. +#[repr(C)] +pub struct RustFfiToken { + pub token_type: u8, + pub code_point: u32, + pub self_closing: bool, + + /// If nonzero, an interned tag-name id (1-based index into + /// `interned_names::INTERNED_TAG_NAMES`). When set, `tag_name_ptr` / + /// `tag_name_len` are unused and the C++ side uses its parallel + /// FlyString table directly. + pub tag_name_id: u16, + pub tag_name_ptr: *const u8, + pub tag_name_len: usize, + + pub comment_ptr: *const u8, + pub comment_len: usize, + + pub doctype_name_ptr: *const u8, + pub doctype_name_len: usize, + pub public_id_ptr: *const u8, + pub public_id_len: usize, + pub system_id_ptr: *const u8, + pub system_id_len: usize, + pub force_quirks: bool, + pub missing_name: bool, + pub missing_public_id: bool, + pub missing_system_id: bool, + + pub attributes_ptr: *const RustFfiAttribute, + pub attributes_len: usize, + + pub start_line: u64, + pub start_column: u64, + pub end_line: u64, + pub end_column: u64, +} + +/// C-compatible attribute representation. +#[repr(C)] +pub struct RustFfiAttribute { + /// If nonzero, an interned attribute-name id (1-based index into + /// `interned_names::INTERNED_ATTR_NAMES`). When set, `name_ptr` / + /// `name_len` are unused. + pub name_id: u16, + pub name_ptr: *const u8, + pub name_len: usize, + pub value_ptr: *const u8, + pub value_len: usize, + pub name_start_line: u64, + pub name_start_column: u64, + pub name_end_line: u64, + pub name_end_column: u64, + pub value_start_line: u64, + pub value_start_column: u64, + pub value_end_line: u64, + pub value_end_column: u64, +} + +impl Default for RustFfiToken { + fn default() -> Self { + RustFfiToken { + token_type: TokenType::Invalid as u8, + code_point: 0, + self_closing: false, + tag_name_id: 0, + tag_name_ptr: ptr::null(), + tag_name_len: 0, + comment_ptr: ptr::null(), + comment_len: 0, + doctype_name_ptr: ptr::null(), + doctype_name_len: 0, + public_id_ptr: ptr::null(), + public_id_len: 0, + system_id_ptr: ptr::null(), + system_id_len: 0, + force_quirks: false, + missing_name: true, + missing_public_id: true, + missing_system_id: true, + attributes_ptr: ptr::null(), + attributes_len: 0, + start_line: 0, + start_column: 0, + end_line: 0, + end_column: 0, + } + } +} + +fn position_to_ffi(pos: &Position) -> (u64, u64) { + (pos.line, pos.column) +} + +/// Create a new Rust HTML tokenizer from UTF-32 code points. +/// +/// # Safety +/// `input` must point to `len` valid u32 values. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn rust_html_tokenizer_create(input: *const u32, len: usize) -> *mut RustFfiTokenizerHandle { + let code_points = if input.is_null() || len == 0 { + Vec::new() + } else { + unsafe { std::slice::from_raw_parts(input, len) }.to_vec() + }; + + make_handle(HtmlTokenizer::new(code_points)) +} + +/// Create a new Rust HTML tokenizer directly from a UTF-8 byte buffer. +/// Rust decodes the bytes to code points internally, skipping the C++ +/// side's 4x-expanded Vec copy. +/// +/// # Safety +/// `bytes` must point to `len` valid UTF-8 bytes. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn rust_html_tokenizer_create_from_utf8( + bytes: *const u8, + len: usize, +) -> *mut RustFfiTokenizerHandle { + let code_points = if bytes.is_null() || len == 0 { + Vec::new() + } else { + let slice = unsafe { std::slice::from_raw_parts(bytes, len) }; + decode_utf8_to_u32(slice) + }; + + make_handle(HtmlTokenizer::new(code_points)) +} + +/// Expand a UTF-8 byte slice into a `Vec` of code points. For the +/// dominant ASCII case we use a tight loop with a single unchecked +/// write per byte and only fall back to `str::chars()` decoding when +/// we see a continuation byte. The caller is responsible for ensuring +/// the input is valid UTF-8. +fn decode_utf8_to_u32(bytes: &[u8]) -> Vec { + let mut out: Vec = Vec::with_capacity(bytes.len()); + // SAFETY: we reserved `bytes.len()` slots and will only write up to + // that many u32s (one per input byte; multi-byte sequences produce + // fewer u32s than bytes, so we stay within bounds). + let out_ptr = out.as_mut_ptr(); + let mut write_idx: usize = 0; + let mut i: usize = 0; + let n = bytes.len(); + while i < n { + let b = bytes[i]; + if b < 0x80 { + unsafe { std::ptr::write(out_ptr.add(write_idx), b as u32) }; + write_idx += 1; + i += 1; + } else { + // Slow path: decode one code point via str::chars. + // SAFETY: the input is valid UTF-8 by precondition. + let tail = unsafe { std::str::from_utf8_unchecked(&bytes[i..]) }; + if let Some(ch) = tail.chars().next() { + unsafe { std::ptr::write(out_ptr.add(write_idx), ch as u32) }; + write_idx += 1; + i += ch.len_utf8(); + } else { + break; + } + } + } + // SAFETY: we wrote exactly `write_idx` elements, all in-bounds. + unsafe { out.set_len(write_idx) }; + out +} + +fn make_handle(tokenizer: HtmlTokenizer) -> *mut RustFfiTokenizerHandle { + let handle = Box::new(RustFfiTokenizerHandle { + tokenizer, + last_tag_name: Vec::new(), + last_comment: Vec::new(), + last_doctype_name: Vec::new(), + last_public_id: Vec::new(), + last_system_id: Vec::new(), + last_attributes: Vec::new(), + last_attr_names: Vec::new(), + last_attr_values: Vec::new(), + last_unparsed_input: Vec::new(), + }); + + Box::into_raw(handle) +} + +/// Get the next token from the tokenizer. +/// Returns true if a token was produced, false if no more tokens. +/// +/// # Safety +/// `handle` must be a valid pointer from `rust_html_tokenizer_create`. +/// `out` must be a valid pointer to an RustFfiToken. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn rust_html_tokenizer_next_token( + handle: *mut RustFfiTokenizerHandle, + out: *mut RustFfiToken, + stop_at_insertion_point: bool, + cdata_allowed: bool, +) -> bool { + if handle.is_null() || out.is_null() { + return false; + } + let handle = unsafe { &mut *handle }; + let out = unsafe { &mut *out }; + + // Fast-path text character in the Data state. For ASCII text runs + // this skips the whole state-machine function call plus Token + // construction/drop. Most real HTML is dominated by such runs. + if !stop_at_insertion_point && let Some((cp, pos)) = handle.tokenizer.try_fast_data_char() { + out.token_type = TokenType::Character as u8; + out.code_point = cp; + out.start_line = pos.line; + out.start_column = pos.column; + out.end_line = pos.line; + out.end_column = pos.column; + return true; + } + + next_token_slow(handle, out, stop_at_insertion_point, cdata_allowed) +} + +/// The state-machine and marshalling portion of the FFI token fetch, +/// pulled out of `rust_html_tokenizer_next_token` so that the fast +/// Data-state path in the outer function keeps a tiny stack frame +/// and a short prologue. Marked `#[inline(never)]` so the compiler +/// doesn't re-inline the whole thing and re-inflate the outer frame. +#[inline(never)] +fn next_token_slow( + handle: &mut RustFfiTokenizerHandle, + out: &mut RustFfiToken, + stop_at_insertion_point: bool, + cdata_allowed: bool, +) -> bool { + let token = match handle.tokenizer.next_token(stop_at_insertion_point, cdata_allowed) { + Some(t) => t, + None => return false, + }; + + // Fast path: Character and EOF tokens only need a few fields. + // Skip zeroing ~200 bytes of RustFfiToken for 95%+ of all tokens. + match token.token_type { + TokenType::Character | TokenType::EndOfFile => { + out.token_type = token.token_type as u8; + out.code_point = token.code_point; + out.start_line = token.start_position.line; + out.start_column = token.start_position.column; + out.end_line = token.end_position.line; + out.end_column = token.end_position.column; + return true; + } + _ => {} + } + + *out = RustFfiToken::default(); + out.token_type = token.token_type as u8; + + let (sl, sc) = position_to_ffi(&token.start_position); + let (el, ec) = position_to_ffi(&token.end_position); + out.start_line = sl; + out.start_column = sc; + out.end_line = el; + out.end_column = ec; + + // Store string data in handle so pointers stay valid. + match token.payload { + TokenPayload::Tag { + tag_name, + tag_name_id, + self_closing, + attributes, + } => { + out.self_closing = self_closing; + // Tokenizer already resolved intern ids, so we trust tag_name_id. + out.tag_name_id = tag_name_id; + if tag_name_id == 0 { + handle.last_tag_name = tag_name.into_bytes(); + out.tag_name_ptr = handle.last_tag_name.as_ptr(); + out.tag_name_len = handle.last_tag_name.len(); + } + + // Convert attributes. Move owned Strings out of each Attribute + // instead of cloning, then point the FfiAttribute at the stable + // heap bytes that now live in handle.last_attr_{names,values} + // (unless the name was interned, in which case skip the copy). + handle.last_attr_names.clear(); + handle.last_attr_values.clear(); + handle.last_attributes.clear(); + handle.last_attr_names.reserve(attributes.len()); + handle.last_attr_values.reserve(attributes.len()); + handle.last_attributes.reserve(attributes.len()); + for attr in attributes { + let Attribute { + local_name, + local_name_id, + value, + name_start_position, + name_end_position, + value_start_position, + value_end_position, + } = attr; + if local_name_id == 0 { + handle.last_attr_names.push(local_name.into_bytes()); + } else { + // Keep the slot aligned with last_attr_values so index math stays valid. + handle.last_attr_names.push(Vec::new()); + } + handle.last_attr_values.push(value.into_bytes()); + let last_idx = handle.last_attr_names.len() - 1; + let name_bytes = &handle.last_attr_names[last_idx]; + let value_bytes = &handle.last_attr_values[last_idx]; + handle.last_attributes.push(RustFfiAttribute { + name_id: local_name_id, + name_ptr: if local_name_id == 0 { + name_bytes.as_ptr() + } else { + ptr::null() + }, + name_len: if local_name_id == 0 { name_bytes.len() } else { 0 }, + value_ptr: value_bytes.as_ptr(), + value_len: value_bytes.len(), + name_start_line: name_start_position.line, + name_start_column: name_start_position.column, + name_end_line: name_end_position.line, + name_end_column: name_end_position.column, + value_start_line: value_start_position.line, + value_start_column: value_start_position.column, + value_end_line: value_end_position.line, + value_end_column: value_end_position.column, + }); + } + out.attributes_ptr = handle.last_attributes.as_ptr(); + out.attributes_len = handle.last_attributes.len(); + } + TokenPayload::Comment(data) => { + handle.last_comment = data.into_bytes(); + out.comment_ptr = handle.last_comment.as_ptr(); + out.comment_len = handle.last_comment.len(); + } + TokenPayload::Doctype(doctype) => { + handle.last_doctype_name = doctype.name.into_bytes(); + handle.last_public_id = doctype.public_identifier.into_bytes(); + handle.last_system_id = doctype.system_identifier.into_bytes(); + out.doctype_name_ptr = handle.last_doctype_name.as_ptr(); + out.doctype_name_len = handle.last_doctype_name.len(); + out.public_id_ptr = handle.last_public_id.as_ptr(); + out.public_id_len = handle.last_public_id.len(); + out.system_id_ptr = handle.last_system_id.as_ptr(); + out.system_id_len = handle.last_system_id.len(); + out.force_quirks = doctype.force_quirks; + out.missing_name = doctype.missing_name; + out.missing_public_id = doctype.missing_public_identifier; + out.missing_system_id = doctype.missing_system_identifier; + } + TokenPayload::None => {} + } + + true +} + +/// Switch the tokenizer to a new state. +/// +/// # Safety +/// `handle` must be a valid pointer from `rust_html_tokenizer_create`. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn rust_html_tokenizer_switch_state(handle: *mut RustFfiTokenizerHandle, state: u8) { + if handle.is_null() { + return; + } + let handle = unsafe { &mut *handle }; + // The state values must match the State enum order. Bound-checked against + // the last known variant so an out-of-range value is rejected instead of + // producing UB via transmute to an invalid discriminant. + const STATE_COUNT: u8 = State::NumericCharacterReferenceEnd as u8 + 1; + if state >= STATE_COUNT { + return; + } + let state: State = unsafe { std::mem::transmute(state) }; + handle.tokenizer.switch_to(state); +} + +/// Insert input (as UTF-32 code points) at the current insertion point. +/// +/// # Safety +/// `handle` must be a valid pointer. `input` must point to `len` valid u32 values. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn rust_html_tokenizer_insert_input( + handle: *mut RustFfiTokenizerHandle, + input: *const u32, + len: usize, +) { + if handle.is_null() { + return; + } + let handle = unsafe { &mut *handle }; + let code_points = if input.is_null() || len == 0 { + &[] + } else { + unsafe { std::slice::from_raw_parts(input, len) } + }; + handle.tokenizer.insert_input_at_insertion_point(code_points); +} + +/// Append input (as UTF-32 code points) to the tokenizer input stream. +/// +/// # Safety +/// `handle` must be a valid pointer. `input` must point to `len` valid u32 values. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn rust_html_tokenizer_append_input( + handle: *mut RustFfiTokenizerHandle, + input: *const u32, + len: usize, +) { + if handle.is_null() { + return; + } + let handle = unsafe { &mut *handle }; + let code_points = if input.is_null() || len == 0 { + &[] + } else { + unsafe { std::slice::from_raw_parts(input, len) } + }; + handle.tokenizer.append_input(code_points); +} + +/// Get the tokenizer input that has not been consumed yet. +/// +/// # Safety +/// `handle`, `out_ptr`, and `out_len` must be valid pointers. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn rust_html_tokenizer_unparsed_input( + handle: *mut RustFfiTokenizerHandle, + out_ptr: *mut *const u8, + out_len: *mut usize, +) { + if handle.is_null() || out_ptr.is_null() || out_len.is_null() { + return; + } + let handle = unsafe { &mut *handle }; + handle.last_unparsed_input = handle.tokenizer.unparsed_input().into_bytes(); + unsafe { + *out_ptr = handle.last_unparsed_input.as_ptr(); + *out_len = handle.last_unparsed_input.len(); + } +} + +/// Compact already-tokenized input after the parser has consumed a chunk. +/// +/// # Safety +/// `handle` must be a valid pointer. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn rust_html_tokenizer_parser_did_run(handle: *mut RustFfiTokenizerHandle) { + if handle.is_null() { + return; + } + let handle = unsafe { &mut *handle }; + handle.tokenizer.parser_did_run(); +} + +/// # Safety +/// `handle` must be a valid pointer. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn rust_html_tokenizer_store_insertion_point(handle: *mut RustFfiTokenizerHandle) { + if handle.is_null() { + return; + } + let handle = unsafe { &mut *handle }; + handle.tokenizer.store_insertion_point(); +} + +/// # Safety +/// `handle` must be a valid pointer. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn rust_html_tokenizer_restore_insertion_point(handle: *mut RustFfiTokenizerHandle) { + if handle.is_null() { + return; + } + let handle = unsafe { &mut *handle }; + handle.tokenizer.restore_insertion_point(); +} + +/// # Safety +/// `handle` must be a valid pointer. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn rust_html_tokenizer_update_insertion_point(handle: *mut RustFfiTokenizerHandle) { + if handle.is_null() { + return; + } + let handle = unsafe { &mut *handle }; + handle.tokenizer.update_insertion_point(); +} + +/// # Safety +/// `handle` must be a valid pointer. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn rust_html_tokenizer_undefine_insertion_point(handle: *mut RustFfiTokenizerHandle) { + if handle.is_null() { + return; + } + let handle = unsafe { &mut *handle }; + handle.tokenizer.undefine_insertion_point(); +} + +/// # Safety +/// `handle` must be a valid pointer. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn rust_html_tokenizer_is_insertion_point_defined(handle: *mut RustFfiTokenizerHandle) -> bool { + if handle.is_null() { + return false; + } + let handle = unsafe { &mut *handle }; + handle.tokenizer.is_insertion_point_defined() +} + +/// # Safety +/// `handle` must be a valid pointer. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn rust_html_tokenizer_is_insertion_point_reached(handle: *mut RustFfiTokenizerHandle) -> bool { + if handle.is_null() { + return false; + } + let handle = unsafe { &mut *handle }; + handle.tokenizer.is_insertion_point_reached() +} + +/// # Safety +/// `handle` must be a valid pointer. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn rust_html_tokenizer_set_blocked(handle: *mut RustFfiTokenizerHandle, blocked: bool) { + if handle.is_null() { + return; + } + let handle = unsafe { &mut *handle }; + handle.tokenizer.set_blocked(blocked); +} + +/// # Safety +/// `handle` must be a valid pointer. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn rust_html_tokenizer_is_blocked(handle: *mut RustFfiTokenizerHandle) -> bool { + if handle.is_null() { + return false; + } + let handle = unsafe { &mut *handle }; + handle.tokenizer.is_blocked() +} + +/// # Safety +/// `handle` must be a valid pointer. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn rust_html_tokenizer_set_input_stream_closed( + handle: *mut RustFfiTokenizerHandle, + closed: bool, +) { + if handle.is_null() { + return; + } + let handle = unsafe { &mut *handle }; + handle.tokenizer.set_input_stream_closed(closed); +} + +/// # Safety +/// `handle` must be a valid pointer. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn rust_html_tokenizer_insert_eof(handle: *mut RustFfiTokenizerHandle) { + if handle.is_null() { + return; + } + let handle = unsafe { &mut *handle }; + handle.tokenizer.insert_eof(); +} + +/// # Safety +/// `handle` must be a valid pointer. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn rust_html_tokenizer_is_eof_inserted(handle: *mut RustFfiTokenizerHandle) -> bool { + if handle.is_null() { + return false; + } + let handle = unsafe { &mut *handle }; + handle.tokenizer.is_eof_inserted() +} + +/// # Safety +/// `handle` must be a valid pointer. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn rust_html_tokenizer_abort(handle: *mut RustFfiTokenizerHandle) { + if handle.is_null() { + return; + } + let handle = unsafe { &mut *handle }; + handle.tokenizer.abort(); +} + +/// Destroy a Rust HTML tokenizer. +/// +/// # Safety +/// `handle` must be a valid pointer from `rust_html_tokenizer_create`, +/// and must not be used after this call. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn rust_html_tokenizer_destroy(handle: *mut RustFfiTokenizerHandle) { + if !handle.is_null() { + drop(unsafe { Box::from_raw(handle) }); + } +} + +// -- Interned name table enumeration -------------------------------------- +// +// The C++ side builds a parallel FlyString array at static-init time by +// enumerating the Rust-owned list once. Ids are 1-based; id 0 is reserved +// for "not interned" in the per-token FFI struct. + +/// Number of interned HTML tag names known to the Rust tokenizer. +#[unsafe(no_mangle)] +pub extern "C" fn rust_html_tokenizer_interned_tag_name_count() -> usize { + interned_names::tag_name_count() +} + +/// Number of interned HTML attribute names known to the Rust tokenizer. +#[unsafe(no_mangle)] +pub extern "C" fn rust_html_tokenizer_interned_attr_name_count() -> usize { + interned_names::attr_name_count() +} + +/// Write the bytes and length of the interned tag name with the given +/// 1-based id to the caller-provided out parameters. On unknown ids the +/// out parameters are set to (null, 0). +/// +/// # Safety +/// `out_ptr` and `out_len` must be valid pointers. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn rust_html_tokenizer_interned_tag_name(id: u16, out_ptr: *mut *const u8, out_len: *mut usize) { + if out_ptr.is_null() || out_len.is_null() { + return; + } + match interned_names::tag_name_by_id(id) { + Some(bytes) => unsafe { + *out_ptr = bytes.as_ptr(); + *out_len = bytes.len(); + }, + None => unsafe { + *out_ptr = ptr::null(); + *out_len = 0; + }, + } +} + +/// Same as `rust_html_tokenizer_interned_tag_name` for attribute names. +/// +/// # Safety +/// `out_ptr` and `out_len` must be valid pointers. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn rust_html_tokenizer_interned_attr_name(id: u16, out_ptr: *mut *const u8, out_len: *mut usize) { + if out_ptr.is_null() || out_len.is_null() { + return; + } + match interned_names::attr_name_by_id(id) { + Some(bytes) => unsafe { + *out_ptr = bytes.as_ptr(); + *out_len = bytes.len(); + }, + None => unsafe { + *out_ptr = ptr::null(); + *out_len = 0; + }, + } +} diff --git a/Libraries/LibWeb/HTML/Parser/Rust/src/token.rs b/Libraries/LibWeb/HTML/Parser/Rust/src/token.rs new file mode 100644 index 0000000000..592aaf5257 --- /dev/null +++ b/Libraries/LibWeb/HTML/Parser/Rust/src/token.rs @@ -0,0 +1,202 @@ +/* + * Copyright (c) 2026-present, the Ladybird developers. + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +/// Source position in the input. +#[derive(Clone, Copy, Debug, Default)] +pub struct Position { + pub line: u64, + pub column: u64, +} + +/// A single attribute on a start or end tag token. +/// +/// If `local_name_id` is non-zero it is an index into +/// `interned_names::INTERNED_ATTR_NAMES` and `local_name` is unused. +/// Otherwise `local_name` holds the owned bytes. +#[derive(Clone, Debug, Default)] +pub struct Attribute { + pub local_name: String, + pub local_name_id: u16, + pub value: String, + pub name_start_position: Position, + pub name_end_position: Position, + pub value_start_position: Position, + pub value_end_position: Position, +} + +/// Data specific to DOCTYPE tokens. +#[derive(Clone, Debug, Default)] +pub struct DoctypeData { + pub name: String, + pub public_identifier: String, + pub system_identifier: String, + pub missing_name: bool, + pub missing_public_identifier: bool, + pub missing_system_identifier: bool, + pub force_quirks: bool, +} + +/// The type of an HTML token. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +#[repr(u8)] +pub enum TokenType { + #[default] + Invalid = 0, + Doctype = 1, + StartTag = 2, + EndTag = 3, + Comment = 4, + Character = 5, + EndOfFile = 6, +} + +/// Type-specific data for an HTML token. +/// +/// If `tag_name_id` is non-zero it is an index into +/// `interned_names::INTERNED_TAG_NAMES` and `tag_name` is unused. +/// Otherwise `tag_name` holds the owned bytes. +#[derive(Clone, Debug, Default)] +pub enum TokenPayload { + #[default] + None, + Tag { + tag_name: String, + tag_name_id: u16, + self_closing: bool, + attributes: Vec, + }, + Comment(String), + Doctype(Box), +} + +/// An HTML token produced by the tokenizer. +#[derive(Clone, Debug, Default)] +pub struct Token { + pub token_type: TokenType, + pub code_point: u32, + pub payload: TokenPayload, + pub start_position: Position, + pub end_position: Position, +} + +impl Token { + pub fn new_character(code_point: u32) -> Self { + Token { + token_type: TokenType::Character, + code_point, + ..Default::default() + } + } + + pub fn new_eof() -> Self { + Token { + token_type: TokenType::EndOfFile, + ..Default::default() + } + } + + /// Return the tag name as a &str. For interned names this resolves + /// through the interned name table; for un-interned names it returns + /// the owned String's contents. + #[inline(always)] + pub fn tag_name(&self) -> &str { + match &self.payload { + TokenPayload::Tag { + tag_name, tag_name_id, .. + } => { + if *tag_name_id != 0 { + // SAFETY: interned names are compile-time ASCII byte + // literals in interned_names_generated.rs, so they are + // always valid UTF-8. + match crate::interned_names::tag_name_by_id(*tag_name_id) { + Some(bytes) => unsafe { std::str::from_utf8_unchecked(bytes) }, + None => "", + } + } else { + tag_name + } + } + _ => "", + } + } + + #[inline(always)] + pub fn tag_name_mut(&mut self) -> &mut String { + match &mut self.payload { + TokenPayload::Tag { + tag_name, tag_name_id, .. + } => { + *tag_name_id = 0; + tag_name + } + _ => panic!("tag_name_mut called on non-tag token"), + } + } + + /// Set the tag name to an interned id, clearing any previously stored + /// owned name. + #[inline(always)] + pub fn set_tag_name_id(&mut self, id: u16) { + match &mut self.payload { + TokenPayload::Tag { + tag_name, tag_name_id, .. + } => { + tag_name.clear(); + *tag_name_id = id; + } + _ => panic!("set_tag_name_id called on non-tag token"), + } + } + + /// Returns the interned tag-name id, or 0 if the name is un-interned. + #[inline(always)] + pub fn tag_name_id(&self) -> u16 { + match &self.payload { + TokenPayload::Tag { tag_name_id, .. } => *tag_name_id, + _ => 0, + } + } + + #[inline(always)] + pub fn set_self_closing(&mut self, value: bool) { + match &mut self.payload { + TokenPayload::Tag { self_closing, .. } => *self_closing = value, + _ => panic!("set_self_closing called on non-tag token"), + } + } + + #[inline(always)] + pub fn is_self_closing(&self) -> bool { + match &self.payload { + TokenPayload::Tag { self_closing, .. } => *self_closing, + _ => false, + } + } + + #[inline(always)] + pub fn attributes_mut(&mut self) -> &mut Vec { + match &mut self.payload { + TokenPayload::Tag { attributes, .. } => attributes, + _ => panic!("attributes_mut called on non-tag token"), + } + } + + #[inline(always)] + pub fn set_comment_data(&mut self, data: String) { + match &mut self.payload { + TokenPayload::Comment(s) => *s = data, + _ => panic!("set_comment_data called on non-comment token"), + } + } + + #[inline(always)] + pub fn doctype_data_mut(&mut self) -> &mut DoctypeData { + match &mut self.payload { + TokenPayload::Doctype(dd) => dd, + _ => panic!("doctype_data_mut called on non-doctype token"), + } + } +} diff --git a/Libraries/LibWeb/HTML/Parser/Rust/src/tokenizer.rs b/Libraries/LibWeb/HTML/Parser/Rust/src/tokenizer.rs new file mode 100644 index 0000000000..02de3347e1 --- /dev/null +++ b/Libraries/LibWeb/HTML/Parser/Rust/src/tokenizer.rs @@ -0,0 +1,3152 @@ +/* + * Copyright (c) 2026-present, the Ladybird developers. + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +use std::collections::VecDeque; + +use crate::entities::NamedCharacterReferenceMatcher; +use crate::token::{Attribute, DoctypeData, Position, Token, TokenPayload, TokenType}; + +/// Tokenizer states per the WHATWG HTML spec section 13.2.5. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[repr(u8)] +pub enum State { + Data = 0, + RCDATA, + RAWTEXT, + ScriptData, + PLAINTEXT, + TagOpen, + EndTagOpen, + TagName, + RCDATALessThanSign, + RCDATAEndTagOpen, + RCDATAEndTagName, + RAWTEXTLessThanSign, + RAWTEXTEndTagOpen, + RAWTEXTEndTagName, + ScriptDataLessThanSign, + ScriptDataEndTagOpen, + ScriptDataEndTagName, + ScriptDataEscapeStart, + ScriptDataEscapeStartDash, + ScriptDataEscaped, + ScriptDataEscapedDash, + ScriptDataEscapedDashDash, + ScriptDataEscapedLessThanSign, + ScriptDataEscapedEndTagOpen, + ScriptDataEscapedEndTagName, + ScriptDataDoubleEscapeStart, + ScriptDataDoubleEscaped, + ScriptDataDoubleEscapedDash, + ScriptDataDoubleEscapedDashDash, + ScriptDataDoubleEscapedLessThanSign, + ScriptDataDoubleEscapeEnd, + BeforeAttributeName, + AttributeName, + AfterAttributeName, + BeforeAttributeValue, + AttributeValueDoubleQuoted, + AttributeValueSingleQuoted, + AttributeValueUnquoted, + AfterAttributeValueQuoted, + SelfClosingStartTag, + BogusComment, + MarkupDeclarationOpen, + CommentStart, + CommentStartDash, + Comment, + CommentLessThanSign, + CommentLessThanSignBang, + CommentLessThanSignBangDash, + CommentLessThanSignBangDashDash, + CommentEndDash, + CommentEnd, + CommentEndBang, + DOCTYPE, + BeforeDOCTYPEName, + DOCTYPEName, + AfterDOCTYPEName, + AfterDOCTYPEPublicKeyword, + BeforeDOCTYPEPublicIdentifier, + DOCTYPEPublicIdentifierDoubleQuoted, + DOCTYPEPublicIdentifierSingleQuoted, + AfterDOCTYPEPublicIdentifier, + BetweenDOCTYPEPublicAndSystemIdentifiers, + AfterDOCTYPESystemKeyword, + BeforeDOCTYPESystemIdentifier, + DOCTYPESystemIdentifierDoubleQuoted, + DOCTYPESystemIdentifierSingleQuoted, + AfterDOCTYPESystemIdentifier, + BogusDOCTYPE, + CDATASection, + CDATASectionBracket, + CDATASectionEnd, + CharacterReference, + NamedCharacterReference, + AmbiguousAmpersand, + NumericCharacterReference, + HexadecimalCharacterReferenceStart, + DecimalCharacterReferenceStart, + HexadecimalCharacterReference, + DecimalCharacterReference, + NumericCharacterReferenceEnd, +} + +/// The HTML tokenizer state machine. +/// +/// Implements the WHATWG HTML tokenizer specification (section 13.2.5). +pub struct HtmlTokenizer { + pub state: State, + return_state: State, + pub input: Vec, + pub current_offset: usize, + prev_offset: usize, + current_token: Token, + current_builder: String, + pub queued_tokens: VecDeque, + temporary_buffer: Vec, + character_reference_code: u32, + last_emitted_start_tag_name: Option, + source_positions: Vec, + // Mirror of the most recent entry in source_positions, kept in sync + // with it on the slow path and updated directly on the fast + // character-emission path so we don't have to push into the Vec per + // character in Data-state text runs. + current_line: u64, + current_column: u64, + has_emitted_eof: bool, + aborted: bool, + insertion_point: Option, + old_insertion_points: Vec>, + explicit_eof_inserted: bool, + input_stream_closed: bool, + blocked: bool, + stop_at_insertion_point: bool, + cdata_allowed: bool, + entity_matcher: NamedCharacterReferenceMatcher, +} + +#[inline] +fn is_ascii_alpha(c: u32) -> bool { + matches!(c, 0x41..=0x5A | 0x61..=0x7A) +} + +#[inline] +fn is_ascii_upper_alpha(c: u32) -> bool { + matches!(c, 0x41..=0x5A) +} + +#[inline] +fn is_ascii_digit(c: u32) -> bool { + matches!(c, 0x30..=0x39) +} + +#[inline] +fn is_ascii_hex_digit(c: u32) -> bool { + matches!(c, 0x30..=0x39 | 0x41..=0x46 | 0x61..=0x66) +} + +#[inline] +fn is_ascii_alphanumeric(c: u32) -> bool { + is_ascii_alpha(c) || is_ascii_digit(c) +} + +#[inline] +fn to_ascii_lowercase(c: u32) -> u32 { + if is_ascii_upper_alpha(c) { c + 0x20 } else { c } +} + +#[inline] +fn is_whitespace(c: u32) -> bool { + matches!(c, 0x09 | 0x0A | 0x0C | 0x20) +} + +/// Numeric character reference replacement table (spec section 13.2.5.5). +fn numeric_char_ref_replacement(code: u32) -> Option { + match code { + 0x80 => Some(0x20AC), + 0x82 => Some(0x201A), + 0x83 => Some(0x0192), + 0x84 => Some(0x201E), + 0x85 => Some(0x2026), + 0x86 => Some(0x2020), + 0x87 => Some(0x2021), + 0x88 => Some(0x02C6), + 0x89 => Some(0x2030), + 0x8A => Some(0x0160), + 0x8B => Some(0x2039), + 0x8C => Some(0x0152), + 0x8E => Some(0x017D), + 0x91 => Some(0x2018), + 0x92 => Some(0x2019), + 0x93 => Some(0x201C), + 0x94 => Some(0x201D), + 0x95 => Some(0x2022), + 0x96 => Some(0x2013), + 0x97 => Some(0x2014), + 0x98 => Some(0x02DC), + 0x99 => Some(0x2122), + 0x9A => Some(0x0161), + 0x9B => Some(0x203A), + 0x9C => Some(0x0153), + 0x9E => Some(0x017E), + 0x9F => Some(0x0178), + _ => None, + } +} + +/// Check if a code point is a Unicode surrogate. +fn is_surrogate(c: u32) -> bool { + (0xD800..=0xDFFF).contains(&c) +} + +/// Check if a code point is a Unicode noncharacter. +fn is_noncharacter(c: u32) -> bool { + (0xFDD0..=0xFDEF).contains(&c) || matches!(c & 0xFFFF, 0xFFFE | 0xFFFF) +} + +#[inline] +fn is_ascii_lower_alpha(c: u32) -> bool { + matches!(c, 0x61..=0x7A) +} + +/// Push a code point onto a `String` as UTF-8 without paying +/// `char::from_u32`'s surrogate check or `String::push`'s len_utf8 branch +/// for the common ASCII case. For `cp < 0x80` this is a single byte push +/// to the underlying Vec; for wider code points we fall back to the +/// normal slow path. +#[inline(always)] +fn push_code_point(buf: &mut String, cp: u32) { + if cp < 0x80 { + // SAFETY: ASCII is always valid UTF-8. + unsafe { buf.as_mut_vec().push(cp as u8) }; + } else { + buf.push(char::from_u32(cp).unwrap_or('\u{FFFD}')); + } +} + +impl HtmlTokenizer { + /// Create a new tokenizer for the given input (as UTF-32 code points). + pub fn new(input: Vec) -> Self { + HtmlTokenizer { + state: State::Data, + return_state: State::Data, + input, + current_offset: 0, + prev_offset: 0, + current_token: Token::default(), + current_builder: String::new(), + queued_tokens: VecDeque::new(), + temporary_buffer: Vec::new(), + character_reference_code: 0, + last_emitted_start_tag_name: None, + source_positions: vec![Position { line: 0, column: 0 }], + current_line: 0, + current_column: 0, + has_emitted_eof: false, + aborted: false, + insertion_point: None, + old_insertion_points: Vec::new(), + explicit_eof_inserted: false, + input_stream_closed: true, + blocked: false, + stop_at_insertion_point: false, + cdata_allowed: false, + entity_matcher: NamedCharacterReferenceMatcher::new(), + } + } + + /// Set the tokenizer state. + pub fn switch_to(&mut self, state: State) { + self.state = state; + } + + // -- Insertion point management -- + + pub fn store_insertion_point(&mut self) { + self.old_insertion_points.push(self.insertion_point); + } + + pub fn restore_insertion_point(&mut self) { + self.insertion_point = self.old_insertion_points.pop().unwrap_or(None); + } + + pub fn update_insertion_point(&mut self) { + self.insertion_point = Some(self.current_offset); + } + + pub fn undefine_insertion_point(&mut self) { + self.insertion_point = None; + } + + pub fn is_insertion_point_defined(&self) -> bool { + self.insertion_point.is_some() + } + + pub fn is_insertion_point_reached(&self) -> bool { + self.insertion_point + .is_some_and(|insertion_point| self.current_offset >= insertion_point) + } + + pub fn append_input(&mut self, code_points: &[u32]) { + self.input.extend_from_slice(code_points); + } + + pub fn insert_input_at_insertion_point(&mut self, code_points: &[u32]) { + if let Some(ip) = self.insertion_point { + let ip = ip.min(self.input.len()); + self.input.splice(ip..ip, code_points.iter().copied()); + self.insertion_point = Some(ip + code_points.len()); + for old_insertion_point in &mut self.old_insertion_points { + if let Some(old_ip) = old_insertion_point + && ip <= *old_ip + { + *old_ip += code_points.len(); + } + } + } + } + + pub fn unparsed_input(&self) -> String { + let mut output = String::new(); + for code_point in self.input[self.current_offset..].iter().copied() { + push_code_point(&mut output, code_point); + } + output + } + + pub fn parser_did_run(&mut self) { + if self.current_offset == 0 + || self.current_offset != self.input.len() + || self.insertion_point.is_some_and(|insertion_point| insertion_point != 0) + || !self.old_insertion_points.is_empty() + { + return; + } + + self.input = Vec::new(); + self.current_offset = 0; + self.prev_offset = 0; + let last_position = *self.source_positions.last().unwrap_or(&Position::default()); + self.source_positions.clear(); + self.source_positions.push(last_position); + self.current_line = last_position.line; + self.current_column = last_position.column; + } + + pub fn set_blocked(&mut self, blocked: bool) { + self.blocked = blocked; + } + + pub fn is_blocked(&self) -> bool { + self.blocked + } + + pub fn insert_eof(&mut self) { + self.explicit_eof_inserted = true; + self.input_stream_closed = true; + } + + pub fn is_eof_inserted(&self) -> bool { + self.explicit_eof_inserted + } + + pub fn set_input_stream_closed(&mut self, closed: bool) { + self.input_stream_closed = closed; + } + + pub fn abort(&mut self) { + self.aborted = true; + } + + // -- Input helpers -- + + #[inline] + fn peek_code_point(&self, offset: isize) -> Option { + let idx = self.current_offset as isize + offset; + if idx < 0 || idx as usize >= self.input.len() { + return None; + } + if self.stop_at_insertion_point + && let Some(ip) = self.insertion_point + && idx as usize >= ip + { + return None; + } + Some(self.input[idx as usize]) + } + + #[inline(always)] + fn skip(&mut self, count: usize) { + if !self.source_positions.is_empty() { + let last = *self.source_positions.last().unwrap(); + self.source_positions.push(last); + } + // Keep position updates in local registers across the loop to + // avoid a store-to-load dependency on the source_positions stack + // top. The stack top is written back once at the end. + let (mut line, mut column) = if let Some(pos) = self.source_positions.last() { + (pos.line, pos.column) + } else { + (self.current_line, self.current_column) + }; + for _ in 0..count { + self.prev_offset = self.current_offset; + let code_point = self.input[self.current_offset]; + if code_point == 0x0A { + line += 1; + column = 0; + } else { + column += 1; + } + self.current_offset += 1; + } + self.current_line = line; + self.current_column = column; + if let Some(pos) = self.source_positions.last_mut() { + pos.line = line; + pos.column = column; + } + } + + #[inline(always)] + fn next_code_point(&mut self) -> Option { + if self.current_offset >= self.input.len() { + self.prev_offset = self.current_offset; + return None; + } + let cp = self.input[self.current_offset]; + if cp != 0x0D { + self.skip(1); + return Some(cp); + } + // Slow path: CR normalization + let next = self.peek_code_point(1).unwrap_or(0); + if next == 0x0A { + self.skip(2); + } else { + self.skip(1); + } + Some(0x0A) + } + + #[inline(always)] + fn nth_last_position(&self, n: usize) -> Position { + if n + 1 > self.source_positions.len() { + return Position { line: 0, column: 0 }; + } + self.source_positions[self.source_positions.len() - 1 - n] + } + + /// Fast-path character emission from the Data state: if the tokenizer + /// is positioned at a plain-text code point (not `<`, `&`, NUL, CR, + /// or any non-ASCII character) and the queue is empty, advance one + /// code point and return the character's position, skipping the + /// full `next_token` state-machine dispatch and Token construction. + /// Returns None if the slow path is required. + #[inline(always)] + pub fn try_fast_data_char(&mut self) -> Option<(u32, Position)> { + if self.aborted { + return None; + } + if self.state as u8 != State::Data as u8 { + return None; + } + if !self.queued_tokens.is_empty() { + return None; + } + if self.current_offset >= self.input.len() { + self.sync_source_positions(); + return None; + } + let cp = self.input[self.current_offset]; + if cp == 0x3C || cp == 0x26 || cp == 0x00 || cp == 0x0D || cp >= 0x80 { + self.sync_source_positions(); + return None; + } + if cp == 0x0A { + self.current_line += 1; + self.current_column = 0; + } else { + self.current_column += 1; + } + let pos = Position { + line: self.current_line, + column: self.current_column, + }; + self.prev_offset = self.current_offset; + self.current_offset += 1; + Some((cp, pos)) + } + + /// Resynchronise `source_positions` with the scalar current_line / + /// current_column after a run of fast-path character emissions. + #[inline(always)] + fn sync_source_positions(&mut self) { + self.source_positions.clear(); + self.source_positions.push(Position { + line: self.current_line, + column: self.current_column, + }); + } + + fn restore_to(&mut self, offset: usize) { + while self.current_offset > offset && self.source_positions.len() > 1 { + self.source_positions.pop(); + self.current_offset -= 1; + } + self.current_offset = offset; + if let Some(pos) = self.source_positions.last() { + self.current_line = pos.line; + self.current_column = pos.column; + } + } + + fn consume_current_builder(&mut self) -> String { + std::mem::take(&mut self.current_builder) + } + + /// Commit `current_builder` into `current_token.tag_name` (or its + /// interned-id slot). If the accumulated bytes hit the intern table + /// we clear the builder in place, preserving its capacity for the + /// next token and avoiding the String allocation entirely. Most real + /// HTML tag names are interned. + #[inline] + fn commit_current_builder_as_tag_name(&mut self) { + let id = crate::interned_names::lookup_tag_name(self.current_builder.as_bytes()); + if id != 0 { + self.current_builder.clear(); + self.current_token.set_tag_name_id(id); + } else { + let name = std::mem::take(&mut self.current_builder); + *self.current_token.tag_name_mut() = name; + } + } + + fn create_new_token(&mut self, token_type: TokenType) { + let payload = match token_type { + TokenType::StartTag | TokenType::EndTag => TokenPayload::Tag { + tag_name: String::new(), + tag_name_id: 0, + self_closing: false, + attributes: Vec::new(), + }, + TokenType::Comment => TokenPayload::Comment(String::new()), + TokenType::Doctype => TokenPayload::Doctype(Box::default()), + _ => TokenPayload::None, + }; + let pos = match token_type { + TokenType::StartTag | TokenType::EndTag => self.nth_last_position(1), + _ => self.nth_last_position(0), + }; + self.current_token = Token { + token_type, + code_point: 0, + payload, + start_position: pos, + end_position: Position::default(), + }; + } + + fn current_end_tag_token_is_appropriate(&self) -> bool { + if self.current_token.token_type != TokenType::EndTag { + return false; + } + match &self.last_emitted_start_tag_name { + Some(name) => self.current_token.tag_name() == name, + None => false, + } + } + + fn consumed_as_part_of_an_attribute(&self) -> bool { + matches!( + self.return_state, + State::AttributeValueDoubleQuoted | State::AttributeValueSingleQuoted | State::AttributeValueUnquoted + ) + } + + fn will_emit(&mut self, token_idx: usize) { + // token_idx: 0 = current_token, 1+ = queued_tokens index + // For simplicity, we handle position setting here. + if token_idx == 0 { + if self.current_token.token_type == TokenType::StartTag { + self.last_emitted_start_tag_name = Some(self.current_token.tag_name().to_string()); + } + let is_start_or_end_tag = self.current_token.token_type == TokenType::StartTag + || self.current_token.token_type == TokenType::EndTag; + self.current_token.end_position = self.nth_last_position(if is_start_or_end_tag { 1 } else { 0 }); + } + } + + fn emit_current_token(&mut self) { + self.will_emit(0); + let token = std::mem::take(&mut self.current_token); + self.queued_tokens.push_back(token); + } + + /// Fast-path version of `emit_current_token(); self.queued_tokens.pop_front()`. + /// When the queue is empty we skip the push/pop round trip entirely, + /// which removes two VecDeque ops and a Token move per emitted token + /// for the overwhelmingly common single-emit case. + #[inline] + fn emit_current_token_direct(&mut self) -> Option { + self.will_emit(0); + let token = std::mem::take(&mut self.current_token); + if self.queued_tokens.is_empty() { + Some(token) + } else { + self.queued_tokens.push_back(token); + self.queued_tokens.pop_front() + } + } + + #[inline(always)] + fn return_character_token(&mut self, code_point: u32) -> Option { + let mut token = Token::new_character(code_point); + token.start_position = self.nth_last_position(0); + if self.queued_tokens.is_empty() { + Some(token) + } else { + self.queued_tokens.push_back(token); + self.queued_tokens.pop_front() + } + } + + fn emit_eof(&mut self) { + if self.has_emitted_eof { + return; + } + self.has_emitted_eof = true; + self.create_new_token(TokenType::EndOfFile); + self.emit_current_token(); + } + + fn emit_current_token_followed_by_eof(&mut self) { + self.emit_current_token(); + self.has_emitted_eof = true; + self.create_new_token(TokenType::EndOfFile); + self.emit_current_token(); + } + + fn flush_codepoints_consumed_as_character_reference(&mut self) { + for i in 0..self.temporary_buffer.len() { + let code_point = self.temporary_buffer[i]; + if self.consumed_as_part_of_an_attribute() { + push_code_point(&mut self.current_builder, code_point); + } else { + let mut token = Token::new_character(code_point); + token.start_position = self.nth_last_position(0); + self.queued_tokens.push_back(token); + } + } + } + + fn can_run_out_at(&self, idx: usize) -> bool { + if self.stop_at_insertion_point + && let Some(ip) = self.insertion_point + && idx >= ip + { + return true; + } + !self.input_stream_closed && idx >= self.input.len() + } + + #[inline] + fn fast_scan_limit(&self) -> usize { + if self.stop_at_insertion_point + && let Some(ip) = self.insertion_point + { + return ip.min(self.input.len()); + } + self.input.len() + } + + fn should_pause_before_next_input_character(&self) -> bool { + if self.stop_at_insertion_point + && let Some(ip) = self.insertion_point + { + if self.current_offset >= ip { + return true; + } + + if self.current_offset < self.input.len() + && self.input[self.current_offset] == 0x0D + && self.current_offset + 1 == ip + { + return true; + } + } + + if self.input_stream_closed { + return false; + } + + if self.current_offset >= self.input.len() { + return true; + } + + self.input[self.current_offset] == 0x0D && self.current_offset + 1 >= self.input.len() + } + + /// Case-insensitive match of upcoming input against a string. + /// Returns Some(true) if matched and consumed, Some(false) if no match, + /// None if ran out of characters at the insertion point. + fn consume_next_if_match(&mut self, s: &str) -> Option { + for (i, b) in s.bytes().enumerate() { + match self.peek_code_point(i as isize) { + None => { + if self.can_run_out_at(self.current_offset + i) { + return None; + } + return Some(false); + } + Some(cp) => { + if to_ascii_lowercase(cp) != to_ascii_lowercase(b as u32) { + return Some(false); + } + } + } + } + // All matched, consume them + self.skip(s.len()); + Some(true) + } + + /// Case-sensitive match of upcoming input against a string. + fn consume_next_if_match_exact(&mut self, s: &str) -> Option { + for (i, b) in s.bytes().enumerate() { + match self.peek_code_point(i as isize) { + None => { + if self.can_run_out_at(self.current_offset + i) { + return None; + } + return Some(false); + } + Some(cp) => { + if cp != b as u32 { + return Some(false); + } + } + } + } + self.skip(s.len()); + Some(true) + } + + /// Get the next token from the tokenizer. + pub fn next_token(&mut self, stop_at_insertion_point: bool, cdata_allowed: bool) -> Option { + self.stop_at_insertion_point = stop_at_insertion_point; + self.cdata_allowed = cdata_allowed; + + // Return queued tokens first. + { + let last = *self.source_positions.last().unwrap_or(&Position::default()); + self.source_positions.clear(); + self.source_positions.push(last); + } + + if let Some(token) = self.queued_tokens.pop_front() { + return Some(token); + } + + if self.aborted { + return None; + } + + loop { + // Check insertion point before consuming. + if self.should_pause_before_next_input_character() { + return None; + } + + let current_input_character = self.next_code_point(); + + match self.state { + // 13.2.5.1 Data state + State::Data => { + match current_input_character { + Some(0x26) => { + // '&' + self.return_state = State::Data; + self.state = State::CharacterReference; + continue; + } + Some(0x3C) => { + // '<' + self.state = State::TagOpen; + continue; + } + Some(0x00) => { + // NULL - parse error + return self.return_character_token(0x00); + } + None => { + self.emit_eof(); + return self.queued_tokens.pop_front(); + } + Some(cp) => { + return self.return_character_token(cp); + } + } + } + + // 13.2.5.2 RCDATA state + State::RCDATA => match current_input_character { + Some(0x26) => { + self.return_state = State::RCDATA; + self.state = State::CharacterReference; + continue; + } + Some(0x3C) => { + self.state = State::RCDATALessThanSign; + continue; + } + Some(0x00) => { + return self.return_character_token(0xFFFD); + } + None => { + self.emit_eof(); + return self.queued_tokens.pop_front(); + } + Some(cp) => { + return self.return_character_token(cp); + } + }, + + // 13.2.5.3 RAWTEXT state + State::RAWTEXT => match current_input_character { + Some(0x3C) => { + self.state = State::RAWTEXTLessThanSign; + continue; + } + Some(0x00) => { + return self.return_character_token(0xFFFD); + } + None => { + self.emit_eof(); + return self.queued_tokens.pop_front(); + } + Some(cp) => { + return self.return_character_token(cp); + } + }, + + // 13.2.5.4 Script data state + State::ScriptData => match current_input_character { + Some(0x3C) => { + self.state = State::ScriptDataLessThanSign; + continue; + } + Some(0x00) => { + return self.return_character_token(0xFFFD); + } + None => { + self.emit_eof(); + return self.queued_tokens.pop_front(); + } + Some(cp) => { + return self.return_character_token(cp); + } + }, + + // 13.2.5.5 PLAINTEXT state + State::PLAINTEXT => match current_input_character { + Some(0x00) => { + return self.return_character_token(0xFFFD); + } + None => { + self.emit_eof(); + return self.queued_tokens.pop_front(); + } + Some(cp) => { + return self.return_character_token(cp); + } + }, + + // 13.2.5.6 Tag open state + State::TagOpen => match current_input_character { + Some(0x21) => { + self.state = State::MarkupDeclarationOpen; + continue; + } + Some(0x2F) => { + self.state = State::EndTagOpen; + continue; + } + Some(cp) if is_ascii_alpha(cp) => { + self.create_new_token(TokenType::StartTag); + self.reconsume(State::TagName); + continue; + } + Some(0x3F) => { + // '?' - parse error + self.create_new_token(TokenType::Comment); + self.current_token.start_position = self.nth_last_position(2); + self.reconsume(State::BogusComment); + continue; + } + None => { + // parse error + self.queued_tokens.push_back(self.make_character_token(0x3C)); + self.emit_eof(); + return self.queued_tokens.pop_front(); + } + Some(_) => { + // parse error + self.queued_tokens.push_back(self.make_character_token(0x3C)); + self.reconsume(State::Data); + continue; + } + }, + + // 13.2.5.7 End tag open state + State::EndTagOpen => match current_input_character { + Some(cp) if is_ascii_alpha(cp) => { + self.create_new_token(TokenType::EndTag); + self.reconsume(State::TagName); + continue; + } + Some(0x3E) => { + // '>' - parse error + self.state = State::Data; + continue; + } + None => { + // parse error + self.queued_tokens.push_back(self.make_character_token(0x3C)); + self.queued_tokens.push_back(self.make_character_token(0x2F)); + self.emit_eof(); + return self.queued_tokens.pop_front(); + } + Some(_) => { + // parse error + self.create_new_token(TokenType::Comment); + self.reconsume(State::BogusComment); + continue; + } + }, + + // 13.2.5.8 Tag name state + State::TagName => { + // Fast-path run: bulk-scan consecutive lowercase + // ASCII characters and append them to current_builder. + // This handles the common case where a tag name is + // a plain `div`, `span`, `a` etc. and avoids paying + // the per-character state-machine dispatch cost. + // We keep the first character the outer state loop + // already consumed, then extend the builder by + // scanning from current_offset until we see a byte + // that needs special handling. + if let Some(cp) = current_input_character + && cp >= b'a' as u32 + && cp <= b'z' as u32 + { + // SAFETY: pushing ASCII bytes into a String is + // always valid UTF-8. We drop into a tight + // inner loop that directly advances + // current_offset and current_column without + // going through skip(1) per character. + unsafe { self.current_builder.as_mut_vec().push(cp as u8) }; + let start_offset = self.current_offset; + let mut off = start_offset; + let input_len = self.fast_scan_limit(); + while off < input_len { + let next = self.input[off]; + if next < b'a' as u32 || next > b'z' as u32 { + break; + } + unsafe { self.current_builder.as_mut_vec().push(next as u8) }; + off += 1; + } + let consumed = off - start_offset; + if consumed > 0 { + self.current_column += consumed as u64; + self.prev_offset = off - 1; + self.current_offset = off; + if let Some(pos) = self.source_positions.last_mut() { + pos.line = self.current_line; + pos.column = self.current_column; + } + } + self.current_token.end_position = self.nth_last_position(0); + continue; + } + match current_input_character { + Some(cp) if is_whitespace(cp) => { + self.commit_current_builder_as_tag_name(); + self.current_token.end_position = self.nth_last_position(1); + self.state = State::BeforeAttributeName; + continue; + } + Some(0x2F) => { + self.commit_current_builder_as_tag_name(); + self.current_token.end_position = self.nth_last_position(0); + self.state = State::SelfClosingStartTag; + continue; + } + Some(0x3E) => { + self.commit_current_builder_as_tag_name(); + self.state = State::Data; + return self.emit_current_token_direct(); + } + Some(cp) if is_ascii_upper_alpha(cp) => { + push_code_point(&mut self.current_builder, to_ascii_lowercase(cp)); + self.current_token.end_position = self.nth_last_position(0); + continue; + } + Some(0x00) => { + self.current_builder.push('\u{FFFD}'); + self.current_token.end_position = self.nth_last_position(0); + continue; + } + None => { + self.emit_eof(); + return self.queued_tokens.pop_front(); + } + Some(cp) => { + push_code_point(&mut self.current_builder, cp); + self.current_token.end_position = self.nth_last_position(0); + continue; + } + } + } + + // 13.2.5.32 Before attribute name state + State::BeforeAttributeName => match current_input_character { + Some(cp) if is_whitespace(cp) => { + continue; + } + Some(0x2F) | Some(0x3E) | None => { + self.reconsume(State::AfterAttributeName); + continue; + } + Some(0x3D) => { + // '=' - parse error + self.current_token.attributes_mut().push(Attribute::default()); + self.current_builder.push('='); + self.state = State::AttributeName; + continue; + } + Some(_) => { + self.current_token.attributes_mut().push(Attribute::default()); + self.reconsume(State::AttributeName); + continue; + } + }, + + // 13.2.5.33 Attribute name state + State::AttributeName => { + // Fast-path run: bulk-scan consecutive lowercase ASCII + // and a few other name-safe bytes (digits, '-', '_'). + if let Some(cp) = current_input_character { + let is_name_char = (cp >= b'a' as u32 && cp <= b'z' as u32) + || (cp >= b'0' as u32 && cp <= b'9' as u32) + || cp == b'-' as u32 + || cp == b'_' as u32; + if is_name_char { + unsafe { self.current_builder.as_mut_vec().push(cp as u8) }; + let start_offset = self.current_offset; + let mut off = start_offset; + let input_len = self.fast_scan_limit(); + while off < input_len { + let next = self.input[off]; + let ok = (next >= b'a' as u32 && next <= b'z' as u32) + || (next >= b'0' as u32 && next <= b'9' as u32) + || next == b'-' as u32 + || next == b'_' as u32; + if !ok { + break; + } + unsafe { self.current_builder.as_mut_vec().push(next as u8) }; + off += 1; + } + let consumed = off - start_offset; + if consumed > 0 { + self.current_column += consumed as u64; + self.prev_offset = off - 1; + self.current_offset = off; + if let Some(pos) = self.source_positions.last_mut() { + pos.line = self.current_line; + pos.column = self.current_column; + } + } + continue; + } + } + match current_input_character { + Some(cp) if is_whitespace(cp) => { + self.set_attribute_name(); + self.reconsume(State::AfterAttributeName); + continue; + } + Some(0x2F) | Some(0x3E) | None => { + self.set_attribute_name(); + self.reconsume(State::AfterAttributeName); + continue; + } + Some(0x3D) => { + self.set_attribute_name(); + self.state = State::BeforeAttributeValue; + continue; + } + Some(cp) if is_ascii_upper_alpha(cp) => { + push_code_point(&mut self.current_builder, to_ascii_lowercase(cp)); + continue; + } + Some(0x00) => { + self.current_builder.push('\u{FFFD}'); + continue; + } + Some(cp @ (0x22 | 0x27 | 0x3C)) => { + // '"', '\'', '<' - parse error + push_code_point(&mut self.current_builder, cp); + continue; + } + Some(cp) => { + push_code_point(&mut self.current_builder, cp); + continue; + } + } + } + + // 13.2.5.34 After attribute name state + State::AfterAttributeName => match current_input_character { + Some(cp) if is_whitespace(cp) => { + continue; + } + Some(0x2F) => { + self.state = State::SelfClosingStartTag; + continue; + } + Some(0x3D) => { + self.state = State::BeforeAttributeValue; + continue; + } + Some(0x3E) => { + self.state = State::Data; + return self.emit_current_token_direct(); + } + None => { + self.emit_eof(); + return self.queued_tokens.pop_front(); + } + Some(_) => { + self.current_token.attributes_mut().push(Attribute::default()); + self.reconsume(State::AttributeName); + continue; + } + }, + + // 13.2.5.35 Before attribute value state + State::BeforeAttributeValue => { + let pos = self.nth_last_position(1); + if let Some(attr) = self.current_token.attributes_mut().last_mut() { + attr.value_start_position = pos; + } + match current_input_character { + Some(cp) if is_whitespace(cp) => { + continue; + } + Some(0x22) => { + self.state = State::AttributeValueDoubleQuoted; + continue; + } + Some(0x27) => { + self.state = State::AttributeValueSingleQuoted; + continue; + } + Some(0x3E) => { + // parse error + self.state = State::Data; + return self.emit_current_token_direct(); + } + _ => { + self.reconsume(State::AttributeValueUnquoted); + continue; + } + } + } + + // 13.2.5.36 Attribute value (double-quoted) state + State::AttributeValueDoubleQuoted => { + // Fast-path run: bulk-append printable ASCII until we + // hit `"`, `&`, NUL, CR or non-ASCII. The inner loop + // advances a local offset directly and only updates + // tokenizer/position state once at the end. + if let Some(cp) = current_input_character + && (0x20..0x80).contains(&cp) + && cp != 0x22 + && cp != 0x26 + { + unsafe { self.current_builder.as_mut_vec().push(cp as u8) }; + let start_offset = self.current_offset; + let mut off = start_offset; + let input_len = self.fast_scan_limit(); + while off < input_len { + let next = self.input[off]; + if !(0x20..0x80).contains(&next) || next == 0x22 || next == 0x26 { + break; + } + unsafe { self.current_builder.as_mut_vec().push(next as u8) }; + off += 1; + } + let consumed = off - start_offset; + if consumed > 0 { + self.current_column += consumed as u64; + self.prev_offset = off - 1; + self.current_offset = off; + if let Some(pos) = self.source_positions.last_mut() { + pos.line = self.current_line; + pos.column = self.current_column; + } + } + continue; + } + match current_input_character { + Some(0x22) => { + self.set_attribute_value(); + self.state = State::AfterAttributeValueQuoted; + continue; + } + Some(0x26) => { + self.return_state = State::AttributeValueDoubleQuoted; + self.state = State::CharacterReference; + continue; + } + Some(0x00) => { + self.current_builder.push('\u{FFFD}'); + continue; + } + None => { + self.emit_eof(); + return self.queued_tokens.pop_front(); + } + Some(cp) => { + push_code_point(&mut self.current_builder, cp); + continue; + } + } + } + + // 13.2.5.37 Attribute value (single-quoted) state + State::AttributeValueSingleQuoted => match current_input_character { + Some(0x27) => { + self.set_attribute_value(); + self.state = State::AfterAttributeValueQuoted; + continue; + } + Some(0x26) => { + self.return_state = State::AttributeValueSingleQuoted; + self.state = State::CharacterReference; + continue; + } + Some(0x00) => { + self.current_builder.push('\u{FFFD}'); + continue; + } + None => { + self.emit_eof(); + return self.queued_tokens.pop_front(); + } + Some(cp) => { + push_code_point(&mut self.current_builder, cp); + continue; + } + }, + + // 13.2.5.38 Attribute value (unquoted) state + State::AttributeValueUnquoted => match current_input_character { + Some(cp) if is_whitespace(cp) => { + self.set_attribute_value(); + self.state = State::BeforeAttributeName; + continue; + } + Some(0x26) => { + self.return_state = State::AttributeValueUnquoted; + self.state = State::CharacterReference; + continue; + } + Some(0x3E) => { + self.set_attribute_value(); + self.state = State::Data; + return self.emit_current_token_direct(); + } + Some(0x00) => { + self.current_builder.push('\u{FFFD}'); + continue; + } + Some(cp @ (0x22 | 0x27 | 0x3C | 0x3D | 0x60)) => { + // parse error, treat as anything else + push_code_point(&mut self.current_builder, cp); + continue; + } + None => { + self.emit_eof(); + return self.queued_tokens.pop_front(); + } + Some(cp) => { + push_code_point(&mut self.current_builder, cp); + continue; + } + }, + + // 13.2.5.39 After attribute value (quoted) state + State::AfterAttributeValueQuoted => { + let pos = self.nth_last_position(1); + if let Some(attr) = self.current_token.attributes_mut().last_mut() { + attr.value_end_position = pos; + } + match current_input_character { + Some(cp) if is_whitespace(cp) => { + self.state = State::BeforeAttributeName; + continue; + } + Some(0x2F) => { + self.state = State::SelfClosingStartTag; + continue; + } + Some(0x3E) => { + self.state = State::Data; + return self.emit_current_token_direct(); + } + None => { + self.emit_eof(); + return self.queued_tokens.pop_front(); + } + Some(_) => { + // parse error + self.reconsume(State::BeforeAttributeName); + continue; + } + } + } + + // 13.2.5.40 Self-closing start tag state + State::SelfClosingStartTag => match current_input_character { + Some(0x3E) => { + self.current_token.set_self_closing(true); + self.state = State::Data; + return self.emit_current_token_direct(); + } + None => { + self.emit_eof(); + return self.queued_tokens.pop_front(); + } + Some(_) => { + // parse error + self.reconsume(State::BeforeAttributeName); + continue; + } + }, + + // 13.2.5.41 Bogus comment state + State::BogusComment => match current_input_character { + Some(0x3E) => { + { + let data = self.consume_current_builder(); + self.current_token.set_comment_data(data); + } + self.state = State::Data; + return self.emit_current_token_direct(); + } + None => { + { + let data = self.consume_current_builder(); + self.current_token.set_comment_data(data); + } + self.emit_current_token_followed_by_eof(); + return self.queued_tokens.pop_front(); + } + Some(0x00) => { + self.current_builder.push('\u{FFFD}'); + continue; + } + Some(cp) => { + push_code_point(&mut self.current_builder, cp); + continue; + } + }, + + // 13.2.5.42 Markup declaration open state + State::MarkupDeclarationOpen => { + // Don't consume next input character (reconsume). + if current_input_character.is_some() { + self.restore_to(self.prev_offset); + } + + match self.consume_next_if_match_exact("--") { + Some(true) => { + self.create_new_token(TokenType::Comment); + self.current_token.start_position = self.nth_last_position(3); + self.state = State::CommentStart; + continue; + } + None => return None, + _ => {} + } + match self.consume_next_if_match("DOCTYPE") { + Some(true) => { + self.state = State::DOCTYPE; + continue; + } + None => return None, + _ => {} + } + match self.consume_next_if_match_exact("[CDATA[") { + Some(true) => { + if self.cdata_allowed { + self.state = State::CDATASection; + } else { + self.create_new_token(TokenType::Comment); + self.current_builder.push_str("[CDATA["); + self.state = State::BogusComment; + } + continue; + } + None => return None, + _ => {} + } + // parse error + self.create_new_token(TokenType::Comment); + self.state = State::BogusComment; + continue; + } + + // 13.2.5.43 Comment start state + State::CommentStart => match current_input_character { + Some(0x2D) => { + self.state = State::CommentStartDash; + continue; + } + Some(0x3E) => { + // parse error + { + let data = self.consume_current_builder(); + self.current_token.set_comment_data(data); + } + self.state = State::Data; + return self.emit_current_token_direct(); + } + _ => { + self.reconsume(State::Comment); + continue; + } + }, + + // 13.2.5.44 Comment start dash state + State::CommentStartDash => match current_input_character { + Some(0x2D) => { + self.state = State::CommentEnd; + continue; + } + Some(0x3E) => { + // parse error + { + let data = self.consume_current_builder(); + self.current_token.set_comment_data(data); + } + self.state = State::Data; + return self.emit_current_token_direct(); + } + None => { + { + let data = self.consume_current_builder(); + self.current_token.set_comment_data(data); + } + self.emit_current_token_followed_by_eof(); + return self.queued_tokens.pop_front(); + } + Some(_) => { + self.current_builder.push('-'); + self.reconsume(State::Comment); + continue; + } + }, + + // 13.2.5.45 Comment state + State::Comment => match current_input_character { + Some(0x3C) => { + self.current_builder.push('<'); + self.state = State::CommentLessThanSign; + continue; + } + Some(0x2D) => { + self.state = State::CommentEndDash; + continue; + } + Some(0x00) => { + self.current_builder.push('\u{FFFD}'); + continue; + } + None => { + { + let data = self.consume_current_builder(); + self.current_token.set_comment_data(data); + } + self.emit_current_token_followed_by_eof(); + return self.queued_tokens.pop_front(); + } + Some(cp) => { + push_code_point(&mut self.current_builder, cp); + continue; + } + }, + + // 13.2.5.46 Comment less-than sign state + State::CommentLessThanSign => match current_input_character { + Some(0x21) => { + self.current_builder.push('!'); + self.state = State::CommentLessThanSignBang; + continue; + } + Some(0x3C) => { + self.current_builder.push('<'); + continue; + } + _ => { + self.reconsume(State::Comment); + continue; + } + }, + + // 13.2.5.47 Comment less-than sign bang state + State::CommentLessThanSignBang => match current_input_character { + Some(0x2D) => { + self.state = State::CommentLessThanSignBangDash; + continue; + } + _ => { + self.reconsume(State::Comment); + continue; + } + }, + + // 13.2.5.48 Comment less-than sign bang dash state + State::CommentLessThanSignBangDash => match current_input_character { + Some(0x2D) => { + self.state = State::CommentLessThanSignBangDashDash; + continue; + } + _ => { + self.reconsume(State::CommentEndDash); + continue; + } + }, + + // 13.2.5.49 Comment less-than sign bang dash dash state + State::CommentLessThanSignBangDashDash => match current_input_character { + Some(0x3E) | None => { + self.reconsume(State::CommentEnd); + continue; + } + _ => { + // parse error + self.reconsume(State::CommentEnd); + continue; + } + }, + + // 13.2.5.50 Comment end dash state + State::CommentEndDash => match current_input_character { + Some(0x2D) => { + self.state = State::CommentEnd; + continue; + } + None => { + { + let data = self.consume_current_builder(); + self.current_token.set_comment_data(data); + } + self.emit_current_token_followed_by_eof(); + return self.queued_tokens.pop_front(); + } + Some(_) => { + self.current_builder.push('-'); + self.reconsume(State::Comment); + continue; + } + }, + + // 13.2.5.51 Comment end state + State::CommentEnd => match current_input_character { + Some(0x3E) => { + { + let data = self.consume_current_builder(); + self.current_token.set_comment_data(data); + } + self.state = State::Data; + return self.emit_current_token_direct(); + } + Some(0x21) => { + self.state = State::CommentEndBang; + continue; + } + Some(0x2D) => { + self.current_builder.push('-'); + continue; + } + None => { + { + let data = self.consume_current_builder(); + self.current_token.set_comment_data(data); + } + self.emit_current_token_followed_by_eof(); + return self.queued_tokens.pop_front(); + } + Some(_) => { + self.current_builder.push_str("--"); + self.reconsume(State::Comment); + continue; + } + }, + + // 13.2.5.52 Comment end bang state + State::CommentEndBang => match current_input_character { + Some(0x2D) => { + self.current_builder.push_str("--!"); + self.state = State::CommentEndDash; + continue; + } + Some(0x3E) => { + // parse error + { + let data = self.consume_current_builder(); + self.current_token.set_comment_data(data); + } + self.state = State::Data; + return self.emit_current_token_direct(); + } + None => { + { + let data = self.consume_current_builder(); + self.current_token.set_comment_data(data); + } + self.emit_current_token_followed_by_eof(); + return self.queued_tokens.pop_front(); + } + Some(_) => { + self.current_builder.push_str("--!"); + self.reconsume(State::Comment); + continue; + } + }, + + // 13.2.5.53 DOCTYPE state + State::DOCTYPE => match current_input_character { + Some(cp) if is_whitespace(cp) => { + self.state = State::BeforeDOCTYPEName; + continue; + } + Some(0x3E) => { + self.reconsume(State::BeforeDOCTYPEName); + continue; + } + None => { + self.create_new_token(TokenType::Doctype); + *self.current_token.doctype_data_mut() = DoctypeData { + force_quirks: true, + missing_name: true, + missing_public_identifier: true, + missing_system_identifier: true, + ..Default::default() + }; + self.emit_current_token_followed_by_eof(); + return self.queued_tokens.pop_front(); + } + Some(_) => { + // parse error + self.reconsume(State::BeforeDOCTYPEName); + continue; + } + }, + + // 13.2.5.54 Before DOCTYPE name state + State::BeforeDOCTYPEName => match current_input_character { + Some(cp) if is_whitespace(cp) => { + continue; + } + Some(cp) if is_ascii_upper_alpha(cp) => { + self.create_new_token(TokenType::Doctype); + *self.current_token.doctype_data_mut() = DoctypeData { + missing_name: false, + missing_public_identifier: true, + missing_system_identifier: true, + ..Default::default() + }; + push_code_point(&mut self.current_builder, to_ascii_lowercase(cp)); + self.state = State::DOCTYPEName; + continue; + } + Some(0x00) => { + self.create_new_token(TokenType::Doctype); + *self.current_token.doctype_data_mut() = DoctypeData { + missing_name: false, + missing_public_identifier: true, + missing_system_identifier: true, + ..Default::default() + }; + self.current_builder.push('\u{FFFD}'); + self.state = State::DOCTYPEName; + continue; + } + Some(0x3E) => { + // parse error + self.create_new_token(TokenType::Doctype); + *self.current_token.doctype_data_mut() = DoctypeData { + force_quirks: true, + missing_name: true, + missing_public_identifier: true, + missing_system_identifier: true, + ..Default::default() + }; + self.state = State::Data; + return self.emit_current_token_direct(); + } + None => { + self.create_new_token(TokenType::Doctype); + *self.current_token.doctype_data_mut() = DoctypeData { + force_quirks: true, + missing_name: true, + missing_public_identifier: true, + missing_system_identifier: true, + ..Default::default() + }; + self.emit_current_token_followed_by_eof(); + return self.queued_tokens.pop_front(); + } + Some(cp) => { + self.create_new_token(TokenType::Doctype); + *self.current_token.doctype_data_mut() = DoctypeData { + missing_name: false, + missing_public_identifier: true, + missing_system_identifier: true, + ..Default::default() + }; + push_code_point(&mut self.current_builder, cp); + self.state = State::DOCTYPEName; + continue; + } + }, + + // 13.2.5.55 DOCTYPE name state + State::DOCTYPEName => match current_input_character { + Some(cp) if is_whitespace(cp) => { + { + let name = self.consume_current_builder(); + { + let dd = self.current_token.doctype_data_mut(); + dd.name = name; + } + } + self.state = State::AfterDOCTYPEName; + continue; + } + Some(0x3E) => { + { + let name = self.consume_current_builder(); + { + let dd = self.current_token.doctype_data_mut(); + dd.name = name; + } + } + self.state = State::Data; + return self.emit_current_token_direct(); + } + Some(cp) if is_ascii_upper_alpha(cp) => { + push_code_point(&mut self.current_builder, to_ascii_lowercase(cp)); + continue; + } + Some(0x00) => { + self.current_builder.push('\u{FFFD}'); + continue; + } + None => { + let name = self.consume_current_builder(); + { + let dd = self.current_token.doctype_data_mut(); + dd.name = name; + dd.force_quirks = true; + } + self.emit_current_token_followed_by_eof(); + return self.queued_tokens.pop_front(); + } + Some(cp) => { + push_code_point(&mut self.current_builder, cp); + continue; + } + }, + + // 13.2.5.56 After DOCTYPE name state + State::AfterDOCTYPEName => match current_input_character { + Some(cp) if is_whitespace(cp) => { + continue; + } + Some(0x3E) => { + self.state = State::Data; + return self.emit_current_token_direct(); + } + None => { + { + let dd = self.current_token.doctype_data_mut(); + dd.force_quirks = true; + } + self.emit_current_token_followed_by_eof(); + return self.queued_tokens.pop_front(); + } + Some(_) => { + // Reconsume and try PUBLIC/SYSTEM + if current_input_character.is_some() { + self.restore_to(self.prev_offset); + } + match self.consume_next_if_match("PUBLIC") { + Some(true) => { + self.state = State::AfterDOCTYPEPublicKeyword; + continue; + } + None => return None, + _ => {} + } + match self.consume_next_if_match("SYSTEM") { + Some(true) => { + self.state = State::AfterDOCTYPESystemKeyword; + continue; + } + None => return None, + _ => {} + } + // Re-consume the character we put back + let _ = self.next_code_point(); + { + let dd = self.current_token.doctype_data_mut(); + dd.force_quirks = true; + } + self.reconsume(State::BogusDOCTYPE); + continue; + } + }, + + // 13.2.5.57 After DOCTYPE public keyword state + State::AfterDOCTYPEPublicKeyword => match current_input_character { + Some(cp) if is_whitespace(cp) => { + self.state = State::BeforeDOCTYPEPublicIdentifier; + continue; + } + Some(0x22) => { + { + let dd = self.current_token.doctype_data_mut(); + dd.missing_public_identifier = false; + } + self.state = State::DOCTYPEPublicIdentifierDoubleQuoted; + continue; + } + Some(0x27) => { + { + let dd = self.current_token.doctype_data_mut(); + dd.missing_public_identifier = false; + } + self.state = State::DOCTYPEPublicIdentifierSingleQuoted; + continue; + } + Some(0x3E) => { + { + let dd = self.current_token.doctype_data_mut(); + dd.force_quirks = true; + } + self.state = State::Data; + return self.emit_current_token_direct(); + } + None => { + { + let dd = self.current_token.doctype_data_mut(); + dd.force_quirks = true; + } + self.emit_current_token_followed_by_eof(); + return self.queued_tokens.pop_front(); + } + Some(_) => { + { + let dd = self.current_token.doctype_data_mut(); + dd.force_quirks = true; + } + self.reconsume(State::BogusDOCTYPE); + continue; + } + }, + + // 13.2.5.58 Before DOCTYPE public identifier state + State::BeforeDOCTYPEPublicIdentifier => match current_input_character { + Some(cp) if is_whitespace(cp) => continue, + Some(0x22) => { + { + let dd = self.current_token.doctype_data_mut(); + dd.missing_public_identifier = false; + } + self.state = State::DOCTYPEPublicIdentifierDoubleQuoted; + continue; + } + Some(0x27) => { + { + let dd = self.current_token.doctype_data_mut(); + dd.missing_public_identifier = false; + } + self.state = State::DOCTYPEPublicIdentifierSingleQuoted; + continue; + } + Some(0x3E) => { + { + let dd = self.current_token.doctype_data_mut(); + dd.force_quirks = true; + } + self.state = State::Data; + return self.emit_current_token_direct(); + } + None => { + { + let dd = self.current_token.doctype_data_mut(); + dd.force_quirks = true; + } + self.emit_current_token_followed_by_eof(); + return self.queued_tokens.pop_front(); + } + Some(_) => { + { + let dd = self.current_token.doctype_data_mut(); + dd.force_quirks = true; + } + self.reconsume(State::BogusDOCTYPE); + continue; + } + }, + + // 13.2.5.59 DOCTYPE public identifier (double-quoted) state + State::DOCTYPEPublicIdentifierDoubleQuoted => match current_input_character { + Some(0x22) => { + { + let val = self.consume_current_builder(); + { + let dd = self.current_token.doctype_data_mut(); + dd.public_identifier = val; + } + } + self.state = State::AfterDOCTYPEPublicIdentifier; + continue; + } + Some(0x00) => { + self.current_builder.push('\u{FFFD}'); + continue; + } + Some(0x3E) => { + { + let val = self.consume_current_builder(); + { + let dd = self.current_token.doctype_data_mut(); + dd.public_identifier = val; + dd.force_quirks = true; + } + } + self.state = State::Data; + return self.emit_current_token_direct(); + } + None => { + { + let val = self.consume_current_builder(); + { + let dd = self.current_token.doctype_data_mut(); + dd.public_identifier = val; + dd.force_quirks = true; + } + } + self.emit_current_token_followed_by_eof(); + return self.queued_tokens.pop_front(); + } + Some(cp) => { + push_code_point(&mut self.current_builder, cp); + continue; + } + }, + + // 13.2.5.60 DOCTYPE public identifier (single-quoted) state + State::DOCTYPEPublicIdentifierSingleQuoted => match current_input_character { + Some(0x27) => { + { + let val = self.consume_current_builder(); + { + let dd = self.current_token.doctype_data_mut(); + dd.public_identifier = val; + } + } + self.state = State::AfterDOCTYPEPublicIdentifier; + continue; + } + Some(0x00) => { + self.current_builder.push('\u{FFFD}'); + continue; + } + Some(0x3E) => { + { + let val = self.consume_current_builder(); + { + let dd = self.current_token.doctype_data_mut(); + dd.public_identifier = val; + dd.force_quirks = true; + } + } + self.state = State::Data; + return self.emit_current_token_direct(); + } + None => { + { + let val = self.consume_current_builder(); + { + let dd = self.current_token.doctype_data_mut(); + dd.public_identifier = val; + dd.force_quirks = true; + } + } + self.emit_current_token_followed_by_eof(); + return self.queued_tokens.pop_front(); + } + Some(cp) => { + push_code_point(&mut self.current_builder, cp); + continue; + } + }, + + // 13.2.5.61 After DOCTYPE public identifier state + State::AfterDOCTYPEPublicIdentifier => match current_input_character { + Some(cp) if is_whitespace(cp) => { + self.state = State::BetweenDOCTYPEPublicAndSystemIdentifiers; + continue; + } + Some(0x3E) => { + self.state = State::Data; + return self.emit_current_token_direct(); + } + Some(0x22) => { + { + let dd = self.current_token.doctype_data_mut(); + dd.system_identifier = String::new(); + dd.missing_system_identifier = false; + } + self.state = State::DOCTYPESystemIdentifierDoubleQuoted; + continue; + } + Some(0x27) => { + { + let dd = self.current_token.doctype_data_mut(); + dd.system_identifier = String::new(); + dd.missing_system_identifier = false; + } + self.state = State::DOCTYPESystemIdentifierSingleQuoted; + continue; + } + None => { + { + let dd = self.current_token.doctype_data_mut(); + dd.force_quirks = true; + } + self.emit_current_token_followed_by_eof(); + return self.queued_tokens.pop_front(); + } + Some(_) => { + { + let dd = self.current_token.doctype_data_mut(); + dd.force_quirks = true; + } + self.reconsume(State::BogusDOCTYPE); + continue; + } + }, + + // 13.2.5.62 Between DOCTYPE public and system identifiers state + State::BetweenDOCTYPEPublicAndSystemIdentifiers => match current_input_character { + Some(cp) if is_whitespace(cp) => continue, + Some(0x3E) => { + self.state = State::Data; + return self.emit_current_token_direct(); + } + Some(0x22) => { + { + let dd = self.current_token.doctype_data_mut(); + dd.system_identifier = String::new(); + dd.missing_system_identifier = false; + } + self.state = State::DOCTYPESystemIdentifierDoubleQuoted; + continue; + } + Some(0x27) => { + { + let dd = self.current_token.doctype_data_mut(); + dd.system_identifier = String::new(); + dd.missing_system_identifier = false; + } + self.state = State::DOCTYPESystemIdentifierSingleQuoted; + continue; + } + None => { + { + let dd = self.current_token.doctype_data_mut(); + dd.force_quirks = true; + } + self.emit_current_token_followed_by_eof(); + return self.queued_tokens.pop_front(); + } + Some(_) => { + { + let dd = self.current_token.doctype_data_mut(); + dd.force_quirks = true; + } + self.reconsume(State::BogusDOCTYPE); + continue; + } + }, + + // 13.2.5.63 After DOCTYPE system keyword state + State::AfterDOCTYPESystemKeyword => match current_input_character { + Some(cp) if is_whitespace(cp) => { + self.state = State::BeforeDOCTYPESystemIdentifier; + continue; + } + Some(0x22) => { + { + let dd = self.current_token.doctype_data_mut(); + dd.system_identifier = String::new(); + dd.missing_system_identifier = false; + } + self.state = State::DOCTYPESystemIdentifierDoubleQuoted; + continue; + } + Some(0x27) => { + { + let dd = self.current_token.doctype_data_mut(); + dd.system_identifier = String::new(); + dd.missing_system_identifier = false; + } + self.state = State::DOCTYPESystemIdentifierSingleQuoted; + continue; + } + Some(0x3E) => { + { + let dd = self.current_token.doctype_data_mut(); + dd.force_quirks = true; + } + self.state = State::Data; + return self.emit_current_token_direct(); + } + None => { + { + let dd = self.current_token.doctype_data_mut(); + dd.force_quirks = true; + } + self.emit_current_token_followed_by_eof(); + return self.queued_tokens.pop_front(); + } + Some(_) => { + { + let dd = self.current_token.doctype_data_mut(); + dd.force_quirks = true; + } + self.reconsume(State::BogusDOCTYPE); + continue; + } + }, + + // 13.2.5.64 Before DOCTYPE system identifier state + State::BeforeDOCTYPESystemIdentifier => match current_input_character { + Some(cp) if is_whitespace(cp) => continue, + Some(0x22) => { + { + let dd = self.current_token.doctype_data_mut(); + dd.system_identifier = String::new(); + dd.missing_system_identifier = false; + } + self.state = State::DOCTYPESystemIdentifierDoubleQuoted; + continue; + } + Some(0x27) => { + { + let dd = self.current_token.doctype_data_mut(); + dd.system_identifier = String::new(); + dd.missing_system_identifier = false; + } + self.state = State::DOCTYPESystemIdentifierSingleQuoted; + continue; + } + Some(0x3E) => { + { + let dd = self.current_token.doctype_data_mut(); + dd.force_quirks = true; + } + self.state = State::Data; + return self.emit_current_token_direct(); + } + None => { + { + let dd = self.current_token.doctype_data_mut(); + dd.force_quirks = true; + } + self.emit_current_token_followed_by_eof(); + return self.queued_tokens.pop_front(); + } + Some(_) => { + { + let dd = self.current_token.doctype_data_mut(); + dd.force_quirks = true; + } + self.reconsume(State::BogusDOCTYPE); + continue; + } + }, + + // 13.2.5.65 DOCTYPE system identifier (double-quoted) state + State::DOCTYPESystemIdentifierDoubleQuoted => match current_input_character { + Some(0x22) => { + let sys_id = self.consume_current_builder(); + { + let dd = self.current_token.doctype_data_mut(); + dd.system_identifier = sys_id; + } + self.state = State::AfterDOCTYPESystemIdentifier; + continue; + } + Some(0x00) => { + self.current_builder.push('\u{FFFD}'); + continue; + } + Some(0x3E) => { + let sys_id = self.consume_current_builder(); + { + let dd = self.current_token.doctype_data_mut(); + dd.system_identifier = sys_id; + dd.force_quirks = true; + } + self.state = State::Data; + return self.emit_current_token_direct(); + } + None => { + let sys_id = self.consume_current_builder(); + { + let dd = self.current_token.doctype_data_mut(); + dd.system_identifier = sys_id; + dd.force_quirks = true; + } + self.emit_current_token_followed_by_eof(); + return self.queued_tokens.pop_front(); + } + Some(cp) => { + push_code_point(&mut self.current_builder, cp); + continue; + } + }, + + // 13.2.5.66 DOCTYPE system identifier (single-quoted) state + State::DOCTYPESystemIdentifierSingleQuoted => match current_input_character { + Some(0x27) => { + let sys_id = self.consume_current_builder(); + { + let dd = self.current_token.doctype_data_mut(); + dd.system_identifier = sys_id; + } + self.state = State::AfterDOCTYPESystemIdentifier; + continue; + } + Some(0x00) => { + self.current_builder.push('\u{FFFD}'); + continue; + } + Some(0x3E) => { + let sys_id = self.consume_current_builder(); + { + let dd = self.current_token.doctype_data_mut(); + dd.system_identifier = sys_id; + dd.force_quirks = true; + } + self.state = State::Data; + return self.emit_current_token_direct(); + } + None => { + let sys_id = self.consume_current_builder(); + { + let dd = self.current_token.doctype_data_mut(); + dd.system_identifier = sys_id; + dd.force_quirks = true; + } + self.emit_current_token_followed_by_eof(); + return self.queued_tokens.pop_front(); + } + Some(cp) => { + push_code_point(&mut self.current_builder, cp); + continue; + } + }, + + // 13.2.5.67 After DOCTYPE system identifier state + State::AfterDOCTYPESystemIdentifier => match current_input_character { + Some(cp) if is_whitespace(cp) => continue, + Some(0x3E) => { + self.state = State::Data; + return self.emit_current_token_direct(); + } + None => { + { + let dd = self.current_token.doctype_data_mut(); + dd.force_quirks = true; + } + self.emit_current_token_followed_by_eof(); + return self.queued_tokens.pop_front(); + } + Some(_) => { + // parse error - do NOT set force_quirks + self.reconsume(State::BogusDOCTYPE); + continue; + } + }, + + // 13.2.5.68 Bogus DOCTYPE state + State::BogusDOCTYPE => match current_input_character { + Some(0x3E) => { + self.state = State::Data; + return self.emit_current_token_direct(); + } + Some(0x00) => continue, + None => { + self.emit_current_token_followed_by_eof(); + return self.queued_tokens.pop_front(); + } + Some(_) => continue, + }, + + // 13.2.5.69 CDATA section state + State::CDATASection => match current_input_character { + Some(0x5D) => { + self.state = State::CDATASectionBracket; + continue; + } + None => { + self.emit_eof(); + return self.queued_tokens.pop_front(); + } + Some(cp) => { + return self.return_character_token(cp); + } + }, + + // 13.2.5.70 CDATA section bracket state + State::CDATASectionBracket => match current_input_character { + Some(0x5D) => { + self.state = State::CDATASectionEnd; + continue; + } + _ => { + self.queued_tokens.push_back(self.make_character_token(0x5D)); + self.reconsume(State::CDATASection); + continue; + } + }, + + // 13.2.5.71 CDATA section end state + State::CDATASectionEnd => match current_input_character { + Some(0x5D) => { + return self.return_character_token(0x5D); + } + Some(0x3E) => { + self.state = State::Data; + continue; + } + _ => { + self.queued_tokens.push_back(self.make_character_token(0x5D)); + self.queued_tokens.push_back(self.make_character_token(0x5D)); + self.reconsume(State::CDATASection); + continue; + } + }, + + // 13.2.5.72 Character reference state + State::CharacterReference => { + self.temporary_buffer.clear(); + self.temporary_buffer.push(0x26); // '&' + self.entity_matcher = NamedCharacterReferenceMatcher::new(); + + match current_input_character { + Some(cp) if is_ascii_alphanumeric(cp) => { + self.reconsume(State::NamedCharacterReference); + continue; + } + Some(0x23) => { + self.temporary_buffer.push(0x23); + self.state = State::NumericCharacterReference; + continue; + } + _ => { + self.flush_codepoints_consumed_as_character_reference(); + self.reconsume_in_return_state(current_input_character); + if !self.queued_tokens.is_empty() { + return self.queued_tokens.pop_front(); + } + continue; + } + } + } + + // 13.2.5.73 Named character reference state + State::NamedCharacterReference => { + // Insertion-point path: feed one character at a time. + if self.stop_at_insertion_point && self.insertion_point.is_some() { + if let Some(cp) = current_input_character { + if self.entity_matcher.try_consume_code_point(cp) { + self.temporary_buffer.push(cp); + continue; + } + // Character not accepted by matcher. Reconsume it. + self.restore_to(self.prev_offset); + } else if self.can_run_out_at(self.current_offset) { + // At insertion point with no more chars -- pause. + return None; + } + // Fall through to resolution. + } else { + // Normal path: feed all remaining chars in a tight loop. + if current_input_character.is_some() { + self.restore_to(self.prev_offset); + } + let limit = self.input.len(); + while self.current_offset < limit { + let cp = self.input[self.current_offset]; + if !self.entity_matcher.try_consume_code_point(cp) { + break; + } + self.temporary_buffer.push(cp); + self.skip(1); + } + if !self.input_stream_closed && self.current_offset >= limit { + return None; + } + } + + // Resolution: backtrack overconsumed characters. + let overconsumed = self.entity_matcher.overconsumed_code_points() as usize; + if overconsumed > 0 { + self.restore_to(self.current_offset - overconsumed); + self.temporary_buffer + .truncate(self.temporary_buffer.len() - overconsumed); + } + + if let Some((first_cp, second_cp)) = self.entity_matcher.code_points() { + let ends_with_semi = self.entity_matcher.last_match_ends_with_semicolon(); + + // Check special attribute handling. + let next_cp = self.peek_code_point(0); + if self.consumed_as_part_of_an_attribute() + && !ends_with_semi + && next_cp.is_some_and(|c| c == 0x3D || is_ascii_alphanumeric(c)) + { + self.flush_codepoints_consumed_as_character_reference(); + self.state = self.return_state; + if !self.queued_tokens.is_empty() { + return self.queued_tokens.pop_front(); + } + continue; + } + + if !ends_with_semi { + // parse error + } + + self.temporary_buffer.clear(); + self.temporary_buffer.push(first_cp); + if second_cp != 0 { + self.temporary_buffer.push(second_cp); + } + self.flush_codepoints_consumed_as_character_reference(); + self.state = self.return_state; + if !self.queued_tokens.is_empty() { + return self.queued_tokens.pop_front(); + } + continue; + } else { + // No match found. + self.flush_codepoints_consumed_as_character_reference(); + self.state = State::AmbiguousAmpersand; + continue; + } + } + + // 13.2.5.74 Ambiguous ampersand state + State::AmbiguousAmpersand => match current_input_character { + Some(cp) if is_ascii_alphanumeric(cp) => { + if self.consumed_as_part_of_an_attribute() { + push_code_point(&mut self.current_builder, cp); + continue; + } else { + return self.return_character_token(cp); + } + } + Some(0x3B) => { + // parse error + self.reconsume_in_return_state(current_input_character); + if !self.queued_tokens.is_empty() { + return self.queued_tokens.pop_front(); + } + continue; + } + _ => { + self.reconsume_in_return_state(current_input_character); + if !self.queued_tokens.is_empty() { + return self.queued_tokens.pop_front(); + } + continue; + } + }, + + // 13.2.5.75 Numeric character reference state + State::NumericCharacterReference => { + self.character_reference_code = 0; + match current_input_character { + Some(0x78) | Some(0x58) => { + // 'x' or 'X' + self.temporary_buffer.push(current_input_character.unwrap()); + self.state = State::HexadecimalCharacterReferenceStart; + continue; + } + _ => { + self.reconsume(State::DecimalCharacterReferenceStart); + continue; + } + } + } + + // 13.2.5.76 Hexadecimal character reference start state + State::HexadecimalCharacterReferenceStart => match current_input_character { + Some(cp) if is_ascii_hex_digit(cp) => { + self.reconsume(State::HexadecimalCharacterReference); + continue; + } + _ => { + // parse error + self.flush_codepoints_consumed_as_character_reference(); + self.reconsume_in_return_state(current_input_character); + if !self.queued_tokens.is_empty() { + return self.queued_tokens.pop_front(); + } + continue; + } + }, + + // 13.2.5.77 Decimal character reference start state + State::DecimalCharacterReferenceStart => match current_input_character { + Some(cp) if is_ascii_digit(cp) => { + self.reconsume(State::DecimalCharacterReference); + continue; + } + _ => { + // parse error + self.flush_codepoints_consumed_as_character_reference(); + self.reconsume_in_return_state(current_input_character); + if !self.queued_tokens.is_empty() { + return self.queued_tokens.pop_front(); + } + continue; + } + }, + + // 13.2.5.78 Hexadecimal character reference state + State::HexadecimalCharacterReference => match current_input_character { + Some(cp) if is_ascii_digit(cp) => { + self.character_reference_code = + self.character_reference_code.wrapping_mul(16).wrapping_add(cp - 0x30); + continue; + } + Some(cp @ 0x41..=0x46) => { + self.character_reference_code = + self.character_reference_code.wrapping_mul(16).wrapping_add(cp - 0x37); + continue; + } + Some(cp @ 0x61..=0x66) => { + self.character_reference_code = + self.character_reference_code.wrapping_mul(16).wrapping_add(cp - 0x57); + continue; + } + Some(0x3B) => { + self.state = State::NumericCharacterReferenceEnd; + continue; + } + _ => { + // parse error + self.reconsume(State::NumericCharacterReferenceEnd); + continue; + } + }, + + // 13.2.5.79 Decimal character reference state + State::DecimalCharacterReference => match current_input_character { + Some(cp) if is_ascii_digit(cp) => { + self.character_reference_code = + self.character_reference_code.wrapping_mul(10).wrapping_add(cp - 0x30); + continue; + } + Some(0x3B) => { + self.state = State::NumericCharacterReferenceEnd; + continue; + } + _ => { + // parse error + self.reconsume(State::NumericCharacterReferenceEnd); + continue; + } + }, + + // 13.2.5.80 Numeric character reference end state + State::NumericCharacterReferenceEnd => { + // Don't consume + if current_input_character.is_some() { + self.restore_to(self.prev_offset); + } + + let code = self.character_reference_code; + let code = if code == 0 || code > 0x10FFFF || is_surrogate(code) { + 0xFFFD + } else if is_noncharacter(code) { + code // parse error but keep value + } else if let Some(replacement) = numeric_char_ref_replacement(code) { + replacement + } else if code == 0x0D { + 0x000D // parse error but keep + } else if code != 0 && code <= 0x1F && code != 0x09 && code != 0x0A && code != 0x0C { + code // control char, parse error but keep + } else if (0x7F..=0x9F).contains(&code) { + if let Some(replacement) = numeric_char_ref_replacement(code) { + replacement + } else { + code + } + } else { + code + }; + + self.temporary_buffer.clear(); + self.temporary_buffer.push(code); + self.flush_codepoints_consumed_as_character_reference(); + self.state = self.return_state; + if !self.queued_tokens.is_empty() { + return self.queued_tokens.pop_front(); + } + continue; + } + + // RCDATA less-than sign state + State::RCDATALessThanSign => match current_input_character { + Some(0x2F) => { + self.temporary_buffer.clear(); + self.state = State::RCDATAEndTagOpen; + continue; + } + _ => { + self.queued_tokens.push_back(self.make_character_token(0x3C)); + self.reconsume(State::RCDATA); + continue; + } + }, + + // RCDATA end tag open state + State::RCDATAEndTagOpen => match current_input_character { + Some(cp) if is_ascii_alpha(cp) => { + self.create_new_token(TokenType::EndTag); + self.reconsume(State::RCDATAEndTagName); + continue; + } + _ => { + self.queued_tokens.push_back(self.make_character_token(0x3C)); + self.queued_tokens.push_back(self.make_character_token(0x2F)); + self.reconsume(State::RCDATA); + continue; + } + }, + + // RCDATA end tag name state + State::RCDATAEndTagName => { + self.handle_rawtext_end_tag_name(current_input_character, State::RCDATA); + if !self.queued_tokens.is_empty() { + return self.queued_tokens.pop_front(); + } + continue; + } + + // RAWTEXT less-than sign state + State::RAWTEXTLessThanSign => match current_input_character { + Some(0x2F) => { + self.temporary_buffer.clear(); + self.state = State::RAWTEXTEndTagOpen; + continue; + } + _ => { + self.queued_tokens.push_back(self.make_character_token(0x3C)); + self.reconsume(State::RAWTEXT); + continue; + } + }, + + // RAWTEXT end tag open state + State::RAWTEXTEndTagOpen => match current_input_character { + Some(cp) if is_ascii_alpha(cp) => { + self.create_new_token(TokenType::EndTag); + self.reconsume(State::RAWTEXTEndTagName); + continue; + } + _ => { + self.queued_tokens.push_back(self.make_character_token(0x3C)); + self.queued_tokens.push_back(self.make_character_token(0x2F)); + self.reconsume(State::RAWTEXT); + continue; + } + }, + + // RAWTEXT end tag name state + State::RAWTEXTEndTagName => { + self.handle_rawtext_end_tag_name(current_input_character, State::RAWTEXT); + if !self.queued_tokens.is_empty() { + return self.queued_tokens.pop_front(); + } + continue; + } + + // Script data less-than sign state + State::ScriptDataLessThanSign => match current_input_character { + Some(0x2F) => { + self.temporary_buffer.clear(); + self.state = State::ScriptDataEndTagOpen; + continue; + } + Some(0x21) => { + self.queued_tokens.push_back(self.make_character_token(0x3C)); + self.queued_tokens.push_back(self.make_character_token(0x21)); + self.state = State::ScriptDataEscapeStart; + return self.queued_tokens.pop_front(); + } + _ => { + self.queued_tokens.push_back(self.make_character_token(0x3C)); + self.reconsume(State::ScriptData); + continue; + } + }, + + // Script data end tag open state + State::ScriptDataEndTagOpen => match current_input_character { + Some(cp) if is_ascii_alpha(cp) => { + self.create_new_token(TokenType::EndTag); + self.reconsume(State::ScriptDataEndTagName); + continue; + } + _ => { + self.queued_tokens.push_back(self.make_character_token(0x3C)); + self.queued_tokens.push_back(self.make_character_token(0x2F)); + self.reconsume(State::ScriptData); + continue; + } + }, + + // Script data end tag name state + State::ScriptDataEndTagName => { + self.handle_rawtext_end_tag_name(current_input_character, State::ScriptData); + if !self.queued_tokens.is_empty() { + return self.queued_tokens.pop_front(); + } + continue; + } + + // Script data escape start state + State::ScriptDataEscapeStart => match current_input_character { + Some(0x2D) => { + self.state = State::ScriptDataEscapeStartDash; + return self.return_character_token(0x2D); + } + _ => { + self.reconsume(State::ScriptData); + continue; + } + }, + + // Script data escape start dash state + State::ScriptDataEscapeStartDash => match current_input_character { + Some(0x2D) => { + self.state = State::ScriptDataEscapedDashDash; + return self.return_character_token(0x2D); + } + _ => { + self.reconsume(State::ScriptData); + continue; + } + }, + + // Script data escaped state + State::ScriptDataEscaped => match current_input_character { + Some(0x2D) => { + self.state = State::ScriptDataEscapedDash; + return self.return_character_token(0x2D); + } + Some(0x3C) => { + self.state = State::ScriptDataEscapedLessThanSign; + continue; + } + Some(0x00) => { + return self.return_character_token(0xFFFD); + } + None => { + self.emit_eof(); + return self.queued_tokens.pop_front(); + } + Some(cp) => { + return self.return_character_token(cp); + } + }, + + // Script data escaped dash state + State::ScriptDataEscapedDash => match current_input_character { + Some(0x2D) => { + self.state = State::ScriptDataEscapedDashDash; + return self.return_character_token(0x2D); + } + Some(0x3C) => { + self.state = State::ScriptDataEscapedLessThanSign; + continue; + } + Some(0x00) => { + self.state = State::ScriptDataEscaped; + return self.return_character_token(0xFFFD); + } + None => { + self.emit_eof(); + return self.queued_tokens.pop_front(); + } + Some(cp) => { + self.state = State::ScriptDataEscaped; + return self.return_character_token(cp); + } + }, + + // Script data escaped dash dash state + State::ScriptDataEscapedDashDash => match current_input_character { + Some(0x2D) => { + return self.return_character_token(0x2D); + } + Some(0x3C) => { + self.state = State::ScriptDataEscapedLessThanSign; + continue; + } + Some(0x3E) => { + self.state = State::ScriptData; + return self.return_character_token(0x3E); + } + Some(0x00) => { + self.state = State::ScriptDataEscaped; + return self.return_character_token(0xFFFD); + } + None => { + self.emit_eof(); + return self.queued_tokens.pop_front(); + } + Some(cp) => { + self.state = State::ScriptDataEscaped; + return self.return_character_token(cp); + } + }, + + // Script data escaped less-than sign state + State::ScriptDataEscapedLessThanSign => match current_input_character { + Some(0x2F) => { + self.temporary_buffer.clear(); + self.state = State::ScriptDataEscapedEndTagOpen; + continue; + } + Some(cp) if is_ascii_alpha(cp) => { + self.temporary_buffer.clear(); + self.queued_tokens.push_back(self.make_character_token(0x3C)); + self.reconsume(State::ScriptDataDoubleEscapeStart); + continue; + } + _ => { + self.queued_tokens.push_back(self.make_character_token(0x3C)); + self.reconsume(State::ScriptDataEscaped); + continue; + } + }, + + // Script data escaped end tag open state + State::ScriptDataEscapedEndTagOpen => match current_input_character { + Some(cp) if is_ascii_alpha(cp) => { + self.create_new_token(TokenType::EndTag); + self.reconsume(State::ScriptDataEscapedEndTagName); + continue; + } + _ => { + self.queued_tokens.push_back(self.make_character_token(0x3C)); + self.queued_tokens.push_back(self.make_character_token(0x2F)); + self.reconsume(State::ScriptDataEscaped); + continue; + } + }, + + // Script data escaped end tag name state + State::ScriptDataEscapedEndTagName => { + self.handle_rawtext_end_tag_name(current_input_character, State::ScriptDataEscaped); + if !self.queued_tokens.is_empty() { + return self.queued_tokens.pop_front(); + } + continue; + } + + // Script data double escape start state + State::ScriptDataDoubleEscapeStart => match current_input_character { + Some(cp) if is_whitespace(cp) || cp == 0x2F || cp == 0x3E => { + if self.temporary_buffer_equals_script() { + self.state = State::ScriptDataDoubleEscaped; + } else { + self.state = State::ScriptDataEscaped; + } + return self.return_character_token(cp); + } + Some(cp) if is_ascii_upper_alpha(cp) => { + self.temporary_buffer.push(to_ascii_lowercase(cp)); + return self.return_character_token(cp); + } + Some(cp) if is_ascii_lower_alpha(cp) => { + self.temporary_buffer.push(cp); + return self.return_character_token(cp); + } + _ => { + self.reconsume(State::ScriptDataEscaped); + continue; + } + }, + + // Script data double escaped state + State::ScriptDataDoubleEscaped => match current_input_character { + Some(0x2D) => { + self.state = State::ScriptDataDoubleEscapedDash; + return self.return_character_token(0x2D); + } + Some(0x3C) => { + self.state = State::ScriptDataDoubleEscapedLessThanSign; + return self.return_character_token(0x3C); + } + Some(0x00) => { + return self.return_character_token(0xFFFD); + } + None => { + self.emit_eof(); + return self.queued_tokens.pop_front(); + } + Some(cp) => { + return self.return_character_token(cp); + } + }, + + // Script data double escaped dash state + State::ScriptDataDoubleEscapedDash => match current_input_character { + Some(0x2D) => { + self.state = State::ScriptDataDoubleEscapedDashDash; + return self.return_character_token(0x2D); + } + Some(0x3C) => { + self.state = State::ScriptDataDoubleEscapedLessThanSign; + return self.return_character_token(0x3C); + } + Some(0x00) => { + self.state = State::ScriptDataDoubleEscaped; + return self.return_character_token(0xFFFD); + } + None => { + self.emit_eof(); + return self.queued_tokens.pop_front(); + } + Some(cp) => { + self.state = State::ScriptDataDoubleEscaped; + return self.return_character_token(cp); + } + }, + + // Script data double escaped dash dash state + State::ScriptDataDoubleEscapedDashDash => match current_input_character { + Some(0x2D) => { + return self.return_character_token(0x2D); + } + Some(0x3C) => { + self.state = State::ScriptDataDoubleEscapedLessThanSign; + return self.return_character_token(0x3C); + } + Some(0x3E) => { + self.state = State::ScriptData; + return self.return_character_token(0x3E); + } + Some(0x00) => { + self.state = State::ScriptDataDoubleEscaped; + return self.return_character_token(0xFFFD); + } + None => { + self.emit_eof(); + return self.queued_tokens.pop_front(); + } + Some(cp) => { + self.state = State::ScriptDataDoubleEscaped; + return self.return_character_token(cp); + } + }, + + // Script data double escaped less-than sign state + State::ScriptDataDoubleEscapedLessThanSign => match current_input_character { + Some(0x2F) => { + self.temporary_buffer.clear(); + self.state = State::ScriptDataDoubleEscapeEnd; + return self.return_character_token(0x2F); + } + _ => { + self.reconsume(State::ScriptDataDoubleEscaped); + continue; + } + }, + + // Script data double escape end state + State::ScriptDataDoubleEscapeEnd => match current_input_character { + Some(cp) if is_whitespace(cp) || cp == 0x2F || cp == 0x3E => { + if self.temporary_buffer_equals_script() { + self.state = State::ScriptDataEscaped; + } else { + self.state = State::ScriptDataDoubleEscaped; + } + return self.return_character_token(cp); + } + Some(cp) if is_ascii_upper_alpha(cp) => { + self.temporary_buffer.push(to_ascii_lowercase(cp)); + return self.return_character_token(cp); + } + Some(cp) if is_ascii_lower_alpha(cp) => { + self.temporary_buffer.push(cp); + return self.return_character_token(cp); + } + _ => { + self.reconsume(State::ScriptDataDoubleEscaped); + continue; + } + }, + } + } + } + + // -- Helper methods -- + + fn reconsume(&mut self, new_state: State) { + self.state = new_state; + if self.current_offset > 0 { + self.restore_to(self.prev_offset); + } + } + + fn reconsume_in_return_state(&mut self, current_input_character: Option) { + self.state = self.return_state; + if current_input_character.is_some() { + self.restore_to(self.prev_offset); + } + } + + #[inline(always)] + fn make_character_token(&self, code_point: u32) -> Token { + Token::new_character(code_point) + } + + fn set_attribute_name(&mut self) { + let id = crate::interned_names::lookup_attr_name(self.current_builder.as_bytes()); + let name_length = self.current_builder.chars().count() as u64; + let name_end = self.nth_last_position(1); + let name_start = Position { + line: name_end.line, + column: name_end.column.saturating_sub(name_length), + }; + if id != 0 { + self.current_builder.clear(); + if let Some(attr) = self.current_token.attributes_mut().last_mut() { + attr.local_name_id = id; + attr.local_name.clear(); + attr.name_start_position = name_start; + attr.name_end_position = name_end; + } + } else { + let name = self.consume_current_builder(); + if let Some(attr) = self.current_token.attributes_mut().last_mut() { + attr.local_name_id = 0; + attr.local_name = name; + attr.name_start_position = name_start; + attr.name_end_position = name_end; + } + } + } + + fn set_attribute_value(&mut self) { + let value = self.consume_current_builder(); + let value_end = self.nth_last_position(1); + if let Some(attr) = self.current_token.attributes_mut().last_mut() { + attr.value = value; + attr.value_end_position = value_end; + } + } + + fn temporary_buffer_equals_script(&self) -> bool { + self.temporary_buffer == [0x73, 0x63, 0x72, 0x69, 0x70, 0x74] + } + + /// Handle the common end-tag-name pattern shared by RCDATA, RAWTEXT, + /// ScriptData, and ScriptDataEscaped end tag name states. + fn handle_rawtext_end_tag_name(&mut self, current_input_character: Option, fallback_state: State) { + match current_input_character { + Some(cp) if is_whitespace(cp) || cp == 0x2F || cp == 0x3E => { + self.commit_current_builder_as_tag_name(); + if self.current_end_tag_token_is_appropriate() { + match cp { + _ if is_whitespace(cp) => { + self.state = State::BeforeAttributeName; + return; + } + 0x2F => { + self.state = State::SelfClosingStartTag; + return; + } + 0x3E => { + self.state = State::Data; + self.emit_current_token(); + return; + } + _ => {} + } + } + // Not appropriate - emit buffered chars + self.queued_tokens.push_back(self.make_character_token(0x3C)); + self.queued_tokens.push_back(self.make_character_token(0x2F)); + for i in 0..self.temporary_buffer.len() { + let cp = self.temporary_buffer[i]; + self.queued_tokens.push_back(self.make_character_token(cp)); + } + self.current_builder.clear(); + self.reconsume(fallback_state); + } + Some(cp) if is_ascii_upper_alpha(cp) => { + push_code_point(&mut self.current_builder, to_ascii_lowercase(cp)); + self.temporary_buffer.push(cp); + } + Some(cp) if is_ascii_lower_alpha(cp) => { + self.current_builder.push(char::from_u32(cp).unwrap()); + self.temporary_buffer.push(cp); + } + _ => { + self.queued_tokens.push_back(self.make_character_token(0x3C)); + self.queued_tokens.push_back(self.make_character_token(0x2F)); + for i in 0..self.temporary_buffer.len() { + let cp = self.temporary_buffer[i]; + self.queued_tokens.push_back(self.make_character_token(cp)); + } + self.current_builder.clear(); + self.reconsume(fallback_state); + } + } + } +} diff --git a/Libraries/LibWeb/Rust/Cargo.toml b/Libraries/LibWeb/Rust/Cargo.toml index e71fddd47b..4e9b5c8b22 100644 --- a/Libraries/LibWeb/Rust/Cargo.toml +++ b/Libraries/LibWeb/Rust/Cargo.toml @@ -9,6 +9,7 @@ crate-type = ["staticlib"] # After changing dependencies, regenerate the Flatpak sources: # python3 Meta/CMake/flatpak/generate-cargo-sources.py [dependencies] +libweb_html_tokenizer = { path = "../HTML/Parser/Rust" } [build-dependencies] cbindgen = "0.29" diff --git a/Libraries/LibWeb/Rust/src/lib.rs b/Libraries/LibWeb/Rust/src/lib.rs index 5f6283820b..30a5e7fcdc 100644 --- a/Libraries/LibWeb/Rust/src/lib.rs +++ b/Libraries/LibWeb/Rust/src/lib.rs @@ -9,6 +9,8 @@ mod rust_allocator; mod css_tokenizer; +pub use libweb_html_tokenizer as html_tokenizer; + use std::ffi::c_void; use std::panic::{AssertUnwindSafe, catch_unwind}; diff --git a/Tests/LibWeb/TestHTMLTokenizer.cpp b/Tests/LibWeb/TestHTMLTokenizer.cpp index f67e9f9a83..f388af39f4 100644 --- a/Tests/LibWeb/TestHTMLTokenizer.cpp +++ b/Tests/LibWeb/TestHTMLTokenizer.cpp @@ -199,6 +199,19 @@ TEST_CASE(character_reference_in_attribute) END_ENUMERATION(); } +TEST_CASE(duplicate_attributes_are_reported) +{ + auto tokens = run_tokenizer(""sv); + auto& token = tokens.first(); + EXPECT_EQ(token.type(), Token::Type::StartTag); + EXPECT(token.had_duplicate_attribute()); + EXPECT_EQ(token.attribute_count(), 1u); + + auto nonce = token.raw_attribute("nonce"_fly_string); + VERIFY(nonce.has_value()); + EXPECT_EQ(nonce->value, "x"); +} + TEST_CASE(named_character_reference) { auto tokens = run_tokenizer("⋶¬it;&cz"sv); @@ -275,3 +288,73 @@ TEST_CASE(ambiguous_ampersand_offset) EXPECT_EQ(token.start_position().line, 0u); EXPECT_EQ(token.start_position().column, 1u); } + +TEST_CASE(insertion_point_inside_fast_tag_name) +{ + Tokenizer tokenizer; + tokenizer.update_insertion_point(); + tokenizer.insert_input_at_insertion_point(""sv); + + EXPECT(!tokenizer.next_token(Tokenizer::StopAtInsertionPoint::Yes).has_value()); + EXPECT(tokenizer.is_insertion_point_reached()); + EXPECT_EQ(tokenizer.unparsed_input(), "def>"sv); + + tokenizer.insert_input_at_insertion_point("x"sv); + tokenizer.undefine_insertion_point(); + tokenizer.close_input_stream(); + + auto token = tokenizer.next_token(); + VERIFY(token.has_value()); + EXPECT_EQ(token->type(), Token::Type::StartTag); + EXPECT_EQ(token->tag_name(), "abcxdef"); +} + +TEST_CASE(insertion_point_inside_fast_attribute_name) +{ + Tokenizer tokenizer; + tokenizer.update_insertion_point(); + tokenizer.insert_input_at_insertion_point("

"sv); + + EXPECT(!tokenizer.next_token(Tokenizer::StopAtInsertionPoint::Yes).has_value()); + EXPECT(tokenizer.is_insertion_point_reached()); + EXPECT_EQ(tokenizer.unparsed_input(), "def=value>"sv); + + tokenizer.insert_input_at_insertion_point("x"sv); + tokenizer.undefine_insertion_point(); + tokenizer.close_input_stream(); + + auto token = tokenizer.next_token(); + VERIFY(token.has_value()); + EXPECT_EQ(token->type(), Token::Type::StartTag); + EXPECT_EQ(token->attribute_count(), 1u); + + auto attribute = token->raw_attribute("abcxdef"_fly_string); + VERIFY(attribute.has_value()); + EXPECT_EQ(attribute->value, "value"); +} + +TEST_CASE(insertion_point_inside_fast_quoted_attribute_value) +{ + Tokenizer tokenizer; + tokenizer.update_insertion_point(); + tokenizer.insert_input_at_insertion_point("

"sv); + + EXPECT(!tokenizer.next_token(Tokenizer::StopAtInsertionPoint::Yes).has_value()); + EXPECT(tokenizer.is_insertion_point_reached()); + EXPECT_EQ(tokenizer.unparsed_input(), "def\">"sv); + + tokenizer.insert_input_at_insertion_point("x"sv); + tokenizer.undefine_insertion_point(); + tokenizer.close_input_stream(); + + auto token = tokenizer.next_token(); + VERIFY(token.has_value()); + EXPECT_EQ(token->type(), Token::Type::StartTag); + + auto attribute = token->raw_attribute("a"_fly_string); + VERIFY(attribute.has_value()); + EXPECT_EQ(attribute->value, "abcxdef"); +} diff --git a/Tests/LibWeb/Text/expected/ContentSecurityPolicy/script-duplicate-nonce-attribute.txt b/Tests/LibWeb/Text/expected/ContentSecurityPolicy/script-duplicate-nonce-attribute.txt new file mode 100644 index 0000000000..c660efa29c --- /dev/null +++ b/Tests/LibWeb/Text/expected/ContentSecurityPolicy/script-duplicate-nonce-attribute.txt @@ -0,0 +1,2 @@ +PASS: duplicate nonce script blocked +PASS: valid nonce script executed diff --git a/Tests/LibWeb/Text/input/ContentSecurityPolicy/script-duplicate-nonce-attribute.html b/Tests/LibWeb/Text/input/ContentSecurityPolicy/script-duplicate-nonce-attribute.html new file mode 100644 index 0000000000..6267ea1f70 --- /dev/null +++ b/Tests/LibWeb/Text/input/ContentSecurityPolicy/script-duplicate-nonce-attribute.html @@ -0,0 +1,15 @@ + + + + + +