diff --git a/Cargo.lock b/Cargo.lock index 58bf1506fc..14dca35439 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -585,11 +585,19 @@ dependencies = [ "target-lexicon", ] +[[package]] +name = "libweb_html_tokenizer" +version = "0.1.0" +dependencies = [ + "cbindgen", +] + [[package]] name = "libweb_rust" version = "0.1.0" dependencies = [ "cbindgen", + "libweb_html_tokenizer", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 0140f68dad..1640bcabbe 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,6 +6,7 @@ members = [ "Libraries/LibUnicode/Rust", "Libraries/LibWasm/Rust", "Libraries/LibWeb/Rust", + "Libraries/LibWeb/HTML/Parser/Rust", ] exclude = [ "Libraries/LibJS/AsmIntGen", diff --git a/Libraries/LibWeb/CMakeLists.txt b/Libraries/LibWeb/CMakeLists.txt index 2b05d1c312..87adff4380 100644 --- a/Libraries/LibWeb/CMakeLists.txt +++ b/Libraries/LibWeb/CMakeLists.txt @@ -1253,6 +1253,8 @@ target_link_libraries(LibWeb PRIVATE LibCore LibCompress LibCrypto LibJS LibHTTP import_rust_crate(MANIFEST_PATH Rust/Cargo.toml CRATE_NAME libweb_rust FFI_HEADER RustFFI.h) target_link_libraries(LibWeb PRIVATE libweb_rust) +get_target_property(_libweb_rust_lib libweb_rust IMPORTED_LOCATION) +set_property(SOURCE HTML/Parser/HTMLTokenizer.cpp APPEND PROPERTY OBJECT_DEPENDS ${_libweb_rust_lib}) if ((LINUX OR BSD) AND NOT BUILD_SHARED_LIBS) target_link_options(LibWeb INTERFACE LINKER:--allow-multiple-definition) endif() @@ -1273,7 +1275,6 @@ if(NOT BUILD_SHARED_LIBS) # stale Rust objects. Force the C++ FFI bridge file to depend on the # Rust archive: when it changes, RustTokenizer.cpp recompiles, LibWeb # re-archives, and POST_BUILD re-merges the fresh Rust objects. - get_target_property(_libweb_rust_lib libweb_rust IMPORTED_LOCATION) set_property(SOURCE CSS/Parser/RustTokenizer.cpp APPEND PROPERTY OBJECT_DEPENDS ${_libweb_rust_lib}) endif() diff --git a/Libraries/LibWeb/HTML/Parser/HTMLTokenizer.cpp b/Libraries/LibWeb/HTML/Parser/HTMLTokenizer.cpp index 37753c7319..7f294062bd 100644 --- a/Libraries/LibWeb/HTML/Parser/HTMLTokenizer.cpp +++ b/Libraries/LibWeb/HTML/Parser/HTMLTokenizer.cpp @@ -5,2918 +5,92 @@ * SPDX-License-Identifier: BSD-2-Clause */ -#include #include -#include -#include -#include -#include +#include +#include #include -#include +#include #include #include #include +#include #include -#include namespace Web::HTML { -#pragma GCC diagnostic ignored "-Wunused-label" - -#define CONSUME_NEXT_INPUT_CHARACTER \ - current_input_character = next_code_point(stop_at_insertion_point); - -#define SWITCH_TO(new_state) \ - do { \ - VERIFY(m_current_builder.is_empty()); \ - SWITCH_TO_WITH_UNCLEAN_BUILDER(new_state); \ - } while (0) - -#define SWITCH_TO_WITH_UNCLEAN_BUILDER(new_state) \ - do { \ - will_switch_to(State::new_state); \ - m_state = State::new_state; \ - if (should_pause_before_next_input_character(stop_at_insertion_point)) \ - return {}; \ - CONSUME_NEXT_INPUT_CHARACTER; \ - goto new_state; \ - } while (0) - -#define RECONSUME_IN(new_state) \ - do { \ - will_reconsume_in(State::new_state); \ - m_state = State::new_state; \ - goto new_state; \ - } while (0) - -#define SWITCH_TO_RETURN_STATE \ - do { \ - will_switch_to(m_return_state); \ - m_state = m_return_state; \ - goto _StartOfFunction; \ - } while (0) - -#define RECONSUME_IN_RETURN_STATE \ - do { \ - will_reconsume_in(m_return_state); \ - m_state = m_return_state; \ - if (current_input_character.has_value()) \ - restore_to(m_prev_offset); \ - goto _StartOfFunction; \ - } while (0) - -#define SWITCH_TO_AND_EMIT_CURRENT_TOKEN(new_state) \ - do { \ - VERIFY(m_current_builder.is_empty()); \ - will_switch_to(State::new_state); \ - m_state = State::new_state; \ - will_emit(m_current_token); \ - m_queued_tokens.enqueue(move(m_current_token)); \ - return m_queued_tokens.dequeue(); \ - } while (0) - -#define EMIT_CHARACTER_AND_RECONSUME_IN(code_point, new_state) \ - do { \ - m_queued_tokens.enqueue(HTMLToken::make_character(code_point)); \ - will_reconsume_in(State::new_state); \ - m_state = State::new_state; \ - goto new_state; \ - } while (0) - -#define FLUSH_CODEPOINTS_CONSUMED_AS_A_CHARACTER_REFERENCE \ - do { \ - for (auto code_point : m_temporary_buffer) { \ - if (consumed_as_part_of_an_attribute()) { \ - m_current_builder.append_code_point(code_point); \ - } else { \ - create_new_token(HTMLToken::Type::Character); \ - m_current_token.set_code_point(code_point); \ - m_queued_tokens.enqueue(move(m_current_token)); \ - } \ - } \ - } while (0) - -#define DONT_CONSUME_NEXT_INPUT_CHARACTER \ - do { \ - if (current_input_character.has_value()) \ - restore_to(m_prev_offset); \ - } while (0) - -#define ON(code_point) \ - if (current_input_character.has_value() && current_input_character.value() == code_point) - -#define ON_EOF \ - if (!current_input_character.has_value()) - -#define ON_ASCII_ALPHA \ - if (current_input_character.has_value() && is_ascii_alpha(current_input_character.value())) - -#define ON_ASCII_ALPHANUMERIC \ - if (current_input_character.has_value() && is_ascii_alphanumeric(current_input_character.value())) - -#define ON_ASCII_UPPER_ALPHA \ - if (current_input_character.has_value() && is_ascii_upper_alpha(current_input_character.value())) - -#define ON_ASCII_LOWER_ALPHA \ - if (current_input_character.has_value() && is_ascii_lower_alpha(current_input_character.value())) - -#define ON_ASCII_DIGIT \ - if (current_input_character.has_value() && is_ascii_digit(current_input_character.value())) - -#define ON_ASCII_HEX_DIGIT \ - if (current_input_character.has_value() && is_ascii_hex_digit(current_input_character.value())) - -#define ON_WHITESPACE \ - if (current_input_character.has_value() && is_ascii(*current_input_character) && first_is_one_of(static_cast(*current_input_character), '\t', '\n', '\f', ' ')) - -#define ANYTHING_ELSE if (1) - -#define EMIT_EOF \ - do { \ - if (m_has_emitted_eof) \ - return {}; \ - m_has_emitted_eof = true; \ - create_new_token(HTMLToken::Type::EndOfFile); \ - will_emit(m_current_token); \ - m_queued_tokens.enqueue(move(m_current_token)); \ - return m_queued_tokens.dequeue(); \ - } while (0) - -#define EMIT_CURRENT_TOKEN_FOLLOWED_BY_EOF \ - do { \ - VERIFY(m_current_builder.is_empty()); \ - will_emit(m_current_token); \ - m_queued_tokens.enqueue(move(m_current_token)); \ - \ - m_has_emitted_eof = true; \ - create_new_token(HTMLToken::Type::EndOfFile); \ - will_emit(m_current_token); \ - m_queued_tokens.enqueue(move(m_current_token)); \ - \ - return m_queued_tokens.dequeue(); \ - } while (0) - -#define EMIT_CHARACTER(code_point) \ - do { \ - create_new_token(HTMLToken::Type::Character); \ - m_current_token.set_code_point(code_point); \ - m_queued_tokens.enqueue(move(m_current_token)); \ - return m_queued_tokens.dequeue(); \ - } while (0) - -#define EMIT_CURRENT_CHARACTER \ - EMIT_CHARACTER(current_input_character.value()); - -#define SWITCH_TO_AND_EMIT_CHARACTER(code_point, new_state) \ - do { \ - will_switch_to(State::new_state); \ - m_state = State::new_state; \ - EMIT_CHARACTER(code_point); \ - } while (0) - -#define SWITCH_TO_AND_EMIT_CURRENT_CHARACTER(new_state) \ - SWITCH_TO_AND_EMIT_CHARACTER(current_input_character.value(), new_state) - -#define BEGIN_STATE(state) \ - state: \ - case State::state: { \ - { \ - { - -#define END_STATE \ - VERIFY_NOT_REACHED(); \ - break; \ - } \ - } \ - } - -static inline void log_parse_error(SourceLocation const& location = SourceLocation::current()) +static Vector code_points_from_string(String const& string) { - dbgln_if(TOKENIZER_TRACE_DEBUG, "Parse error (tokenization) {}", location); + Vector code_points; + code_points.ensure_capacity(string.bytes().size()); + for (auto code_point : string.code_points()) + code_points.append(code_point); + return code_points; } -bool HTMLTokenizer::should_pause_before_next_input_character(StopAtInsertionPoint stop_at_insertion_point) const +static StringView ffi_string_view(u8 const* ptr, size_t len) { - if (stop_at_insertion_point == StopAtInsertionPoint::Yes && m_insertion_point.has_value()) { - if (m_current_offset >= *m_insertion_point) - return true; - - if (m_current_offset < static_cast(m_decoded_input.size()) - && m_decoded_input[m_current_offset] == '\r' - && m_current_offset + 1 == *m_insertion_point) { - // Newline normalization needs one code point of lookahead for CRLF. If document.write() - // stops the tokenizer immediately after a CR, wait for more input before deciding whether - // it is a CRLF pair or a standalone CR. - return true; - } - } - - if (m_input_stream_closed) - return false; - - if (m_current_offset >= static_cast(m_decoded_input.size())) - return true; - - // Newline normalization needs one code point of lookahead for CRLF. If a network chunk ends in - // CR, wait for either another code point or the input stream close before deciding whether it is - // a CRLF pair or a standalone CR. - return m_decoded_input[m_current_offset] == '\r' && m_current_offset + 1 >= static_cast(m_decoded_input.size()); -} - -bool HTMLTokenizer::can_run_out_of_characters(StopAtInsertionPoint stop_at_insertion_point) const -{ - return (stop_at_insertion_point == StopAtInsertionPoint::Yes && m_insertion_point.has_value()) - || !m_input_stream_closed; -} - -Optional HTMLTokenizer::next_code_point(StopAtInsertionPoint stop_at_insertion_point) -{ - if (m_current_offset >= static_cast(m_decoded_input.size())) + if (ptr == nullptr || len == 0) return {}; - - u32 code_point; - // https://html.spec.whatwg.org/multipage/parsing.html#preprocessing-the-input-stream:tokenization - // https://infra.spec.whatwg.org/#normalize-newlines - if (peek_code_point(0, stop_at_insertion_point).value_or(0) == '\r' && peek_code_point(1, stop_at_insertion_point).value_or(0) == '\n') { - // replace every U+000D CR U+000A LF code point pair with a single U+000A LF code point, - skip(2); - code_point = '\n'; - } else if (peek_code_point(0, stop_at_insertion_point).value_or(0) == '\r') { - // replace every remaining U+000D CR code point with a U+000A LF code point. - skip(1); - code_point = '\n'; - } else { - skip(1); - code_point = m_decoded_input[m_prev_offset]; - } - - dbgln_if(TOKENIZER_TRACE_DEBUG, "(Tokenizer) Next code_point: {}", code_point); - return code_point; + return { ptr, len }; } -void HTMLTokenizer::skip(size_t count) +static RustFfiTokenizerHandle* create_tokenizer_from_utf8(StringView utf8_bytes) { - VERIFY(count > 0); - if (!m_source_positions.is_empty()) - m_source_positions.append(m_source_positions.last()); + auto* bytes = reinterpret_cast(utf8_bytes.characters_without_null_termination()); + if (bytes == nullptr) + bytes = reinterpret_cast(""); + return rust_html_tokenizer_create_from_utf8(bytes, utf8_bytes.length()); +} + +static Vector build_interned_name_table(size_t count, void (*fetch)(uint16_t, uint8_t const**, size_t*)) +{ + Vector table; + // Slot 0 is unused (id 0 means "not interned"); store an empty FlyString there. + table.append(FlyString {}); + table.ensure_capacity(count + 1); for (size_t i = 0; i < count; ++i) { - m_prev_offset = m_current_offset; - auto code_point = m_decoded_input[m_current_offset]; - if (!m_source_positions.is_empty()) { - if (code_point == '\n') { - m_source_positions.last().column = 0; - m_source_positions.last().line++; - } else { - m_source_positions.last().column++; - } + uint8_t const* ptr = nullptr; + size_t len = 0; + fetch(static_cast(i + 1), &ptr, &len); + if (ptr == nullptr || len == 0) { + table.append(FlyString {}); + continue; } - ++m_current_offset; + table.append(MUST(FlyString::from_utf8(StringView { ptr, len }))); } + return table; } -Optional HTMLTokenizer::peek_code_point(ssize_t offset, StopAtInsertionPoint stop_at_insertion_point) const +static FlyString const& interned_rust_tag_name(uint16_t id) { - auto it = m_current_offset + offset; - if (it >= static_cast(m_decoded_input.size())) - return {}; - if (stop_at_insertion_point == StopAtInsertionPoint::Yes - && m_insertion_point.has_value() - && it >= *m_insertion_point) { - return {}; - } - return m_decoded_input[it]; + static Vector const s_table = build_interned_name_table( + rust_html_tokenizer_interned_tag_name_count(), + rust_html_tokenizer_interned_tag_name); + if (id == 0 || id >= s_table.size()) + return s_table[0]; + return s_table[id]; } -HTMLToken::Position HTMLTokenizer::nth_last_position(size_t n) +static FlyString const& interned_rust_attr_name(uint16_t id) { - if (n + 1 > m_source_positions.size()) { - dbgln_if(TOKENIZER_TRACE_DEBUG, "(Tokenizer::nth_last_position) Invalid position requested: {}th-last of {}. Returning (0-0).", n, m_source_positions.size()); - return HTMLToken::Position { 0, 0 }; - }; - return m_source_positions.at(m_source_positions.size() - 1 - n); -} - -Optional HTMLTokenizer::next_token(StopAtInsertionPoint stop_at_insertion_point) -{ - if (!m_source_positions.is_empty()) { - auto last_position = m_source_positions.last(); - m_source_positions.clear_with_capacity(); - m_source_positions.append(move(last_position)); - } -_StartOfFunction: - if (!m_queued_tokens.is_empty()) - return m_queued_tokens.dequeue(); - - if (m_aborted) - return {}; - - for (;;) { - if (should_pause_before_next_input_character(stop_at_insertion_point)) - return {}; - - auto current_input_character = next_code_point(stop_at_insertion_point); - switch (m_state) { - // 13.2.5.1 Data state, https://html.spec.whatwg.org/multipage/parsing.html#data-state - BEGIN_STATE(Data) - { - ON('&') - { - m_return_state = State::Data; - SWITCH_TO(CharacterReference); - } - ON('<') - { - SWITCH_TO(TagOpen); - } - ON(0) - { - log_parse_error(); - EMIT_CURRENT_CHARACTER; - } - ON_EOF - { - EMIT_EOF; - } - ANYTHING_ELSE - { - EMIT_CURRENT_CHARACTER; - } - } - END_STATE - - // 13.2.5.6 Tag open state, https://html.spec.whatwg.org/multipage/parsing.html#tag-open-state - BEGIN_STATE(TagOpen) - { - ON('!') - { - SWITCH_TO(MarkupDeclarationOpen); - } - ON('/') - { - SWITCH_TO(EndTagOpen); - } - ON_ASCII_ALPHA - { - create_new_token(HTMLToken::Type::StartTag); - RECONSUME_IN(TagName); - } - ON('?') - { - log_parse_error(); - create_new_token(HTMLToken::Type::Comment); - m_current_token.set_start_position({}, nth_last_position(2)); - RECONSUME_IN(BogusComment); - } - ON_EOF - { - log_parse_error(); - m_queued_tokens.enqueue(HTMLToken::make_character('<')); - EMIT_EOF; - } - ANYTHING_ELSE - { - log_parse_error(); - EMIT_CHARACTER_AND_RECONSUME_IN('<', Data); - } - } - END_STATE - - // 13.2.5.8 Tag name state, https://html.spec.whatwg.org/multipage/parsing.html#tag-name-state - BEGIN_STATE(TagName) - { - ON_WHITESPACE - { - m_current_token.set_tag_name(consume_current_builder()); - m_current_token.set_end_position({}, nth_last_position(1)); - SWITCH_TO(BeforeAttributeName); - } - ON('/') - { - m_current_token.set_tag_name(consume_current_builder()); - m_current_token.set_end_position({}, nth_last_position(0)); - SWITCH_TO(SelfClosingStartTag); - } - ON('>') - { - m_current_token.set_tag_name(consume_current_builder()); - SWITCH_TO_AND_EMIT_CURRENT_TOKEN(Data); - } - ON_ASCII_UPPER_ALPHA - { - m_current_builder.append_code_point(to_ascii_lowercase(current_input_character.value())); - m_current_token.set_end_position({}, nth_last_position(0)); - continue; - } - ON(0) - { - log_parse_error(); - m_current_builder.append_code_point(0xFFFD); - m_current_token.set_end_position({}, nth_last_position(0)); - continue; - } - ON_EOF - { - log_parse_error(); - EMIT_EOF; - } - ANYTHING_ELSE - { - m_current_builder.append_code_point(current_input_character.value()); - m_current_token.set_end_position({}, nth_last_position(0)); - continue; - } - } - END_STATE - - // 13.2.5.7 End tag open state, https://html.spec.whatwg.org/multipage/parsing.html#end-tag-open-state - BEGIN_STATE(EndTagOpen) - { - ON_ASCII_ALPHA - { - create_new_token(HTMLToken::Type::EndTag); - RECONSUME_IN(TagName); - } - ON('>') - { - log_parse_error(); - SWITCH_TO(Data); - } - ON_EOF - { - log_parse_error(); - m_queued_tokens.enqueue(HTMLToken::make_character('<')); - m_queued_tokens.enqueue(HTMLToken::make_character('/')); - EMIT_EOF; - } - ANYTHING_ELSE - { - log_parse_error(); - create_new_token(HTMLToken::Type::Comment); - RECONSUME_IN(BogusComment); - } - } - END_STATE - - // 13.2.5.42 Markup declaration open state, https://html.spec.whatwg.org/multipage/parsing.html#markup-declaration-open-state - BEGIN_STATE(MarkupDeclarationOpen) - { - DONT_CONSUME_NEXT_INPUT_CHARACTER; - - switch (consume_next_if_match("--"sv, stop_at_insertion_point)) { - case ConsumeNextResult::Consumed: - create_new_token(HTMLToken::Type::Comment); - m_current_token.set_start_position({}, nth_last_position(3)); - SWITCH_TO(CommentStart); - break; - case ConsumeNextResult::NotConsumed: - break; - case ConsumeNextResult::RanOutOfCharacters: - return {}; - } - - switch (consume_next_if_match("DOCTYPE"sv, stop_at_insertion_point, CaseSensitivity::CaseInsensitive)) { - case ConsumeNextResult::Consumed: - SWITCH_TO(DOCTYPE); - break; - case ConsumeNextResult::NotConsumed: - break; - case ConsumeNextResult::RanOutOfCharacters: - return {}; - } - - switch (consume_next_if_match("[CDATA["sv, stop_at_insertion_point)) { - case ConsumeNextResult::Consumed: - // We keep the parser optional so that syntax highlighting can be lexer-only. - // The parser registers itself with the lexer it creates. - if (m_parser != nullptr - && m_parser->adjusted_current_node() - && m_parser->adjusted_current_node()->namespace_uri() != Namespace::HTML) { - SWITCH_TO(CDATASection); - } else { - create_new_token(HTMLToken::Type::Comment); - m_current_builder.append("[CDATA["sv); - SWITCH_TO_WITH_UNCLEAN_BUILDER(BogusComment); - } - break; - case ConsumeNextResult::NotConsumed: - break; - case ConsumeNextResult::RanOutOfCharacters: - return {}; - } - ANYTHING_ELSE - { - log_parse_error(); - create_new_token(HTMLToken::Type::Comment); - SWITCH_TO(BogusComment); - } - } - END_STATE - - // 13.2.5.41 Bogus comment state, https://html.spec.whatwg.org/multipage/parsing.html#bogus-comment-state - BEGIN_STATE(BogusComment) - { - ON('>') - { - m_current_token.set_comment(consume_current_builder()); - SWITCH_TO_AND_EMIT_CURRENT_TOKEN(Data); - } - ON_EOF - { - m_current_token.set_comment(consume_current_builder()); - EMIT_CURRENT_TOKEN_FOLLOWED_BY_EOF; - } - ON(0) - { - log_parse_error(); - m_current_builder.append_code_point(0xFFFD); - continue; - } - ANYTHING_ELSE - { - m_current_builder.append_code_point(current_input_character.value()); - continue; - } - } - END_STATE - - // 13.2.5.53 DOCTYPE state, https://html.spec.whatwg.org/multipage/parsing.html#doctype-state - BEGIN_STATE(DOCTYPE) - { - ON_WHITESPACE - { - SWITCH_TO(BeforeDOCTYPEName); - } - ON('>') - { - RECONSUME_IN(BeforeDOCTYPEName); - } - ON_EOF - { - log_parse_error(); - create_new_token(HTMLToken::Type::DOCTYPE); - m_current_token.ensure_doctype_data().force_quirks = true; - m_queued_tokens.enqueue(move(m_current_token)); - EMIT_EOF; - } - ANYTHING_ELSE - { - log_parse_error(); - RECONSUME_IN(BeforeDOCTYPEName); - } - } - END_STATE - - // 13.2.5.54 Before DOCTYPE name state, https://html.spec.whatwg.org/multipage/parsing.html#before-doctype-name-state - BEGIN_STATE(BeforeDOCTYPEName) - { - ON_WHITESPACE - { - continue; - } - ON_ASCII_UPPER_ALPHA - { - create_new_token(HTMLToken::Type::DOCTYPE); - m_current_builder.append_code_point(to_ascii_lowercase(current_input_character.value())); - m_current_token.ensure_doctype_data().missing_name = false; - SWITCH_TO_WITH_UNCLEAN_BUILDER(DOCTYPEName); - } - ON(0) - { - log_parse_error(); - create_new_token(HTMLToken::Type::DOCTYPE); - m_current_builder.append_code_point(0xFFFD); - m_current_token.ensure_doctype_data().missing_name = false; - SWITCH_TO_WITH_UNCLEAN_BUILDER(DOCTYPEName); - } - ON('>') - { - log_parse_error(); - create_new_token(HTMLToken::Type::DOCTYPE); - m_current_token.ensure_doctype_data().force_quirks = true; - SWITCH_TO_AND_EMIT_CURRENT_TOKEN(Data); - } - ON_EOF - { - log_parse_error(); - create_new_token(HTMLToken::Type::DOCTYPE); - m_current_token.ensure_doctype_data().force_quirks = true; - m_queued_tokens.enqueue(move(m_current_token)); - EMIT_EOF; - } - ANYTHING_ELSE - { - create_new_token(HTMLToken::Type::DOCTYPE); - m_current_builder.append_code_point(current_input_character.value()); - m_current_token.ensure_doctype_data().missing_name = false; - SWITCH_TO_WITH_UNCLEAN_BUILDER(DOCTYPEName); - } - } - END_STATE - - // 13.2.5.55 DOCTYPE name state, https://html.spec.whatwg.org/multipage/parsing.html#doctype-name-state - BEGIN_STATE(DOCTYPEName) - { - ON_WHITESPACE - { - m_current_token.ensure_doctype_data().name = consume_current_builder(); - SWITCH_TO(AfterDOCTYPEName); - } - ON('>') - { - m_current_token.ensure_doctype_data().name = consume_current_builder(); - SWITCH_TO_AND_EMIT_CURRENT_TOKEN(Data); - } - ON_ASCII_UPPER_ALPHA - { - m_current_builder.append_code_point(to_ascii_lowercase(current_input_character.value())); - continue; - } - ON(0) - { - log_parse_error(); - m_current_builder.append_code_point(0xFFFD); - continue; - } - ON_EOF - { - log_parse_error(); - m_current_token.ensure_doctype_data().force_quirks = true; - m_queued_tokens.enqueue(move(m_current_token)); - EMIT_EOF; - } - ANYTHING_ELSE - { - m_current_builder.append_code_point(current_input_character.value()); - continue; - } - } - END_STATE - - // 13.2.5.56 After DOCTYPE name state, https://html.spec.whatwg.org/multipage/parsing.html#after-doctype-name-state - BEGIN_STATE(AfterDOCTYPEName) - { - ON_WHITESPACE - { - continue; - } - ON('>') - { - SWITCH_TO_AND_EMIT_CURRENT_TOKEN(Data); - } - ON_EOF - { - log_parse_error(); - m_current_token.ensure_doctype_data().force_quirks = true; - m_queued_tokens.enqueue(move(m_current_token)); - EMIT_EOF; - } - ANYTHING_ELSE - { - if (to_ascii_uppercase(current_input_character.value()) == 'P') { - switch (consume_next_if_match("UBLIC"sv, stop_at_insertion_point, CaseSensitivity::CaseInsensitive)) { - case ConsumeNextResult::Consumed: - SWITCH_TO(AfterDOCTYPEPublicKeyword); - break; - case ConsumeNextResult::NotConsumed: - break; - case ConsumeNextResult::RanOutOfCharacters: - DONT_CONSUME_NEXT_INPUT_CHARACTER; - return {}; - } - } - if (to_ascii_uppercase(current_input_character.value()) == 'S') { - switch (consume_next_if_match("YSTEM"sv, stop_at_insertion_point, CaseSensitivity::CaseInsensitive)) { - case ConsumeNextResult::Consumed: - SWITCH_TO(AfterDOCTYPESystemKeyword); - break; - case ConsumeNextResult::NotConsumed: - break; - case ConsumeNextResult::RanOutOfCharacters: - DONT_CONSUME_NEXT_INPUT_CHARACTER; - return {}; - } - } - log_parse_error(); - m_current_token.ensure_doctype_data().force_quirks = true; - RECONSUME_IN(BogusDOCTYPE); - } - } - END_STATE - - // 13.2.5.57 After DOCTYPE public keyword state, https://html.spec.whatwg.org/multipage/parsing.html#after-doctype-public-keyword-state - BEGIN_STATE(AfterDOCTYPEPublicKeyword) - { - ON_WHITESPACE - { - SWITCH_TO(BeforeDOCTYPEPublicIdentifier); - } - ON('"') - { - log_parse_error(); - m_current_token.ensure_doctype_data().missing_public_identifier = false; - SWITCH_TO(DOCTYPEPublicIdentifierDoubleQuoted); - } - ON('\'') - { - log_parse_error(); - m_current_token.ensure_doctype_data().missing_public_identifier = false; - SWITCH_TO(DOCTYPEPublicIdentifierSingleQuoted); - } - ON('>') - { - log_parse_error(); - m_current_token.ensure_doctype_data().force_quirks = true; - SWITCH_TO_AND_EMIT_CURRENT_TOKEN(Data); - } - ON_EOF - { - log_parse_error(); - m_current_token.ensure_doctype_data().force_quirks = true; - m_queued_tokens.enqueue(move(m_current_token)); - EMIT_EOF; - } - ANYTHING_ELSE - { - log_parse_error(); - m_current_token.ensure_doctype_data().force_quirks = true; - RECONSUME_IN(BogusDOCTYPE); - } - } - END_STATE - - // 13.2.5.63 After DOCTYPE system keyword state, https://html.spec.whatwg.org/multipage/parsing.html#after-doctype-system-keyword-state - BEGIN_STATE(AfterDOCTYPESystemKeyword) - { - ON_WHITESPACE - { - SWITCH_TO(BeforeDOCTYPESystemIdentifier); - } - ON('"') - { - log_parse_error(); - m_current_token.ensure_doctype_data().system_identifier = {}; - m_current_token.ensure_doctype_data().missing_system_identifier = false; - SWITCH_TO(DOCTYPESystemIdentifierDoubleQuoted); - } - ON('\'') - { - log_parse_error(); - m_current_token.ensure_doctype_data().system_identifier = {}; - m_current_token.ensure_doctype_data().missing_system_identifier = false; - SWITCH_TO(DOCTYPESystemIdentifierSingleQuoted); - } - ON('>') - { - log_parse_error(); - m_current_token.ensure_doctype_data().force_quirks = true; - SWITCH_TO_AND_EMIT_CURRENT_TOKEN(Data); - } - ON_EOF - { - log_parse_error(); - m_current_token.ensure_doctype_data().force_quirks = true; - m_queued_tokens.enqueue(move(m_current_token)); - EMIT_EOF; - } - ANYTHING_ELSE - { - log_parse_error(); - m_current_token.ensure_doctype_data().force_quirks = true; - RECONSUME_IN(BogusDOCTYPE); - } - } - END_STATE - - // 13.2.5.58 Before DOCTYPE public identifier state, https://html.spec.whatwg.org/multipage/parsing.html#before-doctype-public-identifier-state - BEGIN_STATE(BeforeDOCTYPEPublicIdentifier) - { - ON_WHITESPACE - { - continue; - } - ON('"') - { - m_current_token.ensure_doctype_data().missing_public_identifier = false; - SWITCH_TO(DOCTYPEPublicIdentifierDoubleQuoted); - } - ON('\'') - { - m_current_token.ensure_doctype_data().missing_public_identifier = false; - SWITCH_TO(DOCTYPEPublicIdentifierSingleQuoted); - } - ON('>') - { - log_parse_error(); - m_current_token.ensure_doctype_data().force_quirks = true; - SWITCH_TO_AND_EMIT_CURRENT_TOKEN(Data); - } - ON_EOF - { - log_parse_error(); - m_current_token.ensure_doctype_data().force_quirks = true; - m_queued_tokens.enqueue(move(m_current_token)); - EMIT_EOF; - } - ANYTHING_ELSE - { - log_parse_error(); - m_current_token.ensure_doctype_data().force_quirks = true; - RECONSUME_IN(BogusDOCTYPE); - } - } - END_STATE - - // 13.2.5.64 Before DOCTYPE system identifier state, https://html.spec.whatwg.org/multipage/parsing.html#before-doctype-system-identifier-state - BEGIN_STATE(BeforeDOCTYPESystemIdentifier) - { - ON_WHITESPACE - { - continue; - } - ON('"') - { - m_current_token.ensure_doctype_data().missing_system_identifier = false; - SWITCH_TO(DOCTYPESystemIdentifierDoubleQuoted); - } - ON('\'') - { - m_current_token.ensure_doctype_data().missing_system_identifier = false; - SWITCH_TO(DOCTYPESystemIdentifierSingleQuoted); - } - ON('>') - { - log_parse_error(); - m_current_token.ensure_doctype_data().force_quirks = true; - SWITCH_TO_AND_EMIT_CURRENT_TOKEN(Data); - } - ON_EOF - { - log_parse_error(); - m_current_token.ensure_doctype_data().force_quirks = true; - m_queued_tokens.enqueue(move(m_current_token)); - EMIT_EOF; - } - ANYTHING_ELSE - { - log_parse_error(); - m_current_token.ensure_doctype_data().force_quirks = true; - RECONSUME_IN(BogusDOCTYPE); - } - } - END_STATE - - // 13.2.5.59 DOCTYPE public identifier (double-quoted) state, https://html.spec.whatwg.org/multipage/parsing.html#doctype-public-identifier-(double-quoted)-state - BEGIN_STATE(DOCTYPEPublicIdentifierDoubleQuoted) - { - ON('"') - { - m_current_token.ensure_doctype_data().public_identifier = consume_current_builder(); - SWITCH_TO(AfterDOCTYPEPublicIdentifier); - } - ON(0) - { - log_parse_error(); - m_current_builder.append_code_point(0xFFFD); - continue; - } - ON('>') - { - log_parse_error(); - m_current_token.ensure_doctype_data().public_identifier = consume_current_builder(); - m_current_token.ensure_doctype_data().force_quirks = true; - SWITCH_TO_AND_EMIT_CURRENT_TOKEN(Data); - } - ON_EOF - { - log_parse_error(); - m_current_token.ensure_doctype_data().force_quirks = true; - m_queued_tokens.enqueue(move(m_current_token)); - EMIT_EOF; - } - ANYTHING_ELSE - { - m_current_builder.append_code_point(current_input_character.value()); - continue; - } - } - END_STATE - - // 13.2.5.60 DOCTYPE public identifier (single-quoted) state, https://html.spec.whatwg.org/multipage/parsing.html#doctype-public-identifier-(single-quoted)-state - BEGIN_STATE(DOCTYPEPublicIdentifierSingleQuoted) - { - ON('\'') - { - m_current_token.ensure_doctype_data().public_identifier = consume_current_builder(); - SWITCH_TO(AfterDOCTYPEPublicIdentifier); - } - ON(0) - { - log_parse_error(); - m_current_builder.append_code_point(0xFFFD); - continue; - } - ON('>') - { - log_parse_error(); - m_current_token.ensure_doctype_data().public_identifier = consume_current_builder(); - m_current_token.ensure_doctype_data().force_quirks = true; - SWITCH_TO_AND_EMIT_CURRENT_TOKEN(Data); - } - ON_EOF - { - log_parse_error(); - m_current_token.ensure_doctype_data().force_quirks = true; - m_queued_tokens.enqueue(move(m_current_token)); - EMIT_EOF; - } - ANYTHING_ELSE - { - m_current_builder.append_code_point(current_input_character.value()); - continue; - } - } - END_STATE - - // 13.2.5.65 DOCTYPE system identifier (double-quoted) state, https://html.spec.whatwg.org/multipage/parsing.html#doctype-system-identifier-(double-quoted)-state - BEGIN_STATE(DOCTYPESystemIdentifierDoubleQuoted) - { - ON('"') - { - m_current_token.ensure_doctype_data().system_identifier = consume_current_builder(); - SWITCH_TO(AfterDOCTYPESystemIdentifier); - } - ON(0) - { - log_parse_error(); - m_current_builder.append_code_point(0xFFFD); - continue; - } - ON('>') - { - log_parse_error(); - m_current_token.ensure_doctype_data().system_identifier = consume_current_builder(); - m_current_token.ensure_doctype_data().force_quirks = true; - SWITCH_TO_AND_EMIT_CURRENT_TOKEN(Data); - } - ON_EOF - { - log_parse_error(); - m_current_token.ensure_doctype_data().force_quirks = true; - m_queued_tokens.enqueue(move(m_current_token)); - EMIT_EOF; - } - ANYTHING_ELSE - { - m_current_builder.append_code_point(current_input_character.value()); - continue; - } - } - END_STATE - - // 13.2.5.66 DOCTYPE system identifier (single-quoted) state, https://html.spec.whatwg.org/multipage/parsing.html#doctype-system-identifier-(single-quoted)-state - BEGIN_STATE(DOCTYPESystemIdentifierSingleQuoted) - { - ON('\'') - { - m_current_token.ensure_doctype_data().system_identifier = consume_current_builder(); - SWITCH_TO(AfterDOCTYPESystemIdentifier); - } - ON(0) - { - log_parse_error(); - m_current_builder.append_code_point(0xFFFD); - continue; - } - ON('>') - { - log_parse_error(); - m_current_token.ensure_doctype_data().system_identifier = consume_current_builder(); - m_current_token.ensure_doctype_data().force_quirks = true; - SWITCH_TO_AND_EMIT_CURRENT_TOKEN(Data); - } - ON_EOF - { - log_parse_error(); - m_current_token.ensure_doctype_data().force_quirks = true; - m_queued_tokens.enqueue(move(m_current_token)); - EMIT_EOF; - } - ANYTHING_ELSE - { - m_current_builder.append_code_point(current_input_character.value()); - continue; - } - } - END_STATE - - // 13.2.5.61 After DOCTYPE public identifier state, https://html.spec.whatwg.org/multipage/parsing.html#after-doctype-public-identifier-state - BEGIN_STATE(AfterDOCTYPEPublicIdentifier) - { - ON_WHITESPACE - { - SWITCH_TO(BetweenDOCTYPEPublicAndSystemIdentifiers); - } - ON('>') - { - SWITCH_TO_AND_EMIT_CURRENT_TOKEN(Data); - } - ON('"') - { - log_parse_error(); - m_current_token.ensure_doctype_data().missing_system_identifier = false; - SWITCH_TO(DOCTYPESystemIdentifierDoubleQuoted); - } - ON('\'') - { - log_parse_error(); - m_current_token.ensure_doctype_data().missing_system_identifier = false; - SWITCH_TO(DOCTYPESystemIdentifierSingleQuoted); - } - ON_EOF - { - log_parse_error(); - m_current_token.ensure_doctype_data().force_quirks = true; - m_queued_tokens.enqueue(move(m_current_token)); - EMIT_EOF; - } - ANYTHING_ELSE - { - log_parse_error(); - m_current_token.ensure_doctype_data().force_quirks = true; - RECONSUME_IN(BogusDOCTYPE); - } - } - END_STATE - - // 13.2.5.62 Between DOCTYPE public and system identifiers state, https://html.spec.whatwg.org/multipage/parsing.html#between-doctype-public-and-system-identifiers-state - BEGIN_STATE(BetweenDOCTYPEPublicAndSystemIdentifiers) - { - ON_WHITESPACE - { - continue; - } - ON('>') - { - SWITCH_TO_AND_EMIT_CURRENT_TOKEN(Data); - } - ON('"') - { - m_current_token.ensure_doctype_data().missing_system_identifier = false; - SWITCH_TO(DOCTYPESystemIdentifierDoubleQuoted); - } - ON('\'') - { - m_current_token.ensure_doctype_data().missing_system_identifier = false; - SWITCH_TO(DOCTYPESystemIdentifierSingleQuoted); - } - ON_EOF - { - log_parse_error(); - m_current_token.ensure_doctype_data().force_quirks = true; - m_queued_tokens.enqueue(move(m_current_token)); - EMIT_EOF; - } - ANYTHING_ELSE - { - log_parse_error(); - m_current_token.ensure_doctype_data().force_quirks = true; - RECONSUME_IN(BogusDOCTYPE); - } - } - END_STATE - - // 13.2.5.67 After DOCTYPE system identifier state, https://html.spec.whatwg.org/multipage/parsing.html#after-doctype-system-identifier-state - BEGIN_STATE(AfterDOCTYPESystemIdentifier) - { - ON_WHITESPACE - { - continue; - } - ON('>') - { - SWITCH_TO_AND_EMIT_CURRENT_TOKEN(Data); - } - ON_EOF - { - log_parse_error(); - m_current_token.ensure_doctype_data().force_quirks = true; - m_queued_tokens.enqueue(move(m_current_token)); - EMIT_EOF; - } - ANYTHING_ELSE - { - log_parse_error(); - RECONSUME_IN(BogusDOCTYPE); - } - } - END_STATE - - // 13.2.5.68 Bogus DOCTYPE state, https://html.spec.whatwg.org/multipage/parsing.html#bogus-doctype-state - BEGIN_STATE(BogusDOCTYPE) - { - ON('>') - { - SWITCH_TO_AND_EMIT_CURRENT_TOKEN(Data); - } - ON(0) - { - log_parse_error(); - continue; - } - ON_EOF - { - m_queued_tokens.enqueue(move(m_current_token)); - EMIT_EOF; - } - ANYTHING_ELSE - { - continue; - } - } - END_STATE - - // 13.2.5.32 Before attribute name state, https://html.spec.whatwg.org/multipage/parsing.html#before-attribute-name-state - BEGIN_STATE(BeforeAttributeName) - { - ON_WHITESPACE - { - continue; - } - ON('/') - { - RECONSUME_IN(AfterAttributeName); - } - ON('>') - { - RECONSUME_IN(AfterAttributeName); - } - ON_EOF - { - RECONSUME_IN(AfterAttributeName); - } - ON('=') - { - log_parse_error(); - HTMLToken::Attribute new_attribute; - new_attribute.name_start_position = nth_last_position(1); - m_current_builder.append_code_point(current_input_character.value()); - m_current_token.add_attribute(move(new_attribute)); - SWITCH_TO_WITH_UNCLEAN_BUILDER(AttributeName); - } - ANYTHING_ELSE - { - HTMLToken::Attribute new_attribute; - new_attribute.name_start_position = nth_last_position(1); - m_current_token.add_attribute(move(new_attribute)); - RECONSUME_IN(AttributeName); - } - } - END_STATE - - // 13.2.5.40 Self-closing start tag state, https://html.spec.whatwg.org/multipage/parsing.html#self-closing-start-tag-state - BEGIN_STATE(SelfClosingStartTag) - { - ON('>') - { - m_current_token.set_self_closing(true); - SWITCH_TO_AND_EMIT_CURRENT_TOKEN(Data); - } - ON_EOF - { - log_parse_error(); - EMIT_EOF; - } - ANYTHING_ELSE - { - log_parse_error(); - RECONSUME_IN(BeforeAttributeName); - } - } - END_STATE - - // 13.2.5.33 Attribute name state, https://html.spec.whatwg.org/multipage/parsing.html#attribute-name-state - BEGIN_STATE(AttributeName) - { - ON_WHITESPACE - { - m_current_token.last_attribute().name_end_position = nth_last_position(1); - m_current_token.last_attribute().local_name = consume_current_builder(); - RECONSUME_IN(AfterAttributeName); - } - ON('/') - { - m_current_token.last_attribute().name_end_position = nth_last_position(1); - m_current_token.last_attribute().local_name = consume_current_builder(); - RECONSUME_IN(AfterAttributeName); - } - ON('>') - { - m_current_token.last_attribute().name_end_position = nth_last_position(1); - m_current_token.last_attribute().local_name = consume_current_builder(); - RECONSUME_IN(AfterAttributeName); - } - ON_EOF - { - m_current_token.last_attribute().name_end_position = nth_last_position(1); - m_current_token.last_attribute().local_name = consume_current_builder(); - RECONSUME_IN(AfterAttributeName); - } - ON('=') - { - m_current_token.last_attribute().name_end_position = nth_last_position(1); - m_current_token.last_attribute().local_name = consume_current_builder(); - SWITCH_TO(BeforeAttributeValue); - } - ON_ASCII_UPPER_ALPHA - { - m_current_builder.append_code_point(to_ascii_lowercase(current_input_character.value())); - continue; - } - ON(0) - { - log_parse_error(); - m_current_builder.append_code_point(0xFFFD); - continue; - } - ON('"') - { - log_parse_error(); - goto AnythingElseAttributeName; - } - ON('\'') - { - log_parse_error(); - goto AnythingElseAttributeName; - } - ON('<') - { - log_parse_error(); - goto AnythingElseAttributeName; - } - ANYTHING_ELSE - { - AnythingElseAttributeName: - m_current_builder.append_code_point(current_input_character.value()); - continue; - } - } - END_STATE - - // 13.2.5.34 After attribute name state, https://html.spec.whatwg.org/multipage/parsing.html#after-attribute-name-state - BEGIN_STATE(AfterAttributeName) - { - ON_WHITESPACE - { - continue; - } - ON('/') - { - SWITCH_TO(SelfClosingStartTag); - } - ON('=') - { - m_current_token.last_attribute().name_end_position = nth_last_position(1); - SWITCH_TO(BeforeAttributeValue); - } - ON('>') - { - SWITCH_TO_AND_EMIT_CURRENT_TOKEN(Data); - } - ON_EOF - { - log_parse_error(); - EMIT_EOF; - } - ANYTHING_ELSE - { - m_current_token.add_attribute({}); - if (!m_source_positions.is_empty()) - m_current_token.last_attribute().name_start_position = nth_last_position(1); - RECONSUME_IN(AttributeName); - } - } - END_STATE - - // 13.2.5.35 Before attribute value state, https://html.spec.whatwg.org/multipage/parsing.html#before-attribute-value-state - BEGIN_STATE(BeforeAttributeValue) - { - m_current_token.last_attribute().value_start_position = nth_last_position(1); - ON_WHITESPACE - { - continue; - } - ON('"') - { - SWITCH_TO(AttributeValueDoubleQuoted); - } - ON('\'') - { - SWITCH_TO(AttributeValueSingleQuoted); - } - ON('>') - { - log_parse_error(); - SWITCH_TO_AND_EMIT_CURRENT_TOKEN(Data); - } - ANYTHING_ELSE - { - RECONSUME_IN(AttributeValueUnquoted); - } - } - END_STATE - - // 13.2.5.36 Attribute value (double-quoted) state, https://html.spec.whatwg.org/multipage/parsing.html#attribute-value-(double-quoted)-state - BEGIN_STATE(AttributeValueDoubleQuoted) - { - ON('"') - { - m_current_token.last_attribute().value = consume_current_builder(); - SWITCH_TO(AfterAttributeValueQuoted); - } - ON('&') - { - m_return_state = State::AttributeValueDoubleQuoted; - SWITCH_TO_WITH_UNCLEAN_BUILDER(CharacterReference); - } - ON(0) - { - log_parse_error(); - m_current_builder.append_code_point(0xFFFD); - continue; - } - ON_EOF - { - log_parse_error(); - EMIT_EOF; - } - ANYTHING_ELSE - { - m_current_builder.append_code_point(current_input_character.value()); - continue; - } - } - END_STATE - - // 13.2.5.37 Attribute value (single-quoted) state, https://html.spec.whatwg.org/multipage/parsing.html#attribute-value-(single-quoted)-state - BEGIN_STATE(AttributeValueSingleQuoted) - { - ON('\'') - { - m_current_token.last_attribute().value = consume_current_builder(); - SWITCH_TO(AfterAttributeValueQuoted); - } - ON('&') - { - m_return_state = State::AttributeValueSingleQuoted; - SWITCH_TO_WITH_UNCLEAN_BUILDER(CharacterReference); - } - ON(0) - { - log_parse_error(); - m_current_builder.append_code_point(0xFFFD); - continue; - } - ON_EOF - { - log_parse_error(); - EMIT_EOF; - } - ANYTHING_ELSE - { - m_current_builder.append_code_point(current_input_character.value()); - continue; - } - } - END_STATE - - // 13.2.5.38 Attribute value (unquoted) state, https://html.spec.whatwg.org/multipage/parsing.html#attribute-value-(single-quoted)-state - BEGIN_STATE(AttributeValueUnquoted) - { - ON_WHITESPACE - { - m_current_token.last_attribute().value = consume_current_builder(); - m_current_token.last_attribute().value_end_position = nth_last_position(1); - SWITCH_TO(BeforeAttributeName); - } - ON('&') - { - m_return_state = State::AttributeValueUnquoted; - SWITCH_TO_WITH_UNCLEAN_BUILDER(CharacterReference); - } - ON('>') - { - m_current_token.last_attribute().value = consume_current_builder(); - m_current_token.last_attribute().value_end_position = nth_last_position(1); - SWITCH_TO_AND_EMIT_CURRENT_TOKEN(Data); - } - ON(0) - { - log_parse_error(); - m_current_builder.append_code_point(0xFFFD); - continue; - } - ON('"') - { - log_parse_error(); - goto AnythingElseAttributeValueUnquoted; - } - ON('\'') - { - log_parse_error(); - goto AnythingElseAttributeValueUnquoted; - } - ON('<') - { - log_parse_error(); - goto AnythingElseAttributeValueUnquoted; - } - ON('=') - { - log_parse_error(); - goto AnythingElseAttributeValueUnquoted; - } - ON('`') - { - log_parse_error(); - goto AnythingElseAttributeValueUnquoted; - } - ON_EOF - { - log_parse_error(); - EMIT_EOF; - } - ANYTHING_ELSE - { - AnythingElseAttributeValueUnquoted: - m_current_builder.append_code_point(current_input_character.value()); - continue; - } - } - END_STATE - - // 13.2.5.39 After attribute value (quoted) state, https://html.spec.whatwg.org/multipage/parsing.html#after-attribute-value-(quoted)-state - BEGIN_STATE(AfterAttributeValueQuoted) - { - m_current_token.last_attribute().value_end_position = nth_last_position(1); - ON_WHITESPACE - { - SWITCH_TO(BeforeAttributeName); - } - ON('/') - { - SWITCH_TO(SelfClosingStartTag); - } - ON('>') - { - SWITCH_TO_AND_EMIT_CURRENT_TOKEN(Data); - } - ON_EOF - { - log_parse_error(); - EMIT_EOF; - } - ANYTHING_ELSE - { - log_parse_error(); - RECONSUME_IN(BeforeAttributeName); - } - } - END_STATE - - // 13.2.5.43 Comment start state, https://html.spec.whatwg.org/multipage/parsing.html#comment-start-state - BEGIN_STATE(CommentStart) - { - ON('-') - { - SWITCH_TO(CommentStartDash); - } - ON('>') - { - log_parse_error(); - SWITCH_TO_AND_EMIT_CURRENT_TOKEN(Data); - } - ANYTHING_ELSE - { - RECONSUME_IN(Comment); - } - } - END_STATE - - // 13.2.5.44 Comment start dash state, https://html.spec.whatwg.org/multipage/parsing.html#comment-start-dash-state - BEGIN_STATE(CommentStartDash) - { - ON('-') - { - SWITCH_TO(CommentEnd); - } - ON('>') - { - log_parse_error(); - SWITCH_TO_AND_EMIT_CURRENT_TOKEN(Data); - } - ON_EOF - { - log_parse_error(); - EMIT_CURRENT_TOKEN_FOLLOWED_BY_EOF; - } - ANYTHING_ELSE - { - m_current_builder.append('-'); - RECONSUME_IN(Comment); - } - } - END_STATE - - // 13.2.5.45 Comment state, https://html.spec.whatwg.org/multipage/parsing.html#comment-state - BEGIN_STATE(Comment) - { - ON('<') - { - m_current_builder.append_code_point(current_input_character.value()); - SWITCH_TO_WITH_UNCLEAN_BUILDER(CommentLessThanSign); - } - ON('-') - { - SWITCH_TO_WITH_UNCLEAN_BUILDER(CommentEndDash); - } - ON(0) - { - log_parse_error(); - m_current_builder.append_code_point(0xFFFD); - continue; - } - ON_EOF - { - log_parse_error(); - m_current_token.set_comment(consume_current_builder()); - EMIT_CURRENT_TOKEN_FOLLOWED_BY_EOF; - } - ANYTHING_ELSE - { - m_current_builder.append_code_point(current_input_character.value()); - continue; - } - } - END_STATE - - // 13.2.5.51 Comment end state, https://html.spec.whatwg.org/multipage/parsing.html#comment-end-state - BEGIN_STATE(CommentEnd) - { - ON('>') - { - m_current_token.set_comment(consume_current_builder()); - SWITCH_TO_AND_EMIT_CURRENT_TOKEN(Data); - } - ON('!') - { - SWITCH_TO_WITH_UNCLEAN_BUILDER(CommentEndBang); - } - ON('-') - { - m_current_builder.append('-'); - continue; - } - ON_EOF - { - log_parse_error(); - m_current_token.set_comment(consume_current_builder()); - EMIT_CURRENT_TOKEN_FOLLOWED_BY_EOF; - } - ANYTHING_ELSE - { - m_current_builder.append("--"sv); - RECONSUME_IN(Comment); - } - } - END_STATE - - // 13.2.5.52 Comment end bang state, https://html.spec.whatwg.org/multipage/parsing.html#comment-end-bang-state - BEGIN_STATE(CommentEndBang) - { - ON('-') - { - m_current_builder.append("--!"sv); - SWITCH_TO_WITH_UNCLEAN_BUILDER(CommentEndDash); - } - ON('>') - { - log_parse_error(); - m_current_token.set_comment(consume_current_builder()); - SWITCH_TO_AND_EMIT_CURRENT_TOKEN(Data); - } - ON_EOF - { - log_parse_error(); - m_current_token.set_comment(consume_current_builder()); - EMIT_CURRENT_TOKEN_FOLLOWED_BY_EOF; - } - ANYTHING_ELSE - { - m_current_builder.append("--!"sv); - RECONSUME_IN(Comment); - } - } - END_STATE - - // 13.2.5.50 Comment end dash state, https://html.spec.whatwg.org/multipage/parsing.html#comment-end-dash-state - BEGIN_STATE(CommentEndDash) - { - ON('-') - { - SWITCH_TO_WITH_UNCLEAN_BUILDER(CommentEnd); - } - ON_EOF - { - log_parse_error(); - m_current_token.set_comment(consume_current_builder()); - EMIT_CURRENT_TOKEN_FOLLOWED_BY_EOF; - } - ANYTHING_ELSE - { - m_current_builder.append('-'); - RECONSUME_IN(Comment); - } - } - END_STATE - - // 13.2.5.46 Comment less-than sign state, https://html.spec.whatwg.org/multipage/parsing.html#comment-less-than-sign-state - BEGIN_STATE(CommentLessThanSign) - { - ON('!') - { - m_current_builder.append_code_point(current_input_character.value()); - SWITCH_TO_WITH_UNCLEAN_BUILDER(CommentLessThanSignBang); - } - ON('<') - { - m_current_builder.append_code_point(current_input_character.value()); - continue; - } - ANYTHING_ELSE - { - RECONSUME_IN(Comment); - } - } - END_STATE - - // 13.2.5.47 Comment less-than sign bang state, https://html.spec.whatwg.org/multipage/parsing.html#comment-less-than-sign-bang-state - BEGIN_STATE(CommentLessThanSignBang) - { - ON('-') - { - SWITCH_TO_WITH_UNCLEAN_BUILDER(CommentLessThanSignBangDash); - } - ANYTHING_ELSE - { - RECONSUME_IN(Comment); - } - } - END_STATE - - // 13.2.5.48 Comment less-than sign bang dash state, https://html.spec.whatwg.org/multipage/parsing.html#comment-less-than-sign-bang-dash-state - BEGIN_STATE(CommentLessThanSignBangDash) - { - ON('-') - { - SWITCH_TO_WITH_UNCLEAN_BUILDER(CommentLessThanSignBangDashDash); - } - ANYTHING_ELSE - { - RECONSUME_IN(CommentEndDash); - } - } - END_STATE - - // 13.2.5.49 Comment less-than sign bang dash dash state, https://html.spec.whatwg.org/multipage/parsing.html#comment-less-than-sign-bang-dash-dash-state - BEGIN_STATE(CommentLessThanSignBangDashDash) - { - ON('>') - { - RECONSUME_IN(CommentEnd); - } - ON_EOF - { - RECONSUME_IN(CommentEnd); - } - ANYTHING_ELSE - { - log_parse_error(); - RECONSUME_IN(CommentEnd); - } - } - END_STATE - - // 13.2.5.72 Character reference state, https://html.spec.whatwg.org/multipage/parsing.html#character-reference-state - BEGIN_STATE(CharacterReference) - { - m_temporary_buffer.clear(); - m_temporary_buffer.append('&'); - - ON_ASCII_ALPHANUMERIC - { - m_named_character_reference_matcher = {}; - RECONSUME_IN(NamedCharacterReference); - } - ON('#') - { - m_temporary_buffer.append(current_input_character.value()); - SWITCH_TO_WITH_UNCLEAN_BUILDER(NumericCharacterReference); - } - ANYTHING_ELSE - { - FLUSH_CODEPOINTS_CONSUMED_AS_A_CHARACTER_REFERENCE; - RECONSUME_IN_RETURN_STATE; - } - } - END_STATE - - // 13.2.5.73 Named character reference state, https://html.spec.whatwg.org/multipage/parsing.html#named-character-reference-state - BEGIN_STATE(NamedCharacterReference) - { - if (can_run_out_of_characters(stop_at_insertion_point)) { - // If there is an insertion point, match code-point-by-code-point to handle the possibility of - // document.write being used to insert a named character reference one-code-point-at-a-time. Do - // the same for an open input stream, since more network input can extend the reference later. - if (current_input_character.has_value()) { - if (m_named_character_reference_matcher.try_consume_code_point(current_input_character.value())) { - m_temporary_buffer.append(current_input_character.value()); - continue; - } else { - DONT_CONSUME_NEXT_INPUT_CHARACTER; - } - } - } else { - // If there's no insertion point (this is the common case), it is safe to look ahead at the rest - // of the input and try to match a named character reference all-at-once. This is worthwhile - // because matching all-at-once ends up being more efficient. - auto starting_consumed_count = m_temporary_buffer.size(); - auto remaining_source = m_decoded_input.span().slice(m_prev_offset); - - for (auto const code_point : remaining_source) { - if (m_named_character_reference_matcher.try_consume_code_point(code_point)) { - m_temporary_buffer.append(code_point); - } else { - break; - } - } - - auto num_consumed = m_temporary_buffer.size() - starting_consumed_count; - if (num_consumed == 0) { - DONT_CONSUME_NEXT_INPUT_CHARACTER; - } else if (num_consumed > 1) { - skip(num_consumed - 1); - } - } - - // Only consume the characters within the longest match. It's possible that we've overconsumed code points, - // though, so we want to backtrack to the longest match found. For example, `¬indo` (which could still - // have lead to `⋵̸`) would need to backtrack back to `¬`. - auto overconsumed_code_points = m_named_character_reference_matcher.overconsumed_code_points(); - if (overconsumed_code_points > 0) { - restore_to(m_current_offset - overconsumed_code_points); - m_temporary_buffer.resize_and_keep_capacity(m_temporary_buffer.size() - overconsumed_code_points); - } - - auto mapped_codepoints = m_named_character_reference_matcher.code_points(); - // If there is a match - if (mapped_codepoints.has_value()) { - if (consumed_as_part_of_an_attribute() && !m_named_character_reference_matcher.last_match_ends_with_semicolon()) { - auto next_code_point = peek_code_point(0, stop_at_insertion_point); - if (next_code_point.has_value() && (next_code_point.value() == '=' || is_ascii_alphanumeric(next_code_point.value()))) { - FLUSH_CODEPOINTS_CONSUMED_AS_A_CHARACTER_REFERENCE; - SWITCH_TO_RETURN_STATE; - } - } - - if (!m_named_character_reference_matcher.last_match_ends_with_semicolon()) { - log_parse_error(); - } - - m_temporary_buffer.clear_with_capacity(); - m_temporary_buffer.append(mapped_codepoints.value().first); - auto second_codepoint = named_character_reference_second_codepoint_value(mapped_codepoints.value().second); - if (second_codepoint.has_value()) { - m_temporary_buffer.append(second_codepoint.value()); - } - - FLUSH_CODEPOINTS_CONSUMED_AS_A_CHARACTER_REFERENCE; - SWITCH_TO_RETURN_STATE; - } else { - FLUSH_CODEPOINTS_CONSUMED_AS_A_CHARACTER_REFERENCE; - SWITCH_TO_WITH_UNCLEAN_BUILDER(AmbiguousAmpersand); - } - } - END_STATE - - // 13.2.5.74 Ambiguous ampersand state, https://html.spec.whatwg.org/multipage/parsing.html#ambiguous-ampersand-state - BEGIN_STATE(AmbiguousAmpersand) - { - ON_ASCII_ALPHANUMERIC - { - if (consumed_as_part_of_an_attribute()) { - m_current_builder.append_code_point(current_input_character.value()); - continue; - } else { - EMIT_CURRENT_CHARACTER; - } - } - ON(';') - { - log_parse_error(); - RECONSUME_IN_RETURN_STATE; - } - ANYTHING_ELSE - { - RECONSUME_IN_RETURN_STATE; - } - } - END_STATE - - // 13.2.5.75 Numeric character reference state, https://html.spec.whatwg.org/multipage/parsing.html#numeric-character-reference-state - BEGIN_STATE(NumericCharacterReference) - { - m_character_reference_code = 0; - - ON('X') - { - m_temporary_buffer.append(current_input_character.value()); - SWITCH_TO_WITH_UNCLEAN_BUILDER(HexadecimalCharacterReferenceStart); - } - ON('x') - { - m_temporary_buffer.append(current_input_character.value()); - SWITCH_TO_WITH_UNCLEAN_BUILDER(HexadecimalCharacterReferenceStart); - } - ANYTHING_ELSE - { - RECONSUME_IN(DecimalCharacterReferenceStart); - } - } - END_STATE - - // 13.2.5.76 Hexadecimal character reference start state, https://html.spec.whatwg.org/multipage/parsing.html#hexadecimal-character-reference-start-state - BEGIN_STATE(HexadecimalCharacterReferenceStart) - { - ON_ASCII_HEX_DIGIT - { - RECONSUME_IN(HexadecimalCharacterReference); - } - ANYTHING_ELSE - { - log_parse_error(); - FLUSH_CODEPOINTS_CONSUMED_AS_A_CHARACTER_REFERENCE; - RECONSUME_IN_RETURN_STATE; - } - } - END_STATE - - // 13.2.5.77 Decimal character reference start state, https://html.spec.whatwg.org/multipage/parsing.html#decimal-character-reference-start-state - BEGIN_STATE(DecimalCharacterReferenceStart) - { - ON_ASCII_DIGIT - { - RECONSUME_IN(DecimalCharacterReference); - } - ANYTHING_ELSE - { - log_parse_error(); - FLUSH_CODEPOINTS_CONSUMED_AS_A_CHARACTER_REFERENCE; - RECONSUME_IN_RETURN_STATE; - } - } - END_STATE - - // 13.2.5.78 Hexadecimal character reference state, https://html.spec.whatwg.org/multipage/parsing.html#decimal-character-reference-start-state - BEGIN_STATE(HexadecimalCharacterReference) - { - ON_ASCII_DIGIT - { - m_character_reference_code *= 16; - m_character_reference_code += current_input_character.value() - 0x30; - continue; - } - ON_ASCII_HEX_DIGIT - { - m_character_reference_code *= 16; - auto hex_digit_min_ascii_value = is_ascii_upper_alpha(current_input_character.value()) ? 0x37 : 0x57; - m_character_reference_code += current_input_character.value() - hex_digit_min_ascii_value; - continue; - } - ON(';') - { - SWITCH_TO_WITH_UNCLEAN_BUILDER(NumericCharacterReferenceEnd); - } - ANYTHING_ELSE - { - log_parse_error(); - RECONSUME_IN(NumericCharacterReferenceEnd); - } - } - END_STATE - - // 13.2.5.79 Decimal character reference state, https://html.spec.whatwg.org/multipage/parsing.html#decimal-character-reference-state - BEGIN_STATE(DecimalCharacterReference) - { - ON_ASCII_DIGIT - { - m_character_reference_code *= 10; - m_character_reference_code += current_input_character.value() - 0x30; - continue; - } - ON(';') - { - SWITCH_TO_WITH_UNCLEAN_BUILDER(NumericCharacterReferenceEnd); - } - ANYTHING_ELSE - { - log_parse_error(); - RECONSUME_IN(NumericCharacterReferenceEnd); - } - } - END_STATE - - // 13.2.5.80 Numeric character reference end state, https://html.spec.whatwg.org/multipage/parsing.html#numeric-character-reference-end-state - BEGIN_STATE(NumericCharacterReferenceEnd) - { - DONT_CONSUME_NEXT_INPUT_CHARACTER; - - if (m_character_reference_code == 0) { - log_parse_error(); - m_character_reference_code = 0xFFFD; - } - if (m_character_reference_code > 0x10ffff) { - log_parse_error(); - m_character_reference_code = 0xFFFD; - } - if (is_unicode_surrogate(m_character_reference_code)) { - log_parse_error(); - m_character_reference_code = 0xFFFD; - } - if (is_unicode_noncharacter(m_character_reference_code)) { - log_parse_error(); - } - if (m_character_reference_code == 0xd || (is_unicode_control(m_character_reference_code) && !is_ascii_space(m_character_reference_code))) { - log_parse_error(); - constexpr struct { - u32 number; - u32 code_point; - } conversion_table[] = { - { 0x80, 0x20AC }, - { 0x82, 0x201A }, - { 0x83, 0x0192 }, - { 0x84, 0x201E }, - { 0x85, 0x2026 }, - { 0x86, 0x2020 }, - { 0x87, 0x2021 }, - { 0x88, 0x02C6 }, - { 0x89, 0x2030 }, - { 0x8A, 0x0160 }, - { 0x8B, 0x2039 }, - { 0x8C, 0x0152 }, - { 0x8E, 0x017D }, - { 0x91, 0x2018 }, - { 0x92, 0x2019 }, - { 0x93, 0x201C }, - { 0x94, 0x201D }, - { 0x95, 0x2022 }, - { 0x96, 0x2013 }, - { 0x97, 0x2014 }, - { 0x98, 0x02DC }, - { 0x99, 0x2122 }, - { 0x9A, 0x0161 }, - { 0x9B, 0x203A }, - { 0x9C, 0x0153 }, - { 0x9E, 0x017E }, - { 0x9F, 0x0178 }, - }; - for (auto& entry : conversion_table) { - if (m_character_reference_code == entry.number) { - m_character_reference_code = entry.code_point; - break; - } - } - } - - m_temporary_buffer.clear(); - m_temporary_buffer.append(m_character_reference_code); - FLUSH_CODEPOINTS_CONSUMED_AS_A_CHARACTER_REFERENCE; - SWITCH_TO_RETURN_STATE; - } - END_STATE - - // 13.2.5.2 RCDATA state, https://html.spec.whatwg.org/multipage/parsing.html#rcdata-state - BEGIN_STATE(RCDATA) - { - ON('&') - { - m_return_state = State::RCDATA; - SWITCH_TO(CharacterReference); - } - ON('<') - { - SWITCH_TO(RCDATALessThanSign); - } - ON(0) - { - log_parse_error(); - EMIT_CHARACTER(0xFFFD); - } - ON_EOF - { - EMIT_EOF; - } - ANYTHING_ELSE - { - EMIT_CURRENT_CHARACTER; - } - } - END_STATE - - // 13.2.5.9 RCDATA less-than sign state, https://html.spec.whatwg.org/multipage/parsing.html#rcdata-less-than-sign-state - BEGIN_STATE(RCDATALessThanSign) - { - ON('/') - { - m_temporary_buffer.clear(); - SWITCH_TO(RCDATAEndTagOpen); - } - ANYTHING_ELSE - { - EMIT_CHARACTER_AND_RECONSUME_IN('<', RCDATA); - } - } - END_STATE - - // 13.2.5.10 RCDATA end tag open state, https://html.spec.whatwg.org/multipage/parsing.html#rcdata-end-tag-open-state - BEGIN_STATE(RCDATAEndTagOpen) - { - ON_ASCII_ALPHA - { - create_new_token(HTMLToken::Type::EndTag); - RECONSUME_IN(RCDATAEndTagName); - } - ANYTHING_ELSE - { - m_queued_tokens.enqueue(HTMLToken::make_character('<')); - m_queued_tokens.enqueue(HTMLToken::make_character('/')); - RECONSUME_IN(RCDATA); - } - } - END_STATE - - // 13.2.5.11 RCDATA end tag name state, https://html.spec.whatwg.org/multipage/parsing.html#rcdata-end-tag-name-state - BEGIN_STATE(RCDATAEndTagName) - { - ON_WHITESPACE - { - m_current_token.set_tag_name(consume_current_builder()); - if (!current_end_tag_token_is_appropriate()) { - m_queued_tokens.enqueue(HTMLToken::make_character('<')); - m_queued_tokens.enqueue(HTMLToken::make_character('/')); - for (auto code_point : m_temporary_buffer) - m_queued_tokens.enqueue(HTMLToken::make_character(code_point)); - RECONSUME_IN(RCDATA); - } - SWITCH_TO(BeforeAttributeName); - } - ON('/') - { - m_current_token.set_tag_name(consume_current_builder()); - if (!current_end_tag_token_is_appropriate()) { - m_queued_tokens.enqueue(HTMLToken::make_character('<')); - m_queued_tokens.enqueue(HTMLToken::make_character('/')); - for (auto code_point : m_temporary_buffer) - m_queued_tokens.enqueue(HTMLToken::make_character(code_point)); - RECONSUME_IN(RCDATA); - } - SWITCH_TO(SelfClosingStartTag); - } - ON('>') - { - m_current_token.set_tag_name(consume_current_builder()); - if (!current_end_tag_token_is_appropriate()) { - m_queued_tokens.enqueue(HTMLToken::make_character('<')); - m_queued_tokens.enqueue(HTMLToken::make_character('/')); - for (auto code_point : m_temporary_buffer) - m_queued_tokens.enqueue(HTMLToken::make_character(code_point)); - RECONSUME_IN(RCDATA); - } - SWITCH_TO_AND_EMIT_CURRENT_TOKEN(Data); - } - ON_ASCII_UPPER_ALPHA - { - m_current_builder.append_code_point(to_ascii_lowercase(current_input_character.value())); - m_temporary_buffer.append(current_input_character.value()); - continue; - } - ON_ASCII_LOWER_ALPHA - { - m_current_builder.append_code_point(current_input_character.value()); - m_temporary_buffer.append(current_input_character.value()); - continue; - } - ANYTHING_ELSE - { - m_queued_tokens.enqueue(HTMLToken::make_character('<')); - m_queued_tokens.enqueue(HTMLToken::make_character('/')); - // NOTE: The spec doesn't mention this, but it seems that m_current_token (an end tag) is just dropped in this case. - m_current_builder.clear(); - for (auto code_point : m_temporary_buffer) - m_queued_tokens.enqueue(HTMLToken::make_character(code_point)); - RECONSUME_IN(RCDATA); - } - } - END_STATE - - // 13.2.5.3 RAWTEXT state, https://html.spec.whatwg.org/multipage/parsing.html#rawtext-state - BEGIN_STATE(RAWTEXT) - { - ON('<') - { - SWITCH_TO(RAWTEXTLessThanSign); - } - ON(0) - { - log_parse_error(); - EMIT_CHARACTER(0xFFFD); - } - ON_EOF - { - EMIT_EOF; - } - ANYTHING_ELSE - { - EMIT_CURRENT_CHARACTER; - } - } - END_STATE - - // 13.2.5.12 RAWTEXT less-than sign state, https://html.spec.whatwg.org/multipage/parsing.html#rawtext-less-than-sign-state - BEGIN_STATE(RAWTEXTLessThanSign) - { - ON('/') - { - m_temporary_buffer.clear(); - SWITCH_TO(RAWTEXTEndTagOpen); - } - ANYTHING_ELSE - { - EMIT_CHARACTER_AND_RECONSUME_IN('<', RAWTEXT); - } - } - END_STATE - - // 13.2.5.13 RAWTEXT end tag open state, https://html.spec.whatwg.org/multipage/parsing.html#rawtext-end-tag-open-state - BEGIN_STATE(RAWTEXTEndTagOpen) - { - ON_ASCII_ALPHA - { - create_new_token(HTMLToken::Type::EndTag); - RECONSUME_IN(RAWTEXTEndTagName); - } - ANYTHING_ELSE - { - m_queued_tokens.enqueue(HTMLToken::make_character('<')); - m_queued_tokens.enqueue(HTMLToken::make_character('/')); - RECONSUME_IN(RAWTEXT); - } - } - END_STATE - - // 13.2.5.14 RAWTEXT end tag name state, https://html.spec.whatwg.org/multipage/parsing.html#rawtext-end-tag-name-state - BEGIN_STATE(RAWTEXTEndTagName) - { - ON_WHITESPACE - { - m_current_token.set_tag_name(consume_current_builder()); - if (!current_end_tag_token_is_appropriate()) { - m_queued_tokens.enqueue(HTMLToken::make_character('<')); - m_queued_tokens.enqueue(HTMLToken::make_character('/')); - for (auto code_point : m_temporary_buffer) - m_queued_tokens.enqueue(HTMLToken::make_character(code_point)); - RECONSUME_IN(RAWTEXT); - } - SWITCH_TO(BeforeAttributeName); - } - ON('/') - { - m_current_token.set_tag_name(consume_current_builder()); - if (!current_end_tag_token_is_appropriate()) { - m_queued_tokens.enqueue(HTMLToken::make_character('<')); - m_queued_tokens.enqueue(HTMLToken::make_character('/')); - for (auto code_point : m_temporary_buffer) - m_queued_tokens.enqueue(HTMLToken::make_character(code_point)); - RECONSUME_IN(RAWTEXT); - } - SWITCH_TO(SelfClosingStartTag); - } - ON('>') - { - m_current_token.set_tag_name(consume_current_builder()); - if (!current_end_tag_token_is_appropriate()) { - m_queued_tokens.enqueue(HTMLToken::make_character('<')); - m_queued_tokens.enqueue(HTMLToken::make_character('/')); - for (auto code_point : m_temporary_buffer) - m_queued_tokens.enqueue(HTMLToken::make_character(code_point)); - RECONSUME_IN(RAWTEXT); - } - SWITCH_TO_AND_EMIT_CURRENT_TOKEN(Data); - } - ON_ASCII_UPPER_ALPHA - { - m_current_builder.append_code_point(to_ascii_lowercase(current_input_character.value())); - m_temporary_buffer.append(current_input_character.value()); - continue; - } - ON_ASCII_LOWER_ALPHA - { - m_current_builder.append(current_input_character.value()); - m_temporary_buffer.append(current_input_character.value()); - continue; - } - ANYTHING_ELSE - { - m_queued_tokens.enqueue(HTMLToken::make_character('<')); - m_queued_tokens.enqueue(HTMLToken::make_character('/')); - // NOTE: The spec doesn't mention this, but it seems that m_current_token (an end tag) is just dropped in this case. - m_current_builder.clear(); - for (auto code_point : m_temporary_buffer) - m_queued_tokens.enqueue(HTMLToken::make_character(code_point)); - RECONSUME_IN(RAWTEXT); - } - } - END_STATE - - // 13.2.5.4 Script data state, https://html.spec.whatwg.org/multipage/parsing.html#script-data-state - BEGIN_STATE(ScriptData) - { - ON('<') - { - SWITCH_TO(ScriptDataLessThanSign); - } - ON(0) - { - log_parse_error(); - EMIT_CHARACTER(0xFFFD); - } - ON_EOF - { - EMIT_EOF; - } - ANYTHING_ELSE - { - EMIT_CURRENT_CHARACTER; - } - } - END_STATE - - // 13.2.5.5 PLAINTEXT state, https://html.spec.whatwg.org/multipage/parsing.html#plaintext-state - BEGIN_STATE(PLAINTEXT) - { - ON(0) - { - log_parse_error(); - EMIT_CHARACTER(0xFFFD); - } - ON_EOF - { - EMIT_EOF; - } - ANYTHING_ELSE - { - EMIT_CURRENT_CHARACTER; - } - } - END_STATE - - // 13.2.5.15 Script data less-than sign state, https://html.spec.whatwg.org/multipage/parsing.html#script-data-less-than-sign-state - BEGIN_STATE(ScriptDataLessThanSign) - { - ON('/') - { - m_temporary_buffer.clear(); - SWITCH_TO(ScriptDataEndTagOpen); - } - ON('!') - { - m_queued_tokens.enqueue(HTMLToken::make_character('<')); - m_queued_tokens.enqueue(HTMLToken::make_character('!')); - SWITCH_TO(ScriptDataEscapeStart); - } - ANYTHING_ELSE - { - EMIT_CHARACTER_AND_RECONSUME_IN('<', ScriptData); - } - } - END_STATE - - // 13.2.5.18 Script data escape start state, https://html.spec.whatwg.org/multipage/parsing.html#script-data-escape-start-state - BEGIN_STATE(ScriptDataEscapeStart) - { - ON('-') - { - SWITCH_TO_AND_EMIT_CHARACTER('-', ScriptDataEscapeStartDash); - } - ANYTHING_ELSE - { - RECONSUME_IN(ScriptData); - } - } - END_STATE - - // 13.2.5.19 Script data escape start dash state, https://html.spec.whatwg.org/multipage/parsing.html#script-data-escape-start-dash-state - BEGIN_STATE(ScriptDataEscapeStartDash) - { - ON('-') - { - SWITCH_TO_AND_EMIT_CHARACTER('-', ScriptDataEscapedDashDash); - } - ANYTHING_ELSE - { - RECONSUME_IN(ScriptData); - } - } - END_STATE - - // 13.2.5.22 Script data escaped dash dash state, https://html.spec.whatwg.org/multipage/parsing.html#script-data-escaped-dash-dash-state - BEGIN_STATE(ScriptDataEscapedDashDash) - { - ON('-') - { - EMIT_CHARACTER('-'); - } - ON('<') - { - SWITCH_TO(ScriptDataEscapedLessThanSign); - } - ON('>') - { - SWITCH_TO_AND_EMIT_CHARACTER('>', ScriptData); - } - ON(0) - { - log_parse_error(); - SWITCH_TO_AND_EMIT_CHARACTER(0xFFFD, ScriptDataEscaped); - } - ON_EOF - { - log_parse_error(); - EMIT_EOF; - } - ANYTHING_ELSE - { - SWITCH_TO_AND_EMIT_CURRENT_CHARACTER(ScriptDataEscaped); - } - } - END_STATE - - // 13.2.5.23 Script data escaped less-than sign state, https://html.spec.whatwg.org/multipage/parsing.html#script-data-escaped-less-than-sign-state - BEGIN_STATE(ScriptDataEscapedLessThanSign) - { - ON('/') - { - m_temporary_buffer.clear(); - SWITCH_TO(ScriptDataEscapedEndTagOpen); - } - ON_ASCII_ALPHA - { - m_temporary_buffer.clear(); - EMIT_CHARACTER_AND_RECONSUME_IN('<', ScriptDataDoubleEscapeStart); - } - ANYTHING_ELSE - { - EMIT_CHARACTER_AND_RECONSUME_IN('<', ScriptDataEscaped); - } - } - END_STATE - - // 13.2.5.24 Script data escaped end tag open state, https://html.spec.whatwg.org/multipage/parsing.html#script-data-escaped-end-tag-open-state - BEGIN_STATE(ScriptDataEscapedEndTagOpen) - { - ON_ASCII_ALPHA - { - create_new_token(HTMLToken::Type::EndTag); - RECONSUME_IN(ScriptDataEscapedEndTagName); - } - ANYTHING_ELSE - { - m_queued_tokens.enqueue(HTMLToken::make_character('<')); - m_queued_tokens.enqueue(HTMLToken::make_character('/')); - RECONSUME_IN(ScriptDataEscaped); - } - } - END_STATE - - // 13.2.5.25 Script data escaped end tag name state, https://html.spec.whatwg.org/multipage/parsing.html#script-data-escaped-end-tag-name-state - BEGIN_STATE(ScriptDataEscapedEndTagName) - { - ON_WHITESPACE - { - m_current_token.set_tag_name(consume_current_builder()); - if (current_end_tag_token_is_appropriate()) - SWITCH_TO(BeforeAttributeName); - - m_queued_tokens.enqueue(HTMLToken::make_character('<')); - m_queued_tokens.enqueue(HTMLToken::make_character('/')); - // NOTE: The spec doesn't mention this, but it seems that m_current_token (an end tag) is just dropped in this case. - m_current_builder.clear(); - for (auto code_point : m_temporary_buffer) { - m_queued_tokens.enqueue(HTMLToken::make_character(code_point)); - } - RECONSUME_IN(ScriptDataEscaped); - } - ON('/') - { - m_current_token.set_tag_name(consume_current_builder()); - if (current_end_tag_token_is_appropriate()) - SWITCH_TO(SelfClosingStartTag); - - m_queued_tokens.enqueue(HTMLToken::make_character('<')); - m_queued_tokens.enqueue(HTMLToken::make_character('/')); - // NOTE: The spec doesn't mention this, but it seems that m_current_token (an end tag) is just dropped in this case. - m_current_builder.clear(); - for (auto code_point : m_temporary_buffer) { - m_queued_tokens.enqueue(HTMLToken::make_character(code_point)); - } - RECONSUME_IN(ScriptDataEscaped); - } - ON('>') - { - m_current_token.set_tag_name(consume_current_builder()); - if (current_end_tag_token_is_appropriate()) - SWITCH_TO_AND_EMIT_CURRENT_TOKEN(Data); - - m_queued_tokens.enqueue(HTMLToken::make_character('<')); - m_queued_tokens.enqueue(HTMLToken::make_character('/')); - // NOTE: The spec doesn't mention this, but it seems that m_current_token (an end tag) is just dropped in this case. - m_current_builder.clear(); - for (auto code_point : m_temporary_buffer) { - m_queued_tokens.enqueue(HTMLToken::make_character(code_point)); - } - RECONSUME_IN(ScriptDataEscaped); - } - ON_ASCII_UPPER_ALPHA - { - m_current_builder.append_code_point(to_ascii_lowercase(current_input_character.value())); - m_temporary_buffer.append(current_input_character.value()); - continue; - } - ON_ASCII_LOWER_ALPHA - { - m_current_builder.append(current_input_character.value()); - m_temporary_buffer.append(current_input_character.value()); - continue; - } - ANYTHING_ELSE - { - m_queued_tokens.enqueue(HTMLToken::make_character('<')); - m_queued_tokens.enqueue(HTMLToken::make_character('/')); - // NOTE: The spec doesn't mention this, but it seems that m_current_token (an end tag) is just dropped in this case. - m_current_builder.clear(); - for (auto code_point : m_temporary_buffer) { - m_queued_tokens.enqueue(HTMLToken::make_character(code_point)); - } - RECONSUME_IN(ScriptDataEscaped); - } - } - END_STATE - - // 13.2.5.26 Script data double escape start state, https://html.spec.whatwg.org/multipage/parsing.html#script-data-double-escape-start-state - BEGIN_STATE(ScriptDataDoubleEscapeStart) - { - auto temporary_buffer_equal_to_script = [this]() -> bool { - if (m_temporary_buffer.size() != 6) - return false; - - // FIXME: Is there a better way of doing this? - return m_temporary_buffer[0] == 's' && m_temporary_buffer[1] == 'c' && m_temporary_buffer[2] == 'r' && m_temporary_buffer[3] == 'i' && m_temporary_buffer[4] == 'p' && m_temporary_buffer[5] == 't'; - }; - ON_WHITESPACE - { - if (temporary_buffer_equal_to_script()) - SWITCH_TO_AND_EMIT_CURRENT_CHARACTER(ScriptDataDoubleEscaped); - else - SWITCH_TO_AND_EMIT_CURRENT_CHARACTER(ScriptDataEscaped); - } - ON('/') - { - if (temporary_buffer_equal_to_script()) - SWITCH_TO_AND_EMIT_CURRENT_CHARACTER(ScriptDataDoubleEscaped); - else - SWITCH_TO_AND_EMIT_CURRENT_CHARACTER(ScriptDataEscaped); - } - ON('>') - { - if (temporary_buffer_equal_to_script()) - SWITCH_TO_AND_EMIT_CURRENT_CHARACTER(ScriptDataDoubleEscaped); - else - SWITCH_TO_AND_EMIT_CURRENT_CHARACTER(ScriptDataEscaped); - } - ON_ASCII_UPPER_ALPHA - { - m_temporary_buffer.append(to_ascii_lowercase(current_input_character.value())); - EMIT_CURRENT_CHARACTER; - } - ON_ASCII_LOWER_ALPHA - { - m_temporary_buffer.append(current_input_character.value()); - EMIT_CURRENT_CHARACTER; - } - ANYTHING_ELSE - { - RECONSUME_IN(ScriptDataEscaped); - } - } - END_STATE - - // 13.2.5.27 Script data double escaped state, https://html.spec.whatwg.org/multipage/parsing.html#script-data-double-escaped-state - BEGIN_STATE(ScriptDataDoubleEscaped) - { - ON('-') - { - SWITCH_TO_AND_EMIT_CHARACTER('-', ScriptDataDoubleEscapedDash); - } - ON('<') - { - SWITCH_TO_AND_EMIT_CHARACTER('<', ScriptDataDoubleEscapedLessThanSign); - } - ON(0) - { - log_parse_error(); - EMIT_CHARACTER(0xFFFD); - } - ON_EOF - { - log_parse_error(); - EMIT_EOF; - } - ANYTHING_ELSE - { - EMIT_CURRENT_CHARACTER; - } - } - END_STATE - - // 13.2.5.28 Script data double escaped dash state, https://html.spec.whatwg.org/multipage/parsing.html#script-data-double-escaped-dash-state - BEGIN_STATE(ScriptDataDoubleEscapedDash) - { - ON('-') - { - SWITCH_TO_AND_EMIT_CHARACTER('-', ScriptDataDoubleEscapedDashDash); - } - ON('<') - { - SWITCH_TO_AND_EMIT_CHARACTER('<', ScriptDataDoubleEscapedLessThanSign); - } - ON(0) - { - log_parse_error(); - SWITCH_TO_AND_EMIT_CHARACTER(0xFFFD, ScriptDataDoubleEscaped); - } - ON_EOF - { - log_parse_error(); - EMIT_EOF; - } - ANYTHING_ELSE - { - SWITCH_TO_AND_EMIT_CURRENT_CHARACTER(ScriptDataDoubleEscaped); - } - } - END_STATE - - // 13.2.5.29 Script data double escaped dash dash state, https://html.spec.whatwg.org/multipage/parsing.html#script-data-double-escaped-dash-dash-state - BEGIN_STATE(ScriptDataDoubleEscapedDashDash) - { - ON('-') - { - EMIT_CHARACTER('-'); - } - ON('<') - { - SWITCH_TO_AND_EMIT_CHARACTER('<', ScriptDataDoubleEscapedLessThanSign); - } - ON('>') - { - SWITCH_TO_AND_EMIT_CHARACTER('>', ScriptData); - } - ON(0) - { - log_parse_error(); - SWITCH_TO_AND_EMIT_CHARACTER(0xFFFD, ScriptDataDoubleEscaped); - } - ON_EOF - { - log_parse_error(); - EMIT_EOF; - } - ANYTHING_ELSE - { - SWITCH_TO_AND_EMIT_CURRENT_CHARACTER(ScriptDataDoubleEscaped); - } - } - END_STATE - - // 13.2.5.30 Script data double escaped less-than sign state, https://html.spec.whatwg.org/multipage/parsing.html#script-data-double-escaped-less-than-sign-state - BEGIN_STATE(ScriptDataDoubleEscapedLessThanSign) - { - ON('/') - { - m_temporary_buffer.clear(); - SWITCH_TO_AND_EMIT_CHARACTER('/', ScriptDataDoubleEscapeEnd); - } - ANYTHING_ELSE - { - RECONSUME_IN(ScriptDataDoubleEscaped); - } - } - END_STATE - - // 13.2.5.31 Script data double escape end state, https://html.spec.whatwg.org/multipage/parsing.html#script-data-double-escape-end-state - BEGIN_STATE(ScriptDataDoubleEscapeEnd) - { - auto temporary_buffer_equal_to_script = [this]() -> bool { - if (m_temporary_buffer.size() != 6) - return false; - - // FIXME: Is there a better way of doing this? - return m_temporary_buffer[0] == 's' && m_temporary_buffer[1] == 'c' && m_temporary_buffer[2] == 'r' && m_temporary_buffer[3] == 'i' && m_temporary_buffer[4] == 'p' && m_temporary_buffer[5] == 't'; - }; - ON_WHITESPACE - { - if (temporary_buffer_equal_to_script()) - SWITCH_TO_AND_EMIT_CURRENT_CHARACTER(ScriptDataEscaped); - else - SWITCH_TO_AND_EMIT_CURRENT_CHARACTER(ScriptDataDoubleEscaped); - } - ON('/') - { - if (temporary_buffer_equal_to_script()) - SWITCH_TO_AND_EMIT_CURRENT_CHARACTER(ScriptDataEscaped); - else - SWITCH_TO_AND_EMIT_CURRENT_CHARACTER(ScriptDataDoubleEscaped); - } - ON('>') - { - if (temporary_buffer_equal_to_script()) - SWITCH_TO_AND_EMIT_CURRENT_CHARACTER(ScriptDataEscaped); - else - SWITCH_TO_AND_EMIT_CURRENT_CHARACTER(ScriptDataDoubleEscaped); - } - ON_ASCII_UPPER_ALPHA - { - m_temporary_buffer.append(to_ascii_lowercase(current_input_character.value())); - EMIT_CURRENT_CHARACTER; - } - ON_ASCII_LOWER_ALPHA - { - m_temporary_buffer.append(current_input_character.value()); - EMIT_CURRENT_CHARACTER; - } - ANYTHING_ELSE - { - RECONSUME_IN(ScriptDataDoubleEscaped); - } - } - END_STATE - - // 13.2.5.21 Script data escaped dash state, https://html.spec.whatwg.org/multipage/parsing.html#script-data-escaped-dash-state - BEGIN_STATE(ScriptDataEscapedDash) - { - ON('-') - { - SWITCH_TO_AND_EMIT_CHARACTER('-', ScriptDataEscapedDashDash); - } - ON('<') - { - SWITCH_TO(ScriptDataEscapedLessThanSign); - } - ON(0) - { - log_parse_error(); - SWITCH_TO_AND_EMIT_CHARACTER(0xFFFD, ScriptDataEscaped); - } - ON_EOF - { - log_parse_error(); - EMIT_EOF; - } - ANYTHING_ELSE - { - SWITCH_TO_AND_EMIT_CURRENT_CHARACTER(ScriptDataEscaped); - } - } - END_STATE - - // 13.2.5.20 Script data escaped state, https://html.spec.whatwg.org/multipage/parsing.html#script-data-escaped-state - BEGIN_STATE(ScriptDataEscaped) - { - ON('-') - { - SWITCH_TO_AND_EMIT_CHARACTER('-', ScriptDataEscapedDash); - } - ON('<') - { - SWITCH_TO(ScriptDataEscapedLessThanSign); - } - ON(0) - { - log_parse_error(); - EMIT_CHARACTER(0xFFFD); - } - ON_EOF - { - log_parse_error(); - EMIT_EOF; - } - ANYTHING_ELSE - { - EMIT_CURRENT_CHARACTER; - } - } - END_STATE - - // 13.2.5.16 Script data end tag open state, https://html.spec.whatwg.org/multipage/parsing.html#script-data-end-tag-open-state - BEGIN_STATE(ScriptDataEndTagOpen) - { - ON_ASCII_ALPHA - { - create_new_token(HTMLToken::Type::EndTag); - RECONSUME_IN(ScriptDataEndTagName); - } - ANYTHING_ELSE - { - m_queued_tokens.enqueue(HTMLToken::make_character('<')); - m_queued_tokens.enqueue(HTMLToken::make_character('/')); - RECONSUME_IN(ScriptData); - } - } - END_STATE - - // 13.2.5.17 Script data end tag name state, https://html.spec.whatwg.org/multipage/parsing.html#script-data-end-tag-name-state - BEGIN_STATE(ScriptDataEndTagName) - { - ON_WHITESPACE - { - m_current_token.set_tag_name(consume_current_builder()); - if (current_end_tag_token_is_appropriate()) - SWITCH_TO(BeforeAttributeName); - m_queued_tokens.enqueue(HTMLToken::make_character('<')); - m_queued_tokens.enqueue(HTMLToken::make_character('/')); - // NOTE: The spec doesn't mention this, but it seems that m_current_token (an end tag) is just dropped in this case. - m_current_builder.clear(); - for (auto code_point : m_temporary_buffer) - m_queued_tokens.enqueue(HTMLToken::make_character(code_point)); - RECONSUME_IN(ScriptData); - } - ON('/') - { - m_current_token.set_tag_name(consume_current_builder()); - if (current_end_tag_token_is_appropriate()) - SWITCH_TO(SelfClosingStartTag); - m_queued_tokens.enqueue(HTMLToken::make_character('<')); - m_queued_tokens.enqueue(HTMLToken::make_character('/')); - // NOTE: The spec doesn't mention this, but it seems that m_current_token (an end tag) is just dropped in this case. - m_current_builder.clear(); - for (auto code_point : m_temporary_buffer) - m_queued_tokens.enqueue(HTMLToken::make_character(code_point)); - RECONSUME_IN(ScriptData); - } - ON('>') - { - m_current_token.set_tag_name(consume_current_builder()); - if (current_end_tag_token_is_appropriate()) - SWITCH_TO_AND_EMIT_CURRENT_TOKEN(Data); - m_queued_tokens.enqueue(HTMLToken::make_character('<')); - m_queued_tokens.enqueue(HTMLToken::make_character('/')); - // NOTE: The spec doesn't mention this, but it seems that m_current_token (an end tag) is just dropped in this case. - m_current_builder.clear(); - for (auto code_point : m_temporary_buffer) - m_queued_tokens.enqueue(HTMLToken::make_character(code_point)); - RECONSUME_IN(ScriptData); - } - ON_ASCII_UPPER_ALPHA - { - m_current_builder.append_code_point(to_ascii_lowercase(current_input_character.value())); - m_temporary_buffer.append(current_input_character.value()); - continue; - } - ON_ASCII_LOWER_ALPHA - { - m_current_builder.append(current_input_character.value()); - m_temporary_buffer.append(current_input_character.value()); - continue; - } - ANYTHING_ELSE - { - m_queued_tokens.enqueue(HTMLToken::make_character('<')); - m_queued_tokens.enqueue(HTMLToken::make_character('/')); - // NOTE: The spec doesn't mention this, but it seems that m_current_token (an end tag) is just dropped in this case. - m_current_builder.clear(); - for (auto code_point : m_temporary_buffer) - m_queued_tokens.enqueue(HTMLToken::make_character(code_point)); - RECONSUME_IN(ScriptData); - } - } - END_STATE - - // 13.2.5.69 CDATA section state, https://html.spec.whatwg.org/multipage/parsing.html#cdata-section-state - BEGIN_STATE(CDATASection) - { - ON(']') - { - SWITCH_TO(CDATASectionBracket); - } - ON_EOF - { - log_parse_error(); - EMIT_EOF; - } - ANYTHING_ELSE - { - EMIT_CURRENT_CHARACTER; - } - } - END_STATE - - // 13.2.5.70 CDATA section bracket state, https://html.spec.whatwg.org/multipage/parsing.html#cdata-section-bracket-state - BEGIN_STATE(CDATASectionBracket) - { - ON(']') - { - SWITCH_TO(CDATASectionEnd); - } - ANYTHING_ELSE - { - EMIT_CHARACTER_AND_RECONSUME_IN(']', CDATASection); - } - } - END_STATE - - // 13.2.5.71 CDATA section end state, https://html.spec.whatwg.org/multipage/parsing.html#cdata-section-end-state - BEGIN_STATE(CDATASectionEnd) - { - ON(']') - { - EMIT_CHARACTER(']'); - } - ON('>') - { - SWITCH_TO(Data); - } - ANYTHING_ELSE - { - m_queued_tokens.enqueue(HTMLToken::make_character(']')); - m_queued_tokens.enqueue(HTMLToken::make_character(']')); - RECONSUME_IN(CDATASection); - } - } - END_STATE - - default: - TODO(); - } - } -} - -HTMLTokenizer::ConsumeNextResult HTMLTokenizer::consume_next_if_match(StringView string, StopAtInsertionPoint stop_at_insertion_point, CaseSensitivity case_sensitivity) -{ - for (size_t i = 0; i < string.length(); ++i) { - auto code_point = peek_code_point(i, stop_at_insertion_point); - if (!code_point.has_value()) { - if (can_run_out_of_characters(stop_at_insertion_point)) - return ConsumeNextResult::RanOutOfCharacters; - return ConsumeNextResult::NotConsumed; - } - // FIXME: This should be more Unicode-aware. - if (case_sensitivity == CaseSensitivity::CaseInsensitive) { - if (code_point.value() < 0x80) { - if (to_ascii_lowercase(code_point.value()) != to_ascii_lowercase(string[i])) - return ConsumeNextResult::NotConsumed; - continue; - } - } - if (code_point.value() != (u32)string[i]) - return ConsumeNextResult::NotConsumed; - } - skip(string.length()); - return ConsumeNextResult::Consumed; -} - -void HTMLTokenizer::create_new_token(HTMLToken::Type type) -{ - m_current_token = { type }; - - auto is_start_or_end_tag = type == HTMLToken::Type::StartTag || type == HTMLToken::Type::EndTag; - m_current_token.set_start_position({}, nth_last_position(is_start_or_end_tag ? 1 : 0)); + static Vector const s_table = build_interned_name_table( + rust_html_tokenizer_interned_attr_name_count(), + rust_html_tokenizer_interned_attr_name); + if (id == 0 || id >= s_table.size()) + return s_table[0]; + return s_table[id]; } HTMLTokenizer::HTMLTokenizer() { - m_decoded_input = {}; - m_current_offset = 0; - m_prev_offset = 0; - m_source_positions.empend(0u, 0u); + m_tokenizer = create_tokenizer_from_utf8({}); + rust_html_tokenizer_set_input_stream_closed(m_tokenizer, false); +} + +HTMLTokenizer::~HTMLTokenizer() +{ + if (m_tokenizer) + rust_html_tokenizer_destroy(m_tokenizer); } HTMLTokenizer::HTMLTokenizer(StringView input, ByteString const& encoding) @@ -2924,148 +98,209 @@ HTMLTokenizer::HTMLTokenizer(StringView input, ByteString const& encoding) auto decoder = TextCodec::decoder_for(encoding); VERIFY(decoder.has_value()); m_source = MUST(decoder->to_utf8(input)); - m_decoded_input.ensure_capacity(m_source.bytes().size()); - for (auto code_point : m_source.code_points()) - m_decoded_input.append(code_point); - m_current_offset = 0; - m_prev_offset = 0; - m_source_positions.empend(0u, 0u); m_input_stream_closed = true; + m_tokenizer = create_tokenizer_from_utf8(m_source.bytes_as_string_view()); +} + +Optional HTMLTokenizer::next_token(StopAtInsertionPoint stop_at_insertion_point) +{ + RustFfiToken ffi; + bool stop = stop_at_insertion_point == StopAtInsertionPoint::Yes; + bool cdata_allowed = m_parser != nullptr + && m_parser->adjusted_current_node() + && m_parser->adjusted_current_node()->namespace_uri() != Namespace::HTML; + if (!rust_html_tokenizer_next_token(m_tokenizer, &ffi, stop, cdata_allowed)) + return {}; + + HTMLToken::Type type; + switch (ffi.token_type) { + case 1: + type = HTMLToken::Type::DOCTYPE; + break; + case 2: + type = HTMLToken::Type::StartTag; + break; + case 3: + type = HTMLToken::Type::EndTag; + break; + case 4: + type = HTMLToken::Type::Comment; + break; + case 5: + type = HTMLToken::Type::Character; + break; + case 6: + type = HTMLToken::Type::EndOfFile; + break; + default: + VERIFY_NOT_REACHED(); + } + + HTMLToken token { type }; + token.set_start_position({}, { ffi.start_line, ffi.start_column }); + token.set_end_position({}, { ffi.end_line, ffi.end_column }); + + switch (type) { + case HTMLToken::Type::Character: + token.set_code_point(ffi.code_point); + break; + case HTMLToken::Type::StartTag: + case HTMLToken::Type::EndTag: { + if (ffi.tag_name_id != 0) + token.set_tag_name(interned_rust_tag_name(ffi.tag_name_id)); + else + token.set_tag_name(MUST(FlyString::from_utf8(ffi_string_view(ffi.tag_name_ptr, ffi.tag_name_len)))); + + token.set_self_closing(ffi.self_closing); + for (size_t i = 0; i < ffi.attributes_len; ++i) { + auto const& ffi_attribute = ffi.attributes_ptr[i]; + HTMLToken::Attribute attribute; + if (ffi_attribute.name_id != 0) + attribute.local_name = interned_rust_attr_name(ffi_attribute.name_id); + else + attribute.local_name = MUST(FlyString::from_utf8(ffi_string_view(ffi_attribute.name_ptr, ffi_attribute.name_len))); + attribute.value = MUST(String::from_utf8(ffi_string_view(ffi_attribute.value_ptr, ffi_attribute.value_len))); + attribute.name_start_position = { ffi_attribute.name_start_line, ffi_attribute.name_start_column }; + attribute.name_end_position = { ffi_attribute.name_end_line, ffi_attribute.name_end_column }; + attribute.value_start_position = { ffi_attribute.value_start_line, ffi_attribute.value_start_column }; + attribute.value_end_position = { ffi_attribute.value_end_line, ffi_attribute.value_end_column }; + token.add_attribute(move(attribute)); + } + token.normalize_attributes(); + break; + } + case HTMLToken::Type::Comment: + token.set_comment(MUST(String::from_utf8(ffi_string_view(ffi.comment_ptr, ffi.comment_len)))); + break; + case HTMLToken::Type::DOCTYPE: { + auto& doctype = token.ensure_doctype_data(); + if (!ffi.missing_name) { + doctype.name = MUST(String::from_utf8(ffi_string_view(ffi.doctype_name_ptr, ffi.doctype_name_len))); + doctype.missing_name = false; + } + if (!ffi.missing_public_id) { + doctype.public_identifier = MUST(String::from_utf8(ffi_string_view(ffi.public_id_ptr, ffi.public_id_len))); + doctype.missing_public_identifier = false; + } + if (!ffi.missing_system_id) { + doctype.system_identifier = MUST(String::from_utf8(ffi_string_view(ffi.system_id_ptr, ffi.system_id_len))); + doctype.missing_system_identifier = false; + } + doctype.force_quirks = ffi.force_quirks; + break; + } + case HTMLToken::Type::EndOfFile: + break; + case HTMLToken::Type::Invalid: + VERIFY_NOT_REACHED(); + } + + return token; } void HTMLTokenizer::parser_did_run(Badge) { - // OPTIMIZATION: If we've consumed all input and the insertion point is at the start, - // we can throw away the decoded input buffer to save memory. - if (m_current_offset > 0 - && static_cast(m_current_offset) == m_decoded_input.size() - && (!m_insertion_point.has_value() || *m_insertion_point == 0) - && m_old_insertion_points.is_empty()) { - m_decoded_input.clear(); - m_current_offset = 0; - m_prev_offset = 0; - } + rust_html_tokenizer_parser_did_run(m_tokenizer); } String HTMLTokenizer::unparsed_input() const { - if (m_current_offset < 0 || static_cast(m_current_offset) >= m_decoded_input.size()) - return {}; - StringBuilder builder; - builder.append(Utf32View { m_decoded_input.span().slice(m_current_offset) }); - return MUST(builder.to_string()); + uint8_t const* ptr = nullptr; + size_t len = 0; + rust_html_tokenizer_unparsed_input(m_tokenizer, &ptr, &len); + return MUST(String::from_utf8(ffi_string_view(ptr, len))); } void HTMLTokenizer::append_to_input_stream(StringView input) { - for (auto code_point : Utf8View(input)) - m_decoded_input.append(code_point); + if (input.is_empty()) + return; + + auto utf8_input = MUST(String::from_utf8(input)); + auto code_points = code_points_from_string(utf8_input); + rust_html_tokenizer_append_input(m_tokenizer, code_points.data(), code_points.size()); } void HTMLTokenizer::close_input_stream() { m_input_stream_closed = true; + rust_html_tokenizer_set_input_stream_closed(m_tokenizer, true); } void HTMLTokenizer::insert_input_at_insertion_point(StringView input) { - Vector new_decoded_input; - new_decoded_input.ensure_capacity(m_decoded_input.size() + input.length()); - - auto insertion_point = *m_insertion_point; - auto before = m_decoded_input.span().slice(0, insertion_point); - new_decoded_input.append(before.data(), before.size()); - - auto utf8_to_insert = MUST(String::from_utf8(input)); - ssize_t code_points_inserted = 0; - for (auto code_point : utf8_to_insert.code_points()) { - new_decoded_input.append(code_point); - ++code_points_inserted; - } - - auto after = m_decoded_input.span().slice(insertion_point); - new_decoded_input.append(after.data(), after.size()); - m_decoded_input = move(new_decoded_input); - - m_insertion_point.value() += code_points_inserted; - for (auto& old_insertion_point : m_old_insertion_points) { - if (old_insertion_point.has_value() && insertion_point <= *old_insertion_point) - old_insertion_point.value() += code_points_inserted; - } + auto utf8_input = MUST(String::from_utf8(input)); + auto code_points = code_points_from_string(utf8_input); + rust_html_tokenizer_insert_input(m_tokenizer, code_points.data(), code_points.size()); } void HTMLTokenizer::insert_eof() { close_input_stream(); - m_explicit_eof_inserted = true; + rust_html_tokenizer_insert_eof(m_tokenizer); } bool HTMLTokenizer::is_eof_inserted() { - return m_explicit_eof_inserted; + return rust_html_tokenizer_is_eof_inserted(m_tokenizer); } -void HTMLTokenizer::will_switch_to([[maybe_unused]] State new_state) +void HTMLTokenizer::set_blocked(bool blocked) { - dbgln_if(TOKENIZER_TRACE_DEBUG, "[{}] Switch to {}", state_name(m_state), state_name(new_state)); + rust_html_tokenizer_set_blocked(m_tokenizer, blocked); } -void HTMLTokenizer::will_reconsume_in([[maybe_unused]] State new_state) +bool HTMLTokenizer::is_blocked() const { - dbgln_if(TOKENIZER_TRACE_DEBUG, "[{}] Reconsume in {}", state_name(m_state), state_name(new_state)); + return rust_html_tokenizer_is_blocked(m_tokenizer); +} + +bool HTMLTokenizer::is_insertion_point_defined() const +{ + return rust_html_tokenizer_is_insertion_point_defined(m_tokenizer); +} + +bool HTMLTokenizer::is_insertion_point_reached() +{ + return rust_html_tokenizer_is_insertion_point_reached(m_tokenizer); +} + +void HTMLTokenizer::undefine_insertion_point() +{ + rust_html_tokenizer_undefine_insertion_point(m_tokenizer); +} + +void HTMLTokenizer::store_insertion_point() +{ + rust_html_tokenizer_store_insertion_point(m_tokenizer); +} + +void HTMLTokenizer::restore_insertion_point() +{ + rust_html_tokenizer_restore_insertion_point(m_tokenizer); +} + +void HTMLTokenizer::update_insertion_point() +{ + rust_html_tokenizer_update_insertion_point(m_tokenizer); +} + +void HTMLTokenizer::abort() +{ + rust_html_tokenizer_abort(m_tokenizer); } void HTMLTokenizer::switch_to(Badge, State new_state) { dbgln_if(TOKENIZER_TRACE_DEBUG, "[{}] Parser switches tokenizer state to {}", state_name(m_state), state_name(new_state)); + switch_to(new_state); +} + +void HTMLTokenizer::switch_to(State new_state) +{ + dbgln_if(TOKENIZER_TRACE_DEBUG, "[{}] Switch to {}", state_name(m_state), state_name(new_state)); m_state = new_state; -} - -void HTMLTokenizer::will_emit(HTMLToken& token) -{ - if (token.is_start_tag()) - m_last_emitted_start_tag_name = token.tag_name(); - - auto is_start_or_end_tag = token.type() == HTMLToken::Type::StartTag || token.type() == HTMLToken::Type::EndTag; - token.set_end_position({}, nth_last_position(is_start_or_end_tag ? 1 : 0)); - - if (is_start_or_end_tag) - token.normalize_attributes(); -} - -bool HTMLTokenizer::current_end_tag_token_is_appropriate() const -{ - VERIFY(m_current_token.is_end_tag()); - if (!m_last_emitted_start_tag_name.has_value()) - return false; - return m_current_token.tag_name() == m_last_emitted_start_tag_name.value(); -} - -bool HTMLTokenizer::consumed_as_part_of_an_attribute() const -{ - return m_return_state == State::AttributeValueUnquoted || m_return_state == State::AttributeValueSingleQuoted || m_return_state == State::AttributeValueDoubleQuoted; -} - -void HTMLTokenizer::restore_to(ssize_t new_iterator) -{ - auto diff = m_current_offset - new_iterator; - if (diff > 0) { - for (ssize_t i = 0; i < diff; ++i) { - if (!m_source_positions.is_empty()) - m_source_positions.take_last(); - } - } else { - // Going forwards...? - TODO(); - } - m_current_offset = new_iterator; -} - -String HTMLTokenizer::consume_current_builder() -{ - auto string = m_current_builder.to_string_without_validation(); - m_current_builder.clear(); - return string; + rust_html_tokenizer_switch_state(m_tokenizer, static_cast(new_state)); } void HTMLTokenizer::visit_edges(GC::Cell::Visitor& visitor) diff --git a/Libraries/LibWeb/HTML/Parser/HTMLTokenizer.h b/Libraries/LibWeb/HTML/Parser/HTMLTokenizer.h index 2f99818f32..dac5355f00 100644 --- a/Libraries/LibWeb/HTML/Parser/HTMLTokenizer.h +++ b/Libraries/LibWeb/HTML/Parser/HTMLTokenizer.h @@ -7,18 +7,17 @@ #pragma once -#include -#include +#include #include #include -#include #include #include #include #include -#include #include +struct RustFfiTokenizerHandle; + namespace Web::HTML { #define ENUMERATE_TOKENIZER_STATES \ @@ -107,6 +106,7 @@ class WEB_API HTMLTokenizer { public: explicit HTMLTokenizer(); explicit HTMLTokenizer(StringView input, ByteString const& encoding); + ~HTMLTokenizer(); enum class State { #define __ENUMERATE_TOKENIZER_STATE(state) state, @@ -123,13 +123,10 @@ public: void set_parser(Badge, HTMLParser& parser) { m_parser = &parser; } void switch_to(Badge, State new_state); - void switch_to(State new_state) - { - m_state = new_state; - } + void switch_to(State new_state); - void set_blocked(bool b) { m_blocked = b; } - bool is_blocked() const { return m_blocked; } + void set_blocked(bool b); + bool is_blocked() const; auto const& source() const { return m_source; } @@ -142,38 +139,23 @@ public: void insert_eof(); bool is_eof_inserted(); - bool is_insertion_point_defined() const { return m_insertion_point.has_value(); } - bool is_insertion_point_reached() { return m_insertion_point.has_value() && m_current_offset >= *m_insertion_point; } - void undefine_insertion_point() { m_insertion_point = {}; } - void store_old_insertion_point() { m_old_insertion_points.append(m_insertion_point); } - void restore_old_insertion_point() { m_insertion_point = m_old_insertion_points.take_last(); } - void update_insertion_point() { m_insertion_point = m_current_offset; } + bool is_insertion_point_defined() const; + bool is_insertion_point_reached(); + void undefine_insertion_point(); + void store_insertion_point(); + void restore_insertion_point(); + void store_old_insertion_point() { store_insertion_point(); } + void restore_old_insertion_point() { restore_insertion_point(); } + void update_insertion_point(); // This permanently cuts off the tokenizer input stream. - void abort() { m_aborted = true; } + void abort(); void parser_did_run(Badge); void visit_edges(GC::Cell::Visitor&); private: - void skip(size_t count); - Optional next_code_point(StopAtInsertionPoint); - Optional peek_code_point(ssize_t offset, StopAtInsertionPoint) const; - - enum class ConsumeNextResult { - Consumed, - NotConsumed, - RanOutOfCharacters, - }; - [[nodiscard]] ConsumeNextResult consume_next_if_match(StringView, StopAtInsertionPoint, CaseSensitivity = CaseSensitivity::CaseSensitive); - bool should_pause_before_next_input_character(StopAtInsertionPoint) const; - bool can_run_out_of_characters(StopAtInsertionPoint) const; - - void create_new_token(HTMLToken::Type); - bool current_end_tag_token_is_appropriate() const; - String consume_current_builder(); - static char const* state_name(State state) { switch (state) { @@ -186,52 +168,13 @@ private: VERIFY_NOT_REACHED(); } - void will_emit(HTMLToken&); - void will_switch_to(State); - void will_reconsume_in(State); - - bool consumed_as_part_of_an_attribute() const; - - void restore_to(ssize_t new_iterator); - HTMLToken::Position nth_last_position(size_t n = 0); - GC::Ptr m_parser; State m_state { State::Data }; - State m_return_state { State::Data }; - - Vector m_temporary_buffer; - String m_source; - Vector m_decoded_input; - - Optional m_insertion_point; - // Spec algorithms have an "old insertion point" local; reentrant script execution can nest those locals. - Vector> m_old_insertion_points; - - ssize_t m_current_offset { 0 }; - ssize_t m_prev_offset { 0 }; - - HTMLToken m_current_token; - StringBuilder m_current_builder; - - NamedCharacterReferenceMatcher m_named_character_reference_matcher; - - Optional m_last_emitted_start_tag_name; - - bool m_explicit_eof_inserted { false }; bool m_input_stream_closed { false }; - bool m_has_emitted_eof { false }; - Queue m_queued_tokens; - - u32 m_character_reference_code { 0 }; - - bool m_blocked { false }; - - bool m_aborted { false }; - - Vector m_source_positions; + RustFfiTokenizerHandle* m_tokenizer { nullptr }; }; } diff --git a/Libraries/LibWeb/HTML/Parser/Rust/Cargo.toml b/Libraries/LibWeb/HTML/Parser/Rust/Cargo.toml new file mode 100644 index 0000000000..25cb66ec95 --- /dev/null +++ b/Libraries/LibWeb/HTML/Parser/Rust/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "libweb_html_tokenizer" +version = "0.1.0" +edition = "2024" + +[lib] +crate-type = ["rlib"] + +[build-dependencies] +cbindgen = "0.29" diff --git a/Libraries/LibWeb/HTML/Parser/Rust/build.rs b/Libraries/LibWeb/HTML/Parser/Rust/build.rs new file mode 100644 index 0000000000..9d7a5874f6 --- /dev/null +++ b/Libraries/LibWeb/HTML/Parser/Rust/build.rs @@ -0,0 +1,821 @@ +/* + * Copyright (c) 2026-present, the Ladybird developers. + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +//! Build script that generates a DAFSA (Deterministic Acyclic Finite State Automaton) +//! for named character reference matching. This is a Rust port of the C++ generator at +//! Meta/Lagom/Tools/CodeGenerators/LibWeb/GenerateNamedCharacterReferences.cpp. + +use std::cell::RefCell; +use std::collections::HashMap; +use std::env; +use std::fs; +use std::path::{Path, PathBuf}; +use std::rc::Rc; + +const FFI_HEADER: &str = "HTMLTokenizerRustFFI.h"; + +fn main() { + let manifest_dir = env::var("CARGO_MANIFEST_DIR").unwrap(); + let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap()); + + println!("cargo:rerun-if-changed=build.rs"); + println!("cargo:rerun-if-changed=cbindgen.toml"); + println!("cargo:rerun-if-changed=src"); + println!("cargo:rerun-if-env-changed=FFI_OUTPUT_DIR"); + + let ffi_out_dir = env::var("FFI_OUTPUT_DIR") + .map(PathBuf::from) + .unwrap_or_else(|_| out_dir.clone()); + + cbindgen::generate(&manifest_dir).map_or_else( + |error| match error { + cbindgen::Error::ParseSyntaxError { .. } => {} + e => panic!("{e:?}"), + }, + |bindings| { + bindings.write_to_file(out_dir.join(FFI_HEADER)); + if ffi_out_dir != out_dir { + bindings.write_to_file(ffi_out_dir.join(FFI_HEADER)); + } + }, + ); + + // Generate interned name tables from the existing C++ headers. + let tag_names_header = Path::new(&manifest_dir).join("../../TagNames.h"); + let attr_names_header = Path::new(&manifest_dir).join("../../AttributeNames.h"); + println!("cargo:rerun-if-changed={}", tag_names_header.display()); + println!("cargo:rerun-if-changed={}", attr_names_header.display()); + let tag_names = parse_enumerate_macro( + &fs::read_to_string(&tag_names_header).expect("Failed to read TagNames.h"), + "__ENUMERATE_HTML_TAG", + ); + let attr_names = parse_enumerate_macro( + &fs::read_to_string(&attr_names_header).expect("Failed to read AttributeNames.h"), + "__ENUMERATE_HTML_ATTRIBUTE", + ); + emit_interned_names(&out_dir.join("interned_names_generated.rs"), &tag_names, &attr_names); + + let json_path = Path::new(&manifest_dir).join("../Entities.json"); + println!("cargo:rerun-if-changed={}", json_path.display()); + + let json_str = fs::read_to_string(&json_path).expect("Failed to read Entities.json"); + let entities = parse_entities_json(&json_str); + + // Build DAFSA. + let mut builder = DafsaBuilder::new(); + for (name, _, _) in &entities { + builder.insert(name); + } + builder.minimize(0); + builder.calc_numbers(); + + // Verify minimal perfect hashing (no collisions). + let mut seen: Vec = vec![false; entities.len() + 1]; + for (name, _, _) in &entities { + let idx = builder.get_unique_index(name).unwrap(); + assert!(!seen[idx], "Hash collision at index {idx} for '{name}'"); + seen[idx] = true; + } + + // Build codepoints lookup table indexed by unique_index. + let mut index_to_codepoints = vec![(0u32, 0u32); entities.len()]; + for (name, first, second) in &entities { + let idx = builder.get_unique_index(name).unwrap(); + index_to_codepoints[idx - 1] = (*first, *second); + } + + // Extract DAFSA layers. + let root = &builder.root; + let root_ref = root.borrow(); + + let mut first_layer: Vec = Vec::new(); + let mut first_to_second_layer: Vec<(u64, u16)> = Vec::new(); + + let mut first_layer_tally: u16 = 0; + let mut second_layer_offset: u16 = 0; + + for c in 0u8..128 { + if root_ref.children[c as usize].is_none() { + continue; + } + assert!(c.is_ascii_alphabetic()); + let child = root_ref.children[c as usize].as_ref().unwrap(); + let child_ref = child.borrow(); + + first_layer.push(first_layer_tally); + first_layer_tally += child_ref.number; + + let mask = child_ref.get_ascii_alphabetic_bit_mask(); + first_to_second_layer.push((mask, second_layer_offset)); + second_layer_offset += child_ref.num_direct_children() as u16; + } + assert_eq!(first_layer.len(), 52); + + // BFS to build DAFSA node array. + // Following the C++ write_node_data three-phase approach: + type NodePtr = Rc>; + let mut queue: Vec = Vec::new(); + let mut child_indexes: HashMap<*const RefCell, u16> = HashMap::new(); + + // Phase 1: Queue root's children (first-layer nodes = 52 A-Z/a-z). + // This assigns temporary child_indexes for first-layer children. + queue_children(root, &mut queue, &mut child_indexes, 1); + + // Phase 2: Clear indexes and re-process. For each first-layer child, + // queue ITS children (second-layer nodes) and assign their child_indexes. + child_indexes.clear(); + let mut first_available_index: u16 = 1; // 0 is reserved (dummy node) + let first_layer_count = queue.len(); + for i in 0..first_layer_count { + let node = Rc::clone(&queue[i]); + first_available_index = queue_children(&node, &mut queue, &mut child_indexes, first_available_index); + } + // Remove first-layer nodes from queue, keep only second-layer+ nodes. + let second_layer_nodes: Vec = queue.drain(first_layer_count..).collect(); + queue.clear(); + queue.extend(second_layer_nodes); + + // Phase 3: BFS remaining nodes, writing node data. + let mut node_data: Vec = Vec::new(); + let mut qi = 0; + #[allow(unused_assignments)] + while qi < queue.len() { + let node = Rc::clone(&queue[qi]); + qi += 1; + first_available_index = write_children_data( + &node, + &mut node_data, + &mut queue, + &mut child_indexes, + first_available_index, + ); + } + + // Build second_layer entries with child_indexes from phase 2. + let mut second_layer: Vec = Vec::new(); + for c in 0u8..128 { + if root_ref.children[c as usize].is_none() { + continue; + } + let first_child = root_ref.children[c as usize].as_ref().unwrap(); + let first_child_ref = first_child.borrow(); + let mut tally: u8 = 0; + for cc in 0u8..128 { + if first_child_ref.children[cc as usize].is_none() { + continue; + } + let second_child = first_child_ref.children[cc as usize].as_ref().unwrap(); + let second_child_ref = second_child.borrow(); + let key = Rc::as_ptr(second_child); + let ci = child_indexes.get(&key).copied().unwrap_or(0); + let children_len = second_child_ref.num_direct_children(); + second_layer.push(SecondLayerEntry { + child_index: ci, + number: tally, + children_len, + end_of_word: second_child_ref.is_terminal, + }); + tally = tally.wrapping_add(second_child_ref.number as u8); + } + } + drop(root_ref); + + // Generate output file. + let out_dir = env::var("OUT_DIR").unwrap(); + let out_path = Path::new(&out_dir).join("named_character_references.rs"); + let mut out = String::new(); + + out.push_str("// Auto-generated by build.rs -- do not edit!\n\n"); + + // Second codepoint enum. + out.push_str("#[derive(Clone, Copy, PartialEq, Eq)]\n"); + out.push_str("#[repr(u8)]\n"); + out.push_str("pub enum SecondCodepoint {\n"); + out.push_str(" None = 0,\n"); + out.push_str(" CombiningLongSolidusOverlay = 1,\n"); + out.push_str(" CombiningLongVerticalLineOverlay = 2,\n"); + out.push_str(" HairSpace = 3,\n"); + out.push_str(" CombiningDoubleLowLine = 4,\n"); + out.push_str(" CombiningReverseSolidusOverlay = 5,\n"); + out.push_str(" VariationSelector1 = 6,\n"); + out.push_str(" LatinSmallLetterJ = 7,\n"); + out.push_str(" CombiningMacronBelow = 8,\n"); + out.push_str("}\n\n"); + + out.push_str("impl SecondCodepoint {\n"); + out.push_str(" pub fn value(self) -> u32 {\n"); + out.push_str(" match self {\n"); + out.push_str(" SecondCodepoint::None => 0,\n"); + out.push_str(" SecondCodepoint::CombiningLongSolidusOverlay => 0x0338,\n"); + out.push_str(" SecondCodepoint::CombiningLongVerticalLineOverlay => 0x20D2,\n"); + out.push_str(" SecondCodepoint::HairSpace => 0x200A,\n"); + out.push_str(" SecondCodepoint::CombiningDoubleLowLine => 0x0333,\n"); + out.push_str(" SecondCodepoint::CombiningReverseSolidusOverlay => 0x20E5,\n"); + out.push_str(" SecondCodepoint::VariationSelector1 => 0xFE00,\n"); + out.push_str(" SecondCodepoint::LatinSmallLetterJ => 0x006A,\n"); + out.push_str(" SecondCodepoint::CombiningMacronBelow => 0x0331,\n"); + out.push_str(" }\n"); + out.push_str(" }\n"); + out.push_str("}\n\n"); + + // Struct definitions. + out.push_str("#[derive(Clone, Copy)]\n"); + out.push_str("pub struct DafsaNode {\n"); + out.push_str(" pub character: u8,\n"); + out.push_str(" pub number: u8,\n"); + out.push_str(" pub end_of_word: bool,\n"); + out.push_str(" pub child_index: u16,\n"); + out.push_str(" pub children_len: u8,\n"); + out.push_str("}\n\n"); + + out.push_str("#[derive(Clone, Copy)]\n"); + out.push_str("pub struct SecondLayerNode {\n"); + out.push_str(" pub child_index: u16,\n"); + out.push_str(" pub number: u8,\n"); + out.push_str(" pub children_len: u8,\n"); + out.push_str(" pub end_of_word: bool,\n"); + out.push_str("}\n\n"); + + // Codepoints lookup table. + out.push_str(&format!( + "pub static CODEPOINTS_LOOKUP: [(u32, SecondCodepoint); {}] = [\n", + index_to_codepoints.len() + )); + for (first, second) in &index_to_codepoints { + let variant = second_codepoint_variant(*second); + out.push_str(&format!(" ({first:#06X}, SecondCodepoint::{variant}),\n")); + } + out.push_str("];\n\n"); + + // DAFSA nodes array (with dummy node at index 0). + out.push_str(&format!( + "pub static DAFSA_NODES: [DafsaNode; {}] = [\n", + node_data.len() + 1 + )); + out.push_str(" DafsaNode { character: 0, number: 0, end_of_word: false, child_index: 0, children_len: 0 },\n"); + for nd in &node_data { + out.push_str(&format!( + " DafsaNode {{ character: b'{}', number: {}, end_of_word: {}, child_index: {}, children_len: {} }},\n", + escape_byte(nd.character), + nd.number, + nd.end_of_word, + nd.child_index, + nd.children_len + )); + } + out.push_str("];\n\n"); + + // First layer. + out.push_str(&format!("pub static FIRST_LAYER: [u16; {}] = [\n", first_layer.len())); + for n in &first_layer { + out.push_str(&format!(" {n},\n")); + } + out.push_str("];\n\n"); + + // First-to-second layer links. + out.push_str(&format!( + "pub static FIRST_TO_SECOND_LAYER: [(u64, u16); {}] = [\n", + first_to_second_layer.len() + )); + for (mask, offset) in &first_to_second_layer { + out.push_str(&format!(" ({mask:#018X}, {offset}),\n")); + } + out.push_str("];\n\n"); + + // Second layer nodes. + out.push_str(&format!( + "pub static SECOND_LAYER: [SecondLayerNode; {}] = [\n", + second_layer.len() + )); + for sl in &second_layer { + out.push_str(&format!( + " SecondLayerNode {{ child_index: {}, number: {}, children_len: {}, end_of_word: {} }},\n", + sl.child_index, sl.number, sl.children_len, sl.end_of_word + )); + } + out.push_str("];\n\n"); + + // Total entity count. + out.push_str(&format!("pub const ENTITY_COUNT: usize = {};\n", entities.len())); + + fs::write(&out_path, &out).expect("Failed to write generated file"); +} + +/// Extract the string literal from `__ENUMERATE_FOO(ident, "string")` macro +/// invocations in a C++ header. +fn parse_enumerate_macro(source: &str, macro_name: &str) -> Vec { + let needle = format!("{macro_name}("); + let mut out = Vec::new(); + for line in source.lines() { + let Some(idx) = line.find(&needle) else { + continue; + }; + let rest = &line[idx + needle.len()..]; + // Take the second argument, which is the quoted string literal. + let Some(first_quote) = rest.find('"') else { + continue; + }; + let after = &rest[first_quote + 1..]; + let Some(end_quote) = after.find('"') else { + continue; + }; + out.push(after[..end_quote].to_string()); + } + out +} + +/// Emit a Rust source file with two const byte-slice arrays and two lookup +/// functions that dispatch on length and then on the exact bytes. rustc +/// compiles this pattern to a jump table + direct memcmp, which beats a +/// HashMap lookup with a cryptographic default hasher by a wide margin for +/// the small, fixed set of HTML names. +fn emit_interned_names(out_path: &Path, tag_names: &[String], attr_names: &[String]) { + let mut out = String::new(); + out.push_str("// Auto-generated by build.rs from TagNames.h / AttributeNames.h.\n"); + out.push_str("// Do not edit by hand.\n\n"); + + out.push_str("pub const INTERNED_TAG_NAMES: &[&[u8]] = &[\n"); + for name in tag_names { + out.push_str(&format!(" b\"{}\",\n", name)); + } + out.push_str("];\n\n"); + + out.push_str("pub const INTERNED_ATTR_NAMES: &[&[u8]] = &[\n"); + for name in attr_names { + out.push_str(&format!(" b\"{}\",\n", name)); + } + out.push_str("];\n\n"); + + emit_lookup_fn(&mut out, "lookup_tag_name_generated", tag_names); + emit_lookup_fn(&mut out, "lookup_attr_name_generated", attr_names); + + fs::write(out_path, out).expect("Failed to write interned_names_generated.rs"); +} + +fn emit_lookup_fn(out: &mut String, fn_name: &str, names: &[String]) { + // Group names by byte length so the outer dispatch can be a single match. + let mut by_length: std::collections::BTreeMap> = std::collections::BTreeMap::new(); + for (i, name) in names.iter().enumerate() { + by_length.entry(name.len()).or_default().push((i, name)); + } + + out.push_str(&format!("#[inline]\npub fn {fn_name}(bytes: &[u8]) -> u16 {{\n")); + out.push_str(" match bytes.len() {\n"); + for (length, entries) in &by_length { + out.push_str(&format!(" {length} => match bytes {{\n")); + for (index, name) in entries { + // id is 1-based. + let id = index + 1; + out.push_str(&format!(" b\"{name}\" => {id},\n")); + } + out.push_str(" _ => 0,\n"); + out.push_str(" },\n"); + } + out.push_str(" _ => 0,\n"); + out.push_str(" }\n"); + out.push_str("}\n\n"); +} + +fn escape_byte(b: u8) -> String { + if b == b'\'' { + "\\'".to_string() + } else if b == b'\\' { + "\\\\".to_string() + } else if b.is_ascii_graphic() || b == b' ' { + String::from(b as char) + } else { + format!("\\x{b:02X}") + } +} + +fn second_codepoint_variant(cp: u32) -> &'static str { + match cp { + 0 => "None", + 0x0338 => "CombiningLongSolidusOverlay", + 0x20D2 => "CombiningLongVerticalLineOverlay", + 0x200A => "HairSpace", + 0x0333 => "CombiningDoubleLowLine", + 0x20E5 => "CombiningReverseSolidusOverlay", + 0xFE00 => "VariationSelector1", + 0x006A => "LatinSmallLetterJ", + 0x0331 => "CombiningMacronBelow", + _ => panic!("Unknown second codepoint: {cp:#X}"), + } +} + +// Minimal JSON parser for Entities.json. +fn parse_entities_json(json: &str) -> Vec<(String, u32, u32)> { + let mut entities = Vec::new(); + let bytes = json.as_bytes(); + let len = bytes.len(); + let mut i = 0; + + while i < len && bytes[i] != b'{' { + i += 1; + } + i += 1; + + loop { + while i < len && bytes[i].is_ascii_whitespace() { + i += 1; + } + if i >= len || bytes[i] == b'}' { + break; + } + if bytes[i] == b',' { + i += 1; + continue; + } + + // Parse key. + assert_eq!(bytes[i], b'"'); + i += 1; + let key_start = i; + while i < len && bytes[i] != b'"' { + if bytes[i] == b'\\' { + i += 1; + } + i += 1; + } + let key = std::str::from_utf8(&bytes[key_start..i]).unwrap().to_string(); + i += 1; + + // Skip ':'. + while i < len && bytes[i].is_ascii_whitespace() { + i += 1; + } + assert_eq!(bytes[i], b':'); + i += 1; + + // Skip to inner '{'. + while i < len && bytes[i] != b'{' { + i += 1; + } + i += 1; + + // Parse inner object for "codepoints". + let mut codepoints: Vec = Vec::new(); + while i < len && bytes[i] != b'}' { + if bytes[i] == b'"' { + i += 1; + let field_start = i; + while i < len && bytes[i] != b'"' { + i += 1; + } + let field_name = std::str::from_utf8(&bytes[field_start..i]).unwrap(); + i += 1; + + while i < len && bytes[i].is_ascii_whitespace() { + i += 1; + } + assert_eq!(bytes[i], b':'); + i += 1; + while i < len && bytes[i].is_ascii_whitespace() { + i += 1; + } + + if field_name == "codepoints" { + assert_eq!(bytes[i], b'['); + i += 1; + loop { + while i < len && (bytes[i].is_ascii_whitespace() || bytes[i] == b',') { + i += 1; + } + if i >= len || bytes[i] == b']' { + i += 1; + break; + } + let num_start = i; + while i < len && bytes[i].is_ascii_digit() { + i += 1; + } + let num_str = std::str::from_utf8(&bytes[num_start..i]).unwrap(); + codepoints.push(num_str.parse().unwrap()); + } + } else { + // Skip value. + if bytes[i] == b'"' { + i += 1; + while i < len && bytes[i] != b'"' { + if bytes[i] == b'\\' { + i += 1; + } + i += 1; + } + i += 1; + } else if bytes[i] == b'[' { + let mut depth = 1; + i += 1; + while i < len && depth > 0 { + if bytes[i] == b'[' { + depth += 1; + } else if bytes[i] == b']' { + depth -= 1; + } + i += 1; + } + } + } + } else { + i += 1; + } + } + i += 1; + + let name = key.strip_prefix('&').unwrap_or(&key).to_string(); + let first = codepoints.first().copied().unwrap_or(0); + let second = if codepoints.len() > 1 { codepoints[1] } else { 0 }; + entities.push((name, first, second)); + } + + entities.sort_by(|a, b| a.0.cmp(&b.0)); + entities +} + +// DAFSA builder using Rc> for shared ownership. + +type NodeRc = Rc>; + +struct Node { + children: Vec>, // 128 slots + is_terminal: bool, + number: u16, +} + +struct SecondLayerEntry { + child_index: u16, + number: u8, + children_len: u8, + end_of_word: bool, +} + +struct NodeData { + character: u8, + number: u8, + end_of_word: bool, + child_index: u16, + children_len: u8, +} + +impl Node { + fn new_rc() -> NodeRc { + Rc::new(RefCell::new(Node { + children: (0..128).map(|_| Option::None).collect(), + is_terminal: false, + number: 0, + })) + } + + fn calc_numbers(&mut self) { + self.number = if self.is_terminal { 1 } else { 0 }; + for child in self.children.iter().flatten() { + child.borrow_mut().calc_numbers(); + self.number += child.borrow().number; + } + } + + fn num_direct_children(&self) -> u8 { + let mut n = 0u8; + for c in &self.children { + if c.is_some() { + n += 1; + } + } + n + } + + fn get_ascii_alphabetic_bit_mask(&self) -> u64 { + let mut mask = 0u64; + for i in 0..128u8 { + if self.children[i as usize].is_some() { + mask |= 1u64 << ascii_alphabetic_to_index(i); + } + } + mask + } + + /// Hash based on child identities (Rc pointer) and terminal status. + fn structure_hash(&self) -> u64 { + let mut h: u64 = if self.is_terminal { 1 } else { 0 }; + for (i, child) in self.children.iter().enumerate() { + if let Some(c) = child { + h = h.wrapping_mul(31).wrapping_add(i as u64); + h = h.wrapping_mul(31).wrapping_add(Rc::as_ptr(c) as u64); + } + } + h + } + + /// Check structural equality via Rc pointer identity. + fn structure_eq(&self, other: &Node) -> bool { + if self.is_terminal != other.is_terminal { + return false; + } + for i in 0..128 { + match (&self.children[i], &other.children[i]) { + (None, None) => {} + (Some(a), Some(b)) => { + if !Rc::ptr_eq(a, b) { + return false; + } + } + _ => return false, + } + } + true + } +} + +fn ascii_alphabetic_to_index(c: u8) -> u8 { + if c <= b'Z' { c - b'A' } else { c - b'a' + 26 } +} + +struct UncheckedNode { + parent: NodeRc, + character: u8, +} + +struct DafsaBuilder { + root: NodeRc, + minimized_nodes: HashMap>, + unchecked_nodes: Vec, + previous_word: String, +} + +impl DafsaBuilder { + fn new() -> Self { + DafsaBuilder { + root: Node::new_rc(), + minimized_nodes: HashMap::new(), + unchecked_nodes: Vec::new(), + previous_word: String::new(), + } + } + + fn insert(&mut self, word: &str) { + assert!( + word > self.previous_word.as_str(), + "Words must be inserted in sorted order: '{word}' <= '{}'", + self.previous_word + ); + + let common_prefix_len = word + .bytes() + .zip(self.previous_word.bytes()) + .take_while(|(a, b)| a == b) + .count(); + + self.minimize(common_prefix_len); + + let node: NodeRc = if self.unchecked_nodes.is_empty() { + Rc::clone(&self.root) + } else { + let last = &self.unchecked_nodes[self.unchecked_nodes.len() - 1]; + let parent = last.parent.borrow(); + Rc::clone(parent.children[last.character as usize].as_ref().unwrap()) + }; + + let remaining = &word[common_prefix_len..]; + let mut current = node; + for c in remaining.bytes() { + let new_child = Node::new_rc(); + { + let mut current_ref = current.borrow_mut(); + assert!(current_ref.children[c as usize].is_none()); + current_ref.children[c as usize] = Some(Rc::clone(&new_child)); + } + self.unchecked_nodes.push(UncheckedNode { + parent: Rc::clone(¤t), + character: c, + }); + current = new_child; + } + current.borrow_mut().is_terminal = true; + + self.previous_word = word.to_string(); + } + + fn minimize(&mut self, down_to: usize) { + while self.unchecked_nodes.len() > down_to { + let unchecked = self.unchecked_nodes.pop().unwrap(); + let parent = &unchecked.parent; + let child = { + let parent_ref = parent.borrow(); + Rc::clone(parent_ref.children[unchecked.character as usize].as_ref().unwrap()) + }; + + let hash = child.borrow().structure_hash(); + let mut found_replacement: Option = Option::None; + + if let Some(bucket) = self.minimized_nodes.get(&hash) { + for existing in bucket { + if child.borrow().structure_eq(&existing.borrow()) { + found_replacement = Some(Rc::clone(existing)); + break; + } + } + } + + if let Some(replacement) = found_replacement { + parent.borrow_mut().children[unchecked.character as usize] = Some(replacement); + } else { + self.minimized_nodes.entry(hash).or_default().push(Rc::clone(&child)); + } + } + } + + fn calc_numbers(&mut self) { + self.root.borrow_mut().calc_numbers(); + } + + fn get_unique_index(&self, word: &str) -> Option { + let mut index: usize = 0; + let mut current = Rc::clone(&self.root); + + for c in word.bytes() { + let next = { + let node = current.borrow(); + let child = node.children[c as usize].as_ref()?; + for sibling_c in 0u8..128 { + if let Some(sibling) = &node.children[sibling_c as usize] + && sibling_c < c + { + index += sibling.borrow().number as usize; + } + } + Rc::clone(child) + }; + if next.borrow().is_terminal { + index += 1; + } + current = next; + } + + Some(index) + } +} + +fn queue_children( + node: &NodeRc, + queue: &mut Vec, + child_indexes: &mut HashMap<*const RefCell, u16>, + first_available_index: u16, +) -> u16 { + let mut current = first_available_index; + let node_ref = node.borrow(); + for c in 0..128u8 { + if let Some(child) = &node_ref.children[c as usize] { + let key = Rc::as_ptr(child); + if let std::collections::hash_map::Entry::Vacant(entry) = child_indexes.entry(key) { + let num_children = child.borrow().num_direct_children(); + if num_children > 0 { + entry.insert(current); + current += num_children as u16; + } + queue.push(Rc::clone(child)); + } + } + } + current +} + +fn write_children_data( + node: &NodeRc, + node_data: &mut Vec, + queue: &mut Vec, + child_indexes: &mut HashMap<*const RefCell, u16>, + first_available_index: u16, +) -> u16 { + let mut current = first_available_index; + let mut unique_index_tally: u8 = 0; + let node_ref = node.borrow(); + for c in 0..128u8 { + if let Some(child) = &node_ref.children[c as usize] { + let key = Rc::as_ptr(child); + let child_ref = child.borrow(); + let num_children = child_ref.num_direct_children(); + + if let std::collections::hash_map::Entry::Vacant(entry) = child_indexes.entry(key) { + if num_children > 0 { + entry.insert(current); + current += num_children as u16; + } + queue.push(Rc::clone(child)); + } + + node_data.push(NodeData { + character: c, + number: unique_index_tally, + end_of_word: child_ref.is_terminal, + child_index: child_indexes.get(&key).copied().unwrap_or(0), + children_len: num_children, + }); + + unique_index_tally = unique_index_tally.wrapping_add(child_ref.number as u8); + } + } + current +} diff --git a/Libraries/LibWeb/HTML/Parser/Rust/cbindgen.toml b/Libraries/LibWeb/HTML/Parser/Rust/cbindgen.toml new file mode 100644 index 0000000000..6b2b44e957 --- /dev/null +++ b/Libraries/LibWeb/HTML/Parser/Rust/cbindgen.toml @@ -0,0 +1,19 @@ +language = "C++" +header = """/* + * Copyright (c) 2026-present, the Ladybird developers. + * + * SPDX-License-Identifier: BSD-2-Clause + */""" +pragma_once = true +include_version = true +line_length = 120 +tab_width = 4 +no_includes = true +sys_includes = ["stdint.h", "stddef.h"] +usize_is_size_t = true + +[parse] +parse_deps = false + +[parse.expand] +all_features = false diff --git a/Libraries/LibWeb/HTML/Parser/Rust/src/entities.rs b/Libraries/LibWeb/HTML/Parser/Rust/src/entities.rs new file mode 100644 index 0000000000..591e1369bd --- /dev/null +++ b/Libraries/LibWeb/HTML/Parser/Rust/src/entities.rs @@ -0,0 +1,151 @@ +/* + * Copyright (c) 2026-present, the Ladybird developers. + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +//! Named character reference matching using a DAFSA (Deterministic Acyclic Finite +//! State Automaton) with minimal perfect hashing. The DAFSA data is generated at +//! build time by build.rs from Entities.json. + +include!(concat!(env!("OUT_DIR"), "/named_character_references.rs")); + +fn ascii_alphabetic_to_index(c: u8) -> u8 { + if c <= b'Z' { c - b'A' } else { c - b'a' + 26 } +} + +#[derive(Clone, Copy)] +enum SearchState { + Init, + FirstToSecondLayer { mask: u64, offset: u16 }, + DafsaChildren { start_index: u16, len: u8 }, +} + +/// Incremental matcher for named character references using the DAFSA. +/// +/// Feed characters one at a time via `try_consume_code_point()`. After each +/// rejected character (returns false), call `code_points()` to get the longest +/// match found so far, and `overconsumed_code_points()` to know how many +/// characters were consumed past the longest match. +pub struct NamedCharacterReferenceMatcher { + search_state: SearchState, + last_matched_unique_index: u16, + pending_unique_index: u16, + overconsumed_code_points: u8, + ends_with_semicolon: bool, +} + +impl NamedCharacterReferenceMatcher { + pub fn new() -> Self { + Self { + search_state: SearchState::Init, + last_matched_unique_index: 0, + pending_unique_index: 0, + overconsumed_code_points: 0, + ends_with_semicolon: false, + } + } + + /// Feed one code point to the matcher. Returns true if the character was + /// consumed (there may still be longer matches), false if no further + /// matches are possible. + pub fn try_consume_code_point(&mut self, c: u32) -> bool { + if c > 0x7F { + return false; + } + self.try_consume_ascii_char(c as u8) + } + + fn try_consume_ascii_char(&mut self, c: u8) -> bool { + match self.search_state { + SearchState::Init => { + if !c.is_ascii_alphabetic() { + return false; + } + let index = ascii_alphabetic_to_index(c) as usize; + self.search_state = SearchState::FirstToSecondLayer { + mask: FIRST_TO_SECOND_LAYER[index].0, + offset: FIRST_TO_SECOND_LAYER[index].1, + }; + self.pending_unique_index = FIRST_LAYER[index]; + self.overconsumed_code_points += 1; + true + } + SearchState::FirstToSecondLayer { mask, offset } => { + if !c.is_ascii_alphabetic() { + return false; + } + let bit_index = ascii_alphabetic_to_index(c); + if ((1u64 << bit_index) & mask) == 0 { + return false; + } + + // Count set bits below bit_index to find the node's position. + let lower_mask = (1u64 << bit_index) - 1; + let char_index = (mask & lower_mask).count_ones() as u16; + let node = &SECOND_LAYER[(offset + char_index) as usize]; + + self.pending_unique_index += node.number as u16; + self.overconsumed_code_points += 1; + if node.end_of_word { + self.pending_unique_index += 1; + self.last_matched_unique_index = self.pending_unique_index; + self.ends_with_semicolon = c == b';'; + self.overconsumed_code_points = 0; + } + self.search_state = SearchState::DafsaChildren { + start_index: node.child_index, + len: node.children_len, + }; + true + } + SearchState::DafsaChildren { start_index, len } => { + for i in 0..len as u16 { + let node = &DAFSA_NODES[(start_index + i) as usize]; + if node.character == c { + self.pending_unique_index += node.number as u16; + self.overconsumed_code_points += 1; + if node.end_of_word { + self.pending_unique_index += 1; + self.last_matched_unique_index = self.pending_unique_index; + self.ends_with_semicolon = c == b';'; + self.overconsumed_code_points = 0; + } + self.search_state = SearchState::DafsaChildren { + start_index: node.child_index, + len: node.children_len, + }; + return true; + } + } + false + } + } + } + + /// Returns the codepoints for the longest match found, or None if no match. + /// Returns (first_codepoint, second_codepoint). second is 0 if there is no second codepoint. + pub fn code_points(&self) -> Option<(u32, u32)> { + if self.last_matched_unique_index == 0 { + return None; + } + let entry = &CODEPOINTS_LOOKUP[(self.last_matched_unique_index - 1) as usize]; + Some((entry.0, entry.1.value())) + } + + /// Number of characters consumed past the longest match. + pub fn overconsumed_code_points(&self) -> u8 { + self.overconsumed_code_points + } + + /// Whether the longest match ended with a semicolon. + pub fn last_match_ends_with_semicolon(&self) -> bool { + self.ends_with_semicolon + } +} + +impl Default for NamedCharacterReferenceMatcher { + fn default() -> Self { + Self::new() + } +} diff --git a/Libraries/LibWeb/HTML/Parser/Rust/src/interned_names.rs b/Libraries/LibWeb/HTML/Parser/Rust/src/interned_names.rs new file mode 100644 index 0000000000..ed7af9fb51 --- /dev/null +++ b/Libraries/LibWeb/HTML/Parser/Rust/src/interned_names.rs @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2026-present, the Ladybird developers. + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +//! Lookup tables for common HTML tag and attribute names. +//! +//! The tables are generated at build time from LibWeb's TagNames.h and +//! AttributeNames.h headers so they stay in sync with the C++ side. Each +//! name gets a 1-based u16 id; id 0 is reserved for "not interned" so the +//! FFI layer can carry either an id or raw bytes in the same slot. +//! +//! The lookup functions themselves are generated as a `match` on byte +//! length and exact byte sequence, which rustc compiles to a jump table +//! plus direct memcmp per length bucket. This is dramatically faster than +//! a HashMap lookup with the default cryptographic hasher. + +include!(concat!(env!("OUT_DIR"), "/interned_names_generated.rs")); + +/// Look up a tag name. Returns 0 if not interned. +#[inline] +pub fn lookup_tag_name(bytes: &[u8]) -> u16 { + lookup_tag_name_generated(bytes) +} + +/// Look up an attribute name. Returns 0 if not interned. +#[inline] +pub fn lookup_attr_name(bytes: &[u8]) -> u16 { + lookup_attr_name_generated(bytes) +} + +/// Number of interned tag names (for C++ table sizing at startup). +#[inline] +pub fn tag_name_count() -> usize { + INTERNED_TAG_NAMES.len() +} + +/// Number of interned attribute names. +#[inline] +pub fn attr_name_count() -> usize { + INTERNED_ATTR_NAMES.len() +} + +/// Fetch a tag name by id (1-based). Returns None for id 0 or out of range. +#[inline] +pub fn tag_name_by_id(id: u16) -> Option<&'static [u8]> { + if id == 0 { + return None; + } + INTERNED_TAG_NAMES.get((id - 1) as usize).copied() +} + +/// Fetch an attribute name by id (1-based). Returns None for id 0 or out of range. +#[inline] +pub fn attr_name_by_id(id: u16) -> Option<&'static [u8]> { + if id == 0 { + return None; + } + INTERNED_ATTR_NAMES.get((id - 1) as usize).copied() +} diff --git a/Libraries/LibWeb/HTML/Parser/Rust/src/lib.rs b/Libraries/LibWeb/HTML/Parser/Rust/src/lib.rs new file mode 100644 index 0000000000..262f7efc17 --- /dev/null +++ b/Libraries/LibWeb/HTML/Parser/Rust/src/lib.rs @@ -0,0 +1,702 @@ +/* + * Copyright (c) 2026-present, the Ladybird developers. + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +pub mod entities; +pub mod interned_names; +pub mod token; +pub mod tokenizer; + +use std::ptr; +use token::{Attribute, Position, TokenPayload, TokenType}; +use tokenizer::{HtmlTokenizer, State}; + +/// Opaque handle for the Rust tokenizer, passed across the FFI boundary. +pub struct RustFfiTokenizerHandle { + tokenizer: HtmlTokenizer, + /// Temporary storage for the last token's string data, kept alive + /// so that pointers in RustFfiToken remain valid until the next call. + last_tag_name: Vec, + last_comment: Vec, + last_doctype_name: Vec, + last_public_id: Vec, + last_system_id: Vec, + last_attributes: Vec, + last_attr_names: Vec>, + last_attr_values: Vec>, + last_unparsed_input: Vec, +} + +/// C-compatible token representation. +/// +/// Pointer fields (`tag_name_ptr`, `comment_ptr`, doctype/attribute pointers) +/// borrow into buffers owned by the `RustFfiTokenizerHandle` that produced this +/// token. They remain valid only until the next call into the tokenizer for +/// that handle (any of `next_token`, `insert_input`, `destroy`, etc.). +/// Callers must consume the data before making the next call. +#[repr(C)] +pub struct RustFfiToken { + pub token_type: u8, + pub code_point: u32, + pub self_closing: bool, + + /// If nonzero, an interned tag-name id (1-based index into + /// `interned_names::INTERNED_TAG_NAMES`). When set, `tag_name_ptr` / + /// `tag_name_len` are unused and the C++ side uses its parallel + /// FlyString table directly. + pub tag_name_id: u16, + pub tag_name_ptr: *const u8, + pub tag_name_len: usize, + + pub comment_ptr: *const u8, + pub comment_len: usize, + + pub doctype_name_ptr: *const u8, + pub doctype_name_len: usize, + pub public_id_ptr: *const u8, + pub public_id_len: usize, + pub system_id_ptr: *const u8, + pub system_id_len: usize, + pub force_quirks: bool, + pub missing_name: bool, + pub missing_public_id: bool, + pub missing_system_id: bool, + + pub attributes_ptr: *const RustFfiAttribute, + pub attributes_len: usize, + + pub start_line: u64, + pub start_column: u64, + pub end_line: u64, + pub end_column: u64, +} + +/// C-compatible attribute representation. +#[repr(C)] +pub struct RustFfiAttribute { + /// If nonzero, an interned attribute-name id (1-based index into + /// `interned_names::INTERNED_ATTR_NAMES`). When set, `name_ptr` / + /// `name_len` are unused. + pub name_id: u16, + pub name_ptr: *const u8, + pub name_len: usize, + pub value_ptr: *const u8, + pub value_len: usize, + pub name_start_line: u64, + pub name_start_column: u64, + pub name_end_line: u64, + pub name_end_column: u64, + pub value_start_line: u64, + pub value_start_column: u64, + pub value_end_line: u64, + pub value_end_column: u64, +} + +impl Default for RustFfiToken { + fn default() -> Self { + RustFfiToken { + token_type: TokenType::Invalid as u8, + code_point: 0, + self_closing: false, + tag_name_id: 0, + tag_name_ptr: ptr::null(), + tag_name_len: 0, + comment_ptr: ptr::null(), + comment_len: 0, + doctype_name_ptr: ptr::null(), + doctype_name_len: 0, + public_id_ptr: ptr::null(), + public_id_len: 0, + system_id_ptr: ptr::null(), + system_id_len: 0, + force_quirks: false, + missing_name: true, + missing_public_id: true, + missing_system_id: true, + attributes_ptr: ptr::null(), + attributes_len: 0, + start_line: 0, + start_column: 0, + end_line: 0, + end_column: 0, + } + } +} + +fn position_to_ffi(pos: &Position) -> (u64, u64) { + (pos.line, pos.column) +} + +/// Create a new Rust HTML tokenizer from UTF-32 code points. +/// +/// # Safety +/// `input` must point to `len` valid u32 values. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn rust_html_tokenizer_create(input: *const u32, len: usize) -> *mut RustFfiTokenizerHandle { + let code_points = if input.is_null() || len == 0 { + Vec::new() + } else { + unsafe { std::slice::from_raw_parts(input, len) }.to_vec() + }; + + make_handle(HtmlTokenizer::new(code_points)) +} + +/// Create a new Rust HTML tokenizer directly from a UTF-8 byte buffer. +/// Rust decodes the bytes to code points internally, skipping the C++ +/// side's 4x-expanded Vec copy. +/// +/// # Safety +/// `bytes` must point to `len` valid UTF-8 bytes. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn rust_html_tokenizer_create_from_utf8( + bytes: *const u8, + len: usize, +) -> *mut RustFfiTokenizerHandle { + let code_points = if bytes.is_null() || len == 0 { + Vec::new() + } else { + let slice = unsafe { std::slice::from_raw_parts(bytes, len) }; + decode_utf8_to_u32(slice) + }; + + make_handle(HtmlTokenizer::new(code_points)) +} + +/// Expand a UTF-8 byte slice into a `Vec` of code points. For the +/// dominant ASCII case we use a tight loop with a single unchecked +/// write per byte and only fall back to `str::chars()` decoding when +/// we see a continuation byte. The caller is responsible for ensuring +/// the input is valid UTF-8. +fn decode_utf8_to_u32(bytes: &[u8]) -> Vec { + let mut out: Vec = Vec::with_capacity(bytes.len()); + // SAFETY: we reserved `bytes.len()` slots and will only write up to + // that many u32s (one per input byte; multi-byte sequences produce + // fewer u32s than bytes, so we stay within bounds). + let out_ptr = out.as_mut_ptr(); + let mut write_idx: usize = 0; + let mut i: usize = 0; + let n = bytes.len(); + while i < n { + let b = bytes[i]; + if b < 0x80 { + unsafe { std::ptr::write(out_ptr.add(write_idx), b as u32) }; + write_idx += 1; + i += 1; + } else { + // Slow path: decode one code point via str::chars. + // SAFETY: the input is valid UTF-8 by precondition. + let tail = unsafe { std::str::from_utf8_unchecked(&bytes[i..]) }; + if let Some(ch) = tail.chars().next() { + unsafe { std::ptr::write(out_ptr.add(write_idx), ch as u32) }; + write_idx += 1; + i += ch.len_utf8(); + } else { + break; + } + } + } + // SAFETY: we wrote exactly `write_idx` elements, all in-bounds. + unsafe { out.set_len(write_idx) }; + out +} + +fn make_handle(tokenizer: HtmlTokenizer) -> *mut RustFfiTokenizerHandle { + let handle = Box::new(RustFfiTokenizerHandle { + tokenizer, + last_tag_name: Vec::new(), + last_comment: Vec::new(), + last_doctype_name: Vec::new(), + last_public_id: Vec::new(), + last_system_id: Vec::new(), + last_attributes: Vec::new(), + last_attr_names: Vec::new(), + last_attr_values: Vec::new(), + last_unparsed_input: Vec::new(), + }); + + Box::into_raw(handle) +} + +/// Get the next token from the tokenizer. +/// Returns true if a token was produced, false if no more tokens. +/// +/// # Safety +/// `handle` must be a valid pointer from `rust_html_tokenizer_create`. +/// `out` must be a valid pointer to an RustFfiToken. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn rust_html_tokenizer_next_token( + handle: *mut RustFfiTokenizerHandle, + out: *mut RustFfiToken, + stop_at_insertion_point: bool, + cdata_allowed: bool, +) -> bool { + if handle.is_null() || out.is_null() { + return false; + } + let handle = unsafe { &mut *handle }; + let out = unsafe { &mut *out }; + + // Fast-path text character in the Data state. For ASCII text runs + // this skips the whole state-machine function call plus Token + // construction/drop. Most real HTML is dominated by such runs. + if !stop_at_insertion_point && let Some((cp, pos)) = handle.tokenizer.try_fast_data_char() { + out.token_type = TokenType::Character as u8; + out.code_point = cp; + out.start_line = pos.line; + out.start_column = pos.column; + out.end_line = pos.line; + out.end_column = pos.column; + return true; + } + + next_token_slow(handle, out, stop_at_insertion_point, cdata_allowed) +} + +/// The state-machine and marshalling portion of the FFI token fetch, +/// pulled out of `rust_html_tokenizer_next_token` so that the fast +/// Data-state path in the outer function keeps a tiny stack frame +/// and a short prologue. Marked `#[inline(never)]` so the compiler +/// doesn't re-inline the whole thing and re-inflate the outer frame. +#[inline(never)] +fn next_token_slow( + handle: &mut RustFfiTokenizerHandle, + out: &mut RustFfiToken, + stop_at_insertion_point: bool, + cdata_allowed: bool, +) -> bool { + let token = match handle.tokenizer.next_token(stop_at_insertion_point, cdata_allowed) { + Some(t) => t, + None => return false, + }; + + // Fast path: Character and EOF tokens only need a few fields. + // Skip zeroing ~200 bytes of RustFfiToken for 95%+ of all tokens. + match token.token_type { + TokenType::Character | TokenType::EndOfFile => { + out.token_type = token.token_type as u8; + out.code_point = token.code_point; + out.start_line = token.start_position.line; + out.start_column = token.start_position.column; + out.end_line = token.end_position.line; + out.end_column = token.end_position.column; + return true; + } + _ => {} + } + + *out = RustFfiToken::default(); + out.token_type = token.token_type as u8; + + let (sl, sc) = position_to_ffi(&token.start_position); + let (el, ec) = position_to_ffi(&token.end_position); + out.start_line = sl; + out.start_column = sc; + out.end_line = el; + out.end_column = ec; + + // Store string data in handle so pointers stay valid. + match token.payload { + TokenPayload::Tag { + tag_name, + tag_name_id, + self_closing, + attributes, + } => { + out.self_closing = self_closing; + // Tokenizer already resolved intern ids, so we trust tag_name_id. + out.tag_name_id = tag_name_id; + if tag_name_id == 0 { + handle.last_tag_name = tag_name.into_bytes(); + out.tag_name_ptr = handle.last_tag_name.as_ptr(); + out.tag_name_len = handle.last_tag_name.len(); + } + + // Convert attributes. Move owned Strings out of each Attribute + // instead of cloning, then point the FfiAttribute at the stable + // heap bytes that now live in handle.last_attr_{names,values} + // (unless the name was interned, in which case skip the copy). + handle.last_attr_names.clear(); + handle.last_attr_values.clear(); + handle.last_attributes.clear(); + handle.last_attr_names.reserve(attributes.len()); + handle.last_attr_values.reserve(attributes.len()); + handle.last_attributes.reserve(attributes.len()); + for attr in attributes { + let Attribute { + local_name, + local_name_id, + value, + name_start_position, + name_end_position, + value_start_position, + value_end_position, + } = attr; + if local_name_id == 0 { + handle.last_attr_names.push(local_name.into_bytes()); + } else { + // Keep the slot aligned with last_attr_values so index math stays valid. + handle.last_attr_names.push(Vec::new()); + } + handle.last_attr_values.push(value.into_bytes()); + let last_idx = handle.last_attr_names.len() - 1; + let name_bytes = &handle.last_attr_names[last_idx]; + let value_bytes = &handle.last_attr_values[last_idx]; + handle.last_attributes.push(RustFfiAttribute { + name_id: local_name_id, + name_ptr: if local_name_id == 0 { + name_bytes.as_ptr() + } else { + ptr::null() + }, + name_len: if local_name_id == 0 { name_bytes.len() } else { 0 }, + value_ptr: value_bytes.as_ptr(), + value_len: value_bytes.len(), + name_start_line: name_start_position.line, + name_start_column: name_start_position.column, + name_end_line: name_end_position.line, + name_end_column: name_end_position.column, + value_start_line: value_start_position.line, + value_start_column: value_start_position.column, + value_end_line: value_end_position.line, + value_end_column: value_end_position.column, + }); + } + out.attributes_ptr = handle.last_attributes.as_ptr(); + out.attributes_len = handle.last_attributes.len(); + } + TokenPayload::Comment(data) => { + handle.last_comment = data.into_bytes(); + out.comment_ptr = handle.last_comment.as_ptr(); + out.comment_len = handle.last_comment.len(); + } + TokenPayload::Doctype(doctype) => { + handle.last_doctype_name = doctype.name.into_bytes(); + handle.last_public_id = doctype.public_identifier.into_bytes(); + handle.last_system_id = doctype.system_identifier.into_bytes(); + out.doctype_name_ptr = handle.last_doctype_name.as_ptr(); + out.doctype_name_len = handle.last_doctype_name.len(); + out.public_id_ptr = handle.last_public_id.as_ptr(); + out.public_id_len = handle.last_public_id.len(); + out.system_id_ptr = handle.last_system_id.as_ptr(); + out.system_id_len = handle.last_system_id.len(); + out.force_quirks = doctype.force_quirks; + out.missing_name = doctype.missing_name; + out.missing_public_id = doctype.missing_public_identifier; + out.missing_system_id = doctype.missing_system_identifier; + } + TokenPayload::None => {} + } + + true +} + +/// Switch the tokenizer to a new state. +/// +/// # Safety +/// `handle` must be a valid pointer from `rust_html_tokenizer_create`. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn rust_html_tokenizer_switch_state(handle: *mut RustFfiTokenizerHandle, state: u8) { + if handle.is_null() { + return; + } + let handle = unsafe { &mut *handle }; + // The state values must match the State enum order. Bound-checked against + // the last known variant so an out-of-range value is rejected instead of + // producing UB via transmute to an invalid discriminant. + const STATE_COUNT: u8 = State::NumericCharacterReferenceEnd as u8 + 1; + if state >= STATE_COUNT { + return; + } + let state: State = unsafe { std::mem::transmute(state) }; + handle.tokenizer.switch_to(state); +} + +/// Insert input (as UTF-32 code points) at the current insertion point. +/// +/// # Safety +/// `handle` must be a valid pointer. `input` must point to `len` valid u32 values. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn rust_html_tokenizer_insert_input( + handle: *mut RustFfiTokenizerHandle, + input: *const u32, + len: usize, +) { + if handle.is_null() { + return; + } + let handle = unsafe { &mut *handle }; + let code_points = if input.is_null() || len == 0 { + &[] + } else { + unsafe { std::slice::from_raw_parts(input, len) } + }; + handle.tokenizer.insert_input_at_insertion_point(code_points); +} + +/// Append input (as UTF-32 code points) to the tokenizer input stream. +/// +/// # Safety +/// `handle` must be a valid pointer. `input` must point to `len` valid u32 values. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn rust_html_tokenizer_append_input( + handle: *mut RustFfiTokenizerHandle, + input: *const u32, + len: usize, +) { + if handle.is_null() { + return; + } + let handle = unsafe { &mut *handle }; + let code_points = if input.is_null() || len == 0 { + &[] + } else { + unsafe { std::slice::from_raw_parts(input, len) } + }; + handle.tokenizer.append_input(code_points); +} + +/// Get the tokenizer input that has not been consumed yet. +/// +/// # Safety +/// `handle`, `out_ptr`, and `out_len` must be valid pointers. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn rust_html_tokenizer_unparsed_input( + handle: *mut RustFfiTokenizerHandle, + out_ptr: *mut *const u8, + out_len: *mut usize, +) { + if handle.is_null() || out_ptr.is_null() || out_len.is_null() { + return; + } + let handle = unsafe { &mut *handle }; + handle.last_unparsed_input = handle.tokenizer.unparsed_input().into_bytes(); + unsafe { + *out_ptr = handle.last_unparsed_input.as_ptr(); + *out_len = handle.last_unparsed_input.len(); + } +} + +/// Compact already-tokenized input after the parser has consumed a chunk. +/// +/// # Safety +/// `handle` must be a valid pointer. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn rust_html_tokenizer_parser_did_run(handle: *mut RustFfiTokenizerHandle) { + if handle.is_null() { + return; + } + let handle = unsafe { &mut *handle }; + handle.tokenizer.parser_did_run(); +} + +/// # Safety +/// `handle` must be a valid pointer. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn rust_html_tokenizer_store_insertion_point(handle: *mut RustFfiTokenizerHandle) { + if handle.is_null() { + return; + } + let handle = unsafe { &mut *handle }; + handle.tokenizer.store_insertion_point(); +} + +/// # Safety +/// `handle` must be a valid pointer. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn rust_html_tokenizer_restore_insertion_point(handle: *mut RustFfiTokenizerHandle) { + if handle.is_null() { + return; + } + let handle = unsafe { &mut *handle }; + handle.tokenizer.restore_insertion_point(); +} + +/// # Safety +/// `handle` must be a valid pointer. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn rust_html_tokenizer_update_insertion_point(handle: *mut RustFfiTokenizerHandle) { + if handle.is_null() { + return; + } + let handle = unsafe { &mut *handle }; + handle.tokenizer.update_insertion_point(); +} + +/// # Safety +/// `handle` must be a valid pointer. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn rust_html_tokenizer_undefine_insertion_point(handle: *mut RustFfiTokenizerHandle) { + if handle.is_null() { + return; + } + let handle = unsafe { &mut *handle }; + handle.tokenizer.undefine_insertion_point(); +} + +/// # Safety +/// `handle` must be a valid pointer. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn rust_html_tokenizer_is_insertion_point_defined(handle: *mut RustFfiTokenizerHandle) -> bool { + if handle.is_null() { + return false; + } + let handle = unsafe { &mut *handle }; + handle.tokenizer.is_insertion_point_defined() +} + +/// # Safety +/// `handle` must be a valid pointer. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn rust_html_tokenizer_is_insertion_point_reached(handle: *mut RustFfiTokenizerHandle) -> bool { + if handle.is_null() { + return false; + } + let handle = unsafe { &mut *handle }; + handle.tokenizer.is_insertion_point_reached() +} + +/// # Safety +/// `handle` must be a valid pointer. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn rust_html_tokenizer_set_blocked(handle: *mut RustFfiTokenizerHandle, blocked: bool) { + if handle.is_null() { + return; + } + let handle = unsafe { &mut *handle }; + handle.tokenizer.set_blocked(blocked); +} + +/// # Safety +/// `handle` must be a valid pointer. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn rust_html_tokenizer_is_blocked(handle: *mut RustFfiTokenizerHandle) -> bool { + if handle.is_null() { + return false; + } + let handle = unsafe { &mut *handle }; + handle.tokenizer.is_blocked() +} + +/// # Safety +/// `handle` must be a valid pointer. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn rust_html_tokenizer_set_input_stream_closed( + handle: *mut RustFfiTokenizerHandle, + closed: bool, +) { + if handle.is_null() { + return; + } + let handle = unsafe { &mut *handle }; + handle.tokenizer.set_input_stream_closed(closed); +} + +/// # Safety +/// `handle` must be a valid pointer. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn rust_html_tokenizer_insert_eof(handle: *mut RustFfiTokenizerHandle) { + if handle.is_null() { + return; + } + let handle = unsafe { &mut *handle }; + handle.tokenizer.insert_eof(); +} + +/// # Safety +/// `handle` must be a valid pointer. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn rust_html_tokenizer_is_eof_inserted(handle: *mut RustFfiTokenizerHandle) -> bool { + if handle.is_null() { + return false; + } + let handle = unsafe { &mut *handle }; + handle.tokenizer.is_eof_inserted() +} + +/// # Safety +/// `handle` must be a valid pointer. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn rust_html_tokenizer_abort(handle: *mut RustFfiTokenizerHandle) { + if handle.is_null() { + return; + } + let handle = unsafe { &mut *handle }; + handle.tokenizer.abort(); +} + +/// Destroy a Rust HTML tokenizer. +/// +/// # Safety +/// `handle` must be a valid pointer from `rust_html_tokenizer_create`, +/// and must not be used after this call. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn rust_html_tokenizer_destroy(handle: *mut RustFfiTokenizerHandle) { + if !handle.is_null() { + drop(unsafe { Box::from_raw(handle) }); + } +} + +// -- Interned name table enumeration -------------------------------------- +// +// The C++ side builds a parallel FlyString array at static-init time by +// enumerating the Rust-owned list once. Ids are 1-based; id 0 is reserved +// for "not interned" in the per-token FFI struct. + +/// Number of interned HTML tag names known to the Rust tokenizer. +#[unsafe(no_mangle)] +pub extern "C" fn rust_html_tokenizer_interned_tag_name_count() -> usize { + interned_names::tag_name_count() +} + +/// Number of interned HTML attribute names known to the Rust tokenizer. +#[unsafe(no_mangle)] +pub extern "C" fn rust_html_tokenizer_interned_attr_name_count() -> usize { + interned_names::attr_name_count() +} + +/// Write the bytes and length of the interned tag name with the given +/// 1-based id to the caller-provided out parameters. On unknown ids the +/// out parameters are set to (null, 0). +/// +/// # Safety +/// `out_ptr` and `out_len` must be valid pointers. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn rust_html_tokenizer_interned_tag_name(id: u16, out_ptr: *mut *const u8, out_len: *mut usize) { + if out_ptr.is_null() || out_len.is_null() { + return; + } + match interned_names::tag_name_by_id(id) { + Some(bytes) => unsafe { + *out_ptr = bytes.as_ptr(); + *out_len = bytes.len(); + }, + None => unsafe { + *out_ptr = ptr::null(); + *out_len = 0; + }, + } +} + +/// Same as `rust_html_tokenizer_interned_tag_name` for attribute names. +/// +/// # Safety +/// `out_ptr` and `out_len` must be valid pointers. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn rust_html_tokenizer_interned_attr_name(id: u16, out_ptr: *mut *const u8, out_len: *mut usize) { + if out_ptr.is_null() || out_len.is_null() { + return; + } + match interned_names::attr_name_by_id(id) { + Some(bytes) => unsafe { + *out_ptr = bytes.as_ptr(); + *out_len = bytes.len(); + }, + None => unsafe { + *out_ptr = ptr::null(); + *out_len = 0; + }, + } +} diff --git a/Libraries/LibWeb/HTML/Parser/Rust/src/token.rs b/Libraries/LibWeb/HTML/Parser/Rust/src/token.rs new file mode 100644 index 0000000000..592aaf5257 --- /dev/null +++ b/Libraries/LibWeb/HTML/Parser/Rust/src/token.rs @@ -0,0 +1,202 @@ +/* + * Copyright (c) 2026-present, the Ladybird developers. + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +/// Source position in the input. +#[derive(Clone, Copy, Debug, Default)] +pub struct Position { + pub line: u64, + pub column: u64, +} + +/// A single attribute on a start or end tag token. +/// +/// If `local_name_id` is non-zero it is an index into +/// `interned_names::INTERNED_ATTR_NAMES` and `local_name` is unused. +/// Otherwise `local_name` holds the owned bytes. +#[derive(Clone, Debug, Default)] +pub struct Attribute { + pub local_name: String, + pub local_name_id: u16, + pub value: String, + pub name_start_position: Position, + pub name_end_position: Position, + pub value_start_position: Position, + pub value_end_position: Position, +} + +/// Data specific to DOCTYPE tokens. +#[derive(Clone, Debug, Default)] +pub struct DoctypeData { + pub name: String, + pub public_identifier: String, + pub system_identifier: String, + pub missing_name: bool, + pub missing_public_identifier: bool, + pub missing_system_identifier: bool, + pub force_quirks: bool, +} + +/// The type of an HTML token. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +#[repr(u8)] +pub enum TokenType { + #[default] + Invalid = 0, + Doctype = 1, + StartTag = 2, + EndTag = 3, + Comment = 4, + Character = 5, + EndOfFile = 6, +} + +/// Type-specific data for an HTML token. +/// +/// If `tag_name_id` is non-zero it is an index into +/// `interned_names::INTERNED_TAG_NAMES` and `tag_name` is unused. +/// Otherwise `tag_name` holds the owned bytes. +#[derive(Clone, Debug, Default)] +pub enum TokenPayload { + #[default] + None, + Tag { + tag_name: String, + tag_name_id: u16, + self_closing: bool, + attributes: Vec, + }, + Comment(String), + Doctype(Box), +} + +/// An HTML token produced by the tokenizer. +#[derive(Clone, Debug, Default)] +pub struct Token { + pub token_type: TokenType, + pub code_point: u32, + pub payload: TokenPayload, + pub start_position: Position, + pub end_position: Position, +} + +impl Token { + pub fn new_character(code_point: u32) -> Self { + Token { + token_type: TokenType::Character, + code_point, + ..Default::default() + } + } + + pub fn new_eof() -> Self { + Token { + token_type: TokenType::EndOfFile, + ..Default::default() + } + } + + /// Return the tag name as a &str. For interned names this resolves + /// through the interned name table; for un-interned names it returns + /// the owned String's contents. + #[inline(always)] + pub fn tag_name(&self) -> &str { + match &self.payload { + TokenPayload::Tag { + tag_name, tag_name_id, .. + } => { + if *tag_name_id != 0 { + // SAFETY: interned names are compile-time ASCII byte + // literals in interned_names_generated.rs, so they are + // always valid UTF-8. + match crate::interned_names::tag_name_by_id(*tag_name_id) { + Some(bytes) => unsafe { std::str::from_utf8_unchecked(bytes) }, + None => "", + } + } else { + tag_name + } + } + _ => "", + } + } + + #[inline(always)] + pub fn tag_name_mut(&mut self) -> &mut String { + match &mut self.payload { + TokenPayload::Tag { + tag_name, tag_name_id, .. + } => { + *tag_name_id = 0; + tag_name + } + _ => panic!("tag_name_mut called on non-tag token"), + } + } + + /// Set the tag name to an interned id, clearing any previously stored + /// owned name. + #[inline(always)] + pub fn set_tag_name_id(&mut self, id: u16) { + match &mut self.payload { + TokenPayload::Tag { + tag_name, tag_name_id, .. + } => { + tag_name.clear(); + *tag_name_id = id; + } + _ => panic!("set_tag_name_id called on non-tag token"), + } + } + + /// Returns the interned tag-name id, or 0 if the name is un-interned. + #[inline(always)] + pub fn tag_name_id(&self) -> u16 { + match &self.payload { + TokenPayload::Tag { tag_name_id, .. } => *tag_name_id, + _ => 0, + } + } + + #[inline(always)] + pub fn set_self_closing(&mut self, value: bool) { + match &mut self.payload { + TokenPayload::Tag { self_closing, .. } => *self_closing = value, + _ => panic!("set_self_closing called on non-tag token"), + } + } + + #[inline(always)] + pub fn is_self_closing(&self) -> bool { + match &self.payload { + TokenPayload::Tag { self_closing, .. } => *self_closing, + _ => false, + } + } + + #[inline(always)] + pub fn attributes_mut(&mut self) -> &mut Vec { + match &mut self.payload { + TokenPayload::Tag { attributes, .. } => attributes, + _ => panic!("attributes_mut called on non-tag token"), + } + } + + #[inline(always)] + pub fn set_comment_data(&mut self, data: String) { + match &mut self.payload { + TokenPayload::Comment(s) => *s = data, + _ => panic!("set_comment_data called on non-comment token"), + } + } + + #[inline(always)] + pub fn doctype_data_mut(&mut self) -> &mut DoctypeData { + match &mut self.payload { + TokenPayload::Doctype(dd) => dd, + _ => panic!("doctype_data_mut called on non-doctype token"), + } + } +} diff --git a/Libraries/LibWeb/HTML/Parser/Rust/src/tokenizer.rs b/Libraries/LibWeb/HTML/Parser/Rust/src/tokenizer.rs new file mode 100644 index 0000000000..02de3347e1 --- /dev/null +++ b/Libraries/LibWeb/HTML/Parser/Rust/src/tokenizer.rs @@ -0,0 +1,3152 @@ +/* + * Copyright (c) 2026-present, the Ladybird developers. + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +use std::collections::VecDeque; + +use crate::entities::NamedCharacterReferenceMatcher; +use crate::token::{Attribute, DoctypeData, Position, Token, TokenPayload, TokenType}; + +/// Tokenizer states per the WHATWG HTML spec section 13.2.5. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[repr(u8)] +pub enum State { + Data = 0, + RCDATA, + RAWTEXT, + ScriptData, + PLAINTEXT, + TagOpen, + EndTagOpen, + TagName, + RCDATALessThanSign, + RCDATAEndTagOpen, + RCDATAEndTagName, + RAWTEXTLessThanSign, + RAWTEXTEndTagOpen, + RAWTEXTEndTagName, + ScriptDataLessThanSign, + ScriptDataEndTagOpen, + ScriptDataEndTagName, + ScriptDataEscapeStart, + ScriptDataEscapeStartDash, + ScriptDataEscaped, + ScriptDataEscapedDash, + ScriptDataEscapedDashDash, + ScriptDataEscapedLessThanSign, + ScriptDataEscapedEndTagOpen, + ScriptDataEscapedEndTagName, + ScriptDataDoubleEscapeStart, + ScriptDataDoubleEscaped, + ScriptDataDoubleEscapedDash, + ScriptDataDoubleEscapedDashDash, + ScriptDataDoubleEscapedLessThanSign, + ScriptDataDoubleEscapeEnd, + BeforeAttributeName, + AttributeName, + AfterAttributeName, + BeforeAttributeValue, + AttributeValueDoubleQuoted, + AttributeValueSingleQuoted, + AttributeValueUnquoted, + AfterAttributeValueQuoted, + SelfClosingStartTag, + BogusComment, + MarkupDeclarationOpen, + CommentStart, + CommentStartDash, + Comment, + CommentLessThanSign, + CommentLessThanSignBang, + CommentLessThanSignBangDash, + CommentLessThanSignBangDashDash, + CommentEndDash, + CommentEnd, + CommentEndBang, + DOCTYPE, + BeforeDOCTYPEName, + DOCTYPEName, + AfterDOCTYPEName, + AfterDOCTYPEPublicKeyword, + BeforeDOCTYPEPublicIdentifier, + DOCTYPEPublicIdentifierDoubleQuoted, + DOCTYPEPublicIdentifierSingleQuoted, + AfterDOCTYPEPublicIdentifier, + BetweenDOCTYPEPublicAndSystemIdentifiers, + AfterDOCTYPESystemKeyword, + BeforeDOCTYPESystemIdentifier, + DOCTYPESystemIdentifierDoubleQuoted, + DOCTYPESystemIdentifierSingleQuoted, + AfterDOCTYPESystemIdentifier, + BogusDOCTYPE, + CDATASection, + CDATASectionBracket, + CDATASectionEnd, + CharacterReference, + NamedCharacterReference, + AmbiguousAmpersand, + NumericCharacterReference, + HexadecimalCharacterReferenceStart, + DecimalCharacterReferenceStart, + HexadecimalCharacterReference, + DecimalCharacterReference, + NumericCharacterReferenceEnd, +} + +/// The HTML tokenizer state machine. +/// +/// Implements the WHATWG HTML tokenizer specification (section 13.2.5). +pub struct HtmlTokenizer { + pub state: State, + return_state: State, + pub input: Vec, + pub current_offset: usize, + prev_offset: usize, + current_token: Token, + current_builder: String, + pub queued_tokens: VecDeque, + temporary_buffer: Vec, + character_reference_code: u32, + last_emitted_start_tag_name: Option, + source_positions: Vec, + // Mirror of the most recent entry in source_positions, kept in sync + // with it on the slow path and updated directly on the fast + // character-emission path so we don't have to push into the Vec per + // character in Data-state text runs. + current_line: u64, + current_column: u64, + has_emitted_eof: bool, + aborted: bool, + insertion_point: Option, + old_insertion_points: Vec>, + explicit_eof_inserted: bool, + input_stream_closed: bool, + blocked: bool, + stop_at_insertion_point: bool, + cdata_allowed: bool, + entity_matcher: NamedCharacterReferenceMatcher, +} + +#[inline] +fn is_ascii_alpha(c: u32) -> bool { + matches!(c, 0x41..=0x5A | 0x61..=0x7A) +} + +#[inline] +fn is_ascii_upper_alpha(c: u32) -> bool { + matches!(c, 0x41..=0x5A) +} + +#[inline] +fn is_ascii_digit(c: u32) -> bool { + matches!(c, 0x30..=0x39) +} + +#[inline] +fn is_ascii_hex_digit(c: u32) -> bool { + matches!(c, 0x30..=0x39 | 0x41..=0x46 | 0x61..=0x66) +} + +#[inline] +fn is_ascii_alphanumeric(c: u32) -> bool { + is_ascii_alpha(c) || is_ascii_digit(c) +} + +#[inline] +fn to_ascii_lowercase(c: u32) -> u32 { + if is_ascii_upper_alpha(c) { c + 0x20 } else { c } +} + +#[inline] +fn is_whitespace(c: u32) -> bool { + matches!(c, 0x09 | 0x0A | 0x0C | 0x20) +} + +/// Numeric character reference replacement table (spec section 13.2.5.5). +fn numeric_char_ref_replacement(code: u32) -> Option { + match code { + 0x80 => Some(0x20AC), + 0x82 => Some(0x201A), + 0x83 => Some(0x0192), + 0x84 => Some(0x201E), + 0x85 => Some(0x2026), + 0x86 => Some(0x2020), + 0x87 => Some(0x2021), + 0x88 => Some(0x02C6), + 0x89 => Some(0x2030), + 0x8A => Some(0x0160), + 0x8B => Some(0x2039), + 0x8C => Some(0x0152), + 0x8E => Some(0x017D), + 0x91 => Some(0x2018), + 0x92 => Some(0x2019), + 0x93 => Some(0x201C), + 0x94 => Some(0x201D), + 0x95 => Some(0x2022), + 0x96 => Some(0x2013), + 0x97 => Some(0x2014), + 0x98 => Some(0x02DC), + 0x99 => Some(0x2122), + 0x9A => Some(0x0161), + 0x9B => Some(0x203A), + 0x9C => Some(0x0153), + 0x9E => Some(0x017E), + 0x9F => Some(0x0178), + _ => None, + } +} + +/// Check if a code point is a Unicode surrogate. +fn is_surrogate(c: u32) -> bool { + (0xD800..=0xDFFF).contains(&c) +} + +/// Check if a code point is a Unicode noncharacter. +fn is_noncharacter(c: u32) -> bool { + (0xFDD0..=0xFDEF).contains(&c) || matches!(c & 0xFFFF, 0xFFFE | 0xFFFF) +} + +#[inline] +fn is_ascii_lower_alpha(c: u32) -> bool { + matches!(c, 0x61..=0x7A) +} + +/// Push a code point onto a `String` as UTF-8 without paying +/// `char::from_u32`'s surrogate check or `String::push`'s len_utf8 branch +/// for the common ASCII case. For `cp < 0x80` this is a single byte push +/// to the underlying Vec; for wider code points we fall back to the +/// normal slow path. +#[inline(always)] +fn push_code_point(buf: &mut String, cp: u32) { + if cp < 0x80 { + // SAFETY: ASCII is always valid UTF-8. + unsafe { buf.as_mut_vec().push(cp as u8) }; + } else { + buf.push(char::from_u32(cp).unwrap_or('\u{FFFD}')); + } +} + +impl HtmlTokenizer { + /// Create a new tokenizer for the given input (as UTF-32 code points). + pub fn new(input: Vec) -> Self { + HtmlTokenizer { + state: State::Data, + return_state: State::Data, + input, + current_offset: 0, + prev_offset: 0, + current_token: Token::default(), + current_builder: String::new(), + queued_tokens: VecDeque::new(), + temporary_buffer: Vec::new(), + character_reference_code: 0, + last_emitted_start_tag_name: None, + source_positions: vec![Position { line: 0, column: 0 }], + current_line: 0, + current_column: 0, + has_emitted_eof: false, + aborted: false, + insertion_point: None, + old_insertion_points: Vec::new(), + explicit_eof_inserted: false, + input_stream_closed: true, + blocked: false, + stop_at_insertion_point: false, + cdata_allowed: false, + entity_matcher: NamedCharacterReferenceMatcher::new(), + } + } + + /// Set the tokenizer state. + pub fn switch_to(&mut self, state: State) { + self.state = state; + } + + // -- Insertion point management -- + + pub fn store_insertion_point(&mut self) { + self.old_insertion_points.push(self.insertion_point); + } + + pub fn restore_insertion_point(&mut self) { + self.insertion_point = self.old_insertion_points.pop().unwrap_or(None); + } + + pub fn update_insertion_point(&mut self) { + self.insertion_point = Some(self.current_offset); + } + + pub fn undefine_insertion_point(&mut self) { + self.insertion_point = None; + } + + pub fn is_insertion_point_defined(&self) -> bool { + self.insertion_point.is_some() + } + + pub fn is_insertion_point_reached(&self) -> bool { + self.insertion_point + .is_some_and(|insertion_point| self.current_offset >= insertion_point) + } + + pub fn append_input(&mut self, code_points: &[u32]) { + self.input.extend_from_slice(code_points); + } + + pub fn insert_input_at_insertion_point(&mut self, code_points: &[u32]) { + if let Some(ip) = self.insertion_point { + let ip = ip.min(self.input.len()); + self.input.splice(ip..ip, code_points.iter().copied()); + self.insertion_point = Some(ip + code_points.len()); + for old_insertion_point in &mut self.old_insertion_points { + if let Some(old_ip) = old_insertion_point + && ip <= *old_ip + { + *old_ip += code_points.len(); + } + } + } + } + + pub fn unparsed_input(&self) -> String { + let mut output = String::new(); + for code_point in self.input[self.current_offset..].iter().copied() { + push_code_point(&mut output, code_point); + } + output + } + + pub fn parser_did_run(&mut self) { + if self.current_offset == 0 + || self.current_offset != self.input.len() + || self.insertion_point.is_some_and(|insertion_point| insertion_point != 0) + || !self.old_insertion_points.is_empty() + { + return; + } + + self.input = Vec::new(); + self.current_offset = 0; + self.prev_offset = 0; + let last_position = *self.source_positions.last().unwrap_or(&Position::default()); + self.source_positions.clear(); + self.source_positions.push(last_position); + self.current_line = last_position.line; + self.current_column = last_position.column; + } + + pub fn set_blocked(&mut self, blocked: bool) { + self.blocked = blocked; + } + + pub fn is_blocked(&self) -> bool { + self.blocked + } + + pub fn insert_eof(&mut self) { + self.explicit_eof_inserted = true; + self.input_stream_closed = true; + } + + pub fn is_eof_inserted(&self) -> bool { + self.explicit_eof_inserted + } + + pub fn set_input_stream_closed(&mut self, closed: bool) { + self.input_stream_closed = closed; + } + + pub fn abort(&mut self) { + self.aborted = true; + } + + // -- Input helpers -- + + #[inline] + fn peek_code_point(&self, offset: isize) -> Option { + let idx = self.current_offset as isize + offset; + if idx < 0 || idx as usize >= self.input.len() { + return None; + } + if self.stop_at_insertion_point + && let Some(ip) = self.insertion_point + && idx as usize >= ip + { + return None; + } + Some(self.input[idx as usize]) + } + + #[inline(always)] + fn skip(&mut self, count: usize) { + if !self.source_positions.is_empty() { + let last = *self.source_positions.last().unwrap(); + self.source_positions.push(last); + } + // Keep position updates in local registers across the loop to + // avoid a store-to-load dependency on the source_positions stack + // top. The stack top is written back once at the end. + let (mut line, mut column) = if let Some(pos) = self.source_positions.last() { + (pos.line, pos.column) + } else { + (self.current_line, self.current_column) + }; + for _ in 0..count { + self.prev_offset = self.current_offset; + let code_point = self.input[self.current_offset]; + if code_point == 0x0A { + line += 1; + column = 0; + } else { + column += 1; + } + self.current_offset += 1; + } + self.current_line = line; + self.current_column = column; + if let Some(pos) = self.source_positions.last_mut() { + pos.line = line; + pos.column = column; + } + } + + #[inline(always)] + fn next_code_point(&mut self) -> Option { + if self.current_offset >= self.input.len() { + self.prev_offset = self.current_offset; + return None; + } + let cp = self.input[self.current_offset]; + if cp != 0x0D { + self.skip(1); + return Some(cp); + } + // Slow path: CR normalization + let next = self.peek_code_point(1).unwrap_or(0); + if next == 0x0A { + self.skip(2); + } else { + self.skip(1); + } + Some(0x0A) + } + + #[inline(always)] + fn nth_last_position(&self, n: usize) -> Position { + if n + 1 > self.source_positions.len() { + return Position { line: 0, column: 0 }; + } + self.source_positions[self.source_positions.len() - 1 - n] + } + + /// Fast-path character emission from the Data state: if the tokenizer + /// is positioned at a plain-text code point (not `<`, `&`, NUL, CR, + /// or any non-ASCII character) and the queue is empty, advance one + /// code point and return the character's position, skipping the + /// full `next_token` state-machine dispatch and Token construction. + /// Returns None if the slow path is required. + #[inline(always)] + pub fn try_fast_data_char(&mut self) -> Option<(u32, Position)> { + if self.aborted { + return None; + } + if self.state as u8 != State::Data as u8 { + return None; + } + if !self.queued_tokens.is_empty() { + return None; + } + if self.current_offset >= self.input.len() { + self.sync_source_positions(); + return None; + } + let cp = self.input[self.current_offset]; + if cp == 0x3C || cp == 0x26 || cp == 0x00 || cp == 0x0D || cp >= 0x80 { + self.sync_source_positions(); + return None; + } + if cp == 0x0A { + self.current_line += 1; + self.current_column = 0; + } else { + self.current_column += 1; + } + let pos = Position { + line: self.current_line, + column: self.current_column, + }; + self.prev_offset = self.current_offset; + self.current_offset += 1; + Some((cp, pos)) + } + + /// Resynchronise `source_positions` with the scalar current_line / + /// current_column after a run of fast-path character emissions. + #[inline(always)] + fn sync_source_positions(&mut self) { + self.source_positions.clear(); + self.source_positions.push(Position { + line: self.current_line, + column: self.current_column, + }); + } + + fn restore_to(&mut self, offset: usize) { + while self.current_offset > offset && self.source_positions.len() > 1 { + self.source_positions.pop(); + self.current_offset -= 1; + } + self.current_offset = offset; + if let Some(pos) = self.source_positions.last() { + self.current_line = pos.line; + self.current_column = pos.column; + } + } + + fn consume_current_builder(&mut self) -> String { + std::mem::take(&mut self.current_builder) + } + + /// Commit `current_builder` into `current_token.tag_name` (or its + /// interned-id slot). If the accumulated bytes hit the intern table + /// we clear the builder in place, preserving its capacity for the + /// next token and avoiding the String allocation entirely. Most real + /// HTML tag names are interned. + #[inline] + fn commit_current_builder_as_tag_name(&mut self) { + let id = crate::interned_names::lookup_tag_name(self.current_builder.as_bytes()); + if id != 0 { + self.current_builder.clear(); + self.current_token.set_tag_name_id(id); + } else { + let name = std::mem::take(&mut self.current_builder); + *self.current_token.tag_name_mut() = name; + } + } + + fn create_new_token(&mut self, token_type: TokenType) { + let payload = match token_type { + TokenType::StartTag | TokenType::EndTag => TokenPayload::Tag { + tag_name: String::new(), + tag_name_id: 0, + self_closing: false, + attributes: Vec::new(), + }, + TokenType::Comment => TokenPayload::Comment(String::new()), + TokenType::Doctype => TokenPayload::Doctype(Box::default()), + _ => TokenPayload::None, + }; + let pos = match token_type { + TokenType::StartTag | TokenType::EndTag => self.nth_last_position(1), + _ => self.nth_last_position(0), + }; + self.current_token = Token { + token_type, + code_point: 0, + payload, + start_position: pos, + end_position: Position::default(), + }; + } + + fn current_end_tag_token_is_appropriate(&self) -> bool { + if self.current_token.token_type != TokenType::EndTag { + return false; + } + match &self.last_emitted_start_tag_name { + Some(name) => self.current_token.tag_name() == name, + None => false, + } + } + + fn consumed_as_part_of_an_attribute(&self) -> bool { + matches!( + self.return_state, + State::AttributeValueDoubleQuoted | State::AttributeValueSingleQuoted | State::AttributeValueUnquoted + ) + } + + fn will_emit(&mut self, token_idx: usize) { + // token_idx: 0 = current_token, 1+ = queued_tokens index + // For simplicity, we handle position setting here. + if token_idx == 0 { + if self.current_token.token_type == TokenType::StartTag { + self.last_emitted_start_tag_name = Some(self.current_token.tag_name().to_string()); + } + let is_start_or_end_tag = self.current_token.token_type == TokenType::StartTag + || self.current_token.token_type == TokenType::EndTag; + self.current_token.end_position = self.nth_last_position(if is_start_or_end_tag { 1 } else { 0 }); + } + } + + fn emit_current_token(&mut self) { + self.will_emit(0); + let token = std::mem::take(&mut self.current_token); + self.queued_tokens.push_back(token); + } + + /// Fast-path version of `emit_current_token(); self.queued_tokens.pop_front()`. + /// When the queue is empty we skip the push/pop round trip entirely, + /// which removes two VecDeque ops and a Token move per emitted token + /// for the overwhelmingly common single-emit case. + #[inline] + fn emit_current_token_direct(&mut self) -> Option { + self.will_emit(0); + let token = std::mem::take(&mut self.current_token); + if self.queued_tokens.is_empty() { + Some(token) + } else { + self.queued_tokens.push_back(token); + self.queued_tokens.pop_front() + } + } + + #[inline(always)] + fn return_character_token(&mut self, code_point: u32) -> Option { + let mut token = Token::new_character(code_point); + token.start_position = self.nth_last_position(0); + if self.queued_tokens.is_empty() { + Some(token) + } else { + self.queued_tokens.push_back(token); + self.queued_tokens.pop_front() + } + } + + fn emit_eof(&mut self) { + if self.has_emitted_eof { + return; + } + self.has_emitted_eof = true; + self.create_new_token(TokenType::EndOfFile); + self.emit_current_token(); + } + + fn emit_current_token_followed_by_eof(&mut self) { + self.emit_current_token(); + self.has_emitted_eof = true; + self.create_new_token(TokenType::EndOfFile); + self.emit_current_token(); + } + + fn flush_codepoints_consumed_as_character_reference(&mut self) { + for i in 0..self.temporary_buffer.len() { + let code_point = self.temporary_buffer[i]; + if self.consumed_as_part_of_an_attribute() { + push_code_point(&mut self.current_builder, code_point); + } else { + let mut token = Token::new_character(code_point); + token.start_position = self.nth_last_position(0); + self.queued_tokens.push_back(token); + } + } + } + + fn can_run_out_at(&self, idx: usize) -> bool { + if self.stop_at_insertion_point + && let Some(ip) = self.insertion_point + && idx >= ip + { + return true; + } + !self.input_stream_closed && idx >= self.input.len() + } + + #[inline] + fn fast_scan_limit(&self) -> usize { + if self.stop_at_insertion_point + && let Some(ip) = self.insertion_point + { + return ip.min(self.input.len()); + } + self.input.len() + } + + fn should_pause_before_next_input_character(&self) -> bool { + if self.stop_at_insertion_point + && let Some(ip) = self.insertion_point + { + if self.current_offset >= ip { + return true; + } + + if self.current_offset < self.input.len() + && self.input[self.current_offset] == 0x0D + && self.current_offset + 1 == ip + { + return true; + } + } + + if self.input_stream_closed { + return false; + } + + if self.current_offset >= self.input.len() { + return true; + } + + self.input[self.current_offset] == 0x0D && self.current_offset + 1 >= self.input.len() + } + + /// Case-insensitive match of upcoming input against a string. + /// Returns Some(true) if matched and consumed, Some(false) if no match, + /// None if ran out of characters at the insertion point. + fn consume_next_if_match(&mut self, s: &str) -> Option { + for (i, b) in s.bytes().enumerate() { + match self.peek_code_point(i as isize) { + None => { + if self.can_run_out_at(self.current_offset + i) { + return None; + } + return Some(false); + } + Some(cp) => { + if to_ascii_lowercase(cp) != to_ascii_lowercase(b as u32) { + return Some(false); + } + } + } + } + // All matched, consume them + self.skip(s.len()); + Some(true) + } + + /// Case-sensitive match of upcoming input against a string. + fn consume_next_if_match_exact(&mut self, s: &str) -> Option { + for (i, b) in s.bytes().enumerate() { + match self.peek_code_point(i as isize) { + None => { + if self.can_run_out_at(self.current_offset + i) { + return None; + } + return Some(false); + } + Some(cp) => { + if cp != b as u32 { + return Some(false); + } + } + } + } + self.skip(s.len()); + Some(true) + } + + /// Get the next token from the tokenizer. + pub fn next_token(&mut self, stop_at_insertion_point: bool, cdata_allowed: bool) -> Option { + self.stop_at_insertion_point = stop_at_insertion_point; + self.cdata_allowed = cdata_allowed; + + // Return queued tokens first. + { + let last = *self.source_positions.last().unwrap_or(&Position::default()); + self.source_positions.clear(); + self.source_positions.push(last); + } + + if let Some(token) = self.queued_tokens.pop_front() { + return Some(token); + } + + if self.aborted { + return None; + } + + loop { + // Check insertion point before consuming. + if self.should_pause_before_next_input_character() { + return None; + } + + let current_input_character = self.next_code_point(); + + match self.state { + // 13.2.5.1 Data state + State::Data => { + match current_input_character { + Some(0x26) => { + // '&' + self.return_state = State::Data; + self.state = State::CharacterReference; + continue; + } + Some(0x3C) => { + // '<' + self.state = State::TagOpen; + continue; + } + Some(0x00) => { + // NULL - parse error + return self.return_character_token(0x00); + } + None => { + self.emit_eof(); + return self.queued_tokens.pop_front(); + } + Some(cp) => { + return self.return_character_token(cp); + } + } + } + + // 13.2.5.2 RCDATA state + State::RCDATA => match current_input_character { + Some(0x26) => { + self.return_state = State::RCDATA; + self.state = State::CharacterReference; + continue; + } + Some(0x3C) => { + self.state = State::RCDATALessThanSign; + continue; + } + Some(0x00) => { + return self.return_character_token(0xFFFD); + } + None => { + self.emit_eof(); + return self.queued_tokens.pop_front(); + } + Some(cp) => { + return self.return_character_token(cp); + } + }, + + // 13.2.5.3 RAWTEXT state + State::RAWTEXT => match current_input_character { + Some(0x3C) => { + self.state = State::RAWTEXTLessThanSign; + continue; + } + Some(0x00) => { + return self.return_character_token(0xFFFD); + } + None => { + self.emit_eof(); + return self.queued_tokens.pop_front(); + } + Some(cp) => { + return self.return_character_token(cp); + } + }, + + // 13.2.5.4 Script data state + State::ScriptData => match current_input_character { + Some(0x3C) => { + self.state = State::ScriptDataLessThanSign; + continue; + } + Some(0x00) => { + return self.return_character_token(0xFFFD); + } + None => { + self.emit_eof(); + return self.queued_tokens.pop_front(); + } + Some(cp) => { + return self.return_character_token(cp); + } + }, + + // 13.2.5.5 PLAINTEXT state + State::PLAINTEXT => match current_input_character { + Some(0x00) => { + return self.return_character_token(0xFFFD); + } + None => { + self.emit_eof(); + return self.queued_tokens.pop_front(); + } + Some(cp) => { + return self.return_character_token(cp); + } + }, + + // 13.2.5.6 Tag open state + State::TagOpen => match current_input_character { + Some(0x21) => { + self.state = State::MarkupDeclarationOpen; + continue; + } + Some(0x2F) => { + self.state = State::EndTagOpen; + continue; + } + Some(cp) if is_ascii_alpha(cp) => { + self.create_new_token(TokenType::StartTag); + self.reconsume(State::TagName); + continue; + } + Some(0x3F) => { + // '?' - parse error + self.create_new_token(TokenType::Comment); + self.current_token.start_position = self.nth_last_position(2); + self.reconsume(State::BogusComment); + continue; + } + None => { + // parse error + self.queued_tokens.push_back(self.make_character_token(0x3C)); + self.emit_eof(); + return self.queued_tokens.pop_front(); + } + Some(_) => { + // parse error + self.queued_tokens.push_back(self.make_character_token(0x3C)); + self.reconsume(State::Data); + continue; + } + }, + + // 13.2.5.7 End tag open state + State::EndTagOpen => match current_input_character { + Some(cp) if is_ascii_alpha(cp) => { + self.create_new_token(TokenType::EndTag); + self.reconsume(State::TagName); + continue; + } + Some(0x3E) => { + // '>' - parse error + self.state = State::Data; + continue; + } + None => { + // parse error + self.queued_tokens.push_back(self.make_character_token(0x3C)); + self.queued_tokens.push_back(self.make_character_token(0x2F)); + self.emit_eof(); + return self.queued_tokens.pop_front(); + } + Some(_) => { + // parse error + self.create_new_token(TokenType::Comment); + self.reconsume(State::BogusComment); + continue; + } + }, + + // 13.2.5.8 Tag name state + State::TagName => { + // Fast-path run: bulk-scan consecutive lowercase + // ASCII characters and append them to current_builder. + // This handles the common case where a tag name is + // a plain `div`, `span`, `a` etc. and avoids paying + // the per-character state-machine dispatch cost. + // We keep the first character the outer state loop + // already consumed, then extend the builder by + // scanning from current_offset until we see a byte + // that needs special handling. + if let Some(cp) = current_input_character + && cp >= b'a' as u32 + && cp <= b'z' as u32 + { + // SAFETY: pushing ASCII bytes into a String is + // always valid UTF-8. We drop into a tight + // inner loop that directly advances + // current_offset and current_column without + // going through skip(1) per character. + unsafe { self.current_builder.as_mut_vec().push(cp as u8) }; + let start_offset = self.current_offset; + let mut off = start_offset; + let input_len = self.fast_scan_limit(); + while off < input_len { + let next = self.input[off]; + if next < b'a' as u32 || next > b'z' as u32 { + break; + } + unsafe { self.current_builder.as_mut_vec().push(next as u8) }; + off += 1; + } + let consumed = off - start_offset; + if consumed > 0 { + self.current_column += consumed as u64; + self.prev_offset = off - 1; + self.current_offset = off; + if let Some(pos) = self.source_positions.last_mut() { + pos.line = self.current_line; + pos.column = self.current_column; + } + } + self.current_token.end_position = self.nth_last_position(0); + continue; + } + match current_input_character { + Some(cp) if is_whitespace(cp) => { + self.commit_current_builder_as_tag_name(); + self.current_token.end_position = self.nth_last_position(1); + self.state = State::BeforeAttributeName; + continue; + } + Some(0x2F) => { + self.commit_current_builder_as_tag_name(); + self.current_token.end_position = self.nth_last_position(0); + self.state = State::SelfClosingStartTag; + continue; + } + Some(0x3E) => { + self.commit_current_builder_as_tag_name(); + self.state = State::Data; + return self.emit_current_token_direct(); + } + Some(cp) if is_ascii_upper_alpha(cp) => { + push_code_point(&mut self.current_builder, to_ascii_lowercase(cp)); + self.current_token.end_position = self.nth_last_position(0); + continue; + } + Some(0x00) => { + self.current_builder.push('\u{FFFD}'); + self.current_token.end_position = self.nth_last_position(0); + continue; + } + None => { + self.emit_eof(); + return self.queued_tokens.pop_front(); + } + Some(cp) => { + push_code_point(&mut self.current_builder, cp); + self.current_token.end_position = self.nth_last_position(0); + continue; + } + } + } + + // 13.2.5.32 Before attribute name state + State::BeforeAttributeName => match current_input_character { + Some(cp) if is_whitespace(cp) => { + continue; + } + Some(0x2F) | Some(0x3E) | None => { + self.reconsume(State::AfterAttributeName); + continue; + } + Some(0x3D) => { + // '=' - parse error + self.current_token.attributes_mut().push(Attribute::default()); + self.current_builder.push('='); + self.state = State::AttributeName; + continue; + } + Some(_) => { + self.current_token.attributes_mut().push(Attribute::default()); + self.reconsume(State::AttributeName); + continue; + } + }, + + // 13.2.5.33 Attribute name state + State::AttributeName => { + // Fast-path run: bulk-scan consecutive lowercase ASCII + // and a few other name-safe bytes (digits, '-', '_'). + if let Some(cp) = current_input_character { + let is_name_char = (cp >= b'a' as u32 && cp <= b'z' as u32) + || (cp >= b'0' as u32 && cp <= b'9' as u32) + || cp == b'-' as u32 + || cp == b'_' as u32; + if is_name_char { + unsafe { self.current_builder.as_mut_vec().push(cp as u8) }; + let start_offset = self.current_offset; + let mut off = start_offset; + let input_len = self.fast_scan_limit(); + while off < input_len { + let next = self.input[off]; + let ok = (next >= b'a' as u32 && next <= b'z' as u32) + || (next >= b'0' as u32 && next <= b'9' as u32) + || next == b'-' as u32 + || next == b'_' as u32; + if !ok { + break; + } + unsafe { self.current_builder.as_mut_vec().push(next as u8) }; + off += 1; + } + let consumed = off - start_offset; + if consumed > 0 { + self.current_column += consumed as u64; + self.prev_offset = off - 1; + self.current_offset = off; + if let Some(pos) = self.source_positions.last_mut() { + pos.line = self.current_line; + pos.column = self.current_column; + } + } + continue; + } + } + match current_input_character { + Some(cp) if is_whitespace(cp) => { + self.set_attribute_name(); + self.reconsume(State::AfterAttributeName); + continue; + } + Some(0x2F) | Some(0x3E) | None => { + self.set_attribute_name(); + self.reconsume(State::AfterAttributeName); + continue; + } + Some(0x3D) => { + self.set_attribute_name(); + self.state = State::BeforeAttributeValue; + continue; + } + Some(cp) if is_ascii_upper_alpha(cp) => { + push_code_point(&mut self.current_builder, to_ascii_lowercase(cp)); + continue; + } + Some(0x00) => { + self.current_builder.push('\u{FFFD}'); + continue; + } + Some(cp @ (0x22 | 0x27 | 0x3C)) => { + // '"', '\'', '<' - parse error + push_code_point(&mut self.current_builder, cp); + continue; + } + Some(cp) => { + push_code_point(&mut self.current_builder, cp); + continue; + } + } + } + + // 13.2.5.34 After attribute name state + State::AfterAttributeName => match current_input_character { + Some(cp) if is_whitespace(cp) => { + continue; + } + Some(0x2F) => { + self.state = State::SelfClosingStartTag; + continue; + } + Some(0x3D) => { + self.state = State::BeforeAttributeValue; + continue; + } + Some(0x3E) => { + self.state = State::Data; + return self.emit_current_token_direct(); + } + None => { + self.emit_eof(); + return self.queued_tokens.pop_front(); + } + Some(_) => { + self.current_token.attributes_mut().push(Attribute::default()); + self.reconsume(State::AttributeName); + continue; + } + }, + + // 13.2.5.35 Before attribute value state + State::BeforeAttributeValue => { + let pos = self.nth_last_position(1); + if let Some(attr) = self.current_token.attributes_mut().last_mut() { + attr.value_start_position = pos; + } + match current_input_character { + Some(cp) if is_whitespace(cp) => { + continue; + } + Some(0x22) => { + self.state = State::AttributeValueDoubleQuoted; + continue; + } + Some(0x27) => { + self.state = State::AttributeValueSingleQuoted; + continue; + } + Some(0x3E) => { + // parse error + self.state = State::Data; + return self.emit_current_token_direct(); + } + _ => { + self.reconsume(State::AttributeValueUnquoted); + continue; + } + } + } + + // 13.2.5.36 Attribute value (double-quoted) state + State::AttributeValueDoubleQuoted => { + // Fast-path run: bulk-append printable ASCII until we + // hit `"`, `&`, NUL, CR or non-ASCII. The inner loop + // advances a local offset directly and only updates + // tokenizer/position state once at the end. + if let Some(cp) = current_input_character + && (0x20..0x80).contains(&cp) + && cp != 0x22 + && cp != 0x26 + { + unsafe { self.current_builder.as_mut_vec().push(cp as u8) }; + let start_offset = self.current_offset; + let mut off = start_offset; + let input_len = self.fast_scan_limit(); + while off < input_len { + let next = self.input[off]; + if !(0x20..0x80).contains(&next) || next == 0x22 || next == 0x26 { + break; + } + unsafe { self.current_builder.as_mut_vec().push(next as u8) }; + off += 1; + } + let consumed = off - start_offset; + if consumed > 0 { + self.current_column += consumed as u64; + self.prev_offset = off - 1; + self.current_offset = off; + if let Some(pos) = self.source_positions.last_mut() { + pos.line = self.current_line; + pos.column = self.current_column; + } + } + continue; + } + match current_input_character { + Some(0x22) => { + self.set_attribute_value(); + self.state = State::AfterAttributeValueQuoted; + continue; + } + Some(0x26) => { + self.return_state = State::AttributeValueDoubleQuoted; + self.state = State::CharacterReference; + continue; + } + Some(0x00) => { + self.current_builder.push('\u{FFFD}'); + continue; + } + None => { + self.emit_eof(); + return self.queued_tokens.pop_front(); + } + Some(cp) => { + push_code_point(&mut self.current_builder, cp); + continue; + } + } + } + + // 13.2.5.37 Attribute value (single-quoted) state + State::AttributeValueSingleQuoted => match current_input_character { + Some(0x27) => { + self.set_attribute_value(); + self.state = State::AfterAttributeValueQuoted; + continue; + } + Some(0x26) => { + self.return_state = State::AttributeValueSingleQuoted; + self.state = State::CharacterReference; + continue; + } + Some(0x00) => { + self.current_builder.push('\u{FFFD}'); + continue; + } + None => { + self.emit_eof(); + return self.queued_tokens.pop_front(); + } + Some(cp) => { + push_code_point(&mut self.current_builder, cp); + continue; + } + }, + + // 13.2.5.38 Attribute value (unquoted) state + State::AttributeValueUnquoted => match current_input_character { + Some(cp) if is_whitespace(cp) => { + self.set_attribute_value(); + self.state = State::BeforeAttributeName; + continue; + } + Some(0x26) => { + self.return_state = State::AttributeValueUnquoted; + self.state = State::CharacterReference; + continue; + } + Some(0x3E) => { + self.set_attribute_value(); + self.state = State::Data; + return self.emit_current_token_direct(); + } + Some(0x00) => { + self.current_builder.push('\u{FFFD}'); + continue; + } + Some(cp @ (0x22 | 0x27 | 0x3C | 0x3D | 0x60)) => { + // parse error, treat as anything else + push_code_point(&mut self.current_builder, cp); + continue; + } + None => { + self.emit_eof(); + return self.queued_tokens.pop_front(); + } + Some(cp) => { + push_code_point(&mut self.current_builder, cp); + continue; + } + }, + + // 13.2.5.39 After attribute value (quoted) state + State::AfterAttributeValueQuoted => { + let pos = self.nth_last_position(1); + if let Some(attr) = self.current_token.attributes_mut().last_mut() { + attr.value_end_position = pos; + } + match current_input_character { + Some(cp) if is_whitespace(cp) => { + self.state = State::BeforeAttributeName; + continue; + } + Some(0x2F) => { + self.state = State::SelfClosingStartTag; + continue; + } + Some(0x3E) => { + self.state = State::Data; + return self.emit_current_token_direct(); + } + None => { + self.emit_eof(); + return self.queued_tokens.pop_front(); + } + Some(_) => { + // parse error + self.reconsume(State::BeforeAttributeName); + continue; + } + } + } + + // 13.2.5.40 Self-closing start tag state + State::SelfClosingStartTag => match current_input_character { + Some(0x3E) => { + self.current_token.set_self_closing(true); + self.state = State::Data; + return self.emit_current_token_direct(); + } + None => { + self.emit_eof(); + return self.queued_tokens.pop_front(); + } + Some(_) => { + // parse error + self.reconsume(State::BeforeAttributeName); + continue; + } + }, + + // 13.2.5.41 Bogus comment state + State::BogusComment => match current_input_character { + Some(0x3E) => { + { + let data = self.consume_current_builder(); + self.current_token.set_comment_data(data); + } + self.state = State::Data; + return self.emit_current_token_direct(); + } + None => { + { + let data = self.consume_current_builder(); + self.current_token.set_comment_data(data); + } + self.emit_current_token_followed_by_eof(); + return self.queued_tokens.pop_front(); + } + Some(0x00) => { + self.current_builder.push('\u{FFFD}'); + continue; + } + Some(cp) => { + push_code_point(&mut self.current_builder, cp); + continue; + } + }, + + // 13.2.5.42 Markup declaration open state + State::MarkupDeclarationOpen => { + // Don't consume next input character (reconsume). + if current_input_character.is_some() { + self.restore_to(self.prev_offset); + } + + match self.consume_next_if_match_exact("--") { + Some(true) => { + self.create_new_token(TokenType::Comment); + self.current_token.start_position = self.nth_last_position(3); + self.state = State::CommentStart; + continue; + } + None => return None, + _ => {} + } + match self.consume_next_if_match("DOCTYPE") { + Some(true) => { + self.state = State::DOCTYPE; + continue; + } + None => return None, + _ => {} + } + match self.consume_next_if_match_exact("[CDATA[") { + Some(true) => { + if self.cdata_allowed { + self.state = State::CDATASection; + } else { + self.create_new_token(TokenType::Comment); + self.current_builder.push_str("[CDATA["); + self.state = State::BogusComment; + } + continue; + } + None => return None, + _ => {} + } + // parse error + self.create_new_token(TokenType::Comment); + self.state = State::BogusComment; + continue; + } + + // 13.2.5.43 Comment start state + State::CommentStart => match current_input_character { + Some(0x2D) => { + self.state = State::CommentStartDash; + continue; + } + Some(0x3E) => { + // parse error + { + let data = self.consume_current_builder(); + self.current_token.set_comment_data(data); + } + self.state = State::Data; + return self.emit_current_token_direct(); + } + _ => { + self.reconsume(State::Comment); + continue; + } + }, + + // 13.2.5.44 Comment start dash state + State::CommentStartDash => match current_input_character { + Some(0x2D) => { + self.state = State::CommentEnd; + continue; + } + Some(0x3E) => { + // parse error + { + let data = self.consume_current_builder(); + self.current_token.set_comment_data(data); + } + self.state = State::Data; + return self.emit_current_token_direct(); + } + None => { + { + let data = self.consume_current_builder(); + self.current_token.set_comment_data(data); + } + self.emit_current_token_followed_by_eof(); + return self.queued_tokens.pop_front(); + } + Some(_) => { + self.current_builder.push('-'); + self.reconsume(State::Comment); + continue; + } + }, + + // 13.2.5.45 Comment state + State::Comment => match current_input_character { + Some(0x3C) => { + self.current_builder.push('<'); + self.state = State::CommentLessThanSign; + continue; + } + Some(0x2D) => { + self.state = State::CommentEndDash; + continue; + } + Some(0x00) => { + self.current_builder.push('\u{FFFD}'); + continue; + } + None => { + { + let data = self.consume_current_builder(); + self.current_token.set_comment_data(data); + } + self.emit_current_token_followed_by_eof(); + return self.queued_tokens.pop_front(); + } + Some(cp) => { + push_code_point(&mut self.current_builder, cp); + continue; + } + }, + + // 13.2.5.46 Comment less-than sign state + State::CommentLessThanSign => match current_input_character { + Some(0x21) => { + self.current_builder.push('!'); + self.state = State::CommentLessThanSignBang; + continue; + } + Some(0x3C) => { + self.current_builder.push('<'); + continue; + } + _ => { + self.reconsume(State::Comment); + continue; + } + }, + + // 13.2.5.47 Comment less-than sign bang state + State::CommentLessThanSignBang => match current_input_character { + Some(0x2D) => { + self.state = State::CommentLessThanSignBangDash; + continue; + } + _ => { + self.reconsume(State::Comment); + continue; + } + }, + + // 13.2.5.48 Comment less-than sign bang dash state + State::CommentLessThanSignBangDash => match current_input_character { + Some(0x2D) => { + self.state = State::CommentLessThanSignBangDashDash; + continue; + } + _ => { + self.reconsume(State::CommentEndDash); + continue; + } + }, + + // 13.2.5.49 Comment less-than sign bang dash dash state + State::CommentLessThanSignBangDashDash => match current_input_character { + Some(0x3E) | None => { + self.reconsume(State::CommentEnd); + continue; + } + _ => { + // parse error + self.reconsume(State::CommentEnd); + continue; + } + }, + + // 13.2.5.50 Comment end dash state + State::CommentEndDash => match current_input_character { + Some(0x2D) => { + self.state = State::CommentEnd; + continue; + } + None => { + { + let data = self.consume_current_builder(); + self.current_token.set_comment_data(data); + } + self.emit_current_token_followed_by_eof(); + return self.queued_tokens.pop_front(); + } + Some(_) => { + self.current_builder.push('-'); + self.reconsume(State::Comment); + continue; + } + }, + + // 13.2.5.51 Comment end state + State::CommentEnd => match current_input_character { + Some(0x3E) => { + { + let data = self.consume_current_builder(); + self.current_token.set_comment_data(data); + } + self.state = State::Data; + return self.emit_current_token_direct(); + } + Some(0x21) => { + self.state = State::CommentEndBang; + continue; + } + Some(0x2D) => { + self.current_builder.push('-'); + continue; + } + None => { + { + let data = self.consume_current_builder(); + self.current_token.set_comment_data(data); + } + self.emit_current_token_followed_by_eof(); + return self.queued_tokens.pop_front(); + } + Some(_) => { + self.current_builder.push_str("--"); + self.reconsume(State::Comment); + continue; + } + }, + + // 13.2.5.52 Comment end bang state + State::CommentEndBang => match current_input_character { + Some(0x2D) => { + self.current_builder.push_str("--!"); + self.state = State::CommentEndDash; + continue; + } + Some(0x3E) => { + // parse error + { + let data = self.consume_current_builder(); + self.current_token.set_comment_data(data); + } + self.state = State::Data; + return self.emit_current_token_direct(); + } + None => { + { + let data = self.consume_current_builder(); + self.current_token.set_comment_data(data); + } + self.emit_current_token_followed_by_eof(); + return self.queued_tokens.pop_front(); + } + Some(_) => { + self.current_builder.push_str("--!"); + self.reconsume(State::Comment); + continue; + } + }, + + // 13.2.5.53 DOCTYPE state + State::DOCTYPE => match current_input_character { + Some(cp) if is_whitespace(cp) => { + self.state = State::BeforeDOCTYPEName; + continue; + } + Some(0x3E) => { + self.reconsume(State::BeforeDOCTYPEName); + continue; + } + None => { + self.create_new_token(TokenType::Doctype); + *self.current_token.doctype_data_mut() = DoctypeData { + force_quirks: true, + missing_name: true, + missing_public_identifier: true, + missing_system_identifier: true, + ..Default::default() + }; + self.emit_current_token_followed_by_eof(); + return self.queued_tokens.pop_front(); + } + Some(_) => { + // parse error + self.reconsume(State::BeforeDOCTYPEName); + continue; + } + }, + + // 13.2.5.54 Before DOCTYPE name state + State::BeforeDOCTYPEName => match current_input_character { + Some(cp) if is_whitespace(cp) => { + continue; + } + Some(cp) if is_ascii_upper_alpha(cp) => { + self.create_new_token(TokenType::Doctype); + *self.current_token.doctype_data_mut() = DoctypeData { + missing_name: false, + missing_public_identifier: true, + missing_system_identifier: true, + ..Default::default() + }; + push_code_point(&mut self.current_builder, to_ascii_lowercase(cp)); + self.state = State::DOCTYPEName; + continue; + } + Some(0x00) => { + self.create_new_token(TokenType::Doctype); + *self.current_token.doctype_data_mut() = DoctypeData { + missing_name: false, + missing_public_identifier: true, + missing_system_identifier: true, + ..Default::default() + }; + self.current_builder.push('\u{FFFD}'); + self.state = State::DOCTYPEName; + continue; + } + Some(0x3E) => { + // parse error + self.create_new_token(TokenType::Doctype); + *self.current_token.doctype_data_mut() = DoctypeData { + force_quirks: true, + missing_name: true, + missing_public_identifier: true, + missing_system_identifier: true, + ..Default::default() + }; + self.state = State::Data; + return self.emit_current_token_direct(); + } + None => { + self.create_new_token(TokenType::Doctype); + *self.current_token.doctype_data_mut() = DoctypeData { + force_quirks: true, + missing_name: true, + missing_public_identifier: true, + missing_system_identifier: true, + ..Default::default() + }; + self.emit_current_token_followed_by_eof(); + return self.queued_tokens.pop_front(); + } + Some(cp) => { + self.create_new_token(TokenType::Doctype); + *self.current_token.doctype_data_mut() = DoctypeData { + missing_name: false, + missing_public_identifier: true, + missing_system_identifier: true, + ..Default::default() + }; + push_code_point(&mut self.current_builder, cp); + self.state = State::DOCTYPEName; + continue; + } + }, + + // 13.2.5.55 DOCTYPE name state + State::DOCTYPEName => match current_input_character { + Some(cp) if is_whitespace(cp) => { + { + let name = self.consume_current_builder(); + { + let dd = self.current_token.doctype_data_mut(); + dd.name = name; + } + } + self.state = State::AfterDOCTYPEName; + continue; + } + Some(0x3E) => { + { + let name = self.consume_current_builder(); + { + let dd = self.current_token.doctype_data_mut(); + dd.name = name; + } + } + self.state = State::Data; + return self.emit_current_token_direct(); + } + Some(cp) if is_ascii_upper_alpha(cp) => { + push_code_point(&mut self.current_builder, to_ascii_lowercase(cp)); + continue; + } + Some(0x00) => { + self.current_builder.push('\u{FFFD}'); + continue; + } + None => { + let name = self.consume_current_builder(); + { + let dd = self.current_token.doctype_data_mut(); + dd.name = name; + dd.force_quirks = true; + } + self.emit_current_token_followed_by_eof(); + return self.queued_tokens.pop_front(); + } + Some(cp) => { + push_code_point(&mut self.current_builder, cp); + continue; + } + }, + + // 13.2.5.56 After DOCTYPE name state + State::AfterDOCTYPEName => match current_input_character { + Some(cp) if is_whitespace(cp) => { + continue; + } + Some(0x3E) => { + self.state = State::Data; + return self.emit_current_token_direct(); + } + None => { + { + let dd = self.current_token.doctype_data_mut(); + dd.force_quirks = true; + } + self.emit_current_token_followed_by_eof(); + return self.queued_tokens.pop_front(); + } + Some(_) => { + // Reconsume and try PUBLIC/SYSTEM + if current_input_character.is_some() { + self.restore_to(self.prev_offset); + } + match self.consume_next_if_match("PUBLIC") { + Some(true) => { + self.state = State::AfterDOCTYPEPublicKeyword; + continue; + } + None => return None, + _ => {} + } + match self.consume_next_if_match("SYSTEM") { + Some(true) => { + self.state = State::AfterDOCTYPESystemKeyword; + continue; + } + None => return None, + _ => {} + } + // Re-consume the character we put back + let _ = self.next_code_point(); + { + let dd = self.current_token.doctype_data_mut(); + dd.force_quirks = true; + } + self.reconsume(State::BogusDOCTYPE); + continue; + } + }, + + // 13.2.5.57 After DOCTYPE public keyword state + State::AfterDOCTYPEPublicKeyword => match current_input_character { + Some(cp) if is_whitespace(cp) => { + self.state = State::BeforeDOCTYPEPublicIdentifier; + continue; + } + Some(0x22) => { + { + let dd = self.current_token.doctype_data_mut(); + dd.missing_public_identifier = false; + } + self.state = State::DOCTYPEPublicIdentifierDoubleQuoted; + continue; + } + Some(0x27) => { + { + let dd = self.current_token.doctype_data_mut(); + dd.missing_public_identifier = false; + } + self.state = State::DOCTYPEPublicIdentifierSingleQuoted; + continue; + } + Some(0x3E) => { + { + let dd = self.current_token.doctype_data_mut(); + dd.force_quirks = true; + } + self.state = State::Data; + return self.emit_current_token_direct(); + } + None => { + { + let dd = self.current_token.doctype_data_mut(); + dd.force_quirks = true; + } + self.emit_current_token_followed_by_eof(); + return self.queued_tokens.pop_front(); + } + Some(_) => { + { + let dd = self.current_token.doctype_data_mut(); + dd.force_quirks = true; + } + self.reconsume(State::BogusDOCTYPE); + continue; + } + }, + + // 13.2.5.58 Before DOCTYPE public identifier state + State::BeforeDOCTYPEPublicIdentifier => match current_input_character { + Some(cp) if is_whitespace(cp) => continue, + Some(0x22) => { + { + let dd = self.current_token.doctype_data_mut(); + dd.missing_public_identifier = false; + } + self.state = State::DOCTYPEPublicIdentifierDoubleQuoted; + continue; + } + Some(0x27) => { + { + let dd = self.current_token.doctype_data_mut(); + dd.missing_public_identifier = false; + } + self.state = State::DOCTYPEPublicIdentifierSingleQuoted; + continue; + } + Some(0x3E) => { + { + let dd = self.current_token.doctype_data_mut(); + dd.force_quirks = true; + } + self.state = State::Data; + return self.emit_current_token_direct(); + } + None => { + { + let dd = self.current_token.doctype_data_mut(); + dd.force_quirks = true; + } + self.emit_current_token_followed_by_eof(); + return self.queued_tokens.pop_front(); + } + Some(_) => { + { + let dd = self.current_token.doctype_data_mut(); + dd.force_quirks = true; + } + self.reconsume(State::BogusDOCTYPE); + continue; + } + }, + + // 13.2.5.59 DOCTYPE public identifier (double-quoted) state + State::DOCTYPEPublicIdentifierDoubleQuoted => match current_input_character { + Some(0x22) => { + { + let val = self.consume_current_builder(); + { + let dd = self.current_token.doctype_data_mut(); + dd.public_identifier = val; + } + } + self.state = State::AfterDOCTYPEPublicIdentifier; + continue; + } + Some(0x00) => { + self.current_builder.push('\u{FFFD}'); + continue; + } + Some(0x3E) => { + { + let val = self.consume_current_builder(); + { + let dd = self.current_token.doctype_data_mut(); + dd.public_identifier = val; + dd.force_quirks = true; + } + } + self.state = State::Data; + return self.emit_current_token_direct(); + } + None => { + { + let val = self.consume_current_builder(); + { + let dd = self.current_token.doctype_data_mut(); + dd.public_identifier = val; + dd.force_quirks = true; + } + } + self.emit_current_token_followed_by_eof(); + return self.queued_tokens.pop_front(); + } + Some(cp) => { + push_code_point(&mut self.current_builder, cp); + continue; + } + }, + + // 13.2.5.60 DOCTYPE public identifier (single-quoted) state + State::DOCTYPEPublicIdentifierSingleQuoted => match current_input_character { + Some(0x27) => { + { + let val = self.consume_current_builder(); + { + let dd = self.current_token.doctype_data_mut(); + dd.public_identifier = val; + } + } + self.state = State::AfterDOCTYPEPublicIdentifier; + continue; + } + Some(0x00) => { + self.current_builder.push('\u{FFFD}'); + continue; + } + Some(0x3E) => { + { + let val = self.consume_current_builder(); + { + let dd = self.current_token.doctype_data_mut(); + dd.public_identifier = val; + dd.force_quirks = true; + } + } + self.state = State::Data; + return self.emit_current_token_direct(); + } + None => { + { + let val = self.consume_current_builder(); + { + let dd = self.current_token.doctype_data_mut(); + dd.public_identifier = val; + dd.force_quirks = true; + } + } + self.emit_current_token_followed_by_eof(); + return self.queued_tokens.pop_front(); + } + Some(cp) => { + push_code_point(&mut self.current_builder, cp); + continue; + } + }, + + // 13.2.5.61 After DOCTYPE public identifier state + State::AfterDOCTYPEPublicIdentifier => match current_input_character { + Some(cp) if is_whitespace(cp) => { + self.state = State::BetweenDOCTYPEPublicAndSystemIdentifiers; + continue; + } + Some(0x3E) => { + self.state = State::Data; + return self.emit_current_token_direct(); + } + Some(0x22) => { + { + let dd = self.current_token.doctype_data_mut(); + dd.system_identifier = String::new(); + dd.missing_system_identifier = false; + } + self.state = State::DOCTYPESystemIdentifierDoubleQuoted; + continue; + } + Some(0x27) => { + { + let dd = self.current_token.doctype_data_mut(); + dd.system_identifier = String::new(); + dd.missing_system_identifier = false; + } + self.state = State::DOCTYPESystemIdentifierSingleQuoted; + continue; + } + None => { + { + let dd = self.current_token.doctype_data_mut(); + dd.force_quirks = true; + } + self.emit_current_token_followed_by_eof(); + return self.queued_tokens.pop_front(); + } + Some(_) => { + { + let dd = self.current_token.doctype_data_mut(); + dd.force_quirks = true; + } + self.reconsume(State::BogusDOCTYPE); + continue; + } + }, + + // 13.2.5.62 Between DOCTYPE public and system identifiers state + State::BetweenDOCTYPEPublicAndSystemIdentifiers => match current_input_character { + Some(cp) if is_whitespace(cp) => continue, + Some(0x3E) => { + self.state = State::Data; + return self.emit_current_token_direct(); + } + Some(0x22) => { + { + let dd = self.current_token.doctype_data_mut(); + dd.system_identifier = String::new(); + dd.missing_system_identifier = false; + } + self.state = State::DOCTYPESystemIdentifierDoubleQuoted; + continue; + } + Some(0x27) => { + { + let dd = self.current_token.doctype_data_mut(); + dd.system_identifier = String::new(); + dd.missing_system_identifier = false; + } + self.state = State::DOCTYPESystemIdentifierSingleQuoted; + continue; + } + None => { + { + let dd = self.current_token.doctype_data_mut(); + dd.force_quirks = true; + } + self.emit_current_token_followed_by_eof(); + return self.queued_tokens.pop_front(); + } + Some(_) => { + { + let dd = self.current_token.doctype_data_mut(); + dd.force_quirks = true; + } + self.reconsume(State::BogusDOCTYPE); + continue; + } + }, + + // 13.2.5.63 After DOCTYPE system keyword state + State::AfterDOCTYPESystemKeyword => match current_input_character { + Some(cp) if is_whitespace(cp) => { + self.state = State::BeforeDOCTYPESystemIdentifier; + continue; + } + Some(0x22) => { + { + let dd = self.current_token.doctype_data_mut(); + dd.system_identifier = String::new(); + dd.missing_system_identifier = false; + } + self.state = State::DOCTYPESystemIdentifierDoubleQuoted; + continue; + } + Some(0x27) => { + { + let dd = self.current_token.doctype_data_mut(); + dd.system_identifier = String::new(); + dd.missing_system_identifier = false; + } + self.state = State::DOCTYPESystemIdentifierSingleQuoted; + continue; + } + Some(0x3E) => { + { + let dd = self.current_token.doctype_data_mut(); + dd.force_quirks = true; + } + self.state = State::Data; + return self.emit_current_token_direct(); + } + None => { + { + let dd = self.current_token.doctype_data_mut(); + dd.force_quirks = true; + } + self.emit_current_token_followed_by_eof(); + return self.queued_tokens.pop_front(); + } + Some(_) => { + { + let dd = self.current_token.doctype_data_mut(); + dd.force_quirks = true; + } + self.reconsume(State::BogusDOCTYPE); + continue; + } + }, + + // 13.2.5.64 Before DOCTYPE system identifier state + State::BeforeDOCTYPESystemIdentifier => match current_input_character { + Some(cp) if is_whitespace(cp) => continue, + Some(0x22) => { + { + let dd = self.current_token.doctype_data_mut(); + dd.system_identifier = String::new(); + dd.missing_system_identifier = false; + } + self.state = State::DOCTYPESystemIdentifierDoubleQuoted; + continue; + } + Some(0x27) => { + { + let dd = self.current_token.doctype_data_mut(); + dd.system_identifier = String::new(); + dd.missing_system_identifier = false; + } + self.state = State::DOCTYPESystemIdentifierSingleQuoted; + continue; + } + Some(0x3E) => { + { + let dd = self.current_token.doctype_data_mut(); + dd.force_quirks = true; + } + self.state = State::Data; + return self.emit_current_token_direct(); + } + None => { + { + let dd = self.current_token.doctype_data_mut(); + dd.force_quirks = true; + } + self.emit_current_token_followed_by_eof(); + return self.queued_tokens.pop_front(); + } + Some(_) => { + { + let dd = self.current_token.doctype_data_mut(); + dd.force_quirks = true; + } + self.reconsume(State::BogusDOCTYPE); + continue; + } + }, + + // 13.2.5.65 DOCTYPE system identifier (double-quoted) state + State::DOCTYPESystemIdentifierDoubleQuoted => match current_input_character { + Some(0x22) => { + let sys_id = self.consume_current_builder(); + { + let dd = self.current_token.doctype_data_mut(); + dd.system_identifier = sys_id; + } + self.state = State::AfterDOCTYPESystemIdentifier; + continue; + } + Some(0x00) => { + self.current_builder.push('\u{FFFD}'); + continue; + } + Some(0x3E) => { + let sys_id = self.consume_current_builder(); + { + let dd = self.current_token.doctype_data_mut(); + dd.system_identifier = sys_id; + dd.force_quirks = true; + } + self.state = State::Data; + return self.emit_current_token_direct(); + } + None => { + let sys_id = self.consume_current_builder(); + { + let dd = self.current_token.doctype_data_mut(); + dd.system_identifier = sys_id; + dd.force_quirks = true; + } + self.emit_current_token_followed_by_eof(); + return self.queued_tokens.pop_front(); + } + Some(cp) => { + push_code_point(&mut self.current_builder, cp); + continue; + } + }, + + // 13.2.5.66 DOCTYPE system identifier (single-quoted) state + State::DOCTYPESystemIdentifierSingleQuoted => match current_input_character { + Some(0x27) => { + let sys_id = self.consume_current_builder(); + { + let dd = self.current_token.doctype_data_mut(); + dd.system_identifier = sys_id; + } + self.state = State::AfterDOCTYPESystemIdentifier; + continue; + } + Some(0x00) => { + self.current_builder.push('\u{FFFD}'); + continue; + } + Some(0x3E) => { + let sys_id = self.consume_current_builder(); + { + let dd = self.current_token.doctype_data_mut(); + dd.system_identifier = sys_id; + dd.force_quirks = true; + } + self.state = State::Data; + return self.emit_current_token_direct(); + } + None => { + let sys_id = self.consume_current_builder(); + { + let dd = self.current_token.doctype_data_mut(); + dd.system_identifier = sys_id; + dd.force_quirks = true; + } + self.emit_current_token_followed_by_eof(); + return self.queued_tokens.pop_front(); + } + Some(cp) => { + push_code_point(&mut self.current_builder, cp); + continue; + } + }, + + // 13.2.5.67 After DOCTYPE system identifier state + State::AfterDOCTYPESystemIdentifier => match current_input_character { + Some(cp) if is_whitespace(cp) => continue, + Some(0x3E) => { + self.state = State::Data; + return self.emit_current_token_direct(); + } + None => { + { + let dd = self.current_token.doctype_data_mut(); + dd.force_quirks = true; + } + self.emit_current_token_followed_by_eof(); + return self.queued_tokens.pop_front(); + } + Some(_) => { + // parse error - do NOT set force_quirks + self.reconsume(State::BogusDOCTYPE); + continue; + } + }, + + // 13.2.5.68 Bogus DOCTYPE state + State::BogusDOCTYPE => match current_input_character { + Some(0x3E) => { + self.state = State::Data; + return self.emit_current_token_direct(); + } + Some(0x00) => continue, + None => { + self.emit_current_token_followed_by_eof(); + return self.queued_tokens.pop_front(); + } + Some(_) => continue, + }, + + // 13.2.5.69 CDATA section state + State::CDATASection => match current_input_character { + Some(0x5D) => { + self.state = State::CDATASectionBracket; + continue; + } + None => { + self.emit_eof(); + return self.queued_tokens.pop_front(); + } + Some(cp) => { + return self.return_character_token(cp); + } + }, + + // 13.2.5.70 CDATA section bracket state + State::CDATASectionBracket => match current_input_character { + Some(0x5D) => { + self.state = State::CDATASectionEnd; + continue; + } + _ => { + self.queued_tokens.push_back(self.make_character_token(0x5D)); + self.reconsume(State::CDATASection); + continue; + } + }, + + // 13.2.5.71 CDATA section end state + State::CDATASectionEnd => match current_input_character { + Some(0x5D) => { + return self.return_character_token(0x5D); + } + Some(0x3E) => { + self.state = State::Data; + continue; + } + _ => { + self.queued_tokens.push_back(self.make_character_token(0x5D)); + self.queued_tokens.push_back(self.make_character_token(0x5D)); + self.reconsume(State::CDATASection); + continue; + } + }, + + // 13.2.5.72 Character reference state + State::CharacterReference => { + self.temporary_buffer.clear(); + self.temporary_buffer.push(0x26); // '&' + self.entity_matcher = NamedCharacterReferenceMatcher::new(); + + match current_input_character { + Some(cp) if is_ascii_alphanumeric(cp) => { + self.reconsume(State::NamedCharacterReference); + continue; + } + Some(0x23) => { + self.temporary_buffer.push(0x23); + self.state = State::NumericCharacterReference; + continue; + } + _ => { + self.flush_codepoints_consumed_as_character_reference(); + self.reconsume_in_return_state(current_input_character); + if !self.queued_tokens.is_empty() { + return self.queued_tokens.pop_front(); + } + continue; + } + } + } + + // 13.2.5.73 Named character reference state + State::NamedCharacterReference => { + // Insertion-point path: feed one character at a time. + if self.stop_at_insertion_point && self.insertion_point.is_some() { + if let Some(cp) = current_input_character { + if self.entity_matcher.try_consume_code_point(cp) { + self.temporary_buffer.push(cp); + continue; + } + // Character not accepted by matcher. Reconsume it. + self.restore_to(self.prev_offset); + } else if self.can_run_out_at(self.current_offset) { + // At insertion point with no more chars -- pause. + return None; + } + // Fall through to resolution. + } else { + // Normal path: feed all remaining chars in a tight loop. + if current_input_character.is_some() { + self.restore_to(self.prev_offset); + } + let limit = self.input.len(); + while self.current_offset < limit { + let cp = self.input[self.current_offset]; + if !self.entity_matcher.try_consume_code_point(cp) { + break; + } + self.temporary_buffer.push(cp); + self.skip(1); + } + if !self.input_stream_closed && self.current_offset >= limit { + return None; + } + } + + // Resolution: backtrack overconsumed characters. + let overconsumed = self.entity_matcher.overconsumed_code_points() as usize; + if overconsumed > 0 { + self.restore_to(self.current_offset - overconsumed); + self.temporary_buffer + .truncate(self.temporary_buffer.len() - overconsumed); + } + + if let Some((first_cp, second_cp)) = self.entity_matcher.code_points() { + let ends_with_semi = self.entity_matcher.last_match_ends_with_semicolon(); + + // Check special attribute handling. + let next_cp = self.peek_code_point(0); + if self.consumed_as_part_of_an_attribute() + && !ends_with_semi + && next_cp.is_some_and(|c| c == 0x3D || is_ascii_alphanumeric(c)) + { + self.flush_codepoints_consumed_as_character_reference(); + self.state = self.return_state; + if !self.queued_tokens.is_empty() { + return self.queued_tokens.pop_front(); + } + continue; + } + + if !ends_with_semi { + // parse error + } + + self.temporary_buffer.clear(); + self.temporary_buffer.push(first_cp); + if second_cp != 0 { + self.temporary_buffer.push(second_cp); + } + self.flush_codepoints_consumed_as_character_reference(); + self.state = self.return_state; + if !self.queued_tokens.is_empty() { + return self.queued_tokens.pop_front(); + } + continue; + } else { + // No match found. + self.flush_codepoints_consumed_as_character_reference(); + self.state = State::AmbiguousAmpersand; + continue; + } + } + + // 13.2.5.74 Ambiguous ampersand state + State::AmbiguousAmpersand => match current_input_character { + Some(cp) if is_ascii_alphanumeric(cp) => { + if self.consumed_as_part_of_an_attribute() { + push_code_point(&mut self.current_builder, cp); + continue; + } else { + return self.return_character_token(cp); + } + } + Some(0x3B) => { + // parse error + self.reconsume_in_return_state(current_input_character); + if !self.queued_tokens.is_empty() { + return self.queued_tokens.pop_front(); + } + continue; + } + _ => { + self.reconsume_in_return_state(current_input_character); + if !self.queued_tokens.is_empty() { + return self.queued_tokens.pop_front(); + } + continue; + } + }, + + // 13.2.5.75 Numeric character reference state + State::NumericCharacterReference => { + self.character_reference_code = 0; + match current_input_character { + Some(0x78) | Some(0x58) => { + // 'x' or 'X' + self.temporary_buffer.push(current_input_character.unwrap()); + self.state = State::HexadecimalCharacterReferenceStart; + continue; + } + _ => { + self.reconsume(State::DecimalCharacterReferenceStart); + continue; + } + } + } + + // 13.2.5.76 Hexadecimal character reference start state + State::HexadecimalCharacterReferenceStart => match current_input_character { + Some(cp) if is_ascii_hex_digit(cp) => { + self.reconsume(State::HexadecimalCharacterReference); + continue; + } + _ => { + // parse error + self.flush_codepoints_consumed_as_character_reference(); + self.reconsume_in_return_state(current_input_character); + if !self.queued_tokens.is_empty() { + return self.queued_tokens.pop_front(); + } + continue; + } + }, + + // 13.2.5.77 Decimal character reference start state + State::DecimalCharacterReferenceStart => match current_input_character { + Some(cp) if is_ascii_digit(cp) => { + self.reconsume(State::DecimalCharacterReference); + continue; + } + _ => { + // parse error + self.flush_codepoints_consumed_as_character_reference(); + self.reconsume_in_return_state(current_input_character); + if !self.queued_tokens.is_empty() { + return self.queued_tokens.pop_front(); + } + continue; + } + }, + + // 13.2.5.78 Hexadecimal character reference state + State::HexadecimalCharacterReference => match current_input_character { + Some(cp) if is_ascii_digit(cp) => { + self.character_reference_code = + self.character_reference_code.wrapping_mul(16).wrapping_add(cp - 0x30); + continue; + } + Some(cp @ 0x41..=0x46) => { + self.character_reference_code = + self.character_reference_code.wrapping_mul(16).wrapping_add(cp - 0x37); + continue; + } + Some(cp @ 0x61..=0x66) => { + self.character_reference_code = + self.character_reference_code.wrapping_mul(16).wrapping_add(cp - 0x57); + continue; + } + Some(0x3B) => { + self.state = State::NumericCharacterReferenceEnd; + continue; + } + _ => { + // parse error + self.reconsume(State::NumericCharacterReferenceEnd); + continue; + } + }, + + // 13.2.5.79 Decimal character reference state + State::DecimalCharacterReference => match current_input_character { + Some(cp) if is_ascii_digit(cp) => { + self.character_reference_code = + self.character_reference_code.wrapping_mul(10).wrapping_add(cp - 0x30); + continue; + } + Some(0x3B) => { + self.state = State::NumericCharacterReferenceEnd; + continue; + } + _ => { + // parse error + self.reconsume(State::NumericCharacterReferenceEnd); + continue; + } + }, + + // 13.2.5.80 Numeric character reference end state + State::NumericCharacterReferenceEnd => { + // Don't consume + if current_input_character.is_some() { + self.restore_to(self.prev_offset); + } + + let code = self.character_reference_code; + let code = if code == 0 || code > 0x10FFFF || is_surrogate(code) { + 0xFFFD + } else if is_noncharacter(code) { + code // parse error but keep value + } else if let Some(replacement) = numeric_char_ref_replacement(code) { + replacement + } else if code == 0x0D { + 0x000D // parse error but keep + } else if code != 0 && code <= 0x1F && code != 0x09 && code != 0x0A && code != 0x0C { + code // control char, parse error but keep + } else if (0x7F..=0x9F).contains(&code) { + if let Some(replacement) = numeric_char_ref_replacement(code) { + replacement + } else { + code + } + } else { + code + }; + + self.temporary_buffer.clear(); + self.temporary_buffer.push(code); + self.flush_codepoints_consumed_as_character_reference(); + self.state = self.return_state; + if !self.queued_tokens.is_empty() { + return self.queued_tokens.pop_front(); + } + continue; + } + + // RCDATA less-than sign state + State::RCDATALessThanSign => match current_input_character { + Some(0x2F) => { + self.temporary_buffer.clear(); + self.state = State::RCDATAEndTagOpen; + continue; + } + _ => { + self.queued_tokens.push_back(self.make_character_token(0x3C)); + self.reconsume(State::RCDATA); + continue; + } + }, + + // RCDATA end tag open state + State::RCDATAEndTagOpen => match current_input_character { + Some(cp) if is_ascii_alpha(cp) => { + self.create_new_token(TokenType::EndTag); + self.reconsume(State::RCDATAEndTagName); + continue; + } + _ => { + self.queued_tokens.push_back(self.make_character_token(0x3C)); + self.queued_tokens.push_back(self.make_character_token(0x2F)); + self.reconsume(State::RCDATA); + continue; + } + }, + + // RCDATA end tag name state + State::RCDATAEndTagName => { + self.handle_rawtext_end_tag_name(current_input_character, State::RCDATA); + if !self.queued_tokens.is_empty() { + return self.queued_tokens.pop_front(); + } + continue; + } + + // RAWTEXT less-than sign state + State::RAWTEXTLessThanSign => match current_input_character { + Some(0x2F) => { + self.temporary_buffer.clear(); + self.state = State::RAWTEXTEndTagOpen; + continue; + } + _ => { + self.queued_tokens.push_back(self.make_character_token(0x3C)); + self.reconsume(State::RAWTEXT); + continue; + } + }, + + // RAWTEXT end tag open state + State::RAWTEXTEndTagOpen => match current_input_character { + Some(cp) if is_ascii_alpha(cp) => { + self.create_new_token(TokenType::EndTag); + self.reconsume(State::RAWTEXTEndTagName); + continue; + } + _ => { + self.queued_tokens.push_back(self.make_character_token(0x3C)); + self.queued_tokens.push_back(self.make_character_token(0x2F)); + self.reconsume(State::RAWTEXT); + continue; + } + }, + + // RAWTEXT end tag name state + State::RAWTEXTEndTagName => { + self.handle_rawtext_end_tag_name(current_input_character, State::RAWTEXT); + if !self.queued_tokens.is_empty() { + return self.queued_tokens.pop_front(); + } + continue; + } + + // Script data less-than sign state + State::ScriptDataLessThanSign => match current_input_character { + Some(0x2F) => { + self.temporary_buffer.clear(); + self.state = State::ScriptDataEndTagOpen; + continue; + } + Some(0x21) => { + self.queued_tokens.push_back(self.make_character_token(0x3C)); + self.queued_tokens.push_back(self.make_character_token(0x21)); + self.state = State::ScriptDataEscapeStart; + return self.queued_tokens.pop_front(); + } + _ => { + self.queued_tokens.push_back(self.make_character_token(0x3C)); + self.reconsume(State::ScriptData); + continue; + } + }, + + // Script data end tag open state + State::ScriptDataEndTagOpen => match current_input_character { + Some(cp) if is_ascii_alpha(cp) => { + self.create_new_token(TokenType::EndTag); + self.reconsume(State::ScriptDataEndTagName); + continue; + } + _ => { + self.queued_tokens.push_back(self.make_character_token(0x3C)); + self.queued_tokens.push_back(self.make_character_token(0x2F)); + self.reconsume(State::ScriptData); + continue; + } + }, + + // Script data end tag name state + State::ScriptDataEndTagName => { + self.handle_rawtext_end_tag_name(current_input_character, State::ScriptData); + if !self.queued_tokens.is_empty() { + return self.queued_tokens.pop_front(); + } + continue; + } + + // Script data escape start state + State::ScriptDataEscapeStart => match current_input_character { + Some(0x2D) => { + self.state = State::ScriptDataEscapeStartDash; + return self.return_character_token(0x2D); + } + _ => { + self.reconsume(State::ScriptData); + continue; + } + }, + + // Script data escape start dash state + State::ScriptDataEscapeStartDash => match current_input_character { + Some(0x2D) => { + self.state = State::ScriptDataEscapedDashDash; + return self.return_character_token(0x2D); + } + _ => { + self.reconsume(State::ScriptData); + continue; + } + }, + + // Script data escaped state + State::ScriptDataEscaped => match current_input_character { + Some(0x2D) => { + self.state = State::ScriptDataEscapedDash; + return self.return_character_token(0x2D); + } + Some(0x3C) => { + self.state = State::ScriptDataEscapedLessThanSign; + continue; + } + Some(0x00) => { + return self.return_character_token(0xFFFD); + } + None => { + self.emit_eof(); + return self.queued_tokens.pop_front(); + } + Some(cp) => { + return self.return_character_token(cp); + } + }, + + // Script data escaped dash state + State::ScriptDataEscapedDash => match current_input_character { + Some(0x2D) => { + self.state = State::ScriptDataEscapedDashDash; + return self.return_character_token(0x2D); + } + Some(0x3C) => { + self.state = State::ScriptDataEscapedLessThanSign; + continue; + } + Some(0x00) => { + self.state = State::ScriptDataEscaped; + return self.return_character_token(0xFFFD); + } + None => { + self.emit_eof(); + return self.queued_tokens.pop_front(); + } + Some(cp) => { + self.state = State::ScriptDataEscaped; + return self.return_character_token(cp); + } + }, + + // Script data escaped dash dash state + State::ScriptDataEscapedDashDash => match current_input_character { + Some(0x2D) => { + return self.return_character_token(0x2D); + } + Some(0x3C) => { + self.state = State::ScriptDataEscapedLessThanSign; + continue; + } + Some(0x3E) => { + self.state = State::ScriptData; + return self.return_character_token(0x3E); + } + Some(0x00) => { + self.state = State::ScriptDataEscaped; + return self.return_character_token(0xFFFD); + } + None => { + self.emit_eof(); + return self.queued_tokens.pop_front(); + } + Some(cp) => { + self.state = State::ScriptDataEscaped; + return self.return_character_token(cp); + } + }, + + // Script data escaped less-than sign state + State::ScriptDataEscapedLessThanSign => match current_input_character { + Some(0x2F) => { + self.temporary_buffer.clear(); + self.state = State::ScriptDataEscapedEndTagOpen; + continue; + } + Some(cp) if is_ascii_alpha(cp) => { + self.temporary_buffer.clear(); + self.queued_tokens.push_back(self.make_character_token(0x3C)); + self.reconsume(State::ScriptDataDoubleEscapeStart); + continue; + } + _ => { + self.queued_tokens.push_back(self.make_character_token(0x3C)); + self.reconsume(State::ScriptDataEscaped); + continue; + } + }, + + // Script data escaped end tag open state + State::ScriptDataEscapedEndTagOpen => match current_input_character { + Some(cp) if is_ascii_alpha(cp) => { + self.create_new_token(TokenType::EndTag); + self.reconsume(State::ScriptDataEscapedEndTagName); + continue; + } + _ => { + self.queued_tokens.push_back(self.make_character_token(0x3C)); + self.queued_tokens.push_back(self.make_character_token(0x2F)); + self.reconsume(State::ScriptDataEscaped); + continue; + } + }, + + // Script data escaped end tag name state + State::ScriptDataEscapedEndTagName => { + self.handle_rawtext_end_tag_name(current_input_character, State::ScriptDataEscaped); + if !self.queued_tokens.is_empty() { + return self.queued_tokens.pop_front(); + } + continue; + } + + // Script data double escape start state + State::ScriptDataDoubleEscapeStart => match current_input_character { + Some(cp) if is_whitespace(cp) || cp == 0x2F || cp == 0x3E => { + if self.temporary_buffer_equals_script() { + self.state = State::ScriptDataDoubleEscaped; + } else { + self.state = State::ScriptDataEscaped; + } + return self.return_character_token(cp); + } + Some(cp) if is_ascii_upper_alpha(cp) => { + self.temporary_buffer.push(to_ascii_lowercase(cp)); + return self.return_character_token(cp); + } + Some(cp) if is_ascii_lower_alpha(cp) => { + self.temporary_buffer.push(cp); + return self.return_character_token(cp); + } + _ => { + self.reconsume(State::ScriptDataEscaped); + continue; + } + }, + + // Script data double escaped state + State::ScriptDataDoubleEscaped => match current_input_character { + Some(0x2D) => { + self.state = State::ScriptDataDoubleEscapedDash; + return self.return_character_token(0x2D); + } + Some(0x3C) => { + self.state = State::ScriptDataDoubleEscapedLessThanSign; + return self.return_character_token(0x3C); + } + Some(0x00) => { + return self.return_character_token(0xFFFD); + } + None => { + self.emit_eof(); + return self.queued_tokens.pop_front(); + } + Some(cp) => { + return self.return_character_token(cp); + } + }, + + // Script data double escaped dash state + State::ScriptDataDoubleEscapedDash => match current_input_character { + Some(0x2D) => { + self.state = State::ScriptDataDoubleEscapedDashDash; + return self.return_character_token(0x2D); + } + Some(0x3C) => { + self.state = State::ScriptDataDoubleEscapedLessThanSign; + return self.return_character_token(0x3C); + } + Some(0x00) => { + self.state = State::ScriptDataDoubleEscaped; + return self.return_character_token(0xFFFD); + } + None => { + self.emit_eof(); + return self.queued_tokens.pop_front(); + } + Some(cp) => { + self.state = State::ScriptDataDoubleEscaped; + return self.return_character_token(cp); + } + }, + + // Script data double escaped dash dash state + State::ScriptDataDoubleEscapedDashDash => match current_input_character { + Some(0x2D) => { + return self.return_character_token(0x2D); + } + Some(0x3C) => { + self.state = State::ScriptDataDoubleEscapedLessThanSign; + return self.return_character_token(0x3C); + } + Some(0x3E) => { + self.state = State::ScriptData; + return self.return_character_token(0x3E); + } + Some(0x00) => { + self.state = State::ScriptDataDoubleEscaped; + return self.return_character_token(0xFFFD); + } + None => { + self.emit_eof(); + return self.queued_tokens.pop_front(); + } + Some(cp) => { + self.state = State::ScriptDataDoubleEscaped; + return self.return_character_token(cp); + } + }, + + // Script data double escaped less-than sign state + State::ScriptDataDoubleEscapedLessThanSign => match current_input_character { + Some(0x2F) => { + self.temporary_buffer.clear(); + self.state = State::ScriptDataDoubleEscapeEnd; + return self.return_character_token(0x2F); + } + _ => { + self.reconsume(State::ScriptDataDoubleEscaped); + continue; + } + }, + + // Script data double escape end state + State::ScriptDataDoubleEscapeEnd => match current_input_character { + Some(cp) if is_whitespace(cp) || cp == 0x2F || cp == 0x3E => { + if self.temporary_buffer_equals_script() { + self.state = State::ScriptDataEscaped; + } else { + self.state = State::ScriptDataDoubleEscaped; + } + return self.return_character_token(cp); + } + Some(cp) if is_ascii_upper_alpha(cp) => { + self.temporary_buffer.push(to_ascii_lowercase(cp)); + return self.return_character_token(cp); + } + Some(cp) if is_ascii_lower_alpha(cp) => { + self.temporary_buffer.push(cp); + return self.return_character_token(cp); + } + _ => { + self.reconsume(State::ScriptDataDoubleEscaped); + continue; + } + }, + } + } + } + + // -- Helper methods -- + + fn reconsume(&mut self, new_state: State) { + self.state = new_state; + if self.current_offset > 0 { + self.restore_to(self.prev_offset); + } + } + + fn reconsume_in_return_state(&mut self, current_input_character: Option) { + self.state = self.return_state; + if current_input_character.is_some() { + self.restore_to(self.prev_offset); + } + } + + #[inline(always)] + fn make_character_token(&self, code_point: u32) -> Token { + Token::new_character(code_point) + } + + fn set_attribute_name(&mut self) { + let id = crate::interned_names::lookup_attr_name(self.current_builder.as_bytes()); + let name_length = self.current_builder.chars().count() as u64; + let name_end = self.nth_last_position(1); + let name_start = Position { + line: name_end.line, + column: name_end.column.saturating_sub(name_length), + }; + if id != 0 { + self.current_builder.clear(); + if let Some(attr) = self.current_token.attributes_mut().last_mut() { + attr.local_name_id = id; + attr.local_name.clear(); + attr.name_start_position = name_start; + attr.name_end_position = name_end; + } + } else { + let name = self.consume_current_builder(); + if let Some(attr) = self.current_token.attributes_mut().last_mut() { + attr.local_name_id = 0; + attr.local_name = name; + attr.name_start_position = name_start; + attr.name_end_position = name_end; + } + } + } + + fn set_attribute_value(&mut self) { + let value = self.consume_current_builder(); + let value_end = self.nth_last_position(1); + if let Some(attr) = self.current_token.attributes_mut().last_mut() { + attr.value = value; + attr.value_end_position = value_end; + } + } + + fn temporary_buffer_equals_script(&self) -> bool { + self.temporary_buffer == [0x73, 0x63, 0x72, 0x69, 0x70, 0x74] + } + + /// Handle the common end-tag-name pattern shared by RCDATA, RAWTEXT, + /// ScriptData, and ScriptDataEscaped end tag name states. + fn handle_rawtext_end_tag_name(&mut self, current_input_character: Option, fallback_state: State) { + match current_input_character { + Some(cp) if is_whitespace(cp) || cp == 0x2F || cp == 0x3E => { + self.commit_current_builder_as_tag_name(); + if self.current_end_tag_token_is_appropriate() { + match cp { + _ if is_whitespace(cp) => { + self.state = State::BeforeAttributeName; + return; + } + 0x2F => { + self.state = State::SelfClosingStartTag; + return; + } + 0x3E => { + self.state = State::Data; + self.emit_current_token(); + return; + } + _ => {} + } + } + // Not appropriate - emit buffered chars + self.queued_tokens.push_back(self.make_character_token(0x3C)); + self.queued_tokens.push_back(self.make_character_token(0x2F)); + for i in 0..self.temporary_buffer.len() { + let cp = self.temporary_buffer[i]; + self.queued_tokens.push_back(self.make_character_token(cp)); + } + self.current_builder.clear(); + self.reconsume(fallback_state); + } + Some(cp) if is_ascii_upper_alpha(cp) => { + push_code_point(&mut self.current_builder, to_ascii_lowercase(cp)); + self.temporary_buffer.push(cp); + } + Some(cp) if is_ascii_lower_alpha(cp) => { + self.current_builder.push(char::from_u32(cp).unwrap()); + self.temporary_buffer.push(cp); + } + _ => { + self.queued_tokens.push_back(self.make_character_token(0x3C)); + self.queued_tokens.push_back(self.make_character_token(0x2F)); + for i in 0..self.temporary_buffer.len() { + let cp = self.temporary_buffer[i]; + self.queued_tokens.push_back(self.make_character_token(cp)); + } + self.current_builder.clear(); + self.reconsume(fallback_state); + } + } + } +} diff --git a/Libraries/LibWeb/Rust/Cargo.toml b/Libraries/LibWeb/Rust/Cargo.toml index e71fddd47b..4e9b5c8b22 100644 --- a/Libraries/LibWeb/Rust/Cargo.toml +++ b/Libraries/LibWeb/Rust/Cargo.toml @@ -9,6 +9,7 @@ crate-type = ["staticlib"] # After changing dependencies, regenerate the Flatpak sources: # python3 Meta/CMake/flatpak/generate-cargo-sources.py [dependencies] +libweb_html_tokenizer = { path = "../HTML/Parser/Rust" } [build-dependencies] cbindgen = "0.29" diff --git a/Libraries/LibWeb/Rust/src/lib.rs b/Libraries/LibWeb/Rust/src/lib.rs index 5f6283820b..30a5e7fcdc 100644 --- a/Libraries/LibWeb/Rust/src/lib.rs +++ b/Libraries/LibWeb/Rust/src/lib.rs @@ -9,6 +9,8 @@ mod rust_allocator; mod css_tokenizer; +pub use libweb_html_tokenizer as html_tokenizer; + use std::ffi::c_void; use std::panic::{AssertUnwindSafe, catch_unwind}; diff --git a/Tests/LibWeb/TestHTMLTokenizer.cpp b/Tests/LibWeb/TestHTMLTokenizer.cpp index f67e9f9a83..f388af39f4 100644 --- a/Tests/LibWeb/TestHTMLTokenizer.cpp +++ b/Tests/LibWeb/TestHTMLTokenizer.cpp @@ -199,6 +199,19 @@ TEST_CASE(character_reference_in_attribute) END_ENUMERATION(); } +TEST_CASE(duplicate_attributes_are_reported) +{ + auto tokens = run_tokenizer(""sv); + auto& token = tokens.first(); + EXPECT_EQ(token.type(), Token::Type::StartTag); + EXPECT(token.had_duplicate_attribute()); + EXPECT_EQ(token.attribute_count(), 1u); + + auto nonce = token.raw_attribute("nonce"_fly_string); + VERIFY(nonce.has_value()); + EXPECT_EQ(nonce->value, "x"); +} + TEST_CASE(named_character_reference) { auto tokens = run_tokenizer("⋶¬it;&cz"sv); @@ -275,3 +288,73 @@ TEST_CASE(ambiguous_ampersand_offset) EXPECT_EQ(token.start_position().line, 0u); EXPECT_EQ(token.start_position().column, 1u); } + +TEST_CASE(insertion_point_inside_fast_tag_name) +{ + Tokenizer tokenizer; + tokenizer.update_insertion_point(); + tokenizer.insert_input_at_insertion_point(""sv); + + EXPECT(!tokenizer.next_token(Tokenizer::StopAtInsertionPoint::Yes).has_value()); + EXPECT(tokenizer.is_insertion_point_reached()); + EXPECT_EQ(tokenizer.unparsed_input(), "def>"sv); + + tokenizer.insert_input_at_insertion_point("x"sv); + tokenizer.undefine_insertion_point(); + tokenizer.close_input_stream(); + + auto token = tokenizer.next_token(); + VERIFY(token.has_value()); + EXPECT_EQ(token->type(), Token::Type::StartTag); + EXPECT_EQ(token->tag_name(), "abcxdef"); +} + +TEST_CASE(insertion_point_inside_fast_attribute_name) +{ + Tokenizer tokenizer; + tokenizer.update_insertion_point(); + tokenizer.insert_input_at_insertion_point("

"sv); + + EXPECT(!tokenizer.next_token(Tokenizer::StopAtInsertionPoint::Yes).has_value()); + EXPECT(tokenizer.is_insertion_point_reached()); + EXPECT_EQ(tokenizer.unparsed_input(), "def=value>"sv); + + tokenizer.insert_input_at_insertion_point("x"sv); + tokenizer.undefine_insertion_point(); + tokenizer.close_input_stream(); + + auto token = tokenizer.next_token(); + VERIFY(token.has_value()); + EXPECT_EQ(token->type(), Token::Type::StartTag); + EXPECT_EQ(token->attribute_count(), 1u); + + auto attribute = token->raw_attribute("abcxdef"_fly_string); + VERIFY(attribute.has_value()); + EXPECT_EQ(attribute->value, "value"); +} + +TEST_CASE(insertion_point_inside_fast_quoted_attribute_value) +{ + Tokenizer tokenizer; + tokenizer.update_insertion_point(); + tokenizer.insert_input_at_insertion_point("

"sv); + + EXPECT(!tokenizer.next_token(Tokenizer::StopAtInsertionPoint::Yes).has_value()); + EXPECT(tokenizer.is_insertion_point_reached()); + EXPECT_EQ(tokenizer.unparsed_input(), "def\">"sv); + + tokenizer.insert_input_at_insertion_point("x"sv); + tokenizer.undefine_insertion_point(); + tokenizer.close_input_stream(); + + auto token = tokenizer.next_token(); + VERIFY(token.has_value()); + EXPECT_EQ(token->type(), Token::Type::StartTag); + + auto attribute = token->raw_attribute("a"_fly_string); + VERIFY(attribute.has_value()); + EXPECT_EQ(attribute->value, "abcxdef"); +} diff --git a/Tests/LibWeb/Text/expected/ContentSecurityPolicy/script-duplicate-nonce-attribute.txt b/Tests/LibWeb/Text/expected/ContentSecurityPolicy/script-duplicate-nonce-attribute.txt new file mode 100644 index 0000000000..c660efa29c --- /dev/null +++ b/Tests/LibWeb/Text/expected/ContentSecurityPolicy/script-duplicate-nonce-attribute.txt @@ -0,0 +1,2 @@ +PASS: duplicate nonce script blocked +PASS: valid nonce script executed diff --git a/Tests/LibWeb/Text/input/ContentSecurityPolicy/script-duplicate-nonce-attribute.html b/Tests/LibWeb/Text/input/ContentSecurityPolicy/script-duplicate-nonce-attribute.html new file mode 100644 index 0000000000..6267ea1f70 --- /dev/null +++ b/Tests/LibWeb/Text/input/ContentSecurityPolicy/script-duplicate-nonce-attribute.html @@ -0,0 +1,15 @@ + + + + + +