dmaax/ladybird
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

LibWeb: Pause tokenizer at a CR right before the insertion point

Browse source 
HTML newline normalization collapses CRLF into a single LF, so
next_code_point() needs one code point of lookahead at a CR to decide
whether the CR stands alone or is the first half of a CRLF pair. When
the tokenizer is paused at the insertion point and the next code point
to consume is a CR sitting one position before it, that lookahead has
not been written yet.

Previously the tokenizer consumed the CR and emitted it as LF, so a
subsequent document.write() that began with LF surfaced as a second
LF instead of being absorbed into the original CRLF pair.

Stop one code point earlier in this case and wait for the next write
to arrive. This makes four html5lib write_single WPT tests pass.
This commit is contained in:
Aliaksandr Kalenik 2026-04-27 20:38:28 +02:00 committed by Alexander Kalenik
parent aad66085b3
commit b6ffd51d1c
6 changed files with 38 additions and 21 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

38
Libraries/LibWeb/HTML/Parser/HTMLTokenizer.cpp
View file

@ -31,14 +31,14 @@ namespace Web::HTML {
SWITCH_TO_WITH_UNCLEAN_BUILDER(new_state); \
} while (0)
#define SWITCH_TO_WITH_UNCLEAN_BUILDER(new_state) \
do { \
will_switch_to(State::new_state); \
m_state = State::new_state; \
if (stop_at_insertion_point == StopAtInsertionPoint::Yes && is_insertion_point_reached()) \
return {}; \
CONSUME_NEXT_INPUT_CHARACTER; \
goto new_state; \
#define SWITCH_TO_WITH_UNCLEAN_BUILDER(new_state) \
do { \
will_switch_to(State::new_state); \
m_state = State::new_state; \
if (should_pause_before_next_input_character(stop_at_insertion_point)) \
return {}; \
CONSUME_NEXT_INPUT_CHARACTER; \
goto new_state; \
} while (0)
#define RECONSUME_IN(new_state) \
@ -194,6 +194,26 @@ static inline void log_parse_error(SourceLocation const& location = SourceLocati
dbgln_if(TOKENIZER_TRACE_DEBUG, "Parse error (tokenization) {}", location);
}
bool HTMLTokenizer::should_pause_before_next_input_character(StopAtInsertionPoint stop_at_insertion_point) const
{
if (stop_at_insertion_point != StopAtInsertionPoint::Yes || !m_insertion_point.has_value())
return false;
if (m_current_offset >= *m_insertion_point)
return true;
if (m_current_offset < static_cast<ssize_t>(m_decoded_input.size())
&& m_decoded_input[m_current_offset] == '\r'
&& m_current_offset + 1 == *m_insertion_point) {
// Newline normalization needs one code point of lookahead for CRLF. If document.write()
// stops the tokenizer immediately after a CR, wait for more input before deciding whether
// it is a CRLF pair or a standalone CR.
return true;
}
return false;
}
Optional<u32> HTMLTokenizer::next_code_point(StopAtInsertionPoint stop_at_insertion_point)
{
if (m_current_offset >= static_cast<ssize_t>(m_decoded_input.size()))
@ -276,7 +296,7 @@ _StartOfFunction:
return {};
for (;;) {
if (stop_at_insertion_point == StopAtInsertionPoint::Yes && is_insertion_point_reached())
if (should_pause_before_next_input_character(stop_at_insertion_point))
return {};
auto current_input_character = next_code_point(stop_at_insertion_point);

1
Libraries/LibWeb/HTML/Parser/HTMLTokenizer.h
View file

@ -164,6 +164,7 @@ private:
RanOutOfCharacters,
};
[[nodiscard]] ConsumeNextResult consume_next_if_match(StringView, StopAtInsertionPoint, CaseSensitivity = CaseSensitivity::CaseSensitive);
bool should_pause_before_next_input_character(StopAtInsertionPoint) const;
void create_new_token(HTMLToken::Type);
bool current_end_tag_token_is_appropriate() const;

5
Tests/LibWeb/Text/expected/wpt-import/html/syntax/parsing/html5lib_domjs-unsafe@run_type=write_single.txt
View file

@ -2,11 +2,10 @@ Harness status: OK
Found 49 tests
48 Pass
1 Fail
49 Pass
Pass html5lib_domjs-unsafe.html b76df4b192c11a5ef7a3e895a7dd2aaea2dcc5c7
Pass html5lib_domjs-unsafe.html da5cf60e732fcf20d288e772e51a7a9ccd3471e4
Fail html5lib_domjs-unsafe.html c32f77685b52841e1685110a8a6fac17b3ae127a
Pass html5lib_domjs-unsafe.html c32f77685b52841e1685110a8a6fac17b3ae127a
Pass html5lib_domjs-unsafe.html 4fd39e063b0ad1fa82274d6de3122859f76934a8
Pass html5lib_domjs-unsafe.html 7208fbe8702c37cf180b1c736c984f5340517408
Pass html5lib_domjs-unsafe.html 7b5dd0bcb2a92ebf1974a0ec6017a798e774ca5b

5
Tests/LibWeb/Text/expected/wpt-import/html/syntax/parsing/html5lib_plain-text-unsafe-write@run_type=write_single.txt
View file

@ -2,8 +2,7 @@ Harness status: OK
Found 33 tests
32 Pass
1 Fail
33 Pass
Pass html5lib_plain-text-unsafe.html 7e4ca4cb5e73852744a876bf8652dd2c8998d94a
Pass html5lib_plain-text-unsafe.html e552342bdd3ac62316bd91126556512683f3d4a9
Pass html5lib_plain-text-unsafe.html 9112cef60139e6988b66334e522777329051442a
@ -27,7 +26,7 @@ Pass html5lib_plain-text-unsafe.html 4b9e317cba617324fea77a4e525602b3ad2717ef
Pass html5lib_plain-text-unsafe.html a1e08cb99d89381a1c997fcd60bad23c029c4500
Pass html5lib_plain-text-unsafe.html 26d850208425cc885d4d0143909cf341f61fa1f1
Pass html5lib_plain-text-unsafe.html 68f0365c01dc386c706edd2b18672f9d85caaa2e
Fail html5lib_plain-text-unsafe.html e415a2e7cf090e2c308af905d52c5f8163ae52ce
Pass html5lib_plain-text-unsafe.html e415a2e7cf090e2c308af905d52c5f8163ae52ce
Pass html5lib_plain-text-unsafe.html 822702de65b80ec8e79da19335ab9d6a49f6ec6a
Pass html5lib_plain-text-unsafe.html ee8b017ab043ff51b593787961626acb4c6488cd
Pass html5lib_plain-text-unsafe.html 304960c795639128844445166238350682ba0516

5
Tests/LibWeb/Text/expected/wpt-import/html/syntax/parsing/html5lib_plain-text-unsafe-write_single@run_type=write_single.txt
View file

@ -2,8 +2,7 @@ Harness status: OK
Found 33 tests
32 Pass
1 Fail
33 Pass
Pass html5lib_plain-text-unsafe.html 7e4ca4cb5e73852744a876bf8652dd2c8998d94a
Pass html5lib_plain-text-unsafe.html e552342bdd3ac62316bd91126556512683f3d4a9
Pass html5lib_plain-text-unsafe.html 9112cef60139e6988b66334e522777329051442a
@ -27,7 +26,7 @@ Pass html5lib_plain-text-unsafe.html 4b9e317cba617324fea77a4e525602b3ad2717ef
Pass html5lib_plain-text-unsafe.html a1e08cb99d89381a1c997fcd60bad23c029c4500
Pass html5lib_plain-text-unsafe.html 26d850208425cc885d4d0143909cf341f61fa1f1
Pass html5lib_plain-text-unsafe.html 68f0365c01dc386c706edd2b18672f9d85caaa2e
Fail html5lib_plain-text-unsafe.html e415a2e7cf090e2c308af905d52c5f8163ae52ce
Pass html5lib_plain-text-unsafe.html e415a2e7cf090e2c308af905d52c5f8163ae52ce
Pass html5lib_plain-text-unsafe.html 822702de65b80ec8e79da19335ab9d6a49f6ec6a
Pass html5lib_plain-text-unsafe.html ee8b017ab043ff51b593787961626acb4c6488cd
Pass html5lib_plain-text-unsafe.html 304960c795639128844445166238350682ba0516

5
Tests/LibWeb/Text/expected/wpt-import/html/syntax/parsing/html5lib_plain-text-unsafe@run_type=write_single.txt
View file

@ -2,8 +2,7 @@ Harness status: OK
Found 33 tests
32 Pass
1 Fail
33 Pass
Pass html5lib_plain-text-unsafe.html 7e4ca4cb5e73852744a876bf8652dd2c8998d94a
Pass html5lib_plain-text-unsafe.html e552342bdd3ac62316bd91126556512683f3d4a9
Pass html5lib_plain-text-unsafe.html 9112cef60139e6988b66334e522777329051442a
@ -27,7 +26,7 @@ Pass html5lib_plain-text-unsafe.html 4b9e317cba617324fea77a4e525602b3ad2717ef
Pass html5lib_plain-text-unsafe.html a1e08cb99d89381a1c997fcd60bad23c029c4500
Pass html5lib_plain-text-unsafe.html 26d850208425cc885d4d0143909cf341f61fa1f1
Pass html5lib_plain-text-unsafe.html 68f0365c01dc386c706edd2b18672f9d85caaa2e
Fail html5lib_plain-text-unsafe.html e415a2e7cf090e2c308af905d52c5f8163ae52ce
Pass html5lib_plain-text-unsafe.html e415a2e7cf090e2c308af905d52c5f8163ae52ce
Pass html5lib_plain-text-unsafe.html 822702de65b80ec8e79da19335ab9d6a49f6ec6a
Pass html5lib_plain-text-unsafe.html ee8b017ab043ff51b593787961626acb4c6488cd
Pass html5lib_plain-text-unsafe.html 304960c795639128844445166238350682ba0516