dmaax/ladybird
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

LibTextCodec: Make StreamingDecoder own decoder lookup

Browse source 
Pass an encoding label to StreamingDecoder instead of requiring
callers to pre-resolve a Decoder reference.

This removes redundant decoder_for() plumbing from callers and lets
StreamingDecoder hide how chunked decoder state is represented.
This commit is contained in:
Shannon Booth 2026-06-20 00:50:19 +02:00 committed by Andreas Kling
parent ebc439f0f2
commit 62e4da6d3e
4 changed files with 19 additions and 26 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
Libraries/LibTextCodec/Decoder.cpp
View file

@ -703,6 +703,11 @@ size_t ISO2022JPDecoder::incomplete_tail_length(ReadonlyBytes bytes) const
return 0;
}
StreamingDecoder::StreamingDecoder(StringView encoding)
: m_decoder(decoder_for(encoding).value())
{
}
ErrorOr<String> StreamingDecoder::to_utf8(ReadonlyBytes input)
{
ReadonlyBytes bytes;

5
Libraries/LibTextCodec/Decoder.h
View file

@ -140,10 +140,7 @@ public:
// Preserves incomplete trailing decoder tokens when callers provide input in chunks.
class TEXTCODEC_API StreamingDecoder final {
public:
explicit StreamingDecoder(Decoder& decoder)
: m_decoder(decoder)
{
}
explicit StreamingDecoder(StringView encoding);
ErrorOr<String> to_utf8(ReadonlyBytes);
ErrorOr<String> finish();

5
Libraries/LibWeb/HTML/Parser/IncrementalDocumentParser.cpp
View file

@ -66,12 +66,9 @@ void IncrementalDocumentParser::initialize_parser(ReadonlyBytes sniff_bytes)
: run_encoding_sniffing_algorithm(m_document, sniff_bytes, m_mime_type);
dbgln_if(HTML_PARSER_DEBUG, "The incremental HTML parser selected encoding '{}'", encoding);
auto decoder = TextCodec::decoder_for(encoding);
VERIFY(decoder.has_value());
auto standardized_encoding = TextCodec::get_standardized_encoding(encoding);
VERIFY(standardized_encoding.has_value());
m_decoder = make<TextCodec::StreamingDecoder>(decoder.value());
m_decoder = make<TextCodec::StreamingDecoder>(standardized_encoding.value());
// https://html.spec.whatwg.org/multipage/parsing.html#determining-the-character-encoding
// The document's character encoding must immediately be set to the value returned from this

30
Tests/LibTextCodec/TestTextDecoders.cpp
View file

@ -127,8 +127,7 @@ TEST_CASE(test_utf16be_process_code_points)
TEST_CASE(test_streaming_decoder_utf8_mid_sequence)
{
auto& decoder = decoder_for("UTF-8"sv);
auto streaming_decoder = TextCodec::StreamingDecoder { decoder };
auto streaming_decoder = TextCodec::StreamingDecoder { "UTF-8"sv };
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 'a', 0xc3 }))), "a"sv);
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0xa9, 'b' }))), "éb"sv);
@ -138,7 +137,7 @@ TEST_CASE(test_streaming_decoder_utf8_mid_sequence)
TEST_CASE(test_streaming_decoder_finishes_incomplete_sequence)
{
auto& decoder = decoder_for("UTF-8"sv);
auto streaming_decoder = TextCodec::StreamingDecoder { decoder };
auto streaming_decoder = TextCodec::StreamingDecoder { "UTF-8"sv };
auto incomplete_sequence = Vector<u8> { 0xc3 };
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes(incomplete_sequence))), ""sv);
@ -147,21 +146,18 @@ TEST_CASE(test_streaming_decoder_finishes_incomplete_sequence)
TEST_CASE(test_streaming_decoder_utf8_invalid_second_byte_tail)
{
auto& decoder = decoder_for("UTF-8"sv);
auto streaming_decoder_for_overlong_three_byte_sequence = TextCodec::StreamingDecoder { decoder };
auto streaming_decoder_for_overlong_three_byte_sequence = TextCodec::StreamingDecoder { "UTF-8"sv };
EXPECT_EQ(MUST(streaming_decoder_for_overlong_three_byte_sequence.to_utf8(bytes({ 0xe0, 0x80 }))), "\xef\xbf\xbd\xef\xbf\xbd"sv);
EXPECT_EQ(MUST(streaming_decoder_for_overlong_three_byte_sequence.finish()), ""sv);
auto streaming_decoder_for_out_of_range_four_byte_sequence = TextCodec::StreamingDecoder { decoder };
auto streaming_decoder_for_out_of_range_four_byte_sequence = TextCodec::StreamingDecoder { "UTF-8"sv };
EXPECT_EQ(MUST(streaming_decoder_for_out_of_range_four_byte_sequence.to_utf8(bytes({ 0xf4, 0x90 }))), "\xef\xbf\xbd\xef\xbf\xbd"sv);
EXPECT_EQ(MUST(streaming_decoder_for_out_of_range_four_byte_sequence.finish()), ""sv);
}
TEST_CASE(test_streaming_decoder_utf8_valid_second_byte_tail)
{
auto& decoder = decoder_for("UTF-8"sv);
auto streaming_decoder = TextCodec::StreamingDecoder { decoder };
auto streaming_decoder = TextCodec::StreamingDecoder { "UTF-8"sv };
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0xe0, 0xa0 }))), ""sv);
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0x80 }))), "\xe0\xa0\x80"sv);
@ -170,8 +166,7 @@ TEST_CASE(test_streaming_decoder_utf8_valid_second_byte_tail)
TEST_CASE(test_streaming_decoder_utf16_odd_byte)
{
auto& decoder = decoder_for("UTF-16LE"sv);
auto streaming_decoder = TextCodec::StreamingDecoder { decoder };
auto streaming_decoder = TextCodec::StreamingDecoder { "UTF-16LE"sv };
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0x41, 0x00, 0x42 }))), "A"sv);
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0x00 }))), "B"sv);
@ -180,8 +175,7 @@ TEST_CASE(test_streaming_decoder_utf16_odd_byte)
TEST_CASE(test_streaming_decoder_utf16_surrogate_pair_split)
{
auto& decoder = decoder_for("UTF-16BE"sv);
auto streaming_decoder = TextCodec::StreamingDecoder { decoder };
auto streaming_decoder = TextCodec::StreamingDecoder { "UTF-16BE"sv };
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0x00, 0x41, 0xd8, 0x3d }))), "A"sv);
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0xde, 0x00 }))), "😀"sv);
@ -191,7 +185,7 @@ TEST_CASE(test_streaming_decoder_utf16_surrogate_pair_split)
TEST_CASE(test_streaming_decoder_gb18030_four_byte_tail)
{
auto& decoder = decoder_for("gb18030"sv);
auto streaming_decoder = TextCodec::StreamingDecoder { decoder };
auto streaming_decoder = TextCodec::StreamingDecoder { "gb18030"sv };
auto gb18030_sequence = Vector<u8> { 0x81, 0x30, 0x81, 0x30 };
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 'a', 0x81, 0x30, 0x81 }))), "a"sv);
@ -202,13 +196,13 @@ TEST_CASE(test_streaming_decoder_gb18030_four_byte_tail)
TEST_CASE(test_streaming_decoder_big5_overlapping_trail)
{
auto& decoder = decoder_for("Big5"sv);
auto streaming_decoder = TextCodec::StreamingDecoder { decoder };
auto streaming_decoder = TextCodec::StreamingDecoder { "Big5"sv };
auto big5_sequence = Vector<u8> { 0xa4, 0xa4 };
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes(big5_sequence))), MUST(decoder.to_utf8(StringView(bytes(big5_sequence)))));
EXPECT_EQ(MUST(streaming_decoder.finish()), ""sv);
auto streaming_decoder_for_split_input = TextCodec::StreamingDecoder { decoder };
auto streaming_decoder_for_split_input = TextCodec::StreamingDecoder { "Big5"sv };
EXPECT_EQ(MUST(streaming_decoder_for_split_input.to_utf8(bytes({ 0xa4 }))), ""sv);
EXPECT_EQ(MUST(streaming_decoder_for_split_input.to_utf8(bytes({ 0xa4 }))), MUST(decoder.to_utf8(StringView(bytes(big5_sequence)))));
EXPECT_EQ(MUST(streaming_decoder_for_split_input.finish()), ""sv);
@ -217,7 +211,7 @@ TEST_CASE(test_streaming_decoder_big5_overlapping_trail)
TEST_CASE(test_streaming_decoder_euc_jp_three_byte_tail)
{
auto& decoder = decoder_for("EUC-JP"sv);
auto streaming_decoder = TextCodec::StreamingDecoder { decoder };
auto streaming_decoder = TextCodec::StreamingDecoder { "EUC-JP"sv };
auto euc_jp_sequence = Vector<u8> { 0x8f, 0xa2, 0xaf };
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0x8f, 0xa2 }))), ""sv);
@ -228,7 +222,7 @@ TEST_CASE(test_streaming_decoder_euc_jp_three_byte_tail)
TEST_CASE(test_streaming_decoder_shift_jis_tail)
{
auto& decoder = decoder_for("Shift_JIS"sv);
auto streaming_decoder = TextCodec::StreamingDecoder { decoder };
auto streaming_decoder = TextCodec::StreamingDecoder { "Shift_JIS"sv };
auto shift_jis_sequence = Vector<u8> { 0x82, 0xa0 };
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0x82 }))), ""sv);