diff --git a/Libraries/LibTextCodec/Decoder.cpp b/Libraries/LibTextCodec/Decoder.cpp index 6441e4ce29..b5d3aa6f8d 100644 --- a/Libraries/LibTextCodec/Decoder.cpp +++ b/Libraries/LibTextCodec/Decoder.cpp @@ -8,7 +8,9 @@ */ #include +#include #include +#include #include #include #include @@ -352,6 +354,341 @@ ErrorOr Decoder::to_utf8(StringView input) return builder.to_string_without_validation(); } +// Tail-length helpers for chunked decoding. Each returns the number of trailing bytes that must +// be buffered until more input arrives, because they form an incomplete trailing sequence per the +// Encoding Standard's decoder handler byte ranges. +// https://encoding.spec.whatwg.org/#interface-textdecoder + +static bool is_utf8_continuation_byte(u8 byte) +{ + return (byte & 0xc0) == 0x80; +} + +static Optional utf8_sequence_length(u8 lead) +{ + if (lead <= 0x7f) + return 1; + if (lead >= 0xc2 && lead <= 0xdf) + return 2; + if (lead >= 0xe0 && lead <= 0xef) + return 3; + if (lead >= 0xf0 && lead <= 0xf4) + return 4; + return {}; +} + +size_t UTF8Decoder::incomplete_tail_length(ReadonlyBytes bytes) const +{ + // The longest UTF-8 sequence is 4 bytes, so the lead of any incomplete trailing sequence is + // at most 3 positions before the end. Scan backward to find the lead and decide. + auto max_back = min(bytes.size(), 4uz); + for (size_t back = 0; back < max_back; ++back) { + auto byte = bytes[bytes.size() - 1 - back]; + if (is_utf8_continuation_byte(byte)) + continue; + + auto seq_len = utf8_sequence_length(byte); + size_t seen = back + 1; + if (!seq_len.has_value() || *seq_len <= seen) + return 0; + return seen; + } + return 0; +} + +static size_t incomplete_utf16_tail_length(ReadonlyBytes bytes, bool big_endian) +{ + if (bytes.size() % 2 != 0) + return 1; + if (bytes.size() < 2) + return 0; + + auto high = bytes[bytes.size() - 2]; + auto low = bytes[bytes.size() - 1]; + u16 code_unit = big_endian + ? (static_cast(high) << 8) | low + : (static_cast(low) << 8) | high; + if (AK::UnicodeUtils::is_utf16_high_surrogate(code_unit)) + return 2; + return 0; +} + +size_t UTF16BEDecoder::incomplete_tail_length(ReadonlyBytes bytes) const +{ + return incomplete_utf16_tail_length(bytes, true); +} + +size_t UTF16LEDecoder::incomplete_tail_length(ReadonlyBytes bytes) const +{ + return incomplete_utf16_tail_length(bytes, false); +} + +static bool is_gb18030_lead_byte(u8 byte) +{ + return byte >= 0x81 && byte <= 0xfe; +} + +static bool is_gb18030_two_byte_trail(u8 byte) +{ + return (byte >= 0x40 && byte <= 0x7e) || (byte >= 0x80 && byte <= 0xfe); +} + +size_t GB18030Decoder::incomplete_tail_length(ReadonlyBytes bytes) const +{ + for (size_t i = 0; i < bytes.size();) { + auto byte = bytes[i]; + if (byte <= 0x80) { + ++i; + continue; + } + + if (!is_gb18030_lead_byte(byte)) { + ++i; + continue; + } + + if (i + 1 >= bytes.size()) + return bytes.size() - i; + + auto second = bytes[i + 1]; + if (is_ascii_digit(second)) { + if (i + 2 >= bytes.size()) + return bytes.size() - i; + + auto third = bytes[i + 2]; + if (is_gb18030_lead_byte(third)) { + if (i + 3 >= bytes.size()) + return bytes.size() - i; + + auto fourth = bytes[i + 3]; + if (is_ascii_digit(fourth)) { + i += 4; + continue; + } + } + + ++i; + continue; + } + + if (is_gb18030_two_byte_trail(second)) { + i += 2; + continue; + } + + ++i; + } + + return 0; +} + +static bool is_big5_lead_byte(u8 byte) +{ + return byte >= 0x81 && byte <= 0xfe; +} + +static bool is_big5_trail_byte(u8 byte) +{ + return (byte >= 0x40 && byte <= 0x7e) || (byte >= 0xa1 && byte <= 0xfe); +} + +size_t Big5Decoder::incomplete_tail_length(ReadonlyBytes bytes) const +{ + for (size_t i = 0; i < bytes.size();) { + auto byte = bytes[i]; + if (byte <= 0x7f) { + ++i; + continue; + } + + if (!is_big5_lead_byte(byte)) { + ++i; + continue; + } + + if (i + 1 >= bytes.size()) + return bytes.size() - i; + + if (is_big5_trail_byte(bytes[i + 1])) + i += 2; + else + ++i; + } + + return 0; +} + +static bool is_euc_jp_lead_byte(u8 byte) +{ + return byte == 0x8e || byte == 0x8f || (byte >= 0xa1 && byte <= 0xfe); +} + +static bool is_euc_jp_trail_byte(u8 byte) +{ + return byte >= 0xa1 && byte <= 0xfe; +} + +size_t EUCJPDecoder::incomplete_tail_length(ReadonlyBytes bytes) const +{ + for (size_t i = 0; i < bytes.size();) { + auto byte = bytes[i]; + if (byte <= 0x7f) { + ++i; + continue; + } + + if (!is_euc_jp_lead_byte(byte)) { + ++i; + continue; + } + + if (byte == 0x8f) { + if (i + 1 >= bytes.size()) + return bytes.size() - i; + if (!is_euc_jp_trail_byte(bytes[i + 1])) { + ++i; + continue; + } + if (i + 2 >= bytes.size()) + return bytes.size() - i; + if (is_euc_jp_trail_byte(bytes[i + 2])) + i += 3; + else + i += 2; + continue; + } + + if (i + 1 >= bytes.size()) + return bytes.size() - i; + + if (byte == 0x8e) { + if (bytes[i + 1] >= 0xa1 && bytes[i + 1] <= 0xdf) + i += 2; + else + ++i; + continue; + } + + if (is_euc_jp_trail_byte(bytes[i + 1])) + i += 2; + else + ++i; + } + + return 0; +} + +static bool is_shift_jis_lead_byte(u8 byte) +{ + return (byte >= 0x81 && byte <= 0x9f) || (byte >= 0xe0 && byte <= 0xfc); +} + +static bool is_shift_jis_trail_byte(u8 byte) +{ + return (byte >= 0x40 && byte <= 0x7e) || (byte >= 0x80 && byte <= 0xfc); +} + +size_t ShiftJISDecoder::incomplete_tail_length(ReadonlyBytes bytes) const +{ + for (size_t i = 0; i < bytes.size();) { + auto byte = bytes[i]; + if (byte <= 0x80 || (byte >= 0xa1 && byte <= 0xdf)) { + ++i; + continue; + } + + if (!is_shift_jis_lead_byte(byte)) { + ++i; + continue; + } + + if (i + 1 >= bytes.size()) + return bytes.size() - i; + + if (is_shift_jis_trail_byte(bytes[i + 1])) + i += 2; + else + ++i; + } + + return 0; +} + +static bool is_euc_kr_lead_byte(u8 byte) +{ + return byte >= 0x81 && byte <= 0xfe; +} + +static bool is_euc_kr_trail_byte(u8 byte) +{ + return byte >= 0x41 && byte <= 0xfe; +} + +size_t EUCKRDecoder::incomplete_tail_length(ReadonlyBytes bytes) const +{ + for (size_t i = 0; i < bytes.size();) { + auto byte = bytes[i]; + if (byte <= 0x7f) { + ++i; + continue; + } + + if (!is_euc_kr_lead_byte(byte)) { + ++i; + continue; + } + + if (i + 1 >= bytes.size()) + return bytes.size() - i; + + if (is_euc_kr_trail_byte(bytes[i + 1])) + i += 2; + else + ++i; + } + + return 0; +} + +size_t ISO2022JPDecoder::incomplete_tail_length(ReadonlyBytes bytes) const +{ + if (bytes.is_empty()) + return 0; + + if (bytes[bytes.size() - 1] == 0x1b) + return 1; + if (bytes.size() >= 2 && bytes[bytes.size() - 2] == 0x1b && (bytes[bytes.size() - 1] == 0x24 || bytes[bytes.size() - 1] == 0x28)) + return 2; + return 0; +} + +ErrorOr StreamingDecoder::to_utf8(ReadonlyBytes input) +{ + ReadonlyBytes bytes; + if (m_pending_input.is_empty()) { + bytes = input; + } else { + TRY(m_pending_input.try_append(input)); + bytes = m_pending_input.bytes(); + } + + auto tail_length = m_decoder.incomplete_tail_length(bytes); + auto decoded = TRY(m_decoder.to_utf8(StringView(bytes.slice(0, bytes.size() - tail_length)))); + + if (tail_length == 0) + m_pending_input.clear(); + else + m_pending_input = TRY(ByteBuffer::copy(bytes.slice(bytes.size() - tail_length, tail_length))); + return decoded; +} + +ErrorOr StreamingDecoder::finish() +{ + auto decoded = TRY(m_decoder.to_utf8(StringView(m_pending_input.bytes()))); + m_pending_input.clear(); + return decoded; +} + ErrorOr UTF8Decoder::process(StringView input, Function(u32)> on_code_point) { for (auto c : Utf8View(input)) { diff --git a/Libraries/LibTextCodec/Decoder.h b/Libraries/LibTextCodec/Decoder.h index a7a52a396b..839579dfbb 100644 --- a/Libraries/LibTextCodec/Decoder.h +++ b/Libraries/LibTextCodec/Decoder.h @@ -8,6 +8,7 @@ #pragma once +#include #include #include #include @@ -22,6 +23,10 @@ public: virtual bool validate(StringView); virtual ErrorOr to_utf8(StringView); + // Returns the number of trailing bytes that form an incomplete sequence and must be buffered + // until more input arrives. Used by StreamingDecoder for chunked decoding. + virtual size_t incomplete_tail_length(ReadonlyBytes) const { return 0; } + protected: virtual ~Decoder() = default; virtual ErrorOr process(StringView, Function(u32)> on_code_point) = 0; @@ -32,12 +37,14 @@ public: virtual ErrorOr process(StringView, Function(u32)> on_code_point) override; virtual bool validate(StringView) override; virtual ErrorOr to_utf8(StringView) override; + virtual size_t incomplete_tail_length(ReadonlyBytes) const override; }; class TEXTCODEC_API UTF16BEDecoder final : public Decoder { public: virtual bool validate(StringView) override; virtual ErrorOr to_utf8(StringView) override; + virtual size_t incomplete_tail_length(ReadonlyBytes) const override; private: virtual ErrorOr process(StringView, Function(u32)>) override { VERIFY_NOT_REACHED(); } @@ -47,6 +54,7 @@ class TEXTCODEC_API UTF16LEDecoder final : public Decoder { public: virtual bool validate(StringView) override; virtual ErrorOr to_utf8(StringView) override; + virtual size_t incomplete_tail_length(ReadonlyBytes) const override; private: virtual ErrorOr process(StringView, Function(u32)>) override { VERIFY_NOT_REACHED(); } @@ -87,31 +95,37 @@ public: class TEXTCODEC_API GB18030Decoder final : public Decoder { public: virtual ErrorOr process(StringView, Function(u32)> on_code_point) override; + virtual size_t incomplete_tail_length(ReadonlyBytes) const override; }; class TEXTCODEC_API Big5Decoder final : public Decoder { public: virtual ErrorOr process(StringView, Function(u32)> on_code_point) override; + virtual size_t incomplete_tail_length(ReadonlyBytes) const override; }; class TEXTCODEC_API EUCJPDecoder final : public Decoder { public: virtual ErrorOr process(StringView, Function(u32)> on_code_point) override; + virtual size_t incomplete_tail_length(ReadonlyBytes) const override; }; class TEXTCODEC_API ISO2022JPDecoder final : public Decoder { public: virtual ErrorOr process(StringView, Function(u32)> on_code_point) override; + virtual size_t incomplete_tail_length(ReadonlyBytes) const override; }; class TEXTCODEC_API ShiftJISDecoder final : public Decoder { public: virtual ErrorOr process(StringView, Function(u32)> on_code_point) override; + virtual size_t incomplete_tail_length(ReadonlyBytes) const override; }; class TEXTCODEC_API EUCKRDecoder final : public Decoder { public: virtual ErrorOr process(StringView, Function(u32)> on_code_point) override; + virtual size_t incomplete_tail_length(ReadonlyBytes) const override; }; class TEXTCODEC_API ReplacementDecoder final : public Decoder { @@ -120,6 +134,22 @@ public: virtual bool validate(StringView input) override { return input.is_empty(); } }; +// Preserves incomplete trailing decoder tokens when callers provide input in chunks. +class TEXTCODEC_API StreamingDecoder final { +public: + explicit StreamingDecoder(Decoder& decoder) + : m_decoder(decoder) + { + } + + ErrorOr to_utf8(ReadonlyBytes); + ErrorOr finish(); + +private: + Decoder& m_decoder; + ByteBuffer m_pending_input; +}; + // This will return a decoder for the exact name specified, skipping get_standardized_encoding. // Use this when you want ISO-8859-1 instead of windows-1252. TEXTCODEC_API Optional decoder_for_exact_name(StringView encoding); diff --git a/Libraries/LibTextCodec/Forward.h b/Libraries/LibTextCodec/Forward.h index 68841c04b9..b6b55c13f2 100644 --- a/Libraries/LibTextCodec/Forward.h +++ b/Libraries/LibTextCodec/Forward.h @@ -10,5 +10,6 @@ namespace TextCodec { class Decoder; class Encoder; +class StreamingDecoder; } diff --git a/Tests/LibTextCodec/TestTextDecoders.cpp b/Tests/LibTextCodec/TestTextDecoders.cpp index f19ce8ba25..011266e2aa 100644 --- a/Tests/LibTextCodec/TestTextDecoders.cpp +++ b/Tests/LibTextCodec/TestTextDecoders.cpp @@ -9,6 +9,18 @@ #include #include +static ReadonlyBytes bytes(Vector const& data) +{ + return data.span(); +} + +static TextCodec::Decoder& decoder_for(StringView encoding) +{ + auto decoder = TextCodec::decoder_for(encoding); + VERIFY(decoder.has_value()); + return decoder.value(); +} + TEST_CASE(test_utf8_decode) { auto decoder = TextCodec::UTF8Decoder(); @@ -38,6 +50,94 @@ TEST_CASE(test_utf16be_decode) EXPECT_EQ(utf8, "säk😀"sv); } +TEST_CASE(test_streaming_decoder_utf8_mid_sequence) +{ + auto& decoder = decoder_for("UTF-8"sv); + auto streaming_decoder = TextCodec::StreamingDecoder { decoder }; + + EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 'a', 0xc3 }))), "a"sv); + EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0xa9, 'b' }))), "éb"sv); + EXPECT_EQ(MUST(streaming_decoder.finish()), ""sv); +} + +TEST_CASE(test_streaming_decoder_finishes_incomplete_sequence) +{ + auto& decoder = decoder_for("UTF-8"sv); + auto streaming_decoder = TextCodec::StreamingDecoder { decoder }; + + auto incomplete_sequence = Vector { 0xc3 }; + EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes(incomplete_sequence))), ""sv); + EXPECT_EQ(MUST(streaming_decoder.finish()), MUST(decoder.to_utf8(StringView(bytes(incomplete_sequence))))); +} + +TEST_CASE(test_streaming_decoder_utf16_odd_byte) +{ + auto& decoder = decoder_for("UTF-16LE"sv); + auto streaming_decoder = TextCodec::StreamingDecoder { decoder }; + + EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0x41, 0x00, 0x42 }))), "A"sv); + EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0x00 }))), "B"sv); + EXPECT_EQ(MUST(streaming_decoder.finish()), ""sv); +} + +TEST_CASE(test_streaming_decoder_utf16_surrogate_pair_split) +{ + auto& decoder = decoder_for("UTF-16BE"sv); + auto streaming_decoder = TextCodec::StreamingDecoder { decoder }; + + EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0x00, 0x41, 0xd8, 0x3d }))), "A"sv); + EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0xde, 0x00 }))), "😀"sv); + EXPECT_EQ(MUST(streaming_decoder.finish()), ""sv); +} + +TEST_CASE(test_streaming_decoder_gb18030_four_byte_tail) +{ + auto& decoder = decoder_for("gb18030"sv); + auto streaming_decoder = TextCodec::StreamingDecoder { decoder }; + auto gb18030_sequence = Vector { 0x81, 0x30, 0x81, 0x30 }; + + EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 'a', 0x81, 0x30, 0x81 }))), "a"sv); + EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0x30 }))), MUST(decoder.to_utf8(StringView(bytes(gb18030_sequence))))); + EXPECT_EQ(MUST(streaming_decoder.finish()), ""sv); +} + +TEST_CASE(test_streaming_decoder_big5_overlapping_trail) +{ + auto& decoder = decoder_for("Big5"sv); + auto streaming_decoder = TextCodec::StreamingDecoder { decoder }; + auto big5_sequence = Vector { 0xa4, 0xa4 }; + + EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes(big5_sequence))), MUST(decoder.to_utf8(StringView(bytes(big5_sequence))))); + EXPECT_EQ(MUST(streaming_decoder.finish()), ""sv); + + auto streaming_decoder_for_split_input = TextCodec::StreamingDecoder { decoder }; + EXPECT_EQ(MUST(streaming_decoder_for_split_input.to_utf8(bytes({ 0xa4 }))), ""sv); + EXPECT_EQ(MUST(streaming_decoder_for_split_input.to_utf8(bytes({ 0xa4 }))), MUST(decoder.to_utf8(StringView(bytes(big5_sequence))))); + EXPECT_EQ(MUST(streaming_decoder_for_split_input.finish()), ""sv); +} + +TEST_CASE(test_streaming_decoder_euc_jp_three_byte_tail) +{ + auto& decoder = decoder_for("EUC-JP"sv); + auto streaming_decoder = TextCodec::StreamingDecoder { decoder }; + auto euc_jp_sequence = Vector { 0x8f, 0xa2, 0xaf }; + + EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0x8f, 0xa2 }))), ""sv); + EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0xaf }))), MUST(decoder.to_utf8(StringView(bytes(euc_jp_sequence))))); + EXPECT_EQ(MUST(streaming_decoder.finish()), ""sv); +} + +TEST_CASE(test_streaming_decoder_shift_jis_tail) +{ + auto& decoder = decoder_for("Shift_JIS"sv); + auto streaming_decoder = TextCodec::StreamingDecoder { decoder }; + auto shift_jis_sequence = Vector { 0x82, 0xa0 }; + + EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0x82 }))), ""sv); + EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0xa0 }))), MUST(decoder.to_utf8(StringView(bytes(shift_jis_sequence))))); + EXPECT_EQ(MUST(streaming_decoder.finish()), ""sv); +} + TEST_CASE(test_utf16le_decode) { auto decoder = TextCodec::UTF16LEDecoder();