LibTextCodec: Add streaming decoder
Introduce a StreamingDecoder wrapper that lets callers feed bytes to a Decoder one chunk at a time. It buffers any incomplete trailing byte sequence at the end of a chunk and prepends it to the next chunk, so a multi-byte code point split across a chunk boundary is decoded correctly once the next chunk arrives. To support that, add an incomplete_tail_length() virtual on Decoder returning the number of trailing bytes that form an incomplete sequence per the Encoding Standard's decoder handler byte ranges, with overrides for UTF-8, UTF-16BE, UTF-16LE, GB18030, Big5, EUC-JP, ISO-2022-JP, Shift_JIS, and EUC-KR. The default implementation returns 0, which keeps single-byte legacy decoders correct. This is the foundation for the upcoming incremental HTML parser, which needs to decode network response bodies as they arrive.
This commit is contained in:
Aliaksandr Kalenik
Andreas Kling
parent
c61066c0ae
commit
9375499e52
4 changed files with 468 additions and 0 deletions
|
|
@ -8,7 +8,9 @@
|
|||
*/
|
||||
|
||||
#include <AK/BinarySearch.h>
|
||||
#include <AK/CharacterTypes.h>
|
||||
#include <AK/StringBuilder.h>
|
||||
#include <AK/UnicodeUtils.h>
|
||||
#include <AK/Utf16View.h>
|
||||
#include <AK/Utf8View.h>
|
||||
#include <LibTextCodec/Decoder.h>
|
||||
|
|
@ -352,6 +354,341 @@ ErrorOr<String> Decoder::to_utf8(StringView input)
|
|||
return builder.to_string_without_validation();
|
||||
}
|
||||
|
||||
// Tail-length helpers for chunked decoding. Each returns the number of trailing bytes that must
|
||||
// be buffered until more input arrives, because they form an incomplete trailing sequence per the
|
||||
// Encoding Standard's decoder handler byte ranges.
|
||||
// https://encoding.spec.whatwg.org/#interface-textdecoder
|
||||
|
||||
static bool is_utf8_continuation_byte(u8 byte)
|
||||
{
|
||||
return (byte & 0xc0) == 0x80;
|
||||
}
|
||||
|
||||
static Optional<size_t> utf8_sequence_length(u8 lead)
|
||||
{
|
||||
if (lead <= 0x7f)
|
||||
return 1;
|
||||
if (lead >= 0xc2 && lead <= 0xdf)
|
||||
return 2;
|
||||
if (lead >= 0xe0 && lead <= 0xef)
|
||||
return 3;
|
||||
if (lead >= 0xf0 && lead <= 0xf4)
|
||||
return 4;
|
||||
return {};
|
||||
}
|
||||
|
||||
size_t UTF8Decoder::incomplete_tail_length(ReadonlyBytes bytes) const
|
||||
{
|
||||
// The longest UTF-8 sequence is 4 bytes, so the lead of any incomplete trailing sequence is
|
||||
// at most 3 positions before the end. Scan backward to find the lead and decide.
|
||||
auto max_back = min(bytes.size(), 4uz);
|
||||
for (size_t back = 0; back < max_back; ++back) {
|
||||
auto byte = bytes[bytes.size() - 1 - back];
|
||||
if (is_utf8_continuation_byte(byte))
|
||||
continue;
|
||||
|
||||
auto seq_len = utf8_sequence_length(byte);
|
||||
size_t seen = back + 1;
|
||||
if (!seq_len.has_value() || *seq_len <= seen)
|
||||
return 0;
|
||||
return seen;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static size_t incomplete_utf16_tail_length(ReadonlyBytes bytes, bool big_endian)
|
||||
{
|
||||
if (bytes.size() % 2 != 0)
|
||||
return 1;
|
||||
if (bytes.size() < 2)
|
||||
return 0;
|
||||
|
||||
auto high = bytes[bytes.size() - 2];
|
||||
auto low = bytes[bytes.size() - 1];
|
||||
u16 code_unit = big_endian
|
||||
? (static_cast<u16>(high) << 8) | low
|
||||
: (static_cast<u16>(low) << 8) | high;
|
||||
if (AK::UnicodeUtils::is_utf16_high_surrogate(code_unit))
|
||||
return 2;
|
||||
return 0;
|
||||
}
|
||||
|
||||
size_t UTF16BEDecoder::incomplete_tail_length(ReadonlyBytes bytes) const
|
||||
{
|
||||
return incomplete_utf16_tail_length(bytes, true);
|
||||
}
|
||||
|
||||
size_t UTF16LEDecoder::incomplete_tail_length(ReadonlyBytes bytes) const
|
||||
{
|
||||
return incomplete_utf16_tail_length(bytes, false);
|
||||
}
|
||||
|
||||
static bool is_gb18030_lead_byte(u8 byte)
|
||||
{
|
||||
return byte >= 0x81 && byte <= 0xfe;
|
||||
}
|
||||
|
||||
static bool is_gb18030_two_byte_trail(u8 byte)
|
||||
{
|
||||
return (byte >= 0x40 && byte <= 0x7e) || (byte >= 0x80 && byte <= 0xfe);
|
||||
}
|
||||
|
||||
size_t GB18030Decoder::incomplete_tail_length(ReadonlyBytes bytes) const
|
||||
{
|
||||
for (size_t i = 0; i < bytes.size();) {
|
||||
auto byte = bytes[i];
|
||||
if (byte <= 0x80) {
|
||||
++i;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!is_gb18030_lead_byte(byte)) {
|
||||
++i;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (i + 1 >= bytes.size())
|
||||
return bytes.size() - i;
|
||||
|
||||
auto second = bytes[i + 1];
|
||||
if (is_ascii_digit(second)) {
|
||||
if (i + 2 >= bytes.size())
|
||||
return bytes.size() - i;
|
||||
|
||||
auto third = bytes[i + 2];
|
||||
if (is_gb18030_lead_byte(third)) {
|
||||
if (i + 3 >= bytes.size())
|
||||
return bytes.size() - i;
|
||||
|
||||
auto fourth = bytes[i + 3];
|
||||
if (is_ascii_digit(fourth)) {
|
||||
i += 4;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
++i;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (is_gb18030_two_byte_trail(second)) {
|
||||
i += 2;
|
||||
continue;
|
||||
}
|
||||
|
||||
++i;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static bool is_big5_lead_byte(u8 byte)
|
||||
{
|
||||
return byte >= 0x81 && byte <= 0xfe;
|
||||
}
|
||||
|
||||
static bool is_big5_trail_byte(u8 byte)
|
||||
{
|
||||
return (byte >= 0x40 && byte <= 0x7e) || (byte >= 0xa1 && byte <= 0xfe);
|
||||
}
|
||||
|
||||
size_t Big5Decoder::incomplete_tail_length(ReadonlyBytes bytes) const
|
||||
{
|
||||
for (size_t i = 0; i < bytes.size();) {
|
||||
auto byte = bytes[i];
|
||||
if (byte <= 0x7f) {
|
||||
++i;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!is_big5_lead_byte(byte)) {
|
||||
++i;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (i + 1 >= bytes.size())
|
||||
return bytes.size() - i;
|
||||
|
||||
if (is_big5_trail_byte(bytes[i + 1]))
|
||||
i += 2;
|
||||
else
|
||||
++i;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static bool is_euc_jp_lead_byte(u8 byte)
|
||||
{
|
||||
return byte == 0x8e || byte == 0x8f || (byte >= 0xa1 && byte <= 0xfe);
|
||||
}
|
||||
|
||||
static bool is_euc_jp_trail_byte(u8 byte)
|
||||
{
|
||||
return byte >= 0xa1 && byte <= 0xfe;
|
||||
}
|
||||
|
||||
size_t EUCJPDecoder::incomplete_tail_length(ReadonlyBytes bytes) const
|
||||
{
|
||||
for (size_t i = 0; i < bytes.size();) {
|
||||
auto byte = bytes[i];
|
||||
if (byte <= 0x7f) {
|
||||
++i;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!is_euc_jp_lead_byte(byte)) {
|
||||
++i;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (byte == 0x8f) {
|
||||
if (i + 1 >= bytes.size())
|
||||
return bytes.size() - i;
|
||||
if (!is_euc_jp_trail_byte(bytes[i + 1])) {
|
||||
++i;
|
||||
continue;
|
||||
}
|
||||
if (i + 2 >= bytes.size())
|
||||
return bytes.size() - i;
|
||||
if (is_euc_jp_trail_byte(bytes[i + 2]))
|
||||
i += 3;
|
||||
else
|
||||
i += 2;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (i + 1 >= bytes.size())
|
||||
return bytes.size() - i;
|
||||
|
||||
if (byte == 0x8e) {
|
||||
if (bytes[i + 1] >= 0xa1 && bytes[i + 1] <= 0xdf)
|
||||
i += 2;
|
||||
else
|
||||
++i;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (is_euc_jp_trail_byte(bytes[i + 1]))
|
||||
i += 2;
|
||||
else
|
||||
++i;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static bool is_shift_jis_lead_byte(u8 byte)
|
||||
{
|
||||
return (byte >= 0x81 && byte <= 0x9f) || (byte >= 0xe0 && byte <= 0xfc);
|
||||
}
|
||||
|
||||
static bool is_shift_jis_trail_byte(u8 byte)
|
||||
{
|
||||
return (byte >= 0x40 && byte <= 0x7e) || (byte >= 0x80 && byte <= 0xfc);
|
||||
}
|
||||
|
||||
size_t ShiftJISDecoder::incomplete_tail_length(ReadonlyBytes bytes) const
|
||||
{
|
||||
for (size_t i = 0; i < bytes.size();) {
|
||||
auto byte = bytes[i];
|
||||
if (byte <= 0x80 || (byte >= 0xa1 && byte <= 0xdf)) {
|
||||
++i;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!is_shift_jis_lead_byte(byte)) {
|
||||
++i;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (i + 1 >= bytes.size())
|
||||
return bytes.size() - i;
|
||||
|
||||
if (is_shift_jis_trail_byte(bytes[i + 1]))
|
||||
i += 2;
|
||||
else
|
||||
++i;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static bool is_euc_kr_lead_byte(u8 byte)
|
||||
{
|
||||
return byte >= 0x81 && byte <= 0xfe;
|
||||
}
|
||||
|
||||
static bool is_euc_kr_trail_byte(u8 byte)
|
||||
{
|
||||
return byte >= 0x41 && byte <= 0xfe;
|
||||
}
|
||||
|
||||
size_t EUCKRDecoder::incomplete_tail_length(ReadonlyBytes bytes) const
|
||||
{
|
||||
for (size_t i = 0; i < bytes.size();) {
|
||||
auto byte = bytes[i];
|
||||
if (byte <= 0x7f) {
|
||||
++i;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!is_euc_kr_lead_byte(byte)) {
|
||||
++i;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (i + 1 >= bytes.size())
|
||||
return bytes.size() - i;
|
||||
|
||||
if (is_euc_kr_trail_byte(bytes[i + 1]))
|
||||
i += 2;
|
||||
else
|
||||
++i;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
size_t ISO2022JPDecoder::incomplete_tail_length(ReadonlyBytes bytes) const
|
||||
{
|
||||
if (bytes.is_empty())
|
||||
return 0;
|
||||
|
||||
if (bytes[bytes.size() - 1] == 0x1b)
|
||||
return 1;
|
||||
if (bytes.size() >= 2 && bytes[bytes.size() - 2] == 0x1b && (bytes[bytes.size() - 1] == 0x24 || bytes[bytes.size() - 1] == 0x28))
|
||||
return 2;
|
||||
return 0;
|
||||
}
|
||||
|
||||
ErrorOr<String> StreamingDecoder::to_utf8(ReadonlyBytes input)
|
||||
{
|
||||
ReadonlyBytes bytes;
|
||||
if (m_pending_input.is_empty()) {
|
||||
bytes = input;
|
||||
} else {
|
||||
TRY(m_pending_input.try_append(input));
|
||||
bytes = m_pending_input.bytes();
|
||||
}
|
||||
|
||||
auto tail_length = m_decoder.incomplete_tail_length(bytes);
|
||||
auto decoded = TRY(m_decoder.to_utf8(StringView(bytes.slice(0, bytes.size() - tail_length))));
|
||||
|
||||
if (tail_length == 0)
|
||||
m_pending_input.clear();
|
||||
else
|
||||
m_pending_input = TRY(ByteBuffer::copy(bytes.slice(bytes.size() - tail_length, tail_length)));
|
||||
return decoded;
|
||||
}
|
||||
|
||||
ErrorOr<String> StreamingDecoder::finish()
|
||||
{
|
||||
auto decoded = TRY(m_decoder.to_utf8(StringView(m_pending_input.bytes())));
|
||||
m_pending_input.clear();
|
||||
return decoded;
|
||||
}
|
||||
|
||||
ErrorOr<void> UTF8Decoder::process(StringView input, Function<ErrorOr<void>(u32)> on_code_point)
|
||||
{
|
||||
for (auto c : Utf8View(input)) {
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@
|
|||
|
||||
#pragma once
|
||||
|
||||
#include <AK/ByteBuffer.h>
|
||||
#include <AK/Forward.h>
|
||||
#include <AK/Function.h>
|
||||
#include <AK/Optional.h>
|
||||
|
|
@ -22,6 +23,10 @@ public:
|
|||
virtual bool validate(StringView);
|
||||
virtual ErrorOr<String> to_utf8(StringView);
|
||||
|
||||
// Returns the number of trailing bytes that form an incomplete sequence and must be buffered
|
||||
// until more input arrives. Used by StreamingDecoder for chunked decoding.
|
||||
virtual size_t incomplete_tail_length(ReadonlyBytes) const { return 0; }
|
||||
|
||||
protected:
|
||||
virtual ~Decoder() = default;
|
||||
virtual ErrorOr<void> process(StringView, Function<ErrorOr<void>(u32)> on_code_point) = 0;
|
||||
|
|
@ -32,12 +37,14 @@ public:
|
|||
virtual ErrorOr<void> process(StringView, Function<ErrorOr<void>(u32)> on_code_point) override;
|
||||
virtual bool validate(StringView) override;
|
||||
virtual ErrorOr<String> to_utf8(StringView) override;
|
||||
virtual size_t incomplete_tail_length(ReadonlyBytes) const override;
|
||||
};
|
||||
|
||||
class TEXTCODEC_API UTF16BEDecoder final : public Decoder {
|
||||
public:
|
||||
virtual bool validate(StringView) override;
|
||||
virtual ErrorOr<String> to_utf8(StringView) override;
|
||||
virtual size_t incomplete_tail_length(ReadonlyBytes) const override;
|
||||
|
||||
private:
|
||||
virtual ErrorOr<void> process(StringView, Function<ErrorOr<void>(u32)>) override { VERIFY_NOT_REACHED(); }
|
||||
|
|
@ -47,6 +54,7 @@ class TEXTCODEC_API UTF16LEDecoder final : public Decoder {
|
|||
public:
|
||||
virtual bool validate(StringView) override;
|
||||
virtual ErrorOr<String> to_utf8(StringView) override;
|
||||
virtual size_t incomplete_tail_length(ReadonlyBytes) const override;
|
||||
|
||||
private:
|
||||
virtual ErrorOr<void> process(StringView, Function<ErrorOr<void>(u32)>) override { VERIFY_NOT_REACHED(); }
|
||||
|
|
@ -87,31 +95,37 @@ public:
|
|||
class TEXTCODEC_API GB18030Decoder final : public Decoder {
|
||||
public:
|
||||
virtual ErrorOr<void> process(StringView, Function<ErrorOr<void>(u32)> on_code_point) override;
|
||||
virtual size_t incomplete_tail_length(ReadonlyBytes) const override;
|
||||
};
|
||||
|
||||
class TEXTCODEC_API Big5Decoder final : public Decoder {
|
||||
public:
|
||||
virtual ErrorOr<void> process(StringView, Function<ErrorOr<void>(u32)> on_code_point) override;
|
||||
virtual size_t incomplete_tail_length(ReadonlyBytes) const override;
|
||||
};
|
||||
|
||||
class TEXTCODEC_API EUCJPDecoder final : public Decoder {
|
||||
public:
|
||||
virtual ErrorOr<void> process(StringView, Function<ErrorOr<void>(u32)> on_code_point) override;
|
||||
virtual size_t incomplete_tail_length(ReadonlyBytes) const override;
|
||||
};
|
||||
|
||||
class TEXTCODEC_API ISO2022JPDecoder final : public Decoder {
|
||||
public:
|
||||
virtual ErrorOr<void> process(StringView, Function<ErrorOr<void>(u32)> on_code_point) override;
|
||||
virtual size_t incomplete_tail_length(ReadonlyBytes) const override;
|
||||
};
|
||||
|
||||
class TEXTCODEC_API ShiftJISDecoder final : public Decoder {
|
||||
public:
|
||||
virtual ErrorOr<void> process(StringView, Function<ErrorOr<void>(u32)> on_code_point) override;
|
||||
virtual size_t incomplete_tail_length(ReadonlyBytes) const override;
|
||||
};
|
||||
|
||||
class TEXTCODEC_API EUCKRDecoder final : public Decoder {
|
||||
public:
|
||||
virtual ErrorOr<void> process(StringView, Function<ErrorOr<void>(u32)> on_code_point) override;
|
||||
virtual size_t incomplete_tail_length(ReadonlyBytes) const override;
|
||||
};
|
||||
|
||||
class TEXTCODEC_API ReplacementDecoder final : public Decoder {
|
||||
|
|
@ -120,6 +134,22 @@ public:
|
|||
virtual bool validate(StringView input) override { return input.is_empty(); }
|
||||
};
|
||||
|
||||
// Preserves incomplete trailing decoder tokens when callers provide input in chunks.
|
||||
class TEXTCODEC_API StreamingDecoder final {
|
||||
public:
|
||||
explicit StreamingDecoder(Decoder& decoder)
|
||||
: m_decoder(decoder)
|
||||
{
|
||||
}
|
||||
|
||||
ErrorOr<String> to_utf8(ReadonlyBytes);
|
||||
ErrorOr<String> finish();
|
||||
|
||||
private:
|
||||
Decoder& m_decoder;
|
||||
ByteBuffer m_pending_input;
|
||||
};
|
||||
|
||||
// This will return a decoder for the exact name specified, skipping get_standardized_encoding.
|
||||
// Use this when you want ISO-8859-1 instead of windows-1252.
|
||||
TEXTCODEC_API Optional<Decoder&> decoder_for_exact_name(StringView encoding);
|
||||
|
|
|
|||
|
|
@ -10,5 +10,6 @@ namespace TextCodec {
|
|||
|
||||
class Decoder;
|
||||
class Encoder;
|
||||
class StreamingDecoder;
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,6 +9,18 @@
|
|||
#include <LibTest/TestCase.h>
|
||||
#include <LibTextCodec/Decoder.h>
|
||||
|
||||
static ReadonlyBytes bytes(Vector<u8> const& data)
|
||||
{
|
||||
return data.span();
|
||||
}
|
||||
|
||||
static TextCodec::Decoder& decoder_for(StringView encoding)
|
||||
{
|
||||
auto decoder = TextCodec::decoder_for(encoding);
|
||||
VERIFY(decoder.has_value());
|
||||
return decoder.value();
|
||||
}
|
||||
|
||||
TEST_CASE(test_utf8_decode)
|
||||
{
|
||||
auto decoder = TextCodec::UTF8Decoder();
|
||||
|
|
@ -38,6 +50,94 @@ TEST_CASE(test_utf16be_decode)
|
|||
EXPECT_EQ(utf8, "säk😀"sv);
|
||||
}
|
||||
|
||||
TEST_CASE(test_streaming_decoder_utf8_mid_sequence)
|
||||
{
|
||||
auto& decoder = decoder_for("UTF-8"sv);
|
||||
auto streaming_decoder = TextCodec::StreamingDecoder { decoder };
|
||||
|
||||
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 'a', 0xc3 }))), "a"sv);
|
||||
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0xa9, 'b' }))), "éb"sv);
|
||||
EXPECT_EQ(MUST(streaming_decoder.finish()), ""sv);
|
||||
}
|
||||
|
||||
TEST_CASE(test_streaming_decoder_finishes_incomplete_sequence)
|
||||
{
|
||||
auto& decoder = decoder_for("UTF-8"sv);
|
||||
auto streaming_decoder = TextCodec::StreamingDecoder { decoder };
|
||||
|
||||
auto incomplete_sequence = Vector<u8> { 0xc3 };
|
||||
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes(incomplete_sequence))), ""sv);
|
||||
EXPECT_EQ(MUST(streaming_decoder.finish()), MUST(decoder.to_utf8(StringView(bytes(incomplete_sequence)))));
|
||||
}
|
||||
|
||||
TEST_CASE(test_streaming_decoder_utf16_odd_byte)
|
||||
{
|
||||
auto& decoder = decoder_for("UTF-16LE"sv);
|
||||
auto streaming_decoder = TextCodec::StreamingDecoder { decoder };
|
||||
|
||||
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0x41, 0x00, 0x42 }))), "A"sv);
|
||||
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0x00 }))), "B"sv);
|
||||
EXPECT_EQ(MUST(streaming_decoder.finish()), ""sv);
|
||||
}
|
||||
|
||||
TEST_CASE(test_streaming_decoder_utf16_surrogate_pair_split)
|
||||
{
|
||||
auto& decoder = decoder_for("UTF-16BE"sv);
|
||||
auto streaming_decoder = TextCodec::StreamingDecoder { decoder };
|
||||
|
||||
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0x00, 0x41, 0xd8, 0x3d }))), "A"sv);
|
||||
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0xde, 0x00 }))), "😀"sv);
|
||||
EXPECT_EQ(MUST(streaming_decoder.finish()), ""sv);
|
||||
}
|
||||
|
||||
TEST_CASE(test_streaming_decoder_gb18030_four_byte_tail)
|
||||
{
|
||||
auto& decoder = decoder_for("gb18030"sv);
|
||||
auto streaming_decoder = TextCodec::StreamingDecoder { decoder };
|
||||
auto gb18030_sequence = Vector<u8> { 0x81, 0x30, 0x81, 0x30 };
|
||||
|
||||
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 'a', 0x81, 0x30, 0x81 }))), "a"sv);
|
||||
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0x30 }))), MUST(decoder.to_utf8(StringView(bytes(gb18030_sequence)))));
|
||||
EXPECT_EQ(MUST(streaming_decoder.finish()), ""sv);
|
||||
}
|
||||
|
||||
TEST_CASE(test_streaming_decoder_big5_overlapping_trail)
|
||||
{
|
||||
auto& decoder = decoder_for("Big5"sv);
|
||||
auto streaming_decoder = TextCodec::StreamingDecoder { decoder };
|
||||
auto big5_sequence = Vector<u8> { 0xa4, 0xa4 };
|
||||
|
||||
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes(big5_sequence))), MUST(decoder.to_utf8(StringView(bytes(big5_sequence)))));
|
||||
EXPECT_EQ(MUST(streaming_decoder.finish()), ""sv);
|
||||
|
||||
auto streaming_decoder_for_split_input = TextCodec::StreamingDecoder { decoder };
|
||||
EXPECT_EQ(MUST(streaming_decoder_for_split_input.to_utf8(bytes({ 0xa4 }))), ""sv);
|
||||
EXPECT_EQ(MUST(streaming_decoder_for_split_input.to_utf8(bytes({ 0xa4 }))), MUST(decoder.to_utf8(StringView(bytes(big5_sequence)))));
|
||||
EXPECT_EQ(MUST(streaming_decoder_for_split_input.finish()), ""sv);
|
||||
}
|
||||
|
||||
TEST_CASE(test_streaming_decoder_euc_jp_three_byte_tail)
|
||||
{
|
||||
auto& decoder = decoder_for("EUC-JP"sv);
|
||||
auto streaming_decoder = TextCodec::StreamingDecoder { decoder };
|
||||
auto euc_jp_sequence = Vector<u8> { 0x8f, 0xa2, 0xaf };
|
||||
|
||||
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0x8f, 0xa2 }))), ""sv);
|
||||
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0xaf }))), MUST(decoder.to_utf8(StringView(bytes(euc_jp_sequence)))));
|
||||
EXPECT_EQ(MUST(streaming_decoder.finish()), ""sv);
|
||||
}
|
||||
|
||||
TEST_CASE(test_streaming_decoder_shift_jis_tail)
|
||||
{
|
||||
auto& decoder = decoder_for("Shift_JIS"sv);
|
||||
auto streaming_decoder = TextCodec::StreamingDecoder { decoder };
|
||||
auto shift_jis_sequence = Vector<u8> { 0x82, 0xa0 };
|
||||
|
||||
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0x82 }))), ""sv);
|
||||
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0xa0 }))), MUST(decoder.to_utf8(StringView(bytes(shift_jis_sequence)))));
|
||||
EXPECT_EQ(MUST(streaming_decoder.finish()), ""sv);
|
||||
}
|
||||
|
||||
TEST_CASE(test_utf16le_decode)
|
||||
{
|
||||
auto decoder = TextCodec::UTF16LEDecoder();
|
||||
|
|
|
|||
Loading…
Reference in a new issue