LibTextCodec: Stop buffering invalid UTF-8 tails

Treat trailing UTF-8 prefixes with an invalid second byte as complete
input for streaming decode, so replacement characters are emitted in the
current chunk instead of being held until later input or finish. Keep
valid incomplete prefixes buffered across chunk boundaries.

Keep TextDecoderStream from holding continuation bytes after an invalid
lead byte at a chunk boundary. Add LibTextCodec and TextDecoderStream
coverage for invalid tails, valid split sequences, EOF partials,
surrogate sequences, and malformed continuation tails.

Libraries/LibTextCodec/Decoder.cpp

@@ -412,6 +412,8 @@ size_t UTF8Decoder::incomplete_tail_length(ReadonlyBytes bytes) const
         size_t seen = back + 1;
         if (!seq_len.has_value() || *seq_len <= seen)
             return 0;
+        if (seen >= 2 && !is_utf8_second_byte_in_range(byte, bytes[bytes.size() - back]))
+            return 0;
         return seen;
     }
     return 0;

Libraries/LibWeb/Encoding/TextDecoderStream.cpp

@@ -24,6 +24,26 @@ namespace Web::Encoding {
 
 GC_DEFINE_ALLOCATOR(TextDecoderStream);
 
+static bool is_utf8_continuation_byte(u8 byte)
+{
+    return (byte & 0xc0) == 0x80;
+}
+
+static bool is_utf8_second_byte_in_range(u8 lead, u8 byte)
+{
+    if (!is_utf8_continuation_byte(byte))
+        return false;
+    if (lead == 0xe0)
+        return byte >= 0xa0;
+    if (lead == 0xed)
+        return byte <= 0x9f;
+    if (lead == 0xf0)
+        return byte >= 0x90;
+    if (lead == 0xf4)
+        return byte <= 0x8f;
+    return true;
+}
+
 // Returns the largest prefix length of `bytes` that can be safely decoded as UTF-8 without splitting an in-progress
 // multi-byte sequence. The remainder (if any) is held over for the next chunk.
 static size_t find_utf8_safe_decode_boundary(ReadonlyBytes bytes)
@@ -36,7 +56,7 @@ static size_t find_utf8_safe_decode_boundary(ReadonlyBytes bytes)
         u8 byte = bytes[pos];
 
         // Continuation byte (10xxxxxx): keep walking back to find the leading byte.
-        if ((byte & 0xC0) == 0x80) {
+        if (is_utf8_continuation_byte(byte)) {
             ++scan;
             continue;
         }
@@ -49,14 +69,16 @@ static size_t find_utf8_safe_decode_boundary(ReadonlyBytes bytes)
         // sequence, cut before it so the next chunk can complete it. Otherwise (recognized and complete, or
         // unrecognized so it'll just become a replacement character) include all bytes up to the end.
         size_t expected_length = 0;
-        if ((byte & 0xE0) == 0xC0)
+        if (byte >= 0xc2 && byte <= 0xdf)
             expected_length = 2;
-        else if ((byte & 0xF0) == 0xE0)
+        else if (byte >= 0xe0 && byte <= 0xef)
             expected_length = 3;
-        else if ((byte & 0xF8) == 0xF0)
+        else if (byte >= 0xf0 && byte <= 0xf4)
             expected_length = 4;
         else
-            return pos + 1;
+            return bytes.size();
+        if (bytes.size() - pos >= 2 && !is_utf8_second_byte_in_range(byte, bytes[pos + 1]))
+            return bytes.size();
         if (bytes.size() - pos >= expected_length)
             return bytes.size();
         return pos;

Tests/LibTextCodec/TestTextDecoders.cpp

@@ -148,6 +148,29 @@ TEST_CASE(test_streaming_decoder_finishes_incomplete_sequence)
     EXPECT_EQ(MUST(streaming_decoder.finish()), MUST(decoder.to_utf8(StringView(bytes(incomplete_sequence)))));
 }
 
+TEST_CASE(test_streaming_decoder_utf8_invalid_second_byte_tail)
+{
+    auto& decoder = decoder_for("UTF-8"sv);
+
+    auto streaming_decoder_for_overlong_three_byte_sequence = TextCodec::StreamingDecoder { decoder };
+    EXPECT_EQ(MUST(streaming_decoder_for_overlong_three_byte_sequence.to_utf8(bytes({ 0xe0, 0x80 }))), "\xef\xbf\xbd\xef\xbf\xbd"sv);
+    EXPECT_EQ(MUST(streaming_decoder_for_overlong_three_byte_sequence.finish()), ""sv);
+
+    auto streaming_decoder_for_out_of_range_four_byte_sequence = TextCodec::StreamingDecoder { decoder };
+    EXPECT_EQ(MUST(streaming_decoder_for_out_of_range_four_byte_sequence.to_utf8(bytes({ 0xf4, 0x90 }))), "\xef\xbf\xbd\xef\xbf\xbd"sv);
+    EXPECT_EQ(MUST(streaming_decoder_for_out_of_range_four_byte_sequence.finish()), ""sv);
+}
+
+TEST_CASE(test_streaming_decoder_utf8_valid_second_byte_tail)
+{
+    auto& decoder = decoder_for("UTF-8"sv);
+    auto streaming_decoder = TextCodec::StreamingDecoder { decoder };
+
+    EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0xe0, 0xa0 }))), ""sv);
+    EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0x80 }))), "\xe0\xa0\x80"sv);
+    EXPECT_EQ(MUST(streaming_decoder.finish()), ""sv);
+}
+
 TEST_CASE(test_streaming_decoder_utf16_odd_byte)
 {
     auto& decoder = decoder_for("UTF-16LE"sv);

Tests/LibWeb/Text/expected/Encoding/TextDecoderStream-decode.txt

@@ -0,0 +1,17 @@
+invalid_second_byte_e0_80: [2, fffd, fffd]
+invalid_second_byte_f4_90: [2, fffd, fffd]
+split_valid_2_byte: [1, e9]
+split_valid_3_byte: [1, 800]
+split_valid_4_byte: [2, 10000]
+eof_incomplete_2_byte: [1, fffd]
+eof_incomplete_3_byte: [1, fffd]
+eof_incomplete_4_byte: [1, fffd]
+ascii_prefix_then_eof_incomplete_3_byte: [1, 78] | [1, fffd]
+incomplete_prefix_then_ascii: [2, fffd, 78]
+complete_sequence_with_extra_continuation: [2, e9, fffd]
+surrogate_sequence: [3, fffd, fffd, fffd]
+invalid_lead_f5_with_continuations: [4, fffd, fffd, fffd, fffd]
+overlong_c0_with_continuations: [3, fffd, fffd, fffd]
+continuation_only_tail: [4, fffd, fffd, fffd, fffd]
+invalid_lead_then_ascii: [2, fffd, 78]
+invalid_lead_with_continuations_then_ascii: [5, fffd, fffd, fffd, fffd, 78]

Tests/LibWeb/Text/input/Encoding/TextDecoderStream-decode.html

@@ -0,0 +1,54 @@
+<!DOCTYPE html>
+<script src="../include.js"></script>
+<script>
+    function describe(string) {
+        const codePoints = Array.from(string, codePoint => codePoint.codePointAt(0).toString(16));
+        return `[${string.length}, ${codePoints.join(", ")}]`;
+    }
+
+    async function decodeChunks(chunks) {
+        const readable = new ReadableStream({
+            start(controller) {
+                for (const chunk of chunks)
+                    controller.enqueue(new Uint8Array(chunk));
+                controller.close();
+            },
+        });
+        const reader = readable.pipeThrough(new TextDecoderStream("utf-8")).getReader();
+        const decoded = [];
+
+        for (;;) {
+            const { value, done } = await reader.read();
+            if (done)
+                break;
+            decoded.push(describe(value));
+        }
+
+        return decoded;
+    }
+
+    promiseTest(async () => {
+        const cases = [
+            ["invalid_second_byte_e0_80", [[0xe0, 0x80]]],
+            ["invalid_second_byte_f4_90", [[0xf4, 0x90]]],
+            ["split_valid_2_byte", [[0xc3], [0xa9]]],
+            ["split_valid_3_byte", [[0xe0, 0xa0], [0x80]]],
+            ["split_valid_4_byte", [[0xf0, 0x90, 0x80], [0x80]]],
+            ["eof_incomplete_2_byte", [[0xc3]]],
+            ["eof_incomplete_3_byte", [[0xe0, 0xa0]]],
+            ["eof_incomplete_4_byte", [[0xf0, 0x90, 0x80]]],
+            ["ascii_prefix_then_eof_incomplete_3_byte", [[0x78, 0xe0, 0xa0]]],
+            ["incomplete_prefix_then_ascii", [[0xe0, 0xa0], [0x78]]],
+            ["complete_sequence_with_extra_continuation", [[0xc3, 0xa9, 0x80]]],
+            ["surrogate_sequence", [[0xed, 0xa0, 0x80]]],
+            ["invalid_lead_f5_with_continuations", [[0xf5, 0x80, 0x80, 0x80]]],
+            ["overlong_c0_with_continuations", [[0xc0, 0x80, 0x80]]],
+            ["continuation_only_tail", [[0x80, 0x80, 0x80, 0x80]]],
+            ["invalid_lead_then_ascii", [[0xf5, 0x78]]],
+            ["invalid_lead_with_continuations_then_ascii", [[0xf5, 0x80, 0x80, 0x80, 0x78]]],
+        ];
+
+        for (const [name, chunks] of cases)
+            println(`${name}: ${(await decodeChunks(chunks)).join(" | ")}`);
+    });
+</script>