01cec162c8
Treat trailing UTF-8 prefixes with an invalid second byte as complete input for streaming decode, so replacement characters are emitted in the current chunk instead of being held until later input or finish. Keep valid incomplete prefixes buffered across chunk boundaries. Keep TextDecoderStream from holding continuation bytes after an invalid lead byte at a chunk boundary. Add LibTextCodec and TextDecoderStream coverage for invalid tails, valid split sequences, EOF partials, surrogate sequences, and malformed continuation tails.
54 lines
2.2 KiB
HTML
54 lines
2.2 KiB
HTML
<!DOCTYPE html>
|
|
<script src="../include.js"></script>
|
|
<script>
|
|
function describe(string) {
|
|
const codePoints = Array.from(string, codePoint => codePoint.codePointAt(0).toString(16));
|
|
return `[${string.length}, ${codePoints.join(", ")}]`;
|
|
}
|
|
|
|
async function decodeChunks(chunks) {
|
|
const readable = new ReadableStream({
|
|
start(controller) {
|
|
for (const chunk of chunks)
|
|
controller.enqueue(new Uint8Array(chunk));
|
|
controller.close();
|
|
},
|
|
});
|
|
const reader = readable.pipeThrough(new TextDecoderStream("utf-8")).getReader();
|
|
const decoded = [];
|
|
|
|
for (;;) {
|
|
const { value, done } = await reader.read();
|
|
if (done)
|
|
break;
|
|
decoded.push(describe(value));
|
|
}
|
|
|
|
return decoded;
|
|
}
|
|
|
|
promiseTest(async () => {
|
|
const cases = [
|
|
["invalid_second_byte_e0_80", [[0xe0, 0x80]]],
|
|
["invalid_second_byte_f4_90", [[0xf4, 0x90]]],
|
|
["split_valid_2_byte", [[0xc3], [0xa9]]],
|
|
["split_valid_3_byte", [[0xe0, 0xa0], [0x80]]],
|
|
["split_valid_4_byte", [[0xf0, 0x90, 0x80], [0x80]]],
|
|
["eof_incomplete_2_byte", [[0xc3]]],
|
|
["eof_incomplete_3_byte", [[0xe0, 0xa0]]],
|
|
["eof_incomplete_4_byte", [[0xf0, 0x90, 0x80]]],
|
|
["ascii_prefix_then_eof_incomplete_3_byte", [[0x78, 0xe0, 0xa0]]],
|
|
["incomplete_prefix_then_ascii", [[0xe0, 0xa0], [0x78]]],
|
|
["complete_sequence_with_extra_continuation", [[0xc3, 0xa9, 0x80]]],
|
|
["surrogate_sequence", [[0xed, 0xa0, 0x80]]],
|
|
["invalid_lead_f5_with_continuations", [[0xf5, 0x80, 0x80, 0x80]]],
|
|
["overlong_c0_with_continuations", [[0xc0, 0x80, 0x80]]],
|
|
["continuation_only_tail", [[0x80, 0x80, 0x80, 0x80]]],
|
|
["invalid_lead_then_ascii", [[0xf5, 0x78]]],
|
|
["invalid_lead_with_continuations_then_ascii", [[0xf5, 0x80, 0x80, 0x80, 0x78]]],
|
|
];
|
|
|
|
for (const [name, chunks] of cases)
|
|
println(`${name}: ${(await decodeChunks(chunks)).join(" | ")}`);
|
|
});
|
|
</script>
|