LibTextCodec: Preserve malformed decoder replacements
Reject UTF-8 second bytes outside the Encoding Standard's per-lead-byte bounds before consuming the rest of each sequence. This keeps surrogate and out-of-range sequences from collapsing multiple malformed bytes into one replacement character. Also report an odd trailing UTF-16 byte as U+FFFD through the streaming code point path and route UTF-16 to_utf8() through the same logic. This keeps lazy and eager script decoding aligned for bytecode cache source hashes. Cover the malformed UTF-8 and UTF-16 cases in LibTextCodec, TextDecoder, and bytecode-cache source decoding tests.
This commit is contained in:
Andreas Kling
Andreas Kling
parent
f4960d9d7d
commit
45da0e4a0e
5 changed files with 70 additions and 18 deletions
|
|
@ -369,6 +369,22 @@ static bool is_utf8_continuation_byte(u8 byte)
|
|||
return (byte & 0xc0) == 0x80;
|
||||
}
|
||||
|
||||
static bool is_utf8_second_byte_in_range(u8 lead, u8 byte)
|
||||
{
|
||||
if (!is_utf8_continuation_byte(byte))
|
||||
return false;
|
||||
|
||||
if (lead == 0xe0)
|
||||
return byte >= 0xa0;
|
||||
if (lead == 0xed)
|
||||
return byte <= 0x9f;
|
||||
if (lead == 0xf0)
|
||||
return byte >= 0x90;
|
||||
if (lead == 0xf4)
|
||||
return byte <= 0x8f;
|
||||
return true;
|
||||
}
|
||||
|
||||
static Optional<size_t> utf8_sequence_length(u8 lead)
|
||||
{
|
||||
if (lead <= 0x7f)
|
||||
|
|
@ -728,6 +744,12 @@ static ErrorOr<void> process_utf8_with_replacement_character(StringView input, F
|
|||
continue;
|
||||
}
|
||||
|
||||
if (i + 1 < bytes.size() && !is_utf8_second_byte_in_range(byte, bytes[i + 1])) {
|
||||
TRY(on_code_point(replacement_code_point));
|
||||
++i;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (i + sequence_length > bytes.size()) {
|
||||
TRY(on_code_point(replacement_code_point));
|
||||
i = bytes.size();
|
||||
|
|
@ -759,7 +781,7 @@ static ErrorOr<void> process_utf8_with_replacement_character(StringView input, F
|
|||
|
||||
if (is_unicode_surrogate(code_point)) {
|
||||
TRY(on_code_point(replacement_code_point));
|
||||
i += sequence_length;
|
||||
++i;
|
||||
continue;
|
||||
}
|
||||
|
||||
|
|
@ -794,6 +816,8 @@ ErrorOr<String> UTF8Decoder::to_utf8(StringView input)
|
|||
|
||||
bool UTF16BEDecoder::validate(StringView input)
|
||||
{
|
||||
if (input.bytes().size() % 2 != 0)
|
||||
return false;
|
||||
return AK::validate_utf16_be(input.bytes());
|
||||
}
|
||||
|
||||
|
|
@ -835,6 +859,9 @@ static ErrorOr<void> process_utf16(StringView input, bool big_endian, Function<E
|
|||
TRY(on_code_point(code_unit));
|
||||
}
|
||||
|
||||
if (bytes.size() % 2 != 0)
|
||||
TRY(on_code_point(replacement_code_point));
|
||||
|
||||
return {};
|
||||
}
|
||||
|
||||
|
|
@ -845,15 +872,13 @@ ErrorOr<void> UTF16BEDecoder::process(StringView input, Function<ErrorOr<void>(u
|
|||
|
||||
ErrorOr<String> UTF16BEDecoder::to_utf8(StringView input)
|
||||
{
|
||||
// Discard the BOM
|
||||
if (auto bytes = input.bytes(); bytes.size() >= 2 && bytes[0] == 0xFE && bytes[1] == 0xFF)
|
||||
input = input.substring_view(2);
|
||||
|
||||
return String::from_utf16_be_with_replacement_character(input.bytes());
|
||||
return Decoder::to_utf8(input);
|
||||
}
|
||||
|
||||
bool UTF16LEDecoder::validate(StringView input)
|
||||
{
|
||||
if (input.bytes().size() % 2 != 0)
|
||||
return false;
|
||||
return AK::validate_utf16_le(input.bytes());
|
||||
}
|
||||
|
||||
|
|
@ -864,11 +889,7 @@ ErrorOr<void> UTF16LEDecoder::process(StringView input, Function<ErrorOr<void>(u
|
|||
|
||||
ErrorOr<String> UTF16LEDecoder::to_utf8(StringView input)
|
||||
{
|
||||
// Discard the BOM
|
||||
if (auto bytes = input.bytes(); bytes.size() >= 2 && bytes[0] == 0xFF && bytes[1] == 0xFE)
|
||||
input = input.substring_view(2);
|
||||
|
||||
return String::from_utf16_le_with_replacement_character(input.bytes());
|
||||
return Decoder::to_utf8(input);
|
||||
}
|
||||
|
||||
ErrorOr<void> Latin1Decoder::process(StringView input, Function<ErrorOr<void>(u32)> on_code_point)
|
||||
|
|
|
|||
|
|
@ -30,10 +30,20 @@ TEST_CASE(lazy_source_code_decoding_replaces_utf8_surrogates)
|
|||
{
|
||||
auto source_data = Vector<u8> { 0xed, 0xa0, 0x80 };
|
||||
auto source_bytes = TRY_OR_FAIL(Core::ImmutableBytes::copy(source_data.span()));
|
||||
auto source_code = JS::SourceCode::create("test.js"_string, 1, "UTF-8"_string, move(source_bytes));
|
||||
auto source_code = JS::SourceCode::create("test.js"_string, 3, "UTF-8"_string, move(source_bytes));
|
||||
|
||||
EXPECT_EQ(source_code->code().to_utf8(), "\xef\xbf\xbd"sv);
|
||||
EXPECT_EQ(source_code->source_text_from_offsets(0, 1).to_utf8(), "\xef\xbf\xbd"sv);
|
||||
EXPECT_EQ(source_code->code().to_utf8(), "\xef\xbf\xbd\xef\xbf\xbd\xef\xbf\xbd"sv);
|
||||
EXPECT_EQ(source_code->source_text_from_offsets(0, 3).to_utf8(), "\xef\xbf\xbd\xef\xbf\xbd\xef\xbf\xbd"sv);
|
||||
}
|
||||
|
||||
TEST_CASE(lazy_source_code_decoding_replaces_odd_trailing_utf16_byte)
|
||||
{
|
||||
auto source_data = Vector<u8> { 'A', 0x00, 0xff };
|
||||
auto source_bytes = TRY_OR_FAIL(Core::ImmutableBytes::copy(source_data.span()));
|
||||
auto source_code = JS::SourceCode::create("test.js"_string, 2, "UTF-16LE"_string, move(source_bytes));
|
||||
|
||||
EXPECT_EQ(source_code->code().to_utf8(), "A\xef\xbf\xbd"sv);
|
||||
EXPECT_EQ(source_code->source_text_from_offsets(0, 2).to_utf8(), "A\xef\xbf\xbd"sv);
|
||||
}
|
||||
|
||||
TEST_CASE(lazy_source_code_decoding_replaces_overlong_utf8_sequences)
|
||||
|
|
|
|||
|
|
@ -64,8 +64,8 @@ TEST_CASE(test_utf8_process_code_points_replaces_surrogates)
|
|||
auto utf8_encoded_surrogate = StringView(bytes(utf8_encoded_surrogate_bytes));
|
||||
|
||||
EXPECT(!decoder.validate(utf8_encoded_surrogate));
|
||||
EXPECT_EQ(MUST(decoder.to_utf8(utf8_encoded_surrogate)), "\xef\xbf\xbd"sv);
|
||||
EXPECT_EQ(process_code_points(decoder, utf8_encoded_surrogate), (Vector<u32> { 0xfffd }));
|
||||
EXPECT_EQ(MUST(decoder.to_utf8(utf8_encoded_surrogate)), "\xef\xbf\xbd\xef\xbf\xbd\xef\xbf\xbd"sv);
|
||||
EXPECT_EQ(process_code_points(decoder, utf8_encoded_surrogate), (Vector<u32> { 0xfffd, 0xfffd, 0xfffd }));
|
||||
}
|
||||
|
||||
TEST_CASE(test_utf8_process_code_points_replaces_truncated_tail_as_single_error)
|
||||
|
|
@ -90,6 +90,23 @@ TEST_CASE(test_utf8_process_code_points_replaces_overlong_sequences)
|
|||
EXPECT_EQ(process_code_points(decoder, overlong_null), (Vector<u32> { 0xfffd, 0xfffd }));
|
||||
}
|
||||
|
||||
TEST_CASE(test_utf8_process_code_points_restores_invalid_second_byte)
|
||||
{
|
||||
auto decoder = TextCodec::UTF8Decoder();
|
||||
auto overlong_three_byte_sequence_bytes = Vector<u8> { 0xe0, 0x80, 0x80 };
|
||||
auto overlong_three_byte_sequence = StringView(bytes(overlong_three_byte_sequence_bytes));
|
||||
auto out_of_range_four_byte_sequence_bytes = Vector<u8> { 0xf4, 0x90, 0x80, 0x80 };
|
||||
auto out_of_range_four_byte_sequence = StringView(bytes(out_of_range_four_byte_sequence_bytes));
|
||||
|
||||
EXPECT(!decoder.validate(overlong_three_byte_sequence));
|
||||
EXPECT_EQ(MUST(decoder.to_utf8(overlong_three_byte_sequence)), "\xef\xbf\xbd\xef\xbf\xbd\xef\xbf\xbd"sv);
|
||||
EXPECT_EQ(process_code_points(decoder, overlong_three_byte_sequence), (Vector<u32> { 0xfffd, 0xfffd, 0xfffd }));
|
||||
|
||||
EXPECT(!decoder.validate(out_of_range_four_byte_sequence));
|
||||
EXPECT_EQ(MUST(decoder.to_utf8(out_of_range_four_byte_sequence)), "\xef\xbf\xbd\xef\xbf\xbd\xef\xbf\xbd\xef\xbf\xbd"sv);
|
||||
EXPECT_EQ(process_code_points(decoder, out_of_range_four_byte_sequence), (Vector<u32> { 0xfffd, 0xfffd, 0xfffd, 0xfffd }));
|
||||
}
|
||||
|
||||
TEST_CASE(test_utf16be_decode)
|
||||
{
|
||||
auto decoder = TextCodec::UTF16BEDecoder();
|
||||
|
|
@ -107,6 +124,8 @@ TEST_CASE(test_utf16be_process_code_points)
|
|||
|
||||
EXPECT_EQ(process_code_points(decoder, StringView(bytes({ 0xfe, 0xff, 0x00, 'A', 0xd8, 0x3d, 0xde, 0x00 }))), (Vector<u32> { 0x41, 0x1F600 }));
|
||||
EXPECT_EQ(process_code_points(decoder, StringView(bytes({ 0xd8, 0x3d, 0x00, 'A', 0xde, 0x00 }))), (Vector<u32> { 0xfffd, 0x41, 0xfffd }));
|
||||
EXPECT_EQ(process_code_points(decoder, StringView(bytes({ 0x00, 'A', 0xff }))), (Vector<u32> { 0x41, 0xfffd }));
|
||||
EXPECT_EQ(MUST(decoder.to_utf8(StringView(bytes({ 0x00, 'A', 0xff })))), "A\xef\xbf\xbd"sv);
|
||||
}
|
||||
|
||||
TEST_CASE(test_streaming_decoder_utf8_mid_sequence)
|
||||
|
|
@ -214,4 +233,6 @@ TEST_CASE(test_utf16le_process_code_points)
|
|||
|
||||
EXPECT_EQ(process_code_points(decoder, StringView(bytes({ 0xff, 0xfe, 'A', 0x00, 0x3d, 0xd8, 0x00, 0xde }))), (Vector<u32> { 0x41, 0x1F600 }));
|
||||
EXPECT_EQ(process_code_points(decoder, StringView(bytes({ 0x3d, 0xd8, 'A', 0x00, 0x00, 0xde }))), (Vector<u32> { 0xfffd, 0x41, 0xfffd }));
|
||||
EXPECT_EQ(process_code_points(decoder, StringView(bytes({ 'A', 0x00, 0xff }))), (Vector<u32> { 0x41, 0xfffd }));
|
||||
EXPECT_EQ(MUST(decoder.to_utf8(StringView(bytes({ 'A', 0x00, 0xff })))), "A\xef\xbf\xbd"sv);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
[ABC]
|
||||
[]
|
||||
[fffd]
|
||||
[3, fffd, fffd, fffd]
|
||||
[1, fffd]
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@
|
|||
println(`[${decoder.decode()}]`);
|
||||
|
||||
const surrogate = decoder.decode(new Uint8Array([0xed, 0xa0, 0x80])); // U+D800
|
||||
println(`[${surrogate.codePointAt(0).toString(16)}]`);
|
||||
println(`[${surrogate.length}, ${surrogate.codePointAt(0).toString(16)}, ${surrogate.codePointAt(1).toString(16)}, ${surrogate.codePointAt(2).toString(16)}]`);
|
||||
|
||||
const truncatedTail = decoder.decode(new Uint8Array([0xf0, 0x9f, 0x98]));
|
||||
println(`[${truncatedTail.length}, ${truncatedTail.codePointAt(0).toString(16)}]`);
|
||||
|
|
|
|||
Loading…
Reference in a new issue