dmaax/ladybird
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

LibTextCodec: Remove decoder validation API

Browse source 
All users instead rely on the 'Fatal' option being passed through.
This commit is contained in:
Shannon Booth 2026-06-22 21:58:47 +02:00 committed by Shannon Booth
parent b9da74d16e
commit d4048aaa96
4 changed files with 9 additions and 106 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

53
Libraries/LibTextCodec/Decoder.cpp
View file

@ -15,8 +15,6 @@
namespace TextCodec {
static constexpr u32 replacement_code_point = 0xfffd;
namespace {
class RustDecoder final : public Decoder {
@ -26,7 +24,6 @@ public:
{
}
virtual bool validate(StringView input) override;
virtual ErrorOr<String> to_utf8(StringView input, IgnoreBOM, ErrorMode) override;
virtual ErrorOr<size_t> length_in_utf16_code_units(StringView input) override;
@ -39,14 +36,12 @@ private:
class UTF8Decoder final : public Decoder {
public:
virtual ErrorOr<void> process(StringView, Function<ErrorOr<void>(u32)> on_code_point) override;
virtual bool validate(StringView) override;
virtual ErrorOr<String> to_utf8(StringView, IgnoreBOM, ErrorMode) override;
virtual ErrorOr<size_t> length_in_utf16_code_units(StringView) override;
};
class UTF16BEDecoder final : public Decoder {
public:
virtual bool validate(StringView) override;
virtual ErrorOr<String> to_utf8(StringView, IgnoreBOM, ErrorMode) override;
virtual ErrorOr<size_t> length_in_utf16_code_units(StringView) override;
@ -56,7 +51,6 @@ private:
class UTF16LEDecoder final : public Decoder {
public:
virtual bool validate(StringView) override;
virtual ErrorOr<String> to_utf8(StringView, IgnoreBOM, ErrorMode) override;
virtual ErrorOr<size_t> length_in_utf16_code_units(StringView) override;
@ -67,7 +61,6 @@ private:
class Latin1Decoder final : public Decoder {
public:
virtual ErrorOr<void> process(StringView, Function<ErrorOr<void>(u32)> on_code_point) override;
virtual bool validate(StringView) override { return true; }
virtual ErrorOr<size_t> length_in_utf16_code_units(StringView) override;
};
@ -151,16 +144,6 @@ ErrorOr<void> rust_process(StringView encoding, StringView input, IgnoreBOM igno
return {};
}
bool rust_validate(StringView encoding, StringView input, IgnoreBOM ignore_bom)
{
return FFI::textcodec_rust_validate(
reinterpret_cast<u8 const*>(encoding.characters_without_null_termination()),
encoding.length(),
reinterpret_cast<u8 const*>(input.characters_without_null_termination()),
input.length(),
ignore_bom == IgnoreBOM::No);
}
ErrorOr<size_t> rust_length_in_utf16_code_units(StringView encoding, StringView input, IgnoreBOM ignore_bom)
{
auto utf8 = TRY(rust_decode_to_utf8(encoding, input, ignore_bom, ErrorMode::Replacement));
@ -407,22 +390,8 @@ StringView get_output_encoding(StringView encoding)
return encoding;
}
bool Decoder::validate(StringView input)
ErrorOr<String> Decoder::to_utf8(StringView input, IgnoreBOM, ErrorMode)
{
auto result = this->process(input, [](auto code_point) -> ErrorOr<void> {
if (code_point == replacement_code_point)
return Error::from_string_literal("Decoded input contains replacement character");
return {};
});
return !result.is_error();
}
ErrorOr<String> Decoder::to_utf8(StringView input, IgnoreBOM, ErrorMode error_mode)
{
if (error_mode == ErrorMode::Fatal && !validate(input))
return Error::from_string_literal("Failed to decode input");
StringBuilder builder(input.length());
TRY(process(input, [&builder](u32 c) { return builder.try_append_code_point(c); }));
return builder.to_string_without_validation();
@ -453,11 +422,6 @@ ErrorOr<void> Decoder::process_code_points(StringView input, Function<ErrorOr<vo
return process(input, move(on_code_point));
}
bool RustDecoder::validate(StringView input)
{
return rust_validate(m_encoding, input, IgnoreBOM::Yes);
}
ErrorOr<String> RustDecoder::to_utf8(StringView input, IgnoreBOM ignore_bom, ErrorMode error_mode)
{
return rust_decode_to_utf8(m_encoding, input, ignore_bom, error_mode);
@ -515,11 +479,6 @@ ErrorOr<void> UTF8Decoder::process(StringView input, Function<ErrorOr<void>(u32)
return rust_process("UTF-8"sv, input, IgnoreBOM::Yes, move(on_code_point));
}
bool UTF8Decoder::validate(StringView input)
{
return rust_validate("UTF-8"sv, input, IgnoreBOM::Yes);
}
ErrorOr<String> UTF8Decoder::to_utf8(StringView input, IgnoreBOM ignore_bom, ErrorMode error_mode)
{
return rust_decode_to_utf8("UTF-8"sv, input, ignore_bom, error_mode);
@ -530,11 +489,6 @@ ErrorOr<size_t> UTF8Decoder::length_in_utf16_code_units(StringView input)
return rust_length_in_utf16_code_units("UTF-8"sv, input, IgnoreBOM::No);
}
bool UTF16BEDecoder::validate(StringView input)
{
return rust_validate("UTF-16BE"sv, input, IgnoreBOM::Yes);
}
ErrorOr<void> UTF16BEDecoder::process(StringView input, Function<ErrorOr<void>(u32)> on_code_point)
{
return rust_process("UTF-16BE"sv, input, IgnoreBOM::No, move(on_code_point));
@ -550,11 +504,6 @@ ErrorOr<size_t> UTF16BEDecoder::length_in_utf16_code_units(StringView input)
return rust_length_in_utf16_code_units("UTF-16BE"sv, input, IgnoreBOM::No);
}
bool UTF16LEDecoder::validate(StringView input)
{
return rust_validate("UTF-16LE"sv, input, IgnoreBOM::Yes);
}
ErrorOr<void> UTF16LEDecoder::process(StringView input, Function<ErrorOr<void>(u32)> on_code_point)
{
return rust_process("UTF-16LE"sv, input, IgnoreBOM::No, move(on_code_point));

1
Libraries/LibTextCodec/Decoder.h
View file

@ -32,7 +32,6 @@ enum class ErrorMode {
class TEXTCODEC_API Decoder {
public:
virtual bool validate(StringView);
virtual ErrorOr<String> to_utf8(StringView, IgnoreBOM, ErrorMode);
virtual ErrorOr<Utf16String> to_utf16(StringView);
virtual ErrorOr<size_t> length_in_utf16_code_units(StringView);

43
Libraries/LibTextCodec/Rust/src/lib.rs
View file

@ -123,49 +123,6 @@ pub unsafe extern "C" fn textcodec_rust_decode_to_utf8(
}
}
/// # Safety
/// - `encoding_label`/`encoding_label_len` and `input`/`input_len` must be valid byte slices.
#[unsafe(no_mangle)]
pub unsafe extern "C" fn textcodec_rust_validate(
encoding_label: *const u8,
encoding_label_len: usize,
input: *const u8,
input_len: usize,
remove_bom: bool,
) -> bool {
unsafe {
abort_on_panic(|| {
let Some(label) = bytes_from_raw(encoding_label, encoding_label_len) else {
return false;
};
let Some(input) = bytes_from_raw(input, input_len) else {
return false;
};
let Some(encoding) = Encoding::for_label(label) else {
return false;
};
let input = if remove_bom {
if encoding == encoding_rs::UTF_8 && input.starts_with(b"\xEF\xBB\xBF") {
&input[3..]
} else if (encoding == encoding_rs::UTF_16LE && input.starts_with(b"\xFF\xFE"))
|| (encoding == encoding_rs::UTF_16BE && input.starts_with(b"\xFE\xFF"))
{
&input[2..]
} else {
input
}
} else {
input
};
encoding
.decode_without_bom_handling_and_without_replacement(input)
.is_some()
})
}
}
/// # Safety
/// - `encoding_label`/`encoding_label_len` must be a valid byte slice.
/// - The returned pointer must be freed with `textcodec_rust_streaming_decoder_free`.

18
Tests/LibTextCodec/TestTextDecoders.cpp
View file

@ -37,7 +37,7 @@ TEST_CASE(test_utf8_decode)
// Bytes for U+1F600 GRINNING FACE
auto test_string = "\xf0\x9f\x98\x80"sv;
EXPECT(decoder.validate(test_string));
EXPECT(!decoder.to_utf8(test_string, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Fatal).is_error());
auto processed_code_points = process_code_points(decoder, test_string);
EXPECT(processed_code_points.size() == 1);
@ -60,7 +60,7 @@ TEST_CASE(test_utf8_process_code_points_replaces_surrogates)
auto utf8_encoded_surrogate_bytes = Vector<u8> { 0xed, 0xa0, 0x80 };
auto utf8_encoded_surrogate = StringView(bytes(utf8_encoded_surrogate_bytes));
EXPECT(!decoder.validate(utf8_encoded_surrogate));
EXPECT(decoder.to_utf8(utf8_encoded_surrogate, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Fatal).is_error());
EXPECT_EQ(MUST(decoder.to_utf8(utf8_encoded_surrogate, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)), "\xef\xbf\xbd\xef\xbf\xbd\xef\xbf\xbd"sv);
EXPECT_EQ(process_code_points(decoder, utf8_encoded_surrogate), (Vector<u32> { 0xfffd, 0xfffd, 0xfffd }));
}
@ -71,7 +71,7 @@ TEST_CASE(test_utf8_process_code_points_replaces_truncated_tail_as_single_error)
auto truncated_tail_bytes = Vector<u8> { 0xf0, 0x9f, 0x98 };
auto truncated_tail = StringView(bytes(truncated_tail_bytes));
EXPECT(!decoder.validate(truncated_tail));
EXPECT(decoder.to_utf8(truncated_tail, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Fatal).is_error());
EXPECT_EQ(MUST(decoder.to_utf8(truncated_tail, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)), "\xef\xbf\xbd"sv);
EXPECT_EQ(process_code_points(decoder, truncated_tail), (Vector<u32> { 0xfffd }));
}
@ -82,7 +82,7 @@ TEST_CASE(test_utf8_process_code_points_replaces_overlong_sequences)
auto overlong_null_bytes = Vector<u8> { 0xc0, 0x80 };
auto overlong_null = StringView(bytes(overlong_null_bytes));
EXPECT(!decoder.validate(overlong_null));
EXPECT(decoder.to_utf8(overlong_null, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Fatal).is_error());
EXPECT_EQ(MUST(decoder.to_utf8(overlong_null, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)), "\xef\xbf\xbd\xef\xbf\xbd"sv);
EXPECT_EQ(process_code_points(decoder, overlong_null), (Vector<u32> { 0xfffd, 0xfffd }));
}
@ -95,11 +95,11 @@ TEST_CASE(test_utf8_process_code_points_restores_invalid_second_byte)
auto out_of_range_four_byte_sequence_bytes = Vector<u8> { 0xf4, 0x90, 0x80, 0x80 };
auto out_of_range_four_byte_sequence = StringView(bytes(out_of_range_four_byte_sequence_bytes));
EXPECT(!decoder.validate(overlong_three_byte_sequence));
EXPECT(decoder.to_utf8(overlong_three_byte_sequence, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Fatal).is_error());
EXPECT_EQ(MUST(decoder.to_utf8(overlong_three_byte_sequence, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)), "\xef\xbf\xbd\xef\xbf\xbd\xef\xbf\xbd"sv);
EXPECT_EQ(process_code_points(decoder, overlong_three_byte_sequence), (Vector<u32> { 0xfffd, 0xfffd, 0xfffd }));
EXPECT(!decoder.validate(out_of_range_four_byte_sequence));
EXPECT(decoder.to_utf8(out_of_range_four_byte_sequence, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Fatal).is_error());
EXPECT_EQ(MUST(decoder.to_utf8(out_of_range_four_byte_sequence, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)), "\xef\xbf\xbd\xef\xbf\xbd\xef\xbf\xbd\xef\xbf\xbd"sv);
EXPECT_EQ(process_code_points(decoder, out_of_range_four_byte_sequence), (Vector<u32> { 0xfffd, 0xfffd, 0xfffd, 0xfffd }));
}
@ -110,8 +110,7 @@ TEST_CASE(test_utf16be_decode)
// This is the output of `python3 -c "print('säk😀'.encode('utf-16be'))"`.
auto test_string = "\x00s\x00\xe4\x00k\xd8=\xde\x00"sv;
EXPECT(decoder.validate(test_string));
auto utf8 = MUST(decoder.to_utf8(test_string, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement));
auto utf8 = MUST(decoder.to_utf8(test_string, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Fatal));
EXPECT_EQ(utf8, "säk😀"sv);
}
@ -236,8 +235,7 @@ TEST_CASE(test_utf16le_decode)
// This is the output of `python3 -c "print('säk😀'.encode('utf-16le'))"`.
auto test_string = "s\x00\xe4\x00k\x00=\xd8\x00\xde"sv;
EXPECT(decoder.validate(test_string));
auto utf8 = MUST(decoder.to_utf8(test_string, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement));
auto utf8 = MUST(decoder.to_utf8(test_string, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Fatal));
EXPECT_EQ(utf8, "säk😀"sv);
}