From d4048aaa9600af7796a311b79cd43a6d3f6654ed Mon Sep 17 00:00:00 2001 From: Shannon Booth Date: Mon, 22 Jun 2026 21:58:47 +0200 Subject: [PATCH] LibTextCodec: Remove decoder validation API All users instead rely on the 'Fatal' option being passed through. --- Libraries/LibTextCodec/Decoder.cpp | 53 +------------------------ Libraries/LibTextCodec/Decoder.h | 1 - Libraries/LibTextCodec/Rust/src/lib.rs | 43 -------------------- Tests/LibTextCodec/TestTextDecoders.cpp | 18 ++++----- 4 files changed, 9 insertions(+), 106 deletions(-) diff --git a/Libraries/LibTextCodec/Decoder.cpp b/Libraries/LibTextCodec/Decoder.cpp index 2a704e9835..b71dfaf542 100644 --- a/Libraries/LibTextCodec/Decoder.cpp +++ b/Libraries/LibTextCodec/Decoder.cpp @@ -15,8 +15,6 @@ namespace TextCodec { -static constexpr u32 replacement_code_point = 0xfffd; - namespace { class RustDecoder final : public Decoder { @@ -26,7 +24,6 @@ public: { } - virtual bool validate(StringView input) override; virtual ErrorOr to_utf8(StringView input, IgnoreBOM, ErrorMode) override; virtual ErrorOr length_in_utf16_code_units(StringView input) override; @@ -39,14 +36,12 @@ private: class UTF8Decoder final : public Decoder { public: virtual ErrorOr process(StringView, Function(u32)> on_code_point) override; - virtual bool validate(StringView) override; virtual ErrorOr to_utf8(StringView, IgnoreBOM, ErrorMode) override; virtual ErrorOr length_in_utf16_code_units(StringView) override; }; class UTF16BEDecoder final : public Decoder { public: - virtual bool validate(StringView) override; virtual ErrorOr to_utf8(StringView, IgnoreBOM, ErrorMode) override; virtual ErrorOr length_in_utf16_code_units(StringView) override; @@ -56,7 +51,6 @@ private: class UTF16LEDecoder final : public Decoder { public: - virtual bool validate(StringView) override; virtual ErrorOr to_utf8(StringView, IgnoreBOM, ErrorMode) override; virtual ErrorOr length_in_utf16_code_units(StringView) override; @@ -67,7 +61,6 @@ private: class Latin1Decoder final : public Decoder { public: virtual ErrorOr process(StringView, Function(u32)> on_code_point) override; - virtual bool validate(StringView) override { return true; } virtual ErrorOr length_in_utf16_code_units(StringView) override; }; @@ -151,16 +144,6 @@ ErrorOr rust_process(StringView encoding, StringView input, IgnoreBOM igno return {}; } -bool rust_validate(StringView encoding, StringView input, IgnoreBOM ignore_bom) -{ - return FFI::textcodec_rust_validate( - reinterpret_cast(encoding.characters_without_null_termination()), - encoding.length(), - reinterpret_cast(input.characters_without_null_termination()), - input.length(), - ignore_bom == IgnoreBOM::No); -} - ErrorOr rust_length_in_utf16_code_units(StringView encoding, StringView input, IgnoreBOM ignore_bom) { auto utf8 = TRY(rust_decode_to_utf8(encoding, input, ignore_bom, ErrorMode::Replacement)); @@ -407,22 +390,8 @@ StringView get_output_encoding(StringView encoding) return encoding; } -bool Decoder::validate(StringView input) +ErrorOr Decoder::to_utf8(StringView input, IgnoreBOM, ErrorMode) { - auto result = this->process(input, [](auto code_point) -> ErrorOr { - if (code_point == replacement_code_point) - return Error::from_string_literal("Decoded input contains replacement character"); - return {}; - }); - - return !result.is_error(); -} - -ErrorOr Decoder::to_utf8(StringView input, IgnoreBOM, ErrorMode error_mode) -{ - if (error_mode == ErrorMode::Fatal && !validate(input)) - return Error::from_string_literal("Failed to decode input"); - StringBuilder builder(input.length()); TRY(process(input, [&builder](u32 c) { return builder.try_append_code_point(c); })); return builder.to_string_without_validation(); @@ -453,11 +422,6 @@ ErrorOr Decoder::process_code_points(StringView input, Function RustDecoder::to_utf8(StringView input, IgnoreBOM ignore_bom, ErrorMode error_mode) { return rust_decode_to_utf8(m_encoding, input, ignore_bom, error_mode); @@ -515,11 +479,6 @@ ErrorOr UTF8Decoder::process(StringView input, Function(u32) return rust_process("UTF-8"sv, input, IgnoreBOM::Yes, move(on_code_point)); } -bool UTF8Decoder::validate(StringView input) -{ - return rust_validate("UTF-8"sv, input, IgnoreBOM::Yes); -} - ErrorOr UTF8Decoder::to_utf8(StringView input, IgnoreBOM ignore_bom, ErrorMode error_mode) { return rust_decode_to_utf8("UTF-8"sv, input, ignore_bom, error_mode); @@ -530,11 +489,6 @@ ErrorOr UTF8Decoder::length_in_utf16_code_units(StringView input) return rust_length_in_utf16_code_units("UTF-8"sv, input, IgnoreBOM::No); } -bool UTF16BEDecoder::validate(StringView input) -{ - return rust_validate("UTF-16BE"sv, input, IgnoreBOM::Yes); -} - ErrorOr UTF16BEDecoder::process(StringView input, Function(u32)> on_code_point) { return rust_process("UTF-16BE"sv, input, IgnoreBOM::No, move(on_code_point)); @@ -550,11 +504,6 @@ ErrorOr UTF16BEDecoder::length_in_utf16_code_units(StringView input) return rust_length_in_utf16_code_units("UTF-16BE"sv, input, IgnoreBOM::No); } -bool UTF16LEDecoder::validate(StringView input) -{ - return rust_validate("UTF-16LE"sv, input, IgnoreBOM::Yes); -} - ErrorOr UTF16LEDecoder::process(StringView input, Function(u32)> on_code_point) { return rust_process("UTF-16LE"sv, input, IgnoreBOM::No, move(on_code_point)); diff --git a/Libraries/LibTextCodec/Decoder.h b/Libraries/LibTextCodec/Decoder.h index 721fa56dee..ea16cc7c63 100644 --- a/Libraries/LibTextCodec/Decoder.h +++ b/Libraries/LibTextCodec/Decoder.h @@ -32,7 +32,6 @@ enum class ErrorMode { class TEXTCODEC_API Decoder { public: - virtual bool validate(StringView); virtual ErrorOr to_utf8(StringView, IgnoreBOM, ErrorMode); virtual ErrorOr to_utf16(StringView); virtual ErrorOr length_in_utf16_code_units(StringView); diff --git a/Libraries/LibTextCodec/Rust/src/lib.rs b/Libraries/LibTextCodec/Rust/src/lib.rs index d35f677af0..94b8b2c787 100644 --- a/Libraries/LibTextCodec/Rust/src/lib.rs +++ b/Libraries/LibTextCodec/Rust/src/lib.rs @@ -123,49 +123,6 @@ pub unsafe extern "C" fn textcodec_rust_decode_to_utf8( } } -/// # Safety -/// - `encoding_label`/`encoding_label_len` and `input`/`input_len` must be valid byte slices. -#[unsafe(no_mangle)] -pub unsafe extern "C" fn textcodec_rust_validate( - encoding_label: *const u8, - encoding_label_len: usize, - input: *const u8, - input_len: usize, - remove_bom: bool, -) -> bool { - unsafe { - abort_on_panic(|| { - let Some(label) = bytes_from_raw(encoding_label, encoding_label_len) else { - return false; - }; - let Some(input) = bytes_from_raw(input, input_len) else { - return false; - }; - let Some(encoding) = Encoding::for_label(label) else { - return false; - }; - - let input = if remove_bom { - if encoding == encoding_rs::UTF_8 && input.starts_with(b"\xEF\xBB\xBF") { - &input[3..] - } else if (encoding == encoding_rs::UTF_16LE && input.starts_with(b"\xFF\xFE")) - || (encoding == encoding_rs::UTF_16BE && input.starts_with(b"\xFE\xFF")) - { - &input[2..] - } else { - input - } - } else { - input - }; - - encoding - .decode_without_bom_handling_and_without_replacement(input) - .is_some() - }) - } -} - /// # Safety /// - `encoding_label`/`encoding_label_len` must be a valid byte slice. /// - The returned pointer must be freed with `textcodec_rust_streaming_decoder_free`. diff --git a/Tests/LibTextCodec/TestTextDecoders.cpp b/Tests/LibTextCodec/TestTextDecoders.cpp index e467b2196c..188d84e9b4 100644 --- a/Tests/LibTextCodec/TestTextDecoders.cpp +++ b/Tests/LibTextCodec/TestTextDecoders.cpp @@ -37,7 +37,7 @@ TEST_CASE(test_utf8_decode) // Bytes for U+1F600 GRINNING FACE auto test_string = "\xf0\x9f\x98\x80"sv; - EXPECT(decoder.validate(test_string)); + EXPECT(!decoder.to_utf8(test_string, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Fatal).is_error()); auto processed_code_points = process_code_points(decoder, test_string); EXPECT(processed_code_points.size() == 1); @@ -60,7 +60,7 @@ TEST_CASE(test_utf8_process_code_points_replaces_surrogates) auto utf8_encoded_surrogate_bytes = Vector { 0xed, 0xa0, 0x80 }; auto utf8_encoded_surrogate = StringView(bytes(utf8_encoded_surrogate_bytes)); - EXPECT(!decoder.validate(utf8_encoded_surrogate)); + EXPECT(decoder.to_utf8(utf8_encoded_surrogate, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Fatal).is_error()); EXPECT_EQ(MUST(decoder.to_utf8(utf8_encoded_surrogate, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)), "\xef\xbf\xbd\xef\xbf\xbd\xef\xbf\xbd"sv); EXPECT_EQ(process_code_points(decoder, utf8_encoded_surrogate), (Vector { 0xfffd, 0xfffd, 0xfffd })); } @@ -71,7 +71,7 @@ TEST_CASE(test_utf8_process_code_points_replaces_truncated_tail_as_single_error) auto truncated_tail_bytes = Vector { 0xf0, 0x9f, 0x98 }; auto truncated_tail = StringView(bytes(truncated_tail_bytes)); - EXPECT(!decoder.validate(truncated_tail)); + EXPECT(decoder.to_utf8(truncated_tail, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Fatal).is_error()); EXPECT_EQ(MUST(decoder.to_utf8(truncated_tail, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)), "\xef\xbf\xbd"sv); EXPECT_EQ(process_code_points(decoder, truncated_tail), (Vector { 0xfffd })); } @@ -82,7 +82,7 @@ TEST_CASE(test_utf8_process_code_points_replaces_overlong_sequences) auto overlong_null_bytes = Vector { 0xc0, 0x80 }; auto overlong_null = StringView(bytes(overlong_null_bytes)); - EXPECT(!decoder.validate(overlong_null)); + EXPECT(decoder.to_utf8(overlong_null, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Fatal).is_error()); EXPECT_EQ(MUST(decoder.to_utf8(overlong_null, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)), "\xef\xbf\xbd\xef\xbf\xbd"sv); EXPECT_EQ(process_code_points(decoder, overlong_null), (Vector { 0xfffd, 0xfffd })); } @@ -95,11 +95,11 @@ TEST_CASE(test_utf8_process_code_points_restores_invalid_second_byte) auto out_of_range_four_byte_sequence_bytes = Vector { 0xf4, 0x90, 0x80, 0x80 }; auto out_of_range_four_byte_sequence = StringView(bytes(out_of_range_four_byte_sequence_bytes)); - EXPECT(!decoder.validate(overlong_three_byte_sequence)); + EXPECT(decoder.to_utf8(overlong_three_byte_sequence, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Fatal).is_error()); EXPECT_EQ(MUST(decoder.to_utf8(overlong_three_byte_sequence, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)), "\xef\xbf\xbd\xef\xbf\xbd\xef\xbf\xbd"sv); EXPECT_EQ(process_code_points(decoder, overlong_three_byte_sequence), (Vector { 0xfffd, 0xfffd, 0xfffd })); - EXPECT(!decoder.validate(out_of_range_four_byte_sequence)); + EXPECT(decoder.to_utf8(out_of_range_four_byte_sequence, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Fatal).is_error()); EXPECT_EQ(MUST(decoder.to_utf8(out_of_range_four_byte_sequence, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)), "\xef\xbf\xbd\xef\xbf\xbd\xef\xbf\xbd\xef\xbf\xbd"sv); EXPECT_EQ(process_code_points(decoder, out_of_range_four_byte_sequence), (Vector { 0xfffd, 0xfffd, 0xfffd, 0xfffd })); } @@ -110,8 +110,7 @@ TEST_CASE(test_utf16be_decode) // This is the output of `python3 -c "print('säk😀'.encode('utf-16be'))"`. auto test_string = "\x00s\x00\xe4\x00k\xd8=\xde\x00"sv; - EXPECT(decoder.validate(test_string)); - auto utf8 = MUST(decoder.to_utf8(test_string, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)); + auto utf8 = MUST(decoder.to_utf8(test_string, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Fatal)); EXPECT_EQ(utf8, "säk😀"sv); } @@ -236,8 +235,7 @@ TEST_CASE(test_utf16le_decode) // This is the output of `python3 -c "print('säk😀'.encode('utf-16le'))"`. auto test_string = "s\x00\xe4\x00k\x00=\xd8\x00\xde"sv; - EXPECT(decoder.validate(test_string)); - auto utf8 = MUST(decoder.to_utf8(test_string, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)); + auto utf8 = MUST(decoder.to_utf8(test_string, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Fatal)); EXPECT_EQ(utf8, "säk😀"sv); }