From ef6753a9f90f72d2605aea0367f94c4367910c2f Mon Sep 17 00:00:00 2001 From: Shannon Booth Date: Mon, 22 Jun 2026 22:07:40 +0200 Subject: [PATCH] LibWeb+LibTextCodec: Wire decoder options through TextDecoder Add explicit IgnoreBOM and ErrorMode options to LibTextCodec decoders, and thread them through TextDecoder and TextDecoderStream. This lets Web-facing decoder APIs preserve BOMs when requested and use fatal error handling without post-processing decoded output. NB: RemoveBOM was renamed to IgnoreBOM as "RemoveBOM" is the name used by encoding_rs and was previously an implementation detail. The new name matches what is used by the encoding standard as it is now also used in LibWeb. --- Libraries/LibTextCodec/Decoder.cpp | 95 ++++++++++--------- Libraries/LibTextCodec/Decoder.h | 16 +++- Libraries/LibTextCodec/Rust/src/lib.rs | 12 ++- Libraries/LibWeb/CSS/Parser/RustTokenizer.cpp | 2 +- Libraries/LibWeb/CSS/Parser/Tokenizer.cpp | 2 +- Libraries/LibWeb/DOM/DocumentLoading.cpp | 4 +- Libraries/LibWeb/Encoding/TextDecoder.cpp | 19 ++-- Libraries/LibWeb/Encoding/TextDecoder.h | 2 +- .../LibWeb/Encoding/TextDecoderCommon.cpp | 2 +- Libraries/LibWeb/Encoding/TextDecoderCommon.h | 17 +--- .../LibWeb/Encoding/TextDecoderStream.cpp | 32 +++---- Libraries/LibWeb/Encoding/TextDecoderStream.h | 2 +- .../LibWeb/HTML/Parser/HTMLTokenizer.cpp | 2 +- .../HTML/Parser/IncrementalDocumentParser.cpp | 2 +- Libraries/LibWebView/Autocomplete.cpp | 2 +- Meta/Fuzzers/FuzzTextDecoder.cpp | 2 +- Services/RequestServer/Request.cpp | 2 +- Tests/LibTextCodec/TestTextDecoders.cpp | 56 +++++------ .../streams/decode-ignore-bom.any.txt | 26 ++--- .../encoding/textdecoder-ignorebom.any.txt | 9 +- .../encoding/textdecoder-mistakes.any.txt | 8 +- 21 files changed, 163 insertions(+), 151 deletions(-) diff --git a/Libraries/LibTextCodec/Decoder.cpp b/Libraries/LibTextCodec/Decoder.cpp index 1a10bf5eda..2a704e9835 100644 --- a/Libraries/LibTextCodec/Decoder.cpp +++ b/Libraries/LibTextCodec/Decoder.cpp @@ -19,11 +19,6 @@ static constexpr u32 replacement_code_point = 0xfffd; namespace { -enum class RemoveBOM { - No, - Yes, -}; - class RustDecoder final : public Decoder { public: explicit RustDecoder(StringView encoding) @@ -32,7 +27,7 @@ public: } virtual bool validate(StringView input) override; - virtual ErrorOr to_utf8(StringView input) override; + virtual ErrorOr to_utf8(StringView input, IgnoreBOM, ErrorMode) override; virtual ErrorOr length_in_utf16_code_units(StringView input) override; private: @@ -45,14 +40,14 @@ class UTF8Decoder final : public Decoder { public: virtual ErrorOr process(StringView, Function(u32)> on_code_point) override; virtual bool validate(StringView) override; - virtual ErrorOr to_utf8(StringView) override; + virtual ErrorOr to_utf8(StringView, IgnoreBOM, ErrorMode) override; virtual ErrorOr length_in_utf16_code_units(StringView) override; }; class UTF16BEDecoder final : public Decoder { public: virtual bool validate(StringView) override; - virtual ErrorOr to_utf8(StringView) override; + virtual ErrorOr to_utf8(StringView, IgnoreBOM, ErrorMode) override; virtual ErrorOr length_in_utf16_code_units(StringView) override; private: @@ -62,7 +57,7 @@ private: class UTF16LEDecoder final : public Decoder { public: virtual bool validate(StringView) override; - virtual ErrorOr to_utf8(StringView) override; + virtual ErrorOr to_utf8(StringView, IgnoreBOM, ErrorMode) override; virtual ErrorOr length_in_utf16_code_units(StringView) override; private: @@ -130,7 +125,7 @@ static void append_decoded_bytes(void* context, u8 const* data, size_t length) decode_context.result = decode_context.builder.try_append(StringView { data, length }); } -ErrorOr rust_decode_to_utf8(StringView encoding, StringView input, RemoveBOM remove_bom) +ErrorOr rust_decode_to_utf8(StringView encoding, StringView input, IgnoreBOM ignore_bom, ErrorMode error_mode) { DecodeContext context { .builder = StringBuilder(input.length()) }; auto succeeded = FFI::textcodec_rust_decode_to_utf8( @@ -138,36 +133,37 @@ ErrorOr rust_decode_to_utf8(StringView encoding, StringView input, Remov encoding.length(), reinterpret_cast(input.characters_without_null_termination()), input.length(), - remove_bom == RemoveBOM::Yes, + ignore_bom == IgnoreBOM::No, + error_mode == ErrorMode::Fatal, &context, append_decoded_bytes); if (!succeeded) - return Error::from_errno(EINVAL); + return Error::from_string_literal("Failed to decode input"); TRY(context.result); return context.builder.to_string_without_validation(); } -ErrorOr rust_process(StringView encoding, StringView input, RemoveBOM remove_bom, Function(u32)> on_code_point) +ErrorOr rust_process(StringView encoding, StringView input, IgnoreBOM ignore_bom, Function(u32)> on_code_point) { - auto utf8 = TRY(rust_decode_to_utf8(encoding, input, remove_bom)); + auto utf8 = TRY(rust_decode_to_utf8(encoding, input, ignore_bom, ErrorMode::Replacement)); for (auto code_point : Utf8View { utf8 }) TRY(on_code_point(code_point)); return {}; } -bool rust_validate(StringView encoding, StringView input, RemoveBOM remove_bom) +bool rust_validate(StringView encoding, StringView input, IgnoreBOM ignore_bom) { return FFI::textcodec_rust_validate( reinterpret_cast(encoding.characters_without_null_termination()), encoding.length(), reinterpret_cast(input.characters_without_null_termination()), input.length(), - remove_bom == RemoveBOM::Yes); + ignore_bom == IgnoreBOM::No); } -ErrorOr rust_length_in_utf16_code_units(StringView encoding, StringView input, RemoveBOM remove_bom) +ErrorOr rust_length_in_utf16_code_units(StringView encoding, StringView input, IgnoreBOM ignore_bom) { - auto utf8 = TRY(rust_decode_to_utf8(encoding, input, remove_bom)); + auto utf8 = TRY(rust_decode_to_utf8(encoding, input, ignore_bom, ErrorMode::Replacement)); size_t length = 0; for (auto code_point : Utf8View { utf8 }) length += code_point <= 0xffff ? 1 : 2; @@ -188,7 +184,7 @@ Optional get_static_encoding_name_from_rust(StringView label) return StringView { encoding_name, encoding_name_length }; } -ErrorOr rust_streaming_decode_to_utf8(FFI::TextCodecRustStreamingDecoder* decoder, ReadonlyBytes input, bool last) +ErrorOr rust_streaming_decode_to_utf8(FFI::TextCodecRustStreamingDecoder* decoder, ReadonlyBytes input, bool last, ErrorMode error_mode) { DecodeContext context { .builder = StringBuilder(input.size()) }; auto succeeded = FFI::textcodec_rust_streaming_decoder_decode_to_utf8( @@ -196,10 +192,11 @@ ErrorOr rust_streaming_decode_to_utf8(FFI::TextCodecRustStreamingDecoder input.data(), input.size(), last, + error_mode == ErrorMode::Fatal, &context, append_decoded_bytes); if (!succeeded) - return Error::from_errno(EINVAL); + return Error::from_string_literal("Failed to decode input"); TRY(context.result); return context.builder.to_string_without_validation(); } @@ -367,7 +364,7 @@ ErrorOr convert_input_to_utf8_using_given_decoder_unless_there_is_a_byte // 3. Process a queue with an instance of encoding’s decoder, ioQueue, output, and "replacement". // FIXME: This isn't the exact same as the spec, which is written in terms of I/O queues. - auto output = TRY(actual_decoder->to_utf8(input)); + auto output = TRY(actual_decoder->to_utf8(input, IgnoreBOM::No, ErrorMode::Replacement)); // 4. Return output. return output; @@ -414,15 +411,18 @@ bool Decoder::validate(StringView input) { auto result = this->process(input, [](auto code_point) -> ErrorOr { if (code_point == replacement_code_point) - return Error::from_errno(EINVAL); + return Error::from_string_literal("Decoded input contains replacement character"); return {}; }); return !result.is_error(); } -ErrorOr Decoder::to_utf8(StringView input) +ErrorOr Decoder::to_utf8(StringView input, IgnoreBOM, ErrorMode error_mode) { + if (error_mode == ErrorMode::Fatal && !validate(input)) + return Error::from_string_literal("Failed to decode input"); + StringBuilder builder(input.length()); TRY(process(input, [&builder](u32 c) { return builder.try_append_code_point(c); })); return builder.to_string_without_validation(); @@ -455,22 +455,22 @@ ErrorOr Decoder::process_code_points(StringView input, Function RustDecoder::to_utf8(StringView input) +ErrorOr RustDecoder::to_utf8(StringView input, IgnoreBOM ignore_bom, ErrorMode error_mode) { - return rust_decode_to_utf8(m_encoding, input, RemoveBOM::No); + return rust_decode_to_utf8(m_encoding, input, ignore_bom, error_mode); } ErrorOr RustDecoder::length_in_utf16_code_units(StringView input) { - return rust_length_in_utf16_code_units(m_encoding, input, RemoveBOM::No); + return rust_length_in_utf16_code_units(m_encoding, input, IgnoreBOM::Yes); } ErrorOr RustDecoder::process(StringView input, Function(u32)> on_code_point) { - return rust_process(m_encoding, input, RemoveBOM::No, move(on_code_point)); + return rust_process(m_encoding, input, IgnoreBOM::Yes, move(on_code_point)); } ErrorOr Latin1Decoder::process(StringView input, Function(u32)> on_code_point) @@ -485,12 +485,13 @@ ErrorOr Latin1Decoder::length_in_utf16_code_units(StringView input) return input.length(); } -StreamingDecoder::StreamingDecoder(StringView encoding) +StreamingDecoder::StreamingDecoder(StringView encoding, IgnoreBOM ignore_bom, ErrorMode error_mode) + : m_error_mode(error_mode) { m_decoder = FFI::textcodec_rust_streaming_decoder_new( reinterpret_cast(encoding.characters_without_null_termination()), encoding.length(), - true); + ignore_bom == IgnoreBOM::No); VERIFY(m_decoder); } @@ -501,72 +502,72 @@ StreamingDecoder::~StreamingDecoder() ErrorOr StreamingDecoder::to_utf8(ReadonlyBytes input) { - return rust_streaming_decode_to_utf8(static_cast(m_decoder), input, false); + return rust_streaming_decode_to_utf8(static_cast(m_decoder), input, false, m_error_mode); } ErrorOr StreamingDecoder::finish() { - return rust_streaming_decode_to_utf8(static_cast(m_decoder), {}, true); + return rust_streaming_decode_to_utf8(static_cast(m_decoder), {}, true, m_error_mode); } ErrorOr UTF8Decoder::process(StringView input, Function(u32)> on_code_point) { - return rust_process("UTF-8"sv, input, RemoveBOM::No, move(on_code_point)); + return rust_process("UTF-8"sv, input, IgnoreBOM::Yes, move(on_code_point)); } bool UTF8Decoder::validate(StringView input) { - return rust_validate("UTF-8"sv, input, RemoveBOM::No); + return rust_validate("UTF-8"sv, input, IgnoreBOM::Yes); } -ErrorOr UTF8Decoder::to_utf8(StringView input) +ErrorOr UTF8Decoder::to_utf8(StringView input, IgnoreBOM ignore_bom, ErrorMode error_mode) { - return rust_decode_to_utf8("UTF-8"sv, input, RemoveBOM::Yes); + return rust_decode_to_utf8("UTF-8"sv, input, ignore_bom, error_mode); } ErrorOr UTF8Decoder::length_in_utf16_code_units(StringView input) { - return rust_length_in_utf16_code_units("UTF-8"sv, input, RemoveBOM::Yes); + return rust_length_in_utf16_code_units("UTF-8"sv, input, IgnoreBOM::No); } bool UTF16BEDecoder::validate(StringView input) { - return rust_validate("UTF-16BE"sv, input, RemoveBOM::No); + return rust_validate("UTF-16BE"sv, input, IgnoreBOM::Yes); } ErrorOr UTF16BEDecoder::process(StringView input, Function(u32)> on_code_point) { - return rust_process("UTF-16BE"sv, input, RemoveBOM::Yes, move(on_code_point)); + return rust_process("UTF-16BE"sv, input, IgnoreBOM::No, move(on_code_point)); } -ErrorOr UTF16BEDecoder::to_utf8(StringView input) +ErrorOr UTF16BEDecoder::to_utf8(StringView input, IgnoreBOM ignore_bom, ErrorMode error_mode) { - return rust_decode_to_utf8("UTF-16BE"sv, input, RemoveBOM::Yes); + return rust_decode_to_utf8("UTF-16BE"sv, input, ignore_bom, error_mode); } ErrorOr UTF16BEDecoder::length_in_utf16_code_units(StringView input) { - return rust_length_in_utf16_code_units("UTF-16BE"sv, input, RemoveBOM::Yes); + return rust_length_in_utf16_code_units("UTF-16BE"sv, input, IgnoreBOM::No); } bool UTF16LEDecoder::validate(StringView input) { - return rust_validate("UTF-16LE"sv, input, RemoveBOM::No); + return rust_validate("UTF-16LE"sv, input, IgnoreBOM::Yes); } ErrorOr UTF16LEDecoder::process(StringView input, Function(u32)> on_code_point) { - return rust_process("UTF-16LE"sv, input, RemoveBOM::Yes, move(on_code_point)); + return rust_process("UTF-16LE"sv, input, IgnoreBOM::No, move(on_code_point)); } -ErrorOr UTF16LEDecoder::to_utf8(StringView input) +ErrorOr UTF16LEDecoder::to_utf8(StringView input, IgnoreBOM ignore_bom, ErrorMode error_mode) { - return rust_decode_to_utf8("UTF-16LE"sv, input, RemoveBOM::Yes); + return rust_decode_to_utf8("UTF-16LE"sv, input, ignore_bom, error_mode); } ErrorOr UTF16LEDecoder::length_in_utf16_code_units(StringView input) { - return rust_length_in_utf16_code_units("UTF-16LE"sv, input, RemoveBOM::Yes); + return rust_length_in_utf16_code_units("UTF-16LE"sv, input, IgnoreBOM::No); } // https://infra.spec.whatwg.org/#isomorphic-decode diff --git a/Libraries/LibTextCodec/Decoder.h b/Libraries/LibTextCodec/Decoder.h index 8319110eff..721fa56dee 100644 --- a/Libraries/LibTextCodec/Decoder.h +++ b/Libraries/LibTextCodec/Decoder.h @@ -19,10 +19,21 @@ namespace TextCodec { +enum class IgnoreBOM { + Yes, + No, +}; + +// https://encoding.spec.whatwg.org/#concept-encoding-error-mode +enum class ErrorMode { + Replacement, + Fatal, +}; + class TEXTCODEC_API Decoder { public: virtual bool validate(StringView); - virtual ErrorOr to_utf8(StringView); + virtual ErrorOr to_utf8(StringView, IgnoreBOM, ErrorMode); virtual ErrorOr to_utf16(StringView); virtual ErrorOr length_in_utf16_code_units(StringView); ErrorOr process_code_points(StringView, Function(u32)>); @@ -36,13 +47,14 @@ class TEXTCODEC_API StreamingDecoder final { AK_MAKE_NONCOPYABLE(StreamingDecoder); public: - explicit StreamingDecoder(StringView encoding); + StreamingDecoder(StringView encoding, IgnoreBOM, ErrorMode); ~StreamingDecoder(); ErrorOr to_utf8(ReadonlyBytes); ErrorOr finish(); private: + ErrorMode m_error_mode { ErrorMode::Replacement }; void* m_decoder { nullptr }; }; diff --git a/Libraries/LibTextCodec/Rust/src/lib.rs b/Libraries/LibTextCodec/Rust/src/lib.rs index 31833bc0dd..d35f677af0 100644 --- a/Libraries/LibTextCodec/Rust/src/lib.rs +++ b/Libraries/LibTextCodec/Rust/src/lib.rs @@ -93,6 +93,7 @@ pub unsafe extern "C" fn textcodec_rust_decode_to_utf8( input: *const u8, input_len: usize, remove_bom: bool, + fatal: bool, ctx: *mut c_void, on_bytes: FfiBytesFn, ) -> bool { @@ -108,11 +109,14 @@ pub unsafe extern "C" fn textcodec_rust_decode_to_utf8( return false; }; - let (output, _) = if remove_bom { + let (output, had_errors) = if remove_bom { encoding.decode_with_bom_removal(input) } else { encoding.decode_without_bom_handling(input) }; + if fatal && had_errors { + return false; + } on_bytes(ctx, output.as_bytes().as_ptr(), output.len()); true }) @@ -212,6 +216,7 @@ pub unsafe extern "C" fn textcodec_rust_streaming_decoder_decode_to_utf8( input: *const u8, input_len: usize, last: bool, + fatal: bool, ctx: *mut c_void, on_bytes: FfiBytesFn, ) -> bool { @@ -230,7 +235,10 @@ pub unsafe extern "C" fn textcodec_rust_streaming_decoder_decode_to_utf8( }; let mut output = String::with_capacity(output_capacity); - let (result, _, _) = decoder.decoder.decode_to_string(input, &mut output, last); + let (result, _, had_errors) = decoder.decoder.decode_to_string(input, &mut output, last); + if fatal && had_errors { + return false; + } if !output.is_empty() { on_bytes(ctx, output.as_ptr(), output.len()); } diff --git a/Libraries/LibWeb/CSS/Parser/RustTokenizer.cpp b/Libraries/LibWeb/CSS/Parser/RustTokenizer.cpp index 50d8dad3d1..750780b6d5 100644 --- a/Libraries/LibWeb/CSS/Parser/RustTokenizer.cpp +++ b/Libraries/LibWeb/CSS/Parser/RustTokenizer.cpp @@ -36,7 +36,7 @@ static String decode_and_filter_code_points(StringView input, StringView encodin input = input.substring_view(3); return String::from_utf8_without_validation(input.bytes()); } - return MUST(decoder->to_utf8(input)); + return MUST(decoder->to_utf8(input, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)); }(); // OPTIMIZATION: If the input doesn't contain any filterable characters, we can skip the filtering diff --git a/Libraries/LibWeb/CSS/Parser/Tokenizer.cpp b/Libraries/LibWeb/CSS/Parser/Tokenizer.cpp index a52bc54ad1..7e8931bcc3 100644 --- a/Libraries/LibWeb/CSS/Parser/Tokenizer.cpp +++ b/Libraries/LibWeb/CSS/Parser/Tokenizer.cpp @@ -175,7 +175,7 @@ Vector Tokenizer::tokenize(StringView input, StringView encoding, Tokeniz input = input.substring_view(3); return String::from_utf8_without_validation(input.bytes()); } - return MUST(decoder->to_utf8(input)); + return MUST(decoder->to_utf8(input, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)); }(); // OPTIMIZATION: If the input doesn't contain any filterable characters, we can skip the filtering diff --git a/Libraries/LibWeb/DOM/DocumentLoading.cpp b/Libraries/LibWeb/DOM/DocumentLoading.cpp index fd15dedaed..0845c455ee 100644 --- a/Libraries/LibWeb/DOM/DocumentLoading.cpp +++ b/Libraries/LibWeb/DOM/DocumentLoading.cpp @@ -77,7 +77,7 @@ bool build_xml_document(DOM::Document& document, ByteBuffer const& data, Optiona convert_to_xml_error_document(document, "XML Document contains improperly-encoded characters"_utf16); return false; } - auto source = decoder->to_utf8(data).release_value_but_fixme_should_propagate_errors(); + auto source = decoder->to_utf8(data, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Fatal).release_value_but_fixme_should_propagate_errors(); XML::Parser parser(source, { .resolve_named_html_entity = resolve_named_html_entity }); XMLDocumentBuilder builder { document }; auto result = parser.parse_with_listener(builder); @@ -204,7 +204,7 @@ static WebIDL::ExceptionOr> load_xml_document(HTML::Navig convert_to_xml_error_document(document, "XML Document contains improperly-encoded characters"_utf16); return; } - auto source = decoder->to_utf8(data); + auto source = decoder->to_utf8(data, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Fatal); if (source.is_error()) { // FIXME: Insert error message into the document. dbgln("Failed to decode XML document: {}", source.error()); diff --git a/Libraries/LibWeb/Encoding/TextDecoder.cpp b/Libraries/LibWeb/Encoding/TextDecoder.cpp index 449e4ec84b..97404f07c1 100644 --- a/Libraries/LibWeb/Encoding/TextDecoder.cpp +++ b/Libraries/LibWeb/Encoding/TextDecoder.cpp @@ -35,7 +35,7 @@ WebIDL::ExceptionOr> TextDecoder::construct_impl(JS::Realm& auto lowercase_encoding_name = encoding.value().to_ascii_lowercase_string(); // 4. If options["fatal"] is true, then set this’s error mode to "fatal". - auto error_mode = options.fatal ? ErrorMode::Fatal : ErrorMode::Replacement; + auto error_mode = options.fatal ? TextCodec::ErrorMode::Fatal : TextCodec::ErrorMode::Replacement; // 5. Set this’s ignore BOM to options["ignoreBOM"]. auto ignore_bom = options.ignore_bom; @@ -48,7 +48,7 @@ WebIDL::ExceptionOr> TextDecoder::construct_impl(JS::Realm& } // https://encoding.spec.whatwg.org/#dom-textdecoder -TextDecoder::TextDecoder(JS::Realm& realm, TextCodec::Decoder& decoder, FlyString encoding, ErrorMode error_mode, bool ignore_bom) +TextDecoder::TextDecoder(JS::Realm& realm, TextCodec::Decoder& decoder, FlyString encoding, TextCodec::ErrorMode error_mode, bool ignore_bom) : PlatformObject(realm) , TextDecoderCommonMixin(decoder, move(encoding), error_mode, ignore_bom) { @@ -65,18 +65,23 @@ void TextDecoder::initialize(JS::Realm& realm) // https://encoding.spec.whatwg.org/#dom-textdecoder-decode WebIDL::ExceptionOr TextDecoder::decode(Optional input, Optional const&) const { - if (!input.has_value()) - return TRY_OR_THROW_OOM(vm(), m_decoder.to_utf8({})); + auto ignore_bom = m_ignore_bom ? TextCodec::IgnoreBOM::Yes : TextCodec::IgnoreBOM::No; + if (!input.has_value()) { + auto result = m_decoder.to_utf8({}, ignore_bom, m_error_mode); + if (result.is_error() && result.error().code() != ENOMEM) + return WebIDL::SimpleException { WebIDL::SimpleExceptionType::TypeError, "Decoding failed"sv }; + return TRY_OR_THROW_OOM(vm(), move(result)); + } // FIXME: Implement the streaming stuff. auto data_buffer_or_error = WebIDL::get_buffer_source_copy(*input); if (data_buffer_or_error.is_error()) return WebIDL::OperationError::create(realm(), "Failed to copy bytes from ArrayBuffer"_utf16); auto& data_buffer = data_buffer_or_error.value(); - auto result = TRY_OR_THROW_OOM(vm(), m_decoder.to_utf8({ data_buffer.data(), data_buffer.size() })); - if (this->fatal() && result.contains(0xfffd)) + auto result = m_decoder.to_utf8({ data_buffer.data(), data_buffer.size() }, ignore_bom, m_error_mode); + if (result.is_error() && result.error().code() != ENOMEM) return WebIDL::SimpleException { WebIDL::SimpleExceptionType::TypeError, "Decoding failed"sv }; - return result; + return TRY_OR_THROW_OOM(vm(), move(result)); } } diff --git a/Libraries/LibWeb/Encoding/TextDecoder.h b/Libraries/LibWeb/Encoding/TextDecoder.h index cbed714218..18fb82d9ab 100644 --- a/Libraries/LibWeb/Encoding/TextDecoder.h +++ b/Libraries/LibWeb/Encoding/TextDecoder.h @@ -34,7 +34,7 @@ public: WebIDL::ExceptionOr decode(Optional, Optional const& options = {}) const; private: - TextDecoder(JS::Realm&, TextCodec::Decoder&, FlyString encoding, ErrorMode error_mode, bool ignore_bom); + TextDecoder(JS::Realm&, TextCodec::Decoder&, FlyString encoding, TextCodec::ErrorMode error_mode, bool ignore_bom); virtual void initialize(JS::Realm&) override; }; diff --git a/Libraries/LibWeb/Encoding/TextDecoderCommon.cpp b/Libraries/LibWeb/Encoding/TextDecoderCommon.cpp index 65146040de..b93ec4fc6e 100644 --- a/Libraries/LibWeb/Encoding/TextDecoderCommon.cpp +++ b/Libraries/LibWeb/Encoding/TextDecoderCommon.cpp @@ -8,7 +8,7 @@ namespace Web::Encoding { -TextDecoderCommonMixin::TextDecoderCommonMixin(TextCodec::Decoder& decoder, FlyString encoding, ErrorMode error_mode, bool ignore_bom) +TextDecoderCommonMixin::TextDecoderCommonMixin(TextCodec::Decoder& decoder, FlyString encoding, TextCodec::ErrorMode error_mode, bool ignore_bom) : m_decoder(decoder) , m_encoding(move(encoding)) , m_error_mode(error_mode) diff --git a/Libraries/LibWeb/Encoding/TextDecoderCommon.h b/Libraries/LibWeb/Encoding/TextDecoderCommon.h index a38e1bbc7e..77f835d66c 100644 --- a/Libraries/LibWeb/Encoding/TextDecoderCommon.h +++ b/Libraries/LibWeb/Encoding/TextDecoderCommon.h @@ -7,7 +7,7 @@ #pragma once #include -#include +#include namespace Web::Encoding { @@ -20,19 +20,13 @@ public: FlyString const& encoding() const { return m_encoding; } // https://encoding.spec.whatwg.org/#dom-textdecoder-fatal - bool fatal() const { return m_error_mode == ErrorMode::Fatal; } + bool fatal() const { return m_error_mode == TextCodec::ErrorMode::Fatal; } // https://encoding.spec.whatwg.org/#dom-textdecoder-ignorebom bool ignore_bom() const { return m_ignore_bom; } protected: - // https://encoding.spec.whatwg.org/#concept-encoding-error-mode - enum class ErrorMode { - Replacement, - Fatal, - }; - - TextDecoderCommonMixin(TextCodec::Decoder& decoder, FlyString encoding, ErrorMode error_mode, bool ignore_bom); + TextDecoderCommonMixin(TextCodec::Decoder& decoder, FlyString encoding, TextCodec::ErrorMode error_mode, bool ignore_bom); // https://encoding.spec.whatwg.org/#textdecodercommon-decoder TextCodec::Decoder& m_decoder; @@ -41,13 +35,10 @@ protected: FlyString m_encoding; // https://encoding.spec.whatwg.org/#textdecoder-error-mode - ErrorMode m_error_mode { ErrorMode::Replacement }; + TextCodec::ErrorMode m_error_mode { TextCodec::ErrorMode::Replacement }; // https://encoding.spec.whatwg.org/#textdecoder-ignore-bom-flag bool m_ignore_bom { false }; - - // https://encoding.spec.whatwg.org/#textdecoder-bom-seen-flag - bool m_bom_seen { false }; }; } diff --git a/Libraries/LibWeb/Encoding/TextDecoderStream.cpp b/Libraries/LibWeb/Encoding/TextDecoderStream.cpp index 7aeda02506..fe43f2cc6a 100644 --- a/Libraries/LibWeb/Encoding/TextDecoderStream.cpp +++ b/Libraries/LibWeb/Encoding/TextDecoderStream.cpp @@ -39,7 +39,7 @@ WebIDL::ExceptionOr> TextDecoderStream::construct_imp auto lowercase_encoding_name = encoding.value().to_ascii_lowercase_string(); // 4. If options["fatal"] is true, then set this’s error mode to "fatal". - auto error_mode = options.fatal ? ErrorMode::Fatal : ErrorMode::Replacement; + auto error_mode = options.fatal ? TextCodec::ErrorMode::Fatal : TextCodec::ErrorMode::Replacement; // 5. Set this’s ignore BOM to options["ignoreBOM"]. auto ignore_bom = options.ignore_bom; @@ -84,11 +84,14 @@ WebIDL::ExceptionOr> TextDecoderStream::construct_imp return stream; } -TextDecoderStream::TextDecoderStream(JS::Realm& realm, GC::Ref transform, TextCodec::Decoder& decoder, FlyString encoding, ErrorMode error_mode, bool ignore_bom) +TextDecoderStream::TextDecoderStream(JS::Realm& realm, GC::Ref transform, TextCodec::Decoder& decoder, FlyString encoding, TextCodec::ErrorMode error_mode, bool ignore_bom) : Bindings::PlatformObject(realm) , Streams::GenericTransformStreamMixin(transform) , TextDecoderCommonMixin(decoder, move(encoding), error_mode, ignore_bom) - , m_streaming_decoder(make(m_encoding)) + , m_streaming_decoder(make( + m_encoding, + m_ignore_bom ? TextCodec::IgnoreBOM::Yes : TextCodec::IgnoreBOM::No, + m_error_mode)) { } @@ -122,7 +125,10 @@ WebIDL::ExceptionOr TextDecoderStream::decode_and_enqueue_chunk(JS::Value return WebIDL::OperationError::create(realm, "Failed to copy bytes from BufferSource"_utf16); auto buffer = buffer_or_error.release_value(); - auto decoded = TRY_OR_THROW_OOM(vm, m_streaming_decoder->to_utf8(buffer.bytes())); + auto decoded_or_error = m_streaming_decoder->to_utf8(buffer.bytes()); + if (decoded_or_error.is_error() && decoded_or_error.error().code() != ENOMEM) + return WebIDL::SimpleException { WebIDL::SimpleExceptionType::TypeError, "Decoding failed"sv }; + auto decoded = TRY_OR_THROW_OOM(vm, move(decoded_or_error)); // 3-4. Run "processing an item" until the input is exhausted, accumulating the output, then enqueue any non-empty // result. If processing returns error, throw a TypeError. @@ -133,7 +139,10 @@ WebIDL::ExceptionOr TextDecoderStream::decode_and_enqueue_chunk(JS::Value WebIDL::ExceptionOr TextDecoderStream::flush_and_enqueue() { // 1-3. Drain decoder's I/O queue and run "processing an item" to completion. - auto decoded = TRY_OR_THROW_OOM(vm(), m_streaming_decoder->finish()); + auto decoded_or_error = m_streaming_decoder->finish(); + if (decoded_or_error.is_error() && decoded_or_error.error().code() != ENOMEM) + return WebIDL::SimpleException { WebIDL::SimpleExceptionType::TypeError, "Decoding failed"sv }; + auto decoded = TRY_OR_THROW_OOM(vm(), move(decoded_or_error)); return enqueue_decoded_output(decoded); } @@ -142,22 +151,9 @@ WebIDL::ExceptionOr TextDecoderStream::enqueue_decoded_output(String const auto& realm = this->realm(); auto& vm = realm.vm(); - // https://encoding.spec.whatwg.org/#concept-td-serialize - // FIXME: The underlying TextCodec decoders currently strip leading BOMs unconditionally for UTF-8 and UTF-16BE/LE, - // so the "ignore BOM" flag is effectively ignored here. Once the decoders accept a "preserve BOM" mode, - // plumb m_ignore_bom through and strip the BOM from `decoded` only when m_ignore_bom is false. - if (!m_bom_seen && !decoded.is_empty()) - m_bom_seen = true; - if (decoded.is_empty()) return {}; - // If decoder's error mode is "fatal" and processing produced any error, throw a TypeError. - // NB: We can only detect this approximately by looking for U+FFFD in the decoded output, which the underlying - // decoder substitutes for invalid sequences. This matches the existing TextDecoder.decode() behavior. - if (fatal() && decoded.contains(0xFFFD)) - return WebIDL::SimpleException { WebIDL::SimpleExceptionType::TypeError, "Decoding failed"sv }; - auto js_string = JS::PrimitiveString::create(vm, Utf16String::from_utf8(decoded)); return Streams::transform_stream_default_controller_enqueue(*m_transform->controller(), js_string); } diff --git a/Libraries/LibWeb/Encoding/TextDecoderStream.h b/Libraries/LibWeb/Encoding/TextDecoderStream.h index b9a39a5b70..cea1236750 100644 --- a/Libraries/LibWeb/Encoding/TextDecoderStream.h +++ b/Libraries/LibWeb/Encoding/TextDecoderStream.h @@ -27,7 +27,7 @@ public: virtual ~TextDecoderStream() override; private: - TextDecoderStream(JS::Realm&, GC::Ref, TextCodec::Decoder&, FlyString encoding, ErrorMode, bool ignore_bom); + TextDecoderStream(JS::Realm&, GC::Ref, TextCodec::Decoder&, FlyString encoding, TextCodec::ErrorMode, bool ignore_bom); virtual void initialize(JS::Realm&) override; virtual void visit_edges(Cell::Visitor&) override; diff --git a/Libraries/LibWeb/HTML/Parser/HTMLTokenizer.cpp b/Libraries/LibWeb/HTML/Parser/HTMLTokenizer.cpp index 967181985b..83605942f4 100644 --- a/Libraries/LibWeb/HTML/Parser/HTMLTokenizer.cpp +++ b/Libraries/LibWeb/HTML/Parser/HTMLTokenizer.cpp @@ -108,7 +108,7 @@ HTMLTokenizer::HTMLTokenizer(StringView input, ByteString const& encoding, Input if (input_type == InputType::EncodedBytes) { auto decoder = TextCodec::decoder_for(encoding); VERIFY(decoder.has_value()); - m_source = MUST(decoder->to_utf8(input)); + m_source = MUST(decoder->to_utf8(input, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)); } else { m_source = decoded_string_for_utf8_tokenizer(input); } diff --git a/Libraries/LibWeb/HTML/Parser/IncrementalDocumentParser.cpp b/Libraries/LibWeb/HTML/Parser/IncrementalDocumentParser.cpp index 919e2c3cab..f0801b4e50 100644 --- a/Libraries/LibWeb/HTML/Parser/IncrementalDocumentParser.cpp +++ b/Libraries/LibWeb/HTML/Parser/IncrementalDocumentParser.cpp @@ -68,7 +68,7 @@ void IncrementalDocumentParser::initialize_parser(ReadonlyBytes sniff_bytes) auto standardized_encoding = TextCodec::get_standardized_encoding(encoding); VERIFY(standardized_encoding.has_value()); - m_decoder = make(standardized_encoding.value()); + m_decoder = make(standardized_encoding.value(), TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement); // https://html.spec.whatwg.org/multipage/parsing.html#determining-the-character-encoding // The document's character encoding must immediately be set to the value returned from this diff --git a/Libraries/LibWebView/Autocomplete.cpp b/Libraries/LibWebView/Autocomplete.cpp index 79c562318e..287835f814 100644 --- a/Libraries/LibWebView/Autocomplete.cpp +++ b/Libraries/LibWebView/Autocomplete.cpp @@ -438,7 +438,7 @@ ErrorOr> Autocomplete::received_autocomplete_respsonse(Autocomple if (!decoder.has_value()) decoder = TextCodec::decoder_for_exact_name("UTF-8"sv); - auto decoded_response = TRY(decoder->to_utf8(response)); + auto decoded_response = TRY(decoder->to_utf8(response, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)); auto json = TRY(JsonValue::from_string(decoded_response)); if (engine.name == "DuckDuckGo") diff --git a/Meta/Fuzzers/FuzzTextDecoder.cpp b/Meta/Fuzzers/FuzzTextDecoder.cpp index 3125527ac7..f78385c178 100644 --- a/Meta/Fuzzers/FuzzTextDecoder.cpp +++ b/Meta/Fuzzers/FuzzTextDecoder.cpp @@ -24,6 +24,6 @@ extern "C" int LLVMFuzzerTestOneInput(uint8_t const* data, size_t size) if (!decoder.has_value()) return 0; - (void)decoder->to_utf8(encoded_data); + (void)decoder->to_utf8(encoded_data, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement); return 0; } diff --git a/Services/RequestServer/Request.cpp b/Services/RequestServer/Request.cpp index 5f193c0b8a..2297573e3b 100644 --- a/Services/RequestServer/Request.cpp +++ b/Services/RequestServer/Request.cpp @@ -1085,7 +1085,7 @@ size_t Request::on_header_received(void* buffer, size_t size, size_t nmemb, void auto decoder = TextCodec::decoder_for_exact_name("ISO-8859-1"sv); VERIFY(decoder.has_value()); - request.m_reason_phrase = MUST(decoder->to_utf8(reason_phrase)); + request.m_reason_phrase = MUST(decoder->to_utf8(reason_phrase, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)); return total_size; } } diff --git a/Tests/LibTextCodec/TestTextDecoders.cpp b/Tests/LibTextCodec/TestTextDecoders.cpp index e46f4e1351..e467b2196c 100644 --- a/Tests/LibTextCodec/TestTextDecoders.cpp +++ b/Tests/LibTextCodec/TestTextDecoders.cpp @@ -43,7 +43,7 @@ TEST_CASE(test_utf8_decode) EXPECT(processed_code_points.size() == 1); EXPECT(processed_code_points[0] == 0x1F600); - EXPECT(MUST(decoder.to_utf8(test_string)) == test_string); + EXPECT(MUST(decoder.to_utf8(test_string, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)) == test_string); } TEST_CASE(test_utf8_process_code_points) @@ -61,7 +61,7 @@ TEST_CASE(test_utf8_process_code_points_replaces_surrogates) auto utf8_encoded_surrogate = StringView(bytes(utf8_encoded_surrogate_bytes)); EXPECT(!decoder.validate(utf8_encoded_surrogate)); - EXPECT_EQ(MUST(decoder.to_utf8(utf8_encoded_surrogate)), "\xef\xbf\xbd\xef\xbf\xbd\xef\xbf\xbd"sv); + EXPECT_EQ(MUST(decoder.to_utf8(utf8_encoded_surrogate, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)), "\xef\xbf\xbd\xef\xbf\xbd\xef\xbf\xbd"sv); EXPECT_EQ(process_code_points(decoder, utf8_encoded_surrogate), (Vector { 0xfffd, 0xfffd, 0xfffd })); } @@ -72,7 +72,7 @@ TEST_CASE(test_utf8_process_code_points_replaces_truncated_tail_as_single_error) auto truncated_tail = StringView(bytes(truncated_tail_bytes)); EXPECT(!decoder.validate(truncated_tail)); - EXPECT_EQ(MUST(decoder.to_utf8(truncated_tail)), "\xef\xbf\xbd"sv); + EXPECT_EQ(MUST(decoder.to_utf8(truncated_tail, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)), "\xef\xbf\xbd"sv); EXPECT_EQ(process_code_points(decoder, truncated_tail), (Vector { 0xfffd })); } @@ -83,7 +83,7 @@ TEST_CASE(test_utf8_process_code_points_replaces_overlong_sequences) auto overlong_null = StringView(bytes(overlong_null_bytes)); EXPECT(!decoder.validate(overlong_null)); - EXPECT_EQ(MUST(decoder.to_utf8(overlong_null)), "\xef\xbf\xbd\xef\xbf\xbd"sv); + EXPECT_EQ(MUST(decoder.to_utf8(overlong_null, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)), "\xef\xbf\xbd\xef\xbf\xbd"sv); EXPECT_EQ(process_code_points(decoder, overlong_null), (Vector { 0xfffd, 0xfffd })); } @@ -96,11 +96,11 @@ TEST_CASE(test_utf8_process_code_points_restores_invalid_second_byte) auto out_of_range_four_byte_sequence = StringView(bytes(out_of_range_four_byte_sequence_bytes)); EXPECT(!decoder.validate(overlong_three_byte_sequence)); - EXPECT_EQ(MUST(decoder.to_utf8(overlong_three_byte_sequence)), "\xef\xbf\xbd\xef\xbf\xbd\xef\xbf\xbd"sv); + EXPECT_EQ(MUST(decoder.to_utf8(overlong_three_byte_sequence, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)), "\xef\xbf\xbd\xef\xbf\xbd\xef\xbf\xbd"sv); EXPECT_EQ(process_code_points(decoder, overlong_three_byte_sequence), (Vector { 0xfffd, 0xfffd, 0xfffd })); EXPECT(!decoder.validate(out_of_range_four_byte_sequence)); - EXPECT_EQ(MUST(decoder.to_utf8(out_of_range_four_byte_sequence)), "\xef\xbf\xbd\xef\xbf\xbd\xef\xbf\xbd\xef\xbf\xbd"sv); + EXPECT_EQ(MUST(decoder.to_utf8(out_of_range_four_byte_sequence, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)), "\xef\xbf\xbd\xef\xbf\xbd\xef\xbf\xbd\xef\xbf\xbd"sv); EXPECT_EQ(process_code_points(decoder, out_of_range_four_byte_sequence), (Vector { 0xfffd, 0xfffd, 0xfffd, 0xfffd })); } @@ -111,7 +111,7 @@ TEST_CASE(test_utf16be_decode) auto test_string = "\x00s\x00\xe4\x00k\xd8=\xde\x00"sv; EXPECT(decoder.validate(test_string)); - auto utf8 = MUST(decoder.to_utf8(test_string)); + auto utf8 = MUST(decoder.to_utf8(test_string, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)); EXPECT_EQ(utf8, "säk😀"sv); } @@ -122,12 +122,12 @@ TEST_CASE(test_utf16be_process_code_points) EXPECT_EQ(process_code_points(decoder, StringView(bytes({ 0xfe, 0xff, 0x00, 'A', 0xd8, 0x3d, 0xde, 0x00 }))), (Vector { 0x41, 0x1F600 })); EXPECT_EQ(process_code_points(decoder, StringView(bytes({ 0xd8, 0x3d, 0x00, 'A', 0xde, 0x00 }))), (Vector { 0xfffd, 0x41, 0xfffd })); EXPECT_EQ(process_code_points(decoder, StringView(bytes({ 0x00, 'A', 0xff }))), (Vector { 0x41, 0xfffd })); - EXPECT_EQ(MUST(decoder.to_utf8(StringView(bytes({ 0x00, 'A', 0xff })))), "A\xef\xbf\xbd"sv); + EXPECT_EQ(MUST(decoder.to_utf8(StringView(bytes({ 0x00, 'A', 0xff })), TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)), "A\xef\xbf\xbd"sv); } TEST_CASE(test_streaming_decoder_utf8_mid_sequence) { - auto streaming_decoder = TextCodec::StreamingDecoder { "UTF-8"sv }; + auto streaming_decoder = TextCodec::StreamingDecoder { "UTF-8"sv, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement }; EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 'a', 0xc3 }))), "a"sv); EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0xa9, 'b' }))), "éb"sv); @@ -137,27 +137,27 @@ TEST_CASE(test_streaming_decoder_utf8_mid_sequence) TEST_CASE(test_streaming_decoder_finishes_incomplete_sequence) { auto& decoder = decoder_for("UTF-8"sv); - auto streaming_decoder = TextCodec::StreamingDecoder { "UTF-8"sv }; + auto streaming_decoder = TextCodec::StreamingDecoder { "UTF-8"sv, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement }; auto incomplete_sequence = Vector { 0xc3 }; EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes(incomplete_sequence))), ""sv); - EXPECT_EQ(MUST(streaming_decoder.finish()), MUST(decoder.to_utf8(StringView(bytes(incomplete_sequence))))); + EXPECT_EQ(MUST(streaming_decoder.finish()), MUST(decoder.to_utf8(StringView(bytes(incomplete_sequence)), TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement))); } TEST_CASE(test_streaming_decoder_utf8_invalid_second_byte_tail) { - auto streaming_decoder_for_overlong_three_byte_sequence = TextCodec::StreamingDecoder { "UTF-8"sv }; + auto streaming_decoder_for_overlong_three_byte_sequence = TextCodec::StreamingDecoder { "UTF-8"sv, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement }; EXPECT_EQ(MUST(streaming_decoder_for_overlong_three_byte_sequence.to_utf8(bytes({ 0xe0, 0x80 }))), "\xef\xbf\xbd\xef\xbf\xbd"sv); EXPECT_EQ(MUST(streaming_decoder_for_overlong_three_byte_sequence.finish()), ""sv); - auto streaming_decoder_for_out_of_range_four_byte_sequence = TextCodec::StreamingDecoder { "UTF-8"sv }; + auto streaming_decoder_for_out_of_range_four_byte_sequence = TextCodec::StreamingDecoder { "UTF-8"sv, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement }; EXPECT_EQ(MUST(streaming_decoder_for_out_of_range_four_byte_sequence.to_utf8(bytes({ 0xf4, 0x90 }))), "\xef\xbf\xbd\xef\xbf\xbd"sv); EXPECT_EQ(MUST(streaming_decoder_for_out_of_range_four_byte_sequence.finish()), ""sv); } TEST_CASE(test_streaming_decoder_utf8_valid_second_byte_tail) { - auto streaming_decoder = TextCodec::StreamingDecoder { "UTF-8"sv }; + auto streaming_decoder = TextCodec::StreamingDecoder { "UTF-8"sv, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement }; EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0xe0, 0xa0 }))), ""sv); EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0x80 }))), "\xe0\xa0\x80"sv); @@ -166,7 +166,7 @@ TEST_CASE(test_streaming_decoder_utf8_valid_second_byte_tail) TEST_CASE(test_streaming_decoder_utf16_odd_byte) { - auto streaming_decoder = TextCodec::StreamingDecoder { "UTF-16LE"sv }; + auto streaming_decoder = TextCodec::StreamingDecoder { "UTF-16LE"sv, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement }; EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0x41, 0x00, 0x42 }))), "A"sv); EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0x00 }))), "B"sv); @@ -175,7 +175,7 @@ TEST_CASE(test_streaming_decoder_utf16_odd_byte) TEST_CASE(test_streaming_decoder_utf16_surrogate_pair_split) { - auto streaming_decoder = TextCodec::StreamingDecoder { "UTF-16BE"sv }; + auto streaming_decoder = TextCodec::StreamingDecoder { "UTF-16BE"sv, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement }; EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0x00, 0x41, 0xd8, 0x3d }))), "A"sv); EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0xde, 0x00 }))), "😀"sv); @@ -185,48 +185,48 @@ TEST_CASE(test_streaming_decoder_utf16_surrogate_pair_split) TEST_CASE(test_streaming_decoder_gb18030_four_byte_tail) { auto& decoder = decoder_for("gb18030"sv); - auto streaming_decoder = TextCodec::StreamingDecoder { "gb18030"sv }; + auto streaming_decoder = TextCodec::StreamingDecoder { "gb18030"sv, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement }; auto gb18030_sequence = Vector { 0x81, 0x30, 0x81, 0x30 }; EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 'a', 0x81, 0x30, 0x81 }))), "a"sv); - EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0x30 }))), MUST(decoder.to_utf8(StringView(bytes(gb18030_sequence))))); + EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0x30 }))), MUST(decoder.to_utf8(StringView(bytes(gb18030_sequence)), TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement))); EXPECT_EQ(MUST(streaming_decoder.finish()), ""sv); } TEST_CASE(test_streaming_decoder_big5_overlapping_trail) { auto& decoder = decoder_for("Big5"sv); - auto streaming_decoder = TextCodec::StreamingDecoder { "Big5"sv }; + auto streaming_decoder = TextCodec::StreamingDecoder { "Big5"sv, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement }; auto big5_sequence = Vector { 0xa4, 0xa4 }; - EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes(big5_sequence))), MUST(decoder.to_utf8(StringView(bytes(big5_sequence))))); + EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes(big5_sequence))), MUST(decoder.to_utf8(StringView(bytes(big5_sequence)), TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement))); EXPECT_EQ(MUST(streaming_decoder.finish()), ""sv); - auto streaming_decoder_for_split_input = TextCodec::StreamingDecoder { "Big5"sv }; + auto streaming_decoder_for_split_input = TextCodec::StreamingDecoder { "Big5"sv, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement }; EXPECT_EQ(MUST(streaming_decoder_for_split_input.to_utf8(bytes({ 0xa4 }))), ""sv); - EXPECT_EQ(MUST(streaming_decoder_for_split_input.to_utf8(bytes({ 0xa4 }))), MUST(decoder.to_utf8(StringView(bytes(big5_sequence))))); + EXPECT_EQ(MUST(streaming_decoder_for_split_input.to_utf8(bytes({ 0xa4 }))), MUST(decoder.to_utf8(StringView(bytes(big5_sequence)), TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement))); EXPECT_EQ(MUST(streaming_decoder_for_split_input.finish()), ""sv); } TEST_CASE(test_streaming_decoder_euc_jp_three_byte_tail) { auto& decoder = decoder_for("EUC-JP"sv); - auto streaming_decoder = TextCodec::StreamingDecoder { "EUC-JP"sv }; + auto streaming_decoder = TextCodec::StreamingDecoder { "EUC-JP"sv, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement }; auto euc_jp_sequence = Vector { 0x8f, 0xa2, 0xaf }; EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0x8f, 0xa2 }))), ""sv); - EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0xaf }))), MUST(decoder.to_utf8(StringView(bytes(euc_jp_sequence))))); + EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0xaf }))), MUST(decoder.to_utf8(StringView(bytes(euc_jp_sequence)), TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement))); EXPECT_EQ(MUST(streaming_decoder.finish()), ""sv); } TEST_CASE(test_streaming_decoder_shift_jis_tail) { auto& decoder = decoder_for("Shift_JIS"sv); - auto streaming_decoder = TextCodec::StreamingDecoder { "Shift_JIS"sv }; + auto streaming_decoder = TextCodec::StreamingDecoder { "Shift_JIS"sv, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement }; auto shift_jis_sequence = Vector { 0x82, 0xa0 }; EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0x82 }))), ""sv); - EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0xa0 }))), MUST(decoder.to_utf8(StringView(bytes(shift_jis_sequence))))); + EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0xa0 }))), MUST(decoder.to_utf8(StringView(bytes(shift_jis_sequence)), TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement))); EXPECT_EQ(MUST(streaming_decoder.finish()), ""sv); } @@ -237,7 +237,7 @@ TEST_CASE(test_utf16le_decode) auto test_string = "s\x00\xe4\x00k\x00=\xd8\x00\xde"sv; EXPECT(decoder.validate(test_string)); - auto utf8 = MUST(decoder.to_utf8(test_string)); + auto utf8 = MUST(decoder.to_utf8(test_string, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)); EXPECT_EQ(utf8, "säk😀"sv); } @@ -248,5 +248,5 @@ TEST_CASE(test_utf16le_process_code_points) EXPECT_EQ(process_code_points(decoder, StringView(bytes({ 0xff, 0xfe, 'A', 0x00, 0x3d, 0xd8, 0x00, 0xde }))), (Vector { 0x41, 0x1F600 })); EXPECT_EQ(process_code_points(decoder, StringView(bytes({ 0x3d, 0xd8, 'A', 0x00, 0x00, 0xde }))), (Vector { 0xfffd, 0x41, 0xfffd })); EXPECT_EQ(process_code_points(decoder, StringView(bytes({ 'A', 0x00, 0xff }))), (Vector { 0x41, 0xfffd })); - EXPECT_EQ(MUST(decoder.to_utf8(StringView(bytes({ 'A', 0x00, 0xff })))), "A\xef\xbf\xbd"sv); + EXPECT_EQ(MUST(decoder.to_utf8(StringView(bytes({ 'A', 0x00, 0xff })), TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)), "A\xef\xbf\xbd"sv); } diff --git a/Tests/LibWeb/Text/expected/wpt-import/encoding/streams/decode-ignore-bom.any.txt b/Tests/LibWeb/Text/expected/wpt-import/encoding/streams/decode-ignore-bom.any.txt index 6c337ae5ef..a56d83d7da 100644 --- a/Tests/LibWeb/Text/expected/wpt-import/encoding/streams/decode-ignore-bom.any.txt +++ b/Tests/LibWeb/Text/expected/wpt-import/encoding/streams/decode-ignore-bom.any.txt @@ -2,16 +2,16 @@ Harness status: OK Found 12 tests -12 Fail -Fail ignoreBOM should work for encoding utf-8, split at character 0 -Fail ignoreBOM should work for encoding utf-8, split at character 1 -Fail ignoreBOM should work for encoding utf-8, split at character 2 -Fail ignoreBOM should work for encoding utf-8, split at character 3 -Fail ignoreBOM should work for encoding utf-16le, split at character 0 -Fail ignoreBOM should work for encoding utf-16le, split at character 1 -Fail ignoreBOM should work for encoding utf-16le, split at character 2 -Fail ignoreBOM should work for encoding utf-16le, split at character 3 -Fail ignoreBOM should work for encoding utf-16be, split at character 0 -Fail ignoreBOM should work for encoding utf-16be, split at character 1 -Fail ignoreBOM should work for encoding utf-16be, split at character 2 -Fail ignoreBOM should work for encoding utf-16be, split at character 3 \ No newline at end of file +12 Pass +Pass ignoreBOM should work for encoding utf-8, split at character 0 +Pass ignoreBOM should work for encoding utf-8, split at character 1 +Pass ignoreBOM should work for encoding utf-8, split at character 2 +Pass ignoreBOM should work for encoding utf-8, split at character 3 +Pass ignoreBOM should work for encoding utf-16le, split at character 0 +Pass ignoreBOM should work for encoding utf-16le, split at character 1 +Pass ignoreBOM should work for encoding utf-16le, split at character 2 +Pass ignoreBOM should work for encoding utf-16le, split at character 3 +Pass ignoreBOM should work for encoding utf-16be, split at character 0 +Pass ignoreBOM should work for encoding utf-16be, split at character 1 +Pass ignoreBOM should work for encoding utf-16be, split at character 2 +Pass ignoreBOM should work for encoding utf-16be, split at character 3 diff --git a/Tests/LibWeb/Text/expected/wpt-import/encoding/textdecoder-ignorebom.any.txt b/Tests/LibWeb/Text/expected/wpt-import/encoding/textdecoder-ignorebom.any.txt index e119116e98..7cd9df9e45 100644 --- a/Tests/LibWeb/Text/expected/wpt-import/encoding/textdecoder-ignorebom.any.txt +++ b/Tests/LibWeb/Text/expected/wpt-import/encoding/textdecoder-ignorebom.any.txt @@ -2,9 +2,8 @@ Harness status: OK Found 4 tests -1 Pass -3 Fail -Fail BOM is ignored if ignoreBOM option is specified: utf-8 -Fail BOM is ignored if ignoreBOM option is specified: utf-16le -Fail BOM is ignored if ignoreBOM option is specified: utf-16be +4 Pass +Pass BOM is ignored if ignoreBOM option is specified: utf-8 +Pass BOM is ignored if ignoreBOM option is specified: utf-16le +Pass BOM is ignored if ignoreBOM option is specified: utf-16be Pass The ignoreBOM attribute of TextDecoder \ No newline at end of file diff --git a/Tests/LibWeb/Text/expected/wpt-import/encoding/textdecoder-mistakes.any.txt b/Tests/LibWeb/Text/expected/wpt-import/encoding/textdecoder-mistakes.any.txt index 78995c943d..7145328f56 100644 --- a/Tests/LibWeb/Text/expected/wpt-import/encoding/textdecoder-mistakes.any.txt +++ b/Tests/LibWeb/Text/expected/wpt-import/encoding/textdecoder-mistakes.any.txt @@ -2,8 +2,8 @@ Harness status: OK Found 87 tests -72 Pass -15 Fail +74 Pass +13 Fail Pass Invalid Unicode input is replaced: utf-8 Pass Invalid Unicode input is replaced: utf-16le Pass Invalid Unicode input is replaced: utf-16be @@ -51,8 +51,8 @@ Pass selected single-byte: iso-8859-8-i Pass selected single-byte: iso-8859-16 Pass selected single-byte: x-mac-cyrillic Pass Concatenating two ISO-2022-JP outputs is not always valid -Fail gb18030 version and ranges -Fail gbk version and ranges +Pass gb18030 version and ranges +Pass gbk version and ranges Pass gbk decoder is gb18030 decoder Pass Replacement, push back ASCII characters: big5 Pass Replacement, push back ASCII characters: iso-2022-jp