LibWeb+LibTextCodec: Wire decoder options through TextDecoder

Add explicit IgnoreBOM and ErrorMode options to LibTextCodec decoders,
and thread them through TextDecoder and TextDecoderStream.

This lets Web-facing decoder APIs preserve BOMs when requested and use
fatal error handling without post-processing decoded output.

NB: RemoveBOM was renamed to IgnoreBOM as "RemoveBOM" is the name
used by encoding_rs and was previously an implementation detail.
The new name matches what is used by the encoding standard as it
is now also used in LibWeb.

Libraries/LibTextCodec/Decoder.cpp

@@ -19,11 +19,6 @@ static constexpr u32 replacement_code_point = 0xfffd;
 
 namespace {
 
-enum class RemoveBOM {
-    No,
-    Yes,
-};
-
 class RustDecoder final : public Decoder {
 public:
     explicit RustDecoder(StringView encoding)
@@ -32,7 +27,7 @@ public:
     }
 
     virtual bool validate(StringView input) override;
-    virtual ErrorOr<String> to_utf8(StringView input) override;
+    virtual ErrorOr<String> to_utf8(StringView input, IgnoreBOM, ErrorMode) override;
     virtual ErrorOr<size_t> length_in_utf16_code_units(StringView input) override;
 
 private:
@@ -45,14 +40,14 @@ class UTF8Decoder final : public Decoder {
 public:
     virtual ErrorOr<void> process(StringView, Function<ErrorOr<void>(u32)> on_code_point) override;
     virtual bool validate(StringView) override;
-    virtual ErrorOr<String> to_utf8(StringView) override;
+    virtual ErrorOr<String> to_utf8(StringView, IgnoreBOM, ErrorMode) override;
     virtual ErrorOr<size_t> length_in_utf16_code_units(StringView) override;
 };
 
 class UTF16BEDecoder final : public Decoder {
 public:
     virtual bool validate(StringView) override;
-    virtual ErrorOr<String> to_utf8(StringView) override;
+    virtual ErrorOr<String> to_utf8(StringView, IgnoreBOM, ErrorMode) override;
     virtual ErrorOr<size_t> length_in_utf16_code_units(StringView) override;
 
 private:
@@ -62,7 +57,7 @@ private:
 class UTF16LEDecoder final : public Decoder {
 public:
     virtual bool validate(StringView) override;
-    virtual ErrorOr<String> to_utf8(StringView) override;
+    virtual ErrorOr<String> to_utf8(StringView, IgnoreBOM, ErrorMode) override;
     virtual ErrorOr<size_t> length_in_utf16_code_units(StringView) override;
 
 private:
@@ -130,7 +125,7 @@ static void append_decoded_bytes(void* context, u8 const* data, size_t length)
     decode_context.result = decode_context.builder.try_append(StringView { data, length });
 }
 
-ErrorOr<String> rust_decode_to_utf8(StringView encoding, StringView input, RemoveBOM remove_bom)
+ErrorOr<String> rust_decode_to_utf8(StringView encoding, StringView input, IgnoreBOM ignore_bom, ErrorMode error_mode)
 {
     DecodeContext context { .builder = StringBuilder(input.length()) };
     auto succeeded = FFI::textcodec_rust_decode_to_utf8(
@@ -138,36 +133,37 @@ ErrorOr<String> rust_decode_to_utf8(StringView encoding, StringView input, Remov
         encoding.length(),
         reinterpret_cast<u8 const*>(input.characters_without_null_termination()),
         input.length(),
-        remove_bom == RemoveBOM::Yes,
+        ignore_bom == IgnoreBOM::No,
+        error_mode == ErrorMode::Fatal,
         &context,
         append_decoded_bytes);
     if (!succeeded)
-        return Error::from_errno(EINVAL);
+        return Error::from_string_literal("Failed to decode input");
     TRY(context.result);
     return context.builder.to_string_without_validation();
 }
 
-ErrorOr<void> rust_process(StringView encoding, StringView input, RemoveBOM remove_bom, Function<ErrorOr<void>(u32)> on_code_point)
+ErrorOr<void> rust_process(StringView encoding, StringView input, IgnoreBOM ignore_bom, Function<ErrorOr<void>(u32)> on_code_point)
 {
-    auto utf8 = TRY(rust_decode_to_utf8(encoding, input, remove_bom));
+    auto utf8 = TRY(rust_decode_to_utf8(encoding, input, ignore_bom, ErrorMode::Replacement));
     for (auto code_point : Utf8View { utf8 })
         TRY(on_code_point(code_point));
     return {};
 }
 
-bool rust_validate(StringView encoding, StringView input, RemoveBOM remove_bom)
+bool rust_validate(StringView encoding, StringView input, IgnoreBOM ignore_bom)
 {
     return FFI::textcodec_rust_validate(
         reinterpret_cast<u8 const*>(encoding.characters_without_null_termination()),
         encoding.length(),
         reinterpret_cast<u8 const*>(input.characters_without_null_termination()),
         input.length(),
-        remove_bom == RemoveBOM::Yes);
+        ignore_bom == IgnoreBOM::No);
 }
 
-ErrorOr<size_t> rust_length_in_utf16_code_units(StringView encoding, StringView input, RemoveBOM remove_bom)
+ErrorOr<size_t> rust_length_in_utf16_code_units(StringView encoding, StringView input, IgnoreBOM ignore_bom)
 {
-    auto utf8 = TRY(rust_decode_to_utf8(encoding, input, remove_bom));
+    auto utf8 = TRY(rust_decode_to_utf8(encoding, input, ignore_bom, ErrorMode::Replacement));
     size_t length = 0;
     for (auto code_point : Utf8View { utf8 })
         length += code_point <= 0xffff ? 1 : 2;
@@ -188,7 +184,7 @@ Optional<StringView> get_static_encoding_name_from_rust(StringView label)
     return StringView { encoding_name, encoding_name_length };
 }
 
-ErrorOr<String> rust_streaming_decode_to_utf8(FFI::TextCodecRustStreamingDecoder* decoder, ReadonlyBytes input, bool last)
+ErrorOr<String> rust_streaming_decode_to_utf8(FFI::TextCodecRustStreamingDecoder* decoder, ReadonlyBytes input, bool last, ErrorMode error_mode)
 {
     DecodeContext context { .builder = StringBuilder(input.size()) };
     auto succeeded = FFI::textcodec_rust_streaming_decoder_decode_to_utf8(
@@ -196,10 +192,11 @@ ErrorOr<String> rust_streaming_decode_to_utf8(FFI::TextCodecRustStreamingDecoder
         input.data(),
         input.size(),
         last,
+        error_mode == ErrorMode::Fatal,
         &context,
         append_decoded_bytes);
     if (!succeeded)
-        return Error::from_errno(EINVAL);
+        return Error::from_string_literal("Failed to decode input");
     TRY(context.result);
     return context.builder.to_string_without_validation();
 }
@@ -367,7 +364,7 @@ ErrorOr<String> convert_input_to_utf8_using_given_decoder_unless_there_is_a_byte
 
     // 3. Process a queue with an instance of encoding’s decoder, ioQueue, output, and "replacement".
     // FIXME: This isn't the exact same as the spec, which is written in terms of I/O queues.
-    auto output = TRY(actual_decoder->to_utf8(input));
+    auto output = TRY(actual_decoder->to_utf8(input, IgnoreBOM::No, ErrorMode::Replacement));
 
     // 4. Return output.
     return output;
@@ -414,15 +411,18 @@ bool Decoder::validate(StringView input)
 {
     auto result = this->process(input, [](auto code_point) -> ErrorOr<void> {
         if (code_point == replacement_code_point)
-            return Error::from_errno(EINVAL);
+            return Error::from_string_literal("Decoded input contains replacement character");
         return {};
     });
 
     return !result.is_error();
 }
 
-ErrorOr<String> Decoder::to_utf8(StringView input)
+ErrorOr<String> Decoder::to_utf8(StringView input, IgnoreBOM, ErrorMode error_mode)
 {
+    if (error_mode == ErrorMode::Fatal && !validate(input))
+        return Error::from_string_literal("Failed to decode input");
+
     StringBuilder builder(input.length());
     TRY(process(input, [&builder](u32 c) { return builder.try_append_code_point(c); }));
     return builder.to_string_without_validation();
@@ -455,22 +455,22 @@ ErrorOr<void> Decoder::process_code_points(StringView input, Function<ErrorOr<vo
 
 bool RustDecoder::validate(StringView input)
 {
-    return rust_validate(m_encoding, input, RemoveBOM::No);
+    return rust_validate(m_encoding, input, IgnoreBOM::Yes);
 }
 
-ErrorOr<String> RustDecoder::to_utf8(StringView input)
+ErrorOr<String> RustDecoder::to_utf8(StringView input, IgnoreBOM ignore_bom, ErrorMode error_mode)
 {
-    return rust_decode_to_utf8(m_encoding, input, RemoveBOM::No);
+    return rust_decode_to_utf8(m_encoding, input, ignore_bom, error_mode);
 }
 
 ErrorOr<size_t> RustDecoder::length_in_utf16_code_units(StringView input)
 {
-    return rust_length_in_utf16_code_units(m_encoding, input, RemoveBOM::No);
+    return rust_length_in_utf16_code_units(m_encoding, input, IgnoreBOM::Yes);
 }
 
 ErrorOr<void> RustDecoder::process(StringView input, Function<ErrorOr<void>(u32)> on_code_point)
 {
-    return rust_process(m_encoding, input, RemoveBOM::No, move(on_code_point));
+    return rust_process(m_encoding, input, IgnoreBOM::Yes, move(on_code_point));
 }
 
 ErrorOr<void> Latin1Decoder::process(StringView input, Function<ErrorOr<void>(u32)> on_code_point)
@@ -485,12 +485,13 @@ ErrorOr<size_t> Latin1Decoder::length_in_utf16_code_units(StringView input)
     return input.length();
 }
 
-StreamingDecoder::StreamingDecoder(StringView encoding)
+StreamingDecoder::StreamingDecoder(StringView encoding, IgnoreBOM ignore_bom, ErrorMode error_mode)
+    : m_error_mode(error_mode)
 {
     m_decoder = FFI::textcodec_rust_streaming_decoder_new(
         reinterpret_cast<u8 const*>(encoding.characters_without_null_termination()),
         encoding.length(),
-        true);
+        ignore_bom == IgnoreBOM::No);
     VERIFY(m_decoder);
 }
 
@@ -501,72 +502,72 @@ StreamingDecoder::~StreamingDecoder()
 
 ErrorOr<String> StreamingDecoder::to_utf8(ReadonlyBytes input)
 {
-    return rust_streaming_decode_to_utf8(static_cast<FFI::TextCodecRustStreamingDecoder*>(m_decoder), input, false);
+    return rust_streaming_decode_to_utf8(static_cast<FFI::TextCodecRustStreamingDecoder*>(m_decoder), input, false, m_error_mode);
 }
 
 ErrorOr<String> StreamingDecoder::finish()
 {
-    return rust_streaming_decode_to_utf8(static_cast<FFI::TextCodecRustStreamingDecoder*>(m_decoder), {}, true);
+    return rust_streaming_decode_to_utf8(static_cast<FFI::TextCodecRustStreamingDecoder*>(m_decoder), {}, true, m_error_mode);
 }
 
 ErrorOr<void> UTF8Decoder::process(StringView input, Function<ErrorOr<void>(u32)> on_code_point)
 {
-    return rust_process("UTF-8"sv, input, RemoveBOM::No, move(on_code_point));
+    return rust_process("UTF-8"sv, input, IgnoreBOM::Yes, move(on_code_point));
 }
 
 bool UTF8Decoder::validate(StringView input)
 {
-    return rust_validate("UTF-8"sv, input, RemoveBOM::No);
+    return rust_validate("UTF-8"sv, input, IgnoreBOM::Yes);
 }
 
-ErrorOr<String> UTF8Decoder::to_utf8(StringView input)
+ErrorOr<String> UTF8Decoder::to_utf8(StringView input, IgnoreBOM ignore_bom, ErrorMode error_mode)
 {
-    return rust_decode_to_utf8("UTF-8"sv, input, RemoveBOM::Yes);
+    return rust_decode_to_utf8("UTF-8"sv, input, ignore_bom, error_mode);
 }
 
 ErrorOr<size_t> UTF8Decoder::length_in_utf16_code_units(StringView input)
 {
-    return rust_length_in_utf16_code_units("UTF-8"sv, input, RemoveBOM::Yes);
+    return rust_length_in_utf16_code_units("UTF-8"sv, input, IgnoreBOM::No);
 }
 
 bool UTF16BEDecoder::validate(StringView input)
 {
-    return rust_validate("UTF-16BE"sv, input, RemoveBOM::No);
+    return rust_validate("UTF-16BE"sv, input, IgnoreBOM::Yes);
 }
 
 ErrorOr<void> UTF16BEDecoder::process(StringView input, Function<ErrorOr<void>(u32)> on_code_point)
 {
-    return rust_process("UTF-16BE"sv, input, RemoveBOM::Yes, move(on_code_point));
+    return rust_process("UTF-16BE"sv, input, IgnoreBOM::No, move(on_code_point));
 }
 
-ErrorOr<String> UTF16BEDecoder::to_utf8(StringView input)
+ErrorOr<String> UTF16BEDecoder::to_utf8(StringView input, IgnoreBOM ignore_bom, ErrorMode error_mode)
 {
-    return rust_decode_to_utf8("UTF-16BE"sv, input, RemoveBOM::Yes);
+    return rust_decode_to_utf8("UTF-16BE"sv, input, ignore_bom, error_mode);
 }
 
 ErrorOr<size_t> UTF16BEDecoder::length_in_utf16_code_units(StringView input)
 {
-    return rust_length_in_utf16_code_units("UTF-16BE"sv, input, RemoveBOM::Yes);
+    return rust_length_in_utf16_code_units("UTF-16BE"sv, input, IgnoreBOM::No);
 }
 
 bool UTF16LEDecoder::validate(StringView input)
 {
-    return rust_validate("UTF-16LE"sv, input, RemoveBOM::No);
+    return rust_validate("UTF-16LE"sv, input, IgnoreBOM::Yes);
 }
 
 ErrorOr<void> UTF16LEDecoder::process(StringView input, Function<ErrorOr<void>(u32)> on_code_point)
 {
-    return rust_process("UTF-16LE"sv, input, RemoveBOM::Yes, move(on_code_point));
+    return rust_process("UTF-16LE"sv, input, IgnoreBOM::No, move(on_code_point));
 }
 
-ErrorOr<String> UTF16LEDecoder::to_utf8(StringView input)
+ErrorOr<String> UTF16LEDecoder::to_utf8(StringView input, IgnoreBOM ignore_bom, ErrorMode error_mode)
 {
-    return rust_decode_to_utf8("UTF-16LE"sv, input, RemoveBOM::Yes);
+    return rust_decode_to_utf8("UTF-16LE"sv, input, ignore_bom, error_mode);
 }
 
 ErrorOr<size_t> UTF16LEDecoder::length_in_utf16_code_units(StringView input)
 {
-    return rust_length_in_utf16_code_units("UTF-16LE"sv, input, RemoveBOM::Yes);
+    return rust_length_in_utf16_code_units("UTF-16LE"sv, input, IgnoreBOM::No);
 }
 
 // https://infra.spec.whatwg.org/#isomorphic-decode

Libraries/LibTextCodec/Decoder.h

@@ -19,10 +19,21 @@
 
 namespace TextCodec {
 
+enum class IgnoreBOM {
+    Yes,
+    No,
+};
+
+// https://encoding.spec.whatwg.org/#concept-encoding-error-mode
+enum class ErrorMode {
+    Replacement,
+    Fatal,
+};
+
 class TEXTCODEC_API Decoder {
 public:
     virtual bool validate(StringView);
-    virtual ErrorOr<String> to_utf8(StringView);
+    virtual ErrorOr<String> to_utf8(StringView, IgnoreBOM, ErrorMode);
     virtual ErrorOr<Utf16String> to_utf16(StringView);
     virtual ErrorOr<size_t> length_in_utf16_code_units(StringView);
     ErrorOr<void> process_code_points(StringView, Function<ErrorOr<void>(u32)>);
@@ -36,13 +47,14 @@ class TEXTCODEC_API StreamingDecoder final {
     AK_MAKE_NONCOPYABLE(StreamingDecoder);
 
 public:
-    explicit StreamingDecoder(StringView encoding);
+    StreamingDecoder(StringView encoding, IgnoreBOM, ErrorMode);
     ~StreamingDecoder();
 
     ErrorOr<String> to_utf8(ReadonlyBytes);
     ErrorOr<String> finish();
 
 private:
+    ErrorMode m_error_mode { ErrorMode::Replacement };
     void* m_decoder { nullptr };
 };
 

Libraries/LibTextCodec/Rust/src/lib.rs

@@ -93,6 +93,7 @@ pub unsafe extern "C" fn textcodec_rust_decode_to_utf8(
     input: *const u8,
     input_len: usize,
     remove_bom: bool,
+    fatal: bool,
     ctx: *mut c_void,
     on_bytes: FfiBytesFn,
 ) -> bool {
@@ -108,11 +109,14 @@ pub unsafe extern "C" fn textcodec_rust_decode_to_utf8(
                 return false;
             };
 
-            let (output, _) = if remove_bom {
+            let (output, had_errors) = if remove_bom {
                 encoding.decode_with_bom_removal(input)
             } else {
                 encoding.decode_without_bom_handling(input)
             };
+            if fatal && had_errors {
+                return false;
+            }
             on_bytes(ctx, output.as_bytes().as_ptr(), output.len());
             true
         })
@@ -212,6 +216,7 @@ pub unsafe extern "C" fn textcodec_rust_streaming_decoder_decode_to_utf8(
     input: *const u8,
     input_len: usize,
     last: bool,
+    fatal: bool,
     ctx: *mut c_void,
     on_bytes: FfiBytesFn,
 ) -> bool {
@@ -230,7 +235,10 @@ pub unsafe extern "C" fn textcodec_rust_streaming_decoder_decode_to_utf8(
             };
             let mut output = String::with_capacity(output_capacity);
 
-            let (result, _, _) = decoder.decoder.decode_to_string(input, &mut output, last);
+            let (result, _, had_errors) = decoder.decoder.decode_to_string(input, &mut output, last);
+            if fatal && had_errors {
+                return false;
+            }
             if !output.is_empty() {
                 on_bytes(ctx, output.as_ptr(), output.len());
             }

Libraries/LibWeb/CSS/Parser/RustTokenizer.cpp

@@ -36,7 +36,7 @@ static String decode_and_filter_code_points(StringView input, StringView encodin
                 input = input.substring_view(3);
             return String::from_utf8_without_validation(input.bytes());
         }
-        return MUST(decoder->to_utf8(input));
+        return MUST(decoder->to_utf8(input, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement));
     }();
 
     // OPTIMIZATION: If the input doesn't contain any filterable characters, we can skip the filtering

Libraries/LibWeb/CSS/Parser/Tokenizer.cpp

@@ -175,7 +175,7 @@ Vector<Token> Tokenizer::tokenize(StringView input, StringView encoding, Tokeniz
                     input = input.substring_view(3);
                 return String::from_utf8_without_validation(input.bytes());
             }
-            return MUST(decoder->to_utf8(input));
+            return MUST(decoder->to_utf8(input, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement));
         }();
 
         // OPTIMIZATION: If the input doesn't contain any filterable characters, we can skip the filtering

Libraries/LibWeb/DOM/DocumentLoading.cpp

@@ -77,7 +77,7 @@ bool build_xml_document(DOM::Document& document, ByteBuffer const& data, Optiona
         convert_to_xml_error_document(document, "XML Document contains improperly-encoded characters"_utf16);
         return false;
     }
-    auto source = decoder->to_utf8(data).release_value_but_fixme_should_propagate_errors();
+    auto source = decoder->to_utf8(data, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Fatal).release_value_but_fixme_should_propagate_errors();
     XML::Parser parser(source, { .resolve_named_html_entity = resolve_named_html_entity });
     XMLDocumentBuilder builder { document };
     auto result = parser.parse_with_listener(builder);
@@ -204,7 +204,7 @@ static WebIDL::ExceptionOr<GC::Ref<DOM::Document>> load_xml_document(HTML::Navig
             convert_to_xml_error_document(document, "XML Document contains improperly-encoded characters"_utf16);
             return;
         }
-        auto source = decoder->to_utf8(data);
+        auto source = decoder->to_utf8(data, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Fatal);
         if (source.is_error()) {
             // FIXME: Insert error message into the document.
             dbgln("Failed to decode XML document: {}", source.error());

Libraries/LibWeb/Encoding/TextDecoder.cpp

@@ -35,7 +35,7 @@ WebIDL::ExceptionOr<GC::Ref<TextDecoder>> TextDecoder::construct_impl(JS::Realm&
     auto lowercase_encoding_name = encoding.value().to_ascii_lowercase_string();
 
     // 4. If options["fatal"] is true, then set this’s error mode to "fatal".
-    auto error_mode = options.fatal ? ErrorMode::Fatal : ErrorMode::Replacement;
+    auto error_mode = options.fatal ? TextCodec::ErrorMode::Fatal : TextCodec::ErrorMode::Replacement;
 
     // 5. Set this’s ignore BOM to options["ignoreBOM"].
     auto ignore_bom = options.ignore_bom;
@@ -48,7 +48,7 @@ WebIDL::ExceptionOr<GC::Ref<TextDecoder>> TextDecoder::construct_impl(JS::Realm&
 }
 
 // https://encoding.spec.whatwg.org/#dom-textdecoder
-TextDecoder::TextDecoder(JS::Realm& realm, TextCodec::Decoder& decoder, FlyString encoding, ErrorMode error_mode, bool ignore_bom)
+TextDecoder::TextDecoder(JS::Realm& realm, TextCodec::Decoder& decoder, FlyString encoding, TextCodec::ErrorMode error_mode, bool ignore_bom)
     : PlatformObject(realm)
     , TextDecoderCommonMixin(decoder, move(encoding), error_mode, ignore_bom)
 {
@@ -65,18 +65,23 @@ void TextDecoder::initialize(JS::Realm& realm)
 // https://encoding.spec.whatwg.org/#dom-textdecoder-decode
 WebIDL::ExceptionOr<String> TextDecoder::decode(Optional<WebIDL::BufferSourceVariant> input, Optional<Bindings::TextDecodeOptions> const&) const
 {
-    if (!input.has_value())
+    auto ignore_bom = m_ignore_bom ? TextCodec::IgnoreBOM::Yes : TextCodec::IgnoreBOM::No;
-        return TRY_OR_THROW_OOM(vm(), m_decoder.to_utf8({}));
+    if (!input.has_value()) {
+        auto result = m_decoder.to_utf8({}, ignore_bom, m_error_mode);
+        if (result.is_error() && result.error().code() != ENOMEM)
+            return WebIDL::SimpleException { WebIDL::SimpleExceptionType::TypeError, "Decoding failed"sv };
+        return TRY_OR_THROW_OOM(vm(), move(result));
+    }
 
     // FIXME: Implement the streaming stuff.
     auto data_buffer_or_error = WebIDL::get_buffer_source_copy(*input);
     if (data_buffer_or_error.is_error())
         return WebIDL::OperationError::create(realm(), "Failed to copy bytes from ArrayBuffer"_utf16);
     auto& data_buffer = data_buffer_or_error.value();
-    auto result = TRY_OR_THROW_OOM(vm(), m_decoder.to_utf8({ data_buffer.data(), data_buffer.size() }));
+    auto result = m_decoder.to_utf8({ data_buffer.data(), data_buffer.size() }, ignore_bom, m_error_mode);
-    if (this->fatal() && result.contains(0xfffd))
+    if (result.is_error() && result.error().code() != ENOMEM)
         return WebIDL::SimpleException { WebIDL::SimpleExceptionType::TypeError, "Decoding failed"sv };
-    return result;
+    return TRY_OR_THROW_OOM(vm(), move(result));
 }
 
 }

Libraries/LibWeb/Encoding/TextDecoder.h

@@ -34,7 +34,7 @@ public:
     WebIDL::ExceptionOr<String> decode(Optional<WebIDL::BufferSourceVariant>, Optional<Bindings::TextDecodeOptions> const& options = {}) const;
 
 private:
-    TextDecoder(JS::Realm&, TextCodec::Decoder&, FlyString encoding, ErrorMode error_mode, bool ignore_bom);
+    TextDecoder(JS::Realm&, TextCodec::Decoder&, FlyString encoding, TextCodec::ErrorMode error_mode, bool ignore_bom);
 
     virtual void initialize(JS::Realm&) override;
 };

Libraries/LibWeb/Encoding/TextDecoderCommon.cpp

@@ -8,7 +8,7 @@
 
 namespace Web::Encoding {
 
-TextDecoderCommonMixin::TextDecoderCommonMixin(TextCodec::Decoder& decoder, FlyString encoding, ErrorMode error_mode, bool ignore_bom)
+TextDecoderCommonMixin::TextDecoderCommonMixin(TextCodec::Decoder& decoder, FlyString encoding, TextCodec::ErrorMode error_mode, bool ignore_bom)
     : m_decoder(decoder)
     , m_encoding(move(encoding))
     , m_error_mode(error_mode)

Libraries/LibWeb/Encoding/TextDecoderCommon.h

@@ -7,7 +7,7 @@
 #pragma once
 
 #include <AK/FlyString.h>
-#include <LibTextCodec/Forward.h>
+#include <LibTextCodec/Decoder.h>
 
 namespace Web::Encoding {
 
@@ -20,19 +20,13 @@ public:
     FlyString const& encoding() const { return m_encoding; }
 
     // https://encoding.spec.whatwg.org/#dom-textdecoder-fatal
-    bool fatal() const { return m_error_mode == ErrorMode::Fatal; }
+    bool fatal() const { return m_error_mode == TextCodec::ErrorMode::Fatal; }
 
     // https://encoding.spec.whatwg.org/#dom-textdecoder-ignorebom
     bool ignore_bom() const { return m_ignore_bom; }
 
 protected:
-    // https://encoding.spec.whatwg.org/#concept-encoding-error-mode
+    TextDecoderCommonMixin(TextCodec::Decoder& decoder, FlyString encoding, TextCodec::ErrorMode error_mode, bool ignore_bom);
-    enum class ErrorMode {
-        Replacement,
-        Fatal,
-    };
-
-    TextDecoderCommonMixin(TextCodec::Decoder& decoder, FlyString encoding, ErrorMode error_mode, bool ignore_bom);
 
     // https://encoding.spec.whatwg.org/#textdecodercommon-decoder
     TextCodec::Decoder& m_decoder;
@@ -41,13 +35,10 @@ protected:
     FlyString m_encoding;
 
     // https://encoding.spec.whatwg.org/#textdecoder-error-mode
-    ErrorMode m_error_mode { ErrorMode::Replacement };
+    TextCodec::ErrorMode m_error_mode { TextCodec::ErrorMode::Replacement };
 
     // https://encoding.spec.whatwg.org/#textdecoder-ignore-bom-flag
     bool m_ignore_bom { false };
-
-    // https://encoding.spec.whatwg.org/#textdecoder-bom-seen-flag
-    bool m_bom_seen { false };
 };
 
 }

Libraries/LibWeb/Encoding/TextDecoderStream.cpp

@@ -39,7 +39,7 @@ WebIDL::ExceptionOr<GC::Ref<TextDecoderStream>> TextDecoderStream::construct_imp
     auto lowercase_encoding_name = encoding.value().to_ascii_lowercase_string();
 
     // 4. If options["fatal"] is true, then set this’s error mode to "fatal".
-    auto error_mode = options.fatal ? ErrorMode::Fatal : ErrorMode::Replacement;
+    auto error_mode = options.fatal ? TextCodec::ErrorMode::Fatal : TextCodec::ErrorMode::Replacement;
 
     // 5. Set this’s ignore BOM to options["ignoreBOM"].
     auto ignore_bom = options.ignore_bom;
@@ -84,11 +84,14 @@ WebIDL::ExceptionOr<GC::Ref<TextDecoderStream>> TextDecoderStream::construct_imp
     return stream;
 }
 
-TextDecoderStream::TextDecoderStream(JS::Realm& realm, GC::Ref<Streams::TransformStream> transform, TextCodec::Decoder& decoder, FlyString encoding, ErrorMode error_mode, bool ignore_bom)
+TextDecoderStream::TextDecoderStream(JS::Realm& realm, GC::Ref<Streams::TransformStream> transform, TextCodec::Decoder& decoder, FlyString encoding, TextCodec::ErrorMode error_mode, bool ignore_bom)
     : Bindings::PlatformObject(realm)
     , Streams::GenericTransformStreamMixin(transform)
     , TextDecoderCommonMixin(decoder, move(encoding), error_mode, ignore_bom)
-    , m_streaming_decoder(make<TextCodec::StreamingDecoder>(m_encoding))
+    , m_streaming_decoder(make<TextCodec::StreamingDecoder>(
+          m_encoding,
+          m_ignore_bom ? TextCodec::IgnoreBOM::Yes : TextCodec::IgnoreBOM::No,
+          m_error_mode))
 {
 }
 
@@ -122,7 +125,10 @@ WebIDL::ExceptionOr<void> TextDecoderStream::decode_and_enqueue_chunk(JS::Value
         return WebIDL::OperationError::create(realm, "Failed to copy bytes from BufferSource"_utf16);
     auto buffer = buffer_or_error.release_value();
 
-    auto decoded = TRY_OR_THROW_OOM(vm, m_streaming_decoder->to_utf8(buffer.bytes()));
+    auto decoded_or_error = m_streaming_decoder->to_utf8(buffer.bytes());
+    if (decoded_or_error.is_error() && decoded_or_error.error().code() != ENOMEM)
+        return WebIDL::SimpleException { WebIDL::SimpleExceptionType::TypeError, "Decoding failed"sv };
+    auto decoded = TRY_OR_THROW_OOM(vm, move(decoded_or_error));
 
     // 3-4. Run "processing an item" until the input is exhausted, accumulating the output, then enqueue any non-empty
     //      result. If processing returns error, throw a TypeError.
@@ -133,7 +139,10 @@ WebIDL::ExceptionOr<void> TextDecoderStream::decode_and_enqueue_chunk(JS::Value
 WebIDL::ExceptionOr<void> TextDecoderStream::flush_and_enqueue()
 {
     // 1-3. Drain decoder's I/O queue and run "processing an item" to completion.
-    auto decoded = TRY_OR_THROW_OOM(vm(), m_streaming_decoder->finish());
+    auto decoded_or_error = m_streaming_decoder->finish();
+    if (decoded_or_error.is_error() && decoded_or_error.error().code() != ENOMEM)
+        return WebIDL::SimpleException { WebIDL::SimpleExceptionType::TypeError, "Decoding failed"sv };
+    auto decoded = TRY_OR_THROW_OOM(vm(), move(decoded_or_error));
     return enqueue_decoded_output(decoded);
 }
 
@@ -142,22 +151,9 @@ WebIDL::ExceptionOr<void> TextDecoderStream::enqueue_decoded_output(String const
     auto& realm = this->realm();
     auto& vm = realm.vm();
 
-    // https://encoding.spec.whatwg.org/#concept-td-serialize
-    // FIXME: The underlying TextCodec decoders currently strip leading BOMs unconditionally for UTF-8 and UTF-16BE/LE,
-    //        so the "ignore BOM" flag is effectively ignored here. Once the decoders accept a "preserve BOM" mode,
-    //        plumb m_ignore_bom through and strip the BOM from `decoded` only when m_ignore_bom is false.
-    if (!m_bom_seen && !decoded.is_empty())
-        m_bom_seen = true;
-
     if (decoded.is_empty())
         return {};
 
-    // If decoder's error mode is "fatal" and processing produced any error, throw a TypeError.
-    // NB: We can only detect this approximately by looking for U+FFFD in the decoded output, which the underlying
-    //     decoder substitutes for invalid sequences. This matches the existing TextDecoder.decode() behavior.
-    if (fatal() && decoded.contains(0xFFFD))
-        return WebIDL::SimpleException { WebIDL::SimpleExceptionType::TypeError, "Decoding failed"sv };
-
     auto js_string = JS::PrimitiveString::create(vm, Utf16String::from_utf8(decoded));
     return Streams::transform_stream_default_controller_enqueue(*m_transform->controller(), js_string);
 }

Libraries/LibWeb/Encoding/TextDecoderStream.h

@@ -27,7 +27,7 @@ public:
     virtual ~TextDecoderStream() override;
 
 private:
-    TextDecoderStream(JS::Realm&, GC::Ref<Streams::TransformStream>, TextCodec::Decoder&, FlyString encoding, ErrorMode, bool ignore_bom);
+    TextDecoderStream(JS::Realm&, GC::Ref<Streams::TransformStream>, TextCodec::Decoder&, FlyString encoding, TextCodec::ErrorMode, bool ignore_bom);
 
     virtual void initialize(JS::Realm&) override;
     virtual void visit_edges(Cell::Visitor&) override;

Libraries/LibWeb/HTML/Parser/HTMLTokenizer.cpp

@@ -108,7 +108,7 @@ HTMLTokenizer::HTMLTokenizer(StringView input, ByteString const& encoding, Input
     if (input_type == InputType::EncodedBytes) {
         auto decoder = TextCodec::decoder_for(encoding);
         VERIFY(decoder.has_value());
-        m_source = MUST(decoder->to_utf8(input));
+        m_source = MUST(decoder->to_utf8(input, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement));
     } else {
         m_source = decoded_string_for_utf8_tokenizer(input);
     }

Libraries/LibWeb/HTML/Parser/IncrementalDocumentParser.cpp

@@ -68,7 +68,7 @@ void IncrementalDocumentParser::initialize_parser(ReadonlyBytes sniff_bytes)
 
     auto standardized_encoding = TextCodec::get_standardized_encoding(encoding);
     VERIFY(standardized_encoding.has_value());
-    m_decoder = make<TextCodec::StreamingDecoder>(standardized_encoding.value());
+    m_decoder = make<TextCodec::StreamingDecoder>(standardized_encoding.value(), TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement);
 
     // https://html.spec.whatwg.org/multipage/parsing.html#determining-the-character-encoding
     // The document's character encoding must immediately be set to the value returned from this

Libraries/LibWebView/Autocomplete.cpp

@@ -438,7 +438,7 @@ ErrorOr<Vector<String>> Autocomplete::received_autocomplete_respsonse(Autocomple
     if (!decoder.has_value())
         decoder = TextCodec::decoder_for_exact_name("UTF-8"sv);
 
-    auto decoded_response = TRY(decoder->to_utf8(response));
+    auto decoded_response = TRY(decoder->to_utf8(response, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement));
     auto json = TRY(JsonValue::from_string(decoded_response));
 
     if (engine.name == "DuckDuckGo")

Meta/Fuzzers/FuzzTextDecoder.cpp

@@ -24,6 +24,6 @@ extern "C" int LLVMFuzzerTestOneInput(uint8_t const* data, size_t size)
     if (!decoder.has_value())
         return 0;
 
-    (void)decoder->to_utf8(encoded_data);
+    (void)decoder->to_utf8(encoded_data, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement);
     return 0;
 }

Services/RequestServer/Request.cpp

@@ -1085,7 +1085,7 @@ size_t Request::on_header_received(void* buffer, size_t size, size_t nmemb, void
                 auto decoder = TextCodec::decoder_for_exact_name("ISO-8859-1"sv);
                 VERIFY(decoder.has_value());
 
-                request.m_reason_phrase = MUST(decoder->to_utf8(reason_phrase));
+                request.m_reason_phrase = MUST(decoder->to_utf8(reason_phrase, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement));
                 return total_size;
             }
         }

Tests/LibTextCodec/TestTextDecoders.cpp

@@ -43,7 +43,7 @@ TEST_CASE(test_utf8_decode)
     EXPECT(processed_code_points.size() == 1);
     EXPECT(processed_code_points[0] == 0x1F600);
 
-    EXPECT(MUST(decoder.to_utf8(test_string)) == test_string);
+    EXPECT(MUST(decoder.to_utf8(test_string, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)) == test_string);
 }
 
 TEST_CASE(test_utf8_process_code_points)
@@ -61,7 +61,7 @@ TEST_CASE(test_utf8_process_code_points_replaces_surrogates)
     auto utf8_encoded_surrogate = StringView(bytes(utf8_encoded_surrogate_bytes));
 
     EXPECT(!decoder.validate(utf8_encoded_surrogate));
-    EXPECT_EQ(MUST(decoder.to_utf8(utf8_encoded_surrogate)), "\xef\xbf\xbd\xef\xbf\xbd\xef\xbf\xbd"sv);
+    EXPECT_EQ(MUST(decoder.to_utf8(utf8_encoded_surrogate, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)), "\xef\xbf\xbd\xef\xbf\xbd\xef\xbf\xbd"sv);
     EXPECT_EQ(process_code_points(decoder, utf8_encoded_surrogate), (Vector<u32> { 0xfffd, 0xfffd, 0xfffd }));
 }
 
@@ -72,7 +72,7 @@ TEST_CASE(test_utf8_process_code_points_replaces_truncated_tail_as_single_error)
     auto truncated_tail = StringView(bytes(truncated_tail_bytes));
 
     EXPECT(!decoder.validate(truncated_tail));
-    EXPECT_EQ(MUST(decoder.to_utf8(truncated_tail)), "\xef\xbf\xbd"sv);
+    EXPECT_EQ(MUST(decoder.to_utf8(truncated_tail, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)), "\xef\xbf\xbd"sv);
     EXPECT_EQ(process_code_points(decoder, truncated_tail), (Vector<u32> { 0xfffd }));
 }
 
@@ -83,7 +83,7 @@ TEST_CASE(test_utf8_process_code_points_replaces_overlong_sequences)
     auto overlong_null = StringView(bytes(overlong_null_bytes));
 
     EXPECT(!decoder.validate(overlong_null));
-    EXPECT_EQ(MUST(decoder.to_utf8(overlong_null)), "\xef\xbf\xbd\xef\xbf\xbd"sv);
+    EXPECT_EQ(MUST(decoder.to_utf8(overlong_null, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)), "\xef\xbf\xbd\xef\xbf\xbd"sv);
     EXPECT_EQ(process_code_points(decoder, overlong_null), (Vector<u32> { 0xfffd, 0xfffd }));
 }
 
@@ -96,11 +96,11 @@ TEST_CASE(test_utf8_process_code_points_restores_invalid_second_byte)
     auto out_of_range_four_byte_sequence = StringView(bytes(out_of_range_four_byte_sequence_bytes));
 
     EXPECT(!decoder.validate(overlong_three_byte_sequence));
-    EXPECT_EQ(MUST(decoder.to_utf8(overlong_three_byte_sequence)), "\xef\xbf\xbd\xef\xbf\xbd\xef\xbf\xbd"sv);
+    EXPECT_EQ(MUST(decoder.to_utf8(overlong_three_byte_sequence, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)), "\xef\xbf\xbd\xef\xbf\xbd\xef\xbf\xbd"sv);
     EXPECT_EQ(process_code_points(decoder, overlong_three_byte_sequence), (Vector<u32> { 0xfffd, 0xfffd, 0xfffd }));
 
     EXPECT(!decoder.validate(out_of_range_four_byte_sequence));
-    EXPECT_EQ(MUST(decoder.to_utf8(out_of_range_four_byte_sequence)), "\xef\xbf\xbd\xef\xbf\xbd\xef\xbf\xbd\xef\xbf\xbd"sv);
+    EXPECT_EQ(MUST(decoder.to_utf8(out_of_range_four_byte_sequence, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)), "\xef\xbf\xbd\xef\xbf\xbd\xef\xbf\xbd\xef\xbf\xbd"sv);
     EXPECT_EQ(process_code_points(decoder, out_of_range_four_byte_sequence), (Vector<u32> { 0xfffd, 0xfffd, 0xfffd, 0xfffd }));
 }
 
@@ -111,7 +111,7 @@ TEST_CASE(test_utf16be_decode)
     auto test_string = "\x00s\x00\xe4\x00k\xd8=\xde\x00"sv;
 
     EXPECT(decoder.validate(test_string));
-    auto utf8 = MUST(decoder.to_utf8(test_string));
+    auto utf8 = MUST(decoder.to_utf8(test_string, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement));
     EXPECT_EQ(utf8, "säk😀"sv);
 }
 
@@ -122,12 +122,12 @@ TEST_CASE(test_utf16be_process_code_points)
     EXPECT_EQ(process_code_points(decoder, StringView(bytes({ 0xfe, 0xff, 0x00, 'A', 0xd8, 0x3d, 0xde, 0x00 }))), (Vector<u32> { 0x41, 0x1F600 }));
     EXPECT_EQ(process_code_points(decoder, StringView(bytes({ 0xd8, 0x3d, 0x00, 'A', 0xde, 0x00 }))), (Vector<u32> { 0xfffd, 0x41, 0xfffd }));
     EXPECT_EQ(process_code_points(decoder, StringView(bytes({ 0x00, 'A', 0xff }))), (Vector<u32> { 0x41, 0xfffd }));
-    EXPECT_EQ(MUST(decoder.to_utf8(StringView(bytes({ 0x00, 'A', 0xff })))), "A\xef\xbf\xbd"sv);
+    EXPECT_EQ(MUST(decoder.to_utf8(StringView(bytes({ 0x00, 'A', 0xff })), TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)), "A\xef\xbf\xbd"sv);
 }
 
 TEST_CASE(test_streaming_decoder_utf8_mid_sequence)
 {
-    auto streaming_decoder = TextCodec::StreamingDecoder { "UTF-8"sv };
+    auto streaming_decoder = TextCodec::StreamingDecoder { "UTF-8"sv, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement };
 
     EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 'a', 0xc3 }))), "a"sv);
     EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0xa9, 'b' }))), "éb"sv);
@@ -137,27 +137,27 @@ TEST_CASE(test_streaming_decoder_utf8_mid_sequence)
 TEST_CASE(test_streaming_decoder_finishes_incomplete_sequence)
 {
     auto& decoder = decoder_for("UTF-8"sv);
-    auto streaming_decoder = TextCodec::StreamingDecoder { "UTF-8"sv };
+    auto streaming_decoder = TextCodec::StreamingDecoder { "UTF-8"sv, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement };
 
     auto incomplete_sequence = Vector<u8> { 0xc3 };
     EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes(incomplete_sequence))), ""sv);
-    EXPECT_EQ(MUST(streaming_decoder.finish()), MUST(decoder.to_utf8(StringView(bytes(incomplete_sequence)))));
+    EXPECT_EQ(MUST(streaming_decoder.finish()), MUST(decoder.to_utf8(StringView(bytes(incomplete_sequence)), TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)));
 }
 
 TEST_CASE(test_streaming_decoder_utf8_invalid_second_byte_tail)
 {
-    auto streaming_decoder_for_overlong_three_byte_sequence = TextCodec::StreamingDecoder { "UTF-8"sv };
+    auto streaming_decoder_for_overlong_three_byte_sequence = TextCodec::StreamingDecoder { "UTF-8"sv, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement };
     EXPECT_EQ(MUST(streaming_decoder_for_overlong_three_byte_sequence.to_utf8(bytes({ 0xe0, 0x80 }))), "\xef\xbf\xbd\xef\xbf\xbd"sv);
     EXPECT_EQ(MUST(streaming_decoder_for_overlong_three_byte_sequence.finish()), ""sv);
 
-    auto streaming_decoder_for_out_of_range_four_byte_sequence = TextCodec::StreamingDecoder { "UTF-8"sv };
+    auto streaming_decoder_for_out_of_range_four_byte_sequence = TextCodec::StreamingDecoder { "UTF-8"sv, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement };
     EXPECT_EQ(MUST(streaming_decoder_for_out_of_range_four_byte_sequence.to_utf8(bytes({ 0xf4, 0x90 }))), "\xef\xbf\xbd\xef\xbf\xbd"sv);
     EXPECT_EQ(MUST(streaming_decoder_for_out_of_range_four_byte_sequence.finish()), ""sv);
 }
 
 TEST_CASE(test_streaming_decoder_utf8_valid_second_byte_tail)
 {
-    auto streaming_decoder = TextCodec::StreamingDecoder { "UTF-8"sv };
+    auto streaming_decoder = TextCodec::StreamingDecoder { "UTF-8"sv, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement };
 
     EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0xe0, 0xa0 }))), ""sv);
     EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0x80 }))), "\xe0\xa0\x80"sv);
@@ -166,7 +166,7 @@ TEST_CASE(test_streaming_decoder_utf8_valid_second_byte_tail)
 
 TEST_CASE(test_streaming_decoder_utf16_odd_byte)
 {
-    auto streaming_decoder = TextCodec::StreamingDecoder { "UTF-16LE"sv };
+    auto streaming_decoder = TextCodec::StreamingDecoder { "UTF-16LE"sv, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement };
 
     EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0x41, 0x00, 0x42 }))), "A"sv);
     EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0x00 }))), "B"sv);
@@ -175,7 +175,7 @@ TEST_CASE(test_streaming_decoder_utf16_odd_byte)
 
 TEST_CASE(test_streaming_decoder_utf16_surrogate_pair_split)
 {
-    auto streaming_decoder = TextCodec::StreamingDecoder { "UTF-16BE"sv };
+    auto streaming_decoder = TextCodec::StreamingDecoder { "UTF-16BE"sv, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement };
 
     EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0x00, 0x41, 0xd8, 0x3d }))), "A"sv);
     EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0xde, 0x00 }))), "😀"sv);
@@ -185,48 +185,48 @@ TEST_CASE(test_streaming_decoder_utf16_surrogate_pair_split)
 TEST_CASE(test_streaming_decoder_gb18030_four_byte_tail)
 {
     auto& decoder = decoder_for("gb18030"sv);
-    auto streaming_decoder = TextCodec::StreamingDecoder { "gb18030"sv };
+    auto streaming_decoder = TextCodec::StreamingDecoder { "gb18030"sv, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement };
     auto gb18030_sequence = Vector<u8> { 0x81, 0x30, 0x81, 0x30 };
 
     EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 'a', 0x81, 0x30, 0x81 }))), "a"sv);
-    EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0x30 }))), MUST(decoder.to_utf8(StringView(bytes(gb18030_sequence)))));
+    EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0x30 }))), MUST(decoder.to_utf8(StringView(bytes(gb18030_sequence)), TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)));
     EXPECT_EQ(MUST(streaming_decoder.finish()), ""sv);
 }
 
 TEST_CASE(test_streaming_decoder_big5_overlapping_trail)
 {
     auto& decoder = decoder_for("Big5"sv);
-    auto streaming_decoder = TextCodec::StreamingDecoder { "Big5"sv };
+    auto streaming_decoder = TextCodec::StreamingDecoder { "Big5"sv, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement };
     auto big5_sequence = Vector<u8> { 0xa4, 0xa4 };
 
-    EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes(big5_sequence))), MUST(decoder.to_utf8(StringView(bytes(big5_sequence)))));
+    EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes(big5_sequence))), MUST(decoder.to_utf8(StringView(bytes(big5_sequence)), TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)));
     EXPECT_EQ(MUST(streaming_decoder.finish()), ""sv);
 
-    auto streaming_decoder_for_split_input = TextCodec::StreamingDecoder { "Big5"sv };
+    auto streaming_decoder_for_split_input = TextCodec::StreamingDecoder { "Big5"sv, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement };
     EXPECT_EQ(MUST(streaming_decoder_for_split_input.to_utf8(bytes({ 0xa4 }))), ""sv);
-    EXPECT_EQ(MUST(streaming_decoder_for_split_input.to_utf8(bytes({ 0xa4 }))), MUST(decoder.to_utf8(StringView(bytes(big5_sequence)))));
+    EXPECT_EQ(MUST(streaming_decoder_for_split_input.to_utf8(bytes({ 0xa4 }))), MUST(decoder.to_utf8(StringView(bytes(big5_sequence)), TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)));
     EXPECT_EQ(MUST(streaming_decoder_for_split_input.finish()), ""sv);
 }
 
 TEST_CASE(test_streaming_decoder_euc_jp_three_byte_tail)
 {
     auto& decoder = decoder_for("EUC-JP"sv);
-    auto streaming_decoder = TextCodec::StreamingDecoder { "EUC-JP"sv };
+    auto streaming_decoder = TextCodec::StreamingDecoder { "EUC-JP"sv, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement };
     auto euc_jp_sequence = Vector<u8> { 0x8f, 0xa2, 0xaf };
 
     EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0x8f, 0xa2 }))), ""sv);
-    EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0xaf }))), MUST(decoder.to_utf8(StringView(bytes(euc_jp_sequence)))));
+    EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0xaf }))), MUST(decoder.to_utf8(StringView(bytes(euc_jp_sequence)), TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)));
     EXPECT_EQ(MUST(streaming_decoder.finish()), ""sv);
 }
 
 TEST_CASE(test_streaming_decoder_shift_jis_tail)
 {
     auto& decoder = decoder_for("Shift_JIS"sv);
-    auto streaming_decoder = TextCodec::StreamingDecoder { "Shift_JIS"sv };
+    auto streaming_decoder = TextCodec::StreamingDecoder { "Shift_JIS"sv, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement };
     auto shift_jis_sequence = Vector<u8> { 0x82, 0xa0 };
 
     EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0x82 }))), ""sv);
-    EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0xa0 }))), MUST(decoder.to_utf8(StringView(bytes(shift_jis_sequence)))));
+    EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0xa0 }))), MUST(decoder.to_utf8(StringView(bytes(shift_jis_sequence)), TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)));
     EXPECT_EQ(MUST(streaming_decoder.finish()), ""sv);
 }
 
@@ -237,7 +237,7 @@ TEST_CASE(test_utf16le_decode)
     auto test_string = "s\x00\xe4\x00k\x00=\xd8\x00\xde"sv;
 
     EXPECT(decoder.validate(test_string));
-    auto utf8 = MUST(decoder.to_utf8(test_string));
+    auto utf8 = MUST(decoder.to_utf8(test_string, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement));
     EXPECT_EQ(utf8, "säk😀"sv);
 }
 
@@ -248,5 +248,5 @@ TEST_CASE(test_utf16le_process_code_points)
     EXPECT_EQ(process_code_points(decoder, StringView(bytes({ 0xff, 0xfe, 'A', 0x00, 0x3d, 0xd8, 0x00, 0xde }))), (Vector<u32> { 0x41, 0x1F600 }));
     EXPECT_EQ(process_code_points(decoder, StringView(bytes({ 0x3d, 0xd8, 'A', 0x00, 0x00, 0xde }))), (Vector<u32> { 0xfffd, 0x41, 0xfffd }));
     EXPECT_EQ(process_code_points(decoder, StringView(bytes({ 'A', 0x00, 0xff }))), (Vector<u32> { 0x41, 0xfffd }));
-    EXPECT_EQ(MUST(decoder.to_utf8(StringView(bytes({ 'A', 0x00, 0xff })))), "A\xef\xbf\xbd"sv);
+    EXPECT_EQ(MUST(decoder.to_utf8(StringView(bytes({ 'A', 0x00, 0xff })), TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)), "A\xef\xbf\xbd"sv);
 }

Tests/LibWeb/Text/expected/wpt-import/encoding/streams/decode-ignore-bom.any.txt

@@ -2,16 +2,16 @@ Harness status: OK
 
 Found 12 tests
 
-12 Fail
+12 Pass
-Fail	ignoreBOM should work for encoding utf-8, split at character 0
+Pass	ignoreBOM should work for encoding utf-8, split at character 0
-Fail	ignoreBOM should work for encoding utf-8, split at character 1
+Pass	ignoreBOM should work for encoding utf-8, split at character 1
-Fail	ignoreBOM should work for encoding utf-8, split at character 2
+Pass	ignoreBOM should work for encoding utf-8, split at character 2
-Fail	ignoreBOM should work for encoding utf-8, split at character 3
+Pass	ignoreBOM should work for encoding utf-8, split at character 3
-Fail	ignoreBOM should work for encoding utf-16le, split at character 0
+Pass	ignoreBOM should work for encoding utf-16le, split at character 0
-Fail	ignoreBOM should work for encoding utf-16le, split at character 1
+Pass	ignoreBOM should work for encoding utf-16le, split at character 1
-Fail	ignoreBOM should work for encoding utf-16le, split at character 2
+Pass	ignoreBOM should work for encoding utf-16le, split at character 2
-Fail	ignoreBOM should work for encoding utf-16le, split at character 3
+Pass	ignoreBOM should work for encoding utf-16le, split at character 3
-Fail	ignoreBOM should work for encoding utf-16be, split at character 0
+Pass	ignoreBOM should work for encoding utf-16be, split at character 0
-Fail	ignoreBOM should work for encoding utf-16be, split at character 1
+Pass	ignoreBOM should work for encoding utf-16be, split at character 1
-Fail	ignoreBOM should work for encoding utf-16be, split at character 2
+Pass	ignoreBOM should work for encoding utf-16be, split at character 2
-Fail	ignoreBOM should work for encoding utf-16be, split at character 3
+Pass	ignoreBOM should work for encoding utf-16be, split at character 3

Tests/LibWeb/Text/expected/wpt-import/encoding/textdecoder-ignorebom.any.txt

@@ -2,9 +2,8 @@ Harness status: OK
 
 Found 4 tests
 
-1 Pass
+4 Pass
-3 Fail
+Pass	BOM is ignored if ignoreBOM option is specified: utf-8
-Fail	BOM is ignored if ignoreBOM option is specified: utf-8
+Pass	BOM is ignored if ignoreBOM option is specified: utf-16le
-Fail	BOM is ignored if ignoreBOM option is specified: utf-16le
+Pass	BOM is ignored if ignoreBOM option is specified: utf-16be
-Fail	BOM is ignored if ignoreBOM option is specified: utf-16be
 Pass	The ignoreBOM attribute of TextDecoder

Tests/LibWeb/Text/expected/wpt-import/encoding/textdecoder-mistakes.any.txt

@@ -2,8 +2,8 @@ Harness status: OK
 
 Found 87 tests
 
-72 Pass
+74 Pass
-15 Fail
+13 Fail
 Pass	Invalid Unicode input is replaced: utf-8
 Pass	Invalid Unicode input is replaced: utf-16le
 Pass	Invalid Unicode input is replaced: utf-16be
@@ -51,8 +51,8 @@ Pass	selected single-byte: iso-8859-8-i
 Pass	selected single-byte: iso-8859-16
 Pass	selected single-byte: x-mac-cyrillic
 Pass	Concatenating two ISO-2022-JP outputs is not always valid
-Fail	gb18030 version and ranges
+Pass	gb18030 version and ranges
-Fail	gbk version and ranges
+Pass	gbk version and ranges
 Pass	gbk decoder is gb18030 decoder
 Pass	Replacement, push back ASCII characters: big5
 Pass	Replacement, push back ASCII characters: iso-2022-jp