LibWeb+LibTextCodec: Wire decoder options through TextDecoder
Add explicit IgnoreBOM and ErrorMode options to LibTextCodec decoders, and thread them through TextDecoder and TextDecoderStream. This lets Web-facing decoder APIs preserve BOMs when requested and use fatal error handling without post-processing decoded output. NB: RemoveBOM was renamed to IgnoreBOM as "RemoveBOM" is the name used by encoding_rs and was previously an implementation detail. The new name matches what is used by the encoding standard as it is now also used in LibWeb.
This commit is contained in:
Shannon Booth
Shannon Booth
parent
ef99632fa7
commit
ef6753a9f9
21 changed files with 163 additions and 151 deletions
|
|
@ -19,11 +19,6 @@ static constexpr u32 replacement_code_point = 0xfffd;
|
|||
|
||||
namespace {
|
||||
|
||||
enum class RemoveBOM {
|
||||
No,
|
||||
Yes,
|
||||
};
|
||||
|
||||
class RustDecoder final : public Decoder {
|
||||
public:
|
||||
explicit RustDecoder(StringView encoding)
|
||||
|
|
@ -32,7 +27,7 @@ public:
|
|||
}
|
||||
|
||||
virtual bool validate(StringView input) override;
|
||||
virtual ErrorOr<String> to_utf8(StringView input) override;
|
||||
virtual ErrorOr<String> to_utf8(StringView input, IgnoreBOM, ErrorMode) override;
|
||||
virtual ErrorOr<size_t> length_in_utf16_code_units(StringView input) override;
|
||||
|
||||
private:
|
||||
|
|
@ -45,14 +40,14 @@ class UTF8Decoder final : public Decoder {
|
|||
public:
|
||||
virtual ErrorOr<void> process(StringView, Function<ErrorOr<void>(u32)> on_code_point) override;
|
||||
virtual bool validate(StringView) override;
|
||||
virtual ErrorOr<String> to_utf8(StringView) override;
|
||||
virtual ErrorOr<String> to_utf8(StringView, IgnoreBOM, ErrorMode) override;
|
||||
virtual ErrorOr<size_t> length_in_utf16_code_units(StringView) override;
|
||||
};
|
||||
|
||||
class UTF16BEDecoder final : public Decoder {
|
||||
public:
|
||||
virtual bool validate(StringView) override;
|
||||
virtual ErrorOr<String> to_utf8(StringView) override;
|
||||
virtual ErrorOr<String> to_utf8(StringView, IgnoreBOM, ErrorMode) override;
|
||||
virtual ErrorOr<size_t> length_in_utf16_code_units(StringView) override;
|
||||
|
||||
private:
|
||||
|
|
@ -62,7 +57,7 @@ private:
|
|||
class UTF16LEDecoder final : public Decoder {
|
||||
public:
|
||||
virtual bool validate(StringView) override;
|
||||
virtual ErrorOr<String> to_utf8(StringView) override;
|
||||
virtual ErrorOr<String> to_utf8(StringView, IgnoreBOM, ErrorMode) override;
|
||||
virtual ErrorOr<size_t> length_in_utf16_code_units(StringView) override;
|
||||
|
||||
private:
|
||||
|
|
@ -130,7 +125,7 @@ static void append_decoded_bytes(void* context, u8 const* data, size_t length)
|
|||
decode_context.result = decode_context.builder.try_append(StringView { data, length });
|
||||
}
|
||||
|
||||
ErrorOr<String> rust_decode_to_utf8(StringView encoding, StringView input, RemoveBOM remove_bom)
|
||||
ErrorOr<String> rust_decode_to_utf8(StringView encoding, StringView input, IgnoreBOM ignore_bom, ErrorMode error_mode)
|
||||
{
|
||||
DecodeContext context { .builder = StringBuilder(input.length()) };
|
||||
auto succeeded = FFI::textcodec_rust_decode_to_utf8(
|
||||
|
|
@ -138,36 +133,37 @@ ErrorOr<String> rust_decode_to_utf8(StringView encoding, StringView input, Remov
|
|||
encoding.length(),
|
||||
reinterpret_cast<u8 const*>(input.characters_without_null_termination()),
|
||||
input.length(),
|
||||
remove_bom == RemoveBOM::Yes,
|
||||
ignore_bom == IgnoreBOM::No,
|
||||
error_mode == ErrorMode::Fatal,
|
||||
&context,
|
||||
append_decoded_bytes);
|
||||
if (!succeeded)
|
||||
return Error::from_errno(EINVAL);
|
||||
return Error::from_string_literal("Failed to decode input");
|
||||
TRY(context.result);
|
||||
return context.builder.to_string_without_validation();
|
||||
}
|
||||
|
||||
ErrorOr<void> rust_process(StringView encoding, StringView input, RemoveBOM remove_bom, Function<ErrorOr<void>(u32)> on_code_point)
|
||||
ErrorOr<void> rust_process(StringView encoding, StringView input, IgnoreBOM ignore_bom, Function<ErrorOr<void>(u32)> on_code_point)
|
||||
{
|
||||
auto utf8 = TRY(rust_decode_to_utf8(encoding, input, remove_bom));
|
||||
auto utf8 = TRY(rust_decode_to_utf8(encoding, input, ignore_bom, ErrorMode::Replacement));
|
||||
for (auto code_point : Utf8View { utf8 })
|
||||
TRY(on_code_point(code_point));
|
||||
return {};
|
||||
}
|
||||
|
||||
bool rust_validate(StringView encoding, StringView input, RemoveBOM remove_bom)
|
||||
bool rust_validate(StringView encoding, StringView input, IgnoreBOM ignore_bom)
|
||||
{
|
||||
return FFI::textcodec_rust_validate(
|
||||
reinterpret_cast<u8 const*>(encoding.characters_without_null_termination()),
|
||||
encoding.length(),
|
||||
reinterpret_cast<u8 const*>(input.characters_without_null_termination()),
|
||||
input.length(),
|
||||
remove_bom == RemoveBOM::Yes);
|
||||
ignore_bom == IgnoreBOM::No);
|
||||
}
|
||||
|
||||
ErrorOr<size_t> rust_length_in_utf16_code_units(StringView encoding, StringView input, RemoveBOM remove_bom)
|
||||
ErrorOr<size_t> rust_length_in_utf16_code_units(StringView encoding, StringView input, IgnoreBOM ignore_bom)
|
||||
{
|
||||
auto utf8 = TRY(rust_decode_to_utf8(encoding, input, remove_bom));
|
||||
auto utf8 = TRY(rust_decode_to_utf8(encoding, input, ignore_bom, ErrorMode::Replacement));
|
||||
size_t length = 0;
|
||||
for (auto code_point : Utf8View { utf8 })
|
||||
length += code_point <= 0xffff ? 1 : 2;
|
||||
|
|
@ -188,7 +184,7 @@ Optional<StringView> get_static_encoding_name_from_rust(StringView label)
|
|||
return StringView { encoding_name, encoding_name_length };
|
||||
}
|
||||
|
||||
ErrorOr<String> rust_streaming_decode_to_utf8(FFI::TextCodecRustStreamingDecoder* decoder, ReadonlyBytes input, bool last)
|
||||
ErrorOr<String> rust_streaming_decode_to_utf8(FFI::TextCodecRustStreamingDecoder* decoder, ReadonlyBytes input, bool last, ErrorMode error_mode)
|
||||
{
|
||||
DecodeContext context { .builder = StringBuilder(input.size()) };
|
||||
auto succeeded = FFI::textcodec_rust_streaming_decoder_decode_to_utf8(
|
||||
|
|
@ -196,10 +192,11 @@ ErrorOr<String> rust_streaming_decode_to_utf8(FFI::TextCodecRustStreamingDecoder
|
|||
input.data(),
|
||||
input.size(),
|
||||
last,
|
||||
error_mode == ErrorMode::Fatal,
|
||||
&context,
|
||||
append_decoded_bytes);
|
||||
if (!succeeded)
|
||||
return Error::from_errno(EINVAL);
|
||||
return Error::from_string_literal("Failed to decode input");
|
||||
TRY(context.result);
|
||||
return context.builder.to_string_without_validation();
|
||||
}
|
||||
|
|
@ -367,7 +364,7 @@ ErrorOr<String> convert_input_to_utf8_using_given_decoder_unless_there_is_a_byte
|
|||
|
||||
// 3. Process a queue with an instance of encoding’s decoder, ioQueue, output, and "replacement".
|
||||
// FIXME: This isn't the exact same as the spec, which is written in terms of I/O queues.
|
||||
auto output = TRY(actual_decoder->to_utf8(input));
|
||||
auto output = TRY(actual_decoder->to_utf8(input, IgnoreBOM::No, ErrorMode::Replacement));
|
||||
|
||||
// 4. Return output.
|
||||
return output;
|
||||
|
|
@ -414,15 +411,18 @@ bool Decoder::validate(StringView input)
|
|||
{
|
||||
auto result = this->process(input, [](auto code_point) -> ErrorOr<void> {
|
||||
if (code_point == replacement_code_point)
|
||||
return Error::from_errno(EINVAL);
|
||||
return Error::from_string_literal("Decoded input contains replacement character");
|
||||
return {};
|
||||
});
|
||||
|
||||
return !result.is_error();
|
||||
}
|
||||
|
||||
ErrorOr<String> Decoder::to_utf8(StringView input)
|
||||
ErrorOr<String> Decoder::to_utf8(StringView input, IgnoreBOM, ErrorMode error_mode)
|
||||
{
|
||||
if (error_mode == ErrorMode::Fatal && !validate(input))
|
||||
return Error::from_string_literal("Failed to decode input");
|
||||
|
||||
StringBuilder builder(input.length());
|
||||
TRY(process(input, [&builder](u32 c) { return builder.try_append_code_point(c); }));
|
||||
return builder.to_string_without_validation();
|
||||
|
|
@ -455,22 +455,22 @@ ErrorOr<void> Decoder::process_code_points(StringView input, Function<ErrorOr<vo
|
|||
|
||||
bool RustDecoder::validate(StringView input)
|
||||
{
|
||||
return rust_validate(m_encoding, input, RemoveBOM::No);
|
||||
return rust_validate(m_encoding, input, IgnoreBOM::Yes);
|
||||
}
|
||||
|
||||
ErrorOr<String> RustDecoder::to_utf8(StringView input)
|
||||
ErrorOr<String> RustDecoder::to_utf8(StringView input, IgnoreBOM ignore_bom, ErrorMode error_mode)
|
||||
{
|
||||
return rust_decode_to_utf8(m_encoding, input, RemoveBOM::No);
|
||||
return rust_decode_to_utf8(m_encoding, input, ignore_bom, error_mode);
|
||||
}
|
||||
|
||||
ErrorOr<size_t> RustDecoder::length_in_utf16_code_units(StringView input)
|
||||
{
|
||||
return rust_length_in_utf16_code_units(m_encoding, input, RemoveBOM::No);
|
||||
return rust_length_in_utf16_code_units(m_encoding, input, IgnoreBOM::Yes);
|
||||
}
|
||||
|
||||
ErrorOr<void> RustDecoder::process(StringView input, Function<ErrorOr<void>(u32)> on_code_point)
|
||||
{
|
||||
return rust_process(m_encoding, input, RemoveBOM::No, move(on_code_point));
|
||||
return rust_process(m_encoding, input, IgnoreBOM::Yes, move(on_code_point));
|
||||
}
|
||||
|
||||
ErrorOr<void> Latin1Decoder::process(StringView input, Function<ErrorOr<void>(u32)> on_code_point)
|
||||
|
|
@ -485,12 +485,13 @@ ErrorOr<size_t> Latin1Decoder::length_in_utf16_code_units(StringView input)
|
|||
return input.length();
|
||||
}
|
||||
|
||||
StreamingDecoder::StreamingDecoder(StringView encoding)
|
||||
StreamingDecoder::StreamingDecoder(StringView encoding, IgnoreBOM ignore_bom, ErrorMode error_mode)
|
||||
: m_error_mode(error_mode)
|
||||
{
|
||||
m_decoder = FFI::textcodec_rust_streaming_decoder_new(
|
||||
reinterpret_cast<u8 const*>(encoding.characters_without_null_termination()),
|
||||
encoding.length(),
|
||||
true);
|
||||
ignore_bom == IgnoreBOM::No);
|
||||
VERIFY(m_decoder);
|
||||
}
|
||||
|
||||
|
|
@ -501,72 +502,72 @@ StreamingDecoder::~StreamingDecoder()
|
|||
|
||||
ErrorOr<String> StreamingDecoder::to_utf8(ReadonlyBytes input)
|
||||
{
|
||||
return rust_streaming_decode_to_utf8(static_cast<FFI::TextCodecRustStreamingDecoder*>(m_decoder), input, false);
|
||||
return rust_streaming_decode_to_utf8(static_cast<FFI::TextCodecRustStreamingDecoder*>(m_decoder), input, false, m_error_mode);
|
||||
}
|
||||
|
||||
ErrorOr<String> StreamingDecoder::finish()
|
||||
{
|
||||
return rust_streaming_decode_to_utf8(static_cast<FFI::TextCodecRustStreamingDecoder*>(m_decoder), {}, true);
|
||||
return rust_streaming_decode_to_utf8(static_cast<FFI::TextCodecRustStreamingDecoder*>(m_decoder), {}, true, m_error_mode);
|
||||
}
|
||||
|
||||
ErrorOr<void> UTF8Decoder::process(StringView input, Function<ErrorOr<void>(u32)> on_code_point)
|
||||
{
|
||||
return rust_process("UTF-8"sv, input, RemoveBOM::No, move(on_code_point));
|
||||
return rust_process("UTF-8"sv, input, IgnoreBOM::Yes, move(on_code_point));
|
||||
}
|
||||
|
||||
bool UTF8Decoder::validate(StringView input)
|
||||
{
|
||||
return rust_validate("UTF-8"sv, input, RemoveBOM::No);
|
||||
return rust_validate("UTF-8"sv, input, IgnoreBOM::Yes);
|
||||
}
|
||||
|
||||
ErrorOr<String> UTF8Decoder::to_utf8(StringView input)
|
||||
ErrorOr<String> UTF8Decoder::to_utf8(StringView input, IgnoreBOM ignore_bom, ErrorMode error_mode)
|
||||
{
|
||||
return rust_decode_to_utf8("UTF-8"sv, input, RemoveBOM::Yes);
|
||||
return rust_decode_to_utf8("UTF-8"sv, input, ignore_bom, error_mode);
|
||||
}
|
||||
|
||||
ErrorOr<size_t> UTF8Decoder::length_in_utf16_code_units(StringView input)
|
||||
{
|
||||
return rust_length_in_utf16_code_units("UTF-8"sv, input, RemoveBOM::Yes);
|
||||
return rust_length_in_utf16_code_units("UTF-8"sv, input, IgnoreBOM::No);
|
||||
}
|
||||
|
||||
bool UTF16BEDecoder::validate(StringView input)
|
||||
{
|
||||
return rust_validate("UTF-16BE"sv, input, RemoveBOM::No);
|
||||
return rust_validate("UTF-16BE"sv, input, IgnoreBOM::Yes);
|
||||
}
|
||||
|
||||
ErrorOr<void> UTF16BEDecoder::process(StringView input, Function<ErrorOr<void>(u32)> on_code_point)
|
||||
{
|
||||
return rust_process("UTF-16BE"sv, input, RemoveBOM::Yes, move(on_code_point));
|
||||
return rust_process("UTF-16BE"sv, input, IgnoreBOM::No, move(on_code_point));
|
||||
}
|
||||
|
||||
ErrorOr<String> UTF16BEDecoder::to_utf8(StringView input)
|
||||
ErrorOr<String> UTF16BEDecoder::to_utf8(StringView input, IgnoreBOM ignore_bom, ErrorMode error_mode)
|
||||
{
|
||||
return rust_decode_to_utf8("UTF-16BE"sv, input, RemoveBOM::Yes);
|
||||
return rust_decode_to_utf8("UTF-16BE"sv, input, ignore_bom, error_mode);
|
||||
}
|
||||
|
||||
ErrorOr<size_t> UTF16BEDecoder::length_in_utf16_code_units(StringView input)
|
||||
{
|
||||
return rust_length_in_utf16_code_units("UTF-16BE"sv, input, RemoveBOM::Yes);
|
||||
return rust_length_in_utf16_code_units("UTF-16BE"sv, input, IgnoreBOM::No);
|
||||
}
|
||||
|
||||
bool UTF16LEDecoder::validate(StringView input)
|
||||
{
|
||||
return rust_validate("UTF-16LE"sv, input, RemoveBOM::No);
|
||||
return rust_validate("UTF-16LE"sv, input, IgnoreBOM::Yes);
|
||||
}
|
||||
|
||||
ErrorOr<void> UTF16LEDecoder::process(StringView input, Function<ErrorOr<void>(u32)> on_code_point)
|
||||
{
|
||||
return rust_process("UTF-16LE"sv, input, RemoveBOM::Yes, move(on_code_point));
|
||||
return rust_process("UTF-16LE"sv, input, IgnoreBOM::No, move(on_code_point));
|
||||
}
|
||||
|
||||
ErrorOr<String> UTF16LEDecoder::to_utf8(StringView input)
|
||||
ErrorOr<String> UTF16LEDecoder::to_utf8(StringView input, IgnoreBOM ignore_bom, ErrorMode error_mode)
|
||||
{
|
||||
return rust_decode_to_utf8("UTF-16LE"sv, input, RemoveBOM::Yes);
|
||||
return rust_decode_to_utf8("UTF-16LE"sv, input, ignore_bom, error_mode);
|
||||
}
|
||||
|
||||
ErrorOr<size_t> UTF16LEDecoder::length_in_utf16_code_units(StringView input)
|
||||
{
|
||||
return rust_length_in_utf16_code_units("UTF-16LE"sv, input, RemoveBOM::Yes);
|
||||
return rust_length_in_utf16_code_units("UTF-16LE"sv, input, IgnoreBOM::No);
|
||||
}
|
||||
|
||||
// https://infra.spec.whatwg.org/#isomorphic-decode
|
||||
|
|
|
|||
|
|
@ -19,10 +19,21 @@
|
|||
|
||||
namespace TextCodec {
|
||||
|
||||
enum class IgnoreBOM {
|
||||
Yes,
|
||||
No,
|
||||
};
|
||||
|
||||
// https://encoding.spec.whatwg.org/#concept-encoding-error-mode
|
||||
enum class ErrorMode {
|
||||
Replacement,
|
||||
Fatal,
|
||||
};
|
||||
|
||||
class TEXTCODEC_API Decoder {
|
||||
public:
|
||||
virtual bool validate(StringView);
|
||||
virtual ErrorOr<String> to_utf8(StringView);
|
||||
virtual ErrorOr<String> to_utf8(StringView, IgnoreBOM, ErrorMode);
|
||||
virtual ErrorOr<Utf16String> to_utf16(StringView);
|
||||
virtual ErrorOr<size_t> length_in_utf16_code_units(StringView);
|
||||
ErrorOr<void> process_code_points(StringView, Function<ErrorOr<void>(u32)>);
|
||||
|
|
@ -36,13 +47,14 @@ class TEXTCODEC_API StreamingDecoder final {
|
|||
AK_MAKE_NONCOPYABLE(StreamingDecoder);
|
||||
|
||||
public:
|
||||
explicit StreamingDecoder(StringView encoding);
|
||||
StreamingDecoder(StringView encoding, IgnoreBOM, ErrorMode);
|
||||
~StreamingDecoder();
|
||||
|
||||
ErrorOr<String> to_utf8(ReadonlyBytes);
|
||||
ErrorOr<String> finish();
|
||||
|
||||
private:
|
||||
ErrorMode m_error_mode { ErrorMode::Replacement };
|
||||
void* m_decoder { nullptr };
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -93,6 +93,7 @@ pub unsafe extern "C" fn textcodec_rust_decode_to_utf8(
|
|||
input: *const u8,
|
||||
input_len: usize,
|
||||
remove_bom: bool,
|
||||
fatal: bool,
|
||||
ctx: *mut c_void,
|
||||
on_bytes: FfiBytesFn,
|
||||
) -> bool {
|
||||
|
|
@ -108,11 +109,14 @@ pub unsafe extern "C" fn textcodec_rust_decode_to_utf8(
|
|||
return false;
|
||||
};
|
||||
|
||||
let (output, _) = if remove_bom {
|
||||
let (output, had_errors) = if remove_bom {
|
||||
encoding.decode_with_bom_removal(input)
|
||||
} else {
|
||||
encoding.decode_without_bom_handling(input)
|
||||
};
|
||||
if fatal && had_errors {
|
||||
return false;
|
||||
}
|
||||
on_bytes(ctx, output.as_bytes().as_ptr(), output.len());
|
||||
true
|
||||
})
|
||||
|
|
@ -212,6 +216,7 @@ pub unsafe extern "C" fn textcodec_rust_streaming_decoder_decode_to_utf8(
|
|||
input: *const u8,
|
||||
input_len: usize,
|
||||
last: bool,
|
||||
fatal: bool,
|
||||
ctx: *mut c_void,
|
||||
on_bytes: FfiBytesFn,
|
||||
) -> bool {
|
||||
|
|
@ -230,7 +235,10 @@ pub unsafe extern "C" fn textcodec_rust_streaming_decoder_decode_to_utf8(
|
|||
};
|
||||
let mut output = String::with_capacity(output_capacity);
|
||||
|
||||
let (result, _, _) = decoder.decoder.decode_to_string(input, &mut output, last);
|
||||
let (result, _, had_errors) = decoder.decoder.decode_to_string(input, &mut output, last);
|
||||
if fatal && had_errors {
|
||||
return false;
|
||||
}
|
||||
if !output.is_empty() {
|
||||
on_bytes(ctx, output.as_ptr(), output.len());
|
||||
}
|
||||
|
|
|
|||
|
|
@ -36,7 +36,7 @@ static String decode_and_filter_code_points(StringView input, StringView encodin
|
|||
input = input.substring_view(3);
|
||||
return String::from_utf8_without_validation(input.bytes());
|
||||
}
|
||||
return MUST(decoder->to_utf8(input));
|
||||
return MUST(decoder->to_utf8(input, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement));
|
||||
}();
|
||||
|
||||
// OPTIMIZATION: If the input doesn't contain any filterable characters, we can skip the filtering
|
||||
|
|
|
|||
|
|
@ -175,7 +175,7 @@ Vector<Token> Tokenizer::tokenize(StringView input, StringView encoding, Tokeniz
|
|||
input = input.substring_view(3);
|
||||
return String::from_utf8_without_validation(input.bytes());
|
||||
}
|
||||
return MUST(decoder->to_utf8(input));
|
||||
return MUST(decoder->to_utf8(input, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement));
|
||||
}();
|
||||
|
||||
// OPTIMIZATION: If the input doesn't contain any filterable characters, we can skip the filtering
|
||||
|
|
|
|||
|
|
@ -77,7 +77,7 @@ bool build_xml_document(DOM::Document& document, ByteBuffer const& data, Optiona
|
|||
convert_to_xml_error_document(document, "XML Document contains improperly-encoded characters"_utf16);
|
||||
return false;
|
||||
}
|
||||
auto source = decoder->to_utf8(data).release_value_but_fixme_should_propagate_errors();
|
||||
auto source = decoder->to_utf8(data, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Fatal).release_value_but_fixme_should_propagate_errors();
|
||||
XML::Parser parser(source, { .resolve_named_html_entity = resolve_named_html_entity });
|
||||
XMLDocumentBuilder builder { document };
|
||||
auto result = parser.parse_with_listener(builder);
|
||||
|
|
@ -204,7 +204,7 @@ static WebIDL::ExceptionOr<GC::Ref<DOM::Document>> load_xml_document(HTML::Navig
|
|||
convert_to_xml_error_document(document, "XML Document contains improperly-encoded characters"_utf16);
|
||||
return;
|
||||
}
|
||||
auto source = decoder->to_utf8(data);
|
||||
auto source = decoder->to_utf8(data, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Fatal);
|
||||
if (source.is_error()) {
|
||||
// FIXME: Insert error message into the document.
|
||||
dbgln("Failed to decode XML document: {}", source.error());
|
||||
|
|
|
|||
|
|
@ -35,7 +35,7 @@ WebIDL::ExceptionOr<GC::Ref<TextDecoder>> TextDecoder::construct_impl(JS::Realm&
|
|||
auto lowercase_encoding_name = encoding.value().to_ascii_lowercase_string();
|
||||
|
||||
// 4. If options["fatal"] is true, then set this’s error mode to "fatal".
|
||||
auto error_mode = options.fatal ? ErrorMode::Fatal : ErrorMode::Replacement;
|
||||
auto error_mode = options.fatal ? TextCodec::ErrorMode::Fatal : TextCodec::ErrorMode::Replacement;
|
||||
|
||||
// 5. Set this’s ignore BOM to options["ignoreBOM"].
|
||||
auto ignore_bom = options.ignore_bom;
|
||||
|
|
@ -48,7 +48,7 @@ WebIDL::ExceptionOr<GC::Ref<TextDecoder>> TextDecoder::construct_impl(JS::Realm&
|
|||
}
|
||||
|
||||
// https://encoding.spec.whatwg.org/#dom-textdecoder
|
||||
TextDecoder::TextDecoder(JS::Realm& realm, TextCodec::Decoder& decoder, FlyString encoding, ErrorMode error_mode, bool ignore_bom)
|
||||
TextDecoder::TextDecoder(JS::Realm& realm, TextCodec::Decoder& decoder, FlyString encoding, TextCodec::ErrorMode error_mode, bool ignore_bom)
|
||||
: PlatformObject(realm)
|
||||
, TextDecoderCommonMixin(decoder, move(encoding), error_mode, ignore_bom)
|
||||
{
|
||||
|
|
@ -65,18 +65,23 @@ void TextDecoder::initialize(JS::Realm& realm)
|
|||
// https://encoding.spec.whatwg.org/#dom-textdecoder-decode
|
||||
WebIDL::ExceptionOr<String> TextDecoder::decode(Optional<WebIDL::BufferSourceVariant> input, Optional<Bindings::TextDecodeOptions> const&) const
|
||||
{
|
||||
if (!input.has_value())
|
||||
return TRY_OR_THROW_OOM(vm(), m_decoder.to_utf8({}));
|
||||
auto ignore_bom = m_ignore_bom ? TextCodec::IgnoreBOM::Yes : TextCodec::IgnoreBOM::No;
|
||||
if (!input.has_value()) {
|
||||
auto result = m_decoder.to_utf8({}, ignore_bom, m_error_mode);
|
||||
if (result.is_error() && result.error().code() != ENOMEM)
|
||||
return WebIDL::SimpleException { WebIDL::SimpleExceptionType::TypeError, "Decoding failed"sv };
|
||||
return TRY_OR_THROW_OOM(vm(), move(result));
|
||||
}
|
||||
|
||||
// FIXME: Implement the streaming stuff.
|
||||
auto data_buffer_or_error = WebIDL::get_buffer_source_copy(*input);
|
||||
if (data_buffer_or_error.is_error())
|
||||
return WebIDL::OperationError::create(realm(), "Failed to copy bytes from ArrayBuffer"_utf16);
|
||||
auto& data_buffer = data_buffer_or_error.value();
|
||||
auto result = TRY_OR_THROW_OOM(vm(), m_decoder.to_utf8({ data_buffer.data(), data_buffer.size() }));
|
||||
if (this->fatal() && result.contains(0xfffd))
|
||||
auto result = m_decoder.to_utf8({ data_buffer.data(), data_buffer.size() }, ignore_bom, m_error_mode);
|
||||
if (result.is_error() && result.error().code() != ENOMEM)
|
||||
return WebIDL::SimpleException { WebIDL::SimpleExceptionType::TypeError, "Decoding failed"sv };
|
||||
return result;
|
||||
return TRY_OR_THROW_OOM(vm(), move(result));
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -34,7 +34,7 @@ public:
|
|||
WebIDL::ExceptionOr<String> decode(Optional<WebIDL::BufferSourceVariant>, Optional<Bindings::TextDecodeOptions> const& options = {}) const;
|
||||
|
||||
private:
|
||||
TextDecoder(JS::Realm&, TextCodec::Decoder&, FlyString encoding, ErrorMode error_mode, bool ignore_bom);
|
||||
TextDecoder(JS::Realm&, TextCodec::Decoder&, FlyString encoding, TextCodec::ErrorMode error_mode, bool ignore_bom);
|
||||
|
||||
virtual void initialize(JS::Realm&) override;
|
||||
};
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@
|
|||
|
||||
namespace Web::Encoding {
|
||||
|
||||
TextDecoderCommonMixin::TextDecoderCommonMixin(TextCodec::Decoder& decoder, FlyString encoding, ErrorMode error_mode, bool ignore_bom)
|
||||
TextDecoderCommonMixin::TextDecoderCommonMixin(TextCodec::Decoder& decoder, FlyString encoding, TextCodec::ErrorMode error_mode, bool ignore_bom)
|
||||
: m_decoder(decoder)
|
||||
, m_encoding(move(encoding))
|
||||
, m_error_mode(error_mode)
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@
|
|||
#pragma once
|
||||
|
||||
#include <AK/FlyString.h>
|
||||
#include <LibTextCodec/Forward.h>
|
||||
#include <LibTextCodec/Decoder.h>
|
||||
|
||||
namespace Web::Encoding {
|
||||
|
||||
|
|
@ -20,19 +20,13 @@ public:
|
|||
FlyString const& encoding() const { return m_encoding; }
|
||||
|
||||
// https://encoding.spec.whatwg.org/#dom-textdecoder-fatal
|
||||
bool fatal() const { return m_error_mode == ErrorMode::Fatal; }
|
||||
bool fatal() const { return m_error_mode == TextCodec::ErrorMode::Fatal; }
|
||||
|
||||
// https://encoding.spec.whatwg.org/#dom-textdecoder-ignorebom
|
||||
bool ignore_bom() const { return m_ignore_bom; }
|
||||
|
||||
protected:
|
||||
// https://encoding.spec.whatwg.org/#concept-encoding-error-mode
|
||||
enum class ErrorMode {
|
||||
Replacement,
|
||||
Fatal,
|
||||
};
|
||||
|
||||
TextDecoderCommonMixin(TextCodec::Decoder& decoder, FlyString encoding, ErrorMode error_mode, bool ignore_bom);
|
||||
TextDecoderCommonMixin(TextCodec::Decoder& decoder, FlyString encoding, TextCodec::ErrorMode error_mode, bool ignore_bom);
|
||||
|
||||
// https://encoding.spec.whatwg.org/#textdecodercommon-decoder
|
||||
TextCodec::Decoder& m_decoder;
|
||||
|
|
@ -41,13 +35,10 @@ protected:
|
|||
FlyString m_encoding;
|
||||
|
||||
// https://encoding.spec.whatwg.org/#textdecoder-error-mode
|
||||
ErrorMode m_error_mode { ErrorMode::Replacement };
|
||||
TextCodec::ErrorMode m_error_mode { TextCodec::ErrorMode::Replacement };
|
||||
|
||||
// https://encoding.spec.whatwg.org/#textdecoder-ignore-bom-flag
|
||||
bool m_ignore_bom { false };
|
||||
|
||||
// https://encoding.spec.whatwg.org/#textdecoder-bom-seen-flag
|
||||
bool m_bom_seen { false };
|
||||
};
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -39,7 +39,7 @@ WebIDL::ExceptionOr<GC::Ref<TextDecoderStream>> TextDecoderStream::construct_imp
|
|||
auto lowercase_encoding_name = encoding.value().to_ascii_lowercase_string();
|
||||
|
||||
// 4. If options["fatal"] is true, then set this’s error mode to "fatal".
|
||||
auto error_mode = options.fatal ? ErrorMode::Fatal : ErrorMode::Replacement;
|
||||
auto error_mode = options.fatal ? TextCodec::ErrorMode::Fatal : TextCodec::ErrorMode::Replacement;
|
||||
|
||||
// 5. Set this’s ignore BOM to options["ignoreBOM"].
|
||||
auto ignore_bom = options.ignore_bom;
|
||||
|
|
@ -84,11 +84,14 @@ WebIDL::ExceptionOr<GC::Ref<TextDecoderStream>> TextDecoderStream::construct_imp
|
|||
return stream;
|
||||
}
|
||||
|
||||
TextDecoderStream::TextDecoderStream(JS::Realm& realm, GC::Ref<Streams::TransformStream> transform, TextCodec::Decoder& decoder, FlyString encoding, ErrorMode error_mode, bool ignore_bom)
|
||||
TextDecoderStream::TextDecoderStream(JS::Realm& realm, GC::Ref<Streams::TransformStream> transform, TextCodec::Decoder& decoder, FlyString encoding, TextCodec::ErrorMode error_mode, bool ignore_bom)
|
||||
: Bindings::PlatformObject(realm)
|
||||
, Streams::GenericTransformStreamMixin(transform)
|
||||
, TextDecoderCommonMixin(decoder, move(encoding), error_mode, ignore_bom)
|
||||
, m_streaming_decoder(make<TextCodec::StreamingDecoder>(m_encoding))
|
||||
, m_streaming_decoder(make<TextCodec::StreamingDecoder>(
|
||||
m_encoding,
|
||||
m_ignore_bom ? TextCodec::IgnoreBOM::Yes : TextCodec::IgnoreBOM::No,
|
||||
m_error_mode))
|
||||
{
|
||||
}
|
||||
|
||||
|
|
@ -122,7 +125,10 @@ WebIDL::ExceptionOr<void> TextDecoderStream::decode_and_enqueue_chunk(JS::Value
|
|||
return WebIDL::OperationError::create(realm, "Failed to copy bytes from BufferSource"_utf16);
|
||||
auto buffer = buffer_or_error.release_value();
|
||||
|
||||
auto decoded = TRY_OR_THROW_OOM(vm, m_streaming_decoder->to_utf8(buffer.bytes()));
|
||||
auto decoded_or_error = m_streaming_decoder->to_utf8(buffer.bytes());
|
||||
if (decoded_or_error.is_error() && decoded_or_error.error().code() != ENOMEM)
|
||||
return WebIDL::SimpleException { WebIDL::SimpleExceptionType::TypeError, "Decoding failed"sv };
|
||||
auto decoded = TRY_OR_THROW_OOM(vm, move(decoded_or_error));
|
||||
|
||||
// 3-4. Run "processing an item" until the input is exhausted, accumulating the output, then enqueue any non-empty
|
||||
// result. If processing returns error, throw a TypeError.
|
||||
|
|
@ -133,7 +139,10 @@ WebIDL::ExceptionOr<void> TextDecoderStream::decode_and_enqueue_chunk(JS::Value
|
|||
WebIDL::ExceptionOr<void> TextDecoderStream::flush_and_enqueue()
|
||||
{
|
||||
// 1-3. Drain decoder's I/O queue and run "processing an item" to completion.
|
||||
auto decoded = TRY_OR_THROW_OOM(vm(), m_streaming_decoder->finish());
|
||||
auto decoded_or_error = m_streaming_decoder->finish();
|
||||
if (decoded_or_error.is_error() && decoded_or_error.error().code() != ENOMEM)
|
||||
return WebIDL::SimpleException { WebIDL::SimpleExceptionType::TypeError, "Decoding failed"sv };
|
||||
auto decoded = TRY_OR_THROW_OOM(vm(), move(decoded_or_error));
|
||||
return enqueue_decoded_output(decoded);
|
||||
}
|
||||
|
||||
|
|
@ -142,22 +151,9 @@ WebIDL::ExceptionOr<void> TextDecoderStream::enqueue_decoded_output(String const
|
|||
auto& realm = this->realm();
|
||||
auto& vm = realm.vm();
|
||||
|
||||
// https://encoding.spec.whatwg.org/#concept-td-serialize
|
||||
// FIXME: The underlying TextCodec decoders currently strip leading BOMs unconditionally for UTF-8 and UTF-16BE/LE,
|
||||
// so the "ignore BOM" flag is effectively ignored here. Once the decoders accept a "preserve BOM" mode,
|
||||
// plumb m_ignore_bom through and strip the BOM from `decoded` only when m_ignore_bom is false.
|
||||
if (!m_bom_seen && !decoded.is_empty())
|
||||
m_bom_seen = true;
|
||||
|
||||
if (decoded.is_empty())
|
||||
return {};
|
||||
|
||||
// If decoder's error mode is "fatal" and processing produced any error, throw a TypeError.
|
||||
// NB: We can only detect this approximately by looking for U+FFFD in the decoded output, which the underlying
|
||||
// decoder substitutes for invalid sequences. This matches the existing TextDecoder.decode() behavior.
|
||||
if (fatal() && decoded.contains(0xFFFD))
|
||||
return WebIDL::SimpleException { WebIDL::SimpleExceptionType::TypeError, "Decoding failed"sv };
|
||||
|
||||
auto js_string = JS::PrimitiveString::create(vm, Utf16String::from_utf8(decoded));
|
||||
return Streams::transform_stream_default_controller_enqueue(*m_transform->controller(), js_string);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -27,7 +27,7 @@ public:
|
|||
virtual ~TextDecoderStream() override;
|
||||
|
||||
private:
|
||||
TextDecoderStream(JS::Realm&, GC::Ref<Streams::TransformStream>, TextCodec::Decoder&, FlyString encoding, ErrorMode, bool ignore_bom);
|
||||
TextDecoderStream(JS::Realm&, GC::Ref<Streams::TransformStream>, TextCodec::Decoder&, FlyString encoding, TextCodec::ErrorMode, bool ignore_bom);
|
||||
|
||||
virtual void initialize(JS::Realm&) override;
|
||||
virtual void visit_edges(Cell::Visitor&) override;
|
||||
|
|
|
|||
|
|
@ -108,7 +108,7 @@ HTMLTokenizer::HTMLTokenizer(StringView input, ByteString const& encoding, Input
|
|||
if (input_type == InputType::EncodedBytes) {
|
||||
auto decoder = TextCodec::decoder_for(encoding);
|
||||
VERIFY(decoder.has_value());
|
||||
m_source = MUST(decoder->to_utf8(input));
|
||||
m_source = MUST(decoder->to_utf8(input, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement));
|
||||
} else {
|
||||
m_source = decoded_string_for_utf8_tokenizer(input);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -68,7 +68,7 @@ void IncrementalDocumentParser::initialize_parser(ReadonlyBytes sniff_bytes)
|
|||
|
||||
auto standardized_encoding = TextCodec::get_standardized_encoding(encoding);
|
||||
VERIFY(standardized_encoding.has_value());
|
||||
m_decoder = make<TextCodec::StreamingDecoder>(standardized_encoding.value());
|
||||
m_decoder = make<TextCodec::StreamingDecoder>(standardized_encoding.value(), TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement);
|
||||
|
||||
// https://html.spec.whatwg.org/multipage/parsing.html#determining-the-character-encoding
|
||||
// The document's character encoding must immediately be set to the value returned from this
|
||||
|
|
|
|||
|
|
@ -438,7 +438,7 @@ ErrorOr<Vector<String>> Autocomplete::received_autocomplete_respsonse(Autocomple
|
|||
if (!decoder.has_value())
|
||||
decoder = TextCodec::decoder_for_exact_name("UTF-8"sv);
|
||||
|
||||
auto decoded_response = TRY(decoder->to_utf8(response));
|
||||
auto decoded_response = TRY(decoder->to_utf8(response, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement));
|
||||
auto json = TRY(JsonValue::from_string(decoded_response));
|
||||
|
||||
if (engine.name == "DuckDuckGo")
|
||||
|
|
|
|||
|
|
@ -24,6 +24,6 @@ extern "C" int LLVMFuzzerTestOneInput(uint8_t const* data, size_t size)
|
|||
if (!decoder.has_value())
|
||||
return 0;
|
||||
|
||||
(void)decoder->to_utf8(encoded_data);
|
||||
(void)decoder->to_utf8(encoded_data, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement);
|
||||
return 0;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1085,7 +1085,7 @@ size_t Request::on_header_received(void* buffer, size_t size, size_t nmemb, void
|
|||
auto decoder = TextCodec::decoder_for_exact_name("ISO-8859-1"sv);
|
||||
VERIFY(decoder.has_value());
|
||||
|
||||
request.m_reason_phrase = MUST(decoder->to_utf8(reason_phrase));
|
||||
request.m_reason_phrase = MUST(decoder->to_utf8(reason_phrase, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement));
|
||||
return total_size;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -43,7 +43,7 @@ TEST_CASE(test_utf8_decode)
|
|||
EXPECT(processed_code_points.size() == 1);
|
||||
EXPECT(processed_code_points[0] == 0x1F600);
|
||||
|
||||
EXPECT(MUST(decoder.to_utf8(test_string)) == test_string);
|
||||
EXPECT(MUST(decoder.to_utf8(test_string, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)) == test_string);
|
||||
}
|
||||
|
||||
TEST_CASE(test_utf8_process_code_points)
|
||||
|
|
@ -61,7 +61,7 @@ TEST_CASE(test_utf8_process_code_points_replaces_surrogates)
|
|||
auto utf8_encoded_surrogate = StringView(bytes(utf8_encoded_surrogate_bytes));
|
||||
|
||||
EXPECT(!decoder.validate(utf8_encoded_surrogate));
|
||||
EXPECT_EQ(MUST(decoder.to_utf8(utf8_encoded_surrogate)), "\xef\xbf\xbd\xef\xbf\xbd\xef\xbf\xbd"sv);
|
||||
EXPECT_EQ(MUST(decoder.to_utf8(utf8_encoded_surrogate, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)), "\xef\xbf\xbd\xef\xbf\xbd\xef\xbf\xbd"sv);
|
||||
EXPECT_EQ(process_code_points(decoder, utf8_encoded_surrogate), (Vector<u32> { 0xfffd, 0xfffd, 0xfffd }));
|
||||
}
|
||||
|
||||
|
|
@ -72,7 +72,7 @@ TEST_CASE(test_utf8_process_code_points_replaces_truncated_tail_as_single_error)
|
|||
auto truncated_tail = StringView(bytes(truncated_tail_bytes));
|
||||
|
||||
EXPECT(!decoder.validate(truncated_tail));
|
||||
EXPECT_EQ(MUST(decoder.to_utf8(truncated_tail)), "\xef\xbf\xbd"sv);
|
||||
EXPECT_EQ(MUST(decoder.to_utf8(truncated_tail, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)), "\xef\xbf\xbd"sv);
|
||||
EXPECT_EQ(process_code_points(decoder, truncated_tail), (Vector<u32> { 0xfffd }));
|
||||
}
|
||||
|
||||
|
|
@ -83,7 +83,7 @@ TEST_CASE(test_utf8_process_code_points_replaces_overlong_sequences)
|
|||
auto overlong_null = StringView(bytes(overlong_null_bytes));
|
||||
|
||||
EXPECT(!decoder.validate(overlong_null));
|
||||
EXPECT_EQ(MUST(decoder.to_utf8(overlong_null)), "\xef\xbf\xbd\xef\xbf\xbd"sv);
|
||||
EXPECT_EQ(MUST(decoder.to_utf8(overlong_null, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)), "\xef\xbf\xbd\xef\xbf\xbd"sv);
|
||||
EXPECT_EQ(process_code_points(decoder, overlong_null), (Vector<u32> { 0xfffd, 0xfffd }));
|
||||
}
|
||||
|
||||
|
|
@ -96,11 +96,11 @@ TEST_CASE(test_utf8_process_code_points_restores_invalid_second_byte)
|
|||
auto out_of_range_four_byte_sequence = StringView(bytes(out_of_range_four_byte_sequence_bytes));
|
||||
|
||||
EXPECT(!decoder.validate(overlong_three_byte_sequence));
|
||||
EXPECT_EQ(MUST(decoder.to_utf8(overlong_three_byte_sequence)), "\xef\xbf\xbd\xef\xbf\xbd\xef\xbf\xbd"sv);
|
||||
EXPECT_EQ(MUST(decoder.to_utf8(overlong_three_byte_sequence, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)), "\xef\xbf\xbd\xef\xbf\xbd\xef\xbf\xbd"sv);
|
||||
EXPECT_EQ(process_code_points(decoder, overlong_three_byte_sequence), (Vector<u32> { 0xfffd, 0xfffd, 0xfffd }));
|
||||
|
||||
EXPECT(!decoder.validate(out_of_range_four_byte_sequence));
|
||||
EXPECT_EQ(MUST(decoder.to_utf8(out_of_range_four_byte_sequence)), "\xef\xbf\xbd\xef\xbf\xbd\xef\xbf\xbd\xef\xbf\xbd"sv);
|
||||
EXPECT_EQ(MUST(decoder.to_utf8(out_of_range_four_byte_sequence, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)), "\xef\xbf\xbd\xef\xbf\xbd\xef\xbf\xbd\xef\xbf\xbd"sv);
|
||||
EXPECT_EQ(process_code_points(decoder, out_of_range_four_byte_sequence), (Vector<u32> { 0xfffd, 0xfffd, 0xfffd, 0xfffd }));
|
||||
}
|
||||
|
||||
|
|
@ -111,7 +111,7 @@ TEST_CASE(test_utf16be_decode)
|
|||
auto test_string = "\x00s\x00\xe4\x00k\xd8=\xde\x00"sv;
|
||||
|
||||
EXPECT(decoder.validate(test_string));
|
||||
auto utf8 = MUST(decoder.to_utf8(test_string));
|
||||
auto utf8 = MUST(decoder.to_utf8(test_string, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement));
|
||||
EXPECT_EQ(utf8, "säk😀"sv);
|
||||
}
|
||||
|
||||
|
|
@ -122,12 +122,12 @@ TEST_CASE(test_utf16be_process_code_points)
|
|||
EXPECT_EQ(process_code_points(decoder, StringView(bytes({ 0xfe, 0xff, 0x00, 'A', 0xd8, 0x3d, 0xde, 0x00 }))), (Vector<u32> { 0x41, 0x1F600 }));
|
||||
EXPECT_EQ(process_code_points(decoder, StringView(bytes({ 0xd8, 0x3d, 0x00, 'A', 0xde, 0x00 }))), (Vector<u32> { 0xfffd, 0x41, 0xfffd }));
|
||||
EXPECT_EQ(process_code_points(decoder, StringView(bytes({ 0x00, 'A', 0xff }))), (Vector<u32> { 0x41, 0xfffd }));
|
||||
EXPECT_EQ(MUST(decoder.to_utf8(StringView(bytes({ 0x00, 'A', 0xff })))), "A\xef\xbf\xbd"sv);
|
||||
EXPECT_EQ(MUST(decoder.to_utf8(StringView(bytes({ 0x00, 'A', 0xff })), TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)), "A\xef\xbf\xbd"sv);
|
||||
}
|
||||
|
||||
TEST_CASE(test_streaming_decoder_utf8_mid_sequence)
|
||||
{
|
||||
auto streaming_decoder = TextCodec::StreamingDecoder { "UTF-8"sv };
|
||||
auto streaming_decoder = TextCodec::StreamingDecoder { "UTF-8"sv, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement };
|
||||
|
||||
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 'a', 0xc3 }))), "a"sv);
|
||||
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0xa9, 'b' }))), "éb"sv);
|
||||
|
|
@ -137,27 +137,27 @@ TEST_CASE(test_streaming_decoder_utf8_mid_sequence)
|
|||
TEST_CASE(test_streaming_decoder_finishes_incomplete_sequence)
|
||||
{
|
||||
auto& decoder = decoder_for("UTF-8"sv);
|
||||
auto streaming_decoder = TextCodec::StreamingDecoder { "UTF-8"sv };
|
||||
auto streaming_decoder = TextCodec::StreamingDecoder { "UTF-8"sv, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement };
|
||||
|
||||
auto incomplete_sequence = Vector<u8> { 0xc3 };
|
||||
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes(incomplete_sequence))), ""sv);
|
||||
EXPECT_EQ(MUST(streaming_decoder.finish()), MUST(decoder.to_utf8(StringView(bytes(incomplete_sequence)))));
|
||||
EXPECT_EQ(MUST(streaming_decoder.finish()), MUST(decoder.to_utf8(StringView(bytes(incomplete_sequence)), TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)));
|
||||
}
|
||||
|
||||
TEST_CASE(test_streaming_decoder_utf8_invalid_second_byte_tail)
|
||||
{
|
||||
auto streaming_decoder_for_overlong_three_byte_sequence = TextCodec::StreamingDecoder { "UTF-8"sv };
|
||||
auto streaming_decoder_for_overlong_three_byte_sequence = TextCodec::StreamingDecoder { "UTF-8"sv, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement };
|
||||
EXPECT_EQ(MUST(streaming_decoder_for_overlong_three_byte_sequence.to_utf8(bytes({ 0xe0, 0x80 }))), "\xef\xbf\xbd\xef\xbf\xbd"sv);
|
||||
EXPECT_EQ(MUST(streaming_decoder_for_overlong_three_byte_sequence.finish()), ""sv);
|
||||
|
||||
auto streaming_decoder_for_out_of_range_four_byte_sequence = TextCodec::StreamingDecoder { "UTF-8"sv };
|
||||
auto streaming_decoder_for_out_of_range_four_byte_sequence = TextCodec::StreamingDecoder { "UTF-8"sv, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement };
|
||||
EXPECT_EQ(MUST(streaming_decoder_for_out_of_range_four_byte_sequence.to_utf8(bytes({ 0xf4, 0x90 }))), "\xef\xbf\xbd\xef\xbf\xbd"sv);
|
||||
EXPECT_EQ(MUST(streaming_decoder_for_out_of_range_four_byte_sequence.finish()), ""sv);
|
||||
}
|
||||
|
||||
TEST_CASE(test_streaming_decoder_utf8_valid_second_byte_tail)
|
||||
{
|
||||
auto streaming_decoder = TextCodec::StreamingDecoder { "UTF-8"sv };
|
||||
auto streaming_decoder = TextCodec::StreamingDecoder { "UTF-8"sv, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement };
|
||||
|
||||
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0xe0, 0xa0 }))), ""sv);
|
||||
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0x80 }))), "\xe0\xa0\x80"sv);
|
||||
|
|
@ -166,7 +166,7 @@ TEST_CASE(test_streaming_decoder_utf8_valid_second_byte_tail)
|
|||
|
||||
TEST_CASE(test_streaming_decoder_utf16_odd_byte)
|
||||
{
|
||||
auto streaming_decoder = TextCodec::StreamingDecoder { "UTF-16LE"sv };
|
||||
auto streaming_decoder = TextCodec::StreamingDecoder { "UTF-16LE"sv, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement };
|
||||
|
||||
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0x41, 0x00, 0x42 }))), "A"sv);
|
||||
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0x00 }))), "B"sv);
|
||||
|
|
@ -175,7 +175,7 @@ TEST_CASE(test_streaming_decoder_utf16_odd_byte)
|
|||
|
||||
TEST_CASE(test_streaming_decoder_utf16_surrogate_pair_split)
|
||||
{
|
||||
auto streaming_decoder = TextCodec::StreamingDecoder { "UTF-16BE"sv };
|
||||
auto streaming_decoder = TextCodec::StreamingDecoder { "UTF-16BE"sv, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement };
|
||||
|
||||
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0x00, 0x41, 0xd8, 0x3d }))), "A"sv);
|
||||
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0xde, 0x00 }))), "😀"sv);
|
||||
|
|
@ -185,48 +185,48 @@ TEST_CASE(test_streaming_decoder_utf16_surrogate_pair_split)
|
|||
TEST_CASE(test_streaming_decoder_gb18030_four_byte_tail)
|
||||
{
|
||||
auto& decoder = decoder_for("gb18030"sv);
|
||||
auto streaming_decoder = TextCodec::StreamingDecoder { "gb18030"sv };
|
||||
auto streaming_decoder = TextCodec::StreamingDecoder { "gb18030"sv, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement };
|
||||
auto gb18030_sequence = Vector<u8> { 0x81, 0x30, 0x81, 0x30 };
|
||||
|
||||
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 'a', 0x81, 0x30, 0x81 }))), "a"sv);
|
||||
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0x30 }))), MUST(decoder.to_utf8(StringView(bytes(gb18030_sequence)))));
|
||||
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0x30 }))), MUST(decoder.to_utf8(StringView(bytes(gb18030_sequence)), TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)));
|
||||
EXPECT_EQ(MUST(streaming_decoder.finish()), ""sv);
|
||||
}
|
||||
|
||||
TEST_CASE(test_streaming_decoder_big5_overlapping_trail)
|
||||
{
|
||||
auto& decoder = decoder_for("Big5"sv);
|
||||
auto streaming_decoder = TextCodec::StreamingDecoder { "Big5"sv };
|
||||
auto streaming_decoder = TextCodec::StreamingDecoder { "Big5"sv, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement };
|
||||
auto big5_sequence = Vector<u8> { 0xa4, 0xa4 };
|
||||
|
||||
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes(big5_sequence))), MUST(decoder.to_utf8(StringView(bytes(big5_sequence)))));
|
||||
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes(big5_sequence))), MUST(decoder.to_utf8(StringView(bytes(big5_sequence)), TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)));
|
||||
EXPECT_EQ(MUST(streaming_decoder.finish()), ""sv);
|
||||
|
||||
auto streaming_decoder_for_split_input = TextCodec::StreamingDecoder { "Big5"sv };
|
||||
auto streaming_decoder_for_split_input = TextCodec::StreamingDecoder { "Big5"sv, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement };
|
||||
EXPECT_EQ(MUST(streaming_decoder_for_split_input.to_utf8(bytes({ 0xa4 }))), ""sv);
|
||||
EXPECT_EQ(MUST(streaming_decoder_for_split_input.to_utf8(bytes({ 0xa4 }))), MUST(decoder.to_utf8(StringView(bytes(big5_sequence)))));
|
||||
EXPECT_EQ(MUST(streaming_decoder_for_split_input.to_utf8(bytes({ 0xa4 }))), MUST(decoder.to_utf8(StringView(bytes(big5_sequence)), TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)));
|
||||
EXPECT_EQ(MUST(streaming_decoder_for_split_input.finish()), ""sv);
|
||||
}
|
||||
|
||||
TEST_CASE(test_streaming_decoder_euc_jp_three_byte_tail)
|
||||
{
|
||||
auto& decoder = decoder_for("EUC-JP"sv);
|
||||
auto streaming_decoder = TextCodec::StreamingDecoder { "EUC-JP"sv };
|
||||
auto streaming_decoder = TextCodec::StreamingDecoder { "EUC-JP"sv, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement };
|
||||
auto euc_jp_sequence = Vector<u8> { 0x8f, 0xa2, 0xaf };
|
||||
|
||||
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0x8f, 0xa2 }))), ""sv);
|
||||
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0xaf }))), MUST(decoder.to_utf8(StringView(bytes(euc_jp_sequence)))));
|
||||
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0xaf }))), MUST(decoder.to_utf8(StringView(bytes(euc_jp_sequence)), TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)));
|
||||
EXPECT_EQ(MUST(streaming_decoder.finish()), ""sv);
|
||||
}
|
||||
|
||||
TEST_CASE(test_streaming_decoder_shift_jis_tail)
|
||||
{
|
||||
auto& decoder = decoder_for("Shift_JIS"sv);
|
||||
auto streaming_decoder = TextCodec::StreamingDecoder { "Shift_JIS"sv };
|
||||
auto streaming_decoder = TextCodec::StreamingDecoder { "Shift_JIS"sv, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement };
|
||||
auto shift_jis_sequence = Vector<u8> { 0x82, 0xa0 };
|
||||
|
||||
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0x82 }))), ""sv);
|
||||
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0xa0 }))), MUST(decoder.to_utf8(StringView(bytes(shift_jis_sequence)))));
|
||||
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0xa0 }))), MUST(decoder.to_utf8(StringView(bytes(shift_jis_sequence)), TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)));
|
||||
EXPECT_EQ(MUST(streaming_decoder.finish()), ""sv);
|
||||
}
|
||||
|
||||
|
|
@ -237,7 +237,7 @@ TEST_CASE(test_utf16le_decode)
|
|||
auto test_string = "s\x00\xe4\x00k\x00=\xd8\x00\xde"sv;
|
||||
|
||||
EXPECT(decoder.validate(test_string));
|
||||
auto utf8 = MUST(decoder.to_utf8(test_string));
|
||||
auto utf8 = MUST(decoder.to_utf8(test_string, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement));
|
||||
EXPECT_EQ(utf8, "säk😀"sv);
|
||||
}
|
||||
|
||||
|
|
@ -248,5 +248,5 @@ TEST_CASE(test_utf16le_process_code_points)
|
|||
EXPECT_EQ(process_code_points(decoder, StringView(bytes({ 0xff, 0xfe, 'A', 0x00, 0x3d, 0xd8, 0x00, 0xde }))), (Vector<u32> { 0x41, 0x1F600 }));
|
||||
EXPECT_EQ(process_code_points(decoder, StringView(bytes({ 0x3d, 0xd8, 'A', 0x00, 0x00, 0xde }))), (Vector<u32> { 0xfffd, 0x41, 0xfffd }));
|
||||
EXPECT_EQ(process_code_points(decoder, StringView(bytes({ 'A', 0x00, 0xff }))), (Vector<u32> { 0x41, 0xfffd }));
|
||||
EXPECT_EQ(MUST(decoder.to_utf8(StringView(bytes({ 'A', 0x00, 0xff })))), "A\xef\xbf\xbd"sv);
|
||||
EXPECT_EQ(MUST(decoder.to_utf8(StringView(bytes({ 'A', 0x00, 0xff })), TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)), "A\xef\xbf\xbd"sv);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,16 +2,16 @@ Harness status: OK
|
|||
|
||||
Found 12 tests
|
||||
|
||||
12 Fail
|
||||
Fail ignoreBOM should work for encoding utf-8, split at character 0
|
||||
Fail ignoreBOM should work for encoding utf-8, split at character 1
|
||||
Fail ignoreBOM should work for encoding utf-8, split at character 2
|
||||
Fail ignoreBOM should work for encoding utf-8, split at character 3
|
||||
Fail ignoreBOM should work for encoding utf-16le, split at character 0
|
||||
Fail ignoreBOM should work for encoding utf-16le, split at character 1
|
||||
Fail ignoreBOM should work for encoding utf-16le, split at character 2
|
||||
Fail ignoreBOM should work for encoding utf-16le, split at character 3
|
||||
Fail ignoreBOM should work for encoding utf-16be, split at character 0
|
||||
Fail ignoreBOM should work for encoding utf-16be, split at character 1
|
||||
Fail ignoreBOM should work for encoding utf-16be, split at character 2
|
||||
Fail ignoreBOM should work for encoding utf-16be, split at character 3
|
||||
12 Pass
|
||||
Pass ignoreBOM should work for encoding utf-8, split at character 0
|
||||
Pass ignoreBOM should work for encoding utf-8, split at character 1
|
||||
Pass ignoreBOM should work for encoding utf-8, split at character 2
|
||||
Pass ignoreBOM should work for encoding utf-8, split at character 3
|
||||
Pass ignoreBOM should work for encoding utf-16le, split at character 0
|
||||
Pass ignoreBOM should work for encoding utf-16le, split at character 1
|
||||
Pass ignoreBOM should work for encoding utf-16le, split at character 2
|
||||
Pass ignoreBOM should work for encoding utf-16le, split at character 3
|
||||
Pass ignoreBOM should work for encoding utf-16be, split at character 0
|
||||
Pass ignoreBOM should work for encoding utf-16be, split at character 1
|
||||
Pass ignoreBOM should work for encoding utf-16be, split at character 2
|
||||
Pass ignoreBOM should work for encoding utf-16be, split at character 3
|
||||
|
|
|
|||
|
|
@ -2,9 +2,8 @@ Harness status: OK
|
|||
|
||||
Found 4 tests
|
||||
|
||||
1 Pass
|
||||
3 Fail
|
||||
Fail BOM is ignored if ignoreBOM option is specified: utf-8
|
||||
Fail BOM is ignored if ignoreBOM option is specified: utf-16le
|
||||
Fail BOM is ignored if ignoreBOM option is specified: utf-16be
|
||||
4 Pass
|
||||
Pass BOM is ignored if ignoreBOM option is specified: utf-8
|
||||
Pass BOM is ignored if ignoreBOM option is specified: utf-16le
|
||||
Pass BOM is ignored if ignoreBOM option is specified: utf-16be
|
||||
Pass The ignoreBOM attribute of TextDecoder
|
||||
|
|
@ -2,8 +2,8 @@ Harness status: OK
|
|||
|
||||
Found 87 tests
|
||||
|
||||
72 Pass
|
||||
15 Fail
|
||||
74 Pass
|
||||
13 Fail
|
||||
Pass Invalid Unicode input is replaced: utf-8
|
||||
Pass Invalid Unicode input is replaced: utf-16le
|
||||
Pass Invalid Unicode input is replaced: utf-16be
|
||||
|
|
@ -51,8 +51,8 @@ Pass selected single-byte: iso-8859-8-i
|
|||
Pass selected single-byte: iso-8859-16
|
||||
Pass selected single-byte: x-mac-cyrillic
|
||||
Pass Concatenating two ISO-2022-JP outputs is not always valid
|
||||
Fail gb18030 version and ranges
|
||||
Fail gbk version and ranges
|
||||
Pass gb18030 version and ranges
|
||||
Pass gbk version and ranges
|
||||
Pass gbk decoder is gb18030 decoder
|
||||
Pass Replacement, push back ASCII characters: big5
|
||||
Pass Replacement, push back ASCII characters: iso-2022-jp
|
||||
|
|
|
|||
Loading…
Reference in a new issue