Compare commits

...

10 commits

Author SHA1 Message Date
b41536f4ef ci(forgejo): add forgejo actions
Some checks failed
Build Ladybird (Ubuntu 24.04) / Build on Ubuntu 24.04 (push) Failing after 10s
2026-06-23 11:02:47 -03:00
sideshowbarker
4af1b9357b LibWeb: Don't crash when evaluating XPath against a rootless document
Problem: Crash when evaluating an XPath expression against a document
that has no root element.

Cause: mirror_node()’s DOCUMENT_NODE branch unconditionally mirrored
document.document_element(). But a document may lack a root element —
in which case, document_element() returns null.

Fix: Return null from mirror_node() when the document has no root
element. (The caller already maps a null mirror result to a failed
evaluation, and raises an error in that case.)

Fixes https://github.com/LadybirdBrowser/ladybird/issues/10004
2026-06-23 08:59:01 +02:00
Timothy Flynn
a7ce007ff4 LibWeb: Define name and length properties on exported wasm functions
We were previously not setting any name or length properties on exported
functions. Some sites like https://squeel.frankmayer.dev/ rely on these
properties.

The name property is a bit strange. Exported wasm functions have a name
property which is its index in its module instance's function-address
list.

This patch is not enough for the squeel site to be fully functional; we
will need to implement navigator.locks.
2026-06-23 08:57:08 +02:00
Shannon Booth
d4048aaa96 LibTextCodec: Remove decoder validation API
All users instead rely on the 'Fatal' option being passed through.
2026-06-23 07:25:11 +02:00
Shannon Booth
b9da74d16e LibWeb: Use fatal decoding when loading XML documents
Decode XML document bytes with TextCodec::ErrorMode::Fatal and treat
decode failure as an XML loading error.

This removes the separate decoder validation pass and lets the decode
operation itself enforce XML's requirement that documents contain only
properly encoded characters.
2026-06-23 07:25:11 +02:00
Shannon Booth
ef6753a9f9 LibWeb+LibTextCodec: Wire decoder options through TextDecoder
Add explicit IgnoreBOM and ErrorMode options to LibTextCodec decoders,
and thread them through TextDecoder and TextDecoderStream.

This lets Web-facing decoder APIs preserve BOMs when requested and use
fatal error handling without post-processing decoded output.

NB: RemoveBOM was renamed to IgnoreBOM as "RemoveBOM" is the name
used by encoding_rs and was previously an implementation detail.
The new name matches what is used by the encoding standard as it
is now also used in LibWeb.
2026-06-23 07:25:11 +02:00
Shannon Booth
ef99632fa7 Tests/LibWeb: Add some encoding options related WPT tests 2026-06-23 07:25:11 +02:00
Shannon Booth
02320c9b58 LibWeb: Run text encoder/decoder stream algorithms in their realm
Temporarily enter the TextEncoderStream/TextDecoderStream realm while
running their transform and flush algorithms.

This ensures objects and exceptions created through those algorithms are
associated with the constructor realm, matching the encoding streams
realm WPT.

I find this behaviour _somewhat_ strange, and this is only very loosely
specified, but all browsers have aligned on this behviour, so we may
as well match it.
2026-06-23 07:25:11 +02:00
Shannon Booth
ae0c7bc097 LibWeb/Encoding: Dont return ExceptionOr for encoder constructors
No exception is thrown.
2026-06-23 07:25:11 +02:00
Shannon Booth
8caee39bac LibWeb/Encoding: Take non-nullable options in TextDecoder construction
Also change to StringView instead of FlyString as we were not
making use of FlyStrings properties here.
2026-06-23 07:25:11 +02:00
39 changed files with 527 additions and 278 deletions

View file

@ -0,0 +1,112 @@
name: Build Ladybird (Ubuntu 24.04)
on:
push:
branches: [ master ]
pull_request:
branches: [ master ]
workflow_dispatch:
env:
LADYBIRD_SOURCE_DIR: ${{ github.workspace }}
CCACHE_DIR: ${{ github.workspace }}/.ccache
VCPKG_ROOT: ${{ github.workspace }}/Build/vcpkg
jobs:
build:
name: Build on Ubuntu 24.04
runs-on: ubuntu-latest
container:
image: ubuntu:24.04
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Install system dependencies
run: |
apt-get update && apt-get install -y \
software-properties-common \
wget \
curl \
gpg \
lsb-release \
git
# Install CMake 3.30+ from Kitware (since Ubuntu 24.04 has 3.28)
wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | tee /usr/share/keyrings/kitware-archive-keyring.gpg >/dev/null
echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ noble main" | tee /etc/apt/sources.list.d/kitware.list
apt-get update && apt-get install -y \
autoconf \
autoconf-archive \
automake \
build-essential \
ccache \
cmake \
fonts-liberation2 \
glslang-tools \
libdrm-dev \
libgl1-mesa-dev \
libncurses-dev \
libtool \
nasm \
ninja-build \
pkg-config \
python3-venv \
qt6-base-dev \
qt6-tools-dev-tools \
qt6-wayland \
tar \
unzip \
zip \
libpulse-dev \
gcc-14 \
g++-14
- name: Configure compiler alternatives
run: |
update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-14 100 \
--slave /usr/bin/g++ g++ /usr/bin/g++-14 \
--slave /usr/bin/gcov gcov /usr/bin/gcov-14
- name: Install Rust
run: |
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain 1.96.0
echo "$HOME/.cargo/bin" >> $GITHUB_PATH
- name: Configure Git Safe Directory
run: |
git config --global safe.directory '*'
- name: Bootstrap vcpkg
run: |
./Meta/Utils/build_vcpkg.py
- name: Configure CMake
env:
CC: gcc-14
CXX: g++-14
run: |
cmake --preset Release -B Build \
-DCMAKE_C_COMPILER=gcc-14 \
-DCMAKE_CXX_COMPILER=g++-14 \
-DENABLE_CI_BASELINE_CPU=ON \
-DLADYBIRD_GUI_FRAMEWORK=Qt
- name: Build
run: |
cmake --build Build
- name: Install
run: |
cmake --install Build --strip --prefix dist
- name: Package Binaries
run: |
tar -czf ladybird-ubuntu-24.04.tar.gz -C dist .
- name: Upload Build Artifacts
uses: actions/upload-artifact@v4
with:
name: ladybird-ubuntu-24.04
path: ladybird-ubuntu-24.04.tar.gz

View file

@ -15,15 +15,8 @@
namespace TextCodec {
static constexpr u32 replacement_code_point = 0xfffd;
namespace {
enum class RemoveBOM {
No,
Yes,
};
class RustDecoder final : public Decoder {
public:
explicit RustDecoder(StringView encoding)
@ -31,8 +24,7 @@ public:
{
}
virtual bool validate(StringView input) override;
virtual ErrorOr<String> to_utf8(StringView input) override;
virtual ErrorOr<String> to_utf8(StringView input, IgnoreBOM, ErrorMode) override;
virtual ErrorOr<size_t> length_in_utf16_code_units(StringView input) override;
private:
@ -44,15 +36,13 @@ private:
class UTF8Decoder final : public Decoder {
public:
virtual ErrorOr<void> process(StringView, Function<ErrorOr<void>(u32)> on_code_point) override;
virtual bool validate(StringView) override;
virtual ErrorOr<String> to_utf8(StringView) override;
virtual ErrorOr<String> to_utf8(StringView, IgnoreBOM, ErrorMode) override;
virtual ErrorOr<size_t> length_in_utf16_code_units(StringView) override;
};
class UTF16BEDecoder final : public Decoder {
public:
virtual bool validate(StringView) override;
virtual ErrorOr<String> to_utf8(StringView) override;
virtual ErrorOr<String> to_utf8(StringView, IgnoreBOM, ErrorMode) override;
virtual ErrorOr<size_t> length_in_utf16_code_units(StringView) override;
private:
@ -61,8 +51,7 @@ private:
class UTF16LEDecoder final : public Decoder {
public:
virtual bool validate(StringView) override;
virtual ErrorOr<String> to_utf8(StringView) override;
virtual ErrorOr<String> to_utf8(StringView, IgnoreBOM, ErrorMode) override;
virtual ErrorOr<size_t> length_in_utf16_code_units(StringView) override;
private:
@ -72,7 +61,6 @@ private:
class Latin1Decoder final : public Decoder {
public:
virtual ErrorOr<void> process(StringView, Function<ErrorOr<void>(u32)> on_code_point) override;
virtual bool validate(StringView) override { return true; }
virtual ErrorOr<size_t> length_in_utf16_code_units(StringView) override;
};
@ -130,7 +118,7 @@ static void append_decoded_bytes(void* context, u8 const* data, size_t length)
decode_context.result = decode_context.builder.try_append(StringView { data, length });
}
ErrorOr<String> rust_decode_to_utf8(StringView encoding, StringView input, RemoveBOM remove_bom)
ErrorOr<String> rust_decode_to_utf8(StringView encoding, StringView input, IgnoreBOM ignore_bom, ErrorMode error_mode)
{
DecodeContext context { .builder = StringBuilder(input.length()) };
auto succeeded = FFI::textcodec_rust_decode_to_utf8(
@ -138,36 +126,27 @@ ErrorOr<String> rust_decode_to_utf8(StringView encoding, StringView input, Remov
encoding.length(),
reinterpret_cast<u8 const*>(input.characters_without_null_termination()),
input.length(),
remove_bom == RemoveBOM::Yes,
ignore_bom == IgnoreBOM::No,
error_mode == ErrorMode::Fatal,
&context,
append_decoded_bytes);
if (!succeeded)
return Error::from_errno(EINVAL);
return Error::from_string_literal("Failed to decode input");
TRY(context.result);
return context.builder.to_string_without_validation();
}
ErrorOr<void> rust_process(StringView encoding, StringView input, RemoveBOM remove_bom, Function<ErrorOr<void>(u32)> on_code_point)
ErrorOr<void> rust_process(StringView encoding, StringView input, IgnoreBOM ignore_bom, Function<ErrorOr<void>(u32)> on_code_point)
{
auto utf8 = TRY(rust_decode_to_utf8(encoding, input, remove_bom));
auto utf8 = TRY(rust_decode_to_utf8(encoding, input, ignore_bom, ErrorMode::Replacement));
for (auto code_point : Utf8View { utf8 })
TRY(on_code_point(code_point));
return {};
}
bool rust_validate(StringView encoding, StringView input, RemoveBOM remove_bom)
ErrorOr<size_t> rust_length_in_utf16_code_units(StringView encoding, StringView input, IgnoreBOM ignore_bom)
{
return FFI::textcodec_rust_validate(
reinterpret_cast<u8 const*>(encoding.characters_without_null_termination()),
encoding.length(),
reinterpret_cast<u8 const*>(input.characters_without_null_termination()),
input.length(),
remove_bom == RemoveBOM::Yes);
}
ErrorOr<size_t> rust_length_in_utf16_code_units(StringView encoding, StringView input, RemoveBOM remove_bom)
{
auto utf8 = TRY(rust_decode_to_utf8(encoding, input, remove_bom));
auto utf8 = TRY(rust_decode_to_utf8(encoding, input, ignore_bom, ErrorMode::Replacement));
size_t length = 0;
for (auto code_point : Utf8View { utf8 })
length += code_point <= 0xffff ? 1 : 2;
@ -188,7 +167,7 @@ Optional<StringView> get_static_encoding_name_from_rust(StringView label)
return StringView { encoding_name, encoding_name_length };
}
ErrorOr<String> rust_streaming_decode_to_utf8(FFI::TextCodecRustStreamingDecoder* decoder, ReadonlyBytes input, bool last)
ErrorOr<String> rust_streaming_decode_to_utf8(FFI::TextCodecRustStreamingDecoder* decoder, ReadonlyBytes input, bool last, ErrorMode error_mode)
{
DecodeContext context { .builder = StringBuilder(input.size()) };
auto succeeded = FFI::textcodec_rust_streaming_decoder_decode_to_utf8(
@ -196,10 +175,11 @@ ErrorOr<String> rust_streaming_decode_to_utf8(FFI::TextCodecRustStreamingDecoder
input.data(),
input.size(),
last,
error_mode == ErrorMode::Fatal,
&context,
append_decoded_bytes);
if (!succeeded)
return Error::from_errno(EINVAL);
return Error::from_string_literal("Failed to decode input");
TRY(context.result);
return context.builder.to_string_without_validation();
}
@ -367,7 +347,7 @@ ErrorOr<String> convert_input_to_utf8_using_given_decoder_unless_there_is_a_byte
// 3. Process a queue with an instance of encodings decoder, ioQueue, output, and "replacement".
// FIXME: This isn't the exact same as the spec, which is written in terms of I/O queues.
auto output = TRY(actual_decoder->to_utf8(input));
auto output = TRY(actual_decoder->to_utf8(input, IgnoreBOM::No, ErrorMode::Replacement));
// 4. Return output.
return output;
@ -410,18 +390,7 @@ StringView get_output_encoding(StringView encoding)
return encoding;
}
bool Decoder::validate(StringView input)
{
auto result = this->process(input, [](auto code_point) -> ErrorOr<void> {
if (code_point == replacement_code_point)
return Error::from_errno(EINVAL);
return {};
});
return !result.is_error();
}
ErrorOr<String> Decoder::to_utf8(StringView input)
ErrorOr<String> Decoder::to_utf8(StringView input, IgnoreBOM, ErrorMode)
{
StringBuilder builder(input.length());
TRY(process(input, [&builder](u32 c) { return builder.try_append_code_point(c); }));
@ -453,24 +422,19 @@ ErrorOr<void> Decoder::process_code_points(StringView input, Function<ErrorOr<vo
return process(input, move(on_code_point));
}
bool RustDecoder::validate(StringView input)
ErrorOr<String> RustDecoder::to_utf8(StringView input, IgnoreBOM ignore_bom, ErrorMode error_mode)
{
return rust_validate(m_encoding, input, RemoveBOM::No);
}
ErrorOr<String> RustDecoder::to_utf8(StringView input)
{
return rust_decode_to_utf8(m_encoding, input, RemoveBOM::No);
return rust_decode_to_utf8(m_encoding, input, ignore_bom, error_mode);
}
ErrorOr<size_t> RustDecoder::length_in_utf16_code_units(StringView input)
{
return rust_length_in_utf16_code_units(m_encoding, input, RemoveBOM::No);
return rust_length_in_utf16_code_units(m_encoding, input, IgnoreBOM::Yes);
}
ErrorOr<void> RustDecoder::process(StringView input, Function<ErrorOr<void>(u32)> on_code_point)
{
return rust_process(m_encoding, input, RemoveBOM::No, move(on_code_point));
return rust_process(m_encoding, input, IgnoreBOM::Yes, move(on_code_point));
}
ErrorOr<void> Latin1Decoder::process(StringView input, Function<ErrorOr<void>(u32)> on_code_point)
@ -485,12 +449,13 @@ ErrorOr<size_t> Latin1Decoder::length_in_utf16_code_units(StringView input)
return input.length();
}
StreamingDecoder::StreamingDecoder(StringView encoding)
StreamingDecoder::StreamingDecoder(StringView encoding, IgnoreBOM ignore_bom, ErrorMode error_mode)
: m_error_mode(error_mode)
{
m_decoder = FFI::textcodec_rust_streaming_decoder_new(
reinterpret_cast<u8 const*>(encoding.characters_without_null_termination()),
encoding.length(),
true);
ignore_bom == IgnoreBOM::No);
VERIFY(m_decoder);
}
@ -501,72 +466,57 @@ StreamingDecoder::~StreamingDecoder()
ErrorOr<String> StreamingDecoder::to_utf8(ReadonlyBytes input)
{
return rust_streaming_decode_to_utf8(static_cast<FFI::TextCodecRustStreamingDecoder*>(m_decoder), input, false);
return rust_streaming_decode_to_utf8(static_cast<FFI::TextCodecRustStreamingDecoder*>(m_decoder), input, false, m_error_mode);
}
ErrorOr<String> StreamingDecoder::finish()
{
return rust_streaming_decode_to_utf8(static_cast<FFI::TextCodecRustStreamingDecoder*>(m_decoder), {}, true);
return rust_streaming_decode_to_utf8(static_cast<FFI::TextCodecRustStreamingDecoder*>(m_decoder), {}, true, m_error_mode);
}
ErrorOr<void> UTF8Decoder::process(StringView input, Function<ErrorOr<void>(u32)> on_code_point)
{
return rust_process("UTF-8"sv, input, RemoveBOM::No, move(on_code_point));
return rust_process("UTF-8"sv, input, IgnoreBOM::Yes, move(on_code_point));
}
bool UTF8Decoder::validate(StringView input)
ErrorOr<String> UTF8Decoder::to_utf8(StringView input, IgnoreBOM ignore_bom, ErrorMode error_mode)
{
return rust_validate("UTF-8"sv, input, RemoveBOM::No);
}
ErrorOr<String> UTF8Decoder::to_utf8(StringView input)
{
return rust_decode_to_utf8("UTF-8"sv, input, RemoveBOM::Yes);
return rust_decode_to_utf8("UTF-8"sv, input, ignore_bom, error_mode);
}
ErrorOr<size_t> UTF8Decoder::length_in_utf16_code_units(StringView input)
{
return rust_length_in_utf16_code_units("UTF-8"sv, input, RemoveBOM::Yes);
}
bool UTF16BEDecoder::validate(StringView input)
{
return rust_validate("UTF-16BE"sv, input, RemoveBOM::No);
return rust_length_in_utf16_code_units("UTF-8"sv, input, IgnoreBOM::No);
}
ErrorOr<void> UTF16BEDecoder::process(StringView input, Function<ErrorOr<void>(u32)> on_code_point)
{
return rust_process("UTF-16BE"sv, input, RemoveBOM::Yes, move(on_code_point));
return rust_process("UTF-16BE"sv, input, IgnoreBOM::No, move(on_code_point));
}
ErrorOr<String> UTF16BEDecoder::to_utf8(StringView input)
ErrorOr<String> UTF16BEDecoder::to_utf8(StringView input, IgnoreBOM ignore_bom, ErrorMode error_mode)
{
return rust_decode_to_utf8("UTF-16BE"sv, input, RemoveBOM::Yes);
return rust_decode_to_utf8("UTF-16BE"sv, input, ignore_bom, error_mode);
}
ErrorOr<size_t> UTF16BEDecoder::length_in_utf16_code_units(StringView input)
{
return rust_length_in_utf16_code_units("UTF-16BE"sv, input, RemoveBOM::Yes);
}
bool UTF16LEDecoder::validate(StringView input)
{
return rust_validate("UTF-16LE"sv, input, RemoveBOM::No);
return rust_length_in_utf16_code_units("UTF-16BE"sv, input, IgnoreBOM::No);
}
ErrorOr<void> UTF16LEDecoder::process(StringView input, Function<ErrorOr<void>(u32)> on_code_point)
{
return rust_process("UTF-16LE"sv, input, RemoveBOM::Yes, move(on_code_point));
return rust_process("UTF-16LE"sv, input, IgnoreBOM::No, move(on_code_point));
}
ErrorOr<String> UTF16LEDecoder::to_utf8(StringView input)
ErrorOr<String> UTF16LEDecoder::to_utf8(StringView input, IgnoreBOM ignore_bom, ErrorMode error_mode)
{
return rust_decode_to_utf8("UTF-16LE"sv, input, RemoveBOM::Yes);
return rust_decode_to_utf8("UTF-16LE"sv, input, ignore_bom, error_mode);
}
ErrorOr<size_t> UTF16LEDecoder::length_in_utf16_code_units(StringView input)
{
return rust_length_in_utf16_code_units("UTF-16LE"sv, input, RemoveBOM::Yes);
return rust_length_in_utf16_code_units("UTF-16LE"sv, input, IgnoreBOM::No);
}
// https://infra.spec.whatwg.org/#isomorphic-decode

View file

@ -19,10 +19,20 @@
namespace TextCodec {
enum class IgnoreBOM {
Yes,
No,
};
// https://encoding.spec.whatwg.org/#concept-encoding-error-mode
enum class ErrorMode {
Replacement,
Fatal,
};
class TEXTCODEC_API Decoder {
public:
virtual bool validate(StringView);
virtual ErrorOr<String> to_utf8(StringView);
virtual ErrorOr<String> to_utf8(StringView, IgnoreBOM, ErrorMode);
virtual ErrorOr<Utf16String> to_utf16(StringView);
virtual ErrorOr<size_t> length_in_utf16_code_units(StringView);
ErrorOr<void> process_code_points(StringView, Function<ErrorOr<void>(u32)>);
@ -36,13 +46,14 @@ class TEXTCODEC_API StreamingDecoder final {
AK_MAKE_NONCOPYABLE(StreamingDecoder);
public:
explicit StreamingDecoder(StringView encoding);
StreamingDecoder(StringView encoding, IgnoreBOM, ErrorMode);
~StreamingDecoder();
ErrorOr<String> to_utf8(ReadonlyBytes);
ErrorOr<String> finish();
private:
ErrorMode m_error_mode { ErrorMode::Replacement };
void* m_decoder { nullptr };
};

View file

@ -93,6 +93,7 @@ pub unsafe extern "C" fn textcodec_rust_decode_to_utf8(
input: *const u8,
input_len: usize,
remove_bom: bool,
fatal: bool,
ctx: *mut c_void,
on_bytes: FfiBytesFn,
) -> bool {
@ -108,60 +109,20 @@ pub unsafe extern "C" fn textcodec_rust_decode_to_utf8(
return false;
};
let (output, _) = if remove_bom {
let (output, had_errors) = if remove_bom {
encoding.decode_with_bom_removal(input)
} else {
encoding.decode_without_bom_handling(input)
};
if fatal && had_errors {
return false;
}
on_bytes(ctx, output.as_bytes().as_ptr(), output.len());
true
})
}
}
/// # Safety
/// - `encoding_label`/`encoding_label_len` and `input`/`input_len` must be valid byte slices.
#[unsafe(no_mangle)]
pub unsafe extern "C" fn textcodec_rust_validate(
encoding_label: *const u8,
encoding_label_len: usize,
input: *const u8,
input_len: usize,
remove_bom: bool,
) -> bool {
unsafe {
abort_on_panic(|| {
let Some(label) = bytes_from_raw(encoding_label, encoding_label_len) else {
return false;
};
let Some(input) = bytes_from_raw(input, input_len) else {
return false;
};
let Some(encoding) = Encoding::for_label(label) else {
return false;
};
let input = if remove_bom {
if encoding == encoding_rs::UTF_8 && input.starts_with(b"\xEF\xBB\xBF") {
&input[3..]
} else if (encoding == encoding_rs::UTF_16LE && input.starts_with(b"\xFF\xFE"))
|| (encoding == encoding_rs::UTF_16BE && input.starts_with(b"\xFE\xFF"))
{
&input[2..]
} else {
input
}
} else {
input
};
encoding
.decode_without_bom_handling_and_without_replacement(input)
.is_some()
})
}
}
/// # Safety
/// - `encoding_label`/`encoding_label_len` must be a valid byte slice.
/// - The returned pointer must be freed with `textcodec_rust_streaming_decoder_free`.
@ -212,6 +173,7 @@ pub unsafe extern "C" fn textcodec_rust_streaming_decoder_decode_to_utf8(
input: *const u8,
input_len: usize,
last: bool,
fatal: bool,
ctx: *mut c_void,
on_bytes: FfiBytesFn,
) -> bool {
@ -230,7 +192,10 @@ pub unsafe extern "C" fn textcodec_rust_streaming_decoder_decode_to_utf8(
};
let mut output = String::with_capacity(output_capacity);
let (result, _, _) = decoder.decoder.decode_to_string(input, &mut output, last);
let (result, _, had_errors) = decoder.decoder.decode_to_string(input, &mut output, last);
if fatal && had_errors {
return false;
}
if !output.is_empty() {
on_bytes(ctx, output.as_ptr(), output.len());
}

View file

@ -36,7 +36,7 @@ static String decode_and_filter_code_points(StringView input, StringView encodin
input = input.substring_view(3);
return String::from_utf8_without_validation(input.bytes());
}
return MUST(decoder->to_utf8(input));
return MUST(decoder->to_utf8(input, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement));
}();
// OPTIMIZATION: If the input doesn't contain any filterable characters, we can skip the filtering

View file

@ -175,7 +175,7 @@ Vector<Token> Tokenizer::tokenize(StringView input, StringView encoding, Tokeniz
input = input.substring_view(3);
return String::from_utf8_without_validation(input.bytes());
}
return MUST(decoder->to_utf8(input));
return MUST(decoder->to_utf8(input, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement));
}();
// OPTIMIZATION: If the input doesn't contain any filterable characters, we can skip the filtering

View file

@ -73,11 +73,12 @@ bool build_xml_document(DOM::Document& document, ByteBuffer const& data, Optiona
}
VERIFY(decoder.has_value());
// Well-formed XML documents contain only properly encoded characters
if (!decoder->validate(data)) {
auto source_or_error = decoder->to_utf8(data, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Fatal);
if (source_or_error.is_error()) {
convert_to_xml_error_document(document, "XML Document contains improperly-encoded characters"_utf16);
return false;
}
auto source = decoder->to_utf8(data).release_value_but_fixme_should_propagate_errors();
auto source = source_or_error.release_value();
XML::Parser parser(source, { .resolve_named_html_entity = resolve_named_html_entity });
XMLDocumentBuilder builder { document };
auto result = parser.parse_with_listener(builder);
@ -198,13 +199,7 @@ static WebIDL::ExceptionOr<GC::Ref<DOM::Document>> load_xml_document(HTML::Navig
}
VERIFY(decoder.has_value());
// Well-formed XML documents contain only properly encoded characters
if (!decoder->validate(data)) {
// FIXME: Insert error message into the document.
dbgln("XML Document contains improperly-encoded characters");
convert_to_xml_error_document(document, "XML Document contains improperly-encoded characters"_utf16);
return;
}
auto source = decoder->to_utf8(data);
auto source = decoder->to_utf8(data, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Fatal);
if (source.is_error()) {
// FIXME: Insert error message into the document.
dbgln("Failed to decode XML document: {}", source.error());

View file

@ -18,7 +18,7 @@ namespace Web::Encoding {
GC_DEFINE_ALLOCATOR(TextDecoder);
// https://encoding.spec.whatwg.org/#dom-textdecoder
WebIDL::ExceptionOr<GC::Ref<TextDecoder>> TextDecoder::construct_impl(JS::Realm& realm, FlyString label, Optional<Bindings::TextDecoderOptions> const& options)
WebIDL::ExceptionOr<GC::Ref<TextDecoder>> TextDecoder::construct_impl(JS::Realm& realm, StringView label, Bindings::TextDecoderOptions const& options)
{
auto& vm = realm.vm();
@ -35,10 +35,10 @@ WebIDL::ExceptionOr<GC::Ref<TextDecoder>> TextDecoder::construct_impl(JS::Realm&
auto lowercase_encoding_name = encoding.value().to_ascii_lowercase_string();
// 4. If options["fatal"] is true, then set thiss error mode to "fatal".
auto error_mode = options.value_or({}).fatal ? ErrorMode::Fatal : ErrorMode::Replacement;
auto error_mode = options.fatal ? TextCodec::ErrorMode::Fatal : TextCodec::ErrorMode::Replacement;
// 5. Set thiss ignore BOM to options["ignoreBOM"].
auto ignore_bom = options.value_or({}).ignore_bom;
auto ignore_bom = options.ignore_bom;
// NOTE: This should happen in decode(), but we don't support streaming yet and share decoders across calls.
auto decoder = TextCodec::decoder_for_exact_name(encoding.value());
@ -48,7 +48,7 @@ WebIDL::ExceptionOr<GC::Ref<TextDecoder>> TextDecoder::construct_impl(JS::Realm&
}
// https://encoding.spec.whatwg.org/#dom-textdecoder
TextDecoder::TextDecoder(JS::Realm& realm, TextCodec::Decoder& decoder, FlyString encoding, ErrorMode error_mode, bool ignore_bom)
TextDecoder::TextDecoder(JS::Realm& realm, TextCodec::Decoder& decoder, FlyString encoding, TextCodec::ErrorMode error_mode, bool ignore_bom)
: PlatformObject(realm)
, TextDecoderCommonMixin(decoder, move(encoding), error_mode, ignore_bom)
{
@ -65,18 +65,23 @@ void TextDecoder::initialize(JS::Realm& realm)
// https://encoding.spec.whatwg.org/#dom-textdecoder-decode
WebIDL::ExceptionOr<String> TextDecoder::decode(Optional<WebIDL::BufferSourceVariant> input, Optional<Bindings::TextDecodeOptions> const&) const
{
if (!input.has_value())
return TRY_OR_THROW_OOM(vm(), m_decoder.to_utf8({}));
auto ignore_bom = m_ignore_bom ? TextCodec::IgnoreBOM::Yes : TextCodec::IgnoreBOM::No;
if (!input.has_value()) {
auto result = m_decoder.to_utf8({}, ignore_bom, m_error_mode);
if (result.is_error() && result.error().code() != ENOMEM)
return WebIDL::SimpleException { WebIDL::SimpleExceptionType::TypeError, "Decoding failed"sv };
return TRY_OR_THROW_OOM(vm(), move(result));
}
// FIXME: Implement the streaming stuff.
auto data_buffer_or_error = WebIDL::get_buffer_source_copy(*input);
if (data_buffer_or_error.is_error())
return WebIDL::OperationError::create(realm(), "Failed to copy bytes from ArrayBuffer"_utf16);
auto& data_buffer = data_buffer_or_error.value();
auto result = TRY_OR_THROW_OOM(vm(), m_decoder.to_utf8({ data_buffer.data(), data_buffer.size() }));
if (this->fatal() && result.contains(0xfffd))
auto result = m_decoder.to_utf8({ data_buffer.data(), data_buffer.size() }, ignore_bom, m_error_mode);
if (result.is_error() && result.error().code() != ENOMEM)
return WebIDL::SimpleException { WebIDL::SimpleExceptionType::TypeError, "Decoding failed"sv };
return result;
return TRY_OR_THROW_OOM(vm(), move(result));
}
}

View file

@ -27,14 +27,14 @@ class TextDecoder
GC_DECLARE_ALLOCATOR(TextDecoder);
public:
static WebIDL::ExceptionOr<GC::Ref<TextDecoder>> construct_impl(JS::Realm&, FlyString encoding, Optional<Bindings::TextDecoderOptions> const& options = {});
static WebIDL::ExceptionOr<GC::Ref<TextDecoder>> construct_impl(JS::Realm&, StringView label, Bindings::TextDecoderOptions const&);
virtual ~TextDecoder() override;
WebIDL::ExceptionOr<String> decode(Optional<WebIDL::BufferSourceVariant>, Optional<Bindings::TextDecodeOptions> const& options = {}) const;
private:
TextDecoder(JS::Realm&, TextCodec::Decoder&, FlyString encoding, ErrorMode error_mode, bool ignore_bom);
TextDecoder(JS::Realm&, TextCodec::Decoder&, FlyString encoding, TextCodec::ErrorMode error_mode, bool ignore_bom);
virtual void initialize(JS::Realm&) override;
};

View file

@ -8,7 +8,7 @@
namespace Web::Encoding {
TextDecoderCommonMixin::TextDecoderCommonMixin(TextCodec::Decoder& decoder, FlyString encoding, ErrorMode error_mode, bool ignore_bom)
TextDecoderCommonMixin::TextDecoderCommonMixin(TextCodec::Decoder& decoder, FlyString encoding, TextCodec::ErrorMode error_mode, bool ignore_bom)
: m_decoder(decoder)
, m_encoding(move(encoding))
, m_error_mode(error_mode)

View file

@ -7,7 +7,7 @@
#pragma once
#include <AK/FlyString.h>
#include <LibTextCodec/Forward.h>
#include <LibTextCodec/Decoder.h>
namespace Web::Encoding {
@ -20,19 +20,13 @@ public:
FlyString const& encoding() const { return m_encoding; }
// https://encoding.spec.whatwg.org/#dom-textdecoder-fatal
bool fatal() const { return m_error_mode == ErrorMode::Fatal; }
bool fatal() const { return m_error_mode == TextCodec::ErrorMode::Fatal; }
// https://encoding.spec.whatwg.org/#dom-textdecoder-ignorebom
bool ignore_bom() const { return m_ignore_bom; }
protected:
// https://encoding.spec.whatwg.org/#concept-encoding-error-mode
enum class ErrorMode {
Replacement,
Fatal,
};
TextDecoderCommonMixin(TextCodec::Decoder& decoder, FlyString encoding, ErrorMode error_mode, bool ignore_bom);
TextDecoderCommonMixin(TextCodec::Decoder& decoder, FlyString encoding, TextCodec::ErrorMode error_mode, bool ignore_bom);
// https://encoding.spec.whatwg.org/#textdecodercommon-decoder
TextCodec::Decoder& m_decoder;
@ -41,13 +35,10 @@ protected:
FlyString m_encoding;
// https://encoding.spec.whatwg.org/#textdecoder-error-mode
ErrorMode m_error_mode { ErrorMode::Replacement };
TextCodec::ErrorMode m_error_mode { TextCodec::ErrorMode::Replacement };
// https://encoding.spec.whatwg.org/#textdecoder-ignore-bom-flag
bool m_ignore_bom { false };
// https://encoding.spec.whatwg.org/#textdecoder-bom-seen-flag
bool m_bom_seen { false };
};
}

View file

@ -15,6 +15,7 @@
#include <LibWeb/Bindings/TextDecoder.h>
#include <LibWeb/Bindings/TextDecoderStream.h>
#include <LibWeb/Encoding/TextDecoderStream.h>
#include <LibWeb/HTML/Scripting/TemporaryExecutionContext.h>
#include <LibWeb/Streams/TransformStream.h>
#include <LibWeb/Streams/TransformStreamOperations.h>
#include <LibWeb/WebIDL/AbstractOperations.h>
@ -38,7 +39,7 @@ WebIDL::ExceptionOr<GC::Ref<TextDecoderStream>> TextDecoderStream::construct_imp
auto lowercase_encoding_name = encoding.value().to_ascii_lowercase_string();
// 4. If options["fatal"] is true, then set thiss error mode to "fatal".
auto error_mode = options.fatal ? ErrorMode::Fatal : ErrorMode::Replacement;
auto error_mode = options.fatal ? TextCodec::ErrorMode::Fatal : TextCodec::ErrorMode::Replacement;
// 5. Set thiss ignore BOM to options["ignoreBOM"].
auto ignore_bom = options.ignore_bom;
@ -59,6 +60,7 @@ WebIDL::ExceptionOr<GC::Ref<TextDecoderStream>> TextDecoderStream::construct_imp
// algorithm with this and chunk.
auto transform_algorithm = GC::create_function(realm.heap(), [stream](JS::Value chunk) -> GC::Ref<WebIDL::Promise> {
auto& realm = stream->realm();
HTML::TemporaryExecutionContext execution_context { realm };
if (auto result = stream->decode_and_enqueue_chunk(chunk); result.is_error())
return WebIDL::create_rejected_promise_from_exception(realm, result.release_error());
return WebIDL::create_resolved_promise(realm, JS::js_undefined());
@ -67,6 +69,7 @@ WebIDL::ExceptionOr<GC::Ref<TextDecoderStream>> TextDecoderStream::construct_imp
// 8. Let flushAlgorithm be an algorithm which takes no arguments and runs the flush and enqueue algorithm with this.
auto flush_algorithm = GC::create_function(realm.heap(), [stream]() -> GC::Ref<WebIDL::Promise> {
auto& realm = stream->realm();
HTML::TemporaryExecutionContext execution_context { realm };
if (auto result = stream->flush_and_enqueue(); result.is_error())
return WebIDL::create_rejected_promise_from_exception(realm, result.release_error());
return WebIDL::create_resolved_promise(realm, JS::js_undefined());
@ -81,11 +84,14 @@ WebIDL::ExceptionOr<GC::Ref<TextDecoderStream>> TextDecoderStream::construct_imp
return stream;
}
TextDecoderStream::TextDecoderStream(JS::Realm& realm, GC::Ref<Streams::TransformStream> transform, TextCodec::Decoder& decoder, FlyString encoding, ErrorMode error_mode, bool ignore_bom)
TextDecoderStream::TextDecoderStream(JS::Realm& realm, GC::Ref<Streams::TransformStream> transform, TextCodec::Decoder& decoder, FlyString encoding, TextCodec::ErrorMode error_mode, bool ignore_bom)
: Bindings::PlatformObject(realm)
, Streams::GenericTransformStreamMixin(transform)
, TextDecoderCommonMixin(decoder, move(encoding), error_mode, ignore_bom)
, m_streaming_decoder(make<TextCodec::StreamingDecoder>(m_encoding))
, m_streaming_decoder(make<TextCodec::StreamingDecoder>(
m_encoding,
m_ignore_bom ? TextCodec::IgnoreBOM::Yes : TextCodec::IgnoreBOM::No,
m_error_mode))
{
}
@ -119,7 +125,10 @@ WebIDL::ExceptionOr<void> TextDecoderStream::decode_and_enqueue_chunk(JS::Value
return WebIDL::OperationError::create(realm, "Failed to copy bytes from BufferSource"_utf16);
auto buffer = buffer_or_error.release_value();
auto decoded = TRY_OR_THROW_OOM(vm, m_streaming_decoder->to_utf8(buffer.bytes()));
auto decoded_or_error = m_streaming_decoder->to_utf8(buffer.bytes());
if (decoded_or_error.is_error() && decoded_or_error.error().code() != ENOMEM)
return WebIDL::SimpleException { WebIDL::SimpleExceptionType::TypeError, "Decoding failed"sv };
auto decoded = TRY_OR_THROW_OOM(vm, move(decoded_or_error));
// 3-4. Run "processing an item" until the input is exhausted, accumulating the output, then enqueue any non-empty
// result. If processing returns error, throw a TypeError.
@ -130,7 +139,10 @@ WebIDL::ExceptionOr<void> TextDecoderStream::decode_and_enqueue_chunk(JS::Value
WebIDL::ExceptionOr<void> TextDecoderStream::flush_and_enqueue()
{
// 1-3. Drain decoder's I/O queue and run "processing an item" to completion.
auto decoded = TRY_OR_THROW_OOM(vm(), m_streaming_decoder->finish());
auto decoded_or_error = m_streaming_decoder->finish();
if (decoded_or_error.is_error() && decoded_or_error.error().code() != ENOMEM)
return WebIDL::SimpleException { WebIDL::SimpleExceptionType::TypeError, "Decoding failed"sv };
auto decoded = TRY_OR_THROW_OOM(vm(), move(decoded_or_error));
return enqueue_decoded_output(decoded);
}
@ -139,22 +151,9 @@ WebIDL::ExceptionOr<void> TextDecoderStream::enqueue_decoded_output(String const
auto& realm = this->realm();
auto& vm = realm.vm();
// https://encoding.spec.whatwg.org/#concept-td-serialize
// FIXME: The underlying TextCodec decoders currently strip leading BOMs unconditionally for UTF-8 and UTF-16BE/LE,
// so the "ignore BOM" flag is effectively ignored here. Once the decoders accept a "preserve BOM" mode,
// plumb m_ignore_bom through and strip the BOM from `decoded` only when m_ignore_bom is false.
if (!m_bom_seen && !decoded.is_empty())
m_bom_seen = true;
if (decoded.is_empty())
return {};
// If decoder's error mode is "fatal" and processing produced any error, throw a TypeError.
// NB: We can only detect this approximately by looking for U+FFFD in the decoded output, which the underlying
// decoder substitutes for invalid sequences. This matches the existing TextDecoder.decode() behavior.
if (fatal() && decoded.contains(0xFFFD))
return WebIDL::SimpleException { WebIDL::SimpleExceptionType::TypeError, "Decoding failed"sv };
auto js_string = JS::PrimitiveString::create(vm, Utf16String::from_utf8(decoded));
return Streams::transform_stream_default_controller_enqueue(*m_transform->controller(), js_string);
}

View file

@ -27,7 +27,7 @@ public:
virtual ~TextDecoderStream() override;
private:
TextDecoderStream(JS::Realm&, GC::Ref<Streams::TransformStream>, TextCodec::Decoder&, FlyString encoding, ErrorMode, bool ignore_bom);
TextDecoderStream(JS::Realm&, GC::Ref<Streams::TransformStream>, TextCodec::Decoder&, FlyString encoding, TextCodec::ErrorMode, bool ignore_bom);
virtual void initialize(JS::Realm&) override;
virtual void visit_edges(Cell::Visitor&) override;

View file

@ -14,7 +14,7 @@ namespace Web::Encoding {
GC_DEFINE_ALLOCATOR(TextEncoder);
WebIDL::ExceptionOr<GC::Ref<TextEncoder>> TextEncoder::construct_impl(JS::Realm& realm)
GC::Ref<TextEncoder> TextEncoder::construct_impl(JS::Realm& realm)
{
return realm.create<TextEncoder>(realm);
}

View file

@ -27,7 +27,7 @@ class TextEncoder final
GC_DECLARE_ALLOCATOR(TextEncoder);
public:
static WebIDL::ExceptionOr<GC::Ref<TextEncoder>> construct_impl(JS::Realm&);
static GC::Ref<TextEncoder> construct_impl(JS::Realm&);
virtual ~TextEncoder() override;

View file

@ -12,6 +12,7 @@
#include <LibWeb/Bindings/Intrinsics.h>
#include <LibWeb/Bindings/TextEncoderStream.h>
#include <LibWeb/Encoding/TextEncoderStream.h>
#include <LibWeb/HTML/Scripting/TemporaryExecutionContext.h>
#include <LibWeb/Streams/TransformStream.h>
#include <LibWeb/Streams/TransformStreamOperations.h>
#include <LibWeb/WebIDL/Promise.h>
@ -21,7 +22,7 @@ namespace Web::Encoding {
GC_DEFINE_ALLOCATOR(TextEncoderStream);
// https://encoding.spec.whatwg.org/#dom-textencoderstream
WebIDL::ExceptionOr<GC::Ref<TextEncoderStream>> TextEncoderStream::construct_impl(JS::Realm& realm)
GC::Ref<TextEncoderStream> TextEncoderStream::construct_impl(JS::Realm& realm)
{
// 1. Set thiss encoder to an instance of the UTF-8 encoder.
// NOTE: No-op, as AK::String is already in UTF-8 format.
@ -38,6 +39,7 @@ WebIDL::ExceptionOr<GC::Ref<TextEncoderStream>> TextEncoderStream::construct_imp
auto transform_algorithm = GC::create_function(realm.heap(), [stream](JS::Value chunk) -> GC::Ref<WebIDL::Promise> {
auto& realm = stream->realm();
auto& vm = realm.vm();
HTML::TemporaryExecutionContext execution_context { realm };
if (auto result = stream->encode_and_enqueue_chunk(chunk); result.is_error()) {
auto throw_completion = Bindings::exception_to_throw_completion(vm, result.exception());
@ -51,6 +53,7 @@ WebIDL::ExceptionOr<GC::Ref<TextEncoderStream>> TextEncoderStream::construct_imp
auto flush_algorithm = GC::create_function(realm.heap(), [stream]() -> GC::Ref<WebIDL::Promise> {
auto& realm = stream->realm();
auto& vm = realm.vm();
HTML::TemporaryExecutionContext execution_context { realm };
if (auto result = stream->encode_and_flush(); result.is_error()) {
auto throw_completion = Bindings::exception_to_throw_completion(vm, result.exception());

View file

@ -20,7 +20,7 @@ class TextEncoderStream final
GC_DECLARE_ALLOCATOR(TextEncoderStream);
public:
static WebIDL::ExceptionOr<GC::Ref<TextEncoderStream>> construct_impl(JS::Realm&);
static GC::Ref<TextEncoderStream> construct_impl(JS::Realm&);
virtual ~TextEncoderStream() override;
private:

View file

@ -108,7 +108,7 @@ HTMLTokenizer::HTMLTokenizer(StringView input, ByteString const& encoding, Input
if (input_type == InputType::EncodedBytes) {
auto decoder = TextCodec::decoder_for(encoding);
VERIFY(decoder.has_value());
m_source = MUST(decoder->to_utf8(input));
m_source = MUST(decoder->to_utf8(input, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement));
} else {
m_source = decoded_string_for_utf8_tokenizer(input);
}

View file

@ -68,7 +68,7 @@ void IncrementalDocumentParser::initialize_parser(ReadonlyBytes sniff_bytes)
auto standardized_encoding = TextCodec::get_standardized_encoding(encoding);
VERIFY(standardized_encoding.has_value());
m_decoder = make<TextCodec::StreamingDecoder>(standardized_encoding.value());
m_decoder = make<TextCodec::StreamingDecoder>(standardized_encoding.value(), TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement);
// https://html.spec.whatwg.org/multipage/parsing.html#determining-the-character-encoding
// The document's character encoding must immediately be set to the value returned from this

View file

@ -54,7 +54,7 @@ void Instance::initialize(JS::Realm& realm)
[&](Wasm::FunctionAddress const& address) {
Optional<GC::Ptr<JS::FunctionObject>> object = m_function_instances.get(address);
if (!object.has_value()) {
object = Detail::create_native_function(vm, address, name, this);
object = Detail::create_native_function(vm, address, this);
m_function_instances.set(address, *object);
}

View file

@ -289,6 +289,7 @@ JS::ThrowCompletionOr<NonnullRefPtr<Wasm::ModuleInstance>> instantiate_module(JS
// 2. Let imports be « ».
HashMap<Wasm::Linker::Name, Wasm::ExternValue> resolved_imports;
size_t imported_function_count = 0;
if (import_object) {
dbgln_if(LIBWEB_WASM_DEBUG, "Trying to resolve stuff because import object was specified");
@ -330,7 +331,7 @@ JS::ThrowCompletionOr<NonnullRefPtr<Wasm::ModuleInstance>> instantiate_module(JS
else {
// 3.4.3.1. Create a host function from v and functype, and let funcaddr be the result.
cache.add_imported_object(function);
auto host_function = create_host_function(vm, function, function_type, ByteString::formatted("func{}", resolved_imports.size()));
auto host_function = create_host_function(vm, function, function_type, ByteString::number(imported_function_count));
address = cache.abstract_machine().store().allocate(move(host_function));
// FIXME: 3.4.3.2. Let index be the number of external functions in imports. This value index is known as the index of the host function funcaddr.
// 'index' doesn't seem to be used anywhere?
@ -342,6 +343,7 @@ JS::ThrowCompletionOr<NonnullRefPtr<Wasm::ModuleInstance>> instantiate_module(JS
// 3.4.4. Let externfunc be the external value func funcaddr.
// 3.4.5. Append externfunc to imports.
resolved_imports.set(import_name, Wasm::ExternValue { Wasm::FunctionAddress { *address } });
++imported_function_count;
return {};
},
// 3.5. If externtype is of the form global mut valtype,
@ -647,14 +649,20 @@ JS::ThrowCompletionOr<JS::HandledByHost> host_grow_shared_array_buffer(JS::VM& v
GC_DEFINE_ALLOCATOR(ExportedWasmFunction);
GC::Ref<ExportedWasmFunction> ExportedWasmFunction::create(JS::Realm& realm, Utf16FlyString name, Function<JS::ThrowCompletionOr<JS::Value>(JS::VM&)> behavior, Wasm::FunctionAddress exported_address)
GC::Ref<ExportedWasmFunction> ExportedWasmFunction::create(JS::Realm& realm, Utf16FlyString name, size_t length, Function<JS::ThrowCompletionOr<JS::Value>(JS::VM&)> behavior, Wasm::FunctionAddress exported_address)
{
auto& vm = realm.vm();
auto prototype = realm.intrinsics().function_prototype();
return realm.create<ExportedWasmFunction>(
auto function = realm.create<ExportedWasmFunction>(
move(name),
move(behavior),
exported_address,
prototype);
function->define_direct_property(vm.names.length, JS::Value { static_cast<double>(length) }, JS::Attribute::Configurable);
function->define_direct_property(vm.names.name, JS::PrimitiveString::create(vm, function->name()), JS::Attribute::Configurable);
return function;
}
ExportedWasmFunction::ExportedWasmFunction(Utf16FlyString name, AK::Function<JS::ThrowCompletionOr<JS::Value>(JS::VM&)> behavior, Wasm::FunctionAddress exported_address, JS::Object& prototype)
@ -676,19 +684,57 @@ JS::ThrowCompletionOr<JS::Value> ExportedWasmFunction::call()
return m_behavior(vm());
}
JS::NativeFunction* create_native_function(JS::VM& vm, Wasm::FunctionAddress address, Utf16FlyString name, Instance* instance)
// https://www.w3.org/TR/wasm-js-api-2/#name-of-the-webassembly-function
static Utf16FlyString name_of_webassembly_function(Wasm::Store& store, Wasm::FunctionAddress function_address)
{
// 1. Let store be the surrounding agents associated store.
// 2. Let funcinst be store.funcs[funcaddr].
auto* function_instance = store.get(function_address);
VERIFY(function_instance);
auto index = function_instance->visit(
// 3. If funcinst is of the form {type functype, hostcode hostfunc},
[&](Wasm::HostFunction const& host_function) {
// 1. Assert: hostfunc is a JavaScript object and IsCallable(hostfunc) is true.
// 2. Let index be the index of the host function funcaddr.
auto index = host_function.name().to_number<size_t>(TrimWhitespace::No);
VERIFY(index.has_value());
return *index;
},
// 4. Otherwise,
[&](Wasm::WasmFunction const& wasm_function) {
// 1. Let moduleinst be funcinst.module.
auto const& module_instance = wasm_function.module();
// 2. Assert: funcaddr is contained in moduleinst.funcaddrs.
// 3. Let index be the index of moduleinst.funcaddrs where funcaddr is found.
auto index = module_instance.functions().find_first_index(function_address);
VERIFY(index.has_value());
return *index;
});
// 5. Return ! ToString(index).
return Utf16String::number(index);
}
JS::NativeFunction* create_native_function(JS::VM& vm, Wasm::FunctionAddress address, Instance* instance)
{
auto& realm = *vm.current_realm();
Optional<Wasm::FunctionType> type;
auto& cache = get_cache(realm);
cache.abstract_machine().store().get(address)->visit([&](auto const& value) { type = value.type(); });
if (auto entry = cache.get_function_instance(address); entry.has_value())
return *entry;
auto& store = cache.abstract_machine().store();
auto type = store.get(address)->visit([&](auto const& value) { return value.type(); });
auto length = type.parameters().size();
auto function = ExportedWasmFunction::create(
realm,
move(name),
[address, type = type.release_value(), instance](JS::VM& vm) -> JS::ThrowCompletionOr<JS::Value> {
name_of_webassembly_function(store, address),
length,
[address, type = move(type), instance](JS::VM& vm) -> JS::ThrowCompletionOr<JS::Value> {
(void)instance;
auto& realm = *vm.current_realm();
Vector<Wasm::Value> values;
@ -861,17 +907,7 @@ JS::Value to_js_value(JS::VM& vm, Wasm::Value& wasm_value, Wasm::ValueType type)
if (ref_.ref().has<Wasm::Reference::Null>())
return JS::js_null();
auto address = ref_.ref().get<Wasm::Reference::Func>().address;
auto& cache = get_cache(realm);
auto* function = cache.abstract_machine().store().get(address);
auto name = function->visit(
[&](Wasm::WasmFunction& wasm_function) {
auto index = *wasm_function.module().functions().find_first_index(address);
return ByteString::formatted("func{}", index);
},
[](Wasm::HostFunction& host_function) {
return host_function.name();
});
return create_native_function(vm, address, Utf16FlyString::from_utf8(name));
return create_native_function(vm, address);
}
case Wasm::ValueType::ExternReference: {
auto ref_ = wasm_value.to<Wasm::Reference>();

View file

@ -101,7 +101,7 @@ class ExportedWasmFunction final : public JS::NativeFunction {
GC_DECLARE_ALLOCATOR(ExportedWasmFunction);
public:
static GC::Ref<ExportedWasmFunction> create(JS::Realm&, Utf16FlyString name, ESCAPING Function<JS::ThrowCompletionOr<JS::Value>(JS::VM&)>, Wasm::FunctionAddress);
static GC::Ref<ExportedWasmFunction> create(JS::Realm&, Utf16FlyString name, size_t length, ESCAPING Function<JS::ThrowCompletionOr<JS::Value>(JS::VM&)>, Wasm::FunctionAddress);
virtual ~ExportedWasmFunction() override = default;
Wasm::FunctionAddress exported_address() const { return m_exported_address; }
@ -122,7 +122,7 @@ WebAssemblyCache& get_cache(JS::Realm&);
JS::ThrowCompletionOr<NonnullRefPtr<Wasm::ModuleInstance>> instantiate_module(JS::VM&, Wasm::Module const&, GC::Ptr<JS::Object> import_object);
JS::ThrowCompletionOr<NonnullRefPtr<CompiledWebAssemblyModule>> compile_a_webassembly_module(JS::VM&, ByteBuffer);
JS::NativeFunction* create_native_function(JS::VM&, Wasm::FunctionAddress address, Utf16FlyString name, Instance* instance = nullptr);
JS::NativeFunction* create_native_function(JS::VM&, Wasm::FunctionAddress address, Instance* instance = nullptr);
JS::ThrowCompletionOr<Wasm::Value> to_webassembly_value(JS::VM&, JS::Value value, Wasm::ValueType const& type);
Wasm::Value default_webassembly_value(JS::VM&, Wasm::ValueType type);
JS::Value to_js_value(JS::VM&, Wasm::Value& wasm_value, Wasm::ValueType type);

View file

@ -92,7 +92,10 @@ static xmlNodePtr mirror_node(xmlDocPtr doc, DOM::Node const& node)
}
case DOM::NodeType::DOCUMENT_NODE: {
auto const& document = static_cast<DOM::Document const&>(node);
return mirror_node(doc, *document.document_element());
auto const* document_element = document.document_element();
if (!document_element)
return nullptr;
return mirror_node(doc, *document_element);
}
case DOM::NodeType::DOCUMENT_TYPE_NODE: {
return nullptr; // Unused in libxml2

View file

@ -438,7 +438,7 @@ ErrorOr<Vector<String>> Autocomplete::received_autocomplete_respsonse(Autocomple
if (!decoder.has_value())
decoder = TextCodec::decoder_for_exact_name("UTF-8"sv);
auto decoded_response = TRY(decoder->to_utf8(response));
auto decoded_response = TRY(decoder->to_utf8(response, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement));
auto json = TRY(JsonValue::from_string(decoded_response));
if (engine.name == "DuckDuckGo")

View file

@ -24,6 +24,6 @@ extern "C" int LLVMFuzzerTestOneInput(uint8_t const* data, size_t size)
if (!decoder.has_value())
return 0;
(void)decoder->to_utf8(encoded_data);
(void)decoder->to_utf8(encoded_data, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement);
return 0;
}

View file

@ -1085,7 +1085,7 @@ size_t Request::on_header_received(void* buffer, size_t size, size_t nmemb, void
auto decoder = TextCodec::decoder_for_exact_name("ISO-8859-1"sv);
VERIFY(decoder.has_value());
request.m_reason_phrase = MUST(decoder->to_utf8(reason_phrase));
request.m_reason_phrase = MUST(decoder->to_utf8(reason_phrase, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement));
return total_size;
}
}

View file

@ -37,13 +37,13 @@ TEST_CASE(test_utf8_decode)
// Bytes for U+1F600 GRINNING FACE
auto test_string = "\xf0\x9f\x98\x80"sv;
EXPECT(decoder.validate(test_string));
EXPECT(!decoder.to_utf8(test_string, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Fatal).is_error());
auto processed_code_points = process_code_points(decoder, test_string);
EXPECT(processed_code_points.size() == 1);
EXPECT(processed_code_points[0] == 0x1F600);
EXPECT(MUST(decoder.to_utf8(test_string)) == test_string);
EXPECT(MUST(decoder.to_utf8(test_string, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)) == test_string);
}
TEST_CASE(test_utf8_process_code_points)
@ -60,8 +60,8 @@ TEST_CASE(test_utf8_process_code_points_replaces_surrogates)
auto utf8_encoded_surrogate_bytes = Vector<u8> { 0xed, 0xa0, 0x80 };
auto utf8_encoded_surrogate = StringView(bytes(utf8_encoded_surrogate_bytes));
EXPECT(!decoder.validate(utf8_encoded_surrogate));
EXPECT_EQ(MUST(decoder.to_utf8(utf8_encoded_surrogate)), "\xef\xbf\xbd\xef\xbf\xbd\xef\xbf\xbd"sv);
EXPECT(decoder.to_utf8(utf8_encoded_surrogate, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Fatal).is_error());
EXPECT_EQ(MUST(decoder.to_utf8(utf8_encoded_surrogate, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)), "\xef\xbf\xbd\xef\xbf\xbd\xef\xbf\xbd"sv);
EXPECT_EQ(process_code_points(decoder, utf8_encoded_surrogate), (Vector<u32> { 0xfffd, 0xfffd, 0xfffd }));
}
@ -71,8 +71,8 @@ TEST_CASE(test_utf8_process_code_points_replaces_truncated_tail_as_single_error)
auto truncated_tail_bytes = Vector<u8> { 0xf0, 0x9f, 0x98 };
auto truncated_tail = StringView(bytes(truncated_tail_bytes));
EXPECT(!decoder.validate(truncated_tail));
EXPECT_EQ(MUST(decoder.to_utf8(truncated_tail)), "\xef\xbf\xbd"sv);
EXPECT(decoder.to_utf8(truncated_tail, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Fatal).is_error());
EXPECT_EQ(MUST(decoder.to_utf8(truncated_tail, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)), "\xef\xbf\xbd"sv);
EXPECT_EQ(process_code_points(decoder, truncated_tail), (Vector<u32> { 0xfffd }));
}
@ -82,8 +82,8 @@ TEST_CASE(test_utf8_process_code_points_replaces_overlong_sequences)
auto overlong_null_bytes = Vector<u8> { 0xc0, 0x80 };
auto overlong_null = StringView(bytes(overlong_null_bytes));
EXPECT(!decoder.validate(overlong_null));
EXPECT_EQ(MUST(decoder.to_utf8(overlong_null)), "\xef\xbf\xbd\xef\xbf\xbd"sv);
EXPECT(decoder.to_utf8(overlong_null, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Fatal).is_error());
EXPECT_EQ(MUST(decoder.to_utf8(overlong_null, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)), "\xef\xbf\xbd\xef\xbf\xbd"sv);
EXPECT_EQ(process_code_points(decoder, overlong_null), (Vector<u32> { 0xfffd, 0xfffd }));
}
@ -95,12 +95,12 @@ TEST_CASE(test_utf8_process_code_points_restores_invalid_second_byte)
auto out_of_range_four_byte_sequence_bytes = Vector<u8> { 0xf4, 0x90, 0x80, 0x80 };
auto out_of_range_four_byte_sequence = StringView(bytes(out_of_range_four_byte_sequence_bytes));
EXPECT(!decoder.validate(overlong_three_byte_sequence));
EXPECT_EQ(MUST(decoder.to_utf8(overlong_three_byte_sequence)), "\xef\xbf\xbd\xef\xbf\xbd\xef\xbf\xbd"sv);
EXPECT(decoder.to_utf8(overlong_three_byte_sequence, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Fatal).is_error());
EXPECT_EQ(MUST(decoder.to_utf8(overlong_three_byte_sequence, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)), "\xef\xbf\xbd\xef\xbf\xbd\xef\xbf\xbd"sv);
EXPECT_EQ(process_code_points(decoder, overlong_three_byte_sequence), (Vector<u32> { 0xfffd, 0xfffd, 0xfffd }));
EXPECT(!decoder.validate(out_of_range_four_byte_sequence));
EXPECT_EQ(MUST(decoder.to_utf8(out_of_range_four_byte_sequence)), "\xef\xbf\xbd\xef\xbf\xbd\xef\xbf\xbd\xef\xbf\xbd"sv);
EXPECT(decoder.to_utf8(out_of_range_four_byte_sequence, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Fatal).is_error());
EXPECT_EQ(MUST(decoder.to_utf8(out_of_range_four_byte_sequence, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)), "\xef\xbf\xbd\xef\xbf\xbd\xef\xbf\xbd\xef\xbf\xbd"sv);
EXPECT_EQ(process_code_points(decoder, out_of_range_four_byte_sequence), (Vector<u32> { 0xfffd, 0xfffd, 0xfffd, 0xfffd }));
}
@ -110,8 +110,7 @@ TEST_CASE(test_utf16be_decode)
// This is the output of `python3 -c "print('säk😀'.encode('utf-16be'))"`.
auto test_string = "\x00s\x00\xe4\x00k\xd8=\xde\x00"sv;
EXPECT(decoder.validate(test_string));
auto utf8 = MUST(decoder.to_utf8(test_string));
auto utf8 = MUST(decoder.to_utf8(test_string, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Fatal));
EXPECT_EQ(utf8, "säk😀"sv);
}
@ -122,12 +121,12 @@ TEST_CASE(test_utf16be_process_code_points)
EXPECT_EQ(process_code_points(decoder, StringView(bytes({ 0xfe, 0xff, 0x00, 'A', 0xd8, 0x3d, 0xde, 0x00 }))), (Vector<u32> { 0x41, 0x1F600 }));
EXPECT_EQ(process_code_points(decoder, StringView(bytes({ 0xd8, 0x3d, 0x00, 'A', 0xde, 0x00 }))), (Vector<u32> { 0xfffd, 0x41, 0xfffd }));
EXPECT_EQ(process_code_points(decoder, StringView(bytes({ 0x00, 'A', 0xff }))), (Vector<u32> { 0x41, 0xfffd }));
EXPECT_EQ(MUST(decoder.to_utf8(StringView(bytes({ 0x00, 'A', 0xff })))), "A\xef\xbf\xbd"sv);
EXPECT_EQ(MUST(decoder.to_utf8(StringView(bytes({ 0x00, 'A', 0xff })), TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)), "A\xef\xbf\xbd"sv);
}
TEST_CASE(test_streaming_decoder_utf8_mid_sequence)
{
auto streaming_decoder = TextCodec::StreamingDecoder { "UTF-8"sv };
auto streaming_decoder = TextCodec::StreamingDecoder { "UTF-8"sv, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement };
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 'a', 0xc3 }))), "a"sv);
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0xa9, 'b' }))), "éb"sv);
@ -137,27 +136,27 @@ TEST_CASE(test_streaming_decoder_utf8_mid_sequence)
TEST_CASE(test_streaming_decoder_finishes_incomplete_sequence)
{
auto& decoder = decoder_for("UTF-8"sv);
auto streaming_decoder = TextCodec::StreamingDecoder { "UTF-8"sv };
auto streaming_decoder = TextCodec::StreamingDecoder { "UTF-8"sv, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement };
auto incomplete_sequence = Vector<u8> { 0xc3 };
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes(incomplete_sequence))), ""sv);
EXPECT_EQ(MUST(streaming_decoder.finish()), MUST(decoder.to_utf8(StringView(bytes(incomplete_sequence)))));
EXPECT_EQ(MUST(streaming_decoder.finish()), MUST(decoder.to_utf8(StringView(bytes(incomplete_sequence)), TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)));
}
TEST_CASE(test_streaming_decoder_utf8_invalid_second_byte_tail)
{
auto streaming_decoder_for_overlong_three_byte_sequence = TextCodec::StreamingDecoder { "UTF-8"sv };
auto streaming_decoder_for_overlong_three_byte_sequence = TextCodec::StreamingDecoder { "UTF-8"sv, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement };
EXPECT_EQ(MUST(streaming_decoder_for_overlong_three_byte_sequence.to_utf8(bytes({ 0xe0, 0x80 }))), "\xef\xbf\xbd\xef\xbf\xbd"sv);
EXPECT_EQ(MUST(streaming_decoder_for_overlong_three_byte_sequence.finish()), ""sv);
auto streaming_decoder_for_out_of_range_four_byte_sequence = TextCodec::StreamingDecoder { "UTF-8"sv };
auto streaming_decoder_for_out_of_range_four_byte_sequence = TextCodec::StreamingDecoder { "UTF-8"sv, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement };
EXPECT_EQ(MUST(streaming_decoder_for_out_of_range_four_byte_sequence.to_utf8(bytes({ 0xf4, 0x90 }))), "\xef\xbf\xbd\xef\xbf\xbd"sv);
EXPECT_EQ(MUST(streaming_decoder_for_out_of_range_four_byte_sequence.finish()), ""sv);
}
TEST_CASE(test_streaming_decoder_utf8_valid_second_byte_tail)
{
auto streaming_decoder = TextCodec::StreamingDecoder { "UTF-8"sv };
auto streaming_decoder = TextCodec::StreamingDecoder { "UTF-8"sv, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement };
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0xe0, 0xa0 }))), ""sv);
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0x80 }))), "\xe0\xa0\x80"sv);
@ -166,7 +165,7 @@ TEST_CASE(test_streaming_decoder_utf8_valid_second_byte_tail)
TEST_CASE(test_streaming_decoder_utf16_odd_byte)
{
auto streaming_decoder = TextCodec::StreamingDecoder { "UTF-16LE"sv };
auto streaming_decoder = TextCodec::StreamingDecoder { "UTF-16LE"sv, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement };
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0x41, 0x00, 0x42 }))), "A"sv);
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0x00 }))), "B"sv);
@ -175,7 +174,7 @@ TEST_CASE(test_streaming_decoder_utf16_odd_byte)
TEST_CASE(test_streaming_decoder_utf16_surrogate_pair_split)
{
auto streaming_decoder = TextCodec::StreamingDecoder { "UTF-16BE"sv };
auto streaming_decoder = TextCodec::StreamingDecoder { "UTF-16BE"sv, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement };
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0x00, 0x41, 0xd8, 0x3d }))), "A"sv);
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0xde, 0x00 }))), "😀"sv);
@ -185,48 +184,48 @@ TEST_CASE(test_streaming_decoder_utf16_surrogate_pair_split)
TEST_CASE(test_streaming_decoder_gb18030_four_byte_tail)
{
auto& decoder = decoder_for("gb18030"sv);
auto streaming_decoder = TextCodec::StreamingDecoder { "gb18030"sv };
auto streaming_decoder = TextCodec::StreamingDecoder { "gb18030"sv, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement };
auto gb18030_sequence = Vector<u8> { 0x81, 0x30, 0x81, 0x30 };
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 'a', 0x81, 0x30, 0x81 }))), "a"sv);
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0x30 }))), MUST(decoder.to_utf8(StringView(bytes(gb18030_sequence)))));
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0x30 }))), MUST(decoder.to_utf8(StringView(bytes(gb18030_sequence)), TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)));
EXPECT_EQ(MUST(streaming_decoder.finish()), ""sv);
}
TEST_CASE(test_streaming_decoder_big5_overlapping_trail)
{
auto& decoder = decoder_for("Big5"sv);
auto streaming_decoder = TextCodec::StreamingDecoder { "Big5"sv };
auto streaming_decoder = TextCodec::StreamingDecoder { "Big5"sv, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement };
auto big5_sequence = Vector<u8> { 0xa4, 0xa4 };
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes(big5_sequence))), MUST(decoder.to_utf8(StringView(bytes(big5_sequence)))));
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes(big5_sequence))), MUST(decoder.to_utf8(StringView(bytes(big5_sequence)), TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)));
EXPECT_EQ(MUST(streaming_decoder.finish()), ""sv);
auto streaming_decoder_for_split_input = TextCodec::StreamingDecoder { "Big5"sv };
auto streaming_decoder_for_split_input = TextCodec::StreamingDecoder { "Big5"sv, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement };
EXPECT_EQ(MUST(streaming_decoder_for_split_input.to_utf8(bytes({ 0xa4 }))), ""sv);
EXPECT_EQ(MUST(streaming_decoder_for_split_input.to_utf8(bytes({ 0xa4 }))), MUST(decoder.to_utf8(StringView(bytes(big5_sequence)))));
EXPECT_EQ(MUST(streaming_decoder_for_split_input.to_utf8(bytes({ 0xa4 }))), MUST(decoder.to_utf8(StringView(bytes(big5_sequence)), TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)));
EXPECT_EQ(MUST(streaming_decoder_for_split_input.finish()), ""sv);
}
TEST_CASE(test_streaming_decoder_euc_jp_three_byte_tail)
{
auto& decoder = decoder_for("EUC-JP"sv);
auto streaming_decoder = TextCodec::StreamingDecoder { "EUC-JP"sv };
auto streaming_decoder = TextCodec::StreamingDecoder { "EUC-JP"sv, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement };
auto euc_jp_sequence = Vector<u8> { 0x8f, 0xa2, 0xaf };
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0x8f, 0xa2 }))), ""sv);
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0xaf }))), MUST(decoder.to_utf8(StringView(bytes(euc_jp_sequence)))));
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0xaf }))), MUST(decoder.to_utf8(StringView(bytes(euc_jp_sequence)), TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)));
EXPECT_EQ(MUST(streaming_decoder.finish()), ""sv);
}
TEST_CASE(test_streaming_decoder_shift_jis_tail)
{
auto& decoder = decoder_for("Shift_JIS"sv);
auto streaming_decoder = TextCodec::StreamingDecoder { "Shift_JIS"sv };
auto streaming_decoder = TextCodec::StreamingDecoder { "Shift_JIS"sv, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement };
auto shift_jis_sequence = Vector<u8> { 0x82, 0xa0 };
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0x82 }))), ""sv);
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0xa0 }))), MUST(decoder.to_utf8(StringView(bytes(shift_jis_sequence)))));
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0xa0 }))), MUST(decoder.to_utf8(StringView(bytes(shift_jis_sequence)), TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)));
EXPECT_EQ(MUST(streaming_decoder.finish()), ""sv);
}
@ -236,8 +235,7 @@ TEST_CASE(test_utf16le_decode)
// This is the output of `python3 -c "print('säk😀'.encode('utf-16le'))"`.
auto test_string = "s\x00\xe4\x00k\x00=\xd8\x00\xde"sv;
EXPECT(decoder.validate(test_string));
auto utf8 = MUST(decoder.to_utf8(test_string));
auto utf8 = MUST(decoder.to_utf8(test_string, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Fatal));
EXPECT_EQ(utf8, "säk😀"sv);
}
@ -248,5 +246,5 @@ TEST_CASE(test_utf16le_process_code_points)
EXPECT_EQ(process_code_points(decoder, StringView(bytes({ 0xff, 0xfe, 'A', 0x00, 0x3d, 0xd8, 0x00, 0xde }))), (Vector<u32> { 0x41, 0x1F600 }));
EXPECT_EQ(process_code_points(decoder, StringView(bytes({ 0x3d, 0xd8, 'A', 0x00, 0x00, 0xde }))), (Vector<u32> { 0xfffd, 0x41, 0xfffd }));
EXPECT_EQ(process_code_points(decoder, StringView(bytes({ 'A', 0x00, 0xff }))), (Vector<u32> { 0x41, 0xfffd }));
EXPECT_EQ(MUST(decoder.to_utf8(StringView(bytes({ 'A', 0x00, 0xff })))), "A\xef\xbf\xbd"sv);
EXPECT_EQ(MUST(decoder.to_utf8(StringView(bytes({ 'A', 0x00, 0xff })), TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)), "A\xef\xbf\xbd"sv);
}

View file

@ -0,0 +1,6 @@
<!DOCTYPE html>
<script>
// createDocument with an empty qualified name yields a document with no root element.
const doc = document.implementation.createDocument(null, "", null);
document.evaluate("/", doc, null, XPathResult.ANY_TYPE, null);
</script>

View file

@ -0,0 +1,6 @@
length: 2
name: 0
export name: sqlite3_aggregate_context
length descriptor: writable=false, enumerable=false, configurable=true
name descriptor: writable=false, enumerable=false, configurable=true
call result: 42

View file

@ -0,0 +1,17 @@
Harness status: OK
Found 12 tests
12 Pass
Pass ignoreBOM should work for encoding utf-8, split at character 0
Pass ignoreBOM should work for encoding utf-8, split at character 1
Pass ignoreBOM should work for encoding utf-8, split at character 2
Pass ignoreBOM should work for encoding utf-8, split at character 3
Pass ignoreBOM should work for encoding utf-16le, split at character 0
Pass ignoreBOM should work for encoding utf-16le, split at character 1
Pass ignoreBOM should work for encoding utf-16le, split at character 2
Pass ignoreBOM should work for encoding utf-16le, split at character 3
Pass ignoreBOM should work for encoding utf-16be, split at character 0
Pass ignoreBOM should work for encoding utf-16be, split at character 1
Pass ignoreBOM should work for encoding utf-16be, split at character 2
Pass ignoreBOM should work for encoding utf-16be, split at character 3

View file

@ -2,17 +2,16 @@ Harness status: OK
Found 12 tests
4 Pass
8 Fail
12 Pass
Pass a TextEncoderStream object should be associated with the realm the constructor came from
Pass TextEncoderStream's readable and writable attributes should come from the same realm as the constructor definition
Fail the output chunks when read is called after write should come from the same realm as the constructor of TextEncoderStream
Fail the output chunks when write is called with a pending read should come from the same realm as the constructor of TextEncoderStream
Fail TypeError for unconvertable chunk should come from constructor realm of TextEncoderStream
Pass the output chunks when read is called after write should come from the same realm as the constructor of TextEncoderStream
Pass the output chunks when write is called with a pending read should come from the same realm as the constructor of TextEncoderStream
Pass TypeError for unconvertable chunk should come from constructor realm of TextEncoderStream
Pass a TextDecoderStream object should be associated with the realm the constructor came from
Pass TextDecoderStream's readable and writable attributes should come from the same realm as the constructor definition
Fail the result object when read is called after write should come from the same realm as the constructor of TextDecoderStream
Fail the result object when write is called with a pending read should come from the same realm as the constructor of TextDecoderStream
Fail TypeError for chunk with the wrong type should come from constructor realm of TextDecoderStream
Fail TypeError for invalid chunk should come from constructor realm of TextDecoderStream
Fail TypeError for incomplete input should come from constructor realm of TextDecoderStream
Pass the result object when read is called after write should come from the same realm as the constructor of TextDecoderStream
Pass the result object when write is called with a pending read should come from the same realm as the constructor of TextDecoderStream
Pass TypeError for chunk with the wrong type should come from constructor realm of TextDecoderStream
Pass TypeError for invalid chunk should come from constructor realm of TextDecoderStream
Pass TypeError for incomplete input should come from constructor realm of TextDecoderStream

View file

@ -0,0 +1,8 @@
Harness status: OK
Found 2 tests
1 Pass
1 Fail
Pass Fatal flag, non-streaming cases
Fail Fatal flag, streaming cases

View file

@ -2,9 +2,8 @@ Harness status: OK
Found 4 tests
1 Pass
3 Fail
Fail BOM is ignored if ignoreBOM option is specified: utf-8
Fail BOM is ignored if ignoreBOM option is specified: utf-16le
Fail BOM is ignored if ignoreBOM option is specified: utf-16be
4 Pass
Pass BOM is ignored if ignoreBOM option is specified: utf-8
Pass BOM is ignored if ignoreBOM option is specified: utf-16le
Pass BOM is ignored if ignoreBOM option is specified: utf-16be
Pass The ignoreBOM attribute of TextDecoder

View file

@ -2,8 +2,8 @@ Harness status: OK
Found 87 tests
72 Pass
15 Fail
74 Pass
13 Fail
Pass Invalid Unicode input is replaced: utf-8
Pass Invalid Unicode input is replaced: utf-16le
Pass Invalid Unicode input is replaced: utf-16be
@ -51,8 +51,8 @@ Pass selected single-byte: iso-8859-8-i
Pass selected single-byte: iso-8859-16
Pass selected single-byte: x-mac-cyrillic
Pass Concatenating two ISO-2022-JP outputs is not always valid
Fail gb18030 version and ranges
Fail gbk version and ranges
Pass gb18030 version and ranges
Pass gbk version and ranges
Pass gbk decoder is gb18030 decoder
Pass Replacement, push back ASCII characters: big5
Pass Replacement, push back ASCII characters: iso-2022-jp

View file

@ -0,0 +1,32 @@
<!DOCTYPE html>
<script src="../include.js"></script>
<script>
asyncTest(async (done) => {
const wasmBytes = new Uint8Array([
0x00, 0x61, 0x73, 0x6d, 0x01, 0x00, 0x00, 0x00,
0x01, 0x07, 0x01, 0x60, 0x02, 0x7f, 0x7f, 0x01, 0x7f,
0x03, 0x02, 0x01, 0x00,
0x07, 0x1d, 0x01, 0x19,
0x73, 0x71, 0x6c, 0x69, 0x74, 0x65, 0x33, 0x5f,
0x61, 0x67, 0x67, 0x72, 0x65, 0x67, 0x61, 0x74,
0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74,
0x00, 0x00,
0x0a, 0x06, 0x01, 0x04, 0x00, 0x20, 0x00, 0x0b,
]);
const { instance } = await WebAssembly.instantiate(wasmBytes);
const fn = instance.exports.sqlite3_aggregate_context;
const exportName = Object.getOwnPropertyNames(instance.exports)[0];
const lengthDescriptor = Object.getOwnPropertyDescriptor(fn, "length");
const nameDescriptor = Object.getOwnPropertyDescriptor(fn, "name");
println(`length: ${fn.length}`);
println(`name: ${fn.name}`);
println(`export name: ${exportName}`);
println(`length descriptor: writable=${lengthDescriptor.writable}, enumerable=${lengthDescriptor.enumerable}, configurable=${lengthDescriptor.configurable}`);
println(`name descriptor: writable=${nameDescriptor.writable}, enumerable=${nameDescriptor.enumerable}, configurable=${nameDescriptor.configurable}`);
println(`call result: ${fn(42, 7)}`);
done();
});
</script>

View file

@ -0,0 +1,16 @@
<!doctype html>
<meta charset=utf-8>
<script>
self.GLOBAL = {
isWindow: function() { return true; },
isWorker: function() { return false; },
isShadowRealm: function() { return false; },
};
</script>
<script src="../../resources/testharness.js"></script>
<script src="../../resources/testharnessreport.js"></script>
<script src="resources/readable-stream-from-array.js"></script>
<script src="resources/readable-stream-to-array.js"></script>
<div id=log></div>
<script src="../../encoding/streams/decode-ignore-bom.any.js"></script>

View file

@ -0,0 +1,38 @@
// META: global=window,worker
// META: script=resources/readable-stream-from-array.js
// META: script=resources/readable-stream-to-array.js
const cases = [
{encoding: 'utf-8', bytes: [0xEF, 0xBB, 0xBF, 0x61, 0x62, 0x63]},
{encoding: 'utf-16le', bytes: [0xFF, 0xFE, 0x61, 0x00, 0x62, 0x00, 0x63, 0x00]},
{encoding: 'utf-16be', bytes: [0xFE, 0xFF, 0x00, 0x61, 0x00, 0x62, 0x00, 0x63]}
];
const BOM = '\uFEFF';
// |inputChunks| is an array of chunks, each represented by an array of
// integers. |ignoreBOM| is true or false. The result value is the output of the
// pipe, concatenated into a single string.
async function pipeAndAssemble(inputChunks, encoding, ignoreBOM) {
const chunksAsUint8 = inputChunks.map(values => new Uint8Array(values));
const readable = readableStreamFromArray(chunksAsUint8);
const outputArray = await readableStreamToArray(readable.pipeThrough(
new TextDecoderStream(encoding, {ignoreBOM})));
return outputArray.join('');
}
for (const testCase of cases) {
for (let splitPoint = 0; splitPoint < 4; ++splitPoint) {
promise_test(async () => {
const inputChunks = [testCase.bytes.slice(0, splitPoint),
testCase.bytes.slice(splitPoint)];
const withIgnoreBOM =
await pipeAndAssemble(inputChunks, testCase.encoding, true);
assert_equals(withIgnoreBOM, BOM + 'abc', 'BOM should be preserved');
const withoutIgnoreBOM =
await pipeAndAssemble(inputChunks, testCase.encoding, false);
assert_equals(withoutIgnoreBOM, 'abc', 'BOM should be stripped')
}, `ignoreBOM should work for encoding ${testCase.encoding}, split at ` +
`character ${splitPoint}`);
}
}

View file

@ -0,0 +1,15 @@
<!doctype html>
<meta charset=utf-8>
<title>Encoding API: End-of-file</title>
<script>
self.GLOBAL = {
isWindow: function() { return true; },
isWorker: function() { return false; },
isShadowRealm: function() { return false; },
};
</script>
<script src="../resources/testharness.js"></script>
<script src="../resources/testharnessreport.js"></script>
<div id=log></div>
<script src="../encoding/textdecoder-fatal-streaming.any.js"></script>

View file

@ -0,0 +1,45 @@
// META: global=window,dedicatedworker
// META: title=Encoding API: End-of-file
test(function() {
[
{encoding: 'utf-8', sequence: [0xC0]},
{encoding: 'utf-16le', sequence: [0x00]},
{encoding: 'utf-16be', sequence: [0x00]}
].forEach(function(testCase) {
assert_throws_js(TypeError, function() {
var decoder = new TextDecoder(testCase.encoding, {fatal: true});
decoder.decode(new Uint8Array(testCase.sequence));
}, 'Unterminated ' + testCase.encoding + ' sequence should throw if fatal flag is set');
assert_equals(
new TextDecoder(testCase.encoding).decode(new Uint8Array([testCase.sequence])),
'\uFFFD',
'Unterminated UTF-8 sequence should emit replacement character if fatal flag is unset');
});
}, 'Fatal flag, non-streaming cases');
test(function() {
var decoder = new TextDecoder('utf-16le', {fatal: true});
var odd = new Uint8Array([0x00]);
var even = new Uint8Array([0x00, 0x00]);
assert_equals(decoder.decode(odd, {stream: true}), '');
assert_equals(decoder.decode(odd), '\u0000');
assert_throws_js(TypeError, function() {
decoder.decode(even, {stream: true});
decoder.decode(odd)
});
assert_throws_js(TypeError, function() {
decoder.decode(odd, {stream: true});
decoder.decode(even);
});
assert_equals(decoder.decode(even, {stream: true}), '\u0000');
assert_equals(decoder.decode(even), '\u0000');
}, 'Fatal flag, streaming cases');