Compare commits
10 commits
b81269e78b
...
b41536f4ef
| Author | SHA1 | Date | |
|---|---|---|---|
| b41536f4ef | |||
|
|
4af1b9357b | ||
|
|
a7ce007ff4 | ||
|
|
d4048aaa96 | ||
|
|
b9da74d16e | ||
|
|
ef6753a9f9 | ||
|
|
ef99632fa7 | ||
|
|
02320c9b58 | ||
|
|
ae0c7bc097 | ||
|
|
8caee39bac |
39 changed files with 527 additions and 278 deletions
112
.forgejo/workflows/build-ubuntu-24-04.yml
Normal file
112
.forgejo/workflows/build-ubuntu-24-04.yml
Normal file
|
|
@ -0,0 +1,112 @@
|
||||||
|
name: Build Ladybird (Ubuntu 24.04)
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
branches: [ master ]
|
||||||
|
pull_request:
|
||||||
|
branches: [ master ]
|
||||||
|
workflow_dispatch:
|
||||||
|
|
||||||
|
env:
|
||||||
|
LADYBIRD_SOURCE_DIR: ${{ github.workspace }}
|
||||||
|
CCACHE_DIR: ${{ github.workspace }}/.ccache
|
||||||
|
VCPKG_ROOT: ${{ github.workspace }}/Build/vcpkg
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
build:
|
||||||
|
name: Build on Ubuntu 24.04
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
container:
|
||||||
|
image: ubuntu:24.04
|
||||||
|
steps:
|
||||||
|
- name: Checkout repository
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- name: Install system dependencies
|
||||||
|
run: |
|
||||||
|
apt-get update && apt-get install -y \
|
||||||
|
software-properties-common \
|
||||||
|
wget \
|
||||||
|
curl \
|
||||||
|
gpg \
|
||||||
|
lsb-release \
|
||||||
|
git
|
||||||
|
|
||||||
|
# Install CMake 3.30+ from Kitware (since Ubuntu 24.04 has 3.28)
|
||||||
|
wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | tee /usr/share/keyrings/kitware-archive-keyring.gpg >/dev/null
|
||||||
|
echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ noble main" | tee /etc/apt/sources.list.d/kitware.list
|
||||||
|
|
||||||
|
apt-get update && apt-get install -y \
|
||||||
|
autoconf \
|
||||||
|
autoconf-archive \
|
||||||
|
automake \
|
||||||
|
build-essential \
|
||||||
|
ccache \
|
||||||
|
cmake \
|
||||||
|
fonts-liberation2 \
|
||||||
|
glslang-tools \
|
||||||
|
libdrm-dev \
|
||||||
|
libgl1-mesa-dev \
|
||||||
|
libncurses-dev \
|
||||||
|
libtool \
|
||||||
|
nasm \
|
||||||
|
ninja-build \
|
||||||
|
pkg-config \
|
||||||
|
python3-venv \
|
||||||
|
qt6-base-dev \
|
||||||
|
qt6-tools-dev-tools \
|
||||||
|
qt6-wayland \
|
||||||
|
tar \
|
||||||
|
unzip \
|
||||||
|
zip \
|
||||||
|
libpulse-dev \
|
||||||
|
gcc-14 \
|
||||||
|
g++-14
|
||||||
|
|
||||||
|
- name: Configure compiler alternatives
|
||||||
|
run: |
|
||||||
|
update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-14 100 \
|
||||||
|
--slave /usr/bin/g++ g++ /usr/bin/g++-14 \
|
||||||
|
--slave /usr/bin/gcov gcov /usr/bin/gcov-14
|
||||||
|
|
||||||
|
- name: Install Rust
|
||||||
|
run: |
|
||||||
|
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain 1.96.0
|
||||||
|
echo "$HOME/.cargo/bin" >> $GITHUB_PATH
|
||||||
|
|
||||||
|
- name: Configure Git Safe Directory
|
||||||
|
run: |
|
||||||
|
git config --global safe.directory '*'
|
||||||
|
|
||||||
|
- name: Bootstrap vcpkg
|
||||||
|
run: |
|
||||||
|
./Meta/Utils/build_vcpkg.py
|
||||||
|
|
||||||
|
- name: Configure CMake
|
||||||
|
env:
|
||||||
|
CC: gcc-14
|
||||||
|
CXX: g++-14
|
||||||
|
run: |
|
||||||
|
cmake --preset Release -B Build \
|
||||||
|
-DCMAKE_C_COMPILER=gcc-14 \
|
||||||
|
-DCMAKE_CXX_COMPILER=g++-14 \
|
||||||
|
-DENABLE_CI_BASELINE_CPU=ON \
|
||||||
|
-DLADYBIRD_GUI_FRAMEWORK=Qt
|
||||||
|
|
||||||
|
- name: Build
|
||||||
|
run: |
|
||||||
|
cmake --build Build
|
||||||
|
|
||||||
|
- name: Install
|
||||||
|
run: |
|
||||||
|
cmake --install Build --strip --prefix dist
|
||||||
|
|
||||||
|
- name: Package Binaries
|
||||||
|
run: |
|
||||||
|
tar -czf ladybird-ubuntu-24.04.tar.gz -C dist .
|
||||||
|
|
||||||
|
- name: Upload Build Artifacts
|
||||||
|
uses: actions/upload-artifact@v4
|
||||||
|
with:
|
||||||
|
name: ladybird-ubuntu-24.04
|
||||||
|
path: ladybird-ubuntu-24.04.tar.gz
|
||||||
|
|
@ -15,15 +15,8 @@
|
||||||
|
|
||||||
namespace TextCodec {
|
namespace TextCodec {
|
||||||
|
|
||||||
static constexpr u32 replacement_code_point = 0xfffd;
|
|
||||||
|
|
||||||
namespace {
|
namespace {
|
||||||
|
|
||||||
enum class RemoveBOM {
|
|
||||||
No,
|
|
||||||
Yes,
|
|
||||||
};
|
|
||||||
|
|
||||||
class RustDecoder final : public Decoder {
|
class RustDecoder final : public Decoder {
|
||||||
public:
|
public:
|
||||||
explicit RustDecoder(StringView encoding)
|
explicit RustDecoder(StringView encoding)
|
||||||
|
|
@ -31,8 +24,7 @@ public:
|
||||||
{
|
{
|
||||||
}
|
}
|
||||||
|
|
||||||
virtual bool validate(StringView input) override;
|
virtual ErrorOr<String> to_utf8(StringView input, IgnoreBOM, ErrorMode) override;
|
||||||
virtual ErrorOr<String> to_utf8(StringView input) override;
|
|
||||||
virtual ErrorOr<size_t> length_in_utf16_code_units(StringView input) override;
|
virtual ErrorOr<size_t> length_in_utf16_code_units(StringView input) override;
|
||||||
|
|
||||||
private:
|
private:
|
||||||
|
|
@ -44,15 +36,13 @@ private:
|
||||||
class UTF8Decoder final : public Decoder {
|
class UTF8Decoder final : public Decoder {
|
||||||
public:
|
public:
|
||||||
virtual ErrorOr<void> process(StringView, Function<ErrorOr<void>(u32)> on_code_point) override;
|
virtual ErrorOr<void> process(StringView, Function<ErrorOr<void>(u32)> on_code_point) override;
|
||||||
virtual bool validate(StringView) override;
|
virtual ErrorOr<String> to_utf8(StringView, IgnoreBOM, ErrorMode) override;
|
||||||
virtual ErrorOr<String> to_utf8(StringView) override;
|
|
||||||
virtual ErrorOr<size_t> length_in_utf16_code_units(StringView) override;
|
virtual ErrorOr<size_t> length_in_utf16_code_units(StringView) override;
|
||||||
};
|
};
|
||||||
|
|
||||||
class UTF16BEDecoder final : public Decoder {
|
class UTF16BEDecoder final : public Decoder {
|
||||||
public:
|
public:
|
||||||
virtual bool validate(StringView) override;
|
virtual ErrorOr<String> to_utf8(StringView, IgnoreBOM, ErrorMode) override;
|
||||||
virtual ErrorOr<String> to_utf8(StringView) override;
|
|
||||||
virtual ErrorOr<size_t> length_in_utf16_code_units(StringView) override;
|
virtual ErrorOr<size_t> length_in_utf16_code_units(StringView) override;
|
||||||
|
|
||||||
private:
|
private:
|
||||||
|
|
@ -61,8 +51,7 @@ private:
|
||||||
|
|
||||||
class UTF16LEDecoder final : public Decoder {
|
class UTF16LEDecoder final : public Decoder {
|
||||||
public:
|
public:
|
||||||
virtual bool validate(StringView) override;
|
virtual ErrorOr<String> to_utf8(StringView, IgnoreBOM, ErrorMode) override;
|
||||||
virtual ErrorOr<String> to_utf8(StringView) override;
|
|
||||||
virtual ErrorOr<size_t> length_in_utf16_code_units(StringView) override;
|
virtual ErrorOr<size_t> length_in_utf16_code_units(StringView) override;
|
||||||
|
|
||||||
private:
|
private:
|
||||||
|
|
@ -72,7 +61,6 @@ private:
|
||||||
class Latin1Decoder final : public Decoder {
|
class Latin1Decoder final : public Decoder {
|
||||||
public:
|
public:
|
||||||
virtual ErrorOr<void> process(StringView, Function<ErrorOr<void>(u32)> on_code_point) override;
|
virtual ErrorOr<void> process(StringView, Function<ErrorOr<void>(u32)> on_code_point) override;
|
||||||
virtual bool validate(StringView) override { return true; }
|
|
||||||
virtual ErrorOr<size_t> length_in_utf16_code_units(StringView) override;
|
virtual ErrorOr<size_t> length_in_utf16_code_units(StringView) override;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
@ -130,7 +118,7 @@ static void append_decoded_bytes(void* context, u8 const* data, size_t length)
|
||||||
decode_context.result = decode_context.builder.try_append(StringView { data, length });
|
decode_context.result = decode_context.builder.try_append(StringView { data, length });
|
||||||
}
|
}
|
||||||
|
|
||||||
ErrorOr<String> rust_decode_to_utf8(StringView encoding, StringView input, RemoveBOM remove_bom)
|
ErrorOr<String> rust_decode_to_utf8(StringView encoding, StringView input, IgnoreBOM ignore_bom, ErrorMode error_mode)
|
||||||
{
|
{
|
||||||
DecodeContext context { .builder = StringBuilder(input.length()) };
|
DecodeContext context { .builder = StringBuilder(input.length()) };
|
||||||
auto succeeded = FFI::textcodec_rust_decode_to_utf8(
|
auto succeeded = FFI::textcodec_rust_decode_to_utf8(
|
||||||
|
|
@ -138,36 +126,27 @@ ErrorOr<String> rust_decode_to_utf8(StringView encoding, StringView input, Remov
|
||||||
encoding.length(),
|
encoding.length(),
|
||||||
reinterpret_cast<u8 const*>(input.characters_without_null_termination()),
|
reinterpret_cast<u8 const*>(input.characters_without_null_termination()),
|
||||||
input.length(),
|
input.length(),
|
||||||
remove_bom == RemoveBOM::Yes,
|
ignore_bom == IgnoreBOM::No,
|
||||||
|
error_mode == ErrorMode::Fatal,
|
||||||
&context,
|
&context,
|
||||||
append_decoded_bytes);
|
append_decoded_bytes);
|
||||||
if (!succeeded)
|
if (!succeeded)
|
||||||
return Error::from_errno(EINVAL);
|
return Error::from_string_literal("Failed to decode input");
|
||||||
TRY(context.result);
|
TRY(context.result);
|
||||||
return context.builder.to_string_without_validation();
|
return context.builder.to_string_without_validation();
|
||||||
}
|
}
|
||||||
|
|
||||||
ErrorOr<void> rust_process(StringView encoding, StringView input, RemoveBOM remove_bom, Function<ErrorOr<void>(u32)> on_code_point)
|
ErrorOr<void> rust_process(StringView encoding, StringView input, IgnoreBOM ignore_bom, Function<ErrorOr<void>(u32)> on_code_point)
|
||||||
{
|
{
|
||||||
auto utf8 = TRY(rust_decode_to_utf8(encoding, input, remove_bom));
|
auto utf8 = TRY(rust_decode_to_utf8(encoding, input, ignore_bom, ErrorMode::Replacement));
|
||||||
for (auto code_point : Utf8View { utf8 })
|
for (auto code_point : Utf8View { utf8 })
|
||||||
TRY(on_code_point(code_point));
|
TRY(on_code_point(code_point));
|
||||||
return {};
|
return {};
|
||||||
}
|
}
|
||||||
|
|
||||||
bool rust_validate(StringView encoding, StringView input, RemoveBOM remove_bom)
|
ErrorOr<size_t> rust_length_in_utf16_code_units(StringView encoding, StringView input, IgnoreBOM ignore_bom)
|
||||||
{
|
{
|
||||||
return FFI::textcodec_rust_validate(
|
auto utf8 = TRY(rust_decode_to_utf8(encoding, input, ignore_bom, ErrorMode::Replacement));
|
||||||
reinterpret_cast<u8 const*>(encoding.characters_without_null_termination()),
|
|
||||||
encoding.length(),
|
|
||||||
reinterpret_cast<u8 const*>(input.characters_without_null_termination()),
|
|
||||||
input.length(),
|
|
||||||
remove_bom == RemoveBOM::Yes);
|
|
||||||
}
|
|
||||||
|
|
||||||
ErrorOr<size_t> rust_length_in_utf16_code_units(StringView encoding, StringView input, RemoveBOM remove_bom)
|
|
||||||
{
|
|
||||||
auto utf8 = TRY(rust_decode_to_utf8(encoding, input, remove_bom));
|
|
||||||
size_t length = 0;
|
size_t length = 0;
|
||||||
for (auto code_point : Utf8View { utf8 })
|
for (auto code_point : Utf8View { utf8 })
|
||||||
length += code_point <= 0xffff ? 1 : 2;
|
length += code_point <= 0xffff ? 1 : 2;
|
||||||
|
|
@ -188,7 +167,7 @@ Optional<StringView> get_static_encoding_name_from_rust(StringView label)
|
||||||
return StringView { encoding_name, encoding_name_length };
|
return StringView { encoding_name, encoding_name_length };
|
||||||
}
|
}
|
||||||
|
|
||||||
ErrorOr<String> rust_streaming_decode_to_utf8(FFI::TextCodecRustStreamingDecoder* decoder, ReadonlyBytes input, bool last)
|
ErrorOr<String> rust_streaming_decode_to_utf8(FFI::TextCodecRustStreamingDecoder* decoder, ReadonlyBytes input, bool last, ErrorMode error_mode)
|
||||||
{
|
{
|
||||||
DecodeContext context { .builder = StringBuilder(input.size()) };
|
DecodeContext context { .builder = StringBuilder(input.size()) };
|
||||||
auto succeeded = FFI::textcodec_rust_streaming_decoder_decode_to_utf8(
|
auto succeeded = FFI::textcodec_rust_streaming_decoder_decode_to_utf8(
|
||||||
|
|
@ -196,10 +175,11 @@ ErrorOr<String> rust_streaming_decode_to_utf8(FFI::TextCodecRustStreamingDecoder
|
||||||
input.data(),
|
input.data(),
|
||||||
input.size(),
|
input.size(),
|
||||||
last,
|
last,
|
||||||
|
error_mode == ErrorMode::Fatal,
|
||||||
&context,
|
&context,
|
||||||
append_decoded_bytes);
|
append_decoded_bytes);
|
||||||
if (!succeeded)
|
if (!succeeded)
|
||||||
return Error::from_errno(EINVAL);
|
return Error::from_string_literal("Failed to decode input");
|
||||||
TRY(context.result);
|
TRY(context.result);
|
||||||
return context.builder.to_string_without_validation();
|
return context.builder.to_string_without_validation();
|
||||||
}
|
}
|
||||||
|
|
@ -367,7 +347,7 @@ ErrorOr<String> convert_input_to_utf8_using_given_decoder_unless_there_is_a_byte
|
||||||
|
|
||||||
// 3. Process a queue with an instance of encoding’s decoder, ioQueue, output, and "replacement".
|
// 3. Process a queue with an instance of encoding’s decoder, ioQueue, output, and "replacement".
|
||||||
// FIXME: This isn't the exact same as the spec, which is written in terms of I/O queues.
|
// FIXME: This isn't the exact same as the spec, which is written in terms of I/O queues.
|
||||||
auto output = TRY(actual_decoder->to_utf8(input));
|
auto output = TRY(actual_decoder->to_utf8(input, IgnoreBOM::No, ErrorMode::Replacement));
|
||||||
|
|
||||||
// 4. Return output.
|
// 4. Return output.
|
||||||
return output;
|
return output;
|
||||||
|
|
@ -410,18 +390,7 @@ StringView get_output_encoding(StringView encoding)
|
||||||
return encoding;
|
return encoding;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool Decoder::validate(StringView input)
|
ErrorOr<String> Decoder::to_utf8(StringView input, IgnoreBOM, ErrorMode)
|
||||||
{
|
|
||||||
auto result = this->process(input, [](auto code_point) -> ErrorOr<void> {
|
|
||||||
if (code_point == replacement_code_point)
|
|
||||||
return Error::from_errno(EINVAL);
|
|
||||||
return {};
|
|
||||||
});
|
|
||||||
|
|
||||||
return !result.is_error();
|
|
||||||
}
|
|
||||||
|
|
||||||
ErrorOr<String> Decoder::to_utf8(StringView input)
|
|
||||||
{
|
{
|
||||||
StringBuilder builder(input.length());
|
StringBuilder builder(input.length());
|
||||||
TRY(process(input, [&builder](u32 c) { return builder.try_append_code_point(c); }));
|
TRY(process(input, [&builder](u32 c) { return builder.try_append_code_point(c); }));
|
||||||
|
|
@ -453,24 +422,19 @@ ErrorOr<void> Decoder::process_code_points(StringView input, Function<ErrorOr<vo
|
||||||
return process(input, move(on_code_point));
|
return process(input, move(on_code_point));
|
||||||
}
|
}
|
||||||
|
|
||||||
bool RustDecoder::validate(StringView input)
|
ErrorOr<String> RustDecoder::to_utf8(StringView input, IgnoreBOM ignore_bom, ErrorMode error_mode)
|
||||||
{
|
{
|
||||||
return rust_validate(m_encoding, input, RemoveBOM::No);
|
return rust_decode_to_utf8(m_encoding, input, ignore_bom, error_mode);
|
||||||
}
|
|
||||||
|
|
||||||
ErrorOr<String> RustDecoder::to_utf8(StringView input)
|
|
||||||
{
|
|
||||||
return rust_decode_to_utf8(m_encoding, input, RemoveBOM::No);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
ErrorOr<size_t> RustDecoder::length_in_utf16_code_units(StringView input)
|
ErrorOr<size_t> RustDecoder::length_in_utf16_code_units(StringView input)
|
||||||
{
|
{
|
||||||
return rust_length_in_utf16_code_units(m_encoding, input, RemoveBOM::No);
|
return rust_length_in_utf16_code_units(m_encoding, input, IgnoreBOM::Yes);
|
||||||
}
|
}
|
||||||
|
|
||||||
ErrorOr<void> RustDecoder::process(StringView input, Function<ErrorOr<void>(u32)> on_code_point)
|
ErrorOr<void> RustDecoder::process(StringView input, Function<ErrorOr<void>(u32)> on_code_point)
|
||||||
{
|
{
|
||||||
return rust_process(m_encoding, input, RemoveBOM::No, move(on_code_point));
|
return rust_process(m_encoding, input, IgnoreBOM::Yes, move(on_code_point));
|
||||||
}
|
}
|
||||||
|
|
||||||
ErrorOr<void> Latin1Decoder::process(StringView input, Function<ErrorOr<void>(u32)> on_code_point)
|
ErrorOr<void> Latin1Decoder::process(StringView input, Function<ErrorOr<void>(u32)> on_code_point)
|
||||||
|
|
@ -485,12 +449,13 @@ ErrorOr<size_t> Latin1Decoder::length_in_utf16_code_units(StringView input)
|
||||||
return input.length();
|
return input.length();
|
||||||
}
|
}
|
||||||
|
|
||||||
StreamingDecoder::StreamingDecoder(StringView encoding)
|
StreamingDecoder::StreamingDecoder(StringView encoding, IgnoreBOM ignore_bom, ErrorMode error_mode)
|
||||||
|
: m_error_mode(error_mode)
|
||||||
{
|
{
|
||||||
m_decoder = FFI::textcodec_rust_streaming_decoder_new(
|
m_decoder = FFI::textcodec_rust_streaming_decoder_new(
|
||||||
reinterpret_cast<u8 const*>(encoding.characters_without_null_termination()),
|
reinterpret_cast<u8 const*>(encoding.characters_without_null_termination()),
|
||||||
encoding.length(),
|
encoding.length(),
|
||||||
true);
|
ignore_bom == IgnoreBOM::No);
|
||||||
VERIFY(m_decoder);
|
VERIFY(m_decoder);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -501,72 +466,57 @@ StreamingDecoder::~StreamingDecoder()
|
||||||
|
|
||||||
ErrorOr<String> StreamingDecoder::to_utf8(ReadonlyBytes input)
|
ErrorOr<String> StreamingDecoder::to_utf8(ReadonlyBytes input)
|
||||||
{
|
{
|
||||||
return rust_streaming_decode_to_utf8(static_cast<FFI::TextCodecRustStreamingDecoder*>(m_decoder), input, false);
|
return rust_streaming_decode_to_utf8(static_cast<FFI::TextCodecRustStreamingDecoder*>(m_decoder), input, false, m_error_mode);
|
||||||
}
|
}
|
||||||
|
|
||||||
ErrorOr<String> StreamingDecoder::finish()
|
ErrorOr<String> StreamingDecoder::finish()
|
||||||
{
|
{
|
||||||
return rust_streaming_decode_to_utf8(static_cast<FFI::TextCodecRustStreamingDecoder*>(m_decoder), {}, true);
|
return rust_streaming_decode_to_utf8(static_cast<FFI::TextCodecRustStreamingDecoder*>(m_decoder), {}, true, m_error_mode);
|
||||||
}
|
}
|
||||||
|
|
||||||
ErrorOr<void> UTF8Decoder::process(StringView input, Function<ErrorOr<void>(u32)> on_code_point)
|
ErrorOr<void> UTF8Decoder::process(StringView input, Function<ErrorOr<void>(u32)> on_code_point)
|
||||||
{
|
{
|
||||||
return rust_process("UTF-8"sv, input, RemoveBOM::No, move(on_code_point));
|
return rust_process("UTF-8"sv, input, IgnoreBOM::Yes, move(on_code_point));
|
||||||
}
|
}
|
||||||
|
|
||||||
bool UTF8Decoder::validate(StringView input)
|
ErrorOr<String> UTF8Decoder::to_utf8(StringView input, IgnoreBOM ignore_bom, ErrorMode error_mode)
|
||||||
{
|
{
|
||||||
return rust_validate("UTF-8"sv, input, RemoveBOM::No);
|
return rust_decode_to_utf8("UTF-8"sv, input, ignore_bom, error_mode);
|
||||||
}
|
|
||||||
|
|
||||||
ErrorOr<String> UTF8Decoder::to_utf8(StringView input)
|
|
||||||
{
|
|
||||||
return rust_decode_to_utf8("UTF-8"sv, input, RemoveBOM::Yes);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
ErrorOr<size_t> UTF8Decoder::length_in_utf16_code_units(StringView input)
|
ErrorOr<size_t> UTF8Decoder::length_in_utf16_code_units(StringView input)
|
||||||
{
|
{
|
||||||
return rust_length_in_utf16_code_units("UTF-8"sv, input, RemoveBOM::Yes);
|
return rust_length_in_utf16_code_units("UTF-8"sv, input, IgnoreBOM::No);
|
||||||
}
|
|
||||||
|
|
||||||
bool UTF16BEDecoder::validate(StringView input)
|
|
||||||
{
|
|
||||||
return rust_validate("UTF-16BE"sv, input, RemoveBOM::No);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
ErrorOr<void> UTF16BEDecoder::process(StringView input, Function<ErrorOr<void>(u32)> on_code_point)
|
ErrorOr<void> UTF16BEDecoder::process(StringView input, Function<ErrorOr<void>(u32)> on_code_point)
|
||||||
{
|
{
|
||||||
return rust_process("UTF-16BE"sv, input, RemoveBOM::Yes, move(on_code_point));
|
return rust_process("UTF-16BE"sv, input, IgnoreBOM::No, move(on_code_point));
|
||||||
}
|
}
|
||||||
|
|
||||||
ErrorOr<String> UTF16BEDecoder::to_utf8(StringView input)
|
ErrorOr<String> UTF16BEDecoder::to_utf8(StringView input, IgnoreBOM ignore_bom, ErrorMode error_mode)
|
||||||
{
|
{
|
||||||
return rust_decode_to_utf8("UTF-16BE"sv, input, RemoveBOM::Yes);
|
return rust_decode_to_utf8("UTF-16BE"sv, input, ignore_bom, error_mode);
|
||||||
}
|
}
|
||||||
|
|
||||||
ErrorOr<size_t> UTF16BEDecoder::length_in_utf16_code_units(StringView input)
|
ErrorOr<size_t> UTF16BEDecoder::length_in_utf16_code_units(StringView input)
|
||||||
{
|
{
|
||||||
return rust_length_in_utf16_code_units("UTF-16BE"sv, input, RemoveBOM::Yes);
|
return rust_length_in_utf16_code_units("UTF-16BE"sv, input, IgnoreBOM::No);
|
||||||
}
|
|
||||||
|
|
||||||
bool UTF16LEDecoder::validate(StringView input)
|
|
||||||
{
|
|
||||||
return rust_validate("UTF-16LE"sv, input, RemoveBOM::No);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
ErrorOr<void> UTF16LEDecoder::process(StringView input, Function<ErrorOr<void>(u32)> on_code_point)
|
ErrorOr<void> UTF16LEDecoder::process(StringView input, Function<ErrorOr<void>(u32)> on_code_point)
|
||||||
{
|
{
|
||||||
return rust_process("UTF-16LE"sv, input, RemoveBOM::Yes, move(on_code_point));
|
return rust_process("UTF-16LE"sv, input, IgnoreBOM::No, move(on_code_point));
|
||||||
}
|
}
|
||||||
|
|
||||||
ErrorOr<String> UTF16LEDecoder::to_utf8(StringView input)
|
ErrorOr<String> UTF16LEDecoder::to_utf8(StringView input, IgnoreBOM ignore_bom, ErrorMode error_mode)
|
||||||
{
|
{
|
||||||
return rust_decode_to_utf8("UTF-16LE"sv, input, RemoveBOM::Yes);
|
return rust_decode_to_utf8("UTF-16LE"sv, input, ignore_bom, error_mode);
|
||||||
}
|
}
|
||||||
|
|
||||||
ErrorOr<size_t> UTF16LEDecoder::length_in_utf16_code_units(StringView input)
|
ErrorOr<size_t> UTF16LEDecoder::length_in_utf16_code_units(StringView input)
|
||||||
{
|
{
|
||||||
return rust_length_in_utf16_code_units("UTF-16LE"sv, input, RemoveBOM::Yes);
|
return rust_length_in_utf16_code_units("UTF-16LE"sv, input, IgnoreBOM::No);
|
||||||
}
|
}
|
||||||
|
|
||||||
// https://infra.spec.whatwg.org/#isomorphic-decode
|
// https://infra.spec.whatwg.org/#isomorphic-decode
|
||||||
|
|
|
||||||
|
|
@ -19,10 +19,20 @@
|
||||||
|
|
||||||
namespace TextCodec {
|
namespace TextCodec {
|
||||||
|
|
||||||
|
enum class IgnoreBOM {
|
||||||
|
Yes,
|
||||||
|
No,
|
||||||
|
};
|
||||||
|
|
||||||
|
// https://encoding.spec.whatwg.org/#concept-encoding-error-mode
|
||||||
|
enum class ErrorMode {
|
||||||
|
Replacement,
|
||||||
|
Fatal,
|
||||||
|
};
|
||||||
|
|
||||||
class TEXTCODEC_API Decoder {
|
class TEXTCODEC_API Decoder {
|
||||||
public:
|
public:
|
||||||
virtual bool validate(StringView);
|
virtual ErrorOr<String> to_utf8(StringView, IgnoreBOM, ErrorMode);
|
||||||
virtual ErrorOr<String> to_utf8(StringView);
|
|
||||||
virtual ErrorOr<Utf16String> to_utf16(StringView);
|
virtual ErrorOr<Utf16String> to_utf16(StringView);
|
||||||
virtual ErrorOr<size_t> length_in_utf16_code_units(StringView);
|
virtual ErrorOr<size_t> length_in_utf16_code_units(StringView);
|
||||||
ErrorOr<void> process_code_points(StringView, Function<ErrorOr<void>(u32)>);
|
ErrorOr<void> process_code_points(StringView, Function<ErrorOr<void>(u32)>);
|
||||||
|
|
@ -36,13 +46,14 @@ class TEXTCODEC_API StreamingDecoder final {
|
||||||
AK_MAKE_NONCOPYABLE(StreamingDecoder);
|
AK_MAKE_NONCOPYABLE(StreamingDecoder);
|
||||||
|
|
||||||
public:
|
public:
|
||||||
explicit StreamingDecoder(StringView encoding);
|
StreamingDecoder(StringView encoding, IgnoreBOM, ErrorMode);
|
||||||
~StreamingDecoder();
|
~StreamingDecoder();
|
||||||
|
|
||||||
ErrorOr<String> to_utf8(ReadonlyBytes);
|
ErrorOr<String> to_utf8(ReadonlyBytes);
|
||||||
ErrorOr<String> finish();
|
ErrorOr<String> finish();
|
||||||
|
|
||||||
private:
|
private:
|
||||||
|
ErrorMode m_error_mode { ErrorMode::Replacement };
|
||||||
void* m_decoder { nullptr };
|
void* m_decoder { nullptr };
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -93,6 +93,7 @@ pub unsafe extern "C" fn textcodec_rust_decode_to_utf8(
|
||||||
input: *const u8,
|
input: *const u8,
|
||||||
input_len: usize,
|
input_len: usize,
|
||||||
remove_bom: bool,
|
remove_bom: bool,
|
||||||
|
fatal: bool,
|
||||||
ctx: *mut c_void,
|
ctx: *mut c_void,
|
||||||
on_bytes: FfiBytesFn,
|
on_bytes: FfiBytesFn,
|
||||||
) -> bool {
|
) -> bool {
|
||||||
|
|
@ -108,60 +109,20 @@ pub unsafe extern "C" fn textcodec_rust_decode_to_utf8(
|
||||||
return false;
|
return false;
|
||||||
};
|
};
|
||||||
|
|
||||||
let (output, _) = if remove_bom {
|
let (output, had_errors) = if remove_bom {
|
||||||
encoding.decode_with_bom_removal(input)
|
encoding.decode_with_bom_removal(input)
|
||||||
} else {
|
} else {
|
||||||
encoding.decode_without_bom_handling(input)
|
encoding.decode_without_bom_handling(input)
|
||||||
};
|
};
|
||||||
|
if fatal && had_errors {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
on_bytes(ctx, output.as_bytes().as_ptr(), output.len());
|
on_bytes(ctx, output.as_bytes().as_ptr(), output.len());
|
||||||
true
|
true
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// # Safety
|
|
||||||
/// - `encoding_label`/`encoding_label_len` and `input`/`input_len` must be valid byte slices.
|
|
||||||
#[unsafe(no_mangle)]
|
|
||||||
pub unsafe extern "C" fn textcodec_rust_validate(
|
|
||||||
encoding_label: *const u8,
|
|
||||||
encoding_label_len: usize,
|
|
||||||
input: *const u8,
|
|
||||||
input_len: usize,
|
|
||||||
remove_bom: bool,
|
|
||||||
) -> bool {
|
|
||||||
unsafe {
|
|
||||||
abort_on_panic(|| {
|
|
||||||
let Some(label) = bytes_from_raw(encoding_label, encoding_label_len) else {
|
|
||||||
return false;
|
|
||||||
};
|
|
||||||
let Some(input) = bytes_from_raw(input, input_len) else {
|
|
||||||
return false;
|
|
||||||
};
|
|
||||||
let Some(encoding) = Encoding::for_label(label) else {
|
|
||||||
return false;
|
|
||||||
};
|
|
||||||
|
|
||||||
let input = if remove_bom {
|
|
||||||
if encoding == encoding_rs::UTF_8 && input.starts_with(b"\xEF\xBB\xBF") {
|
|
||||||
&input[3..]
|
|
||||||
} else if (encoding == encoding_rs::UTF_16LE && input.starts_with(b"\xFF\xFE"))
|
|
||||||
|| (encoding == encoding_rs::UTF_16BE && input.starts_with(b"\xFE\xFF"))
|
|
||||||
{
|
|
||||||
&input[2..]
|
|
||||||
} else {
|
|
||||||
input
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
input
|
|
||||||
};
|
|
||||||
|
|
||||||
encoding
|
|
||||||
.decode_without_bom_handling_and_without_replacement(input)
|
|
||||||
.is_some()
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// # Safety
|
/// # Safety
|
||||||
/// - `encoding_label`/`encoding_label_len` must be a valid byte slice.
|
/// - `encoding_label`/`encoding_label_len` must be a valid byte slice.
|
||||||
/// - The returned pointer must be freed with `textcodec_rust_streaming_decoder_free`.
|
/// - The returned pointer must be freed with `textcodec_rust_streaming_decoder_free`.
|
||||||
|
|
@ -212,6 +173,7 @@ pub unsafe extern "C" fn textcodec_rust_streaming_decoder_decode_to_utf8(
|
||||||
input: *const u8,
|
input: *const u8,
|
||||||
input_len: usize,
|
input_len: usize,
|
||||||
last: bool,
|
last: bool,
|
||||||
|
fatal: bool,
|
||||||
ctx: *mut c_void,
|
ctx: *mut c_void,
|
||||||
on_bytes: FfiBytesFn,
|
on_bytes: FfiBytesFn,
|
||||||
) -> bool {
|
) -> bool {
|
||||||
|
|
@ -230,7 +192,10 @@ pub unsafe extern "C" fn textcodec_rust_streaming_decoder_decode_to_utf8(
|
||||||
};
|
};
|
||||||
let mut output = String::with_capacity(output_capacity);
|
let mut output = String::with_capacity(output_capacity);
|
||||||
|
|
||||||
let (result, _, _) = decoder.decoder.decode_to_string(input, &mut output, last);
|
let (result, _, had_errors) = decoder.decoder.decode_to_string(input, &mut output, last);
|
||||||
|
if fatal && had_errors {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
if !output.is_empty() {
|
if !output.is_empty() {
|
||||||
on_bytes(ctx, output.as_ptr(), output.len());
|
on_bytes(ctx, output.as_ptr(), output.len());
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -36,7 +36,7 @@ static String decode_and_filter_code_points(StringView input, StringView encodin
|
||||||
input = input.substring_view(3);
|
input = input.substring_view(3);
|
||||||
return String::from_utf8_without_validation(input.bytes());
|
return String::from_utf8_without_validation(input.bytes());
|
||||||
}
|
}
|
||||||
return MUST(decoder->to_utf8(input));
|
return MUST(decoder->to_utf8(input, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement));
|
||||||
}();
|
}();
|
||||||
|
|
||||||
// OPTIMIZATION: If the input doesn't contain any filterable characters, we can skip the filtering
|
// OPTIMIZATION: If the input doesn't contain any filterable characters, we can skip the filtering
|
||||||
|
|
|
||||||
|
|
@ -175,7 +175,7 @@ Vector<Token> Tokenizer::tokenize(StringView input, StringView encoding, Tokeniz
|
||||||
input = input.substring_view(3);
|
input = input.substring_view(3);
|
||||||
return String::from_utf8_without_validation(input.bytes());
|
return String::from_utf8_without_validation(input.bytes());
|
||||||
}
|
}
|
||||||
return MUST(decoder->to_utf8(input));
|
return MUST(decoder->to_utf8(input, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement));
|
||||||
}();
|
}();
|
||||||
|
|
||||||
// OPTIMIZATION: If the input doesn't contain any filterable characters, we can skip the filtering
|
// OPTIMIZATION: If the input doesn't contain any filterable characters, we can skip the filtering
|
||||||
|
|
|
||||||
|
|
@ -73,11 +73,12 @@ bool build_xml_document(DOM::Document& document, ByteBuffer const& data, Optiona
|
||||||
}
|
}
|
||||||
VERIFY(decoder.has_value());
|
VERIFY(decoder.has_value());
|
||||||
// Well-formed XML documents contain only properly encoded characters
|
// Well-formed XML documents contain only properly encoded characters
|
||||||
if (!decoder->validate(data)) {
|
auto source_or_error = decoder->to_utf8(data, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Fatal);
|
||||||
|
if (source_or_error.is_error()) {
|
||||||
convert_to_xml_error_document(document, "XML Document contains improperly-encoded characters"_utf16);
|
convert_to_xml_error_document(document, "XML Document contains improperly-encoded characters"_utf16);
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
auto source = decoder->to_utf8(data).release_value_but_fixme_should_propagate_errors();
|
auto source = source_or_error.release_value();
|
||||||
XML::Parser parser(source, { .resolve_named_html_entity = resolve_named_html_entity });
|
XML::Parser parser(source, { .resolve_named_html_entity = resolve_named_html_entity });
|
||||||
XMLDocumentBuilder builder { document };
|
XMLDocumentBuilder builder { document };
|
||||||
auto result = parser.parse_with_listener(builder);
|
auto result = parser.parse_with_listener(builder);
|
||||||
|
|
@ -198,13 +199,7 @@ static WebIDL::ExceptionOr<GC::Ref<DOM::Document>> load_xml_document(HTML::Navig
|
||||||
}
|
}
|
||||||
VERIFY(decoder.has_value());
|
VERIFY(decoder.has_value());
|
||||||
// Well-formed XML documents contain only properly encoded characters
|
// Well-formed XML documents contain only properly encoded characters
|
||||||
if (!decoder->validate(data)) {
|
auto source = decoder->to_utf8(data, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Fatal);
|
||||||
// FIXME: Insert error message into the document.
|
|
||||||
dbgln("XML Document contains improperly-encoded characters");
|
|
||||||
convert_to_xml_error_document(document, "XML Document contains improperly-encoded characters"_utf16);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
auto source = decoder->to_utf8(data);
|
|
||||||
if (source.is_error()) {
|
if (source.is_error()) {
|
||||||
// FIXME: Insert error message into the document.
|
// FIXME: Insert error message into the document.
|
||||||
dbgln("Failed to decode XML document: {}", source.error());
|
dbgln("Failed to decode XML document: {}", source.error());
|
||||||
|
|
|
||||||
|
|
@ -18,7 +18,7 @@ namespace Web::Encoding {
|
||||||
GC_DEFINE_ALLOCATOR(TextDecoder);
|
GC_DEFINE_ALLOCATOR(TextDecoder);
|
||||||
|
|
||||||
// https://encoding.spec.whatwg.org/#dom-textdecoder
|
// https://encoding.spec.whatwg.org/#dom-textdecoder
|
||||||
WebIDL::ExceptionOr<GC::Ref<TextDecoder>> TextDecoder::construct_impl(JS::Realm& realm, FlyString label, Optional<Bindings::TextDecoderOptions> const& options)
|
WebIDL::ExceptionOr<GC::Ref<TextDecoder>> TextDecoder::construct_impl(JS::Realm& realm, StringView label, Bindings::TextDecoderOptions const& options)
|
||||||
{
|
{
|
||||||
auto& vm = realm.vm();
|
auto& vm = realm.vm();
|
||||||
|
|
||||||
|
|
@ -35,10 +35,10 @@ WebIDL::ExceptionOr<GC::Ref<TextDecoder>> TextDecoder::construct_impl(JS::Realm&
|
||||||
auto lowercase_encoding_name = encoding.value().to_ascii_lowercase_string();
|
auto lowercase_encoding_name = encoding.value().to_ascii_lowercase_string();
|
||||||
|
|
||||||
// 4. If options["fatal"] is true, then set this’s error mode to "fatal".
|
// 4. If options["fatal"] is true, then set this’s error mode to "fatal".
|
||||||
auto error_mode = options.value_or({}).fatal ? ErrorMode::Fatal : ErrorMode::Replacement;
|
auto error_mode = options.fatal ? TextCodec::ErrorMode::Fatal : TextCodec::ErrorMode::Replacement;
|
||||||
|
|
||||||
// 5. Set this’s ignore BOM to options["ignoreBOM"].
|
// 5. Set this’s ignore BOM to options["ignoreBOM"].
|
||||||
auto ignore_bom = options.value_or({}).ignore_bom;
|
auto ignore_bom = options.ignore_bom;
|
||||||
|
|
||||||
// NOTE: This should happen in decode(), but we don't support streaming yet and share decoders across calls.
|
// NOTE: This should happen in decode(), but we don't support streaming yet and share decoders across calls.
|
||||||
auto decoder = TextCodec::decoder_for_exact_name(encoding.value());
|
auto decoder = TextCodec::decoder_for_exact_name(encoding.value());
|
||||||
|
|
@ -48,7 +48,7 @@ WebIDL::ExceptionOr<GC::Ref<TextDecoder>> TextDecoder::construct_impl(JS::Realm&
|
||||||
}
|
}
|
||||||
|
|
||||||
// https://encoding.spec.whatwg.org/#dom-textdecoder
|
// https://encoding.spec.whatwg.org/#dom-textdecoder
|
||||||
TextDecoder::TextDecoder(JS::Realm& realm, TextCodec::Decoder& decoder, FlyString encoding, ErrorMode error_mode, bool ignore_bom)
|
TextDecoder::TextDecoder(JS::Realm& realm, TextCodec::Decoder& decoder, FlyString encoding, TextCodec::ErrorMode error_mode, bool ignore_bom)
|
||||||
: PlatformObject(realm)
|
: PlatformObject(realm)
|
||||||
, TextDecoderCommonMixin(decoder, move(encoding), error_mode, ignore_bom)
|
, TextDecoderCommonMixin(decoder, move(encoding), error_mode, ignore_bom)
|
||||||
{
|
{
|
||||||
|
|
@ -65,18 +65,23 @@ void TextDecoder::initialize(JS::Realm& realm)
|
||||||
// https://encoding.spec.whatwg.org/#dom-textdecoder-decode
|
// https://encoding.spec.whatwg.org/#dom-textdecoder-decode
|
||||||
WebIDL::ExceptionOr<String> TextDecoder::decode(Optional<WebIDL::BufferSourceVariant> input, Optional<Bindings::TextDecodeOptions> const&) const
|
WebIDL::ExceptionOr<String> TextDecoder::decode(Optional<WebIDL::BufferSourceVariant> input, Optional<Bindings::TextDecodeOptions> const&) const
|
||||||
{
|
{
|
||||||
if (!input.has_value())
|
auto ignore_bom = m_ignore_bom ? TextCodec::IgnoreBOM::Yes : TextCodec::IgnoreBOM::No;
|
||||||
return TRY_OR_THROW_OOM(vm(), m_decoder.to_utf8({}));
|
if (!input.has_value()) {
|
||||||
|
auto result = m_decoder.to_utf8({}, ignore_bom, m_error_mode);
|
||||||
|
if (result.is_error() && result.error().code() != ENOMEM)
|
||||||
|
return WebIDL::SimpleException { WebIDL::SimpleExceptionType::TypeError, "Decoding failed"sv };
|
||||||
|
return TRY_OR_THROW_OOM(vm(), move(result));
|
||||||
|
}
|
||||||
|
|
||||||
// FIXME: Implement the streaming stuff.
|
// FIXME: Implement the streaming stuff.
|
||||||
auto data_buffer_or_error = WebIDL::get_buffer_source_copy(*input);
|
auto data_buffer_or_error = WebIDL::get_buffer_source_copy(*input);
|
||||||
if (data_buffer_or_error.is_error())
|
if (data_buffer_or_error.is_error())
|
||||||
return WebIDL::OperationError::create(realm(), "Failed to copy bytes from ArrayBuffer"_utf16);
|
return WebIDL::OperationError::create(realm(), "Failed to copy bytes from ArrayBuffer"_utf16);
|
||||||
auto& data_buffer = data_buffer_or_error.value();
|
auto& data_buffer = data_buffer_or_error.value();
|
||||||
auto result = TRY_OR_THROW_OOM(vm(), m_decoder.to_utf8({ data_buffer.data(), data_buffer.size() }));
|
auto result = m_decoder.to_utf8({ data_buffer.data(), data_buffer.size() }, ignore_bom, m_error_mode);
|
||||||
if (this->fatal() && result.contains(0xfffd))
|
if (result.is_error() && result.error().code() != ENOMEM)
|
||||||
return WebIDL::SimpleException { WebIDL::SimpleExceptionType::TypeError, "Decoding failed"sv };
|
return WebIDL::SimpleException { WebIDL::SimpleExceptionType::TypeError, "Decoding failed"sv };
|
||||||
return result;
|
return TRY_OR_THROW_OOM(vm(), move(result));
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -27,14 +27,14 @@ class TextDecoder
|
||||||
GC_DECLARE_ALLOCATOR(TextDecoder);
|
GC_DECLARE_ALLOCATOR(TextDecoder);
|
||||||
|
|
||||||
public:
|
public:
|
||||||
static WebIDL::ExceptionOr<GC::Ref<TextDecoder>> construct_impl(JS::Realm&, FlyString encoding, Optional<Bindings::TextDecoderOptions> const& options = {});
|
static WebIDL::ExceptionOr<GC::Ref<TextDecoder>> construct_impl(JS::Realm&, StringView label, Bindings::TextDecoderOptions const&);
|
||||||
|
|
||||||
virtual ~TextDecoder() override;
|
virtual ~TextDecoder() override;
|
||||||
|
|
||||||
WebIDL::ExceptionOr<String> decode(Optional<WebIDL::BufferSourceVariant>, Optional<Bindings::TextDecodeOptions> const& options = {}) const;
|
WebIDL::ExceptionOr<String> decode(Optional<WebIDL::BufferSourceVariant>, Optional<Bindings::TextDecodeOptions> const& options = {}) const;
|
||||||
|
|
||||||
private:
|
private:
|
||||||
TextDecoder(JS::Realm&, TextCodec::Decoder&, FlyString encoding, ErrorMode error_mode, bool ignore_bom);
|
TextDecoder(JS::Realm&, TextCodec::Decoder&, FlyString encoding, TextCodec::ErrorMode error_mode, bool ignore_bom);
|
||||||
|
|
||||||
virtual void initialize(JS::Realm&) override;
|
virtual void initialize(JS::Realm&) override;
|
||||||
};
|
};
|
||||||
|
|
|
||||||
|
|
@ -8,7 +8,7 @@
|
||||||
|
|
||||||
namespace Web::Encoding {
|
namespace Web::Encoding {
|
||||||
|
|
||||||
TextDecoderCommonMixin::TextDecoderCommonMixin(TextCodec::Decoder& decoder, FlyString encoding, ErrorMode error_mode, bool ignore_bom)
|
TextDecoderCommonMixin::TextDecoderCommonMixin(TextCodec::Decoder& decoder, FlyString encoding, TextCodec::ErrorMode error_mode, bool ignore_bom)
|
||||||
: m_decoder(decoder)
|
: m_decoder(decoder)
|
||||||
, m_encoding(move(encoding))
|
, m_encoding(move(encoding))
|
||||||
, m_error_mode(error_mode)
|
, m_error_mode(error_mode)
|
||||||
|
|
|
||||||
|
|
@ -7,7 +7,7 @@
|
||||||
#pragma once
|
#pragma once
|
||||||
|
|
||||||
#include <AK/FlyString.h>
|
#include <AK/FlyString.h>
|
||||||
#include <LibTextCodec/Forward.h>
|
#include <LibTextCodec/Decoder.h>
|
||||||
|
|
||||||
namespace Web::Encoding {
|
namespace Web::Encoding {
|
||||||
|
|
||||||
|
|
@ -20,19 +20,13 @@ public:
|
||||||
FlyString const& encoding() const { return m_encoding; }
|
FlyString const& encoding() const { return m_encoding; }
|
||||||
|
|
||||||
// https://encoding.spec.whatwg.org/#dom-textdecoder-fatal
|
// https://encoding.spec.whatwg.org/#dom-textdecoder-fatal
|
||||||
bool fatal() const { return m_error_mode == ErrorMode::Fatal; }
|
bool fatal() const { return m_error_mode == TextCodec::ErrorMode::Fatal; }
|
||||||
|
|
||||||
// https://encoding.spec.whatwg.org/#dom-textdecoder-ignorebom
|
// https://encoding.spec.whatwg.org/#dom-textdecoder-ignorebom
|
||||||
bool ignore_bom() const { return m_ignore_bom; }
|
bool ignore_bom() const { return m_ignore_bom; }
|
||||||
|
|
||||||
protected:
|
protected:
|
||||||
// https://encoding.spec.whatwg.org/#concept-encoding-error-mode
|
TextDecoderCommonMixin(TextCodec::Decoder& decoder, FlyString encoding, TextCodec::ErrorMode error_mode, bool ignore_bom);
|
||||||
enum class ErrorMode {
|
|
||||||
Replacement,
|
|
||||||
Fatal,
|
|
||||||
};
|
|
||||||
|
|
||||||
TextDecoderCommonMixin(TextCodec::Decoder& decoder, FlyString encoding, ErrorMode error_mode, bool ignore_bom);
|
|
||||||
|
|
||||||
// https://encoding.spec.whatwg.org/#textdecodercommon-decoder
|
// https://encoding.spec.whatwg.org/#textdecodercommon-decoder
|
||||||
TextCodec::Decoder& m_decoder;
|
TextCodec::Decoder& m_decoder;
|
||||||
|
|
@ -41,13 +35,10 @@ protected:
|
||||||
FlyString m_encoding;
|
FlyString m_encoding;
|
||||||
|
|
||||||
// https://encoding.spec.whatwg.org/#textdecoder-error-mode
|
// https://encoding.spec.whatwg.org/#textdecoder-error-mode
|
||||||
ErrorMode m_error_mode { ErrorMode::Replacement };
|
TextCodec::ErrorMode m_error_mode { TextCodec::ErrorMode::Replacement };
|
||||||
|
|
||||||
// https://encoding.spec.whatwg.org/#textdecoder-ignore-bom-flag
|
// https://encoding.spec.whatwg.org/#textdecoder-ignore-bom-flag
|
||||||
bool m_ignore_bom { false };
|
bool m_ignore_bom { false };
|
||||||
|
|
||||||
// https://encoding.spec.whatwg.org/#textdecoder-bom-seen-flag
|
|
||||||
bool m_bom_seen { false };
|
|
||||||
};
|
};
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -15,6 +15,7 @@
|
||||||
#include <LibWeb/Bindings/TextDecoder.h>
|
#include <LibWeb/Bindings/TextDecoder.h>
|
||||||
#include <LibWeb/Bindings/TextDecoderStream.h>
|
#include <LibWeb/Bindings/TextDecoderStream.h>
|
||||||
#include <LibWeb/Encoding/TextDecoderStream.h>
|
#include <LibWeb/Encoding/TextDecoderStream.h>
|
||||||
|
#include <LibWeb/HTML/Scripting/TemporaryExecutionContext.h>
|
||||||
#include <LibWeb/Streams/TransformStream.h>
|
#include <LibWeb/Streams/TransformStream.h>
|
||||||
#include <LibWeb/Streams/TransformStreamOperations.h>
|
#include <LibWeb/Streams/TransformStreamOperations.h>
|
||||||
#include <LibWeb/WebIDL/AbstractOperations.h>
|
#include <LibWeb/WebIDL/AbstractOperations.h>
|
||||||
|
|
@ -38,7 +39,7 @@ WebIDL::ExceptionOr<GC::Ref<TextDecoderStream>> TextDecoderStream::construct_imp
|
||||||
auto lowercase_encoding_name = encoding.value().to_ascii_lowercase_string();
|
auto lowercase_encoding_name = encoding.value().to_ascii_lowercase_string();
|
||||||
|
|
||||||
// 4. If options["fatal"] is true, then set this’s error mode to "fatal".
|
// 4. If options["fatal"] is true, then set this’s error mode to "fatal".
|
||||||
auto error_mode = options.fatal ? ErrorMode::Fatal : ErrorMode::Replacement;
|
auto error_mode = options.fatal ? TextCodec::ErrorMode::Fatal : TextCodec::ErrorMode::Replacement;
|
||||||
|
|
||||||
// 5. Set this’s ignore BOM to options["ignoreBOM"].
|
// 5. Set this’s ignore BOM to options["ignoreBOM"].
|
||||||
auto ignore_bom = options.ignore_bom;
|
auto ignore_bom = options.ignore_bom;
|
||||||
|
|
@ -59,6 +60,7 @@ WebIDL::ExceptionOr<GC::Ref<TextDecoderStream>> TextDecoderStream::construct_imp
|
||||||
// algorithm with this and chunk.
|
// algorithm with this and chunk.
|
||||||
auto transform_algorithm = GC::create_function(realm.heap(), [stream](JS::Value chunk) -> GC::Ref<WebIDL::Promise> {
|
auto transform_algorithm = GC::create_function(realm.heap(), [stream](JS::Value chunk) -> GC::Ref<WebIDL::Promise> {
|
||||||
auto& realm = stream->realm();
|
auto& realm = stream->realm();
|
||||||
|
HTML::TemporaryExecutionContext execution_context { realm };
|
||||||
if (auto result = stream->decode_and_enqueue_chunk(chunk); result.is_error())
|
if (auto result = stream->decode_and_enqueue_chunk(chunk); result.is_error())
|
||||||
return WebIDL::create_rejected_promise_from_exception(realm, result.release_error());
|
return WebIDL::create_rejected_promise_from_exception(realm, result.release_error());
|
||||||
return WebIDL::create_resolved_promise(realm, JS::js_undefined());
|
return WebIDL::create_resolved_promise(realm, JS::js_undefined());
|
||||||
|
|
@ -67,6 +69,7 @@ WebIDL::ExceptionOr<GC::Ref<TextDecoderStream>> TextDecoderStream::construct_imp
|
||||||
// 8. Let flushAlgorithm be an algorithm which takes no arguments and runs the flush and enqueue algorithm with this.
|
// 8. Let flushAlgorithm be an algorithm which takes no arguments and runs the flush and enqueue algorithm with this.
|
||||||
auto flush_algorithm = GC::create_function(realm.heap(), [stream]() -> GC::Ref<WebIDL::Promise> {
|
auto flush_algorithm = GC::create_function(realm.heap(), [stream]() -> GC::Ref<WebIDL::Promise> {
|
||||||
auto& realm = stream->realm();
|
auto& realm = stream->realm();
|
||||||
|
HTML::TemporaryExecutionContext execution_context { realm };
|
||||||
if (auto result = stream->flush_and_enqueue(); result.is_error())
|
if (auto result = stream->flush_and_enqueue(); result.is_error())
|
||||||
return WebIDL::create_rejected_promise_from_exception(realm, result.release_error());
|
return WebIDL::create_rejected_promise_from_exception(realm, result.release_error());
|
||||||
return WebIDL::create_resolved_promise(realm, JS::js_undefined());
|
return WebIDL::create_resolved_promise(realm, JS::js_undefined());
|
||||||
|
|
@ -81,11 +84,14 @@ WebIDL::ExceptionOr<GC::Ref<TextDecoderStream>> TextDecoderStream::construct_imp
|
||||||
return stream;
|
return stream;
|
||||||
}
|
}
|
||||||
|
|
||||||
TextDecoderStream::TextDecoderStream(JS::Realm& realm, GC::Ref<Streams::TransformStream> transform, TextCodec::Decoder& decoder, FlyString encoding, ErrorMode error_mode, bool ignore_bom)
|
TextDecoderStream::TextDecoderStream(JS::Realm& realm, GC::Ref<Streams::TransformStream> transform, TextCodec::Decoder& decoder, FlyString encoding, TextCodec::ErrorMode error_mode, bool ignore_bom)
|
||||||
: Bindings::PlatformObject(realm)
|
: Bindings::PlatformObject(realm)
|
||||||
, Streams::GenericTransformStreamMixin(transform)
|
, Streams::GenericTransformStreamMixin(transform)
|
||||||
, TextDecoderCommonMixin(decoder, move(encoding), error_mode, ignore_bom)
|
, TextDecoderCommonMixin(decoder, move(encoding), error_mode, ignore_bom)
|
||||||
, m_streaming_decoder(make<TextCodec::StreamingDecoder>(m_encoding))
|
, m_streaming_decoder(make<TextCodec::StreamingDecoder>(
|
||||||
|
m_encoding,
|
||||||
|
m_ignore_bom ? TextCodec::IgnoreBOM::Yes : TextCodec::IgnoreBOM::No,
|
||||||
|
m_error_mode))
|
||||||
{
|
{
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -119,7 +125,10 @@ WebIDL::ExceptionOr<void> TextDecoderStream::decode_and_enqueue_chunk(JS::Value
|
||||||
return WebIDL::OperationError::create(realm, "Failed to copy bytes from BufferSource"_utf16);
|
return WebIDL::OperationError::create(realm, "Failed to copy bytes from BufferSource"_utf16);
|
||||||
auto buffer = buffer_or_error.release_value();
|
auto buffer = buffer_or_error.release_value();
|
||||||
|
|
||||||
auto decoded = TRY_OR_THROW_OOM(vm, m_streaming_decoder->to_utf8(buffer.bytes()));
|
auto decoded_or_error = m_streaming_decoder->to_utf8(buffer.bytes());
|
||||||
|
if (decoded_or_error.is_error() && decoded_or_error.error().code() != ENOMEM)
|
||||||
|
return WebIDL::SimpleException { WebIDL::SimpleExceptionType::TypeError, "Decoding failed"sv };
|
||||||
|
auto decoded = TRY_OR_THROW_OOM(vm, move(decoded_or_error));
|
||||||
|
|
||||||
// 3-4. Run "processing an item" until the input is exhausted, accumulating the output, then enqueue any non-empty
|
// 3-4. Run "processing an item" until the input is exhausted, accumulating the output, then enqueue any non-empty
|
||||||
// result. If processing returns error, throw a TypeError.
|
// result. If processing returns error, throw a TypeError.
|
||||||
|
|
@ -130,7 +139,10 @@ WebIDL::ExceptionOr<void> TextDecoderStream::decode_and_enqueue_chunk(JS::Value
|
||||||
WebIDL::ExceptionOr<void> TextDecoderStream::flush_and_enqueue()
|
WebIDL::ExceptionOr<void> TextDecoderStream::flush_and_enqueue()
|
||||||
{
|
{
|
||||||
// 1-3. Drain decoder's I/O queue and run "processing an item" to completion.
|
// 1-3. Drain decoder's I/O queue and run "processing an item" to completion.
|
||||||
auto decoded = TRY_OR_THROW_OOM(vm(), m_streaming_decoder->finish());
|
auto decoded_or_error = m_streaming_decoder->finish();
|
||||||
|
if (decoded_or_error.is_error() && decoded_or_error.error().code() != ENOMEM)
|
||||||
|
return WebIDL::SimpleException { WebIDL::SimpleExceptionType::TypeError, "Decoding failed"sv };
|
||||||
|
auto decoded = TRY_OR_THROW_OOM(vm(), move(decoded_or_error));
|
||||||
return enqueue_decoded_output(decoded);
|
return enqueue_decoded_output(decoded);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -139,22 +151,9 @@ WebIDL::ExceptionOr<void> TextDecoderStream::enqueue_decoded_output(String const
|
||||||
auto& realm = this->realm();
|
auto& realm = this->realm();
|
||||||
auto& vm = realm.vm();
|
auto& vm = realm.vm();
|
||||||
|
|
||||||
// https://encoding.spec.whatwg.org/#concept-td-serialize
|
|
||||||
// FIXME: The underlying TextCodec decoders currently strip leading BOMs unconditionally for UTF-8 and UTF-16BE/LE,
|
|
||||||
// so the "ignore BOM" flag is effectively ignored here. Once the decoders accept a "preserve BOM" mode,
|
|
||||||
// plumb m_ignore_bom through and strip the BOM from `decoded` only when m_ignore_bom is false.
|
|
||||||
if (!m_bom_seen && !decoded.is_empty())
|
|
||||||
m_bom_seen = true;
|
|
||||||
|
|
||||||
if (decoded.is_empty())
|
if (decoded.is_empty())
|
||||||
return {};
|
return {};
|
||||||
|
|
||||||
// If decoder's error mode is "fatal" and processing produced any error, throw a TypeError.
|
|
||||||
// NB: We can only detect this approximately by looking for U+FFFD in the decoded output, which the underlying
|
|
||||||
// decoder substitutes for invalid sequences. This matches the existing TextDecoder.decode() behavior.
|
|
||||||
if (fatal() && decoded.contains(0xFFFD))
|
|
||||||
return WebIDL::SimpleException { WebIDL::SimpleExceptionType::TypeError, "Decoding failed"sv };
|
|
||||||
|
|
||||||
auto js_string = JS::PrimitiveString::create(vm, Utf16String::from_utf8(decoded));
|
auto js_string = JS::PrimitiveString::create(vm, Utf16String::from_utf8(decoded));
|
||||||
return Streams::transform_stream_default_controller_enqueue(*m_transform->controller(), js_string);
|
return Streams::transform_stream_default_controller_enqueue(*m_transform->controller(), js_string);
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -27,7 +27,7 @@ public:
|
||||||
virtual ~TextDecoderStream() override;
|
virtual ~TextDecoderStream() override;
|
||||||
|
|
||||||
private:
|
private:
|
||||||
TextDecoderStream(JS::Realm&, GC::Ref<Streams::TransformStream>, TextCodec::Decoder&, FlyString encoding, ErrorMode, bool ignore_bom);
|
TextDecoderStream(JS::Realm&, GC::Ref<Streams::TransformStream>, TextCodec::Decoder&, FlyString encoding, TextCodec::ErrorMode, bool ignore_bom);
|
||||||
|
|
||||||
virtual void initialize(JS::Realm&) override;
|
virtual void initialize(JS::Realm&) override;
|
||||||
virtual void visit_edges(Cell::Visitor&) override;
|
virtual void visit_edges(Cell::Visitor&) override;
|
||||||
|
|
|
||||||
|
|
@ -14,7 +14,7 @@ namespace Web::Encoding {
|
||||||
|
|
||||||
GC_DEFINE_ALLOCATOR(TextEncoder);
|
GC_DEFINE_ALLOCATOR(TextEncoder);
|
||||||
|
|
||||||
WebIDL::ExceptionOr<GC::Ref<TextEncoder>> TextEncoder::construct_impl(JS::Realm& realm)
|
GC::Ref<TextEncoder> TextEncoder::construct_impl(JS::Realm& realm)
|
||||||
{
|
{
|
||||||
return realm.create<TextEncoder>(realm);
|
return realm.create<TextEncoder>(realm);
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -27,7 +27,7 @@ class TextEncoder final
|
||||||
GC_DECLARE_ALLOCATOR(TextEncoder);
|
GC_DECLARE_ALLOCATOR(TextEncoder);
|
||||||
|
|
||||||
public:
|
public:
|
||||||
static WebIDL::ExceptionOr<GC::Ref<TextEncoder>> construct_impl(JS::Realm&);
|
static GC::Ref<TextEncoder> construct_impl(JS::Realm&);
|
||||||
|
|
||||||
virtual ~TextEncoder() override;
|
virtual ~TextEncoder() override;
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -12,6 +12,7 @@
|
||||||
#include <LibWeb/Bindings/Intrinsics.h>
|
#include <LibWeb/Bindings/Intrinsics.h>
|
||||||
#include <LibWeb/Bindings/TextEncoderStream.h>
|
#include <LibWeb/Bindings/TextEncoderStream.h>
|
||||||
#include <LibWeb/Encoding/TextEncoderStream.h>
|
#include <LibWeb/Encoding/TextEncoderStream.h>
|
||||||
|
#include <LibWeb/HTML/Scripting/TemporaryExecutionContext.h>
|
||||||
#include <LibWeb/Streams/TransformStream.h>
|
#include <LibWeb/Streams/TransformStream.h>
|
||||||
#include <LibWeb/Streams/TransformStreamOperations.h>
|
#include <LibWeb/Streams/TransformStreamOperations.h>
|
||||||
#include <LibWeb/WebIDL/Promise.h>
|
#include <LibWeb/WebIDL/Promise.h>
|
||||||
|
|
@ -21,7 +22,7 @@ namespace Web::Encoding {
|
||||||
GC_DEFINE_ALLOCATOR(TextEncoderStream);
|
GC_DEFINE_ALLOCATOR(TextEncoderStream);
|
||||||
|
|
||||||
// https://encoding.spec.whatwg.org/#dom-textencoderstream
|
// https://encoding.spec.whatwg.org/#dom-textencoderstream
|
||||||
WebIDL::ExceptionOr<GC::Ref<TextEncoderStream>> TextEncoderStream::construct_impl(JS::Realm& realm)
|
GC::Ref<TextEncoderStream> TextEncoderStream::construct_impl(JS::Realm& realm)
|
||||||
{
|
{
|
||||||
// 1. Set this’s encoder to an instance of the UTF-8 encoder.
|
// 1. Set this’s encoder to an instance of the UTF-8 encoder.
|
||||||
// NOTE: No-op, as AK::String is already in UTF-8 format.
|
// NOTE: No-op, as AK::String is already in UTF-8 format.
|
||||||
|
|
@ -38,6 +39,7 @@ WebIDL::ExceptionOr<GC::Ref<TextEncoderStream>> TextEncoderStream::construct_imp
|
||||||
auto transform_algorithm = GC::create_function(realm.heap(), [stream](JS::Value chunk) -> GC::Ref<WebIDL::Promise> {
|
auto transform_algorithm = GC::create_function(realm.heap(), [stream](JS::Value chunk) -> GC::Ref<WebIDL::Promise> {
|
||||||
auto& realm = stream->realm();
|
auto& realm = stream->realm();
|
||||||
auto& vm = realm.vm();
|
auto& vm = realm.vm();
|
||||||
|
HTML::TemporaryExecutionContext execution_context { realm };
|
||||||
|
|
||||||
if (auto result = stream->encode_and_enqueue_chunk(chunk); result.is_error()) {
|
if (auto result = stream->encode_and_enqueue_chunk(chunk); result.is_error()) {
|
||||||
auto throw_completion = Bindings::exception_to_throw_completion(vm, result.exception());
|
auto throw_completion = Bindings::exception_to_throw_completion(vm, result.exception());
|
||||||
|
|
@ -51,6 +53,7 @@ WebIDL::ExceptionOr<GC::Ref<TextEncoderStream>> TextEncoderStream::construct_imp
|
||||||
auto flush_algorithm = GC::create_function(realm.heap(), [stream]() -> GC::Ref<WebIDL::Promise> {
|
auto flush_algorithm = GC::create_function(realm.heap(), [stream]() -> GC::Ref<WebIDL::Promise> {
|
||||||
auto& realm = stream->realm();
|
auto& realm = stream->realm();
|
||||||
auto& vm = realm.vm();
|
auto& vm = realm.vm();
|
||||||
|
HTML::TemporaryExecutionContext execution_context { realm };
|
||||||
|
|
||||||
if (auto result = stream->encode_and_flush(); result.is_error()) {
|
if (auto result = stream->encode_and_flush(); result.is_error()) {
|
||||||
auto throw_completion = Bindings::exception_to_throw_completion(vm, result.exception());
|
auto throw_completion = Bindings::exception_to_throw_completion(vm, result.exception());
|
||||||
|
|
|
||||||
|
|
@ -20,7 +20,7 @@ class TextEncoderStream final
|
||||||
GC_DECLARE_ALLOCATOR(TextEncoderStream);
|
GC_DECLARE_ALLOCATOR(TextEncoderStream);
|
||||||
|
|
||||||
public:
|
public:
|
||||||
static WebIDL::ExceptionOr<GC::Ref<TextEncoderStream>> construct_impl(JS::Realm&);
|
static GC::Ref<TextEncoderStream> construct_impl(JS::Realm&);
|
||||||
virtual ~TextEncoderStream() override;
|
virtual ~TextEncoderStream() override;
|
||||||
|
|
||||||
private:
|
private:
|
||||||
|
|
|
||||||
|
|
@ -108,7 +108,7 @@ HTMLTokenizer::HTMLTokenizer(StringView input, ByteString const& encoding, Input
|
||||||
if (input_type == InputType::EncodedBytes) {
|
if (input_type == InputType::EncodedBytes) {
|
||||||
auto decoder = TextCodec::decoder_for(encoding);
|
auto decoder = TextCodec::decoder_for(encoding);
|
||||||
VERIFY(decoder.has_value());
|
VERIFY(decoder.has_value());
|
||||||
m_source = MUST(decoder->to_utf8(input));
|
m_source = MUST(decoder->to_utf8(input, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement));
|
||||||
} else {
|
} else {
|
||||||
m_source = decoded_string_for_utf8_tokenizer(input);
|
m_source = decoded_string_for_utf8_tokenizer(input);
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -68,7 +68,7 @@ void IncrementalDocumentParser::initialize_parser(ReadonlyBytes sniff_bytes)
|
||||||
|
|
||||||
auto standardized_encoding = TextCodec::get_standardized_encoding(encoding);
|
auto standardized_encoding = TextCodec::get_standardized_encoding(encoding);
|
||||||
VERIFY(standardized_encoding.has_value());
|
VERIFY(standardized_encoding.has_value());
|
||||||
m_decoder = make<TextCodec::StreamingDecoder>(standardized_encoding.value());
|
m_decoder = make<TextCodec::StreamingDecoder>(standardized_encoding.value(), TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement);
|
||||||
|
|
||||||
// https://html.spec.whatwg.org/multipage/parsing.html#determining-the-character-encoding
|
// https://html.spec.whatwg.org/multipage/parsing.html#determining-the-character-encoding
|
||||||
// The document's character encoding must immediately be set to the value returned from this
|
// The document's character encoding must immediately be set to the value returned from this
|
||||||
|
|
|
||||||
|
|
@ -54,7 +54,7 @@ void Instance::initialize(JS::Realm& realm)
|
||||||
[&](Wasm::FunctionAddress const& address) {
|
[&](Wasm::FunctionAddress const& address) {
|
||||||
Optional<GC::Ptr<JS::FunctionObject>> object = m_function_instances.get(address);
|
Optional<GC::Ptr<JS::FunctionObject>> object = m_function_instances.get(address);
|
||||||
if (!object.has_value()) {
|
if (!object.has_value()) {
|
||||||
object = Detail::create_native_function(vm, address, name, this);
|
object = Detail::create_native_function(vm, address, this);
|
||||||
m_function_instances.set(address, *object);
|
m_function_instances.set(address, *object);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -289,6 +289,7 @@ JS::ThrowCompletionOr<NonnullRefPtr<Wasm::ModuleInstance>> instantiate_module(JS
|
||||||
|
|
||||||
// 2. Let imports be « ».
|
// 2. Let imports be « ».
|
||||||
HashMap<Wasm::Linker::Name, Wasm::ExternValue> resolved_imports;
|
HashMap<Wasm::Linker::Name, Wasm::ExternValue> resolved_imports;
|
||||||
|
size_t imported_function_count = 0;
|
||||||
if (import_object) {
|
if (import_object) {
|
||||||
dbgln_if(LIBWEB_WASM_DEBUG, "Trying to resolve stuff because import object was specified");
|
dbgln_if(LIBWEB_WASM_DEBUG, "Trying to resolve stuff because import object was specified");
|
||||||
|
|
||||||
|
|
@ -330,7 +331,7 @@ JS::ThrowCompletionOr<NonnullRefPtr<Wasm::ModuleInstance>> instantiate_module(JS
|
||||||
else {
|
else {
|
||||||
// 3.4.3.1. Create a host function from v and functype, and let funcaddr be the result.
|
// 3.4.3.1. Create a host function from v and functype, and let funcaddr be the result.
|
||||||
cache.add_imported_object(function);
|
cache.add_imported_object(function);
|
||||||
auto host_function = create_host_function(vm, function, function_type, ByteString::formatted("func{}", resolved_imports.size()));
|
auto host_function = create_host_function(vm, function, function_type, ByteString::number(imported_function_count));
|
||||||
address = cache.abstract_machine().store().allocate(move(host_function));
|
address = cache.abstract_machine().store().allocate(move(host_function));
|
||||||
// FIXME: 3.4.3.2. Let index be the number of external functions in imports. This value index is known as the index of the host function funcaddr.
|
// FIXME: 3.4.3.2. Let index be the number of external functions in imports. This value index is known as the index of the host function funcaddr.
|
||||||
// 'index' doesn't seem to be used anywhere?
|
// 'index' doesn't seem to be used anywhere?
|
||||||
|
|
@ -342,6 +343,7 @@ JS::ThrowCompletionOr<NonnullRefPtr<Wasm::ModuleInstance>> instantiate_module(JS
|
||||||
// 3.4.4. Let externfunc be the external value func funcaddr.
|
// 3.4.4. Let externfunc be the external value func funcaddr.
|
||||||
// 3.4.5. Append externfunc to imports.
|
// 3.4.5. Append externfunc to imports.
|
||||||
resolved_imports.set(import_name, Wasm::ExternValue { Wasm::FunctionAddress { *address } });
|
resolved_imports.set(import_name, Wasm::ExternValue { Wasm::FunctionAddress { *address } });
|
||||||
|
++imported_function_count;
|
||||||
return {};
|
return {};
|
||||||
},
|
},
|
||||||
// 3.5. If externtype is of the form global mut valtype,
|
// 3.5. If externtype is of the form global mut valtype,
|
||||||
|
|
@ -647,14 +649,20 @@ JS::ThrowCompletionOr<JS::HandledByHost> host_grow_shared_array_buffer(JS::VM& v
|
||||||
|
|
||||||
GC_DEFINE_ALLOCATOR(ExportedWasmFunction);
|
GC_DEFINE_ALLOCATOR(ExportedWasmFunction);
|
||||||
|
|
||||||
GC::Ref<ExportedWasmFunction> ExportedWasmFunction::create(JS::Realm& realm, Utf16FlyString name, Function<JS::ThrowCompletionOr<JS::Value>(JS::VM&)> behavior, Wasm::FunctionAddress exported_address)
|
GC::Ref<ExportedWasmFunction> ExportedWasmFunction::create(JS::Realm& realm, Utf16FlyString name, size_t length, Function<JS::ThrowCompletionOr<JS::Value>(JS::VM&)> behavior, Wasm::FunctionAddress exported_address)
|
||||||
{
|
{
|
||||||
|
auto& vm = realm.vm();
|
||||||
|
|
||||||
auto prototype = realm.intrinsics().function_prototype();
|
auto prototype = realm.intrinsics().function_prototype();
|
||||||
return realm.create<ExportedWasmFunction>(
|
auto function = realm.create<ExportedWasmFunction>(
|
||||||
move(name),
|
move(name),
|
||||||
move(behavior),
|
move(behavior),
|
||||||
exported_address,
|
exported_address,
|
||||||
prototype);
|
prototype);
|
||||||
|
function->define_direct_property(vm.names.length, JS::Value { static_cast<double>(length) }, JS::Attribute::Configurable);
|
||||||
|
function->define_direct_property(vm.names.name, JS::PrimitiveString::create(vm, function->name()), JS::Attribute::Configurable);
|
||||||
|
|
||||||
|
return function;
|
||||||
}
|
}
|
||||||
|
|
||||||
ExportedWasmFunction::ExportedWasmFunction(Utf16FlyString name, AK::Function<JS::ThrowCompletionOr<JS::Value>(JS::VM&)> behavior, Wasm::FunctionAddress exported_address, JS::Object& prototype)
|
ExportedWasmFunction::ExportedWasmFunction(Utf16FlyString name, AK::Function<JS::ThrowCompletionOr<JS::Value>(JS::VM&)> behavior, Wasm::FunctionAddress exported_address, JS::Object& prototype)
|
||||||
|
|
@ -676,19 +684,57 @@ JS::ThrowCompletionOr<JS::Value> ExportedWasmFunction::call()
|
||||||
return m_behavior(vm());
|
return m_behavior(vm());
|
||||||
}
|
}
|
||||||
|
|
||||||
JS::NativeFunction* create_native_function(JS::VM& vm, Wasm::FunctionAddress address, Utf16FlyString name, Instance* instance)
|
// https://www.w3.org/TR/wasm-js-api-2/#name-of-the-webassembly-function
|
||||||
|
static Utf16FlyString name_of_webassembly_function(Wasm::Store& store, Wasm::FunctionAddress function_address)
|
||||||
|
{
|
||||||
|
// 1. Let store be the surrounding agent’s associated store.
|
||||||
|
|
||||||
|
// 2. Let funcinst be store.funcs[funcaddr].
|
||||||
|
auto* function_instance = store.get(function_address);
|
||||||
|
VERIFY(function_instance);
|
||||||
|
|
||||||
|
auto index = function_instance->visit(
|
||||||
|
// 3. If funcinst is of the form {type functype, hostcode hostfunc},
|
||||||
|
[&](Wasm::HostFunction const& host_function) {
|
||||||
|
// 1. Assert: hostfunc is a JavaScript object and IsCallable(hostfunc) is true.
|
||||||
|
// 2. Let index be the index of the host function funcaddr.
|
||||||
|
auto index = host_function.name().to_number<size_t>(TrimWhitespace::No);
|
||||||
|
VERIFY(index.has_value());
|
||||||
|
return *index;
|
||||||
|
},
|
||||||
|
// 4. Otherwise,
|
||||||
|
[&](Wasm::WasmFunction const& wasm_function) {
|
||||||
|
// 1. Let moduleinst be funcinst.module.
|
||||||
|
auto const& module_instance = wasm_function.module();
|
||||||
|
|
||||||
|
// 2. Assert: funcaddr is contained in moduleinst.funcaddrs.
|
||||||
|
// 3. Let index be the index of moduleinst.funcaddrs where funcaddr is found.
|
||||||
|
auto index = module_instance.functions().find_first_index(function_address);
|
||||||
|
VERIFY(index.has_value());
|
||||||
|
return *index;
|
||||||
|
});
|
||||||
|
|
||||||
|
// 5. Return ! ToString(index).
|
||||||
|
return Utf16String::number(index);
|
||||||
|
}
|
||||||
|
|
||||||
|
JS::NativeFunction* create_native_function(JS::VM& vm, Wasm::FunctionAddress address, Instance* instance)
|
||||||
{
|
{
|
||||||
auto& realm = *vm.current_realm();
|
auto& realm = *vm.current_realm();
|
||||||
Optional<Wasm::FunctionType> type;
|
|
||||||
auto& cache = get_cache(realm);
|
auto& cache = get_cache(realm);
|
||||||
cache.abstract_machine().store().get(address)->visit([&](auto const& value) { type = value.type(); });
|
|
||||||
if (auto entry = cache.get_function_instance(address); entry.has_value())
|
if (auto entry = cache.get_function_instance(address); entry.has_value())
|
||||||
return *entry;
|
return *entry;
|
||||||
|
|
||||||
|
auto& store = cache.abstract_machine().store();
|
||||||
|
auto type = store.get(address)->visit([&](auto const& value) { return value.type(); });
|
||||||
|
auto length = type.parameters().size();
|
||||||
|
|
||||||
auto function = ExportedWasmFunction::create(
|
auto function = ExportedWasmFunction::create(
|
||||||
realm,
|
realm,
|
||||||
move(name),
|
name_of_webassembly_function(store, address),
|
||||||
[address, type = type.release_value(), instance](JS::VM& vm) -> JS::ThrowCompletionOr<JS::Value> {
|
length,
|
||||||
|
[address, type = move(type), instance](JS::VM& vm) -> JS::ThrowCompletionOr<JS::Value> {
|
||||||
(void)instance;
|
(void)instance;
|
||||||
auto& realm = *vm.current_realm();
|
auto& realm = *vm.current_realm();
|
||||||
Vector<Wasm::Value> values;
|
Vector<Wasm::Value> values;
|
||||||
|
|
@ -861,17 +907,7 @@ JS::Value to_js_value(JS::VM& vm, Wasm::Value& wasm_value, Wasm::ValueType type)
|
||||||
if (ref_.ref().has<Wasm::Reference::Null>())
|
if (ref_.ref().has<Wasm::Reference::Null>())
|
||||||
return JS::js_null();
|
return JS::js_null();
|
||||||
auto address = ref_.ref().get<Wasm::Reference::Func>().address;
|
auto address = ref_.ref().get<Wasm::Reference::Func>().address;
|
||||||
auto& cache = get_cache(realm);
|
return create_native_function(vm, address);
|
||||||
auto* function = cache.abstract_machine().store().get(address);
|
|
||||||
auto name = function->visit(
|
|
||||||
[&](Wasm::WasmFunction& wasm_function) {
|
|
||||||
auto index = *wasm_function.module().functions().find_first_index(address);
|
|
||||||
return ByteString::formatted("func{}", index);
|
|
||||||
},
|
|
||||||
[](Wasm::HostFunction& host_function) {
|
|
||||||
return host_function.name();
|
|
||||||
});
|
|
||||||
return create_native_function(vm, address, Utf16FlyString::from_utf8(name));
|
|
||||||
}
|
}
|
||||||
case Wasm::ValueType::ExternReference: {
|
case Wasm::ValueType::ExternReference: {
|
||||||
auto ref_ = wasm_value.to<Wasm::Reference>();
|
auto ref_ = wasm_value.to<Wasm::Reference>();
|
||||||
|
|
|
||||||
|
|
@ -101,7 +101,7 @@ class ExportedWasmFunction final : public JS::NativeFunction {
|
||||||
GC_DECLARE_ALLOCATOR(ExportedWasmFunction);
|
GC_DECLARE_ALLOCATOR(ExportedWasmFunction);
|
||||||
|
|
||||||
public:
|
public:
|
||||||
static GC::Ref<ExportedWasmFunction> create(JS::Realm&, Utf16FlyString name, ESCAPING Function<JS::ThrowCompletionOr<JS::Value>(JS::VM&)>, Wasm::FunctionAddress);
|
static GC::Ref<ExportedWasmFunction> create(JS::Realm&, Utf16FlyString name, size_t length, ESCAPING Function<JS::ThrowCompletionOr<JS::Value>(JS::VM&)>, Wasm::FunctionAddress);
|
||||||
virtual ~ExportedWasmFunction() override = default;
|
virtual ~ExportedWasmFunction() override = default;
|
||||||
|
|
||||||
Wasm::FunctionAddress exported_address() const { return m_exported_address; }
|
Wasm::FunctionAddress exported_address() const { return m_exported_address; }
|
||||||
|
|
@ -122,7 +122,7 @@ WebAssemblyCache& get_cache(JS::Realm&);
|
||||||
|
|
||||||
JS::ThrowCompletionOr<NonnullRefPtr<Wasm::ModuleInstance>> instantiate_module(JS::VM&, Wasm::Module const&, GC::Ptr<JS::Object> import_object);
|
JS::ThrowCompletionOr<NonnullRefPtr<Wasm::ModuleInstance>> instantiate_module(JS::VM&, Wasm::Module const&, GC::Ptr<JS::Object> import_object);
|
||||||
JS::ThrowCompletionOr<NonnullRefPtr<CompiledWebAssemblyModule>> compile_a_webassembly_module(JS::VM&, ByteBuffer);
|
JS::ThrowCompletionOr<NonnullRefPtr<CompiledWebAssemblyModule>> compile_a_webassembly_module(JS::VM&, ByteBuffer);
|
||||||
JS::NativeFunction* create_native_function(JS::VM&, Wasm::FunctionAddress address, Utf16FlyString name, Instance* instance = nullptr);
|
JS::NativeFunction* create_native_function(JS::VM&, Wasm::FunctionAddress address, Instance* instance = nullptr);
|
||||||
JS::ThrowCompletionOr<Wasm::Value> to_webassembly_value(JS::VM&, JS::Value value, Wasm::ValueType const& type);
|
JS::ThrowCompletionOr<Wasm::Value> to_webassembly_value(JS::VM&, JS::Value value, Wasm::ValueType const& type);
|
||||||
Wasm::Value default_webassembly_value(JS::VM&, Wasm::ValueType type);
|
Wasm::Value default_webassembly_value(JS::VM&, Wasm::ValueType type);
|
||||||
JS::Value to_js_value(JS::VM&, Wasm::Value& wasm_value, Wasm::ValueType type);
|
JS::Value to_js_value(JS::VM&, Wasm::Value& wasm_value, Wasm::ValueType type);
|
||||||
|
|
|
||||||
|
|
@ -92,7 +92,10 @@ static xmlNodePtr mirror_node(xmlDocPtr doc, DOM::Node const& node)
|
||||||
}
|
}
|
||||||
case DOM::NodeType::DOCUMENT_NODE: {
|
case DOM::NodeType::DOCUMENT_NODE: {
|
||||||
auto const& document = static_cast<DOM::Document const&>(node);
|
auto const& document = static_cast<DOM::Document const&>(node);
|
||||||
return mirror_node(doc, *document.document_element());
|
auto const* document_element = document.document_element();
|
||||||
|
if (!document_element)
|
||||||
|
return nullptr;
|
||||||
|
return mirror_node(doc, *document_element);
|
||||||
}
|
}
|
||||||
case DOM::NodeType::DOCUMENT_TYPE_NODE: {
|
case DOM::NodeType::DOCUMENT_TYPE_NODE: {
|
||||||
return nullptr; // Unused in libxml2
|
return nullptr; // Unused in libxml2
|
||||||
|
|
|
||||||
|
|
@ -438,7 +438,7 @@ ErrorOr<Vector<String>> Autocomplete::received_autocomplete_respsonse(Autocomple
|
||||||
if (!decoder.has_value())
|
if (!decoder.has_value())
|
||||||
decoder = TextCodec::decoder_for_exact_name("UTF-8"sv);
|
decoder = TextCodec::decoder_for_exact_name("UTF-8"sv);
|
||||||
|
|
||||||
auto decoded_response = TRY(decoder->to_utf8(response));
|
auto decoded_response = TRY(decoder->to_utf8(response, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement));
|
||||||
auto json = TRY(JsonValue::from_string(decoded_response));
|
auto json = TRY(JsonValue::from_string(decoded_response));
|
||||||
|
|
||||||
if (engine.name == "DuckDuckGo")
|
if (engine.name == "DuckDuckGo")
|
||||||
|
|
|
||||||
|
|
@ -24,6 +24,6 @@ extern "C" int LLVMFuzzerTestOneInput(uint8_t const* data, size_t size)
|
||||||
if (!decoder.has_value())
|
if (!decoder.has_value())
|
||||||
return 0;
|
return 0;
|
||||||
|
|
||||||
(void)decoder->to_utf8(encoded_data);
|
(void)decoder->to_utf8(encoded_data, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement);
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1085,7 +1085,7 @@ size_t Request::on_header_received(void* buffer, size_t size, size_t nmemb, void
|
||||||
auto decoder = TextCodec::decoder_for_exact_name("ISO-8859-1"sv);
|
auto decoder = TextCodec::decoder_for_exact_name("ISO-8859-1"sv);
|
||||||
VERIFY(decoder.has_value());
|
VERIFY(decoder.has_value());
|
||||||
|
|
||||||
request.m_reason_phrase = MUST(decoder->to_utf8(reason_phrase));
|
request.m_reason_phrase = MUST(decoder->to_utf8(reason_phrase, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement));
|
||||||
return total_size;
|
return total_size;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -37,13 +37,13 @@ TEST_CASE(test_utf8_decode)
|
||||||
// Bytes for U+1F600 GRINNING FACE
|
// Bytes for U+1F600 GRINNING FACE
|
||||||
auto test_string = "\xf0\x9f\x98\x80"sv;
|
auto test_string = "\xf0\x9f\x98\x80"sv;
|
||||||
|
|
||||||
EXPECT(decoder.validate(test_string));
|
EXPECT(!decoder.to_utf8(test_string, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Fatal).is_error());
|
||||||
|
|
||||||
auto processed_code_points = process_code_points(decoder, test_string);
|
auto processed_code_points = process_code_points(decoder, test_string);
|
||||||
EXPECT(processed_code_points.size() == 1);
|
EXPECT(processed_code_points.size() == 1);
|
||||||
EXPECT(processed_code_points[0] == 0x1F600);
|
EXPECT(processed_code_points[0] == 0x1F600);
|
||||||
|
|
||||||
EXPECT(MUST(decoder.to_utf8(test_string)) == test_string);
|
EXPECT(MUST(decoder.to_utf8(test_string, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)) == test_string);
|
||||||
}
|
}
|
||||||
|
|
||||||
TEST_CASE(test_utf8_process_code_points)
|
TEST_CASE(test_utf8_process_code_points)
|
||||||
|
|
@ -60,8 +60,8 @@ TEST_CASE(test_utf8_process_code_points_replaces_surrogates)
|
||||||
auto utf8_encoded_surrogate_bytes = Vector<u8> { 0xed, 0xa0, 0x80 };
|
auto utf8_encoded_surrogate_bytes = Vector<u8> { 0xed, 0xa0, 0x80 };
|
||||||
auto utf8_encoded_surrogate = StringView(bytes(utf8_encoded_surrogate_bytes));
|
auto utf8_encoded_surrogate = StringView(bytes(utf8_encoded_surrogate_bytes));
|
||||||
|
|
||||||
EXPECT(!decoder.validate(utf8_encoded_surrogate));
|
EXPECT(decoder.to_utf8(utf8_encoded_surrogate, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Fatal).is_error());
|
||||||
EXPECT_EQ(MUST(decoder.to_utf8(utf8_encoded_surrogate)), "\xef\xbf\xbd\xef\xbf\xbd\xef\xbf\xbd"sv);
|
EXPECT_EQ(MUST(decoder.to_utf8(utf8_encoded_surrogate, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)), "\xef\xbf\xbd\xef\xbf\xbd\xef\xbf\xbd"sv);
|
||||||
EXPECT_EQ(process_code_points(decoder, utf8_encoded_surrogate), (Vector<u32> { 0xfffd, 0xfffd, 0xfffd }));
|
EXPECT_EQ(process_code_points(decoder, utf8_encoded_surrogate), (Vector<u32> { 0xfffd, 0xfffd, 0xfffd }));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -71,8 +71,8 @@ TEST_CASE(test_utf8_process_code_points_replaces_truncated_tail_as_single_error)
|
||||||
auto truncated_tail_bytes = Vector<u8> { 0xf0, 0x9f, 0x98 };
|
auto truncated_tail_bytes = Vector<u8> { 0xf0, 0x9f, 0x98 };
|
||||||
auto truncated_tail = StringView(bytes(truncated_tail_bytes));
|
auto truncated_tail = StringView(bytes(truncated_tail_bytes));
|
||||||
|
|
||||||
EXPECT(!decoder.validate(truncated_tail));
|
EXPECT(decoder.to_utf8(truncated_tail, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Fatal).is_error());
|
||||||
EXPECT_EQ(MUST(decoder.to_utf8(truncated_tail)), "\xef\xbf\xbd"sv);
|
EXPECT_EQ(MUST(decoder.to_utf8(truncated_tail, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)), "\xef\xbf\xbd"sv);
|
||||||
EXPECT_EQ(process_code_points(decoder, truncated_tail), (Vector<u32> { 0xfffd }));
|
EXPECT_EQ(process_code_points(decoder, truncated_tail), (Vector<u32> { 0xfffd }));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -82,8 +82,8 @@ TEST_CASE(test_utf8_process_code_points_replaces_overlong_sequences)
|
||||||
auto overlong_null_bytes = Vector<u8> { 0xc0, 0x80 };
|
auto overlong_null_bytes = Vector<u8> { 0xc0, 0x80 };
|
||||||
auto overlong_null = StringView(bytes(overlong_null_bytes));
|
auto overlong_null = StringView(bytes(overlong_null_bytes));
|
||||||
|
|
||||||
EXPECT(!decoder.validate(overlong_null));
|
EXPECT(decoder.to_utf8(overlong_null, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Fatal).is_error());
|
||||||
EXPECT_EQ(MUST(decoder.to_utf8(overlong_null)), "\xef\xbf\xbd\xef\xbf\xbd"sv);
|
EXPECT_EQ(MUST(decoder.to_utf8(overlong_null, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)), "\xef\xbf\xbd\xef\xbf\xbd"sv);
|
||||||
EXPECT_EQ(process_code_points(decoder, overlong_null), (Vector<u32> { 0xfffd, 0xfffd }));
|
EXPECT_EQ(process_code_points(decoder, overlong_null), (Vector<u32> { 0xfffd, 0xfffd }));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -95,12 +95,12 @@ TEST_CASE(test_utf8_process_code_points_restores_invalid_second_byte)
|
||||||
auto out_of_range_four_byte_sequence_bytes = Vector<u8> { 0xf4, 0x90, 0x80, 0x80 };
|
auto out_of_range_four_byte_sequence_bytes = Vector<u8> { 0xf4, 0x90, 0x80, 0x80 };
|
||||||
auto out_of_range_four_byte_sequence = StringView(bytes(out_of_range_four_byte_sequence_bytes));
|
auto out_of_range_four_byte_sequence = StringView(bytes(out_of_range_four_byte_sequence_bytes));
|
||||||
|
|
||||||
EXPECT(!decoder.validate(overlong_three_byte_sequence));
|
EXPECT(decoder.to_utf8(overlong_three_byte_sequence, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Fatal).is_error());
|
||||||
EXPECT_EQ(MUST(decoder.to_utf8(overlong_three_byte_sequence)), "\xef\xbf\xbd\xef\xbf\xbd\xef\xbf\xbd"sv);
|
EXPECT_EQ(MUST(decoder.to_utf8(overlong_three_byte_sequence, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)), "\xef\xbf\xbd\xef\xbf\xbd\xef\xbf\xbd"sv);
|
||||||
EXPECT_EQ(process_code_points(decoder, overlong_three_byte_sequence), (Vector<u32> { 0xfffd, 0xfffd, 0xfffd }));
|
EXPECT_EQ(process_code_points(decoder, overlong_three_byte_sequence), (Vector<u32> { 0xfffd, 0xfffd, 0xfffd }));
|
||||||
|
|
||||||
EXPECT(!decoder.validate(out_of_range_four_byte_sequence));
|
EXPECT(decoder.to_utf8(out_of_range_four_byte_sequence, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Fatal).is_error());
|
||||||
EXPECT_EQ(MUST(decoder.to_utf8(out_of_range_four_byte_sequence)), "\xef\xbf\xbd\xef\xbf\xbd\xef\xbf\xbd\xef\xbf\xbd"sv);
|
EXPECT_EQ(MUST(decoder.to_utf8(out_of_range_four_byte_sequence, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)), "\xef\xbf\xbd\xef\xbf\xbd\xef\xbf\xbd\xef\xbf\xbd"sv);
|
||||||
EXPECT_EQ(process_code_points(decoder, out_of_range_four_byte_sequence), (Vector<u32> { 0xfffd, 0xfffd, 0xfffd, 0xfffd }));
|
EXPECT_EQ(process_code_points(decoder, out_of_range_four_byte_sequence), (Vector<u32> { 0xfffd, 0xfffd, 0xfffd, 0xfffd }));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -110,8 +110,7 @@ TEST_CASE(test_utf16be_decode)
|
||||||
// This is the output of `python3 -c "print('säk😀'.encode('utf-16be'))"`.
|
// This is the output of `python3 -c "print('säk😀'.encode('utf-16be'))"`.
|
||||||
auto test_string = "\x00s\x00\xe4\x00k\xd8=\xde\x00"sv;
|
auto test_string = "\x00s\x00\xe4\x00k\xd8=\xde\x00"sv;
|
||||||
|
|
||||||
EXPECT(decoder.validate(test_string));
|
auto utf8 = MUST(decoder.to_utf8(test_string, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Fatal));
|
||||||
auto utf8 = MUST(decoder.to_utf8(test_string));
|
|
||||||
EXPECT_EQ(utf8, "säk😀"sv);
|
EXPECT_EQ(utf8, "säk😀"sv);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -122,12 +121,12 @@ TEST_CASE(test_utf16be_process_code_points)
|
||||||
EXPECT_EQ(process_code_points(decoder, StringView(bytes({ 0xfe, 0xff, 0x00, 'A', 0xd8, 0x3d, 0xde, 0x00 }))), (Vector<u32> { 0x41, 0x1F600 }));
|
EXPECT_EQ(process_code_points(decoder, StringView(bytes({ 0xfe, 0xff, 0x00, 'A', 0xd8, 0x3d, 0xde, 0x00 }))), (Vector<u32> { 0x41, 0x1F600 }));
|
||||||
EXPECT_EQ(process_code_points(decoder, StringView(bytes({ 0xd8, 0x3d, 0x00, 'A', 0xde, 0x00 }))), (Vector<u32> { 0xfffd, 0x41, 0xfffd }));
|
EXPECT_EQ(process_code_points(decoder, StringView(bytes({ 0xd8, 0x3d, 0x00, 'A', 0xde, 0x00 }))), (Vector<u32> { 0xfffd, 0x41, 0xfffd }));
|
||||||
EXPECT_EQ(process_code_points(decoder, StringView(bytes({ 0x00, 'A', 0xff }))), (Vector<u32> { 0x41, 0xfffd }));
|
EXPECT_EQ(process_code_points(decoder, StringView(bytes({ 0x00, 'A', 0xff }))), (Vector<u32> { 0x41, 0xfffd }));
|
||||||
EXPECT_EQ(MUST(decoder.to_utf8(StringView(bytes({ 0x00, 'A', 0xff })))), "A\xef\xbf\xbd"sv);
|
EXPECT_EQ(MUST(decoder.to_utf8(StringView(bytes({ 0x00, 'A', 0xff })), TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)), "A\xef\xbf\xbd"sv);
|
||||||
}
|
}
|
||||||
|
|
||||||
TEST_CASE(test_streaming_decoder_utf8_mid_sequence)
|
TEST_CASE(test_streaming_decoder_utf8_mid_sequence)
|
||||||
{
|
{
|
||||||
auto streaming_decoder = TextCodec::StreamingDecoder { "UTF-8"sv };
|
auto streaming_decoder = TextCodec::StreamingDecoder { "UTF-8"sv, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement };
|
||||||
|
|
||||||
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 'a', 0xc3 }))), "a"sv);
|
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 'a', 0xc3 }))), "a"sv);
|
||||||
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0xa9, 'b' }))), "éb"sv);
|
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0xa9, 'b' }))), "éb"sv);
|
||||||
|
|
@ -137,27 +136,27 @@ TEST_CASE(test_streaming_decoder_utf8_mid_sequence)
|
||||||
TEST_CASE(test_streaming_decoder_finishes_incomplete_sequence)
|
TEST_CASE(test_streaming_decoder_finishes_incomplete_sequence)
|
||||||
{
|
{
|
||||||
auto& decoder = decoder_for("UTF-8"sv);
|
auto& decoder = decoder_for("UTF-8"sv);
|
||||||
auto streaming_decoder = TextCodec::StreamingDecoder { "UTF-8"sv };
|
auto streaming_decoder = TextCodec::StreamingDecoder { "UTF-8"sv, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement };
|
||||||
|
|
||||||
auto incomplete_sequence = Vector<u8> { 0xc3 };
|
auto incomplete_sequence = Vector<u8> { 0xc3 };
|
||||||
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes(incomplete_sequence))), ""sv);
|
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes(incomplete_sequence))), ""sv);
|
||||||
EXPECT_EQ(MUST(streaming_decoder.finish()), MUST(decoder.to_utf8(StringView(bytes(incomplete_sequence)))));
|
EXPECT_EQ(MUST(streaming_decoder.finish()), MUST(decoder.to_utf8(StringView(bytes(incomplete_sequence)), TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)));
|
||||||
}
|
}
|
||||||
|
|
||||||
TEST_CASE(test_streaming_decoder_utf8_invalid_second_byte_tail)
|
TEST_CASE(test_streaming_decoder_utf8_invalid_second_byte_tail)
|
||||||
{
|
{
|
||||||
auto streaming_decoder_for_overlong_three_byte_sequence = TextCodec::StreamingDecoder { "UTF-8"sv };
|
auto streaming_decoder_for_overlong_three_byte_sequence = TextCodec::StreamingDecoder { "UTF-8"sv, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement };
|
||||||
EXPECT_EQ(MUST(streaming_decoder_for_overlong_three_byte_sequence.to_utf8(bytes({ 0xe0, 0x80 }))), "\xef\xbf\xbd\xef\xbf\xbd"sv);
|
EXPECT_EQ(MUST(streaming_decoder_for_overlong_three_byte_sequence.to_utf8(bytes({ 0xe0, 0x80 }))), "\xef\xbf\xbd\xef\xbf\xbd"sv);
|
||||||
EXPECT_EQ(MUST(streaming_decoder_for_overlong_three_byte_sequence.finish()), ""sv);
|
EXPECT_EQ(MUST(streaming_decoder_for_overlong_three_byte_sequence.finish()), ""sv);
|
||||||
|
|
||||||
auto streaming_decoder_for_out_of_range_four_byte_sequence = TextCodec::StreamingDecoder { "UTF-8"sv };
|
auto streaming_decoder_for_out_of_range_four_byte_sequence = TextCodec::StreamingDecoder { "UTF-8"sv, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement };
|
||||||
EXPECT_EQ(MUST(streaming_decoder_for_out_of_range_four_byte_sequence.to_utf8(bytes({ 0xf4, 0x90 }))), "\xef\xbf\xbd\xef\xbf\xbd"sv);
|
EXPECT_EQ(MUST(streaming_decoder_for_out_of_range_four_byte_sequence.to_utf8(bytes({ 0xf4, 0x90 }))), "\xef\xbf\xbd\xef\xbf\xbd"sv);
|
||||||
EXPECT_EQ(MUST(streaming_decoder_for_out_of_range_four_byte_sequence.finish()), ""sv);
|
EXPECT_EQ(MUST(streaming_decoder_for_out_of_range_four_byte_sequence.finish()), ""sv);
|
||||||
}
|
}
|
||||||
|
|
||||||
TEST_CASE(test_streaming_decoder_utf8_valid_second_byte_tail)
|
TEST_CASE(test_streaming_decoder_utf8_valid_second_byte_tail)
|
||||||
{
|
{
|
||||||
auto streaming_decoder = TextCodec::StreamingDecoder { "UTF-8"sv };
|
auto streaming_decoder = TextCodec::StreamingDecoder { "UTF-8"sv, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement };
|
||||||
|
|
||||||
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0xe0, 0xa0 }))), ""sv);
|
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0xe0, 0xa0 }))), ""sv);
|
||||||
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0x80 }))), "\xe0\xa0\x80"sv);
|
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0x80 }))), "\xe0\xa0\x80"sv);
|
||||||
|
|
@ -166,7 +165,7 @@ TEST_CASE(test_streaming_decoder_utf8_valid_second_byte_tail)
|
||||||
|
|
||||||
TEST_CASE(test_streaming_decoder_utf16_odd_byte)
|
TEST_CASE(test_streaming_decoder_utf16_odd_byte)
|
||||||
{
|
{
|
||||||
auto streaming_decoder = TextCodec::StreamingDecoder { "UTF-16LE"sv };
|
auto streaming_decoder = TextCodec::StreamingDecoder { "UTF-16LE"sv, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement };
|
||||||
|
|
||||||
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0x41, 0x00, 0x42 }))), "A"sv);
|
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0x41, 0x00, 0x42 }))), "A"sv);
|
||||||
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0x00 }))), "B"sv);
|
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0x00 }))), "B"sv);
|
||||||
|
|
@ -175,7 +174,7 @@ TEST_CASE(test_streaming_decoder_utf16_odd_byte)
|
||||||
|
|
||||||
TEST_CASE(test_streaming_decoder_utf16_surrogate_pair_split)
|
TEST_CASE(test_streaming_decoder_utf16_surrogate_pair_split)
|
||||||
{
|
{
|
||||||
auto streaming_decoder = TextCodec::StreamingDecoder { "UTF-16BE"sv };
|
auto streaming_decoder = TextCodec::StreamingDecoder { "UTF-16BE"sv, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement };
|
||||||
|
|
||||||
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0x00, 0x41, 0xd8, 0x3d }))), "A"sv);
|
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0x00, 0x41, 0xd8, 0x3d }))), "A"sv);
|
||||||
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0xde, 0x00 }))), "😀"sv);
|
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0xde, 0x00 }))), "😀"sv);
|
||||||
|
|
@ -185,48 +184,48 @@ TEST_CASE(test_streaming_decoder_utf16_surrogate_pair_split)
|
||||||
TEST_CASE(test_streaming_decoder_gb18030_four_byte_tail)
|
TEST_CASE(test_streaming_decoder_gb18030_four_byte_tail)
|
||||||
{
|
{
|
||||||
auto& decoder = decoder_for("gb18030"sv);
|
auto& decoder = decoder_for("gb18030"sv);
|
||||||
auto streaming_decoder = TextCodec::StreamingDecoder { "gb18030"sv };
|
auto streaming_decoder = TextCodec::StreamingDecoder { "gb18030"sv, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement };
|
||||||
auto gb18030_sequence = Vector<u8> { 0x81, 0x30, 0x81, 0x30 };
|
auto gb18030_sequence = Vector<u8> { 0x81, 0x30, 0x81, 0x30 };
|
||||||
|
|
||||||
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 'a', 0x81, 0x30, 0x81 }))), "a"sv);
|
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 'a', 0x81, 0x30, 0x81 }))), "a"sv);
|
||||||
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0x30 }))), MUST(decoder.to_utf8(StringView(bytes(gb18030_sequence)))));
|
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0x30 }))), MUST(decoder.to_utf8(StringView(bytes(gb18030_sequence)), TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)));
|
||||||
EXPECT_EQ(MUST(streaming_decoder.finish()), ""sv);
|
EXPECT_EQ(MUST(streaming_decoder.finish()), ""sv);
|
||||||
}
|
}
|
||||||
|
|
||||||
TEST_CASE(test_streaming_decoder_big5_overlapping_trail)
|
TEST_CASE(test_streaming_decoder_big5_overlapping_trail)
|
||||||
{
|
{
|
||||||
auto& decoder = decoder_for("Big5"sv);
|
auto& decoder = decoder_for("Big5"sv);
|
||||||
auto streaming_decoder = TextCodec::StreamingDecoder { "Big5"sv };
|
auto streaming_decoder = TextCodec::StreamingDecoder { "Big5"sv, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement };
|
||||||
auto big5_sequence = Vector<u8> { 0xa4, 0xa4 };
|
auto big5_sequence = Vector<u8> { 0xa4, 0xa4 };
|
||||||
|
|
||||||
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes(big5_sequence))), MUST(decoder.to_utf8(StringView(bytes(big5_sequence)))));
|
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes(big5_sequence))), MUST(decoder.to_utf8(StringView(bytes(big5_sequence)), TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)));
|
||||||
EXPECT_EQ(MUST(streaming_decoder.finish()), ""sv);
|
EXPECT_EQ(MUST(streaming_decoder.finish()), ""sv);
|
||||||
|
|
||||||
auto streaming_decoder_for_split_input = TextCodec::StreamingDecoder { "Big5"sv };
|
auto streaming_decoder_for_split_input = TextCodec::StreamingDecoder { "Big5"sv, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement };
|
||||||
EXPECT_EQ(MUST(streaming_decoder_for_split_input.to_utf8(bytes({ 0xa4 }))), ""sv);
|
EXPECT_EQ(MUST(streaming_decoder_for_split_input.to_utf8(bytes({ 0xa4 }))), ""sv);
|
||||||
EXPECT_EQ(MUST(streaming_decoder_for_split_input.to_utf8(bytes({ 0xa4 }))), MUST(decoder.to_utf8(StringView(bytes(big5_sequence)))));
|
EXPECT_EQ(MUST(streaming_decoder_for_split_input.to_utf8(bytes({ 0xa4 }))), MUST(decoder.to_utf8(StringView(bytes(big5_sequence)), TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)));
|
||||||
EXPECT_EQ(MUST(streaming_decoder_for_split_input.finish()), ""sv);
|
EXPECT_EQ(MUST(streaming_decoder_for_split_input.finish()), ""sv);
|
||||||
}
|
}
|
||||||
|
|
||||||
TEST_CASE(test_streaming_decoder_euc_jp_three_byte_tail)
|
TEST_CASE(test_streaming_decoder_euc_jp_three_byte_tail)
|
||||||
{
|
{
|
||||||
auto& decoder = decoder_for("EUC-JP"sv);
|
auto& decoder = decoder_for("EUC-JP"sv);
|
||||||
auto streaming_decoder = TextCodec::StreamingDecoder { "EUC-JP"sv };
|
auto streaming_decoder = TextCodec::StreamingDecoder { "EUC-JP"sv, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement };
|
||||||
auto euc_jp_sequence = Vector<u8> { 0x8f, 0xa2, 0xaf };
|
auto euc_jp_sequence = Vector<u8> { 0x8f, 0xa2, 0xaf };
|
||||||
|
|
||||||
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0x8f, 0xa2 }))), ""sv);
|
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0x8f, 0xa2 }))), ""sv);
|
||||||
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0xaf }))), MUST(decoder.to_utf8(StringView(bytes(euc_jp_sequence)))));
|
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0xaf }))), MUST(decoder.to_utf8(StringView(bytes(euc_jp_sequence)), TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)));
|
||||||
EXPECT_EQ(MUST(streaming_decoder.finish()), ""sv);
|
EXPECT_EQ(MUST(streaming_decoder.finish()), ""sv);
|
||||||
}
|
}
|
||||||
|
|
||||||
TEST_CASE(test_streaming_decoder_shift_jis_tail)
|
TEST_CASE(test_streaming_decoder_shift_jis_tail)
|
||||||
{
|
{
|
||||||
auto& decoder = decoder_for("Shift_JIS"sv);
|
auto& decoder = decoder_for("Shift_JIS"sv);
|
||||||
auto streaming_decoder = TextCodec::StreamingDecoder { "Shift_JIS"sv };
|
auto streaming_decoder = TextCodec::StreamingDecoder { "Shift_JIS"sv, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement };
|
||||||
auto shift_jis_sequence = Vector<u8> { 0x82, 0xa0 };
|
auto shift_jis_sequence = Vector<u8> { 0x82, 0xa0 };
|
||||||
|
|
||||||
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0x82 }))), ""sv);
|
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0x82 }))), ""sv);
|
||||||
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0xa0 }))), MUST(decoder.to_utf8(StringView(bytes(shift_jis_sequence)))));
|
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0xa0 }))), MUST(decoder.to_utf8(StringView(bytes(shift_jis_sequence)), TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)));
|
||||||
EXPECT_EQ(MUST(streaming_decoder.finish()), ""sv);
|
EXPECT_EQ(MUST(streaming_decoder.finish()), ""sv);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -236,8 +235,7 @@ TEST_CASE(test_utf16le_decode)
|
||||||
// This is the output of `python3 -c "print('säk😀'.encode('utf-16le'))"`.
|
// This is the output of `python3 -c "print('säk😀'.encode('utf-16le'))"`.
|
||||||
auto test_string = "s\x00\xe4\x00k\x00=\xd8\x00\xde"sv;
|
auto test_string = "s\x00\xe4\x00k\x00=\xd8\x00\xde"sv;
|
||||||
|
|
||||||
EXPECT(decoder.validate(test_string));
|
auto utf8 = MUST(decoder.to_utf8(test_string, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Fatal));
|
||||||
auto utf8 = MUST(decoder.to_utf8(test_string));
|
|
||||||
EXPECT_EQ(utf8, "säk😀"sv);
|
EXPECT_EQ(utf8, "säk😀"sv);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -248,5 +246,5 @@ TEST_CASE(test_utf16le_process_code_points)
|
||||||
EXPECT_EQ(process_code_points(decoder, StringView(bytes({ 0xff, 0xfe, 'A', 0x00, 0x3d, 0xd8, 0x00, 0xde }))), (Vector<u32> { 0x41, 0x1F600 }));
|
EXPECT_EQ(process_code_points(decoder, StringView(bytes({ 0xff, 0xfe, 'A', 0x00, 0x3d, 0xd8, 0x00, 0xde }))), (Vector<u32> { 0x41, 0x1F600 }));
|
||||||
EXPECT_EQ(process_code_points(decoder, StringView(bytes({ 0x3d, 0xd8, 'A', 0x00, 0x00, 0xde }))), (Vector<u32> { 0xfffd, 0x41, 0xfffd }));
|
EXPECT_EQ(process_code_points(decoder, StringView(bytes({ 0x3d, 0xd8, 'A', 0x00, 0x00, 0xde }))), (Vector<u32> { 0xfffd, 0x41, 0xfffd }));
|
||||||
EXPECT_EQ(process_code_points(decoder, StringView(bytes({ 'A', 0x00, 0xff }))), (Vector<u32> { 0x41, 0xfffd }));
|
EXPECT_EQ(process_code_points(decoder, StringView(bytes({ 'A', 0x00, 0xff }))), (Vector<u32> { 0x41, 0xfffd }));
|
||||||
EXPECT_EQ(MUST(decoder.to_utf8(StringView(bytes({ 'A', 0x00, 0xff })))), "A\xef\xbf\xbd"sv);
|
EXPECT_EQ(MUST(decoder.to_utf8(StringView(bytes({ 'A', 0x00, 0xff })), TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)), "A\xef\xbf\xbd"sv);
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,6 @@
|
||||||
|
<!DOCTYPE html>
|
||||||
|
<script>
|
||||||
|
// createDocument with an empty qualified name yields a document with no root element.
|
||||||
|
const doc = document.implementation.createDocument(null, "", null);
|
||||||
|
document.evaluate("/", doc, null, XPathResult.ANY_TYPE, null);
|
||||||
|
</script>
|
||||||
|
|
@ -0,0 +1,6 @@
|
||||||
|
length: 2
|
||||||
|
name: 0
|
||||||
|
export name: sqlite3_aggregate_context
|
||||||
|
length descriptor: writable=false, enumerable=false, configurable=true
|
||||||
|
name descriptor: writable=false, enumerable=false, configurable=true
|
||||||
|
call result: 42
|
||||||
|
|
@ -0,0 +1,17 @@
|
||||||
|
Harness status: OK
|
||||||
|
|
||||||
|
Found 12 tests
|
||||||
|
|
||||||
|
12 Pass
|
||||||
|
Pass ignoreBOM should work for encoding utf-8, split at character 0
|
||||||
|
Pass ignoreBOM should work for encoding utf-8, split at character 1
|
||||||
|
Pass ignoreBOM should work for encoding utf-8, split at character 2
|
||||||
|
Pass ignoreBOM should work for encoding utf-8, split at character 3
|
||||||
|
Pass ignoreBOM should work for encoding utf-16le, split at character 0
|
||||||
|
Pass ignoreBOM should work for encoding utf-16le, split at character 1
|
||||||
|
Pass ignoreBOM should work for encoding utf-16le, split at character 2
|
||||||
|
Pass ignoreBOM should work for encoding utf-16le, split at character 3
|
||||||
|
Pass ignoreBOM should work for encoding utf-16be, split at character 0
|
||||||
|
Pass ignoreBOM should work for encoding utf-16be, split at character 1
|
||||||
|
Pass ignoreBOM should work for encoding utf-16be, split at character 2
|
||||||
|
Pass ignoreBOM should work for encoding utf-16be, split at character 3
|
||||||
|
|
@ -2,17 +2,16 @@ Harness status: OK
|
||||||
|
|
||||||
Found 12 tests
|
Found 12 tests
|
||||||
|
|
||||||
4 Pass
|
12 Pass
|
||||||
8 Fail
|
|
||||||
Pass a TextEncoderStream object should be associated with the realm the constructor came from
|
Pass a TextEncoderStream object should be associated with the realm the constructor came from
|
||||||
Pass TextEncoderStream's readable and writable attributes should come from the same realm as the constructor definition
|
Pass TextEncoderStream's readable and writable attributes should come from the same realm as the constructor definition
|
||||||
Fail the output chunks when read is called after write should come from the same realm as the constructor of TextEncoderStream
|
Pass the output chunks when read is called after write should come from the same realm as the constructor of TextEncoderStream
|
||||||
Fail the output chunks when write is called with a pending read should come from the same realm as the constructor of TextEncoderStream
|
Pass the output chunks when write is called with a pending read should come from the same realm as the constructor of TextEncoderStream
|
||||||
Fail TypeError for unconvertable chunk should come from constructor realm of TextEncoderStream
|
Pass TypeError for unconvertable chunk should come from constructor realm of TextEncoderStream
|
||||||
Pass a TextDecoderStream object should be associated with the realm the constructor came from
|
Pass a TextDecoderStream object should be associated with the realm the constructor came from
|
||||||
Pass TextDecoderStream's readable and writable attributes should come from the same realm as the constructor definition
|
Pass TextDecoderStream's readable and writable attributes should come from the same realm as the constructor definition
|
||||||
Fail the result object when read is called after write should come from the same realm as the constructor of TextDecoderStream
|
Pass the result object when read is called after write should come from the same realm as the constructor of TextDecoderStream
|
||||||
Fail the result object when write is called with a pending read should come from the same realm as the constructor of TextDecoderStream
|
Pass the result object when write is called with a pending read should come from the same realm as the constructor of TextDecoderStream
|
||||||
Fail TypeError for chunk with the wrong type should come from constructor realm of TextDecoderStream
|
Pass TypeError for chunk with the wrong type should come from constructor realm of TextDecoderStream
|
||||||
Fail TypeError for invalid chunk should come from constructor realm of TextDecoderStream
|
Pass TypeError for invalid chunk should come from constructor realm of TextDecoderStream
|
||||||
Fail TypeError for incomplete input should come from constructor realm of TextDecoderStream
|
Pass TypeError for incomplete input should come from constructor realm of TextDecoderStream
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,8 @@
|
||||||
|
Harness status: OK
|
||||||
|
|
||||||
|
Found 2 tests
|
||||||
|
|
||||||
|
1 Pass
|
||||||
|
1 Fail
|
||||||
|
Pass Fatal flag, non-streaming cases
|
||||||
|
Fail Fatal flag, streaming cases
|
||||||
|
|
@ -2,9 +2,8 @@ Harness status: OK
|
||||||
|
|
||||||
Found 4 tests
|
Found 4 tests
|
||||||
|
|
||||||
1 Pass
|
4 Pass
|
||||||
3 Fail
|
Pass BOM is ignored if ignoreBOM option is specified: utf-8
|
||||||
Fail BOM is ignored if ignoreBOM option is specified: utf-8
|
Pass BOM is ignored if ignoreBOM option is specified: utf-16le
|
||||||
Fail BOM is ignored if ignoreBOM option is specified: utf-16le
|
Pass BOM is ignored if ignoreBOM option is specified: utf-16be
|
||||||
Fail BOM is ignored if ignoreBOM option is specified: utf-16be
|
|
||||||
Pass The ignoreBOM attribute of TextDecoder
|
Pass The ignoreBOM attribute of TextDecoder
|
||||||
|
|
@ -2,8 +2,8 @@ Harness status: OK
|
||||||
|
|
||||||
Found 87 tests
|
Found 87 tests
|
||||||
|
|
||||||
72 Pass
|
74 Pass
|
||||||
15 Fail
|
13 Fail
|
||||||
Pass Invalid Unicode input is replaced: utf-8
|
Pass Invalid Unicode input is replaced: utf-8
|
||||||
Pass Invalid Unicode input is replaced: utf-16le
|
Pass Invalid Unicode input is replaced: utf-16le
|
||||||
Pass Invalid Unicode input is replaced: utf-16be
|
Pass Invalid Unicode input is replaced: utf-16be
|
||||||
|
|
@ -51,8 +51,8 @@ Pass selected single-byte: iso-8859-8-i
|
||||||
Pass selected single-byte: iso-8859-16
|
Pass selected single-byte: iso-8859-16
|
||||||
Pass selected single-byte: x-mac-cyrillic
|
Pass selected single-byte: x-mac-cyrillic
|
||||||
Pass Concatenating two ISO-2022-JP outputs is not always valid
|
Pass Concatenating two ISO-2022-JP outputs is not always valid
|
||||||
Fail gb18030 version and ranges
|
Pass gb18030 version and ranges
|
||||||
Fail gbk version and ranges
|
Pass gbk version and ranges
|
||||||
Pass gbk decoder is gb18030 decoder
|
Pass gbk decoder is gb18030 decoder
|
||||||
Pass Replacement, push back ASCII characters: big5
|
Pass Replacement, push back ASCII characters: big5
|
||||||
Pass Replacement, push back ASCII characters: iso-2022-jp
|
Pass Replacement, push back ASCII characters: iso-2022-jp
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,32 @@
|
||||||
|
<!DOCTYPE html>
|
||||||
|
<script src="../include.js"></script>
|
||||||
|
<script>
|
||||||
|
asyncTest(async (done) => {
|
||||||
|
const wasmBytes = new Uint8Array([
|
||||||
|
0x00, 0x61, 0x73, 0x6d, 0x01, 0x00, 0x00, 0x00,
|
||||||
|
0x01, 0x07, 0x01, 0x60, 0x02, 0x7f, 0x7f, 0x01, 0x7f,
|
||||||
|
0x03, 0x02, 0x01, 0x00,
|
||||||
|
0x07, 0x1d, 0x01, 0x19,
|
||||||
|
0x73, 0x71, 0x6c, 0x69, 0x74, 0x65, 0x33, 0x5f,
|
||||||
|
0x61, 0x67, 0x67, 0x72, 0x65, 0x67, 0x61, 0x74,
|
||||||
|
0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74,
|
||||||
|
0x00, 0x00,
|
||||||
|
0x0a, 0x06, 0x01, 0x04, 0x00, 0x20, 0x00, 0x0b,
|
||||||
|
]);
|
||||||
|
|
||||||
|
const { instance } = await WebAssembly.instantiate(wasmBytes);
|
||||||
|
const fn = instance.exports.sqlite3_aggregate_context;
|
||||||
|
const exportName = Object.getOwnPropertyNames(instance.exports)[0];
|
||||||
|
const lengthDescriptor = Object.getOwnPropertyDescriptor(fn, "length");
|
||||||
|
const nameDescriptor = Object.getOwnPropertyDescriptor(fn, "name");
|
||||||
|
|
||||||
|
println(`length: ${fn.length}`);
|
||||||
|
println(`name: ${fn.name}`);
|
||||||
|
println(`export name: ${exportName}`);
|
||||||
|
println(`length descriptor: writable=${lengthDescriptor.writable}, enumerable=${lengthDescriptor.enumerable}, configurable=${lengthDescriptor.configurable}`);
|
||||||
|
println(`name descriptor: writable=${nameDescriptor.writable}, enumerable=${nameDescriptor.enumerable}, configurable=${nameDescriptor.configurable}`);
|
||||||
|
println(`call result: ${fn(42, 7)}`);
|
||||||
|
|
||||||
|
done();
|
||||||
|
});
|
||||||
|
</script>
|
||||||
|
|
@ -0,0 +1,16 @@
|
||||||
|
<!doctype html>
|
||||||
|
<meta charset=utf-8>
|
||||||
|
|
||||||
|
<script>
|
||||||
|
self.GLOBAL = {
|
||||||
|
isWindow: function() { return true; },
|
||||||
|
isWorker: function() { return false; },
|
||||||
|
isShadowRealm: function() { return false; },
|
||||||
|
};
|
||||||
|
</script>
|
||||||
|
<script src="../../resources/testharness.js"></script>
|
||||||
|
<script src="../../resources/testharnessreport.js"></script>
|
||||||
|
<script src="resources/readable-stream-from-array.js"></script>
|
||||||
|
<script src="resources/readable-stream-to-array.js"></script>
|
||||||
|
<div id=log></div>
|
||||||
|
<script src="../../encoding/streams/decode-ignore-bom.any.js"></script>
|
||||||
|
|
@ -0,0 +1,38 @@
|
||||||
|
// META: global=window,worker
|
||||||
|
// META: script=resources/readable-stream-from-array.js
|
||||||
|
// META: script=resources/readable-stream-to-array.js
|
||||||
|
|
||||||
|
const cases = [
|
||||||
|
{encoding: 'utf-8', bytes: [0xEF, 0xBB, 0xBF, 0x61, 0x62, 0x63]},
|
||||||
|
{encoding: 'utf-16le', bytes: [0xFF, 0xFE, 0x61, 0x00, 0x62, 0x00, 0x63, 0x00]},
|
||||||
|
{encoding: 'utf-16be', bytes: [0xFE, 0xFF, 0x00, 0x61, 0x00, 0x62, 0x00, 0x63]}
|
||||||
|
];
|
||||||
|
const BOM = '\uFEFF';
|
||||||
|
|
||||||
|
// |inputChunks| is an array of chunks, each represented by an array of
|
||||||
|
// integers. |ignoreBOM| is true or false. The result value is the output of the
|
||||||
|
// pipe, concatenated into a single string.
|
||||||
|
async function pipeAndAssemble(inputChunks, encoding, ignoreBOM) {
|
||||||
|
const chunksAsUint8 = inputChunks.map(values => new Uint8Array(values));
|
||||||
|
const readable = readableStreamFromArray(chunksAsUint8);
|
||||||
|
const outputArray = await readableStreamToArray(readable.pipeThrough(
|
||||||
|
new TextDecoderStream(encoding, {ignoreBOM})));
|
||||||
|
return outputArray.join('');
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const testCase of cases) {
|
||||||
|
for (let splitPoint = 0; splitPoint < 4; ++splitPoint) {
|
||||||
|
promise_test(async () => {
|
||||||
|
const inputChunks = [testCase.bytes.slice(0, splitPoint),
|
||||||
|
testCase.bytes.slice(splitPoint)];
|
||||||
|
const withIgnoreBOM =
|
||||||
|
await pipeAndAssemble(inputChunks, testCase.encoding, true);
|
||||||
|
assert_equals(withIgnoreBOM, BOM + 'abc', 'BOM should be preserved');
|
||||||
|
|
||||||
|
const withoutIgnoreBOM =
|
||||||
|
await pipeAndAssemble(inputChunks, testCase.encoding, false);
|
||||||
|
assert_equals(withoutIgnoreBOM, 'abc', 'BOM should be stripped')
|
||||||
|
}, `ignoreBOM should work for encoding ${testCase.encoding}, split at ` +
|
||||||
|
`character ${splitPoint}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,15 @@
|
||||||
|
<!doctype html>
|
||||||
|
<meta charset=utf-8>
|
||||||
|
<title>Encoding API: End-of-file</title>
|
||||||
|
<script>
|
||||||
|
self.GLOBAL = {
|
||||||
|
isWindow: function() { return true; },
|
||||||
|
isWorker: function() { return false; },
|
||||||
|
isShadowRealm: function() { return false; },
|
||||||
|
};
|
||||||
|
</script>
|
||||||
|
<script src="../resources/testharness.js"></script>
|
||||||
|
<script src="../resources/testharnessreport.js"></script>
|
||||||
|
|
||||||
|
<div id=log></div>
|
||||||
|
<script src="../encoding/textdecoder-fatal-streaming.any.js"></script>
|
||||||
|
|
@ -0,0 +1,45 @@
|
||||||
|
// META: global=window,dedicatedworker
|
||||||
|
// META: title=Encoding API: End-of-file
|
||||||
|
|
||||||
|
test(function() {
|
||||||
|
[
|
||||||
|
{encoding: 'utf-8', sequence: [0xC0]},
|
||||||
|
{encoding: 'utf-16le', sequence: [0x00]},
|
||||||
|
{encoding: 'utf-16be', sequence: [0x00]}
|
||||||
|
].forEach(function(testCase) {
|
||||||
|
|
||||||
|
assert_throws_js(TypeError, function() {
|
||||||
|
var decoder = new TextDecoder(testCase.encoding, {fatal: true});
|
||||||
|
decoder.decode(new Uint8Array(testCase.sequence));
|
||||||
|
}, 'Unterminated ' + testCase.encoding + ' sequence should throw if fatal flag is set');
|
||||||
|
|
||||||
|
assert_equals(
|
||||||
|
new TextDecoder(testCase.encoding).decode(new Uint8Array([testCase.sequence])),
|
||||||
|
'\uFFFD',
|
||||||
|
'Unterminated UTF-8 sequence should emit replacement character if fatal flag is unset');
|
||||||
|
});
|
||||||
|
}, 'Fatal flag, non-streaming cases');
|
||||||
|
|
||||||
|
test(function() {
|
||||||
|
|
||||||
|
var decoder = new TextDecoder('utf-16le', {fatal: true});
|
||||||
|
var odd = new Uint8Array([0x00]);
|
||||||
|
var even = new Uint8Array([0x00, 0x00]);
|
||||||
|
|
||||||
|
assert_equals(decoder.decode(odd, {stream: true}), '');
|
||||||
|
assert_equals(decoder.decode(odd), '\u0000');
|
||||||
|
|
||||||
|
assert_throws_js(TypeError, function() {
|
||||||
|
decoder.decode(even, {stream: true});
|
||||||
|
decoder.decode(odd)
|
||||||
|
});
|
||||||
|
|
||||||
|
assert_throws_js(TypeError, function() {
|
||||||
|
decoder.decode(odd, {stream: true});
|
||||||
|
decoder.decode(even);
|
||||||
|
});
|
||||||
|
|
||||||
|
assert_equals(decoder.decode(even, {stream: true}), '\u0000');
|
||||||
|
assert_equals(decoder.decode(even), '\u0000');
|
||||||
|
|
||||||
|
}, 'Fatal flag, streaming cases');
|
||||||
Loading…
Reference in a new issue