Compare commits
10 commits
b81269e78b
...
b41536f4ef
| Author | SHA1 | Date | |
|---|---|---|---|
| b41536f4ef | |||
|
|
4af1b9357b | ||
|
|
a7ce007ff4 | ||
|
|
d4048aaa96 | ||
|
|
b9da74d16e | ||
|
|
ef6753a9f9 | ||
|
|
ef99632fa7 | ||
|
|
02320c9b58 | ||
|
|
ae0c7bc097 | ||
|
|
8caee39bac |
39 changed files with 527 additions and 278 deletions
112
.forgejo/workflows/build-ubuntu-24-04.yml
Normal file
112
.forgejo/workflows/build-ubuntu-24-04.yml
Normal file
|
|
@ -0,0 +1,112 @@
|
|||
name: Build Ladybird (Ubuntu 24.04)
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
workflow_dispatch:
|
||||
|
||||
env:
|
||||
LADYBIRD_SOURCE_DIR: ${{ github.workspace }}
|
||||
CCACHE_DIR: ${{ github.workspace }}/.ccache
|
||||
VCPKG_ROOT: ${{ github.workspace }}/Build/vcpkg
|
||||
|
||||
jobs:
|
||||
build:
|
||||
name: Build on Ubuntu 24.04
|
||||
runs-on: ubuntu-latest
|
||||
container:
|
||||
image: ubuntu:24.04
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Install system dependencies
|
||||
run: |
|
||||
apt-get update && apt-get install -y \
|
||||
software-properties-common \
|
||||
wget \
|
||||
curl \
|
||||
gpg \
|
||||
lsb-release \
|
||||
git
|
||||
|
||||
# Install CMake 3.30+ from Kitware (since Ubuntu 24.04 has 3.28)
|
||||
wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | tee /usr/share/keyrings/kitware-archive-keyring.gpg >/dev/null
|
||||
echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ noble main" | tee /etc/apt/sources.list.d/kitware.list
|
||||
|
||||
apt-get update && apt-get install -y \
|
||||
autoconf \
|
||||
autoconf-archive \
|
||||
automake \
|
||||
build-essential \
|
||||
ccache \
|
||||
cmake \
|
||||
fonts-liberation2 \
|
||||
glslang-tools \
|
||||
libdrm-dev \
|
||||
libgl1-mesa-dev \
|
||||
libncurses-dev \
|
||||
libtool \
|
||||
nasm \
|
||||
ninja-build \
|
||||
pkg-config \
|
||||
python3-venv \
|
||||
qt6-base-dev \
|
||||
qt6-tools-dev-tools \
|
||||
qt6-wayland \
|
||||
tar \
|
||||
unzip \
|
||||
zip \
|
||||
libpulse-dev \
|
||||
gcc-14 \
|
||||
g++-14
|
||||
|
||||
- name: Configure compiler alternatives
|
||||
run: |
|
||||
update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-14 100 \
|
||||
--slave /usr/bin/g++ g++ /usr/bin/g++-14 \
|
||||
--slave /usr/bin/gcov gcov /usr/bin/gcov-14
|
||||
|
||||
- name: Install Rust
|
||||
run: |
|
||||
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain 1.96.0
|
||||
echo "$HOME/.cargo/bin" >> $GITHUB_PATH
|
||||
|
||||
- name: Configure Git Safe Directory
|
||||
run: |
|
||||
git config --global safe.directory '*'
|
||||
|
||||
- name: Bootstrap vcpkg
|
||||
run: |
|
||||
./Meta/Utils/build_vcpkg.py
|
||||
|
||||
- name: Configure CMake
|
||||
env:
|
||||
CC: gcc-14
|
||||
CXX: g++-14
|
||||
run: |
|
||||
cmake --preset Release -B Build \
|
||||
-DCMAKE_C_COMPILER=gcc-14 \
|
||||
-DCMAKE_CXX_COMPILER=g++-14 \
|
||||
-DENABLE_CI_BASELINE_CPU=ON \
|
||||
-DLADYBIRD_GUI_FRAMEWORK=Qt
|
||||
|
||||
- name: Build
|
||||
run: |
|
||||
cmake --build Build
|
||||
|
||||
- name: Install
|
||||
run: |
|
||||
cmake --install Build --strip --prefix dist
|
||||
|
||||
- name: Package Binaries
|
||||
run: |
|
||||
tar -czf ladybird-ubuntu-24.04.tar.gz -C dist .
|
||||
|
||||
- name: Upload Build Artifacts
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: ladybird-ubuntu-24.04
|
||||
path: ladybird-ubuntu-24.04.tar.gz
|
||||
|
|
@ -15,15 +15,8 @@
|
|||
|
||||
namespace TextCodec {
|
||||
|
||||
static constexpr u32 replacement_code_point = 0xfffd;
|
||||
|
||||
namespace {
|
||||
|
||||
enum class RemoveBOM {
|
||||
No,
|
||||
Yes,
|
||||
};
|
||||
|
||||
class RustDecoder final : public Decoder {
|
||||
public:
|
||||
explicit RustDecoder(StringView encoding)
|
||||
|
|
@ -31,8 +24,7 @@ public:
|
|||
{
|
||||
}
|
||||
|
||||
virtual bool validate(StringView input) override;
|
||||
virtual ErrorOr<String> to_utf8(StringView input) override;
|
||||
virtual ErrorOr<String> to_utf8(StringView input, IgnoreBOM, ErrorMode) override;
|
||||
virtual ErrorOr<size_t> length_in_utf16_code_units(StringView input) override;
|
||||
|
||||
private:
|
||||
|
|
@ -44,15 +36,13 @@ private:
|
|||
class UTF8Decoder final : public Decoder {
|
||||
public:
|
||||
virtual ErrorOr<void> process(StringView, Function<ErrorOr<void>(u32)> on_code_point) override;
|
||||
virtual bool validate(StringView) override;
|
||||
virtual ErrorOr<String> to_utf8(StringView) override;
|
||||
virtual ErrorOr<String> to_utf8(StringView, IgnoreBOM, ErrorMode) override;
|
||||
virtual ErrorOr<size_t> length_in_utf16_code_units(StringView) override;
|
||||
};
|
||||
|
||||
class UTF16BEDecoder final : public Decoder {
|
||||
public:
|
||||
virtual bool validate(StringView) override;
|
||||
virtual ErrorOr<String> to_utf8(StringView) override;
|
||||
virtual ErrorOr<String> to_utf8(StringView, IgnoreBOM, ErrorMode) override;
|
||||
virtual ErrorOr<size_t> length_in_utf16_code_units(StringView) override;
|
||||
|
||||
private:
|
||||
|
|
@ -61,8 +51,7 @@ private:
|
|||
|
||||
class UTF16LEDecoder final : public Decoder {
|
||||
public:
|
||||
virtual bool validate(StringView) override;
|
||||
virtual ErrorOr<String> to_utf8(StringView) override;
|
||||
virtual ErrorOr<String> to_utf8(StringView, IgnoreBOM, ErrorMode) override;
|
||||
virtual ErrorOr<size_t> length_in_utf16_code_units(StringView) override;
|
||||
|
||||
private:
|
||||
|
|
@ -72,7 +61,6 @@ private:
|
|||
class Latin1Decoder final : public Decoder {
|
||||
public:
|
||||
virtual ErrorOr<void> process(StringView, Function<ErrorOr<void>(u32)> on_code_point) override;
|
||||
virtual bool validate(StringView) override { return true; }
|
||||
virtual ErrorOr<size_t> length_in_utf16_code_units(StringView) override;
|
||||
};
|
||||
|
||||
|
|
@ -130,7 +118,7 @@ static void append_decoded_bytes(void* context, u8 const* data, size_t length)
|
|||
decode_context.result = decode_context.builder.try_append(StringView { data, length });
|
||||
}
|
||||
|
||||
ErrorOr<String> rust_decode_to_utf8(StringView encoding, StringView input, RemoveBOM remove_bom)
|
||||
ErrorOr<String> rust_decode_to_utf8(StringView encoding, StringView input, IgnoreBOM ignore_bom, ErrorMode error_mode)
|
||||
{
|
||||
DecodeContext context { .builder = StringBuilder(input.length()) };
|
||||
auto succeeded = FFI::textcodec_rust_decode_to_utf8(
|
||||
|
|
@ -138,36 +126,27 @@ ErrorOr<String> rust_decode_to_utf8(StringView encoding, StringView input, Remov
|
|||
encoding.length(),
|
||||
reinterpret_cast<u8 const*>(input.characters_without_null_termination()),
|
||||
input.length(),
|
||||
remove_bom == RemoveBOM::Yes,
|
||||
ignore_bom == IgnoreBOM::No,
|
||||
error_mode == ErrorMode::Fatal,
|
||||
&context,
|
||||
append_decoded_bytes);
|
||||
if (!succeeded)
|
||||
return Error::from_errno(EINVAL);
|
||||
return Error::from_string_literal("Failed to decode input");
|
||||
TRY(context.result);
|
||||
return context.builder.to_string_without_validation();
|
||||
}
|
||||
|
||||
ErrorOr<void> rust_process(StringView encoding, StringView input, RemoveBOM remove_bom, Function<ErrorOr<void>(u32)> on_code_point)
|
||||
ErrorOr<void> rust_process(StringView encoding, StringView input, IgnoreBOM ignore_bom, Function<ErrorOr<void>(u32)> on_code_point)
|
||||
{
|
||||
auto utf8 = TRY(rust_decode_to_utf8(encoding, input, remove_bom));
|
||||
auto utf8 = TRY(rust_decode_to_utf8(encoding, input, ignore_bom, ErrorMode::Replacement));
|
||||
for (auto code_point : Utf8View { utf8 })
|
||||
TRY(on_code_point(code_point));
|
||||
return {};
|
||||
}
|
||||
|
||||
bool rust_validate(StringView encoding, StringView input, RemoveBOM remove_bom)
|
||||
ErrorOr<size_t> rust_length_in_utf16_code_units(StringView encoding, StringView input, IgnoreBOM ignore_bom)
|
||||
{
|
||||
return FFI::textcodec_rust_validate(
|
||||
reinterpret_cast<u8 const*>(encoding.characters_without_null_termination()),
|
||||
encoding.length(),
|
||||
reinterpret_cast<u8 const*>(input.characters_without_null_termination()),
|
||||
input.length(),
|
||||
remove_bom == RemoveBOM::Yes);
|
||||
}
|
||||
|
||||
ErrorOr<size_t> rust_length_in_utf16_code_units(StringView encoding, StringView input, RemoveBOM remove_bom)
|
||||
{
|
||||
auto utf8 = TRY(rust_decode_to_utf8(encoding, input, remove_bom));
|
||||
auto utf8 = TRY(rust_decode_to_utf8(encoding, input, ignore_bom, ErrorMode::Replacement));
|
||||
size_t length = 0;
|
||||
for (auto code_point : Utf8View { utf8 })
|
||||
length += code_point <= 0xffff ? 1 : 2;
|
||||
|
|
@ -188,7 +167,7 @@ Optional<StringView> get_static_encoding_name_from_rust(StringView label)
|
|||
return StringView { encoding_name, encoding_name_length };
|
||||
}
|
||||
|
||||
ErrorOr<String> rust_streaming_decode_to_utf8(FFI::TextCodecRustStreamingDecoder* decoder, ReadonlyBytes input, bool last)
|
||||
ErrorOr<String> rust_streaming_decode_to_utf8(FFI::TextCodecRustStreamingDecoder* decoder, ReadonlyBytes input, bool last, ErrorMode error_mode)
|
||||
{
|
||||
DecodeContext context { .builder = StringBuilder(input.size()) };
|
||||
auto succeeded = FFI::textcodec_rust_streaming_decoder_decode_to_utf8(
|
||||
|
|
@ -196,10 +175,11 @@ ErrorOr<String> rust_streaming_decode_to_utf8(FFI::TextCodecRustStreamingDecoder
|
|||
input.data(),
|
||||
input.size(),
|
||||
last,
|
||||
error_mode == ErrorMode::Fatal,
|
||||
&context,
|
||||
append_decoded_bytes);
|
||||
if (!succeeded)
|
||||
return Error::from_errno(EINVAL);
|
||||
return Error::from_string_literal("Failed to decode input");
|
||||
TRY(context.result);
|
||||
return context.builder.to_string_without_validation();
|
||||
}
|
||||
|
|
@ -367,7 +347,7 @@ ErrorOr<String> convert_input_to_utf8_using_given_decoder_unless_there_is_a_byte
|
|||
|
||||
// 3. Process a queue with an instance of encoding’s decoder, ioQueue, output, and "replacement".
|
||||
// FIXME: This isn't the exact same as the spec, which is written in terms of I/O queues.
|
||||
auto output = TRY(actual_decoder->to_utf8(input));
|
||||
auto output = TRY(actual_decoder->to_utf8(input, IgnoreBOM::No, ErrorMode::Replacement));
|
||||
|
||||
// 4. Return output.
|
||||
return output;
|
||||
|
|
@ -410,18 +390,7 @@ StringView get_output_encoding(StringView encoding)
|
|||
return encoding;
|
||||
}
|
||||
|
||||
bool Decoder::validate(StringView input)
|
||||
{
|
||||
auto result = this->process(input, [](auto code_point) -> ErrorOr<void> {
|
||||
if (code_point == replacement_code_point)
|
||||
return Error::from_errno(EINVAL);
|
||||
return {};
|
||||
});
|
||||
|
||||
return !result.is_error();
|
||||
}
|
||||
|
||||
ErrorOr<String> Decoder::to_utf8(StringView input)
|
||||
ErrorOr<String> Decoder::to_utf8(StringView input, IgnoreBOM, ErrorMode)
|
||||
{
|
||||
StringBuilder builder(input.length());
|
||||
TRY(process(input, [&builder](u32 c) { return builder.try_append_code_point(c); }));
|
||||
|
|
@ -453,24 +422,19 @@ ErrorOr<void> Decoder::process_code_points(StringView input, Function<ErrorOr<vo
|
|||
return process(input, move(on_code_point));
|
||||
}
|
||||
|
||||
bool RustDecoder::validate(StringView input)
|
||||
ErrorOr<String> RustDecoder::to_utf8(StringView input, IgnoreBOM ignore_bom, ErrorMode error_mode)
|
||||
{
|
||||
return rust_validate(m_encoding, input, RemoveBOM::No);
|
||||
}
|
||||
|
||||
ErrorOr<String> RustDecoder::to_utf8(StringView input)
|
||||
{
|
||||
return rust_decode_to_utf8(m_encoding, input, RemoveBOM::No);
|
||||
return rust_decode_to_utf8(m_encoding, input, ignore_bom, error_mode);
|
||||
}
|
||||
|
||||
ErrorOr<size_t> RustDecoder::length_in_utf16_code_units(StringView input)
|
||||
{
|
||||
return rust_length_in_utf16_code_units(m_encoding, input, RemoveBOM::No);
|
||||
return rust_length_in_utf16_code_units(m_encoding, input, IgnoreBOM::Yes);
|
||||
}
|
||||
|
||||
ErrorOr<void> RustDecoder::process(StringView input, Function<ErrorOr<void>(u32)> on_code_point)
|
||||
{
|
||||
return rust_process(m_encoding, input, RemoveBOM::No, move(on_code_point));
|
||||
return rust_process(m_encoding, input, IgnoreBOM::Yes, move(on_code_point));
|
||||
}
|
||||
|
||||
ErrorOr<void> Latin1Decoder::process(StringView input, Function<ErrorOr<void>(u32)> on_code_point)
|
||||
|
|
@ -485,12 +449,13 @@ ErrorOr<size_t> Latin1Decoder::length_in_utf16_code_units(StringView input)
|
|||
return input.length();
|
||||
}
|
||||
|
||||
StreamingDecoder::StreamingDecoder(StringView encoding)
|
||||
StreamingDecoder::StreamingDecoder(StringView encoding, IgnoreBOM ignore_bom, ErrorMode error_mode)
|
||||
: m_error_mode(error_mode)
|
||||
{
|
||||
m_decoder = FFI::textcodec_rust_streaming_decoder_new(
|
||||
reinterpret_cast<u8 const*>(encoding.characters_without_null_termination()),
|
||||
encoding.length(),
|
||||
true);
|
||||
ignore_bom == IgnoreBOM::No);
|
||||
VERIFY(m_decoder);
|
||||
}
|
||||
|
||||
|
|
@ -501,72 +466,57 @@ StreamingDecoder::~StreamingDecoder()
|
|||
|
||||
ErrorOr<String> StreamingDecoder::to_utf8(ReadonlyBytes input)
|
||||
{
|
||||
return rust_streaming_decode_to_utf8(static_cast<FFI::TextCodecRustStreamingDecoder*>(m_decoder), input, false);
|
||||
return rust_streaming_decode_to_utf8(static_cast<FFI::TextCodecRustStreamingDecoder*>(m_decoder), input, false, m_error_mode);
|
||||
}
|
||||
|
||||
ErrorOr<String> StreamingDecoder::finish()
|
||||
{
|
||||
return rust_streaming_decode_to_utf8(static_cast<FFI::TextCodecRustStreamingDecoder*>(m_decoder), {}, true);
|
||||
return rust_streaming_decode_to_utf8(static_cast<FFI::TextCodecRustStreamingDecoder*>(m_decoder), {}, true, m_error_mode);
|
||||
}
|
||||
|
||||
ErrorOr<void> UTF8Decoder::process(StringView input, Function<ErrorOr<void>(u32)> on_code_point)
|
||||
{
|
||||
return rust_process("UTF-8"sv, input, RemoveBOM::No, move(on_code_point));
|
||||
return rust_process("UTF-8"sv, input, IgnoreBOM::Yes, move(on_code_point));
|
||||
}
|
||||
|
||||
bool UTF8Decoder::validate(StringView input)
|
||||
ErrorOr<String> UTF8Decoder::to_utf8(StringView input, IgnoreBOM ignore_bom, ErrorMode error_mode)
|
||||
{
|
||||
return rust_validate("UTF-8"sv, input, RemoveBOM::No);
|
||||
}
|
||||
|
||||
ErrorOr<String> UTF8Decoder::to_utf8(StringView input)
|
||||
{
|
||||
return rust_decode_to_utf8("UTF-8"sv, input, RemoveBOM::Yes);
|
||||
return rust_decode_to_utf8("UTF-8"sv, input, ignore_bom, error_mode);
|
||||
}
|
||||
|
||||
ErrorOr<size_t> UTF8Decoder::length_in_utf16_code_units(StringView input)
|
||||
{
|
||||
return rust_length_in_utf16_code_units("UTF-8"sv, input, RemoveBOM::Yes);
|
||||
}
|
||||
|
||||
bool UTF16BEDecoder::validate(StringView input)
|
||||
{
|
||||
return rust_validate("UTF-16BE"sv, input, RemoveBOM::No);
|
||||
return rust_length_in_utf16_code_units("UTF-8"sv, input, IgnoreBOM::No);
|
||||
}
|
||||
|
||||
ErrorOr<void> UTF16BEDecoder::process(StringView input, Function<ErrorOr<void>(u32)> on_code_point)
|
||||
{
|
||||
return rust_process("UTF-16BE"sv, input, RemoveBOM::Yes, move(on_code_point));
|
||||
return rust_process("UTF-16BE"sv, input, IgnoreBOM::No, move(on_code_point));
|
||||
}
|
||||
|
||||
ErrorOr<String> UTF16BEDecoder::to_utf8(StringView input)
|
||||
ErrorOr<String> UTF16BEDecoder::to_utf8(StringView input, IgnoreBOM ignore_bom, ErrorMode error_mode)
|
||||
{
|
||||
return rust_decode_to_utf8("UTF-16BE"sv, input, RemoveBOM::Yes);
|
||||
return rust_decode_to_utf8("UTF-16BE"sv, input, ignore_bom, error_mode);
|
||||
}
|
||||
|
||||
ErrorOr<size_t> UTF16BEDecoder::length_in_utf16_code_units(StringView input)
|
||||
{
|
||||
return rust_length_in_utf16_code_units("UTF-16BE"sv, input, RemoveBOM::Yes);
|
||||
}
|
||||
|
||||
bool UTF16LEDecoder::validate(StringView input)
|
||||
{
|
||||
return rust_validate("UTF-16LE"sv, input, RemoveBOM::No);
|
||||
return rust_length_in_utf16_code_units("UTF-16BE"sv, input, IgnoreBOM::No);
|
||||
}
|
||||
|
||||
ErrorOr<void> UTF16LEDecoder::process(StringView input, Function<ErrorOr<void>(u32)> on_code_point)
|
||||
{
|
||||
return rust_process("UTF-16LE"sv, input, RemoveBOM::Yes, move(on_code_point));
|
||||
return rust_process("UTF-16LE"sv, input, IgnoreBOM::No, move(on_code_point));
|
||||
}
|
||||
|
||||
ErrorOr<String> UTF16LEDecoder::to_utf8(StringView input)
|
||||
ErrorOr<String> UTF16LEDecoder::to_utf8(StringView input, IgnoreBOM ignore_bom, ErrorMode error_mode)
|
||||
{
|
||||
return rust_decode_to_utf8("UTF-16LE"sv, input, RemoveBOM::Yes);
|
||||
return rust_decode_to_utf8("UTF-16LE"sv, input, ignore_bom, error_mode);
|
||||
}
|
||||
|
||||
ErrorOr<size_t> UTF16LEDecoder::length_in_utf16_code_units(StringView input)
|
||||
{
|
||||
return rust_length_in_utf16_code_units("UTF-16LE"sv, input, RemoveBOM::Yes);
|
||||
return rust_length_in_utf16_code_units("UTF-16LE"sv, input, IgnoreBOM::No);
|
||||
}
|
||||
|
||||
// https://infra.spec.whatwg.org/#isomorphic-decode
|
||||
|
|
|
|||
|
|
@ -19,10 +19,20 @@
|
|||
|
||||
namespace TextCodec {
|
||||
|
||||
enum class IgnoreBOM {
|
||||
Yes,
|
||||
No,
|
||||
};
|
||||
|
||||
// https://encoding.spec.whatwg.org/#concept-encoding-error-mode
|
||||
enum class ErrorMode {
|
||||
Replacement,
|
||||
Fatal,
|
||||
};
|
||||
|
||||
class TEXTCODEC_API Decoder {
|
||||
public:
|
||||
virtual bool validate(StringView);
|
||||
virtual ErrorOr<String> to_utf8(StringView);
|
||||
virtual ErrorOr<String> to_utf8(StringView, IgnoreBOM, ErrorMode);
|
||||
virtual ErrorOr<Utf16String> to_utf16(StringView);
|
||||
virtual ErrorOr<size_t> length_in_utf16_code_units(StringView);
|
||||
ErrorOr<void> process_code_points(StringView, Function<ErrorOr<void>(u32)>);
|
||||
|
|
@ -36,13 +46,14 @@ class TEXTCODEC_API StreamingDecoder final {
|
|||
AK_MAKE_NONCOPYABLE(StreamingDecoder);
|
||||
|
||||
public:
|
||||
explicit StreamingDecoder(StringView encoding);
|
||||
StreamingDecoder(StringView encoding, IgnoreBOM, ErrorMode);
|
||||
~StreamingDecoder();
|
||||
|
||||
ErrorOr<String> to_utf8(ReadonlyBytes);
|
||||
ErrorOr<String> finish();
|
||||
|
||||
private:
|
||||
ErrorMode m_error_mode { ErrorMode::Replacement };
|
||||
void* m_decoder { nullptr };
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -93,6 +93,7 @@ pub unsafe extern "C" fn textcodec_rust_decode_to_utf8(
|
|||
input: *const u8,
|
||||
input_len: usize,
|
||||
remove_bom: bool,
|
||||
fatal: bool,
|
||||
ctx: *mut c_void,
|
||||
on_bytes: FfiBytesFn,
|
||||
) -> bool {
|
||||
|
|
@ -108,60 +109,20 @@ pub unsafe extern "C" fn textcodec_rust_decode_to_utf8(
|
|||
return false;
|
||||
};
|
||||
|
||||
let (output, _) = if remove_bom {
|
||||
let (output, had_errors) = if remove_bom {
|
||||
encoding.decode_with_bom_removal(input)
|
||||
} else {
|
||||
encoding.decode_without_bom_handling(input)
|
||||
};
|
||||
if fatal && had_errors {
|
||||
return false;
|
||||
}
|
||||
on_bytes(ctx, output.as_bytes().as_ptr(), output.len());
|
||||
true
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// # Safety
|
||||
/// - `encoding_label`/`encoding_label_len` and `input`/`input_len` must be valid byte slices.
|
||||
#[unsafe(no_mangle)]
|
||||
pub unsafe extern "C" fn textcodec_rust_validate(
|
||||
encoding_label: *const u8,
|
||||
encoding_label_len: usize,
|
||||
input: *const u8,
|
||||
input_len: usize,
|
||||
remove_bom: bool,
|
||||
) -> bool {
|
||||
unsafe {
|
||||
abort_on_panic(|| {
|
||||
let Some(label) = bytes_from_raw(encoding_label, encoding_label_len) else {
|
||||
return false;
|
||||
};
|
||||
let Some(input) = bytes_from_raw(input, input_len) else {
|
||||
return false;
|
||||
};
|
||||
let Some(encoding) = Encoding::for_label(label) else {
|
||||
return false;
|
||||
};
|
||||
|
||||
let input = if remove_bom {
|
||||
if encoding == encoding_rs::UTF_8 && input.starts_with(b"\xEF\xBB\xBF") {
|
||||
&input[3..]
|
||||
} else if (encoding == encoding_rs::UTF_16LE && input.starts_with(b"\xFF\xFE"))
|
||||
|| (encoding == encoding_rs::UTF_16BE && input.starts_with(b"\xFE\xFF"))
|
||||
{
|
||||
&input[2..]
|
||||
} else {
|
||||
input
|
||||
}
|
||||
} else {
|
||||
input
|
||||
};
|
||||
|
||||
encoding
|
||||
.decode_without_bom_handling_and_without_replacement(input)
|
||||
.is_some()
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// # Safety
|
||||
/// - `encoding_label`/`encoding_label_len` must be a valid byte slice.
|
||||
/// - The returned pointer must be freed with `textcodec_rust_streaming_decoder_free`.
|
||||
|
|
@ -212,6 +173,7 @@ pub unsafe extern "C" fn textcodec_rust_streaming_decoder_decode_to_utf8(
|
|||
input: *const u8,
|
||||
input_len: usize,
|
||||
last: bool,
|
||||
fatal: bool,
|
||||
ctx: *mut c_void,
|
||||
on_bytes: FfiBytesFn,
|
||||
) -> bool {
|
||||
|
|
@ -230,7 +192,10 @@ pub unsafe extern "C" fn textcodec_rust_streaming_decoder_decode_to_utf8(
|
|||
};
|
||||
let mut output = String::with_capacity(output_capacity);
|
||||
|
||||
let (result, _, _) = decoder.decoder.decode_to_string(input, &mut output, last);
|
||||
let (result, _, had_errors) = decoder.decoder.decode_to_string(input, &mut output, last);
|
||||
if fatal && had_errors {
|
||||
return false;
|
||||
}
|
||||
if !output.is_empty() {
|
||||
on_bytes(ctx, output.as_ptr(), output.len());
|
||||
}
|
||||
|
|
|
|||
|
|
@ -36,7 +36,7 @@ static String decode_and_filter_code_points(StringView input, StringView encodin
|
|||
input = input.substring_view(3);
|
||||
return String::from_utf8_without_validation(input.bytes());
|
||||
}
|
||||
return MUST(decoder->to_utf8(input));
|
||||
return MUST(decoder->to_utf8(input, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement));
|
||||
}();
|
||||
|
||||
// OPTIMIZATION: If the input doesn't contain any filterable characters, we can skip the filtering
|
||||
|
|
|
|||
|
|
@ -175,7 +175,7 @@ Vector<Token> Tokenizer::tokenize(StringView input, StringView encoding, Tokeniz
|
|||
input = input.substring_view(3);
|
||||
return String::from_utf8_without_validation(input.bytes());
|
||||
}
|
||||
return MUST(decoder->to_utf8(input));
|
||||
return MUST(decoder->to_utf8(input, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement));
|
||||
}();
|
||||
|
||||
// OPTIMIZATION: If the input doesn't contain any filterable characters, we can skip the filtering
|
||||
|
|
|
|||
|
|
@ -73,11 +73,12 @@ bool build_xml_document(DOM::Document& document, ByteBuffer const& data, Optiona
|
|||
}
|
||||
VERIFY(decoder.has_value());
|
||||
// Well-formed XML documents contain only properly encoded characters
|
||||
if (!decoder->validate(data)) {
|
||||
auto source_or_error = decoder->to_utf8(data, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Fatal);
|
||||
if (source_or_error.is_error()) {
|
||||
convert_to_xml_error_document(document, "XML Document contains improperly-encoded characters"_utf16);
|
||||
return false;
|
||||
}
|
||||
auto source = decoder->to_utf8(data).release_value_but_fixme_should_propagate_errors();
|
||||
auto source = source_or_error.release_value();
|
||||
XML::Parser parser(source, { .resolve_named_html_entity = resolve_named_html_entity });
|
||||
XMLDocumentBuilder builder { document };
|
||||
auto result = parser.parse_with_listener(builder);
|
||||
|
|
@ -198,13 +199,7 @@ static WebIDL::ExceptionOr<GC::Ref<DOM::Document>> load_xml_document(HTML::Navig
|
|||
}
|
||||
VERIFY(decoder.has_value());
|
||||
// Well-formed XML documents contain only properly encoded characters
|
||||
if (!decoder->validate(data)) {
|
||||
// FIXME: Insert error message into the document.
|
||||
dbgln("XML Document contains improperly-encoded characters");
|
||||
convert_to_xml_error_document(document, "XML Document contains improperly-encoded characters"_utf16);
|
||||
return;
|
||||
}
|
||||
auto source = decoder->to_utf8(data);
|
||||
auto source = decoder->to_utf8(data, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Fatal);
|
||||
if (source.is_error()) {
|
||||
// FIXME: Insert error message into the document.
|
||||
dbgln("Failed to decode XML document: {}", source.error());
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ namespace Web::Encoding {
|
|||
GC_DEFINE_ALLOCATOR(TextDecoder);
|
||||
|
||||
// https://encoding.spec.whatwg.org/#dom-textdecoder
|
||||
WebIDL::ExceptionOr<GC::Ref<TextDecoder>> TextDecoder::construct_impl(JS::Realm& realm, FlyString label, Optional<Bindings::TextDecoderOptions> const& options)
|
||||
WebIDL::ExceptionOr<GC::Ref<TextDecoder>> TextDecoder::construct_impl(JS::Realm& realm, StringView label, Bindings::TextDecoderOptions const& options)
|
||||
{
|
||||
auto& vm = realm.vm();
|
||||
|
||||
|
|
@ -35,10 +35,10 @@ WebIDL::ExceptionOr<GC::Ref<TextDecoder>> TextDecoder::construct_impl(JS::Realm&
|
|||
auto lowercase_encoding_name = encoding.value().to_ascii_lowercase_string();
|
||||
|
||||
// 4. If options["fatal"] is true, then set this’s error mode to "fatal".
|
||||
auto error_mode = options.value_or({}).fatal ? ErrorMode::Fatal : ErrorMode::Replacement;
|
||||
auto error_mode = options.fatal ? TextCodec::ErrorMode::Fatal : TextCodec::ErrorMode::Replacement;
|
||||
|
||||
// 5. Set this’s ignore BOM to options["ignoreBOM"].
|
||||
auto ignore_bom = options.value_or({}).ignore_bom;
|
||||
auto ignore_bom = options.ignore_bom;
|
||||
|
||||
// NOTE: This should happen in decode(), but we don't support streaming yet and share decoders across calls.
|
||||
auto decoder = TextCodec::decoder_for_exact_name(encoding.value());
|
||||
|
|
@ -48,7 +48,7 @@ WebIDL::ExceptionOr<GC::Ref<TextDecoder>> TextDecoder::construct_impl(JS::Realm&
|
|||
}
|
||||
|
||||
// https://encoding.spec.whatwg.org/#dom-textdecoder
|
||||
TextDecoder::TextDecoder(JS::Realm& realm, TextCodec::Decoder& decoder, FlyString encoding, ErrorMode error_mode, bool ignore_bom)
|
||||
TextDecoder::TextDecoder(JS::Realm& realm, TextCodec::Decoder& decoder, FlyString encoding, TextCodec::ErrorMode error_mode, bool ignore_bom)
|
||||
: PlatformObject(realm)
|
||||
, TextDecoderCommonMixin(decoder, move(encoding), error_mode, ignore_bom)
|
||||
{
|
||||
|
|
@ -65,18 +65,23 @@ void TextDecoder::initialize(JS::Realm& realm)
|
|||
// https://encoding.spec.whatwg.org/#dom-textdecoder-decode
|
||||
WebIDL::ExceptionOr<String> TextDecoder::decode(Optional<WebIDL::BufferSourceVariant> input, Optional<Bindings::TextDecodeOptions> const&) const
|
||||
{
|
||||
if (!input.has_value())
|
||||
return TRY_OR_THROW_OOM(vm(), m_decoder.to_utf8({}));
|
||||
auto ignore_bom = m_ignore_bom ? TextCodec::IgnoreBOM::Yes : TextCodec::IgnoreBOM::No;
|
||||
if (!input.has_value()) {
|
||||
auto result = m_decoder.to_utf8({}, ignore_bom, m_error_mode);
|
||||
if (result.is_error() && result.error().code() != ENOMEM)
|
||||
return WebIDL::SimpleException { WebIDL::SimpleExceptionType::TypeError, "Decoding failed"sv };
|
||||
return TRY_OR_THROW_OOM(vm(), move(result));
|
||||
}
|
||||
|
||||
// FIXME: Implement the streaming stuff.
|
||||
auto data_buffer_or_error = WebIDL::get_buffer_source_copy(*input);
|
||||
if (data_buffer_or_error.is_error())
|
||||
return WebIDL::OperationError::create(realm(), "Failed to copy bytes from ArrayBuffer"_utf16);
|
||||
auto& data_buffer = data_buffer_or_error.value();
|
||||
auto result = TRY_OR_THROW_OOM(vm(), m_decoder.to_utf8({ data_buffer.data(), data_buffer.size() }));
|
||||
if (this->fatal() && result.contains(0xfffd))
|
||||
auto result = m_decoder.to_utf8({ data_buffer.data(), data_buffer.size() }, ignore_bom, m_error_mode);
|
||||
if (result.is_error() && result.error().code() != ENOMEM)
|
||||
return WebIDL::SimpleException { WebIDL::SimpleExceptionType::TypeError, "Decoding failed"sv };
|
||||
return result;
|
||||
return TRY_OR_THROW_OOM(vm(), move(result));
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -27,14 +27,14 @@ class TextDecoder
|
|||
GC_DECLARE_ALLOCATOR(TextDecoder);
|
||||
|
||||
public:
|
||||
static WebIDL::ExceptionOr<GC::Ref<TextDecoder>> construct_impl(JS::Realm&, FlyString encoding, Optional<Bindings::TextDecoderOptions> const& options = {});
|
||||
static WebIDL::ExceptionOr<GC::Ref<TextDecoder>> construct_impl(JS::Realm&, StringView label, Bindings::TextDecoderOptions const&);
|
||||
|
||||
virtual ~TextDecoder() override;
|
||||
|
||||
WebIDL::ExceptionOr<String> decode(Optional<WebIDL::BufferSourceVariant>, Optional<Bindings::TextDecodeOptions> const& options = {}) const;
|
||||
|
||||
private:
|
||||
TextDecoder(JS::Realm&, TextCodec::Decoder&, FlyString encoding, ErrorMode error_mode, bool ignore_bom);
|
||||
TextDecoder(JS::Realm&, TextCodec::Decoder&, FlyString encoding, TextCodec::ErrorMode error_mode, bool ignore_bom);
|
||||
|
||||
virtual void initialize(JS::Realm&) override;
|
||||
};
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@
|
|||
|
||||
namespace Web::Encoding {
|
||||
|
||||
TextDecoderCommonMixin::TextDecoderCommonMixin(TextCodec::Decoder& decoder, FlyString encoding, ErrorMode error_mode, bool ignore_bom)
|
||||
TextDecoderCommonMixin::TextDecoderCommonMixin(TextCodec::Decoder& decoder, FlyString encoding, TextCodec::ErrorMode error_mode, bool ignore_bom)
|
||||
: m_decoder(decoder)
|
||||
, m_encoding(move(encoding))
|
||||
, m_error_mode(error_mode)
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@
|
|||
#pragma once
|
||||
|
||||
#include <AK/FlyString.h>
|
||||
#include <LibTextCodec/Forward.h>
|
||||
#include <LibTextCodec/Decoder.h>
|
||||
|
||||
namespace Web::Encoding {
|
||||
|
||||
|
|
@ -20,19 +20,13 @@ public:
|
|||
FlyString const& encoding() const { return m_encoding; }
|
||||
|
||||
// https://encoding.spec.whatwg.org/#dom-textdecoder-fatal
|
||||
bool fatal() const { return m_error_mode == ErrorMode::Fatal; }
|
||||
bool fatal() const { return m_error_mode == TextCodec::ErrorMode::Fatal; }
|
||||
|
||||
// https://encoding.spec.whatwg.org/#dom-textdecoder-ignorebom
|
||||
bool ignore_bom() const { return m_ignore_bom; }
|
||||
|
||||
protected:
|
||||
// https://encoding.spec.whatwg.org/#concept-encoding-error-mode
|
||||
enum class ErrorMode {
|
||||
Replacement,
|
||||
Fatal,
|
||||
};
|
||||
|
||||
TextDecoderCommonMixin(TextCodec::Decoder& decoder, FlyString encoding, ErrorMode error_mode, bool ignore_bom);
|
||||
TextDecoderCommonMixin(TextCodec::Decoder& decoder, FlyString encoding, TextCodec::ErrorMode error_mode, bool ignore_bom);
|
||||
|
||||
// https://encoding.spec.whatwg.org/#textdecodercommon-decoder
|
||||
TextCodec::Decoder& m_decoder;
|
||||
|
|
@ -41,13 +35,10 @@ protected:
|
|||
FlyString m_encoding;
|
||||
|
||||
// https://encoding.spec.whatwg.org/#textdecoder-error-mode
|
||||
ErrorMode m_error_mode { ErrorMode::Replacement };
|
||||
TextCodec::ErrorMode m_error_mode { TextCodec::ErrorMode::Replacement };
|
||||
|
||||
// https://encoding.spec.whatwg.org/#textdecoder-ignore-bom-flag
|
||||
bool m_ignore_bom { false };
|
||||
|
||||
// https://encoding.spec.whatwg.org/#textdecoder-bom-seen-flag
|
||||
bool m_bom_seen { false };
|
||||
};
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@
|
|||
#include <LibWeb/Bindings/TextDecoder.h>
|
||||
#include <LibWeb/Bindings/TextDecoderStream.h>
|
||||
#include <LibWeb/Encoding/TextDecoderStream.h>
|
||||
#include <LibWeb/HTML/Scripting/TemporaryExecutionContext.h>
|
||||
#include <LibWeb/Streams/TransformStream.h>
|
||||
#include <LibWeb/Streams/TransformStreamOperations.h>
|
||||
#include <LibWeb/WebIDL/AbstractOperations.h>
|
||||
|
|
@ -38,7 +39,7 @@ WebIDL::ExceptionOr<GC::Ref<TextDecoderStream>> TextDecoderStream::construct_imp
|
|||
auto lowercase_encoding_name = encoding.value().to_ascii_lowercase_string();
|
||||
|
||||
// 4. If options["fatal"] is true, then set this’s error mode to "fatal".
|
||||
auto error_mode = options.fatal ? ErrorMode::Fatal : ErrorMode::Replacement;
|
||||
auto error_mode = options.fatal ? TextCodec::ErrorMode::Fatal : TextCodec::ErrorMode::Replacement;
|
||||
|
||||
// 5. Set this’s ignore BOM to options["ignoreBOM"].
|
||||
auto ignore_bom = options.ignore_bom;
|
||||
|
|
@ -59,6 +60,7 @@ WebIDL::ExceptionOr<GC::Ref<TextDecoderStream>> TextDecoderStream::construct_imp
|
|||
// algorithm with this and chunk.
|
||||
auto transform_algorithm = GC::create_function(realm.heap(), [stream](JS::Value chunk) -> GC::Ref<WebIDL::Promise> {
|
||||
auto& realm = stream->realm();
|
||||
HTML::TemporaryExecutionContext execution_context { realm };
|
||||
if (auto result = stream->decode_and_enqueue_chunk(chunk); result.is_error())
|
||||
return WebIDL::create_rejected_promise_from_exception(realm, result.release_error());
|
||||
return WebIDL::create_resolved_promise(realm, JS::js_undefined());
|
||||
|
|
@ -67,6 +69,7 @@ WebIDL::ExceptionOr<GC::Ref<TextDecoderStream>> TextDecoderStream::construct_imp
|
|||
// 8. Let flushAlgorithm be an algorithm which takes no arguments and runs the flush and enqueue algorithm with this.
|
||||
auto flush_algorithm = GC::create_function(realm.heap(), [stream]() -> GC::Ref<WebIDL::Promise> {
|
||||
auto& realm = stream->realm();
|
||||
HTML::TemporaryExecutionContext execution_context { realm };
|
||||
if (auto result = stream->flush_and_enqueue(); result.is_error())
|
||||
return WebIDL::create_rejected_promise_from_exception(realm, result.release_error());
|
||||
return WebIDL::create_resolved_promise(realm, JS::js_undefined());
|
||||
|
|
@ -81,11 +84,14 @@ WebIDL::ExceptionOr<GC::Ref<TextDecoderStream>> TextDecoderStream::construct_imp
|
|||
return stream;
|
||||
}
|
||||
|
||||
TextDecoderStream::TextDecoderStream(JS::Realm& realm, GC::Ref<Streams::TransformStream> transform, TextCodec::Decoder& decoder, FlyString encoding, ErrorMode error_mode, bool ignore_bom)
|
||||
TextDecoderStream::TextDecoderStream(JS::Realm& realm, GC::Ref<Streams::TransformStream> transform, TextCodec::Decoder& decoder, FlyString encoding, TextCodec::ErrorMode error_mode, bool ignore_bom)
|
||||
: Bindings::PlatformObject(realm)
|
||||
, Streams::GenericTransformStreamMixin(transform)
|
||||
, TextDecoderCommonMixin(decoder, move(encoding), error_mode, ignore_bom)
|
||||
, m_streaming_decoder(make<TextCodec::StreamingDecoder>(m_encoding))
|
||||
, m_streaming_decoder(make<TextCodec::StreamingDecoder>(
|
||||
m_encoding,
|
||||
m_ignore_bom ? TextCodec::IgnoreBOM::Yes : TextCodec::IgnoreBOM::No,
|
||||
m_error_mode))
|
||||
{
|
||||
}
|
||||
|
||||
|
|
@ -119,7 +125,10 @@ WebIDL::ExceptionOr<void> TextDecoderStream::decode_and_enqueue_chunk(JS::Value
|
|||
return WebIDL::OperationError::create(realm, "Failed to copy bytes from BufferSource"_utf16);
|
||||
auto buffer = buffer_or_error.release_value();
|
||||
|
||||
auto decoded = TRY_OR_THROW_OOM(vm, m_streaming_decoder->to_utf8(buffer.bytes()));
|
||||
auto decoded_or_error = m_streaming_decoder->to_utf8(buffer.bytes());
|
||||
if (decoded_or_error.is_error() && decoded_or_error.error().code() != ENOMEM)
|
||||
return WebIDL::SimpleException { WebIDL::SimpleExceptionType::TypeError, "Decoding failed"sv };
|
||||
auto decoded = TRY_OR_THROW_OOM(vm, move(decoded_or_error));
|
||||
|
||||
// 3-4. Run "processing an item" until the input is exhausted, accumulating the output, then enqueue any non-empty
|
||||
// result. If processing returns error, throw a TypeError.
|
||||
|
|
@ -130,7 +139,10 @@ WebIDL::ExceptionOr<void> TextDecoderStream::decode_and_enqueue_chunk(JS::Value
|
|||
WebIDL::ExceptionOr<void> TextDecoderStream::flush_and_enqueue()
|
||||
{
|
||||
// 1-3. Drain decoder's I/O queue and run "processing an item" to completion.
|
||||
auto decoded = TRY_OR_THROW_OOM(vm(), m_streaming_decoder->finish());
|
||||
auto decoded_or_error = m_streaming_decoder->finish();
|
||||
if (decoded_or_error.is_error() && decoded_or_error.error().code() != ENOMEM)
|
||||
return WebIDL::SimpleException { WebIDL::SimpleExceptionType::TypeError, "Decoding failed"sv };
|
||||
auto decoded = TRY_OR_THROW_OOM(vm(), move(decoded_or_error));
|
||||
return enqueue_decoded_output(decoded);
|
||||
}
|
||||
|
||||
|
|
@ -139,22 +151,9 @@ WebIDL::ExceptionOr<void> TextDecoderStream::enqueue_decoded_output(String const
|
|||
auto& realm = this->realm();
|
||||
auto& vm = realm.vm();
|
||||
|
||||
// https://encoding.spec.whatwg.org/#concept-td-serialize
|
||||
// FIXME: The underlying TextCodec decoders currently strip leading BOMs unconditionally for UTF-8 and UTF-16BE/LE,
|
||||
// so the "ignore BOM" flag is effectively ignored here. Once the decoders accept a "preserve BOM" mode,
|
||||
// plumb m_ignore_bom through and strip the BOM from `decoded` only when m_ignore_bom is false.
|
||||
if (!m_bom_seen && !decoded.is_empty())
|
||||
m_bom_seen = true;
|
||||
|
||||
if (decoded.is_empty())
|
||||
return {};
|
||||
|
||||
// If decoder's error mode is "fatal" and processing produced any error, throw a TypeError.
|
||||
// NB: We can only detect this approximately by looking for U+FFFD in the decoded output, which the underlying
|
||||
// decoder substitutes for invalid sequences. This matches the existing TextDecoder.decode() behavior.
|
||||
if (fatal() && decoded.contains(0xFFFD))
|
||||
return WebIDL::SimpleException { WebIDL::SimpleExceptionType::TypeError, "Decoding failed"sv };
|
||||
|
||||
auto js_string = JS::PrimitiveString::create(vm, Utf16String::from_utf8(decoded));
|
||||
return Streams::transform_stream_default_controller_enqueue(*m_transform->controller(), js_string);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -27,7 +27,7 @@ public:
|
|||
virtual ~TextDecoderStream() override;
|
||||
|
||||
private:
|
||||
TextDecoderStream(JS::Realm&, GC::Ref<Streams::TransformStream>, TextCodec::Decoder&, FlyString encoding, ErrorMode, bool ignore_bom);
|
||||
TextDecoderStream(JS::Realm&, GC::Ref<Streams::TransformStream>, TextCodec::Decoder&, FlyString encoding, TextCodec::ErrorMode, bool ignore_bom);
|
||||
|
||||
virtual void initialize(JS::Realm&) override;
|
||||
virtual void visit_edges(Cell::Visitor&) override;
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ namespace Web::Encoding {
|
|||
|
||||
GC_DEFINE_ALLOCATOR(TextEncoder);
|
||||
|
||||
WebIDL::ExceptionOr<GC::Ref<TextEncoder>> TextEncoder::construct_impl(JS::Realm& realm)
|
||||
GC::Ref<TextEncoder> TextEncoder::construct_impl(JS::Realm& realm)
|
||||
{
|
||||
return realm.create<TextEncoder>(realm);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -27,7 +27,7 @@ class TextEncoder final
|
|||
GC_DECLARE_ALLOCATOR(TextEncoder);
|
||||
|
||||
public:
|
||||
static WebIDL::ExceptionOr<GC::Ref<TextEncoder>> construct_impl(JS::Realm&);
|
||||
static GC::Ref<TextEncoder> construct_impl(JS::Realm&);
|
||||
|
||||
virtual ~TextEncoder() override;
|
||||
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@
|
|||
#include <LibWeb/Bindings/Intrinsics.h>
|
||||
#include <LibWeb/Bindings/TextEncoderStream.h>
|
||||
#include <LibWeb/Encoding/TextEncoderStream.h>
|
||||
#include <LibWeb/HTML/Scripting/TemporaryExecutionContext.h>
|
||||
#include <LibWeb/Streams/TransformStream.h>
|
||||
#include <LibWeb/Streams/TransformStreamOperations.h>
|
||||
#include <LibWeb/WebIDL/Promise.h>
|
||||
|
|
@ -21,7 +22,7 @@ namespace Web::Encoding {
|
|||
GC_DEFINE_ALLOCATOR(TextEncoderStream);
|
||||
|
||||
// https://encoding.spec.whatwg.org/#dom-textencoderstream
|
||||
WebIDL::ExceptionOr<GC::Ref<TextEncoderStream>> TextEncoderStream::construct_impl(JS::Realm& realm)
|
||||
GC::Ref<TextEncoderStream> TextEncoderStream::construct_impl(JS::Realm& realm)
|
||||
{
|
||||
// 1. Set this’s encoder to an instance of the UTF-8 encoder.
|
||||
// NOTE: No-op, as AK::String is already in UTF-8 format.
|
||||
|
|
@ -38,6 +39,7 @@ WebIDL::ExceptionOr<GC::Ref<TextEncoderStream>> TextEncoderStream::construct_imp
|
|||
auto transform_algorithm = GC::create_function(realm.heap(), [stream](JS::Value chunk) -> GC::Ref<WebIDL::Promise> {
|
||||
auto& realm = stream->realm();
|
||||
auto& vm = realm.vm();
|
||||
HTML::TemporaryExecutionContext execution_context { realm };
|
||||
|
||||
if (auto result = stream->encode_and_enqueue_chunk(chunk); result.is_error()) {
|
||||
auto throw_completion = Bindings::exception_to_throw_completion(vm, result.exception());
|
||||
|
|
@ -51,6 +53,7 @@ WebIDL::ExceptionOr<GC::Ref<TextEncoderStream>> TextEncoderStream::construct_imp
|
|||
auto flush_algorithm = GC::create_function(realm.heap(), [stream]() -> GC::Ref<WebIDL::Promise> {
|
||||
auto& realm = stream->realm();
|
||||
auto& vm = realm.vm();
|
||||
HTML::TemporaryExecutionContext execution_context { realm };
|
||||
|
||||
if (auto result = stream->encode_and_flush(); result.is_error()) {
|
||||
auto throw_completion = Bindings::exception_to_throw_completion(vm, result.exception());
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ class TextEncoderStream final
|
|||
GC_DECLARE_ALLOCATOR(TextEncoderStream);
|
||||
|
||||
public:
|
||||
static WebIDL::ExceptionOr<GC::Ref<TextEncoderStream>> construct_impl(JS::Realm&);
|
||||
static GC::Ref<TextEncoderStream> construct_impl(JS::Realm&);
|
||||
virtual ~TextEncoderStream() override;
|
||||
|
||||
private:
|
||||
|
|
|
|||
|
|
@ -108,7 +108,7 @@ HTMLTokenizer::HTMLTokenizer(StringView input, ByteString const& encoding, Input
|
|||
if (input_type == InputType::EncodedBytes) {
|
||||
auto decoder = TextCodec::decoder_for(encoding);
|
||||
VERIFY(decoder.has_value());
|
||||
m_source = MUST(decoder->to_utf8(input));
|
||||
m_source = MUST(decoder->to_utf8(input, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement));
|
||||
} else {
|
||||
m_source = decoded_string_for_utf8_tokenizer(input);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -68,7 +68,7 @@ void IncrementalDocumentParser::initialize_parser(ReadonlyBytes sniff_bytes)
|
|||
|
||||
auto standardized_encoding = TextCodec::get_standardized_encoding(encoding);
|
||||
VERIFY(standardized_encoding.has_value());
|
||||
m_decoder = make<TextCodec::StreamingDecoder>(standardized_encoding.value());
|
||||
m_decoder = make<TextCodec::StreamingDecoder>(standardized_encoding.value(), TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement);
|
||||
|
||||
// https://html.spec.whatwg.org/multipage/parsing.html#determining-the-character-encoding
|
||||
// The document's character encoding must immediately be set to the value returned from this
|
||||
|
|
|
|||
|
|
@ -54,7 +54,7 @@ void Instance::initialize(JS::Realm& realm)
|
|||
[&](Wasm::FunctionAddress const& address) {
|
||||
Optional<GC::Ptr<JS::FunctionObject>> object = m_function_instances.get(address);
|
||||
if (!object.has_value()) {
|
||||
object = Detail::create_native_function(vm, address, name, this);
|
||||
object = Detail::create_native_function(vm, address, this);
|
||||
m_function_instances.set(address, *object);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -289,6 +289,7 @@ JS::ThrowCompletionOr<NonnullRefPtr<Wasm::ModuleInstance>> instantiate_module(JS
|
|||
|
||||
// 2. Let imports be « ».
|
||||
HashMap<Wasm::Linker::Name, Wasm::ExternValue> resolved_imports;
|
||||
size_t imported_function_count = 0;
|
||||
if (import_object) {
|
||||
dbgln_if(LIBWEB_WASM_DEBUG, "Trying to resolve stuff because import object was specified");
|
||||
|
||||
|
|
@ -330,7 +331,7 @@ JS::ThrowCompletionOr<NonnullRefPtr<Wasm::ModuleInstance>> instantiate_module(JS
|
|||
else {
|
||||
// 3.4.3.1. Create a host function from v and functype, and let funcaddr be the result.
|
||||
cache.add_imported_object(function);
|
||||
auto host_function = create_host_function(vm, function, function_type, ByteString::formatted("func{}", resolved_imports.size()));
|
||||
auto host_function = create_host_function(vm, function, function_type, ByteString::number(imported_function_count));
|
||||
address = cache.abstract_machine().store().allocate(move(host_function));
|
||||
// FIXME: 3.4.3.2. Let index be the number of external functions in imports. This value index is known as the index of the host function funcaddr.
|
||||
// 'index' doesn't seem to be used anywhere?
|
||||
|
|
@ -342,6 +343,7 @@ JS::ThrowCompletionOr<NonnullRefPtr<Wasm::ModuleInstance>> instantiate_module(JS
|
|||
// 3.4.4. Let externfunc be the external value func funcaddr.
|
||||
// 3.4.5. Append externfunc to imports.
|
||||
resolved_imports.set(import_name, Wasm::ExternValue { Wasm::FunctionAddress { *address } });
|
||||
++imported_function_count;
|
||||
return {};
|
||||
},
|
||||
// 3.5. If externtype is of the form global mut valtype,
|
||||
|
|
@ -647,14 +649,20 @@ JS::ThrowCompletionOr<JS::HandledByHost> host_grow_shared_array_buffer(JS::VM& v
|
|||
|
||||
GC_DEFINE_ALLOCATOR(ExportedWasmFunction);
|
||||
|
||||
GC::Ref<ExportedWasmFunction> ExportedWasmFunction::create(JS::Realm& realm, Utf16FlyString name, Function<JS::ThrowCompletionOr<JS::Value>(JS::VM&)> behavior, Wasm::FunctionAddress exported_address)
|
||||
GC::Ref<ExportedWasmFunction> ExportedWasmFunction::create(JS::Realm& realm, Utf16FlyString name, size_t length, Function<JS::ThrowCompletionOr<JS::Value>(JS::VM&)> behavior, Wasm::FunctionAddress exported_address)
|
||||
{
|
||||
auto& vm = realm.vm();
|
||||
|
||||
auto prototype = realm.intrinsics().function_prototype();
|
||||
return realm.create<ExportedWasmFunction>(
|
||||
auto function = realm.create<ExportedWasmFunction>(
|
||||
move(name),
|
||||
move(behavior),
|
||||
exported_address,
|
||||
prototype);
|
||||
function->define_direct_property(vm.names.length, JS::Value { static_cast<double>(length) }, JS::Attribute::Configurable);
|
||||
function->define_direct_property(vm.names.name, JS::PrimitiveString::create(vm, function->name()), JS::Attribute::Configurable);
|
||||
|
||||
return function;
|
||||
}
|
||||
|
||||
ExportedWasmFunction::ExportedWasmFunction(Utf16FlyString name, AK::Function<JS::ThrowCompletionOr<JS::Value>(JS::VM&)> behavior, Wasm::FunctionAddress exported_address, JS::Object& prototype)
|
||||
|
|
@ -676,19 +684,57 @@ JS::ThrowCompletionOr<JS::Value> ExportedWasmFunction::call()
|
|||
return m_behavior(vm());
|
||||
}
|
||||
|
||||
JS::NativeFunction* create_native_function(JS::VM& vm, Wasm::FunctionAddress address, Utf16FlyString name, Instance* instance)
|
||||
// https://www.w3.org/TR/wasm-js-api-2/#name-of-the-webassembly-function
|
||||
static Utf16FlyString name_of_webassembly_function(Wasm::Store& store, Wasm::FunctionAddress function_address)
|
||||
{
|
||||
// 1. Let store be the surrounding agent’s associated store.
|
||||
|
||||
// 2. Let funcinst be store.funcs[funcaddr].
|
||||
auto* function_instance = store.get(function_address);
|
||||
VERIFY(function_instance);
|
||||
|
||||
auto index = function_instance->visit(
|
||||
// 3. If funcinst is of the form {type functype, hostcode hostfunc},
|
||||
[&](Wasm::HostFunction const& host_function) {
|
||||
// 1. Assert: hostfunc is a JavaScript object and IsCallable(hostfunc) is true.
|
||||
// 2. Let index be the index of the host function funcaddr.
|
||||
auto index = host_function.name().to_number<size_t>(TrimWhitespace::No);
|
||||
VERIFY(index.has_value());
|
||||
return *index;
|
||||
},
|
||||
// 4. Otherwise,
|
||||
[&](Wasm::WasmFunction const& wasm_function) {
|
||||
// 1. Let moduleinst be funcinst.module.
|
||||
auto const& module_instance = wasm_function.module();
|
||||
|
||||
// 2. Assert: funcaddr is contained in moduleinst.funcaddrs.
|
||||
// 3. Let index be the index of moduleinst.funcaddrs where funcaddr is found.
|
||||
auto index = module_instance.functions().find_first_index(function_address);
|
||||
VERIFY(index.has_value());
|
||||
return *index;
|
||||
});
|
||||
|
||||
// 5. Return ! ToString(index).
|
||||
return Utf16String::number(index);
|
||||
}
|
||||
|
||||
JS::NativeFunction* create_native_function(JS::VM& vm, Wasm::FunctionAddress address, Instance* instance)
|
||||
{
|
||||
auto& realm = *vm.current_realm();
|
||||
Optional<Wasm::FunctionType> type;
|
||||
auto& cache = get_cache(realm);
|
||||
cache.abstract_machine().store().get(address)->visit([&](auto const& value) { type = value.type(); });
|
||||
|
||||
if (auto entry = cache.get_function_instance(address); entry.has_value())
|
||||
return *entry;
|
||||
|
||||
auto& store = cache.abstract_machine().store();
|
||||
auto type = store.get(address)->visit([&](auto const& value) { return value.type(); });
|
||||
auto length = type.parameters().size();
|
||||
|
||||
auto function = ExportedWasmFunction::create(
|
||||
realm,
|
||||
move(name),
|
||||
[address, type = type.release_value(), instance](JS::VM& vm) -> JS::ThrowCompletionOr<JS::Value> {
|
||||
name_of_webassembly_function(store, address),
|
||||
length,
|
||||
[address, type = move(type), instance](JS::VM& vm) -> JS::ThrowCompletionOr<JS::Value> {
|
||||
(void)instance;
|
||||
auto& realm = *vm.current_realm();
|
||||
Vector<Wasm::Value> values;
|
||||
|
|
@ -861,17 +907,7 @@ JS::Value to_js_value(JS::VM& vm, Wasm::Value& wasm_value, Wasm::ValueType type)
|
|||
if (ref_.ref().has<Wasm::Reference::Null>())
|
||||
return JS::js_null();
|
||||
auto address = ref_.ref().get<Wasm::Reference::Func>().address;
|
||||
auto& cache = get_cache(realm);
|
||||
auto* function = cache.abstract_machine().store().get(address);
|
||||
auto name = function->visit(
|
||||
[&](Wasm::WasmFunction& wasm_function) {
|
||||
auto index = *wasm_function.module().functions().find_first_index(address);
|
||||
return ByteString::formatted("func{}", index);
|
||||
},
|
||||
[](Wasm::HostFunction& host_function) {
|
||||
return host_function.name();
|
||||
});
|
||||
return create_native_function(vm, address, Utf16FlyString::from_utf8(name));
|
||||
return create_native_function(vm, address);
|
||||
}
|
||||
case Wasm::ValueType::ExternReference: {
|
||||
auto ref_ = wasm_value.to<Wasm::Reference>();
|
||||
|
|
|
|||
|
|
@ -101,7 +101,7 @@ class ExportedWasmFunction final : public JS::NativeFunction {
|
|||
GC_DECLARE_ALLOCATOR(ExportedWasmFunction);
|
||||
|
||||
public:
|
||||
static GC::Ref<ExportedWasmFunction> create(JS::Realm&, Utf16FlyString name, ESCAPING Function<JS::ThrowCompletionOr<JS::Value>(JS::VM&)>, Wasm::FunctionAddress);
|
||||
static GC::Ref<ExportedWasmFunction> create(JS::Realm&, Utf16FlyString name, size_t length, ESCAPING Function<JS::ThrowCompletionOr<JS::Value>(JS::VM&)>, Wasm::FunctionAddress);
|
||||
virtual ~ExportedWasmFunction() override = default;
|
||||
|
||||
Wasm::FunctionAddress exported_address() const { return m_exported_address; }
|
||||
|
|
@ -122,7 +122,7 @@ WebAssemblyCache& get_cache(JS::Realm&);
|
|||
|
||||
JS::ThrowCompletionOr<NonnullRefPtr<Wasm::ModuleInstance>> instantiate_module(JS::VM&, Wasm::Module const&, GC::Ptr<JS::Object> import_object);
|
||||
JS::ThrowCompletionOr<NonnullRefPtr<CompiledWebAssemblyModule>> compile_a_webassembly_module(JS::VM&, ByteBuffer);
|
||||
JS::NativeFunction* create_native_function(JS::VM&, Wasm::FunctionAddress address, Utf16FlyString name, Instance* instance = nullptr);
|
||||
JS::NativeFunction* create_native_function(JS::VM&, Wasm::FunctionAddress address, Instance* instance = nullptr);
|
||||
JS::ThrowCompletionOr<Wasm::Value> to_webassembly_value(JS::VM&, JS::Value value, Wasm::ValueType const& type);
|
||||
Wasm::Value default_webassembly_value(JS::VM&, Wasm::ValueType type);
|
||||
JS::Value to_js_value(JS::VM&, Wasm::Value& wasm_value, Wasm::ValueType type);
|
||||
|
|
|
|||
|
|
@ -92,7 +92,10 @@ static xmlNodePtr mirror_node(xmlDocPtr doc, DOM::Node const& node)
|
|||
}
|
||||
case DOM::NodeType::DOCUMENT_NODE: {
|
||||
auto const& document = static_cast<DOM::Document const&>(node);
|
||||
return mirror_node(doc, *document.document_element());
|
||||
auto const* document_element = document.document_element();
|
||||
if (!document_element)
|
||||
return nullptr;
|
||||
return mirror_node(doc, *document_element);
|
||||
}
|
||||
case DOM::NodeType::DOCUMENT_TYPE_NODE: {
|
||||
return nullptr; // Unused in libxml2
|
||||
|
|
|
|||
|
|
@ -438,7 +438,7 @@ ErrorOr<Vector<String>> Autocomplete::received_autocomplete_respsonse(Autocomple
|
|||
if (!decoder.has_value())
|
||||
decoder = TextCodec::decoder_for_exact_name("UTF-8"sv);
|
||||
|
||||
auto decoded_response = TRY(decoder->to_utf8(response));
|
||||
auto decoded_response = TRY(decoder->to_utf8(response, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement));
|
||||
auto json = TRY(JsonValue::from_string(decoded_response));
|
||||
|
||||
if (engine.name == "DuckDuckGo")
|
||||
|
|
|
|||
|
|
@ -24,6 +24,6 @@ extern "C" int LLVMFuzzerTestOneInput(uint8_t const* data, size_t size)
|
|||
if (!decoder.has_value())
|
||||
return 0;
|
||||
|
||||
(void)decoder->to_utf8(encoded_data);
|
||||
(void)decoder->to_utf8(encoded_data, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement);
|
||||
return 0;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1085,7 +1085,7 @@ size_t Request::on_header_received(void* buffer, size_t size, size_t nmemb, void
|
|||
auto decoder = TextCodec::decoder_for_exact_name("ISO-8859-1"sv);
|
||||
VERIFY(decoder.has_value());
|
||||
|
||||
request.m_reason_phrase = MUST(decoder->to_utf8(reason_phrase));
|
||||
request.m_reason_phrase = MUST(decoder->to_utf8(reason_phrase, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement));
|
||||
return total_size;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -37,13 +37,13 @@ TEST_CASE(test_utf8_decode)
|
|||
// Bytes for U+1F600 GRINNING FACE
|
||||
auto test_string = "\xf0\x9f\x98\x80"sv;
|
||||
|
||||
EXPECT(decoder.validate(test_string));
|
||||
EXPECT(!decoder.to_utf8(test_string, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Fatal).is_error());
|
||||
|
||||
auto processed_code_points = process_code_points(decoder, test_string);
|
||||
EXPECT(processed_code_points.size() == 1);
|
||||
EXPECT(processed_code_points[0] == 0x1F600);
|
||||
|
||||
EXPECT(MUST(decoder.to_utf8(test_string)) == test_string);
|
||||
EXPECT(MUST(decoder.to_utf8(test_string, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)) == test_string);
|
||||
}
|
||||
|
||||
TEST_CASE(test_utf8_process_code_points)
|
||||
|
|
@ -60,8 +60,8 @@ TEST_CASE(test_utf8_process_code_points_replaces_surrogates)
|
|||
auto utf8_encoded_surrogate_bytes = Vector<u8> { 0xed, 0xa0, 0x80 };
|
||||
auto utf8_encoded_surrogate = StringView(bytes(utf8_encoded_surrogate_bytes));
|
||||
|
||||
EXPECT(!decoder.validate(utf8_encoded_surrogate));
|
||||
EXPECT_EQ(MUST(decoder.to_utf8(utf8_encoded_surrogate)), "\xef\xbf\xbd\xef\xbf\xbd\xef\xbf\xbd"sv);
|
||||
EXPECT(decoder.to_utf8(utf8_encoded_surrogate, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Fatal).is_error());
|
||||
EXPECT_EQ(MUST(decoder.to_utf8(utf8_encoded_surrogate, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)), "\xef\xbf\xbd\xef\xbf\xbd\xef\xbf\xbd"sv);
|
||||
EXPECT_EQ(process_code_points(decoder, utf8_encoded_surrogate), (Vector<u32> { 0xfffd, 0xfffd, 0xfffd }));
|
||||
}
|
||||
|
||||
|
|
@ -71,8 +71,8 @@ TEST_CASE(test_utf8_process_code_points_replaces_truncated_tail_as_single_error)
|
|||
auto truncated_tail_bytes = Vector<u8> { 0xf0, 0x9f, 0x98 };
|
||||
auto truncated_tail = StringView(bytes(truncated_tail_bytes));
|
||||
|
||||
EXPECT(!decoder.validate(truncated_tail));
|
||||
EXPECT_EQ(MUST(decoder.to_utf8(truncated_tail)), "\xef\xbf\xbd"sv);
|
||||
EXPECT(decoder.to_utf8(truncated_tail, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Fatal).is_error());
|
||||
EXPECT_EQ(MUST(decoder.to_utf8(truncated_tail, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)), "\xef\xbf\xbd"sv);
|
||||
EXPECT_EQ(process_code_points(decoder, truncated_tail), (Vector<u32> { 0xfffd }));
|
||||
}
|
||||
|
||||
|
|
@ -82,8 +82,8 @@ TEST_CASE(test_utf8_process_code_points_replaces_overlong_sequences)
|
|||
auto overlong_null_bytes = Vector<u8> { 0xc0, 0x80 };
|
||||
auto overlong_null = StringView(bytes(overlong_null_bytes));
|
||||
|
||||
EXPECT(!decoder.validate(overlong_null));
|
||||
EXPECT_EQ(MUST(decoder.to_utf8(overlong_null)), "\xef\xbf\xbd\xef\xbf\xbd"sv);
|
||||
EXPECT(decoder.to_utf8(overlong_null, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Fatal).is_error());
|
||||
EXPECT_EQ(MUST(decoder.to_utf8(overlong_null, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)), "\xef\xbf\xbd\xef\xbf\xbd"sv);
|
||||
EXPECT_EQ(process_code_points(decoder, overlong_null), (Vector<u32> { 0xfffd, 0xfffd }));
|
||||
}
|
||||
|
||||
|
|
@ -95,12 +95,12 @@ TEST_CASE(test_utf8_process_code_points_restores_invalid_second_byte)
|
|||
auto out_of_range_four_byte_sequence_bytes = Vector<u8> { 0xf4, 0x90, 0x80, 0x80 };
|
||||
auto out_of_range_four_byte_sequence = StringView(bytes(out_of_range_four_byte_sequence_bytes));
|
||||
|
||||
EXPECT(!decoder.validate(overlong_three_byte_sequence));
|
||||
EXPECT_EQ(MUST(decoder.to_utf8(overlong_three_byte_sequence)), "\xef\xbf\xbd\xef\xbf\xbd\xef\xbf\xbd"sv);
|
||||
EXPECT(decoder.to_utf8(overlong_three_byte_sequence, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Fatal).is_error());
|
||||
EXPECT_EQ(MUST(decoder.to_utf8(overlong_three_byte_sequence, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)), "\xef\xbf\xbd\xef\xbf\xbd\xef\xbf\xbd"sv);
|
||||
EXPECT_EQ(process_code_points(decoder, overlong_three_byte_sequence), (Vector<u32> { 0xfffd, 0xfffd, 0xfffd }));
|
||||
|
||||
EXPECT(!decoder.validate(out_of_range_four_byte_sequence));
|
||||
EXPECT_EQ(MUST(decoder.to_utf8(out_of_range_four_byte_sequence)), "\xef\xbf\xbd\xef\xbf\xbd\xef\xbf\xbd\xef\xbf\xbd"sv);
|
||||
EXPECT(decoder.to_utf8(out_of_range_four_byte_sequence, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Fatal).is_error());
|
||||
EXPECT_EQ(MUST(decoder.to_utf8(out_of_range_four_byte_sequence, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)), "\xef\xbf\xbd\xef\xbf\xbd\xef\xbf\xbd\xef\xbf\xbd"sv);
|
||||
EXPECT_EQ(process_code_points(decoder, out_of_range_four_byte_sequence), (Vector<u32> { 0xfffd, 0xfffd, 0xfffd, 0xfffd }));
|
||||
}
|
||||
|
||||
|
|
@ -110,8 +110,7 @@ TEST_CASE(test_utf16be_decode)
|
|||
// This is the output of `python3 -c "print('säk😀'.encode('utf-16be'))"`.
|
||||
auto test_string = "\x00s\x00\xe4\x00k\xd8=\xde\x00"sv;
|
||||
|
||||
EXPECT(decoder.validate(test_string));
|
||||
auto utf8 = MUST(decoder.to_utf8(test_string));
|
||||
auto utf8 = MUST(decoder.to_utf8(test_string, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Fatal));
|
||||
EXPECT_EQ(utf8, "säk😀"sv);
|
||||
}
|
||||
|
||||
|
|
@ -122,12 +121,12 @@ TEST_CASE(test_utf16be_process_code_points)
|
|||
EXPECT_EQ(process_code_points(decoder, StringView(bytes({ 0xfe, 0xff, 0x00, 'A', 0xd8, 0x3d, 0xde, 0x00 }))), (Vector<u32> { 0x41, 0x1F600 }));
|
||||
EXPECT_EQ(process_code_points(decoder, StringView(bytes({ 0xd8, 0x3d, 0x00, 'A', 0xde, 0x00 }))), (Vector<u32> { 0xfffd, 0x41, 0xfffd }));
|
||||
EXPECT_EQ(process_code_points(decoder, StringView(bytes({ 0x00, 'A', 0xff }))), (Vector<u32> { 0x41, 0xfffd }));
|
||||
EXPECT_EQ(MUST(decoder.to_utf8(StringView(bytes({ 0x00, 'A', 0xff })))), "A\xef\xbf\xbd"sv);
|
||||
EXPECT_EQ(MUST(decoder.to_utf8(StringView(bytes({ 0x00, 'A', 0xff })), TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)), "A\xef\xbf\xbd"sv);
|
||||
}
|
||||
|
||||
TEST_CASE(test_streaming_decoder_utf8_mid_sequence)
|
||||
{
|
||||
auto streaming_decoder = TextCodec::StreamingDecoder { "UTF-8"sv };
|
||||
auto streaming_decoder = TextCodec::StreamingDecoder { "UTF-8"sv, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement };
|
||||
|
||||
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 'a', 0xc3 }))), "a"sv);
|
||||
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0xa9, 'b' }))), "éb"sv);
|
||||
|
|
@ -137,27 +136,27 @@ TEST_CASE(test_streaming_decoder_utf8_mid_sequence)
|
|||
TEST_CASE(test_streaming_decoder_finishes_incomplete_sequence)
|
||||
{
|
||||
auto& decoder = decoder_for("UTF-8"sv);
|
||||
auto streaming_decoder = TextCodec::StreamingDecoder { "UTF-8"sv };
|
||||
auto streaming_decoder = TextCodec::StreamingDecoder { "UTF-8"sv, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement };
|
||||
|
||||
auto incomplete_sequence = Vector<u8> { 0xc3 };
|
||||
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes(incomplete_sequence))), ""sv);
|
||||
EXPECT_EQ(MUST(streaming_decoder.finish()), MUST(decoder.to_utf8(StringView(bytes(incomplete_sequence)))));
|
||||
EXPECT_EQ(MUST(streaming_decoder.finish()), MUST(decoder.to_utf8(StringView(bytes(incomplete_sequence)), TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)));
|
||||
}
|
||||
|
||||
TEST_CASE(test_streaming_decoder_utf8_invalid_second_byte_tail)
|
||||
{
|
||||
auto streaming_decoder_for_overlong_three_byte_sequence = TextCodec::StreamingDecoder { "UTF-8"sv };
|
||||
auto streaming_decoder_for_overlong_three_byte_sequence = TextCodec::StreamingDecoder { "UTF-8"sv, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement };
|
||||
EXPECT_EQ(MUST(streaming_decoder_for_overlong_three_byte_sequence.to_utf8(bytes({ 0xe0, 0x80 }))), "\xef\xbf\xbd\xef\xbf\xbd"sv);
|
||||
EXPECT_EQ(MUST(streaming_decoder_for_overlong_three_byte_sequence.finish()), ""sv);
|
||||
|
||||
auto streaming_decoder_for_out_of_range_four_byte_sequence = TextCodec::StreamingDecoder { "UTF-8"sv };
|
||||
auto streaming_decoder_for_out_of_range_four_byte_sequence = TextCodec::StreamingDecoder { "UTF-8"sv, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement };
|
||||
EXPECT_EQ(MUST(streaming_decoder_for_out_of_range_four_byte_sequence.to_utf8(bytes({ 0xf4, 0x90 }))), "\xef\xbf\xbd\xef\xbf\xbd"sv);
|
||||
EXPECT_EQ(MUST(streaming_decoder_for_out_of_range_four_byte_sequence.finish()), ""sv);
|
||||
}
|
||||
|
||||
TEST_CASE(test_streaming_decoder_utf8_valid_second_byte_tail)
|
||||
{
|
||||
auto streaming_decoder = TextCodec::StreamingDecoder { "UTF-8"sv };
|
||||
auto streaming_decoder = TextCodec::StreamingDecoder { "UTF-8"sv, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement };
|
||||
|
||||
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0xe0, 0xa0 }))), ""sv);
|
||||
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0x80 }))), "\xe0\xa0\x80"sv);
|
||||
|
|
@ -166,7 +165,7 @@ TEST_CASE(test_streaming_decoder_utf8_valid_second_byte_tail)
|
|||
|
||||
TEST_CASE(test_streaming_decoder_utf16_odd_byte)
|
||||
{
|
||||
auto streaming_decoder = TextCodec::StreamingDecoder { "UTF-16LE"sv };
|
||||
auto streaming_decoder = TextCodec::StreamingDecoder { "UTF-16LE"sv, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement };
|
||||
|
||||
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0x41, 0x00, 0x42 }))), "A"sv);
|
||||
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0x00 }))), "B"sv);
|
||||
|
|
@ -175,7 +174,7 @@ TEST_CASE(test_streaming_decoder_utf16_odd_byte)
|
|||
|
||||
TEST_CASE(test_streaming_decoder_utf16_surrogate_pair_split)
|
||||
{
|
||||
auto streaming_decoder = TextCodec::StreamingDecoder { "UTF-16BE"sv };
|
||||
auto streaming_decoder = TextCodec::StreamingDecoder { "UTF-16BE"sv, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement };
|
||||
|
||||
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0x00, 0x41, 0xd8, 0x3d }))), "A"sv);
|
||||
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0xde, 0x00 }))), "😀"sv);
|
||||
|
|
@ -185,48 +184,48 @@ TEST_CASE(test_streaming_decoder_utf16_surrogate_pair_split)
|
|||
TEST_CASE(test_streaming_decoder_gb18030_four_byte_tail)
|
||||
{
|
||||
auto& decoder = decoder_for("gb18030"sv);
|
||||
auto streaming_decoder = TextCodec::StreamingDecoder { "gb18030"sv };
|
||||
auto streaming_decoder = TextCodec::StreamingDecoder { "gb18030"sv, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement };
|
||||
auto gb18030_sequence = Vector<u8> { 0x81, 0x30, 0x81, 0x30 };
|
||||
|
||||
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 'a', 0x81, 0x30, 0x81 }))), "a"sv);
|
||||
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0x30 }))), MUST(decoder.to_utf8(StringView(bytes(gb18030_sequence)))));
|
||||
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0x30 }))), MUST(decoder.to_utf8(StringView(bytes(gb18030_sequence)), TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)));
|
||||
EXPECT_EQ(MUST(streaming_decoder.finish()), ""sv);
|
||||
}
|
||||
|
||||
TEST_CASE(test_streaming_decoder_big5_overlapping_trail)
|
||||
{
|
||||
auto& decoder = decoder_for("Big5"sv);
|
||||
auto streaming_decoder = TextCodec::StreamingDecoder { "Big5"sv };
|
||||
auto streaming_decoder = TextCodec::StreamingDecoder { "Big5"sv, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement };
|
||||
auto big5_sequence = Vector<u8> { 0xa4, 0xa4 };
|
||||
|
||||
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes(big5_sequence))), MUST(decoder.to_utf8(StringView(bytes(big5_sequence)))));
|
||||
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes(big5_sequence))), MUST(decoder.to_utf8(StringView(bytes(big5_sequence)), TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)));
|
||||
EXPECT_EQ(MUST(streaming_decoder.finish()), ""sv);
|
||||
|
||||
auto streaming_decoder_for_split_input = TextCodec::StreamingDecoder { "Big5"sv };
|
||||
auto streaming_decoder_for_split_input = TextCodec::StreamingDecoder { "Big5"sv, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement };
|
||||
EXPECT_EQ(MUST(streaming_decoder_for_split_input.to_utf8(bytes({ 0xa4 }))), ""sv);
|
||||
EXPECT_EQ(MUST(streaming_decoder_for_split_input.to_utf8(bytes({ 0xa4 }))), MUST(decoder.to_utf8(StringView(bytes(big5_sequence)))));
|
||||
EXPECT_EQ(MUST(streaming_decoder_for_split_input.to_utf8(bytes({ 0xa4 }))), MUST(decoder.to_utf8(StringView(bytes(big5_sequence)), TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)));
|
||||
EXPECT_EQ(MUST(streaming_decoder_for_split_input.finish()), ""sv);
|
||||
}
|
||||
|
||||
TEST_CASE(test_streaming_decoder_euc_jp_three_byte_tail)
|
||||
{
|
||||
auto& decoder = decoder_for("EUC-JP"sv);
|
||||
auto streaming_decoder = TextCodec::StreamingDecoder { "EUC-JP"sv };
|
||||
auto streaming_decoder = TextCodec::StreamingDecoder { "EUC-JP"sv, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement };
|
||||
auto euc_jp_sequence = Vector<u8> { 0x8f, 0xa2, 0xaf };
|
||||
|
||||
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0x8f, 0xa2 }))), ""sv);
|
||||
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0xaf }))), MUST(decoder.to_utf8(StringView(bytes(euc_jp_sequence)))));
|
||||
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0xaf }))), MUST(decoder.to_utf8(StringView(bytes(euc_jp_sequence)), TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)));
|
||||
EXPECT_EQ(MUST(streaming_decoder.finish()), ""sv);
|
||||
}
|
||||
|
||||
TEST_CASE(test_streaming_decoder_shift_jis_tail)
|
||||
{
|
||||
auto& decoder = decoder_for("Shift_JIS"sv);
|
||||
auto streaming_decoder = TextCodec::StreamingDecoder { "Shift_JIS"sv };
|
||||
auto streaming_decoder = TextCodec::StreamingDecoder { "Shift_JIS"sv, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement };
|
||||
auto shift_jis_sequence = Vector<u8> { 0x82, 0xa0 };
|
||||
|
||||
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0x82 }))), ""sv);
|
||||
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0xa0 }))), MUST(decoder.to_utf8(StringView(bytes(shift_jis_sequence)))));
|
||||
EXPECT_EQ(MUST(streaming_decoder.to_utf8(bytes({ 0xa0 }))), MUST(decoder.to_utf8(StringView(bytes(shift_jis_sequence)), TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)));
|
||||
EXPECT_EQ(MUST(streaming_decoder.finish()), ""sv);
|
||||
}
|
||||
|
||||
|
|
@ -236,8 +235,7 @@ TEST_CASE(test_utf16le_decode)
|
|||
// This is the output of `python3 -c "print('säk😀'.encode('utf-16le'))"`.
|
||||
auto test_string = "s\x00\xe4\x00k\x00=\xd8\x00\xde"sv;
|
||||
|
||||
EXPECT(decoder.validate(test_string));
|
||||
auto utf8 = MUST(decoder.to_utf8(test_string));
|
||||
auto utf8 = MUST(decoder.to_utf8(test_string, TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Fatal));
|
||||
EXPECT_EQ(utf8, "säk😀"sv);
|
||||
}
|
||||
|
||||
|
|
@ -248,5 +246,5 @@ TEST_CASE(test_utf16le_process_code_points)
|
|||
EXPECT_EQ(process_code_points(decoder, StringView(bytes({ 0xff, 0xfe, 'A', 0x00, 0x3d, 0xd8, 0x00, 0xde }))), (Vector<u32> { 0x41, 0x1F600 }));
|
||||
EXPECT_EQ(process_code_points(decoder, StringView(bytes({ 0x3d, 0xd8, 'A', 0x00, 0x00, 0xde }))), (Vector<u32> { 0xfffd, 0x41, 0xfffd }));
|
||||
EXPECT_EQ(process_code_points(decoder, StringView(bytes({ 'A', 0x00, 0xff }))), (Vector<u32> { 0x41, 0xfffd }));
|
||||
EXPECT_EQ(MUST(decoder.to_utf8(StringView(bytes({ 'A', 0x00, 0xff })))), "A\xef\xbf\xbd"sv);
|
||||
EXPECT_EQ(MUST(decoder.to_utf8(StringView(bytes({ 'A', 0x00, 0xff })), TextCodec::IgnoreBOM::No, TextCodec::ErrorMode::Replacement)), "A\xef\xbf\xbd"sv);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,6 @@
|
|||
<!DOCTYPE html>
|
||||
<script>
|
||||
// createDocument with an empty qualified name yields a document with no root element.
|
||||
const doc = document.implementation.createDocument(null, "", null);
|
||||
document.evaluate("/", doc, null, XPathResult.ANY_TYPE, null);
|
||||
</script>
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
length: 2
|
||||
name: 0
|
||||
export name: sqlite3_aggregate_context
|
||||
length descriptor: writable=false, enumerable=false, configurable=true
|
||||
name descriptor: writable=false, enumerable=false, configurable=true
|
||||
call result: 42
|
||||
|
|
@ -0,0 +1,17 @@
|
|||
Harness status: OK
|
||||
|
||||
Found 12 tests
|
||||
|
||||
12 Pass
|
||||
Pass ignoreBOM should work for encoding utf-8, split at character 0
|
||||
Pass ignoreBOM should work for encoding utf-8, split at character 1
|
||||
Pass ignoreBOM should work for encoding utf-8, split at character 2
|
||||
Pass ignoreBOM should work for encoding utf-8, split at character 3
|
||||
Pass ignoreBOM should work for encoding utf-16le, split at character 0
|
||||
Pass ignoreBOM should work for encoding utf-16le, split at character 1
|
||||
Pass ignoreBOM should work for encoding utf-16le, split at character 2
|
||||
Pass ignoreBOM should work for encoding utf-16le, split at character 3
|
||||
Pass ignoreBOM should work for encoding utf-16be, split at character 0
|
||||
Pass ignoreBOM should work for encoding utf-16be, split at character 1
|
||||
Pass ignoreBOM should work for encoding utf-16be, split at character 2
|
||||
Pass ignoreBOM should work for encoding utf-16be, split at character 3
|
||||
|
|
@ -2,17 +2,16 @@ Harness status: OK
|
|||
|
||||
Found 12 tests
|
||||
|
||||
4 Pass
|
||||
8 Fail
|
||||
12 Pass
|
||||
Pass a TextEncoderStream object should be associated with the realm the constructor came from
|
||||
Pass TextEncoderStream's readable and writable attributes should come from the same realm as the constructor definition
|
||||
Fail the output chunks when read is called after write should come from the same realm as the constructor of TextEncoderStream
|
||||
Fail the output chunks when write is called with a pending read should come from the same realm as the constructor of TextEncoderStream
|
||||
Fail TypeError for unconvertable chunk should come from constructor realm of TextEncoderStream
|
||||
Pass the output chunks when read is called after write should come from the same realm as the constructor of TextEncoderStream
|
||||
Pass the output chunks when write is called with a pending read should come from the same realm as the constructor of TextEncoderStream
|
||||
Pass TypeError for unconvertable chunk should come from constructor realm of TextEncoderStream
|
||||
Pass a TextDecoderStream object should be associated with the realm the constructor came from
|
||||
Pass TextDecoderStream's readable and writable attributes should come from the same realm as the constructor definition
|
||||
Fail the result object when read is called after write should come from the same realm as the constructor of TextDecoderStream
|
||||
Fail the result object when write is called with a pending read should come from the same realm as the constructor of TextDecoderStream
|
||||
Fail TypeError for chunk with the wrong type should come from constructor realm of TextDecoderStream
|
||||
Fail TypeError for invalid chunk should come from constructor realm of TextDecoderStream
|
||||
Fail TypeError for incomplete input should come from constructor realm of TextDecoderStream
|
||||
Pass the result object when read is called after write should come from the same realm as the constructor of TextDecoderStream
|
||||
Pass the result object when write is called with a pending read should come from the same realm as the constructor of TextDecoderStream
|
||||
Pass TypeError for chunk with the wrong type should come from constructor realm of TextDecoderStream
|
||||
Pass TypeError for invalid chunk should come from constructor realm of TextDecoderStream
|
||||
Pass TypeError for incomplete input should come from constructor realm of TextDecoderStream
|
||||
|
|
|
|||
|
|
@ -0,0 +1,8 @@
|
|||
Harness status: OK
|
||||
|
||||
Found 2 tests
|
||||
|
||||
1 Pass
|
||||
1 Fail
|
||||
Pass Fatal flag, non-streaming cases
|
||||
Fail Fatal flag, streaming cases
|
||||
|
|
@ -2,9 +2,8 @@ Harness status: OK
|
|||
|
||||
Found 4 tests
|
||||
|
||||
1 Pass
|
||||
3 Fail
|
||||
Fail BOM is ignored if ignoreBOM option is specified: utf-8
|
||||
Fail BOM is ignored if ignoreBOM option is specified: utf-16le
|
||||
Fail BOM is ignored if ignoreBOM option is specified: utf-16be
|
||||
4 Pass
|
||||
Pass BOM is ignored if ignoreBOM option is specified: utf-8
|
||||
Pass BOM is ignored if ignoreBOM option is specified: utf-16le
|
||||
Pass BOM is ignored if ignoreBOM option is specified: utf-16be
|
||||
Pass The ignoreBOM attribute of TextDecoder
|
||||
|
|
@ -2,8 +2,8 @@ Harness status: OK
|
|||
|
||||
Found 87 tests
|
||||
|
||||
72 Pass
|
||||
15 Fail
|
||||
74 Pass
|
||||
13 Fail
|
||||
Pass Invalid Unicode input is replaced: utf-8
|
||||
Pass Invalid Unicode input is replaced: utf-16le
|
||||
Pass Invalid Unicode input is replaced: utf-16be
|
||||
|
|
@ -51,8 +51,8 @@ Pass selected single-byte: iso-8859-8-i
|
|||
Pass selected single-byte: iso-8859-16
|
||||
Pass selected single-byte: x-mac-cyrillic
|
||||
Pass Concatenating two ISO-2022-JP outputs is not always valid
|
||||
Fail gb18030 version and ranges
|
||||
Fail gbk version and ranges
|
||||
Pass gb18030 version and ranges
|
||||
Pass gbk version and ranges
|
||||
Pass gbk decoder is gb18030 decoder
|
||||
Pass Replacement, push back ASCII characters: big5
|
||||
Pass Replacement, push back ASCII characters: iso-2022-jp
|
||||
|
|
|
|||
|
|
@ -0,0 +1,32 @@
|
|||
<!DOCTYPE html>
|
||||
<script src="../include.js"></script>
|
||||
<script>
|
||||
asyncTest(async (done) => {
|
||||
const wasmBytes = new Uint8Array([
|
||||
0x00, 0x61, 0x73, 0x6d, 0x01, 0x00, 0x00, 0x00,
|
||||
0x01, 0x07, 0x01, 0x60, 0x02, 0x7f, 0x7f, 0x01, 0x7f,
|
||||
0x03, 0x02, 0x01, 0x00,
|
||||
0x07, 0x1d, 0x01, 0x19,
|
||||
0x73, 0x71, 0x6c, 0x69, 0x74, 0x65, 0x33, 0x5f,
|
||||
0x61, 0x67, 0x67, 0x72, 0x65, 0x67, 0x61, 0x74,
|
||||
0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74,
|
||||
0x00, 0x00,
|
||||
0x0a, 0x06, 0x01, 0x04, 0x00, 0x20, 0x00, 0x0b,
|
||||
]);
|
||||
|
||||
const { instance } = await WebAssembly.instantiate(wasmBytes);
|
||||
const fn = instance.exports.sqlite3_aggregate_context;
|
||||
const exportName = Object.getOwnPropertyNames(instance.exports)[0];
|
||||
const lengthDescriptor = Object.getOwnPropertyDescriptor(fn, "length");
|
||||
const nameDescriptor = Object.getOwnPropertyDescriptor(fn, "name");
|
||||
|
||||
println(`length: ${fn.length}`);
|
||||
println(`name: ${fn.name}`);
|
||||
println(`export name: ${exportName}`);
|
||||
println(`length descriptor: writable=${lengthDescriptor.writable}, enumerable=${lengthDescriptor.enumerable}, configurable=${lengthDescriptor.configurable}`);
|
||||
println(`name descriptor: writable=${nameDescriptor.writable}, enumerable=${nameDescriptor.enumerable}, configurable=${nameDescriptor.configurable}`);
|
||||
println(`call result: ${fn(42, 7)}`);
|
||||
|
||||
done();
|
||||
});
|
||||
</script>
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
<!doctype html>
|
||||
<meta charset=utf-8>
|
||||
|
||||
<script>
|
||||
self.GLOBAL = {
|
||||
isWindow: function() { return true; },
|
||||
isWorker: function() { return false; },
|
||||
isShadowRealm: function() { return false; },
|
||||
};
|
||||
</script>
|
||||
<script src="../../resources/testharness.js"></script>
|
||||
<script src="../../resources/testharnessreport.js"></script>
|
||||
<script src="resources/readable-stream-from-array.js"></script>
|
||||
<script src="resources/readable-stream-to-array.js"></script>
|
||||
<div id=log></div>
|
||||
<script src="../../encoding/streams/decode-ignore-bom.any.js"></script>
|
||||
|
|
@ -0,0 +1,38 @@
|
|||
// META: global=window,worker
|
||||
// META: script=resources/readable-stream-from-array.js
|
||||
// META: script=resources/readable-stream-to-array.js
|
||||
|
||||
const cases = [
|
||||
{encoding: 'utf-8', bytes: [0xEF, 0xBB, 0xBF, 0x61, 0x62, 0x63]},
|
||||
{encoding: 'utf-16le', bytes: [0xFF, 0xFE, 0x61, 0x00, 0x62, 0x00, 0x63, 0x00]},
|
||||
{encoding: 'utf-16be', bytes: [0xFE, 0xFF, 0x00, 0x61, 0x00, 0x62, 0x00, 0x63]}
|
||||
];
|
||||
const BOM = '\uFEFF';
|
||||
|
||||
// |inputChunks| is an array of chunks, each represented by an array of
|
||||
// integers. |ignoreBOM| is true or false. The result value is the output of the
|
||||
// pipe, concatenated into a single string.
|
||||
async function pipeAndAssemble(inputChunks, encoding, ignoreBOM) {
|
||||
const chunksAsUint8 = inputChunks.map(values => new Uint8Array(values));
|
||||
const readable = readableStreamFromArray(chunksAsUint8);
|
||||
const outputArray = await readableStreamToArray(readable.pipeThrough(
|
||||
new TextDecoderStream(encoding, {ignoreBOM})));
|
||||
return outputArray.join('');
|
||||
}
|
||||
|
||||
for (const testCase of cases) {
|
||||
for (let splitPoint = 0; splitPoint < 4; ++splitPoint) {
|
||||
promise_test(async () => {
|
||||
const inputChunks = [testCase.bytes.slice(0, splitPoint),
|
||||
testCase.bytes.slice(splitPoint)];
|
||||
const withIgnoreBOM =
|
||||
await pipeAndAssemble(inputChunks, testCase.encoding, true);
|
||||
assert_equals(withIgnoreBOM, BOM + 'abc', 'BOM should be preserved');
|
||||
|
||||
const withoutIgnoreBOM =
|
||||
await pipeAndAssemble(inputChunks, testCase.encoding, false);
|
||||
assert_equals(withoutIgnoreBOM, 'abc', 'BOM should be stripped')
|
||||
}, `ignoreBOM should work for encoding ${testCase.encoding}, split at ` +
|
||||
`character ${splitPoint}`);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
<!doctype html>
|
||||
<meta charset=utf-8>
|
||||
<title>Encoding API: End-of-file</title>
|
||||
<script>
|
||||
self.GLOBAL = {
|
||||
isWindow: function() { return true; },
|
||||
isWorker: function() { return false; },
|
||||
isShadowRealm: function() { return false; },
|
||||
};
|
||||
</script>
|
||||
<script src="../resources/testharness.js"></script>
|
||||
<script src="../resources/testharnessreport.js"></script>
|
||||
|
||||
<div id=log></div>
|
||||
<script src="../encoding/textdecoder-fatal-streaming.any.js"></script>
|
||||
|
|
@ -0,0 +1,45 @@
|
|||
// META: global=window,dedicatedworker
|
||||
// META: title=Encoding API: End-of-file
|
||||
|
||||
test(function() {
|
||||
[
|
||||
{encoding: 'utf-8', sequence: [0xC0]},
|
||||
{encoding: 'utf-16le', sequence: [0x00]},
|
||||
{encoding: 'utf-16be', sequence: [0x00]}
|
||||
].forEach(function(testCase) {
|
||||
|
||||
assert_throws_js(TypeError, function() {
|
||||
var decoder = new TextDecoder(testCase.encoding, {fatal: true});
|
||||
decoder.decode(new Uint8Array(testCase.sequence));
|
||||
}, 'Unterminated ' + testCase.encoding + ' sequence should throw if fatal flag is set');
|
||||
|
||||
assert_equals(
|
||||
new TextDecoder(testCase.encoding).decode(new Uint8Array([testCase.sequence])),
|
||||
'\uFFFD',
|
||||
'Unterminated UTF-8 sequence should emit replacement character if fatal flag is unset');
|
||||
});
|
||||
}, 'Fatal flag, non-streaming cases');
|
||||
|
||||
test(function() {
|
||||
|
||||
var decoder = new TextDecoder('utf-16le', {fatal: true});
|
||||
var odd = new Uint8Array([0x00]);
|
||||
var even = new Uint8Array([0x00, 0x00]);
|
||||
|
||||
assert_equals(decoder.decode(odd, {stream: true}), '');
|
||||
assert_equals(decoder.decode(odd), '\u0000');
|
||||
|
||||
assert_throws_js(TypeError, function() {
|
||||
decoder.decode(even, {stream: true});
|
||||
decoder.decode(odd)
|
||||
});
|
||||
|
||||
assert_throws_js(TypeError, function() {
|
||||
decoder.decode(odd, {stream: true});
|
||||
decoder.decode(even);
|
||||
});
|
||||
|
||||
assert_equals(decoder.decode(even, {stream: true}), '\u0000');
|
||||
assert_equals(decoder.decode(even), '\u0000');
|
||||
|
||||
}, 'Fatal flag, streaming cases');
|
||||
Loading…
Reference in a new issue