dmaax/ladybird
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

LibTextCodec: Use encoding_rs for legacy codecs

Browse source 
Replace the generated C++ legacy codec implementations with a
small Rust wrapper around encoding_rs.

This keeps the existing LibTextCodec API while moving label lookup,
legacy decode/encode, validation, and streaming decoder state to Rust.
The generated index data and generator are no longer needed.

It also fixes several TextDecoder EOF cases due to a more correct
implementation. encoding_rs finalizes decoders according to the
Encoding Standard, so incomplete UTF-8/Big5 tails and malformed
UTF-16 surrogate tails produce the required single replacement at
end-of-queue instead of being dropped, buffered, or double-counted
by our old hand-written decoders.
This commit is contained in:
Shannon Booth 2026-06-20 00:51:56 +02:00 committed by Andreas Kling
parent 250842d36a
commit 7b0dd6ab30
18 changed files with 753 additions and 2831 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
Cargo.lock generated
View file

@ -727,6 +727,14 @@ dependencies = [
"libunicode_rust",
]
[[package]]
name = "libtextcodec_rust"
version = "0.1.0"
dependencies = [
"cbindgen",
"encoding_rs",
]
[[package]]
name = "libunicode_rust"
version = "0.1.0"

1
Cargo.toml
View file

@ -3,6 +3,7 @@ members = [
"Libraries/LibGfx/Rust",
"Libraries/LibJS/Rust",
"Libraries/LibRegex/Rust",
"Libraries/LibTextCodec/Rust",
"Libraries/LibUnicode/Rust",
"Libraries/LibURL/Rust",
"Libraries/LibWasm/Rust",

15
Libraries/LibTextCodec/CMakeLists.txt
View file

@ -1,14 +1,13 @@
include(libtextcodec_generators)
set(SOURCES
Decoder.cpp
Encoder.cpp
)
generate_encoding_indexes()
set(GENERATED_SOURCES
LookupTables.cpp
)
ladybird_lib(LibTextCodec textcodec EXPLICIT_SYMBOL_EXPORT)
import_rust_crate(MANIFEST_PATH Rust/Cargo.toml CRATE_NAME libtextcodec_rust FFI_HEADER RustFFI.h)
target_link_libraries(LibTextCodec PRIVATE libtextcodec_rust)
get_target_property(_libtextcodec_rust_lib libtextcodec_rust IMPORTED_LOCATION)
set_property(SOURCE Decoder.cpp APPEND PROPERTY OBJECT_DEPENDS ${_libtextcodec_rust_lib})
set_property(SOURCE Encoder.cpp APPEND PROPERTY OBJECT_DEPENDS ${_libtextcodec_rust_lib})

1791
Libraries/LibTextCodec/Decoder.cpp
View file

File diff suppressed because it is too large Load diff

116
Libraries/LibTextCodec/Decoder.h
View file

@ -8,9 +8,9 @@
#pragma once
#include <AK/ByteBuffer.h>
#include <AK/Forward.h>
#include <AK/Function.h>
#include <AK/Noncopyable.h>
#include <AK/Optional.h>
#include <AK/String.h>
#include <LibTextCodec/Export.h>
@ -25,129 +25,23 @@ public:
virtual ErrorOr<size_t> length_in_utf16_code_units(StringView);
ErrorOr<void> process_code_points(StringView, Function<ErrorOr<void>(u32)>);
// Returns the number of trailing bytes that form an incomplete sequence and must be buffered
// until more input arrives. Used by StreamingDecoder for chunked decoding.
virtual size_t incomplete_tail_length(ReadonlyBytes) const { return 0; }
protected:
virtual ~Decoder() = default;
virtual ErrorOr<void> process(StringView, Function<ErrorOr<void>(u32)> on_code_point) = 0;
};
class TEXTCODEC_API UTF8Decoder final : public Decoder {
public:
virtual ErrorOr<void> process(StringView, Function<ErrorOr<void>(u32)> on_code_point) override;
virtual bool validate(StringView) override;
virtual ErrorOr<String> to_utf8(StringView) override;
virtual ErrorOr<size_t> length_in_utf16_code_units(StringView) override;
virtual size_t incomplete_tail_length(ReadonlyBytes) const override;
};
class TEXTCODEC_API UTF16BEDecoder final : public Decoder {
public:
virtual bool validate(StringView) override;
virtual ErrorOr<String> to_utf8(StringView) override;
virtual ErrorOr<size_t> length_in_utf16_code_units(StringView) override;
virtual size_t incomplete_tail_length(ReadonlyBytes) const override;
private:
virtual ErrorOr<void> process(StringView, Function<ErrorOr<void>(u32)>) override;
};
class TEXTCODEC_API UTF16LEDecoder final : public Decoder {
public:
virtual bool validate(StringView) override;
virtual ErrorOr<String> to_utf8(StringView) override;
virtual ErrorOr<size_t> length_in_utf16_code_units(StringView) override;
virtual size_t incomplete_tail_length(ReadonlyBytes) const override;
private:
virtual ErrorOr<void> process(StringView, Function<ErrorOr<void>(u32)>) override;
};
template<Integral ArrayType = u32>
class SingleByteDecoder final : public Decoder {
public:
SingleByteDecoder(Array<ArrayType, 128> translation_table)
: m_translation_table(translation_table)
{
}
virtual ErrorOr<void> process(StringView, Function<ErrorOr<void>(u32)> on_code_point) override;
virtual ErrorOr<size_t> length_in_utf16_code_units(StringView) override;
private:
Array<ArrayType, 128> m_translation_table;
};
class TEXTCODEC_API Latin1Decoder final : public Decoder {
public:
virtual ErrorOr<void> process(StringView, Function<ErrorOr<void>(u32)> on_code_point) override;
virtual bool validate(StringView) override { return true; }
virtual ErrorOr<size_t> length_in_utf16_code_units(StringView) override;
};
class TEXTCODEC_API XUserDefinedDecoder final : public Decoder {
public:
virtual ErrorOr<void> process(StringView, Function<ErrorOr<void>(u32)> on_code_point) override;
virtual bool validate(StringView) override { return true; }
virtual ErrorOr<size_t> length_in_utf16_code_units(StringView) override;
};
class TEXTCODEC_API GB18030Decoder final : public Decoder {
public:
virtual ErrorOr<void> process(StringView, Function<ErrorOr<void>(u32)> on_code_point) override;
virtual size_t incomplete_tail_length(ReadonlyBytes) const override;
};
class TEXTCODEC_API Big5Decoder final : public Decoder {
public:
virtual ErrorOr<void> process(StringView, Function<ErrorOr<void>(u32)> on_code_point) override;
virtual size_t incomplete_tail_length(ReadonlyBytes) const override;
};
class TEXTCODEC_API EUCJPDecoder final : public Decoder {
public:
virtual ErrorOr<void> process(StringView, Function<ErrorOr<void>(u32)> on_code_point) override;
virtual size_t incomplete_tail_length(ReadonlyBytes) const override;
};
class TEXTCODEC_API ISO2022JPDecoder final : public Decoder {
public:
virtual ErrorOr<void> process(StringView, Function<ErrorOr<void>(u32)> on_code_point) override;
virtual size_t incomplete_tail_length(ReadonlyBytes) const override;
};
class TEXTCODEC_API ShiftJISDecoder final : public Decoder {
public:
virtual ErrorOr<void> process(StringView, Function<ErrorOr<void>(u32)> on_code_point) override;
virtual size_t incomplete_tail_length(ReadonlyBytes) const override;
};
class TEXTCODEC_API EUCKRDecoder final : public Decoder {
public:
virtual ErrorOr<void> process(StringView, Function<ErrorOr<void>(u32)> on_code_point) override;
virtual size_t incomplete_tail_length(ReadonlyBytes) const override;
};
class TEXTCODEC_API ReplacementDecoder final : public Decoder {
public:
virtual ErrorOr<void> process(StringView, Function<ErrorOr<void>(u32)> on_code_point) override;
virtual bool validate(StringView input) override { return input.is_empty(); }
virtual ErrorOr<size_t> length_in_utf16_code_units(StringView) override;
};
// Preserves incomplete trailing decoder tokens when callers provide input in chunks.
class TEXTCODEC_API StreamingDecoder final {
AK_MAKE_NONCOPYABLE(StreamingDecoder);
public:
explicit StreamingDecoder(StringView encoding);
~StreamingDecoder();
ErrorOr<String> to_utf8(ReadonlyBytes);
ErrorOr<String> finish();
private:
Decoder& m_decoder;
ByteBuffer m_pending_input;
void* m_decoder { nullptr };
};
// This will return a decoder for the exact name specified, skipping get_standardized_encoding.

765
Libraries/LibTextCodec/Encoder.cpp
View file

@ -4,55 +4,120 @@
* SPDX-License-Identifier: BSD-2-Clause
*/
#include <AK/BinarySearch.h>
#include <AK/Error.h>
#include <AK/StringBuilder.h>
#include <AK/Utf8View.h>
#include <LibTextCodec/Decoder.h>
#include <LibTextCodec/Encoder.h>
#include <LibTextCodec/LookupTables.h>
#include <RustFFI.h>
namespace TextCodec {
namespace {
UTF8Encoder s_utf8_encoder;
GB18030Encoder s_gb18030_encoder;
GB18030Encoder s_gbk_encoder(GB18030Encoder::IsGBK::Yes);
Big5Encoder s_big5_encoder;
EUCJPEncoder s_euc_jp_encoder;
ISO2022JPEncoder s_iso_2022_jp_encoder;
ShiftJISEncoder s_shift_jis_encoder;
EUCKREncoder s_euc_kr_encoder;
class RustEncoder final : public Encoder {
public:
explicit RustEncoder(StringView encoding)
: m_encoding(encoding)
{
}
// s_{encoding}_index is generated from https://encoding.spec.whatwg.org/indexes.json
// Found separately in https://encoding.spec.whatwg.org/index-{encoding}.txt
SingleByteEncoder s_ibm866_encoder { s_ibm866_index };
SingleByteEncoder s_latin2_encoder { s_iso_8859_2_index };
SingleByteEncoder s_latin3_encoder { s_iso_8859_3_index };
SingleByteEncoder s_latin4_encoder { s_iso_8859_4_index };
SingleByteEncoder s_latin_cyrillic_encoder { s_iso_8859_5_index };
SingleByteEncoder s_latin_arabic_encoder { s_iso_8859_6_index };
SingleByteEncoder s_latin_greek_encoder { s_iso_8859_7_index };
SingleByteEncoder s_latin_hebrew_encoder { s_iso_8859_8_index };
SingleByteEncoder s_latin6_encoder { s_iso_8859_10_index };
SingleByteEncoder s_latin7_encoder { s_iso_8859_13_index };
SingleByteEncoder s_latin8_encoder { s_iso_8859_14_index };
SingleByteEncoder s_latin9_encoder { s_iso_8859_15_index };
SingleByteEncoder s_latin10_encoder { s_iso_8859_16_index };
SingleByteEncoder s_centraleurope_encoder { s_windows_1250_index };
SingleByteEncoder s_cyrillic_encoder { s_windows_1251_index };
SingleByteEncoder s_hebrew_encoder { s_windows_1255_index };
SingleByteEncoder s_koi8r_encoder { s_koi8_r_index };
SingleByteEncoder s_koi8u_encoder { s_koi8_u_index };
SingleByteEncoder s_mac_roman_encoder { s_macintosh_index };
SingleByteEncoder s_windows874_encoder { s_windows_874_index };
SingleByteEncoder s_windows1252_encoder { s_windows_1252_index };
SingleByteEncoder s_windows1253_encoder { s_windows_1253_index };
SingleByteEncoder s_turkish_encoder { s_windows_1254_index };
SingleByteEncoder s_windows1256_encoder { s_windows_1256_index };
SingleByteEncoder s_windows1257_encoder { s_windows_1257_index };
SingleByteEncoder s_windows1258_encoder { s_windows_1258_index };
SingleByteEncoder s_mac_cyrillic_encoder { s_x_mac_cyrillic_index };
virtual ErrorOr<void> process(Utf8View input, Function<ErrorOr<void>(u8)> on_byte, Function<ErrorOr<void>(u32)> on_error) override;
private:
StringView m_encoding;
};
class UTF8Encoder final : public Encoder {
public:
virtual ErrorOr<void> process(Utf8View, Function<ErrorOr<void>(u8)> on_byte, Function<ErrorOr<void>(u32)> on_error) override;
};
UTF8Encoder s_utf8_encoder;
RustEncoder s_gb18030_encoder { "gb18030"sv };
RustEncoder s_gbk_encoder { "GBK"sv };
RustEncoder s_big5_encoder { "Big5"sv };
RustEncoder s_euc_jp_encoder { "EUC-JP"sv };
RustEncoder s_iso_2022_jp_encoder { "ISO-2022-JP"sv };
RustEncoder s_shift_jis_encoder { "Shift_JIS"sv };
RustEncoder s_euc_kr_encoder { "EUC-KR"sv };
RustEncoder s_ibm866_encoder { "IBM866"sv };
RustEncoder s_latin2_encoder { "ISO-8859-2"sv };
RustEncoder s_latin3_encoder { "ISO-8859-3"sv };
RustEncoder s_latin4_encoder { "ISO-8859-4"sv };
RustEncoder s_latin_cyrillic_encoder { "ISO-8859-5"sv };
RustEncoder s_latin_arabic_encoder { "ISO-8859-6"sv };
RustEncoder s_latin_greek_encoder { "ISO-8859-7"sv };
RustEncoder s_latin_hebrew_encoder { "ISO-8859-8"sv };
RustEncoder s_latin6_encoder { "ISO-8859-10"sv };
RustEncoder s_latin7_encoder { "ISO-8859-13"sv };
RustEncoder s_latin8_encoder { "ISO-8859-14"sv };
RustEncoder s_latin9_encoder { "ISO-8859-15"sv };
RustEncoder s_latin10_encoder { "ISO-8859-16"sv };
RustEncoder s_centraleurope_encoder { "windows-1250"sv };
RustEncoder s_cyrillic_encoder { "windows-1251"sv };
RustEncoder s_hebrew_encoder { "windows-1255"sv };
RustEncoder s_koi8r_encoder { "KOI8-R"sv };
RustEncoder s_koi8u_encoder { "KOI8-U"sv };
RustEncoder s_mac_roman_encoder { "macintosh"sv };
RustEncoder s_windows874_encoder { "windows-874"sv };
RustEncoder s_windows1252_encoder { "windows-1252"sv };
RustEncoder s_windows1253_encoder { "windows-1253"sv };
RustEncoder s_turkish_encoder { "windows-1254"sv };
RustEncoder s_windows1256_encoder { "windows-1256"sv };
RustEncoder s_windows1257_encoder { "windows-1257"sv };
RustEncoder s_windows1258_encoder { "windows-1258"sv };
RustEncoder s_mac_cyrillic_encoder { "x-mac-cyrillic"sv };
struct EncodeContext {
Function<ErrorOr<void>(u8)>* on_byte { nullptr };
Function<ErrorOr<void>(u32)>* on_error { nullptr };
ErrorOr<void> result {};
};
static void append_encoded_bytes(void* context, u8 const* data, size_t length)
{
auto& encode_context = *static_cast<EncodeContext*>(context);
if (encode_context.result.is_error())
return;
for (size_t i = 0; i < length; ++i) {
encode_context.result = (*encode_context.on_byte)(data[i]);
if (encode_context.result.is_error())
return;
}
}
static void report_unmappable_code_point(void* context, u32 code_point)
{
auto& encode_context = *static_cast<EncodeContext*>(context);
if (encode_context.result.is_error())
return;
encode_context.result = (*encode_context.on_error)(code_point);
}
ErrorOr<void> rust_encode(StringView encoding, Utf8View input, Function<ErrorOr<void>(u8)> on_byte, Function<ErrorOr<void>(u32)> on_error)
{
auto input_view = StringView { input.bytes(), input.byte_length() };
EncodeContext context {
.on_byte = &on_byte,
.on_error = &on_error,
};
auto succeeded = FFI::textcodec_rust_encode_from_utf8(
reinterpret_cast<u8 const*>(encoding.characters_without_null_termination()),
encoding.length(),
reinterpret_cast<u8 const*>(input_view.characters_without_null_termination()),
input_view.length(),
&context,
append_encoded_bytes,
report_unmappable_code_point);
if (!succeeded)
return Error::from_errno(EINVAL);
TRY(context.result);
return {};
}
}
@ -138,7 +203,11 @@ Optional<Encoder&> encoder_for(StringView label)
return encoding.has_value() ? encoder_for_exact_name(encoding.value()) : Optional<Encoder&> {};
}
// https://encoding.spec.whatwg.org/#utf-8-encoder
ErrorOr<void> RustEncoder::process(Utf8View input, Function<ErrorOr<void>(u8)> on_byte, Function<ErrorOr<void>(u32)> on_error)
{
return rust_encode(m_encoding, input, move(on_byte), move(on_error));
}
ErrorOr<void> UTF8Encoder::process(Utf8View input, Function<ErrorOr<void>(u8)> on_byte, Function<ErrorOr<void>(u32)>)
{
ReadonlyBytes bytes { input.bytes(), input.byte_length() };
@ -147,622 +216,6 @@ ErrorOr<void> UTF8Encoder::process(Utf8View input, Function<ErrorOr<void>(u8)> o
return {};
}
// https://encoding.spec.whatwg.org/#euc-jp-encoder
ErrorOr<void> EUCJPEncoder::process(Utf8View input, Function<ErrorOr<void>(u8)> on_byte, Function<ErrorOr<void>(u32)> on_error)
{
for (auto item : input) {
// 1. If code point is end-of-queue, return finished.
// 2. If code point is an ASCII code point, return a byte whose value is code point.
if (is_ascii(item)) {
TRY(on_byte(static_cast<u8>(item)));
continue;
}
// 3. If code point is U+00A5, return byte 0x5C.
if (item == 0x00A5) {
TRY(on_byte(static_cast<u8>(0x5C)));
continue;
}
// 4. If code point is U+203E, return byte 0x7E.
if (item == 0x203E) {
TRY(on_byte(static_cast<u8>(0x7E)));
continue;
}
// 5. If code point is in the range U+FF61 to U+FF9F, inclusive, return two bytes whose values are 0x8E and code point − 0xFF61 + 0xA1.
if (item >= 0xFF61 && item <= 0xFF9F) {
TRY(on_byte(0x8E));
TRY(on_byte(static_cast<u8>(item - 0xFF61 + 0xA1)));
continue;
}
// 6. If code point is U+2212, set it to U+FF0D.
if (item == 0x2212)
item = 0xFF0D;
// 7. Let pointer be the index pointer for code point in index jis0208.
auto pointer = code_point_jis0208_index(item);
// 8. If pointer is null, return error with code point.
if (!pointer.has_value()) {
TRY(on_error(item));
continue;
}
// 9. Let lead be pointer / 94 + 0xA1.
auto lead = *pointer / 94 + 0xA1;
// 10. Let trail be pointer % 94 + 0xA1.
auto trail = *pointer % 94 + 0xA1;
// 11. Return two bytes whose values are lead and trail.
TRY(on_byte(static_cast<u8>(lead)));
TRY(on_byte(static_cast<u8>(trail)));
}
return {};
}
// https://encoding.spec.whatwg.org/#iso-2022-jp-encoder
ErrorOr<ISO2022JPEncoder::State> ISO2022JPEncoder::process_item(u32 item, State state, Function<ErrorOr<void>(u8)>& on_byte, Function<ErrorOr<void>(u32)>& on_error)
{
// 3. If ISO-2022-JP encoder state is ASCII or Roman, and code point is U+000E, U+000F, or U+001B, return error with U+FFFD.
if (state == State::ASCII || state == State::Roman) {
if (item == 0x000E || item == 0x000F || item == 0x001B) {
TRY(on_error(0xFFFD));
return state;
}
}
// 4. If ISO-2022-JP encoder state is ASCII and code point is an ASCII code point, return a byte whose value is code point.
if (state == State::ASCII && is_ascii(item)) {
TRY(on_byte(static_cast<u8>(item)));
return state;
}
// 5. If ISO-2022-JP encoder state is Roman and code point is an ASCII code point, excluding U+005C and U+007E, or is U+00A5 or U+203E, then:
if (state == State::Roman && ((is_ascii(item) && item != 0x005C && item != 0x007E) || (item == 0x00A5 || item == 0x203E))) {
// 1. If code point is an ASCII code point, return a byte whose value is code point.
if (is_ascii(item)) {
TRY(on_byte(static_cast<u8>(item)));
return state;
}
// 2. If code point is U+00A5, return byte 0x5C.
if (item == 0x00A5) {
TRY(on_byte(0x5C));
return state;
}
// 3. If code point is U+203E, return byte 0x7E.
if (item == 0x203E) {
TRY(on_byte(0x7E));
return state;
}
}
// 6. If code point is an ASCII code point, and ISO-2022-JP encoder state is not ASCII, restore code point to ioQueue, set
// ISO-2022-JP encoder state to ASCII, and return three bytes 0x1B 0x28 0x42.
if (is_ascii(item) && state != State::ASCII) {
TRY(on_byte(0x1B));
TRY(on_byte(0x28));
TRY(on_byte(0x42));
return process_item(item, State::ASCII, on_byte, on_error);
}
// 7. If code point is either U+00A5 or U+203E, and ISO-2022-JP encoder state is not Roman, restore code point to ioQueue,
// set ISO-2022-JP encoder state to Roman, and return three bytes 0x1B 0x28 0x4A.
if ((item == 0x00A5 || item == 0x203E) && state != State::Roman) {
TRY(on_byte(0x1B));
TRY(on_byte(0x28));
TRY(on_byte(0x4A));
return process_item(item, State::Roman, on_byte, on_error);
}
// 8. If code point is U+2212, set it to U+FF0D.
if (item == 0x2212)
item = 0xFF0D;
// 9. If code point is in the range U+FF61 to U+FF9F, inclusive, set it to the index code point for code point − 0xFF61
// in index ISO-2022-JP katakana.
if (item >= 0xFF61 && item <= 0xFF9F) {
item = *index_iso_2022_jp_katakana_code_point(item - 0xFF61);
}
// 10. Let pointer be the index pointer for code point in index jis0208.
auto pointer = code_point_jis0208_index(item);
// 11. If pointer is null, then:
if (!pointer.has_value()) {
// 1. If ISO-2022-JP encoder state is jis0208, then restore code point to ioQueue, set ISO-2022-JP encoder state to
// ASCII, and return three bytes 0x1B 0x28 0x42.
if (state == State::jis0208) {
TRY(on_byte(0x1B));
TRY(on_byte(0x28));
TRY(on_byte(0x42));
return process_item(item, State::ASCII, on_byte, on_error);
}
// 2. Return error with code point.
TRY(on_error(item));
return state;
}
// 12. If ISO-2022-JP encoder state is not jis0208, restore code point to ioQueue, set ISO-2022-JP encoder state to
// jis0208, and return three bytes 0x1B 0x24 0x42.
if (state != State::jis0208) {
TRY(on_byte(0x1B));
TRY(on_byte(0x24));
TRY(on_byte(0x42));
return process_item(item, State::jis0208, on_byte, on_error);
}
// 13. Let lead be pointer / 94 + 0x21.
auto lead = *pointer / 94 + 0x21;
// 14. Let trail be pointer % 94 + 0x21.
auto trail = *pointer % 94 + 0x21;
// 15. Return two bytes whose values are lead and trail.
TRY(on_byte(static_cast<u8>(lead)));
TRY(on_byte(static_cast<u8>(trail)));
return state;
}
// https://encoding.spec.whatwg.org/#iso-2022-jp-encoder
ErrorOr<void> ISO2022JPEncoder::process(Utf8View input, Function<ErrorOr<void>(u8)> on_byte, Function<ErrorOr<void>(u32)> on_error)
{
// ISO-2022-JP’s encoder has an associated ISO-2022-JP encoder state which is ASCII, Roman, or jis0208 (initially ASCII).
auto state = State::ASCII;
for (u32 item : input) {
state = TRY(process_item(item, state, on_byte, on_error));
}
// 1. If code point is end-of-queue and ISO-2022-JP encoder state is not ASCII, set ISO-2022-JP
// encoder state to ASCII, and return three bytes 0x1B 0x28 0x42.
if (state != State::ASCII) {
state = State::ASCII;
TRY(on_byte(0x1B));
TRY(on_byte(0x28));
TRY(on_byte(0x42));
return {};
}
// 2. If code point is end-of-queue and ISO-2022-JP encoder state is ASCII, return finished.
return {};
}
static Optional<u32> code_point_jis0208_index_skipping_range(u32 code_point, u32 skip_from, u32 skip_to)
{
VERIFY(skip_to >= skip_from);
for (u32 i = 0; i < s_jis0208_index.size(); ++i) {
if (i >= skip_from && i <= skip_to)
continue;
if (s_jis0208_index[i] == code_point)
return i;
}
return {};
}
// https://encoding.spec.whatwg.org/#index-shift_jis-pointer
static Optional<u32> index_shift_jis_pointer(u32 code_point)
{
// 1. Let index be index jis0208 excluding all entries whose pointer is in the range 8272 to 8835, inclusive.
auto pointer = code_point_jis0208_index_skipping_range(code_point, 8272, 8835);
if (!pointer.has_value())
return {};
// 2. Return the index pointer for code point in index.
return *pointer;
}
// https://encoding.spec.whatwg.org/#shift_jis-encoder
ErrorOr<void> ShiftJISEncoder::process(Utf8View input, Function<ErrorOr<void>(u8)> on_byte, Function<ErrorOr<void>(u32)> on_error)
{
for (u32 item : input) {
// 1. If code point is end-of-queue, return finished.
// 2. If code point is an ASCII code point or U+0080, return a byte whose value is code point.
if (is_ascii(item) || item == 0x0080) {
TRY(on_byte(static_cast<u8>(item)));
continue;
}
// 3. If code point is U+00A5, return byte 0x5C.
if (item == 0x00A5) {
TRY(on_byte(0x5C));
continue;
}
// 4. If code point is U+203E, return byte 0x7E.
if (item == 0x203E) {
TRY(on_byte(0x7E));
continue;
}
// 5. If code point is in the range U+FF61 to U+FF9F, inclusive, return a byte whose value is code point − 0xFF61 + 0xA1.
if (item >= 0xFF61 && item <= 0xFF9F) {
TRY(on_byte(static_cast<u8>(item - 0xFF61 + 0xA1)));
continue;
}
// 6. If code point is U+2212, set it to U+FF0D.
if (item == 0x2212)
item = 0xFF0D;
// 7. Let pointer be the index Shift_JIS pointer for code point.
auto pointer = index_shift_jis_pointer(item);
// 8. If pointer is null, return error with code point.
if (!pointer.has_value()) {
TRY(on_error(item));
continue;
}
// 9. Let lead be pointer / 188.
auto lead = *pointer / 188;
// 10. Let lead offset be 0x81 if lead is less than 0x1F, otherwise 0xC1.
auto lead_offset = 0xC1;
if (lead < 0x1F)
lead_offset = 0x81;
// 11. Let trail be pointer % 188.
auto trail = *pointer % 188;
// 12. Let offset be 0x40 if trail is less than 0x3F, otherwise 0x41.
auto offset = 0x41;
if (trail < 0x3F)
offset = 0x40;
// 13. Return two bytes whose values are lead + lead offset and trail + offset.
TRY(on_byte(static_cast<u8>(lead + lead_offset)));
TRY(on_byte(static_cast<u8>(trail + offset)));
}
return {};
}
// https://encoding.spec.whatwg.org/#euc-kr-encoder
ErrorOr<void> EUCKREncoder::process(Utf8View input, Function<ErrorOr<void>(u8)> on_byte, Function<ErrorOr<void>(u32)> on_error)
{
for (u32 item : input) {
// 1. If code point is end-of-queue, return finished.
// 2. If code point is an ASCII code point, return a byte whose value is code point.
if (is_ascii(item)) {
TRY(on_byte(static_cast<u8>(item)));
continue;
}
// 3. Let pointer be the index pointer for code point in index EUC-KR.
auto pointer = code_point_euc_kr_index(item);
// 4. If pointer is null, return error with code point.
if (!pointer.has_value()) {
TRY(on_error(item));
continue;
}
// 5. Let lead be pointer / 190 + 0x81.
auto lead = *pointer / 190 + 0x81;
// 6. Let trail be pointer % 190 + 0x41.
auto trail = *pointer % 190 + 0x41;
// 7. Return two bytes whose values are lead and trail.
TRY(on_byte(static_cast<u8>(lead)));
TRY(on_byte(static_cast<u8>(trail)));
}
return {};
}
// https://encoding.spec.whatwg.org/#index-big5-pointer
static Optional<u32> index_big5_pointer(u32 code_point)
{
// 1. Let index be index Big5 excluding all entries whose pointer is less than (0xA1 - 0x81) × 157.
auto start_index = (0xA1 - 0x81) * 157 - s_big5_index_first_pointer;
// 2. If code point is U+2550, U+255E, U+2561, U+256A, U+5341, or U+5345, return the last pointer
// corresponding to code point in index.
if (Array<u32, 6> { 0x2550, 0x255E, 0x2561, 0x256A, 0x5341, 0x5345 }.contains_slow(code_point)) {
for (u32 i = s_big5_index.size() - 1; i >= start_index; --i) {
if (s_big5_index[i] == code_point) {
return s_big5_index_first_pointer + i;
}
}
return {};
}
// 3. Return the index pointer for code point in index.
for (u32 i = start_index; i < s_big5_index.size(); ++i) {
if (s_big5_index[i] == code_point) {
return s_big5_index_first_pointer + i;
}
}
return {};
}
// https://encoding.spec.whatwg.org/#big5-encoder
ErrorOr<void> Big5Encoder::process(Utf8View input, Function<ErrorOr<void>(u8)> on_byte, Function<ErrorOr<void>(u32)> on_error)
{
for (u32 item : input) {
// 1. If code point is end-of-queue, return finished.
// 2. If code point is an ASCII code point, return a byte whose value is code point.
if (is_ascii(item)) {
TRY(on_byte(static_cast<u8>(item)));
continue;
}
// 3. Let pointer be the index Big5 pointer for code point.
auto pointer = index_big5_pointer(item);
// 4. If pointer is null, return error with code point.
if (!pointer.has_value()) {
TRY(on_error(item));
continue;
}
// 5. Let lead be pointer / 157 + 0x81.
auto lead = *pointer / 157 + 0x81;
// 6. Let trail be pointer % 157.
auto trail = *pointer % 157;
// 7. Let offset be 0x40 if trail is less than 0x3F, otherwise 0x62.
auto offset = 0x62;
if (trail < 0x3f)
offset = 0x40;
// 8. Return two bytes whose values are lead and trail + offset.
TRY(on_byte(static_cast<u8>(lead)));
TRY(on_byte(static_cast<u8>(trail + offset)));
}
return {};
}
// https://encoding.spec.whatwg.org/#index-gb18030-ranges-pointer
static u32 index_gb18030_ranges_pointer(u32 code_point)
{
// 1. If code point is U+E7C7, return pointer 7457.
if (code_point == 0xe7c7)
return 7457;
// 2. Let offset be the last code point in index gb18030 ranges that is less than
// or equal to code point and let pointer offset be its corresponding pointer.
size_t last_index;
binary_search(s_gb18030_ranges, code_point, &last_index, [](auto const code_point, auto const& entry) {
return code_point - entry.code_point;
});
auto offset = s_gb18030_ranges[last_index].code_point;
auto pointer_offset = s_gb18030_ranges[last_index].pointer;
// 3. Return a pointer whose value is pointer offset + code point − offset.
return pointer_offset + code_point - offset;
}
GB18030Encoder::GB18030Encoder(IsGBK is_gbk)
: m_is_gbk(is_gbk)
{
}
// https://encoding.spec.whatwg.org/#gb18030-encoder
ErrorOr<void> GB18030Encoder::process(Utf8View input, Function<ErrorOr<void>(u8)> on_byte, Function<ErrorOr<void>(u32)> on_error)
{
bool gbk = (m_is_gbk == IsGBK::Yes);
for (u32 item : input) {
// 1. If code point is end-of-queue, return finished.
// 2. If code point is an ASCII code point, return a byte whose value is code point.
if (is_ascii(item)) {
TRY(on_byte(static_cast<u8>(item)));
continue;
}
// 3. If code point is U+E5E5, return error with code point.
if (item == 0xE5E5) {
TRY(on_error(item));
continue;
}
// 4. If is GBK is true and code point is U+20AC, return byte 0x80.
if (gbk && item == 0x20AC) {
TRY(on_byte(0x80));
continue;
}
// 5. If there is a row in the table below whose first column is codePoint, then return the two bytes on the same row listed in the second column:
// Code point Bytes
// U+E78D 0xA6 0xD9
// U+E78E 0xA6 0xDA
// U+E78F 0xA6 0xDB
// U+E790 0xA6 0xDC
// U+E791 0xA6 0xDD
// U+E792 0xA6 0xDE
// U+E793 0xA6 0xDF
// U+E794 0xA6 0xEC
// U+E795 0xA6 0xED
// U+E796 0xA6 0xF3
// U+E81E 0xFE 0x59
// U+E826 0xFE 0x61
// U+E82B 0xFE 0x66
// U+E82C 0xFE 0x67
// U+E832 0xFE 0x6D
// U+E843 0xFE 0x7E
// U+E854 0xFE 0x90
// U+E864 0xFE 0xA0
switch (item) {
case 0xE78D:
TRY(on_byte(0xA6));
TRY(on_byte(0xD9));
continue;
case 0xE78E:
TRY(on_byte(0xA6));
TRY(on_byte(0xDA));
continue;
case 0xE78F:
TRY(on_byte(0xA6));
TRY(on_byte(0xDB));
continue;
case 0xE790:
TRY(on_byte(0xA6));
TRY(on_byte(0xDC));
continue;
case 0xE791:
TRY(on_byte(0xA6));
TRY(on_byte(0xDD));
continue;
case 0xE792:
TRY(on_byte(0xA6));
TRY(on_byte(0xDE));
continue;
case 0xE793:
TRY(on_byte(0xA6));
TRY(on_byte(0xDF));
continue;
case 0xE794:
TRY(on_byte(0xA6));
TRY(on_byte(0xEC));
continue;
case 0xE795:
TRY(on_byte(0xA6));
TRY(on_byte(0xED));
continue;
case 0xE796:
TRY(on_byte(0xA6));
TRY(on_byte(0xF3));
continue;
case 0xE81E:
TRY(on_byte(0xFE));
TRY(on_byte(0x59));
continue;
case 0xE826:
TRY(on_byte(0xFE));
TRY(on_byte(0x61));
continue;
case 0xE82B:
TRY(on_byte(0xFE));
TRY(on_byte(0x66));
continue;
case 0xE82C:
TRY(on_byte(0xFE));
TRY(on_byte(0x67));
continue;
case 0xE832:
TRY(on_byte(0xFE));
TRY(on_byte(0x6D));
continue;
case 0xE843:
TRY(on_byte(0xFE));
TRY(on_byte(0x7E));
continue;
case 0xE854:
TRY(on_byte(0xFE));
TRY(on_byte(0x90));
continue;
case 0xE864:
TRY(on_byte(0xFE));
TRY(on_byte(0xA0));
continue;
}
// 6. Let pointer be the index pointer for code point in index gb18030.
auto pointer = code_point_gb18030_index(item);
// 7. If pointer is non-null, then:
if (pointer.has_value()) {
// 1. Let lead be pointer / 190 + 0x81.
auto lead = *pointer / 190 + 0x81;
// 2. Let trail be pointer % 190.
auto trail = *pointer % 190;
// 3. Let offset be 0x40 if trail is less than 0x3F, otherwise 0x41.
auto offset = 0x41;
if (trail < 0x3f)
offset = 0x40;
// 4. Return two bytes whose values are lead and trail + offset.
TRY(on_byte(static_cast<u8>(lead)));
TRY(on_byte(static_cast<u8>(trail + offset)));
continue;
}
// 8. If is GBK is true, return error with code point.
if (gbk) {
TRY(on_error(item));
continue;
}
// 9. Set pointer to the index gb18030 ranges pointer for code point.
pointer = index_gb18030_ranges_pointer(item);
// 10. Let byte1 be pointer / (10 × 126 × 10).
auto byte1 = *pointer / (10 * 126 * 10);
// 11. Set pointer to pointer % (10 × 126 × 10).
pointer = *pointer % (10 * 126 * 10);
// 12. Let byte2 be pointer / (10 × 126).
auto byte2 = *pointer / (10 * 126);
// 13. Set pointer to pointer % (10 × 126).
pointer = *pointer % (10 * 126);
// 14. Let byte3 be pointer / 10.
auto byte3 = *pointer / 10;
// 15. Let byte4 be pointer % 10.
auto byte4 = *pointer % 10;
// 16. Return four bytes whose values are byte1 + 0x81, byte2 + 0x30, byte3 + 0x81, byte4 + 0x30.
TRY(on_byte(static_cast<u8>(byte1 + 0x81)));
TRY(on_byte(static_cast<u8>(byte2 + 0x30)));
TRY(on_byte(static_cast<u8>(byte3 + 0x81)));
TRY(on_byte(static_cast<u8>(byte4 + 0x30)));
}
return {};
}
// https://encoding.spec.whatwg.org/#single-byte-encoder
template<Integral ArrayType>
ErrorOr<void> SingleByteEncoder<ArrayType>::process(Utf8View input, Function<ErrorOr<void>(u8)> on_byte, Function<ErrorOr<void>(u32)> on_error)
{
for (u32 const code_point : input) {
if (code_point < 0x80) {
// 2. If code point is an ASCII code point, return a byte whose value is code point.
TRY(on_byte(static_cast<u8>(code_point)));
} else {
Optional<u8> pointer = {};
for (u8 i = 0; i < m_translation_table.size(); i++) {
if (m_translation_table[i] == code_point) {
// 3. Let pointer be the index pointer for code point in index single-byte.
pointer = i;
break;
}
}
if (pointer.has_value()) {
// 5. Return a byte whose value is pointer + 0x80.
TRY(on_byte(pointer.value() + 0x80));
} else {
// 4. If pointer is null, return error with code point.
TRY(on_error(code_point));
}
}
}
// 1. If code point is end-of-queue, return finished.
return {};
}
// https://infra.spec.whatwg.org/#isomorphic-encode
ByteString isomorphic_encode(StringView input)
{

67
Libraries/LibTextCodec/Encoder.h
View file

@ -21,73 +21,6 @@ protected:
virtual ~Encoder() = default;
};
class TEXTCODEC_API UTF8Encoder final : public Encoder {
public:
virtual ErrorOr<void> process(Utf8View, Function<ErrorOr<void>(u8)> on_byte, Function<ErrorOr<void>(u32)> on_error) override;
};
class TEXTCODEC_API EUCJPEncoder final : public Encoder {
public:
virtual ErrorOr<void> process(Utf8View, Function<ErrorOr<void>(u8)> on_byte, Function<ErrorOr<void>(u32)> on_error) override;
};
class TEXTCODEC_API ISO2022JPEncoder final : public Encoder {
public:
virtual ErrorOr<void> process(Utf8View, Function<ErrorOr<void>(u8)> on_byte, Function<ErrorOr<void>(u32)> on_error) override;
private:
enum class State {
ASCII,
Roman,
jis0208,
};
ErrorOr<State> process_item(u32 item, State, Function<ErrorOr<void>(u8)>& on_byte, Function<ErrorOr<void>(u32)>& on_error);
};
class TEXTCODEC_API ShiftJISEncoder final : public Encoder {
public:
virtual ErrorOr<void> process(Utf8View, Function<ErrorOr<void>(u8)> on_byte, Function<ErrorOr<void>(u32)> on_error) override;
};
class TEXTCODEC_API EUCKREncoder final : public Encoder {
public:
virtual ErrorOr<void> process(Utf8View, Function<ErrorOr<void>(u8)> on_byte, Function<ErrorOr<void>(u32)> on_error) override;
};
class TEXTCODEC_API Big5Encoder final : public Encoder {
public:
virtual ErrorOr<void> process(Utf8View, Function<ErrorOr<void>(u8)> on_byte, Function<ErrorOr<void>(u32)> on_error) override;
};
class TEXTCODEC_API GB18030Encoder final : public Encoder {
public:
enum class IsGBK {
Yes,
No,
};
GB18030Encoder(IsGBK is_gbk = IsGBK::No);
virtual ErrorOr<void> process(Utf8View, Function<ErrorOr<void>(u8)> on_byte, Function<ErrorOr<void>(u32)> on_error) override;
private:
IsGBK m_is_gbk { IsGBK::No };
};
template<Integral ArrayType = u32>
class SingleByteEncoder final : public Encoder {
public:
SingleByteEncoder(Array<ArrayType, 128> translation_table)
: m_translation_table(translation_table)
{
}
virtual ErrorOr<void> process(Utf8View, Function<ErrorOr<void>(u8)> on_byte, Function<ErrorOr<void>(u32)> on_error) override;
private:
Array<ArrayType, 128> m_translation_table;
};
TEXTCODEC_API Optional<Encoder&> encoder_for_exact_name(StringView encoding);
TEXTCODEC_API Optional<Encoder&> encoder_for(StringView label);

16
Libraries/LibTextCodec/Rust/Cargo.toml Normal file
View file

@ -0,0 +1,16 @@
[package]
name = "libtextcodec_rust"
version = "0.1.0"
edition = "2024"
[lib]
crate-type = ["staticlib"]
[dependencies]
encoding_rs = "0.8.35"
[build-dependencies]
cbindgen = "0.29"
[lints]
workspace = true

40
Libraries/LibTextCodec/Rust/build.rs Normal file
View file

@ -0,0 +1,40 @@
/*
* Copyright (c) 2026-present, the Ladybird developers.
*
* SPDX-License-Identifier: BSD-2-Clause
*/
use std::env;
use std::error::Error;
use std::path::PathBuf;
fn main() -> Result<(), Box<dyn Error>> {
let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR")?);
let out_dir = PathBuf::from(env::var("OUT_DIR")?);
println!("cargo:rerun-if-changed=build.rs");
println!("cargo:rerun-if-changed=cbindgen.toml");
println!("cargo:rerun-if-env-changed=FFI_OUTPUT_DIR");
println!("cargo:rerun-if-changed=src");
let ffi_out_dir = env::var("FFI_OUTPUT_DIR")
.map(PathBuf::from)
.unwrap_or_else(|_| out_dir.clone());
cbindgen::generate(manifest_dir).map_or_else(
|error| match error {
cbindgen::Error::ParseSyntaxError { .. } => {}
e => panic!("{e:?}"),
},
|bindings| {
let header_path = out_dir.join("RustFFI.h");
bindings.write_to_file(&header_path);
if ffi_out_dir != out_dir {
bindings.write_to_file(ffi_out_dir.join("RustFFI.h"));
}
},
);
Ok(())
}

17
Libraries/LibTextCodec/Rust/cbindgen.toml Normal file
View file

@ -0,0 +1,17 @@
language = "C++"
header = """/*
* Copyright (c) 2026-present, the Ladybird developers.
*
* SPDX-License-Identifier: BSD-2-Clause
*/"""
pragma_once = true
include_version = true
namespaces = ["TextCodec", "FFI"]
line_length = 120
tab_width = 4
no_includes = true
sys_includes = ["stdint.h", "stddef.h"]
usize_is_size_t = true
[export.mangle]
rename_types = "PascalCase"

298
Libraries/LibTextCodec/Rust/src/lib.rs Normal file
View file

@ -0,0 +1,298 @@
/*
* Copyright (c) 2026-present, the Ladybird developers.
*
* SPDX-License-Identifier: BSD-2-Clause
*/
use encoding_rs::CoderResult;
use encoding_rs::EncoderResult;
use encoding_rs::Encoding;
use std::ffi::c_void;
use std::panic::AssertUnwindSafe;
use std::panic::catch_unwind;
type FfiBytesFn = unsafe extern "C" fn(ctx: *mut c_void, data: *const u8, len: usize);
type FfiCodePointFn = unsafe extern "C" fn(ctx: *mut c_void, code_point: u32);
pub struct TextCodecRustStreamingDecoder {
decoder: encoding_rs::Decoder,
}
fn abort_on_panic<F: FnOnce() -> R, R>(f: F) -> R {
match catch_unwind(AssertUnwindSafe(f)) {
Ok(result) => result,
Err(payload) => {
let message = if let Some(message) = payload.downcast_ref::<&str>() {
(*message).to_string()
} else if let Some(message) = payload.downcast_ref::<String>() {
message.clone()
} else {
"unknown panic".to_string()
};
eprintln!("Rust panic at FFI boundary: {message}");
std::process::abort();
}
}
}
unsafe fn bytes_from_raw<'a>(bytes: *const u8, len: usize) -> Option<&'a [u8]> {
unsafe {
if len == 0 {
return Some(&[]);
}
if bytes.is_null() {
eprintln!("bytes_from_raw: null pointer with non-zero length {len}");
return None;
}
Some(std::slice::from_raw_parts(bytes, len))
}
}
unsafe fn set_static_string(out: *mut *const u8, out_len: *mut usize, value: &'static str) -> bool {
unsafe {
if out.is_null() || out_len.is_null() {
eprintln!("set_static_string: null output pointer");
return false;
}
*out = value.as_ptr();
*out_len = value.len();
true
}
}
/// # Safety
/// - `label`/`label_len` must be a valid byte slice.
/// - `out_name` and `out_name_len` must be valid writable pointers.
#[unsafe(no_mangle)]
pub unsafe extern "C" fn textcodec_rust_get_standardized_encoding(
label: *const u8,
label_len: usize,
out_name: *mut *const u8,
out_name_len: *mut usize,
) -> bool {
unsafe {
abort_on_panic(|| {
let Some(label) = bytes_from_raw(label, label_len) else {
return false;
};
let Some(encoding) = Encoding::for_label(label) else {
return false;
};
set_static_string(out_name, out_name_len, encoding.name())
})
}
}
/// # Safety
/// - `encoding_label`/`encoding_label_len` and `input`/`input_len` must be valid byte slices.
/// - `on_bytes` must not retain `data` beyond the duration of the callback.
#[unsafe(no_mangle)]
pub unsafe extern "C" fn textcodec_rust_decode_to_utf8(
encoding_label: *const u8,
encoding_label_len: usize,
input: *const u8,
input_len: usize,
remove_bom: bool,
ctx: *mut c_void,
on_bytes: FfiBytesFn,
) -> bool {
unsafe {
abort_on_panic(|| {
let Some(label) = bytes_from_raw(encoding_label, encoding_label_len) else {
return false;
};
let Some(input) = bytes_from_raw(input, input_len) else {
return false;
};
let Some(encoding) = Encoding::for_label(label) else {
return false;
};
let (output, _) = if remove_bom {
encoding.decode_with_bom_removal(input)
} else {
encoding.decode_without_bom_handling(input)
};
on_bytes(ctx, output.as_bytes().as_ptr(), output.len());
true
})
}
}
/// # Safety
/// - `encoding_label`/`encoding_label_len` and `input`/`input_len` must be valid byte slices.
#[unsafe(no_mangle)]
pub unsafe extern "C" fn textcodec_rust_validate(
encoding_label: *const u8,
encoding_label_len: usize,
input: *const u8,
input_len: usize,
remove_bom: bool,
) -> bool {
unsafe {
abort_on_panic(|| {
let Some(label) = bytes_from_raw(encoding_label, encoding_label_len) else {
return false;
};
let Some(input) = bytes_from_raw(input, input_len) else {
return false;
};
let Some(encoding) = Encoding::for_label(label) else {
return false;
};
let input = if remove_bom {
if encoding == encoding_rs::UTF_8 && input.starts_with(b"\xEF\xBB\xBF") {
&input[3..]
} else if (encoding == encoding_rs::UTF_16LE && input.starts_with(b"\xFF\xFE"))
|| (encoding == encoding_rs::UTF_16BE && input.starts_with(b"\xFE\xFF"))
{
&input[2..]
} else {
input
}
} else {
input
};
encoding
.decode_without_bom_handling_and_without_replacement(input)
.is_some()
})
}
}
/// # Safety
/// - `encoding_label`/`encoding_label_len` must be a valid byte slice.
/// - The returned pointer must be freed with `textcodec_rust_streaming_decoder_free`.
#[unsafe(no_mangle)]
pub unsafe extern "C" fn textcodec_rust_streaming_decoder_new(
encoding_label: *const u8,
encoding_label_len: usize,
remove_bom: bool,
) -> *mut TextCodecRustStreamingDecoder {
unsafe {
abort_on_panic(|| {
let Some(label) = bytes_from_raw(encoding_label, encoding_label_len) else {
return std::ptr::null_mut();
};
let Some(encoding) = Encoding::for_label(label) else {
return std::ptr::null_mut();
};
let decoder = if remove_bom {
encoding.new_decoder_with_bom_removal()
} else {
encoding.new_decoder_without_bom_handling()
};
Box::into_raw(Box::new(TextCodecRustStreamingDecoder { decoder }))
})
}
}
/// # Safety
/// - `decoder` must be null or a pointer returned by `textcodec_rust_streaming_decoder_new`.
#[unsafe(no_mangle)]
pub unsafe extern "C" fn textcodec_rust_streaming_decoder_free(decoder: *mut TextCodecRustStreamingDecoder) {
unsafe {
abort_on_panic(|| {
if !decoder.is_null() {
drop(Box::from_raw(decoder));
}
});
}
}
/// # Safety
/// - `decoder` must be a valid pointer returned by `textcodec_rust_streaming_decoder_new`.
/// - `input`/`input_len` must be a valid byte slice.
/// - `on_bytes` must not retain `data` beyond the duration of the callback.
#[unsafe(no_mangle)]
pub unsafe extern "C" fn textcodec_rust_streaming_decoder_decode_to_utf8(
decoder: *mut TextCodecRustStreamingDecoder,
input: *const u8,
input_len: usize,
last: bool,
ctx: *mut c_void,
on_bytes: FfiBytesFn,
) -> bool {
unsafe {
abort_on_panic(|| {
if decoder.is_null() {
eprintln!("textcodec_rust_streaming_decoder_decode_to_utf8: null decoder pointer");
return false;
}
let Some(input) = bytes_from_raw(input, input_len) else {
return false;
};
let decoder = &mut *decoder;
let Some(output_capacity) = decoder.decoder.max_utf8_buffer_length(input.len()) else {
return false;
};
let mut output = String::with_capacity(output_capacity);
let (result, _, _) = decoder.decoder.decode_to_string(input, &mut output, last);
if !output.is_empty() {
on_bytes(ctx, output.as_ptr(), output.len());
}
matches!(result, CoderResult::InputEmpty)
})
}
}
/// # Safety
/// - `encoding_label`/`encoding_label_len` and `input`/`input_len` must be valid byte slices.
/// - `on_bytes` and `on_error` must not retain data beyond their callback duration.
#[unsafe(no_mangle)]
pub unsafe extern "C" fn textcodec_rust_encode_from_utf8(
encoding_label: *const u8,
encoding_label_len: usize,
input: *const u8,
input_len: usize,
ctx: *mut c_void,
on_bytes: FfiBytesFn,
on_error: FfiCodePointFn,
) -> bool {
unsafe {
abort_on_panic(|| {
let Some(label) = bytes_from_raw(encoding_label, encoding_label_len) else {
return false;
};
let Some(input) = bytes_from_raw(input, input_len) else {
return false;
};
let Ok(input) = std::str::from_utf8(input) else {
return false;
};
let Some(encoding) = Encoding::for_label(label) else {
return false;
};
let mut encoder = encoding.new_encoder();
let mut total_read = 0usize;
let Some(output_capacity) = encoder.max_buffer_length_from_utf8_without_replacement(input.len()) else {
return false;
};
let mut output = Vec::with_capacity(output_capacity);
loop {
let (result, read) =
encoder.encode_from_utf8_to_vec_without_replacement(&input[total_read..], &mut output, true);
total_read += read;
if !output.is_empty() {
on_bytes(ctx, output.as_ptr(), output.len());
output.clear();
}
match result {
EncoderResult::InputEmpty => return true,
EncoderResult::OutputFull => return false,
EncoderResult::Unmappable(unmappable) => {
on_error(ctx, unmappable as u32);
}
}
}
})
}
}

36
Libraries/LibTextCodec/indexes.json
View file

File diff suppressed because one or more lines are too long

97
Libraries/LibWeb/Encoding/TextDecoderStream.cpp
View file

@ -24,72 +24,6 @@ namespace Web::Encoding {
GC_DEFINE_ALLOCATOR(TextDecoderStream);
static bool is_utf8_continuation_byte(u8 byte)
{
return (byte & 0xc0) == 0x80;
}
static bool is_utf8_second_byte_in_range(u8 lead, u8 byte)
{
if (!is_utf8_continuation_byte(byte))
return false;
if (lead == 0xe0)
return byte >= 0xa0;
if (lead == 0xed)
return byte <= 0x9f;
if (lead == 0xf0)
return byte >= 0x90;
if (lead == 0xf4)
return byte <= 0x8f;
return true;
}
// Returns the largest prefix length of `bytes` that can be safely decoded as UTF-8 without splitting an in-progress
// multi-byte sequence. The remainder (if any) is held over for the next chunk.
static size_t find_utf8_safe_decode_boundary(ReadonlyBytes bytes)
{
// A valid UTF-8 sequence is at most 4 bytes long, so we never need to look back more than 3 continuation bytes
// to find the leading byte of the trailing sequence.
size_t scan = 0;
while (scan < bytes.size() && scan < 4) {
size_t pos = bytes.size() - scan - 1;
u8 byte = bytes[pos];
// Continuation byte (10xxxxxx): keep walking back to find the leading byte.
if (is_utf8_continuation_byte(byte)) {
++scan;
continue;
}
// ASCII byte (0xxxxxxx): the trailing sequence ends here and is complete.
if ((byte & 0x80) == 0)
return pos + 1;
// Multi-byte leading byte. If it's a recognized leading byte and the buffer doesn't yet hold the full
// sequence, cut before it so the next chunk can complete it. Otherwise (recognized and complete, or
// unrecognized so it'll just become a replacement character) include all bytes up to the end.
size_t expected_length = 0;
if (byte >= 0xc2 && byte <= 0xdf)
expected_length = 2;
else if (byte >= 0xe0 && byte <= 0xef)
expected_length = 3;
else if (byte >= 0xf0 && byte <= 0xf4)
expected_length = 4;
else
return bytes.size();
if (bytes.size() - pos >= 2 && !is_utf8_second_byte_in_range(byte, bytes[pos + 1]))
return bytes.size();
if (bytes.size() - pos >= expected_length)
return bytes.size();
return pos;
}
// No leading byte found within the last 4 bytes. Either the buffer is shorter than that, or it ends with 4
// continuation bytes (malformed UTF-8). Either way, decode everything; the decoder will produce replacement
// characters as needed.
return bytes.size();
}
// https://encoding.spec.whatwg.org/#dom-textdecoderstream
WebIDL::ExceptionOr<GC::Ref<TextDecoderStream>> TextDecoderStream::construct_impl(JS::Realm& realm, FlyString label, Bindings::TextDecoderOptions const& options)
{
@ -151,6 +85,7 @@ TextDecoderStream::TextDecoderStream(JS::Realm& realm, GC::Ref<Streams::Transfor
: Bindings::PlatformObject(realm)
, Streams::GenericTransformStreamMixin(transform)
, TextDecoderCommonMixin(decoder, move(encoding), error_mode, ignore_bom)
, m_streaming_decoder(make<TextCodec::StreamingDecoder>(m_encoding))
{
}
@ -183,24 +118,8 @@ WebIDL::ExceptionOr<void> TextDecoderStream::decode_and_enqueue_chunk(JS::Value
if (buffer_or_error.is_error())
return WebIDL::OperationError::create(realm, "Failed to copy bytes from BufferSource"_utf16);
auto buffer = buffer_or_error.release_value();
m_io_queue.append(buffer.bytes());
// NB: Only decode the prefix of m_io_queue that doesn't end mid-multi-byte-sequence; the remainder is held over
// for the next chunk so we don't emit spurious replacement characters at chunk boundaries. We currently only
// do this boundary search for UTF-8; the underlying decoders for other encodings are stateless across calls
// so for those we just decode whatever's in the queue and don't carry anything over.
auto safe_length = (m_encoding == "utf-8"_fly_string)
? find_utf8_safe_decode_boundary(m_io_queue.bytes())
: m_io_queue.size();
if (safe_length == 0)
return {};
auto decoded = TRY_OR_THROW_OOM(vm, m_decoder.to_utf8(StringView { m_io_queue.data(), safe_length }));
auto remaining = m_io_queue.size() - safe_length;
if (remaining > 0)
memmove(m_io_queue.data(), m_io_queue.data() + safe_length, remaining);
m_io_queue.resize(remaining);
auto decoded = TRY_OR_THROW_OOM(vm, m_streaming_decoder->to_utf8(buffer.bytes()));
// 3-4. Run "processing an item" until the input is exhausted, accumulating the output, then enqueue any non-empty
// result. If processing returns error, throw a TypeError.
@ -211,17 +130,7 @@ WebIDL::ExceptionOr<void> TextDecoderStream::decode_and_enqueue_chunk(JS::Value
WebIDL::ExceptionOr<void> TextDecoderStream::flush_and_enqueue()
{
// 1-3. Drain decoder's I/O queue and run "processing an item" to completion.
// NB: For UTF-8, anything still in the I/O queue here is exactly the trailing partial sequence that
// decode_and_enqueue_chunk held back at the safe boundary. The WHATWG UTF-8 decoder emits a single replacement
// character for the whole incomplete sequence, so emit exactly one rather than letting the underlying decoder
// produce one per stray byte.
String decoded;
if (!m_io_queue.is_empty()) {
decoded = "\xEF\xBF\xBD"_string;
m_io_queue.clear();
}
auto decoded = TRY_OR_THROW_OOM(vm(), m_streaming_decoder->finish());
return enqueue_decoded_output(decoded);
}

6
Libraries/LibWeb/Encoding/TextDecoderStream.h
View file

@ -6,7 +6,7 @@
#pragma once
#include <AK/ByteBuffer.h>
#include <AK/OwnPtr.h>
#include <LibWeb/Bindings/PlatformObject.h>
#include <LibWeb/Encoding/TextDecoder.h>
#include <LibWeb/Encoding/TextDecoderCommon.h>
@ -38,9 +38,7 @@ private:
WebIDL::ExceptionOr<void> enqueue_decoded_output(String const&);
// https://encoding.spec.whatwg.org/#textdecodercommon-i-o-queue
// NB: We accumulate input bytes that have been pushed to the I/O queue but not yet decoded, so that a multi-byte
// sequence which is split across chunks can be reassembled.
ByteBuffer m_io_queue;
NonnullOwnPtr<TextCodec::StreamingDecoder> m_streaming_decoder;
};
}

17
Meta/CMake/libtextcodec_generators.cmake
View file

@ -1,17 +0,0 @@
function(generate_encoding_indexes)
set(LIBTEXTCODEC_INPUT_FOLDER "${CMAKE_CURRENT_SOURCE_DIR}")
# indexes.json can be found at https://encoding.spec.whatwg.org/indexes.json
invoke_py_generator(
"LookupTables.cpp"
"generate_encoding_indexes.py"
"${LIBTEXTCODEC_INPUT_FOLDER}/indexes.json"
"LookupTables.h"
"LookupTables.cpp"
arguments -j "${LIBTEXTCODEC_INPUT_FOLDER}/indexes.json"
)
if(ENABLE_INSTALL_HEADERS)
install(FILES "${CMAKE_CURRENT_BINARY_DIR}/LookupTables.h" DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/LibTextCodec/")
endif()
endfunction()

281
Meta/Generators/generate_encoding_indexes.py
View file

@ -1,281 +0,0 @@
#!/usr/bin/env python3
# Copyright (c) 2024, Simon Wanner <simon@skyrising.xyz>
# Copyright (c) 2025, ayeteadoe <ayeteadoe@gmail.com>
#
# SPDX-License-Identifier: BSD-2-Clause
import argparse
import json
from dataclasses import dataclass
from enum import Enum
from pathlib import Path
from typing import Any
class GenerateAccessor(Enum):
NO = False
YES = True
class GenerateInverseAccessor(Enum):
NO = False
YES = True
@dataclass
class LookupTable:
first_pointer: int
max_code_point: int
code_points: list[int]
generate_accessor: GenerateAccessor
generate_inverse_accessor: GenerateInverseAccessor
@dataclass
class LookupTables:
gb18030_ranges: list[Any]
indexes: dict[str, LookupTable]
def prepare_table(
data: list[Any],
generate_accessor: GenerateAccessor = GenerateAccessor.NO,
) -> LookupTable:
code_points = []
max_code_point = 0
first_pointer = 0
for entry in data:
if entry is None:
if not code_points:
first_pointer += 1
else:
code_points.append(0xFFFD)
max_code_point = max(max_code_point, code_points[-1])
else:
code_points.append(int(entry))
max_code_point = max(max_code_point, code_points[-1])
if generate_accessor == GenerateAccessor.YES:
while code_points and code_points[-1] == 0xFFFD:
code_points.pop()
else:
assert first_pointer == 0
return LookupTable(
first_pointer=first_pointer,
max_code_point=max_code_point,
code_points=code_points,
generate_accessor=GenerateAccessor.YES,
generate_inverse_accessor=GenerateInverseAccessor.YES,
)
def generate_table(name: str, table: LookupTable) -> str:
max_u16 = (1 << 16) - 1
value_type = "u32" if table.max_code_point > max_u16 else "u16"
size = len(table.code_points)
lines = []
if table.first_pointer > 0:
lines.append(f"static constexpr u32 s_{name}_index_first_pointer = {table.first_pointer};")
lines.append(f"static constexpr Array<{value_type}, {size}> s_{name}_index {{")
formatted_points = []
for i, point in enumerate(table.code_points):
formatted_points.append(f"0x{point:04x}")
if i != len(table.code_points) - 1:
if i % 16 == 15:
formatted_points.append(",\n ")
else:
formatted_points.append(", ")
lines.append(f" {' '.join(formatted_points)}")
lines.append("};")
if table.generate_accessor:
lines.append(f"Optional<u32> index_{name}_code_point(u32 pointer);")
if table.generate_inverse_accessor:
lines.append(f"Optional<u32> code_point_{name}_index(u32 code_point);")
return "\n".join(lines)
def generate_header_file(tables: LookupTables, output_path: Path) -> None:
gb18030_ranges_size = len(tables.gb18030_ranges)
content = f"""#pragma once
#include <AK/Array.h>
#include <AK/Types.h>
namespace TextCodec {{
struct Gb18030RangeEntry {{
u32 pointer;
u32 code_point;
}};
static constexpr Array<Gb18030RangeEntry, {gb18030_ranges_size}> s_gb18030_ranges {{ {{
"""
for range_entry in tables.gb18030_ranges:
pointer = range_entry[0]
code_point = range_entry[1]
content += f" {{ {pointer}, 0x{code_point:04x} }},\n"
content += "} };\n\n"
for name, table in tables.indexes.items():
content += generate_table(name, table) + "\n\n"
content += "}\n"
with open(output_path, "w") as f:
f.write(content)
def generate_table_accessor(name: str, table: LookupTable) -> str:
if table.first_pointer > 0:
return f"""
Optional<u32> index_{name}_code_point(u32 pointer)
{{
if (pointer < s_{name}_index_first_pointer || pointer - s_{name}_index_first_pointer >= s_{name}_index.size())
return {{}};
auto value = s_{name}_index[pointer - s_{name}_index_first_pointer];
if (value == 0xfffd)
return {{}};
return value;
}}
"""
else:
return f"""
Optional<u32> index_{name}_code_point(u32 pointer)
{{
if (pointer >= s_{name}_index.size())
return {{}};
auto value = s_{name}_index[pointer];
if (value == 0xfffd)
return {{}};
return value;
}}
"""
def generate_inverse_table_accessor(name: str, table: LookupTable) -> str:
if table.first_pointer > 0:
return f"""
Optional<u32> code_point_{name}_index(u32 code_point)
{{
for (u32 i = 0; i < s_{name}_index.size(); ++i) {{
if (s_{name}_index[i] == code_point) {{
return s_{name}_index_first_pointer + i;
}}
}}
return {{}};
}}
"""
else:
return f"""
Optional<u32> code_point_{name}_index(u32 code_point)
{{
for (u32 i = 0; i < s_{name}_index.size(); ++i) {{
if (s_{name}_index[i] == code_point) {{
return i;
}}
}}
return {{}};
}}
"""
def generate_implementation_file(tables: LookupTables, output_path: Path) -> None:
content = """
#include <LibTextCodec/LookupTables.h>
namespace TextCodec {
"""
for name, table in tables.indexes.items():
if table.generate_accessor:
content += generate_table_accessor(name, table)
if table.generate_inverse_accessor:
content += generate_inverse_table_accessor(name, table)
content += "\n}\n"
with open(output_path, "w") as f:
f.write(content)
def main():
parser = argparse.ArgumentParser(description="Generate text codec lookup tables", add_help=False)
parser.add_argument("--help", action="help", help="Show this help message and exit")
parser.add_argument(
"-h", "--generated-header-path", required=True, help="Path to the lookup table header file to generate"
)
parser.add_argument(
"-c",
"--generated-implementation-path",
required=True,
help="Path to the lookup table implementation file to generate",
)
parser.add_argument("-j", "--json-path", required=True, help="Path to the JSON file to read from")
args = parser.parse_args()
with open(args.json_path, "r") as f:
data = json.load(f)
gb18030_table = prepare_table(data["gb18030"], GenerateAccessor.YES)
tables = LookupTables(
gb18030_ranges=data["gb18030-ranges"],
indexes={
"gb18030": gb18030_table,
"big5": prepare_table(data["big5"], GenerateAccessor.YES),
"jis0208": prepare_table(data["jis0208"], GenerateAccessor.YES),
"jis0212": prepare_table(data["jis0212"], GenerateAccessor.YES),
"euc_kr": prepare_table(data["euc-kr"], GenerateAccessor.YES),
"ibm866": prepare_table(data["ibm866"]),
"iso_2022_jp_katakana": prepare_table(data["iso-2022-jp-katakana"], GenerateAccessor.YES),
"iso_8859_2": prepare_table(data["iso-8859-2"]),
"iso_8859_3": prepare_table(data["iso-8859-3"]),
"iso_8859_4": prepare_table(data["iso-8859-4"]),
"iso_8859_5": prepare_table(data["iso-8859-5"]),
"iso_8859_6": prepare_table(data["iso-8859-6"]),
"iso_8859_7": prepare_table(data["iso-8859-7"]),
"iso_8859_8": prepare_table(data["iso-8859-8"]),
"iso_8859_10": prepare_table(data["iso-8859-10"]),
"iso_8859_13": prepare_table(data["iso-8859-13"]),
"iso_8859_14": prepare_table(data["iso-8859-14"]),
"iso_8859_15": prepare_table(data["iso-8859-15"]),
"iso_8859_16": prepare_table(data["iso-8859-16"]),
"koi8_r": prepare_table(data["koi8-r"]),
"koi8_u": prepare_table(data["koi8-u"]),
"macintosh": prepare_table(data["macintosh"]),
"windows_874": prepare_table(data["windows-874"]),
"windows_1250": prepare_table(data["windows-1250"]),
"windows_1251": prepare_table(data["windows-1251"]),
"windows_1252": prepare_table(data["windows-1252"]),
"windows_1253": prepare_table(data["windows-1253"]),
"windows_1254": prepare_table(data["windows-1254"]),
"windows_1255": prepare_table(data["windows-1255"]),
"windows_1256": prepare_table(data["windows-1256"]),
"windows_1257": prepare_table(data["windows-1257"]),
"windows_1258": prepare_table(data["windows-1258"]),
"x_mac_cyrillic": prepare_table(data["x-mac-cyrillic"]),
},
)
generate_header_file(tables, Path(args.generated_header_path))
generate_implementation_file(tables, Path(args.generated_implementation_path))
if __name__ == "__main__":
main()

5
Tests/LibWeb/Text/expected/wpt-import/encoding/textdecoder-eof.any.txt
View file

@ -2,6 +2,7 @@ Harness status: OK
Found 2 tests
2 Fail
Fail TextDecoder end-of-queue handling
1 Pass
1 Fail
Pass TextDecoder end-of-queue handling
Fail TextDecoder end-of-queue handling using stream: true

8
Tests/LibWeb/Text/expected/wpt-import/encoding/textdecoder-mistakes.any.txt
View file

@ -2,8 +2,8 @@ Harness status: OK
Found 87 tests
70 Pass
17 Fail
72 Pass
15 Fail
Pass Invalid Unicode input is replaced: utf-8
Pass Invalid Unicode input is replaced: utf-16le
Pass Invalid Unicode input is replaced: utf-16be
@ -25,8 +25,8 @@ Pass Fast path misdetection: windows-1257
Pass Fast path misdetection: windows-1258
Pass Fast path misdetection: latin1
Pass Fast path misdetection: ascii
Fail utf-16le does not produce more chars than truncated
Fail utf-16be does not produce more chars than truncated
Pass utf-16le does not produce more chars than truncated
Pass utf-16be does not produce more chars than truncated
Pass windows-1252 maps bytes outside of latin1: windows-1252
Pass windows-1252 maps bytes outside of latin1: latin1
Pass windows-1252 maps bytes outside of latin1: ascii