AK: Remove UTF-16 mode from StringBuilder

Remove the UTF-16 storage mode and make StringBuilder solely build
UTF-8 strings again. The remaining UTF-16 adoption path now goes
through Utf16StringBuilder, keeping the direct-adoption optimization
with the type that owns UTF-16 construction.
This commit is contained in:
Andreas Kling 2026-06-22 14:28:38 +02:00 committed by Andreas Kling
parent a8193e7572
commit 503453eb9c
7 changed files with 28 additions and 244 deletions

View file

@ -252,20 +252,14 @@ ErrorOr<void> StringBuilder::Buffer::try_ensure_capacity_slowpath(size_t new_cap
return {};
}
static constexpr size_t string_builder_prefix_size(StringBuilder::Mode mode)
static constexpr size_t string_builder_prefix_size()
{
switch (mode) {
case StringBuilder::Mode::UTF8:
return sizeof(Detail::StringData);
case StringBuilder::Mode::UTF16:
return Detail::Utf16StringData::offset_of_string_storage();
}
VERIFY_NOT_REACHED();
return sizeof(Detail::StringData);
}
void StringBuilder::initialize_buffer(Mode mode, size_t capacity)
void StringBuilder::initialize_buffer(size_t capacity)
{
auto prefix_size = string_builder_prefix_size(mode);
auto prefix_size = string_builder_prefix_size();
if (capacity > StringBuilder::inline_capacity)
m_buffer.ensure_capacity(prefix_size + capacity);
m_buffer.resize(prefix_size);
@ -273,27 +267,15 @@ void StringBuilder::initialize_buffer(Mode mode, size_t capacity)
StringBuilder::StringBuilder()
{
static constexpr auto prefix_size = string_builder_prefix_size(DEFAULT_MODE);
static constexpr auto prefix_size = string_builder_prefix_size();
static_assert(inline_capacity > prefix_size);
initialize_buffer(m_mode, inline_capacity);
initialize_buffer(inline_capacity);
}
StringBuilder::StringBuilder(size_t initial_capacity)
{
initialize_buffer(m_mode, initial_capacity);
}
StringBuilder::StringBuilder(Mode mode)
: m_mode(mode)
{
initialize_buffer(m_mode, inline_capacity);
}
StringBuilder::StringBuilder(Mode mode, size_t initial_capacity_in_code_units)
: m_mode(mode)
{
initialize_buffer(mode, initial_capacity_in_code_units * (mode == Mode::UTF8 ? 1 : 2));
initialize_buffer(initial_capacity);
}
inline ErrorOr<void> StringBuilder::will_append(size_t size_in_bytes)
@ -311,32 +293,9 @@ inline ErrorOr<void> StringBuilder::will_append(size_t size_in_bytes)
return {};
}
ErrorOr<void> StringBuilder::ensure_storage_is_utf16()
{
if (!exchange(m_utf16_builder_is_ascii, false))
return {};
if (is_empty())
return {};
auto ascii_length = this->length();
TRY(m_buffer.try_resize(m_buffer.size() + ascii_length));
Bytes source { data(), ascii_length };
Span<char16_t> target { reinterpret_cast<char16_t*>(data()), ascii_length };
for (size_t i = ascii_length; i > 0; --i) {
auto index = i - 1;
auto ch = static_cast<char16_t>(source[index]);
target.overwrite(index, &ch, sizeof(char16_t));
}
return {};
}
size_t StringBuilder::length() const
{
return m_buffer.size() - string_builder_prefix_size(m_mode);
return m_buffer.size() - string_builder_prefix_size();
}
bool StringBuilder::is_empty() const
@ -346,9 +305,6 @@ bool StringBuilder::is_empty() const
void StringBuilder::trim(size_t count)
{
if (m_mode == Mode::UTF16)
count *= 2;
auto decrease_count = min(m_buffer.size(), count);
m_buffer.resize(m_buffer.size() - decrease_count);
}
@ -358,17 +314,8 @@ ErrorOr<void> StringBuilder::try_append(StringView string)
if (string.is_empty())
return {};
if (m_mode == Mode::UTF8 || (m_utf16_builder_is_ascii && string.is_ascii())) {
TRY(will_append(string.length()));
TRY(m_buffer.try_append(string.characters_without_null_termination(), string.length()));
} else {
TRY(ensure_storage_is_utf16());
TRY(will_append(string.length() * 2));
for (auto code_point : Utf8View { string })
TRY(try_append_code_point(code_point));
}
TRY(will_append(string.length()));
TRY(m_buffer.try_append(string.characters_without_null_termination(), string.length()));
return {};
}
@ -382,52 +329,26 @@ ErrorOr<void> StringBuilder::try_append_ascii_without_validation(ReadonlyBytes s
if (string.is_empty())
return {};
if (m_mode == Mode::UTF8 || m_utf16_builder_is_ascii) {
TRY(m_buffer.try_append(string));
} else {
if (m_mode == Mode::UTF16) {
TRY(ensure_storage_is_utf16());
TRY(will_append(string.size() * 2));
} else {
TRY(will_append(string.size()));
}
for (auto code_point : Utf8View { string })
TRY(try_append_code_point(code_point));
}
TRY(will_append(string.size()));
TRY(m_buffer.try_append(string));
return {};
}
ErrorOr<void> StringBuilder::try_append(char ch)
{
if (m_mode == Mode::UTF8 || (m_utf16_builder_is_ascii && is_ascii(ch))) {
TRY(will_append(1));
TRY(m_buffer.try_append(ch));
} else {
TRY(ensure_storage_is_utf16());
TRY(try_append_code_unit(ch));
}
TRY(will_append(1));
TRY(m_buffer.try_append(ch));
return {};
}
ErrorOr<void> StringBuilder::try_append_code_unit(char16_t ch)
{
if (m_mode == Mode::UTF8 || (m_utf16_builder_is_ascii && is_ascii(ch))) {
TRY(try_append_code_point(ch));
} else {
TRY(ensure_storage_is_utf16());
TRY(will_append(2));
TRY(m_buffer.try_append(&ch, sizeof(ch)));
}
return {};
return try_append_code_point(ch);
}
ErrorOr<void> StringBuilder::try_append_repeated(char ch, size_t n)
{
auto append_as_utf8 = m_mode == Mode::UTF8 || (m_utf16_builder_is_ascii && is_ascii(ch));
TRY(will_append(n * (append_as_utf8 ? 1 : 2)));
TRY(will_append(n));
for (size_t i = 0; i < n; ++i)
TRY(try_append(ch));
@ -440,12 +361,7 @@ ErrorOr<void> StringBuilder::try_append_repeated(StringView string, size_t n)
if (string.is_empty())
return {};
if (m_mode == Mode::UTF8 || (m_utf16_builder_is_ascii && string.is_ascii())) {
TRY(will_append(string.length() * n));
} else {
auto utf16_length = simdutf::utf16_length_from_utf8(string.characters_without_null_termination(), string.length());
TRY(will_append(utf16_length * n * 2));
}
TRY(will_append(string.length() * n));
for (size_t i = 0; i < n; ++i)
TRY(try_append(string));
@ -458,15 +374,11 @@ ErrorOr<void> StringBuilder::try_append_repeated(Utf16View const& string, size_t
if (string.is_empty())
return {};
if (m_mode == Mode::UTF8 || (m_utf16_builder_is_ascii && string.is_ascii())) {
if (string.has_ascii_storage()) {
TRY(will_append(string.length_in_code_units() * n));
} else {
auto utf8_length = simdutf::utf8_length_from_utf16(string.utf16_span().data(), string.length_in_code_units());
TRY(will_append(utf8_length * n));
}
if (string.has_ascii_storage()) {
TRY(will_append(string.length_in_code_units() * n));
} else {
TRY(will_append(string.length_in_code_units() * n * 2));
auto utf8_length = simdutf::utf8_length_from_utf16(string.utf16_span().data(), string.length_in_code_units());
TRY(will_append(utf8_length * n));
}
for (size_t i = 0; i < n; ++i)
@ -522,7 +434,6 @@ ErrorOr<ByteBuffer> StringBuilder::to_byte_buffer() const
ByteString StringBuilder::to_byte_string() const
{
VERIFY(m_mode == Mode::UTF8);
if (is_empty())
return ByteString::empty();
return ByteString((char const*)data(), length());
@ -530,7 +441,6 @@ ByteString StringBuilder::to_byte_string() const
ErrorOr<String> StringBuilder::to_string()
{
VERIFY(m_mode == Mode::UTF8);
if (m_buffer.is_inline())
return String::from_utf8(string_view());
return String::from_string_builder({}, *this);
@ -538,7 +448,6 @@ ErrorOr<String> StringBuilder::to_string()
String StringBuilder::to_string_without_validation()
{
VERIFY(m_mode == Mode::UTF8);
if (m_buffer.is_inline())
return String::from_utf8_without_validation(string_view().bytes());
return String::from_string_builder_without_validation({}, *this);
@ -546,53 +455,32 @@ String StringBuilder::to_string_without_validation()
FlyString StringBuilder::to_fly_string_without_validation() const
{
VERIFY(m_mode == Mode::UTF8);
return FlyString::from_utf8_without_validation(string_view().bytes());
}
ErrorOr<FlyString> StringBuilder::to_fly_string() const
{
VERIFY(m_mode == Mode::UTF8);
return FlyString::from_utf8(string_view());
}
Utf16String StringBuilder::to_utf16_string()
{
VERIFY(m_mode == Mode::UTF16);
return Utf16String::from_string_builder({}, *this);
}
u8* StringBuilder::data()
{
return m_buffer.data() + string_builder_prefix_size(m_mode);
return m_buffer.data() + string_builder_prefix_size();
}
u8 const* StringBuilder::data() const
{
return m_buffer.data() + string_builder_prefix_size(m_mode);
return m_buffer.data() + string_builder_prefix_size();
}
StringView StringBuilder::string_view() const
{
VERIFY(m_mode == Mode::UTF8);
return m_buffer.span().slice(string_builder_prefix_size(m_mode));
}
Utf16View StringBuilder::utf16_string_view() const
{
VERIFY(m_mode == Mode::UTF16);
auto view = m_buffer.span().slice(string_builder_prefix_size(m_mode));
if (m_utf16_builder_is_ascii)
return { reinterpret_cast<char const*>(view.data()), view.size() };
return { reinterpret_cast<char16_t const*>(view.data()), view.size() / 2 };
return m_buffer.span().slice(string_builder_prefix_size());
}
void StringBuilder::clear()
{
m_buffer.resize(string_builder_prefix_size(m_mode));
if (m_mode == Mode::UTF16)
m_utf16_builder_is_ascii = true;
m_buffer.resize(string_builder_prefix_size());
}
ErrorOr<void> StringBuilder::try_append_code_point(u32 code_point)
@ -602,14 +490,7 @@ ErrorOr<void> StringBuilder::try_append_code_point(u32 code_point)
return {};
}
if (m_mode == Mode::UTF8 || (m_utf16_builder_is_ascii && is_ascii(code_point))) {
TRY(AK::UnicodeUtils::try_code_point_to_utf8(code_point, [this](char c) { return try_append(c); }));
} else {
TRY(ensure_storage_is_utf16());
TRY(AK::UnicodeUtils::try_code_point_to_utf16(code_point, [this](char16_t c) { return m_buffer.try_append(&c, sizeof(c)); }));
}
TRY(AK::UnicodeUtils::try_code_point_to_utf8(code_point, [this](char c) { return try_append(c); }));
return {};
}
@ -620,30 +501,6 @@ void StringBuilder::append_code_point(u32 code_point)
return;
}
auto append_as_utf8 = m_mode == Mode::UTF8 || (m_utf16_builder_is_ascii && is_ascii(code_point));
if (!append_as_utf8) {
MUST(ensure_storage_is_utf16());
(void)(will_append(2));
if (code_point < UnicodeUtils::FIRST_SUPPLEMENTARY_PLANE_CODE_POINT) {
auto code_unit = static_cast<char16_t>(code_point);
m_buffer.append(&code_unit, sizeof(code_unit));
return;
}
(void)(will_append(2));
code_point -= UnicodeUtils::FIRST_SUPPLEMENTARY_PLANE_CODE_POINT;
auto code_unit = static_cast<u16>(UnicodeUtils::HIGH_SURROGATE_MIN | (code_point >> 10));
m_buffer.append(&code_unit, sizeof(code_unit));
code_unit = static_cast<u16>(UnicodeUtils::LOW_SURROGATE_MIN | (code_point & 0x3ff));
m_buffer.append(&code_unit, sizeof(code_unit));
return;
}
if (code_point <= 0x7f) {
m_buffer.append(static_cast<char>(code_point));
} else if (code_point <= 0x07ff) {
@ -671,18 +528,6 @@ ErrorOr<void> StringBuilder::try_append(Utf16View const& utf16_view)
if (utf16_view.has_ascii_storage())
return try_append_ascii_without_validation(utf16_view.bytes());
auto append_as_utf8 = m_mode == Mode::UTF8 || (m_utf16_builder_is_ascii && utf16_view.is_ascii());
if (!append_as_utf8) {
TRY(ensure_storage_is_utf16());
TRY(will_append(utf16_view.length_in_code_units() * 2));
for (size_t i = 0; i < utf16_view.length_in_code_units(); ++i)
TRY(try_append_code_unit(utf16_view.code_unit_at(i)));
return {};
}
auto remaining_view = utf16_view.utf16_span();
auto maximum_utf8_length = UnicodeUtils::maximum_utf8_length_from_utf16(remaining_view);

View file

@ -16,12 +16,6 @@ namespace AK {
class StringBuilder {
public:
enum class Mode {
UTF8,
UTF16,
};
static constexpr auto DEFAULT_MODE = Mode::UTF8;
static constexpr size_t inline_capacity = 256;
class Buffer {
@ -86,9 +80,6 @@ public:
StringBuilder();
explicit StringBuilder(size_t initial_capacity);
explicit StringBuilder(Mode);
StringBuilder(Mode, size_t initial_capacity_in_code_units);
~StringBuilder() = default;
ErrorOr<void> try_append(StringView);
@ -139,12 +130,9 @@ public:
[[nodiscard]] FlyString to_fly_string_without_validation() const;
ErrorOr<FlyString> to_fly_string() const;
Utf16String to_utf16_string();
[[nodiscard]] ErrorOr<ByteBuffer> to_byte_buffer() const;
[[nodiscard]] StringView string_view() const;
[[nodiscard]] Utf16View utf16_string_view() const;
void clear();
[[nodiscard]] size_t length() const;
@ -176,19 +164,16 @@ public:
}
private:
void initialize_buffer(Mode, size_t capacity);
void initialize_buffer(size_t capacity);
Optional<Buffer::OutlineBuffer> leak_buffer_for_string_construction();
ErrorOr<void> will_append(size_t);
ErrorOr<void> ensure_storage_is_utf16();
u8* data();
u8 const* data() const;
Buffer m_buffer;
Mode m_mode { DEFAULT_MODE };
bool m_utf16_builder_is_ascii { true };
};
}

View file

@ -87,23 +87,6 @@ Utf16String Utf16String::from_utf16(Utf16View const& utf16_string)
return Utf16String { Detail::Utf16StringData::from_utf16(utf16_string) };
}
Utf16String Utf16String::from_string_builder(Badge<StringBuilder>, StringBuilder& builder)
{
auto view = builder.utf16_string_view();
if (view.length_in_code_units() <= Detail::MAX_SHORT_STRING_BYTE_COUNT && view.has_ascii_storage()) {
Utf16String string;
string.m_value.short_ascii_string = Detail::ShortString::create_with_byte_count(view.length_in_code_units());
auto result = view.bytes().copy_to(string.m_value.short_ascii_string.storage);
VERIFY(result == view.length_in_code_units());
return string;
}
return Utf16String { Detail::Utf16StringData::from_string_builder(builder) };
}
Utf16String Utf16String::from_string_builder(Badge<Utf16StringBuilder>, Utf16StringBuilder& builder)
{
auto view = builder.view();

View file

@ -236,7 +236,6 @@ public:
ALWAYS_INLINE Utf16String escape_html_entities() const { return utf16_view().escape_html_entities(); }
static Utf16String from_string_builder(Badge<StringBuilder>, StringBuilder& builder);
static Utf16String from_string_builder(Badge<Utf16StringBuilder>, Utf16StringBuilder& builder);
static ErrorOr<Utf16String> from_ipc_stream(Stream&, size_t length_in_code_units, bool is_ascii);

View file

@ -110,33 +110,6 @@ NonnullRefPtr<Utf16StringData> Utf16StringData::from_utf16(Utf16View const& utf1
return string.release_nonnull();
}
NonnullRefPtr<Utf16StringData> Utf16StringData::from_string_builder(StringBuilder& builder)
{
auto view = builder.utf16_string_view();
auto code_unit_length = view.length_in_code_units();
VERIFY_UTF16_LENGTH(code_unit_length);
RefPtr<Utf16StringData> string;
if (auto buffer = builder.leak_buffer_for_string_construction(Badge<Utf16StringData> {}); buffer.has_value()) {
auto storage_type = view.has_ascii_storage() ? StorageType::ASCII : StorageType::UTF16;
string = adopt_ref(*new (buffer->buffer.data()) Utf16StringData { storage_type, code_unit_length });
} else {
if (view.has_ascii_storage()) {
string = create_uninitialized(StorageType::ASCII, code_unit_length);
TypedTransfer<char>::copy(string->m_ascii_data, view.ascii_span().data(), code_unit_length);
} else {
string = create_uninitialized(StorageType::UTF16, code_unit_length);
TypedTransfer<char16_t>::copy(string->m_utf16_data, view.utf16_span().data(), code_unit_length);
string->m_length_in_code_points = view.m_length_in_code_points;
}
}
return string.release_nonnull();
}
NonnullRefPtr<Utf16StringData> Utf16StringData::from_string_builder(Utf16StringBuilder& builder)
{
auto view = builder.view();

View file

@ -34,7 +34,6 @@ public:
static NonnullRefPtr<Utf16StringData> from_utf8(StringView, AllowASCIIStorage);
static NonnullRefPtr<Utf16StringData> from_ascii(ReadonlyBytes);
static NonnullRefPtr<Utf16StringData> from_utf16(Utf16View const&);
static NonnullRefPtr<Utf16StringData> from_string_builder(StringBuilder&);
static NonnullRefPtr<Utf16StringData> from_string_builder(Utf16StringBuilder&);
static ErrorOr<NonnullRefPtr<Utf16StringData>> from_ipc_stream(Stream&, size_t length_in_code_units, bool is_ascii);

View file

@ -1249,7 +1249,7 @@ TEST_CASE(utf16_string_builder_clear_resets_ascii_flag)
// subsequent ASCII as char16_t, corrupting the first code unit when adopted.
Utf16StringBuilder builder;
// 1. Append a non-ASCII code point to force m_utf16_builder_is_ascii = false.
// 1. Append a non-ASCII code point to force UTF-16 storage.
builder.append_code_point(0x00D7); // U+00D7 MULTIPLICATION SIGN (×)
auto first = builder.to_string();
EXPECT_EQ(first.length_in_code_units(), 1u);