AK: Add Utf16StringBuilder

Add a purpose-built builder for constructing Utf16String values.
It keeps ASCII storage while possible, widens to UTF-16 when needed,
and can hand outline storage to Utf16StringData for direct adoption.

Add AK coverage for ASCII strings, UTF-16 widening, code points,
trimming, and long string construction.
This commit is contained in:
Andreas Kling 2026-06-22 11:46:25 +02:00 committed by Andreas Kling
parent a87f7b9561
commit 488006d276
11 changed files with 691 additions and 0 deletions

View file

@ -31,6 +31,7 @@ set(SOURCES
Time.cpp
Utf16FlyString.cpp
Utf16String.cpp
Utf16StringBuilder.cpp
Utf16StringData.cpp
Utf16View.cpp
Utf8View.cpp

View file

@ -57,6 +57,7 @@ class StringView;
class UnixDateTime;
class Utf16FlyString;
class Utf16String;
class Utf16StringBuilder;
class Utf16View;
class Utf8CodePointIterator;
class Utf8View;

View file

@ -7,6 +7,7 @@
#include <AK/Stream.h>
#include <AK/StringNumber.h>
#include <AK/Utf16String.h>
#include <AK/Utf16StringBuilder.h>
#include <simdutf.h>
@ -103,6 +104,23 @@ Utf16String Utf16String::from_string_builder(Badge<StringBuilder>, StringBuilder
return Utf16String { Detail::Utf16StringData::from_string_builder(builder) };
}
Utf16String Utf16String::from_string_builder(Badge<Utf16StringBuilder>, Utf16StringBuilder& builder)
{
auto view = builder.view();
if (view.length_in_code_units() <= Detail::MAX_SHORT_STRING_BYTE_COUNT && view.has_ascii_storage()) {
Utf16String string;
string.m_value.short_ascii_string = Detail::ShortString::create_with_byte_count(view.length_in_code_units());
auto result = view.bytes().copy_to(string.m_value.short_ascii_string.storage);
VERIFY(result == view.length_in_code_units());
return string;
}
return Utf16String { Detail::Utf16StringData::from_string_builder(builder) };
}
ErrorOr<Utf16String> Utf16String::from_ipc_stream(Stream& stream, size_t length_in_code_units, bool is_ascii)
{
if (is_ascii && length_in_code_units <= Detail::MAX_SHORT_STRING_BYTE_COUNT) {

View file

@ -230,6 +230,7 @@ public:
ALWAYS_INLINE Utf16String escape_html_entities() const { return utf16_view().escape_html_entities(); }
static Utf16String from_string_builder(Badge<StringBuilder>, StringBuilder& builder);
static Utf16String from_string_builder(Badge<Utf16StringBuilder>, Utf16StringBuilder& builder);
static ErrorOr<Utf16String> from_ipc_stream(Stream&, size_t length_in_code_units, bool is_ascii);
constexpr Utf16String(Badge<Optional<Utf16String>>, nullptr_t)

409
AK/Utf16StringBuilder.cpp Normal file
View file

@ -0,0 +1,409 @@
/*
* Copyright (c) 2026-present, the Ladybird developers.
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#include <AK/CharacterTypes.h>
#include <AK/Checked.h>
#include <AK/UnicodeUtils.h>
#include <AK/Utf16String.h>
#include <AK/Utf16StringBuilder.h>
#include <AK/Utf16StringData.h>
#include <AK/kmalloc.h>
namespace AK {
Utf16StringBuilder::Buffer::~Buffer()
{
clear();
}
u8* Utf16StringBuilder::Buffer::data()
{
return m_inline ? m_inline_buffer : m_outline_buffer;
}
u8 const* Utf16StringBuilder::Buffer::data() const
{
return m_inline ? m_inline_buffer : m_outline_buffer;
}
Bytes Utf16StringBuilder::Buffer::span()
{
return { data(), size() };
}
ReadonlyBytes Utf16StringBuilder::Buffer::span() const
{
return { data(), size() };
}
void Utf16StringBuilder::Buffer::clear()
{
if (!m_inline) {
kfree(m_outline_buffer);
m_inline = true;
}
m_size = 0;
}
void Utf16StringBuilder::Buffer::resize(size_t new_size)
{
if (new_size <= m_size) {
trim(new_size, false);
return;
}
ensure_capacity(new_size);
set_size(new_size);
}
void Utf16StringBuilder::Buffer::set_size(size_t new_size)
{
ASSERT(new_size <= capacity());
m_size = new_size;
}
void Utf16StringBuilder::Buffer::ensure_capacity(size_t new_capacity)
{
if (new_capacity <= capacity())
return;
ensure_capacity_slowpath(new_capacity);
}
void Utf16StringBuilder::Buffer::append(void const* data, size_t data_size)
{
if (data_size == 0)
return;
VERIFY(data != nullptr);
auto old_size = size();
Checked<size_t> new_size = old_size;
new_size += data_size;
VERIFY(!new_size.has_overflow());
resize(new_size.value());
__builtin_memcpy(this->data() + old_size, data, data_size);
}
auto Utf16StringBuilder::Buffer::leak_outline_buffer() -> Optional<OutlineBuffer>
{
if (m_inline)
return {};
auto* outline_buffer = m_outline_buffer;
auto size = m_size;
auto outline_capacity = m_outline_capacity;
m_inline = true;
m_size = 0;
return OutlineBuffer { Bytes { outline_buffer, size }, outline_capacity };
}
void Utf16StringBuilder::Buffer::trim(size_t size, bool may_discard_existing_data)
{
VERIFY(size <= m_size);
if (!m_inline && size <= inline_capacity)
shrink_into_inline_buffer(size, may_discard_existing_data);
m_size = size;
}
void Utf16StringBuilder::Buffer::shrink_into_inline_buffer(size_t size, bool may_discard_existing_data)
{
auto* outline_buffer = m_outline_buffer;
if (!may_discard_existing_data)
__builtin_memcpy(m_inline_buffer, outline_buffer, size);
kfree(outline_buffer);
m_inline = true;
}
void Utf16StringBuilder::Buffer::ensure_capacity_slowpath(size_t new_capacity)
{
Checked<size_t> grown_capacity = capacity();
grown_capacity *= 3;
VERIFY(!grown_capacity.has_overflow());
new_capacity = max(new_capacity, grown_capacity.value() / 2);
new_capacity = kmalloc_good_size(new_capacity);
if (m_inline) {
auto* new_buffer = static_cast<u8*>(kmalloc(HeapPartition::String, new_capacity));
VERIFY(new_buffer);
__builtin_memcpy(new_buffer, data(), m_size);
m_outline_buffer = new_buffer;
} else {
auto* new_buffer = static_cast<u8*>(krealloc(HeapPartition::String, m_outline_buffer, new_capacity));
VERIFY(new_buffer);
m_outline_buffer = new_buffer;
}
m_outline_capacity = new_capacity;
m_inline = false;
}
static constexpr size_t string_builder_prefix_size()
{
return Detail::Utf16StringData::offset_of_string_storage();
}
static size_t utf16_byte_count(size_t length_in_code_units)
{
Checked<size_t> byte_count = length_in_code_units;
byte_count *= sizeof(char16_t);
VERIFY(!byte_count.has_overflow());
return byte_count.value();
}
Utf16StringBuilder::Utf16StringBuilder()
{
initialize_buffer(inline_capacity / sizeof(char16_t));
}
Utf16StringBuilder::Utf16StringBuilder(size_t initial_capacity_in_code_units)
{
initialize_buffer(initial_capacity_in_code_units);
}
void Utf16StringBuilder::initialize_buffer(size_t capacity_in_code_units)
{
auto prefix_size = string_builder_prefix_size();
auto capacity_in_bytes = utf16_byte_count(capacity_in_code_units);
if (capacity_in_bytes > inline_capacity) {
Checked<size_t> capacity = prefix_size;
capacity += capacity_in_bytes;
VERIFY(!capacity.has_overflow());
m_buffer.ensure_capacity(capacity.value());
}
m_buffer.resize(prefix_size);
}
void Utf16StringBuilder::will_append(size_t size_in_bytes)
{
Checked<size_t> needed_capacity = m_buffer.size();
needed_capacity += size_in_bytes;
VERIFY(!needed_capacity.has_overflow());
if (needed_capacity <= m_buffer.capacity())
return;
Checked<size_t> expanded_capacity = needed_capacity;
expanded_capacity *= 2;
VERIFY(!expanded_capacity.has_overflow());
m_buffer.ensure_capacity(expanded_capacity.value());
}
void Utf16StringBuilder::ensure_storage_is_utf16()
{
if (!m_is_ascii)
return;
auto ascii_length = length_in_bytes();
m_is_ascii = false;
if (ascii_length == 0)
return;
m_buffer.resize(m_buffer.size() + ascii_length);
Bytes source { data(), ascii_length };
Span<char16_t> target { reinterpret_cast<char16_t*>(data()), ascii_length };
for (size_t i = ascii_length; i > 0; --i) {
auto index = i - 1;
auto code_unit = static_cast<char16_t>(source[index]);
target[index] = code_unit;
}
}
void Utf16StringBuilder::append(Utf16View const& view)
{
if (view.is_empty())
return;
if (m_is_ascii && view.is_ascii()) {
will_append(view.length_in_code_units());
if (view.has_ascii_storage()) {
m_buffer.append(view.ascii_span().data(), view.length_in_code_units());
} else {
for (auto code_unit : view.utf16_span()) {
auto ascii_code_unit = static_cast<char>(code_unit);
m_buffer.append(&ascii_code_unit, sizeof(ascii_code_unit));
}
}
return;
}
ensure_storage_is_utf16();
will_append(utf16_byte_count(view.length_in_code_units()));
if (view.has_ascii_storage()) {
for (auto code_unit : view.ascii_span()) {
auto utf16_code_unit = static_cast<char16_t>(code_unit);
m_buffer.append(&utf16_code_unit, sizeof(utf16_code_unit));
}
} else {
m_buffer.append(view.utf16_span().data(), utf16_byte_count(view.length_in_code_units()));
}
}
void Utf16StringBuilder::append_ascii(char code_unit)
{
VERIFY(is_ascii(code_unit));
if (m_is_ascii) {
will_append(1);
m_buffer.append(&code_unit, sizeof(code_unit));
} else {
auto utf16_code_unit = static_cast<char16_t>(code_unit);
will_append(sizeof(utf16_code_unit));
m_buffer.append(&utf16_code_unit, sizeof(utf16_code_unit));
}
}
void Utf16StringBuilder::append_ascii(StringView string)
{
VERIFY(string.is_ascii());
if (string.is_empty())
return;
if (m_is_ascii) {
will_append(string.length());
m_buffer.append(string.characters_without_null_termination(), string.length());
return;
}
will_append(utf16_byte_count(string.length()));
for (auto code_unit : string) {
auto utf16_code_unit = static_cast<char16_t>(code_unit);
m_buffer.append(&utf16_code_unit, sizeof(utf16_code_unit));
}
}
void Utf16StringBuilder::append_code_unit(char16_t code_unit)
{
if (m_is_ascii && is_ascii(code_unit)) {
append_ascii(static_cast<char>(code_unit));
return;
}
ensure_storage_is_utf16();
will_append(sizeof(code_unit));
m_buffer.append(&code_unit, sizeof(code_unit));
}
void Utf16StringBuilder::append_code_point(u32 code_point)
{
if (!is_unicode(code_point)) {
append_code_point(UnicodeUtils::REPLACEMENT_CODE_POINT);
return;
}
if (m_is_ascii && is_ascii(code_point)) {
append_ascii(static_cast<char>(code_point));
return;
}
ensure_storage_is_utf16();
(void)UnicodeUtils::code_point_to_utf16(code_point, [this](char16_t code_unit) {
will_append(sizeof(code_unit));
m_buffer.append(&code_unit, sizeof(code_unit));
});
}
void Utf16StringBuilder::append_repeated_ascii(char code_unit, size_t count)
{
VERIFY(is_ascii(code_unit));
if (m_is_ascii) {
will_append(count);
for (size_t i = 0; i < count; ++i)
m_buffer.append(&code_unit, sizeof(code_unit));
return;
}
auto utf16_code_unit = static_cast<char16_t>(code_unit);
will_append(utf16_byte_count(count));
for (size_t i = 0; i < count; ++i)
m_buffer.append(&utf16_code_unit, sizeof(utf16_code_unit));
}
void Utf16StringBuilder::append_repeated(Utf16View const& view, size_t count)
{
for (size_t i = 0; i < count; ++i)
append(view);
}
Utf16String Utf16StringBuilder::to_string()
{
return Utf16String::from_string_builder({}, *this);
}
Utf16View Utf16StringBuilder::view() const
{
auto data = m_buffer.span().slice(string_builder_prefix_size());
if (m_is_ascii)
return { reinterpret_cast<char const*>(data.data()), data.size() };
return { reinterpret_cast<char16_t const*>(data.data()), data.size() / sizeof(char16_t) };
}
size_t Utf16StringBuilder::length_in_code_units() const
{
auto length = length_in_bytes();
if (m_is_ascii)
return length;
return length / sizeof(char16_t);
}
void Utf16StringBuilder::trim(size_t count_in_code_units)
{
auto count_in_bytes = count_in_code_units;
if (!m_is_ascii)
count_in_bytes = utf16_byte_count(count_in_code_units);
auto decrease_count = min(length_in_bytes(), count_in_bytes);
m_buffer.resize(m_buffer.size() - decrease_count);
}
void Utf16StringBuilder::clear()
{
m_buffer.resize(string_builder_prefix_size());
m_is_ascii = true;
}
auto Utf16StringBuilder::leak_buffer_for_string_construction() -> Optional<Buffer::OutlineBuffer>
{
if (auto buffer = m_buffer.leak_outline_buffer(); buffer.has_value()) {
clear();
return buffer;
}
return {};
}
u8* Utf16StringBuilder::data()
{
return m_buffer.data() + string_builder_prefix_size();
}
u8 const* Utf16StringBuilder::data() const
{
return m_buffer.data() + string_builder_prefix_size();
}
size_t Utf16StringBuilder::length_in_bytes() const
{
return m_buffer.size() - string_builder_prefix_size();
}
}

120
AK/Utf16StringBuilder.h Normal file
View file

@ -0,0 +1,120 @@
/*
* Copyright (c) 2026-present, the Ladybird developers.
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#pragma once
#include <AK/Badge.h>
#include <AK/Forward.h>
#include <AK/Optional.h>
#include <AK/Span.h>
#include <AK/StringView.h>
#include <AK/Types.h>
#include <AK/Utf16View.h>
namespace AK {
class Utf16StringBuilder {
public:
static constexpr size_t inline_capacity = 256;
class Buffer {
public:
Buffer() = default;
~Buffer();
Buffer(Buffer const&) = delete;
Buffer(Buffer&&) = delete;
Buffer& operator=(Buffer const&) = delete;
Buffer& operator=(Buffer&&) = delete;
[[nodiscard]] u8* data();
[[nodiscard]] u8 const* data() const;
[[nodiscard]] Bytes span() LIFETIME_BOUND;
[[nodiscard]] ReadonlyBytes span() const LIFETIME_BOUND;
[[nodiscard]] size_t size() const { return m_size; }
[[nodiscard]] size_t capacity() const { return m_inline ? inline_capacity : m_outline_capacity; }
[[nodiscard]] bool is_inline() const { return m_inline; }
void clear();
void resize(size_t);
void set_size(size_t);
void ensure_capacity(size_t);
void append(void const*, size_t);
struct OutlineBuffer {
Bytes buffer;
size_t capacity { 0 };
};
Optional<OutlineBuffer> leak_outline_buffer();
private:
void trim(size_t, bool may_discard_existing_data);
void shrink_into_inline_buffer(size_t, bool may_discard_existing_data);
void ensure_capacity_slowpath(size_t);
union {
u8 m_inline_buffer[inline_capacity];
struct {
u8* m_outline_buffer;
size_t m_outline_capacity;
};
};
size_t m_size { 0 };
bool m_inline { true };
};
Utf16StringBuilder();
explicit Utf16StringBuilder(size_t initial_capacity_in_code_units);
~Utf16StringBuilder() = default;
Utf16StringBuilder(Utf16StringBuilder const&) = delete;
Utf16StringBuilder(Utf16StringBuilder&&) = delete;
Utf16StringBuilder& operator=(Utf16StringBuilder const&) = delete;
Utf16StringBuilder& operator=(Utf16StringBuilder&&) = delete;
void append(Utf16View const&);
void append_ascii(char);
void append_ascii(StringView);
void append_code_unit(char16_t);
void append_code_point(u32);
void append_repeated_ascii(char, size_t);
void append_repeated(Utf16View const&, size_t);
[[nodiscard]] Utf16String to_string();
[[nodiscard]] Utf16View view() const;
[[nodiscard]] size_t length_in_code_units() const;
[[nodiscard]] bool is_empty() const { return length_in_code_units() == 0; }
void trim(size_t count_in_code_units);
void clear();
Optional<Buffer::OutlineBuffer> leak_buffer_for_string_construction(Badge<Detail::Utf16StringData>)
{
return leak_buffer_for_string_construction();
}
private:
void initialize_buffer(size_t capacity_in_code_units);
void will_append(size_t size_in_bytes);
void ensure_storage_is_utf16();
Optional<Buffer::OutlineBuffer> leak_buffer_for_string_construction();
[[nodiscard]] u8* data();
[[nodiscard]] u8 const* data() const;
[[nodiscard]] size_t length_in_bytes() const;
Buffer m_buffer;
bool m_is_ascii { true };
};
}
#if USING_AK_GLOBALLY
using AK::Utf16StringBuilder;
#endif

View file

@ -7,6 +7,7 @@
#include <AK/RefPtr.h>
#include <AK/Stream.h>
#include <AK/TypedTransfer.h>
#include <AK/Utf16StringBuilder.h>
#include <AK/Utf16StringData.h>
#include <AK/Utf8View.h>
@ -136,6 +137,33 @@ NonnullRefPtr<Utf16StringData> Utf16StringData::from_string_builder(StringBuilde
return string.release_nonnull();
}
NonnullRefPtr<Utf16StringData> Utf16StringData::from_string_builder(Utf16StringBuilder& builder)
{
auto view = builder.view();
auto code_unit_length = view.length_in_code_units();
VERIFY_UTF16_LENGTH(code_unit_length);
RefPtr<Utf16StringData> string;
if (auto buffer = builder.leak_buffer_for_string_construction(Badge<Utf16StringData> {}); buffer.has_value()) {
auto storage_type = view.has_ascii_storage() ? StorageType::ASCII : StorageType::UTF16;
string = adopt_ref(*new (buffer->buffer.data()) Utf16StringData { storage_type, code_unit_length });
} else {
if (view.has_ascii_storage()) {
string = create_uninitialized(StorageType::ASCII, code_unit_length);
TypedTransfer<char>::copy(string->m_ascii_data, view.ascii_span().data(), code_unit_length);
} else {
string = create_uninitialized(StorageType::UTF16, code_unit_length);
TypedTransfer<char16_t>::copy(string->m_utf16_data, view.utf16_span().data(), code_unit_length);
string->m_length_in_code_points = view.m_length_in_code_points;
}
}
return string.release_nonnull();
}
ErrorOr<NonnullRefPtr<Utf16StringData>> Utf16StringData::from_ipc_stream(Stream& stream, size_t length_in_code_units, bool is_ascii)
{
RefPtr<Utf16StringData> string;

View file

@ -35,6 +35,7 @@ public:
static NonnullRefPtr<Utf16StringData> from_ascii(ReadonlyBytes);
static NonnullRefPtr<Utf16StringData> from_utf16(Utf16View const&);
static NonnullRefPtr<Utf16StringData> from_string_builder(StringBuilder&);
static NonnullRefPtr<Utf16StringData> from_string_builder(Utf16StringBuilder&);
static ErrorOr<NonnullRefPtr<Utf16StringData>> from_ipc_stream(Stream&, size_t length_in_code_units, bool is_ascii);
static NonnullRefPtr<Utf16StringData> to_well_formed(Utf16View const&);

View file

@ -660,6 +660,7 @@ public:
private:
friend StringBuilder;
friend Utf16StringBuilder;
friend Detail::Utf16StringBase;
friend Detail::Utf16StringData;

View file

@ -74,6 +74,7 @@ set(AK_TEST_SOURCES
TestUFixedBigInt.cpp
TestUtf16FlyString.cpp
TestUtf16String.cpp
TestUtf16StringBuilder.cpp
TestUtf16View.cpp
TestUtf8View.cpp
TestVariant.cpp

View file

@ -0,0 +1,110 @@
/*
* Copyright (c) 2026-present, the Ladybird developers.
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#include <AK/Utf16String.h>
#include <AK/Utf16StringBuilder.h>
#include <LibTest/TestCase.h>
TEST_CASE(build_ascii_string)
{
Utf16StringBuilder builder;
builder.append_ascii("hello"sv);
builder.append_ascii('!');
EXPECT_EQ(builder.view(), "hello!"sv);
auto string = builder.to_string();
EXPECT_EQ(string, "hello!"sv);
EXPECT(string.has_short_ascii_storage());
}
TEST_CASE(widen_to_utf16_storage)
{
Utf16StringBuilder builder;
builder.append_ascii("hello "sv);
builder.append_code_unit(0x263A);
EXPECT_EQ(builder.view(), u"hello \u263A"sv);
auto string = builder.to_string();
EXPECT_EQ(string, u"hello \u263A"sv);
EXPECT(!string.is_ascii());
}
TEST_CASE(append_code_point)
{
Utf16StringBuilder builder;
builder.append_ascii("wave "sv);
builder.append_code_point(0x1F44B);
auto string = builder.to_string();
EXPECT_EQ(string, u"wave \U0001F44B"sv);
EXPECT_EQ(string.length_in_code_units(), 7uz);
EXPECT_EQ(string.length_in_code_points(), 6uz);
}
TEST_CASE(trim_ascii_storage)
{
Utf16StringBuilder builder;
builder.append_ascii("abc"sv);
builder.trim(1);
EXPECT_EQ(builder.view(), "ab"sv);
EXPECT_EQ(builder.to_string(), "ab"sv);
}
TEST_CASE(trim_utf16_storage)
{
Utf16StringBuilder builder;
builder.append(u"ab\u0100"sv);
builder.trim(1);
EXPECT_EQ(builder.view(), "ab"sv);
EXPECT_EQ(builder.to_string(), "ab"sv);
}
TEST_CASE(long_ascii_string)
{
Utf16StringBuilder builder;
builder.append_repeated_ascii('x', 300);
auto string = builder.to_string();
EXPECT_EQ(string.length_in_code_units(), 300uz);
EXPECT(string.has_long_ascii_storage());
EXPECT_EQ(string.utf16_view().code_unit_at(0), 'x');
EXPECT_EQ(string.utf16_view().code_unit_at(299), 'x');
}
TEST_CASE(long_utf16_string)
{
Utf16StringBuilder builder;
builder.append_repeated(u"\u0100"sv, 300);
auto string = builder.to_string();
EXPECT_EQ(string.length_in_code_units(), 300uz);
EXPECT(!string.is_ascii());
EXPECT_EQ(string.utf16_view().code_unit_at(0), 0x0100);
EXPECT_EQ(string.utf16_view().code_unit_at(299), 0x0100);
}
TEST_CASE(reuse_after_creating_long_utf16_string)
{
Utf16StringBuilder builder;
builder.append_repeated(u"\u0100"sv, 300);
auto string = builder.to_string();
EXPECT_EQ(string.length_in_code_units(), 300uz);
EXPECT(!string.is_ascii());
EXPECT(builder.is_empty());
builder.append_ascii("ok"sv);
EXPECT_EQ(builder.view(), "ok"sv);
EXPECT_EQ(builder.to_string(), "ok"sv);
}