diff --git a/AK/CMakeLists.txt b/AK/CMakeLists.txt index ffde0fdf1f..3dcbd19b1f 100644 --- a/AK/CMakeLists.txt +++ b/AK/CMakeLists.txt @@ -31,6 +31,7 @@ set(SOURCES Time.cpp Utf16FlyString.cpp Utf16String.cpp + Utf16StringBuilder.cpp Utf16StringData.cpp Utf16View.cpp Utf8View.cpp diff --git a/AK/Forward.h b/AK/Forward.h index d88397d236..400f872390 100644 --- a/AK/Forward.h +++ b/AK/Forward.h @@ -57,6 +57,7 @@ class StringView; class UnixDateTime; class Utf16FlyString; class Utf16String; +class Utf16StringBuilder; class Utf16View; class Utf8CodePointIterator; class Utf8View; diff --git a/AK/Utf16String.cpp b/AK/Utf16String.cpp index d1b24f1da2..b984d8fb93 100644 --- a/AK/Utf16String.cpp +++ b/AK/Utf16String.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include @@ -103,6 +104,23 @@ Utf16String Utf16String::from_string_builder(Badge, StringBuilder return Utf16String { Detail::Utf16StringData::from_string_builder(builder) }; } +Utf16String Utf16String::from_string_builder(Badge, Utf16StringBuilder& builder) +{ + auto view = builder.view(); + + if (view.length_in_code_units() <= Detail::MAX_SHORT_STRING_BYTE_COUNT && view.has_ascii_storage()) { + Utf16String string; + string.m_value.short_ascii_string = Detail::ShortString::create_with_byte_count(view.length_in_code_units()); + + auto result = view.bytes().copy_to(string.m_value.short_ascii_string.storage); + VERIFY(result == view.length_in_code_units()); + + return string; + } + + return Utf16String { Detail::Utf16StringData::from_string_builder(builder) }; +} + ErrorOr Utf16String::from_ipc_stream(Stream& stream, size_t length_in_code_units, bool is_ascii) { if (is_ascii && length_in_code_units <= Detail::MAX_SHORT_STRING_BYTE_COUNT) { diff --git a/AK/Utf16String.h b/AK/Utf16String.h index dd93b7daa5..1bf1380969 100644 --- a/AK/Utf16String.h +++ b/AK/Utf16String.h @@ -230,6 +230,7 @@ public: ALWAYS_INLINE Utf16String escape_html_entities() const { return utf16_view().escape_html_entities(); } static Utf16String from_string_builder(Badge, StringBuilder& builder); + static Utf16String from_string_builder(Badge, Utf16StringBuilder& builder); static ErrorOr from_ipc_stream(Stream&, size_t length_in_code_units, bool is_ascii); constexpr Utf16String(Badge>, nullptr_t) diff --git a/AK/Utf16StringBuilder.cpp b/AK/Utf16StringBuilder.cpp new file mode 100644 index 0000000000..11cd7de1bf --- /dev/null +++ b/AK/Utf16StringBuilder.cpp @@ -0,0 +1,409 @@ +/* + * Copyright (c) 2026-present, the Ladybird developers. + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include +#include +#include +#include +#include +#include +#include + +namespace AK { + +Utf16StringBuilder::Buffer::~Buffer() +{ + clear(); +} + +u8* Utf16StringBuilder::Buffer::data() +{ + return m_inline ? m_inline_buffer : m_outline_buffer; +} + +u8 const* Utf16StringBuilder::Buffer::data() const +{ + return m_inline ? m_inline_buffer : m_outline_buffer; +} + +Bytes Utf16StringBuilder::Buffer::span() +{ + return { data(), size() }; +} + +ReadonlyBytes Utf16StringBuilder::Buffer::span() const +{ + return { data(), size() }; +} + +void Utf16StringBuilder::Buffer::clear() +{ + if (!m_inline) { + kfree(m_outline_buffer); + m_inline = true; + } + m_size = 0; +} + +void Utf16StringBuilder::Buffer::resize(size_t new_size) +{ + if (new_size <= m_size) { + trim(new_size, false); + return; + } + + ensure_capacity(new_size); + set_size(new_size); +} + +void Utf16StringBuilder::Buffer::set_size(size_t new_size) +{ + ASSERT(new_size <= capacity()); + m_size = new_size; +} + +void Utf16StringBuilder::Buffer::ensure_capacity(size_t new_capacity) +{ + if (new_capacity <= capacity()) + return; + ensure_capacity_slowpath(new_capacity); +} + +void Utf16StringBuilder::Buffer::append(void const* data, size_t data_size) +{ + if (data_size == 0) + return; + VERIFY(data != nullptr); + + auto old_size = size(); + Checked new_size = old_size; + new_size += data_size; + VERIFY(!new_size.has_overflow()); + + resize(new_size.value()); + __builtin_memcpy(this->data() + old_size, data, data_size); +} + +auto Utf16StringBuilder::Buffer::leak_outline_buffer() -> Optional +{ + if (m_inline) + return {}; + + auto* outline_buffer = m_outline_buffer; + auto size = m_size; + auto outline_capacity = m_outline_capacity; + + m_inline = true; + m_size = 0; + + return OutlineBuffer { Bytes { outline_buffer, size }, outline_capacity }; +} + +void Utf16StringBuilder::Buffer::trim(size_t size, bool may_discard_existing_data) +{ + VERIFY(size <= m_size); + if (!m_inline && size <= inline_capacity) + shrink_into_inline_buffer(size, may_discard_existing_data); + m_size = size; +} + +void Utf16StringBuilder::Buffer::shrink_into_inline_buffer(size_t size, bool may_discard_existing_data) +{ + auto* outline_buffer = m_outline_buffer; + if (!may_discard_existing_data) + __builtin_memcpy(m_inline_buffer, outline_buffer, size); + kfree(outline_buffer); + m_inline = true; +} + +void Utf16StringBuilder::Buffer::ensure_capacity_slowpath(size_t new_capacity) +{ + Checked grown_capacity = capacity(); + grown_capacity *= 3; + VERIFY(!grown_capacity.has_overflow()); + + new_capacity = max(new_capacity, grown_capacity.value() / 2); + new_capacity = kmalloc_good_size(new_capacity); + + if (m_inline) { + auto* new_buffer = static_cast(kmalloc(HeapPartition::String, new_capacity)); + VERIFY(new_buffer); + + __builtin_memcpy(new_buffer, data(), m_size); + m_outline_buffer = new_buffer; + } else { + auto* new_buffer = static_cast(krealloc(HeapPartition::String, m_outline_buffer, new_capacity)); + VERIFY(new_buffer); + + m_outline_buffer = new_buffer; + } + + m_outline_capacity = new_capacity; + m_inline = false; +} + +static constexpr size_t string_builder_prefix_size() +{ + return Detail::Utf16StringData::offset_of_string_storage(); +} + +static size_t utf16_byte_count(size_t length_in_code_units) +{ + Checked byte_count = length_in_code_units; + byte_count *= sizeof(char16_t); + VERIFY(!byte_count.has_overflow()); + return byte_count.value(); +} + +Utf16StringBuilder::Utf16StringBuilder() +{ + initialize_buffer(inline_capacity / sizeof(char16_t)); +} + +Utf16StringBuilder::Utf16StringBuilder(size_t initial_capacity_in_code_units) +{ + initialize_buffer(initial_capacity_in_code_units); +} + +void Utf16StringBuilder::initialize_buffer(size_t capacity_in_code_units) +{ + auto prefix_size = string_builder_prefix_size(); + auto capacity_in_bytes = utf16_byte_count(capacity_in_code_units); + + if (capacity_in_bytes > inline_capacity) { + Checked capacity = prefix_size; + capacity += capacity_in_bytes; + VERIFY(!capacity.has_overflow()); + + m_buffer.ensure_capacity(capacity.value()); + } + m_buffer.resize(prefix_size); +} + +void Utf16StringBuilder::will_append(size_t size_in_bytes) +{ + Checked needed_capacity = m_buffer.size(); + needed_capacity += size_in_bytes; + VERIFY(!needed_capacity.has_overflow()); + + if (needed_capacity <= m_buffer.capacity()) + return; + + Checked expanded_capacity = needed_capacity; + expanded_capacity *= 2; + VERIFY(!expanded_capacity.has_overflow()); + + m_buffer.ensure_capacity(expanded_capacity.value()); +} + +void Utf16StringBuilder::ensure_storage_is_utf16() +{ + if (!m_is_ascii) + return; + + auto ascii_length = length_in_bytes(); + m_is_ascii = false; + if (ascii_length == 0) + return; + + m_buffer.resize(m_buffer.size() + ascii_length); + + Bytes source { data(), ascii_length }; + Span target { reinterpret_cast(data()), ascii_length }; + + for (size_t i = ascii_length; i > 0; --i) { + auto index = i - 1; + + auto code_unit = static_cast(source[index]); + target[index] = code_unit; + } +} + +void Utf16StringBuilder::append(Utf16View const& view) +{ + if (view.is_empty()) + return; + + if (m_is_ascii && view.is_ascii()) { + will_append(view.length_in_code_units()); + + if (view.has_ascii_storage()) { + m_buffer.append(view.ascii_span().data(), view.length_in_code_units()); + } else { + for (auto code_unit : view.utf16_span()) { + auto ascii_code_unit = static_cast(code_unit); + m_buffer.append(&ascii_code_unit, sizeof(ascii_code_unit)); + } + } + + return; + } + + ensure_storage_is_utf16(); + will_append(utf16_byte_count(view.length_in_code_units())); + + if (view.has_ascii_storage()) { + for (auto code_unit : view.ascii_span()) { + auto utf16_code_unit = static_cast(code_unit); + m_buffer.append(&utf16_code_unit, sizeof(utf16_code_unit)); + } + } else { + m_buffer.append(view.utf16_span().data(), utf16_byte_count(view.length_in_code_units())); + } +} + +void Utf16StringBuilder::append_ascii(char code_unit) +{ + VERIFY(is_ascii(code_unit)); + + if (m_is_ascii) { + will_append(1); + m_buffer.append(&code_unit, sizeof(code_unit)); + } else { + auto utf16_code_unit = static_cast(code_unit); + will_append(sizeof(utf16_code_unit)); + m_buffer.append(&utf16_code_unit, sizeof(utf16_code_unit)); + } +} + +void Utf16StringBuilder::append_ascii(StringView string) +{ + VERIFY(string.is_ascii()); + if (string.is_empty()) + return; + + if (m_is_ascii) { + will_append(string.length()); + m_buffer.append(string.characters_without_null_termination(), string.length()); + return; + } + + will_append(utf16_byte_count(string.length())); + for (auto code_unit : string) { + auto utf16_code_unit = static_cast(code_unit); + m_buffer.append(&utf16_code_unit, sizeof(utf16_code_unit)); + } +} + +void Utf16StringBuilder::append_code_unit(char16_t code_unit) +{ + if (m_is_ascii && is_ascii(code_unit)) { + append_ascii(static_cast(code_unit)); + return; + } + + ensure_storage_is_utf16(); + will_append(sizeof(code_unit)); + m_buffer.append(&code_unit, sizeof(code_unit)); +} + +void Utf16StringBuilder::append_code_point(u32 code_point) +{ + if (!is_unicode(code_point)) { + append_code_point(UnicodeUtils::REPLACEMENT_CODE_POINT); + return; + } + + if (m_is_ascii && is_ascii(code_point)) { + append_ascii(static_cast(code_point)); + return; + } + + ensure_storage_is_utf16(); + + (void)UnicodeUtils::code_point_to_utf16(code_point, [this](char16_t code_unit) { + will_append(sizeof(code_unit)); + m_buffer.append(&code_unit, sizeof(code_unit)); + }); +} + +void Utf16StringBuilder::append_repeated_ascii(char code_unit, size_t count) +{ + VERIFY(is_ascii(code_unit)); + + if (m_is_ascii) { + will_append(count); + for (size_t i = 0; i < count; ++i) + m_buffer.append(&code_unit, sizeof(code_unit)); + return; + } + + auto utf16_code_unit = static_cast(code_unit); + will_append(utf16_byte_count(count)); + for (size_t i = 0; i < count; ++i) + m_buffer.append(&utf16_code_unit, sizeof(utf16_code_unit)); +} + +void Utf16StringBuilder::append_repeated(Utf16View const& view, size_t count) +{ + for (size_t i = 0; i < count; ++i) + append(view); +} + +Utf16String Utf16StringBuilder::to_string() +{ + return Utf16String::from_string_builder({}, *this); +} + +Utf16View Utf16StringBuilder::view() const +{ + auto data = m_buffer.span().slice(string_builder_prefix_size()); + + if (m_is_ascii) + return { reinterpret_cast(data.data()), data.size() }; + return { reinterpret_cast(data.data()), data.size() / sizeof(char16_t) }; +} + +size_t Utf16StringBuilder::length_in_code_units() const +{ + auto length = length_in_bytes(); + if (m_is_ascii) + return length; + return length / sizeof(char16_t); +} + +void Utf16StringBuilder::trim(size_t count_in_code_units) +{ + auto count_in_bytes = count_in_code_units; + if (!m_is_ascii) + count_in_bytes = utf16_byte_count(count_in_code_units); + + auto decrease_count = min(length_in_bytes(), count_in_bytes); + m_buffer.resize(m_buffer.size() - decrease_count); +} + +void Utf16StringBuilder::clear() +{ + m_buffer.resize(string_builder_prefix_size()); + m_is_ascii = true; +} + +auto Utf16StringBuilder::leak_buffer_for_string_construction() -> Optional +{ + if (auto buffer = m_buffer.leak_outline_buffer(); buffer.has_value()) { + clear(); + return buffer; + } + + return {}; +} + +u8* Utf16StringBuilder::data() +{ + return m_buffer.data() + string_builder_prefix_size(); +} + +u8 const* Utf16StringBuilder::data() const +{ + return m_buffer.data() + string_builder_prefix_size(); +} + +size_t Utf16StringBuilder::length_in_bytes() const +{ + return m_buffer.size() - string_builder_prefix_size(); +} + +} diff --git a/AK/Utf16StringBuilder.h b/AK/Utf16StringBuilder.h new file mode 100644 index 0000000000..ae6b476d49 --- /dev/null +++ b/AK/Utf16StringBuilder.h @@ -0,0 +1,120 @@ +/* + * Copyright (c) 2026-present, the Ladybird developers. + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +namespace AK { + +class Utf16StringBuilder { +public: + static constexpr size_t inline_capacity = 256; + + class Buffer { + public: + Buffer() = default; + ~Buffer(); + + Buffer(Buffer const&) = delete; + Buffer(Buffer&&) = delete; + Buffer& operator=(Buffer const&) = delete; + Buffer& operator=(Buffer&&) = delete; + + [[nodiscard]] u8* data(); + [[nodiscard]] u8 const* data() const; + + [[nodiscard]] Bytes span() LIFETIME_BOUND; + [[nodiscard]] ReadonlyBytes span() const LIFETIME_BOUND; + + [[nodiscard]] size_t size() const { return m_size; } + [[nodiscard]] size_t capacity() const { return m_inline ? inline_capacity : m_outline_capacity; } + [[nodiscard]] bool is_inline() const { return m_inline; } + + void clear(); + void resize(size_t); + void set_size(size_t); + void ensure_capacity(size_t); + void append(void const*, size_t); + + struct OutlineBuffer { + Bytes buffer; + size_t capacity { 0 }; + }; + Optional leak_outline_buffer(); + + private: + void trim(size_t, bool may_discard_existing_data); + void shrink_into_inline_buffer(size_t, bool may_discard_existing_data); + void ensure_capacity_slowpath(size_t); + + union { + u8 m_inline_buffer[inline_capacity]; + struct { + u8* m_outline_buffer; + size_t m_outline_capacity; + }; + }; + size_t m_size { 0 }; + bool m_inline { true }; + }; + + Utf16StringBuilder(); + explicit Utf16StringBuilder(size_t initial_capacity_in_code_units); + ~Utf16StringBuilder() = default; + + Utf16StringBuilder(Utf16StringBuilder const&) = delete; + Utf16StringBuilder(Utf16StringBuilder&&) = delete; + Utf16StringBuilder& operator=(Utf16StringBuilder const&) = delete; + Utf16StringBuilder& operator=(Utf16StringBuilder&&) = delete; + + void append(Utf16View const&); + void append_ascii(char); + void append_ascii(StringView); + void append_code_unit(char16_t); + void append_code_point(u32); + void append_repeated_ascii(char, size_t); + void append_repeated(Utf16View const&, size_t); + + [[nodiscard]] Utf16String to_string(); + + [[nodiscard]] Utf16View view() const; + [[nodiscard]] size_t length_in_code_units() const; + [[nodiscard]] bool is_empty() const { return length_in_code_units() == 0; } + void trim(size_t count_in_code_units); + void clear(); + + Optional leak_buffer_for_string_construction(Badge) + { + return leak_buffer_for_string_construction(); + } + +private: + void initialize_buffer(size_t capacity_in_code_units); + void will_append(size_t size_in_bytes); + void ensure_storage_is_utf16(); + + Optional leak_buffer_for_string_construction(); + + [[nodiscard]] u8* data(); + [[nodiscard]] u8 const* data() const; + [[nodiscard]] size_t length_in_bytes() const; + + Buffer m_buffer; + bool m_is_ascii { true }; +}; + +} + +#if USING_AK_GLOBALLY +using AK::Utf16StringBuilder; +#endif diff --git a/AK/Utf16StringData.cpp b/AK/Utf16StringData.cpp index 9337dc4f8d..eea93d5e2c 100644 --- a/AK/Utf16StringData.cpp +++ b/AK/Utf16StringData.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include @@ -136,6 +137,33 @@ NonnullRefPtr Utf16StringData::from_string_builder(StringBuilde return string.release_nonnull(); } +NonnullRefPtr Utf16StringData::from_string_builder(Utf16StringBuilder& builder) +{ + auto view = builder.view(); + + auto code_unit_length = view.length_in_code_units(); + VERIFY_UTF16_LENGTH(code_unit_length); + + RefPtr string; + + if (auto buffer = builder.leak_buffer_for_string_construction(Badge {}); buffer.has_value()) { + auto storage_type = view.has_ascii_storage() ? StorageType::ASCII : StorageType::UTF16; + string = adopt_ref(*new (buffer->buffer.data()) Utf16StringData { storage_type, code_unit_length }); + } else { + if (view.has_ascii_storage()) { + string = create_uninitialized(StorageType::ASCII, code_unit_length); + TypedTransfer::copy(string->m_ascii_data, view.ascii_span().data(), code_unit_length); + } else { + string = create_uninitialized(StorageType::UTF16, code_unit_length); + TypedTransfer::copy(string->m_utf16_data, view.utf16_span().data(), code_unit_length); + + string->m_length_in_code_points = view.m_length_in_code_points; + } + } + + return string.release_nonnull(); +} + ErrorOr> Utf16StringData::from_ipc_stream(Stream& stream, size_t length_in_code_units, bool is_ascii) { RefPtr string; diff --git a/AK/Utf16StringData.h b/AK/Utf16StringData.h index f39fab1211..14cda660c1 100644 --- a/AK/Utf16StringData.h +++ b/AK/Utf16StringData.h @@ -35,6 +35,7 @@ public: static NonnullRefPtr from_ascii(ReadonlyBytes); static NonnullRefPtr from_utf16(Utf16View const&); static NonnullRefPtr from_string_builder(StringBuilder&); + static NonnullRefPtr from_string_builder(Utf16StringBuilder&); static ErrorOr> from_ipc_stream(Stream&, size_t length_in_code_units, bool is_ascii); static NonnullRefPtr to_well_formed(Utf16View const&); diff --git a/AK/Utf16View.h b/AK/Utf16View.h index 3660174a19..78b67bdc34 100644 --- a/AK/Utf16View.h +++ b/AK/Utf16View.h @@ -660,6 +660,7 @@ public: private: friend StringBuilder; + friend Utf16StringBuilder; friend Detail::Utf16StringBase; friend Detail::Utf16StringData; diff --git a/Tests/AK/CMakeLists.txt b/Tests/AK/CMakeLists.txt index 19395c9e77..d482af9d0c 100644 --- a/Tests/AK/CMakeLists.txt +++ b/Tests/AK/CMakeLists.txt @@ -74,6 +74,7 @@ set(AK_TEST_SOURCES TestUFixedBigInt.cpp TestUtf16FlyString.cpp TestUtf16String.cpp + TestUtf16StringBuilder.cpp TestUtf16View.cpp TestUtf8View.cpp TestVariant.cpp diff --git a/Tests/AK/TestUtf16StringBuilder.cpp b/Tests/AK/TestUtf16StringBuilder.cpp new file mode 100644 index 0000000000..e11e0dcdd1 --- /dev/null +++ b/Tests/AK/TestUtf16StringBuilder.cpp @@ -0,0 +1,110 @@ +/* + * Copyright (c) 2026-present, the Ladybird developers. + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include +#include +#include + +TEST_CASE(build_ascii_string) +{ + Utf16StringBuilder builder; + builder.append_ascii("hello"sv); + builder.append_ascii('!'); + + EXPECT_EQ(builder.view(), "hello!"sv); + + auto string = builder.to_string(); + EXPECT_EQ(string, "hello!"sv); + EXPECT(string.has_short_ascii_storage()); +} + +TEST_CASE(widen_to_utf16_storage) +{ + Utf16StringBuilder builder; + builder.append_ascii("hello "sv); + builder.append_code_unit(0x263A); + + EXPECT_EQ(builder.view(), u"hello \u263A"sv); + + auto string = builder.to_string(); + EXPECT_EQ(string, u"hello \u263A"sv); + EXPECT(!string.is_ascii()); +} + +TEST_CASE(append_code_point) +{ + Utf16StringBuilder builder; + builder.append_ascii("wave "sv); + builder.append_code_point(0x1F44B); + + auto string = builder.to_string(); + EXPECT_EQ(string, u"wave \U0001F44B"sv); + EXPECT_EQ(string.length_in_code_units(), 7uz); + EXPECT_EQ(string.length_in_code_points(), 6uz); +} + +TEST_CASE(trim_ascii_storage) +{ + Utf16StringBuilder builder; + builder.append_ascii("abc"sv); + + builder.trim(1); + + EXPECT_EQ(builder.view(), "ab"sv); + EXPECT_EQ(builder.to_string(), "ab"sv); +} + +TEST_CASE(trim_utf16_storage) +{ + Utf16StringBuilder builder; + builder.append(u"ab\u0100"sv); + + builder.trim(1); + + EXPECT_EQ(builder.view(), "ab"sv); + EXPECT_EQ(builder.to_string(), "ab"sv); +} + +TEST_CASE(long_ascii_string) +{ + Utf16StringBuilder builder; + builder.append_repeated_ascii('x', 300); + + auto string = builder.to_string(); + EXPECT_EQ(string.length_in_code_units(), 300uz); + EXPECT(string.has_long_ascii_storage()); + EXPECT_EQ(string.utf16_view().code_unit_at(0), 'x'); + EXPECT_EQ(string.utf16_view().code_unit_at(299), 'x'); +} + +TEST_CASE(long_utf16_string) +{ + Utf16StringBuilder builder; + builder.append_repeated(u"\u0100"sv, 300); + + auto string = builder.to_string(); + EXPECT_EQ(string.length_in_code_units(), 300uz); + EXPECT(!string.is_ascii()); + EXPECT_EQ(string.utf16_view().code_unit_at(0), 0x0100); + EXPECT_EQ(string.utf16_view().code_unit_at(299), 0x0100); +} + +TEST_CASE(reuse_after_creating_long_utf16_string) +{ + Utf16StringBuilder builder; + builder.append_repeated(u"\u0100"sv, 300); + + auto string = builder.to_string(); + EXPECT_EQ(string.length_in_code_units(), 300uz); + EXPECT(!string.is_ascii()); + + EXPECT(builder.is_empty()); + + builder.append_ascii("ok"sv); + + EXPECT_EQ(builder.view(), "ok"sv); + EXPECT_EQ(builder.to_string(), "ok"sv); +}