From 503453eb9c33068a8122daf6e9b029b8585cded4 Mon Sep 17 00:00:00 2001 From: Andreas Kling Date: Mon, 22 Jun 2026 14:28:38 +0200 Subject: [PATCH] AK: Remove UTF-16 mode from StringBuilder Remove the UTF-16 storage mode and make StringBuilder solely build UTF-8 strings again. The remaining UTF-16 adoption path now goes through Utf16StringBuilder, keeping the direct-adoption optimization with the type that owns UTF-16 construction. --- AK/StringBuilder.cpp | 207 +++++------------------------------ AK/StringBuilder.h | 17 +-- AK/Utf16String.cpp | 17 --- AK/Utf16String.h | 1 - AK/Utf16StringData.cpp | 27 ----- AK/Utf16StringData.h | 1 - Tests/AK/TestUtf16String.cpp | 2 +- 7 files changed, 28 insertions(+), 244 deletions(-) diff --git a/AK/StringBuilder.cpp b/AK/StringBuilder.cpp index 39a52fa790..3f7e99bf78 100644 --- a/AK/StringBuilder.cpp +++ b/AK/StringBuilder.cpp @@ -252,20 +252,14 @@ ErrorOr StringBuilder::Buffer::try_ensure_capacity_slowpath(size_t new_cap return {}; } -static constexpr size_t string_builder_prefix_size(StringBuilder::Mode mode) +static constexpr size_t string_builder_prefix_size() { - switch (mode) { - case StringBuilder::Mode::UTF8: - return sizeof(Detail::StringData); - case StringBuilder::Mode::UTF16: - return Detail::Utf16StringData::offset_of_string_storage(); - } - VERIFY_NOT_REACHED(); + return sizeof(Detail::StringData); } -void StringBuilder::initialize_buffer(Mode mode, size_t capacity) +void StringBuilder::initialize_buffer(size_t capacity) { - auto prefix_size = string_builder_prefix_size(mode); + auto prefix_size = string_builder_prefix_size(); if (capacity > StringBuilder::inline_capacity) m_buffer.ensure_capacity(prefix_size + capacity); m_buffer.resize(prefix_size); @@ -273,27 +267,15 @@ void StringBuilder::initialize_buffer(Mode mode, size_t capacity) StringBuilder::StringBuilder() { - static constexpr auto prefix_size = string_builder_prefix_size(DEFAULT_MODE); + static constexpr auto prefix_size = string_builder_prefix_size(); static_assert(inline_capacity > prefix_size); - initialize_buffer(m_mode, inline_capacity); + initialize_buffer(inline_capacity); } StringBuilder::StringBuilder(size_t initial_capacity) { - initialize_buffer(m_mode, initial_capacity); -} - -StringBuilder::StringBuilder(Mode mode) - : m_mode(mode) -{ - initialize_buffer(m_mode, inline_capacity); -} - -StringBuilder::StringBuilder(Mode mode, size_t initial_capacity_in_code_units) - : m_mode(mode) -{ - initialize_buffer(mode, initial_capacity_in_code_units * (mode == Mode::UTF8 ? 1 : 2)); + initialize_buffer(initial_capacity); } inline ErrorOr StringBuilder::will_append(size_t size_in_bytes) @@ -311,32 +293,9 @@ inline ErrorOr StringBuilder::will_append(size_t size_in_bytes) return {}; } -ErrorOr StringBuilder::ensure_storage_is_utf16() -{ - if (!exchange(m_utf16_builder_is_ascii, false)) - return {}; - if (is_empty()) - return {}; - - auto ascii_length = this->length(); - TRY(m_buffer.try_resize(m_buffer.size() + ascii_length)); - - Bytes source { data(), ascii_length }; - Span target { reinterpret_cast(data()), ascii_length }; - - for (size_t i = ascii_length; i > 0; --i) { - auto index = i - 1; - - auto ch = static_cast(source[index]); - target.overwrite(index, &ch, sizeof(char16_t)); - } - - return {}; -} - size_t StringBuilder::length() const { - return m_buffer.size() - string_builder_prefix_size(m_mode); + return m_buffer.size() - string_builder_prefix_size(); } bool StringBuilder::is_empty() const @@ -346,9 +305,6 @@ bool StringBuilder::is_empty() const void StringBuilder::trim(size_t count) { - if (m_mode == Mode::UTF16) - count *= 2; - auto decrease_count = min(m_buffer.size(), count); m_buffer.resize(m_buffer.size() - decrease_count); } @@ -358,17 +314,8 @@ ErrorOr StringBuilder::try_append(StringView string) if (string.is_empty()) return {}; - if (m_mode == Mode::UTF8 || (m_utf16_builder_is_ascii && string.is_ascii())) { - TRY(will_append(string.length())); - TRY(m_buffer.try_append(string.characters_without_null_termination(), string.length())); - } else { - TRY(ensure_storage_is_utf16()); - - TRY(will_append(string.length() * 2)); - for (auto code_point : Utf8View { string }) - TRY(try_append_code_point(code_point)); - } - + TRY(will_append(string.length())); + TRY(m_buffer.try_append(string.characters_without_null_termination(), string.length())); return {}; } @@ -382,52 +329,26 @@ ErrorOr StringBuilder::try_append_ascii_without_validation(ReadonlyBytes s if (string.is_empty()) return {}; - if (m_mode == Mode::UTF8 || m_utf16_builder_is_ascii) { - TRY(m_buffer.try_append(string)); - } else { - if (m_mode == Mode::UTF16) { - TRY(ensure_storage_is_utf16()); - TRY(will_append(string.size() * 2)); - } else { - TRY(will_append(string.size())); - } - for (auto code_point : Utf8View { string }) - TRY(try_append_code_point(code_point)); - } - + TRY(will_append(string.size())); + TRY(m_buffer.try_append(string)); return {}; } ErrorOr StringBuilder::try_append(char ch) { - if (m_mode == Mode::UTF8 || (m_utf16_builder_is_ascii && is_ascii(ch))) { - TRY(will_append(1)); - TRY(m_buffer.try_append(ch)); - } else { - TRY(ensure_storage_is_utf16()); - TRY(try_append_code_unit(ch)); - } - + TRY(will_append(1)); + TRY(m_buffer.try_append(ch)); return {}; } ErrorOr StringBuilder::try_append_code_unit(char16_t ch) { - if (m_mode == Mode::UTF8 || (m_utf16_builder_is_ascii && is_ascii(ch))) { - TRY(try_append_code_point(ch)); - } else { - TRY(ensure_storage_is_utf16()); - TRY(will_append(2)); - TRY(m_buffer.try_append(&ch, sizeof(ch))); - } - - return {}; + return try_append_code_point(ch); } ErrorOr StringBuilder::try_append_repeated(char ch, size_t n) { - auto append_as_utf8 = m_mode == Mode::UTF8 || (m_utf16_builder_is_ascii && is_ascii(ch)); - TRY(will_append(n * (append_as_utf8 ? 1 : 2))); + TRY(will_append(n)); for (size_t i = 0; i < n; ++i) TRY(try_append(ch)); @@ -440,12 +361,7 @@ ErrorOr StringBuilder::try_append_repeated(StringView string, size_t n) if (string.is_empty()) return {}; - if (m_mode == Mode::UTF8 || (m_utf16_builder_is_ascii && string.is_ascii())) { - TRY(will_append(string.length() * n)); - } else { - auto utf16_length = simdutf::utf16_length_from_utf8(string.characters_without_null_termination(), string.length()); - TRY(will_append(utf16_length * n * 2)); - } + TRY(will_append(string.length() * n)); for (size_t i = 0; i < n; ++i) TRY(try_append(string)); @@ -458,15 +374,11 @@ ErrorOr StringBuilder::try_append_repeated(Utf16View const& string, size_t if (string.is_empty()) return {}; - if (m_mode == Mode::UTF8 || (m_utf16_builder_is_ascii && string.is_ascii())) { - if (string.has_ascii_storage()) { - TRY(will_append(string.length_in_code_units() * n)); - } else { - auto utf8_length = simdutf::utf8_length_from_utf16(string.utf16_span().data(), string.length_in_code_units()); - TRY(will_append(utf8_length * n)); - } + if (string.has_ascii_storage()) { + TRY(will_append(string.length_in_code_units() * n)); } else { - TRY(will_append(string.length_in_code_units() * n * 2)); + auto utf8_length = simdutf::utf8_length_from_utf16(string.utf16_span().data(), string.length_in_code_units()); + TRY(will_append(utf8_length * n)); } for (size_t i = 0; i < n; ++i) @@ -522,7 +434,6 @@ ErrorOr StringBuilder::to_byte_buffer() const ByteString StringBuilder::to_byte_string() const { - VERIFY(m_mode == Mode::UTF8); if (is_empty()) return ByteString::empty(); return ByteString((char const*)data(), length()); @@ -530,7 +441,6 @@ ByteString StringBuilder::to_byte_string() const ErrorOr StringBuilder::to_string() { - VERIFY(m_mode == Mode::UTF8); if (m_buffer.is_inline()) return String::from_utf8(string_view()); return String::from_string_builder({}, *this); @@ -538,7 +448,6 @@ ErrorOr StringBuilder::to_string() String StringBuilder::to_string_without_validation() { - VERIFY(m_mode == Mode::UTF8); if (m_buffer.is_inline()) return String::from_utf8_without_validation(string_view().bytes()); return String::from_string_builder_without_validation({}, *this); @@ -546,53 +455,32 @@ String StringBuilder::to_string_without_validation() FlyString StringBuilder::to_fly_string_without_validation() const { - VERIFY(m_mode == Mode::UTF8); return FlyString::from_utf8_without_validation(string_view().bytes()); } ErrorOr StringBuilder::to_fly_string() const { - VERIFY(m_mode == Mode::UTF8); return FlyString::from_utf8(string_view()); } -Utf16String StringBuilder::to_utf16_string() -{ - VERIFY(m_mode == Mode::UTF16); - return Utf16String::from_string_builder({}, *this); -} - u8* StringBuilder::data() { - return m_buffer.data() + string_builder_prefix_size(m_mode); + return m_buffer.data() + string_builder_prefix_size(); } u8 const* StringBuilder::data() const { - return m_buffer.data() + string_builder_prefix_size(m_mode); + return m_buffer.data() + string_builder_prefix_size(); } StringView StringBuilder::string_view() const { - VERIFY(m_mode == Mode::UTF8); - return m_buffer.span().slice(string_builder_prefix_size(m_mode)); -} - -Utf16View StringBuilder::utf16_string_view() const -{ - VERIFY(m_mode == Mode::UTF16); - auto view = m_buffer.span().slice(string_builder_prefix_size(m_mode)); - - if (m_utf16_builder_is_ascii) - return { reinterpret_cast(view.data()), view.size() }; - return { reinterpret_cast(view.data()), view.size() / 2 }; + return m_buffer.span().slice(string_builder_prefix_size()); } void StringBuilder::clear() { - m_buffer.resize(string_builder_prefix_size(m_mode)); - if (m_mode == Mode::UTF16) - m_utf16_builder_is_ascii = true; + m_buffer.resize(string_builder_prefix_size()); } ErrorOr StringBuilder::try_append_code_point(u32 code_point) @@ -602,14 +490,7 @@ ErrorOr StringBuilder::try_append_code_point(u32 code_point) return {}; } - if (m_mode == Mode::UTF8 || (m_utf16_builder_is_ascii && is_ascii(code_point))) { - TRY(AK::UnicodeUtils::try_code_point_to_utf8(code_point, [this](char c) { return try_append(c); })); - } else { - TRY(ensure_storage_is_utf16()); - - TRY(AK::UnicodeUtils::try_code_point_to_utf16(code_point, [this](char16_t c) { return m_buffer.try_append(&c, sizeof(c)); })); - } - + TRY(AK::UnicodeUtils::try_code_point_to_utf8(code_point, [this](char c) { return try_append(c); })); return {}; } @@ -620,30 +501,6 @@ void StringBuilder::append_code_point(u32 code_point) return; } - auto append_as_utf8 = m_mode == Mode::UTF8 || (m_utf16_builder_is_ascii && is_ascii(code_point)); - - if (!append_as_utf8) { - MUST(ensure_storage_is_utf16()); - (void)(will_append(2)); - - if (code_point < UnicodeUtils::FIRST_SUPPLEMENTARY_PLANE_CODE_POINT) { - auto code_unit = static_cast(code_point); - m_buffer.append(&code_unit, sizeof(code_unit)); - return; - } - - (void)(will_append(2)); - code_point -= UnicodeUtils::FIRST_SUPPLEMENTARY_PLANE_CODE_POINT; - - auto code_unit = static_cast(UnicodeUtils::HIGH_SURROGATE_MIN | (code_point >> 10)); - m_buffer.append(&code_unit, sizeof(code_unit)); - - code_unit = static_cast(UnicodeUtils::LOW_SURROGATE_MIN | (code_point & 0x3ff)); - m_buffer.append(&code_unit, sizeof(code_unit)); - - return; - } - if (code_point <= 0x7f) { m_buffer.append(static_cast(code_point)); } else if (code_point <= 0x07ff) { @@ -671,18 +528,6 @@ ErrorOr StringBuilder::try_append(Utf16View const& utf16_view) if (utf16_view.has_ascii_storage()) return try_append_ascii_without_validation(utf16_view.bytes()); - auto append_as_utf8 = m_mode == Mode::UTF8 || (m_utf16_builder_is_ascii && utf16_view.is_ascii()); - - if (!append_as_utf8) { - TRY(ensure_storage_is_utf16()); - TRY(will_append(utf16_view.length_in_code_units() * 2)); - - for (size_t i = 0; i < utf16_view.length_in_code_units(); ++i) - TRY(try_append_code_unit(utf16_view.code_unit_at(i))); - - return {}; - } - auto remaining_view = utf16_view.utf16_span(); auto maximum_utf8_length = UnicodeUtils::maximum_utf8_length_from_utf16(remaining_view); diff --git a/AK/StringBuilder.h b/AK/StringBuilder.h index feefe6102e..2a145ac2e4 100644 --- a/AK/StringBuilder.h +++ b/AK/StringBuilder.h @@ -16,12 +16,6 @@ namespace AK { class StringBuilder { public: - enum class Mode { - UTF8, - UTF16, - }; - - static constexpr auto DEFAULT_MODE = Mode::UTF8; static constexpr size_t inline_capacity = 256; class Buffer { @@ -86,9 +80,6 @@ public: StringBuilder(); explicit StringBuilder(size_t initial_capacity); - explicit StringBuilder(Mode); - StringBuilder(Mode, size_t initial_capacity_in_code_units); - ~StringBuilder() = default; ErrorOr try_append(StringView); @@ -139,12 +130,9 @@ public: [[nodiscard]] FlyString to_fly_string_without_validation() const; ErrorOr to_fly_string() const; - Utf16String to_utf16_string(); - [[nodiscard]] ErrorOr to_byte_buffer() const; [[nodiscard]] StringView string_view() const; - [[nodiscard]] Utf16View utf16_string_view() const; void clear(); [[nodiscard]] size_t length() const; @@ -176,19 +164,16 @@ public: } private: - void initialize_buffer(Mode, size_t capacity); + void initialize_buffer(size_t capacity); Optional leak_buffer_for_string_construction(); ErrorOr will_append(size_t); - ErrorOr ensure_storage_is_utf16(); u8* data(); u8 const* data() const; Buffer m_buffer; - Mode m_mode { DEFAULT_MODE }; - bool m_utf16_builder_is_ascii { true }; }; } diff --git a/AK/Utf16String.cpp b/AK/Utf16String.cpp index 2fc5087abd..d9a095d230 100644 --- a/AK/Utf16String.cpp +++ b/AK/Utf16String.cpp @@ -87,23 +87,6 @@ Utf16String Utf16String::from_utf16(Utf16View const& utf16_string) return Utf16String { Detail::Utf16StringData::from_utf16(utf16_string) }; } -Utf16String Utf16String::from_string_builder(Badge, StringBuilder& builder) -{ - auto view = builder.utf16_string_view(); - - if (view.length_in_code_units() <= Detail::MAX_SHORT_STRING_BYTE_COUNT && view.has_ascii_storage()) { - Utf16String string; - string.m_value.short_ascii_string = Detail::ShortString::create_with_byte_count(view.length_in_code_units()); - - auto result = view.bytes().copy_to(string.m_value.short_ascii_string.storage); - VERIFY(result == view.length_in_code_units()); - - return string; - } - - return Utf16String { Detail::Utf16StringData::from_string_builder(builder) }; -} - Utf16String Utf16String::from_string_builder(Badge, Utf16StringBuilder& builder) { auto view = builder.view(); diff --git a/AK/Utf16String.h b/AK/Utf16String.h index 3d3c4b8173..26fed6dfaf 100644 --- a/AK/Utf16String.h +++ b/AK/Utf16String.h @@ -236,7 +236,6 @@ public: ALWAYS_INLINE Utf16String escape_html_entities() const { return utf16_view().escape_html_entities(); } - static Utf16String from_string_builder(Badge, StringBuilder& builder); static Utf16String from_string_builder(Badge, Utf16StringBuilder& builder); static ErrorOr from_ipc_stream(Stream&, size_t length_in_code_units, bool is_ascii); diff --git a/AK/Utf16StringData.cpp b/AK/Utf16StringData.cpp index eea93d5e2c..70bce6e81a 100644 --- a/AK/Utf16StringData.cpp +++ b/AK/Utf16StringData.cpp @@ -110,33 +110,6 @@ NonnullRefPtr Utf16StringData::from_utf16(Utf16View const& utf1 return string.release_nonnull(); } -NonnullRefPtr Utf16StringData::from_string_builder(StringBuilder& builder) -{ - auto view = builder.utf16_string_view(); - - auto code_unit_length = view.length_in_code_units(); - VERIFY_UTF16_LENGTH(code_unit_length); - - RefPtr string; - - if (auto buffer = builder.leak_buffer_for_string_construction(Badge {}); buffer.has_value()) { - auto storage_type = view.has_ascii_storage() ? StorageType::ASCII : StorageType::UTF16; - string = adopt_ref(*new (buffer->buffer.data()) Utf16StringData { storage_type, code_unit_length }); - } else { - if (view.has_ascii_storage()) { - string = create_uninitialized(StorageType::ASCII, code_unit_length); - TypedTransfer::copy(string->m_ascii_data, view.ascii_span().data(), code_unit_length); - } else { - string = create_uninitialized(StorageType::UTF16, code_unit_length); - TypedTransfer::copy(string->m_utf16_data, view.utf16_span().data(), code_unit_length); - - string->m_length_in_code_points = view.m_length_in_code_points; - } - } - - return string.release_nonnull(); -} - NonnullRefPtr Utf16StringData::from_string_builder(Utf16StringBuilder& builder) { auto view = builder.view(); diff --git a/AK/Utf16StringData.h b/AK/Utf16StringData.h index 14cda660c1..a84c83a197 100644 --- a/AK/Utf16StringData.h +++ b/AK/Utf16StringData.h @@ -34,7 +34,6 @@ public: static NonnullRefPtr from_utf8(StringView, AllowASCIIStorage); static NonnullRefPtr from_ascii(ReadonlyBytes); static NonnullRefPtr from_utf16(Utf16View const&); - static NonnullRefPtr from_string_builder(StringBuilder&); static NonnullRefPtr from_string_builder(Utf16StringBuilder&); static ErrorOr> from_ipc_stream(Stream&, size_t length_in_code_units, bool is_ascii); diff --git a/Tests/AK/TestUtf16String.cpp b/Tests/AK/TestUtf16String.cpp index 691925b38b..253109b401 100644 --- a/Tests/AK/TestUtf16String.cpp +++ b/Tests/AK/TestUtf16String.cpp @@ -1249,7 +1249,7 @@ TEST_CASE(utf16_string_builder_clear_resets_ascii_flag) // subsequent ASCII as char16_t, corrupting the first code unit when adopted. Utf16StringBuilder builder; - // 1. Append a non-ASCII code point to force m_utf16_builder_is_ascii = false. + // 1. Append a non-ASCII code point to force UTF-16 storage. builder.append_code_point(0x00D7); // U+00D7 MULTIPLICATION SIGN (×) auto first = builder.to_string(); EXPECT_EQ(first.length_in_code_units(), 1u);