From f161215f53323ecc91ce409c8b28ebd6381b18f2 Mon Sep 17 00:00:00 2001 From: Tim Ledbetter Date: Mon, 27 Apr 2026 16:25:50 +0100 Subject: [PATCH] LibUnicode: Add an ASCII fast path for line break segmentation Add a Segmenter implementation that implements the UAX#14 line breaking rules applicable to ASCII text. This avoids the need to build an ICU BreakIterator for the majority of text on the web. --- Libraries/LibUnicode/Segmenter.cpp | 414 +++++++++++++++++++++++++++++ Libraries/LibUnicode/Segmenter.h | 2 + Tests/LibUnicode/TestSegmenter.cpp | 224 ++++++++++++++++ 3 files changed, 640 insertions(+) diff --git a/Libraries/LibUnicode/Segmenter.cpp b/Libraries/LibUnicode/Segmenter.cpp index d7e05f94ce..14a7f9da6d 100644 --- a/Libraries/LibUnicode/Segmenter.cpp +++ b/Libraries/LibUnicode/Segmenter.cpp @@ -4,6 +4,9 @@ * SPDX-License-Identifier: BSD-2-Clause */ +#include +#include +#include #include #include #include @@ -131,6 +134,404 @@ private: size_t m_current { 0 }; }; +static bool can_use_ascii_line_breaking_fast_path(ReadonlyBytes bytes) +{ + return all_of(bytes, [](u8 byte) { return is_ascii_printable(byte) || is_ascii_space(byte); }); +} + +// UAX#14 line-break classes that occur in printable ASCII plus ASCII whitespace. +// https://www.unicode.org/reports/tr14/#Table1 +enum class AsciiLineBreakClass : u8 { + AL, // Alphabetic (alphabets and regular symbols) + BA, // Break After (TAB, '|') + BK, // Mandatory Break (VT, FF) + CL, // Close Punctuation ('}') + CP, // Close Parenthesis (')', ']') + CR, // Carriage Return + EX, // Exclamation/Interrogation ('!', '?') + HY, // Hyphen ('-') + IS, // Infix Numeric Separator (',', '.', ':', ';') + LF, // Line Feed + NU, // Numeric (digits) + OP, // Open Punctuation ('(', '[', '{') + PO, // Postfix Numeric ('%') + PR, // Prefix Numeric ('$', '+', '\\') + QU, // Quotation ('"', '\'') + SP, // Space + SY, // Symbols Allowing Break After ('/') +}; + +static constexpr AsciiLineBreakClass classify_ascii_byte(u8 byte) +{ + using enum AsciiLineBreakClass; + switch (byte) { + case '\t': + case '|': + return BA; + case '\n': + return LF; + case '\v': + case '\f': + return BK; + case '\r': + return CR; + case ' ': + return SP; + case '!': + case '?': + return EX; + case '"': + case '\'': + return QU; + case '$': + case '+': + case '\\': + return PR; + case '%': + return PO; + case '(': + case '[': + case '{': + return OP; + case ')': + case ']': + return CP; + case ',': + case '.': + case ':': + case ';': + return IS; + case '-': + return HY; + case '/': + return SY; + case '}': + return CL; + default: + return is_ascii_digit(byte) ? NU : AL; + } +} + +static void mark_ascii_numeric_expression_interiors(ReadonlyBytes text, Vector& interior_of_numeric_expression) +{ + // Identify spans matching UAX#14 LB25's numeric-expression grammar: + // (PR | PO)? (OP | HY)? IS? NU (NU | SY | IS)* (CL | CP)? (PR | PO)? + // Positions strictly inside such a span are kept atomic by LB25 — positions at the very start or end of a span are + // governed by surrounding rules so adjacent numeric expressions can still break apart. + + using enum AsciiLineBreakClass; + interior_of_numeric_expression.clear_with_capacity(); + interior_of_numeric_expression.resize(text.size() + 1); + + size_t i = 0; + while (i < text.size()) { + if (classify_ascii_byte(text[i]) != NU) { + ++i; + continue; + } + + // Walk the prefix backwards: NU is preceded by an optional IS, then optional OP/HY, then optional PR/PO. + size_t start = i; + if (start > 0 && classify_ascii_byte(text[start - 1]) == IS) + --start; + if (start > 0 && first_is_one_of(classify_ascii_byte(text[start - 1]), OP, HY)) + --start; + if (start > 0 && first_is_one_of(classify_ascii_byte(text[start - 1]), PR, PO)) + --start; + + // Walk the body and suffix forward: NU (NU | SY | IS)* (CL | CP)? (PR | PO)? + size_t end = i + 1; + while (end < text.size() && first_is_one_of(classify_ascii_byte(text[end]), NU, SY, IS)) + ++end; + if (end < text.size() && first_is_one_of(classify_ascii_byte(text[end]), CL, CP)) + ++end; + if (end < text.size() && first_is_one_of(classify_ascii_byte(text[end]), PR, PO)) + ++end; + + // Mark positions strictly between `start` and `end` as interior. Position `start` and position `end` + // are left alone so adjacent expressions can still break against each other. + for (size_t position = start + 1; position < end; ++position) + interior_of_numeric_expression[position] = true; + + i = end; + } +} + +static void compute_ascii_line_boundaries(ReadonlyBytes text, Vector& is_boundary) +{ + // Compute soft and mandatory line-break opportunities for ASCII text following UAX#14: https://www.unicode.org/reports/tr14/) + // Only the rules whose left- and right-hand classes can occur in printable ASCII plus ASCII whitespace are + // implemented. + using enum AsciiLineBreakClass; + + is_boundary.clear_with_capacity(); + is_boundary.resize(text.size() + 1); + is_boundary[0] = true; + if (text.is_empty()) + return; + is_boundary[text.size()] = true; + + // LB25: precompute positions interior to numeric expressions. + Vector interior_of_numeric_expression; + mark_ascii_numeric_expression_interiors(text, interior_of_numeric_expression); + + // LB14 state: have we just seen `OP SP*` (with no intervening non-SP)? + bool in_op_sp_run = false; + + for (size_t i = 1; i < text.size(); ++i) { + auto previous_class = classify_ascii_byte(text[i - 1]); + auto current_class = classify_ascii_byte(text[i]); + + // LB5: CR × LF; CR ! ; LF ! . + if (previous_class == CR && current_class == LF) { + is_boundary[i] = false; + continue; + } + // LB4: BK ! . LB5 (continued): CR ! and LF ! . + if (first_is_one_of(previous_class, BK, CR, LF)) { + is_boundary[i] = true; + in_op_sp_run = false; + continue; + } + // LB6: × ( BK | CR | LF ). + if (first_is_one_of(current_class, BK, CR, LF)) { + is_boundary[i] = false; + continue; + } + + // LB14 trigger: `OP` starts an `OP SP*` run that suppresses breaks until the next non-SP. + if (previous_class == OP) + in_op_sp_run = true; + + // LB7: × SP. The OP-SP* run is preserved across spaces. + if (current_class == SP) { + is_boundary[i] = false; + continue; + } + + // LB14: OP SP* × (no break after OP, even across intervening spaces). + if (in_op_sp_run) { + is_boundary[i] = false; + in_op_sp_run = false; + continue; + } + + // LB13: × CL × CP × EX × SY (IS is handled by LB15c/d below). + if (first_is_one_of(current_class, CL, CP, EX, SY)) { + is_boundary[i] = false; + continue; + } + + // LB15d: do not break before IS unless preceded by SP, and even then only when SP IS is followedby NU. + if (current_class == IS) { + if (previous_class != SP) { + is_boundary[i] = false; + continue; + } + auto next_class = i + 1 < text.size() ? classify_ascii_byte(text[i + 1]) : AL; + bool next_forces_break = i + 1 < text.size() && next_class == NU; + if (!next_forces_break) { + is_boundary[i] = false; + continue; + } + // Fall through: SP × IS NU → break before IS (per LB15c "leading decimal point"). + is_boundary[i] = true; + continue; + } + + // LB18: SP ÷ (default break opportunity after spaces). + if (previous_class == SP) { + is_boundary[i] = true; + continue; + } + + // LB19: × QU; QU × (treat ASCII straight quotes as ambiguous QU). + if (current_class == QU || previous_class == QU) { + is_boundary[i] = false; + continue; + } + + // LB20a: do not break between a hyphen and a following letter when the hyphen comes at the start of a line - + // that is, at the start of the text, after a space, or after a forced break. + if (previous_class == HY && current_class == AL) { + bool hy_starts_line = i == 1; + if (!hy_starts_line && i >= 2) + hy_starts_line = first_is_one_of(classify_ascii_byte(text[i - 2]), SP, BK, CR, LF); + if (hy_starts_line) { + is_boundary[i] = false; + continue; + } + } + + // LB21: × BA; × HY (break-before suppression for BA and HY). + if (first_is_one_of(current_class, BA, HY)) { + is_boundary[i] = false; + continue; + } + + // LB23: AL × NU; NU × AL. + if ((previous_class == AL && current_class == NU) || (previous_class == NU && current_class == AL)) { + is_boundary[i] = false; + continue; + } + + // LB24: (PR | PO) × AL; AL × (PR | PO). + if ((first_is_one_of(previous_class, PR, PO) && current_class == AL) + || (previous_class == AL && first_is_one_of(current_class, PR, PO))) { + is_boundary[i] = false; + continue; + } + + // LB25: positions strictly inside a numeric expression are kept atomic. + if (interior_of_numeric_expression[i]) { + is_boundary[i] = false; + continue; + } + + // LB28: AL × AL. + if (previous_class == AL && current_class == AL) { + is_boundary[i] = false; + continue; + } + + // LB29: IS × AL. + if (previous_class == IS && current_class == AL) { + is_boundary[i] = false; + continue; + } + + // LB30: (AL | NU) × OP; CP × (AL | NU). All ASCII OP/CP characters are non-East-Asian. + if (current_class == OP && first_is_one_of(previous_class, AL, NU)) { + is_boundary[i] = false; + continue; + } + if (previous_class == CP && first_is_one_of(current_class, AL, NU)) { + is_boundary[i] = false; + continue; + } + + // LB31: ÷ (default). + is_boundary[i] = true; + } +} + +// Implements UAX#14 line breaking rules for ASCII text: https://www.unicode.org/reports/tr14/tr14-39.html +class AsciiLineSegmenter : public Segmenter { +public: + AsciiLineSegmenter() + : Segmenter(SegmenterGranularity::Line) + { + } + + virtual ~AsciiLineSegmenter() override = default; + + virtual NonnullOwnPtr clone() const override + { + return make(); + } + + virtual void set_segmented_text(String text) override + { + apply(text.bytes()); + } + + virtual void set_segmented_text(Utf16View const& text) override + { + VERIFY(text.has_ascii_storage()); + auto span = text.ascii_span(); + apply({ span.data(), span.size() }); + } + + virtual size_t current_boundary() override + { + return m_current; + } + + virtual Optional previous_boundary(size_t index, Inclusive inclusive) override + { + if (inclusive == Inclusive::Yes && index <= text_size() && is_boundary(index)) + return index; + if (index == 0) + return {}; + size_t i = min(index, text_size() + 1); + while (i > 0) { + --i; + if (is_boundary(i)) + return i; + } + return {}; + } + + virtual Optional next_boundary(size_t index, Inclusive inclusive) override + { + if (inclusive == Inclusive::Yes && index <= text_size() && is_boundary(index)) + return index; + if (index >= text_size()) + return {}; + for (size_t i = index + 1; i <= text_size(); ++i) { + if (is_boundary(i)) + return i; + } + return {}; + } + + virtual void for_each_boundary(String text, SegmentationCallback callback) override + { + if (text.is_empty()) + return; + set_segmented_text(move(text)); + iterate(callback); + } + + virtual void for_each_boundary(Utf16View const& text, SegmentationCallback callback) override + { + if (text.is_empty()) + return; + set_segmented_text(text); + iterate(callback); + } + + virtual void for_each_boundary(Utf32View const&, SegmentationCallback) override + { + VERIFY_NOT_REACHED(); + } + + virtual bool is_current_boundary_word_like() const override + { + return false; + } + +private: + void apply(ReadonlyBytes bytes) + { + VERIFY(can_use_ascii_line_breaking_fast_path(bytes)); + compute_ascii_line_boundaries(bytes, m_is_boundary); + m_current = 0; + } + + size_t text_size() const { return m_is_boundary.size() - 1; } + + bool is_boundary(size_t index) const + { + VERIFY(index < m_is_boundary.size()); + return m_is_boundary[index]; + } + + void iterate(SegmentationCallback& callback) + { + for (size_t i = 0; i <= text_size(); ++i) { + if (!m_is_boundary[i]) + continue; + m_current = i; + if (callback(i) == IterationDecision::Break) + return; + } + } + + Vector m_is_boundary; + size_t m_current { 0 }; +}; + class SegmenterImpl : public Segmenter { public: SegmenterImpl(NonnullOwnPtr segmenter, SegmenterGranularity segmenter_granularity) @@ -344,6 +745,19 @@ NonnullOwnPtr Segmenter::create_for_ascii_grapheme(size_t length) return make(length); } +OwnPtr Segmenter::try_create_for_ascii_line(Utf16View const& text) +{ + if (!text.has_ascii_storage()) + return {}; + auto span = text.ascii_span(); + ReadonlyBytes bytes { span.data(), span.size() }; + if (!can_use_ascii_line_breaking_fast_path(bytes)) + return {}; + auto segmenter = make(); + segmenter->set_segmented_text(text); + return segmenter; +} + bool Segmenter::should_continue_beyond_word(Utf16View const& word) { for (auto code_point : word) { diff --git a/Libraries/LibUnicode/Segmenter.h b/Libraries/LibUnicode/Segmenter.h index 292b1af2d6..924f2587c1 100644 --- a/Libraries/LibUnicode/Segmenter.h +++ b/Libraries/LibUnicode/Segmenter.h @@ -9,6 +9,7 @@ #include #include #include +#include #include namespace Unicode { @@ -27,6 +28,7 @@ public: static NonnullOwnPtr create(SegmenterGranularity segmenter_granularity); static NonnullOwnPtr create(StringView locale, SegmenterGranularity segmenter_granularity); static NonnullOwnPtr create_for_ascii_grapheme(size_t length); + static OwnPtr try_create_for_ascii_line(Utf16View const&); virtual ~Segmenter() = default; static bool should_continue_beyond_word(Utf16View const&); diff --git a/Tests/LibUnicode/TestSegmenter.cpp b/Tests/LibUnicode/TestSegmenter.cpp index 2b1cb169a4..928b2ca9aa 100644 --- a/Tests/LibUnicode/TestSegmenter.cpp +++ b/Tests/LibUnicode/TestSegmenter.cpp @@ -175,6 +175,230 @@ TEST_CASE(line_segmentation) test_line_segmentation("ab你好cd"sv, { 0u, 2u, 5u, 8u, 10u }); } +template +static void test_ascii_line_segmentation(Utf16String const& string, size_t const (&expected_boundaries)[N]) +{ + auto segmenter = Unicode::Segmenter::try_create_for_ascii_line(string.utf16_view()); + VERIFY(segmenter); + + Vector boundaries; + segmenter->for_each_boundary(string.utf16_view(), [&](auto boundary) { + boundaries.append(boundary); + return IterationDecision::Continue; + }); + + EXPECT_EQ(boundaries, ReadonlySpan { expected_boundaries }); +} + +static void expect_ascii_line_segmentation_matches_icu(Utf16String const& string, Unicode::Segmenter& icu_segmenter) +{ + auto ascii_segmenter = Unicode::Segmenter::try_create_for_ascii_line(string.utf16_view()); + VERIFY(ascii_segmenter); + + Vector icu_boundaries; + icu_segmenter.for_each_boundary(string.utf16_view(), [&](auto boundary) { + icu_boundaries.append(boundary); + return IterationDecision::Continue; + }); + + Vector ascii_boundaries; + ascii_segmenter->for_each_boundary(string.utf16_view(), [&](auto boundary) { + ascii_boundaries.append(boundary); + return IterationDecision::Continue; + }); + + EXPECT_EQ(ascii_boundaries, icu_boundaries); +} + +TEST_CASE(ascii_line_segmentation) +{ + auto empty_segmenter = Unicode::Segmenter::try_create_for_ascii_line({}); + VERIFY(empty_segmenter); + empty_segmenter->for_each_boundary(String {}, [&](auto) { + VERIFY_NOT_REACHED(); + return IterationDecision::Break; + }); + + // Single characters and atomic words. + test_ascii_line_segmentation("a"_utf16, { 0u, 1u }); + test_ascii_line_segmentation("abc"_utf16, { 0u, 3u }); + + // Break opportunity after whitespace. + test_ascii_line_segmentation("ab cd"_utf16, { 0u, 3u, 5u }); + test_ascii_line_segmentation("ab cd"_utf16, { 0u, 4u, 6u }); + test_ascii_line_segmentation("ab\tcd"_utf16, { 0u, 3u, 5u }); + + // Hard line breaks. + test_ascii_line_segmentation("ab\ncd"_utf16, { 0u, 3u, 5u }); + test_ascii_line_segmentation("ab\r\ncd"_utf16, { 0u, 4u, 6u }); + test_ascii_line_segmentation("ab\rcd"_utf16, { 0u, 3u, 5u }); + + // Alphanumerics are atomic across letter/digit boundaries. + test_ascii_line_segmentation("abc123"_utf16, { 0u, 6u }); + test_ascii_line_segmentation("123abc"_utf16, { 0u, 6u }); + test_ascii_line_segmentation("a 1 b"_utf16, { 0u, 2u, 4u, 5u }); + + // Printable ASCII punctuation follows UAX#14 line-breaking rules. + test_ascii_line_segmentation("example.com"_utf16, { 0u, 11u }); // LB15d (× IS) and LB29 (IS × AL) keep the dotted name atomic. + test_ascii_line_segmentation("hello, world"_utf16, { 0u, 7u, 12u }); // Break after the space (LB18). + test_ascii_line_segmentation("a/b/c"_utf16, { 0u, 2u, 4u, 5u }); // LB13 × SY, default ÷ after SY. + test_ascii_line_segmentation("http://a/b"_utf16, { 0u, 7u, 9u, 10u }); + test_ascii_line_segmentation("\"ab\""_utf16, { 0u, 4u }); // LB19 around QU. + test_ascii_line_segmentation("$100"_utf16, { 0u, 4u }); // LB25 PR × NU and NU × NU. + test_ascii_line_segmentation("5%"_utf16, { 0u, 2u }); // LB25 NU × PO. + test_ascii_line_segmentation("a+$b"_utf16, { 0u, 2u, 4u }); // LB24 AL × PR; PR × PR has no rule. + test_ascii_line_segmentation("x%20y"_utf16, { 0u, 5u }); // LB24 AL × PO; LB25 PO × NU; LB23 NU × AL. + test_ascii_line_segmentation(" .23"_utf16, { 0u, 1u, 4u }); // LB15c forces a break before a leading decimal point. + test_ascii_line_segmentation("1/2"_utf16, { 0u, 3u }); // LB25 NU × SY × NU. + test_ascii_line_segmentation("1)$"_utf16, { 0u, 3u }); // LB25 NU × CP and CP × PR. + test_ascii_line_segmentation("(ab)"_utf16, { 0u, 4u }); // LB14 OP × ; LB13 × CP. + test_ascii_line_segmentation(") ("_utf16, { 0u, 2u, 3u }); // LB18 break after SP. + test_ascii_line_segmentation("a(b)c"_utf16, { 0u, 5u }); // LB30 AL × OP and CP × AL. + test_ascii_line_segmentation("a-\"b"_utf16, { 0u, 4u }); // LB21 × HY and LB19 around QU. + test_ascii_line_segmentation("what?yes"_utf16, { 0u, 5u, 8u }); // LB13 × EX, default ÷ after EX. + test_ascii_line_segmentation("what?\"yes\""_utf16, { 0u, 10u }); // LB19 keeps the quoted segment atomic. + + // U+007C VERTICAL LINE is UAX#14 class BA: no break before, break after. + test_ascii_line_segmentation("a|b"_utf16, { 0u, 2u, 3u }); + test_ascii_line_segmentation("aaa|bbb"_utf16, { 0u, 4u, 7u }); + test_ascii_line_segmentation("a||b"_utf16, { 0u, 3u, 4u }); + + // Hyphen behavior follows UAX#14: × HY before, ÷ after when not in a numeric expression. + // LB20a additionally glues a leading hyphen to the following letter. + test_ascii_line_segmentation("-foo"_utf16, { 0u, 4u }); + test_ascii_line_segmentation("x -foo"_utf16, { 0u, 2u, 6u }); + test_ascii_line_segmentation("word-break"_utf16, { 0u, 5u, 10u }); + // LB25 keeps numeric expressions atomic across hyphens (HY × NU). + test_ascii_line_segmentation("-2"_utf16, { 0u, 2u }); + test_ascii_line_segmentation("foo-2"_utf16, { 0u, 5u }); + test_ascii_line_segmentation("ABCD-1234"_utf16, { 0u, 9u }); + test_ascii_line_segmentation("-#tag"_utf16, { 0u, 5u }); +} + +TEST_CASE(ascii_line_segmentation_matches_icu_for_context_independent_text) +{ + // The fast path implements UAX#14 for ASCII inputs and should agree with ICU on every input it accepts. + Array test_strings = { + "a"_utf16, + "abc"_utf16, + "ab cd"_utf16, + "ab cd"_utf16, + "ab\tcd"_utf16, + "ab\ncd"_utf16, + "ab\r\ncd"_utf16, + "abc123"_utf16, + "a 1 b 2"_utf16, + "The quick brown fox jumps over the lazy dog"_utf16, + "example.com"_utf16, + "hello, world"_utf16, + "a/b/c"_utf16, + "http://a/b"_utf16, + "\"ab\""_utf16, + "$100"_utf16, + "5%"_utf16, + "a+$b"_utf16, + "x%20y"_utf16, + " .23"_utf16, + "1/2"_utf16, + "1)$"_utf16, + "(ab)"_utf16, + ") ("_utf16, + "a(b)c"_utf16, + "a-\"b"_utf16, + "what?yes"_utf16, + "what?\"yes\""_utf16, + "a|b"_utf16, + "-foo"_utf16, + "word-break"_utf16, + "foo-2"_utf16, + "ABCD-1234"_utf16, + "a[b]c"_utf16, + "$(100)"_utf16, + "10/%"_utf16, + "/0"_utf16, + ")%"_utf16, + "x -foo"_utf16, + "\"[0-9]{8,19}\""_utf16, + }; + + auto icu_segmenter = Unicode::Segmenter::create(Unicode::SegmenterGranularity::Line); + + for (auto const& string : test_strings) { + expect_ascii_line_segmentation_matches_icu(string, *icu_segmenter); + } +} + +TEST_CASE(try_create_for_ascii_line) +{ + auto eligible = [](StringView text) { + auto string = Utf16String::from_utf8(text); + return Unicode::Segmenter::try_create_for_ascii_line(string.utf16_view()) != nullptr; + }; + + // Letters, digits, and whitespace are admitted. + EXPECT(eligible(""sv)); + EXPECT(eligible("hello"sv)); + EXPECT(eligible("hello world"sv)); + EXPECT(eligible("123"sv)); + EXPECT(eligible("abc 123\nxyz"sv)); + EXPECT(eligible("a\tb\r\nc"sv)); + + // Printable ASCII punctuation is admitted. + EXPECT(eligible("hello, world"sv)); + EXPECT(eligible("a/b"sv)); + EXPECT(eligible("example.com"sv)); + EXPECT(eligible("word-break"sv)); + EXPECT(eligible("$100"sv)); + EXPECT(eligible("\"quoted\""sv)); + EXPECT(eligible("1/2"sv)); + EXPECT(eligible(" .23"sv)); + EXPECT(eligible("-#tag"sv)); + EXPECT(eligible("(ab)"sv)); + EXPECT(eligible("[ab]"sv)); + EXPECT(eligible("{ab}"sv)); + + // Unsupported controls and non-ASCII are rejected so that ICU handles them. + EXPECT(!eligible("\x1b"sv)); + EXPECT(!eligible("你好"sv)); +} + +TEST_CASE(ascii_line_segmenter_clone_independence) +{ + auto utf16_a = "ab cd"_utf16; + auto segmenter_a = Unicode::Segmenter::try_create_for_ascii_line(utf16_a.utf16_view()); + VERIFY(segmenter_a); + + auto segmenter_b = segmenter_a->clone(); + segmenter_b->set_segmented_text("xy"_string); + + EXPECT_EQ(segmenter_a->next_boundary(0).value_or(0u), 3u); + EXPECT_EQ(segmenter_b->next_boundary(0).value_or(0u), 2u); +} + +TEST_CASE(ascii_line_segmenter_next_boundary_inclusive) +{ + auto utf16 = "ab cd"_utf16; + auto segmenter = Unicode::Segmenter::try_create_for_ascii_line(utf16.utf16_view()); + VERIFY(segmenter); + + // Inclusive::Yes returns the index itself when it is a boundary. + EXPECT_EQ(segmenter->next_boundary(0, Unicode::Segmenter::Inclusive::Yes).value_or(0u), 0u); + EXPECT_EQ(segmenter->next_boundary(3, Unicode::Segmenter::Inclusive::Yes).value_or(0u), 3u); + EXPECT_EQ(segmenter->next_boundary(5, Unicode::Segmenter::Inclusive::Yes).value_or(0u), 5u); + + // Inclusive::Yes at a non-boundary returns the next boundary. + EXPECT_EQ(segmenter->next_boundary(1, Unicode::Segmenter::Inclusive::Yes).value_or(0u), 3u); + EXPECT_EQ(segmenter->next_boundary(2, Unicode::Segmenter::Inclusive::Yes).value_or(0u), 3u); + EXPECT_EQ(segmenter->next_boundary(4, Unicode::Segmenter::Inclusive::Yes).value_or(0u), 5u); + + // Past the end returns empty. + EXPECT(!segmenter->next_boundary(5).has_value()); + EXPECT(!segmenter->next_boundary(10).has_value()); + + // Previous boundary clamps from past the end, matching the ICU-backed segmenter. + EXPECT_EQ(segmenter->previous_boundary(10).value_or(0u), 5u); +} + TEST_CASE(out_of_bounds) { {