LibUnicode: Add an ASCII fast path for line break segmentation
Add a Segmenter implementation that implements the UAX#14 line breaking rules applicable to ASCII text. This avoids the need to build an ICU BreakIterator for the majority of text on the web.
This commit is contained in:
Tim Ledbetter
Jelle Raaijmakers
parent
2f39b2dd63
commit
f161215f53
3 changed files with 640 additions and 0 deletions
|
|
@ -4,6 +4,9 @@
|
|||
* SPDX-License-Identifier: BSD-2-Clause
|
||||
*/
|
||||
|
||||
#include <AK/CharacterTypes.h>
|
||||
#include <AK/GenericShorthands.h>
|
||||
#include <AK/OwnPtr.h>
|
||||
#include <AK/Utf16View.h>
|
||||
#include <AK/Utf32View.h>
|
||||
#include <LibUnicode/CharacterTypes.h>
|
||||
|
|
@ -131,6 +134,404 @@ private:
|
|||
size_t m_current { 0 };
|
||||
};
|
||||
|
||||
static bool can_use_ascii_line_breaking_fast_path(ReadonlyBytes bytes)
|
||||
{
|
||||
return all_of(bytes, [](u8 byte) { return is_ascii_printable(byte) || is_ascii_space(byte); });
|
||||
}
|
||||
|
||||
// UAX#14 line-break classes that occur in printable ASCII plus ASCII whitespace.
|
||||
// https://www.unicode.org/reports/tr14/#Table1
|
||||
enum class AsciiLineBreakClass : u8 {
|
||||
AL, // Alphabetic (alphabets and regular symbols)
|
||||
BA, // Break After (TAB, '|')
|
||||
BK, // Mandatory Break (VT, FF)
|
||||
CL, // Close Punctuation ('}')
|
||||
CP, // Close Parenthesis (')', ']')
|
||||
CR, // Carriage Return
|
||||
EX, // Exclamation/Interrogation ('!', '?')
|
||||
HY, // Hyphen ('-')
|
||||
IS, // Infix Numeric Separator (',', '.', ':', ';')
|
||||
LF, // Line Feed
|
||||
NU, // Numeric (digits)
|
||||
OP, // Open Punctuation ('(', '[', '{')
|
||||
PO, // Postfix Numeric ('%')
|
||||
PR, // Prefix Numeric ('$', '+', '\\')
|
||||
QU, // Quotation ('"', '\'')
|
||||
SP, // Space
|
||||
SY, // Symbols Allowing Break After ('/')
|
||||
};
|
||||
|
||||
static constexpr AsciiLineBreakClass classify_ascii_byte(u8 byte)
|
||||
{
|
||||
using enum AsciiLineBreakClass;
|
||||
switch (byte) {
|
||||
case '\t':
|
||||
case '|':
|
||||
return BA;
|
||||
case '\n':
|
||||
return LF;
|
||||
case '\v':
|
||||
case '\f':
|
||||
return BK;
|
||||
case '\r':
|
||||
return CR;
|
||||
case ' ':
|
||||
return SP;
|
||||
case '!':
|
||||
case '?':
|
||||
return EX;
|
||||
case '"':
|
||||
case '\'':
|
||||
return QU;
|
||||
case '$':
|
||||
case '+':
|
||||
case '\\':
|
||||
return PR;
|
||||
case '%':
|
||||
return PO;
|
||||
case '(':
|
||||
case '[':
|
||||
case '{':
|
||||
return OP;
|
||||
case ')':
|
||||
case ']':
|
||||
return CP;
|
||||
case ',':
|
||||
case '.':
|
||||
case ':':
|
||||
case ';':
|
||||
return IS;
|
||||
case '-':
|
||||
return HY;
|
||||
case '/':
|
||||
return SY;
|
||||
case '}':
|
||||
return CL;
|
||||
default:
|
||||
return is_ascii_digit(byte) ? NU : AL;
|
||||
}
|
||||
}
|
||||
|
||||
static void mark_ascii_numeric_expression_interiors(ReadonlyBytes text, Vector<bool>& interior_of_numeric_expression)
|
||||
{
|
||||
// Identify spans matching UAX#14 LB25's numeric-expression grammar:
|
||||
// (PR | PO)? (OP | HY)? IS? NU (NU | SY | IS)* (CL | CP)? (PR | PO)?
|
||||
// Positions strictly inside such a span are kept atomic by LB25 — positions at the very start or end of a span are
|
||||
// governed by surrounding rules so adjacent numeric expressions can still break apart.
|
||||
|
||||
using enum AsciiLineBreakClass;
|
||||
interior_of_numeric_expression.clear_with_capacity();
|
||||
interior_of_numeric_expression.resize(text.size() + 1);
|
||||
|
||||
size_t i = 0;
|
||||
while (i < text.size()) {
|
||||
if (classify_ascii_byte(text[i]) != NU) {
|
||||
++i;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Walk the prefix backwards: NU is preceded by an optional IS, then optional OP/HY, then optional PR/PO.
|
||||
size_t start = i;
|
||||
if (start > 0 && classify_ascii_byte(text[start - 1]) == IS)
|
||||
--start;
|
||||
if (start > 0 && first_is_one_of(classify_ascii_byte(text[start - 1]), OP, HY))
|
||||
--start;
|
||||
if (start > 0 && first_is_one_of(classify_ascii_byte(text[start - 1]), PR, PO))
|
||||
--start;
|
||||
|
||||
// Walk the body and suffix forward: NU (NU | SY | IS)* (CL | CP)? (PR | PO)?
|
||||
size_t end = i + 1;
|
||||
while (end < text.size() && first_is_one_of(classify_ascii_byte(text[end]), NU, SY, IS))
|
||||
++end;
|
||||
if (end < text.size() && first_is_one_of(classify_ascii_byte(text[end]), CL, CP))
|
||||
++end;
|
||||
if (end < text.size() && first_is_one_of(classify_ascii_byte(text[end]), PR, PO))
|
||||
++end;
|
||||
|
||||
// Mark positions strictly between `start` and `end` as interior. Position `start` and position `end`
|
||||
// are left alone so adjacent expressions can still break against each other.
|
||||
for (size_t position = start + 1; position < end; ++position)
|
||||
interior_of_numeric_expression[position] = true;
|
||||
|
||||
i = end;
|
||||
}
|
||||
}
|
||||
|
||||
static void compute_ascii_line_boundaries(ReadonlyBytes text, Vector<bool>& is_boundary)
|
||||
{
|
||||
// Compute soft and mandatory line-break opportunities for ASCII text following UAX#14: https://www.unicode.org/reports/tr14/)
|
||||
// Only the rules whose left- and right-hand classes can occur in printable ASCII plus ASCII whitespace are
|
||||
// implemented.
|
||||
using enum AsciiLineBreakClass;
|
||||
|
||||
is_boundary.clear_with_capacity();
|
||||
is_boundary.resize(text.size() + 1);
|
||||
is_boundary[0] = true;
|
||||
if (text.is_empty())
|
||||
return;
|
||||
is_boundary[text.size()] = true;
|
||||
|
||||
// LB25: precompute positions interior to numeric expressions.
|
||||
Vector<bool> interior_of_numeric_expression;
|
||||
mark_ascii_numeric_expression_interiors(text, interior_of_numeric_expression);
|
||||
|
||||
// LB14 state: have we just seen `OP SP*` (with no intervening non-SP)?
|
||||
bool in_op_sp_run = false;
|
||||
|
||||
for (size_t i = 1; i < text.size(); ++i) {
|
||||
auto previous_class = classify_ascii_byte(text[i - 1]);
|
||||
auto current_class = classify_ascii_byte(text[i]);
|
||||
|
||||
// LB5: CR × LF; CR ! ; LF ! .
|
||||
if (previous_class == CR && current_class == LF) {
|
||||
is_boundary[i] = false;
|
||||
continue;
|
||||
}
|
||||
// LB4: BK ! . LB5 (continued): CR ! and LF ! .
|
||||
if (first_is_one_of(previous_class, BK, CR, LF)) {
|
||||
is_boundary[i] = true;
|
||||
in_op_sp_run = false;
|
||||
continue;
|
||||
}
|
||||
// LB6: × ( BK | CR | LF ).
|
||||
if (first_is_one_of(current_class, BK, CR, LF)) {
|
||||
is_boundary[i] = false;
|
||||
continue;
|
||||
}
|
||||
|
||||
// LB14 trigger: `OP` starts an `OP SP*` run that suppresses breaks until the next non-SP.
|
||||
if (previous_class == OP)
|
||||
in_op_sp_run = true;
|
||||
|
||||
// LB7: × SP. The OP-SP* run is preserved across spaces.
|
||||
if (current_class == SP) {
|
||||
is_boundary[i] = false;
|
||||
continue;
|
||||
}
|
||||
|
||||
// LB14: OP SP* × (no break after OP, even across intervening spaces).
|
||||
if (in_op_sp_run) {
|
||||
is_boundary[i] = false;
|
||||
in_op_sp_run = false;
|
||||
continue;
|
||||
}
|
||||
|
||||
// LB13: × CL × CP × EX × SY (IS is handled by LB15c/d below).
|
||||
if (first_is_one_of(current_class, CL, CP, EX, SY)) {
|
||||
is_boundary[i] = false;
|
||||
continue;
|
||||
}
|
||||
|
||||
// LB15d: do not break before IS unless preceded by SP, and even then only when SP IS is followedby NU.
|
||||
if (current_class == IS) {
|
||||
if (previous_class != SP) {
|
||||
is_boundary[i] = false;
|
||||
continue;
|
||||
}
|
||||
auto next_class = i + 1 < text.size() ? classify_ascii_byte(text[i + 1]) : AL;
|
||||
bool next_forces_break = i + 1 < text.size() && next_class == NU;
|
||||
if (!next_forces_break) {
|
||||
is_boundary[i] = false;
|
||||
continue;
|
||||
}
|
||||
// Fall through: SP × IS NU → break before IS (per LB15c "leading decimal point").
|
||||
is_boundary[i] = true;
|
||||
continue;
|
||||
}
|
||||
|
||||
// LB18: SP ÷ (default break opportunity after spaces).
|
||||
if (previous_class == SP) {
|
||||
is_boundary[i] = true;
|
||||
continue;
|
||||
}
|
||||
|
||||
// LB19: × QU; QU × (treat ASCII straight quotes as ambiguous QU).
|
||||
if (current_class == QU || previous_class == QU) {
|
||||
is_boundary[i] = false;
|
||||
continue;
|
||||
}
|
||||
|
||||
// LB20a: do not break between a hyphen and a following letter when the hyphen comes at the start of a line -
|
||||
// that is, at the start of the text, after a space, or after a forced break.
|
||||
if (previous_class == HY && current_class == AL) {
|
||||
bool hy_starts_line = i == 1;
|
||||
if (!hy_starts_line && i >= 2)
|
||||
hy_starts_line = first_is_one_of(classify_ascii_byte(text[i - 2]), SP, BK, CR, LF);
|
||||
if (hy_starts_line) {
|
||||
is_boundary[i] = false;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
// LB21: × BA; × HY (break-before suppression for BA and HY).
|
||||
if (first_is_one_of(current_class, BA, HY)) {
|
||||
is_boundary[i] = false;
|
||||
continue;
|
||||
}
|
||||
|
||||
// LB23: AL × NU; NU × AL.
|
||||
if ((previous_class == AL && current_class == NU) || (previous_class == NU && current_class == AL)) {
|
||||
is_boundary[i] = false;
|
||||
continue;
|
||||
}
|
||||
|
||||
// LB24: (PR | PO) × AL; AL × (PR | PO).
|
||||
if ((first_is_one_of(previous_class, PR, PO) && current_class == AL)
|
||||
|| (previous_class == AL && first_is_one_of(current_class, PR, PO))) {
|
||||
is_boundary[i] = false;
|
||||
continue;
|
||||
}
|
||||
|
||||
// LB25: positions strictly inside a numeric expression are kept atomic.
|
||||
if (interior_of_numeric_expression[i]) {
|
||||
is_boundary[i] = false;
|
||||
continue;
|
||||
}
|
||||
|
||||
// LB28: AL × AL.
|
||||
if (previous_class == AL && current_class == AL) {
|
||||
is_boundary[i] = false;
|
||||
continue;
|
||||
}
|
||||
|
||||
// LB29: IS × AL.
|
||||
if (previous_class == IS && current_class == AL) {
|
||||
is_boundary[i] = false;
|
||||
continue;
|
||||
}
|
||||
|
||||
// LB30: (AL | NU) × OP; CP × (AL | NU). All ASCII OP/CP characters are non-East-Asian.
|
||||
if (current_class == OP && first_is_one_of(previous_class, AL, NU)) {
|
||||
is_boundary[i] = false;
|
||||
continue;
|
||||
}
|
||||
if (previous_class == CP && first_is_one_of(current_class, AL, NU)) {
|
||||
is_boundary[i] = false;
|
||||
continue;
|
||||
}
|
||||
|
||||
// LB31: ÷ (default).
|
||||
is_boundary[i] = true;
|
||||
}
|
||||
}
|
||||
|
||||
// Implements UAX#14 line breaking rules for ASCII text: https://www.unicode.org/reports/tr14/tr14-39.html
|
||||
class AsciiLineSegmenter : public Segmenter {
|
||||
public:
|
||||
AsciiLineSegmenter()
|
||||
: Segmenter(SegmenterGranularity::Line)
|
||||
{
|
||||
}
|
||||
|
||||
virtual ~AsciiLineSegmenter() override = default;
|
||||
|
||||
virtual NonnullOwnPtr<Segmenter> clone() const override
|
||||
{
|
||||
return make<AsciiLineSegmenter>();
|
||||
}
|
||||
|
||||
virtual void set_segmented_text(String text) override
|
||||
{
|
||||
apply(text.bytes());
|
||||
}
|
||||
|
||||
virtual void set_segmented_text(Utf16View const& text) override
|
||||
{
|
||||
VERIFY(text.has_ascii_storage());
|
||||
auto span = text.ascii_span();
|
||||
apply({ span.data(), span.size() });
|
||||
}
|
||||
|
||||
virtual size_t current_boundary() override
|
||||
{
|
||||
return m_current;
|
||||
}
|
||||
|
||||
virtual Optional<size_t> previous_boundary(size_t index, Inclusive inclusive) override
|
||||
{
|
||||
if (inclusive == Inclusive::Yes && index <= text_size() && is_boundary(index))
|
||||
return index;
|
||||
if (index == 0)
|
||||
return {};
|
||||
size_t i = min(index, text_size() + 1);
|
||||
while (i > 0) {
|
||||
--i;
|
||||
if (is_boundary(i))
|
||||
return i;
|
||||
}
|
||||
return {};
|
||||
}
|
||||
|
||||
virtual Optional<size_t> next_boundary(size_t index, Inclusive inclusive) override
|
||||
{
|
||||
if (inclusive == Inclusive::Yes && index <= text_size() && is_boundary(index))
|
||||
return index;
|
||||
if (index >= text_size())
|
||||
return {};
|
||||
for (size_t i = index + 1; i <= text_size(); ++i) {
|
||||
if (is_boundary(i))
|
||||
return i;
|
||||
}
|
||||
return {};
|
||||
}
|
||||
|
||||
virtual void for_each_boundary(String text, SegmentationCallback callback) override
|
||||
{
|
||||
if (text.is_empty())
|
||||
return;
|
||||
set_segmented_text(move(text));
|
||||
iterate(callback);
|
||||
}
|
||||
|
||||
virtual void for_each_boundary(Utf16View const& text, SegmentationCallback callback) override
|
||||
{
|
||||
if (text.is_empty())
|
||||
return;
|
||||
set_segmented_text(text);
|
||||
iterate(callback);
|
||||
}
|
||||
|
||||
virtual void for_each_boundary(Utf32View const&, SegmentationCallback) override
|
||||
{
|
||||
VERIFY_NOT_REACHED();
|
||||
}
|
||||
|
||||
virtual bool is_current_boundary_word_like() const override
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
private:
|
||||
void apply(ReadonlyBytes bytes)
|
||||
{
|
||||
VERIFY(can_use_ascii_line_breaking_fast_path(bytes));
|
||||
compute_ascii_line_boundaries(bytes, m_is_boundary);
|
||||
m_current = 0;
|
||||
}
|
||||
|
||||
size_t text_size() const { return m_is_boundary.size() - 1; }
|
||||
|
||||
bool is_boundary(size_t index) const
|
||||
{
|
||||
VERIFY(index < m_is_boundary.size());
|
||||
return m_is_boundary[index];
|
||||
}
|
||||
|
||||
void iterate(SegmentationCallback& callback)
|
||||
{
|
||||
for (size_t i = 0; i <= text_size(); ++i) {
|
||||
if (!m_is_boundary[i])
|
||||
continue;
|
||||
m_current = i;
|
||||
if (callback(i) == IterationDecision::Break)
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
Vector<bool> m_is_boundary;
|
||||
size_t m_current { 0 };
|
||||
};
|
||||
|
||||
class SegmenterImpl : public Segmenter {
|
||||
public:
|
||||
SegmenterImpl(NonnullOwnPtr<icu::BreakIterator> segmenter, SegmenterGranularity segmenter_granularity)
|
||||
|
|
@ -344,6 +745,19 @@ NonnullOwnPtr<Segmenter> Segmenter::create_for_ascii_grapheme(size_t length)
|
|||
return make<AsciiGraphemeSegmenter>(length);
|
||||
}
|
||||
|
||||
OwnPtr<Segmenter> Segmenter::try_create_for_ascii_line(Utf16View const& text)
|
||||
{
|
||||
if (!text.has_ascii_storage())
|
||||
return {};
|
||||
auto span = text.ascii_span();
|
||||
ReadonlyBytes bytes { span.data(), span.size() };
|
||||
if (!can_use_ascii_line_breaking_fast_path(bytes))
|
||||
return {};
|
||||
auto segmenter = make<AsciiLineSegmenter>();
|
||||
segmenter->set_segmented_text(text);
|
||||
return segmenter;
|
||||
}
|
||||
|
||||
bool Segmenter::should_continue_beyond_word(Utf16View const& word)
|
||||
{
|
||||
for (auto code_point : word) {
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@
|
|||
#include <AK/Function.h>
|
||||
#include <AK/NonnullOwnPtr.h>
|
||||
#include <AK/Optional.h>
|
||||
#include <AK/OwnPtr.h>
|
||||
#include <AK/StringView.h>
|
||||
|
||||
namespace Unicode {
|
||||
|
|
@ -27,6 +28,7 @@ public:
|
|||
static NonnullOwnPtr<Segmenter> create(SegmenterGranularity segmenter_granularity);
|
||||
static NonnullOwnPtr<Segmenter> create(StringView locale, SegmenterGranularity segmenter_granularity);
|
||||
static NonnullOwnPtr<Segmenter> create_for_ascii_grapheme(size_t length);
|
||||
static OwnPtr<Segmenter> try_create_for_ascii_line(Utf16View const&);
|
||||
virtual ~Segmenter() = default;
|
||||
|
||||
static bool should_continue_beyond_word(Utf16View const&);
|
||||
|
|
|
|||
|
|
@ -175,6 +175,230 @@ TEST_CASE(line_segmentation)
|
|||
test_line_segmentation("ab你好cd"sv, { 0u, 2u, 5u, 8u, 10u });
|
||||
}
|
||||
|
||||
template<size_t N>
|
||||
static void test_ascii_line_segmentation(Utf16String const& string, size_t const (&expected_boundaries)[N])
|
||||
{
|
||||
auto segmenter = Unicode::Segmenter::try_create_for_ascii_line(string.utf16_view());
|
||||
VERIFY(segmenter);
|
||||
|
||||
Vector<size_t> boundaries;
|
||||
segmenter->for_each_boundary(string.utf16_view(), [&](auto boundary) {
|
||||
boundaries.append(boundary);
|
||||
return IterationDecision::Continue;
|
||||
});
|
||||
|
||||
EXPECT_EQ(boundaries, ReadonlySpan<size_t> { expected_boundaries });
|
||||
}
|
||||
|
||||
static void expect_ascii_line_segmentation_matches_icu(Utf16String const& string, Unicode::Segmenter& icu_segmenter)
|
||||
{
|
||||
auto ascii_segmenter = Unicode::Segmenter::try_create_for_ascii_line(string.utf16_view());
|
||||
VERIFY(ascii_segmenter);
|
||||
|
||||
Vector<size_t> icu_boundaries;
|
||||
icu_segmenter.for_each_boundary(string.utf16_view(), [&](auto boundary) {
|
||||
icu_boundaries.append(boundary);
|
||||
return IterationDecision::Continue;
|
||||
});
|
||||
|
||||
Vector<size_t> ascii_boundaries;
|
||||
ascii_segmenter->for_each_boundary(string.utf16_view(), [&](auto boundary) {
|
||||
ascii_boundaries.append(boundary);
|
||||
return IterationDecision::Continue;
|
||||
});
|
||||
|
||||
EXPECT_EQ(ascii_boundaries, icu_boundaries);
|
||||
}
|
||||
|
||||
TEST_CASE(ascii_line_segmentation)
|
||||
{
|
||||
auto empty_segmenter = Unicode::Segmenter::try_create_for_ascii_line({});
|
||||
VERIFY(empty_segmenter);
|
||||
empty_segmenter->for_each_boundary(String {}, [&](auto) {
|
||||
VERIFY_NOT_REACHED();
|
||||
return IterationDecision::Break;
|
||||
});
|
||||
|
||||
// Single characters and atomic words.
|
||||
test_ascii_line_segmentation("a"_utf16, { 0u, 1u });
|
||||
test_ascii_line_segmentation("abc"_utf16, { 0u, 3u });
|
||||
|
||||
// Break opportunity after whitespace.
|
||||
test_ascii_line_segmentation("ab cd"_utf16, { 0u, 3u, 5u });
|
||||
test_ascii_line_segmentation("ab cd"_utf16, { 0u, 4u, 6u });
|
||||
test_ascii_line_segmentation("ab\tcd"_utf16, { 0u, 3u, 5u });
|
||||
|
||||
// Hard line breaks.
|
||||
test_ascii_line_segmentation("ab\ncd"_utf16, { 0u, 3u, 5u });
|
||||
test_ascii_line_segmentation("ab\r\ncd"_utf16, { 0u, 4u, 6u });
|
||||
test_ascii_line_segmentation("ab\rcd"_utf16, { 0u, 3u, 5u });
|
||||
|
||||
// Alphanumerics are atomic across letter/digit boundaries.
|
||||
test_ascii_line_segmentation("abc123"_utf16, { 0u, 6u });
|
||||
test_ascii_line_segmentation("123abc"_utf16, { 0u, 6u });
|
||||
test_ascii_line_segmentation("a 1 b"_utf16, { 0u, 2u, 4u, 5u });
|
||||
|
||||
// Printable ASCII punctuation follows UAX#14 line-breaking rules.
|
||||
test_ascii_line_segmentation("example.com"_utf16, { 0u, 11u }); // LB15d (× IS) and LB29 (IS × AL) keep the dotted name atomic.
|
||||
test_ascii_line_segmentation("hello, world"_utf16, { 0u, 7u, 12u }); // Break after the space (LB18).
|
||||
test_ascii_line_segmentation("a/b/c"_utf16, { 0u, 2u, 4u, 5u }); // LB13 × SY, default ÷ after SY.
|
||||
test_ascii_line_segmentation("http://a/b"_utf16, { 0u, 7u, 9u, 10u });
|
||||
test_ascii_line_segmentation("\"ab\""_utf16, { 0u, 4u }); // LB19 around QU.
|
||||
test_ascii_line_segmentation("$100"_utf16, { 0u, 4u }); // LB25 PR × NU and NU × NU.
|
||||
test_ascii_line_segmentation("5%"_utf16, { 0u, 2u }); // LB25 NU × PO.
|
||||
test_ascii_line_segmentation("a+$b"_utf16, { 0u, 2u, 4u }); // LB24 AL × PR; PR × PR has no rule.
|
||||
test_ascii_line_segmentation("x%20y"_utf16, { 0u, 5u }); // LB24 AL × PO; LB25 PO × NU; LB23 NU × AL.
|
||||
test_ascii_line_segmentation(" .23"_utf16, { 0u, 1u, 4u }); // LB15c forces a break before a leading decimal point.
|
||||
test_ascii_line_segmentation("1/2"_utf16, { 0u, 3u }); // LB25 NU × SY × NU.
|
||||
test_ascii_line_segmentation("1)$"_utf16, { 0u, 3u }); // LB25 NU × CP and CP × PR.
|
||||
test_ascii_line_segmentation("(ab)"_utf16, { 0u, 4u }); // LB14 OP × ; LB13 × CP.
|
||||
test_ascii_line_segmentation(") ("_utf16, { 0u, 2u, 3u }); // LB18 break after SP.
|
||||
test_ascii_line_segmentation("a(b)c"_utf16, { 0u, 5u }); // LB30 AL × OP and CP × AL.
|
||||
test_ascii_line_segmentation("a-\"b"_utf16, { 0u, 4u }); // LB21 × HY and LB19 around QU.
|
||||
test_ascii_line_segmentation("what?yes"_utf16, { 0u, 5u, 8u }); // LB13 × EX, default ÷ after EX.
|
||||
test_ascii_line_segmentation("what?\"yes\""_utf16, { 0u, 10u }); // LB19 keeps the quoted segment atomic.
|
||||
|
||||
// U+007C VERTICAL LINE is UAX#14 class BA: no break before, break after.
|
||||
test_ascii_line_segmentation("a|b"_utf16, { 0u, 2u, 3u });
|
||||
test_ascii_line_segmentation("aaa|bbb"_utf16, { 0u, 4u, 7u });
|
||||
test_ascii_line_segmentation("a||b"_utf16, { 0u, 3u, 4u });
|
||||
|
||||
// Hyphen behavior follows UAX#14: × HY before, ÷ after when not in a numeric expression.
|
||||
// LB20a additionally glues a leading hyphen to the following letter.
|
||||
test_ascii_line_segmentation("-foo"_utf16, { 0u, 4u });
|
||||
test_ascii_line_segmentation("x -foo"_utf16, { 0u, 2u, 6u });
|
||||
test_ascii_line_segmentation("word-break"_utf16, { 0u, 5u, 10u });
|
||||
// LB25 keeps numeric expressions atomic across hyphens (HY × NU).
|
||||
test_ascii_line_segmentation("-2"_utf16, { 0u, 2u });
|
||||
test_ascii_line_segmentation("foo-2"_utf16, { 0u, 5u });
|
||||
test_ascii_line_segmentation("ABCD-1234"_utf16, { 0u, 9u });
|
||||
test_ascii_line_segmentation("-#tag"_utf16, { 0u, 5u });
|
||||
}
|
||||
|
||||
TEST_CASE(ascii_line_segmentation_matches_icu_for_context_independent_text)
|
||||
{
|
||||
// The fast path implements UAX#14 for ASCII inputs and should agree with ICU on every input it accepts.
|
||||
Array test_strings = {
|
||||
"a"_utf16,
|
||||
"abc"_utf16,
|
||||
"ab cd"_utf16,
|
||||
"ab cd"_utf16,
|
||||
"ab\tcd"_utf16,
|
||||
"ab\ncd"_utf16,
|
||||
"ab\r\ncd"_utf16,
|
||||
"abc123"_utf16,
|
||||
"a 1 b 2"_utf16,
|
||||
"The quick brown fox jumps over the lazy dog"_utf16,
|
||||
"example.com"_utf16,
|
||||
"hello, world"_utf16,
|
||||
"a/b/c"_utf16,
|
||||
"http://a/b"_utf16,
|
||||
"\"ab\""_utf16,
|
||||
"$100"_utf16,
|
||||
"5%"_utf16,
|
||||
"a+$b"_utf16,
|
||||
"x%20y"_utf16,
|
||||
" .23"_utf16,
|
||||
"1/2"_utf16,
|
||||
"1)$"_utf16,
|
||||
"(ab)"_utf16,
|
||||
") ("_utf16,
|
||||
"a(b)c"_utf16,
|
||||
"a-\"b"_utf16,
|
||||
"what?yes"_utf16,
|
||||
"what?\"yes\""_utf16,
|
||||
"a|b"_utf16,
|
||||
"-foo"_utf16,
|
||||
"word-break"_utf16,
|
||||
"foo-2"_utf16,
|
||||
"ABCD-1234"_utf16,
|
||||
"a[b]c"_utf16,
|
||||
"$(100)"_utf16,
|
||||
"10/%"_utf16,
|
||||
"/0"_utf16,
|
||||
")%"_utf16,
|
||||
"x -foo"_utf16,
|
||||
"\"[0-9]{8,19}\""_utf16,
|
||||
};
|
||||
|
||||
auto icu_segmenter = Unicode::Segmenter::create(Unicode::SegmenterGranularity::Line);
|
||||
|
||||
for (auto const& string : test_strings) {
|
||||
expect_ascii_line_segmentation_matches_icu(string, *icu_segmenter);
|
||||
}
|
||||
}
|
||||
|
||||
TEST_CASE(try_create_for_ascii_line)
|
||||
{
|
||||
auto eligible = [](StringView text) {
|
||||
auto string = Utf16String::from_utf8(text);
|
||||
return Unicode::Segmenter::try_create_for_ascii_line(string.utf16_view()) != nullptr;
|
||||
};
|
||||
|
||||
// Letters, digits, and whitespace are admitted.
|
||||
EXPECT(eligible(""sv));
|
||||
EXPECT(eligible("hello"sv));
|
||||
EXPECT(eligible("hello world"sv));
|
||||
EXPECT(eligible("123"sv));
|
||||
EXPECT(eligible("abc 123\nxyz"sv));
|
||||
EXPECT(eligible("a\tb\r\nc"sv));
|
||||
|
||||
// Printable ASCII punctuation is admitted.
|
||||
EXPECT(eligible("hello, world"sv));
|
||||
EXPECT(eligible("a/b"sv));
|
||||
EXPECT(eligible("example.com"sv));
|
||||
EXPECT(eligible("word-break"sv));
|
||||
EXPECT(eligible("$100"sv));
|
||||
EXPECT(eligible("\"quoted\""sv));
|
||||
EXPECT(eligible("1/2"sv));
|
||||
EXPECT(eligible(" .23"sv));
|
||||
EXPECT(eligible("-#tag"sv));
|
||||
EXPECT(eligible("(ab)"sv));
|
||||
EXPECT(eligible("[ab]"sv));
|
||||
EXPECT(eligible("{ab}"sv));
|
||||
|
||||
// Unsupported controls and non-ASCII are rejected so that ICU handles them.
|
||||
EXPECT(!eligible("\x1b"sv));
|
||||
EXPECT(!eligible("你好"sv));
|
||||
}
|
||||
|
||||
TEST_CASE(ascii_line_segmenter_clone_independence)
|
||||
{
|
||||
auto utf16_a = "ab cd"_utf16;
|
||||
auto segmenter_a = Unicode::Segmenter::try_create_for_ascii_line(utf16_a.utf16_view());
|
||||
VERIFY(segmenter_a);
|
||||
|
||||
auto segmenter_b = segmenter_a->clone();
|
||||
segmenter_b->set_segmented_text("xy"_string);
|
||||
|
||||
EXPECT_EQ(segmenter_a->next_boundary(0).value_or(0u), 3u);
|
||||
EXPECT_EQ(segmenter_b->next_boundary(0).value_or(0u), 2u);
|
||||
}
|
||||
|
||||
TEST_CASE(ascii_line_segmenter_next_boundary_inclusive)
|
||||
{
|
||||
auto utf16 = "ab cd"_utf16;
|
||||
auto segmenter = Unicode::Segmenter::try_create_for_ascii_line(utf16.utf16_view());
|
||||
VERIFY(segmenter);
|
||||
|
||||
// Inclusive::Yes returns the index itself when it is a boundary.
|
||||
EXPECT_EQ(segmenter->next_boundary(0, Unicode::Segmenter::Inclusive::Yes).value_or(0u), 0u);
|
||||
EXPECT_EQ(segmenter->next_boundary(3, Unicode::Segmenter::Inclusive::Yes).value_or(0u), 3u);
|
||||
EXPECT_EQ(segmenter->next_boundary(5, Unicode::Segmenter::Inclusive::Yes).value_or(0u), 5u);
|
||||
|
||||
// Inclusive::Yes at a non-boundary returns the next boundary.
|
||||
EXPECT_EQ(segmenter->next_boundary(1, Unicode::Segmenter::Inclusive::Yes).value_or(0u), 3u);
|
||||
EXPECT_EQ(segmenter->next_boundary(2, Unicode::Segmenter::Inclusive::Yes).value_or(0u), 3u);
|
||||
EXPECT_EQ(segmenter->next_boundary(4, Unicode::Segmenter::Inclusive::Yes).value_or(0u), 5u);
|
||||
|
||||
// Past the end returns empty.
|
||||
EXPECT(!segmenter->next_boundary(5).has_value());
|
||||
EXPECT(!segmenter->next_boundary(10).has_value());
|
||||
|
||||
// Previous boundary clamps from past the end, matching the ICU-backed segmenter.
|
||||
EXPECT_EQ(segmenter->previous_boundary(10).value_or(0u), 5u);
|
||||
}
|
||||
|
||||
TEST_CASE(out_of_bounds)
|
||||
{
|
||||
{
|
||||
|
|
|
|||
Loading…
Reference in a new issue