LibUnicode: Add Unicode case folding support for regex matching
This commit is contained in:
aplefull
Tim Flynn
parent
2085bdc361
commit
e1682424aa
3 changed files with 376 additions and 25 deletions
|
|
@ -7,6 +7,8 @@
|
|||
#include <AK/Array.h>
|
||||
#include <AK/CharacterTypes.h>
|
||||
#include <AK/Find.h>
|
||||
#include <AK/HashMap.h>
|
||||
#include <AK/NonnullOwnPtr.h>
|
||||
#include <AK/Traits.h>
|
||||
#include <LibUnicode/CharacterTypes.h>
|
||||
#include <LibUnicode/ICU.h>
|
||||
|
|
@ -51,6 +53,9 @@ static constexpr GeneralCategory GENERAL_CATEGORY_SEPARATOR = U_CHAR_CATEGORY_CO
|
|||
static constexpr GeneralCategory GENERAL_CATEGORY_OTHER = U_CHAR_CATEGORY_COUNT + 8;
|
||||
static constexpr GeneralCategory GENERAL_CATEGORY_LIMIT = U_CHAR_CATEGORY_COUNT + 9;
|
||||
|
||||
static HashMap<GeneralCategory, NonnullOwnPtr<icu::UnicodeSet>> s_category_sets_with_case_closure;
|
||||
static HashMap<Property, NonnullOwnPtr<icu::UnicodeSet>> s_property_sets_with_case_closure;
|
||||
|
||||
Optional<GeneralCategory> general_category_from_string(StringView general_category)
|
||||
{
|
||||
static auto general_category_names = []() {
|
||||
|
|
@ -85,29 +90,49 @@ Optional<GeneralCategory> general_category_from_string(StringView general_catego
|
|||
return {};
|
||||
}
|
||||
|
||||
bool code_point_has_general_category(u32 code_point, GeneralCategory general_category)
|
||||
static uint32_t get_icu_mask(GeneralCategory general_category)
|
||||
{
|
||||
if (general_category == GENERAL_CATEGORY_CASED_LETTER)
|
||||
return U_GC_LC_MASK;
|
||||
if (general_category == GENERAL_CATEGORY_LETTER)
|
||||
return U_GC_L_MASK;
|
||||
if (general_category == GENERAL_CATEGORY_MARK)
|
||||
return U_GC_M_MASK;
|
||||
if (general_category == GENERAL_CATEGORY_NUMBER)
|
||||
return U_GC_N_MASK;
|
||||
if (general_category == GENERAL_CATEGORY_PUNCTUATION)
|
||||
return U_GC_P_MASK;
|
||||
if (general_category == GENERAL_CATEGORY_SYMBOL)
|
||||
return U_GC_S_MASK;
|
||||
if (general_category == GENERAL_CATEGORY_SEPARATOR)
|
||||
return U_GC_Z_MASK;
|
||||
if (general_category == GENERAL_CATEGORY_OTHER)
|
||||
return U_GC_C_MASK;
|
||||
|
||||
return U_MASK(static_cast<UCharCategory>(general_category.value()));
|
||||
}
|
||||
|
||||
bool code_point_has_general_category(u32 code_point, GeneralCategory general_category, CaseSensitivity case_sensitivity)
|
||||
{
|
||||
auto icu_code_point = static_cast<UChar32>(code_point);
|
||||
auto icu_general_category = static_cast<UCharCategory>(general_category.value());
|
||||
auto category_mask = get_icu_mask(general_category);
|
||||
|
||||
if (general_category == GENERAL_CATEGORY_CASED_LETTER)
|
||||
return (U_GET_GC_MASK(icu_code_point) & U_GC_LC_MASK) != 0;
|
||||
if (general_category == GENERAL_CATEGORY_LETTER)
|
||||
return (U_GET_GC_MASK(icu_code_point) & U_GC_L_MASK) != 0;
|
||||
if (general_category == GENERAL_CATEGORY_MARK)
|
||||
return (U_GET_GC_MASK(icu_code_point) & U_GC_M_MASK) != 0;
|
||||
if (general_category == GENERAL_CATEGORY_NUMBER)
|
||||
return (U_GET_GC_MASK(icu_code_point) & U_GC_N_MASK) != 0;
|
||||
if (general_category == GENERAL_CATEGORY_PUNCTUATION)
|
||||
return (U_GET_GC_MASK(icu_code_point) & U_GC_P_MASK) != 0;
|
||||
if (general_category == GENERAL_CATEGORY_SYMBOL)
|
||||
return (U_GET_GC_MASK(icu_code_point) & U_GC_S_MASK) != 0;
|
||||
if (general_category == GENERAL_CATEGORY_SEPARATOR)
|
||||
return (U_GET_GC_MASK(icu_code_point) & U_GC_Z_MASK) != 0;
|
||||
if (general_category == GENERAL_CATEGORY_OTHER)
|
||||
return (U_GET_GC_MASK(icu_code_point) & U_GC_C_MASK) != 0;
|
||||
if ((U_GET_GC_MASK(icu_code_point) & category_mask) != 0)
|
||||
return true;
|
||||
|
||||
return u_charType(icu_code_point) == icu_general_category;
|
||||
if (case_sensitivity == CaseSensitivity::CaseSensitive)
|
||||
return false;
|
||||
|
||||
auto& set = s_category_sets_with_case_closure.ensure(general_category, [&] {
|
||||
UErrorCode status = U_ZERO_ERROR;
|
||||
auto new_set = make<icu::UnicodeSet>();
|
||||
new_set->applyIntPropertyValue(UCHAR_GENERAL_CATEGORY_MASK, static_cast<int32_t>(category_mask), status);
|
||||
new_set->closeOver(USET_CASE_INSENSITIVE);
|
||||
new_set->freeze();
|
||||
return new_set;
|
||||
});
|
||||
|
||||
return set->contains(icu_code_point);
|
||||
}
|
||||
|
||||
bool code_point_is_printable(u32 code_point)
|
||||
|
|
@ -188,19 +213,34 @@ Optional<Property> property_from_string(StringView property)
|
|||
return {};
|
||||
}
|
||||
|
||||
bool code_point_has_property(u32 code_point, Property property)
|
||||
bool code_point_has_property(u32 code_point, Property property, CaseSensitivity case_sensitivity)
|
||||
{
|
||||
auto icu_code_point = static_cast<UChar32>(code_point);
|
||||
auto icu_property = static_cast<UProperty>(property.value());
|
||||
|
||||
if (property == PROPERTY_ANY)
|
||||
return is_unicode(code_point);
|
||||
if (property == PROPERTY_ASCII)
|
||||
return is_ascii(code_point);
|
||||
if (property == PROPERTY_ASSIGNED)
|
||||
return u_isdefined(icu_code_point);
|
||||
return u_isdefined(icu_code_point) != 0;
|
||||
|
||||
return static_cast<bool>(u_hasBinaryProperty(icu_code_point, icu_property));
|
||||
auto icu_property = static_cast<UProperty>(property.value());
|
||||
if (u_hasBinaryProperty(icu_code_point, icu_property))
|
||||
return true;
|
||||
|
||||
if (case_sensitivity == CaseSensitivity::CaseSensitive)
|
||||
return false;
|
||||
|
||||
auto& set = s_property_sets_with_case_closure.ensure(property, [&] {
|
||||
UErrorCode status = U_ZERO_ERROR;
|
||||
auto new_set = make<icu::UnicodeSet>();
|
||||
new_set->applyIntPropertyValue(icu_property, 1, status);
|
||||
new_set->closeOver(USET_CASE_INSENSITIVE);
|
||||
new_set->freeze();
|
||||
return new_set;
|
||||
});
|
||||
|
||||
return set->contains(icu_code_point);
|
||||
}
|
||||
|
||||
bool code_point_has_emoji_property(u32 code_point)
|
||||
|
|
@ -489,4 +529,105 @@ LineBreakClass line_break_class(u32 code_point)
|
|||
}
|
||||
}
|
||||
|
||||
// 22.2.2.7.3 Canonicalize ( rer, ch ), https://tc39.es/ecma262/#sec-runtime-semantics-canonicalize-ch
|
||||
u32 canonicalize(u32 code_point, bool unicode_mode)
|
||||
{
|
||||
// 1. If HasEitherUnicodeFlag(rer) is true and rer.[[IgnoreCase]] is true, then
|
||||
// a. If the file CaseFolding.txt of the Unicode Character Database provides a simple or common case folding mapping for ch, return the result of applying that mapping to ch.
|
||||
// b. Return ch.
|
||||
if (unicode_mode)
|
||||
return u_foldCase(static_cast<UChar32>(code_point), U_FOLD_CASE_DEFAULT);
|
||||
|
||||
// 2. If rer.[[IgnoreCase]] is false, return ch.
|
||||
// NOTE: This is handled by the caller.
|
||||
|
||||
// 3. Assert: ch is a UTF-16 code unit.
|
||||
// 4. Let cp be the code point whose numeric value is the numeric value of ch.
|
||||
// NOTE: We already have a code point.
|
||||
|
||||
// 5. Let u be toUppercase(« cp »), according to the Unicode Default Case Conversion algorithm.
|
||||
// 6. Let uStr be CodePointsToString(u).
|
||||
auto code_point_string = String::from_code_point(code_point);
|
||||
auto uppercased = code_point_string.to_uppercase();
|
||||
if (uppercased.is_error())
|
||||
return code_point;
|
||||
|
||||
auto code_points = uppercased.value().code_points();
|
||||
|
||||
// 7. If the length of uStr ≠ 1, return ch.
|
||||
if (code_points.length() != 1)
|
||||
return code_point;
|
||||
|
||||
// 8. Let cu be uStr's single code unit element.
|
||||
auto it = code_points.begin();
|
||||
auto uppercased_code_point = *it;
|
||||
|
||||
// 9. If the numeric value of ch ≥ 128 and the numeric value of cu < 128, return ch.
|
||||
if (code_point >= 128 && uppercased_code_point < 128)
|
||||
return code_point;
|
||||
|
||||
// 10. Return cu.
|
||||
return uppercased_code_point;
|
||||
}
|
||||
|
||||
Vector<CodePointRange> expand_range_case_insensitive(u32 from, u32 to)
|
||||
{
|
||||
icu::UnicodeSet set(static_cast<UChar32>(from), static_cast<UChar32>(to));
|
||||
set.closeOver(USET_CASE_INSENSITIVE);
|
||||
|
||||
Vector<CodePointRange> result;
|
||||
auto range_count = set.getRangeCount();
|
||||
result.ensure_capacity(range_count);
|
||||
|
||||
for (int32_t i = 0; i < range_count; ++i)
|
||||
result.unchecked_append({ static_cast<u32>(set.getRangeStart(i)), static_cast<u32>(set.getRangeEnd(i)) });
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
void for_each_case_folded_code_point(u32 code_point, Function<IterationDecision(u32)> callback)
|
||||
{
|
||||
u32 canonical = canonicalize(code_point, true);
|
||||
|
||||
icu::UnicodeSet closure(static_cast<UChar32>(canonical), static_cast<UChar32>(canonical));
|
||||
closure.closeOver(USET_CASE_INSENSITIVE);
|
||||
|
||||
auto range_count = closure.getRangeCount();
|
||||
for (int32_t i = 0; i < range_count; ++i) {
|
||||
auto start = closure.getRangeStart(i);
|
||||
auto end = closure.getRangeEnd(i);
|
||||
for (auto cp = start; cp <= end; ++cp) {
|
||||
if (callback(static_cast<u32>(cp)) == IterationDecision::Break)
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
bool code_point_matches_range_ignoring_case(u32 code_point, u32 from, u32 to, bool unicode_mode)
|
||||
{
|
||||
if (code_point >= from && code_point <= to)
|
||||
return true;
|
||||
|
||||
icu::UnicodeSet candidates(static_cast<UChar32>(code_point), static_cast<UChar32>(code_point));
|
||||
candidates.closeOver(USET_CASE_INSENSITIVE);
|
||||
candidates.retain(static_cast<UChar32>(from), static_cast<UChar32>(to));
|
||||
|
||||
if (candidates.isEmpty())
|
||||
return false;
|
||||
|
||||
auto canonical_ch = canonicalize(code_point, unicode_mode);
|
||||
auto range_count = candidates.getRangeCount();
|
||||
for (auto i = 0; i < range_count; ++i) {
|
||||
auto start = candidates.getRangeStart(i);
|
||||
auto end = candidates.getRangeEnd(i);
|
||||
|
||||
for (auto candidate_cp = start; candidate_cp <= end; ++candidate_cp) {
|
||||
if (canonicalize(candidate_cp, unicode_mode) == canonical_ch)
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,14 +7,17 @@
|
|||
#pragma once
|
||||
|
||||
#include <AK/Forward.h>
|
||||
#include <AK/Function.h>
|
||||
#include <AK/IterationDecision.h>
|
||||
#include <AK/Optional.h>
|
||||
#include <AK/Types.h>
|
||||
#include <AK/Vector.h>
|
||||
#include <LibUnicode/Forward.h>
|
||||
|
||||
namespace Unicode {
|
||||
|
||||
Optional<GeneralCategory> general_category_from_string(StringView);
|
||||
bool code_point_has_general_category(u32 code_point, GeneralCategory general_category);
|
||||
bool code_point_has_general_category(u32 code_point, GeneralCategory general_category, CaseSensitivity case_sensitivity = CaseSensitivity::CaseSensitive);
|
||||
|
||||
bool code_point_is_printable(u32 code_point);
|
||||
bool code_point_has_control_general_category(u32 code_point);
|
||||
|
|
@ -27,7 +30,7 @@ bool code_point_has_space_separator_general_category(u32 code_point);
|
|||
bool code_point_has_symbol_general_category(u32 code_point);
|
||||
|
||||
Optional<Property> property_from_string(StringView);
|
||||
bool code_point_has_property(u32 code_point, Property property);
|
||||
bool code_point_has_property(u32 code_point, Property property, CaseSensitivity case_sensitivity = CaseSensitivity::CaseSensitive);
|
||||
|
||||
bool code_point_has_emoji_property(u32 code_point);
|
||||
bool code_point_has_emoji_modifier_base_property(u32 code_point);
|
||||
|
|
@ -86,4 +89,33 @@ enum class LineBreakClass {
|
|||
|
||||
LineBreakClass line_break_class(u32 code_point);
|
||||
|
||||
struct CodePointRange {
|
||||
u32 from { 0 };
|
||||
u32 to { 0 };
|
||||
};
|
||||
|
||||
u32 canonicalize(u32 code_point, bool unicode_mode);
|
||||
|
||||
bool code_point_matches_range_ignoring_case(u32 code_point, u32 from, u32 to, bool unicode_mode);
|
||||
|
||||
Vector<CodePointRange> expand_range_case_insensitive(u32 from, u32 to);
|
||||
|
||||
void for_each_case_folded_code_point(u32 code_point, Function<IterationDecision(u32)> callback);
|
||||
|
||||
template<typename Range1, typename Range2>
|
||||
bool ranges_equal_ignoring_case(Range1 const& range1, Range2 const& range2, bool unicode_mode)
|
||||
{
|
||||
auto it1 = range1.begin();
|
||||
auto it2 = range2.begin();
|
||||
auto end1 = range1.end();
|
||||
auto end2 = range2.end();
|
||||
|
||||
for (; it1 != end1 && it2 != end2; ++it1, ++it2) {
|
||||
if (canonicalize(*it1, unicode_mode) != canonicalize(*it2, unicode_mode))
|
||||
return false;
|
||||
}
|
||||
|
||||
return it1 == end1 && it2 == end2;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@
|
|||
#include <LibTest/TestCase.h>
|
||||
|
||||
#include <AK/StringView.h>
|
||||
#include <AK/Utf16View.h>
|
||||
#include <LibUnicode/CharacterTypes.h>
|
||||
|
||||
TEST_CASE(general_category)
|
||||
|
|
@ -99,6 +100,26 @@ TEST_CASE(general_category)
|
|||
EXPECT(!Unicode::code_point_has_general_category(code_point, general_category_cn));
|
||||
EXPECT(!Unicode::code_point_has_general_category(code_point, general_category_ll));
|
||||
}
|
||||
|
||||
for (u32 code_point = 0x61; code_point <= 0x7a; ++code_point) {
|
||||
EXPECT(!Unicode::code_point_has_general_category(code_point, general_category_lu, CaseSensitivity::CaseSensitive));
|
||||
EXPECT(Unicode::code_point_has_general_category(code_point, general_category_lu, CaseSensitivity::CaseInsensitive));
|
||||
}
|
||||
|
||||
for (u32 code_point = 0x41; code_point <= 0x5a; ++code_point) {
|
||||
EXPECT(!Unicode::code_point_has_general_category(code_point, general_category_ll, CaseSensitivity::CaseSensitive));
|
||||
EXPECT(Unicode::code_point_has_general_category(code_point, general_category_ll, CaseSensitivity::CaseInsensitive));
|
||||
}
|
||||
|
||||
for (u32 code_point = 0x0410; code_point <= 0x042F; ++code_point) {
|
||||
EXPECT(!Unicode::code_point_has_general_category(code_point, general_category_ll, CaseSensitivity::CaseSensitive));
|
||||
EXPECT(Unicode::code_point_has_general_category(code_point, general_category_ll, CaseSensitivity::CaseInsensitive));
|
||||
}
|
||||
|
||||
for (u32 code_point = 0x0430; code_point <= 0x044F; ++code_point) {
|
||||
EXPECT(!Unicode::code_point_has_general_category(code_point, general_category_lu, CaseSensitivity::CaseSensitive));
|
||||
EXPECT(Unicode::code_point_has_general_category(code_point, general_category_lu, CaseSensitivity::CaseInsensitive));
|
||||
}
|
||||
}
|
||||
|
||||
BENCHMARK_CASE(general_category_performance)
|
||||
|
|
@ -134,6 +155,8 @@ TEST_CASE(property)
|
|||
auto property_any = property("Any"sv);
|
||||
auto property_assigned = property("Assigned"sv);
|
||||
auto property_ascii = property("ASCII"sv);
|
||||
auto property_uppercase = property("Uppercase"sv);
|
||||
auto property_lowercase = property("Lowercase"sv);
|
||||
|
||||
auto property_white_space = property("White_Space"sv);
|
||||
auto property_wspace = property("WSpace"sv);
|
||||
|
|
@ -191,6 +214,26 @@ TEST_CASE(property)
|
|||
EXPECT(!Unicode::code_point_has_property(code_point, property_ascii));
|
||||
EXPECT(!Unicode::code_point_has_property(code_point, property_white_space));
|
||||
}
|
||||
|
||||
for (u32 code_point = 0x61; code_point <= 0x7a; ++code_point) {
|
||||
EXPECT(!Unicode::code_point_has_property(code_point, property_uppercase, CaseSensitivity::CaseSensitive));
|
||||
EXPECT(Unicode::code_point_has_property(code_point, property_uppercase, CaseSensitivity::CaseInsensitive));
|
||||
}
|
||||
|
||||
for (u32 code_point = 0x41; code_point <= 0x5a; ++code_point) {
|
||||
EXPECT(!Unicode::code_point_has_property(code_point, property_lowercase, CaseSensitivity::CaseSensitive));
|
||||
EXPECT(Unicode::code_point_has_property(code_point, property_lowercase, CaseSensitivity::CaseInsensitive));
|
||||
}
|
||||
|
||||
for (u32 code_point = 0x0430; code_point <= 0x044F; ++code_point) {
|
||||
EXPECT(!Unicode::code_point_has_property(code_point, property_uppercase, CaseSensitivity::CaseSensitive));
|
||||
EXPECT(Unicode::code_point_has_property(code_point, property_uppercase, CaseSensitivity::CaseInsensitive));
|
||||
}
|
||||
|
||||
for (u32 code_point = 0x0410; code_point <= 0x042F; ++code_point) {
|
||||
EXPECT(!Unicode::code_point_has_property(code_point, property_lowercase, CaseSensitivity::CaseSensitive));
|
||||
EXPECT(Unicode::code_point_has_property(code_point, property_lowercase, CaseSensitivity::CaseInsensitive));
|
||||
}
|
||||
}
|
||||
|
||||
TEST_CASE(script)
|
||||
|
|
@ -309,3 +352,138 @@ TEST_CASE(code_point_bidirectional_character_type)
|
|||
// Arabic right-to-left (U+FEB4 ARABIC LETTER SEEN MEDIAL FORM)
|
||||
EXPECT_EQ(Unicode::bidirectional_class(0xFEB4), Unicode::BidiClass::RightToLeftArabic);
|
||||
}
|
||||
|
||||
TEST_CASE(canonicalize)
|
||||
{
|
||||
constexpr u32 LATIN_CAPITAL_A_GRAVE = 0x00C0; // À
|
||||
constexpr u32 LATIN_SMALL_A_GRAVE = 0x00E0; // à
|
||||
constexpr u32 LATIN_CAPITAL_SHARP_S = 0x1E9E; // ẞ
|
||||
constexpr u32 LATIN_SMALL_SHARP_S = 0x00DF; // ß
|
||||
constexpr u32 LATIN_CAPITAL_OE = 0x0152; // Œ
|
||||
constexpr u32 LATIN_SMALL_OE = 0x0153; // œ
|
||||
constexpr u32 GREEK_CAPITAL_SIGMA = 0x03A3; // Σ
|
||||
constexpr u32 GREEK_SMALL_SIGMA = 0x03C3; // σ
|
||||
constexpr u32 GREEK_SMALL_FINAL_SIGMA = 0x03C2; // ς
|
||||
constexpr u32 KELVIN_SIGN = 0x212A; // K
|
||||
|
||||
EXPECT_EQ(Unicode::canonicalize('A', true), static_cast<u32>('a'));
|
||||
EXPECT_EQ(Unicode::canonicalize('a', false), static_cast<u32>('A'));
|
||||
|
||||
EXPECT_EQ(Unicode::canonicalize(KELVIN_SIGN, true), static_cast<u32>('k'));
|
||||
EXPECT_EQ(Unicode::canonicalize(KELVIN_SIGN, false), KELVIN_SIGN);
|
||||
|
||||
EXPECT_EQ(Unicode::canonicalize(LATIN_CAPITAL_A_GRAVE, true), LATIN_SMALL_A_GRAVE);
|
||||
EXPECT_EQ(Unicode::canonicalize(LATIN_SMALL_A_GRAVE, false), LATIN_CAPITAL_A_GRAVE);
|
||||
|
||||
EXPECT_EQ(Unicode::canonicalize(LATIN_CAPITAL_SHARP_S, true), LATIN_SMALL_SHARP_S);
|
||||
EXPECT_EQ(Unicode::canonicalize(LATIN_SMALL_SHARP_S, false), LATIN_SMALL_SHARP_S);
|
||||
|
||||
EXPECT_EQ(Unicode::canonicalize(GREEK_CAPITAL_SIGMA, true), GREEK_SMALL_SIGMA);
|
||||
EXPECT_EQ(Unicode::canonicalize(GREEK_SMALL_FINAL_SIGMA, true), GREEK_SMALL_SIGMA);
|
||||
|
||||
EXPECT_EQ(Unicode::canonicalize(LATIN_CAPITAL_OE, true), LATIN_SMALL_OE);
|
||||
EXPECT_EQ(Unicode::canonicalize(LATIN_SMALL_OE, false), LATIN_CAPITAL_OE);
|
||||
}
|
||||
|
||||
TEST_CASE(expand_range_case_insensitive)
|
||||
{
|
||||
auto latin_ranges = Unicode::expand_range_case_insensitive('a', 'z');
|
||||
EXPECT_EQ(latin_ranges.size(), 4uz);
|
||||
|
||||
EXPECT(any_of(latin_ranges, [](auto const& range) {
|
||||
return range.from == 'a' && range.to == 'z';
|
||||
}));
|
||||
|
||||
EXPECT(any_of(latin_ranges, [](auto const& range) {
|
||||
return range.from == 'A' && range.to == 'Z';
|
||||
}));
|
||||
|
||||
// LATIN SMALL LETTER LONG S (ſ)
|
||||
EXPECT(any_of(latin_ranges, [](auto const& range) {
|
||||
return range.from == 0x017F && range.to == 0x017F;
|
||||
}));
|
||||
|
||||
// KELVIN SIGN (K)
|
||||
EXPECT(any_of(latin_ranges, [](auto const& range) {
|
||||
return range.from == 0x212A && range.to == 0x212A;
|
||||
}));
|
||||
|
||||
auto k_ranges = Unicode::expand_range_case_insensitive('k', 'k');
|
||||
EXPECT_EQ(k_ranges.size(), 3uz);
|
||||
|
||||
// KELVIN SIGN (K)
|
||||
EXPECT(any_of(k_ranges, [](auto const& range) {
|
||||
return range.from == 0x212A && range.to == 0x212A;
|
||||
}));
|
||||
}
|
||||
|
||||
TEST_CASE(for_each_case_folded_code_point)
|
||||
{
|
||||
constexpr u32 GREEK_SMALL_SIGMA = 0x03C3; // σ
|
||||
constexpr u32 GREEK_SMALL_FINAL_SIGMA = 0x03C2; // ς
|
||||
constexpr u32 GREEK_CAPITAL_SIGMA = 0x03A3; // Σ
|
||||
constexpr u32 KELVIN_SIGN = 0x212A; // K
|
||||
|
||||
Vector<u32> folded_A;
|
||||
Unicode::for_each_case_folded_code_point('A', [&](u32 cp) {
|
||||
folded_A.append(cp);
|
||||
return IterationDecision::Continue;
|
||||
});
|
||||
EXPECT(folded_A.contains_slow('A'));
|
||||
EXPECT(folded_A.contains_slow('a'));
|
||||
|
||||
Vector<u32> folded_sigma;
|
||||
Unicode::for_each_case_folded_code_point(GREEK_CAPITAL_SIGMA, [&](u32 cp) {
|
||||
folded_sigma.append(cp);
|
||||
return IterationDecision::Continue;
|
||||
});
|
||||
EXPECT(folded_sigma.contains_slow(GREEK_CAPITAL_SIGMA));
|
||||
EXPECT(folded_sigma.contains_slow(GREEK_SMALL_SIGMA));
|
||||
EXPECT(folded_sigma.contains_slow(GREEK_SMALL_FINAL_SIGMA));
|
||||
|
||||
Vector<u32> folded_kelvin;
|
||||
Unicode::for_each_case_folded_code_point(KELVIN_SIGN, [&](u32 cp) {
|
||||
folded_kelvin.append(cp);
|
||||
return IterationDecision::Continue;
|
||||
});
|
||||
EXPECT(folded_kelvin.contains_slow(KELVIN_SIGN));
|
||||
EXPECT(folded_kelvin.contains_slow('K'));
|
||||
EXPECT(folded_kelvin.contains_slow('k'));
|
||||
}
|
||||
|
||||
TEST_CASE(code_point_matches_range_ignoring_case)
|
||||
{
|
||||
constexpr u32 LATIN_CAPITAL_A_GRAVE = 0x00C0; // À
|
||||
constexpr u32 LATIN_SMALL_A_GRAVE = 0x00E0; // à
|
||||
constexpr u32 GREEK_SMALL_SIGMA = 0x03C3; // σ
|
||||
constexpr u32 GREEK_SMALL_FINAL_SIGMA = 0x03C2; // ς
|
||||
constexpr u32 MICRO_SIGN = 0x00B5; // µ
|
||||
constexpr u32 GREEK_SMALL_MU = 0x03BC; // μ
|
||||
constexpr u32 KELVIN_SIGN = 0x212A; // K
|
||||
|
||||
EXPECT(Unicode::code_point_matches_range_ignoring_case('B', 'a', 'z', true));
|
||||
EXPECT(Unicode::code_point_matches_range_ignoring_case('b', 'A', 'Z', true));
|
||||
|
||||
EXPECT(Unicode::code_point_matches_range_ignoring_case(KELVIN_SIGN, 'a', 'z', true));
|
||||
EXPECT(!Unicode::code_point_matches_range_ignoring_case(KELVIN_SIGN, 'a', 'z', false));
|
||||
|
||||
EXPECT(Unicode::code_point_matches_range_ignoring_case(LATIN_SMALL_A_GRAVE, LATIN_CAPITAL_A_GRAVE, LATIN_CAPITAL_A_GRAVE, true));
|
||||
EXPECT(Unicode::code_point_matches_range_ignoring_case(LATIN_CAPITAL_A_GRAVE, LATIN_SMALL_A_GRAVE, LATIN_SMALL_A_GRAVE, true));
|
||||
|
||||
EXPECT(Unicode::code_point_matches_range_ignoring_case(GREEK_SMALL_FINAL_SIGMA, GREEK_SMALL_SIGMA, GREEK_SMALL_SIGMA, true));
|
||||
EXPECT(Unicode::code_point_matches_range_ignoring_case(MICRO_SIGN, GREEK_SMALL_MU, GREEK_SMALL_MU, true));
|
||||
}
|
||||
|
||||
TEST_CASE(ranges_equal_ignoring_case)
|
||||
{
|
||||
EXPECT(Unicode::ranges_equal_ignoring_case(Utf8View("Hello"sv), Utf16View("HELLO"sv), true));
|
||||
EXPECT(Unicode::ranges_equal_ignoring_case(Utf16View("Hello"sv), Utf8View("hello"sv), true));
|
||||
|
||||
EXPECT(Unicode::ranges_equal_ignoring_case(Utf8View("Σσς"sv), Utf8View("ΣΣΣ"sv), true));
|
||||
EXPECT(Unicode::ranges_equal_ignoring_case(Utf8View("straße"sv), Utf8View("STRAẞE"sv), true));
|
||||
EXPECT(Unicode::ranges_equal_ignoring_case(Utf8View("CAFÉ"sv), Utf8View("café"sv), true));
|
||||
EXPECT(Unicode::ranges_equal_ignoring_case(Utf8View("Œ"sv), Utf8View("œ"sv), true));
|
||||
|
||||
EXPECT(Unicode::ranges_equal_ignoring_case(Utf8View("K"sv), Utf8View("K"sv), true));
|
||||
EXPECT(!Unicode::ranges_equal_ignoring_case(Utf8View("K"sv), Utf8View("K"sv), false));
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue