From 11719369e834ecf83840be77456fac6fd5a077d3 Mon Sep 17 00:00:00 2001 From: Timothy Flynn Date: Sat, 18 Apr 2026 13:29:10 -0400 Subject: [PATCH] LibRegex+LibUnicode: Migrate Unicode Rust FFI methods to LibUnicode Let's not have LibRegex be the home of LibUnicode FFI. Move these to LibUnicode so that we can: 1. Use these helpers in other libraries more easily. 2. Swap out icu4c methods with icu4x methods all within LibUnicode. --- Cargo.lock | 1 + Libraries/LibRegex/Rust/Cargo.toml | 5 + Libraries/LibRegex/Rust/cbindgen.toml | 15 - Libraries/LibRegex/Rust/src/bytecode.rs | 34 +- Libraries/LibRegex/Rust/src/compiler.rs | 21 +- Libraries/LibRegex/Rust/src/lib.rs | 1 - Libraries/LibRegex/Rust/src/parser.rs | 22 +- Libraries/LibRegex/Rust/src/vm.rs | 18 +- Libraries/LibRegex/RustRegex.cpp | 383 ------------------ Libraries/LibUnicode/CharacterTypes.cpp | 302 +++++++++++++- Libraries/LibUnicode/Rust/Cargo.toml | 2 +- .../Rust/src/character_types.rs} | 300 ++++++++------ Libraries/LibUnicode/Rust/src/lib.rs | 1 + 13 files changed, 516 insertions(+), 589 deletions(-) rename Libraries/{LibRegex/Rust/src/unicode_ffi.rs => LibUnicode/Rust/src/character_types.rs} (53%) diff --git a/Cargo.lock b/Cargo.lock index 7111fd9661..a76c50e177 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -392,6 +392,7 @@ name = "libregex_rust" version = "0.1.0" dependencies = [ "cbindgen", + "libunicode_rust", ] [[package]] diff --git a/Libraries/LibRegex/Rust/Cargo.toml b/Libraries/LibRegex/Rust/Cargo.toml index 695e5b13c0..e01db02dc1 100644 --- a/Libraries/LibRegex/Rust/Cargo.toml +++ b/Libraries/LibRegex/Rust/Cargo.toml @@ -6,5 +6,10 @@ edition = "2024" [lib] crate-type = ["staticlib"] +# After changing dependencies, regenerate the Flatpak sources: +# python3 Meta/CMake/flatpak/generate-cargo-sources.py +[dependencies] +libunicode_rust = { path = "../../LibUnicode/Rust", default-features = false } + [build-dependencies] cbindgen = "0.29" diff --git a/Libraries/LibRegex/Rust/cbindgen.toml b/Libraries/LibRegex/Rust/cbindgen.toml index a6bf184dbe..b355945eab 100644 --- a/Libraries/LibRegex/Rust/cbindgen.toml +++ b/Libraries/LibRegex/Rust/cbindgen.toml @@ -18,20 +18,5 @@ parse_deps = false [parse.expand] all_features = false -[export] -exclude = [ - "unicode_property_matches", - "unicode_simple_case_fold", - "unicode_code_point_matches_range_ignoring_case", - "unicode_property_matches_case_insensitive", - "unicode_get_case_closure", - "unicode_is_string_property", - "unicode_property_all_case_equivalents_match", - "unicode_resolved_property_matches", - "unicode_is_valid_ecma262_property", - "unicode_get_string_property_data", - "unicode_resolve_property", -] - [export.mangle] rename_types = "PascalCase" diff --git a/Libraries/LibRegex/Rust/src/bytecode.rs b/Libraries/LibRegex/Rust/src/bytecode.rs index 0b0a500c3a..0a39fe8d84 100644 --- a/Libraries/LibRegex/Rust/src/bytecode.rs +++ b/Libraries/LibRegex/Rust/src/bytecode.rs @@ -18,6 +18,8 @@ //! - A set of registers for capture group positions //! - A backtrack stack for saving/restoring state +pub use libunicode_rust::character_types::{PropertyKind, ResolvedProperty}; + /// A named capture group mapping derived from the pattern's named captures. /// #[derive(Debug, Clone)] @@ -186,38 +188,6 @@ pub enum Instruction { }, } -/// The kind of a resolved Unicode property. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -#[repr(u8)] -pub enum PropertyKind { - Script = 0, - ScriptExtension = 1, - GeneralCategory = 2, - BinaryProperty = 3, -} - -impl PropertyKind { - pub fn from_u8(v: u8) -> Option { - match v { - 0 => Some(Self::Script), - 1 => Some(Self::ScriptExtension), - 2 => Some(Self::GeneralCategory), - 3 => Some(Self::BinaryProperty), - _ => None, - } - } -} - -/// A resolved Unicode property — the string name/value has been resolved to -/// an ICU enum at compile time, so match-time lookups avoid string parsing. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct ResolvedProperty { - /// The kind of Unicode property (Script, GeneralCategory, etc.). - pub kind: PropertyKind, - /// ICU enum value (e.g. script code, general category, binary property ID). - pub id: u32, -} - /// Data for a Unicode property match instruction. #[derive(Debug, Clone, PartialEq, Eq)] pub struct UnicodePropertyData { diff --git a/Libraries/LibRegex/Rust/src/compiler.rs b/Libraries/LibRegex/Rust/src/compiler.rs index e204599e88..4529465bb0 100644 --- a/Libraries/LibRegex/Rust/src/compiler.rs +++ b/Libraries/LibRegex/Rust/src/compiler.rs @@ -24,7 +24,7 @@ use std::collections::BTreeSet; /// - /// - pub fn resolve_property(name: &str, value: Option<&str>) -> Option { - crate::unicode_ffi::resolve_property(name, value) + libunicode_rust::character_types::resolve_property(name, value) } /// Compile a parsed pattern into a bytecode program. @@ -88,7 +88,7 @@ impl Compiler { /// match atomically, as described by `CompileClassSetString`. /// fn get_string_property_strings(name: &str) -> Vec> { - let buf = crate::unicode_ffi::get_string_property_data(name); + let buf = libunicode_rust::character_types::get_string_property_data(name); if buf.is_empty() { return Vec::new(); } @@ -233,7 +233,10 @@ impl Compiler { } } ClassSetOperand::UnicodeProperty(up) => { - if self.program.unicode_sets && !up.negated && crate::unicode_ffi::is_string_property(&up.name) { + if self.program.unicode_sets + && !up.negated + && libunicode_rust::character_types::is_string_property(&up.name) + { let mut lengths = Self::singleton_length_set(); for string in Self::get_string_property_strings(&up.name) { lengths.insert(string.len()); @@ -501,7 +504,7 @@ impl Compiler { Atom::UnicodeProperty(up) => { // String properties (e.g. Basic_Emoji) can match multi-character // sequences and cannot use simple matching. - if self.program.unicode_sets && crate::unicode_ffi::is_string_property(&up.name) { + if self.program.unicode_sets && libunicode_rust::character_types::is_string_property(&up.name) { return None; } Some(SimpleMatch::UnicodeProperty(Box::new(UnicodePropertyData { @@ -813,7 +816,10 @@ impl Compiler { } Atom::UnicodeProperty(up) => { - if self.program.unicode_sets && !up.negated && crate::unicode_ffi::is_string_property(&up.name) { + if self.program.unicode_sets + && !up.negated + && libunicode_rust::character_types::is_string_property(&up.name) + { self.emit_string_property_match(&up.name, up.value.as_deref()); return; } @@ -1231,7 +1237,10 @@ impl Compiler { self.emit(Instruction::BuiltinClass(*bc)); } ClassSetOperand::UnicodeProperty(up) => { - if self.program.unicode_sets && !up.negated && crate::unicode_ffi::is_string_property(&up.name) { + if self.program.unicode_sets + && !up.negated + && libunicode_rust::character_types::is_string_property(&up.name) + { if length == 1 { self.emit_unicode_property(up.negated, &up.name, up.value.as_deref()); } else { diff --git a/Libraries/LibRegex/Rust/src/lib.rs b/Libraries/LibRegex/Rust/src/lib.rs index 4da383bbb9..594f9130d9 100644 --- a/Libraries/LibRegex/Rust/src/lib.rs +++ b/Libraries/LibRegex/Rust/src/lib.rs @@ -13,5 +13,4 @@ pub mod compiler; pub mod ffi; pub mod parser; pub mod regex; -mod unicode_ffi; pub mod vm; diff --git a/Libraries/LibRegex/Rust/src/parser.rs b/Libraries/LibRegex/Rust/src/parser.rs index 159fa233a9..15b66d16be 100644 --- a/Libraries/LibRegex/Rust/src/parser.rs +++ b/Libraries/LibRegex/Rust/src/parser.rs @@ -791,7 +791,7 @@ impl Parser { // Validate against the ECMA-262 allow-list up front so later compiler // and VM stages can assume the escape names already match spec. - if !crate::unicode_ffi::is_valid_ecma262_property(&name, value.as_deref()) { + if !libunicode_rust::character_types::is_valid_ecma262_property(&name, value.as_deref()) { return Err(Error::InvalidUnicodeProperty(name)); } @@ -800,7 +800,7 @@ impl Parser { /// Check if a property name refers to a Unicode string property (e.g. Basic_Emoji). fn is_string_property(name: &str) -> bool { - crate::unicode_ffi::is_string_property(name) + libunicode_rust::character_types::is_string_property(name) } fn parse_unicode_property_name(&mut self) -> Result { @@ -1010,7 +1010,7 @@ impl Parser { fn parse_group_name(&mut self) -> Result { let mut name = String::new(); let first = self.parse_group_name_char()?; - if !is_id_start(first) { + if !is_identifier_start_char(first) { return Err(Error::InvalidGroupName); } name.push(first); @@ -1019,7 +1019,7 @@ impl Parser { break; } let ch = self.parse_group_name_char()?; - if !is_id_continue(ch) { + if !is_identifier_continue_char(ch) { return Err(Error::InvalidGroupName); } name.push(ch); @@ -1422,7 +1422,7 @@ impl Parser { } fn get_string_property_strings(name: &str) -> std::collections::BTreeSet> { - let buf = crate::unicode_ffi::get_string_property_data(name); + let buf = libunicode_rust::character_types::get_string_property_data(name); if buf.is_empty() { return std::collections::BTreeSet::new(); } @@ -1782,12 +1782,16 @@ fn is_modifier_flag(ch: char) -> bool { matches!(ch, 'i' | 'm' | 's') } -fn is_id_start(ch: char) -> bool { - ch == '$' || ch == '_' || crate::unicode_ffi::is_id_start(ch as u32) +// https://tc39.es/ecma262/#prod-IdentifierStartChar +// IdentifierStartChar :: UnicodeIDStart | $ | _ +fn is_identifier_start_char(ch: char) -> bool { + ch == '$' || ch == '_' || libunicode_rust::character_types::code_point_has_identifier_start_property(ch as u32) } -fn is_id_continue(ch: char) -> bool { - matches!(ch, '$' | '_' | '\u{200C}' | '\u{200D}') || crate::unicode_ffi::is_id_continue(ch as u32) +// https://tc39.es/ecma262/#prod-IdentifierPartChar +// IdentifierPartChar :: UnicodeIDContinue | $ +fn is_identifier_continue_char(ch: char) -> bool { + ch == '$' || libunicode_rust::character_types::code_point_has_identifier_continue_property(ch as u32) } /// Extract a single code point from a class atom, for use in ranges. diff --git a/Libraries/LibRegex/Rust/src/vm.rs b/Libraries/LibRegex/Rust/src/vm.rs index 81cdff68d8..dc4d349678 100644 --- a/Libraries/LibRegex/Rust/src/vm.rs +++ b/Libraries/LibRegex/Rust/src/vm.rs @@ -3381,7 +3381,7 @@ fn case_fold(cp: u32, unicode_mode: bool) -> u32 { return cp; } // Non-ASCII: call FFI which handles both modes correctly. - crate::unicode_ffi::simple_case_fold(cp, unicode_mode) + libunicode_rust::character_types::simple_case_fold(cp, unicode_mode) } /// Compare two code points for case-insensitive equality. @@ -3464,7 +3464,7 @@ pub(crate) fn match_char_class( // v-flag case-insensitive: check full case closure. // Get all case-equivalent code points and check if any falls in the ranges. let mut closure_buf = [0u32; 16]; - let count = crate::unicode_ffi::get_case_closure(cp, &mut closure_buf); + let count = libunicode_rust::character_types::get_case_closure(cp, &mut closure_buf); for item in closure_buf.iter().take(count) { let equiv = *item; if char_in_ranges(equiv, ranges) { @@ -3488,9 +3488,9 @@ pub(crate) fn match_char_class( // other characters that share the same canonical form as cp. // Always use the ICU-based range matcher which correctly handles // cross-script case folding (e.g. U+017F ſ folds to ASCII s). - ranges - .iter() - .any(|r| crate::unicode_ffi::code_point_matches_range_ignoring_case(cp, r.start, r.end, unicode_mode)) + ranges.iter().any(|r| { + libunicode_rust::character_types::code_point_matches_range_ignoring_case(cp, r.start, r.end, unicode_mode) + }) } else { char_in_ranges(cp, ranges) } @@ -3511,20 +3511,20 @@ pub(crate) fn match_builtin_class(cp: u32, class: BuiltinCharacterClass, unicode /// Match a Unicode property considering case closure for ignore-case matching. /// pub(crate) fn match_unicode_property_case_insensitive(cp: u32, name: &str, value: Option<&str>) -> bool { - crate::unicode_ffi::property_matches_case_insensitive(cp, name, value) + libunicode_rust::character_types::property_matches_case_insensitive(cp, name, value) } /// Check if all case-equivalents of `cp` have the property. /// pub(crate) fn match_unicode_property_all_case_equivalents(cp: u32, name: &str, value: Option<&str>) -> bool { - crate::unicode_ffi::property_all_case_equivalents_match(cp, name, value) + libunicode_rust::character_types::property_all_case_equivalents_match(cp, name, value) } /// Match a Unicode property via FFI to C++ LibUnicode. /// - /// - fn match_unicode_property(cp: u32, name: &str, value: Option<&str>) -> bool { - crate::unicode_ffi::property_matches(cp, name, value) + libunicode_rust::character_types::property_matches(cp, name, value) } /// Match a Unicode property using a resolved ICU property ID if available, @@ -3539,7 +3539,7 @@ pub(crate) fn match_unicode_property_resolved( resolved: Option<&ResolvedProperty>, ) -> bool { if let Some(r) = resolved { - return crate::unicode_ffi::resolved_property_matches(cp, *r); + return libunicode_rust::character_types::resolved_property_matches(cp, *r); } match_unicode_property(cp, name, value) } diff --git a/Libraries/LibRegex/RustRegex.cpp b/Libraries/LibRegex/RustRegex.cpp index 1b55bc2d35..8f725798fc 100644 --- a/Libraries/LibRegex/RustRegex.cpp +++ b/Libraries/LibRegex/RustRegex.cpp @@ -5,389 +5,6 @@ */ #include -#include - -// Forward declarations for C++ functions called from Rust. -extern "C" { -bool unicode_property_matches(uint32_t, unsigned char const*, size_t, unsigned char const*, size_t, int); -uint32_t unicode_simple_case_fold(uint32_t, int); -int unicode_code_point_matches_range_ignoring_case(uint32_t, uint32_t, uint32_t, int); -int unicode_property_matches_case_insensitive(uint32_t, unsigned char const*, size_t, unsigned char const*, size_t, int); -int unicode_property_all_case_equivalents_match(uint32_t, unsigned char const*, size_t, unsigned char const*, size_t, int); -unsigned int unicode_get_case_closure(uint32_t, uint32_t*, unsigned int); -int unicode_is_string_property(unsigned char const*, size_t); -int unicode_is_valid_ecma262_property(unsigned char const*, size_t, unsigned char const*, size_t, int); -uint32_t unicode_get_string_property_data(unsigned char const*, size_t, uint32_t*, uint32_t); -int unicode_resolve_property(unsigned char const*, size_t, unsigned char const*, size_t, int, unsigned char*, uint32_t*); -int unicode_resolved_property_matches(uint32_t, unsigned char, uint32_t); -int unicode_is_id_start(uint32_t); -int unicode_is_id_continue(uint32_t); -} - -extern "C" bool unicode_property_matches( - uint32_t code_point, - unsigned char const* name_ptr, size_t name_len, - unsigned char const* value_ptr, size_t value_len, - int has_value) -{ - auto name = StringView { reinterpret_cast(name_ptr), name_len }; - auto value = has_value - ? StringView { reinterpret_cast(value_ptr), value_len } - : StringView {}; - - // If there's a value, the name is a category key like "Script", "General_Category", etc. - if (has_value) { - // Script or Script_Extensions - if (name.is_one_of("sc"sv, "Script"sv, "scx"sv, "Script_Extensions"sv)) { - auto script = Unicode::script_from_string(value); - if (!script.has_value()) - return false; - if (name.is_one_of("sc"sv, "Script"sv)) - return Unicode::code_point_has_script(code_point, *script); - return Unicode::code_point_has_script_extension(code_point, *script); - } - // General_Category - if (name.is_one_of("gc"sv, "General_Category"sv)) { - auto category = Unicode::general_category_from_string(value); - if (!category.has_value()) - return false; - return Unicode::code_point_has_general_category(code_point, *category); - } - return false; - } - - // No value: could be a property name, general category, or script. - // Try as a binary property first. - auto prop = Unicode::property_from_string(name); - if (prop.has_value()) - return Unicode::code_point_has_property(code_point, *prop); - - // Try as a general category. - auto category = Unicode::general_category_from_string(name); - if (category.has_value()) - return Unicode::code_point_has_general_category(code_point, *category); - - // Try as a script. - auto script = Unicode::script_from_string(name); - if (script.has_value()) - return Unicode::code_point_has_script(code_point, *script); - - return false; -} - -// Property kind constants (must match Rust ResolvedProperty::kind). -enum ResolvedPropertyKind : uint8_t { - Script = 0, - ScriptExtension = 1, - GeneralCategory = 2, - BinaryProperty = 3, -}; - -/// Resolve a Unicode property name/value pair to a (kind, id) pair. -/// Returns 1 on success, 0 if the property is not recognized. -extern "C" int unicode_resolve_property( - unsigned char const* name_ptr, size_t name_len, - unsigned char const* value_ptr, size_t value_len, - int has_value, - unsigned char* out_kind, uint32_t* out_id) -{ - auto name = StringView { reinterpret_cast(name_ptr), name_len }; - auto value = has_value - ? StringView { reinterpret_cast(value_ptr), value_len } - : StringView {}; - - if (has_value) { - if (name.is_one_of("sc"sv, "Script"sv, "scx"sv, "Script_Extensions"sv)) { - auto script = Unicode::script_from_string(value); - if (!script.has_value()) - return 0; - *out_kind = name.is_one_of("scx"sv, "Script_Extensions"sv) - ? ResolvedPropertyKind::ScriptExtension - : ResolvedPropertyKind::Script; - *out_id = script->value(); - return 1; - } - if (name.is_one_of("gc"sv, "General_Category"sv)) { - auto category = Unicode::general_category_from_string(value); - if (!category.has_value()) - return 0; - *out_kind = ResolvedPropertyKind::GeneralCategory; - *out_id = category->value(); - return 1; - } - return 0; - } - - // No value: try binary property, general category, script. - auto prop = Unicode::property_from_string(name); - if (prop.has_value()) { - *out_kind = ResolvedPropertyKind::BinaryProperty; - *out_id = prop->value(); - return 1; - } - auto category = Unicode::general_category_from_string(name); - if (category.has_value()) { - *out_kind = ResolvedPropertyKind::GeneralCategory; - *out_id = category->value(); - return 1; - } - auto script = Unicode::script_from_string(name); - if (script.has_value()) { - *out_kind = ResolvedPropertyKind::Script; - *out_id = script->value(); - return 1; - } - return 0; -} - -/// Check if a code point matches a resolved property. Direct ICU trie lookup, no string parsing. -extern "C" int unicode_resolved_property_matches(uint32_t code_point, unsigned char kind, uint32_t id) -{ - switch (kind) { - case ResolvedPropertyKind::Script: - return Unicode::code_point_has_script(code_point, Unicode::Script { id }) ? 1 : 0; - case ResolvedPropertyKind::ScriptExtension: - return Unicode::code_point_has_script_extension(code_point, Unicode::Script { id }) ? 1 : 0; - case ResolvedPropertyKind::GeneralCategory: - return Unicode::code_point_has_general_category(code_point, Unicode::GeneralCategory { id }) ? 1 : 0; - case ResolvedPropertyKind::BinaryProperty: - return Unicode::code_point_has_property(code_point, Unicode::Property { id }) ? 1 : 0; - default: - return 0; - } -} - -extern "C" uint32_t unicode_simple_case_fold(uint32_t code_point, int unicode_mode) -{ - return Unicode::canonicalize(code_point, unicode_mode != 0); -} - -extern "C" int unicode_code_point_matches_range_ignoring_case(uint32_t code_point, uint32_t from, uint32_t to, int unicode_mode) -{ - return Unicode::code_point_matches_range_ignoring_case(code_point, from, to, unicode_mode != 0) ? 1 : 0; -} - -// Check if a code point matches a Unicode property, considering case closure. -// Returns 1 if the code point or any of its case equivalents has the property. -extern "C" int unicode_property_matches_case_insensitive( - uint32_t code_point, - unsigned char const* name_ptr, size_t name_len, - unsigned char const* value_ptr, size_t value_len, - int has_value) -{ - // First check the code point itself. - auto name = StringView { reinterpret_cast(name_ptr), name_len }; - auto value = has_value - ? StringView { reinterpret_cast(value_ptr), value_len } - : StringView {}; - - auto check_property = [&](u32 cp) -> bool { - if (has_value) { - if (name.is_one_of("sc"sv, "Script"sv, "scx"sv, "Script_Extensions"sv)) { - auto script = Unicode::script_from_string(value); - if (!script.has_value()) - return false; - if (name.is_one_of("sc"sv, "Script"sv)) - return Unicode::code_point_has_script(cp, *script); - return Unicode::code_point_has_script_extension(cp, *script); - } - if (name.is_one_of("gc"sv, "General_Category"sv)) { - auto category = Unicode::general_category_from_string(value); - if (!category.has_value()) - return false; - return Unicode::code_point_has_general_category(cp, *category); - } - return false; - } - auto prop = Unicode::property_from_string(name); - if (prop.has_value()) - return Unicode::code_point_has_property(cp, *prop); - auto category = Unicode::general_category_from_string(name); - if (category.has_value()) - return Unicode::code_point_has_general_category(cp, *category); - auto script = Unicode::script_from_string(name); - if (script.has_value()) - return Unicode::code_point_has_script(cp, *script); - return false; - }; - - // Check using case closure — all case-equivalent code points. - bool found = false; - Unicode::for_each_case_folded_code_point(code_point, [&](u32 cp) { - if (check_property(cp)) { - found = true; - return IterationDecision::Break; - } - return IterationDecision::Continue; - }); - return found ? 1 : 0; -} - -// Check if ALL case-equivalents of a code point have the property. -// Returns 1 only if every case-equivalent has the property. -// Used for u-flag negated case-insensitive: \P{X}/iu matches if NOT all have X. -extern "C" int unicode_property_all_case_equivalents_match( - uint32_t code_point, - unsigned char const* name_ptr, size_t name_len, - unsigned char const* value_ptr, size_t value_len, - int has_value) -{ - auto name = StringView { reinterpret_cast(name_ptr), name_len }; - auto value = has_value - ? StringView { reinterpret_cast(value_ptr), value_len } - : StringView {}; - - auto check_property = [&](u32 cp) -> bool { - if (has_value) { - if (name.is_one_of("sc"sv, "Script"sv, "scx"sv, "Script_Extensions"sv)) { - auto script = Unicode::script_from_string(value); - if (!script.has_value()) - return false; - if (name.is_one_of("sc"sv, "Script"sv)) - return Unicode::code_point_has_script(cp, *script); - return Unicode::code_point_has_script_extension(cp, *script); - } - if (name.is_one_of("gc"sv, "General_Category"sv)) { - auto category = Unicode::general_category_from_string(value); - if (!category.has_value()) - return false; - return Unicode::code_point_has_general_category(cp, *category); - } - return false; - } - auto prop = Unicode::property_from_string(name); - if (prop.has_value()) - return Unicode::code_point_has_property(cp, *prop); - auto category = Unicode::general_category_from_string(name); - if (category.has_value()) - return Unicode::code_point_has_general_category(cp, *category); - auto script = Unicode::script_from_string(name); - if (script.has_value()) - return Unicode::code_point_has_script(cp, *script); - return false; - }; - - bool all_match = true; - Unicode::for_each_case_folded_code_point(code_point, [&](u32 cp) { - if (!check_property(cp)) { - all_match = false; - return IterationDecision::Break; - } - return IterationDecision::Continue; - }); - return all_match ? 1 : 0; -} - -extern "C" unsigned int unicode_get_case_closure( - uint32_t code_point, - uint32_t* out_buffer, - unsigned int buffer_capacity) -{ - unsigned int count = 0; - Unicode::for_each_case_folded_code_point(code_point, [&](u32 cp) { - if (count < buffer_capacity) { - out_buffer[count] = cp; - count++; - } - return IterationDecision::Continue; - }); - return count; -} - -extern "C" int unicode_is_string_property( - unsigned char const* name_ptr, size_t name_len) -{ - auto name = StringView { reinterpret_cast(name_ptr), name_len }; - auto prop = Unicode::property_from_string(name); - if (!prop.has_value()) - return 0; - return Unicode::is_ecma262_string_property(*prop) ? 1 : 0; -} - -extern "C" int unicode_is_valid_ecma262_property( - unsigned char const* name_ptr, size_t name_len, - unsigned char const* value_ptr, size_t value_len, - int has_value) -{ - auto name = StringView { reinterpret_cast(name_ptr), name_len }; - auto value = has_value - ? StringView { reinterpret_cast(value_ptr), value_len } - : StringView {}; - - if (has_value) { - // Key=Value form: Script, Script_Extensions, General_Category - if (name.is_one_of("sc"sv, "Script"sv, "scx"sv, "Script_Extensions"sv)) - return Unicode::script_from_string(value).has_value() ? 1 : 0; - if (name.is_one_of("gc"sv, "General_Category"sv)) - return Unicode::general_category_from_string(value).has_value() ? 1 : 0; - return 0; - } - - // Lone name: try as ECMA-262 binary property or General_Category value. - // Note: Script names (e.g. "Hiragana") are NOT valid as lone names per - // ECMA-262 -- they require Script= or sc= prefix. - auto prop = Unicode::property_from_string(name); - if (prop.has_value()) - return (Unicode::is_ecma262_property(*prop) || Unicode::is_ecma262_string_property(*prop)) ? 1 : 0; - if (Unicode::general_category_from_string(name).has_value()) - return 1; - return 0; -} - -/// Get all multi-codepoint strings for a Unicode string property. -/// Writes packed data: [string_count, len1, cp1_0, cp1_1, ..., len2, cp2_0, ...] -/// Returns total number of u32 values written. -/// If out is null, returns the total size needed. -extern "C" uint32_t unicode_get_string_property_data( - unsigned char const* name_ptr, size_t name_len, - uint32_t* out, uint32_t capacity) -{ - auto name = StringView { reinterpret_cast(name_ptr), name_len }; - auto prop = Unicode::property_from_string(name); - if (!prop.has_value() || !Unicode::is_ecma262_string_property(*prop)) - return 0; - - auto strings = Unicode::get_property_strings(*prop); - - // Filter to multi-codepoint strings and compute total size needed. - // Format: [count, len1, cp1..., len2, cp2..., ...] - Vector> multi_cp_strings; - for (auto const& str : strings) { - Vector code_points; - for (auto cp : str.code_points()) - code_points.append(cp); - if (code_points.size() > 1) - multi_cp_strings.append(move(code_points)); - } - - // Calculate total size: 1 (count) + sum(1 + len) for each string - uint32_t total_size = 1; - for (auto const& cps : multi_cp_strings) - total_size += 1 + static_cast(cps.size()); - - if (!out || capacity < total_size) - return total_size; - - uint32_t offset = 0; - out[offset++] = static_cast(multi_cp_strings.size()); - for (auto const& cps : multi_cp_strings) { - out[offset++] = static_cast(cps.size()); - for (auto cp : cps) - out[offset++] = cp; - } - - return total_size; -} - -extern "C" int unicode_is_id_start(uint32_t code_point) -{ - return Unicode::code_point_has_identifier_start_property(code_point) ? 1 : 0; -} - -extern "C" int unicode_is_id_continue(uint32_t code_point) -{ - return Unicode::code_point_has_identifier_continue_property(code_point) ? 1 : 0; -} namespace regex { diff --git a/Libraries/LibUnicode/CharacterTypes.cpp b/Libraries/LibUnicode/CharacterTypes.cpp index 9a684ef71e..21c9ee0001 100644 --- a/Libraries/LibUnicode/CharacterTypes.cpp +++ b/Libraries/LibUnicode/CharacterTypes.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2024, Tim Flynn + * Copyright (c) 2021-2026, Tim Flynn * * SPDX-License-Identifier: BSD-2-Clause */ @@ -637,3 +637,303 @@ bool code_point_matches_range_ignoring_case(u32 code_point, u32 from, u32 to, bo } } + +enum class ResolvedPropertyKind : u8 { + Script, + ScriptExtension, + GeneralCategory, + BinaryProperty, +}; + +static constexpr StringView string_view_from_ffi(unsigned char const* string, size_t length) +{ + VERIFY(string); + return { reinterpret_cast(string), length }; +} + +static constexpr Optional optional_string_view_from_ffi(unsigned char const* string, size_t length) +{ + if (string) + return string_view_from_ffi(string, length); + return {}; +} + +static bool has_property(u32 code_point, StringView name, Optional value) +{ + if (value.has_value()) { + if (name.is_one_of("sc"sv, "Script"sv, "scx"sv, "Script_Extensions"sv)) { + if (auto script = Unicode::script_from_string(*value); script.has_value()) { + if (name.is_one_of("sc"sv, "Script"sv)) + return Unicode::code_point_has_script(code_point, *script); + return Unicode::code_point_has_script_extension(code_point, *script); + } + } else if (name.is_one_of("gc"sv, "General_Category"sv)) { + if (auto category = Unicode::general_category_from_string(*value); category.has_value()) + return Unicode::code_point_has_general_category(code_point, *category); + } + + return false; + } + + if (auto property = Unicode::property_from_string(name); property.has_value()) + return Unicode::code_point_has_property(code_point, *property); + + if (auto category = Unicode::general_category_from_string(name); category.has_value()) + return Unicode::code_point_has_general_category(code_point, *category); + + if (auto script = Unicode::script_from_string(name); script.has_value()) + return Unicode::code_point_has_script(code_point, *script); + + return false; +} + +static bool resolve_property(StringView name, Optional value, unsigned char* out_kind, u32* out_id) +{ + if (value.has_value()) { + if (name.is_one_of("sc"sv, "Script"sv, "scx"sv, "Script_Extensions"sv)) { + if (auto script = Unicode::script_from_string(*value); script.has_value()) { + *out_kind = to_underlying(name.is_one_of("scx"sv, "Script_Extensions"sv) + ? ResolvedPropertyKind::ScriptExtension + : ResolvedPropertyKind::Script); + *out_id = script->value(); + return true; + } + } else if (name.is_one_of("gc"sv, "General_Category"sv)) { + if (auto category = Unicode::general_category_from_string(*value); category.has_value()) { + *out_kind = to_underlying(ResolvedPropertyKind::GeneralCategory); + *out_id = category->value(); + return true; + } + } + + return false; + } + + if (auto property = Unicode::property_from_string(name); property.has_value()) { + *out_kind = to_underlying(ResolvedPropertyKind::BinaryProperty); + *out_id = property->value(); + return true; + } + + if (auto category = Unicode::general_category_from_string(name); category.has_value()) { + *out_kind = to_underlying(ResolvedPropertyKind::GeneralCategory); + *out_id = category->value(); + return true; + } + + if (auto script = Unicode::script_from_string(name); script.has_value()) { + *out_kind = to_underlying(ResolvedPropertyKind::Script); + *out_id = script->value(); + return true; + } + + return false; +} + +extern "C" { + +bool unicode_property_matches(u32, unsigned char const*, size_t, unsigned char const*, size_t); +bool unicode_property_matches_case_insensitive(u32, unsigned char const*, size_t, unsigned char const*, size_t); +bool unicode_property_all_case_equivalents_match(u32, unsigned char const*, size_t, unsigned char const*, size_t); + +bool unicode_resolve_property(unsigned char const*, size_t, unsigned char const*, size_t, unsigned char*, u32*); +bool unicode_resolved_property_matches(u32, unsigned char, u32); + +bool unicode_code_point_has_identifier_start_property(u32); +bool unicode_code_point_has_identifier_continue_property(u32); + +bool unicode_is_string_property(unsigned char const*, size_t); +bool unicode_is_valid_ecma262_property(unsigned char const*, size_t, unsigned char const*, size_t); +u32 unicode_get_string_property_data(unsigned char const*, size_t, u32*, u32); + +u32 unicode_simple_case_fold(u32, bool); + +bool unicode_code_point_matches_range_ignoring_case(u32, u32, u32, bool); +u32 unicode_get_case_closure(u32, u32*, u32); +} + +extern "C" bool unicode_property_matches( + u32 code_point, + unsigned char const* name_ptr, size_t name_len, + unsigned char const* value_ptr, size_t value_len) +{ + auto name = string_view_from_ffi(name_ptr, name_len); + auto value = optional_string_view_from_ffi(value_ptr, value_len); + + return has_property(code_point, name, value); +} + +extern "C" bool unicode_property_matches_case_insensitive( + u32 code_point, + unsigned char const* name_ptr, size_t name_len, + unsigned char const* value_ptr, size_t value_len) +{ + auto name = string_view_from_ffi(name_ptr, name_len); + auto value = optional_string_view_from_ffi(value_ptr, value_len); + bool found = false; + + Unicode::for_each_case_folded_code_point(code_point, [&](u32 cp) { + if (has_property(cp, name, value)) { + found = true; + return IterationDecision::Break; + } + return IterationDecision::Continue; + }); + + return found; +} + +extern "C" bool unicode_property_all_case_equivalents_match( + u32 code_point, + unsigned char const* name_ptr, size_t name_len, + unsigned char const* value_ptr, size_t value_len) +{ + auto name = string_view_from_ffi(name_ptr, name_len); + auto value = optional_string_view_from_ffi(value_ptr, value_len); + bool all_match = true; + + Unicode::for_each_case_folded_code_point(code_point, [&](u32 cp) { + if (!has_property(cp, name, value)) { + all_match = false; + return IterationDecision::Break; + } + return IterationDecision::Continue; + }); + + return all_match; +} + +extern "C" bool unicode_resolve_property( + unsigned char const* name_ptr, size_t name_len, + unsigned char const* value_ptr, size_t value_len, + unsigned char* out_kind, u32* out_id) +{ + auto name = string_view_from_ffi(name_ptr, name_len); + auto value = optional_string_view_from_ffi(value_ptr, value_len); + + return resolve_property(name, value, out_kind, out_id); +} + +extern "C" bool unicode_resolved_property_matches(u32 code_point, unsigned char kind, u32 id) +{ + switch (static_cast(kind)) { + case ResolvedPropertyKind::Script: + return Unicode::code_point_has_script(code_point, Unicode::Script { id }); + case ResolvedPropertyKind::ScriptExtension: + return Unicode::code_point_has_script_extension(code_point, Unicode::Script { id }); + case ResolvedPropertyKind::GeneralCategory: + return Unicode::code_point_has_general_category(code_point, Unicode::GeneralCategory { id }); + case ResolvedPropertyKind::BinaryProperty: + return Unicode::code_point_has_property(code_point, Unicode::Property { id }); + } + VERIFY_NOT_REACHED(); +} + +extern "C" bool unicode_code_point_has_identifier_start_property(u32 code_point) +{ + return Unicode::code_point_has_identifier_start_property(code_point); +} + +extern "C" bool unicode_code_point_has_identifier_continue_property(u32 code_point) +{ + return Unicode::code_point_has_identifier_continue_property(code_point); +} + +extern "C" bool unicode_is_string_property(unsigned char const* name_ptr, size_t name_len) +{ + auto name = string_view_from_ffi(name_ptr, name_len); + + if (auto property = Unicode::property_from_string(name); property.has_value()) + return Unicode::is_ecma262_string_property(*property); + + return false; +} + +extern "C" bool unicode_is_valid_ecma262_property( + unsigned char const* name_ptr, size_t name_len, + unsigned char const* value_ptr, size_t value_len) +{ + auto name = string_view_from_ffi(name_ptr, name_len); + auto value = optional_string_view_from_ffi(value_ptr, value_len); + + if (value.has_value()) { + if (name.is_one_of("sc"sv, "Script"sv, "scx"sv, "Script_Extensions"sv)) + return Unicode::script_from_string(*value).has_value(); + if (name.is_one_of("gc"sv, "General_Category"sv)) + return Unicode::general_category_from_string(*value).has_value(); + return false; + } + + if (auto property = Unicode::property_from_string(name); property.has_value()) + return Unicode::is_ecma262_property(*property) || Unicode::is_ecma262_string_property(*property); + + return Unicode::general_category_from_string(name).has_value(); +} + +extern "C" u32 unicode_get_string_property_data( + unsigned char const* name_ptr, size_t name_len, + u32* out, u32 capacity) +{ + auto name = string_view_from_ffi(name_ptr, name_len); + auto property = Unicode::property_from_string(name); + if (!property.has_value() || !Unicode::is_ecma262_string_property(*property)) + return 0; + + auto strings = Unicode::get_property_strings(*property); + + Vector> multi_code_point_strings; + for (auto const& string : strings) { + Vector code_points; + for (auto code_point : string.code_points()) + code_points.append(code_point); + if (code_points.size() > 1) + multi_code_point_strings.append(move(code_points)); + } + + u32 total_size = 1; + for (auto const& code_points : multi_code_point_strings) + total_size += 1 + static_cast(code_points.size()); + + if (!out || capacity < total_size) + return total_size; + + u32 offset = 0; + out[offset++] = static_cast(multi_code_point_strings.size()); + + for (auto const& code_points : multi_code_point_strings) { + out[offset++] = static_cast(code_points.size()); + + for (auto code_point : code_points) + out[offset++] = code_point; + } + + return total_size; +} + +extern "C" u32 unicode_simple_case_fold(u32 code_point, bool unicode_mode) +{ + return Unicode::canonicalize(code_point, unicode_mode); +} + +extern "C" bool unicode_code_point_matches_range_ignoring_case(u32 code_point, u32 from, u32 to, bool unicode_mode) +{ + return Unicode::code_point_matches_range_ignoring_case(code_point, from, to, unicode_mode); +} + +extern "C" u32 unicode_get_case_closure( + u32 code_point, + u32* out_buffer, + u32 buffer_capacity) +{ + u32 count = 0; + + Unicode::for_each_case_folded_code_point(code_point, [&](u32 cp) { + if (count < buffer_capacity) { + out_buffer[count++] = cp; + return IterationDecision::Continue; + } + return IterationDecision::Break; + }); + + return count; +} diff --git a/Libraries/LibUnicode/Rust/Cargo.toml b/Libraries/LibUnicode/Rust/Cargo.toml index 5790321e4b..bd710161c1 100644 --- a/Libraries/LibUnicode/Rust/Cargo.toml +++ b/Libraries/LibUnicode/Rust/Cargo.toml @@ -8,7 +8,7 @@ default = [] allocator = [] [lib] -crate-type = ["staticlib"] +crate-type = ["staticlib", "rlib"] # After changing dependencies, regenerate the Flatpak sources: # python3 Meta/CMake/flatpak/generate-cargo-sources.py diff --git a/Libraries/LibRegex/Rust/src/unicode_ffi.rs b/Libraries/LibUnicode/Rust/src/character_types.rs similarity index 53% rename from Libraries/LibRegex/Rust/src/unicode_ffi.rs rename to Libraries/LibUnicode/Rust/src/character_types.rs index d68dac0c52..4cf00b7d15 100644 --- a/Libraries/LibRegex/Rust/src/unicode_ffi.rs +++ b/Libraries/LibUnicode/Rust/src/character_types.rs @@ -4,7 +4,32 @@ * SPDX-License-Identifier: BSD-2-Clause */ -use crate::bytecode::{PropertyKind, ResolvedProperty}; +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[repr(u8)] +pub enum PropertyKind { + Script = 0, + ScriptExtension = 1, + GeneralCategory = 2, + BinaryProperty = 3, +} + +impl PropertyKind { + pub fn from_u8(value: u8) -> Option { + match value { + 0 => Some(Self::Script), + 1 => Some(Self::ScriptExtension), + 2 => Some(Self::GeneralCategory), + 3 => Some(Self::BinaryProperty), + _ => None, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ResolvedProperty { + pub kind: PropertyKind, + pub id: u32, +} unsafe extern "C" { fn unicode_property_matches( @@ -13,25 +38,15 @@ unsafe extern "C" { name_len: usize, value_ptr: *const u8, value_len: usize, - has_value: i32, ) -> bool; - fn unicode_simple_case_fold(code_point: u32, unicode_mode: i32) -> u32; - - fn unicode_code_point_matches_range_ignoring_case(code_point: u32, from: u32, to: u32, unicode_mode: i32) -> i32; - fn unicode_property_matches_case_insensitive( code_point: u32, name_ptr: *const u8, name_len: usize, value_ptr: *const u8, value_len: usize, - has_value: i32, - ) -> i32; - - fn unicode_get_case_closure(code_point: u32, out_buffer: *mut u32, buffer_capacity: u32) -> u32; - - fn unicode_is_string_property(name_ptr: *const u8, name_len: usize) -> i32; + ) -> bool; fn unicode_property_all_case_equivalents_match( code_point: u32, @@ -39,145 +54,98 @@ unsafe extern "C" { name_len: usize, value_ptr: *const u8, value_len: usize, - has_value: i32, - ) -> i32; - - fn unicode_resolved_property_matches(code_point: u32, kind: u8, id: u32) -> i32; - - fn unicode_is_valid_ecma262_property( - name_ptr: *const u8, - name_len: usize, - value_ptr: *const u8, - value_len: usize, - has_value: i32, - ) -> i32; - - fn unicode_get_string_property_data(name_ptr: *const u8, name_len: usize, out: *mut u32, capacity: u32) -> u32; + ) -> bool; fn unicode_resolve_property( name_ptr: *const u8, name_len: usize, value_ptr: *const u8, value_len: usize, - has_value: i32, out_kind: *mut u8, out_id: *mut u32, - ) -> i32; + ) -> bool; + + fn unicode_resolved_property_matches(code_point: u32, kind: u8, id: u32) -> bool; + + fn unicode_code_point_has_identifier_start_property(code_point: u32) -> bool; + fn unicode_code_point_has_identifier_continue_property(code_point: u32) -> bool; + + fn unicode_is_string_property(name_ptr: *const u8, name_len: usize) -> bool; + + fn unicode_is_valid_ecma262_property( + name_ptr: *const u8, + name_len: usize, + value_ptr: *const u8, + value_len: usize, + ) -> bool; + + fn unicode_get_string_property_data(name_ptr: *const u8, name_len: usize, out: *mut u32, capacity: u32) -> u32; + + fn unicode_simple_case_fold(code_point: u32, unicode_mode: bool) -> u32; + + fn unicode_code_point_matches_range_ignoring_case(code_point: u32, from: u32, to: u32, unicode_mode: bool) -> bool; + + fn unicode_get_case_closure(code_point: u32, out_buffer: *mut u32, buffer_capacity: u32) -> u32; - fn unicode_is_id_start(code_point: u32) -> i32; - fn unicode_is_id_continue(code_point: u32) -> i32; } #[inline(always)] -fn optional_value_parts(value: Option<&str>) -> (*const u8, usize, i32) { +fn optional_value_parts(value: Option<&str>) -> (*const u8, usize) { match value { - Some(value) => (value.as_ptr(), value.len(), 1), - None => (std::ptr::null(), 0, 0), - } -} - -pub(crate) fn property_matches(code_point: u32, name: &str, value: Option<&str>) -> bool { - let (value_ptr, value_len, has_value) = optional_value_parts(value); - // SAFETY: `name` and `value` remain valid for the duration of this call, - // and the C++ helper only reads through those pointers. - unsafe { unicode_property_matches(code_point, name.as_ptr(), name.len(), value_ptr, value_len, has_value) } -} - -#[inline(always)] -pub(crate) fn simple_case_fold(code_point: u32, unicode_mode: bool) -> u32 { - // SAFETY: This forwards only scalar values to the C++ helper. - unsafe { unicode_simple_case_fold(code_point, if unicode_mode { 1 } else { 0 }) } -} - -#[inline(always)] -pub(crate) fn code_point_matches_range_ignoring_case(code_point: u32, from: u32, to: u32, unicode_mode: bool) -> bool { - // SAFETY: This forwards only scalar values to the C++ helper. - unsafe { - unicode_code_point_matches_range_ignoring_case(code_point, from, to, if unicode_mode { 1 } else { 0 }) != 0 - } -} - -pub(crate) fn property_matches_case_insensitive(code_point: u32, name: &str, value: Option<&str>) -> bool { - let (value_ptr, value_len, has_value) = optional_value_parts(value); - // SAFETY: `name` and `value` remain valid for the duration of this call, - // and the C++ helper only reads through those pointers. - unsafe { - unicode_property_matches_case_insensitive( - code_point, - name.as_ptr(), - name.len(), - value_ptr, - value_len, - has_value, - ) != 0 - } -} - -pub(crate) fn get_case_closure(code_point: u32, out_buffer: &mut [u32]) -> usize { - let capacity = out_buffer.len().min(u32::MAX as usize) as u32; - // SAFETY: `out_buffer` is writable for `capacity` elements and the C++ - // helper writes at most that many entries. - unsafe { unicode_get_case_closure(code_point, out_buffer.as_mut_ptr(), capacity) as usize } -} - -pub(crate) fn is_string_property(name: &str) -> bool { - // SAFETY: `name` remains valid for the duration of this call and the C++ - // helper only reads through that pointer. - unsafe { unicode_is_string_property(name.as_ptr(), name.len()) != 0 } -} - -pub(crate) fn property_all_case_equivalents_match(code_point: u32, name: &str, value: Option<&str>) -> bool { - let (value_ptr, value_len, has_value) = optional_value_parts(value); - // SAFETY: `name` and `value` remain valid for the duration of this call, - // and the C++ helper only reads through those pointers. - unsafe { - unicode_property_all_case_equivalents_match( - code_point, - name.as_ptr(), - name.len(), - value_ptr, - value_len, - has_value, - ) != 0 + Some(value) => (value.as_ptr(), value.len()), + None => (std::ptr::null(), 0), } } #[inline(always)] -pub(crate) fn resolved_property_matches(code_point: u32, property: ResolvedProperty) -> bool { - // SAFETY: This forwards only scalar values to the C++ helper. - unsafe { unicode_resolved_property_matches(code_point, property.kind as u8, property.id) != 0 } +fn is_ascii(code_point: u32) -> bool { + code_point < 128 } -pub(crate) fn is_valid_ecma262_property(name: &str, value: Option<&str>) -> bool { - let (value_ptr, value_len, has_value) = optional_value_parts(value); +#[inline(always)] +fn is_ascii_alpha(code_point: u32) -> bool { + is_ascii(code_point) && (code_point as u8).is_ascii_alphabetic() +} + +#[inline(always)] +fn is_ascii_digit(code_point: u32) -> bool { + code_point >= b'0' as u32 && code_point <= b'9' as u32 +} + +#[inline(always)] +fn is_ascii_alphanumeric(code_point: u32) -> bool { + is_ascii_alpha(code_point) || is_ascii_digit(code_point) +} + +pub fn property_matches(code_point: u32, name: &str, value: Option<&str>) -> bool { + let (value_ptr, value_len) = optional_value_parts(value); + // SAFETY: `name` and `value` remain valid for the duration of this call, // and the C++ helper only reads through those pointers. - unsafe { unicode_is_valid_ecma262_property(name.as_ptr(), name.len(), value_ptr, value_len, has_value) != 0 } + unsafe { unicode_property_matches(code_point, name.as_ptr(), name.len(), value_ptr, value_len) } } -pub(crate) fn get_string_property_data(name: &str) -> Vec { - // SAFETY: Passing a null output buffer with zero capacity is the API's - // documented query mode, and `name` remains valid during the call. - let needed = unsafe { unicode_get_string_property_data(name.as_ptr(), name.len(), std::ptr::null_mut(), 0) }; - if needed == 0 { - return Vec::new(); - } +pub fn property_matches_case_insensitive(code_point: u32, name: &str, value: Option<&str>) -> bool { + let (value_ptr, value_len) = optional_value_parts(value); - let mut buffer = vec![0u32; needed as usize]; - // SAFETY: `buffer` has room for `needed` elements and the C++ helper - // writes at most that many u32 values. - let written = unsafe { unicode_get_string_property_data(name.as_ptr(), name.len(), buffer.as_mut_ptr(), needed) }; - if written == 0 || written > needed { - return Vec::new(); - } - buffer.truncate(written as usize); - buffer + // SAFETY: `name` and `value` remain valid for the duration of this call, + // and the C++ helper only reads through those pointers. + unsafe { unicode_property_matches_case_insensitive(code_point, name.as_ptr(), name.len(), value_ptr, value_len) } } -pub(crate) fn resolve_property(name: &str, value: Option<&str>) -> Option { - let (value_ptr, value_len, has_value) = optional_value_parts(value); +pub fn property_all_case_equivalents_match(code_point: u32, name: &str, value: Option<&str>) -> bool { + let (value_ptr, value_len) = optional_value_parts(value); + + // SAFETY: `name` and `value` remain valid for the duration of this call, + // and the C++ helper only reads through those pointers. + unsafe { unicode_property_all_case_equivalents_match(code_point, name.as_ptr(), name.len(), value_ptr, value_len) } +} + +pub fn resolve_property(name: &str, value: Option<&str>) -> Option { + let (value_ptr, value_len) = optional_value_parts(value); let mut kind: u8 = 0; let mut id: u32 = 0; + // SAFETY: `name` and `value` remain valid for the duration of this call, // and `kind`/`id` are valid out-pointers to local storage. let ok = unsafe { @@ -186,12 +154,12 @@ pub(crate) fn resolve_property(name: &str, value: Option<&str>) -> Option) -> Option bool { - // SAFETY: This forwards only a scalar value to the C++ helper. - unsafe { unicode_is_id_start(code_point) != 0 } +pub fn resolved_property_matches(code_point: u32, property: ResolvedProperty) -> bool { + // SAFETY: This forwards only scalar values to the C++ helper. + unsafe { unicode_resolved_property_matches(code_point, property.kind as u8, property.id) } } #[inline(always)] -pub(crate) fn is_id_continue(code_point: u32) -> bool { +pub fn code_point_has_identifier_start_property(code_point: u32) -> bool { + if is_ascii(code_point) { + return is_ascii_alpha(code_point); + } + // SAFETY: This forwards only a scalar value to the C++ helper. - unsafe { unicode_is_id_continue(code_point) != 0 } + unsafe { unicode_code_point_has_identifier_start_property(code_point) } +} + +#[inline(always)] +pub fn code_point_has_identifier_continue_property(code_point: u32) -> bool { + if is_ascii(code_point) { + return is_ascii_alphanumeric(code_point) || code_point == '_' as u32; + } + + // SAFETY: This forwards only a scalar value to the C++ helper. + unsafe { unicode_code_point_has_identifier_continue_property(code_point) } +} + +pub fn is_string_property(name: &str) -> bool { + // SAFETY: `name` remains valid for the duration of this call and the C++ + // helper only reads through that pointer. + unsafe { unicode_is_string_property(name.as_ptr(), name.len()) } +} + +pub fn is_valid_ecma262_property(name: &str, value: Option<&str>) -> bool { + let (value_ptr, value_len) = optional_value_parts(value); + // SAFETY: `name` and `value` remain valid for the duration of this call, + // and the C++ helper only reads through those pointers. + unsafe { unicode_is_valid_ecma262_property(name.as_ptr(), name.len(), value_ptr, value_len) } +} + +pub fn get_string_property_data(name: &str) -> Vec { + // SAFETY: Passing a null output buffer with zero capacity is the API's + // documented query mode, and `name` remains valid during the call. + let needed = unsafe { unicode_get_string_property_data(name.as_ptr(), name.len(), std::ptr::null_mut(), 0) }; + if needed == 0 { + return Vec::new(); + } + + let mut buffer = vec![0u32; needed as usize]; + + // SAFETY: `buffer` has room for `needed` elements and the C++ helper + // writes at most that many u32 values. + let written = unsafe { unicode_get_string_property_data(name.as_ptr(), name.len(), buffer.as_mut_ptr(), needed) }; + if written == 0 || written > needed { + return Vec::new(); + } + + buffer.truncate(written as usize); + buffer +} + +#[inline(always)] +pub fn simple_case_fold(code_point: u32, unicode_mode: bool) -> u32 { + // SAFETY: This forwards only scalar values to the C++ helper. + unsafe { unicode_simple_case_fold(code_point, unicode_mode) } +} + +#[inline(always)] +pub fn code_point_matches_range_ignoring_case(code_point: u32, from: u32, to: u32, unicode_mode: bool) -> bool { + // SAFETY: This forwards only scalar values to the C++ helper. + unsafe { unicode_code_point_matches_range_ignoring_case(code_point, from, to, unicode_mode) } +} + +pub fn get_case_closure(code_point: u32, out_buffer: &mut [u32]) -> usize { + let capacity = out_buffer.len().min(u32::MAX as usize) as u32; + + // SAFETY: `out_buffer` is writable for `capacity` elements and the C++ + // helper writes at most that many entries. + unsafe { unicode_get_case_closure(code_point, out_buffer.as_mut_ptr(), capacity) as usize } } diff --git a/Libraries/LibUnicode/Rust/src/lib.rs b/Libraries/LibUnicode/Rust/src/lib.rs index 976d0629da..9c9ebfe57a 100644 --- a/Libraries/LibUnicode/Rust/src/lib.rs +++ b/Libraries/LibUnicode/Rust/src/lib.rs @@ -9,3 +9,4 @@ mod rust_allocator; pub mod calendar; +pub mod character_types;