From 7025dd1fa76f0c8d71ca70ebf8c3790aa117dc4f Mon Sep 17 00:00:00 2001 From: Andreas Kling Date: Sun, 21 Jun 2026 19:03:06 +0200 Subject: [PATCH] Libraries: Parse JS strings from UTF-16 Thread UTF-16 string input through JSON, script parsing, Date parsing, Intl option parsing, Temporal parsing, and the helper library boundaries that feed those parsers. Preserve ASCII fast paths where the source data is known to be ASCII. --- AK/Base64.cpp | 82 ++++-- AK/Base64.h | 4 + AK/Utf16FlyString.h | 2 +- .../LibCrypto/BigInt/SignedBigInteger.cpp | 27 ++ Libraries/LibCrypto/BigInt/SignedBigInteger.h | 3 +- .../LibCrypto/BigInt/UnsignedBigInteger.cpp | 27 ++ .../LibCrypto/BigInt/UnsignedBigInteger.h | 4 +- Libraries/LibJS/Bytecode/PropertyAccess.h | 2 +- Libraries/LibJS/Console.cpp | 90 ++++--- Libraries/LibJS/Console.h | 21 +- Libraries/LibJS/Contrib/Test262/262Object.cpp | 2 +- Libraries/LibJS/Print.cpp | 39 +-- .../LibJS/Runtime/AbstractOperations.cpp | 6 +- Libraries/LibJS/Runtime/Date.cpp | 18 +- Libraries/LibJS/Runtime/Date.h | 7 +- Libraries/LibJS/Runtime/DateConstructor.cpp | 8 +- Libraries/LibJS/Runtime/DateParser.h | 19 +- Libraries/LibJS/Runtime/DatePrototype.cpp | 19 +- Libraries/LibJS/Runtime/DatePrototype.h | 5 +- .../LibJS/Runtime/FunctionConstructor.cpp | 14 +- .../LibJS/Runtime/Intl/AbstractOperations.cpp | 97 +++++-- .../LibJS/Runtime/Intl/AbstractOperations.h | 6 +- Libraries/LibJS/Runtime/Intl/Collator.h | 3 +- .../Runtime/Intl/CollatorCompareFunction.cpp | 1 - .../Runtime/Intl/CollatorConstructor.cpp | 4 +- Libraries/LibJS/Runtime/Intl/DateTimeFormat.h | 5 +- .../Intl/DateTimeFormatConstructor.cpp | 24 +- Libraries/LibJS/Runtime/Intl/DisplayNames.cpp | 33 ++- Libraries/LibJS/Runtime/Intl/DisplayNames.h | 13 +- .../Runtime/Intl/DisplayNamesConstructor.cpp | 8 +- .../Runtime/Intl/DisplayNamesPrototype.cpp | 4 +- .../LibJS/Runtime/Intl/DurationFormat.cpp | 15 +- Libraries/LibJS/Runtime/Intl/DurationFormat.h | 9 +- .../Intl/DurationFormatConstructor.cpp | 2 +- Libraries/LibJS/Runtime/Intl/ListFormat.h | 3 + .../Runtime/Intl/ListFormatConstructor.cpp | 4 +- .../LibJS/Runtime/Intl/LocaleConstructor.cpp | 79 ++++-- .../LibJS/Runtime/Intl/MathematicalValue.cpp | 8 +- .../LibJS/Runtime/Intl/MathematicalValue.h | 9 +- Libraries/LibJS/Runtime/Intl/NumberFormat.cpp | 8 +- Libraries/LibJS/Runtime/Intl/NumberFormat.h | 10 + .../Runtime/Intl/NumberFormatConstructor.cpp | 42 ++- Libraries/LibJS/Runtime/Intl/PluralRules.h | 2 + .../Runtime/Intl/PluralRulesConstructor.cpp | 6 +- .../LibJS/Runtime/Intl/RelativeTimeFormat.cpp | 8 +- .../LibJS/Runtime/Intl/RelativeTimeFormat.h | 11 +- .../Intl/RelativeTimeFormatConstructor.cpp | 4 +- .../Intl/RelativeTimeFormatPrototype.cpp | 8 +- Libraries/LibJS/Runtime/Intl/Segmenter.h | 2 + .../Runtime/Intl/SegmenterConstructor.cpp | 2 +- Libraries/LibJS/Runtime/JSONObject.cpp | 40 +-- Libraries/LibJS/Runtime/JSONObject.h | 2 +- Libraries/LibJS/Runtime/ObjectPrototype.cpp | 38 +-- Libraries/LibJS/Runtime/PropertyDescriptor.h | 20 +- Libraries/LibJS/Runtime/PropertyKey.h | 2 +- Libraries/LibJS/Runtime/RegExpConstructor.cpp | 4 +- Libraries/LibJS/Runtime/StringIterator.cpp | 11 +- Libraries/LibJS/Runtime/StringIterator.h | 11 +- Libraries/LibJS/Runtime/StringPrototype.cpp | 50 ++-- .../Runtime/Temporal/AbstractOperations.cpp | 48 ++-- .../Runtime/Temporal/AbstractOperations.h | 4 +- Libraries/LibJS/Runtime/Temporal/Calendar.cpp | 35 ++- Libraries/LibJS/Runtime/Temporal/Calendar.h | 10 +- Libraries/LibJS/Runtime/Temporal/ISORecords.h | 7 +- .../LibJS/Runtime/Temporal/PlainDate.cpp | 7 +- .../LibJS/Runtime/Temporal/PlainDateTime.cpp | 7 +- .../LibJS/Runtime/Temporal/PlainMonthDay.cpp | 7 +- .../LibJS/Runtime/Temporal/PlainYearMonth.cpp | 7 +- Libraries/LibJS/Runtime/Temporal/TimeZone.cpp | 25 +- .../LibJS/Runtime/Temporal/ZonedDateTime.cpp | 15 +- .../Temporal/ZonedDateTimePrototype.cpp | 6 +- Libraries/LibJS/Runtime/Uint8Array.cpp | 6 +- Libraries/LibJS/Runtime/Uint8Array.h | 2 +- Libraries/LibJS/Runtime/VM.cpp | 4 +- Libraries/LibJS/Runtime/VM.h | 4 +- Libraries/LibJS/Runtime/Value.cpp | 39 +-- Libraries/LibJS/Runtime/Value.h | 6 +- Libraries/LibJS/Rust/src/ast_dump.rs | 2 +- Libraries/LibJS/RustIntegration.cpp | 21 +- Libraries/LibJS/RustIntegration.h | 5 +- Libraries/LibJS/Script.cpp | 2 +- Libraries/LibJS/Script.h | 3 +- Libraries/LibJS/SyntheticModule.cpp | 5 +- Libraries/LibTest/JavaScriptTestRunner.h | 18 +- Libraries/LibUnicode/Calendar.cpp | 53 ++++ Libraries/LibUnicode/Calendar.h | 2 + Libraries/LibUnicode/Collator.cpp | 4 +- Libraries/LibUnicode/Collator.h | 5 +- Libraries/LibUnicode/DateTimeFormat.cpp | 4 +- Libraries/LibUnicode/DateTimeFormat.h | 5 +- Libraries/LibUnicode/DisplayNames.cpp | 2 +- Libraries/LibUnicode/DisplayNames.h | 3 +- Libraries/LibUnicode/ListFormat.cpp | 11 + Libraries/LibUnicode/ListFormat.h | 2 + Libraries/LibUnicode/Locale.cpp | 251 +++++++++++++----- Libraries/LibUnicode/Locale.h | 42 +++ Libraries/LibUnicode/Normalize.cpp | 64 +++-- Libraries/LibUnicode/Normalize.h | 4 + Libraries/LibUnicode/NumberFormat.cpp | 137 +++++++++- Libraries/LibUnicode/NumberFormat.h | 11 +- Libraries/LibUnicode/PluralRules.cpp | 9 + Libraries/LibUnicode/PluralRules.h | 1 + Libraries/LibUnicode/RelativeTimeFormat.cpp | 30 +++ Libraries/LibUnicode/RelativeTimeFormat.h | 3 + Libraries/LibUnicode/Segmenter.cpp | 13 + Libraries/LibUnicode/Segmenter.h | 2 + Libraries/LibWeb/Bindings/MainThreadVM.cpp | 4 +- .../BlockingAlgorithms.cpp | 8 +- .../BlockingAlgorithms.h | 2 +- Libraries/LibWeb/DOM/EventTarget.cpp | 17 +- .../LibWeb/Geometry/DOMMatrixReadOnly.cpp | 12 +- Libraries/LibWeb/HTML/ErrorInformation.cpp | 2 +- .../LibWeb/HTML/Scripting/ClassicScript.cpp | 3 +- .../HTML/Scripting/ExceptionReporter.cpp | 9 +- .../LibWeb/HTML/WindowOrWorkerGlobalScope.cpp | 3 +- Libraries/LibWeb/WebAssembly/WebAssembly.cpp | 4 +- Libraries/LibWeb/WebDriver/ExecuteScript.cpp | 7 +- Libraries/LibWeb/WebIDL/ExceptionOr.h | 4 +- Libraries/LibWeb/WebIDL/Tracing.cpp | 12 +- Meta/Fuzzers/FuzzJs.cpp | 4 +- Meta/Fuzzers/FuzzilliJs.cpp | 4 +- Services/WebContent/DevToolsConsoleClient.cpp | 2 +- Services/WebWorker/WorkerHost.cpp | 3 +- Tests/AK/TestBase64.cpp | 23 ++ Tests/LibCrypto/TestBigInteger.cpp | 11 + .../builtins/Intl/Intl.getCanonicalLocales.js | 8 + Tests/LibJS/test-js.cpp | 8 +- Tests/LibUnicode/TestLocale.cpp | 12 + Utilities/js.cpp | 12 +- Utilities/test262-runner.cpp | 18 +- Utilities/wasm.cpp | 16 +- 131 files changed, 1498 insertions(+), 670 deletions(-) diff --git a/AK/Base64.cpp b/AK/Base64.cpp index ba1847c221..f4a21a57a8 100644 --- a/AK/Base64.cpp +++ b/AK/Base64.cpp @@ -16,6 +16,13 @@ size_t size_required_to_decode_base64(StringView input) return simdutf::maximal_binary_length_from_base64(input.characters_without_null_termination(), input.length()); } +size_t size_required_to_decode_base64(Utf16View input) +{ + if (input.has_ascii_storage()) + return size_required_to_decode_base64(StringView { input.bytes() }); + return simdutf::maximal_binary_length_from_base64(input.utf16_span().data(), input.length_in_code_units()); +} + static constexpr simdutf::last_chunk_handling_options to_simdutf_last_chunk_handling(LastChunkHandling last_chunk_handling) { switch (last_chunk_handling) { @@ -30,6 +37,29 @@ static constexpr simdutf::last_chunk_handling_options to_simdutf_last_chunk_hand VERIFY_NOT_REACHED(); } +static Optional base64_error_from_result(simdutf::result result, ByteBuffer& output) +{ + if (result.error == simdutf::SUCCESS || result.error == simdutf::OUTPUT_BUFFER_TOO_SMALL) + return {}; + + output.resize((result.count / 4) * 3); + + auto error = [&]() { + switch (result.error) { + case simdutf::BASE64_EXTRA_BITS: + return Error::from_string_literal("Extra bits found at end of chunk"); + case simdutf::BASE64_INPUT_REMAINDER: + return Error::from_string_literal("Invalid trailing data"); + case simdutf::INVALID_BASE64_CHARACTER: + return Error::from_string_literal("Invalid base64 character"); + default: + return Error::from_string_literal("Invalid base64-encoded data"); + } + }(); + + return InvalidBase64 { .error = move(error), .valid_input_bytes = result.count }; +} + static ErrorOr decode_base64_into_impl(StringView input, ByteBuffer& output, LastChunkHandling last_chunk_handling, simdutf::base64_options options) { static constexpr auto decode_up_to_bad_character = true; @@ -44,24 +74,34 @@ static ErrorOr decode_base64_into_impl(StringView input, to_simdutf_last_chunk_handling(last_chunk_handling), decode_up_to_bad_character); - if (result.error != simdutf::SUCCESS && result.error != simdutf::OUTPUT_BUFFER_TOO_SMALL) { - output.resize((result.count / 4) * 3); + if (auto error = base64_error_from_result(result, output); error.has_value()) + return error.release_value(); - auto error = [&]() { - switch (result.error) { - case simdutf::BASE64_EXTRA_BITS: - return Error::from_string_literal("Extra bits found at end of chunk"); - case simdutf::BASE64_INPUT_REMAINDER: - return Error::from_string_literal("Invalid trailing data"); - case simdutf::INVALID_BASE64_CHARACTER: - return Error::from_string_literal("Invalid base64 character"); - default: - return Error::from_string_literal("Invalid base64-encoded data"); - } - }(); + VERIFY(output_length <= output.size()); + output.resize(output_length); - return InvalidBase64 { .error = move(error), .valid_input_bytes = result.count }; - } + return result.count; +} + +static ErrorOr decode_base64_into_impl(Utf16View input, ByteBuffer& output, LastChunkHandling last_chunk_handling, simdutf::base64_options options) +{ + if (input.has_ascii_storage()) + return decode_base64_into_impl(StringView { input.bytes() }, output, last_chunk_handling, options); + + static constexpr auto decode_up_to_bad_character = true; + auto output_length = output.size(); + + auto result = simdutf::base64_to_binary_safe( + input.utf16_span().data(), + input.length_in_code_units(), + reinterpret_cast(output.data()), + output_length, + options, + to_simdutf_last_chunk_handling(last_chunk_handling), + decode_up_to_bad_character); + + if (auto error = base64_error_from_result(result, output); error.has_value()) + return error.release_value(); VERIFY(output_length <= output.size()); output.resize(output_length); @@ -118,6 +158,16 @@ ErrorOr decode_base64url_into(StringView input, ByteBuffe return decode_base64_into_impl(input, output, last_chunk_handling, simdutf::base64_url); } +ErrorOr decode_base64_into(Utf16View input, ByteBuffer& output, LastChunkHandling last_chunk_handling) +{ + return decode_base64_into_impl(input, output, last_chunk_handling, simdutf::base64_default); +} + +ErrorOr decode_base64url_into(Utf16View input, ByteBuffer& output, LastChunkHandling last_chunk_handling) +{ + return decode_base64_into_impl(input, output, last_chunk_handling, simdutf::base64_url); +} + ErrorOr encode_base64(ReadonlyBytes input, OmitPadding omit_padding) { auto options = omit_padding == OmitPadding::Yes diff --git a/AK/Base64.h b/AK/Base64.h index 3c988e8390..63ef72a901 100644 --- a/AK/Base64.h +++ b/AK/Base64.h @@ -10,10 +10,12 @@ #include #include #include +#include namespace AK { size_t size_required_to_decode_base64(StringView); +size_t size_required_to_decode_base64(Utf16View); enum class LastChunkHandling { Loose, @@ -33,6 +35,8 @@ struct InvalidBase64 { // string length if the output buffer was not large enough. ErrorOr decode_base64_into(StringView, ByteBuffer&, LastChunkHandling = LastChunkHandling::Loose); ErrorOr decode_base64url_into(StringView, ByteBuffer&, LastChunkHandling = LastChunkHandling::Loose); +ErrorOr decode_base64_into(Utf16View, ByteBuffer&, LastChunkHandling = LastChunkHandling::Loose); +ErrorOr decode_base64url_into(Utf16View, ByteBuffer&, LastChunkHandling = LastChunkHandling::Loose); enum class OmitPadding { No, diff --git a/AK/Utf16FlyString.h b/AK/Utf16FlyString.h index 535d8473af..31c0bc594d 100644 --- a/AK/Utf16FlyString.h +++ b/AK/Utf16FlyString.h @@ -198,7 +198,7 @@ template<> struct Formatter : Formatter { ErrorOr format(FormatBuilder& builder, Utf16FlyString const& string) { - return Formatter::format(builder, string.to_utf16_string()); + return Formatter {}.format(builder, string.to_utf16_string()); } }; diff --git a/Libraries/LibCrypto/BigInt/SignedBigInteger.cpp b/Libraries/LibCrypto/BigInt/SignedBigInteger.cpp index 9770d86dee..9da6344245 100644 --- a/Libraries/LibCrypto/BigInt/SignedBigInteger.cpp +++ b/Libraries/LibCrypto/BigInt/SignedBigInteger.cpp @@ -127,6 +127,33 @@ ErrorOr SignedBigInteger::from_base(u16 N, StringView str) return result; } +ErrorOr SignedBigInteger::from_base(u16 N, Utf16View str) +{ + VERIFY(N <= 36); + if (str.is_empty()) + return SignedBigInteger(0); + + auto buffer = TRY(ByteBuffer::create_zeroed(str.length_in_code_units() + 1)); + + size_t idx = 0; + for (size_t i = 0; i < str.length_in_code_units(); ++i) { + auto code_unit = str.code_unit_at(i); + if (code_unit > NumericLimits::max()) + return Error::from_string_literal("Invalid number"); + if (code_unit == '_') { + // Skip underscores + continue; + } + + buffer[idx++] = static_cast(code_unit); + } + + SignedBigInteger result; + if (mp_read_radix(&result.m_mp, reinterpret_cast(buffer.data()), N) != MP_OKAY) + return Error::from_string_literal("Invalid number"); + return result; +} + ErrorOr SignedBigInteger::to_base(u16 N) const { VERIFY(N <= 36); diff --git a/Libraries/LibCrypto/BigInt/SignedBigInteger.h b/Libraries/LibCrypto/BigInt/SignedBigInteger.h index cdba078f2a..bf30198c65 100644 --- a/Libraries/LibCrypto/BigInt/SignedBigInteger.h +++ b/Libraries/LibCrypto/BigInt/SignedBigInteger.h @@ -42,6 +42,7 @@ public: [[nodiscard]] Bytes export_data(Bytes) const; [[nodiscard]] static ErrorOr from_base(u16 N, StringView str); + [[nodiscard]] static ErrorOr from_base(u16 N, Utf16View str); [[nodiscard]] ErrorOr to_base(u16 N) const; [[nodiscard]] i64 to_i64() const; @@ -117,5 +118,5 @@ struct AK::Formatter : AK::Formatter UnsignedBigInteger::from_base(u16 N, StringView str) return result; } +ErrorOr UnsignedBigInteger::from_base(u16 N, Utf16View str) +{ + VERIFY(N <= 36); + if (str.is_empty()) + return UnsignedBigInteger(0); + + auto buffer = TRY(ByteBuffer::create_zeroed(str.length_in_code_units() + 1)); + + size_t idx = 0; + for (size_t i = 0; i < str.length_in_code_units(); ++i) { + auto code_unit = str.code_unit_at(i); + if (code_unit > NumericLimits::max()) + return Error::from_string_literal("Invalid number"); + if (code_unit == '_') { + // Skip underscores + continue; + } + + buffer[idx++] = static_cast(code_unit); + } + + UnsignedBigInteger result; + if (mp_read_radix(&result.m_mp, reinterpret_cast(buffer.data()), N) != MP_OKAY) + return Error::from_string_literal("Invalid number"); + return result; +} + ErrorOr UnsignedBigInteger::to_base(u16 N) const { VERIFY(N <= 36); diff --git a/Libraries/LibCrypto/BigInt/UnsignedBigInteger.h b/Libraries/LibCrypto/BigInt/UnsignedBigInteger.h index ed59baaf9a..1d6ca5fb7d 100644 --- a/Libraries/LibCrypto/BigInt/UnsignedBigInteger.h +++ b/Libraries/LibCrypto/BigInt/UnsignedBigInteger.h @@ -10,6 +10,7 @@ #pragma once #include +#include #include namespace Crypto { @@ -44,6 +45,7 @@ public: [[nodiscard]] Bytes export_data(Bytes) const; [[nodiscard]] static ErrorOr from_base(u16 N, StringView str); + [[nodiscard]] static ErrorOr from_base(u16 N, Utf16View str); [[nodiscard]] ErrorOr to_base(u16 N) const; [[nodiscard]] size_t count_digits_in_base(u16 base) const; @@ -125,7 +127,7 @@ struct AK::Formatter : Formatter { inline Crypto::UnsignedBigInteger operator""_bigint(char const* string, size_t length) { - return MUST(Crypto::UnsignedBigInteger::from_base(10, { string, length })); + return MUST(Crypto::UnsignedBigInteger::from_base(10, StringView { string, length })); } inline Crypto::UnsignedBigInteger operator""_bigint(unsigned long long value) diff --git a/Libraries/LibJS/Bytecode/PropertyAccess.h b/Libraries/LibJS/Bytecode/PropertyAccess.h index 3e327fe988..9c846e3cba 100644 --- a/Libraries/LibJS/Bytecode/PropertyAccess.h +++ b/Libraries/LibJS/Bytecode/PropertyAccess.h @@ -401,7 +401,7 @@ inline ThrowCompletionOr put_by_property_key(VM& vm, Value base, Value thi if (!succeeded && strict == Strict::Yes) [[unlikely]] { if (base.is_object()) return vm.throw_completion(ErrorType::ReferenceNullishSetProperty, name, base); - return vm.throw_completion(ErrorType::ReferencePrimitiveSetProperty, name, base.typeof_(vm)->utf16_string_view().to_utf8_but_should_be_ported_to_utf16(), base); + return vm.throw_completion(ErrorType::ReferencePrimitiveSetProperty, name, base.typeof_(vm)->utf16_string_view(), base); } break; } diff --git a/Libraries/LibJS/Console.cpp b/Libraries/LibJS/Console.cpp index 684be9737e..c601b958fd 100644 --- a/Libraries/LibJS/Console.cpp +++ b/Libraries/LibJS/Console.cpp @@ -446,11 +446,11 @@ ThrowCompletionOr Console::dirxml() return js_undefined(); } -static ThrowCompletionOr label_or_fallback(VM& vm, StringView fallback) +static ThrowCompletionOr label_or_fallback(VM& vm, Utf16View fallback) { return vm.argument_count() > 0 && !vm.argument(0).is_undefined() - ? TRY(vm.argument(0).to_utf16_string(vm)).to_utf8_but_should_be_ported_to_utf16() - : TRY_OR_THROW_OOM(vm, String::from_utf8(fallback)); + ? TRY(vm.argument(0).to_utf16_string(vm)) + : Utf16String::from_utf16(fallback); } // 1.2.1. count(label), https://console.spec.whatwg.org/#count @@ -474,7 +474,7 @@ ThrowCompletionOr Console::count() } // 4. Let concat be the concatenation of label, U+003A (:), U+0020 SPACE, and ToString(map[label]). - auto concat = TRY_OR_THROW_OOM(vm, String::formatted("{}: {}", label, map.get(label).value())); + auto concat = Utf16String::formatted("{}: {}", label, map.get(label).value()); // 5. Perform Logger("count", « concat »). GC::RootVector concat_as_vector; @@ -503,7 +503,7 @@ ThrowCompletionOr Console::count_reset() else { // 1. Let message be a string without any formatting specifiers indicating generically // that the given label does not have an associated count. - auto message = TRY_OR_THROW_OOM(vm, String::formatted("\"{}\" doesn't have a count", label)); + auto message = Utf16String::formatted("\"{}\" doesn't have a count", label); // 2. Perform Logger("countReset", « message »); GC::RootVector message_as_vector; message_as_vector.append(PrimitiveString::create(vm, move(message))); @@ -521,7 +521,7 @@ ThrowCompletionOr Console::group() Group group; // 2. If data is not empty, let groupLabel be the result of Formatter(data). - String group_label {}; + Utf16String group_label {}; auto data = vm_arguments(); if (!data.is_empty()) { if (m_client) { @@ -533,7 +533,7 @@ ThrowCompletionOr Console::group() } // ... Otherwise, let groupLabel be an implementation-chosen label representing a group. else { - group_label = "Group"_string; + group_label = "Group"_utf16; } // 3. Incorporate groupLabel as a label for group. @@ -559,7 +559,7 @@ ThrowCompletionOr Console::group_collapsed() Group group; // 2. If data is not empty, let groupLabel be the result of Formatter(data). - String group_label {}; + Utf16String group_label {}; auto data = vm_arguments(); if (!data.is_empty()) { if (m_client) { @@ -571,7 +571,7 @@ ThrowCompletionOr Console::group_collapsed() } // ... Otherwise, let groupLabel be an implementation-chosen label representing a group. else { - group_label = "Group"_string; + group_label = "Group"_utf16; } // 3. Incorporate groupLabel as a label for group. @@ -618,7 +618,7 @@ ThrowCompletionOr Console::time() if (m_client) { GC::RootVector timer_already_exists_warning_message_as_vector; - auto message = TRY_OR_THROW_OOM(vm, String::formatted("Timer '{}' already exists.", label)); + auto message = Utf16String::formatted("Timer '{}' already exists.", label); timer_already_exists_warning_message_as_vector.append(PrimitiveString::create(vm, move(message))); TRY(m_client->printer(LogLevel::Warn, move(timer_already_exists_warning_message_as_vector))); @@ -649,7 +649,7 @@ ThrowCompletionOr Console::time_log() if (m_client) { GC::RootVector timer_does_not_exist_warning_message_as_vector; - auto message = TRY_OR_THROW_OOM(vm, String::formatted("Timer '{}' does not exist.", label)); + auto message = Utf16String::formatted("Timer '{}' does not exist.", label); timer_does_not_exist_warning_message_as_vector.append(PrimitiveString::create(vm, move(message))); TRY(m_client->printer(LogLevel::Warn, move(timer_does_not_exist_warning_message_as_vector))); @@ -662,7 +662,7 @@ ThrowCompletionOr Console::time_log() auto duration = AK::human_readable_time(start_time.elapsed_time()); // 4. Let concat be the concatenation of label, U+003A (:), U+0020 SPACE, and duration. - auto concat = TRY_OR_THROW_OOM(vm, String::formatted("{}: {}", label, duration)); + auto concat = Utf16String::formatted("{}: {}", label, duration); // 5. Prepend concat to data. GC::RootVector data; @@ -695,7 +695,7 @@ ThrowCompletionOr Console::time_end() if (m_client) { GC::RootVector timer_does_not_exist_warning_message_as_vector; - auto message = TRY_OR_THROW_OOM(vm, String::formatted("Timer '{}' does not exist.", label)); + auto message = Utf16String::formatted("Timer '{}' does not exist.", label); timer_does_not_exist_warning_message_as_vector.append(PrimitiveString::create(vm, move(message))); TRY(m_client->printer(LogLevel::Warn, move(timer_does_not_exist_warning_message_as_vector))); @@ -711,7 +711,7 @@ ThrowCompletionOr Console::time_end() auto duration = AK::human_readable_time(start_time.elapsed_time()); // 5. Let concat be the concatenation of label, U+003A (:), U+0020 SPACE, and duration. - auto concat = TRY_OR_THROW_OOM(vm, String::formatted("{}: {}", label, duration)); + auto concat = Utf16String::formatted("{}: {}", label, duration); // 6. Perform Printer("timeEnd", « concat »). if (m_client) { @@ -758,25 +758,49 @@ void Console::output_debug_message(LogLevel log_level, StringView output) const } } +void Console::output_debug_message(LogLevel log_level, Utf16View output) const +{ + switch (log_level) { + case Console::LogLevel::Debug: + dbgln("\033[32;1m(js debug)\033[0m {}", output); + break; + case Console::LogLevel::Error: + dbgln("\033[32;1m(js error)\033[0m {}", output); + break; + case Console::LogLevel::Info: + dbgln("\033[32;1m(js info)\033[0m {}", output); + break; + case Console::LogLevel::Log: + dbgln("\033[32;1m(js log)\033[0m {}", output); + break; + case Console::LogLevel::Warn: + dbgln("\033[32;1m(js warn)\033[0m {}", output); + break; + default: + dbgln("\033[32;1m(js)\033[0m {}", output); + break; + } +} + void Console::report_exception(String const& name, String const& message, JS::ErrorData const& error_data, bool in_promise) const { if (m_client) m_client->report_exception(name, message, error_data, in_promise); } -ThrowCompletionOr Console::value_vector_to_string(GC::RootVector const& values) +ThrowCompletionOr Console::value_vector_to_string(GC::RootVector const& values) { auto& vm = realm().vm(); - StringBuilder builder; + Utf16StringBuilder builder; for (auto const& item : values) { if (!builder.is_empty()) - builder.append(' '); + builder.append_ascii(' '); builder.append(TRY(item.to_utf16_string(vm))); } - return MUST(builder.to_string()); + return builder.to_string(); } ConsoleClient::ConsoleClient(Console& console) @@ -832,24 +856,24 @@ ThrowCompletionOr> ConsoleClient::formatter(GC::RootVector return args; // 2. Let target be the first element of args. - auto target = (!args.is_empty()) ? TRY(args.first().to_utf16_string(vm)).to_utf8_but_should_be_ported_to_utf16() : String {}; + auto target = (!args.is_empty()) ? TRY(args.first().to_utf16_string(vm)) : Utf16String {}; // 3. Let current be the second element of args. auto current = (args.size() > 1) ? args[1] : js_undefined(); // 4. Find the first possible format specifier specifier, from the left to the right in target. - auto find_specifier = [](StringView target) -> Optional { + auto find_specifier = [](Utf16View target) -> Optional { size_t start_index = 0; - while (start_index < target.length()) { - auto maybe_index = target.find('%', start_index); + while (start_index < target.length_in_code_units()) { + auto maybe_index = target.find_code_unit_offset('%', start_index); if (!maybe_index.has_value()) return {}; auto index = maybe_index.value(); - if (index + 1 >= target.length()) + if (index + 1 >= target.length_in_code_units()) return {}; - switch (target[index + 1]) { + switch (target.code_unit_at(index + 1)) { case 'c': case 'd': case 'f': @@ -864,7 +888,7 @@ ThrowCompletionOr> ConsoleClient::formatter(GC::RootVector } return {}; }; - auto maybe_specifier = find_specifier(target); + auto maybe_specifier = find_specifier(target.utf16_view()); // 5. If no format specifier was found, return args. if (!maybe_specifier.has_value()) { @@ -914,13 +938,16 @@ ThrowCompletionOr> ConsoleClient::formatter(GC::RootVector // 6. TODO: process %c else if (specifier == "%c"sv) { // NOTE: This has no spec yet. `%c` specifiers treat the argument as CSS styling for the log message. - add_css_style_to_current_message(TRY(current.to_utf16_string(vm)).to_utf8_but_should_be_ported_to_utf16()); - converted = PrimitiveString::create(vm, String {}); + auto css_style = TRY(current.to_utf16_string(vm)); + add_css_style_to_current_message(css_style.utf16_view()); + converted = PrimitiveString::create(vm, Utf16String {}); } // 7. If any of the previous steps set converted, replace specifier in target with converted. - if (converted.has_value()) - target = TRY_OR_THROW_OOM(vm, target.replace(specifier, TRY(converted->to_utf16_string(vm)).to_utf8_but_should_be_ported_to_utf16(), ReplaceMode::FirstOnly)); + if (converted.has_value()) { + auto converted_string = TRY(converted->to_utf16_string(vm)); + target = target.replace(specifier, converted_string.utf16_view(), ReplaceMode::FirstOnly); + } } // 7. Let result be a list containing target together with the elements of args starting from the third onward. @@ -934,7 +961,7 @@ ThrowCompletionOr> ConsoleClient::formatter(GC::RootVector return formatter(result); } -ThrowCompletionOr ConsoleClient::generically_format_values(GC::RootVector const& values) +ThrowCompletionOr ConsoleClient::generically_format_values(GC::RootVector const& values) { AllocatingMemoryStream stream; auto& vm = m_console->realm().vm(); @@ -947,7 +974,8 @@ ThrowCompletionOr ConsoleClient::generically_format_values(GC::RootVecto first = false; } // FIXME: Is it possible we could end up serializing objects to invalid UTF-8? - return TRY_OR_THROW_OOM(vm, String::from_stream(stream, stream.used_buffer_size())); + auto output = TRY_OR_THROW_OOM(vm, String::from_stream(stream, stream.used_buffer_size())); + return Utf16String::from_utf8(output); } } diff --git a/Libraries/LibJS/Console.h b/Libraries/LibJS/Console.h index 92e10d0511..4e9dec68c0 100644 --- a/Libraries/LibJS/Console.h +++ b/Libraries/LibJS/Console.h @@ -11,6 +11,8 @@ #include #include #include +#include +#include #include #include #include @@ -52,7 +54,7 @@ public: }; struct Group { - String label; + Utf16String label; }; struct TraceFrame { @@ -63,7 +65,7 @@ public: }; struct Trace { - String label; + Utf16String label; Vector stack; }; @@ -73,8 +75,8 @@ public: GC::RootVector vm_arguments(); - HashMap& counters() { return m_counters; } - HashMap const& counters() const { return m_counters; } + HashMap& counters() { return m_counters; } + HashMap const& counters() const { return m_counters; } ThrowCompletionOr assert_(); Value clear(); @@ -97,6 +99,7 @@ public: ThrowCompletionOr time_end(); void output_debug_message(LogLevel log_level, StringView output) const; + void output_debug_message(LogLevel log_level, Utf16View output) const; void report_exception(String const& name, String const& message, JS::ErrorData const&, bool) const; private: @@ -104,13 +107,13 @@ private: virtual void visit_edges(Visitor&) override; - ThrowCompletionOr value_vector_to_string(GC::RootVector const&); + ThrowCompletionOr value_vector_to_string(GC::RootVector const&); GC::Ref m_realm; GC::Ptr m_client; - HashMap m_counters; - HashMap m_timer_table; + HashMap m_counters; + HashMap m_timer_table; Vector m_group_stack; }; @@ -125,13 +128,13 @@ public: ThrowCompletionOr> formatter(GC::RootVector const& args); virtual ThrowCompletionOr printer(Console::LogLevel log_level, PrinterArguments) = 0; - virtual void add_css_style_to_current_message(StringView) { } + virtual void add_css_style_to_current_message(Utf16View) { } virtual void report_exception(String const&, String const&, JS::ErrorData const&, bool) { } virtual void clear() = 0; virtual void end_group() = 0; - ThrowCompletionOr generically_format_values(GC::RootVector const&); + ThrowCompletionOr generically_format_values(GC::RootVector const&); protected: explicit ConsoleClient(Console&); diff --git a/Libraries/LibJS/Contrib/Test262/262Object.cpp b/Libraries/LibJS/Contrib/Test262/262Object.cpp index 3e6c3ca1bb..c7cf4e87cf 100644 --- a/Libraries/LibJS/Contrib/Test262/262Object.cpp +++ b/Libraries/LibJS/Contrib/Test262/262Object.cpp @@ -100,7 +100,7 @@ JS_DEFINE_NATIVE_FUNCTION($262Object::eval_script) auto& realm = *vm.current_realm(); // 3. Let s be ParseScript(sourceText, realm, hostDefined). - auto script_or_error = Script::parse(source_text.to_utf8_but_should_be_ported_to_utf16(), realm); + auto script_or_error = Script::parse(source_text.utf16_view(), realm); // 4. If s is a List of errors, then if (script_or_error.is_error()) { diff --git a/Libraries/LibJS/Print.cpp b/Libraries/LibJS/Print.cpp index 76e1a81c8d..e5e5c317a5 100644 --- a/Libraries/LibJS/Print.cpp +++ b/Libraries/LibJS/Print.cpp @@ -8,6 +8,7 @@ #include #include +#include #include #include #include @@ -62,31 +63,32 @@ namespace { -static ErrorOr escape_for_string_literal(StringView string) +static ErrorOr escape_for_string_literal(Utf16View string) { - StringBuilder builder; - for (auto byte : string.bytes()) { - switch (byte) { + Utf16StringBuilder builder; + for (size_t i = 0; i < string.length_in_code_units(); ++i) { + auto code_unit = string.code_unit_at(i); + switch (code_unit) { case '\r': - TRY(builder.try_append("\\r"sv)); + builder.append_ascii("\\r"sv); continue; case '\v': - TRY(builder.try_append("\\v"sv)); + builder.append_ascii("\\v"sv); continue; case '\f': - TRY(builder.try_append("\\f"sv)); + builder.append_ascii("\\f"sv); continue; case '\b': - TRY(builder.try_append("\\b"sv)); + builder.append_ascii("\\b"sv); continue; case '\n': - TRY(builder.try_append("\\n"sv)); + builder.append_ascii("\\n"sv); continue; case '\\': - TRY(builder.try_append("\\\\"sv)); + builder.append_ascii("\\\\"sv); continue; default: - TRY(builder.try_append(byte)); + builder.append_code_unit(code_unit); continue; } } @@ -151,6 +153,11 @@ ErrorOr print_type(JS::PrintContext& print_context, StringView name) return js_out(print_context, "[\033[36;1m{}\033[0m]", name); } +ErrorOr print_type(JS::PrintContext& print_context, Utf16View name) +{ + return js_out(print_context, "[\033[36;1m{}\033[0m]", name); +} + ErrorOr print_separator(JS::PrintContext& print_context, bool& first) { TRY(js_out(print_context, "{}", first ? " "sv : ", "sv)); @@ -278,9 +285,9 @@ ErrorOr print_error(JS::PrintContext& print_context, JS::Object const& obj if (name.is_accessor() || message.is_accessor()) { TRY(print_value(print_context, &object, seen_objects)); } else { - auto name_string = name.to_string_without_side_effects(); - auto message_string = message.to_string_without_side_effects(); - TRY(print_type(print_context, name_string)); + auto name_string = name.to_utf16_string_without_side_effects(); + auto message_string = message.to_utf16_string_without_side_effects(); + TRY(print_type(print_context, name_string.utf16_view())); if (!message_string.is_empty()) TRY(js_out(print_context, " \033[31;1m{}\033[0m", message_string)); } @@ -1033,9 +1040,9 @@ ErrorOr print_value(JS::PrintContext& print_context, JS::Value value, GC:: else if (value.is_negative_zero()) TRY(js_out(print_context, "-")); - auto contents = value.to_string_without_side_effects(); + auto contents = value.to_utf16_string_without_side_effects(); if (value.is_string() && !print_context.raw_strings) - TRY(js_out(print_context, "{}", TRY(escape_for_string_literal(contents)))); + TRY(js_out(print_context, "{}", TRY(escape_for_string_literal(contents.utf16_view())))); else TRY(js_out(print_context, "{}", contents)); diff --git a/Libraries/LibJS/Runtime/AbstractOperations.cpp b/Libraries/LibJS/Runtime/AbstractOperations.cpp index e37ecaadb2..b6186fde88 100644 --- a/Libraries/LibJS/Runtime/AbstractOperations.cpp +++ b/Libraries/LibJS/Runtime/AbstractOperations.cpp @@ -646,8 +646,8 @@ ThrowCompletionOr perform_eval(VM& vm, Value x, CallerMode strict_caller, // 6. NOTE: In the case of a direct eval, evalRealm is the realm of both the caller of eval and of the eval function itself. // 7. Perform ? HostEnsureCanCompileStrings(evalRealm, « », xStr, xStr, direct, « », x). - auto code_string_utf8 = code_string->utf16_string_view().to_utf8_but_should_be_ported_to_utf16(); - TRY(vm.host_ensure_can_compile_strings(eval_realm, {}, code_string_utf8, code_string_utf8, direct == EvalMode::Direct ? CompilationType::DirectEval : CompilationType::IndirectEval, {}, x)); + auto code_string_view = code_string->utf16_string_view(); + TRY(vm.host_ensure_can_compile_strings(eval_realm, {}, code_string_view, code_string_view, direct == EvalMode::Direct ? CompilationType::DirectEval : CompilationType::IndirectEval, {}, x)); // 8. Let inFunction be false. bool in_function = false; @@ -1959,7 +1959,7 @@ ThrowCompletionOr get_option(VM& vm, Object const& options, PropertyKey c auto value_string = value.as_string().utf16_string_view(); auto it = find_if(values.begin(), values.end(), [&](auto allowed_value) { return value_string == allowed_value; }); if (it == values.end()) - return vm.throw_completion(ErrorType::OptionIsNotValidValue, value_string.to_utf8_but_should_be_ported_to_utf16(), property.as_string()); + return vm.throw_completion(ErrorType::OptionIsNotValidValue, value_string, property.as_string()); value = PrimitiveString::create(vm, *it); } diff --git a/Libraries/LibJS/Runtime/Date.cpp b/Libraries/LibJS/Runtime/Date.cpp index 811391ab7b..a462c53e7e 100644 --- a/Libraries/LibJS/Runtime/Date.cpp +++ b/Libraries/LibJS/Runtime/Date.cpp @@ -457,7 +457,8 @@ String system_time_zone_identifier() // time zone identifier or an offset time zone identifier. auto system_time_zone_string = Unicode::current_time_zone(); - if (!is_offset_time_zone_identifier(system_time_zone_string)) { + auto utf16_system_time_zone_string = Utf16String::from_utf8(system_time_zone_string); + if (!is_offset_time_zone_identifier(utf16_system_time_zone_string)) { auto time_zone_identifier = Intl::get_available_named_time_zone_identifier(system_time_zone_string); if (!time_zone_identifier.has_value()) return "UTC"_string; @@ -662,11 +663,10 @@ double time_clip(double time) // 21.4.1.33.1 IsTimeZoneOffsetString ( offsetString ), https://tc39.es/ecma262/#sec-istimezoneoffsetstring // 14.5.10 IsOffsetTimeZoneIdentifier ( offsetString ), https://tc39.es/proposal-temporal/#sec-isoffsettimezoneidentifier -bool is_offset_time_zone_identifier(StringView offset_string) +bool is_offset_time_zone_identifier(Utf16View offset_string) { // 1. Let parseResult be ParseText(StringToCodePoints(offsetString), UTCOffset[~SubMinutePrecision]). - auto utf16_offset_string = Utf16String::from_utf8(offset_string); - auto parse_result = Temporal::parse_utc_offset(utf16_offset_string, Temporal::SubMinutePrecision::No); + auto parse_result = Temporal::parse_utc_offset(offset_string, Temporal::SubMinutePrecision::No); // 2. If parseResult is a List of errors, return false. // 3. Return true. @@ -675,11 +675,10 @@ bool is_offset_time_zone_identifier(StringView offset_string) // 21.4.1.33.2 ParseTimeZoneOffsetString ( offsetString ), https://tc39.es/ecma262/#sec-parsetimezoneoffsetstring // 14.5.11 ParseDateTimeUTCOffset ( offsetString ), https://tc39.es/proposal-temporal/#sec-parsedatetimeutcoffset -ThrowCompletionOr parse_date_time_utc_offset(VM& vm, StringView offset_string) +ThrowCompletionOr parse_date_time_utc_offset(VM& vm, Utf16View offset_string) { // 1. Let parseResult be ParseText(offsetString, UTCOffset[+SubMinutePrecision]). - auto utf16_offset_string = Utf16String::from_utf8(offset_string); - auto parse_result = Temporal::parse_utc_offset(utf16_offset_string, Temporal::SubMinutePrecision::Yes); + auto parse_result = Temporal::parse_utc_offset(offset_string, Temporal::SubMinutePrecision::Yes); // 2. If parseResult is a List of errors, throw a RangeError exception. if (!parse_result.has_value()) @@ -690,13 +689,12 @@ ThrowCompletionOr parse_date_time_utc_offset(VM& vm, StringView offset_s // 21.4.1.33.2 ParseTimeZoneOffsetString ( offsetString ), https://tc39.es/ecma262/#sec-parsetimezoneoffsetstring // 14.5.11 ParseDateTimeUTCOffset ( offsetString ), https://tc39.es/proposal-temporal/#sec-parsedatetimeutcoffset -double parse_date_time_utc_offset(StringView offset_string) +double parse_date_time_utc_offset(Utf16View offset_string) { // OPTIMIZATION: Some callers can assume that parsing will succeed. // 1. Let parseResult be ParseText(offsetString, UTCOffset[+SubMinutePrecision]). - auto utf16_offset_string = Utf16String::from_utf8(offset_string); - auto parse_result = Temporal::parse_utc_offset(utf16_offset_string, Temporal::SubMinutePrecision::Yes); + auto parse_result = Temporal::parse_utc_offset(offset_string, Temporal::SubMinutePrecision::Yes); VERIFY(parse_result.has_value()); return parse_date_time_utc_offset(*parse_result); diff --git a/Libraries/LibJS/Runtime/Date.h b/Libraries/LibJS/Runtime/Date.h index ceec41c261..4c92ce1895 100644 --- a/Libraries/LibJS/Runtime/Date.h +++ b/Libraries/LibJS/Runtime/Date.h @@ -7,6 +7,7 @@ #pragma once +#include #include #include #include @@ -97,9 +98,9 @@ JS_API double make_time(double hour, double min, double sec, double ms); JS_API double make_day(double year, double month, double date); JS_API double make_date(double day, double time); double time_clip(double time); -bool is_offset_time_zone_identifier(StringView offset_string); -ThrowCompletionOr parse_date_time_utc_offset(VM&, StringView offset_string); -double parse_date_time_utc_offset(StringView offset_string); +bool is_offset_time_zone_identifier(Utf16View offset_string); +ThrowCompletionOr parse_date_time_utc_offset(VM&, Utf16View offset_string); +double parse_date_time_utc_offset(Utf16View offset_string); double parse_date_time_utc_offset(Temporal::TimeZoneOffset const&); } diff --git a/Libraries/LibJS/Runtime/DateConstructor.cpp b/Libraries/LibJS/Runtime/DateConstructor.cpp index 14f0d5bb87..9b7498bfc7 100644 --- a/Libraries/LibJS/Runtime/DateConstructor.cpp +++ b/Libraries/LibJS/Runtime/DateConstructor.cpp @@ -24,7 +24,7 @@ namespace JS { GC_DEFINE_ALLOCATOR(DateConstructor); -static double parse_date_string(VM& vm, StringView date_string) +static double parse_date_string(VM& vm, Utf16View date_string) { double result = DateParser::parse(date_string); if (result == NAN) @@ -33,12 +33,6 @@ static double parse_date_string(VM& vm, StringView date_string) return result; } -static double parse_date_string(VM& vm, Utf16View date_string) -{ - auto utf8_date_string = date_string.to_utf8_but_should_be_ported_to_utf16(); - return parse_date_string(vm, utf8_date_string.bytes_as_string_view()); -} - DateConstructor::DateConstructor(Realm& realm) : NativeFunction(realm.vm().names.Date.as_string(), realm.intrinsics().function_prototype()) { diff --git a/Libraries/LibJS/Runtime/DateParser.h b/Libraries/LibJS/Runtime/DateParser.h index 7a474f5502..ce6cb251ce 100644 --- a/Libraries/LibJS/Runtime/DateParser.h +++ b/Libraries/LibJS/Runtime/DateParser.h @@ -6,6 +6,7 @@ #pragma once +#include #include #include #include @@ -45,18 +46,18 @@ // - We always parse as "Month 01, Year". // - Support Firefox less permissive punctuation but more permissive punctuation // syntax. -class DateParser : public GenericLexer { +class DateParser : public Utf16GenericLexer { public: - ALWAYS_INLINE static double parse(StringView string) + ALWAYS_INLINE static double parse(Utf16View string) { - if (!string.is_ascii()) + if (!all_of(string, is_ascii)) return NAN; return DateParser(string).parse().value_or(NAN); } private: - explicit DateParser(StringView string) - : GenericLexer(string) + explicit DateParser(Utf16View string) + : Utf16GenericLexer(string) { } @@ -589,7 +590,7 @@ private: m_timezone_utc = true; - bool space = consume_while(is_ascii_space).length() > 0; + bool space = !consume_while(is_ascii_space).is_empty(); switch (peek()) { case '+': case '-': @@ -953,10 +954,10 @@ private: // Convert the input string to uppercase only ~after~ parsing ISO8601 failed. // This saves some time (two string copies) if parsing a ISO8601 date succeeds. // The index stays exactly where it was before converting to uppercase. - auto str_uppercase = m_input.to_ascii_uppercase_string(); - m_input = str_uppercase; + auto str_uppercase = m_input.to_ascii_uppercase(); + m_input = str_uppercase.utf16_view(); // FIXME: Two full string copies could be avoided, if to_uppercase can be done in place. - // The underlying StringView m_input protects itself from modifying its contents. Bummer. + // The underlying Utf16View m_input protects itself from modifying its contents. Bummer. while (!is_eof()) if (!loop()) diff --git a/Libraries/LibJS/Runtime/DatePrototype.cpp b/Libraries/LibJS/Runtime/DatePrototype.cpp index 9b5dee5f8f..77c4630b28 100644 --- a/Libraries/LibJS/Runtime/DatePrototype.cpp +++ b/Libraries/LibJS/Runtime/DatePrototype.cpp @@ -1105,7 +1105,7 @@ ByteString date_string(double time) // 21.4.4.41.3 TimeZoneString ( tv ), https://tc39.es/ecma262/#sec-timezoneestring // 14.5.9 TimeZoneString ( tv ), https://tc39.es/proposal-temporal/#sec-timezoneestring -ByteString time_zone_string(double time) +Utf16String time_zone_string(double time) { // 1. Let systemTimeZoneIdentifier be SystemTimeZoneIdentifier(). auto system_time_zone_identifier = JS::system_time_zone_identifier(); @@ -1130,28 +1130,29 @@ ByteString time_zone_string(double time) // 5. Let tzName be an implementation-defined string that is either the empty String or the string-concatenation of // the code unit 0x0020 (SPACE), the code unit 0x0028 (LEFT PARENTHESIS), an implementation-defined timezone name, // and the code unit 0x0029 (RIGHT PARENTHESIS). - auto tz_name = Unicode::current_time_zone(); + auto time_zone_identifier = Unicode::current_time_zone(); + auto tz_name = Utf16String::from_utf8(time_zone_identifier); // Most implementations seem to prefer the long-form display name of the time zone. Not super important, but we may as well match that behavior. - if (auto name = Unicode::time_zone_display_name(Unicode::default_locale(), tz_name, in_dst, time); name.has_value()) - tz_name = name->to_utf8_but_should_be_ported_to_utf16(); + if (auto name = Unicode::time_zone_display_name(Unicode::default_locale(), time_zone_identifier, in_dst, time); name.has_value()) + tz_name = name.release_value(); // 10. Return the string-concatenation of offsetString and tzName. - return ByteString::formatted("{} ({})", offset_string, tz_name); + return Utf16String::formatted("{} ({})", offset_string, tz_name); } // 21.4.4.41.4 ToDateString ( tv ), https://tc39.es/ecma262/#sec-todatestring -ByteString to_date_string(double time) +Utf16String to_date_string(double time) { // 1. If tv is NaN, return "Invalid Date". if (Value(time).is_nan()) - return "Invalid Date"sv; + return "Invalid Date"_utf16; // 2. Let t be LocalTime(tv). time = local_time(time); // 3. Return the string-concatenation of DateString(t), the code unit 0x0020 (SPACE), TimeString(t), and TimeZoneString(tv). - return ByteString::formatted("{} {}{}", date_string(time), time_string(time), time_zone_string(time)); + return Utf16String::formatted("{} {}{}", date_string(time), time_string(time), time_zone_string(time)); } // 21.4.4.42 Date.prototype.toTimeString ( ), https://tc39.es/ecma262/#sec-date.prototype.totimestring @@ -1167,7 +1168,7 @@ JS_DEFINE_NATIVE_FUNCTION(DatePrototype::to_time_string) // 4. Let t be LocalTime(tv). // 5. Return the string-concatenation of TimeString(t) and TimeZoneString(tv). - auto string = ByteString::formatted("{}{}", time_string(local_time(time)), time_zone_string(time)); + auto string = Utf16String::formatted("{}{}", time_string(local_time(time)), time_zone_string(time)); return PrimitiveString::create(vm, move(string)); } diff --git a/Libraries/LibJS/Runtime/DatePrototype.h b/Libraries/LibJS/Runtime/DatePrototype.h index bb28e97080..90d1495fe6 100644 --- a/Libraries/LibJS/Runtime/DatePrototype.h +++ b/Libraries/LibJS/Runtime/DatePrototype.h @@ -6,6 +6,7 @@ #pragma once +#include #include #include @@ -77,7 +78,7 @@ private: ThrowCompletionOr this_time_value(VM&, Value value); ByteString time_string(double time); ByteString date_string(double time); -ByteString time_zone_string(double time); -ByteString to_date_string(double time); +Utf16String time_zone_string(double time); +Utf16String to_date_string(double time); } diff --git a/Libraries/LibJS/Runtime/FunctionConstructor.cpp b/Libraries/LibJS/Runtime/FunctionConstructor.cpp index f772934441..6425f1252a 100644 --- a/Libraries/LibJS/Runtime/FunctionConstructor.cpp +++ b/Libraries/LibJS/Runtime/FunctionConstructor.cpp @@ -109,23 +109,23 @@ ThrowCompletionOr> FunctionConstructor::create auto arg_count = parameter_args.size(); // 7. Let parameterStrings be a new empty List. - Vector parameter_strings; + Vector parameter_strings; parameter_strings.ensure_capacity(arg_count); // 8. For each element arg of parameterArgs, do for (auto const& parameter_value : parameter_args) { // a. Append ? ToString(arg) to parameterStrings. - parameter_strings.unchecked_append(TRY(parameter_value.to_utf16_string(vm)).to_utf8_but_should_be_ported_to_utf16()); + parameter_strings.unchecked_append(TRY(parameter_value.to_utf16_string(vm))); } // 9. Let bodyString be ? ToString(bodyArg). - auto body_string = TRY(body_arg.to_utf16_string(vm)).to_utf8_but_should_be_ported_to_utf16(); + auto body_string = TRY(body_arg.to_utf16_string(vm)); // 10. Let currentRealm be the current Realm Record. auto& realm = *vm.current_realm(); // 11. Let P be the empty String. - String parameters_string; + Utf16String parameters_string; // 12. If argCount > 0, then if (arg_count > 0) { @@ -135,15 +135,15 @@ ThrowCompletionOr> FunctionConstructor::create // i. Let nextArgString be parameterStrings[k]. // ii. Set P to the string-concatenation of P, "," (a comma), and nextArgString. // iii. Set k to k + 1. - parameters_string = MUST(String::join(',', parameter_strings)); + parameters_string = Utf16String::join(',', parameter_strings); } // 13. Let bodyParseString be the string-concatenation of 0x000A (LINE FEED), bodyString, and 0x000A (LINE FEED). - auto body_parse_string = ByteString::formatted("\n{}\n", body_string); + auto body_parse_string = Utf16String::formatted("\n{}\n", body_string); // 14. Let sourceString be the string-concatenation of prefix, " anonymous(", P, 0x000A (LINE FEED), ") {", bodyParseString, and "}". // 15. Let sourceText be StringToCodePoints(sourceString). - auto source_text = ByteString::formatted("{} anonymous({}\n) {{{}}}", prefix, parameters_string, body_parse_string); + auto source_text = Utf16String::formatted("{} anonymous({}\n) {{{}}}", prefix, parameters_string, body_parse_string); // 16. Perform ? HostEnsureCanCompileStrings(currentRealm, parameterStrings, bodyString, sourceString, FUNCTION, parameterArgs, bodyArg). TRY(vm.host_ensure_can_compile_strings(realm, parameter_strings, body_string, source_text, CompilationType::Function, parameter_args, body_arg)); diff --git a/Libraries/LibJS/Runtime/Intl/AbstractOperations.cpp b/Libraries/LibJS/Runtime/Intl/AbstractOperations.cpp index b0dcdc8ef6..81cf582036 100644 --- a/Libraries/LibJS/Runtime/Intl/AbstractOperations.cpp +++ b/Libraries/LibJS/Runtime/Intl/AbstractOperations.cpp @@ -24,8 +24,8 @@ namespace JS::Intl { -// 6.2.1 IsWellFormedLanguageTag ( locale ), https://tc39.es/ecma402/#sec-iswellformedlanguagetag -bool is_well_formed_language_tag(StringView locale) +template +static bool is_well_formed_language_tag_impl(ViewType locale) { auto contains_duplicate_variant = [&](auto& variants) { if (variants.is_empty()) @@ -84,7 +84,7 @@ bool is_well_formed_language_tag(StringView locale) // b. Let transformExtension be the longest substring of extensions matched by the transformed_extensions Unicode // locale nonterminal. If there is no such substring, return true. - if (auto* transformed = extension.get_pointer()) { + if (auto* transformed = extension.template get_pointer()) { // c. Assert: The substring of transformExtension from 0 to 3 is "-t-". // d. Let tPrefix be the substring of transformExtension from 3. @@ -109,12 +109,28 @@ bool is_well_formed_language_tag(StringView locale) return true; } +// 6.2.1 IsWellFormedLanguageTag ( locale ), https://tc39.es/ecma402/#sec-iswellformedlanguagetag +bool is_well_formed_language_tag(StringView locale) +{ + return is_well_formed_language_tag_impl(locale); +} + +bool is_well_formed_language_tag(Utf16View locale) +{ + return is_well_formed_language_tag_impl(locale); +} + // 6.2.2 CanonicalizeUnicodeLocaleId ( locale ), https://tc39.es/ecma402/#sec-canonicalizeunicodelocaleid String canonicalize_unicode_locale_id(StringView locale) { return Unicode::canonicalize_unicode_locale_id(locale); } +String canonicalize_unicode_locale_id(Utf16View locale) +{ + return Unicode::canonicalize_unicode_locale_id(locale); +} + // 6.3.1 IsWellFormedCurrencyCode ( currency ), https://tc39.es/ecma402/#sec-iswellformedcurrencycode bool is_well_formed_currency_code(StringView currency) { @@ -131,6 +147,23 @@ bool is_well_formed_currency_code(StringView currency) return true; } +bool is_well_formed_currency_code(Utf16View currency) +{ + // 1. If the length of currency is not 3, return false. + if (currency.length_in_code_units() != 3) + return false; + + // 2. Let normalized be the ASCII-uppercase of currency. + // 3. If normalized contains any code unit outside of 0x0041 through 0x005A (corresponding to Unicode characters LATIN CAPITAL LETTER A through LATIN CAPITAL LETTER Z), return false. + for (size_t i = 0; i < currency.length_in_code_units(); ++i) { + if (!is_ascii_alpha(currency.code_unit_at(i))) + return false; + } + + // 4. Return true. + return true; +} + // 6.5.1 AvailableNamedTimeZoneIdentifiers ( ), https://tc39.es/ecma402/#sup-availablenamedtimezoneidentifiers Vector const& available_named_time_zone_identifiers() { @@ -205,14 +238,18 @@ Optional get_available_named_time_zone_identifier(Str } // 6.6.1 IsWellFormedUnitIdentifier ( unitIdentifier ), https://tc39.es/ecma402/#sec-iswellformedunitidentifier -bool is_well_formed_unit_identifier(StringView unit_identifier) +bool is_well_formed_unit_identifier(Utf16View unit_identifier) { // 6.6.2 IsSanctionedSingleUnitIdentifier ( unitIdentifier ), https://tc39.es/ecma402/#sec-issanctionedsingleunitidentifier - constexpr auto is_sanctioned_single_unit_identifier = [](StringView unit_identifier) { + constexpr auto is_sanctioned_single_unit_identifier = [](Utf16View unit_identifier) { // 1. If unitIdentifier is listed in Table 2 below, return true. // 2. Else, return false. static constexpr auto sanctioned_units = sanctioned_single_unit_identifiers(); - return find(sanctioned_units.begin(), sanctioned_units.end(), unit_identifier) != sanctioned_units.end(); + for (auto sanctioned_unit : sanctioned_units) { + if (unit_identifier == sanctioned_unit) + return true; + } + return false; }; // 1. If ! IsSanctionedSingleUnitIdentifier(unitIdentifier) is true, then @@ -222,22 +259,22 @@ bool is_well_formed_unit_identifier(StringView unit_identifier) } // 2. Let i be StringIndexOf(unitIdentifier, "-per-", 0). - auto indices = unit_identifier.find_all("-per-"sv); + auto index = unit_identifier.find_code_unit_offset("-per-"sv); // 3. If i is -1 or StringIndexOf(unitIdentifier, "-per-", i + 1) is not -1, then - if (indices.size() != 1) { + if (!index.has_value() || unit_identifier.find_code_unit_offset("-per-"sv, *index + 1).has_value()) { // a. Return false. return false; } // 4. Assert: The five-character substring "-per-" occurs exactly once in unitIdentifier, at index i. - // NOTE: We skip this because the indices vector being of size 1 already verifies this invariant. + // NOTE: We skip this because the checks above already verify this invariant. // 5. Let numerator be the substring of unitIdentifier from 0 to i. - auto numerator = unit_identifier.substring_view(0, indices[0]); + auto numerator = unit_identifier.substring_view(0, *index); // 6. Let denominator be the substring of unitIdentifier from i + 5. - auto denominator = unit_identifier.substring_view(indices[0] + 5); + auto denominator = unit_identifier.substring_view(*index + 5); // 7. If ! IsSanctionedSingleUnitIdentifier(numerator) and ! IsSanctionedSingleUnitIdentifier(denominator) are both true, then if (is_sanctioned_single_unit_identifier(numerator) && is_sanctioned_single_unit_identifier(denominator)) { @@ -297,26 +334,33 @@ ThrowCompletionOr> canonicalize_locale_list(VM& vm, Value locales if (!key_value.is_string() && !key_value.is_object()) return vm.throw_completion(ErrorType::NotAnObjectOrString, key_value); - String tag; + String canonicalized_tag; // iii. If Type(kValue) is Object and kValue has an [[InitializedLocale]] internal slot, then if (auto locale = key_value.as_if()) { // 1. Let tag be kValue.[[Locale]]. - tag = locale->locale(); + auto tag = locale->locale(); + + // v. If IsWellFormedLanguageTag(tag) is false, throw a RangeError exception. + if (!is_well_formed_language_tag(tag)) + return vm.throw_completion(ErrorType::IntlInvalidLanguageTag, tag); + + // vi. Let canonicalizedTag be ! CanonicalizeUnicodeLocaleId(tag). + canonicalized_tag = canonicalize_unicode_locale_id(tag); } // iv. Else, else { // 1. Let tag be ? ToString(kValue). - tag = TRY(key_value.to_utf16_string(vm)).to_utf8_but_should_be_ported_to_utf16(); + auto tag = TRY(key_value.to_utf16_string(vm)); + + // v. If IsWellFormedLanguageTag(tag) is false, throw a RangeError exception. + if (!is_well_formed_language_tag(tag.utf16_view())) + return vm.throw_completion(ErrorType::IntlInvalidLanguageTag, tag); + + // vi. Let canonicalizedTag be ! CanonicalizeUnicodeLocaleId(tag). + canonicalized_tag = canonicalize_unicode_locale_id(tag.utf16_view()); } - // v. If IsWellFormedLanguageTag(tag) is false, throw a RangeError exception. - if (!is_well_formed_language_tag(tag)) - return vm.throw_completion(ErrorType::IntlInvalidLanguageTag, tag); - - // vi. Let canonicalizedTag be ! CanonicalizeUnicodeLocaleId(tag). - auto canonicalized_tag = canonicalize_unicode_locale_id(tag); - // vii. If canonicalizedTag is not an element of seen, append canonicalizedTag as the last element of seen. if (!seen.contains_slow(canonicalized_tag)) seen.append(move(canonicalized_tag)); @@ -639,13 +683,14 @@ ThrowCompletionOr resolve_options(VM& vm, IntlObject& object, V // d. If value is not undefined, then if (!value.is_undefined()) { // i. Set value to ! ToString(value). - auto value_string = MUST(value.to_utf16_string(vm)).to_utf8_but_should_be_ported_to_utf16(); + auto value_string = MUST(value.to_utf16_string(vm)); + auto value_string_view = value_string.utf16_view(); // ii. If value cannot be matched by the type Unicode locale nonterminal, throw a RangeError exception. - if (!Unicode::is_type_identifier(value_string)) - return vm.throw_completion(ErrorType::OptionIsNotValidValue, value_string, descriptor.property); + if (!value_string_view.is_ascii() || !Unicode::is_type_identifier(value_string_view)) + return vm.throw_completion(ErrorType::OptionIsNotValidValue, value_string_view, descriptor.property); - locale_key = move(value_string); + locale_key = MUST(value_string_view.to_utf8()); } // e. Let key be desc.[[Key]]. @@ -749,7 +794,7 @@ ThrowCompletionOr get_boolean_or_string_number_format_option(VM auto value_string_view = value_string.utf16_view(); auto it = find_if(string_values.begin(), string_values.end(), [&](auto allowed_value) { return value_string_view == allowed_value; }); if (it == string_values.end()) - return vm.throw_completion(ErrorType::OptionIsNotValidValue, value_string_view.to_utf8_but_should_be_ported_to_utf16(), property.as_string()); + return vm.throw_completion(ErrorType::OptionIsNotValidValue, value_string_view, property.as_string()); // 7. Return value. return StringOrBoolean { *it }; diff --git a/Libraries/LibJS/Runtime/Intl/AbstractOperations.h b/Libraries/LibJS/Runtime/Intl/AbstractOperations.h index 41aeb2c310..25b2377408 100644 --- a/Libraries/LibJS/Runtime/Intl/AbstractOperations.h +++ b/Libraries/LibJS/Runtime/Intl/AbstractOperations.h @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -64,11 +65,14 @@ AK_ENUM_BITWISE_OPERATORS(SpecialBehaviors); using StringOrBoolean = Variant; bool is_well_formed_language_tag(StringView locale); +bool is_well_formed_language_tag(Utf16View locale); String canonicalize_unicode_locale_id(StringView locale); +String canonicalize_unicode_locale_id(Utf16View locale); bool is_well_formed_currency_code(StringView currency); +bool is_well_formed_currency_code(Utf16View currency); Vector const& available_named_time_zone_identifiers(); Optional get_available_named_time_zone_identifier(StringView time_zone_identifier); -bool is_well_formed_unit_identifier(StringView unit_identifier); +bool is_well_formed_unit_identifier(Utf16View unit_identifier); ThrowCompletionOr> canonicalize_locale_list(VM&, Value locales); Optional lookup_matching_locale_by_prefix(ReadonlySpan requested_locales); Optional lookup_matching_locale_by_best_fit(ReadonlySpan requested_locales); diff --git a/Libraries/LibJS/Runtime/Intl/Collator.h b/Libraries/LibJS/Runtime/Intl/Collator.h index 58decc07ea..6b397faaac 100644 --- a/Libraries/LibJS/Runtime/Intl/Collator.h +++ b/Libraries/LibJS/Runtime/Intl/Collator.h @@ -8,6 +8,7 @@ #include #include +#include #include #include #include @@ -28,7 +29,7 @@ public: void set_locale(String locale) { m_locale = move(locale); } Unicode::Usage usage() const { return m_usage; } - void set_usage(StringView usage) { m_usage = Unicode::usage_from_string(usage); } + void set_usage(Utf16View usage) { m_usage = Unicode::usage_from_string(usage); } StringView usage_string() const LIFETIME_BOUND { return Unicode::usage_to_string(m_usage); } Unicode::Sensitivity sensitivity() const { return m_sensitivity; } diff --git a/Libraries/LibJS/Runtime/Intl/CollatorCompareFunction.cpp b/Libraries/LibJS/Runtime/Intl/CollatorCompareFunction.cpp index cbc60350ce..48d5cec5c3 100644 --- a/Libraries/LibJS/Runtime/Intl/CollatorCompareFunction.cpp +++ b/Libraries/LibJS/Runtime/Intl/CollatorCompareFunction.cpp @@ -4,7 +4,6 @@ * SPDX-License-Identifier: BSD-2-Clause */ -#include #include #include #include diff --git a/Libraries/LibJS/Runtime/Intl/CollatorConstructor.cpp b/Libraries/LibJS/Runtime/Intl/CollatorConstructor.cpp index 1ed97a853a..ff3b281068 100644 --- a/Libraries/LibJS/Runtime/Intl/CollatorConstructor.cpp +++ b/Libraries/LibJS/Runtime/Intl/CollatorConstructor.cpp @@ -69,7 +69,7 @@ ThrowCompletionOr> CollatorConstructor::construct(FunctionObject auto usage = TRY(get_option(vm, options, vm.names.usage, OptionType::String, { "sort"sv, "search"sv }, "sort"sv)); // 8. Set collator.[[Usage]] to usage. - collator->set_usage(usage.as_string().utf16_string_view().to_utf8_but_should_be_ported_to_utf16()); + collator->set_usage(usage.as_string().utf16_string_view()); // 9. If usage is "sort", then // a. Let localeData be %Intl.Collator%.[[SortLocaleData]]. @@ -113,7 +113,7 @@ ThrowCompletionOr> CollatorConstructor::construct(FunctionObject Optional sensitivity; if (!sensitivity_value.is_undefined()) - sensitivity = Unicode::sensitivity_from_string(sensitivity_value.as_string().utf16_string_view().to_utf8_but_should_be_ported_to_utf16()); + sensitivity = Unicode::sensitivity_from_string(sensitivity_value.as_string().utf16_string_view()); // 21. Let defaultIgnorePunctuation be resolvedLocaleData.[[ignorePunctuation]]. // NOTE: We do not acquire resolvedLocaleData.[[ignorePunctuation]] here. Instead, we let LibUnicode fill in the diff --git a/Libraries/LibJS/Runtime/Intl/DateTimeFormat.h b/Libraries/LibJS/Runtime/Intl/DateTimeFormat.h index b86e280fec..4a36843663 100644 --- a/Libraries/LibJS/Runtime/Intl/DateTimeFormat.h +++ b/Libraries/LibJS/Runtime/Intl/DateTimeFormat.h @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -49,12 +50,12 @@ public: bool has_date_style() const { return m_date_style.has_value(); } Optional const& date_style() const { return m_date_style; } StringView date_style_string() const { return Unicode::date_time_style_to_string(*m_date_style); } - void set_date_style(StringView style) { m_date_style = Unicode::date_time_style_from_string(style); } + void set_date_style(Utf16View style) { m_date_style = Unicode::date_time_style_from_string(style); } bool has_time_style() const { return m_time_style.has_value(); } Optional const& time_style() const { return m_time_style; } StringView time_style_string() const { return Unicode::date_time_style_to_string(*m_time_style); } - void set_time_style(StringView style) { m_time_style = Unicode::date_time_style_from_string(style); } + void set_time_style(Utf16View style) { m_time_style = Unicode::date_time_style_from_string(style); } Unicode::CalendarPattern& date_time_format() { return m_date_time_format; } void set_date_time_format(Unicode::CalendarPattern date_time_format) { m_date_time_format = move(date_time_format); } diff --git a/Libraries/LibJS/Runtime/Intl/DateTimeFormatConstructor.cpp b/Libraries/LibJS/Runtime/Intl/DateTimeFormatConstructor.cpp index 849abb22bc..3bfef10276 100644 --- a/Libraries/LibJS/Runtime/Intl/DateTimeFormatConstructor.cpp +++ b/Libraries/LibJS/Runtime/Intl/DateTimeFormatConstructor.cpp @@ -171,6 +171,7 @@ ThrowCompletionOr> create_date_time_format(VM& vm, Funct // 17. Let timeZone be ? Get(options, "timeZone"). auto time_zone_value = TRY(options->get(vm.names.timeZone)); String time_zone; + Utf16String time_zone_string; // 18. If timeZone is undefined, then if (time_zone_value.is_undefined()) { @@ -178,11 +179,13 @@ ThrowCompletionOr> create_date_time_format(VM& vm, Funct if (to_locale_string_time_zone.has_value()) { // i. Set timeZone to toLocaleStringTimeZone. time_zone = *to_locale_string_time_zone; + time_zone_string = Utf16String::from_utf8(time_zone); } // b. Else, else { // i. Set timeZone to SystemTimeZoneIdentifier(). time_zone = system_time_zone_identifier(); + time_zone_string = Utf16String::from_utf8(time_zone); } } // 19. Else, @@ -192,22 +195,23 @@ ThrowCompletionOr> create_date_time_format(VM& vm, Funct return vm.throw_completion(ErrorType::IntlInvalidDateTimeFormatOption, vm.names.timeZone, "a toLocaleString time zone"sv); // b. Set timeZone to ? ToString(timeZone). - time_zone = TRY(time_zone_value.to_utf16_string(vm)).to_utf8_but_should_be_ported_to_utf16(); + time_zone_string = TRY(time_zone_value.to_utf16_string(vm)); } + auto time_zone_view = time_zone_string.utf16_view(); + // 20. If IsTimeZoneOffsetString(timeZone) is true, then - bool is_time_zone_offset_string = JS::is_offset_time_zone_identifier(time_zone); + auto parse_result = Temporal::parse_utc_offset(time_zone_view, Temporal::SubMinutePrecision::No); + bool is_time_zone_offset_string = parse_result.has_value(); if (is_time_zone_offset_string) { // a. Let parseResult be ParseText(StringToCodePoints(timeZone), UTCOffset[~SubMinutePrecision]). - auto utf16_time_zone = Utf16String::from_utf8(time_zone); - auto parse_result = Temporal::parse_utc_offset(utf16_time_zone, Temporal::SubMinutePrecision::No); // b. Assert: parseResult is a Parse Node. VERIFY(parse_result.has_value()); // c. Let offsetNanoseconds be ? ParseDateTimeUTCOffset(timeZone). - auto offset_nanoseconds = TRY(parse_date_time_utc_offset(vm, time_zone)); + auto offset_nanoseconds = parse_date_time_utc_offset(*parse_result); // d. Let offsetMinutes be offsetNanoseconds / (6 × 10**10). auto offset_minutes = offset_nanoseconds / 60'000'000'000; @@ -218,6 +222,10 @@ ThrowCompletionOr> create_date_time_format(VM& vm, Funct // 21. Else, else { // a. Let timeZoneIdentifierRecord be GetAvailableNamedTimeZoneIdentifier(timeZone). + if (!time_zone_view.is_ascii()) + return vm.throw_completion(ErrorType::OptionIsNotValidValue, time_zone_view, vm.names.timeZone); + + time_zone = MUST(time_zone_view.to_utf8()); auto time_zone_identifier_record = get_available_named_time_zone_identifier(time_zone); // b. If timeZoneIdentifierRecord is EMPTY, throw a RangeError exception. @@ -277,7 +285,7 @@ ThrowCompletionOr> create_date_time_format(VM& vm, Funct // d. Set formatOptions.[[]] to value. if (!value.is_undefined()) { - option = Unicode::calendar_pattern_style_from_string(value.as_string().utf16_string_view().to_utf8_but_should_be_ported_to_utf16()); + option = Unicode::calendar_pattern_style_from_string(value.as_string().utf16_string_view()); // e. If value is not undefined, then // i. Set hasExplicitFormatComponents to true. @@ -296,14 +304,14 @@ ThrowCompletionOr> create_date_time_format(VM& vm, Funct // 29. Set dateTimeFormat.[[DateStyle]] to dateStyle. if (!date_style.is_undefined()) - date_time_format->set_date_style(date_style.as_string().utf16_string_view().to_utf8_but_should_be_ported_to_utf16()); + date_time_format->set_date_style(date_style.as_string().utf16_string_view()); // 30. Let timeStyle be ? GetOption(options, "timeStyle", string, « "full", "long", "medium", "short" », undefined). auto time_style = TRY(get_option(vm, *options, vm.names.timeStyle, OptionType::String, AK::Array { "full"sv, "long"sv, "medium"sv, "short"sv }, Empty {})); // 31. Set dateTimeFormat.[[TimeStyle]] to timeStyle. if (!time_style.is_undefined()) - date_time_format->set_time_style(time_style.as_string().utf16_string_view().to_utf8_but_should_be_ported_to_utf16()); + date_time_format->set_time_style(time_style.as_string().utf16_string_view()); // 32. Let formats be resolvedLocaleData.[[formats]].[[]]. diff --git a/Libraries/LibJS/Runtime/Intl/DisplayNames.cpp b/Libraries/LibJS/Runtime/Intl/DisplayNames.cpp index 80addf84bc..2b00413345 100644 --- a/Libraries/LibJS/Runtime/Intl/DisplayNames.cpp +++ b/Libraries/LibJS/Runtime/Intl/DisplayNames.cpp @@ -32,7 +32,7 @@ ReadonlySpan DisplayNames::resolution_option_descrip return {}; } -void DisplayNames::set_type(StringView type) +void DisplayNames::set_type(Utf16View type) { if (type == "language"sv) m_type = Type::Language; @@ -70,7 +70,7 @@ StringView DisplayNames::type_string() const } } -void DisplayNames::set_fallback(StringView fallback) +void DisplayNames::set_fallback(Utf16View fallback) { if (fallback == "none"sv) m_fallback = Fallback::None; @@ -93,20 +93,25 @@ StringView DisplayNames::fallback_string() const } // 12.5.1 CanonicalCodeForDisplayNames ( type, code ), https://tc39.es/ecma402/#sec-canonicalcodefordisplaynames -ThrowCompletionOr canonical_code_for_display_names(VM& vm, DisplayNames::Type type, StringView code) +ThrowCompletionOr canonical_code_for_display_names(VM& vm, DisplayNames::Type type, Utf16View code) { // 1. If type is "language", then if (type == DisplayNames::Type::Language) { + if (!code.is_ascii()) + return vm.throw_completion(ErrorType::OptionIsNotValidValue, code, "language"sv); + + auto code_string = MUST(code.to_utf8()); + // a. If code does not match the unicode_language_id production, throw a RangeError exception. - if (!Unicode::parse_unicode_language_id(code).has_value()) + if (!Unicode::parse_unicode_language_id(code_string).has_value()) return vm.throw_completion(ErrorType::OptionIsNotValidValue, code, "language"sv); // b. If IsWellFormedLanguageTag(code) is false, throw a RangeError exception. - if (!is_well_formed_language_tag(code)) + if (!is_well_formed_language_tag(code_string)) return vm.throw_completion(ErrorType::IntlInvalidLanguageTag, code); // c. Return ! CanonicalizeUnicodeLocaleId(code). - auto canonicalized_tag = canonicalize_unicode_locale_id(code); + auto canonicalized_tag = canonicalize_unicode_locale_id(code_string); return PrimitiveString::create(vm, move(canonicalized_tag)); } @@ -117,7 +122,7 @@ ThrowCompletionOr canonical_code_for_display_names(VM& vm, DisplayNames:: return vm.throw_completion(ErrorType::OptionIsNotValidValue, code, "region"sv); // b. Return the ASCII-uppercase of code. - return PrimitiveString::create(vm, code.to_ascii_uppercase_string()); + return PrimitiveString::create(vm, code.to_ascii_uppercase()); } // 3. If type is "script", then @@ -127,13 +132,13 @@ ThrowCompletionOr canonical_code_for_display_names(VM& vm, DisplayNames:: return vm.throw_completion(ErrorType::OptionIsNotValidValue, code, "script"sv); // Assert: The length of code is 4, and every code unit of code represents an ASCII letter (0x0041 through 0x005A and 0x0061 through 0x007A, both inclusive). - VERIFY(code.length() == 4); + VERIFY(code.length_in_code_units() == 4); VERIFY(all_of(code, is_ascii_alpha)); // c. Let first be the ASCII-uppercase of the substring of code from 0 to 1. // d. Let rest be the ASCII-lowercase of the substring of code from 1. // e. Return the string-concatenation of first and rest. - return PrimitiveString::create(vm, code.to_ascii_titlecase_string()); + return PrimitiveString::create(vm, code.to_ascii_titlecase()); } // 4. If type is "calendar", then @@ -143,11 +148,11 @@ ThrowCompletionOr canonical_code_for_display_names(VM& vm, DisplayNames:: return vm.throw_completion(ErrorType::OptionIsNotValidValue, code, "calendar"sv); // b. If code uses any of the backwards compatibility syntax described in Unicode Technical Standard #35 LDML § 3.3 BCP 47 Conformance, throw a RangeError exception. - if (code.contains('_')) + if (code.contains(u"_"sv)) return vm.throw_completion(ErrorType::OptionIsNotValidValue, code, "calendar"sv); // c. Return the ASCII-lowercase of code. - return PrimitiveString::create(vm, code.to_ascii_lowercase_string()); + return PrimitiveString::create(vm, code.to_ascii_lowercase()); } // 5. If type is "dateTimeField", then @@ -157,7 +162,7 @@ ThrowCompletionOr canonical_code_for_display_names(VM& vm, DisplayNames:: return vm.throw_completion(ErrorType::OptionIsNotValidValue, code, "dateTimeField"sv); // b. Return code. - return PrimitiveString::create(vm, code); + return PrimitiveString::create(vm, Utf16String::from_utf16(code)); } // 6. Assert: type is "currency". @@ -168,11 +173,11 @@ ThrowCompletionOr canonical_code_for_display_names(VM& vm, DisplayNames:: return vm.throw_completion(ErrorType::OptionIsNotValidValue, code, "currency"sv); // 8. Return the ASCII-uppercase of code. - return PrimitiveString::create(vm, code.to_ascii_uppercase_string()); + return PrimitiveString::create(vm, code.to_ascii_uppercase()); } // 12.5.2 IsValidDateTimeFieldCode ( field ), https://tc39.es/ecma402/#sec-isvaliddatetimefieldcode -bool is_valid_date_time_field_code(StringView field) +bool is_valid_date_time_field_code(Utf16View field) { // 1. If field is listed in the Code column of Table 19, return true. // 2. Return false. diff --git a/Libraries/LibJS/Runtime/Intl/DisplayNames.h b/Libraries/LibJS/Runtime/Intl/DisplayNames.h index 371c5a85b1..712f8b8458 100644 --- a/Libraries/LibJS/Runtime/Intl/DisplayNames.h +++ b/Libraries/LibJS/Runtime/Intl/DisplayNames.h @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -45,20 +46,20 @@ public: void set_locale(String locale) { m_locale = move(locale); } Unicode::Style style() const { return m_style; } - void set_style(StringView style) { m_style = Unicode::style_from_string(style); } + void set_style(Utf16View style) { m_style = Unicode::style_from_string(style); } StringView style_string() const { return Unicode::style_to_string(m_style); } Type type() const { return m_type; } - void set_type(StringView type); + void set_type(Utf16View type); StringView type_string() const; Fallback fallback() const { return m_fallback; } - void set_fallback(StringView fallback); + void set_fallback(Utf16View fallback); StringView fallback_string() const; bool has_language_display() const { return m_language_display.has_value(); } Unicode::LanguageDisplay language_display() const { return *m_language_display; } - void set_language_display(StringView language_display) { m_language_display = Unicode::language_display_from_string(language_display); } + void set_language_display(Utf16View language_display) { m_language_display = Unicode::language_display_from_string(language_display); } StringView language_display_string() const { return Unicode::language_display_to_string(*m_language_display); } private: @@ -71,7 +72,7 @@ private: Optional m_language_display; // [[LanguageDisplay]] }; -ThrowCompletionOr canonical_code_for_display_names(VM&, DisplayNames::Type, StringView code); -bool is_valid_date_time_field_code(StringView field); +ThrowCompletionOr canonical_code_for_display_names(VM&, DisplayNames::Type, Utf16View code); +bool is_valid_date_time_field_code(Utf16View field); } diff --git a/Libraries/LibJS/Runtime/Intl/DisplayNamesConstructor.cpp b/Libraries/LibJS/Runtime/Intl/DisplayNamesConstructor.cpp index 7e8d7ebf1e..75bf0d9d06 100644 --- a/Libraries/LibJS/Runtime/Intl/DisplayNamesConstructor.cpp +++ b/Libraries/LibJS/Runtime/Intl/DisplayNamesConstructor.cpp @@ -64,7 +64,7 @@ ThrowCompletionOr> DisplayNamesConstructor::construct(FunctionOb auto style = TRY(get_option(vm, *options, vm.names.style, OptionType::String, { "narrow"sv, "short"sv, "long"sv }, "long"sv)); // 7. Set displayNames.[[Style]] to style. - display_names->set_style(style.as_string().utf16_string_view().to_utf8_but_should_be_ported_to_utf16()); + display_names->set_style(style.as_string().utf16_string_view()); // 8. Let type be ? GetOption(options, "type", string, « "language", "region", "script", "currency", "calendar", "dateTimeField" », undefined). auto type = TRY(get_option(vm, *options, vm.names.type, OptionType::String, { "language"sv, "region"sv, "script"sv, "currency"sv, "calendar"sv, "dateTimeField"sv }, Empty {})); @@ -74,13 +74,13 @@ ThrowCompletionOr> DisplayNamesConstructor::construct(FunctionOb return vm.throw_completion(ErrorType::IsUndefined, "options.type"sv); // 10. Set displayNames.[[Type]] to type. - display_names->set_type(type.as_string().utf16_string_view().to_utf8_but_should_be_ported_to_utf16()); + display_names->set_type(type.as_string().utf16_string_view()); // 11. Let fallback be ? GetOption(options, "fallback", string, « "code", "none" », "code"). auto fallback = TRY(get_option(vm, *options, vm.names.fallback, OptionType::String, { "code"sv, "none"sv }, "code"sv)); // 12. Set displayNames.[[Fallback]] to fallback. - display_names->set_fallback(fallback.as_string().utf16_string_view().to_utf8_but_should_be_ported_to_utf16()); + display_names->set_fallback(fallback.as_string().utf16_string_view()); // 13. Set displayNames.[[Locale]] to r.[[Locale]]. display_names->set_locale(move(result.locale)); @@ -98,7 +98,7 @@ ThrowCompletionOr> DisplayNamesConstructor::construct(FunctionOb // 20. If type is "language", then if (display_names->type() == DisplayNames::Type::Language) { // a. Set displayNames.[[LanguageDisplay]] to languageDisplay. - display_names->set_language_display(language_display.as_string().utf16_string_view().to_utf8_but_should_be_ported_to_utf16()); + display_names->set_language_display(language_display.as_string().utf16_string_view()); // b. Set typeFields to typeFields.[[]]. // c. Assert: typeFields is a Record (see 12.2.3). diff --git a/Libraries/LibJS/Runtime/Intl/DisplayNamesPrototype.cpp b/Libraries/LibJS/Runtime/Intl/DisplayNamesPrototype.cpp index fafe52c10b..b827db8c0e 100644 --- a/Libraries/LibJS/Runtime/Intl/DisplayNamesPrototype.cpp +++ b/Libraries/LibJS/Runtime/Intl/DisplayNamesPrototype.cpp @@ -77,8 +77,8 @@ JS_DEFINE_NATIVE_FUNCTION(DisplayNamesPrototype::of) code = PrimitiveString::create(vm, TRY(code.to_utf16_string(vm))); // 4. Let code be ? CanonicalCodeForDisplayNames(displayNames.[[Type]], code). - code = TRY(canonical_code_for_display_names(vm, display_names->type(), code.as_string().utf16_string_view().to_utf8_but_should_be_ported_to_utf16())); - auto code_string = code.as_string().utf16_string_view().to_utf8_but_should_be_ported_to_utf16(); + code = TRY(canonical_code_for_display_names(vm, display_names->type(), code.as_string().utf16_string_view())); + auto code_string = MUST(code.as_string().utf16_string_view().to_utf8()); // 5. Let fields be displayNames.[[Fields]]. // 6. If fields has a field [[]], return fields.[[]]. diff --git a/Libraries/LibJS/Runtime/Intl/DurationFormat.cpp b/Libraries/LibJS/Runtime/Intl/DurationFormat.cpp index 8c25c1f1b5..b2dc11818e 100644 --- a/Libraries/LibJS/Runtime/Intl/DurationFormat.cpp +++ b/Libraries/LibJS/Runtime/Intl/DurationFormat.cpp @@ -50,7 +50,7 @@ ReadonlySpan DurationFormat::resolution_option_descr return *descriptors; } -DurationFormat::Style DurationFormat::style_from_string(StringView style) +DurationFormat::Style DurationFormat::style_from_string(Utf16View style) { if (style == "long"sv) return Style::Long; @@ -79,7 +79,7 @@ StringView DurationFormat::style_to_string(Style style) } } -DurationFormat::Display DurationFormat::display_from_string(StringView display) +DurationFormat::Display DurationFormat::display_from_string(Utf16View display) { if (display == "auto"sv) return Display::Auto; @@ -88,7 +88,7 @@ DurationFormat::Display DurationFormat::display_from_string(StringView display) VERIFY_NOT_REACHED(); } -DurationFormat::ValueStyle DurationFormat::value_style_from_string(StringView value_style) +DurationFormat::ValueStyle DurationFormat::value_style_from_string(Utf16View value_style) { if (value_style == "long"sv) return ValueStyle::Long; @@ -269,7 +269,7 @@ ThrowCompletionOr get_duration_unit_options display_default = "auto"sv; } } else { - style = DurationFormat::value_style_from_string(style_value.as_string().utf16_string_view().to_utf8_but_should_be_ported_to_utf16()); + style = DurationFormat::value_style_from_string(style_value.as_string().utf16_string_view()); } // 4. If style is "numeric" and IsFractionalSecondUnitName(unit) is true, then @@ -286,7 +286,7 @@ ThrowCompletionOr get_duration_unit_options // 6. Let display be ? GetOption(options, displayField, STRING, « "auto", "always" », displayDefault). auto display_value = TRY(get_option(vm, options, display_field, OptionType::String, { "auto"sv, "always"sv }, display_default)); - auto display = DurationFormat::display_from_string(display_value.as_string().utf16_string_view().to_utf8_but_should_be_ported_to_utf16()); + auto display = DurationFormat::display_from_string(display_value.as_string().utf16_string_view()); // 7. Perform ? ValidateDurationUnitStyle(unit, style, display, prevStyle). TRY(validate_duration_unit_style(vm, unit_property_key, style, display, previous_style, display_field)); @@ -690,7 +690,8 @@ Vector format_numeric_units(VM& vm, DurationFormat const& du // 18. If secondsFormatted is true, then if (seconds_formatted) { // a. Let secondsParts be FormatNumericSeconds(durationFormat, secondsValue, minutesFormatted, signDisplayed). - auto seconds_parts = format_numeric_seconds(vm, duration_format, MathematicalValue { seconds_value.to_string(9) }, minutes_formatted, sign_displayed); + auto seconds_value_mv = MathematicalValue { Utf16String::from_utf8(seconds_value.to_string(9)) }; + auto seconds_parts = format_numeric_seconds(vm, duration_format, seconds_value_mv, minutes_formatted, sign_displayed); // b. Set numericPartsList to the list-concatenation of numericPartsList and secondsParts. numeric_parts_list.extend(move(seconds_parts)); @@ -883,7 +884,7 @@ Vector partition_duration_format_pattern(VM& vm, DurationFor // iii. If display is "always" or value is not 0, then if (display == DurationFormat::Display::Always || !value.is_zero()) { - MathematicalValue value_mv { value.to_string(9) }; + auto value_mv = MathematicalValue { Utf16String::from_utf8(value.to_string(9)) }; // 1. Perform ! CreateDataPropertyOrThrow(nfOpts, "numberingSystem", durationFormat.[[NumberingSystem]]). MUST(number_format_options->create_data_property_or_throw(vm.names.numberingSystem, PrimitiveString::create(vm, duration_format.numbering_system()))); diff --git a/Libraries/LibJS/Runtime/Intl/DurationFormat.h b/Libraries/LibJS/Runtime/Intl/DurationFormat.h index bd5c9e100c..8da8fc81f9 100644 --- a/Libraries/LibJS/Runtime/Intl/DurationFormat.h +++ b/Libraries/LibJS/Runtime/Intl/DurationFormat.h @@ -9,6 +9,7 @@ #include #include +#include #include #include #include @@ -27,7 +28,7 @@ public: Narrow, Digital, }; - static Style style_from_string(StringView style); + static Style style_from_string(Utf16View style); static StringView style_to_string(Style); enum class ValueStyle { @@ -38,7 +39,7 @@ public: TwoDigit, Fractional, }; - static ValueStyle value_style_from_string(StringView); + static ValueStyle value_style_from_string(Utf16View); static StringView value_style_to_string(ValueStyle); static_assert(to_underlying(ValueStyle::Long) == to_underlying(Unicode::Style::Long)); @@ -49,7 +50,7 @@ public: Auto, Always, }; - static Display display_from_string(StringView display); + static Display display_from_string(Utf16View display); static StringView display_to_string(Display); enum class Unit { @@ -88,7 +89,7 @@ public: void set_minute_second_separator(Utf16String minute_second_separator) { m_minute_second_separator = move(minute_second_separator); } Utf16String const& minute_second_separator() const { return m_minute_second_separator; } - void set_style(StringView style) { m_style = style_from_string(style); } + void set_style(Utf16View style) { m_style = style_from_string(style); } Style style() const { return m_style; } StringView style_string() const { return style_to_string(m_style); } diff --git a/Libraries/LibJS/Runtime/Intl/DurationFormatConstructor.cpp b/Libraries/LibJS/Runtime/Intl/DurationFormatConstructor.cpp index c8ae799b1d..c904f21c0e 100644 --- a/Libraries/LibJS/Runtime/Intl/DurationFormatConstructor.cpp +++ b/Libraries/LibJS/Runtime/Intl/DurationFormatConstructor.cpp @@ -83,7 +83,7 @@ ThrowCompletionOr> DurationFormatConstructor::construct(Function auto style = TRY(get_option(vm, *options, vm.names.style, OptionType::String, { "long"sv, "short"sv, "narrow"sv, "digital"sv }, "short"sv)); // 13. Set durationFormat.[[Style]] to style. - duration_format->set_style(style.as_string().utf16_string_view().to_utf8_but_should_be_ported_to_utf16()); + duration_format->set_style(style.as_string().utf16_string_view()); // 14. Let prevStyle be the empty String. Optional previous_style; diff --git a/Libraries/LibJS/Runtime/Intl/ListFormat.h b/Libraries/LibJS/Runtime/Intl/ListFormat.h index 8d1c486141..1cdea60398 100644 --- a/Libraries/LibJS/Runtime/Intl/ListFormat.h +++ b/Libraries/LibJS/Runtime/Intl/ListFormat.h @@ -8,6 +8,7 @@ #include #include +#include #include #include #include @@ -37,10 +38,12 @@ public: Unicode::ListFormatType type() const { return m_type; } void set_type(StringView type) { m_type = Unicode::list_format_type_from_string(type); } + void set_type(Utf16View type) { m_type = Unicode::list_format_type_from_string(type); } StringView type_string() const { return Unicode::list_format_type_to_string(m_type); } Unicode::Style style() const { return m_style; } void set_style(StringView style) { m_style = Unicode::style_from_string(style); } + void set_style(Utf16View style) { m_style = Unicode::style_from_string(style); } StringView style_string() const { return Unicode::style_to_string(m_style); } Unicode::ListFormat const& formatter() const { return *m_formatter; } diff --git a/Libraries/LibJS/Runtime/Intl/ListFormatConstructor.cpp b/Libraries/LibJS/Runtime/Intl/ListFormatConstructor.cpp index d15de16036..d8edc70336 100644 --- a/Libraries/LibJS/Runtime/Intl/ListFormatConstructor.cpp +++ b/Libraries/LibJS/Runtime/Intl/ListFormatConstructor.cpp @@ -66,13 +66,13 @@ ThrowCompletionOr> ListFormatConstructor::construct(FunctionObje auto type = TRY(get_option(vm, *options, vm.names.type, OptionType::String, { "conjunction"sv, "disjunction"sv, "unit"sv }, "conjunction"sv)); // 8. Set listFormat.[[Type]] to type. - list_format->set_type(type.as_string().utf16_string_view().to_utf8_but_should_be_ported_to_utf16()); + list_format->set_type(type.as_string().utf16_string_view()); // 9. Let style be ? GetOption(options, "style", string, « "long", "short", "narrow" », "long"). auto style = TRY(get_option(vm, *options, vm.names.style, OptionType::String, { "long"sv, "short"sv, "narrow"sv }, "long"sv)); // 10. Set listFormat.[[Style]] to style. - list_format->set_style(style.as_string().utf16_string_view().to_utf8_but_should_be_ported_to_utf16()); + list_format->set_style(style.as_string().utf16_string_view()); // 11. Let resolvedLocaleData be r.[[LocaleData]]. // 12. Let dataLocaleTypes be resolvedLocaleData.[[]]. diff --git a/Libraries/LibJS/Runtime/Intl/LocaleConstructor.cpp b/Libraries/LibJS/Runtime/Intl/LocaleConstructor.cpp index cd4d5d4ac6..d52f4405b2 100644 --- a/Libraries/LibJS/Runtime/Intl/LocaleConstructor.cpp +++ b/Libraries/LibJS/Runtime/Intl/LocaleConstructor.cpp @@ -28,8 +28,28 @@ struct LocaleAndKeys { Optional nu; }; +static bool is_unicode_language_subtag(Utf16View subtag) +{ + return Unicode::is_unicode_language_subtag(subtag); +} + +static bool is_unicode_script_subtag(Utf16View subtag) +{ + return Unicode::is_unicode_script_subtag(subtag); +} + +static bool is_unicode_region_subtag(Utf16View subtag) +{ + return Unicode::is_unicode_region_subtag(subtag); +} + +static bool is_type_identifier(Utf16View identifier) +{ + return Unicode::is_type_identifier(identifier); +} + // NOTE: This is not an AO in the spec. This just serves to abstract very similar steps in UpdateLanguageId and the Intl.Locale constructor. -static ThrowCompletionOr> get_string_option(VM& vm, Object const& options, PropertyKey const& property, Function validator, ReadonlySpan values = {}, Optional const& fallback = {}) +static ThrowCompletionOr> get_string_option(VM& vm, Object const& options, PropertyKey const& property, Function validator, ReadonlySpan values = {}, Optional const& fallback = {}) { auto option_default = fallback.has_value() ? OptionDefault { *fallback } : Empty {}; @@ -37,10 +57,14 @@ static ThrowCompletionOr> get_string_option(VM& vm, Object cons if (option.is_undefined()) return OptionalNone {}; - if (validator && !validator(option.as_string().utf16_string_view().to_utf8_but_should_be_ported_to_utf16())) + auto option_string = option.as_string().utf16_string_view(); + if (!option_string.is_ascii()) return vm.throw_completion(ErrorType::OptionIsNotValidValue, option, property); - return option.as_string().utf16_string_view().to_utf8_but_should_be_ported_to_utf16(); + if (validator && !validator(option_string)) + return vm.throw_completion(ErrorType::OptionIsNotValidValue, option, property); + + return MUST(option_string.to_utf8()); } // 15.1.2 UpdateLanguageId ( tag, options ), https://tc39.es/ecma402/#sec-updatelanguageid @@ -54,17 +78,17 @@ static ThrowCompletionOr update_language_id(VM& vm, StringView tag, Obje // 2. Let language be ? GetOption(options, "language", STRING, EMPTY, GetLocaleLanguage(baseName)). // 3. If language cannot be matched by the unicode_language_subtag Unicode locale nonterminal, throw a RangeError exception. - auto language = TRY(get_string_option(vm, options, vm.names.language, Unicode::is_unicode_language_subtag, {}, *base_name.language)); + auto language = TRY(get_string_option(vm, options, vm.names.language, is_unicode_language_subtag, {}, *base_name.language)); // 4. Let script be ? GetOption(options, "script", STRING, EMPTY, GetLocaleScript(baseName)). // 5. If script is not undefined, then // a. If script cannot be matched by the unicode_script_subtag Unicode locale nonterminal, throw a RangeError exception. - auto script = TRY(get_string_option(vm, options, vm.names.script, Unicode::is_unicode_script_subtag, {}, base_name.script)); + auto script = TRY(get_string_option(vm, options, vm.names.script, is_unicode_script_subtag, {}, base_name.script)); // 6. Let region be ? GetOption(options, "region", STRING, EMPTY, GetLocaleRegion(baseName)). // 7. If region is not undefined, then // a. If region cannot be matched by the unicode_region_subtag Unicode locale nonterminal, throw a RangeError exception. - auto region = TRY(get_string_option(vm, options, vm.names.region, Unicode::is_unicode_region_subtag, {}, base_name.region)); + auto region = TRY(get_string_option(vm, options, vm.names.region, is_unicode_region_subtag, {}, base_name.region)); // 8. Let variants be ? GetOption(options, "variants", STRING, EMPTY, GetLocaleVariants(baseName)). auto variants = TRY(get_string_option(vm, options, vm.names.variants, nullptr, {}, get_locale_variants(*locale_id))); @@ -279,21 +303,33 @@ ThrowCompletionOr> LocaleConstructor::construct(FunctionObject& if (!tag_value.is_string() && !tag_value.is_object()) return vm.throw_completion(ErrorType::NotAnObjectOrString, "tag"sv); - auto tag = TRY([&]() -> ThrowCompletionOr { - // 8. If tag is an Object and tag has an [[InitializedLocale]] internal slot, then - // a. Let tag be tag.[[Locale]]. - if (auto locale_tag = tag_value.as_if()) - return locale_tag->locale(); - // 9. Else, - // a. Let tag be ? ToString(tag). - return TRY(tag_value.to_utf16_string(vm)).to_utf8_but_should_be_ported_to_utf16(); - }()); + String tag; + bool tag_is_canonicalized = false; + + // 8. If tag is an Object and tag has an [[InitializedLocale]] internal slot, then + // a. Let tag be tag.[[Locale]]. + if (auto locale_tag = tag_value.as_if()) { + tag = locale_tag->locale(); + } + // 9. Else, + else { + // a. Let tag be ? ToString(tag). + auto tag_string = TRY(tag_value.to_utf16_string(vm)); + + // 11. If IsWellFormedLanguageTag(tag) is false, throw a RangeError exception. + if (!is_well_formed_language_tag(tag_string.utf16_view())) + return vm.throw_completion(ErrorType::IntlInvalidLanguageTag, tag_string); + + // 13. Set tag to CanonicalizeUnicodeLocaleId(tag). + tag = canonicalize_unicode_locale_id(tag_string.utf16_view()); + tag_is_canonicalized = true; + } // 10. Set options to ? CoerceOptionsToObject(options). auto options = TRY(coerce_options_to_object(vm, options_value)); // 11. If IsWellFormedLanguageTag(tag) is false, throw a RangeError exception. - if (!is_well_formed_language_tag(tag)) + if (!tag_is_canonicalized && !is_well_formed_language_tag(tag)) return vm.throw_completion(ErrorType::IntlInvalidLanguageTag, tag); // 12. NOTE: Because LanguageId canonicalization can alter tag in arbitrary ways according to Alias Rules from @@ -301,7 +337,8 @@ ThrowCompletionOr> LocaleConstructor::construct(FunctionObject& // options. // 13. Set tag to CanonicalizeUnicodeLocaleId(tag). - tag = canonicalize_unicode_locale_id(tag); + if (!tag_is_canonicalized) + tag = canonicalize_unicode_locale_id(tag); // 14. Set tag to ? UpdateLanguageId(tag, options). tag = TRY(update_language_id(vm, tag, options)); @@ -313,13 +350,13 @@ ThrowCompletionOr> LocaleConstructor::construct(FunctionObject& // 17. If calendar is not undefined, then // a. If calendar cannot be matched by the type Unicode locale nonterminal, throw a RangeError exception. // 18. Set opt.[[ca]] to calendar. - opt.ca = TRY(get_string_option(vm, options, vm.names.calendar, Unicode::is_type_identifier)); + opt.ca = TRY(get_string_option(vm, options, vm.names.calendar, is_type_identifier)); // 19. Let collation be ? GetOption(options, "collation", STRING, EMPTY, undefined). // 20. If collation is not undefined, then // a. If collation cannot be matched by the type Unicode locale nonterminal, throw a RangeError exception. // 21. Set opt.[[co]] to collation. - opt.co = TRY(get_string_option(vm, options, vm.names.collation, Unicode::is_type_identifier)); + opt.co = TRY(get_string_option(vm, options, vm.names.collation, is_type_identifier)); // 22. Let fw be ? GetOption(options, "firstDayOfWeek", STRING, EMPTY, undefined). auto first_day_of_week = TRY(get_string_option(vm, options, vm.names.firstDayOfWeek, nullptr)); @@ -351,13 +388,13 @@ ThrowCompletionOr> LocaleConstructor::construct(FunctionObject& // 30. If kn is not undefined, set kn to ! ToString(kn). // 31. Set opt.[[kn]] to kn. if (!kn.is_undefined()) - opt.kn = TRY(kn.to_utf16_string(vm)).to_utf8_but_should_be_ported_to_utf16(); + opt.kn = kn.as_bool() ? "true"_string : "false"_string; // 32. Let numberingSystem be ? GetOption(options, "numberingSystem", STRING, EMPTY, undefined). // 33. If numberingSystem is not undefined, then // a. If numberingSystem cannot be matched by the type Unicode locale nonterminal, throw a RangeError exception. // 34. Set opt.[[nu]] to numberingSystem. - opt.nu = TRY(get_string_option(vm, options, vm.names.numberingSystem, Unicode::is_type_identifier)); + opt.nu = TRY(get_string_option(vm, options, vm.names.numberingSystem, is_type_identifier)); // 35. Let r be MakeLocaleRecord(tag, opt, localeExtensionKeys). auto result = make_locale_record(tag, move(opt), locale_extension_keys); diff --git a/Libraries/LibJS/Runtime/Intl/MathematicalValue.cpp b/Libraries/LibJS/Runtime/Intl/MathematicalValue.cpp index e35187092b..4d0170e0b6 100644 --- a/Libraries/LibJS/Runtime/Intl/MathematicalValue.cpp +++ b/Libraries/LibJS/Runtime/Intl/MathematicalValue.cpp @@ -21,13 +21,13 @@ double MathematicalValue::as_number() const bool MathematicalValue::is_string() const { - return m_value.has(); + return m_value.has(); } -String const& MathematicalValue::as_string() const +Utf16String const& MathematicalValue::as_string() const { VERIFY(is_string()); - return m_value.get(); + return m_value.get(); } bool MathematicalValue::is_mathematical_value() const @@ -69,7 +69,7 @@ Unicode::NumberFormat::Value MathematicalValue::to_value() const [](double value) -> Unicode::NumberFormat::Value { return value; }, - [](String const& value) -> Unicode::NumberFormat::Value { + [](Utf16String const& value) -> Unicode::NumberFormat::Value { return value; }, [](auto symbol) -> Unicode::NumberFormat::Value { diff --git a/Libraries/LibJS/Runtime/Intl/MathematicalValue.h b/Libraries/LibJS/Runtime/Intl/MathematicalValue.h index 23f5451d48..e687f25224 100644 --- a/Libraries/LibJS/Runtime/Intl/MathematicalValue.h +++ b/Libraries/LibJS/Runtime/Intl/MathematicalValue.h @@ -7,6 +7,7 @@ #pragma once #include +#include #include #include #include @@ -33,7 +34,7 @@ public: { } - explicit MathematicalValue(String value) + explicit MathematicalValue(Utf16String value) : m_value(move(value)) { } @@ -46,7 +47,7 @@ public: MathematicalValue(Value value) : m_value(value.is_number() ? value_from_number(value.as_double()) - : ValueType(MUST(value.as_bigint().big_integer().to_base(10)))) + : ValueType(Utf16String::from_utf8(MUST(value.as_bigint().big_integer().to_base(10))))) { } @@ -54,7 +55,7 @@ public: double as_number() const; bool is_string() const; - String const& as_string() const; + Utf16String const& as_string() const; bool is_mathematical_value() const; bool is_positive_infinity() const; @@ -65,7 +66,7 @@ public: Unicode::NumberFormat::Value to_value() const; private: - using ValueType = Variant; + using ValueType = Variant; static ValueType value_from_number(double number); diff --git a/Libraries/LibJS/Runtime/Intl/NumberFormat.cpp b/Libraries/LibJS/Runtime/Intl/NumberFormat.cpp index 3fe3c9a2ed..898c319dbf 100644 --- a/Libraries/LibJS/Runtime/Intl/NumberFormat.cpp +++ b/Libraries/LibJS/Runtime/Intl/NumberFormat.cpp @@ -210,7 +210,7 @@ ThrowCompletionOr to_intl_mathematical_value(VM& vm, Value va // 2. If Type(primValue) is BigInt, return the mathematical value of primValue. if (primitive_value.is_bigint()) - return MUST(value.as_bigint().big_integer().to_base(10)); + return Utf16String::from_utf8(MUST(value.as_bigint().big_integer().to_base(10))); // FIXME: The remaining steps are being refactored into a new Runtime Semantic, StringIntlMV. // We short-circuit some of these steps to avoid known pitfalls. @@ -222,7 +222,7 @@ ThrowCompletionOr to_intl_mathematical_value(VM& vm, Value va // 3. If Type(primValue) is String, // a. Let str be primValue. - auto string = primitive_value.as_string().utf16_string_view().to_utf8_but_should_be_ported_to_utf16(); + auto string = primitive_value.as_string().utf16_string_view(); // Step 4 handled separately by the FIXME above. @@ -234,7 +234,7 @@ ThrowCompletionOr to_intl_mathematical_value(VM& vm, Value va return MathematicalValue::Symbol::NotANumber; // 7. If mv is 0 and the first non white space code point in str is -, return negative-zero. - if (mathematical_value == 0.0 && string.bytes_as_string_view().trim_whitespace(TrimMode::Left).starts_with('-')) + if (mathematical_value == 0.0 && string.trim_ascii_whitespace(TrimMode::Left).starts_with('-')) return MathematicalValue::Symbol::NegativeZero; // 8. If mv is 10^10000 and str contains Infinity, return positive-infinity. @@ -246,7 +246,7 @@ ThrowCompletionOr to_intl_mathematical_value(VM& vm, Value va return MathematicalValue::Symbol::NegativeInfinity; // 10. Return mv. - return string; + return Utf16String::from_utf16(string); } // 16.5.19 PartitionNumberRangePattern ( numberFormat, x, y ), https://tc39.es/ecma402/#sec-partitionnumberrangepattern diff --git a/Libraries/LibJS/Runtime/Intl/NumberFormat.h b/Libraries/LibJS/Runtime/Intl/NumberFormat.h index e88ddcf2ae..d1743ebe57 100644 --- a/Libraries/LibJS/Runtime/Intl/NumberFormat.h +++ b/Libraries/LibJS/Runtime/Intl/NumberFormat.h @@ -8,6 +8,7 @@ #include #include +#include #include #include #include @@ -56,11 +57,13 @@ public: Unicode::Notation notation() const { return m_notation; } StringView notation_string() const { return Unicode::notation_to_string(m_notation); } void set_notation(StringView notation) { m_notation = Unicode::notation_from_string(notation); } + void set_notation(Utf16View notation) { m_notation = Unicode::notation_from_string(notation); } bool has_compact_display() const { return m_compact_display.has_value(); } Unicode::CompactDisplay compact_display() const { return *m_compact_display; } StringView compact_display_string() const { return Unicode::compact_display_to_string(*m_compact_display); } void set_compact_display(StringView compact_display) { m_compact_display = Unicode::compact_display_from_string(compact_display); } + void set_compact_display(Utf16View compact_display) { m_compact_display = Unicode::compact_display_from_string(compact_display); } Unicode::RoundingType rounding_type() const { return m_rounding_type; } StringView rounding_type_string() const { return Unicode::rounding_type_to_string(m_rounding_type); } @@ -73,6 +76,7 @@ public: Unicode::RoundingMode rounding_mode() const { return m_rounding_mode; } StringView rounding_mode_string() const { return Unicode::rounding_mode_to_string(m_rounding_mode); } void set_rounding_mode(StringView rounding_mode) { m_rounding_mode = Unicode::rounding_mode_from_string(rounding_mode); } + void set_rounding_mode(Utf16View rounding_mode) { m_rounding_mode = Unicode::rounding_mode_from_string(rounding_mode); } int rounding_increment() const { return m_rounding_increment; } void set_rounding_increment(int rounding_increment) { m_rounding_increment = rounding_increment; } @@ -80,6 +84,7 @@ public: Unicode::TrailingZeroDisplay trailing_zero_display() const { return m_trailing_zero_display; } StringView trailing_zero_display_string() const { return Unicode::trailing_zero_display_to_string(m_trailing_zero_display); } void set_trailing_zero_display(StringView trailing_zero_display) { m_trailing_zero_display = Unicode::trailing_zero_display_from_string(trailing_zero_display); } + void set_trailing_zero_display(Utf16View trailing_zero_display) { m_trailing_zero_display = Unicode::trailing_zero_display_from_string(trailing_zero_display); } virtual Unicode::DisplayOptions display_options() const; Unicode::RoundingOptions rounding_options() const; @@ -125,6 +130,7 @@ public: Unicode::NumberFormatStyle style() const { return m_style; } StringView style_string() const { return Unicode::number_format_style_to_string(m_style); } void set_style(StringView style) { m_style = Unicode::number_format_style_from_string(style); } + void set_style(Utf16View style) { m_style = Unicode::number_format_style_from_string(style); } bool has_currency() const { return m_currency.has_value(); } String const& currency() const { return m_currency.value(); } @@ -134,11 +140,13 @@ public: Unicode::CurrencyDisplay currency_display() const { return *m_currency_display; } StringView currency_display_string() const { return Unicode::currency_display_to_string(*m_currency_display); } void set_currency_display(StringView currency_display) { m_currency_display = Unicode::currency_display_from_string(currency_display); } + void set_currency_display(Utf16View currency_display) { m_currency_display = Unicode::currency_display_from_string(currency_display); } bool has_currency_sign() const { return m_currency_sign.has_value(); } Unicode::CurrencySign currency_sign() const { return *m_currency_sign; } StringView currency_sign_string() const { return Unicode::currency_sign_to_string(*m_currency_sign); } void set_currency_sign(StringView currency_sign) { m_currency_sign = Unicode::currency_sign_from_string(currency_sign); } + void set_currency_sign(Utf16View currency_sign) { m_currency_sign = Unicode::currency_sign_from_string(currency_sign); } bool has_unit() const { return m_unit.has_value(); } String const& unit() const { return m_unit.value(); } @@ -148,6 +156,7 @@ public: Unicode::Style unit_display() const { return *m_unit_display; } StringView unit_display_string() const { return Unicode::style_to_string(*m_unit_display); } void set_unit_display(StringView unit_display) { m_unit_display = Unicode::style_from_string(unit_display); } + void set_unit_display(Utf16View unit_display) { m_unit_display = Unicode::style_from_string(unit_display); } Unicode::Grouping use_grouping() const { return m_use_grouping; } Value use_grouping_to_value(VM&) const; @@ -156,6 +165,7 @@ public: Unicode::SignDisplay sign_display() const { return m_sign_display; } StringView sign_display_string() const { return Unicode::sign_display_to_string(m_sign_display); } void set_sign_display(StringView sign_display) { m_sign_display = Unicode::sign_display_from_string(sign_display); } + void set_sign_display(Utf16View sign_display) { m_sign_display = Unicode::sign_display_from_string(sign_display); } NativeFunction* bound_format() const { return m_bound_format; } void set_bound_format(NativeFunction* bound_format) { m_bound_format = bound_format; } diff --git a/Libraries/LibJS/Runtime/Intl/NumberFormatConstructor.cpp b/Libraries/LibJS/Runtime/Intl/NumberFormatConstructor.cpp index e54efad7c9..28ea596077 100644 --- a/Libraries/LibJS/Runtime/Intl/NumberFormatConstructor.cpp +++ b/Libraries/LibJS/Runtime/Intl/NumberFormatConstructor.cpp @@ -4,6 +4,7 @@ * SPDX-License-Identifier: BSD-2-Clause */ +#include #include #include #include @@ -15,6 +16,19 @@ namespace JS::Intl { GC_DEFINE_ALLOCATOR(NumberFormatConstructor); +static String ascii_uppercase_currency_code(Utf16View currency) +{ + VERIFY(currency.length_in_code_units() == 3); + + char code[3]; + for (size_t i = 0; i < currency.length_in_code_units(); ++i) { + VERIFY(is_ascii_alpha(currency.code_unit_at(i))); + code[i] = static_cast(to_ascii_uppercase(currency.code_unit_at(i))); + } + + return String::from_ascii_short_string_without_validation(code, 3); +} + // 16.1 The Intl.NumberFormat Constructor, https://tc39.es/ecma402/#sec-intl-numberformat-constructor NumberFormatConstructor::NumberFormatConstructor(Realm& realm) : NativeFunction(realm.vm().names.NumberFormat.as_string(), realm.intrinsics().function_prototype()) @@ -78,7 +92,7 @@ ThrowCompletionOr> NumberFormatConstructor::construct(FunctionOb auto notation = TRY(get_option(vm, *options, vm.names.notation, OptionType::String, { "standard"sv, "scientific"sv, "engineering"sv, "compact"sv }, "standard"sv)); // 12. Set numberFormat.[[Notation]] to notation. - number_format->set_notation(notation.as_string().utf16_string_view().to_utf8_but_should_be_ported_to_utf16()); + number_format->set_notation(notation.as_string().utf16_string_view()); int default_min_fraction_digits = 0; int default_max_fraction_digits = 0; @@ -121,7 +135,7 @@ ThrowCompletionOr> NumberFormatConstructor::construct(FunctionOb // 18. If notation is "compact", then if (number_format->notation() == Unicode::Notation::Compact) { // a. Set numberFormat.[[CompactDisplay]] to compactDisplay. - number_format->set_compact_display(compact_display.as_string().utf16_string_view().to_utf8_but_should_be_ported_to_utf16()); + number_format->set_compact_display(compact_display.as_string().utf16_string_view()); // b. Set defaultUseGrouping to "min2". default_use_grouping = "min2"sv; @@ -150,7 +164,7 @@ ThrowCompletionOr> NumberFormatConstructor::construct(FunctionOb auto sign_display = TRY(get_option(vm, *options, vm.names.signDisplay, OptionType::String, { "auto"sv, "never"sv, "always"sv, "exceptZero"sv, "negative"sv }, "auto"sv)); // 25. Set numberFormat.[[SignDisplay]] to signDisplay. - number_format->set_sign_display(sign_display.as_string().utf16_string_view().to_utf8_but_should_be_ported_to_utf16()); + number_format->set_sign_display(sign_display.as_string().utf16_string_view()); // 26. If the implementation supports the normative optional constructor mode of 4.3 Note 1, then // a. Let this be the this value. @@ -202,7 +216,7 @@ ThrowCompletionOr set_number_format_digit_options(VM& vm, NumberFormatBase // 10. Let roundingPriority be ? GetOption(options, "roundingPriority", STRING, « "auto", "morePrecision", "lessPrecision" », "auto"). auto rounding_priority_option = TRY(get_option(vm, options, vm.names.roundingPriority, OptionType::String, { "auto"sv, "morePrecision"sv, "lessPrecision"sv }, "auto"sv)); - auto rounding_priority = rounding_priority_option.as_string().utf16_string_view().to_utf8_but_should_be_ported_to_utf16(); + auto rounding_priority = rounding_priority_option.as_string().utf16_string_view(); // 11. Let trailingZeroDisplay be ? GetOption(options, "trailingZeroDisplay", STRING, « "auto", "stripIfInteger" », "auto"). auto trailing_zero_display = TRY(get_option(vm, options, vm.names.trailingZeroDisplay, OptionType::String, { "auto"sv, "stripIfInteger"sv }, "auto"sv)); @@ -217,10 +231,10 @@ ThrowCompletionOr set_number_format_digit_options(VM& vm, NumberFormatBase intl_object.set_rounding_increment(*rounding_increment); // 15. Set intlObj.[[RoundingMode]] to roundingMode. - intl_object.set_rounding_mode(rounding_mode.as_string().utf16_string_view().to_utf8_but_should_be_ported_to_utf16()); + intl_object.set_rounding_mode(rounding_mode.as_string().utf16_string_view()); // 16. Set intlObj.[[TrailingZeroDisplay]] to trailingZeroDisplay. - intl_object.set_trailing_zero_display(trailing_zero_display.as_string().utf16_string_view().to_utf8_but_should_be_ported_to_utf16()); + intl_object.set_trailing_zero_display(trailing_zero_display.as_string().utf16_string_view()); // 17. If mnsd is undefined and mxsd is undefined, let hasSd be false. Otherwise, let hasSd be true. bool has_significant_digits = !min_significant_digits.is_undefined() || !max_significant_digits.is_undefined(); @@ -379,7 +393,7 @@ ThrowCompletionOr set_number_format_unit_options(VM& vm, NumberFormat& int auto style = TRY(get_option(vm, options, vm.names.style, OptionType::String, { "decimal"sv, "percent"sv, "currency"sv, "unit"sv }, "decimal"sv)); // 2. Set intlObj.[[Style]] to style. - intl_object.set_style(style.as_string().utf16_string_view().to_utf8_but_should_be_ported_to_utf16()); + intl_object.set_style(style.as_string().utf16_string_view()); // 3. Let currency be ? GetOption(options, "currency", STRING, EMPTY, undefined). auto currency = TRY(get_option(vm, options, vm.names.currency, OptionType::String, {}, Empty {})); @@ -392,7 +406,7 @@ ThrowCompletionOr set_number_format_unit_options(VM& vm, NumberFormat& int } // 5. Else, // a. If IsWellFormedCurrencyCode(currency) is false, throw a RangeError exception. - else if (!is_well_formed_currency_code(currency.as_string().utf16_string_view().to_utf8_but_should_be_ported_to_utf16())) { + else if (!is_well_formed_currency_code(currency.as_string().utf16_string_view())) { return vm.throw_completion(ErrorType::OptionIsNotValidValue, currency, "currency"sv); } @@ -413,7 +427,7 @@ ThrowCompletionOr set_number_format_unit_options(VM& vm, NumberFormat& int } // 10. Else, // a. If IsWellFormedUnitIdentifier(unit) is false, throw a RangeError exception. - else if (!is_well_formed_unit_identifier(unit.as_string().utf16_string_view().to_utf8_but_should_be_ported_to_utf16())) { + else if (!is_well_formed_unit_identifier(unit.as_string().utf16_string_view())) { return vm.throw_completion(ErrorType::OptionIsNotValidValue, unit, "unit"sv); } @@ -423,22 +437,22 @@ ThrowCompletionOr set_number_format_unit_options(VM& vm, NumberFormat& int // 12. If style is "currency", then if (intl_object.style() == Unicode::NumberFormatStyle::Currency) { // a. Set intlObj.[[Currency]] to the ASCII-uppercase of currency. - intl_object.set_currency(MUST(currency.as_string().utf16_string_view().to_utf8_but_should_be_ported_to_utf16().to_uppercase())); + intl_object.set_currency(ascii_uppercase_currency_code(currency.as_string().utf16_string_view())); // c. Set intlObj.[[CurrencyDisplay]] to currencyDisplay. - intl_object.set_currency_display(currency_display.as_string().utf16_string_view().to_utf8_but_should_be_ported_to_utf16()); + intl_object.set_currency_display(currency_display.as_string().utf16_string_view()); // d. Set intlObj.[[CurrencySign]] to currencySign. - intl_object.set_currency_sign(currency_sign.as_string().utf16_string_view().to_utf8_but_should_be_ported_to_utf16()); + intl_object.set_currency_sign(currency_sign.as_string().utf16_string_view()); } // 13. If style is "unit", then if (intl_object.style() == Unicode::NumberFormatStyle::Unit) { // a. Set intlObj.[[Unit]] to unit. - intl_object.set_unit(unit.as_string().utf16_string_view().to_utf8_but_should_be_ported_to_utf16()); + intl_object.set_unit(MUST(unit.as_string().utf16_string_view().to_utf8())); // b. Set intlObj.[[UnitDisplay]] to unitDisplay. - intl_object.set_unit_display(unit_display.as_string().utf16_string_view().to_utf8_but_should_be_ported_to_utf16()); + intl_object.set_unit_display(unit_display.as_string().utf16_string_view()); } // 14. Return UNUSED. diff --git a/Libraries/LibJS/Runtime/Intl/PluralRules.h b/Libraries/LibJS/Runtime/Intl/PluralRules.h index 8fb19d88b1..8ae1d885b4 100644 --- a/Libraries/LibJS/Runtime/Intl/PluralRules.h +++ b/Libraries/LibJS/Runtime/Intl/PluralRules.h @@ -7,6 +7,7 @@ #pragma once #include +#include #include #include #include @@ -27,6 +28,7 @@ public: Unicode::PluralForm type() const { return m_type; } StringView type_string() const { return Unicode::plural_form_to_string(m_type); } void set_type(StringView type) { m_type = Unicode::plural_form_from_string(type); } + void set_type(Utf16View type) { m_type = Unicode::plural_form_from_string(type); } private: explicit PluralRules(Object& prototype); diff --git a/Libraries/LibJS/Runtime/Intl/PluralRulesConstructor.cpp b/Libraries/LibJS/Runtime/Intl/PluralRulesConstructor.cpp index 7b5f9bf2c6..4389e549a5 100644 --- a/Libraries/LibJS/Runtime/Intl/PluralRulesConstructor.cpp +++ b/Libraries/LibJS/Runtime/Intl/PluralRulesConstructor.cpp @@ -67,13 +67,13 @@ ThrowCompletionOr> PluralRulesConstructor::construct(FunctionObj auto type = TRY(get_option(vm, *options, vm.names.type, OptionType::String, AK::Array { "cardinal"sv, "ordinal"sv }, "cardinal"sv)); // 8. Set pluralRules.[[Type]] to t. - plural_rules->set_type(type.as_string().utf16_string_view().to_utf8_but_should_be_ported_to_utf16()); + plural_rules->set_type(type.as_string().utf16_string_view()); // 9. Let notation be ? GetOption(options, "notation", string, « "standard", "scientific", "engineering", "compact" », "standard"). auto notation = TRY(get_option(vm, *options, vm.names.notation, OptionType::String, { "standard"sv, "scientific"sv, "engineering"sv, "compact"sv }, "standard"sv)); // 10. Set pluralRules.[[Notation]] to notation. - plural_rules->set_notation(notation.as_string().utf16_string_view().to_utf8_but_should_be_ported_to_utf16()); + plural_rules->set_notation(notation.as_string().utf16_string_view()); // 11. Let compactDisplay be ? GetOption(options, "compactDisplay", string, « "short", "long" », "short"). auto compact_display = TRY(get_option(vm, *options, vm.names.compactDisplay, OptionType::String, { "short"sv, "long"sv }, "short"sv)); @@ -81,7 +81,7 @@ ThrowCompletionOr> PluralRulesConstructor::construct(FunctionObj // 12. If notation is "compact", then if (plural_rules->notation() == Unicode::Notation::Compact) { // a. Set pluralRules.[[CompactDisplay]] to compactDisplay. - plural_rules->set_compact_display(compact_display.as_string().utf16_string_view().to_utf8_but_should_be_ported_to_utf16()); + plural_rules->set_compact_display(compact_display.as_string().utf16_string_view()); } // 13. Perform ? SetNumberFormatDigitOptions(pluralRules, options, 0, 3, notation). diff --git a/Libraries/LibJS/Runtime/Intl/RelativeTimeFormat.cpp b/Libraries/LibJS/Runtime/Intl/RelativeTimeFormat.cpp index 9cd8e19891..80ce3e3299 100644 --- a/Libraries/LibJS/Runtime/Intl/RelativeTimeFormat.cpp +++ b/Libraries/LibJS/Runtime/Intl/RelativeTimeFormat.cpp @@ -43,7 +43,7 @@ ReadonlySpan RelativeTimeFormat::resolution_option_d } // 18.5.1 SingularRelativeTimeUnit ( unit ), https://tc39.es/ecma402/#sec-singularrelativetimeunit -ThrowCompletionOr singular_relative_time_unit(VM& vm, StringView unit) +ThrowCompletionOr singular_relative_time_unit(VM& vm, Utf16View unit) { // 1. If unit is "seconds", return "second". if (unit == "seconds"sv) @@ -78,7 +78,7 @@ ThrowCompletionOr singular_relative_time_unit(VM& vm, StringV } // 18.5.2 PartitionRelativeTimePattern ( relativeTimeFormat, value, unit ), https://tc39.es/ecma402/#sec-PartitionRelativeTimePattern -ThrowCompletionOr> partition_relative_time_pattern(VM& vm, RelativeTimeFormat& relative_time_format, double value, StringView unit) +ThrowCompletionOr> partition_relative_time_pattern(VM& vm, RelativeTimeFormat& relative_time_format, double value, Utf16View unit) { // 1. If value is NaN, +∞𝔽, or -∞𝔽, throw a RangeError exception. if (!Value(value).is_finite_number()) @@ -91,7 +91,7 @@ ThrowCompletionOr> partition_rela } // 18.5.4 FormatRelativeTime ( relativeTimeFormat, value, unit ), https://tc39.es/ecma402/#sec-FormatRelativeTime -ThrowCompletionOr format_relative_time(VM& vm, RelativeTimeFormat& relative_time_format, double value, StringView unit) +ThrowCompletionOr format_relative_time(VM& vm, RelativeTimeFormat& relative_time_format, double value, Utf16View unit) { // 1. Let parts be ? PartitionRelativeTimePattern(relativeTimeFormat, value, unit). auto time_unit = TRY([&]() -> ThrowCompletionOr { @@ -114,7 +114,7 @@ ThrowCompletionOr format_relative_time(VM& vm, RelativeTimeFormat& } // 18.5.5 FormatRelativeTimeToParts ( relativeTimeFormat, value, unit ), https://tc39.es/ecma402/#sec-FormatRelativeTimeToParts -ThrowCompletionOr> format_relative_time_to_parts(VM& vm, RelativeTimeFormat& relative_time_format, double value, StringView unit) +ThrowCompletionOr> format_relative_time_to_parts(VM& vm, RelativeTimeFormat& relative_time_format, double value, Utf16View unit) { auto& realm = *vm.current_realm(); diff --git a/Libraries/LibJS/Runtime/Intl/RelativeTimeFormat.h b/Libraries/LibJS/Runtime/Intl/RelativeTimeFormat.h index bfeb1f19ce..eb7f713f82 100644 --- a/Libraries/LibJS/Runtime/Intl/RelativeTimeFormat.h +++ b/Libraries/LibJS/Runtime/Intl/RelativeTimeFormat.h @@ -8,6 +8,7 @@ #include #include +#include #include #include #include @@ -33,10 +34,12 @@ public: Unicode::Style style() const { return m_style; } void set_style(StringView style) { m_style = Unicode::style_from_string(style); } + void set_style(Utf16View style) { m_style = Unicode::style_from_string(style); } StringView style_string() const { return Unicode::style_to_string(m_style); } Unicode::NumericDisplay numeric() const { return m_numeric; } void set_numeric(StringView numeric) { m_numeric = Unicode::numeric_display_from_string(numeric); } + void set_numeric(Utf16View numeric) { m_numeric = Unicode::numeric_display_from_string(numeric); } StringView numeric_string() const { return Unicode::numeric_display_to_string(m_numeric); } Unicode::RelativeTimeFormat const& formatter() const { return *m_formatter; } @@ -54,9 +57,9 @@ private: OwnPtr m_formatter; }; -ThrowCompletionOr singular_relative_time_unit(VM&, StringView unit); -ThrowCompletionOr> partition_relative_time_pattern(VM&, RelativeTimeFormat&, double value, StringView unit); -ThrowCompletionOr format_relative_time(VM&, RelativeTimeFormat&, double value, StringView unit); -ThrowCompletionOr> format_relative_time_to_parts(VM&, RelativeTimeFormat&, double value, StringView unit); +ThrowCompletionOr singular_relative_time_unit(VM&, Utf16View unit); +ThrowCompletionOr> partition_relative_time_pattern(VM&, RelativeTimeFormat&, double value, Utf16View unit); +ThrowCompletionOr format_relative_time(VM&, RelativeTimeFormat&, double value, Utf16View unit); +ThrowCompletionOr> format_relative_time_to_parts(VM&, RelativeTimeFormat&, double value, Utf16View unit); } diff --git a/Libraries/LibJS/Runtime/Intl/RelativeTimeFormatConstructor.cpp b/Libraries/LibJS/Runtime/Intl/RelativeTimeFormatConstructor.cpp index d4607b5a1b..576e0fe791 100644 --- a/Libraries/LibJS/Runtime/Intl/RelativeTimeFormatConstructor.cpp +++ b/Libraries/LibJS/Runtime/Intl/RelativeTimeFormatConstructor.cpp @@ -75,13 +75,13 @@ ThrowCompletionOr> RelativeTimeFormatConstructor::construct(Func auto style = TRY(get_option(vm, *options, vm.names.style, OptionType::String, { "long"sv, "short"sv, "narrow"sv }, "long"sv)); // 11. Set relativeTimeFormat.[[Style]] to style. - relative_time_format->set_style(style.as_string().utf16_string_view().to_utf8_but_should_be_ported_to_utf16()); + relative_time_format->set_style(style.as_string().utf16_string_view()); // 12. Let numeric be ? GetOption(options, "numeric", STRING, « "always", "auto" », "always"). auto numeric = TRY(get_option(vm, *options, vm.names.numeric, OptionType::String, { "always"sv, "auto"sv }, "always"sv)); // 13. Set relativeTimeFormat.[[Numeric]] to numeric. - relative_time_format->set_numeric(numeric.as_string().utf16_string_view().to_utf8_but_should_be_ported_to_utf16()); + relative_time_format->set_numeric(numeric.as_string().utf16_string_view()); // 14. Let nfOptions be OrdinaryObjectCreate(null). // 15. Perform ! CreateDataPropertyOrThrow(nfOptions, "numberingSystem", relativeTimeFormat.[[NumberingSystem]]). diff --git a/Libraries/LibJS/Runtime/Intl/RelativeTimeFormatPrototype.cpp b/Libraries/LibJS/Runtime/Intl/RelativeTimeFormatPrototype.cpp index a7ffd8b05c..7c9a6671ae 100644 --- a/Libraries/LibJS/Runtime/Intl/RelativeTimeFormatPrototype.cpp +++ b/Libraries/LibJS/Runtime/Intl/RelativeTimeFormatPrototype.cpp @@ -71,10 +71,10 @@ JS_DEFINE_NATIVE_FUNCTION(RelativeTimeFormatPrototype::format) auto value = TRY(vm.argument(0).to_number(vm)); // 4. Let unit be ? ToString(unit). - auto unit = TRY(vm.argument(1).to_utf16_string(vm)).to_utf8_but_should_be_ported_to_utf16(); + auto unit = TRY(vm.argument(1).to_utf16_string(vm)); // 5. Return ? FormatRelativeTime(relativeTimeFormat, value, unit). - auto formatted = TRY(format_relative_time(vm, relative_time_format, value.as_double(), unit)); + auto formatted = TRY(format_relative_time(vm, relative_time_format, value.as_double(), unit.utf16_view())); return PrimitiveString::create(vm, move(formatted)); } @@ -89,10 +89,10 @@ JS_DEFINE_NATIVE_FUNCTION(RelativeTimeFormatPrototype::format_to_parts) auto value = TRY(vm.argument(0).to_number(vm)); // 4. Let unit be ? ToString(unit). - auto unit = TRY(vm.argument(1).to_utf16_string(vm)).to_utf8_but_should_be_ported_to_utf16(); + auto unit = TRY(vm.argument(1).to_utf16_string(vm)); // 5. Return ? FormatRelativeTimeToParts(relativeTimeFormat, value, unit). - return TRY(format_relative_time_to_parts(vm, relative_time_format, value.as_double(), unit)); + return TRY(format_relative_time_to_parts(vm, relative_time_format, value.as_double(), unit.utf16_view())); } } diff --git a/Libraries/LibJS/Runtime/Intl/Segmenter.h b/Libraries/LibJS/Runtime/Intl/Segmenter.h index fe1b373b2e..5b6da25cc4 100644 --- a/Libraries/LibJS/Runtime/Intl/Segmenter.h +++ b/Libraries/LibJS/Runtime/Intl/Segmenter.h @@ -8,6 +8,7 @@ #pragma once #include +#include #include #include @@ -28,6 +29,7 @@ public: Unicode::SegmenterGranularity segmenter_granularity() const { return m_segmenter_granularity; } void set_segmenter_granularity(StringView segmenter_granularity) { m_segmenter_granularity = Unicode::segmenter_granularity_from_string(segmenter_granularity); } + void set_segmenter_granularity(Utf16View segmenter_granularity) { m_segmenter_granularity = Unicode::segmenter_granularity_from_string(segmenter_granularity); } StringView segmenter_granularity_string() const { return Unicode::segmenter_granularity_to_string(m_segmenter_granularity); } Unicode::Segmenter const& segmenter() const { return *m_segmenter; } diff --git a/Libraries/LibJS/Runtime/Intl/SegmenterConstructor.cpp b/Libraries/LibJS/Runtime/Intl/SegmenterConstructor.cpp index 7768cc020f..c48feb2fec 100644 --- a/Libraries/LibJS/Runtime/Intl/SegmenterConstructor.cpp +++ b/Libraries/LibJS/Runtime/Intl/SegmenterConstructor.cpp @@ -67,7 +67,7 @@ ThrowCompletionOr> SegmenterConstructor::construct(FunctionObjec auto granularity = TRY(get_option(vm, *options, vm.names.granularity, OptionType::String, { "grapheme"sv, "word"sv, "sentence"sv }, "grapheme"sv)); // 9. Set segmenter.[[SegmenterGranularity]] to granularity. - segmenter->set_segmenter_granularity(granularity.as_string().utf16_string_view().to_utf8_but_should_be_ported_to_utf16()); + segmenter->set_segmenter_granularity(granularity.as_string().utf16_string_view()); auto locale_segmenter = Unicode::Segmenter::create(segmenter->locale(), segmenter->segmenter_granularity()); segmenter->set_segmenter(move(locale_segmenter)); diff --git a/Libraries/LibJS/Runtime/JSONObject.cpp b/Libraries/LibJS/Runtime/JSONObject.cpp index a65eba362f..3d5d98df63 100644 --- a/Libraries/LibJS/Runtime/JSONObject.cpp +++ b/Libraries/LibJS/Runtime/JSONObject.cpp @@ -10,7 +10,6 @@ #include #include #include -#include #include #include #include @@ -445,7 +444,7 @@ JS_DEFINE_NATIVE_FUNCTION(JSONObject::parse) auto reviver = vm.argument(1); // 1. Let jsonString be ? ToString(text). - auto json_string = TRY(text.to_utf16_string(vm)).to_utf8_but_should_be_ported_to_utf16(); + auto json_string = TRY(text.to_utf16_string(vm)); // 2. Let parseResult be ? ParseJSON(jsonString). // 3. Let unfiltered be parseResult.[[Value]]. @@ -868,20 +867,28 @@ static ThrowCompletionOr parse_simdjson_document(VM& vm, simdjson::ondema VERIFY_NOT_REACHED(); } +static StringView utf8_json_text_bytes(Utf16View text, Optional& utf8_text) +{ + if (text.has_ascii_storage()) + return { text.bytes() }; + + utf8_text = MUST(text.to_utf8()); + return utf8_text->bytes_as_string_view(); +} + // 25.5.1.1 ParseJSON ( text ), https://tc39.es/ecma262/#sec-ParseJSON -ThrowCompletionOr JSONObject::parse_json(VM& vm, StringView text, JSONParseRecord* root_record) +ThrowCompletionOr JSONObject::parse_json(VM& vm, Utf16View text, JSONParseRecord* root_record) { // 1. If StringToCodePoints(text) is not a valid JSON text as specified in ECMA-404, throw a SyntaxError exception. // NB: Per ECMA-404, the BOM is not valid JSON whitespace. simdjson silently skips it, so we must reject it explicitly. - if (text.length() >= 3 - && static_cast(text[0]) == 0xEF - && static_cast(text[1]) == 0xBB - && static_cast(text[2]) == 0xBF) { + if (text.length_in_code_units() >= 1 && text.code_unit_at(0) == 0xFEFF) return vm.throw_completion(ErrorType::JsonMalformed); - } + + Optional utf8_text; + auto text_bytes = utf8_json_text_bytes(text, utf8_text); simdjson::ondemand::parser parser; - simdjson::padded_string padded(text.characters_without_null_termination(), text.length()); + simdjson::padded_string padded(text_bytes.characters_without_null_termination(), text_bytes.length()); simdjson::ondemand::document document; if (parser.iterate(padded).get(document)) @@ -996,18 +1003,18 @@ JS_DEFINE_NATIVE_FUNCTION(JSONObject::raw_json) auto& realm = *vm.current_realm(); // 1. Let jsonString be ? ToString(text). - auto json_string = TRY(vm.argument(0).to_utf16_string(vm)).to_utf8_but_should_be_ported_to_utf16(); + auto json_string = TRY(vm.argument(0).to_utf16_string(vm)); // 2. Throw a SyntaxError exception if jsonString is the empty String, or if either the first or last code unit of // jsonString is any of 0x0009 (CHARACTER TABULATION), 0x000A (LINE FEED), 0x000D (CARRIAGE RETURN), or // 0x0020 (SPACE). - auto bytes = json_string.bytes_as_string_view(); - if (bytes.is_empty()) + auto json_string_view = json_string.utf16_view(); + if (json_string_view.is_empty()) return vm.throw_completion(ErrorType::JsonMalformed); static constexpr AK::Array invalid_code_points { 0x09, 0x0A, 0x0D, 0x20 }; - auto first_char = bytes[0]; - auto last_char = bytes[bytes.length() - 1]; + auto first_char = json_string_view.code_unit_at(0); + auto last_char = json_string_view.code_unit_at(json_string_view.length_in_code_units() - 1); if (invalid_code_points.contains_slow(first_char) || invalid_code_points.contains_slow(last_char)) return vm.throw_completion(ErrorType::JsonMalformed); @@ -1015,8 +1022,11 @@ JS_DEFINE_NATIVE_FUNCTION(JSONObject::raw_json) // 3. Parse StringToCodePoints(jsonString) as a JSON text as specified in ECMA-404. Throw a SyntaxError exception // if it is not a valid JSON text as defined in that specification, or if its outermost value is an object or // array as defined in that specification. + Optional utf8_text; + auto text_bytes = utf8_json_text_bytes(json_string_view, utf8_text); + simdjson::ondemand::parser parser; - simdjson::padded_string padded(json_string.bytes_as_string_view().characters_without_null_termination(), json_string.bytes_as_string_view().length()); + simdjson::padded_string padded(text_bytes.characters_without_null_termination(), text_bytes.length()); simdjson::ondemand::document doc; if (parser.iterate(padded).get(doc)) diff --git a/Libraries/LibJS/Runtime/JSONObject.h b/Libraries/LibJS/Runtime/JSONObject.h index 15a17a8669..801e8e2701 100644 --- a/Libraries/LibJS/Runtime/JSONObject.h +++ b/Libraries/LibJS/Runtime/JSONObject.h @@ -41,7 +41,7 @@ public: // test-js to communicate between the JS tests and the C++ test runner. static ThrowCompletionOr> stringify_impl(VM&, Value value, Value replacer, Value space); - static ThrowCompletionOr parse_json(VM&, StringView text, JSONParseRecord* root_record = nullptr); + static ThrowCompletionOr parse_json(VM&, Utf16View text, JSONParseRecord* root_record = nullptr); private: explicit JSONObject(Realm&); diff --git a/Libraries/LibJS/Runtime/ObjectPrototype.cpp b/Libraries/LibJS/Runtime/ObjectPrototype.cpp index ac6dfa14f9..1b8828edf9 100644 --- a/Libraries/LibJS/Runtime/ObjectPrototype.cpp +++ b/Libraries/LibJS/Runtime/ObjectPrototype.cpp @@ -148,61 +148,61 @@ JS_DEFINE_NATIVE_FUNCTION(ObjectPrototype::to_string) // 4. Let isArray be ? IsArray(O). auto is_array = TRY(Value(object).is_array(vm)); - StringView builtin_tag; + Utf16View builtin_tag; // 5. If isArray is true, let builtinTag be "Array". if (is_array) - builtin_tag = "Array"sv; + builtin_tag = u"Array"sv; // 6. Else if O has a [[ParameterMap]] internal slot, let builtinTag be "Arguments". else if (object->has_parameter_map()) - builtin_tag = "Arguments"sv; + builtin_tag = u"Arguments"sv; // 7. Else if O has a [[Call]] internal method, let builtinTag be "Function". else if (object->is_function()) - builtin_tag = "Function"sv; + builtin_tag = u"Function"sv; // 8. Else if O has an [[ErrorData]] internal slot, let builtinTag be "Error". else if (object->has_error_data()) - builtin_tag = "Error"sv; + builtin_tag = u"Error"sv; // 9. Else if O has a [[BooleanData]] internal slot, let builtinTag be "Boolean". else if (is(*object)) - builtin_tag = "Boolean"sv; + builtin_tag = u"Boolean"sv; // 10. Else if O has a [[NumberData]] internal slot, let builtinTag be "Number". else if (is(*object)) - builtin_tag = "Number"sv; + builtin_tag = u"Number"sv; // 11. Else if O has a [[StringData]] internal slot, let builtinTag be "String". else if (is(*object)) - builtin_tag = "String"sv; + builtin_tag = u"String"sv; // 12. Else if O has a [[DateValue]] internal slot, let builtinTag be "Date". else if (is(*object)) - builtin_tag = "Date"sv; + builtin_tag = u"Date"sv; // 13. Else if O has a [[RegExpMatcher]] internal slot, let builtinTag be "RegExp". else if (is(*object)) - builtin_tag = "RegExp"sv; + builtin_tag = u"RegExp"sv; // 14. Else, let builtinTag be "Object". else - builtin_tag = "Object"sv; + builtin_tag = u"Object"sv; // 15. Let tag be ? Get(O, @@toStringTag). static auto& cache = *new Bytecode::StaticPropertyLookupCache; auto to_string_tag = TRY(object->get(vm.well_known_symbol_to_string_tag(), cache)); // Optimization: Instead of creating another PrimitiveString from builtin_tag, we separate tag and to_string_tag and add an additional branch to step 16. - StringView tag; - String custom_tag; + Utf16View tag; + Utf16String custom_tag; // 16. If Type(tag) is not String, set tag to builtinTag. if (!to_string_tag.is_string()) tag = builtin_tag; else { - custom_tag = to_string_tag.as_string().utf16_string_view().to_utf8_but_should_be_ported_to_utf16(); - tag = custom_tag; + custom_tag = to_string_tag.as_string().utf16_string(); + tag = custom_tag.utf16_view(); } // 17. Return the string-concatenation of "[object ", tag, and "]". // OPTIMIZATION: The VM has a cache for the extremely common "[object Object]" string. - if (tag == "Object"sv) + if (tag == u"Object"sv) return vm.cached_strings.object_Object; - return PrimitiveString::create(vm, MUST(String::formatted("[object {}]", tag))); + return PrimitiveString::create(vm, Utf16String::formatted("[object {}]", tag)); } // 20.1.3.7 Object.prototype.valueOf ( ), https://tc39.es/ecma262/#sec-object.prototype.valueof @@ -262,7 +262,7 @@ JS_DEFINE_NATIVE_FUNCTION(ObjectPrototype::define_getter) // 2. If IsCallable(getter) is false, throw a TypeError exception. if (!getter.is_function()) - return vm.throw_completion(ErrorType::NotAFunction, getter.to_string_without_side_effects()); + return vm.throw_completion(ErrorType::NotAFunction, getter.to_utf16_string_without_side_effects()); // 3. Let desc be PropertyDescriptor { [[Get]]: getter, [[Enumerable]]: true, [[Configurable]]: true }. auto descriptor = PropertyDescriptor { .get = &getter.as_function(), .enumerable = true, .configurable = true }; @@ -288,7 +288,7 @@ JS_DEFINE_NATIVE_FUNCTION(ObjectPrototype::define_setter) // 2. If IsCallable(setter) is false, throw a TypeError exception. if (!setter.is_function()) - return vm.throw_completion(ErrorType::NotAFunction, setter.to_string_without_side_effects()); + return vm.throw_completion(ErrorType::NotAFunction, setter.to_utf16_string_without_side_effects()); // 3. Let desc be PropertyDescriptor { [[Set]]: setter, [[Enumerable]]: true, [[Configurable]]: true }. auto descriptor = PropertyDescriptor { .set = &setter.as_function(), .enumerable = true, .configurable = true }; diff --git a/Libraries/LibJS/Runtime/PropertyDescriptor.h b/Libraries/LibJS/Runtime/PropertyDescriptor.h index 75e1f6b0ac..54d85498b9 100644 --- a/Libraries/LibJS/Runtime/PropertyDescriptor.h +++ b/Libraries/LibJS/Runtime/PropertyDescriptor.h @@ -54,25 +54,25 @@ public: namespace AK { template<> -struct Formatter : Formatter { +struct Formatter : Formatter { ErrorOr format(FormatBuilder& builder, JS::PropertyDescriptor const& property_descriptor) { - Vector parts; + Vector parts; if (property_descriptor.value.has_value()) - TRY(parts.try_append(TRY(String::formatted("[[Value]]: {}", property_descriptor.value->to_string_without_side_effects())))); + TRY(parts.try_append(Utf16String::formatted("[[Value]]: {}", property_descriptor.value->to_utf16_string_without_side_effects()))); if (property_descriptor.get.has_value()) - TRY(parts.try_append(TRY(String::formatted("[[Get]]: JS::Function* @ {:p}", property_descriptor.get->ptr())))); + TRY(parts.try_append(Utf16String::formatted("[[Get]]: JS::Function* @ {:p}", property_descriptor.get->ptr()))); if (property_descriptor.set.has_value()) - TRY(parts.try_append(TRY(String::formatted("[[Set]]: JS::Function* @ {:p}", property_descriptor.set->ptr())))); + TRY(parts.try_append(Utf16String::formatted("[[Set]]: JS::Function* @ {:p}", property_descriptor.set->ptr()))); if (property_descriptor.writable.has_value()) - TRY(parts.try_append(TRY(String::formatted("[[Writable]]: {}", *property_descriptor.writable)))); + TRY(parts.try_append(Utf16String::formatted("[[Writable]]: {}", *property_descriptor.writable))); if (property_descriptor.enumerable.has_value()) - TRY(parts.try_append(TRY(String::formatted("[[Enumerable]]: {}", *property_descriptor.enumerable)))); + TRY(parts.try_append(Utf16String::formatted("[[Enumerable]]: {}", *property_descriptor.enumerable))); if (property_descriptor.configurable.has_value()) - TRY(parts.try_append(TRY(String::formatted("[[Configurable]]: {}", *property_descriptor.configurable)))); + TRY(parts.try_append(Utf16String::formatted("[[Configurable]]: {}", *property_descriptor.configurable))); if (property_descriptor.unimplemented.has_value()) - TRY(parts.try_append(TRY(String::formatted("[[Unimplemented]]: {}", *property_descriptor.unimplemented)))); - return Formatter::format(builder, TRY(String::formatted("PropertyDescriptor {{ {} }}", TRY(String::join(", "sv, parts))))); + TRY(parts.try_append(Utf16String::formatted("[[Unimplemented]]: {}", *property_descriptor.unimplemented))); + return Formatter {}.format(builder, Utf16String::formatted("PropertyDescriptor {{ {} }}", Utf16String::join(", "sv, parts))); } }; diff --git a/Libraries/LibJS/Runtime/PropertyKey.h b/Libraries/LibJS/Runtime/PropertyKey.h index 38b74617be..2b9105655d 100644 --- a/Libraries/LibJS/Runtime/PropertyKey.h +++ b/Libraries/LibJS/Runtime/PropertyKey.h @@ -239,7 +239,7 @@ struct Formatter : Formatter { { if (property_key.is_number()) return builder.put_u64(property_key.as_number()); - return Formatter::format(builder, property_key.to_string()); + return Formatter {}.format(builder, property_key.to_string()); } }; diff --git a/Libraries/LibJS/Runtime/RegExpConstructor.cpp b/Libraries/LibJS/Runtime/RegExpConstructor.cpp index a1510cf829..1e9d9e1b52 100644 --- a/Libraries/LibJS/Runtime/RegExpConstructor.cpp +++ b/Libraries/LibJS/Runtime/RegExpConstructor.cpp @@ -221,11 +221,11 @@ static Utf16String encode_for_regexp_escape(u32 code_point) // 3. Let otherPunctuators be the string-concatenation of ",-=<>#&!%:;@~'`" and the code unit 0x0022 (QUOTATION MARK). // 4. Let toEscape be StringToCodePoints(otherPunctuators). - static constexpr Utf8View to_escape { ",-=<>#&!%:;@~'`\""sv }; + static constexpr auto to_escape = ",-=<>#&!%:;@~'`\""sv; // 5. If toEscape contains c, c is matched by either WhiteSpace or LineTerminator, or c has the same numeric value // as a leading surrogate or trailing surrogate, then - if (to_escape.contains(code_point) || is_whitespace(code_point) || is_line_terminator(code_point) || is_unicode_surrogate(code_point)) { + if ((is_ascii(code_point) && to_escape.contains(static_cast(code_point))) || is_whitespace(code_point) || is_line_terminator(code_point) || is_unicode_surrogate(code_point)) { // a. Let cNum be the numeric value of c. // b. If cNum ≤ 0xFF, then if (code_point <= 0xFF) { diff --git a/Libraries/LibJS/Runtime/StringIterator.cpp b/Libraries/LibJS/Runtime/StringIterator.cpp index 0f12058fe1..97cbf48e4c 100644 --- a/Libraries/LibJS/Runtime/StringIterator.cpp +++ b/Libraries/LibJS/Runtime/StringIterator.cpp @@ -4,7 +4,6 @@ * SPDX-License-Identifier: BSD-2-Clause */ -#include #include #include #include @@ -13,15 +12,15 @@ namespace JS { GC_DEFINE_ALLOCATOR(StringIterator); -GC::Ref StringIterator::create(Realm& realm, String string) +GC::Ref StringIterator::create(Realm& realm, Utf16String string) { return realm.create(move(string), realm.intrinsics().string_iterator_prototype()); } -StringIterator::StringIterator(String string, Object& prototype) +StringIterator::StringIterator(Utf16String string, Object& prototype) : Object(ConstructWithPrototypeTag::Tag, prototype) , m_string(move(string)) - , m_iterator(Utf8View(m_string).begin()) + , m_iterator(m_string.begin()) { } @@ -42,14 +41,14 @@ ThrowCompletionOr StringIterator::next(VM& vm, bool& done, Value& value) return {}; } - if (m_iterator.done()) { + if (m_iterator == m_string.end()) { m_done = true; done = true; value = js_undefined(); return {}; } - auto code_point = String::from_code_point(*m_iterator); + auto code_point = Utf16String::from_code_point(*m_iterator); ++m_iterator; value = PrimitiveString::create(vm, move(code_point)); diff --git a/Libraries/LibJS/Runtime/StringIterator.h b/Libraries/LibJS/Runtime/StringIterator.h index e73cb35cbe..31590136fc 100644 --- a/Libraries/LibJS/Runtime/StringIterator.h +++ b/Libraries/LibJS/Runtime/StringIterator.h @@ -6,8 +6,7 @@ #pragma once -#include -#include +#include #include #include @@ -19,7 +18,7 @@ class StringIterator final : public Object GC_DECLARE_ALLOCATOR(StringIterator); public: - static GC::Ref create(Realm&, String string); + static GC::Ref create(Realm&, Utf16String string); virtual ~StringIterator() override = default; @@ -27,12 +26,12 @@ public: ThrowCompletionOr next(VM&, bool& done, Value& value) override; private: - explicit StringIterator(String string, Object& prototype); + explicit StringIterator(Utf16String string, Object& prototype); friend class StringIteratorPrototype; - String m_string; - Utf8CodePointIterator m_iterator; + Utf16String m_string; + AK::Utf16CodePointIterator m_iterator; bool m_done { false }; }; diff --git a/Libraries/LibJS/Runtime/StringPrototype.cpp b/Libraries/LibJS/Runtime/StringPrototype.cpp index 23447708a8..43115d4e6d 100644 --- a/Libraries/LibJS/Runtime/StringPrototype.cpp +++ b/Libraries/LibJS/Runtime/StringPrototype.cpp @@ -37,12 +37,6 @@ namespace JS { GC_DEFINE_ALLOCATOR(StringPrototype); -static ThrowCompletionOr utf8_string_from(VM& vm) -{ - auto this_value = TRY(require_object_coercible(vm, vm.this_value())); - return TRY(this_value.to_utf16_string(vm)).to_utf8_but_should_be_ported_to_utf16(); -} - static ThrowCompletionOr> primitive_string_from(VM& vm) { auto this_value = TRY(require_object_coercible(vm, vm.this_value())); @@ -704,26 +698,26 @@ JS_DEFINE_NATIVE_FUNCTION(StringPrototype::normalize) { // 1. Let O be ? RequireObjectCoercible(this value). // 2. Let S be ? ToString(O). - auto string = TRY(utf8_string_from(vm)); + auto string = TRY(primitive_string_from(vm)); - String form; + Utf16String form; // 3. If form is undefined, let f be "NFC". if (auto form_value = vm.argument(0); form_value.is_undefined()) { - form = "NFC"_string; + form = "NFC"_utf16; } // 4. Else, let f be ? ToString(form). else { - form = TRY(form_value.to_utf16_string(vm)).to_utf8_but_should_be_ported_to_utf16(); + form = TRY(form_value.to_utf16_string(vm)); } // 5. If f is not one of "NFC", "NFD", "NFKC", or "NFKD", throw a RangeError exception. - if (!form.is_one_of("NFC"sv, "NFD"sv, "NFKC"sv, "NFKD"sv)) + if (!form.utf16_view().is_one_of(u"NFC"sv, u"NFD"sv, u"NFKC"sv, u"NFKD"sv)) return vm.throw_completion(ErrorType::InvalidNormalizationForm, form); // 6. Let ns be the String value that is the result of normalizing S into the normalization form named by f as specified in https://unicode.org/reports/tr15/. - auto unicode_form = Unicode::normalization_form_from_string(form); - auto ns = Unicode::normalize(string, unicode_form); + auto unicode_form = Unicode::normalization_form_from_string(form.utf16_view()); + auto ns = Unicode::normalize(string->utf16_string_view(), unicode_form); // 7. Return ns. return PrimitiveString::create(vm, move(ns)); @@ -824,11 +818,11 @@ JS_DEFINE_NATIVE_FUNCTION(StringPrototype::repeat) // 5. If n = 0, return the empty String. if (n == 0) - return PrimitiveString::create(vm, String {}); + return PrimitiveString::create(vm, Utf16String {}); // OPTIMIZATION: If the string is empty, the result will be empty as well. if (string->is_empty()) - return PrimitiveString::create(vm, String {}); + return PrimitiveString::create(vm, Utf16String {}); if (n > static_cast(NumericLimits::max())) return vm.throw_completion(ErrorType::StringRepeatCountMustNotOverflow); @@ -1348,7 +1342,7 @@ enum class TargetCase { }; // 20.1.2.1 TransformCase ( S, locales, targetCase ), https://tc39.es/ecma402/#sec-transform-case -static ThrowCompletionOr transform_case(VM& vm, String const& string, Value locales, TargetCase target_case) +static ThrowCompletionOr transform_case(VM& vm, Utf16String const& string, Value locales, TargetCase target_case) { // 1. Let requestedLocales be ? CanonicalizeLocaleList(locales). auto requested_locales = TRY(Intl::canonicalize_locale_list(vm, locales)); @@ -1375,19 +1369,19 @@ static ThrowCompletionOr transform_case(VM& vm, String const& string, Va // 7. Let codePoints be StringToCodePoints(S). - String new_code_points; + Utf16String new_code_points; switch (target_case) { // 8. If targetCase is lower, then case TargetCase::Lower: // a. Let newCodePoints be a List whose elements are the result of a lowercase transformation of codePoints according to an implementation-derived algorithm using locale or the Unicode Default Case Conversion algorithm. - new_code_points = MUST(string.to_lowercase(locale)); + new_code_points = string.to_lowercase(locale); break; // 9. Else, case TargetCase::Upper: // a. Assert: targetCase is upper. // b. Let newCodePoints be a List whose elements are the result of an uppercase transformation of codePoints according to an implementation-derived algorithm using locale or the Unicode Default Case Conversion algorithm. - new_code_points = MUST(string.to_uppercase(locale)); + new_code_points = string.to_uppercase(locale); break; default: VERIFY_NOT_REACHED(); @@ -1405,10 +1399,10 @@ JS_DEFINE_NATIVE_FUNCTION(StringPrototype::to_locale_lowercase) // 1. Let O be ? RequireObjectCoercible(this value). // 2. Let S be ? ToString(O). - auto string = TRY(utf8_string_from(vm)); + auto string = TRY(primitive_string_from(vm)); // 3. Return ? TransformCase(S, locales, lower). - return PrimitiveString::create(vm, TRY(transform_case(vm, string, locales, TargetCase::Lower))); + return PrimitiveString::create(vm, TRY(transform_case(vm, string->utf16_string(), locales, TargetCase::Lower))); } // 22.1.3.27 String.prototype.toLocaleUpperCase ( [ reserved1 [ , reserved2 ] ] ), https://tc39.es/ecma262/#sec-string.prototype.tolocaleuppercase @@ -1419,10 +1413,10 @@ JS_DEFINE_NATIVE_FUNCTION(StringPrototype::to_locale_uppercase) // 1. Let O be ? RequireObjectCoercible(this value). // 2. Let S be ? ToString(O). - auto string = TRY(utf8_string_from(vm)); + auto string = TRY(primitive_string_from(vm)); // 3. Return ? TransformCase(S, locales, upper). - return PrimitiveString::create(vm, TRY(transform_case(vm, string, locales, TargetCase::Upper))); + return PrimitiveString::create(vm, TRY(transform_case(vm, string->utf16_string(), locales, TargetCase::Upper))); } // 22.1.3.28 String.prototype.toLowerCase ( ), https://tc39.es/ecma262/#sec-string.prototype.tolowercase @@ -1431,10 +1425,10 @@ JS_DEFINE_NATIVE_FUNCTION(StringPrototype::to_lowercase) // 1. Let O be ? RequireObjectCoercible(this value). // 2. Let S be ? ToString(O). // 3. Let sText be StringToCodePoints(S). - auto string = TRY(utf8_string_from(vm)); + auto string = TRY(primitive_string_from(vm)); // 4. Let lowerText be the result of toLowercase(sText), according to the Unicode Default Case Conversion algorithm. - auto lowercase = MUST(string.to_lowercase()); + auto lowercase = string->utf16_string().to_lowercase(); // 5. Let L be CodePointsToString(lowerText). // 6. Return L. @@ -1453,8 +1447,8 @@ JS_DEFINE_NATIVE_FUNCTION(StringPrototype::to_uppercase) { // This method interprets a String value as a sequence of UTF-16 encoded code points, as described in 6.1.4. // It behaves in exactly the same way as String.prototype.toLowerCase, except that the String is mapped using the toUppercase algorithm of the Unicode Default Case Conversion. - auto string = TRY(utf8_string_from(vm)); - auto uppercase = MUST(string.to_uppercase()); + auto string = TRY(primitive_string_from(vm)); + auto uppercase = string->utf16_string().to_uppercase(); return PrimitiveString::create(vm, move(uppercase)); } @@ -1537,7 +1531,7 @@ JS_DEFINE_NATIVE_FUNCTION(StringPrototype::symbol_iterator) auto this_object = TRY(require_object_coercible(vm, vm.this_value())); // 2. Let s be ? ToString(O). - auto string = TRY(this_object.to_utf16_string(vm)).to_utf8_but_should_be_ported_to_utf16(); + auto string = TRY(this_object.to_utf16_string(vm)); // 3. Let closure be a new Abstract Closure with no parameters that captures s and performs the following steps when called: // ... diff --git a/Libraries/LibJS/Runtime/Temporal/AbstractOperations.cpp b/Libraries/LibJS/Runtime/Temporal/AbstractOperations.cpp index 278f3dd509..d0d1c92189 100644 --- a/Libraries/LibJS/Runtime/Temporal/AbstractOperations.cpp +++ b/Libraries/LibJS/Runtime/Temporal/AbstractOperations.cpp @@ -480,7 +480,7 @@ ThrowCompletionOr get_temporal_relative_to_option(VM& vm, Object con String calendar; Optional time_zone; - Optional offset_string; + Optional offset_string; ISODate iso_date; Variant time { Time {} }; @@ -560,8 +560,7 @@ ThrowCompletionOr get_temporal_relative_to_option(VM& vm, Object con // f. Else, else { // i. Let timeZone be ? ToTemporalTimeZoneIdentifier(annotation). - auto utf16_annotation = Utf16String::from_utf8(*annotation); - time_zone = TRY(to_temporal_time_zone_identifier(vm, utf16_annotation)); + time_zone = TRY(to_temporal_time_zone_identifier(vm, *annotation)); // ii. If result.[[TimeZone]].[[Z]] is true, then if (result.time_zone.z_designator) { @@ -580,8 +579,7 @@ ThrowCompletionOr get_temporal_relative_to_option(VM& vm, Object con // v. If offsetString is not EMPTY, then if (offset_string.has_value()) { // 1. Let offsetParseResult be ParseText(StringToCodePoints(offsetString), UTCOffset[+SubMinutePrecision]). - auto utf16_offset_string = Utf16String::from_utf8(*offset_string); - auto offset_parse_result = parse_utc_offset(utf16_offset_string, SubMinutePrecision::Yes); + auto offset_parse_result = parse_utc_offset(*offset_string, SubMinutePrecision::Yes); // 2. Assert: offsetParseResult is a Parse Node. VERIFY(offset_parse_result.has_value()); @@ -594,10 +592,10 @@ ThrowCompletionOr get_temporal_relative_to_option(VM& vm, Object con // g. Let calendar be result.[[Calendar]]. // h. If calendar is EMPTY, set calendar to "iso8601". - calendar = result.calendar.value_or("iso8601"_string); - - // i. Set calendar to ? CanonicalizeCalendar(calendar). - calendar = TRY(canonicalize_calendar(vm, calendar)); + if (result.calendar.has_value()) + calendar = TRY(canonicalize_calendar(vm, *result.calendar)); + else + calendar = TRY(canonicalize_calendar(vm, "iso8601"sv)); // j. Let isoDate be CreateISODateRecord(result.[[Year]], result.[[Month]], result.[[Day]]). iso_date = create_iso_date_record(*result.year, result.month, result.day); @@ -620,7 +618,7 @@ ThrowCompletionOr get_temporal_relative_to_option(VM& vm, Object con // 8. If offsetBehaviour is OPTION, then if (offset_behavior == OffsetBehavior::Option) { // a. Let offsetNs be ! ParseDateTimeUTCOffset(offsetString). - offset_nanoseconds = parse_date_time_utc_offset(offset_string->bytes_as_string_view()); + offset_nanoseconds = parse_date_time_utc_offset(*offset_string); } // 9. Else, else { @@ -1103,7 +1101,7 @@ ThrowCompletionOr parse_iso_date_time(VM& vm, Utf16View iso_s Optional parse_result; // 2. Let calendar be EMPTY. - Optional calendar; + Optional calendar; // 3. Let yearAbsent be false. auto year_absent = false; @@ -1135,7 +1133,7 @@ ThrowCompletionOr parse_iso_date_time(VM& vm, Utf16View iso_s // i. If calendar is EMPTY, then if (!calendar.has_value()) { // i. Set calendar to CodePointsToString(value). - calendar = value.to_utf8_but_should_be_ported_to_utf16(); + calendar = Utf16String::from_utf16(value); // ii. If annotation contains an AnnotationCriticalFlag Parse Node, set calendarWasCritical to true. if (annotation.critical) @@ -1160,14 +1158,14 @@ ThrowCompletionOr parse_iso_date_time(VM& vm, Utf16View iso_s // 3. If goal is TemporalYearMonthString and parseResult does not contain a DateDay Parse Node, then if (goal == Production::TemporalYearMonthString && !parse_result->date_day.has_value()) { // a. If calendar is not EMPTY and the ASCII-lowercase of calendar is not "iso8601", throw a RangeError exception. - if (calendar.has_value() && !calendar->equals_ignoring_ascii_case(ISO8601_CALENDAR)) + if (calendar.has_value() && !calendar->equals_ignoring_ascii_case("iso8601"sv)) return vm.throw_completion(ErrorType::TemporalInvalidCalendarIdentifier, *calendar); } // 4. If goal is TemporalMonthDayString and parseResult does not contain a DateYear Parse Node, then if (goal == Production::TemporalMonthDayString && !parse_result->date_year.has_value()) { // a. If calendar is not EMPTY and the ASCII-lowercase of calendar is not "iso8601", throw a RangeError exception. - if (calendar.has_value() && !calendar->equals_ignoring_ascii_case(ISO8601_CALENDAR)) + if (calendar.has_value() && !calendar->equals_ignoring_ascii_case("iso8601"sv)) return vm.throw_completion(ErrorType::TemporalInvalidCalendarIdentifier, *calendar); // b. Set yearAbsent to true. @@ -1291,7 +1289,7 @@ ThrowCompletionOr parse_iso_date_time(VM& vm, Utf16View iso_s if (parse_result->time_zone_identifier.has_value()) { // a. Let identifier be the source text matched by the TimeZoneIdentifier Parse Node contained within parseResult. // b. Set timeZoneResult.[[TimeZoneAnnotation]] to CodePointsToString(identifier). - time_zone_result.time_zone_annotation = parse_result->time_zone_identifier->to_utf8_but_should_be_ported_to_utf16(); + time_zone_result.time_zone_annotation = Utf16String::from_utf16(*parse_result->time_zone_identifier); } // 26. If parseResult contains a UTCDesignator Parse Node, then @@ -1303,7 +1301,7 @@ ThrowCompletionOr parse_iso_date_time(VM& vm, Utf16View iso_s else if (parse_result->date_time_offset.has_value()) { // a. Let offset be the source text matched by the UTCOffset[+SubMinutePrecision] Parse Node contained within parseResult. // b. Set timeZoneResult.[[OffsetString]] to CodePointsToString(offset). - time_zone_result.offset_string = parse_result->date_time_offset->source_text.to_utf8_but_should_be_ported_to_utf16(); + time_zone_result.offset_string = Utf16String::from_utf16(parse_result->date_time_offset->source_text); } // 28. If yearAbsent is true, let yearReturn be EMPTY; else let yearReturn be yearMV. @@ -1316,7 +1314,7 @@ ThrowCompletionOr parse_iso_date_time(VM& vm, Utf16View iso_s } // 13.36 ParseTemporalCalendarString ( string ), https://tc39.es/proposal-temporal/#sec-temporal-parsetemporalcalendarstring -ThrowCompletionOr parse_temporal_calendar_string(VM& vm, Utf16View string) +ThrowCompletionOr parse_temporal_calendar_string(VM& vm, Utf16View string) { // 1. Let parseResult be Completion(ParseISODateTime(string, « TemporalDateTimeString[+Zoned], TemporalDateTimeString[~Zoned], // TemporalInstantString, TemporalTimeString, TemporalMonthDayString, TemporalYearMonthString »)). @@ -1338,7 +1336,7 @@ ThrowCompletionOr parse_temporal_calendar_string(VM& vm, Utf16View strin // b. If calendar is EMPTY, return "iso8601". // c. Else, return calendar. - return calendar.value_or("iso8601"_string); + return calendar.value_or(Utf16String::from_utf8_without_validation("iso8601"sv)); } // 3. Set parseResult to ParseText(StringToCodePoints(string), AnnotationValue). @@ -1346,10 +1344,10 @@ ThrowCompletionOr parse_temporal_calendar_string(VM& vm, Utf16View strin // 4. If parseResult is a List of errors, throw a RangeError exception. if (!annotation_parse_result.has_value()) - return vm.throw_completion(ErrorType::TemporalInvalidCalendarString, string.to_utf8_but_should_be_ported_to_utf16()); + return vm.throw_completion(ErrorType::TemporalInvalidCalendarString, string); // 5. Return string. - return string.to_utf8_but_should_be_ported_to_utf16(); + return Utf16String::from_utf16(string); } // 13.37 ParseTemporalDurationString ( isoString ), https://tc39.es/proposal-temporal/#sec-temporal-parsetemporaldurationstring @@ -1600,7 +1598,7 @@ ThrowCompletionOr parse_temporal_time_zone_string(VM& // 5. If timeZoneResult.[[TimeZoneAnnotation]] is not EMPTY, return ! ParseTimeZoneIdentifier(timeZoneResult.[[TimeZoneAnnotation]]). if (time_zone_result.time_zone_annotation.has_value()) - return parse_time_zone_identifier(*time_zone_result.time_zone_annotation); + return TRY(parse_time_zone_identifier(vm, *time_zone_result.time_zone_annotation)); // 6. If timeZoneResult.[[Z]] is true, return ! ParseTimeZoneIdentifier("UTC"). if (time_zone_result.z_designator) @@ -1615,7 +1613,7 @@ ThrowCompletionOr parse_temporal_time_zone_string(VM& } // 13.41 ToOffsetString ( argument ), https://tc39.es/proposal-temporal/#sec-temporal-tooffsetstring -ThrowCompletionOr to_offset_string(VM& vm, Value argument) +ThrowCompletionOr to_offset_string(VM& vm, Value argument) { // 1. Let offset be ? ToPrimitive(argument, STRING). auto offset = TRY(argument.to_primitive(vm, Value::PreferredType::String)); @@ -1625,11 +1623,11 @@ ThrowCompletionOr to_offset_string(VM& vm, Value argument) return vm.throw_completion(ErrorType::TemporalInvalidTimeZoneString, offset); // 3. Perform ? ParseDateTimeUTCOffset(offset). - auto offset_string = offset.as_string().utf16_string_view().to_utf8_but_should_be_ported_to_utf16(); - TRY(parse_date_time_utc_offset(vm, offset_string.bytes_as_string_view())); + auto offset_string = offset.as_string().utf16_string_view(); + TRY(parse_date_time_utc_offset(vm, offset_string)); // 4. Return offset. - return offset_string; + return Utf16String::from_utf16(offset_string); } // 13.42 ISODateToFields ( calendar, isoDate, type ), https://tc39.es/proposal-temporal/#sec-temporal-isodatetofields diff --git a/Libraries/LibJS/Runtime/Temporal/AbstractOperations.h b/Libraries/LibJS/Runtime/Temporal/AbstractOperations.h index 2fcd42df37..32ab735cc8 100644 --- a/Libraries/LibJS/Runtime/Temporal/AbstractOperations.h +++ b/Libraries/LibJS/Runtime/Temporal/AbstractOperations.h @@ -192,10 +192,10 @@ double round_number_to_increment(double, u64 increment, RoundingMode); Crypto::SignedBigInteger round_number_to_increment(Crypto::SignedBigInteger const&, Crypto::UnsignedBigInteger const& increment, RoundingMode); Crypto::SignedBigInteger round_number_to_increment_as_if_positive(Crypto::SignedBigInteger const&, Crypto::UnsignedBigInteger const& increment, RoundingMode); ThrowCompletionOr parse_iso_date_time(VM&, Utf16View iso_string, ReadonlySpan allowed_formats); -ThrowCompletionOr parse_temporal_calendar_string(VM&, Utf16View); +ThrowCompletionOr parse_temporal_calendar_string(VM&, Utf16View); ThrowCompletionOr> parse_temporal_duration_string(VM&, Utf16View iso_string); ThrowCompletionOr parse_temporal_time_zone_string(VM&, Utf16View time_zone_string); -ThrowCompletionOr to_offset_string(VM&, Value argument); +ThrowCompletionOr to_offset_string(VM&, Value argument); CalendarFields iso_date_to_fields(String const& calendar, ISODate, DateType); ThrowCompletionOr get_difference_settings(VM&, DurationOperation, Object const& options, UnitGroup, ReadonlySpan disallowed_units, Unit fallback_smallest_unit, Unit smallest_largest_default_unit); diff --git a/Libraries/LibJS/Runtime/Temporal/Calendar.cpp b/Libraries/LibJS/Runtime/Temporal/Calendar.cpp index cbd58b852c..0027837b9b 100644 --- a/Libraries/LibJS/Runtime/Temporal/Calendar.cpp +++ b/Libraries/LibJS/Runtime/Temporal/Calendar.cpp @@ -286,8 +286,11 @@ ThrowCompletionOr canonicalize_calendar(VM& vm, StringView id) ThrowCompletionOr canonicalize_calendar(VM& vm, Utf16View id) { - auto utf8_id = id.to_utf8_but_should_be_ported_to_utf16(); - return canonicalize_calendar(vm, utf8_id.bytes_as_string_view()); + if (!id.is_ascii()) + return vm.throw_completion(ErrorType::TemporalInvalidCalendarIdentifier, id); + + auto id_string = MUST(id.to_utf8()); + return canonicalize_calendar(vm, id_string.bytes_as_string_view()); } // 12.1.2 AvailableCalendars ( ), https://tc39.es/proposal-temporal/#sec-availablecalendars @@ -324,8 +327,7 @@ ThrowCompletionOr parse_month_code(VM& vm, Value argument) if (!month_code.is_string()) return vm.throw_completion(ErrorType::NotAString, month_code); - auto month_code_string = month_code.as_string().utf16_string_view().to_utf8_but_should_be_ported_to_utf16(); - return parse_month_code(vm, month_code_string.bytes_as_string_view()); + return parse_month_code(vm, month_code.as_string().utf16_string_view()); } // 12.2.1 ParseMonthCode ( argument ), https://tc39.es/proposal-temporal/#sec-temporal-parsemonthcode @@ -337,6 +339,15 @@ ThrowCompletionOr parse_month_code(VM& vm, StringView month_code) return vm.throw_completion(ErrorType::TemporalInvalidMonthCode); } +// 12.2.1 ParseMonthCode ( argument ), https://tc39.es/proposal-temporal/#sec-temporal-parsemonthcode +ThrowCompletionOr parse_month_code(VM& vm, Utf16View month_code) +{ + // 3. If ParseText(StringToCodePoints(monthCode), MonthCode) is a List of errors, throw a RangeError exception. + if (auto result = Unicode::parse_month_code(month_code); result.has_value()) + return result.release_value(); + return vm.throw_completion(ErrorType::TemporalInvalidMonthCode); +} + // 12.3.3 PrepareCalendarFields ( calendar, fields, calendarFieldNames, nonCalendarFieldNames, requiredFieldNames ), https://tc39.es/proposal-temporal/#sec-temporal-preparecalendarfields ThrowCompletionOr prepare_calendar_fields(VM& vm, String const& calendar, Object const& fields, CalendarFieldList calendar_field_names, CalendarFieldList non_calendar_field_names, CalendarFieldListOrPartial required_field_names) { @@ -395,7 +406,7 @@ ThrowCompletionOr prepare_calendar_fields(VM& vm, String const& // v. Else if Conversion is TO-STRING, then case CalendarFieldConversion::ToString: // 1. Set value to ? ToString(value). - set_field_value(key, result, TRY(value.to_utf16_string(vm)).to_utf8_but_should_be_ported_to_utf16()); + set_field_value(key, result, TRY(value.to_utf16_string(vm))); break; // vi. Else if Conversion is TO-TEMPORAL-TIME-ZONE-IDENTIFIER, then case CalendarFieldConversion::ToTemporalTimeZoneIdentifier: @@ -1510,7 +1521,7 @@ ThrowCompletionOr non_iso_resolve_fields(VM& vm, String const& calendar, C return vm.throw_completion(ErrorType::TemporalInvalidCalendarFieldName, "era"sv); // c. Let arithmeticYear be CalendarDateArithmeticYearForEraYear(calendar, canonicalEra, fields.[[EraYear]]). - auto arithmetic_year = calendar_date_arithmetic_year_for_era_year(calendar, *canonical_era, *fields.era_year); + auto arithmetic_year = calendar_date_arithmetic_year_for_era_year(calendar, *fields.era, *fields.era_year); // d. If fields.[[Year]] is not UNSET, and fields.[[Year]] ≠ arithmeticYear, throw a RangeError exception. if (fields.year.has_value() && *fields.year != arithmetic_year) @@ -1650,7 +1661,7 @@ bool calendar_supports_era(String const& calendar) } // 4.1.2 CanonicalizeEraInCalendar ( calendar, era ), https://tc39.es/proposal-intl-era-monthcode/#sec-temporal-canonicalizeeraincalendar -Optional canonicalize_era_in_calendar(String const& calendar, StringView era) +Optional canonicalize_era_in_calendar(String const& calendar, Utf16View era) { // 1. For each row of Table 2, except the header row, do for (auto const& row : CALENDAR_ERA_DATA) { @@ -1661,12 +1672,12 @@ Optional canonicalize_era_in_calendar(String const& calendar, String auto canonical_name = row.era; // ii. If canonicalName is equal to era, return canonicalName. - if (canonical_name == era) + if (era == canonical_name) return canonical_name; // iii. Let aliases be a List whose elements are the strings given in the "Aliases" column of the row. // iv. If aliases contains era, return canonicalName. - if (row.alias == era) + if (era == row.alias) return canonical_name; } } @@ -1824,16 +1835,16 @@ u8 calendar_days_in_month(String const& calendar, i32 arithmetic_year, u8 ordina } // 4.1.12 CalendarDateArithmeticYearForEraYear ( calendar, era, eraYear ), https://tc39.es/proposal-intl-era-monthcode/#sec-temporal-calendardatearithmeticyearforerayear -i32 calendar_date_arithmetic_year_for_era_year(String const& calendar, StringView era, i32 era_year) +i32 calendar_date_arithmetic_year_for_era_year(String const& calendar, Utf16View era, i32 era_year) { // 1. Let era be CanonicalizeEraInCalendar(calendar, era). // 2. Assert: era is not undefined. - era = canonicalize_era_in_calendar(calendar, era).release_value(); + auto canonical_era = canonicalize_era_in_calendar(calendar, era).release_value(); // 3. If calendar is not listed in the "Calendar Type" column of Table 1, return an implementation-defined value. // 4. Let r be the row in Table 2 with a value in the Calendar column matching calendar and a value in the Era // column matching era. - auto row = find_value(CALENDAR_ERA_DATA, [&](auto const& row) { return row.calendar == calendar && row.era == era; }); + auto row = find_value(CALENDAR_ERA_DATA, [&](auto const& row) { return row.calendar == calendar && row.era == canonical_era; }); if (!row.has_value()) return era_year; diff --git a/Libraries/LibJS/Runtime/Temporal/Calendar.h b/Libraries/LibJS/Runtime/Temporal/Calendar.h index 5836e8ba4e..754fd2db37 100644 --- a/Libraries/LibJS/Runtime/Temporal/Calendar.h +++ b/Libraries/LibJS/Runtime/Temporal/Calendar.h @@ -11,6 +11,7 @@ #include #include +#include #include #include #include @@ -71,7 +72,7 @@ struct CalendarFields { }; } - Optional era; + Optional era; Optional era_year; Optional year; Optional month; @@ -83,7 +84,7 @@ struct CalendarFields { Optional millisecond { 0 }; Optional microsecond { 0 }; Optional nanosecond { 0 }; - Optional offset_string; + Optional offset_string; Optional time_zone; }; @@ -103,6 +104,7 @@ Vector const& available_calendars(); ThrowCompletionOr parse_month_code(VM&, Value argument); ThrowCompletionOr parse_month_code(VM&, StringView month_code); +ThrowCompletionOr parse_month_code(VM&, Utf16View month_code); ThrowCompletionOr prepare_calendar_fields(VM&, String const& calendar, Object const& fields, CalendarFieldList calendar_field_names, CalendarFieldList non_calendar_field_names, CalendarFieldListOrPartial required_field_names); ThrowCompletionOr calendar_date_from_fields(VM&, String const& calendar, CalendarFields&, Overflow); @@ -135,14 +137,14 @@ ThrowCompletionOr non_iso_resolve_fields(VM&, String const& calendar, Cale ThrowCompletionOr calendar_resolve_fields(VM&, String const& calendar, CalendarFields&, DateType); bool calendar_supports_era(String const& calendar); -Optional canonicalize_era_in_calendar(String const& calendar, StringView era); +Optional canonicalize_era_in_calendar(String const& calendar, Utf16View era); bool calendar_has_mid_year_eras(String const& calendar); bool is_valid_month_code_for_calendar(String const& calendar, StringView month_code); bool year_contains_month_code(String const& calendar, i32 arithmetic_year, StringView month_code); ThrowCompletionOr constrain_month_code(VM&, String const& calendar, i32 arithmetic_year, String const& month_code, Overflow overflow); u8 month_code_to_ordinal(String const& calendar, i32 arithmetic_year, StringView month_code); u8 calendar_days_in_month(String const& calendar, i32 arithmetic_year, u8 ordinal_month); -i32 calendar_date_arithmetic_year_for_era_year(String const& calendar, StringView era, i32 era_year); +i32 calendar_date_arithmetic_year_for_era_year(String const& calendar, Utf16View era, i32 era_year); ThrowCompletionOr calendar_integers_to_iso(VM&, String const& calendar, i32 arithmetic_year, u8 ordinal_month, u8 day); u8 calendar_months_in_year(String const& calendar, i32 arithmetic_year); BalancedDate balance_non_iso_date(String const& calendar, i32 arithmetic_year, i32 ordinal_month, i32 day); diff --git a/Libraries/LibJS/Runtime/Temporal/ISORecords.h b/Libraries/LibJS/Runtime/Temporal/ISORecords.h index 6aaee2184f..d150ab90d2 100644 --- a/Libraries/LibJS/Runtime/Temporal/ISORecords.h +++ b/Libraries/LibJS/Runtime/Temporal/ISORecords.h @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -50,8 +51,8 @@ struct ISOYearMonth { // 13.32 ISO String Time Zone Parse Records, https://tc39.es/proposal-temporal/#sec-temporal-iso-string-time-zone-parse-records struct ParsedISOTimeZone { bool z_designator { false }; - Optional offset_string; - Optional time_zone_annotation; + Optional offset_string; + Optional time_zone_annotation; }; // 13.33 Time Zone Identifier Parse Records, https://tc39.es/proposal-temporal/#sec-temporal-time-zone-identifier-parse-records @@ -69,7 +70,7 @@ struct ParsedISODateTime { u8 day { 0 }; Variant time; ParsedISOTimeZone time_zone; - Optional calendar; + Optional calendar; }; } diff --git a/Libraries/LibJS/Runtime/Temporal/PlainDate.cpp b/Libraries/LibJS/Runtime/Temporal/PlainDate.cpp index 5142c12921..0d80e82183 100644 --- a/Libraries/LibJS/Runtime/Temporal/PlainDate.cpp +++ b/Libraries/LibJS/Runtime/Temporal/PlainDate.cpp @@ -139,10 +139,9 @@ ThrowCompletionOr> to_temporal_date(VM& vm, Value item, Value // 5. Let calendar be result.[[Calendar]]. // 6. If calendar is empty, set calendar to "iso8601". - auto calendar = result.calendar.value_or("iso8601"_string); - - // 7. Set calendar to ? CanonicalizeCalendar(calendar). - calendar = TRY(canonicalize_calendar(vm, calendar)); + auto calendar = result.calendar.has_value() + ? TRY(canonicalize_calendar(vm, *result.calendar)) + : TRY(canonicalize_calendar(vm, "iso8601"sv)); // 8. Let resolvedOptions be ? GetOptionsObject(options). auto resolved_options = TRY(get_options_object(vm, options)); diff --git a/Libraries/LibJS/Runtime/Temporal/PlainDateTime.cpp b/Libraries/LibJS/Runtime/Temporal/PlainDateTime.cpp index 1cbc378f83..ec13588dcf 100644 --- a/Libraries/LibJS/Runtime/Temporal/PlainDateTime.cpp +++ b/Libraries/LibJS/Runtime/Temporal/PlainDateTime.cpp @@ -182,10 +182,9 @@ ThrowCompletionOr> to_temporal_date_time(VM& vm, Value it // 6. Let calendar be result.[[Calendar]]. // 7. If calendar is empty, set calendar to "iso8601". - auto calendar = result.calendar.value_or("iso8601"_string); - - // 8. Set calendar to ? CanonicalizeCalendar(calendar). - calendar = TRY(canonicalize_calendar(vm, calendar)); + auto calendar = result.calendar.has_value() + ? TRY(canonicalize_calendar(vm, *result.calendar)) + : TRY(canonicalize_calendar(vm, "iso8601"sv)); // 9. Let resolvedOptions be ? GetOptionsObject(options). auto resolved_options = TRY(get_options_object(vm, options)); diff --git a/Libraries/LibJS/Runtime/Temporal/PlainMonthDay.cpp b/Libraries/LibJS/Runtime/Temporal/PlainMonthDay.cpp index 373ce1a59f..0349b07c35 100644 --- a/Libraries/LibJS/Runtime/Temporal/PlainMonthDay.cpp +++ b/Libraries/LibJS/Runtime/Temporal/PlainMonthDay.cpp @@ -73,10 +73,9 @@ ThrowCompletionOr> to_temporal_month_day(VM& vm, Value it // 5. Let calendar be result.[[Calendar]]. // 6. If calendar is empty, set calendar to "iso8601". - auto calendar = parse_result.calendar.value_or("iso8601"_string); - - // 7. Set calendar to ? CanonicalizeCalendar(calendar). - calendar = TRY(canonicalize_calendar(vm, calendar)); + auto calendar = parse_result.calendar.has_value() + ? TRY(canonicalize_calendar(vm, *parse_result.calendar)) + : TRY(canonicalize_calendar(vm, "iso8601"sv)); // 8. Let resolvedOptions be ? GetOptionsObject(options). auto resolved_options = TRY(get_options_object(vm, options)); diff --git a/Libraries/LibJS/Runtime/Temporal/PlainYearMonth.cpp b/Libraries/LibJS/Runtime/Temporal/PlainYearMonth.cpp index 85ef4bf53c..fc97e57fd7 100644 --- a/Libraries/LibJS/Runtime/Temporal/PlainYearMonth.cpp +++ b/Libraries/LibJS/Runtime/Temporal/PlainYearMonth.cpp @@ -76,10 +76,9 @@ ThrowCompletionOr> to_temporal_year_month(VM& vm, Value // 5. Let calendar be result.[[Calendar]]. // 6. If calendar is empty, set calendar to "iso8601". - auto calendar = parse_result.calendar.value_or("iso8601"_string); - - // 7. Set calendar to ? CanonicalizeCalendar(calendar). - calendar = TRY(canonicalize_calendar(vm, calendar)); + auto calendar = parse_result.calendar.has_value() + ? TRY(canonicalize_calendar(vm, *parse_result.calendar)) + : TRY(canonicalize_calendar(vm, "iso8601"sv)); // 8. Let resolvedOptions be ? GetOptionsObject(options). auto resolved_options = TRY(get_options_object(vm, options)); diff --git a/Libraries/LibJS/Runtime/Temporal/TimeZone.cpp b/Libraries/LibJS/Runtime/Temporal/TimeZone.cpp index 530c91434a..ae3b8151c4 100644 --- a/Libraries/LibJS/Runtime/Temporal/TimeZone.cpp +++ b/Libraries/LibJS/Runtime/Temporal/TimeZone.cpp @@ -453,7 +453,8 @@ ThrowCompletionOr get_start_of_day(VM& vm, String cons return move(possible_epoch_nanoseconds[0]); // 4. Assert: IsOffsetTimeZoneIdentifier(timeZone) is false. - VERIFY(!is_offset_time_zone_identifier(time_zone)); + auto utf16_time_zone = Utf16String::from_utf8(time_zone); + VERIFY(!is_offset_time_zone_identifier(utf16_time_zone)); // 5. Let possibleEpochNsAfter be GetNamedTimeZoneEpochNanoseconds(timeZone, isoDateTimeAfter), where isoDateTimeAfter // is the ISO Date-Time Record for which DifferenceISODateTime(isoDateTime, isoDateTimeAfter, "iso8601", hour).[[Time]] @@ -542,7 +543,25 @@ ThrowCompletionOr parse_time_zone_identifier(VM& vm, S ThrowCompletionOr parse_time_zone_identifier(VM& vm, Utf16View identifier) { - return parse_time_zone_identifier(vm, identifier.to_utf8_but_should_be_ported_to_utf16()); + Optional cache_key; + if (identifier.is_ascii()) { + cache_key = MUST(identifier.to_utf8()); + if (auto result = time_zone_id_cache().get(*cache_key); result.has_value()) + return *result; + } + + // 1. Let parseResult be ParseText(StringToCodePoints(identifier), TimeZoneIdentifier). + auto parse_result = parse_iso8601(Production::TimeZoneIdentifier, identifier); + + // 2. If parseResult is a List of errors, throw a RangeError exception. + if (!parse_result.has_value()) + return vm.throw_completion(ErrorType::TemporalInvalidTimeZoneString, identifier); + + auto result = parse_time_zone_identifier(*parse_result); + if (cache_key.has_value()) + time_zone_id_cache().set(*cache_key, result); + + return result; } // 11.1.16 ParseTimeZoneIdentifier ( identifier ), https://tc39.es/proposal-temporal/#sec-parsetimezoneidentifier @@ -570,7 +589,7 @@ ParsedTimeZoneIdentifier parse_time_zone_identifier(ParseResult const& parse_res // b. NOTE: name is syntactically valid, but does not necessarily conform to IANA Time Zone Database naming // guidelines or correspond with an available named time zone identifier. // c. Return Time Zone Identifier Parse Record { [[Name]]: CodePointsToString(name), [[OffsetMinutes]]: EMPTY }. - return ParsedTimeZoneIdentifier { .name = parse_result.time_zone_iana_name->to_utf8_but_should_be_ported_to_utf16(), .offset_minutes = {} }; + return ParsedTimeZoneIdentifier { .name = MUST(parse_result.time_zone_iana_name->to_utf8()), .offset_minutes = {} }; } // 4. Assert: parseResult contains a UTCOffset[~SubMinutePrecision] Parse Node. diff --git a/Libraries/LibJS/Runtime/Temporal/ZonedDateTime.cpp b/Libraries/LibJS/Runtime/Temporal/ZonedDateTime.cpp index c1b3914fc7..bd80450416 100644 --- a/Libraries/LibJS/Runtime/Temporal/ZonedDateTime.cpp +++ b/Libraries/LibJS/Runtime/Temporal/ZonedDateTime.cpp @@ -139,7 +139,7 @@ ThrowCompletionOr> to_temporal_zoned_date_time(VM& vm, Va String calendar; String time_zone; - Optional offset_string; + Optional offset_string; Disambiguation disambiguation; OffsetOption offset_option; @@ -223,8 +223,7 @@ ThrowCompletionOr> to_temporal_zoned_date_time(VM& vm, Va VERIFY(annotation.has_value()); // e. Let timeZone be ? ToTemporalTimeZoneIdentifier(annotation). - auto utf16_annotation = Utf16String::from_utf8(*annotation); - time_zone = TRY(to_temporal_time_zone_identifier(vm, utf16_annotation)); + time_zone = TRY(to_temporal_time_zone_identifier(vm, *annotation)); // f. Let offsetString be result.[[TimeZone]].[[OffsetString]]. offset_string = move(result.time_zone.offset_string); @@ -237,10 +236,9 @@ ThrowCompletionOr> to_temporal_zoned_date_time(VM& vm, Va // h. Let calendar be result.[[Calendar]]. // i. If calendar is EMPTY, set calendar to "iso8601". - calendar = result.calendar.value_or("iso8601"_string); - - // j. Set calendar to ? CanonicalizeCalendar(calendar). - calendar = TRY(canonicalize_calendar(vm, calendar)); + calendar = result.calendar.has_value() + ? TRY(canonicalize_calendar(vm, *result.calendar)) + : TRY(canonicalize_calendar(vm, "iso8601"sv)); // k. Set matchBehaviour to MATCH-MINUTES. match_behavior = MatchBehavior::MatchMinutes; @@ -248,8 +246,7 @@ ThrowCompletionOr> to_temporal_zoned_date_time(VM& vm, Va // l. If offsetString is not EMPTY, then if (offset_string.has_value()) { // i. Let offsetParseResult be ParseText(StringToCodePoints(offsetString), UTCOffset[+SubMinutePrecision]). - auto utf16_offset_string = Utf16String::from_utf8(*offset_string); - auto offset_parse_result = parse_utc_offset(utf16_offset_string, SubMinutePrecision::Yes); + auto offset_parse_result = parse_utc_offset(*offset_string, SubMinutePrecision::Yes); // ii. Assert: offsetParseResult is a Parse Node. VERIFY(offset_parse_result.has_value()); diff --git a/Libraries/LibJS/Runtime/Temporal/ZonedDateTimePrototype.cpp b/Libraries/LibJS/Runtime/Temporal/ZonedDateTimePrototype.cpp index 6d17aa1aae..146d724271 100644 --- a/Libraries/LibJS/Runtime/Temporal/ZonedDateTimePrototype.cpp +++ b/Libraries/LibJS/Runtime/Temporal/ZonedDateTimePrototype.cpp @@ -6,6 +6,7 @@ * SPDX-License-Identifier: BSD-2-Clause */ +#include #include #include #include @@ -415,7 +416,7 @@ JS_DEFINE_NATIVE_FUNCTION(ZonedDateTimePrototype::with) fields.nanosecond = iso_date_time.time.nanosecond; // 16. Set fields.[[OffsetString]] to FormatUTCOffsetNanoseconds(offsetNanoseconds). - fields.offset_string = format_utc_offset_nanoseconds(offset_nanoseconds); + fields.offset_string = Utf16String::from_utf8(format_utc_offset_nanoseconds(offset_nanoseconds)); // 17. Let partialZonedDateTime be ? PrepareCalendarFields(calendar, temporalZonedDateTimeLike, « YEAR, MONTH, MONTH-CODE, DAY », « HOUR, MINUTE, SECOND, MILLISECOND, MICROSECOND, NANOSECOND, OFFSET », PARTIAL). static constexpr auto calendar_field_names = to_array({ CalendarField::Year, CalendarField::Month, CalendarField::MonthCode, CalendarField::Day }); @@ -900,7 +901,8 @@ JS_DEFINE_NATIVE_FUNCTION(ZonedDateTimePrototype::get_time_zone_transition) auto direction = TRY(get_direction_option(vm, *direction_param)); // 8. If IsOffsetTimeZoneIdentifier(timeZone) is true, return null. - if (is_offset_time_zone_identifier(time_zone)) + auto utf16_time_zone = Utf16String::from_utf8(time_zone); + if (is_offset_time_zone_identifier(utf16_time_zone)) return js_null(); Optional transition; diff --git a/Libraries/LibJS/Runtime/Uint8Array.cpp b/Libraries/LibJS/Runtime/Uint8Array.cpp index 685b964266..9e86c2a133 100644 --- a/Libraries/LibJS/Runtime/Uint8Array.cpp +++ b/Libraries/LibJS/Runtime/Uint8Array.cpp @@ -116,7 +116,7 @@ JS_DEFINE_NATIVE_FUNCTION(Uint8ArrayConstructorHelpers::from_base64) } // 9. Let result be FromBase64(string, alphabet, lastChunkHandling). - auto result = JS::from_base64(vm, string_value.as_string().utf16_string_view().to_utf8_but_should_be_ported_to_utf16(), alphabet, last_chunk_handling); + auto result = JS::from_base64(vm, string_value.as_string().utf16_string_view(), alphabet, last_chunk_handling); // 10. If result.[[Error]] is not NONE, then if (result.error.has_value()) { @@ -220,7 +220,7 @@ JS_DEFINE_NATIVE_FUNCTION(Uint8ArrayPrototypeHelpers::set_from_base64) auto byte_length = typed_array_length(typed_array_record); // 14. Let result be FromBase64(string, alphabet, lastChunkHandling, byteLength). - auto result = JS::from_base64(vm, string_value.as_string().utf16_string_view().to_utf8_but_should_be_ported_to_utf16(), alphabet, last_chunk_handling, byte_length); + auto result = JS::from_base64(vm, string_value.as_string().utf16_string_view(), alphabet, last_chunk_handling, byte_length); // 15. Let bytes be result.[[Bytes]]. auto bytes = move(result.bytes); @@ -505,7 +505,7 @@ void set_uint8_array_bytes(TypedArrayBase& into, ReadonlyBytes bytes) } // 23.3.3.7 FromBase64 ( string, alphabet, lastChunkHandling [ , maxLength ] ), https://tc39.es/ecma262/#sec-frombase64 -DecodeResult from_base64(VM& vm, StringView string, Alphabet alphabet, AK::LastChunkHandling last_chunk_handling, Optional max_length) +DecodeResult from_base64(VM& vm, Utf16View string, Alphabet alphabet, AK::LastChunkHandling last_chunk_handling, Optional max_length) { auto output = MUST(ByteBuffer::create_uninitialized(max_length.value_or_lazy_evaluated([&]() { return AK::size_required_to_decode_base64(string); diff --git a/Libraries/LibJS/Runtime/Uint8Array.h b/Libraries/LibJS/Runtime/Uint8Array.h index 3b6cbf295f..3db6a427f7 100644 --- a/Libraries/LibJS/Runtime/Uint8Array.h +++ b/Libraries/LibJS/Runtime/Uint8Array.h @@ -53,7 +53,7 @@ ThrowCompletionOr> validate_uint8_array(VM&); ThrowCompletionOr get_uint8_array_bytes(VM&, TypedArrayBase const&); ThrowCompletionOr get_uint8_array_bytes_view(VM&, TypedArrayBase const&); void set_uint8_array_bytes(TypedArrayBase&, ReadonlyBytes); -DecodeResult from_base64(VM&, StringView string, Alphabet alphabet, AK::LastChunkHandling last_chunk_handling, Optional max_length = {}); +DecodeResult from_base64(VM&, Utf16View string, Alphabet alphabet, AK::LastChunkHandling last_chunk_handling, Optional max_length = {}); DecodeResult from_hex(VM&, Utf16View string, Optional max_length = {}); } diff --git a/Libraries/LibJS/Runtime/VM.cpp b/Libraries/LibJS/Runtime/VM.cpp index 00bdbb24ed..5c78b22a26 100644 --- a/Libraries/LibJS/Runtime/VM.cpp +++ b/Libraries/LibJS/Runtime/VM.cpp @@ -154,7 +154,7 @@ VM::VM(ErrorMessages error_messages) }; // 2 HostEnsureCanCompileStrings ( calleeRealm, parameterStrings, bodyString, codeString, compilationType, parameterArgs, bodyArg ), https://tc39.es/proposal-dynamic-code-brand-checks/#sec-hostensurecancompilestrings - host_ensure_can_compile_strings = [](Realm&, ReadonlySpan, StringView, StringView, CompilationType, ReadonlySpan, Value) -> ThrowCompletionOr { + host_ensure_can_compile_strings = [](Realm&, ReadonlySpan, Utf16View, Utf16View, CompilationType, ReadonlySpan, Value) -> ThrowCompletionOr { // The host-defined abstract operation HostEnsureCanCompileStrings takes arguments calleeRealm (a Realm Record), // parameterStrings (a List of Strings), bodyString (a String), and direct (a Boolean) and returns either a normal // completion containing unused or a throw completion. @@ -230,7 +230,7 @@ VM::VM(ErrorMessages error_messages) }; // AD-HOC: Inform the host that we received a date string we were unable to parse. - host_unrecognized_date_string = [](StringView) { + host_unrecognized_date_string = [](Utf16View) { }; } diff --git a/Libraries/LibJS/Runtime/VM.h b/Libraries/LibJS/Runtime/VM.h index 77b0eaf5ad..797e9a6868 100644 --- a/Libraries/LibJS/Runtime/VM.h +++ b/Libraries/LibJS/Runtime/VM.h @@ -435,11 +435,11 @@ public: Function()>>, Realm*)> host_enqueue_promise_job; Function(FunctionObject&)> host_make_job_callback; Function(Object const&)> host_get_code_for_eval; - Function(Realm&, ReadonlySpan, StringView, StringView, CompilationType, ReadonlySpan, Value)> host_ensure_can_compile_strings; + Function(Realm&, ReadonlySpan, Utf16View, Utf16View, CompilationType, ReadonlySpan, Value)> host_ensure_can_compile_strings; Function(Object&)> host_ensure_can_add_private_element; Function(ArrayBuffer&, size_t)> host_resize_array_buffer; Function(ArrayBuffer&, size_t)> host_grow_shared_array_buffer; - Function host_unrecognized_date_string; + Function host_unrecognized_date_string; Function host_system_utc_epoch_nanoseconds; Function host_promise_job_queue_is_empty; diff --git a/Libraries/LibJS/Runtime/Value.cpp b/Libraries/LibJS/Runtime/Value.cpp index 64e8f996b9..bd282d5556 100644 --- a/Libraries/LibJS/Runtime/Value.cpp +++ b/Libraries/LibJS/Runtime/Value.cpp @@ -412,39 +412,6 @@ GC::Ref Value::typeof_(VM& vm) const } } -String Value::to_string_without_side_effects() const -{ - if (is_double()) - return number_to_string(m_value.as_double); - - switch (m_value.tag) { - case UNDEFINED_TAG: - return "undefined"_string; - case NULL_TAG: - return "null"_string; - case BOOLEAN_TAG: - return as_bool() ? "true"_string : "false"_string; - case INT32_TAG: - return String::number(as_i32()); - case STRING_TAG: - return as_string().utf16_string_view().to_utf8_but_should_be_ported_to_utf16(); - case SYMBOL_TAG: - return as_symbol().descriptive_string().to_utf8_but_should_be_ported_to_utf16(); - case BIGINT_TAG: - return as_bigint().to_string().release_value(); - case OBJECT_TAG: - return String::formatted("[object {}]", as_object().class_name()).release_value(); - case ACCESSOR_TAG: - return ""_string; - case EMPTY_TAG: - return ""_string; - default: - if (is_cell()) - return String::formatted("[internal object {}]", as_cell().class_name()).release_value(); - VERIFY_NOT_REACHED(); - } -} - Utf16String Value::to_utf16_string_without_side_effects() const { if (is_double()) @@ -613,7 +580,7 @@ ThrowCompletionOr Value::to_primitive_slow_case(VM& vm, PreferredType pre return result; // vi. Throw a TypeError exception. - return vm.throw_completion(ErrorType::ToPrimitiveReturnedObject, to_string_without_side_effects(), hint); + return vm.throw_completion(ErrorType::ToPrimitiveReturnedObject, to_utf16_string_without_side_effects(), hint); } // c. If preferredType is not present, let preferredType be number. @@ -801,7 +768,7 @@ double string_to_number(Utf16View string) // 4. Return StringNumericValue of literal. if (result->base != 10) { - auto bigint = MUST(Crypto::UnsignedBigInteger::from_base(result->base, result->literal.to_utf8_but_should_be_ported_to_utf16())); + auto bigint = MUST(Crypto::UnsignedBigInteger::from_base(result->base, result->literal)); return bigint.to_double(); } @@ -970,7 +937,7 @@ static Optional string_to_bigint(VM& vm, Utf16View string) // 4. Let mv be the MV of literal. // 5. Assert: mv is an integer. - auto bigint = MUST(Crypto::SignedBigInteger::from_base(result->base, result->literal.to_utf8_but_should_be_ported_to_utf16())); + auto bigint = MUST(Crypto::SignedBigInteger::from_base(result->base, result->literal)); if (result->is_negative && (bigint != bigint_zero())) bigint.negate(); diff --git a/Libraries/LibJS/Runtime/Value.h b/Libraries/LibJS/Runtime/Value.h index 5ffcf05fdf..3841b34d93 100644 --- a/Libraries/LibJS/Runtime/Value.h +++ b/Libraries/LibJS/Runtime/Value.h @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -433,7 +434,6 @@ public: ThrowCompletionOr> get_method(VM&, PropertyKey const&) const; ThrowCompletionOr> get_method(VM&, PropertyKey const&, Bytecode::PropertyLookupCache&) const; - [[nodiscard]] String to_string_without_side_effects() const; [[nodiscard]] Utf16String to_utf16_string_without_side_effects() const; [[nodiscard]] GC::Ref typeof_(VM&) const; @@ -669,12 +669,12 @@ inline Root make_root(JS::Value value, SourceLocation location = Sour namespace AK { template<> -struct Formatter : Formatter { +struct Formatter : Formatter { ErrorOr format(FormatBuilder& builder, JS::Value value) { if (value.is_special_empty_value()) return Formatter::format(builder, ""sv); - return Formatter::format(builder, value.to_string_without_side_effects()); + return Formatter {}.format(builder, value.to_utf16_string_without_side_effects()); } }; diff --git a/Libraries/LibJS/Rust/src/ast_dump.rs b/Libraries/LibJS/Rust/src/ast_dump.rs index 037780c3fa..7153f620dc 100644 --- a/Libraries/LibJS/Rust/src/ast_dump.rs +++ b/Libraries/LibJS/Rust/src/ast_dump.rs @@ -230,7 +230,7 @@ fn utf16_to_string(s: &[u16]) -> String { /// Once the C++ pipeline is removed, this can be replaced with /// a native implementation. fn format_f64(value: f64) -> String { - // C++ AST dump formats JS::Value which uses to_string_without_side_effects(), + // C++ AST dump formats JS::Value which uses to_utf16_string_without_side_effects(), // producing "Infinity"/"-Infinity"/"NaN". The rust_format_double FFI uses // AK's double formatter which produces "inf"/"-inf"/"nan" instead. if value.is_nan() { diff --git a/Libraries/LibJS/RustIntegration.cpp b/Libraries/LibJS/RustIntegration.cpp index be5e763678..9530ee9bb9 100644 --- a/Libraries/LibJS/RustIntegration.cpp +++ b/Libraries/LibJS/RustIntegration.cpp @@ -704,11 +704,11 @@ Optional>> materialize_bytecode_cache_s return builder.result; } -Optional>> compile_script(StringView source_text, Realm& realm, StringView filename, size_t line_number_offset) +Optional>> compile_script(Utf16View source_text, Realm& realm, StringView filename, size_t line_number_offset) { auto source_code = SourceCode::create( String::from_utf8(filename).release_value_but_fixme_should_propagate_errors(), - Utf16String::from_utf8(source_text)); + Utf16String::from_utf16(source_text)); auto const* source_ptr = source_code->utf16_data(); auto length = source_code->length_in_code_units(); @@ -1001,16 +1001,13 @@ Optional>> compile_module(StringView so } Optional, String>> compile_dynamic_function( - VM& vm, StringView source_text, StringView parameters_string, StringView body_parse_string, + VM& vm, Utf16View source_text, Utf16View parameters_string, Utf16View body_parse_string, FunctionKind kind) { - auto source_code = SourceCode::create({}, Utf16String::from_utf8(source_text)); + auto source_code = SourceCode::create({}, Utf16String::from_utf16(source_text)); auto const& code_view = source_code->code_view(); auto full_length = code_view.length_in_code_units(); - auto params_utf16 = Utf16String::from_utf8(parameters_string); - auto body_utf16 = Utf16String::from_utf8(body_parse_string); - auto prepare_utf16 = [](Utf16View const& view, Vector& buf) -> u16 const* { if (view.has_ascii_storage()) { auto ascii = view.ascii_span(); @@ -1024,16 +1021,16 @@ Optional, String>> compile_dynamic_fu Vector full_buf, params_buf, body_buf; auto const* full_data = prepare_utf16(code_view, full_buf); - auto const* params_data = prepare_utf16(params_utf16.utf16_view(), params_buf); - auto const* body_data = prepare_utf16(body_utf16.utf16_view(), body_buf); + auto const* params_data = prepare_utf16(parameters_string, params_buf); + auto const* body_data = prepare_utf16(body_parse_string, body_buf); GC::DeferGC defer_gc(vm.heap()); String parse_error; void* sfd_ptr = rust_compile_dynamic_function( full_data, full_length, - params_data, params_utf16.utf16_view().length_in_code_units(), - body_data, body_utf16.utf16_view().length_in_code_units(), + params_data, parameters_string.length_in_code_units(), + body_data, body_parse_string.length_in_code_units(), &vm, source_code.ptr(), static_cast(kind), &parse_error, collect_single_parse_error, @@ -1043,7 +1040,7 @@ Optional, String>> compile_dynamic_fu return parse_error; auto& function_data = *static_cast(sfd_ptr); - function_data.m_source_text_owner = Utf16String::from_utf8(source_text); + function_data.m_source_text_owner = Utf16String::from_utf16(source_text); return GC::Ref { function_data }; } diff --git a/Libraries/LibJS/RustIntegration.h b/Libraries/LibJS/RustIntegration.h index 2cf2246521..f830d9a9d3 100644 --- a/Libraries/LibJS/RustIntegration.h +++ b/Libraries/LibJS/RustIntegration.h @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -157,7 +158,7 @@ Optional>> compile_parsed_script(FFI::P Optional>> materialize_compiled_script(FFI::CompiledProgram* compiled, NonnullRefPtr source_code, Realm& realm); // Compile a script. Returns nullopt if Rust is not available. -Optional>> compile_script(StringView source_text, Realm& realm, StringView filename, size_t line_number_offset); +Optional>> compile_script(Utf16View source_text, Realm& realm, StringView filename, size_t line_number_offset); // Compile eval code. Returns nullopt if Rust is not available. // On success, the executable's name is set to "eval". @@ -181,7 +182,7 @@ Optional>> compile_module(StringView so // Compile a dynamic function (new Function()). // On success, returns a SharedFunctionInstanceData with source_text set. JS_API Optional, String>> compile_dynamic_function( - VM& vm, StringView source_text, StringView parameters_string, StringView body_parse_string, + VM& vm, Utf16View source_text, Utf16View parameters_string, Utf16View body_parse_string, FunctionKind kind); // Compile a builtin JS file. Returns nullopt if Rust is not available. diff --git a/Libraries/LibJS/Script.cpp b/Libraries/LibJS/Script.cpp index 1c8acc016b..b69aecdc36 100644 --- a/Libraries/LibJS/Script.cpp +++ b/Libraries/LibJS/Script.cpp @@ -22,7 +22,7 @@ bool g_dump_ast_use_color = false; GC_DEFINE_ALLOCATOR(Script); // 16.1.5 ParseScript ( sourceText, realm, hostDefined ), https://tc39.es/ecma262/#sec-parse-script -Result, Vector> Script::parse(StringView source_text, Realm& realm, StringView filename, HostDefined* host_defined, size_t line_number_offset) +Result, Vector> Script::parse(Utf16View source_text, Realm& realm, StringView filename, HostDefined* host_defined, size_t line_number_offset) { auto rust_compilation = RustIntegration::compile_script(source_text, realm, filename, line_number_offset); if (!rust_compilation.has_value()) diff --git a/Libraries/LibJS/Script.h b/Libraries/LibJS/Script.h index 1485b233d3..b954b7706b 100644 --- a/Libraries/LibJS/Script.h +++ b/Libraries/LibJS/Script.h @@ -8,6 +8,7 @@ #include #include +#include #include #include #include @@ -57,7 +58,7 @@ public: }; virtual ~Script() override; - static Result, Vector> parse(StringView source_text, Realm&, StringView filename = {}, HostDefined* = nullptr, size_t line_number_offset = 1); + static Result, Vector> parse(Utf16View source_text, Realm&, StringView filename = {}, HostDefined* = nullptr, size_t line_number_offset = 1); static Result, Vector> create_from_parsed(FFI::ParsedProgram* parsed, NonnullRefPtr source_code, Realm&, HostDefined* = nullptr); static Result, Vector> create_from_compiled(FFI::CompiledProgram* compiled, NonnullRefPtr source_code, Realm&, HostDefined* = nullptr); static Result, Vector> create_from_bytecode_cache(NonnullRefPtr, NonnullRefPtr source_code, Realm&, HostDefined* = nullptr); diff --git a/Libraries/LibJS/SyntheticModule.cpp b/Libraries/LibJS/SyntheticModule.cpp index 5e6046b0b5..22088cfed9 100644 --- a/Libraries/LibJS/SyntheticModule.cpp +++ b/Libraries/LibJS/SyntheticModule.cpp @@ -63,7 +63,10 @@ ThrowCompletionOr> parse_json_module(Realm& realm, Stri auto& vm = realm.vm(); // 1. Let json be ? ParseJSON(source). - auto json = TRY(JSONObject::parse_json(vm, source_text)); + auto json_text = Utf16String::try_from_utf8(source_text); + if (json_text.is_error()) + return vm.throw_completion(ErrorType::JsonMalformed); + auto json = TRY(JSONObject::parse_json(vm, json_text.release_value())); // 3. Return CreateDefaultExportSyntheticModule(json). return SyntheticModule::create_default_export_synthetic_module(realm, json, move(filename)); diff --git a/Libraries/LibTest/JavaScriptTestRunner.h b/Libraries/LibTest/JavaScriptTestRunner.h index 3dd71afa79..350db9a4db 100644 --- a/Libraries/LibTest/JavaScriptTestRunner.h +++ b/Libraries/LibTest/JavaScriptTestRunner.h @@ -235,11 +235,12 @@ inline ByteBuffer load_entire_file(StringView path) inline AK::Result, ParserError> parse_script(StringView path, JS::Realm& realm) { auto contents = load_entire_file(path); - auto script_or_errors = JS::Script::parse(contents, realm, path); + auto source_text = Utf16String::from_utf8(StringView { contents.bytes() }); + auto script_or_errors = JS::Script::parse(source_text.utf16_view(), realm, path); if (script_or_errors.is_error()) { auto errors = script_or_errors.release_error(); - return ParserError { errors[0], errors[0].source_location_hint(Utf16String::from_utf8(contents)) }; + return ParserError { errors[0], errors[0].source_location_hint(source_text) }; } return script_or_errors.release_value(); @@ -419,7 +420,7 @@ inline JSFileResult TestRunner::run_file_test(ByteString const& test_path) auto& arr = user_output.as_array_exotic_object(); for (u32 i = 0; i < arr.indexed_array_like_size(); ++i) { auto message = MUST(arr.get(i)); - file_result.logged_messages.append(message.to_string_without_side_effects().to_byte_string()); + file_result.logged_messages.append(message.to_utf16_string_without_side_effects().to_utf8().to_byte_string()); } test_json.value().as_object().for_each_member([&](String const& suite_name, JsonValue const& suite_value) { @@ -492,11 +493,14 @@ inline JSFileResult TestRunner::run_file_test(ByteString const& test_path) auto message = error_object.get_without_side_effects(g_vm->names.message); if (name.is_accessor() || message.is_accessor()) { - detail_builder.append(error.to_string_without_side_effects()); + auto error_string = error.to_utf16_string_without_side_effects(); + detail_builder.append(error_string.utf16_view()); } else { - detail_builder.append(name.to_string_without_side_effects()); + auto name_string = name.to_utf16_string_without_side_effects(); + detail_builder.append(name_string.utf16_view()); detail_builder.append(": "sv); - detail_builder.append(message.to_string_without_side_effects()); + auto message_string = message.to_utf16_string_without_side_effects(); + detail_builder.append(message_string.utf16_view()); } if (is(error_object)) { @@ -507,7 +511,7 @@ inline JSFileResult TestRunner::run_file_test(ByteString const& test_path) test_case.details = MUST(detail_builder.to_string()); } else { - test_case.details = error.to_string_without_side_effects(); + test_case.details = error.to_utf16_string_without_side_effects().to_utf8(); } suite.tests.append(move(test_case)); diff --git a/Libraries/LibUnicode/Calendar.cpp b/Libraries/LibUnicode/Calendar.cpp index 0f783e4c5d..bf30f31867 100644 --- a/Libraries/LibUnicode/Calendar.cpp +++ b/Libraries/LibUnicode/Calendar.cpp @@ -35,6 +35,31 @@ static constexpr bool is_valid_month_code_string(StringView month_code) return true; } +static constexpr bool is_valid_month_code_string(Utf16View month_code) +{ + // MonthCode ::: + // M00L + // M0 NonZeroDigit L[opt] + // M NonZeroDigit DecimalDigit L[opt] + auto length = month_code.length_in_code_units(); + + if (length != 3 && length != 4) + return false; + + if (month_code.code_unit_at(0) != 'M') + return false; + + if (!is_ascii_digit(month_code.code_unit_at(1)) || !is_ascii_digit(month_code.code_unit_at(2))) + return false; + + if (length == 3 && month_code.code_unit_at(1) == '0' && month_code.code_unit_at(2) == '0') + return false; + if (length == 4 && month_code.code_unit_at(3) != 'L') + return false; + + return true; +} + // 12.2.1 ParseMonthCode ( argument ), https://tc39.es/proposal-temporal/#sec-temporal-parsemonthcode Optional parse_month_code(StringView month_code) { @@ -64,6 +89,34 @@ Optional parse_month_code(StringView month_code) return MonthCode { month_number, is_leap_month }; } +// 12.2.1 ParseMonthCode ( argument ), https://tc39.es/proposal-temporal/#sec-temporal-parsemonthcode +Optional parse_month_code(Utf16View month_code) +{ + // 3. If ParseText(StringToCodePoints(monthCode), MonthCode) is a List of errors, throw a RangeError exception. + if (!is_valid_month_code_string(month_code)) + return {}; + + // 4. Let isLeapMonth be false. + auto is_leap_month = false; + + // 5. If the length of monthCode = 4, then + if (month_code.length_in_code_units() == 4) { + // a. Assert: The fourth code unit of monthCode is 0x004C (LATIN CAPITAL LETTER L). + VERIFY(month_code.code_unit_at(3) == 'L'); + + // b. Set isLeapMonth to true. + is_leap_month = true; + } + + // 6. Let monthCodeDigits be the substring of monthCode from 1 to 3. + + // 7. Let monthNumber be ℝ(StringToNumber(monthCodeDigits)). + auto month_number = static_cast((month_code.code_unit_at(1) - '0') * 10 + (month_code.code_unit_at(2) - '0')); + + // 8. Return the Record { [[MonthNumber]]: monthNumber, [[IsLeapMonth]]: isLeapMonth }. + return MonthCode { month_number, is_leap_month }; +} + // 12.2.2 CreateMonthCode ( monthNumber, isLeapMonth ), https://tc39.es/proposal-temporal/#sec-temporal-createmonthcode String create_month_code(u8 month_number, bool is_leap_month) { diff --git a/Libraries/LibUnicode/Calendar.h b/Libraries/LibUnicode/Calendar.h index 2b353e4617..6cce86f1c1 100644 --- a/Libraries/LibUnicode/Calendar.h +++ b/Libraries/LibUnicode/Calendar.h @@ -9,6 +9,7 @@ #include #include #include +#include namespace Unicode { @@ -50,6 +51,7 @@ struct CalendarDate { }; Optional parse_month_code(StringView month_code); +Optional parse_month_code(Utf16View month_code); String create_month_code(u8 month_number, bool is_leap_month); CalendarDate iso_date_to_calendar_date(String const& calendar, ISODate); diff --git a/Libraries/LibUnicode/Collator.cpp b/Libraries/LibUnicode/Collator.cpp index 55d94cfc7f..55e08a0a1c 100644 --- a/Libraries/LibUnicode/Collator.cpp +++ b/Libraries/LibUnicode/Collator.cpp @@ -11,7 +11,7 @@ namespace Unicode { -Usage usage_from_string(StringView usage) +Usage usage_from_string(Utf16View usage) { if (usage == "sort"sv) return Usage::Sort; @@ -49,7 +49,7 @@ static NonnullOwnPtr apply_usage_to_locale(icu::Locale const& local return result; } -Sensitivity sensitivity_from_string(StringView sensitivity) +Sensitivity sensitivity_from_string(Utf16View sensitivity) { if (sensitivity == "base"sv) return Sensitivity::Base; diff --git a/Libraries/LibUnicode/Collator.h b/Libraries/LibUnicode/Collator.h index e0e2c9a5f2..c683af3047 100644 --- a/Libraries/LibUnicode/Collator.h +++ b/Libraries/LibUnicode/Collator.h @@ -8,6 +8,7 @@ #include #include +#include namespace Unicode { @@ -15,7 +16,7 @@ enum class Usage { Sort, Search, }; -Usage usage_from_string(StringView); +Usage usage_from_string(Utf16View); StringView usage_to_string(Usage); enum class Sensitivity { @@ -24,7 +25,7 @@ enum class Sensitivity { Case, Variant, }; -Sensitivity sensitivity_from_string(StringView); +Sensitivity sensitivity_from_string(Utf16View); StringView sensitivity_to_string(Sensitivity); enum class CaseFirst { diff --git a/Libraries/LibUnicode/DateTimeFormat.cpp b/Libraries/LibUnicode/DateTimeFormat.cpp index 5e9e967f76..8adaacb43a 100644 --- a/Libraries/LibUnicode/DateTimeFormat.cpp +++ b/Libraries/LibUnicode/DateTimeFormat.cpp @@ -30,7 +30,7 @@ namespace Unicode { -DateTimeStyle date_time_style_from_string(StringView style) +DateTimeStyle date_time_style_from_string(Utf16View style) { if (style == "full"sv) return DateTimeStyle::Full; @@ -146,7 +146,7 @@ static constexpr char icu_hour_cycle(Optional const& hour_cycle, Opti VERIFY_NOT_REACHED(); } -CalendarPatternStyle calendar_pattern_style_from_string(StringView style) +CalendarPatternStyle calendar_pattern_style_from_string(Utf16View style) { if (style == "narrow"sv) return CalendarPatternStyle::Narrow; diff --git a/Libraries/LibUnicode/DateTimeFormat.h b/Libraries/LibUnicode/DateTimeFormat.h index 048ba5f1d1..52d48ba2e9 100644 --- a/Libraries/LibUnicode/DateTimeFormat.h +++ b/Libraries/LibUnicode/DateTimeFormat.h @@ -13,6 +13,7 @@ #include #include #include +#include #include #include @@ -24,7 +25,7 @@ enum class DateTimeStyle { Medium, Short, }; -DateTimeStyle date_time_style_from_string(StringView); +DateTimeStyle date_time_style_from_string(Utf16View); StringView date_time_style_to_string(DateTimeStyle); enum class Weekday { @@ -58,7 +59,7 @@ enum class CalendarPatternStyle { ShortGeneric, LongGeneric, }; -CalendarPatternStyle calendar_pattern_style_from_string(StringView style); +CalendarPatternStyle calendar_pattern_style_from_string(Utf16View style); StringView calendar_pattern_style_to_string(CalendarPatternStyle style); struct CalendarPattern { diff --git a/Libraries/LibUnicode/DisplayNames.cpp b/Libraries/LibUnicode/DisplayNames.cpp index 14574c14a9..0f8a55ec48 100644 --- a/Libraries/LibUnicode/DisplayNames.cpp +++ b/Libraries/LibUnicode/DisplayNames.cpp @@ -16,7 +16,7 @@ namespace Unicode { -LanguageDisplay language_display_from_string(StringView language_display) +LanguageDisplay language_display_from_string(Utf16View language_display) { if (language_display == "standard"sv) return LanguageDisplay::Standard; diff --git a/Libraries/LibUnicode/DisplayNames.h b/Libraries/LibUnicode/DisplayNames.h index bd9ad1e850..d337949b41 100644 --- a/Libraries/LibUnicode/DisplayNames.h +++ b/Libraries/LibUnicode/DisplayNames.h @@ -9,6 +9,7 @@ #include #include #include +#include #include #include @@ -19,7 +20,7 @@ enum class LanguageDisplay { Dialect, }; -LanguageDisplay language_display_from_string(StringView language_display); +LanguageDisplay language_display_from_string(Utf16View language_display); StringView language_display_to_string(LanguageDisplay language_display); Optional language_display_name(StringView locale, StringView language, LanguageDisplay); diff --git a/Libraries/LibUnicode/ListFormat.cpp b/Libraries/LibUnicode/ListFormat.cpp index feef4c391d..e12d11b2bb 100644 --- a/Libraries/LibUnicode/ListFormat.cpp +++ b/Libraries/LibUnicode/ListFormat.cpp @@ -23,6 +23,17 @@ ListFormatType list_format_type_from_string(StringView list_format_type) VERIFY_NOT_REACHED(); } +ListFormatType list_format_type_from_string(Utf16View list_format_type) +{ + if (list_format_type == "conjunction"sv) + return ListFormatType::Conjunction; + if (list_format_type == "disjunction"sv) + return ListFormatType::Disjunction; + if (list_format_type == "unit"sv) + return ListFormatType::Unit; + VERIFY_NOT_REACHED(); +} + StringView list_format_type_to_string(ListFormatType list_format_type) { switch (list_format_type) { diff --git a/Libraries/LibUnicode/ListFormat.h b/Libraries/LibUnicode/ListFormat.h index fc522a8b11..8a8a6172d9 100644 --- a/Libraries/LibUnicode/ListFormat.h +++ b/Libraries/LibUnicode/ListFormat.h @@ -7,6 +7,7 @@ #pragma once #include +#include #include #include @@ -18,6 +19,7 @@ enum class ListFormatType { Unit, }; ListFormatType list_format_type_from_string(StringView); +ListFormatType list_format_type_from_string(Utf16View); StringView list_format_type_to_string(ListFormatType); class ListFormat { diff --git a/Libraries/LibUnicode/Locale.cpp b/Libraries/LibUnicode/Locale.cpp index edfa404c21..d92e3052bd 100644 --- a/Libraries/LibUnicode/Locale.cpp +++ b/Libraries/LibUnicode/Locale.cpp @@ -18,12 +18,41 @@ namespace Unicode { -static bool is_key(StringView key) +template +static constexpr size_t view_length(ViewType const& view) +{ + if constexpr (IsSame) + return view.length_in_code_units(); + else + return view.length(); +} + +template +static constexpr u32 view_code_unit_at(ViewType const& view, size_t index) +{ + if constexpr (IsSame) + return view.code_unit_at(index); + else + return static_cast(view[index]); +} + +static String string_from_ascii_view(StringView string) +{ + return MUST(String::from_utf8(string)); +} + +static String string_from_ascii_view(Utf16View string) +{ + return MUST(string.to_utf8()); +} + +template +static bool is_key(ViewType key) { // key = alphanum alpha - if (key.length() != 2) + if (view_length(key) != 2) return false; - return is_ascii_alphanumeric(key[0]) && is_ascii_alpha(key[1]); + return is_ascii_alphanumeric(view_code_unit_at(key, 0)) && is_ascii_alpha(view_code_unit_at(key, 1)); } static bool is_single_type(StringView type) @@ -35,34 +64,47 @@ static bool is_single_type(StringView type) return all_of(type, is_ascii_alphanumeric); } -static bool is_attribute(StringView type) +static bool is_single_type(Utf16View type) { - // attribute = alphanum{3,8} - if ((type.length() < 3) || (type.length() > 8)) + // type = alphanum{3,8} (sep alphanum{3,8})* + // Note: Consecutive types are not handled here, that is left to the caller. + if ((type.length_in_code_units() < 3) || (type.length_in_code_units() > 8)) return false; return all_of(type, is_ascii_alphanumeric); } -static bool is_transformed_key(StringView key) +template +static bool is_attribute(ViewType type) { - // tkey = alpha digit - if (key.length() != 2) + // attribute = alphanum{3,8} + if ((view_length(type) < 3) || (view_length(type) > 8)) return false; - return is_ascii_alpha(key[0]) && is_ascii_digit(key[1]); + return all_of(type, is_ascii_alphanumeric); } -static bool is_single_transformed_value(StringView value) +template +static bool is_transformed_key(ViewType key) +{ + // tkey = alpha digit + if (view_length(key) != 2) + return false; + return is_ascii_alpha(view_code_unit_at(key, 0)) && is_ascii_digit(view_code_unit_at(key, 1)); +} + +template +static bool is_single_transformed_value(ViewType value) { // tvalue = (sep alphanum{3,8})+ // Note: Consecutive values are not handled here, that is left to the caller. - if ((value.length() < 3) || (value.length() > 8)) + if ((view_length(value) < 3) || (view_length(value) > 8)) return false; return all_of(value, is_ascii_alphanumeric); } -static Optional consume_next_segment(GenericLexer& lexer, bool with_separator = true) +template +static Optional consume_next_segment(Lexer& lexer, bool with_separator = true) { - constexpr auto is_separator = is_any_of("-_"sv); + constexpr auto is_separator = [](auto code_unit) { return code_unit == '-' || code_unit == '_'; }; if (with_separator) { if (!lexer.next_is(is_separator)) @@ -95,7 +137,40 @@ bool is_type_identifier(StringView identifier) return lexer.is_eof() && (lexer.tell() > 0); } -static Optional parse_unicode_language_id(GenericLexer& lexer) +bool is_type_identifier(Utf16View identifier) +{ + // type = alphanum{3,8} (sep alphanum{3,8})* + bool saw_type = false; + bool is_valid = true; + size_t start = 0; + + auto validate_type = [&](Utf16View type) { + saw_type = true; + if (type.is_empty() || !is_single_type(type)) { + is_valid = false; + return IterationDecision::Break; + } + return IterationDecision::Continue; + }; + + for (size_t i = 0; i < identifier.length_in_code_units(); ++i) { + auto code_unit = identifier.code_unit_at(i); + if (code_unit != '-' && code_unit != '_') + continue; + + if (validate_type(identifier.substring_view(start, i - start)) == IterationDecision::Break) + return false; + start = i + 1; + } + + if (validate_type(identifier.substring_view(start)) == IterationDecision::Break) + return false; + + return saw_type && is_valid; +} + +template +static Optional parse_unicode_language_id_from_lexer(Lexer& lexer) { // https://unicode.org/reports/tr35/#Unicode_language_identifier // @@ -130,10 +205,10 @@ static Optional parse_unicode_language_id(GenericLexer& lexer) case ParseState::ParsingLanguageOrScript: if (is_unicode_language_subtag(*segment)) { state = ParseState::ParsingScript; - language_id.language = MUST(String::from_utf8(*segment)); + language_id.language = string_from_ascii_view(*segment); } else if (is_unicode_script_subtag(*segment)) { state = ParseState::ParsingRegion; - language_id.script = MUST(String::from_utf8(*segment)); + language_id.script = string_from_ascii_view(*segment); } else { return {}; } @@ -142,7 +217,7 @@ static Optional parse_unicode_language_id(GenericLexer& lexer) case ParseState::ParsingScript: if (is_unicode_script_subtag(*segment)) { state = ParseState::ParsingRegion; - language_id.script = MUST(String::from_utf8(*segment)); + language_id.script = string_from_ascii_view(*segment); break; } @@ -152,7 +227,7 @@ static Optional parse_unicode_language_id(GenericLexer& lexer) case ParseState::ParsingRegion: if (is_unicode_region_subtag(*segment)) { state = ParseState::ParsingVariant; - language_id.region = MUST(String::from_utf8(*segment)); + language_id.region = string_from_ascii_view(*segment); break; } @@ -161,9 +236,9 @@ static Optional parse_unicode_language_id(GenericLexer& lexer) case ParseState::ParsingVariant: if (is_unicode_variant_subtag(*segment)) { - language_id.variants.append(MUST(String::from_utf8(*segment))); + language_id.variants.append(string_from_ascii_view(*segment)); } else { - lexer.retreat(segment->length() + 1); + lexer.retreat(view_length(*segment) + 1); state = ParseState::Done; } break; @@ -176,7 +251,8 @@ static Optional parse_unicode_language_id(GenericLexer& lexer) return language_id; } -static Optional parse_unicode_locale_extension(GenericLexer& lexer) +template +static Optional parse_unicode_locale_extension(Lexer& lexer) { // https://unicode.org/reports/tr35/#unicode_locale_extensions // @@ -203,7 +279,7 @@ static Optional parse_unicode_locale_extension(GenericLexer& le switch (state) { case ParseState::ParsingAttribute: if (is_attribute(*segment)) { - locale_extension.attributes.append(MUST(String::from_utf8(*segment))); + locale_extension.attributes.append(string_from_ascii_view(*segment)); break; } @@ -212,11 +288,11 @@ static Optional parse_unicode_locale_extension(GenericLexer& le case ParseState::ParsingKeyword: { // keyword = key (sep type)? - Keyword keyword { .key = MUST(String::from_utf8(*segment)) }; - Vector keyword_values; + Keyword keyword { .key = string_from_ascii_view(*segment) }; + Vector keyword_values; if (!is_key(*segment)) { - lexer.retreat(segment->length() + 1); + lexer.retreat(view_length(*segment) + 1); state = ParseState::Done; break; } @@ -226,11 +302,11 @@ static Optional parse_unicode_locale_extension(GenericLexer& le if (!type.has_value() || !is_single_type(*type)) { if (type.has_value()) - lexer.retreat(type->length() + 1); + lexer.retreat(view_length(*type) + 1); break; } - keyword_values.append(*type); + keyword_values.append(string_from_ascii_view(*type)); } StringBuilder builder; @@ -251,7 +327,8 @@ static Optional parse_unicode_locale_extension(GenericLexer& le return locale_extension; } -static Optional parse_transformed_extension(GenericLexer& lexer) +template +static Optional parse_transformed_extension(Lexer& lexer) { // https://unicode.org/reports/tr35/#transformed_extensions // @@ -277,9 +354,9 @@ static Optional parse_transformed_extension(GenericLexer& switch (state) { case ParseState::ParsingLanguage: - lexer.retreat(segment->length()); + lexer.retreat(view_length(*segment)); - if (auto language_id = parse_unicode_language_id(lexer); language_id.has_value()) { + if (auto language_id = parse_unicode_language_id_from_lexer(lexer); language_id.has_value()) { transformed_extension.language = language_id.release_value(); state = ParseState::ParsingField; break; @@ -289,11 +366,11 @@ static Optional parse_transformed_extension(GenericLexer& case ParseState::ParsingField: { // tfield = tkey tvalue; - TransformedField field { .key = MUST(String::from_utf8(*segment)) }; - Vector field_values; + TransformedField field { .key = string_from_ascii_view(*segment) }; + Vector field_values; if (!is_transformed_key(*segment)) { - lexer.retreat(segment->length() + 1); + lexer.retreat(view_length(*segment) + 1); state = ParseState::Done; break; } @@ -303,11 +380,11 @@ static Optional parse_transformed_extension(GenericLexer& if (!value.has_value() || !is_single_transformed_value(*value)) { if (value.has_value()) - lexer.retreat(value->length() + 1); + lexer.retreat(view_length(*value) + 1); break; } - field_values.append(*value); + field_values.append(string_from_ascii_view(*value)); } if (field_values.is_empty()) @@ -331,28 +408,30 @@ static Optional parse_transformed_extension(GenericLexer& return transformed_extension; } -static Optional parse_other_extension(char key, GenericLexer& lexer) +template +static Optional parse_other_extension(u32 key, Lexer& lexer) { // https://unicode.org/reports/tr35/#other_extensions // // other_extensions = sep [alphanum-[tTuUxX]] (sep alphanum{2,8})+ ; - OtherExtension other_extension { .key = key }; - Vector other_values; + Vector other_values; if (!is_ascii_alphanumeric(key) || (key == 'x') || (key == 'X')) return {}; + OtherExtension other_extension { .key = static_cast(key) }; + while (true) { auto segment = consume_next_segment(lexer); if (!segment.has_value()) break; - if ((segment->length() < 2) || (segment->length() > 8) || !all_of(*segment, is_ascii_alphanumeric)) { - lexer.retreat(segment->length() + 1); + if ((view_length(*segment) < 2) || (view_length(*segment) > 8) || !all_of(*segment, is_ascii_alphanumeric)) { + lexer.retreat(view_length(*segment) + 1); break; } - other_values.append(*segment); + other_values.append(string_from_ascii_view(*segment)); } if (other_values.is_empty()) @@ -365,15 +444,16 @@ static Optional parse_other_extension(char key, GenericLexer& le return other_extension; } -static Optional parse_extension(GenericLexer& lexer) +template +static Optional parse_extension(Lexer& lexer) { // https://unicode.org/reports/tr35/#extensions // // extensions = unicode_locale_extensions | transformed_extensions | other_extensions size_t starting_position = lexer.tell(); - if (auto header = consume_next_segment(lexer); header.has_value() && (header->length() == 1)) { - switch (char key = (*header)[0]) { + if (auto header = consume_next_segment(lexer); header.has_value() && (view_length(*header) == 1)) { + switch (auto key = view_code_unit_at(*header, 0)) { case 'u': case 'U': if (auto extension = parse_unicode_locale_extension(lexer); extension.has_value()) @@ -397,7 +477,8 @@ static Optional parse_extension(GenericLexer& lexer) return {}; } -static Vector parse_private_use_extensions(GenericLexer& lexer) +template +static Vector parse_private_use_extensions(Lexer& lexer) { // https://unicode.org/reports/tr35/#pu_extensions // @@ -416,18 +497,18 @@ static Vector parse_private_use_extensions(GenericLexer& lexer) if (!segment.has_value()) break; - if ((segment->length() < 1) || (segment->length() > 8) || !all_of(*segment, is_ascii_alphanumeric)) { - lexer.retreat(segment->length() + 1); + if ((view_length(*segment) < 1) || (view_length(*segment) > 8) || !all_of(*segment, is_ascii_alphanumeric)) { + lexer.retreat(view_length(*segment) + 1); break; } - extensions.append(MUST(String::from_utf8(*segment))); + extensions.append(string_from_ascii_view(*segment)); } return extensions; }; - if ((header->length() == 1) && (((*header)[0] == 'x') || ((*header)[0] == 'X'))) { + if ((view_length(*header) == 1) && ((view_code_unit_at(*header, 0) == 'x') || (view_code_unit_at(*header, 0) == 'X'))) { if (auto extensions = parse_values(); !extensions.is_empty()) return extensions; } @@ -436,27 +517,15 @@ static Vector parse_private_use_extensions(GenericLexer& lexer) return {}; } -Optional parse_unicode_language_id(StringView language) +template +static Optional parse_unicode_locale_id_from_lexer(Lexer& lexer) { - GenericLexer lexer { language }; - - auto language_id = parse_unicode_language_id(lexer); - if (!lexer.is_eof()) - return {}; - - return language_id; -} - -Optional parse_unicode_locale_id(StringView locale) -{ - GenericLexer lexer { locale }; - // https://unicode.org/reports/tr35/#Unicode_locale_identifier // // unicode_locale_id = unicode_language_id // extensions* // pu_extensions? - auto language_id = parse_unicode_language_id(lexer); + auto language_id = parse_unicode_language_id_from_lexer(lexer); if (!language_id.has_value()) return {}; @@ -477,11 +546,52 @@ Optional parse_unicode_locale_id(StringView locale) return locale_id; } +Optional parse_unicode_language_id(StringView language) +{ + GenericLexer lexer { language }; + + auto language_id = parse_unicode_language_id_from_lexer(lexer); + if (!lexer.is_eof()) + return {}; + + return language_id; +} + +Optional parse_unicode_language_id(Utf16View language) +{ + Utf16GenericLexer lexer { language }; + + auto language_id = parse_unicode_language_id_from_lexer(lexer); + if (!lexer.is_eof()) + return {}; + + return language_id; +} + +Optional parse_unicode_locale_id(StringView locale) +{ + GenericLexer lexer { locale }; + + return parse_unicode_locale_id_from_lexer(lexer); +} + +Optional parse_unicode_locale_id(Utf16View locale) +{ + Utf16GenericLexer lexer { locale }; + + return parse_unicode_locale_id_from_lexer(lexer); +} + String canonicalize_unicode_locale_id(StringView locale) { return LocaleData::canonicalize(locale); } +String canonicalize_unicode_locale_id(Utf16View locale) +{ + return LocaleData::canonicalize(string_from_ascii_view(locale)); +} + String canonicalize_unicode_extension_values(StringView key, StringView value) { UErrorCode status = U_ZERO_ERROR; @@ -566,6 +676,17 @@ Style style_from_string(StringView style) VERIFY_NOT_REACHED(); } +Style style_from_string(Utf16View style) +{ + if (style == "narrow"sv) + return Style::Narrow; + if (style == "short"sv) + return Style::Short; + if (style == "long"sv) + return Style::Long; + VERIFY_NOT_REACHED(); +} + StringView style_to_string(Style style) { switch (style) { diff --git a/Libraries/LibUnicode/Locale.h b/Libraries/LibUnicode/Locale.h index 6269fc3485..0ba404a1e3 100644 --- a/Libraries/LibUnicode/Locale.h +++ b/Libraries/LibUnicode/Locale.h @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -106,6 +107,14 @@ constexpr bool is_unicode_language_subtag(StringView subtag) return all_of(subtag, is_ascii_alpha); } +constexpr bool is_unicode_language_subtag(Utf16View subtag) +{ + // unicode_language_subtag = alpha{2,3} | alpha{5,8} + if ((subtag.length_in_code_units() < 2) || (subtag.length_in_code_units() == 4) || (subtag.length_in_code_units() > 8)) + return false; + return all_of(subtag, is_ascii_alpha); +} + constexpr bool is_unicode_script_subtag(StringView subtag) { // unicode_script_subtag = alpha{4} @@ -114,6 +123,14 @@ constexpr bool is_unicode_script_subtag(StringView subtag) return all_of(subtag, is_ascii_alpha); } +constexpr bool is_unicode_script_subtag(Utf16View subtag) +{ + // unicode_script_subtag = alpha{4} + if (subtag.length_in_code_units() != 4) + return false; + return all_of(subtag, is_ascii_alpha); +} + constexpr bool is_unicode_region_subtag(StringView subtag) { // unicode_region_subtag = (alpha{2} | digit{3}) @@ -124,6 +141,16 @@ constexpr bool is_unicode_region_subtag(StringView subtag) return false; } +constexpr bool is_unicode_region_subtag(Utf16View subtag) +{ + // unicode_region_subtag = (alpha{2} | digit{3}) + if (subtag.length_in_code_units() == 2) + return all_of(subtag, is_ascii_alpha); + if (subtag.length_in_code_units() == 3) + return all_of(subtag, is_ascii_digit); + return false; +} + constexpr bool is_unicode_variant_subtag(StringView subtag) { // unicode_variant_subtag = (alphanum{5,8} | digit alphanum{3}) @@ -134,18 +161,33 @@ constexpr bool is_unicode_variant_subtag(StringView subtag) return false; } +constexpr bool is_unicode_variant_subtag(Utf16View subtag) +{ + // unicode_variant_subtag = (alphanum{5,8} | digit alphanum{3}) + if ((subtag.length_in_code_units() >= 5) && (subtag.length_in_code_units() <= 8)) + return all_of(subtag, is_ascii_alphanumeric); + if (subtag.length_in_code_units() == 4) + return is_ascii_digit(subtag.code_unit_at(0)) && all_of(subtag.substring_view(1), is_ascii_alphanumeric); + return false; +} + bool is_type_identifier(StringView); +bool is_type_identifier(Utf16View); Optional parse_unicode_language_id(StringView); +Optional parse_unicode_language_id(Utf16View); Optional parse_unicode_locale_id(StringView); +Optional parse_unicode_locale_id(Utf16View); String canonicalize_unicode_locale_id(StringView); +String canonicalize_unicode_locale_id(Utf16View); String canonicalize_unicode_extension_values(StringView key, StringView value); StringView default_locale(); bool is_locale_available(StringView locale); Style style_from_string(StringView style); +Style style_from_string(Utf16View style); StringView style_to_string(Style style); Optional add_likely_subtags(StringView); diff --git a/Libraries/LibUnicode/Normalize.cpp b/Libraries/LibUnicode/Normalize.cpp index c5b5b1cb97..4bf943abd7 100644 --- a/Libraries/LibUnicode/Normalize.cpp +++ b/Libraries/LibUnicode/Normalize.cpp @@ -26,6 +26,19 @@ NormalizationForm normalization_form_from_string(StringView form) VERIFY_NOT_REACHED(); } +NormalizationForm normalization_form_from_string(Utf16View form) +{ + if (form == u"NFD"sv) + return NormalizationForm::NFD; + if (form == u"NFC"sv) + return NormalizationForm::NFC; + if (form == u"NFKD"sv) + return NormalizationForm::NFKD; + if (form == u"NFKC"sv) + return NormalizationForm::NFKC; + VERIFY_NOT_REACHED(); +} + StringView normalization_form_to_string(NormalizationForm form) { switch (form) { @@ -41,25 +54,25 @@ StringView normalization_form_to_string(NormalizationForm form) VERIFY_NOT_REACHED(); } +static icu::Normalizer2 const* normalizer_for_form(NormalizationForm form, UErrorCode& status) +{ + switch (form) { + case NormalizationForm::NFD: + return icu::Normalizer2::getNFDInstance(status); + case NormalizationForm::NFC: + return icu::Normalizer2::getNFCInstance(status); + case NormalizationForm::NFKD: + return icu::Normalizer2::getNFKDInstance(status); + case NormalizationForm::NFKC: + return icu::Normalizer2::getNFKCInstance(status); + } + VERIFY_NOT_REACHED(); +} + String normalize(StringView string, NormalizationForm form) { UErrorCode status = U_ZERO_ERROR; - icu::Normalizer2 const* normalizer = nullptr; - - switch (form) { - case NormalizationForm::NFD: - normalizer = icu::Normalizer2::getNFDInstance(status); - break; - case NormalizationForm::NFC: - normalizer = icu::Normalizer2::getNFCInstance(status); - break; - case NormalizationForm::NFKD: - normalizer = icu::Normalizer2::getNFKDInstance(status); - break; - case NormalizationForm::NFKC: - normalizer = icu::Normalizer2::getNFKCInstance(status); - break; - } + auto const* normalizer = normalizer_for_form(form, status); if (icu_failure(status)) return MUST(String::from_utf8(string)); @@ -76,4 +89,23 @@ String normalize(StringView string, NormalizationForm form) return MUST(builder.to_string()); } +Utf16String normalize(Utf16View string, NormalizationForm form) +{ + UErrorCode status = U_ZERO_ERROR; + auto const* normalizer = normalizer_for_form(form, status); + + if (icu_failure(status)) + return Utf16String::from_utf16(string); + + VERIFY(normalizer); + + auto icu_input = icu_string(string); + UErrorCode normalize_status = U_ZERO_ERROR; + auto icu_output = normalizer->normalize(icu_input, normalize_status); + if (icu_failure(normalize_status)) + return Utf16String::from_utf16(string); + + return icu_string_to_utf16_string(icu_output); +} + } diff --git a/Libraries/LibUnicode/Normalize.h b/Libraries/LibUnicode/Normalize.h index ef3b43c94e..b9c33885a0 100644 --- a/Libraries/LibUnicode/Normalize.h +++ b/Libraries/LibUnicode/Normalize.h @@ -9,6 +9,8 @@ #include #include +#include +#include namespace Unicode { @@ -19,8 +21,10 @@ enum class NormalizationForm { NFKC }; NormalizationForm normalization_form_from_string(StringView); +NormalizationForm normalization_form_from_string(Utf16View); StringView normalization_form_to_string(NormalizationForm); String normalize(StringView string, NormalizationForm form); +Utf16String normalize(Utf16View string, NormalizationForm form); } diff --git a/Libraries/LibUnicode/NumberFormat.cpp b/Libraries/LibUnicode/NumberFormat.cpp index 89e1f6d259..d8b48d9ebd 100644 --- a/Libraries/LibUnicode/NumberFormat.cpp +++ b/Libraries/LibUnicode/NumberFormat.cpp @@ -6,7 +6,6 @@ #include #include -#include #include #include #include @@ -32,6 +31,19 @@ NumberFormatStyle number_format_style_from_string(StringView number_format_style VERIFY_NOT_REACHED(); } +NumberFormatStyle number_format_style_from_string(Utf16View number_format_style) +{ + if (number_format_style == "decimal"sv) + return NumberFormatStyle::Decimal; + if (number_format_style == "percent"sv) + return NumberFormatStyle::Percent; + if (number_format_style == "currency"sv) + return NumberFormatStyle::Currency; + if (number_format_style == "unit"sv) + return NumberFormatStyle::Unit; + VERIFY_NOT_REACHED(); +} + StringView number_format_style_to_string(NumberFormatStyle number_format_style) { switch (number_format_style) { @@ -62,6 +74,21 @@ SignDisplay sign_display_from_string(StringView sign_display) VERIFY_NOT_REACHED(); } +SignDisplay sign_display_from_string(Utf16View sign_display) +{ + if (sign_display == "auto"sv) + return SignDisplay::Auto; + if (sign_display == "never"sv) + return SignDisplay::Never; + if (sign_display == "always"sv) + return SignDisplay::Always; + if (sign_display == "exceptZero"sv) + return SignDisplay::ExceptZero; + if (sign_display == "negative"sv) + return SignDisplay::Negative; + VERIFY_NOT_REACHED(); +} + StringView sign_display_to_string(SignDisplay sign_display) { switch (sign_display) { @@ -109,6 +136,19 @@ Notation notation_from_string(StringView notation) VERIFY_NOT_REACHED(); } +Notation notation_from_string(Utf16View notation) +{ + if (notation == "standard"sv) + return Notation::Standard; + if (notation == "scientific"sv) + return Notation::Scientific; + if (notation == "engineering"sv) + return Notation::Engineering; + if (notation == "compact"sv) + return Notation::Compact; + VERIFY_NOT_REACHED(); +} + StringView notation_to_string(Notation notation) { switch (notation) { @@ -153,6 +193,15 @@ CompactDisplay compact_display_from_string(StringView compact_display) VERIFY_NOT_REACHED(); } +CompactDisplay compact_display_from_string(Utf16View compact_display) +{ + if (compact_display == "short"sv) + return CompactDisplay::Short; + if (compact_display == "long"sv) + return CompactDisplay::Long; + VERIFY_NOT_REACHED(); +} + StringView compact_display_to_string(CompactDisplay compact_display) { switch (compact_display) { @@ -220,6 +269,19 @@ CurrencyDisplay currency_display_from_string(StringView currency_display) VERIFY_NOT_REACHED(); } +CurrencyDisplay currency_display_from_string(Utf16View currency_display) +{ + if (currency_display == "code"sv) + return CurrencyDisplay::Code; + if (currency_display == "symbol"sv) + return CurrencyDisplay::Symbol; + if (currency_display == "narrowSymbol"sv) + return CurrencyDisplay::NarrowSymbol; + if (currency_display == "name"sv) + return CurrencyDisplay::Name; + VERIFY_NOT_REACHED(); +} + StringView currency_display_to_string(CurrencyDisplay currency_display) { switch (currency_display) { @@ -259,6 +321,15 @@ CurrencySign currency_sign_from_string(StringView currency_sign) VERIFY_NOT_REACHED(); } +CurrencySign currency_sign_from_string(Utf16View currency_sign) +{ + if (currency_sign == "standard"sv) + return CurrencySign::Standard; + if (currency_sign == "accounting"sv) + return CurrencySign::Accounting; + VERIFY_NOT_REACHED(); +} + StringView currency_sign_to_string(CurrencySign currency_sign) { switch (currency_sign) { @@ -321,6 +392,29 @@ RoundingMode rounding_mode_from_string(StringView rounding_mode) VERIFY_NOT_REACHED(); } +RoundingMode rounding_mode_from_string(Utf16View rounding_mode) +{ + if (rounding_mode == "ceil"sv) + return RoundingMode::Ceil; + if (rounding_mode == "expand"sv) + return RoundingMode::Expand; + if (rounding_mode == "floor"sv) + return RoundingMode::Floor; + if (rounding_mode == "halfCeil"sv) + return RoundingMode::HalfCeil; + if (rounding_mode == "halfEven"sv) + return RoundingMode::HalfEven; + if (rounding_mode == "halfExpand"sv) + return RoundingMode::HalfExpand; + if (rounding_mode == "halfFloor"sv) + return RoundingMode::HalfFloor; + if (rounding_mode == "halfTrunc"sv) + return RoundingMode::HalfTrunc; + if (rounding_mode == "trunc"sv) + return RoundingMode::Trunc; + VERIFY_NOT_REACHED(); +} + StringView rounding_mode_to_string(RoundingMode rounding_mode) { switch (rounding_mode) { @@ -380,6 +474,15 @@ TrailingZeroDisplay trailing_zero_display_from_string(StringView trailing_zero_d VERIFY_NOT_REACHED(); } +TrailingZeroDisplay trailing_zero_display_from_string(Utf16View trailing_zero_display) +{ + if (trailing_zero_display == "auto"sv) + return TrailingZeroDisplay::Auto; + if (trailing_zero_display == "stripIfInteger"sv) + return TrailingZeroDisplay::StripIfInteger; + VERIFY_NOT_REACHED(); +} + StringView trailing_zero_display_to_string(TrailingZeroDisplay trailing_zero_display) { switch (trailing_zero_display) { @@ -522,7 +625,7 @@ static constexpr StringView icu_number_format_field_to_string(i32 field, NumberF case UNUM_SIGN_FIELD: { auto is_negative = value.visit( [&](double number) { return signbit(number); }, - [&](String const& number) { return number.starts_with('-'); }); + [&](Utf16String const& number) { return number.starts_with('-'); }); return is_negative ? "minusSign"sv : "plusSign"sv; } case UNUM_MEASURE_UNIT_FIELD: @@ -723,13 +826,36 @@ public: } private: + struct DecimalStringPiece { + String utf8_storage; + icu::StringPiece string_piece; + }; + + static DecimalStringPiece decimal_string_piece(Utf16String const& number) + { + auto number_view = number.utf16_view(); + + if (number_view.has_ascii_storage()) { + auto bytes = number_view.bytes(); + return { {}, { reinterpret_cast(bytes.data()), static_cast(bytes.size()) } }; + } + + DecimalStringPiece result; + result.utf8_storage = MUST(number_view.to_utf8()); + result.string_piece = icu_string_piece(result.utf8_storage); + return result; + } + static icu::Formattable value_to_formattable(Value const& value) { UErrorCode status = U_ZERO_ERROR; auto formattable = value.visit( [&](double number) { return icu::Formattable { number }; }, - [&](String const& number) { return icu::Formattable(icu_string_piece(number), status); }); + [&](Utf16String const& number) { + auto decimal_number = decimal_string_piece(number); + return icu::Formattable(decimal_number.string_piece, status); + }); verify_icu_success(status); return formattable; @@ -743,8 +869,9 @@ private: [&](double number) { return m_formatter.formatDouble(number, status); }, - [&](String const& number) { - return m_formatter.formatDecimal(icu_string_piece(number), status); + [&](Utf16String const& number) { + auto decimal_number = decimal_string_piece(number); + return m_formatter.formatDecimal(decimal_number.string_piece, status); }); if (icu_failure(status)) diff --git a/Libraries/LibUnicode/NumberFormat.h b/Libraries/LibUnicode/NumberFormat.h index 02211b3ff1..fff34b7629 100644 --- a/Libraries/LibUnicode/NumberFormat.h +++ b/Libraries/LibUnicode/NumberFormat.h @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -24,6 +25,7 @@ enum class NumberFormatStyle { Unit, }; NumberFormatStyle number_format_style_from_string(StringView); +NumberFormatStyle number_format_style_from_string(Utf16View); StringView number_format_style_to_string(NumberFormatStyle); enum class SignDisplay { @@ -34,6 +36,7 @@ enum class SignDisplay { Negative, }; SignDisplay sign_display_from_string(StringView); +SignDisplay sign_display_from_string(Utf16View); StringView sign_display_to_string(SignDisplay); enum class Notation { @@ -43,6 +46,7 @@ enum class Notation { Compact, }; Notation notation_from_string(StringView); +Notation notation_from_string(Utf16View); StringView notation_to_string(Notation); enum class CompactDisplay { @@ -50,6 +54,7 @@ enum class CompactDisplay { Long, }; CompactDisplay compact_display_from_string(StringView); +CompactDisplay compact_display_from_string(Utf16View); StringView compact_display_to_string(CompactDisplay); enum class Grouping { @@ -68,6 +73,7 @@ enum class CurrencyDisplay { Name, }; CurrencyDisplay currency_display_from_string(StringView); +CurrencyDisplay currency_display_from_string(Utf16View); StringView currency_display_to_string(CurrencyDisplay); enum class CurrencySign { @@ -75,6 +81,7 @@ enum class CurrencySign { Accounting, }; CurrencySign currency_sign_from_string(StringView); +CurrencySign currency_sign_from_string(Utf16View); StringView currency_sign_to_string(CurrencySign); struct DisplayOptions { @@ -115,6 +122,7 @@ enum class RoundingMode { Trunc, }; RoundingMode rounding_mode_from_string(StringView); +RoundingMode rounding_mode_from_string(Utf16View); StringView rounding_mode_to_string(RoundingMode); enum class TrailingZeroDisplay { @@ -122,6 +130,7 @@ enum class TrailingZeroDisplay { StripIfInteger, }; TrailingZeroDisplay trailing_zero_display_from_string(StringView); +TrailingZeroDisplay trailing_zero_display_from_string(Utf16View); StringView trailing_zero_display_to_string(TrailingZeroDisplay); struct RoundingOptions { @@ -154,7 +163,7 @@ public: StringView source; }; - using Value = Variant; + using Value = Variant; virtual Utf16String format(Value const&) const = 0; virtual Vector format_to_parts(Value const&) const = 0; diff --git a/Libraries/LibUnicode/PluralRules.cpp b/Libraries/LibUnicode/PluralRules.cpp index e229aa06a7..444d30dc70 100644 --- a/Libraries/LibUnicode/PluralRules.cpp +++ b/Libraries/LibUnicode/PluralRules.cpp @@ -17,6 +17,15 @@ PluralForm plural_form_from_string(StringView plural_form) VERIFY_NOT_REACHED(); } +PluralForm plural_form_from_string(Utf16View plural_form) +{ + if (plural_form == "cardinal"sv) + return PluralForm::Cardinal; + if (plural_form == "ordinal"sv) + return PluralForm::Ordinal; + VERIFY_NOT_REACHED(); +} + StringView plural_form_to_string(PluralForm plural_form) { switch (plural_form) { diff --git a/Libraries/LibUnicode/PluralRules.h b/Libraries/LibUnicode/PluralRules.h index 7b3a21eb1b..ba1ef24a1a 100644 --- a/Libraries/LibUnicode/PluralRules.h +++ b/Libraries/LibUnicode/PluralRules.h @@ -16,6 +16,7 @@ enum class PluralForm { Ordinal, }; PluralForm plural_form_from_string(StringView); +PluralForm plural_form_from_string(Utf16View); StringView plural_form_to_string(PluralForm); enum class PluralCategory { diff --git a/Libraries/LibUnicode/RelativeTimeFormat.cpp b/Libraries/LibUnicode/RelativeTimeFormat.cpp index 51d90a02b6..51bb0f9b6c 100644 --- a/Libraries/LibUnicode/RelativeTimeFormat.cpp +++ b/Libraries/LibUnicode/RelativeTimeFormat.cpp @@ -37,6 +37,27 @@ Optional time_unit_from_string(StringView time_unit) return {}; } +Optional time_unit_from_string(Utf16View time_unit) +{ + if (time_unit == "second"sv) + return TimeUnit::Second; + if (time_unit == "minute"sv) + return TimeUnit::Minute; + if (time_unit == "hour"sv) + return TimeUnit::Hour; + if (time_unit == "day"sv) + return TimeUnit::Day; + if (time_unit == "week"sv) + return TimeUnit::Week; + if (time_unit == "month"sv) + return TimeUnit::Month; + if (time_unit == "quarter"sv) + return TimeUnit::Quarter; + if (time_unit == "year"sv) + return TimeUnit::Year; + return {}; +} + StringView time_unit_to_string(TimeUnit time_unit) { switch (time_unit) { @@ -92,6 +113,15 @@ NumericDisplay numeric_display_from_string(StringView numeric_display) VERIFY_NOT_REACHED(); } +NumericDisplay numeric_display_from_string(Utf16View numeric_display) +{ + if (numeric_display == "always"sv) + return NumericDisplay::Always; + if (numeric_display == "auto"sv) + return NumericDisplay::Auto; + VERIFY_NOT_REACHED(); +} + StringView numeric_display_to_string(NumericDisplay numeric_display) { switch (numeric_display) { diff --git a/Libraries/LibUnicode/RelativeTimeFormat.h b/Libraries/LibUnicode/RelativeTimeFormat.h index aa4aa1b1a9..f538ca8e0c 100644 --- a/Libraries/LibUnicode/RelativeTimeFormat.h +++ b/Libraries/LibUnicode/RelativeTimeFormat.h @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -28,6 +29,7 @@ enum class TimeUnit { Year, }; Optional time_unit_from_string(StringView); +Optional time_unit_from_string(Utf16View); StringView time_unit_to_string(TimeUnit); enum class NumericDisplay { @@ -35,6 +37,7 @@ enum class NumericDisplay { Auto, }; NumericDisplay numeric_display_from_string(StringView); +NumericDisplay numeric_display_from_string(Utf16View); StringView numeric_display_to_string(NumericDisplay); class RelativeTimeFormat { diff --git a/Libraries/LibUnicode/Segmenter.cpp b/Libraries/LibUnicode/Segmenter.cpp index 2fdcc66bb5..41829c20f8 100644 --- a/Libraries/LibUnicode/Segmenter.cpp +++ b/Libraries/LibUnicode/Segmenter.cpp @@ -32,6 +32,19 @@ SegmenterGranularity segmenter_granularity_from_string(StringView segmenter_gran VERIFY_NOT_REACHED(); } +SegmenterGranularity segmenter_granularity_from_string(Utf16View segmenter_granularity) +{ + if (segmenter_granularity == "grapheme"sv) + return SegmenterGranularity::Grapheme; + if (segmenter_granularity == "line"sv) + return SegmenterGranularity::Line; + if (segmenter_granularity == "sentence"sv) + return SegmenterGranularity::Sentence; + if (segmenter_granularity == "word"sv) + return SegmenterGranularity::Word; + VERIFY_NOT_REACHED(); +} + StringView segmenter_granularity_to_string(SegmenterGranularity segmenter_granularity) { switch (segmenter_granularity) { diff --git a/Libraries/LibUnicode/Segmenter.h b/Libraries/LibUnicode/Segmenter.h index 8466117d79..d9b85f122b 100644 --- a/Libraries/LibUnicode/Segmenter.h +++ b/Libraries/LibUnicode/Segmenter.h @@ -11,6 +11,7 @@ #include #include #include +#include namespace Unicode { @@ -21,6 +22,7 @@ enum class SegmenterGranularity { Word, }; SegmenterGranularity segmenter_granularity_from_string(StringView); +SegmenterGranularity segmenter_granularity_from_string(Utf16View); StringView segmenter_granularity_to_string(SegmenterGranularity); class Segmenter { diff --git a/Libraries/LibWeb/Bindings/MainThreadVM.cpp b/Libraries/LibWeb/Bindings/MainThreadVM.cpp index bcf67bc0e4..5ef4a42391 100644 --- a/Libraries/LibWeb/Bindings/MainThreadVM.cpp +++ b/Libraries/LibWeb/Bindings/MainThreadVM.cpp @@ -122,7 +122,7 @@ void initialize_main_thread_vm(AgentType type) }; // 8.1.6.2 HostEnsureCanCompileStrings(realm, parameterStrings, bodyString, codeString, compilationType, parameterArgs, bodyArg), https://html.spec.whatwg.org/multipage/webappapis.html#hostensurecancompilestrings(realm,-parameterstrings,-bodystring,-codestring,-compilationtype,-parameterargs,-bodyarg) - main_thread_vm_ptr()->host_ensure_can_compile_strings = [](JS::Realm& realm, ReadonlySpan parameter_strings, StringView body_string, StringView code_string, JS::CompilationType compilation_type, ReadonlySpan parameter_args, JS::Value body_arg) -> JS::ThrowCompletionOr { + main_thread_vm_ptr()->host_ensure_can_compile_strings = [](JS::Realm& realm, ReadonlySpan parameter_strings, Utf16View body_string, Utf16View code_string, JS::CompilationType compilation_type, ReadonlySpan parameter_args, JS::Value body_arg) -> JS::ThrowCompletionOr { // 1. Perform ? EnsureCSPDoesNotBlockStringCompilation(realm, parameterStrings, bodyString, codeString, compilationType, parameterArgs, bodyArg). [CSP] return ContentSecurityPolicy::ensure_csp_does_not_block_string_compilation(realm, parameter_strings, body_string, code_string, compilation_type, parameter_args, body_arg); }; @@ -654,7 +654,7 @@ void initialize_main_thread_vm(AgentType type) HTML::fetch_single_imported_module_script(settings_object->realm(), url.release_value(), *fetch_client, destination, fetch_options, settings_object, fetch_referrer, module_request, perform_fetch, on_single_fetch_complete); }; - main_thread_vm_ptr()->host_unrecognized_date_string = [](StringView date) { + main_thread_vm_ptr()->host_unrecognized_date_string = [](Utf16View date) { dbgln("Unable to parse date string: \"{}\"", date); }; diff --git a/Libraries/LibWeb/ContentSecurityPolicy/BlockingAlgorithms.cpp b/Libraries/LibWeb/ContentSecurityPolicy/BlockingAlgorithms.cpp index 7c9760b6c7..f012daeff0 100644 --- a/Libraries/LibWeb/ContentSecurityPolicy/BlockingAlgorithms.cpp +++ b/Libraries/LibWeb/ContentSecurityPolicy/BlockingAlgorithms.cpp @@ -464,14 +464,14 @@ Directives::Directive::Result should_elements_inline_type_behavior_be_blocked_by } // https://w3c.github.io/webappsec-csp/#can-compile-strings -JS::ThrowCompletionOr ensure_csp_does_not_block_string_compilation(JS::Realm& realm, ReadonlySpan parameter_strings, StringView body_string, StringView code_string, JS::CompilationType compilation_type, ReadonlySpan parameter_args, JS::Value body_arg) +JS::ThrowCompletionOr ensure_csp_does_not_block_string_compilation(JS::Realm& realm, ReadonlySpan parameter_strings, Utf16View body_string, Utf16View code_string, JS::CompilationType compilation_type, ReadonlySpan parameter_args, JS::Value body_arg) { Utf16String source_string; // 1. If compilationType is "TIMER", then: if (compilation_type == JS::CompilationType::Timer) { // 1. Let sourceString be codeString. - source_string = Utf16String::from_utf8(code_string); + source_string = Utf16String::from_utf16(code_string); } // 2. Else: else { @@ -517,8 +517,8 @@ JS::ThrowCompletionOr ensure_csp_does_not_block_string_compilation(JS::Rea // 5. Let sourceToValidate be a new TrustedScript object created in realm whose data is set to codeString // if isTrusted is true, and codeString otherwise. auto const source_to_validate = is_trusted - ? TrustedTypes::TrustedScriptOrString(realm.create(realm, Utf16String::from_utf8(code_string))) - : Utf16String::from_utf8(code_string); + ? TrustedTypes::TrustedScriptOrString(realm.create(realm, Utf16String::from_utf16(code_string))) + : Utf16String::from_utf16(code_string); // 6. Let sourceString be the result of executing the Get Trusted Type compliant string algorithm, // with TrustedScript, realm, sourceToValidate, compilationSink, and 'script'. diff --git a/Libraries/LibWeb/ContentSecurityPolicy/BlockingAlgorithms.h b/Libraries/LibWeb/ContentSecurityPolicy/BlockingAlgorithms.h index 139255f591..3f3ff36ee3 100644 --- a/Libraries/LibWeb/ContentSecurityPolicy/BlockingAlgorithms.h +++ b/Libraries/LibWeb/ContentSecurityPolicy/BlockingAlgorithms.h @@ -26,7 +26,7 @@ Directives::Directive::Result should_navigation_response_to_navigation_request_o GC::Ref target); Directives::Directive::Result should_elements_inline_type_behavior_be_blocked_by_content_security_policy(JS::Realm&, GC::Ref element, Directives::Directive::InlineType type, String const& source); -JS::ThrowCompletionOr ensure_csp_does_not_block_string_compilation(JS::Realm& realm, ReadonlySpan parameter_strings, StringView body_string, StringView code_string, JS::CompilationType compilation_type, ReadonlySpan parameter_args, JS::Value body_arg); +JS::ThrowCompletionOr ensure_csp_does_not_block_string_compilation(JS::Realm& realm, ReadonlySpan parameter_strings, Utf16View body_string, Utf16View code_string, JS::CompilationType compilation_type, ReadonlySpan parameter_args, JS::Value body_arg); JS::ThrowCompletionOr ensure_csp_does_not_block_wasm_byte_compilation(JS::Realm&); [[nodiscard]] Directives::Directive::Result is_base_allowed_for_document(JS::Realm&, URL::URL const& base, GC::Ref document); diff --git a/Libraries/LibWeb/DOM/EventTarget.cpp b/Libraries/LibWeb/DOM/EventTarget.cpp index 39fe1f6191..25842c495e 100644 --- a/Libraries/LibWeb/DOM/EventTarget.cpp +++ b/Libraries/LibWeb/DOM/EventTarget.cpp @@ -479,6 +479,7 @@ WebIDL::CallbackType* EventTarget::get_current_value_of_event_handler(FlyString // 3. Let body be the uncompiled script body in eventHandler's value. auto& body = event_handler->value.get(); + auto body_utf16 = Utf16String::from_utf8(body); // FIXME: 4. Let location be the location where the script body originated, as given by eventHandler's value. @@ -493,28 +494,28 @@ WebIDL::CallbackType* EventTarget::get_current_value_of_event_handler(FlyString auto& settings_object = document->relevant_settings_object(); // Build source text and parameter strings for the event handler function. - StringBuilder source_builder; - StringView parameters_string; + StringBuilder source_builder(StringBuilder::Mode::UTF16); + Utf16String parameters_string; // sourceText / ParameterList if (name == HTML::EventNames::error && is(this)) { // -> If name is onerror and eventTarget is a Window object // Let the function have five arguments, named event, source, lineno, colno, and error. - source_builder.appendff("function on{}(event, source, lineno, colno, error) {{\n{}\n}}", name, body); - parameters_string = "event, source, lineno, colno, error"sv; + source_builder.appendff("function on{}(event, source, lineno, colno, error) {{\n{}\n}}", name, body_utf16); + parameters_string = "event, source, lineno, colno, error"_utf16; } else { // -> Otherwise // Let the function have a single argument called event. - source_builder.appendff("function on{}(event) {{\n{}\n}}", name, body); - parameters_string = "event"sv; + source_builder.appendff("function on{}(event) {{\n{}\n}}", name, body_utf16); + parameters_string = "event"_utf16; } - auto source_text = source_builder.to_byte_string(); + auto source_text = source_builder.to_utf16_string(); auto& vm = Bindings::main_thread_vm(); auto rust_compilation = JS::RustIntegration::compile_dynamic_function( - vm, source_text, parameters_string, body, JS::FunctionKind::Normal); + vm, source_text, parameters_string, body_utf16, JS::FunctionKind::Normal); // 7. If body is not parsable as FunctionBody or if parsing detects an early error, then follow these substeps: if (!rust_compilation.has_value() || rust_compilation->is_error()) { diff --git a/Libraries/LibWeb/Geometry/DOMMatrixReadOnly.cpp b/Libraries/LibWeb/Geometry/DOMMatrixReadOnly.cpp index a69f7e240d..6b6d054cda 100644 --- a/Libraries/LibWeb/Geometry/DOMMatrixReadOnly.cpp +++ b/Libraries/LibWeb/Geometry/DOMMatrixReadOnly.cpp @@ -567,37 +567,37 @@ WebIDL::ExceptionOr DOMMatrixReadOnly::to_string() const TRY_OR_THROW_OOM(vm, builder.try_append("matrix("sv)); // 2. Append ! ToString(m11 element) to string. - TRY_OR_THROW_OOM(vm, builder.try_append(JS::Value(m11()).to_string_without_side_effects())); + TRY_OR_THROW_OOM(vm, builder.try_append(JS::number_to_string(m11()))); // 3. Append ", " to string. TRY_OR_THROW_OOM(vm, builder.try_append(", "sv)); // 4. Append ! ToString(m12 element) to string. - TRY_OR_THROW_OOM(vm, builder.try_append(JS::Value(m12()).to_string_without_side_effects())); + TRY_OR_THROW_OOM(vm, builder.try_append(JS::number_to_string(m12()))); // 5. Append ", " to string. TRY_OR_THROW_OOM(vm, builder.try_append(", "sv)); // 6. Append ! ToString(m21 element) to string. - TRY_OR_THROW_OOM(vm, builder.try_append(JS::Value(m21()).to_string_without_side_effects())); + TRY_OR_THROW_OOM(vm, builder.try_append(JS::number_to_string(m21()))); // 7. Append ", " to string. TRY_OR_THROW_OOM(vm, builder.try_append(", "sv)); // 8. Append ! ToString(m22 element) to string. - TRY_OR_THROW_OOM(vm, builder.try_append(JS::Value(m22()).to_string_without_side_effects())); + TRY_OR_THROW_OOM(vm, builder.try_append(JS::number_to_string(m22()))); // 9. Append ", " to string. TRY_OR_THROW_OOM(vm, builder.try_append(", "sv)); // 10. Append ! ToString(m41 element) to string. - TRY_OR_THROW_OOM(vm, builder.try_append(JS::Value(m41()).to_string_without_side_effects())); + TRY_OR_THROW_OOM(vm, builder.try_append(JS::number_to_string(m41()))); // 11. Append ", " to string. TRY_OR_THROW_OOM(vm, builder.try_append(", "sv)); // 12. Append ! ToString(m42 element) to string. - TRY_OR_THROW_OOM(vm, builder.try_append(JS::Value(m42()).to_string_without_side_effects())); + TRY_OR_THROW_OOM(vm, builder.try_append(JS::number_to_string(m42()))); // 13. Append ")" to string. TRY_OR_THROW_OOM(vm, builder.try_append(")"sv)); diff --git a/Libraries/LibWeb/HTML/ErrorInformation.cpp b/Libraries/LibWeb/HTML/ErrorInformation.cpp index a9093d7805..5705464094 100644 --- a/Libraries/LibWeb/HTML/ErrorInformation.cpp +++ b/Libraries/LibWeb/HTML/ErrorInformation.cpp @@ -25,7 +25,7 @@ ErrorInformation extract_error_information(JS::VM& vm, JS::Value exception) if (auto object = exception.as_if()) { if (MUST(object->has_own_property(vm.names.message))) { auto message = object->get_without_side_effects(vm.names.message); - return message.to_string_without_side_effects(); + return message.to_utf16_string_without_side_effects().to_utf8(); } } diff --git a/Libraries/LibWeb/HTML/Scripting/ClassicScript.cpp b/Libraries/LibWeb/HTML/Scripting/ClassicScript.cpp index f0919cd786..d17b539899 100644 --- a/Libraries/LibWeb/HTML/Scripting/ClassicScript.cpp +++ b/Libraries/LibWeb/HTML/Scripting/ClassicScript.cpp @@ -48,8 +48,9 @@ GC::Ref ClassicScript::create(ByteString filename, StringView sou // FIXME: 9. Record classic script creation time given script and sourceURLForWindowScripts . // 10. Let result be ParseScript(source, settings's realm, script). + auto source_text = Utf16String::from_utf8(source); auto parse_timer = Core::ElapsedTimer::start_new(); - auto result = JS::Script::parse(source, settings.realm(), script->filename(), script, source_line_number); + auto result = JS::Script::parse(source_text.utf16_view(), settings.realm(), script->filename(), script, source_line_number); dbgln_if(HTML_SCRIPT_DEBUG, "ClassicScript: Parsed {} in {}ms", script->filename(), parse_timer.elapsed_milliseconds()); // 11. If result is a list of errors, then: diff --git a/Libraries/LibWeb/HTML/Scripting/ExceptionReporter.cpp b/Libraries/LibWeb/HTML/Scripting/ExceptionReporter.cpp index 40b21b39b5..cf96707483 100644 --- a/Libraries/LibWeb/HTML/Scripting/ExceptionReporter.cpp +++ b/Libraries/LibWeb/HTML/Scripting/ExceptionReporter.cpp @@ -42,8 +42,8 @@ void report_exception_to_console(JS::Value value, JS::Realm& realm, ErrorInPromi exception_name = exception->name().to_string(); exception_message = MUST(exception->message().view().to_utf8()); } else { - exception_name = name.to_string_without_side_effects(); - exception_message = message.to_string_without_side_effects(); + exception_name = name.to_utf16_string_without_side_effects().to_utf8(); + exception_message = message.to_utf16_string_without_side_effects().to_utf8(); } dbgln("{}", error_data->stack_string(JS::CompactTraceback::Yes)); console.report_exception(exception_name, exception_message, *error_data, error_in_promise == ErrorInPromise::Yes); @@ -53,8 +53,9 @@ void report_exception_to_console(JS::Value value, JS::Realm& realm, ErrorInPromi dbgln("\033[31;1mUnhandled JavaScript exception{}:\033[0m {}", error_in_promise == ErrorInPromise::Yes ? " (in promise)" : "", value); } - auto message = value.to_string_without_side_effects(); - auto error = JS::Error::create(realm, Utf16String::from_utf8(message)); + auto utf16_message = value.to_utf16_string_without_side_effects(); + auto message = utf16_message.to_utf8(); + auto error = JS::Error::create(realm, utf16_message); console.report_exception("Error"_string, message, *error, error_in_promise == ErrorInPromise::Yes); } diff --git a/Libraries/LibWeb/HTML/WindowOrWorkerGlobalScope.cpp b/Libraries/LibWeb/HTML/WindowOrWorkerGlobalScope.cpp index f94db85a3c..da5af9356d 100644 --- a/Libraries/LibWeb/HTML/WindowOrWorkerGlobalScope.cpp +++ b/Libraries/LibWeb/HTML/WindowOrWorkerGlobalScope.cpp @@ -623,7 +623,8 @@ i32 WindowOrWorkerGlobalScopeMixin::run_timer_initialization_steps(TimerHandler // 3. Perform EnsureCSPDoesNotBlockStringCompilation(realm, « », handler, handler, timer, « », handler). // If this throws an exception, catch it, report it for global, and abort these steps. auto handler_primitive_string = JS::PrimitiveString::create(vm, source); - if (auto result = ContentSecurityPolicy::ensure_csp_does_not_block_string_compilation(realm, {}, source, source, JS::CompilationType::Timer, {}, handler_primitive_string); result.is_throw_completion()) { + auto source_utf16 = Utf16String::from_utf8(source); + if (auto result = ContentSecurityPolicy::ensure_csp_does_not_block_string_compilation(realm, {}, source_utf16, source_utf16, JS::CompilationType::Timer, {}, handler_primitive_string); result.is_throw_completion()) { report_exception(result, realm); return false; } diff --git a/Libraries/LibWeb/WebAssembly/WebAssembly.cpp b/Libraries/LibWeb/WebAssembly/WebAssembly.cpp index 4707c158d2..7e92e09ccd 100644 --- a/Libraries/LibWeb/WebAssembly/WebAssembly.cpp +++ b/Libraries/LibWeb/WebAssembly/WebAssembly.cpp @@ -255,7 +255,7 @@ Wasm::HostFunction create_host_function(JS::VM& vm, JS::FunctionObject& function auto method = TRY_OR_RETURN_TRAP(result.get_method(vm, vm.names.iterator)); if (!method) - return Wasm::Trap::from_external_object(vm.throw_completion(JS::ErrorType::NotIterable, result.to_string_without_side_effects())); + return Wasm::Trap::from_external_object(vm.throw_completion(JS::ErrorType::NotIterable, result.to_utf16_string_without_side_effects())); auto values = TRY_OR_RETURN_TRAP(JS::iterator_to_list(vm, TRY_OR_RETURN_TRAP(JS::get_iterator_from_method(vm, result, *method)))); @@ -300,7 +300,7 @@ JS::ThrowCompletionOr> instantiate_module(JS // 3.2. If o is not an Object, throw a TypeError exception. if (!value.is_object()) - return vm.throw_completion(JS::ErrorType::IsNotAEvaluatedFrom, value.to_string_without_side_effects(), "Object"_string, MUST(String::formatted("[wasm import object][\"{}\"]", import_name.module))); + return vm.throw_completion(JS::ErrorType::IsNotAEvaluatedFrom, value.to_utf16_string_without_side_effects(), "Object"_string, MUST(String::formatted("[wasm import object][\"{}\"]", import_name.module))); auto const& object = value.as_object(); // 3.3. Let v be ? Get(o, componentName). diff --git a/Libraries/LibWeb/WebDriver/ExecuteScript.cpp b/Libraries/LibWeb/WebDriver/ExecuteScript.cpp index 391214d375..76fb882549 100644 --- a/Libraries/LibWeb/WebDriver/ExecuteScript.cpp +++ b/Libraries/LibWeb/WebDriver/ExecuteScript.cpp @@ -50,14 +50,15 @@ static JS::ThrowCompletionOr execute_a_function_body(HTML::BrowsingCo // FIXME: This does not handle scripts which contain `await` statements. It is not as as simple as declaring this // function async, unfortunately. See: https://github.com/w3c/webdriver/issues/1436 - auto source_text = ByteString::formatted( + auto body_utf16 = Utf16String::from_utf8(body); + auto source_text = Utf16String::formatted( R"~~~(function() {{ {} }})~~~", - body); + body_utf16); auto rust_compilation = JS::RustIntegration::compile_dynamic_function( - realm.vm(), source_text, ""sv, body, JS::FunctionKind::Normal); + realm.vm(), source_text, Utf16String {}, body_utf16, JS::FunctionKind::Normal); // 4. If body is not parsable as a FunctionBody or if parsing detects an early error, return Completion { [[Type]]: normal, [[Value]]: null, [[Target]]: empty }. if (!rust_compilation.has_value() || rust_compilation->is_error()) diff --git a/Libraries/LibWeb/WebIDL/ExceptionOr.h b/Libraries/LibWeb/WebIDL/ExceptionOr.h index 1fb9c3a4cc..f966b3fd79 100644 --- a/Libraries/LibWeb/WebIDL/ExceptionOr.h +++ b/Libraries/LibWeb/WebIDL/ExceptionOr.h @@ -181,11 +181,11 @@ struct Formatter : Formatter { auto has_message_or_error = object->has_own_property(*message_property_key); if (!has_message_or_error.is_error() && has_message_or_error.value()) { auto message_object = object->get_without_side_effects(*message_property_key); - return Formatter::format(builder, message_object.to_string_without_side_effects()); + return Formatter {}.format(builder, message_object.to_utf16_string_without_side_effects()); } } - return Formatter::format(builder, value.to_string_without_side_effects()); + return Formatter {}.format(builder, value.to_utf16_string_without_side_effects()); }); } }; diff --git a/Libraries/LibWeb/WebIDL/Tracing.cpp b/Libraries/LibWeb/WebIDL/Tracing.cpp index f88f18fa4e..ce769eaf62 100644 --- a/Libraries/LibWeb/WebIDL/Tracing.cpp +++ b/Libraries/LibWeb/WebIDL/Tracing.cpp @@ -26,13 +26,15 @@ void log_trace_impl(JS::VM& vm, char const* function) auto argument = vm.argument(i); if (argument.is_string()) builder.append_code_point('"'); - auto string = argument.to_string_without_side_effects(); - for (auto code_point : string.code_points()) { - if (code_point < 0x20) { - builder.appendff("\\u{:04x}", code_point); + auto string = argument.to_utf16_string_without_side_effects(); + auto view = string.utf16_view(); + for (size_t code_unit_index = 0; code_unit_index < view.length_in_code_units(); ++code_unit_index) { + auto code_unit = view.code_unit_at(code_unit_index); + if (code_unit < 0x20) { + builder.appendff("\\u{:04x}", code_unit); continue; } - builder.append_code_point(code_point); + builder.append_code_unit(code_unit); } if (argument.is_string()) builder.append_code_point('"'); diff --git a/Meta/Fuzzers/FuzzJs.cpp b/Meta/Fuzzers/FuzzJs.cpp index 6740260de8..3596d19da5 100644 --- a/Meta/Fuzzers/FuzzJs.cpp +++ b/Meta/Fuzzers/FuzzJs.cpp @@ -6,6 +6,7 @@ */ #include +#include #include #include #include @@ -19,10 +20,11 @@ extern "C" int LLVMFuzzerTestOneInput(uint8_t const* data, size_t size) // FIXME: https://github.com/SerenityOS/serenity/issues/17899 if (!Utf8View(js).validate()) return 0; + auto source_text = Utf16String::from_utf8_without_validation(js); auto vm = JS::VM::create(); auto root_execution_context = JS::create_simple_execution_context(*vm); auto& realm = *root_execution_context->realm; - auto parse_result = JS::Script::parse(js, realm); + auto parse_result = JS::Script::parse(source_text.utf16_view(), realm); if (!parse_result.is_error()) (void)vm->run(parse_result.value()); diff --git a/Meta/Fuzzers/FuzzilliJs.cpp b/Meta/Fuzzers/FuzzilliJs.cpp index f5e92785b5..54bf62d4a8 100644 --- a/Meta/Fuzzers/FuzzilliJs.cpp +++ b/Meta/Fuzzers/FuzzilliJs.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -220,7 +221,8 @@ int main(int, char**) if (!Utf8View(js).validate()) { result = 1; } else { - auto parse_result = JS::Script::parse(js, realm); + auto source_text = Utf16String::from_utf8_without_validation(js); + auto parse_result = JS::Script::parse(source_text.utf16_view(), realm); if (parse_result.is_error()) { result = 1; } else { diff --git a/Services/WebContent/DevToolsConsoleClient.cpp b/Services/WebContent/DevToolsConsoleClient.cpp index 587c808a1f..912fc7566e 100644 --- a/Services/WebContent/DevToolsConsoleClient.cpp +++ b/Services/WebContent/DevToolsConsoleClient.cpp @@ -161,7 +161,7 @@ JS::ThrowCompletionOr DevToolsConsoleClient::printer(JS::Console::Log send_console_output({ .timestamp = UnixDateTime::now(), .output = WebView::ConsoleTrace { - .label = trace.label, + .label = trace.label.to_utf8(), .stack = move(stack_frames), }, }); diff --git a/Services/WebWorker/WorkerHost.cpp b/Services/WebWorker/WorkerHost.cpp index 859e520687..9c41b58d5e 100644 --- a/Services/WebWorker/WorkerHost.cpp +++ b/Services/WebWorker/WorkerHost.cpp @@ -220,7 +220,8 @@ void WorkerHost::run(GC::Ref page, Web::HTML::TransferDataEncoder mes inside_settings->discard_environment(); // 3. Abort these steps. - dbgln("DedicatedWorkerHost: Unable to fetch script {} because {}", url, script ? script->error_to_rethrow().to_string_without_side_effects() : "script was null"_string); + auto reason = script ? script->error_to_rethrow().to_utf16_string_without_side_effects().to_utf8() : "script was null"_string; + dbgln("DedicatedWorkerHost: Unable to fetch script {} because {}", url, reason); return; } diff --git a/Tests/AK/TestBase64.cpp b/Tests/AK/TestBase64.cpp index 383938d5ea..90cb371f5f 100644 --- a/Tests/AK/TestBase64.cpp +++ b/Tests/AK/TestBase64.cpp @@ -66,6 +66,29 @@ TEST_CASE(test_decode_into) decode_equal("Zm9vYmFy"sv, "foobar"sv, 7); } +TEST_CASE(test_decode_into_utf16) +{ + ByteBuffer buffer; + + auto decode_equal = [&](Utf16View input, StringView expected, Optional buffer_size = {}) { + buffer.resize(buffer_size.value_or_lazy_evaluated([&]() { + return AK::size_required_to_decode_base64(input); + })); + + auto result = AK::decode_base64_into(input, buffer); + VERIFY(!result.is_error()); + + EXPECT_EQ(StringView { buffer }, expected); + }; + + decode_equal(u"Zm9vYmFy"sv, "foobar"sv); + decode_equal(u"Zm9vYmFy"sv, "foo"sv, 3); + decode_equal(u"aGVsbG8/d29ybGQ="sv, "hello?world"sv); + + char16_t input[] = u"Zm9vYmFy"; + decode_equal({ input, 8 }, "foobar"sv); +} + TEST_CASE(test_decode_invalid) { EXPECT(decode_base64(("asdf\xffqwe"sv)).is_error()); diff --git a/Tests/LibCrypto/TestBigInteger.cpp b/Tests/LibCrypto/TestBigInteger.cpp index 8ff95c56cf..f3d72d6de0 100644 --- a/Tests/LibCrypto/TestBigInteger.cpp +++ b/Tests/LibCrypto/TestBigInteger.cpp @@ -200,6 +200,17 @@ TEST_CASE(test_unsigned_bigint_base10_from_string) EXPECT_EQ(Crypto::UnsignedBigInteger::from_base(10, invalid_base10_number_string).is_error(), true); } +TEST_CASE(test_bigint_from_utf16_string) +{ + auto unsigned_result = TRY_OR_FAIL(Crypto::UnsignedBigInteger::from_base(16, u"ffffffffffff"sv)); + EXPECT_EQ(MUST(unsigned_result.to_base(16)), "ffffffffffff"); + + auto signed_result = TRY_OR_FAIL(Crypto::SignedBigInteger::from_base(10, u"-123456789"sv)); + EXPECT_EQ(MUST(signed_result.to_base(10)), "-123456789"); + + EXPECT(Crypto::UnsignedBigInteger::from_base(10, u"12\u00a0"sv).is_error()); +} + TEST_CASE(test_unsigned_bigint_base10_to_string) { auto bigint = Crypto::UnsignedBigInteger { diff --git a/Tests/LibJS/Runtime/builtins/Intl/Intl.getCanonicalLocales.js b/Tests/LibJS/Runtime/builtins/Intl/Intl.getCanonicalLocales.js index e6c47f7ced..8fd026c79d 100644 --- a/Tests/LibJS/Runtime/builtins/Intl/Intl.getCanonicalLocales.js +++ b/Tests/LibJS/Runtime/builtins/Intl/Intl.getCanonicalLocales.js @@ -66,6 +66,14 @@ describe("errors", () => { Intl.getCanonicalLocales("en-t-en-POSIX-POSIX"); }).toThrowWithMessage(RangeError, "en-t-en-POSIX-POSIX is not a structurally valid language tag"); }); + + test("non-ASCII UTF-16 extension subtags", () => { + ["en-\uff55-ab-foo", "en-u-\uff41b-foo", "en-t-\uff4b0-foo"].forEach(locale => { + expect(() => { + Intl.getCanonicalLocales(locale); + }).toThrowWithMessage(RangeError, `${locale} is not a structurally valid language tag`); + }); + }); }); describe("normal behavior", () => { diff --git a/Tests/LibJS/test-js.cpp b/Tests/LibJS/test-js.cpp index 8ca9427c81..0e2538d0c0 100644 --- a/Tests/LibJS/test-js.cpp +++ b/Tests/LibJS/test-js.cpp @@ -22,8 +22,8 @@ TESTJS_PROGRAM_FLAG(test262_parser_tests, "Run test262 parser tests", "test262-p TESTJS_GLOBAL_FUNCTION(can_parse_source, canParseSource) { auto& realm = *vm.current_realm(); - auto source = TRY(vm.argument(0).to_utf16_string(vm)).to_utf8_but_should_be_ported_to_utf16(); - auto script = JS::Script::parse(source, realm); + auto source = TRY(vm.argument(0).to_utf16_string(vm)); + auto script = JS::Script::parse(source.utf16_view(), realm); return JS::Value(!script.is_error()); } @@ -38,9 +38,9 @@ TESTJS_GLOBAL_FUNCTION(evaluate_source, evaluateSource) { auto& realm = *vm.current_realm(); - auto source = TRY(vm.argument(0).to_utf16_string(vm)).to_utf8_but_should_be_ported_to_utf16(); + auto source = TRY(vm.argument(0).to_utf16_string(vm)); - auto script = JS::Script::parse(source, realm); + auto script = JS::Script::parse(source.utf16_view(), realm); if (script.is_error()) return vm.throw_completion(script.error().first().to_string()); diff --git a/Tests/LibUnicode/TestLocale.cpp b/Tests/LibUnicode/TestLocale.cpp index 1f45f37ad1..6796ba6ea9 100644 --- a/Tests/LibUnicode/TestLocale.cpp +++ b/Tests/LibUnicode/TestLocale.cpp @@ -188,6 +188,18 @@ TEST_CASE(parse_unicode_locale_id_with_unicode_locale_extension) pass("en-u-fff-gggg-xx-yyyy"sv, { { "fff"sv, "gggg"sv }, { { "xx"sv, "yyyy"sv } } }); } +TEST_CASE(parse_unicode_locale_id_rejects_non_ascii_utf16_extension_parts) +{ + auto fail = [](Utf16View locale) { + auto locale_id = Unicode::parse_unicode_locale_id(locale); + EXPECT(!locale_id.has_value()); + }; + + fail(u"en-\uFF55-ab-foo"sv); + fail(u"en-u-\uFF41b-foo"sv); + fail(u"en-t-\uFF4B0-foo"sv); +} + TEST_CASE(parse_unicode_locale_id_with_transformed_extension) { struct TransformedExtension { diff --git a/Utilities/js.cpp b/Utilities/js.cpp index 5b4a65eaf9..ff16db731c 100644 --- a/Utilities/js.cpp +++ b/Utilities/js.cpp @@ -201,10 +201,9 @@ static ErrorOr parse_and_run(JS::Realm& realm, StringView source, StringVi JS::ThrowCompletionOr result { JS::js_undefined() }; if (!s_as_module) { - auto script_or_error = JS::Script::parse(source, realm, source_name); + auto utf16_source = Utf16String::from_utf8(source); + auto script_or_error = JS::Script::parse(utf16_source.utf16_view(), realm, source_name); if (script_or_error.is_error()) { - auto utf16_source = Utf16String::from_utf8(source); - auto error = script_or_error.error()[0]; auto hint = error.source_location_hint(utf16_source); if (!hint.is_empty()) @@ -296,7 +295,12 @@ static JS::ThrowCompletionOr load_json_impl(JS::VM& vm) if (file_contents_or_error.is_error()) return vm.throw_completion(TRY_OR_THROW_OOM(vm, String::formatted("Failed to read '{}': {}", filename, file_contents_or_error.error()))); - return JS::JSONObject::parse_json(vm, file_contents_or_error.value()); + auto file_contents = file_contents_or_error.release_value(); + auto json_text = Utf16String::try_from_utf8(StringView { file_contents.bytes() }); + if (json_text.is_error()) + return vm.throw_completion(JS::ErrorType::JsonMalformed); + + return JS::JSONObject::parse_json(vm, json_text.release_value()); } void ReplObject::initialize(JS::Realm& realm) diff --git a/Utilities/test262-runner.cpp b/Utilities/test262-runner.cpp index daaf6d0ae7..05b9e2d3a1 100644 --- a/Utilities/test262-runner.cpp +++ b/Utilities/test262-runner.cpp @@ -44,17 +44,23 @@ struct TestError { ByteString harness_file; }; +static String value_to_utf8_string_without_side_effects(JS::Value value) +{ + return value.to_utf16_string_without_side_effects().to_utf8(); +} + using ScriptOrModuleProgram = Variant, GC::Ref>; template static ErrorOr parse_program(JS::Realm& realm, StringView source, StringView filepath) { - auto script_or_error = ScriptType::parse(source, realm, filepath); + auto source_utf16 = Utf16String::from_utf8(source); + auto script_or_error = ScriptType::parse(source_utf16, realm, filepath); if (script_or_error.is_error()) { return TestError { NegativePhase::ParseOrEarly, "SyntaxError"_string, - script_or_error.error()[0].to_string(), + script_or_error.error()[0].to_utf16_string().to_utf8(), "" }; } @@ -103,22 +109,22 @@ static ErrorOr run_program(InterpreterT& interpreter, ScriptOrM if (auto object = error_value.template as_if()) { auto name = object->get_without_side_effects("name"_utf16_fly_string); if (!name.is_undefined() && !name.is_accessor()) { - error.type = name.to_string_without_side_effects(); + error.type = value_to_utf8_string_without_side_effects(name); } else { auto constructor_value = object->get_without_side_effects("constructor"_utf16_fly_string); if (auto constructor = constructor_value.template as_if()) { name = constructor->get_without_side_effects("name"_utf16_fly_string); if (!name.is_undefined()) - error.type = name.to_string_without_side_effects(); + error.type = value_to_utf8_string_without_side_effects(name); } } auto message = object->get_without_side_effects("message"_utf16_fly_string); if (!message.is_undefined() && !message.is_accessor()) - error.details = message.to_string_without_side_effects(); + error.details = value_to_utf8_string_without_side_effects(message); } if (error.type.is_empty()) - error.type = error_value.to_string_without_side_effects(); + error.type = value_to_utf8_string_without_side_effects(error_value); return error; } return {}; diff --git a/Utilities/wasm.cpp b/Utilities/wasm.cpp index f961e25aed..2927181eb1 100644 --- a/Utilities/wasm.cpp +++ b/Utilities/wasm.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -414,17 +415,20 @@ ErrorOr ladybird_main(Main::Arguments arguments) } auto source_text = lexer.consume_all().trim_whitespace(); - StringBuilder builder; - builder.append("("sv); + Utf16StringBuilder builder; + builder.append_ascii("("sv); auto first = true; for (auto& arg : formal_params) { if (!first) - builder.append(", "sv); + builder.append_ascii(", "sv); first = false; - builder.append(arg.name); + auto argument_name = Utf16String::from_utf8(arg.name); + builder.append(argument_name.utf16_view()); } - builder.appendff(") => {}", source_text); - auto js_function = builder.to_byte_string(); + builder.append_ascii(") => "sv); + auto source_text_utf16 = Utf16String::from_utf8(source_text); + builder.append(source_text_utf16.utf16_view()); + auto js_function = builder.to_string(); auto name = ByteString::formatted("{}.{}", module, fn_name); auto script = JS::Script::parse(js_function, realm, name); if (script.is_error()) {