diff --git a/Libraries/LibJS/SyntaxHighlighter.cpp b/Libraries/LibJS/SyntaxHighlighter.cpp index 5dbb3c513a..7581f4ef8c 100644 --- a/Libraries/LibJS/SyntaxHighlighter.cpp +++ b/Libraries/LibJS/SyntaxHighlighter.cpp @@ -6,6 +6,7 @@ */ #include +#include #include #include #include @@ -49,11 +50,17 @@ struct RehighlightState { static void advance_position(Syntax::TextPosition& position, u16 const* source, u32 start, u32 len) { for (u32 i = 0; i < len; ++i) { - if (source[start + i] == '\n') { + if (auto code_unit = source[start + i]; code_unit == '\n') { position.set_line(position.line() + 1); position.set_column(0); } else { position.set_column(position.column() + 1); + + if (AK::UnicodeUtils::is_utf16_high_surrogate(code_unit) + && i + 1 < len + && AK::UnicodeUtils::is_utf16_low_surrogate(source[start + i + 1])) { + ++i; + } } } } diff --git a/Libraries/LibSyntax/Document.cpp b/Libraries/LibSyntax/Document.cpp index cce86c1c26..84f5b76ee6 100644 --- a/Libraries/LibSyntax/Document.cpp +++ b/Libraries/LibSyntax/Document.cpp @@ -23,7 +23,8 @@ TextDocumentLine::TextDocumentLine(Document& document, StringView text) void TextDocumentLine::clear(Document& document) { - m_text.clear(); + m_text = {}; + m_length = 0; document.update_views({}); } @@ -33,13 +34,16 @@ bool TextDocumentLine::set_text(Document& document, StringView text) clear(document); return true; } - m_text.clear(); + + m_text = {}; + m_length = 0; + Utf8View utf8_view(text); - if (!utf8_view.validate()) { + if (!utf8_view.validate()) return false; - } - for (auto code_point : utf8_view) - m_text.append(code_point); + + m_text = String::from_utf8_without_validation(text.bytes()); + m_length = utf8_view.length(); document.update_views({}); return true; } diff --git a/Libraries/LibSyntax/Document.h b/Libraries/LibSyntax/Document.h index 123435ad62..5ef4d23925 100644 --- a/Libraries/LibSyntax/Document.h +++ b/Libraries/LibSyntax/Document.h @@ -7,7 +7,7 @@ #pragma once #include -#include +#include #include #include #include @@ -27,16 +27,14 @@ public: explicit TextDocumentLine(Document&); explicit TextDocumentLine(Document&, StringView); - Utf32View view() const LIFETIME_BOUND { return { code_points(), length() }; } - u32 const* code_points() const { return m_text.data(); } - bool is_empty() const { return length() == 0; } - size_t length() const { return m_text.size(); } + Utf8View view() const LIFETIME_BOUND { return m_text.code_points(); } + size_t length() const { return m_length; } bool set_text(Document&, StringView); void clear(Document&); private: - // NOTE: This vector is null terminated. - Vector m_text; + String m_text; + size_t m_length { 0 }; }; class Document : public RefCounted { diff --git a/Libraries/LibWebView/SourceHighlighter.cpp b/Libraries/LibWebView/SourceHighlighter.cpp index eae0b3aa78..e221aae2cc 100644 --- a/Libraries/LibWebView/SourceHighlighter.cpp +++ b/Libraries/LibWebView/SourceHighlighter.cpp @@ -232,7 +232,7 @@ String SourceHighlighterClient::to_html_string(Optional const& url, UR { StringBuilder builder; - auto append_escaped = [&](Utf32View text) { + auto append_escaped = [&](Utf8View const& text) { for (auto code_point : text) { if (code_point == '&') { builder.append("&"sv); @@ -259,6 +259,7 @@ String SourceHighlighterClient::to_html_string(Optional const& url, UR + )~~~"sv); if (url.has_value()) @@ -272,16 +273,15 @@ String SourceHighlighterClient::to_html_string(Optional const& url, UR 
)~~~"sv);
 
-    static constexpr auto href = to_array({ 'h', 'r', 'e', 'f' });
-    static constexpr auto src = to_array({ 's', 'r', 'c' });
+    static constexpr auto href = "href"sv;
+    static constexpr auto src = "src"sv;
     bool linkify_attribute = false;
 
-    auto resolve_url_for_attribute = [&](Utf32View const& attribute_value) -> Optional {
+    auto resolve_url_for_attribute = [&](Utf8View const& attribute_value) -> Optional {
         if (!linkify_attribute)
             return {};
 
-        auto attribute_url = MUST(String::formatted("{}", attribute_value));
-        auto attribute_url_without_quotes = attribute_url.bytes_as_string_view().trim("\""sv);
+        auto attribute_url_without_quotes = attribute_value.as_string().trim("\""sv);
 
         return Web::DOMURL::parse(attribute_url_without_quotes, base_url);
     };
@@ -299,13 +299,13 @@ String SourceHighlighterClient::to_html_string(Optional const& url, UR
             if (length == 0)
                 return;
 
-            auto text = line_view.substring_view(start, length);
+            auto text = line_view.unicode_substring_view(start, length);
 
             if (span.has_value()) {
                 bool append_anchor_close = false;
 
                 if (span->data == to_underlying(Web::HTML::AugmentedTokenKind::AttributeName)) {
-                    linkify_attribute = text == Utf32View { href } || text == Utf32View { src };
+                    linkify_attribute = text.as_string() == href || text.as_string() == src;
                 } else if (span->data == to_underlying(Web::HTML::AugmentedTokenKind::AttributeValue)) {
                     if (auto href = resolve_url_for_attribute(text); href.has_value()) {
                         builder.appendff("", *href);
diff --git a/Tests/LibWeb/TestSourceHighlighter.cpp b/Tests/LibWeb/TestSourceHighlighter.cpp
index 3704b59ce3..ed0daf5004 100644
--- a/Tests/LibWeb/TestSourceHighlighter.cpp
+++ b/Tests/LibWeb/TestSourceHighlighter.cpp
@@ -5,14 +5,60 @@
  */
 
 #include 
+#include 
 #include 
 #include 
 
+static String highlight_source(String const& source, Syntax::Language language)
+{
+    auto url = URL::Parser::basic_parse("https://example.com/source.html"sv).release_value();
+    auto base_url = URL::Parser::basic_parse("https://example.com/base/"sv).release_value();
+    return WebView::highlight_source(url, base_url, source, language);
+}
+
 TEST_CASE(highlight_script_with_braces)
 {
     // Regression test for https://github.com/LadybirdBrowser/ladybird/issues/8529
     auto source = ""_string;
-    URL::URL base_url {};
-    auto result = WebView::highlight_source({}, base_url, source, Syntax::Language::HTML);
+    auto result = highlight_source(source, Syntax::Language::HTML);
     EXPECT(!result.is_empty());
 }
+
+TEST_CASE(highlight_html_with_non_ascii_before_linkified_attribute)
+{
+    auto source = "
ok
"_string;
+    auto result = highlight_source(source, Syntax::Language::HTML);
+
+    EXPECT(result.contains("\xF0\x9F\x98\x80"sv));
+    EXPECT(result.contains("href"sv));
+    EXPECT(result.contains("\"next.html\""sv));
+}
+
+TEST_CASE(highlight_css_with_non_ascii_before_token)
+{
+    auto source = ".caf\xC3\xA9 { color: red; }"_string;
+    auto result = highlight_source(source, Syntax::Language::CSS);
+
+    EXPECT(result.contains(".caf\xC3\xA9"sv));
+    EXPECT(result.contains("color"sv));
+}
+
+TEST_CASE(highlight_javascript_with_non_bmp_before_token)
+{
+    auto source = "const smile = \"\xF0\x9F\x98\x80\"; const answer = 42;"_string;
+    auto result = highlight_source(source, Syntax::Language::JavaScript);
+
+    EXPECT(result.contains("\xF0\x9F\x98\x80"sv));
+    EXPECT(result.contains("answer"sv));
+    EXPECT(result.contains("42"sv));
+}
+
+TEST_CASE(declares_utf8_and_preserves_non_ascii_text)
+{
+    auto source = ""_string;
+    auto result = highlight_source(source, Syntax::Language::HTML);
+
+    EXPECT(result.contains(""sv));
+    EXPECT(result.contains("Caf\xC3\xA9 cr\xC3\xA8me - \xE6\x9D\xB1\xE4\xBA\xAC - \xF0\x9F\x98\x80"sv));
+    EXPECT(!result.contains("Caf\xC3\x83\xC2\xA9"sv));
+}