dmaax/ladybird
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

LibSyntax+LibJS+LibWebView: Port the syntax highlighter to UTF-8

Browse source 
LibSyntax is the only remaining user of UTF-32 in the code base. Let's
use UTF-8 here.

Bonus: The tests added here for non-ASCII sources actually used to
crash the old UTF-32 implementation.
This commit is contained in:
Timothy Flynn 2026-06-06 09:38:27 -04:00 committed by Shannon Booth
parent 87cdeda93c
commit 4d51499809
5 changed files with 79 additions and 24 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
Libraries/LibJS/SyntaxHighlighter.cpp
View file

@ -6,6 +6,7 @@
*/
#include <AK/Debug.h>
#include <AK/UnicodeUtils.h>
#include <AK/Utf16String.h>
#include <LibGfx/Palette.h>
#include <LibJS/RustFFI.h>
@ -49,11 +50,17 @@ struct RehighlightState {
static void advance_position(Syntax::TextPosition& position, u16 const* source, u32 start, u32 len)
{
for (u32 i = 0; i < len; ++i) {
if (source[start + i] == '\n') {
if (auto code_unit = source[start + i]; code_unit == '\n') {
position.set_line(position.line() + 1);
position.set_column(0);
} else {
position.set_column(position.column() + 1);
if (AK::UnicodeUtils::is_utf16_high_surrogate(code_unit)
&& i + 1 < len
&& AK::UnicodeUtils::is_utf16_low_surrogate(source[start + i + 1])) {
++i;
}
}
}
}

16
Libraries/LibSyntax/Document.cpp
View file

@ -23,7 +23,8 @@ TextDocumentLine::TextDocumentLine(Document& document, StringView text)
void TextDocumentLine::clear(Document& document)
{
m_text.clear();
m_text = {};
m_length = 0;
document.update_views({});
}
@ -33,13 +34,16 @@ bool TextDocumentLine::set_text(Document& document, StringView text)
clear(document);
return true;
}
m_text.clear();
m_text = {};
m_length = 0;
Utf8View utf8_view(text);
if (!utf8_view.validate()) {
if (!utf8_view.validate())
return false;
}
for (auto code_point : utf8_view)
m_text.append(code_point);
m_text = String::from_utf8_without_validation(text.bytes());
m_length = utf8_view.length();
document.update_views({});
return true;
}

12
Libraries/LibSyntax/Document.h
View file

@ -7,7 +7,7 @@
#pragma once
#include <AK/RefCounted.h>
#include <AK/Utf32View.h>
#include <AK/String.h>
#include <AK/Vector.h>
#include <LibGfx/TextAttributes.h>
#include <LibSyntax/Forward.h>
@ -27,16 +27,14 @@ public:
explicit TextDocumentLine(Document&);
explicit TextDocumentLine(Document&, StringView);
Utf32View view() const LIFETIME_BOUND { return { code_points(), length() }; }
u32 const* code_points() const { return m_text.data(); }
bool is_empty() const { return length() == 0; }
size_t length() const { return m_text.size(); }
Utf8View view() const LIFETIME_BOUND { return m_text.code_points(); }
size_t length() const { return m_length; }
bool set_text(Document&, StringView);
void clear(Document&);
private:
// NOTE: This vector is null terminated.
Vector<u32> m_text;
String m_text;
size_t m_length { 0 };
};
class Document : public RefCounted<Document> {

16
Libraries/LibWebView/SourceHighlighter.cpp
View file

@ -232,7 +232,7 @@ String SourceHighlighterClient::to_html_string(Optional<URL::URL> const& url, UR
{
StringBuilder builder;
auto append_escaped = [&](Utf32View text) {
auto append_escaped = [&](Utf8View const& text) {
for (auto code_point : text) {
if (code_point == '&') {
builder.append("&amp;"sv);
@ -259,6 +259,7 @@ String SourceHighlighterClient::to_html_string(Optional<URL::URL> const& url, UR
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<meta name="color-scheme" content="dark light">)~~~"sv);
if (url.has_value())
@ -272,16 +273,15 @@ String SourceHighlighterClient::to_html_string(Optional<URL::URL> const& url, UR
<body>
<pre class=\"html\">)~~~"sv);
static constexpr auto href = to_array<u32>({ 'h', 'r', 'e', 'f' });
static constexpr auto src = to_array<u32>({ 's', 'r', 'c' });
static constexpr auto href = "href"sv;
static constexpr auto src = "src"sv;
bool linkify_attribute = false;
auto resolve_url_for_attribute = [&](Utf32View const& attribute_value) -> Optional<URL::URL> {
auto resolve_url_for_attribute = [&](Utf8View const& attribute_value) -> Optional<URL::URL> {
if (!linkify_attribute)
return {};
auto attribute_url = MUST(String::formatted("{}", attribute_value));
auto attribute_url_without_quotes = attribute_url.bytes_as_string_view().trim("\""sv);
auto attribute_url_without_quotes = attribute_value.as_string().trim("\""sv);
return Web::DOMURL::parse(attribute_url_without_quotes, base_url);
};
@ -299,13 +299,13 @@ String SourceHighlighterClient::to_html_string(Optional<URL::URL> const& url, UR
if (length == 0)
return;
auto text = line_view.substring_view(start, length);
auto text = line_view.unicode_substring_view(start, length);
if (span.has_value()) {
bool append_anchor_close = false;
if (span->data == to_underlying(Web::HTML::AugmentedTokenKind::AttributeName)) {
linkify_attribute = text == Utf32View { href } || text == Utf32View { src };
linkify_attribute = text.as_string() == href || text.as_string() == src;
} else if (span->data == to_underlying(Web::HTML::AugmentedTokenKind::AttributeValue)) {
if (auto href = resolve_url_for_attribute(text); href.has_value()) {
builder.appendff("<a href=\"{}\">", *href);

50
Tests/LibWeb/TestSourceHighlighter.cpp
View file

@ -5,14 +5,60 @@
*/
#include <LibTest/TestCase.h>
#include <LibURL/Parser.h>
#include <LibURL/URL.h>
#include <LibWebView/SourceHighlighter.h>
static String highlight_source(String const& source, Syntax::Language language)
{
auto url = URL::Parser::basic_parse("https://example.com/source.html"sv).release_value();
auto base_url = URL::Parser::basic_parse("https://example.com/base/"sv).release_value();
return WebView::highlight_source(url, base_url, source, language);
}
TEST_CASE(highlight_script_with_braces)
{
// Regression test for https://github.com/LadybirdBrowser/ladybird/issues/8529
auto source = "<script>\nfunction foo() {\n return 1;\n}\n</script>"_string;
URL::URL base_url {};
auto result = WebView::highlight_source({}, base_url, source, Syntax::Language::HTML);
auto result = highlight_source(source, Syntax::Language::HTML);
EXPECT(!result.is_empty());
}
TEST_CASE(highlight_html_with_non_ascii_before_linkified_attribute)
{
auto source = "<p title=\"\xF0\x9F\x98\x80\"><a href=\"next.html\">ok</a></p>"_string;
auto result = highlight_source(source, Syntax::Language::HTML);
EXPECT(result.contains("\xF0\x9F\x98\x80"sv));
EXPECT(result.contains("<span class=\"attribute-name\">href</span>"sv));
EXPECT(result.contains("<a href=\"https://example.com/base/next.html\"><span class=\"attribute-value\">\"next.html\"</span></a>"sv));
}
TEST_CASE(highlight_css_with_non_ascii_before_token)
{
auto source = ".caf\xC3\xA9 { color: red; }"_string;
auto result = highlight_source(source, Syntax::Language::CSS);
EXPECT(result.contains("<span class=\"delimiter\">.</span><span class=\"identifier\">caf\xC3\xA9</span>"sv));
EXPECT(result.contains("<span class=\"identifier\">color</span>"sv));
}
TEST_CASE(highlight_javascript_with_non_bmp_before_token)
{
auto source = "const smile = \"\xF0\x9F\x98\x80\"; const answer = 42;"_string;
auto result = highlight_source(source, Syntax::Language::JavaScript);
EXPECT(result.contains("\xF0\x9F\x98\x80"sv));
EXPECT(result.contains("<span class=\"identifier\">answer</span>"sv));
EXPECT(result.contains("<span class=\"number\">42</span>"sv));
}
TEST_CASE(declares_utf8_and_preserves_non_ascii_text)
{
auto source = "<style>:root { --label-text: \"Caf\xC3\xA9 cr\xC3\xA8me - \xE6\x9D\xB1\xE4\xBA\xAC - \xF0\x9F\x98\x80\"; }</style>"_string;
auto result = highlight_source(source, Syntax::Language::HTML);
EXPECT(result.contains("<meta charset=\"utf-8\">"sv));
EXPECT(result.contains("Caf\xC3\xA9 cr\xC3\xA8me - \xE6\x9D\xB1\xE4\xBA\xAC - \xF0\x9F\x98\x80"sv));
EXPECT(!result.contains("Caf\xC3\x83\xC2\xA9"sv));
}