LibWeb: Reduce CSS parser token memory usage

Store CSS token payloads in a variant so each token only carries the
state needed by its type. Keep delimiter, number, hash, string, and
dimension data separate instead of storing every possible payload on
every token.

Use a smaller component-value token for function and block boundary
metadata. These component values only need token type, original source
text, and source positions, so avoid embedding full token payload
storage inside every Function and SimpleBlock.

Shrink CSS source positions to explicit 32-bit counters. Guard the C++
and Rust tokenizer paths against overflow. Add size assertions for the
hot Token and ComponentValue types so future growth is intentional.

Libraries/LibWeb/CSS/Parser/ComponentValue.h

@@ -60,6 +60,8 @@ private:
     bool m_attr_tainted { false };
 };
 
+static_assert(sizeof(ComponentValue) <= 112, "Keep the size of CSS component values down!");
+
 }
 
 template<>

Libraries/LibWeb/CSS/Parser/RustTokenizer.cpp

@@ -104,7 +104,9 @@ static Number::Type css_number_type_from_ffi(FFI::CssNumberType number_type)
 
 static SourcePosition position_from_ffi(size_t line, size_t column)
 {
-    return { line, column };
+    VERIFY(line <= NumericLimits<u32>::max());
+    VERIFY(column <= NumericLimits<u32>::max());
+    return { static_cast<u32>(line), static_cast<u32>(column) };
 }
 
 Token RustTokenizer::token_from_ffi(FFI::CssToken const& ffi_token)

Libraries/LibWeb/CSS/Parser/SourcePosition.h

@@ -9,6 +9,6 @@
 #include <AK/Types.h>
 
 struct SourcePosition {
-    size_t line { 0 };
-    size_t column { 0 };
+    u32 line { 0 };
+    u32 column { 0 };
 };

Libraries/LibWeb/CSS/Parser/Token.cpp

@@ -67,8 +67,7 @@ Token Token::create_hash(FlyString value, HashType hash_type, String original_so
 {
     Token token;
     token.m_type = Type::Hash;
-    token.m_value = move(value);
-    token.m_hash_type = hash_type;
+    token.m_value = HashValue { move(value), hash_type };
     token.m_original_source_text = move(original_source_text);
     return token;
 }
@@ -95,7 +94,7 @@ Token Token::create_delim(u32 delim, String original_source_text)
 {
     Token token;
     token.m_type = Type::Delim;
-    token.m_value = String::from_code_point(delim);
+    token.m_value = delim;
     token.m_original_source_text = move(original_source_text);
     return token;
 }
@@ -104,7 +103,7 @@ Token Token::create_number(Number value, String original_source_text)
 {
     Token token;
     token.m_type = Type::Number;
-    token.m_number_value = value;
+    token.m_value = value;
     token.m_original_source_text = move(original_source_text);
     return token;
 }
@@ -113,7 +112,7 @@ Token Token::create_percentage(Number value, String original_source_text)
 {
     Token token;
     token.m_type = Type::Percentage;
-    token.m_number_value = value;
+    token.m_value = value;
     token.m_original_source_text = move(original_source_text);
     return token;
 }
@@ -122,8 +121,7 @@ Token Token::create_dimension(Number value, FlyString unit, String original_sour
 {
     Token token;
     token.m_type = Type::Dimension;
-    token.m_number_value = value;
-    token.m_value = move(unit);
+    token.m_value = DimensionValue { value, move(unit) };
     token.m_original_source_text = move(original_source_text);
     return token;
 }
@@ -150,7 +148,7 @@ String Token::to_string() const
     case Type::AtKeyword:
         return MUST(String::formatted("@{}", serialize_an_identifier(at_keyword())));
     case Type::Hash: {
-        switch (m_hash_type) {
+        switch (hash_type()) {
         case HashType::Id:
             return MUST(String::formatted("#{}", serialize_an_identifier(hash_value())));
         case HashType::Unrestricted:
@@ -167,13 +165,13 @@ String Token::to_string() const
     case Type::BadUrl:
         return "url()"_string;
     case Type::Delim:
-        return String { m_value };
+        return String::from_code_point(delim());
     case Type::Number:
-        return String::number(m_number_value.value());
+        return String::number(m_value.get<Number>().value());
     case Type::Percentage:
-        return MUST(String::formatted("{}%", m_number_value.value()));
+        return MUST(String::formatted("{}%", m_value.get<Number>().value()));
     case Type::Dimension:
-        return MUST(String::formatted("{}{}", m_number_value.value(), dimension_unit()));
+        return MUST(String::formatted("{}{}", m_value.get<DimensionValue>().number.value(), dimension_unit()));
     case Type::Whitespace:
         return " "_string;
     case Type::CDO:
@@ -261,7 +259,7 @@ String Token::to_debug_string() const
     case Type::Hash:
         builder.append("Hash(value="sv);
         append_quoted_string(builder, hash_value().bytes_as_string_view());
-        builder.appendff(", hash_type={}", hash_type_name(m_hash_type));
+        builder.appendff(", hash_type={}", hash_type_name(hash_type()));
         has_type_specific_fields = true;
         break;
     case Type::String:
@@ -282,20 +280,20 @@ String Token::to_debug_string() const
         break;
     case Type::Delim:
         builder.append("Delim(value="sv);
-        append_quoted_string(builder, m_value.bytes_as_string_view());
+        append_quoted_string(builder, String::from_code_point(delim()));
         builder.appendff(", code_point=U+{:04X}", delim());
         has_type_specific_fields = true;
         break;
     case Type::Number:
-        builder.appendff("Number(value={}, number_type={}", number_value(), number_type_name(m_number_value.type()));
+        builder.appendff("Number(value={}, number_type={}", number_value(), number_type_name(m_value.get<Number>().type()));
         has_type_specific_fields = true;
         break;
     case Type::Percentage:
-        builder.appendff("Percentage(value={}, number_type={}", percentage(), number_type_name(m_number_value.type()));
+        builder.appendff("Percentage(value={}, number_type={}", percentage(), number_type_name(m_value.get<Number>().type()));
         has_type_specific_fields = true;
         break;
     case Type::Dimension:
-        builder.appendff("Dimension(value={}, number_type={}, unit=", dimension_value(), number_type_name(m_number_value.type()));
+        builder.appendff("Dimension(value={}, number_type={}, unit=", dimension_value(), number_type_name(m_value.get<DimensionValue>().number.type()));
         append_quoted_string(builder, dimension_unit().bytes_as_string_view());
         has_type_specific_fields = true;
         break;
@@ -424,4 +422,16 @@ void Token::set_position_range(Badge<Tokenizer, RustTokenizer>, SourcePosition s
     m_end_position = end;
 }
 
+FlyString const& Token::string_value() const
+{
+    return m_value.get<FlyString>();
+}
+
+Number const& Token::number_value_for_type() const
+{
+    if (m_type == Type::Dimension)
+        return m_value.get<DimensionValue>().number;
+    return m_value.get<Number>();
+}
+
 }

Libraries/LibWeb/CSS/Parser/Token.h

@@ -8,6 +8,7 @@
 #pragma once
 
 #include <AK/FlyString.h>
+#include <AK/Variant.h>
 #include <LibWeb/CSS/Number.h>
 #include <LibWeb/CSS/Parser/SourcePosition.h>
 #include <LibWeb/Export.h>
@@ -87,89 +88,89 @@ public:
     FlyString const& ident() const
     {
         VERIFY(m_type == Type::Ident);
-        return m_value;
+        return string_value();
     }
 
     FlyString const& function() const
     {
         VERIFY(m_type == Type::Function);
-        return m_value;
+        return string_value();
     }
 
     u32 delim() const
     {
         VERIFY(m_type == Type::Delim);
-        return *m_value.code_points().begin();
+        return m_value.get<u32>();
     }
 
     FlyString const& string() const
     {
         VERIFY(m_type == Type::String);
-        return m_value;
+        return string_value();
     }
 
     FlyString const& url() const
     {
         VERIFY(m_type == Type::Url);
-        return m_value;
+        return string_value();
     }
 
     FlyString const& at_keyword() const
     {
         VERIFY(m_type == Type::AtKeyword);
-        return m_value;
+        return string_value();
     }
 
     HashType hash_type() const
     {
         VERIFY(m_type == Type::Hash);
-        return m_hash_type;
+        return m_value.get<HashValue>().type;
     }
     FlyString const& hash_value() const
     {
         VERIFY(m_type == Type::Hash);
-        return m_value;
+        return m_value.get<HashValue>().value;
     }
 
     bool is_integer() const
     {
         VERIFY(m_type == Type::Number || m_type == Type::Dimension || m_type == Type::Percentage);
-        return m_number_value.is_integer();
+        return number_value_for_type().is_integer();
     }
 
     bool is_integer_with_explicit_sign() const
     {
         VERIFY(m_type == Type::Number || m_type == Type::Dimension || m_type == Type::Percentage);
-        return m_number_value.is_integer_with_explicit_sign();
+        return number_value_for_type().is_integer_with_explicit_sign();
     }
 
     double number_value() const
     {
         VERIFY(m_type == Type::Number);
-        return clamp_to_single_precision(m_number_value.value());
+        return clamp_to_single_precision(m_value.get<Number>().value());
     }
     i32 to_integer() const
     {
-        VERIFY(m_type == Type::Number && m_number_value.is_integer());
-        return m_number_value.integer_value();
+        VERIFY(m_type == Type::Number && m_value.get<Number>().is_integer());
+        return m_value.get<Number>().integer_value();
     }
 
     FlyString const& dimension_unit() const
     {
         VERIFY(m_type == Type::Dimension);
-        return m_value;
+        return m_value.get<DimensionValue>().unit;
     }
     double dimension_value() const
     {
         VERIFY(m_type == Type::Dimension);
-        return clamp_to_single_precision(m_number_value.value());
+        return clamp_to_single_precision(m_value.get<DimensionValue>().number.value());
     }
-    i32 dimension_value_int() const { return m_number_value.integer_value(); }
+    i32 dimension_value_int() const { return m_value.get<DimensionValue>().number.integer_value(); }
 
     double percentage() const
     {
         VERIFY(m_type == Type::Percentage);
-        return clamp_to_single_precision(m_number_value.value());
+        return clamp_to_single_precision(m_value.get<Number>().value());
     }
 
     Type mirror_variant() const;
@@ -186,21 +187,38 @@ public:
 
     bool operator==(Token const& other) const
     {
-        return m_type == other.m_type && m_value == other.m_value && m_number_value == other.m_number_value && m_hash_type == other.m_hash_type;
+        return m_type == other.m_type && m_value == other.m_value;
     }
 
 private:
+    struct HashValue {
+        FlyString value;
+        HashType type { HashType::Unrestricted };
+
+        bool operator==(HashValue const&) const = default;
+    };
+
+    struct DimensionValue {
+        Number number;
+        FlyString unit;
+
+        bool operator==(DimensionValue const&) const = default;
+    };
+
+    FlyString const& string_value() const;
+    Number const& number_value_for_type() const;
+
     Type m_type { Type::Invalid };
 
-    FlyString m_value;
-    Number m_number_value;
-    HashType m_hash_type { HashType::Unrestricted };
+    Variant<Empty, FlyString, u32, Number, HashValue, DimensionValue> m_value;
 
     String m_original_source_text;
     SourcePosition m_start_position;
     SourcePosition m_end_position;
 };
 
+static_assert(sizeof(Token) <= 64, "Keep the size of CSS parser tokens down!");
+
 }
 
 template<>

Libraries/LibWeb/CSS/Parser/Tokenizer.cpp

@@ -264,9 +264,11 @@ u32 Tokenizer::next_code_point()
 
     m_prev_position = m_position;
     if (is_newline(code_point)) {
+        VERIFY(m_position.line < NumericLimits<u32>::max());
         m_position.line++;
         m_position.column = 0;
     } else {
+        VERIFY(m_position.column < NumericLimits<u32>::max());
         m_position.column++;
     }
 

Libraries/LibWeb/CSS/Parser/Types.cpp

@@ -12,6 +12,89 @@
 
 namespace Web::CSS::Parser {
 
+ComponentValueToken::ComponentValueToken(Token const& token)
+    : m_type(token.type())
+    , m_original_source_text(token.original_source_text())
+    , m_start_position(token.start_position())
+    , m_end_position(token.end_position())
+{
+}
+
+Token::Type ComponentValueToken::mirror_variant() const
+{
+    if (is(Token::Type::OpenCurly)) {
+        return Token::Type::CloseCurly;
+    }
+
+    if (is(Token::Type::OpenSquare)) {
+        return Token::Type::CloseSquare;
+    }
+
+    if (is(Token::Type::OpenParen)) {
+        return Token::Type::CloseParen;
+    }
+
+    return Token::Type::Invalid;
+}
+
+StringView ComponentValueToken::bracket_string() const
+{
+    if (is(Token::Type::OpenCurly)) {
+        return "{"sv;
+    }
+
+    if (is(Token::Type::CloseCurly)) {
+        return "}"sv;
+    }
+
+    if (is(Token::Type::OpenSquare)) {
+        return "["sv;
+    }
+
+    if (is(Token::Type::CloseSquare)) {
+        return "]"sv;
+    }
+
+    if (is(Token::Type::OpenParen)) {
+        return "("sv;
+    }
+
+    if (is(Token::Type::CloseParen)) {
+        return ")"sv;
+    }
+
+    return ""sv;
+}
+
+StringView ComponentValueToken::bracket_mirror_string() const
+{
+    if (is(Token::Type::OpenCurly)) {
+        return "}"sv;
+    }
+
+    if (is(Token::Type::CloseCurly)) {
+        return "{"sv;
+    }
+
+    if (is(Token::Type::OpenSquare)) {
+        return "]"sv;
+    }
+
+    if (is(Token::Type::CloseSquare)) {
+        return "["sv;
+    }
+
+    if (is(Token::Type::OpenParen)) {
+        return ")"sv;
+    }
+
+    if (is(Token::Type::CloseParen)) {
+        return "("sv;
+    }
+
+    return ""sv;
+}
+
 String SimpleBlock::to_string() const
 {
     StringBuilder builder;

Libraries/LibWeb/CSS/Parser/Types.h

@@ -72,11 +72,35 @@ struct SubstitutionFunctionsPresence {
     bool has_any() const { return attr || env || if_ || inherit || var; }
 };
 
+class ComponentValueToken {
+public:
+    ComponentValueToken() = default;
+    ComponentValueToken(Token const&);
+
+    bool is(Token::Type type) const { return m_type == type; }
+    Token::Type type() const { return m_type; }
+    Token::Type mirror_variant() const;
+    StringView bracket_string() const;
+    StringView bracket_mirror_string() const;
+
+    String const& original_source_text() const { return m_original_source_text; }
+    SourcePosition const& start_position() const { return m_start_position; }
+    SourcePosition const& end_position() const { return m_end_position; }
+
+    bool operator==(ComponentValueToken const& other) const { return m_type == other.m_type; }
+
+private:
+    Token::Type m_type { Token::Type::Invalid };
+    String m_original_source_text;
+    SourcePosition m_start_position;
+    SourcePosition m_end_position;
+};
+
 // https://drafts.csswg.org/css-syntax/#simple-block
 struct SimpleBlock {
-    Token token;
+    ComponentValueToken token;
     Vector<ComponentValue> value;
-    Token end_token = {};
+    ComponentValueToken end_token = {};
 
     bool is_curly() const { return token.is(Token::Type::OpenCurly); }
     bool is_paren() const { return token.is(Token::Type::OpenParen); }
@@ -92,8 +116,8 @@ struct SimpleBlock {
 struct Function {
     FlyString name;
     Vector<ComponentValue> value;
-    Token name_token = {};
-    Token end_token = {};
+    ComponentValueToken name_token = {};
+    ComponentValueToken end_token = {};
 
     String to_string() const;
     String original_source_text() const;

Libraries/LibWeb/CSS/StyleValues/UnresolvedStyleValue.cpp

@@ -176,17 +176,17 @@ private:
 
             if (component_value.is_function()) {
                 auto& function = component_value.function();
-                m_unserialized_values.append(function.name_token);
+                m_unserialized_values.append(Parser::Token::create_function(function.name, function.name_token.original_source_text()));
                 process_values(realm, function.value);
-                m_unserialized_values.append(function.end_token);
+                m_unserialized_values.append(Parser::Token::create(function.end_token.type(), function.end_token.original_source_text()));
                 continue;
             }
 
             if (component_value.is_block()) {
                 auto& block = component_value.block();
-                m_unserialized_values.append(block.token);
+                m_unserialized_values.append(Parser::Token::create(block.token.type(), block.token.original_source_text()));
                 process_values(realm, block.value);
-                m_unserialized_values.append(block.end_token);
+                m_unserialized_values.append(Parser::Token::create(block.end_token.type(), block.end_token.original_source_text()));
                 continue;
             }