From 4278194d962044dba9d24ec5ff80ab911b405a47 Mon Sep 17 00:00:00 2001 From: Sam Atkins Date: Wed, 15 Apr 2026 11:53:15 +0100 Subject: [PATCH] LibWeb/CSS: Port the CSS Tokenizer to Rust test-css-tokenizer is updated to run both the C++ and Rust tokenizers and compare their output, to ensure they behave identically. The Parser still uses the C++ Tokenizer. The LibWeb crate, FFI layer etc are all based on the existing ones for other libraries. This is a direct AI translation to get us started, and not idiomatic Rust. Future work can be done to make it more sensible. --- Cargo.lock | 7 + Cargo.toml | 1 + Libraries/LibWeb/CMakeLists.txt | 19 + Libraries/LibWeb/CSS/Parser/RustTokenizer.cpp | 255 +++ Libraries/LibWeb/CSS/Parser/RustTokenizer.h | 30 + Libraries/LibWeb/CSS/Parser/Token.cpp | 6 + Libraries/LibWeb/CSS/Parser/Token.h | 1 + Libraries/LibWeb/CSS/Parser/Tokenizer.cpp | 2 +- Libraries/LibWeb/Forward.h | 1 + Libraries/LibWeb/Rust/Cargo.toml | 17 + Libraries/LibWeb/Rust/build.rs | 40 + Libraries/LibWeb/Rust/cbindgen.toml | 26 + Libraries/LibWeb/Rust/src/css_tokenizer.rs | 1460 +++++++++++++++++ Libraries/LibWeb/Rust/src/lib.rs | 71 + Tests/LibWeb/css-tokenizer.cpp | 12 +- Tests/LibWeb/test-css-tokenizer.py | 64 +- 16 files changed, 1997 insertions(+), 15 deletions(-) create mode 100644 Libraries/LibWeb/CSS/Parser/RustTokenizer.cpp create mode 100644 Libraries/LibWeb/CSS/Parser/RustTokenizer.h create mode 100644 Libraries/LibWeb/Rust/Cargo.toml create mode 100644 Libraries/LibWeb/Rust/build.rs create mode 100644 Libraries/LibWeb/Rust/cbindgen.toml create mode 100644 Libraries/LibWeb/Rust/src/css_tokenizer.rs create mode 100644 Libraries/LibWeb/Rust/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 100375979b..a70b5a4648 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -405,6 +405,13 @@ dependencies = [ "icu_calendar", ] +[[package]] +name = "libweb_rust" +version = "0.1.0" +dependencies = [ + "cbindgen", +] + [[package]] name = "linux-raw-sys" version = "0.12.1" diff --git a/Cargo.toml b/Cargo.toml index 6d5a3dd2a3..4b7cc78864 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,6 +4,7 @@ members = [ "Libraries/LibJS/Rust", "Libraries/LibRegex/Rust", "Libraries/LibUnicode/Rust", + "Libraries/LibWeb/Rust", ] exclude = [ "Libraries/LibJS/AsmIntGen", diff --git a/Libraries/LibWeb/CMakeLists.txt b/Libraries/LibWeb/CMakeLists.txt index a2ff4cdf14..11d4dc9ad7 100644 --- a/Libraries/LibWeb/CMakeLists.txt +++ b/Libraries/LibWeb/CMakeLists.txt @@ -213,6 +213,7 @@ set(SOURCES CSS/Parser/Helpers.cpp CSS/Parser/MediaParsing.cpp CSS/Parser/Parser.cpp + CSS/Parser/RustTokenizer.cpp CSS/Parser/PropertyParsing.cpp CSS/Parser/RuleContext.cpp CSS/Parser/RuleParsing.cpp @@ -1250,6 +1251,24 @@ ladybird_lib(LibWeb web EXPLICIT_SYMBOL_EXPORT) target_link_libraries(LibWeb PRIVATE LibCore LibCompress LibCrypto LibJS LibHTTP LibGfx LibIPC LibRegex LibSyntax LibTextCodec LibUnicode LibMedia LibWasm LibXML LibIDL LibURL LibTLS LibRequests LibGC LibThreading skia ${ANGLE_TARGETS} SDL3::SDL3 LibXml2::LibXml2) +import_rust_crate(MANIFEST_PATH Rust/Cargo.toml CRATE_NAME libweb_rust FFI_HEADER RustFFI.h) +target_link_libraries(LibWeb PRIVATE libweb_rust) +if ((LINUX OR BSD) AND NOT BUILD_SHARED_LIBS) + target_link_options(LibWeb INTERFACE LINKER:--allow-multiple-definition) +endif() + +if(NOT BUILD_SHARED_LIBS) + add_custom_command(TARGET LibWeb POST_BUILD + COMMAND ${CMAKE_AR} -x $ + COMMAND ${CMAKE_AR} -qS $ *.o + COMMAND ${CMAKE_RANLIB} $ + COMMAND ${CMAKE_COMMAND} -E remove -f *.o + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/rust_merge_tmp + COMMENT "Merging Rust archive into LibWeb" + ) + file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/rust_merge_tmp) +endif() + if (HAS_FONTCONFIG) target_link_libraries(LibWeb PRIVATE Fontconfig::Fontconfig) endif() diff --git a/Libraries/LibWeb/CSS/Parser/RustTokenizer.cpp b/Libraries/LibWeb/CSS/Parser/RustTokenizer.cpp new file mode 100644 index 0000000000..295671fa30 --- /dev/null +++ b/Libraries/LibWeb/CSS/Parser/RustTokenizer.cpp @@ -0,0 +1,255 @@ +/* + * Copyright (c) 2026, the Ladybird developers. + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include +#include +#include +#include +#include +#include + +namespace Web::CSS::Parser { + +// U+FFFD REPLACEMENT CHARACTER (�) +static constexpr u32 REPLACEMENT_CHARACTER = 0xFFFD; + +static String decode_and_filter_code_points(StringView input, StringView encoding) +{ + // https://www.w3.org/TR/css-syntax-3/#css-filter-code-points + auto decoder = TextCodec::decoder_for(encoding); + VERIFY(decoder.has_value()); + + auto decoded_input = MUST(decoder->to_utf8(input)); + + // OPTIMIZATION: If the input doesn't contain any filterable characters, we can skip the filtering + bool const contains_filterable = [&] { + for (auto code_point : decoded_input.code_points()) { + if (code_point == '\r' || code_point == '\f' || code_point == 0x00 || is_unicode_surrogate(code_point)) + return true; + } + return false; + }(); + if (!contains_filterable) + return decoded_input; + + StringBuilder builder { input.length() }; + bool last_was_carriage_return = false; + + // To filter code points from a stream of (unfiltered) code points input: + for (auto code_point : decoded_input.code_points()) { + // Replace any U+000D CARRIAGE RETURN (CR) code points, + // U+000C FORM FEED (FF) code points, + // or pairs of U+000D CARRIAGE RETURN (CR) followed by U+000A LINE FEED (LF) + // in input by a single U+000A LINE FEED (LF) code point. + if (code_point == '\r') { + if (last_was_carriage_return) { + builder.append('\n'); + } else { + last_was_carriage_return = true; + } + } else { + if (last_was_carriage_return) + builder.append('\n'); + + if (code_point == '\n') { + if (!last_was_carriage_return) + builder.append('\n'); + + } else if (code_point == '\f') { + builder.append('\n'); + // Replace any U+0000 NULL or surrogate code points in input with U+FFFD REPLACEMENT CHARACTER (�). + } else if (code_point == 0x00 || is_unicode_surrogate(code_point)) { + builder.append_code_point(REPLACEMENT_CHARACTER); + } else { + builder.append_code_point(code_point); + } + + last_was_carriage_return = false; + } + } + + return builder.to_string_without_validation(); +} + +static String string_from_ffi_bytes(u8 const* bytes, size_t length) +{ + if (length == 0) + return {}; + return String::from_utf8_without_validation({ bytes, length }); +} + +static FlyString fly_string_from_ffi_bytes(u8 const* bytes, size_t length) +{ + if (length == 0) + return {}; + return FlyString::from_utf8_without_validation({ bytes, length }); +} + +static Number::Type css_number_type_from_ffi(FFI::CssNumberType number_type) +{ + switch (number_type) { + case FFI::CssNumberType::Number: + return Number::Type::Number; + case FFI::CssNumberType::IntegerWithExplicitSign: + return Number::Type::IntegerWithExplicitSign; + case FFI::CssNumberType::Integer: + return Number::Type::Integer; + } + VERIFY_NOT_REACHED(); +} + +static Token::Position position_from_ffi(size_t line, size_t column) +{ + return { line, column }; +} + +Token RustTokenizer::token_from_ffi(FFI::CssToken const& ffi_token) +{ + auto original_source_text = string_from_ffi_bytes(ffi_token.original_source_ptr, ffi_token.original_source_len); + auto payload = fly_string_from_ffi_bytes(ffi_token.value_ptr, ffi_token.value_len); + + Token token; + switch (ffi_token.token_type) { + case FFI::CssTokenType::Invalid: + VERIFY_NOT_REACHED(); + case FFI::CssTokenType::EndOfFile: + token = Token::create(Token::Type::EndOfFile, move(original_source_text)); + break; + case FFI::CssTokenType::Ident: + token = Token::create_ident(move(payload), move(original_source_text)); + break; + case FFI::CssTokenType::Function: + token = Token::create_function(move(payload), move(original_source_text)); + break; + case FFI::CssTokenType::AtKeyword: + token = Token::create_at_keyword(move(payload), move(original_source_text)); + break; + case FFI::CssTokenType::Hash: + token = Token::create_hash( + move(payload), + ffi_token.hash_type == FFI::CssHashType::Id ? Token::HashType::Id : Token::HashType::Unrestricted, + move(original_source_text)); + break; + case FFI::CssTokenType::String: + token = Token::create_string(move(payload), move(original_source_text)); + break; + case FFI::CssTokenType::BadString: + token = Token::create(Token::Type::BadString, move(original_source_text)); + break; + case FFI::CssTokenType::Url: + token = Token::create_url(move(payload), move(original_source_text)); + break; + case FFI::CssTokenType::BadUrl: + token = Token::create(Token::Type::BadUrl, move(original_source_text)); + break; + case FFI::CssTokenType::Delim: + token = Token::create_delim(ffi_token.delim, move(original_source_text)); + break; + case FFI::CssTokenType::Number: + token = Token::create_number(Number { css_number_type_from_ffi(ffi_token.number_type), ffi_token.number_value }, move(original_source_text)); + break; + case FFI::CssTokenType::Percentage: + token = Token::create_percentage(Number { css_number_type_from_ffi(ffi_token.number_type), ffi_token.number_value }, move(original_source_text)); + break; + case FFI::CssTokenType::Dimension: + token = Token::create_dimension(Number { css_number_type_from_ffi(ffi_token.number_type), ffi_token.number_value }, move(payload), move(original_source_text)); + break; + case FFI::CssTokenType::Whitespace: + token = Token::create_whitespace(move(original_source_text)); + break; + case FFI::CssTokenType::CDO: + token = Token::create(Token::Type::CDO, move(original_source_text)); + break; + case FFI::CssTokenType::CDC: + token = Token::create(Token::Type::CDC, move(original_source_text)); + break; + case FFI::CssTokenType::Colon: + token = Token::create(Token::Type::Colon, move(original_source_text)); + break; + case FFI::CssTokenType::Semicolon: + token = Token::create(Token::Type::Semicolon, move(original_source_text)); + break; + case FFI::CssTokenType::Comma: + token = Token::create(Token::Type::Comma, move(original_source_text)); + break; + case FFI::CssTokenType::OpenSquare: + token = Token::create(Token::Type::OpenSquare, move(original_source_text)); + break; + case FFI::CssTokenType::CloseSquare: + token = Token::create(Token::Type::CloseSquare, move(original_source_text)); + break; + case FFI::CssTokenType::OpenParen: + token = Token::create(Token::Type::OpenParen, move(original_source_text)); + break; + case FFI::CssTokenType::CloseParen: + token = Token::create(Token::Type::CloseParen, move(original_source_text)); + break; + case FFI::CssTokenType::OpenCurly: + token = Token::create(Token::Type::OpenCurly, move(original_source_text)); + break; + case FFI::CssTokenType::CloseCurly: + token = Token::create(Token::Type::CloseCurly, move(original_source_text)); + break; + } + + token.set_position_range(Badge {}, position_from_ffi(ffi_token.start_line, ffi_token.start_column), position_from_ffi(ffi_token.end_line, ffi_token.end_column)); + return token; +} + +static_assert(static_cast(FFI::CssTokenType::Invalid) == static_cast(Token::Type::Invalid)); +static_assert(static_cast(FFI::CssTokenType::EndOfFile) == static_cast(Token::Type::EndOfFile)); +static_assert(static_cast(FFI::CssTokenType::Ident) == static_cast(Token::Type::Ident)); +static_assert(static_cast(FFI::CssTokenType::Function) == static_cast(Token::Type::Function)); +static_assert(static_cast(FFI::CssTokenType::AtKeyword) == static_cast(Token::Type::AtKeyword)); +static_assert(static_cast(FFI::CssTokenType::Hash) == static_cast(Token::Type::Hash)); +static_assert(static_cast(FFI::CssTokenType::String) == static_cast(Token::Type::String)); +static_assert(static_cast(FFI::CssTokenType::BadString) == static_cast(Token::Type::BadString)); +static_assert(static_cast(FFI::CssTokenType::Url) == static_cast(Token::Type::Url)); +static_assert(static_cast(FFI::CssTokenType::BadUrl) == static_cast(Token::Type::BadUrl)); +static_assert(static_cast(FFI::CssTokenType::Delim) == static_cast(Token::Type::Delim)); +static_assert(static_cast(FFI::CssTokenType::Number) == static_cast(Token::Type::Number)); +static_assert(static_cast(FFI::CssTokenType::Percentage) == static_cast(Token::Type::Percentage)); +static_assert(static_cast(FFI::CssTokenType::Dimension) == static_cast(Token::Type::Dimension)); +static_assert(static_cast(FFI::CssTokenType::Whitespace) == static_cast(Token::Type::Whitespace)); +static_assert(static_cast(FFI::CssTokenType::CDO) == static_cast(Token::Type::CDO)); +static_assert(static_cast(FFI::CssTokenType::CDC) == static_cast(Token::Type::CDC)); +static_assert(static_cast(FFI::CssTokenType::Colon) == static_cast(Token::Type::Colon)); +static_assert(static_cast(FFI::CssTokenType::Semicolon) == static_cast(Token::Type::Semicolon)); +static_assert(static_cast(FFI::CssTokenType::Comma) == static_cast(Token::Type::Comma)); +static_assert(static_cast(FFI::CssTokenType::OpenSquare) == static_cast(Token::Type::OpenSquare)); +static_assert(static_cast(FFI::CssTokenType::CloseSquare) == static_cast(Token::Type::CloseSquare)); +static_assert(static_cast(FFI::CssTokenType::OpenParen) == static_cast(Token::Type::OpenParen)); +static_assert(static_cast(FFI::CssTokenType::CloseParen) == static_cast(Token::Type::CloseParen)); +static_assert(static_cast(FFI::CssTokenType::OpenCurly) == static_cast(Token::Type::OpenCurly)); +static_assert(static_cast(FFI::CssTokenType::CloseCurly) == static_cast(Token::Type::CloseCurly)); +static_assert(static_cast(FFI::CssHashType::Id) == static_cast(Token::HashType::Id)); +static_assert(static_cast(FFI::CssHashType::Unrestricted) == static_cast(Token::HashType::Unrestricted)); +static_assert(static_cast(FFI::CssNumberType::Number) == static_cast(Number::Type::Number)); +static_assert(static_cast(FFI::CssNumberType::IntegerWithExplicitSign) == static_cast(Number::Type::IntegerWithExplicitSign)); +static_assert(static_cast(FFI::CssNumberType::Integer) == static_cast(Number::Type::Integer)); + +Vector RustTokenizer::tokenize(StringView input, StringView encoding) +{ + struct CallbackContext { + Vector tokens; + }; + + auto filtered_input = decode_and_filter_code_points(input, encoding); + auto filtered_input_bytes = filtered_input.bytes(); + CallbackContext context; + FFI::rust_css_tokenize( + filtered_input_bytes.data(), + filtered_input_bytes.size(), + &context, + [](void* raw_context, FFI::CssToken const* ffi_token) { + auto& context = *static_cast(raw_context); + context.tokens.append(token_from_ffi(*ffi_token)); + }); + + return move(context.tokens); +} + +} diff --git a/Libraries/LibWeb/CSS/Parser/RustTokenizer.h b/Libraries/LibWeb/CSS/Parser/RustTokenizer.h new file mode 100644 index 0000000000..efa55e1338 --- /dev/null +++ b/Libraries/LibWeb/CSS/Parser/RustTokenizer.h @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2026, the Ladybird developers. + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#pragma once + +#include +#include +#include +#include + +namespace Web::CSS::Parser::FFI { + +struct CssToken; + +} + +namespace Web::CSS::Parser { + +class WEB_API RustTokenizer { +public: + static Vector tokenize(StringView input, StringView encoding); + +private: + static Token token_from_ffi(FFI::CssToken const&); +}; + +} diff --git a/Libraries/LibWeb/CSS/Parser/Token.cpp b/Libraries/LibWeb/CSS/Parser/Token.cpp index 67ac7fed99..9ecd7fb6b0 100644 --- a/Libraries/LibWeb/CSS/Parser/Token.cpp +++ b/Libraries/LibWeb/CSS/Parser/Token.cpp @@ -424,4 +424,10 @@ void Token::set_position_range(Badge, Position start, Position end) m_end_position = end; } +void Token::set_position_range(Badge, Position start, Position end) +{ + m_start_position = start; + m_end_position = end; +} + } diff --git a/Libraries/LibWeb/CSS/Parser/Token.h b/Libraries/LibWeb/CSS/Parser/Token.h index 461174faf1..9052a4917d 100644 --- a/Libraries/LibWeb/CSS/Parser/Token.h +++ b/Libraries/LibWeb/CSS/Parser/Token.h @@ -187,6 +187,7 @@ public: Position const& start_position() const { return m_start_position; } Position const& end_position() const { return m_end_position; } void set_position_range(Badge, Position start, Position end); + void set_position_range(Badge, Position start, Position end); bool operator==(Token const& other) const { diff --git a/Libraries/LibWeb/CSS/Parser/Tokenizer.cpp b/Libraries/LibWeb/CSS/Parser/Tokenizer.cpp index 902f8e0477..0d20b274bb 100644 --- a/Libraries/LibWeb/CSS/Parser/Tokenizer.cpp +++ b/Libraries/LibWeb/CSS/Parser/Tokenizer.cpp @@ -231,7 +231,7 @@ Vector Tokenizer::tokenize() for (;;) { auto token_start = m_position; auto token = consume_a_token(); - token.set_position_range({}, token_start, m_position); + token.set_position_range(Badge {}, token_start, m_position); tokens.append(token); if (token.is(Token::Type::EndOfFile)) { diff --git a/Libraries/LibWeb/Forward.h b/Libraries/LibWeb/Forward.h index 29b93d8e50..4c52d54f66 100644 --- a/Libraries/LibWeb/Forward.h +++ b/Libraries/LibWeb/Forward.h @@ -508,6 +508,7 @@ namespace Web::CSS::Parser { class ComponentValue; class GuardedSubstitutionContexts; class Parser; +class RustTokenizer; class SyntaxNode; class Token; class Tokenizer; diff --git a/Libraries/LibWeb/Rust/Cargo.toml b/Libraries/LibWeb/Rust/Cargo.toml new file mode 100644 index 0000000000..e71fddd47b --- /dev/null +++ b/Libraries/LibWeb/Rust/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "libweb_rust" +version = "0.1.0" +edition = "2024" + +[lib] +crate-type = ["staticlib"] + +# After changing dependencies, regenerate the Flatpak sources: +# python3 Meta/CMake/flatpak/generate-cargo-sources.py +[dependencies] + +[build-dependencies] +cbindgen = "0.29" + +[lints] +workspace = true diff --git a/Libraries/LibWeb/Rust/build.rs b/Libraries/LibWeb/Rust/build.rs new file mode 100644 index 0000000000..172dc068de --- /dev/null +++ b/Libraries/LibWeb/Rust/build.rs @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2026-present, the Ladybird developers. + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +use std::env; +use std::error::Error; +use std::path::PathBuf; + +fn main() -> Result<(), Box> { + let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR")?); + let out_dir = PathBuf::from(env::var("OUT_DIR")?); + + println!("cargo:rerun-if-changed=build.rs"); + println!("cargo:rerun-if-changed=cbindgen.toml"); + println!("cargo:rerun-if-env-changed=FFI_OUTPUT_DIR"); + println!("cargo:rerun-if-changed=src"); + + let ffi_out_dir = env::var("FFI_OUTPUT_DIR") + .map(PathBuf::from) + .unwrap_or_else(|_| out_dir.clone()); + + cbindgen::generate(manifest_dir).map_or_else( + |error| match error { + cbindgen::Error::ParseSyntaxError { .. } => {} + other => panic!("{other:?}"), + }, + |bindings| { + let header_path = out_dir.join("RustFFI.h"); + bindings.write_to_file(&header_path); + + if ffi_out_dir != out_dir { + bindings.write_to_file(ffi_out_dir.join("RustFFI.h")); + } + }, + ); + + Ok(()) +} diff --git a/Libraries/LibWeb/Rust/cbindgen.toml b/Libraries/LibWeb/Rust/cbindgen.toml new file mode 100644 index 0000000000..68c11b9886 --- /dev/null +++ b/Libraries/LibWeb/Rust/cbindgen.toml @@ -0,0 +1,26 @@ +language = "C++" +header = """/* + * Copyright (c) 2026-present, the Ladybird developers. + * + * SPDX-License-Identifier: BSD-2-Clause + */""" +pragma_once = true +include_version = true +namespaces = ["Web", "CSS", "Parser", "FFI"] +line_length = 120 +tab_width = 4 +no_includes = true +sys_includes = ["stdint.h", "stddef.h"] +usize_is_size_t = true + +[parse] +parse_deps = false + +[parse.expand] +all_features = false + +[export] +include = ["CssHashType", "CssNumberType", "CssToken", "CssTokenType"] + +[export.mangle] +rename_types = "PascalCase" diff --git a/Libraries/LibWeb/Rust/src/css_tokenizer.rs b/Libraries/LibWeb/Rust/src/css_tokenizer.rs new file mode 100644 index 0000000000..73d66a90c4 --- /dev/null +++ b/Libraries/LibWeb/Rust/src/css_tokenizer.rs @@ -0,0 +1,1460 @@ +/* + * Copyright (c) 2026-present, the Ladybird developers. + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +use std::char; +use std::ptr; + +const REPLACEMENT_CHARACTER: u32 = 0xFFFD; +const TOKENIZER_EOF: u32 = u32::MAX; + +// NB: Keep this in sync with Web::CSS::Parser::Token::Type in Token.h. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[repr(C)] +pub enum CssTokenType { + Invalid, + EndOfFile, + Ident, + Function, + AtKeyword, + Hash, + String, + BadString, + Url, + BadUrl, + Delim, + Number, + Percentage, + Dimension, + Whitespace, + CDO, + CDC, + Colon, + Semicolon, + Comma, + OpenSquare, + CloseSquare, + OpenParen, + CloseParen, + OpenCurly, + CloseCurly, +} + +// NB: Keep this in sync with Web::CSS::Parser::Token::HashType in Token.h. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[repr(C)] +pub enum CssHashType { + Id, + Unrestricted, +} + +// NB: Keep this in sync with Web::CSS::Number::Type in Number.h. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[repr(C)] +pub enum CssNumberType { + Number, + IntegerWithExplicitSign, + Integer, +} + +#[repr(C)] +pub struct CssToken { + pub token_type: CssTokenType, + pub hash_type: CssHashType, + pub number_type: CssNumberType, + pub number_value: f64, + pub delim: u32, + pub value_ptr: *const u8, + pub value_len: usize, + pub original_source_ptr: *const u8, + pub original_source_len: usize, + pub start_line: usize, + pub start_column: usize, + pub end_line: usize, + pub end_column: usize, +} + +#[derive(Clone, Copy, Debug, Default)] +struct Position { + line: usize, + column: usize, +} + +#[derive(Clone, Copy)] +struct U32Twin { + first: u32, + second: u32, +} + +impl Default for U32Twin { + fn default() -> Self { + Self { + first: TOKENIZER_EOF, + second: TOKENIZER_EOF, + } + } +} + +#[derive(Clone, Copy)] +struct U32Triplet { + first: u32, + second: u32, + third: u32, +} + +impl Default for U32Triplet { + fn default() -> Self { + Self { + first: TOKENIZER_EOF, + second: TOKENIZER_EOF, + third: TOKENIZER_EOF, + } + } +} + +impl U32Triplet { + fn to_twin_12(self) -> U32Twin { + U32Twin { + first: self.first, + second: self.second, + } + } + + fn to_twin_23(self) -> U32Twin { + U32Twin { + first: self.second, + second: self.third, + } + } +} + +#[derive(Clone, Copy)] +struct NumericValue { + number_type: CssNumberType, + value: f64, +} + +pub(crate) struct Token { + token_type: CssTokenType, + value: String, + number_value: f64, + number_type: CssNumberType, + hash_type: CssHashType, + delim: u32, + original_source_start: usize, + original_source_end: usize, + start: Position, + end: Position, +} + +impl Token { + fn create(token_type: CssTokenType, original_source_start: usize, original_source_end: usize) -> Self { + Self { + token_type, + value: String::new(), + number_value: 0.0, + number_type: CssNumberType::Number, + hash_type: CssHashType::Unrestricted, + delim: 0, + original_source_start, + original_source_end, + start: Position::default(), + end: Position::default(), + } + } + + fn create_ident(value: String, original_source_start: usize, original_source_end: usize) -> Self { + Self { + token_type: CssTokenType::Ident, + value, + ..Self::create(CssTokenType::Ident, original_source_start, original_source_end) + } + } + + fn create_function(value: String, original_source_start: usize, original_source_end: usize) -> Self { + Self { + token_type: CssTokenType::Function, + value, + ..Self::create(CssTokenType::Function, original_source_start, original_source_end) + } + } + + fn create_at_keyword(value: String, original_source_start: usize, original_source_end: usize) -> Self { + Self { + token_type: CssTokenType::AtKeyword, + value, + ..Self::create(CssTokenType::AtKeyword, original_source_start, original_source_end) + } + } + + fn create_hash( + value: String, + hash_type: CssHashType, + original_source_start: usize, + original_source_end: usize, + ) -> Self { + Self { + token_type: CssTokenType::Hash, + value, + hash_type, + ..Self::create(CssTokenType::Hash, original_source_start, original_source_end) + } + } + + fn create_string(value: String, original_source_start: usize, original_source_end: usize) -> Self { + Self { + token_type: CssTokenType::String, + value, + ..Self::create(CssTokenType::String, original_source_start, original_source_end) + } + } + + fn create_url(value: String, original_source_start: usize, original_source_end: usize) -> Self { + Self { + token_type: CssTokenType::Url, + value, + ..Self::create(CssTokenType::Url, original_source_start, original_source_end) + } + } + + fn create_delim(delim: u32, original_source_start: usize, original_source_end: usize) -> Self { + Self { + token_type: CssTokenType::Delim, + delim, + ..Self::create(CssTokenType::Delim, original_source_start, original_source_end) + } + } + + fn create_number(number: NumericValue, original_source_start: usize, original_source_end: usize) -> Self { + Self { + token_type: CssTokenType::Number, + number_value: number.value, + number_type: number.number_type, + ..Self::create(CssTokenType::Number, original_source_start, original_source_end) + } + } + + fn create_percentage(number: NumericValue, original_source_start: usize, original_source_end: usize) -> Self { + Self { + token_type: CssTokenType::Percentage, + number_value: number.value, + number_type: number.number_type, + ..Self::create(CssTokenType::Percentage, original_source_start, original_source_end) + } + } + + fn create_dimension( + number: NumericValue, + unit: String, + original_source_start: usize, + original_source_end: usize, + ) -> Self { + Self { + token_type: CssTokenType::Dimension, + value: unit, + number_value: number.value, + number_type: number.number_type, + ..Self::create(CssTokenType::Dimension, original_source_start, original_source_end) + } + } + + fn create_whitespace(original_source_start: usize, original_source_end: usize) -> Self { + Self::create(CssTokenType::Whitespace, original_source_start, original_source_end) + } + + pub(crate) fn as_ffi(&self, filtered_input: &str) -> CssToken { + let (value_ptr, value_len) = string_parts(&self.value); + let original_source = &filtered_input.as_bytes()[self.original_source_start..self.original_source_end]; + let (original_source_ptr, original_source_len) = bytes_parts(original_source); + + CssToken { + token_type: self.token_type, + hash_type: self.hash_type, + number_type: self.number_type, + number_value: self.number_value, + delim: self.delim, + value_ptr, + value_len, + original_source_ptr, + original_source_len, + start_line: self.start.line, + start_column: self.start.column, + end_line: self.end.line, + end_column: self.end.column, + } + } +} + +pub(crate) struct TokenizationResult { + pub filtered_input: String, + pub tokens: Vec, +} + +pub fn tokenize(filtered_input: &[u8]) -> TokenizationResult { + let filtered_input = std::str::from_utf8(filtered_input) + .expect("rust_css_tokenize received non-UTF-8 input after C++ decoding") + .to_owned(); + Tokenizer::new(filtered_input).tokenize() +} + +struct Tokenizer { + input: String, + code_points: Vec<(usize, u32)>, + index: usize, + prev_index: usize, + position: Position, + prev_position: Position, +} + +impl Tokenizer { + fn new(input: String) -> Self { + let code_points = input + .char_indices() + .map(|(offset, code_point)| (offset, code_point as u32)) + .collect(); + + Self { + input, + code_points, + index: 0, + prev_index: 0, + position: Position::default(), + prev_position: Position::default(), + } + } + + fn tokenize(mut self) -> TokenizationResult { + let mut tokens = Vec::new(); + + loop { + let token_start = self.position; + let mut token = self.consume_a_token(); + token.start = token_start; + token.end = self.position; + let is_eof = token.token_type == CssTokenType::EndOfFile; + tokens.push(token); + + if is_eof { + return TokenizationResult { + filtered_input: self.input, + tokens, + }; + } + } + } + + fn current_byte_offset(&self) -> usize { + if let Some((offset, _)) = self.code_points.get(self.index) { + *offset + } else { + self.input.len() + } + } + + fn next_code_point(&mut self) -> u32 { + if self.index >= self.code_points.len() { + return TOKENIZER_EOF; + } + + self.prev_index = self.index; + self.prev_position = self.position; + + let (_, code_point) = self.code_points[self.index]; + self.index += 1; + + if is_newline(code_point) { + self.position.line += 1; + self.position.column = 0; + } else { + self.position.column += 1; + } + + code_point + } + + fn peek_code_point(&self, offset: usize) -> u32 { + self.code_points + .get(self.index + offset) + .map(|(_, code_point)| *code_point) + .unwrap_or(TOKENIZER_EOF) + } + + fn peek_twin(&self) -> U32Twin { + U32Twin { + first: self.peek_code_point(0), + second: self.peek_code_point(1), + } + } + + fn peek_triplet(&self) -> U32Triplet { + U32Triplet { + first: self.peek_code_point(0), + second: self.peek_code_point(1), + third: self.peek_code_point(2), + } + } + + fn start_of_input_stream_twin(&mut self) -> U32Twin { + // FIXME: Reconsuming just to read the current code point again is weird. + self.reconsume_current_input_code_point(); + U32Twin { + first: self.next_code_point(), + second: self.peek_code_point(0), + } + } + + fn start_of_input_stream_triplet(&mut self) -> U32Triplet { + // FIXME: Reconsuming just to read the current code point again is weird. + self.reconsume_current_input_code_point(); + let first = self.next_code_point(); + let next_two = self.peek_twin(); + U32Triplet { + first, + second: next_two.first, + third: next_two.second, + } + } + + fn reconsume_current_input_code_point(&mut self) { + self.index = self.prev_index; + self.position = self.prev_position; + } + + // https://www.w3.org/TR/css-syntax-3/#consume-comment + fn consume_comments(&mut self) { + // This section describes how to consume comments from a stream of code points. + // It returns nothing. + + loop { + // If the next two input code point are U+002F SOLIDUS (/) followed by a U+002A ASTERISK (*), + // consume them and all following code points up to and including the first U+002A ASTERISK (*) + // followed by a U+002F SOLIDUS (/), or up to an EOF code point. Return to the start of this step. + // + // If the preceding paragraph ended by consuming an EOF code point, this is a parse error. + // + // Return nothing. + let twin = self.peek_twin(); + if !(is_solidus(twin.first) && is_asterisk(twin.second)) { + return; + } + + self.next_code_point(); + self.next_code_point(); + + loop { + let twin = self.peek_twin(); + if is_eof(twin.first) || is_eof(twin.second) { + return; + } + + if is_asterisk(twin.first) && is_solidus(twin.second) { + self.next_code_point(); + self.next_code_point(); + break; + } + + self.next_code_point(); + } + } + } + + fn consume_as_much_whitespace_as_possible(&mut self) { + while is_whitespace(self.peek_code_point(0)) { + self.next_code_point(); + } + } + + // https://www.w3.org/TR/css-syntax-3/#consume-escaped-code-point + fn consume_escaped_code_point(&mut self) -> u32 { + // This section describes how to consume an escaped code point. + // It assumes that the U+005C REVERSE SOLIDUS (\) has already been consumed and that the next + // input code point has already been verified to be part of a valid escape. + // It will return a code point. + + // Consume the next input code point. + let input = self.next_code_point(); + + // hex digit + if is_hex_digit(input) { + let mut repr = String::new(); + append_code_point(&mut repr, input); + + // Consume as many hex digits as possible, but no more than 5. + // Note that this means 1-6 hex digits have been consumed in total. + let mut counter = 0usize; + while is_hex_digit(self.peek_code_point(0)) && counter < 5 { + counter += 1; + append_code_point(&mut repr, self.next_code_point()); + } + + // If the next input code point is whitespace, consume it as well. + if is_whitespace(self.peek_code_point(0)) { + self.next_code_point(); + } + + // Interpret the hex digits as a hexadecimal number. + let unhexed = u32::from_str_radix(&repr, 16).unwrap_or(0); + // If this number is zero, or is for a surrogate, or is greater than the maximum allowed + // code point, return U+FFFD REPLACEMENT CHARACTER (�). + if unhexed == 0 || is_unicode_surrogate(unhexed) || is_greater_than_maximum_allowed_code_point(unhexed) { + return REPLACEMENT_CHARACTER; + } + + // Otherwise, return the code point with that value. + return unhexed; + } + + // EOF + if is_eof(input) { + // This is a parse error. Return U+FFFD REPLACEMENT CHARACTER (�). + return REPLACEMENT_CHARACTER; + } + + // anything else + // Return the current input code point. + input + } + + // https://www.w3.org/TR/css-syntax-3/#consume-ident-like-token + fn consume_an_ident_like_token(&mut self) -> Token { + // This section describes how to consume an ident-like token from a stream of code points. + // It returns an , , , or . + + // Consume an ident sequence, and let string be the result. + let start_byte_offset = self.current_byte_offset(); + let string = self.consume_an_ident_sequence(); + + // If string’s value is an ASCII case-insensitive match for "url", and the next input code + // point is U+0028 LEFT PARENTHESIS ((), consume it. + if string.eq_ignore_ascii_case("url") && is_left_paren(self.peek_code_point(0)) { + self.next_code_point(); + + // While the next two input code points are whitespace, consume the next input code point. + loop { + let maybe_whitespace = self.peek_twin(); + if !(is_whitespace(maybe_whitespace.first) && is_whitespace(maybe_whitespace.second)) { + break; + } + self.next_code_point(); + } + + // If the next one or two input code points are U+0022 QUOTATION MARK ("), U+0027 APOSTROPHE ('), + // or whitespace followed by U+0022 QUOTATION MARK (") or U+0027 APOSTROPHE ('), then create a + // with its value set to string and return it. + let next_two = self.peek_twin(); + if is_quotation_mark(next_two.first) + || is_apostrophe(next_two.first) + || (is_whitespace(next_two.first) + && (is_quotation_mark(next_two.second) || is_apostrophe(next_two.second))) + { + return Token::create_function(string, start_byte_offset, self.current_byte_offset()); + } + + // Otherwise, consume a url token, and return it. + return self.consume_a_url_token(start_byte_offset); + } + + // Otherwise, if the next input code point is U+0028 LEFT PARENTHESIS ((), consume it. + if is_left_paren(self.peek_code_point(0)) { + self.next_code_point(); + + // Create a with its value set to string and return it. + return Token::create_function(string, start_byte_offset, self.current_byte_offset()); + } + + // Otherwise, create an with its value set to string and return it. + Token::create_ident(string, start_byte_offset, self.current_byte_offset()) + } + + // https://www.w3.org/TR/css-syntax-3/#consume-number + fn consume_a_number(&mut self) -> NumericValue { + // This section describes how to consume a number from a stream of code points. + // It returns a numeric value, and a type which is either "integer" or "number". + // + // Note: This algorithm does not do the verification of the first few code points + // that are necessary to ensure a number can be obtained from the stream. Ensure + // that the stream starts with a number before calling this algorithm. + + // Execute the following steps in order: + + // 1. Initially set type to "integer". Let repr be the empty string. + let mut repr = String::new(); + let mut number_type = CssNumberType::Integer; + + // 2. If the next input code point is U+002B PLUS SIGN (+) or U+002D HYPHEN-MINUS (-), + // consume it and append it to repr. + let mut has_explicit_sign = false; + let next_input = self.peek_code_point(0); + if is_plus_sign(next_input) || is_hyphen_minus(next_input) { + has_explicit_sign = true; + append_code_point(&mut repr, self.next_code_point()); + } + + // 3. While the next input code point is a digit, consume it and append it to repr. + while is_digit(self.peek_code_point(0)) { + append_code_point(&mut repr, self.next_code_point()); + } + + // 4. If the next 2 input code points are U+002E FULL STOP (.) followed by a digit, then: + let maybe_number = self.peek_twin(); + if is_full_stop(maybe_number.first) && is_digit(maybe_number.second) { + // 1. Consume them. + // 2. Append them to repr. + append_code_point(&mut repr, self.next_code_point()); + append_code_point(&mut repr, self.next_code_point()); + + // 3. Set type to "number". + number_type = CssNumberType::Number; + + // 4. While the next input code point is a digit, consume it and append it to repr. + while is_digit(self.peek_code_point(0)) { + append_code_point(&mut repr, self.next_code_point()); + } + } + + // 5. If the next 2 or 3 input code points are U+0045 LATIN CAPITAL LETTER E (E) or + // U+0065 LATIN SMALL LETTER E (e), optionally followed by U+002D HYPHEN-MINUS (-) + // or U+002B PLUS SIGN (+), followed by a digit, then: + let maybe_exponent = self.peek_triplet(); + if (is_e(maybe_exponent.first) || is_uppercase_e(maybe_exponent.first)) + && (((is_plus_sign(maybe_exponent.second) || is_hyphen_minus(maybe_exponent.second)) + && is_digit(maybe_exponent.third)) + || is_digit(maybe_exponent.second)) + { + // 1. Consume them. + // 2. Append them to repr. + if (is_plus_sign(maybe_exponent.second) || is_hyphen_minus(maybe_exponent.second)) + && is_digit(maybe_exponent.third) + { + append_code_point(&mut repr, self.next_code_point()); + append_code_point(&mut repr, self.next_code_point()); + append_code_point(&mut repr, self.next_code_point()); + } else if is_digit(maybe_exponent.second) { + append_code_point(&mut repr, self.next_code_point()); + append_code_point(&mut repr, self.next_code_point()); + } + + // 3. Set type to "number". + number_type = CssNumberType::Number; + + // 4. While the next input code point is a digit, consume it and append it to repr. + while is_digit(self.peek_code_point(0)) { + append_code_point(&mut repr, self.next_code_point()); + } + } + + // 6. Convert repr to a number, and set the value to the returned value. + let value = repr.parse::().unwrap(); + + // 7. Return value and type. + if number_type == CssNumberType::Integer && has_explicit_sign { + return NumericValue { + number_type: CssNumberType::IntegerWithExplicitSign, + value, + }; + } + + NumericValue { number_type, value } + } + + // https://www.w3.org/TR/css-syntax-3/#consume-name + fn consume_an_ident_sequence(&mut self) -> String { + // This section describes how to consume an ident sequence from a stream of code points. + // It returns a string containing the largest name that can be formed from adjacent + // code points in the stream, starting from the first. + // + // Note: This algorithm does not do the verification of the first few code points that + // are necessary to ensure the returned code points would constitute an . + // If that is the intended use, ensure that the stream starts with an ident sequence before + // calling this algorithm. + + // Let result initially be an empty string. + let mut result = String::new(); + + // Repeatedly consume the next input code point from the stream: + loop { + let input = self.next_code_point(); + + if is_eof(input) { + break; + } + + // name code point + if is_ident_code_point(input) { + // Append the code point to result. + append_code_point(&mut result, input); + continue; + } + + // the stream starts with a valid escape + if is_valid_escape_sequence(self.start_of_input_stream_twin()) { + // Consume an escaped code point. Append the returned code point to result. + append_code_point(&mut result, self.consume_escaped_code_point()); + continue; + } + + // anything else + // Reconsume the current input code point. Return result. + self.reconsume_current_input_code_point(); + break; + } + + result + } + + // https://www.w3.org/TR/css-syntax-3/#consume-url-token + fn consume_a_url_token(&mut self, start_byte_offset: usize) -> Token { + // This section describes how to consume a url token from a stream of code points. + // It returns either a or a . + // + // Note: This algorithm assumes that the initial "url(" has already been consumed. + // This algorithm also assumes that it’s being called to consume an "unquoted" value, + // like url(foo). A quoted value, like url("foo"), is parsed as a . + // Consume an ident-like token automatically handles this distinction; this algorithm + // shouldn’t be called directly otherwise. + + // 1. Initially create a with its value set to the empty string. + let mut value = String::new(); + + // 2. Consume as much whitespace as possible. + self.consume_as_much_whitespace_as_possible(); + + // 3. Repeatedly consume the next input code point from the stream: + loop { + let input = self.next_code_point(); + + // U+0029 RIGHT PARENTHESIS ()) + if is_right_paren(input) { + // Return the . + return Token::create_url(value, start_byte_offset, self.current_byte_offset()); + } + + // EOF + if is_eof(input) { + // This is a parse error. Return the . + return Token::create_url(value, start_byte_offset, self.current_byte_offset()); + } + + // whitespace + if is_whitespace(input) { + // Consume as much whitespace as possible. + self.consume_as_much_whitespace_as_possible(); + let next_input = self.peek_code_point(0); + + // If the next input code point is U+0029 RIGHT PARENTHESIS ()) or EOF, consume it + // and return the (if EOF was encountered, this is a parse error); + if is_right_paren(next_input) { + self.next_code_point(); + return Token::create_url(value, start_byte_offset, self.current_byte_offset()); + } + + if is_eof(next_input) { + self.next_code_point(); + return Token::create_url(value, start_byte_offset, self.current_byte_offset()); + } + + // otherwise, consume the remnants of a bad url, create a , and return it. + self.consume_the_remnants_of_a_bad_url(); + return Token::create(CssTokenType::BadUrl, start_byte_offset, self.current_byte_offset()); + } + + // U+0022 QUOTATION MARK (") + // U+0027 APOSTROPHE (') + // U+0028 LEFT PARENTHESIS (() + // non-printable code point + if is_quotation_mark(input) + || is_apostrophe(input) + || is_left_paren(input) + || is_non_printable_code_point(input) + { + // This is a parse error. Consume the remnants of a bad url, create a , and return it. + self.consume_the_remnants_of_a_bad_url(); + return Token::create(CssTokenType::BadUrl, start_byte_offset, self.current_byte_offset()); + } + + // U+005C REVERSE SOLIDUS (\) + if is_reverse_solidus(input) { + // If the stream starts with a valid escape, + if is_valid_escape_sequence(self.start_of_input_stream_twin()) { + // consume an escaped code point and append the returned code point to the ’s value. + append_code_point(&mut value, self.consume_escaped_code_point()); + continue; + } + + // Otherwise, this is a parse error. + // Consume the remnants of a bad url, create a , and return it. + self.consume_the_remnants_of_a_bad_url(); + return Token::create(CssTokenType::BadUrl, start_byte_offset, self.current_byte_offset()); + } + + // anything else + // Append the current input code point to the ’s value. + append_code_point(&mut value, input); + } + } + + // https://www.w3.org/TR/css-syntax-3/#consume-remnants-of-bad-url + fn consume_the_remnants_of_a_bad_url(&mut self) { + // This section describes how to consume the remnants of a bad url from a stream of code points, + // "cleaning up" after the tokenizer realizes that it’s in the middle of a rather + // than a . It returns nothing; its sole use is to consume enough of the input stream + // to reach a recovery point where normal tokenizing can resume. + + // Repeatedly consume the next input code point from the stream: + loop { + let input = self.next_code_point(); + + // U+0029 RIGHT PARENTHESIS ()) + // EOF + if is_eof(input) || is_right_paren(input) { + // Return. + return; + } + + // the input stream starts with a valid escape + if is_valid_escape_sequence(self.start_of_input_stream_twin()) { + // Consume an escaped code point. + // This allows an escaped right parenthesis ("\)") to be encountered without ending + // the . This is otherwise identical to the "anything else" clause. + self.consume_escaped_code_point(); + } + + // anything else + // Do nothing. + } + } + + // https://www.w3.org/TR/css-syntax-3/#consume-numeric-token + fn consume_a_numeric_token(&mut self) -> Token { + // This section describes how to consume a numeric token from a stream of code points. + // It returns either a , , or . + + let start_byte_offset = self.current_byte_offset(); + + // Consume a number and let number be the result. + let number = self.consume_a_number(); + + // If the next 3 input code points would start an ident sequence, then: + if would_start_an_ident_sequence(self.peek_triplet()) { + // 1. Create a with the same value and type flag as number, + // and a unit set initially to the empty string. + + // 2. Consume an ident sequence. Set the ’s unit to the returned value. + let unit = self.consume_an_ident_sequence(); + + // 3. Return the . + return Token::create_dimension(number, unit, start_byte_offset, self.current_byte_offset()); + } + + // Otherwise, if the next input code point is U+0025 PERCENTAGE SIGN (%), consume it. + if is_percent(self.peek_code_point(0)) { + self.next_code_point(); + + // Create a with the same value as number, and return it. + return Token::create_percentage(number, start_byte_offset, self.current_byte_offset()); + } + + // Otherwise, create a with the same value and type flag as number, and return it. + Token::create_number(number, start_byte_offset, self.current_byte_offset()) + } + + // https://www.w3.org/TR/css-syntax-3/#consume-string-token + fn consume_string_token(&mut self, ending_code_point: u32) -> Token { + // This section describes how to consume a string token from a stream of code points. + // It returns either a or . + // + // This algorithm may be called with an ending code point, which denotes the code point + // that ends the string. If an ending code point is not specified, the current input + // code point is used. + + // Initially create a with its value set to the empty string. + let start_byte_offset = self.current_byte_offset() - 1; + let mut value = String::new(); + + // Repeatedly consume the next input code point from the stream: + loop { + let input = self.next_code_point(); + + // ending code point + if input == ending_code_point { + // Return the . + return Token::create_string(value, start_byte_offset, self.current_byte_offset()); + } + + // EOF + if is_eof(input) { + // This is a parse error. Return the . + return Token::create_string(value, start_byte_offset, self.current_byte_offset()); + } + + // newline + if is_newline(input) { + // This is a parse error. Reconsume the current input code point, create a + // , and return it. + self.reconsume_current_input_code_point(); + return Token::create(CssTokenType::BadString, start_byte_offset, self.current_byte_offset()); + } + + // U+005C REVERSE SOLIDUS (\) + if is_reverse_solidus(input) { + // If the next input code point is EOF, do nothing. + let next_input = self.peek_code_point(0); + if is_eof(next_input) { + continue; + } + + // Otherwise, if the next input code point is a newline, consume it. + if is_newline(next_input) { + self.next_code_point(); + continue; + } + + // Otherwise, (the stream starts with a valid escape) consume an escaped code + // point and append the returned code point to the ’s value. + append_code_point(&mut value, self.consume_escaped_code_point()); + continue; + } + + // anything else + // Append the current input code point to the ’s value. + append_code_point(&mut value, input); + } + } + + // https://www.w3.org/TR/css-syntax-3/#consume-token + fn consume_a_token(&mut self) -> Token { + // This section describes how to consume a token from a stream of code points. + // It will return a single token of any type. + + let start_byte_offset = self.current_byte_offset(); + + // Consume comments. + self.consume_comments(); + + // AD-HOC: Preserve comments as whitespace tokens, for serializing custom properties. + let after_comments_byte_offset = self.current_byte_offset(); + if after_comments_byte_offset != start_byte_offset { + return Token::create_whitespace(start_byte_offset, self.current_byte_offset()); + } + + // Consume the next input code point. + let input = self.next_code_point(); + + // whitespace + if is_whitespace(input) { + // Consume as much whitespace as possible. Return a . + self.consume_as_much_whitespace_as_possible(); + return Token::create_whitespace(start_byte_offset, self.current_byte_offset()); + } + + // U+0022 QUOTATION MARK (") + if is_quotation_mark(input) { + // Consume a string token and return it. + return self.consume_string_token(input); + } + + // U+0023 NUMBER SIGN (#) + if is_number_sign(input) { + // If the next input code point is an ident code point or the next two input code points + // are a valid escape, then: + let next_input = self.peek_code_point(0); + let maybe_escape = self.peek_twin(); + + if is_ident_code_point(next_input) || is_valid_escape_sequence(maybe_escape) { + // 1. Create a . + let mut hash_type = CssHashType::Unrestricted; + + // 2. If the next 3 input code points would start an ident sequence, set the ’s + // type flag to "id". + if would_start_an_ident_sequence(self.peek_triplet()) { + hash_type = CssHashType::Id; + } + + // 3. Consume an ident sequence, and set the ’s value to the returned string. + let value = self.consume_an_ident_sequence(); + + // 4. Return the . + return Token::create_hash(value, hash_type, start_byte_offset, self.current_byte_offset()); + } + + // Otherwise, return a with its value set to the current input code point. + return Token::create_delim(input, start_byte_offset, self.current_byte_offset()); + } + + // U+0027 APOSTROPHE (') + if is_apostrophe(input) { + // Consume a string token and return it. + return self.consume_string_token(input); + } + + // U+0028 LEFT PARENTHESIS (() + if is_left_paren(input) { + // Return a <(-token>. + return Token::create(CssTokenType::OpenParen, start_byte_offset, self.current_byte_offset()); + } + + // U+0029 RIGHT PARENTHESIS ()) + if is_right_paren(input) { + // Return a <)-token>. + return Token::create(CssTokenType::CloseParen, start_byte_offset, self.current_byte_offset()); + } + + // U+002B PLUS SIGN (+) + if is_plus_sign(input) { + // If the input stream starts with a number, reconsume the current input code point, + // consume a numeric token and return it. + if would_start_a_number(self.start_of_input_stream_triplet()) { + self.reconsume_current_input_code_point(); + return self.consume_a_numeric_token(); + } + + // Otherwise, return a with its value set to the current input code point. + return Token::create_delim(input, start_byte_offset, self.current_byte_offset()); + } + + // U+002C COMMA (,) + if is_comma(input) { + // Return a . + return Token::create(CssTokenType::Comma, start_byte_offset, self.current_byte_offset()); + } + + // U+002D HYPHEN-MINUS (-) + if is_hyphen_minus(input) { + // If the input stream starts with a number, reconsume the current input code point, + // consume a numeric token, and return it. + if would_start_a_number(self.start_of_input_stream_triplet()) { + self.reconsume_current_input_code_point(); + return self.consume_a_numeric_token(); + } + + // Otherwise, if the next 2 input code points are U+002D HYPHEN-MINUS U+003E + // GREATER-THAN SIGN (->), consume them and return a . + let next_twin = self.peek_twin(); + if is_hyphen_minus(next_twin.first) && is_greater_than_sign(next_twin.second) { + self.next_code_point(); + self.next_code_point(); + return Token::create(CssTokenType::CDC, start_byte_offset, self.current_byte_offset()); + } + + // Otherwise, if the input stream starts with an identifier, reconsume the current + // input code point, consume an ident-like token, and return it. + if would_start_an_ident_sequence(self.start_of_input_stream_triplet()) { + self.reconsume_current_input_code_point(); + return self.consume_an_ident_like_token(); + } + + // Otherwise, return a with its value set to the current input code point. + return Token::create_delim(input, start_byte_offset, self.current_byte_offset()); + } + + // U+002E FULL STOP (.) + if is_full_stop(input) { + // If the input stream starts with a number, reconsume the current input code point, + // consume a numeric token, and return it. + if would_start_a_number(self.start_of_input_stream_triplet()) { + self.reconsume_current_input_code_point(); + return self.consume_a_numeric_token(); + } + + // Otherwise, return a with its value set to the current input code point. + return Token::create_delim(input, start_byte_offset, self.current_byte_offset()); + } + + // U+003A COLON (:) + if is_colon(input) { + // Return a . + return Token::create(CssTokenType::Colon, start_byte_offset, self.current_byte_offset()); + } + + // U+003B SEMICOLON (;) + if is_semicolon(input) { + // Return a . + return Token::create(CssTokenType::Semicolon, start_byte_offset, self.current_byte_offset()); + } + + // U+003C LESS-THAN SIGN (<) + if is_less_than_sign(input) { + // If the next 3 input code points are U+0021 EXCLAMATION MARK U+002D HYPHEN-MINUS + // U+002D HYPHEN-MINUS (!--), consume them and return a . + let maybe_cdo = self.peek_triplet(); + if is_exclamation_mark(maybe_cdo.first) + && is_hyphen_minus(maybe_cdo.second) + && is_hyphen_minus(maybe_cdo.third) + { + self.next_code_point(); + self.next_code_point(); + self.next_code_point(); + return Token::create(CssTokenType::CDO, start_byte_offset, self.current_byte_offset()); + } + + // Otherwise, return a with its value set to the current input code point. + return Token::create_delim(input, start_byte_offset, self.current_byte_offset()); + } + + // U+0040 COMMERCIAL AT (@) + if is_at(input) { + // If the next 3 input code points would start an ident sequence, consume an ident sequence, create + // an with its value set to the returned value, and return it. + if would_start_an_ident_sequence(self.peek_triplet()) { + return Token::create_at_keyword( + self.consume_an_ident_sequence(), + start_byte_offset, + self.current_byte_offset(), + ); + } + + // Otherwise, return a with its value set to the current input code point. + return Token::create_delim(input, start_byte_offset, self.current_byte_offset()); + } + + // U+005B LEFT SQUARE BRACKET ([) + if is_open_square_bracket(input) { + // Return a <[-token>. + return Token::create(CssTokenType::OpenSquare, start_byte_offset, self.current_byte_offset()); + } + + // U+005C REVERSE SOLIDUS (\) + if is_reverse_solidus(input) { + // If the input stream starts with a valid escape, reconsume the current input code point, + // consume an ident-like token, and return it. + if is_valid_escape_sequence(self.start_of_input_stream_twin()) { + self.reconsume_current_input_code_point(); + return self.consume_an_ident_like_token(); + } + + // Otherwise, this is a parse error. Return a with its value set to the + // current input code point. + return Token::create_delim(input, start_byte_offset, self.current_byte_offset()); + } + + // U+005D RIGHT SQUARE BRACKET (]) + if is_closed_square_bracket(input) { + // Return a <]-token>. + return Token::create(CssTokenType::CloseSquare, start_byte_offset, self.current_byte_offset()); + } + + // U+007B LEFT CURLY BRACKET ({) + if is_open_curly_bracket(input) { + // Return a <{-token>. + return Token::create(CssTokenType::OpenCurly, start_byte_offset, self.current_byte_offset()); + } + + // U+007D RIGHT CURLY BRACKET (}) + if is_closed_curly_bracket(input) { + // Return a <}-token>. + return Token::create(CssTokenType::CloseCurly, start_byte_offset, self.current_byte_offset()); + } + + // digit + if is_digit(input) { + // Reconsume the current input code point, consume a numeric token, and return it. + self.reconsume_current_input_code_point(); + return self.consume_a_numeric_token(); + } + + // name-start code point + if is_ident_start_code_point(input) { + // Reconsume the current input code point, consume an ident-like token, and return it. + self.reconsume_current_input_code_point(); + return self.consume_an_ident_like_token(); + } + + // EOF + if is_eof(input) { + // Return an . + return Token::create(CssTokenType::EndOfFile, start_byte_offset, self.current_byte_offset()); + } + + // anything else + // Return a with its value set to the current input code point. + Token::create_delim(input, start_byte_offset, self.current_byte_offset()) + } +} + +fn string_parts(string: &str) -> (*const u8, usize) { + bytes_parts(string.as_bytes()) +} + +fn bytes_parts(bytes: &[u8]) -> (*const u8, usize) { + if bytes.is_empty() { + (ptr::null(), 0) + } else { + (bytes.as_ptr(), bytes.len()) + } +} + +fn append_code_point(builder: &mut String, code_point: u32) { + builder.push(char::from_u32(code_point).unwrap_or(char::REPLACEMENT_CHARACTER)); +} + +fn is_eof(code_point: u32) -> bool { + code_point == TOKENIZER_EOF +} + +fn is_ascii(code_point: u32) -> bool { + code_point <= 0x7F +} + +fn is_ascii_alpha(code_point: u32) -> bool { + (0x41..=0x5A).contains(&code_point) || (0x61..=0x7A).contains(&code_point) +} + +fn is_unicode(code_point: u32) -> bool { + code_point <= 0x10FFFF +} + +fn is_unicode_surrogate(code_point: u32) -> bool { + (0xD800..=0xDFFF).contains(&code_point) +} + +fn is_digit(code_point: u32) -> bool { + (0x30..=0x39).contains(&code_point) +} + +fn is_hex_digit(code_point: u32) -> bool { + is_digit(code_point) || (0x41..=0x46).contains(&code_point) || (0x61..=0x66).contains(&code_point) +} + +fn is_ident_start_code_point(code_point: u32) -> bool { + is_ascii_alpha(code_point) || (!is_ascii(code_point) && is_unicode(code_point)) || code_point == '_' as u32 +} + +fn is_ident_code_point(code_point: u32) -> bool { + is_ident_start_code_point(code_point) || is_digit(code_point) || code_point == '-' as u32 +} + +fn is_non_printable_code_point(code_point: u32) -> bool { + code_point <= 0x8 || code_point == 0xB || (0xE..=0x1F).contains(&code_point) || code_point == 0x7F +} + +fn is_newline(code_point: u32) -> bool { + code_point == 0x0A +} + +fn is_whitespace(code_point: u32) -> bool { + is_newline(code_point) || code_point == '\t' as u32 || code_point == ' ' as u32 +} + +fn is_greater_than_maximum_allowed_code_point(code_point: u32) -> bool { + code_point > 0x10FFFF +} + +fn is_quotation_mark(code_point: u32) -> bool { + code_point == 0x22 +} + +fn is_hyphen_minus(code_point: u32) -> bool { + code_point == 0x2D +} + +fn is_number_sign(code_point: u32) -> bool { + code_point == 0x23 +} + +fn is_reverse_solidus(code_point: u32) -> bool { + code_point == 0x5C +} + +fn is_apostrophe(code_point: u32) -> bool { + code_point == 0x27 +} + +fn is_left_paren(code_point: u32) -> bool { + code_point == 0x28 +} + +fn is_right_paren(code_point: u32) -> bool { + code_point == 0x29 +} + +fn is_plus_sign(code_point: u32) -> bool { + code_point == 0x2B +} + +fn is_comma(code_point: u32) -> bool { + code_point == 0x2C +} + +fn is_full_stop(code_point: u32) -> bool { + code_point == 0x2E +} + +fn is_asterisk(code_point: u32) -> bool { + code_point == 0x2A +} + +fn is_solidus(code_point: u32) -> bool { + code_point == 0x2F +} + +fn is_colon(code_point: u32) -> bool { + code_point == 0x3A +} + +fn is_semicolon(code_point: u32) -> bool { + code_point == 0x3B +} + +fn is_less_than_sign(code_point: u32) -> bool { + code_point == 0x3C +} + +fn is_greater_than_sign(code_point: u32) -> bool { + code_point == 0x3E +} + +fn is_at(code_point: u32) -> bool { + code_point == 0x40 +} + +fn is_open_square_bracket(code_point: u32) -> bool { + code_point == 0x5B +} + +fn is_closed_square_bracket(code_point: u32) -> bool { + code_point == 0x5D +} + +fn is_open_curly_bracket(code_point: u32) -> bool { + code_point == 0x7B +} + +fn is_closed_curly_bracket(code_point: u32) -> bool { + code_point == 0x7D +} + +fn is_percent(code_point: u32) -> bool { + code_point == 0x25 +} + +fn is_exclamation_mark(code_point: u32) -> bool { + code_point == 0x21 +} + +fn is_e(code_point: u32) -> bool { + code_point == 0x65 +} + +fn is_uppercase_e(code_point: u32) -> bool { + code_point == 0x45 +} + +// https://www.w3.org/TR/css-syntax-3/#starts-with-a-valid-escape +fn is_valid_escape_sequence(values: U32Twin) -> bool { + // This section describes how to check if two code points are a valid escape. + // The algorithm described here can be called explicitly with two code points, + // or can be called with the input stream itself. In the latter case, the two + // code points in question are the current input code point and the next input + // code point, in that order. + // + // Note: This algorithm will not consume any additional code point. + + // If the first code point is not U+005C REVERSE SOLIDUS (\), return false. + if !is_reverse_solidus(values.first) { + return false; + } + + // Otherwise, if the second code point is a newline, return false. + if is_newline(values.second) { + return false; + } + + // Otherwise, return true. + true +} + +// https://www.w3.org/TR/css-syntax-3/#would-start-an-identifier +fn would_start_an_ident_sequence(values: U32Triplet) -> bool { + // This section describes how to check if three code points would start an ident sequence. + // The algorithm described here can be called explicitly with three code points, or + // can be called with the input stream itself. In the latter case, the three code + // points in question are the current input code point and the next two input code + // points, in that order. + // + // Note: This algorithm will not consume any additional code points. + + // Look at the first code point: + + // U+002D HYPHEN-MINUS + if is_hyphen_minus(values.first) { + // If the second code point is a name-start code point or a U+002D HYPHEN-MINUS, + // or the second and third code points are a valid escape, return true. + if is_ident_start_code_point(values.second) + || is_hyphen_minus(values.second) + || is_valid_escape_sequence(values.to_twin_23()) + { + return true; + } + // Otherwise, return false. + return false; + } + + // name-start code point + if is_ident_start_code_point(values.first) { + // Return true. + return true; + } + + // U+005C REVERSE SOLIDUS (\) + if is_reverse_solidus(values.first) { + // If the first and second code points are a valid escape, return true. + if is_valid_escape_sequence(values.to_twin_12()) { + return true; + } + // Otherwise, return false. + return false; + } + + // anything else + // Return false. + false +} + +// https://www.w3.org/TR/css-syntax-3/#starts-with-a-number +fn would_start_a_number(values: U32Triplet) -> bool { + // This section describes how to check if three code points would start a number. + // The algorithm described here can be called explicitly with three code points, + // or can be called with the input stream itself. In the latter case, the three + // code points in question are the current input code point and the next two input + // code points, in that order. + // + // Note: This algorithm will not consume any additional code points. + + // Look at the first code point: + + // U+002B PLUS SIGN (+) + // U+002D HYPHEN-MINUS (-) + if is_plus_sign(values.first) || is_hyphen_minus(values.first) { + // If the second code point is a digit, return true. + if is_digit(values.second) { + return true; + } + + // Otherwise, if the second code point is a U+002E FULL STOP (.) and the third + // code point is a digit, return true. + if is_full_stop(values.second) && is_digit(values.third) { + return true; + } + + // Otherwise, return false. + return false; + } + + // U+002E FULL STOP (.) + if is_full_stop(values.first) { + // If the second code point is a digit, return true. Otherwise, return false. + return is_digit(values.second); + } + + // digit + if is_digit(values.first) { + // Return true. + return true; + } + + // anything else + // Return false. + false +} diff --git a/Libraries/LibWeb/Rust/src/lib.rs b/Libraries/LibWeb/Rust/src/lib.rs new file mode 100644 index 0000000000..78956682ee --- /dev/null +++ b/Libraries/LibWeb/Rust/src/lib.rs @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2026-present, the Ladybird developers. + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#[path = "../../../RustAllocator.rs"] +mod rust_allocator; + +mod css_tokenizer; + +use std::ffi::c_void; +use std::panic::{AssertUnwindSafe, catch_unwind}; + +pub use css_tokenizer::{CssHashType, CssNumberType, CssToken, CssTokenType}; + +fn abort_on_panic R, R>(f: F) -> R { + match catch_unwind(AssertUnwindSafe(f)) { + Ok(result) => result, + Err(payload) => { + let message = if let Some(message) = payload.downcast_ref::<&str>() { + (*message).to_string() + } else if let Some(message) = payload.downcast_ref::() { + message.clone() + } else { + "unknown panic".to_string() + }; + eprintln!("Rust panic at FFI boundary: {message}"); + std::process::abort(); + } + } +} + +unsafe fn bytes_from_raw<'a>(bytes: *const u8, len: usize) -> Option<&'a [u8]> { + unsafe { + if len == 0 { + return Some(&[]); + } + if bytes.is_null() { + eprintln!("bytes_from_raw: null pointer with non-zero length {len}"); + return None; + } + Some(std::slice::from_raw_parts(bytes, len)) + } +} + +/// # Safety +/// - `input` and `input_len` must point to a valid string +/// - `ctx` must be a valid pointer to a CallbackContext +/// - Parameters provided to `callback` must be valid pointers +#[unsafe(no_mangle)] +pub unsafe extern "C" fn rust_css_tokenize( + input: *const u8, + input_len: usize, + ctx: *mut c_void, + callback: unsafe extern "C" fn(ctx: *mut c_void, token: *const CssToken), +) { + unsafe { + abort_on_panic(|| { + let Some(input) = bytes_from_raw(input, input_len) else { + return; + }; + + let tokenization_result = css_tokenizer::tokenize(input); + for token in &tokenization_result.tokens { + let ffi_token = token.as_ffi(&tokenization_result.filtered_input); + callback(ctx, &raw const ffi_token); + } + }); + } +} diff --git a/Tests/LibWeb/css-tokenizer.cpp b/Tests/LibWeb/css-tokenizer.cpp index 6e804b8859..e324e7880e 100644 --- a/Tests/LibWeb/css-tokenizer.cpp +++ b/Tests/LibWeb/css-tokenizer.cpp @@ -8,21 +8,31 @@ #include #include #include +#include #include ErrorOr ladybird_main(Main::Arguments arguments) { + StringView backend = "cpp"sv; StringView encoding = "utf-8"sv; StringView input_path; Core::ArgsParser args_parser; + args_parser.add_option(backend, "Tokenizer backend to use (cpp or rust)", "backend", 'b', "backend"); args_parser.add_option(encoding, "Source encoding label", "encoding", 'e', "encoding"); args_parser.add_positional_argument(input_path, "Path to the CSS input file", "input", Core::ArgsParser::Required::Yes); args_parser.parse(arguments); + if (backend != "cpp"sv && backend != "rust"sv) { + warnln("Unknown backend '{}'. Expected 'cpp' or 'rust'.", backend); + return 1; + } + auto file = TRY(Core::File::open(input_path, Core::File::OpenMode::Read)); auto input = TRY(file->read_until_eof()); - auto tokens = Web::CSS::Parser::Tokenizer::tokenize(input, encoding); + auto tokens = backend == "rust"sv + ? Web::CSS::Parser::RustTokenizer::tokenize(input, encoding) + : Web::CSS::Parser::Tokenizer::tokenize(input, encoding); for (auto const& token : tokens) outln("{}", token.to_debug_string()); diff --git a/Tests/LibWeb/test-css-tokenizer.py b/Tests/LibWeb/test-css-tokenizer.py index 229eafc570..3b10199ecd 100755 --- a/Tests/LibWeb/test-css-tokenizer.py +++ b/Tests/LibWeb/test-css-tokenizer.py @@ -47,6 +47,10 @@ def diff(a: str, a_file: Path, b: str, b_file: Path) -> None: print(f"{color_prefix}{line}\x1b[0m") +def output_file_for(file: Path, backend: str) -> Path: + return CSS_TOKENIZER_TEST_DIR / "output" / file.with_suffix(f".{backend}.txt") + + def encoding_for(file: Path) -> str: encoding_file = CSS_TOKENIZER_TEST_DIR / "input" / Path(f"{file.name}.encoding") if not encoding_file.exists(): @@ -54,11 +58,13 @@ def encoding_for(file: Path) -> str: return encoding_file.read_text(encoding="utf8").strip() -def test(file: Path, rebaseline: bool) -> bool: +def run_backend(file: Path, backend: str, encoding: str) -> tuple[str, Path]: args = [ str(BUILD_DIR / "bin/css-tokenizer"), + "--backend", + backend, "--encoding", - encoding_for(file), + encoding, str(CSS_TOKENIZER_TEST_DIR / "input" / file), ] process = subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) @@ -69,25 +75,55 @@ def test(file: Path, rebaseline: bool) -> bool: print(stdout) sys.exit(1) - expected_file = CSS_TOKENIZER_TEST_DIR / "expected" / file.with_suffix(".txt") - output_file = CSS_TOKENIZER_TEST_DIR / "output" / file.with_suffix(".txt") - + output_file = output_file_for(file, backend) output_file.write_text(stdout + "\n", encoding="utf8") + return stdout, output_file + + +def test(file: Path, backend: str, rebaseline: bool) -> bool: + requested_backends = ["cpp", "rust"] if backend == "both" else [backend] + encoding = encoding_for(file) + results = { + requested_backend: run_backend(file, requested_backend, encoding) for requested_backend in requested_backends + } + expected_file = CSS_TOKENIZER_TEST_DIR / "expected" / file.with_suffix(".txt") if rebaseline: - expected_file.write_text(stdout + "\n", encoding="utf8") + cpp_stdout = results["cpp"][0] if "cpp" in results else run_backend(file, "cpp", encoding)[0] + expected_file.write_text(cpp_stdout + "\n", encoding="utf8") + + if "rust" in results and results["rust"][0] != cpp_stdout: + print(f"\nRust tokenizer output does not match C++ for {file} after rebaseline!\n") + diff( + a=cpp_stdout, + a_file=output_file_for(file, "cpp"), + b=results["rust"][0], + b_file=results["rust"][1], + ) + return True + return False expected = expected_file.read_text(encoding="utf8").strip() + failed = False - if stdout != expected: - print(f"\nCSS tokens do not match for {file}!\n") + for current_backend, (stdout, output_file) in results.items(): + if stdout != expected: + print(f"\nCSS tokens do not match for {file} with backend '{current_backend}'!\n") + diff(a=expected, a_file=expected_file, b=stdout, b_file=output_file) + failed = True - diff(a=expected, a_file=expected_file, b=stdout, b_file=output_file) + if "cpp" in results and "rust" in results and results["cpp"][0] != results["rust"][0]: + print(f"\nBackends disagree for {file}!\n") + diff( + a=results["cpp"][0], + a_file=results["cpp"][1], + b=results["rust"][0], + b_file=results["rust"][1], + ) + failed = True - return True - - return False + return failed def main() -> int: @@ -95,6 +131,7 @@ def main() -> int: parser = ArgumentParser() parser.add_argument("-j", "--jobs", type=int) + parser.add_argument("--backend", choices=("cpp", "rust", "both"), default="both") parser.add_argument("--rebaseline", action="store_true") args = parser.parse_args() @@ -108,7 +145,8 @@ def main() -> int: with ThreadPoolExecutor(max_workers=args.jobs) as executor: executables = [ - executor.submit(test, css_file.relative_to(input_dir), args.rebaseline) for css_file in css_files + executor.submit(test, css_file.relative_to(input_dir), args.backend, args.rebaseline) + for css_file in css_files ] for executable in as_completed(executables):