dmaax/ladybird
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

LibWeb/CSS: Port the CSS Tokenizer to Rust

Browse source 
test-css-tokenizer is updated to run both the C++ and Rust tokenizers
and compare their output, to ensure they behave identically. The Parser
still uses the C++ Tokenizer.

The LibWeb crate, FFI layer etc are all based on the existing ones for
other libraries.

This is a direct AI translation to get us started, and not idiomatic
Rust. Future work can be done to make it more sensible.
This commit is contained in:
Sam Atkins 2026-04-15 11:53:15 +01:00 committed by Andreas Kling
parent 8b09a8e568
commit 4278194d96
16 changed files with 1997 additions and 15 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
Cargo.lock generated
View file

@ -405,6 +405,13 @@ dependencies = [
"icu_calendar",
]
[[package]]
name = "libweb_rust"
version = "0.1.0"
dependencies = [
"cbindgen",
]
[[package]]
name = "linux-raw-sys"
version = "0.12.1"

1
Cargo.toml
View file

@ -4,6 +4,7 @@ members = [
"Libraries/LibJS/Rust",
"Libraries/LibRegex/Rust",
"Libraries/LibUnicode/Rust",
"Libraries/LibWeb/Rust",
]
exclude = [
"Libraries/LibJS/AsmIntGen",

19
Libraries/LibWeb/CMakeLists.txt
View file

@ -213,6 +213,7 @@ set(SOURCES
CSS/Parser/Helpers.cpp
CSS/Parser/MediaParsing.cpp
CSS/Parser/Parser.cpp
CSS/Parser/RustTokenizer.cpp
CSS/Parser/PropertyParsing.cpp
CSS/Parser/RuleContext.cpp
CSS/Parser/RuleParsing.cpp
@ -1250,6 +1251,24 @@ ladybird_lib(LibWeb web EXPLICIT_SYMBOL_EXPORT)
target_link_libraries(LibWeb PRIVATE LibCore LibCompress LibCrypto LibJS LibHTTP LibGfx LibIPC LibRegex LibSyntax LibTextCodec LibUnicode LibMedia LibWasm LibXML LibIDL LibURL LibTLS LibRequests LibGC LibThreading skia ${ANGLE_TARGETS} SDL3::SDL3 LibXml2::LibXml2)
import_rust_crate(MANIFEST_PATH Rust/Cargo.toml CRATE_NAME libweb_rust FFI_HEADER RustFFI.h)
target_link_libraries(LibWeb PRIVATE libweb_rust)
if ((LINUX OR BSD) AND NOT BUILD_SHARED_LIBS)
target_link_options(LibWeb INTERFACE LINKER:--allow-multiple-definition)
endif()
if(NOT BUILD_SHARED_LIBS)
add_custom_command(TARGET LibWeb POST_BUILD
COMMAND ${CMAKE_AR} -x $<TARGET_FILE:libweb_rust>
COMMAND ${CMAKE_AR} -qS $<TARGET_FILE:LibWeb> *.o
COMMAND ${CMAKE_RANLIB} $<TARGET_FILE:LibWeb>
COMMAND ${CMAKE_COMMAND} -E remove -f *.o
WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/rust_merge_tmp
COMMENT "Merging Rust archive into LibWeb"
)
file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/rust_merge_tmp)
endif()
if (HAS_FONTCONFIG)
target_link_libraries(LibWeb PRIVATE Fontconfig::Fontconfig)
endif()

255
Libraries/LibWeb/CSS/Parser/RustTokenizer.cpp Normal file
View file

@ -0,0 +1,255 @@
/*
* Copyright (c) 2026, the Ladybird developers.
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#include <AK/StringBuilder.h>
#include <LibTextCodec/Decoder.h>
#include <LibWeb/CSS/CharacterTypes.h>
#include <LibWeb/CSS/Number.h>
#include <LibWeb/CSS/Parser/RustTokenizer.h>
#include <LibWeb/RustFFI.h>
namespace Web::CSS::Parser {
// U+FFFD REPLACEMENT CHARACTER (<28>)
static constexpr u32 REPLACEMENT_CHARACTER = 0xFFFD;
static String decode_and_filter_code_points(StringView input, StringView encoding)
{
// https://www.w3.org/TR/css-syntax-3/#css-filter-code-points
auto decoder = TextCodec::decoder_for(encoding);
VERIFY(decoder.has_value());
auto decoded_input = MUST(decoder->to_utf8(input));
// OPTIMIZATION: If the input doesn't contain any filterable characters, we can skip the filtering
bool const contains_filterable = [&] {
for (auto code_point : decoded_input.code_points()) {
if (code_point == '\r' || code_point == '\f' || code_point == 0x00 || is_unicode_surrogate(code_point))
return true;
}
return false;
}();
if (!contains_filterable)
return decoded_input;
StringBuilder builder { input.length() };
bool last_was_carriage_return = false;
// To filter code points from a stream of (unfiltered) code points input:
for (auto code_point : decoded_input.code_points()) {
// Replace any U+000D CARRIAGE RETURN (CR) code points,
// U+000C FORM FEED (FF) code points,
// or pairs of U+000D CARRIAGE RETURN (CR) followed by U+000A LINE FEED (LF)
// in input by a single U+000A LINE FEED (LF) code point.
if (code_point == '\r') {
if (last_was_carriage_return) {
builder.append('\n');
} else {
last_was_carriage_return = true;
}
} else {
if (last_was_carriage_return)
builder.append('\n');
if (code_point == '\n') {
if (!last_was_carriage_return)
builder.append('\n');
} else if (code_point == '\f') {
builder.append('\n');
// Replace any U+0000 NULL or surrogate code points in input with U+FFFD REPLACEMENT CHARACTER (<28>).
} else if (code_point == 0x00 || is_unicode_surrogate(code_point)) {
builder.append_code_point(REPLACEMENT_CHARACTER);
} else {
builder.append_code_point(code_point);
}
last_was_carriage_return = false;
}
}
return builder.to_string_without_validation();
}
static String string_from_ffi_bytes(u8 const* bytes, size_t length)
{
if (length == 0)
return {};
return String::from_utf8_without_validation({ bytes, length });
}
static FlyString fly_string_from_ffi_bytes(u8 const* bytes, size_t length)
{
if (length == 0)
return {};
return FlyString::from_utf8_without_validation({ bytes, length });
}
static Number::Type css_number_type_from_ffi(FFI::CssNumberType number_type)
{
switch (number_type) {
case FFI::CssNumberType::Number:
return Number::Type::Number;
case FFI::CssNumberType::IntegerWithExplicitSign:
return Number::Type::IntegerWithExplicitSign;
case FFI::CssNumberType::Integer:
return Number::Type::Integer;
}
VERIFY_NOT_REACHED();
}
static Token::Position position_from_ffi(size_t line, size_t column)
{
return { line, column };
}
Token RustTokenizer::token_from_ffi(FFI::CssToken const& ffi_token)
{
auto original_source_text = string_from_ffi_bytes(ffi_token.original_source_ptr, ffi_token.original_source_len);
auto payload = fly_string_from_ffi_bytes(ffi_token.value_ptr, ffi_token.value_len);
Token token;
switch (ffi_token.token_type) {
case FFI::CssTokenType::Invalid:
VERIFY_NOT_REACHED();
case FFI::CssTokenType::EndOfFile:
token = Token::create(Token::Type::EndOfFile, move(original_source_text));
break;
case FFI::CssTokenType::Ident:
token = Token::create_ident(move(payload), move(original_source_text));
break;
case FFI::CssTokenType::Function:
token = Token::create_function(move(payload), move(original_source_text));
break;
case FFI::CssTokenType::AtKeyword:
token = Token::create_at_keyword(move(payload), move(original_source_text));
break;
case FFI::CssTokenType::Hash:
token = Token::create_hash(
move(payload),
ffi_token.hash_type == FFI::CssHashType::Id ? Token::HashType::Id : Token::HashType::Unrestricted,
move(original_source_text));
break;
case FFI::CssTokenType::String:
token = Token::create_string(move(payload), move(original_source_text));
break;
case FFI::CssTokenType::BadString:
token = Token::create(Token::Type::BadString, move(original_source_text));
break;
case FFI::CssTokenType::Url:
token = Token::create_url(move(payload), move(original_source_text));
break;
case FFI::CssTokenType::BadUrl:
token = Token::create(Token::Type::BadUrl, move(original_source_text));
break;
case FFI::CssTokenType::Delim:
token = Token::create_delim(ffi_token.delim, move(original_source_text));
break;
case FFI::CssTokenType::Number:
token = Token::create_number(Number { css_number_type_from_ffi(ffi_token.number_type), ffi_token.number_value }, move(original_source_text));
break;
case FFI::CssTokenType::Percentage:
token = Token::create_percentage(Number { css_number_type_from_ffi(ffi_token.number_type), ffi_token.number_value }, move(original_source_text));
break;
case FFI::CssTokenType::Dimension:
token = Token::create_dimension(Number { css_number_type_from_ffi(ffi_token.number_type), ffi_token.number_value }, move(payload), move(original_source_text));
break;
case FFI::CssTokenType::Whitespace:
token = Token::create_whitespace(move(original_source_text));
break;
case FFI::CssTokenType::CDO:
token = Token::create(Token::Type::CDO, move(original_source_text));
break;
case FFI::CssTokenType::CDC:
token = Token::create(Token::Type::CDC, move(original_source_text));
break;
case FFI::CssTokenType::Colon:
token = Token::create(Token::Type::Colon, move(original_source_text));
break;
case FFI::CssTokenType::Semicolon:
token = Token::create(Token::Type::Semicolon, move(original_source_text));
break;
case FFI::CssTokenType::Comma:
token = Token::create(Token::Type::Comma, move(original_source_text));
break;
case FFI::CssTokenType::OpenSquare:
token = Token::create(Token::Type::OpenSquare, move(original_source_text));
break;
case FFI::CssTokenType::CloseSquare:
token = Token::create(Token::Type::CloseSquare, move(original_source_text));
break;
case FFI::CssTokenType::OpenParen:
token = Token::create(Token::Type::OpenParen, move(original_source_text));
break;
case FFI::CssTokenType::CloseParen:
token = Token::create(Token::Type::CloseParen, move(original_source_text));
break;
case FFI::CssTokenType::OpenCurly:
token = Token::create(Token::Type::OpenCurly, move(original_source_text));
break;
case FFI::CssTokenType::CloseCurly:
token = Token::create(Token::Type::CloseCurly, move(original_source_text));
break;
}
token.set_position_range(Badge<RustTokenizer> {}, position_from_ffi(ffi_token.start_line, ffi_token.start_column), position_from_ffi(ffi_token.end_line, ffi_token.end_column));
return token;
}
static_assert(static_cast<u8>(FFI::CssTokenType::Invalid) == static_cast<u8>(Token::Type::Invalid));
static_assert(static_cast<u8>(FFI::CssTokenType::EndOfFile) == static_cast<u8>(Token::Type::EndOfFile));
static_assert(static_cast<u8>(FFI::CssTokenType::Ident) == static_cast<u8>(Token::Type::Ident));
static_assert(static_cast<u8>(FFI::CssTokenType::Function) == static_cast<u8>(Token::Type::Function));
static_assert(static_cast<u8>(FFI::CssTokenType::AtKeyword) == static_cast<u8>(Token::Type::AtKeyword));
static_assert(static_cast<u8>(FFI::CssTokenType::Hash) == static_cast<u8>(Token::Type::Hash));
static_assert(static_cast<u8>(FFI::CssTokenType::String) == static_cast<u8>(Token::Type::String));
static_assert(static_cast<u8>(FFI::CssTokenType::BadString) == static_cast<u8>(Token::Type::BadString));
static_assert(static_cast<u8>(FFI::CssTokenType::Url) == static_cast<u8>(Token::Type::Url));
static_assert(static_cast<u8>(FFI::CssTokenType::BadUrl) == static_cast<u8>(Token::Type::BadUrl));
static_assert(static_cast<u8>(FFI::CssTokenType::Delim) == static_cast<u8>(Token::Type::Delim));
static_assert(static_cast<u8>(FFI::CssTokenType::Number) == static_cast<u8>(Token::Type::Number));
static_assert(static_cast<u8>(FFI::CssTokenType::Percentage) == static_cast<u8>(Token::Type::Percentage));
static_assert(static_cast<u8>(FFI::CssTokenType::Dimension) == static_cast<u8>(Token::Type::Dimension));
static_assert(static_cast<u8>(FFI::CssTokenType::Whitespace) == static_cast<u8>(Token::Type::Whitespace));
static_assert(static_cast<u8>(FFI::CssTokenType::CDO) == static_cast<u8>(Token::Type::CDO));
static_assert(static_cast<u8>(FFI::CssTokenType::CDC) == static_cast<u8>(Token::Type::CDC));
static_assert(static_cast<u8>(FFI::CssTokenType::Colon) == static_cast<u8>(Token::Type::Colon));
static_assert(static_cast<u8>(FFI::CssTokenType::Semicolon) == static_cast<u8>(Token::Type::Semicolon));
static_assert(static_cast<u8>(FFI::CssTokenType::Comma) == static_cast<u8>(Token::Type::Comma));
static_assert(static_cast<u8>(FFI::CssTokenType::OpenSquare) == static_cast<u8>(Token::Type::OpenSquare));
static_assert(static_cast<u8>(FFI::CssTokenType::CloseSquare) == static_cast<u8>(Token::Type::CloseSquare));
static_assert(static_cast<u8>(FFI::CssTokenType::OpenParen) == static_cast<u8>(Token::Type::OpenParen));
static_assert(static_cast<u8>(FFI::CssTokenType::CloseParen) == static_cast<u8>(Token::Type::CloseParen));
static_assert(static_cast<u8>(FFI::CssTokenType::OpenCurly) == static_cast<u8>(Token::Type::OpenCurly));
static_assert(static_cast<u8>(FFI::CssTokenType::CloseCurly) == static_cast<u8>(Token::Type::CloseCurly));
static_assert(static_cast<u8>(FFI::CssHashType::Id) == static_cast<u8>(Token::HashType::Id));
static_assert(static_cast<u8>(FFI::CssHashType::Unrestricted) == static_cast<u8>(Token::HashType::Unrestricted));
static_assert(static_cast<u8>(FFI::CssNumberType::Number) == static_cast<u8>(Number::Type::Number));
static_assert(static_cast<u8>(FFI::CssNumberType::IntegerWithExplicitSign) == static_cast<u8>(Number::Type::IntegerWithExplicitSign));
static_assert(static_cast<u8>(FFI::CssNumberType::Integer) == static_cast<u8>(Number::Type::Integer));
Vector<Token> RustTokenizer::tokenize(StringView input, StringView encoding)
{
struct CallbackContext {
Vector<Token> tokens;
};
auto filtered_input = decode_and_filter_code_points(input, encoding);
auto filtered_input_bytes = filtered_input.bytes();
CallbackContext context;
FFI::rust_css_tokenize(
filtered_input_bytes.data(),
filtered_input_bytes.size(),
&context,
[](void* raw_context, FFI::CssToken const* ffi_token) {
auto& context = *static_cast<CallbackContext*>(raw_context);
context.tokens.append(token_from_ffi(*ffi_token));
});
return move(context.tokens);
}
}

30
Libraries/LibWeb/CSS/Parser/RustTokenizer.h Normal file
View file

@ -0,0 +1,30 @@
/*
* Copyright (c) 2026, the Ladybird developers.
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#pragma once
#include <AK/StringView.h>
#include <AK/Vector.h>
#include <LibWeb/CSS/Parser/Token.h>
#include <LibWeb/Export.h>
namespace Web::CSS::Parser::FFI {
struct CssToken;
}
namespace Web::CSS::Parser {
class WEB_API RustTokenizer {
public:
static Vector<Token> tokenize(StringView input, StringView encoding);
private:
static Token token_from_ffi(FFI::CssToken const&);
};
}

6
Libraries/LibWeb/CSS/Parser/Token.cpp
View file

@ -424,4 +424,10 @@ void Token::set_position_range(Badge<Tokenizer>, Position start, Position end)
m_end_position = end;
}
void Token::set_position_range(Badge<RustTokenizer>, Position start, Position end)
{
m_start_position = start;
m_end_position = end;
}
}

1
Libraries/LibWeb/CSS/Parser/Token.h
View file

@ -187,6 +187,7 @@ public:
Position const& start_position() const { return m_start_position; }
Position const& end_position() const { return m_end_position; }
void set_position_range(Badge<Tokenizer>, Position start, Position end);
void set_position_range(Badge<RustTokenizer>, Position start, Position end);
bool operator==(Token const& other) const
{

2
Libraries/LibWeb/CSS/Parser/Tokenizer.cpp
View file

@ -231,7 +231,7 @@ Vector<Token> Tokenizer::tokenize()
for (;;) {
auto token_start = m_position;
auto token = consume_a_token();
token.set_position_range({}, token_start, m_position);
token.set_position_range(Badge<Tokenizer> {}, token_start, m_position);
tokens.append(token);
if (token.is(Token::Type::EndOfFile)) {

1
Libraries/LibWeb/Forward.h
View file

@ -508,6 +508,7 @@ namespace Web::CSS::Parser {
class ComponentValue;
class GuardedSubstitutionContexts;
class Parser;
class RustTokenizer;
class SyntaxNode;
class Token;
class Tokenizer;

17
Libraries/LibWeb/Rust/Cargo.toml Normal file
View file

@ -0,0 +1,17 @@
[package]
name = "libweb_rust"
version = "0.1.0"
edition = "2024"
[lib]
crate-type = ["staticlib"]
# After changing dependencies, regenerate the Flatpak sources:
# python3 Meta/CMake/flatpak/generate-cargo-sources.py
[dependencies]
[build-dependencies]
cbindgen = "0.29"
[lints]
workspace = true

40
Libraries/LibWeb/Rust/build.rs Normal file
View file

@ -0,0 +1,40 @@
/*
* Copyright (c) 2026-present, the Ladybird developers.
*
* SPDX-License-Identifier: BSD-2-Clause
*/
use std::env;
use std::error::Error;
use std::path::PathBuf;
fn main() -> Result<(), Box<dyn Error>> {
let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR")?);
let out_dir = PathBuf::from(env::var("OUT_DIR")?);
println!("cargo:rerun-if-changed=build.rs");
println!("cargo:rerun-if-changed=cbindgen.toml");
println!("cargo:rerun-if-env-changed=FFI_OUTPUT_DIR");
println!("cargo:rerun-if-changed=src");
let ffi_out_dir = env::var("FFI_OUTPUT_DIR")
.map(PathBuf::from)
.unwrap_or_else(|_| out_dir.clone());
cbindgen::generate(manifest_dir).map_or_else(
|error| match error {
cbindgen::Error::ParseSyntaxError { .. } => {}
other => panic!("{other:?}"),
},
|bindings| {
let header_path = out_dir.join("RustFFI.h");
bindings.write_to_file(&header_path);
if ffi_out_dir != out_dir {
bindings.write_to_file(ffi_out_dir.join("RustFFI.h"));
}
},
);
Ok(())
}

26
Libraries/LibWeb/Rust/cbindgen.toml Normal file
View file

@ -0,0 +1,26 @@
language = "C++"
header = """/*
* Copyright (c) 2026-present, the Ladybird developers.
*
* SPDX-License-Identifier: BSD-2-Clause
*/"""
pragma_once = true
include_version = true
namespaces = ["Web", "CSS", "Parser", "FFI"]
line_length = 120
tab_width = 4
no_includes = true
sys_includes = ["stdint.h", "stddef.h"]
usize_is_size_t = true
[parse]
parse_deps = false
[parse.expand]
all_features = false
[export]
include = ["CssHashType", "CssNumberType", "CssToken", "CssTokenType"]
[export.mangle]
rename_types = "PascalCase"

1460
Libraries/LibWeb/Rust/src/css_tokenizer.rs Normal file
View file

File diff suppressed because it is too large Load diff

71
Libraries/LibWeb/Rust/src/lib.rs Normal file
View file

@ -0,0 +1,71 @@
/*
* Copyright (c) 2026-present, the Ladybird developers.
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#[path = "../../../RustAllocator.rs"]
mod rust_allocator;
mod css_tokenizer;
use std::ffi::c_void;
use std::panic::{AssertUnwindSafe, catch_unwind};
pub use css_tokenizer::{CssHashType, CssNumberType, CssToken, CssTokenType};
fn abort_on_panic<F: FnOnce() -> R, R>(f: F) -> R {
match catch_unwind(AssertUnwindSafe(f)) {
Ok(result) => result,
Err(payload) => {
let message = if let Some(message) = payload.downcast_ref::<&str>() {
(*message).to_string()
} else if let Some(message) = payload.downcast_ref::<String>() {
message.clone()
} else {
"unknown panic".to_string()
};
eprintln!("Rust panic at FFI boundary: {message}");
std::process::abort();
}
}
}
unsafe fn bytes_from_raw<'a>(bytes: *const u8, len: usize) -> Option<&'a [u8]> {
unsafe {
if len == 0 {
return Some(&[]);
}
if bytes.is_null() {
eprintln!("bytes_from_raw: null pointer with non-zero length {len}");
return None;
}
Some(std::slice::from_raw_parts(bytes, len))
}
}
/// # Safety
/// - `input` and `input_len` must point to a valid string
/// - `ctx` must be a valid pointer to a CallbackContext
/// - Parameters provided to `callback` must be valid pointers
#[unsafe(no_mangle)]
pub unsafe extern "C" fn rust_css_tokenize(
input: *const u8,
input_len: usize,
ctx: *mut c_void,
callback: unsafe extern "C" fn(ctx: *mut c_void, token: *const CssToken),
) {
unsafe {
abort_on_panic(|| {
let Some(input) = bytes_from_raw(input, input_len) else {
return;
};
let tokenization_result = css_tokenizer::tokenize(input);
for token in &tokenization_result.tokens {
let ffi_token = token.as_ffi(&tokenization_result.filtered_input);
callback(ctx, &raw const ffi_token);
}
});
}
}

12
Tests/LibWeb/css-tokenizer.cpp
View file

@ -8,21 +8,31 @@
#include <LibCore/ArgsParser.h>
#include <LibCore/File.h>
#include <LibMain/Main.h>
#include <LibWeb/CSS/Parser/RustTokenizer.h>
#include <LibWeb/CSS/Parser/Tokenizer.h>
ErrorOr<int> ladybird_main(Main::Arguments arguments)
{
StringView backend = "cpp"sv;
StringView encoding = "utf-8"sv;
StringView input_path;
Core::ArgsParser args_parser;
args_parser.add_option(backend, "Tokenizer backend to use (cpp or rust)", "backend", 'b', "backend");
args_parser.add_option(encoding, "Source encoding label", "encoding", 'e', "encoding");
args_parser.add_positional_argument(input_path, "Path to the CSS input file", "input", Core::ArgsParser::Required::Yes);
args_parser.parse(arguments);
if (backend != "cpp"sv && backend != "rust"sv) {
warnln("Unknown backend '{}'. Expected 'cpp' or 'rust'.", backend);
return 1;
}
auto file = TRY(Core::File::open(input_path, Core::File::OpenMode::Read));
auto input = TRY(file->read_until_eof());
auto tokens = Web::CSS::Parser::Tokenizer::tokenize(input, encoding);
auto tokens = backend == "rust"sv
? Web::CSS::Parser::RustTokenizer::tokenize(input, encoding)
: Web::CSS::Parser::Tokenizer::tokenize(input, encoding);
for (auto const& token : tokens)
outln("{}", token.to_debug_string());

64
Tests/LibWeb/test-css-tokenizer.py
View file

@ -47,6 +47,10 @@ def diff(a: str, a_file: Path, b: str, b_file: Path) -> None:
print(f"{color_prefix}{line}\x1b[0m")
def output_file_for(file: Path, backend: str) -> Path:
return CSS_TOKENIZER_TEST_DIR / "output" / file.with_suffix(f".{backend}.txt")
def encoding_for(file: Path) -> str:
encoding_file = CSS_TOKENIZER_TEST_DIR / "input" / Path(f"{file.name}.encoding")
if not encoding_file.exists():
@ -54,11 +58,13 @@ def encoding_for(file: Path) -> str:
return encoding_file.read_text(encoding="utf8").strip()
def test(file: Path, rebaseline: bool) -> bool:
def run_backend(file: Path, backend: str, encoding: str) -> tuple[str, Path]:
args = [
str(BUILD_DIR / "bin/css-tokenizer"),
"--backend",
backend,
"--encoding",
encoding_for(file),
encoding,
str(CSS_TOKENIZER_TEST_DIR / "input" / file),
]
process = subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
@ -69,25 +75,55 @@ def test(file: Path, rebaseline: bool) -> bool:
print(stdout)
sys.exit(1)
expected_file = CSS_TOKENIZER_TEST_DIR / "expected" / file.with_suffix(".txt")
output_file = CSS_TOKENIZER_TEST_DIR / "output" / file.with_suffix(".txt")
output_file = output_file_for(file, backend)
output_file.write_text(stdout + "\n", encoding="utf8")
return stdout, output_file
def test(file: Path, backend: str, rebaseline: bool) -> bool:
requested_backends = ["cpp", "rust"] if backend == "both" else [backend]
encoding = encoding_for(file)
results = {
requested_backend: run_backend(file, requested_backend, encoding) for requested_backend in requested_backends
}
expected_file = CSS_TOKENIZER_TEST_DIR / "expected" / file.with_suffix(".txt")
if rebaseline:
expected_file.write_text(stdout + "\n", encoding="utf8")
cpp_stdout = results["cpp"][0] if "cpp" in results else run_backend(file, "cpp", encoding)[0]
expected_file.write_text(cpp_stdout + "\n", encoding="utf8")
if "rust" in results and results["rust"][0] != cpp_stdout:
print(f"\nRust tokenizer output does not match C++ for {file} after rebaseline!\n")
diff(
a=cpp_stdout,
a_file=output_file_for(file, "cpp"),
b=results["rust"][0],
b_file=results["rust"][1],
)
return True
return False
expected = expected_file.read_text(encoding="utf8").strip()
failed = False
if stdout != expected:
print(f"\nCSS tokens do not match for {file}!\n")
for current_backend, (stdout, output_file) in results.items():
if stdout != expected:
print(f"\nCSS tokens do not match for {file} with backend '{current_backend}'!\n")
diff(a=expected, a_file=expected_file, b=stdout, b_file=output_file)
failed = True
diff(a=expected, a_file=expected_file, b=stdout, b_file=output_file)
if "cpp" in results and "rust" in results and results["cpp"][0] != results["rust"][0]:
print(f"\nBackends disagree for {file}!\n")
diff(
a=results["cpp"][0],
a_file=results["cpp"][1],
b=results["rust"][0],
b_file=results["rust"][1],
)
failed = True
return True
return False
return failed
def main() -> int:
@ -95,6 +131,7 @@ def main() -> int:
parser = ArgumentParser()
parser.add_argument("-j", "--jobs", type=int)
parser.add_argument("--backend", choices=("cpp", "rust", "both"), default="both")
parser.add_argument("--rebaseline", action="store_true")
args = parser.parse_args()
@ -108,7 +145,8 @@ def main() -> int:
with ThreadPoolExecutor(max_workers=args.jobs) as executor:
executables = [
executor.submit(test, css_file.relative_to(input_dir), args.rebaseline) for css_file in css_files
executor.submit(test, css_file.relative_to(input_dir), args.backend, args.rebaseline)
for css_file in css_files
]
for executable in as_completed(executables):