f4960d9d7d
Keep decoded CSS text separate from tokenizer byte input. CSSOM and already-decoded stylesheet text preserve code point preprocessing, so a lone surrogate maps to one replacement character instead of being re-decoded as malformed UTF-8 bytes. Decode tokenizer byte input with the requested encoding unless that encoding is UTF-8 and the byte stream is strictly valid UTF-8. Keep the fast path by constructing the decoded string without validating twice after strict validation succeeds. Preserve UTF-8 decoder behavior on the byte fast path by stripping an initial UTF-8 BOM and rejecting encoded surrogate bytes. Invalid UTF-8 still goes through the decoder. Add tokenizer coverage for both the C++ and Rust backends across decoded text, UTF-8 aliases, BOM-prefixed input, invalid UTF-8, and non-UTF requested encodings.
45 lines
1.6 KiB
C++
45 lines
1.6 KiB
C++
/*
|
|
* Copyright (c) 2026, Sam Atkins <sam@ladybird.org>
|
|
*
|
|
* SPDX-License-Identifier: BSD-2-Clause
|
|
*/
|
|
|
|
#include <AK/Format.h>
|
|
#include <LibCore/ArgsParser.h>
|
|
#include <LibCore/File.h>
|
|
#include <LibMain/Main.h>
|
|
#include <LibWeb/CSS/Parser/RustTokenizer.h>
|
|
#include <LibWeb/CSS/Parser/Tokenizer.h>
|
|
|
|
ErrorOr<int> ladybird_main(Main::Arguments arguments)
|
|
{
|
|
StringView backend = "cpp"sv;
|
|
StringView encoding = "utf-8"sv;
|
|
StringView input_path;
|
|
bool silent = false;
|
|
|
|
Core::ArgsParser args_parser;
|
|
args_parser.add_option(backend, "Tokenizer backend to use (cpp or rust)", "backend", 'b', "backend");
|
|
args_parser.add_option(encoding, "Source encoding label", "encoding", 'e', "encoding");
|
|
args_parser.add_option(silent, "Don't print the tokens (useful for perf testing)", "silent", 's');
|
|
args_parser.add_positional_argument(input_path, "Path to the CSS input file", "input", Core::ArgsParser::Required::Yes);
|
|
args_parser.parse(arguments);
|
|
|
|
if (backend != "cpp"sv && backend != "rust"sv) {
|
|
warnln("Unknown backend '{}'. Expected 'cpp' or 'rust'.", backend);
|
|
return 1;
|
|
}
|
|
|
|
auto file = TRY(Core::File::open(input_path, Core::File::OpenMode::Read));
|
|
auto input = TRY(file->read_until_eof());
|
|
auto tokens = backend == "rust"sv
|
|
? Web::CSS::Parser::RustTokenizer::tokenize(input, encoding, Web::CSS::Parser::TokenizerInput::EncodedBytes)
|
|
: Web::CSS::Parser::Tokenizer::tokenize(input, encoding, Web::CSS::Parser::TokenizerInput::EncodedBytes);
|
|
|
|
if (!silent) {
|
|
for (auto const& token : tokens)
|
|
outln("{}", token.to_debug_string());
|
|
}
|
|
|
|
return 0;
|
|
}
|