dmaax/ladybird
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

LibWeb: Replace the HTML tokenizer with Rust

Browse source 
Replace the C++ HTML tokenizer with a Rust implementation behind the
existing HTMLTokenizer API.

Keep the parser-facing integration points for streaming input,
insertion points, document.write(), EOF insertion, parser aborts,
speculative parser input, and last start tag tracking. The generated
FFI handle stays an implementation detail of HTMLTokenizer, so callers
keep a single tokenizer class.

Preserve duplicate attributes through FFI so C++ token normalization can
record the duplicate-attribute signal used by CSP nonce checks. Keep
bulk tag-name and attribute scans capped at the active insertion point
so streamed parser input is spliced at the right offset.

Use generated DAFSA tables for named character references and intern
common tag and attribute names to reduce FFI marshalling overhead. This
also fixes attribute name source positions, nested old insertion points,
and aborted fast-path handling.

TestHTMLTokenizer covers duplicate attributes and insertion points in
fast tag-name, attribute-name, and quoted-value scans. A CSP text test
covers duplicate nonce attributes on parser-created script elements.
The tokenizer dump fixtures still match, TestHTMLTokenizer passes, and
the full release test-web run passes with 6981 tests and 226 skipped.
This commit is contained in:
Andreas Kling 2026-05-15 15:13:43 +02:00 committed by Alexander Kalenik
parent 54a172f3b5
commit 171e3adf01
18 changed files with 5465 additions and 3056 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
Cargo.lock generated
View file

@ -585,11 +585,19 @@ dependencies = [
"target-lexicon",
]
[[package]]
name = "libweb_html_tokenizer"
version = "0.1.0"
dependencies = [
"cbindgen",
]
[[package]]
name = "libweb_rust"
version = "0.1.0"
dependencies = [
"cbindgen",
"libweb_html_tokenizer",
]
[[package]]

1
Cargo.toml
View file

@ -6,6 +6,7 @@ members = [
"Libraries/LibUnicode/Rust",
"Libraries/LibWasm/Rust",
"Libraries/LibWeb/Rust",
"Libraries/LibWeb/HTML/Parser/Rust",
]
exclude = [
"Libraries/LibJS/AsmIntGen",

3
Libraries/LibWeb/CMakeLists.txt
View file

@ -1253,6 +1253,8 @@ target_link_libraries(LibWeb PRIVATE LibCore LibCompress LibCrypto LibJS LibHTTP
import_rust_crate(MANIFEST_PATH Rust/Cargo.toml CRATE_NAME libweb_rust FFI_HEADER RustFFI.h)
target_link_libraries(LibWeb PRIVATE libweb_rust)
get_target_property(_libweb_rust_lib libweb_rust IMPORTED_LOCATION)
set_property(SOURCE HTML/Parser/HTMLTokenizer.cpp APPEND PROPERTY OBJECT_DEPENDS ${_libweb_rust_lib})
if ((LINUX OR BSD) AND NOT BUILD_SHARED_LIBS)
target_link_options(LibWeb INTERFACE LINKER:--allow-multiple-definition)
endif()
@ -1273,7 +1275,6 @@ if(NOT BUILD_SHARED_LIBS)
# stale Rust objects. Force the C++ FFI bridge file to depend on the
# Rust archive: when it changes, RustTokenizer.cpp recompiles, LibWeb
# re-archives, and POST_BUILD re-merges the fresh Rust objects.
get_target_property(_libweb_rust_lib libweb_rust IMPORTED_LOCATION)
set_property(SOURCE CSS/Parser/RustTokenizer.cpp APPEND PROPERTY OBJECT_DEPENDS ${_libweb_rust_lib})
endif()

3197
Libraries/LibWeb/HTML/Parser/HTMLTokenizer.cpp
View file

File diff suppressed because it is too large Load diff

91
Libraries/LibWeb/HTML/Parser/HTMLTokenizer.h
View file

@ -7,18 +7,17 @@
#pragma once
#include <AK/Queue.h>
#include <AK/StringBuilder.h>
#include <AK/String.h>
#include <AK/StringView.h>
#include <AK/Types.h>
#include <AK/Vector.h>
#include <LibGC/Cell.h>
#include <LibGC/Ptr.h>
#include <LibWeb/Export.h>
#include <LibWeb/Forward.h>
#include <LibWeb/HTML/Parser/Entities.h>
#include <LibWeb/HTML/Parser/HTMLToken.h>
struct RustFfiTokenizerHandle;
namespace Web::HTML {
#define ENUMERATE_TOKENIZER_STATES \
@ -107,6 +106,7 @@ class WEB_API HTMLTokenizer {
public:
explicit HTMLTokenizer();
explicit HTMLTokenizer(StringView input, ByteString const& encoding);
~HTMLTokenizer();
enum class State {
#define __ENUMERATE_TOKENIZER_STATE(state) state,
@ -123,13 +123,10 @@ public:
void set_parser(Badge<HTMLParser>, HTMLParser& parser) { m_parser = &parser; }
void switch_to(Badge<HTMLParser>, State new_state);
void switch_to(State new_state)
{
m_state = new_state;
}
void switch_to(State new_state);
void set_blocked(bool b) { m_blocked = b; }
bool is_blocked() const { return m_blocked; }
void set_blocked(bool b);
bool is_blocked() const;
auto const& source() const { return m_source; }
@ -142,38 +139,23 @@ public:
void insert_eof();
bool is_eof_inserted();
bool is_insertion_point_defined() const { return m_insertion_point.has_value(); }
bool is_insertion_point_reached() { return m_insertion_point.has_value() && m_current_offset >= *m_insertion_point; }
void undefine_insertion_point() { m_insertion_point = {}; }
void store_old_insertion_point() { m_old_insertion_points.append(m_insertion_point); }
void restore_old_insertion_point() { m_insertion_point = m_old_insertion_points.take_last(); }
void update_insertion_point() { m_insertion_point = m_current_offset; }
bool is_insertion_point_defined() const;
bool is_insertion_point_reached();
void undefine_insertion_point();
void store_insertion_point();
void restore_insertion_point();
void store_old_insertion_point() { store_insertion_point(); }
void restore_old_insertion_point() { restore_insertion_point(); }
void update_insertion_point();
// This permanently cuts off the tokenizer input stream.
void abort() { m_aborted = true; }
void abort();
void parser_did_run(Badge<HTMLParser>);
void visit_edges(GC::Cell::Visitor&);
private:
void skip(size_t count);
Optional<u32> next_code_point(StopAtInsertionPoint);
Optional<u32> peek_code_point(ssize_t offset, StopAtInsertionPoint) const;
enum class ConsumeNextResult {
Consumed,
NotConsumed,
RanOutOfCharacters,
};
[[nodiscard]] ConsumeNextResult consume_next_if_match(StringView, StopAtInsertionPoint, CaseSensitivity = CaseSensitivity::CaseSensitive);
bool should_pause_before_next_input_character(StopAtInsertionPoint) const;
bool can_run_out_of_characters(StopAtInsertionPoint) const;
void create_new_token(HTMLToken::Type);
bool current_end_tag_token_is_appropriate() const;
String consume_current_builder();
static char const* state_name(State state)
{
switch (state) {
@ -186,52 +168,13 @@ private:
VERIFY_NOT_REACHED();
}
void will_emit(HTMLToken&);
void will_switch_to(State);
void will_reconsume_in(State);
bool consumed_as_part_of_an_attribute() const;
void restore_to(ssize_t new_iterator);
HTMLToken::Position nth_last_position(size_t n = 0);
GC::Ptr<HTMLParser> m_parser;
State m_state { State::Data };
State m_return_state { State::Data };
Vector<u32> m_temporary_buffer;
String m_source;
Vector<u32> m_decoded_input;
Optional<ssize_t> m_insertion_point;
// Spec algorithms have an "old insertion point" local; reentrant script execution can nest those locals.
Vector<Optional<ssize_t>> m_old_insertion_points;
ssize_t m_current_offset { 0 };
ssize_t m_prev_offset { 0 };
HTMLToken m_current_token;
StringBuilder m_current_builder;
NamedCharacterReferenceMatcher m_named_character_reference_matcher;
Optional<FlyString> m_last_emitted_start_tag_name;
bool m_explicit_eof_inserted { false };
bool m_input_stream_closed { false };
bool m_has_emitted_eof { false };
Queue<HTMLToken> m_queued_tokens;
u32 m_character_reference_code { 0 };
bool m_blocked { false };
bool m_aborted { false };
Vector<HTMLToken::Position> m_source_positions;
RustFfiTokenizerHandle* m_tokenizer { nullptr };
};
}

10
Libraries/LibWeb/HTML/Parser/Rust/Cargo.toml Normal file
View file

@ -0,0 +1,10 @@
[package]
name = "libweb_html_tokenizer"
version = "0.1.0"
edition = "2024"
[lib]
crate-type = ["rlib"]
[build-dependencies]
cbindgen = "0.29"

821
Libraries/LibWeb/HTML/Parser/Rust/build.rs Normal file
View file

@ -0,0 +1,821 @@
/*
* Copyright (c) 2026-present, the Ladybird developers.
*
* SPDX-License-Identifier: BSD-2-Clause
*/
//! Build script that generates a DAFSA (Deterministic Acyclic Finite State Automaton)
//! for named character reference matching. This is a Rust port of the C++ generator at
//! Meta/Lagom/Tools/CodeGenerators/LibWeb/GenerateNamedCharacterReferences.cpp.
use std::cell::RefCell;
use std::collections::HashMap;
use std::env;
use std::fs;
use std::path::{Path, PathBuf};
use std::rc::Rc;
const FFI_HEADER: &str = "HTMLTokenizerRustFFI.h";
fn main() {
let manifest_dir = env::var("CARGO_MANIFEST_DIR").unwrap();
let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap());
println!("cargo:rerun-if-changed=build.rs");
println!("cargo:rerun-if-changed=cbindgen.toml");
println!("cargo:rerun-if-changed=src");
println!("cargo:rerun-if-env-changed=FFI_OUTPUT_DIR");
let ffi_out_dir = env::var("FFI_OUTPUT_DIR")
.map(PathBuf::from)
.unwrap_or_else(|_| out_dir.clone());
cbindgen::generate(&manifest_dir).map_or_else(
|error| match error {
cbindgen::Error::ParseSyntaxError { .. } => {}
e => panic!("{e:?}"),
},
|bindings| {
bindings.write_to_file(out_dir.join(FFI_HEADER));
if ffi_out_dir != out_dir {
bindings.write_to_file(ffi_out_dir.join(FFI_HEADER));
}
},
);
// Generate interned name tables from the existing C++ headers.
let tag_names_header = Path::new(&manifest_dir).join("../../TagNames.h");
let attr_names_header = Path::new(&manifest_dir).join("../../AttributeNames.h");
println!("cargo:rerun-if-changed={}", tag_names_header.display());
println!("cargo:rerun-if-changed={}", attr_names_header.display());
let tag_names = parse_enumerate_macro(
&fs::read_to_string(&tag_names_header).expect("Failed to read TagNames.h"),
"__ENUMERATE_HTML_TAG",
);
let attr_names = parse_enumerate_macro(
&fs::read_to_string(&attr_names_header).expect("Failed to read AttributeNames.h"),
"__ENUMERATE_HTML_ATTRIBUTE",
);
emit_interned_names(&out_dir.join("interned_names_generated.rs"), &tag_names, &attr_names);
let json_path = Path::new(&manifest_dir).join("../Entities.json");
println!("cargo:rerun-if-changed={}", json_path.display());
let json_str = fs::read_to_string(&json_path).expect("Failed to read Entities.json");
let entities = parse_entities_json(&json_str);
// Build DAFSA.
let mut builder = DafsaBuilder::new();
for (name, _, _) in &entities {
builder.insert(name);
}
builder.minimize(0);
builder.calc_numbers();
// Verify minimal perfect hashing (no collisions).
let mut seen: Vec<bool> = vec![false; entities.len() + 1];
for (name, _, _) in &entities {
let idx = builder.get_unique_index(name).unwrap();
assert!(!seen[idx], "Hash collision at index {idx} for '{name}'");
seen[idx] = true;
}
// Build codepoints lookup table indexed by unique_index.
let mut index_to_codepoints = vec![(0u32, 0u32); entities.len()];
for (name, first, second) in &entities {
let idx = builder.get_unique_index(name).unwrap();
index_to_codepoints[idx - 1] = (*first, *second);
}
// Extract DAFSA layers.
let root = &builder.root;
let root_ref = root.borrow();
let mut first_layer: Vec<u16> = Vec::new();
let mut first_to_second_layer: Vec<(u64, u16)> = Vec::new();
let mut first_layer_tally: u16 = 0;
let mut second_layer_offset: u16 = 0;
for c in 0u8..128 {
if root_ref.children[c as usize].is_none() {
continue;
}
assert!(c.is_ascii_alphabetic());
let child = root_ref.children[c as usize].as_ref().unwrap();
let child_ref = child.borrow();
first_layer.push(first_layer_tally);
first_layer_tally += child_ref.number;
let mask = child_ref.get_ascii_alphabetic_bit_mask();
first_to_second_layer.push((mask, second_layer_offset));
second_layer_offset += child_ref.num_direct_children() as u16;
}
assert_eq!(first_layer.len(), 52);
// BFS to build DAFSA node array.
// Following the C++ write_node_data three-phase approach:
type NodePtr = Rc<RefCell<Node>>;
let mut queue: Vec<NodePtr> = Vec::new();
let mut child_indexes: HashMap<*const RefCell<Node>, u16> = HashMap::new();
// Phase 1: Queue root's children (first-layer nodes = 52 A-Z/a-z).
// This assigns temporary child_indexes for first-layer children.
queue_children(root, &mut queue, &mut child_indexes, 1);
// Phase 2: Clear indexes and re-process. For each first-layer child,
// queue ITS children (second-layer nodes) and assign their child_indexes.
child_indexes.clear();
let mut first_available_index: u16 = 1; // 0 is reserved (dummy node)
let first_layer_count = queue.len();
for i in 0..first_layer_count {
let node = Rc::clone(&queue[i]);
first_available_index = queue_children(&node, &mut queue, &mut child_indexes, first_available_index);
}
// Remove first-layer nodes from queue, keep only second-layer+ nodes.
let second_layer_nodes: Vec<NodePtr> = queue.drain(first_layer_count..).collect();
queue.clear();
queue.extend(second_layer_nodes);
// Phase 3: BFS remaining nodes, writing node data.
let mut node_data: Vec<NodeData> = Vec::new();
let mut qi = 0;
#[allow(unused_assignments)]
while qi < queue.len() {
let node = Rc::clone(&queue[qi]);
qi += 1;
first_available_index = write_children_data(
&node,
&mut node_data,
&mut queue,
&mut child_indexes,
first_available_index,
);
}
// Build second_layer entries with child_indexes from phase 2.
let mut second_layer: Vec<SecondLayerEntry> = Vec::new();
for c in 0u8..128 {
if root_ref.children[c as usize].is_none() {
continue;
}
let first_child = root_ref.children[c as usize].as_ref().unwrap();
let first_child_ref = first_child.borrow();
let mut tally: u8 = 0;
for cc in 0u8..128 {
if first_child_ref.children[cc as usize].is_none() {
continue;
}
let second_child = first_child_ref.children[cc as usize].as_ref().unwrap();
let second_child_ref = second_child.borrow();
let key = Rc::as_ptr(second_child);
let ci = child_indexes.get(&key).copied().unwrap_or(0);
let children_len = second_child_ref.num_direct_children();
second_layer.push(SecondLayerEntry {
child_index: ci,
number: tally,
children_len,
end_of_word: second_child_ref.is_terminal,
});
tally = tally.wrapping_add(second_child_ref.number as u8);
}
}
drop(root_ref);
// Generate output file.
let out_dir = env::var("OUT_DIR").unwrap();
let out_path = Path::new(&out_dir).join("named_character_references.rs");
let mut out = String::new();
out.push_str("// Auto-generated by build.rs -- do not edit!\n\n");
// Second codepoint enum.
out.push_str("#[derive(Clone, Copy, PartialEq, Eq)]\n");
out.push_str("#[repr(u8)]\n");
out.push_str("pub enum SecondCodepoint {\n");
out.push_str(" None = 0,\n");
out.push_str(" CombiningLongSolidusOverlay = 1,\n");
out.push_str(" CombiningLongVerticalLineOverlay = 2,\n");
out.push_str(" HairSpace = 3,\n");
out.push_str(" CombiningDoubleLowLine = 4,\n");
out.push_str(" CombiningReverseSolidusOverlay = 5,\n");
out.push_str(" VariationSelector1 = 6,\n");
out.push_str(" LatinSmallLetterJ = 7,\n");
out.push_str(" CombiningMacronBelow = 8,\n");
out.push_str("}\n\n");
out.push_str("impl SecondCodepoint {\n");
out.push_str(" pub fn value(self) -> u32 {\n");
out.push_str(" match self {\n");
out.push_str(" SecondCodepoint::None => 0,\n");
out.push_str(" SecondCodepoint::CombiningLongSolidusOverlay => 0x0338,\n");
out.push_str(" SecondCodepoint::CombiningLongVerticalLineOverlay => 0x20D2,\n");
out.push_str(" SecondCodepoint::HairSpace => 0x200A,\n");
out.push_str(" SecondCodepoint::CombiningDoubleLowLine => 0x0333,\n");
out.push_str(" SecondCodepoint::CombiningReverseSolidusOverlay => 0x20E5,\n");
out.push_str(" SecondCodepoint::VariationSelector1 => 0xFE00,\n");
out.push_str(" SecondCodepoint::LatinSmallLetterJ => 0x006A,\n");
out.push_str(" SecondCodepoint::CombiningMacronBelow => 0x0331,\n");
out.push_str(" }\n");
out.push_str(" }\n");
out.push_str("}\n\n");
// Struct definitions.
out.push_str("#[derive(Clone, Copy)]\n");
out.push_str("pub struct DafsaNode {\n");
out.push_str(" pub character: u8,\n");
out.push_str(" pub number: u8,\n");
out.push_str(" pub end_of_word: bool,\n");
out.push_str(" pub child_index: u16,\n");
out.push_str(" pub children_len: u8,\n");
out.push_str("}\n\n");
out.push_str("#[derive(Clone, Copy)]\n");
out.push_str("pub struct SecondLayerNode {\n");
out.push_str(" pub child_index: u16,\n");
out.push_str(" pub number: u8,\n");
out.push_str(" pub children_len: u8,\n");
out.push_str(" pub end_of_word: bool,\n");
out.push_str("}\n\n");
// Codepoints lookup table.
out.push_str(&format!(
"pub static CODEPOINTS_LOOKUP: [(u32, SecondCodepoint); {}] = [\n",
index_to_codepoints.len()
));
for (first, second) in &index_to_codepoints {
let variant = second_codepoint_variant(*second);
out.push_str(&format!(" ({first:#06X}, SecondCodepoint::{variant}),\n"));
}
out.push_str("];\n\n");
// DAFSA nodes array (with dummy node at index 0).
out.push_str(&format!(
"pub static DAFSA_NODES: [DafsaNode; {}] = [\n",
node_data.len() + 1
));
out.push_str(" DafsaNode { character: 0, number: 0, end_of_word: false, child_index: 0, children_len: 0 },\n");
for nd in &node_data {
out.push_str(&format!(
" DafsaNode {{ character: b'{}', number: {}, end_of_word: {}, child_index: {}, children_len: {} }},\n",
escape_byte(nd.character),
nd.number,
nd.end_of_word,
nd.child_index,
nd.children_len
));
}
out.push_str("];\n\n");
// First layer.
out.push_str(&format!("pub static FIRST_LAYER: [u16; {}] = [\n", first_layer.len()));
for n in &first_layer {
out.push_str(&format!(" {n},\n"));
}
out.push_str("];\n\n");
// First-to-second layer links.
out.push_str(&format!(
"pub static FIRST_TO_SECOND_LAYER: [(u64, u16); {}] = [\n",
first_to_second_layer.len()
));
for (mask, offset) in &first_to_second_layer {
out.push_str(&format!(" ({mask:#018X}, {offset}),\n"));
}
out.push_str("];\n\n");
// Second layer nodes.
out.push_str(&format!(
"pub static SECOND_LAYER: [SecondLayerNode; {}] = [\n",
second_layer.len()
));
for sl in &second_layer {
out.push_str(&format!(
" SecondLayerNode {{ child_index: {}, number: {}, children_len: {}, end_of_word: {} }},\n",
sl.child_index, sl.number, sl.children_len, sl.end_of_word
));
}
out.push_str("];\n\n");
// Total entity count.
out.push_str(&format!("pub const ENTITY_COUNT: usize = {};\n", entities.len()));
fs::write(&out_path, &out).expect("Failed to write generated file");
}
/// Extract the string literal from `__ENUMERATE_FOO(ident, "string")` macro
/// invocations in a C++ header.
fn parse_enumerate_macro(source: &str, macro_name: &str) -> Vec<String> {
let needle = format!("{macro_name}(");
let mut out = Vec::new();
for line in source.lines() {
let Some(idx) = line.find(&needle) else {
continue;
};
let rest = &line[idx + needle.len()..];
// Take the second argument, which is the quoted string literal.
let Some(first_quote) = rest.find('"') else {
continue;
};
let after = &rest[first_quote + 1..];
let Some(end_quote) = after.find('"') else {
continue;
};
out.push(after[..end_quote].to_string());
}
out
}
/// Emit a Rust source file with two const byte-slice arrays and two lookup
/// functions that dispatch on length and then on the exact bytes. rustc
/// compiles this pattern to a jump table + direct memcmp, which beats a
/// HashMap lookup with a cryptographic default hasher by a wide margin for
/// the small, fixed set of HTML names.
fn emit_interned_names(out_path: &Path, tag_names: &[String], attr_names: &[String]) {
let mut out = String::new();
out.push_str("// Auto-generated by build.rs from TagNames.h / AttributeNames.h.\n");
out.push_str("// Do not edit by hand.\n\n");
out.push_str("pub const INTERNED_TAG_NAMES: &[&[u8]] = &[\n");
for name in tag_names {
out.push_str(&format!(" b\"{}\",\n", name));
}
out.push_str("];\n\n");
out.push_str("pub const INTERNED_ATTR_NAMES: &[&[u8]] = &[\n");
for name in attr_names {
out.push_str(&format!(" b\"{}\",\n", name));
}
out.push_str("];\n\n");
emit_lookup_fn(&mut out, "lookup_tag_name_generated", tag_names);
emit_lookup_fn(&mut out, "lookup_attr_name_generated", attr_names);
fs::write(out_path, out).expect("Failed to write interned_names_generated.rs");
}
fn emit_lookup_fn(out: &mut String, fn_name: &str, names: &[String]) {
// Group names by byte length so the outer dispatch can be a single match.
let mut by_length: std::collections::BTreeMap<usize, Vec<(usize, &String)>> = std::collections::BTreeMap::new();
for (i, name) in names.iter().enumerate() {
by_length.entry(name.len()).or_default().push((i, name));
}
out.push_str(&format!("#[inline]\npub fn {fn_name}(bytes: &[u8]) -> u16 {{\n"));
out.push_str(" match bytes.len() {\n");
for (length, entries) in &by_length {
out.push_str(&format!(" {length} => match bytes {{\n"));
for (index, name) in entries {
// id is 1-based.
let id = index + 1;
out.push_str(&format!(" b\"{name}\" => {id},\n"));
}
out.push_str(" _ => 0,\n");
out.push_str(" },\n");
}
out.push_str(" _ => 0,\n");
out.push_str(" }\n");
out.push_str("}\n\n");
}
fn escape_byte(b: u8) -> String {
if b == b'\'' {
"\\'".to_string()
} else if b == b'\\' {
"\\\\".to_string()
} else if b.is_ascii_graphic() || b == b' ' {
String::from(b as char)
} else {
format!("\\x{b:02X}")
}
}
fn second_codepoint_variant(cp: u32) -> &'static str {
match cp {
0 => "None",
0x0338 => "CombiningLongSolidusOverlay",
0x20D2 => "CombiningLongVerticalLineOverlay",
0x200A => "HairSpace",
0x0333 => "CombiningDoubleLowLine",
0x20E5 => "CombiningReverseSolidusOverlay",
0xFE00 => "VariationSelector1",
0x006A => "LatinSmallLetterJ",
0x0331 => "CombiningMacronBelow",
_ => panic!("Unknown second codepoint: {cp:#X}"),
}
}
// Minimal JSON parser for Entities.json.
fn parse_entities_json(json: &str) -> Vec<(String, u32, u32)> {
let mut entities = Vec::new();
let bytes = json.as_bytes();
let len = bytes.len();
let mut i = 0;
while i < len && bytes[i] != b'{' {
i += 1;
}
i += 1;
loop {
while i < len && bytes[i].is_ascii_whitespace() {
i += 1;
}
if i >= len || bytes[i] == b'}' {
break;
}
if bytes[i] == b',' {
i += 1;
continue;
}
// Parse key.
assert_eq!(bytes[i], b'"');
i += 1;
let key_start = i;
while i < len && bytes[i] != b'"' {
if bytes[i] == b'\\' {
i += 1;
}
i += 1;
}
let key = std::str::from_utf8(&bytes[key_start..i]).unwrap().to_string();
i += 1;
// Skip ':'.
while i < len && bytes[i].is_ascii_whitespace() {
i += 1;
}
assert_eq!(bytes[i], b':');
i += 1;
// Skip to inner '{'.
while i < len && bytes[i] != b'{' {
i += 1;
}
i += 1;
// Parse inner object for "codepoints".
let mut codepoints: Vec<u32> = Vec::new();
while i < len && bytes[i] != b'}' {
if bytes[i] == b'"' {
i += 1;
let field_start = i;
while i < len && bytes[i] != b'"' {
i += 1;
}
let field_name = std::str::from_utf8(&bytes[field_start..i]).unwrap();
i += 1;
while i < len && bytes[i].is_ascii_whitespace() {
i += 1;
}
assert_eq!(bytes[i], b':');
i += 1;
while i < len && bytes[i].is_ascii_whitespace() {
i += 1;
}
if field_name == "codepoints" {
assert_eq!(bytes[i], b'[');
i += 1;
loop {
while i < len && (bytes[i].is_ascii_whitespace() || bytes[i] == b',') {
i += 1;
}
if i >= len || bytes[i] == b']' {
i += 1;
break;
}
let num_start = i;
while i < len && bytes[i].is_ascii_digit() {
i += 1;
}
let num_str = std::str::from_utf8(&bytes[num_start..i]).unwrap();
codepoints.push(num_str.parse().unwrap());
}
} else {
// Skip value.
if bytes[i] == b'"' {
i += 1;
while i < len && bytes[i] != b'"' {
if bytes[i] == b'\\' {
i += 1;
}
i += 1;
}
i += 1;
} else if bytes[i] == b'[' {
let mut depth = 1;
i += 1;
while i < len && depth > 0 {
if bytes[i] == b'[' {
depth += 1;
} else if bytes[i] == b']' {
depth -= 1;
}
i += 1;
}
}
}
} else {
i += 1;
}
}
i += 1;
let name = key.strip_prefix('&').unwrap_or(&key).to_string();
let first = codepoints.first().copied().unwrap_or(0);
let second = if codepoints.len() > 1 { codepoints[1] } else { 0 };
entities.push((name, first, second));
}
entities.sort_by(|a, b| a.0.cmp(&b.0));
entities
}
// DAFSA builder using Rc<RefCell<Node>> for shared ownership.
type NodeRc = Rc<RefCell<Node>>;
struct Node {
children: Vec<Option<NodeRc>>, // 128 slots
is_terminal: bool,
number: u16,
}
struct SecondLayerEntry {
child_index: u16,
number: u8,
children_len: u8,
end_of_word: bool,
}
struct NodeData {
character: u8,
number: u8,
end_of_word: bool,
child_index: u16,
children_len: u8,
}
impl Node {
fn new_rc() -> NodeRc {
Rc::new(RefCell::new(Node {
children: (0..128).map(|_| Option::None).collect(),
is_terminal: false,
number: 0,
}))
}
fn calc_numbers(&mut self) {
self.number = if self.is_terminal { 1 } else { 0 };
for child in self.children.iter().flatten() {
child.borrow_mut().calc_numbers();
self.number += child.borrow().number;
}
}
fn num_direct_children(&self) -> u8 {
let mut n = 0u8;
for c in &self.children {
if c.is_some() {
n += 1;
}
}
n
}
fn get_ascii_alphabetic_bit_mask(&self) -> u64 {
let mut mask = 0u64;
for i in 0..128u8 {
if self.children[i as usize].is_some() {
mask |= 1u64 << ascii_alphabetic_to_index(i);
}
}
mask
}
/// Hash based on child identities (Rc pointer) and terminal status.
fn structure_hash(&self) -> u64 {
let mut h: u64 = if self.is_terminal { 1 } else { 0 };
for (i, child) in self.children.iter().enumerate() {
if let Some(c) = child {
h = h.wrapping_mul(31).wrapping_add(i as u64);
h = h.wrapping_mul(31).wrapping_add(Rc::as_ptr(c) as u64);
}
}
h
}
/// Check structural equality via Rc pointer identity.
fn structure_eq(&self, other: &Node) -> bool {
if self.is_terminal != other.is_terminal {
return false;
}
for i in 0..128 {
match (&self.children[i], &other.children[i]) {
(None, None) => {}
(Some(a), Some(b)) => {
if !Rc::ptr_eq(a, b) {
return false;
}
}
_ => return false,
}
}
true
}
}
fn ascii_alphabetic_to_index(c: u8) -> u8 {
if c <= b'Z' { c - b'A' } else { c - b'a' + 26 }
}
struct UncheckedNode {
parent: NodeRc,
character: u8,
}
struct DafsaBuilder {
root: NodeRc,
minimized_nodes: HashMap<u64, Vec<NodeRc>>,
unchecked_nodes: Vec<UncheckedNode>,
previous_word: String,
}
impl DafsaBuilder {
fn new() -> Self {
DafsaBuilder {
root: Node::new_rc(),
minimized_nodes: HashMap::new(),
unchecked_nodes: Vec::new(),
previous_word: String::new(),
}
}
fn insert(&mut self, word: &str) {
assert!(
word > self.previous_word.as_str(),
"Words must be inserted in sorted order: '{word}' <= '{}'",
self.previous_word
);
let common_prefix_len = word
.bytes()
.zip(self.previous_word.bytes())
.take_while(|(a, b)| a == b)
.count();
self.minimize(common_prefix_len);
let node: NodeRc = if self.unchecked_nodes.is_empty() {
Rc::clone(&self.root)
} else {
let last = &self.unchecked_nodes[self.unchecked_nodes.len() - 1];
let parent = last.parent.borrow();
Rc::clone(parent.children[last.character as usize].as_ref().unwrap())
};
let remaining = &word[common_prefix_len..];
let mut current = node;
for c in remaining.bytes() {
let new_child = Node::new_rc();
{
let mut current_ref = current.borrow_mut();
assert!(current_ref.children[c as usize].is_none());
current_ref.children[c as usize] = Some(Rc::clone(&new_child));
}
self.unchecked_nodes.push(UncheckedNode {
parent: Rc::clone(&current),
character: c,
});
current = new_child;
}
current.borrow_mut().is_terminal = true;
self.previous_word = word.to_string();
}
fn minimize(&mut self, down_to: usize) {
while self.unchecked_nodes.len() > down_to {
let unchecked = self.unchecked_nodes.pop().unwrap();
let parent = &unchecked.parent;
let child = {
let parent_ref = parent.borrow();
Rc::clone(parent_ref.children[unchecked.character as usize].as_ref().unwrap())
};
let hash = child.borrow().structure_hash();
let mut found_replacement: Option<NodeRc> = Option::None;
if let Some(bucket) = self.minimized_nodes.get(&hash) {
for existing in bucket {
if child.borrow().structure_eq(&existing.borrow()) {
found_replacement = Some(Rc::clone(existing));
break;
}
}
}
if let Some(replacement) = found_replacement {
parent.borrow_mut().children[unchecked.character as usize] = Some(replacement);
} else {
self.minimized_nodes.entry(hash).or_default().push(Rc::clone(&child));
}
}
}
fn calc_numbers(&mut self) {
self.root.borrow_mut().calc_numbers();
}
fn get_unique_index(&self, word: &str) -> Option<usize> {
let mut index: usize = 0;
let mut current = Rc::clone(&self.root);
for c in word.bytes() {
let next = {
let node = current.borrow();
let child = node.children[c as usize].as_ref()?;
for sibling_c in 0u8..128 {
if let Some(sibling) = &node.children[sibling_c as usize]
&& sibling_c < c
{
index += sibling.borrow().number as usize;
}
}
Rc::clone(child)
};
if next.borrow().is_terminal {
index += 1;
}
current = next;
}
Some(index)
}
}
fn queue_children(
node: &NodeRc,
queue: &mut Vec<NodeRc>,
child_indexes: &mut HashMap<*const RefCell<Node>, u16>,
first_available_index: u16,
) -> u16 {
let mut current = first_available_index;
let node_ref = node.borrow();
for c in 0..128u8 {
if let Some(child) = &node_ref.children[c as usize] {
let key = Rc::as_ptr(child);
if let std::collections::hash_map::Entry::Vacant(entry) = child_indexes.entry(key) {
let num_children = child.borrow().num_direct_children();
if num_children > 0 {
entry.insert(current);
current += num_children as u16;
}
queue.push(Rc::clone(child));
}
}
}
current
}
fn write_children_data(
node: &NodeRc,
node_data: &mut Vec<NodeData>,
queue: &mut Vec<NodeRc>,
child_indexes: &mut HashMap<*const RefCell<Node>, u16>,
first_available_index: u16,
) -> u16 {
let mut current = first_available_index;
let mut unique_index_tally: u8 = 0;
let node_ref = node.borrow();
for c in 0..128u8 {
if let Some(child) = &node_ref.children[c as usize] {
let key = Rc::as_ptr(child);
let child_ref = child.borrow();
let num_children = child_ref.num_direct_children();
if let std::collections::hash_map::Entry::Vacant(entry) = child_indexes.entry(key) {
if num_children > 0 {
entry.insert(current);
current += num_children as u16;
}
queue.push(Rc::clone(child));
}
node_data.push(NodeData {
character: c,
number: unique_index_tally,
end_of_word: child_ref.is_terminal,
child_index: child_indexes.get(&key).copied().unwrap_or(0),
children_len: num_children,
});
unique_index_tally = unique_index_tally.wrapping_add(child_ref.number as u8);
}
}
current
}

19
Libraries/LibWeb/HTML/Parser/Rust/cbindgen.toml Normal file
View file

@ -0,0 +1,19 @@
language = "C++"
header = """/*
* Copyright (c) 2026-present, the Ladybird developers.
*
* SPDX-License-Identifier: BSD-2-Clause
*/"""
pragma_once = true
include_version = true
line_length = 120
tab_width = 4
no_includes = true
sys_includes = ["stdint.h", "stddef.h"]
usize_is_size_t = true
[parse]
parse_deps = false
[parse.expand]
all_features = false

151
Libraries/LibWeb/HTML/Parser/Rust/src/entities.rs Normal file
View file

@ -0,0 +1,151 @@
/*
* Copyright (c) 2026-present, the Ladybird developers.
*
* SPDX-License-Identifier: BSD-2-Clause
*/
//! Named character reference matching using a DAFSA (Deterministic Acyclic Finite
//! State Automaton) with minimal perfect hashing. The DAFSA data is generated at
//! build time by build.rs from Entities.json.
include!(concat!(env!("OUT_DIR"), "/named_character_references.rs"));
fn ascii_alphabetic_to_index(c: u8) -> u8 {
if c <= b'Z' { c - b'A' } else { c - b'a' + 26 }
}
#[derive(Clone, Copy)]
enum SearchState {
Init,
FirstToSecondLayer { mask: u64, offset: u16 },
DafsaChildren { start_index: u16, len: u8 },
}
/// Incremental matcher for named character references using the DAFSA.
///
/// Feed characters one at a time via `try_consume_code_point()`. After each
/// rejected character (returns false), call `code_points()` to get the longest
/// match found so far, and `overconsumed_code_points()` to know how many
/// characters were consumed past the longest match.
pub struct NamedCharacterReferenceMatcher {
search_state: SearchState,
last_matched_unique_index: u16,
pending_unique_index: u16,
overconsumed_code_points: u8,
ends_with_semicolon: bool,
}
impl NamedCharacterReferenceMatcher {
pub fn new() -> Self {
Self {
search_state: SearchState::Init,
last_matched_unique_index: 0,
pending_unique_index: 0,
overconsumed_code_points: 0,
ends_with_semicolon: false,
}
}
/// Feed one code point to the matcher. Returns true if the character was
/// consumed (there may still be longer matches), false if no further
/// matches are possible.
pub fn try_consume_code_point(&mut self, c: u32) -> bool {
if c > 0x7F {
return false;
}
self.try_consume_ascii_char(c as u8)
}
fn try_consume_ascii_char(&mut self, c: u8) -> bool {
match self.search_state {
SearchState::Init => {
if !c.is_ascii_alphabetic() {
return false;
}
let index = ascii_alphabetic_to_index(c) as usize;
self.search_state = SearchState::FirstToSecondLayer {
mask: FIRST_TO_SECOND_LAYER[index].0,
offset: FIRST_TO_SECOND_LAYER[index].1,
};
self.pending_unique_index = FIRST_LAYER[index];
self.overconsumed_code_points += 1;
true
}
SearchState::FirstToSecondLayer { mask, offset } => {
if !c.is_ascii_alphabetic() {
return false;
}
let bit_index = ascii_alphabetic_to_index(c);
if ((1u64 << bit_index) & mask) == 0 {
return false;
}
// Count set bits below bit_index to find the node's position.
let lower_mask = (1u64 << bit_index) - 1;
let char_index = (mask & lower_mask).count_ones() as u16;
let node = &SECOND_LAYER[(offset + char_index) as usize];
self.pending_unique_index += node.number as u16;
self.overconsumed_code_points += 1;
if node.end_of_word {
self.pending_unique_index += 1;
self.last_matched_unique_index = self.pending_unique_index;
self.ends_with_semicolon = c == b';';
self.overconsumed_code_points = 0;
}
self.search_state = SearchState::DafsaChildren {
start_index: node.child_index,
len: node.children_len,
};
true
}
SearchState::DafsaChildren { start_index, len } => {
for i in 0..len as u16 {
let node = &DAFSA_NODES[(start_index + i) as usize];
if node.character == c {
self.pending_unique_index += node.number as u16;
self.overconsumed_code_points += 1;
if node.end_of_word {
self.pending_unique_index += 1;
self.last_matched_unique_index = self.pending_unique_index;
self.ends_with_semicolon = c == b';';
self.overconsumed_code_points = 0;
}
self.search_state = SearchState::DafsaChildren {
start_index: node.child_index,
len: node.children_len,
};
return true;
}
}
false
}
}
}
/// Returns the codepoints for the longest match found, or None if no match.
/// Returns (first_codepoint, second_codepoint). second is 0 if there is no second codepoint.
pub fn code_points(&self) -> Option<(u32, u32)> {
if self.last_matched_unique_index == 0 {
return None;
}
let entry = &CODEPOINTS_LOOKUP[(self.last_matched_unique_index - 1) as usize];
Some((entry.0, entry.1.value()))
}
/// Number of characters consumed past the longest match.
pub fn overconsumed_code_points(&self) -> u8 {
self.overconsumed_code_points
}
/// Whether the longest match ended with a semicolon.
pub fn last_match_ends_with_semicolon(&self) -> bool {
self.ends_with_semicolon
}
}
impl Default for NamedCharacterReferenceMatcher {
fn default() -> Self {
Self::new()
}
}

61
Libraries/LibWeb/HTML/Parser/Rust/src/interned_names.rs Normal file
View file

@ -0,0 +1,61 @@
/*
* Copyright (c) 2026-present, the Ladybird developers.
*
* SPDX-License-Identifier: BSD-2-Clause
*/
//! Lookup tables for common HTML tag and attribute names.
//!
//! The tables are generated at build time from LibWeb's TagNames.h and
//! AttributeNames.h headers so they stay in sync with the C++ side. Each
//! name gets a 1-based u16 id; id 0 is reserved for "not interned" so the
//! FFI layer can carry either an id or raw bytes in the same slot.
//!
//! The lookup functions themselves are generated as a `match` on byte
//! length and exact byte sequence, which rustc compiles to a jump table
//! plus direct memcmp per length bucket. This is dramatically faster than
//! a HashMap lookup with the default cryptographic hasher.
include!(concat!(env!("OUT_DIR"), "/interned_names_generated.rs"));
/// Look up a tag name. Returns 0 if not interned.
#[inline]
pub fn lookup_tag_name(bytes: &[u8]) -> u16 {
lookup_tag_name_generated(bytes)
}
/// Look up an attribute name. Returns 0 if not interned.
#[inline]
pub fn lookup_attr_name(bytes: &[u8]) -> u16 {
lookup_attr_name_generated(bytes)
}
/// Number of interned tag names (for C++ table sizing at startup).
#[inline]
pub fn tag_name_count() -> usize {
INTERNED_TAG_NAMES.len()
}
/// Number of interned attribute names.
#[inline]
pub fn attr_name_count() -> usize {
INTERNED_ATTR_NAMES.len()
}
/// Fetch a tag name by id (1-based). Returns None for id 0 or out of range.
#[inline]
pub fn tag_name_by_id(id: u16) -> Option<&'static [u8]> {
if id == 0 {
return None;
}
INTERNED_TAG_NAMES.get((id - 1) as usize).copied()
}
/// Fetch an attribute name by id (1-based). Returns None for id 0 or out of range.
#[inline]
pub fn attr_name_by_id(id: u16) -> Option<&'static [u8]> {
if id == 0 {
return None;
}
INTERNED_ATTR_NAMES.get((id - 1) as usize).copied()
}

702
Libraries/LibWeb/HTML/Parser/Rust/src/lib.rs Normal file
View file

@ -0,0 +1,702 @@
/*
* Copyright (c) 2026-present, the Ladybird developers.
*
* SPDX-License-Identifier: BSD-2-Clause
*/
pub mod entities;
pub mod interned_names;
pub mod token;
pub mod tokenizer;
use std::ptr;
use token::{Attribute, Position, TokenPayload, TokenType};
use tokenizer::{HtmlTokenizer, State};
/// Opaque handle for the Rust tokenizer, passed across the FFI boundary.
pub struct RustFfiTokenizerHandle {
tokenizer: HtmlTokenizer,
/// Temporary storage for the last token's string data, kept alive
/// so that pointers in RustFfiToken remain valid until the next call.
last_tag_name: Vec<u8>,
last_comment: Vec<u8>,
last_doctype_name: Vec<u8>,
last_public_id: Vec<u8>,
last_system_id: Vec<u8>,
last_attributes: Vec<RustFfiAttribute>,
last_attr_names: Vec<Vec<u8>>,
last_attr_values: Vec<Vec<u8>>,
last_unparsed_input: Vec<u8>,
}
/// C-compatible token representation.
///
/// Pointer fields (`tag_name_ptr`, `comment_ptr`, doctype/attribute pointers)
/// borrow into buffers owned by the `RustFfiTokenizerHandle` that produced this
/// token. They remain valid only until the next call into the tokenizer for
/// that handle (any of `next_token`, `insert_input`, `destroy`, etc.).
/// Callers must consume the data before making the next call.
#[repr(C)]
pub struct RustFfiToken {
pub token_type: u8,
pub code_point: u32,
pub self_closing: bool,
/// If nonzero, an interned tag-name id (1-based index into
/// `interned_names::INTERNED_TAG_NAMES`). When set, `tag_name_ptr` /
/// `tag_name_len` are unused and the C++ side uses its parallel
/// FlyString table directly.
pub tag_name_id: u16,
pub tag_name_ptr: *const u8,
pub tag_name_len: usize,
pub comment_ptr: *const u8,
pub comment_len: usize,
pub doctype_name_ptr: *const u8,
pub doctype_name_len: usize,
pub public_id_ptr: *const u8,
pub public_id_len: usize,
pub system_id_ptr: *const u8,
pub system_id_len: usize,
pub force_quirks: bool,
pub missing_name: bool,
pub missing_public_id: bool,
pub missing_system_id: bool,
pub attributes_ptr: *const RustFfiAttribute,
pub attributes_len: usize,
pub start_line: u64,
pub start_column: u64,
pub end_line: u64,
pub end_column: u64,
}
/// C-compatible attribute representation.
#[repr(C)]
pub struct RustFfiAttribute {
/// If nonzero, an interned attribute-name id (1-based index into
/// `interned_names::INTERNED_ATTR_NAMES`). When set, `name_ptr` /
/// `name_len` are unused.
pub name_id: u16,
pub name_ptr: *const u8,
pub name_len: usize,
pub value_ptr: *const u8,
pub value_len: usize,
pub name_start_line: u64,
pub name_start_column: u64,
pub name_end_line: u64,
pub name_end_column: u64,
pub value_start_line: u64,
pub value_start_column: u64,
pub value_end_line: u64,
pub value_end_column: u64,
}
impl Default for RustFfiToken {
fn default() -> Self {
RustFfiToken {
token_type: TokenType::Invalid as u8,
code_point: 0,
self_closing: false,
tag_name_id: 0,
tag_name_ptr: ptr::null(),
tag_name_len: 0,
comment_ptr: ptr::null(),
comment_len: 0,
doctype_name_ptr: ptr::null(),
doctype_name_len: 0,
public_id_ptr: ptr::null(),
public_id_len: 0,
system_id_ptr: ptr::null(),
system_id_len: 0,
force_quirks: false,
missing_name: true,
missing_public_id: true,
missing_system_id: true,
attributes_ptr: ptr::null(),
attributes_len: 0,
start_line: 0,
start_column: 0,
end_line: 0,
end_column: 0,
}
}
}
fn position_to_ffi(pos: &Position) -> (u64, u64) {
(pos.line, pos.column)
}
/// Create a new Rust HTML tokenizer from UTF-32 code points.
///
/// # Safety
/// `input` must point to `len` valid u32 values.
#[unsafe(no_mangle)]
pub unsafe extern "C" fn rust_html_tokenizer_create(input: *const u32, len: usize) -> *mut RustFfiTokenizerHandle {
let code_points = if input.is_null() || len == 0 {
Vec::new()
} else {
unsafe { std::slice::from_raw_parts(input, len) }.to_vec()
};
make_handle(HtmlTokenizer::new(code_points))
}
/// Create a new Rust HTML tokenizer directly from a UTF-8 byte buffer.
/// Rust decodes the bytes to code points internally, skipping the C++
/// side's 4x-expanded Vec<u32> copy.
///
/// # Safety
/// `bytes` must point to `len` valid UTF-8 bytes.
#[unsafe(no_mangle)]
pub unsafe extern "C" fn rust_html_tokenizer_create_from_utf8(
bytes: *const u8,
len: usize,
) -> *mut RustFfiTokenizerHandle {
let code_points = if bytes.is_null() || len == 0 {
Vec::new()
} else {
let slice = unsafe { std::slice::from_raw_parts(bytes, len) };
decode_utf8_to_u32(slice)
};
make_handle(HtmlTokenizer::new(code_points))
}
/// Expand a UTF-8 byte slice into a `Vec<u32>` of code points. For the
/// dominant ASCII case we use a tight loop with a single unchecked
/// write per byte and only fall back to `str::chars()` decoding when
/// we see a continuation byte. The caller is responsible for ensuring
/// the input is valid UTF-8.
fn decode_utf8_to_u32(bytes: &[u8]) -> Vec<u32> {
let mut out: Vec<u32> = Vec::with_capacity(bytes.len());
// SAFETY: we reserved `bytes.len()` slots and will only write up to
// that many u32s (one per input byte; multi-byte sequences produce
// fewer u32s than bytes, so we stay within bounds).
let out_ptr = out.as_mut_ptr();
let mut write_idx: usize = 0;
let mut i: usize = 0;
let n = bytes.len();
while i < n {
let b = bytes[i];
if b < 0x80 {
unsafe { std::ptr::write(out_ptr.add(write_idx), b as u32) };
write_idx += 1;
i += 1;
} else {
// Slow path: decode one code point via str::chars.
// SAFETY: the input is valid UTF-8 by precondition.
let tail = unsafe { std::str::from_utf8_unchecked(&bytes[i..]) };
if let Some(ch) = tail.chars().next() {
unsafe { std::ptr::write(out_ptr.add(write_idx), ch as u32) };
write_idx += 1;
i += ch.len_utf8();
} else {
break;
}
}
}
// SAFETY: we wrote exactly `write_idx` elements, all in-bounds.
unsafe { out.set_len(write_idx) };
out
}
fn make_handle(tokenizer: HtmlTokenizer) -> *mut RustFfiTokenizerHandle {
let handle = Box::new(RustFfiTokenizerHandle {
tokenizer,
last_tag_name: Vec::new(),
last_comment: Vec::new(),
last_doctype_name: Vec::new(),
last_public_id: Vec::new(),
last_system_id: Vec::new(),
last_attributes: Vec::new(),
last_attr_names: Vec::new(),
last_attr_values: Vec::new(),
last_unparsed_input: Vec::new(),
});
Box::into_raw(handle)
}
/// Get the next token from the tokenizer.
/// Returns true if a token was produced, false if no more tokens.
///
/// # Safety
/// `handle` must be a valid pointer from `rust_html_tokenizer_create`.
/// `out` must be a valid pointer to an RustFfiToken.
#[unsafe(no_mangle)]
pub unsafe extern "C" fn rust_html_tokenizer_next_token(
handle: *mut RustFfiTokenizerHandle,
out: *mut RustFfiToken,
stop_at_insertion_point: bool,
cdata_allowed: bool,
) -> bool {
if handle.is_null() || out.is_null() {
return false;
}
let handle = unsafe { &mut *handle };
let out = unsafe { &mut *out };
// Fast-path text character in the Data state. For ASCII text runs
// this skips the whole state-machine function call plus Token
// construction/drop. Most real HTML is dominated by such runs.
if !stop_at_insertion_point && let Some((cp, pos)) = handle.tokenizer.try_fast_data_char() {
out.token_type = TokenType::Character as u8;
out.code_point = cp;
out.start_line = pos.line;
out.start_column = pos.column;
out.end_line = pos.line;
out.end_column = pos.column;
return true;
}
next_token_slow(handle, out, stop_at_insertion_point, cdata_allowed)
}
/// The state-machine and marshalling portion of the FFI token fetch,
/// pulled out of `rust_html_tokenizer_next_token` so that the fast
/// Data-state path in the outer function keeps a tiny stack frame
/// and a short prologue. Marked `#[inline(never)]` so the compiler
/// doesn't re-inline the whole thing and re-inflate the outer frame.
#[inline(never)]
fn next_token_slow(
handle: &mut RustFfiTokenizerHandle,
out: &mut RustFfiToken,
stop_at_insertion_point: bool,
cdata_allowed: bool,
) -> bool {
let token = match handle.tokenizer.next_token(stop_at_insertion_point, cdata_allowed) {
Some(t) => t,
None => return false,
};
// Fast path: Character and EOF tokens only need a few fields.
// Skip zeroing ~200 bytes of RustFfiToken for 95%+ of all tokens.
match token.token_type {
TokenType::Character | TokenType::EndOfFile => {
out.token_type = token.token_type as u8;
out.code_point = token.code_point;
out.start_line = token.start_position.line;
out.start_column = token.start_position.column;
out.end_line = token.end_position.line;
out.end_column = token.end_position.column;
return true;
}
_ => {}
}
*out = RustFfiToken::default();
out.token_type = token.token_type as u8;
let (sl, sc) = position_to_ffi(&token.start_position);
let (el, ec) = position_to_ffi(&token.end_position);
out.start_line = sl;
out.start_column = sc;
out.end_line = el;
out.end_column = ec;
// Store string data in handle so pointers stay valid.
match token.payload {
TokenPayload::Tag {
tag_name,
tag_name_id,
self_closing,
attributes,
} => {
out.self_closing = self_closing;
// Tokenizer already resolved intern ids, so we trust tag_name_id.
out.tag_name_id = tag_name_id;
if tag_name_id == 0 {
handle.last_tag_name = tag_name.into_bytes();
out.tag_name_ptr = handle.last_tag_name.as_ptr();
out.tag_name_len = handle.last_tag_name.len();
}
// Convert attributes. Move owned Strings out of each Attribute
// instead of cloning, then point the FfiAttribute at the stable
// heap bytes that now live in handle.last_attr_{names,values}
// (unless the name was interned, in which case skip the copy).
handle.last_attr_names.clear();
handle.last_attr_values.clear();
handle.last_attributes.clear();
handle.last_attr_names.reserve(attributes.len());
handle.last_attr_values.reserve(attributes.len());
handle.last_attributes.reserve(attributes.len());
for attr in attributes {
let Attribute {
local_name,
local_name_id,
value,
name_start_position,
name_end_position,
value_start_position,
value_end_position,
} = attr;
if local_name_id == 0 {
handle.last_attr_names.push(local_name.into_bytes());
} else {
// Keep the slot aligned with last_attr_values so index math stays valid.
handle.last_attr_names.push(Vec::new());
}
handle.last_attr_values.push(value.into_bytes());
let last_idx = handle.last_attr_names.len() - 1;
let name_bytes = &handle.last_attr_names[last_idx];
let value_bytes = &handle.last_attr_values[last_idx];
handle.last_attributes.push(RustFfiAttribute {
name_id: local_name_id,
name_ptr: if local_name_id == 0 {
name_bytes.as_ptr()
} else {
ptr::null()
},
name_len: if local_name_id == 0 { name_bytes.len() } else { 0 },
value_ptr: value_bytes.as_ptr(),
value_len: value_bytes.len(),
name_start_line: name_start_position.line,
name_start_column: name_start_position.column,
name_end_line: name_end_position.line,
name_end_column: name_end_position.column,
value_start_line: value_start_position.line,
value_start_column: value_start_position.column,
value_end_line: value_end_position.line,
value_end_column: value_end_position.column,
});
}
out.attributes_ptr = handle.last_attributes.as_ptr();
out.attributes_len = handle.last_attributes.len();
}
TokenPayload::Comment(data) => {
handle.last_comment = data.into_bytes();
out.comment_ptr = handle.last_comment.as_ptr();
out.comment_len = handle.last_comment.len();
}
TokenPayload::Doctype(doctype) => {
handle.last_doctype_name = doctype.name.into_bytes();
handle.last_public_id = doctype.public_identifier.into_bytes();
handle.last_system_id = doctype.system_identifier.into_bytes();
out.doctype_name_ptr = handle.last_doctype_name.as_ptr();
out.doctype_name_len = handle.last_doctype_name.len();
out.public_id_ptr = handle.last_public_id.as_ptr();
out.public_id_len = handle.last_public_id.len();
out.system_id_ptr = handle.last_system_id.as_ptr();
out.system_id_len = handle.last_system_id.len();
out.force_quirks = doctype.force_quirks;
out.missing_name = doctype.missing_name;
out.missing_public_id = doctype.missing_public_identifier;
out.missing_system_id = doctype.missing_system_identifier;
}
TokenPayload::None => {}
}
true
}
/// Switch the tokenizer to a new state.
///
/// # Safety
/// `handle` must be a valid pointer from `rust_html_tokenizer_create`.
#[unsafe(no_mangle)]
pub unsafe extern "C" fn rust_html_tokenizer_switch_state(handle: *mut RustFfiTokenizerHandle, state: u8) {
if handle.is_null() {
return;
}
let handle = unsafe { &mut *handle };
// The state values must match the State enum order. Bound-checked against
// the last known variant so an out-of-range value is rejected instead of
// producing UB via transmute to an invalid discriminant.
const STATE_COUNT: u8 = State::NumericCharacterReferenceEnd as u8 + 1;
if state >= STATE_COUNT {
return;
}
let state: State = unsafe { std::mem::transmute(state) };
handle.tokenizer.switch_to(state);
}
/// Insert input (as UTF-32 code points) at the current insertion point.
///
/// # Safety
/// `handle` must be a valid pointer. `input` must point to `len` valid u32 values.
#[unsafe(no_mangle)]
pub unsafe extern "C" fn rust_html_tokenizer_insert_input(
handle: *mut RustFfiTokenizerHandle,
input: *const u32,
len: usize,
) {
if handle.is_null() {
return;
}
let handle = unsafe { &mut *handle };
let code_points = if input.is_null() || len == 0 {
&[]
} else {
unsafe { std::slice::from_raw_parts(input, len) }
};
handle.tokenizer.insert_input_at_insertion_point(code_points);
}
/// Append input (as UTF-32 code points) to the tokenizer input stream.
///
/// # Safety
/// `handle` must be a valid pointer. `input` must point to `len` valid u32 values.
#[unsafe(no_mangle)]
pub unsafe extern "C" fn rust_html_tokenizer_append_input(
handle: *mut RustFfiTokenizerHandle,
input: *const u32,
len: usize,
) {
if handle.is_null() {
return;
}
let handle = unsafe { &mut *handle };
let code_points = if input.is_null() || len == 0 {
&[]
} else {
unsafe { std::slice::from_raw_parts(input, len) }
};
handle.tokenizer.append_input(code_points);
}
/// Get the tokenizer input that has not been consumed yet.
///
/// # Safety
/// `handle`, `out_ptr`, and `out_len` must be valid pointers.
#[unsafe(no_mangle)]
pub unsafe extern "C" fn rust_html_tokenizer_unparsed_input(
handle: *mut RustFfiTokenizerHandle,
out_ptr: *mut *const u8,
out_len: *mut usize,
) {
if handle.is_null() || out_ptr.is_null() || out_len.is_null() {
return;
}
let handle = unsafe { &mut *handle };
handle.last_unparsed_input = handle.tokenizer.unparsed_input().into_bytes();
unsafe {
*out_ptr = handle.last_unparsed_input.as_ptr();
*out_len = handle.last_unparsed_input.len();
}
}
/// Compact already-tokenized input after the parser has consumed a chunk.
///
/// # Safety
/// `handle` must be a valid pointer.
#[unsafe(no_mangle)]
pub unsafe extern "C" fn rust_html_tokenizer_parser_did_run(handle: *mut RustFfiTokenizerHandle) {
if handle.is_null() {
return;
}
let handle = unsafe { &mut *handle };
handle.tokenizer.parser_did_run();
}
/// # Safety
/// `handle` must be a valid pointer.
#[unsafe(no_mangle)]
pub unsafe extern "C" fn rust_html_tokenizer_store_insertion_point(handle: *mut RustFfiTokenizerHandle) {
if handle.is_null() {
return;
}
let handle = unsafe { &mut *handle };
handle.tokenizer.store_insertion_point();
}
/// # Safety
/// `handle` must be a valid pointer.
#[unsafe(no_mangle)]
pub unsafe extern "C" fn rust_html_tokenizer_restore_insertion_point(handle: *mut RustFfiTokenizerHandle) {
if handle.is_null() {
return;
}
let handle = unsafe { &mut *handle };
handle.tokenizer.restore_insertion_point();
}
/// # Safety
/// `handle` must be a valid pointer.
#[unsafe(no_mangle)]
pub unsafe extern "C" fn rust_html_tokenizer_update_insertion_point(handle: *mut RustFfiTokenizerHandle) {
if handle.is_null() {
return;
}
let handle = unsafe { &mut *handle };
handle.tokenizer.update_insertion_point();
}
/// # Safety
/// `handle` must be a valid pointer.
#[unsafe(no_mangle)]
pub unsafe extern "C" fn rust_html_tokenizer_undefine_insertion_point(handle: *mut RustFfiTokenizerHandle) {
if handle.is_null() {
return;
}
let handle = unsafe { &mut *handle };
handle.tokenizer.undefine_insertion_point();
}
/// # Safety
/// `handle` must be a valid pointer.
#[unsafe(no_mangle)]
pub unsafe extern "C" fn rust_html_tokenizer_is_insertion_point_defined(handle: *mut RustFfiTokenizerHandle) -> bool {
if handle.is_null() {
return false;
}
let handle = unsafe { &mut *handle };
handle.tokenizer.is_insertion_point_defined()
}
/// # Safety
/// `handle` must be a valid pointer.
#[unsafe(no_mangle)]
pub unsafe extern "C" fn rust_html_tokenizer_is_insertion_point_reached(handle: *mut RustFfiTokenizerHandle) -> bool {
if handle.is_null() {
return false;
}
let handle = unsafe { &mut *handle };
handle.tokenizer.is_insertion_point_reached()
}
/// # Safety
/// `handle` must be a valid pointer.
#[unsafe(no_mangle)]
pub unsafe extern "C" fn rust_html_tokenizer_set_blocked(handle: *mut RustFfiTokenizerHandle, blocked: bool) {
if handle.is_null() {
return;
}
let handle = unsafe { &mut *handle };
handle.tokenizer.set_blocked(blocked);
}
/// # Safety
/// `handle` must be a valid pointer.
#[unsafe(no_mangle)]
pub unsafe extern "C" fn rust_html_tokenizer_is_blocked(handle: *mut RustFfiTokenizerHandle) -> bool {
if handle.is_null() {
return false;
}
let handle = unsafe { &mut *handle };
handle.tokenizer.is_blocked()
}
/// # Safety
/// `handle` must be a valid pointer.
#[unsafe(no_mangle)]
pub unsafe extern "C" fn rust_html_tokenizer_set_input_stream_closed(
handle: *mut RustFfiTokenizerHandle,
closed: bool,
) {
if handle.is_null() {
return;
}
let handle = unsafe { &mut *handle };
handle.tokenizer.set_input_stream_closed(closed);
}
/// # Safety
/// `handle` must be a valid pointer.
#[unsafe(no_mangle)]
pub unsafe extern "C" fn rust_html_tokenizer_insert_eof(handle: *mut RustFfiTokenizerHandle) {
if handle.is_null() {
return;
}
let handle = unsafe { &mut *handle };
handle.tokenizer.insert_eof();
}
/// # Safety
/// `handle` must be a valid pointer.
#[unsafe(no_mangle)]
pub unsafe extern "C" fn rust_html_tokenizer_is_eof_inserted(handle: *mut RustFfiTokenizerHandle) -> bool {
if handle.is_null() {
return false;
}
let handle = unsafe { &mut *handle };
handle.tokenizer.is_eof_inserted()
}
/// # Safety
/// `handle` must be a valid pointer.
#[unsafe(no_mangle)]
pub unsafe extern "C" fn rust_html_tokenizer_abort(handle: *mut RustFfiTokenizerHandle) {
if handle.is_null() {
return;
}
let handle = unsafe { &mut *handle };
handle.tokenizer.abort();
}
/// Destroy a Rust HTML tokenizer.
///
/// # Safety
/// `handle` must be a valid pointer from `rust_html_tokenizer_create`,
/// and must not be used after this call.
#[unsafe(no_mangle)]
pub unsafe extern "C" fn rust_html_tokenizer_destroy(handle: *mut RustFfiTokenizerHandle) {
if !handle.is_null() {
drop(unsafe { Box::from_raw(handle) });
}
}
// -- Interned name table enumeration --------------------------------------
//
// The C++ side builds a parallel FlyString array at static-init time by
// enumerating the Rust-owned list once. Ids are 1-based; id 0 is reserved
// for "not interned" in the per-token FFI struct.
/// Number of interned HTML tag names known to the Rust tokenizer.
#[unsafe(no_mangle)]
pub extern "C" fn rust_html_tokenizer_interned_tag_name_count() -> usize {
interned_names::tag_name_count()
}
/// Number of interned HTML attribute names known to the Rust tokenizer.
#[unsafe(no_mangle)]
pub extern "C" fn rust_html_tokenizer_interned_attr_name_count() -> usize {
interned_names::attr_name_count()
}
/// Write the bytes and length of the interned tag name with the given
/// 1-based id to the caller-provided out parameters. On unknown ids the
/// out parameters are set to (null, 0).
///
/// # Safety
/// `out_ptr` and `out_len` must be valid pointers.
#[unsafe(no_mangle)]
pub unsafe extern "C" fn rust_html_tokenizer_interned_tag_name(id: u16, out_ptr: *mut *const u8, out_len: *mut usize) {
if out_ptr.is_null() || out_len.is_null() {
return;
}
match interned_names::tag_name_by_id(id) {
Some(bytes) => unsafe {
*out_ptr = bytes.as_ptr();
*out_len = bytes.len();
},
None => unsafe {
*out_ptr = ptr::null();
*out_len = 0;
},
}
}
/// Same as `rust_html_tokenizer_interned_tag_name` for attribute names.
///
/// # Safety
/// `out_ptr` and `out_len` must be valid pointers.
#[unsafe(no_mangle)]
pub unsafe extern "C" fn rust_html_tokenizer_interned_attr_name(id: u16, out_ptr: *mut *const u8, out_len: *mut usize) {
if out_ptr.is_null() || out_len.is_null() {
return;
}
match interned_names::attr_name_by_id(id) {
Some(bytes) => unsafe {
*out_ptr = bytes.as_ptr();
*out_len = bytes.len();
},
None => unsafe {
*out_ptr = ptr::null();
*out_len = 0;
},
}
}

202
Libraries/LibWeb/HTML/Parser/Rust/src/token.rs Normal file
View file

@ -0,0 +1,202 @@
/*
* Copyright (c) 2026-present, the Ladybird developers.
*
* SPDX-License-Identifier: BSD-2-Clause
*/
/// Source position in the input.
#[derive(Clone, Copy, Debug, Default)]
pub struct Position {
pub line: u64,
pub column: u64,
}
/// A single attribute on a start or end tag token.
///
/// If `local_name_id` is non-zero it is an index into
/// `interned_names::INTERNED_ATTR_NAMES` and `local_name` is unused.
/// Otherwise `local_name` holds the owned bytes.
#[derive(Clone, Debug, Default)]
pub struct Attribute {
pub local_name: String,
pub local_name_id: u16,
pub value: String,
pub name_start_position: Position,
pub name_end_position: Position,
pub value_start_position: Position,
pub value_end_position: Position,
}
/// Data specific to DOCTYPE tokens.
#[derive(Clone, Debug, Default)]
pub struct DoctypeData {
pub name: String,
pub public_identifier: String,
pub system_identifier: String,
pub missing_name: bool,
pub missing_public_identifier: bool,
pub missing_system_identifier: bool,
pub force_quirks: bool,
}
/// The type of an HTML token.
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
#[repr(u8)]
pub enum TokenType {
#[default]
Invalid = 0,
Doctype = 1,
StartTag = 2,
EndTag = 3,
Comment = 4,
Character = 5,
EndOfFile = 6,
}
/// Type-specific data for an HTML token.
///
/// If `tag_name_id` is non-zero it is an index into
/// `interned_names::INTERNED_TAG_NAMES` and `tag_name` is unused.
/// Otherwise `tag_name` holds the owned bytes.
#[derive(Clone, Debug, Default)]
pub enum TokenPayload {
#[default]
None,
Tag {
tag_name: String,
tag_name_id: u16,
self_closing: bool,
attributes: Vec<Attribute>,
},
Comment(String),
Doctype(Box<DoctypeData>),
}
/// An HTML token produced by the tokenizer.
#[derive(Clone, Debug, Default)]
pub struct Token {
pub token_type: TokenType,
pub code_point: u32,
pub payload: TokenPayload,
pub start_position: Position,
pub end_position: Position,
}
impl Token {
pub fn new_character(code_point: u32) -> Self {
Token {
token_type: TokenType::Character,
code_point,
..Default::default()
}
}
pub fn new_eof() -> Self {
Token {
token_type: TokenType::EndOfFile,
..Default::default()
}
}
/// Return the tag name as a &str. For interned names this resolves
/// through the interned name table; for un-interned names it returns
/// the owned String's contents.
#[inline(always)]
pub fn tag_name(&self) -> &str {
match &self.payload {
TokenPayload::Tag {
tag_name, tag_name_id, ..
} => {
if *tag_name_id != 0 {
// SAFETY: interned names are compile-time ASCII byte
// literals in interned_names_generated.rs, so they are
// always valid UTF-8.
match crate::interned_names::tag_name_by_id(*tag_name_id) {
Some(bytes) => unsafe { std::str::from_utf8_unchecked(bytes) },
None => "",
}
} else {
tag_name
}
}
_ => "",
}
}
#[inline(always)]
pub fn tag_name_mut(&mut self) -> &mut String {
match &mut self.payload {
TokenPayload::Tag {
tag_name, tag_name_id, ..
} => {
*tag_name_id = 0;
tag_name
}
_ => panic!("tag_name_mut called on non-tag token"),
}
}
/// Set the tag name to an interned id, clearing any previously stored
/// owned name.
#[inline(always)]
pub fn set_tag_name_id(&mut self, id: u16) {
match &mut self.payload {
TokenPayload::Tag {
tag_name, tag_name_id, ..
} => {
tag_name.clear();
*tag_name_id = id;
}
_ => panic!("set_tag_name_id called on non-tag token"),
}
}
/// Returns the interned tag-name id, or 0 if the name is un-interned.
#[inline(always)]
pub fn tag_name_id(&self) -> u16 {
match &self.payload {
TokenPayload::Tag { tag_name_id, .. } => *tag_name_id,
_ => 0,
}
}
#[inline(always)]
pub fn set_self_closing(&mut self, value: bool) {
match &mut self.payload {
TokenPayload::Tag { self_closing, .. } => *self_closing = value,
_ => panic!("set_self_closing called on non-tag token"),
}
}
#[inline(always)]
pub fn is_self_closing(&self) -> bool {
match &self.payload {
TokenPayload::Tag { self_closing, .. } => *self_closing,
_ => false,
}
}
#[inline(always)]
pub fn attributes_mut(&mut self) -> &mut Vec<Attribute> {
match &mut self.payload {
TokenPayload::Tag { attributes, .. } => attributes,
_ => panic!("attributes_mut called on non-tag token"),
}
}
#[inline(always)]
pub fn set_comment_data(&mut self, data: String) {
match &mut self.payload {
TokenPayload::Comment(s) => *s = data,
_ => panic!("set_comment_data called on non-comment token"),
}
}
#[inline(always)]
pub fn doctype_data_mut(&mut self) -> &mut DoctypeData {
match &mut self.payload {
TokenPayload::Doctype(dd) => dd,
_ => panic!("doctype_data_mut called on non-doctype token"),
}
}
}

3152
Libraries/LibWeb/HTML/Parser/Rust/src/tokenizer.rs Normal file
View file

File diff suppressed because it is too large Load diff

1
Libraries/LibWeb/Rust/Cargo.toml
View file

@ -9,6 +9,7 @@ crate-type = ["staticlib"]
# After changing dependencies, regenerate the Flatpak sources:
# python3 Meta/CMake/flatpak/generate-cargo-sources.py
[dependencies]
libweb_html_tokenizer = { path = "../HTML/Parser/Rust" }
[build-dependencies]
cbindgen = "0.29"

2
Libraries/LibWeb/Rust/src/lib.rs
View file

@ -9,6 +9,8 @@ mod rust_allocator;
mod css_tokenizer;
pub use libweb_html_tokenizer as html_tokenizer;
use std::ffi::c_void;
use std::panic::{AssertUnwindSafe, catch_unwind};

83
Tests/LibWeb/TestHTMLTokenizer.cpp
View file

@ -199,6 +199,19 @@ TEST_CASE(character_reference_in_attribute)
END_ENUMERATION();
}
TEST_CASE(duplicate_attributes_are_reported)
{
auto tokens = run_tokenizer("<script nonce=x nonce=y></script>"sv);
auto& token = tokens.first();
EXPECT_EQ(token.type(), Token::Type::StartTag);
EXPECT(token.had_duplicate_attribute());
EXPECT_EQ(token.attribute_count(), 1u);
auto nonce = token.raw_attribute("nonce"_fly_string);
VERIFY(nonce.has_value());
EXPECT_EQ(nonce->value, "x");
}
TEST_CASE(named_character_reference)
{
auto tokens = run_tokenizer("&notinvc;&notit;&cz"sv);
@ -275,3 +288,73 @@ TEST_CASE(ambiguous_ampersand_offset)
EXPECT_EQ(token.start_position().line, 0u);
EXPECT_EQ(token.start_position().column, 1u);
}
TEST_CASE(insertion_point_inside_fast_tag_name)
{
Tokenizer tokenizer;
tokenizer.update_insertion_point();
tokenizer.insert_input_at_insertion_point("<abc"sv);
tokenizer.append_to_input_stream("def>"sv);
EXPECT(!tokenizer.next_token(Tokenizer::StopAtInsertionPoint::Yes).has_value());
EXPECT(tokenizer.is_insertion_point_reached());
EXPECT_EQ(tokenizer.unparsed_input(), "def>"sv);
tokenizer.insert_input_at_insertion_point("x"sv);
tokenizer.undefine_insertion_point();
tokenizer.close_input_stream();
auto token = tokenizer.next_token();
VERIFY(token.has_value());
EXPECT_EQ(token->type(), Token::Type::StartTag);
EXPECT_EQ(token->tag_name(), "abcxdef");
}
TEST_CASE(insertion_point_inside_fast_attribute_name)
{
Tokenizer tokenizer;
tokenizer.update_insertion_point();
tokenizer.insert_input_at_insertion_point("<p abc"sv);
tokenizer.append_to_input_stream("def=value>"sv);
EXPECT(!tokenizer.next_token(Tokenizer::StopAtInsertionPoint::Yes).has_value());
EXPECT(tokenizer.is_insertion_point_reached());
EXPECT_EQ(tokenizer.unparsed_input(), "def=value>"sv);
tokenizer.insert_input_at_insertion_point("x"sv);
tokenizer.undefine_insertion_point();
tokenizer.close_input_stream();
auto token = tokenizer.next_token();
VERIFY(token.has_value());
EXPECT_EQ(token->type(), Token::Type::StartTag);
EXPECT_EQ(token->attribute_count(), 1u);
auto attribute = token->raw_attribute("abcxdef"_fly_string);
VERIFY(attribute.has_value());
EXPECT_EQ(attribute->value, "value");
}
TEST_CASE(insertion_point_inside_fast_quoted_attribute_value)
{
Tokenizer tokenizer;
tokenizer.update_insertion_point();
tokenizer.insert_input_at_insertion_point("<p a=\"abc"sv);
tokenizer.append_to_input_stream("def\">"sv);
EXPECT(!tokenizer.next_token(Tokenizer::StopAtInsertionPoint::Yes).has_value());
EXPECT(tokenizer.is_insertion_point_reached());
EXPECT_EQ(tokenizer.unparsed_input(), "def\">"sv);
tokenizer.insert_input_at_insertion_point("x"sv);
tokenizer.undefine_insertion_point();
tokenizer.close_input_stream();
auto token = tokenizer.next_token();
VERIFY(token.has_value());
EXPECT_EQ(token->type(), Token::Type::StartTag);
auto attribute = token->raw_attribute("a"_fly_string);
VERIFY(attribute.has_value());
EXPECT_EQ(attribute->value, "abcxdef");
}

2
Tests/LibWeb/Text/expected/ContentSecurityPolicy/script-duplicate-nonce-attribute.txt Normal file
View file

@ -0,0 +1,2 @@
PASS: duplicate nonce script blocked
PASS: valid nonce script executed

15
Tests/LibWeb/Text/input/ContentSecurityPolicy/script-duplicate-nonce-attribute.html Normal file
View file

@ -0,0 +1,15 @@
<!DOCTYPE html>
<script src="../include.js"></script>
<meta http-equiv="Content-Security-Policy" content="script-src 'nonce-good'">
<script nonce=good nonce=bad>
globalThis.duplicateNonceScriptExecuted = true;
</script>
<script nonce=good>
globalThis.validNonceScriptExecuted = true;
</script>
<script nonce=good>
test(() => {
println(globalThis.duplicateNonceScriptExecuted ? "FAIL: duplicate nonce script executed" : "PASS: duplicate nonce script blocked");
println(globalThis.validNonceScriptExecuted ? "PASS: valid nonce script executed" : "FAIL: valid nonce script blocked");
});
</script>