dmaax/ladybird
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Tests: Add a CSS Tokenizer test suite

Browse source 
Based on test-js-ast and test-js-bytecode, test-css-tokenizer finds CSS
files in CSSTokenizer/input, tokenizes them, and then dumps the output,
to compare against files in the expected/ directory. Our Tokenizer is
in a good state currently, but this will let us track regressions, and
also allow us to rewrite this in Rust and ensure that the output
matches.

To support test-css-tokenizer.py, we also have css-tokenizer.cpp which
exposes the Tokenizer, which is usually only used internally.

We'll want to expand this test suite over time, but this initial set of
tests covers each type of Token.
This commit is contained in:
Sam Atkins 2026-04-10 12:14:15 +01:00
parent 5385d90e3c
commit 59a860249b
35 changed files with 438 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
Meta/check-newlines-at-eof.py
View file

@ -13,6 +13,8 @@ def should_check_file(filename):
return False
if filename.startswith("Tests/LibWeb/Crash/"):
return False
if filename.startswith("Tests/LibWeb/CSSTokenizer/"):
return False
if filename.startswith("Tests/LibWeb/Layout/"):
return False
if filename.startswith("Tests/LibWeb/Ref/"):

13
Tests/LibWeb/CMakeLists.txt
View file

@ -19,8 +19,21 @@ foreach(source IN LISTS TEST_SOURCES)
ladybird_test("${source}" LibWeb LIBS LibWeb)
endforeach()
lagom_utility(css-tokenizer SOURCES css-tokenizer.cpp LIBS LibFileSystem LibMain LibWeb)
target_link_libraries(TestContentFilter PRIVATE LibURL)
target_link_libraries(TestFetchURL PRIVATE LibURL)
target_link_libraries(TestSourceHighlighter PRIVATE LibURL LibWebView)
if (NOT WIN32)
add_custom_target(test-css-tokenizer ALL DEPENDS css-tokenizer "${CMAKE_BINARY_DIR}/bin/test-css-tokenizer")
add_custom_command(
OUTPUT "${CMAKE_BINARY_DIR}/bin/test-css-tokenizer"
COMMAND "${CMAKE_COMMAND}" -E copy "${ladybird_SOURCE_DIR}/Tests/LibWeb/test-css-tokenizer.py" "${CMAKE_BINARY_DIR}/bin/test-css-tokenizer"
DEPENDS "${ladybird_SOURCE_DIR}/Tests/LibWeb/test-css-tokenizer.py"
)
add_test(NAME test-css-tokenizer COMMAND "${CMAKE_BINARY_DIR}/bin/test-css-tokenizer")
set_tests_properties(test-css-tokenizer PROPERTIES ENVIRONMENT LADYBIRD_SOURCE_DIR=${LADYBIRD_SOURCE_DIR})
endif()
add_subdirectory("test-web")

40
Tests/LibWeb/CSSTokenizer/expected/at-hash-function-delim.txt Normal file
View file

@ -0,0 +1,40 @@
AtKeyword(value="media", source="@media", start=0:0, end=0:6)
Whitespace(source=" ", start=0:6, end=0:7)
Ident(value="screen", source="screen", start=0:7, end=0:13)
Whitespace(source=" ", start=0:13, end=0:14)
Ident(value="and", source="and", start=0:14, end=0:17)
Whitespace(source=" ", start=0:17, end=0:18)
OpenParen(source="(", start=0:18, end=0:19)
Ident(value="color", source="color", start=0:19, end=0:24)
CloseParen(source=")", start=0:24, end=0:25)
Whitespace(source=" ", start=0:25, end=0:26)
OpenCurly(source="{", start=0:26, end=0:27)
Whitespace(source="\n ", start=0:27, end=1:2)
Hash(value="id", hash_type=Id, source="#id", start=1:2, end=1:5)
Delim(value=".", code_point=U+002E, source=".", start=1:5, end=1:6)
Ident(value="class", source="class", start=1:6, end=1:11)
Whitespace(source=" ", start=1:11, end=1:12)
OpenCurly(source="{", start=1:12, end=1:13)
Whitespace(source=" ", start=1:13, end=1:14)
Ident(value="background", source="background", start=1:14, end=1:24)
Colon(source=":", start=1:24, end=1:25)
Whitespace(source=" ", start=1:25, end=1:26)
Function(value="rgb", source="rgb(", start=1:26, end=1:30)
Number(value=1, number_type=Integer, source="1", start=1:30, end=1:31)
Comma(source=",", start=1:31, end=1:32)
Whitespace(source=" ", start=1:32, end=1:33)
Number(value=2, number_type=Integer, source="2", start=1:33, end=1:34)
Comma(source=",", start=1:34, end=1:35)
Whitespace(source=" ", start=1:35, end=1:36)
Number(value=3, number_type=Integer, source="3", start=1:36, end=1:37)
CloseParen(source=")", start=1:37, end=1:38)
Whitespace(source=" ", start=1:38, end=1:39)
Delim(value="!", code_point=U+0021, source="!", start=1:39, end=1:40)
Ident(value="important", source="important", start=1:40, end=1:49)
Semicolon(source=";", start=1:49, end=1:50)
Whitespace(source=" ", start=1:50, end=1:51)
CloseCurly(source="}", start=1:51, end=1:52)
Whitespace(source="\n", start=1:52, end=2:0)
CloseCurly(source="}", start=2:0, end=2:1)
Whitespace(source="\n", start=2:1, end=3:0)
__EOF__(source="", start=3:0, end=3:0)

13
Tests/LibWeb/CSSTokenizer/expected/basic-rules.txt Normal file
View file

@ -0,0 +1,13 @@
Ident(value="body", source="body", start=0:0, end=0:4)
Whitespace(source=" ", start=0:4, end=0:5)
OpenCurly(source="{", start=0:5, end=0:6)
Whitespace(source=" ", start=0:6, end=0:7)
Ident(value="color", source="color", start=0:7, end=0:12)
Colon(source=":", start=0:12, end=0:13)
Whitespace(source=" ", start=0:13, end=0:14)
Ident(value="red", source="red", start=0:14, end=0:17)
Semicolon(source=";", start=0:17, end=0:18)
Whitespace(source=" ", start=0:18, end=0:19)
CloseCurly(source="}", start=0:19, end=0:20)
Whitespace(source="\n", start=0:20, end=1:0)
__EOF__(source="", start=1:0, end=1:0)

10
Tests/LibWeb/CSSTokenizer/expected/brackets-and-cdo-cdc.txt Normal file
View file

@ -0,0 +1,10 @@
CDO(source="<!--", start=0:0, end=0:4)
Whitespace(source=" ", start=0:4, end=0:5)
OpenSquare(source="[", start=0:5, end=0:6)
OpenParen(source="(", start=0:6, end=0:7)
CloseParen(source=")", start=0:7, end=0:8)
CloseSquare(source="]", start=0:8, end=0:9)
Whitespace(source=" ", start=0:9, end=0:10)
CDC(source="-->", start=0:10, end=0:13)
Whitespace(source="\n", start=0:13, end=1:0)
__EOF__(source="", start=1:0, end=1:0)

16
Tests/LibWeb/CSSTokenizer/expected/comments-and-whitespace.txt Normal file
View file

@ -0,0 +1,16 @@
Whitespace(source=" ", start=0:0, end=0:2)
Whitespace(source="/* comment */", start=0:2, end=0:15)
Whitespace(source="\n", start=0:15, end=1:0)
Delim(value=".", code_point=U+002E, source=".", start=1:0, end=1:1)
Ident(value="foo", source="foo", start=1:1, end=1:4)
Whitespace(source=" \n", start=1:4, end=2:0)
OpenCurly(source="{", start=2:0, end=2:1)
Whitespace(source="\n\t", start=2:1, end=3:1)
Ident(value="color", source="color", start=3:1, end=3:6)
Colon(source=":", start=3:6, end=3:7)
Whitespace(source=" ", start=3:7, end=3:8)
Ident(value="blue", source="blue", start=3:8, end=3:12)
Whitespace(source="\n", start=3:12, end=4:0)
CloseCurly(source="}", start=4:0, end=4:1)
Whitespace(source="\n", start=4:1, end=5:0)
__EOF__(source="", start=5:0, end=5:0)

18
Tests/LibWeb/CSSTokenizer/expected/malformed-bad-string-recovery.txt Normal file
View file

@ -0,0 +1,18 @@
Ident(value="a", source="a", start=0:0, end=0:1)
Whitespace(source=" ", start=0:1, end=0:2)
OpenCurly(source="{", start=0:2, end=0:3)
Whitespace(source="\n ", start=0:3, end=1:2)
Ident(value="content", source="content", start=1:2, end=1:9)
Colon(source=":", start=1:9, end=1:10)
Whitespace(source=" ", start=1:10, end=1:11)
BadString(source="\"hello", start=1:11, end=1:17)
Whitespace(source="\n ", start=1:17, end=2:2)
Ident(value="color", source="color", start=2:2, end=2:7)
Colon(source=":", start=2:7, end=2:8)
Whitespace(source=" ", start=2:8, end=2:9)
Ident(value="red", source="red", start=2:9, end=2:12)
Semicolon(source=";", start=2:12, end=2:13)
Whitespace(source="\n", start=2:13, end=3:0)
CloseCurly(source="}", start=3:0, end=3:1)
Whitespace(source="\n", start=3:1, end=4:0)
__EOF__(source="", start=4:0, end=4:0)

11
Tests/LibWeb/CSSTokenizer/expected/malformed-bad-url-recovery.txt Normal file
View file

@ -0,0 +1,11 @@
Ident(value="a", source="a", start=0:0, end=0:1)
OpenCurly(source="{", start=0:1, end=0:2)
Ident(value="background", source="background", start=0:2, end=0:12)
Colon(source=":", start=0:12, end=0:13)
BadUrl(source="url(bad'broken)", start=0:13, end=0:28)
Ident(value="color", source="color", start=0:28, end=0:33)
Colon(source=":", start=0:33, end=0:34)
Ident(value="red", source="red", start=0:34, end=0:37)
CloseCurly(source="}", start=0:37, end=0:38)
Whitespace(source="\n", start=0:38, end=1:0)
__EOF__(source="", start=1:0, end=1:0)

14
Tests/LibWeb/CSSTokenizer/expected/malformed-invalid-escape-extra-braces.txt Normal file
View file

@ -0,0 +1,14 @@
Ident(value="a", source="a", start=0:0, end=0:1)
OpenCurly(source="{", start=0:1, end=0:2)
Whitespace(source="\n", start=0:2, end=1:0)
Delim(value="\\", code_point=U+005C, source="\\", start=1:0, end=1:1)
Whitespace(source="\n", start=1:1, end=2:0)
Ident(value="color", source="color", start=2:0, end=2:5)
Colon(source=":", start=2:5, end=2:6)
Ident(value="red", source="red", start=2:6, end=2:9)
Semicolon(source=";", start=2:9, end=2:10)
Whitespace(source="\n", start=2:10, end=3:0)
CloseCurly(source="}", start=3:0, end=3:1)
CloseCurly(source="}", start=3:1, end=3:2)
Whitespace(source="\n", start=3:2, end=4:0)
__EOF__(source="", start=4:0, end=4:0)

9
Tests/LibWeb/CSSTokenizer/expected/malformed-unterminated-comment.txt Normal file
View file

@ -0,0 +1,9 @@
Ident(value="a", source="a", start=0:0, end=0:1)
OpenCurly(source="{", start=0:1, end=0:2)
Ident(value="color", source="color", start=0:2, end=0:7)
Colon(source=":", start=0:7, end=0:8)
Ident(value="red", source="red", start=0:8, end=0:11)
Semicolon(source=";", start=0:11, end=0:12)
Whitespace(source="/* unterminated comment", start=0:12, end=0:35)
Whitespace(source="\n", start=0:35, end=1:0)
__EOF__(source="", start=1:0, end=1:0)

7
Tests/LibWeb/CSSTokenizer/expected/malformed-unterminated-string-eof.txt Normal file
View file

@ -0,0 +1,7 @@
Ident(value="a", source="a", start=0:0, end=0:1)
OpenCurly(source="{", start=0:1, end=0:2)
Ident(value="content", source="content", start=0:2, end=0:9)
Colon(source=":", start=0:9, end=0:10)
BadString(source="\"unterminated", start=0:10, end=0:23)
Whitespace(source="\n", start=0:23, end=1:0)
__EOF__(source="", start=1:0, end=1:0)

30
Tests/LibWeb/CSSTokenizer/expected/numeric-tokens.txt Normal file
View file

@ -0,0 +1,30 @@
Delim(value=".", code_point=U+002E, source=".", start=0:0, end=0:1)
Ident(value="a", source="a", start=0:1, end=0:2)
Whitespace(source=" ", start=0:2, end=0:3)
OpenCurly(source="{", start=0:3, end=0:4)
Whitespace(source=" ", start=0:4, end=0:5)
Ident(value="width", source="width", start=0:5, end=0:10)
Colon(source=":", start=0:10, end=0:11)
Whitespace(source=" ", start=0:11, end=0:12)
Percentage(value=1.25, number_type=Number, source="+12.5e-1%", start=0:12, end=0:21)
Semicolon(source=";", start=0:21, end=0:22)
Whitespace(source=" ", start=0:22, end=0:23)
Ident(value="margin", source="margin", start=0:23, end=0:29)
Colon(source=":", start=0:29, end=0:30)
Whitespace(source=" ", start=0:30, end=0:31)
Dimension(value=-3, number_type=IntegerWithExplicitSign, unit="px", source="-3px", start=0:31, end=0:35)
Whitespace(source=" ", start=0:35, end=0:36)
Number(value=0, number_type=Integer, source="0", start=0:36, end=0:37)
Whitespace(source=" ", start=0:37, end=0:38)
Dimension(value=120, number_type=Number, unit="rem", source="1.2e+2rem", start=0:38, end=0:47)
Semicolon(source=";", start=0:47, end=0:48)
Whitespace(source=" ", start=0:48, end=0:49)
Ident(value="z-index", source="z-index", start=0:49, end=0:56)
Colon(source=":", start=0:56, end=0:57)
Whitespace(source=" ", start=0:57, end=0:58)
Number(value=7, number_type=Integer, source="7", start=0:58, end=0:59)
Semicolon(source=";", start=0:59, end=0:60)
Whitespace(source=" ", start=0:60, end=0:61)
CloseCurly(source="}", start=0:61, end=0:62)
Whitespace(source="\n", start=0:62, end=1:0)
__EOF__(source="", start=1:0, end=1:0)

20
Tests/LibWeb/CSSTokenizer/expected/strings-and-escapes.txt Normal file
View file

@ -0,0 +1,20 @@
OpenSquare(source="[", start=0:0, end=0:1)
Ident(value="data-name", source="data-name", start=0:1, end=0:10)
Delim(value="=", code_point=U+003D, source="=", start=0:10, end=0:11)
String(value="a&b", source="\"a\\26 b\"", start=0:11, end=0:19)
CloseSquare(source="]", start=0:19, end=0:20)
Colon(source=":", start=0:20, end=0:21)
Colon(source=":", start=0:21, end=0:22)
Ident(value="before", source="before", start=0:22, end=0:28)
Whitespace(source=" ", start=0:28, end=0:29)
OpenCurly(source="{", start=0:29, end=0:30)
Whitespace(source=" ", start=0:30, end=0:31)
Ident(value="content", source="content", start=0:31, end=0:38)
Colon(source=":", start=0:38, end=0:39)
Whitespace(source=" ", start=0:39, end=0:40)
String(value="hello!", source="'hello\\000021'", start=0:40, end=0:54)
Semicolon(source=";", start=0:54, end=0:55)
Whitespace(source=" ", start=0:55, end=0:56)
CloseCurly(source="}", start=0:56, end=0:57)
Whitespace(source="\n", start=0:57, end=1:0)
__EOF__(source="", start=1:0, end=1:0)

19
Tests/LibWeb/CSSTokenizer/expected/urls-and-bad-url.txt Normal file
View file

@ -0,0 +1,19 @@
Ident(value="a", source="a", start=0:0, end=0:1)
Whitespace(source=" ", start=0:1, end=0:2)
OpenCurly(source="{", start=0:2, end=0:3)
Whitespace(source=" ", start=0:3, end=0:4)
Ident(value="background", source="background", start=0:4, end=0:14)
Colon(source=":", start=0:14, end=0:15)
Whitespace(source=" ", start=0:15, end=0:16)
Url(value="foo.png", source="url(foo.png)", start=0:16, end=0:28)
Semicolon(source=";", start=0:28, end=0:29)
Whitespace(source=" ", start=0:29, end=0:30)
Ident(value="mask", source="mask", start=0:30, end=0:34)
Colon(source=":", start=0:34, end=0:35)
Whitespace(source=" ", start=0:35, end=0:36)
BadUrl(source="url(bad' )", start=0:36, end=0:46)
Semicolon(source=";", start=0:46, end=0:47)
Whitespace(source=" ", start=0:47, end=0:48)
CloseCurly(source="}", start=0:48, end=0:49)
Whitespace(source="\n", start=0:49, end=1:0)
__EOF__(source="", start=1:0, end=1:0)

8
Tests/LibWeb/CSSTokenizer/expected/utf-16le-crlf.txt Normal file
View file

@ -0,0 +1,8 @@
Ident(value="a", source="a", start=0:0, end=0:1)
OpenCurly(source="{", start=0:1, end=0:2)
Whitespace(source="\n", start=0:2, end=1:0)
Ident(value="b", source="b", start=1:0, end=1:1)
Colon(source=":", start=1:1, end=1:2)
Number(value=1, number_type=Integer, source="1", start=1:2, end=1:3)
CloseCurly(source="}", start=1:3, end=1:4)
__EOF__(source="", start=1:4, end=1:4)

15
Tests/LibWeb/CSSTokenizer/expected/windows-1252-curly-quotes.txt Normal file
View file

@ -0,0 +1,15 @@
Ident(value="p", source="p", start=0:0, end=0:1)
Colon(source=":", start=0:1, end=0:2)
Colon(source=":", start=0:2, end=0:3)
Ident(value="before", source="before", start=0:3, end=0:9)
Whitespace(source=" ", start=0:9, end=0:10)
OpenCurly(source="{", start=0:10, end=0:11)
Whitespace(source=" ", start=0:11, end=0:12)
Ident(value="content", source="content", start=0:12, end=0:19)
Colon(source=":", start=0:19, end=0:20)
Whitespace(source=" ", start=0:20, end=0:21)
Ident(value="“x”", source="“x”", start=0:21, end=0:24)
Semicolon(source=";", start=0:24, end=0:25)
Whitespace(source=" ", start=0:25, end=0:26)
CloseCurly(source="}", start=0:26, end=0:27)
__EOF__(source="", start=0:27, end=0:27)

3
Tests/LibWeb/CSSTokenizer/input/at-hash-function-delim.css Normal file
View file

@ -0,0 +1,3 @@
@media screen and (color) {
#id.class { background: rgb(1, 2, 3) !important; }
}

1
Tests/LibWeb/CSSTokenizer/input/basic-rules.css Normal file
View file

@ -0,0 +1 @@
body { color: red; }

1
Tests/LibWeb/CSSTokenizer/input/brackets-and-cdo-cdc.css Normal file
View file

@ -0,0 +1 @@
<!-- [()] -->

5
Tests/LibWeb/CSSTokenizer/input/comments-and-whitespace.css Normal file
View file

@ -0,0 +1,5 @@
/* comment */
.foo
{
color: blue
}

4
Tests/LibWeb/CSSTokenizer/input/malformed-bad-string-recovery.css Normal file
View file

@ -0,0 +1,4 @@
a {
content: "hello
color: red;
}

1
Tests/LibWeb/CSSTokenizer/input/malformed-bad-url-recovery.css Normal file
View file

@ -0,0 +1 @@
a{background:url(bad'broken)color:red}

4
Tests/LibWeb/CSSTokenizer/input/malformed-invalid-escape-extra-braces.css Normal file
View file

@ -0,0 +1,4 @@
a{
\
color:red;
}}

1
Tests/LibWeb/CSSTokenizer/input/malformed-unterminated-comment.css Normal file
View file

@ -0,0 +1 @@
a{color:red;/* unterminated comment

1
Tests/LibWeb/CSSTokenizer/input/malformed-unterminated-string-eof.css Normal file
View file

@ -0,0 +1 @@
a{content:"unterminated

1
Tests/LibWeb/CSSTokenizer/input/numeric-tokens.css Normal file
View file

@ -0,0 +1 @@
.a { width: +12.5e-1%; margin: -3px 0 1.2e+2rem; z-index: 7; }

1
Tests/LibWeb/CSSTokenizer/input/strings-and-escapes.css Normal file
View file

@ -0,0 +1 @@
[data-name="a\26 b"]::before { content: 'hello\000021'; }

1
Tests/LibWeb/CSSTokenizer/input/urls-and-bad-url.css Normal file
View file

@ -0,0 +1 @@
a { background: url(foo.png); mask: url(bad' ); }

BIN
Tests/LibWeb/CSSTokenizer/input/utf-16le-crlf.css Normal file
View file

Binary file not shown.

1
Tests/LibWeb/CSSTokenizer/input/utf-16le-crlf.css.encoding Normal file
View file

@ -0,0 +1 @@
utf-16le

1
Tests/LibWeb/CSSTokenizer/input/windows-1252-curly-quotes.css Normal file
View file

@ -0,0 +1 @@
p::before { content: “x”; }

1
Tests/LibWeb/CSSTokenizer/input/windows-1252-curly-quotes.css.encoding Normal file
View file

@ -0,0 +1 @@
windows-1252

1
Tests/LibWeb/CSSTokenizer/output/.gitignore vendored Normal file
View file

@ -0,0 +1 @@
*

31
Tests/LibWeb/css-tokenizer.cpp Normal file
View file

@ -0,0 +1,31 @@
/*
* Copyright (c) 2026, Sam Atkins <sam@ladybird.org>
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#include <AK/Format.h>
#include <LibCore/ArgsParser.h>
#include <LibCore/File.h>
#include <LibMain/Main.h>
#include <LibWeb/CSS/Parser/Tokenizer.h>
ErrorOr<int> ladybird_main(Main::Arguments arguments)
{
StringView encoding = "utf-8"sv;
StringView input_path;
Core::ArgsParser args_parser;
args_parser.add_option(encoding, "Source encoding label", "encoding", 'e', "encoding");
args_parser.add_positional_argument(input_path, "Path to the CSS input file", "input", Core::ArgsParser::Required::Yes);
args_parser.parse(arguments);
auto file = TRY(Core::File::open(input_path, Core::File::OpenMode::Read));
auto input = TRY(file->read_until_eof());
auto tokens = Web::CSS::Parser::Tokenizer::tokenize(input, encoding);
for (auto const& token : tokens)
outln("{}", token.to_debug_string());
return 0;
}

134
Tests/LibWeb/test-css-tokenizer.py Executable file
View file

@ -0,0 +1,134 @@
#!/usr/bin/env python3
import difflib
import os
import subprocess
import sys
from argparse import ArgumentParser
from concurrent.futures import ThreadPoolExecutor
from concurrent.futures import as_completed
from pathlib import Path
LADYBIRD_SOURCE_DIR: Path
CSS_TOKENIZER_TEST_DIR: Path
BUILD_DIR: Path
def setup() -> None:
global LADYBIRD_SOURCE_DIR, CSS_TOKENIZER_TEST_DIR, BUILD_DIR
ladybird_source_dir = os.getenv("LADYBIRD_SOURCE_DIR")
if ladybird_source_dir is None:
print("LADYBIRD_SOURCE_DIR must be set!")
sys.exit(1)
LADYBIRD_SOURCE_DIR = Path(ladybird_source_dir)
CSS_TOKENIZER_TEST_DIR = LADYBIRD_SOURCE_DIR / "Tests/LibWeb/CSSTokenizer/"
# The script is copied to bin/test-css-tokenizer, so the build dir is one level up
BUILD_DIR = Path(__file__).parent.parent.resolve()
DIFF_PREFIX_ESCAPES = {
"@": "\x1b[36m",
"+": "\x1b[32m",
"-": "\x1b[31m",
}
def diff(a: str, a_file: Path, b: str, b_file: Path) -> None:
for line in difflib.unified_diff(a.splitlines(), b.splitlines(), fromfile=str(a_file), tofile=str(b_file)):
line = line.rstrip()
color_prefix = DIFF_PREFIX_ESCAPES.get((line or " ")[0], "")
print(f"{color_prefix}{line}\x1b[0m")
def encoding_for(file: Path) -> str:
encoding_file = CSS_TOKENIZER_TEST_DIR / "input" / Path(f"{file.name}.encoding")
if not encoding_file.exists():
return "utf-8"
return encoding_file.read_text(encoding="utf8").strip()
def test(file: Path, rebaseline: bool) -> bool:
args = [
str(BUILD_DIR / "bin/css-tokenizer"),
"--encoding",
encoding_for(file),
str(CSS_TOKENIZER_TEST_DIR / "input" / file),
]
process = subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
stdout = process.stdout.decode().strip()
if process.returncode != 0:
print(stdout)
sys.exit(1)
expected_file = CSS_TOKENIZER_TEST_DIR / "expected" / file.with_suffix(".txt")
output_file = CSS_TOKENIZER_TEST_DIR / "output" / file.with_suffix(".txt")
output_file.write_text(stdout + "\n", encoding="utf8")
if rebaseline:
expected_file.write_text(stdout + "\n", encoding="utf8")
return False
expected = expected_file.read_text(encoding="utf8").strip()
if stdout != expected:
print(f"\nCSS tokens do not match for {file}!\n")
diff(a=expected, a_file=expected_file, b=stdout, b_file=output_file)
return True
return False
def main() -> int:
setup()
parser = ArgumentParser()
parser.add_argument("-j", "--jobs", type=int)
parser.add_argument("--rebaseline", action="store_true")
args = parser.parse_args()
input_dir = CSS_TOKENIZER_TEST_DIR / "input"
failed = 0
css_files = [
css_file for css_file in sorted(input_dir.iterdir()) if css_file.is_file() and css_file.suffix == ".css"
]
with ThreadPoolExecutor(max_workers=args.jobs) as executor:
executables = [
executor.submit(test, css_file.relative_to(input_dir), args.rebaseline) for css_file in css_files
]
for executable in as_completed(executables):
if executable.result():
failed += 1
total = len(css_files)
passed = total - failed
if args.rebaseline:
print(f"Rebaselined {total} tests.")
return 0
if failed:
print(f"\nTests: {passed} passed, {failed} failed, {total} total")
return 1
print(f"All tests passed! ({total} total)")
return 0
if __name__ == "__main__":
sys.exit(main())