dmaax/ladybird
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Tests: Support WPT reftests with multiple references

Browse source 
Store reftest expectations as a list of match/mismatch reference URLs
and evaluate each reference in sequence.

This lets test-web run WPT reftests that specify multiple reference
relations, such as tests with both match and mismatch links.
This commit is contained in:
Shannon Booth 2026-06-13 08:56:22 +02:00 committed by Shannon Booth
parent de90aeb07c
commit a8cb98ccf7
2 changed files with 69 additions and 61 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
Tests/LibWeb/test-web/TestWeb.h
View file

@ -72,11 +72,15 @@ constexpr StringView test_result_to_string(TestResult result)
}
enum class RefTestExpectationType {
None, // Test did not specify an expection file
Match,
Mismatch,
};
struct RefTestExpectation {
RefTestExpectationType type;
URL::URL url;
};
struct Test {
TestMode mode;
@ -98,8 +102,8 @@ struct Test {
bool did_finish_loading { false };
bool did_inject_js { false };
RefTestExpectationType ref_test_expectation_type {};
Optional<URL::URL> ref_test_expectation_url {};
Vector<RefTestExpectation> ref_test_expectations {};
size_t ref_test_expectation_index { 0 };
Vector<FuzzyMatch> fuzzy_matches {};
RefPtr<Gfx::Bitmap const> actual_screenshot {};

120
Tests/LibWeb/test-web/main.cpp
View file

@ -627,45 +627,35 @@ static ErrorOr<void> write_screenshot_failure_results(Test& test, Gfx::Bitmap co
return {};
}
static ErrorOr<bool> compare_current_ref_test_expectation(Test& test, URL::URL const& test_url)
{
VERIFY(test.ref_test_expectation_index < test.ref_test_expectations.size());
auto const& expectation = test.ref_test_expectations[test.ref_test_expectation_index];
auto should_match = expectation.type == RefTestExpectationType::Match;
auto screenshot_matches = fuzzy_screenshot_match(test_url, expectation.url, *test.actual_screenshot,
*test.expectation_screenshot, test.fuzzy_matches, should_match);
if (should_match == screenshot_matches)
return true;
TRY(write_screenshot_failure_results(test, *test.actual_screenshot, *test.expectation_screenshot));
return false;
}
static void run_ref_test(TestWebView& view, TestRunContext& context, Test& test, URL::URL const& url)
{
auto test_index = test.index;
auto handle_completed_test = [&view, &context, test_index, url]() -> ErrorOr<TestResult> {
auto& test = context.tests[test_index];
switch (test.ref_test_expectation_type) {
case RefTestExpectationType::None:
return TestResult::Fail;
case RefTestExpectationType::Match:
case RefTestExpectationType::Mismatch: {
auto should_match = test.ref_test_expectation_type == RefTestExpectationType::Match;
auto screenshot_matches = fuzzy_screenshot_match(url, view.url(), *test.actual_screenshot,
*test.expectation_screenshot, test.fuzzy_matches, should_match);
if (should_match == screenshot_matches)
return TestResult::Pass;
TRY(write_screenshot_failure_results(test, *test.actual_screenshot, *test.expectation_screenshot));
return TestResult::Fail;
}
}
VERIFY_NOT_REACHED();
};
auto on_test_complete = [&view, test_index, handle_completed_test]() {
if (auto result = handle_completed_test(); result.is_error())
view.on_test_complete({ test_index, TestResult::Fail });
else
view.on_test_complete({ test_index, result.value() });
};
view.on_load_finish = [&view, &context, test_index, url](auto const& loaded_url) {
auto& test = context.tests[test_index];
// We don't want subframe loads to trigger this.
if (test.ref_test_expectation_url.has_value()) {
if (!test.ref_test_expectations.is_empty()) {
VERIFY(test.ref_test_expectation_index < test.ref_test_expectations.size());
// Match against the expectation URL.
if (!test.ref_test_expectation_url->equals(loaded_url, URL::ExcludeFragment::Yes))
auto const& expectation_url = test.ref_test_expectations[test.ref_test_expectation_index].url;
if (!expectation_url.equals(loaded_url, URL::ExcludeFragment::Yes))
return;
} else {
// Match against the test URL.
@ -679,14 +669,30 @@ static void run_ref_test(TestWebView& view, TestRunContext& context, Test& test,
}
};
view.on_test_finish = [&view, &context, test_index, on_test_complete = move(on_test_complete)](auto const&) {
view.on_test_finish = [&view, &context, test_index, url](auto const&) {
auto& test = context.tests[test_index];
if (test.actual_screenshot) {
// The reference has finished loading; take another screenshot and move on to handling the result.
view.take_screenshot()->when_resolved([&view, &context, test_index, on_test_complete = move(on_test_complete)](RefPtr<Gfx::Bitmap const> screenshot) {
context.tests[test_index].expectation_screenshot = move(screenshot);
view.take_screenshot()->when_resolved([&view, &context, test_index, url](RefPtr<Gfx::Bitmap const> screenshot) {
auto& test = context.tests[test_index];
test.expectation_screenshot = move(screenshot);
view.reset_zoom();
on_test_complete();
auto expectation_passed = compare_current_ref_test_expectation(test, url);
if (expectation_passed.is_error() || !expectation_passed.value()) {
view.on_test_complete({ test_index, TestResult::Fail });
return;
}
++test.ref_test_expectation_index;
if (test.ref_test_expectation_index >= test.ref_test_expectations.size()) {
view.on_test_complete({ test_index, TestResult::Pass });
return;
}
test.expectation_screenshot.clear();
test.did_inject_js = false;
view.load(test.ref_test_expectations[test.ref_test_expectation_index].url);
});
} else {
// When the test initially finishes, we take a screenshot and request the reference test metadata.
@ -698,7 +704,7 @@ static void run_ref_test(TestWebView& view, TestRunContext& context, Test& test,
}
};
view.on_reference_test_metadata = [&view, &context, test_index, on_test_complete = move(on_test_complete)](JsonValue const& metadata) {
view.on_reference_test_metadata = [&view, &context, test_index](JsonValue const& metadata) {
auto& test = context.tests[test_index];
auto metadata_object = metadata.as_object();
@ -726,33 +732,31 @@ static void run_ref_test(TestWebView& view, TestRunContext& context, Test& test,
test.fuzzy_matches.append(fuzzy_match_or_error.release_value());
}
// Read (mis)match reference tests to load.
// FIXME: Currently we only support single match or mismatch reference.
String reference_to_load;
if (!match_references->is_empty()) {
if (match_references->size() > 1)
dbgln("FIXME: Only a single ref test match reference is supported");
test.ref_test_expectations.clear_with_capacity();
test.ref_test_expectation_index = 0;
test.ref_test_expectation_type = RefTestExpectationType::Match;
reference_to_load = match_references->at(0).as_string();
} else if (!mismatch_references->is_empty()) {
if (mismatch_references->size() > 1)
dbgln("FIXME: Only a single ref test mismatch reference is supported");
auto append_references = [&](auto const& references, RefTestExpectationType type) {
for (size_t i = 0; i < references.size(); ++i) {
auto reference_url = URL::Parser::basic_parse(references.at(i).as_string());
if (!reference_url.has_value()) {
warnln("Failed to parse ref test reference URL '{}'", references.at(i).as_string());
continue;
}
test.ref_test_expectations.append({ type, reference_url.release_value() });
}
};
append_references(*match_references, RefTestExpectationType::Match);
append_references(*mismatch_references, RefTestExpectationType::Mismatch);
test.ref_test_expectation_type = RefTestExpectationType::Mismatch;
reference_to_load = mismatch_references->at(0).as_string();
}
if (test.ref_test_expectation_type == RefTestExpectationType::None) {
// Skip loading the expectation page entirely since none was specified
if (test.ref_test_expectations.is_empty()) {
warnln("Test '{}' does not specify an expectation file (e.g. '<link rel=\"match\" href=\"...\" />' tag), test will fail.", test.input_path);
on_test_complete();
} else {
// Clear flag so we can inject the JS into the reference page.
test.ref_test_expectation_url = URL::Parser::basic_parse(reference_to_load).release_value();
test.did_inject_js = false;
view.load(test.ref_test_expectation_url.value());
view.on_test_complete({ test_index, TestResult::Fail });
return;
}
// Clear flag so we can inject the JS into the reference page.
test.did_inject_js = false;
view.load(test.ref_test_expectations[test.ref_test_expectation_index].url);
};
view.load(url);