Meta: Generate Wasm tests using wasm-tools instead of wabt

This commit is contained in:
Ali Mohammad Pur 2025-12-04 02:20:37 +01:00 committed by Ali Mohammad Pur
parent b93c17e5e7
commit cf7767395f
4 changed files with 420 additions and 41 deletions

View file

@ -15,7 +15,7 @@ if(INCLUDE_WASM_SPEC_TESTS)
set(SKIP_PRETTIER true)
endif()
find_program(WAT2WASM wat2wasm REQUIRED)
find_program(WASMTOOLS wasm-tools REQUIRED)
find_program(PRETTIER prettier OPTIONAL)
if (NOT SKIP_PRETTIER AND PRETTIER EQUAL "PRETTIER-NOTFOUND")
message(FATAL_ERROR "Prettier required to format Wasm spec tests! Install prettier or set WASM_SPEC_TEST_SKIP_FORMATTING to ON")

View file

@ -7,8 +7,21 @@ from dataclasses import dataclass
from pathlib import Path
from typing import Any
from typing import Literal
from typing import Optional
from typing import Union
TEST_MODULES_TO_SKIP: set[str] = set()
TESTS_TO_SKIP: set[str] = set()
with open(Path(__file__).parent / "wasm_unimplemented_tests.txt", "r") as f:
all_skipped_tests = (x.strip() for x in f.readlines() if not x.startswith("#"))
for test in all_skipped_tests:
if test.startswith("module "):
TEST_MODULES_TO_SKIP.add(test[len("module ") :] + ".wasm")
elif test.startswith("test "):
TESTS_TO_SKIP.add(test[len("test ") :])
class ParseException(Exception):
pass
@ -30,7 +43,18 @@ class WasmVector:
num_bits: int
WasmValue = Union[WasmPrimitiveValue, WasmVector]
@dataclass
class WasmGCValue:
kind: str
value: Optional[str] = None
@dataclass
class EitherOf:
options: list["WasmValue"]
WasmValue = Union[WasmPrimitiveValue, WasmVector, WasmGCValue, EitherOf]
@dataclass
@ -77,6 +101,12 @@ class AssertTrap:
action: Action
@dataclass
class AssertException:
line: int
action: Action
@dataclass
class ActionCommand:
line: int
@ -86,7 +116,7 @@ class ActionCommand:
@dataclass
class AssertInvalid:
line: int
filename: str
filename: Path
message: str
@ -96,6 +126,7 @@ Command = Union[
AssertTrap,
ActionCommand,
AssertInvalid,
AssertException,
Register,
]
@ -116,7 +147,24 @@ class GeneratedVector:
num_bits: int
GeneratedValue = Union[str, ArithmeticNan, CanonicalNan, GeneratedVector]
@dataclass
class GeneratedEitherOf:
options: list["GeneratedValue"]
@dataclass
class GeneratedAnyFuncRef:
pass
GeneratedValue = Union[
str,
ArithmeticNan,
CanonicalNan,
GeneratedVector,
GeneratedEitherOf,
GeneratedAnyFuncRef,
]
@dataclass
@ -134,13 +182,33 @@ class Context:
def parse_value(arg: dict[str, str]) -> WasmValue:
type_ = arg["type"]
match type_:
case "i32" | "i64" | "f32" | "f64" | "externref" | "funcref":
case "i32" | "i64" | "f32" | "f64":
return WasmPrimitiveValue(type_, arg["value"])
case "externref" | "funcref":
return WasmPrimitiveValue(type_, arg["value"] if "value" in arg else None)
case "refnull":
return WasmPrimitiveValue("externref", "null")
case "nullfuncref":
return WasmPrimitiveValue("funcref", "null")
case "v128":
if not isinstance(arg["value"], list):
raise ParseException("Got unknown type for Wasm value")
num_bits = int(arg["lane_type"][1:])
return WasmVector(arg["value"], num_bits)
case (
"arrayref"
| "structref"
| "eqref"
| "anyref"
| "i31ref"
| "exnref"
| "nullref"
| "nullexnref"
| "nullexternref"
):
return WasmGCValue(type_, arg.get("value"))
case "either":
return EitherOf([parse_value(opt) for opt in arg["values"]])
case _:
raise ParseException(f"Unknown value type: {type_}")
@ -159,14 +227,27 @@ def parse_action(action: dict[str, Any]) -> Action:
raise ParseException(f"Action not implemented: {action['type']}")
def module_binary_filename(raw_cmd: dict[str, str]) -> Path:
return Path(raw_cmd["filename"] if raw_cmd.get("module_type") != "text" else raw_cmd["binary_filename"])
def parse(raw: dict[str, Any]) -> WastDescription:
commands: list[Command] = []
defined_modules: dict[str, Path] = {}
for raw_cmd in raw["commands"]:
line = raw_cmd["line"]
cmd: Command
match raw_cmd["type"]:
case "module":
cmd = ModuleCommand(line, Path(raw_cmd["filename"]), raw_cmd.get("name"))
cmd = ModuleCommand(line, module_binary_filename(raw_cmd), raw_cmd.get("name"))
case "module_definition":
if "name" in raw_cmd:
defined_modules[raw_cmd["name"]] = module_binary_filename(raw_cmd)
continue
else:
cmd = ModuleCommand(line, module_binary_filename(raw_cmd), None)
case "module_instance":
cmd = ModuleCommand(line, defined_modules[raw_cmd["module"]], raw_cmd.get("instance"))
case "action":
cmd = ActionCommand(line, parse_action(raw_cmd["action"]))
case "register":
@ -175,14 +256,16 @@ def parse(raw: dict[str, Any]) -> WastDescription:
cmd = AssertReturn(
line,
parse_action(raw_cmd["action"]),
parse_value(raw_cmd["expected"][0]) if len(raw_cmd["expected"]) == 1 else None,
(parse_value(raw_cmd["expected"][0]) if len(raw_cmd["expected"]) == 1 else None),
)
case "assert_trap" | "assert_exhaustion":
cmd = AssertTrap(line, raw_cmd["text"], parse_action(raw_cmd["action"]))
case "assert_invalid" | "assert_malformed" | "assert_uninstantiable" | "assert_unlinkable":
if raw_cmd.get("module_type") == "text":
continue
cmd = AssertInvalid(line, raw_cmd["filename"], raw_cmd["text"])
cmd = AssertInvalid(line, module_binary_filename(raw_cmd), raw_cmd["text"])
case "assert_exception":
cmd = AssertException(line, parse_action(raw_cmd["action"]))
case _:
raise ParseException(f"Unknown command type: {raw_cmd['type']}")
commands.append(cmd)
@ -197,7 +280,15 @@ def escape(s: str) -> str:
def make_description(input_path: Path, name: str, out_path: Path) -> WastDescription:
out_json_path = out_path / f"{name}.json"
result = subprocess.run(
["wast2json", "--enable-all", input_path, f"--output={out_json_path}", "--no-check"],
[
"wasm-tools",
"json-from-wast",
input_path,
"-o",
out_json_path,
"--wasm-dir",
str(out_path),
],
)
result.check_returncode()
with open(out_json_path, "r") as f:
@ -205,9 +296,18 @@ def make_description(input_path: Path, name: str, out_path: Path) -> WastDescrip
return parse(description)
def to_vector_element(value: str, bits: int, addition: str) -> str:
if value.isdigit():
return value + addition
if value.startswith("-") and value[1:].isdigit():
unsigned_value = (1 << bits) + int(value)
return str(unsigned_value) + addition
return f'"{value}"'
def gen_vector(vec: WasmVector, *, array=False) -> str:
addition = "n" if vec.num_bits == 64 else ""
vals = ", ".join(v + addition if v.isdigit() else f'"{v}"' for v in vec.lanes)
vals = ", ".join(to_vector_element(v, vec.num_bits, addition) for v in vec.lanes)
if not array:
type_ = "BigUint64Array" if vec.num_bits == 64 else f"Uint{vec.num_bits}Array"
return f"new {type_}([{vals}])"
@ -215,9 +315,15 @@ def gen_vector(vec: WasmVector, *, array=False) -> str:
def gen_value_arg(value: WasmValue) -> str:
if isinstance(value, WasmGCValue):
return "null"
if isinstance(value, WasmVector):
return gen_vector(value)
if isinstance(value, EitherOf):
raise AssertionError("EitherOf should not appear here")
def unsigned_to_signed(uint: int, bits: int) -> int:
max_value = 2**bits
if uint >= 2 ** (bits - 1):
@ -242,7 +348,7 @@ def gen_value_arg(value: WasmValue) -> str:
f = int_to_float64_bitcast(bits) if double else int_to_float_bitcast(bits)
return str(f)
if value.value.startswith("nan"):
if value.value is not None and value.value.startswith("nan"):
raise GenerateException("Should not get indeterminate nan value as an argument")
if value.value == "inf":
return "Infinity"
@ -268,6 +374,12 @@ def gen_value_result(value: WasmValue) -> GeneratedValue:
if isinstance(value, WasmVector):
return GeneratedVector(gen_vector(value, array=True), value.num_bits)
if isinstance(value, EitherOf):
return GeneratedEitherOf([gen_value_result(option) for option in value.options])
if value.kind == "funcref" and value.value is None:
return GeneratedAnyFuncRef()
if (value.kind == "f32" or value.kind == "f64") and value.value.startswith("nan"):
num_bits = int(value.kind[1:])
match value.value:
@ -284,6 +396,18 @@ def gen_args(args: list[WasmValue]) -> str:
return ",".join(gen_value_arg(arg) for arg in args)
def gen_test_command_for_module(file_name):
if str(file_name) in TEST_MODULES_TO_SKIP:
return "_test.skip"
return "_test"
def gen_test_command_for_invoke(module_name):
if module_name in TESTS_TO_SKIP:
return "_test.skip"
return "_test"
def gen_module_command(command: ModuleCommand, ctx: Context):
if ctx.has_unclosed:
print("});")
@ -295,7 +419,7 @@ try {{
content = readBinaryWasmFile("Fixtures/SpecTests/{command.file_name}");
module = parseWebAssemblyModule(content, globalImportObject);
}} catch (e) {{
_test("parse", () => expect().fail(e));
{gen_test_command_for_module(command.file_name)}("parse (line {command.line})", () => expect().fail(e));
_test = test.skip;
_test.skip = test.skip;
}}
@ -316,12 +440,12 @@ def gen_invalid(invalid: AssertInvalid, ctx: Context):
if ctx.has_unclosed:
print("});")
ctx.has_unclosed = False
stem = Path(invalid.filename).stem
stem = invalid.filename.stem
print(
f"""
describe("{stem}", () => {{
let _test = test;
_test("parse of {stem} (line {invalid.line})", () => {{
{gen_test_command_for_module(invalid.filename)}("parse of {stem} (line {invalid.line})", () => {{
content = readBinaryWasmFile("Fixtures/SpecTests/{invalid.filename}");
expect(() => parseWebAssemblyModule(content, globalImportObject)).toThrow(Error, "{invalid.message}");
}});
@ -333,6 +457,67 @@ def gen_pretty_expect(expr: str, got: str, expect: str):
print(f"if (!{expr}) {{ expect().fail(`Failed with ${{{got}}}, expected {expect}`); }}")
def gen_expectation(gen_result: GeneratedValue, module: str):
match gen_result:
case str():
print(f"expect(_result).toBe({gen_result});")
case GeneratedAnyFuncRef():
print(f"/* {gen_result} */ ", end="")
gen_pretty_expect(
f"isValidFuncrefIn(_result, {module})",
"_result",
"(ref.func)",
)
case ArithmeticNan():
print(f"/* {gen_result} */ ", end="")
gen_pretty_expect(
f"isArithmeticNaN{gen_result.num_bits}(_result)",
"_result",
"nan:arithmetic",
)
case CanonicalNan():
print(f"/* {gen_result} */ ", end="")
gen_pretty_expect(
f"isCanonicalNaN{gen_result.num_bits}(_result)",
"_result",
"nan:canonical",
)
case GeneratedVector():
if gen_result.num_bits == 64:
array = "new BigUint64Array(_result)"
else:
array = f"new Uint{gen_result.num_bits}Array(_result)"
print(f"/* {gen_result} */ ", end="")
gen_pretty_expect(
f"testSIMDVector({gen_result.repr}, {array})",
array,
gen_result.repr,
)
case GeneratedEitherOf():
print("let matched = false;")
print("let error_sample = null;")
expectations = []
for option in gen_result.options:
print("try {")
gen_expectation(option, module)
print("matched = true;")
print("} catch (e) { error_sample = e; }")
expectation = "unknown"
match option:
case str():
expectation = option
case ArithmeticNan():
expectation = "nan:arithmetic"
case CanonicalNan():
expectation = "nan:canonical"
case GeneratedVector():
expectation = option.repr
expectations.append(expectation)
print(
f"if (!matched) {{ expect().fail(`Expected one of {', '.join(expectations)}, got ${{_result}}: ${{error_sample}}`); }}"
)
def gen_invoke(
line: int,
invoke: Invoke,
@ -348,7 +533,7 @@ def gen_invoke(
module = f'namedModules["{invoke.module}"]'
utf8 = str(invoke.field.encode("utf8"))[2:-1].replace("\\'", "'").replace("`", "${'`'}")
print(
f"""_test(`execution of {ctx.current_module_name}: {utf8} (line {line})`, () => {{
f"""{gen_test_command_for_invoke(ctx.current_module_name)}(`execution of {ctx.current_module_name}: {utf8} (line {line})`, () => {{
let _field = {module}.getExport(decodeURIComponent(escape(`{utf8}`)));
expect(_field).not.toBeUndefined();"""
)
@ -358,31 +543,7 @@ expect(_field).not.toBeUndefined();"""
print(f"let _result = {module}.invoke(_field, {gen_args(invoke.args)});")
if result is not None:
gen_result = gen_value_result(result)
match gen_result:
case str():
print(f"expect(_result).toBe({gen_result});")
case ArithmeticNan():
gen_pretty_expect(
f"isArithmeticNaN{gen_result.num_bits}(_result)",
"_result",
"nan:arithmetic",
)
case CanonicalNan():
gen_pretty_expect(
f"isCanonicalNaN{gen_result.num_bits}(_result)",
"_result",
"nan:canonical",
)
case GeneratedVector():
if gen_result.num_bits == 64:
array = "new BigUint64Array(_result)"
else:
array = f"new Uint{gen_result.num_bits}Array(_result)"
gen_pretty_expect(
f"testSIMDVector({gen_result.repr}, {array})",
array,
gen_result.repr,
)
gen_expectation(gen_result, module)
print("});")
if not ctx.has_unclosed:
print("});")
@ -393,7 +554,7 @@ def gen_get(line: int, get: Get, result: WasmValue | None, ctx: Context):
if get.module is not None:
module = f'namedModules["{get.module}"]'
print(
f"""_test("execution of {ctx.current_module_name}: get-{get.field} (line {line})", () => {{
f"""{gen_test_command_for_invoke(ctx.current_module_name)}("execution of {ctx.current_module_name}: get-{get.field} (line {line})", () => {{
let _field = {module}.getExport("{get.field}");"""
)
if result is not None:
@ -431,6 +592,10 @@ def gen_command(command: Command, ctx: Context):
if not isinstance(command.action, Invoke):
raise GenerateException(f"Not implemented: {type(command.action)}")
gen_invoke(command.line, command.action, None, ctx, fail_msg=command.messsage)
case AssertException():
if not isinstance(command.action, Invoke):
raise GenerateException(f"Not implemented: {type(command.action)}")
gen_invoke(command.line, command.action, None, ctx, fail_msg="exception")
def generate(description: WastDescription):

View file

@ -0,0 +1,204 @@
module array.0
module array.12
module array.2
module array.5
module array.6
module array.7
module array.8
module array_copy.4
module array_fill.3
module array_init_data.2
module array_init_data.3
module array_init_elem.3
module array_new_data.0
module array_new_data.1
module array_new_data.2
module array_new_data.3
module array_new_data.4
module array_new_elem.0
module array_new_elem.1
module array_new_elem.2
module array_new_elem.3
module br_on_cast.0
module br_on_cast.1
module br_on_cast.2
module br_on_cast_fail.0
module br_on_cast_fail.1
module br_on_cast_fail.2
module br_on_non_null.0
module br_on_non_null.1
module br_on_non_null.2
module br_on_null.0
module br_on_null.1
module br_on_null.2
module br_table.0
module call_ref.0
module call_ref.1
module call_ref.2
module call_ref.3
module elem.2
module elem.31
module elem.47
module elem.48
module elem.49
module elem.50
module elem.51
module elem.52
module elem.53
module elem.54
module elem.57
module elem.58
module elem.59
module elem.60
module elem.61
module elem.62
module extern.0
module func.21
module global.50
module i31.0
module i31.1
module i31.3
module i31.4
module i31.5
module i31.6
module instance.0
module instance.1
module instance.2
module instance.3
module instance.4
module linking.10
module linking.9
module local_init.0
module local_init.1
module local_init.2
module local_init.4
module local_init.5
module ref.1
module ref.12
module ref.2
module ref.4
module ref.5
module ref.6
module ref.8
module ref_as_non_null.0
module ref_as_non_null.2
module ref_cast.0
module ref_cast.1
module ref_eq.0
module ref_is_null.0
module ref_null.0
module ref_null.1
module ref_test.0
module ref_test.1
module relaxed_dot_product.0
module relaxed_madd_nmadd.0
module return_call_ref.0
module return_call_ref.1
module return_call_ref.10
module return_call_ref.8
module return_call_ref.9
module select.0
module struct.0
module struct.10
module struct.2
module struct.5
module struct.7
module struct.9
module table-sub.0
module table.13
module table.14
module table.15
module table.29
module table.30
module table.31
module table.32
module table.33
module table.34
module table.35
module table.36
module table64.11
module tag.2
module tag.4
module tag.5
module throw.0
module throw_ref.0
module try_table.1
module try_table.13
module try_table.2
module try_table.5
module type-canon.0
module type-canon.1
module type-equivalence.14
module type-equivalence.15
module type-equivalence.16
module type-equivalence.17
module type-equivalence.18
module type-equivalence.19
module type-equivalence.2
module type-equivalence.20
module type-equivalence.21
module type-equivalence.4
module type-equivalence.5
module type-equivalence.7
module type-equivalence.8
module type-equivalence.9
module type-rec.0
module type-rec.1
module type-rec.13
module type-rec.14
module type-rec.17
module type-rec.18
module type-rec.19
module type-rec.20
module type-rec.3
module type-rec.4
module type-rec.7
module type-rec.8
module type-subtyping.0
module type-subtyping.1
module type-subtyping.11
module type-subtyping.12
module type-subtyping.13
module type-subtyping.14
module type-subtyping.17
module type-subtyping.18
module type-subtyping.19
module type-subtyping.2
module type-subtyping.20
module type-subtyping.21
module type-subtyping.22
module type-subtyping.23
module type-subtyping.24
module type-subtyping.25
module type-subtyping.26
module type-subtyping.27
module type-subtyping.28
module type-subtyping.29
module type-subtyping.3
module type-subtyping.30
module type-subtyping.34
module type-subtyping.37
module type-subtyping.38
module type-subtyping.39
module type-subtyping.4
module type-subtyping.40
module type-subtyping.41
module type-subtyping.43
module type-subtyping.44
module type-subtyping.45
module type-subtyping.46
module type-subtyping.47
module type-subtyping.48
module type-subtyping.49
module type-subtyping.5
module type-subtyping.50
module type-subtyping.51
module type-subtyping.53
module type-subtyping.6
module type-subtyping.7
module type-subtyping.76
module type-subtyping.8
module type-subtyping.9
module unreached-valid.0
module unreached-valid.2
test local_init.5

View file

@ -236,6 +236,16 @@ TESTJS_GLOBAL_FUNCTION(is_arithmetic_nan64, isArithmeticNaN64)
return (bits & 0x7FF0000000000000ull) == 0x7FF0000000000000ull && payload >= 0x0008000000000000ull;
}
TESTJS_GLOBAL_FUNCTION(is_valid_funcref_in, isValidFuncrefIn)
{
auto value = TRY(vm.argument(0).to_index(vm));
auto module_object = TRY(vm.argument(1).to_object(vm));
if (!is<WebAssemblyModule>(*module_object))
return vm.throw_completion<JS::TypeError>("Expected a WebAssemblyModule"sv);
auto& module = static_cast<WebAssemblyModule&>(*module_object);
return JS::Value(module.machine().store().get(Wasm::FunctionAddress { value }) != nullptr);
}
TESTJS_GLOBAL_FUNCTION(test_simd_vector, testSIMDVector)
{
auto expected = TRY(vm.argument(0).to_object(vm));