LibWasm: Implement call argument forwarding using call records
This commit is contained in:
Ali Mohammad Pur
Ali Mohammad Pur
parent
f180d90c20
commit
921373a045
13 changed files with 546 additions and 136 deletions
|
|
@ -188,6 +188,7 @@ InstantiationResult AbstractMachine::instantiate(Module const& module, Vector<Ex
|
|||
return InstantiationError { ByteString::formatted("Validation failed: {}", result.error()) };
|
||||
|
||||
auto main_module_instance_pointer = make<ModuleInstance>();
|
||||
main_module_instance_pointer->cached_minimum_call_record_allocation_size = module.minimum_call_record_allocation_size();
|
||||
auto& main_module_instance = *main_module_instance_pointer;
|
||||
|
||||
main_module_instance.types() = module.type_section().types();
|
||||
|
|
@ -196,6 +197,8 @@ InstantiationResult AbstractMachine::instantiate(Module const& module, Vector<Ex
|
|||
Vector<Vector<Reference>> elements;
|
||||
ModuleInstance auxiliary_instance;
|
||||
|
||||
auxiliary_instance.cached_minimum_call_record_allocation_size = module.minimum_call_record_allocation_size();
|
||||
|
||||
for (auto [i, import_] : enumerate(module.import_section().imports())) {
|
||||
auto extern_ = externs.at(i);
|
||||
auto invalid = import_.description().visit(
|
||||
|
|
@ -294,7 +297,6 @@ InstantiationResult AbstractMachine::instantiate(Module const& module, Vector<Ex
|
|||
config.set_frame(IsTailcall::No,
|
||||
auxiliary_instance,
|
||||
Vector<Value, ArgumentsStaticSize> {},
|
||||
Vector<Value, 8> {},
|
||||
entry.expression(),
|
||||
1);
|
||||
auto result = config.execute(interpreter);
|
||||
|
|
@ -316,7 +318,6 @@ InstantiationResult AbstractMachine::instantiate(Module const& module, Vector<Ex
|
|||
config.set_frame(IsTailcall::No,
|
||||
main_module_instance,
|
||||
Vector<Value, ArgumentsStaticSize> {},
|
||||
Vector<Value, 8> {},
|
||||
entry,
|
||||
entry.instructions().size() - 1);
|
||||
auto result = config.execute(interpreter);
|
||||
|
|
@ -351,7 +352,6 @@ InstantiationResult AbstractMachine::instantiate(Module const& module, Vector<Ex
|
|||
config.set_frame(IsTailcall::No,
|
||||
main_module_instance,
|
||||
Vector<Value, ArgumentsStaticSize> {},
|
||||
Vector<Value, 8> {},
|
||||
active_ptr->expression,
|
||||
1);
|
||||
auto result = config.execute(interpreter);
|
||||
|
|
@ -386,7 +386,6 @@ InstantiationResult AbstractMachine::instantiate(Module const& module, Vector<Ex
|
|||
config.set_frame(IsTailcall::No,
|
||||
main_module_instance,
|
||||
Vector<Value, ArgumentsStaticSize> {},
|
||||
Vector<Value, 8> {},
|
||||
data.offset,
|
||||
1);
|
||||
auto result = config.execute(interpreter);
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@
|
|||
|
||||
namespace Wasm {
|
||||
|
||||
constexpr inline size_t ArgumentsStaticSize = 8;
|
||||
constexpr inline size_t ArgumentsStaticSize = 3;
|
||||
|
||||
class Configuration;
|
||||
class Result;
|
||||
|
|
@ -320,8 +320,9 @@ private:
|
|||
class ModuleInstance {
|
||||
public:
|
||||
explicit ModuleInstance(
|
||||
Vector<FunctionType> types, Vector<FunctionAddress> function_addresses, Vector<TableAddress> table_addresses, Vector<MemoryAddress> memory_addresses, Vector<GlobalAddress> global_addresses, Vector<DataAddress> data_addresses, Vector<TagAddress> tag_addresses, Vector<TagType> tag_types, Vector<ExportInstance> exports)
|
||||
: m_types(move(types))
|
||||
Vector<FunctionType> types, Vector<FunctionAddress> function_addresses, Vector<TableAddress> table_addresses, Vector<MemoryAddress> memory_addresses, Vector<GlobalAddress> global_addresses, Vector<DataAddress> data_addresses, Vector<TagAddress> tag_addresses, Vector<TagType> tag_types, Vector<ExportInstance> exports, size_t minimum_call_record_allocation_size)
|
||||
: cached_minimum_call_record_allocation_size(minimum_call_record_allocation_size)
|
||||
, m_types(move(types))
|
||||
, m_tag_types(move(tag_types))
|
||||
, m_functions(move(function_addresses))
|
||||
, m_tables(move(table_addresses))
|
||||
|
|
@ -357,6 +358,8 @@ public:
|
|||
auto& tags() { return m_tags; }
|
||||
auto& tag_types() { return m_tag_types; }
|
||||
|
||||
size_t cached_minimum_call_record_allocation_size { 0 };
|
||||
|
||||
private:
|
||||
Vector<FunctionType> m_types;
|
||||
Vector<TagType> m_tag_types;
|
||||
|
|
@ -674,9 +677,8 @@ private:
|
|||
|
||||
class Frame {
|
||||
public:
|
||||
explicit Frame(ModuleInstance const& module, Vector<Value, ArgumentsStaticSize> arguments, Vector<Value, 8> locals, Expression const& expression, size_t arity)
|
||||
explicit Frame(ModuleInstance const& module, Vector<Value, ArgumentsStaticSize> locals, Expression const& expression, size_t arity)
|
||||
: m_module(module)
|
||||
, m_arguments(move(arguments))
|
||||
, m_locals(move(locals))
|
||||
, m_expression(expression)
|
||||
, m_arity(arity)
|
||||
|
|
@ -686,24 +688,14 @@ public:
|
|||
auto& module() const { return m_module; }
|
||||
auto& locals() const { return m_locals; }
|
||||
auto& locals() { return m_locals; }
|
||||
auto& arguments() const { return m_arguments; }
|
||||
auto& arguments() { return m_arguments; }
|
||||
auto& expression() const { return m_expression; }
|
||||
auto arity() const { return m_arity; }
|
||||
auto label_index() const { return m_label_index; }
|
||||
auto& label_index() { return m_label_index; }
|
||||
|
||||
Value& local_or_argument(LocalIndex index)
|
||||
{
|
||||
if (index.value() & LocalArgumentMarker)
|
||||
return m_arguments[index.value() & ~LocalArgumentMarker];
|
||||
return m_locals[index.value()];
|
||||
}
|
||||
|
||||
private:
|
||||
ModuleInstance const& m_module;
|
||||
Vector<Value, ArgumentsStaticSize> m_arguments;
|
||||
Vector<Value, 8> m_locals;
|
||||
Vector<Value, ArgumentsStaticSize> m_locals;
|
||||
Expression const& m_expression;
|
||||
size_t m_arity { 0 };
|
||||
size_t m_label_index { 0 };
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@
|
|||
#include <AK/ByteReader.h>
|
||||
#include <AK/Debug.h>
|
||||
#include <AK/Endian.h>
|
||||
#include <AK/GenericShorthands.h>
|
||||
#include <AK/MemoryStream.h>
|
||||
#include <AK/NumericLimits.h>
|
||||
#include <AK/QuickSort.h>
|
||||
|
|
@ -172,12 +173,12 @@ static_assert(sizeof(ShortenedIPAndAddresses) == sizeof(u64));
|
|||
static Outcome operator()(HANDLER_PARAMS(DECOMPOSE_PARAMS)); \
|
||||
}; \
|
||||
template<bool HasDynamicInsnLimit, typename Continue, SourceAddressMix source_address_mix> \
|
||||
Outcome InstructionHandler<Instructions::name.value()>::operator()(HANDLER_PARAMS(DECOMPOSE_PARAMS))
|
||||
FLATTEN Outcome InstructionHandler<Instructions::name.value()>::operator()(HANDLER_PARAMS(DECOMPOSE_PARAMS))
|
||||
#define ALIAS_INSTRUCTION(new_name, existing_name) \
|
||||
template<> \
|
||||
struct InstructionHandler<Instructions::new_name.value()> { \
|
||||
template<bool HasDynamicInsnLimit, typename Continue, SourceAddressMix source_address_mix> \
|
||||
static Outcome operator()(HANDLER_PARAMS(DECOMPOSE_PARAMS)) \
|
||||
FLATTEN static Outcome operator()(HANDLER_PARAMS(DECOMPOSE_PARAMS)) \
|
||||
{ \
|
||||
TAILCALL return InstructionHandler<Instructions::existing_name.value()>::operator()<HasDynamicInsnLimit, Continue, source_address_mix>( \
|
||||
HANDLER_PARAMS(DECOMPOSE_PARAMS_NAME_ONLY)); \
|
||||
|
|
@ -1475,7 +1476,7 @@ HANDLE_INSTRUCTION(local_set)
|
|||
{
|
||||
LOG_INSN;
|
||||
// bounds checked by verifier.
|
||||
configuration.local_or_argument(instruction->local_index()) = configuration.take_source<source_address_mix>(0, ip_and_addresses.addresses.sources);
|
||||
configuration.local(instruction->local_index()) = configuration.take_source<source_address_mix>(0, ip_and_addresses.addresses.sources);
|
||||
TAILCALL return continue_(HANDLER_PARAMS(DECOMPOSE_PARAMS_NAME_ONLY));
|
||||
}
|
||||
|
||||
|
|
@ -1639,6 +1640,28 @@ HANDLE_INSTRUCTION(call)
|
|||
TAILCALL return continue_(HANDLER_PARAMS(DECOMPOSE_PARAMS_NAME_ONLY));
|
||||
}
|
||||
|
||||
HANDLE_INSTRUCTION(synthetic_call_with_record_0)
|
||||
{
|
||||
LOG_INSN;
|
||||
auto index = instruction->arguments().get<FunctionIndex>();
|
||||
auto address = configuration.frame().module().functions()[index.value()];
|
||||
dbgln_if(WASM_TRACE_DEBUG, "call.with_record.0({})", address.value());
|
||||
if (interpreter.call_address(configuration, address, ip_and_addresses.addresses, BytecodeInterpreter::CallAddressSource::DirectCall, BytecodeInterpreter::CallType::UsingCallRecord) == Outcome::Return)
|
||||
return Outcome::Return;
|
||||
TAILCALL return continue_(HANDLER_PARAMS(DECOMPOSE_PARAMS_NAME_ONLY));
|
||||
}
|
||||
|
||||
HANDLE_INSTRUCTION(synthetic_call_with_record_1)
|
||||
{
|
||||
LOG_INSN;
|
||||
auto index = instruction->arguments().get<FunctionIndex>();
|
||||
auto address = configuration.frame().module().functions()[index.value()];
|
||||
dbgln_if(WASM_TRACE_DEBUG, "call.with_record.1({})", address.value());
|
||||
if (interpreter.call_address(configuration, address, ip_and_addresses.addresses, BytecodeInterpreter::CallAddressSource::DirectCall, BytecodeInterpreter::CallType::UsingCallRecord) == Outcome::Return)
|
||||
return Outcome::Return;
|
||||
TAILCALL return continue_(HANDLER_PARAMS(DECOMPOSE_PARAMS_NAME_ONLY));
|
||||
}
|
||||
|
||||
HANDLE_INSTRUCTION(return_call)
|
||||
{
|
||||
LOG_INSN;
|
||||
|
|
@ -1913,7 +1936,17 @@ HANDLE_INSTRUCTION(local_tee)
|
|||
auto value = configuration.source_value<source_address_mix>(0, ip_and_addresses.addresses.sources); // bounds checked by verifier.
|
||||
auto local_index = instruction->local_index();
|
||||
dbgln_if(WASM_TRACE_DEBUG, "stack:peek -> locals({})", local_index.value());
|
||||
configuration.frame().local_or_argument(local_index) = value;
|
||||
configuration.local(local_index) = value;
|
||||
TAILCALL return continue_(HANDLER_PARAMS(DECOMPOSE_PARAMS_NAME_ONLY));
|
||||
}
|
||||
|
||||
HANDLE_INSTRUCTION(synthetic_argument_tee)
|
||||
{
|
||||
LOG_INSN;
|
||||
auto value = configuration.source_value<source_address_mix>(0, ip_and_addresses.addresses.sources); // bounds checked by verifier.
|
||||
auto local_index = instruction->local_index();
|
||||
dbgln_if(WASM_TRACE_DEBUG, "stack:peek -> locals({})", local_index.value());
|
||||
configuration.local(local_index) = value;
|
||||
TAILCALL return continue_(HANDLER_PARAMS(DECOMPOSE_PARAMS_NAME_ONLY));
|
||||
}
|
||||
|
||||
|
|
@ -4668,7 +4701,7 @@ Outcome BytecodeInterpreter::call_address(Configuration& configuration, Function
|
|||
{
|
||||
Optional<ScopedValueRollback<decltype(configuration.regs)>> regs_rollback;
|
||||
|
||||
if (call_type == CallType::UsingRegisters)
|
||||
if (call_type == CallType::UsingRegisters || call_type == CallType::UsingCallRecord)
|
||||
regs_rollback = ScopedValueRollback { configuration.regs };
|
||||
|
||||
auto instance = configuration.store().get(address);
|
||||
|
|
@ -4678,22 +4711,28 @@ Outcome BytecodeInterpreter::call_address(Configuration& configuration, Function
|
|||
TRAP_IF_NOT(type->parameters().size() <= configuration.value_stack().size());
|
||||
}
|
||||
Vector<Value, ArgumentsStaticSize> args;
|
||||
configuration.get_arguments_allocation_if_possible(args, type->parameters().size());
|
||||
|
||||
{
|
||||
auto param_count = type->parameters().size();
|
||||
if (param_count) {
|
||||
args.ensure_capacity(param_count);
|
||||
if (call_type == CallType::UsingRegisters) {
|
||||
args.resize_with_default_value(param_count, Value(0));
|
||||
for (size_t i = 0; i < param_count; ++i)
|
||||
args[param_count - i - 1] = configuration.take_source<SourceAddressMix::Any>(i, addresses.sources);
|
||||
} else {
|
||||
auto span = configuration.value_stack().span().slice_from_end(param_count);
|
||||
for (auto& value : span)
|
||||
args.unchecked_append(value);
|
||||
if (call_type == CallType::UsingCallRecord) {
|
||||
configuration.take_call_record(args);
|
||||
args.shrink(type->parameters().size(), true);
|
||||
} else {
|
||||
configuration.get_arguments_allocation_if_possible(args, type->parameters().size());
|
||||
|
||||
configuration.value_stack().remove(configuration.value_stack().size() - span.size(), span.size());
|
||||
{
|
||||
auto param_count = type->parameters().size();
|
||||
if (param_count) {
|
||||
args.ensure_capacity(param_count);
|
||||
if (call_type == CallType::UsingRegisters) {
|
||||
args.resize_and_keep_capacity(param_count);
|
||||
for (size_t i = 0; i < param_count; ++i)
|
||||
args[param_count - i - 1] = configuration.take_source<SourceAddressMix::Any>(i, addresses.sources);
|
||||
} else {
|
||||
auto span = configuration.value_stack().span().slice_from_end(param_count);
|
||||
for (auto& value : span)
|
||||
args.unchecked_append(value);
|
||||
|
||||
configuration.value_stack().remove(configuration.value_stack().size() - span.size(), span.size());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -4730,7 +4769,7 @@ Outcome BytecodeInterpreter::call_address(Configuration& configuration, Function
|
|||
}
|
||||
|
||||
if (!result.values().is_empty()) {
|
||||
if (call_type == CallType::UsingRegisters) {
|
||||
if (call_type == CallType::UsingRegisters || call_type == CallType::UsingCallRecord || result.values().size() == 1) {
|
||||
configuration.push_to_destination<SourceAddressMix::Any>(result.values().take_first(), addresses.destination);
|
||||
} else {
|
||||
configuration.value_stack().ensure_capacity(configuration.value_stack().size() + result.values().size());
|
||||
|
|
@ -4888,6 +4927,8 @@ CompiledInstructions try_compile_instructions(Expression const& expression, Span
|
|||
} pattern_state { InsnPatternState::Nothing };
|
||||
static Instruction nop { Instructions::nop };
|
||||
|
||||
size_t calls_in_expression = 0;
|
||||
|
||||
auto const set_default_dispatch = [&result](Instruction const& instruction, size_t index = NumericLimits<size_t>::max()) {
|
||||
if (index < result.dispatches.size()) {
|
||||
result.dispatches[index] = { { .instruction_opcode = instruction.opcode() }, &instruction };
|
||||
|
|
@ -4910,6 +4951,8 @@ CompiledInstructions try_compile_instructions(Expression const& expression, Span
|
|||
set_default_dispatch(result.extra_instruction_storage.unsafe_last());
|
||||
continue;
|
||||
}
|
||||
|
||||
calls_in_expression++;
|
||||
}
|
||||
|
||||
switch (pattern_state) {
|
||||
|
|
@ -5133,7 +5176,7 @@ CompiledInstructions try_compile_instructions(Expression const& expression, Span
|
|||
result.dispatches.remove_all(nops_to_remove, [](auto const& it) { return it.key(); });
|
||||
result.src_dst_mappings.remove_all(nops_to_remove, [](auto const& it) { return it.key(); });
|
||||
|
||||
// Rewrite local.get of arguments to argument.get to keep local.get for locals only.
|
||||
// Rewrite local.* of arguments to argument.* to keep local.* for locals only.
|
||||
for (size_t i = 0; i < result.dispatches.size(); ++i) {
|
||||
auto& dispatch = result.dispatches[i];
|
||||
if (dispatch.instruction->opcode() == Instructions::local_get) {
|
||||
|
|
@ -5145,6 +5188,24 @@ CompiledInstructions try_compile_instructions(Expression const& expression, Span
|
|||
result.dispatches[i].instruction = &result.extra_instruction_storage.unsafe_last();
|
||||
result.dispatches[i].instruction_opcode = result.dispatches[i].instruction->opcode();
|
||||
}
|
||||
} else if (dispatch.instruction->opcode() == Instructions::local_set) {
|
||||
auto local_index = dispatch.instruction->local_index();
|
||||
if (local_index.value() & LocalArgumentMarker) {
|
||||
result.extra_instruction_storage.unchecked_append(Instruction(
|
||||
Instructions::synthetic_argument_set,
|
||||
local_index));
|
||||
result.dispatches[i].instruction = &result.extra_instruction_storage.unsafe_last();
|
||||
result.dispatches[i].instruction_opcode = result.dispatches[i].instruction->opcode();
|
||||
}
|
||||
} else if (dispatch.instruction->opcode() == Instructions::local_tee) {
|
||||
auto local_index = dispatch.instruction->local_index();
|
||||
if (local_index.value() & LocalArgumentMarker) {
|
||||
result.extra_instruction_storage.unchecked_append(Instruction(
|
||||
Instructions::synthetic_argument_tee,
|
||||
local_index));
|
||||
result.dispatches[i].instruction = &result.extra_instruction_storage.unsafe_last();
|
||||
result.dispatches[i].instruction_opcode = result.dispatches[i].instruction->opcode();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -5161,6 +5222,7 @@ CompiledInstructions try_compile_instructions(Expression const& expression, Span
|
|||
IP definition_index;
|
||||
Vector<IP> uses;
|
||||
IP last_use = 0;
|
||||
bool was_created_as_a_result_of_polymorphic_stack = false;
|
||||
};
|
||||
|
||||
struct ActiveReg {
|
||||
|
|
@ -5230,6 +5292,20 @@ CompiledInstructions try_compile_instructions(Expression const& expression, Span
|
|||
Vector<Vector<ValueID>> live_at_instr;
|
||||
live_at_instr.resize(result.dispatches.size());
|
||||
|
||||
// Track call record constraints
|
||||
HashMap<ValueID, u8> value_to_callrec_slot;
|
||||
|
||||
struct CallInfo {
|
||||
size_t call_index;
|
||||
size_t param_count;
|
||||
size_t result_count;
|
||||
size_t earliest_arg_index;
|
||||
Vector<ValueID> arg_values;
|
||||
};
|
||||
Vector<CallInfo> eligible_calls;
|
||||
|
||||
eligible_calls.ensure_capacity(calls_in_expression);
|
||||
|
||||
for (size_t i = 0; i < result.dispatches.size(); ++i) {
|
||||
auto& dispatch = result.dispatches[i];
|
||||
auto opcode = dispatch.instruction->opcode();
|
||||
|
|
@ -5238,16 +5314,16 @@ CompiledInstructions try_compile_instructions(Expression const& expression, Span
|
|||
Vector<ValueID> dependent_ids;
|
||||
|
||||
bool variadic_or_unknown = false;
|
||||
bool requires_aliased_destination = true;
|
||||
|
||||
switch (opcode.value()) {
|
||||
#define M(name, _, ins, outs) \
|
||||
case Instructions::name.value(): \
|
||||
if constexpr (ins == -1 || outs == -1) { \
|
||||
variadic_or_unknown = true; \
|
||||
} else { \
|
||||
inputs = ins; \
|
||||
outputs = outs; \
|
||||
} \
|
||||
inputs = max(ins, 0); \
|
||||
outputs = max(outs, 0); \
|
||||
break;
|
||||
ENUMERATE_WASM_OPCODES(M)
|
||||
#undef M
|
||||
|
|
@ -5255,6 +5331,113 @@ CompiledInstructions try_compile_instructions(Expression const& expression, Span
|
|||
|
||||
Vector<ValueID> input_ids;
|
||||
|
||||
if (opcode == Instructions::call) {
|
||||
auto& type = functions[dispatch.instruction->arguments().get<FunctionIndex>().value()];
|
||||
|
||||
if (type.parameters().size() <= (Dispatch::LastCallRecord - Dispatch::CallRecord + 1)
|
||||
&& type.results().size() <= 1
|
||||
&& type.parameters().size() <= value_stack.size()) {
|
||||
|
||||
inputs = type.parameters().size();
|
||||
outputs = type.results().size();
|
||||
variadic_or_unknown = false;
|
||||
requires_aliased_destination = false;
|
||||
|
||||
auto value_stack_copy = value_stack;
|
||||
|
||||
for (size_t j = 0; j < inputs; ++j) {
|
||||
auto input_value = value_stack.take_last();
|
||||
auto& value = values.get(input_value).value();
|
||||
|
||||
// if this value was created as a result of a polymorphic stack,
|
||||
// we can't actually go and force it to a call record again, so disqualify this call.
|
||||
if (value.was_created_as_a_result_of_polymorphic_stack) {
|
||||
inputs = 0;
|
||||
outputs = 0;
|
||||
variadic_or_unknown = true;
|
||||
value_stack = move(value_stack_copy);
|
||||
goto avoid_optimizing_this_call;
|
||||
}
|
||||
|
||||
input_ids.append(input_value);
|
||||
dependent_ids.append(input_value);
|
||||
value.uses.append(i);
|
||||
value.last_use = max(value.last_use, i);
|
||||
forced_stack_values.append(input_value);
|
||||
}
|
||||
instr_to_input_values.set(i, input_ids);
|
||||
instr_to_dependent_values.set(i, dependent_ids);
|
||||
|
||||
for (size_t j = 0; j < outputs; ++j) {
|
||||
auto id = next_value_id++;
|
||||
values.set(id, Value { id, i, {}, i });
|
||||
value_stack.append(id);
|
||||
instr_to_output_value.set(i, id);
|
||||
ensure_id_space(id);
|
||||
}
|
||||
|
||||
size_t earliest = i;
|
||||
ValueID earliest_arg_value = NumericLimits<size_t>::max();
|
||||
for (auto value_id : input_ids) {
|
||||
auto& value = values.get(value_id).value();
|
||||
if (earliest > value.definition_index.value()) {
|
||||
earliest = value.definition_index.value();
|
||||
earliest_arg_value = value_id;
|
||||
}
|
||||
}
|
||||
|
||||
// Reverse the input_ids to match stack order
|
||||
Vector<ValueID> reversed_args;
|
||||
for (size_t j = 0; j < inputs; ++j) {
|
||||
reversed_args.append(input_ids[inputs - 1 - j]);
|
||||
}
|
||||
|
||||
// Follow the alias root of the earliest arg value to find the first instruction that produced it.
|
||||
auto new_earliest = earliest;
|
||||
while (true) {
|
||||
auto maybe_inputs = instr_to_input_values.get(new_earliest);
|
||||
if (!maybe_inputs.has_value())
|
||||
break;
|
||||
bool found_earliest = false;
|
||||
for (auto val : maybe_inputs.value()) {
|
||||
auto root = find_root(val);
|
||||
if (root == find_root(earliest_arg_value)) {
|
||||
auto& value = values.get(val).value();
|
||||
if (value.definition_index.value() < new_earliest) {
|
||||
new_earliest = value.definition_index.value();
|
||||
found_earliest = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!found_earliest)
|
||||
break;
|
||||
}
|
||||
|
||||
eligible_calls.append({ .call_index = i,
|
||||
.param_count = inputs,
|
||||
.result_count = outputs,
|
||||
.earliest_arg_index = new_earliest,
|
||||
.arg_values = reversed_args });
|
||||
|
||||
continue;
|
||||
}
|
||||
}
|
||||
avoid_optimizing_this_call:;
|
||||
|
||||
// Handle the inputs we actually know about.
|
||||
size_t j = 0;
|
||||
for (; j < inputs && !value_stack.is_empty(); ++j) {
|
||||
auto input_value = value_stack.take_last();
|
||||
input_ids.append(input_value);
|
||||
dependent_ids.append(input_value);
|
||||
auto& value = values.get(input_value).value();
|
||||
value.uses.append(i);
|
||||
value.last_use = max(value.last_use, i);
|
||||
}
|
||||
|
||||
inputs -= j;
|
||||
|
||||
if (variadic_or_unknown) {
|
||||
for (auto val : value_stack) {
|
||||
auto& value = values.get(val).value();
|
||||
|
|
@ -5267,7 +5450,7 @@ CompiledInstructions try_compile_instructions(Expression const& expression, Span
|
|||
value_stack.clear_with_capacity();
|
||||
}
|
||||
|
||||
if (!variadic_or_unknown && value_stack.size() < inputs) {
|
||||
if (value_stack.size() < inputs) {
|
||||
size_t j = 0;
|
||||
for (; j < inputs && !value_stack.is_empty(); ++j) {
|
||||
auto input_value = value_stack.take_last();
|
||||
|
|
@ -5280,7 +5463,7 @@ CompiledInstructions try_compile_instructions(Expression const& expression, Span
|
|||
|
||||
for (; j < inputs; ++j) {
|
||||
auto val_id = next_value_id++;
|
||||
values.set(val_id, Value { val_id, i, {}, i });
|
||||
values.set(val_id, Value { val_id, i, {}, i, true });
|
||||
input_ids.append(val_id);
|
||||
forced_stack_values.append(val_id);
|
||||
ensure_id_space(val_id);
|
||||
|
|
@ -5311,7 +5494,7 @@ CompiledInstructions try_compile_instructions(Expression const& expression, Span
|
|||
}
|
||||
|
||||
// Alias the output with the last input, if one exists.
|
||||
if (outputs > 0) {
|
||||
if (outputs > 0 && requires_aliased_destination) {
|
||||
auto maybe_input_ids = instr_to_input_values.get(i);
|
||||
if (maybe_input_ids.has_value() && !maybe_input_ids->is_empty()) {
|
||||
auto last_input_id = maybe_input_ids->last();
|
||||
|
|
@ -5319,6 +5502,12 @@ CompiledInstructions try_compile_instructions(Expression const& expression, Span
|
|||
|
||||
auto alias_root = find_root(last_input_id);
|
||||
|
||||
// If the last input was created as a result of polymorphic stack, propagate that to the output (as they're aliased).
|
||||
auto& output_value = values.get(output_id).value();
|
||||
auto const& input_value = values.get(last_input_id).value();
|
||||
if (input_value.was_created_as_a_result_of_polymorphic_stack)
|
||||
output_value.was_created_as_a_result_of_polymorphic_stack = true;
|
||||
|
||||
// If any *other* input is forced to alias the output, we have no choice but to place all three on the stack.
|
||||
for (size_t j = 0; j < maybe_input_ids->size() - 1; ++j) {
|
||||
auto input_root = find_root((*maybe_input_ids)[j]);
|
||||
|
|
@ -5333,9 +5522,104 @@ CompiledInstructions try_compile_instructions(Expression const& expression, Span
|
|||
|
||||
forced_stack_values.extend(value_stack);
|
||||
|
||||
// Build conflict graph and select maximum set of non-conflicting calls
|
||||
// Prefer calls with more arguments, and among those with equal args, prefer shorter spans
|
||||
|
||||
struct CallScore {
|
||||
size_t index;
|
||||
size_t param_count;
|
||||
size_t span;
|
||||
};
|
||||
|
||||
Vector<CallScore> scored_calls;
|
||||
for (size_t i = 0; i < eligible_calls.size(); ++i) {
|
||||
auto& call = eligible_calls[i];
|
||||
size_t span = call.call_index - call.earliest_arg_index;
|
||||
scored_calls.append({ i, call.param_count, span });
|
||||
}
|
||||
|
||||
// Sort by: more params first, then shorter span
|
||||
quick_sort(scored_calls, [](auto const& a, auto const& b) {
|
||||
if (a.param_count != b.param_count)
|
||||
return a.param_count > b.param_count;
|
||||
return a.span < b.span;
|
||||
});
|
||||
|
||||
// Greedily select non-conflicting calls in priority order
|
||||
Vector<CallInfo*> valid_calls;
|
||||
HashTable<size_t> selected_indices;
|
||||
size_t max_call_record_size = 0;
|
||||
|
||||
for (auto const& score : scored_calls) {
|
||||
auto& call_info = eligible_calls[score.index];
|
||||
size_t call_start = call_info.earliest_arg_index;
|
||||
size_t call_end = call_info.call_index;
|
||||
|
||||
bool conflicts = false;
|
||||
for (auto* other_call : valid_calls) {
|
||||
size_t other_start = other_call->earliest_arg_index;
|
||||
size_t other_end = other_call->call_index;
|
||||
|
||||
// Check if the ranges overlap
|
||||
// Two ranges [a,b] and [c,d] overlap if: NOT (b < c OR d < a)
|
||||
if (!(call_end < other_start || other_end < call_start)) {
|
||||
conflicts = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (!conflicts) {
|
||||
valid_calls.append(&call_info);
|
||||
selected_indices.set(score.index);
|
||||
max_call_record_size = max(max_call_record_size, call_info.param_count);
|
||||
}
|
||||
}
|
||||
|
||||
// Only apply call record optimization to non-conflicting calls
|
||||
HashTable<size_t> calls_with_records;
|
||||
for (auto* call_info : valid_calls) {
|
||||
calls_with_records.set(call_info->call_index);
|
||||
|
||||
// Mark values for call record slots
|
||||
for (size_t j = 0; j < call_info->param_count; ++j) {
|
||||
value_to_callrec_slot.set(call_info->arg_values[j], Dispatch::CallRecord + j);
|
||||
}
|
||||
|
||||
auto new_call_opcode = call_info->result_count == 0
|
||||
? Instructions::synthetic_call_with_record_0
|
||||
: Instructions::synthetic_call_with_record_1;
|
||||
|
||||
auto new_call_insn = Instruction(
|
||||
new_call_opcode,
|
||||
result.dispatches[call_info->call_index].instruction->arguments());
|
||||
|
||||
result.extra_instruction_storage.unchecked_append(new_call_insn);
|
||||
result.dispatches[call_info->call_index].instruction = &result.extra_instruction_storage.unsafe_last();
|
||||
result.dispatches[call_info->call_index].instruction_opcode = new_call_opcode;
|
||||
}
|
||||
|
||||
result.max_call_rec_size = max_call_record_size;
|
||||
|
||||
for (size_t i = 0; i < final_roots.size(); ++i)
|
||||
final_roots[i] = find_root(i);
|
||||
|
||||
HashMap<ValueID, u8> root_to_callrec_slot;
|
||||
for (auto const& [value_id, slot] : value_to_callrec_slot) {
|
||||
auto root = final_roots[value_id.value()];
|
||||
if (auto existing = root_to_callrec_slot.get(root); existing.has_value()) {
|
||||
VERIFY(*existing == slot);
|
||||
}
|
||||
root_to_callrec_slot.set(root, slot);
|
||||
}
|
||||
|
||||
value_to_callrec_slot.clear_with_capacity();
|
||||
for (size_t i = 0; i < final_roots.size(); ++i) {
|
||||
auto root = final_roots[i];
|
||||
if (auto slot = root_to_callrec_slot.get(root); slot.has_value()) {
|
||||
value_to_callrec_slot.set(ValueID { i }, *slot);
|
||||
}
|
||||
}
|
||||
|
||||
struct LiveInterval {
|
||||
ValueID value_id;
|
||||
IP start;
|
||||
|
|
@ -5417,6 +5701,36 @@ CompiledInstructions try_compile_instructions(Expression const& expression, Span
|
|||
}
|
||||
|
||||
for (auto& [key, group] : alias_groups) {
|
||||
// Check if any value in this group needs a call record slot
|
||||
Dispatch::RegisterOrStack forced_slot = Dispatch::RegisterOrStack::Stack;
|
||||
bool has_callrec_constraint = false;
|
||||
|
||||
for (auto* interval : group) {
|
||||
if (auto slot = value_to_callrec_slot.get(interval->value_id); slot.has_value()) {
|
||||
forced_slot = static_cast<Dispatch::RegisterOrStack>(*slot);
|
||||
has_callrec_constraint = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (has_callrec_constraint) {
|
||||
// Force all values in this alias group to use the call record slot
|
||||
for (auto* interval : group) {
|
||||
value_alloc.set(interval->value_id, forced_slot);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
auto has_fixed_allocation = false;
|
||||
for (auto* interval : group) {
|
||||
if (value_alloc.contains(interval->value_id)) {
|
||||
has_fixed_allocation = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (has_fixed_allocation)
|
||||
continue;
|
||||
|
||||
IP group_start = NumericLimits<size_t>::max();
|
||||
IP group_end = 0;
|
||||
auto group_forced_to_stack = false;
|
||||
|
|
@ -5507,17 +5821,30 @@ CompiledInstructions try_compile_instructions(Expression const& expression, Span
|
|||
|
||||
return true;
|
||||
};
|
||||
constexpr auto all_sources_are_callrec = [](SourcesAndDestination const& addrs, ssize_t expected_source_count, ssize_t expected_dest_count) -> bool {
|
||||
if (expected_source_count < 0 || expected_dest_count > 1 || expected_dest_count < 0)
|
||||
return false;
|
||||
for (ssize_t i = 0; i < expected_source_count; ++i) {
|
||||
if (addrs.sources[i] < Dispatch::CallRecord)
|
||||
return false;
|
||||
}
|
||||
if (expected_dest_count == 1 && addrs.destination < Dispatch::CallRecord)
|
||||
return false;
|
||||
return true;
|
||||
};
|
||||
|
||||
for (size_t i = 0; i < result.dispatches.size(); ++i) {
|
||||
auto& dispatch = result.dispatches[i];
|
||||
auto& addrs = result.src_dst_mappings[i];
|
||||
|
||||
#define CASE(name, _, inputs, outputs) \
|
||||
case Instructions::name.value(): \
|
||||
if (all_sources_are_registers(addrs, inputs, outputs)) \
|
||||
dispatch.handler_ptr = bit_cast<FlatPtr>(&InstructionHandler<Instructions::name.value()>::template operator()<false, Continue, SourceAddressMix::AllRegisters>); \
|
||||
else \
|
||||
dispatch.handler_ptr = bit_cast<FlatPtr>(&InstructionHandler<Instructions::name.value()>::template operator()<false, Continue, SourceAddressMix::Any>); \
|
||||
#define CASE(name, _, inputs, outputs) \
|
||||
case Instructions::name.value(): \
|
||||
if (all_sources_are_registers(addrs, inputs, outputs)) \
|
||||
dispatch.handler_ptr = bit_cast<FlatPtr>(&InstructionHandler<Instructions::name.value()>::template operator()<false, Continue, SourceAddressMix::AllRegisters>); \
|
||||
else if (all_sources_are_callrec(addrs, inputs, outputs)) \
|
||||
dispatch.handler_ptr = bit_cast<FlatPtr>(&InstructionHandler<Instructions::name.value()>::template operator()<false, Continue, SourceAddressMix::AllCallRecords>); \
|
||||
else \
|
||||
dispatch.handler_ptr = bit_cast<FlatPtr>(&InstructionHandler<Instructions::name.value()>::template operator()<false, Continue, SourceAddressMix::Any>); \
|
||||
break;
|
||||
|
||||
switch (dispatch.instruction->opcode().value()) {
|
||||
|
|
@ -5561,32 +5888,29 @@ CompiledInstructions try_compile_instructions(Expression const& expression, Span
|
|||
ENUMERATE_WASM_OPCODES(XM)
|
||||
}
|
||||
for (ssize_t i = 0; i < in_count; ++i) {
|
||||
if (addresses.sources[i] < Dispatch::Stack) {
|
||||
warnln(" arg{} [reg{}]", i, to_underlying(addresses.sources[i]));
|
||||
} else {
|
||||
warnln(" arg{} [stack]", i);
|
||||
}
|
||||
warnln(" arg{} [{}]", i, regname(addresses.sources[i]));
|
||||
}
|
||||
if (out_count == 1) {
|
||||
auto dest = addresses.destination;
|
||||
if (dest < Dispatch::Stack) {
|
||||
warnln(" dest [reg{}]", to_underlying(dest));
|
||||
} else {
|
||||
warnln(" dest [stack]");
|
||||
}
|
||||
warnln(" dest [{}]", regname(dest));
|
||||
} else if (out_count > 1) {
|
||||
warnln(" dest [multiple outputs]");
|
||||
} else if (instruction->opcode() == Instructions::call || instruction->opcode() == Instructions::call_indirect) {
|
||||
if (addresses.destination != Dispatch::Stack)
|
||||
warnln(" dest [{}]", regname(addresses.destination));
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
if (start_ish > end_ish)
|
||||
swap(start_ish, end_ish);
|
||||
auto start_ip = start_ish >= 5 ? start_ish - 5 : 0;
|
||||
auto start_ip = start_ish >= 40 ? start_ish - 40 : 0;
|
||||
auto end_ip = min(result.dispatches.size(), end_ish + 10);
|
||||
auto skip_start = Optional<size_t> {};
|
||||
for (auto ip = start_ip; ip < end_ip; ip += 5) {
|
||||
size_t chunk_end = min(end_ip, ip + 5);
|
||||
print_range(ip, chunk_end);
|
||||
continue;
|
||||
bool has_mark = false;
|
||||
for (auto const& mark : { marks... }) {
|
||||
if (mark.ip >= ip && mark.ip < chunk_end) {
|
||||
|
|
@ -5606,6 +5930,7 @@ CompiledInstructions try_compile_instructions(Expression const& expression, Span
|
|||
}
|
||||
};
|
||||
|
||||
bool used[256] = { false };
|
||||
for (size_t i = 0; i < result.dispatches.size(); ++i) {
|
||||
auto& dispatch = result.dispatches[i];
|
||||
if (dispatch.instruction->opcode() == Instructions::if_) {
|
||||
|
|
@ -5628,6 +5953,45 @@ CompiledInstructions try_compile_instructions(Expression const& expression, Span
|
|||
VERIFY_NOT_REACHED();
|
||||
}
|
||||
}
|
||||
// If the instruction is a call with a callrec, clear used[] for the callrec registers.
|
||||
if (dispatch.instruction->opcode() == Instructions::synthetic_call_with_record_0 || dispatch.instruction->opcode() == Instructions::synthetic_call_with_record_1) {
|
||||
for (size_t j = to_underlying(Dispatch::CallRecord); j <= to_underlying(Dispatch::LastCallRecord); ++j)
|
||||
used[j] = false;
|
||||
}
|
||||
|
||||
auto& addr = result.src_dst_mappings[i];
|
||||
|
||||
// for each input, ensure it's not reading from a register that is not marked as used (unless stack).
|
||||
ssize_t in_count = 0;
|
||||
ssize_t out_count = 0;
|
||||
switch (dispatch.instruction->opcode().value()) {
|
||||
ENUMERATE_WASM_OPCODES(XM)
|
||||
}
|
||||
for (ssize_t j = 0; j < in_count; ++j) {
|
||||
auto src = addr.sources[j];
|
||||
if (src == Dispatch::Stack)
|
||||
continue;
|
||||
if (!used[to_underlying(src)]) {
|
||||
dbgln("Instruction {} reads from register {} which is not populated", i, to_underlying(src));
|
||||
dbgln("Instructions around the invalid read:");
|
||||
print_instructions_around(i, i, Mark { i, "invalid read here"sv });
|
||||
VERIFY_NOT_REACHED();
|
||||
}
|
||||
used[to_underlying(src)] = false;
|
||||
}
|
||||
// if the instruction has an output, ensure it's not writing to a register that is marked used.
|
||||
if (out_count == 1 || dispatch.instruction->opcode() == Instructions::call || dispatch.instruction->opcode() == Instructions::call_indirect) {
|
||||
auto dest = addr.destination;
|
||||
if (dest != Dispatch::Stack) {
|
||||
if (used[to_underlying(dest)]) {
|
||||
dbgln("Instruction {} writes to register {} which is already populated", i, to_underlying(dest));
|
||||
dbgln("Instructions around the invalid write:");
|
||||
print_instructions_around(i, i, Mark { i, "invalid write here"sv });
|
||||
VERIFY_NOT_REACHED();
|
||||
}
|
||||
used[to_underlying(dest)] = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
|
|
|
|||
|
|
@ -63,6 +63,7 @@ struct WASM_API BytecodeInterpreter final : public Interpreter {
|
|||
|
||||
enum class CallType {
|
||||
UsingRegisters,
|
||||
UsingCallRecord,
|
||||
UsingStack,
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -13,13 +13,10 @@ namespace Wasm {
|
|||
|
||||
void Configuration::unwind_impl()
|
||||
{
|
||||
release_arguments_allocation(m_frame_stack.last().arguments());
|
||||
m_frame_stack.last().locals().clear();
|
||||
|
||||
auto last_frame = m_frame_stack.take_last();
|
||||
m_depth--;
|
||||
m_locals_base = m_frame_stack.is_empty() ? nullptr : m_frame_stack.unchecked_last().locals().data();
|
||||
m_arguments_base = m_frame_stack.is_empty() ? nullptr : m_frame_stack.unchecked_last().arguments().data();
|
||||
release_arguments_allocation(last_frame.locals());
|
||||
}
|
||||
|
||||
Result Configuration::call(Interpreter& interpreter, FunctionAddress address, Vector<Value, ArgumentsStaticSize>& arguments)
|
||||
|
|
@ -39,18 +36,16 @@ ErrorOr<Optional<HostFunction&>, Trap> Configuration::prepare_call(FunctionAddre
|
|||
if (auto* wasm_function = function->get_pointer<WasmFunction>()) {
|
||||
if (is_tailcall)
|
||||
unwind_impl(); // Unwind the current frame, the "return" in the tail-called function will unwind the frame we're gonna push now.
|
||||
Vector<Value, 8> locals;
|
||||
locals.ensure_capacity(wasm_function->code().func().total_local_count());
|
||||
arguments.ensure_capacity(arguments.size() + wasm_function->code().func().total_local_count());
|
||||
for (auto& local : wasm_function->code().func().locals()) {
|
||||
for (size_t i = 0; i < local.n(); ++i)
|
||||
locals.unchecked_append(Value(local.type()));
|
||||
arguments.unchecked_append(Value(local.type()));
|
||||
}
|
||||
|
||||
set_frame(
|
||||
is_tailcall ? IsTailcall::Yes : IsTailcall::No,
|
||||
wasm_function->module(),
|
||||
move(arguments),
|
||||
move(locals),
|
||||
wasm_function->code().func().body(),
|
||||
wasm_function->type().results().size());
|
||||
return OptionalNone {};
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ namespace Wasm {
|
|||
|
||||
enum class SourceAddressMix {
|
||||
AllRegisters,
|
||||
AllCallRecords,
|
||||
Any,
|
||||
};
|
||||
|
||||
|
|
@ -36,7 +37,6 @@ public:
|
|||
|
||||
auto& frame = m_frame_stack.unchecked_last();
|
||||
m_locals_base = frame.locals().data();
|
||||
m_arguments_base = frame.arguments().data();
|
||||
|
||||
auto continuation = frame.expression().instructions().size() - 1;
|
||||
if (auto size = frame.expression().compiled_instructions.dispatches.size(); size > 0)
|
||||
|
|
@ -50,9 +50,14 @@ public:
|
|||
m_label_stack.ensure_capacity(*hint + m_label_stack.size());
|
||||
}
|
||||
m_label_stack.append(label);
|
||||
if (auto max_count = frame.expression().compiled_instructions.max_call_arg_count; max_count > ArgumentsStaticSize) {
|
||||
if (!m_call_argument_freelist.is_empty() && !any_of(m_call_argument_freelist, [&](auto const& entry) { return entry.capacity() >= frame.expression().compiled_instructions.max_call_arg_count; }))
|
||||
m_call_argument_freelist.last().ensure_capacity(max_count);
|
||||
|
||||
auto max_call_rec_size = frame.expression().compiled_instructions.max_call_rec_size;
|
||||
if (max_call_rec_size > 0) {
|
||||
get_arguments_allocation_if_possible(m_current_call_record, max_call_rec_size);
|
||||
m_current_call_record.resize_and_keep_capacity(max_call_rec_size);
|
||||
m_call_record_base = m_current_call_record.data();
|
||||
} else {
|
||||
m_call_record_base = nullptr;
|
||||
}
|
||||
}
|
||||
ALWAYS_INLINE auto& frame() const { return m_frame_stack.unchecked_last(); }
|
||||
|
|
@ -68,15 +73,6 @@ public:
|
|||
ALWAYS_INLINE auto& store() const { return m_store; }
|
||||
ALWAYS_INLINE auto& store() { return m_store; }
|
||||
|
||||
ALWAYS_INLINE Value& local_or_argument(LocalIndex index)
|
||||
{
|
||||
if (index.value() & LocalArgumentMarker)
|
||||
return m_arguments_base[index.value() & ~LocalArgumentMarker];
|
||||
return m_locals_base[index.value()];
|
||||
}
|
||||
ALWAYS_INLINE Value const& argument(LocalIndex index) const { return m_arguments_base[index.value() & ~LocalArgumentMarker]; }
|
||||
ALWAYS_INLINE Value& argument(LocalIndex index) { return m_arguments_base[index.value() & ~LocalArgumentMarker]; }
|
||||
|
||||
ALWAYS_INLINE Value const& local(LocalIndex index) const { return m_locals_base[index.value()]; }
|
||||
ALWAYS_INLINE Value& local(LocalIndex index) { return m_locals_base[index.value()]; }
|
||||
|
||||
|
|
@ -84,15 +80,25 @@ public:
|
|||
explicit CallFrameHandle(Configuration& configuration)
|
||||
: configuration(configuration)
|
||||
{
|
||||
if (configuration.m_call_record_base)
|
||||
moved_call_record = move(configuration.m_current_call_record);
|
||||
configuration.depth()++;
|
||||
configuration.m_call_record_base = nullptr;
|
||||
}
|
||||
|
||||
~CallFrameHandle()
|
||||
{
|
||||
if (moved_call_record.has_value()) {
|
||||
configuration.m_current_call_record = moved_call_record.release_value();
|
||||
configuration.m_call_record_base = configuration.m_current_call_record.data();
|
||||
} else {
|
||||
configuration.m_call_record_base = nullptr;
|
||||
}
|
||||
configuration.unwind({}, *this);
|
||||
}
|
||||
|
||||
Configuration& configuration;
|
||||
Optional<Vector<Value, ArgumentsStaticSize>> moved_call_record;
|
||||
};
|
||||
|
||||
void unwind(Badge<CallFrameHandle>, CallFrameHandle const&) { unwind_impl(); }
|
||||
|
|
@ -110,24 +116,49 @@ public:
|
|||
if (arguments.capacity() != ArgumentsStaticSize || max_size <= ArgumentsStaticSize)
|
||||
return; // Already heap allocated, or we just don't need to allocate anything.
|
||||
|
||||
// _arguments_ is still in static storage, pull something from the freelist if possible; otherwise allocate a new one.
|
||||
// _arguments_ is still in static storage, pull something from the freelist if it fits.
|
||||
if (auto index = m_call_argument_freelist.find_first_index_if([&](auto& entry) { return entry.capacity() >= max_size; }); index.has_value()) {
|
||||
arguments = m_call_argument_freelist.take(*index);
|
||||
return;
|
||||
}
|
||||
|
||||
arguments.ensure_capacity(max_size);
|
||||
if (!m_call_argument_freelist.is_empty())
|
||||
arguments = m_call_argument_freelist.take_last();
|
||||
|
||||
arguments.ensure_capacity(max(max_size, frame().module().cached_minimum_call_record_allocation_size));
|
||||
}
|
||||
|
||||
void release_arguments_allocation(Vector<Value, ArgumentsStaticSize>& arguments)
|
||||
{
|
||||
arguments.clear_with_capacity(); // Clear to avoid copying, but keep capacity for reuse.
|
||||
if (arguments.capacity() != ArgumentsStaticSize) {
|
||||
if (m_call_argument_freelist.size() >= 16) // Don't grow to heap.
|
||||
return;
|
||||
auto size = frame().expression().compiled_instructions.max_call_rec_size;
|
||||
|
||||
m_call_argument_freelist.append(move(arguments));
|
||||
if (size > 0) {
|
||||
// If we need a call record, keep this as the current one.
|
||||
if (!m_call_record_base) {
|
||||
m_current_call_record = move(arguments);
|
||||
m_current_call_record.resize_and_keep_capacity(size);
|
||||
m_call_record_base = m_current_call_record.data();
|
||||
return;
|
||||
}
|
||||
|
||||
VERIFY(m_current_call_record.size() >= size);
|
||||
}
|
||||
|
||||
if (arguments.capacity() != ArgumentsStaticSize) {
|
||||
if (m_call_argument_freelist.size() >= 16) {
|
||||
// Don't grow to heap.
|
||||
return;
|
||||
}
|
||||
|
||||
m_call_argument_freelist.unchecked_append(move(arguments));
|
||||
}
|
||||
}
|
||||
|
||||
void take_call_record(Vector<Value, ArgumentsStaticSize>& call_record)
|
||||
{
|
||||
call_record = move(m_current_call_record);
|
||||
m_call_record_base = nullptr;
|
||||
}
|
||||
|
||||
template<SourceAddressMix mix>
|
||||
|
|
@ -136,6 +167,9 @@ public:
|
|||
if constexpr (mix == SourceAddressMix::AllRegisters) {
|
||||
regs.data()[to_underlying(destination)] = value;
|
||||
return;
|
||||
} else if constexpr (mix == SourceAddressMix::AllCallRecords) {
|
||||
m_call_record_base[to_underlying(destination) - Dispatch::RegisterOrStack::CallRecord] = value;
|
||||
return;
|
||||
} else if constexpr (mix == SourceAddressMix::Any) {
|
||||
if (!(destination & ~(Dispatch::Stack - 1))) [[likely]] {
|
||||
regs.data()[to_underlying(destination)] = value;
|
||||
|
|
@ -148,6 +182,9 @@ public:
|
|||
value_stack().unchecked_append(value);
|
||||
return;
|
||||
}
|
||||
|
||||
m_call_record_base[to_underlying(destination) - Dispatch::RegisterOrStack::CallRecord] = value;
|
||||
return;
|
||||
}
|
||||
|
||||
VERIFY_NOT_REACHED();
|
||||
|
|
@ -161,6 +198,8 @@ public:
|
|||
|
||||
if constexpr (mix == SourceAddressMix::AllRegisters) {
|
||||
return regs.data()[to_underlying(source)];
|
||||
} else if constexpr (mix == SourceAddressMix::AllCallRecords) {
|
||||
return m_call_record_base[to_underlying(source) - Dispatch::RegisterOrStack::CallRecord];
|
||||
} else if constexpr (mix == SourceAddressMix::Any) {
|
||||
if (!(source & ~(Dispatch::Stack - 1))) [[likely]]
|
||||
return regs.data()[to_underlying(source)];
|
||||
|
|
@ -169,6 +208,8 @@ public:
|
|||
if constexpr (mix == SourceAddressMix::Any) {
|
||||
if (source == Dispatch::RegisterOrStack::Stack) [[unlikely]]
|
||||
return value_stack().unsafe_last();
|
||||
|
||||
return m_call_record_base[to_underlying(source) - Dispatch::RegisterOrStack::CallRecord];
|
||||
}
|
||||
|
||||
VERIFY_NOT_REACHED();
|
||||
|
|
@ -180,6 +221,8 @@ public:
|
|||
auto const source = sources[index];
|
||||
if constexpr (mix == SourceAddressMix::AllRegisters) {
|
||||
return regs.data()[to_underlying(source)];
|
||||
} else if constexpr (mix == SourceAddressMix::AllCallRecords) {
|
||||
return m_call_record_base[to_underlying(source) - Dispatch::RegisterOrStack::CallRecord];
|
||||
} else if constexpr (mix == SourceAddressMix::Any) {
|
||||
if (!(source & ~(Dispatch::Stack - 1))) [[likely]]
|
||||
return regs.data()[to_underlying(source)];
|
||||
|
|
@ -188,6 +231,8 @@ public:
|
|||
if constexpr (mix == SourceAddressMix::Any) {
|
||||
if (source == Dispatch::RegisterOrStack::Stack) [[unlikely]]
|
||||
return value_stack().unsafe_take_last();
|
||||
|
||||
return m_call_record_base[to_underlying(source) - Dispatch::RegisterOrStack::CallRecord];
|
||||
}
|
||||
|
||||
VERIFY_NOT_REACHED();
|
||||
|
|
@ -211,12 +256,13 @@ private:
|
|||
Vector<Value, 64, FastLastAccess::Yes> m_value_stack;
|
||||
Vector<Label, 64> m_label_stack;
|
||||
DoublyLinkedList<Frame, 512> m_frame_stack;
|
||||
Vector<Value, ArgumentsStaticSize> m_current_call_record;
|
||||
Vector<Vector<Value, ArgumentsStaticSize>, 16, FastLastAccess::Yes> m_call_argument_freelist;
|
||||
size_t m_depth { 0 };
|
||||
u64 m_ip { 0 };
|
||||
bool m_should_limit_instruction_count { false };
|
||||
Value* m_locals_base { nullptr };
|
||||
Value* m_arguments_base { nullptr };
|
||||
Value* m_call_record_base { nullptr };
|
||||
};
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -97,6 +97,8 @@ ErrorOr<void, ValidationError> Validator::validate(Module& module)
|
|||
for (auto& tag : module.tag_section().tags())
|
||||
m_context.tags.append(TagType(tag.type(), tag.flags()));
|
||||
|
||||
m_context.current_module = &module;
|
||||
|
||||
// We need to build the set of declared functions to check that `ref.func` uses a specific set of predetermined functions, found in:
|
||||
// - Element initializer expressions
|
||||
// - Global initializer expressions
|
||||
|
|
@ -133,6 +135,9 @@ ErrorOr<void, ValidationError> Validator::validate(Module& module)
|
|||
TRY(validate(module.code_section()));
|
||||
TRY(validate(module.tag_section()));
|
||||
|
||||
for (auto& entry : module.code_section().functions())
|
||||
module.set_minimum_call_record_allocation_size(max(entry.func().body().compiled_instructions.max_call_rec_size, module.minimum_call_record_allocation_size()));
|
||||
|
||||
module.set_validation_status(Module::ValidationStatus::Valid, {});
|
||||
return {};
|
||||
}
|
||||
|
|
@ -273,6 +278,19 @@ ErrorOr<void, ValidationError> Validator::validate(CodeSection const& section)
|
|||
auto results = TRY(function_validator.validate(function.body(), function_type.results()));
|
||||
if (results.result_types.size() != function_type.results().size())
|
||||
return Errors::invalid("function result"sv, function_type.results(), results.result_types);
|
||||
|
||||
if (function.body().compiled_instructions.max_call_rec_size != 0) {
|
||||
size_t max_callee_locals = 0;
|
||||
for (auto& insn : function.body().instructions()) {
|
||||
if (!first_is_one_of(insn.opcode(), Instructions::call, Instructions::synthetic_call_with_record_0, Instructions::synthetic_call_with_record_1))
|
||||
continue;
|
||||
auto callee_index = insn.arguments().template get<FunctionIndex>();
|
||||
if (callee_index.value() - m_context.imported_function_count < section.functions().size())
|
||||
max_callee_locals = max(max_callee_locals, section.functions()[callee_index.value() - m_context.imported_function_count].func().total_local_count());
|
||||
}
|
||||
|
||||
function.body().compiled_instructions.max_call_rec_size += max_callee_locals;
|
||||
}
|
||||
}
|
||||
|
||||
return {};
|
||||
|
|
|
|||
|
|
@ -127,11 +127,6 @@ public:
|
|||
|
||||
ErrorOr<LocalIndex, ValidationError> validate(LocalIndex index) const
|
||||
{
|
||||
if (index.value() & LocalArgumentMarker)
|
||||
index = index.value() & ~LocalArgumentMarker;
|
||||
else
|
||||
index = index.value() + m_context.current_function_parameter_count;
|
||||
|
||||
if (index.value() < m_context.locals.size())
|
||||
return index;
|
||||
return Errors::invalid("LocalIndex"sv);
|
||||
|
|
|
|||
|
|
@ -500,7 +500,9 @@ namespace Instructions {
|
|||
M(synthetic_call_30, 0xfe0000000000000cull, 3, 0) \
|
||||
M(synthetic_call_31, 0xfe0000000000000dull, 3, 1) \
|
||||
M(synthetic_end_expression, 0xfe0000000000000eull, 0, 0) \
|
||||
M(synthetic_argument_get, 0xfe0000000000000full, 0, 1)
|
||||
M(synthetic_argument_get, 0xfe0000000000000full, 0, 1) \
|
||||
M(synthetic_call_with_record_0, 0xfe00000000000012ull, 0, 0) \
|
||||
M(synthetic_call_with_record_1, 0xfe00000000000013ull, 0, 1)
|
||||
|
||||
#define ENUMERATE_WASM_OPCODES(M) \
|
||||
ENUMERATE_SINGLE_BYTE_WASM_OPCODES(M) \
|
||||
|
|
@ -511,7 +513,7 @@ ENUMERATE_WASM_OPCODES(M)
|
|||
#undef M
|
||||
|
||||
static constexpr inline OpCode SyntheticInstructionBase = 0xfe00000000000000ull;
|
||||
static constexpr inline size_t SyntheticInstructionCount = 16;
|
||||
static constexpr inline size_t SyntheticInstructionCount = 18;
|
||||
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1480,22 +1480,6 @@ ParseResult<NonnullRefPtr<Module>> Module::parse(Stream& stream)
|
|||
|
||||
void Module::preprocess()
|
||||
{
|
||||
for (auto const [i, type_id] : enumerate(function_section().types())) {
|
||||
if (type_id.value() >= type_section().types().size() || i >= code_section().functions().size()) {
|
||||
dbgln("WASM Module preprocessing: skipping function {} with invalid type id {}", i, type_id.value());
|
||||
continue;
|
||||
}
|
||||
|
||||
auto& function = code_section().functions()[i];
|
||||
auto const parameter_count = type_section().types()[type_id.value()].parameters().size();
|
||||
for (auto& instruction : function.func().body().instructions()) {
|
||||
auto& mutable_instruction = const_cast<Instruction&>(instruction);
|
||||
if (instruction.local_index() < parameter_count)
|
||||
mutable_instruction.set_local_index({}, instruction.local_index().value() | LocalArgumentMarker);
|
||||
else
|
||||
mutable_instruction.set_local_index({}, instruction.local_index().value() - parameter_count);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
ByteString parse_error_to_byte_string(ParseError error)
|
||||
|
|
|
|||
|
|
@ -475,27 +475,20 @@ void Printer::print(Wasm::Instruction const& instruction)
|
|||
print("({}", instruction_name(instruction.opcode()));
|
||||
if (instruction.arguments().has<u8>()) {
|
||||
if (first_is_one_of(instruction.opcode(), Instructions::local_get, Instructions::local_set, Instructions::local_tee, Instructions::synthetic_argument_get)) {
|
||||
if (instruction.local_index().value() & LocalArgumentMarker)
|
||||
print(" (argument index {})", instruction.local_index().value() & ~LocalArgumentMarker);
|
||||
else
|
||||
print(" (local index {})", instruction.local_index().value());
|
||||
print(" (local index {})", instruction.local_index().value());
|
||||
}
|
||||
print(")\n");
|
||||
} else {
|
||||
print(" ");
|
||||
instruction.arguments().visit(
|
||||
[&](Instruction::BranchArgs const& branch) { print("(label index {})", branch.label.value()); },
|
||||
[&](BlockType const& type) { print(type); },
|
||||
[&](DataIndex const& index) { print("(data index {})", index.value()); },
|
||||
[&](ElementIndex const& index) { print("(element index {})", index.value()); },
|
||||
[&](FunctionIndex const& index) { print("(function index {})", index.value()); },
|
||||
[&](GlobalIndex const& index) { print("(global index {})", index.value()); },
|
||||
[&](LabelIndex const& index) { print("(label index {})", index.value()); },
|
||||
[&](LocalIndex const& index) {
|
||||
if (index.value() & LocalArgumentMarker)
|
||||
print("(argument index {})", index.value() & ~LocalArgumentMarker);
|
||||
else
|
||||
print("(local index {})", index.value());
|
||||
},
|
||||
[&](LocalIndex const& index) { print("(local index {})", index.value()); },
|
||||
[&](TableIndex const& index) { print("(table index {})", index.value()); },
|
||||
[&](Instruction::IndirectCallArgs const& args) { print("(indirect (type index {}) (table index {}))", args.type.value(), args.table.value()); },
|
||||
[&](Instruction::MemoryArgument const& args) { print("(memory index {} (align {}) (offset {}))", args.memory_index.value(), args.align, args.offset); },
|
||||
|
|
@ -516,7 +509,12 @@ void Printer::print(Wasm::Instruction const& instruction)
|
|||
TemporaryChange change { m_indent, m_indent + 1 };
|
||||
print(args.block_type);
|
||||
print_indent();
|
||||
print("(else {}) (end {}))", args.else_ip.has_value() ? ByteString::number(args.else_ip->value()) : "(none)", args.end_ip.value());
|
||||
print("(else {}) (end {})", args.else_ip.has_value() ? ByteString::number(args.else_ip->value()) : "(none)", args.end_ip.value());
|
||||
if (args.meta.has_value())
|
||||
print(" (meta arity {} params {})", args.meta->arity, args.meta->parameter_count);
|
||||
else
|
||||
print(" (meta none)");
|
||||
print(")");
|
||||
},
|
||||
[&](Instruction::TryTableArgs const& args) {
|
||||
print("(try_table ");
|
||||
|
|
@ -540,12 +538,8 @@ void Printer::print(Wasm::Instruction const& instruction)
|
|||
[&](Vector<ValueType> const&) { print("(types...)"); },
|
||||
[&](auto const& value) { print("(const {})", value); });
|
||||
|
||||
if (first_is_one_of(instruction.opcode(), Instructions::local_get, Instructions::local_set, Instructions::local_tee, Instructions::synthetic_argument_get, Instructions::synthetic_local_seti32_const, Instructions::synthetic_i32_storelocal)) {
|
||||
if (instruction.local_index().value() & LocalArgumentMarker)
|
||||
print(" (argument index {})", instruction.local_index().value() & ~LocalArgumentMarker);
|
||||
else
|
||||
print(" (local index {})", instruction.local_index().value());
|
||||
}
|
||||
if (first_is_one_of(instruction.opcode(), Instructions::local_get, Instructions::local_set, Instructions::local_tee, Instructions::synthetic_argument_get, Instructions::synthetic_local_seti32_const, Instructions::synthetic_i32_storelocal))
|
||||
print(" (local index {})", instruction.local_index().value());
|
||||
|
||||
print(")\n");
|
||||
}
|
||||
|
|
@ -1274,5 +1268,7 @@ HashMap<Wasm::OpCode, ByteString> Wasm::Names::instruction_names {
|
|||
{ Instructions::synthetic_call_31, "synthetic:call.31" },
|
||||
{ Instructions::synthetic_end_expression, "synthetic:expression.end" },
|
||||
{ Instructions::synthetic_argument_get, "synthetic:argument.get" },
|
||||
{ Instructions::synthetic_call_with_record_0, "synthetic:call.with_record.0" },
|
||||
{ Instructions::synthetic_call_with_record_1, "synthetic:call.with_record.1" },
|
||||
};
|
||||
HashMap<ByteString, Wasm::OpCode> Wasm::Names::instructions_by_name;
|
||||
|
|
|
|||
|
|
@ -620,6 +620,8 @@ struct Dispatch {
|
|||
R7,
|
||||
CountRegisters,
|
||||
Stack = CountRegisters,
|
||||
CallRecord,
|
||||
LastCallRecord = NumericLimits<u8>::max(),
|
||||
};
|
||||
|
||||
static_assert(is_power_of_two(to_underlying(Stack)), "Stack marker must be a single bit");
|
||||
|
|
@ -645,6 +647,7 @@ struct CompiledInstructions {
|
|||
Vector<Instruction, 0, FastLastAccess::Yes> extra_instruction_storage;
|
||||
bool direct = false; // true if all dispatches contain handler_ptr, otherwise false and all contain instruction_opcode.
|
||||
size_t max_call_arg_count = 0;
|
||||
size_t max_call_rec_size = 0;
|
||||
};
|
||||
|
||||
template<Enum auto... Vs>
|
||||
|
|
@ -1251,6 +1254,9 @@ public:
|
|||
|
||||
static ParseResult<NonnullRefPtr<Module>> parse(Stream& stream);
|
||||
|
||||
size_t minimum_call_record_allocation_size() const { return m_minimum_call_record_allocation_size; }
|
||||
void set_minimum_call_record_allocation_size(size_t size) { m_minimum_call_record_allocation_size = size; }
|
||||
|
||||
private:
|
||||
void set_validation_status(ValidationStatus status) { m_validation_status = status; }
|
||||
void preprocess();
|
||||
|
|
@ -1272,6 +1278,8 @@ private:
|
|||
|
||||
ValidationStatus m_validation_status { ValidationStatus::Unchecked };
|
||||
Optional<ByteString> m_validation_error;
|
||||
|
||||
size_t m_minimum_call_record_allocation_size { 0 };
|
||||
};
|
||||
|
||||
CompiledInstructions try_compile_instructions(Expression const&, Span<FunctionType const> functions);
|
||||
|
|
|
|||
|
|
@ -747,11 +747,11 @@ ErrorOr<int> ladybird_main(Main::Arguments arguments)
|
|||
ByteString regs;
|
||||
auto first = true;
|
||||
ssize_t in_count = 0;
|
||||
bool has_out = false;
|
||||
ssize_t out_count = 0;
|
||||
#define M(name, _, ins, outs) \
|
||||
case Wasm::Instructions::name.value(): \
|
||||
in_count = ins; \
|
||||
has_out = outs != 0; \
|
||||
out_count = outs; \
|
||||
break;
|
||||
switch (dispatch.instruction->opcode().value()) {
|
||||
ENUMERATE_WASM_OPCODES(M)
|
||||
|
|
@ -760,6 +760,8 @@ ErrorOr<int> ladybird_main(Main::Arguments arguments)
|
|||
constexpr auto reg_name = [](Wasm::Dispatch::RegisterOrStack reg) -> ByteString {
|
||||
if (reg == Wasm::Dispatch::RegisterOrStack::Stack)
|
||||
return "stack"sv;
|
||||
if (reg >= Wasm::Dispatch::RegisterOrStack::CallRecord)
|
||||
return ByteString::formatted("cr{}", to_underlying(reg) - to_underlying(Wasm::Dispatch::RegisterOrStack::CallRecord));
|
||||
return ByteString::formatted("reg{}", to_underlying(reg));
|
||||
};
|
||||
if (in_count > -1) {
|
||||
|
|
@ -770,17 +772,25 @@ ErrorOr<int> ladybird_main(Main::Arguments arguments)
|
|||
regs = ByteString::formatted("{}, {}", regs, reg_name(addresses.sources[index]));
|
||||
first = false;
|
||||
}
|
||||
if (has_out) {
|
||||
if (out_count > 0) {
|
||||
if (first)
|
||||
regs = ByteString::formatted(" () -> {}", reg_name(addresses.destination));
|
||||
else
|
||||
regs = ByteString::formatted("{}) -> {}", regs, reg_name(addresses.destination));
|
||||
} else {
|
||||
} else if (out_count == 0) {
|
||||
if (first)
|
||||
regs = ByteString::formatted(" () -x");
|
||||
else
|
||||
regs = ByteString::formatted("{}) -x", regs);
|
||||
} else {
|
||||
if (first)
|
||||
regs = ByteString::formatted(" () -?");
|
||||
else
|
||||
regs = ByteString::formatted("{}) -?", regs);
|
||||
}
|
||||
} else if (dispatch.instruction->opcode() == Wasm::Instructions::call || dispatch.instruction->opcode() == Wasm::Instructions::call_indirect) {
|
||||
if (addresses.destination != Wasm::Dispatch::RegisterOrStack::Stack)
|
||||
regs = ByteString::formatted("(?) -> {}", reg_name(addresses.destination));
|
||||
}
|
||||
|
||||
if (regs.is_empty())
|
||||
|
|
|
|||
Loading…
Reference in a new issue