LibWasm: Prepare for VM and direct function calls
Introduce MemoryBuffer, a memory backing store that uses mmap to reserve the full wasm32 address space (4 GiB + guard pages) upfront, growing without a copy and falling back to a ByteBuffer when mmap fails. Also let Frame know how to handle non-owned locals (to e.g. allow allocating them on the native stack.)
This commit is contained in:
Ali Mohammad Pur
Ali Mohammad Pur
parent
0bb987e809
commit
f181d24dc0
5 changed files with 426 additions and 84 deletions
|
|
@ -6,6 +6,7 @@
|
|||
|
||||
#include <AK/Enumerate.h>
|
||||
#include <AK/SaturatingMath.h>
|
||||
#include <LibCore/System.h>
|
||||
#include <LibWasm/AbstractMachine/AbstractMachine.h>
|
||||
#include <LibWasm/AbstractMachine/BytecodeInterpreter.h>
|
||||
#include <LibWasm/AbstractMachine/Configuration.h>
|
||||
|
|
@ -15,6 +16,154 @@
|
|||
|
||||
namespace Wasm {
|
||||
|
||||
MemoryBuffer::~MemoryBuffer()
|
||||
{
|
||||
clear();
|
||||
}
|
||||
|
||||
MemoryBuffer::MemoryBuffer(MemoryBuffer&& other)
|
||||
: m_size(exchange(other.m_size, 0))
|
||||
, m_reserved_capacity(exchange(other.m_reserved_capacity, 0))
|
||||
, m_mapping_size(exchange(other.m_mapping_size, 0))
|
||||
, m_host_page_size(exchange(other.m_host_page_size, 0))
|
||||
, m_mapping_base(exchange(other.m_mapping_base, nullptr))
|
||||
, m_data(exchange(other.m_data, nullptr))
|
||||
, m_fallback(move(other.m_fallback))
|
||||
{
|
||||
}
|
||||
|
||||
MemoryBuffer& MemoryBuffer::operator=(MemoryBuffer&& other)
|
||||
{
|
||||
if (this != &other) {
|
||||
clear();
|
||||
m_size = exchange(other.m_size, 0);
|
||||
m_reserved_capacity = exchange(other.m_reserved_capacity, 0);
|
||||
m_mapping_size = exchange(other.m_mapping_size, 0);
|
||||
m_host_page_size = exchange(other.m_host_page_size, 0);
|
||||
m_mapping_base = exchange(other.m_mapping_base, nullptr);
|
||||
m_data = exchange(other.m_data, nullptr);
|
||||
m_fallback = move(other.m_fallback);
|
||||
}
|
||||
return *this;
|
||||
}
|
||||
|
||||
void MemoryBuffer::clear()
|
||||
{
|
||||
if (m_mapping_base) {
|
||||
VERIFY(m_reserved_capacity);
|
||||
VERIFY(m_mapping_size);
|
||||
VERIFY(m_host_page_size);
|
||||
auto reservation_size = m_mapping_size + 2 * m_host_page_size;
|
||||
[[maybe_unused]] auto result = Core::System::release_address_space(m_mapping_base, reservation_size);
|
||||
VERIFY(!result.is_error());
|
||||
}
|
||||
m_mapping_base = nullptr;
|
||||
m_data = nullptr;
|
||||
m_reserved_capacity = 0;
|
||||
m_mapping_size = 0;
|
||||
m_host_page_size = 0;
|
||||
m_size = 0;
|
||||
m_fallback.clear();
|
||||
}
|
||||
|
||||
void MemoryBuffer::try_reserve_wasm32_address_space()
|
||||
{
|
||||
if (m_mapping_base)
|
||||
return;
|
||||
|
||||
auto host_page_size = static_cast<size_t>(PAGE_SIZE);
|
||||
auto reserved_capacity = static_cast<size_t>(Constants::page_size) * 65536;
|
||||
auto mapping_size = reserved_capacity * 2;
|
||||
auto reservation_size = mapping_size + 2 * host_page_size;
|
||||
|
||||
auto mapping_or_error = Core::System::reserve_address_space(reservation_size);
|
||||
if (mapping_or_error.is_error())
|
||||
return;
|
||||
|
||||
m_mapping_base = mapping_or_error.value();
|
||||
m_data = reinterpret_cast<u8*>(m_mapping_base) + host_page_size;
|
||||
m_reserved_capacity = reserved_capacity;
|
||||
m_mapping_size = mapping_size;
|
||||
m_host_page_size = host_page_size;
|
||||
}
|
||||
|
||||
ErrorOr<void> MemoryBuffer::try_resize(size_t new_size)
|
||||
{
|
||||
if (m_data) {
|
||||
VERIFY(new_size >= m_size);
|
||||
VERIFY(m_host_page_size);
|
||||
if (new_size > m_reserved_capacity)
|
||||
return Error::from_errno(ENOMEM);
|
||||
if (new_size == m_size)
|
||||
return {};
|
||||
|
||||
auto* grow_base = m_data + m_size;
|
||||
auto grow_size = new_size - m_size;
|
||||
TRY(Core::System::commit_memory(grow_base, grow_size));
|
||||
|
||||
m_size = new_size;
|
||||
return {};
|
||||
}
|
||||
|
||||
TRY(m_fallback.try_resize(new_size));
|
||||
m_size = m_fallback.size();
|
||||
return {};
|
||||
}
|
||||
|
||||
bool MemoryBuffer::contains_virtual_address(void const* address) const
|
||||
{
|
||||
if (!m_mapping_base)
|
||||
return false;
|
||||
|
||||
auto fault_address = bit_cast<FlatPtr>(address);
|
||||
auto base = bit_cast<FlatPtr>(m_data);
|
||||
return fault_address >= base && fault_address < base + m_mapping_size;
|
||||
}
|
||||
|
||||
ErrorOr<MemoryInstance> MemoryInstance::create(MemoryType const& type)
|
||||
{
|
||||
MemoryInstance instance { type };
|
||||
|
||||
if (!instance.grow(type.limits().min() * Constants::page_size, GrowType::No))
|
||||
return Error::from_string_literal("Failed to grow to requested size");
|
||||
|
||||
return { move(instance) };
|
||||
}
|
||||
|
||||
MemoryInstance::MemoryInstance(MemoryType const& type)
|
||||
: m_type(type)
|
||||
{
|
||||
if (type.limits().address_type() == AddressType::I32)
|
||||
m_data.try_reserve_wasm32_address_space();
|
||||
}
|
||||
|
||||
bool MemoryInstance::grow(size_t size_to_grow, GrowType grow_type, InhibitGrowCallback inhibit_callback)
|
||||
{
|
||||
if (size_to_grow == 0)
|
||||
return true;
|
||||
u64 new_size = m_data.size() + size_to_grow;
|
||||
if (new_size >= Constants::page_size * 65536)
|
||||
return false;
|
||||
if (auto max = m_type.limits().max(); max.has_value()) {
|
||||
if (max.value() * Constants::page_size < new_size)
|
||||
return false;
|
||||
}
|
||||
|
||||
auto previous_size = m_data.size();
|
||||
if (m_data.try_resize(new_size).is_error())
|
||||
return false;
|
||||
if (!m_data.is_virtual())
|
||||
m_data.span().slice(previous_size, size_to_grow).fill(0);
|
||||
|
||||
if (inhibit_callback == InhibitGrowCallback::No && successful_grow_hook)
|
||||
successful_grow_hook();
|
||||
|
||||
if (grow_type == GrowType::Yes)
|
||||
m_type = MemoryType { Limits(m_type.limits().address_type(), m_type.limits().min() + size_to_grow / Constants::page_size, m_type.limits().max()) };
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
Optional<FunctionAddress> Store::allocate(ModuleInstance& instance, Module const& module, CodeSection::Code const& code, TypeIndex type_index)
|
||||
{
|
||||
FunctionAddress address { m_functions.size() };
|
||||
|
|
@ -55,7 +204,7 @@ Optional<MemoryAddress> Store::allocate(MemoryType const& type)
|
|||
if (instance.is_error())
|
||||
return {};
|
||||
|
||||
m_memories.append(instance.release_value());
|
||||
m_memories.append(make<MemoryInstance>(instance.release_value()));
|
||||
return address;
|
||||
}
|
||||
|
||||
|
|
@ -123,7 +272,7 @@ MemoryInstance* Store::get(MemoryAddress address)
|
|||
auto value = address.value();
|
||||
if (m_memories.size() <= value)
|
||||
return nullptr;
|
||||
return &m_memories[value];
|
||||
return m_memories[value].ptr();
|
||||
}
|
||||
|
||||
GlobalInstance* Store::get(GlobalAddress address)
|
||||
|
|
@ -299,7 +448,7 @@ InstantiationResult AbstractMachine::instantiate(Module const& module, Vector<Ex
|
|||
auxiliary_instance,
|
||||
Vector<Value, ArgumentsStaticSize> {},
|
||||
entry.expression(),
|
||||
1);
|
||||
1uz);
|
||||
auto result = config.execute(interpreter);
|
||||
if (result.is_trap())
|
||||
return InstantiationError { "Global instantiation trapped", move(result.trap()) };
|
||||
|
|
@ -354,7 +503,7 @@ InstantiationResult AbstractMachine::instantiate(Module const& module, Vector<Ex
|
|||
main_module_instance,
|
||||
Vector<Value, ArgumentsStaticSize> {},
|
||||
active_ptr->expression,
|
||||
1);
|
||||
1uz);
|
||||
auto result = config.execute(interpreter);
|
||||
if (result.is_trap())
|
||||
return InstantiationError { "Element section initialisation trapped", move(result.trap()) };
|
||||
|
|
@ -387,7 +536,7 @@ InstantiationResult AbstractMachine::instantiate(Module const& module, Vector<Ex
|
|||
main_module_instance,
|
||||
Vector<Value, ArgumentsStaticSize> {},
|
||||
data.offset,
|
||||
1);
|
||||
1uz);
|
||||
auto result = config.execute(interpreter);
|
||||
if (result.is_trap())
|
||||
return InstantiationError { "Data section initialisation trapped", move(result.trap()) };
|
||||
|
|
|
|||
|
|
@ -457,22 +457,61 @@ private:
|
|||
TableType m_type;
|
||||
};
|
||||
|
||||
class MemoryInstance {
|
||||
class WASM_API MemoryBuffer {
|
||||
public:
|
||||
static ErrorOr<MemoryInstance> create(MemoryType const& type)
|
||||
MemoryBuffer() = default;
|
||||
~MemoryBuffer();
|
||||
|
||||
MemoryBuffer(MemoryBuffer&&);
|
||||
MemoryBuffer& operator=(MemoryBuffer&&);
|
||||
|
||||
MemoryBuffer(MemoryBuffer const&) = delete;
|
||||
MemoryBuffer& operator=(MemoryBuffer const&) = delete;
|
||||
|
||||
void try_reserve_wasm32_address_space();
|
||||
ErrorOr<void> try_resize(size_t new_size);
|
||||
|
||||
auto size() const { return m_size; }
|
||||
auto data() const { return m_data ? m_data : m_fallback.data(); }
|
||||
auto data() { return m_data ? m_data : m_fallback.data(); }
|
||||
Bytes bytes() { return { data(), size() }; }
|
||||
ReadonlyBytes bytes() const { return { data(), size() }; }
|
||||
Bytes span() { return bytes(); }
|
||||
ReadonlyBytes span() const { return bytes(); }
|
||||
u8* offset_pointer(size_t offset) { return data() + offset; }
|
||||
u8 const* offset_pointer(size_t offset) const { return data() + offset; }
|
||||
u8& operator[](size_t index) { return data()[index]; }
|
||||
u8 const& operator[](size_t index) const { return data()[index]; }
|
||||
void overwrite(size_t offset, void const* source, size_t count)
|
||||
{
|
||||
MemoryInstance instance { type };
|
||||
|
||||
if (!instance.grow(type.limits().min() * Constants::page_size, GrowType::No))
|
||||
return Error::from_string_literal("Failed to grow to requested size");
|
||||
|
||||
return { move(instance) };
|
||||
VERIFY(offset <= size());
|
||||
VERIFY(count <= size() - offset);
|
||||
__builtin_memcpy(offset_pointer(offset), source, count);
|
||||
}
|
||||
bool is_virtual() const { return m_data != nullptr; }
|
||||
bool contains_virtual_address(void const* address) const;
|
||||
|
||||
private:
|
||||
void clear();
|
||||
|
||||
size_t m_size { 0 };
|
||||
size_t m_reserved_capacity { 0 };
|
||||
size_t m_mapping_size { 0 };
|
||||
size_t m_host_page_size { 0 };
|
||||
void* m_mapping_base { nullptr };
|
||||
u8* m_data { nullptr };
|
||||
ByteBuffer m_fallback;
|
||||
};
|
||||
|
||||
class WASM_API MemoryInstance {
|
||||
public:
|
||||
static ErrorOr<MemoryInstance> create(MemoryType const& type);
|
||||
|
||||
auto& type() const { return m_type; }
|
||||
auto size() const { return m_size; }
|
||||
auto size() const { return m_data.size(); }
|
||||
auto& data() const { return m_data; }
|
||||
auto& data() { return m_data; }
|
||||
bool contains_virtual_address(void const* address) const { return m_data.contains_virtual_address(address); }
|
||||
|
||||
enum class InhibitGrowCallback {
|
||||
No,
|
||||
|
|
@ -484,52 +523,15 @@ public:
|
|||
Yes,
|
||||
};
|
||||
|
||||
bool grow(size_t size_to_grow, GrowType grow_type = GrowType::Yes, InhibitGrowCallback inhibit_callback = InhibitGrowCallback::No)
|
||||
{
|
||||
if (size_to_grow == 0)
|
||||
return true;
|
||||
u64 new_size = m_data.size() + size_to_grow;
|
||||
// Can't grow past 2^16 pages.
|
||||
if (new_size >= Constants::page_size * 65536)
|
||||
return false;
|
||||
if (auto max = m_type.limits().max(); max.has_value()) {
|
||||
if (max.value() * Constants::page_size < new_size)
|
||||
return false;
|
||||
}
|
||||
auto previous_size = m_size;
|
||||
if (m_data.try_resize(new_size).is_error())
|
||||
return false;
|
||||
m_size = new_size;
|
||||
// The spec requires that we zero out everything on grow
|
||||
__builtin_memset(m_data.offset_pointer(previous_size), 0, size_to_grow);
|
||||
|
||||
// NOTE: This exists because wasm-js-api wants to execute code after a successful grow,
|
||||
// See [this issue](https://github.com/WebAssembly/spec/issues/1635) for more details.
|
||||
if (inhibit_callback == InhibitGrowCallback::No && successful_grow_hook)
|
||||
successful_grow_hook();
|
||||
|
||||
if (grow_type == GrowType::Yes) {
|
||||
// Grow the memory's type. We do this when encountering a `memory.grow`.
|
||||
//
|
||||
// See relevant spec link:
|
||||
// https://www.w3.org/TR/wasm-core-2/#growing-memories%E2%91%A0
|
||||
m_type = MemoryType { Limits(m_type.limits().address_type(), m_type.limits().min() + size_to_grow / Constants::page_size, m_type.limits().max()) };
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
bool grow(size_t size_to_grow, GrowType grow_type = GrowType::Yes, InhibitGrowCallback inhibit_callback = InhibitGrowCallback::No);
|
||||
|
||||
Function<void()> successful_grow_hook;
|
||||
|
||||
private:
|
||||
explicit MemoryInstance(MemoryType const& type)
|
||||
: m_type(type)
|
||||
{
|
||||
}
|
||||
explicit MemoryInstance(MemoryType const& type);
|
||||
|
||||
MemoryType m_type;
|
||||
size_t m_size { 0 };
|
||||
ByteBuffer m_data;
|
||||
MemoryBuffer m_data;
|
||||
};
|
||||
|
||||
class GlobalInstance {
|
||||
|
|
@ -644,12 +646,13 @@ public:
|
|||
TagInstance* get(TagAddress);
|
||||
ExceptionInstance* get(ExceptionAddress);
|
||||
|
||||
MemoryInstance* unsafe_get(MemoryAddress address) { return &m_memories.data()[address.value()]; }
|
||||
ALWAYS_INLINE FunctionInstance* unsafe_get(FunctionAddress address) { return &m_functions.data()[address.value()]; }
|
||||
ALWAYS_INLINE MemoryInstance* unsafe_get(MemoryAddress address) { return m_memories[address.value()].ptr(); }
|
||||
|
||||
private:
|
||||
Vector<FunctionInstance> m_functions;
|
||||
Vector<TableInstance> m_tables;
|
||||
Vector<MemoryInstance> m_memories;
|
||||
Vector<NonnullOwnPtr<MemoryInstance>> m_memories;
|
||||
Vector<GlobalInstance> m_globals;
|
||||
Vector<ElementInstance> m_elements;
|
||||
Vector<DataInstance> m_datas;
|
||||
|
|
@ -678,17 +681,30 @@ private:
|
|||
|
||||
class Frame {
|
||||
public:
|
||||
// Owning constructor (slow path).
|
||||
explicit Frame(ModuleInstance const& module, Vector<Value, ArgumentsStaticSize> locals, Expression const& expression, size_t arity)
|
||||
: m_module(module)
|
||||
, m_locals(move(locals))
|
||||
, m_owned_locals(move(locals))
|
||||
, m_locals_ptr(m_owned_locals.data())
|
||||
, m_expression(expression)
|
||||
, m_arity(arity)
|
||||
, m_owns_locals(true)
|
||||
{
|
||||
}
|
||||
|
||||
// Non-owning constructor (fast path).
|
||||
explicit Frame(ModuleInstance const& module, Value* locals_ptr, Expression const& expression, size_t arity)
|
||||
: m_module(module)
|
||||
, m_locals_ptr(locals_ptr)
|
||||
, m_expression(expression)
|
||||
, m_arity(arity)
|
||||
{
|
||||
}
|
||||
|
||||
auto& module() const { return m_module; }
|
||||
auto& locals() const { return m_locals; }
|
||||
auto& locals() { return m_locals; }
|
||||
Value* locals_data() const { return m_locals_ptr; }
|
||||
bool owns_locals() const { return m_owns_locals; }
|
||||
Vector<Value, ArgumentsStaticSize>& owned_locals() { return m_owned_locals; }
|
||||
auto& expression() const { return m_expression; }
|
||||
auto arity() const { return m_arity; }
|
||||
auto label_index() const { return m_label_index; }
|
||||
|
|
@ -696,10 +712,12 @@ public:
|
|||
|
||||
private:
|
||||
ModuleInstance const& m_module;
|
||||
Vector<Value, ArgumentsStaticSize> m_locals;
|
||||
Vector<Value, ArgumentsStaticSize> m_owned_locals;
|
||||
Value* m_locals_ptr { nullptr };
|
||||
Expression const& m_expression;
|
||||
size_t m_arity { 0 };
|
||||
size_t m_label_index { 0 };
|
||||
bool m_owns_locals { false };
|
||||
};
|
||||
|
||||
using InstantiationResult = AK::ErrorOr<NonnullOwnPtr<ModuleInstance>, InstantiationError>;
|
||||
|
|
|
|||
|
|
@ -13,10 +13,31 @@ namespace Wasm {
|
|||
|
||||
void Configuration::unwind_impl()
|
||||
{
|
||||
if (m_compiled_direct_call_depth > 0) {
|
||||
m_compiled_direct_call_depth--;
|
||||
m_depth--;
|
||||
return;
|
||||
}
|
||||
auto last_frame = m_frame_stack.take_last();
|
||||
m_depth--;
|
||||
m_locals_base = m_frame_stack.is_empty() ? nullptr : m_frame_stack.unchecked_last().locals().data();
|
||||
release_arguments_allocation(last_frame.locals(), m_locals_base != nullptr);
|
||||
|
||||
m_locals_base = m_frame_stack.is_empty() ? nullptr : m_frame_stack.last().locals_data();
|
||||
if (m_frame_stack.is_empty()) {
|
||||
m_default_memory = nullptr;
|
||||
m_default_memory_base = nullptr;
|
||||
} else {
|
||||
auto const& memories = m_frame_stack.last().module().memories();
|
||||
m_default_memory = memories.is_empty() ? nullptr : m_store.unsafe_get(memories[0]);
|
||||
m_default_memory_base = m_default_memory ? m_default_memory->data().data() : nullptr;
|
||||
}
|
||||
|
||||
if (!last_frame.owns_locals()) {
|
||||
// Non-owning frame: just restore the caller's runtime state.
|
||||
return;
|
||||
}
|
||||
|
||||
// Owning frame: full cleanup.
|
||||
release_arguments_allocation(last_frame.owned_locals(), m_locals_base != nullptr);
|
||||
}
|
||||
|
||||
Result Configuration::call(Interpreter& interpreter, FunctionAddress address, Vector<Value, ArgumentsStaticSize>& arguments)
|
||||
|
|
@ -34,20 +55,7 @@ ErrorOr<Optional<HostFunction&>, Trap> Configuration::prepare_call(FunctionAddre
|
|||
return Trap::from_string("Attempt to call nonexistent function by address");
|
||||
|
||||
if (auto* wasm_function = function->get_pointer<WasmFunction>()) {
|
||||
if (is_tailcall)
|
||||
unwind_impl(); // Unwind the current frame, the "return" in the tail-called function will unwind the frame we're gonna push now.
|
||||
arguments.ensure_capacity(arguments.size() + wasm_function->code().func().total_local_count());
|
||||
for (auto& local : wasm_function->code().func().locals()) {
|
||||
for (size_t i = 0; i < local.n(); ++i)
|
||||
arguments.unchecked_append(Value(local.type()));
|
||||
}
|
||||
|
||||
set_frame(
|
||||
is_tailcall ? IsTailcall::Yes : IsTailcall::No,
|
||||
wasm_function->module(),
|
||||
move(arguments),
|
||||
wasm_function->code().func().body(),
|
||||
wasm_function->type().results().size());
|
||||
TRY(prepare_wasm_call(*wasm_function, arguments, is_tailcall));
|
||||
return OptionalNone {};
|
||||
}
|
||||
|
||||
|
|
@ -71,6 +79,63 @@ Result Configuration::execute(Interpreter& interpreter)
|
|||
return Result { move(results) };
|
||||
}
|
||||
|
||||
ErrorOr<void, Trap> Configuration::execute_for_compiled_call(Interpreter& interpreter, Value* single_result)
|
||||
{
|
||||
interpreter.interpret(*this);
|
||||
if (interpreter.did_trap())
|
||||
return interpreter.trap();
|
||||
|
||||
VERIFY(frame().arity() <= 1);
|
||||
if (frame().arity() == 1) {
|
||||
auto result = value_stack().unsafe_take_last();
|
||||
if (single_result)
|
||||
*single_result = result;
|
||||
}
|
||||
|
||||
if (!label_stack().is_empty())
|
||||
label_stack().take_last();
|
||||
|
||||
return {};
|
||||
}
|
||||
|
||||
void Configuration::build_compiled_function_table()
|
||||
{
|
||||
if (m_frame_stack.is_empty())
|
||||
return;
|
||||
auto const* current_module = &frame().module();
|
||||
if (m_compiled_fn_table_module == current_module && !m_compiled_fn_table.is_empty())
|
||||
return;
|
||||
|
||||
m_compiled_fn_table.clear();
|
||||
m_compiled_fn_table_module = current_module;
|
||||
|
||||
auto const& functions = frame().module().functions();
|
||||
auto count = functions.size();
|
||||
if (count == 0)
|
||||
return;
|
||||
|
||||
for (size_t i = 0; i < count; i++) {
|
||||
auto* instance = m_store.unsafe_get(functions[i]);
|
||||
auto* wasm_fn = instance->get_pointer<WasmFunction>();
|
||||
if (!wasm_fn)
|
||||
continue;
|
||||
auto& ci = wasm_fn->code().func().body().compiled_instructions;
|
||||
if (!ci.cranelift_compiled)
|
||||
continue;
|
||||
CompiledFunctionEntry entry;
|
||||
entry.handler_ptr = ci.dispatches[0].handler_ptr;
|
||||
entry.dispatches_ptr = bit_cast<FlatPtr>(ci.dispatches.data());
|
||||
entry.src_dst_ptr = bit_cast<FlatPtr>(ci.src_dst_mappings.data());
|
||||
entry.first_insn = ci.dispatches[0].instruction;
|
||||
entry.expression = &wasm_fn->code().func().body();
|
||||
entry.module = &wasm_fn->module();
|
||||
entry.total_local_count = static_cast<u32>(wasm_fn->code().func().total_local_count());
|
||||
entry.arity = static_cast<u32>(wasm_fn->type().results().size());
|
||||
entry.max_call_rec_size = static_cast<u32>(ci.max_call_rec_size);
|
||||
m_compiled_fn_table.set(static_cast<u32>(i), entry);
|
||||
}
|
||||
}
|
||||
|
||||
void Configuration::dump_stack()
|
||||
{
|
||||
auto print_value = []<typename... Ts>(CheckedFormatString<Ts...> format, Ts... vs) {
|
||||
|
|
|
|||
|
|
@ -34,10 +34,13 @@ public:
|
|||
template<typename... Args>
|
||||
void set_frame(IsTailcall is_tailcall, Args&&... frame_init)
|
||||
{
|
||||
m_frame_stack.append(forward<Args>(frame_init)...);
|
||||
m_frame_stack.empend(forward<Args>(frame_init)...);
|
||||
|
||||
auto& frame = m_frame_stack.unchecked_last();
|
||||
m_locals_base = frame.locals().data();
|
||||
auto& frame = m_frame_stack.last();
|
||||
m_locals_base = frame.locals_data();
|
||||
auto const& memories = frame.module().memories();
|
||||
m_default_memory = memories.is_empty() ? nullptr : m_store.unsafe_get(memories[0]);
|
||||
m_default_memory_base = m_default_memory ? m_default_memory->data().data() : nullptr;
|
||||
|
||||
auto continuation = frame.expression().instructions().size() - 1;
|
||||
if (auto size = frame.expression().compiled_instructions.dispatches.size(); size > 0)
|
||||
|
|
@ -61,8 +64,33 @@ public:
|
|||
m_call_record_base = nullptr;
|
||||
}
|
||||
}
|
||||
ALWAYS_INLINE auto& frame() const { return m_frame_stack.unchecked_last(); }
|
||||
ALWAYS_INLINE auto& frame() { return m_frame_stack.unchecked_last(); }
|
||||
// Lightweight set_frame for direct Cranelift-to-Cranelift calls.
|
||||
void set_frame_lightweight(ModuleInstance const& module, Value* locals_ptr,
|
||||
Expression const& expression, size_t arity)
|
||||
{
|
||||
m_frame_stack.empend(module, locals_ptr, expression, arity);
|
||||
m_locals_base = locals_ptr;
|
||||
auto const& memories = module.memories();
|
||||
m_default_memory = memories.is_empty() ? nullptr : m_store.unsafe_get(memories[0]);
|
||||
m_default_memory_base = m_default_memory ? m_default_memory->data().data() : nullptr;
|
||||
// Skip capacity hints and label push (Cranelift uses its own structured control flow).
|
||||
}
|
||||
|
||||
void setup_call_record_for_current_frame()
|
||||
{
|
||||
auto max_call_rec_size = m_frame_stack.last().expression().compiled_instructions.max_call_rec_size;
|
||||
if (max_call_rec_size > 0) {
|
||||
m_current_call_record.clear_with_capacity();
|
||||
m_current_call_record.ensure_capacity(max_call_rec_size);
|
||||
m_current_call_record.resize_and_keep_capacity(max_call_rec_size);
|
||||
m_call_record_base = m_current_call_record.data();
|
||||
} else {
|
||||
m_call_record_base = nullptr;
|
||||
}
|
||||
}
|
||||
|
||||
ALWAYS_INLINE auto& frame() const { return m_frame_stack.last(); }
|
||||
ALWAYS_INLINE auto& frame() { return m_frame_stack.last(); }
|
||||
ALWAYS_INLINE auto& ip() const { return m_ip; }
|
||||
ALWAYS_INLINE auto& ip() { return m_ip; }
|
||||
ALWAYS_INLINE auto& depth() const { return m_depth; }
|
||||
|
|
@ -73,16 +101,71 @@ public:
|
|||
ALWAYS_INLINE auto& label_stack() { return m_label_stack; }
|
||||
ALWAYS_INLINE auto& store() const { return m_store; }
|
||||
ALWAYS_INLINE auto& store() { return m_store; }
|
||||
ALWAYS_INLINE MemoryInstance* default_memory() const { return m_default_memory; }
|
||||
ALWAYS_INLINE u8* default_memory_base() const { return m_default_memory_base; }
|
||||
ALWAYS_INLINE void refresh_default_memory_base() { m_default_memory_base = m_default_memory ? m_default_memory->data().data() : nullptr; }
|
||||
ALWAYS_INLINE Value& compiled_call_result_scratch() { return m_compiled_call_result_scratch; }
|
||||
ALWAYS_INLINE Value const& compiled_call_result_scratch() const { return m_compiled_call_result_scratch; }
|
||||
|
||||
ALWAYS_INLINE Value const& local(LocalIndex index) const { return m_locals_base[index.value()]; }
|
||||
ALWAYS_INLINE Value& local(LocalIndex index) { return m_locals_base[index.value()]; }
|
||||
ALWAYS_INLINE Value* locals_base() const { return m_locals_base; }
|
||||
ALWAYS_INLINE void set_locals_base(Value* base) { m_locals_base = base; }
|
||||
|
||||
// When > 0, unwind_impl skips the frame pop (the direct call didn't push a frame).
|
||||
size_t m_compiled_direct_call_depth { 0 };
|
||||
|
||||
// Per-function entry for the compiled function table (direct calls from Cranelift).
|
||||
struct CompiledFunctionEntry {
|
||||
FlatPtr handler_ptr { 0 }; // 0 = not compiled, use slow path
|
||||
FlatPtr dispatches_ptr { 0 }; // Dispatch const* for the handler call
|
||||
FlatPtr src_dst_ptr { 0 }; // SourcesAndDestination const*
|
||||
Instruction const* first_insn { nullptr };
|
||||
Expression const* expression { nullptr };
|
||||
ModuleInstance const* module { nullptr };
|
||||
u32 total_local_count { 0 };
|
||||
u32 arity { 0 };
|
||||
u32 max_call_rec_size { 0 };
|
||||
};
|
||||
HashMap<u32, CompiledFunctionEntry> m_compiled_fn_table;
|
||||
ModuleInstance const* m_compiled_fn_table_module { nullptr };
|
||||
|
||||
void build_compiled_function_table();
|
||||
|
||||
static constexpr size_t locals_base_offset() { return __builtin_offsetof(Configuration, m_locals_base); }
|
||||
static constexpr size_t default_memory_base_offset() { return __builtin_offsetof(Configuration, m_default_memory_base); }
|
||||
static constexpr size_t compiled_call_result_scratch_offset() { return __builtin_offsetof(Configuration, m_compiled_call_result_scratch); }
|
||||
|
||||
ALWAYS_INLINE Value& call_record_entry(size_t index) { return m_call_record_base[index]; }
|
||||
ALWAYS_INLINE Value const& call_record_entry(size_t index) const { return m_call_record_base[index]; }
|
||||
ALWAYS_INLINE Value* call_record_base() const { return m_call_record_base; }
|
||||
ALWAYS_INLINE void set_call_record_base(Value* base) { m_call_record_base = base; }
|
||||
ALWAYS_INLINE void setup_call_record(size_t max_call_rec_size)
|
||||
{
|
||||
get_arguments_allocation_if_possible(m_current_call_record, max_call_rec_size);
|
||||
m_current_call_record.resize_and_keep_capacity(max_call_rec_size);
|
||||
m_call_record_base = m_current_call_record.data();
|
||||
}
|
||||
ALWAYS_INLINE Vector<Value, ArgumentsStaticSize> take_call_record_vector()
|
||||
{
|
||||
auto result = move(m_current_call_record);
|
||||
m_call_record_base = nullptr;
|
||||
return result;
|
||||
}
|
||||
ALWAYS_INLINE void restore_call_record_vector(Vector<Value, ArgumentsStaticSize>&& vec)
|
||||
{
|
||||
m_current_call_record = move(vec);
|
||||
m_call_record_base = m_current_call_record.data();
|
||||
}
|
||||
|
||||
struct CallFrameHandle {
|
||||
explicit CallFrameHandle(Configuration& configuration)
|
||||
: configuration(configuration)
|
||||
, saved_direct_call_depth(configuration.m_compiled_direct_call_depth)
|
||||
{
|
||||
if (configuration.m_call_record_base)
|
||||
moved_call_record = move(configuration.m_current_call_record);
|
||||
configuration.m_compiled_direct_call_depth = 0;
|
||||
configuration.depth()++;
|
||||
configuration.m_call_record_base = nullptr;
|
||||
}
|
||||
|
|
@ -96,16 +179,38 @@ public:
|
|||
configuration.m_call_record_base = nullptr;
|
||||
}
|
||||
configuration.unwind({}, *this);
|
||||
configuration.m_compiled_direct_call_depth = saved_direct_call_depth;
|
||||
}
|
||||
|
||||
Configuration& configuration;
|
||||
Optional<Vector<Value, ArgumentsStaticSize>> moved_call_record;
|
||||
size_t saved_direct_call_depth;
|
||||
};
|
||||
|
||||
void unwind(Badge<CallFrameHandle>, CallFrameHandle const&) { unwind_impl(); }
|
||||
ErrorOr<Optional<HostFunction&>, Trap> prepare_call(FunctionAddress, Vector<Value, ArgumentsStaticSize>& arguments, bool is_tailcall = false);
|
||||
ALWAYS_INLINE ErrorOr<void, Trap> prepare_wasm_call(WasmFunction const& wasm_function, Vector<Value, ArgumentsStaticSize>& arguments, bool is_tailcall = false)
|
||||
{
|
||||
if (is_tailcall)
|
||||
unwind_impl();
|
||||
|
||||
arguments.ensure_capacity(arguments.size() + wasm_function.code().func().total_local_count());
|
||||
for (auto const& local : wasm_function.code().func().locals()) {
|
||||
for (size_t i = 0; i < local.n(); ++i)
|
||||
arguments.unchecked_append(Value(local.type()));
|
||||
}
|
||||
|
||||
set_frame(
|
||||
is_tailcall ? IsTailcall::Yes : IsTailcall::No,
|
||||
wasm_function.module(),
|
||||
move(arguments),
|
||||
wasm_function.code().func().body(),
|
||||
wasm_function.type().results().size());
|
||||
return {};
|
||||
}
|
||||
Result call(Interpreter&, FunctionAddress, Vector<Value, ArgumentsStaticSize>& arguments);
|
||||
Result execute(Interpreter&);
|
||||
ErrorOr<void, Trap> execute_for_compiled_call(Interpreter&, Value* single_result = nullptr);
|
||||
|
||||
void enable_instruction_count_limit() { m_should_limit_instruction_count = true; }
|
||||
bool should_limit_instruction_count() const { return m_should_limit_instruction_count; }
|
||||
|
|
@ -143,7 +248,8 @@ public:
|
|||
return;
|
||||
}
|
||||
|
||||
VERIFY(m_current_call_record.size() >= size);
|
||||
if (m_current_call_record.size() < size)
|
||||
m_current_call_record.resize_and_keep_capacity(size);
|
||||
}
|
||||
|
||||
if (arguments.capacity() != ArgumentsStaticSize) {
|
||||
|
|
@ -257,13 +363,13 @@ public:
|
|||
Value(0),
|
||||
};
|
||||
|
||||
private:
|
||||
// Public for CraneliftBridge direct call pop_frame.
|
||||
void unwind_impl();
|
||||
|
||||
Store& m_store;
|
||||
Vector<Value, 64, FastLastAccess::Yes> m_value_stack;
|
||||
Vector<Label, 64, FastLastAccess::Yes> m_label_stack;
|
||||
DoublyLinkedList<Frame, 512> m_frame_stack;
|
||||
Vector<Frame> m_frame_stack;
|
||||
Vector<Value, ArgumentsStaticSize> m_current_call_record;
|
||||
Vector<Vector<Value, ArgumentsStaticSize>, 16, FastLastAccess::Yes> m_call_argument_freelist;
|
||||
size_t m_depth { 0 };
|
||||
|
|
@ -271,6 +377,9 @@ private:
|
|||
bool m_should_limit_instruction_count { false };
|
||||
Value* m_locals_base { nullptr };
|
||||
Value* m_call_record_base { nullptr };
|
||||
MemoryInstance* m_default_memory { nullptr };
|
||||
u8* m_default_memory_base { nullptr };
|
||||
Value m_compiled_call_result_scratch;
|
||||
};
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ class AbstractMachine;
|
|||
class Validator;
|
||||
struct ValidationError;
|
||||
struct Interpreter;
|
||||
class MemoryBuffer;
|
||||
|
||||
namespace Wasi {
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue