From f181d24dc02dd509a67ff48a4c6bdbd3605ad491 Mon Sep 17 00:00:00 2001 From: Ali Mohammad Pur Date: Sat, 4 Apr 2026 19:26:41 +0200 Subject: [PATCH] LibWasm: Prepare for VM and direct function calls Introduce MemoryBuffer, a memory backing store that uses mmap to reserve the full wasm32 address space (4 GiB + guard pages) upfront, growing without a copy and falling back to a ByteBuffer when mmap fails. Also let Frame know how to handle non-owned locals (to e.g. allow allocating them on the native stack.) --- .../AbstractMachine/AbstractMachine.cpp | 159 +++++++++++++++++- .../LibWasm/AbstractMachine/AbstractMachine.h | 128 ++++++++------ .../LibWasm/AbstractMachine/Configuration.cpp | 97 +++++++++-- .../LibWasm/AbstractMachine/Configuration.h | 125 +++++++++++++- Libraries/LibWasm/Forward.h | 1 + 5 files changed, 426 insertions(+), 84 deletions(-) diff --git a/Libraries/LibWasm/AbstractMachine/AbstractMachine.cpp b/Libraries/LibWasm/AbstractMachine/AbstractMachine.cpp index de7b143948..522644ecfa 100644 --- a/Libraries/LibWasm/AbstractMachine/AbstractMachine.cpp +++ b/Libraries/LibWasm/AbstractMachine/AbstractMachine.cpp @@ -6,6 +6,7 @@ #include #include +#include #include #include #include @@ -15,6 +16,154 @@ namespace Wasm { +MemoryBuffer::~MemoryBuffer() +{ + clear(); +} + +MemoryBuffer::MemoryBuffer(MemoryBuffer&& other) + : m_size(exchange(other.m_size, 0)) + , m_reserved_capacity(exchange(other.m_reserved_capacity, 0)) + , m_mapping_size(exchange(other.m_mapping_size, 0)) + , m_host_page_size(exchange(other.m_host_page_size, 0)) + , m_mapping_base(exchange(other.m_mapping_base, nullptr)) + , m_data(exchange(other.m_data, nullptr)) + , m_fallback(move(other.m_fallback)) +{ +} + +MemoryBuffer& MemoryBuffer::operator=(MemoryBuffer&& other) +{ + if (this != &other) { + clear(); + m_size = exchange(other.m_size, 0); + m_reserved_capacity = exchange(other.m_reserved_capacity, 0); + m_mapping_size = exchange(other.m_mapping_size, 0); + m_host_page_size = exchange(other.m_host_page_size, 0); + m_mapping_base = exchange(other.m_mapping_base, nullptr); + m_data = exchange(other.m_data, nullptr); + m_fallback = move(other.m_fallback); + } + return *this; +} + +void MemoryBuffer::clear() +{ + if (m_mapping_base) { + VERIFY(m_reserved_capacity); + VERIFY(m_mapping_size); + VERIFY(m_host_page_size); + auto reservation_size = m_mapping_size + 2 * m_host_page_size; + [[maybe_unused]] auto result = Core::System::release_address_space(m_mapping_base, reservation_size); + VERIFY(!result.is_error()); + } + m_mapping_base = nullptr; + m_data = nullptr; + m_reserved_capacity = 0; + m_mapping_size = 0; + m_host_page_size = 0; + m_size = 0; + m_fallback.clear(); +} + +void MemoryBuffer::try_reserve_wasm32_address_space() +{ + if (m_mapping_base) + return; + + auto host_page_size = static_cast(PAGE_SIZE); + auto reserved_capacity = static_cast(Constants::page_size) * 65536; + auto mapping_size = reserved_capacity * 2; + auto reservation_size = mapping_size + 2 * host_page_size; + + auto mapping_or_error = Core::System::reserve_address_space(reservation_size); + if (mapping_or_error.is_error()) + return; + + m_mapping_base = mapping_or_error.value(); + m_data = reinterpret_cast(m_mapping_base) + host_page_size; + m_reserved_capacity = reserved_capacity; + m_mapping_size = mapping_size; + m_host_page_size = host_page_size; +} + +ErrorOr MemoryBuffer::try_resize(size_t new_size) +{ + if (m_data) { + VERIFY(new_size >= m_size); + VERIFY(m_host_page_size); + if (new_size > m_reserved_capacity) + return Error::from_errno(ENOMEM); + if (new_size == m_size) + return {}; + + auto* grow_base = m_data + m_size; + auto grow_size = new_size - m_size; + TRY(Core::System::commit_memory(grow_base, grow_size)); + + m_size = new_size; + return {}; + } + + TRY(m_fallback.try_resize(new_size)); + m_size = m_fallback.size(); + return {}; +} + +bool MemoryBuffer::contains_virtual_address(void const* address) const +{ + if (!m_mapping_base) + return false; + + auto fault_address = bit_cast(address); + auto base = bit_cast(m_data); + return fault_address >= base && fault_address < base + m_mapping_size; +} + +ErrorOr MemoryInstance::create(MemoryType const& type) +{ + MemoryInstance instance { type }; + + if (!instance.grow(type.limits().min() * Constants::page_size, GrowType::No)) + return Error::from_string_literal("Failed to grow to requested size"); + + return { move(instance) }; +} + +MemoryInstance::MemoryInstance(MemoryType const& type) + : m_type(type) +{ + if (type.limits().address_type() == AddressType::I32) + m_data.try_reserve_wasm32_address_space(); +} + +bool MemoryInstance::grow(size_t size_to_grow, GrowType grow_type, InhibitGrowCallback inhibit_callback) +{ + if (size_to_grow == 0) + return true; + u64 new_size = m_data.size() + size_to_grow; + if (new_size >= Constants::page_size * 65536) + return false; + if (auto max = m_type.limits().max(); max.has_value()) { + if (max.value() * Constants::page_size < new_size) + return false; + } + + auto previous_size = m_data.size(); + if (m_data.try_resize(new_size).is_error()) + return false; + if (!m_data.is_virtual()) + m_data.span().slice(previous_size, size_to_grow).fill(0); + + if (inhibit_callback == InhibitGrowCallback::No && successful_grow_hook) + successful_grow_hook(); + + if (grow_type == GrowType::Yes) + m_type = MemoryType { Limits(m_type.limits().address_type(), m_type.limits().min() + size_to_grow / Constants::page_size, m_type.limits().max()) }; + + return true; +} + Optional Store::allocate(ModuleInstance& instance, Module const& module, CodeSection::Code const& code, TypeIndex type_index) { FunctionAddress address { m_functions.size() }; @@ -55,7 +204,7 @@ Optional Store::allocate(MemoryType const& type) if (instance.is_error()) return {}; - m_memories.append(instance.release_value()); + m_memories.append(make(instance.release_value())); return address; } @@ -123,7 +272,7 @@ MemoryInstance* Store::get(MemoryAddress address) auto value = address.value(); if (m_memories.size() <= value) return nullptr; - return &m_memories[value]; + return m_memories[value].ptr(); } GlobalInstance* Store::get(GlobalAddress address) @@ -299,7 +448,7 @@ InstantiationResult AbstractMachine::instantiate(Module const& module, Vector {}, entry.expression(), - 1); + 1uz); auto result = config.execute(interpreter); if (result.is_trap()) return InstantiationError { "Global instantiation trapped", move(result.trap()) }; @@ -354,7 +503,7 @@ InstantiationResult AbstractMachine::instantiate(Module const& module, Vector {}, active_ptr->expression, - 1); + 1uz); auto result = config.execute(interpreter); if (result.is_trap()) return InstantiationError { "Element section initialisation trapped", move(result.trap()) }; @@ -387,7 +536,7 @@ InstantiationResult AbstractMachine::instantiate(Module const& module, Vector {}, data.offset, - 1); + 1uz); auto result = config.execute(interpreter); if (result.is_trap()) return InstantiationError { "Data section initialisation trapped", move(result.trap()) }; diff --git a/Libraries/LibWasm/AbstractMachine/AbstractMachine.h b/Libraries/LibWasm/AbstractMachine/AbstractMachine.h index db2d42b377..ac9b4fc32c 100644 --- a/Libraries/LibWasm/AbstractMachine/AbstractMachine.h +++ b/Libraries/LibWasm/AbstractMachine/AbstractMachine.h @@ -457,22 +457,61 @@ private: TableType m_type; }; -class MemoryInstance { +class WASM_API MemoryBuffer { public: - static ErrorOr create(MemoryType const& type) + MemoryBuffer() = default; + ~MemoryBuffer(); + + MemoryBuffer(MemoryBuffer&&); + MemoryBuffer& operator=(MemoryBuffer&&); + + MemoryBuffer(MemoryBuffer const&) = delete; + MemoryBuffer& operator=(MemoryBuffer const&) = delete; + + void try_reserve_wasm32_address_space(); + ErrorOr try_resize(size_t new_size); + + auto size() const { return m_size; } + auto data() const { return m_data ? m_data : m_fallback.data(); } + auto data() { return m_data ? m_data : m_fallback.data(); } + Bytes bytes() { return { data(), size() }; } + ReadonlyBytes bytes() const { return { data(), size() }; } + Bytes span() { return bytes(); } + ReadonlyBytes span() const { return bytes(); } + u8* offset_pointer(size_t offset) { return data() + offset; } + u8 const* offset_pointer(size_t offset) const { return data() + offset; } + u8& operator[](size_t index) { return data()[index]; } + u8 const& operator[](size_t index) const { return data()[index]; } + void overwrite(size_t offset, void const* source, size_t count) { - MemoryInstance instance { type }; - - if (!instance.grow(type.limits().min() * Constants::page_size, GrowType::No)) - return Error::from_string_literal("Failed to grow to requested size"); - - return { move(instance) }; + VERIFY(offset <= size()); + VERIFY(count <= size() - offset); + __builtin_memcpy(offset_pointer(offset), source, count); } + bool is_virtual() const { return m_data != nullptr; } + bool contains_virtual_address(void const* address) const; + +private: + void clear(); + + size_t m_size { 0 }; + size_t m_reserved_capacity { 0 }; + size_t m_mapping_size { 0 }; + size_t m_host_page_size { 0 }; + void* m_mapping_base { nullptr }; + u8* m_data { nullptr }; + ByteBuffer m_fallback; +}; + +class WASM_API MemoryInstance { +public: + static ErrorOr create(MemoryType const& type); auto& type() const { return m_type; } - auto size() const { return m_size; } + auto size() const { return m_data.size(); } auto& data() const { return m_data; } auto& data() { return m_data; } + bool contains_virtual_address(void const* address) const { return m_data.contains_virtual_address(address); } enum class InhibitGrowCallback { No, @@ -484,52 +523,15 @@ public: Yes, }; - bool grow(size_t size_to_grow, GrowType grow_type = GrowType::Yes, InhibitGrowCallback inhibit_callback = InhibitGrowCallback::No) - { - if (size_to_grow == 0) - return true; - u64 new_size = m_data.size() + size_to_grow; - // Can't grow past 2^16 pages. - if (new_size >= Constants::page_size * 65536) - return false; - if (auto max = m_type.limits().max(); max.has_value()) { - if (max.value() * Constants::page_size < new_size) - return false; - } - auto previous_size = m_size; - if (m_data.try_resize(new_size).is_error()) - return false; - m_size = new_size; - // The spec requires that we zero out everything on grow - __builtin_memset(m_data.offset_pointer(previous_size), 0, size_to_grow); - - // NOTE: This exists because wasm-js-api wants to execute code after a successful grow, - // See [this issue](https://github.com/WebAssembly/spec/issues/1635) for more details. - if (inhibit_callback == InhibitGrowCallback::No && successful_grow_hook) - successful_grow_hook(); - - if (grow_type == GrowType::Yes) { - // Grow the memory's type. We do this when encountering a `memory.grow`. - // - // See relevant spec link: - // https://www.w3.org/TR/wasm-core-2/#growing-memories%E2%91%A0 - m_type = MemoryType { Limits(m_type.limits().address_type(), m_type.limits().min() + size_to_grow / Constants::page_size, m_type.limits().max()) }; - } - - return true; - } + bool grow(size_t size_to_grow, GrowType grow_type = GrowType::Yes, InhibitGrowCallback inhibit_callback = InhibitGrowCallback::No); Function successful_grow_hook; private: - explicit MemoryInstance(MemoryType const& type) - : m_type(type) - { - } + explicit MemoryInstance(MemoryType const& type); MemoryType m_type; - size_t m_size { 0 }; - ByteBuffer m_data; + MemoryBuffer m_data; }; class GlobalInstance { @@ -644,12 +646,13 @@ public: TagInstance* get(TagAddress); ExceptionInstance* get(ExceptionAddress); - MemoryInstance* unsafe_get(MemoryAddress address) { return &m_memories.data()[address.value()]; } + ALWAYS_INLINE FunctionInstance* unsafe_get(FunctionAddress address) { return &m_functions.data()[address.value()]; } + ALWAYS_INLINE MemoryInstance* unsafe_get(MemoryAddress address) { return m_memories[address.value()].ptr(); } private: Vector m_functions; Vector m_tables; - Vector m_memories; + Vector> m_memories; Vector m_globals; Vector m_elements; Vector m_datas; @@ -678,17 +681,30 @@ private: class Frame { public: + // Owning constructor (slow path). explicit Frame(ModuleInstance const& module, Vector locals, Expression const& expression, size_t arity) : m_module(module) - , m_locals(move(locals)) + , m_owned_locals(move(locals)) + , m_locals_ptr(m_owned_locals.data()) + , m_expression(expression) + , m_arity(arity) + , m_owns_locals(true) + { + } + + // Non-owning constructor (fast path). + explicit Frame(ModuleInstance const& module, Value* locals_ptr, Expression const& expression, size_t arity) + : m_module(module) + , m_locals_ptr(locals_ptr) , m_expression(expression) , m_arity(arity) { } auto& module() const { return m_module; } - auto& locals() const { return m_locals; } - auto& locals() { return m_locals; } + Value* locals_data() const { return m_locals_ptr; } + bool owns_locals() const { return m_owns_locals; } + Vector& owned_locals() { return m_owned_locals; } auto& expression() const { return m_expression; } auto arity() const { return m_arity; } auto label_index() const { return m_label_index; } @@ -696,10 +712,12 @@ public: private: ModuleInstance const& m_module; - Vector m_locals; + Vector m_owned_locals; + Value* m_locals_ptr { nullptr }; Expression const& m_expression; size_t m_arity { 0 }; size_t m_label_index { 0 }; + bool m_owns_locals { false }; }; using InstantiationResult = AK::ErrorOr, InstantiationError>; diff --git a/Libraries/LibWasm/AbstractMachine/Configuration.cpp b/Libraries/LibWasm/AbstractMachine/Configuration.cpp index 2d8b559c99..674975b834 100644 --- a/Libraries/LibWasm/AbstractMachine/Configuration.cpp +++ b/Libraries/LibWasm/AbstractMachine/Configuration.cpp @@ -13,10 +13,31 @@ namespace Wasm { void Configuration::unwind_impl() { + if (m_compiled_direct_call_depth > 0) { + m_compiled_direct_call_depth--; + m_depth--; + return; + } auto last_frame = m_frame_stack.take_last(); m_depth--; - m_locals_base = m_frame_stack.is_empty() ? nullptr : m_frame_stack.unchecked_last().locals().data(); - release_arguments_allocation(last_frame.locals(), m_locals_base != nullptr); + + m_locals_base = m_frame_stack.is_empty() ? nullptr : m_frame_stack.last().locals_data(); + if (m_frame_stack.is_empty()) { + m_default_memory = nullptr; + m_default_memory_base = nullptr; + } else { + auto const& memories = m_frame_stack.last().module().memories(); + m_default_memory = memories.is_empty() ? nullptr : m_store.unsafe_get(memories[0]); + m_default_memory_base = m_default_memory ? m_default_memory->data().data() : nullptr; + } + + if (!last_frame.owns_locals()) { + // Non-owning frame: just restore the caller's runtime state. + return; + } + + // Owning frame: full cleanup. + release_arguments_allocation(last_frame.owned_locals(), m_locals_base != nullptr); } Result Configuration::call(Interpreter& interpreter, FunctionAddress address, Vector& arguments) @@ -34,20 +55,7 @@ ErrorOr, Trap> Configuration::prepare_call(FunctionAddre return Trap::from_string("Attempt to call nonexistent function by address"); if (auto* wasm_function = function->get_pointer()) { - if (is_tailcall) - unwind_impl(); // Unwind the current frame, the "return" in the tail-called function will unwind the frame we're gonna push now. - arguments.ensure_capacity(arguments.size() + wasm_function->code().func().total_local_count()); - for (auto& local : wasm_function->code().func().locals()) { - for (size_t i = 0; i < local.n(); ++i) - arguments.unchecked_append(Value(local.type())); - } - - set_frame( - is_tailcall ? IsTailcall::Yes : IsTailcall::No, - wasm_function->module(), - move(arguments), - wasm_function->code().func().body(), - wasm_function->type().results().size()); + TRY(prepare_wasm_call(*wasm_function, arguments, is_tailcall)); return OptionalNone {}; } @@ -71,6 +79,63 @@ Result Configuration::execute(Interpreter& interpreter) return Result { move(results) }; } +ErrorOr Configuration::execute_for_compiled_call(Interpreter& interpreter, Value* single_result) +{ + interpreter.interpret(*this); + if (interpreter.did_trap()) + return interpreter.trap(); + + VERIFY(frame().arity() <= 1); + if (frame().arity() == 1) { + auto result = value_stack().unsafe_take_last(); + if (single_result) + *single_result = result; + } + + if (!label_stack().is_empty()) + label_stack().take_last(); + + return {}; +} + +void Configuration::build_compiled_function_table() +{ + if (m_frame_stack.is_empty()) + return; + auto const* current_module = &frame().module(); + if (m_compiled_fn_table_module == current_module && !m_compiled_fn_table.is_empty()) + return; + + m_compiled_fn_table.clear(); + m_compiled_fn_table_module = current_module; + + auto const& functions = frame().module().functions(); + auto count = functions.size(); + if (count == 0) + return; + + for (size_t i = 0; i < count; i++) { + auto* instance = m_store.unsafe_get(functions[i]); + auto* wasm_fn = instance->get_pointer(); + if (!wasm_fn) + continue; + auto& ci = wasm_fn->code().func().body().compiled_instructions; + if (!ci.cranelift_compiled) + continue; + CompiledFunctionEntry entry; + entry.handler_ptr = ci.dispatches[0].handler_ptr; + entry.dispatches_ptr = bit_cast(ci.dispatches.data()); + entry.src_dst_ptr = bit_cast(ci.src_dst_mappings.data()); + entry.first_insn = ci.dispatches[0].instruction; + entry.expression = &wasm_fn->code().func().body(); + entry.module = &wasm_fn->module(); + entry.total_local_count = static_cast(wasm_fn->code().func().total_local_count()); + entry.arity = static_cast(wasm_fn->type().results().size()); + entry.max_call_rec_size = static_cast(ci.max_call_rec_size); + m_compiled_fn_table.set(static_cast(i), entry); + } +} + void Configuration::dump_stack() { auto print_value = [](CheckedFormatString format, Ts... vs) { diff --git a/Libraries/LibWasm/AbstractMachine/Configuration.h b/Libraries/LibWasm/AbstractMachine/Configuration.h index c96ed0700f..e77667c349 100644 --- a/Libraries/LibWasm/AbstractMachine/Configuration.h +++ b/Libraries/LibWasm/AbstractMachine/Configuration.h @@ -34,10 +34,13 @@ public: template void set_frame(IsTailcall is_tailcall, Args&&... frame_init) { - m_frame_stack.append(forward(frame_init)...); + m_frame_stack.empend(forward(frame_init)...); - auto& frame = m_frame_stack.unchecked_last(); - m_locals_base = frame.locals().data(); + auto& frame = m_frame_stack.last(); + m_locals_base = frame.locals_data(); + auto const& memories = frame.module().memories(); + m_default_memory = memories.is_empty() ? nullptr : m_store.unsafe_get(memories[0]); + m_default_memory_base = m_default_memory ? m_default_memory->data().data() : nullptr; auto continuation = frame.expression().instructions().size() - 1; if (auto size = frame.expression().compiled_instructions.dispatches.size(); size > 0) @@ -61,8 +64,33 @@ public: m_call_record_base = nullptr; } } - ALWAYS_INLINE auto& frame() const { return m_frame_stack.unchecked_last(); } - ALWAYS_INLINE auto& frame() { return m_frame_stack.unchecked_last(); } + // Lightweight set_frame for direct Cranelift-to-Cranelift calls. + void set_frame_lightweight(ModuleInstance const& module, Value* locals_ptr, + Expression const& expression, size_t arity) + { + m_frame_stack.empend(module, locals_ptr, expression, arity); + m_locals_base = locals_ptr; + auto const& memories = module.memories(); + m_default_memory = memories.is_empty() ? nullptr : m_store.unsafe_get(memories[0]); + m_default_memory_base = m_default_memory ? m_default_memory->data().data() : nullptr; + // Skip capacity hints and label push (Cranelift uses its own structured control flow). + } + + void setup_call_record_for_current_frame() + { + auto max_call_rec_size = m_frame_stack.last().expression().compiled_instructions.max_call_rec_size; + if (max_call_rec_size > 0) { + m_current_call_record.clear_with_capacity(); + m_current_call_record.ensure_capacity(max_call_rec_size); + m_current_call_record.resize_and_keep_capacity(max_call_rec_size); + m_call_record_base = m_current_call_record.data(); + } else { + m_call_record_base = nullptr; + } + } + + ALWAYS_INLINE auto& frame() const { return m_frame_stack.last(); } + ALWAYS_INLINE auto& frame() { return m_frame_stack.last(); } ALWAYS_INLINE auto& ip() const { return m_ip; } ALWAYS_INLINE auto& ip() { return m_ip; } ALWAYS_INLINE auto& depth() const { return m_depth; } @@ -73,16 +101,71 @@ public: ALWAYS_INLINE auto& label_stack() { return m_label_stack; } ALWAYS_INLINE auto& store() const { return m_store; } ALWAYS_INLINE auto& store() { return m_store; } + ALWAYS_INLINE MemoryInstance* default_memory() const { return m_default_memory; } + ALWAYS_INLINE u8* default_memory_base() const { return m_default_memory_base; } + ALWAYS_INLINE void refresh_default_memory_base() { m_default_memory_base = m_default_memory ? m_default_memory->data().data() : nullptr; } + ALWAYS_INLINE Value& compiled_call_result_scratch() { return m_compiled_call_result_scratch; } + ALWAYS_INLINE Value const& compiled_call_result_scratch() const { return m_compiled_call_result_scratch; } ALWAYS_INLINE Value const& local(LocalIndex index) const { return m_locals_base[index.value()]; } ALWAYS_INLINE Value& local(LocalIndex index) { return m_locals_base[index.value()]; } + ALWAYS_INLINE Value* locals_base() const { return m_locals_base; } + ALWAYS_INLINE void set_locals_base(Value* base) { m_locals_base = base; } + + // When > 0, unwind_impl skips the frame pop (the direct call didn't push a frame). + size_t m_compiled_direct_call_depth { 0 }; + + // Per-function entry for the compiled function table (direct calls from Cranelift). + struct CompiledFunctionEntry { + FlatPtr handler_ptr { 0 }; // 0 = not compiled, use slow path + FlatPtr dispatches_ptr { 0 }; // Dispatch const* for the handler call + FlatPtr src_dst_ptr { 0 }; // SourcesAndDestination const* + Instruction const* first_insn { nullptr }; + Expression const* expression { nullptr }; + ModuleInstance const* module { nullptr }; + u32 total_local_count { 0 }; + u32 arity { 0 }; + u32 max_call_rec_size { 0 }; + }; + HashMap m_compiled_fn_table; + ModuleInstance const* m_compiled_fn_table_module { nullptr }; + + void build_compiled_function_table(); + + static constexpr size_t locals_base_offset() { return __builtin_offsetof(Configuration, m_locals_base); } + static constexpr size_t default_memory_base_offset() { return __builtin_offsetof(Configuration, m_default_memory_base); } + static constexpr size_t compiled_call_result_scratch_offset() { return __builtin_offsetof(Configuration, m_compiled_call_result_scratch); } + + ALWAYS_INLINE Value& call_record_entry(size_t index) { return m_call_record_base[index]; } + ALWAYS_INLINE Value const& call_record_entry(size_t index) const { return m_call_record_base[index]; } + ALWAYS_INLINE Value* call_record_base() const { return m_call_record_base; } + ALWAYS_INLINE void set_call_record_base(Value* base) { m_call_record_base = base; } + ALWAYS_INLINE void setup_call_record(size_t max_call_rec_size) + { + get_arguments_allocation_if_possible(m_current_call_record, max_call_rec_size); + m_current_call_record.resize_and_keep_capacity(max_call_rec_size); + m_call_record_base = m_current_call_record.data(); + } + ALWAYS_INLINE Vector take_call_record_vector() + { + auto result = move(m_current_call_record); + m_call_record_base = nullptr; + return result; + } + ALWAYS_INLINE void restore_call_record_vector(Vector&& vec) + { + m_current_call_record = move(vec); + m_call_record_base = m_current_call_record.data(); + } struct CallFrameHandle { explicit CallFrameHandle(Configuration& configuration) : configuration(configuration) + , saved_direct_call_depth(configuration.m_compiled_direct_call_depth) { if (configuration.m_call_record_base) moved_call_record = move(configuration.m_current_call_record); + configuration.m_compiled_direct_call_depth = 0; configuration.depth()++; configuration.m_call_record_base = nullptr; } @@ -96,16 +179,38 @@ public: configuration.m_call_record_base = nullptr; } configuration.unwind({}, *this); + configuration.m_compiled_direct_call_depth = saved_direct_call_depth; } Configuration& configuration; Optional> moved_call_record; + size_t saved_direct_call_depth; }; void unwind(Badge, CallFrameHandle const&) { unwind_impl(); } ErrorOr, Trap> prepare_call(FunctionAddress, Vector& arguments, bool is_tailcall = false); + ALWAYS_INLINE ErrorOr prepare_wasm_call(WasmFunction const& wasm_function, Vector& arguments, bool is_tailcall = false) + { + if (is_tailcall) + unwind_impl(); + + arguments.ensure_capacity(arguments.size() + wasm_function.code().func().total_local_count()); + for (auto const& local : wasm_function.code().func().locals()) { + for (size_t i = 0; i < local.n(); ++i) + arguments.unchecked_append(Value(local.type())); + } + + set_frame( + is_tailcall ? IsTailcall::Yes : IsTailcall::No, + wasm_function.module(), + move(arguments), + wasm_function.code().func().body(), + wasm_function.type().results().size()); + return {}; + } Result call(Interpreter&, FunctionAddress, Vector& arguments); Result execute(Interpreter&); + ErrorOr execute_for_compiled_call(Interpreter&, Value* single_result = nullptr); void enable_instruction_count_limit() { m_should_limit_instruction_count = true; } bool should_limit_instruction_count() const { return m_should_limit_instruction_count; } @@ -143,7 +248,8 @@ public: return; } - VERIFY(m_current_call_record.size() >= size); + if (m_current_call_record.size() < size) + m_current_call_record.resize_and_keep_capacity(size); } if (arguments.capacity() != ArgumentsStaticSize) { @@ -257,13 +363,13 @@ public: Value(0), }; -private: + // Public for CraneliftBridge direct call pop_frame. void unwind_impl(); Store& m_store; Vector m_value_stack; Vector m_label_stack; - DoublyLinkedList m_frame_stack; + Vector m_frame_stack; Vector m_current_call_record; Vector, 16, FastLastAccess::Yes> m_call_argument_freelist; size_t m_depth { 0 }; @@ -271,6 +377,9 @@ private: bool m_should_limit_instruction_count { false }; Value* m_locals_base { nullptr }; Value* m_call_record_base { nullptr }; + MemoryInstance* m_default_memory { nullptr }; + u8* m_default_memory_base { nullptr }; + Value m_compiled_call_result_scratch; }; } diff --git a/Libraries/LibWasm/Forward.h b/Libraries/LibWasm/Forward.h index 9263fbad56..6dd4f7a442 100644 --- a/Libraries/LibWasm/Forward.h +++ b/Libraries/LibWasm/Forward.h @@ -12,6 +12,7 @@ class AbstractMachine; class Validator; struct ValidationError; struct Interpreter; +class MemoryBuffer; namespace Wasi {