/* * Copyright (c) 2021, Ali Mohammad Pur * * SPDX-License-Identifier: BSD-2-Clause */ #pragma once #include #include #include namespace Wasm { enum class SourceAddressMix { AllRegisters, AllCallRecords, AllStack, Any, }; enum class IsTailcall { No, Yes, }; class Configuration { AK_MAKE_NONCOPYABLE(Configuration); AK_MAKE_NONMOVABLE(Configuration); public: explicit Configuration(Store& store) : m_store(store) { m_store.register_configuration({}, *this); } ~Configuration() { m_store.unregister_configuration({}, *this); } template void set_frame(IsTailcall is_tailcall, Args&&... frame_init) { m_frame_stack.empend(forward(frame_init)...); auto& frame = m_frame_stack.last(); m_locals_base = frame.locals_data(); auto const& memories = frame.module().memories(); m_default_memory = memories.is_empty() ? nullptr : m_store.unsafe_get(memories[0]); m_default_memory_base = m_default_memory ? m_default_memory->data().data() : nullptr; frame.set_compiled_fn_table(&frame.module().compiled_fn_table(m_store)); auto continuation = frame.expression().instructions().size() - 1; if (auto size = frame.expression().compiled_instructions.dispatches.size(); size > 0) continuation = size - 1; Label label(frame.arity(), continuation, m_value_stack.size()); frame.label_index() = m_label_stack.size(); if (auto hint = frame.expression().stack_usage_hint(); hint.has_value()) m_value_stack.ensure_capacity(*hint + m_value_stack.size()); if (is_tailcall == IsTailcall::No) { if (auto hint = frame.expression().frame_usage_hint(); hint.has_value()) m_label_stack.ensure_capacity(*hint + m_label_stack.size()); } m_label_stack.append(label); auto max_call_rec_size = frame.expression().compiled_instructions.max_call_rec_size; if (max_call_rec_size > 0) { get_arguments_allocation_if_possible(m_current_call_record, max_call_rec_size); m_current_call_record.resize_and_keep_capacity(max_call_rec_size); m_call_record_base = m_current_call_record.data(); } else { m_call_record_base = nullptr; } } // Lightweight set_frame for direct Cranelift-to-Cranelift calls. void set_frame_lightweight(ModuleInstance const& module, Value* locals_ptr, Expression const& expression, size_t arity) { VERIFY(bit_cast(&module) != 0); m_frame_stack.empend(module, locals_ptr, expression, arity); m_frame_stack.last().set_compiled_fn_table(&module.compiled_fn_table(m_store)); m_locals_base = locals_ptr; auto const& memories = module.memories(); m_default_memory = memories.is_empty() ? nullptr : m_store.unsafe_get(memories[0]); m_default_memory_base = m_default_memory ? m_default_memory->data().data() : nullptr; // Skip capacity hints and label push (Cranelift uses its own structured control flow). } void setup_call_record_for_current_frame() { auto max_call_rec_size = m_frame_stack.last().expression().compiled_instructions.max_call_rec_size; if (max_call_rec_size > 0) { m_current_call_record.clear_with_capacity(); m_current_call_record.ensure_capacity(max_call_rec_size); m_current_call_record.resize_and_keep_capacity(max_call_rec_size); m_call_record_base = m_current_call_record.data(); } else { m_call_record_base = nullptr; } } ALWAYS_INLINE auto& frame() const { return m_frame_stack.last(); } ALWAYS_INLINE auto& frame() { return m_frame_stack.last(); } ALWAYS_INLINE auto& ip() const { return m_ip; } ALWAYS_INLINE auto& ip() { return m_ip; } ALWAYS_INLINE auto& depth() const { return m_depth; } ALWAYS_INLINE auto& depth() { return m_depth; } ALWAYS_INLINE auto& value_stack() const { return m_value_stack; } ALWAYS_INLINE auto& value_stack() { return m_value_stack; } ALWAYS_INLINE auto& label_stack() const { return m_label_stack; } ALWAYS_INLINE auto& label_stack() { return m_label_stack; } ALWAYS_INLINE auto& store() const { return m_store; } ALWAYS_INLINE auto& store() { return m_store; } ALWAYS_INLINE MemoryInstance* default_memory() const { return m_default_memory; } ALWAYS_INLINE u8* default_memory_base() const { return m_default_memory_base; } ALWAYS_INLINE void refresh_default_memory_base() { m_default_memory_base = m_default_memory ? m_default_memory->data().data() : nullptr; } ALWAYS_INLINE Value& compiled_call_result_scratch() { return m_compiled_call_result_scratch; } ALWAYS_INLINE Value const& compiled_call_result_scratch() const { return m_compiled_call_result_scratch; } ALWAYS_INLINE Value const& local(LocalIndex index) const { return m_locals_base[index.value()]; } ALWAYS_INLINE Value& local(LocalIndex index) { return m_locals_base[index.value()]; } ALWAYS_INLINE Value* locals_base() const { return m_locals_base; } ALWAYS_INLINE void set_locals_base(Value* base) { m_locals_base = base; } // When > 0, unwind_impl skips the frame pop (the direct call didn't push a frame). size_t m_compiled_direct_call_depth { 0 }; static constexpr size_t locals_base_offset() { return __builtin_offsetof(Configuration, m_locals_base); } static constexpr size_t default_memory_base_offset() { return __builtin_offsetof(Configuration, m_default_memory_base); } static constexpr size_t compiled_call_result_scratch_offset() { return __builtin_offsetof(Configuration, m_compiled_call_result_scratch); } ALWAYS_INLINE Value& call_record_entry(size_t index) { return m_call_record_base[index]; } ALWAYS_INLINE Value const& call_record_entry(size_t index) const { return m_call_record_base[index]; } ALWAYS_INLINE Value* call_record_base() const { return m_call_record_base; } ALWAYS_INLINE void set_call_record_base(Value* base) { m_call_record_base = base; } ALWAYS_INLINE void setup_call_record(size_t max_call_rec_size) { get_arguments_allocation_if_possible(m_current_call_record, max_call_rec_size); m_current_call_record.resize_and_keep_capacity(max_call_rec_size); m_call_record_base = m_current_call_record.data(); } ALWAYS_INLINE Vector take_call_record_vector() { auto result = move(m_current_call_record); m_call_record_base = nullptr; return result; } ALWAYS_INLINE void restore_call_record_vector(Vector&& vec) { m_current_call_record = move(vec); m_call_record_base = m_current_call_record.data(); } struct CallFrameHandle { explicit CallFrameHandle(Configuration& configuration) : configuration(configuration) , saved_direct_call_depth(configuration.m_compiled_direct_call_depth) { if (configuration.m_call_record_base) moved_call_record = move(configuration.m_current_call_record); configuration.m_compiled_direct_call_depth = 0; configuration.depth()++; configuration.m_call_record_base = nullptr; } ~CallFrameHandle() { if (moved_call_record.has_value()) { configuration.m_current_call_record = moved_call_record.release_value(); configuration.m_call_record_base = configuration.m_current_call_record.data(); } else { configuration.m_call_record_base = nullptr; } configuration.unwind({}, *this); configuration.m_compiled_direct_call_depth = saved_direct_call_depth; } Configuration& configuration; Optional> moved_call_record; size_t saved_direct_call_depth; }; void unwind(Badge, CallFrameHandle const&) { unwind_impl(); } ErrorOr, Trap> prepare_call(FunctionAddress, Vector& arguments, bool is_tailcall = false); ErrorOr prepare_wasm_call(WasmFunction const& wasm_function, Vector& arguments, bool is_tailcall = false); Result call(Interpreter&, FunctionAddress, Vector& arguments); Result execute(Interpreter&); ErrorOr execute_for_compiled_call(Interpreter&, Value* single_result = nullptr); void enable_instruction_count_limit() { m_should_limit_instruction_count = true; } bool should_limit_instruction_count() const { return m_should_limit_instruction_count; } void dump_stack(); void get_arguments_allocation_if_possible(Vector& arguments, size_t max_size) { if (arguments.capacity() != ArgumentsStaticSize || max_size <= ArgumentsStaticSize) return; // Already heap allocated, or we just don't need to allocate anything. // _arguments_ is still in static storage, pull something from the freelist if it fits. if (auto index = m_call_argument_freelist.find_first_index_if([&](auto& entry) { return entry.capacity() >= max_size; }); index.has_value()) { arguments = m_call_argument_freelist.take(*index); return; } if (!m_call_argument_freelist.is_empty()) arguments = m_call_argument_freelist.take_last(); arguments.ensure_capacity(max(max_size, frame().module().cached_minimum_call_record_allocation_size)); } void release_arguments_allocation(Vector& arguments, bool expect_frame = true) { arguments.clear_with_capacity(); // Clear to avoid copying, but keep capacity for reuse. auto size = expect_frame ? frame().expression().compiled_instructions.max_call_rec_size : 0; if (size > 0) { // If we need a call record, keep this as the current one. if (!m_call_record_base) { m_current_call_record = move(arguments); m_current_call_record.resize_and_keep_capacity(size); m_call_record_base = m_current_call_record.data(); return; } if (m_current_call_record.size() < size) m_current_call_record.resize_and_keep_capacity(size); } if (arguments.capacity() != ArgumentsStaticSize) { if (m_call_argument_freelist.size() >= 16) { // Don't grow to heap. return; } m_call_argument_freelist.unchecked_append(move(arguments)); } } void take_call_record(Vector& call_record) { call_record = move(m_current_call_record); m_call_record_base = nullptr; } template ALWAYS_INLINE FLATTEN void push_to_destination(Value value, Dispatch::RegisterOrStack destination) { if constexpr (mix == SourceAddressMix::AllRegisters) { regs.data()[to_underlying(destination)] = value; return; } else if constexpr (mix == SourceAddressMix::AllCallRecords) { m_call_record_base[to_underlying(destination) - Dispatch::RegisterOrStack::CallRecord] = value; return; } else if constexpr (mix == SourceAddressMix::AllStack) { value_stack().unchecked_append(value); return; } else if constexpr (mix == SourceAddressMix::Any) { if (!(destination & ~(Dispatch::Stack - 1))) [[likely]] { regs.data()[to_underlying(destination)] = value; return; } } if constexpr (mix == SourceAddressMix::Any) { if (destination == Dispatch::RegisterOrStack::Stack) [[unlikely]] { value_stack().unchecked_append(value); return; } m_call_record_base[to_underlying(destination) - Dispatch::RegisterOrStack::CallRecord] = value; return; } VERIFY_NOT_REACHED(); } template ALWAYS_INLINE FLATTEN Value& source_value(u8 index, Dispatch::RegisterOrStack const* sources) { // Note: The last source in a dispatch *must* be equal to the destination for this to be valid. auto const source = sources[index]; if constexpr (mix == SourceAddressMix::AllRegisters) { return regs.data()[to_underlying(source)]; } else if constexpr (mix == SourceAddressMix::AllCallRecords) { return m_call_record_base[to_underlying(source) - Dispatch::RegisterOrStack::CallRecord]; } else if constexpr (mix == SourceAddressMix::AllStack) { return value_stack().unsafe_last(); } else if constexpr (mix == SourceAddressMix::Any) { if (!(source & ~(Dispatch::Stack - 1))) [[likely]] return regs.data()[to_underlying(source)]; } if constexpr (mix == SourceAddressMix::Any) { if (source == Dispatch::RegisterOrStack::Stack) [[unlikely]] return value_stack().unsafe_last(); return m_call_record_base[to_underlying(source) - Dispatch::RegisterOrStack::CallRecord]; } VERIFY_NOT_REACHED(); } template ALWAYS_INLINE FLATTEN Value take_source(u8 index, Dispatch::RegisterOrStack const* sources) { auto const source = sources[index]; if constexpr (mix == SourceAddressMix::AllRegisters) { return regs.data()[to_underlying(source)]; } else if constexpr (mix == SourceAddressMix::AllCallRecords) { return m_call_record_base[to_underlying(source) - Dispatch::RegisterOrStack::CallRecord]; } else if constexpr (mix == SourceAddressMix::AllStack) { return value_stack().unsafe_take_last(); } else if constexpr (mix == SourceAddressMix::Any) { if (!(source & ~(Dispatch::Stack - 1))) [[likely]] return regs.data()[to_underlying(source)]; } if constexpr (mix == SourceAddressMix::Any) { if (source == Dispatch::RegisterOrStack::Stack) [[unlikely]] return value_stack().unsafe_take_last(); return m_call_record_base[to_underlying(source) - Dispatch::RegisterOrStack::CallRecord]; } VERIFY_NOT_REACHED(); } Array regs = { Value(0), Value(0), Value(0), Value(0), Value(0), Value(0), Value(0), Value(0), }; // Public for CraneliftBridge direct call pop_frame. void unwind_impl(); Store& m_store; Vector m_value_stack; Vector m_label_stack; Vector m_frame_stack; Vector m_current_call_record; Vector, 16, FastLastAccess::Yes> m_call_argument_freelist; size_t m_depth { 0 }; u64 m_ip { 0 }; bool m_should_limit_instruction_count { false }; Value* m_locals_base { nullptr }; Value* m_call_record_base { nullptr }; MemoryInstance* m_default_memory { nullptr }; u8* m_default_memory_base { nullptr }; Value m_compiled_call_result_scratch; }; }