1668 lines
70 KiB
C++
1668 lines
70 KiB
C++
/*
|
|
* Copyright (c) 2026-present, Ali Mohammad Pur <ali@ladybird.org>
|
|
*
|
|
* SPDX-License-Identifier: BSD-2-Clause
|
|
*/
|
|
|
|
#include <AK/ByteString.h>
|
|
#include <AK/Checked.h>
|
|
#include <AK/LexicalPath.h>
|
|
#include <AK/NeverDestroyed.h>
|
|
#include <AK/Platform.h>
|
|
#include <AK/ScopeGuard.h>
|
|
#include <CraneliftFFI.h>
|
|
#include <LibCore/Process.h>
|
|
#include <LibCore/System.h>
|
|
#include <LibWasm/AbstractMachine/BytecodeInterpreter.h>
|
|
#include <LibWasm/AbstractMachine/Configuration.h>
|
|
#include <LibWasm/Printer/Printer.h>
|
|
#include <LibWasm/Types.h>
|
|
#include <errno.h>
|
|
#include <stdlib.h>
|
|
|
|
#if defined(AK_OS_WINDOWS)
|
|
# include <AK/Windows.h>
|
|
#else
|
|
# include <fcntl.h>
|
|
# include <sys/mman.h>
|
|
# include <unistd.h>
|
|
#endif
|
|
|
|
#if defined(AK_OS_MACOS)
|
|
# include <libkern/OSCacheControl.h>
|
|
# include <pthread.h>
|
|
#endif
|
|
|
|
using namespace Wasm;
|
|
using namespace Cranelift;
|
|
|
|
namespace {
|
|
|
|
struct InputHeader {
|
|
u32 function_count;
|
|
u32 helpers_offset;
|
|
u64 outcome_return;
|
|
u64 code_region_start;
|
|
u64 reloc_region_start;
|
|
u64 total_size;
|
|
};
|
|
|
|
struct InputFunctionEntry {
|
|
u32 insn_offset;
|
|
u32 insn_count;
|
|
u32 result_arity;
|
|
};
|
|
|
|
struct OutputFunctionEntry {
|
|
u64 code_offset;
|
|
u32 code_size;
|
|
u32 compiled;
|
|
// Offset (relative to the start of the reloc region) and count of `HelperReloc`
|
|
// entries describing the absolute helper addresses baked into this function's code.
|
|
// On cache install we walk these and rewrite the 8 bytes at code+code_offset+offset
|
|
// with the live address of helper N for the current process.
|
|
u64 reloc_offset;
|
|
u32 reloc_count;
|
|
u64 trap_offset;
|
|
u32 trap_count;
|
|
u32 _pad;
|
|
};
|
|
|
|
struct CodeMapping {
|
|
void* mapping;
|
|
size_t size;
|
|
Vector<CraneliftTrap> traps;
|
|
};
|
|
|
|
static constexpr size_t oop_code_region_min_size = 256 * KiB;
|
|
static constexpr size_t oop_code_bytes_per_insn = 256;
|
|
static constexpr size_t oop_reloc_region_min_size = 64 * KiB;
|
|
static constexpr size_t oop_reloc_bytes_per_insn = 128;
|
|
|
|
static size_t align_up(size_t value, size_t alignment)
|
|
{
|
|
VERIFY(alignment > 0);
|
|
auto remainder = value % alignment;
|
|
return remainder == 0 ? value : value + (alignment - remainder);
|
|
}
|
|
|
|
struct BatchInput {
|
|
Vector<CraneliftInsn> insns;
|
|
u32 result_arity;
|
|
u32 function_index;
|
|
CompiledInstructions* target;
|
|
};
|
|
|
|
// Disk-cache blob format. Stable: cached files name format_version + layout_hash so
|
|
// any rebuild that changes those will simply miss the cache rather than try to
|
|
// execute incompatible bytes.
|
|
constexpr u64 cache_blob_magic = 0x4354494A4D534157ULL; // "WASMJITC" little-endian
|
|
constexpr u32 cache_blob_format_version = 3;
|
|
|
|
struct CacheBlobHeader {
|
|
u64 magic;
|
|
u32 format_version;
|
|
u32 helper_count;
|
|
u64 layout_hash;
|
|
u8 wasm_hash[32];
|
|
u32 function_count;
|
|
u32 _pad;
|
|
};
|
|
static_assert(sizeof(CacheBlobHeader) == 64);
|
|
|
|
struct CacheBlobFunctionEntry {
|
|
u32 function_index;
|
|
u32 code_size;
|
|
u32 reloc_count;
|
|
u32 trap_count;
|
|
};
|
|
static_assert(sizeof(CacheBlobFunctionEntry) == 16);
|
|
|
|
struct CacheRecord {
|
|
u32 function_index;
|
|
ByteBuffer unpatched_code;
|
|
Vector<HelperReloc> relocs;
|
|
Vector<CraneliftTrap> traps;
|
|
};
|
|
|
|
// On a cache miss we capture every successful compile so we can hand the blob to a
|
|
// store callback after validation finishes. On a cache hit we populate the install
|
|
// map up front; the per-function lookup happens inside try_cranelift_compile when
|
|
// the dispatch table for that function has just been built and is ready to receive
|
|
// a handler_ptr.
|
|
struct CacheCaptureState {
|
|
bool capturing { false };
|
|
Vector<CacheRecord> records;
|
|
};
|
|
struct PendingInstallState {
|
|
bool active { false };
|
|
HashMap<u32, CacheRecord> records;
|
|
};
|
|
struct CacheState {
|
|
CacheCaptureState cache_capture;
|
|
PendingInstallState pending_install;
|
|
Vector<BatchInput> pending_batch;
|
|
};
|
|
|
|
static thread_local u32 s_active_function_index = NumericLimits<u32>::max();
|
|
|
|
static CacheState& cranelift_cache_state()
|
|
{
|
|
static thread_local auto* state = new CacheState;
|
|
return *state;
|
|
}
|
|
|
|
static u64 compute_layout_hash(RuntimeHelpers const& h)
|
|
{
|
|
auto fnv1a = [](u64 hash, u64 value) {
|
|
for (int i = 0; i < 8; ++i) {
|
|
hash ^= (value >> (i * 8)) & 0xff;
|
|
hash *= 0x100000001b3ULL;
|
|
}
|
|
return hash;
|
|
};
|
|
u64 hash = 0xcbf29ce484222325ULL;
|
|
hash = fnv1a(hash, h.regs_offset);
|
|
hash = fnv1a(hash, h.value_size);
|
|
hash = fnv1a(hash, h.locals_base_offset);
|
|
hash = fnv1a(hash, h.default_memory_base_offset);
|
|
hash = fnv1a(hash, h.compiled_call_result_scratch_offset);
|
|
return hash;
|
|
}
|
|
|
|
// `HelperId` values are assigned in lockstep with the field order of `RuntimeHelpers`,
|
|
// so the helper address for id N is simply the N-th `size_t` field of the struct.
|
|
static_assert(offsetof(RuntimeHelpers, call_function) == 0);
|
|
static_assert(offsetof(RuntimeHelpers, memory_fill) == sizeof(size_t) * 30);
|
|
static_assert(HELPER_COUNT == 31);
|
|
|
|
static bool apply_helper_relocs(u8* code_bytes, size_t code_size, HelperReloc const* relocs, size_t reloc_count, RuntimeHelpers const& helpers)
|
|
{
|
|
auto const* helper_table = reinterpret_cast<size_t const*>(&helpers);
|
|
for (size_t i = 0; i < reloc_count; ++i) {
|
|
auto const& r = relocs[i];
|
|
if (r.helper_id >= HELPER_COUNT)
|
|
return false;
|
|
if (static_cast<size_t>(r.code_offset) + sizeof(u64) > code_size)
|
|
return false;
|
|
u64 addr = static_cast<u64>(helper_table[r.helper_id]) + static_cast<u64>(r.addend);
|
|
__builtin_memcpy(code_bytes + r.code_offset, &addr, sizeof(addr));
|
|
}
|
|
return true;
|
|
}
|
|
|
|
// Allocate an RX-able page, copy the (still unpatched) machine code into it, apply the
|
|
// helper-address patches, and install the resulting function pointer into `target`.
|
|
// Used by both the fresh-compile path (bytes come from the subprocess shm) and the
|
|
// cache-install path (bytes come from a `.wasmjit` blob).
|
|
static bool install_compiled_function(CompiledInstructions& target, ReadonlyBytes code_bytes, HelperReloc const* relocs, size_t reloc_count, ReadonlySpan<CraneliftTrap> traps, RuntimeHelpers const& helpers)
|
|
{
|
|
if (target.dispatches.is_empty())
|
|
return false;
|
|
|
|
auto const code_size = code_bytes.size();
|
|
if (code_size == 0)
|
|
return false;
|
|
|
|
#if defined(AK_OS_WINDOWS)
|
|
SYSTEM_INFO si;
|
|
GetSystemInfo(&si);
|
|
auto const page_size = static_cast<size_t>(si.dwPageSize);
|
|
auto const rx_aligned_size = (code_size + page_size - 1) & ~(page_size - 1);
|
|
auto* jit_mem = VirtualAlloc(nullptr, rx_aligned_size, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE);
|
|
if (!jit_mem)
|
|
return false;
|
|
__builtin_memcpy(jit_mem, code_bytes.data(), code_size);
|
|
if (!apply_helper_relocs(static_cast<u8*>(jit_mem), code_size, relocs, reloc_count, helpers)) {
|
|
VirtualFree(jit_mem, 0, MEM_RELEASE);
|
|
return false;
|
|
}
|
|
DWORD old_protect;
|
|
VirtualProtect(jit_mem, rx_aligned_size, PAGE_EXECUTE_READ, &old_protect);
|
|
FlushInstructionCache(GetCurrentProcess(), jit_mem, code_size);
|
|
auto* func_ptr = static_cast<u8 const*>(jit_mem);
|
|
auto* handle = new CodeMapping { jit_mem, rx_aligned_size, {} };
|
|
#elif defined(AK_OS_MACOS)
|
|
auto const page_size = static_cast<size_t>(sysconf(_SC_PAGESIZE));
|
|
auto const rx_aligned_size = (code_size + page_size - 1) & ~(page_size - 1);
|
|
auto* jit_mapping = mmap(nullptr, rx_aligned_size, PROT_READ | PROT_WRITE | PROT_EXEC, MAP_PRIVATE | MAP_ANON | MAP_JIT, -1, 0);
|
|
if (jit_mapping == MAP_FAILED)
|
|
return false;
|
|
|
|
pthread_jit_write_protect_np(0);
|
|
__builtin_memcpy(jit_mapping, code_bytes.data(), code_size);
|
|
if (!apply_helper_relocs(static_cast<u8*>(jit_mapping), code_size, relocs, reloc_count, helpers)) {
|
|
munmap(jit_mapping, rx_aligned_size);
|
|
return false;
|
|
}
|
|
pthread_jit_write_protect_np(1);
|
|
sys_icache_invalidate(jit_mapping, code_size);
|
|
auto* func_ptr = static_cast<u8 const*>(jit_mapping);
|
|
auto* handle = new CodeMapping { jit_mapping, rx_aligned_size, {} };
|
|
#else
|
|
auto const page_size = static_cast<size_t>(sysconf(_SC_PAGESIZE));
|
|
auto const rx_aligned_size = (code_size + page_size - 1) & ~(page_size - 1);
|
|
auto* rw_mapping = mmap(nullptr, rx_aligned_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0);
|
|
if (rw_mapping == MAP_FAILED)
|
|
return false;
|
|
__builtin_memcpy(rw_mapping, code_bytes.data(), code_size);
|
|
if (!apply_helper_relocs(static_cast<u8*>(rw_mapping), code_size, relocs, reloc_count, helpers) || mprotect(rw_mapping, rx_aligned_size, PROT_READ | PROT_EXEC) != 0) {
|
|
munmap(rw_mapping, rx_aligned_size);
|
|
return false;
|
|
}
|
|
__builtin___clear_cache(static_cast<char*>(rw_mapping), static_cast<char*>(rw_mapping) + code_size);
|
|
auto* func_ptr = static_cast<u8 const*>(rw_mapping);
|
|
auto* handle = new CodeMapping { rw_mapping, rx_aligned_size, {} };
|
|
#endif
|
|
|
|
handle->traps.ensure_capacity(traps.size());
|
|
for (auto const& trap : traps)
|
|
handle->traps.unchecked_append(trap);
|
|
|
|
target.cranelift_code_handle = handle;
|
|
target.cranelift_code_size = code_size;
|
|
target.cranelift_traps = handle->traps.data();
|
|
target.cranelift_trap_count = handle->traps.size();
|
|
target.cranelift_compiled = true;
|
|
publish_cranelift_entry(target, bit_cast<FlatPtr>(func_ptr));
|
|
return true;
|
|
}
|
|
|
|
}
|
|
|
|
extern "C" {
|
|
|
|
static ALWAYS_INLINE i32 wasm_cl_finish_call(BytecodeInterpreter& interpreter, Configuration& config, FunctionAddress address, Vector<Value, ArgumentsStaticSize>& args)
|
|
{
|
|
if (interpreter.trap_if_insufficient_native_stack_space())
|
|
return 1;
|
|
|
|
auto* instance = config.store().unsafe_get(address);
|
|
|
|
if (auto* wasm_function = instance->get_pointer<WasmFunction>(); wasm_function
|
|
&& !config.should_limit_instruction_count()
|
|
&& cranelift_entry_acquire(wasm_function->code().func().body().compiled_instructions) != 0) {
|
|
|
|
// Fast compiled-to-compiled call: stack-allocate locals + non-owning frame.
|
|
auto& func = wasm_function->code().func();
|
|
auto arg_count = args.size();
|
|
auto local_count = func.total_local_count();
|
|
auto total = arg_count + local_count;
|
|
auto callee_arity = wasm_function->type().results().size();
|
|
|
|
Value callee_buf[64];
|
|
Value* callee_locals = callee_buf;
|
|
Vector<Value, ArgumentsStaticSize> heap_fallback;
|
|
if (total > 64) [[unlikely]] {
|
|
heap_fallback.ensure_capacity(total);
|
|
heap_fallback.resize_and_keep_capacity(total);
|
|
callee_locals = heap_fallback.data();
|
|
}
|
|
for (size_t i = 0; i < arg_count; i++)
|
|
callee_locals[i] = args[i];
|
|
__builtin_memset(callee_locals + arg_count, 0, local_count * sizeof(Value));
|
|
|
|
auto& ci = func.body().compiled_instructions;
|
|
{
|
|
BytecodeInterpreter::CallFrameHandle handle { interpreter, config };
|
|
config.set_frame_lightweight(wasm_function->module(), callee_locals, func.body(), callee_arity);
|
|
config.setup_call_record_for_current_frame();
|
|
config.ip() = 0;
|
|
|
|
interpreter.clear_trap();
|
|
auto const* cc = ci.dispatches.data();
|
|
auto const* addrs = ci.src_dst_mappings.data();
|
|
using HandlerFn = Outcome (*)(BytecodeInterpreter&, Configuration&, Instruction const*, u32, Dispatch const*, SourcesAndDestination const*);
|
|
auto const handler = bit_cast<HandlerFn>(cranelift_entry_acquire(ci));
|
|
auto outcome = handler(interpreter, config, cc[0].instruction, 0, cc, addrs);
|
|
|
|
if (outcome != Outcome::Return) {
|
|
interpreter.set_trap("Compiled function returned unexpectedly"sv);
|
|
return 1;
|
|
}
|
|
if (interpreter.did_trap())
|
|
return 1;
|
|
|
|
if (callee_arity == 1)
|
|
config.compiled_call_result_scratch() = config.value_stack().unsafe_take_last();
|
|
// No label pop: set_frame_lightweight doesn't push labels.
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
// direct-threaded interpreter path:
|
|
// CallFrameHandle saves/zeros/restores the direct call counter automatically.
|
|
if (auto* wasm_function = instance->get_pointer<WasmFunction>(); wasm_function && !config.should_limit_instruction_count() && wasm_function->code().func().body().compiled_instructions.direct) {
|
|
BytecodeInterpreter::CallFrameHandle handle { interpreter, config };
|
|
if (auto prepare_result = config.prepare_wasm_call(*wasm_function, args); prepare_result.is_error()) {
|
|
interpreter.set_trap(prepare_result.release_error());
|
|
return 1;
|
|
}
|
|
config.ip() = 0;
|
|
auto outcome = interpreter.run_compiled_function_direct(config);
|
|
if (outcome != Outcome::Return) {
|
|
interpreter.set_trap("Compiled function returned unexpectedly"sv);
|
|
return 1;
|
|
}
|
|
if (interpreter.did_trap())
|
|
return 1;
|
|
if (config.frame().arity() == 1)
|
|
config.compiled_call_result_scratch() = config.value_stack().unsafe_take_last();
|
|
if (!config.label_stack().is_empty())
|
|
config.label_stack().take_last();
|
|
return 0;
|
|
}
|
|
|
|
// non-compiled call (interpreter or host function)
|
|
Wasm::Result result { Vector<Value> {} };
|
|
if (instance->has<WasmFunction>()) {
|
|
BytecodeInterpreter::CallFrameHandle handle { interpreter, config };
|
|
result = config.call(interpreter, address, args);
|
|
} else {
|
|
result = config.call(interpreter, address, args);
|
|
config.release_arguments_allocation(args);
|
|
}
|
|
|
|
if (result.is_trap()) {
|
|
interpreter.set_trap(move(result.trap()));
|
|
return 1;
|
|
}
|
|
|
|
if (!result.values().is_empty())
|
|
config.compiled_call_result_scratch() = result.values().first();
|
|
return 0;
|
|
}
|
|
|
|
i32 wasm_cl_call_function(void* interp_ptr, void* config_ptr, i32 func_index);
|
|
i32 wasm_cl_call_function(void* interp_ptr, void* config_ptr, i32 func_index)
|
|
{
|
|
auto& interpreter = *static_cast<BytecodeInterpreter*>(interp_ptr);
|
|
auto& config = *static_cast<Configuration*>(config_ptr);
|
|
|
|
auto const& module = config.frame().module();
|
|
auto const& functions = module.functions();
|
|
if (static_cast<size_t>(func_index) >= functions.size())
|
|
return 1;
|
|
|
|
auto address = functions[func_index];
|
|
|
|
SourcesAndDestination addrs {};
|
|
addrs.sources[0] = Dispatch::RegisterOrStack::Stack;
|
|
addrs.sources[1] = Dispatch::RegisterOrStack::Stack;
|
|
addrs.sources[2] = Dispatch::RegisterOrStack::Stack;
|
|
addrs.destination = Dispatch::RegisterOrStack::Stack;
|
|
|
|
auto outcome = interpreter.call_address(config, address, addrs,
|
|
BytecodeInterpreter::CallAddressSource::DirectCall,
|
|
BytecodeInterpreter::CallType::UsingStack);
|
|
|
|
return outcome == Outcome::Return && interpreter.did_trap() ? 1 : 0;
|
|
}
|
|
|
|
void wasm_cl_set_trap(void* interp_ptr, u8 const* msg, i32 len);
|
|
void wasm_cl_set_trap(void* interp_ptr, u8 const* msg, i32 len)
|
|
{
|
|
auto& interpreter = *static_cast<BytecodeInterpreter*>(interp_ptr);
|
|
interpreter.set_trap(StringView(reinterpret_cast<char const*>(msg), len));
|
|
}
|
|
|
|
static inline MemoryInstance* wasm_cl_get_memory(void* config_ptr, i32 mem_idx)
|
|
{
|
|
auto& config = *static_cast<Configuration*>(config_ptr);
|
|
if (mem_idx == 0) [[likely]] {
|
|
if (auto* memory = config.default_memory())
|
|
return memory;
|
|
}
|
|
auto const& module = config.frame().module();
|
|
auto const& mem_address = module.memories().data()[mem_idx];
|
|
return config.store().unsafe_get(mem_address);
|
|
}
|
|
|
|
static inline u8 const* wasm_cl_memory_data_if_in_bounds(MemoryInstance* memory, u64 instance_addr, size_t size)
|
|
{
|
|
if (instance_addr > memory->size() || size > memory->size() - instance_addr)
|
|
return nullptr;
|
|
return memory->data().offset_pointer(instance_addr);
|
|
}
|
|
|
|
i32 wasm_cl_memory_load8_s(void* interp_ptr, void* config_ptr, i32 mem_idx, i64 addr, i64* out);
|
|
i32 wasm_cl_memory_load8_s(void* interp_ptr, void* config_ptr, i32 mem_idx, i64 addr, i64* out)
|
|
{
|
|
auto* memory = wasm_cl_get_memory(config_ptr, mem_idx);
|
|
auto const* data = wasm_cl_memory_data_if_in_bounds(memory, static_cast<u64>(addr), 1);
|
|
if (!data) {
|
|
static_cast<BytecodeInterpreter*>(interp_ptr)->set_trap("Memory access out of bounds"sv);
|
|
return 1;
|
|
}
|
|
*out = static_cast<i64>(static_cast<i8>(data[0]));
|
|
return 0;
|
|
}
|
|
|
|
i32 wasm_cl_memory_load8_u(void* interp_ptr, void* config_ptr, i32 mem_idx, i64 addr, i64* out);
|
|
i32 wasm_cl_memory_load8_u(void* interp_ptr, void* config_ptr, i32 mem_idx, i64 addr, i64* out)
|
|
{
|
|
auto* memory = wasm_cl_get_memory(config_ptr, mem_idx);
|
|
auto const* data = wasm_cl_memory_data_if_in_bounds(memory, static_cast<u64>(addr), 1);
|
|
if (!data) {
|
|
static_cast<BytecodeInterpreter*>(interp_ptr)->set_trap("Memory access out of bounds"sv);
|
|
return 1;
|
|
}
|
|
*out = static_cast<i64>(data[0]);
|
|
return 0;
|
|
}
|
|
|
|
i32 wasm_cl_memory_load16_s(void* interp_ptr, void* config_ptr, i32 mem_idx, i64 addr, i64* out);
|
|
i32 wasm_cl_memory_load16_s(void* interp_ptr, void* config_ptr, i32 mem_idx, i64 addr, i64* out)
|
|
{
|
|
auto* memory = wasm_cl_get_memory(config_ptr, mem_idx);
|
|
auto const* data = wasm_cl_memory_data_if_in_bounds(memory, static_cast<u64>(addr), 2);
|
|
if (!data) {
|
|
static_cast<BytecodeInterpreter*>(interp_ptr)->set_trap("Memory access out of bounds"sv);
|
|
return 1;
|
|
}
|
|
u16 val;
|
|
__builtin_memcpy(&val, data, sizeof(val));
|
|
*out = static_cast<i64>(static_cast<i16>(val));
|
|
return 0;
|
|
}
|
|
|
|
i32 wasm_cl_memory_load16_u(void* interp_ptr, void* config_ptr, i32 mem_idx, i64 addr, i64* out);
|
|
i32 wasm_cl_memory_load16_u(void* interp_ptr, void* config_ptr, i32 mem_idx, i64 addr, i64* out)
|
|
{
|
|
auto* memory = wasm_cl_get_memory(config_ptr, mem_idx);
|
|
auto const* data = wasm_cl_memory_data_if_in_bounds(memory, static_cast<u64>(addr), 2);
|
|
if (!data) {
|
|
static_cast<BytecodeInterpreter*>(interp_ptr)->set_trap("Memory access out of bounds"sv);
|
|
return 1;
|
|
}
|
|
u16 val;
|
|
__builtin_memcpy(&val, data, sizeof(val));
|
|
*out = static_cast<i64>(val);
|
|
return 0;
|
|
}
|
|
|
|
i32 wasm_cl_memory_load32_s(void* interp_ptr, void* config_ptr, i32 mem_idx, i64 addr, i64* out);
|
|
i32 wasm_cl_memory_load32_s(void* interp_ptr, void* config_ptr, i32 mem_idx, i64 addr, i64* out)
|
|
{
|
|
auto* memory = wasm_cl_get_memory(config_ptr, mem_idx);
|
|
auto const* data = wasm_cl_memory_data_if_in_bounds(memory, static_cast<u64>(addr), 4);
|
|
if (!data) {
|
|
static_cast<BytecodeInterpreter*>(interp_ptr)->set_trap("Memory access out of bounds"sv);
|
|
return 1;
|
|
}
|
|
u32 val;
|
|
__builtin_memcpy(&val, data, sizeof(val));
|
|
*out = static_cast<i64>(static_cast<i32>(val));
|
|
return 0;
|
|
}
|
|
|
|
i32 wasm_cl_memory_load32_u(void* interp_ptr, void* config_ptr, i32 mem_idx, i64 addr, i64* out);
|
|
i32 wasm_cl_memory_load32_u(void* interp_ptr, void* config_ptr, i32 mem_idx, i64 addr, i64* out)
|
|
{
|
|
auto* memory = wasm_cl_get_memory(config_ptr, mem_idx);
|
|
auto const* data = wasm_cl_memory_data_if_in_bounds(memory, static_cast<u64>(addr), 4);
|
|
if (!data) {
|
|
static_cast<BytecodeInterpreter*>(interp_ptr)->set_trap("Memory access out of bounds"sv);
|
|
return 1;
|
|
}
|
|
u32 val;
|
|
__builtin_memcpy(&val, data, sizeof(val));
|
|
*out = static_cast<i64>(val);
|
|
return 0;
|
|
}
|
|
|
|
i32 wasm_cl_memory_load64(void* interp_ptr, void* config_ptr, i32 mem_idx, i64 addr, i64* out);
|
|
i32 wasm_cl_memory_load64(void* interp_ptr, void* config_ptr, i32 mem_idx, i64 addr, i64* out)
|
|
{
|
|
auto* memory = wasm_cl_get_memory(config_ptr, mem_idx);
|
|
auto const* data = wasm_cl_memory_data_if_in_bounds(memory, static_cast<u64>(addr), 8);
|
|
if (!data) {
|
|
static_cast<BytecodeInterpreter*>(interp_ptr)->set_trap("Memory access out of bounds"sv);
|
|
return 1;
|
|
}
|
|
u64 val;
|
|
__builtin_memcpy(&val, data, sizeof(val));
|
|
*out = static_cast<i64>(val);
|
|
return 0;
|
|
}
|
|
|
|
static inline bool wasm_cl_memory_store_in_bounds(void* interp_ptr, void* config_ptr, i32 mem_idx, i64 addr, size_t size, u8*& data)
|
|
{
|
|
auto* memory = wasm_cl_get_memory(config_ptr, mem_idx);
|
|
auto instance_addr = static_cast<u64>(addr);
|
|
if (instance_addr > memory->size() || size > memory->size() - instance_addr) {
|
|
static_cast<BytecodeInterpreter*>(interp_ptr)->set_trap("Memory access out of bounds"sv);
|
|
return false;
|
|
}
|
|
data = memory->data().offset_pointer(instance_addr);
|
|
return true;
|
|
}
|
|
|
|
i32 wasm_cl_memory_store8(void* interp_ptr, void* config_ptr, i32 mem_idx, i64 addr, i64 value);
|
|
i32 wasm_cl_memory_store8(void* interp_ptr, void* config_ptr, i32 mem_idx, i64 addr, i64 value)
|
|
{
|
|
u8* data;
|
|
if (!wasm_cl_memory_store_in_bounds(interp_ptr, config_ptr, mem_idx, addr, 1, data))
|
|
return 1; // OOB trap
|
|
data[0] = static_cast<u8>(value);
|
|
return 0;
|
|
}
|
|
|
|
i32 wasm_cl_memory_store16(void* interp_ptr, void* config_ptr, i32 mem_idx, i64 addr, i64 value);
|
|
i32 wasm_cl_memory_store16(void* interp_ptr, void* config_ptr, i32 mem_idx, i64 addr, i64 value)
|
|
{
|
|
u8* data;
|
|
if (!wasm_cl_memory_store_in_bounds(interp_ptr, config_ptr, mem_idx, addr, 2, data))
|
|
return 1;
|
|
u16 val = static_cast<u16>(value);
|
|
__builtin_memcpy(data, &val, sizeof(val));
|
|
return 0;
|
|
}
|
|
|
|
i32 wasm_cl_memory_store32(void* interp_ptr, void* config_ptr, i32 mem_idx, i64 addr, i64 value);
|
|
i32 wasm_cl_memory_store32(void* interp_ptr, void* config_ptr, i32 mem_idx, i64 addr, i64 value)
|
|
{
|
|
u8* data;
|
|
if (!wasm_cl_memory_store_in_bounds(interp_ptr, config_ptr, mem_idx, addr, 4, data))
|
|
return 1;
|
|
u32 val = static_cast<u32>(value);
|
|
__builtin_memcpy(data, &val, sizeof(val));
|
|
return 0;
|
|
}
|
|
|
|
i32 wasm_cl_memory_store64(void* interp_ptr, void* config_ptr, i32 mem_idx, i64 addr, i64 value);
|
|
i32 wasm_cl_memory_store64(void* interp_ptr, void* config_ptr, i32 mem_idx, i64 addr, i64 value)
|
|
{
|
|
u8* data;
|
|
if (!wasm_cl_memory_store_in_bounds(interp_ptr, config_ptr, mem_idx, addr, 8, data))
|
|
return 1;
|
|
u64 val = static_cast<u64>(value);
|
|
__builtin_memcpy(data, &val, sizeof(val));
|
|
return 0;
|
|
}
|
|
|
|
i64 wasm_cl_memory_size(void* config_ptr, i32 mem_idx);
|
|
i64 wasm_cl_memory_size(void* config_ptr, i32 mem_idx)
|
|
{
|
|
auto& config = *static_cast<Configuration*>(config_ptr);
|
|
auto const& module = config.frame().module();
|
|
auto const& mem_address = module.memories().data()[mem_idx];
|
|
auto* memory = config.store().unsafe_get(mem_address);
|
|
return static_cast<i64>(memory->size() / Constants::page_size);
|
|
}
|
|
|
|
i32 wasm_cl_memory_grow(void* config_ptr, i32 mem_idx, i32 pages);
|
|
i32 wasm_cl_memory_grow(void* config_ptr, i32 mem_idx, i32 pages)
|
|
{
|
|
auto& config = *static_cast<Configuration*>(config_ptr);
|
|
auto const& module = config.frame().module();
|
|
auto const& mem_address = module.memories().data()[mem_idx];
|
|
auto* memory = config.store().unsafe_get(mem_address);
|
|
auto old_pages = memory->size() / Constants::page_size;
|
|
if (!memory->grow(pages * Constants::page_size))
|
|
return -1;
|
|
if (mem_idx == 0)
|
|
config.refresh_default_memory_base();
|
|
return static_cast<i32>(old_pages);
|
|
}
|
|
|
|
i64 wasm_cl_read_global(void* config_ptr, i32 index);
|
|
i64 wasm_cl_read_global(void* config_ptr, i32 index)
|
|
{
|
|
auto& config = *static_cast<Configuration*>(config_ptr);
|
|
auto const& module = config.frame().module();
|
|
auto global_address = module.globals().data()[index];
|
|
auto* global = config.store().get(global_address);
|
|
return global->value().to<i64>();
|
|
}
|
|
|
|
void wasm_cl_write_global(void* config_ptr, i32 index, i64 value);
|
|
void wasm_cl_write_global(void* config_ptr, i32 index, i64 value)
|
|
{
|
|
auto& config = *static_cast<Configuration*>(config_ptr);
|
|
auto const& module = config.frame().module();
|
|
auto global_address = module.globals().data()[index];
|
|
auto* global = config.store().get(global_address);
|
|
global->set_value(Value(value));
|
|
}
|
|
|
|
i32 wasm_cl_call_indirect(void* interp_ptr, void* config_ptr, i32 table_idx, i32 type_idx, i32 element_index);
|
|
i32 wasm_cl_call_indirect(void* interp_ptr, void* config_ptr, i32 table_idx, i32 type_idx, i32 element_index)
|
|
{
|
|
auto& interpreter = *static_cast<BytecodeInterpreter*>(interp_ptr);
|
|
auto& config = *static_cast<Configuration*>(config_ptr);
|
|
|
|
auto const& module = config.frame().module();
|
|
auto table_address = module.tables()[table_idx];
|
|
auto* table_instance = config.store().get(table_address);
|
|
if (!table_instance || element_index < 0 || static_cast<size_t>(element_index) >= table_instance->elements().size())
|
|
return interpreter.set_trap(Trap::from_string("Table index out of bounds"));
|
|
|
|
auto& element = table_instance->elements()[element_index];
|
|
if (!element.ref().has<Reference::Func>())
|
|
return interpreter.set_trap(Trap::from_string("Table element is not a function reference"));
|
|
|
|
auto address = element.ref().get<Reference::Func>().address;
|
|
auto* function = config.store().get(address);
|
|
if (!function)
|
|
return interpreter.set_trap(Trap::from_string("Indirect call to freed function"));
|
|
// https://webassembly.github.io/spec/core/exec/instructions.html#xref-syntax-instructions-syntax-instr-control-mathsf-call-indirect-x-y
|
|
// call_indirect's runtime check is a defined-type match (a downcast), not structural equality.
|
|
auto const* type_actual = function->visit([](auto& f) { return f.defined_type(); });
|
|
auto const* type_expected = module.canonical_types()[type_idx];
|
|
if (!type_actual || !matches_defined_type(*type_actual, *type_expected))
|
|
return interpreter.set_trap(Trap::from_string("Indirect call type mismatch"));
|
|
|
|
SourcesAndDestination addrs {};
|
|
addrs.sources[0] = Dispatch::RegisterOrStack::Stack;
|
|
addrs.sources[1] = Dispatch::RegisterOrStack::Stack;
|
|
addrs.sources[2] = Dispatch::RegisterOrStack::Stack;
|
|
addrs.destination = Dispatch::RegisterOrStack::Stack;
|
|
|
|
auto outcome = interpreter.call_address(config, address, addrs, BytecodeInterpreter::CallAddressSource::IndirectCall, BytecodeInterpreter::CallType::UsingStack);
|
|
|
|
return outcome == Outcome::Return && interpreter.did_trap() ? 1 : 0;
|
|
}
|
|
|
|
i32 wasm_cl_memory_copy(void* interp_ptr, void* config_ptr, i32 dst_mem, i32 src_mem, i32 dst_offset, i32 src_offset, i32 count);
|
|
i32 wasm_cl_memory_copy(void* interp_ptr, void* config_ptr, i32 dst_mem, i32 src_mem, i32 dst_offset, i32 src_offset, i32 count)
|
|
{
|
|
auto& interpreter = *static_cast<BytecodeInterpreter*>(interp_ptr);
|
|
auto& config = *static_cast<Configuration*>(config_ptr);
|
|
|
|
auto const& module = config.frame().module();
|
|
auto* src_instance = config.store().unsafe_get(module.memories().data()[src_mem]);
|
|
auto* dst_instance = config.store().unsafe_get(module.memories().data()[dst_mem]);
|
|
|
|
auto src_end = static_cast<u64>(static_cast<u32>(src_offset)) + static_cast<u32>(count);
|
|
auto dst_end = static_cast<u64>(static_cast<u32>(dst_offset)) + static_cast<u32>(count);
|
|
if (src_end > src_instance->size() || dst_end > dst_instance->size())
|
|
return interpreter.set_trap(Trap::from_string("Memory access out of bounds"));
|
|
|
|
if (count > 0)
|
|
__builtin_memmove(dst_instance->data().data() + static_cast<u32>(dst_offset), src_instance->data().data() + static_cast<u32>(src_offset), static_cast<u32>(count));
|
|
|
|
return 0;
|
|
}
|
|
|
|
i32 wasm_cl_memory_fill(void* interp_ptr, void* config_ptr, i32 mem_idx, i32 offset, i32 value, i32 count);
|
|
i32 wasm_cl_memory_fill(void* interp_ptr, void* config_ptr, i32 mem_idx, i32 offset, i32 value, i32 count)
|
|
{
|
|
auto& interpreter = *static_cast<BytecodeInterpreter*>(interp_ptr);
|
|
auto& config = *static_cast<Configuration*>(config_ptr);
|
|
|
|
auto const& module = config.frame().module();
|
|
auto* instance = config.store().unsafe_get(module.memories().data()[mem_idx]);
|
|
|
|
auto end = static_cast<u64>(static_cast<u32>(offset)) + static_cast<u32>(count);
|
|
if (end > instance->size())
|
|
return interpreter.set_trap(Trap::from_string("Memory access out of bounds"));
|
|
|
|
if (count > 0)
|
|
__builtin_memset(instance->data().data() + static_cast<u32>(offset), static_cast<u8>(value), static_cast<u32>(count));
|
|
|
|
return 0;
|
|
}
|
|
|
|
void wasm_cl_stack_push(void* config_ptr, i64 value);
|
|
void wasm_cl_stack_push(void* config_ptr, i64 value)
|
|
{
|
|
auto& config = *static_cast<Configuration*>(config_ptr);
|
|
config.value_stack().append(Value(value));
|
|
}
|
|
|
|
i64 wasm_cl_stack_pop(void* config_ptr);
|
|
i64 wasm_cl_stack_pop(void* config_ptr)
|
|
{
|
|
auto& config = *static_cast<Configuration*>(config_ptr);
|
|
return config.value_stack().unsafe_take_last().to<i64>();
|
|
}
|
|
|
|
i64 wasm_cl_stack_size(void* config_ptr);
|
|
i64 wasm_cl_stack_size(void* config_ptr)
|
|
{
|
|
auto& config = *static_cast<Configuration*>(config_ptr);
|
|
return static_cast<i64>(config.value_stack().size());
|
|
}
|
|
|
|
void wasm_cl_stack_cleanup(void* config_ptr, i64 initial_size, i32 result_arity);
|
|
void wasm_cl_stack_cleanup(void* config_ptr, i64 initial_size, i32 result_arity)
|
|
{
|
|
auto& config = *static_cast<Configuration*>(config_ptr);
|
|
auto& stack = config.value_stack();
|
|
auto expected = static_cast<size_t>(initial_size) + static_cast<size_t>(result_arity);
|
|
if (stack.size() == expected)
|
|
return;
|
|
if (stack.size() < expected) {
|
|
// Under-push (e.g. trap path), fill with zeros so the caller has something to pop.
|
|
while (stack.size() < expected)
|
|
stack.append(Value(static_cast<i64>(0)));
|
|
return;
|
|
}
|
|
|
|
// Take results, trim stack, and put them back.
|
|
Value saved[8];
|
|
auto n = min(static_cast<size_t>(result_arity), size_t(8));
|
|
for (size_t i = 0; i < n; i++)
|
|
saved[i] = stack.unsafe_take_last();
|
|
|
|
while (stack.size() > static_cast<size_t>(initial_size))
|
|
stack.unsafe_take_last();
|
|
|
|
for (size_t i = n; i > 0; i--)
|
|
stack.append(saved[i - 1]);
|
|
}
|
|
|
|
i64 wasm_cl_callrec_read(void* config_ptr, i32 index);
|
|
i64 wasm_cl_callrec_read(void* config_ptr, i32 index)
|
|
{
|
|
auto& config = *static_cast<Configuration*>(config_ptr);
|
|
return config.call_record_entry(index).to<i64>();
|
|
}
|
|
|
|
void wasm_cl_callrec_write(void* config_ptr, i32 index, i64 value);
|
|
void wasm_cl_callrec_write(void* config_ptr, i32 index, i64 value)
|
|
{
|
|
auto& config = *static_cast<Configuration*>(config_ptr);
|
|
if (!config.call_record_base()) [[unlikely]] {
|
|
// No call record yet, allocate now.
|
|
auto size = config.frame().expression().compiled_instructions.max_call_rec_size;
|
|
if (size == 0)
|
|
size = static_cast<size_t>(index) + 1;
|
|
config.setup_call_record(size);
|
|
}
|
|
config.call_record_entry(index) = Value(value);
|
|
}
|
|
|
|
i32 wasm_cl_call_with_record(void* interp_ptr, void* config_ptr, i32 func_index);
|
|
i32 wasm_cl_call_with_record(void* interp_ptr, void* config_ptr, i32 func_index)
|
|
{
|
|
auto& interpreter = *static_cast<BytecodeInterpreter*>(interp_ptr);
|
|
auto& config = *static_cast<Configuration*>(config_ptr);
|
|
|
|
auto const& module = config.frame().module();
|
|
auto const& functions = module.functions();
|
|
if (static_cast<size_t>(func_index) >= functions.size())
|
|
return 1;
|
|
|
|
auto address = functions[func_index];
|
|
auto* instance = config.store().get(address);
|
|
if (!instance) {
|
|
interpreter.set_trap("Attempt to call nonexistent function by address"sv);
|
|
return 1;
|
|
}
|
|
|
|
FunctionType const* type { nullptr };
|
|
instance->visit([&](auto const& function) { type = &function.type(); });
|
|
|
|
Vector<Value, ArgumentsStaticSize> args;
|
|
config.take_call_record(args);
|
|
args.shrink(type->parameters().size(), true);
|
|
return wasm_cl_finish_call(interpreter, config, address, args);
|
|
}
|
|
|
|
// Direct compiled-to-compiled call. Falls back to wasm_cl_finish_call for non-compiled targets.
|
|
static ALWAYS_INLINE i32 wasm_cl_direct_call_impl(BytecodeInterpreter& interpreter, Configuration& config, i32 func_index, Value* args, size_t arg_count)
|
|
{
|
|
auto const* table = config.frame().compiled_fn_table();
|
|
auto index = static_cast<size_t>(func_index);
|
|
if (!table || index >= table->size() || !(*table)[index].module) [[unlikely]] {
|
|
// Not Cranelift-compiled, fall back to full path.
|
|
Vector<Value, ArgumentsStaticSize> args_vec;
|
|
args_vec.ensure_capacity(arg_count);
|
|
for (size_t i = 0; i < arg_count; i++)
|
|
args_vec.unchecked_append(args[i]);
|
|
return wasm_cl_finish_call(interpreter, config, config.frame().module().functions()[func_index], args_vec);
|
|
}
|
|
|
|
auto const& entry = (*table)[index];
|
|
|
|
if (config.depth() > 500) [[unlikely]] {
|
|
interpreter.set_trap(Constants::stack_exhaustion_message);
|
|
return 1;
|
|
}
|
|
|
|
// Stack-allocate callee locals: args + zero-initialized locals.
|
|
auto total = arg_count + entry.total_local_count;
|
|
Value callee_locals_buf[64];
|
|
Value* callee_locals = callee_locals_buf;
|
|
Vector<Value, ArgumentsStaticSize> heap_buf;
|
|
if (total > 64) [[unlikely]] {
|
|
heap_buf.ensure_capacity(total);
|
|
heap_buf.resize_and_keep_capacity(total);
|
|
callee_locals = heap_buf.data();
|
|
}
|
|
for (size_t i = 0; i < arg_count; i++)
|
|
callee_locals[i] = args[i];
|
|
__builtin_memset(callee_locals + arg_count, 0, entry.total_local_count * sizeof(Value));
|
|
|
|
// Lightweight non-owning frame push + direct handler call.
|
|
BytecodeInterpreter::CallFrameHandle handle { interpreter, config };
|
|
config.set_frame_lightweight(*entry.module, callee_locals, *entry.expression, entry.arity);
|
|
config.setup_call_record_for_current_frame();
|
|
config.ip() = 0;
|
|
|
|
interpreter.clear_trap();
|
|
using HandlerFn = Outcome (*)(BytecodeInterpreter&, Configuration&, Instruction const*, u32, Dispatch const*, SourcesAndDestination const*);
|
|
auto const handler = bit_cast<HandlerFn>(entry.handler_ptr);
|
|
auto outcome = handler(interpreter, config, entry.first_insn, 0, bit_cast<Dispatch const*>(entry.dispatches_ptr), bit_cast<SourcesAndDestination const*>(entry.src_dst_ptr));
|
|
|
|
if (outcome != Outcome::Return) {
|
|
interpreter.set_trap("Compiled function returned unexpectedly"sv);
|
|
return 1;
|
|
}
|
|
if (interpreter.did_trap())
|
|
return 1;
|
|
if (entry.arity == 1)
|
|
config.compiled_call_result_scratch() = config.value_stack().unsafe_take_last();
|
|
// No label pop: set_frame_lightweight doesn't push labels.
|
|
return 0;
|
|
}
|
|
|
|
i32 wasm_cl_direct_call_0(void* interp_ptr, void* config_ptr, i32 func_index);
|
|
i32 wasm_cl_direct_call_0(void* interp_ptr, void* config_ptr, i32 func_index)
|
|
{
|
|
auto& interpreter = *static_cast<BytecodeInterpreter*>(interp_ptr);
|
|
auto& config = *static_cast<Configuration*>(config_ptr);
|
|
return wasm_cl_direct_call_impl(interpreter, config, func_index, nullptr, 0);
|
|
}
|
|
|
|
i32 wasm_cl_direct_call_1(void* interp_ptr, void* config_ptr, i32 func_index, i64 arg0);
|
|
i32 wasm_cl_direct_call_1(void* interp_ptr, void* config_ptr, i32 func_index, i64 arg0)
|
|
{
|
|
auto& interpreter = *static_cast<BytecodeInterpreter*>(interp_ptr);
|
|
auto& config = *static_cast<Configuration*>(config_ptr);
|
|
Value args[] = { Value(arg0) };
|
|
return wasm_cl_direct_call_impl(interpreter, config, func_index, args, 1);
|
|
}
|
|
|
|
i32 wasm_cl_direct_call_2(void* interp_ptr, void* config_ptr, i32 func_index, i64 arg0, i64 arg1);
|
|
i32 wasm_cl_direct_call_2(void* interp_ptr, void* config_ptr, i32 func_index, i64 arg0, i64 arg1)
|
|
{
|
|
auto& interpreter = *static_cast<BytecodeInterpreter*>(interp_ptr);
|
|
auto& config = *static_cast<Configuration*>(config_ptr);
|
|
Value args[] = { Value(arg0), Value(arg1) };
|
|
return wasm_cl_direct_call_impl(interpreter, config, func_index, args, 2);
|
|
}
|
|
|
|
i32 wasm_cl_direct_call_3(void* interp_ptr, void* config_ptr, i32 func_index, i64 arg0, i64 arg1, i64 arg2);
|
|
i32 wasm_cl_direct_call_3(void* interp_ptr, void* config_ptr, i32 func_index, i64 arg0, i64 arg1, i64 arg2)
|
|
{
|
|
auto& interpreter = *static_cast<BytecodeInterpreter*>(interp_ptr);
|
|
auto& config = *static_cast<Configuration*>(config_ptr);
|
|
Value args[] = { Value(arg0), Value(arg1), Value(arg2) };
|
|
return wasm_cl_direct_call_impl(interpreter, config, func_index, args, 3);
|
|
}
|
|
|
|
// Thin frame push for direct compiled-to-compiled calls. Returns 1 on trap, 0 on success.
|
|
i32 wasm_cl_push_frame(void* interp_ptr, void* config_ptr, Value* locals_ptr, u32, void const* module_ptr, void const* expression_ptr, u32 arity, u32 max_call_rec_size);
|
|
i32 wasm_cl_push_frame(void* interp_ptr, void* config_ptr, Value* locals_ptr, u32 /* total_locals */, void const* module_ptr, void const* expression_ptr, u32 arity, u32 max_call_rec_size)
|
|
{
|
|
auto& interpreter = *static_cast<BytecodeInterpreter*>(interp_ptr);
|
|
auto& config = *static_cast<Configuration*>(config_ptr);
|
|
auto const& module = *static_cast<ModuleInstance const*>(module_ptr);
|
|
auto const& expression = *static_cast<Expression const*>(expression_ptr);
|
|
|
|
if (interpreter.trap_if_insufficient_native_stack_space())
|
|
return 1;
|
|
|
|
config.set_frame_lightweight(module, locals_ptr, expression, arity);
|
|
config.depth()++;
|
|
|
|
// Set up call record for the callee if needed.
|
|
if (max_call_rec_size > 0) {
|
|
config.set_call_record_base(nullptr);
|
|
config.setup_call_record(max_call_rec_size);
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
// Thin frame pop for direct compiled-to-compiled calls.
|
|
void wasm_cl_pop_frame(void* config_ptr, u32 arity);
|
|
void wasm_cl_pop_frame(void* config_ptr, u32 arity)
|
|
{
|
|
auto& config = *static_cast<Configuration*>(config_ptr);
|
|
if (arity == 1)
|
|
config.compiled_call_result_scratch() = config.value_stack().unsafe_take_last();
|
|
if (!config.label_stack().is_empty())
|
|
config.label_stack().take_last();
|
|
config.unwind_impl();
|
|
}
|
|
}
|
|
|
|
namespace Wasm {
|
|
|
|
static RuntimeHelpers make_runtime_helpers()
|
|
{
|
|
return RuntimeHelpers {
|
|
.call_function = bit_cast<uintptr_t>(&wasm_cl_call_function),
|
|
.set_trap = bit_cast<uintptr_t>(&wasm_cl_set_trap),
|
|
.memory_load8_s = bit_cast<uintptr_t>(&wasm_cl_memory_load8_s),
|
|
.memory_load8_u = bit_cast<uintptr_t>(&wasm_cl_memory_load8_u),
|
|
.memory_load16_s = bit_cast<uintptr_t>(&wasm_cl_memory_load16_s),
|
|
.memory_load16_u = bit_cast<uintptr_t>(&wasm_cl_memory_load16_u),
|
|
.memory_load32_s = bit_cast<uintptr_t>(&wasm_cl_memory_load32_s),
|
|
.memory_load32_u = bit_cast<uintptr_t>(&wasm_cl_memory_load32_u),
|
|
.memory_load64 = bit_cast<uintptr_t>(&wasm_cl_memory_load64),
|
|
.memory_store8 = bit_cast<uintptr_t>(&wasm_cl_memory_store8),
|
|
.memory_store16 = bit_cast<uintptr_t>(&wasm_cl_memory_store16),
|
|
.memory_store32 = bit_cast<uintptr_t>(&wasm_cl_memory_store32),
|
|
.memory_store64 = bit_cast<uintptr_t>(&wasm_cl_memory_store64),
|
|
.memory_size = bit_cast<uintptr_t>(&wasm_cl_memory_size),
|
|
.memory_grow = bit_cast<uintptr_t>(&wasm_cl_memory_grow),
|
|
.read_global = bit_cast<uintptr_t>(&wasm_cl_read_global),
|
|
.write_global = bit_cast<uintptr_t>(&wasm_cl_write_global),
|
|
.stack_push = bit_cast<uintptr_t>(&wasm_cl_stack_push),
|
|
.stack_pop = bit_cast<uintptr_t>(&wasm_cl_stack_pop),
|
|
.stack_size = bit_cast<uintptr_t>(&wasm_cl_stack_size),
|
|
.stack_cleanup = bit_cast<uintptr_t>(&wasm_cl_stack_cleanup),
|
|
.callrec_read = bit_cast<uintptr_t>(&wasm_cl_callrec_read),
|
|
.callrec_write = bit_cast<uintptr_t>(&wasm_cl_callrec_write),
|
|
.call_with_record = bit_cast<uintptr_t>(&wasm_cl_call_with_record),
|
|
.direct_call_0 = bit_cast<uintptr_t>(&wasm_cl_direct_call_0),
|
|
.direct_call_1 = bit_cast<uintptr_t>(&wasm_cl_direct_call_1),
|
|
.direct_call_2 = bit_cast<uintptr_t>(&wasm_cl_direct_call_2),
|
|
.direct_call_3 = bit_cast<uintptr_t>(&wasm_cl_direct_call_3),
|
|
.call_indirect = bit_cast<uintptr_t>(&wasm_cl_call_indirect),
|
|
.memory_copy = bit_cast<uintptr_t>(&wasm_cl_memory_copy),
|
|
.memory_fill = bit_cast<uintptr_t>(&wasm_cl_memory_fill),
|
|
.regs_offset = static_cast<u32>(offsetof(Configuration, regs)),
|
|
.value_size = static_cast<u32>(sizeof(Value)),
|
|
.locals_base_offset = static_cast<u32>(Configuration::locals_base_offset()),
|
|
.default_memory_base_offset = static_cast<u32>(Configuration::default_memory_base_offset()),
|
|
.compiled_call_result_scratch_offset = static_cast<u32>(Configuration::compiled_call_result_scratch_offset()),
|
|
};
|
|
}
|
|
|
|
static CraneliftInsn serialize_insn(Dispatch const& dispatch, SourcesAndDestination const& addr)
|
|
{
|
|
CraneliftInsn out {};
|
|
auto const* insn = dispatch.instruction;
|
|
out.opcode = insn->opcode().value();
|
|
out.sources[0] = addr.sources[0];
|
|
out.sources[1] = addr.sources[1];
|
|
out.sources[2] = addr.sources[2];
|
|
out.destination = addr.destination;
|
|
out.imm1 = 0;
|
|
out.imm2 = 0;
|
|
out.imm3 = 0;
|
|
|
|
auto const& args = insn->arguments();
|
|
u32 opc = out.opcode;
|
|
|
|
if (opc == Instructions::i32_const.value()) {
|
|
out.imm1 = static_cast<i64>(args.get<i32>());
|
|
} else if (opc == Instructions::i64_const.value()) {
|
|
out.imm1 = args.get<i64>();
|
|
} else if (opc == Instructions::f32_const.value()) {
|
|
out.imm1 = static_cast<i64>(bit_cast<i32>(args.get<float>()));
|
|
} else if (opc == Instructions::f64_const.value()) {
|
|
out.imm1 = bit_cast<i64>(args.get<double>());
|
|
} else if (opc == Instructions::local_get.value() || opc == Instructions::local_set.value() || opc == Instructions::local_tee.value()) {
|
|
out.imm1 = static_cast<i64>(insn->local_index().value());
|
|
} else if (opc == Instructions::global_get.value() || opc == Instructions::global_set.value()) {
|
|
out.imm1 = static_cast<i64>(args.get<GlobalIndex>().value());
|
|
} else if (opc == Instructions::br.value() || opc == Instructions::br_if.value()) {
|
|
auto const& br_args = args.get<Instruction::BranchArgs>();
|
|
out.imm1 = static_cast<i64>(br_args.label.value());
|
|
} else if (opc == Instructions::block.value() || opc == Instructions::loop.value() || opc == Instructions::if_.value()) {
|
|
auto const& struct_args = args.get<Instruction::StructuredInstructionArgs>();
|
|
out.imm1 = static_cast<i64>(struct_args.end_ip.value());
|
|
out.imm2 = struct_args.else_ip().has_value()
|
|
? static_cast<i64>(struct_args.else_ip()->value())
|
|
: -1;
|
|
u32 arity = struct_args.meta.arity;
|
|
u32 param_count = struct_args.meta.parameter_count;
|
|
out.imm3 = arity | (param_count << 16);
|
|
} else if (opc == Instructions::br_table.value()) {
|
|
auto const& table_args = args.get<Instruction::TableBranchArgs>();
|
|
if (table_args.default_.value() > NumericLimits<u16>::max()) {
|
|
out.imm3 = 0xff;
|
|
return out;
|
|
}
|
|
for (auto const& label : table_args.labels) {
|
|
if (label.value() > NumericLimits<u16>::max()) {
|
|
out.imm3 = 0xff;
|
|
return out;
|
|
}
|
|
}
|
|
|
|
// Pack: imm3 low byte = min(label_count, 8), upper bits = default label.
|
|
// First 4 labels in imm1, next 4 in imm2 (16 bits each).
|
|
// If more than 8 labels, continuation instructions carry the rest.
|
|
auto const total_labels = table_args.labels.size();
|
|
auto const inline_count = min(total_labels, static_cast<size_t>(8));
|
|
out.imm3 = static_cast<u32>(inline_count) | (static_cast<u32>(table_args.default_.value()) << 8);
|
|
for (size_t i = 0; i < inline_count; ++i) {
|
|
auto const encoded = static_cast<u64>(table_args.labels[i].value()) << ((i % 4) * 16);
|
|
if (i < 4)
|
|
out.imm1 |= static_cast<i64>(encoded);
|
|
else
|
|
out.imm2 |= static_cast<i64>(encoded);
|
|
}
|
|
} else if (opc == Instructions::call.value()) {
|
|
out.imm1 = static_cast<i64>(args.get<FunctionIndex>().value());
|
|
} else if (opc == Instructions::call_indirect.value()) {
|
|
auto const& indirect_args = args.get<Instruction::IndirectCallArgs>();
|
|
out.imm1 = static_cast<i64>(indirect_args.type.value());
|
|
out.imm2 = static_cast<i64>(indirect_args.table.value());
|
|
} else if (opc == Instructions::memory_copy.value()) {
|
|
auto const& copy_args = args.get<Instruction::MemoryCopyArgs>();
|
|
out.imm1 = static_cast<i64>(copy_args.dst_index.value());
|
|
out.imm2 = static_cast<i64>(copy_args.src_index.value());
|
|
} else if (opc == Instructions::memory_fill.value()) {
|
|
auto const& fill_args = args.get<Instruction::MemoryIndexArgument>();
|
|
out.imm1 = static_cast<i64>(fill_args.memory_index.value());
|
|
} else if (opc >= Instructions::i32_load.value() && opc <= Instructions::i64_store32.value()) {
|
|
auto const& mem_arg = args.get<Instruction::MemoryArgument>();
|
|
out.imm1 = static_cast<i64>(mem_arg.offset);
|
|
out.imm3 = static_cast<u32>(mem_arg.memory_index.value()) | (mem_arg.memory_index.value() == 0 ? (1u << 31) : 0);
|
|
} else if (opc == Instructions::memory_size.value()
|
|
|| opc == Instructions::memory_grow.value()) {
|
|
auto const& mem_idx_arg = args.get<Instruction::MemoryIndexArgument>();
|
|
out.imm1 = static_cast<i64>(mem_idx_arg.memory_index.value());
|
|
}
|
|
|
|
auto is_syn = [opc](OpCode op) { return opc == op.value(); };
|
|
auto syn_between = [opc](OpCode lo, OpCode hi) { return opc >= lo.value() && opc <= hi.value(); };
|
|
|
|
if (opc >= Instructions::SyntheticInstructionBase.value()) {
|
|
if (syn_between(Instructions::synthetic_call_00, Instructions::synthetic_call_31)) {
|
|
out.imm1 = static_cast<i64>(args.get<FunctionIndex>().value());
|
|
} else if (is_syn(Instructions::synthetic_call_with_record_0) || is_syn(Instructions::synthetic_call_with_record_1)) {
|
|
out.imm1 = static_cast<i64>(args.get<FunctionIndex>().value());
|
|
} else if (is_syn(Instructions::synthetic_br_nostack) || is_syn(Instructions::synthetic_br_if_nostack)) {
|
|
auto const& br_args = args.get<Instruction::BranchArgs>();
|
|
out.imm1 = static_cast<i64>(br_args.label.value());
|
|
} else if (is_syn(Instructions::synthetic_local_copy)) {
|
|
out.imm1 = static_cast<i64>(insn->local_index().value());
|
|
out.imm2 = static_cast<i64>(args.get<LocalIndex>().value());
|
|
} else if (is_syn(Instructions::synthetic_i32_add2local)
|
|
|| syn_between(Instructions::synthetic_i32_sub2local, Instructions::synthetic_i32_shrs2local)
|
|
|| is_syn(Instructions::synthetic_i64_add2local)
|
|
|| syn_between(Instructions::synthetic_i64_sub2local, Instructions::synthetic_i64_shrs2local)) {
|
|
out.imm1 = static_cast<i64>(insn->local_index().value());
|
|
out.imm2 = static_cast<i64>(args.get<LocalIndex>().value());
|
|
} else if (is_syn(Instructions::synthetic_i32_addconstlocal) || is_syn(Instructions::synthetic_i32_andconstlocal)) {
|
|
out.imm1 = static_cast<i64>(args.get<i32>());
|
|
out.imm2 = static_cast<i64>(insn->local_index().value());
|
|
} else if (is_syn(Instructions::synthetic_i64_addconstlocal) || is_syn(Instructions::synthetic_i64_andconstlocal)) {
|
|
out.imm1 = args.get<i64>();
|
|
out.imm2 = static_cast<i64>(insn->local_index().value());
|
|
} else if (is_syn(Instructions::synthetic_i32_storelocal) || is_syn(Instructions::synthetic_i64_storelocal)) {
|
|
auto const& mem_arg = args.get<Instruction::MemoryArgument>();
|
|
out.imm1 = static_cast<i64>(mem_arg.offset);
|
|
out.imm2 = static_cast<i64>(insn->local_index().value());
|
|
out.imm3 = static_cast<u32>(mem_arg.memory_index.value()) | (mem_arg.memory_index.value() == 0 ? (1u << 31) : 0);
|
|
} else if (is_syn(Instructions::synthetic_local_seti32_const)) {
|
|
out.imm1 = static_cast<i64>(args.get<i32>());
|
|
out.imm2 = static_cast<i64>(insn->local_index().value());
|
|
} else if (is_syn(Instructions::synthetic_local_seti64_const)) {
|
|
out.imm1 = args.get<i64>();
|
|
out.imm2 = static_cast<i64>(insn->local_index().value());
|
|
} else if (syn_between(Instructions::synthetic_argument_get, Instructions::synthetic_argument_tee)) {
|
|
out.imm1 = static_cast<i64>(insn->local_index().value());
|
|
}
|
|
}
|
|
|
|
return out;
|
|
}
|
|
|
|
static StringView resolve_cranelift_compiler_path()
|
|
{
|
|
// Lookup order: LADYBIRD_CRANELIFT_COMPILER, compile-time path, sibling-of-self.
|
|
static NeverDestroyed<ByteString> s_path = []() -> ByteString {
|
|
auto file_exists = [](ByteString const& path) {
|
|
return !Core::System::stat(path).is_error();
|
|
};
|
|
|
|
if (auto const* env = getenv("LADYBIRD_CRANELIFT_COMPILER"); env && *env) {
|
|
if (file_exists(env))
|
|
return ByteString { env };
|
|
}
|
|
|
|
if (file_exists(WASM_CRANELIFT_COMPILER_PATH))
|
|
return WASM_CRANELIFT_COMPILER_PATH;
|
|
|
|
if (auto self_path = Core::System::current_executable_path(); !self_path.is_error()) {
|
|
auto sibling = LexicalPath::join(LexicalPath::dirname(self_path.value()), "cranelift-compiler"sv).string();
|
|
if (file_exists(sibling))
|
|
return sibling;
|
|
}
|
|
|
|
return WASM_CRANELIFT_COMPILER_PATH;
|
|
}();
|
|
return s_path->view();
|
|
}
|
|
|
|
static void try_cranelift_compile_batch(Vector<BatchInput>& batch)
|
|
{
|
|
if (batch.is_empty())
|
|
return;
|
|
|
|
static auto helpers = make_runtime_helpers();
|
|
u64 outcome_return = to_underlying(Outcome::Return);
|
|
|
|
size_t function_count = batch.size();
|
|
auto const entries_offset = sizeof(InputHeader);
|
|
auto const entries_size = sizeof(InputFunctionEntry) * function_count;
|
|
|
|
size_t total_insn_count = 0;
|
|
for (auto& entry : batch)
|
|
total_insn_count += entry.insns.size();
|
|
|
|
auto const insn_region_offset = align_up(entries_offset + entries_size, alignof(CraneliftInsn));
|
|
auto const insn_bytes = total_insn_count * sizeof(CraneliftInsn);
|
|
auto const helpers_offset = align_up(insn_region_offset + insn_bytes, alignof(RuntimeHelpers));
|
|
auto const code_region_start = align_up(helpers_offset + sizeof(RuntimeHelpers), alignof(OutputFunctionEntry));
|
|
auto const code_region_size = max(oop_code_region_min_size, total_insn_count * oop_code_bytes_per_insn);
|
|
auto const reloc_region_start = align_up(code_region_start + sizeof(OutputFunctionEntry) * function_count + code_region_size, alignof(HelperReloc));
|
|
auto const reloc_region_size = max(oop_reloc_region_min_size, total_insn_count * oop_reloc_bytes_per_insn);
|
|
auto const total_size = reloc_region_start + reloc_region_size;
|
|
|
|
#if defined(AK_OS_WINDOWS)
|
|
DWORD size_hi = static_cast<DWORD>(static_cast<u64>(total_size) >> 32);
|
|
DWORD size_lo = static_cast<DWORD>(total_size & 0xFFFFFFFF);
|
|
HANDLE section_handle = CreateFileMapping(INVALID_HANDLE_VALUE, NULL, PAGE_READWRITE, size_hi, size_lo, NULL);
|
|
if (!section_handle)
|
|
return;
|
|
SetHandleInformation(section_handle, HANDLE_FLAG_INHERIT, HANDLE_FLAG_INHERIT);
|
|
ScopeGuard close_handle = [section_handle] { CloseHandle(section_handle); };
|
|
|
|
auto* mapping = MapViewOfFile(section_handle, FILE_MAP_ALL_ACCESS, 0, 0, total_size);
|
|
if (!mapping)
|
|
return;
|
|
ScopeGuard unmap = [mapping] { UnmapViewOfFile(mapping); };
|
|
#elif defined(AK_OS_MACOS)
|
|
// macOS lacks memfd_create; use shm_open + shm_unlink for an anonymous fd.
|
|
char shm_name[] = "/libwasm-cranelift-XXXXXX";
|
|
arc4random_buf(shm_name + 21, 6);
|
|
for (int i = 21; i < 27; ++i)
|
|
shm_name[i] = 'A' + (static_cast<unsigned char>(shm_name[i]) % 26);
|
|
int fd = shm_open(shm_name, O_RDWR | O_CREAT | O_EXCL, 0600);
|
|
if (fd < 0)
|
|
return;
|
|
shm_unlink(shm_name);
|
|
// POSIX shm_open sets FD_CLOEXEC on the returned fd, which would close it
|
|
// in the spawned cranelift-compiler child. Clear it so the child inherits.
|
|
if (auto flags = fcntl(fd, F_GETFD); flags >= 0)
|
|
fcntl(fd, F_SETFD, flags & ~FD_CLOEXEC);
|
|
ScopeGuard close_fd = [fd] { close(fd); };
|
|
if (ftruncate(fd, static_cast<off_t>(total_size)) < 0)
|
|
return;
|
|
|
|
auto* mapping = mmap(nullptr, total_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
|
|
if (mapping == MAP_FAILED)
|
|
return;
|
|
ScopeGuard unmap = [mapping, total_size] { munmap(mapping, total_size); };
|
|
#else
|
|
int fd = memfd_create("libwasm-cranelift", 0);
|
|
if (fd < 0)
|
|
return;
|
|
ScopeGuard close_fd = [fd] { close(fd); };
|
|
if (ftruncate(fd, static_cast<off_t>(total_size)) < 0)
|
|
return;
|
|
|
|
auto* mapping = mmap(nullptr, total_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
|
|
if (mapping == MAP_FAILED)
|
|
return;
|
|
ScopeGuard unmap = [mapping, total_size] { munmap(mapping, total_size); };
|
|
#endif
|
|
|
|
auto* base = static_cast<u8*>(mapping);
|
|
__builtin_memset(base, 0, total_size);
|
|
|
|
auto* header = reinterpret_cast<InputHeader*>(base);
|
|
*header = InputHeader {
|
|
.function_count = static_cast<u32>(function_count),
|
|
.helpers_offset = static_cast<u32>(helpers_offset),
|
|
.outcome_return = outcome_return,
|
|
.code_region_start = code_region_start,
|
|
.reloc_region_start = reloc_region_start,
|
|
.total_size = total_size,
|
|
};
|
|
|
|
size_t insn_cursor = insn_region_offset;
|
|
for (size_t i = 0; i < function_count; ++i) {
|
|
auto& input = batch[i];
|
|
auto* entry = reinterpret_cast<InputFunctionEntry*>(base + entries_offset + i * sizeof(InputFunctionEntry));
|
|
*entry = InputFunctionEntry {
|
|
.insn_offset = static_cast<u32>(insn_cursor),
|
|
.insn_count = static_cast<u32>(input.insns.size()),
|
|
.result_arity = input.result_arity,
|
|
};
|
|
__builtin_memcpy(base + insn_cursor, input.insns.data(), input.insns.size() * sizeof(CraneliftInsn));
|
|
insn_cursor += input.insns.size() * sizeof(CraneliftInsn);
|
|
}
|
|
|
|
__builtin_memcpy(base + helpers_offset, &helpers, sizeof(helpers));
|
|
|
|
Vector<ByteString> arguments;
|
|
#if defined(AK_OS_WINDOWS)
|
|
arguments.append(ByteString::number(reinterpret_cast<uintptr_t>(section_handle)));
|
|
#else
|
|
arguments.append(ByteString::number(fd));
|
|
#endif
|
|
|
|
auto process_result = Core::Process::spawn({
|
|
.name = "cranelift-compiler"sv,
|
|
.executable = resolve_cranelift_compiler_path(),
|
|
.arguments = arguments,
|
|
});
|
|
if (process_result.is_error())
|
|
return;
|
|
auto status_result = process_result.release_value().wait_for_termination();
|
|
if (status_result.is_error() || status_result.value() != 0)
|
|
return;
|
|
|
|
// Extract results for each function.
|
|
auto const code_base_offset = code_region_start + sizeof(OutputFunctionEntry) * function_count;
|
|
|
|
for (size_t i = 0; i < function_count; ++i) {
|
|
auto const* output = reinterpret_cast<OutputFunctionEntry const*>(base + code_region_start + i * sizeof(OutputFunctionEntry));
|
|
if (!output->compiled)
|
|
continue;
|
|
|
|
auto code_offset = static_cast<size_t>(output->code_offset);
|
|
auto code_size = static_cast<size_t>(output->code_size);
|
|
if (code_offset > code_region_size || code_size > code_region_size - code_offset)
|
|
continue;
|
|
auto code_start = code_base_offset + code_offset;
|
|
if (code_start + code_size > total_size)
|
|
continue;
|
|
|
|
auto const reloc_offset = static_cast<size_t>(output->reloc_offset);
|
|
auto const reloc_count = static_cast<size_t>(output->reloc_count);
|
|
auto const reloc_bytes = reloc_count * sizeof(HelperReloc);
|
|
if (reloc_count != 0 && reloc_bytes / sizeof(HelperReloc) != reloc_count)
|
|
continue;
|
|
if (reloc_offset > reloc_region_size || reloc_bytes > reloc_region_size - reloc_offset)
|
|
continue;
|
|
if (reloc_region_start + reloc_offset + reloc_bytes > total_size)
|
|
continue;
|
|
|
|
auto const trap_offset = static_cast<size_t>(output->trap_offset);
|
|
auto const trap_count = static_cast<size_t>(output->trap_count);
|
|
auto const trap_bytes = trap_count * sizeof(CraneliftTrap);
|
|
if (trap_count != 0 && trap_bytes / sizeof(CraneliftTrap) != trap_count)
|
|
continue;
|
|
if (trap_offset > reloc_region_size || trap_bytes > reloc_region_size - trap_offset)
|
|
continue;
|
|
if (reloc_region_start + trap_offset + trap_bytes > total_size)
|
|
continue;
|
|
|
|
auto code_bytes = ReadonlyBytes { base + code_start, code_size };
|
|
auto const* relocs = reloc_count == 0
|
|
? nullptr
|
|
: reinterpret_cast<HelperReloc const*>(base + reloc_region_start + reloc_offset);
|
|
auto traps = trap_count == 0
|
|
? ReadonlySpan<CraneliftTrap> {}
|
|
: ReadonlySpan<CraneliftTrap> { reinterpret_cast<CraneliftTrap const*>(base + reloc_region_start + trap_offset), trap_count };
|
|
|
|
auto& capture = cranelift_cache_state().cache_capture;
|
|
if (capture.capturing && batch[i].function_index != NumericLimits<u32>::max()) {
|
|
if (auto copy = ByteBuffer::copy(code_bytes.data(), code_bytes.size()); !copy.is_error()) {
|
|
CacheRecord rec;
|
|
rec.function_index = batch[i].function_index;
|
|
rec.unpatched_code = copy.release_value();
|
|
rec.relocs.ensure_capacity(reloc_count);
|
|
for (size_t j = 0; j < reloc_count; ++j)
|
|
rec.relocs.unchecked_append(relocs[j]);
|
|
rec.traps.ensure_capacity(trap_count);
|
|
for (size_t j = 0; j < trap_count; ++j)
|
|
rec.traps.unchecked_append(traps[j]);
|
|
capture.records.append(move(rec));
|
|
}
|
|
}
|
|
|
|
install_compiled_function(*batch[i].target, code_bytes, relocs, reloc_count, traps, helpers);
|
|
}
|
|
}
|
|
|
|
bool try_cranelift_compile(CompiledInstructions& compiled, u32 result_arity)
|
|
{
|
|
#if !WASM_COMPILED_FAULT_RECOVERY_SUPPORTED
|
|
(void)compiled;
|
|
(void)result_arity;
|
|
return false;
|
|
#else
|
|
auto const& dispatches = compiled.dispatches;
|
|
auto const& addresses = compiled.src_dst_mappings;
|
|
|
|
if (dispatches.is_empty())
|
|
return false;
|
|
|
|
// Already installed (either from a prior compile or a previous cache install in this
|
|
// same validation pass) -- nothing to do.
|
|
if (compiled.cranelift_compiled)
|
|
return true;
|
|
|
|
if (s_active_function_index == NumericLimits<u32>::max())
|
|
return false;
|
|
|
|
// Cache hit: install from the parsed blob instead of going through cranelift.
|
|
// dispatches[] has just been populated by try_compile_instructions, so handler_ptr is ready to be set.
|
|
if (cranelift_cache_state().pending_install.active && s_active_function_index != NumericLimits<u32>::max()) {
|
|
auto record = cranelift_cache_state().pending_install.records.take(s_active_function_index);
|
|
if (record.has_value()) {
|
|
static auto cache_install_helpers = make_runtime_helpers();
|
|
if (install_compiled_function(
|
|
compiled,
|
|
record->unpatched_code.bytes(),
|
|
record->relocs.is_empty() ? nullptr : record->relocs.data(),
|
|
record->relocs.size(),
|
|
record->traps.span(),
|
|
cache_install_helpers)) {
|
|
return true;
|
|
}
|
|
// Put it back so we can try later.
|
|
cranelift_cache_state().pending_install.records.set(s_active_function_index, record.release_value());
|
|
}
|
|
}
|
|
|
|
if constexpr (WASM_CRANELIFT_DEBUG) {
|
|
// CRANELIFT_MAX_INSNS=N skip functions with more than N dispatches.
|
|
// CRANELIFT_MIN_INSNS=N skip functions with fewer than N dispatches.
|
|
// CRANELIFT_MIN_FN=N skip functions with id < N.
|
|
// CRANELIFT_MAX_FN=N skip functions with id > N.
|
|
// CRANELIFT_SKIP_FN=a,b,c skip listed function ids.
|
|
// CRANELIFT_ONLY_FN=a,b,c only compile listed function ids.
|
|
// CRANELIFT_TRACE=1 log a line per compiled function.
|
|
static auto const read_size_env = [](char const* name, size_t fallback) {
|
|
if (auto* env = getenv(name))
|
|
return static_cast<size_t>(atol(env));
|
|
return fallback;
|
|
};
|
|
static auto const read_set_env = [](char const* name) {
|
|
HashTable<size_t> out;
|
|
auto* env = getenv(name);
|
|
if (!env || !*env)
|
|
return out;
|
|
StringView view { env, strlen(env) };
|
|
view.for_each_split_view(',', SplitBehavior::Nothing, [&](auto part) {
|
|
if (auto n = part.template to_number<size_t>(); n.has_value())
|
|
out.set(n.value());
|
|
});
|
|
return out;
|
|
};
|
|
static size_t s_max_insns = read_size_env("CRANELIFT_MAX_INSNS", NumericLimits<size_t>::max());
|
|
static size_t s_min_insns = read_size_env("CRANELIFT_MIN_INSNS", 0);
|
|
static size_t s_min_fn = read_size_env("CRANELIFT_MIN_FN", 0);
|
|
static size_t s_max_fn = read_size_env("CRANELIFT_MAX_FN", NumericLimits<size_t>::max());
|
|
static auto& s_skip_fn = *new HashTable<size_t>(read_set_env("CRANELIFT_SKIP_FN"));
|
|
static auto& s_only_fn = *new HashTable<size_t>(read_set_env("CRANELIFT_ONLY_FN"));
|
|
static auto& s_dump_fn = *new HashTable<size_t>(read_set_env("CRANELIFT_DUMP_FN"));
|
|
static bool s_trace = getenv("CRANELIFT_TRACE") != nullptr;
|
|
|
|
static size_t s_func_counter = 0;
|
|
size_t func_id = s_func_counter++;
|
|
|
|
if (dispatches.size() > s_max_insns || dispatches.size() < s_min_insns)
|
|
return false;
|
|
if (func_id < s_min_fn || func_id > s_max_fn)
|
|
return false;
|
|
if (s_skip_fn.contains(func_id))
|
|
return false;
|
|
if (!s_only_fn.is_empty() && !s_only_fn.contains(func_id))
|
|
return false;
|
|
if (s_trace)
|
|
warnln("cranelift: compiling fn#{} ({} dispatches)", func_id, dispatches.size());
|
|
|
|
if (s_dump_fn.contains(func_id)) {
|
|
warnln("cranelift: dump fn#{} ({} dispatches)", func_id, dispatches.size());
|
|
auto reg_name = [](Dispatch::RegisterOrStack reg) -> ByteString {
|
|
if (reg == Dispatch::RegisterOrStack::Stack)
|
|
return "stack";
|
|
if (reg >= Dispatch::RegisterOrStack::CallRecord)
|
|
return ByteString::formatted("cr{}", to_underlying(reg) - to_underlying(Dispatch::RegisterOrStack::CallRecord));
|
|
return ByteString::formatted("reg{}", to_underlying(reg));
|
|
};
|
|
for (size_t ip = 0; ip < dispatches.size(); ++ip) {
|
|
auto const& dispatch = dispatches[ip];
|
|
auto const& addr = addresses[ip];
|
|
ssize_t in_count = 0;
|
|
ssize_t out_count = 0;
|
|
# define M(name, _, ins, outs) \
|
|
case Instructions::name.value(): \
|
|
in_count = ins; \
|
|
out_count = outs; \
|
|
break;
|
|
switch (dispatch.instruction->opcode().value()) {
|
|
ENUMERATE_WASM_OPCODES(M)
|
|
}
|
|
# undef M
|
|
StringBuilder regs;
|
|
regs.append('(');
|
|
for (ssize_t j = 0; j < (in_count < 0 ? 3 : in_count); ++j) {
|
|
if (j > 0)
|
|
regs.append(", "sv);
|
|
regs.append(reg_name(addr.sources[j]));
|
|
}
|
|
regs.append(')');
|
|
if (out_count > 0) {
|
|
regs.appendff(" -> {}", reg_name(addr.destination));
|
|
}
|
|
warnln(" [{:>03}] {} {} dst={}", ip, instruction_name(dispatch.instruction->opcode()), regs.to_byte_string(), reg_name(addr.destination));
|
|
}
|
|
}
|
|
}
|
|
|
|
Vector<CraneliftInsn> flat;
|
|
flat.ensure_capacity(dispatches.size());
|
|
for (size_t i = 0; i < dispatches.size(); ++i) {
|
|
flat.append(serialize_insn(dispatches[i], addresses[i]));
|
|
|
|
if (dispatches[i].instruction->opcode().value() == Instructions::synthetic_tier_up.value())
|
|
flat.last().imm1 = static_cast<i64>(i);
|
|
|
|
if (dispatches[i].instruction->opcode().value() == Instructions::br_table.value()) {
|
|
auto const& table_args = dispatches[i].instruction->arguments().get<Instruction::TableBranchArgs>();
|
|
auto const total = table_args.labels.size();
|
|
for (size_t base = 8; base < total; base += 8) {
|
|
CraneliftInsn cont {};
|
|
cont.opcode = Instructions::synthetic_br_table_cont.value();
|
|
auto const chunk = min(total - base, static_cast<size_t>(8));
|
|
cont.imm3 = static_cast<u32>(chunk);
|
|
for (size_t j = 0; j < chunk; ++j) {
|
|
auto const encoded = static_cast<u64>(table_args.labels[base + j].value()) << ((j % 4) * 16);
|
|
if (j < 4)
|
|
cont.imm1 |= static_cast<i64>(encoded);
|
|
else
|
|
cont.imm2 |= static_cast<i64>(encoded);
|
|
}
|
|
flat.append(cont);
|
|
}
|
|
}
|
|
}
|
|
|
|
cranelift_cache_state().pending_batch.append({ move(flat), result_arity, s_active_function_index, &compiled });
|
|
return false; // Not compiled yet, will be compiled in flush.
|
|
#endif
|
|
}
|
|
|
|
void flush_cranelift_batch()
|
|
{
|
|
if (cranelift_cache_state().pending_batch.is_empty())
|
|
return;
|
|
try_cranelift_compile_batch(cranelift_cache_state().pending_batch);
|
|
cranelift_cache_state().pending_batch.clear();
|
|
}
|
|
|
|
void discard_cranelift_batch()
|
|
{
|
|
cranelift_cache_state().pending_batch.clear();
|
|
}
|
|
|
|
void free_cranelift_code(void* handle)
|
|
{
|
|
if (handle) {
|
|
auto* mapping = static_cast<CodeMapping*>(handle);
|
|
#if defined(AK_OS_WINDOWS)
|
|
VirtualFree(mapping->mapping, 0, MEM_RELEASE);
|
|
#else
|
|
munmap(mapping->mapping, mapping->size);
|
|
#endif
|
|
delete mapping;
|
|
}
|
|
}
|
|
|
|
void set_cranelift_active_function_index(u32 function_index)
|
|
{
|
|
s_active_function_index = function_index;
|
|
}
|
|
|
|
void begin_cranelift_cache_capture()
|
|
{
|
|
cranelift_cache_state().cache_capture.capturing = true;
|
|
cranelift_cache_state().cache_capture.records.clear();
|
|
}
|
|
|
|
void abort_cranelift_cache_capture()
|
|
{
|
|
cranelift_cache_state().cache_capture.capturing = false;
|
|
cranelift_cache_state().cache_capture.records.clear();
|
|
}
|
|
|
|
void abort_cranelift_cache_install()
|
|
{
|
|
cranelift_cache_state().pending_install.active = false;
|
|
cranelift_cache_state().pending_install.records.clear();
|
|
}
|
|
|
|
Optional<ByteBuffer> serialize_cranelift_cache_blob(ReadonlyBytes wasm_hash)
|
|
{
|
|
ScopeGuard reset = [] {
|
|
cranelift_cache_state().cache_capture.capturing = false;
|
|
cranelift_cache_state().cache_capture.records.clear();
|
|
};
|
|
|
|
auto const& capture = cranelift_cache_state().cache_capture;
|
|
|
|
if (!capture.capturing || capture.records.is_empty())
|
|
return {};
|
|
if (wasm_hash.size() != 32)
|
|
return {};
|
|
|
|
static auto helpers = make_runtime_helpers();
|
|
|
|
size_t total_size = sizeof(CacheBlobHeader);
|
|
for (auto const& r : capture.records) {
|
|
total_size += sizeof(CacheBlobFunctionEntry);
|
|
total_size += align_up(r.unpatched_code.size(), 16);
|
|
total_size += r.relocs.size() * sizeof(HelperReloc);
|
|
total_size += r.traps.size() * sizeof(CraneliftTrap);
|
|
}
|
|
|
|
auto blob_or_error = ByteBuffer::create_zeroed(total_size);
|
|
if (blob_or_error.is_error())
|
|
return {};
|
|
auto blob = blob_or_error.release_value();
|
|
auto* out = blob.data();
|
|
|
|
auto* header = reinterpret_cast<CacheBlobHeader*>(out);
|
|
header->magic = cache_blob_magic;
|
|
header->format_version = cache_blob_format_version;
|
|
header->helper_count = HELPER_COUNT;
|
|
header->layout_hash = compute_layout_hash(helpers);
|
|
__builtin_memcpy(header->wasm_hash, wasm_hash.data(), 32);
|
|
header->function_count = static_cast<u32>(capture.records.size());
|
|
|
|
size_t offset = sizeof(CacheBlobHeader);
|
|
for (auto const& r : capture.records) {
|
|
auto* entry = reinterpret_cast<CacheBlobFunctionEntry*>(out + offset);
|
|
entry->function_index = r.function_index;
|
|
entry->code_size = static_cast<u32>(r.unpatched_code.size());
|
|
entry->reloc_count = static_cast<u32>(r.relocs.size());
|
|
entry->trap_count = static_cast<u32>(r.traps.size());
|
|
offset += sizeof(CacheBlobFunctionEntry);
|
|
|
|
__builtin_memcpy(out + offset, r.unpatched_code.data(), r.unpatched_code.size());
|
|
offset += align_up(r.unpatched_code.size(), 16);
|
|
|
|
auto reloc_bytes = r.relocs.size() * sizeof(HelperReloc);
|
|
if (reloc_bytes > 0)
|
|
__builtin_memcpy(out + offset, r.relocs.data(), reloc_bytes);
|
|
offset += reloc_bytes;
|
|
|
|
auto trap_bytes = r.traps.size() * sizeof(CraneliftTrap);
|
|
if (trap_bytes > 0)
|
|
__builtin_memcpy(out + offset, r.traps.data(), trap_bytes);
|
|
offset += trap_bytes;
|
|
}
|
|
|
|
return blob;
|
|
}
|
|
|
|
bool try_install_cranelift_cache_blob(ReadonlyBytes expected_wasm_hash, ReadonlyBytes blob)
|
|
{
|
|
abort_cranelift_cache_install();
|
|
|
|
if (expected_wasm_hash.size() != 32 || blob.size() < sizeof(CacheBlobHeader))
|
|
return false;
|
|
|
|
auto const* header = reinterpret_cast<CacheBlobHeader const*>(blob.data());
|
|
if (header->magic != cache_blob_magic)
|
|
return false;
|
|
if (header->format_version != cache_blob_format_version)
|
|
return false;
|
|
if (header->helper_count != HELPER_COUNT)
|
|
return false;
|
|
if (__builtin_memcmp(header->wasm_hash, expected_wasm_hash.data(), 32) != 0)
|
|
return false;
|
|
|
|
static auto helpers = make_runtime_helpers();
|
|
if (header->layout_hash != compute_layout_hash(helpers))
|
|
return false;
|
|
|
|
size_t offset = sizeof(CacheBlobHeader);
|
|
for (u32 i = 0; i < header->function_count; ++i) {
|
|
if (offset + sizeof(CacheBlobFunctionEntry) > blob.size())
|
|
return false;
|
|
auto const* entry = reinterpret_cast<CacheBlobFunctionEntry const*>(blob.data() + offset);
|
|
offset += sizeof(CacheBlobFunctionEntry);
|
|
|
|
auto code_off = offset;
|
|
auto aligned_code_size = align_up(entry->code_size, 16);
|
|
if (code_off + aligned_code_size > blob.size())
|
|
return false;
|
|
offset += aligned_code_size;
|
|
|
|
auto reloc_off = offset;
|
|
auto reloc_bytes = static_cast<size_t>(entry->reloc_count) * sizeof(HelperReloc);
|
|
if (reloc_off + reloc_bytes > blob.size())
|
|
return false;
|
|
offset += reloc_bytes;
|
|
|
|
auto trap_off = offset;
|
|
auto trap_bytes = static_cast<size_t>(entry->trap_count) * sizeof(CraneliftTrap);
|
|
if (trap_off + trap_bytes > blob.size())
|
|
return false;
|
|
offset += trap_bytes;
|
|
|
|
auto code_copy = ByteBuffer::copy(blob.data() + code_off, entry->code_size);
|
|
if (code_copy.is_error())
|
|
return false;
|
|
|
|
CacheRecord rec;
|
|
rec.function_index = entry->function_index;
|
|
rec.unpatched_code = code_copy.release_value();
|
|
rec.relocs.ensure_capacity(entry->reloc_count);
|
|
for (u32 j = 0; j < entry->reloc_count; ++j) {
|
|
HelperReloc reloc;
|
|
__builtin_memcpy(&reloc, blob.data() + reloc_off + j * sizeof(HelperReloc), sizeof(HelperReloc));
|
|
rec.relocs.unchecked_append(reloc);
|
|
}
|
|
rec.traps.ensure_capacity(entry->trap_count);
|
|
for (u32 j = 0; j < entry->trap_count; ++j) {
|
|
CraneliftTrap trap;
|
|
__builtin_memcpy(&trap, blob.data() + trap_off + j * sizeof(CraneliftTrap), sizeof(CraneliftTrap));
|
|
rec.traps.unchecked_append(trap);
|
|
}
|
|
cranelift_cache_state().pending_install.records.set(entry->function_index, move(rec));
|
|
}
|
|
|
|
cranelift_cache_state().pending_install.active = true;
|
|
return true;
|
|
}
|
|
|
|
}
|