LibWasm+LibWeb: Serialize and save compiled wasm as a cache blob
This commit is contained in:
Ali Mohammad Pur
Ali Mohammad Pur
parent
ab6eac02ff
commit
2419cce8d8
15 changed files with 737 additions and 146 deletions
|
|
@ -117,6 +117,8 @@ static constexpr StringView cache_entry_associated_data_suffix(CacheEntryAssocia
|
|||
switch (associated_data) {
|
||||
case CacheEntryAssociatedData::JavaScriptBytecode:
|
||||
return "jsbc"sv;
|
||||
case CacheEntryAssociatedData::WebAssemblyCompiledCode:
|
||||
return "wasmjit"sv;
|
||||
}
|
||||
VERIFY_NOT_REACHED();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -26,8 +26,12 @@ constexpr inline u64 DEFAULT_MAXIMUM_DISK_CACHE_SIZE = 5 * GiB;
|
|||
|
||||
enum class CacheEntryAssociatedData {
|
||||
JavaScriptBytecode,
|
||||
WebAssemblyCompiledCode,
|
||||
};
|
||||
constexpr inline Array CACHE_ENTRY_ASSOCIATED_DATA_TYPES {
|
||||
CacheEntryAssociatedData::JavaScriptBytecode,
|
||||
CacheEntryAssociatedData::WebAssemblyCompiledCode,
|
||||
};
|
||||
constexpr inline Array CACHE_ENTRY_ASSOCIATED_DATA_TYPES { CacheEntryAssociatedData::JavaScriptBytecode };
|
||||
|
||||
u64 compute_maximum_disk_cache_size(u64 free_bytes, u64 limit_maximum_disk_cache_size = DEFAULT_MAXIMUM_DISK_CACHE_SIZE);
|
||||
u64 compute_maximum_disk_cache_entry_size(u64 maximum_disk_cache_size);
|
||||
|
|
|
|||
|
|
@ -328,7 +328,7 @@ ExceptionInstance* Store::get(ExceptionAddress address)
|
|||
return &m_exceptions[value];
|
||||
}
|
||||
|
||||
ErrorOr<void, ValidationError> AbstractMachine::validate(Module& module)
|
||||
ErrorOr<void, ValidationError> AbstractMachine::validate(Module& module, Optional<CompileCacheConfig> cache_config)
|
||||
{
|
||||
if (module.validation_status() != Module::ValidationStatus::Unchecked) {
|
||||
if (module.validation_status() == Module::ValidationStatus::Valid)
|
||||
|
|
@ -337,7 +337,10 @@ ErrorOr<void, ValidationError> AbstractMachine::validate(Module& module)
|
|||
return ValidationError { module.validation_error() };
|
||||
}
|
||||
|
||||
auto result = Validator {}.validate(module);
|
||||
Validator validator;
|
||||
if (cache_config.has_value())
|
||||
validator.set_cache_config(cache_config.release_value());
|
||||
auto result = validator.validate(module);
|
||||
if (result.is_error()) {
|
||||
module.set_validation_error(result.error().error_string);
|
||||
return result.release_error();
|
||||
|
|
|
|||
|
|
@ -780,7 +780,7 @@ public:
|
|||
explicit AbstractMachine() = default;
|
||||
|
||||
// Validate a module; permanently sets the module's validity status.
|
||||
ErrorOr<void, ValidationError> validate(Module&);
|
||||
ErrorOr<void, ValidationError> validate(Module&, Optional<CompileCacheConfig> cache_config = {});
|
||||
// Load and instantiate a module, and link it into this interpreter.
|
||||
InstantiationResult instantiate(Module const&, Vector<ExternValue>);
|
||||
Result invoke(FunctionAddress, Vector<Value>);
|
||||
|
|
|
|||
|
|
@ -269,7 +269,38 @@ ErrorOr<void, ValidationError> Validator::validate(TableSection const& section)
|
|||
|
||||
ErrorOr<void, ValidationError> Validator::validate(CodeSection const& section)
|
||||
{
|
||||
ScopeGuard flush_batch = [] { flush_cranelift_batch(); };
|
||||
bool installing = false;
|
||||
bool capturing = false;
|
||||
bool validation_succeeded = false;
|
||||
|
||||
if (m_cache_config.has_value()) {
|
||||
auto hash = ReadonlyBytes { m_cache_config->wasm_hash.data(), 32 };
|
||||
if (!m_cache_config->existing_blob.is_empty())
|
||||
installing = try_install_cranelift_cache_blob(hash, m_cache_config->existing_blob);
|
||||
if (!installing && m_cache_config->on_compiled) {
|
||||
begin_cranelift_cache_capture();
|
||||
capturing = true;
|
||||
}
|
||||
}
|
||||
|
||||
ScopeGuard cleanup = [&] {
|
||||
flush_cranelift_batch();
|
||||
if (installing)
|
||||
abort_cranelift_cache_install();
|
||||
if (capturing) {
|
||||
if (validation_succeeded) {
|
||||
auto hash = ReadonlyBytes { m_cache_config->wasm_hash.data(), 32 };
|
||||
if (auto blob = serialize_cranelift_cache_blob(hash); blob.has_value() && m_cache_config->on_compiled) {
|
||||
m_cache_config->on_compiled(blob.release_value());
|
||||
} else {
|
||||
abort_cranelift_cache_capture();
|
||||
}
|
||||
} else {
|
||||
abort_cranelift_cache_capture();
|
||||
}
|
||||
}
|
||||
set_cranelift_active_function_index(NumericLimits<u32>::max());
|
||||
};
|
||||
|
||||
size_t index = m_context.imported_function_count;
|
||||
for (auto& entry : section.functions()) {
|
||||
|
|
@ -291,6 +322,8 @@ ErrorOr<void, ValidationError> Validator::validate(CodeSection const& section)
|
|||
function_validator.m_frames.empend(function_type, FrameKind::Function, (size_t)0);
|
||||
function_validator.m_max_frame_size = max(function_validator.m_max_frame_size, function_validator.m_frames.size());
|
||||
|
||||
set_cranelift_active_function_index(static_cast<u32>(function_index));
|
||||
|
||||
auto results = TRY(function_validator.validate(function.body(), function_type.results()));
|
||||
if (results.result_types.size() != function_type.results().size())
|
||||
return Errors::invalid("function result"sv, function_type.results(), results.result_types);
|
||||
|
|
@ -309,6 +342,7 @@ ErrorOr<void, ValidationError> Validator::validate(CodeSection const& section)
|
|||
}
|
||||
}
|
||||
|
||||
validation_succeeded = true;
|
||||
return {};
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -62,6 +62,10 @@ public:
|
|||
return Validator { m_context };
|
||||
}
|
||||
|
||||
// Drives the Cranelift disk-cache hooks around validate(CodeSection). Set on the
|
||||
// top-level Validator (the one AbstractMachine creates); forks do not propagate it.
|
||||
void set_cache_config(CompileCacheConfig config) { m_cache_config = move(config); }
|
||||
|
||||
// Module
|
||||
ErrorOr<void, ValidationError> validate(Module&);
|
||||
ErrorOr<void, ValidationError> validate(ImportSection const&);
|
||||
|
|
@ -420,6 +424,7 @@ private:
|
|||
Vector<Frame, 16> m_frames;
|
||||
size_t m_max_frame_size { 0 };
|
||||
COWVector<GlobalType> m_globals_without_internal_globals;
|
||||
Optional<CompileCacheConfig> m_cache_config;
|
||||
};
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -40,6 +40,7 @@ struct InputHeader {
|
|||
u32 helpers_offset;
|
||||
u64 outcome_return;
|
||||
u64 code_region_start;
|
||||
u64 reloc_region_start;
|
||||
u64 total_size;
|
||||
};
|
||||
|
||||
|
|
@ -53,6 +54,13 @@ struct OutputFunctionEntry {
|
|||
u64 code_offset;
|
||||
u32 code_size;
|
||||
u32 compiled;
|
||||
// Offset (relative to the start of the reloc region) and count of `HelperReloc`
|
||||
// entries describing the absolute helper addresses baked into this function's code.
|
||||
// On cache install we walk these and rewrite the 8 bytes at code+code_offset+offset
|
||||
// with the live address of helper N for the current process.
|
||||
u64 reloc_offset;
|
||||
u32 reloc_count;
|
||||
u32 _pad;
|
||||
};
|
||||
|
||||
struct CodeMapping {
|
||||
|
|
@ -62,6 +70,8 @@ struct CodeMapping {
|
|||
|
||||
static constexpr size_t oop_code_region_min_size = 256 * KiB;
|
||||
static constexpr size_t oop_code_bytes_per_insn = 256;
|
||||
static constexpr size_t oop_reloc_region_min_size = 64 * KiB;
|
||||
static constexpr size_t oop_reloc_bytes_per_insn = 128;
|
||||
|
||||
static size_t align_up(size_t value, size_t alignment)
|
||||
{
|
||||
|
|
@ -73,9 +83,173 @@ static size_t align_up(size_t value, size_t alignment)
|
|||
struct BatchInput {
|
||||
Vector<CraneliftInsn> insns;
|
||||
u32 result_arity;
|
||||
u32 function_index;
|
||||
CompiledInstructions* target;
|
||||
};
|
||||
|
||||
// Disk-cache blob format. Stable: cached files name format_version + layout_hash so
|
||||
// any rebuild that changes those will simply miss the cache rather than try to
|
||||
// execute incompatible bytes.
|
||||
constexpr u64 cache_blob_magic = 0x4354494A4D534157ULL; // "WASMJITC" little-endian
|
||||
constexpr u32 cache_blob_format_version = 1;
|
||||
|
||||
struct CacheBlobHeader {
|
||||
u64 magic;
|
||||
u32 format_version;
|
||||
u32 helper_count;
|
||||
u64 layout_hash;
|
||||
u8 wasm_hash[32];
|
||||
u32 function_count;
|
||||
u32 _pad;
|
||||
};
|
||||
static_assert(sizeof(CacheBlobHeader) == 64);
|
||||
|
||||
struct CacheBlobFunctionEntry {
|
||||
u32 function_index;
|
||||
u32 code_size;
|
||||
u32 reloc_count;
|
||||
u32 _pad;
|
||||
};
|
||||
static_assert(sizeof(CacheBlobFunctionEntry) == 16);
|
||||
|
||||
struct CacheRecord {
|
||||
u32 function_index;
|
||||
ByteBuffer unpatched_code;
|
||||
Vector<HelperReloc> relocs;
|
||||
};
|
||||
|
||||
// On a cache miss we capture every successful compile so we can hand the blob to a
|
||||
// store callback after validation finishes. On a cache hit we populate the install
|
||||
// map up front; the per-function lookup happens inside try_cranelift_compile when
|
||||
// the dispatch table for that function has just been built and is ready to receive
|
||||
// a handler_ptr.
|
||||
struct CacheCaptureState {
|
||||
bool capturing { false };
|
||||
Vector<CacheRecord> records;
|
||||
};
|
||||
struct PendingInstallState {
|
||||
bool active { false };
|
||||
HashMap<u32, CacheRecord> records;
|
||||
};
|
||||
struct CacheState {
|
||||
CacheCaptureState cache_capture;
|
||||
PendingInstallState pending_install;
|
||||
Vector<BatchInput> pending_batch;
|
||||
};
|
||||
|
||||
static CacheState s_cranelift_cache_state;
|
||||
static thread_local u32 s_active_function_index = NumericLimits<u32>::max();
|
||||
|
||||
static u64 compute_layout_hash(RuntimeHelpers const& h)
|
||||
{
|
||||
auto fnv1a = [](u64 hash, u64 value) {
|
||||
for (int i = 0; i < 8; ++i) {
|
||||
hash ^= (value >> (i * 8)) & 0xff;
|
||||
hash *= 0x100000001b3ULL;
|
||||
}
|
||||
return hash;
|
||||
};
|
||||
u64 hash = 0xcbf29ce484222325ULL;
|
||||
hash = fnv1a(hash, h.regs_offset);
|
||||
hash = fnv1a(hash, h.value_size);
|
||||
hash = fnv1a(hash, h.locals_base_offset);
|
||||
hash = fnv1a(hash, h.default_memory_base_offset);
|
||||
hash = fnv1a(hash, h.compiled_call_result_scratch_offset);
|
||||
return hash;
|
||||
}
|
||||
|
||||
// `HelperId` values are assigned in lockstep with the field order of `RuntimeHelpers`,
|
||||
// so the helper address for id N is simply the N-th `size_t` field of the struct.
|
||||
static_assert(offsetof(RuntimeHelpers, call_function) == 0);
|
||||
static_assert(offsetof(RuntimeHelpers, memory_fill) == sizeof(size_t) * 30);
|
||||
static_assert(HELPER_COUNT == 31);
|
||||
|
||||
static bool apply_helper_relocs(u8* code_bytes, size_t code_size, HelperReloc const* relocs, size_t reloc_count, RuntimeHelpers const& helpers)
|
||||
{
|
||||
auto const* helper_table = reinterpret_cast<size_t const*>(&helpers);
|
||||
for (size_t i = 0; i < reloc_count; ++i) {
|
||||
auto const& r = relocs[i];
|
||||
if (r.helper_id >= HELPER_COUNT)
|
||||
return false;
|
||||
if (static_cast<size_t>(r.code_offset) + sizeof(u64) > code_size)
|
||||
return false;
|
||||
u64 addr = static_cast<u64>(helper_table[r.helper_id]) + static_cast<u64>(r.addend);
|
||||
__builtin_memcpy(code_bytes + r.code_offset, &addr, sizeof(addr));
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
// Allocate an RX-able page, copy the (still unpatched) machine code into it, apply the
|
||||
// helper-address patches, and install the resulting function pointer into `target`.
|
||||
// Used by both the fresh-compile path (bytes come from the subprocess shm) and the
|
||||
// cache-install path (bytes come from a `.wasmjit` blob).
|
||||
static bool install_compiled_function(CompiledInstructions& target, ReadonlyBytes code_bytes, HelperReloc const* relocs, size_t reloc_count, RuntimeHelpers const& helpers)
|
||||
{
|
||||
if (target.dispatches.is_empty())
|
||||
return false;
|
||||
|
||||
auto const code_size = code_bytes.size();
|
||||
if (code_size == 0)
|
||||
return false;
|
||||
|
||||
#if defined(AK_OS_WINDOWS)
|
||||
SYSTEM_INFO si;
|
||||
GetSystemInfo(&si);
|
||||
auto const page_size = static_cast<size_t>(si.dwPageSize);
|
||||
auto const rx_aligned_size = (code_size + page_size - 1) & ~(page_size - 1);
|
||||
auto* jit_mem = VirtualAlloc(nullptr, rx_aligned_size, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE);
|
||||
if (!jit_mem)
|
||||
return false;
|
||||
__builtin_memcpy(jit_mem, code_bytes.data(), code_size);
|
||||
if (!apply_helper_relocs(static_cast<u8*>(jit_mem), code_size, relocs, reloc_count, helpers)) {
|
||||
VirtualFree(jit_mem, 0, MEM_RELEASE);
|
||||
return false;
|
||||
}
|
||||
DWORD old_protect;
|
||||
VirtualProtect(jit_mem, rx_aligned_size, PAGE_EXECUTE_READ, &old_protect);
|
||||
FlushInstructionCache(GetCurrentProcess(), jit_mem, code_size);
|
||||
auto* func_ptr = static_cast<u8 const*>(jit_mem);
|
||||
auto* handle = new CodeMapping { jit_mem, rx_aligned_size };
|
||||
#elif defined(AK_OS_MACOS)
|
||||
auto const page_size = static_cast<size_t>(sysconf(_SC_PAGESIZE));
|
||||
auto const rx_aligned_size = (code_size + page_size - 1) & ~(page_size - 1);
|
||||
auto* jit_mapping = mmap(nullptr, rx_aligned_size, PROT_READ | PROT_WRITE | PROT_EXEC, MAP_PRIVATE | MAP_ANON | MAP_JIT, -1, 0);
|
||||
if (jit_mapping == MAP_FAILED)
|
||||
return false;
|
||||
|
||||
pthread_jit_write_protect_np(0);
|
||||
__builtin_memcpy(jit_mapping, code_bytes.data(), code_size);
|
||||
if (!apply_helper_relocs(static_cast<u8*>(jit_mapping), code_size, relocs, reloc_count, helpers)) {
|
||||
munmap(jit_mapping, rx_aligned_size);
|
||||
return false;
|
||||
}
|
||||
pthread_jit_write_protect_np(1);
|
||||
sys_icache_invalidate(jit_mapping, code_size);
|
||||
auto* func_ptr = static_cast<u8 const*>(jit_mapping);
|
||||
auto* handle = new CodeMapping { jit_mapping, rx_aligned_size };
|
||||
#else
|
||||
auto const page_size = static_cast<size_t>(sysconf(_SC_PAGESIZE));
|
||||
auto const rx_aligned_size = (code_size + page_size - 1) & ~(page_size - 1);
|
||||
auto* rw_mapping = mmap(nullptr, rx_aligned_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0);
|
||||
if (rw_mapping == MAP_FAILED)
|
||||
return false;
|
||||
__builtin_memcpy(rw_mapping, code_bytes.data(), code_size);
|
||||
if (!apply_helper_relocs(static_cast<u8*>(rw_mapping), code_size, relocs, reloc_count, helpers) || mprotect(rw_mapping, rx_aligned_size, PROT_READ | PROT_EXEC) != 0) {
|
||||
munmap(rw_mapping, rx_aligned_size);
|
||||
return false;
|
||||
}
|
||||
__builtin___clear_cache(static_cast<char*>(rw_mapping), static_cast<char*>(rw_mapping) + code_size);
|
||||
auto* func_ptr = static_cast<u8 const*>(rw_mapping);
|
||||
auto* handle = new CodeMapping { rw_mapping, rx_aligned_size };
|
||||
#endif
|
||||
|
||||
target.dispatches[0].handler_ptr = bit_cast<FlatPtr>(func_ptr);
|
||||
target.cranelift_code_handle = handle;
|
||||
target.cranelift_code_size = code_size;
|
||||
target.cranelift_compiled = true;
|
||||
return true;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// C helpers called by cranelift-generated code (need C linkage but not external visibility).
|
||||
|
|
@ -887,7 +1061,9 @@ static void try_cranelift_compile_batch(Vector<BatchInput>& batch)
|
|||
auto const helpers_offset = align_up(insn_region_offset + insn_bytes, alignof(RuntimeHelpers));
|
||||
auto const code_region_start = align_up(helpers_offset + sizeof(RuntimeHelpers), alignof(OutputFunctionEntry));
|
||||
auto const code_region_size = max(oop_code_region_min_size, total_insn_count * oop_code_bytes_per_insn);
|
||||
auto const total_size = code_region_start + sizeof(OutputFunctionEntry) * function_count + code_region_size;
|
||||
auto const reloc_region_start = align_up(code_region_start + sizeof(OutputFunctionEntry) * function_count + code_region_size, alignof(HelperReloc));
|
||||
auto const reloc_region_size = max(oop_reloc_region_min_size, total_insn_count * oop_reloc_bytes_per_insn);
|
||||
auto const total_size = reloc_region_start + reloc_region_size;
|
||||
|
||||
#if defined(AK_OS_WINDOWS)
|
||||
DWORD size_hi = static_cast<DWORD>(static_cast<u64>(total_size) >> 32);
|
||||
|
|
@ -947,6 +1123,7 @@ static void try_cranelift_compile_batch(Vector<BatchInput>& batch)
|
|||
.helpers_offset = static_cast<u32>(helpers_offset),
|
||||
.outcome_return = outcome_return,
|
||||
.code_region_start = code_region_start,
|
||||
.reloc_region_start = reloc_region_start,
|
||||
.total_size = total_size,
|
||||
};
|
||||
|
||||
|
|
@ -997,61 +1174,34 @@ static void try_cranelift_compile_batch(Vector<BatchInput>& batch)
|
|||
if (code_start + code_size > total_size)
|
||||
continue;
|
||||
|
||||
#if defined(AK_OS_WINDOWS)
|
||||
SYSTEM_INFO si;
|
||||
GetSystemInfo(&si);
|
||||
auto const page_size = static_cast<size_t>(si.dwPageSize);
|
||||
auto const rx_aligned_size = (code_size + page_size - 1) & ~(page_size - 1);
|
||||
auto* jit_mem = VirtualAlloc(nullptr, rx_aligned_size, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE);
|
||||
if (!jit_mem)
|
||||
continue;
|
||||
__builtin_memcpy(jit_mem, base + code_start, code_size);
|
||||
DWORD old_protect;
|
||||
VirtualProtect(jit_mem, rx_aligned_size, PAGE_EXECUTE_READ, &old_protect);
|
||||
FlushInstructionCache(GetCurrentProcess(), jit_mem, code_size);
|
||||
|
||||
auto* func_ptr = static_cast<u8 const*>(jit_mem);
|
||||
auto* handle = new CodeMapping { jit_mem, rx_aligned_size };
|
||||
#elif defined(AK_OS_MACOS)
|
||||
// We can't pull the map-as-rx/rw-across-processes trick on macos, so just do MAP_JIT with the typical jit mapping dance.
|
||||
auto const page_size = static_cast<size_t>(sysconf(_SC_PAGESIZE));
|
||||
auto const rx_aligned_size = (code_size + page_size - 1) & ~(page_size - 1);
|
||||
auto* jit_mapping = mmap(nullptr, rx_aligned_size, PROT_READ | PROT_WRITE | PROT_EXEC, MAP_PRIVATE | MAP_ANON | MAP_JIT, -1, 0);
|
||||
if (jit_mapping == MAP_FAILED)
|
||||
auto const reloc_offset = static_cast<size_t>(output->reloc_offset);
|
||||
auto const reloc_count = static_cast<size_t>(output->reloc_count);
|
||||
auto const reloc_bytes = reloc_count * sizeof(HelperReloc);
|
||||
if (reloc_region_start + reloc_offset + reloc_bytes > total_size)
|
||||
continue;
|
||||
|
||||
pthread_jit_write_protect_np(0);
|
||||
__builtin_memcpy(jit_mapping, base + code_start, code_size);
|
||||
pthread_jit_write_protect_np(1);
|
||||
sys_icache_invalidate(jit_mapping, code_size);
|
||||
auto code_bytes = ReadonlyBytes { base + code_start, code_size };
|
||||
auto const* relocs = reloc_count == 0
|
||||
? nullptr
|
||||
: reinterpret_cast<HelperReloc const*>(base + reloc_region_start + reloc_offset);
|
||||
|
||||
auto* func_ptr = static_cast<u8 const*>(jit_mapping);
|
||||
auto* handle = new CodeMapping { jit_mapping, rx_aligned_size };
|
||||
#else
|
||||
auto const page_size = static_cast<size_t>(sysconf(_SC_PAGESIZE));
|
||||
auto const page_aligned_offset = code_start & ~(page_size - 1);
|
||||
auto const offset_within_page = code_start - page_aligned_offset;
|
||||
auto const rx_map_size = offset_within_page + code_size;
|
||||
auto const rx_aligned_size = (rx_map_size + page_size - 1) & ~(page_size - 1);
|
||||
auto& capture = s_cranelift_cache_state.cache_capture;
|
||||
if (capture.capturing && batch[i].function_index != NumericLimits<u32>::max()) {
|
||||
if (auto copy = ByteBuffer::copy(code_bytes.data(), code_bytes.size()); !copy.is_error()) {
|
||||
CacheRecord rec;
|
||||
rec.function_index = batch[i].function_index;
|
||||
rec.unpatched_code = copy.release_value();
|
||||
rec.relocs.ensure_capacity(reloc_count);
|
||||
for (size_t j = 0; j < reloc_count; ++j)
|
||||
rec.relocs.unchecked_append(relocs[j]);
|
||||
capture.records.append(move(rec));
|
||||
}
|
||||
}
|
||||
|
||||
auto* rx_mapping = mmap(nullptr, rx_aligned_size, PROT_READ | PROT_EXEC, MAP_PRIVATE, fd, static_cast<off_t>(page_aligned_offset));
|
||||
if (rx_mapping == MAP_FAILED)
|
||||
continue;
|
||||
|
||||
auto* func_ptr = static_cast<u8 const*>(rx_mapping) + offset_within_page;
|
||||
auto* handle = new CodeMapping { rx_mapping, rx_aligned_size };
|
||||
#endif
|
||||
|
||||
auto& compiled = *batch[i].target;
|
||||
compiled.dispatches[0].handler_ptr = bit_cast<FlatPtr>(func_ptr);
|
||||
compiled.cranelift_code_handle = handle;
|
||||
compiled.cranelift_code_size = code_size;
|
||||
compiled.cranelift_compiled = true;
|
||||
install_compiled_function(*batch[i].target, code_bytes, relocs, reloc_count, helpers);
|
||||
}
|
||||
}
|
||||
|
||||
static Vector<BatchInput> s_pending_batch;
|
||||
|
||||
bool try_cranelift_compile(CompiledInstructions& compiled, u32 result_arity)
|
||||
{
|
||||
#if !WASM_COMPILED_FAULT_RECOVERY_SUPPORTED
|
||||
|
|
@ -1065,6 +1215,29 @@ bool try_cranelift_compile(CompiledInstructions& compiled, u32 result_arity)
|
|||
if (dispatches.is_empty())
|
||||
return false;
|
||||
|
||||
// Already installed (either from a prior compile or a previous cache install in this
|
||||
// same validation pass) -- nothing to do.
|
||||
if (compiled.cranelift_compiled)
|
||||
return true;
|
||||
|
||||
// Cache hit: install from the parsed blob instead of going through cranelift.
|
||||
// dispatches[] has just been populated by try_compile_instructions, so handler_ptr is ready to be set.
|
||||
if (s_cranelift_cache_state.pending_install.active && s_active_function_index != NumericLimits<u32>::max()) {
|
||||
auto record = s_cranelift_cache_state.pending_install.records.take(s_active_function_index);
|
||||
if (record.has_value()) {
|
||||
static auto cache_install_helpers = make_runtime_helpers();
|
||||
if (install_compiled_function(
|
||||
compiled,
|
||||
record->unpatched_code.bytes(),
|
||||
record->relocs.is_empty() ? nullptr : record->relocs.data(),
|
||||
record->relocs.size(), cache_install_helpers)) {
|
||||
return true;
|
||||
}
|
||||
// Put it back so we can try later.
|
||||
s_cranelift_cache_state.pending_install.records.set(s_active_function_index, record.release_value());
|
||||
}
|
||||
}
|
||||
|
||||
if constexpr (WASM_CRANELIFT_DEBUG) {
|
||||
// CRANELIFT_MAX_INSNS=N skip functions with more than N dispatches.
|
||||
// CRANELIFT_MIN_INSNS=N skip functions with fewer than N dispatches.
|
||||
|
|
@ -1177,17 +1350,17 @@ bool try_cranelift_compile(CompiledInstructions& compiled, u32 result_arity)
|
|||
}
|
||||
}
|
||||
|
||||
s_pending_batch.append({ move(flat), result_arity, &compiled });
|
||||
s_cranelift_cache_state.pending_batch.append({ move(flat), result_arity, s_active_function_index, &compiled });
|
||||
return false; // Not compiled yet, will be compiled in flush.
|
||||
#endif
|
||||
}
|
||||
|
||||
void flush_cranelift_batch()
|
||||
{
|
||||
if (s_pending_batch.is_empty())
|
||||
if (s_cranelift_cache_state.pending_batch.is_empty())
|
||||
return;
|
||||
try_cranelift_compile_batch(s_pending_batch);
|
||||
s_pending_batch.clear();
|
||||
try_cranelift_compile_batch(s_cranelift_cache_state.pending_batch);
|
||||
s_cranelift_cache_state.pending_batch.clear();
|
||||
}
|
||||
|
||||
void free_cranelift_code(void* handle)
|
||||
|
|
@ -1203,4 +1376,144 @@ void free_cranelift_code(void* handle)
|
|||
}
|
||||
}
|
||||
|
||||
void set_cranelift_active_function_index(u32 function_index)
|
||||
{
|
||||
s_active_function_index = function_index;
|
||||
}
|
||||
|
||||
void begin_cranelift_cache_capture()
|
||||
{
|
||||
s_cranelift_cache_state.cache_capture.capturing = true;
|
||||
s_cranelift_cache_state.cache_capture.records.clear();
|
||||
}
|
||||
|
||||
void abort_cranelift_cache_capture()
|
||||
{
|
||||
s_cranelift_cache_state.cache_capture.capturing = false;
|
||||
s_cranelift_cache_state.cache_capture.records.clear();
|
||||
}
|
||||
|
||||
void abort_cranelift_cache_install()
|
||||
{
|
||||
s_cranelift_cache_state.pending_install.active = false;
|
||||
s_cranelift_cache_state.pending_install.records.clear();
|
||||
}
|
||||
|
||||
Optional<ByteBuffer> serialize_cranelift_cache_blob(ReadonlyBytes wasm_hash)
|
||||
{
|
||||
ScopeGuard reset = [] {
|
||||
s_cranelift_cache_state.cache_capture.capturing = false;
|
||||
s_cranelift_cache_state.cache_capture.records.clear();
|
||||
};
|
||||
|
||||
auto const& capture = s_cranelift_cache_state.cache_capture;
|
||||
|
||||
if (!capture.capturing || capture.records.is_empty())
|
||||
return {};
|
||||
if (wasm_hash.size() != 32)
|
||||
return {};
|
||||
|
||||
static auto helpers = make_runtime_helpers();
|
||||
|
||||
size_t total_size = sizeof(CacheBlobHeader);
|
||||
for (auto const& r : capture.records) {
|
||||
total_size += sizeof(CacheBlobFunctionEntry);
|
||||
total_size += align_up(r.unpatched_code.size(), 16);
|
||||
total_size += r.relocs.size() * sizeof(HelperReloc);
|
||||
}
|
||||
|
||||
auto blob_or_error = ByteBuffer::create_zeroed(total_size);
|
||||
if (blob_or_error.is_error())
|
||||
return {};
|
||||
auto blob = blob_or_error.release_value();
|
||||
auto* out = blob.data();
|
||||
|
||||
auto* header = reinterpret_cast<CacheBlobHeader*>(out);
|
||||
header->magic = cache_blob_magic;
|
||||
header->format_version = cache_blob_format_version;
|
||||
header->helper_count = HELPER_COUNT;
|
||||
header->layout_hash = compute_layout_hash(helpers);
|
||||
__builtin_memcpy(header->wasm_hash, wasm_hash.data(), 32);
|
||||
header->function_count = static_cast<u32>(capture.records.size());
|
||||
|
||||
size_t offset = sizeof(CacheBlobHeader);
|
||||
for (auto const& r : capture.records) {
|
||||
auto* entry = reinterpret_cast<CacheBlobFunctionEntry*>(out + offset);
|
||||
entry->function_index = r.function_index;
|
||||
entry->code_size = static_cast<u32>(r.unpatched_code.size());
|
||||
entry->reloc_count = static_cast<u32>(r.relocs.size());
|
||||
offset += sizeof(CacheBlobFunctionEntry);
|
||||
|
||||
__builtin_memcpy(out + offset, r.unpatched_code.data(), r.unpatched_code.size());
|
||||
offset += align_up(r.unpatched_code.size(), 16);
|
||||
|
||||
auto reloc_bytes = r.relocs.size() * sizeof(HelperReloc);
|
||||
if (reloc_bytes > 0)
|
||||
__builtin_memcpy(out + offset, r.relocs.data(), reloc_bytes);
|
||||
offset += reloc_bytes;
|
||||
}
|
||||
|
||||
return blob;
|
||||
}
|
||||
|
||||
bool try_install_cranelift_cache_blob(ReadonlyBytes expected_wasm_hash, ReadonlyBytes blob)
|
||||
{
|
||||
abort_cranelift_cache_install();
|
||||
|
||||
if (expected_wasm_hash.size() != 32 || blob.size() < sizeof(CacheBlobHeader))
|
||||
return false;
|
||||
|
||||
auto const* header = reinterpret_cast<CacheBlobHeader const*>(blob.data());
|
||||
if (header->magic != cache_blob_magic)
|
||||
return false;
|
||||
if (header->format_version != cache_blob_format_version)
|
||||
return false;
|
||||
if (header->helper_count != HELPER_COUNT)
|
||||
return false;
|
||||
if (__builtin_memcmp(header->wasm_hash, expected_wasm_hash.data(), 32) != 0)
|
||||
return false;
|
||||
|
||||
static auto helpers = make_runtime_helpers();
|
||||
if (header->layout_hash != compute_layout_hash(helpers))
|
||||
return false;
|
||||
|
||||
size_t offset = sizeof(CacheBlobHeader);
|
||||
for (u32 i = 0; i < header->function_count; ++i) {
|
||||
if (offset + sizeof(CacheBlobFunctionEntry) > blob.size())
|
||||
return false;
|
||||
auto const* entry = reinterpret_cast<CacheBlobFunctionEntry const*>(blob.data() + offset);
|
||||
offset += sizeof(CacheBlobFunctionEntry);
|
||||
|
||||
auto code_off = offset;
|
||||
auto aligned_code_size = align_up(entry->code_size, 16);
|
||||
if (code_off + aligned_code_size > blob.size())
|
||||
return false;
|
||||
offset += aligned_code_size;
|
||||
|
||||
auto reloc_off = offset;
|
||||
auto reloc_bytes = static_cast<size_t>(entry->reloc_count) * sizeof(HelperReloc);
|
||||
if (reloc_off + reloc_bytes > blob.size())
|
||||
return false;
|
||||
offset += reloc_bytes;
|
||||
|
||||
auto code_copy = ByteBuffer::copy(blob.data() + code_off, entry->code_size);
|
||||
if (code_copy.is_error())
|
||||
return false;
|
||||
|
||||
CacheRecord rec;
|
||||
rec.function_index = entry->function_index;
|
||||
rec.unpatched_code = code_copy.release_value();
|
||||
rec.relocs.ensure_capacity(entry->reloc_count);
|
||||
for (u32 j = 0; j < entry->reloc_count; ++j) {
|
||||
HelperReloc reloc;
|
||||
__builtin_memcpy(&reloc, blob.data() + reloc_off + j * sizeof(HelperReloc), sizeof(HelperReloc));
|
||||
rec.relocs.unchecked_append(reloc);
|
||||
}
|
||||
s_cranelift_cache_state.pending_install.records.set(entry->function_index, move(rec));
|
||||
}
|
||||
|
||||
s_cranelift_cache_state.pending_install.active = true;
|
||||
return true;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -11,5 +11,11 @@ namespace Wasm {
|
|||
bool try_cranelift_compile(CompiledInstructions&, u32) { return false; }
|
||||
void flush_cranelift_batch() { }
|
||||
void free_cranelift_code(void*) { }
|
||||
void set_cranelift_active_function_index(u32) { }
|
||||
void begin_cranelift_cache_capture() { }
|
||||
void abort_cranelift_cache_capture() { }
|
||||
void abort_cranelift_cache_install() { }
|
||||
Optional<ByteBuffer> serialize_cranelift_cache_blob(ReadonlyBytes) { return {}; }
|
||||
bool try_install_cranelift_cache_blob(ReadonlyBytes, ReadonlyBytes) { return false; }
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ sys_includes = ["stdint.h", "stddef.h"]
|
|||
usize_is_size_t = true
|
||||
|
||||
[export]
|
||||
include = ["CraneliftInsn", "RuntimeHelpers"]
|
||||
include = ["CraneliftInsn", "RuntimeHelpers", "HelperReloc", "HelperId", "HELPER_COUNT"]
|
||||
|
||||
[export.mangle]
|
||||
rename_types = "PascalCase"
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@
|
|||
|
||||
#![allow(clippy::manual_let_else)]
|
||||
|
||||
use libwasm_cranelift::{CraneliftInsn, RuntimeHelpers, compile_to_bytes};
|
||||
use libwasm_cranelift::{CompiledFunction, CraneliftInsn, HelperReloc, RuntimeHelpers, compile_to_bytes};
|
||||
use std::env;
|
||||
use std::mem::{size_of, size_of_val};
|
||||
|
||||
|
|
@ -26,6 +26,7 @@ struct InputHeader {
|
|||
helpers_offset: u32,
|
||||
outcome_return: u64,
|
||||
code_region_start: u64,
|
||||
reloc_region_start: u64,
|
||||
total_size: u64,
|
||||
}
|
||||
|
||||
|
|
@ -43,6 +44,9 @@ struct OutputFunctionEntry {
|
|||
code_offset: u64,
|
||||
code_size: u32,
|
||||
compiled: u32,
|
||||
reloc_offset: u64,
|
||||
reloc_count: u32,
|
||||
_pad: u32,
|
||||
}
|
||||
|
||||
fn as_bytes_slice<T>(value: &[T]) -> &[u8] {
|
||||
|
|
@ -230,7 +234,11 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
|
|||
|
||||
let out_entries_offset = code_region_start;
|
||||
let code_base_offset = out_entries_offset + func_count * size_of::<OutputFunctionEntry>();
|
||||
let code_capacity = mapped.len().checked_sub(code_base_offset).ok_or("bad code region")?;
|
||||
let reloc_region_start = usize::try_from(header.reloc_region_start).map_err(|_| "reloc_region_start overflow")?;
|
||||
let code_capacity = reloc_region_start
|
||||
.checked_sub(code_base_offset)
|
||||
.ok_or("bad code region")?;
|
||||
let reloc_capacity = mapped.len().checked_sub(reloc_region_start).ok_or("bad reloc region")?;
|
||||
|
||||
let mut entries: Vec<InputFunctionEntry> = Vec::with_capacity(func_count);
|
||||
for i in 0..func_count {
|
||||
|
|
@ -247,7 +255,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
|
|||
let helpers_ref = &helpers;
|
||||
let outcome_return = header.outcome_return;
|
||||
|
||||
let compiled_chunks: Vec<Vec<(usize, Vec<u8>)>> = std::thread::scope(|scope| {
|
||||
let compiled_chunks: Vec<Vec<(usize, CompiledFunction)>> = std::thread::scope(|scope| {
|
||||
let mut handles = Vec::with_capacity(thread_count);
|
||||
for chunk_idx in 0..thread_count {
|
||||
let start = chunk_idx * chunk_size;
|
||||
|
|
@ -257,7 +265,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
|
|||
let end = (start + chunk_size).min(func_count);
|
||||
let chunk_entries = &entries[start..end];
|
||||
handles.push(scope.spawn(move || {
|
||||
let mut out: Vec<(usize, Vec<u8>)> = Vec::with_capacity(end - start);
|
||||
let mut out: Vec<(usize, CompiledFunction)> = Vec::with_capacity(end - start);
|
||||
for (offset_in_chunk, entry) in chunk_entries.iter().enumerate() {
|
||||
let i = start + offset_in_chunk;
|
||||
if entry.insn_count == 0 {
|
||||
|
|
@ -278,8 +286,8 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
|
|||
};
|
||||
let insns =
|
||||
unsafe { std::slice::from_raw_parts(insn_bytes.as_ptr().cast::<CraneliftInsn>(), insn_count) };
|
||||
if let Ok(code) = compile_to_bytes(insns, helpers_ref, outcome_return, entry.result_arity) {
|
||||
out.push((i, code));
|
||||
if let Ok(compiled) = compile_to_bytes(insns, helpers_ref, outcome_return, entry.result_arity) {
|
||||
out.push((i, compiled));
|
||||
}
|
||||
}
|
||||
out
|
||||
|
|
@ -289,26 +297,43 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
|
|||
});
|
||||
|
||||
let mut code_cursor = 0usize;
|
||||
let mut reloc_cursor = 0usize;
|
||||
for chunk in compiled_chunks {
|
||||
for (i, code) in chunk {
|
||||
for (i, compiled) in chunk {
|
||||
let code = compiled.code;
|
||||
let relocs = compiled.relocs;
|
||||
let aligned = (code.len() + 15) & !15;
|
||||
let reloc_bytes_len = relocs.len() * size_of::<HelperReloc>();
|
||||
if code_cursor + aligned > code_capacity {
|
||||
continue;
|
||||
}
|
||||
if reloc_cursor + reloc_bytes_len > reloc_capacity {
|
||||
continue;
|
||||
}
|
||||
let code_offset = code_cursor;
|
||||
let code_dst = code_base_offset + code_offset;
|
||||
mapped[code_dst..code_dst + code.len()].copy_from_slice(&code);
|
||||
|
||||
let reloc_offset = reloc_cursor;
|
||||
if !relocs.is_empty() {
|
||||
let reloc_dst = reloc_region_start + reloc_offset;
|
||||
mapped[reloc_dst..reloc_dst + reloc_bytes_len].copy_from_slice(as_bytes_slice(&relocs));
|
||||
}
|
||||
|
||||
let entry = OutputFunctionEntry {
|
||||
code_offset: u64::try_from(code_offset).map_err(|_| "code offset overflow")?,
|
||||
code_size: u32::try_from(code.len()).map_err(|_| "code size overflow")?,
|
||||
compiled: 1,
|
||||
reloc_offset: u64::try_from(reloc_offset).map_err(|_| "reloc offset overflow")?,
|
||||
reloc_count: u32::try_from(relocs.len()).map_err(|_| "reloc count overflow")?,
|
||||
_pad: 0,
|
||||
};
|
||||
let entry_dst = out_entries_offset + i * size_of::<OutputFunctionEntry>();
|
||||
let entry_bytes = as_bytes_slice(std::slice::from_ref(&entry));
|
||||
mapped[entry_dst..entry_dst + size_of::<OutputFunctionEntry>()].copy_from_slice(entry_bytes);
|
||||
|
||||
code_cursor += aligned;
|
||||
reloc_cursor += reloc_bytes_len;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -324,13 +349,26 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
|
|||
&mapped[code_base_offset..code_base_offset + code_cursor],
|
||||
u64::try_from(code_base_offset)?,
|
||||
)?;
|
||||
if reloc_cursor > 0 {
|
||||
write_all_at_offset(
|
||||
&file,
|
||||
&mapped[reloc_region_start..reloc_region_start + reloc_cursor],
|
||||
u64::try_from(reloc_region_start)?,
|
||||
)?;
|
||||
}
|
||||
let _ = file.sync_all();
|
||||
}
|
||||
// On macOS we mmap'd the parent's shm fd with MAP_SHARED, so the writes
|
||||
// above are already visible in the parent's mapping. Nothing to flush.
|
||||
#[cfg(target_os = "macos")]
|
||||
{
|
||||
let _ = (out_entries_offset, code_base_offset, code_cursor);
|
||||
let _ = (
|
||||
out_entries_offset,
|
||||
code_base_offset,
|
||||
code_cursor,
|
||||
reloc_region_start,
|
||||
reloc_cursor,
|
||||
);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
|
|
|
|||
|
|
@ -6,16 +6,21 @@
|
|||
|
||||
use crate::{CraneliftInsn, RuntimeHelpers};
|
||||
|
||||
use cranelift_codegen::FinalizedRelocTarget;
|
||||
use cranelift_codegen::binemit::Reloc;
|
||||
use cranelift_codegen::ir::condcodes::{FloatCC, IntCC};
|
||||
use cranelift_codegen::ir::types;
|
||||
use cranelift_codegen::ir::{
|
||||
AbiParam, Function, InstBuilder, MemFlags, Signature, StackSlotData, StackSlotKind, UserFuncName,
|
||||
AbiParam, ExtFuncData, ExternalName, Function, InstBuilder, MemFlags, Signature, StackSlotData, StackSlotKind,
|
||||
UserExternalName, UserFuncName,
|
||||
};
|
||||
use cranelift_codegen::settings::{self, Configurable};
|
||||
use cranelift_codegen::{self, Context};
|
||||
use cranelift_frontend::{FunctionBuilder, FunctionBuilderContext, Variable};
|
||||
use cranelift_native;
|
||||
|
||||
use crate::{CompiledFunction, HelperId, HelperReloc};
|
||||
|
||||
// Opcode constants generated from Opcode.h (see build.rs.)
|
||||
#[allow(dead_code)]
|
||||
mod op {
|
||||
|
|
@ -61,7 +66,7 @@ impl CraneliftCompiler {
|
|||
helpers: &RuntimeHelpers,
|
||||
outcome_return_value: u64,
|
||||
result_arity: u32,
|
||||
) -> Result<Vec<u8>, &'static str> {
|
||||
) -> Result<CompiledFunction, &'static str> {
|
||||
for insn in insns {
|
||||
if !Self::is_supported(insn) {
|
||||
return Err("unsupported instruction");
|
||||
|
|
@ -179,43 +184,54 @@ impl CraneliftCompiler {
|
|||
callrec_write_sig: void fn(ptr, i32, i64);
|
||||
}
|
||||
|
||||
// Helper function pointers, rematerialized as iconst at each use site.
|
||||
macro_rules! h {
|
||||
($($name:ident = helpers.$field:ident;)*) => { $(let $name = helpers.$field as i64;)* };
|
||||
}
|
||||
h! {
|
||||
h_call_fn = helpers.call_function;
|
||||
h_direct_call_0 = helpers.direct_call_0;
|
||||
h_direct_call_1 = helpers.direct_call_1;
|
||||
h_direct_call_2 = helpers.direct_call_2;
|
||||
h_direct_call_3 = helpers.direct_call_3;
|
||||
h_set_trap = helpers.set_trap;
|
||||
h_mem_load8_s = helpers.memory_load8_s;
|
||||
h_mem_load8_u = helpers.memory_load8_u;
|
||||
h_mem_load16_s = helpers.memory_load16_s;
|
||||
h_mem_load16_u = helpers.memory_load16_u;
|
||||
h_mem_load32_s = helpers.memory_load32_s;
|
||||
h_mem_load32_u = helpers.memory_load32_u;
|
||||
h_mem_load64 = helpers.memory_load64;
|
||||
h_mem_store8 = helpers.memory_store8;
|
||||
h_mem_store16 = helpers.memory_store16;
|
||||
h_mem_store32 = helpers.memory_store32;
|
||||
h_mem_store64 = helpers.memory_store64;
|
||||
h_mem_size = helpers.memory_size;
|
||||
h_mem_grow = helpers.memory_grow;
|
||||
h_read_global = helpers.read_global;
|
||||
h_write_global = helpers.write_global;
|
||||
h_stack_push = helpers.stack_push;
|
||||
h_stack_pop = helpers.stack_pop;
|
||||
h_stack_size = helpers.stack_size;
|
||||
h_stack_cleanup = helpers.stack_cleanup;
|
||||
h_callrec_read = helpers.callrec_read;
|
||||
h_callrec_write = helpers.callrec_write;
|
||||
h_call_wr = helpers.call_with_record;
|
||||
h_call_indirect = helpers.call_indirect;
|
||||
h_memory_copy = helpers.memory_copy;
|
||||
h_memory_fill = helpers.memory_fill;
|
||||
// Declare each runtime helper as an imported external function. At every use site
|
||||
// we emit `func_addr` which lowers (with is_pic=false) to a load from an inline
|
||||
// 8-byte literal marked with a `Reloc::Abs8` relocation -- the cache install path
|
||||
// walks these and rewrites the 8 bytes with the current process's helper address.
|
||||
macro_rules! decl_helper {
|
||||
($sig:expr, $id:expr) => {{
|
||||
let user_ref = builder.func.declare_imported_user_function(UserExternalName {
|
||||
namespace: 0,
|
||||
index: $id as u32,
|
||||
});
|
||||
builder.func.import_function(ExtFuncData {
|
||||
name: ExternalName::user(user_ref),
|
||||
signature: $sig,
|
||||
colocated: false,
|
||||
})
|
||||
}};
|
||||
}
|
||||
let h_call_fn = decl_helper!(call_fn_sig, HelperId::call_function);
|
||||
let h_direct_call_0 = decl_helper!(call_fn_sig, HelperId::direct_call_0);
|
||||
let h_direct_call_1 = decl_helper!(call_fn1_sig, HelperId::direct_call_1);
|
||||
let h_direct_call_2 = decl_helper!(call_fn2_sig, HelperId::direct_call_2);
|
||||
let h_direct_call_3 = decl_helper!(call_fn3_sig, HelperId::direct_call_3);
|
||||
let h_set_trap = decl_helper!(set_trap_sig, HelperId::set_trap);
|
||||
let h_mem_load8_s = decl_helper!(mem_load_sig, HelperId::memory_load8_s);
|
||||
let h_mem_load8_u = decl_helper!(mem_load_sig, HelperId::memory_load8_u);
|
||||
let h_mem_load16_s = decl_helper!(mem_load_sig, HelperId::memory_load16_s);
|
||||
let h_mem_load16_u = decl_helper!(mem_load_sig, HelperId::memory_load16_u);
|
||||
let h_mem_load32_s = decl_helper!(mem_load_sig, HelperId::memory_load32_s);
|
||||
let h_mem_load32_u = decl_helper!(mem_load_sig, HelperId::memory_load32_u);
|
||||
let h_mem_load64 = decl_helper!(mem_load_sig, HelperId::memory_load64);
|
||||
let h_mem_store8 = decl_helper!(mem_store_sig, HelperId::memory_store8);
|
||||
let h_mem_store16 = decl_helper!(mem_store_sig, HelperId::memory_store16);
|
||||
let h_mem_store32 = decl_helper!(mem_store_sig, HelperId::memory_store32);
|
||||
let h_mem_store64 = decl_helper!(mem_store_sig, HelperId::memory_store64);
|
||||
let h_mem_size = decl_helper!(mem_size_sig, HelperId::memory_size);
|
||||
let h_mem_grow = decl_helper!(mem_grow_sig, HelperId::memory_grow);
|
||||
let h_read_global = decl_helper!(read_global_sig, HelperId::read_global);
|
||||
let h_write_global = decl_helper!(write_global_sig, HelperId::write_global);
|
||||
let h_stack_push = decl_helper!(stack_push_sig, HelperId::stack_push);
|
||||
let h_stack_pop = decl_helper!(stack_pop_sig, HelperId::stack_pop);
|
||||
let h_stack_size = decl_helper!(stack_size_sig, HelperId::stack_size);
|
||||
let h_stack_cleanup = decl_helper!(stack_cleanup_sig, HelperId::stack_cleanup);
|
||||
let h_callrec_read = decl_helper!(callrec_read_sig, HelperId::callrec_read);
|
||||
let h_callrec_write = decl_helper!(callrec_write_sig, HelperId::callrec_write);
|
||||
let h_call_wr = decl_helper!(call_wr_sig, HelperId::call_with_record);
|
||||
let h_call_indirect = decl_helper!(call_indirect_sig, HelperId::call_indirect);
|
||||
let h_memory_copy = decl_helper!(memory_copy_sig, HelperId::memory_copy);
|
||||
let h_memory_fill = decl_helper!(memory_fill_sig, HelperId::memory_fill);
|
||||
let locals_base_offset = helpers.locals_base_offset as i32;
|
||||
let default_memory_base_offset = helpers.default_memory_base_offset as i32;
|
||||
let compiled_call_result_scratch_offset = helpers.compiled_call_result_scratch_offset as i32;
|
||||
|
|
@ -273,7 +289,7 @@ impl CraneliftCompiler {
|
|||
let mut next_var_id: u32 = VSTACK_VAR_BASE + max_stack_depth as u32 + 1;
|
||||
|
||||
if has_raw_call {
|
||||
let stack_size_fp = builder.ins().iconst(ptr_type, h_stack_size);
|
||||
let stack_size_fp = builder.ins().func_addr(ptr_type, h_stack_size);
|
||||
let cfg_for_size = builder.use_var(config_var);
|
||||
let stack_size_call = builder
|
||||
.ins()
|
||||
|
|
@ -297,13 +313,13 @@ impl CraneliftCompiler {
|
|||
sp -= 1;
|
||||
$builder.use_var(stack_vars[sp])
|
||||
} else {
|
||||
let fp = $builder.ins().iconst(ptr_type, h_stack_pop);
|
||||
let fp = $builder.ins().func_addr(ptr_type, h_stack_pop);
|
||||
let cfg = $builder.use_var(config_var);
|
||||
let call = $builder.ins().call_indirect(stack_pop_sig, fp, &[cfg]);
|
||||
$builder.inst_results(call)[0]
|
||||
}
|
||||
} else {
|
||||
let fp = $builder.ins().iconst(ptr_type, h_callrec_read);
|
||||
let fp = $builder.ins().func_addr(ptr_type, h_callrec_read);
|
||||
let cfg = $builder.use_var(config_var);
|
||||
let idx = $builder.ins().iconst(types::I32, i64::from(src - CALLREC_BASE));
|
||||
let call = $builder.ins().call_indirect(callrec_read_sig, fp, &[cfg, idx]);
|
||||
|
|
@ -318,7 +334,7 @@ impl CraneliftCompiler {
|
|||
if max_stack_depth > 0 {
|
||||
for i in 0..sp {
|
||||
let val = $builder.use_var(stack_vars[i]);
|
||||
let fp = $builder.ins().iconst(ptr_type, h_stack_push);
|
||||
let fp = $builder.ins().func_addr(ptr_type, h_stack_push);
|
||||
let cfg = $builder.use_var(config_var);
|
||||
$builder.ins().call_indirect(stack_push_sig, fp, &[cfg, val]);
|
||||
}
|
||||
|
|
@ -335,7 +351,7 @@ impl CraneliftCompiler {
|
|||
for i in 0..n {
|
||||
let idx = sp - n + i;
|
||||
let val = $builder.use_var(stack_vars[idx]);
|
||||
let fp = $builder.ins().iconst(ptr_type, h_stack_push);
|
||||
let fp = $builder.ins().func_addr(ptr_type, h_stack_push);
|
||||
let cfg = $builder.use_var(config_var);
|
||||
$builder.ins().call_indirect(stack_push_sig, fp, &[cfg, val]);
|
||||
}
|
||||
|
|
@ -355,12 +371,12 @@ impl CraneliftCompiler {
|
|||
$builder.def_var(stack_vars[sp], val);
|
||||
sp += 1;
|
||||
} else {
|
||||
let fp = $builder.ins().iconst(ptr_type, h_stack_push);
|
||||
let fp = $builder.ins().func_addr(ptr_type, h_stack_push);
|
||||
let cfg = $builder.use_var(config_var);
|
||||
$builder.ins().call_indirect(stack_push_sig, fp, &[cfg, val]);
|
||||
}
|
||||
} else {
|
||||
let fp = $builder.ins().iconst(ptr_type, h_callrec_write);
|
||||
let fp = $builder.ins().func_addr(ptr_type, h_callrec_write);
|
||||
let cfg = $builder.use_var(config_var);
|
||||
let idx = $builder.ins().iconst(types::I32, i64::from(dst - CALLREC_BASE));
|
||||
$builder
|
||||
|
|
@ -564,7 +580,7 @@ impl CraneliftCompiler {
|
|||
}
|
||||
let msg_ptr = builder.ins().stack_addr(ptr_type, ss, 0);
|
||||
let msg_len = builder.ins().iconst(types::I32, msg.len() as i64);
|
||||
let st_ptr = builder.ins().iconst(ptr_type, h_set_trap);
|
||||
let st_ptr = builder.ins().func_addr(ptr_type, h_set_trap);
|
||||
let interp = builder.use_var(interp_var);
|
||||
builder
|
||||
.ins()
|
||||
|
|
@ -584,7 +600,7 @@ impl CraneliftCompiler {
|
|||
let var = Variable::from_u32(next_var_id);
|
||||
next_var_id += 1;
|
||||
builder.declare_var(var, types::I64);
|
||||
let stack_size_fp = builder.ins().iconst(ptr_type, h_stack_size);
|
||||
let stack_size_fp = builder.ins().func_addr(ptr_type, h_stack_size);
|
||||
let cfg = builder.use_var(config_var);
|
||||
let call = builder.ins().call_indirect(stack_size_sig, stack_size_fp, &[cfg]);
|
||||
let cur = builder.inst_results(call)[0];
|
||||
|
|
@ -614,7 +630,7 @@ impl CraneliftCompiler {
|
|||
let var = Variable::from_u32(next_var_id);
|
||||
next_var_id += 1;
|
||||
builder.declare_var(var, types::I64);
|
||||
let stack_size_fp = builder.ins().iconst(ptr_type, h_stack_size);
|
||||
let stack_size_fp = builder.ins().func_addr(ptr_type, h_stack_size);
|
||||
let cfg = builder.use_var(config_var);
|
||||
let call = builder.ins().call_indirect(stack_size_sig, stack_size_fp, &[cfg]);
|
||||
let cur = builder.inst_results(call)[0];
|
||||
|
|
@ -651,7 +667,7 @@ impl CraneliftCompiler {
|
|||
let var = Variable::from_u32(next_var_id);
|
||||
next_var_id += 1;
|
||||
builder.declare_var(var, types::I64);
|
||||
let stack_size_fp = builder.ins().iconst(ptr_type, h_stack_size);
|
||||
let stack_size_fp = builder.ins().func_addr(ptr_type, h_stack_size);
|
||||
let cfg = builder.use_var(config_var);
|
||||
let call = builder.ins().call_indirect(stack_size_sig, stack_size_fp, &[cfg]);
|
||||
let cur = builder.inst_results(call)[0];
|
||||
|
|
@ -745,7 +761,7 @@ impl CraneliftCompiler {
|
|||
let result = if sp > 0 {
|
||||
builder.use_var(stack_vars[sp - 1])
|
||||
} else {
|
||||
let fp = builder.ins().iconst(ptr_type, h_stack_pop);
|
||||
let fp = builder.ins().func_addr(ptr_type, h_stack_pop);
|
||||
let cfg = builder.use_var(config_var);
|
||||
let call = builder.ins().call_indirect(stack_pop_sig, fp, &[cfg]);
|
||||
builder.inst_results(call)[0]
|
||||
|
|
@ -760,7 +776,7 @@ impl CraneliftCompiler {
|
|||
let target_size = builder.use_var(entry_depth_var);
|
||||
let arity_val = builder.ins().iconst(types::I32, arity as i64);
|
||||
let cfg = builder.use_var(config_var);
|
||||
let cleanup_fp = builder.ins().iconst(ptr_type, h_stack_cleanup);
|
||||
let cleanup_fp = builder.ins().func_addr(ptr_type, h_stack_cleanup);
|
||||
builder
|
||||
.ins()
|
||||
.call_indirect(stack_cleanup_sig, cleanup_fp, &[cfg, target_size, arity_val]);
|
||||
|
|
@ -808,7 +824,7 @@ impl CraneliftCompiler {
|
|||
let target_size = builder.use_var(entry_depth_var);
|
||||
let arity_val = builder.ins().iconst(types::I32, arity as i64);
|
||||
let cfg = builder.use_var(config_var);
|
||||
let cleanup_fp = builder.ins().iconst(ptr_type, h_stack_cleanup);
|
||||
let cleanup_fp = builder.ins().func_addr(ptr_type, h_stack_cleanup);
|
||||
builder
|
||||
.ins()
|
||||
.call_indirect(stack_cleanup_sig, cleanup_fp, &[cfg, target_size, arity_val]);
|
||||
|
|
@ -902,7 +918,7 @@ impl CraneliftCompiler {
|
|||
op::GLOBAL_GET => {
|
||||
let idx = builder.ins().iconst(types::I32, insn.imm1);
|
||||
let _uv_config_var = builder.use_var(config_var);
|
||||
let _ic_0 = builder.ins().iconst(ptr_type, h_read_global);
|
||||
let _ic_0 = builder.ins().func_addr(ptr_type, h_read_global);
|
||||
let call = builder
|
||||
.ins()
|
||||
.call_indirect(read_global_sig, _ic_0, &[_uv_config_var, idx]);
|
||||
|
|
@ -913,7 +929,7 @@ impl CraneliftCompiler {
|
|||
let val = read_src!(builder, insn.sources[0]);
|
||||
let idx = builder.ins().iconst(types::I32, insn.imm1);
|
||||
let _uv_config_var = builder.use_var(config_var);
|
||||
let _ic_0 = builder.ins().iconst(ptr_type, h_write_global);
|
||||
let _ic_0 = builder.ins().func_addr(ptr_type, h_write_global);
|
||||
builder
|
||||
.ins()
|
||||
.call_indirect(write_global_sig, _ic_0, &[_uv_config_var, idx, val]);
|
||||
|
|
@ -977,7 +993,7 @@ impl CraneliftCompiler {
|
|||
let result = if sp > 0 {
|
||||
builder.use_var(stack_vars[sp - 1])
|
||||
} else {
|
||||
let fp = builder.ins().iconst(ptr_type, h_stack_pop);
|
||||
let fp = builder.ins().func_addr(ptr_type, h_stack_pop);
|
||||
let cfg = builder.use_var(config_var);
|
||||
let call = builder.ins().call_indirect(stack_pop_sig, fp, &[cfg]);
|
||||
builder.inst_results(call)[0]
|
||||
|
|
@ -990,7 +1006,7 @@ impl CraneliftCompiler {
|
|||
let target_size = builder.use_var(entry_depth_var);
|
||||
let arity_val = builder.ins().iconst(types::I32, arity as i64);
|
||||
let cfg = builder.use_var(config_var);
|
||||
let cleanup_fp = builder.ins().iconst(ptr_type, h_stack_cleanup);
|
||||
let cleanup_fp = builder.ins().func_addr(ptr_type, h_stack_cleanup);
|
||||
builder.ins().call_indirect(
|
||||
stack_cleanup_sig,
|
||||
cleanup_fp,
|
||||
|
|
@ -1447,7 +1463,7 @@ impl CraneliftCompiler {
|
|||
} else {
|
||||
let mem_idx = builder.ins().iconst(types::I32, i64::from(insn.imm3 & 0x7fff_ffff));
|
||||
let _xv_config_var = builder.use_var(config_var);
|
||||
let _xc_0 = builder.ins().iconst(ptr_type, memory_load_helper);
|
||||
let _xc_0 = builder.ins().func_addr(ptr_type, memory_load_helper);
|
||||
let call = builder
|
||||
.ins()
|
||||
.call_indirect(mem_load_sig, _xc_0, &[_xv_config_var, mem_idx, addr]);
|
||||
|
|
@ -1510,7 +1526,7 @@ impl CraneliftCompiler {
|
|||
} else {
|
||||
let mem_idx = builder.ins().iconst(types::I32, i64::from(insn.imm3 & 0x7fff_ffff));
|
||||
let _xv_config_var = builder.use_var(config_var);
|
||||
let _xc_0 = builder.ins().iconst(ptr_type, memory_store_helper);
|
||||
let _xc_0 = builder.ins().func_addr(ptr_type, memory_store_helper);
|
||||
let call =
|
||||
builder
|
||||
.ins()
|
||||
|
|
@ -1527,7 +1543,7 @@ impl CraneliftCompiler {
|
|||
op::MEMORY_SIZE => {
|
||||
let mem_idx = builder.ins().iconst(types::I32, insn.imm1);
|
||||
let _xv_config_var = builder.use_var(config_var);
|
||||
let _xc_0 = builder.ins().iconst(ptr_type, h_mem_size);
|
||||
let _xc_0 = builder.ins().func_addr(ptr_type, h_mem_size);
|
||||
let call = builder
|
||||
.ins()
|
||||
.call_indirect(mem_size_sig, _xc_0, &[_xv_config_var, mem_idx]);
|
||||
|
|
@ -1540,7 +1556,7 @@ impl CraneliftCompiler {
|
|||
let pages_i32 = builder.ins().ireduce(types::I32, pages);
|
||||
let mem_idx = builder.ins().iconst(types::I32, insn.imm1);
|
||||
let _xv_config_var = builder.use_var(config_var);
|
||||
let _xc_0 = builder.ins().iconst(ptr_type, h_mem_grow);
|
||||
let _xc_0 = builder.ins().func_addr(ptr_type, h_mem_grow);
|
||||
let call = builder
|
||||
.ins()
|
||||
.call_indirect(mem_grow_sig, _xc_0, &[_xv_config_var, mem_idx, pages_i32]);
|
||||
|
|
@ -1567,7 +1583,7 @@ impl CraneliftCompiler {
|
|||
let dst_i32 = builder.ins().ireduce(types::I32, dst_offset);
|
||||
let dst_mem = builder.ins().iconst(types::I32, insn.imm1);
|
||||
let src_mem = builder.ins().iconst(types::I32, insn.imm2);
|
||||
let cfp = builder.ins().iconst(ptr_type, h_memory_copy);
|
||||
let cfp = builder.ins().func_addr(ptr_type, h_memory_copy);
|
||||
let iv = builder.use_var(interp_var);
|
||||
let cv = builder.use_var(config_var);
|
||||
do_call_and_check!(
|
||||
|
|
@ -1588,7 +1604,7 @@ impl CraneliftCompiler {
|
|||
let value_i32 = builder.ins().ireduce(types::I32, value);
|
||||
let offset_i32 = builder.ins().ireduce(types::I32, offset);
|
||||
let mem_idx = builder.ins().iconst(types::I32, insn.imm1);
|
||||
let cfp = builder.ins().iconst(ptr_type, h_memory_fill);
|
||||
let cfp = builder.ins().func_addr(ptr_type, h_memory_fill);
|
||||
let iv = builder.use_var(interp_var);
|
||||
let cv = builder.use_var(config_var);
|
||||
do_call_and_check!(
|
||||
|
|
@ -1604,13 +1620,13 @@ impl CraneliftCompiler {
|
|||
// Flush virtual stack, args are already on it from previous instructions.
|
||||
flush_vstack_to_real!(builder);
|
||||
let func_idx = builder.ins().iconst(types::I32, insn.imm1);
|
||||
let cfp = builder.ins().iconst(ptr_type, h_call_fn);
|
||||
let cfp = builder.ins().func_addr(ptr_type, h_call_fn);
|
||||
let iv = builder.use_var(interp_var);
|
||||
let cv = builder.use_var(config_var);
|
||||
do_call_and_check!(builder, call_fn_sig, cfp, &[iv, cv, func_idx]);
|
||||
// The helper pushes results to value_stack; pop to the actual destination.
|
||||
if insn.destination != STACK_MARKER {
|
||||
let pop_fp = builder.ins().iconst(ptr_type, h_stack_pop);
|
||||
let pop_fp = builder.ins().func_addr(ptr_type, h_stack_pop);
|
||||
let cfg = builder.use_var(config_var);
|
||||
let call = builder.ins().call_indirect(stack_pop_sig, pop_fp, &[cfg]);
|
||||
let result = builder.inst_results(call)[0];
|
||||
|
|
@ -1624,7 +1640,7 @@ impl CraneliftCompiler {
|
|||
let element_index = builder.ins().ireduce(types::I32, element_index);
|
||||
let type_idx = builder.ins().iconst(types::I32, insn.imm1);
|
||||
let table_idx = builder.ins().iconst(types::I32, insn.imm2);
|
||||
let cfp = builder.ins().iconst(ptr_type, h_call_indirect);
|
||||
let cfp = builder.ins().func_addr(ptr_type, h_call_indirect);
|
||||
let iv = builder.use_var(interp_var);
|
||||
let cv = builder.use_var(config_var);
|
||||
do_call_and_check!(
|
||||
|
|
@ -1634,7 +1650,7 @@ impl CraneliftCompiler {
|
|||
&[iv, cv, table_idx, type_idx, element_index]
|
||||
);
|
||||
if insn.destination != STACK_MARKER {
|
||||
let pop_fp = builder.ins().iconst(ptr_type, h_stack_pop);
|
||||
let pop_fp = builder.ins().func_addr(ptr_type, h_stack_pop);
|
||||
let cfg = builder.use_var(config_var);
|
||||
let call = builder.ins().call_indirect(stack_pop_sig, pop_fp, &[cfg]);
|
||||
let result = builder.inst_results(call)[0];
|
||||
|
|
@ -1651,25 +1667,25 @@ impl CraneliftCompiler {
|
|||
let cv = builder.use_var(config_var);
|
||||
match param_count {
|
||||
0 => {
|
||||
let cfp = builder.ins().iconst(ptr_type, h_direct_call_0);
|
||||
let cfp = builder.ins().func_addr(ptr_type, h_direct_call_0);
|
||||
do_call_and_check!(builder, call_fn_sig, cfp, &[iv, cv, func_idx]);
|
||||
}
|
||||
1 => {
|
||||
let arg0 = read_src!(builder, insn.sources[0]);
|
||||
let cfp = builder.ins().iconst(ptr_type, h_direct_call_1);
|
||||
let cfp = builder.ins().func_addr(ptr_type, h_direct_call_1);
|
||||
do_call_and_check!(builder, call_fn1_sig, cfp, &[iv, cv, func_idx, arg0]);
|
||||
}
|
||||
2 => {
|
||||
let s0 = read_src!(builder, insn.sources[0]); // last param (top)
|
||||
let s1 = read_src!(builder, insn.sources[1]); // first param
|
||||
let cfp = builder.ins().iconst(ptr_type, h_direct_call_2);
|
||||
let cfp = builder.ins().func_addr(ptr_type, h_direct_call_2);
|
||||
do_call_and_check!(builder, call_fn2_sig, cfp, &[iv, cv, func_idx, s1, s0]);
|
||||
}
|
||||
3 => {
|
||||
let s0 = read_src!(builder, insn.sources[0]); // last param (top)
|
||||
let s1 = read_src!(builder, insn.sources[1]); // middle param
|
||||
let s2 = read_src!(builder, insn.sources[2]); // first param
|
||||
let cfp = builder.ins().iconst(ptr_type, h_direct_call_3);
|
||||
let cfp = builder.ins().func_addr(ptr_type, h_direct_call_3);
|
||||
do_call_and_check!(builder, call_fn3_sig, cfp, &[iv, cv, func_idx, s2, s1, s0]);
|
||||
}
|
||||
_ => unreachable!(),
|
||||
|
|
@ -1689,7 +1705,7 @@ impl CraneliftCompiler {
|
|||
|
||||
op::SYNTHETIC_CALL_WITH_RECORD_0 | op::SYNTHETIC_CALL_WITH_RECORD_1 => {
|
||||
let func_idx = builder.ins().iconst(types::I32, insn.imm1);
|
||||
let cwp = builder.ins().iconst(ptr_type, h_call_wr);
|
||||
let cwp = builder.ins().func_addr(ptr_type, h_call_wr);
|
||||
let iv = builder.use_var(interp_var);
|
||||
let cv = builder.use_var(config_var);
|
||||
do_call_and_check!(builder, call_wr_sig, cwp, &[iv, cv, func_idx]);
|
||||
|
|
@ -1823,7 +1839,7 @@ impl CraneliftCompiler {
|
|||
h_mem_store64
|
||||
};
|
||||
let _uv_config_var = builder.use_var(config_var);
|
||||
let _ic_0 = builder.ins().iconst(ptr_type, memory_store_helper);
|
||||
let _ic_0 = builder.ins().func_addr(ptr_type, memory_store_helper);
|
||||
let call =
|
||||
builder
|
||||
.ins()
|
||||
|
|
@ -1861,7 +1877,7 @@ impl CraneliftCompiler {
|
|||
builder.seal_block(epilogue_block);
|
||||
// Clean up excess values on the real stack (e.g. from BR out of nested blocks); nothing to touch if we have vstack info.
|
||||
if has_raw_call {
|
||||
let cleanup_fp = builder.ins().iconst(ptr_type, h_stack_cleanup);
|
||||
let cleanup_fp = builder.ins().func_addr(ptr_type, h_stack_cleanup);
|
||||
let cfg = builder.use_var(config_var);
|
||||
let init_size = builder.use_var(initial_stack_size_var);
|
||||
let arity = builder.ins().iconst(types::I32, result_arity as i64);
|
||||
|
|
@ -1883,11 +1899,41 @@ impl CraneliftCompiler {
|
|||
builder.finalize();
|
||||
|
||||
let mut ctx = Context::for_function(func);
|
||||
let code = ctx
|
||||
.compile(&*isa, &mut Default::default())
|
||||
.map_err(|_| "cranelift compilation failed")?;
|
||||
// Snapshot the code bytes + raw reloc list before we drop the borrow on ctx so we
|
||||
// can map UserExternalNameRefs back to helper ids via ctx.func.params below.
|
||||
let (bytes, raw_relocs) = {
|
||||
let code = ctx
|
||||
.compile(&*isa, &mut Default::default())
|
||||
.map_err(|_| "cranelift compilation failed")?;
|
||||
let bytes = code.code_buffer().to_vec();
|
||||
let raw = code.buffer.relocs().to_vec();
|
||||
(bytes, raw)
|
||||
};
|
||||
|
||||
Ok(code.code_buffer().to_vec())
|
||||
let mut relocs: Vec<HelperReloc> = Vec::with_capacity(raw_relocs.len());
|
||||
let user_names = ctx.func.params.user_named_funcs();
|
||||
for r in &raw_relocs {
|
||||
// We only ever ask cranelift to relocate helper-function addresses, so any
|
||||
// other reloc means it lowered something we didn't expect -- bail rather than
|
||||
// produce machine code that the cache layer can't faithfully reproduce.
|
||||
if r.kind != Reloc::Abs8 {
|
||||
return Err("unexpected non-Abs8 relocation");
|
||||
}
|
||||
let name = match &r.target {
|
||||
FinalizedRelocTarget::ExternalName(ExternalName::User(user_ref)) => &user_names[*user_ref],
|
||||
_ => return Err("unexpected relocation target"),
|
||||
};
|
||||
if name.namespace != 0 || name.index >= crate::HELPER_COUNT {
|
||||
return Err("relocation refers to an unknown helper id");
|
||||
}
|
||||
relocs.push(HelperReloc {
|
||||
code_offset: r.offset,
|
||||
helper_id: name.index,
|
||||
addend: r.addend,
|
||||
});
|
||||
}
|
||||
|
||||
Ok(CompiledFunction { code: bytes, relocs })
|
||||
}
|
||||
|
||||
fn is_supported(insn: &CraneliftInsn) -> bool {
|
||||
|
|
|
|||
|
|
@ -87,11 +87,71 @@ pub struct RuntimeHelpers {
|
|||
pub compiled_call_result_scratch_offset: u32,
|
||||
}
|
||||
|
||||
/// Stable index assigned to each runtime helper. Embedded in cranelift `ExternalName`
|
||||
/// entries so we can recover, post-codegen, which helper a given relocation targets.
|
||||
/// The numeric values are part of the cache blob format -- do NOT reorder.
|
||||
#[repr(u32)]
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
#[allow(non_camel_case_types)]
|
||||
pub enum HelperId {
|
||||
call_function = 0,
|
||||
set_trap = 1,
|
||||
memory_load8_s = 2,
|
||||
memory_load8_u = 3,
|
||||
memory_load16_s = 4,
|
||||
memory_load16_u = 5,
|
||||
memory_load32_s = 6,
|
||||
memory_load32_u = 7,
|
||||
memory_load64 = 8,
|
||||
memory_store8 = 9,
|
||||
memory_store16 = 10,
|
||||
memory_store32 = 11,
|
||||
memory_store64 = 12,
|
||||
memory_size = 13,
|
||||
memory_grow = 14,
|
||||
read_global = 15,
|
||||
write_global = 16,
|
||||
stack_push = 17,
|
||||
stack_pop = 18,
|
||||
stack_size = 19,
|
||||
stack_cleanup = 20,
|
||||
callrec_read = 21,
|
||||
callrec_write = 22,
|
||||
call_with_record = 23,
|
||||
direct_call_0 = 24,
|
||||
direct_call_1 = 25,
|
||||
direct_call_2 = 26,
|
||||
direct_call_3 = 27,
|
||||
call_indirect = 28,
|
||||
memory_copy = 29,
|
||||
memory_fill = 30,
|
||||
}
|
||||
|
||||
pub const HELPER_COUNT: u32 = 31;
|
||||
|
||||
/// One relocation slot in the generated machine code. `code_offset` is the byte offset
|
||||
/// from the start of the function where 8 contiguous bytes hold the absolute helper
|
||||
/// address; on cache install, those bytes get rewritten to the current process's
|
||||
/// helper pointer.
|
||||
#[repr(C)]
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub struct HelperReloc {
|
||||
pub code_offset: u32,
|
||||
pub helper_id: u32,
|
||||
pub addend: i64,
|
||||
}
|
||||
|
||||
/// Output of `compile_to_bytes`: the machine code plus the patch table.
|
||||
pub struct CompiledFunction {
|
||||
pub code: Vec<u8>,
|
||||
pub relocs: Vec<HelperReloc>,
|
||||
}
|
||||
|
||||
pub fn compile_to_bytes(
|
||||
insns: &[CraneliftInsn],
|
||||
helpers: &RuntimeHelpers,
|
||||
outcome_return_value: u64,
|
||||
result_arity: u32,
|
||||
) -> Result<Vec<u8>, &'static str> {
|
||||
) -> Result<CompiledFunction, &'static str> {
|
||||
CraneliftCompiler::compile_to_bytes(insns, helpers, outcome_return_value, result_arity)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@
|
|||
#include <AK/ByteString.h>
|
||||
#include <AK/DistinctNumeric.h>
|
||||
#include <AK/FixedArray.h>
|
||||
#include <AK/Function.h>
|
||||
#include <AK/LEB128.h>
|
||||
#include <AK/NumericLimits.h>
|
||||
#include <AK/Optional.h>
|
||||
|
|
@ -1547,4 +1548,34 @@ CompiledInstructions try_compile_instructions(Expression const&, Span<FunctionTy
|
|||
bool try_cranelift_compile(CompiledInstructions& compiled, u32 result_arity = 0);
|
||||
void flush_cranelift_batch();
|
||||
|
||||
// Caller-supplied hooks for the Cranelift on-disk cache.
|
||||
// - `wasm_hash` is a 32-byte digest of the wasm bytes; embedded in produced blobs
|
||||
// and verified against `existing_blob` before any install.
|
||||
// - `existing_blob` is the prior cache hit (or empty for a miss); validator tries
|
||||
// to install it before falling through to cranelift.
|
||||
// - `on_compiled` is invoked exactly once if a fresh blob was produced; never
|
||||
// invoked on a cache hit nor when no cranelift output was captured.
|
||||
struct CompileCacheConfig {
|
||||
Array<u8, 32> wasm_hash {};
|
||||
ReadonlyBytes existing_blob;
|
||||
AK::Function<void(ByteBuffer)> on_compiled;
|
||||
};
|
||||
|
||||
// Cranelift disk-cache plumbing. Validator drives these around CodeSection validation:
|
||||
// 1. set_cranelift_active_function_index() before each function so cache-hit installs
|
||||
// and post-compile capture know which function they're talking about.
|
||||
// 2. begin_cranelift_cache_capture() to start collecting compiled bytes + relocs.
|
||||
// 3. try_install_cranelift_cache_blob() with the wasm-bytes hash + stored blob; if
|
||||
// it returns true, individual try_cranelift_compile calls will short-circuit by
|
||||
// installing from the stashed records instead of queueing fresh compiles.
|
||||
// 4. After flush, serialize_cranelift_cache_blob() returns a blob to hand to the
|
||||
// cache store (or {} if nothing was captured); abort_cranelift_cache_capture()
|
||||
// throws the capture away.
|
||||
void set_cranelift_active_function_index(u32 function_index);
|
||||
void begin_cranelift_cache_capture();
|
||||
void abort_cranelift_cache_capture();
|
||||
void abort_cranelift_cache_install();
|
||||
Optional<ByteBuffer> serialize_cranelift_cache_blob(ReadonlyBytes wasm_hash);
|
||||
bool try_install_cranelift_cache_blob(ReadonlyBytes expected_wasm_hash, ReadonlyBytes blob);
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,6 +9,8 @@
|
|||
#include <AK/MemoryStream.h>
|
||||
#include <AK/ScopeGuard.h>
|
||||
#include <AK/StringBuilder.h>
|
||||
#include <LibCrypto/Hash/SHA2.h>
|
||||
#include <LibHTTP/Cache/Utilities.h>
|
||||
#include <LibJS/Runtime/Array.h>
|
||||
#include <LibJS/Runtime/ArrayBuffer.h>
|
||||
#include <LibJS/Runtime/BigInt.h>
|
||||
|
|
@ -19,13 +21,17 @@
|
|||
#include <LibJS/Runtime/Object.h>
|
||||
#include <LibJS/Runtime/VM.h>
|
||||
#include <LibJS/Runtime/ValueInlines.h>
|
||||
#include <LibRequests/RequestClient.h>
|
||||
#include <LibURL/Parser.h>
|
||||
#include <LibWasm/AbstractMachine/Validator.h>
|
||||
#include <LibWeb/Bindings/Intrinsics.h>
|
||||
#include <LibWeb/Bindings/Response.h>
|
||||
#include <LibWeb/ContentSecurityPolicy/BlockingAlgorithms.h>
|
||||
#include <LibWeb/Fetch/Infrastructure/HTTP/MIME.h>
|
||||
#include <LibWeb/Fetch/Infrastructure/URL.h>
|
||||
#include <LibWeb/Fetch/Response.h>
|
||||
#include <LibWeb/HTML/Scripting/TemporaryExecutionContext.h>
|
||||
#include <LibWeb/Loader/ResourceLoader.h>
|
||||
#include <LibWeb/Platform/EventLoopPlugin.h>
|
||||
#include <LibWeb/WebAssembly/Global.h>
|
||||
#include <LibWeb/WebAssembly/Instance.h>
|
||||
|
|
@ -432,8 +438,50 @@ JS::ThrowCompletionOr<NonnullRefPtr<CompiledWebAssemblyModule>> compile_a_webass
|
|||
return vm.throw_completion<CompileError>(Wasm::parse_error_to_byte_string(module_result.error()));
|
||||
}
|
||||
|
||||
// Content-keyed disk cache: hash the wasm bytes, slot into the HTTP side-data
|
||||
// shelf under a synthetic wasm-cache://<hex> URL (with a stub index entry to
|
||||
// satisfy the shelf's "associated data needs a real entry" invariant). Works
|
||||
// regardless of whether the caller had a URL.
|
||||
// existing_blob view is borrowed below; the AnonymousBuffer must outlive validate().
|
||||
Optional<Core::AnonymousBuffer> existing_buf;
|
||||
Optional<Wasm::CompileCacheConfig> wasm_cache_config;
|
||||
if (ResourceLoader::is_initialized() && ResourceLoader::the().request_client()) {
|
||||
auto digest = ::Crypto::Hash::SHA256::hash(data.data(), data.size());
|
||||
|
||||
StringBuilder hex_builder;
|
||||
for (auto byte : digest.bytes())
|
||||
hex_builder.appendff("{:02x}", byte);
|
||||
auto synthetic_url = URL::Parser::basic_parse(ByteString::formatted("wasm-cache://{}", hex_builder.to_byte_string()));
|
||||
if (synthetic_url.has_value()) {
|
||||
auto method = "GET"_string.to_byte_string();
|
||||
(void)ResourceLoader::the().request_client()->create_synthetic_cache_entry(*synthetic_url, method);
|
||||
|
||||
Wasm::CompileCacheConfig config;
|
||||
__builtin_memcpy(config.wasm_hash.data(), digest.bytes().data(), 32);
|
||||
|
||||
auto retrieve_result = ResourceLoader::the().request_client()->retrieve_cache_associated_data(
|
||||
*synthetic_url, method, OptionalNone {}, 0u,
|
||||
HTTP::CacheEntryAssociatedData::WebAssemblyCompiledCode);
|
||||
if (!retrieve_result.is_error()) {
|
||||
if (auto buf = retrieve_result.release_value(); buf.has_value()) {
|
||||
existing_buf = buf.release_value();
|
||||
config.existing_blob = existing_buf->bytes();
|
||||
}
|
||||
}
|
||||
|
||||
config.on_compiled = [url = *synthetic_url, method = move(method)](ByteBuffer blob) mutable {
|
||||
if (!ResourceLoader::is_initialized() || !ResourceLoader::the().request_client())
|
||||
return;
|
||||
(void)ResourceLoader::the().request_client()->store_cache_associated_data(
|
||||
url, method, OptionalNone {}, 0u,
|
||||
HTTP::CacheEntryAssociatedData::WebAssemblyCompiledCode, blob.bytes());
|
||||
};
|
||||
wasm_cache_config = move(config);
|
||||
}
|
||||
}
|
||||
|
||||
auto& cache = get_cache(*vm.current_realm());
|
||||
if (auto validation_result = cache.abstract_machine().validate(module_result.value()); validation_result.is_error()) {
|
||||
if (auto validation_result = cache.abstract_machine().validate(module_result.value(), move(wasm_cache_config)); validation_result.is_error()) {
|
||||
return vm.throw_completion<CompileError>(validation_result.error().error_string);
|
||||
}
|
||||
auto compiled_module = make_ref_counted<CompiledWebAssemblyModule>(module_result.release_value());
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@
|
|||
#include <LibJS/Runtime/NativeFunction.h>
|
||||
#include <LibJS/Runtime/PrototypeObject.h>
|
||||
#include <LibJS/Runtime/Value.h>
|
||||
#include <LibURL/URL.h>
|
||||
#include <LibWasm/AbstractMachine/AbstractMachine.h>
|
||||
#include <LibWeb/Export.h>
|
||||
#include <LibWeb/Forward.h>
|
||||
|
|
|
|||
Loading…
Reference in a new issue