LibWasm: Start execution immediately after validation
This adds a tier-up mechanism at loop edges, making it so we can seamlessly (ish) transition between interpreted and native code so we can start running wasm code immediately after validation while compilation happens in the background, and switching to native code eventually once we hit a big enough function that would benefit from being compiled to begin with.
This commit is contained in:
Ali Mohammad Pur
Ali Mohammad Pur
parent
3c89c36158
commit
08221aebab
9 changed files with 249 additions and 37 deletions
|
|
@ -40,7 +40,7 @@ void dump_module_stats()
|
|||
}
|
||||
|
||||
warnln("wasm-stats: {} module(s) compiled", v.size());
|
||||
warnln("wasm-stats: hash input KiB parse ms validate ms cl ms cl blob KiB funcs cache");
|
||||
warnln("wasm-stats: hash input KiB parse ms validate ms cl ms cl blob KiB funcs tu fns tu pts cache");
|
||||
|
||||
AK::Duration total_parse;
|
||||
AK::Duration total_validate;
|
||||
|
|
@ -48,13 +48,15 @@ void dump_module_stats()
|
|||
size_t total_input = 0;
|
||||
size_t total_blob = 0;
|
||||
size_t total_hits = 0;
|
||||
size_t total_tier_up_functions = 0;
|
||||
size_t total_tier_up_checkpoints = 0;
|
||||
|
||||
for (auto const& s : v) {
|
||||
StringBuilder hash_prefix;
|
||||
for (size_t i = 0; i < 4; ++i)
|
||||
hash_prefix.appendff("{:02x}", s.wasm_hash[i]);
|
||||
|
||||
warnln("wasm-stats: {} {:>9} {:>8} {:>11} {:>5} {:>11} {:>5} {}",
|
||||
warnln("wasm-stats: {} {:>9} {:>8} {:>11} {:>5} {:>11} {:>5} {:>7} {:>7} {}",
|
||||
hash_prefix.to_byte_string(),
|
||||
s.input_size_bytes / 1024,
|
||||
s.parse_time.to_milliseconds(),
|
||||
|
|
@ -62,6 +64,8 @@ void dump_module_stats()
|
|||
s.cranelift_time.to_milliseconds(),
|
||||
s.cranelift_blob_size_bytes / 1024,
|
||||
s.function_count,
|
||||
s.tier_up_function_count,
|
||||
s.tier_up_checkpoint_count,
|
||||
s.cache_hit ? "HIT" : "miss");
|
||||
|
||||
total_parse = total_parse + s.parse_time;
|
||||
|
|
@ -69,16 +73,21 @@ void dump_module_stats()
|
|||
total_cranelift = total_cranelift + s.cranelift_time;
|
||||
total_input += s.input_size_bytes;
|
||||
total_blob += s.cranelift_blob_size_bytes;
|
||||
total_tier_up_functions += s.tier_up_function_count;
|
||||
total_tier_up_checkpoints += s.tier_up_checkpoint_count;
|
||||
if (s.cache_hit)
|
||||
++total_hits;
|
||||
}
|
||||
|
||||
warnln("wasm-stats: ---- {:>9} {:>8} {:>11} {:>5} {:>11} hits={}",
|
||||
warnln("wasm-stats: ---- {:>9} {:>8} {:>11} {:>5} {:>11} {:>5} {:>7} {:>7} hits={}",
|
||||
total_input / 1024,
|
||||
total_parse.to_milliseconds(),
|
||||
total_validate.to_milliseconds(),
|
||||
total_cranelift.to_milliseconds(),
|
||||
total_blob / 1024,
|
||||
""sv,
|
||||
total_tier_up_functions,
|
||||
total_tier_up_checkpoints,
|
||||
total_hits);
|
||||
});
|
||||
}
|
||||
|
|
@ -235,26 +244,32 @@ Vector<CompiledFunctionEntry> const& ModuleInstance::compiled_fn_table(Store& st
|
|||
{
|
||||
if (m_compiled_fn_table_built)
|
||||
return m_compiled_fn_table;
|
||||
m_compiled_fn_table_built = true;
|
||||
|
||||
auto count = m_functions.size();
|
||||
if (count == 0)
|
||||
if (count == 0) {
|
||||
m_compiled_fn_table_built = true;
|
||||
return m_compiled_fn_table;
|
||||
}
|
||||
|
||||
m_compiled_fn_table.resize_with_default_value_and_keep_capacity(count, {});
|
||||
auto* entries = m_compiled_fn_table.data();
|
||||
|
||||
// Since we asynchronously compile the code to native, we'll need to rebuild this table incrementally until all functions have been compiled.
|
||||
bool all_ready = true;
|
||||
for (size_t i = 0; i < count; i++) {
|
||||
auto* instance = store.unsafe_get(m_functions[i]);
|
||||
auto* wasm_fn = instance->get_pointer<WasmFunction>();
|
||||
if (!wasm_fn)
|
||||
continue;
|
||||
if (auto src = wasm_fn->module_ref(); src && !src->has_attempted_cranelift_compilation())
|
||||
all_ready = false;
|
||||
auto& ci = wasm_fn->code().func().body().compiled_instructions;
|
||||
if (!ci.cranelift_compiled)
|
||||
auto native = cranelift_entry_acquire(ci);
|
||||
if (native == 0)
|
||||
continue;
|
||||
|
||||
auto& entry = entries[i];
|
||||
entry.handler_ptr = ci.dispatches[0].handler_ptr;
|
||||
entry.handler_ptr = native;
|
||||
entry.dispatches_ptr = bit_cast<FlatPtr>(ci.dispatches.data());
|
||||
entry.src_dst_ptr = bit_cast<FlatPtr>(ci.src_dst_mappings.data());
|
||||
entry.first_insn = ci.dispatches[0].instruction;
|
||||
|
|
@ -264,6 +279,7 @@ Vector<CompiledFunctionEntry> const& ModuleInstance::compiled_fn_table(Store& st
|
|||
entry.arity = static_cast<u32>(wasm_fn->type().results().size());
|
||||
entry.max_call_rec_size = static_cast<u32>(ci.max_call_rec_size);
|
||||
}
|
||||
m_compiled_fn_table_built = all_ready;
|
||||
return m_compiled_fn_table;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -218,7 +218,7 @@ static void compiled_fault_signal_handler(int signal, siginfo_t* info, void* con
|
|||
# endif
|
||||
|
||||
auto const& compiled = recovery->configuration->frame().expression().compiled_instructions;
|
||||
auto const code_start = compiled.dispatches.is_empty() ? 0 : compiled.dispatches[0].handler_ptr;
|
||||
auto const code_start = compiled.cranelift_entry;
|
||||
auto const code_size = compiled.cranelift_code_size;
|
||||
if (compiled.cranelift_compiled && code_start != 0 && pc >= code_start && pc < code_start + code_size) {
|
||||
auto const offset = static_cast<u32>(pc - code_start);
|
||||
|
|
@ -393,9 +393,12 @@ void BytecodeInterpreter::interpret(Configuration& configuration)
|
|||
{
|
||||
m_trap = Empty {};
|
||||
auto& expression = configuration.frame().expression();
|
||||
auto const native_entry = cranelift_entry_acquire(expression.compiled_instructions);
|
||||
// We may end up running native code either at entry (native_entry != 0) or mid-loop via a tier-up checkpoint, so install fault recovery in either case.
|
||||
bool const may_run_native = native_entry != 0 || expression.compiled_instructions.has_tier_up_checkpoints;
|
||||
CompiledFaultRecoveryContext compiled_fault_recovery;
|
||||
bool did_install_compiled_fault_recovery = false;
|
||||
if (expression.compiled_instructions.cranelift_compiled && !s_compiled_fault_recovery) {
|
||||
if (may_run_native && !s_compiled_fault_recovery) {
|
||||
install_compiled_fault_handlers();
|
||||
compiled_fault_recovery.interpreter = this;
|
||||
compiled_fault_recovery.configuration = &configuration;
|
||||
|
|
@ -411,24 +414,30 @@ void BytecodeInterpreter::interpret(Configuration& configuration)
|
|||
return;
|
||||
}
|
||||
}
|
||||
auto const should_limit_instruction_count = configuration.should_limit_instruction_count();
|
||||
if (!expression.compiled_instructions.dispatches.is_empty()) {
|
||||
if (expression.compiled_instructions.direct) {
|
||||
if (should_limit_instruction_count) {
|
||||
interpret_impl<true, true, true>(configuration, expression);
|
||||
if (native_entry != 0) {
|
||||
(void)run_native_entry(configuration);
|
||||
goto done;
|
||||
}
|
||||
{
|
||||
auto const should_limit_instruction_count = configuration.should_limit_instruction_count();
|
||||
if (!expression.compiled_instructions.dispatches.is_empty()) {
|
||||
if (expression.compiled_instructions.direct) {
|
||||
if (should_limit_instruction_count) {
|
||||
interpret_impl<true, true, true>(configuration, expression);
|
||||
goto done;
|
||||
}
|
||||
interpret_impl<true, false, true>(configuration, expression);
|
||||
goto done;
|
||||
}
|
||||
interpret_impl<true, false, true>(configuration, expression);
|
||||
interpret_impl<true, false, false>(configuration, expression);
|
||||
goto done;
|
||||
}
|
||||
interpret_impl<true, false, false>(configuration, expression);
|
||||
goto done;
|
||||
if (should_limit_instruction_count) {
|
||||
interpret_impl<false, true, false>(configuration, expression);
|
||||
goto done;
|
||||
}
|
||||
interpret_impl<false, false, false>(configuration, expression);
|
||||
}
|
||||
if (should_limit_instruction_count) {
|
||||
interpret_impl<false, true, false>(configuration, expression);
|
||||
goto done;
|
||||
}
|
||||
interpret_impl<false, false, false>(configuration, expression);
|
||||
|
||||
done:
|
||||
if (did_install_compiled_fault_recovery)
|
||||
|
|
@ -471,6 +480,21 @@ Outcome BytecodeInterpreter::run_compiled_function_direct(Configuration& configu
|
|||
return handler(*this, configuration, instruction, short_ip, cc, addresses_ptr);
|
||||
}
|
||||
|
||||
// Enter the Cranelift-compiled native code for this function. The native entry conforms to the
|
||||
// same handler ABI as the direct-threaded interpreter, but lives in CompiledInstructions::cranelift_entry
|
||||
// (dispatches[0].handler_ptr stays the C++ handler). Caller must have confirmed the entry is non-zero.
|
||||
Outcome BytecodeInterpreter::run_native_entry(Configuration& configuration)
|
||||
{
|
||||
m_trap = Empty {};
|
||||
auto& expression = configuration.frame().expression();
|
||||
auto const* cc = expression.compiled_instructions.dispatches.data();
|
||||
auto const* addresses_ptr = expression.compiled_instructions.src_dst_mappings.data();
|
||||
ShortenedIP short_ip { .current_ip_value = 0 };
|
||||
auto const instruction = cc[0].instruction;
|
||||
auto const handler = bit_cast<Outcome (*)(HANDLER_PARAMS(DECOMPOSE_PARAMS_TYPE_ONLY))>(cranelift_entry_acquire(expression.compiled_instructions));
|
||||
return handler(*this, configuration, instruction, short_ip, cc, addresses_ptr);
|
||||
}
|
||||
|
||||
#define HANDLE_INSTRUCTION(name, ...) \
|
||||
template<> \
|
||||
struct InstructionHandler<Instructions::name.value()> { \
|
||||
|
|
@ -1952,6 +1976,20 @@ HANDLE_INSTRUCTION(synthetic_br_table_cont)
|
|||
VERIFY_NOT_REACHED();
|
||||
}
|
||||
|
||||
HANDLE_INSTRUCTION(synthetic_tier_up)
|
||||
{
|
||||
LOG_INSN;
|
||||
auto& ci = configuration.frame().expression().compiled_instructions;
|
||||
auto const native_entry = cranelift_entry_acquire(ci);
|
||||
if (native_entry != 0) {
|
||||
// If we have native code for this block, jump into it.
|
||||
// The code is set up such that the target checkpoint is recovered from short_ip and nothing else needs to be passed as the stack is empty and all live state is in the shared locals.
|
||||
auto const handler = bit_cast<Outcome (*)(HANDLER_PARAMS(DECOMPOSE_PARAMS_TYPE_ONLY))>(native_entry);
|
||||
return handler(interpreter, configuration, cc[short_ip.current_ip_value].instruction, short_ip, cc, addresses_ptr);
|
||||
}
|
||||
TAILCALL return continue_(HANDLER_PARAMS(DECOMPOSE_PARAMS_NAME_ONLY));
|
||||
}
|
||||
|
||||
HANDLE_INSTRUCTION(synthetic_call_00)
|
||||
{
|
||||
LOG_INSN;
|
||||
|
|
@ -6410,6 +6448,70 @@ CompiledInstructions try_compile_instructions(Expression const& expression, Span
|
|||
result.dispatches.remove_all(nops_to_remove, [](auto const& it) { return it.key(); });
|
||||
result.src_dst_mappings.remove_all(nops_to_remove, [](auto const& it) { return it.key(); });
|
||||
|
||||
// Every time we have a large-enough function, drop a synthetic_tier_up checkpoint right after each loop header that's eligible for tier-up (empty stack at the header, so the back-edge hits it every iteration).
|
||||
// This allows us to start running code immediately in the interpreter, and switch to native code on paths that matter (or eventually) once compiled code is ready and the tier-up check hits.
|
||||
constexpr size_t tier_up_instruction_threshold = 32;
|
||||
if (result.dispatches.size() >= tier_up_instruction_threshold) {
|
||||
Vector<size_t> loop_positions;
|
||||
for (size_t i = 0; i < result.dispatches.size(); ++i) {
|
||||
if (result.dispatches[i].instruction->opcode() != Instructions::loop)
|
||||
continue;
|
||||
auto& sa = result.dispatches[i].instruction->arguments().get<Instruction::StructuredInstructionArgs>();
|
||||
if (sa.meta.tier_up_eligible)
|
||||
loop_positions.append(i);
|
||||
}
|
||||
|
||||
if (!loop_positions.is_empty()) {
|
||||
// Number of tier-up ops inserted strictly before a given old dispatch index. A tier-up
|
||||
// sits between its loop (at L) and L+1, so it precedes any old index > L.
|
||||
auto shift_before = [&](size_t old_index) {
|
||||
size_t shift = 0;
|
||||
for (auto position : loop_positions) {
|
||||
if (position < old_index)
|
||||
++shift;
|
||||
else
|
||||
break;
|
||||
}
|
||||
return shift;
|
||||
};
|
||||
|
||||
Vector<Dispatch> new_dispatches;
|
||||
Vector<SourcesAndDestination> new_src_dst;
|
||||
new_dispatches.ensure_capacity(result.dispatches.size() + loop_positions.size());
|
||||
new_src_dst.ensure_capacity(result.src_dst_mappings.size() + loop_positions.size());
|
||||
|
||||
size_t next_loop = 0;
|
||||
for (size_t i = 0; i < result.dispatches.size(); ++i) {
|
||||
new_dispatches.append(result.dispatches[i]);
|
||||
new_src_dst.append(result.src_dst_mappings[i]);
|
||||
if (next_loop < loop_positions.size() && loop_positions[next_loop] == i) {
|
||||
auto& tier_up = append_extra_instruction(Instructions::synthetic_tier_up);
|
||||
new_dispatches.append({ { .instruction_opcode = tier_up.opcode() }, &tier_up });
|
||||
new_src_dst.append({ .sources = { Dispatch::Stack, Dispatch::Stack, Dispatch::Stack }, .destination = Dispatch::Stack });
|
||||
++next_loop;
|
||||
}
|
||||
}
|
||||
|
||||
// Re-point absolute IPs in structured args (end_ip / else_ip) past the inserted ops.
|
||||
for (size_t i = 0; i < new_dispatches.size(); ++i) {
|
||||
auto* sa = new_dispatches[i].instruction->arguments().get_pointer<Instruction::StructuredInstructionArgs>();
|
||||
if (!sa)
|
||||
continue;
|
||||
InstructionPointer new_end_ip = sa->end_ip.value() + shift_before(sa->end_ip.value());
|
||||
auto new_else_ip = sa->else_ip().map([&](InstructionPointer ip) -> InstructionPointer { return ip.value() + shift_before(ip.value()); });
|
||||
auto rebuilt = *new_dispatches[i].instruction;
|
||||
rebuilt.arguments() = Instruction::StructuredInstructionArgs { sa->block_type, new_end_ip, new_else_ip, sa->meta };
|
||||
auto& extra_instruction = append_extra_instruction(move(rebuilt));
|
||||
new_dispatches[i].instruction = &extra_instruction;
|
||||
new_dispatches[i].instruction_opcode = extra_instruction.opcode();
|
||||
}
|
||||
|
||||
result.dispatches = move(new_dispatches);
|
||||
result.src_dst_mappings = move(new_src_dst);
|
||||
result.has_tier_up_checkpoints = true;
|
||||
}
|
||||
}
|
||||
|
||||
// Rewrite local.* of arguments to argument.* to keep local.* for locals only.
|
||||
for (size_t i = 0; i < result.dispatches.size(); ++i) {
|
||||
auto& dispatch = result.dispatches[i];
|
||||
|
|
|
|||
|
|
@ -97,6 +97,7 @@ struct WASM_API BytecodeInterpreter final : public Interpreter {
|
|||
bool store_to_memory(Configuration&, Instruction::MemoryArgument const&, ReadonlyBytes data, u32 base);
|
||||
Outcome call_address(Configuration&, FunctionAddress, SourcesAndDestination const&, CallAddressSource = CallAddressSource::DirectCall, CallType = CallType::UsingStack);
|
||||
Outcome run_compiled_function_direct(Configuration&);
|
||||
Outcome run_native_entry(Configuration&);
|
||||
bool trap_if_insufficient_native_stack_space(size_t minimum_native_stack_space_to_keep_free = 2 * MiB);
|
||||
|
||||
template<typename T>
|
||||
|
|
|
|||
|
|
@ -7,7 +7,6 @@
|
|||
#include <AK/MemoryStream.h>
|
||||
#include <LibWasm/AbstractMachine/Configuration.h>
|
||||
#include <LibWasm/AbstractMachine/Interpreter.h>
|
||||
#include <LibWasm/AbstractMachine/Validator.h>
|
||||
#include <LibWasm/Printer/Printer.h>
|
||||
|
||||
namespace Wasm {
|
||||
|
|
@ -65,11 +64,10 @@ ErrorOr<Optional<HostFunction&>, Trap> Configuration::prepare_call(FunctionAddre
|
|||
|
||||
ErrorOr<void, Trap> Configuration::prepare_wasm_call(WasmFunction const& wasm_function, Vector<Value, ArgumentsStaticSize>& arguments, bool is_tailcall)
|
||||
{
|
||||
if (auto module = wasm_function.module_ref()) {
|
||||
if (auto result = ensure_cranelift_compiled(const_cast<Module&>(*module)); result.is_error())
|
||||
return Trap::from_string(ByteString::formatted("Cranelift compilation failed: {}", result.error().error_string));
|
||||
}
|
||||
|
||||
// Tier-0 by default: don't block the call waiting for native compilation. Non-Web embedders
|
||||
// compile synchronously at instantiate time (so the JIT is already live here); the Web path
|
||||
// compiles in the background and the interpreter picks up the native entry on a later call
|
||||
// once it's published. Either way, execution falls back to the interpreter until then.
|
||||
if (is_tailcall)
|
||||
unwind_impl();
|
||||
|
||||
|
|
|
|||
|
|
@ -198,11 +198,23 @@ void compile_module_to_native(Module& module)
|
|||
stats->cranelift_blob_size_bytes = produced_blob_size;
|
||||
stats->cache_hit = installing;
|
||||
size_t count = 0;
|
||||
size_t tier_up_functions = 0;
|
||||
size_t tier_up_checkpoints = 0;
|
||||
for (auto& entry : module.code_section().functions()) {
|
||||
if (entry.func().body().compiled_instructions.cranelift_compiled)
|
||||
auto const& ci = entry.func().body().compiled_instructions;
|
||||
if (ci.cranelift_compiled)
|
||||
++count;
|
||||
if (ci.has_tier_up_checkpoints) {
|
||||
++tier_up_functions;
|
||||
for (auto const& dispatch : ci.dispatches) {
|
||||
if (dispatch.instruction->opcode() == Instructions::synthetic_tier_up)
|
||||
++tier_up_checkpoints;
|
||||
}
|
||||
}
|
||||
}
|
||||
stats->function_count = count;
|
||||
stats->tier_up_function_count = tier_up_functions;
|
||||
stats->tier_up_checkpoint_count = tier_up_checkpoints;
|
||||
record_module_stats(stats.release_value());
|
||||
}
|
||||
|
||||
|
|
@ -2294,6 +2306,7 @@ VALIDATE_INSTRUCTION(block)
|
|||
args.meta = Instruction::StructuredInstructionArgs::Meta {
|
||||
.arity = static_cast<u32>(block_type.results().size()),
|
||||
.parameter_count = static_cast<u32>(parameters.size()),
|
||||
.tier_up_eligible = false,
|
||||
};
|
||||
|
||||
return {};
|
||||
|
|
@ -2308,6 +2321,8 @@ VALIDATE_INSTRUCTION(loop)
|
|||
for (size_t i = 1; i <= parameters.size(); ++i)
|
||||
TRY(stack.take(parameters[parameters.size() - i]));
|
||||
|
||||
auto const tier_up_eligible = parameters.is_empty() && stack.size() == 0;
|
||||
|
||||
m_frames.empend(block_type, FrameKind::Loop, stack.size());
|
||||
m_max_frame_size = max(m_max_frame_size, m_frames.size());
|
||||
for (auto& parameter : parameters)
|
||||
|
|
@ -2316,6 +2331,7 @@ VALIDATE_INSTRUCTION(loop)
|
|||
args.meta = Instruction::StructuredInstructionArgs::Meta {
|
||||
.arity = static_cast<u32>(block_type.results().size()),
|
||||
.parameter_count = static_cast<u32>(parameters.size()),
|
||||
.tier_up_eligible = tier_up_eligible,
|
||||
};
|
||||
|
||||
return {};
|
||||
|
|
@ -2342,6 +2358,7 @@ VALIDATE_INSTRUCTION(if_)
|
|||
args.meta = Instruction::StructuredInstructionArgs::Meta {
|
||||
.arity = static_cast<u32>(block_type.results().size()),
|
||||
.parameter_count = static_cast<u32>(parameters.size()),
|
||||
.tier_up_eligible = false,
|
||||
};
|
||||
|
||||
return {};
|
||||
|
|
@ -2400,6 +2417,7 @@ VALIDATE_INSTRUCTION(try_table)
|
|||
args.meta = Instruction::TryTableArgs::Meta {
|
||||
.arity = static_cast<u32>(block_type.results().size()),
|
||||
.parameter_count = static_cast<u32>(parameters.size()),
|
||||
.tier_up_eligible = false,
|
||||
};
|
||||
|
||||
m_frames.empend(block_type, FrameKind::TryTable, stack.size());
|
||||
|
|
|
|||
|
|
@ -256,12 +256,12 @@ static bool install_compiled_function(CompiledInstructions& target, ReadonlyByte
|
|||
for (auto const& trap : traps)
|
||||
handle->traps.unchecked_append(trap);
|
||||
|
||||
target.dispatches[0].handler_ptr = bit_cast<FlatPtr>(func_ptr);
|
||||
target.cranelift_code_handle = handle;
|
||||
target.cranelift_code_size = code_size;
|
||||
target.cranelift_traps = handle->traps.data();
|
||||
target.cranelift_trap_count = handle->traps.size();
|
||||
target.cranelift_compiled = true;
|
||||
publish_cranelift_entry(target, bit_cast<FlatPtr>(func_ptr));
|
||||
return true;
|
||||
}
|
||||
|
||||
|
|
@ -278,7 +278,7 @@ static ALWAYS_INLINE i32 wasm_cl_finish_call(BytecodeInterpreter& interpreter, C
|
|||
|
||||
if (auto* wasm_function = instance->get_pointer<WasmFunction>(); wasm_function
|
||||
&& !config.should_limit_instruction_count()
|
||||
&& wasm_function->code().func().body().compiled_instructions.cranelift_compiled) {
|
||||
&& cranelift_entry_acquire(wasm_function->code().func().body().compiled_instructions) != 0) {
|
||||
|
||||
// Fast compiled-to-compiled call: stack-allocate locals + non-owning frame.
|
||||
auto& func = wasm_function->code().func();
|
||||
|
|
@ -310,7 +310,7 @@ static ALWAYS_INLINE i32 wasm_cl_finish_call(BytecodeInterpreter& interpreter, C
|
|||
auto const* cc = ci.dispatches.data();
|
||||
auto const* addrs = ci.src_dst_mappings.data();
|
||||
using HandlerFn = Outcome (*)(BytecodeInterpreter&, Configuration&, Instruction const*, u32, Dispatch const*, SourcesAndDestination const*);
|
||||
auto const handler = bit_cast<HandlerFn>(cc[0].handler_ptr);
|
||||
auto const handler = bit_cast<HandlerFn>(cranelift_entry_acquire(ci));
|
||||
auto outcome = handler(interpreter, config, cc[0].instruction, 0, cc, addrs);
|
||||
|
||||
if (outcome != Outcome::Return) {
|
||||
|
|
@ -1420,6 +1420,9 @@ bool try_cranelift_compile(CompiledInstructions& compiled, u32 result_arity)
|
|||
for (size_t i = 0; i < dispatches.size(); ++i) {
|
||||
flat.append(serialize_insn(dispatches[i], addresses[i]));
|
||||
|
||||
if (dispatches[i].instruction->opcode().value() == Instructions::synthetic_tier_up.value())
|
||||
flat.last().imm1 = static_cast<i64>(i);
|
||||
|
||||
if (dispatches[i].instruction->opcode().value() == Instructions::br_table.value()) {
|
||||
auto const& table_args = dispatches[i].instruction->arguments().get<Instruction::TableBranchArgs>();
|
||||
auto const total = table_args.labels.size();
|
||||
|
|
|
|||
|
|
@ -548,7 +548,8 @@ namespace Instructions {
|
|||
M(synthetic_local_seti64_const, 0xfe00003bu, 0, 0) \
|
||||
/* Continuation data for br_table with >8 labels. \
|
||||
* Only consumed by the Cranelift compiler; */ \
|
||||
M(synthetic_br_table_cont, 0xfe00003cu, 0, 0)
|
||||
M(synthetic_br_table_cont, 0xfe00003cu, 0, 0) \
|
||||
M(synthetic_tier_up, 0xfe00003du, 0, 0)
|
||||
|
||||
#define ENUMERATE_WASM_OPCODES(M) \
|
||||
ENUMERATE_SINGLE_BYTE_WASM_OPCODES(M) \
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ use cranelift_codegen::Context;
|
|||
use cranelift_codegen::FinalizedRelocTarget;
|
||||
use cranelift_codegen::binemit::Reloc;
|
||||
use cranelift_codegen::ir::AbiParam;
|
||||
use cranelift_codegen::ir::Block;
|
||||
use cranelift_codegen::ir::ExtFuncData;
|
||||
use cranelift_codegen::ir::ExternalName;
|
||||
use cranelift_codegen::ir::Function;
|
||||
|
|
@ -587,6 +588,23 @@ impl CraneliftCompiler {
|
|||
}};
|
||||
}
|
||||
|
||||
// If we have any tier-up checkpoints, the interpreter will eventually need to jump to some point in the function other than the entry block, so prepare dispatch blocks for that.
|
||||
// Note that the initial block will already have the correct register state loaded, so we don't need to sync registers for the tier-up dispatch targets.
|
||||
let has_tier_up = insns.iter().any(|i| i.opcode == op::SYNTHETIC_TIER_UP);
|
||||
let tier_up_target_ip = builder.block_params(entry_block)[3];
|
||||
let mut tier_up_dispatch_tail: Option<Block> = None;
|
||||
let tier_up_body_start: Option<Block> = if has_tier_up {
|
||||
let body_start = builder.create_block();
|
||||
let dispatch = builder.create_block();
|
||||
let is_tier_up = builder.ins().icmp_imm(IntCC::NotEqual, tier_up_target_ip, 0);
|
||||
builder.ins().brif(is_tier_up, dispatch, &[], body_start, &[]);
|
||||
builder.switch_to_block(body_start);
|
||||
tier_up_dispatch_tail = Some(dispatch);
|
||||
Some(body_start)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let mut ip = 0usize;
|
||||
while ip < insns.len() {
|
||||
let insn = &insns[ip];
|
||||
|
|
@ -1889,6 +1907,22 @@ impl CraneliftCompiler {
|
|||
}
|
||||
}
|
||||
|
||||
op::SYNTHETIC_TIER_UP => {
|
||||
if let Some(tail) = tier_up_dispatch_tail {
|
||||
let header = control_stack
|
||||
.last()
|
||||
.expect("tier-up checkpoint must be inside a loop")
|
||||
.branch_target;
|
||||
let next_tail = builder.create_block();
|
||||
builder.switch_to_block(tail);
|
||||
let matches = builder.ins().icmp_imm(IntCC::Equal, tier_up_target_ip, insn.imm1);
|
||||
builder.ins().brif(matches, header, &[], next_tail, &[]);
|
||||
builder.seal_block(tail);
|
||||
tier_up_dispatch_tail = Some(next_tail);
|
||||
builder.switch_to_block(header);
|
||||
}
|
||||
}
|
||||
|
||||
_ => {
|
||||
return Err("unsupported instruction during codegen");
|
||||
}
|
||||
|
|
@ -1903,6 +1937,14 @@ impl CraneliftCompiler {
|
|||
builder.ins().jump(epilogue_block, &[]);
|
||||
}
|
||||
|
||||
if let Some(tail) = tier_up_dispatch_tail {
|
||||
let body_start = tier_up_body_start.expect("tier_up_body_start set when dispatch tail exists");
|
||||
builder.switch_to_block(tail);
|
||||
builder.ins().jump(body_start, &[]);
|
||||
builder.seal_block(tail);
|
||||
builder.seal_block(body_start);
|
||||
}
|
||||
|
||||
builder.switch_to_block(trap_block);
|
||||
builder.seal_block(trap_block);
|
||||
// Helper already set the trap for us.
|
||||
|
|
@ -2048,6 +2090,7 @@ impl CraneliftCompiler {
|
|||
| op::SYNTHETIC_I32_SUB2LOCAL..=op::SYNTHETIC_I32_SHRS2LOCAL
|
||||
| op::SYNTHETIC_I64_ADD2LOCAL..=op::SYNTHETIC_LOCAL_SETI64_CONST
|
||||
| op::SYNTHETIC_BR_TABLE_CONT
|
||||
| op::SYNTHETIC_TIER_UP
|
||||
)
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -589,6 +589,7 @@ public:
|
|||
struct Meta {
|
||||
u32 arity;
|
||||
u32 parameter_count;
|
||||
bool tier_up_eligible;
|
||||
};
|
||||
mutable Meta meta {};
|
||||
};
|
||||
|
|
@ -882,18 +883,45 @@ struct CompiledInstructions {
|
|||
Vector<Dispatch> dispatches;
|
||||
Vector<SourcesAndDestination> src_dst_mappings;
|
||||
InstructionStorage extra_instruction_storage;
|
||||
bool direct = false; // true if all dispatches contain handler_ptr, otherwise false and all contain instruction_opcode.
|
||||
bool cranelift_eligible = false; // true if this expression cleared the Cranelift type/shape checks during validation.
|
||||
u32 cranelift_result_arity = 0; // result count to hand to try_cranelift_compile(); only meaningful when cranelift_eligible.
|
||||
bool cranelift_compiled = false;
|
||||
|
||||
// Pointer/size_t-sized members first, then the u32, then the bools, so the trailing scalars pack
|
||||
// into one word instead of scattering padding between them.
|
||||
|
||||
// Native entry point for this function (conforms to the interpreter handler ABI). Zero until
|
||||
// the background/AOT compile has fully installed the code. Published with an atomic store-release
|
||||
// as the LAST step of install_compiled_function() and read with an atomic load-acquire at every
|
||||
// execution-decision site, so a function can tier up to JIT concurrently with execution without
|
||||
// a reader ever observing a half-installed function. dispatches[0].handler_ptr always stays the
|
||||
// C++ interpreter handler, so the interpreter path is valid regardless of compilation state.
|
||||
FlatPtr cranelift_entry = 0;
|
||||
void* cranelift_code_handle = nullptr; // Owned; freed when the owning Module is destroyed.
|
||||
size_t cranelift_code_size = 0;
|
||||
CraneliftTrap const* cranelift_traps = nullptr; // Owned by cranelift_code_handle.
|
||||
size_t cranelift_trap_count = 0;
|
||||
size_t max_call_arg_count = 0;
|
||||
size_t max_call_rec_size = 0;
|
||||
|
||||
u32 cranelift_result_arity = 0; // result count to hand to try_cranelift_compile(); only meaningful when cranelift_eligible.
|
||||
|
||||
bool direct = false; // true if all dispatches contain handler_ptr, otherwise false and all contain instruction_opcode.
|
||||
bool cranelift_eligible = false; // true if this expression cleared the Cranelift type/shape checks during validation.
|
||||
bool has_tier_up_checkpoints = false; // true if try_compile_instructions inserted synthetic_tier_up ops (Tier-Up sites).
|
||||
bool cranelift_compiled = false;
|
||||
};
|
||||
|
||||
// Read the native entry with acquire ordering: a non-zero result means the function is fully
|
||||
// installed and every cranelift_* field written before publication is visible to this thread.
|
||||
inline FlatPtr cranelift_entry_acquire(CompiledInstructions const& ci)
|
||||
{
|
||||
return AK::atomic_load(const_cast<FlatPtr volatile*>(&ci.cranelift_entry), AK::MemoryOrder::memory_order_acquire);
|
||||
}
|
||||
|
||||
// Publish the native entry with release ordering. Must be the LAST write of install.
|
||||
inline void publish_cranelift_entry(CompiledInstructions& ci, FlatPtr entry)
|
||||
{
|
||||
AK::atomic_store(&ci.cranelift_entry, entry, AK::MemoryOrder::memory_order_release);
|
||||
}
|
||||
|
||||
template<Enum auto... Vs>
|
||||
consteval auto as_ordered()
|
||||
{
|
||||
|
|
@ -1496,6 +1524,8 @@ struct ModuleStats {
|
|||
AK::Duration cranelift_time;
|
||||
size_t cranelift_blob_size_bytes { 0 };
|
||||
size_t function_count { 0 };
|
||||
size_t tier_up_function_count { 0 }; // functions instrumented with tier-up checkpoints
|
||||
size_t tier_up_checkpoint_count { 0 }; // total tier-up checkpoints inserted across the module
|
||||
bool cache_hit { false };
|
||||
};
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue