diff --git a/Libraries/LibJS/Bytecode/AsmInterpreter/AsmInterpreter.cpp b/Libraries/LibJS/Bytecode/AsmInterpreter/AsmInterpreter.cpp index dc4a8010c9..d16a5b980c 100644 --- a/Libraries/LibJS/Bytecode/AsmInterpreter/AsmInterpreter.cpp +++ b/Libraries/LibJS/Bytecode/AsmInterpreter/AsmInterpreter.cpp @@ -915,9 +915,8 @@ i64 asm_try_put_by_id_cache(VM* vm, u32 pc) auto value = vm->get(insn.src()); auto& cache = vm->current_executable().property_lookup_caches[insn.cache()]; - for (size_t i = 0; i < cache.entries.size(); ++i) { - auto& entry = cache.entries[i]; - switch (cache.types[i]) { + for (auto& entry : cache.entries()) { + switch (entry.type) { case PropertyLookupCache::Entry::Type::ChangeOwnProperty: { auto cached_shape = entry.shape.ptr(); if (cached_shape != &object.shape()) [[unlikely]] @@ -970,7 +969,12 @@ i64 asm_try_get_by_id_cache(VM* vm, u32 pc) auto& shape = object.shape(); auto& cache = vm->current_executable().property_lookup_caches[insn.cache()]; - for (auto& entry : cache.entries) { + for (auto& entry : cache.entries()) { + if (entry.type != PropertyLookupCache::Entry::Type::GetOwnProperty + && entry.type != PropertyLookupCache::Entry::Type::GetPropertyInPrototypeChain) { + continue; + } + auto cached_prototype = entry.prototype.ptr(); if (cached_prototype) { if (&shape != entry.shape) [[unlikely]] diff --git a/Libraries/LibJS/Bytecode/AsmInterpreter/asmint.asm b/Libraries/LibJS/Bytecode/AsmInterpreter/asmint.asm index 34f8cc998c..cb21a55ac4 100644 --- a/Libraries/LibJS/Bytecode/AsmInterpreter/asmint.asm +++ b/Libraries/LibJS/Bytecode/AsmInterpreter/asmint.asm @@ -499,13 +499,18 @@ macro pop_inline_frame_and_resume(caller_frame, value_reg) dispatch_current end -macro load_property_lookup_cache(cache) +macro load_property_lookup_cache(cache, fail_label) temp exe, caches load32 cache, [pb, pc, m_cache] mul cache, cache, PROPERTY_LOOKUP_CACHE_SIZE load64 exe, [exec_ctx, EXECUTION_CONTEXT_EXECUTABLE] load64 caches, [exe, EXECUTABLE_PROPERTY_LOOKUP_CACHES_DATA] add cache, caches + load64 cache, [cache, PROPERTY_LOOKUP_CACHE_DATA] + branch_zero cache, fail_label + branch_bits_clear cache, PROPERTY_LOOKUP_CACHE_POLYMORPHIC_DATA_TAG, .data_loaded + sub cache, PROPERTY_LOOKUP_CACHE_POLYMORPHIC_DATA_TAG +.data_loaded: end macro load_global_variable_cache(cache) @@ -1685,13 +1690,13 @@ handler GetById unbox_object obj, base load64 shape, [obj, OBJECT_SHAPE] assert_nonzero shape - load_property_lookup_cache plc + load_property_lookup_cache plc, .try_cache assert_nonzero plc - load_pair64 cache_shape, cache_proto, [plc, PROPERTY_LOOKUP_CACHE_ENTRY0_SHAPE], [plc, PROPERTY_LOOKUP_CACHE_ENTRY0_PROTOTYPE] + load_pair64 cache_shape, cache_proto, [plc, PROPERTY_LOOKUP_CACHE_ENTRY_SHAPE], [plc, PROPERTY_LOOKUP_CACHE_ENTRY_PROTOTYPE] branch_ne cache_shape, shape, .try_cache branch_nonzero cache_proto, .proto # Check dictionary generation matches - load_pair32 prop_offset, dict_gen, [plc, PROPERTY_LOOKUP_CACHE_ENTRY0_PROPERTY_OFFSET], [plc, PROPERTY_LOOKUP_CACHE_ENTRY0_DICTIONARY_GENERATION] + load_pair32 prop_offset, dict_gen, [plc, PROPERTY_LOOKUP_CACHE_ENTRY_PROPERTY_OFFSET], [plc, PROPERTY_LOOKUP_CACHE_ENTRY_DICTIONARY_GENERATION] load32 cur_dict_gen, [shape, SHAPE_DICTIONARY_GENERATION] branch_ne dict_gen, cur_dict_gen, .try_cache # IC hit! Load property value via get_direct (own property) @@ -1705,11 +1710,11 @@ handler GetById dispatch_next .proto: # cache_proto = prototype Object*, shape = object's shape, plc = PLC base - load64 prop_offset, [plc, PROPERTY_LOOKUP_CACHE_ENTRY0_PROTOTYPE_CHAIN_VALIDITY] + load64 prop_offset, [plc, PROPERTY_LOOKUP_CACHE_ENTRY_PROTOTYPE_CHAIN_VALIDITY] branch_zero prop_offset, .try_cache load8 tag, [prop_offset, PROTOTYPE_CHAIN_VALIDITY_VALID] branch_zero tag, .try_cache - load_pair32 prop_offset, dict_gen, [plc, PROPERTY_LOOKUP_CACHE_ENTRY0_PROPERTY_OFFSET], [plc, PROPERTY_LOOKUP_CACHE_ENTRY0_DICTIONARY_GENERATION] + load_pair32 prop_offset, dict_gen, [plc, PROPERTY_LOOKUP_CACHE_ENTRY_PROPERTY_OFFSET], [plc, PROPERTY_LOOKUP_CACHE_ENTRY_DICTIONARY_GENERATION] load32 cur_dict_gen, [shape, SHAPE_DICTIONARY_GENERATION] branch_ne dict_gen, cur_dict_gen, .try_cache load64 props, [cache_proto, OBJECT_NAMED_PROPERTIES] @@ -1738,12 +1743,12 @@ handler PutById unbox_object obj, base load64 shape, [obj, OBJECT_SHAPE] assert_nonzero shape - load_property_lookup_cache plc + load_property_lookup_cache plc, .try_cache assert_nonzero plc - load_pair64 cache_shape, cache_proto, [plc, PROPERTY_LOOKUP_CACHE_ENTRY0_SHAPE], [plc, PROPERTY_LOOKUP_CACHE_ENTRY0_PROTOTYPE] + load_pair64 cache_shape, cache_proto, [plc, PROPERTY_LOOKUP_CACHE_ENTRY_SHAPE], [plc, PROPERTY_LOOKUP_CACHE_ENTRY_PROTOTYPE] branch_ne cache_shape, shape, .try_cache branch_nonzero cache_proto, .try_cache - load_pair32 prop_offset, dict_gen, [plc, PROPERTY_LOOKUP_CACHE_ENTRY0_PROPERTY_OFFSET], [plc, PROPERTY_LOOKUP_CACHE_ENTRY0_DICTIONARY_GENERATION] + load_pair32 prop_offset, dict_gen, [plc, PROPERTY_LOOKUP_CACHE_ENTRY_PROPERTY_OFFSET], [plc, PROPERTY_LOOKUP_CACHE_ENTRY_DICTIONARY_GENERATION] load32 cur_dict_gen, [shape, SHAPE_DICTIONARY_GENERATION] branch_ne dict_gen, cur_dict_gen, .try_cache # Check current value at prop_offset is not an accessor @@ -1916,12 +1921,12 @@ handler GetLength # Non-magical length: IC fast path (same as GetById) load64 shape, [obj, OBJECT_SHAPE] assert_nonzero shape - load_property_lookup_cache plc + load_property_lookup_cache plc, .slow assert_nonzero plc - load_pair64 cache_shape, cache_proto, [plc, PROPERTY_LOOKUP_CACHE_ENTRY0_SHAPE], [plc, PROPERTY_LOOKUP_CACHE_ENTRY0_PROTOTYPE] + load_pair64 cache_shape, cache_proto, [plc, PROPERTY_LOOKUP_CACHE_ENTRY_SHAPE], [plc, PROPERTY_LOOKUP_CACHE_ENTRY_PROTOTYPE] branch_ne cache_shape, shape, .slow branch_nonzero cache_proto, .slow - load_pair32 prop_offset, dict_gen, [plc, PROPERTY_LOOKUP_CACHE_ENTRY0_PROPERTY_OFFSET], [plc, PROPERTY_LOOKUP_CACHE_ENTRY0_DICTIONARY_GENERATION] + load_pair32 prop_offset, dict_gen, [plc, PROPERTY_LOOKUP_CACHE_ENTRY_PROPERTY_OFFSET], [plc, PROPERTY_LOOKUP_CACHE_ENTRY_DICTIONARY_GENERATION] load32 cur_dict_gen, [shape, SHAPE_DICTIONARY_GENERATION] branch_ne dict_gen, cur_dict_gen, .slow load64 props, [obj, OBJECT_NAMED_PROPERTIES] @@ -1967,10 +1972,10 @@ handler GetGlobal # (falls through to env binding path on shape mismatch) load64 shape, [global_object, OBJECT_SHAPE] assert_nonzero shape - load64 cache_shape, [gvc, PROPERTY_LOOKUP_CACHE_ENTRY0_SHAPE] + load64 cache_shape, [gvc, GLOBAL_VARIABLE_CACHE_ENTRY_SHAPE] branch_ne cache_shape, shape, .try_env_binding load32 cur_dict_gen, [shape, SHAPE_DICTIONARY_GENERATION] - load_pair32 prop_offset, dict_gen, [gvc, PROPERTY_LOOKUP_CACHE_ENTRY0_PROPERTY_OFFSET], [gvc, PROPERTY_LOOKUP_CACHE_ENTRY0_DICTIONARY_GENERATION] + load_pair32 prop_offset, dict_gen, [gvc, GLOBAL_VARIABLE_CACHE_ENTRY_PROPERTY_OFFSET], [gvc, GLOBAL_VARIABLE_CACHE_ENTRY_DICTIONARY_GENERATION] branch_ne dict_gen, cur_dict_gen, .try_env_binding # IC hit! Load property value via get_direct load64 props, [global_object, OBJECT_NAMED_PROPERTIES] @@ -2013,10 +2018,10 @@ handler SetGlobal branch_ne cache_serial, env_serial, .slow load64 shape, [global_object, OBJECT_SHAPE] assert_nonzero shape - load64 cache_shape, [gvc, PROPERTY_LOOKUP_CACHE_ENTRY0_SHAPE] + load64 cache_shape, [gvc, GLOBAL_VARIABLE_CACHE_ENTRY_SHAPE] branch_ne cache_shape, shape, .try_env_binding load32 cur_dict_gen, [shape, SHAPE_DICTIONARY_GENERATION] - load_pair32 prop_offset, dict_gen, [gvc, PROPERTY_LOOKUP_CACHE_ENTRY0_PROPERTY_OFFSET], [gvc, PROPERTY_LOOKUP_CACHE_ENTRY0_DICTIONARY_GENERATION] + load_pair32 prop_offset, dict_gen, [gvc, GLOBAL_VARIABLE_CACHE_ENTRY_PROPERTY_OFFSET], [gvc, GLOBAL_VARIABLE_CACHE_ENTRY_DICTIONARY_GENERATION] branch_ne dict_gen, cur_dict_gen, .try_env_binding # IC hit! Load current value to check it's not an accessor. load64 props, [global_object, OBJECT_NAMED_PROPERTIES] diff --git a/Libraries/LibJS/Bytecode/AsmInterpreter/gen_asm_offsets.cpp b/Libraries/LibJS/Bytecode/AsmInterpreter/gen_asm_offsets.cpp index 955ab0a0c9..00b4019d1c 100644 --- a/Libraries/LibJS/Bytecode/AsmInterpreter/gen_asm_offsets.cpp +++ b/Libraries/LibJS/Bytecode/AsmInterpreter/gen_asm_offsets.cpp @@ -76,7 +76,8 @@ int main() // PropertyLookupCache layout outln("\n# PropertyLookupCache layout"); - EMIT_OFFSET(PROPERTY_LOOKUP_CACHE_ENTRIES, PropertyLookupCache, entries); + EMIT_OFFSET(PROPERTY_LOOKUP_CACHE_DATA, PropertyLookupCache, m_data); + outln("const PROPERTY_LOOKUP_CACHE_POLYMORPHIC_DATA_TAG = {}", PropertyLookupCache::polymorphic_data_tag); EMIT_SIZEOF(PROPERTY_LOOKUP_CACHE_SIZE, PropertyLookupCache); // PropertyLookupCache::Entry layout @@ -89,15 +90,6 @@ int main() EMIT_OFFSET(PROPERTY_LOOKUP_CACHE_ENTRY_PROTOTYPE_CHAIN_VALIDITY, PropertyLookupCache::Entry, prototype_chain_validity); EMIT_SIZEOF(PROPERTY_LOOKUP_CACHE_ENTRY_SIZE, PropertyLookupCache::Entry); - // Composite offsets for entry[0] within a PropertyLookupCache - outln("\n# Entry[0] offsets within PropertyLookupCache"); - auto plc_entries = offsetof(PropertyLookupCache, entries); - outln("const PROPERTY_LOOKUP_CACHE_ENTRY0_PROPERTY_OFFSET = {}", plc_entries + offsetof(PropertyLookupCache::Entry, property_offset)); - outln("const PROPERTY_LOOKUP_CACHE_ENTRY0_DICTIONARY_GENERATION = {}", plc_entries + offsetof(PropertyLookupCache::Entry, shape_dictionary_generation)); - outln("const PROPERTY_LOOKUP_CACHE_ENTRY0_SHAPE = {}", plc_entries + offsetof(PropertyLookupCache::Entry, shape)); - outln("const PROPERTY_LOOKUP_CACHE_ENTRY0_PROTOTYPE = {}", plc_entries + offsetof(PropertyLookupCache::Entry, prototype)); - outln("const PROPERTY_LOOKUP_CACHE_ENTRY0_PROTOTYPE_CHAIN_VALIDITY = {}", plc_entries + offsetof(PropertyLookupCache::Entry, prototype_chain_validity)); - // ObjectPropertyIteratorCacheData layout outln("\n# ObjectPropertyIteratorCacheData layout"); EMIT_OFFSET(OBJECT_PROPERTY_ITERATOR_CACHE_DATA_PROPERTIES, ObjectPropertyIteratorCacheData, m_properties); @@ -240,11 +232,15 @@ int main() // GlobalVariableCache layout outln("\n# GlobalVariableCache layout"); + EMIT_OFFSET(GLOBAL_VARIABLE_CACHE_ENTRY, GlobalVariableCache, entry); EMIT_OFFSET(GLOBAL_VARIABLE_CACHE_ENVIRONMENT_SERIAL, GlobalVariableCache, environment_serial_number); EMIT_OFFSET(GLOBAL_VARIABLE_CACHE_ENVIRONMENT_BINDING_INDEX, GlobalVariableCache, environment_binding_index); EMIT_OFFSET(GLOBAL_VARIABLE_CACHE_HAS_ENVIRONMENT_BINDING, GlobalVariableCache, has_environment_binding_index); EMIT_OFFSET(GLOBAL_VARIABLE_CACHE_IN_MODULE_ENVIRONMENT, GlobalVariableCache, in_module_environment); EMIT_SIZEOF(GLOBAL_VARIABLE_CACHE_SIZE, GlobalVariableCache); + outln("const GLOBAL_VARIABLE_CACHE_ENTRY_PROPERTY_OFFSET = {}", offsetof(GlobalVariableCache, entry) + offsetof(PropertyLookupCache::Entry, property_offset)); + outln("const GLOBAL_VARIABLE_CACHE_ENTRY_DICTIONARY_GENERATION = {}", offsetof(GlobalVariableCache, entry) + offsetof(PropertyLookupCache::Entry, shape_dictionary_generation)); + outln("const GLOBAL_VARIABLE_CACHE_ENTRY_SHAPE = {}", offsetof(GlobalVariableCache, entry) + offsetof(PropertyLookupCache::Entry, shape)); // Builtin enum values outln("\n# Builtin enum values"); diff --git a/Libraries/LibJS/Bytecode/Executable.cpp b/Libraries/LibJS/Bytecode/Executable.cpp index 6664ee9f4f..37924ec905 100644 --- a/Libraries/LibJS/Bytecode/Executable.cpp +++ b/Libraries/LibJS/Bytecode/Executable.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -67,6 +68,152 @@ size_t InstructionStream::external_memory_size() const }); } +static_assert(alignof(PropertyLookupCache::MonomorphicData) > PropertyLookupCache::polymorphic_data_tag); +static_assert(alignof(PropertyLookupCache::PolymorphicData) > PropertyLookupCache::polymorphic_data_tag); +static_assert(offsetof(PropertyLookupCache::MonomorphicData, entry) == 0); +static_assert(offsetof(PropertyLookupCache::PolymorphicData, entries) == 0); + +PropertyLookupCache::PropertyLookupCache(PropertyLookupCache&& other) + : m_data(exchange(other.m_data, 0)) +{ +} + +PropertyLookupCache& PropertyLookupCache::operator=(PropertyLookupCache&& other) +{ + if (this != &other) { + clear(); + m_data = exchange(other.m_data, 0); + } + return *this; +} + +PropertyLookupCache::~PropertyLookupCache() +{ + clear(); +} + +PropertyLookupCache::MonomorphicData* PropertyLookupCache::monomorphic_data() +{ + if (!m_data || (m_data & polymorphic_data_tag)) + return nullptr; + return reinterpret_cast(m_data); +} + +PropertyLookupCache::MonomorphicData const* PropertyLookupCache::monomorphic_data() const +{ + if (!m_data || (m_data & polymorphic_data_tag)) + return nullptr; + return reinterpret_cast(m_data); +} + +PropertyLookupCache::PolymorphicData* PropertyLookupCache::polymorphic_data() +{ + if (!(m_data & polymorphic_data_tag)) + return nullptr; + return reinterpret_cast(m_data & ~polymorphic_data_tag); +} + +PropertyLookupCache::PolymorphicData const* PropertyLookupCache::polymorphic_data() const +{ + if (!(m_data & polymorphic_data_tag)) + return nullptr; + return reinterpret_cast(m_data & ~polymorphic_data_tag); +} + +void PropertyLookupCache::set_monomorphic_data(MonomorphicData* data) +{ + VERIFY(data); + VERIFY(!(reinterpret_cast(data) & polymorphic_data_tag)); + m_data = reinterpret_cast(data); +} + +void PropertyLookupCache::set_polymorphic_data(PolymorphicData* data) +{ + VERIFY(data); + VERIFY(!(reinterpret_cast(data) & polymorphic_data_tag)); + m_data = reinterpret_cast(data) | polymorphic_data_tag; +} + +PropertyLookupCache::Entry* PropertyLookupCache::first_entry() +{ + if (auto* data = monomorphic_data()) + return &data->entry; + if (auto* data = polymorphic_data()) + return &data->entries[0]; + return nullptr; +} + +PropertyLookupCache::Entry const* PropertyLookupCache::first_entry() const +{ + if (auto* data = monomorphic_data()) + return &data->entry; + if (auto* data = polymorphic_data()) + return &data->entries[0]; + return nullptr; +} + +Span PropertyLookupCache::entries() +{ + if (auto* data = monomorphic_data()) + return { &data->entry, 1 }; + if (auto* data = polymorphic_data()) + return data->entries.span(); + return {}; +} + +ReadonlySpan PropertyLookupCache::entries() const +{ + if (auto* data = monomorphic_data()) + return { &data->entry, 1 }; + if (auto* data = polymorphic_data()) + return data->entries.span(); + return {}; +} + +size_t PropertyLookupCache::external_memory_size() const +{ + if (monomorphic_data()) + return sizeof(MonomorphicData); + if (polymorphic_data()) + return sizeof(PolymorphicData); + return 0; +} + +void PropertyLookupCache::clear() +{ + if (auto* data = monomorphic_data()) { + delete data; + m_data = 0; + return; + } + if (auto* data = polymorphic_data()) { + delete data; + m_data = 0; + } +} + +bool PropertyLookupCache::entries_have_same_cache_key(Entry const& a, Entry const& b) +{ + if (a.type == Entry::Type::Empty || b.type == Entry::Type::Empty) + return false; + if (a.type != b.type) + return false; + + switch (a.type) { + case Entry::Type::AddOwnProperty: + return a.from_shape == b.from_shape && a.shape == b.shape; + case Entry::Type::ChangeOwnProperty: + case Entry::Type::GetOwnProperty: + return a.shape == b.shape; + case Entry::Type::ChangePropertyInPrototypeChain: + case Entry::Type::GetPropertyInPrototypeChain: + return a.shape == b.shape && a.prototype == b.prototype; + case Entry::Type::Empty: + VERIFY_NOT_REACHED(); + } + VERIFY_NOT_REACHED(); +} + ObjectPropertyIteratorCacheData::ObjectPropertyIteratorCacheData(VM& vm, Vector properties, ObjectPropertyIteratorFastPath fast_path, u32 indexed_property_count, bool receiver_has_magical_length_property, GC::Ref shape, GC::Ptr prototype_chain_validity) : m_properties(move(properties)) , m_shape(shape) @@ -413,6 +560,8 @@ size_t Executable::external_memory_size() const { size_t size = bytecode.external_memory_size(); size = saturating_add_external_memory_size(size, vector_external_memory_size(property_lookup_caches)); + for (auto const& cache : property_lookup_caches) + size = saturating_add_external_memory_size(size, cache.external_memory_size()); size = saturating_add_external_memory_size(size, vector_external_memory_size(global_variable_caches)); size = saturating_add_external_memory_size(size, vector_external_memory_size(template_object_caches)); size = saturating_add_external_memory_size(size, vector_external_memory_size(object_shape_caches)); @@ -469,7 +618,7 @@ static void clear_cache_entry_if_dead(PropertyLookupCache::Entry& entry) void StaticPropertyLookupCache::sweep_all() { for (auto* cache : static_property_lookup_caches()) { - for (auto& entry : cache->entries) + for (auto& entry : cache->entries()) clear_cache_entry_if_dead(entry); } } @@ -477,13 +626,11 @@ void StaticPropertyLookupCache::sweep_all() void Executable::remove_dead_cells(Badge) { for (auto& cache : property_lookup_caches) { - for (auto& entry : cache.entries) - clear_cache_entry_if_dead(entry); - } - for (auto& cache : global_variable_caches) { - for (auto& entry : cache.entries) + for (auto& entry : cache.entries()) clear_cache_entry_if_dead(entry); } + for (auto& cache : global_variable_caches) + clear_cache_entry_if_dead(cache.entry); for (auto& cache : object_shape_caches) { auto* shape = cache.shape.ptr(); if (shape && cell_is_dead(shape)) diff --git a/Libraries/LibJS/Bytecode/Executable.h b/Libraries/LibJS/Bytecode/Executable.h index 05a2d705b3..707658b8cf 100644 --- a/Libraries/LibJS/Bytecode/Executable.h +++ b/Libraries/LibJS/Bytecode/Executable.h @@ -9,7 +9,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -55,7 +57,7 @@ private: size_t m_size { 0 }; }; -// Represents one polymorphic inline cache used for property lookups. +// Represents one tiered inline cache used for property lookups. struct PropertyLookupCache { static constexpr size_t max_number_of_shapes_to_remember = 4; struct Entry { @@ -67,6 +69,7 @@ struct PropertyLookupCache { ChangePropertyInPrototypeChain, GetPropertyInPrototypeChain, }; + Type type { Type::Empty }; u32 property_offset { 0 }; u32 shape_dictionary_generation { 0 }; GC::RawPtr from_shape; @@ -75,20 +78,82 @@ struct PropertyLookupCache { GC::RawPtr prototype_chain_validity; }; + struct MonomorphicData { + Entry entry; + }; + + struct PolymorphicData { + AK::Array entries; + }; + + PropertyLookupCache() = default; + PropertyLookupCache(PropertyLookupCache const&) = delete; + PropertyLookupCache& operator=(PropertyLookupCache const&) = delete; + PropertyLookupCache(PropertyLookupCache&&); + PropertyLookupCache& operator=(PropertyLookupCache&&); + ~PropertyLookupCache(); + + [[nodiscard]] Entry* first_entry(); + [[nodiscard]] Entry const* first_entry() const; + [[nodiscard]] Span entries(); + [[nodiscard]] ReadonlySpan entries() const; + [[nodiscard]] size_t external_memory_size() const; + void update(Entry::Type type, auto callback) { - // First, move all entries one step back. - for (size_t i = entries.size() - 1; i >= 1; --i) { - types[i] = types[i - 1]; - entries[i] = entries[i - 1]; + Entry new_entry; + new_entry.type = type; + callback(new_entry); + + if (!m_data) { + auto data = make(); + data->entry = new_entry; + set_monomorphic_data(data.leak_ptr()); + return; } - types[0] = type; - entries[0] = {}; - callback(entries[0]); + + if (auto* data = monomorphic_data()) { + if (entries_have_same_cache_key(data->entry, new_entry)) { + data->entry = new_entry; + return; + } + + auto old_entry = data->entry; + auto new_data = make(); + new_data->entries[0] = new_entry; + new_data->entries[1] = old_entry; + clear(); + set_polymorphic_data(new_data.leak_ptr()); + return; + } + + auto& entries = polymorphic_data()->entries; + size_t insertion_index = entries.size() - 1; + for (size_t i = 0; i < entries.size(); ++i) { + if (entries_have_same_cache_key(entries[i], new_entry)) { + insertion_index = i; + break; + } + } + + for (size_t i = insertion_index; i > 0; --i) + entries[i] = entries[i - 1]; + entries[0] = new_entry; } - AK::Array types; - AK::Array entries; + void clear(); + + static constexpr FlatPtr polymorphic_data_tag = 1; + FlatPtr m_data { 0 }; + +private: + [[nodiscard]] MonomorphicData* monomorphic_data(); + [[nodiscard]] MonomorphicData const* monomorphic_data() const; + [[nodiscard]] PolymorphicData* polymorphic_data(); + [[nodiscard]] PolymorphicData const* polymorphic_data() const; + void set_monomorphic_data(MonomorphicData*); + void set_polymorphic_data(PolymorphicData*); + static bool entries_have_same_cache_key(Entry const&, Entry const&); }; // A PropertyLookupCache for use as a static local variable. @@ -98,7 +163,29 @@ struct StaticPropertyLookupCache : public PropertyLookupCache { static void sweep_all(); }; -struct GlobalVariableCache : public PropertyLookupCache { +struct GlobalVariableCache { + PropertyLookupCache::Entry* first_entry() + { + if (entry.type == PropertyLookupCache::Entry::Type::Empty) + return nullptr; + return &entry; + } + + PropertyLookupCache::Entry const* first_entry() const + { + if (entry.type == PropertyLookupCache::Entry::Type::Empty) + return nullptr; + return &entry; + } + + void update(PropertyLookupCache::Entry::Type type, auto callback) + { + entry = {}; + entry.type = type; + callback(entry); + } + + PropertyLookupCache::Entry entry; u64 environment_serial_number { 0 }; u32 environment_binding_index { 0 }; bool has_environment_binding_index { false }; diff --git a/Libraries/LibJS/Bytecode/Interpreter.cpp b/Libraries/LibJS/Bytecode/Interpreter.cpp index b358acbdc4..dc8697f061 100644 --- a/Libraries/LibJS/Bytecode/Interpreter.cpp +++ b/Libraries/LibJS/Bytecode/Interpreter.cpp @@ -1046,8 +1046,9 @@ inline ThrowCompletionOr get_global(VM& vm, IdentifierTableIndex identifi // OPTIMIZATION: For global var bindings, if the shape of the global object hasn't changed, // we can use the cached property offset. - if (&shape == cache.entries[0].shape && (!shape.is_dictionary() || shape.dictionary_generation() == cache.entries[0].shape_dictionary_generation)) { - auto value = binding_object.get_direct(cache.entries[0].property_offset); + auto* entry = cache.first_entry(); + if (entry && &shape == entry->shape && (!shape.is_dictionary() || shape.dictionary_generation() == entry->shape_dictionary_generation)) { + auto value = binding_object.get_direct(entry->property_offset); if (value.is_accessor()) return TRY(call(vm, value.as_accessor().getter(), &binding_object)); return value; @@ -1096,12 +1097,14 @@ inline ThrowCompletionOr get_global(VM& vm, IdentifierTableIndex identifi CacheableGetPropertyMetadata cacheable_metadata; auto value = TRY(binding_object.internal_get(identifier, &binding_object, &cacheable_metadata)); if (cacheable_metadata.type == CacheableGetPropertyMetadata::Type::GetOwnProperty) { - cache.entries[0].shape = shape; - cache.entries[0].property_offset = cacheable_metadata.property_offset.value(); + cache.update(PropertyLookupCache::Entry::Type::GetOwnProperty, [&](auto& entry) { + entry.shape = shape; + entry.property_offset = cacheable_metadata.property_offset.value(); - if (shape.is_dictionary()) { - cache.entries[0].shape_dictionary_generation = shape.dictionary_generation(); - } + if (shape.is_dictionary()) { + entry.shape_dictionary_generation = shape.dictionary_generation(); + } + }); } return value; } @@ -2396,12 +2399,13 @@ ThrowCompletionOr SetGlobal::execute_impl(VM& vm) const if (cache.environment_serial_number == declarative_record.environment_serial_number()) { // OPTIMIZATION: For global var bindings, if the shape of the global object hasn't changed, // we can use the cached property offset. - if (&shape == cache.entries[0].shape && (!shape.is_dictionary() || shape.dictionary_generation() == cache.entries[0].shape_dictionary_generation)) { - auto value = binding_object.get_direct(cache.entries[0].property_offset); + auto* entry = cache.first_entry(); + if (entry && &shape == entry->shape && (!shape.is_dictionary() || shape.dictionary_generation() == entry->shape_dictionary_generation)) { + auto value = binding_object.get_direct(entry->property_offset); if (value.is_accessor()) TRY(call(vm, value.as_accessor().setter(), &binding_object, src)); else - binding_object.put_direct(cache.entries[0].property_offset, src); + binding_object.put_direct(entry->property_offset, src); return {}; } @@ -2463,12 +2467,14 @@ ThrowCompletionOr SetGlobal::execute_impl(VM& vm) const return vm.throw_completion(ErrorType::ObjectSetReturnedFalse); } if (cacheable_metadata.type == CacheableSetPropertyMetadata::Type::ChangeOwnProperty) { - cache.entries[0].shape = shape; - cache.entries[0].property_offset = cacheable_metadata.property_offset.value(); + cache.update(PropertyLookupCache::Entry::Type::ChangeOwnProperty, [&](auto& entry) { + entry.shape = shape; + entry.property_offset = cacheable_metadata.property_offset.value(); - if (shape.is_dictionary()) { - cache.entries[0].shape_dictionary_generation = shape.dictionary_generation(); - } + if (shape.is_dictionary()) { + entry.shape_dictionary_generation = shape.dictionary_generation(); + } + }); } return {}; } diff --git a/Libraries/LibJS/Bytecode/PropertyAccess.h b/Libraries/LibJS/Bytecode/PropertyAccess.h index d6b4cc37de..28ce064723 100644 --- a/Libraries/LibJS/Bytecode/PropertyAccess.h +++ b/Libraries/LibJS/Bytecode/PropertyAccess.h @@ -89,7 +89,12 @@ ALWAYS_INLINE ThrowCompletionOr get_by_id(VM& vm, GetBaseIdentifier get_b auto& shape = base_obj->shape(); - for (auto& cache_entry : cache.entries) { + for (auto& cache_entry : cache.entries()) { + if (cache_entry.type != PropertyLookupCache::Entry::Type::GetOwnProperty + && cache_entry.type != PropertyLookupCache::Entry::Type::GetPropertyInPrototypeChain) { + continue; + } + auto cached_prototype = cache_entry.prototype.ptr(); if (cached_prototype) { // OPTIMIZATION: If the prototype chain hasn't been mutated in a way that would invalidate the cache, we can use it. @@ -145,31 +150,26 @@ ALWAYS_INLINE ThrowCompletionOr get_by_id(VM& vm, GetBaseIdentifier get_b // that collected metadata is valid, e.g. if getter in prototype chain added // property with the same name into the object itself. if (&shape == &base_obj->shape()) { - auto get_cache_slot = [&] -> PropertyLookupCache::Entry& { - for (size_t i = cache.entries.size() - 1; i >= 1; --i) { - cache.entries[i] = cache.entries[i - 1]; - } - cache.entries[0] = {}; - return cache.entries[0]; - }; if (cacheable_metadata.type == CacheableGetPropertyMetadata::Type::GetOwnProperty) { - auto& entry = get_cache_slot(); - entry.shape = shape; - entry.property_offset = cacheable_metadata.property_offset.value(); + cache.update(PropertyLookupCache::Entry::Type::GetOwnProperty, [&](auto& entry) { + entry.shape = shape; + entry.property_offset = cacheable_metadata.property_offset.value(); - if (shape.is_dictionary()) { - entry.shape_dictionary_generation = shape.dictionary_generation(); - } + if (shape.is_dictionary()) { + entry.shape_dictionary_generation = shape.dictionary_generation(); + } + }); } else if (cacheable_metadata.type == CacheableGetPropertyMetadata::Type::GetPropertyInPrototypeChain) { - auto& entry = get_cache_slot(); - entry.shape = &base_obj->shape(); - entry.property_offset = cacheable_metadata.property_offset.value(); - entry.prototype = const_cast(cacheable_metadata.prototype.ptr()); - entry.prototype_chain_validity = prototype_chain_validity; + cache.update(PropertyLookupCache::Entry::Type::GetPropertyInPrototypeChain, [&](auto& entry) { + entry.shape = &base_obj->shape(); + entry.property_offset = cacheable_metadata.property_offset.value(); + entry.prototype = const_cast(cacheable_metadata.prototype.ptr()); + entry.prototype_chain_validity = prototype_chain_validity; - if (shape.is_dictionary()) { - entry.shape_dictionary_generation = shape.dictionary_generation(); - } + if (shape.is_dictionary()) { + entry.shape_dictionary_generation = shape.dictionary_generation(); + } + }); } } @@ -233,12 +233,11 @@ inline ThrowCompletionOr put_by_property_key(VM& vm, Value base, Value thi auto this_value_object = MUST(this_value.to_object(vm)); auto& from_shape = this_value_object->shape(); if (caches) [[likely]] { - for (size_t i = 0; i < caches->entries.size(); ++i) { - switch (caches->types[i]) { + for (auto& cache : caches->entries()) { + switch (cache.type) { case PropertyLookupCache::Entry::Type::Empty: break; case PropertyLookupCache::Entry::Type::ChangePropertyInPrototypeChain: { - auto& cache = caches->entries[i]; auto cached_prototype = cache.prototype.ptr(); if (!cached_prototype) [[unlikely]] break; @@ -270,7 +269,6 @@ inline ThrowCompletionOr put_by_property_key(VM& vm, Value base, Value thi break; } case PropertyLookupCache::Entry::Type::ChangeOwnProperty: { - auto& cache = caches->entries[i]; auto cached_shape = cache.shape.ptr(); if (cached_shape != &object->shape()) [[unlikely]] break; @@ -289,7 +287,6 @@ inline ThrowCompletionOr put_by_property_key(VM& vm, Value base, Value thi return {}; } case PropertyLookupCache::Entry::Type::AddOwnProperty: { - auto& cache = caches->entries[i]; // OPTIMIZATION: If the object's shape is the same as the one cached before adding the new property, we can // reuse the resulting shape from the cache. if (cache.from_shape != &object->shape()) [[unlikely]] @@ -315,8 +312,9 @@ inline ThrowCompletionOr put_by_property_key(VM& vm, Value base, Value thi object->put_direct(cache.property_offset, value); return {}; } - default: - VERIFY_NOT_REACHED(); + case PropertyLookupCache::Entry::Type::GetOwnProperty: + case PropertyLookupCache::Entry::Type::GetPropertyInPrototypeChain: + break; } } } @@ -385,9 +383,8 @@ inline ThrowCompletionOr put_by_property_key(VM& vm, Value base, Value thi } case PutKind::Own: { if (caches) [[likely]] { - for (size_t i = 0; i < caches->entries.size(); ++i) { - if (caches->types[i] == PropertyLookupCache::Entry::Type::AddOwnProperty) { - auto& cache = caches->entries[i]; + for (auto& cache : caches->entries()) { + if (cache.type == PropertyLookupCache::Entry::Type::AddOwnProperty) { if (cache.from_shape != &object->shape()) [[unlikely]] continue; auto cached_shape = cache.shape.ptr();