LibJS: Reduce inline cache memory usage
Store bytecode property lookup caches as tiered handles instead of eagerly allocating four entries for every cache slot. Each slot starts empty, grows to one monomorphic entry after the first cacheable lookup, and promotes to the existing four-entry table when another cache key appears. Global variable caches keep one inline property entry because they usually warm up as global object property accesses. This keeps common global access allocation-free while still avoiding the inherited polymorphic cache cost. Generated asm offsets now point at the lazy property-cache storage and the inline global entry, and asm fast paths bail out for empty property cache slots.
This commit is contained in:
Andreas Kling
Andreas Kling
parent
75b2bcd7e4
commit
0a49dd1c28
7 changed files with 336 additions and 94 deletions
|
|
@ -915,9 +915,8 @@ i64 asm_try_put_by_id_cache(VM* vm, u32 pc)
|
|||
auto value = vm->get(insn.src());
|
||||
auto& cache = vm->current_executable().property_lookup_caches[insn.cache()];
|
||||
|
||||
for (size_t i = 0; i < cache.entries.size(); ++i) {
|
||||
auto& entry = cache.entries[i];
|
||||
switch (cache.types[i]) {
|
||||
for (auto& entry : cache.entries()) {
|
||||
switch (entry.type) {
|
||||
case PropertyLookupCache::Entry::Type::ChangeOwnProperty: {
|
||||
auto cached_shape = entry.shape.ptr();
|
||||
if (cached_shape != &object.shape()) [[unlikely]]
|
||||
|
|
@ -970,7 +969,12 @@ i64 asm_try_get_by_id_cache(VM* vm, u32 pc)
|
|||
auto& shape = object.shape();
|
||||
auto& cache = vm->current_executable().property_lookup_caches[insn.cache()];
|
||||
|
||||
for (auto& entry : cache.entries) {
|
||||
for (auto& entry : cache.entries()) {
|
||||
if (entry.type != PropertyLookupCache::Entry::Type::GetOwnProperty
|
||||
&& entry.type != PropertyLookupCache::Entry::Type::GetPropertyInPrototypeChain) {
|
||||
continue;
|
||||
}
|
||||
|
||||
auto cached_prototype = entry.prototype.ptr();
|
||||
if (cached_prototype) {
|
||||
if (&shape != entry.shape) [[unlikely]]
|
||||
|
|
|
|||
|
|
@ -499,13 +499,18 @@ macro pop_inline_frame_and_resume(caller_frame, value_reg)
|
|||
dispatch_current
|
||||
end
|
||||
|
||||
macro load_property_lookup_cache(cache)
|
||||
macro load_property_lookup_cache(cache, fail_label)
|
||||
temp exe, caches
|
||||
load32 cache, [pb, pc, m_cache]
|
||||
mul cache, cache, PROPERTY_LOOKUP_CACHE_SIZE
|
||||
load64 exe, [exec_ctx, EXECUTION_CONTEXT_EXECUTABLE]
|
||||
load64 caches, [exe, EXECUTABLE_PROPERTY_LOOKUP_CACHES_DATA]
|
||||
add cache, caches
|
||||
load64 cache, [cache, PROPERTY_LOOKUP_CACHE_DATA]
|
||||
branch_zero cache, fail_label
|
||||
branch_bits_clear cache, PROPERTY_LOOKUP_CACHE_POLYMORPHIC_DATA_TAG, .data_loaded
|
||||
sub cache, PROPERTY_LOOKUP_CACHE_POLYMORPHIC_DATA_TAG
|
||||
.data_loaded:
|
||||
end
|
||||
|
||||
macro load_global_variable_cache(cache)
|
||||
|
|
@ -1685,13 +1690,13 @@ handler GetById
|
|||
unbox_object obj, base
|
||||
load64 shape, [obj, OBJECT_SHAPE]
|
||||
assert_nonzero shape
|
||||
load_property_lookup_cache plc
|
||||
load_property_lookup_cache plc, .try_cache
|
||||
assert_nonzero plc
|
||||
load_pair64 cache_shape, cache_proto, [plc, PROPERTY_LOOKUP_CACHE_ENTRY0_SHAPE], [plc, PROPERTY_LOOKUP_CACHE_ENTRY0_PROTOTYPE]
|
||||
load_pair64 cache_shape, cache_proto, [plc, PROPERTY_LOOKUP_CACHE_ENTRY_SHAPE], [plc, PROPERTY_LOOKUP_CACHE_ENTRY_PROTOTYPE]
|
||||
branch_ne cache_shape, shape, .try_cache
|
||||
branch_nonzero cache_proto, .proto
|
||||
# Check dictionary generation matches
|
||||
load_pair32 prop_offset, dict_gen, [plc, PROPERTY_LOOKUP_CACHE_ENTRY0_PROPERTY_OFFSET], [plc, PROPERTY_LOOKUP_CACHE_ENTRY0_DICTIONARY_GENERATION]
|
||||
load_pair32 prop_offset, dict_gen, [plc, PROPERTY_LOOKUP_CACHE_ENTRY_PROPERTY_OFFSET], [plc, PROPERTY_LOOKUP_CACHE_ENTRY_DICTIONARY_GENERATION]
|
||||
load32 cur_dict_gen, [shape, SHAPE_DICTIONARY_GENERATION]
|
||||
branch_ne dict_gen, cur_dict_gen, .try_cache
|
||||
# IC hit! Load property value via get_direct (own property)
|
||||
|
|
@ -1705,11 +1710,11 @@ handler GetById
|
|||
dispatch_next
|
||||
.proto:
|
||||
# cache_proto = prototype Object*, shape = object's shape, plc = PLC base
|
||||
load64 prop_offset, [plc, PROPERTY_LOOKUP_CACHE_ENTRY0_PROTOTYPE_CHAIN_VALIDITY]
|
||||
load64 prop_offset, [plc, PROPERTY_LOOKUP_CACHE_ENTRY_PROTOTYPE_CHAIN_VALIDITY]
|
||||
branch_zero prop_offset, .try_cache
|
||||
load8 tag, [prop_offset, PROTOTYPE_CHAIN_VALIDITY_VALID]
|
||||
branch_zero tag, .try_cache
|
||||
load_pair32 prop_offset, dict_gen, [plc, PROPERTY_LOOKUP_CACHE_ENTRY0_PROPERTY_OFFSET], [plc, PROPERTY_LOOKUP_CACHE_ENTRY0_DICTIONARY_GENERATION]
|
||||
load_pair32 prop_offset, dict_gen, [plc, PROPERTY_LOOKUP_CACHE_ENTRY_PROPERTY_OFFSET], [plc, PROPERTY_LOOKUP_CACHE_ENTRY_DICTIONARY_GENERATION]
|
||||
load32 cur_dict_gen, [shape, SHAPE_DICTIONARY_GENERATION]
|
||||
branch_ne dict_gen, cur_dict_gen, .try_cache
|
||||
load64 props, [cache_proto, OBJECT_NAMED_PROPERTIES]
|
||||
|
|
@ -1738,12 +1743,12 @@ handler PutById
|
|||
unbox_object obj, base
|
||||
load64 shape, [obj, OBJECT_SHAPE]
|
||||
assert_nonzero shape
|
||||
load_property_lookup_cache plc
|
||||
load_property_lookup_cache plc, .try_cache
|
||||
assert_nonzero plc
|
||||
load_pair64 cache_shape, cache_proto, [plc, PROPERTY_LOOKUP_CACHE_ENTRY0_SHAPE], [plc, PROPERTY_LOOKUP_CACHE_ENTRY0_PROTOTYPE]
|
||||
load_pair64 cache_shape, cache_proto, [plc, PROPERTY_LOOKUP_CACHE_ENTRY_SHAPE], [plc, PROPERTY_LOOKUP_CACHE_ENTRY_PROTOTYPE]
|
||||
branch_ne cache_shape, shape, .try_cache
|
||||
branch_nonzero cache_proto, .try_cache
|
||||
load_pair32 prop_offset, dict_gen, [plc, PROPERTY_LOOKUP_CACHE_ENTRY0_PROPERTY_OFFSET], [plc, PROPERTY_LOOKUP_CACHE_ENTRY0_DICTIONARY_GENERATION]
|
||||
load_pair32 prop_offset, dict_gen, [plc, PROPERTY_LOOKUP_CACHE_ENTRY_PROPERTY_OFFSET], [plc, PROPERTY_LOOKUP_CACHE_ENTRY_DICTIONARY_GENERATION]
|
||||
load32 cur_dict_gen, [shape, SHAPE_DICTIONARY_GENERATION]
|
||||
branch_ne dict_gen, cur_dict_gen, .try_cache
|
||||
# Check current value at prop_offset is not an accessor
|
||||
|
|
@ -1916,12 +1921,12 @@ handler GetLength
|
|||
# Non-magical length: IC fast path (same as GetById)
|
||||
load64 shape, [obj, OBJECT_SHAPE]
|
||||
assert_nonzero shape
|
||||
load_property_lookup_cache plc
|
||||
load_property_lookup_cache plc, .slow
|
||||
assert_nonzero plc
|
||||
load_pair64 cache_shape, cache_proto, [plc, PROPERTY_LOOKUP_CACHE_ENTRY0_SHAPE], [plc, PROPERTY_LOOKUP_CACHE_ENTRY0_PROTOTYPE]
|
||||
load_pair64 cache_shape, cache_proto, [plc, PROPERTY_LOOKUP_CACHE_ENTRY_SHAPE], [plc, PROPERTY_LOOKUP_CACHE_ENTRY_PROTOTYPE]
|
||||
branch_ne cache_shape, shape, .slow
|
||||
branch_nonzero cache_proto, .slow
|
||||
load_pair32 prop_offset, dict_gen, [plc, PROPERTY_LOOKUP_CACHE_ENTRY0_PROPERTY_OFFSET], [plc, PROPERTY_LOOKUP_CACHE_ENTRY0_DICTIONARY_GENERATION]
|
||||
load_pair32 prop_offset, dict_gen, [plc, PROPERTY_LOOKUP_CACHE_ENTRY_PROPERTY_OFFSET], [plc, PROPERTY_LOOKUP_CACHE_ENTRY_DICTIONARY_GENERATION]
|
||||
load32 cur_dict_gen, [shape, SHAPE_DICTIONARY_GENERATION]
|
||||
branch_ne dict_gen, cur_dict_gen, .slow
|
||||
load64 props, [obj, OBJECT_NAMED_PROPERTIES]
|
||||
|
|
@ -1967,10 +1972,10 @@ handler GetGlobal
|
|||
# (falls through to env binding path on shape mismatch)
|
||||
load64 shape, [global_object, OBJECT_SHAPE]
|
||||
assert_nonzero shape
|
||||
load64 cache_shape, [gvc, PROPERTY_LOOKUP_CACHE_ENTRY0_SHAPE]
|
||||
load64 cache_shape, [gvc, GLOBAL_VARIABLE_CACHE_ENTRY_SHAPE]
|
||||
branch_ne cache_shape, shape, .try_env_binding
|
||||
load32 cur_dict_gen, [shape, SHAPE_DICTIONARY_GENERATION]
|
||||
load_pair32 prop_offset, dict_gen, [gvc, PROPERTY_LOOKUP_CACHE_ENTRY0_PROPERTY_OFFSET], [gvc, PROPERTY_LOOKUP_CACHE_ENTRY0_DICTIONARY_GENERATION]
|
||||
load_pair32 prop_offset, dict_gen, [gvc, GLOBAL_VARIABLE_CACHE_ENTRY_PROPERTY_OFFSET], [gvc, GLOBAL_VARIABLE_CACHE_ENTRY_DICTIONARY_GENERATION]
|
||||
branch_ne dict_gen, cur_dict_gen, .try_env_binding
|
||||
# IC hit! Load property value via get_direct
|
||||
load64 props, [global_object, OBJECT_NAMED_PROPERTIES]
|
||||
|
|
@ -2013,10 +2018,10 @@ handler SetGlobal
|
|||
branch_ne cache_serial, env_serial, .slow
|
||||
load64 shape, [global_object, OBJECT_SHAPE]
|
||||
assert_nonzero shape
|
||||
load64 cache_shape, [gvc, PROPERTY_LOOKUP_CACHE_ENTRY0_SHAPE]
|
||||
load64 cache_shape, [gvc, GLOBAL_VARIABLE_CACHE_ENTRY_SHAPE]
|
||||
branch_ne cache_shape, shape, .try_env_binding
|
||||
load32 cur_dict_gen, [shape, SHAPE_DICTIONARY_GENERATION]
|
||||
load_pair32 prop_offset, dict_gen, [gvc, PROPERTY_LOOKUP_CACHE_ENTRY0_PROPERTY_OFFSET], [gvc, PROPERTY_LOOKUP_CACHE_ENTRY0_DICTIONARY_GENERATION]
|
||||
load_pair32 prop_offset, dict_gen, [gvc, GLOBAL_VARIABLE_CACHE_ENTRY_PROPERTY_OFFSET], [gvc, GLOBAL_VARIABLE_CACHE_ENTRY_DICTIONARY_GENERATION]
|
||||
branch_ne dict_gen, cur_dict_gen, .try_env_binding
|
||||
# IC hit! Load current value to check it's not an accessor.
|
||||
load64 props, [global_object, OBJECT_NAMED_PROPERTIES]
|
||||
|
|
|
|||
|
|
@ -76,7 +76,8 @@ int main()
|
|||
|
||||
// PropertyLookupCache layout
|
||||
outln("\n# PropertyLookupCache layout");
|
||||
EMIT_OFFSET(PROPERTY_LOOKUP_CACHE_ENTRIES, PropertyLookupCache, entries);
|
||||
EMIT_OFFSET(PROPERTY_LOOKUP_CACHE_DATA, PropertyLookupCache, m_data);
|
||||
outln("const PROPERTY_LOOKUP_CACHE_POLYMORPHIC_DATA_TAG = {}", PropertyLookupCache::polymorphic_data_tag);
|
||||
EMIT_SIZEOF(PROPERTY_LOOKUP_CACHE_SIZE, PropertyLookupCache);
|
||||
|
||||
// PropertyLookupCache::Entry layout
|
||||
|
|
@ -89,15 +90,6 @@ int main()
|
|||
EMIT_OFFSET(PROPERTY_LOOKUP_CACHE_ENTRY_PROTOTYPE_CHAIN_VALIDITY, PropertyLookupCache::Entry, prototype_chain_validity);
|
||||
EMIT_SIZEOF(PROPERTY_LOOKUP_CACHE_ENTRY_SIZE, PropertyLookupCache::Entry);
|
||||
|
||||
// Composite offsets for entry[0] within a PropertyLookupCache
|
||||
outln("\n# Entry[0] offsets within PropertyLookupCache");
|
||||
auto plc_entries = offsetof(PropertyLookupCache, entries);
|
||||
outln("const PROPERTY_LOOKUP_CACHE_ENTRY0_PROPERTY_OFFSET = {}", plc_entries + offsetof(PropertyLookupCache::Entry, property_offset));
|
||||
outln("const PROPERTY_LOOKUP_CACHE_ENTRY0_DICTIONARY_GENERATION = {}", plc_entries + offsetof(PropertyLookupCache::Entry, shape_dictionary_generation));
|
||||
outln("const PROPERTY_LOOKUP_CACHE_ENTRY0_SHAPE = {}", plc_entries + offsetof(PropertyLookupCache::Entry, shape));
|
||||
outln("const PROPERTY_LOOKUP_CACHE_ENTRY0_PROTOTYPE = {}", plc_entries + offsetof(PropertyLookupCache::Entry, prototype));
|
||||
outln("const PROPERTY_LOOKUP_CACHE_ENTRY0_PROTOTYPE_CHAIN_VALIDITY = {}", plc_entries + offsetof(PropertyLookupCache::Entry, prototype_chain_validity));
|
||||
|
||||
// ObjectPropertyIteratorCacheData layout
|
||||
outln("\n# ObjectPropertyIteratorCacheData layout");
|
||||
EMIT_OFFSET(OBJECT_PROPERTY_ITERATOR_CACHE_DATA_PROPERTIES, ObjectPropertyIteratorCacheData, m_properties);
|
||||
|
|
@ -240,11 +232,15 @@ int main()
|
|||
|
||||
// GlobalVariableCache layout
|
||||
outln("\n# GlobalVariableCache layout");
|
||||
EMIT_OFFSET(GLOBAL_VARIABLE_CACHE_ENTRY, GlobalVariableCache, entry);
|
||||
EMIT_OFFSET(GLOBAL_VARIABLE_CACHE_ENVIRONMENT_SERIAL, GlobalVariableCache, environment_serial_number);
|
||||
EMIT_OFFSET(GLOBAL_VARIABLE_CACHE_ENVIRONMENT_BINDING_INDEX, GlobalVariableCache, environment_binding_index);
|
||||
EMIT_OFFSET(GLOBAL_VARIABLE_CACHE_HAS_ENVIRONMENT_BINDING, GlobalVariableCache, has_environment_binding_index);
|
||||
EMIT_OFFSET(GLOBAL_VARIABLE_CACHE_IN_MODULE_ENVIRONMENT, GlobalVariableCache, in_module_environment);
|
||||
EMIT_SIZEOF(GLOBAL_VARIABLE_CACHE_SIZE, GlobalVariableCache);
|
||||
outln("const GLOBAL_VARIABLE_CACHE_ENTRY_PROPERTY_OFFSET = {}", offsetof(GlobalVariableCache, entry) + offsetof(PropertyLookupCache::Entry, property_offset));
|
||||
outln("const GLOBAL_VARIABLE_CACHE_ENTRY_DICTIONARY_GENERATION = {}", offsetof(GlobalVariableCache, entry) + offsetof(PropertyLookupCache::Entry, shape_dictionary_generation));
|
||||
outln("const GLOBAL_VARIABLE_CACHE_ENTRY_SHAPE = {}", offsetof(GlobalVariableCache, entry) + offsetof(PropertyLookupCache::Entry, shape));
|
||||
|
||||
// Builtin enum values
|
||||
outln("\n# Builtin enum values");
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@
|
|||
#include <AK/BinarySearch.h>
|
||||
#include <AK/NumericLimits.h>
|
||||
#include <AK/QuickSort.h>
|
||||
#include <AK/StdLibExtras.h>
|
||||
#include <LibGC/Heap.h>
|
||||
#include <LibGC/HeapBlock.h>
|
||||
#include <LibJS/Bytecode/BasicBlock.h>
|
||||
|
|
@ -67,6 +68,152 @@ size_t InstructionStream::external_memory_size() const
|
|||
});
|
||||
}
|
||||
|
||||
static_assert(alignof(PropertyLookupCache::MonomorphicData) > PropertyLookupCache::polymorphic_data_tag);
|
||||
static_assert(alignof(PropertyLookupCache::PolymorphicData) > PropertyLookupCache::polymorphic_data_tag);
|
||||
static_assert(offsetof(PropertyLookupCache::MonomorphicData, entry) == 0);
|
||||
static_assert(offsetof(PropertyLookupCache::PolymorphicData, entries) == 0);
|
||||
|
||||
PropertyLookupCache::PropertyLookupCache(PropertyLookupCache&& other)
|
||||
: m_data(exchange(other.m_data, 0))
|
||||
{
|
||||
}
|
||||
|
||||
PropertyLookupCache& PropertyLookupCache::operator=(PropertyLookupCache&& other)
|
||||
{
|
||||
if (this != &other) {
|
||||
clear();
|
||||
m_data = exchange(other.m_data, 0);
|
||||
}
|
||||
return *this;
|
||||
}
|
||||
|
||||
PropertyLookupCache::~PropertyLookupCache()
|
||||
{
|
||||
clear();
|
||||
}
|
||||
|
||||
PropertyLookupCache::MonomorphicData* PropertyLookupCache::monomorphic_data()
|
||||
{
|
||||
if (!m_data || (m_data & polymorphic_data_tag))
|
||||
return nullptr;
|
||||
return reinterpret_cast<MonomorphicData*>(m_data);
|
||||
}
|
||||
|
||||
PropertyLookupCache::MonomorphicData const* PropertyLookupCache::monomorphic_data() const
|
||||
{
|
||||
if (!m_data || (m_data & polymorphic_data_tag))
|
||||
return nullptr;
|
||||
return reinterpret_cast<MonomorphicData const*>(m_data);
|
||||
}
|
||||
|
||||
PropertyLookupCache::PolymorphicData* PropertyLookupCache::polymorphic_data()
|
||||
{
|
||||
if (!(m_data & polymorphic_data_tag))
|
||||
return nullptr;
|
||||
return reinterpret_cast<PolymorphicData*>(m_data & ~polymorphic_data_tag);
|
||||
}
|
||||
|
||||
PropertyLookupCache::PolymorphicData const* PropertyLookupCache::polymorphic_data() const
|
||||
{
|
||||
if (!(m_data & polymorphic_data_tag))
|
||||
return nullptr;
|
||||
return reinterpret_cast<PolymorphicData const*>(m_data & ~polymorphic_data_tag);
|
||||
}
|
||||
|
||||
void PropertyLookupCache::set_monomorphic_data(MonomorphicData* data)
|
||||
{
|
||||
VERIFY(data);
|
||||
VERIFY(!(reinterpret_cast<FlatPtr>(data) & polymorphic_data_tag));
|
||||
m_data = reinterpret_cast<FlatPtr>(data);
|
||||
}
|
||||
|
||||
void PropertyLookupCache::set_polymorphic_data(PolymorphicData* data)
|
||||
{
|
||||
VERIFY(data);
|
||||
VERIFY(!(reinterpret_cast<FlatPtr>(data) & polymorphic_data_tag));
|
||||
m_data = reinterpret_cast<FlatPtr>(data) | polymorphic_data_tag;
|
||||
}
|
||||
|
||||
PropertyLookupCache::Entry* PropertyLookupCache::first_entry()
|
||||
{
|
||||
if (auto* data = monomorphic_data())
|
||||
return &data->entry;
|
||||
if (auto* data = polymorphic_data())
|
||||
return &data->entries[0];
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
PropertyLookupCache::Entry const* PropertyLookupCache::first_entry() const
|
||||
{
|
||||
if (auto* data = monomorphic_data())
|
||||
return &data->entry;
|
||||
if (auto* data = polymorphic_data())
|
||||
return &data->entries[0];
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
Span<PropertyLookupCache::Entry> PropertyLookupCache::entries()
|
||||
{
|
||||
if (auto* data = monomorphic_data())
|
||||
return { &data->entry, 1 };
|
||||
if (auto* data = polymorphic_data())
|
||||
return data->entries.span();
|
||||
return {};
|
||||
}
|
||||
|
||||
ReadonlySpan<PropertyLookupCache::Entry> PropertyLookupCache::entries() const
|
||||
{
|
||||
if (auto* data = monomorphic_data())
|
||||
return { &data->entry, 1 };
|
||||
if (auto* data = polymorphic_data())
|
||||
return data->entries.span();
|
||||
return {};
|
||||
}
|
||||
|
||||
size_t PropertyLookupCache::external_memory_size() const
|
||||
{
|
||||
if (monomorphic_data())
|
||||
return sizeof(MonomorphicData);
|
||||
if (polymorphic_data())
|
||||
return sizeof(PolymorphicData);
|
||||
return 0;
|
||||
}
|
||||
|
||||
void PropertyLookupCache::clear()
|
||||
{
|
||||
if (auto* data = monomorphic_data()) {
|
||||
delete data;
|
||||
m_data = 0;
|
||||
return;
|
||||
}
|
||||
if (auto* data = polymorphic_data()) {
|
||||
delete data;
|
||||
m_data = 0;
|
||||
}
|
||||
}
|
||||
|
||||
bool PropertyLookupCache::entries_have_same_cache_key(Entry const& a, Entry const& b)
|
||||
{
|
||||
if (a.type == Entry::Type::Empty || b.type == Entry::Type::Empty)
|
||||
return false;
|
||||
if (a.type != b.type)
|
||||
return false;
|
||||
|
||||
switch (a.type) {
|
||||
case Entry::Type::AddOwnProperty:
|
||||
return a.from_shape == b.from_shape && a.shape == b.shape;
|
||||
case Entry::Type::ChangeOwnProperty:
|
||||
case Entry::Type::GetOwnProperty:
|
||||
return a.shape == b.shape;
|
||||
case Entry::Type::ChangePropertyInPrototypeChain:
|
||||
case Entry::Type::GetPropertyInPrototypeChain:
|
||||
return a.shape == b.shape && a.prototype == b.prototype;
|
||||
case Entry::Type::Empty:
|
||||
VERIFY_NOT_REACHED();
|
||||
}
|
||||
VERIFY_NOT_REACHED();
|
||||
}
|
||||
|
||||
ObjectPropertyIteratorCacheData::ObjectPropertyIteratorCacheData(VM& vm, Vector<PropertyKey> properties, ObjectPropertyIteratorFastPath fast_path, u32 indexed_property_count, bool receiver_has_magical_length_property, GC::Ref<Shape> shape, GC::Ptr<PrototypeChainValidity> prototype_chain_validity)
|
||||
: m_properties(move(properties))
|
||||
, m_shape(shape)
|
||||
|
|
@ -413,6 +560,8 @@ size_t Executable::external_memory_size() const
|
|||
{
|
||||
size_t size = bytecode.external_memory_size();
|
||||
size = saturating_add_external_memory_size(size, vector_external_memory_size(property_lookup_caches));
|
||||
for (auto const& cache : property_lookup_caches)
|
||||
size = saturating_add_external_memory_size(size, cache.external_memory_size());
|
||||
size = saturating_add_external_memory_size(size, vector_external_memory_size(global_variable_caches));
|
||||
size = saturating_add_external_memory_size(size, vector_external_memory_size(template_object_caches));
|
||||
size = saturating_add_external_memory_size(size, vector_external_memory_size(object_shape_caches));
|
||||
|
|
@ -469,7 +618,7 @@ static void clear_cache_entry_if_dead(PropertyLookupCache::Entry& entry)
|
|||
void StaticPropertyLookupCache::sweep_all()
|
||||
{
|
||||
for (auto* cache : static_property_lookup_caches()) {
|
||||
for (auto& entry : cache->entries)
|
||||
for (auto& entry : cache->entries())
|
||||
clear_cache_entry_if_dead(entry);
|
||||
}
|
||||
}
|
||||
|
|
@ -477,13 +626,11 @@ void StaticPropertyLookupCache::sweep_all()
|
|||
void Executable::remove_dead_cells(Badge<GC::Heap>)
|
||||
{
|
||||
for (auto& cache : property_lookup_caches) {
|
||||
for (auto& entry : cache.entries)
|
||||
clear_cache_entry_if_dead(entry);
|
||||
}
|
||||
for (auto& cache : global_variable_caches) {
|
||||
for (auto& entry : cache.entries)
|
||||
for (auto& entry : cache.entries())
|
||||
clear_cache_entry_if_dead(entry);
|
||||
}
|
||||
for (auto& cache : global_variable_caches)
|
||||
clear_cache_entry_if_dead(cache.entry);
|
||||
for (auto& cache : object_shape_caches) {
|
||||
auto* shape = cache.shape.ptr();
|
||||
if (shape && cell_is_dead(shape))
|
||||
|
|
|
|||
|
|
@ -9,7 +9,9 @@
|
|||
#include <AK/NonnullOwnPtr.h>
|
||||
#include <AK/Optional.h>
|
||||
#include <AK/OwnPtr.h>
|
||||
#include <AK/Span.h>
|
||||
#include <AK/String.h>
|
||||
#include <AK/Types.h>
|
||||
#include <AK/Utf16FlyString.h>
|
||||
#include <AK/Variant.h>
|
||||
#include <AK/Vector.h>
|
||||
|
|
@ -55,7 +57,7 @@ private:
|
|||
size_t m_size { 0 };
|
||||
};
|
||||
|
||||
// Represents one polymorphic inline cache used for property lookups.
|
||||
// Represents one tiered inline cache used for property lookups.
|
||||
struct PropertyLookupCache {
|
||||
static constexpr size_t max_number_of_shapes_to_remember = 4;
|
||||
struct Entry {
|
||||
|
|
@ -67,6 +69,7 @@ struct PropertyLookupCache {
|
|||
ChangePropertyInPrototypeChain,
|
||||
GetPropertyInPrototypeChain,
|
||||
};
|
||||
Type type { Type::Empty };
|
||||
u32 property_offset { 0 };
|
||||
u32 shape_dictionary_generation { 0 };
|
||||
GC::RawPtr<Shape> from_shape;
|
||||
|
|
@ -75,20 +78,82 @@ struct PropertyLookupCache {
|
|||
GC::RawPtr<PrototypeChainValidity> prototype_chain_validity;
|
||||
};
|
||||
|
||||
struct MonomorphicData {
|
||||
Entry entry;
|
||||
};
|
||||
|
||||
struct PolymorphicData {
|
||||
AK::Array<Entry, max_number_of_shapes_to_remember> entries;
|
||||
};
|
||||
|
||||
PropertyLookupCache() = default;
|
||||
PropertyLookupCache(PropertyLookupCache const&) = delete;
|
||||
PropertyLookupCache& operator=(PropertyLookupCache const&) = delete;
|
||||
PropertyLookupCache(PropertyLookupCache&&);
|
||||
PropertyLookupCache& operator=(PropertyLookupCache&&);
|
||||
~PropertyLookupCache();
|
||||
|
||||
[[nodiscard]] Entry* first_entry();
|
||||
[[nodiscard]] Entry const* first_entry() const;
|
||||
[[nodiscard]] Span<Entry> entries();
|
||||
[[nodiscard]] ReadonlySpan<Entry> entries() const;
|
||||
[[nodiscard]] size_t external_memory_size() const;
|
||||
|
||||
void update(Entry::Type type, auto callback)
|
||||
{
|
||||
// First, move all entries one step back.
|
||||
for (size_t i = entries.size() - 1; i >= 1; --i) {
|
||||
types[i] = types[i - 1];
|
||||
entries[i] = entries[i - 1];
|
||||
Entry new_entry;
|
||||
new_entry.type = type;
|
||||
callback(new_entry);
|
||||
|
||||
if (!m_data) {
|
||||
auto data = make<MonomorphicData>();
|
||||
data->entry = new_entry;
|
||||
set_monomorphic_data(data.leak_ptr());
|
||||
return;
|
||||
}
|
||||
types[0] = type;
|
||||
entries[0] = {};
|
||||
callback(entries[0]);
|
||||
|
||||
if (auto* data = monomorphic_data()) {
|
||||
if (entries_have_same_cache_key(data->entry, new_entry)) {
|
||||
data->entry = new_entry;
|
||||
return;
|
||||
}
|
||||
|
||||
auto old_entry = data->entry;
|
||||
auto new_data = make<PolymorphicData>();
|
||||
new_data->entries[0] = new_entry;
|
||||
new_data->entries[1] = old_entry;
|
||||
clear();
|
||||
set_polymorphic_data(new_data.leak_ptr());
|
||||
return;
|
||||
}
|
||||
|
||||
auto& entries = polymorphic_data()->entries;
|
||||
size_t insertion_index = entries.size() - 1;
|
||||
for (size_t i = 0; i < entries.size(); ++i) {
|
||||
if (entries_have_same_cache_key(entries[i], new_entry)) {
|
||||
insertion_index = i;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
for (size_t i = insertion_index; i > 0; --i)
|
||||
entries[i] = entries[i - 1];
|
||||
entries[0] = new_entry;
|
||||
}
|
||||
|
||||
AK::Array<Entry::Type, max_number_of_shapes_to_remember> types;
|
||||
AK::Array<Entry, max_number_of_shapes_to_remember> entries;
|
||||
void clear();
|
||||
|
||||
static constexpr FlatPtr polymorphic_data_tag = 1;
|
||||
FlatPtr m_data { 0 };
|
||||
|
||||
private:
|
||||
[[nodiscard]] MonomorphicData* monomorphic_data();
|
||||
[[nodiscard]] MonomorphicData const* monomorphic_data() const;
|
||||
[[nodiscard]] PolymorphicData* polymorphic_data();
|
||||
[[nodiscard]] PolymorphicData const* polymorphic_data() const;
|
||||
void set_monomorphic_data(MonomorphicData*);
|
||||
void set_polymorphic_data(PolymorphicData*);
|
||||
static bool entries_have_same_cache_key(Entry const&, Entry const&);
|
||||
};
|
||||
|
||||
// A PropertyLookupCache for use as a static local variable.
|
||||
|
|
@ -98,7 +163,29 @@ struct StaticPropertyLookupCache : public PropertyLookupCache {
|
|||
static void sweep_all();
|
||||
};
|
||||
|
||||
struct GlobalVariableCache : public PropertyLookupCache {
|
||||
struct GlobalVariableCache {
|
||||
PropertyLookupCache::Entry* first_entry()
|
||||
{
|
||||
if (entry.type == PropertyLookupCache::Entry::Type::Empty)
|
||||
return nullptr;
|
||||
return &entry;
|
||||
}
|
||||
|
||||
PropertyLookupCache::Entry const* first_entry() const
|
||||
{
|
||||
if (entry.type == PropertyLookupCache::Entry::Type::Empty)
|
||||
return nullptr;
|
||||
return &entry;
|
||||
}
|
||||
|
||||
void update(PropertyLookupCache::Entry::Type type, auto callback)
|
||||
{
|
||||
entry = {};
|
||||
entry.type = type;
|
||||
callback(entry);
|
||||
}
|
||||
|
||||
PropertyLookupCache::Entry entry;
|
||||
u64 environment_serial_number { 0 };
|
||||
u32 environment_binding_index { 0 };
|
||||
bool has_environment_binding_index { false };
|
||||
|
|
|
|||
|
|
@ -1046,8 +1046,9 @@ inline ThrowCompletionOr<Value> get_global(VM& vm, IdentifierTableIndex identifi
|
|||
|
||||
// OPTIMIZATION: For global var bindings, if the shape of the global object hasn't changed,
|
||||
// we can use the cached property offset.
|
||||
if (&shape == cache.entries[0].shape && (!shape.is_dictionary() || shape.dictionary_generation() == cache.entries[0].shape_dictionary_generation)) {
|
||||
auto value = binding_object.get_direct(cache.entries[0].property_offset);
|
||||
auto* entry = cache.first_entry();
|
||||
if (entry && &shape == entry->shape && (!shape.is_dictionary() || shape.dictionary_generation() == entry->shape_dictionary_generation)) {
|
||||
auto value = binding_object.get_direct(entry->property_offset);
|
||||
if (value.is_accessor())
|
||||
return TRY(call(vm, value.as_accessor().getter(), &binding_object));
|
||||
return value;
|
||||
|
|
@ -1096,12 +1097,14 @@ inline ThrowCompletionOr<Value> get_global(VM& vm, IdentifierTableIndex identifi
|
|||
CacheableGetPropertyMetadata cacheable_metadata;
|
||||
auto value = TRY(binding_object.internal_get(identifier, &binding_object, &cacheable_metadata));
|
||||
if (cacheable_metadata.type == CacheableGetPropertyMetadata::Type::GetOwnProperty) {
|
||||
cache.entries[0].shape = shape;
|
||||
cache.entries[0].property_offset = cacheable_metadata.property_offset.value();
|
||||
cache.update(PropertyLookupCache::Entry::Type::GetOwnProperty, [&](auto& entry) {
|
||||
entry.shape = shape;
|
||||
entry.property_offset = cacheable_metadata.property_offset.value();
|
||||
|
||||
if (shape.is_dictionary()) {
|
||||
cache.entries[0].shape_dictionary_generation = shape.dictionary_generation();
|
||||
}
|
||||
if (shape.is_dictionary()) {
|
||||
entry.shape_dictionary_generation = shape.dictionary_generation();
|
||||
}
|
||||
});
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
|
@ -2396,12 +2399,13 @@ ThrowCompletionOr<void> SetGlobal::execute_impl(VM& vm) const
|
|||
if (cache.environment_serial_number == declarative_record.environment_serial_number()) {
|
||||
// OPTIMIZATION: For global var bindings, if the shape of the global object hasn't changed,
|
||||
// we can use the cached property offset.
|
||||
if (&shape == cache.entries[0].shape && (!shape.is_dictionary() || shape.dictionary_generation() == cache.entries[0].shape_dictionary_generation)) {
|
||||
auto value = binding_object.get_direct(cache.entries[0].property_offset);
|
||||
auto* entry = cache.first_entry();
|
||||
if (entry && &shape == entry->shape && (!shape.is_dictionary() || shape.dictionary_generation() == entry->shape_dictionary_generation)) {
|
||||
auto value = binding_object.get_direct(entry->property_offset);
|
||||
if (value.is_accessor())
|
||||
TRY(call(vm, value.as_accessor().setter(), &binding_object, src));
|
||||
else
|
||||
binding_object.put_direct(cache.entries[0].property_offset, src);
|
||||
binding_object.put_direct(entry->property_offset, src);
|
||||
return {};
|
||||
}
|
||||
|
||||
|
|
@ -2463,12 +2467,14 @@ ThrowCompletionOr<void> SetGlobal::execute_impl(VM& vm) const
|
|||
return vm.throw_completion<TypeError>(ErrorType::ObjectSetReturnedFalse);
|
||||
}
|
||||
if (cacheable_metadata.type == CacheableSetPropertyMetadata::Type::ChangeOwnProperty) {
|
||||
cache.entries[0].shape = shape;
|
||||
cache.entries[0].property_offset = cacheable_metadata.property_offset.value();
|
||||
cache.update(PropertyLookupCache::Entry::Type::ChangeOwnProperty, [&](auto& entry) {
|
||||
entry.shape = shape;
|
||||
entry.property_offset = cacheable_metadata.property_offset.value();
|
||||
|
||||
if (shape.is_dictionary()) {
|
||||
cache.entries[0].shape_dictionary_generation = shape.dictionary_generation();
|
||||
}
|
||||
if (shape.is_dictionary()) {
|
||||
entry.shape_dictionary_generation = shape.dictionary_generation();
|
||||
}
|
||||
});
|
||||
}
|
||||
return {};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -89,7 +89,12 @@ ALWAYS_INLINE ThrowCompletionOr<Value> get_by_id(VM& vm, GetBaseIdentifier get_b
|
|||
|
||||
auto& shape = base_obj->shape();
|
||||
|
||||
for (auto& cache_entry : cache.entries) {
|
||||
for (auto& cache_entry : cache.entries()) {
|
||||
if (cache_entry.type != PropertyLookupCache::Entry::Type::GetOwnProperty
|
||||
&& cache_entry.type != PropertyLookupCache::Entry::Type::GetPropertyInPrototypeChain) {
|
||||
continue;
|
||||
}
|
||||
|
||||
auto cached_prototype = cache_entry.prototype.ptr();
|
||||
if (cached_prototype) {
|
||||
// OPTIMIZATION: If the prototype chain hasn't been mutated in a way that would invalidate the cache, we can use it.
|
||||
|
|
@ -145,31 +150,26 @@ ALWAYS_INLINE ThrowCompletionOr<Value> get_by_id(VM& vm, GetBaseIdentifier get_b
|
|||
// that collected metadata is valid, e.g. if getter in prototype chain added
|
||||
// property with the same name into the object itself.
|
||||
if (&shape == &base_obj->shape()) {
|
||||
auto get_cache_slot = [&] -> PropertyLookupCache::Entry& {
|
||||
for (size_t i = cache.entries.size() - 1; i >= 1; --i) {
|
||||
cache.entries[i] = cache.entries[i - 1];
|
||||
}
|
||||
cache.entries[0] = {};
|
||||
return cache.entries[0];
|
||||
};
|
||||
if (cacheable_metadata.type == CacheableGetPropertyMetadata::Type::GetOwnProperty) {
|
||||
auto& entry = get_cache_slot();
|
||||
entry.shape = shape;
|
||||
entry.property_offset = cacheable_metadata.property_offset.value();
|
||||
cache.update(PropertyLookupCache::Entry::Type::GetOwnProperty, [&](auto& entry) {
|
||||
entry.shape = shape;
|
||||
entry.property_offset = cacheable_metadata.property_offset.value();
|
||||
|
||||
if (shape.is_dictionary()) {
|
||||
entry.shape_dictionary_generation = shape.dictionary_generation();
|
||||
}
|
||||
if (shape.is_dictionary()) {
|
||||
entry.shape_dictionary_generation = shape.dictionary_generation();
|
||||
}
|
||||
});
|
||||
} else if (cacheable_metadata.type == CacheableGetPropertyMetadata::Type::GetPropertyInPrototypeChain) {
|
||||
auto& entry = get_cache_slot();
|
||||
entry.shape = &base_obj->shape();
|
||||
entry.property_offset = cacheable_metadata.property_offset.value();
|
||||
entry.prototype = const_cast<Object*>(cacheable_metadata.prototype.ptr());
|
||||
entry.prototype_chain_validity = prototype_chain_validity;
|
||||
cache.update(PropertyLookupCache::Entry::Type::GetPropertyInPrototypeChain, [&](auto& entry) {
|
||||
entry.shape = &base_obj->shape();
|
||||
entry.property_offset = cacheable_metadata.property_offset.value();
|
||||
entry.prototype = const_cast<Object*>(cacheable_metadata.prototype.ptr());
|
||||
entry.prototype_chain_validity = prototype_chain_validity;
|
||||
|
||||
if (shape.is_dictionary()) {
|
||||
entry.shape_dictionary_generation = shape.dictionary_generation();
|
||||
}
|
||||
if (shape.is_dictionary()) {
|
||||
entry.shape_dictionary_generation = shape.dictionary_generation();
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -233,12 +233,11 @@ inline ThrowCompletionOr<void> put_by_property_key(VM& vm, Value base, Value thi
|
|||
auto this_value_object = MUST(this_value.to_object(vm));
|
||||
auto& from_shape = this_value_object->shape();
|
||||
if (caches) [[likely]] {
|
||||
for (size_t i = 0; i < caches->entries.size(); ++i) {
|
||||
switch (caches->types[i]) {
|
||||
for (auto& cache : caches->entries()) {
|
||||
switch (cache.type) {
|
||||
case PropertyLookupCache::Entry::Type::Empty:
|
||||
break;
|
||||
case PropertyLookupCache::Entry::Type::ChangePropertyInPrototypeChain: {
|
||||
auto& cache = caches->entries[i];
|
||||
auto cached_prototype = cache.prototype.ptr();
|
||||
if (!cached_prototype) [[unlikely]]
|
||||
break;
|
||||
|
|
@ -270,7 +269,6 @@ inline ThrowCompletionOr<void> put_by_property_key(VM& vm, Value base, Value thi
|
|||
break;
|
||||
}
|
||||
case PropertyLookupCache::Entry::Type::ChangeOwnProperty: {
|
||||
auto& cache = caches->entries[i];
|
||||
auto cached_shape = cache.shape.ptr();
|
||||
if (cached_shape != &object->shape()) [[unlikely]]
|
||||
break;
|
||||
|
|
@ -289,7 +287,6 @@ inline ThrowCompletionOr<void> put_by_property_key(VM& vm, Value base, Value thi
|
|||
return {};
|
||||
}
|
||||
case PropertyLookupCache::Entry::Type::AddOwnProperty: {
|
||||
auto& cache = caches->entries[i];
|
||||
// OPTIMIZATION: If the object's shape is the same as the one cached before adding the new property, we can
|
||||
// reuse the resulting shape from the cache.
|
||||
if (cache.from_shape != &object->shape()) [[unlikely]]
|
||||
|
|
@ -315,8 +312,9 @@ inline ThrowCompletionOr<void> put_by_property_key(VM& vm, Value base, Value thi
|
|||
object->put_direct(cache.property_offset, value);
|
||||
return {};
|
||||
}
|
||||
default:
|
||||
VERIFY_NOT_REACHED();
|
||||
case PropertyLookupCache::Entry::Type::GetOwnProperty:
|
||||
case PropertyLookupCache::Entry::Type::GetPropertyInPrototypeChain:
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -385,9 +383,8 @@ inline ThrowCompletionOr<void> put_by_property_key(VM& vm, Value base, Value thi
|
|||
}
|
||||
case PutKind::Own: {
|
||||
if (caches) [[likely]] {
|
||||
for (size_t i = 0; i < caches->entries.size(); ++i) {
|
||||
if (caches->types[i] == PropertyLookupCache::Entry::Type::AddOwnProperty) {
|
||||
auto& cache = caches->entries[i];
|
||||
for (auto& cache : caches->entries()) {
|
||||
if (cache.type == PropertyLookupCache::Entry::Type::AddOwnProperty) {
|
||||
if (cache.from_shape != &object->shape()) [[unlikely]]
|
||||
continue;
|
||||
auto cached_shape = cache.shape.ptr();
|
||||
|
|
|
|||
Loading…
Reference in a new issue