/* * Copyright (c) 2020-2024, Andreas Kling * * SPDX-License-Identifier: BSD-2-Clause */ #pragma once #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include namespace GC { struct StackFrameInfo { String label; size_t size_bytes { 0 }; }; class GC_API Heap { AK_MAKE_NONCOPYABLE(Heap); AK_MAKE_NONMOVABLE(Heap); friend class HeapGroup; public: enum class BecomeProcessDefault { No, Yes, }; explicit Heap(AK::Function&)> gather_embedder_roots, BecomeProcessDefault = BecomeProcessDefault::Yes); ~Heap(); static Heap& the(); static void set_default_heap_for_testing(Heap&); template Ref allocate(Args&&... args) { VERIFY(!m_collecting_garbage); auto* memory = allocate_cell(); defer_gc(); new (memory) T(forward(args)...); auto* cell = static_cast(memory); // Cells allocated during incremental sweep must be marked so they // survive until the next GC cycle clears and re-establishes marks. if (m_incremental_sweep_active) { cell->set_marked(true); m_cells_allocated_during_sweep.append(cell); } undefer_gc(); return *cell; } enum class CollectionType { CollectGarbage, CollectEverything, }; void collect_garbage(CollectionType = CollectionType::CollectGarbage, bool print_report = false); AK::JsonObject dump_graph(); bool should_collect_on_every_allocation() const { return m_should_collect_on_every_allocation; } void set_incremental_sweep_enabled(bool enabled) { m_incremental_sweep_enabled = enabled; } void set_should_collect_on_every_allocation(bool b) { m_should_collect_on_every_allocation = b; } void did_create_root(Badge, RootImpl&); void did_destroy_root(Badge, RootImpl&); void did_create_root_vector(Badge, RootVectorBase&); void did_destroy_root_vector(Badge, RootVectorBase&); void did_create_root_hash_map(Badge, RootHashMapBase&); void did_destroy_root_hash_map(Badge, RootHashMapBase&); void did_create_root_hash_table(Badge, RootHashTableBase&); void did_destroy_root_hash_table(Badge, RootHashTableBase&); void did_create_conservative_hash_map(Badge, ConservativeHashMapBase&); void did_destroy_conservative_hash_map(Badge, ConservativeHashMapBase&); void did_create_conservative_hash_table(Badge, ConservativeHashTableBase&); void did_destroy_conservative_hash_table(Badge, ConservativeHashTableBase&); void did_create_conservative_vector(Badge, ConservativeVectorBase&); void did_destroy_conservative_vector(Badge, ConservativeVectorBase&); void did_create_conservative_range_provider(Badge, ConservativeRangeProvider&); void did_destroy_conservative_range_provider(Badge, ConservativeRangeProvider&); void did_create_cross_heap_member(Badge, CrossHeapMemberBase&); void did_destroy_cross_heap_member(Badge, CrossHeapMemberBase&); void did_create_weak_container(Badge, WeakContainer&); void did_destroy_weak_container(Badge, WeakContainer&); void register_sweep_callback(AK::Function); void register_cell_allocator(Badge, CellAllocator&); CellAllocator& cell_allocator_for(Badge, CellAllocatorDescriptorBase&); void uproot_cell(Cell* cell); bool is_gc_deferred() const { return m_gc_deferrals > 0; } bool is_incremental_sweep_active() const { return m_incremental_sweep_active; } void sweep_block(HeapBlock&); bool is_live_heap_block(HeapBlock* block) const { return m_live_heap_blocks.contains(block); } void enqueue_post_gc_task(AK::Function); WeakImpl* create_weak_impl(void*); void did_allocate_external_memory(size_t); void did_free_external_memory(size_t); private: friend class CellAllocator; friend class HeapBlock; friend class MarkingVisitor; friend class GraphConstructorVisitor; friend class DeferGC; void defer_gc(); void undefer_gc(); void dump_allocators(); template Cell* allocate_cell() { static_assert(requires { T::cell_allocator.for_heap(*this).allocate_cell(*this); }, "GC cell type must declare its own allocator using GC_DECLARE_ALLOCATOR(ClassName)"); static_assert(IsSame, "GC cell allocator type mismatch"); will_allocate(sizeof(T)); return T::cell_allocator.for_heap(*this).allocate_cell(*this); } void will_allocate(size_t); void update_gc_bytes_threshold(size_t live_cell_bytes, size_t live_external_bytes); void find_min_and_max_block_addresses(FlatPtr& min_address, FlatPtr& max_address); enum class IncludeIncomingCrossHeapMembers { No, Yes, }; void gather_roots(HashMap&, Vector* out_stack_frames = nullptr, IncludeIncomingCrossHeapMembers = IncludeIncomingCrossHeapMembers::Yes); static void mark_live_cells_across(ReadonlySpan, HashMap const& roots); void run_post_mark_phases(bool report); void gather_conservative_roots(HashMap&, Vector* out_stack_frames = nullptr); void gather_asan_fake_stack_roots(HashMap&, FlatPtr, FlatPtr min_block_address, FlatPtr max_block_address, FlatPtr stack_reference, FlatPtr stack_top); void mark_live_cells(HashMap const& live_cells); void finalize_unmarked_cells(); void sweep_dead_cells(bool print_report, Core::ElapsedTimer const&); void sweep_weak_blocks(); void run_post_gc_tasks(); bool sweep_next_block(); void start_incremental_sweep(); void finish_incremental_sweep(); void finish_pending_incremental_sweep(); void start_incremental_sweep_timer(); void stop_incremental_sweep_timer(); void sweep_on_timer(); void start_idle_gc_timer(); void idle_gc_on_timer(); template void for_each_block(Callback callback) { for (auto& allocator : m_all_cell_allocators) { if (allocator.for_each_block(callback) == IterationDecision::Break) return; } } size_t m_gc_bytes_threshold { 0 }; size_t m_allocated_bytes_since_last_gc { 0 }; bool m_should_collect_on_every_allocation { false }; CellAllocator::List m_all_cell_allocators; HashMap> m_cell_allocators_by_type; RootImpl::List m_roots; RootVectorBase::List m_root_vectors; RootHashMapBase::List m_root_hash_maps; RootHashTableBase::List m_root_hash_tables; ConservativeHashMapBase::List m_conservative_hash_maps; ConservativeHashTableBase::List m_conservative_hash_tables; ConservativeVectorBase::List m_conservative_vectors; ConservativeRangeProvider::List m_conservative_range_providers; HashTable m_incoming_cross_heap_members; HeapGroup* m_group { nullptr }; bool m_incremental_sweep_enabled { true }; WeakContainer::List m_weak_containers; Vector> m_uprooted_cells; size_t m_gc_deferrals { 0 }; bool m_should_gc_when_deferral_ends { false }; bool m_collecting_garbage { false }; StackInfo m_stack_info; AK::Function&)> m_gather_embedder_roots; Vector> m_post_gc_tasks; Vector> m_sweep_callbacks; HashTable m_live_heap_blocks; WeakBlock::List m_usable_weak_blocks; WeakBlock::List m_full_weak_blocks; bool m_incremental_sweep_active { false }; size_t m_sweep_live_cell_bytes { 0 }; size_t m_sweep_live_external_bytes { 0 }; Vector> m_cells_allocated_during_sweep; CellAllocator::SweepList m_allocators_to_sweep; RefPtr m_incremental_sweep_timer; RefPtr m_idle_gc_timer; u64 m_total_allocated_bytes { 0 }; IdleCollectionPolicy m_idle_collection_policy; }; inline void Heap::did_create_root(Badge, RootImpl& impl) { VERIFY(!m_roots.contains(impl)); m_roots.append(impl); } inline void Heap::did_destroy_root(Badge, RootImpl& impl) { VERIFY(m_roots.contains(impl)); m_roots.remove(impl); } inline void Heap::did_create_root_vector(Badge, RootVectorBase& vector) { VERIFY(!m_root_vectors.contains(vector)); m_root_vectors.append(vector); } inline void Heap::did_destroy_root_vector(Badge, RootVectorBase& vector) { VERIFY(m_root_vectors.contains(vector)); m_root_vectors.remove(vector); } inline void Heap::did_create_root_hash_map(Badge, RootHashMapBase& hash_map) { VERIFY(!m_root_hash_maps.contains(hash_map)); m_root_hash_maps.append(hash_map); } inline void Heap::did_destroy_root_hash_map(Badge, RootHashMapBase& hash_map) { VERIFY(m_root_hash_maps.contains(hash_map)); m_root_hash_maps.remove(hash_map); } inline void Heap::did_create_root_hash_table(Badge, RootHashTableBase& hash_table) { VERIFY(!m_root_hash_tables.contains(hash_table)); m_root_hash_tables.append(hash_table); } inline void Heap::did_destroy_root_hash_table(Badge, RootHashTableBase& hash_table) { VERIFY(m_root_hash_tables.contains(hash_table)); m_root_hash_tables.remove(hash_table); } inline void Heap::did_create_conservative_hash_map(Badge, ConservativeHashMapBase& hash_map) { VERIFY(!m_conservative_hash_maps.contains(hash_map)); m_conservative_hash_maps.append(hash_map); } inline void Heap::did_destroy_conservative_hash_map(Badge, ConservativeHashMapBase& hash_map) { VERIFY(m_conservative_hash_maps.contains(hash_map)); m_conservative_hash_maps.remove(hash_map); } inline void Heap::did_create_conservative_hash_table(Badge, ConservativeHashTableBase& hash_table) { VERIFY(!m_conservative_hash_tables.contains(hash_table)); m_conservative_hash_tables.append(hash_table); } inline void Heap::did_destroy_conservative_hash_table(Badge, ConservativeHashTableBase& hash_table) { VERIFY(m_conservative_hash_tables.contains(hash_table)); m_conservative_hash_tables.remove(hash_table); } inline void Heap::did_create_conservative_vector(Badge, ConservativeVectorBase& vector) { VERIFY(!m_conservative_vectors.contains(vector)); m_conservative_vectors.append(vector); } inline void Heap::did_destroy_conservative_vector(Badge, ConservativeVectorBase& vector) { VERIFY(m_conservative_vectors.contains(vector)); m_conservative_vectors.remove(vector); } inline void Heap::did_create_conservative_range_provider(Badge, ConservativeRangeProvider& provider) { VERIFY(!m_conservative_range_providers.contains(provider)); m_conservative_range_providers.append(provider); } inline void Heap::did_destroy_conservative_range_provider(Badge, ConservativeRangeProvider& provider) { VERIFY(m_conservative_range_providers.contains(provider)); m_conservative_range_providers.remove(provider); } inline void Heap::did_create_cross_heap_member(Badge, CrossHeapMemberBase& member) { VERIFY(!m_incoming_cross_heap_members.contains(&member)); m_incoming_cross_heap_members.set(&member); } inline void Heap::did_destroy_cross_heap_member(Badge, CrossHeapMemberBase& member) { VERIFY(m_incoming_cross_heap_members.contains(&member)); m_incoming_cross_heap_members.remove(&member); } inline void Heap::did_create_weak_container(Badge, WeakContainer& set) { VERIFY(!m_weak_containers.contains(set)); m_weak_containers.append(set); } inline void Heap::did_destroy_weak_container(Badge, WeakContainer& set) { VERIFY(m_weak_containers.contains(set)); m_weak_containers.remove(set); } inline void Heap::register_cell_allocator(Badge, CellAllocator& allocator) { m_all_cell_allocators.append(allocator); } }