diff --git a/Libraries/LibGC/CMakeLists.txt b/Libraries/LibGC/CMakeLists.txt index 10e65745bb..39830c17ac 100644 --- a/Libraries/LibGC/CMakeLists.txt +++ b/Libraries/LibGC/CMakeLists.txt @@ -4,12 +4,15 @@ set(SOURCES CellAllocator.cpp ConservativeHashMap.cpp ConservativeHashTable.cpp + ConservativeRangeProvider.cpp ConservativeVector.cpp + CrossHeapMember.cpp Root.cpp RootHashMap.cpp RootHashTable.cpp RootVector.cpp Heap.cpp + HeapGroup.cpp HeapBlock.cpp Timer.cpp WeakBlock.cpp diff --git a/Libraries/LibGC/CellAllocator.cpp b/Libraries/LibGC/CellAllocator.cpp index 45927ea6a7..f20b3275d4 100644 --- a/Libraries/LibGC/CellAllocator.cpp +++ b/Libraries/LibGC/CellAllocator.cpp @@ -20,6 +20,16 @@ CellAllocator::CellAllocator(size_t cell_size, Optional class_name, { } +CellAllocator& CellAllocatorDescriptorBase::for_heap(Heap& heap) +{ + if (m_last_heap == &heap) [[likely]] + return *m_last_allocator; + auto& allocator = heap.cell_allocator_for({}, *this); + m_last_heap = &heap; + m_last_allocator = &allocator; + return allocator; +} + Cell* CellAllocator::allocate_cell(Heap& heap) { if (!m_list_node.is_in_list()) diff --git a/Libraries/LibGC/CellAllocator.h b/Libraries/LibGC/CellAllocator.h index 05056caf66..5149dbb85e 100644 --- a/Libraries/LibGC/CellAllocator.h +++ b/Libraries/LibGC/CellAllocator.h @@ -22,6 +22,45 @@ namespace GC { +class GC_API CellAllocatorDescriptorBase { + AK_MAKE_NONCOPYABLE(CellAllocatorDescriptorBase); + AK_MAKE_NONMOVABLE(CellAllocatorDescriptorBase); + +public: + Optional class_name() const { return m_class_name; } + size_t cell_size() const { return m_cell_size; } + bool overrides_must_survive_garbage_collection() const { return m_overrides_must_survive_garbage_collection; } + bool overrides_finalize() const { return m_overrides_finalize; } + + CellAllocator& for_heap(Heap&); + + void forget_heap(Badge, Heap& heap) + { + if (m_last_heap == &heap) { + m_last_heap = nullptr; + m_last_allocator = nullptr; + } + } + +protected: + CellAllocatorDescriptorBase(size_t cell_size, StringView class_name, bool overrides_must_survive_garbage_collection, bool overrides_finalize) + : m_class_name(class_name) + , m_cell_size(cell_size) + , m_overrides_must_survive_garbage_collection(overrides_must_survive_garbage_collection) + , m_overrides_finalize(overrides_finalize) + { + } + +private: + Optional m_class_name; + size_t m_cell_size { 0 }; + bool m_overrides_must_survive_garbage_collection { false }; + bool m_overrides_finalize { false }; + + Heap* m_last_heap { nullptr }; + CellAllocator* m_last_allocator { nullptr }; +}; + class GC_API CellAllocator { public: CellAllocator(size_t cell_size, Optional = {}, bool overrides_must_survive_garbage_collection = false, bool overrides_finalize = false); @@ -81,16 +120,14 @@ private: }; template -class GC_API TypeIsolatingCellAllocator { +class GC_API TypeIsolatingCellAllocator final : public CellAllocatorDescriptorBase { public: using CellType = T; TypeIsolatingCellAllocator(StringView class_name, bool overrides_must_survive_garbage_collection, bool overrides_finalize) - : allocator(sizeof(T), class_name, overrides_must_survive_garbage_collection, overrides_finalize) + : CellAllocatorDescriptorBase(sizeof(T), class_name, overrides_must_survive_garbage_collection, overrides_finalize) { } - - NeverDestroyed allocator; }; } diff --git a/Libraries/LibGC/ConservativeRangeProvider.cpp b/Libraries/LibGC/ConservativeRangeProvider.cpp new file mode 100644 index 0000000000..a0d3bbc9d2 --- /dev/null +++ b/Libraries/LibGC/ConservativeRangeProvider.cpp @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2026, Ali Mohammad Pur + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include +#include + +namespace GC { + +ConservativeRangeProvider::ConservativeRangeProvider(Heap& heap) + : m_heap(&heap) +{ + m_heap->did_create_conservative_range_provider({}, *this); +} + +ConservativeRangeProvider::~ConservativeRangeProvider() +{ + if (m_heap) + m_heap->did_destroy_conservative_range_provider({}, *this); +} + +} diff --git a/Libraries/LibGC/ConservativeRangeProvider.h b/Libraries/LibGC/ConservativeRangeProvider.h new file mode 100644 index 0000000000..2004450a7b --- /dev/null +++ b/Libraries/LibGC/ConservativeRangeProvider.h @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2026, Ali Mohammad Pur + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#pragma once + +#include +#include +#include +#include + +namespace GC { + +class GC_API ConservativeRangeProvider { + AK_MAKE_NONCOPYABLE(ConservativeRangeProvider); + AK_MAKE_NONMOVABLE(ConservativeRangeProvider); + +public: + virtual ~ConservativeRangeProvider(); + + // Called while gathering roots; every word of every reported range is treated as a possible cell pointer. + virtual void for_each_conservative_range(AK::Function)> const&) const = 0; + + void detach_from_heap(Badge) { m_heap = nullptr; } + +protected: + explicit ConservativeRangeProvider(Heap&); + + Heap* m_heap { nullptr }; + IntrusiveListNode m_list_node; + +public: + using List = IntrusiveList<&ConservativeRangeProvider::m_list_node>; +}; + +} diff --git a/Libraries/LibGC/CrossHeapMember.cpp b/Libraries/LibGC/CrossHeapMember.cpp new file mode 100644 index 0000000000..c89600a184 --- /dev/null +++ b/Libraries/LibGC/CrossHeapMember.cpp @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2026, Ali Mohammad Pur + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include +#include + +namespace GC { + +void CrossHeapMemberBase::reset(Cell* cell) +{ + auto* new_heap = cell ? &HeapBlockBase::from_cell(cell)->heap() : nullptr; + if (m_registered_heap == new_heap) { + m_cell = cell; + return; + } + + if (m_registered_heap) + m_registered_heap->did_destroy_cross_heap_member({}, *this); + m_cell = cell; + m_registered_heap = new_heap; + if (m_registered_heap) + m_registered_heap->did_create_cross_heap_member({}, *this); +} + +} diff --git a/Libraries/LibGC/CrossHeapMember.h b/Libraries/LibGC/CrossHeapMember.h new file mode 100644 index 0000000000..d825bee50a --- /dev/null +++ b/Libraries/LibGC/CrossHeapMember.h @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2026, Ali Mohammad Pur + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#pragma once + +#include +#include +#include + +namespace GC { + +// A strong reference from a cell on one heap to a cell on another heap in the same HeapGroup. +class GC_API CrossHeapMemberBase { + AK_MAKE_NONCOPYABLE(CrossHeapMemberBase); + AK_MAKE_NONMOVABLE(CrossHeapMemberBase); + +public: + Cell* cell_base() const { return static_cast(m_cell); } + + void detach_from_heap(Badge) + { + m_registered_heap = nullptr; + m_cell = nullptr; + } + +protected: + explicit CrossHeapMemberBase(Cell* cell) + { + reset(cell); + } + + ~CrossHeapMemberBase() + { + reset(nullptr); + } + + void reset(Cell* cell); + + void* m_cell { nullptr }; // Not a member of "this" heap. + Heap* m_registered_heap { nullptr }; +}; + +template +class CrossHeapMember final : public CrossHeapMemberBase { +public: + CrossHeapMember() + : CrossHeapMemberBase(nullptr) + { + } + + explicit CrossHeapMember(T* cell) + : CrossHeapMemberBase(cell) + { + } + + CrossHeapMember& operator=(T* cell) + { + reset(cell); + return *this; + } + + T* ptr() const { return static_cast(m_cell); } + T* operator->() const { return ptr(); } + explicit operator bool() const { return m_cell != nullptr; } + + // Called by the holder heap. + void visit(Cell::Visitor& visitor) const + { + if (m_cell) + visitor.visit(cell_base()); + } +}; + +} diff --git a/Libraries/LibGC/Forward.h b/Libraries/LibGC/Forward.h index 5f87da0e78..6d7d3590ed 100644 --- a/Libraries/LibGC/Forward.h +++ b/Libraries/LibGC/Forward.h @@ -16,6 +16,8 @@ class CellAllocator; class DeferGC; class RootImpl; class Heap; +class HeapGroup; +class CrossHeapMemberBase; class HeapBlock; class NanBoxedValue; class Timer; diff --git a/Libraries/LibGC/Heap.cpp b/Libraries/LibGC/Heap.cpp index cf17eff14b..69161aba22 100644 --- a/Libraries/LibGC/Heap.cpp +++ b/Libraries/LibGC/Heap.cpp @@ -285,10 +285,18 @@ void Heap::set_default_heap_for_testing(Heap& heap) s_the = &heap; } -Heap::Heap(AK::Function&)> gather_embedder_roots) +CellAllocator& Heap::cell_allocator_for(Badge, CellAllocatorDescriptorBase& descriptor) +{ + return *m_cell_allocators_by_type.ensure(&descriptor, [&] { + return make(descriptor.cell_size(), descriptor.class_name(), descriptor.overrides_must_survive_garbage_collection(), descriptor.overrides_finalize()); + }); +} + +Heap::Heap(AK::Function&)> gather_embedder_roots, BecomeProcessDefault become_process_default) : m_gather_embedder_roots(move(gather_embedder_roots)) { - s_the = this; + if (become_process_default == BecomeProcessDefault::Yes) + s_the = this; m_gc_bytes_threshold = GC_MIN_BYTES_THRESHOLD; static_assert(HeapBlock::min_possible_cell_size <= 32, "Heap Cell tracking uses too much data!"); } @@ -296,6 +304,20 @@ Heap::Heap(AK::Function&)> gather_embedder_roo Heap::~Heap() { collect_garbage(CollectionType::CollectEverything); + + for (auto& entry : m_cell_allocators_by_type) + entry.key->forget_heap({}, *this); + while (auto* allocator = m_all_cell_allocators.first()) + m_all_cell_allocators.remove(*allocator); + + while (auto* provider = m_conservative_range_providers.first()) { + m_conservative_range_providers.remove(*provider); + provider->detach_from_heap({}); + } + + for (auto* member : m_incoming_cross_heap_members) + member->detach_from_heap({}); + m_incoming_cross_heap_members.clear(); } void Heap::will_allocate(size_t size) @@ -492,6 +514,9 @@ public: case HeapRoot::Type::ConservativeVector: node.set("root"sv, "ConservativeVector"sv); break; + case HeapRoot::Type::CrossHeapMember: + node.set("root"sv, "CrossHeapMember"sv); + break; case HeapRoot::Type::HeapFunctionCapturedPointer: node.set("root"sv, "HeapFunctionCapturedPointer"sv); break; @@ -579,6 +604,40 @@ AK::JsonObject Heap::dump_graph() return graph; } +void Heap::run_post_mark_phases(bool report) +{ + { + ScopedPhaseTimer timer { report, g_phase_timings.finalize_unmarked_cells_us }; + finalize_unmarked_cells(); + } + { + ScopedPhaseTimer timer { report, g_phase_timings.sweep_weak_blocks_us }; + sweep_weak_blocks(); + } + + // Prune weak containers while we're still stop-the-world; doing this + // during incremental sweep risks reading cells that have already been + // freed and ASAN-poisoned. + { + ScopedPhaseTimer timer { report, g_phase_timings.prune_weak_containers_us }; + for (auto& weak_container : m_weak_containers) { + if (!weak_container.owner_cell({}).is_marked()) + continue; + weak_container.remove_dead_cells({}); + } + } + + // Run sweep callbacks at STW so they fire for every collection, + // not just CollectEverything. Static caches like + // StaticPropertyLookupCache prune by mark state and must see valid + // marks before incremental sweep starts freeing cells. + { + ScopedPhaseTimer timer { report, g_phase_timings.sweep_callbacks_us }; + for (auto& callback : m_sweep_callbacks) + callback(); + } +} + void Heap::collect_garbage(CollectionType collection_type, bool print_report) { VERIFY(!m_collecting_garbage); @@ -618,41 +677,13 @@ void Heap::collect_garbage(CollectionType collection_type, bool print_report) mark_live_cells(roots); } } - { - ScopedPhaseTimer timer { report, g_phase_timings.finalize_unmarked_cells_us }; - finalize_unmarked_cells(); - } - { - ScopedPhaseTimer timer { report, g_phase_timings.sweep_weak_blocks_us }; - sweep_weak_blocks(); - } - - // Prune weak containers while we're still stop-the-world; doing this - // during incremental sweep risks reading cells that have already been - // freed and ASAN-poisoned. - { - ScopedPhaseTimer timer { report, g_phase_timings.prune_weak_containers_us }; - for (auto& weak_container : m_weak_containers) { - if (!weak_container.owner_cell({}).is_marked()) - continue; - weak_container.remove_dead_cells({}); - } - } - - // Run sweep callbacks at STW so they fire for every collection, - // not just CollectEverything. Static caches like - // StaticPropertyLookupCache prune by mark state and must see valid - // marks before incremental sweep starts freeing cells. - { - ScopedPhaseTimer timer { report, g_phase_timings.sweep_callbacks_us }; - for (auto& callback : m_sweep_callbacks) - callback(); - } + run_post_mark_phases(report); // For CollectEverything we must finish sweeping synchronously so that // every cell is collected before the Heap destructor returns. All - // other collection types defer sweeping to incremental work below. - if (collection_type == CollectionType::CollectEverything) { + // other collection types defer sweeping to incremental work below, + // unless incremental sweeping is disabled (no event loop to run it). + if (collection_type == CollectionType::CollectEverything || !m_incremental_sweep_enabled) { ScopedPhaseTimer timer { report, g_phase_timings.sweep_dead_cells_us }; sweep_dead_cells(report, collection_measurement_timer); } @@ -674,7 +705,7 @@ void Heap::collect_garbage(CollectionType collection_type, bool print_report) // Arm incremental sweep before running post-GC tasks so any cells those // tasks allocate get tagged as allocated-during-sweep and aren't freed // by sweep_block before the next mark phase reaches them. - if (collection_type != CollectionType::CollectEverything) + if (collection_type != CollectionType::CollectEverything && m_incremental_sweep_enabled) start_incremental_sweep(); else g_next_incremental_sweep_should_report = false; @@ -771,8 +802,16 @@ void Heap::register_sweep_callback(AK::Function callback) m_sweep_callbacks.append(move(callback)); } -void Heap::gather_roots(HashMap& roots, Vector* out_stack_frames) +void Heap::gather_roots(HashMap& roots, Vector* out_stack_frames, IncludeIncomingCrossHeapMembers include_incoming_cross_heap_members) { + // Cross-heap members targeting this heap act as roots for local collections (as the foreign holder is invisible to a local mark). + if (include_incoming_cross_heap_members == IncludeIncomingCrossHeapMembers::Yes) { + for (auto* member : m_incoming_cross_heap_members) { + if (auto* cell = member->cell_base()) + roots.set(cell, HeapRoot { .type = HeapRoot::Type::CrossHeapMember }); + } + } + { ScopedPhaseTimer timer { g_recording_phase_timings, g_phase_timings.gather_must_survive_roots_us }; for_each_block([&](auto& block) { @@ -979,6 +1018,13 @@ NO_SANITIZE_ADDRESS void Heap::gather_conservative_roots(HashMap range) { + for (auto possible_value : range) + add_possible_value(possible_pointers, possible_value, HeapRoot { .type = HeapRoot::Type::ConservativeVector }, min_block_address, max_block_address); + }); + } } for (auto& hash_map : m_conservative_hash_maps) { @@ -1008,19 +1054,41 @@ NO_SANITIZE_ADDRESS void Heap::gather_conservative_roots(HashMap const& roots) - : m_heap(heap) + // The domain is a set of heaps whose cells this mark phase is responsible for; cells outside the domain are not visited. + explicit MarkingVisitor(ReadonlySpan domain, HashMap const& roots) + : m_domain(domain) { - m_heap.find_min_and_max_block_addresses(m_min_block_address, m_max_block_address); + m_min_block_address = explode_byte(0xff); + m_max_block_address = 0; + for (auto* heap : m_domain) { + FlatPtr min_block_address, max_block_address; + heap->find_min_and_max_block_addresses(min_block_address, max_block_address); + m_min_block_address = min(m_min_block_address, min_block_address); + m_max_block_address = max(m_max_block_address, max_block_address); + } for (auto* root : roots.keys()) { visit(root); } } + bool cell_is_in_domain(Cell const& cell) const + { + auto& heap = HeapBlockBase::from_cell(&cell)->heap(); + if (m_domain.size() == 1) [[likely]] + return m_domain.data()[0] == &heap; + for (auto* domain_heap : m_domain) { + if (domain_heap == &heap) + return true; + } + return false; + } + virtual void visit_impl(Cell& cell) override { if (cell.is_marked()) return; + if (!cell_is_in_domain(cell)) + return; dbgln_if(HEAP_DEBUG, " ! {}", &cell); cell.set_marked(true); @@ -1037,6 +1105,8 @@ public: auto& cell = value.as_cell(); if (cell.is_marked()) continue; + if (!cell_is_in_domain(cell)) + continue; dbgln_if(HEAP_DEBUG, " ! {}", &cell); cell.set_marked(true); @@ -1052,14 +1122,16 @@ public: for (size_t i = 0; i < (bytes.size() / sizeof(FlatPtr)); ++i) add_possible_value(possible_pointers, raw_pointer_sized_values[i], HeapRoot { .type = HeapRoot::Type::HeapFunctionCapturedPointer }, m_min_block_address, m_max_block_address); - for_each_cell_among_possible_pointers(m_heap.m_live_heap_blocks, possible_pointers, [&](Cell* cell, FlatPtr) { - if (cell->is_marked()) - return; - if (cell->state() != Cell::State::Live) - return; - cell->set_marked(true); - m_work_queue.append(*cell); - }); + for (auto* heap : m_domain) { + for_each_cell_among_possible_pointers(heap->m_live_heap_blocks, possible_pointers, [&](Cell* cell, FlatPtr) { + if (cell->is_marked()) + return; + if (cell->state() != Cell::State::Live) + return; + cell->set_marked(true); + m_work_queue.append(*cell); + }); + } } void mark_all_live_cells() @@ -1070,20 +1142,26 @@ public: } private: - Heap& m_heap; + ReadonlySpan m_domain; Vector> m_work_queue; FlatPtr m_min_block_address; FlatPtr m_max_block_address; }; void Heap::mark_live_cells(HashMap const& roots) +{ + Heap* domain[] = { this }; + mark_live_cells_across(domain, roots); +} + +void Heap::mark_live_cells_across(ReadonlySpan heaps, HashMap const& roots) { dbgln_if(HEAP_DEBUG, "mark_live_cells:"); Optional visitor; { ScopedPhaseTimer timer { g_recording_phase_timings, g_phase_timings.mark_initial_visit_us }; - visitor.emplace(*this, roots); + visitor.emplace(heaps, roots); } { @@ -1093,10 +1171,12 @@ void Heap::mark_live_cells(HashMap const& roots) { ScopedPhaseTimer timer { g_recording_phase_timings, g_phase_timings.mark_clear_uprooted_us }; - for (auto& inverse_root : m_uprooted_cells) - inverse_root->set_marked(false); + for (auto* heap : heaps) { + for (auto& inverse_root : heap->m_uprooted_cells) + inverse_root->set_marked(false); - m_uprooted_cells.clear(); + heap->m_uprooted_cells.clear(); + } } } diff --git a/Libraries/LibGC/Heap.h b/Libraries/LibGC/Heap.h index ac2ffaf479..95228df562 100644 --- a/Libraries/LibGC/Heap.h +++ b/Libraries/LibGC/Heap.h @@ -21,7 +21,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -43,8 +45,15 @@ class GC_API Heap { AK_MAKE_NONCOPYABLE(Heap); AK_MAKE_NONMOVABLE(Heap); + friend class HeapGroup; + public: - explicit Heap(AK::Function&)> gather_embedder_roots); + enum class BecomeProcessDefault { + No, + Yes, + }; + + explicit Heap(AK::Function&)> gather_embedder_roots, BecomeProcessDefault = BecomeProcessDefault::Yes); ~Heap(); static Heap& the(); @@ -77,6 +86,8 @@ public: AK::JsonObject dump_graph(); bool should_collect_on_every_allocation() const { return m_should_collect_on_every_allocation; } + + void set_incremental_sweep_enabled(bool enabled) { m_incremental_sweep_enabled = enabled; } void set_should_collect_on_every_allocation(bool b) { m_should_collect_on_every_allocation = b; } void did_create_root(Badge, RootImpl&); @@ -96,6 +107,10 @@ public: void did_destroy_conservative_hash_table(Badge, ConservativeHashTableBase&); void did_create_conservative_vector(Badge, ConservativeVectorBase&); void did_destroy_conservative_vector(Badge, ConservativeVectorBase&); + void did_create_conservative_range_provider(Badge, ConservativeRangeProvider&); + void did_destroy_conservative_range_provider(Badge, ConservativeRangeProvider&); + void did_create_cross_heap_member(Badge, CrossHeapMemberBase&); + void did_destroy_cross_heap_member(Badge, CrossHeapMemberBase&); void did_create_weak_container(Badge, WeakContainer&); void did_destroy_weak_container(Badge, WeakContainer&); @@ -103,6 +118,7 @@ public: void register_sweep_callback(AK::Function); void register_cell_allocator(Badge, CellAllocator&); + CellAllocator& cell_allocator_for(Badge, CellAllocatorDescriptorBase&); void uproot_cell(Cell* cell); @@ -135,19 +151,25 @@ private: template Cell* allocate_cell() { - static_assert(requires { T::cell_allocator.allocator.get().allocate_cell(*this); }, "GC cell type must declare its own allocator using GC_DECLARE_ALLOCATOR(ClassName)"); + static_assert(requires { T::cell_allocator.for_heap(*this).allocate_cell(*this); }, "GC cell type must declare its own allocator using GC_DECLARE_ALLOCATOR(ClassName)"); static_assert(IsSame, "GC cell allocator type mismatch"); will_allocate(sizeof(T)); - return T::cell_allocator.allocator.get().allocate_cell(*this); + return T::cell_allocator.for_heap(*this).allocate_cell(*this); } void will_allocate(size_t); void update_gc_bytes_threshold(size_t live_cell_bytes, size_t live_external_bytes); void find_min_and_max_block_addresses(FlatPtr& min_address, FlatPtr& max_address); - void gather_roots(HashMap&, Vector* out_stack_frames = nullptr); + enum class IncludeIncomingCrossHeapMembers { + No, + Yes, + }; + void gather_roots(HashMap&, Vector* out_stack_frames = nullptr, IncludeIncomingCrossHeapMembers = IncludeIncomingCrossHeapMembers::Yes); + static void mark_live_cells_across(ReadonlySpan, HashMap const& roots); + void run_post_mark_phases(bool report); void gather_conservative_roots(HashMap&, Vector* out_stack_frames = nullptr); void gather_asan_fake_stack_roots(HashMap&, FlatPtr, FlatPtr min_block_address, FlatPtr max_block_address, FlatPtr stack_reference, FlatPtr stack_top); void mark_live_cells(HashMap const& live_cells); @@ -182,6 +204,7 @@ private: bool m_should_collect_on_every_allocation { false }; CellAllocator::List m_all_cell_allocators; + HashMap> m_cell_allocators_by_type; RootImpl::List m_roots; RootVectorBase::List m_root_vectors; @@ -190,6 +213,10 @@ private: ConservativeHashMapBase::List m_conservative_hash_maps; ConservativeHashTableBase::List m_conservative_hash_tables; ConservativeVectorBase::List m_conservative_vectors; + ConservativeRangeProvider::List m_conservative_range_providers; + HashTable m_incoming_cross_heap_members; + HeapGroup* m_group { nullptr }; + bool m_incremental_sweep_enabled { true }; WeakContainer::List m_weak_containers; Vector> m_uprooted_cells; @@ -305,6 +332,30 @@ inline void Heap::did_destroy_conservative_vector(Badge, m_conservative_vectors.remove(vector); } +inline void Heap::did_create_conservative_range_provider(Badge, ConservativeRangeProvider& provider) +{ + VERIFY(!m_conservative_range_providers.contains(provider)); + m_conservative_range_providers.append(provider); +} + +inline void Heap::did_destroy_conservative_range_provider(Badge, ConservativeRangeProvider& provider) +{ + VERIFY(m_conservative_range_providers.contains(provider)); + m_conservative_range_providers.remove(provider); +} + +inline void Heap::did_create_cross_heap_member(Badge, CrossHeapMemberBase& member) +{ + VERIFY(!m_incoming_cross_heap_members.contains(&member)); + m_incoming_cross_heap_members.set(&member); +} + +inline void Heap::did_destroy_cross_heap_member(Badge, CrossHeapMemberBase& member) +{ + VERIFY(m_incoming_cross_heap_members.contains(&member)); + m_incoming_cross_heap_members.remove(&member); +} + inline void Heap::did_create_weak_container(Badge, WeakContainer& set) { VERIFY(!m_weak_containers.contains(set)); diff --git a/Libraries/LibGC/HeapGroup.cpp b/Libraries/LibGC/HeapGroup.cpp new file mode 100644 index 0000000000..5522572a0e --- /dev/null +++ b/Libraries/LibGC/HeapGroup.cpp @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2026, Ali Mohammad Pur + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include +#include +#include +#include + +namespace GC { + +HeapGroup::~HeapGroup() +{ + for (auto* heap : m_heaps) + heap->m_group = nullptr; +} + +void HeapGroup::add(Heap& heap) +{ + VERIFY(!heap.m_group); + heap.m_group = this; + m_heaps.append(&heap); +} + +void HeapGroup::remove(Heap& heap) +{ + VERIFY(heap.m_group == this); + heap.m_group = nullptr; + m_heaps.remove_first_matching([&](auto* entry) { return entry == &heap; }); +} + +void HeapGroup::collect_garbage(bool print_report) +{ + // Defer all member heaps' collections until the last one, so that cross-heap edges are visible to the mark phase. + for (auto* heap : m_heaps) { + VERIFY(!heap->m_collecting_garbage); + if (heap->m_gc_deferrals) { + heap->m_should_gc_when_deferral_ends = true; + return; + } + } + + for (auto* heap : m_heaps) { + heap->finish_pending_incremental_sweep(); + heap->m_collecting_garbage = true; + } + ScopeGuard unset_collecting = [&] { + for (auto* heap : m_heaps) + heap->m_collecting_garbage = false; + }; + + HashMap roots; + for (auto* heap : m_heaps) + heap->gather_roots(roots, nullptr, Heap::IncludeIncomingCrossHeapMembers::No); + + Heap::mark_live_cells_across(m_heaps, roots); + + for (auto* heap : m_heaps) + heap->run_post_mark_phases(print_report); + + for (auto* heap : m_heaps) { + Core::ElapsedTimer measurement_timer { Core::TimerType::Precise }; + measurement_timer.start(); + heap->sweep_dead_cells(print_report, measurement_timer); + } +} + +} diff --git a/Libraries/LibGC/HeapGroup.h b/Libraries/LibGC/HeapGroup.h new file mode 100644 index 0000000000..2194570e5c --- /dev/null +++ b/Libraries/LibGC/HeapGroup.h @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2026, Ali Mohammad Pur + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#pragma once + +#include +#include + +namespace GC { + +// A group of heaps whose objects may reference each other through CrossHeapMember edges. +class GC_API HeapGroup { + AK_MAKE_NONCOPYABLE(HeapGroup); + AK_MAKE_NONMOVABLE(HeapGroup); + +public: + HeapGroup() = default; + ~HeapGroup(); + + void add(Heap&); + void remove(Heap&); + + void collect_garbage(bool print_report = false); + +private: + Vector m_heaps; +}; + +} diff --git a/Libraries/LibGC/HeapRoot.h b/Libraries/LibGC/HeapRoot.h index 124224d790..d5725a27d5 100644 --- a/Libraries/LibGC/HeapRoot.h +++ b/Libraries/LibGC/HeapRoot.h @@ -17,6 +17,7 @@ struct GC_API HeapRoot { ConservativeHashMap, ConservativeHashTable, ConservativeVector, + CrossHeapMember, HeapFunctionCapturedPointer, MustSurviveGC, RegisterPointer, diff --git a/Tests/LibGC/CMakeLists.txt b/Tests/LibGC/CMakeLists.txt index bd03943f2e..d631a3a630 100644 --- a/Tests/LibGC/CMakeLists.txt +++ b/Tests/LibGC/CMakeLists.txt @@ -1,5 +1,6 @@ set(TEST_SOURCES TestGCContainers.cpp + TestGCHeapGroup.cpp TestGCIdleCollection.cpp TestGCVisitor.cpp ) diff --git a/Tests/LibGC/TestGCHeapGroup.cpp b/Tests/LibGC/TestGCHeapGroup.cpp new file mode 100644 index 0000000000..13bf65823a --- /dev/null +++ b/Tests/LibGC/TestGCHeapGroup.cpp @@ -0,0 +1,188 @@ +/* + * Copyright (c) 2026, Ali Mohammad Pur + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +size_t s_live_linked_cells = 0; + +class LinkedCell final : public GC::Cell { + GC_CELL(LinkedCell, GC::Cell); + GC_DECLARE_ALLOCATOR(LinkedCell); + +public: + virtual ~LinkedCell() override { --s_live_linked_cells; } + + GC::CrossHeapMember& foreign() { return m_foreign; } + GC::Ptr& local() { return m_local; } + +private: + LinkedCell() { ++s_live_linked_cells; } + + virtual void visit_edges(Visitor& visitor) override + { + Base::visit_edges(visitor); + visitor.visit(m_local); + m_foreign.visit(visitor); + } + + GC::CrossHeapMember m_foreign; + GC::Ptr m_local; +}; + +GC_DEFINE_ALLOCATOR(LinkedCell); + +NEVER_INLINE void scrub_stack() +{ + u8 volatile filler[8 * KiB]; + for (size_t i = 0; i < sizeof(filler); ++i) + filler[i] = 0; +} + +NEVER_INLINE GC::Root allocate_holder_and_foreign_target(GC::Heap& holder_heap, GC::Heap& target_heap) +{ + auto holder = GC::make_root(holder_heap.allocate()); + auto target = target_heap.allocate(); + holder->foreign() = target.ptr(); + return holder; +} + +NEVER_INLINE void allocate_cross_heap_cycle(GC::Heap& heap_a, GC::Heap& heap_b) +{ + auto cell_on_a = heap_a.allocate(); + auto cell_on_b = heap_b.allocate(); + cell_on_a->foreign() = cell_on_b.ptr(); + cell_on_b->foreign() = cell_on_a.ptr(); +} + +NEVER_INLINE GC::Root allocate_cross_heap_chain(GC::Heap& heap_a, GC::Heap& heap_b) +{ + // root -> A -> B -> A(second) + auto holder = GC::make_root(heap_a.allocate()); + auto middle = heap_b.allocate(); + auto tail = heap_a.allocate(); + middle->foreign() = tail.ptr(); + holder->foreign() = middle.ptr(); + return holder; +} + +NEVER_INLINE void allocate_garbage(GC::Heap& heap) +{ + (void)heap.allocate(); +} + +} + +TEST_CASE(sanity_single_heap_frees_garbage) +{ + GC::Heap heap([](auto&) { }, GC::Heap::BecomeProcessDefault::No); + heap.set_incremental_sweep_enabled(false); + + allocate_garbage(heap); + EXPECT_EQ(s_live_linked_cells, 1u); + scrub_stack(); + heap.collect_garbage(); + EXPECT_EQ(s_live_linked_cells, 0u); +} + +TEST_CASE(incoming_cross_heap_member_roots_local_collection) +{ + GC::Heap heap_a([](auto&) { }, GC::Heap::BecomeProcessDefault::No); + GC::Heap heap_b([](auto&) { }, GC::Heap::BecomeProcessDefault::No); + // No event loop runs during this test; sweep synchronously so frees are observable. + heap_a.set_incremental_sweep_enabled(false); + heap_b.set_incremental_sweep_enabled(false); + GC::HeapGroup group; + group.add(heap_a); + group.add(heap_b); + + auto holder = allocate_holder_and_foreign_target(heap_a, heap_b); + EXPECT_EQ(s_live_linked_cells, 2u); + + // The only thing keeping the target alive is the incoming cross-heap member; B's local collection cannot see the holder on A, so the member registration must root it. + scrub_stack(); + heap_b.collect_garbage(); + EXPECT_EQ(s_live_linked_cells, 2u); + + // Once the edge is dropped, B's next local collection should free the target. + holder->foreign() = nullptr; + scrub_stack(); + heap_b.collect_garbage(); + EXPECT_EQ(s_live_linked_cells, 1u); + + holder = {}; + scrub_stack(); + heap_a.collect_garbage(); + EXPECT_EQ(s_live_linked_cells, 0u); + + group.remove(heap_a); + group.remove(heap_b); +} + +TEST_CASE(group_collection_breaks_cross_heap_cycles) +{ + GC::Heap heap_a([](auto&) { }, GC::Heap::BecomeProcessDefault::No); + GC::Heap heap_b([](auto&) { }, GC::Heap::BecomeProcessDefault::No); + + heap_a.set_incremental_sweep_enabled(false); + heap_b.set_incremental_sweep_enabled(false); + GC::HeapGroup group; + group.add(heap_a); + group.add(heap_b); + + allocate_cross_heap_cycle(heap_a, heap_b); + EXPECT_EQ(s_live_linked_cells, 2u); + + // Local collections see the incoming members as roots, so the boundary cycle should survive collection. + scrub_stack(); + heap_a.collect_garbage(); + heap_b.collect_garbage(); + EXPECT_EQ(s_live_linked_cells, 2u); + + // The unified mark should not reach the cycle, so both cells should be freed by the group collection. + scrub_stack(); + group.collect_garbage(); + EXPECT_EQ(s_live_linked_cells, 0u); + + group.remove(heap_a); + group.remove(heap_b); +} + +TEST_CASE(group_collection_traces_live_cross_heap_chains) +{ + GC::Heap heap_a([](auto&) { }, GC::Heap::BecomeProcessDefault::No); + GC::Heap heap_b([](auto&) { }, GC::Heap::BecomeProcessDefault::No); + + heap_a.set_incremental_sweep_enabled(false); + heap_b.set_incremental_sweep_enabled(false); + GC::HeapGroup group; + group.add(heap_a); + group.add(heap_b); + + auto holder = allocate_cross_heap_chain(heap_a, heap_b); + EXPECT_EQ(s_live_linked_cells, 3u); + + // The whole chain must survive a group collection, including the second A cell that is only reachable through the B cell. + scrub_stack(); + group.collect_garbage(); + EXPECT_EQ(s_live_linked_cells, 3u); + + holder = {}; + scrub_stack(); + group.collect_garbage(); + EXPECT_EQ(s_live_linked_cells, 0u); + + group.remove(heap_a); + group.remove(heap_b); +}