LibGC: Add support for coordinated multi-heap collection

Allow having separate GC heaps and implement coordinated marking between
them; this is useful for keeping wasm and js GC heaps separated with a
clear boundary.
This commit is contained in:
Ali Mohammad Pur 2026-06-13 13:18:54 +02:00 committed by Ali Mohammad Pur
parent 62cb073ada
commit 76f17f7703
15 changed files with 702 additions and 60 deletions

View file

@ -4,12 +4,15 @@ set(SOURCES
CellAllocator.cpp
ConservativeHashMap.cpp
ConservativeHashTable.cpp
ConservativeRangeProvider.cpp
ConservativeVector.cpp
CrossHeapMember.cpp
Root.cpp
RootHashMap.cpp
RootHashTable.cpp
RootVector.cpp
Heap.cpp
HeapGroup.cpp
HeapBlock.cpp
Timer.cpp
WeakBlock.cpp

View file

@ -20,6 +20,16 @@ CellAllocator::CellAllocator(size_t cell_size, Optional<StringView> class_name,
{
}
CellAllocator& CellAllocatorDescriptorBase::for_heap(Heap& heap)
{
if (m_last_heap == &heap) [[likely]]
return *m_last_allocator;
auto& allocator = heap.cell_allocator_for({}, *this);
m_last_heap = &heap;
m_last_allocator = &allocator;
return allocator;
}
Cell* CellAllocator::allocate_cell(Heap& heap)
{
if (!m_list_node.is_in_list())

View file

@ -22,6 +22,45 @@
namespace GC {
class GC_API CellAllocatorDescriptorBase {
AK_MAKE_NONCOPYABLE(CellAllocatorDescriptorBase);
AK_MAKE_NONMOVABLE(CellAllocatorDescriptorBase);
public:
Optional<StringView> class_name() const { return m_class_name; }
size_t cell_size() const { return m_cell_size; }
bool overrides_must_survive_garbage_collection() const { return m_overrides_must_survive_garbage_collection; }
bool overrides_finalize() const { return m_overrides_finalize; }
CellAllocator& for_heap(Heap&);
void forget_heap(Badge<Heap>, Heap& heap)
{
if (m_last_heap == &heap) {
m_last_heap = nullptr;
m_last_allocator = nullptr;
}
}
protected:
CellAllocatorDescriptorBase(size_t cell_size, StringView class_name, bool overrides_must_survive_garbage_collection, bool overrides_finalize)
: m_class_name(class_name)
, m_cell_size(cell_size)
, m_overrides_must_survive_garbage_collection(overrides_must_survive_garbage_collection)
, m_overrides_finalize(overrides_finalize)
{
}
private:
Optional<StringView> m_class_name;
size_t m_cell_size { 0 };
bool m_overrides_must_survive_garbage_collection { false };
bool m_overrides_finalize { false };
Heap* m_last_heap { nullptr };
CellAllocator* m_last_allocator { nullptr };
};
class GC_API CellAllocator {
public:
CellAllocator(size_t cell_size, Optional<StringView> = {}, bool overrides_must_survive_garbage_collection = false, bool overrides_finalize = false);
@ -81,16 +120,14 @@ private:
};
template<typename T>
class GC_API TypeIsolatingCellAllocator {
class GC_API TypeIsolatingCellAllocator final : public CellAllocatorDescriptorBase {
public:
using CellType = T;
TypeIsolatingCellAllocator(StringView class_name, bool overrides_must_survive_garbage_collection, bool overrides_finalize)
: allocator(sizeof(T), class_name, overrides_must_survive_garbage_collection, overrides_finalize)
: CellAllocatorDescriptorBase(sizeof(T), class_name, overrides_must_survive_garbage_collection, overrides_finalize)
{
}
NeverDestroyed<CellAllocator> allocator;
};
}

View file

@ -0,0 +1,24 @@
/*
* Copyright (c) 2026, Ali Mohammad Pur <ali@ladybird.org>
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#include <LibGC/ConservativeRangeProvider.h>
#include <LibGC/Heap.h>
namespace GC {
ConservativeRangeProvider::ConservativeRangeProvider(Heap& heap)
: m_heap(&heap)
{
m_heap->did_create_conservative_range_provider({}, *this);
}
ConservativeRangeProvider::~ConservativeRangeProvider()
{
if (m_heap)
m_heap->did_destroy_conservative_range_provider({}, *this);
}
}

View file

@ -0,0 +1,38 @@
/*
* Copyright (c) 2026, Ali Mohammad Pur <ali@ladybird.org>
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#pragma once
#include <AK/Function.h>
#include <AK/IntrusiveList.h>
#include <AK/Span.h>
#include <LibGC/Forward.h>
namespace GC {
class GC_API ConservativeRangeProvider {
AK_MAKE_NONCOPYABLE(ConservativeRangeProvider);
AK_MAKE_NONMOVABLE(ConservativeRangeProvider);
public:
virtual ~ConservativeRangeProvider();
// Called while gathering roots; every word of every reported range is treated as a possible cell pointer.
virtual void for_each_conservative_range(AK::Function<void(ReadonlySpan<FlatPtr>)> const&) const = 0;
void detach_from_heap(Badge<Heap>) { m_heap = nullptr; }
protected:
explicit ConservativeRangeProvider(Heap&);
Heap* m_heap { nullptr };
IntrusiveListNode<ConservativeRangeProvider> m_list_node;
public:
using List = IntrusiveList<&ConservativeRangeProvider::m_list_node>;
};
}

View file

@ -0,0 +1,28 @@
/*
* Copyright (c) 2026, Ali Mohammad Pur <ali@ladybird.org>
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#include <LibGC/CrossHeapMember.h>
#include <LibGC/Heap.h>
namespace GC {
void CrossHeapMemberBase::reset(Cell* cell)
{
auto* new_heap = cell ? &HeapBlockBase::from_cell(cell)->heap() : nullptr;
if (m_registered_heap == new_heap) {
m_cell = cell;
return;
}
if (m_registered_heap)
m_registered_heap->did_destroy_cross_heap_member({}, *this);
m_cell = cell;
m_registered_heap = new_heap;
if (m_registered_heap)
m_registered_heap->did_create_cross_heap_member({}, *this);
}
}

View file

@ -0,0 +1,77 @@
/*
* Copyright (c) 2026, Ali Mohammad Pur <ali@ladybird.org>
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#pragma once
#include <LibGC/Cell.h>
#include <LibGC/Forward.h>
#include <LibGC/Internals.h>
namespace GC {
// A strong reference from a cell on one heap to a cell on another heap in the same HeapGroup.
class GC_API CrossHeapMemberBase {
AK_MAKE_NONCOPYABLE(CrossHeapMemberBase);
AK_MAKE_NONMOVABLE(CrossHeapMemberBase);
public:
Cell* cell_base() const { return static_cast<Cell*>(m_cell); }
void detach_from_heap(Badge<Heap>)
{
m_registered_heap = nullptr;
m_cell = nullptr;
}
protected:
explicit CrossHeapMemberBase(Cell* cell)
{
reset(cell);
}
~CrossHeapMemberBase()
{
reset(nullptr);
}
void reset(Cell* cell);
void* m_cell { nullptr }; // Not a member of "this" heap.
Heap* m_registered_heap { nullptr };
};
template<typename T>
class CrossHeapMember final : public CrossHeapMemberBase {
public:
CrossHeapMember()
: CrossHeapMemberBase(nullptr)
{
}
explicit CrossHeapMember(T* cell)
: CrossHeapMemberBase(cell)
{
}
CrossHeapMember& operator=(T* cell)
{
reset(cell);
return *this;
}
T* ptr() const { return static_cast<T*>(m_cell); }
T* operator->() const { return ptr(); }
explicit operator bool() const { return m_cell != nullptr; }
// Called by the holder heap.
void visit(Cell::Visitor& visitor) const
{
if (m_cell)
visitor.visit(cell_base());
}
};
}

View file

@ -16,6 +16,8 @@ class CellAllocator;
class DeferGC;
class RootImpl;
class Heap;
class HeapGroup;
class CrossHeapMemberBase;
class HeapBlock;
class NanBoxedValue;
class Timer;

View file

@ -285,9 +285,17 @@ void Heap::set_default_heap_for_testing(Heap& heap)
s_the = &heap;
}
Heap::Heap(AK::Function<void(HashMap<Cell*, GC::HeapRoot>&)> gather_embedder_roots)
CellAllocator& Heap::cell_allocator_for(Badge<CellAllocatorDescriptorBase>, CellAllocatorDescriptorBase& descriptor)
{
return *m_cell_allocators_by_type.ensure(&descriptor, [&] {
return make<CellAllocator>(descriptor.cell_size(), descriptor.class_name(), descriptor.overrides_must_survive_garbage_collection(), descriptor.overrides_finalize());
});
}
Heap::Heap(AK::Function<void(HashMap<Cell*, GC::HeapRoot>&)> gather_embedder_roots, BecomeProcessDefault become_process_default)
: m_gather_embedder_roots(move(gather_embedder_roots))
{
if (become_process_default == BecomeProcessDefault::Yes)
s_the = this;
m_gc_bytes_threshold = GC_MIN_BYTES_THRESHOLD;
static_assert(HeapBlock::min_possible_cell_size <= 32, "Heap Cell tracking uses too much data!");
@ -296,6 +304,20 @@ Heap::Heap(AK::Function<void(HashMap<Cell*, GC::HeapRoot>&)> gather_embedder_roo
Heap::~Heap()
{
collect_garbage(CollectionType::CollectEverything);
for (auto& entry : m_cell_allocators_by_type)
entry.key->forget_heap({}, *this);
while (auto* allocator = m_all_cell_allocators.first())
m_all_cell_allocators.remove(*allocator);
while (auto* provider = m_conservative_range_providers.first()) {
m_conservative_range_providers.remove(*provider);
provider->detach_from_heap({});
}
for (auto* member : m_incoming_cross_heap_members)
member->detach_from_heap({});
m_incoming_cross_heap_members.clear();
}
void Heap::will_allocate(size_t size)
@ -492,6 +514,9 @@ public:
case HeapRoot::Type::ConservativeVector:
node.set("root"sv, "ConservativeVector"sv);
break;
case HeapRoot::Type::CrossHeapMember:
node.set("root"sv, "CrossHeapMember"sv);
break;
case HeapRoot::Type::HeapFunctionCapturedPointer:
node.set("root"sv, "HeapFunctionCapturedPointer"sv);
break;
@ -579,6 +604,40 @@ AK::JsonObject Heap::dump_graph()
return graph;
}
void Heap::run_post_mark_phases(bool report)
{
{
ScopedPhaseTimer timer { report, g_phase_timings.finalize_unmarked_cells_us };
finalize_unmarked_cells();
}
{
ScopedPhaseTimer timer { report, g_phase_timings.sweep_weak_blocks_us };
sweep_weak_blocks();
}
// Prune weak containers while we're still stop-the-world; doing this
// during incremental sweep risks reading cells that have already been
// freed and ASAN-poisoned.
{
ScopedPhaseTimer timer { report, g_phase_timings.prune_weak_containers_us };
for (auto& weak_container : m_weak_containers) {
if (!weak_container.owner_cell({}).is_marked())
continue;
weak_container.remove_dead_cells({});
}
}
// Run sweep callbacks at STW so they fire for every collection,
// not just CollectEverything. Static caches like
// StaticPropertyLookupCache prune by mark state and must see valid
// marks before incremental sweep starts freeing cells.
{
ScopedPhaseTimer timer { report, g_phase_timings.sweep_callbacks_us };
for (auto& callback : m_sweep_callbacks)
callback();
}
}
void Heap::collect_garbage(CollectionType collection_type, bool print_report)
{
VERIFY(!m_collecting_garbage);
@ -618,41 +677,13 @@ void Heap::collect_garbage(CollectionType collection_type, bool print_report)
mark_live_cells(roots);
}
}
{
ScopedPhaseTimer timer { report, g_phase_timings.finalize_unmarked_cells_us };
finalize_unmarked_cells();
}
{
ScopedPhaseTimer timer { report, g_phase_timings.sweep_weak_blocks_us };
sweep_weak_blocks();
}
// Prune weak containers while we're still stop-the-world; doing this
// during incremental sweep risks reading cells that have already been
// freed and ASAN-poisoned.
{
ScopedPhaseTimer timer { report, g_phase_timings.prune_weak_containers_us };
for (auto& weak_container : m_weak_containers) {
if (!weak_container.owner_cell({}).is_marked())
continue;
weak_container.remove_dead_cells({});
}
}
// Run sweep callbacks at STW so they fire for every collection,
// not just CollectEverything. Static caches like
// StaticPropertyLookupCache prune by mark state and must see valid
// marks before incremental sweep starts freeing cells.
{
ScopedPhaseTimer timer { report, g_phase_timings.sweep_callbacks_us };
for (auto& callback : m_sweep_callbacks)
callback();
}
run_post_mark_phases(report);
// For CollectEverything we must finish sweeping synchronously so that
// every cell is collected before the Heap destructor returns. All
// other collection types defer sweeping to incremental work below.
if (collection_type == CollectionType::CollectEverything) {
// other collection types defer sweeping to incremental work below,
// unless incremental sweeping is disabled (no event loop to run it).
if (collection_type == CollectionType::CollectEverything || !m_incremental_sweep_enabled) {
ScopedPhaseTimer timer { report, g_phase_timings.sweep_dead_cells_us };
sweep_dead_cells(report, collection_measurement_timer);
}
@ -674,7 +705,7 @@ void Heap::collect_garbage(CollectionType collection_type, bool print_report)
// Arm incremental sweep before running post-GC tasks so any cells those
// tasks allocate get tagged as allocated-during-sweep and aren't freed
// by sweep_block before the next mark phase reaches them.
if (collection_type != CollectionType::CollectEverything)
if (collection_type != CollectionType::CollectEverything && m_incremental_sweep_enabled)
start_incremental_sweep();
else
g_next_incremental_sweep_should_report = false;
@ -771,8 +802,16 @@ void Heap::register_sweep_callback(AK::Function<void()> callback)
m_sweep_callbacks.append(move(callback));
}
void Heap::gather_roots(HashMap<Cell*, HeapRoot>& roots, Vector<StackFrameInfo>* out_stack_frames)
void Heap::gather_roots(HashMap<Cell*, HeapRoot>& roots, Vector<StackFrameInfo>* out_stack_frames, IncludeIncomingCrossHeapMembers include_incoming_cross_heap_members)
{
// Cross-heap members targeting this heap act as roots for local collections (as the foreign holder is invisible to a local mark).
if (include_incoming_cross_heap_members == IncludeIncomingCrossHeapMembers::Yes) {
for (auto* member : m_incoming_cross_heap_members) {
if (auto* cell = member->cell_base())
roots.set(cell, HeapRoot { .type = HeapRoot::Type::CrossHeapMember });
}
}
{
ScopedPhaseTimer timer { g_recording_phase_timings, g_phase_timings.gather_must_survive_roots_us };
for_each_block([&](auto& block) {
@ -979,6 +1018,13 @@ NO_SANITIZE_ADDRESS void Heap::gather_conservative_roots(HashMap<Cell*, HeapRoot
add_possible_value(possible_pointers, possible_value, HeapRoot { .type = HeapRoot::Type::ConservativeVector }, min_block_address, max_block_address);
}
}
for (auto& provider : m_conservative_range_providers) {
provider.for_each_conservative_range([&](ReadonlySpan<FlatPtr> range) {
for (auto possible_value : range)
add_possible_value(possible_pointers, possible_value, HeapRoot { .type = HeapRoot::Type::ConservativeVector }, min_block_address, max_block_address);
});
}
}
for (auto& hash_map : m_conservative_hash_maps) {
@ -1008,19 +1054,41 @@ NO_SANITIZE_ADDRESS void Heap::gather_conservative_roots(HashMap<Cell*, HeapRoot
class MarkingVisitor final : public Cell::Visitor {
public:
explicit MarkingVisitor(Heap& heap, HashMap<Cell*, HeapRoot> const& roots)
: m_heap(heap)
// The domain is a set of heaps whose cells this mark phase is responsible for; cells outside the domain are not visited.
explicit MarkingVisitor(ReadonlySpan<Heap* const> domain, HashMap<Cell*, HeapRoot> const& roots)
: m_domain(domain)
{
m_heap.find_min_and_max_block_addresses(m_min_block_address, m_max_block_address);
m_min_block_address = explode_byte(0xff);
m_max_block_address = 0;
for (auto* heap : m_domain) {
FlatPtr min_block_address, max_block_address;
heap->find_min_and_max_block_addresses(min_block_address, max_block_address);
m_min_block_address = min(m_min_block_address, min_block_address);
m_max_block_address = max(m_max_block_address, max_block_address);
}
for (auto* root : roots.keys()) {
visit(root);
}
}
bool cell_is_in_domain(Cell const& cell) const
{
auto& heap = HeapBlockBase::from_cell(&cell)->heap();
if (m_domain.size() == 1) [[likely]]
return m_domain.data()[0] == &heap;
for (auto* domain_heap : m_domain) {
if (domain_heap == &heap)
return true;
}
return false;
}
virtual void visit_impl(Cell& cell) override
{
if (cell.is_marked())
return;
if (!cell_is_in_domain(cell))
return;
dbgln_if(HEAP_DEBUG, " ! {}", &cell);
cell.set_marked(true);
@ -1037,6 +1105,8 @@ public:
auto& cell = value.as_cell();
if (cell.is_marked())
continue;
if (!cell_is_in_domain(cell))
continue;
dbgln_if(HEAP_DEBUG, " ! {}", &cell);
cell.set_marked(true);
@ -1052,7 +1122,8 @@ public:
for (size_t i = 0; i < (bytes.size() / sizeof(FlatPtr)); ++i)
add_possible_value(possible_pointers, raw_pointer_sized_values[i], HeapRoot { .type = HeapRoot::Type::HeapFunctionCapturedPointer }, m_min_block_address, m_max_block_address);
for_each_cell_among_possible_pointers(m_heap.m_live_heap_blocks, possible_pointers, [&](Cell* cell, FlatPtr) {
for (auto* heap : m_domain) {
for_each_cell_among_possible_pointers(heap->m_live_heap_blocks, possible_pointers, [&](Cell* cell, FlatPtr) {
if (cell->is_marked())
return;
if (cell->state() != Cell::State::Live)
@ -1061,6 +1132,7 @@ public:
m_work_queue.append(*cell);
});
}
}
void mark_all_live_cells()
{
@ -1070,20 +1142,26 @@ public:
}
private:
Heap& m_heap;
ReadonlySpan<Heap* const> m_domain;
Vector<Ref<Cell>> m_work_queue;
FlatPtr m_min_block_address;
FlatPtr m_max_block_address;
};
void Heap::mark_live_cells(HashMap<Cell*, HeapRoot> const& roots)
{
Heap* domain[] = { this };
mark_live_cells_across(domain, roots);
}
void Heap::mark_live_cells_across(ReadonlySpan<Heap* const> heaps, HashMap<Cell*, HeapRoot> const& roots)
{
dbgln_if(HEAP_DEBUG, "mark_live_cells:");
Optional<MarkingVisitor> visitor;
{
ScopedPhaseTimer timer { g_recording_phase_timings, g_phase_timings.mark_initial_visit_us };
visitor.emplace(*this, roots);
visitor.emplace(heaps, roots);
}
{
@ -1093,10 +1171,12 @@ void Heap::mark_live_cells(HashMap<Cell*, HeapRoot> const& roots)
{
ScopedPhaseTimer timer { g_recording_phase_timings, g_phase_timings.mark_clear_uprooted_us };
for (auto& inverse_root : m_uprooted_cells)
for (auto* heap : heaps) {
for (auto& inverse_root : heap->m_uprooted_cells)
inverse_root->set_marked(false);
m_uprooted_cells.clear();
heap->m_uprooted_cells.clear();
}
}
}

View file

@ -21,7 +21,9 @@
#include <LibGC/CellAllocator.h>
#include <LibGC/ConservativeHashMap.h>
#include <LibGC/ConservativeHashTable.h>
#include <LibGC/ConservativeRangeProvider.h>
#include <LibGC/ConservativeVector.h>
#include <LibGC/CrossHeapMember.h>
#include <LibGC/Forward.h>
#include <LibGC/HeapRoot.h>
#include <LibGC/IdleCollectionPolicy.h>
@ -43,8 +45,15 @@ class GC_API Heap {
AK_MAKE_NONCOPYABLE(Heap);
AK_MAKE_NONMOVABLE(Heap);
friend class HeapGroup;
public:
explicit Heap(AK::Function<void(HashMap<Cell*, GC::HeapRoot>&)> gather_embedder_roots);
enum class BecomeProcessDefault {
No,
Yes,
};
explicit Heap(AK::Function<void(HashMap<Cell*, GC::HeapRoot>&)> gather_embedder_roots, BecomeProcessDefault = BecomeProcessDefault::Yes);
~Heap();
static Heap& the();
@ -77,6 +86,8 @@ public:
AK::JsonObject dump_graph();
bool should_collect_on_every_allocation() const { return m_should_collect_on_every_allocation; }
void set_incremental_sweep_enabled(bool enabled) { m_incremental_sweep_enabled = enabled; }
void set_should_collect_on_every_allocation(bool b) { m_should_collect_on_every_allocation = b; }
void did_create_root(Badge<RootImpl>, RootImpl&);
@ -96,6 +107,10 @@ public:
void did_destroy_conservative_hash_table(Badge<ConservativeHashTableBase>, ConservativeHashTableBase&);
void did_create_conservative_vector(Badge<ConservativeVectorBase>, ConservativeVectorBase&);
void did_destroy_conservative_vector(Badge<ConservativeVectorBase>, ConservativeVectorBase&);
void did_create_conservative_range_provider(Badge<ConservativeRangeProvider>, ConservativeRangeProvider&);
void did_destroy_conservative_range_provider(Badge<ConservativeRangeProvider>, ConservativeRangeProvider&);
void did_create_cross_heap_member(Badge<CrossHeapMemberBase>, CrossHeapMemberBase&);
void did_destroy_cross_heap_member(Badge<CrossHeapMemberBase>, CrossHeapMemberBase&);
void did_create_weak_container(Badge<WeakContainer>, WeakContainer&);
void did_destroy_weak_container(Badge<WeakContainer>, WeakContainer&);
@ -103,6 +118,7 @@ public:
void register_sweep_callback(AK::Function<void()>);
void register_cell_allocator(Badge<CellAllocator>, CellAllocator&);
CellAllocator& cell_allocator_for(Badge<CellAllocatorDescriptorBase>, CellAllocatorDescriptorBase&);
void uproot_cell(Cell* cell);
@ -135,19 +151,25 @@ private:
template<typename T>
Cell* allocate_cell()
{
static_assert(requires { T::cell_allocator.allocator.get().allocate_cell(*this); }, "GC cell type must declare its own allocator using GC_DECLARE_ALLOCATOR(ClassName)");
static_assert(requires { T::cell_allocator.for_heap(*this).allocate_cell(*this); }, "GC cell type must declare its own allocator using GC_DECLARE_ALLOCATOR(ClassName)");
static_assert(IsSame<T, typename decltype(T::cell_allocator)::CellType>,
"GC cell allocator type mismatch");
will_allocate(sizeof(T));
return T::cell_allocator.allocator.get().allocate_cell(*this);
return T::cell_allocator.for_heap(*this).allocate_cell(*this);
}
void will_allocate(size_t);
void update_gc_bytes_threshold(size_t live_cell_bytes, size_t live_external_bytes);
void find_min_and_max_block_addresses(FlatPtr& min_address, FlatPtr& max_address);
void gather_roots(HashMap<Cell*, HeapRoot>&, Vector<StackFrameInfo>* out_stack_frames = nullptr);
enum class IncludeIncomingCrossHeapMembers {
No,
Yes,
};
void gather_roots(HashMap<Cell*, HeapRoot>&, Vector<StackFrameInfo>* out_stack_frames = nullptr, IncludeIncomingCrossHeapMembers = IncludeIncomingCrossHeapMembers::Yes);
static void mark_live_cells_across(ReadonlySpan<Heap* const>, HashMap<Cell*, HeapRoot> const& roots);
void run_post_mark_phases(bool report);
void gather_conservative_roots(HashMap<Cell*, HeapRoot>&, Vector<StackFrameInfo>* out_stack_frames = nullptr);
void gather_asan_fake_stack_roots(HashMap<FlatPtr, HeapRoot>&, FlatPtr, FlatPtr min_block_address, FlatPtr max_block_address, FlatPtr stack_reference, FlatPtr stack_top);
void mark_live_cells(HashMap<Cell*, HeapRoot> const& live_cells);
@ -182,6 +204,7 @@ private:
bool m_should_collect_on_every_allocation { false };
CellAllocator::List m_all_cell_allocators;
HashMap<CellAllocatorDescriptorBase*, NonnullOwnPtr<CellAllocator>> m_cell_allocators_by_type;
RootImpl::List m_roots;
RootVectorBase::List m_root_vectors;
@ -190,6 +213,10 @@ private:
ConservativeHashMapBase::List m_conservative_hash_maps;
ConservativeHashTableBase::List m_conservative_hash_tables;
ConservativeVectorBase::List m_conservative_vectors;
ConservativeRangeProvider::List m_conservative_range_providers;
HashTable<CrossHeapMemberBase*> m_incoming_cross_heap_members;
HeapGroup* m_group { nullptr };
bool m_incremental_sweep_enabled { true };
WeakContainer::List m_weak_containers;
Vector<Ptr<Cell>> m_uprooted_cells;
@ -305,6 +332,30 @@ inline void Heap::did_destroy_conservative_vector(Badge<ConservativeVectorBase>,
m_conservative_vectors.remove(vector);
}
inline void Heap::did_create_conservative_range_provider(Badge<ConservativeRangeProvider>, ConservativeRangeProvider& provider)
{
VERIFY(!m_conservative_range_providers.contains(provider));
m_conservative_range_providers.append(provider);
}
inline void Heap::did_destroy_conservative_range_provider(Badge<ConservativeRangeProvider>, ConservativeRangeProvider& provider)
{
VERIFY(m_conservative_range_providers.contains(provider));
m_conservative_range_providers.remove(provider);
}
inline void Heap::did_create_cross_heap_member(Badge<CrossHeapMemberBase>, CrossHeapMemberBase& member)
{
VERIFY(!m_incoming_cross_heap_members.contains(&member));
m_incoming_cross_heap_members.set(&member);
}
inline void Heap::did_destroy_cross_heap_member(Badge<CrossHeapMemberBase>, CrossHeapMemberBase& member)
{
VERIFY(m_incoming_cross_heap_members.contains(&member));
m_incoming_cross_heap_members.remove(&member);
}
inline void Heap::did_create_weak_container(Badge<WeakContainer>, WeakContainer& set)
{
VERIFY(!m_weak_containers.contains(set));

View file

@ -0,0 +1,70 @@
/*
* Copyright (c) 2026, Ali Mohammad Pur <ali@ladybird.org>
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#include <AK/TemporaryChange.h>
#include <LibCore/ElapsedTimer.h>
#include <LibGC/Heap.h>
#include <LibGC/HeapGroup.h>
namespace GC {
HeapGroup::~HeapGroup()
{
for (auto* heap : m_heaps)
heap->m_group = nullptr;
}
void HeapGroup::add(Heap& heap)
{
VERIFY(!heap.m_group);
heap.m_group = this;
m_heaps.append(&heap);
}
void HeapGroup::remove(Heap& heap)
{
VERIFY(heap.m_group == this);
heap.m_group = nullptr;
m_heaps.remove_first_matching([&](auto* entry) { return entry == &heap; });
}
void HeapGroup::collect_garbage(bool print_report)
{
// Defer all member heaps' collections until the last one, so that cross-heap edges are visible to the mark phase.
for (auto* heap : m_heaps) {
VERIFY(!heap->m_collecting_garbage);
if (heap->m_gc_deferrals) {
heap->m_should_gc_when_deferral_ends = true;
return;
}
}
for (auto* heap : m_heaps) {
heap->finish_pending_incremental_sweep();
heap->m_collecting_garbage = true;
}
ScopeGuard unset_collecting = [&] {
for (auto* heap : m_heaps)
heap->m_collecting_garbage = false;
};
HashMap<Cell*, HeapRoot> roots;
for (auto* heap : m_heaps)
heap->gather_roots(roots, nullptr, Heap::IncludeIncomingCrossHeapMembers::No);
Heap::mark_live_cells_across(m_heaps, roots);
for (auto* heap : m_heaps)
heap->run_post_mark_phases(print_report);
for (auto* heap : m_heaps) {
Core::ElapsedTimer measurement_timer { Core::TimerType::Precise };
measurement_timer.start();
heap->sweep_dead_cells(print_report, measurement_timer);
}
}
}

View file

@ -0,0 +1,32 @@
/*
* Copyright (c) 2026, Ali Mohammad Pur <ali@ladybird.org>
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#pragma once
#include <AK/Vector.h>
#include <LibGC/Forward.h>
namespace GC {
// A group of heaps whose objects may reference each other through CrossHeapMember edges.
class GC_API HeapGroup {
AK_MAKE_NONCOPYABLE(HeapGroup);
AK_MAKE_NONMOVABLE(HeapGroup);
public:
HeapGroup() = default;
~HeapGroup();
void add(Heap&);
void remove(Heap&);
void collect_garbage(bool print_report = false);
private:
Vector<Heap*> m_heaps;
};
}

View file

@ -17,6 +17,7 @@ struct GC_API HeapRoot {
ConservativeHashMap,
ConservativeHashTable,
ConservativeVector,
CrossHeapMember,
HeapFunctionCapturedPointer,
MustSurviveGC,
RegisterPointer,

View file

@ -1,5 +1,6 @@
set(TEST_SOURCES
TestGCContainers.cpp
TestGCHeapGroup.cpp
TestGCIdleCollection.cpp
TestGCVisitor.cpp
)

View file

@ -0,0 +1,188 @@
/*
* Copyright (c) 2026, Ali Mohammad Pur <ali@ladybird.org>
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#include <LibGC/Cell.h>
#include <LibGC/CellAllocator.h>
#include <LibGC/CrossHeapMember.h>
#include <LibGC/Heap.h>
#include <LibGC/HeapGroup.h>
#include <LibGC/Ptr.h>
#include <LibGC/Root.h>
#include <LibTest/TestCase.h>
namespace {
size_t s_live_linked_cells = 0;
class LinkedCell final : public GC::Cell {
GC_CELL(LinkedCell, GC::Cell);
GC_DECLARE_ALLOCATOR(LinkedCell);
public:
virtual ~LinkedCell() override { --s_live_linked_cells; }
GC::CrossHeapMember<LinkedCell>& foreign() { return m_foreign; }
GC::Ptr<LinkedCell>& local() { return m_local; }
private:
LinkedCell() { ++s_live_linked_cells; }
virtual void visit_edges(Visitor& visitor) override
{
Base::visit_edges(visitor);
visitor.visit(m_local);
m_foreign.visit(visitor);
}
GC::CrossHeapMember<LinkedCell> m_foreign;
GC::Ptr<LinkedCell> m_local;
};
GC_DEFINE_ALLOCATOR(LinkedCell);
NEVER_INLINE void scrub_stack()
{
u8 volatile filler[8 * KiB];
for (size_t i = 0; i < sizeof(filler); ++i)
filler[i] = 0;
}
NEVER_INLINE GC::Root<LinkedCell> allocate_holder_and_foreign_target(GC::Heap& holder_heap, GC::Heap& target_heap)
{
auto holder = GC::make_root(holder_heap.allocate<LinkedCell>());
auto target = target_heap.allocate<LinkedCell>();
holder->foreign() = target.ptr();
return holder;
}
NEVER_INLINE void allocate_cross_heap_cycle(GC::Heap& heap_a, GC::Heap& heap_b)
{
auto cell_on_a = heap_a.allocate<LinkedCell>();
auto cell_on_b = heap_b.allocate<LinkedCell>();
cell_on_a->foreign() = cell_on_b.ptr();
cell_on_b->foreign() = cell_on_a.ptr();
}
NEVER_INLINE GC::Root<LinkedCell> allocate_cross_heap_chain(GC::Heap& heap_a, GC::Heap& heap_b)
{
// root -> A -> B -> A(second)
auto holder = GC::make_root(heap_a.allocate<LinkedCell>());
auto middle = heap_b.allocate<LinkedCell>();
auto tail = heap_a.allocate<LinkedCell>();
middle->foreign() = tail.ptr();
holder->foreign() = middle.ptr();
return holder;
}
NEVER_INLINE void allocate_garbage(GC::Heap& heap)
{
(void)heap.allocate<LinkedCell>();
}
}
TEST_CASE(sanity_single_heap_frees_garbage)
{
GC::Heap heap([](auto&) { }, GC::Heap::BecomeProcessDefault::No);
heap.set_incremental_sweep_enabled(false);
allocate_garbage(heap);
EXPECT_EQ(s_live_linked_cells, 1u);
scrub_stack();
heap.collect_garbage();
EXPECT_EQ(s_live_linked_cells, 0u);
}
TEST_CASE(incoming_cross_heap_member_roots_local_collection)
{
GC::Heap heap_a([](auto&) { }, GC::Heap::BecomeProcessDefault::No);
GC::Heap heap_b([](auto&) { }, GC::Heap::BecomeProcessDefault::No);
// No event loop runs during this test; sweep synchronously so frees are observable.
heap_a.set_incremental_sweep_enabled(false);
heap_b.set_incremental_sweep_enabled(false);
GC::HeapGroup group;
group.add(heap_a);
group.add(heap_b);
auto holder = allocate_holder_and_foreign_target(heap_a, heap_b);
EXPECT_EQ(s_live_linked_cells, 2u);
// The only thing keeping the target alive is the incoming cross-heap member; B's local collection cannot see the holder on A, so the member registration must root it.
scrub_stack();
heap_b.collect_garbage();
EXPECT_EQ(s_live_linked_cells, 2u);
// Once the edge is dropped, B's next local collection should free the target.
holder->foreign() = nullptr;
scrub_stack();
heap_b.collect_garbage();
EXPECT_EQ(s_live_linked_cells, 1u);
holder = {};
scrub_stack();
heap_a.collect_garbage();
EXPECT_EQ(s_live_linked_cells, 0u);
group.remove(heap_a);
group.remove(heap_b);
}
TEST_CASE(group_collection_breaks_cross_heap_cycles)
{
GC::Heap heap_a([](auto&) { }, GC::Heap::BecomeProcessDefault::No);
GC::Heap heap_b([](auto&) { }, GC::Heap::BecomeProcessDefault::No);
heap_a.set_incremental_sweep_enabled(false);
heap_b.set_incremental_sweep_enabled(false);
GC::HeapGroup group;
group.add(heap_a);
group.add(heap_b);
allocate_cross_heap_cycle(heap_a, heap_b);
EXPECT_EQ(s_live_linked_cells, 2u);
// Local collections see the incoming members as roots, so the boundary cycle should survive collection.
scrub_stack();
heap_a.collect_garbage();
heap_b.collect_garbage();
EXPECT_EQ(s_live_linked_cells, 2u);
// The unified mark should not reach the cycle, so both cells should be freed by the group collection.
scrub_stack();
group.collect_garbage();
EXPECT_EQ(s_live_linked_cells, 0u);
group.remove(heap_a);
group.remove(heap_b);
}
TEST_CASE(group_collection_traces_live_cross_heap_chains)
{
GC::Heap heap_a([](auto&) { }, GC::Heap::BecomeProcessDefault::No);
GC::Heap heap_b([](auto&) { }, GC::Heap::BecomeProcessDefault::No);
heap_a.set_incremental_sweep_enabled(false);
heap_b.set_incremental_sweep_enabled(false);
GC::HeapGroup group;
group.add(heap_a);
group.add(heap_b);
auto holder = allocate_cross_heap_chain(heap_a, heap_b);
EXPECT_EQ(s_live_linked_cells, 3u);
// The whole chain must survive a group collection, including the second A cell that is only reachable through the B cell.
scrub_stack();
group.collect_garbage();
EXPECT_EQ(s_live_linked_cells, 3u);
holder = {};
scrub_stack();
group.collect_garbage();
EXPECT_EQ(s_live_linked_cells, 0u);
group.remove(heap_a);
group.remove(heap_b);
}