LibGC: Implement incremental sweeping for reduced GC pause times
Instead of sweeping all heap blocks in one go after marking, sweep incrementally, one block at a time, interleaved with program execution. This significantly reduces worst-case GC pause times by spreading sweep work across multiple smaller time slices. Sweep is driven by two complementary mechanisms: 1. Timer-based sweeping: A 16ms repeating timer drives background sweep work, processing blocks for up to 5ms per timer fire. 2. Allocation-directed sweeping: Each allocator sweeps its own pending blocks before creating new ones, ensuring forward progress even without timer events. Each allocator maintains its own list of blocks pending sweep, and allocators with pending work are tracked in a separate list for efficient timer-driven sweeping. Key implementation details: - Newly allocated cells during sweep are marked immediately to prevent premature collection. - Mark bits are cleared incrementally as each block is swept, rather than in a separate pass over the entire heap. - Finalization and weak reference processing remain stop-the-world since they must complete atomically before any sweeping occurs.
This commit is contained in:
Andreas Kling
Andreas Kling
parent
e23ec1f841
commit
fb4095ae50
7 changed files with 258 additions and 7 deletions
|
|
@ -90,6 +90,10 @@
|
|||
# cmakedefine01 HEAP_DEBUG
|
||||
#endif
|
||||
|
||||
#ifndef INCREMENTAL_SWEEP_DEBUG
|
||||
# cmakedefine01 INCREMENTAL_SWEEP_DEBUG
|
||||
#endif
|
||||
|
||||
#ifndef HIGHLIGHT_FOCUSED_FRAME_DEBUG
|
||||
# cmakedefine01 HIGHLIGHT_FOCUSED_FRAME_DEBUG
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -25,6 +25,16 @@ Cell* CellAllocator::allocate_cell(Heap& heap)
|
|||
if (!m_list_node.is_in_list())
|
||||
heap.register_cell_allocator({}, *this);
|
||||
|
||||
if (m_usable_blocks.is_empty() && heap.is_incremental_sweep_active() && !heap.is_gc_deferred()) {
|
||||
// Sweep our own pending blocks first to try to find free cells
|
||||
// before allocating a new block.
|
||||
while (!m_usable_blocks.is_empty() || !m_blocks_pending_sweep.is_empty()) {
|
||||
if (!m_usable_blocks.is_empty())
|
||||
break;
|
||||
heap.sweep_block(*m_blocks_pending_sweep.first());
|
||||
}
|
||||
}
|
||||
|
||||
if (m_usable_blocks.is_empty()) {
|
||||
auto block = HeapBlock::create_with_cell_size(heap, *this, m_cell_size, m_overrides_must_survive_garbage_collection, m_overrides_finalize);
|
||||
auto block_ptr = reinterpret_cast<FlatPtr>(block.ptr());
|
||||
|
|
|
|||
|
|
@ -49,22 +49,31 @@ public:
|
|||
void block_did_become_empty(Badge<Heap>, HeapBlock&);
|
||||
void block_did_become_usable(Badge<Heap>, HeapBlock&);
|
||||
|
||||
bool has_blocks_pending_sweep() const { return !m_blocks_pending_sweep.is_empty(); }
|
||||
|
||||
IntrusiveListNode<CellAllocator> m_list_node;
|
||||
using List = IntrusiveList<&CellAllocator::m_list_node>;
|
||||
|
||||
IntrusiveListNode<CellAllocator> m_sweep_list_node;
|
||||
using SweepList = IntrusiveList<&CellAllocator::m_sweep_list_node>;
|
||||
|
||||
BlockAllocator& block_allocator() { return m_block_allocator; }
|
||||
FlatPtr min_block_address() const { return m_min_block_address; }
|
||||
FlatPtr max_block_address() const { return m_max_block_address; }
|
||||
|
||||
private:
|
||||
friend class Heap;
|
||||
|
||||
Optional<StringView> m_class_name;
|
||||
size_t const m_cell_size;
|
||||
|
||||
BlockAllocator m_block_allocator;
|
||||
|
||||
using BlockList = IntrusiveList<&HeapBlock::m_list_node>;
|
||||
using SweepBlockList = IntrusiveList<&HeapBlock::m_sweep_list_node>;
|
||||
BlockList m_full_blocks;
|
||||
BlockList m_usable_blocks;
|
||||
SweepBlockList m_blocks_pending_sweep;
|
||||
FlatPtr m_min_block_address { explode_byte(0xff) };
|
||||
FlatPtr m_max_block_address { 0 };
|
||||
bool m_overrides_must_survive_garbage_collection { false };
|
||||
|
|
|
|||
|
|
@ -20,9 +20,11 @@
|
|||
#include <AK/StackInfo.h>
|
||||
#include <AK/StackUnwinder.h>
|
||||
#include <AK/TemporaryChange.h>
|
||||
#include <AK/Time.h>
|
||||
#include <LibCore/ElapsedTimer.h>
|
||||
#include <LibCore/File.h>
|
||||
#include <LibCore/StandardPaths.h>
|
||||
#include <LibCore/Timer.h>
|
||||
#include <LibGC/BlockAllocator.h>
|
||||
#include <LibGC/CellAllocator.h>
|
||||
#include <LibGC/Heap.h>
|
||||
|
|
@ -46,6 +48,9 @@ static constexpr size_t GC_MIN_BYTES_THRESHOLD { 8 * 1024 * 1024 };
|
|||
static constexpr size_t GC_HEAP_GROWTH_FACTOR_NUMERATOR { 7 };
|
||||
static constexpr size_t GC_HEAP_GROWTH_FACTOR_DENOMINATOR { 4 };
|
||||
|
||||
static constexpr int GC_INCREMENTAL_SWEEP_INTERVAL_MS = 16;
|
||||
static constexpr int GC_INCREMENTAL_SWEEP_SLICE_MS = 5;
|
||||
|
||||
static Heap* s_the;
|
||||
|
||||
namespace {
|
||||
|
|
@ -452,6 +457,12 @@ private:
|
|||
|
||||
AK::JsonObject Heap::dump_graph()
|
||||
{
|
||||
// An in-progress incremental sweep would leave parts of the heap as freelist
|
||||
// entries while the conservative scan in gather_roots() can still pick up
|
||||
// not-yet-swept (but unreachable) cells whose internal pointers lead to
|
||||
// those freelist entries. Drain the sweep so we operate on a stable heap.
|
||||
finish_pending_incremental_sweep();
|
||||
|
||||
HashMap<Cell*, HeapRoot> roots;
|
||||
HashTable<HeapBlock*> all_live_heap_blocks;
|
||||
Vector<StackFrameInfo> stack_frames;
|
||||
|
|
@ -478,6 +489,13 @@ void Heap::collect_garbage(CollectionType collection_type, bool print_report)
|
|||
{
|
||||
VERIFY(!m_collecting_garbage);
|
||||
|
||||
// If an incremental sweep is still in progress, finish it first.
|
||||
if (m_incremental_sweep_active && !is_gc_deferred()) {
|
||||
dbgln_if(INCREMENTAL_SWEEP_DEBUG, "[sweep] New GC triggered, finishing current sweep...");
|
||||
while (m_incremental_sweep_active)
|
||||
sweep_next_block();
|
||||
}
|
||||
|
||||
{
|
||||
TemporaryChange change(m_collecting_garbage, true);
|
||||
|
||||
|
|
@ -519,7 +537,21 @@ void Heap::collect_garbage(CollectionType collection_type, bool print_report)
|
|||
ScopedPhaseTimer timer { report, g_phase_timings.sweep_weak_blocks_us };
|
||||
sweep_weak_blocks();
|
||||
}
|
||||
|
||||
// Run sweep callbacks at STW so they fire for every collection,
|
||||
// not just CollectEverything. Static caches like
|
||||
// StaticPropertyLookupCache prune by mark state and must see valid
|
||||
// marks before incremental sweep starts freeing cells.
|
||||
{
|
||||
ScopedPhaseTimer timer { report, g_phase_timings.sweep_callbacks_us };
|
||||
for (auto& callback : m_sweep_callbacks)
|
||||
callback();
|
||||
}
|
||||
|
||||
// For CollectEverything we must finish sweeping synchronously so that
|
||||
// every cell is collected before the Heap destructor returns. All
|
||||
// other collection types defer sweeping to incremental work below.
|
||||
if (collection_type == CollectionType::CollectEverything) {
|
||||
ScopedPhaseTimer timer { report, g_phase_timings.sweep_dead_cells_us };
|
||||
sweep_dead_cells(report, collection_measurement_timer);
|
||||
}
|
||||
|
|
@ -536,6 +568,12 @@ void Heap::collect_garbage(CollectionType collection_type, bool print_report)
|
|||
}
|
||||
}
|
||||
|
||||
// Arm incremental sweep before running post-GC tasks so any cells those
|
||||
// tasks allocate get tagged as allocated-during-sweep and aren't freed
|
||||
// by sweep_block before the next mark phase reaches them.
|
||||
if (collection_type != CollectionType::CollectEverything)
|
||||
start_incremental_sweep();
|
||||
|
||||
run_post_gc_tasks();
|
||||
}
|
||||
|
||||
|
|
@ -1019,12 +1057,6 @@ void Heap::sweep_dead_cells(bool print_report, Core::ElapsedTimer const& measure
|
|||
weak_container.remove_dead_cells({});
|
||||
}
|
||||
|
||||
{
|
||||
ScopedPhaseTimer timer { g_recording_phase_timings, g_phase_timings.sweep_callbacks_us };
|
||||
for (auto& callback : m_sweep_callbacks)
|
||||
callback();
|
||||
}
|
||||
|
||||
{
|
||||
ScopedPhaseTimer timer { g_recording_phase_timings, g_phase_timings.sweep_block_reclassify_us };
|
||||
for (auto* block : empty_blocks) {
|
||||
|
|
@ -1067,6 +1099,174 @@ void Heap::sweep_dead_cells(bool print_report, Core::ElapsedTimer const& measure
|
|||
BlockAllocator::wake_decommit_worker_async();
|
||||
}
|
||||
|
||||
void Heap::sweep_block(HeapBlock& block)
|
||||
{
|
||||
// Remove from the allocator's pending sweep list.
|
||||
block.m_sweep_list_node.remove();
|
||||
|
||||
bool block_has_live_cells = false;
|
||||
bool block_was_full = block.is_full();
|
||||
size_t collected_cells = 0;
|
||||
size_t live_cells = 0;
|
||||
|
||||
block.for_each_cell_in_state<Cell::State::Live>([&](Cell* cell) {
|
||||
if (!cell->is_marked()) {
|
||||
dbgln_if(HEAP_DEBUG, " ~ {}", cell);
|
||||
block.deallocate(cell);
|
||||
++collected_cells;
|
||||
} else {
|
||||
cell->set_marked(false);
|
||||
block_has_live_cells = true;
|
||||
m_sweep_live_cell_bytes += block.cell_size();
|
||||
auto cell_external_memory_size = cell->external_memory_size();
|
||||
m_sweep_live_external_bytes = cell_external_memory_size > NumericLimits<size_t>::max() - m_sweep_live_external_bytes
|
||||
? NumericLimits<size_t>::max()
|
||||
: m_sweep_live_external_bytes + cell_external_memory_size;
|
||||
++live_cells;
|
||||
}
|
||||
});
|
||||
|
||||
if (!block_has_live_cells) {
|
||||
dbgln_if(HEAP_DEBUG, " - HeapBlock empty @ {}: cell_size={}", &block, block.cell_size());
|
||||
dbgln_if(INCREMENTAL_SWEEP_DEBUG, "[sweep] Block @ {} freed ({} cells collected)",
|
||||
&block, collected_cells);
|
||||
block.cell_allocator().block_did_become_empty({}, block);
|
||||
} else if (block_was_full && !block.is_full()) {
|
||||
dbgln_if(HEAP_DEBUG, " - HeapBlock usable again @ {}: cell_size={}", &block, block.cell_size());
|
||||
dbgln_if(INCREMENTAL_SWEEP_DEBUG, "[sweep] Block @ {} now usable (live: {}, collected: {})",
|
||||
&block, live_cells, collected_cells);
|
||||
block.cell_allocator().block_did_become_usable({}, block);
|
||||
} else if constexpr (INCREMENTAL_SWEEP_DEBUG) {
|
||||
dbgln("[sweep] Block @ {} swept (live: {}, collected: {})",
|
||||
&block, live_cells, collected_cells);
|
||||
}
|
||||
}
|
||||
|
||||
bool Heap::sweep_next_block()
|
||||
{
|
||||
if (!m_incremental_sweep_active)
|
||||
return true;
|
||||
|
||||
if (is_gc_deferred())
|
||||
return true;
|
||||
|
||||
// Find the next allocator that has blocks pending sweep.
|
||||
while (auto* allocator = m_allocators_to_sweep.first()) {
|
||||
if (auto* block = allocator->m_blocks_pending_sweep.first()) {
|
||||
sweep_block(*block);
|
||||
if (!allocator->has_blocks_pending_sweep())
|
||||
allocator->m_sweep_list_node.remove();
|
||||
return false;
|
||||
}
|
||||
// Allocator was drained by allocation-directed sweeping.
|
||||
allocator->m_sweep_list_node.remove();
|
||||
}
|
||||
|
||||
// No more blocks to sweep.
|
||||
finish_incremental_sweep();
|
||||
return true;
|
||||
}
|
||||
|
||||
void Heap::start_incremental_sweep()
|
||||
{
|
||||
dbgln_if(INCREMENTAL_SWEEP_DEBUG, "[sweep] === Starting incremental sweep ===");
|
||||
|
||||
m_incremental_sweep_active = true;
|
||||
m_sweep_live_cell_bytes = 0;
|
||||
m_sweep_live_external_bytes = 0;
|
||||
|
||||
for (auto& weak_container : m_weak_containers)
|
||||
weak_container.remove_dead_cells({});
|
||||
|
||||
// Populate each allocator's pending sweep list with its current blocks.
|
||||
// Blocks allocated during incremental sweep won't be on these lists
|
||||
// and don't need sweeping.
|
||||
size_t total_blocks = 0;
|
||||
for (auto& allocator : m_all_cell_allocators) {
|
||||
allocator.for_each_block([&](HeapBlock& block) {
|
||||
allocator.m_blocks_pending_sweep.append(block);
|
||||
++total_blocks;
|
||||
return IterationDecision::Continue;
|
||||
});
|
||||
if (allocator.has_blocks_pending_sweep())
|
||||
m_allocators_to_sweep.append(allocator);
|
||||
}
|
||||
|
||||
dbgln_if(INCREMENTAL_SWEEP_DEBUG, "[sweep] {} blocks to sweep", total_blocks);
|
||||
|
||||
start_incremental_sweep_timer();
|
||||
}
|
||||
|
||||
void Heap::finish_incremental_sweep()
|
||||
{
|
||||
update_gc_bytes_threshold(m_sweep_live_cell_bytes, m_sweep_live_external_bytes);
|
||||
|
||||
dbgln_if(INCREMENTAL_SWEEP_DEBUG, "[sweep] === Sweep complete ===");
|
||||
dbgln_if(INCREMENTAL_SWEEP_DEBUG, "[sweep] Live cell bytes: {} ({} KiB)", m_sweep_live_cell_bytes, m_sweep_live_cell_bytes / KiB);
|
||||
dbgln_if(INCREMENTAL_SWEEP_DEBUG, "[sweep] Live external bytes: {} ({} KiB)", m_sweep_live_external_bytes, m_sweep_live_external_bytes / KiB);
|
||||
dbgln_if(INCREMENTAL_SWEEP_DEBUG, "[sweep] Next GC threshold: {} ({} KiB)", m_gc_bytes_threshold, m_gc_bytes_threshold / KiB);
|
||||
|
||||
// Clear marks on cells allocated during sweep. Sweep already cleared
|
||||
// marks on cells it visited, so only these remain marked.
|
||||
for (auto cell : m_cells_allocated_during_sweep)
|
||||
cell->set_marked(false);
|
||||
m_cells_allocated_during_sweep.clear();
|
||||
|
||||
m_incremental_sweep_active = false;
|
||||
|
||||
stop_incremental_sweep_timer();
|
||||
}
|
||||
|
||||
void Heap::finish_pending_incremental_sweep()
|
||||
{
|
||||
if (!m_incremental_sweep_active || is_gc_deferred())
|
||||
return;
|
||||
|
||||
dbgln_if(INCREMENTAL_SWEEP_DEBUG, "[sweep] Finishing pending sweep...");
|
||||
while (m_incremental_sweep_active)
|
||||
sweep_next_block();
|
||||
}
|
||||
|
||||
void Heap::start_incremental_sweep_timer()
|
||||
{
|
||||
if (!m_incremental_sweep_timer) {
|
||||
m_incremental_sweep_timer = Core::Timer::create_repeating(GC_INCREMENTAL_SWEEP_INTERVAL_MS, [this] {
|
||||
sweep_on_timer();
|
||||
});
|
||||
}
|
||||
m_incremental_sweep_timer->start();
|
||||
}
|
||||
|
||||
void Heap::stop_incremental_sweep_timer()
|
||||
{
|
||||
if (m_incremental_sweep_timer)
|
||||
m_incremental_sweep_timer->stop();
|
||||
}
|
||||
|
||||
void Heap::sweep_on_timer()
|
||||
{
|
||||
if (!m_incremental_sweep_active)
|
||||
return;
|
||||
|
||||
if (is_gc_deferred())
|
||||
return;
|
||||
|
||||
size_t blocks_swept = 0;
|
||||
auto start_time = MonotonicTime::now();
|
||||
auto deadline = start_time + AK::Duration::from_milliseconds(GC_INCREMENTAL_SWEEP_SLICE_MS);
|
||||
while (MonotonicTime::now() < deadline) {
|
||||
if (sweep_next_block())
|
||||
break;
|
||||
++blocks_swept;
|
||||
}
|
||||
|
||||
if (blocks_swept > 0) {
|
||||
auto elapsed = MonotonicTime::now() - start_time;
|
||||
dbgln_if(INCREMENTAL_SWEEP_DEBUG, "[sweep] Timer slice: {} blocks in {}ms",
|
||||
blocks_swept, elapsed.to_milliseconds());
|
||||
}
|
||||
}
|
||||
|
||||
void Heap::defer_gc()
|
||||
{
|
||||
++m_gc_deferrals;
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@
|
|||
#include <AK/Function.h>
|
||||
#include <AK/Noncopyable.h>
|
||||
#include <AK/NonnullOwnPtr.h>
|
||||
#include <AK/RefPtr.h>
|
||||
#include <AK/StackInfo.h>
|
||||
#include <AK/String.h>
|
||||
#include <AK/Types.h>
|
||||
|
|
@ -49,8 +50,15 @@ public:
|
|||
auto* memory = allocate_cell<T>();
|
||||
defer_gc();
|
||||
new (memory) T(forward<Args>(args)...);
|
||||
auto* cell = static_cast<T*>(memory);
|
||||
// Cells allocated during incremental sweep must be marked so they
|
||||
// survive until the next GC cycle clears and re-establishes marks.
|
||||
if (m_incremental_sweep_active) {
|
||||
cell->set_marked(true);
|
||||
m_cells_allocated_during_sweep.append(cell);
|
||||
}
|
||||
undefer_gc();
|
||||
return *static_cast<T*>(memory);
|
||||
return *cell;
|
||||
}
|
||||
|
||||
enum class CollectionType {
|
||||
|
|
@ -86,6 +94,9 @@ public:
|
|||
void uproot_cell(Cell* cell);
|
||||
|
||||
bool is_gc_deferred() const { return m_gc_deferrals > 0; }
|
||||
bool is_incremental_sweep_active() const { return m_incremental_sweep_active; }
|
||||
|
||||
void sweep_block(HeapBlock&);
|
||||
|
||||
void enqueue_post_gc_task(AK::Function<void()>);
|
||||
|
||||
|
|
@ -128,6 +139,14 @@ private:
|
|||
void sweep_weak_blocks();
|
||||
void run_post_gc_tasks();
|
||||
|
||||
bool sweep_next_block();
|
||||
void start_incremental_sweep();
|
||||
void finish_incremental_sweep();
|
||||
void finish_pending_incremental_sweep();
|
||||
void start_incremental_sweep_timer();
|
||||
void stop_incremental_sweep_timer();
|
||||
void sweep_on_timer();
|
||||
|
||||
template<typename Callback>
|
||||
void for_each_block(Callback callback)
|
||||
{
|
||||
|
|
@ -164,6 +183,13 @@ private:
|
|||
|
||||
WeakBlock::List m_usable_weak_blocks;
|
||||
WeakBlock::List m_full_weak_blocks;
|
||||
|
||||
bool m_incremental_sweep_active { false };
|
||||
size_t m_sweep_live_cell_bytes { 0 };
|
||||
size_t m_sweep_live_external_bytes { 0 };
|
||||
Vector<GC::Ptr<Cell>> m_cells_allocated_during_sweep;
|
||||
CellAllocator::SweepList m_allocators_to_sweep;
|
||||
RefPtr<Core::Timer> m_incremental_sweep_timer;
|
||||
};
|
||||
|
||||
inline void Heap::did_create_root(Badge<RootImpl>, RootImpl& impl)
|
||||
|
|
|
|||
|
|
@ -89,6 +89,7 @@ public:
|
|||
}
|
||||
|
||||
IntrusiveListNode<HeapBlock> m_list_node;
|
||||
IntrusiveListNode<HeapBlock> m_sweep_list_node;
|
||||
|
||||
CellAllocator& cell_allocator() { return m_cell_allocator; }
|
||||
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ set(FLAC_ENCODER_DEBUG ON)
|
|||
set(FORMATTING_CONTEXT_TRACE_DEBUG ON)
|
||||
set(GIF_DEBUG ON)
|
||||
set(HEAP_DEBUG ON)
|
||||
set(INCREMENTAL_SWEEP_DEBUG ON)
|
||||
set(HIGHLIGHT_FOCUSED_FRAME_DEBUG ON)
|
||||
set(HTML_SCRIPT_DEBUG ON)
|
||||
set(HTTP_DISK_CACHE_DEBUG ON)
|
||||
|
|
|
|||
Loading…
Reference in a new issue