diff --git a/AK/Debug.h.in b/AK/Debug.h.in index 2adcea2ee2..90b05fb1bc 100644 --- a/AK/Debug.h.in +++ b/AK/Debug.h.in @@ -90,6 +90,10 @@ # cmakedefine01 HEAP_DEBUG #endif +#ifndef INCREMENTAL_SWEEP_DEBUG +# cmakedefine01 INCREMENTAL_SWEEP_DEBUG +#endif + #ifndef HIGHLIGHT_FOCUSED_FRAME_DEBUG # cmakedefine01 HIGHLIGHT_FOCUSED_FRAME_DEBUG #endif diff --git a/Libraries/LibGC/CellAllocator.cpp b/Libraries/LibGC/CellAllocator.cpp index 31960084c0..7d5bbcbaec 100644 --- a/Libraries/LibGC/CellAllocator.cpp +++ b/Libraries/LibGC/CellAllocator.cpp @@ -25,6 +25,16 @@ Cell* CellAllocator::allocate_cell(Heap& heap) if (!m_list_node.is_in_list()) heap.register_cell_allocator({}, *this); + if (m_usable_blocks.is_empty() && heap.is_incremental_sweep_active() && !heap.is_gc_deferred()) { + // Sweep our own pending blocks first to try to find free cells + // before allocating a new block. + while (!m_usable_blocks.is_empty() || !m_blocks_pending_sweep.is_empty()) { + if (!m_usable_blocks.is_empty()) + break; + heap.sweep_block(*m_blocks_pending_sweep.first()); + } + } + if (m_usable_blocks.is_empty()) { auto block = HeapBlock::create_with_cell_size(heap, *this, m_cell_size, m_overrides_must_survive_garbage_collection, m_overrides_finalize); auto block_ptr = reinterpret_cast(block.ptr()); diff --git a/Libraries/LibGC/CellAllocator.h b/Libraries/LibGC/CellAllocator.h index ec0013db2a..fd5fcb51ce 100644 --- a/Libraries/LibGC/CellAllocator.h +++ b/Libraries/LibGC/CellAllocator.h @@ -49,22 +49,31 @@ public: void block_did_become_empty(Badge, HeapBlock&); void block_did_become_usable(Badge, HeapBlock&); + bool has_blocks_pending_sweep() const { return !m_blocks_pending_sweep.is_empty(); } + IntrusiveListNode m_list_node; using List = IntrusiveList<&CellAllocator::m_list_node>; + IntrusiveListNode m_sweep_list_node; + using SweepList = IntrusiveList<&CellAllocator::m_sweep_list_node>; + BlockAllocator& block_allocator() { return m_block_allocator; } FlatPtr min_block_address() const { return m_min_block_address; } FlatPtr max_block_address() const { return m_max_block_address; } private: + friend class Heap; + Optional m_class_name; size_t const m_cell_size; BlockAllocator m_block_allocator; using BlockList = IntrusiveList<&HeapBlock::m_list_node>; + using SweepBlockList = IntrusiveList<&HeapBlock::m_sweep_list_node>; BlockList m_full_blocks; BlockList m_usable_blocks; + SweepBlockList m_blocks_pending_sweep; FlatPtr m_min_block_address { explode_byte(0xff) }; FlatPtr m_max_block_address { 0 }; bool m_overrides_must_survive_garbage_collection { false }; diff --git a/Libraries/LibGC/Heap.cpp b/Libraries/LibGC/Heap.cpp index cd236f907b..e5777343b2 100644 --- a/Libraries/LibGC/Heap.cpp +++ b/Libraries/LibGC/Heap.cpp @@ -20,9 +20,11 @@ #include #include #include +#include #include #include #include +#include #include #include #include @@ -46,6 +48,9 @@ static constexpr size_t GC_MIN_BYTES_THRESHOLD { 8 * 1024 * 1024 }; static constexpr size_t GC_HEAP_GROWTH_FACTOR_NUMERATOR { 7 }; static constexpr size_t GC_HEAP_GROWTH_FACTOR_DENOMINATOR { 4 }; +static constexpr int GC_INCREMENTAL_SWEEP_INTERVAL_MS = 16; +static constexpr int GC_INCREMENTAL_SWEEP_SLICE_MS = 5; + static Heap* s_the; namespace { @@ -452,6 +457,12 @@ private: AK::JsonObject Heap::dump_graph() { + // An in-progress incremental sweep would leave parts of the heap as freelist + // entries while the conservative scan in gather_roots() can still pick up + // not-yet-swept (but unreachable) cells whose internal pointers lead to + // those freelist entries. Drain the sweep so we operate on a stable heap. + finish_pending_incremental_sweep(); + HashMap roots; HashTable all_live_heap_blocks; Vector stack_frames; @@ -478,6 +489,13 @@ void Heap::collect_garbage(CollectionType collection_type, bool print_report) { VERIFY(!m_collecting_garbage); + // If an incremental sweep is still in progress, finish it first. + if (m_incremental_sweep_active && !is_gc_deferred()) { + dbgln_if(INCREMENTAL_SWEEP_DEBUG, "[sweep] New GC triggered, finishing current sweep..."); + while (m_incremental_sweep_active) + sweep_next_block(); + } + { TemporaryChange change(m_collecting_garbage, true); @@ -519,7 +537,21 @@ void Heap::collect_garbage(CollectionType collection_type, bool print_report) ScopedPhaseTimer timer { report, g_phase_timings.sweep_weak_blocks_us }; sweep_weak_blocks(); } + + // Run sweep callbacks at STW so they fire for every collection, + // not just CollectEverything. Static caches like + // StaticPropertyLookupCache prune by mark state and must see valid + // marks before incremental sweep starts freeing cells. { + ScopedPhaseTimer timer { report, g_phase_timings.sweep_callbacks_us }; + for (auto& callback : m_sweep_callbacks) + callback(); + } + + // For CollectEverything we must finish sweeping synchronously so that + // every cell is collected before the Heap destructor returns. All + // other collection types defer sweeping to incremental work below. + if (collection_type == CollectionType::CollectEverything) { ScopedPhaseTimer timer { report, g_phase_timings.sweep_dead_cells_us }; sweep_dead_cells(report, collection_measurement_timer); } @@ -536,6 +568,12 @@ void Heap::collect_garbage(CollectionType collection_type, bool print_report) } } + // Arm incremental sweep before running post-GC tasks so any cells those + // tasks allocate get tagged as allocated-during-sweep and aren't freed + // by sweep_block before the next mark phase reaches them. + if (collection_type != CollectionType::CollectEverything) + start_incremental_sweep(); + run_post_gc_tasks(); } @@ -1019,12 +1057,6 @@ void Heap::sweep_dead_cells(bool print_report, Core::ElapsedTimer const& measure weak_container.remove_dead_cells({}); } - { - ScopedPhaseTimer timer { g_recording_phase_timings, g_phase_timings.sweep_callbacks_us }; - for (auto& callback : m_sweep_callbacks) - callback(); - } - { ScopedPhaseTimer timer { g_recording_phase_timings, g_phase_timings.sweep_block_reclassify_us }; for (auto* block : empty_blocks) { @@ -1067,6 +1099,174 @@ void Heap::sweep_dead_cells(bool print_report, Core::ElapsedTimer const& measure BlockAllocator::wake_decommit_worker_async(); } +void Heap::sweep_block(HeapBlock& block) +{ + // Remove from the allocator's pending sweep list. + block.m_sweep_list_node.remove(); + + bool block_has_live_cells = false; + bool block_was_full = block.is_full(); + size_t collected_cells = 0; + size_t live_cells = 0; + + block.for_each_cell_in_state([&](Cell* cell) { + if (!cell->is_marked()) { + dbgln_if(HEAP_DEBUG, " ~ {}", cell); + block.deallocate(cell); + ++collected_cells; + } else { + cell->set_marked(false); + block_has_live_cells = true; + m_sweep_live_cell_bytes += block.cell_size(); + auto cell_external_memory_size = cell->external_memory_size(); + m_sweep_live_external_bytes = cell_external_memory_size > NumericLimits::max() - m_sweep_live_external_bytes + ? NumericLimits::max() + : m_sweep_live_external_bytes + cell_external_memory_size; + ++live_cells; + } + }); + + if (!block_has_live_cells) { + dbgln_if(HEAP_DEBUG, " - HeapBlock empty @ {}: cell_size={}", &block, block.cell_size()); + dbgln_if(INCREMENTAL_SWEEP_DEBUG, "[sweep] Block @ {} freed ({} cells collected)", + &block, collected_cells); + block.cell_allocator().block_did_become_empty({}, block); + } else if (block_was_full && !block.is_full()) { + dbgln_if(HEAP_DEBUG, " - HeapBlock usable again @ {}: cell_size={}", &block, block.cell_size()); + dbgln_if(INCREMENTAL_SWEEP_DEBUG, "[sweep] Block @ {} now usable (live: {}, collected: {})", + &block, live_cells, collected_cells); + block.cell_allocator().block_did_become_usable({}, block); + } else if constexpr (INCREMENTAL_SWEEP_DEBUG) { + dbgln("[sweep] Block @ {} swept (live: {}, collected: {})", + &block, live_cells, collected_cells); + } +} + +bool Heap::sweep_next_block() +{ + if (!m_incremental_sweep_active) + return true; + + if (is_gc_deferred()) + return true; + + // Find the next allocator that has blocks pending sweep. + while (auto* allocator = m_allocators_to_sweep.first()) { + if (auto* block = allocator->m_blocks_pending_sweep.first()) { + sweep_block(*block); + if (!allocator->has_blocks_pending_sweep()) + allocator->m_sweep_list_node.remove(); + return false; + } + // Allocator was drained by allocation-directed sweeping. + allocator->m_sweep_list_node.remove(); + } + + // No more blocks to sweep. + finish_incremental_sweep(); + return true; +} + +void Heap::start_incremental_sweep() +{ + dbgln_if(INCREMENTAL_SWEEP_DEBUG, "[sweep] === Starting incremental sweep ==="); + + m_incremental_sweep_active = true; + m_sweep_live_cell_bytes = 0; + m_sweep_live_external_bytes = 0; + + for (auto& weak_container : m_weak_containers) + weak_container.remove_dead_cells({}); + + // Populate each allocator's pending sweep list with its current blocks. + // Blocks allocated during incremental sweep won't be on these lists + // and don't need sweeping. + size_t total_blocks = 0; + for (auto& allocator : m_all_cell_allocators) { + allocator.for_each_block([&](HeapBlock& block) { + allocator.m_blocks_pending_sweep.append(block); + ++total_blocks; + return IterationDecision::Continue; + }); + if (allocator.has_blocks_pending_sweep()) + m_allocators_to_sweep.append(allocator); + } + + dbgln_if(INCREMENTAL_SWEEP_DEBUG, "[sweep] {} blocks to sweep", total_blocks); + + start_incremental_sweep_timer(); +} + +void Heap::finish_incremental_sweep() +{ + update_gc_bytes_threshold(m_sweep_live_cell_bytes, m_sweep_live_external_bytes); + + dbgln_if(INCREMENTAL_SWEEP_DEBUG, "[sweep] === Sweep complete ==="); + dbgln_if(INCREMENTAL_SWEEP_DEBUG, "[sweep] Live cell bytes: {} ({} KiB)", m_sweep_live_cell_bytes, m_sweep_live_cell_bytes / KiB); + dbgln_if(INCREMENTAL_SWEEP_DEBUG, "[sweep] Live external bytes: {} ({} KiB)", m_sweep_live_external_bytes, m_sweep_live_external_bytes / KiB); + dbgln_if(INCREMENTAL_SWEEP_DEBUG, "[sweep] Next GC threshold: {} ({} KiB)", m_gc_bytes_threshold, m_gc_bytes_threshold / KiB); + + // Clear marks on cells allocated during sweep. Sweep already cleared + // marks on cells it visited, so only these remain marked. + for (auto cell : m_cells_allocated_during_sweep) + cell->set_marked(false); + m_cells_allocated_during_sweep.clear(); + + m_incremental_sweep_active = false; + + stop_incremental_sweep_timer(); +} + +void Heap::finish_pending_incremental_sweep() +{ + if (!m_incremental_sweep_active || is_gc_deferred()) + return; + + dbgln_if(INCREMENTAL_SWEEP_DEBUG, "[sweep] Finishing pending sweep..."); + while (m_incremental_sweep_active) + sweep_next_block(); +} + +void Heap::start_incremental_sweep_timer() +{ + if (!m_incremental_sweep_timer) { + m_incremental_sweep_timer = Core::Timer::create_repeating(GC_INCREMENTAL_SWEEP_INTERVAL_MS, [this] { + sweep_on_timer(); + }); + } + m_incremental_sweep_timer->start(); +} + +void Heap::stop_incremental_sweep_timer() +{ + if (m_incremental_sweep_timer) + m_incremental_sweep_timer->stop(); +} + +void Heap::sweep_on_timer() +{ + if (!m_incremental_sweep_active) + return; + + if (is_gc_deferred()) + return; + + size_t blocks_swept = 0; + auto start_time = MonotonicTime::now(); + auto deadline = start_time + AK::Duration::from_milliseconds(GC_INCREMENTAL_SWEEP_SLICE_MS); + while (MonotonicTime::now() < deadline) { + if (sweep_next_block()) + break; + ++blocks_swept; + } + + if (blocks_swept > 0) { + auto elapsed = MonotonicTime::now() - start_time; + dbgln_if(INCREMENTAL_SWEEP_DEBUG, "[sweep] Timer slice: {} blocks in {}ms", + blocks_swept, elapsed.to_milliseconds()); + } +} + void Heap::defer_gc() { ++m_gc_deferrals; diff --git a/Libraries/LibGC/Heap.h b/Libraries/LibGC/Heap.h index 575126694c..c81926da5a 100644 --- a/Libraries/LibGC/Heap.h +++ b/Libraries/LibGC/Heap.h @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -49,8 +50,15 @@ public: auto* memory = allocate_cell(); defer_gc(); new (memory) T(forward(args)...); + auto* cell = static_cast(memory); + // Cells allocated during incremental sweep must be marked so they + // survive until the next GC cycle clears and re-establishes marks. + if (m_incremental_sweep_active) { + cell->set_marked(true); + m_cells_allocated_during_sweep.append(cell); + } undefer_gc(); - return *static_cast(memory); + return *cell; } enum class CollectionType { @@ -86,6 +94,9 @@ public: void uproot_cell(Cell* cell); bool is_gc_deferred() const { return m_gc_deferrals > 0; } + bool is_incremental_sweep_active() const { return m_incremental_sweep_active; } + + void sweep_block(HeapBlock&); void enqueue_post_gc_task(AK::Function); @@ -128,6 +139,14 @@ private: void sweep_weak_blocks(); void run_post_gc_tasks(); + bool sweep_next_block(); + void start_incremental_sweep(); + void finish_incremental_sweep(); + void finish_pending_incremental_sweep(); + void start_incremental_sweep_timer(); + void stop_incremental_sweep_timer(); + void sweep_on_timer(); + template void for_each_block(Callback callback) { @@ -164,6 +183,13 @@ private: WeakBlock::List m_usable_weak_blocks; WeakBlock::List m_full_weak_blocks; + + bool m_incremental_sweep_active { false }; + size_t m_sweep_live_cell_bytes { 0 }; + size_t m_sweep_live_external_bytes { 0 }; + Vector> m_cells_allocated_during_sweep; + CellAllocator::SweepList m_allocators_to_sweep; + RefPtr m_incremental_sweep_timer; }; inline void Heap::did_create_root(Badge, RootImpl& impl) diff --git a/Libraries/LibGC/HeapBlock.h b/Libraries/LibGC/HeapBlock.h index df8024e7db..f2ee85b5d6 100644 --- a/Libraries/LibGC/HeapBlock.h +++ b/Libraries/LibGC/HeapBlock.h @@ -89,6 +89,7 @@ public: } IntrusiveListNode m_list_node; + IntrusiveListNode m_sweep_list_node; CellAllocator& cell_allocator() { return m_cell_allocator; } diff --git a/Meta/CMake/all_the_debug_macros.cmake b/Meta/CMake/all_the_debug_macros.cmake index 982044a04a..55f1683e34 100644 --- a/Meta/CMake/all_the_debug_macros.cmake +++ b/Meta/CMake/all_the_debug_macros.cmake @@ -18,6 +18,7 @@ set(FLAC_ENCODER_DEBUG ON) set(FORMATTING_CONTEXT_TRACE_DEBUG ON) set(GIF_DEBUG ON) set(HEAP_DEBUG ON) +set(INCREMENTAL_SWEEP_DEBUG ON) set(HIGHLIGHT_FOCUSED_FRAME_DEBUG ON) set(HTML_SCRIPT_DEBUG ON) set(HTTP_DISK_CACHE_DEBUG ON)