From adfc9d263f92105620c6ad862b2819eb5aa163bc Mon Sep 17 00:00:00 2001 From: Andreas Kling Date: Thu, 7 May 2026 10:48:06 +0200 Subject: [PATCH] LibGC: Defer per-block madvise to a global background worker deallocate_block() used to call MADV_FREE_REUSABLE / MADV_FREE / MADV_DONTNEED inline on every freed block. With sweep typically freeing many blocks per GC, the cumulative syscall cost shows up as real GC pause time. Move the work onto a single global "decommit worker" thread: - deallocate_block now just poisons the slot and pushes it onto a per-allocator m_freshly_freed queue. No syscalls. - allocate_block prefers m_freshly_freed over m_blocks, so a slot that's recycled before the worker sees it skips the REUSABLE/REUSE pair entirely. This is the main payoff. - Heap::sweep_dead_cells kicks the worker at the end of sweep. The worker sleeps 50 ms after each kick to give the JS thread breathing room, then drains each registered allocator's m_freshly_freed, madvises slots in batches of 64 with sched_yield between batches, and splices them onto m_blocks. - Per-allocator refcount + condvar lets ~BlockAllocator wait until the worker has dropped its reference before our storage goes away. (Chunks themselves remain leaked: type-isolated VM is permanent, so we never tear them down.) --- Libraries/LibGC/BlockAllocator.cpp | 295 ++++++++++++++++++++++++----- Libraries/LibGC/BlockAllocator.h | 37 +++- Libraries/LibGC/CMakeLists.txt | 1 + Libraries/LibGC/Heap.cpp | 7 +- Libraries/LibTest/CMakeLists.txt | 1 + 5 files changed, 293 insertions(+), 48 deletions(-) diff --git a/Libraries/LibGC/BlockAllocator.cpp b/Libraries/LibGC/BlockAllocator.cpp index 72d5e5ffa6..e8a703c7ac 100644 --- a/Libraries/LibGC/BlockAllocator.cpp +++ b/Libraries/LibGC/BlockAllocator.cpp @@ -6,11 +6,13 @@ */ #include +#include #include #include #include #include #include +#include #include #if defined(AK_OS_MACOS) @@ -26,30 +28,241 @@ #if defined(AK_OS_WINDOWS) # include # include +#else +# include +# include #endif namespace GC { // Each BlockAllocator carves its 16 KiB HeapBlock slots out of 2 MiB -// chunks, so the kernel sees one mmap per 128 blocks instead of one per -// block. Chunks are owned exclusively by a single BlockAllocator and are -// never released back to the OS or shared across allocators -- that's how -// we keep the heap's VM permanently type-isolated, where a virtual address -// used for a cell of type T is never reused for any other type. +// chunks. Chunks are owned exclusively by a single BlockAllocator and are +// never released back to the OS or shared across allocators -- the heap's +// VM is permanently type-isolated. // -// We do not hint MADV_HUGEPAGE: per-block madvise() in deallocate_block -// would split any THP backing the chunk anyway. Per-block memory return -// matches what V8, SpiderMonkey, and WebKit's libpas all do. +// Per-block madvise() is deferred to a single global background "decommit +// worker" so it never costs us GC pause time, and slots that are recycled +// before the worker sees them skip the madvise pair entirely. static constexpr size_t CHUNK_SIZE = 2 * MiB; static constexpr size_t BLOCKS_PER_CHUNK = CHUNK_SIZE / HeapBlock::BLOCK_SIZE; static_assert(CHUNK_SIZE % HeapBlock::BLOCK_SIZE == 0); static_assert(BLOCKS_PER_CHUNK == 128); -BlockAllocator::~BlockAllocator() = default; +static void madvise_block_for_decommit(void* block) +{ +#if defined(AK_OS_WINDOWS) + DWORD ret = DiscardVirtualMemory(block, HeapBlock::BLOCK_SIZE); + if (ret != ERROR_SUCCESS) { + warnln("{}", Error::from_windows_error(ret)); + VERIFY_NOT_REACHED(); + } +#elif defined(MADV_FREE_REUSE) && defined(MADV_FREE_REUSABLE) + if (madvise(block, HeapBlock::BLOCK_SIZE, MADV_FREE_REUSABLE) < 0) { + perror("madvise(MADV_FREE_REUSABLE)"); + VERIFY_NOT_REACHED(); + } +#elif defined(MADV_FREE) + if (madvise(block, HeapBlock::BLOCK_SIZE, MADV_FREE) < 0) { + perror("madvise(MADV_FREE)"); + VERIFY_NOT_REACHED(); + } +#elif defined(MADV_DONTNEED) + if (madvise(block, HeapBlock::BLOCK_SIZE, MADV_DONTNEED) < 0) { + perror("madvise(MADV_DONTNEED)"); + VERIFY_NOT_REACHED(); + } +#endif +} + +static void sleep_before_decommit() +{ +#if defined(AK_OS_WINDOWS) + Sleep(50); +#else + usleep(50 * 1000); +#endif +} + +static void yield_during_decommit() +{ +#if defined(AK_OS_WINDOWS) + Sleep(0); +#else + sched_yield(); +#endif +} + +class DecommitWorker { +public: + static DecommitWorker& the(); + + void register_pending(BlockAllocator&); + void deregister(BlockAllocator&); + void kick(); + + DecommitWorker(); + +private: + void run(); + void process_one(BlockAllocator&); + + Threading::Mutex m_mutex; + Threading::ConditionVariable m_cv { m_mutex }; + RefPtr m_thread; + Vector m_pending; + bool m_kicked { false }; +}; + +DecommitWorker& DecommitWorker::the() +{ + static AK::NeverDestroyed instance; + return *instance; +} + +DecommitWorker::DecommitWorker() +{ + m_thread = Threading::Thread::construct("DecommitWorker"sv, [this] { + run(); + return static_cast(0); + }); + m_thread->start(); + m_thread->detach(); +} + +void DecommitWorker::register_pending(BlockAllocator& a) +{ + Threading::MutexLocker locker(m_mutex); + m_pending.append(&a); +} + +void DecommitWorker::deregister(BlockAllocator& a) +{ + Threading::MutexLocker locker(m_mutex); + m_pending.remove_first_matching([&](auto* p) { return p == &a; }); +} + +void DecommitWorker::kick() +{ + { + Threading::MutexLocker locker(m_mutex); + m_kicked = true; + } + m_cv.signal(); +} + +void DecommitWorker::run() +{ + while (true) { + Vector snapshot; + { + Threading::MutexLocker locker(m_mutex); + while (!m_kicked) + m_cv.wait(); + m_kicked = false; + snapshot = move(m_pending); + // Pin every allocator we're about to process so destructors + // block until we drop our reference. + for (auto* a : snapshot) + a->m_worker_refcount.fetch_add(1); + } + + if (snapshot.is_empty()) + continue; + + // Stagger: give the JS thread some breathing room after the kick + // (typically right after sweep ends) before we consume CPU and + // syscall bandwidth. + sleep_before_decommit(); + + for (auto* a : snapshot) { + process_one(*a); + int prev_refcount = a->m_worker_refcount.fetch_sub(1); + if (prev_refcount == 1) { + Threading::MutexLocker locker(a->m_mutex); + a->m_worker_cv.broadcast(); + } + } + } +} + +void DecommitWorker::process_one(BlockAllocator& a) +{ + Vector to_process; + { + Threading::MutexLocker locker(a.m_mutex); + a.m_in_decommit_registry = false; + to_process = move(a.m_freshly_freed); + } + + // Madvise each slot outside the per-allocator lock so the JS thread can + // continue to allocate/free; yield every 64 slots to avoid hogging the + // kernel's mm subsystem. + constexpr size_t BATCH = 64; + for (size_t i = 0; i < to_process.size(); ++i) { + madvise_block_for_decommit(to_process[i]); + if ((i + 1) % BATCH == 0) + yield_during_decommit(); + } + + { + Threading::MutexLocker locker(a.m_mutex); + for (auto* slot : to_process) + a.m_blocks.append(slot); + } +} + +void BlockAllocator::wake_decommit_worker_async() +{ + DecommitWorker::the().kick(); +} + +BlockAllocator::BlockAllocator() + : m_worker_cv(m_mutex) +{ +} + +BlockAllocator::~BlockAllocator() +{ + // Chunks are permanent -- we never tear them down. The destructor only + // exists to make sure the global decommit worker has finished any + // in-flight processing of *this before our storage goes away. + DecommitWorker::the().deregister(*this); + + Threading::MutexLocker locker(m_mutex); + while (m_worker_refcount.load() != 0) + m_worker_cv.wait(); +} + +size_t BlockAllocator::block_count() +{ + Threading::MutexLocker locker(m_mutex); + return m_blocks.size(); +} void* BlockAllocator::allocate_block([[maybe_unused]] char const* name) { - if (m_blocks.is_empty()) { + void* block = nullptr; + bool needs_madvise_reuse = false; + + { + Threading::MutexLocker locker(m_mutex); + + // Prefer m_freshly_freed: those slots were never madvised, so we + // can hand them back out with zero syscalls. This is the deferred- + // decommit payoff -- hot recycle skips both MADV_FREE_REUSABLE + // and MADV_FREE_REUSE. + if (!m_freshly_freed.is_empty()) { + size_t random_index = get_random_uniform(m_freshly_freed.size()); + block = m_freshly_freed.unstable_take(random_index); + } else if (!m_blocks.is_empty()) { + size_t random_index = get_random_uniform(m_blocks.size()); + block = m_blocks.unstable_take(random_index); + needs_madvise_reuse = true; + } + } + + if (block == nullptr) { + // Both pools empty: allocate a fresh 2 MiB chunk and slice it. void* chunk_base = nullptr; #if defined(AK_OS_MACOS) mach_vm_address_t address = 0; @@ -77,7 +290,7 @@ void* BlockAllocator::allocate_block([[maybe_unused]] char const* name) #if defined(MADV_FREE_REUSE) && defined(MADV_FREE_REUSABLE) // Mark the whole chunk reusable upfront so MADV_FREE_REUSE pairs - // symmetrically when slots are popped from m_blocks below. (Linux + // symmetrically when slots are popped from m_blocks later. (Linux // and Windows fall through with no-op.) if (madvise(chunk_base, CHUNK_SIZE, MADV_FREE_REUSABLE) < 0) { perror("madvise(MADV_FREE_REUSABLE)"); @@ -86,21 +299,26 @@ void* BlockAllocator::allocate_block([[maybe_unused]] char const* name) #endif ASAN_POISON_MEMORY_REGION(chunk_base, CHUNK_SIZE); + + Threading::MutexLocker locker(m_mutex); for (size_t i = 0; i < BLOCKS_PER_CHUNK; ++i) m_blocks.append(static_cast(chunk_base) + i * HeapBlock::BLOCK_SIZE); + size_t random_index = get_random_uniform(m_blocks.size()); + block = m_blocks.unstable_take(random_index); + needs_madvise_reuse = true; } - // Random pick to preserve the previous anti-predictability behavior. - size_t random_index = get_random_uniform(m_blocks.size()); - auto* block = m_blocks.unstable_take(random_index); - ASAN_UNPOISON_MEMORY_REGION(block, HeapBlock::BLOCK_SIZE); LSAN_REGISTER_ROOT_REGION(block, HeapBlock::BLOCK_SIZE); #if defined(MADV_FREE_REUSE) && defined(MADV_FREE_REUSABLE) - if (madvise(block, HeapBlock::BLOCK_SIZE, MADV_FREE_REUSE) < 0) { - perror("madvise(MADV_FREE_REUSE)"); - VERIFY_NOT_REACHED(); + if (needs_madvise_reuse) { + if (madvise(block, HeapBlock::BLOCK_SIZE, MADV_FREE_REUSE) < 0) { + perror("madvise(MADV_FREE_REUSE)"); + VERIFY_NOT_REACHED(); + } } +#else + (void)needs_madvise_reuse; #endif return block; } @@ -109,35 +327,22 @@ void BlockAllocator::deallocate_block(void* block) { VERIFY(block); - // Tell the kernel it can reclaim physical pages backing this 16 KiB - // slot. The slot stays in m_blocks for reuse by this same - // BlockAllocator -- never seen by a different cell type. -#if defined(AK_OS_WINDOWS) - DWORD ret = DiscardVirtualMemory(block, HeapBlock::BLOCK_SIZE); - if (ret != ERROR_SUCCESS) { - warnln("{}", Error::from_windows_error(ret)); - VERIFY_NOT_REACHED(); - } -#elif defined(MADV_FREE_REUSE) && defined(MADV_FREE_REUSABLE) - if (madvise(block, HeapBlock::BLOCK_SIZE, MADV_FREE_REUSABLE) < 0) { - perror("madvise(MADV_FREE_REUSABLE)"); - VERIFY_NOT_REACHED(); - } -#elif defined(MADV_FREE) - if (madvise(block, HeapBlock::BLOCK_SIZE, MADV_FREE) < 0) { - perror("madvise(MADV_FREE)"); - VERIFY_NOT_REACHED(); - } -#elif defined(MADV_DONTNEED) - if (madvise(block, HeapBlock::BLOCK_SIZE, MADV_DONTNEED) < 0) { - perror("madvise(MADV_DONTNEED)"); - VERIFY_NOT_REACHED(); - } -#endif - + // Fast path: bookkeep only. The actual madvise is deferred to the + // global decommit worker, which the GC kicks at the end of sweep. ASAN_POISON_MEMORY_REGION(block, HeapBlock::BLOCK_SIZE); LSAN_UNREGISTER_ROOT_REGION(block, HeapBlock::BLOCK_SIZE); - m_blocks.append(block); + + bool need_to_register = false; + { + Threading::MutexLocker locker(m_mutex); + m_freshly_freed.append(block); + if (!m_in_decommit_registry) { + m_in_decommit_registry = true; + need_to_register = true; + } + } + if (need_to_register) + DecommitWorker::the().register_pending(*this); } } diff --git a/Libraries/LibGC/BlockAllocator.h b/Libraries/LibGC/BlockAllocator.h index 00887f3302..d4e9c69ccd 100644 --- a/Libraries/LibGC/BlockAllocator.h +++ b/Libraries/LibGC/BlockAllocator.h @@ -6,23 +6,56 @@ #pragma once +#include #include #include +#include +#include namespace GC { +class DecommitWorker; + class GC_API BlockAllocator { public: - BlockAllocator() = default; + BlockAllocator(); ~BlockAllocator(); void* allocate_block(char const* name); void deallocate_block(void*); - auto const& blocks() const { return m_blocks; } + size_t block_count(); + + // Wake the global decommit worker so it processes any deferred madvise + // work that's piled up. Call this at the end of a GC sweep. + static void wake_decommit_worker_async(); private: + friend class DecommitWorker; + + // Slots in "ready to reuse" state -- have been MADV_FREE_REUSABLE'd by + // the worker (Darwin) so allocate_block pairs them with MADV_FREE_REUSE. Vector m_blocks; + + // Slots freed by deallocate_block but not yet madvised. allocate_block + // pops directly from here to skip the madvise round-trip on hot recycle + // paths -- this is the main payoff of deferring decommit. + Vector m_freshly_freed; + + // Protects m_blocks, m_freshly_freed, and m_in_decommit_registry. Held + // briefly on the alloc/dealloc hot path; uncontended in the common case. + Threading::Mutex m_mutex; + + // Refcount the decommit worker bumps while it has a reference to this + // allocator. The destructor waits on m_worker_cv until it hits zero so + // we never let our storage go away while the worker is still running. + AK::Atomic m_worker_refcount { 0 }; + Threading::ConditionVariable m_worker_cv; + + // True iff this allocator is currently in the worker's pending list. + // Avoids re-registering on every dealloc; cleared by the worker at the + // start of process_one, set by deallocate_block under m_mutex. + bool m_in_decommit_registry { false }; }; } diff --git a/Libraries/LibGC/CMakeLists.txt b/Libraries/LibGC/CMakeLists.txt index 65259fe8e6..b446cbdd91 100644 --- a/Libraries/LibGC/CMakeLists.txt +++ b/Libraries/LibGC/CMakeLists.txt @@ -14,6 +14,7 @@ set(SOURCES ladybird_lib(LibGC gc EXPLICIT_SYMBOL_EXPORT) target_link_libraries(LibGC PRIVATE LibCore) +target_link_libraries(LibGC PUBLIC LibThreading) if(cpptrace_FOUND AND LADYBIRD_ENABLE_CPPTRACE) target_link_libraries(LibGC PRIVATE cpptrace::cpptrace) diff --git a/Libraries/LibGC/Heap.cpp b/Libraries/LibGC/Heap.cpp index c863acf9c1..cf8126e0c9 100644 --- a/Libraries/LibGC/Heap.cpp +++ b/Libraries/LibGC/Heap.cpp @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -424,7 +425,7 @@ void Heap::dump_allocators() builder.appendff(" x {}", total_live_cells); size_t cost = blocks.size() * HeapBlock::BLOCK_SIZE / KiB; - size_t reserved = allocator.block_allocator().blocks().size() * HeapBlock::BLOCK_SIZE / KiB; + size_t reserved = allocator.block_allocator().block_count() * HeapBlock::BLOCK_SIZE / KiB; builder.appendff(", cost: {} KiB, reserved: {} KiB", cost, reserved); size_t total_dead_bytes = ((blocks.size() * cell_count) - total_live_cells) * allocator.cell_size(); @@ -847,6 +848,10 @@ void Heap::sweep_dead_cells(bool print_report, Core::ElapsedTimer const& measure dbgln(" Freed blocks: {} ({} bytes)", empty_blocks.size(), empty_blocks.size() * HeapBlock::BLOCK_SIZE); dbgln("============================================="); } + + // Sweep is done; kick the global decommit worker so the slots we just + // freed get madvise()'d off the GC pause path. + BlockAllocator::wake_decommit_worker_async(); } void Heap::defer_gc() diff --git a/Libraries/LibTest/CMakeLists.txt b/Libraries/LibTest/CMakeLists.txt index 400b41d7f6..56cfd78476 100644 --- a/Libraries/LibTest/CMakeLists.txt +++ b/Libraries/LibTest/CMakeLists.txt @@ -2,6 +2,7 @@ add_library(LibTestMain OBJECT TestMain.cpp AssertionHandler.cpp) target_link_libraries(LibTestMain PUBLIC GenericClangPlugin) add_library(JavaScriptTestRunnerMain OBJECT JavaScriptTestRunnerMain.cpp) +target_link_libraries(JavaScriptTestRunnerMain PRIVATE LibJS LibGC) set(SOURCES TestSuite.cpp