dmaax/ladybird
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

LibGC: Defer per-block madvise to a global background worker

Browse source 
deallocate_block() used to call MADV_FREE_REUSABLE / MADV_FREE /
MADV_DONTNEED inline on every freed block. With sweep typically
freeing many blocks per GC, the cumulative syscall cost shows up
as real GC pause time.

Move the work onto a single global "decommit worker" thread:

- deallocate_block now just poisons the slot and pushes it onto a
  per-allocator m_freshly_freed queue. No syscalls.
- allocate_block prefers m_freshly_freed over m_blocks, so a slot
  that's recycled before the worker sees it skips the
  REUSABLE/REUSE pair entirely. This is the main payoff.
- Heap::sweep_dead_cells kicks the worker at the end of sweep.
  The worker sleeps 50 ms after each kick to give the JS thread
  breathing room, then drains each registered allocator's
  m_freshly_freed, madvises slots in batches of 64 with
  sched_yield between batches, and splices them onto m_blocks.
- Per-allocator refcount + condvar lets ~BlockAllocator wait
  until the worker has dropped its reference before our storage
  goes away. (Chunks themselves remain leaked: type-isolated VM
  is permanent, so we never tear them down.)
This commit is contained in:
Andreas Kling 2026-05-07 10:48:06 +02:00 committed by Andreas Kling
parent 788bfe0f24
commit adfc9d263f
5 changed files with 293 additions and 48 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

295
Libraries/LibGC/BlockAllocator.cpp
View file

@ -6,11 +6,13 @@
*/
#include <AK/Assertions.h>
#include <AK/NeverDestroyed.h>
#include <AK/Platform.h>
#include <AK/Random.h>
#include <AK/Vector.h>
#include <LibGC/BlockAllocator.h>
#include <LibGC/HeapBlock.h>
#include <LibThreading/Thread.h>
#include <sys/mman.h>
#if defined(AK_OS_MACOS)
@ -26,30 +28,241 @@
#if defined(AK_OS_WINDOWS)
# include <AK/Windows.h>
# include <memoryapi.h>
#else
# include <sched.h>
# include <unistd.h>
#endif
namespace GC {
// Each BlockAllocator carves its 16 KiB HeapBlock slots out of 2 MiB
// chunks, so the kernel sees one mmap per 128 blocks instead of one per
// block. Chunks are owned exclusively by a single BlockAllocator and are
// never released back to the OS or shared across allocators -- that's how
// we keep the heap's VM permanently type-isolated, where a virtual address
// used for a cell of type T is never reused for any other type.
// chunks. Chunks are owned exclusively by a single BlockAllocator and are
// never released back to the OS or shared across allocators -- the heap's
// VM is permanently type-isolated.
//
// We do not hint MADV_HUGEPAGE: per-block madvise() in deallocate_block
// would split any THP backing the chunk anyway. Per-block memory return
// matches what V8, SpiderMonkey, and WebKit's libpas all do.
// Per-block madvise() is deferred to a single global background "decommit
// worker" so it never costs us GC pause time, and slots that are recycled
// before the worker sees them skip the madvise pair entirely.
static constexpr size_t CHUNK_SIZE = 2 * MiB;
static constexpr size_t BLOCKS_PER_CHUNK = CHUNK_SIZE / HeapBlock::BLOCK_SIZE;
static_assert(CHUNK_SIZE % HeapBlock::BLOCK_SIZE == 0);
static_assert(BLOCKS_PER_CHUNK == 128);
BlockAllocator::~BlockAllocator() = default;
static void madvise_block_for_decommit(void* block)
{
#if defined(AK_OS_WINDOWS)
DWORD ret = DiscardVirtualMemory(block, HeapBlock::BLOCK_SIZE);
if (ret != ERROR_SUCCESS) {
warnln("{}", Error::from_windows_error(ret));
VERIFY_NOT_REACHED();
}
#elif defined(MADV_FREE_REUSE) && defined(MADV_FREE_REUSABLE)
if (madvise(block, HeapBlock::BLOCK_SIZE, MADV_FREE_REUSABLE) < 0) {
perror("madvise(MADV_FREE_REUSABLE)");
VERIFY_NOT_REACHED();
}
#elif defined(MADV_FREE)
if (madvise(block, HeapBlock::BLOCK_SIZE, MADV_FREE) < 0) {
perror("madvise(MADV_FREE)");
VERIFY_NOT_REACHED();
}
#elif defined(MADV_DONTNEED)
if (madvise(block, HeapBlock::BLOCK_SIZE, MADV_DONTNEED) < 0) {
perror("madvise(MADV_DONTNEED)");
VERIFY_NOT_REACHED();
}
#endif
}
static void sleep_before_decommit()
{
#if defined(AK_OS_WINDOWS)
Sleep(50);
#else
usleep(50 * 1000);
#endif
}
static void yield_during_decommit()
{
#if defined(AK_OS_WINDOWS)
Sleep(0);
#else
sched_yield();
#endif
}
class DecommitWorker {
public:
static DecommitWorker& the();
void register_pending(BlockAllocator&);
void deregister(BlockAllocator&);
void kick();
DecommitWorker();
private:
void run();
void process_one(BlockAllocator&);
Threading::Mutex m_mutex;
Threading::ConditionVariable m_cv { m_mutex };
RefPtr<Threading::Thread> m_thread;
Vector<BlockAllocator*> m_pending;
bool m_kicked { false };
};
DecommitWorker& DecommitWorker::the()
{
static AK::NeverDestroyed<DecommitWorker> instance;
return *instance;
}
DecommitWorker::DecommitWorker()
{
m_thread = Threading::Thread::construct("DecommitWorker"sv, [this] {
run();
return static_cast<intptr_t>(0);
});
m_thread->start();
m_thread->detach();
}
void DecommitWorker::register_pending(BlockAllocator& a)
{
Threading::MutexLocker locker(m_mutex);
m_pending.append(&a);
}
void DecommitWorker::deregister(BlockAllocator& a)
{
Threading::MutexLocker locker(m_mutex);
m_pending.remove_first_matching([&](auto* p) { return p == &a; });
}
void DecommitWorker::kick()
{
{
Threading::MutexLocker locker(m_mutex);
m_kicked = true;
}
m_cv.signal();
}
void DecommitWorker::run()
{
while (true) {
Vector<BlockAllocator*> snapshot;
{
Threading::MutexLocker locker(m_mutex);
while (!m_kicked)
m_cv.wait();
m_kicked = false;
snapshot = move(m_pending);
// Pin every allocator we're about to process so destructors
// block until we drop our reference.
for (auto* a : snapshot)
a->m_worker_refcount.fetch_add(1);
}
if (snapshot.is_empty())
continue;
// Stagger: give the JS thread some breathing room after the kick
// (typically right after sweep ends) before we consume CPU and
// syscall bandwidth.
sleep_before_decommit();
for (auto* a : snapshot) {
process_one(*a);
int prev_refcount = a->m_worker_refcount.fetch_sub(1);
if (prev_refcount == 1) {
Threading::MutexLocker locker(a->m_mutex);
a->m_worker_cv.broadcast();
}
}
}
}
void DecommitWorker::process_one(BlockAllocator& a)
{
Vector<void*> to_process;
{
Threading::MutexLocker locker(a.m_mutex);
a.m_in_decommit_registry = false;
to_process = move(a.m_freshly_freed);
}
// Madvise each slot outside the per-allocator lock so the JS thread can
// continue to allocate/free; yield every 64 slots to avoid hogging the
// kernel's mm subsystem.
constexpr size_t BATCH = 64;
for (size_t i = 0; i < to_process.size(); ++i) {
madvise_block_for_decommit(to_process[i]);
if ((i + 1) % BATCH == 0)
yield_during_decommit();
}
{
Threading::MutexLocker locker(a.m_mutex);
for (auto* slot : to_process)
a.m_blocks.append(slot);
}
}
void BlockAllocator::wake_decommit_worker_async()
{
DecommitWorker::the().kick();
}
BlockAllocator::BlockAllocator()
: m_worker_cv(m_mutex)
{
}
BlockAllocator::~BlockAllocator()
{
// Chunks are permanent -- we never tear them down. The destructor only
// exists to make sure the global decommit worker has finished any
// in-flight processing of *this before our storage goes away.
DecommitWorker::the().deregister(*this);
Threading::MutexLocker locker(m_mutex);
while (m_worker_refcount.load() != 0)
m_worker_cv.wait();
}
size_t BlockAllocator::block_count()
{
Threading::MutexLocker locker(m_mutex);
return m_blocks.size();
}
void* BlockAllocator::allocate_block([[maybe_unused]] char const* name)
{
if (m_blocks.is_empty()) {
void* block = nullptr;
bool needs_madvise_reuse = false;
{
Threading::MutexLocker locker(m_mutex);
// Prefer m_freshly_freed: those slots were never madvised, so we
// can hand them back out with zero syscalls. This is the deferred-
// decommit payoff -- hot recycle skips both MADV_FREE_REUSABLE
// and MADV_FREE_REUSE.
if (!m_freshly_freed.is_empty()) {
size_t random_index = get_random_uniform(m_freshly_freed.size());
block = m_freshly_freed.unstable_take(random_index);
} else if (!m_blocks.is_empty()) {
size_t random_index = get_random_uniform(m_blocks.size());
block = m_blocks.unstable_take(random_index);
needs_madvise_reuse = true;
}
}
if (block == nullptr) {
// Both pools empty: allocate a fresh 2 MiB chunk and slice it.
void* chunk_base = nullptr;
#if defined(AK_OS_MACOS)
mach_vm_address_t address = 0;
@ -77,7 +290,7 @@ void* BlockAllocator::allocate_block([[maybe_unused]] char const* name)
#if defined(MADV_FREE_REUSE) && defined(MADV_FREE_REUSABLE)
// Mark the whole chunk reusable upfront so MADV_FREE_REUSE pairs
// symmetrically when slots are popped from m_blocks below. (Linux
// symmetrically when slots are popped from m_blocks later. (Linux
// and Windows fall through with no-op.)
if (madvise(chunk_base, CHUNK_SIZE, MADV_FREE_REUSABLE) < 0) {
perror("madvise(MADV_FREE_REUSABLE)");
@ -86,21 +299,26 @@ void* BlockAllocator::allocate_block([[maybe_unused]] char const* name)
#endif
ASAN_POISON_MEMORY_REGION(chunk_base, CHUNK_SIZE);
Threading::MutexLocker locker(m_mutex);
for (size_t i = 0; i < BLOCKS_PER_CHUNK; ++i)
m_blocks.append(static_cast<u8*>(chunk_base) + i * HeapBlock::BLOCK_SIZE);
size_t random_index = get_random_uniform(m_blocks.size());
block = m_blocks.unstable_take(random_index);
needs_madvise_reuse = true;
}
// Random pick to preserve the previous anti-predictability behavior.
size_t random_index = get_random_uniform(m_blocks.size());
auto* block = m_blocks.unstable_take(random_index);
ASAN_UNPOISON_MEMORY_REGION(block, HeapBlock::BLOCK_SIZE);
LSAN_REGISTER_ROOT_REGION(block, HeapBlock::BLOCK_SIZE);
#if defined(MADV_FREE_REUSE) && defined(MADV_FREE_REUSABLE)
if (madvise(block, HeapBlock::BLOCK_SIZE, MADV_FREE_REUSE) < 0) {
perror("madvise(MADV_FREE_REUSE)");
VERIFY_NOT_REACHED();
if (needs_madvise_reuse) {
if (madvise(block, HeapBlock::BLOCK_SIZE, MADV_FREE_REUSE) < 0) {
perror("madvise(MADV_FREE_REUSE)");
VERIFY_NOT_REACHED();
}
}
#else
(void)needs_madvise_reuse;
#endif
return block;
}
@ -109,35 +327,22 @@ void BlockAllocator::deallocate_block(void* block)
{
VERIFY(block);
// Tell the kernel it can reclaim physical pages backing this 16 KiB
// slot. The slot stays in m_blocks for reuse by this same
// BlockAllocator -- never seen by a different cell type.
#if defined(AK_OS_WINDOWS)
DWORD ret = DiscardVirtualMemory(block, HeapBlock::BLOCK_SIZE);
if (ret != ERROR_SUCCESS) {
warnln("{}", Error::from_windows_error(ret));
VERIFY_NOT_REACHED();
}
#elif defined(MADV_FREE_REUSE) && defined(MADV_FREE_REUSABLE)
if (madvise(block, HeapBlock::BLOCK_SIZE, MADV_FREE_REUSABLE) < 0) {
perror("madvise(MADV_FREE_REUSABLE)");
VERIFY_NOT_REACHED();
}
#elif defined(MADV_FREE)
if (madvise(block, HeapBlock::BLOCK_SIZE, MADV_FREE) < 0) {
perror("madvise(MADV_FREE)");
VERIFY_NOT_REACHED();
}
#elif defined(MADV_DONTNEED)
if (madvise(block, HeapBlock::BLOCK_SIZE, MADV_DONTNEED) < 0) {
perror("madvise(MADV_DONTNEED)");
VERIFY_NOT_REACHED();
}
#endif
// Fast path: bookkeep only. The actual madvise is deferred to the
// global decommit worker, which the GC kicks at the end of sweep.
ASAN_POISON_MEMORY_REGION(block, HeapBlock::BLOCK_SIZE);
LSAN_UNREGISTER_ROOT_REGION(block, HeapBlock::BLOCK_SIZE);
m_blocks.append(block);
bool need_to_register = false;
{
Threading::MutexLocker locker(m_mutex);
m_freshly_freed.append(block);
if (!m_in_decommit_registry) {
m_in_decommit_registry = true;
need_to_register = true;
}
}
if (need_to_register)
DecommitWorker::the().register_pending(*this);
}
}

37
Libraries/LibGC/BlockAllocator.h
View file

@ -6,23 +6,56 @@
#pragma once
#include <AK/Atomic.h>
#include <AK/Vector.h>
#include <LibGC/Forward.h>
#include <LibThreading/ConditionVariable.h>
#include <LibThreading/Mutex.h>
namespace GC {
class DecommitWorker;
class GC_API BlockAllocator {
public:
BlockAllocator() = default;
BlockAllocator();
~BlockAllocator();
void* allocate_block(char const* name);
void deallocate_block(void*);
auto const& blocks() const { return m_blocks; }
size_t block_count();
// Wake the global decommit worker so it processes any deferred madvise
// work that's piled up. Call this at the end of a GC sweep.
static void wake_decommit_worker_async();
private:
friend class DecommitWorker;
// Slots in "ready to reuse" state -- have been MADV_FREE_REUSABLE'd by
// the worker (Darwin) so allocate_block pairs them with MADV_FREE_REUSE.
Vector<void*> m_blocks;
// Slots freed by deallocate_block but not yet madvised. allocate_block
// pops directly from here to skip the madvise round-trip on hot recycle
// paths -- this is the main payoff of deferring decommit.
Vector<void*> m_freshly_freed;
// Protects m_blocks, m_freshly_freed, and m_in_decommit_registry. Held
// briefly on the alloc/dealloc hot path; uncontended in the common case.
Threading::Mutex m_mutex;
// Refcount the decommit worker bumps while it has a reference to this
// allocator. The destructor waits on m_worker_cv until it hits zero so
// we never let our storage go away while the worker is still running.
AK::Atomic<int> m_worker_refcount { 0 };
Threading::ConditionVariable m_worker_cv;
// True iff this allocator is currently in the worker's pending list.
// Avoids re-registering on every dealloc; cleared by the worker at the
// start of process_one, set by deallocate_block under m_mutex.
bool m_in_decommit_registry { false };
};
}

1
Libraries/LibGC/CMakeLists.txt
View file

@ -14,6 +14,7 @@ set(SOURCES
ladybird_lib(LibGC gc EXPLICIT_SYMBOL_EXPORT)
target_link_libraries(LibGC PRIVATE LibCore)
target_link_libraries(LibGC PUBLIC LibThreading)
if(cpptrace_FOUND AND LADYBIRD_ENABLE_CPPTRACE)
target_link_libraries(LibGC PRIVATE cpptrace::cpptrace)

7
Libraries/LibGC/Heap.cpp
View file

@ -21,6 +21,7 @@
#include <LibCore/ElapsedTimer.h>
#include <LibCore/File.h>
#include <LibCore/StandardPaths.h>
#include <LibGC/BlockAllocator.h>
#include <LibGC/CellAllocator.h>
#include <LibGC/Heap.h>
#include <LibGC/HeapBlock.h>
@ -424,7 +425,7 @@ void Heap::dump_allocators()
builder.appendff(" x {}", total_live_cells);
size_t cost = blocks.size() * HeapBlock::BLOCK_SIZE / KiB;
size_t reserved = allocator.block_allocator().blocks().size() * HeapBlock::BLOCK_SIZE / KiB;
size_t reserved = allocator.block_allocator().block_count() * HeapBlock::BLOCK_SIZE / KiB;
builder.appendff(", cost: {} KiB, reserved: {} KiB", cost, reserved);
size_t total_dead_bytes = ((blocks.size() * cell_count) - total_live_cells) * allocator.cell_size();
@ -847,6 +848,10 @@ void Heap::sweep_dead_cells(bool print_report, Core::ElapsedTimer const& measure
dbgln(" Freed blocks: {} ({} bytes)", empty_blocks.size(), empty_blocks.size() * HeapBlock::BLOCK_SIZE);
dbgln("=============================================");
}
// Sweep is done; kick the global decommit worker so the slots we just
// freed get madvise()'d off the GC pause path.
BlockAllocator::wake_decommit_worker_async();
}
void Heap::defer_gc()

1
Libraries/LibTest/CMakeLists.txt
View file

@ -2,6 +2,7 @@ add_library(LibTestMain OBJECT TestMain.cpp AssertionHandler.cpp)
target_link_libraries(LibTestMain PUBLIC GenericClangPlugin)
add_library(JavaScriptTestRunnerMain OBJECT JavaScriptTestRunnerMain.cpp)
target_link_libraries(JavaScriptTestRunnerMain PRIVATE LibJS LibGC)
set(SOURCES
TestSuite.cpp