From 716e5f72f2e6890e2ac2cb195236f0076af8d576 Mon Sep 17 00:00:00 2001 From: Andreas Kling Date: Thu, 18 Dec 2025 14:16:18 -0600 Subject: [PATCH] LibGC: Always use 16 KiB as HeapBlock size Before this change, we'd use the system page size as the HeapBlock size. This caused it to vary on different platforms, going as low as 4 KiB on most Linux systems. To make this work, we now use posix_memalign() to ensure we get size-aligned allocations on every platform. Also nice: HeapBlock::BLOCK_SIZE is now a constant. --- Libraries/LibGC/BlockAllocator.cpp | 30 ++++++++++++++---------------- Libraries/LibGC/Heap.cpp | 6 +++--- Libraries/LibGC/HeapBlock.cpp | 4 +--- Libraries/LibGC/HeapBlock.h | 4 ++-- Libraries/LibGC/Internals.h | 6 +++--- 5 files changed, 23 insertions(+), 27 deletions(-) diff --git a/Libraries/LibGC/BlockAllocator.cpp b/Libraries/LibGC/BlockAllocator.cpp index 234f7032b5..d228488bbe 100644 --- a/Libraries/LibGC/BlockAllocator.cpp +++ b/Libraries/LibGC/BlockAllocator.cpp @@ -29,12 +29,9 @@ namespace GC { BlockAllocator::~BlockAllocator() { for (auto* block : m_blocks) { - ASAN_UNPOISON_MEMORY_REGION(block, HeapBlock::block_size); + ASAN_UNPOISON_MEMORY_REGION(block, HeapBlock::BLOCK_SIZE); #if !defined(AK_OS_WINDOWS) - if (munmap(block, HeapBlock::block_size) < 0) { - perror("munmap"); - VERIFY_NOT_REACHED(); - } + free(block); #else if (!VirtualFree(block, 0, MEM_RELEASE)) { warnln("{}", Error::from_windows_error()); @@ -50,19 +47,20 @@ void* BlockAllocator::allocate_block([[maybe_unused]] char const* name) // To reduce predictability, take a random block from the cache. size_t random_index = get_random_uniform(m_blocks.size()); auto* block = m_blocks.unstable_take(random_index); - ASAN_UNPOISON_MEMORY_REGION(block, HeapBlock::block_size); - LSAN_REGISTER_ROOT_REGION(block, HeapBlock::block_size); + ASAN_UNPOISON_MEMORY_REGION(block, HeapBlock::BLOCK_SIZE); + LSAN_REGISTER_ROOT_REGION(block, HeapBlock::BLOCK_SIZE); return block; } #if !defined(AK_OS_WINDOWS) - auto* block = (HeapBlock*)mmap(nullptr, HeapBlock::block_size, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); - VERIFY(block != MAP_FAILED); + void* block = nullptr; + auto rc = posix_memalign(&block, HeapBlock::BLOCK_SIZE, HeapBlock::BLOCK_SIZE); + VERIFY(rc == 0); #else - auto* block = (HeapBlock*)VirtualAlloc(NULL, HeapBlock::block_size, MEM_COMMIT, PAGE_READWRITE); + auto* block = VirtualAlloc(NULL, HeapBlock::BLOCK_SIZE, MEM_COMMIT, PAGE_READWRITE); VERIFY(block); #endif - LSAN_REGISTER_ROOT_REGION(block, HeapBlock::block_size); + LSAN_REGISTER_ROOT_REGION(block, HeapBlock::BLOCK_SIZE); return block; } @@ -71,25 +69,25 @@ void BlockAllocator::deallocate_block(void* block) VERIFY(block); #if defined(AK_OS_WINDOWS) - DWORD ret = DiscardVirtualMemory(block, HeapBlock::block_size); + DWORD ret = DiscardVirtualMemory(block, HeapBlock::BLOCK_SIZE); if (ret != ERROR_SUCCESS) { warnln("{}", Error::from_windows_error(ret)); VERIFY_NOT_REACHED(); } #elif defined(MADV_FREE) - if (madvise(block, HeapBlock::block_size, MADV_FREE) < 0) { + if (madvise(block, HeapBlock::BLOCK_SIZE, MADV_FREE) < 0) { perror("madvise(MADV_FREE)"); VERIFY_NOT_REACHED(); } #elif defined(MADV_DONTNEED) - if (madvise(block, HeapBlock::block_size, MADV_DONTNEED) < 0) { + if (madvise(block, HeapBlock::BLOCK_SIZE, MADV_DONTNEED) < 0) { perror("madvise(MADV_DONTNEED)"); VERIFY_NOT_REACHED(); } #endif - ASAN_POISON_MEMORY_REGION(block, HeapBlock::block_size); - LSAN_UNREGISTER_ROOT_REGION(block, HeapBlock::block_size); + ASAN_POISON_MEMORY_REGION(block, HeapBlock::BLOCK_SIZE); + LSAN_UNREGISTER_ROOT_REGION(block, HeapBlock::BLOCK_SIZE); m_blocks.append(block); } diff --git a/Libraries/LibGC/Heap.cpp b/Libraries/LibGC/Heap.cpp index 68f45a5ac9..cf583e7a1d 100644 --- a/Libraries/LibGC/Heap.cpp +++ b/Libraries/LibGC/Heap.cpp @@ -99,7 +99,7 @@ void Heap::find_min_and_max_block_addresses(FlatPtr& min_address, FlatPtr& max_a max_address = 0; for (auto& allocator : m_all_cell_allocators) { min_address = min(min_address, allocator.min_block_address()); - max_address = max(max_address, allocator.max_block_address() + HeapBlockBase::block_size); + max_address = max(max_address, allocator.max_block_address() + HeapBlock::BLOCK_SIZE); } } @@ -557,8 +557,8 @@ void Heap::sweep_dead_cells(bool print_report, Core::ElapsedTimer const& measure dbgln(" Time spent: {} ms", time_spent.to_milliseconds()); dbgln(" Live cells: {} ({} bytes)", live_cells, live_cell_bytes); dbgln("Collected cells: {} ({} bytes)", collected_cells, collected_cell_bytes); - dbgln(" Live blocks: {} ({} bytes)", live_block_count, live_block_count * HeapBlock::block_size); - dbgln(" Freed blocks: {} ({} bytes)", empty_blocks.size(), empty_blocks.size() * HeapBlock::block_size); + dbgln(" Live blocks: {} ({} bytes)", live_block_count, live_block_count * HeapBlock::BLOCK_SIZE); + dbgln(" Freed blocks: {} ({} bytes)", empty_blocks.size(), empty_blocks.size() * HeapBlock::BLOCK_SIZE); dbgln("============================================="); } } diff --git a/Libraries/LibGC/HeapBlock.cpp b/Libraries/LibGC/HeapBlock.cpp index 61373dc48e..8b678a15b2 100644 --- a/Libraries/LibGC/HeapBlock.cpp +++ b/Libraries/LibGC/HeapBlock.cpp @@ -18,8 +18,6 @@ namespace GC { -size_t HeapBlockBase::block_size = PAGE_SIZE; - NonnullOwnPtr HeapBlock::create_with_cell_size(Heap& heap, CellAllocator& cell_allocator, size_t cell_size, [[maybe_unused]] char const* class_name) { char const* name = nullptr; @@ -34,7 +32,7 @@ HeapBlock::HeapBlock(Heap& heap, CellAllocator& cell_allocator, size_t cell_size , m_cell_size(cell_size) { VERIFY(cell_size >= sizeof(FreelistEntry)); - ASAN_POISON_MEMORY_REGION(m_storage, block_size - sizeof(HeapBlock)); + ASAN_POISON_MEMORY_REGION(m_storage, BLOCK_SIZE - sizeof(HeapBlock)); } void HeapBlock::deallocate(Cell* cell) diff --git a/Libraries/LibGC/HeapBlock.h b/Libraries/LibGC/HeapBlock.h index c4b0ee283c..81f51b287b 100644 --- a/Libraries/LibGC/HeapBlock.h +++ b/Libraries/LibGC/HeapBlock.h @@ -25,11 +25,11 @@ class GC_API HeapBlock : public HeapBlockBase { AK_MAKE_NONMOVABLE(HeapBlock); public: - using HeapBlockBase::block_size; + using HeapBlockBase::BLOCK_SIZE; static NonnullOwnPtr create_with_cell_size(Heap&, CellAllocator&, size_t cell_size, char const* class_name); size_t cell_size() const { return m_cell_size; } - size_t cell_count() const { return (block_size - sizeof(HeapBlock)) / m_cell_size; } + size_t cell_count() const { return (HeapBlock::BLOCK_SIZE - sizeof(HeapBlock)) / m_cell_size; } bool is_full() const { return !has_lazy_freelist() && !m_freelist; } ALWAYS_INLINE Cell* allocate() diff --git a/Libraries/LibGC/Internals.h b/Libraries/LibGC/Internals.h index f40d56239d..56601c8075 100644 --- a/Libraries/LibGC/Internals.h +++ b/Libraries/LibGC/Internals.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2024, Andreas Kling + * Copyright (c) 2020-2025, Andreas Kling * Copyright (c) 2020-2023, the SerenityOS developers. * * SPDX-License-Identifier: BSD-2-Clause @@ -18,10 +18,10 @@ class GC_API HeapBlockBase { AK_MAKE_NONCOPYABLE(HeapBlockBase); public: - static size_t block_size; + static constexpr size_t BLOCK_SIZE = 16 * KiB; static HeapBlockBase* from_cell(Cell const* cell) { - return reinterpret_cast(bit_cast(cell) & ~(HeapBlockBase::block_size - 1)); + return reinterpret_cast(bit_cast(cell) & ~(BLOCK_SIZE - 1)); } Heap& heap() { return m_heap; }