diff --git a/AK/Platform.h b/AK/Platform.h index 1000e7f472..2aefdda712 100644 --- a/AK/Platform.h +++ b/AK/Platform.h @@ -276,11 +276,13 @@ # define ASAN_UNPOISON_MEMORY_REGION(addr, size) __asan_unpoison_memory_region(addr, size) # define LSAN_REGISTER_ROOT_REGION(base, size) __lsan_register_root_region(base, size) # define LSAN_UNREGISTER_ROOT_REGION(base, size) __lsan_unregister_root_region(base, size) +# define LSAN_IGNORE_OBJECT(base) __lsan_ignore_object(base) #else # define ASAN_POISON_MEMORY_REGION(addr, size) # define ASAN_UNPOISON_MEMORY_REGION(addr, size) # define LSAN_REGISTER_ROOT_REGION(base, size) # define LSAN_UNREGISTER_ROOT_REGION(base, size) +# define LSAN_IGNORE_OBJECT(base) #endif #if __has_feature(blocks) diff --git a/Libraries/LibGC/CMakeLists.txt b/Libraries/LibGC/CMakeLists.txt index c89319960a..378f0790b5 100644 --- a/Libraries/LibGC/CMakeLists.txt +++ b/Libraries/LibGC/CMakeLists.txt @@ -9,6 +9,7 @@ set(SOURCES RootVector.cpp Heap.cpp HeapBlock.cpp + WeakBlock.cpp WeakContainer.cpp ) diff --git a/Libraries/LibGC/Forward.h b/Libraries/LibGC/Forward.h index b531242a68..ddf12dc528 100644 --- a/Libraries/LibGC/Forward.h +++ b/Libraries/LibGC/Forward.h @@ -20,6 +20,7 @@ class Heap; class HeapBlock; class NanBoxedValue; class WeakContainer; +class WeakImpl; template class Function; diff --git a/Libraries/LibGC/Heap.cpp b/Libraries/LibGC/Heap.cpp index 0c25853778..fe661af006 100644 --- a/Libraries/LibGC/Heap.cpp +++ b/Libraries/LibGC/Heap.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2022, Andreas Kling + * Copyright (c) 2020-2025, Andreas Kling * Copyright (c) 2023, Aliaksandr Kalenik * * SPDX-License-Identifier: BSD-2-Clause @@ -20,6 +20,8 @@ #include #include #include +#include +#include #include #ifdef HAS_ADDRESS_SANITIZER @@ -258,6 +260,7 @@ void Heap::collect_garbage(CollectionType collection_type, bool print_report) mark_live_cells(roots); } finalize_unmarked_cells(); + sweep_weak_blocks(); sweep_dead_cells(print_report, collection_measurement_timer); } @@ -462,6 +465,22 @@ void Heap::finalize_unmarked_cells() }); } +void Heap::sweep_weak_blocks() +{ + for (auto& weak_block : m_usable_weak_blocks) { + weak_block.sweep(); + } + Vector now_usable_weak_blocks; + for (auto& weak_block : m_full_weak_blocks) { + weak_block.sweep(); + if (weak_block.can_allocate()) + now_usable_weak_blocks.append(weak_block); + } + for (auto& weak_block : now_usable_weak_blocks) { + m_usable_weak_blocks.append(weak_block); + } +} + void Heap::sweep_dead_cells(bool print_report, Core::ElapsedTimer const& measurement_timer) { dbgln_if(HEAP_DEBUG, "sweep_dead_cells:"); @@ -559,4 +578,21 @@ void Heap::uproot_cell(Cell* cell) m_uprooted_cells.append(cell); } +WeakImpl* Heap::create_weak_impl(void* ptr) +{ + if (m_usable_weak_blocks.is_empty()) { + // NOTE: These are leaked on Heap destruction, but that's fine since Heap is tied to process lifetime. + auto* weak_block = WeakBlock::create(); + m_usable_weak_blocks.append(*weak_block); + } + + auto* weak_block = m_usable_weak_blocks.first(); + auto* new_weak_impl = weak_block->allocate(static_cast(ptr)); + if (!weak_block->can_allocate()) { + m_full_weak_blocks.append(*weak_block); + } + + return new_weak_impl; +} + } diff --git a/Libraries/LibGC/Heap.h b/Libraries/LibGC/Heap.h index 193ce4ecc1..91b15c4c7f 100644 --- a/Libraries/LibGC/Heap.h +++ b/Libraries/LibGC/Heap.h @@ -25,6 +25,7 @@ #include #include #include +#include #include namespace GC { @@ -81,6 +82,8 @@ public: void enqueue_post_gc_task(AK::Function); + WeakImpl* create_weak_impl(void*); + private: friend class MarkingVisitor; friend class GraphConstructorVisitor; @@ -113,6 +116,7 @@ private: void mark_live_cells(HashMap const& live_cells); void finalize_unmarked_cells(); void sweep_dead_cells(bool print_report, Core::ElapsedTimer const&); + void sweep_weak_blocks(); ALWAYS_INLINE CellAllocator& allocator_for_size(size_t cell_size) { @@ -159,6 +163,9 @@ private: AK::Function&)> m_gather_embedder_roots; Vector> m_post_gc_tasks; + + WeakBlock::List m_usable_weak_blocks; + WeakBlock::List m_full_weak_blocks; } SWIFT_IMMORTAL_REFERENCE; inline void Heap::did_create_root(Badge, RootImpl& impl) diff --git a/Libraries/LibGC/Weak.h b/Libraries/LibGC/Weak.h new file mode 100644 index 0000000000..b1d9a60eed --- /dev/null +++ b/Libraries/LibGC/Weak.h @@ -0,0 +1,166 @@ +/* + * Copyright (c) 2025, Andreas Kling + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#pragma once + +#include +#include +#include +#include + +namespace GC { + +class WeakBlock; + +class WeakImpl { +public: + // NOTE: Null GC::Weaks point at this WeakImpl. This allows Weak to always chase the impl pointer without null-checking it. + static GC_API WeakImpl the_null_weak_impl; + + WeakImpl() = default; + WeakImpl(void* ptr) + : m_ptr(ptr) + { + } + + void* ptr() const { return m_ptr; } + void set_ptr(Badge, void* ptr) { m_ptr = ptr; } + + bool operator==(WeakImpl const& other) const { return m_ptr == other.m_ptr; } + bool operator!=(WeakImpl const& other) const { return m_ptr != other.m_ptr; } + + void ref() const { ++m_ref_count; } + void unref() const + { + VERIFY(m_ref_count); + --m_ref_count; + } + + size_t ref_count() const { return m_ref_count; } + + enum class State { + Allocated, + Freelist, + }; + + void set_state(State state) { m_state = state; } + State state() const { return m_state; } + +private: + mutable size_t m_ref_count { 0 }; + State m_state { State::Allocated }; + void* m_ptr { nullptr }; +}; + +template +class Weak { +public: + constexpr Weak() = default; + Weak(nullptr_t) { } + + Weak(T const* ptr); + Weak(T const& ptr); + + template + Weak(Weak const& other) + requires(IsConvertible); + + Weak(Ref const& other); + + template + Weak(Ref const& other) + requires(IsConvertible); + + template + Weak& operator=(Weak const& other) + requires(IsConvertible) + { + m_impl = other.impl(); + return *this; + } + + Weak& operator=(Ref const& other); + + template + Weak& operator=(Ref const& other) + requires(IsConvertible); + + Weak& operator=(T const& other); + + template + Weak& operator=(U const& other) + requires(IsConvertible); + + Weak& operator=(T const* other); + + template + Weak& operator=(U const* other) + requires(IsConvertible); + + T* operator->() const + { + ASSERT(ptr()); + return ptr(); + } + + [[nodiscard]] T& operator*() const + { + ASSERT(ptr()); + return *ptr(); + } + + Ptr ptr() const { return static_cast(impl().ptr()); } + + explicit operator bool() const { return !!ptr(); } + bool operator!() const { return !ptr(); } + + operator T*() const { return ptr(); } + + Ref as_nonnull() const + { + ASSERT(ptr()); + return *ptr(); + } + + WeakImpl& impl() const { return *m_impl; } + +private: + NonnullRefPtr m_impl { WeakImpl::the_null_weak_impl }; +}; + +template +inline bool operator==(Weak const& a, Ptr const& b) +{ + return a.ptr() == b.ptr(); +} + +template +inline bool operator==(Weak const& a, Ref const& b) +{ + return a.ptr() == b.ptr(); +} + +} + +namespace AK { + +template +struct Traits> : public DefaultTraits> { + static unsigned hash(GC::Weak const& value) + { + return Traits::hash(value.ptr()); + } +}; + +template +struct Formatter> : Formatter { + ErrorOr format(FormatBuilder& builder, GC::Weak const& value) + { + return Formatter::format(builder, value.ptr()); + } +}; + +} diff --git a/Libraries/LibGC/WeakBlock.cpp b/Libraries/LibGC/WeakBlock.cpp new file mode 100644 index 0000000000..5a00dbd7ac --- /dev/null +++ b/Libraries/LibGC/WeakBlock.cpp @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2025, Andreas Kling + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include +#include +#include + +#if defined(AK_OS_WINDOWS) +# include +# include +#endif + +namespace GC { + +WeakImpl WeakImpl::the_null_weak_impl; + +WeakBlock* WeakBlock::create() +{ +#if !defined(AK_OS_WINDOWS) + auto* block = (HeapBlock*)mmap(nullptr, WeakBlock::BLOCK_SIZE, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); + VERIFY(block != MAP_FAILED); +#else + auto* block = (HeapBlock*)VirtualAlloc(NULL, WeakBlock::BLOCK_SIZE, MEM_COMMIT, PAGE_READWRITE); + VERIFY(block); +#endif + return new (block) WeakBlock; +} + +WeakBlock::WeakBlock() +{ + for (size_t i = 0; i < IMPL_COUNT; ++i) { + m_impls[i].set_ptr({}, i + 1 < IMPL_COUNT ? &m_impls[i + 1] : nullptr); + m_impls[i].set_state(WeakImpl::State::Freelist); + } + m_freelist = &m_impls[0]; +} + +WeakBlock::~WeakBlock() = default; + +WeakImpl* WeakBlock::allocate(Cell* cell) +{ + auto* impl = m_freelist; + if (!impl) + return nullptr; + VERIFY(impl->ref_count() == 0); + m_freelist = impl->ptr() ? static_cast(impl->ptr()) : nullptr; + impl->set_ptr({}, cell); + impl->set_state(WeakImpl::State::Allocated); + return impl; +} + +void WeakBlock::deallocate(WeakImpl* impl) +{ + VERIFY(impl->ref_count() == 0); + impl->set_ptr({}, m_freelist); + impl->set_state(WeakImpl::State::Freelist); + m_freelist = impl; +} + +void WeakBlock::sweep() +{ + for (size_t i = 0; i < IMPL_COUNT; ++i) { + auto& impl = m_impls[i]; + if (impl.state() == WeakImpl::State::Freelist) + continue; + auto* cell = static_cast(impl.ptr()); + if (!cell || !cell->is_marked()) + impl.set_ptr({}, nullptr); + if (impl.ref_count() == 0) + deallocate(&impl); + } +} + +} diff --git a/Libraries/LibGC/WeakBlock.h b/Libraries/LibGC/WeakBlock.h new file mode 100644 index 0000000000..c475ddd0e6 --- /dev/null +++ b/Libraries/LibGC/WeakBlock.h @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2025, Andreas Kling + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#pragma once + +#include +#include +#include + +namespace GC { + +class GC_API WeakBlock { +public: + static constexpr size_t BLOCK_SIZE = 16 * KiB; + + static WeakBlock* create(); + + WeakImpl* allocate(Cell*); + void deallocate(WeakImpl*); + + bool can_allocate() const { return m_freelist != nullptr; } + + void sweep(); + +private: + WeakBlock(); + ~WeakBlock(); + + IntrusiveListNode m_list_node; + +public: + using List = IntrusiveList<&WeakBlock::m_list_node>; + + WeakImpl* m_freelist { nullptr }; + + static constexpr size_t IMPL_COUNT = (BLOCK_SIZE - sizeof(m_list_node) - sizeof(WeakImpl*)) / sizeof(WeakImpl); + WeakImpl m_impls[IMPL_COUNT]; +}; + +static_assert(sizeof(WeakBlock) <= WeakBlock::BLOCK_SIZE); + +} diff --git a/Libraries/LibGC/WeakInlines.h b/Libraries/LibGC/WeakInlines.h new file mode 100644 index 0000000000..9a8fa00619 --- /dev/null +++ b/Libraries/LibGC/WeakInlines.h @@ -0,0 +1,113 @@ +/* + * Copyright (c) 2025, Andreas Kling + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#pragma once + +#include + +namespace GC { + +template +Weak::Weak(T const* ptr) + : m_impl(ptr ? *ptr->heap().create_weak_impl(const_cast(static_cast(ptr))) : WeakImpl::the_null_weak_impl) +{ +} + +template +Weak::Weak(T const& ptr) + : m_impl(*ptr.heap().create_weak_impl(const_cast(static_cast(&ptr)))) +{ +} + +template +template +Weak::Weak(Weak const& other) +requires(IsConvertible) + : m_impl(other.impl()) +{ +} + +template +Weak::Weak(Ref const& other) + : m_impl(*other.ptr()->heap().create_weak_impl(other.ptr())) +{ +} + +template +template +Weak::Weak(Ref const& other) +requires(IsConvertible) + : m_impl(*other.ptr()->heap().create_weak_impl(other.ptr())) +{ +} + +template +template +Weak& Weak::operator=(U const& other) +requires(IsConvertible) +{ + if (ptr() != other) { + m_impl = *other.heap().create_weak_impl(const_cast(static_cast(&other))); + } + return *this; +} + +template +Weak& Weak::operator=(Ref const& other) +{ + if (ptr() != other.ptr()) { + m_impl = *other.ptr()->heap().create_weak_impl(other.ptr()); + } + return *this; +} + +template +template +Weak& Weak::operator=(Ref const& other) +requires(IsConvertible) +{ + if (ptr() != other.ptr()) { + m_impl = *other.ptr()->heap().create_weak_impl(other.ptr()); + } + return *this; +} + +template +Weak& Weak::operator=(T const& other) +{ + if (ptr() != &other) { + m_impl = *other.heap().create_weak_impl(const_cast(static_cast(&other))); + } + return *this; +} + +template +Weak& Weak::operator=(T const* other) +{ + if (ptr() != other) { + if (other) + m_impl = *other->heap().create_weak_impl(const_cast(static_cast(other))); + else + m_impl = WeakImpl::the_null_weak_impl; + } + return *this; +} + +template +template +Weak& Weak::operator=(U const* other) +requires(IsConvertible) +{ + if (ptr() != other) { + if (other) + m_impl = *other->heap().create_weak_impl(const_cast(static_cast(other))); + else + m_impl = WeakImpl::the_null_weak_impl; + } + return *this; +} + +}