ladybird/Libraries/LibWeb/Compositor/CompositorThread.cpp
Andreas Kling 1b810a5e15 LibWeb: Present frames through Compositor IPC
Introduce a dedicated Compositor IPC channel between the UI process and
WebContent. Use it for backing-store setup, presented bitmap delivery,
and bitmap-specific ready_to_paint acknowledgements.

This makes CompositorThread the single owner of frame presentation
bookkeeping before async scrolling starts producing frames without the
main thread.

Remove old paint and backing-store messages from WebContentClient and
PageClient so the UI process no longer observes two presentation
protocols.
2026-05-12 20:57:08 +02:00

640 lines
24 KiB
C++

/*
* Copyright (c) 2025-2026, Aliaksandr Kalenik <kalenik.aliaksandr@gmail.com>
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#include <LibCore/EventLoop.h>
#include <LibGfx/DecodedImageFrame.h>
#include <LibGfx/PaintingSurface.h>
#include <LibGfx/SharedImage.h>
#include <LibGfx/SharedImageBuffer.h>
#include <LibGfx/SkiaBackendContext.h>
#include <LibThreading/Thread.h>
#include <LibWeb/Compositor/CompositorThread.h>
#include <LibWeb/Painting/DisplayListPlayerSkia.h>
#include <LibWeb/Painting/ExternalContentSource.h>
#include <AK/HashMap.h>
#include <AK/NeverDestroyed.h>
#include <AK/Queue.h>
#ifdef USE_VULKAN_DMABUF_IMAGES
# include <AK/Array.h>
# include <LibGfx/VulkanImage.h>
# include <libdrm/drm_fourcc.h>
#endif
#include <core/SkCanvas.h>
#include <core/SkColor.h>
#include <LibCore/Platform/ScopedAutoreleasePool.h>
namespace Web::Compositor {
struct BackingStoreState {
RefPtr<Gfx::PaintingSurface> front_store;
RefPtr<Gfx::PaintingSurface> back_store;
i32 front_bitmap_id { -1 };
i32 back_bitmap_id { -1 };
void swap()
{
AK::swap(front_store, back_store);
AK::swap(front_bitmap_id, back_bitmap_id);
}
bool is_valid() const { return front_store && back_store; }
};
struct UpdateDisplayListCommand {
NonnullRefPtr<Painting::DisplayList> display_list;
Painting::ScrollStateSnapshot scroll_state_snapshot;
};
struct UpdateScrollStateCommand {
Painting::ScrollStateSnapshot scroll_state_snapshot;
};
struct UpdateBackingStoresCommand {
Gfx::IntSize size;
i32 front_bitmap_id;
i32 back_bitmap_id;
};
struct ScreenshotCommand {
NonnullRefPtr<Gfx::PaintingSurface> target_surface;
Function<void()> callback;
};
using CompositorCommand = Variant<UpdateDisplayListCommand, UpdateScrollStateCommand, UpdateBackingStoresCommand, ScreenshotCommand>;
struct BackingStorePair {
RefPtr<Gfx::PaintingSurface> front;
RefPtr<Gfx::PaintingSurface> back;
};
#ifdef USE_VULKAN
static NonnullRefPtr<Gfx::PaintingSurface> create_gpu_painting_surface_with_bitmap_flush(Gfx::IntSize size, Gfx::SharedImageBuffer& buffer, RefPtr<Gfx::SkiaBackendContext> const& skia_backend_context)
{
auto surface = Gfx::PaintingSurface::create_with_size(size, Gfx::BitmapFormat::BGRA8888, Gfx::AlphaType::Premultiplied, skia_backend_context);
auto bitmap = buffer.bitmap();
surface->on_flush = [bitmap = move(bitmap)](auto& surface) {
surface.read_into_bitmap(*bitmap);
};
return surface;
}
#endif
static BackingStorePair create_shareable_bitmap_backing_stores([[maybe_unused]] Gfx::IntSize size, Gfx::SharedImageBuffer& front_buffer, Gfx::SharedImageBuffer& back_buffer, RefPtr<Gfx::SkiaBackendContext> const& skia_backend_context)
{
#ifdef AK_OS_MACOS
if (skia_backend_context) {
return {
.front = Gfx::PaintingSurface::create_from_shared_image_buffer(front_buffer, *skia_backend_context),
.back = Gfx::PaintingSurface::create_from_shared_image_buffer(back_buffer, *skia_backend_context),
};
}
#else
# ifdef USE_VULKAN
if (skia_backend_context) {
return {
.front = create_gpu_painting_surface_with_bitmap_flush(size, front_buffer, skia_backend_context),
.back = create_gpu_painting_surface_with_bitmap_flush(size, back_buffer, skia_backend_context),
};
}
# else
(void)skia_backend_context;
# endif
#endif
return {
.front = Gfx::PaintingSurface::wrap_bitmap(*front_buffer.bitmap()),
.back = Gfx::PaintingSurface::wrap_bitmap(*back_buffer.bitmap()),
};
}
#ifdef USE_VULKAN_DMABUF_IMAGES
struct DMABufBackingStorePair {
RefPtr<Gfx::PaintingSurface> front;
RefPtr<Gfx::PaintingSurface> back;
Gfx::SharedImage front_shared_image;
Gfx::SharedImage back_shared_image;
};
static ErrorOr<DMABufBackingStorePair> create_linear_dmabuf_backing_stores(Gfx::IntSize size, Gfx::SkiaBackendContext& skia_backend_context)
{
auto const& vulkan_context = skia_backend_context.vulkan_context();
static constexpr Array<uint64_t, 1> linear_modifiers = { DRM_FORMAT_MOD_LINEAR };
auto front_image = TRY(Gfx::create_shared_vulkan_image(vulkan_context, size.width(), size.height(), VK_FORMAT_B8G8R8A8_UNORM, linear_modifiers.span()));
auto back_image = TRY(Gfx::create_shared_vulkan_image(vulkan_context, size.width(), size.height(), VK_FORMAT_B8G8R8A8_UNORM, linear_modifiers.span()));
auto front_shared_image = Gfx::duplicate_shared_image(*front_image);
auto back_shared_image = Gfx::duplicate_shared_image(*back_image);
return DMABufBackingStorePair {
.front = Gfx::PaintingSurface::create_from_vkimage(skia_backend_context, move(front_image), Gfx::PaintingSurface::Origin::TopLeft),
.back = Gfx::PaintingSurface::create_from_vkimage(skia_backend_context, move(back_image), Gfx::PaintingSurface::Origin::TopLeft),
.front_shared_image = move(front_shared_image),
.back_shared_image = move(back_shared_image),
};
}
#endif
class CompositorThread::ThreadData final : public AtomicRefCounted<ThreadData> {
public:
ThreadData(u64 page_id, NonnullRefPtr<Core::WeakEventLoopReference>&& main_thread_event_loop, CompositorThread::PagePresentationRegistration page_presentation_registration)
: m_page_id(page_id)
, m_main_thread_event_loop(move(main_thread_event_loop))
, m_presents_to_client(page_presentation_registration == CompositorThread::PagePresentationRegistration::Yes)
{
}
~ThreadData() = default;
u64 page_id() const { return m_page_id; }
bool presents_to_client() const { return m_presents_to_client; }
void stop_presenting_to_client()
{
Sync::MutexLocker const locker { m_mutex };
m_presents_to_client = false;
}
void set_presentation_mode(CompositorThread::PresentationMode mode)
{
Sync::MutexLocker const locker { m_mutex };
m_presentation_mode = move(mode);
}
void exit()
{
Sync::MutexLocker const locker { m_mutex };
m_exit = true;
m_command_ready.signal();
m_ready_to_paint.signal();
m_frame_completed.broadcast();
}
void enqueue_command(CompositorCommand&& command)
{
Sync::MutexLocker const locker { m_mutex };
m_command_queue.enqueue(move(command));
m_command_ready.signal();
}
u64 set_needs_present(Gfx::IntRect viewport_rect)
{
Sync::MutexLocker const locker { m_mutex };
m_needs_present = true;
m_pending_viewport_rect = viewport_rect;
m_submitted_frame_id++;
m_command_ready.signal();
return m_submitted_frame_id;
}
void mark_frame_complete(u64 frame_id)
{
Sync::MutexLocker const locker { m_mutex };
m_completed_frame_id = frame_id;
m_frame_completed.broadcast();
}
void wait_for_frame(u64 frame_id)
{
Sync::MutexLocker const locker { m_mutex };
while (m_completed_frame_id < frame_id && !m_exit)
m_frame_completed.wait();
}
void compositor_loop(DisplayListPlayerType display_list_player_type)
{
initialize_skia_player(display_list_player_type);
while (true) {
{
Sync::MutexLocker const locker { m_mutex };
while (m_command_queue.is_empty() && !m_needs_present && !m_exit) {
m_command_ready.wait();
}
if (m_exit)
break;
}
// Drain autoreleased Objective-C objects created by Metal/Skia each iteration,
// since this background thread has no autorelease pool.
Core::ScopedAutoreleasePool autorelease_pool;
while (true) {
auto command = [this]() -> Optional<CompositorCommand> {
Sync::MutexLocker const locker { m_mutex };
if (m_command_queue.is_empty())
return {};
return m_command_queue.dequeue();
}();
if (!command.has_value())
break;
command->visit(
[this](UpdateDisplayListCommand& cmd) {
m_cached_display_list = move(cmd.display_list);
m_cached_scroll_state_snapshot = move(cmd.scroll_state_snapshot);
},
[this](UpdateScrollStateCommand& cmd) {
m_cached_scroll_state_snapshot = move(cmd.scroll_state_snapshot);
},
[this](UpdateBackingStoresCommand& cmd) {
allocate_backing_stores(cmd);
m_backing_stores.front_bitmap_id = cmd.front_bitmap_id;
m_backing_stores.back_bitmap_id = cmd.back_bitmap_id;
},
[this](ScreenshotCommand& cmd) {
if (!m_cached_display_list)
return;
m_skia_player->execute(*m_cached_display_list, m_cached_scroll_state_snapshot, *cmd.target_surface);
if (cmd.callback) {
invoke_on_main_thread([callback = move(cmd.callback)]() mutable {
callback();
});
}
});
if (m_exit)
break;
}
if (m_exit)
break;
bool should_present = false;
Gfx::IntRect viewport_rect;
u64 presenting_frame_id = 0;
{
Sync::MutexLocker const locker { m_mutex };
if (m_needs_present) {
should_present = true;
viewport_rect = m_pending_viewport_rect;
presenting_frame_id = m_submitted_frame_id;
m_needs_present = false;
}
}
if (should_present) {
// Block if we already have a frame queued (back pressure).
{
Sync::MutexLocker const locker { m_mutex };
while (m_queued_rasterization_tasks > 0 && !m_exit) {
m_ready_to_paint.wait();
}
if (m_exit)
break;
}
auto presentation_mode = [this] {
Sync::MutexLocker const locker { m_mutex };
return m_presentation_mode;
}();
if (m_cached_display_list && m_backing_stores.is_valid()) {
auto should_clear_back_store = presentation_mode.visit(
[](CompositorThread::PresentToUI) { return false; },
[](CompositorThread::PublishToExternalContent const&) { return true; });
if (should_clear_back_store) {
// Embedded navigables leave their PaintConfig canvas unfilled, so double-buffered back stores
// must be cleared before repainting.
m_backing_stores.back_store->canvas().clear(SK_ColorTRANSPARENT);
}
m_skia_player->execute(*m_cached_display_list, m_cached_scroll_state_snapshot, *m_backing_stores.back_store);
i32 rendered_bitmap_id = m_backing_stores.back_bitmap_id;
m_backing_stores.swap();
presentation_mode.visit(
[this, viewport_rect, rendered_bitmap_id](CompositorThread::PresentToUI) {
if (m_presents_to_client) {
finish_rasterizing(rendered_bitmap_id);
VERIFY(CompositorThread::present_frame_to_client(m_page_id, viewport_rect, rendered_bitmap_id));
}
},
[this](CompositorThread::PublishToExternalContent const& mode) {
auto snapshot = Gfx::DecodedImageFrame { *m_backing_stores.front_store->snapshot_bitmap() };
mode.source->update(move(snapshot));
});
}
mark_frame_complete(presenting_frame_id);
}
}
}
private:
void initialize_skia_player(DisplayListPlayerType display_list_player_type)
{
switch (display_list_player_type) {
case DisplayListPlayerType::SkiaGPUIfAvailable:
m_skia_backend_context = Gfx::SkiaBackendContext::create_independent_gpu_backend();
break;
case DisplayListPlayerType::SkiaCPU:
break;
}
m_skia_player = make<Painting::DisplayListPlayerSkia>(m_skia_backend_context);
}
void publish_backing_store_pair(UpdateBackingStoresCommand& cmd, Gfx::SharedImage front_shared_image, Gfx::SharedImage back_shared_image)
{
if (!m_presents_to_client)
return;
VERIFY(CompositorThread::present_backing_stores_to_client(m_page_id, cmd.front_bitmap_id, move(front_shared_image), cmd.back_bitmap_id, move(back_shared_image)));
}
void allocate_backing_stores(UpdateBackingStoresCommand& cmd)
{
#ifdef USE_VULKAN_DMABUF_IMAGES
if (m_skia_backend_context && m_presents_to_client) {
auto backing_stores = create_linear_dmabuf_backing_stores(cmd.size, *m_skia_backend_context);
if (!backing_stores.is_error()) {
auto backing_store_pair = backing_stores.release_value();
m_backing_stores.front_store = move(backing_store_pair.front);
m_backing_stores.back_store = move(backing_store_pair.back);
publish_backing_store_pair(cmd, move(backing_store_pair.front_shared_image), move(backing_store_pair.back_shared_image));
return;
}
}
#endif
auto front_buffer = Gfx::SharedImageBuffer::create(cmd.size);
auto back_buffer = Gfx::SharedImageBuffer::create(cmd.size);
auto front_shared_image = front_buffer.export_shared_image();
auto back_shared_image = back_buffer.export_shared_image();
auto backing_store_pair = create_shareable_bitmap_backing_stores(cmd.size, front_buffer, back_buffer, m_skia_backend_context);
m_backing_stores.front_store = move(backing_store_pair.front);
m_backing_stores.back_store = move(backing_store_pair.back);
publish_backing_store_pair(cmd, move(front_shared_image), move(back_shared_image));
}
template<typename Invokee>
void invoke_on_main_thread(Invokee invokee)
{
if (m_exit)
return;
auto event_loop = m_main_thread_event_loop->take();
if (!event_loop)
return;
event_loop->deferred_invoke([self = NonnullRefPtr(*this), invokee = move(invokee)]() mutable {
invokee();
});
}
u64 m_page_id { 0 };
NonnullRefPtr<Core::WeakEventLoopReference> m_main_thread_event_loop;
bool m_presents_to_client { false };
mutable Sync::Mutex m_mutex;
mutable Sync::ConditionVariable m_command_ready { m_mutex };
Atomic<bool> m_exit { false };
Queue<CompositorCommand> m_command_queue;
OwnPtr<Painting::DisplayListPlayerSkia> m_skia_player;
RefPtr<Gfx::SkiaBackendContext> m_skia_backend_context;
RefPtr<Painting::DisplayList> m_cached_display_list;
Painting::ScrollStateSnapshot m_cached_scroll_state_snapshot;
BackingStoreState m_backing_stores;
CompositorThread::PresentationMode m_presentation_mode { CompositorThread::PresentToUI {} };
Atomic<i32> m_queued_rasterization_tasks { 0 };
Optional<i32> m_presented_bitmap_id_awaiting_ack;
mutable Sync::ConditionVariable m_ready_to_paint { m_mutex };
bool m_needs_present { false };
Gfx::IntRect m_pending_viewport_rect;
u64 m_submitted_frame_id { 0 };
u64 m_completed_frame_id { 0 };
mutable Sync::ConditionVariable m_frame_completed { m_mutex };
public:
void finish_rasterizing(i32 bitmap_id)
{
Sync::MutexLocker const locker { m_mutex };
VERIFY(!m_presented_bitmap_id_awaiting_ack.has_value());
m_presented_bitmap_id_awaiting_ack = bitmap_id;
m_queued_rasterization_tasks++;
VERIFY(m_queued_rasterization_tasks == 1);
}
void decrement_queued_tasks(i32 bitmap_id)
{
Sync::MutexLocker const locker { m_mutex };
if (m_presented_bitmap_id_awaiting_ack != bitmap_id)
return;
VERIFY(m_queued_rasterization_tasks == 1);
m_presented_bitmap_id_awaiting_ack.clear();
m_queued_rasterization_tasks--;
m_ready_to_paint.signal();
m_command_ready.signal();
}
};
struct FramePresentationState {
RefPtr<Core::WeakEventLoopReference> event_loop;
CompositorThread::BackingStorePresentationCallback backing_store_callback;
CompositorThread::FramePresentationCallback frame_callback;
};
static Sync::Mutex& compositor_presentation_state_mutex()
{
static NeverDestroyed<Sync::Mutex> mutex;
return *mutex;
}
static HashMap<u64, NonnullRefPtr<CompositorThread::ThreadData>>& page_compositors()
{
static NeverDestroyed<HashMap<u64, NonnullRefPtr<CompositorThread::ThreadData>>> compositors;
return *compositors;
}
static FramePresentationState& frame_presentation_state()
{
static NeverDestroyed<FramePresentationState> state;
return *state;
}
CompositorThread::CompositorThread(u64 page_id, PagePresentationRegistration page_presentation_registration)
: m_thread_data(adopt_ref(*new ThreadData(page_id, Core::EventLoop::current_weak(), page_presentation_registration)))
{
if (page_presentation_registration == PagePresentationRegistration::Yes)
register_page_compositor(page_id, m_thread_data);
}
CompositorThread::~CompositorThread()
{
unregister_page_compositor(m_thread_data->page_id(), *m_thread_data);
m_thread_data->exit();
}
void CompositorThread::register_page_compositor(u64 page_id, NonnullRefPtr<ThreadData> thread_data)
{
Sync::MutexLocker const locker { compositor_presentation_state_mutex() };
page_compositors().set(page_id, move(thread_data));
}
void CompositorThread::unregister_page_compositor(u64 page_id, ThreadData& thread_data)
{
Sync::MutexLocker const locker { compositor_presentation_state_mutex() };
auto compositor = page_compositors().find(page_id);
if (compositor == page_compositors().end())
return;
if (compositor->value.ptr() != &thread_data)
return;
page_compositors().remove(compositor);
}
void CompositorThread::set_frame_presentation_callbacks(NonnullRefPtr<Core::WeakEventLoopReference> event_loop, BackingStorePresentationCallback backing_store_callback, FramePresentationCallback frame_callback)
{
Sync::MutexLocker const locker { compositor_presentation_state_mutex() };
auto& state = frame_presentation_state();
state.event_loop = move(event_loop);
state.backing_store_callback = move(backing_store_callback);
state.frame_callback = move(frame_callback);
}
void CompositorThread::clear_frame_presentation_callbacks()
{
Sync::MutexLocker const locker { compositor_presentation_state_mutex() };
auto& state = frame_presentation_state();
state.event_loop = nullptr;
state.backing_store_callback = {};
state.frame_callback = {};
}
bool CompositorThread::present_backing_stores_to_client(u64 page_id, i32 front_bitmap_id, Gfx::SharedImage&& front_shared_image, i32 back_bitmap_id, Gfx::SharedImage&& back_shared_image)
{
RefPtr<Core::WeakEventLoopReference> event_loop_reference;
{
Sync::MutexLocker const locker { compositor_presentation_state_mutex() };
if (!page_compositors().contains(page_id))
return false;
auto& state = frame_presentation_state();
if (!state.backing_store_callback)
return false;
event_loop_reference = state.event_loop;
}
if (!event_loop_reference)
return false;
auto event_loop = event_loop_reference->take();
if (!event_loop)
return false;
event_loop->deferred_invoke([page_id, front_bitmap_id, front_shared_image = move(front_shared_image), back_bitmap_id, back_shared_image = move(back_shared_image)]() mutable {
Sync::MutexLocker const locker { compositor_presentation_state_mutex() };
if (!page_compositors().contains(page_id)) {
dbgln_if(COMPOSITOR_DEBUG, "[Compositor] Dropping queued UI backing stores for page {} front={} back={}: page unregistered",
page_id, front_bitmap_id, back_bitmap_id);
return;
}
auto& state = frame_presentation_state();
if (state.backing_store_callback)
state.backing_store_callback(page_id, front_bitmap_id, move(front_shared_image), back_bitmap_id, move(back_shared_image));
});
return true;
}
bool CompositorThread::present_frame_to_client(u64 page_id, Gfx::IntRect const& viewport_rect, i32 bitmap_id)
{
RefPtr<Core::WeakEventLoopReference> event_loop_reference;
{
Sync::MutexLocker const locker { compositor_presentation_state_mutex() };
if (!page_compositors().contains(page_id))
return false;
auto& state = frame_presentation_state();
if (!state.frame_callback)
return false;
event_loop_reference = state.event_loop;
}
if (!event_loop_reference)
return false;
auto event_loop = event_loop_reference->take();
if (!event_loop)
return false;
event_loop->deferred_invoke([page_id, viewport_rect, bitmap_id] {
Sync::MutexLocker const locker { compositor_presentation_state_mutex() };
if (!page_compositors().contains(page_id)) {
dbgln_if(COMPOSITOR_DEBUG, "[Compositor] Dropping queued UI present for page {} bitmap {}: page unregistered",
page_id, bitmap_id);
return;
}
auto& state = frame_presentation_state();
if (state.frame_callback)
state.frame_callback(page_id, viewport_rect, bitmap_id);
});
return true;
}
void CompositorThread::presented_bitmap_ready_to_paint(u64 page_id, i32 bitmap_id)
{
RefPtr<ThreadData> thread_data;
{
Sync::MutexLocker const locker { compositor_presentation_state_mutex() };
auto compositor = page_compositors().find(page_id);
if (compositor == page_compositors().end())
return;
thread_data = compositor->value;
}
thread_data->decrement_queued_tasks(bitmap_id);
}
void CompositorThread::start(DisplayListPlayerType display_list_player_type)
{
m_thread = Threading::Thread::construct("Compositor"sv, [thread_data = m_thread_data, display_list_player_type] {
thread_data->compositor_loop(display_list_player_type);
return static_cast<intptr_t>(0);
});
m_thread->start();
m_thread->detach();
}
void CompositorThread::set_presentation_mode(PresentationMode mode)
{
m_thread_data->set_presentation_mode(move(mode));
}
void CompositorThread::stop_presenting_to_client()
{
m_thread_data->stop_presenting_to_client();
unregister_page_compositor(m_thread_data->page_id(), *m_thread_data);
}
void CompositorThread::update_display_list(NonnullRefPtr<Painting::DisplayList> display_list, Painting::ScrollStateSnapshot&& scroll_state_snapshot)
{
m_thread_data->enqueue_command(UpdateDisplayListCommand { move(display_list), move(scroll_state_snapshot) });
}
void CompositorThread::update_scroll_state(Painting::ScrollStateSnapshot&& scroll_state_snapshot)
{
m_thread_data->enqueue_command(UpdateScrollStateCommand { move(scroll_state_snapshot) });
}
void CompositorThread::update_backing_stores(Gfx::IntSize size, i32 front_id, i32 back_id)
{
m_thread_data->enqueue_command(UpdateBackingStoresCommand { size, front_id, back_id });
}
u64 CompositorThread::present_frame(Gfx::IntRect viewport_rect)
{
return m_thread_data->set_needs_present(viewport_rect);
}
void CompositorThread::wait_for_frame(u64 frame_id)
{
m_thread_data->wait_for_frame(frame_id);
}
void CompositorThread::request_screenshot(NonnullRefPtr<Gfx::PaintingSurface> target_surface, Function<void()>&& callback)
{
m_thread_data->enqueue_command(ScreenshotCommand { move(target_surface), move(callback) });
}
}