LibIPC: Adopt Mach OOL payloads on receive

Mach transport already sends payloads as out-of-line virtual-copy
regions, but the receive path immediately copied each payload into a new
Vector and deallocated the kernel mapping. That made the IPC IO thread
touch every byte before the main thread could decode the message.

Add ReceivedMessageBytes as the raw-message byte storage and let the
Mach transport adopt the OOL region directly. The mapping now lives
until the raw message storage is destroyed, so invalid descriptor paths
and normal queue teardown both release it through the same destructor.
Socket transports keep their existing receive copy path by wrapping
vectors in the same storage type, and the direct raw-message consumers
now decode from its ReadonlyBytes view.
This commit is contained in:
Aliaksandr Kalenik 2026-06-12 16:30:54 +02:00 committed by Alexander Kalenik
parent f0ed472429
commit 60f27523c6
11 changed files with 162 additions and 17 deletions

View file

@ -5,6 +5,7 @@ set(SOURCES
Encoder.cpp
File.cpp
Message.cpp
ReceivedMessageBytes.cpp
TransportHandle.cpp
)

View file

@ -97,10 +97,11 @@ ConnectionBase::PeerEOF ConnectionBase::drain_messages_from_peer()
{
bool parse_error = false;
auto schedule_shutdown = m_transport->read_as_many_messages_as_possible_without_blocking([&](auto&& raw_message) {
if (auto message = try_parse_message(raw_message.bytes, raw_message.attachments)) {
auto bytes = raw_message.bytes.bytes();
if (auto message = try_parse_message(bytes, raw_message.attachments)) {
m_unprocessed_messages.append(message.release_nonnull());
} else {
dbgln("Failed to parse IPC message {:hex-dump}", raw_message.bytes);
dbgln("Failed to parse IPC message {:hex-dump}", bytes);
parse_error = true;
}
});

View file

@ -0,0 +1,100 @@
/*
* Copyright (c) 2026, Aliaksandr Kalenik <kalenik.aliaksandr@gmail.com>
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#include <AK/RefCounted.h>
#include <LibIPC/ReceivedMessageBytes.h>
#if defined(AK_OS_MACOS)
# include <mach/mach.h>
#endif
namespace IPC {
class ReceivedMessageBytes::Impl final : public RefCounted<ReceivedMessageBytes::Impl> {
public:
explicit Impl(Vector<u8> bytes)
: m_storage_type(StorageType::Vector)
, m_vector(move(bytes))
{
}
#if defined(AK_OS_MACOS)
Impl(void* vm_region_address, size_t vm_region_size)
: m_storage_type(StorageType::VMRegion)
, m_vm_region_address(vm_region_address)
, m_vm_region_size(vm_region_size)
{
}
#endif
~Impl()
{
#if defined(AK_OS_MACOS)
if (m_storage_type == StorageType::VMRegion && m_vm_region_size > 0)
vm_deallocate(mach_task_self(), reinterpret_cast<vm_address_t>(m_vm_region_address), m_vm_region_size);
#endif
}
ReadonlyBytes bytes() const
{
switch (m_storage_type) {
case StorageType::Vector:
return m_vector;
case StorageType::VMRegion:
return { static_cast<u8 const*>(m_vm_region_address), m_vm_region_size };
}
VERIFY_NOT_REACHED();
}
private:
enum class StorageType {
Vector,
VMRegion,
};
StorageType m_storage_type { StorageType::Vector };
Vector<u8> m_vector;
void* m_vm_region_address { nullptr };
size_t m_vm_region_size { 0 };
};
ReceivedMessageBytes::ReceivedMessageBytes() = default;
ReceivedMessageBytes::ReceivedMessageBytes(ReceivedMessageBytes const&) = default;
ReceivedMessageBytes::ReceivedMessageBytes(ReceivedMessageBytes&&) = default;
ReceivedMessageBytes::~ReceivedMessageBytes() = default;
ReceivedMessageBytes& ReceivedMessageBytes::operator=(ReceivedMessageBytes const&) = default;
ReceivedMessageBytes& ReceivedMessageBytes::operator=(ReceivedMessageBytes&&) = default;
ReceivedMessageBytes::ReceivedMessageBytes(NonnullRefPtr<Impl> impl)
: m_impl(move(impl))
{
}
ReceivedMessageBytes ReceivedMessageBytes::from_vector(Vector<u8> bytes)
{
if (bytes.is_empty())
return {};
return ReceivedMessageBytes { adopt_ref(*new Impl(move(bytes))) };
}
#if defined(AK_OS_MACOS)
ReceivedMessageBytes ReceivedMessageBytes::adopt_vm_region(void* address, size_t size)
{
if (size == 0)
return {};
return ReceivedMessageBytes { adopt_ref(*new Impl(address, size)) };
}
#endif
ReadonlyBytes ReceivedMessageBytes::bytes() const
{
if (!m_impl)
return {};
return m_impl->bytes();
}
}

View file

@ -0,0 +1,41 @@
/*
* Copyright (c) 2026, Aliaksandr Kalenik <kalenik.aliaksandr@gmail.com>
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#pragma once
#include <AK/Platform.h>
#include <AK/RefPtr.h>
#include <AK/Vector.h>
namespace IPC {
class ReceivedMessageBytes {
public:
ReceivedMessageBytes();
ReceivedMessageBytes(ReceivedMessageBytes const&);
ReceivedMessageBytes(ReceivedMessageBytes&&);
~ReceivedMessageBytes();
ReceivedMessageBytes& operator=(ReceivedMessageBytes const&);
ReceivedMessageBytes& operator=(ReceivedMessageBytes&&);
static ReceivedMessageBytes from_vector(Vector<u8>);
#if defined(AK_OS_MACOS)
static ReceivedMessageBytes adopt_vm_region(void*, size_t);
#endif
ReadonlyBytes bytes() const;
bool is_empty() const { return bytes().is_empty(); }
private:
class Impl;
explicit ReceivedMessageBytes(NonnullRefPtr<Impl>);
RefPtr<Impl> m_impl;
};
}

View file

@ -297,27 +297,22 @@ void TransportMachPort::process_received_message(u8* buffer)
auto* descriptors = reinterpret_cast<mach_msg_port_descriptor_t*>(body + 1);
auto const* payload = reinterpret_cast<mach_msg_ool_descriptor_t const*>(&descriptors[attachment_count]);
VERIFY(payload->type == MACH_MSG_OOL_DESCRIPTOR);
auto payload_bytes = ReceivedMessageBytes::adopt_vm_region(payload->address, payload->size);
auto message = make<Message>();
for (unsigned int i = 0; i < attachment_count; ++i) {
auto attachment = attachment_from_descriptor(descriptors[i]);
if (attachment.is_error()) {
dbgln("TransportMachPort: dropping message with invalid port descriptor {} of {}: {} (name={:x}, disposition={}, type={})",
i + 1, attachment_count, attachment.error(), descriptors[i].name, descriptors[i].disposition, descriptors[i].type);
if (payload->type == MACH_MSG_OOL_DESCRIPTOR && payload->size > 0)
vm_deallocate(mach_task_self(), reinterpret_cast<vm_address_t>(payload->address), payload->size);
mark_peer_eof();
return;
}
message->attachments.enqueue(attachment.release_value());
}
VERIFY(payload->type == MACH_MSG_OOL_DESCRIPTOR);
if (payload->size > 0) {
message->bytes.append(static_cast<u8 const*>(payload->address), payload->size);
// The out-of-line payload arrives as a temporary mapping in this task. After copying it into our queue,
// unmap that region so each receive does not leak virtual memory.
vm_deallocate(mach_task_self(), reinterpret_cast<vm_address_t>(payload->address), payload->size);
}
message->bytes = move(payload_bytes);
if (message->bytes.is_empty() && message->attachments.is_empty())
return;

View file

@ -21,6 +21,7 @@
#include <LibIPC/Attachment.h>
#include <LibIPC/AutoCloseFileDescriptor.h>
#include <LibIPC/Forward.h>
#include <LibIPC/ReceivedMessageBytes.h>
#include <LibIPC/TransportHandle.h>
#include <LibSync/ConditionVariable.h>
#include <LibSync/Mutex.h>
@ -57,7 +58,7 @@ public:
Yes,
};
struct Message {
Vector<u8> bytes;
ReceivedMessageBytes bytes;
Queue<Attachment> attachments;
};
ShouldShutdown read_as_many_messages_as_possible_without_blocking(Function<void(Message&&)>&&);

View file

@ -448,11 +448,13 @@ void TransportSocket::read_incoming_messages()
}
for (size_t i = 0; i < header.fd_count; ++i)
message->attachments.enqueue(m_unprocessed_attachments.dequeue());
if (message->bytes.try_append(m_unprocessed_bytes.data() + index + sizeof(SocketMessageHeader), header.payload_size).is_error()) {
Vector<u8> payload_bytes;
if (payload_bytes.try_append(m_unprocessed_bytes.data() + index + sizeof(SocketMessageHeader), header.payload_size).is_error()) {
dbgln("TransportSocket: Failed to allocate message buffer for payload_size {}", header.payload_size);
m_peer_eof = true;
break;
}
message->bytes = ReceivedMessageBytes::from_vector(move(payload_bytes));
batch.append(move(message));
} else if (header.type == SocketMessageHeader::Type::FileDescriptorAcknowledgement) {
if (header.payload_size != 0) {

View file

@ -14,6 +14,7 @@
#include <LibIPC/Attachment.h>
#include <LibIPC/AutoCloseFileDescriptor.h>
#include <LibIPC/Forward.h>
#include <LibIPC/ReceivedMessageBytes.h>
#include <LibIPC/TransportHandle.h>
#include <LibSync/ConditionVariable.h>
#include <LibSync/Mutex.h>
@ -88,7 +89,7 @@ public:
Yes,
};
struct Message {
Vector<u8> bytes;
ReceivedMessageBytes bytes;
Queue<Attachment> attachments;
};
ShouldShutdown read_as_many_messages_as_possible_without_blocking(Function<void(Message&&)>&&);

View file

@ -296,11 +296,13 @@ TransportSocketWindows::ShouldShutdown TransportSocketWindows::read_as_many_mess
VERIFY(attachment_bytes.is_empty());
auto const* payload = m_unprocessed_bytes.data() + index + sizeof(MessageHeader) + header.attachment_data_size;
if (message.bytes.try_append(payload, header.payload_size).is_error()) {
Vector<u8> payload_bytes;
if (payload_bytes.try_append(payload, header.payload_size).is_error()) {
dbgln("TransportSocketWindows: Failed to allocate message buffer for payload_size {}", header.payload_size);
should_shutdown = ShouldShutdown::Yes;
break;
}
message.bytes = ReceivedMessageBytes::from_vector(move(payload_bytes));
callback(move(message));
Checked<size_t> new_index = index;
new_index += header.payload_size;

View file

@ -11,6 +11,7 @@
#include <LibCore/Socket.h>
#include <LibIPC/Attachment.h>
#include <LibIPC/Forward.h>
#include <LibIPC/ReceivedMessageBytes.h>
#include <LibIPC/TransportHandle.h>
namespace IPC {
@ -44,7 +45,7 @@ public:
Yes,
};
struct Message {
Vector<u8> bytes;
ReceivedMessageBytes bytes;
Queue<Attachment> attachments;
};
ShouldShutdown read_as_many_messages_as_possible_without_blocking(Function<void(Message&&)>&&);

View file

@ -351,7 +351,7 @@ void MessagePort::drain_transport()
return;
auto schedule_shutdown = m_transport->read_as_many_messages_as_possible_without_blocking([this](auto&& raw_message) {
FixedMemoryStream stream { raw_message.bytes.span(), FixedMemoryStream::Mode::ReadOnly };
FixedMemoryStream stream { raw_message.bytes.bytes() };
IPC::Decoder decoder { stream, raw_message.attachments };
m_pending_incoming_messages.append(MUST(decoder.decode<SerializedTransferRecord>()));