From b5bace2391ce6eebde31f765eb2b1412d93a2154 Mon Sep 17 00:00:00 2001 From: Aliaksandr Kalenik Date: Wed, 20 May 2026 15:01:36 +0200 Subject: [PATCH] LibGfx+LibMedia: Add IPC serialization for VideoFrame Compositor video frame transport should not flatten frames into a shareable bitmap before crossing an IPC boundary. That conversion loses the native YUV representation and makes Web-side transport code own the frame wire format. Teach VideoFrame to encode its YUV planes into a shared anonymous buffer with the color space, timing, subsampling, bit depth, and CICP metadata needed to rebuild the frame on decode. Add YUVData helpers for checked plane sizing and construction from validated plane bytes so malformed buffers are rejected before a frame is created. This is preparatory work required to add IPC between the main and compositor threads. --- Libraries/LibGfx/YUVData.cpp | 79 ++++++++++++++++++--- Libraries/LibGfx/YUVData.h | 12 ++++ Libraries/LibMedia/VideoFrame.cpp | 114 ++++++++++++++++++++++++++++++ Libraries/LibMedia/VideoFrame.h | 15 ++++ 4 files changed, 210 insertions(+), 10 deletions(-) diff --git a/Libraries/LibGfx/YUVData.cpp b/Libraries/LibGfx/YUVData.cpp index 11605b1f85..d56f31b4a9 100644 --- a/Libraries/LibGfx/YUVData.cpp +++ b/Libraries/LibGfx/YUVData.cpp @@ -4,6 +4,7 @@ * SPDX-License-Identifier: BSD-2-Clause */ +#include #include #include #include @@ -35,19 +36,49 @@ struct YUVDataImpl { } +static ErrorOr checked_plane_size(Gfx::IntSize size, size_t component_size) +{ + Checked plane_size = static_cast(size.width()); + plane_size *= static_cast(size.height()); + plane_size *= component_size; + if (plane_size.has_overflow()) + return Error::from_string_literal("YUVData plane size overflow"); + return plane_size.value(); +} + +ErrorOr YUVData::plane_sizes(IntSize size, u8 bit_depth, Media::Subsampling subsampling) +{ + if (size.is_empty()) + return Error::from_string_literal("YUVData size is empty"); + if (bit_depth == 0 || bit_depth > 16) + return Error::from_string_literal("Invalid YUVData bit depth"); + + auto component_size = bit_depth <= 8 ? 1 : 2; + auto y_buffer_size = TRY(checked_plane_size(size, component_size)); + auto uv_size = subsampling.subsampled_size(size); + auto uv_buffer_size = TRY(checked_plane_size(uv_size, component_size)); + + Checked total_size = y_buffer_size; + total_size += uv_buffer_size; + total_size += uv_buffer_size; + if (total_size.has_overflow()) + return Error::from_string_literal("YUVData total size overflow"); + + return PlaneSizes { + .y = y_buffer_size, + .u = uv_buffer_size, + .v = uv_buffer_size, + .total = total_size.value(), + }; +} + ErrorOr> YUVData::create(IntSize size, u8 bit_depth, Media::Subsampling subsampling, Media::CodingIndependentCodePoints cicp) { - VERIFY(bit_depth <= 16); - auto component_size = bit_depth <= 8 ? 1 : 2; + auto sizes = TRY(plane_sizes(size, bit_depth, subsampling)); - auto y_buffer_size = static_cast(size.width()) * size.height() * component_size; - - auto uv_size = subsampling.subsampled_size(size); - auto uv_buffer_size = static_cast(uv_size.width()) * uv_size.height() * component_size; - - auto y_buffer = TRY(FixedArray::create(y_buffer_size)); - auto u_buffer = TRY(FixedArray::create(uv_buffer_size)); - auto v_buffer = TRY(FixedArray::create(uv_buffer_size)); + auto y_buffer = TRY(FixedArray::create(sizes.y)); + auto u_buffer = TRY(FixedArray::create(sizes.u)); + auto v_buffer = TRY(FixedArray::create(sizes.v)); auto impl = TRY(try_make(Details::YUVDataImpl { .size = size, @@ -62,6 +93,19 @@ ErrorOr> YUVData::create(IntSize size, u8 bit_depth, Medi return adopt_nonnull_own_or_enomem(new (nothrow) YUVData(move(impl))); } +ErrorOr> YUVData::create_from_data(IntSize size, u8 bit_depth, Media::Subsampling subsampling, Media::CodingIndependentCodePoints cicp, ReadonlyBytes y_data, ReadonlyBytes u_data, ReadonlyBytes v_data) +{ + auto sizes = TRY(plane_sizes(size, bit_depth, subsampling)); + if (y_data.size() != sizes.y || u_data.size() != sizes.u || v_data.size() != sizes.v) + return Error::from_string_literal("YUVData plane data size mismatch"); + + auto yuv_data = TRY(create(size, bit_depth, subsampling, cicp)); + y_data.copy_to(yuv_data->y_data()); + u_data.copy_to(yuv_data->u_data()); + v_data.copy_to(yuv_data->v_data()); + return yuv_data; +} + YUVData::YUVData(NonnullOwnPtr impl) : m_impl(move(impl)) { @@ -104,6 +148,21 @@ Bytes YUVData::v_data() return m_impl->v_buffer.span(); } +ReadonlyBytes YUVData::y_data() const +{ + return m_impl->y_buffer.span(); +} + +ReadonlyBytes YUVData::u_data() const +{ + return m_impl->u_buffer.span(); +} + +ReadonlyBytes YUVData::v_data() const +{ + return m_impl->v_buffer.span(); +} + static FFI::YUVMatrix yuv_matrix_for_cicp(Media::CodingIndependentCodePoints const& cicp) { switch (cicp.matrix_coefficients()) { diff --git a/Libraries/LibGfx/YUVData.h b/Libraries/LibGfx/YUVData.h index 2c5da3c408..f42e226bbf 100644 --- a/Libraries/LibGfx/YUVData.h +++ b/Libraries/LibGfx/YUVData.h @@ -30,7 +30,16 @@ struct YUVDataImpl; // Not ref-counted - owned directly by decoded video frame objects via NonnullOwnPtr. class YUVData final { public: + struct PlaneSizes { + size_t y; + size_t u; + size_t v; + size_t total; + }; + + static ErrorOr plane_sizes(IntSize size, u8 bit_depth, Media::Subsampling); static ErrorOr> create(IntSize size, u8 bit_depth, Media::Subsampling, Media::CodingIndependentCodePoints); + static ErrorOr> create_from_data(IntSize size, u8 bit_depth, Media::Subsampling, Media::CodingIndependentCodePoints, ReadonlyBytes y_data, ReadonlyBytes u_data, ReadonlyBytes v_data); ~YUVData(); @@ -43,6 +52,9 @@ public: Bytes y_data(); Bytes u_data(); Bytes v_data(); + ReadonlyBytes y_data() const; + ReadonlyBytes u_data() const; + ReadonlyBytes v_data() const; ErrorOr> to_bitmap() const; diff --git a/Libraries/LibMedia/VideoFrame.cpp b/Libraries/LibMedia/VideoFrame.cpp index eb5636bca4..ea54f51d0f 100644 --- a/Libraries/LibMedia/VideoFrame.cpp +++ b/Libraries/LibMedia/VideoFrame.cpp @@ -5,9 +5,14 @@ */ #include +#include +#include #include "VideoFrame.h" +#include +#include + namespace Media { VideoFrame::VideoFrame( @@ -29,3 +34,112 @@ VideoFrame::VideoFrame( VideoFrame::~VideoFrame() = default; } + +namespace IPC { + +static bool color_primaries_ipc_value_valid(Media::ColorPrimaries color_primaries) +{ + return color_primaries == Media::ColorPrimaries::Unspecified + || Media::color_primaries_valid(color_primaries); +} + +static bool transfer_characteristics_ipc_value_valid(Media::TransferCharacteristics transfer_characteristics) +{ + return transfer_characteristics == Media::TransferCharacteristics::Unspecified + || Media::transfer_characteristics_valid(transfer_characteristics); +} + +static bool matrix_coefficients_ipc_value_valid(Media::MatrixCoefficients matrix_coefficients) +{ + return matrix_coefficients == Media::MatrixCoefficients::Unspecified + || Media::matrix_coefficients_valid(matrix_coefficients); +} + +static bool video_full_range_flag_ipc_value_valid(Media::VideoFullRangeFlag video_full_range_flag) +{ + return video_full_range_flag == Media::VideoFullRangeFlag::Unspecified + || Media::video_full_range_flag_valid(video_full_range_flag); +} + +static ErrorOr encode_yuv_data(Gfx::YUVData const& yuv_data) +{ + auto sizes = TRY(Gfx::YUVData::plane_sizes(yuv_data.size(), yuv_data.bit_depth(), yuv_data.subsampling())); + auto buffer = TRY(Core::AnonymousBuffer::create_with_size(sizes.total)); + + auto bytes = Bytes { buffer.data(), buffer.size() }; + yuv_data.y_data().copy_to(bytes.slice(0, sizes.y)); + yuv_data.u_data().copy_to(bytes.slice(sizes.y, sizes.u)); + yuv_data.v_data().copy_to(bytes.slice(sizes.y + sizes.u, sizes.v)); + return buffer; +} + +template<> +ErrorOr encode(Encoder& encoder, Media::VideoFrame const& frame) +{ + auto const& yuv_data = frame.yuv_data(); + auto yuv_data_buffer = TRY(encode_yuv_data(yuv_data)); + TRY(encoder.encode(yuv_data_buffer)); + TRY(encoder.encode(frame.color_space())); + TRY(encoder.encode(frame.timestamp())); + TRY(encoder.encode(frame.duration())); + TRY(encoder.encode(yuv_data.size())); + TRY(encoder.encode(yuv_data.bit_depth())); + TRY(encoder.encode(yuv_data.subsampling().x())); + TRY(encoder.encode(yuv_data.subsampling().y())); + TRY(encoder.encode(yuv_data.cicp().color_primaries())); + TRY(encoder.encode(yuv_data.cicp().transfer_characteristics())); + TRY(encoder.encode(yuv_data.cicp().matrix_coefficients())); + TRY(encoder.encode(yuv_data.cicp().video_full_range_flag())); + return {}; +} + +template<> +ErrorOr encode(Encoder& encoder, NonnullRefPtr const& frame) +{ + return encoder.encode(*frame); +} + +template<> +ErrorOr> decode(Decoder& decoder) +{ + auto yuv_data_buffer = TRY(decoder.decode()); + if (!yuv_data_buffer.is_valid()) + return Error::from_string_literal("IPC: VideoFrame contained invalid YUV data"); + + auto color_space = TRY(decoder.decode()); + auto timestamp = TRY(decoder.decode()); + auto duration = TRY(decoder.decode()); + auto size = TRY(decoder.decode()); + auto bit_depth = TRY(decoder.decode()); + auto subsampling = Media::Subsampling { + TRY(decoder.decode()), + TRY(decoder.decode()), + }; + auto cicp = Media::CodingIndependentCodePoints { + TRY(decoder.decode()), + TRY(decoder.decode()), + TRY(decoder.decode()), + TRY(decoder.decode()), + }; + + if (!color_primaries_ipc_value_valid(cicp.color_primaries()) + || !transfer_characteristics_ipc_value_valid(cicp.transfer_characteristics()) + || !matrix_coefficients_ipc_value_valid(cicp.matrix_coefficients()) + || !video_full_range_flag_ipc_value_valid(cicp.video_full_range_flag())) + return Error::from_string_literal("IPC: VideoFrame contained invalid CICP metadata"); + + auto sizes = TRY(Gfx::YUVData::plane_sizes(size, bit_depth, subsampling)); + if (yuv_data_buffer.size() != sizes.total) + return Error::from_string_literal("IPC: VideoFrame contained invalid YUV data size"); + + auto bytes = yuv_data_buffer.bytes(); + auto y_data = bytes.slice(0, sizes.y); + auto u_data = bytes.slice(sizes.y, sizes.u); + auto v_data = bytes.slice(sizes.y + sizes.u, sizes.v); + + auto yuv_data = TRY(Gfx::YUVData::create_from_data(size, bit_depth, subsampling, cicp, y_data, u_data, v_data)); + auto frame = TRY(try_make_ref_counted(timestamp, duration, size.to_type(), bit_depth, move(color_space), move(yuv_data))); + return NonnullRefPtr { *frame }; +} + +} diff --git a/Libraries/LibMedia/VideoFrame.h b/Libraries/LibMedia/VideoFrame.h index 78f26e5ddb..d6e6d4b4d4 100644 --- a/Libraries/LibMedia/VideoFrame.h +++ b/Libraries/LibMedia/VideoFrame.h @@ -7,11 +7,14 @@ #pragma once #include +#include #include +#include #include #include #include #include +#include #include namespace Media { @@ -50,3 +53,15 @@ private: }; } + +namespace IPC { + +template<> +MEDIA_API ErrorOr encode(Encoder&, Media::VideoFrame const&); +template<> +MEDIA_API ErrorOr encode(Encoder&, NonnullRefPtr const&); + +template<> +MEDIA_API ErrorOr> decode(Decoder&); + +}