LibGfx+LibMedia: Add IPC serialization for VideoFrame

Compositor video frame transport should not flatten frames into a
shareable bitmap before crossing an IPC boundary. That conversion loses
the native YUV representation and makes Web-side transport code own the
frame wire format.

Teach VideoFrame to encode its YUV planes into a shared anonymous buffer
with the color space, timing, subsampling, bit depth, and CICP metadata
needed to rebuild the frame on decode. Add YUVData helpers for checked
plane sizing and construction from validated plane bytes so malformed
buffers are rejected before a frame is created.

This is preparatory work required to add IPC between the main and
compositor threads.

Libraries/LibGfx/YUVData.cpp

@@ -4,6 +4,7 @@
  * SPDX-License-Identifier: BSD-2-Clause
  */
 
+#include <AK/Checked.h>
 #include <AK/Time.h>
 #include <LibGfx/Bitmap.h>
 #include <LibGfx/SkiaBackendContext.h>
@@ -35,19 +36,49 @@ struct YUVDataImpl {
 
 }
 
+static ErrorOr<size_t> checked_plane_size(Gfx::IntSize size, size_t component_size)
+{
+    Checked<size_t> plane_size = static_cast<size_t>(size.width());
+    plane_size *= static_cast<size_t>(size.height());
+    plane_size *= component_size;
+    if (plane_size.has_overflow())
+        return Error::from_string_literal("YUVData plane size overflow");
+    return plane_size.value();
+}
+
+ErrorOr<YUVData::PlaneSizes> YUVData::plane_sizes(IntSize size, u8 bit_depth, Media::Subsampling subsampling)
+{
+    if (size.is_empty())
+        return Error::from_string_literal("YUVData size is empty");
+    if (bit_depth == 0 || bit_depth > 16)
+        return Error::from_string_literal("Invalid YUVData bit depth");
+
+    auto component_size = bit_depth <= 8 ? 1 : 2;
+    auto y_buffer_size = TRY(checked_plane_size(size, component_size));
+    auto uv_size = subsampling.subsampled_size(size);
+    auto uv_buffer_size = TRY(checked_plane_size(uv_size, component_size));
+
+    Checked<size_t> total_size = y_buffer_size;
+    total_size += uv_buffer_size;
+    total_size += uv_buffer_size;
+    if (total_size.has_overflow())
+        return Error::from_string_literal("YUVData total size overflow");
+
+    return PlaneSizes {
+        .y = y_buffer_size,
+        .u = uv_buffer_size,
+        .v = uv_buffer_size,
+        .total = total_size.value(),
+    };
+}
+
 ErrorOr<NonnullOwnPtr<YUVData>> YUVData::create(IntSize size, u8 bit_depth, Media::Subsampling subsampling, Media::CodingIndependentCodePoints cicp)
 {
-    VERIFY(bit_depth <= 16);
-    auto component_size = bit_depth <= 8 ? 1 : 2;
+    auto sizes = TRY(plane_sizes(size, bit_depth, subsampling));
 
-    auto y_buffer_size = static_cast<size_t>(size.width()) * size.height() * component_size;
-
-    auto uv_size = subsampling.subsampled_size(size);
-    auto uv_buffer_size = static_cast<size_t>(uv_size.width()) * uv_size.height() * component_size;
-
-    auto y_buffer = TRY(FixedArray<u8>::create(y_buffer_size));
-    auto u_buffer = TRY(FixedArray<u8>::create(uv_buffer_size));
-    auto v_buffer = TRY(FixedArray<u8>::create(uv_buffer_size));
+    auto y_buffer = TRY(FixedArray<u8>::create(sizes.y));
+    auto u_buffer = TRY(FixedArray<u8>::create(sizes.u));
+    auto v_buffer = TRY(FixedArray<u8>::create(sizes.v));
 
     auto impl = TRY(try_make<Details::YUVDataImpl>(Details::YUVDataImpl {
         .size = size,
@@ -62,6 +93,19 @@ ErrorOr<NonnullOwnPtr<YUVData>> YUVData::create(IntSize size, u8 bit_depth, Medi
     return adopt_nonnull_own_or_enomem(new (nothrow) YUVData(move(impl)));
 }
 
+ErrorOr<NonnullOwnPtr<YUVData>> YUVData::create_from_data(IntSize size, u8 bit_depth, Media::Subsampling subsampling, Media::CodingIndependentCodePoints cicp, ReadonlyBytes y_data, ReadonlyBytes u_data, ReadonlyBytes v_data)
+{
+    auto sizes = TRY(plane_sizes(size, bit_depth, subsampling));
+    if (y_data.size() != sizes.y || u_data.size() != sizes.u || v_data.size() != sizes.v)
+        return Error::from_string_literal("YUVData plane data size mismatch");
+
+    auto yuv_data = TRY(create(size, bit_depth, subsampling, cicp));
+    y_data.copy_to(yuv_data->y_data());
+    u_data.copy_to(yuv_data->u_data());
+    v_data.copy_to(yuv_data->v_data());
+    return yuv_data;
+}
+
 YUVData::YUVData(NonnullOwnPtr<Details::YUVDataImpl> impl)
     : m_impl(move(impl))
 {
@@ -104,6 +148,21 @@ Bytes YUVData::v_data()
     return m_impl->v_buffer.span();
 }
 
+ReadonlyBytes YUVData::y_data() const
+{
+    return m_impl->y_buffer.span();
+}
+
+ReadonlyBytes YUVData::u_data() const
+{
+    return m_impl->u_buffer.span();
+}
+
+ReadonlyBytes YUVData::v_data() const
+{
+    return m_impl->v_buffer.span();
+}
+
 static FFI::YUVMatrix yuv_matrix_for_cicp(Media::CodingIndependentCodePoints const& cicp)
 {
     switch (cicp.matrix_coefficients()) {

Libraries/LibGfx/YUVData.h

@@ -30,7 +30,16 @@ struct YUVDataImpl;
 // Not ref-counted - owned directly by decoded video frame objects via NonnullOwnPtr.
 class YUVData final {
 public:
+    struct PlaneSizes {
+        size_t y;
+        size_t u;
+        size_t v;
+        size_t total;
+    };
+
+    static ErrorOr<PlaneSizes> plane_sizes(IntSize size, u8 bit_depth, Media::Subsampling);
     static ErrorOr<NonnullOwnPtr<YUVData>> create(IntSize size, u8 bit_depth, Media::Subsampling, Media::CodingIndependentCodePoints);
+    static ErrorOr<NonnullOwnPtr<YUVData>> create_from_data(IntSize size, u8 bit_depth, Media::Subsampling, Media::CodingIndependentCodePoints, ReadonlyBytes y_data, ReadonlyBytes u_data, ReadonlyBytes v_data);
 
     ~YUVData();
 
@@ -43,6 +52,9 @@ public:
     Bytes y_data();
     Bytes u_data();
     Bytes v_data();
+    ReadonlyBytes y_data() const;
+    ReadonlyBytes u_data() const;
+    ReadonlyBytes v_data() const;
 
     ErrorOr<NonnullRefPtr<Bitmap>> to_bitmap() const;
 

Libraries/LibMedia/VideoFrame.cpp

@@ -5,9 +5,14 @@
  */
 
 #include <LibGfx/YUVData.h>
+#include <LibIPC/Decoder.h>
+#include <LibIPC/Encoder.h>
 
 #include "VideoFrame.h"
 
+#include <LibCore/AnonymousBuffer.h>
+#include <LibMedia/Color/CodingIndependentCodePoints.h>
+
 namespace Media {
 
 VideoFrame::VideoFrame(
@@ -29,3 +34,112 @@ VideoFrame::VideoFrame(
 VideoFrame::~VideoFrame() = default;
 
 }
+
+namespace IPC {
+
+static bool color_primaries_ipc_value_valid(Media::ColorPrimaries color_primaries)
+{
+    return color_primaries == Media::ColorPrimaries::Unspecified
+        || Media::color_primaries_valid(color_primaries);
+}
+
+static bool transfer_characteristics_ipc_value_valid(Media::TransferCharacteristics transfer_characteristics)
+{
+    return transfer_characteristics == Media::TransferCharacteristics::Unspecified
+        || Media::transfer_characteristics_valid(transfer_characteristics);
+}
+
+static bool matrix_coefficients_ipc_value_valid(Media::MatrixCoefficients matrix_coefficients)
+{
+    return matrix_coefficients == Media::MatrixCoefficients::Unspecified
+        || Media::matrix_coefficients_valid(matrix_coefficients);
+}
+
+static bool video_full_range_flag_ipc_value_valid(Media::VideoFullRangeFlag video_full_range_flag)
+{
+    return video_full_range_flag == Media::VideoFullRangeFlag::Unspecified
+        || Media::video_full_range_flag_valid(video_full_range_flag);
+}
+
+static ErrorOr<Core::AnonymousBuffer> encode_yuv_data(Gfx::YUVData const& yuv_data)
+{
+    auto sizes = TRY(Gfx::YUVData::plane_sizes(yuv_data.size(), yuv_data.bit_depth(), yuv_data.subsampling()));
+    auto buffer = TRY(Core::AnonymousBuffer::create_with_size(sizes.total));
+
+    auto bytes = Bytes { buffer.data<u8>(), buffer.size() };
+    yuv_data.y_data().copy_to(bytes.slice(0, sizes.y));
+    yuv_data.u_data().copy_to(bytes.slice(sizes.y, sizes.u));
+    yuv_data.v_data().copy_to(bytes.slice(sizes.y + sizes.u, sizes.v));
+    return buffer;
+}
+
+template<>
+ErrorOr<void> encode(Encoder& encoder, Media::VideoFrame const& frame)
+{
+    auto const& yuv_data = frame.yuv_data();
+    auto yuv_data_buffer = TRY(encode_yuv_data(yuv_data));
+    TRY(encoder.encode(yuv_data_buffer));
+    TRY(encoder.encode(frame.color_space()));
+    TRY(encoder.encode(frame.timestamp()));
+    TRY(encoder.encode(frame.duration()));
+    TRY(encoder.encode(yuv_data.size()));
+    TRY(encoder.encode(yuv_data.bit_depth()));
+    TRY(encoder.encode(yuv_data.subsampling().x()));
+    TRY(encoder.encode(yuv_data.subsampling().y()));
+    TRY(encoder.encode(yuv_data.cicp().color_primaries()));
+    TRY(encoder.encode(yuv_data.cicp().transfer_characteristics()));
+    TRY(encoder.encode(yuv_data.cicp().matrix_coefficients()));
+    TRY(encoder.encode(yuv_data.cicp().video_full_range_flag()));
+    return {};
+}
+
+template<>
+ErrorOr<void> encode(Encoder& encoder, NonnullRefPtr<Media::VideoFrame const> const& frame)
+{
+    return encoder.encode(*frame);
+}
+
+template<>
+ErrorOr<NonnullRefPtr<Media::VideoFrame const>> decode(Decoder& decoder)
+{
+    auto yuv_data_buffer = TRY(decoder.decode<Core::AnonymousBuffer>());
+    if (!yuv_data_buffer.is_valid())
+        return Error::from_string_literal("IPC: VideoFrame contained invalid YUV data");
+
+    auto color_space = TRY(decoder.decode<Gfx::ColorSpace>());
+    auto timestamp = TRY(decoder.decode<AK::Duration>());
+    auto duration = TRY(decoder.decode<AK::Duration>());
+    auto size = TRY(decoder.decode<Gfx::IntSize>());
+    auto bit_depth = TRY(decoder.decode<u8>());
+    auto subsampling = Media::Subsampling {
+        TRY(decoder.decode<bool>()),
+        TRY(decoder.decode<bool>()),
+    };
+    auto cicp = Media::CodingIndependentCodePoints {
+        TRY(decoder.decode<Media::ColorPrimaries>()),
+        TRY(decoder.decode<Media::TransferCharacteristics>()),
+        TRY(decoder.decode<Media::MatrixCoefficients>()),
+        TRY(decoder.decode<Media::VideoFullRangeFlag>()),
+    };
+
+    if (!color_primaries_ipc_value_valid(cicp.color_primaries())
+        || !transfer_characteristics_ipc_value_valid(cicp.transfer_characteristics())
+        || !matrix_coefficients_ipc_value_valid(cicp.matrix_coefficients())
+        || !video_full_range_flag_ipc_value_valid(cicp.video_full_range_flag()))
+        return Error::from_string_literal("IPC: VideoFrame contained invalid CICP metadata");
+
+    auto sizes = TRY(Gfx::YUVData::plane_sizes(size, bit_depth, subsampling));
+    if (yuv_data_buffer.size() != sizes.total)
+        return Error::from_string_literal("IPC: VideoFrame contained invalid YUV data size");
+
+    auto bytes = yuv_data_buffer.bytes();
+    auto y_data = bytes.slice(0, sizes.y);
+    auto u_data = bytes.slice(sizes.y, sizes.u);
+    auto v_data = bytes.slice(sizes.y + sizes.u, sizes.v);
+
+    auto yuv_data = TRY(Gfx::YUVData::create_from_data(size, bit_depth, subsampling, cicp, y_data, u_data, v_data));
+    auto frame = TRY(try_make_ref_counted<Media::VideoFrame>(timestamp, duration, size.to_type<u32>(), bit_depth, move(color_space), move(yuv_data)));
+    return NonnullRefPtr<Media::VideoFrame const> { *frame };
+}
+
+}

Libraries/LibMedia/VideoFrame.h

@@ -7,11 +7,14 @@
 #pragma once
 
 #include <AK/AtomicRefCounted.h>
+#include <AK/Error.h>
 #include <AK/NonnullOwnPtr.h>
+#include <AK/NonnullRefPtr.h>
 #include <AK/Time.h>
 #include <LibGfx/ColorSpace.h>
 #include <LibGfx/Forward.h>
 #include <LibGfx/Size.h>
+#include <LibIPC/Forward.h>
 #include <LibMedia/Export.h>
 
 namespace Media {
@@ -50,3 +53,15 @@ private:
 };
 
 }
+
+namespace IPC {
+
+template<>
+MEDIA_API ErrorOr<void> encode(Encoder&, Media::VideoFrame const&);
+template<>
+MEDIA_API ErrorOr<void> encode(Encoder&, NonnullRefPtr<Media::VideoFrame const> const&);
+
+template<>
+MEDIA_API ErrorOr<NonnullRefPtr<Media::VideoFrame const>> decode(Decoder&);
+
+}