Start adding infrastructure to retrieve video frames as texture

We're using RHI here, as that's what we need for Qt Quick anyway.
You can now set a QRhi object on QVideoSink. This can then be
used internally to create textures of the required format instead
of memory buffers.

QVideoSurfaceFormat will now tell you how many planes are required
for the format, and you can retrieve the textures for each plane
from QVideoFrame.

Change-Id: I86430db60a8f1aba07ec3b38e22b977cdaefaa0a
Reviewed-by: Doris Verria <doris.verria@qt.io>
Reviewed-by: Lars Knoll <lars.knoll@qt.io>

7 files changed, 82 insertions, 18 deletions

diff --git a/src/multimedia/video/qabstractvideobuffer_p.h b/src/multimedia/video/qabstractvideobuffer_p.h
index 3ab1b60e4..e3a0fb86c 100644
--- a/src/multimedia/video/qabstractvideobuffer_p.h
+++ b/src/multimedia/video/qabstractvideobuffer_p.h
@@ -82,6 +82,7 @@ public:
     virtual void unmap() = 0;
 
     virtual QVariant handle() const;
+    virtual quint64 textureHandle(int /*plane*/) const { return 0; }
 
 protected:
     QVideoFrame::HandleType m_type;
diff --git a/src/multimedia/video/qvideoframe.cpp b/src/multimedia/video/qvideoframe.cpp
index 0ff9d2ace..316366d53 100644
--- a/src/multimedia/video/qvideoframe.cpp
+++ b/src/multimedia/video/qvideoframe.cpp
@@ -633,7 +633,11 @@ bool QVideoFrame::map(QVideoFrame::MapMode mode)
         case QVideoSurfaceFormat::Format_NV12:
         case QVideoSurfaceFormat::Format_NV21:
         case QVideoSurfaceFormat::Format_IMC2:
-        case QVideoSurfaceFormat::Format_IMC4: {
+        case QVideoSurfaceFormat::Format_IMC4:
+        case QVideoSurfaceFormat::Format_P010BE:
+        case QVideoSurfaceFormat::Format_P010LE:
+        case QVideoSurfaceFormat::Format_P016BE:
+        case QVideoSurfaceFormat::Format_P016LE: {
             // Semi planar, Full resolution Y plane with interleaved subsampled U and V planes.
             d->mapData.nPlanes = 2;
             d->mapData.bytesPerLine[1] = d->mapData.bytesPerLine[0];
@@ -650,8 +654,6 @@ bool QVideoFrame::map(QVideoFrame::MapMode mode)
             d->mapData.data[2] = d->mapData.data[1] + (d->mapData.bytesPerLine[1] * height() / 2);
             break;
         }
-        default:
-            break;
         }
     }
 
@@ -797,15 +799,18 @@ int QVideoFrame::mappedBytes() const
 /*!
     Returns the number of planes in the video frame.
 
-    This value is only valid while the frame data is \l {map()}{mapped}.
-
-    \sa map()
+    \sa map(), textureHandle()
     \since 5.4
 */
 
 int QVideoFrame::planeCount() const
 {
-    return d->mapData.nPlanes;
+    return d->format.nPlanes();
+}
+
+quint64 QVideoFrame::textureHandle(int plane)
+{
+    return d->buffer->textureHandle(plane);
 }
 
 /*!
@@ -982,10 +987,8 @@ QDebug operator<<(QDebug dbg, QVideoFrame::HandleType type)
     switch (type) {
     case QVideoFrame::NoHandle:
         return dbg << "NoHandle";
-    case QVideoFrame::GLTextureHandle:
-        return dbg << "GLTextureHandle";
-    case QVideoFrame::MTLTextureHandle:
-        return dbg << "MTLTextureHandle";
+    case QVideoFrame::RhiTextureHandle:
+        return dbg << "RhiTextureHandle";
     }
 }
 
diff --git a/src/multimedia/video/qvideoframe.h b/src/multimedia/video/qvideoframe.h
index c3eba5585..5e90b6ca1 100644
--- a/src/multimedia/video/qvideoframe.h
+++ b/src/multimedia/video/qvideoframe.h
@@ -61,8 +61,7 @@ public:
     enum HandleType
     {
         NoHandle,
-        GLTextureHandle,
-        MTLTextureHandle,
+        RhiTextureHandle
     };
 
     enum MapMode
@@ -114,6 +113,8 @@ public:
     int mappedBytes() const;
     int planeCount() const;
 
+    quint64 textureHandle(int plane);
+
     QVariant handle() const;
 
     qint64 startTime() const;
diff --git a/src/multimedia/video/qvideosink.cpp b/src/multimedia/video/qvideosink.cpp
index 639d00583..f5112b173 100644
--- a/src/multimedia/video/qvideosink.cpp
+++ b/src/multimedia/video/qvideosink.cpp
@@ -69,6 +69,7 @@ public:
     QSize nativeResolution;
     bool active = false;
     WId window = 0;
+    QRhi *rhi = nullptr;
     Qt::AspectRatioMode aspectRatioMode = Qt::KeepAspectRatio;
     QRectF targetRect;
     int brightness = 0;
@@ -118,6 +119,16 @@ void QVideoSink::setNativeWindowId(WId id)
     d->videoSink->setWinId(id);
 }
 
+QRhi *QVideoSink::rhi() const
+{
+    return d->rhi;
+}
+
+void QVideoSink::setRhi(QRhi *rhi)
+{
+    d->rhi = rhi;
+}
+
 void QVideoSink::setFullScreen(bool fullscreen)
 {
     d->videoSink->setFullScreen(fullscreen);
diff --git a/src/multimedia/video/qvideosink.h b/src/multimedia/video/qvideosink.h
index ec89c425d..f94a43d2e 100644
--- a/src/multimedia/video/qvideosink.h
+++ b/src/multimedia/video/qvideosink.h
@@ -52,6 +52,7 @@ class QVideoFrame;
 
 class QVideoSinkPrivate;
 class QPlatformVideoSink;
+class QRhi;
 
 class Q_MULTIMEDIA_EXPORT QVideoSink : public QObject
 {
@@ -61,11 +62,7 @@ public:
     {
         Memory,
         NativeWindow,
-        NativeTexture,
-        OpenGL,
-        Metal,
-        Direct3D11,
-        Vulkan
+        RhiTexture
     };
 
     QVideoSink(QObject *parent = nullptr);
@@ -80,6 +77,9 @@ public:
     WId nativeWindowId() const;
     void setNativeWindowId(WId id);
 
+    QRhi *rhi() const;
+    void setRhi(QRhi *rhi);
+
     void setFullScreen(bool fullscreen);
     bool isFullscreen() const;
 
diff --git a/src/multimedia/video/qvideosurfaceformat.cpp b/src/multimedia/video/qvideosurfaceformat.cpp
index 3ba143079..ad83aa954 100644
--- a/src/multimedia/video/qvideosurfaceformat.cpp
+++ b/src/multimedia/video/qvideosurfaceformat.cpp
@@ -251,6 +251,52 @@ int QVideoSurfaceFormat::frameHeight() const
     return d->frameSize.height();
 }
 
+int QVideoSurfaceFormat::nPlanes() const
+{
+    switch (d->pixelFormat) {
+    case Format_Invalid:
+    case Format_ARGB32:
+    case Format_ARGB32_Premultiplied:
+    case Format_RGB32:
+    case Format_RGB24:
+    case Format_RGB565:
+    case Format_RGB555:
+    case Format_ARGB8565_Premultiplied:
+    case Format_BGRA32:
+    case Format_BGRA32_Premultiplied:
+    case Format_ABGR32:
+    case Format_BGR32:
+    case Format_BGR24:
+    case Format_BGR565:
+    case Format_BGR555:
+    case Format_BGRA5658_Premultiplied:
+    case Format_AYUV444:
+    case Format_AYUV444_Premultiplied:
+    case Format_YUV444:
+    case Format_UYVY:
+    case Format_YUYV:
+    case Format_Y8:
+    case Format_Y16:
+    case Format_Jpeg:
+        return 1;
+    case Format_NV12:
+    case Format_NV21:
+    case Format_IMC2:
+    case Format_IMC4:
+    case Format_P010LE:
+    case Format_P010BE:
+    case Format_P016LE:
+    case Format_P016BE:
+        return 2;
+    case Format_YUV420P:
+    case Format_YUV422P:
+    case Format_YV12:
+    case Format_IMC1:
+    case Format_IMC3:
+        return 3;
+    }
+}
+
 /*!
     Sets the size of frames in a video stream to \a size.
 
diff --git a/src/multimedia/video/qvideosurfaceformat.h b/src/multimedia/video/qvideosurfaceformat.h
index 5b9c9fcb8..837d9545a 100644
--- a/src/multimedia/video/qvideosurfaceformat.h
+++ b/src/multimedia/video/qvideosurfaceformat.h
@@ -144,6 +144,8 @@ public:
     int frameWidth() const;
     int frameHeight() const;
 
+    int nPlanes() const;
+
     QRect viewport() const;
     void setViewport(const QRect &viewport);