Quick: Add support of RHI for video frames

Added RHI shaders for RGB*, YUV*  and opengl texture video frames.

Task-number: QTBUG-78678
Change-Id: I045d6a806fea059a80b8e5d9817b6997af8d0f41
Reviewed-by: Laszlo Agocs <laszlo.agocs@qt.io>

1 files changed, 220 insertions, 2 deletions

diff --git a/src/qtmultimediaquicktools/qsgvideonode_yuv.cpp b/src/qtmultimediaquicktools/qsgvideonode_yuv.cpp
index 2b50e9cb0..006704a23 100644
--- a/src/qtmultimediaquicktools/qsgvideonode_yuv.cpp
+++ b/src/qtmultimediaquicktools/qsgvideonode_yuv.cpp
@@ -37,6 +37,7 @@
 **
 ****************************************************************************/
 #include "qsgvideonode_yuv_p.h"
+#include "qsgvideotexture_p.h"
 #include <QtCore/qmutex.h>
 #include <QtQuick/qsgtexturematerial.h>
 #include <QtQuick/qsgmaterial.h>
@@ -181,7 +182,6 @@ public:
     }
 };
 
-
 class QSGVideoMaterialShader_YUV_BiPlanar_swizzle : public QSGVideoMaterialShader_YUV_BiPlanar
 {
 public:
@@ -192,7 +192,6 @@ public:
     }
 };
 
-
 class QSGVideoMaterialShader_YUV_TriPlanar : public QSGVideoMaterialShader_YUV_BiPlanar
 {
 public:
@@ -216,6 +215,77 @@ protected:
     int m_id_plane3Texture;
 };
 
+class QSGVideoMaterialRhiShader_YUV : public QSGMaterialRhiShader
+{
+public:
+    QSGVideoMaterialRhiShader_YUV()
+    {
+        setShaderFileName(VertexStage, QStringLiteral(":/qtmultimediaquicktools/shaders_ng/yuv.vert.qsb"));
+    }
+
+    bool updateUniformData(RenderState &state, QSGMaterial *newMaterial,
+                           QSGMaterial *oldMaterial) override;
+
+    void updateSampledImage(RenderState &state, int binding, QSGTexture **texture,
+                            QSGMaterial *newMaterial, QSGMaterial *oldMaterial) override;
+
+    virtual void mapFrame(QSGVideoMaterial_YUV *) = 0;
+
+protected:
+    GLfloat m_planeWidth[3] = {0, 0, 0};
+    QMatrix4x4 m_colorMatrix;
+};
+
+class QSGVideoMaterialRhiShader_UYVY : public QSGVideoMaterialRhiShader_YUV
+{
+public:
+    QSGVideoMaterialRhiShader_UYVY()
+    {
+        setShaderFileName(FragmentStage, QStringLiteral(":/qtmultimediaquicktools/shaders_ng/uyvy.frag.qsb"));
+    }
+
+    void mapFrame(QSGVideoMaterial_YUV *m) override;
+};
+
+class QSGVideoMaterialRhiShader_YUYV : public QSGVideoMaterialRhiShader_UYVY
+{
+public:
+    QSGVideoMaterialRhiShader_YUYV()
+    {
+        setShaderFileName(FragmentStage, QStringLiteral(":/qtmultimediaquicktools/shaders_ng/yuyv.frag.qsb"));
+    }
+};
+
+class QSGVideoMaterialRhiShader_YUV_YV : public QSGVideoMaterialRhiShader_YUV
+{
+public:
+    QSGVideoMaterialRhiShader_YUV_YV()
+    {
+        setShaderFileName(FragmentStage, QStringLiteral(":/qtmultimediaquicktools/shaders_ng/yuv_yv.frag.qsb"));
+    }
+
+    void mapFrame(QSGVideoMaterial_YUV *m) override;
+};
+
+class QSGVideoMaterialRhiShader_NV12 : public QSGVideoMaterialRhiShader_YUV
+{
+public:
+    QSGVideoMaterialRhiShader_NV12()
+    {
+        setShaderFileName(FragmentStage, QStringLiteral(":/qtmultimediaquicktools/shaders_ng/nv12.frag.qsb"));
+    }
+
+    void mapFrame(QSGVideoMaterial_YUV *m) override;
+};
+
+class QSGVideoMaterialRhiShader_NV21 : public QSGVideoMaterialRhiShader_NV12
+{
+public:
+    QSGVideoMaterialRhiShader_NV21()
+    {
+        setShaderFileName(FragmentStage, QStringLiteral(":/qtmultimediaquicktools/shaders_ng/nv21.frag.qsb"));
+    }
+};
 
 class QSGVideoMaterial_YUV : public QSGMaterial
 {
@@ -241,6 +311,21 @@ public:
     }
 
     QSGMaterialShader *createShader() const override {
+        if (flags().testFlag(RhiShaderWanted)) {
+            switch (m_format.pixelFormat()) {
+            case QVideoFrame::Format_NV12:
+                return new QSGVideoMaterialRhiShader_NV12;
+            case QVideoFrame::Format_NV21:
+                return new QSGVideoMaterialRhiShader_NV21;
+            case QVideoFrame::Format_UYVY:
+                return new QSGVideoMaterialRhiShader_UYVY;
+            case QVideoFrame::Format_YUYV:
+                return new QSGVideoMaterialRhiShader_YUYV;
+            default: // Currently: YUV420P, YUV422P and YV12
+                return new QSGVideoMaterialRhiShader_YUV_YV;
+            }
+        }
+
         switch (m_format.pixelFormat()) {
         case QVideoFrame::Format_NV12:
             return new QSGVideoMaterialShader_YUV_BiPlanar;
@@ -293,12 +378,145 @@ public:
 
     QVideoFrame m_frame;
     QMutex m_frameMutex;
+
+    QScopedPointer<QSGVideoTexture> m_textures[3];
 };
 
+bool QSGVideoMaterialRhiShader_YUV::updateUniformData(RenderState &state, QSGMaterial *newMaterial,
+                                                      QSGMaterial *oldMaterial)
+{
+    Q_UNUSED(oldMaterial);
+
+    auto m = static_cast<QSGVideoMaterial_YUV *>(newMaterial);
+    bool changed = false;
+    QByteArray *buf = state.uniformData();
+
+    if (state.isMatrixDirty()) {
+        memcpy(buf->data(), state.combinedMatrix().constData(), 64);
+        changed = true;
+    }
+
+    if (m->m_colorMatrix != m_colorMatrix) {
+        memcpy(buf->data() + 64, m->m_colorMatrix.constData(), 64);
+        changed = true;
+    }
+    m_colorMatrix = m->m_colorMatrix;
+
+    if (state.isOpacityDirty()) {
+        const float opacity = state.opacity();
+        memcpy(buf->data() + 64 + 64, &opacity, 4);
+        changed = true;
+    }
+
+    if (!m->m_textures[0]) {
+        m->m_textures[0].reset(new QSGVideoTexture);
+        m->m_textures[1].reset(new QSGVideoTexture);
+    }
+
+    mapFrame(m);
+
+    if (m->m_planeWidth[0] != m_planeWidth[0]
+        || m->m_planeWidth[1] != m_planeWidth[1]
+        || m->m_planeWidth[2] != m_planeWidth[2])
+    {
+        memcpy(buf->data() + 64 + 64 + 4, &m->m_planeWidth[0], 4);
+        memcpy(buf->data() + 64 + 64 + 4 + 4, &m->m_planeWidth[1], 4);
+        memcpy(buf->data() + 64 + 64 + 4 + 4 + 4, &m->m_planeWidth[2], 4);
+        changed = true;
+    }
+    m_planeWidth[0] = m->m_planeWidth[0];
+    m_planeWidth[1] = m->m_planeWidth[1];
+    m_planeWidth[2] = m->m_planeWidth[2];
+
+    return changed;
+}
+
+void QSGVideoMaterialRhiShader_YUV::updateSampledImage(RenderState &state, int binding, QSGTexture **texture,
+                                                       QSGMaterial *newMaterial, QSGMaterial *oldMaterial)
+{
+    Q_UNUSED(oldMaterial);
+    if (binding < 1 || binding > 3)
+        return;
+
+    auto m = static_cast<QSGVideoMaterial_YUV *>(newMaterial);
+    *texture = m->m_textures[binding - 1].data();
+    (*texture)->commitTextureOperations(state.rhi(), state.resourceUpdateBatch());
+}
+
+void QSGVideoMaterialRhiShader_UYVY::mapFrame(QSGVideoMaterial_YUV *m)
+{
+    if (!m->m_frame.isValid() || !m->m_frame.map(QAbstractVideoBuffer::ReadOnly))
+        return;
+
+    int fw = m->m_frame.width();
+    int fh = m->m_frame.height();
+
+    m->m_planeWidth[0] = 1;
+    m->m_planeWidth[1] = 1;
+
+    // Either r,b (YUYV) or g,a (UYVY) values are used as source of UV.
+    // Additionally U and V are set per 2 pixels hence only 1/2 of image width is used.
+    m->m_textures[0]->setData(QRhiTexture::RG8, m->m_frame.size(),
+        m->m_frame.bits(), m->m_frame.bytesPerLine() * fh);
+    m->m_textures[1]->setData(QRhiTexture::RGBA8, QSize(fw / 2, fh),
+        m->m_frame.bits(), m->m_frame.bytesPerLine() * fh);
+
+    m->m_frame.unmap();
+}
+
+void QSGVideoMaterialRhiShader_YUV_YV::mapFrame(QSGVideoMaterial_YUV *m)
+{
+    if (!m->m_frame.isValid() || !m->m_frame.map(QAbstractVideoBuffer::ReadOnly))
+        return;
+
+    if (!m->m_textures[2])
+        m->m_textures[2].reset(new QSGVideoTexture);
+
+    int y = 0;
+    int u = m->m_frame.pixelFormat() == QVideoFrame::Format_YV12 ? 2 : 1;
+    int v = m->m_frame.pixelFormat() == QVideoFrame::Format_YV12 ? 1 : 2;
+    int fw = m->m_frame.width();
+    int fh = m->m_frame.height();
+    int uvHeight = m->m_frame.pixelFormat() == QVideoFrame::Format_YUV422P ? fh : fh / 2;
+
+    m->m_planeWidth[0] = qreal(fw) / m->m_frame.bytesPerLine(y);
+    m->m_planeWidth[1] = m->m_planeWidth[2] = qreal(fw) / (2 * m->m_frame.bytesPerLine(u));
+
+    m->m_textures[0]->setData(QRhiTexture::R8, m->m_frame.size(),
+        m->m_frame.bits(y), m->m_frame.bytesPerLine(y) * fh);
+    m->m_textures[1]->setData(QRhiTexture::R8, QSize(m->m_frame.bytesPerLine(u), uvHeight),
+        m->m_frame.bits(u), m->m_frame.bytesPerLine(u) * uvHeight);
+    m->m_textures[2]->setData(QRhiTexture::R8, QSize(m->m_frame.bytesPerLine(v), uvHeight),
+        m->m_frame.bits(v), m->m_frame.bytesPerLine(v) * uvHeight);
+
+    m->m_frame.unmap();
+}
+
+void QSGVideoMaterialRhiShader_NV12::mapFrame(QSGVideoMaterial_YUV *m)
+{
+    if (!m->m_frame.isValid() || !m->m_frame.map(QAbstractVideoBuffer::ReadOnly))
+        return;
+
+    int y = 0;
+    int uv = 1;
+    int fw = m->m_frame.width();
+    int fh = m->m_frame.height();
+
+    m->m_planeWidth[0] = m->m_planeWidth[1] = qreal(fw) / m->m_frame.bytesPerLine(y);
+
+    m->m_textures[0]->setData(QRhiTexture::R8, m->m_frame.size(),
+        m->m_frame.bits(y), m->m_frame.bytesPerLine(y) * fh);
+    m->m_textures[1]->setData(QRhiTexture::RG8, QSize(m->m_frame.bytesPerLine(uv) / 2 , fh / 2),
+        m->m_frame.bits(uv), m->m_frame.bytesPerLine(uv) * fh / 2);
+
+    m->m_frame.unmap();
+}
+
 QSGVideoMaterial_YUV::QSGVideoMaterial_YUV(const QVideoSurfaceFormat &format) :
     m_format(format),
     m_opacity(1.0)
 {
+    setFlag(SupportsRhiShader, true);
     memset(m_textureIds, 0, sizeof(m_textureIds));
 
     switch (format.pixelFormat()) {