Use AVPlayerItemVideoOutput to generate video frames

This fixes rendering problems on M1 based Macs.

It also unifies the rendering pipeline between macOS and iOS as
much as possible, and avoids an intermediate copy to an FBO,

Since AVPlayerItemVideoOutput produces GL_TEXTURE_RECTANGLE
textures on macOS a new QAbstractVideoBuffer handle has been
added that explicitly maps to GL_TEXTURE_RECTANGLE.

We use this handle type internally in QSGVideoMaterial_Texture
where we know how to blit GL_TEXTURE_RECTANGLE textures.

To maintain compatibility for QAbstractVideoSurface consumers who
expect GL_TEXTURE_2D textures we blit the rectangle texture to
an FBO returned as QAbstractVideoBuffer::GLTextureHandle.

Fixes: QTBUG-89803
Done-with: Lars Knoll <lars.knoll@qt.io>
Change-Id: I36d22eafb63902ecc1097e138705812ef6a8cb71
Reviewed-by: Lars Knoll <lars.knoll@qt.io>
Reviewed-by: Doris Verria <doris.verria@qt.io>

6 files changed, 165 insertions, 26 deletions

diff --git a/src/qtmultimediaquicktools/qsgvideonode_texture.cpp b/src/qtmultimediaquicktools/qsgvideonode_texture.cpp
index 589b1359a..65c57f903 100644
--- a/src/qtmultimediaquicktools/qsgvideonode_texture.cpp
+++ b/src/qtmultimediaquicktools/qsgvideonode_texture.cpp
@@ -53,6 +53,13 @@ QList<QVideoFrame::PixelFormat> QSGVideoNodeFactory_Texture::supportedPixelForma
 {
     QList<QVideoFrame::PixelFormat> pixelFormats;
 
+#ifdef Q_OS_MACOS
+    if (handleType == QAbstractVideoBuffer::GLTextureRectangleHandle) {
+        pixelFormats.append(QVideoFrame::Format_BGR32);
+        pixelFormats.append(QVideoFrame::Format_BGRA32);
+    }
+#endif
+
     if (handleType == QAbstractVideoBuffer::GLTextureHandle) {
         pixelFormats.append(QVideoFrame::Format_RGB565);
         pixelFormats.append(QVideoFrame::Format_RGB32);
@@ -82,8 +89,6 @@ public:
     QSGVideoMaterialShader_Texture()
         : QSGMaterialShader()
     {
-        setShaderSourceFile(QOpenGLShader::Vertex, QStringLiteral(":/qtmultimediaquicktools/shaders/monoplanarvideo.vert"));
-        setShaderSourceFile(QOpenGLShader::Fragment, QStringLiteral(":/qtmultimediaquicktools/shaders/rgbvideo.frag"));
     }
 
     void updateState(const RenderState &state, QSGMaterial *newMaterial, QSGMaterial *oldMaterial) override;
@@ -109,10 +114,20 @@ protected:
     int m_id_opacity;
 };
 
-class QSGVideoMaterialShader_Texture_swizzle : public QSGVideoMaterialShader_Texture
+class QSGVideoMaterialShader_Texture_2D : public QSGVideoMaterialShader_Texture
 {
 public:
-    QSGVideoMaterialShader_Texture_swizzle(bool hasAlpha)
+    QSGVideoMaterialShader_Texture_2D()
+    {
+        setShaderSourceFile(QOpenGLShader::Vertex, QStringLiteral(":/qtmultimediaquicktools/shaders/monoplanarvideo.vert"));
+        setShaderSourceFile(QOpenGLShader::Fragment, QStringLiteral(":/qtmultimediaquicktools/shaders/rgbvideo.frag"));
+    }
+};
+
+class QSGVideoMaterialShader_Texture_2D_swizzle : public QSGVideoMaterialShader_Texture_2D
+{
+public:
+    QSGVideoMaterialShader_Texture_2D_swizzle(bool hasAlpha)
         : m_hasAlpha(hasAlpha)
     {
         setShaderSourceFile(QOpenGLShader::Fragment, QStringLiteral(":/qtmultimediaquicktools/shaders/rgbvideo_swizzle.frag"));
@@ -120,14 +135,13 @@ public:
 
 protected:
     void initialize() override {
-        QSGVideoMaterialShader_Texture::initialize();
+        QSGVideoMaterialShader_Texture_2D::initialize();
         program()->setUniformValue(program()->uniformLocation("hasAlpha"), GLboolean(m_hasAlpha));
     }
 
     int m_hasAlpha;
 };
 
-
 class QSGVideoMaterial_Texture : public QSGMaterial
 {
 public:
@@ -149,12 +163,6 @@ public:
         return needsSwizzling() ? &swizzleType : &normalType;
     }
 
-    QSGMaterialShader *createShader() const override {
-        const bool hasAlpha = m_format.pixelFormat() == QVideoFrame::Format_ARGB32;
-        return needsSwizzling() ? new QSGVideoMaterialShader_Texture_swizzle(hasAlpha)
-                                : new QSGVideoMaterialShader_Texture;
-    }
-
     int compare(const QSGMaterial *other) const override {
         const QSGVideoMaterial_Texture *m = static_cast<const QSGVideoMaterial_Texture *>(other);
 
@@ -179,9 +187,42 @@ public:
     void setVideoFrame(const QVideoFrame &frame) {
         QMutexLocker lock(&m_frameMutex);
         m_frame = frame;
+        m_textureSize = frame.size();
+    }
+
+    virtual void bind() = 0;
+
+    QVideoFrame m_frame;
+    QMutex m_frameMutex;
+    QSize m_textureSize;
+    QVideoSurfaceFormat m_format;
+    GLuint m_textureId;
+    qreal m_opacity;
+
+protected:
+    bool needsSwizzling() const {
+        return !QMediaOpenGLHelper::isANGLE()
+                && (m_format.pixelFormat() == QVideoFrame::Format_RGB32
+                    || m_format.pixelFormat() == QVideoFrame::Format_ARGB32);
+    }
+};
+
+class QSGVideoMaterial_Texture_2D : public QSGVideoMaterial_Texture
+{
+public:
+    QSGVideoMaterial_Texture_2D(const QVideoSurfaceFormat &format) :
+        QSGVideoMaterial_Texture(format)
+    {
     }
 
-    void bind()
+    QSGMaterialShader *createShader() const override
+    {
+        const bool hasAlpha = m_format.pixelFormat() == QVideoFrame::Format_ARGB32;
+        return needsSwizzling() ? new QSGVideoMaterialShader_Texture_2D_swizzle(hasAlpha)
+                                : new QSGVideoMaterialShader_Texture_2D;
+    }
+
+    void bind() override
     {
         QMutexLocker lock(&m_frameMutex);
         if (m_frame.isValid()) {
@@ -197,28 +238,84 @@ public:
             m_textureId = 0;
         }
     }
+};
+
+#ifdef Q_OS_MACOS
+class QSGVideoMaterialShader_Texture_Rectangle : public QSGVideoMaterialShader_Texture
+{
+public:
+    QSGVideoMaterialShader_Texture_Rectangle()
+    {
+        setShaderSourceFile(QOpenGLShader::Vertex, QStringLiteral(":/qtmultimediaquicktools/shaders/rectsampler.vert"));
+        setShaderSourceFile(QOpenGLShader::Fragment, QStringLiteral(":/qtmultimediaquicktools/shaders/rectsampler_rgb.frag"));
+    }
 
-    QVideoFrame m_frame;
-    QMutex m_frameMutex;
-    QSize m_textureSize;
-    QVideoSurfaceFormat m_format;
-    GLuint m_textureId;
-    qreal m_opacity;
+    void updateState(const RenderState &state, QSGMaterial *newMaterial, QSGMaterial *oldMaterial) override
+    {
+        QSGVideoMaterial_Texture *mat = static_cast<QSGVideoMaterial_Texture *>(newMaterial);
+        QVector2D size(mat->m_textureSize.width(), mat->m_textureSize.height());
+        program()->setUniformValue(m_id_videoSize, size);
 
-private:
-    bool needsSwizzling() const {
-        return !QMediaOpenGLHelper::isANGLE()
-                && (m_format.pixelFormat() == QVideoFrame::Format_RGB32
-                    || m_format.pixelFormat() == QVideoFrame::Format_ARGB32);
+        QSGVideoMaterialShader_Texture::updateState(state, newMaterial, oldMaterial);
     }
+
+protected:
+    void initialize() override
+    {
+        QSGVideoMaterialShader_Texture::initialize();
+        m_id_videoSize = program()->uniformLocation("qt_videoSize");
+    }
+
+    int m_id_videoSize;
 };
 
+class QSGVideoMaterial_Texture_Rectangle : public QSGVideoMaterial_Texture
+{
+public:
+    QSGVideoMaterial_Texture_Rectangle(const QVideoSurfaceFormat &format) :
+        QSGVideoMaterial_Texture(format)
+    {
+    }
+
+    QSGMaterialShader *createShader() const override
+    {
+        Q_ASSERT(!needsSwizzling());
+        return new QSGVideoMaterialShader_Texture_Rectangle;
+    }
+
+    void bind() override
+    {
+        QMutexLocker lock(&m_frameMutex);
+        if (m_frame.isValid()) {
+            m_textureId = m_frame.handle().toUInt();
+            QOpenGLFunctions *functions = QOpenGLContext::currentContext()->functions();
+            functions->glActiveTexture(GL_TEXTURE0);
+            functions->glBindTexture(GL_TEXTURE_RECTANGLE, m_textureId);
+
+            functions->glTexParameteri(GL_TEXTURE_RECTANGLE, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
+            functions->glTexParameteri(GL_TEXTURE_RECTANGLE, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
+            functions->glTexParameteri(GL_TEXTURE_RECTANGLE, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
+            functions->glTexParameteri(GL_TEXTURE_RECTANGLE, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
+        } else {
+            m_textureId = 0;
+        }
+    }
+};
+#endif
 
 QSGVideoNode_Texture::QSGVideoNode_Texture(const QVideoSurfaceFormat &format) :
     m_format(format)
 {
     setFlag(QSGNode::OwnsMaterial);
-    m_material = new QSGVideoMaterial_Texture(format);
+
+#ifdef Q_OS_MACOS
+    if (format.handleType() == QAbstractVideoBuffer::GLTextureRectangleHandle)
+        m_material = new QSGVideoMaterial_Texture_Rectangle(format);
+#endif
+
+    if (!m_material)
+        m_material = new QSGVideoMaterial_Texture_2D(format);
+
     setMaterial(m_material);
 }
 
diff --git a/src/qtmultimediaquicktools/qtmultimediaquicktools.qrc b/src/qtmultimediaquicktools/qtmultimediaquicktools.qrc
index b8180e31f..86523e771 100644
--- a/src/qtmultimediaquicktools/qtmultimediaquicktools.qrc
+++ b/src/qtmultimediaquicktools/qtmultimediaquicktools.qrc
@@ -11,7 +11,6 @@
         <file>shaders/triplanaryuvvideo.vert</file>
         <file>shaders/uyvyvideo.frag</file>
         <file>shaders/yuyvvideo.frag</file>
-
         <file>shaders/monoplanarvideo_core.vert</file>
         <file>shaders/rgbvideo_core.frag</file>
         <file>shaders/rgbvideo_swizzle_core.frag</file>
@@ -23,5 +22,9 @@
         <file>shaders/triplanaryuvvideo_core.vert</file>
         <file>shaders/uyvyvideo_core.frag</file>
         <file>shaders/yuyvvideo_core.frag</file>
+        <file>shaders/rectsampler.vert</file>
+        <file>shaders/rectsampler_rgb.frag</file>
+        <file>shaders/rectsampler_core.vert</file>
+        <file>shaders/rectsampler_rgb_core.frag</file>
     </qresource>
 </RCC>
diff --git a/src/qtmultimediaquicktools/shaders/rectsampler.vert b/src/qtmultimediaquicktools/shaders/rectsampler.vert
new file mode 100644
index 000000000..762ec7e7e
--- /dev/null
+++ b/src/qtmultimediaquicktools/shaders/rectsampler.vert
@@ -0,0 +1,10 @@
+uniform highp mat4 qt_Matrix;
+uniform highp vec2 qt_videoSize;
+attribute highp vec4 qt_VertexPosition;
+attribute highp vec2 qt_VertexTexCoord;
+varying highp vec2 qt_TexCoord;
+
+void main() {
+    qt_TexCoord = vec2(qt_VertexTexCoord.x * qt_videoSize.x, qt_VertexTexCoord.y * qt_videoSize.y);
+    gl_Position = qt_Matrix * qt_VertexPosition;
+}
diff --git a/src/qtmultimediaquicktools/shaders/rectsampler_core.vert b/src/qtmultimediaquicktools/shaders/rectsampler_core.vert
new file mode 100644
index 000000000..f0fe02349
--- /dev/null
+++ b/src/qtmultimediaquicktools/shaders/rectsampler_core.vert
@@ -0,0 +1,11 @@
+#version 150 core
+uniform highp mat4 qt_Matrix;
+uniform highp vec2 qt_videoSize;
+in highp vec4 qt_VertexPosition;
+in highp vec2 qt_VertexTexCoord;
+out highp vec2 qt_TexCoord;
+
+void main() {
+    qt_TexCoord = vec2(qt_VertexTexCoord.x * qt_videoSize.x, qt_VertexTexCoord.y * qt_videoSize.y);
+    gl_Position = qt_Matrix * qt_VertexPosition;
+}
diff --git a/src/qtmultimediaquicktools/shaders/rectsampler_rgb.frag b/src/qtmultimediaquicktools/shaders/rectsampler_rgb.frag
new file mode 100644
index 000000000..2a30f7c3d
--- /dev/null
+++ b/src/qtmultimediaquicktools/shaders/rectsampler_rgb.frag
@@ -0,0 +1,8 @@
+uniform sampler2DRect rgbTexture;
+uniform lowp float opacity;
+varying highp vec2 qt_TexCoord;
+
+void main()
+{
+    gl_FragColor = texture2DRect(rgbTexture, qt_TexCoord) * opacity;
+}
diff --git a/src/qtmultimediaquicktools/shaders/rectsampler_rgb_core.frag b/src/qtmultimediaquicktools/shaders/rectsampler_rgb_core.frag
new file mode 100644
index 000000000..17f306456
--- /dev/null
+++ b/src/qtmultimediaquicktools/shaders/rectsampler_rgb_core.frag
@@ -0,0 +1,10 @@
+#version 150 core
+uniform sampler2DRect rgbTexture;
+uniform lowp float opacity;
+in highp vec2 qt_TexCoord;
+out vec4 fragColor;
+
+void main()
+{
+    fragColor = texture(rgbTexture, qt_TexCoord) * opacity;
+}