macOS: Fix video rendering with OpenGL based RHI

Linking against Qt Webengine forces the RHI to be OpenGL based on macOS.
This in turn caused video rendering to fail and only render black or
corrupted rectangles.

The underlying reason is that the OpenGL textures we get from the video
pipeline on macOS are GL_TEXTURE_RECTANGLE based while RHI expects
GL_TEXTURE_2D. As the fix for this requires a bit of plumbing in
QtGui, this change works around the problem by going through a CPU based
texture copy. This works, but leads to higher CPU loads and some dropped
frames at least for 4k videos. Work is ongoing to get this fixed properly
for 6.2.1.

Also submit a two related fixes that were found while debugging the
problem. Ask for OpenGL compatible pixel buffers if we're using OpenGL,
and Flush the texture cache regularly (this was done in Qt 5 as well).

Task-number: QTBUG-96406
Change-Id: I0a5f8da1d3d885c40e8b65048da9093d685848aa
Reviewed-by: Laszlo Agocs <laszlo.agocs@qt.io>
(cherry picked from commit 62c8e983e10dc300406e24e0dfabf895521a70fa)
Reviewed-by: Qt Cherry-pick Bot <cherrypick_bot@qt-project.org>

3 files changed, 12 insertions, 2 deletions

diff --git a/src/multimedia/platform/darwin/avfvideobuffer.mm b/src/multimedia/platform/darwin/avfvideobuffer.mm
index 6b3bae51c..0f687b924 100644
--- a/src/multimedia/platform/darwin/avfvideobuffer.mm
+++ b/src/multimedia/platform/darwin/avfvideobuffer.mm
@@ -71,7 +71,7 @@ AVFVideoBuffer::~AVFVideoBuffer()
             CFRelease(cvMetalTexture[i]);
 #if defined(Q_OS_MACOS)
     if (cvOpenGLTexture)
-        CFRelease(cvOpenGLTexture);
+        CVOpenGLTextureRelease(cvOpenGLTexture);
 #elif defined(Q_OS_IOS)
     if (cvOpenGLESTexture)
         CFRelease(cvOpenGLESTexture);
@@ -191,6 +191,7 @@ quint64 AVFVideoBuffer::textureHandle(int plane) const
         return cvMetalTexture[plane] ? quint64(CVMetalTextureGetTexture(cvMetalTexture[plane])) : 0;
     } else if (rhi->backend() == QRhi::OpenGLES2) {
 #ifdef Q_OS_MACOS
+        CVOpenGLTextureCacheFlush(sink->cvOpenGLTextureCache, 0);
         CVReturn cvret;
         // Create a CVPixelBuffer-backed OpenGL texture image from the texture cache.
         cvret = CVOpenGLTextureCacheCreateTextureFromImage(
@@ -200,10 +201,12 @@ quint64 AVFVideoBuffer::textureHandle(int plane) const
                         nil,
                         &cvOpenGLTexture);
 
+        Q_ASSERT(CVOpenGLTextureGetTarget(cvOpenGLTexture) == GL_TEXTURE_RECTANGLE);
         // Get an OpenGL texture name from the CVPixelBuffer-backed OpenGL texture image.
         return CVOpenGLTextureGetName(cvOpenGLTexture);
 #endif
 #ifdef Q_OS_IOS
+        CVOpenGLESTextureCacheFlush(sink->cvOpenGLESTextureCache, 0);
         CVReturn cvret;
         // Create a CVPixelBuffer-backed OpenGL texture image from the texture cache.
         cvret = CVOpenGLESTextureCacheCreateTextureFromImage(
diff --git a/src/multimedia/platform/darwin/avfvideosink.mm b/src/multimedia/platform/darwin/avfvideosink.mm
index ac4367140..bbf5b0432 100644
--- a/src/multimedia/platform/darwin/avfvideosink.mm
+++ b/src/multimedia/platform/darwin/avfvideosink.mm
@@ -151,6 +151,10 @@ void AVFVideoSinkInterface::setRhi(QRhi *rhi)
         }
     } else if (rhi->backend() == QRhi::OpenGLES2) {
 #ifdef Q_OS_MACOS
+        // ### FIXME: Workaround a limitation on macOS for now. The OpenGL textures we get a GL_TEXTURE_RECTANGLE and those ar
+        // not currently supported by RHI. So disable the RHI code path for the moment and rather copy the texture data
+        // on the CPU time until we have a solution.
+#if 0
         const auto *gl = static_cast<const QRhiGles2NativeHandles *>(rhi->nativeHandles());
 
         auto nsGLContext = gl->context->nativeInterface<QNativeInterface::QCocoaGLContext>()->nativeContext();
@@ -167,6 +171,9 @@ void AVFVideoSinkInterface::setRhi(QRhi *rhi)
             qWarning() << "OpenGL texture cache creation failed";
             m_rhi = nullptr;
         }
+#else
+        m_rhi = nullptr;
+#endif
 #endif
 #ifdef Q_OS_IOS
         // Create an OpenGL CoreVideo texture cache from the pixel buffer.
diff --git a/src/multimedia/platform/darwin/mediaplayer/avfvideorenderercontrol.mm b/src/multimedia/platform/darwin/mediaplayer/avfvideorenderercontrol.mm
index 2eda39936..05654efde 100644
--- a/src/multimedia/platform/darwin/mediaplayer/avfvideorenderercontrol.mm
+++ b/src/multimedia/platform/darwin/mediaplayer/avfvideorenderercontrol.mm
@@ -209,7 +209,7 @@ static NSDictionary* const AVF_OUTPUT_SETTINGS_OPENGL = @{
             @(kCVPixelFormatType_32BGRA),
             @(kCVPixelFormatType_32RGBA),
         ],
-        (NSString *)kCVPixelBufferMetalCompatibilityKey: @true
+        (NSString *)kCVPixelBufferOpenGLCompatibilityKey: @true
 };
 
 CVPixelBufferRef AVFVideoRendererControl::copyPixelBufferFromLayer(size_t& width, size_t& height)