Fix the direct texture path

The external texture can now be sampled directly. The external matrix
from the SurfaceTexture needs some adjustments to be correct for us.

The slow path can now be toggled via a build time define (handy for
testing).

There is an additional performance improvement in the uniform data
update for the video node (no need to create a QByteArray - and so
malloc - on every frame, we can rather write into the provided one).

Also a crash fix for when opening a new video in e.g. the mediaplayer
example. (querying the externalMatrix() from the videobuffer should
be skipped when there is no buffer due to switching video sources)

That said, there may be some other issues left because switching videos
in the mediaplayer example does crash sometimes still.

Change-Id: I6542782f4c6e32e0ad8fa0276bc8baac6dda7cad
Reviewed-by: Lars Knoll <lars.knoll@qt.io>

7 files changed, 66 insertions, 35 deletions

diff --git a/src/multimedia/platform/android/common/qandroidvideooutput.cpp b/src/multimedia/platform/android/common/qandroidvideooutput.cpp
index e5fb3b878..957734d85 100644
--- a/src/multimedia/platform/android/common/qandroidvideooutput.cpp
+++ b/src/multimedia/platform/android/common/qandroidvideooutput.cpp
@@ -112,15 +112,20 @@ QAbstractVideoBuffer::MapData AndroidTextureVideoBuffer::map(QVideoFrame::MapMod
 
 quint64 AndroidTextureVideoBuffer::textureHandle(int plane) const
 {
-    qDebug() << "AndroidTextureVideoBuffer::textureHandle()";
-    if (plane != 0)
+    if (plane != 0 || !rhi)
         return 0;
-    if (rhi) {
-        m_output->m_surfaceTexture->updateTexImage();
-        m_externalMatrix = m_output->m_surfaceTexture->getTransformMatrix();
-        return m_output->m_externalTex;
-    }
-    return 0;
+
+    m_output->ensureCommonGLResources();
+    m_output->m_surfaceTexture->updateTexImage();
+    m_externalMatrix = m_output->m_surfaceTexture->getTransformMatrix();
+    // flip it back, see http://androidxref.com/9.0.0_r3/xref/frameworks/native/libs/gui/GLConsumer.cpp#866
+    // (NB our matrix ctor takes row major)
+    static const QMatrix4x4 flipV(1.0f,  0.0f, 0.0f, 0.0f,
+                                  0.0f, -1.0f, 0.0f, 1.0f,
+                                  0.0f,  0.0f, 1.0f, 0.0f,
+                                  0.0f,  0.0f, 0.0f, 1.0f);
+    m_externalMatrix *= flipV;
+    return m_output->m_externalTex;
 }
 
 QAndroidTextureVideoOutput::QAndroidTextureVideoOutput(QObject *parent)
@@ -235,10 +240,14 @@ void QAndroidTextureVideoOutput::onFrameAvailable()
         return;
 
     QVideoFrameFormat::PixelFormat format = QVideoFrameFormat::Format_ARGB32_Premultiplied;
-    auto rhi = m_sink ? m_sink->rhi() : nullptr;
+#ifdef QANDROIDVIDEOUTPUT_NO_DIRECT_TEXTURE_USAGE
+    QRhi *rhi = nullptr;
+#else
+    QRhi *rhi = m_sink ? m_sink->rhi() : nullptr;
+#endif
     if (rhi && rhi->backend() != QRhi::OpenGLES2)
         rhi = nullptr;
-    else
+    if (rhi)
         format = QVideoFrameFormat::Format_SamplerExternalOES;
 
     auto *buffer = new AndroidTextureVideoBuffer(rhi, this, m_nativeSize);
@@ -286,7 +295,9 @@ bool QAndroidTextureVideoOutput::renderFrameToFbo()
 
     m_glContext->makeCurrent(m_offscreenSurface);
 
-    createGLResources();
+    ensureFboGLResources();
+
+    m_surfaceTexture->updateTexImage();
 
     // save current render states
     GLboolean stencilTestEnabled;
@@ -341,9 +352,9 @@ bool QAndroidTextureVideoOutput::renderFrameToFbo()
     return true;
 }
 
-void QAndroidTextureVideoOutput::createGLResources()
+void QAndroidTextureVideoOutput::ensureCommonGLResources()
 {
-    Q_ASSERT(QOpenGLContext::currentContext() != NULL);
+    Q_ASSERT(QOpenGLContext::currentContext());
 
     if (!m_glDeleter)
         m_glDeleter = new OpenGLResourcesDeleter;
@@ -353,6 +364,11 @@ void QAndroidTextureVideoOutput::createGLResources()
         glGenTextures(1, &m_externalTex);
         m_surfaceTexture->attachToGLContext(m_externalTex);
     }
+}
+
+void QAndroidTextureVideoOutput::ensureFboGLResources()
+{
+    ensureCommonGLResources();
 
     if (!m_fbo || m_fbo->size() != m_nativeSize) {
         delete m_fbo;
diff --git a/src/multimedia/platform/android/common/qandroidvideooutput_p.h b/src/multimedia/platform/android/common/qandroidvideooutput_p.h
index ddf98dc04..b651c1dcc 100644
--- a/src/multimedia/platform/android/common/qandroidvideooutput_p.h
+++ b/src/multimedia/platform/android/common/qandroidvideooutput_p.h
@@ -59,6 +59,11 @@
 
 QT_BEGIN_NAMESPACE
 
+// Enable this to prevent using the external texture directly (bound as
+// GL_TEXTURE_EXTERNAL_OES), but rather do a readback on every frame and
+// upload the QImage data into a plain 2D texture.
+//#define QANDROIDVIDEOUTPUT_NO_DIRECT_TEXTURE_USAGE
+
 class AndroidSurfaceTexture;
 class AndroidSurfaceHolder;
 class QOpenGLFramebufferObject;
@@ -129,7 +134,8 @@ private Q_SLOTS:
 private:
     void initSurfaceTexture();
     bool renderFrameToFbo();
-    void createGLResources();
+    void ensureCommonGLResources();
+    void ensureFboGLResources();
 
     QMutex m_mutex;
     void clearSurfaceTexture();
diff --git a/src/multimedia/video/qvideoframeformat.cpp b/src/multimedia/video/qvideoframeformat.cpp
index e8043870f..940fdaf98 100644
--- a/src/multimedia/video/qvideoframeformat.cpp
+++ b/src/multimedia/video/qvideoframeformat.cpp
@@ -548,9 +548,9 @@ QString QVideoFrameFormat::fragmentShaderFileName() const
 /*!
     \internal
 */
-QByteArray QVideoFrameFormat::uniformData(const QVideoFrame &frame, const QMatrix4x4 &transform, float opacity) const
+void QVideoFrameFormat::updateUniformData(QByteArray *dst, const QVideoFrame &frame, const QMatrix4x4 &transform, float opacity) const
 {
-    return QVideoTextureHelper::uniformData(*this, frame, transform, opacity);
+    QVideoTextureHelper::updateUniformData(dst, *this, frame, transform, opacity);
 }
 
 
diff --git a/src/multimedia/video/qvideoframeformat.h b/src/multimedia/video/qvideoframeformat.h
index 514a4713c..bfab9ebb3 100644
--- a/src/multimedia/video/qvideoframeformat.h
+++ b/src/multimedia/video/qvideoframeformat.h
@@ -162,7 +162,7 @@ public:
 
     QString vertexShaderFileName() const;
     QString fragmentShaderFileName() const;
-    QByteArray uniformData(const QVideoFrame &frame, const QMatrix4x4 &transform, float opacity) const;
+    void updateUniformData(QByteArray *dst, const QVideoFrame &frame, const QMatrix4x4 &transform, float opacity) const;
 
     static PixelFormat pixelFormatFromImageFormat(QImage::Format format);
     static QImage::Format imageFormatFromPixelFormat(PixelFormat format);
diff --git a/src/multimedia/video/qvideotexturehelper.cpp b/src/multimedia/video/qvideotexturehelper.cpp
index e4c74e9c2..cca4720ae 100644
--- a/src/multimedia/video/qvideotexturehelper.cpp
+++ b/src/multimedia/video/qvideotexturehelper.cpp
@@ -369,7 +369,7 @@ static QMatrix4x4 yuvColorCorrectionMatrix(float brightness, float contrast, flo
 }
 #endif
 
-QByteArray uniformData(const QVideoFrameFormat &format, const QVideoFrame &frame, const QMatrix4x4 &transform, float opacity)
+void updateUniformData(QByteArray *dst, const QVideoFrameFormat &format, const QVideoFrame &frame, const QMatrix4x4 &transform, float opacity)
 {
     Q_UNUSED(frame);
 
@@ -377,7 +377,7 @@ QByteArray uniformData(const QVideoFrameFormat &format, const QVideoFrame &frame
     switch (format.pixelFormat()) {
     case QVideoFrameFormat::Format_Invalid:
     case QVideoFrameFormat::Format_Jpeg:
-        return QByteArray();
+        return;
 
     case QVideoFrameFormat::Format_ARGB32:
     case QVideoFrameFormat::Format_ARGB32_Premultiplied:
@@ -411,22 +411,20 @@ QByteArray uniformData(const QVideoFrameFormat &format, const QVideoFrame &frame
 #ifdef Q_OS_ANDROID
     {
         // get Android specific transform for the externalsampler texture
-        auto *buffer = static_cast<AndroidTextureVideoBuffer *>(frame.videoBuffer());
-        Q_ASSERT(buffer);
-        cmat = buffer->externalTextureMatrix();
+        if (auto *buffer = static_cast<AndroidTextureVideoBuffer *>(frame.videoBuffer()))
+            cmat = buffer->externalTextureMatrix();
     }
 #endif
         break;
     }
-    // { matrix4x4, colorMatrix, opacity, planeWidth[3] }
-    QByteArray buf(64*2 + 4 + 4, Qt::Uninitialized);
-    char *data = buf.data();
+    // { matrix, colorMatrix, opacity, width }
+    Q_ASSERT(dst->size() >= 64 + 64 + 4 + 4);
+    char *data = dst->data();
     memcpy(data, transform.constData(), 64);
     memcpy(data + 64, cmat.constData(), 64);
     memcpy(data + 64 + 64, &opacity, 4);
     float width = format.frameWidth();
     memcpy(data + 64 + 64 + 4, &width, 4);
-    return buf;
 }
 
 int updateRhiTextures(QVideoFrame frame, QRhi *rhi, QRhiResourceUpdateBatch *resourceUpdates, QRhiTexture **textures)
@@ -442,11 +440,20 @@ int updateRhiTextures(QVideoFrame frame, QRhi *rhi, QRhiResourceUpdateBatch *res
         planeSizes[plane] = QSize(size.width()/description->sizeScale[plane].x, size.height()/description->sizeScale[plane].y);
 
     if (frame.handleType() == QVideoFrame::RhiTextureHandle) {
+        QRhiTexture::Flags textureFlags = {};
+        if (pixelFormat == QVideoFrameFormat::Format_SamplerExternalOES) {
+#ifdef Q_OS_ANDROID
+            if (rhi->backend() == QRhi::OpenGLES2)
+                textureFlags |= QRhiTexture::ExternalOES;
+#endif
+        }
         for (int plane = 0; plane < description->nplanes; ++plane) {
             quint64 nativeTexture = frame.textureHandle(plane);
-//            Q_ASSERT(nativeTexture);
-            textures[plane] = rhi->newTexture(description->textureFormat[plane], planeSizes[plane], 1, {});
-            textures[plane]->createFrom({nativeTexture, 0});
+            if (!nativeTexture)
+                qWarning("Texture from QVideoFrame is 0, this cannot be right");
+            textures[plane] = rhi->newTexture(description->textureFormat[plane], planeSizes[plane], 1, textureFlags);
+            if (!textures[plane]->createFrom({nativeTexture, 0}))
+                qWarning("Failed to initialize QRhiTexture wrapper for native texture object %llu", nativeTexture);
         }
         return description->nplanes;
     }
diff --git a/src/multimedia/video/qvideotexturehelper_p.h b/src/multimedia/video/qvideotexturehelper_p.h
index e1159b88b..9f8d95d3a 100644
--- a/src/multimedia/video/qvideotexturehelper_p.h
+++ b/src/multimedia/video/qvideotexturehelper_p.h
@@ -84,7 +84,7 @@ Q_MULTIMEDIA_EXPORT const TextureDescription *textureDescription(QVideoFrameForm
 
 Q_MULTIMEDIA_EXPORT QString vertexShaderFileName(QVideoFrameFormat::PixelFormat format);
 Q_MULTIMEDIA_EXPORT QString fragmentShaderFileName(QVideoFrameFormat::PixelFormat format);
-Q_MULTIMEDIA_EXPORT QByteArray uniformData(const QVideoFrameFormat &format, const QVideoFrame &frame, const QMatrix4x4 &transform, float opacity);
+Q_MULTIMEDIA_EXPORT void updateUniformData(QByteArray *dst, const QVideoFrameFormat &format, const QVideoFrame &frame, const QMatrix4x4 &transform, float opacity);
 Q_MULTIMEDIA_EXPORT int updateRhiTextures(QVideoFrame frame, QRhi *rhi,
                                            QRhiResourceUpdateBatch *resourceUpdates, QRhiTexture **textures);
 
diff --git a/src/multimediaquick/qsgvideonode_p.cpp b/src/multimediaquick/qsgvideonode_p.cpp
index c010d8e9c..9445f3d40 100644
--- a/src/multimediaquick/qsgvideonode_p.cpp
+++ b/src/multimediaquick/qsgvideonode_p.cpp
@@ -237,8 +237,12 @@ bool QSGVideoMaterialRhiShader::updateUniformData(RenderState &state, QSGMateria
         m->updateBlending();
     }
 
-    QByteArray *buf = state.uniformData();
-    *buf = m_format.uniformData(m_material->m_frame, state.combinedMatrix(), state.opacity());
+    // Do this here, not in updateSampledImage. First, with multiple textures we want to
+    // do this once. More importantly, on some platforms (Android) the externalMatrix is
+    // updated by this function and we need that already in updateUniformData.
+    m->updateTextures(state.rhi(), state.resourceUpdateBatch());
+
+    m_format.updateUniformData(state.uniformData(), m_material->m_frame, state.combinedMatrix(), state.opacity());
 
     return true;
 }
@@ -246,14 +250,12 @@ bool QSGVideoMaterialRhiShader::updateUniformData(RenderState &state, QSGMateria
 void QSGVideoMaterialRhiShader::updateSampledImage(RenderState &state, int binding, QSGTexture **texture,
                                                        QSGMaterial *newMaterial, QSGMaterial *oldMaterial)
 {
+    Q_UNUSED(state);
     Q_UNUSED(oldMaterial);
     if (binding < 1 || binding > 3)
         return;
 
     auto m = static_cast<QSGVideoMaterial *>(newMaterial);
-
-    m->updateTextures(state.rhi(), state.resourceUpdateBatch());
-
     *texture = m->m_textures[binding - 1].data();
 }