Android: fixed video rendering with multiple media players.

It seems all Android media players share the same video buffers, which
results in textures containing frames for the wrong media player.
When getting a new frame, we now copy it into a FBO in order to avoid
that another media player overwrites the frame being shown on screen.

Task-number: QTBUG-35868
Change-Id: I6701cf7368a3ef9e73d649c3ece1f206cafd5bb3
Reviewed-by: Christian Stromme <christian.stromme@digia.com>

2 files changed, 175 insertions, 28 deletions

diff --git a/src/plugins/android/src/common/qandroidvideorendercontrol.cpp b/src/plugins/android/src/common/qandroidvideorendercontrol.cpp
index 5306fe918..55f71d735 100644
--- a/src/plugins/android/src/common/qandroidvideorendercontrol.cpp
+++ b/src/plugins/android/src/common/qandroidvideorendercontrol.cpp
@@ -49,22 +49,39 @@
 #include <qcoreapplication.h>
 #include <qopenglcontext.h>
 #include <qopenglfunctions.h>
+#include <qopenglshaderprogram.h>
+#include <qopenglframebufferobject.h>
 
 QT_BEGIN_NAMESPACE
 
-#define ExternalGLTextureHandle QAbstractVideoBuffer::HandleType(QAbstractVideoBuffer::UserHandle + 1)
+static const GLfloat g_vertex_data[] = {
+    -1.f, 1.f,
+    1.f, 1.f,
+    1.f, -1.f,
+    -1.f, -1.f
+};
+
+static const GLfloat g_texture_data[] = {
+    0.f, 0.f,
+    1.f, 0.f,
+    1.f, 1.f,
+    0.f, 1.f
+};
 
-TextureDeleter::~TextureDeleter()
+OpenGLResourcesDeleter::~OpenGLResourcesDeleter()
 {
-    glDeleteTextures(1, &m_id);
+    glDeleteTextures(1, &m_textureID);
+    delete m_fbo;
+    delete m_program;
 }
 
 class AndroidTextureVideoBuffer : public QAbstractVideoBuffer
 {
 public:
-    AndroidTextureVideoBuffer(JSurfaceTexture *surface)
-        : QAbstractVideoBuffer(ExternalGLTextureHandle)
-        , m_surfaceTexture(surface)
+    AndroidTextureVideoBuffer(QAndroidVideoRendererControl *control)
+        : QAbstractVideoBuffer(GLTextureHandle)
+        , m_control(control)
+        , m_textureUpdated(false)
     {
     }
 
@@ -76,18 +93,18 @@ public:
 
     QVariant handle() const
     {
-        if (m_data.isEmpty()) {
+        if (!m_textureUpdated) {
             // update the video texture (called from the render thread)
-            m_surfaceTexture->updateTexImage();
-            m_data << (uint)m_surfaceTexture->textureID() << m_surfaceTexture->getTransformMatrix();
+            m_control->renderFrameToFbo();
+            m_textureUpdated = true;
         }
 
-        return m_data;
+        return m_control->m_fbo->texture();
     }
 
 private:
-    mutable JSurfaceTexture *m_surfaceTexture;
-    mutable QVariantList m_data;
+    mutable QAndroidVideoRendererControl *m_control;
+    mutable bool m_textureUpdated;
 };
 
 QAndroidVideoRendererControl::QAndroidVideoRendererControl(QObject *parent)
@@ -97,7 +114,9 @@ QAndroidVideoRendererControl::QAndroidVideoRendererControl(QObject *parent)
     , m_surfaceTexture(0)
     , m_surfaceHolder(0)
     , m_externalTex(0)
-    , m_textureDeleter(0)
+    , m_fbo(0)
+    , m_program(0)
+    , m_glDeleter(0)
 {
 }
 
@@ -117,8 +136,8 @@ QAndroidVideoRendererControl::~QAndroidVideoRendererControl()
         delete m_surfaceHolder;
         m_surfaceHolder = 0;
     }
-    if (m_textureDeleter)
-        m_textureDeleter->deleteLater();
+    if (m_glDeleter)
+        m_glDeleter->deleteLater();
 }
 
 QAbstractVideoSurface *QAndroidVideoRendererControl::surface() const
@@ -162,7 +181,8 @@ bool QAndroidVideoRendererControl::initSurfaceTexture()
     // for the GL render thread to call us back to do it.
     if (QOpenGLContext::currentContext()) {
         glGenTextures(1, &m_externalTex);
-        m_textureDeleter = new TextureDeleter(m_externalTex);
+        m_glDeleter = new OpenGLResourcesDeleter;
+        m_glDeleter->setTexture(m_externalTex);
     } else if (!m_externalTex) {
         return false;
     }
@@ -174,9 +194,9 @@ bool QAndroidVideoRendererControl::initSurfaceTexture()
     } else {
         delete m_surfaceTexture;
         m_surfaceTexture = 0;
-        m_textureDeleter->deleteLater();
+        m_glDeleter->deleteLater();
         m_externalTex = 0;
-        m_textureDeleter = 0;
+        m_glDeleter = 0;
     }
 
     return m_surfaceTexture != 0;
@@ -208,6 +228,8 @@ jobject QAndroidVideoRendererControl::surfaceTexture()
 
 void QAndroidVideoRendererControl::setVideoSize(const QSize &size)
 {
+     QMutexLocker locker(&m_mutex);
+
     if (m_nativeSize == size)
         return;
 
@@ -228,7 +250,7 @@ void QAndroidVideoRendererControl::onFrameAvailable()
     if (!m_nativeSize.isValid() || !m_surface)
         return;
 
-    QAbstractVideoBuffer *buffer = new AndroidTextureVideoBuffer(m_surfaceTexture);
+    QAbstractVideoBuffer *buffer = new AndroidTextureVideoBuffer(this);
     QVideoFrame frame(buffer, m_nativeSize, QVideoFrame::Format_BGR32);
 
     if (m_surface->isActive() && (m_surface->surfaceFormat().pixelFormat() != frame.pixelFormat()
@@ -237,8 +259,8 @@ void QAndroidVideoRendererControl::onFrameAvailable()
     }
 
     if (!m_surface->isActive()) {
-        QVideoSurfaceFormat format(frame.size(), frame.pixelFormat(), ExternalGLTextureHandle);
-        format.setScanLineDirection(QVideoSurfaceFormat::BottomToTop);
+        QVideoSurfaceFormat format(frame.size(), frame.pixelFormat(),
+                                   QAbstractVideoBuffer::GLTextureHandle);
 
         m_surface->start(format);
     }
@@ -247,13 +269,114 @@ void QAndroidVideoRendererControl::onFrameAvailable()
         m_surface->present(frame);
 }
 
+void QAndroidVideoRendererControl::renderFrameToFbo()
+{
+    QMutexLocker locker(&m_mutex);
+
+    createGLResources();
+
+    m_surfaceTexture->updateTexImage();
+
+    // save current render states
+    GLboolean stencilTestEnabled;
+    GLboolean depthTestEnabled;
+    GLboolean scissorTestEnabled;
+    GLboolean blendEnabled;
+    glGetBooleanv(GL_STENCIL_TEST, &stencilTestEnabled);
+    glGetBooleanv(GL_DEPTH_TEST, &depthTestEnabled);
+    glGetBooleanv(GL_SCISSOR_TEST, &scissorTestEnabled);
+    glGetBooleanv(GL_BLEND, &blendEnabled);
+
+    if (stencilTestEnabled)
+        glDisable(GL_STENCIL_TEST);
+    if (depthTestEnabled)
+        glDisable(GL_DEPTH_TEST);
+    if (scissorTestEnabled)
+        glDisable(GL_SCISSOR_TEST);
+    if (blendEnabled)
+        glDisable(GL_BLEND);
+
+    m_fbo->bind();
+
+    glViewport(0, 0, m_nativeSize.width(), m_nativeSize.height());
+
+    m_program->bind();
+    m_program->enableAttributeArray(0);
+    m_program->enableAttributeArray(1);
+    m_program->setUniformValue("frameTexture", GLuint(0));
+    m_program->setUniformValue("texMatrix", m_surfaceTexture->getTransformMatrix());
+
+    glVertexAttribPointer(0, 2, GL_FLOAT, GL_FALSE, 0, g_vertex_data);
+    glVertexAttribPointer(1, 2, GL_FLOAT, GL_FALSE, 0, g_texture_data);
+
+    glDrawArrays(GL_TRIANGLE_FAN, 0, 4);
+
+    m_program->disableAttributeArray(0);
+    m_program->disableAttributeArray(1);
+
+    glBindTexture(GL_TEXTURE_EXTERNAL_OES, 0);
+    m_fbo->release();
+
+    // restore render states
+    if (stencilTestEnabled)
+        glEnable(GL_STENCIL_TEST);
+    if (depthTestEnabled)
+        glEnable(GL_DEPTH_TEST);
+    if (scissorTestEnabled)
+        glEnable(GL_SCISSOR_TEST);
+    if (blendEnabled)
+        glEnable(GL_BLEND);
+}
+
+void QAndroidVideoRendererControl::createGLResources()
+{
+    if (!m_fbo || m_fbo->size() != m_nativeSize) {
+        delete m_fbo;
+        m_fbo = new QOpenGLFramebufferObject(m_nativeSize);
+        m_glDeleter->setFbo(m_fbo);
+    }
+
+    if (!m_program) {
+        m_program = new QOpenGLShaderProgram;
+
+        QOpenGLShader *vertexShader = new QOpenGLShader(QOpenGLShader::Vertex, m_program);
+        vertexShader->compileSourceCode("attribute highp vec4 vertexCoordsArray; \n" \
+                                        "attribute highp vec2 textureCoordArray; \n" \
+                                        "uniform   highp mat4 texMatrix; \n" \
+                                        "varying   highp vec2 textureCoords; \n" \
+                                        "void main(void) \n" \
+                                        "{ \n" \
+                                        "    gl_Position = vertexCoordsArray; \n" \
+                                        "    textureCoords = (texMatrix * vec4(textureCoordArray, 0.0, 1.0)).xy; \n" \
+                                        "}\n");
+        m_program->addShader(vertexShader);
+
+        QOpenGLShader *fragmentShader = new QOpenGLShader(QOpenGLShader::Fragment, m_program);
+        fragmentShader->compileSourceCode("#extension GL_OES_EGL_image_external : require \n" \
+                                          "varying highp vec2         textureCoords; \n" \
+                                          "uniform samplerExternalOES frameTexture; \n" \
+                                          "void main() \n" \
+                                          "{ \n" \
+                                          "    gl_FragColor = texture2D(frameTexture, textureCoords); \n" \
+                                          "}\n");
+        m_program->addShader(fragmentShader);
+
+        m_program->bindAttributeLocation("vertexCoordsArray", 0);
+        m_program->bindAttributeLocation("textureCoordArray", 1);
+        m_program->link();
+
+        m_glDeleter->setShaderProgram(m_program);
+    }
+}
+
 void QAndroidVideoRendererControl::customEvent(QEvent *e)
 {
     if (e->type() == QEvent::User) {
         // This is running in the render thread (OpenGL enabled)
         if (!m_externalTex) {
             glGenTextures(1, &m_externalTex);
-            m_textureDeleter = new TextureDeleter(m_externalTex); // will be deleted in the correct thread
+            m_glDeleter = new OpenGLResourcesDeleter; // will cleanup GL resources in the correct thread
+            m_glDeleter->setTexture(m_externalTex);
             emit readyChanged(true);
         }
     }
diff --git a/src/plugins/android/src/common/qandroidvideorendercontrol.h b/src/plugins/android/src/common/qandroidvideorendercontrol.h
index 5d9130c07..6ce1e2dd4 100644
--- a/src/plugins/android/src/common/qandroidvideorendercontrol.h
+++ b/src/plugins/android/src/common/qandroidvideorendercontrol.h
@@ -43,22 +43,37 @@
 #define QANDROIDVIDEORENDERCONTROL_H
 
 #include <qvideorenderercontrol.h>
+#include <qmutex.h>
 #include "qandroidvideooutput.h"
 #include "jsurfacetexture.h"
 
 QT_BEGIN_NAMESPACE
 
 class JSurfaceTextureHolder;
+class QOpenGLTexture;
+class QOpenGLFramebufferObject;
+class QOpenGLShaderProgram;
 
-class TextureDeleter : public QObject
+class OpenGLResourcesDeleter : public QObject
 {
     Q_OBJECT
 public:
-    TextureDeleter(uint id) : m_id(id) { }
-    ~TextureDeleter();
+    OpenGLResourcesDeleter()
+        : m_textureID(0)
+        , m_fbo(0)
+        , m_program(0)
+    { }
+
+    ~OpenGLResourcesDeleter();
+
+    void setTexture(quint32 id) { m_textureID = id; }
+    void setFbo(QOpenGLFramebufferObject *fbo) { m_fbo = fbo; }
+    void setShaderProgram(QOpenGLShaderProgram *prog) { m_program = prog; }
 
 private:
-    uint m_id;
+    quint32 m_textureID;
+    QOpenGLFramebufferObject *m_fbo;
+    QOpenGLShaderProgram *m_program;
 };
 
 class QAndroidVideoRendererControl : public QVideoRendererControl, public QAndroidVideoOutput
@@ -88,6 +103,10 @@ private Q_SLOTS:
 
 private:
     bool initSurfaceTexture();
+    void renderFrameToFbo();
+    void createGLResources();
+
+    QMutex m_mutex;
 
     QAbstractVideoSurface *m_surface;
     QSize m_nativeSize;
@@ -95,8 +114,13 @@ private:
     QJNIObjectPrivate *m_androidSurface;
     JSurfaceTexture *m_surfaceTexture;
     JSurfaceTextureHolder *m_surfaceHolder;
-    uint m_externalTex;
-    TextureDeleter *m_textureDeleter;
+
+    quint32 m_externalTex;
+    QOpenGLFramebufferObject *m_fbo;
+    QOpenGLShaderProgram *m_program;
+    OpenGLResourcesDeleter *m_glDeleter;
+
+    friend class AndroidTextureVideoBuffer;
 };
 
 QT_END_NAMESPACE