From f7e5f12149b73b60c594be626eeac8c2aa34beda Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christian=20Str=C3=B8mme?= <christian.stromme@qt.io>
Date: Thu, 20 Apr 2017 17:23:11 +0200
Subject: Android: Fix crash caused by stale video-frame objects

Make sure that the frame being processed is still valid, that is, the
size is valid and correct, and that the frame buffer is accessible.

Task-number: QTBUG-60115
Change-Id: Ic64b6f7d45d92b7923d97d6ecdc630da31abee17
Reviewed-by: Michael Dippold <mike@dippold.com>
Reviewed-by: Eirik Aavitsland <eirik.aavitsland@qt.io>
---
 .../android/src/common/qandroidvideooutput.cpp     | 52 +++++++++++++++-------
 .../android/src/common/qandroidvideooutput.h       |  2 +-
 2 files changed, 36 insertions(+), 18 deletions(-)

(limited to 'src')

diff --git a/src/plugins/android/src/common/qandroidvideooutput.cpp b/src/plugins/android/src/common/qandroidvideooutput.cpp
index 1d0df27f2..b425b9d89 100644
--- a/src/plugins/android/src/common/qandroidvideooutput.cpp
+++ b/src/plugins/android/src/common/qandroidvideooutput.cpp
@@ -86,11 +86,12 @@ void OpenGLResourcesDeleter::deleteShaderProgramHelper(void *prog)
 class AndroidTextureVideoBuffer : public QAbstractVideoBuffer
 {
 public:
-    AndroidTextureVideoBuffer(QAndroidTextureVideoOutput *output)
+    AndroidTextureVideoBuffer(QAndroidTextureVideoOutput *output, const QSize &size)
         : QAbstractVideoBuffer(GLTextureHandle)
+        , m_mapMode(NotMapped)
         , m_output(output)
+        , m_size(size)
         , m_textureUpdated(false)
-        , m_mapMode(NotMapped)
     {
     }
 
@@ -100,8 +101,7 @@ public:
 
     uchar *map(MapMode mode, int *numBytes, int *bytesPerLine)
     {
-        if (m_mapMode == NotMapped && mode == ReadOnly) {
-            updateFrame();
+        if (m_mapMode == NotMapped && mode == ReadOnly && updateFrame()) {
             m_mapMode = mode;
             m_image = m_output->m_fbo->toImage();
 
@@ -126,24 +126,41 @@ public:
     QVariant handle() const
     {
         AndroidTextureVideoBuffer *that = const_cast<AndroidTextureVideoBuffer*>(this);
-        that->updateFrame();
+        if (!that->updateFrame())
+            return QVariant();
+
         return m_output->m_fbo->texture();
     }
 
 private:
-    void updateFrame()
+    bool updateFrame()
     {
-        if (!m_textureUpdated) {
-            // update the video texture (called from the render thread)
-            m_output->renderFrameToFbo();
-            m_textureUpdated = true;
-        }
+        // Even though the texture was updated in a previous call, we need to re-check
+        // that this has not become a stale buffer, e.g., if the output size changed or
+        // has since became invalid.
+        if (!m_output->m_nativeSize.isValid())
+            return false;
+
+        // Size changed
+        if (m_output->m_nativeSize != m_size)
+            return false;
+
+        // In the unlikely event that we don't have a valid fbo, but have a valid size,
+        // force an update.
+        const bool forceUpdate = !m_output->m_fbo;
+
+        if (m_textureUpdated && !forceUpdate)
+            return true;
+
+        // update the video texture (called from the render thread)
+        return (m_textureUpdated = m_output->renderFrameToFbo());
     }
 
-    QAndroidTextureVideoOutput *m_output;
-    bool m_textureUpdated;
     MapMode m_mapMode;
+    QAndroidTextureVideoOutput *m_output;
     QImage m_image;
+    QSize m_size;
+    bool m_textureUpdated;
 };
 
 QAndroidTextureVideoOutput::QAndroidTextureVideoOutput(QObject *parent)
@@ -267,7 +284,6 @@ AndroidSurfaceTexture *QAndroidTextureVideoOutput::surfaceTexture()
 void QAndroidTextureVideoOutput::setVideoSize(const QSize &size)
 {
      QMutexLocker locker(&m_mutex);
-
     if (m_nativeSize == size)
         return;
 
@@ -297,7 +313,7 @@ void QAndroidTextureVideoOutput::onFrameAvailable()
     if (!m_nativeSize.isValid() || !m_surface)
         return;
 
-    QAbstractVideoBuffer *buffer = new AndroidTextureVideoBuffer(this);
+    QAbstractVideoBuffer *buffer = new AndroidTextureVideoBuffer(this, m_nativeSize);
     QVideoFrame frame(buffer, m_nativeSize, QVideoFrame::Format_BGR32);
 
     if (m_surface->isActive() && (m_surface->surfaceFormat().pixelFormat() != frame.pixelFormat()
@@ -316,12 +332,12 @@ void QAndroidTextureVideoOutput::onFrameAvailable()
         m_surface->present(frame);
 }
 
-void QAndroidTextureVideoOutput::renderFrameToFbo()
+bool QAndroidTextureVideoOutput::renderFrameToFbo()
 {
     QMutexLocker locker(&m_mutex);
 
     if (!m_nativeSize.isValid() || !m_surfaceTexture)
-        return;
+        return false;
 
     createGLResources();
 
@@ -376,6 +392,8 @@ void QAndroidTextureVideoOutput::renderFrameToFbo()
         glEnable(GL_SCISSOR_TEST);
     if (blendEnabled)
         glEnable(GL_BLEND);
+
+    return true;
 }
 
 void QAndroidTextureVideoOutput::createGLResources()
diff --git a/src/plugins/android/src/common/qandroidvideooutput.h b/src/plugins/android/src/common/qandroidvideooutput.h
index a12db75a2..936e4c40b 100644
--- a/src/plugins/android/src/common/qandroidvideooutput.h
+++ b/src/plugins/android/src/common/qandroidvideooutput.h
@@ -112,7 +112,7 @@ private Q_SLOTS:
 
 private:
     bool initSurfaceTexture();
-    void renderFrameToFbo();
+    bool renderFrameToFbo();
     void createGLResources();
 
     QMutex m_mutex;
-- 
cgit v1.2.3