Studio3D: Stabilize

On Linux at least it is now able to start up and even change the window
both with the basic and threaded render loops. There are some issues
left in the threaded case, the window change can deadlock sometimes.
Also, in the threaded case the engine is not actually destructed when
exiting the application (since the deleteLater() request has no chance
to run). We can probably live with the latter for now. The former needs
more testing.

Change-Id: I7dac8a8b603973ce56cd8afee7075177aef4deda
Reviewed-by: Andy Nichols <andy.nichols@qt.io>

1 files changed, 62 insertions, 22 deletions

diff --git a/src/imports/studio3d/q3dsstudio3ditem.cpp b/src/imports/studio3d/q3dsstudio3ditem.cpp
index c61c18e..5957ba1 100644
--- a/src/imports/studio3d/q3dsstudio3ditem.cpp
+++ b/src/imports/studio3d/q3dsstudio3ditem.cpp
@@ -32,6 +32,7 @@
 #include "q3dsstudio3dnode_p.h"
 #include <QSGNode>
 #include <QLoggingCategory>
+#include <QThread>
 #include <QRunnable>
 #include <QQuickWindow>
 #include <QQuickRenderControl>
@@ -105,16 +106,21 @@ void Q3DSStudio3DItem::createEngine()
         } else {
             m_engine->setSurface(w);
         }
+
+        qCDebug(lcStudio3D, "created engine %p", m_engine);
     }
 
     const QString fn = QQmlFile::urlToLocalFileOrQrc(m_source);
     qCDebug(lcStudio3D) << "source is now" << fn;
     const QSize sz(width(), height());
     if (!sz.isEmpty())
-        sendResize(sz);
+        m_engine->resize(sz);
 
     m_engine->setSource(fn);
 
+    if (!sz.isEmpty())
+        sendResizeToQt3D(sz);
+
     // cannot start() here, that must be deferred
 
     update();
@@ -152,7 +158,7 @@ void Q3DSStudio3DItem::itemChange(QQuickItem::ItemChange change,
     if (change == QQuickItem::ItemSceneChange) {
         if (changeData.window) {
             connect(changeData.window, &QQuickWindow::sceneGraphInvalidated, this, [this]() {
-                // render thread
+                // render thread (or main if non-threaded render loop)
                 qCDebug(lcStudio3D, "[R] sceneGraphInvalidated");
                 delete m_renderer;
                 m_renderer = nullptr;
@@ -174,35 +180,67 @@ void Q3DSStudio3DItem::startEngine()
 void Q3DSStudio3DItem::destroyEngine()
 {
     if (m_engine) {
-        qCDebug(lcStudio3D, "destroying engine");
+        Q_ASSERT(!m_renderer);
+        qCDebug(lcStudio3D, "destroying engine %p", m_engine);
         delete m_engine; // recreate on next window change - if we are still around, that is
         m_engine = nullptr;
     }
 }
 
-class Releaser : public QRunnable
+class EngineReleaser : public QObject
+{
+public:
+    EngineReleaser(Q3DSEngine *engine)
+        : m_engine(engine)
+    { }
+    ~EngineReleaser() {
+        qCDebug(lcStudio3D, "async release: destroying engine %p", m_engine);
+        delete m_engine;
+    }
+
+private:
+    Q3DSEngine *m_engine;
+};
+
+class RendererReleaser : public QRunnable
 {
 public:
-    Releaser(Q3DSStudio3DRenderer *r) : m_renderer(r) { }
+    RendererReleaser(Q3DSStudio3DRenderer *r, EngineReleaser *er)
+        : m_renderer(r),
+          m_engineReleaser(er)
+    { }
     void run() override {
         delete m_renderer;
+        m_engineReleaser->deleteLater();
     }
+
 private:
     Q3DSStudio3DRenderer *m_renderer;
+    EngineReleaser *m_engineReleaser;
 };
 
 void Q3DSStudio3DItem::releaseResources()
 {
     qCDebug(lcStudio3D, "releaseResources");
 
-    // by the time the runnable runs we (the item) may already be gone; that's fine, just pass the renderer ref
+    // this may not be an application exit; if this is just a window change then allow continuing
+    // by eventually creating new engine and renderer objects
+
     if (window()) {
-        destroyEngine();
+        // renderer must be destroyed first (on the Quick render thread)
         if (m_renderer) {
-            window()->scheduleRenderJob(new Releaser(m_renderer), QQuickWindow::BeforeSynchronizingStage);
-            // this may not be an application exit; if this is just a window change then allow continuing
-            // by eventually creating a new renderer object
-            m_renderer = nullptr;
+            if (m_renderer->thread() == QThread::currentThread()) {
+                delete m_renderer;
+                m_renderer = nullptr;
+                destroyEngine();
+            } else {
+                // by the time the runnable runs we (the item) may already be gone; that's fine, just pass the renderer ref
+                EngineReleaser *er = new EngineReleaser(m_engine);
+                RendererReleaser *rr = new RendererReleaser(m_renderer, er);
+                window()->scheduleRenderJob(rr, QQuickWindow::BeforeSynchronizingStage);
+                m_renderer = nullptr;
+                m_engine = nullptr;
+            }
         }
     }
 }
@@ -211,21 +249,23 @@ void Q3DSStudio3DItem::geometryChanged(const QRectF &newGeometry, const QRectF &
 {
     QQuickItem::geometryChanged(newGeometry, oldGeometry);
 
-    if (!newGeometry.isEmpty() && m_engine)
-        sendResize(newGeometry.size().toSize());
+    if (!newGeometry.isEmpty() && m_engine) {
+        const QSize sz = newGeometry.size().toSize();
+        m_engine->resize(sz);
+        sendResizeToQt3D(sz);
+    }
 }
 
-void Q3DSStudio3DItem::sendResize(const QSize &size)
+void Q3DSStudio3DItem::sendResizeToQt3D(const QSize &size)
 {
-    Q_ASSERT(m_engine);
-    m_engine->resize(size);
-
     Qt3DCore::QEntity *rootEntity = m_engine->rootEntity();
-    Qt3DRender::QRenderSurfaceSelector *surfaceSelector = Qt3DRender::QRenderSurfaceSelectorPrivate::find(rootEntity);
-    qCDebug(lcStudio3D, "Setting external render target size on surface selector %p", surfaceSelector);
-    if (surfaceSelector) {
-        surfaceSelector->setExternalRenderTargetSize(size);
-        surfaceSelector->setSurfacePixelRatio(window()->effectiveDevicePixelRatio());
+    if (rootEntity) {
+        Qt3DRender::QRenderSurfaceSelector *surfaceSelector = Qt3DRender::QRenderSurfaceSelectorPrivate::find(rootEntity);
+        qCDebug(lcStudio3D, "Setting external render target size on surface selector %p", surfaceSelector);
+        if (surfaceSelector) {
+            surfaceSelector->setExternalRenderTargetSize(size);
+            surfaceSelector->setSurfacePixelRatio(window()->effectiveDevicePixelRatio());
+        }
     }
 }