Scene3D: introduce compositingMode (FBO or Underlay)

The default compositing mode is FBO.

One of the problematic aspects of Scene3D is its round trip
through a FBO, which is moderately expensive on low-end hardware,
although it makes it a fully fledged Qt Quick 2 item. If one
wants MSAA then things are even worse, as an intermediate MS'ed
FBO is needed, then resolving into the final one, whose color
texture is then sampled.

However, there's a significant use case for which these FBOs
can be avoided, and that's the case of a 3D scene "below" other
QQ2 content. In this setup, Qt3D can simply render to the
screen, driven by QQ2; then QQ2 can draw on top.

(It's the typical "underlay" scenario.) This can be enabled by setting
the compositing mode to Underlay

[ChangeLog] Scene3D add compositingMode property. Allows underlay rendering.

Task-number: QTBUG-74977
Change-Id: I1ec5f5d60eab45835dbdb2596a7bf1b2ac3624e0
Reviewed-by: Mike Krus <mike.krus@kdab.com>

5 files changed, 146 insertions, 38 deletions

diff --git a/src/quick3d/imports/scene3d/qtquickscene3dplugin.cpp b/src/quick3d/imports/scene3d/qtquickscene3dplugin.cpp
index d7b985e9f..f93a8fdd1 100644
--- a/src/quick3d/imports/scene3d/qtquickscene3dplugin.cpp
+++ b/src/quick3d/imports/scene3d/qtquickscene3dplugin.cpp
@@ -48,6 +48,7 @@ QT_BEGIN_NAMESPACE
 void QtQuickScene3DPlugin::registerTypes(const char *uri)
 {
     qmlRegisterType<Qt3DRender::Scene3DItem>(uri, 2, 0, "Scene3D");
+    qmlRegisterType<Qt3DRender::Scene3DItem, 14>(uri, 2, 14, "Scene3D");
 
     // Auto-increment the import to stay in sync with ALL future Qt minor versions
     qmlRegisterModule(uri, 2, QT_VERSION_MINOR);
diff --git a/src/quick3d/imports/scene3d/scene3ditem.cpp b/src/quick3d/imports/scene3d/scene3ditem.cpp
index 9da35a34f..54f1554fd 100644
--- a/src/quick3d/imports/scene3d/scene3ditem.cpp
+++ b/src/quick3d/imports/scene3d/scene3ditem.cpp
@@ -132,7 +132,10 @@ Scene3DItem::Scene3DItem(QQuickItem *parent)
     , m_rendererCleaner(new Scene3DCleaner())
     , m_multisample(true)
     , m_dirty(true)
+    , m_clearsWindowByDefault(true)
+    , m_disableClearWindow(false)
     , m_cameraAspectRatioMode(AutomaticAspectRatio)
+    , m_compositingMode(FBO)
 {
     setFlag(QQuickItem::ItemHasContents, true);
     setAcceptedMouseButtons(Qt::MouseButtonMask);
@@ -140,6 +143,11 @@ Scene3DItem::Scene3DItem(QQuickItem *parent)
 
     // Use manual drive mode when using Scene3D
     m_aspectEngine->setRunMode(Qt3DCore::QAspectEngine::Manual);
+
+    // Give a default size so that if nothing is specified by the user
+    // we still won't get ignored by the QtQuick SG when in Underlay mode
+    setWidth(1);
+    setHeight(1);
 }
 
 Scene3DItem::~Scene3DItem()
@@ -246,6 +254,45 @@ void Scene3DItem::setHoverEnabled(bool enabled)
 }
 
 /*!
+    \qmlproperty enumeration Scene3D::compositingMode
+
+    \value FBO
+           Scene is rendered into a Frame Buffer Object which can be costly on
+           some platform and hardware but allows a greater amount of
+           flexibility. Automatic aspect ratio. This is the compositing mode to
+           choose if your Scene3D element shouldn't occupy the entire screen
+           and if you optionally plan on having it resized or animated. In this
+           mode, the position of the Scene3D in the QML file controls its
+           stacking order with regard to the other Qt Quick elements.
+
+    \value Underlay
+           Suitable for full screen 3D scenes where using an FBO might be too
+           resource intensive. Scene3D behaves as a QtQuick underlay.
+
+           Please note that when using this mode, the size of the Scene3D and
+           its transformations are ignored and the rendering will occupy the
+           whole screen. The position of the Scene3D in the QML file won't have
+           any effect either. The Qt 3D content will be drawn prior to any Qt
+           Quick content. Care has to be taken not to overdraw and hide the Qt
+           3D content by overlapping Qt Quick content.
+
+           Additionally when using this mode, the window clearBeforeRendering
+           will be set to false automatically.
+
+    \since 5.14
+    \default FBO
+ */
+void Scene3DItem::setCompositingMode(Scene3DItem::CompositingMode mode)
+{
+    if (m_compositingMode == mode)
+        return;
+    m_compositingMode = mode;
+    emit compositingModeChanged();
+
+    QQuickItem::update();
+}
+
+/*!
     \qmlproperty enumeration Scene3D::cameraAspectRatioMode
 
     \value Scene3D.AutomaticAspectRatio
@@ -260,6 +307,11 @@ Scene3DItem::CameraAspectRatioMode Scene3DItem::cameraAspectRatioMode() const
     return m_cameraAspectRatioMode;
 }
 
+Scene3DItem::CompositingMode Scene3DItem::compositingMode() const
+{
+    return m_compositingMode;
+}
+
 void Scene3DItem::applyRootEntityChange()
 {
     if (m_aspectEngine->rootEntity() != m_entity) {
@@ -351,7 +403,13 @@ void Scene3DItem::onBeforeSync()
         m_renderer->allowRender();
 
     // Request refresh for next frame
-    QQuickItem::update();
+
+    // When using the FBO mode, only the QQuickItem needs to be updated
+    // When using the Underlay mode, the whole windows needs updating
+    if (m_compositingMode == FBO)
+        QQuickItem::update();
+    else
+        window()->update();
 }
 
 void Scene3DItem::setWindowSurface(QObject *rootObject)
@@ -474,13 +532,35 @@ QSGNode *Scene3DItem::updatePaintNode(QSGNode *node, QQuickItem::UpdatePaintNode
         m_renderer = new Scene3DRenderer(this, m_aspectEngine, m_renderAspect);
         m_renderer->setCleanerHelper(m_rendererCleaner);
     }
+    m_renderer->setCompositingMode(m_compositingMode);
 
     Scene3DSGNode *fboNode = static_cast<Scene3DSGNode *>(node);
-    if (fboNode == nullptr) {
-        fboNode = new Scene3DSGNode();
-        m_renderer->setSGNode(fboNode);
+    const bool usesFBO = m_compositingMode == FBO;
+    if (usesFBO) {
+        if (fboNode == nullptr) {
+            fboNode = new Scene3DSGNode();
+            m_renderer->setSGNode(fboNode);
+        }
+        fboNode->setRect(boundingRect());
+
+        // Reset clear flag if we've set it to false it's still set to that
+        if (m_disableClearWindow && !window()->clearBeforeRendering())
+            window()->setClearBeforeRendering(m_clearsWindowByDefault);
+        m_disableClearWindow = false;
+    } else {
+        // In FBOLess node the Scene3DItem doesn't have any QSGNode to actually
+        // manager
+        if (fboNode != nullptr) {
+            delete fboNode;
+            fboNode = nullptr;
+            m_renderer->setSGNode(fboNode);
+        }
+        // Record clearBeforeRendering value before we force it to false
+        m_clearsWindowByDefault = window()->clearBeforeRendering();
+        m_disableClearWindow = true;
+        if (m_clearsWindowByDefault)
+            window()->setClearBeforeRendering(false);
     }
-    fboNode->setRect(boundingRect());
 
     return fboNode;
 }
diff --git a/src/quick3d/imports/scene3d/scene3ditem_p.h b/src/quick3d/imports/scene3d/scene3ditem_p.h
index a23bb9e5b..ae7a4600e 100644
--- a/src/quick3d/imports/scene3d/scene3ditem_p.h
+++ b/src/quick3d/imports/scene3d/scene3ditem_p.h
@@ -78,6 +78,7 @@ class Scene3DItem : public QQuickItem
     Q_PROPERTY(bool multisample READ multisample WRITE setMultisample NOTIFY multisampleChanged)
     Q_PROPERTY(CameraAspectRatioMode cameraAspectRatioMode READ cameraAspectRatioMode WRITE setCameraAspectRatioMode NOTIFY cameraAspectRatioModeChanged)
     Q_PROPERTY(bool hoverEnabled READ isHoverEnabled WRITE setHoverEnabled NOTIFY hoverEnabledChanged)
+    Q_PROPERTY(CompositingMode compositingMode READ compositingMode WRITE setCompositingMode NOTIFY compositingModeChanged REVISION 14)
     Q_CLASSINFO("DefaultProperty", "entity")
 public:
     explicit Scene3DItem(QQuickItem *parent = 0);
@@ -98,11 +99,19 @@ public:
     Q_ENUM(CameraAspectRatioMode); // LCOV_EXCL_LINE
     CameraAspectRatioMode cameraAspectRatioMode() const;
 
+    enum CompositingMode {
+        FBO,
+        Underlay
+    };
+    Q_ENUM(CompositingMode) // LCOV_EXCL_LINE
+    CompositingMode compositingMode() const;
+
 public Q_SLOTS:
     void setAspects(const QStringList &aspects);
     void setEntity(Qt3DCore::QEntity *entity);
     void setCameraAspectRatioMode(CameraAspectRatioMode mode);
     void setHoverEnabled(bool enabled);
+    void setCompositingMode(CompositingMode mode);
 
 Q_SIGNALS:
     void aspectsChanged();
@@ -110,6 +119,7 @@ Q_SIGNALS:
     void multisampleChanged();
     void cameraAspectRatioModeChanged(CameraAspectRatioMode mode);
     void hoverEnabledChanged();
+    void compositingModeChanged();
 
 private Q_SLOTS:
     void applyRootEntityChange();
@@ -133,9 +143,12 @@ private:
 
     bool m_multisample;
     bool m_dirty;
+    bool m_clearsWindowByDefault;
+    bool m_disableClearWindow;
 
     QPointer<Qt3DRender::QCamera> m_camera;
     CameraAspectRatioMode m_cameraAspectRatioMode;
+    CompositingMode m_compositingMode;
     QOffscreenSurface *m_dummySurface;
 };
 
diff --git a/src/quick3d/imports/scene3d/scene3drenderer.cpp b/src/quick3d/imports/scene3d/scene3drenderer.cpp
index 596ad0b84..da63c32d8 100644
--- a/src/quick3d/imports/scene3d/scene3drenderer.cpp
+++ b/src/quick3d/imports/scene3d/scene3drenderer.cpp
@@ -311,6 +311,11 @@ void Scene3DRenderer::allowRender()
     m_allowRendering.release(1);
 }
 
+void Scene3DRenderer::setCompositingMode(Scene3DItem::CompositingMode mode)
+{
+    m_compositingMode = mode;
+}
+
 void Scene3DRenderer::setSGNode(Scene3DSGNode *node)
 {
     m_node = node;
@@ -331,51 +336,58 @@ void Scene3DRenderer::render()
     // it here to give Qt3D the clean state it expects
     m_window->resetOpenGLState();
 
-    // Rebuild FBO and textures if never created or a resize has occurred
-    if ((m_multisampledFBO.isNull() || m_forceRecreate) && m_multisample) {
-        m_multisampledFBO.reset(createMultisampledFramebufferObject(m_lastSize));
-        if (m_multisampledFBO->format().samples() == 0 || !QOpenGLFramebufferObject::hasOpenGLFramebufferBlit()) {
-            m_multisample = false;
-            m_multisampledFBO.reset(nullptr);
+    // Create and bind FBO if using the FBO compositing mode
+    const bool usesFBO = m_compositingMode == Scene3DItem::FBO;
+    if (usesFBO) {
+        // Rebuild FBO and textures if never created or a resize has occurred
+        if ((m_multisampledFBO.isNull() || m_forceRecreate) && m_multisample) {
+            m_multisampledFBO.reset(createMultisampledFramebufferObject(m_lastSize));
+            if (m_multisampledFBO->format().samples() == 0 || !QOpenGLFramebufferObject::hasOpenGLFramebufferBlit()) {
+                m_multisample = false;
+                m_multisampledFBO.reset(nullptr);
+            }
         }
-    }
 
-    if (m_finalFBO.isNull() || m_forceRecreate) {
-        m_finalFBO.reset(createFramebufferObject(m_lastSize));
-        m_texture.reset(m_window->createTextureFromId(m_finalFBO->texture(), m_finalFBO->size(), QQuickWindow::TextureHasAlphaChannel));
-        m_node->setTexture(m_texture.data());
-    }
+        if (m_finalFBO.isNull() || m_forceRecreate) {
+            m_finalFBO.reset(createFramebufferObject(m_lastSize));
+            m_texture.reset(m_window->createTextureFromId(m_finalFBO->texture(), m_finalFBO->size(), QQuickWindow::TextureHasAlphaChannel));
+            m_node->setTexture(m_texture.data());
+        }
 
-    m_forceRecreate = false;
 
-    // Bind FBO
-    if (m_multisample) //Only try to use MSAA when available
-        m_multisampledFBO->bind();
-    else
-        m_finalFBO->bind();
+        m_forceRecreate = false;
+
+        // Bind FBO
+        if (m_multisample) //Only try to use MSAA when available
+            m_multisampledFBO->bind();
+        else
+            m_finalFBO->bind();
+    }
 
     // Render Qt3D Scene
-    static_cast<QRenderAspectPrivate*>(QRenderAspectPrivate::get(m_renderAspect))->renderSynchronous();
+    static_cast<QRenderAspectPrivate*>(QRenderAspectPrivate::get(m_renderAspect))->renderSynchronous(usesFBO);
 
     // We may have called doneCurrent() so restore the context if the rendering surface was changed
     // Note: keep in mind that the ContextSave also restores the surface when destroyed
     if (saver.context()->surface() != saver.surface())
         saver.context()->makeCurrent(saver.surface());
 
-    if (m_multisample) {
-        // Blit multisampled FBO with non multisampled FBO with texture attachment
-        const QRect dstRect(QPoint(0, 0), m_finalFBO->size());
-        const QRect srcRect(QPoint(0, 0), m_multisampledFBO->size());
-        QOpenGLFramebufferObject::blitFramebuffer(m_finalFBO.data(), dstRect,
-                                                  m_multisampledFBO.data(), srcRect,
-                                                  GL_COLOR_BUFFER_BIT,
-                                                  GL_NEAREST,
-                                                  0, 0,
-                                                  QOpenGLFramebufferObject::DontRestoreFramebufferBinding);
-    }
+    if (usesFBO) {
+        if (m_multisample) {
+            // Blit multisampled FBO with non multisampled FBO with texture attachment
+            const QRect dstRect(QPoint(0, 0), m_finalFBO->size());
+            const QRect srcRect(QPoint(0, 0), m_multisampledFBO->size());
+            QOpenGLFramebufferObject::blitFramebuffer(m_finalFBO.data(), dstRect,
+                                                      m_multisampledFBO.data(), srcRect,
+                                                      GL_COLOR_BUFFER_BIT,
+                                                      GL_NEAREST,
+                                                      0, 0,
+                                                      QOpenGLFramebufferObject::DontRestoreFramebufferBinding);
+        }
 
-    // Restore QtQuick FBO
-    QOpenGLFramebufferObject::bindDefault();
+        // Restore QtQuick FBO
+        QOpenGLFramebufferObject::bindDefault();
+    }
 
     // Reset the state used by the Qt Quick scenegraph to avoid any
     // interference when rendering the rest of the UI.
diff --git a/src/quick3d/imports/scene3d/scene3drenderer_p.h b/src/quick3d/imports/scene3d/scene3drenderer_p.h
index cfb7f23d2..a2deed61e 100644
--- a/src/quick3d/imports/scene3d/scene3drenderer_p.h
+++ b/src/quick3d/imports/scene3d/scene3drenderer_p.h
@@ -55,6 +55,7 @@
 #include <QtCore/qsize.h>
 #include <QtCore/QMutex>
 #include <QtCore/QSemaphore>
+#include <scene3ditem_p.h>
 
 QT_BEGIN_NAMESPACE
 
@@ -70,7 +71,6 @@ namespace Qt3DRender {
 
 class QRenderAspect;
 class Scene3DCleaner;
-class Scene3DItem;
 class Scene3DSGNode;
 
 class Scene3DRenderer : public QObject
@@ -85,6 +85,7 @@ public:
     void setSGNode(Scene3DSGNode *node);
     void setCleanerHelper(Scene3DCleaner *cleaner);
     void allowRender();
+    void setCompositingMode(Scene3DItem::CompositingMode mode);
 
 public Q_SLOTS:
     void render();
@@ -115,6 +116,7 @@ private:
     bool m_forceRecreate;
     bool m_shouldRender;
     QSemaphore m_allowRendering;
+    Scene3DItem::CompositingMode m_compositingMode;
 
     friend class Scene3DCleaner;
 };