Move resetOpenGLState() to QQuickFbo

QQuickWindow should not have OpenGL specifics in its API in Qt 6.

However, resetOpenGLState() is used by applications commonly in
combination with QQuickFramebufferObject (not the least because the
documentation recommends it!). This is no problem in practice because
QQuickFramebufferObject remains an OpenGL-only feature. So to minimize
the breaks, move the function into QQuickFramebufferObject::Renderer so
any application that calls the function can continue to do so by just
calling it on 'this' instead.

The rendercontrol_opengl example used to call this function as well, but
unnecessarily, it will still function the same way without it.

Note that there is a chance that there are applications that call
resetOpenGLState() in other contexts, for example in slots connected to
before or afterRendering(). For these it will need to be determined if
the call is necessary at all, and if it is, should be replaced by
(re)setting the appropriate OpenGL state manually instead.

Task-number: QTBUG-84523
Change-Id: I335599f77e8a84e347a44427eb1a1bf917796ee8
Reviewed-by: Christian Strømme <christian.stromme@qt.io>

6 files changed, 83 insertions, 123 deletions

diff --git a/examples/quick/rendercontrol/rendercontrol_opengl/window_singlethreaded.cpp b/examples/quick/rendercontrol/rendercontrol_opengl/window_singlethreaded.cpp
index 9435c004a7..0d29a155dd 100644
--- a/examples/quick/rendercontrol/rendercontrol_opengl/window_singlethreaded.cpp
+++ b/examples/quick/rendercontrol/rendercontrol_opengl/window_singlethreaded.cpp
@@ -206,9 +206,7 @@ void WindowSingleThreaded::render()
     m_renderControl->render();
     m_renderControl->endFrame();
 
-    m_quickWindow->resetOpenGLState();
     QOpenGLFramebufferObject::bindDefault();
-
     m_context->functions()->glFlush();
 
     m_quickReady = true;
diff --git a/src/quick/items/qquickframebufferobject.cpp b/src/quick/items/qquickframebufferobject.cpp
index 9026591119..81ea49f4de 100644
--- a/src/quick/items/qquickframebufferobject.cpp
+++ b/src/quick/items/qquickframebufferobject.cpp
@@ -41,6 +41,7 @@
 
 #include <QOpenGLFramebufferObject>
 #include <QOpenGLFunctions>
+#include <private/qopenglvertexarrayobject_p.h>
 #include <private/qquickitem_p.h>
 #include <private/qsgadaptationlayer_p.h>
 #include <qsgtextureprovider.h>
@@ -208,6 +209,7 @@ public:
         , renderPending(true)
         , invalidatePending(false)
         , devicePixelRatio(1)
+        , vaoHelper(nullptr)
     {
         qsgnode_set_description(this, QStringLiteral("fbonode"));
     }
@@ -218,6 +220,7 @@ public:
         delete texture();
         delete fbo;
         delete msDisplayFbo;
+        delete vaoHelper;
     }
 
     void scheduleRender()
@@ -238,7 +241,7 @@ public Q_SLOTS:
             renderPending = false;
 
             window->beginExternalCommands();
-            window->resetOpenGLState();
+            renderer->resetOpenGLState();
 
             fbo->bind();
             QOpenGLContext::currentContext()->functions()->glViewport(0, 0, fbo->width(), fbo->height());
@@ -274,6 +277,7 @@ public:
     bool invalidatePending;
 
     qreal devicePixelRatio;
+    QOpenGLVertexArrayObjectHelper *vaoHelper;
 };
 
 static inline bool isOpenGL(QSGRenderContext *rc)
@@ -473,7 +477,7 @@ QOpenGLFramebufferObject *QQuickFramebufferObject::Renderer::framebufferObject()
  * context. This means that the state might have been modified by Quick before
  * invoking this function.
  *
- * \note It is recommended to call QQuickWindow::resetOpenGLState() before
+ * \note It is recommended to call resetOpenGLState() before
  * returning. This resets OpenGL state used by the Qt Quick renderer and thus
  * avoids interference from the state changes made by the rendering code in this
  * function.
@@ -549,6 +553,76 @@ void QQuickFramebufferObject::Renderer::update()
         ((QSGFramebufferObjectNode *) data)->scheduleRender();
 }
 
+/*!
+    Call this function to reset the OpenGL context its default state.
+
+    The scene graph uses the OpenGL context and will both rely on and
+    clobber its state. When mixing raw OpenGL commands with scene
+    graph rendering, this function provides a convenient way of
+    resetting the OpenGL context state back to its default values.
+
+    This function does not touch state in the fixed-function pipeline.
+
+    \warning This function will only reset the OpenGL context in
+    relation to what may be changed internally as part of the OpenGL
+    scene graph. It does not reset anything that has been changed
+    externally such as direct OpenGL calls done inside the application
+    code if those same calls are not used internally. (for example,
+    various OpenGL 3.x or 4.x specific state)
+
+    \since 6.0
+ */
+void QQuickFramebufferObject::Renderer::resetOpenGLState()
+{
+    QOpenGLContext *ctx = QOpenGLContext::currentContext();
+    if (!ctx || !data)
+        return;
+
+    QOpenGLFunctions *gl = ctx->functions();
+
+    gl->glBindBuffer(GL_ARRAY_BUFFER, 0);
+    gl->glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, 0);
+
+    QOpenGLVertexArrayObjectHelper *&vaoHelper(static_cast<QSGFramebufferObjectNode *>(data)->vaoHelper);
+    if (!vaoHelper)
+        vaoHelper = new QOpenGLVertexArrayObjectHelper(ctx);
+    if (vaoHelper->isValid())
+        vaoHelper->glBindVertexArray(0);
+
+    if (ctx->isOpenGLES() || (gl->openGLFeatures() & QOpenGLFunctions::FixedFunctionPipeline)) {
+        int maxAttribs;
+        gl->glGetIntegerv(GL_MAX_VERTEX_ATTRIBS, &maxAttribs);
+        for (int i=0; i<maxAttribs; ++i) {
+            gl->glVertexAttribPointer(i, 4, GL_FLOAT, GL_FALSE, 0, nullptr);
+            gl->glDisableVertexAttribArray(i);
+        }
+    }
+
+    gl->glActiveTexture(GL_TEXTURE0);
+    gl->glBindTexture(GL_TEXTURE_2D, 0);
+
+    gl->glDisable(GL_DEPTH_TEST);
+    gl->glDisable(GL_STENCIL_TEST);
+    gl->glDisable(GL_SCISSOR_TEST);
+
+    gl->glColorMask(true, true, true, true);
+    gl->glClearColor(0, 0, 0, 0);
+
+    gl->glDepthMask(true);
+    gl->glDepthFunc(GL_LESS);
+    gl->glClearDepthf(1);
+
+    gl->glStencilMask(0xff);
+    gl->glStencilOp(GL_KEEP, GL_KEEP, GL_KEEP);
+    gl->glStencilFunc(GL_ALWAYS, 0, 0xff);
+
+    gl->glDisable(GL_BLEND);
+    gl->glBlendFunc(GL_ONE, GL_ZERO);
+
+    gl->glUseProgram(0);
+
+    QOpenGLFramebufferObject::bindDefault();
+}
 
 #include "qquickframebufferobject.moc"
 #include "moc_qquickframebufferobject.cpp"
diff --git a/src/quick/items/qquickframebufferobject.h b/src/quick/items/qquickframebufferobject.h
index 4bb335c6ed..e8aa4a24fb 100644
--- a/src/quick/items/qquickframebufferobject.h
+++ b/src/quick/items/qquickframebufferobject.h
@@ -68,6 +68,7 @@ public:
         QOpenGLFramebufferObject *framebufferObject() const;
         void update();
         void invalidateFramebufferObject();
+        void resetOpenGLState();
     private:
         friend class QSGFramebufferObjectNode;
         friend class QQuickFramebufferObject;
diff --git a/src/quick/items/qquickwindow.cpp b/src/quick/items/qquickwindow.cpp
index 7925d1a483..aaf89b27e9 100644
--- a/src/quick/items/qquickwindow.cpp
+++ b/src/quick/items/qquickwindow.cpp
@@ -81,12 +81,9 @@
 #include <private/qqmldebugserviceinterfaces_p.h>
 #include <private/qqmldebugconnector_p.h>
 #if QT_CONFIG(opengl)
-#include <private/qopenglvertexarrayobject_p.h>
 #include <private/qsgdefaultrendercontext_p.h>
 #include <private/qopengl_p.h>
 #include <QOpenGLContext>
-#include <QOpenGLFunctions>
-#include <QOpenGLFramebufferObject>
 #endif
 #ifndef QT_NO_DEBUG_STREAM
 #include <private/qdebug_p.h>
@@ -753,7 +750,6 @@ QQuickWindowPrivate::QQuickWindowPrivate()
     , allowChildEventFiltering(true)
     , allowDoubleClick(true)
     , lastFocusReason(Qt::OtherFocusReason)
-    , vaoHelper(nullptr)
     , incubationController(nullptr)
     , hasActiveSwapchain(false)
     , hasRenderableSwapchain(false)
@@ -3986,10 +3982,6 @@ void QQuickWindow::maybeUpdate()
 void QQuickWindow::cleanupSceneGraph()
 {
     Q_D(QQuickWindow);
-#if QT_CONFIG(opengl)
-    delete d->vaoHelper;
-    d->vaoHelper = nullptr;
-#endif
     if (!d->renderer)
         return;
 
@@ -4389,8 +4381,6 @@ QQmlIncubationController *QQuickWindow::incubationController() const
     \warning Make very sure that a signal handler for beforeSynchronizing leaves the GL
     context in the same state as it was when the signal handler was entered. Failing to
     do so can result in the scene not rendering properly.
-
-    \sa resetOpenGLState()
 */
 
 /*!
@@ -4418,7 +4408,6 @@ QQmlIncubationController *QQuickWindow::incubationController() const
     properly.
 
     \since 5.3
-    \sa resetOpenGLState()
  */
 
 /*!
@@ -4458,8 +4447,6 @@ QQmlIncubationController *QQuickWindow::incubationController() const
     \warning Make very sure that a signal handler for beforeRendering leaves the OpenGL
     context in the same state as it was when the signal handler was entered. Failing to
     do so can result in the scene not rendering properly.
-
-    \sa resetOpenGLState()
 */
 
 /*!
@@ -4496,8 +4483,6 @@ QQmlIncubationController *QQuickWindow::incubationController() const
     \warning Make very sure that a signal handler for afterRendering() leaves the OpenGL
     context in the same state as it was when the signal handler was entered. Failing to
     do so can result in the scene not rendering properly.
-
-    \sa resetOpenGLState()
  */
 
 /*!
@@ -4676,7 +4661,7 @@ QQmlIncubationController *QQuickWindow::incubationController() const
     graphics context in the same state as it was when the signal handler was entered.
     Failing to do so can result in the scene not rendering properly.
 
-    \sa sceneGraphInvalidated(), resetOpenGLState()
+    \sa sceneGraphInvalidated()
     \since 5.3
  */
 
@@ -4937,89 +4922,6 @@ void QQuickWindow::setDefaultAlphaBuffer(bool useAlpha)
 {
     QQuickWindowPrivate::defaultAlphaBuffer = useAlpha;
 }
-#if QT_CONFIG(opengl)
-/*!
-    \since 5.2
-
-    Call this function to reset the OpenGL context its default state.
-
-    The scene graph uses the OpenGL context and will both rely on and
-    clobber its state. When mixing raw OpenGL commands with scene
-    graph rendering, this function provides a convenient way of
-    resetting the OpenGL context state back to its default values.
-
-    This function does not touch state in the fixed-function pipeline.
-
-    This function does not clear the color, depth and stencil buffers. Use
-    QQuickWindow::setClearBeforeRendering to control clearing of the color
-    buffer. The depth and stencil buffer might be clobbered by the scene
-    graph renderer. Clear these manually on demand.
-
-    \note This function only has an effect when using the default OpenGL scene graph
-    adaptation.
-
-    \note This function will only reset the OpenGL context in relation to what has been changed
-    internally as part of the OpenGL scene graph. It does not reset anything that has been changed
-    externally such as direct OpenGL calls done inside the application code if those same calls are
-    not used internally.
-
-    \note This function has no effect when running on the RHI graphics
-    abstraction and the underlying RHI backend is not OpenGL.
-
-    \sa QQuickWindow::beforeRendering(), beginExternalCommands(), endExternalCommands()
- */
-void QQuickWindow::resetOpenGLState()
-{
-    Q_D(QQuickWindow);
-
-    QOpenGLContext *ctx = d->openglContext();
-    if (!ctx)
-        return;
-
-    QOpenGLFunctions *gl = ctx->functions();
-    gl->glBindBuffer(GL_ARRAY_BUFFER, 0);
-    gl->glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, 0);
-
-    if (!d->vaoHelper)
-        d->vaoHelper = new QOpenGLVertexArrayObjectHelper(ctx);
-    if (d->vaoHelper->isValid())
-        d->vaoHelper->glBindVertexArray(0);
-
-    if (ctx->isOpenGLES() || (gl->openGLFeatures() & QOpenGLFunctions::FixedFunctionPipeline)) {
-        int maxAttribs;
-        gl->glGetIntegerv(GL_MAX_VERTEX_ATTRIBS, &maxAttribs);
-        for (int i=0; i<maxAttribs; ++i) {
-            gl->glVertexAttribPointer(i, 4, GL_FLOAT, GL_FALSE, 0, nullptr);
-            gl->glDisableVertexAttribArray(i);
-        }
-    }
-
-    gl->glActiveTexture(GL_TEXTURE0);
-    gl->glBindTexture(GL_TEXTURE_2D, 0);
-
-    gl->glDisable(GL_DEPTH_TEST);
-    gl->glDisable(GL_STENCIL_TEST);
-    gl->glDisable(GL_SCISSOR_TEST);
-
-    gl->glColorMask(true, true, true, true);
-    gl->glClearColor(0, 0, 0, 0);
-
-    gl->glDepthMask(true);
-    gl->glDepthFunc(GL_LESS);
-    gl->glClearDepthf(1);
-
-    gl->glStencilMask(0xff);
-    gl->glStencilOp(GL_KEEP, GL_KEEP, GL_KEEP);
-    gl->glStencilFunc(GL_ALWAYS, 0, 0xff);
-
-    gl->glDisable(GL_BLEND);
-    gl->glBlendFunc(GL_ONE, GL_ZERO);
-
-    gl->glUseProgram(0);
-
-    QOpenGLFramebufferObject::bindDefault();
-}
-#endif
 
 /*!
     \struct QQuickWindow::GraphicsStateInfo
@@ -5064,7 +4966,7 @@ const QQuickWindow::GraphicsStateInfo &QQuickWindow::graphicsStateInfo()
     application, not retrieved from the scene graph). With graphics APIs where
     no native command buffer concept is exposed (OpenGL, Direct 3D 11),
     beginExternalCommands() and endExternalCommands() together provide a
-    replacement for resetOpenGLState().
+    replacement for the removed Qt 5 resetOpenGLState() function.
 
     Calling this function and endExternalCommands() is not necessary within the
     \l{QSGRenderNode::render()}{render()} implementation of a QSGRenderNode
@@ -5082,17 +4984,12 @@ const QQuickWindow::GraphicsStateInfo &QQuickWindow::graphicsStateInfo()
     Therefore, always query CommandListResource after calling this function. Do
     not attempt to reuse an object from an earlier query.
 
-    \note This function has no effect when the scene graph is using OpenGL
-    directly and the RHI graphics abstraction layer is not in use. Refer to
-    resetOpenGLState() in that case.
-
     \note When the scenegraph is using the RHI graphics abstraction layer with
     the OpenGL backend underneath, pay attention to the fact that the OpenGL
     state in the context can have arbitrary settings, and this function does not
-    perform any resetting of the state back to defaults. Call
-    resetOpenGLState() if that is seen necessary.
+    perform any resetting of the state back to defaults.
 
-    \sa endExternalCommands(), resetOpenGLState()
+    \sa endExternalCommands()
 
     \since 5.14
  */
@@ -5124,17 +5021,13 @@ void QQuickWindow::beginExternalCommands()
     application, not retrieved from the scene graph). With graphics APIs where
     no native command buffer concept is exposed (OpenGL, Direct 3D 11),
     beginExternalCommands() and endExternalCommands() together provide a
-    replacement for resetOpenGLState().
+    replacement for the removed Qt 5 resetOpenGLState() function.
 
     Calling this function and beginExternalCommands() is not necessary within the
     \l{QSGRenderNode::render()}{render()} implementation of a QSGRenderNode
     because the scene graph performs the necessary steps implicitly for render
     nodes.
 
-    \note This function has no effect when the scene graph is using OpenGL
-    directly and the RHI graphics abstraction layer is not in use. Refer to
-    resetOpenGLState() in that case.
-
     \sa beginExternalCommands()
 
     \since 5.14
diff --git a/src/quick/items/qquickwindow.h b/src/quick/items/qquickwindow.h
index acd289ff4a..1b87d01f96 100644
--- a/src/quick/items/qquickwindow.h
+++ b/src/quick/items/qquickwindow.h
@@ -134,9 +134,6 @@ public:
 
     void setRenderTarget(const QQuickRenderTarget &target);
 
-#if QT_CONFIG(opengl)
-    void resetOpenGLState(); // ### Qt 6 remove
-#endif
     struct GraphicsStateInfo {
         int currentFrameSlot;
         int framesInFlight;
diff --git a/src/quick/items/qquickwindow_p.h b/src/quick/items/qquickwindow_p.h
index 6d2ff3753b..3141ef6b55 100644
--- a/src/quick/items/qquickwindow_p.h
+++ b/src/quick/items/qquickwindow_p.h
@@ -71,8 +71,7 @@
 
 QT_BEGIN_NAMESPACE
 
-class QOpenContext;
-class QOpenGLVertexArrayObjectHelper;
+class QOpenGLContext;
 class QQuickAnimatorController;
 class QQuickDragGrabber;
 class QQuickItemPrivate;
@@ -298,8 +297,6 @@ public:
 
     QQuickGraphicsDevice customDeviceObjects;
 
-    QOpenGLVertexArrayObjectHelper *vaoHelper; // ### Qt 6 remove
-
     mutable QQuickWindowIncubationController *incubationController;
 
     static bool defaultAlphaBuffer;