Fix composition for windows with alpha

While we will cover the entire surface with our textured quads, the clear
is still necessary in order to make top-levels with TranslucentBackground
render correctly: We don't want to blend transparent areas with undefined
content that is in the surface's framebuffer.

Blending is problematic for alpha values. We now prevent the blended alpha
from being written out. This ensures that in examples like qquickviewcomparison,
where the backingstore image contains an alpha of 0.5 while the QQuickWidget
texture 1.0, the result is still an alpha value of 1.0 in the final image.
Writing out an alpha of 0.5 would break on systems where windows get an alpha
buffer by default.

hellogl2 can now take a --transparent parameter which makes the QOpenGLWidget
being cleared to transparent in order to verify it works in combination
with Qt::WA_TranslucentBackground.

The swapped red and blue problem is also corrected. RGBA8888 does not need
swizzling. The only format that needs this is RGB32.

Task-number: QTBUG-40716
Change-Id: I54a9fd3a91a1b59575b38cdb908835315514e40f
Reviewed-by: Jørgen Lind <jorgen.lind@digia.com>

10 files changed, 71 insertions, 21 deletions

diff --git a/examples/opengl/hellogl2/glwidget.cpp b/examples/opengl/hellogl2/glwidget.cpp
index c8db3047a1..7764fa8dfc 100644
--- a/examples/opengl/hellogl2/glwidget.cpp
+++ b/examples/opengl/hellogl2/glwidget.cpp
@@ -52,6 +52,11 @@ GLWidget::GLWidget(QWidget *parent)
       m_program(0)
 {
     m_core = QCoreApplication::arguments().contains(QStringLiteral("--coreprofile"));
+    // --transparent causes the clear color to be transparent. Therefore, on systems that
+    // support it, the widget will become transparent apart from the logo.
+    m_transparent = QCoreApplication::arguments().contains(QStringLiteral("--transparent"));
+    if (m_transparent)
+        setAttribute(Qt::WA_TranslucentBackground);
 }
 
 GLWidget::~GLWidget()
@@ -183,6 +188,7 @@ void GLWidget::initializeGL()
     connect(context(), &QOpenGLContext::aboutToBeDestroyed, this, &GLWidget::cleanup);
 
     initializeOpenGLFunctions();
+    glClearColor(0, 0, 0, m_transparent ? 0 : 1);
 
     m_program = new QOpenGLShaderProgram;
     m_program->addShaderFromSourceCode(QOpenGLShader::Vertex, m_core ? vertexShaderSourceCore : vertexShaderSource);
diff --git a/examples/opengl/hellogl2/glwidget.h b/examples/opengl/hellogl2/glwidget.h
index fcc6225999..2d642649f1 100644
--- a/examples/opengl/hellogl2/glwidget.h
+++ b/examples/opengl/hellogl2/glwidget.h
@@ -98,6 +98,7 @@ private:
     QMatrix4x4 m_proj;
     QMatrix4x4 m_camera;
     QMatrix4x4 m_world;
+    bool m_transparent;
 };
 
 #endif
diff --git a/src/gui/painting/qplatformbackingstore.cpp b/src/gui/painting/qplatformbackingstore.cpp
index e5b06f499c..66a56ac32f 100644
--- a/src/gui/painting/qplatformbackingstore.cpp
+++ b/src/gui/painting/qplatformbackingstore.cpp
@@ -80,6 +80,7 @@ public:
 #ifndef QT_NO_OPENGL
     mutable GLuint textureId;
     mutable QSize textureSize;
+    mutable bool needsSwizzle;
     QOpenGLTextureBlitter *blitter;
 #endif
 };
@@ -229,13 +230,16 @@ static QRegion deviceRegion(const QRegion &region, QWindow *window)
 
 void QPlatformBackingStore::composeAndFlush(QWindow *window, const QRegion &region,
                                             const QPoint &offset,
-                                            QPlatformTextureList *textures, QOpenGLContext *context)
+                                            QPlatformTextureList *textures, QOpenGLContext *context,
+                                            bool translucentBackground)
 {
     Q_UNUSED(offset);
 
     context->makeCurrent(window);
     QOpenGLFunctions *funcs = context->functions();
     funcs->glViewport(0, 0, window->width() * window->devicePixelRatio(), window->height() * window->devicePixelRatio());
+    funcs->glClearColor(0, 0, 0, translucentBackground ? 0 : 1);
+    funcs->glClear(GL_COLOR_BUFFER_BIT);
 
     if (!d_ptr->blitter) {
         d_ptr->blitter = new QOpenGLTextureBlitter;
@@ -257,15 +261,22 @@ void QPlatformBackingStore::composeAndFlush(QWindow *window, const QRegion &regi
 
     funcs->glEnable(GL_BLEND);
     funcs->glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA);
+    // Do not write out alpha. We need blending, but only for RGB. The toplevel may have
+    // alpha enabled in which case blending (writing out < 1.0 alpha values) would lead to
+    // semi-transparency even when it is not wanted.
+    funcs->glColorMask(GL_TRUE, GL_TRUE, GL_TRUE, GL_FALSE);
 
     // Backingstore texture with the normal widgets.
-    GLuint textureId = toTexture(deviceRegion(region, window), &d_ptr->textureSize);
+    GLuint textureId = toTexture(deviceRegion(region, window), &d_ptr->textureSize, &d_ptr->needsSwizzle);
     if (textureId) {
         QMatrix4x4 target = QOpenGLTextureBlitter::targetTransform(QRect(QPoint(), d_ptr->textureSize), windowRect);
-        d_ptr->blitter->setSwizzleRB(true);
+        if (d_ptr->needsSwizzle)
+            d_ptr->blitter->setSwizzleRB(true);
         d_ptr->blitter->blit(textureId, target, QOpenGLTextureBlitter::OriginTopLeft);
-        d_ptr->blitter->setSwizzleRB(false);
+        if (d_ptr->needsSwizzle)
+            d_ptr->blitter->setSwizzleRB(false);
     }
+    funcs->glColorMask(GL_TRUE, GL_TRUE, GL_TRUE, GL_TRUE);
 
     // Textures for renderToTexture widgets that have WA_AlwaysStackOnTop set.
     for (int i = 0; i < textures->count(); ++i) {
@@ -308,9 +319,15 @@ QImage QPlatformBackingStore::toImage() const
   The default implementation returns a cached texture if \a dirtyRegion is
   empty and the window has not been resized, otherwise it retrieves the
   content using toImage() and performs a texture upload.
- */
 
-GLuint QPlatformBackingStore::toTexture(const QRegion &dirtyRegion, QSize *textureSize) const
+  If the red and blue components have to swapped, \a needsSwizzle will be set to \c true.
+  This allows creating textures from images in formats like QImage::Format_RGB32 without
+  any further image conversion. Instead, the swizzling will be done in the shaders when
+  performing composition. Other formats, that do not need such swizzling due to being
+  already byte ordered RGBA, for example QImage::Format_RGBA8888, must result in having \a
+  needsSwizzle set to false.
+ */
+GLuint QPlatformBackingStore::toTexture(const QRegion &dirtyRegion, QSize *textureSize, bool *needsSwizzle) const
 {
     QImage image = toImage();
     QSize imageSize = image.size();
@@ -321,8 +338,16 @@ GLuint QPlatformBackingStore::toTexture(const QRegion &dirtyRegion, QSize *textu
     if (dirtyRegion.isEmpty() && !resized)
         return d_ptr->textureId;
 
-    if (image.format() != QImage::Format_RGB32 && image.format() != QImage::Format_RGBA8888)
-        image = image.convertToFormat(QImage::Format_RGBA8888);
+    // Fast path for RGB32 and RGBA8888, convert everything else to RGBA8888.
+    if (image.format() == QImage::Format_RGB32) {
+        if (needsSwizzle)
+            *needsSwizzle = true;
+    } else {
+        if (needsSwizzle)
+            *needsSwizzle = false;
+        if (image.format() != QImage::Format_RGBA8888)
+            image = image.convertToFormat(QImage::Format_RGBA8888);
+    }
 
     QOpenGLFunctions *funcs = QOpenGLContext::currentContext()->functions();
 
diff --git a/src/gui/painting/qplatformbackingstore.h b/src/gui/painting/qplatformbackingstore.h
index fe0f460ed9..2c183f6d6f 100644
--- a/src/gui/painting/qplatformbackingstore.h
+++ b/src/gui/painting/qplatformbackingstore.h
@@ -110,9 +110,11 @@ public:
     // offset is the (child) window's offset in relation to the window surface.
     virtual void flush(QWindow *window, const QRegion &region, const QPoint &offset) = 0;
 #ifndef QT_NO_OPENGL
-    virtual void composeAndFlush(QWindow *window, const QRegion &region, const QPoint &offset, QPlatformTextureList *textures, QOpenGLContext *context);
+    virtual void composeAndFlush(QWindow *window, const QRegion &region, const QPoint &offset,
+                                 QPlatformTextureList *textures, QOpenGLContext *context,
+                                 bool translucentBackground);
     virtual QImage toImage() const;
-    virtual GLuint toTexture(const QRegion &dirtyRegion, QSize *textureSize) const;
+    virtual GLuint toTexture(const QRegion &dirtyRegion, QSize *textureSize, bool *needsSwizzle) const;
 #endif
 
     virtual void resize(const QSize &size, const QRegion &staticContents) = 0;
diff --git a/src/platformsupport/eglconvenience/qeglplatformbackingstore.cpp b/src/platformsupport/eglconvenience/qeglplatformbackingstore.cpp
index 7b627f85ae..7923ef1cab 100644
--- a/src/platformsupport/eglconvenience/qeglplatformbackingstore.cpp
+++ b/src/platformsupport/eglconvenience/qeglplatformbackingstore.cpp
@@ -165,13 +165,15 @@ void QEGLPlatformBackingStore::flush(QWindow *window, const QRegion &region, con
 }
 
 void QEGLPlatformBackingStore::composeAndFlush(QWindow *window, const QRegion &region, const QPoint &offset,
-                                               QPlatformTextureList *textures, QOpenGLContext *context)
+                                               QPlatformTextureList *textures, QOpenGLContext *context,
+                                               bool translucentBackground)
 {
     // QOpenGLWidget content provided as textures. The raster content should go on top.
 
     Q_UNUSED(region);
     Q_UNUSED(offset);
     Q_UNUSED(context);
+    Q_UNUSED(translucentBackground);
 
     QEGLPlatformScreen *screen = static_cast<QEGLPlatformScreen *>(m_window->screen());
     QEGLPlatformWindow *dstWin = screen->compositingWindow();
diff --git a/src/platformsupport/eglconvenience/qeglplatformbackingstore_p.h b/src/platformsupport/eglconvenience/qeglplatformbackingstore_p.h
index 8b0566afa6..f3c4b9e559 100644
--- a/src/platformsupport/eglconvenience/qeglplatformbackingstore_p.h
+++ b/src/platformsupport/eglconvenience/qeglplatformbackingstore_p.h
@@ -68,7 +68,8 @@ public:
 
     QImage toImage() const Q_DECL_OVERRIDE;
     void composeAndFlush(QWindow *window, const QRegion &region, const QPoint &offset,
-                         QPlatformTextureList *textures, QOpenGLContext *context) Q_DECL_OVERRIDE;
+                         QPlatformTextureList *textures, QOpenGLContext *context,
+                         bool translucentBackground) Q_DECL_OVERRIDE;
 
     const QPlatformTextureList *textures() const { return m_textures; }
 
diff --git a/src/plugins/platforms/xcb/qxcbbackingstore.cpp b/src/plugins/platforms/xcb/qxcbbackingstore.cpp
index ada5b0eedf..6258b29fc7 100644
--- a/src/plugins/platforms/xcb/qxcbbackingstore.cpp
+++ b/src/plugins/platforms/xcb/qxcbbackingstore.cpp
@@ -337,9 +337,10 @@ void QXcbBackingStore::flush(QWindow *window, const QRegion &region, const QPoin
 
 #ifndef QT_NO_OPENGL
 void QXcbBackingStore::composeAndFlush(QWindow *window, const QRegion &region, const QPoint &offset,
-                                       QPlatformTextureList *textures, QOpenGLContext *context)
+                                       QPlatformTextureList *textures, QOpenGLContext *context,
+                                       bool translucentBackground)
 {
-    QPlatformBackingStore::composeAndFlush(window, region, offset, textures, context);
+    QPlatformBackingStore::composeAndFlush(window, region, offset, textures, context, translucentBackground);
 
     Q_XCB_NOOP(connection());
 
diff --git a/src/plugins/platforms/xcb/qxcbbackingstore.h b/src/plugins/platforms/xcb/qxcbbackingstore.h
index af3c004c2d..725d0511af 100644
--- a/src/plugins/platforms/xcb/qxcbbackingstore.h
+++ b/src/plugins/platforms/xcb/qxcbbackingstore.h
@@ -62,7 +62,8 @@ public:
     void flush(QWindow *window, const QRegion &region, const QPoint &offset);
 #ifndef QT_NO_OPENGL
     void composeAndFlush(QWindow *window, const QRegion &region, const QPoint &offset,
-                         QPlatformTextureList *textures, QOpenGLContext *context);
+                         QPlatformTextureList *textures, QOpenGLContext *context,
+                         bool translucentBackground);
 #endif
     QImage toImage() const;
     void resize(const QSize &size, const QRegion &staticContents);
diff --git a/src/widgets/kernel/qopenglwidget.cpp b/src/widgets/kernel/qopenglwidget.cpp
index 68e3082c14..d8f31bfcc8 100644
--- a/src/widgets/kernel/qopenglwidget.cpp
+++ b/src/widgets/kernel/qopenglwidget.cpp
@@ -365,6 +365,10 @@ QT_BEGIN_NAMESPACE
   where a semi-transparent QOpenGLWidget with other widgets visible underneath
   is required.
 
+  Note that this does not apply when there are no other widgets underneath and
+  the intention is to have a semi-transparent window. In that case the
+  traditional approach of setting Qt::WA_TranslucentBackground is sufficient.
+
   \e{OpenGL is a trademark of Silicon Graphics, Inc. in the United States and other
   countries.}
 
@@ -670,11 +674,13 @@ QOpenGLWidget::~QOpenGLWidget()
   OpenGL widgets, individual calls to this function can be replaced by one single call to
   QSurfaceFormat::setDefaultFormat() before creating the first widget.
 
-  \note Requesting an alpha buffer via this function will not lead to the desired results
-  and should be avoided. Instead, use Qt::WA_AlwaysStackOnTop to enable semi-transparent
-  QOpenGLWidget instances with other widgets visible underneath. Keep in mind however that
-  this breaks the stacking order, so it will no longer be possible to have other widgets
-  on top of the QOpenGLWidget.
+  \note Requesting an alpha buffer via this function, or by setting
+  Qt::WA_TranslucentBackground, will not lead to the desired results when the intention is
+  to make other widgets beneath visible. Instead, use Qt::WA_AlwaysStackOnTop to enable
+  semi-transparent QOpenGLWidget instances with other widgets visible underneath. Keep in
+  mind however that this breaks the stacking order, so it will no longer be possible to
+  have other widgets on top of the QOpenGLWidget. When the intention is to have a
+  semi-transparent top-level window, Qt::WA_TranslucentBackground is sufficient.
 
   \sa format(), Qt::WA_AlwaysStackOnTop, QSurfaceFormat::setDefaultFormat()
  */
diff --git a/src/widgets/kernel/qwidgetbackingstore.cpp b/src/widgets/kernel/qwidgetbackingstore.cpp
index 6f635611f0..bb4518ec5e 100644
--- a/src/widgets/kernel/qwidgetbackingstore.cpp
+++ b/src/widgets/kernel/qwidgetbackingstore.cpp
@@ -113,7 +113,12 @@ void QWidgetBackingStore::qt_flush(QWidget *widget, const QRegion &region, QBack
 #ifndef QT_NO_OPENGL
     if (widgetTextures) {
         widget->window()->d_func()->sendComposeStatus(widget->window(), false);
-        backingStore->handle()->composeAndFlush(widget->windowHandle(), region, offset, widgetTextures, tlw->d_func()->shareContext());
+        // A window may have alpha even when the app did not request
+        // WA_TranslucentBackground. Therefore the compositor needs to know whether the app intends
+        // to rely on translucency, in order to decide if it should clear to transparent or opaque.
+        const bool translucentBackground = widget->testAttribute(Qt::WA_TranslucentBackground);
+        backingStore->handle()->composeAndFlush(widget->windowHandle(), region, offset, widgetTextures,
+                                                widget->d_func()->shareContext(), translucentBackground);
         widget->window()->d_func()->sendComposeStatus(widget->window(), true);
     } else
 #endif