Windows: make TranslucentBackground functional always

QOpenGLWidget and QQuickWidget was not functional when
WA_TranslucentBackground was set. This is due to the
static "isGL" type of checks that are not suitable since 5.3
due to RasterGLSurface windows which may or may not be
OpenGL windows, depending on their content.

To handle this, we have to do some check on every makeCurrent
and perform the necessary calls (most importantly
SetLayeredWindowAttributes).

Task-number: QTBUG-43854
Change-Id: If19c79482ec4f0a8b795ee710d52ed7e08b52563
Reviewed-by: Friedemann Kleint <Friedemann.Kleint@theqtcompany.com>

6 files changed, 59 insertions, 21 deletions

diff --git a/src/plugins/platforms/windows/qwindowscontext.cpp b/src/plugins/platforms/windows/qwindowscontext.cpp
index 13a3d044a0..2dab4f699f 100644
--- a/src/plugins/platforms/windows/qwindowscontext.cpp
+++ b/src/plugins/platforms/windows/qwindowscontext.cpp
@@ -430,7 +430,7 @@ void QWindowsContext::setKeyGrabber(QWindow *w)
 
 // Window class registering code (from qapplication_win.cpp)
 
-QString QWindowsContext::registerWindowClass(const QWindow *w, bool isGL)
+QString QWindowsContext::registerWindowClass(const QWindow *w)
 {
     Q_ASSERT(w);
     const Qt::WindowFlags flags = w->flags();
@@ -438,7 +438,9 @@ QString QWindowsContext::registerWindowClass(const QWindow *w, bool isGL)
     // Determine style and icon.
     uint style = CS_DBLCLKS;
     bool icon = true;
-    if (isGL || (flags & Qt::MSWindowsOwnDC))
+    // The following will not set CS_OWNDC for any widget window, even if it contains a
+    // QOpenGLWidget or QQuickWidget later on. That cannot be detected at this stage.
+    if (w->surfaceType() == QSurface::OpenGLSurface || (flags & Qt::MSWindowsOwnDC))
         style |= CS_OWNDC;
     if (!(flags & Qt::NoDropShadowWindowHint) && (QSysInfo::WindowsVersion & QSysInfo::WV_NT_based)
         && (type == Qt::Popup || w->property("_q_windowsDropShadow").toBool())) {
@@ -471,8 +473,6 @@ QString QWindowsContext::registerWindowClass(const QWindow *w, bool isGL)
     default:
         break;
     }
-    if (isGL)
-        cname += QStringLiteral("GL");
     if (style & CS_DROPSHADOW)
         cname += QStringLiteral("DropShadow");
     if (style & CS_SAVEBITS)
diff --git a/src/plugins/platforms/windows/qwindowscontext.h b/src/plugins/platforms/windows/qwindowscontext.h
index 28202219d6..5c2e21192b 100644
--- a/src/plugins/platforms/windows/qwindowscontext.h
+++ b/src/plugins/platforms/windows/qwindowscontext.h
@@ -161,7 +161,7 @@ public:
 
     int defaultDPI() const;
 
-    QString registerWindowClass(const QWindow *w, bool isGL);
+    QString registerWindowClass(const QWindow *w);
     QString registerWindowClass(QString cname, WNDPROC proc,
                                 unsigned style = 0, HBRUSH brush = 0,
                                 bool icon = false);
diff --git a/src/plugins/platforms/windows/qwindowseglcontext.cpp b/src/plugins/platforms/windows/qwindowseglcontext.cpp
index c0d0c1f77c..fbd56e2b02 100644
--- a/src/plugins/platforms/windows/qwindowseglcontext.cpp
+++ b/src/plugins/platforms/windows/qwindowseglcontext.cpp
@@ -573,6 +573,7 @@ bool QWindowsEGLContext::makeCurrent(QPlatformSurface *surface)
     QWindowsEGLStaticContext::libEGL.eglBindAPI(m_api);
 
     QWindowsWindow *window = static_cast<QWindowsWindow *>(surface);
+    window->aboutToMakeCurrent();
     EGLSurface eglSurface = static_cast<EGLSurface>(window->surface(m_eglConfig));
     Q_ASSERT(eglSurface);
 
diff --git a/src/plugins/platforms/windows/qwindowsglcontext.cpp b/src/plugins/platforms/windows/qwindowsglcontext.cpp
index 0e3e6826ec..3348241d37 100644
--- a/src/plugins/platforms/windows/qwindowsglcontext.cpp
+++ b/src/plugins/platforms/windows/qwindowsglcontext.cpp
@@ -1267,6 +1267,7 @@ bool QWindowsGLContext::makeCurrent(QPlatformSurface *surface)
 
     // Do we already have a DC entry for that window?
     QWindowsWindow *window = static_cast<QWindowsWindow *>(surface);
+    window->aboutToMakeCurrent();
     const HWND hwnd = window->handle();
     if (const QOpenGLContextData *contextData = findByHWND(m_windowContexts, hwnd)) {
         // Repeated calls to wglMakeCurrent when vsync is enabled in the driver will
diff --git a/src/plugins/platforms/windows/qwindowswindow.cpp b/src/plugins/platforms/windows/qwindowswindow.cpp
index e7061dbfde..0b72f6db4d 100644
--- a/src/plugins/platforms/windows/qwindowswindow.cpp
+++ b/src/plugins/platforms/windows/qwindowswindow.cpp
@@ -205,6 +205,18 @@ static inline QSize clientSize(HWND hwnd)
     return qSizeOfRect(rect);
 }
 
+static inline bool windowIsOpenGL(const QWindow *w)
+{
+    switch (w->surfaceType()) {
+    case QSurface::OpenGLSurface:
+        return true;
+    case QSurface::RasterGLSurface:
+        return qt_window_private(const_cast<QWindow *>(w))->compositing;
+    default:
+        return false;
+    }
+}
+
 static bool applyBlurBehindWindow(HWND hwnd)
 {
 #ifdef Q_OS_WINCE
@@ -328,6 +340,17 @@ static void setWindowOpacity(HWND hwnd, Qt::WindowFlags flags, bool hasAlpha, bo
 #endif // !Q_OS_WINCE
 }
 
+static inline void updateGLWindowSettings(const QWindow *w, HWND hwnd, Qt::WindowFlags flags, qreal opacity)
+{
+    const bool isGL = windowIsOpenGL(w);
+    const bool hasAlpha = w->format().hasAlpha();
+
+    if (isGL && hasAlpha)
+        applyBlurBehindWindow(hwnd);
+
+    setWindowOpacity(hwnd, flags, hasAlpha, isGL, opacity);
+}
+
 /*!
     \class WindowCreationData
     \brief Window creation code.
@@ -369,14 +392,13 @@ struct WindowCreationData
     void fromWindow(const QWindow *w, const Qt::WindowFlags flags, unsigned creationFlags = 0);
     inline WindowData create(const QWindow *w, const WindowData &data, QString title) const;
     inline void applyWindowFlags(HWND hwnd) const;
-    void initialize(HWND h, bool frameChange, qreal opacityLevel) const;
+    void initialize(const QWindow *w, HWND h, bool frameChange, qreal opacityLevel) const;
 
     Qt::WindowFlags flags;
     HWND parentHandle;
     Qt::WindowType type;
     unsigned style;
     unsigned exStyle;
-    bool isGL;
     bool topLevel;
     bool popup;
     bool dialog;
@@ -389,7 +411,7 @@ struct WindowCreationData
 QDebug operator<<(QDebug debug, const WindowCreationData &d)
 {
     debug.nospace() << QWindowsWindow::debugWindowFlags(d.flags)
-        << " GL=" << d.isGL << " topLevel=" << d.topLevel << " popup="
+        << " topLevel=" << d.topLevel << " popup="
         << d.popup << " dialog=" << d.dialog << " desktop=" << d.desktop
         << " embedded=" << d.embedded
         << " tool=" << d.tool << " style=" << debugWinStyle(d.style)
@@ -420,8 +442,6 @@ static inline void fixTopLevelWindowFlags(Qt::WindowFlags &flags)
 void WindowCreationData::fromWindow(const QWindow *w, const Qt::WindowFlags flagsIn,
                                     unsigned creationFlags)
 {
-    isGL = w->surfaceType() == QWindow::OpenGLSurface;
-    hasAlpha = w->format().hasAlpha();
     flags = flagsIn;
 
     // Sometimes QWindow doesn't have a QWindow parent but does have a native parent window,
@@ -494,7 +514,7 @@ void WindowCreationData::fromWindow(const QWindow *w, const Qt::WindowFlags flag
         // ### Commented out for now as it causes some problems, but
         // this should be correct anyway, so dig some more into this
 #ifdef Q_FLATTEN_EXPOSE
-        if (isGL)
+        if (windowIsOpenGL(w)) // a bit incorrect since the is-opengl status may change from false to true at any time later on
             style |= WS_CLIPSIBLINGS | WS_CLIPCHILDREN; // see SetPixelFormat
 #else
         style |= WS_CLIPSIBLINGS | WS_CLIPCHILDREN ;
@@ -569,7 +589,7 @@ QWindowsWindowData
 
     const HINSTANCE appinst = (HINSTANCE)GetModuleHandle(0);
 
-    const QString windowClassName = QWindowsContext::instance()->registerWindowClass(w, isGL);
+    const QString windowClassName = QWindowsContext::instance()->registerWindowClass(w);
 
     const QRect geometryDip = QWindowsScaling::mapFromNative(data.geometry);
     QRect fixedGeometryDip = QPlatformWindow::initialGeometry(w, geometryDip, defaultWindowWidth, defaultWindowHeight);
@@ -612,9 +632,6 @@ QWindowsWindowData
     result.embedded = embedded;
     result.customMargins = context->customMargins;
 
-    if (isGL && hasAlpha)
-        applyBlurBehindWindow(result.hwnd);
-
     return result;
 }
 
@@ -637,7 +654,7 @@ void WindowCreationData::applyWindowFlags(HWND hwnd) const
         << debugWinExStyle(newExStyle);
 }
 
-void WindowCreationData::initialize(HWND hwnd, bool frameChange, qreal opacityLevel) const
+void WindowCreationData::initialize(const QWindow *w, HWND hwnd, bool frameChange, qreal opacityLevel) const
 {
     if (desktop || !hwnd)
         return;
@@ -662,8 +679,7 @@ void WindowCreationData::initialize(HWND hwnd, bool frameChange, qreal opacityLe
             else
                 EnableMenuItem(systemMenu, SC_CLOSE, MF_BYCOMMAND|MF_GRAYED);
         }
-
-        setWindowOpacity(hwnd, flags, hasAlpha, isGL, opacityLevel);
+        updateGLWindowSettings(w, hwnd, flags, opacityLevel);
     } else { // child.
         SetWindowPos(hwnd, HWND_TOP, 0, 0, 0, 0, swpFlags);
     }
@@ -1045,7 +1061,7 @@ QWindowsWindowData
     creationData.fromWindow(w, parameters.flags);
     QWindowsWindowData result = creationData.create(w, parameters, title);
     // Force WM_NCCALCSIZE (with wParam=1) via SWP_FRAMECHANGED for custom margin.
-    creationData.initialize(result.hwnd, !parameters.customMargins.isNull(), 1);
+    creationData.initialize(w, result.hwnd, !parameters.customMargins.isNull(), 1);
     return result;
 }
 
@@ -1532,7 +1548,7 @@ QWindowsWindowData QWindowsWindow::setWindowFlags_sys(Qt::WindowFlags wt,
     WindowCreationData creationData;
     creationData.fromWindow(window(), wt, flags);
     creationData.applyWindowFlags(m_data.hwnd);
-    creationData.initialize(m_data.hwnd, true, m_opacity);
+    creationData.initialize(window(), m_data.hwnd, true, m_opacity);
 
     QWindowsWindowData result = m_data;
     result.flags = creationData.flags;
@@ -2303,4 +2319,22 @@ void *QWindowsWindow::surface(void *nativeConfig)
 #endif
 }
 
+void QWindowsWindow::aboutToMakeCurrent()
+{
+#ifndef QT_NO_OPENGL
+    // For RasterGLSurface windows, that become OpenGL windows dynamically, it might be
+    // time to set up some GL specifics.  This is particularly important for layered
+    // windows (WS_EX_LAYERED due to alpha > 0).
+    const bool isCompositing = qt_window_private(window())->compositing;
+    if (isCompositing != testFlag(Compositing)) {
+        if (isCompositing)
+            setFlag(Compositing);
+        else
+            clearFlag(Compositing);
+
+        updateGLWindowSettings(window(), m_data.hwnd, m_data.flags, m_opacity);
+    }
+#endif
+}
+
 QT_END_NAMESPACE
diff --git a/src/plugins/platforms/windows/qwindowswindow.h b/src/plugins/platforms/windows/qwindowswindow.h
index 9822ebce45..33be287f81 100644
--- a/src/plugins/platforms/windows/qwindowswindow.h
+++ b/src/plugins/platforms/windows/qwindowswindow.h
@@ -140,7 +140,8 @@ public:
         WithinCreate = 0x20000,
         WithinMaximize = 0x40000,
         MaximizeToFullScreen = 0x80000,
-        InputMethodDisabled =0x100000
+        InputMethodDisabled = 0x100000,
+        Compositing = 0x200000
     };
 
     QWindowsWindow(QWindow *window, const QWindowsWindowData &data);
@@ -253,6 +254,7 @@ public:
     void setWindowIcon(const QIcon &icon);
 
     void *surface(void *nativeConfig);
+    void aboutToMakeCurrent();
 
 #ifndef Q_OS_WINCE
     void setAlertState(bool enabled);