macOS: Make sure NSResizableWindowMask is set when toggling fullscreen

Instead of setting the mask in toggleFullScreen(), which is only hit
when going to fullscreen via our own API, we do it in the window
notification callbacks, which also includes going to full screen via
the native macOS title bar buttons. This allows making customized
windows without Qt::WindowMaximizeButtonHint full screen with the
full geometry of the screen.

Change-Id: I63c3e4582ea7c4fe8c0008265793c5f656b830b2
Reviewed-by: Timur Pocheptsov <timur.pocheptsov@qt.io>

2 files changed, 30 insertions, 14 deletions

diff --git a/src/plugins/platforms/cocoa/qcocoawindow.h b/src/plugins/platforms/cocoa/qcocoawindow.h
index 8e2c441465..567eb7438b 100644
--- a/src/plugins/platforms/cocoa/qcocoawindow.h
+++ b/src/plugins/platforms/cocoa/qcocoawindow.h
@@ -208,7 +208,9 @@ public:
     Q_NOTIFICATION_HANDLER(NSWindowDidResignKeyNotification) void windowDidResignKey();
     Q_NOTIFICATION_HANDLER(NSWindowDidMiniaturizeNotification) void windowDidMiniaturize();
     Q_NOTIFICATION_HANDLER(NSWindowDidDeminiaturizeNotification) void windowDidDeminiaturize();
+    Q_NOTIFICATION_HANDLER(NSWindowWillEnterFullScreenNotification) void windowWillEnterFullScreen();
     Q_NOTIFICATION_HANDLER(NSWindowDidEnterFullScreenNotification) void windowDidEnterFullScreen();
+    Q_NOTIFICATION_HANDLER(NSWindowWillExitFullScreenNotification) void windowWillExitFullScreen();
     Q_NOTIFICATION_HANDLER(NSWindowDidExitFullScreenNotification) void windowDidExitFullScreen();
     Q_NOTIFICATION_HANDLER(NSWindowDidOrderOffScreenNotification) void windowDidOrderOffScreen();
     Q_NOTIFICATION_HANDLER(NSWindowDidOrderOnScreenAndFinishAnimatingNotification) void windowDidOrderOnScreen();
diff --git a/src/plugins/platforms/cocoa/qcocoawindow.mm b/src/plugins/platforms/cocoa/qcocoawindow.mm
index 2dd151e324..75c13d6bbc 100644
--- a/src/plugins/platforms/cocoa/qcocoawindow.mm
+++ b/src/plugins/platforms/cocoa/qcocoawindow.mm
@@ -947,6 +947,10 @@ NSUInteger QCocoaWindow::windowStyleMask(Qt::WindowFlags flags)
     if (m_drawContentBorderGradient)
         styleMask |= NSTexturedBackgroundWindowMask;
 
+    // Don't wipe fullscreen state
+    if (m_nsWindow.styleMask & NSFullScreenWindowMask)
+        styleMask |= NSFullScreenWindowMask;
+
     return styleMask;
 }
 
@@ -1369,19 +1373,40 @@ void QCocoaWindow::windowDidDeminiaturize()
     reportCurrentWindowState();
 }
 
+void QCocoaWindow::windowWillEnterFullScreen()
+{
+    // The NSWindow needs to be resizable, otherwise we'll end up with
+    // the normal window geometry, centered in the middle of the screen
+    // on a black background. The styleMask will be reset below.
+    m_nsWindow.styleMask |= NSResizableWindowMask;
+}
+
 void QCocoaWindow::windowDidEnterFullScreen()
 {
     Q_ASSERT_X(m_nsWindow.qt_fullScreen, "QCocoaWindow",
         "FullScreen category processes window notifications first");
 
+    // Reset to original styleMask
+    setWindowFlags(m_windowFlags);
+
     reportCurrentWindowState();
 }
 
+void QCocoaWindow::windowWillExitFullScreen()
+{
+    // The NSWindow needs to be resizable, otherwise we'll end up with
+    // a weird zoom animation. The styleMask will be reset below.
+    m_nsWindow.styleMask |= NSResizableWindowMask;
+}
+
 void QCocoaWindow::windowDidExitFullScreen()
 {
     Q_ASSERT_X(!m_nsWindow.qt_fullScreen, "QCocoaWindow",
         "FullScreen category processes window notifications first");
 
+    // Reset to original styleMask
+    setWindowFlags(m_windowFlags);
+
     Qt::WindowState requestedState = window()->windowState();
 
     // Deliver update of QWindow state
@@ -1875,24 +1900,13 @@ void QCocoaWindow::toggleMaximized()
 
 void QCocoaWindow::toggleFullScreen()
 {
-    // The NSWindow needs to be resizable, otherwise we'll end up with
-    // the normal window geometry, centered in the middle of the screen
-    // on a black background.
-    const bool wasResizable = m_nsWindow.styleMask & NSResizableWindowMask;
-    m_nsWindow.styleMask |= NSResizableWindowMask;
-
-    // It also needs to have the correct collection behavior for the
-    // toggleFullScreen call to have an effect.
-    const bool wasFullScreenEnabled = m_nsWindow.collectionBehavior & NSWindowCollectionBehaviorFullScreenPrimary;
+    // The window needs to have the correct collection behavior for the
+    // toggleFullScreen call to have an effect. The collection behavior
+    // will be reset in windowDidEnterFullScreen/windowDidLeaveFullScreen.
     m_nsWindow.collectionBehavior |= NSWindowCollectionBehaviorFullScreenPrimary;
 
     const id sender = m_nsWindow;
     [m_nsWindow toggleFullScreen:sender];
-
-    if (!wasResizable)
-        m_nsWindow.styleMask &= ~NSResizableWindowMask;
-    if (!wasFullScreenEnabled)
-        m_nsWindow.collectionBehavior &= ~NSWindowCollectionBehaviorFullScreenPrimary;
 }
 
 bool QCocoaWindow::isTransitioningToFullScreen() const