macOS: Rewrite window state handling

Instead of relying on specific notifications to change the window
state we now evaluate the state based on the current window state.

This allows us to get rid of windowShouldZoom in the window delegate,
making window state handling work for foreign windows as well, and
also allows us to re-evaluate the state in more places, such as
when moving a window, which may bring it out of maximized state.

The full screen state is tracked by a helper category that doesn't
just rely on the styleFlag, but also on the full screen notifications.
This is needed as macOS will complain if you try to go in or out of
fullscreen while a transition is in effect.

The differentiation between performFoo: and foo: has been removed,
as the latter works in both cases and doesn't rely on the button
being visible/enabled.

These changes fixes many observed quirks in the window state handling
that also resulted in making it hard to write tests that relied on
the fullscreen/maximized operations always working.

Change-Id: I0538c42d9223a56f20ec9156f4939288e0750552
Reviewed-by: Timur Pocheptsov <timur.pocheptsov@qt.io>
Reviewed-by: Tor Arne Vestbø <tor.arne.vestbo@qt.io>

6 files changed, 208 insertions, 110 deletions

diff --git a/src/plugins/platforms/cocoa/qcocoawindow.h b/src/plugins/platforms/cocoa/qcocoawindow.h
index b057299d0e..f60597fe42 100644
--- a/src/plugins/platforms/cocoa/qcocoawindow.h
+++ b/src/plugins/platforms/cocoa/qcocoawindow.h
@@ -217,7 +217,7 @@ public:
     bool windowShouldClose();
     bool windowIsPopupType(Qt::WindowType type = Qt::Widget) const;
 
-    void handleWindowStateChanged(Qt::WindowState);
+    void reportCurrentWindowState(bool unconditionally = false);
 
     NSInteger windowLevel(Qt::WindowFlags flags);
     NSUInteger windowStyleMask(Qt::WindowFlags flags);
@@ -284,10 +284,15 @@ protected:
     QCocoaNSWindow *createNSWindow(bool shouldBeChildNSWindow, bool shouldBePanel);
 
     QRect nativeWindowGeometry() const;
-    void syncWindowState(Qt::WindowState newState);
     void reinsertChildWindow(QCocoaWindow *child);
     void removeChildWindow(QCocoaWindow *child);
 
+    Qt::WindowState windowState() const;
+    void applyWindowState(Qt::WindowState newState);
+    void toggleMaximized();
+    void toggleFullScreen();
+    bool isTransitioningToFullScreen() const;
+
 // private:
 public: // for QNSView
     friend class QCocoaBackingStore;
@@ -304,8 +309,7 @@ public: // for QNSView
     bool m_viewIsToBeEmbedded; // true if the m_view is intended to be embedded in a "foreign" NSView hiearchy
 
     Qt::WindowFlags m_windowFlags;
-    bool m_effectivelyMaximized;
-    Qt::WindowState m_synchedWindowState;
+    Qt::WindowState m_lastReportedWindowState;
     Qt::WindowModality m_windowModality;
     QPointer<QWindow> m_enterLeaveTargetWindow;
     bool m_windowUnderMouse;
@@ -339,11 +343,6 @@ public: // for QNSView
     int m_topContentBorderThickness;
     int m_bottomContentBorderThickness;
 
-    // used by showFullScreen in fake mode
-    QRect m_normalGeometry;
-    Qt::WindowFlags m_oldWindowFlags;
-    NSApplicationPresentationOptions m_presentationOptions;
-
     struct BorderRange {
         BorderRange(quintptr i, int u, int l) : identifier(i), upper(u), lower(l) { }
         quintptr identifier;
diff --git a/src/plugins/platforms/cocoa/qcocoawindow.mm b/src/plugins/platforms/cocoa/qcocoawindow.mm
index 40159a6e01..8b8700eca8 100644
--- a/src/plugins/platforms/cocoa/qcocoawindow.mm
+++ b/src/plugins/platforms/cocoa/qcocoawindow.mm
@@ -87,6 +87,36 @@ static void qt_closePopups()
     }
 }
 
+@interface NSWindow (FullScreenProperty)
+@property(readonly) BOOL qt_fullScreen;
+@end
+
+@implementation NSWindow (FullScreenProperty)
+
++ (void)load
+{
+    NSNotificationCenter *center = [NSNotificationCenter defaultCenter];
+    [center addObserverForName:NSWindowDidEnterFullScreenNotification object:nil queue:nil
+        usingBlock:^(NSNotification *notification) {
+            objc_setAssociatedObject(notification.object, @selector(qt_fullScreen),
+                [NSNumber numberWithBool:YES], OBJC_ASSOCIATION_RETAIN);
+        }
+    ];
+    [center addObserverForName:NSWindowDidExitFullScreenNotification object:nil queue:nil
+        usingBlock:^(NSNotification *notification) {
+            objc_setAssociatedObject(notification.object, @selector(qt_fullScreen),
+                nil, OBJC_ASSOCIATION_RETAIN);
+        }
+    ];
+}
+
+- (BOOL)qt_fullScreen
+{
+    NSNumber *number = objc_getAssociatedObject(self, @selector(qt_fullScreen));
+    return [number boolValue];
+}
+@end
+
 @implementation QNSWindowHelper
 
 @synthesize window = _window;
@@ -409,8 +439,7 @@ QCocoaWindow::QCocoaWindow(QWindow *tlw)
     , m_nsWindow(0)
     , m_viewIsEmbedded(false)
     , m_viewIsToBeEmbedded(false)
-    , m_effectivelyMaximized(false)
-    , m_synchedWindowState(Qt::WindowActive)
+    , m_lastReportedWindowState(Qt::WindowNoState)
     , m_windowModality(Qt::NonModal)
     , m_windowUnderMouse(false)
     , m_inConstructor(true)
@@ -435,7 +464,6 @@ QCocoaWindow::QCocoaWindow(QWindow *tlw)
     , m_drawContentBorderGradient(false)
     , m_topContentBorderThickness(0)
     , m_bottomContentBorderThickness(0)
-    , m_normalGeometry(QRect(0,0,-1,-1))
     , m_hasWindowFilePath(false)
 {
     qCDebug(lcQpaCocoaWindow) << "QCocoaWindow::QCocoaWindow" << window();
@@ -728,7 +756,7 @@ void QCocoaWindow::setVisible(bool visible)
 
             // setWindowState might have been called while the window was hidden and
             // will not change the NSWindow state in that case. Sync up here:
-            syncWindowState(window()->windowState());
+            applyWindowState(window()->windowState());
 
             if (window()->windowState() != Qt::WindowMinimized) {
                 if ((window()->modality() == Qt::WindowModal
@@ -972,7 +1000,7 @@ void QCocoaWindow::setWindowFlags(Qt::WindowFlags flags)
 void QCocoaWindow::setWindowState(Qt::WindowState state)
 {
     if (window()->isVisible())
-        syncWindowState(state);  // Window state set for hidden windows take effect when show() is called.
+        applyWindowState(state); // Window state set for hidden windows take effect when show() is called
 }
 
 void QCocoaWindow::setWindowTitle(const QString &title)
@@ -1248,6 +1276,9 @@ void QCocoaWindow::windowDidMove()
         return;
 
     [qnsview_cast(m_view) updateGeometry];
+
+    // Moving a window might bring it out of maximized state
+    reportCurrentWindowState();
 }
 
 void QCocoaWindow::windowDidResize()
@@ -1260,6 +1291,9 @@ void QCocoaWindow::windowDidResize()
 
     clipChildWindows();
     [qnsview_cast(m_view) updateGeometry];
+
+    if (!m_view.inLiveResize)
+        reportCurrentWindowState();
 }
 
 void QCocoaWindow::viewDidChangeFrame()
@@ -1281,10 +1315,7 @@ void QCocoaWindow::viewDidChangeGlobalFrame()
 
 void QCocoaWindow::windowDidEndLiveResize()
 {
-    if (m_synchedWindowState == Qt::WindowMaximized && ![m_nsWindow isZoomed]) {
-        m_effectivelyMaximized = false;
-        handleWindowStateChanged(Qt::WindowNoState);
-    }
+    reportCurrentWindowState();
 }
 
 void QCocoaWindow::windowDidBecomeKey()
@@ -1321,22 +1352,37 @@ void QCocoaWindow::windowDidResignKey()
 
 void QCocoaWindow::windowDidMiniaturize()
 {
-    handleWindowStateChanged(Qt::WindowMinimized);
+    reportCurrentWindowState();
 }
 
 void QCocoaWindow::windowDidDeminiaturize()
 {
-    handleWindowStateChanged(Qt::WindowNoState);
+    reportCurrentWindowState();
 }
 
 void QCocoaWindow::windowDidEnterFullScreen()
 {
-    handleWindowStateChanged(Qt::WindowFullScreen);
+    Q_ASSERT_X(m_nsWindow.qt_fullScreen, "QCocoaWindow",
+        "FullScreen category processes window notifications first");
+
+    reportCurrentWindowState();
 }
 
 void QCocoaWindow::windowDidExitFullScreen()
 {
-    handleWindowStateChanged(Qt::WindowNoState);
+    Q_ASSERT_X(!m_nsWindow.qt_fullScreen, "QCocoaWindow",
+        "FullScreen category processes window notifications first");
+
+    Qt::WindowState requestedState = window()->windowState();
+
+    // Deliver update of QWindow state
+    reportCurrentWindowState();
+
+    if (requestedState != windowState() && requestedState != Qt::WindowFullScreen) {
+        // We were only going out of full screen as an intermediate step before
+        // progressing into the final step, so re-sync the desired state.
+       applyWindowState(requestedState);
+    }
 }
 
 void QCocoaWindow::windowDidOrderOffScreen()
@@ -1748,99 +1794,155 @@ QRect QCocoaWindow::nativeWindowGeometry() const
     return qRect;
 }
 
-// Syncs the NSWindow minimize/maximize/fullscreen state with the current QWindow state
-void QCocoaWindow::syncWindowState(Qt::WindowState newState)
+/*!
+    Applies the given state to the NSWindow, going in/out of minimize/zoomed/fullscreen
+
+    When this is called from QWindow::setWindowState(), the QWindow state has not been
+    updated yet, so window()->windowState() will reflect the previous state that was
+    reported to QtGui.
+*/
+void QCocoaWindow::applyWindowState(Qt::WindowState newState)
 {
+    const Qt::WindowState currentState = windowState();
+    if (newState == currentState)
+        return;
+
     if (!m_nsWindow)
         return;
-    // if content view width or height is 0 then the window animations will crash so
-    // do nothing except set the new state
-    NSRect contentRect = m_view.frame;
-    if (contentRect.size.width <= 0 || contentRect.size.height <= 0) {
+
+    const NSSize contentSize = m_view.frame.size;
+    if (contentSize.width <= 0 || contentSize.height <= 0) {
+        // If content view width or height is 0 then the window animations will crash so
+        // do nothing. We report the current state back to reflect the failed operation.
         qWarning("invalid window content view size, check your window geometry");
-        m_synchedWindowState = newState;
+        reportCurrentWindowState(true);
         return;
     }
 
-    Qt::WindowState predictedState = newState;
-    if ((m_synchedWindowState & Qt::WindowMaximized) != (newState & Qt::WindowMaximized)) {
-        const int styleMask = [m_nsWindow styleMask];
-        const bool usePerform = styleMask & NSResizableWindowMask;
-        [m_nsWindow setStyleMask:styleMask | NSResizableWindowMask];
-        if (usePerform)
-            [m_nsWindow performZoom : m_nsWindow]; // toggles
-        else
-            [m_nsWindow zoom : m_nsWindow]; // toggles
-        [m_nsWindow setStyleMask:styleMask];
+    if (m_nsWindow.styleMask & NSUtilityWindowMask) {
+        // Utility panels cannot be fullscreen
+        qWarning() << window()->type() << "windows can not be made full screen";
+        reportCurrentWindowState(true);
+        return;
     }
 
-    if ((m_synchedWindowState & Qt::WindowMinimized) != (newState & Qt::WindowMinimized)) {
-        if (newState & Qt::WindowMinimized) {
-            if ([m_nsWindow styleMask] & NSMiniaturizableWindowMask)
-                [m_nsWindow performMiniaturize : m_nsWindow];
-            else
-                [m_nsWindow miniaturize : m_nsWindow];
-        } else {
-            [m_nsWindow deminiaturize : m_nsWindow];
-        }
+    const id sender = m_nsWindow;
+
+    // First we need to exit states that can't transition directly to other states
+    switch (currentState) {
+    case Qt::WindowMinimized:
+        [m_nsWindow deminiaturize:sender];
+        Q_ASSERT_X(windowState() != Qt::WindowMinimized, "QCocoaWindow",
+            "[NSWindow deminiaturize:] is synchronous");
+        break;
+    case Qt::WindowFullScreen: {
+        toggleFullScreen();
+        // Exiting fullscreen is not synchronous, so we need to wait for the
+        // NSWindowDidExitFullScreenNotification before continuing to apply
+        // the new state.
+        return;
     }
-
-    const bool effMax = m_effectivelyMaximized;
-    if ((m_synchedWindowState & Qt::WindowMaximized) != (newState & Qt::WindowMaximized) || (m_effectivelyMaximized && newState == Qt::WindowNoState)) {
-        if ((m_synchedWindowState & Qt::WindowFullScreen) == (newState & Qt::WindowFullScreen)) {
-            [m_nsWindow zoom : m_nsWindow]; // toggles
-            m_effectivelyMaximized = !effMax;
-        } else if (!(newState & Qt::WindowMaximized)) {
-            // it would be nice to change the target geometry that toggleFullScreen will animate toward
-            // but there is no known way, so the maximized state is not possible at this time
-            predictedState = static_cast<Qt::WindowState>(static_cast<int>(newState) | Qt::WindowMaximized);
-            m_effectivelyMaximized = true;
-        }
+    default:
+        Q_FALLTHROUGH();
     }
 
-    if ((m_synchedWindowState & Qt::WindowFullScreen) != (newState & Qt::WindowFullScreen)) {
-        if (window()->flags() & Qt::WindowFullscreenButtonHint) {
-            if (m_effectivelyMaximized && m_synchedWindowState == Qt::WindowFullScreen)
-                predictedState = Qt::WindowMaximized;
-            [m_nsWindow toggleFullScreen : m_nsWindow];
-        } else {
-            if (newState & Qt::WindowFullScreen) {
-                QScreen *screen = window()->screen();
-                if (screen) {
-                    if (m_normalGeometry.width() < 0) {
-                        m_oldWindowFlags = m_windowFlags;
-                        window()->setFlags(window()->flags() | Qt::FramelessWindowHint);
-                        m_normalGeometry = nativeWindowGeometry();
-                        setGeometry(screen->geometry());
-                        m_presentationOptions = [NSApp presentationOptions];
-                        [NSApp setPresentationOptions : m_presentationOptions | NSApplicationPresentationAutoHideMenuBar | NSApplicationPresentationAutoHideDock];
-                    }
-                }
-            } else {
-                window()->setFlags(m_oldWindowFlags);
-                setGeometry(m_normalGeometry);
-                m_normalGeometry.setRect(0, 0, -1, -1);
-                [NSApp setPresentationOptions : m_presentationOptions];
-            }
+    // Then we apply the new state if needed
+    if (newState == windowState())
+        return;
+
+    switch (newState) {
+    case Qt::WindowFullScreen:
+        toggleFullScreen();
+        break;
+    case Qt::WindowMaximized:
+        toggleMaximized();
+        break;
+    case Qt::WindowMinimized:
+        [m_nsWindow miniaturize:sender];
+        break;
+    case Qt::WindowNoState:
+        switch (windowState()) {
+        case Qt::WindowMaximized:
+            toggleMaximized();
+        default:
+            Q_FALLTHROUGH();
         }
+        break;
+    default:
+        Q_UNREACHABLE();
     }
+}
+
+void QCocoaWindow::toggleMaximized()
+{
+    // The NSWindow needs to be resizable, otherwise the window will
+    // not be possible to zoom back to non-zoomed state.
+    const bool wasResizable = m_nsWindow.styleMask & NSResizableWindowMask;
+    m_nsWindow.styleMask |= NSResizableWindowMask;
+
+    const id sender = m_nsWindow;
+    [m_nsWindow zoom:sender];
 
-    // New state is now the current synched state
-    m_synchedWindowState = predictedState;
+    if (!wasResizable)
+        m_nsWindow.styleMask &= ~NSResizableWindowMask;
 }
 
-void QCocoaWindow::handleWindowStateChanged(Qt::WindowState newState)
+void QCocoaWindow::toggleFullScreen()
 {
-    // If the window was maximized, then fullscreen, then tried to go directly to "normal" state,
-    // this notification will say that it is "normal", but it will still look maximized, and
-    // if you called performZoom it would actually take it back to "normal".
-    // So we should say that it is maximized because it actually is.
-    if (newState == Qt::WindowNoState && m_effectivelyMaximized)
-        newState = Qt::WindowMaximized;
+    // The NSWindow needs to be resizable, otherwise we'll end up with
+    // the normal window geometry, centered in the middle of the screen
+    // on a black background.
+    const bool wasResizable = m_nsWindow.styleMask & NSResizableWindowMask;
+    m_nsWindow.styleMask |= NSResizableWindowMask;
 
-    QWindowSystemInterface::handleWindowStateChanged<QWindowSystemInterface::SynchronousDelivery>(window(), newState);
+    // It also needs to have the correct collection behavior for the
+    // toggleFullScreen call to have an effect.
+    const bool wasFullScreenEnabled = m_nsWindow.collectionBehavior & NSWindowCollectionBehaviorFullScreenPrimary;
+    m_nsWindow.collectionBehavior |= NSWindowCollectionBehaviorFullScreenPrimary;
+
+    const id sender = m_nsWindow;
+    [m_nsWindow toggleFullScreen:sender];
+
+    if (!wasResizable)
+        m_nsWindow.styleMask &= ~NSResizableWindowMask;
+    if (!wasFullScreenEnabled)
+        m_nsWindow.collectionBehavior &= ~NSWindowCollectionBehaviorFullScreenPrimary;
+}
+
+bool QCocoaWindow::isTransitioningToFullScreen() const
+{
+    NSWindow *window = m_view.window;
+    return window.styleMask & NSFullScreenWindowMask && !window.qt_fullScreen;
+}
+
+Qt::WindowState QCocoaWindow::windowState() const
+{
+    // FIXME: Support compound states (Qt::WindowStates)
+
+    NSWindow *window = m_view.window;
+    if (window.miniaturized)
+        return Qt::WindowMinimized;
+    if (window.qt_fullScreen)
+        return Qt::WindowFullScreen;
+    if ((window.zoomed && !isTransitioningToFullScreen())
+        || (m_lastReportedWindowState == Qt::WindowMaximized && isTransitioningToFullScreen()))
+        return Qt::WindowMaximized;
+
+    // Note: We do not report Qt::WindowActive, even if isActive()
+    // is true, as QtGui does not expect this window state to be set.
+
+    return Qt::WindowNoState;
+}
+
+void QCocoaWindow::reportCurrentWindowState(bool unconditionally)
+{
+    Qt::WindowState currentState = windowState();
+    if (!unconditionally && currentState == m_lastReportedWindowState)
+        return;
 
-    m_synchedWindowState = window()->windowState();
+    QWindowSystemInterface::handleWindowStateChanged<QWindowSystemInterface::SynchronousDelivery>(
+        window(), currentState, m_lastReportedWindowState);
+    m_lastReportedWindowState = currentState;
 }
 
 bool QCocoaWindow::setWindowModified(bool modified)
diff --git a/src/plugins/platforms/cocoa/qnsview.h b/src/plugins/platforms/cocoa/qnsview.h
index a33e1a9de8..75a508370f 100644
--- a/src/plugins/platforms/cocoa/qnsview.h
+++ b/src/plugins/platforms/cocoa/qnsview.h
@@ -100,7 +100,6 @@ Q_FORWARD_DECLARE_OBJC_CLASS(QT_MANGLE_NAMESPACE(QNSViewMouseMoveHelper));
 - (void)drawRect:(NSRect)dirtyRect;
 - (void)drawBackingStoreUsingCoreGraphics:(NSRect)dirtyRect;
 - (void)updateGeometry;
-- (void)notifyWindowWillZoom:(BOOL)willZoom;
 - (void)textInputContextKeyboardSelectionDidChangeNotification : (NSNotification *) textInputContextKeyboardSelectionDidChangeNotification;
 - (void)viewDidHide;
 - (void)viewDidUnhide;
diff --git a/src/plugins/platforms/cocoa/qnsview.mm b/src/plugins/platforms/cocoa/qnsview.mm
index 2009096aaa..fbf4f99f2e 100644
--- a/src/plugins/platforms/cocoa/qnsview.mm
+++ b/src/plugins/platforms/cocoa/qnsview.mm
@@ -348,14 +348,6 @@ static bool _q_dontOverrideCtrlLMB = false;
     }
 }
 
-- (void)notifyWindowWillZoom:(BOOL)willZoom
-{
-    Qt::WindowState newState = willZoom ? Qt::WindowMaximized : Qt::WindowNoState;
-    if (!willZoom)
-        m_platformWindow->m_effectivelyMaximized = false;
-    m_platformWindow->handleWindowStateChanged(newState);
-}
-
 - (void)viewDidHide
 {
     m_platformWindow->obscureWindow();
diff --git a/src/plugins/platforms/cocoa/qnswindowdelegate.h b/src/plugins/platforms/cocoa/qnswindowdelegate.h
index a465b249c5..d2078b5786 100644
--- a/src/plugins/platforms/cocoa/qnswindowdelegate.h
+++ b/src/plugins/platforms/cocoa/qnswindowdelegate.h
@@ -52,7 +52,6 @@
 - (id)initWithQCocoaWindow:(QCocoaWindow *)cocoaWindow;
 
 - (BOOL)windowShouldClose:(NSNotification *)notification;
-- (BOOL)windowShouldZoom:(NSWindow *)window toFrame:(NSRect)newFrame;
 
 - (BOOL)window:(NSWindow *)window shouldPopUpDocumentPathMenu:(NSMenu *)menu;
 - (BOOL)window:(NSWindow *)window shouldDragDocumentWithEvent:(NSEvent *)event from:(NSPoint)dragImageLocation withPasteboard:(NSPasteboard *)pasteboard;
diff --git a/src/plugins/platforms/cocoa/qnswindowdelegate.mm b/src/plugins/platforms/cocoa/qnswindowdelegate.mm
index 2d5afa9375..ce74aa9973 100644
--- a/src/plugins/platforms/cocoa/qnswindowdelegate.mm
+++ b/src/plugins/platforms/cocoa/qnswindowdelegate.mm
@@ -41,6 +41,7 @@
 #include "qcocoahelpers.h"
 
 #include <QDebug>
+#include <qpa/qplatformscreen.h>
 #include <qpa/qwindowsysteminterface.h>
 
 @implementation QNSWindowDelegate
@@ -62,13 +63,19 @@
 
     return YES;
 }
-
-- (BOOL)windowShouldZoom:(NSWindow *)window toFrame:(NSRect)newFrame
+/*!
+    Overridden to ensure that the zoomed state always results in a maximized
+    window, which would otherwise not be the case for borderless windows.
+*/
+- (NSRect)windowWillUseStandardFrame:(NSWindow *)window defaultFrame:(NSRect)newFrame
 {
     Q_UNUSED(newFrame);
-    if (m_cocoaWindow && !m_cocoaWindow->isForeignWindow())
-        [qnsview_cast(m_cocoaWindow->view()) notifyWindowWillZoom:![window isZoomed]];
-    return YES;
+
+    // We explicitly go through the QScreen API here instead of just using
+    // window.screen.visibleFrame directly, as that ensures we have the same
+    // behavior for both use-cases/APIs.
+    Q_ASSERT(window == m_cocoaWindow->nativeWindow());
+    return m_cocoaWindow->screen()->availableGeometry().toCGRect();
 }
 
 - (BOOL)window:(NSWindow *)window shouldPopUpDocumentPathMenu:(NSMenu *)menu