macOS: Handle window titlebar buttons independently from style mask

Style masks such as NSWindowStyleMask{Resizable,Miniaturizable}
affect whether the window has a title bar button for the action,
but also whether the window can be resized or minimized through
other means, for example if the window border can be dragged
to resize.

By decoupling the visibility and enablement of the title bar buttons
from the style mask we can individually control the buttons, and
leave the style mask set to enable behaviors we always want. We
were already doing this for the NSWindowZoomButton.

Unfortunately AppKit not only checks NSWindowStyleMaskMiniaturizable
during a call to miniaturize, but also whether the title bar button
is enabled. To allow minimizing windows without the titlebar button
we detect the situation and give AppKit a NSWindowMiniaturizeButton
that we haven't disabled. The alternative would be to temporarily
enable the NSWindowMiniaturizeButton during the minimize, but this
results in the button flashing yellow for the duration of the
animation.

Task-number: QTBUG-65637
Task-number: QTBUG-46882
Task-number: QTBUG-64994
Task-number: QTBUG-71485
Change-Id: I2c1a9564d8b7516476aa018b2820670199124bc2
Reviewed-by: Volker Hilsheimer <volker.hilsheimer@qt.io>

3 files changed, 71 insertions, 36 deletions

diff --git a/src/plugins/platforms/cocoa/qcocoawindow.h b/src/plugins/platforms/cocoa/qcocoawindow.h
index e097a69c4d..a36f85d620 100644
--- a/src/plugins/platforms/cocoa/qcocoawindow.h
+++ b/src/plugins/platforms/cocoa/qcocoawindow.h
@@ -174,7 +174,8 @@ public:
 
     NSInteger windowLevel(Qt::WindowFlags flags);
     NSUInteger windowStyleMask(Qt::WindowFlags flags);
-    void setWindowZoomButton(Qt::WindowFlags flags);
+    void updateTitleBarButtons(Qt::WindowFlags flags);
+    bool isFixedSize() const;
 
     bool setWindowModified(bool modified) override;
 
diff --git a/src/plugins/platforms/cocoa/qcocoawindow.mm b/src/plugins/platforms/cocoa/qcocoawindow.mm
index f5f7ae6bff..16e33ee014 100644
--- a/src/plugins/platforms/cocoa/qcocoawindow.mm
+++ b/src/plugins/platforms/cocoa/qcocoawindow.mm
@@ -511,29 +511,14 @@ NSUInteger QCocoaWindow::windowStyleMask(Qt::WindowFlags flags)
         }
     }();
 
-    // FIXME: Control visibility of buttons directly, instead of affecting styleMask
-    if (styleMask == NSWindowStyleMaskBorderless) {
-        // Frameless windows do not display the traffic lights buttons for
-        // e.g. minimize, however StyleMaskMiniaturizable is required to allow
-        // programmatic minimize.
-        styleMask |= NSWindowStyleMaskMiniaturizable;
-    } else if (flags & Qt::CustomizeWindowHint) {
-        if (flags & Qt::WindowCloseButtonHint)
-            styleMask |= NSWindowStyleMaskClosable;
-        if (flags & Qt::WindowMinimizeButtonHint)
-            styleMask |= NSWindowStyleMaskMiniaturizable;
-        if (flags & Qt::WindowMaximizeButtonHint)
-            styleMask |= NSWindowStyleMaskResizable;
-
-        // Force tool windows to be resizable
-        if (type == Qt::Tool)
-            styleMask |= NSWindowStyleMaskResizable;
-    } else {
-        styleMask |= NSWindowStyleMaskClosable | NSWindowStyleMaskResizable;
-
-        if (type != Qt::Dialog)
-            styleMask |= NSWindowStyleMaskMiniaturizable;
-    }
+    // We determine which buttons to show in updateTitleBarButtons,
+    // so we can enable all the relevant style masks here to ensure
+    // that behaviors that don't involve the title bar buttons are
+    // working (for example minimizing frameless windows, or resizing
+    // windows that don't have zoom or fullscreen titlebar buttons).
+    styleMask |= NSWindowStyleMaskClosable
+               | NSWindowStyleMaskResizable
+               | NSWindowStyleMaskMiniaturizable;
 
     if (type == Qt::Tool)
         styleMask |= NSWindowStyleMaskUtilityWindow;
@@ -551,20 +536,41 @@ NSUInteger QCocoaWindow::windowStyleMask(Qt::WindowFlags flags)
     return styleMask;
 }
 
-void QCocoaWindow::setWindowZoomButton(Qt::WindowFlags flags)
+bool QCocoaWindow::isFixedSize() const
+{
+    return windowMinimumSize().isValid() && windowMaximumSize().isValid()
+        && windowMinimumSize() == windowMaximumSize();
+}
+
+void QCocoaWindow::updateTitleBarButtons(Qt::WindowFlags windowFlags)
 {
     if (!isContentView())
         return;
 
-    // Disable the zoom (maximize) button for fixed-sized windows and customized
-    // no-WindowMaximizeButtonHint windows. From a Qt perspective it migth be expected
-    // that the button would be removed in the latter case, but disabling it is more
-    // in line with the platform style guidelines.
-    bool fixedSizeNoZoom = (windowMinimumSize().isValid() && windowMaximumSize().isValid()
-                            && windowMinimumSize() == windowMaximumSize());
-    bool customizeNoZoom = ((flags & Qt::CustomizeWindowHint)
-        && !(flags & (Qt::WindowMaximizeButtonHint | Qt::WindowFullscreenButtonHint)));
-    [[m_view.window standardWindowButton:NSWindowZoomButton] setEnabled:!(fixedSizeNoZoom || customizeNoZoom)];
+    NSWindow *window = m_view.window;
+
+    static constexpr std::pair<NSWindowButton, Qt::WindowFlags> buttons[] = {
+        { NSWindowCloseButton, Qt::WindowCloseButtonHint },
+        { NSWindowMiniaturizeButton, Qt::WindowMinimizeButtonHint},
+        { NSWindowZoomButton, Qt::WindowMaximizeButtonHint | Qt::WindowFullscreenButtonHint }
+    };
+
+    bool hideButtons = true;
+    for (const auto &[button, buttonHint] : buttons) {
+        bool enabled = true;
+        if (windowFlags & Qt::CustomizeWindowHint)
+            enabled = windowFlags & buttonHint;
+
+        if (button == NSWindowZoomButton && isFixedSize())
+            enabled = false;
+
+        [window standardWindowButton:button].enabled = enabled;
+        hideButtons &= !enabled;
+    }
+
+    // Hide buttons in case we disabled all of them
+    for (const auto &[button, buttonHint] : buttons)
+        [window standardWindowButton:button].hidden = hideButtons;
 }
 
 void QCocoaWindow::setWindowFlags(Qt::WindowFlags flags)
@@ -609,7 +615,7 @@ void QCocoaWindow::setWindowFlags(Qt::WindowFlags flags)
     if (!(flags & Qt::FramelessWindowHint))
         setWindowTitle(window()->title());
 
-    setWindowZoomButton(flags);
+    updateTitleBarButtons(flags);
 
     // Make window ignore mouse events if WindowTransparentForInput is set.
     // Note that ignoresMouseEvents has a special initial state where events
@@ -1016,7 +1022,7 @@ void QCocoaWindow::propagateSizeHints()
     window.contentMaxSize = NSSizeFromCGSize(windowMaximumSize().toCGSize());
 
     // The window may end up with a fixed size; in this case the zoom button should be disabled.
-    setWindowZoomButton(this->window()->flags());
+    updateTitleBarButtons(this->window()->flags());
 
     // sizeIncrement is observed to take values of (-1, -1) and (0, 0) for windows that should be
     // resizable and that have no specific size increment set. Cocoa expects (1.0, 1.0) in this case.
diff --git a/src/plugins/platforms/cocoa/qnswindow.mm b/src/plugins/platforms/cocoa/qnswindow.mm
index bdd2c07f02..7c9e0dce2d 100644
--- a/src/plugins/platforms/cocoa/qnswindow.mm
+++ b/src/plugins/platforms/cocoa/qnswindow.mm
@@ -249,6 +249,7 @@ OSStatus CGSClearWindowTags(const CGSConnectionID, const CGSWindowID, int *, int
 {
     // Member variables
     QPointer<QCocoaWindow> m_platformWindow;
+    bool m_isMinimizing;
 }
 
 - (instancetype)initWithContentRect:(NSRect)contentRect styleMask:(NSWindowStyleMask)style
@@ -260,6 +261,8 @@ OSStatus CGSClearWindowTags(const CGSConnectionID, const CGSWindowID, int *, int
     // we can properly reflect the window's state during initialization.
     m_platformWindow = window;
 
+    m_isMinimizing = false;
+
     return [super initWithContentRect:contentRect styleMask:style backing:backingStoreType defer:defer screen:screen];
 }
 
@@ -395,6 +398,31 @@ OSStatus CGSClearWindowTags(const CGSConnectionID, const CGSWindowID, int *, int
         [qnsview_cast(m_platformWindow->view()) handleFrameStrutMouseEvent:theEvent];
 }
 
+- (void)miniaturize:(id)sender
+{
+    QBoolBlocker miniaturizeTracker(m_isMinimizing, true);
+    [super miniaturize:sender];
+}
+
+- (NSButton *)standardWindowButton:(NSWindowButton)buttonType
+{
+    NSButton *button = [super standardWindowButton:buttonType];
+
+    // When an NSWindow is asked to minimize it will check the
+    // NSWindowMiniaturizeButton for enablement before continuing,
+    // even if the style mask includes NSWindowStyleMaskMiniaturizable.
+    // To ensure that a window can be minimized, even when the
+    // minimize button has been disabled in response to the user
+    // setting CustomizeWindowHint, we temporarily return a default
+    // minimize-button that we haven't modified in updateTitleBarButtons.
+    // This ensures the window can be minimized, without visually
+    // toggling the actual minimize-button on and off.
+    if (buttonType == NSWindowMiniaturizeButton && m_isMinimizing && !button.enabled)
+        return [NSWindow standardWindowButton:buttonType forStyleMask:self.styleMask];
+
+    return button;
+}
+
 - (void)closeAndRelease
 {
     qCDebug(lcQpaWindow) << "Closing and releasing" << self;