macOS: Fix logic for determining NSWindowStyleMaskTitled/Borderless

We were working under the assumption that the NSWindowStyleMaskBorderless
style mask excluded all others, on account of having the value 0. But in
practice this does not seem to be the case, and you can combine the mask
with many of the other masks. The only mask that is mutually exclusive with
the borderless mask is NSWindowStyleMaskTitled (with a value of 1).

Clarify this be restructuring QCocoaWindow::windowStyleMask().

Task-number: QTBUG-71485
Change-Id: I4bbd603fd2373c11f76e84b72a2a60aa2356b032
Reviewed-by: Doris Verria <doris.verria@qt.io>
Reviewed-by: Øystein Heskestad <oystein.heskestad@qt.io>
Reviewed-by: Volker Hilsheimer <volker.hilsheimer@qt.io>

1 files changed, 32 insertions, 11 deletions

diff --git a/src/plugins/platforms/cocoa/qcocoawindow.mm b/src/plugins/platforms/cocoa/qcocoawindow.mm
index 6e9166a05a..a1cc8052f1 100644
--- a/src/plugins/platforms/cocoa/qcocoawindow.mm
+++ b/src/plugins/platforms/cocoa/qcocoawindow.mm
@@ -485,31 +485,51 @@ NSInteger QCocoaWindow::windowLevel(Qt::WindowFlags flags)
 NSUInteger QCocoaWindow::windowStyleMask(Qt::WindowFlags flags)
 {
     const Qt::WindowType type = static_cast<Qt::WindowType>(int(flags & Qt::WindowType_Mask));
-    const bool frameless = (flags & Qt::FramelessWindowHint) || windowIsPopupType(type);
 
-    // Remove zoom button by disabling resize for CustomizeWindowHint windows, except for
-    // Qt::Tool windows (e.g. dock windows) which should always be resizable.
-    const bool resizable = !(flags & Qt::CustomizeWindowHint) || (type == Qt::Tool);
-
-    // Select base window type. Note that the value of NSBorderlessWindowMask is 0.
-    NSUInteger styleMask = (frameless || !resizable) ? NSWindowStyleMaskBorderless : NSWindowStyleMaskResizable;
+    // Determine initial style mask based on whether the window should
+    // have a frame and title or not. The NSWindowStyleMaskBorderless
+    // and NSWindowStyleMaskTitled styles are mutually exclusive, with
+    // values of 0 and 1 correspondingly.
+    NSUInteger styleMask = [&]{
+        // Honor explicit requests for borderless windows
+        if (flags & Qt::FramelessWindowHint)
+            return NSWindowStyleMaskBorderless;
+
+        // Popup windows should always be borderless
+        if (windowIsPopupType(type))
+            return NSWindowStyleMaskBorderless;
+
+        if (flags & Qt::CustomizeWindowHint) {
+            // CustomizeWindowHint turns off the default window title hints,
+            // so the choice is then up to the user via Qt::WindowTitleHint.
+            return flags & Qt::WindowTitleHint
+                ? NSWindowStyleMaskTitled
+                : NSWindowStyleMaskBorderless;
+        } else {
+            // Otherwise, default to using titled windows
+            return NSWindowStyleMaskTitled;
+        }
+    }();
 
-    if (frameless) {
+    // FIXME: Control visibility of buttons directly, instead of affecting styleMask
+    if (styleMask == NSWindowStyleMaskBorderless) {
         // Frameless windows do not display the traffic lights buttons for
         // e.g. minimize, however StyleMaskMiniaturizable is required to allow
         // programmatic minimize.
         styleMask |= NSWindowStyleMaskMiniaturizable;
     } else if (flags & Qt::CustomizeWindowHint) {
-        if (flags & Qt::WindowTitleHint)
-            styleMask |= NSWindowStyleMaskTitled;
         if (flags & Qt::WindowCloseButtonHint)
             styleMask |= NSWindowStyleMaskClosable;
         if (flags & Qt::WindowMinimizeButtonHint)
             styleMask |= NSWindowStyleMaskMiniaturizable;
         if (flags & Qt::WindowMaximizeButtonHint)
             styleMask |= NSWindowStyleMaskResizable;
+
+        // Force tool windows to be resizable
+        if (type == Qt::Tool)
+            styleMask |= NSWindowStyleMaskResizable;
     } else {
-        styleMask |= NSWindowStyleMaskClosable | NSWindowStyleMaskTitled;
+        styleMask |= NSWindowStyleMaskClosable | NSWindowStyleMaskResizable;
 
         if (type != Qt::Dialog)
             styleMask |= NSWindowStyleMaskMiniaturizable;
@@ -518,6 +538,7 @@ NSUInteger QCocoaWindow::windowStyleMask(Qt::WindowFlags flags)
     if (type == Qt::Tool)
         styleMask |= NSWindowStyleMaskUtilityWindow;
 
+    // FIXME: Remove use of deprecated style mask
     if (m_drawContentBorderGradient)
         styleMask |= NSWindowStyleMaskTexturedBackground;