macOS: Simplify and correct style mask determination

This function had accumulated a fair bit of accidental
complexity over the years.

- No early returns, make sure to preserve fullscreen
  state for all windows.

- Use windowIsPopupType() directly to set borderless
  for Qt::Popup (but not Qt::Tool).

- Handle Qt::Tool explicitly.

- Deduplicate Qt::CustomizeWindowMask handling.

- Remove case that used the absence of NSResizableWindowMask
  to remove the maximize button. Maximize is now
  disabled elsewhere (setWindowZoomButton). All windows
  now get NSResizableWindowMask by default.

- Qt::ForeignWindow now gets a standard window style
  mask instead of NSBorderlessWindowMask. The old
  code did not handle this case and left the mask
  value unmodified.

Change-Id: I56499e9f05c3f481b5a96e0507da2fb195f207fa
Reviewed-by: Tor Arne Vestbø <tor.arne.vestbo@qt.io>

1 files changed, 22 insertions, 43 deletions

diff --git a/src/plugins/platforms/cocoa/qcocoawindow.mm b/src/plugins/platforms/cocoa/qcocoawindow.mm
index e8550887cb..d3f26df6c5 100644
--- a/src/plugins/platforms/cocoa/qcocoawindow.mm
+++ b/src/plugins/platforms/cocoa/qcocoawindow.mm
@@ -485,52 +485,31 @@ NSInteger QCocoaWindow::windowLevel(Qt::WindowFlags flags)
 
 NSUInteger QCocoaWindow::windowStyleMask(Qt::WindowFlags flags)
 {
-    Qt::WindowType type = static_cast<Qt::WindowType>(int(flags & Qt::WindowType_Mask));
-    NSInteger styleMask = NSBorderlessWindowMask;
-    if (flags & Qt::FramelessWindowHint)
-        return styleMask;
-    if ((type & Qt::Popup) == Qt::Popup) {
-        if (!windowIsPopupType(type)) {
-            styleMask = NSUtilityWindowMask | NSResizableWindowMask;
-            if (!(flags & Qt::CustomizeWindowHint)) {
-                styleMask |= NSClosableWindowMask | NSMiniaturizableWindowMask | NSTitledWindowMask;
-            } else {
-                if (flags & Qt::WindowTitleHint)
-                    styleMask |= NSTitledWindowMask;
-                if (flags & Qt::WindowCloseButtonHint)
-                    styleMask |= NSClosableWindowMask;
-                if (flags & Qt::WindowMinimizeButtonHint)
-                    styleMask |= NSMiniaturizableWindowMask;
-            }
-        }
+    const Qt::WindowType type = static_cast<Qt::WindowType>(int(flags & Qt::WindowType_Mask));
+    const bool frameless = (flags & Qt::FramelessWindowHint) || windowIsPopupType(type);
+
+    // Select base window type.
+    NSUInteger styleMask = frameless ? NSBorderlessWindowMask : NSResizableWindowMask;
+
+    if (frameless) {
+        // No further customizations for frameless since there are no window decorations.
+    } else if (flags & Qt::CustomizeWindowHint) {
+        if (flags & Qt::WindowTitleHint)
+            styleMask |= NSTitledWindowMask;
+        if (flags & Qt::WindowCloseButtonHint)
+            styleMask |= NSClosableWindowMask;
+        if (flags & Qt::WindowMinimizeButtonHint)
+            styleMask |= NSMiniaturizableWindowMask;
     } else {
-        if (type == Qt::Window && !(flags & Qt::CustomizeWindowHint)) {
-            styleMask = (NSResizableWindowMask | NSClosableWindowMask | NSMiniaturizableWindowMask | NSTitledWindowMask);
-        } else if (type == Qt::Dialog) {
-            if (flags & Qt::CustomizeWindowHint) {
-                if (flags & Qt::WindowMaximizeButtonHint)
-                    styleMask = NSResizableWindowMask;
-                if (flags & Qt::WindowTitleHint)
-                    styleMask |= NSTitledWindowMask;
-                if (flags & Qt::WindowCloseButtonHint)
-                    styleMask |= NSClosableWindowMask;
-                if (flags & Qt::WindowMinimizeButtonHint)
-                    styleMask |= NSMiniaturizableWindowMask;
-            } else {
-                styleMask = NSResizableWindowMask | NSClosableWindowMask | NSTitledWindowMask;
-            }
-        } else {
-            if (flags & Qt::WindowMaximizeButtonHint)
-                styleMask |= NSResizableWindowMask;
-            if (flags & Qt::WindowTitleHint)
-                styleMask |= NSTitledWindowMask;
-            if (flags & Qt::WindowCloseButtonHint)
-                styleMask |= NSClosableWindowMask;
-            if (flags & Qt::WindowMinimizeButtonHint)
-                styleMask |= NSMiniaturizableWindowMask;
-        }
+        styleMask |= NSClosableWindowMask | NSTitledWindowMask;
+
+        if (type != Qt::Dialog)
+            styleMask |= NSMiniaturizableWindowMask;
     }
 
+    if (type == Qt::Tool)
+        styleMask |= NSUtilityWindowMask;
+
     if (m_drawContentBorderGradient)
         styleMask |= NSTexturedBackgroundWindowMask;