macOS: Don't exec file dialogs via runModal unless they are app modal

Non-modal or window modal dialogs are shown at show(), via AppKit APIs
that are non-blocking. If we want to block execution at this point, we
need to spin our own event loop. The runModal API of NSSavePanel is not
meant to be used for blocking execution for already shown dialogs, but
is reserved for application modal dialogs.

This means we no longer trip over AppKit's understanding of what state
the dialog is in, which would result in the dialog not reporting back
any files. It also allows us to remove the guard for closing dialogs
twice.

We now also correctly close and end the application modal session if
the dialog is closed programmatically using Qt APIs.

Task-number: QTBUG-89959
Fixes: QTBUG-85547
Change-Id: Ida3dc404417789d4823822ecfbf0935591c23878
Reviewed-by: Morten Johan Sørvig <morten.sorvig@qt.io>
(cherry picked from commit aa3b42d6342c6756c3ce941db0bbf1122ae48224)
Reviewed-by: Qt Cherry-pick Bot <cherrypick_bot@qt-project.org>

2 files changed, 40 insertions, 16 deletions

diff --git a/src/plugins/platforms/cocoa/qcocoafiledialoghelper.h b/src/plugins/platforms/cocoa/qcocoafiledialoghelper.h
index 20663fb231..08ad056c50 100644
--- a/src/plugins/platforms/cocoa/qcocoafiledialoghelper.h
+++ b/src/plugins/platforms/cocoa/qcocoafiledialoghelper.h
@@ -56,6 +56,7 @@ QT_NAMESPACE_ALIAS_OBJC_CLASS(QNSOpenSavePanelDelegate);
 
 QT_BEGIN_NAMESPACE
 
+class QEventLoop;
 class QFileDialog;
 class QFileDialogPrivate;
 
@@ -92,6 +93,7 @@ public:
 private:
     QNSOpenSavePanelDelegate *mDelegate;
     QUrl mDir;
+    QEventLoop *m_eventLoop = nullptr;
 };
 
 QT_END_NAMESPACE
diff --git a/src/plugins/platforms/cocoa/qcocoafiledialoghelper.mm b/src/plugins/platforms/cocoa/qcocoafiledialoghelper.mm
index a14320f049..49fd28828d 100644
--- a/src/plugins/platforms/cocoa/qcocoafiledialoghelper.mm
+++ b/src/plugins/platforms/cocoa/qcocoafiledialoghelper.mm
@@ -179,10 +179,13 @@ static QString strippedText(QString s)
 - (void)closePanel
 {
     *mCurrentSelection = QString::fromNSString([[mSavePanel URL] path]).normalized(QString::NormalizationForm_C);
-    if ([mSavePanel respondsToSelector:@selector(close)])
+
+    if (mSavePanel.sheet)
+        [NSApp endSheet:mSavePanel];
+    else if (NSApp.modalWindow == mSavePanel)
+        [NSApp stopModal];
+    else
         [mSavePanel close];
-    if ([mSavePanel isSheet])
-        [NSApp endSheet: mSavePanel];
 }
 
 - (void)showModelessPanel
@@ -715,11 +718,14 @@ bool QCocoaFileDialogHelper::showCocoaFilePanel(Qt::WindowModality windowModalit
     createNSOpenSavePanelDelegate();
     if (!mDelegate)
         return false;
-    if (windowModality == Qt::NonModal)
-        [mDelegate showModelessPanel];
-    else if (windowModality == Qt::WindowModal && parent)
+
+    if (windowModality == Qt::WindowModal && parent)
         [mDelegate showWindowModalSheet:parent];
-    // no need to show a Qt::ApplicationModal dialog here, since it will be done in _q_platformRunNativeAppModalPanel()
+    else if (windowModality == Qt::ApplicationModal)
+        return true; // Defer until exec()
+    else
+        [mDelegate showModelessPanel];
+
     return true;
 }
 
@@ -731,6 +737,10 @@ bool QCocoaFileDialogHelper::hideCocoaFilePanel()
         return false;
     } else {
         [mDelegate closePanel];
+
+        if (m_eventLoop)
+            m_eventLoop->exit();
+
         // Even when we hide it, we are still using a
         // native dialog, so return true:
         return true;
@@ -739,16 +749,28 @@ bool QCocoaFileDialogHelper::hideCocoaFilePanel()
 
 void QCocoaFileDialogHelper::exec()
 {
-    // Note: If NSApp is not running (which is the case if e.g a top-most
-    // QEventLoop has been interrupted, and the second-most event loop has not
-    // yet been reactivated (regardless if [NSApp run] is still on the stack)),
-    // showing a native modal dialog will fail.
-    QMacAutoReleasePool pool;
-    if ([mDelegate runApplicationModalPanel])
-        emit accept();
-    else
-        emit reject();
+    Q_ASSERT(mDelegate);
 
+    if (mDelegate->mSavePanel.visible) {
+        // WindowModal or NonModal, so already shown above
+        QEventLoop eventLoop;
+        m_eventLoop = &eventLoop;
+        eventLoop.exec(QEventLoop::DialogExec);
+        m_eventLoop = nullptr;
+    } else {
+        // ApplicationModal, so show and block using native APIs
+
+        // Note: If NSApp is not running (which is the case if e.g a top-most
+        // QEventLoop has been interrupted, and the second-most event loop has not
+        // yet been reactivated (regardless if [NSApp run] is still on the stack)),
+        // showing a native modal dialog will fail.
+
+        QMacAutoReleasePool pool;
+        if ([mDelegate runApplicationModalPanel])
+            emit accept();
+        else
+            emit reject();
+    }
 }
 
 bool QCocoaFileDialogHelper::defaultNameFilterDisables() const