Prevent takeStandardSnapshots() from stopping on first mismatch

Force takeStandardSnapshots() to take and log all snapshots before
launching QFAIL. Macros QBASELINE_CHECK_DEFERRED and
QBASELINE_CHECK_DEFERRED_SUM have been added in qbaselinetest.h

Task-number: QTBUG-99790
Change-Id: Ia015de808f354e842ac4029c5c84be18c4a4e209
Reviewed-by: Volker Hilsheimer <volker.hilsheimer@qt.io>
(cherry picked from commit fd93c298d6fab70db5a1ce65eed41d929635e8b9)
Reviewed-by: Qt Cherry-pick Bot <cherrypick_bot@qt-project.org>

2 files changed, 18 insertions, 4 deletions

diff --git a/tests/baseline/shared/qbaselinetest.h b/tests/baseline/shared/qbaselinetest.h
index df99348488..68f2efe461 100644
--- a/tests/baseline/shared/qbaselinetest.h
+++ b/tests/baseline/shared/qbaselinetest.h
@@ -58,8 +58,21 @@ do {\
     }\
 } while (0)
 
+#define QBASELINE_CHECK_SUM_DEFERRED(image, name, checksum)\
+do {\
+    QByteArray _msg;\
+    bool _err = false;\
+    if (!QBaselineTest::checkImage((image), (name), (checksum), &_msg, &_err)) {\
+        QTest::qFail(_msg.constData(), __FILE__, __LINE__);\
+    } else if (_err) {\
+        QSKIP(_msg.constData());\
+    }\
+} while (0)
+
 #define QBASELINE_CHECK(image, name) QBASELINE_CHECK_SUM(image, name, 0)
 
+#define QBASELINE_CHECK_DEFERRED(image, name) QBASELINE_CHECK_SUM_DEFERRED(image, name, 0)
+
 #define QBASELINE_TEST(image)\
 do {\
     QByteArray _msg;\
diff --git a/tests/baseline/shared/qwidgetbaselinetest.cpp b/tests/baseline/shared/qwidgetbaselinetest.cpp
index 1553a2d82f..e4b36ddb69 100644
--- a/tests/baseline/shared/qwidgetbaselinetest.cpp
+++ b/tests/baseline/shared/qwidgetbaselinetest.cpp
@@ -149,7 +149,7 @@ void QWidgetBaselineTest::takeStandardSnapshots()
         bool focusNextPrevChild(bool next) override { return QWidget::focusNextPrevChild(next); }
     };
 
-    QBASELINE_CHECK(takeSnapshot(), "default");
+    QBASELINE_CHECK_DEFERRED(takeSnapshot(), "default");
 
     // try hard to set focus
     static_cast<PublicWidget*>(window)->focusNextPrevChild(true);
@@ -158,14 +158,15 @@ void QWidgetBaselineTest::takeStandardSnapshots()
         if (firstChild)
             firstChild->setFocus();
     }
+
     if (testWindow()->focusWidget()) {
-        QBASELINE_CHECK(takeSnapshot(), "focused");
+        QBASELINE_CHECK_DEFERRED(takeSnapshot(), "focused");
         testWindow()->focusWidget()->clearFocus();
     }
 
     // this disables all children
     window->setEnabled(false);
-    QBASELINE_CHECK(takeSnapshot(), "disabled");
+    QBASELINE_CHECK_DEFERRED(takeSnapshot(), "disabled");
     window->setEnabled(true);
 
     // show and activate another window so that our test window becomes inactive
@@ -176,7 +177,7 @@ void QWidgetBaselineTest::takeStandardSnapshots()
     otherWindow.show();
     otherWindow.windowHandle()->requestActivate();
     QVERIFY(QTest::qWaitForWindowActive(&otherWindow));
-    QBASELINE_CHECK(takeSnapshot(), "inactive");
+    QBASELINE_CHECK_DEFERRED(takeSnapshot(), "inactive");
 
     window->windowHandle()->requestActivate();
     QVERIFY(QTest::qWaitForWindowActive(window));