Tests: Un-crash image comparison tests

When (naively) running the tests with High-DPI scaling active, the
test would assert on mismatching image sizes caused by the scaling.
Add an error message parameter to
QQuickVisualTestUtil::compareImages() for use with QVERIFY2, return
false instead of asserting on format/size mismatches and adapt the
usages. Remove duplicate code in tst_qquickwindow.

Change-Id: I76564f4125798fa1e065a0202b686bc7f5ec5680
Reviewed-by: Mitch Curtis <mitch.curtis@qt.io>

6 files changed, 48 insertions, 60 deletions

diff --git a/tests/auto/quick/qquickborderimage/tst_qquickborderimage.cpp b/tests/auto/quick/qquickborderimage/tst_qquickborderimage.cpp
index 02e89ba0a7..9292e1886a 100644
--- a/tests/auto/quick/qquickborderimage/tst_qquickborderimage.cpp
+++ b/tests/auto/quick/qquickborderimage/tst_qquickborderimage.cpp
@@ -596,7 +596,9 @@ void tst_qquickborderimage::borderImageMesh()
     window->setSource(testFileUrl("mesh.qml"));
     QImage mesh = window->grabWindow();
 
-    QVERIFY(QQuickVisualTestUtil::compareImages(mesh, nonmesh));
+    QString errorMessage;
+    QVERIFY2(QQuickVisualTestUtil::compareImages(mesh, nonmesh, &errorMessage),
+             qPrintable(errorMessage));
 }
 #endif
 QTEST_MAIN(tst_qquickborderimage)
diff --git a/tests/auto/quick/qquickshape/tst_qquickshape.cpp b/tests/auto/quick/qquickshape/tst_qquickshape.cpp
index 3206129e72..61fb260612 100644
--- a/tests/auto/quick/qquickshape/tst_qquickshape.cpp
+++ b/tests/auto/quick/qquickshape/tst_qquickshape.cpp
@@ -233,7 +233,10 @@ void tst_QQuickShape::render()
     QImage refImg(testFileUrl("pathitem3.png").toLocalFile());
     QVERIFY(!refImg.isNull());
 
-    QVERIFY(QQuickVisualTestUtil::compareImages(img.convertToFormat(refImg.format()), refImg));
+    QString errorMessage;
+    const QImage actualImg = img.convertToFormat(refImg.format());
+    QVERIFY2(QQuickVisualTestUtil::compareImages(actualImg, refImg, &errorMessage),
+             qPrintable(errorMessage));
 }
 
 void tst_QQuickShape::renderWithMultipleSp()
@@ -254,7 +257,10 @@ void tst_QQuickShape::renderWithMultipleSp()
     QImage refImg(testFileUrl("pathitem4.png").toLocalFile());
     QVERIFY(!refImg.isNull());
 
-    QVERIFY(QQuickVisualTestUtil::compareImages(img.convertToFormat(refImg.format()), refImg));
+    QString errorMessage;
+    const QImage actualImg = img.convertToFormat(refImg.format());
+    QVERIFY2(QQuickVisualTestUtil::compareImages(actualImg, refImg, &errorMessage),
+             qPrintable(errorMessage));
 }
 
 void tst_QQuickShape::radialGrad()
@@ -275,7 +281,10 @@ void tst_QQuickShape::radialGrad()
     QImage refImg(testFileUrl("pathitem5.png").toLocalFile());
     QVERIFY(!refImg.isNull());
 
-    QVERIFY(QQuickVisualTestUtil::compareImages(img.convertToFormat(refImg.format()), refImg));
+    QString errorMessage;
+    const QImage actualImg = img.convertToFormat(refImg.format());
+    QVERIFY2(QQuickVisualTestUtil::compareImages(actualImg, refImg, &errorMessage),
+             qPrintable(errorMessage));
 }
 
 void tst_QQuickShape::conicalGrad()
@@ -296,7 +305,10 @@ void tst_QQuickShape::conicalGrad()
     QImage refImg(testFileUrl("pathitem6.png").toLocalFile());
     QVERIFY(!refImg.isNull());
 
-    QVERIFY(QQuickVisualTestUtil::compareImages(img.convertToFormat(refImg.format()), refImg));
+    QString errorMessage;
+    const QImage actualImg = img.convertToFormat(refImg.format());
+    QVERIFY2(QQuickVisualTestUtil::compareImages(actualImg, refImg, &errorMessage),
+             qPrintable(errorMessage));
 }
 
 QTEST_MAIN(tst_QQuickShape)
diff --git a/tests/auto/quick/qquickwindow/tst_qquickwindow.cpp b/tests/auto/quick/qquickwindow/tst_qquickwindow.cpp
index 578ae56cca..ab101dc1be 100644
--- a/tests/auto/quick/qquickwindow/tst_qquickwindow.cpp
+++ b/tests/auto/quick/qquickwindow/tst_qquickwindow.cpp
@@ -1506,44 +1506,6 @@ void tst_qquickwindow::animationsWhileHidden()
     QTRY_VERIFY(window->isVisible());
 }
 
-// When running on native Nvidia graphics cards on linux, the
-// distance field glyph pixels have a measurable, but not visible
-// pixel error. Use a custom compare function to avoid
-//
-// This was GT-216 with the ubuntu "nvidia-319" driver package.
-// llvmpipe does not show the same issue.
-//
-bool compareImages(const QImage &ia, const QImage &ib)
-{
-    if (ia.size() != ib.size())
-        qDebug() << "images are of different size" << ia.size() << ib.size();
-    Q_ASSERT(ia.size() == ib.size());
-    Q_ASSERT(ia.format() == ib.format());
-
-    int w = ia.width();
-    int h = ia.height();
-    const int tolerance = 5;
-    for (int y=0; y<h; ++y) {
-        const uint *as= (const uint *) ia.constScanLine(y);
-        const uint *bs= (const uint *) ib.constScanLine(y);
-        for (int x=0; x<w; ++x) {
-            uint a = as[x];
-            uint b = bs[x];
-
-            // No tolerance for error in the alpha.
-            if ((a & 0xff000000) != (b & 0xff000000))
-                return false;
-            if (qAbs(qRed(a) - qRed(b)) > tolerance)
-                return false;
-            if (qAbs(qRed(a) - qRed(b)) > tolerance)
-                return false;
-            if (qAbs(qRed(a) - qRed(b)) > tolerance)
-                return false;
-        }
-    }
-    return true;
-}
-
 void tst_qquickwindow::headless()
 {
     QQmlEngine engine;
@@ -1597,7 +1559,9 @@ void tst_qquickwindow::headless()
 
     // Verify that the visual output is the same
     QImage newContent = window->grabWindow();
-    QVERIFY(compareImages(newContent, originalContent));
+    QString errorMessage;
+    QVERIFY2(QQuickVisualTestUtil::compareImages(newContent, originalContent, &errorMessage),
+             qPrintable(errorMessage));
 }
 
 void tst_qquickwindow::noUpdateWhenNothingChanges()
diff --git a/tests/auto/quick/scenegraph/tst_scenegraph.cpp b/tests/auto/quick/scenegraph/tst_scenegraph.cpp
index 2479450287..063358c795 100644
--- a/tests/auto/quick/scenegraph/tst_scenegraph.cpp
+++ b/tests/auto/quick/scenegraph/tst_scenegraph.cpp
@@ -264,6 +264,7 @@ void tst_SceneGraph::manyWindows()
     const int COUNT = 4;
 
     QImage baseLine;
+    QString errorMessage;
     for (int i=0; i<COUNT; ++i) {
         views << createView(file, parent.data(), (i % 2) * 100, (i / 2) * 100, 100, 100);
     }
@@ -275,7 +276,8 @@ void tst_SceneGraph::manyWindows()
             baseLine = content;
             QVERIFY(containsSomethingOtherThanWhite(baseLine));
         } else {
-            QVERIFY(compareImages(content, baseLine));
+            QVERIFY2(compareImages(content, baseLine, &errorMessage),
+                     qPrintable(errorMessage));
         }
     }
 
@@ -284,7 +286,8 @@ void tst_SceneGraph::manyWindows()
     QQuickView *last = createView(file, parent.data(), 100, 100, 100, 100);
     QVERIFY(QTest::qWaitForWindowExposed(last));
     views << last;
-    QVERIFY(compareImages(baseLine, last->grabWindow()));
+    QVERIFY2(compareImages(baseLine, last->grabWindow(), &errorMessage),
+             qPrintable(errorMessage));
 
     // Wipe and recreate all
     qDeleteAll(views);
@@ -297,7 +300,8 @@ void tst_SceneGraph::manyWindows()
         QQuickView *view = views.at(i);
         QVERIFY(QTest::qWaitForWindowExposed(view));
         QImage content = view->grabWindow();
-        QVERIFY(compareImages(content, baseLine));
+        QVERIFY2(compareImages(content, baseLine, &errorMessage),
+                 qPrintable(errorMessage));
     }
 }
 
diff --git a/tests/auto/quick/shared/visualtestutil.cpp b/tests/auto/quick/shared/visualtestutil.cpp
index eabfe5368b..de2cf2bd5b 100644
--- a/tests/auto/quick/shared/visualtestutil.cpp
+++ b/tests/auto/quick/shared/visualtestutil.cpp
@@ -66,12 +66,18 @@ void QQuickVisualTestUtil::dumpTree(QQuickItem *parent, int depth)
 // distance field glyph pixels have a measurable, but not visible
 // pixel error. This was GT-216 with the ubuntu "nvidia-319" driver package.
 // llvmpipe does not show the same issue.
-bool QQuickVisualTestUtil::compareImages(const QImage &ia, const QImage &ib)
+
+bool QQuickVisualTestUtil::compareImages(const QImage &ia, const QImage &ib, QString *errorMessage)
 {
-    if (ia.size() != ib.size())
-        qDebug() << "images are of different size" << ia.size() << ib.size();
-    Q_ASSERT(ia.size() == ib.size());
-    Q_ASSERT(ia.format() == ib.format());
+    if (ia.size() != ib.size()) {
+        QDebug(errorMessage) << "Images are of different size:" << ia.size() << ib.size()
+            << "DPR:" << ia.devicePixelRatio() << ib.devicePixelRatio();
+        return false;
+    }
+    if (ia.format() != ib.format()) {
+        QDebug(errorMessage) << "Images are of different formats:" << ia.format() << ib.format();
+        return false;
+    }
 
     int w = ia.width();
     int h = ia.height();
@@ -84,14 +90,14 @@ bool QQuickVisualTestUtil::compareImages(const QImage &ia, const QImage &ib)
             uint b = bs[x];
 
             // No tolerance for error in the alpha.
-            if ((a & 0xff000000) != (b & 0xff000000))
-                return false;
-            if (qAbs(qRed(a) - qRed(b)) > tolerance)
-                return false;
-            if (qAbs(qRed(a) - qRed(b)) > tolerance)
-                return false;
-            if (qAbs(qRed(a) - qRed(b)) > tolerance)
+            if ((a & 0xff000000) != (b & 0xff000000)
+                || qAbs(qRed(a) - qRed(b)) > tolerance
+                || qAbs(qRed(a) - qRed(b)) > tolerance
+                || qAbs(qRed(a) - qRed(b)) > tolerance) {
+                QDebug(errorMessage) << "Mismatch at:" << x << y << ':'
+                    << hex << showbase << a << b;
                 return false;
+            }
         }
     }
     return true;
diff --git a/tests/auto/quick/shared/visualtestutil.h b/tests/auto/quick/shared/visualtestutil.h
index 2daf86cd83..1cdbaf838b 100644
--- a/tests/auto/quick/shared/visualtestutil.h
+++ b/tests/auto/quick/shared/visualtestutil.h
@@ -97,7 +97,7 @@ namespace QQuickVisualTestUtil
         return items;
     }
 
-    bool compareImages(const QImage &ia, const QImage &ib);
+    bool compareImages(const QImage &ia, const QImage &ib, QString *errorMessage);
 }
 
 #define QQUICK_VERIFY_POLISH(item) \