QTestlib: Make QImage comparison more verbose.

Introduce a specialization for qCompare(QImage,QImage) that
checks isNull, size and format and outputs verbose messages.

Check isNull, size similarly for QPixmap.

Add an autotest:
- Add test to cmptest and make it a GUI application since
  QImage requires QGuiApplication.
- Make testlib/selftests capable of running X11-GUI applications
  by passing DISPLAY.
- Ignore stderr output for cmptest
- Add test data

Change-Id: I2b29c7822fbeedf2b22c90889739ed7ff859ce92
Reviewed-by: Jason McDonald <jason.mcdonald@nokia.com>

7 files changed, 357 insertions, 36 deletions

diff --git a/tests/auto/testlib/selftests/cmptest/cmptest.pro b/tests/auto/testlib/selftests/cmptest/cmptest.pro
index bc72f23696..1da4646023 100644
--- a/tests/auto/testlib/selftests/cmptest/cmptest.pro
+++ b/tests/auto/testlib/selftests/cmptest/cmptest.pro
@@ -1,5 +1,5 @@
 SOURCES += tst_cmptest.cpp
-QT = core testlib
+QT = core gui testlib
 
 mac:CONFIG -= app_bundle
 CONFIG -= debug_and_release_target
diff --git a/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp b/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp
index 6e19d60cbb..f6773cc4c5 100644
--- a/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp
+++ b/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp
@@ -42,6 +42,88 @@
 
 #include <QtCore>
 #include <QtTest/QtTest>
+#include <QtGui/QImage>
+#include <QtGui/QPixmap>
+
+/* XPM test data for QPixmap, QImage tests (use drag cursors as example) */
+
+static const char * const xpmPixmapData1[] = {
+"11 20 3 1",
+".        c None",
+"a        c #FFFFFF",
+"X        c #000000", // X11 cursor is traditionally black
+"aa.........",
+"aXa........",
+"aXXa.......",
+"aXXXa......",
+"aXXXXa.....",
+"aXXXXXa....",
+"aXXXXXXa...",
+"aXXXXXXXa..",
+"aXXXXXXXXa.",
+"aXXXXXXXXXa",
+"aXXXXXXaaaa",
+"aXXXaXXa...",
+"aXXaaXXa...",
+"aXa..aXXa..",
+"aa...aXXa..",
+"a.....aXXa.",
+"......aXXa.",
+".......aXXa",
+".......aXXa",
+"........aa."};
+
+static const char * const xpmPixmapData2[] = {
+"11 20 4 1",
+".        c None",
+"a        c #FFFFFF",
+"b        c #0000FF",
+"X        c #000000",
+"aab........",
+"aXab.......",
+"aXXab......",
+"aXXXab.....",
+"aXXXXab....",
+"aXXXXXab...",
+"aXXXXXXab..",
+"aXXXXXXXa..",
+"aXXXXXXXXa.",
+"aXXXXXXXXXa",
+"aXXXXXXaaaa",
+"aXXXaXXa...",
+"aXXaaXXa...",
+"aXa..aXXa..",
+"aa...aXXa..",
+"a.....aXXa.",
+"......aXXa.",
+".......aXXa",
+".......aXXa",
+"........aa."};
+
+static const char * const xpmPixmapData3[] = {
+"20 20 2 1",
+"       c #000000",
+".      c #C32D2D",
+"          ..........",
+"            ........",
+"             .......",
+"              ......",
+"                ....",
+"                  ..",
+"                   .",
+"                    ",
+"                    ",
+".                   ",
+"...                 ",
+".....               ",
+"......              ",
+".......             ",
+".........           ",
+"...........         ",
+"...........         ",
+"............        ",
+"............        ",
+".............       "};
 
 class tst_Cmptest: public QObject
 {
@@ -54,6 +136,10 @@ private slots:
     void compare_tostring_data();
     void compareQStringLists();
     void compareQStringLists_data();
+    void compareQPixmaps();
+    void compareQPixmaps_data();
+    void compareQImages();
+    void compareQImages_data();
 };
 
 static bool boolfunc() { return true; }
@@ -222,5 +308,57 @@ void tst_Cmptest::compareQStringLists()
     QCOMPARE(opA, opB);
 }
 
+void tst_Cmptest::compareQPixmaps_data()
+{
+    QTest::addColumn<QPixmap>("opA");
+    QTest::addColumn<QPixmap>("opB");
+
+    const QPixmap pixmap1(xpmPixmapData1);
+    const QPixmap pixmap2(xpmPixmapData2);
+    const QPixmap pixmap3(xpmPixmapData3);
+
+    QTest::newRow("both null") << QPixmap() << QPixmap();
+    QTest::newRow("one null") << QPixmap() << pixmap1;
+    QTest::newRow("other null") << pixmap1 << QPixmap();
+    QTest::newRow("equal") << pixmap1 << pixmap1;
+    QTest::newRow("different size") << pixmap1 << pixmap3;
+    QTest::newRow("different pixels") << pixmap1 << pixmap2;
+}
+
+void tst_Cmptest::compareQPixmaps()
+{
+    QFETCH(QPixmap, opA);
+    QFETCH(QPixmap, opB);
+
+    QCOMPARE(opA, opB);
+}
+
+void tst_Cmptest::compareQImages_data()
+{
+    QTest::addColumn<QImage>("opA");
+    QTest::addColumn<QImage>("opB");
+
+    const QImage image1(QPixmap(xpmPixmapData1).toImage());
+    const QImage image2(QPixmap(xpmPixmapData2).toImage());
+    const QImage image1Indexed = image1.convertToFormat(QImage::Format_Indexed8);
+    const QImage image3(QPixmap(xpmPixmapData3).toImage());
+
+    QTest::newRow("both null") << QImage() << QImage();
+    QTest::newRow("one null") << QImage() << image1;
+    QTest::newRow("other null") << image1 << QImage();
+    QTest::newRow("equal") << image1 << image1;
+    QTest::newRow("different size") << image1 << image3;
+    QTest::newRow("different format") << image1 << image1Indexed;
+    QTest::newRow("different pixels") << image1 << image2;
+}
+
+void tst_Cmptest::compareQImages()
+{
+    QFETCH(QImage, opA);
+    QFETCH(QImage, opB);
+
+    QCOMPARE(opA, opB);
+}
+
 QTEST_MAIN(tst_Cmptest)
 #include "tst_cmptest.moc"
diff --git a/tests/auto/testlib/selftests/expected_cmptest.lightxml b/tests/auto/testlib/selftests/expected_cmptest.lightxml
index d027dcde11..17f3e6ec3a 100644
--- a/tests/auto/testlib/selftests/expected_cmptest.lightxml
+++ b/tests/auto/testlib/selftests/expected_cmptest.lightxml
@@ -12,25 +12,25 @@
 <Incident type="pass" file="" line="0" />
 </TestFunction>
 <TestFunction name="compare_tostring">
-<Incident type="fail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp" line="122">
+<Incident type="fail" file="tst_cmptest.cpp" line="214">
     <DataTag><![CDATA[int, string]]></DataTag>
     <Description><![CDATA[Compared values are not the same
    Actual (actual): QVariant(int,123)
    Expected (expected): QVariant(QString,hi)]]></Description>
 </Incident>
-<Incident type="fail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp" line="122">
+<Incident type="fail" file="tst_cmptest.cpp" line="214">
     <DataTag><![CDATA[null hash, invalid]]></DataTag>
     <Description><![CDATA[Compared values are not the same
    Actual (actual): QVariant(QVariantHash)
    Expected (expected): QVariant()]]></Description>
 </Incident>
-<Incident type="fail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp" line="122">
+<Incident type="fail" file="tst_cmptest.cpp" line="214">
     <DataTag><![CDATA[string, null user type]]></DataTag>
     <Description><![CDATA[Compared values are not the same
    Actual (actual): QVariant(QString,A simple string)
    Expected (expected): QVariant(PhonyClass)]]></Description>
 </Incident>
-<Incident type="fail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp" line="122">
+<Incident type="fail" file="tst_cmptest.cpp" line="214">
     <DataTag><![CDATA[both non-null user type]]></DataTag>
     <Description><![CDATA[Compared values are not the same
    Actual (actual): QVariant(PhonyClass,<value not representable as string>)
@@ -38,37 +38,91 @@
 </Incident>
 </TestFunction>
 <TestFunction name="compareQStringLists">
-<Incident type="fail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp" line="203">
+<Incident type="fail" file="tst_cmptest.cpp" line="308">
     <DataTag><![CDATA[last item different]]></DataTag>
     <Description><![CDATA[Compared QStringLists differ at index 2.
    Actual (opA) : 'string3'
    Expected (opB) : 'DIFFERS']]></Description>
 </Incident>
-<Incident type="fail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp" line="203">
+<Incident type="fail" file="tst_cmptest.cpp" line="308">
     <DataTag><![CDATA[second-last item different]]></DataTag>
     <Description><![CDATA[Compared QStringLists differ at index 2.
    Actual (opA) : 'string3'
    Expected (opB) : 'DIFFERS']]></Description>
 </Incident>
-<Incident type="fail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp" line="203">
+<Incident type="fail" file="tst_cmptest.cpp" line="308">
     <DataTag><![CDATA[prefix]]></DataTag>
     <Description><![CDATA[Compared QStringLists have different sizes.
    Actual (opA) size  : '2'
    Expected (opB) size: '1']]></Description>
 </Incident>
-<Incident type="fail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp" line="203">
+<Incident type="fail" file="tst_cmptest.cpp" line="308">
     <DataTag><![CDATA[short list second]]></DataTag>
     <Description><![CDATA[Compared QStringLists have different sizes.
    Actual (opA) size  : '12'
    Expected (opB) size: '1']]></Description>
 </Incident>
-<Incident type="fail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp" line="203">
+<Incident type="fail" file="tst_cmptest.cpp" line="308">
     <DataTag><![CDATA[short list first]]></DataTag>
     <Description><![CDATA[Compared QStringLists have different sizes.
    Actual (opA) size  : '1'
    Expected (opB) size: '12']]></Description>
 </Incident>
 </TestFunction>
+<TestFunction name="compareQPixmaps">
+<Incident type="fail" file="tst_cmptest.cpp" line="333">
+    <DataTag><![CDATA[one null]]></DataTag>
+    <Description><![CDATA[Compared QPixmaps differ.
+   Actual (opA).isNull()  : 1
+   Expected (opB).isNull(): 0]]></Description>
+</Incident>
+<Incident type="fail" file="tst_cmptest.cpp" line="333">
+    <DataTag><![CDATA[other null]]></DataTag>
+    <Description><![CDATA[Compared QPixmaps differ.
+   Actual (opA).isNull()  : 0
+   Expected (opB).isNull(): 1]]></Description>
+</Incident>
+<Incident type="fail" file="tst_cmptest.cpp" line="333">
+    <DataTag><![CDATA[different size]]></DataTag>
+    <Description><![CDATA[Compared QPixmaps differ in size.
+   Actual (opA)  : 11x20
+   Expected (opB): 20x20]]></Description>
+</Incident>
+<Incident type="fail" file="tst_cmptest.cpp" line="333">
+    <DataTag><![CDATA[different pixels]]></DataTag>
+    <Description><![CDATA[Compared values are not the same]]></Description>
+</Incident>
+</TestFunction>
+<TestFunction name="compareQImages">
+<Incident type="fail" file="tst_cmptest.cpp" line="360">
+    <DataTag><![CDATA[one null]]></DataTag>
+    <Description><![CDATA[Compared QImages differ.
+   Actual (opA).isNull()  : 1
+   Expected (opB).isNull(): 0]]></Description>
+</Incident>
+<Incident type="fail" file="tst_cmptest.cpp" line="360">
+    <DataTag><![CDATA[other null]]></DataTag>
+    <Description><![CDATA[Compared QImages differ.
+   Actual (opA).isNull()  : 0
+   Expected (opB).isNull(): 1]]></Description>
+</Incident>
+<Incident type="fail" file="tst_cmptest.cpp" line="360">
+    <DataTag><![CDATA[different size]]></DataTag>
+    <Description><![CDATA[Compared QImages differ in size.
+   Actual (opA)  : 11x20
+   Expected (opB): 20x20]]></Description>
+</Incident>
+<Incident type="fail" file="tst_cmptest.cpp" line="360">
+    <DataTag><![CDATA[different format]]></DataTag>
+    <Description><![CDATA[Compared QImages differ in format.
+   Actual (opA)  : 6
+   Expected (opB): 3]]></Description>
+</Incident>
+<Incident type="fail" file="tst_cmptest.cpp" line="360">
+    <DataTag><![CDATA[different pixels]]></DataTag>
+    <Description><![CDATA[Compared values are not the same]]></Description>
+</Incident>
+</TestFunction>
 <TestFunction name="cleanupTestCase">
 <Incident type="pass" file="" line="0" />
 </TestFunction>
diff --git a/tests/auto/testlib/selftests/expected_cmptest.txt b/tests/auto/testlib/selftests/expected_cmptest.txt
index f7dc5cfed3..36a4995d91 100644
--- a/tests/auto/testlib/selftests/expected_cmptest.txt
+++ b/tests/auto/testlib/selftests/expected_cmptest.txt
@@ -6,39 +6,71 @@ PASS   : tst_Cmptest::compare_pointerfuncs()
 FAIL!  : tst_Cmptest::compare_tostring(int, string) Compared values are not the same
    Actual (actual): QVariant(int,123)
    Expected (expected): QVariant(QString,hi)
-   Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp(122)]
+   Loc: [tst_cmptest.cpp(214)]
 FAIL!  : tst_Cmptest::compare_tostring(null hash, invalid) Compared values are not the same
    Actual (actual): QVariant(QVariantHash)
    Expected (expected): QVariant()
-   Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp(122)]
+   Loc: [tst_cmptest.cpp(214)]
 FAIL!  : tst_Cmptest::compare_tostring(string, null user type) Compared values are not the same
    Actual (actual): QVariant(QString,A simple string)
    Expected (expected): QVariant(PhonyClass)
-   Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp(122)]
+   Loc: [tst_cmptest.cpp(214)]
 FAIL!  : tst_Cmptest::compare_tostring(both non-null user type) Compared values are not the same
    Actual (actual): QVariant(PhonyClass,<value not representable as string>)
    Expected (expected): QVariant(PhonyClass,<value not representable as string>)
-   Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp(122)]
+   Loc: [tst_cmptest.cpp(214)]
 FAIL!  : tst_Cmptest::compareQStringLists(last item different) Compared QStringLists differ at index 2.
    Actual (opA) : 'string3'
    Expected (opB) : 'DIFFERS'
-   Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/alive/tst_alive.cpp(68)]
+   Loc: [tst_cmptest.cpp(308)]
 FAIL!  : tst_Cmptest::compareQStringLists(second-last item different) Compared QStringLists differ at index 2.
    Actual (opA) : 'string3'
    Expected (opB) : 'DIFFERS'
-   Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/alive/tst_alive.cpp(68)]
+   Loc: [tst_cmptest.cpp(308)]
 FAIL!  : tst_Cmptest::compareQStringLists(prefix) Compared QStringLists have different sizes.
    Actual (opA) size  : '2'
    Expected (opB) size: '1'
-   Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/alive/tst_alive.cpp(68)]
+   Loc: [tst_cmptest.cpp(308)]
 FAIL!  : tst_Cmptest::compareQStringLists(short list second) Compared QStringLists have different sizes.
    Actual (opA) size  : '12'
    Expected (opB) size: '1'
-   Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/alive/tst_alive.cpp(68)]
+   Loc: [tst_cmptest.cpp(308)]
 FAIL!  : tst_Cmptest::compareQStringLists(short list first) Compared QStringLists have different sizes.
    Actual (opA) size  : '1'
    Expected (opB) size: '12'
-   Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/alive/tst_alive.cpp(68)]
+   Loc: [tst_cmptest.cpp(308)]
+FAIL!  : tst_Cmptest::compareQPixmaps(one null) Compared QPixmaps differ.
+   Actual (opA).isNull()  : 1
+   Expected (opB).isNull(): 0
+   Loc: [tst_cmptest.cpp(333)]
+FAIL!  : tst_Cmptest::compareQPixmaps(other null) Compared QPixmaps differ.
+   Actual (opA).isNull()  : 0
+   Expected (opB).isNull(): 1
+   Loc: [tst_cmptest.cpp(333)]
+FAIL!  : tst_Cmptest::compareQPixmaps(different size) Compared QPixmaps differ in size.
+   Actual (opA)  : 11x20
+   Expected (opB): 20x20
+   Loc: [tst_cmptest.cpp(333)]
+FAIL!  : tst_Cmptest::compareQPixmaps(different pixels) Compared values are not the same
+   Loc: [tst_cmptest.cpp(333)]
+FAIL!  : tst_Cmptest::compareQImages(one null) Compared QImages differ.
+   Actual (opA).isNull()  : 1
+   Expected (opB).isNull(): 0
+   Loc: [tst_cmptest.cpp(360)]
+FAIL!  : tst_Cmptest::compareQImages(other null) Compared QImages differ.
+   Actual (opA).isNull()  : 0
+   Expected (opB).isNull(): 1
+   Loc: [tst_cmptest.cpp(360)]
+FAIL!  : tst_Cmptest::compareQImages(different size) Compared QImages differ in size.
+   Actual (opA)  : 11x20
+   Expected (opB): 20x20
+   Loc: [tst_cmptest.cpp(360)]
+FAIL!  : tst_Cmptest::compareQImages(different format) Compared QImages differ in format.
+   Actual (opA)  : 6
+   Expected (opB): 3
+   Loc: [tst_cmptest.cpp(360)]
+FAIL!  : tst_Cmptest::compareQImages(different pixels) Compared values are not the same
+   Loc: [tst_cmptest.cpp(360)]
 PASS   : tst_Cmptest::cleanupTestCase()
-Totals: 4 passed, 9 failed, 0 skipped
+Totals: 4 passed, 18 failed, 0 skipped
 ********* Finished testing of tst_Cmptest *********
diff --git a/tests/auto/testlib/selftests/expected_cmptest.xml b/tests/auto/testlib/selftests/expected_cmptest.xml
index 7faa3a979d..aba1ce5edd 100644
--- a/tests/auto/testlib/selftests/expected_cmptest.xml
+++ b/tests/auto/testlib/selftests/expected_cmptest.xml
@@ -14,25 +14,25 @@
 <Incident type="pass" file="" line="0" />
 </TestFunction>
 <TestFunction name="compare_tostring">
-<Incident type="fail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp" line="122">
+<Incident type="fail" file="tst_cmptest.cpp" line="214">
     <DataTag><![CDATA[int, string]]></DataTag>
     <Description><![CDATA[Compared values are not the same
    Actual (actual): QVariant(int,123)
    Expected (expected): QVariant(QString,hi)]]></Description>
 </Incident>
-<Incident type="fail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp" line="122">
+<Incident type="fail" file="tst_cmptest.cpp" line="214">
     <DataTag><![CDATA[null hash, invalid]]></DataTag>
     <Description><![CDATA[Compared values are not the same
    Actual (actual): QVariant(QVariantHash)
    Expected (expected): QVariant()]]></Description>
 </Incident>
-<Incident type="fail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp" line="122">
+<Incident type="fail" file="tst_cmptest.cpp" line="214">
     <DataTag><![CDATA[string, null user type]]></DataTag>
     <Description><![CDATA[Compared values are not the same
    Actual (actual): QVariant(QString,A simple string)
    Expected (expected): QVariant(PhonyClass)]]></Description>
 </Incident>
-<Incident type="fail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp" line="122">
+<Incident type="fail" file="tst_cmptest.cpp" line="214">
     <DataTag><![CDATA[both non-null user type]]></DataTag>
     <Description><![CDATA[Compared values are not the same
    Actual (actual): QVariant(PhonyClass,<value not representable as string>)
@@ -40,37 +40,91 @@
 </Incident>
 </TestFunction>
 <TestFunction name="compareQStringLists">
-<Incident type="fail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp" line="203">
+<Incident type="fail" file="tst_cmptest.cpp" line="308">
     <DataTag><![CDATA[last item different]]></DataTag>
     <Description><![CDATA[Compared QStringLists differ at index 2.
    Actual (opA) : 'string3'
    Expected (opB) : 'DIFFERS']]></Description>
 </Incident>
-<Incident type="fail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp" line="203">
+<Incident type="fail" file="tst_cmptest.cpp" line="308">
     <DataTag><![CDATA[second-last item different]]></DataTag>
     <Description><![CDATA[Compared QStringLists differ at index 2.
    Actual (opA) : 'string3'
    Expected (opB) : 'DIFFERS']]></Description>
 </Incident>
-<Incident type="fail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp" line="203">
+<Incident type="fail" file="tst_cmptest.cpp" line="308">
     <DataTag><![CDATA[prefix]]></DataTag>
     <Description><![CDATA[Compared QStringLists have different sizes.
    Actual (opA) size  : '2'
    Expected (opB) size: '1']]></Description>
 </Incident>
-<Incident type="fail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp" line="203">
+<Incident type="fail" file="tst_cmptest.cpp" line="308">
     <DataTag><![CDATA[short list second]]></DataTag>
     <Description><![CDATA[Compared QStringLists have different sizes.
    Actual (opA) size  : '12'
    Expected (opB) size: '1']]></Description>
 </Incident>
-<Incident type="fail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp" line="203">
+<Incident type="fail" file="tst_cmptest.cpp" line="308">
     <DataTag><![CDATA[short list first]]></DataTag>
     <Description><![CDATA[Compared QStringLists have different sizes.
    Actual (opA) size  : '1'
    Expected (opB) size: '12']]></Description>
 </Incident>
 </TestFunction>
+<TestFunction name="compareQPixmaps">
+<Incident type="fail" file="tst_cmptest.cpp" line="333">
+    <DataTag><![CDATA[one null]]></DataTag>
+    <Description><![CDATA[Compared QPixmaps differ.
+   Actual (opA).isNull()  : 1
+   Expected (opB).isNull(): 0]]></Description>
+</Incident>
+<Incident type="fail" file="tst_cmptest.cpp" line="333">
+    <DataTag><![CDATA[other null]]></DataTag>
+    <Description><![CDATA[Compared QPixmaps differ.
+   Actual (opA).isNull()  : 0
+   Expected (opB).isNull(): 1]]></Description>
+</Incident>
+<Incident type="fail" file="tst_cmptest.cpp" line="333">
+    <DataTag><![CDATA[different size]]></DataTag>
+    <Description><![CDATA[Compared QPixmaps differ in size.
+   Actual (opA)  : 11x20
+   Expected (opB): 20x20]]></Description>
+</Incident>
+<Incident type="fail" file="tst_cmptest.cpp" line="333">
+    <DataTag><![CDATA[different pixels]]></DataTag>
+    <Description><![CDATA[Compared values are not the same]]></Description>
+</Incident>
+</TestFunction>
+<TestFunction name="compareQImages">
+<Incident type="fail" file="tst_cmptest.cpp" line="360">
+    <DataTag><![CDATA[one null]]></DataTag>
+    <Description><![CDATA[Compared QImages differ.
+   Actual (opA).isNull()  : 1
+   Expected (opB).isNull(): 0]]></Description>
+</Incident>
+<Incident type="fail" file="tst_cmptest.cpp" line="360">
+    <DataTag><![CDATA[other null]]></DataTag>
+    <Description><![CDATA[Compared QImages differ.
+   Actual (opA).isNull()  : 0
+   Expected (opB).isNull(): 1]]></Description>
+</Incident>
+<Incident type="fail" file="tst_cmptest.cpp" line="360">
+    <DataTag><![CDATA[different size]]></DataTag>
+    <Description><![CDATA[Compared QImages differ in size.
+   Actual (opA)  : 11x20
+   Expected (opB): 20x20]]></Description>
+</Incident>
+<Incident type="fail" file="tst_cmptest.cpp" line="360">
+    <DataTag><![CDATA[different format]]></DataTag>
+    <Description><![CDATA[Compared QImages differ in format.
+   Actual (opA)  : 6
+   Expected (opB): 3]]></Description>
+</Incident>
+<Incident type="fail" file="tst_cmptest.cpp" line="360">
+    <DataTag><![CDATA[different pixels]]></DataTag>
+    <Description><![CDATA[Compared values are not the same]]></Description>
+</Incident>
+</TestFunction>
 <TestFunction name="cleanupTestCase">
 <Incident type="pass" file="" line="0" />
 </TestFunction>
diff --git a/tests/auto/testlib/selftests/expected_cmptest.xunitxml b/tests/auto/testlib/selftests/expected_cmptest.xunitxml
index aa765a65e0..00f5f7f480 100644
--- a/tests/auto/testlib/selftests/expected_cmptest.xunitxml
+++ b/tests/auto/testlib/selftests/expected_cmptest.xunitxml
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" ?>
-<testsuite errors="0" failures="9" tests="6" name="tst_Cmptest">
+<testsuite errors="0" failures="18" tests="8" name="tst_Cmptest">
   <properties>
     <property value="@INSERT_QT_VERSION_HERE@" name="QTestVersion"/>
     <property value="@INSERT_QT_VERSION_HERE@" name="QtVersion"/>
@@ -38,6 +38,33 @@
    Actual (opA) size  : &apos;1&apos;
    Expected (opB) size: &apos;12&apos;" result="fail"/>
   </testcase>
+  <testcase result="fail" name="compareQPixmaps">
+    <failure tag="one null" message="Compared QPixmaps differ.
+   Actual (opA).isNull()  : 1
+   Expected (opB).isNull(): 0" result="fail"/>
+    <failure tag="other null" message="Compared QPixmaps differ.
+   Actual (opA).isNull()  : 0
+   Expected (opB).isNull(): 1" result="fail"/>
+    <failure tag="different size" message="Compared QPixmaps differ in size.
+   Actual (opA)  : 11x20
+   Expected (opB): 20x20" result="fail"/>
+    <failure tag="different pixels" message="Compared values are not the same" result="fail"/>
+  </testcase>
+  <testcase result="fail" name="compareQImages">
+    <failure tag="one null" message="Compared QImages differ.
+   Actual (opA).isNull()  : 1
+   Expected (opB).isNull(): 0" result="fail"/>
+    <failure tag="other null" message="Compared QImages differ.
+   Actual (opA).isNull()  : 0
+   Expected (opB).isNull(): 1" result="fail"/>
+    <failure tag="different size" message="Compared QImages differ in size.
+   Actual (opA)  : 11x20
+   Expected (opB): 20x20" result="fail"/>
+    <failure tag="different format" message="Compared QImages differ in format.
+   Actual (opA)  : 6
+   Expected (opB): 3" result="fail"/>
+    <failure tag="different pixels" message="Compared values are not the same" result="fail"/>
+  </testcase>
   <testcase result="pass" name="cleanupTestCase"/>
   <system-err/>
 </testsuite>
diff --git a/tests/auto/testlib/selftests/tst_selftests.cpp b/tests/auto/testlib/selftests/tst_selftests.cpp
index ea47678248..7825ade183 100644
--- a/tests/auto/testlib/selftests/tst_selftests.cpp
+++ b/tests/auto/testlib/selftests/tst_selftests.cpp
@@ -445,7 +445,15 @@ static inline QProcessEnvironment processEnvironment()
 {
     QProcessEnvironment result;
     const QString path = QStringLiteral("PATH");
-    result.insert(path, QProcessEnvironment::systemEnvironment().value(path));
+    const QProcessEnvironment systemEnvironment = QProcessEnvironment::systemEnvironment();
+    result.insert(path, systemEnvironment.value(path));
+    // Preserve DISPLAY for X11 as some tests use QtGui.
+#if defined(Q_OS_UNIX) && !defined(Q_OS_MAC)
+    const QString display = QStringLiteral("DISPLAY");
+    const QString displayValue = systemEnvironment.value(display);
+    if (!displayValue.isEmpty())
+        result.insert(display, displayValue);
+#endif
     return result;
 }
 
@@ -486,6 +494,7 @@ void tst_Selftests::doRunSubTest(QString const& subdir, QStringList const& logge
     // newer than the valgrind version, such that valgrind can't understand the
     // debug information on the binary.
     if (subdir != QLatin1String("exceptionthrow")
+        && subdir != QLatin1String("cmptest") // QImage comparison requires QGuiApplication
         && subdir != QLatin1String("fetchbogus")
         && subdir != QLatin1String("xunit")
         && subdir != QLatin1String("benchlibcallgrind"))
@@ -524,7 +533,9 @@ void tst_Selftests::doRunSubTest(QString const& subdir, QStringList const& logge
                 }
             }
         } else {
-            QCOMPARE(res.count(), exp.count());
+            QVERIFY2(res.count() == exp.count(),
+                     qPrintable(QString::fromLatin1("Mismatch in line count: %1 != %2 (%3).")
+                                .arg(res.count()).arg(exp.count()).arg(loggers.at(n))));
         }
 
         // For xml output formats, verify that the log is valid XML.
@@ -585,7 +596,9 @@ void tst_Selftests::doRunSubTest(QString const& subdir, QStringList const& logge
 
                 QCOMPARE(actualResult, expectedResult);
             } else {
-                QCOMPARE(output, expected);
+                QVERIFY2(output == expected,
+                         qPrintable(QString::fromLatin1("Mismatch at line %1 (%2): '%3' != '%4'")
+                                    .arg(i).arg(loggers.at(n), output, expected)));
             }
 
             benchmark = line.startsWith("RESULT : ");
@@ -758,9 +771,12 @@ void tst_Selftests::cleanup()
 
     // Remove the test output files
     for (int i = 0; i < loggers.count(); ++i) {
-        QString logFile = logName(loggers[i]);
-        if (!logFile.isEmpty())
-            QVERIFY(QFile::remove(logFile));
+        QString logFileName = logName(loggers[i]);
+        if (!logFileName.isEmpty()) {
+            QFile logFile(logFileName);
+            if (logFile.exists())
+                QVERIFY2(logFile.remove(), qPrintable(QString::fromLatin1("Cannot remove file '%1': %2: ").arg(logFileName, logFile.errorString())));
+        }
     }
 }