testlib: Don't print QCOMPARE values if they lack string representation

Before 0681a2dd5a8095baddb5905fb21a58ce19b958c5, QCOMPARE'ing types
for which no QTest::toString specialization exists did not output
Actual and Expected lines on failure, as that would only print <null>
for both values (which then look like the same value, confusingly).

Commit 0681a2dd5a8095baddb5905fb21a58ce19b958c5 changed that behavior,
and started printing the confusing <null> values.

Take care of the logic in the formatFailMessage function: if both values
are nullptr, then print only the variable names, but not the confusing
<null> text representation of the values.

Remove dead and duplicated code related to the formatting logic, add a
self-test function, and update the expected_cmptest files.

Fixes: QTBUG-104867
Pick-to: 6.4
Change-Id: I4be98e79f91196b14690a2cc0a68ffd50b431a45
Reviewed-by: Tor Arne Vestbø <tor.arne.vestbo@qt.io>
Reviewed-by: Thiago Macieira <thiago.macieira@intel.com>

9 files changed, 128 insertions, 88 deletions

diff --git a/src/testlib/qtestcase.h b/src/testlib/qtestcase.h
index 7db64af8ab..0412f872f7 100644
--- a/src/testlib/qtestcase.h
+++ b/src/testlib/qtestcase.h
@@ -437,17 +437,6 @@ namespace QTest
     Q_TESTLIB_EXPORT QTestData &newRow(const char *dataTag);
     Q_TESTLIB_EXPORT QTestData &addRow(const char *format, ...) Q_ATTRIBUTE_FORMAT_PRINTF(1, 2);
 
-#if QT_VERSION < QT_VERSION_CHECK(6, 0, 0)
-    // kept after adding implementation of <T1, T2> out of paranoia:
-    template <typename T>
-    inline bool qCompare(T const &t1, T const &t2, const char *actual, const char *expected,
-                        const char *file, int line)
-    {
-        return compare_helper(t1 == t2, "Compared values are not the same",
-                              toString(t1), toString(t2), actual, expected, file, line);
-    }
-#endif
-
     Q_TESTLIB_EXPORT bool qCompare(qfloat16 const &t1, qfloat16 const &t2,
                     const char *actual, const char *expected, const char *file, int line);
 
diff --git a/src/testlib/qtestresult.cpp b/src/testlib/qtestresult.cpp
index 14d344f381..556cafd4d4 100644
--- a/src/testlib/qtestresult.cpp
+++ b/src/testlib/qtestresult.cpp
@@ -321,43 +321,47 @@ static const char *rightArgNameForOp(QTest::ComparisonOperation op)
     return op == QTest::ComparisonOperation::CustomCompare ? "Expected " : "Right  ";
 }
 
-// Format failures using the toString() template
-template <class Actual, class Expected>
+// Overload to format failures for "const char *" - no need to strdup().
 void formatFailMessage(char *msg, size_t maxMsgLen,
                        const char *failureMsg,
-                       const Actual &val1, const Expected &val2,
+                       const char *val1, const char *val2,
                        const char *actual, const char *expected,
                        QTest::ComparisonOperation op)
 {
-    auto val1S = QTest::toString(val1);
-    auto val2S = QTest::toString(val2);
-
     size_t len1 = mbstowcs(nullptr, actual, maxMsgLen);    // Last parameter is not ignored on QNX
     size_t len2 = mbstowcs(nullptr, expected, maxMsgLen);  // (result is never larger than this).
-    qsnprintf(msg, maxMsgLen, "%s\n   %s(%s)%*s %s\n   %s(%s)%*s %s", failureMsg,
-              leftArgNameForOp(op), actual, qMax(len1, len2) - len1 + 1, ":",
-              val1S ? val1S : "<null>",
-              rightArgNameForOp(op), expected, qMax(len1, len2) - len2 + 1, ":",
-              val2S ? val2S : "<null>");
-
-    delete [] val1S;
-    delete [] val2S;
+    const int written = qsnprintf(msg, maxMsgLen, "%s\n", failureMsg);
+    msg += written;
+    maxMsgLen -= written;
+
+    if (val1 || val2) {
+        qsnprintf(msg, maxMsgLen, "   %s(%s)%*s %s\n   %s(%s)%*s %s",
+                    leftArgNameForOp(op), actual, qMax(len1, len2) - len1 + 1, ":",
+                    val1 ? val1 : "<null>",
+                    rightArgNameForOp(op), expected, qMax(len1, len2) - len2 + 1, ":",
+                    val2 ? val2 : "<null>");
+    } else {
+        // only print variable names if neither value can be represented as a string
+        qsnprintf(msg, maxMsgLen, "   %s: %s\n   %s: %s",
+                    leftArgNameForOp(op), actual, rightArgNameForOp(op), expected);
+    }
 }
 
-// Overload to format failures for "const char *" - no need to strdup().
+// Format failures using the toString() template
+template <class Actual, class Expected>
 void formatFailMessage(char *msg, size_t maxMsgLen,
                        const char *failureMsg,
-                       const char *val1, const char *val2,
+                       const Actual &val1, const Expected &val2,
                        const char *actual, const char *expected,
                        QTest::ComparisonOperation op)
 {
-    size_t len1 = mbstowcs(nullptr, actual, maxMsgLen);    // Last parameter is not ignored on QNX
-    size_t len2 = mbstowcs(nullptr, expected, maxMsgLen);  // (result is never larger than this).
-    qsnprintf(msg, maxMsgLen, "%s\n   %s(%s)%*s %s\n   %s(%s)%*s %s", failureMsg,
-              leftArgNameForOp(op), actual, qMax(len1, len2) - len1 + 1, ":",
-              val1 ? val1 : "<null>",
-              rightArgNameForOp(op), expected, qMax(len1, len2) - len2 + 1, ":",
-              val2 ? val2 : "<null>");
+    const char *val1S = QTest::toString(val1);
+    const char *val2S = QTest::toString(val2);
+
+    formatFailMessage(msg, maxMsgLen, failureMsg, val1S, val2S, actual, expected, op);
+
+    delete [] val1S;
+    delete [] val2S;
 }
 
 template <class Actual, class Expected>
diff --git a/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp b/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp
index 5633a0252b..5842d7b3b1 100644
--- a/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp
+++ b/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp
@@ -114,6 +114,7 @@ private slots:
     void compare_pointerfuncs();
     void compare_tostring();
     void compare_tostring_data();
+    void compare_unknown();
     void compareQObjects();
     void compareQStringLists();
     void compareQStringLists_data();
@@ -323,6 +324,14 @@ void tst_Cmptest::compare_tostring()
     QCOMPARE(actual, expected);
 }
 
+void tst_Cmptest::compare_unknown()
+{
+    std::string a("a");
+    std::string b("b");
+
+    QCOMPARE(a, b);
+}
+
 void tst_Cmptest::compareQStringLists_data()
 {
     QTest::addColumn<QStringList>("opA");
diff --git a/tests/auto/testlib/selftests/expected_cmptest.junitxml b/tests/auto/testlib/selftests/expected_cmptest.junitxml
index 9509c067e2..2c92cc66fd 100644
--- a/tests/auto/testlib/selftests/expected_cmptest.junitxml
+++ b/tests/auto/testlib/selftests/expected_cmptest.junitxml
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" ?>
-<testsuite name="tst_Cmptest" timestamp="@TEST_START_TIME@" hostname="@HOSTNAME@" tests="67" failures="46" errors="0" skipped="0" time="@TEST_DURATION@">
+<testsuite name="tst_Cmptest" timestamp="@TEST_START_TIME@" hostname="@HOSTNAME@" tests="68" failures="47" errors="0" skipped="0" time="@TEST_DURATION@">
   <properties>
     <property name="QTestVersion" value="@INSERT_QT_VERSION_HERE@"/>
     <property name="QtVersion" value="@INSERT_QT_VERSION_HERE@"/>
@@ -78,6 +78,12 @@
    Expected (expected): QVariant(PhonyClass,<value not representable as string>)]]>
     </failure>
   </testcase>
+  <testcase name="compare_unknown" classname="tst_Cmptest" time="@TEST_DURATION@">
+    <failure type="fail" message="Compared values are not the same">
+      <![CDATA[   Actual   : a
+   Expected : b]]>
+    </failure>
+  </testcase>
   <testcase name="compareQObjects" classname="tst_Cmptest" time="@TEST_DURATION@">
     <failure type="fail" message="Compared QObject pointers are not the same">
       <![CDATA[   Actual   (&object1): QObject/"object1"
diff --git a/tests/auto/testlib/selftests/expected_cmptest.lightxml b/tests/auto/testlib/selftests/expected_cmptest.lightxml
index df00b1a884..9082dd56d3 100644
--- a/tests/auto/testlib/selftests/expected_cmptest.lightxml
+++ b/tests/auto/testlib/selftests/expected_cmptest.lightxml
@@ -109,6 +109,14 @@
     </Incident>
     <Duration msecs="0"/>
   </TestFunction>
+  <TestFunction name="compare_unknown">
+    <Incident type="fail" file="qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp" line="0">
+      <Description><![CDATA[Compared values are not the same
+   Actual   : a
+   Expected : b]]></Description>
+    </Incident>
+    <Duration msecs="0"/>
+  </TestFunction>
   <TestFunction name="compareQObjects">
     <Incident type="fail" file="qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp" line="0">
       <Description><![CDATA[Compared QObject pointers are not the same
diff --git a/tests/auto/testlib/selftests/expected_cmptest.tap b/tests/auto/testlib/selftests/expected_cmptest.tap
index a282fb1f4e..9a7f13fe7e 100644
--- a/tests/auto/testlib/selftests/expected_cmptest.tap
+++ b/tests/auto/testlib/selftests/expected_cmptest.tap
@@ -139,7 +139,16 @@ not ok 18 - compare_tostring(both non-null user type)
   file: qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp
   line: 0
   ...
-not ok 19 - compareQObjects()
+not ok 19 - compare_unknown()
+  ---
+  # Compared values are not the same
+   Actual   : a
+   Expected : b
+  at: tst_Cmptest::compare_unknown() (qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp:0)
+  file: qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp
+  line: 0
+  ...
+not ok 20 - compareQObjects()
   ---
   type: QCOMPARE
   message: Compared QObject pointers are not the same
@@ -151,9 +160,9 @@ not ok 19 - compareQObjects()
   file: qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp
   line: 0
   ...
-ok 20 - compareQStringLists(empty lists)
-ok 21 - compareQStringLists(equal lists)
-not ok 22 - compareQStringLists(last item different)
+ok 21 - compareQStringLists(empty lists)
+ok 22 - compareQStringLists(equal lists)
+not ok 23 - compareQStringLists(last item different)
   ---
   type: QCOMPARE
   message: Compared lists differ at index 2.
@@ -165,7 +174,7 @@ not ok 22 - compareQStringLists(last item different)
   file: qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp
   line: 0
   ...
-not ok 23 - compareQStringLists(second-last item different)
+not ok 24 - compareQStringLists(second-last item different)
   ---
   type: QCOMPARE
   message: Compared lists differ at index 2.
@@ -177,7 +186,7 @@ not ok 23 - compareQStringLists(second-last item different)
   file: qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp
   line: 0
   ...
-not ok 24 - compareQStringLists(prefix)
+not ok 25 - compareQStringLists(prefix)
   ---
   # Compared lists have different sizes.
    Actual   (opA) size: 2
@@ -186,7 +195,7 @@ not ok 24 - compareQStringLists(prefix)
   file: qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp
   line: 0
   ...
-not ok 25 - compareQStringLists(short list second)
+not ok 26 - compareQStringLists(short list second)
   ---
   # Compared lists have different sizes.
    Actual   (opA) size: 12
@@ -195,7 +204,7 @@ not ok 25 - compareQStringLists(short list second)
   file: qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp
   line: 0
   ...
-not ok 26 - compareQStringLists(short list first)
+not ok 27 - compareQStringLists(short list first)
   ---
   # Compared lists have different sizes.
    Actual   (opA) size: 1
@@ -204,8 +213,8 @@ not ok 26 - compareQStringLists(short list first)
   file: qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp
   line: 0
   ...
-ok 27 - compareQListInt(match)
-not ok 28 - compareQListInt(size mismatch)
+ok 28 - compareQListInt(match)
+not ok 29 - compareQListInt(size mismatch)
   ---
   # Compared lists have different sizes.
    Actual   (actual) size: 2
@@ -214,7 +223,7 @@ not ok 28 - compareQListInt(size mismatch)
   file: qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp
   line: 0
   ...
-not ok 29 - compareQListInt(value mismatch)
+not ok 30 - compareQListInt(value mismatch)
   ---
   type: QCOMPARE
   message: Compared lists differ at index 2.
@@ -226,8 +235,8 @@ not ok 29 - compareQListInt(value mismatch)
   file: qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp
   line: 0
   ...
-ok 30 - compareQListIntToArray(match)
-not ok 31 - compareQListIntToArray(size mismatch)
+ok 31 - compareQListIntToArray(match)
+not ok 32 - compareQListIntToArray(size mismatch)
   ---
   # Compared lists have different sizes.
    Actual   (actual) size: 2
@@ -236,7 +245,7 @@ not ok 31 - compareQListIntToArray(size mismatch)
   file: qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp
   line: 0
   ...
-not ok 32 - compareQListIntToArray(value mismatch)
+not ok 33 - compareQListIntToArray(value mismatch)
   ---
   type: QCOMPARE
   message: Compared lists differ at index 2.
@@ -248,8 +257,8 @@ not ok 32 - compareQListIntToArray(value mismatch)
   file: qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp
   line: 0
   ...
-ok 33 - compareQListIntToInitializerList(match)
-not ok 34 - compareQListIntToInitializerList(size mismatch)
+ok 34 - compareQListIntToInitializerList(match)
+not ok 35 - compareQListIntToInitializerList(size mismatch)
   ---
   # Compared lists have different sizes.
    Actual   (actual) size: 2
@@ -258,7 +267,7 @@ not ok 34 - compareQListIntToInitializerList(size mismatch)
   file: qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp
   line: 0
   ...
-not ok 35 - compareQListIntToInitializerList(value mismatch)
+not ok 36 - compareQListIntToInitializerList(value mismatch)
   ---
   type: QCOMPARE
   message: Compared lists differ at index 2.
@@ -270,7 +279,7 @@ not ok 35 - compareQListIntToInitializerList(value mismatch)
   file: qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp
   line: 0
   ...
-not ok 36 - compareQListDouble()
+not ok 37 - compareQListDouble()
   ---
   type: QCOMPARE
   message: Compared lists differ at index 0.
@@ -282,8 +291,8 @@ not ok 36 - compareQListDouble()
   file: qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp
   line: 0
   ...
-ok 37 - compareQColor(Qt::yellow vs "yellow")
-not ok 38 - compareQColor(Qt::yellow vs Qt::green)
+ok 38 - compareQColor(Qt::yellow vs "yellow")
+not ok 39 - compareQColor(Qt::yellow vs Qt::green)
   ---
   type: QCOMPARE
   message: Compared values are not the same
@@ -295,7 +304,7 @@ not ok 38 - compareQColor(Qt::yellow vs Qt::green)
   file: qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp
   line: 0
   ...
-not ok 39 - compareQColor(0x88ff0000 vs 0xffff0000)
+not ok 40 - compareQColor(0x88ff0000 vs 0xffff0000)
   ---
   type: QCOMPARE
   message: Compared values are not the same
@@ -307,8 +316,8 @@ not ok 39 - compareQColor(0x88ff0000 vs 0xffff0000)
   file: qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp
   line: 0
   ...
-ok 40 - compareQPixmaps(both null)
-not ok 41 - compareQPixmaps(one null)
+ok 41 - compareQPixmaps(both null)
+not ok 42 - compareQPixmaps(one null)
   ---
   type: QCOMPARE
   message: Compared QPixmaps differ.
@@ -320,7 +329,7 @@ not ok 41 - compareQPixmaps(one null)
   file: qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp
   line: 0
   ...
-not ok 42 - compareQPixmaps(other null)
+not ok 43 - compareQPixmaps(other null)
   ---
   type: QCOMPARE
   message: Compared QPixmaps differ.
@@ -332,8 +341,8 @@ not ok 42 - compareQPixmaps(other null)
   file: qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp
   line: 0
   ...
-ok 43 - compareQPixmaps(equal)
-not ok 44 - compareQPixmaps(different size)
+ok 44 - compareQPixmaps(equal)
+not ok 45 - compareQPixmaps(different size)
   ---
   type: QCOMPARE
   message: Compared QPixmaps differ in size.
@@ -345,14 +354,14 @@ not ok 44 - compareQPixmaps(different size)
   file: qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp
   line: 0
   ...
-not ok 45 - compareQPixmaps(different pixels)
+not ok 46 - compareQPixmaps(different pixels)
   ---
   # Compared values are not the same
   at: tst_Cmptest::compareQPixmaps() (qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp:0)
   file: qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp
   line: 0
   ...
-not ok 46 - compareQPixmaps(different dpr)
+not ok 47 - compareQPixmaps(different dpr)
   ---
   type: QCOMPARE
   message: Compared QPixmaps differ in device pixel ratio.
@@ -364,8 +373,8 @@ not ok 46 - compareQPixmaps(different dpr)
   file: qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp
   line: 0
   ...
-ok 47 - compareQImages(both null)
-not ok 48 - compareQImages(one null)
+ok 48 - compareQImages(both null)
+not ok 49 - compareQImages(one null)
   ---
   type: QCOMPARE
   message: Compared QImages differ.
@@ -377,7 +386,7 @@ not ok 48 - compareQImages(one null)
   file: qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp
   line: 0
   ...
-not ok 49 - compareQImages(other null)
+not ok 50 - compareQImages(other null)
   ---
   type: QCOMPARE
   message: Compared QImages differ.
@@ -389,8 +398,8 @@ not ok 49 - compareQImages(other null)
   file: qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp
   line: 0
   ...
-ok 50 - compareQImages(equal)
-not ok 51 - compareQImages(different size)
+ok 51 - compareQImages(equal)
+not ok 52 - compareQImages(different size)
   ---
   type: QCOMPARE
   message: Compared QImages differ in size.
@@ -402,7 +411,7 @@ not ok 51 - compareQImages(different size)
   file: qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp
   line: 0
   ...
-not ok 52 - compareQImages(different format)
+not ok 53 - compareQImages(different format)
   ---
   type: QCOMPARE
   message: Compared QImages differ in format.
@@ -414,14 +423,14 @@ not ok 52 - compareQImages(different format)
   file: qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp
   line: 0
   ...
-not ok 53 - compareQImages(different pixels)
+not ok 54 - compareQImages(different pixels)
   ---
   # Compared values are not the same
   at: tst_Cmptest::compareQImages() (qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp:0)
   file: qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp
   line: 0
   ...
-not ok 54 - compareQImages(different dpr)
+not ok 55 - compareQImages(different dpr)
   ---
   type: QCOMPARE
   message: Compared QImages differ in device pixel ratio.
@@ -433,8 +442,8 @@ not ok 54 - compareQImages(different dpr)
   file: qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp
   line: 0
   ...
-ok 55 - compareQRegion(equal-empty)
-not ok 56 - compareQRegion(1-empty)
+ok 56 - compareQRegion(equal-empty)
+not ok 57 - compareQRegion(1-empty)
   ---
   type: QCOMPARE
   message: Compared values are not the same
@@ -446,8 +455,8 @@ not ok 56 - compareQRegion(1-empty)
   file: qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp
   line: 0
   ...
-ok 57 - compareQRegion(equal)
-not ok 58 - compareQRegion(different lists)
+ok 58 - compareQRegion(equal)
+not ok 59 - compareQRegion(different lists)
   ---
   type: QCOMPARE
   message: Compared values are not the same
@@ -459,7 +468,7 @@ not ok 58 - compareQRegion(different lists)
   file: qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp
   line: 0
   ...
-not ok 59 - compareQVector2D()
+not ok 60 - compareQVector2D()
   ---
   type: QCOMPARE
   message: Compared values are not the same
@@ -471,7 +480,7 @@ not ok 59 - compareQVector2D()
   file: qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp
   line: 0
   ...
-not ok 60 - compareQVector3D()
+not ok 61 - compareQVector3D()
   ---
   type: QCOMPARE
   message: Compared values are not the same
@@ -483,7 +492,7 @@ not ok 60 - compareQVector3D()
   file: qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp
   line: 0
   ...
-not ok 61 - compareQVector4D()
+not ok 62 - compareQVector4D()
   ---
   type: QCOMPARE
   message: Compared values are not the same
@@ -495,7 +504,7 @@ not ok 61 - compareQVector4D()
   file: qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp
   line: 0
   ...
-not ok 62 - verify()
+not ok 63 - verify()
   ---
   type: QVERIFY
   message: Verification failed
@@ -507,7 +516,7 @@ not ok 62 - verify()
   file: qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp
   line: 0
   ...
-not ok 63 - verify2()
+not ok 64 - verify2()
   ---
   type: QVERIFY
   message: 42
@@ -519,7 +528,7 @@ not ok 63 - verify2()
   file: qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp
   line: 0
   ...
-not ok 64 - tryVerify()
+not ok 65 - tryVerify()
   ---
   type: QVERIFY
   message: Verification failed
@@ -531,7 +540,7 @@ not ok 64 - tryVerify()
   file: qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp
   line: 0
   ...
-not ok 65 - tryVerify2()
+not ok 66 - tryVerify2()
   ---
   type: QVERIFY
   message: 42
@@ -543,9 +552,9 @@ not ok 65 - tryVerify2()
   file: qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp
   line: 0
   ...
-ok 66 - verifyExplicitOperatorBool()
-ok 67 - cleanupTestCase()
-1..67
-# tests 67
+ok 67 - verifyExplicitOperatorBool()
+ok 68 - cleanupTestCase()
+1..68
+# tests 68
 # pass 21
-# fail 46
+# fail 47
diff --git a/tests/auto/testlib/selftests/expected_cmptest.teamcity b/tests/auto/testlib/selftests/expected_cmptest.teamcity
index 6bddfef86f..2a0c3b9ff9 100644
--- a/tests/auto/testlib/selftests/expected_cmptest.teamcity
+++ b/tests/auto/testlib/selftests/expected_cmptest.teamcity
@@ -46,6 +46,9 @@
 ##teamcity[testStarted name='compare_tostring(both non-null user type)' flowId='tst_Cmptest']
 ##teamcity[testFailed name='compare_tostring(both non-null user type)' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp(0)|]' details='Compared values are not the same|n Actual (actual) : QVariant(PhonyClass,<value not representable as string>)|n Expected (expected): QVariant(PhonyClass,<value not representable as string>)' flowId='tst_Cmptest']
 ##teamcity[testFinished name='compare_tostring(both non-null user type)' flowId='tst_Cmptest']
+##teamcity[testStarted name='compare_unknown()' flowId='tst_Cmptest']
+##teamcity[testFailed name='compare_unknown()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp(0)|]' details='Compared values are not the same|n Actual : a|n Expected : b' flowId='tst_Cmptest']
+##teamcity[testFinished name='compare_unknown()' flowId='tst_Cmptest']
 ##teamcity[testStarted name='compareQObjects()' flowId='tst_Cmptest']
 ##teamcity[testFailed name='compareQObjects()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp(0)|]' details='Compared QObject pointers are not the same|n Actual (&object1): QObject/"object1"|n Expected (&object2): QObject/"object2"' flowId='tst_Cmptest']
 ##teamcity[testFinished name='compareQObjects()' flowId='tst_Cmptest']
diff --git a/tests/auto/testlib/selftests/expected_cmptest.txt b/tests/auto/testlib/selftests/expected_cmptest.txt
index 0fe70cdef6..993cbfa53b 100644
--- a/tests/auto/testlib/selftests/expected_cmptest.txt
+++ b/tests/auto/testlib/selftests/expected_cmptest.txt
@@ -51,6 +51,10 @@ FAIL!  : tst_Cmptest::compare_tostring(both non-null user type) Compared values
    Actual   (actual)  : QVariant(PhonyClass,<value not representable as string>)
    Expected (expected): QVariant(PhonyClass,<value not representable as string>)
    Loc: [qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp(0)]
+FAIL!  : tst_Cmptest::compare_unknown() Compared values are not the same
+   Actual   : a
+   Expected : b
+   Loc: [qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp(0)]
 FAIL!  : tst_Cmptest::compareQObjects() Compared QObject pointers are not the same
    Actual   (&object1): QObject/"object1"
    Expected (&object2): QObject/"object2"
@@ -193,5 +197,5 @@ FAIL!  : tst_Cmptest::tryVerify2() 'opaqueFunc() < 2' returned FALSE. (42)
    Loc: [qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp(0)]
 PASS   : tst_Cmptest::verifyExplicitOperatorBool()
 PASS   : tst_Cmptest::cleanupTestCase()
-Totals: 21 passed, 46 failed, 0 skipped, 0 blacklisted, 0ms
+Totals: 21 passed, 47 failed, 0 skipped, 0 blacklisted, 0ms
 ********* Finished testing of tst_Cmptest *********
diff --git a/tests/auto/testlib/selftests/expected_cmptest.xml b/tests/auto/testlib/selftests/expected_cmptest.xml
index 509dca88b7..08c8d4694f 100644
--- a/tests/auto/testlib/selftests/expected_cmptest.xml
+++ b/tests/auto/testlib/selftests/expected_cmptest.xml
@@ -111,6 +111,14 @@
     </Incident>
     <Duration msecs="0"/>
   </TestFunction>
+  <TestFunction name="compare_unknown">
+    <Incident type="fail" file="qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp" line="0">
+      <Description><![CDATA[Compared values are not the same
+   Actual   : a
+   Expected : b]]></Description>
+    </Incident>
+    <Duration msecs="0"/>
+  </TestFunction>
   <TestFunction name="compareQObjects">
     <Incident type="fail" file="qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp" line="0">
       <Description><![CDATA[Compared QObject pointers are not the same