testlib: Improve verbose and XPASS output

Previously, verbose (-v2) and XPASS test output showed all QCOMPAREs as
"COMPARE()", making it impossible to see what was compared and difficult
to match the output to the source of a test containing many calls to
QCOMPARE.

This commit changes testlib's internal compare_helper API so that string
representations of the compared expressions are always passed to
QTestResult::compare() when available, and can thus be shown in the
verbose and XPASS output.  The XPASS output has also been changed to
state explicitly that the comparison succeeded unexpectedly, bringing it
in line with the XPASS output resulting from a call to QVERIFY.

This commit also changes all calls to compare_helper() to call the
eight-argument version of the function, which simplifies much of the
calling code.  The now obsolete four-argument version of
compare_helper() has been changed to output a warning that it is
obsolete.  It will be removed once other modules have had some time to
catch up.

The improved XPASS and verbose output is demonstrated by the expectfail
and verbose2 selftests.

Change-Id: I8baa46d5dd30e6c43b26f366c34dc5b64aab5f7c
Reviewed-by: Rohan McGovern <rohan.mcgovern@nokia.com>

6 files changed, 73 insertions, 48 deletions

diff --git a/src/testlib/qtest.h b/src/testlib/qtest.h
index 32b2f0ca9c..477d344d71 100644
--- a/src/testlib/qtest.h
+++ b/src/testlib/qtest.h
@@ -210,7 +210,7 @@ inline bool qCompare(QStringList const &t1, QStringList const &t2,
             isOk = false;
         }
     }
-    return compare_helper(isOk, msg, file, line);
+    return compare_helper(isOk, msg, 0, 0, actual, expected, file, line);
 }
 
 template <typename T>
diff --git a/src/testlib/qtest_gui.h b/src/testlib/qtest_gui.h
index d39e943926..3067c35e02 100644
--- a/src/testlib/qtest_gui.h
+++ b/src/testlib/qtest_gui.h
@@ -102,29 +102,27 @@ inline bool qCompare(QImage const &t1, QImage const &t2,
         qsnprintf(msg, 1024, "Compared QImages differ.\n"
                   "   Actual (%s).isNull()  : %d\n"
                   "   Expected (%s).isNull(): %d", actual, t1Null, expected, t2Null);
-        return compare_helper(false, msg, file, line);
+        return compare_helper(false, msg, 0, 0, actual, expected, file, line);
     }
     if (t1Null && t2Null)
-        return compare_helper(true, "COMPARE()", file, line);
+        return compare_helper(true, 0, 0, 0, actual, expected, file, line);
     if (t1.width() != t2.width() || t2.height() != t2.height()) {
         qsnprintf(msg, 1024, "Compared QImages differ in size.\n"
                   "   Actual (%s)  : %dx%d\n"
                   "   Expected (%s): %dx%d",
                   actual, t1.width(), t1.height(),
                   expected, t2.width(), t2.height());
-        return compare_helper(false, msg, file, line);
+        return compare_helper(false, msg, 0, 0, actual, expected, file, line);
     }
     if (t1.format() != t2.format()) {
         qsnprintf(msg, 1024, "Compared QImages differ in format.\n"
                   "   Actual (%s)  : %d\n"
                   "   Expected (%s): %d",
                   actual, t1.format(), expected, t2.format());
-        return compare_helper(false, msg, file, line);
+        return compare_helper(false, msg, 0, 0, actual, expected, file, line);
     }
-    return (t1 == t2)
-            ? compare_helper(true, "COMPARE()", file, line)
-            : compare_helper(false, "Compared values are not the same",
-                             toString(t1), toString(t2), actual, expected, file, line);
+    return compare_helper(t1 == t2, "Compared values are not the same",
+                          toString(t1), toString(t2), actual, expected, file, line);
 }
 
 #ifndef QTEST_NO_SPECIALIZATIONS
@@ -141,17 +139,17 @@ inline bool qCompare(QPixmap const &t1, QPixmap const &t2, const char *actual, c
         qsnprintf(msg, 1024, "Compared QPixmaps differ.\n"
                   "   Actual (%s).isNull()  : %d\n"
                   "   Expected (%s).isNull(): %d", actual, t1Null, expected, t2Null);
-        return compare_helper(false, msg, file, line);
+        return compare_helper(false, msg, 0, 0, actual, expected, file, line);
     }
     if (t1Null && t2Null)
-        return compare_helper(true, "COMPARE()", file, line);
+        return compare_helper(true, 0, 0, 0, actual, expected, file, line);
     if (t1.width() != t2.width() || t2.height() != t2.height()) {
         qsnprintf(msg, 1024, "Compared QPixmaps differ in size.\n"
                   "   Actual (%s)  : %dx%d\n"
                   "   Expected (%s): %dx%d",
                   actual, t1.width(), t1.height(),
                   expected, t2.width(), t2.height());
-        return compare_helper(false, msg, file, line);
+        return compare_helper(false, msg, 0, 0, actual, expected, file, line);
     }
     return qCompare(t1.toImage(), t2.toImage(), actual, expected, file, line);
 }
diff --git a/src/testlib/qtestcase.cpp b/src/testlib/qtestcase.cpp
index ace080ef94..001a14a121 100644
--- a/src/testlib/qtestcase.cpp
+++ b/src/testlib/qtestcase.cpp
@@ -2412,15 +2412,34 @@ QObject *QTest::testObject()
  */
 bool QTest::compare_helper(bool success, const char *msg, const char *file, int line)
 {
+    static bool warned = false;
+    if (!warned) {
+        warned = true;
+        QTest::qWarn("QTest::compare_helper(bool, const char *, const char *, int) is obsolete "
+                     "and will soon be removed. Please update your code to use the other "
+                     "version of this function.");
+    }
+
     return QTestResult::compare(success, msg, file, line);
 }
 
 /*! \internal
+    This function is called by various specializations of QTest::qCompare
+    to decide whether to report a failure and to produce verbose test output.
+
+    The failureMsg parameter can be null, in which case a default message
+    will be output if the compare fails.  If the compare succeeds, failureMsg
+    will not be output.
+
+    If the caller has already passed a failure message showing the compared
+    values, or if those values cannot be stringified, val1 and val2 can be null.
  */
-bool QTest::compare_helper(bool success, const char *msg, char *val1, char *val2,
-                    const char *actual, const char *expected, const char *file, int line)
+bool QTest::compare_helper(bool success, const char *failureMsg,
+                           char *val1, char *val2,
+                           const char *actual, const char *expected,
+                           const char *file, int line)
 {
-    return QTestResult::compare(success, msg, val1, val2, actual, expected, file, line);
+    return QTestResult::compare(success, failureMsg, val1, val2, actual, expected, file, line);
 }
 
 /*! \fn bool QTest::qCompare<float>(float const &t1, float const &t2, const char *actual, const char *expected, const char *file, int line)
@@ -2430,10 +2449,8 @@ template <>
 Q_TESTLIB_EXPORT bool QTest::qCompare<float>(float const &t1, float const &t2, const char *actual, const char *expected,
                     const char *file, int line)
 {
-    return qFuzzyCompare(t1, t2)
-            ? compare_helper(true, "COMPARE()", file, line)
-            : compare_helper(false, "Compared floats are not the same (fuzzy compare)",
-                             toString(t1), toString(t2), actual, expected, file, line);
+    return compare_helper(qFuzzyCompare(t1, t2), "Compared floats are not the same (fuzzy compare)",
+                          toString(t1), toString(t2), actual, expected, file, line);
 }
 
 /*! \fn bool QTest::qCompare<double>(double const &t1, double const &t2, const char *actual, const char *expected, const char *file, int line)
@@ -2443,10 +2460,8 @@ template <>
 Q_TESTLIB_EXPORT bool QTest::qCompare<double>(double const &t1, double const &t2, const char *actual, const char *expected,
                     const char *file, int line)
 {
-    return qFuzzyCompare(t1, t2)
-            ? compare_helper(true, "COMPARE()", file, line)
-            : compare_helper(false, "Compared doubles are not the same (fuzzy compare)",
-                             toString(t1), toString(t2), actual, expected, file, line);
+    return compare_helper(qFuzzyCompare(t1, t2), "Compared doubles are not the same (fuzzy compare)",
+                          toString(t1), toString(t2), actual, expected, file, line);
 }
 
 #define TO_STRING_IMPL(TYPE, FORMAT) \
@@ -2499,10 +2514,8 @@ char *QTest::toString(const void *p)
 bool QTest::compare_string_helper(const char *t1, const char *t2, const char *actual,
                                   const char *expected, const char *file, int line)
 {
-    return (qstrcmp(t1, t2) == 0)
-            ? compare_helper(true, "COMPARE()", file, line)
-            : compare_helper(false, "Compared strings are not the same",
-                             toString(t1), toString(t2), actual, expected, file, line);
+    return compare_helper(qstrcmp(t1, t2) == 0, "Compared strings are not the same",
+                          toString(t1), toString(t2), actual, expected, file, line);
 }
 
 /*! \fn bool QTest::compare_ptr_helper(const void *t1, const void *t2, const char *actual, const char *expected, const char *file, int line);
diff --git a/src/testlib/qtestcase.h b/src/testlib/qtestcase.h
index c64420b000..a344736f7b 100644
--- a/src/testlib/qtestcase.h
+++ b/src/testlib/qtestcase.h
@@ -200,7 +200,8 @@ namespace QTest
 
     Q_TESTLIB_EXPORT bool compare_helper(bool success, const char *msg, const char *file,
                                           int line);
-    Q_TESTLIB_EXPORT bool compare_helper(bool success, const char *msg, char *val1, char *val2,
+    Q_TESTLIB_EXPORT bool compare_helper(bool success, const char *failureMsg,
+                                         char *val1, char *val2,
                                          const char *actual, const char *expected,
                                          const char *file, int line);
     Q_TESTLIB_EXPORT void qSleep(int ms);
@@ -217,13 +218,10 @@ namespace QTest
     inline bool qCompare(T const &t1, T const &t2, const char *actual, const char *expected,
                         const char *file, int line)
     {
-        return (t1 == t2)
-            ? compare_helper(true, "COMPARE()", file, line)
-            : compare_helper(false, "Compared values are not the same",
-                             toString<T>(t1), toString<T>(t2), actual, expected, file, line);
+        return compare_helper(t1 == t2, "Compared values are not the same",
+                              toString<T>(t1), toString<T>(t2), actual, expected, file, line);
     }
 
-
     template <>
     Q_TESTLIB_EXPORT bool qCompare<float>(float const &t1, float const &t2,
                     const char *actual, const char *expected, const char *file, int line);
@@ -235,10 +233,8 @@ namespace QTest
     inline bool compare_ptr_helper(const void *t1, const void *t2, const char *actual,
                                    const char *expected, const char *file, int line)
     {
-        return (t1 == t2)
-            ? compare_helper(true, "COMPARE()", file, line)
-            : compare_helper(false, "Compared pointers are not the same",
-                             toString(t1), toString(t2), actual, expected, file, line);
+        return compare_helper(t1 == t2, "Compared pointers are not the same",
+                              toString(t1), toString(t2), actual, expected, file, line);
     }
 
     Q_TESTLIB_EXPORT bool compare_string_helper(const char *t1, const char *t2, const char *actual,
diff --git a/src/testlib/qtestresult.cpp b/src/testlib/qtestresult.cpp
index 14ab29af2b..231fc8f3b1 100644
--- a/src/testlib/qtestresult.cpp
+++ b/src/testlib/qtestresult.cpp
@@ -254,22 +254,38 @@ bool QTestResult::compare(bool success, const char *msg, const char *file, int l
     return checkStatement(success, msg, file, line);
 }
 
-bool QTestResult::compare(bool success, const char *msg, char *val1, char *val2,
-                          const char *actual, const char *expected, const char *file, int line)
+bool QTestResult::compare(bool success, const char *failureMsg,
+                          char *val1, char *val2,
+                          const char *actual, const char *expected,
+                          const char *file, int line)
 {
     QTEST_ASSERT(expected);
     QTEST_ASSERT(actual);
 
-    if (!val1 && !val2)
-        return compare(success, msg, file, line);
+    char msg[1024];
+
+    if (QTestLog::verboseLevel() >= 2) {
+        qsnprintf(msg, 1024, "QCOMPARE(%s, %s)", actual, expected);
+        QTestLog::info(msg, file, line);
+    }
+
+    if (!failureMsg)
+        failureMsg = "Compared values are not the same";
+
+    if (success && QTest::expectFailMode) {
+        qsnprintf(msg, 1024, "QCOMPARE(%s, %s) returned TRUE unexpectedly.", actual, expected);
+    } else if (val1 || val2) {
+        qsnprintf(msg, 1024, "%s\n   Actual (%s): %s\n   Expected (%s): %s",
+                  failureMsg,
+                  actual, val1 ? val1 : "<null>",
+                  expected, val2 ? val2 : "<null>");
+    } else
+        qsnprintf(msg, 1024, "%s", failureMsg);
 
-    char buf[1024];
-    qsnprintf(buf, 1024, "%s\n   Actual (%s): %s\n   Expected (%s): %s", msg,
-              actual, val1 ? val1 : "<null>",
-              expected, val2 ? val2 : "<null>");
     delete [] val1;
     delete [] val2;
-    return compare(success, buf, file, line);
+
+    return checkStatement(success, msg, file, line);
 }
 
 void QTestResult::addFailure(const char *message, const char *file, int line)
diff --git a/src/testlib/qtestresult_p.h b/src/testlib/qtestresult_p.h
index b060926f2a..1bf070f6cf 100644
--- a/src/testlib/qtestresult_p.h
+++ b/src/testlib/qtestresult_p.h
@@ -77,8 +77,10 @@ public:
 
     static void addFailure(const char *message, const char *file, int line);
     static bool compare(bool success, const char *msg, const char *file, int line);
-    static bool compare(bool success, const char *msg, char *val1, char *val2,
-                        const char *actual, const char *expected, const char *file, int line);
+    static bool compare(bool success, const char *failureMsg,
+                        char *val1, char *val2,
+                        const char *actual, const char *expected,
+                        const char *file, int line);
 
     static void setCurrentGlobalTestData(QTestData *data);
     static void setCurrentTestData(QTestData *data);