From d81b065bb3cea85a98f2299354d05ce440032f1f Mon Sep 17 00:00:00 2001 From: Jason McDonald Date: Wed, 29 Feb 2012 15:43:42 +1000 Subject: testlib: Improve verbose and XPASS output Previously, verbose (-v2) and XPASS test output showed all QCOMPAREs as "COMPARE()", making it impossible to see what was compared and difficult to match the output to the source of a test containing many calls to QCOMPARE. This commit changes testlib's internal compare_helper API so that string representations of the compared expressions are always passed to QTestResult::compare() when available, and can thus be shown in the verbose and XPASS output. The XPASS output has also been changed to state explicitly that the comparison succeeded unexpectedly, bringing it in line with the XPASS output resulting from a call to QVERIFY. This commit also changes all calls to compare_helper() to call the eight-argument version of the function, which simplifies much of the calling code. The now obsolete four-argument version of compare_helper() has been changed to output a warning that it is obsolete. It will be removed once other modules have had some time to catch up. The improved XPASS and verbose output is demonstrated by the expectfail and verbose2 selftests. Change-Id: I8baa46d5dd30e6c43b26f366c34dc5b64aab5f7c Reviewed-by: Rohan McGovern --- tests/auto/testlib/selftests/tst_selftests.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'tests/auto/testlib/selftests/tst_selftests.cpp') diff --git a/tests/auto/testlib/selftests/tst_selftests.cpp b/tests/auto/testlib/selftests/tst_selftests.cpp index 5d216992d7..a9ec4e31f2 100644 --- a/tests/auto/testlib/selftests/tst_selftests.cpp +++ b/tests/auto/testlib/selftests/tst_selftests.cpp @@ -104,7 +104,7 @@ inline bool qCompare } if (qAbs(qreal(r1.total) - qreal(r2.total)) <= qreal(r1.total)*variance) { - return compare_helper(true, "COMPARE()", file, line); + return compare_helper(true, 0, 0, 0, actual, expected, file, line); } // Whoops, didn't match. Compare the whole string for the most useful failure message. -- cgit v1.2.3 From 8854b74cf4511c758b998abfc269cc1f3a11d4a1 Mon Sep 17 00:00:00 2001 From: Jason McDonald Date: Wed, 7 Mar 2012 16:17:46 +1000 Subject: selftests: Improve reporting of problems with loading expected output If the expected output file was missing (e.g. not included in selftests.qrc), tst_selftests would trigger an assert inside QList by calling QList::at() on an empty list. Make tst_selftests detect this error instead and give a meaningful error message. When loading expected output for the crashes selftest, where there are several alternative versions of the expected output, the code reused the "exp" variable when loading the alternative test output files. This caused the last file loaded to be used unintentionally if none of the alternative files had the correct number of lines. Use a different variable so that exp remains empty if none of the alternatives are valid and a failure can be reported. Change-Id: I35b2a3d905d069d3ee8dcb1447836eb68d5c8612 Reviewed-by: Rohan McGovern --- tests/auto/testlib/selftests/tst_selftests.cpp | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) (limited to 'tests/auto/testlib/selftests/tst_selftests.cpp') diff --git a/tests/auto/testlib/selftests/tst_selftests.cpp b/tests/auto/testlib/selftests/tst_selftests.cpp index a9ec4e31f2..97458abfbf 100644 --- a/tests/auto/testlib/selftests/tst_selftests.cpp +++ b/tests/auto/testlib/selftests/tst_selftests.cpp @@ -571,12 +571,13 @@ void tst_Selftests::doRunSubTest(QString const& subdir, QStringList const& logge // the actual output. if (exp.count() == 0) { QList > expArr; + QList tmp; int i = 1; do { - exp = expectedResult(subdir + QString("_%1").arg(i++), logger); - if (exp.count()) - expArr += exp; - } while (exp.count()); + tmp = expectedResult(subdir + QString("_%1").arg(i++), logger); + if (tmp.count()) + expArr += tmp; + } while (tmp.count()); for (int j = 0; j < expArr.count(); ++j) { if (res.count() == expArr.at(j).count()) { @@ -584,12 +585,24 @@ void tst_Selftests::doRunSubTest(QString const& subdir, QStringList const& logge break; } } + + if (expArr.count()) { + QVERIFY2(exp.count(), + qPrintable(QString::fromLatin1("None of the expected output files for " + "%1 format has matching line count.") + .arg(loggers.at(n)))); + } } else { QVERIFY2(res.count() == exp.count(), qPrintable(QString::fromLatin1("Mismatch in line count: %1 != %2 (%3).") .arg(res.count()).arg(exp.count()).arg(loggers.at(n)))); } + // By this point, we should have loaded a non-empty expected data file. + QVERIFY2(exp.count(), + qPrintable(QString::fromLatin1("Expected test data for %1 format is empty or not found.") + .arg(loggers.at(n)))); + // For xml output formats, verify that the log is valid XML. if (logFormat(logger) == "xunitxml" || logFormat(logger) == "xml" || logFormat(logger) == "lightxml") { QByteArray xml(actualOutputs[n]); -- cgit v1.2.3