diff options
Diffstat (limited to 'tests/auto/testlib/selftests/tst_selftests.cpp')
| -rw-r--r-- | tests/auto/testlib/selftests/tst_selftests.cpp | 395 |
1 files changed, 233 insertions, 162 deletions
diff --git a/tests/auto/testlib/selftests/tst_selftests.cpp b/tests/auto/testlib/selftests/tst_selftests.cpp index 63e5721e7e..c5f847562e 100644 --- a/tests/auto/testlib/selftests/tst_selftests.cpp +++ b/tests/auto/testlib/selftests/tst_selftests.cpp @@ -62,11 +62,22 @@ private slots: private: void doRunSubTest(QString const& subdir, QStringList const& loggers, QStringList const& arguments, bool crashes); + bool compareOutput(const QString &logger, const QString &subdir, + const QByteArray &rawOutput, const QByteArrayList &actual, + const QByteArrayList &expected, + QString *errorMessage) const; + bool compareLine(const QString &logger, const QString &subdir, bool benchmark, + const QString &actualLine, const QString &expLine, + QString *errorMessage) const; + bool checkXml(const QString &logger, QByteArray rawOutput, + QString *errorMessage) const; + QString logName(const QString &logger) const; QList<LoggerSet> allLoggerSets() const; QTemporaryDir tempDir; QRegularExpression durationRegExp; + QRegularExpression teamcityLocRegExp; }; struct BenchmarkResult @@ -81,46 +92,50 @@ struct BenchmarkResult static BenchmarkResult parse(QString const&, QString*); }; -QT_BEGIN_NAMESPACE -namespace QTest +static QString msgMismatch(const QString &actual, const QString &expected) { -template <> -inline bool qCompare - (BenchmarkResult const &r1, BenchmarkResult const &r2, - const char* actual, const char* expected, const char* file, int line) + return QLatin1String("Mismatch:\n'") + actual + QLatin1String("'\n !=\n'") + + expected + QLatin1Char('\''); +} + +static bool compareBenchmarkResult(BenchmarkResult const &r1, BenchmarkResult const &r2, + QString *errorMessage) { // First make sure the iterations and unit match. if (r1.iterations != r2.iterations || r1.unit != r2.unit) { // Nope - compare whole string for best failure message - return qCompare(r1.toString(), r2.toString(), actual, expected, file, line); + *errorMessage = msgMismatch(r1.toString(), r2.toString()); + return false; } // Now check the value. Some variance is allowed, and how much depends on // the measured unit. qreal variance = 0.; - if (r1.unit == "msecs" || r1.unit == "WalltimeMilliseconds") { + if (r1.unit == QLatin1String("msecs") || r1.unit == QLatin1String("WalltimeMilliseconds")) variance = 0.1; - } - else if (r1.unit == "instruction reads") { + else if (r1.unit == QLatin1String("instruction reads")) variance = 0.001; - } - else if (r1.unit == "CPU ticks" || r1.unit == "CPUTicks") { + else if (r1.unit == QLatin1String("CPU ticks") || r1.unit == QLatin1String("CPUTicks")) variance = 0.001; - } + if (variance == 0.) { // No variance allowed - compare whole string - return qCompare(r1.toString(), r2.toString(), actual, expected, file, line); + const QString r1S = r1.toString(); + const QString r2S = r2.toString(); + if (r1S != r2S) { + *errorMessage = msgMismatch(r1S, r2S); + return false; + } + return true; } - if (qAbs(qreal(r1.total) - qreal(r2.total)) <= qreal(r1.total)*variance) { - return compare_helper(true, 0, 0, 0, actual, expected, file, line); + if (qAbs(qreal(r1.total) - qreal(r2.total)) > qreal(r1.total) * variance) { + // Whoops, didn't match. Compare the whole string for the most useful failure message. + *errorMessage = msgMismatch(r1.toString(), r2.toString()); + return false; } - - // Whoops, didn't match. Compare the whole string for the most useful failure message. - return qCompare(r1.toString(), r2.toString(), actual, expected, file, line); -} + return true; } -QT_END_NAMESPACE // Split the passed block of text into an array of lines, replacing any // filenames and line numbers with generic markers to avoid failing the test @@ -227,26 +242,6 @@ static QByteArray runDiff(const QByteArrayList &expected, const QByteArrayList & return result; } -// Print the difference preferably using 'diff', else just print lines -static void printDifference(const QByteArrayList &expected, const QByteArrayList &actual) -{ - QDebug info = qInfo(); - info.noquote(); - info.nospace(); - const QByteArray diff = runDiff(expected, actual); - if (diff.isEmpty()) { - info << "<<<<<<\n"; - for (const QByteArray &line : actual) - info << line << '\n'; - info << "======\n"; - for (const QByteArray &line : expected) - info << line << '\n'; - info << ">>>>>>\n"; - } else { - info << diff; - } -} - // Each test is run with a set of one or more test output loggers. // This struct holds information about one such test. struct LoggerSet @@ -315,6 +310,14 @@ QList<LoggerSet> tst_Selftests::allLoggerSets() const QStringList() << "teamcity", QStringList() << "-teamcity" << "-o" << logName("teamcity") ) + << LoggerSet("old stdout tap", + QStringList() << "stdout tap", + QStringList() << "-tap" + ) + << LoggerSet("old tap", + QStringList() << "tap", + QStringList() << "-tap" << "-o" << logName("tap") + ) // Test with new-style options for a single logger << LoggerSet("new stdout txt", QStringList() << "stdout txt", @@ -362,6 +365,14 @@ QList<LoggerSet> tst_Selftests::allLoggerSets() const QStringList() << "teamcity", QStringList() << "-o" << logName("teamcity")+",teamcity" ) + << LoggerSet("new stdout tap", + QStringList() << "stdout tap", + QStringList() << "-o" << "-,tap" + ) + << LoggerSet("new tap", + QStringList() << "tap", + QStringList() << "-o" << logName("tap")+",tap" + ) // Test with two loggers (don't test all 32 combinations, just a sample) << LoggerSet("stdout txt + txt", QStringList() << "stdout txt" << "txt", @@ -385,13 +396,14 @@ QList<LoggerSet> tst_Selftests::allLoggerSets() const ) // All loggers at the same time (except csv) << LoggerSet("all loggers", - QStringList() << "txt" << "xml" << "lightxml" << "stdout txt" << "xunitxml", + QStringList() << "txt" << "xml" << "lightxml" << "stdout txt" << "xunitxml" << "tap", QStringList() << "-o" << logName("txt")+",txt" << "-o" << logName("xml")+",xml" << "-o" << logName("lightxml")+",lightxml" << "-o" << "-,txt" << "-o" << logName("xunitxml")+",xunitxml" << "-o" << logName("teamcity")+",teamcity" + << "-o" << logName("tap")+",tap" ) ; } @@ -399,6 +411,7 @@ QList<LoggerSet> tst_Selftests::allLoggerSets() const tst_Selftests::tst_Selftests() : tempDir(QDir::tempPath() + "/tst_selftests.XXXXXX") , durationRegExp("<Duration msecs=\"[\\d\\.]+\"/>") + , teamcityLocRegExp("\\|\\[Loc: .*\\(\\d*\\)\\|\\]") {} void tst_Selftests::initTestCase() @@ -464,6 +477,7 @@ void tst_Selftests::runSubTest_data() << "findtestdata" << "float" << "globaldata" + << "keyboard" << "longstring" << "maxwarnings" << "multiexec" @@ -480,6 +494,7 @@ void tst_Selftests::runSubTest_data() << "sleep" << "strcmp" << "subtest" + << "tuplediagnostics" << "verbose1" << "verbose2" #ifndef QT_NO_EXCEPTIONS @@ -760,147 +775,203 @@ void tst_Selftests::doRunSubTest(QString const& subdir, QStringList const& logge } QList<QByteArray> res = splitLines(actualOutputs[n]); - const QString expectedFileName = expectedFileNameFromTest(subdir, logger); - QList<QByteArray> exp = expectedResult(expectedFileName); + QString errorMessage; + QString expectedFileName = expectedFileNameFromTest(subdir, logger); + QByteArrayList exp = expectedResult(expectedFileName); + if (!exp.isEmpty()) { #ifdef Q_CC_MINGW - // MinGW formats double numbers differently - if (n == 0 && subdir == QStringLiteral("float")) { - for (int i = 0; i < exp.size(); ++i) { - exp[i].replace("e-07", "e-007"); - exp[i].replace("e+07", "e+007"); - } - } -#endif - - // For the "crashes" test, there are multiple versions of the - // expected output. Load the one with the same line count as - // the actual output. - if (exp.count() == 0) { - QList<QList<QByteArray> > expArr; - QList<QByteArray> tmp; - int i = 1; - do { - tmp = expectedResult(expectedFileNameFromTest(subdir + QLatin1Char('_') + QString::number(i++), logger)); - if (tmp.count()) - expArr += tmp; - } while (tmp.count()); - - for (int j = 0; j < expArr.count(); ++j) { - if (res.count() == expArr.at(j).count()) { - exp = expArr.at(j); - break; + // MinGW formats double numbers differently (last verified with 7.1) + if (n == 0 && subdir == QStringLiteral("float")) { + for (int i = 0; i < exp.size(); ++i) { + exp[i].replace("e-07", "e-007"); + exp[i].replace("e+07", "e+007"); } } - - if (expArr.count()) { - QVERIFY2(exp.count(), - qPrintable(QString::fromLatin1("None of the expected output files for " - "%1 format has matching line count.") - .arg(loggers.at(n)))); +#endif + if (!compareOutput(logger, subdir, actualOutputs[n], res, exp, &errorMessage)) { + errorMessage.prepend(QLatin1Char('"') + logger + QLatin1String("\", ") + + expectedFileName + QLatin1Char(' ')); + errorMessage += QLatin1String("\nActual:\n") + QLatin1String(actualOutputs[n]); + const QByteArray diff = runDiff(exp, res); + if (!diff.isEmpty()) + errorMessage += QLatin1String("\nDiff:\n") + QLatin1String(diff); + QFAIL(qPrintable(errorMessage)); } } else { - if (res.count() != exp.count()) { - printDifference(exp, res); - QVERIFY2(res.count() == exp.count(), - qPrintable(QString::fromLatin1("Mismatch in line count: %1 != %2 (%3, %4).") - .arg(res.count()).arg(exp.count()).arg(loggers.at(n), expectedFileName))); + // For the "crashes" and other tests, there are multiple versions of the + // expected output. Loop until a matching one is found. + bool ok = false; + for (int i = 1; !ok; ++i) { + expectedFileName = expectedFileNameFromTest(subdir + QLatin1Char('_') + QString::number(i), logger); + const QByteArrayList exp = expectedResult(expectedFileName); + if (exp.isEmpty()) + break; + QString errorMessage2; + ok = compareOutput(logger, subdir, actualOutputs[n], res, exp, &errorMessage2); + if (!ok) + errorMessage += QLatin1Char('\n') + expectedFileName + QLatin1String(": ") + errorMessage2; + } + if (!ok) { // Use QDebug's quote mechanism to report potentially garbled output. + errorMessage.prepend(QLatin1String("Cannot find a matching file for ") + subdir); + errorMessage += QLatin1String("\nActual:\n"); + QDebug(&errorMessage) << actualOutputs[n]; + QFAIL(qPrintable(errorMessage)); } } + } +} - // By this point, we should have loaded a non-empty expected data file. - QVERIFY2(exp.count(), - qPrintable(QString::fromLatin1("Expected test data for %1 format is empty or not found.") - .arg(loggers.at(n)))); - - // For xml output formats, verify that the log is valid XML. - if (logFormat(logger) == "xunitxml" || logFormat(logger) == "xml" || logFormat(logger) == "lightxml") { - QByteArray xml(actualOutputs[n]); - // lightxml intentionally skips the root element, which technically makes it - // not valid XML. - // We'll add that ourselves for the purpose of validation. - if (logFormat(logger) == "lightxml") { - xml.prepend("<root>"); - xml.append("</root>"); - } +static QString teamCityLocation() { return QStringLiteral("|[Loc: _FILE_(_LINE_)|]"); } +static QString qtVersionPlaceHolder() { return QStringLiteral("@INSERT_QT_VERSION_HERE@"); } - QXmlStreamReader reader(xml); +bool tst_Selftests::compareOutput(const QString &logger, const QString &subdir, + const QByteArray &rawOutput, const QByteArrayList &actual, + const QByteArrayList &expected, + QString *errorMessage) const +{ - while (!reader.atEnd()) - reader.readNext(); + if (actual.size() != expected.size()) { + *errorMessage = QString::fromLatin1("Mismatch in line count: %1 != %2.") + .arg(actual.size()).arg(expected.size()); + return false; + } - QVERIFY2(!reader.error(), qPrintable(QString("line %1, col %2: %3") - .arg(reader.lineNumber()) - .arg(reader.columnNumber()) - .arg(reader.errorString()) - )); + // For xml output formats, verify that the log is valid XML. + if (logger.endsWith(QLatin1String("xml")) && !checkXml(logger, rawOutput, errorMessage)) + return false; + + // Verify that the actual output is an acceptable match for the + // expected output. + + const QString qtVersion = QLatin1String(QT_VERSION_STR); + bool benchmark = false; + for (int i = 0, size = actual.size(); i < size; ++i) { + const QByteArray &actualLineBA = actual.at(i); + // the __FILE__ __LINE__ output is compiler dependent, skip it + if (actualLineBA.startsWith(" Loc: [") && actualLineBA.endsWith(")]")) + continue; + if (actualLineBA.endsWith(" : failure location")) + continue; + + if (actualLineBA.startsWith("Config: Using QtTest library") // Text build string + || actualLineBA.startsWith(" <QtBuild") // XML, Light XML build string + || (actualLineBA.startsWith(" <property value=") && actualLineBA.endsWith("name=\"QtBuild\"/>"))) { // XUNIT-XML build string + continue; } - // Verify that the actual output is an acceptable match for the - // expected output. - bool benchmark = false; - for (int i = 0; i < res.count(); ++i) { - QByteArray line = res.at(i); - // the __FILE__ __LINE__ output is compiler dependent, skip it - if (line.startsWith(" Loc: [") && line.endsWith(")]")) - continue; - if (line.endsWith(" : failure location")) - continue; + QString actualLine = QString::fromLatin1(actualLineBA); + QString expectedLine = QString::fromLatin1(expected.at(i)); + expectedLine.replace(qtVersionPlaceHolder(), qtVersion); - if (line.startsWith("Config: Using QtTest library") // Text build string - || line.startsWith(" <QtBuild") // XML, Light XML build string - || (line.startsWith(" <property value=") && line.endsWith("name=\"QtBuild\"/>"))) { // XUNIT-XML build string - continue; - } + // Special handling for ignoring _FILE_ and _LINE_ if logger is teamcity + if (logger.endsWith(QLatin1String("teamcity"))) { + actualLine.replace(teamcityLocRegExp, teamCityLocation()); + expectedLine.replace(teamcityLocRegExp, teamCityLocation()); + } - QByteArray expLine = exp.at(i); + if (logger.endsWith(QLatin1String("tap"))) { + if (expectedLine.contains(QLatin1String("at:")) + || expectedLine.contains(QLatin1String("file:")) + || expectedLine.contains(QLatin1String("line:"))) + actualLine = expectedLine; + } - // Special handling for ignoring _FILE_ and _LINE_ if logger is teamcity - if (logFormat(logger) == "teamcity") { - QRegularExpression teamcityLocRegExp("\\|\\[Loc: .*\\(\\d*\\)\\|\\]"); - line = QString(line).replace(teamcityLocRegExp, "|[Loc: _FILE_(_LINE_)|]").toLatin1(); - expLine = QString(expLine).replace(teamcityLocRegExp, "|[Loc: _FILE_(_LINE_)|]").toLatin1(); - } + if (!compareLine(logger, subdir, benchmark, actualLine, + expectedLine, errorMessage)) { + errorMessage->prepend(QLatin1String("Line ") + QString::number(i + 1) + + QLatin1String(": ")); + return false; + } - const QString output(QString::fromLatin1(line)); - const QString expected(QString::fromLatin1(expLine).replace("@INSERT_QT_VERSION_HERE@", QT_VERSION_STR)); + benchmark = actualLineBA.startsWith("RESULT : "); + } + return true; +} - if (subdir == "assert" && output.contains("ASSERT: ") && expected.contains("ASSERT: ") && output != expected) - // Q_ASSERT uses __FILE__, the exact contents of which are - // undefined. If we something that looks like a Q_ASSERT and we - // were expecting to see a Q_ASSERT, we'll skip the line. - continue; - else if (expected.startsWith(QLatin1String("FAIL! : tst_Exception::throwException() Caught unhandled exce")) && expected != output) - // On some platforms we compile without RTTI, and as a result we never throw an exception. - QCOMPARE(output.simplified(), QString::fromLatin1("tst_Exception::throwException()").simplified()); - else if (benchmark || line.startsWith("<BenchmarkResult") || (logFormat(logger) == "csv" && line.startsWith('"'))) { - // Don't do a literal comparison for benchmark results, since - // results have some natural variance. - QString error; - - BenchmarkResult actualResult = BenchmarkResult::parse(output, &error); - QVERIFY2(error.isEmpty(), qPrintable(QString("Actual line didn't parse as benchmark result: %1\nLine: %2").arg(error).arg(output))); - - BenchmarkResult expectedResult = BenchmarkResult::parse(expected, &error); - QVERIFY2(error.isEmpty(), qPrintable(QString("Expected line didn't parse as benchmark result: %1\nLine: %2").arg(error).arg(expected))); - - QCOMPARE(actualResult, expectedResult); - } else if (line.startsWith(" <Duration msecs=") || line.startsWith("<Duration msecs=")) { - QRegularExpressionMatch match = durationRegExp.match(line); - QVERIFY2(match.hasMatch(), qPrintable(QString::fromLatin1("Invalid Duration tag at line %1 (%2): '%3'") - .arg(i).arg(loggers.at(n), output))); - } else if (line.startsWith("Totals:")) { - const int lastCommaPos = line.lastIndexOf(','); - if (lastCommaPos > 0) - line.truncate(lastCommaPos); // Plain text logger: strip time (", 2323dms"). - } else { - QVERIFY2(output == expected, - qPrintable(QString::fromLatin1("Mismatch at line %1 (%2, %3):\n'%4'\n !=\n'%5'") - .arg(i + 1).arg(loggers.at(n), expectedFileName, output, expected))); - } +bool tst_Selftests::compareLine(const QString &logger, const QString &subdir, + bool benchmark, + const QString &actualLine, const QString &expectedLine, + QString *errorMessage) const +{ + if (subdir == QLatin1String("assert") && actualLine.contains(QLatin1String("ASSERT: ")) + && expectedLine.contains(QLatin1String("ASSERT: ")) && actualLine != expectedLine) { + // Q_ASSERT uses __FILE__, the exact contents of which are + // undefined. If have we something that looks like a Q_ASSERT and we + // were expecting to see a Q_ASSERT, we'll skip the line. + return true; + } - benchmark = line.startsWith("RESULT : "); + if (expectedLine.startsWith(QLatin1String("FAIL! : tst_Exception::throwException() Caught unhandled exce")) + && actualLine != expectedLine) { + // On some platforms we compile without RTTI, and as a result we never throw an exception + if (actualLine.simplified() != QLatin1String("tst_Exception::throwException()")) { + *errorMessage = QString::fromLatin1("'%1' != 'tst_Exception::throwException()'").arg(actualLine); + return false; } + return true; + } + + if (benchmark || actualLine.startsWith(QLatin1String("<BenchmarkResult")) + || (logger == QLatin1String("csv") && actualLine.startsWith(QLatin1Char('"')))) { + // Don't do a literal comparison for benchmark results, since + // results have some natural variance. + QString error; + BenchmarkResult actualResult = BenchmarkResult::parse(actualLine, &error); + if (!error.isEmpty()) { + *errorMessage = QString::fromLatin1("Actual line didn't parse as benchmark result: %1\nLine: %2").arg(error, actualLine); + return false; + } + BenchmarkResult expectedResult = BenchmarkResult::parse(expectedLine, &error); + if (!error.isEmpty()) { + *errorMessage = QString::fromLatin1("Expected line didn't parse as benchmark result: %1\nLine: %2").arg(error, expectedLine); + return false; + } + return compareBenchmarkResult(actualResult, expectedResult, errorMessage); + } + + if (actualLine.startsWith(QLatin1String(" <Duration msecs=")) + || actualLine.startsWith(QLatin1String("<Duration msecs="))) { + QRegularExpressionMatch match = durationRegExp.match(actualLine); + if (match.hasMatch()) + return true; + *errorMessage = QString::fromLatin1("Invalid Duration tag: '%1'").arg(actualLine); + return false; + } + + if (actualLine.startsWith(QLatin1String("Totals:")) && expectedLine.startsWith(QLatin1String("Totals:"))) + return true; + + if (actualLine == expectedLine) + return true; + + *errorMessage = msgMismatch(actualLine, expectedLine); + return false; +} + +bool tst_Selftests::checkXml(const QString &logger, QByteArray xml, + QString *errorMessage) const +{ + // lightxml intentionally skips the root element, which technically makes it + // not valid XML. + // We'll add that ourselves for the purpose of validation. + if (logger.endsWith(QLatin1String("lightxml"))) { + xml.prepend("<root>"); + xml.append("</root>"); + } + + QXmlStreamReader reader(xml); + while (!reader.atEnd()) + reader.readNext(); + + if (reader.hasError()) { + const int lineNumber = int(reader.lineNumber()); + const QByteArray line = xml.split('\n').value(lineNumber - 1); + *errorMessage = QString::fromLatin1("line %1, col %2 '%3': %4") + .arg(lineNumber).arg(reader.columnNumber()) + .arg(QString::fromLatin1(line), reader.errorString()); + return false; } + return true; } #endif // QT_CONFIG(process) |