From 157fc88f90c42c6ff1b47b1bad018565fe63f626 Mon Sep 17 00:00:00 2001 From: Jason McDonald Date: Wed, 21 Sep 2011 11:39:07 +1000 Subject: Enable testlib self-testing with multiple loggers Modify the selftest to be able to run each subtest with a list of one or more test loggers. The addition of tests that use this capability will be part of a subsequent commit. Task-number: QTBUG-20615 Change-Id: Iac3efe8220e8245aa7e5589348d2c86b8034dd28 Reviewed-on: http://codereview.qt-project.org/5292 Reviewed-by: Qt Sanity Bot Reviewed-by: Rohan McGovern --- tests/auto/testlib/selftests/tst_selftests.cpp | 293 ++++++++++++++----------- 1 file changed, 164 insertions(+), 129 deletions(-) diff --git a/tests/auto/testlib/selftests/tst_selftests.cpp b/tests/auto/testlib/selftests/tst_selftests.cpp index 5efaedbd86..17727435ea 100644 --- a/tests/auto/testlib/selftests/tst_selftests.cpp +++ b/tests/auto/testlib/selftests/tst_selftests.cpp @@ -53,7 +53,7 @@ private slots: void cleanup(); private: - void doRunSubTest(QString const& subdir, QString const& logger, QStringList const& arguments ); + void doRunSubTest(QString const& subdir, QStringList const& loggers, QStringList const& arguments); }; struct BenchmarkResult @@ -142,45 +142,73 @@ static QList splitLines(QByteArray ba) return out; } +// Return the log format, e.g. for both "stdout txt" and "txt", return "txt'. +static inline QString logFormat(const QString &logger) +{ + return (logger.startsWith("stdout") ? logger.mid(7) : logger); +} + +// Return the log file name, or an empty string if the log goes to stdout. +static inline QString logName(const QString &logger) +{ + return (logger.startsWith("stdout") ? "" : "test_output." + logger); +} + // Load the expected test output for the nominated test (subdir) and logger // as an array of lines. If there is no expected output file, return an // empty array. static QList expectedResult(const QString &subdir, const QString &logger) { - QFile file(":/expected_" + subdir + "." + logger); + QFile file(":/expected_" + subdir + "." + logFormat(logger)); if (!file.open(QIODevice::ReadOnly)) return QList(); return splitLines(file.readAll()); } -struct Logger +// Each test is run with a set of one or more test output loggers. +// This struct holds information about one such test. +struct LoggerSet { - Logger(QString const&, QStringList const&); + LoggerSet(QString const& _name, QStringList const& _loggers, QStringList const& _arguments) + : name(_name), loggers(_loggers), arguments(_arguments) + { } QString name; + QStringList loggers; QStringList arguments; }; -Logger::Logger(QString const& _name, QStringList const& _arguments) - : name(_name) - , arguments(_arguments) -{ -} - -static QList allLoggers() +// This function returns a list of all sets of loggers to be used for +// running each subtest. +static QList allLoggerSets() { - return QList() - << Logger("txt", QStringList()) - << Logger("xml", QStringList() << "-xml") - << Logger("xunitxml", QStringList() << "-xunitxml") - << Logger("lightxml", QStringList() << "-lightxml") + // For the plain text logger, we'll test logging to file and to standard + // output. For all other loggers (XML), we'll tell testlib to redirect to + // file. The reason is that tests are allowed to print to standard output, + // and that means the test log is no longer guaranteed to be valid XML. + return QList() + << LoggerSet("stdout txt", + QStringList() << "stdout txt", + QStringList()) + << LoggerSet("txt", + QStringList() << "txt", + QStringList() << "-o" << logName("txt")) + << LoggerSet("xml", + QStringList() << "xml", + QStringList() << "-xml" << "-o" << logName("xml")) + << LoggerSet("xunitxml", + QStringList() << "xunitxml", + QStringList() << "-xunitxml" << "-o" << logName("xunitxml")) + << LoggerSet("lightxml", + QStringList() << "lightxml", + QStringList() << "-lightxml" << "-o" << logName("lightxml")) ; } void tst_Selftests::runSubTest_data() { QTest::addColumn("subdir"); - QTest::addColumn("logger"); + QTest::addColumn("loggers"); QTest::addColumn("arguments"); QStringList tests = QStringList() @@ -243,14 +271,11 @@ void tst_Selftests::runSubTest_data() << "badxml" ; - foreach (Logger const& logger, allLoggers()) { - QString rowSuffix; - if (logger.name != "txt") { - rowSuffix = QString(" %1").arg(logger.name); - } + foreach (LoggerSet const& loggerSet, allLoggerSets()) { + QStringList loggers = loggerSet.loggers; foreach (QString const& subtest, tests) { - QStringList arguments = logger.arguments; + QStringList arguments = loggerSet.arguments; if (subtest == "commandlinedata") { arguments << QString("fiveTablePasses fiveTablePasses:fiveTablePasses_data1 -v2").split(' '); } @@ -274,7 +299,7 @@ void tst_Selftests::runSubTest_data() // standard output, either because they execute multiple test // objects or because they internally supply arguments to // themselves. - if (logger.name != "txt") { + if (loggerSet.name != "stdout txt") { if (subtest == "differentexec") { continue; } @@ -298,40 +323,34 @@ void tst_Selftests::runSubTest_data() } } - QTest::newRow(qPrintable(QString("%1%2").arg(subtest).arg(rowSuffix))) + QTest::newRow(qPrintable(QString("%1 %2").arg(subtest).arg(loggerSet.name))) << subtest - << logger.name + << loggers << arguments ; } } } -void tst_Selftests::doRunSubTest(QString const& subdir, QString const& logger, QStringList const& arguments ) +void tst_Selftests::doRunSubTest(QString const& subdir, QStringList const& loggers, QStringList const& arguments) { - // For the plain text logger, we'll read straight from standard output. - // For all other loggers (XML), we'll tell testlib to redirect to a file. - // The reason is that tests are allowed to print to standard output, and - // that means the test log is no longer guaranteed to be valid XML. - QStringList extraArguments; - QString logfile; - if (logger != "txt") { - logfile = "test_output"; - extraArguments << "-o" << logfile; - } - QProcess proc; proc.setEnvironment(QStringList("")); - proc.start(subdir + "/" + subdir, QStringList() << arguments << extraArguments); + proc.start(subdir + "/" + subdir, arguments); QVERIFY2(proc.waitForFinished(), qPrintable(proc.errorString())); - QByteArray out; - if (logfile.isEmpty()) { - out = proc.readAllStandardOutput(); - } else { - QFile file(logfile); - if (file.open(QIODevice::ReadOnly)) - out = file.readAll(); + QList actualOutputs; + for (int i = 0; i < loggers.count(); ++i) { + QString logFile = logName(loggers[i]); + QByteArray out; + if (logFile.isEmpty()) { + out = proc.readAllStandardOutput(); + } else { + QFile file(logFile); + if (file.open(QIODevice::ReadOnly)) + out = file.readAll(); + } + actualOutputs << out; } const QByteArray err(proc.readAllStandardError()); @@ -352,107 +371,117 @@ void tst_Selftests::doRunSubTest(QString const& subdir, QString const& logger, Q && subdir != QLatin1String("benchlibcallgrind")) QVERIFY2(err.isEmpty(), err.constData()); - QList res = splitLines(out); - QList exp = expectedResult(subdir, logger); - - if (exp.count() == 0) { - QList > expArr; - int i = 1; - do { - exp = expectedResult(subdir + QString("_%1").arg(i++), logger); - if (exp.count()) - expArr += exp; - } while (exp.count()); - - for (int j = 0; j < expArr.count(); ++j) { - if (res.count() == expArr.at(j).count()) { - exp = expArr.at(j); - break; + for (int n = 0; n < loggers.count(); ++n) { + QString logger = loggers[n]; + QList res = splitLines(actualOutputs[n]); + QList exp = expectedResult(subdir, logger); + + // For the "crashes" test, there are multiple versions of the + // expected output. Load the one with the same line count as + // the actual output. + if (exp.count() == 0) { + QList > expArr; + int i = 1; + do { + exp = expectedResult(subdir + QString("_%1").arg(i++), logger); + if (exp.count()) + expArr += exp; + } while (exp.count()); + + for (int j = 0; j < expArr.count(); ++j) { + if (res.count() == expArr.at(j).count()) { + exp = expArr.at(j); + break; + } } - } - } else { - QCOMPARE(res.count(), exp.count()); - } - - if (logger == "xunitxml" || logger == "xml" || logger == "lightxml") { - QByteArray xml(out); - // lightxml intentionally skips the root element, which technically makes it - // not valid XML. - // We'll add that ourselves for the purpose of validation. - if (logger == "lightxml") { - xml.prepend(""); - xml.append(""); + } else { + QCOMPARE(res.count(), exp.count()); } - QXmlStreamReader reader(xml); + // For xml output formats, verify that the log is valid XML. + if (logFormat(logger) == "xunitxml" || logFormat(logger) == "xml" || logFormat(logger) == "lightxml") { + QByteArray xml(actualOutputs[n]); + // lightxml intentionally skips the root element, which technically makes it + // not valid XML. + // We'll add that ourselves for the purpose of validation. + if (logFormat(logger) == "lightxml") { + xml.prepend(""); + xml.append(""); + } - while(!reader.atEnd()) - reader.readNext(); + QXmlStreamReader reader(xml); - QVERIFY2(!reader.error(), qPrintable(QString("line %1, col %2: %3") - .arg(reader.lineNumber()) - .arg(reader.columnNumber()) - .arg(reader.errorString()) - )); - } + while (!reader.atEnd()) + reader.readNext(); - bool benchmark = false; - for (int i = 0; i < res.count(); ++i) { - QByteArray line = res.at(i); - if (line.startsWith("Config: Using QTest")) - continue; - // the __FILE__ __LINE__ output is compiler dependent, skip it - if (line.startsWith(" Loc: [") && line.endsWith(")]")) - continue; - if (line.endsWith(" : failure location")) - continue; - - const QString output(QString::fromLatin1(line)); - const QString expected(QString::fromLatin1(exp.at(i)).replace("@INSERT_QT_VERSION_HERE@", QT_VERSION_STR)); - - // Q_ASSERT uses __FILE__. Some compilers include the absolute path in - // __FILE__, while others do not. - if (line.contains("ASSERT") && output != expected) { - const char msg[] = "Q_ASSERT prints out the absolute path on this platform."; - QEXPECT_FAIL("assert", msg, Continue); - QEXPECT_FAIL("assert xml", msg, Continue); - QEXPECT_FAIL("assert lightxml", msg, Continue); - QEXPECT_FAIL("assert xunitxml", msg, Continue); + QVERIFY2(!reader.error(), qPrintable(QString("line %1, col %2: %3") + .arg(reader.lineNumber()) + .arg(reader.columnNumber()) + .arg(reader.errorString()) + )); } - if (expected.startsWith(QLatin1String("FAIL! : tst_Exception::throwException() Caught unhandled exce")) && expected != output) - // On some platforms we compile without RTTI, and as a result we never throw an exception. - QCOMPARE(output.simplified(), QString::fromLatin1("tst_Exception::throwException()").simplified()); - else if (output != expected && qstrcmp(QTest::currentDataTag(), "float") == 0) - // The floating point formatting differs between platforms, so let's just skip it. - continue; - else if (benchmark || line.startsWith("