diff options
author | Jason McDonald <jason.mcdonald@nokia.com> | 2011-09-02 19:08:21 +1000 |
---|---|---|
committer | Qt by Nokia <qt-info@nokia.com> | 2011-09-23 00:34:23 +0200 |
commit | 51589e834e086de93a121a1c628c3492d88a15a7 (patch) | |
tree | 6d1506f12c3dc9eb37d8471e8e0b1d1a821c2cac | |
parent | 157fc88f90c42c6ff1b47b1bad018565fe63f626 (diff) |
Allow tests to log to multiple destinations
Each destination and the format of output to write there is specified by
adding "-o filename,format" to the command-line. The special filename
"-" indicates that the log output is written to the standard output
stream, though standard output can be used as a destination at most
once.
The old-style testlib output options are still supported, but can only
be used to specify one logging destination, as before.
If no logging options are given on the command-line, a plain text log
will go to the console, as before.
To log to the console in plain text and to the file "test_output" in
xunit format, one would invoke a test in the following way:
tst_foo -o test_output,xunitxml -o -,txt
This commit also enhances the selftests to test with multiple loggers,
but negative tests (e.g. bad combinations of command-line options) are
left for future task QTBUG-21567.
Task-number: QTBUG-20615
Change-Id: If91e752bc7001657e15e427aba9d25ab0a29a0b0
Reviewed-on: http://codereview.qt-project.org/4125
Reviewed-by: Qt Sanity Bot <qt_sanity_bot@ovi.com>
Reviewed-by: Rohan McGovern <rohan.mcgovern@nokia.com>
-rw-r--r-- | src/testlib/qtestcase.cpp | 67 | ||||
-rw-r--r-- | src/testlib/qtestlog.cpp | 192 | ||||
-rw-r--r-- | src/testlib/qtestlog_p.h | 5 | ||||
-rw-r--r-- | tests/auto/testlib/selftests/tst_selftests.cpp | 136 |
4 files changed, 316 insertions, 84 deletions
diff --git a/src/testlib/qtestcase.cpp b/src/testlib/qtestcase.cpp index 9bb591989d..b49d5c2aa5 100644 --- a/src/testlib/qtestcase.cpp +++ b/src/testlib/qtestcase.cpp @@ -1004,11 +1004,28 @@ Q_TESTLIB_EXPORT void qtest_qParseArgs(int argc, char *argv[], bool qml) const char *logFilename = 0; const char *testOptions = - " Output options:\n" - " -xunitxml : Outputs results as XML XUnit document\n" - " -xml : Outputs results as XML document\n" - " -lightxml : Outputs results as stream of XML tags\n" - " -o filename : Writes all output into a file\n" + " New-style logging options:\n" + " -o filename,format : Output results to file in the specified format\n" + " Use - to output to stdout\n" + " Valid formats are:\n" + " txt : Plain text\n" + " xunitxml : XML XUnit document\n" + " xml : XML document\n" + " lightxml : A stream of XML tags\n" + "\n" + " *** Multiple loggers can be specified, but at most one can log to stdout.\n" + "\n" + " Old-style logging options:\n" + " -o filename : Write the output into file\n" + " -txt : Output results in Plain Text\n" + " -xunitxml : Output results as XML XUnit document\n" + " -xml : Output results as XML document\n" + " -lightxml : Output results as stream of XML tags\n" + "\n" + " *** If no output file is specified, stdout is assumed.\n" + " *** If no output format is specified, -txt is assumed.\n" + "\n" + " Detail options:\n" " -silent : Only outputs warnings and failures\n" " -v1 : Print enter messages for each testfunction\n" " -v2 : Also print out each QVERIFY/QCOMPARE/QTEST\n" @@ -1065,6 +1082,8 @@ Q_TESTLIB_EXPORT void qtest_qParseArgs(int argc, char *argv[], bool qml) qPrintTestSlots(stdout); exit(0); } + } else if (strcmp(argv[i], "-txt") == 0) { + logFormat = QTestLog::Plain; } else if (strcmp(argv[i], "-xunitxml") == 0) { logFormat = QTestLog::XunitXML; } else if (strcmp(argv[i], "-xml") == 0) { @@ -1081,11 +1100,38 @@ Q_TESTLIB_EXPORT void qtest_qParseArgs(int argc, char *argv[], bool qml) QSignalDumper::startDump(); } else if (strcmp(argv[i], "-o") == 0) { if (i + 1 >= argc) { - fprintf(stderr, "-o needs an extra parameter specifying the filename\n"); + fprintf(stderr, "-o needs an extra parameter specifying the filename and optional format\n"); exit(1); + } + ++i; + // Do we have the old or new style -o option? + char *filename = new char[strlen(argv[i])+1]; + char *format = new char[strlen(argv[i])+1]; + if (sscanf(argv[i], "%[^,],%s", filename, format) == 1) { + // Old-style + logFilename = argv[i]; } else { - logFilename = argv[++i]; + // New-style + if (strcmp(format, "txt") == 0) + logFormat = QTestLog::Plain; + else if (strcmp(format, "lightxml") == 0) + logFormat = QTestLog::LightXML; + else if (strcmp(format, "xml") == 0) + logFormat = QTestLog::XML; + else if (strcmp(format, "xunitxml") == 0) + logFormat = QTestLog::XunitXML; + else { + fprintf(stderr, "output format must be one of txt, lightxml, xml or xunitxml\n"); + exit(1); + } + if (strcmp(filename, "-") == 0 && QTestLog::loggerUsingStdout()) { + fprintf(stderr, "only one logger can log to stdout\n"); + exit(1); + } + QTestLog::addLogger(logFormat, filename); } + delete [] filename; + delete [] format; } else if (strcmp(argv[i], "-eventdelay") == 0) { if (i + 1 >= argc) { fprintf(stderr, "-eventdelay needs an extra parameter to indicate the delay(ms)\n"); @@ -1248,8 +1294,11 @@ Q_TESTLIB_EXPORT void qtest_qParseArgs(int argc, char *argv[], bool qml) } } - // Create the logger - QTestLog::initLogger(logFormat, logFilename); + // If no loggers were created by the long version of the -o command-line + // option, create a logger using whatever filename and format were + // set using the old-style command-line options. + if (QTestLog::loggerCount() == 0) + QTestLog::addLogger(logFormat, logFilename); } QBenchmarkResult qMedian(const QList<QBenchmarkResult> &container) diff --git a/src/testlib/qtestlog.cpp b/src/testlib/qtestlog.cpp index d53b00ca7f..f3b416d3d1 100644 --- a/src/testlib/qtestlog.cpp +++ b/src/testlib/qtestlog.cpp @@ -82,12 +82,103 @@ namespace QTest { static IgnoreResultList *ignoreResultList = 0; - static QTestLog::LogMode logMode = QTestLog::Plain; + struct LoggerList + { + QAbstractTestLogger *logger; + LoggerList *next; + }; + + class TestLoggers + { + public: + static void addLogger(QAbstractTestLogger *logger) + { + LoggerList *l = new LoggerList; + l->logger = logger; + l->next = loggers; + loggers = l; + } + + static void destroyLoggers() + { + while (loggers) { + LoggerList *l = loggers; + loggers = loggers->next; + delete l->logger; + delete l; + } + } + +#define FOREACH_LOGGER(operation) \ + LoggerList *l = loggers; \ + while (l) { \ + QAbstractTestLogger *logger = l->logger; \ + Q_UNUSED(logger); \ + operation; \ + l = l->next; \ + } + + static void startLogging() + { + FOREACH_LOGGER(logger->startLogging()); + } + + static void stopLogging() + { + FOREACH_LOGGER(logger->stopLogging()); + } + + static void enterTestFunction(const char *function) + { + FOREACH_LOGGER(logger->enterTestFunction(function)); + } + + static void leaveTestFunction() + { + FOREACH_LOGGER(logger->leaveTestFunction()); + } + + static void addIncident(QAbstractTestLogger::IncidentTypes type, const char *description, + const char *file = 0, int line = 0) + { + FOREACH_LOGGER(logger->addIncident(type, description, file, line)); + } + + static void addBenchmarkResult(const QBenchmarkResult &result) + { + FOREACH_LOGGER(logger->addBenchmarkResult(result)); + } + + static void addMessage(QAbstractTestLogger::MessageTypes type, const char *message, + const char *file = 0, int line = 0) + { + FOREACH_LOGGER(logger->addMessage(type, message, file, line)); + } + + static void outputString(const char *msg) + { + FOREACH_LOGGER(logger->outputString(msg)); + } + + static int loggerCount() + { + int count = 0; + FOREACH_LOGGER(++count); + return count; + } + + private: + static LoggerList *loggers; + }; + +#undef FOREACH_LOGGER + + LoggerList *TestLoggers::loggers = 0; + static bool loggerUsingStdout = false; + static int verbosity = 0; static int maxWarnings = 2002; - static QAbstractTestLogger *testLogger = 0; - static QtMsgHandler oldMessageHandler; static bool handleIgnoredMessage(QtMsgType type, const char *msg) @@ -118,11 +209,11 @@ namespace QTest { { static QBasicAtomicInt counter = Q_BASIC_ATOMIC_INITIALIZER(QTest::maxWarnings); - if (!msg || !QTest::testLogger) { + if (!msg || QTest::TestLoggers::loggerCount() == 0) { // if this goes wrong, something is seriously broken. qInstallMsgHandler(oldMessageHandler); QTEST_ASSERT(msg); - QTEST_ASSERT(QTest::testLogger); + QTEST_ASSERT(QTest::TestLoggers::loggerCount() != 0); } if (handleIgnoredMessage(type, msg)) @@ -134,7 +225,7 @@ namespace QTest { return; if (!counter.deref()) { - QTest::testLogger->addMessage(QAbstractTestLogger::QSystem, + QTest::TestLoggers::addMessage(QAbstractTestLogger::QSystem, "Maximum amount of warnings exceeded. Use -maxwarnings to override."); return; } @@ -142,16 +233,16 @@ namespace QTest { switch (type) { case QtDebugMsg: - QTest::testLogger->addMessage(QAbstractTestLogger::QDebug, msg); + QTest::TestLoggers::addMessage(QAbstractTestLogger::QDebug, msg); break; case QtCriticalMsg: - QTest::testLogger->addMessage(QAbstractTestLogger::QSystem, msg); + QTest::TestLoggers::addMessage(QAbstractTestLogger::QSystem, msg); break; case QtWarningMsg: - QTest::testLogger->addMessage(QAbstractTestLogger::QWarning, msg); + QTest::TestLoggers::addMessage(QAbstractTestLogger::QWarning, msg); break; case QtFatalMsg: - QTest::testLogger->addMessage(QAbstractTestLogger::QFatal, msg); + QTest::TestLoggers::addMessage(QAbstractTestLogger::QFatal, msg); /* Right now, we're inside the custom message handler and we're * being qt_message_output in qglobal.cpp. After we return from * this function, it will proceed with calling exit() and abort() @@ -167,10 +258,9 @@ namespace QTest { void QTestLog::enterTestFunction(const char* function) { - QTEST_ASSERT(QTest::testLogger); QTEST_ASSERT(function); - QTest::testLogger->enterTestFunction(function); + QTest::TestLoggers::enterTestFunction(function); } int QTestLog::unhandledIgnoreMessages() @@ -186,21 +276,17 @@ int QTestLog::unhandledIgnoreMessages() void QTestLog::leaveTestFunction() { - QTEST_ASSERT(QTest::testLogger); - QTest::IgnoreResultList::clearList(QTest::ignoreResultList); - QTest::testLogger->leaveTestFunction(); + QTest::TestLoggers::leaveTestFunction(); } void QTestLog::printUnhandledIgnoreMessages() { - QTEST_ASSERT(QTest::testLogger); - char msg[1024]; QTest::IgnoreResultList *list = QTest::ignoreResultList; while (list) { QTest::qt_snprintf(msg, 1024, "Did not receive message: \"%s\"", list->msg); - QTest::testLogger->addMessage(QAbstractTestLogger::Info, msg); + QTest::TestLoggers::addMessage(QAbstractTestLogger::Info, msg); list = list->next; } @@ -208,110 +294,110 @@ void QTestLog::printUnhandledIgnoreMessages() void QTestLog::addPass(const char *msg) { - QTEST_ASSERT(QTest::testLogger); QTEST_ASSERT(msg); - QTest::testLogger->addIncident(QAbstractTestLogger::Pass, msg); + QTest::TestLoggers::addIncident(QAbstractTestLogger::Pass, msg); } void QTestLog::addFail(const char *msg, const char *file, int line) { - QTEST_ASSERT(QTest::testLogger); QTEST_ASSERT(msg); - QTest::testLogger->addIncident(QAbstractTestLogger::Fail, msg, file, line); + QTest::TestLoggers::addIncident(QAbstractTestLogger::Fail, msg, file, line); } void QTestLog::addXFail(const char *msg, const char *file, int line) { - QTEST_ASSERT(QTest::testLogger); QTEST_ASSERT(msg); QTEST_ASSERT(file); - QTest::testLogger->addIncident(QAbstractTestLogger::XFail, msg, file, line); + QTest::TestLoggers::addIncident(QAbstractTestLogger::XFail, msg, file, line); } void QTestLog::addXPass(const char *msg, const char *file, int line) { - QTEST_ASSERT(QTest::testLogger); QTEST_ASSERT(msg); QTEST_ASSERT(file); - QTest::testLogger->addIncident(QAbstractTestLogger::XPass, msg, file, line); + QTest::TestLoggers::addIncident(QAbstractTestLogger::XPass, msg, file, line); } void QTestLog::addSkip(const char *msg, const char *file, int line) { - QTEST_ASSERT(QTest::testLogger); QTEST_ASSERT(msg); QTEST_ASSERT(file); - QTest::testLogger->addMessage(QAbstractTestLogger::Skip, msg, file, line); + QTest::TestLoggers::addMessage(QAbstractTestLogger::Skip, msg, file, line); } void QTestLog::addBenchmarkResult(const QBenchmarkResult &result) { - QTEST_ASSERT(QTest::testLogger); - QTest::testLogger->addBenchmarkResult(result); + QTest::TestLoggers::addBenchmarkResult(result); } void QTestLog::startLogging() { - QTEST_ASSERT(QTest::testLogger); - QTest::testLogger->startLogging(); + QTest::TestLoggers::startLogging(); QTest::oldMessageHandler = qInstallMsgHandler(QTest::messageHandler); } void QTestLog::stopLogging() { qInstallMsgHandler(QTest::oldMessageHandler); - - QTEST_ASSERT(QTest::testLogger); - QTest::testLogger->stopLogging(); - delete QTest::testLogger; - QTest::testLogger = 0; + QTest::TestLoggers::stopLogging(); + QTest::TestLoggers::destroyLoggers(); + QTest::loggerUsingStdout = false; } -void QTestLog::initLogger(LogMode mode, const char *filename) +void QTestLog::addLogger(LogMode mode, const char *filename) { - QTest::logMode = mode; + if (filename && strcmp(filename, "-") == 0) + filename = 0; + if (!filename) + QTest::loggerUsingStdout = true; + QAbstractTestLogger *logger = 0; switch (mode) { case QTestLog::Plain: - QTest::testLogger = new QPlainTestLogger(filename); + logger = new QPlainTestLogger(filename); break; case QTestLog::XML: - QTest::testLogger = new QXmlTestLogger(QXmlTestLogger::Complete, filename); + logger = new QXmlTestLogger(QXmlTestLogger::Complete, filename); break; case QTestLog::LightXML: - QTest::testLogger = new QXmlTestLogger(QXmlTestLogger::Light, filename); + logger = new QXmlTestLogger(QXmlTestLogger::Light, filename); break; case QTestLog::XunitXML: - QTest::testLogger = new QXunitTestLogger(filename); + logger = new QXunitTestLogger(filename); break; } - QTEST_ASSERT(QTest::testLogger); + QTEST_ASSERT(logger); + QTest::TestLoggers::addLogger(logger); } -void QTestLog::warn(const char *msg) +int QTestLog::loggerCount() { - QTEST_ASSERT(QTest::testLogger); - QTEST_ASSERT(msg); + return QTest::TestLoggers::loggerCount(); +} - QTest::testLogger->addMessage(QAbstractTestLogger::Warn, msg); +bool QTestLog::loggerUsingStdout() +{ + return QTest::loggerUsingStdout; } -void QTestLog::info(const char *msg, const char *file, int line) +void QTestLog::warn(const char *msg) { QTEST_ASSERT(msg); - if (QTest::testLogger) - QTest::testLogger->addMessage(QAbstractTestLogger::Info, msg, file, line); + if (QTest::TestLoggers::loggerCount() > 0) + QTest::TestLoggers::addMessage(QAbstractTestLogger::Warn, msg); } -QTestLog::LogMode QTestLog::logMode() +void QTestLog::info(const char *msg, const char *file, int line) { - return QTest::logMode; + QTEST_ASSERT(msg); + + QTest::TestLoggers::addMessage(QAbstractTestLogger::Info, msg, file, line); } void QTestLog::setVerboseLevel(int level) diff --git a/src/testlib/qtestlog_p.h b/src/testlib/qtestlog_p.h index f06382978f..fe37469dc6 100644 --- a/src/testlib/qtestlog_p.h +++ b/src/testlib/qtestlog_p.h @@ -83,9 +83,10 @@ public: static void startLogging(); static void stopLogging(); - static void initLogger(LogMode mode, const char *filename); + static void addLogger(LogMode mode, const char *filename); - static LogMode logMode(); + static int loggerCount(); + static bool loggerUsingStdout(); static void setVerboseLevel(int level); static int verboseLevel(); diff --git a/tests/auto/testlib/selftests/tst_selftests.cpp b/tests/auto/testlib/selftests/tst_selftests.cpp index 17727435ea..7c71a292c6 100644 --- a/tests/auto/testlib/selftests/tst_selftests.cpp +++ b/tests/auto/testlib/selftests/tst_selftests.cpp @@ -182,26 +182,106 @@ struct LoggerSet // running each subtest. static QList<LoggerSet> allLoggerSets() { - // For the plain text logger, we'll test logging to file and to standard - // output. For all other loggers (XML), we'll tell testlib to redirect to - // file. The reason is that tests are allowed to print to standard output, - // and that means the test log is no longer guaranteed to be valid XML. + // Note that in order to test XML output to standard output, the subtests + // must not send output directly to stdout, bypassing Qt's output mechanisms + // (e.g. via printf), otherwise the output may not be well-formed XML. return QList<LoggerSet>() - << LoggerSet("stdout txt", + // Test with old-style options for a single logger + << LoggerSet("old stdout txt", QStringList() << "stdout txt", - QStringList()) - << LoggerSet("txt", + QStringList() + ) + << LoggerSet("old txt", QStringList() << "txt", - QStringList() << "-o" << logName("txt")) - << LoggerSet("xml", + QStringList() << "-o" << logName("txt") + ) + << LoggerSet("old stdout xml", + QStringList() << "stdout xml", + QStringList() << "-xml" + ) + << LoggerSet("old xml", QStringList() << "xml", - QStringList() << "-xml" << "-o" << logName("xml")) - << LoggerSet("xunitxml", + QStringList() << "-xml" << "-o" << logName("xml") + ) + << LoggerSet("old stdout xunitxml", + QStringList() << "stdout xunitxml", + QStringList() << "-xunitxml" + ) + << LoggerSet("old xunitxml", QStringList() << "xunitxml", - QStringList() << "-xunitxml" << "-o" << logName("xunitxml")) - << LoggerSet("lightxml", + QStringList() << "-xunitxml" << "-o" << logName("xunitxml") + ) + << LoggerSet("old stdout lightxml", + QStringList() << "stdout lightxml", + QStringList() << "-lightxml" + ) + << LoggerSet("old lightxml", QStringList() << "lightxml", - QStringList() << "-lightxml" << "-o" << logName("lightxml")) + QStringList() << "-lightxml" << "-o" << logName("lightxml") + ) + // Test with new-style options for a single logger + << LoggerSet("new stdout txt", + QStringList() << "stdout txt", + QStringList() << "-o" << "-,txt" + ) + << LoggerSet("new txt", + QStringList() << "txt", + QStringList() << "-o" << logName("txt")+",txt" + ) + << LoggerSet("new stdout xml", + QStringList() << "stdout xml", + QStringList() << "-o" << "-,xml" + ) + << LoggerSet("new xml", + QStringList() << "xml", + QStringList() << "-o" << logName("xml")+",xml" + ) + << LoggerSet("new stdout xunitxml", + QStringList() << "stdout xunitxml", + QStringList() << "-o" << "-,xunitxml" + ) + << LoggerSet("new xunitxml", + QStringList() << "xunitxml", + QStringList() << "-o" << logName("xunitxml")+",xunitxml" + ) + << LoggerSet("new stdout lightxml", + QStringList() << "stdout lightxml", + QStringList() << "-o" << "-,lightxml" + ) + << LoggerSet("new lightxml", + QStringList() << "lightxml", + QStringList() << "-o" << logName("lightxml")+",lightxml" + ) + // Test with two loggers (don't test all 32 combinations, just a sample) + << LoggerSet("stdout txt + txt", + QStringList() << "stdout txt" << "txt", + QStringList() << "-o" << "-,txt" + << "-o" << logName("txt")+",txt" + ) + << LoggerSet("xml + stdout txt", + QStringList() << "xml" << "stdout txt", + QStringList() << "-o" << logName("xml")+",xml" + << "-o" << "-,txt" + ) + << LoggerSet("txt + xunitxml", + QStringList() << "txt" << "xunitxml", + QStringList() << "-o" << logName("txt")+",txt" + << "-o" << logName("xunitxml")+",xunitxml" + ) + << LoggerSet("lightxml + stdout xunitxml", + QStringList() << "lightxml" << "stdout xunitxml", + QStringList() << "-o" << logName("lightxml")+",lightxml" + << "-o" << "-,xunitxml" + ) + // All loggers at the same time + << LoggerSet("all loggers", + QStringList() << "txt" << "xml" << "lightxml" << "stdout txt" << "xunitxml", + QStringList() << "-o" << logName("txt")+",txt" + << "-o" << logName("xml")+",xml" + << "-o" << logName("lightxml")+",lightxml" + << "-o" << "-,txt" + << "-o" << logName("xunitxml")+",xunitxml" + ) ; } @@ -299,7 +379,7 @@ void tst_Selftests::runSubTest_data() // standard output, either because they execute multiple test // objects or because they internally supply arguments to // themselves. - if (loggerSet.name != "stdout txt") { + if (loggerSet.name != "old stdout txt" && loggerSet.name != "new stdout txt") { if (subtest == "differentexec") { continue; } @@ -441,11 +521,27 @@ void tst_Selftests::doRunSubTest(QString const& subdir, QStringList const& logge // __FILE__, while others do not. if (line.contains("ASSERT") && output != expected) { const char msg[] = "Q_ASSERT prints out the absolute path on this platform."; - QEXPECT_FAIL("assert stdout txt", msg, Continue); - QEXPECT_FAIL("assert txt", msg, Continue); - QEXPECT_FAIL("assert xml", msg, Continue); - QEXPECT_FAIL("assert lightxml", msg, Continue); - QEXPECT_FAIL("assert xunitxml", msg, Continue); + QEXPECT_FAIL("assert old stdout txt", msg, Continue); + QEXPECT_FAIL("assert old txt", msg, Continue); + QEXPECT_FAIL("assert old stdout xml", msg, Continue); + QEXPECT_FAIL("assert old xml", msg, Continue); + QEXPECT_FAIL("assert old stdout lightxml", msg, Continue); + QEXPECT_FAIL("assert old lightxml", msg, Continue); + QEXPECT_FAIL("assert old stdout xunitxml", msg, Continue); + QEXPECT_FAIL("assert old xunitxml", msg, Continue); + QEXPECT_FAIL("assert new stdout txt", msg, Continue); + QEXPECT_FAIL("assert new txt", msg, Continue); + QEXPECT_FAIL("assert new stdout xml", msg, Continue); + QEXPECT_FAIL("assert new xml", msg, Continue); + QEXPECT_FAIL("assert new stdout lightxml", msg, Continue); + QEXPECT_FAIL("assert new lightxml", msg, Continue); + QEXPECT_FAIL("assert new stdout xunitxml", msg, Continue); + QEXPECT_FAIL("assert new xunitxml", msg, Continue); + QEXPECT_FAIL("assert stdout txt + txt", msg, Continue); + QEXPECT_FAIL("assert xml + stdout txt", msg, Continue); + QEXPECT_FAIL("assert txt + xunitxml", msg, Continue); + QEXPECT_FAIL("assert lightxml + stdout xunitxml", msg, Continue); + QEXPECT_FAIL("assert all loggers", msg, Continue); } if (expected.startsWith(QLatin1String("FAIL! : tst_Exception::throwException() Caught unhandled exce")) && expected != output) |