From 51589e834e086de93a121a1c628c3492d88a15a7 Mon Sep 17 00:00:00 2001 From: Jason McDonald Date: Fri, 2 Sep 2011 19:08:21 +1000 Subject: Allow tests to log to multiple destinations Each destination and the format of output to write there is specified by adding "-o filename,format" to the command-line. The special filename "-" indicates that the log output is written to the standard output stream, though standard output can be used as a destination at most once. The old-style testlib output options are still supported, but can only be used to specify one logging destination, as before. If no logging options are given on the command-line, a plain text log will go to the console, as before. To log to the console in plain text and to the file "test_output" in xunit format, one would invoke a test in the following way: tst_foo -o test_output,xunitxml -o -,txt This commit also enhances the selftests to test with multiple loggers, but negative tests (e.g. bad combinations of command-line options) are left for future task QTBUG-21567. Task-number: QTBUG-20615 Change-Id: If91e752bc7001657e15e427aba9d25ab0a29a0b0 Reviewed-on: http://codereview.qt-project.org/4125 Reviewed-by: Qt Sanity Bot Reviewed-by: Rohan McGovern --- tests/auto/testlib/selftests/tst_selftests.cpp | 136 +++++++++++++++++++++---- 1 file changed, 116 insertions(+), 20 deletions(-) (limited to 'tests/auto') diff --git a/tests/auto/testlib/selftests/tst_selftests.cpp b/tests/auto/testlib/selftests/tst_selftests.cpp index 17727435ea..7c71a292c6 100644 --- a/tests/auto/testlib/selftests/tst_selftests.cpp +++ b/tests/auto/testlib/selftests/tst_selftests.cpp @@ -182,26 +182,106 @@ struct LoggerSet // running each subtest. static QList allLoggerSets() { - // For the plain text logger, we'll test logging to file and to standard - // output. For all other loggers (XML), we'll tell testlib to redirect to - // file. The reason is that tests are allowed to print to standard output, - // and that means the test log is no longer guaranteed to be valid XML. + // Note that in order to test XML output to standard output, the subtests + // must not send output directly to stdout, bypassing Qt's output mechanisms + // (e.g. via printf), otherwise the output may not be well-formed XML. return QList() - << LoggerSet("stdout txt", + // Test with old-style options for a single logger + << LoggerSet("old stdout txt", QStringList() << "stdout txt", - QStringList()) - << LoggerSet("txt", + QStringList() + ) + << LoggerSet("old txt", QStringList() << "txt", - QStringList() << "-o" << logName("txt")) - << LoggerSet("xml", + QStringList() << "-o" << logName("txt") + ) + << LoggerSet("old stdout xml", + QStringList() << "stdout xml", + QStringList() << "-xml" + ) + << LoggerSet("old xml", QStringList() << "xml", - QStringList() << "-xml" << "-o" << logName("xml")) - << LoggerSet("xunitxml", + QStringList() << "-xml" << "-o" << logName("xml") + ) + << LoggerSet("old stdout xunitxml", + QStringList() << "stdout xunitxml", + QStringList() << "-xunitxml" + ) + << LoggerSet("old xunitxml", QStringList() << "xunitxml", - QStringList() << "-xunitxml" << "-o" << logName("xunitxml")) - << LoggerSet("lightxml", + QStringList() << "-xunitxml" << "-o" << logName("xunitxml") + ) + << LoggerSet("old stdout lightxml", + QStringList() << "stdout lightxml", + QStringList() << "-lightxml" + ) + << LoggerSet("old lightxml", QStringList() << "lightxml", - QStringList() << "-lightxml" << "-o" << logName("lightxml")) + QStringList() << "-lightxml" << "-o" << logName("lightxml") + ) + // Test with new-style options for a single logger + << LoggerSet("new stdout txt", + QStringList() << "stdout txt", + QStringList() << "-o" << "-,txt" + ) + << LoggerSet("new txt", + QStringList() << "txt", + QStringList() << "-o" << logName("txt")+",txt" + ) + << LoggerSet("new stdout xml", + QStringList() << "stdout xml", + QStringList() << "-o" << "-,xml" + ) + << LoggerSet("new xml", + QStringList() << "xml", + QStringList() << "-o" << logName("xml")+",xml" + ) + << LoggerSet("new stdout xunitxml", + QStringList() << "stdout xunitxml", + QStringList() << "-o" << "-,xunitxml" + ) + << LoggerSet("new xunitxml", + QStringList() << "xunitxml", + QStringList() << "-o" << logName("xunitxml")+",xunitxml" + ) + << LoggerSet("new stdout lightxml", + QStringList() << "stdout lightxml", + QStringList() << "-o" << "-,lightxml" + ) + << LoggerSet("new lightxml", + QStringList() << "lightxml", + QStringList() << "-o" << logName("lightxml")+",lightxml" + ) + // Test with two loggers (don't test all 32 combinations, just a sample) + << LoggerSet("stdout txt + txt", + QStringList() << "stdout txt" << "txt", + QStringList() << "-o" << "-,txt" + << "-o" << logName("txt")+",txt" + ) + << LoggerSet("xml + stdout txt", + QStringList() << "xml" << "stdout txt", + QStringList() << "-o" << logName("xml")+",xml" + << "-o" << "-,txt" + ) + << LoggerSet("txt + xunitxml", + QStringList() << "txt" << "xunitxml", + QStringList() << "-o" << logName("txt")+",txt" + << "-o" << logName("xunitxml")+",xunitxml" + ) + << LoggerSet("lightxml + stdout xunitxml", + QStringList() << "lightxml" << "stdout xunitxml", + QStringList() << "-o" << logName("lightxml")+",lightxml" + << "-o" << "-,xunitxml" + ) + // All loggers at the same time + << LoggerSet("all loggers", + QStringList() << "txt" << "xml" << "lightxml" << "stdout txt" << "xunitxml", + QStringList() << "-o" << logName("txt")+",txt" + << "-o" << logName("xml")+",xml" + << "-o" << logName("lightxml")+",lightxml" + << "-o" << "-,txt" + << "-o" << logName("xunitxml")+",xunitxml" + ) ; } @@ -299,7 +379,7 @@ void tst_Selftests::runSubTest_data() // standard output, either because they execute multiple test // objects or because they internally supply arguments to // themselves. - if (loggerSet.name != "stdout txt") { + if (loggerSet.name != "old stdout txt" && loggerSet.name != "new stdout txt") { if (subtest == "differentexec") { continue; } @@ -441,11 +521,27 @@ void tst_Selftests::doRunSubTest(QString const& subdir, QStringList const& logge // __FILE__, while others do not. if (line.contains("ASSERT") && output != expected) { const char msg[] = "Q_ASSERT prints out the absolute path on this platform."; - QEXPECT_FAIL("assert stdout txt", msg, Continue); - QEXPECT_FAIL("assert txt", msg, Continue); - QEXPECT_FAIL("assert xml", msg, Continue); - QEXPECT_FAIL("assert lightxml", msg, Continue); - QEXPECT_FAIL("assert xunitxml", msg, Continue); + QEXPECT_FAIL("assert old stdout txt", msg, Continue); + QEXPECT_FAIL("assert old txt", msg, Continue); + QEXPECT_FAIL("assert old stdout xml", msg, Continue); + QEXPECT_FAIL("assert old xml", msg, Continue); + QEXPECT_FAIL("assert old stdout lightxml", msg, Continue); + QEXPECT_FAIL("assert old lightxml", msg, Continue); + QEXPECT_FAIL("assert old stdout xunitxml", msg, Continue); + QEXPECT_FAIL("assert old xunitxml", msg, Continue); + QEXPECT_FAIL("assert new stdout txt", msg, Continue); + QEXPECT_FAIL("assert new txt", msg, Continue); + QEXPECT_FAIL("assert new stdout xml", msg, Continue); + QEXPECT_FAIL("assert new xml", msg, Continue); + QEXPECT_FAIL("assert new stdout lightxml", msg, Continue); + QEXPECT_FAIL("assert new lightxml", msg, Continue); + QEXPECT_FAIL("assert new stdout xunitxml", msg, Continue); + QEXPECT_FAIL("assert new xunitxml", msg, Continue); + QEXPECT_FAIL("assert stdout txt + txt", msg, Continue); + QEXPECT_FAIL("assert xml + stdout txt", msg, Continue); + QEXPECT_FAIL("assert txt + xunitxml", msg, Continue); + QEXPECT_FAIL("assert lightxml + stdout xunitxml", msg, Continue); + QEXPECT_FAIL("assert all loggers", msg, Continue); } if (expected.startsWith(QLatin1String("FAIL! : tst_Exception::throwException() Caught unhandled exce")) && expected != output) -- cgit v1.2.3