From 8dcb72fd1e2c17ad000474496b6d000580e0a943 Mon Sep 17 00:00:00 2001 From: Thiago Macieira Date: Fri, 20 Dec 2013 11:14:04 -0800 Subject: Add a CSV logging feature to the benchlib This is only useful for logging benchmarks, since it won't print test passes, failures, etc. It's useful for importing to spreadsheets to do number-crunching. [ChangeLog][QtTest]Added a CSV logging mode that is suitable for importing benchmark results into spreadsheets. This can be enabled by the -csv option on the command-line. The CSV logging mode will not print test failures, debug messages, warnings, etc. Change-Id: I245d6f86bb380645c9bc0d748cf474b3ed42cab8 Reviewed-by: Sergio Ahumada Reviewed-by: Jason McDonald --- src/testlib/doc/src/qttestlib-manual.qdoc | 3 + src/testlib/qcsvbenchmarklogger.cpp | 103 +++++++++++++++++++++ src/testlib/qcsvbenchmarklogger_p.h | 82 ++++++++++++++++ src/testlib/qtestcase.cpp | 8 +- src/testlib/qtestlog.cpp | 4 + src/testlib/qtestlog_p.h | 2 +- src/testlib/testlib.pro | 3 +- .../selftests/expected_benchlibcallgrind.csv | 1 + .../selftests/expected_benchlibcounting.csv | 1 + .../selftests/expected_benchlibeventcounter.csv | 7 ++ .../testlib/selftests/expected_benchliboptions.csv | 3 + .../selftests/expected_benchlibtickcounter.csv | 1 + .../selftests/expected_benchlibwalltime.csv | 3 + tests/auto/testlib/selftests/selftests.qrc | 5 + tests/auto/testlib/selftests/tst_selftests.cpp | 49 +++++++++- 15 files changed, 270 insertions(+), 5 deletions(-) create mode 100644 src/testlib/qcsvbenchmarklogger.cpp create mode 100644 src/testlib/qcsvbenchmarklogger_p.h create mode 100644 tests/auto/testlib/selftests/expected_benchlibcallgrind.csv create mode 100644 tests/auto/testlib/selftests/expected_benchlibcounting.csv create mode 100644 tests/auto/testlib/selftests/expected_benchlibeventcounter.csv create mode 100644 tests/auto/testlib/selftests/expected_benchliboptions.csv create mode 100644 tests/auto/testlib/selftests/expected_benchlibtickcounter.csv create mode 100644 tests/auto/testlib/selftests/expected_benchlibwalltime.csv diff --git a/src/testlib/doc/src/qttestlib-manual.qdoc b/src/testlib/doc/src/qttestlib-manual.qdoc index bdd9769175..13654972ec 100644 --- a/src/testlib/doc/src/qttestlib-manual.qdoc +++ b/src/testlib/doc/src/qttestlib-manual.qdoc @@ -186,6 +186,9 @@ Outputs results as a stream of XML tags. \li \c -xunitxml \br Outputs results as an Xunit XML document. + \li \c -csv \br + Outputs results as comma-separated values (CSV). This mode is only suitable for + benchmarks, since it suppresses normal pass/fail messages. \endlist The first version of the \c -o option may be repeated in order to log diff --git a/src/testlib/qcsvbenchmarklogger.cpp b/src/testlib/qcsvbenchmarklogger.cpp new file mode 100644 index 0000000000..ae5c3183dc --- /dev/null +++ b/src/testlib/qcsvbenchmarklogger.cpp @@ -0,0 +1,103 @@ +/**************************************************************************** +** +** Copyright (C) 2013 Intel Corporation +** Contact: http://www.qt-project.org/legal +** +** This file is part of the QtTest module of the Qt Toolkit. +** +** $QT_BEGIN_LICENSE:LGPL$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and Digia. For licensing terms and +** conditions see http://qt.digia.com/licensing. For further information +** use the contact form at http://qt.digia.com/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 2.1 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 2.1 requirements +** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. +** +** In addition, as a special exception, Digia gives you certain additional +** rights. These rights are described in the Digia Qt LGPL Exception +** version 1.1, included in the file LGPL_EXCEPTION.txt in this package. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 3.0 as published by the Free Software +** Foundation and appearing in the file LICENSE.GPL included in the +** packaging of this file. Please review the following information to +** ensure the GNU General Public License version 3.0 requirements will be +** met: http://www.gnu.org/copyleft/gpl.html. +** +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#include "qcsvbenchmarklogger_p.h" +#include "qtestresult_p.h" +#include "qbenchmark_p.h" + +QCsvBenchmarkLogger::QCsvBenchmarkLogger(const char *filename) + : QAbstractTestLogger(filename) +{ +} + +QCsvBenchmarkLogger::~QCsvBenchmarkLogger() +{ +} + +void QCsvBenchmarkLogger::startLogging() +{ + // don't print anything +} + +void QCsvBenchmarkLogger::stopLogging() +{ + // don't print anything +} + +void QCsvBenchmarkLogger::enterTestFunction(const char *) +{ + // don't print anything +} + +void QCsvBenchmarkLogger::leaveTestFunction() +{ + // don't print anything +} + +void QCsvBenchmarkLogger::addIncident(QAbstractTestLogger::IncidentTypes, const char *, const char *, int) +{ + // don't print anything +} + +void QCsvBenchmarkLogger::addBenchmarkResult(const QBenchmarkResult &result) +{ + const char *fn = QTestResult::currentTestFunction() ? QTestResult::currentTestFunction() + : "UnknownTestFunc"; + const char *tag = QTestResult::currentDataTag() ? QTestResult::currentDataTag() : ""; + const char *gtag = QTestResult::currentGlobalDataTag() + ? QTestResult::currentGlobalDataTag() + : ""; + const char *filler = (tag[0] && gtag[0]) ? ":" : ""; + + const char *metric = QTest::benchmarkMetricName(result.metric); + + char buf[1024]; + // "function","[globaltag:]tag","metric",value_per_iteration,total,iterations + qsnprintf(buf, sizeof(buf), "\"%s\",\"%s%s%s\",\"%s\",%.13g,%.13g,%u\n", + fn, gtag, filler, tag, metric, + result.value / result.iterations, result.value, result.iterations); + outputString(buf); +} + +void QCsvBenchmarkLogger::addMessage(QAbstractTestLogger::MessageTypes, const QString &, const char *, int) +{ + // don't print anything +} diff --git a/src/testlib/qcsvbenchmarklogger_p.h b/src/testlib/qcsvbenchmarklogger_p.h new file mode 100644 index 0000000000..89692de972 --- /dev/null +++ b/src/testlib/qcsvbenchmarklogger_p.h @@ -0,0 +1,82 @@ +/**************************************************************************** +** +** Copyright (C) 2013 Intel Corporation +** Contact: http://www.qt-project.org/legal +** +** This file is part of the QtTest module of the Qt Toolkit. +** +** $QT_BEGIN_LICENSE:LGPL$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and Digia. For licensing terms and +** conditions see http://qt.digia.com/licensing. For further information +** use the contact form at http://qt.digia.com/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 2.1 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 2.1 requirements +** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. +** +** In addition, as a special exception, Digia gives you certain additional +** rights. These rights are described in the Digia Qt LGPL Exception +** version 1.1, included in the file LGPL_EXCEPTION.txt in this package. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 3.0 as published by the Free Software +** Foundation and appearing in the file LICENSE.GPL included in the +** packaging of this file. Please review the following information to +** ensure the GNU General Public License version 3.0 requirements will be +** met: http://www.gnu.org/copyleft/gpl.html. +** +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#ifndef QCSVBENCHMARKLOGGER_P_H +#define QCSVBENCHMARKLOGGER_P_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include "qabstracttestlogger_p.h" + +QT_BEGIN_NAMESPACE + +class QCsvBenchmarkLogger : public QAbstractTestLogger +{ +public: + QCsvBenchmarkLogger(const char *filename); + ~QCsvBenchmarkLogger(); + + void startLogging() Q_DECL_OVERRIDE; + void stopLogging() Q_DECL_OVERRIDE; + + void enterTestFunction(const char *function) Q_DECL_OVERRIDE; + void leaveTestFunction() Q_DECL_OVERRIDE; + + void addIncident(IncidentTypes type, const char *description, + const char *file = 0, int line = 0) Q_DECL_OVERRIDE; + void addBenchmarkResult(const QBenchmarkResult &result) Q_DECL_OVERRIDE; + + void addMessage(MessageTypes type, const QString &message, + const char *file = 0, int line = 0) Q_DECL_OVERRIDE; +}; + +QT_END_NAMESPACE + +#endif // QCSVBENCHMARKLOGGER_P_H diff --git a/src/testlib/qtestcase.cpp b/src/testlib/qtestcase.cpp index 224357dd85..0d7a017f89 100644 --- a/src/testlib/qtestcase.cpp +++ b/src/testlib/qtestcase.cpp @@ -1320,6 +1320,7 @@ Q_TESTLIB_EXPORT void qtest_qParseArgs(int argc, char *argv[], bool qml) " Use - to output to stdout\n" " Valid formats are:\n" " txt : Plain text\n" + " csv : CSV format (suitable for benchmarks)\n" " xunitxml : XML XUnit document\n" " xml : XML document\n" " lightxml : A stream of XML tags\n" @@ -1329,6 +1330,7 @@ Q_TESTLIB_EXPORT void qtest_qParseArgs(int argc, char *argv[], bool qml) " Old-style logging options:\n" " -o filename : Write the output into file\n" " -txt : Output results in Plain Text\n" + " -csv : Output results in a CSV format (suitable for benchmarks)\n" " -xunitxml : Output results as XML XUnit document\n" " -xml : Output results as XML document\n" " -lightxml : Output results as stream of XML tags\n" @@ -1408,6 +1410,8 @@ Q_TESTLIB_EXPORT void qtest_qParseArgs(int argc, char *argv[], bool qml) } } else if (strcmp(argv[i], "-txt") == 0) { logFormat = QTestLog::Plain; + } else if (strcmp(argv[i], "-csv") == 0) { + logFormat = QTestLog::CSV; } else if (strcmp(argv[i], "-xunitxml") == 0) { logFormat = QTestLog::XunitXML; } else if (strcmp(argv[i], "-xml") == 0) { @@ -1438,6 +1442,8 @@ Q_TESTLIB_EXPORT void qtest_qParseArgs(int argc, char *argv[], bool qml) // New-style if (strcmp(format, "txt") == 0) logFormat = QTestLog::Plain; + else if (strcmp(format, "csv") == 0) + logFormat = QTestLog::CSV; else if (strcmp(format, "lightxml") == 0) logFormat = QTestLog::LightXML; else if (strcmp(format, "xml") == 0) @@ -1445,7 +1451,7 @@ Q_TESTLIB_EXPORT void qtest_qParseArgs(int argc, char *argv[], bool qml) else if (strcmp(format, "xunitxml") == 0) logFormat = QTestLog::XunitXML; else { - fprintf(stderr, "output format must be one of txt, lightxml, xml or xunitxml\n"); + fprintf(stderr, "output format must be one of txt, csv, lightxml, xml or xunitxml\n"); exit(1); } if (strcmp(filename, "-") == 0 && QTestLog::loggerUsingStdout()) { diff --git a/src/testlib/qtestlog.cpp b/src/testlib/qtestlog.cpp index 74947b3f3a..8cca892a4d 100644 --- a/src/testlib/qtestlog.cpp +++ b/src/testlib/qtestlog.cpp @@ -45,6 +45,7 @@ #include #include #include +#include #include #include #include @@ -452,6 +453,9 @@ void QTestLog::addLogger(LogMode mode, const char *filename) case QTestLog::Plain: logger = new QPlainTestLogger(filename); break; + case QTestLog::CSV: + logger = new QCsvBenchmarkLogger(filename); + break; case QTestLog::XML: logger = new QXmlTestLogger(QXmlTestLogger::Complete, filename); break; diff --git a/src/testlib/qtestlog_p.h b/src/testlib/qtestlog_p.h index bd83870934..a987c45806 100644 --- a/src/testlib/qtestlog_p.h +++ b/src/testlib/qtestlog_p.h @@ -63,7 +63,7 @@ class QRegularExpression; class Q_TESTLIB_EXPORT QTestLog { public: - enum LogMode { Plain = 0, XML, LightXML, XunitXML }; + enum LogMode { Plain = 0, XML, LightXML, XunitXML, CSV }; static void enterTestFunction(const char* function); static void leaveTestFunction(); diff --git a/src/testlib/testlib.pro b/src/testlib/testlib.pro index 571e3b2fca..1fb6d6df18 100644 --- a/src/testlib/testlib.pro +++ b/src/testlib/testlib.pro @@ -33,7 +33,7 @@ HEADERS = qbenchmark.h \ qtestmouse.h \ qtestspontaneevent.h \ qtestsystem.h \ - qtesttouch.h \ + qtesttouch.h SOURCES = qtestcase.cpp \ qtestlog.cpp \ @@ -51,6 +51,7 @@ SOURCES = qtestcase.cpp \ qbenchmarkevent.cpp \ qbenchmarkperfevents.cpp \ qbenchmarkmetric.cpp \ + qcsvbenchmarklogger.cpp \ qtestelement.cpp \ qtestelementattribute.cpp \ qtestxunitstreamer.cpp \ diff --git a/tests/auto/testlib/selftests/expected_benchlibcallgrind.csv b/tests/auto/testlib/selftests/expected_benchlibcallgrind.csv new file mode 100644 index 0000000000..6ce2e2ced8 --- /dev/null +++ b/tests/auto/testlib/selftests/expected_benchlibcallgrind.csv @@ -0,0 +1 @@ +"twoHundredMillionInstructions","","InstructionReads",200000158,200000158,1 diff --git a/tests/auto/testlib/selftests/expected_benchlibcounting.csv b/tests/auto/testlib/selftests/expected_benchlibcounting.csv new file mode 100644 index 0000000000..f3368b6854 --- /dev/null +++ b/tests/auto/testlib/selftests/expected_benchlibcounting.csv @@ -0,0 +1 @@ +"passingBenchmark","","Events",0,0,1 diff --git a/tests/auto/testlib/selftests/expected_benchlibeventcounter.csv b/tests/auto/testlib/selftests/expected_benchlibeventcounter.csv new file mode 100644 index 0000000000..2627687289 --- /dev/null +++ b/tests/auto/testlib/selftests/expected_benchlibeventcounter.csv @@ -0,0 +1,7 @@ +"events","0","Events",0,0,1 +"events","1","Events",1,1,1 +"events","10","Events",10,10,1 +"events","100","Events",100,100,1 +"events","500","Events",500,500,1 +"events","5000","Events",5000,5000,1 +"events","100000","Events",100000,100000,1 diff --git a/tests/auto/testlib/selftests/expected_benchliboptions.csv b/tests/auto/testlib/selftests/expected_benchliboptions.csv new file mode 100644 index 0000000000..9b899aed90 --- /dev/null +++ b/tests/auto/testlib/selftests/expected_benchliboptions.csv @@ -0,0 +1,3 @@ +"threeEvents","","Events",3,3,1 +"threeEvents","","Events",3,45,15 +"threeEvents","","Events",3,3,1 diff --git a/tests/auto/testlib/selftests/expected_benchlibtickcounter.csv b/tests/auto/testlib/selftests/expected_benchlibtickcounter.csv new file mode 100644 index 0000000000..fe5af1e7c8 --- /dev/null +++ b/tests/auto/testlib/selftests/expected_benchlibtickcounter.csv @@ -0,0 +1 @@ +"threeBillionTicks","","CPUTicks",3000000000,3000000000,1 diff --git a/tests/auto/testlib/selftests/expected_benchlibwalltime.csv b/tests/auto/testlib/selftests/expected_benchlibwalltime.csv new file mode 100644 index 0000000000..0dc2dee876 --- /dev/null +++ b/tests/auto/testlib/selftests/expected_benchlibwalltime.csv @@ -0,0 +1,3 @@ +"waitForOneThousand","","WalltimeMilliseconds",1000,1000,1 +"waitForFourThousand","","WalltimeMilliseconds",4000,4000,1 +"qbenchmark_once","","WalltimeMilliseconds",0,0,1 diff --git a/tests/auto/testlib/selftests/selftests.qrc b/tests/auto/testlib/selftests/selftests.qrc index e7ca7138b2..ba567f1fb4 100644 --- a/tests/auto/testlib/selftests/selftests.qrc +++ b/tests/auto/testlib/selftests/selftests.qrc @@ -10,23 +10,28 @@ expected_badxml.xml expected_badxml.xunitxml expected_benchlibcallgrind.txt + expected_benchlibcallgrind.csv expected_benchlibcounting.lightxml expected_benchlibcounting.txt expected_benchlibcounting.xml expected_benchlibcounting.xunitxml + expected_benchlibcounting.csv expected_benchlibeventcounter.lightxml expected_benchlibeventcounter.txt expected_benchlibeventcounter.xml expected_benchlibeventcounter.xunitxml + expected_benchlibeventcounter.csv expected_benchliboptions.txt expected_benchlibtickcounter.lightxml expected_benchlibtickcounter.txt expected_benchlibtickcounter.xml expected_benchlibtickcounter.xunitxml + expected_benchlibtickcounter.csv expected_benchlibwalltime.lightxml expected_benchlibwalltime.txt expected_benchlibwalltime.xml expected_benchlibwalltime.xunitxml + expected_benchlibwalltime.csv expected_cmptest.lightxml expected_cmptest.txt expected_cmptest.xml diff --git a/tests/auto/testlib/selftests/tst_selftests.cpp b/tests/auto/testlib/selftests/tst_selftests.cpp index c2265ad198..8167a96eaa 100644 --- a/tests/auto/testlib/selftests/tst_selftests.cpp +++ b/tests/auto/testlib/selftests/tst_selftests.cpp @@ -234,6 +234,12 @@ QList tst_Selftests::allLoggerSets() const QStringList() << "lightxml", QStringList() << "-lightxml" << "-o" << logName("lightxml") ) + << LoggerSet("old stdout csv", // benchmarks only + QStringList() << "stdout csv", + QStringList() << "-csv") + << LoggerSet("old csv", // benchmarks only + QStringList() << "csv", + QStringList() << "-csv" << "-o" << logName("csv")) // Test with new-style options for a single logger << LoggerSet("new stdout txt", QStringList() << "stdout txt", @@ -267,6 +273,12 @@ QList tst_Selftests::allLoggerSets() const QStringList() << "lightxml", QStringList() << "-o" << logName("lightxml")+",lightxml" ) + << LoggerSet("new stdout csv", // benchmarks only + QStringList() << "stdout csv", + QStringList() << "-o" << "-,csv") + << LoggerSet("new csv", // benchmarks only + QStringList() << "csv", + QStringList() << "-o" << logName("csv")+",csv") // Test with two loggers (don't test all 32 combinations, just a sample) << LoggerSet("stdout txt + txt", QStringList() << "stdout txt" << "txt", @@ -288,7 +300,7 @@ QList tst_Selftests::allLoggerSets() const QStringList() << "-o" << logName("lightxml")+",lightxml" << "-o" << "-,xunitxml" ) - // All loggers at the same time + // All loggers at the same time (except csv) << LoggerSet("all loggers", QStringList() << "txt" << "xml" << "lightxml" << "stdout txt" << "xunitxml", QStringList() << "-o" << logName("txt")+",txt" @@ -488,6 +500,10 @@ void tst_Selftests::runSubTest_data() } if (subtest == "badxml" && (loggerSet.name == "all loggers" || loggerSet.name.contains("txt"))) continue; // XML only, do not mix txt and XML for encoding test. + + if (loggerSet.name.contains("csv") && !subtest.startsWith("benchlib")) + continue; + const bool crashes = subtest == QLatin1String("assert") || subtest == QLatin1String("exceptionthrow") || subtest == QLatin1String("fetchbogus") || subtest == QLatin1String("crashedterminate") || subtest == QLatin1String("crashes") || subtest == QLatin1String("silent"); @@ -706,7 +722,7 @@ void tst_Selftests::doRunSubTest(QString const& subdir, QStringList const& logge else if (expected.startsWith(QLatin1String("FAIL! : tst_Exception::throwException() Caught unhandled exce")) && expected != output) // On some platforms we compile without RTTI, and as a result we never throw an exception. QCOMPARE(output.simplified(), QString::fromLatin1("tst_Exception::throwException()").simplified()); - else if (benchmark || line.startsWith("