From 8dcb72fd1e2c17ad000474496b6d000580e0a943 Mon Sep 17 00:00:00 2001 From: Thiago Macieira Date: Fri, 20 Dec 2013 11:14:04 -0800 Subject: Add a CSV logging feature to the benchlib This is only useful for logging benchmarks, since it won't print test passes, failures, etc. It's useful for importing to spreadsheets to do number-crunching. [ChangeLog][QtTest]Added a CSV logging mode that is suitable for importing benchmark results into spreadsheets. This can be enabled by the -csv option on the command-line. The CSV logging mode will not print test failures, debug messages, warnings, etc. Change-Id: I245d6f86bb380645c9bc0d748cf474b3ed42cab8 Reviewed-by: Sergio Ahumada Reviewed-by: Jason McDonald --- .../selftests/expected_benchlibcallgrind.csv | 1 + .../selftests/expected_benchlibcounting.csv | 1 + .../selftests/expected_benchlibeventcounter.csv | 7 ++++ .../testlib/selftests/expected_benchliboptions.csv | 3 ++ .../selftests/expected_benchlibtickcounter.csv | 1 + .../selftests/expected_benchlibwalltime.csv | 3 ++ tests/auto/testlib/selftests/selftests.qrc | 5 +++ tests/auto/testlib/selftests/tst_selftests.cpp | 49 +++++++++++++++++++++- 8 files changed, 68 insertions(+), 2 deletions(-) create mode 100644 tests/auto/testlib/selftests/expected_benchlibcallgrind.csv create mode 100644 tests/auto/testlib/selftests/expected_benchlibcounting.csv create mode 100644 tests/auto/testlib/selftests/expected_benchlibeventcounter.csv create mode 100644 tests/auto/testlib/selftests/expected_benchliboptions.csv create mode 100644 tests/auto/testlib/selftests/expected_benchlibtickcounter.csv create mode 100644 tests/auto/testlib/selftests/expected_benchlibwalltime.csv (limited to 'tests/auto/testlib/selftests') diff --git a/tests/auto/testlib/selftests/expected_benchlibcallgrind.csv b/tests/auto/testlib/selftests/expected_benchlibcallgrind.csv new file mode 100644 index 0000000000..6ce2e2ced8 --- /dev/null +++ b/tests/auto/testlib/selftests/expected_benchlibcallgrind.csv @@ -0,0 +1 @@ +"twoHundredMillionInstructions","","InstructionReads",200000158,200000158,1 diff --git a/tests/auto/testlib/selftests/expected_benchlibcounting.csv b/tests/auto/testlib/selftests/expected_benchlibcounting.csv new file mode 100644 index 0000000000..f3368b6854 --- /dev/null +++ b/tests/auto/testlib/selftests/expected_benchlibcounting.csv @@ -0,0 +1 @@ +"passingBenchmark","","Events",0,0,1 diff --git a/tests/auto/testlib/selftests/expected_benchlibeventcounter.csv b/tests/auto/testlib/selftests/expected_benchlibeventcounter.csv new file mode 100644 index 0000000000..2627687289 --- /dev/null +++ b/tests/auto/testlib/selftests/expected_benchlibeventcounter.csv @@ -0,0 +1,7 @@ +"events","0","Events",0,0,1 +"events","1","Events",1,1,1 +"events","10","Events",10,10,1 +"events","100","Events",100,100,1 +"events","500","Events",500,500,1 +"events","5000","Events",5000,5000,1 +"events","100000","Events",100000,100000,1 diff --git a/tests/auto/testlib/selftests/expected_benchliboptions.csv b/tests/auto/testlib/selftests/expected_benchliboptions.csv new file mode 100644 index 0000000000..9b899aed90 --- /dev/null +++ b/tests/auto/testlib/selftests/expected_benchliboptions.csv @@ -0,0 +1,3 @@ +"threeEvents","","Events",3,3,1 +"threeEvents","","Events",3,45,15 +"threeEvents","","Events",3,3,1 diff --git a/tests/auto/testlib/selftests/expected_benchlibtickcounter.csv b/tests/auto/testlib/selftests/expected_benchlibtickcounter.csv new file mode 100644 index 0000000000..fe5af1e7c8 --- /dev/null +++ b/tests/auto/testlib/selftests/expected_benchlibtickcounter.csv @@ -0,0 +1 @@ +"threeBillionTicks","","CPUTicks",3000000000,3000000000,1 diff --git a/tests/auto/testlib/selftests/expected_benchlibwalltime.csv b/tests/auto/testlib/selftests/expected_benchlibwalltime.csv new file mode 100644 index 0000000000..0dc2dee876 --- /dev/null +++ b/tests/auto/testlib/selftests/expected_benchlibwalltime.csv @@ -0,0 +1,3 @@ +"waitForOneThousand","","WalltimeMilliseconds",1000,1000,1 +"waitForFourThousand","","WalltimeMilliseconds",4000,4000,1 +"qbenchmark_once","","WalltimeMilliseconds",0,0,1 diff --git a/tests/auto/testlib/selftests/selftests.qrc b/tests/auto/testlib/selftests/selftests.qrc index e7ca7138b2..ba567f1fb4 100644 --- a/tests/auto/testlib/selftests/selftests.qrc +++ b/tests/auto/testlib/selftests/selftests.qrc @@ -10,23 +10,28 @@ expected_badxml.xml expected_badxml.xunitxml expected_benchlibcallgrind.txt + expected_benchlibcallgrind.csv expected_benchlibcounting.lightxml expected_benchlibcounting.txt expected_benchlibcounting.xml expected_benchlibcounting.xunitxml + expected_benchlibcounting.csv expected_benchlibeventcounter.lightxml expected_benchlibeventcounter.txt expected_benchlibeventcounter.xml expected_benchlibeventcounter.xunitxml + expected_benchlibeventcounter.csv expected_benchliboptions.txt expected_benchlibtickcounter.lightxml expected_benchlibtickcounter.txt expected_benchlibtickcounter.xml expected_benchlibtickcounter.xunitxml + expected_benchlibtickcounter.csv expected_benchlibwalltime.lightxml expected_benchlibwalltime.txt expected_benchlibwalltime.xml expected_benchlibwalltime.xunitxml + expected_benchlibwalltime.csv expected_cmptest.lightxml expected_cmptest.txt expected_cmptest.xml diff --git a/tests/auto/testlib/selftests/tst_selftests.cpp b/tests/auto/testlib/selftests/tst_selftests.cpp index c2265ad198..8167a96eaa 100644 --- a/tests/auto/testlib/selftests/tst_selftests.cpp +++ b/tests/auto/testlib/selftests/tst_selftests.cpp @@ -234,6 +234,12 @@ QList tst_Selftests::allLoggerSets() const QStringList() << "lightxml", QStringList() << "-lightxml" << "-o" << logName("lightxml") ) + << LoggerSet("old stdout csv", // benchmarks only + QStringList() << "stdout csv", + QStringList() << "-csv") + << LoggerSet("old csv", // benchmarks only + QStringList() << "csv", + QStringList() << "-csv" << "-o" << logName("csv")) // Test with new-style options for a single logger << LoggerSet("new stdout txt", QStringList() << "stdout txt", @@ -267,6 +273,12 @@ QList tst_Selftests::allLoggerSets() const QStringList() << "lightxml", QStringList() << "-o" << logName("lightxml")+",lightxml" ) + << LoggerSet("new stdout csv", // benchmarks only + QStringList() << "stdout csv", + QStringList() << "-o" << "-,csv") + << LoggerSet("new csv", // benchmarks only + QStringList() << "csv", + QStringList() << "-o" << logName("csv")+",csv") // Test with two loggers (don't test all 32 combinations, just a sample) << LoggerSet("stdout txt + txt", QStringList() << "stdout txt" << "txt", @@ -288,7 +300,7 @@ QList tst_Selftests::allLoggerSets() const QStringList() << "-o" << logName("lightxml")+",lightxml" << "-o" << "-,xunitxml" ) - // All loggers at the same time + // All loggers at the same time (except csv) << LoggerSet("all loggers", QStringList() << "txt" << "xml" << "lightxml" << "stdout txt" << "xunitxml", QStringList() << "-o" << logName("txt")+",txt" @@ -488,6 +500,10 @@ void tst_Selftests::runSubTest_data() } if (subtest == "badxml" && (loggerSet.name == "all loggers" || loggerSet.name.contains("txt"))) continue; // XML only, do not mix txt and XML for encoding test. + + if (loggerSet.name.contains("csv") && !subtest.startsWith("benchlib")) + continue; + const bool crashes = subtest == QLatin1String("assert") || subtest == QLatin1String("exceptionthrow") || subtest == QLatin1String("fetchbogus") || subtest == QLatin1String("crashedterminate") || subtest == QLatin1String("crashes") || subtest == QLatin1String("silent"); @@ -706,7 +722,7 @@ void tst_Selftests::doRunSubTest(QString const& subdir, QStringList const& logge else if (expected.startsWith(QLatin1String("FAIL! : tst_Exception::throwException() Caught unhandled exce")) && expected != output) // On some platforms we compile without RTTI, and as a result we never throw an exception. QCOMPARE(output.simplified(), QString::fromLatin1("tst_Exception::throwException()").simplified()); - else if (benchmark || line.startsWith("