Add a CSV logging feature to the benchlib

This is only useful for logging benchmarks, since it won't print test
passes, failures, etc. It's useful for importing to spreadsheets to do
number-crunching.

[ChangeLog][QtTest]Added a CSV logging mode that is suitable for
importing benchmark results into spreadsheets. This can be enabled by
the -csv option on the command-line. The CSV logging mode will not print
test failures, debug messages, warnings, etc.

Change-Id: I245d6f86bb380645c9bc0d748cf474b3ed42cab8
Reviewed-by: Sergio Ahumada <sahumada@blackberry.com>
Reviewed-by: Jason McDonald <macadder1@gmail.com>

8 files changed, 68 insertions, 2 deletions

diff --git a/tests/auto/testlib/selftests/expected_benchlibcallgrind.csv b/tests/auto/testlib/selftests/expected_benchlibcallgrind.csv
new file mode 100644
index 0000000000..6ce2e2ced8
--- /dev/null
+++ b/tests/auto/testlib/selftests/expected_benchlibcallgrind.csv
@@ -0,0 +1 @@
+"twoHundredMillionInstructions","","InstructionReads",200000158,200000158,1
diff --git a/tests/auto/testlib/selftests/expected_benchlibcounting.csv b/tests/auto/testlib/selftests/expected_benchlibcounting.csv
new file mode 100644
index 0000000000..f3368b6854
--- /dev/null
+++ b/tests/auto/testlib/selftests/expected_benchlibcounting.csv
@@ -0,0 +1 @@
+"passingBenchmark","","Events",0,0,1
diff --git a/tests/auto/testlib/selftests/expected_benchlibeventcounter.csv b/tests/auto/testlib/selftests/expected_benchlibeventcounter.csv
new file mode 100644
index 0000000000..2627687289
--- /dev/null
+++ b/tests/auto/testlib/selftests/expected_benchlibeventcounter.csv
@@ -0,0 +1,7 @@
+"events","0","Events",0,0,1
+"events","1","Events",1,1,1
+"events","10","Events",10,10,1
+"events","100","Events",100,100,1
+"events","500","Events",500,500,1
+"events","5000","Events",5000,5000,1
+"events","100000","Events",100000,100000,1
diff --git a/tests/auto/testlib/selftests/expected_benchliboptions.csv b/tests/auto/testlib/selftests/expected_benchliboptions.csv
new file mode 100644
index 0000000000..9b899aed90
--- /dev/null
+++ b/tests/auto/testlib/selftests/expected_benchliboptions.csv
@@ -0,0 +1,3 @@
+"threeEvents","","Events",3,3,1
+"threeEvents","","Events",3,45,15
+"threeEvents","","Events",3,3,1
diff --git a/tests/auto/testlib/selftests/expected_benchlibtickcounter.csv b/tests/auto/testlib/selftests/expected_benchlibtickcounter.csv
new file mode 100644
index 0000000000..fe5af1e7c8
--- /dev/null
+++ b/tests/auto/testlib/selftests/expected_benchlibtickcounter.csv
@@ -0,0 +1 @@
+"threeBillionTicks","","CPUTicks",3000000000,3000000000,1
diff --git a/tests/auto/testlib/selftests/expected_benchlibwalltime.csv b/tests/auto/testlib/selftests/expected_benchlibwalltime.csv
new file mode 100644
index 0000000000..0dc2dee876
--- /dev/null
+++ b/tests/auto/testlib/selftests/expected_benchlibwalltime.csv
@@ -0,0 +1,3 @@
+"waitForOneThousand","","WalltimeMilliseconds",1000,1000,1
+"waitForFourThousand","","WalltimeMilliseconds",4000,4000,1
+"qbenchmark_once","","WalltimeMilliseconds",0,0,1
diff --git a/tests/auto/testlib/selftests/selftests.qrc b/tests/auto/testlib/selftests/selftests.qrc
index e7ca7138b2..ba567f1fb4 100644
--- a/tests/auto/testlib/selftests/selftests.qrc
+++ b/tests/auto/testlib/selftests/selftests.qrc
@@ -10,23 +10,28 @@
         <file>expected_badxml.xml</file>
         <file>expected_badxml.xunitxml</file>
         <file>expected_benchlibcallgrind.txt</file>
+        <file>expected_benchlibcallgrind.csv</file>
         <file>expected_benchlibcounting.lightxml</file>
         <file>expected_benchlibcounting.txt</file>
         <file>expected_benchlibcounting.xml</file>
         <file>expected_benchlibcounting.xunitxml</file>
+        <file>expected_benchlibcounting.csv</file>
         <file>expected_benchlibeventcounter.lightxml</file>
         <file>expected_benchlibeventcounter.txt</file>
         <file>expected_benchlibeventcounter.xml</file>
         <file>expected_benchlibeventcounter.xunitxml</file>
+        <file>expected_benchlibeventcounter.csv</file>
         <file>expected_benchliboptions.txt</file>
         <file>expected_benchlibtickcounter.lightxml</file>
         <file>expected_benchlibtickcounter.txt</file>
         <file>expected_benchlibtickcounter.xml</file>
         <file>expected_benchlibtickcounter.xunitxml</file>
+        <file>expected_benchlibtickcounter.csv</file>
         <file>expected_benchlibwalltime.lightxml</file>
         <file>expected_benchlibwalltime.txt</file>
         <file>expected_benchlibwalltime.xml</file>
         <file>expected_benchlibwalltime.xunitxml</file>
+        <file>expected_benchlibwalltime.csv</file>
         <file>expected_cmptest.lightxml</file>
         <file>expected_cmptest.txt</file>
         <file>expected_cmptest.xml</file>
diff --git a/tests/auto/testlib/selftests/tst_selftests.cpp b/tests/auto/testlib/selftests/tst_selftests.cpp
index c2265ad198..8167a96eaa 100644
--- a/tests/auto/testlib/selftests/tst_selftests.cpp
+++ b/tests/auto/testlib/selftests/tst_selftests.cpp
@@ -234,6 +234,12 @@ QList<LoggerSet> tst_Selftests::allLoggerSets() const
                      QStringList() << "lightxml",
                      QStringList() << "-lightxml" << "-o" << logName("lightxml")
                     )
+        << LoggerSet("old stdout csv", // benchmarks only
+                     QStringList() << "stdout csv",
+                     QStringList() << "-csv")
+        << LoggerSet("old csv", // benchmarks only
+                     QStringList() << "csv",
+                     QStringList() << "-csv" << "-o" << logName("csv"))
         // Test with new-style options for a single logger
         << LoggerSet("new stdout txt",
                      QStringList() << "stdout txt",
@@ -267,6 +273,12 @@ QList<LoggerSet> tst_Selftests::allLoggerSets() const
                      QStringList() << "lightxml",
                      QStringList() << "-o" << logName("lightxml")+",lightxml"
                     )
+        << LoggerSet("new stdout csv", // benchmarks only
+                     QStringList() << "stdout csv",
+                     QStringList() << "-o" << "-,csv")
+        << LoggerSet("new csv", // benchmarks only
+                     QStringList() << "csv",
+                     QStringList() << "-o" << logName("csv")+",csv")
         // Test with two loggers (don't test all 32 combinations, just a sample)
         << LoggerSet("stdout txt + txt",
                      QStringList() << "stdout txt" << "txt",
@@ -288,7 +300,7 @@ QList<LoggerSet> tst_Selftests::allLoggerSets() const
                      QStringList() << "-o" << logName("lightxml")+",lightxml"
                                    << "-o" << "-,xunitxml"
                     )
-        // All loggers at the same time
+        // All loggers at the same time (except csv)
         << LoggerSet("all loggers",
                      QStringList() << "txt" << "xml" << "lightxml" << "stdout txt" << "xunitxml",
                      QStringList() << "-o" << logName("txt")+",txt"
@@ -488,6 +500,10 @@ void tst_Selftests::runSubTest_data()
             }
             if (subtest == "badxml" && (loggerSet.name == "all loggers" || loggerSet.name.contains("txt")))
                 continue; // XML only, do not mix txt and XML for encoding test.
+
+            if (loggerSet.name.contains("csv") && !subtest.startsWith("benchlib"))
+                continue;
+
             const bool crashes = subtest == QLatin1String("assert") || subtest == QLatin1String("exceptionthrow")
                 || subtest == QLatin1String("fetchbogus") || subtest == QLatin1String("crashedterminate")
                 || subtest == QLatin1String("crashes") || subtest == QLatin1String("silent");
@@ -706,7 +722,7 @@ void tst_Selftests::doRunSubTest(QString const& subdir, QStringList const& logge
             else if (expected.startsWith(QLatin1String("FAIL!  : tst_Exception::throwException() Caught unhandled exce")) && expected != output)
                 // On some platforms we compile without RTTI, and as a result we never throw an exception.
                 QCOMPARE(output.simplified(), QString::fromLatin1("tst_Exception::throwException()").simplified());
-            else if (benchmark || line.startsWith("<BenchmarkResult")) {
+            else if (benchmark || line.startsWith("<BenchmarkResult") || (logFormat(logger) == "csv" && line.startsWith('"'))) {
                 // Don't do a literal comparison for benchmark results, since
                 // results have some natural variance.
                 QString error;
@@ -813,6 +829,35 @@ BenchmarkResult BenchmarkResult::parse(QString const& line, QString* error)
         out.iterations = iterations;
         return out;
     }
+
+    if (line.startsWith('"')) {
+        // CSV result
+        // format:
+        //  "function","[globaltag:]tag","metric",value_per_iteration,total,iterations
+        QStringList split = line.split(',');
+        if (split.count() != 6) {
+            if (error) *error = QString("Wrong number of columns (%1)").arg(split.count());
+            return out;
+        }
+
+        bool ok;
+        double total = split.at(4).toDouble(&ok);
+        if (!ok) {
+            if (error) *error = split.at(4) + " is not a valid number";
+            return out;
+        }
+        double iterations = split.at(5).toDouble(&ok);
+        if (!ok) {
+            if (error) *error = split.at(5) + " is not a valid number";
+            return out;
+        }
+
+        out.unit = split.at(2);
+        out.total = total;
+        out.iterations = iterations;
+        return out;
+    }
+
     // Text result
     // This code avoids using a QRegExp because QRegExp might be broken.
     // Sample format: 4,000 msec per iteration (total: 4,000, iterations: 1)