Add minimumtotal option to improve accuracy of short-lived benchmarks.

Short-lived benchmarks (benchmarks that complete in a very short period
of measured time) are often more affected by jitter and warm-up effects
than longer running benchmarks. Since QBENCHMARK stores the median
result of all accepted benchmark runs, a larger number of aggregation
runs is preferable for short-lived tests, but not necessarily for longer
running ones. The minimumtotal option, specified in units of the
selected measurement, will make the benchmark repeat a benchmark until
the total measured cost exceeds the specified threshold. The displayed
median result will then tend to be more accurate. This is especially
useful for data-driven benchmarks in case the data tags scale the
benchmarked operation from little to large cost.

Change-Id: Ib857de64aaffc77715a0000d36f0245f31d86b9a
Reviewed-by: Jason McDonald <macadder1@gmail.com>

4 files changed, 33 insertions, 1 deletions

diff --git a/src/testlib/doc/src/qttestlib-manual.qdoc b/src/testlib/doc/src/qttestlib-manual.qdoc
index 9d4a34c9d4..c44bb49ae3 100644
--- a/src/testlib/doc/src/qttestlib-manual.qdoc
+++ b/src/testlib/doc/src/qttestlib-manual.qdoc
@@ -268,6 +268,8 @@
     Counts events received during benchmarks.
     \li \c -minimumvalue \e n \br
     Sets the minimum acceptable measurement value.
+    \li \c -minimumtotal \e n \br
+    Sets the minimum acceptable total for repeated executions of a test function.
     \li \c -iterations \e n \br
     Sets the number of accumulation iterations.
     \li \c -median \e n \br
diff --git a/src/testlib/qbenchmark.cpp b/src/testlib/qbenchmark.cpp
index 91cafc99e3..3ef29e19ce 100644
--- a/src/testlib/qbenchmark.cpp
+++ b/src/testlib/qbenchmark.cpp
@@ -59,6 +59,7 @@ QBenchmarkGlobalData::QBenchmarkGlobalData()
     , medianIterationCount(-1)
     , createChart(false)
     , verboseOutput(false)
+    , minimumTotal(-1)
     , mode_(WallTime)
 {
     setMode(mode_);
diff --git a/src/testlib/qbenchmark_p.h b/src/testlib/qbenchmark_p.h
index c5dac38fec..351287b103 100644
--- a/src/testlib/qbenchmark_p.h
+++ b/src/testlib/qbenchmark_p.h
@@ -160,6 +160,7 @@ public:
     bool createChart;
     bool verboseOutput;
     QString callgrindOutFileBase;
+    int minimumTotal;
 private:
     Mode mode_;
 };
diff --git a/src/testlib/qtestcase.cpp b/src/testlib/qtestcase.cpp
index c1ab574291..27fcc64ceb 100644
--- a/src/testlib/qtestcase.cpp
+++ b/src/testlib/qtestcase.cpp
@@ -64,6 +64,8 @@
 #include <QtTest/private/qbenchmark_p.h>
 #include <QtTest/private/cycle_p.h>
 
+#include <numeric>
+
 #include <stdarg.h>
 #include <stdio.h>
 #include <stdlib.h>
@@ -1351,6 +1353,7 @@ Q_TESTLIB_EXPORT void qtest_qParseArgs(int argc, char *argv[], bool qml)
 #endif
          " -eventcounter       : Counts events received during benchmarks\n"
          " -minimumvalue n     : Sets the minimum acceptable measurement value\n"
+         " -minimumtotal n     : Sets the minimum acceptable total for repeated executions of a test function\n"
          " -iterations  n      : Sets the number of accumulation iterations.\n"
          " -median  n          : Sets the number of median iterations.\n"
          " -vb                 : Print out verbose benchmarking information.\n";
@@ -1518,6 +1521,13 @@ Q_TESTLIB_EXPORT void qtest_qParseArgs(int argc, char *argv[], bool qml)
             } else {
                 QBenchmarkGlobalData::current->walltimeMinimum = qToInt(argv[++i]);
             }
+        } else if (strcmp(argv[i], "-minimumtotal") == 0) {
+            if (i + 1 >= argc) {
+                fprintf(stderr, "-minimumtotal needs an extra parameter to indicate the minimum total measurement\n");
+                exit(1);
+            } else {
+                QBenchmarkGlobalData::current->minimumTotal = qToInt(argv[++i]);
+            }
         } else if (strcmp(argv[i], "-iterations") == 0) {
             if (i + 1 >= argc) {
                 fprintf(stderr, "-iterations needs an extra parameter to indicate the number of iterations\n");
@@ -1647,6 +1657,15 @@ struct QTestDataSetter
     }
 };
 
+namespace {
+
+qreal addResult(qreal current, const QBenchmarkResult& r)
+{
+    return current + r.value;
+}
+
+}
+
 static void qInvokeTestMethodDataEntry(char *slot)
 {
     /* Benchmarking: for each median iteration*/
@@ -1655,6 +1674,7 @@ static void qInvokeTestMethodDataEntry(char *slot)
     int i = (QBenchmarkGlobalData::current->measurer->needsWarmupIteration()) ? -1 : 0;
 
     QList<QBenchmarkResult> results;
+    bool minimumTotalReached = false;
     do {
         QBenchmarkTestMethodData::current->beginDataRun();
 
@@ -1713,8 +1733,16 @@ static void qInvokeTestMethodDataEntry(char *slot)
                 }
             }
         }
+
+        // Verify if the minimum total measurement is reached, if it was specified:
+        if (QBenchmarkGlobalData::current->minimumTotal == -1) {
+            minimumTotalReached = true;
+        } else {
+            const qreal total = std::accumulate(results.begin(), results.end(), 0.0, addResult);
+            minimumTotalReached = (total >= QBenchmarkGlobalData::current->minimumTotal);
+        }
     } while (isBenchmark
-             && (++i < QBenchmarkGlobalData::current->adjustMedianIterationCount())
+             && ((++i < QBenchmarkGlobalData::current->adjustMedianIterationCount()) || !minimumTotalReached)
              && !QTestResult::skipCurrentTest() && !QTestResult::currentTestFailed());
 
     // If the test is a benchmark, finalize the result after all iterations have finished.