diff options
author | Mirko Boehm (AWS) <mirko@agile-workers.com> | 2013-03-14 08:32:38 +0100 |
---|---|---|
committer | The Qt Project <gerrit-noreply@qt-project.org> | 2013-03-19 10:29:23 +0100 |
commit | 1a708277c587031d4d00e095be8251e3893b0c45 (patch) | |
tree | 39fd606c2b6f335281bcdc76e9b5df8739ece9e5 /src/testlib/qtestcase.cpp | |
parent | 9817707c715e87f470df0f25024333a01b94b0cc (diff) |
Add minimumtotal option to improve accuracy of short-lived benchmarks.
Short-lived benchmarks (benchmarks that complete in a very short period
of measured time) are often more affected by jitter and warm-up effects
than longer running benchmarks. Since QBENCHMARK stores the median
result of all accepted benchmark runs, a larger number of aggregation
runs is preferable for short-lived tests, but not necessarily for longer
running ones. The minimumtotal option, specified in units of the
selected measurement, will make the benchmark repeat a benchmark until
the total measured cost exceeds the specified threshold. The displayed
median result will then tend to be more accurate. This is especially
useful for data-driven benchmarks in case the data tags scale the
benchmarked operation from little to large cost.
Change-Id: Ib857de64aaffc77715a0000d36f0245f31d86b9a
Reviewed-by: Jason McDonald <macadder1@gmail.com>
Diffstat (limited to 'src/testlib/qtestcase.cpp')
-rw-r--r-- | src/testlib/qtestcase.cpp | 30 |
1 files changed, 29 insertions, 1 deletions
diff --git a/src/testlib/qtestcase.cpp b/src/testlib/qtestcase.cpp index c1ab574291..27fcc64ceb 100644 --- a/src/testlib/qtestcase.cpp +++ b/src/testlib/qtestcase.cpp @@ -64,6 +64,8 @@ #include <QtTest/private/qbenchmark_p.h> #include <QtTest/private/cycle_p.h> +#include <numeric> + #include <stdarg.h> #include <stdio.h> #include <stdlib.h> @@ -1351,6 +1353,7 @@ Q_TESTLIB_EXPORT void qtest_qParseArgs(int argc, char *argv[], bool qml) #endif " -eventcounter : Counts events received during benchmarks\n" " -minimumvalue n : Sets the minimum acceptable measurement value\n" + " -minimumtotal n : Sets the minimum acceptable total for repeated executions of a test function\n" " -iterations n : Sets the number of accumulation iterations.\n" " -median n : Sets the number of median iterations.\n" " -vb : Print out verbose benchmarking information.\n"; @@ -1518,6 +1521,13 @@ Q_TESTLIB_EXPORT void qtest_qParseArgs(int argc, char *argv[], bool qml) } else { QBenchmarkGlobalData::current->walltimeMinimum = qToInt(argv[++i]); } + } else if (strcmp(argv[i], "-minimumtotal") == 0) { + if (i + 1 >= argc) { + fprintf(stderr, "-minimumtotal needs an extra parameter to indicate the minimum total measurement\n"); + exit(1); + } else { + QBenchmarkGlobalData::current->minimumTotal = qToInt(argv[++i]); + } } else if (strcmp(argv[i], "-iterations") == 0) { if (i + 1 >= argc) { fprintf(stderr, "-iterations needs an extra parameter to indicate the number of iterations\n"); @@ -1647,6 +1657,15 @@ struct QTestDataSetter } }; +namespace { + +qreal addResult(qreal current, const QBenchmarkResult& r) +{ + return current + r.value; +} + +} + static void qInvokeTestMethodDataEntry(char *slot) { /* Benchmarking: for each median iteration*/ @@ -1655,6 +1674,7 @@ static void qInvokeTestMethodDataEntry(char *slot) int i = (QBenchmarkGlobalData::current->measurer->needsWarmupIteration()) ? -1 : 0; QList<QBenchmarkResult> results; + bool minimumTotalReached = false; do { QBenchmarkTestMethodData::current->beginDataRun(); @@ -1713,8 +1733,16 @@ static void qInvokeTestMethodDataEntry(char *slot) } } } + + // Verify if the minimum total measurement is reached, if it was specified: + if (QBenchmarkGlobalData::current->minimumTotal == -1) { + minimumTotalReached = true; + } else { + const qreal total = std::accumulate(results.begin(), results.end(), 0.0, addResult); + minimumTotalReached = (total >= QBenchmarkGlobalData::current->minimumTotal); + } } while (isBenchmark - && (++i < QBenchmarkGlobalData::current->adjustMedianIterationCount()) + && ((++i < QBenchmarkGlobalData::current->adjustMedianIterationCount()) || !minimumTotalReached) && !QTestResult::skipCurrentTest() && !QTestResult::currentTestFailed()); // If the test is a benchmark, finalize the result after all iterations have finished. |