From 200a8ac92afa0d8c5ff54c929a345dd3e05d3dd7 Mon Sep 17 00:00:00 2001 From: Rhys Weatherley Date: Thu, 9 Dec 2010 13:51:29 +1000 Subject: Add benchmark support to the QML test framework Test functions called "benchmark_foo()" will be run as benchmarks inside an implicit QBENCHMARK {} block. --- doc/testcases.txt | 29 +++++ src/imports/testlib/TestCase.qml | 45 +++++++- src/quicktestlib/quicktestresult.cpp | 119 +++++++++++++++++++++ src/quicktestlib/quicktestresult_p.h | 20 +++- tests/qmlauto/createbenchmark/item.qml | 75 +++++++++++++ .../createbenchmark/tst_createbenchmark.qml | 55 ++++++++++ 6 files changed, 339 insertions(+), 4 deletions(-) create mode 100644 tests/qmlauto/createbenchmark/item.qml create mode 100644 tests/qmlauto/createbenchmark/tst_createbenchmark.qml diff --git a/doc/testcases.txt b/doc/testcases.txt index fe655dd..3ba44bc 100644 --- a/doc/testcases.txt +++ b/doc/testcases.txt @@ -276,3 +276,32 @@ Button { } } ---------------------- + +Benchmarks +========== + +If the test function name starts with "benchmark_", then it will be +run multiple times with the Qt benchmark framework, with a average +timing value reported for the runs. This is equivalent to using the +QBENCHMARK macro in QTestLib. + +---------------------- +TestCase { + id: top + name: "CreateBenchmark" + + function benchmark_create_component() { + var component = Qt.createComponent("item.qml") + var obj = component.createObject(top) + obj.destroy() + component.destroy() + } +} +---------------------- + +RESULT : CreateBenchmark::benchmark_create_component: + 0.23 msecs per iteration (total: 60, iterations: 256) +PASS : CreateBenchmark::benchmark_create_component() + +To get the effect of the QBENCHMARK_ONCE macro, prefix the test +function name with "benchmark_once_". diff --git a/src/imports/testlib/TestCase.qml b/src/imports/testlib/TestCase.qml index 37a6e66..9e57b2e 100644 --- a/src/imports/testlib/TestCase.qml +++ b/src/imports/testlib/TestCase.qml @@ -233,6 +233,39 @@ Item { results.functionType = TestResult.NoWhere } + function runBenchmarkFunction(prop, arg) { + results.startMeasurement() + do { + results.beginDataRun() + do { + // Run the initialization function. + results.functionType = TestResult.InitFunc + runInternal("init") + if (results.skipped) + break + + // Execute the benchmark function. + results.functionType = TestResult.Func + if (prop.indexOf("benchmark_once_") != 0) + results.startBenchmark(TestResult.RepeatUntilValidMeasurement, results.dataTag) + else + results.startBenchmark(TestResult.RunOnce, results.dataTag) + while (!results.isBenchmarkDone()) { + if (!runInternal(prop, arg)) + break + results.nextBenchmark() + } + results.stopBenchmark() + + // Run the cleanup function. + results.functionType = TestResult.CleanupFunc + runInternal("cleanup") + results.functionType = TestResult.NoWhere + } while (!results.measurementAccepted()) + results.endDataRun() + } while (results.needsMoreMeasurements()) + } + function run() { if (TestLogger.log_start_test()) { results.reset() @@ -255,7 +288,7 @@ Item { var testList = [] if (runTests) { for (var prop in testCase) { - if (prop.indexOf("test_") != 0) + if (prop.indexOf("test_") != 0 && prop.indexOf("benchmark_") != 0) continue var tail = prop.lastIndexOf("_data"); if (tail != -1 && tail == (prop.length - 5)) @@ -267,6 +300,7 @@ Item { for (var index in testList) { var prop = testList[index] var datafunc = prop + "_data" + var isBenchmark = (prop.indexOf("benchmark_") == 0) results.functionName = prop if (datafunc in testCase) { results.functionType = TestResult.DataFunc @@ -280,15 +314,20 @@ Item { if (!row.tag) row.tag = "row " + index // Must have something results.dataTag = row.tag - runFunction(prop, row) + if (isBenchmark) + runBenchmarkFunction(prop, row) + else + runFunction(prop, row) results.dataTag = "" } if (!haveData) results.warn("no data supplied for " + prop + "() by " + datafunc + "()") results.clearTestTable() } + } else if (isBenchmark) { + runBenchmarkFunction(prop, null, isBenchmark) } else { - runFunction(prop) + runFunction(prop, null, isBenchmark) } results.finishTestFunction() results.skipped = false diff --git a/src/quicktestlib/quicktestresult.cpp b/src/quicktestlib/quicktestresult.cpp index b54f64f..bb4ba64 100644 --- a/src/quicktestlib/quicktestresult.cpp +++ b/src/quicktestlib/quicktestresult.cpp @@ -45,26 +45,35 @@ #include "qtestresult_p.h" #include "qtesttable_p.h" #include "qtestlog_p.h" +#include "qbenchmark.h" +#include "qbenchmark_p.h" #include #include #include #include +#include QT_BEGIN_NAMESPACE static const char *globalProgramName = 0; static bool loggingStarted = false; +static QBenchmarkGlobalData globalBenchmarkData; class QuickTestResultPrivate { public: QuickTestResultPrivate() : table(0) + , benchmarkIter(0) + , benchmarkData(0) + , iterCount(0) { } ~QuickTestResultPrivate() { delete table; + delete benchmarkIter; + delete benchmarkData; } QByteArray intern(const QString &str); @@ -74,6 +83,10 @@ public: QString functionName; QSet internedStrings; QTestTable *table; + QTest::QBenchmarkIterationController *benchmarkIter; + QBenchmarkTestMethodData *benchmarkData; + int iterCount; + QList results; }; QByteArray QuickTestResultPrivate::intern(const QString &str) @@ -104,6 +117,8 @@ void QuickTestResultPrivate::updateTestObjectName() QuickTestResult::QuickTestResult(QObject *parent) : QObject(parent), d_ptr(new QuickTestResultPrivate) { + if (!QBenchmarkGlobalData::current) + QBenchmarkGlobalData::current = &globalBenchmarkData; } QuickTestResult::~QuickTestResult() @@ -166,6 +181,7 @@ void QuickTestResult::setFunctionName(const QString &name) } else { QTestResult::setCurrentTestFunction(0); } + d->functionName = name; emit functionNameChanged(); } @@ -450,12 +466,115 @@ void QuickTestResult::sleep(int ms) QTest::qSleep(ms); } +void QuickTestResult::startMeasurement() +{ + Q_D(QuickTestResult); + delete d->benchmarkData; + d->benchmarkData = new QBenchmarkTestMethodData(); + QBenchmarkTestMethodData::current = d->benchmarkData; + d->iterCount = (QBenchmarkGlobalData::current->measurer->needsWarmupIteration()) ? -1 : 0; + d->results.clear(); +} + +void QuickTestResult::beginDataRun() +{ + QBenchmarkTestMethodData::current->beginDataRun(); +} + +void QuickTestResult::endDataRun() +{ + Q_D(QuickTestResult); + QBenchmarkTestMethodData::current->endDataRun(); + if (d->iterCount > -1) // iteration -1 is the warmup iteration. + d->results.append(QBenchmarkTestMethodData::current->result); + + if (QBenchmarkGlobalData::current->verboseOutput) { + if (d->iterCount == -1) { + qDebug() << "warmup stage result :" << QBenchmarkTestMethodData::current->result.value; + } else { + qDebug() << "accumulation stage result:" << QBenchmarkTestMethodData::current->result.value; + } + } +} + +bool QuickTestResult::measurementAccepted() +{ + return QBenchmarkTestMethodData::current->resultsAccepted(); +} + +static QBenchmarkResult qMedian(const QList &container) +{ + const int count = container.count(); + if (count == 0) + return QBenchmarkResult(); + + if (count == 1) + return container.at(0); + + QList containerCopy = container; + qSort(containerCopy); + + const int middle = count / 2; + + // ### handle even-sized containers here by doing an aritmetic mean of the two middle items. + return containerCopy.at(middle); +} + +bool QuickTestResult::needsMoreMeasurements() +{ + Q_D(QuickTestResult); + ++(d->iterCount); + if (d->iterCount < QBenchmarkGlobalData::current->adjustMedianIterationCount()) + return true; + if (QBenchmarkTestMethodData::current->resultsAccepted()) + QTestLog::addBenchmarkResult(qMedian(d->results)); + return false; +} + +void QuickTestResult::startBenchmark(RunMode runMode, const QString &tag) +{ + QBenchmarkTestMethodData::current->result = QBenchmarkResult(); + QBenchmarkTestMethodData::current->resultAccepted = false; + QBenchmarkGlobalData::current->context.tag = tag; + QBenchmarkGlobalData::current->context.slotName = functionName(); + + Q_D(QuickTestResult); + delete d->benchmarkIter; + d->benchmarkIter = new QTest::QBenchmarkIterationController + (QTest::QBenchmarkIterationController::RunMode(runMode)); +} + +bool QuickTestResult::isBenchmarkDone() const +{ + Q_D(const QuickTestResult); + if (d->benchmarkIter) + return d->benchmarkIter->isDone(); + else + return true; +} + +void QuickTestResult::nextBenchmark() +{ + Q_D(QuickTestResult); + if (d->benchmarkIter) + d->benchmarkIter->next(); +} + +void QuickTestResult::stopBenchmark() +{ + Q_D(QuickTestResult); + delete d->benchmarkIter; + d->benchmarkIter = 0; +} + namespace QTest { void qtest_qParseArgs(int argc, char *argv[]); }; void QuickTestResult::parseArgs(int argc, char *argv[]) { + if (!QBenchmarkGlobalData::current) + QBenchmarkGlobalData::current = &globalBenchmarkData; QTest::qtest_qParseArgs(argc, argv); } diff --git a/src/quicktestlib/quicktestresult_p.h b/src/quicktestlib/quicktestresult_p.h index 4a283ea..a3c67be 100644 --- a/src/quicktestlib/quicktestresult_p.h +++ b/src/quicktestlib/quicktestresult_p.h @@ -54,7 +54,7 @@ class QuickTestResultPrivate; class Q_QUICK_TEST_EXPORT QuickTestResult : public QObject { Q_OBJECT - Q_ENUMS(FunctionType) + Q_ENUMS(FunctionType RunMode) Q_PROPERTY(QString testCaseName READ testCaseName WRITE setTestCaseName NOTIFY testCaseNameChanged) Q_PROPERTY(QString functionName READ functionName WRITE setFunctionName NOTIFY functionNameChanged) Q_PROPERTY(FunctionType functionType READ functionType WRITE setFunctionType NOTIFY functionTypeChanged) @@ -79,6 +79,13 @@ public: CleanupFunc = 4 }; + // Values must match QBenchmarkIterationController::RunMode. + enum RunMode + { + RepeatUntilValidMeasurement, + RunOnce + }; + QString testCaseName() const; void setTestCaseName(const QString &name); @@ -131,6 +138,17 @@ public Q_SLOTS: void wait(int ms); void sleep(int ms); + void startMeasurement(); + void beginDataRun(); + void endDataRun(); + bool measurementAccepted(); + bool needsMoreMeasurements(); + + void startBenchmark(RunMode runMode, const QString &tag); + bool isBenchmarkDone() const; + void nextBenchmark(); + void stopBenchmark(); + public: // Helper functions for the C++ main() shell. static void parseArgs(int argc, char *argv[]); diff --git a/tests/qmlauto/createbenchmark/item.qml b/tests/qmlauto/createbenchmark/item.qml new file mode 100644 index 0000000..a6223d0 --- /dev/null +++ b/tests/qmlauto/createbenchmark/item.qml @@ -0,0 +1,75 @@ +/**************************************************************************** +** +** Copyright (C) 2010 Nokia Corporation and/or its subsidiary(-ies). +** All rights reserved. +** Contact: Nokia Corporation (qt-info@nokia.com) +** +** This file is part of the QtDeclarative module of the Qt Toolkit. +** +** $QT_BEGIN_LICENSE:LGPL$ +** No Commercial Usage +** This file contains pre-release code and may not be distributed. +** You may use this file in accordance with the terms and conditions +** contained in the Technology Preview License Agreement accompanying +** this package. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 2.1 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 2.1 requirements +** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. +** +** In addition, as a special exception, Nokia gives you certain additional +** rights. These rights are described in the Nokia Qt LGPL Exception +** version 1.1, included in the file LGPL_EXCEPTION.txt in this package. +** +** If you have questions regarding the use of this file, please contact +** Nokia at qt-info@nokia.com. +** +** +** +** +** +** +** +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +import QtQuick 1.0 + +Item { + Item {} + Item {} + Item {} + Item {} + Item {} + Item {} + Item {} + Item {} + Item {} + Item {} + Item {} + Item {} + Item {} + Item {} + Item {} + Item {} + Item {} + Item {} + Item {} + Item {} + Item {} + Item {} + Item {} + Item {} + Item {} + Item {} + Item {} + Item {} + Item {} + Item {} +} diff --git a/tests/qmlauto/createbenchmark/tst_createbenchmark.qml b/tests/qmlauto/createbenchmark/tst_createbenchmark.qml new file mode 100644 index 0000000..bfc8ff2 --- /dev/null +++ b/tests/qmlauto/createbenchmark/tst_createbenchmark.qml @@ -0,0 +1,55 @@ +/**************************************************************************** +** +** Copyright (C) 2010 Nokia Corporation and/or its subsidiary(-ies). +** All rights reserved. +** Contact: Nokia Corporation (qt-info@nokia.com) +** +** This file is part of the test suite of the Qt Toolkit. +** +** $QT_BEGIN_LICENSE:LGPL$ +** No Commercial Usage +** This file contains pre-release code and may not be distributed. +** You may use this file in accordance with the terms and conditions +** contained in the Technology Preview License Agreement accompanying +** this package. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 2.1 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 2.1 requirements +** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. +** +** In addition, as a special exception, Nokia gives you certain additional +** rights. These rights are described in the Nokia Qt LGPL Exception +** version 1.1, included in the file LGPL_EXCEPTION.txt in this package. +** +** If you have questions regarding the use of this file, please contact +** Nokia at qt-info@nokia.com. +** +** +** +** +** +** +** +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +import QtQuick 1.0 +import QtQuickTest 1.0 + +TestCase { + id: top + name: "CreateBenchmark" + + function benchmark_create_component() { + var component = Qt.createComponent("item.qml") + var obj = component.createObject(top) + obj.destroy() + component.destroy() + } +} -- cgit v1.2.3