/**************************************************************************** ** ** Copyright (C) 2011 Nokia Corporation and/or its subsidiary(-ies). ** All rights reserved. ** Contact: Nokia Corporation (qt-info@nokia.com) ** ** This file is part of the test suite of the Qt Toolkit. ** ** $QT_BEGIN_LICENSE:LGPL$ ** No Commercial Usage ** This file contains pre-release code and may not be distributed. ** You may use this file in accordance with the terms and conditions ** contained in the Technology Preview License Agreement accompanying ** this package. ** ** GNU Lesser General Public License Usage ** Alternatively, this file may be used under the terms of the GNU Lesser ** General Public License version 2.1 as published by the Free Software ** Foundation and appearing in the file LICENSE.LGPL included in the ** packaging of this file. Please review the following information to ** ensure the GNU Lesser General Public License version 2.1 requirements ** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. ** ** In addition, as a special exception, Nokia gives you certain additional ** rights. These rights are described in the Nokia Qt LGPL Exception ** version 1.1, included in the file LGPL_EXCEPTION.txt in this package. ** ** If you have questions regarding the use of this file, please contact ** Nokia at qt-info@nokia.com. ** ** ** ** ** ** ** ** ** $QT_END_LICENSE$ ** ****************************************************************************/ #include "testtypes.h" #include #include #include #include #include #include #include #ifdef Q_OS_SYMBIAN // In Symbian OS test data is located in applications private dir #define SRCDIR "." #endif // Conceptually, there are several different "holistic" areas to benchmark: // 1) Loading // - read file from disk // - parse/lex file // - handle nested imports // 2) Compilation // - create meta object templates etc // - compile to bytecode and cache it // 3) Instantiation // - running the bytecode to create an object tree, assign properties, etc // - and, importantly, to evaluate bindings for the first time (incl. js expressions) // 4) Dynamicism // - bindings evaluation // - signal handlers // // Aside from this, we also need to determine: // 1) JavaScript Metrics // - simple expressions // - complex expressions // - instantiation vs evaluation time // - imports and nested imports // 2) Context-switch costs // - how expensive is it to call a cpp function from QML // - how expensive is it to call a js function from cpp via QML // - how expensive is it to pass around objects between them // 3) Complete creation time. // - loading + compilation + instantiation (for "application startup time" metric) // // In some cases, we want to include "initialization costs"; // i.e., we need to tell the engine not to cache type data resulting // in compilation between rounds, and we need to tell the engine not // to cache whatever it caches between instantiations of components. // The reason for this is that it is often the "first start of application" // performance which we're attempting to benchmark. // define some custom types we use in test data functions. typedef QList PropertyNameList; Q_DECLARE_METATYPE(PropertyNameList); typedef QList PropertyValueList; Q_DECLARE_METATYPE(PropertyValueList); class tst_holistic : public QObject { Q_OBJECT public: tst_holistic(); private slots: void initTestCase() { registerTypes(); qRegisterMetaType("PropertyNameList"); qRegisterMetaType("PropertyValueList"); } void compilation_data(); void compilation(); void instantiation_data() { compilation_data(); } void instantiation(); void creation_data() { compilation_data(); } void creation(); void dynamicity_data(); void dynamicity(); void cppToJsDirect_data(); void cppToJsDirect(); void cppToJsIndirect(); void typeResolution_data(); void typeResolution(); private: QDeclarativeEngine engine; }; tst_holistic::tst_holistic() { } inline QUrl TEST_FILE(const QString &filename) { return QUrl::fromLocalFile(QLatin1String(SRCDIR) + QLatin1String("/data/") + filename); } void tst_holistic::compilation_data() { QTest::addColumn("files"); QTest::addColumn("repetitions"); QStringList f; // Benchmarks: a single, small component once with no caching. f << QString(SRCDIR + QLatin1String("/data/smallTargets/SmallOne.qml")); QTest::newRow("single small component") << f << 1; // Benchmarks: a single, small component ten times with caching. QTest::newRow("single small component cached") << f << 10; f.clear(); // Benchmarks: a single, large component once with no caching. f << QString(SRCDIR + QLatin1String("/data/largeTargets/mousearea-example.qml")); QTest::newRow("single large component") << f << 1; // Benchmarks: a single, large component ten times with caching. QTest::newRow("single large component cached") << f << 10; f.clear(); // Benchmarks: 4 small components once each with no caching f << QString(SRCDIR + QLatin1String("/data/smallTargets/SmallOne.qml")); f << QString(SRCDIR + QLatin1String("/data/smallTargets/SmallTwo.qml")); f << QString(SRCDIR + QLatin1String("/data/smallTargets/SmallThree.qml")); f << QString(SRCDIR + QLatin1String("/data/smallTargets/SmallFour.qml")); QTest::newRow("multiple small components") << f << 1; // Benchmarks: 4 small components ten times each with caching QTest::newRow("multiple small components cached") << f << 10; f.clear(); // Benchmarks: 3 large components once each with no caching. f << QString(SRCDIR + QLatin1String("/data/largeTargets/mousearea-example.qml")); f << QString(SRCDIR + QLatin1String("/data/largeTargets/gridview-example.qml")); f << QString(SRCDIR + QLatin1String("/data/largeTargets/layoutdirection.qml")); QTest::newRow("multiple large components") << f << 1; // Benchmarks: 3 large components ten times each with caching. QTest::newRow("multiple large components cached") << f << 10; f.clear(); // Benchmarks: single small component which imports a single small js file, no caching f << QString(SRCDIR + QLatin1String("/data/jsImports/Sssi.qml")); QTest::newRow("single small js import") << f << 1; // Benchmarks: single small component which imports a single small js file, 10 reps, with caching QTest::newRow("single small js import, cached") << f << 10; f.clear(); // Benchmarks: single small component which imports multiple small js files (no deep nesting), no caching f << QString(SRCDIR + QLatin1String("/data/jsImports/Msbsi.qml")); QTest::newRow("multiple small js imports, shallow") << f << 1; // Benchmarks: single small component which imports multiple small js files (no deep nesting), 10 reps, with caching QTest::newRow("multiple small js imports, shallow, cached") << f << 10; f.clear(); // Benchmarks: single small component which imports multiple small js files (with deep nesting), no caching f << QString(SRCDIR + QLatin1String("/data/jsImports/Msdsi.qml")); QTest::newRow("multiple small js imports, deeply nested") << f << 1; // Benchmarks: single small component which imports multiple small js files (with deep nesting), 10 reps, with caching QTest::newRow("multiple small js imports, deeply nested, cached") << f << 10; f.clear(); // Benchmarks: single small component which imports multiple small js files (nested and unnested), no caching f << QString(SRCDIR + QLatin1String("/data/jsImports/Mssi.qml")); QTest::newRow("muliple small js imports, both") << f << 1; // Benchmarks: single small component which imports multiple small js files (nested and unnested), 10 reps, with caching QTest::newRow("muliple small js imports, both, cached") << f << 10; f.clear(); // Benchmarks: single small component which imports a single large js file, no caching f << QString(SRCDIR + QLatin1String("/data/jsImports/Slsi.qml")); QTest::newRow("single large js import") << f << 1; // Benchmarks: single small component which imports a single large js file, 10 reps, with caching QTest::newRow("single large js import, cached") << f << 10; f.clear(); // Benchmarks: single small component which imports multiple large js files (no deep nesting), no caching f << QString(SRCDIR + QLatin1String("/data/jsImports/Mlbsi.qml")); QTest::newRow("multiple large js imports, shallow") << f << 1; // Benchmarks: single small component which imports multiple large js files (no deep nesting), 10 reps, with caching QTest::newRow("multiple large js imports, shallow, cached") << f << 10; f.clear(); // Benchmarks: single small component which imports multiple large js files (with deep nesting), no caching f << QString(SRCDIR + QLatin1String("/data/jsImports/Mldsi.qml")); QTest::newRow("multiple large js imports, deeply nested") << f << 1; // Benchmarks: single small component which imports multiple large js files (with deep nesting), 10 reps, with caching QTest::newRow("multiple large js imports, deeply nested, cached") << f << 10; f.clear(); // Benchmarks: single small component which imports multiple large js files (nested and unnested), no caching f << QString(SRCDIR + QLatin1String("/data/jsImports/Mlsi.qml")); QTest::newRow("multiple large js imports, both") << f << 1; // Benchmarks: single small component which imports multiple large js files (nested and unnested), 10 reps, with caching QTest::newRow("multiple large js imports, both, cached") << f << 10; f.clear(); // Benchmarks: single small component which imports multiple js files which all import a .pragma library js file, no caching f << QString(SRCDIR + QLatin1String("/data/jsImports/PragmaBm.qml")); QTest::newRow(".pragma library js import") << f << 1; // Benchmarks: single small component which imports multiple js files which all import a .pragma library js file, 10 reps, with caching QTest::newRow(".pragma library js import, cached") << f << 10; f.clear(); // Benchmarks: single small component which imports a js file which imports a QML module, no caching f << QString(SRCDIR + QLatin1String("/data/jsImports/ModuleBm.qml")); QTest::newRow("import js with QML import") << f << 1; // Benchmarks: single small component which imports a js file which imports a QML module, 10 reps, with caching QTest::newRow("import js with QML import, cached") << f << 10; f.clear(); // Benchmarks: single small component which imports multiple js files which all import a .pragma library js file and a QML module, no caching f << QString(SRCDIR + QLatin1String("/data/jsImports/PragmaModuleBm.qml")); QTest::newRow("import js with QML import and .pragma library") << f << 1; // Benchmarks: single small component which imports multiple js files which all import a .pragma library js file and a QML module, 10 reps, with caching QTest::newRow("import js with QML import and .pragma library, cached") << f << 10; f.clear(); } void tst_holistic::compilation() { // This function benchmarks the cost of loading and compiling specified QML files. // If "repetitions" is non-zero, each file from "files" will be compiled "repetitions" // times, without clearing the engine's component cache between compilations. QFETCH(QStringList, files); QFETCH(int, repetitions); Q_ASSERT(files.size() > 0); Q_ASSERT(repetitions > 0); QBENCHMARK { engine.clearComponentCache(); for (int i = 0; i < repetitions; ++i) { for (int j = 0; j < files.size(); ++j) { QDeclarativeComponent c(&engine, QUrl::fromLocalFile(files.at(j))); } } } } void tst_holistic::instantiation() { // This function benchmarks the cost of instantiating components compiled from specified QML files. // If "repetitions" is non-zero, each component compiled from "files" will be instantiated "repetitions" // times, without clearing the component's instantiation cache between instantiations. QFETCH(QStringList, files); QFETCH(int, repetitions); Q_ASSERT(files.size() > 0); Q_ASSERT(repetitions > 0); QList components; for (int i = 0; i < files.size(); ++i) { QDeclarativeComponent *c = new QDeclarativeComponent(&engine, QUrl::fromLocalFile(files.at(i))); components.append(c); } QBENCHMARK { // XXX TODO: clear each component's instantiation cache for (int i = 0; i < repetitions; ++i) { for (int j = 0; j < components.size(); ++j) { QObject *obj = components.at(j)->create(); delete obj; } } } // cleanup for (int i = 0; i < components.size(); ++i) { delete components.at(i); } } void tst_holistic::creation() { // This function benchmarks the cost of loading, compiling and instantiating specified QML files. // If "repetitions" is non-zero, each file from "files" will be created "repetitions" // times, without clearing the engine's component cache between component creation. QFETCH(QStringList, files); QFETCH(int, repetitions); Q_ASSERT(files.size() > 0); Q_ASSERT(repetitions > 0); QBENCHMARK { engine.clearComponentCache(); for (int i = 0; i < repetitions; ++i) { for (int j = 0; j < files.size(); ++j) { QDeclarativeComponent c(&engine, QUrl::fromLocalFile(files.at(j))); QObject *obj = c.create(); delete obj; } } } } void tst_holistic::dynamicity_data() { QTest::addColumn("file"); QTest::addColumn("writeProperty"); QTest::addColumn("writeValueOne"); QTest::addColumn("writeValueTwo"); QTest::addColumn("readProperty"); QString f; // Benchmarks: single simple property binding f = QString(SRCDIR + QLatin1String("/data/dynamicTargets/DynamicOne.qml")); QTest::newRow("single simple property binding") << f << QString(QLatin1String("dynamicWidth")) << QVariant(300) << QVariant(500) << QString(QLatin1String("height")); // Benchmarks: multiple simple property bindings in one component f = QString(SRCDIR + QLatin1String("/data/dynamicTargets/DynamicTwo.qml")); QTest::newRow("multiple simple property bindings") << f << QString(QLatin1String("dynamicWidth")) << QVariant(300) << QVariant(500) << QString(QLatin1String("dynamicWidth")); // Benchmarks: single simple property binding plus onPropertyChanged slot f = QString(SRCDIR + QLatin1String("/data/dynamicTargets/DynamicThree.qml")); QTest::newRow("single simple plus slot") << f << QString(QLatin1String("dynamicWidth")) << QVariant(300) << QVariant(500) << QString(QLatin1String("dynamicWidth")); // Benchmarks: multiple simple property bindings plus multiple onPropertyChanged slots in one component f = QString(SRCDIR + QLatin1String("/data/dynamicTargets/DynamicFour.qml")); QTest::newRow("multiple simple plus slots") << f << QString(QLatin1String("dynamicWidth")) << QVariant(300) << QVariant(500) << QString(QLatin1String("dynamicHeight")); // Benchmarks: single simple js expression in a slot f = QString(SRCDIR + QLatin1String("/data/jsTargets/JsOne.qml")); QTest::newRow("single simple js expression slot") << f << QString(QLatin1String("dynamicWidth")) << QVariant(300) << QVariant(500) << QString(QLatin1String("dynamicWidth")); // Benchmarks: single complex js expression in a slot f = QString(SRCDIR + QLatin1String("/data/jsTargets/JsTwo.qml")); QTest::newRow("single complex js expression slot") << f << QString(QLatin1String("dynamicWidth")) << QVariant(300) << QVariant(500) << QString(QLatin1String("dynamicWidth")); // Benchmarks: simple property assignment and bindings update f = QString(SRCDIR + QLatin1String("/data/scopeSwitching/CppToQml.qml")); QTest::newRow("single simple property binding") << f << QString(QLatin1String("arbitrary")) << QVariant(36) << QVariant(35) << QString(QLatin1String("arbitrary")); } void tst_holistic::dynamicity() { // This function benchmarks the cost of "continued operation" - signal invocation, // updating bindings, etc. Note that we take two different writeValues in order // to force updates to occur, and we read to force lazy evaluation to occur. QFETCH(QString, file); QFETCH(QString, writeProperty); QFETCH(QVariant, writeValueOne); QFETCH(QVariant, writeValueTwo); QFETCH(QString, readProperty); QDeclarativeComponent c(&engine, file); QObject *obj = c.create(); QVariant readValue; QVariant writeValue; bool usedFirst = false; QBENCHMARK { if (usedFirst) { writeValue = writeValueTwo; usedFirst = false; } else { writeValue = writeValueOne; usedFirst = true; } obj->setProperty(writeProperty.toAscii().constData(), writeValue); readValue = obj->property(readProperty.toAscii().constData()); } delete obj; } void tst_holistic::cppToJsDirect_data() { QTest::addColumn("file"); QTest::addColumn("methodName"); QString f; // Benchmarks: cost of calling a js function from cpp directly f = QString(SRCDIR + QLatin1String("/data/scopeSwitching/CppToJs.qml")); QTest::newRow("cpp-to-js") << f << QString(QLatin1String("callJsFunction")); // Benchmarks: cost of calling js function which calls cpp function: // const CPP function with no return value and no arguments. f = QString(SRCDIR + QLatin1String("/data/scopeSwitching/JsToCppOne.qml")); QTest::newRow("cpp-to-js-to-cpp: no retn, no args") << f << QString(QLatin1String("callCppFunction")); // Benchmarks: cost of calling js function which calls cpp function: // nonconst CPP function with no return value and no arguments. f = QString(SRCDIR + QLatin1String("/data/scopeSwitching/JsToCppTwo.qml")); QTest::newRow("cpp-to-js-to-cpp: nonconst, no retn, no args") << f << QString(QLatin1String("callCppFunction")); // Benchmarks: cost of calling js function which calls cpp function: // const CPP function with no return value and a single integer argument. f = QString(SRCDIR + QLatin1String("/data/scopeSwitching/JsToCppThree.qml")); QTest::newRow("cpp-to-js-to-cpp: const, no retn, int arg") << f << QString(QLatin1String("callCppFunction")); // Benchmarks: cost of calling js function which calls cpp function: // nonconst CPP function with no return value and a single integer argument. f = QString(SRCDIR + QLatin1String("/data/scopeSwitching/JsToCppFour.qml")); QTest::newRow("cpp-to-js-to-cpp: nonconst, no retn, int arg") << f << QString(QLatin1String("callCppFunction")); // Benchmarks: cost of calling js function which calls cpp function: // const CPP function with an integer return value and no arguments. f = QString(SRCDIR + QLatin1String("/data/scopeSwitching/JsToCppFive.qml")); QTest::newRow("cpp-to-js-to-cpp: const, int retn, no args") << f << QString(QLatin1String("callCppFunction")); // Benchmarks: cost of calling js function which calls cpp function: // nonconst CPP function with an integer return value and no arguments. f = QString(SRCDIR + QLatin1String("/data/scopeSwitching/JsToCppSix.qml")); QTest::newRow("cpp-to-js-to-cpp: nonconst, int retn, no args") << f << QString(QLatin1String("callCppFunction")); // Benchmarks: cost of calling js function which calls cpp function: // const CPP function with an integer return value and a single integer argument. f = QString(SRCDIR + QLatin1String("/data/scopeSwitching/JsToCppSeven.qml")); QTest::newRow("cpp-to-js-to-cpp: const, int retn, int arg") << f << QString(QLatin1String("callCppFunction")); // Benchmarks: cost of calling js function which calls cpp function: // nonconst CPP function with an integer return value and a single integer argument. f = QString(SRCDIR + QLatin1String("/data/scopeSwitching/JsToCppEight.qml")); QTest::newRow("cpp-to-js-to-cpp: nonconst, int retn, int arg") << f << QString(QLatin1String("callCppFunction")); // Benchmarks: cost of calling js function which calls cpp function: // const CPP function with a variant return value and multiple integer arguments. f = QString(SRCDIR + QLatin1String("/data/scopeSwitching/JsToCppNine.qml")); QTest::newRow("cpp-to-js-to-cpp: const, variant retn, int args") << f << QString(QLatin1String("callCppFunction")); // Benchmarks: cost of calling js function which calls cpp function: // nonconst CPP function with a variant return value and multiple integer arguments. f = QString(SRCDIR + QLatin1String("/data/scopeSwitching/JsToCppTen.qml")); QTest::newRow("cpp-to-js-to-cpp: nonconst, variant retn, int args") << f << QString(QLatin1String("callCppFunction")); // Benchmarks: cost of calling js function which calls cpp function: // nonconst CPP function with a variant return value and multiple integer arguments. f = QString(SRCDIR + QLatin1String("/data/scopeSwitching/JsToCppEleven.qml")); QTest::newRow("cpp-to-js-to-cpp: nonconst, variant retn, variant + int args") << f << QString(QLatin1String("callCppFunction")); // Benchmarks: calling js function which copies scarce resources by calling back into cpp scope f = QString(SRCDIR + QLatin1String("/data/scopeSwitching/ScarceOne.qml")); QTest::newRow("cpp-to-js-to-coo: copy scarce resources") << f << QString(QLatin1String("copyScarceResources")); } void tst_holistic::cppToJsDirect() { // This function benchmarks the cost of calling from CPP scope to JS scope // (and possibly vice versa, if the invoked js method then calls to cpp). QFETCH(QString, file); QFETCH(QString, methodName); QDeclarativeComponent c(&engine, file); QObject *obj = c.create(); QBENCHMARK { QMetaObject::invokeMethod(obj, methodName.toLatin1().constData()); } delete obj; } void tst_holistic::cppToJsIndirect() { // This function benchmarks the cost of binding scarce resources // to properties of a QML component. The engine should automatically release such // resources when they are no longer used. // The benchmark deliberately causes change signals to be emitted (and // modifies the scarce resources) so that the properties are updated. QDeclarativeComponent c(&engine, QString(SRCDIR + QLatin1String("/data/scopeSwitching/ScarceTwo.qml"))); QObject *obj = c.create(); ScarceResourceProvider *srp = 0; srp = qobject_cast(QDeclarativeProperty::read(obj, "a").value()); QBENCHMARK { srp->changeResources(); // will cause small+large scarce resources changed signals to be emitted. } delete obj; } void tst_holistic::typeResolution_data() { QTest::addColumn("file"); QTest::addColumn("propertyNameOne"); QTest::addColumn("propertyValueOne"); QTest::addColumn("propertyNameTwo"); QTest::addColumn("propertyValueTwo"); QTest::addColumn("repetitions"); QString f; PropertyNameList pn1; PropertyValueList pv1; PropertyNameList pn2; PropertyValueList pv2; // Benchmarks: resolving nested ids and types, no caching f = QString(SRCDIR + QLatin1String("/data/resolutionTargets/ResolveOne.qml")); pn1 << QString(QLatin1String("baseWidth")) << QString(QLatin1String("baseHeight")) << QString(QLatin1String("baseColor")); pv1 << QVariant(401) << QVariant(402) << QVariant(QString(QLatin1String("brown"))); pn2 << QString(QLatin1String("baseWidth")) << QString(QLatin1String("baseHeight")) << QString(QLatin1String("baseColor")); pv2 << QVariant(403) << QVariant(404) << QVariant(QString(QLatin1String("orange"))); QTest::newRow("nested id resolution") << f << pn1 << pv1 << pn2 << pv2 << 1; // Benchmarks: resolving nested ids and types, 10 reps with caching QTest::newRow("nested id resolution, cached") << f << pn1 << pv1 << pn2 << pv2 << 10; pn1.clear(); pn2.clear(); pv1.clear(); pv2.clear(); } void tst_holistic::typeResolution() { // This function benchmarks the cost of "continued operation" (signal invocation, // updating bindings, etc) where the component has lots of nested items with // lots of resolving required. Note that we take two different writeValues in order // to force updates to occur. QFETCH(QString, file); QFETCH(PropertyNameList, propertyNameOne); QFETCH(PropertyValueList, propertyValueOne); QFETCH(PropertyNameList, propertyNameTwo); QFETCH(PropertyValueList, propertyValueTwo); QFETCH(int, repetitions); Q_ASSERT(propertyNameOne.size() == propertyValueOne.size()); Q_ASSERT(propertyNameTwo.size() == propertyValueTwo.size()); Q_ASSERT(repetitions > 0); QDeclarativeComponent c(&engine, file); QObject *obj = c.create(); PropertyNameList writeProperty; PropertyValueList writeValue; bool usedFirst = false; QBENCHMARK { for (int i = 0; i < repetitions; ++i) { if (usedFirst) { writeProperty = propertyNameOne; writeValue = propertyValueOne; usedFirst = false; } else { writeProperty = propertyNameTwo; writeValue = propertyValueTwo; usedFirst = true; } for (int j = 0; j < writeProperty.size(); ++j) { obj->setProperty(writeProperty.at(j).toAscii().constData(), writeValue.at(j)); } } } delete obj; } QTEST_MAIN(tst_holistic) #include "tst_holistic.moc"