diff options
Diffstat (limited to 'src/testlib')
-rw-r--r-- | src/testlib/qbenchmark.cpp | 22 | ||||
-rw-r--r-- | src/testlib/qbenchmark_p.h | 13 | ||||
-rw-r--r-- | src/testlib/qbenchmarkevent.cpp | 4 | ||||
-rw-r--r-- | src/testlib/qbenchmarkevent_p.h | 2 | ||||
-rw-r--r-- | src/testlib/qbenchmarkmeasurement.cpp | 8 | ||||
-rw-r--r-- | src/testlib/qbenchmarkmeasurement_p.h | 3 | ||||
-rw-r--r-- | src/testlib/qbenchmarkperfevents.cpp | 93 | ||||
-rw-r--r-- | src/testlib/qbenchmarkperfevents_p.h | 7 | ||||
-rw-r--r-- | src/testlib/qbenchmarktimemeasurers_p.h | 4 | ||||
-rw-r--r-- | src/testlib/qbenchmarkvalgrind.cpp | 4 | ||||
-rw-r--r-- | src/testlib/qbenchmarkvalgrind_p.h | 2 | ||||
-rw-r--r-- | src/testlib/qtestcase.cpp | 56 |
12 files changed, 132 insertions, 86 deletions
diff --git a/src/testlib/qbenchmark.cpp b/src/testlib/qbenchmark.cpp index 48f22aec34..4a8bd72ee9 100644 --- a/src/testlib/qbenchmark.cpp +++ b/src/testlib/qbenchmark.cpp @@ -96,10 +96,13 @@ int QBenchmarkTestMethodData::adjustIterationCount(int suggestion) return iterationCount; } -void QBenchmarkTestMethodData::setResult(QBenchmarkMeasurerBase::Measurement m, - bool setByMacro) +void QBenchmarkTestMethodData::setResults(const QList<QBenchmarkMeasurerBase::Measurement> &list, + bool setByMacro) { bool accepted = false; + QBenchmarkMeasurerBase::Measurement firstMeasurement = {}; + if (!list.isEmpty()) + firstMeasurement = list.constFirst(); // Always accept the result if the iteration count has been // specified on the command line with -iterations. @@ -114,9 +117,9 @@ void QBenchmarkTestMethodData::setResult(QBenchmarkMeasurerBase::Measurement m, // Test the result directly without calling the measurer if the minimum time // has been specified on the command line with -minimumvalue. else if (QBenchmarkGlobalData::current->walltimeMinimum != -1) - accepted = (m.value > QBenchmarkGlobalData::current->walltimeMinimum); + accepted = (firstMeasurement.value > QBenchmarkGlobalData::current->walltimeMinimum); else - accepted = QBenchmarkGlobalData::current->measurer->isMeasurementAccepted(m); + accepted = QBenchmarkGlobalData::current->measurer->isMeasurementAccepted(firstMeasurement); // Accept the result or double the number of iterations. if (accepted) @@ -124,8 +127,10 @@ void QBenchmarkTestMethodData::setResult(QBenchmarkMeasurerBase::Measurement m, else iterationCount *= 2; - this->result = QBenchmarkResult(QBenchmarkGlobalData::current->context, m, - iterationCount, setByMacro); + valid = true; + results.reserve(list.size()); + for (auto m : list) + results.emplaceBack(QBenchmarkGlobalData::current->context, m, iterationCount, setByMacro); } /*! @@ -157,8 +162,7 @@ QTest::QBenchmarkIterationController::QBenchmarkIterationController() */ QTest::QBenchmarkIterationController::~QBenchmarkIterationController() { - QBenchmarkMeasurerBase::Measurement measurement = QTest::endBenchmarkMeasurement(); - QBenchmarkTestMethodData::current->setResult(measurement); + QBenchmarkTestMethodData::current->setResults(QTest::endBenchmarkMeasurement()); } /*! \internal @@ -209,7 +213,7 @@ void QTest::beginBenchmarkMeasurement() /*! \internal */ -QBenchmarkMeasurerBase::Measurement QTest::endBenchmarkMeasurement() +QList<QBenchmarkMeasurerBase::Measurement> QTest::endBenchmarkMeasurement() { // the clock is ticking before the line below, don't add code here. return QBenchmarkGlobalData::current->measurer->stop(); diff --git a/src/testlib/qbenchmark_p.h b/src/testlib/qbenchmark_p.h index dca3de8441..09379aac0f 100644 --- a/src/testlib/qbenchmark_p.h +++ b/src/testlib/qbenchmark_p.h @@ -64,7 +64,6 @@ public: QBenchmarkMeasurerBase::Measurement measurement = { -1, QTest::FramesPerSecond }; int iterations = -1; bool setByMacro = true; - bool valid = false; QBenchmarkResult() = default; @@ -75,7 +74,6 @@ public: , measurement(m) , iterations(iterations) , setByMacro(setByMacro) - , valid(true) { } bool operator<(const QBenchmarkResult &other) const @@ -134,12 +132,15 @@ public: void beginDataRun(); void endDataRun(); - bool isBenchmark() const { return result.valid; } + bool isBenchmark() const { return valid; } bool resultsAccepted() const { return resultAccepted; } int adjustIterationCount(int suggestion); - void setResult(QBenchmarkMeasurerBase::Measurement m, bool setByMacro = true); + void setResults(const QList<QBenchmarkMeasurerBase::Measurement> &m, bool setByMacro = true); + void setResult(QBenchmarkMeasurerBase::Measurement m, bool setByMacro = true) + { setResults({ m }, setByMacro); } - QBenchmarkResult result; + QList<QBenchmarkResult> results; + bool valid = false; bool resultAccepted = false; bool runOnce = false; int iterationCount = -1; @@ -153,7 +154,7 @@ namespace QTest void setIterationCount(int count); void beginBenchmarkMeasurement(); - QBenchmarkMeasurerBase::Measurement endBenchmarkMeasurement(); + QList<QBenchmarkMeasurerBase::Measurement> endBenchmarkMeasurement(); } QT_END_NAMESPACE diff --git a/src/testlib/qbenchmarkevent.cpp b/src/testlib/qbenchmarkevent.cpp index dc7a592afc..5a895106a2 100644 --- a/src/testlib/qbenchmarkevent.cpp +++ b/src/testlib/qbenchmarkevent.cpp @@ -18,10 +18,10 @@ void QBenchmarkEvent::start() QAbstractEventDispatcher::instance()->installNativeEventFilter(this); } -QBenchmarkMeasurerBase::Measurement QBenchmarkEvent::stop() +QList<QBenchmarkMeasurerBase::Measurement> QBenchmarkEvent::stop() { QAbstractEventDispatcher::instance()->removeNativeEventFilter(this); - return { qreal(eventCounter), QTest::Events }; + return { { qreal(eventCounter), QTest::Events } }; } // It's very tempting to simply reject a measurement if 0 events diff --git a/src/testlib/qbenchmarkevent_p.h b/src/testlib/qbenchmarkevent_p.h index bdea221fae..7e997e7779 100644 --- a/src/testlib/qbenchmarkevent_p.h +++ b/src/testlib/qbenchmarkevent_p.h @@ -28,7 +28,7 @@ public: QBenchmarkEvent(); ~QBenchmarkEvent(); void start() override; - Measurement stop() override; + QList<Measurement> stop() override; bool isMeasurementAccepted(Measurement measurement) override; int adjustIterationCount(int suggestion) override; int adjustMedianCount(int suggestion) override; diff --git a/src/testlib/qbenchmarkmeasurement.cpp b/src/testlib/qbenchmarkmeasurement.cpp index 4bd1477640..99d5b1dd4f 100644 --- a/src/testlib/qbenchmarkmeasurement.cpp +++ b/src/testlib/qbenchmarkmeasurement.cpp @@ -16,9 +16,9 @@ void QBenchmarkTimeMeasurer::start() time.start(); } -QBenchmarkMeasurerBase::Measurement QBenchmarkTimeMeasurer::stop() +QList<QBenchmarkMeasurerBase::Measurement> QBenchmarkTimeMeasurer::stop() { - return { qreal(time.elapsed()), QTest::WalltimeMilliseconds }; + return { { qreal(time.elapsed()), QTest::WalltimeMilliseconds } }; } bool QBenchmarkTimeMeasurer::isMeasurementAccepted(Measurement measurement) @@ -48,10 +48,10 @@ void QBenchmarkTickMeasurer::start() startTicks = getticks(); } -QBenchmarkMeasurerBase::Measurement QBenchmarkTickMeasurer::stop() +QList<QBenchmarkMeasurerBase::Measurement> QBenchmarkTickMeasurer::stop() { CycleCounterTicks now = getticks(); - return { elapsed(now, startTicks), QTest::CPUTicks }; + return { { elapsed(now, startTicks), QTest::CPUTicks } }; } bool QBenchmarkTickMeasurer::isMeasurementAccepted(QBenchmarkMeasurerBase::Measurement) diff --git a/src/testlib/qbenchmarkmeasurement_p.h b/src/testlib/qbenchmarkmeasurement_p.h index 459a1d5c90..56b978228b 100644 --- a/src/testlib/qbenchmarkmeasurement_p.h +++ b/src/testlib/qbenchmarkmeasurement_p.h @@ -16,6 +16,7 @@ // #include <QtTest/qbenchmark.h> +#include <QtCore/qlist.h> #include <QtCore/private/qglobal_p.h> QT_BEGIN_NAMESPACE @@ -31,7 +32,7 @@ public: virtual ~QBenchmarkMeasurerBase() = default; virtual void init() {} virtual void start() = 0; - virtual Measurement stop() = 0; + virtual QList<Measurement> stop() = 0; virtual bool isMeasurementAccepted(Measurement m) = 0; virtual int adjustIterationCount(int suggestion) = 0; virtual int adjustMedianCount(int suggestion) = 0; diff --git a/src/testlib/qbenchmarkperfevents.cpp b/src/testlib/qbenchmarkperfevents.cpp index a6295d6e11..6808d465e2 100644 --- a/src/testlib/qbenchmarkperfevents.cpp +++ b/src/testlib/qbenchmarkperfevents.cpp @@ -47,7 +47,13 @@ QT_BEGIN_NAMESPACE +struct PerfEvent +{ + quint32 type; + quint64 config; +}; static perf_event_attr attr; +Q_GLOBAL_STATIC(QList<PerfEvent>, eventTypes); static void initPerf() { @@ -62,14 +68,20 @@ static void initPerf() attr.inherit_stat = true; // aggregate all the info from child processes attr.task = true; // trace fork/exits - // set a default performance counter: CPU cycles - attr.type = PERF_TYPE_HARDWARE; - attr.config = PERF_COUNT_HW_CPU_CYCLES; // default - done = true; } } +static QList<PerfEvent> defaultCounters() +{ + return { + { .type = PERF_TYPE_SOFTWARE, .config = PERF_COUNT_SW_TASK_CLOCK }, + { .type = PERF_TYPE_HARDWARE, .config = PERF_COUNT_HW_CPU_CYCLES }, + { .type = PERF_TYPE_HARDWARE, .config = PERF_COUNT_HW_INSTRUCTIONS }, + { .type = PERF_TYPE_HARDWARE, .config = PERF_COUNT_HW_BRANCH_INSTRUCTIONS }, + }; +} + // This class does not exist in the API so it's qdoc comment marker was removed. /* @@ -383,11 +395,11 @@ static const Events eventlist[] = { }; /* -- END GENERATED CODE -- */ -QTest::QBenchmarkMetric QBenchmarkPerfEventsMeasurer::metricForEvent(quint32 type, quint64 event_id) +static QTest::QBenchmarkMetric metricForEvent(PerfEvent counter) { const Events *ptr = eventlist; for ( ; ptr->type != PERF_TYPE_MAX; ++ptr) { - if (ptr->type == type && ptr->event_id == event_id) + if (ptr->type == counter.type && ptr->event_id == counter.config) return ptr->metric; } return QTest::Events; @@ -396,6 +408,7 @@ QTest::QBenchmarkMetric QBenchmarkPerfEventsMeasurer::metricForEvent(quint32 typ void QBenchmarkPerfEventsMeasurer::setCounter(const char *name) { initPerf(); + eventTypes->clear(); const char *colon = strchr(name, ':'); int n = colon ? colon - name : strlen(name); const Events *ptr = eventlist; @@ -409,8 +422,7 @@ void QBenchmarkPerfEventsMeasurer::setCounter(const char *name) } } - attr.type = ptr->type; - attr.config = ptr->event_id; + *eventTypes = { { ptr->type, ptr->event_id } }; // We used to support attributes, but our code was the opposite of what // perf(1) does, plus QBenchlib isn't exactly expected to be used to @@ -441,7 +453,8 @@ QBenchmarkPerfEventsMeasurer::QBenchmarkPerfEventsMeasurer() = default; QBenchmarkPerfEventsMeasurer::~QBenchmarkPerfEventsMeasurer() { - qt_safe_close(fd); + for (int fd : std::as_const(fds)) + qt_safe_close(fd); } void QBenchmarkPerfEventsMeasurer::init() @@ -451,34 +464,54 @@ void QBenchmarkPerfEventsMeasurer::init() void QBenchmarkPerfEventsMeasurer::start() { initPerf(); - if (fd == -1) { + QList<PerfEvent> &counters = *eventTypes; + if (counters.isEmpty()) + counters = defaultCounters(); + if (fds.isEmpty()) { pid_t pid = 0; // attach to the current process only int cpu = -1; // on any CPU int group_fd = -1; int flags = PERF_FLAG_FD_CLOEXEC; - fd = perf_event_open(&attr, pid, cpu, group_fd, flags); - if (fd == -1) { - // probably a paranoid kernel (/proc/sys/kernel/perf_event_paranoid) - attr.exclude_kernel = true; - attr.exclude_hv = true; - fd = perf_event_open(&attr, pid, cpu, group_fd, flags); - } - if (fd == -1) { - perror("QBenchmarkPerfEventsMeasurer::start: perf_event_open"); - exit(1); + + fds.reserve(counters.size()); + for (PerfEvent counter : std::as_const(counters)) { + attr.type = counter.type; + attr.config = counter.config; + int fd = perf_event_open(&attr, pid, cpu, group_fd, flags); + if (fd == -1) { + // probably a paranoid kernel (/proc/sys/kernel/perf_event_paranoid) + attr.exclude_kernel = true; + attr.exclude_hv = true; + fd = perf_event_open(&attr, pid, cpu, group_fd, flags); + } + if (fd == -1) { + perror("QBenchmarkPerfEventsMeasurer::start: perf_event_open"); + exit(1); + } + + fds.append(fd); } } - // enable the counter - ::ioctl(fd, PERF_EVENT_IOC_RESET); - ::ioctl(fd, PERF_EVENT_IOC_ENABLE); + // enable the counters + for (int fd : std::as_const(fds)) + ::ioctl(fd, PERF_EVENT_IOC_RESET); + for (int fd : std::as_const(fds)) + ::ioctl(fd, PERF_EVENT_IOC_ENABLE); } -QBenchmarkMeasurerBase::Measurement QBenchmarkPerfEventsMeasurer::stop() +QList<QBenchmarkMeasurerBase::Measurement> QBenchmarkPerfEventsMeasurer::stop() { - // disable the counter - ::ioctl(fd, PERF_EVENT_IOC_DISABLE); - return readValue(); + // disable the counters + for (int fd : std::as_const(fds)) + ::ioctl(fd, PERF_EVENT_IOC_DISABLE); + + const QList<PerfEvent> &counters = *eventTypes; + QList<Measurement> result(counters.size(), {}); + for (qsizetype i = 0; i < counters.size(); ++i) { + result[i] = readValue(i); + } + return result; } bool QBenchmarkPerfEventsMeasurer::isMeasurementAccepted(Measurement) @@ -531,10 +564,10 @@ static quint64 rawReadValue(int fd) return results.value * (double(results.time_running) / double(results.time_enabled)); } -QBenchmarkMeasurerBase::Measurement QBenchmarkPerfEventsMeasurer::readValue() +QBenchmarkMeasurerBase::Measurement QBenchmarkPerfEventsMeasurer::readValue(qsizetype idx) { - quint64 raw = rawReadValue(fd); - return { qreal(qint64(raw)), metricForEvent(attr.type, attr.config) }; + quint64 raw = rawReadValue(fds.at(idx)); + return { qreal(qint64(raw)), metricForEvent(eventTypes->at(idx)) }; } QT_END_NAMESPACE diff --git a/src/testlib/qbenchmarkperfevents_p.h b/src/testlib/qbenchmarkperfevents_p.h index fa4c397d6f..5f94ea7794 100644 --- a/src/testlib/qbenchmarkperfevents_p.h +++ b/src/testlib/qbenchmarkperfevents_p.h @@ -26,20 +26,19 @@ public: ~QBenchmarkPerfEventsMeasurer(); void init() override; void start() override; - Measurement stop() override; + QList<Measurement> stop() override; bool isMeasurementAccepted(Measurement measurement) override; int adjustIterationCount(int suggestion) override; int adjustMedianCount(int suggestion) override; bool needsWarmupIteration() override { return true; } static bool isAvailable(); - static QTest::QBenchmarkMetric metricForEvent(quint32 type, quint64 event_id); static void setCounter(const char *name); static void listCounters(); private: - int fd = -1; + QList<int> fds; - Measurement readValue(); + Measurement readValue(qsizetype idx = 0); }; QT_END_NAMESPACE diff --git a/src/testlib/qbenchmarktimemeasurers_p.h b/src/testlib/qbenchmarktimemeasurers_p.h index 10fb23e118..2f1364db02 100644 --- a/src/testlib/qbenchmarktimemeasurers_p.h +++ b/src/testlib/qbenchmarktimemeasurers_p.h @@ -25,7 +25,7 @@ class QBenchmarkTimeMeasurer : public QBenchmarkMeasurerBase { public: void start() override; - Measurement stop() override; + QList<Measurement> stop() override; bool isMeasurementAccepted(Measurement measurement) override; int adjustIterationCount(int sugestion) override; int adjustMedianCount(int suggestion) override; @@ -40,7 +40,7 @@ class QBenchmarkTickMeasurer : public QBenchmarkMeasurerBase { public: void start() override; - Measurement stop() override; + QList<Measurement> stop() override; bool isMeasurementAccepted(Measurement measurement) override; int adjustIterationCount(int) override; int adjustMedianCount(int suggestion) override; diff --git a/src/testlib/qbenchmarkvalgrind.cpp b/src/testlib/qbenchmarkvalgrind.cpp index 732c8bbf42..253108bd9d 100644 --- a/src/testlib/qbenchmarkvalgrind.cpp +++ b/src/testlib/qbenchmarkvalgrind.cpp @@ -170,11 +170,11 @@ void QBenchmarkCallgrindMeasurer::start() CALLGRIND_ZERO_STATS; } -QBenchmarkMeasurerBase::Measurement QBenchmarkCallgrindMeasurer::stop() +QList<QBenchmarkMeasurerBase::Measurement> QBenchmarkCallgrindMeasurer::stop() { CALLGRIND_DUMP_STATS; const qint64 result = QBenchmarkValgrindUtils::extractLastResult(); - return { qreal(result), QTest::InstructionReads }; + return { { qreal(result), QTest::InstructionReads } }; } bool QBenchmarkCallgrindMeasurer::isMeasurementAccepted(Measurement measurement) diff --git a/src/testlib/qbenchmarkvalgrind_p.h b/src/testlib/qbenchmarkvalgrind_p.h index 9f0accd842..f9925c9211 100644 --- a/src/testlib/qbenchmarkvalgrind_p.h +++ b/src/testlib/qbenchmarkvalgrind_p.h @@ -41,7 +41,7 @@ class QBenchmarkCallgrindMeasurer : public QBenchmarkMeasurerBase { public: void start() override; - Measurement stop() override; + QList<Measurement> stop() override; bool isMeasurementAccepted(Measurement measurement) override; int adjustIterationCount(int) override; int adjustMedianCount(int) override; diff --git a/src/testlib/qtestcase.cpp b/src/testlib/qtestcase.cpp index 24581011c2..63691b0c0b 100644 --- a/src/testlib/qtestcase.cpp +++ b/src/testlib/qtestcase.cpp @@ -1067,17 +1067,20 @@ Q_TESTLIB_EXPORT void qtest_qParseArgs(int argc, char *argv[], bool qml) { qtest_qParseArgs(argc, const_cast<const char *const *>(argv), qml); } -QBenchmarkResult qMedian(const QList<QBenchmarkResult> &container) +static QList<QBenchmarkResult> qMedian(const QList<QList<QBenchmarkResult>> &container) { const int count = container.size(); if (count == 0) - return QBenchmarkResult(); + return {}; if (count == 1) return container.front(); - QList<QBenchmarkResult> containerCopy = container; - std::sort(containerCopy.begin(), containerCopy.end()); + QList<QList<QBenchmarkResult>> containerCopy = container; + std::sort(containerCopy.begin(), containerCopy.end(), + [](const QList<QBenchmarkResult> &a, const QList<QBenchmarkResult> &b) { + return a.first() < b.first(); + }); const int middle = count / 2; @@ -1104,7 +1107,7 @@ void TestMethods::invokeTestOnData(int index) const bool isBenchmark = false; int i = (QBenchmarkGlobalData::current->measurer->needsWarmupIteration()) ? -1 : 0; - QList<QBenchmarkResult> results; + QList<QList<QBenchmarkResult>> resultsList; bool minimumTotalReached = false; do { QBenchmarkTestMethodData::current->beginDataRun(); @@ -1121,8 +1124,9 @@ void TestMethods::invokeTestOnData(int index) const const bool initQuit = QTestResult::skipCurrentTest() || QTestResult::currentTestFailed(); if (!initQuit) { - QBenchmarkTestMethodData::current->result = QBenchmarkResult(); + QBenchmarkTestMethodData::current->results.clear(); QBenchmarkTestMethodData::current->resultAccepted = false; + QBenchmarkTestMethodData::current->valid = false; QBenchmarkGlobalData::current->context.tag = QLatin1StringView( QTestResult::currentDataTag() ? QTestResult::currentDataTag() : ""); @@ -1164,29 +1168,29 @@ void TestMethods::invokeTestOnData(int index) const QBenchmarkTestMethodData::current->endDataRun(); if (!QTestResult::skipCurrentTest() && !QTestResult::currentTestFailed()) { if (i > -1) // iteration -1 is the warmup iteration. - results.append(QBenchmarkTestMethodData::current->result); - - if (isBenchmark && QBenchmarkGlobalData::current->verboseOutput) { - if (i == -1) { - QTestLog::info(qPrintable( - QString::fromLatin1("warmup stage result : %1") - .arg(QBenchmarkTestMethodData::current->result.measurement.value)), nullptr, 0); - } else { - QTestLog::info(qPrintable( - QString::fromLatin1("accumulation stage result: %1") - .arg(QBenchmarkTestMethodData::current->result.measurement.value)), nullptr, 0); - } + resultsList.append(QBenchmarkTestMethodData::current->results); + + if (isBenchmark && QBenchmarkGlobalData::current->verboseOutput && + !QBenchmarkTestMethodData::current->results.isEmpty()) { + // we only print the first result + const QBenchmarkResult &first = QBenchmarkTestMethodData::current->results.constFirst(); + QString pattern = i < 0 ? "warmup stage result : %1"_L1 + : "accumulation stage result: %1"_L1; + QTestLog::info(qPrintable(pattern.arg(first.measurement.value)), nullptr, 0); } } - // Verify if the minimum total measurement is reached, if it was specified: + // Verify if the minimum total measurement (for the first measurement) + // was reached, if it was specified: if (QBenchmarkGlobalData::current->minimumTotal == -1) { minimumTotalReached = true; } else { - auto addResult = [](qreal current, const QBenchmarkResult& r) { - return current + r.measurement.value; + auto addResult = [](qreal current, const QList<QBenchmarkResult> &r) { + if (!r.isEmpty()) + current += r.first().measurement.value; + return current; }; - const qreal total = std::accumulate(results.begin(), results.end(), 0.0, addResult); + const qreal total = std::accumulate(resultsList.begin(), resultsList.end(), 0.0, addResult); minimumTotalReached = (total >= QBenchmarkGlobalData::current->minimumTotal); } } while (isBenchmark @@ -1198,8 +1202,12 @@ void TestMethods::invokeTestOnData(int index) const bool testPassed = !QTestResult::skipCurrentTest() && !QTestResult::currentTestFailed(); QTestResult::finishedCurrentTestDataCleanup(); // Only report benchmark figures if the test passed - if (testPassed && QBenchmarkTestMethodData::current->resultsAccepted()) - QTestLog::addBenchmarkResult(qMedian(results)); + if (testPassed && QBenchmarkTestMethodData::current->resultsAccepted()) { + const QList<QBenchmarkResult> median = qMedian(resultsList); + for (auto m : median) { + QTestLog::addBenchmarkResult(m); + } + } } } |