From d3ba45b3c2955bef141d06863030e09ce2658e36 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Mon, 18 Feb 2019 10:55:52 +0100 Subject: Add simple helper program to compare JSON output Change-Id: I7a21c214e302a5b2422185659e261db11374f62c Reviewed-by: Erik Verbruggen Reviewed-by: Michael Brasser --- .gitignore | 1 + tools/compareresults/compareresults.pro | 8 ++ tools/compareresults/main.cpp | 180 ++++++++++++++++++++++++++++++++ tools/tools.pro | 3 +- 4 files changed, 191 insertions(+), 1 deletion(-) create mode 100644 tools/compareresults/compareresults.pro create mode 100644 tools/compareresults/main.cpp diff --git a/.gitignore b/.gitignore index 401dda0..3e8a476 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ moc_predefs.h qrc_qmlbench.cpp src/qmlbench tools/decidefps/decidefps +tools/compareresults/compareresults .qmake.stash Makefile *.pro.user diff --git a/tools/compareresults/compareresults.pro b/tools/compareresults/compareresults.pro new file mode 100644 index 0000000..33ab909 --- /dev/null +++ b/tools/compareresults/compareresults.pro @@ -0,0 +1,8 @@ +TEMPLATE = app +QT = core +CONFIG -= app_bundle +SOURCES = \ + main.cpp + +target.path = /root +INSTALLS += target diff --git a/tools/compareresults/main.cpp b/tools/compareresults/main.cpp new file mode 100644 index 0000000..899a6bc --- /dev/null +++ b/tools/compareresults/main.cpp @@ -0,0 +1,180 @@ +/**************************************************************************** +** +** Copyright (C) 2019 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of the qmlbench tool. +** +** $QT_BEGIN_LICENSE:GPL-EXCEPT$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 3 as published by the Free Software +** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#include +#include +#include +#include +#include +#include +#include + +#include + +static QJsonObject loadFile(const QString &fileName, QString *error) +{ + QFile f(fileName); + if (!f.open(QIODevice::ReadOnly)) { + *error = f.errorString(); + return QJsonObject(); + } + QJsonParseError jsonError; + auto document = QJsonDocument::fromJson(f.readAll(), &jsonError); + if (jsonError.error != QJsonParseError::NoError) { + *error = jsonError.errorString(); + return QJsonObject(); + } + + if (!document.isObject()) { + *error = QLatin1String("JSON data is an array, expected an object"); + return QJsonObject(); + } + + return document.object(); +} + +struct Result +{ + QString name; + double differenceInPercent = 0; +}; + +static QString trimPrefix(const QString &str, const QString &prefix) +{ + if (!str.startsWith(prefix)) + return str; + QString result(str); + result.remove(0, prefix.length()); + return result; +} + +int main(int argc, char **argv) +{ + QCoreApplication app(argc, argv); + + QCommandLineParser parser; + parser.setApplicationDescription("Compare results of json output of qmlbench"); + parser.addHelpOption(); + + QCommandLineOption keyOption("key"); + keyOption.setDescription(QCoreApplication::translate("main", "Result key in JSON data to compare")); + keyOption.setDefaultValue("average"); + parser.addOption(keyOption); + + QCommandLineOption prefixOption("prefix"); + prefixOption.setDescription(QCoreApplication::translate("main", "Prefix to strip from test case names")); + prefixOption.setValueName(QString("path")); + parser.addOption(prefixOption); + + parser.addPositionalArgument("baseline", QCoreApplication::translate("main", "Base line or reference performance results, in JSON format")); + parser.addPositionalArgument("results", QCoreApplication::translate("main", "New results to compare against the base line, in JSON format")); + + parser.process(app); + + const QString key = parser.value(keyOption); + const QString prefix = parser.value(prefixOption); + + QStringList args = parser.positionalArguments(); + if (args.count() != 2) { + qFatal("Internal error, two arguments are required, but QCommandLineParser should have checked for that..."); + return EXIT_FAILURE; + } + + const QString baseLineFileName = args.at(0); + const QString newResultsFileName = args.at(1); + + QString error; + QJsonObject baseLine = loadFile(baseLineFileName, &error); + if (!error.isEmpty()) { + qWarning("Error loading reference file %s: %s", qPrintable(baseLineFileName), qPrintable(error)); + return EXIT_FAILURE; + } + + QJsonObject newResults = loadFile(newResultsFileName, &error); + if (!error.isEmpty()) { + qWarning("Error loading new results file %s: %s", qPrintable(baseLineFileName), qPrintable(error)); + return EXIT_FAILURE; + } + + const QSet unrelatedKeys{"command-line", "id", "opengl", "os", "qt", "windowSize"}; + + auto filter = [&unrelatedKeys](const QStringList &list) -> QStringList { + return list.toSet().subtract(unrelatedKeys).toList(); + }; + + QStringList tests = filter(baseLine.keys()); + QStringList newResultKeys = filter(newResults.keys()); + if (tests != newResultKeys) { + QSet baseLineSet = tests.toSet(); + QSet resultsSet = newResultKeys.toSet(); + + qWarning("Error: The two result files do not cover the same set of tests"); + qWarning("Tests existing in the base line but missing from the new results: %s", qPrintable(baseLineSet.subtract(resultsSet).toList().join("\t\n"))); + qWarning("Tests existing in the new results but missing from the base line: %s", qPrintable(resultsSet.subtract(baseLineSet).toList().join("\t\n"))); + + return EXIT_FAILURE; + } + + QVector differencesInPercent; + + for (const QString &test: qAsConst(tests)) { + const QJsonObject baseLineResult = baseLine[test].toObject(); + const QJsonObject newResult = newResults[test].toObject(); + const QString testName = trimPrefix(test, prefix); + + const double oldValue = baseLineResult[key].toDouble(); + const double newValue = newResult[key].toDouble(); + + const double differenceInPercent = (newValue - oldValue) * 100 / oldValue; + + if (differenceInPercent > 0) { + printf("%s: improvement by %.2f%%\n", qPrintable(testName), differenceInPercent); + } else if (differenceInPercent < 0) { + printf("%s: regression by %.2f%%\n", qPrintable(testName), differenceInPercent); + } + + differencesInPercent << Result{testName, differenceInPercent}; + } + + auto minMax = std::minmax_element(differencesInPercent.constBegin(), differencesInPercent.constEnd(), [](const auto &lhs, const auto &rhs) { + return lhs.differenceInPercent < rhs.differenceInPercent; + }); + + const Result biggestLoser = *minMax.first; + const Result biggestWinner = *minMax.second; + + printf("Biggest improvement: %s with %.2f%%\n", qPrintable(biggestWinner.name), biggestWinner.differenceInPercent); + printf("Biggest regression: %s with %.2f%%\n", qPrintable(biggestLoser.name), biggestLoser.differenceInPercent); + + double overallAverage = std::accumulate(differencesInPercent.constBegin(), differencesInPercent.constEnd(), 0.0, [](double v, const Result &r) { + return v + r.differenceInPercent; }) + / double(differencesInPercent.count()); + printf("Overall average of differences: %.2f%%\n", overallAverage); + + return EXIT_SUCCESS; +} diff --git a/tools/tools.pro b/tools/tools.pro index 90f4364..f328bce 100644 --- a/tools/tools.pro +++ b/tools/tools.pro @@ -1,3 +1,4 @@ TEMPLATE = subdirs SUBDIRS = \ - decidefps + decidefps \ + compareresults -- cgit v1.2.3