summaryrefslogtreecommitdiffstats
path: root/tests/baseline/shared/qbaselinetest.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'tests/baseline/shared/qbaselinetest.cpp')
-rw-r--r--tests/baseline/shared/qbaselinetest.cpp77
1 files changed, 58 insertions, 19 deletions
diff --git a/tests/baseline/shared/qbaselinetest.cpp b/tests/baseline/shared/qbaselinetest.cpp
index 95da71d9e7..964266f1b4 100644
--- a/tests/baseline/shared/qbaselinetest.cpp
+++ b/tests/baseline/shared/qbaselinetest.cpp
@@ -1,5 +1,5 @@
// Copyright (C) 2016 The Qt Company Ltd.
-// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0
+// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only
#include "qbaselinetest.h"
#include "baselineprotocol.h"
@@ -11,6 +11,7 @@
namespace QBaselineTest {
static char *fargv[MAXCMDLINEARGS];
+static QString server;
static bool simfail = false;
static PlatformInfo customInfo;
static bool customAutoModeSet = false;
@@ -33,6 +34,7 @@ void handleCmdLineArgs(int *argcp, char ***argvp)
return;
bool showHelp = false;
+ bool abortOnHelp = true;
int fargc = 0;
int numArgs = *argcp;
@@ -41,7 +43,16 @@ void handleCmdLineArgs(int *argcp, char ***argvp)
QByteArray arg = (*argvp)[i];
QByteArray nextArg = (i+1 < numArgs) ? (*argvp)[i+1] : nullptr;
- if (arg == "-simfail") {
+ if (arg == "-server") {
+ i++;
+ if (!nextArg.isEmpty()) {
+ server = QString::fromLocal8Bit(nextArg);
+ } else {
+ qWarning() << "-server requires parameter";
+ showHelp = true;
+ break;
+ }
+ } else if (arg == "-simfail") {
simfail = true;
} else if (arg == "-fuzzlevel") {
i++;
@@ -77,10 +88,13 @@ void handleCmdLineArgs(int *argcp, char ***argvp)
}
customInfo.addOverride(key, value);
} else {
- if ( (arg == "-help") || (arg == "--help") )
+ if ( (arg == "-help") || (arg == "--help") ) {
showHelp = true;
+ abortOnHelp = false;
+ }
if (fargc >= MAXCMDLINEARGS) {
qWarning() << "Too many command line arguments!";
+ showHelp = true;
break;
}
fargv[fargc++] = (*argvp)[i];
@@ -93,7 +107,9 @@ void handleCmdLineArgs(int *argcp, char ***argvp)
// TBD: arrange for this to be printed *after* QTest's help
QTextStream out(stdout);
out << "\n Baseline testing (lancelot) options:\n";
- out << " -simfail : Force an image comparison mismatch. For testing purposes.\n";
+ out << " -server <host> : Set the network baseline server to connect to.\n";
+ out << " The default is taken from the environment variable QT_LANCELOT_SERVER.\n";
+ out << " -simfail : Force an image comparison mismatch. For development purposes.\n";
out << " -fuzzlevel <int> : Specify the percentage of fuzziness in comparison. Overrides server default. 0 means exact match.\n";
out << " -auto : Inform server that this run is done by a daemon, CI system or similar.\n";
out << " -adhoc (default) : The inverse of -auto; this run is done by human, e.g. for testing.\n";
@@ -104,6 +120,9 @@ void handleCmdLineArgs(int *argcp, char ***argvp)
out << " for example: -compareto QtVersion=4.8.0\n";
out << " Multiple -compareto client specifications may be given.\n";
out << "\n";
+ out.flush();
+ if (abortOnHelp)
+ std::exit(1);
}
}
@@ -139,7 +158,7 @@ void fetchCustomClientProperties()
key = line.left(colonPos).simplified().replace(' ', '_');
val = line.mid(colonPos+1).trimmed();
}
- if (!key.isEmpty() && key.length() < 64 && val.length() < 256) // ###TBD: maximum 256 chars in value?
+ if (!key.isEmpty() && key.size() < 64 && val.size() < 256) // ###TBD: maximum 256 chars in value?
addClientProperty(key, val);
else
qDebug() << "Unparseable script output ignored:" << line;
@@ -182,7 +201,7 @@ bool connect(QByteArray *msg, bool *error)
return false;
}
- if (!proto.connect(testCase, &dryRunMode, clientInfo)) {
+ if (!proto.connect(testCase, &dryRunMode, clientInfo, server)) {
*msg += "Failed to connect to baseline server: " + proto.errorMessage().toLatin1();
*error = true;
return false;
@@ -249,6 +268,7 @@ void modifyImage(QImage *img)
bool compareItem(const ImageItem &baseline, const QImage &img, QByteArray *msg, bool *error)
{
+ *error = false;
ImageItem item = baseline;
if (simfail) {
// Simulate test failure by forcing image mismatch; for testing purposes
@@ -259,6 +279,7 @@ bool compareItem(const ImageItem &baseline, const QImage &img, QByteArray *msg,
} else {
item.image = img;
}
+ bool isNewItem = false;
item.imageChecksums.clear();
item.imageChecksums.prepend(ImageItem::computeChecksum(item.image));
QByteArray srvMsg;
@@ -270,9 +291,11 @@ bool compareItem(const ImageItem &baseline, const QImage &img, QByteArray *msg,
return true;
break;
case ImageItem::BaselineNotFound:
- if (!customInfo.overrides().isEmpty() || baselinePolicy == UploadNone) {
- qWarning() << "Cannot compare to baseline: No such baseline found on server.";
+ if (!customInfo.overrides().isEmpty())
return true;
+ if (baselinePolicy == UploadNone) {
+ isNewItem = true;
+ break;
}
if (proto.submitNewBaseline(item, &srvMsg))
qDebug() << msg->constData() << "Baseline not found on server. New baseline uploaded.";
@@ -285,7 +308,6 @@ bool compareItem(const ImageItem &baseline, const QImage &img, QByteArray *msg,
return true;
break;
}
- *error = false;
// The actual comparison of the given image with the baseline:
if (baseline.imageChecksums.contains(item.imageChecksums.at(0))) {
if (!proto.submitMatch(item, &srvMsg))
@@ -306,7 +328,11 @@ bool compareItem(const ImageItem &baseline, const QImage &img, QByteArray *msg,
qInfo() << "Baseline server reports:" << srvMsg;
return true; // The server decides: a fuzzy match means no mismatch
}
- *msg += "Mismatch. See report:\n " + srvMsg;
+ if (isNewItem)
+ *msg += "No baseline on server, so cannot compare.";
+ else
+ *msg += "Mismatch.";
+ *msg += " See report:\n " + srvMsg;
if (dryRunMode) {
qDebug() << "Dryrun, so ignoring" << *msg;
return true;
@@ -372,22 +398,21 @@ QTestData &newRow(const char *dataTag, quint16 checksum)
return QTest::newRow(dataTag);
}
-
-bool testImage(const QImage& img, QByteArray *msg, bool *error)
+const ImageItem *findCurrentItem(QByteArray *msg, bool *error)
{
if (!connected && !connect(msg, error))
- return true;
+ return nullptr;
if (QTest::currentTestFunction() != curFunction || itemList.isEmpty()) {
- qWarning() << "Usage error: QBASELINE_TEST used without corresponding QBaselineTest::newRow()";
- return true;
+ qWarning() << "Usage error: QBASELINE_ macro used without corresponding QBaselineTest::newRow()";
+ return nullptr;
}
if (!gotBaselines) {
if (!proto.requestBaselineChecksums(QString::fromLatin1(QTest::currentTestFunction()), &itemList) || itemList.isEmpty()) {
*msg = "Communication with baseline server failed: " + proto.errorMessage().toLatin1();
*error = true;
- return true;
+ return nullptr;
}
gotBaselines = true;
}
@@ -397,10 +422,24 @@ bool testImage(const QImage& img, QByteArray *msg, bool *error)
while (it != itemList.constEnd() && it->itemName != curTag)
++it;
if (it == itemList.constEnd()) {
- qWarning() << "Usage error: QBASELINE_TEST used without corresponding QBaselineTest::newRow() for row" << curTag;
- return true;
+ qWarning() << "Usage error: QBASELINE_ macro used without corresponding QBaselineTest::newRow() for row" << curTag;
+ return nullptr;
}
- return compareItem(*it, img, msg, error);
+ return &(*it);
+}
+
+bool testImage(const QImage &img, QByteArray *msg, bool *error)
+{
+ const ImageItem *item = findCurrentItem(msg, error);
+ return item ? compareItem(*item, img, msg, error) : true;
+}
+
+bool isCurrentItemBlacklisted()
+{
+ QByteArray msg;
+ bool error = false;
+ const ImageItem *item = findCurrentItem(&msg, &error);
+ return item ? (item->status == ImageItem::IgnoreItem) : false;
}
}
generated by cgit v1.2.3 (git 2.25.1) at 2024-06-16 09:25:21 +0000