diff options
Diffstat (limited to 'tests/baseline/shared/qbaselinetest.cpp')
-rw-r--r-- | tests/baseline/shared/qbaselinetest.cpp | 128 |
1 files changed, 71 insertions, 57 deletions
diff --git a/tests/baseline/shared/qbaselinetest.cpp b/tests/baseline/shared/qbaselinetest.cpp index 004d8013f1..964266f1b4 100644 --- a/tests/baseline/shared/qbaselinetest.cpp +++ b/tests/baseline/shared/qbaselinetest.cpp @@ -1,30 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the test suite of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:GPL-EXCEPT$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 3 as published by the Free Software -** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only #include "qbaselinetest.h" #include "baselineprotocol.h" @@ -36,6 +11,7 @@ namespace QBaselineTest { static char *fargv[MAXCMDLINEARGS]; +static QString server; static bool simfail = false; static PlatformInfo customInfo; static bool customAutoModeSet = false; @@ -51,9 +27,6 @@ static QByteArray curFunction; static ImageItemList itemList; static bool gotBaselines; -static QString definedTestProject; -static QString definedTestCase; - void handleCmdLineArgs(int *argcp, char ***argvp) { @@ -61,6 +34,7 @@ void handleCmdLineArgs(int *argcp, char ***argvp) return; bool showHelp = false; + bool abortOnHelp = true; int fargc = 0; int numArgs = *argcp; @@ -69,7 +43,16 @@ void handleCmdLineArgs(int *argcp, char ***argvp) QByteArray arg = (*argvp)[i]; QByteArray nextArg = (i+1 < numArgs) ? (*argvp)[i+1] : nullptr; - if (arg == "-simfail") { + if (arg == "-server") { + i++; + if (!nextArg.isEmpty()) { + server = QString::fromLocal8Bit(nextArg); + } else { + qWarning() << "-server requires parameter"; + showHelp = true; + break; + } + } else if (arg == "-simfail") { simfail = true; } else if (arg == "-fuzzlevel") { i++; @@ -105,10 +88,13 @@ void handleCmdLineArgs(int *argcp, char ***argvp) } customInfo.addOverride(key, value); } else { - if ( (arg == "-help") || (arg == "--help") ) + if ( (arg == "-help") || (arg == "--help") ) { showHelp = true; + abortOnHelp = false; + } if (fargc >= MAXCMDLINEARGS) { qWarning() << "Too many command line arguments!"; + showHelp = true; break; } fargv[fargc++] = (*argvp)[i]; @@ -121,7 +107,9 @@ void handleCmdLineArgs(int *argcp, char ***argvp) // TBD: arrange for this to be printed *after* QTest's help QTextStream out(stdout); out << "\n Baseline testing (lancelot) options:\n"; - out << " -simfail : Force an image comparison mismatch. For testing purposes.\n"; + out << " -server <host> : Set the network baseline server to connect to.\n"; + out << " The default is taken from the environment variable QT_LANCELOT_SERVER.\n"; + out << " -simfail : Force an image comparison mismatch. For development purposes.\n"; out << " -fuzzlevel <int> : Specify the percentage of fuzziness in comparison. Overrides server default. 0 means exact match.\n"; out << " -auto : Inform server that this run is done by a daemon, CI system or similar.\n"; out << " -adhoc (default) : The inverse of -auto; this run is done by human, e.g. for testing.\n"; @@ -132,6 +120,9 @@ void handleCmdLineArgs(int *argcp, char ***argvp) out << " for example: -compareto QtVersion=4.8.0\n"; out << " Multiple -compareto client specifications may be given.\n"; out << "\n"; + out.flush(); + if (abortOnHelp) + std::exit(1); } } @@ -167,7 +158,7 @@ void fetchCustomClientProperties() key = line.left(colonPos).simplified().replace(' ', '_'); val = line.mid(colonPos+1).trimmed(); } - if (!key.isEmpty() && key.length() < 64 && val.length() < 256) // ###TBD: maximum 256 chars in value? + if (!key.isEmpty() && key.size() < 64 && val.size() < 256) // ###TBD: maximum 256 chars in value? addClientProperty(key, val); else qDebug() << "Unparseable script output ignored:" << line; @@ -200,10 +191,7 @@ bool connect(QByteArray *msg, bool *error) if (!customAutoModeSet) clientInfo.setAdHocRun(defaultInfo.isAdHocRun()); - if (!definedTestProject.isEmpty()) - clientInfo.insert(PI_Project, definedTestProject); - - QString testCase = definedTestCase; + QString testCase = clientInfo.value(PI_TestCase); if (testCase.isEmpty() && QTest::testObject() && QTest::testObject()->metaObject()) { //qDebug() << "Trying to Read TestCaseName from Testlib!"; testCase = QTest::testObject()->metaObject()->className(); @@ -213,7 +201,7 @@ bool connect(QByteArray *msg, bool *error) return false; } - if (!proto.connect(testCase, &dryRunMode, clientInfo)) { + if (!proto.connect(testCase, &dryRunMode, clientInfo, server)) { *msg += "Failed to connect to baseline server: " + proto.errorMessage().toLatin1(); *error = true; return false; @@ -233,14 +221,10 @@ bool disconnectFromBaselineServer() return false; } -bool connectToBaselineServer(QByteArray *msg, const QString &testProject, const QString &testCase) +bool connectToBaselineServer(QByteArray *msg) { bool dummy; QByteArray dummyMsg; - - definedTestProject = testProject; - definedTestCase = testCase; - return connect(msg ? msg : &dummyMsg, &dummy); } @@ -255,6 +239,16 @@ void setSimFail(bool fail) simfail = fail; } +void setProject(const QString &projectName) +{ + addClientProperty(PI_Project, projectName); +} + +void setProjectImageKeys(const QStringList &keys) +{ + QString keyList = keys.join(QLC(',')); + addClientProperty(PI_ProjectImageKeys, keyList); +} void modifyImage(QImage *img) { @@ -274,6 +268,7 @@ void modifyImage(QImage *img) bool compareItem(const ImageItem &baseline, const QImage &img, QByteArray *msg, bool *error) { + *error = false; ImageItem item = baseline; if (simfail) { // Simulate test failure by forcing image mismatch; for testing purposes @@ -284,6 +279,7 @@ bool compareItem(const ImageItem &baseline, const QImage &img, QByteArray *msg, } else { item.image = img; } + bool isNewItem = false; item.imageChecksums.clear(); item.imageChecksums.prepend(ImageItem::computeChecksum(item.image)); QByteArray srvMsg; @@ -295,9 +291,11 @@ bool compareItem(const ImageItem &baseline, const QImage &img, QByteArray *msg, return true; break; case ImageItem::BaselineNotFound: - if (!customInfo.overrides().isEmpty() || baselinePolicy == UploadNone) { - qWarning() << "Cannot compare to baseline: No such baseline found on server."; + if (!customInfo.overrides().isEmpty()) return true; + if (baselinePolicy == UploadNone) { + isNewItem = true; + break; } if (proto.submitNewBaseline(item, &srvMsg)) qDebug() << msg->constData() << "Baseline not found on server. New baseline uploaded."; @@ -310,7 +308,6 @@ bool compareItem(const ImageItem &baseline, const QImage &img, QByteArray *msg, return true; break; } - *error = false; // The actual comparison of the given image with the baseline: if (baseline.imageChecksums.contains(item.imageChecksums.at(0))) { if (!proto.submitMatch(item, &srvMsg)) @@ -331,7 +328,11 @@ bool compareItem(const ImageItem &baseline, const QImage &img, QByteArray *msg, qInfo() << "Baseline server reports:" << srvMsg; return true; // The server decides: a fuzzy match means no mismatch } - *msg += "Mismatch. See report:\n " + srvMsg; + if (isNewItem) + *msg += "No baseline on server, so cannot compare."; + else + *msg += "Mismatch."; + *msg += " See report:\n " + srvMsg; if (dryRunMode) { qDebug() << "Dryrun, so ignoring" << *msg; return true; @@ -397,22 +398,21 @@ QTestData &newRow(const char *dataTag, quint16 checksum) return QTest::newRow(dataTag); } - -bool testImage(const QImage& img, QByteArray *msg, bool *error) +const ImageItem *findCurrentItem(QByteArray *msg, bool *error) { if (!connected && !connect(msg, error)) - return true; + return nullptr; if (QTest::currentTestFunction() != curFunction || itemList.isEmpty()) { - qWarning() << "Usage error: QBASELINE_TEST used without corresponding QBaselineTest::newRow()"; - return true; + qWarning() << "Usage error: QBASELINE_ macro used without corresponding QBaselineTest::newRow()"; + return nullptr; } if (!gotBaselines) { if (!proto.requestBaselineChecksums(QString::fromLatin1(QTest::currentTestFunction()), &itemList) || itemList.isEmpty()) { *msg = "Communication with baseline server failed: " + proto.errorMessage().toLatin1(); *error = true; - return true; + return nullptr; } gotBaselines = true; } @@ -422,10 +422,24 @@ bool testImage(const QImage& img, QByteArray *msg, bool *error) while (it != itemList.constEnd() && it->itemName != curTag) ++it; if (it == itemList.constEnd()) { - qWarning() << "Usage error: QBASELINE_TEST used without corresponding QBaselineTest::newRow() for row" << curTag; - return true; + qWarning() << "Usage error: QBASELINE_ macro used without corresponding QBaselineTest::newRow() for row" << curTag; + return nullptr; } - return compareItem(*it, img, msg, error); + return &(*it); +} + +bool testImage(const QImage &img, QByteArray *msg, bool *error) +{ + const ImageItem *item = findCurrentItem(msg, error); + return item ? compareItem(*item, img, msg, error) : true; +} + +bool isCurrentItemBlacklisted() +{ + QByteArray msg; + bool error = false; + const ImageItem *item = findCurrentItem(&msg, &error); + return item ? (item->status == ImageItem::IgnoreItem) : false; } } |