diff options
Diffstat (limited to 'tests/baselineserver/shared/qbaselinetest.cpp')
| -rw-r--r-- | tests/baselineserver/shared/qbaselinetest.cpp | 423 |
1 files changed, 0 insertions, 423 deletions
diff --git a/tests/baselineserver/shared/qbaselinetest.cpp b/tests/baselineserver/shared/qbaselinetest.cpp deleted file mode 100644 index 3587cd01ea..0000000000 --- a/tests/baselineserver/shared/qbaselinetest.cpp +++ /dev/null @@ -1,423 +0,0 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the test suite of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:GPL-EXCEPT$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 3 as published by the Free Software -** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ - -#include "qbaselinetest.h" -#include "baselineprotocol.h" -#include <QtCore/QDir> -#include <QFile> - -#define MAXCMDLINEARGS 128 - -namespace QBaselineTest { - -static char *fargv[MAXCMDLINEARGS]; -static bool simfail = false; -static PlatformInfo customInfo; -static bool customAutoModeSet = false; - -static BaselineProtocol proto; -static bool connected = false; -static bool triedConnecting = false; -static bool dryRunMode = false; -static enum { UploadMissing, UploadAll, UploadNone } baselinePolicy = UploadMissing; - -static QByteArray curFunction; -static ImageItemList itemList; -static bool gotBaselines; - -static QString definedTestProject; -static QString definedTestCase; - - -void handleCmdLineArgs(int *argcp, char ***argvp) -{ - if (!argcp || !argvp) - return; - - bool showHelp = false; - - int fargc = 0; - int numArgs = *argcp; - - for (int i = 0; i < numArgs; i++) { - QByteArray arg = (*argvp)[i]; - QByteArray nextArg = (i+1 < numArgs) ? (*argvp)[i+1] : 0; - - if (arg == "-simfail") { - simfail = true; - } else if (arg == "-fuzzlevel") { - i++; - bool ok = false; - (void)nextArg.toInt(&ok); - if (!ok) { - qWarning() << "-fuzzlevel requires integer parameter"; - showHelp = true; - break; - } - customInfo.insert("FuzzLevel", QString::fromLatin1(nextArg)); - } else if (arg == "-auto") { - customAutoModeSet = true; - customInfo.setAdHocRun(false); - } else if (arg == "-adhoc") { - customAutoModeSet = true; - customInfo.setAdHocRun(true); - } else if (arg == "-setbaselines") { - baselinePolicy = UploadAll; - } else if (arg == "-nosetbaselines") { - baselinePolicy = UploadNone; - } else if (arg == "-compareto") { - i++; - int split = qMax(0, nextArg.indexOf('=')); - QByteArray key = nextArg.left(split).trimmed(); - QByteArray value = nextArg.mid(split+1).trimmed(); - if (key.isEmpty() || value.isEmpty()) { - qWarning() << "-compareto requires parameter of the form <key>=<value>"; - showHelp = true; - break; - } - customInfo.addOverride(key, value); - } else { - if ( (arg == "-help") || (arg == "--help") ) - showHelp = true; - if (fargc >= MAXCMDLINEARGS) { - qWarning() << "Too many command line arguments!"; - break; - } - fargv[fargc++] = (*argvp)[i]; - } - } - *argcp = fargc; - *argvp = fargv; - - if (showHelp) { - // TBD: arrange for this to be printed *after* QTest's help - QTextStream out(stdout); - out << "\n Baseline testing (lancelot) options:\n"; - out << " -simfail : Force an image comparison mismatch. For testing purposes.\n"; - out << " -fuzzlevel <int> : Specify the percentage of fuzziness in comparison. Overrides server default. 0 means exact match.\n"; - out << " -auto : Inform server that this run is done by a daemon, CI system or similar.\n"; - out << " -adhoc (default) : The inverse of -auto; this run is done by human, e.g. for testing.\n"; - out << " -setbaselines : Store ALL rendered images as new baselines. Forces replacement of previous baselines.\n"; - out << " -nosetbaselines : Do not store rendered images as new baselines when previous baselines are missing.\n"; - out << " -compareto KEY=VAL : Force comparison to baselines from a different client,\n"; - out << " for example: -compareto QtVersion=4.8.0\n"; - out << " Multiple -compareto client specifications may be given.\n"; - out << "\n"; - } -} - - -void addClientProperty(const QString& key, const QString& value) -{ - customInfo.insert(key, value); -} - - -/* - If a client property script is present, run it and accept its output - in the form of one 'key: value' property per line -*/ -void fetchCustomClientProperties() -{ - QFile file("hostinfo.txt"); - if (!file.open(QIODevice::ReadOnly | QIODevice::Text)) - return; - QTextStream in(&file); - - while (!in.atEnd()) { - QString line = in.readLine().trimmed(); // ###local8bit? utf8? - if (line.startsWith(QLatin1Char('#'))) // Ignore comments in file - continue; - QString key, val; - int colonPos = line.indexOf(':'); - if (colonPos > 0) { - key = line.left(colonPos).simplified().replace(' ', '_'); - val = line.mid(colonPos+1).trimmed(); - } - if (!key.isEmpty() && key.length() < 64 && val.length() < 256) // ###TBD: maximum 256 chars in value? - addClientProperty(key, val); - else - qDebug() << "Unparseable script output ignored:" << line; - } -} - - -bool connect(QByteArray *msg, bool *error) -{ - if (connected) { - return true; - } - else if (triedConnecting) { - // Avoid repeated connection attempts, to avoid the program using Timeout * #testItems seconds before giving up - *msg = "Not connected to baseline server."; - *error = true; - return false; - } - - triedConnecting = true; - fetchCustomClientProperties(); - // Merge the platform info set by the program with the protocols default info - PlatformInfo clientInfo = customInfo; - PlatformInfo defaultInfo = PlatformInfo::localHostInfo(); - foreach (QString key, defaultInfo.keys()) { - if (!clientInfo.contains(key)) - clientInfo.insert(key, defaultInfo.value(key)); - } - if (!customAutoModeSet) - clientInfo.setAdHocRun(defaultInfo.isAdHocRun()); - - if (!definedTestProject.isEmpty()) - clientInfo.insert(PI_Project, definedTestProject); - - QString testCase = definedTestCase; - if (testCase.isEmpty() && QTest::testObject() && QTest::testObject()->metaObject()) { - //qDebug() << "Trying to Read TestCaseName from Testlib!"; - testCase = QTest::testObject()->metaObject()->className(); - } - if (testCase.isEmpty()) { - qWarning("QBaselineTest::connect: No test case name specified, cannot connect."); - return false; - } - - if (!proto.connect(testCase, &dryRunMode, clientInfo)) { - *msg += "Failed to connect to baseline server: " + proto.errorMessage().toLatin1(); - *error = true; - return false; - } - connected = true; - return true; -} - -bool disconnectFromBaselineServer() -{ - if (proto.disconnect()) { - connected = false; - triedConnecting = false; - return true; - } - - return false; -} - -bool connectToBaselineServer(QByteArray *msg, const QString &testProject, const QString &testCase) -{ - bool dummy; - QByteArray dummyMsg; - - definedTestProject = testProject; - definedTestCase = testCase; - - return connect(msg ? msg : &dummyMsg, &dummy); -} - -void setAutoMode(bool mode) -{ - customInfo.setAdHocRun(!mode); - customAutoModeSet = true; -} - -void setSimFail(bool fail) -{ - simfail = fail; -} - - -void modifyImage(QImage *img) -{ - uint c0 = 0x0000ff00; - uint c1 = 0x0080ff00; - img->setPixel(1,1,c0); - img->setPixel(2,1,c1); - img->setPixel(3,1,c0); - img->setPixel(1,2,c1); - img->setPixel(1,3,c0); - img->setPixel(2,3,c1); - img->setPixel(3,3,c0); - img->setPixel(1,4,c1); - img->setPixel(1,5,c0); -} - - -bool compareItem(const ImageItem &baseline, const QImage &img, QByteArray *msg, bool *error) -{ - ImageItem item = baseline; - if (simfail) { - // Simulate test failure by forcing image mismatch; for testing purposes - QImage misImg = img; - modifyImage(&misImg); - item.image = misImg; - simfail = false; // One failure is typically enough - } else { - item.image = img; - } - item.imageChecksums.clear(); - item.imageChecksums.prepend(ImageItem::computeChecksum(item.image)); - QByteArray srvMsg; - switch (baseline.status) { - case ImageItem::Ok: - break; - case ImageItem::IgnoreItem : - qDebug() << msg->constData() << "Ignored, blacklisted on server."; - return true; - break; - case ImageItem::BaselineNotFound: - if (!customInfo.overrides().isEmpty() || baselinePolicy == UploadNone) { - qWarning() << "Cannot compare to baseline: No such baseline found on server."; - return true; - } - if (proto.submitNewBaseline(item, &srvMsg)) - qDebug() << msg->constData() << "Baseline not found on server. New baseline uploaded."; - else - qDebug() << msg->constData() << "Baseline not found on server. Uploading of new baseline failed:" << srvMsg; - return true; - break; - default: - qWarning() << "Unexpected reply from baseline server."; - return true; - break; - } - *error = false; - // The actual comparison of the given image with the baseline: - if (baseline.imageChecksums.contains(item.imageChecksums.at(0))) { - if (!proto.submitMatch(item, &srvMsg)) - qWarning() << "Failed to report image match to server:" << srvMsg; - return true; - } - // At this point, we have established a legitimate mismatch - if (baselinePolicy == UploadAll) { - if (proto.submitNewBaseline(item, &srvMsg)) - qDebug() << msg->constData() << "Forcing new baseline; uploaded ok."; - else - qDebug() << msg->constData() << "Forcing new baseline; uploading failed:" << srvMsg; - return true; - } - bool fuzzyMatch = false; - bool res = proto.submitMismatch(item, &srvMsg, &fuzzyMatch); - if (res && fuzzyMatch) { - *error = true; // To force a QSKIP/debug output; somewhat kludgy - *msg += srvMsg; - return true; // The server decides: a fuzzy match means no mismatch - } - *msg += "Mismatch. See report:\n " + srvMsg; - if (dryRunMode) { - qDebug() << "Dryrun, so ignoring" << *msg; - return true; - } - return false; -} - -bool checkImage(const QImage &img, const char *name, quint16 checksum, QByteArray *msg, bool *error, int manualdatatag) -{ - if (!connected && !connect(msg, error)) - return true; - - QByteArray itemName; - bool hasName = qstrlen(name); - - const char *tag = QTest::currentDataTag(); - if (qstrlen(tag)) { - itemName = tag; - if (hasName) - itemName.append('_').append(name); - } else { - itemName = hasName ? name : "default_name"; - } - - if (manualdatatag > 0) - { - itemName.prepend("_"); - itemName.prepend(QByteArray::number(manualdatatag)); - } - - *msg = "Baseline check of image '" + itemName + "': "; - - - ImageItem item; - item.itemName = QString::fromLatin1(itemName); - item.itemChecksum = checksum; - item.testFunction = QString::fromLatin1(QTest::currentTestFunction()); - ImageItemList list; - list.append(item); - if (!proto.requestBaselineChecksums(QLatin1String(QTest::currentTestFunction()), &list) || list.isEmpty()) { - *msg = "Communication with baseline server failed: " + proto.errorMessage().toLatin1(); - *error = true; - return true; - } - - return compareItem(list.at(0), img, msg, error); -} - - -QTestData &newRow(const char *dataTag, quint16 checksum) -{ - if (QTest::currentTestFunction() != curFunction) { - curFunction = QTest::currentTestFunction(); - itemList.clear(); - gotBaselines = false; - } - ImageItem item; - item.itemName = QString::fromLatin1(dataTag); - item.itemChecksum = checksum; - item.testFunction = QString::fromLatin1(QTest::currentTestFunction()); - itemList.append(item); - - return QTest::newRow(dataTag); -} - - -bool testImage(const QImage& img, QByteArray *msg, bool *error) -{ - if (!connected && !connect(msg, error)) - return true; - - if (QTest::currentTestFunction() != curFunction || itemList.isEmpty()) { - qWarning() << "Usage error: QBASELINE_TEST used without corresponding QBaselineTest::newRow()"; - return true; - } - - if (!gotBaselines) { - if (!proto.requestBaselineChecksums(QString::fromLatin1(QTest::currentTestFunction()), &itemList) || itemList.isEmpty()) { - *msg = "Communication with baseline server failed: " + proto.errorMessage().toLatin1(); - *error = true; - return true; - } - gotBaselines = true; - } - - QString curTag = QString::fromLatin1(QTest::currentDataTag()); - ImageItemList::const_iterator it = itemList.constBegin(); - while (it != itemList.constEnd() && it->itemName != curTag) - ++it; - if (it == itemList.constEnd()) { - qWarning() << "Usage error: QBASELINE_TEST used without corresponding QBaselineTest::newRow() for row" << curTag; - return true; - } - return compareItem(*it, img, msg, error); -} - -} |