diff options
Diffstat (limited to 'tests/baselineserver/shared')
-rw-r--r-- | tests/baselineserver/shared/baselineprotocol.cpp | 29 | ||||
-rw-r--r-- | tests/baselineserver/shared/baselineprotocol.h | 13 | ||||
-rw-r--r-- | tests/baselineserver/shared/qbaselinetest.cpp | 248 | ||||
-rw-r--r-- | tests/baselineserver/shared/qbaselinetest.h | 8 |
4 files changed, 268 insertions, 30 deletions
diff --git a/tests/baselineserver/shared/baselineprotocol.cpp b/tests/baselineserver/shared/baselineprotocol.cpp index b8e141374f..e800b76fec 100644 --- a/tests/baselineserver/shared/baselineprotocol.cpp +++ b/tests/baselineserver/shared/baselineprotocol.cpp @@ -50,6 +50,7 @@ #include <QTime> #include <QPointer> +const QString PI_Project(QLS("Project")); const QString PI_TestCase(QLS("TestCase")); const QString PI_HostName(QLS("HostName")); const QString PI_HostAddress(QLS("HostAddress")); @@ -357,9 +358,13 @@ BaselineProtocol::BaselineProtocol() BaselineProtocol::~BaselineProtocol() { + disconnect(); +} + +bool BaselineProtocol::disconnect() +{ socket.close(); - if (socket.state() != QTcpSocket::UnconnectedState) - socket.waitForDisconnected(Timeout); + return (socket.state() == QTcpSocket::UnconnectedState) ? true : socket.waitForDisconnected(Timeout); } @@ -372,7 +377,7 @@ bool BaselineProtocol::connect(const QString &testCase, bool *dryrun, const Plat socket.connectToHost(serverName, ServerPort); if (!socket.waitForConnected(Timeout)) { - sysSleep(Timeout); // Wait a bit and try again, the server might just be restarting + sysSleep(3000); // Wait a bit and try again, the server might just be restarting if (!socket.waitForConnected(Timeout)) { errMsg += QLS("TCP connectToHost failed. Host:") + serverName + QLS(" port:") + QString::number(ServerPort); return false; @@ -456,6 +461,15 @@ bool BaselineProtocol::requestBaselineChecksums(const QString &testFunction, Ima } +bool BaselineProtocol::submitMatch(const ImageItem &item, QByteArray *serverMsg) +{ + Command cmd; + ImageItem smallItem = item; + smallItem.image = QImage(); // No need to waste bandwith sending image (identical to baseline) to server + return (sendItem(AcceptMatch, smallItem) && receiveBlock(&cmd, serverMsg) && cmd == Ack); +} + + bool BaselineProtocol::submitNewBaseline(const ImageItem &item, QByteArray *serverMsg) { Command cmd; @@ -463,10 +477,15 @@ bool BaselineProtocol::submitNewBaseline(const ImageItem &item, QByteArray *serv } -bool BaselineProtocol::submitMismatch(const ImageItem &item, QByteArray *serverMsg) +bool BaselineProtocol::submitMismatch(const ImageItem &item, QByteArray *serverMsg, bool *fuzzyMatch) { Command cmd; - return (sendItem(AcceptMismatch, item) && receiveBlock(&cmd, serverMsg) && cmd == Ack); + if (sendItem(AcceptMismatch, item) && receiveBlock(&cmd, serverMsg) && (cmd == Ack || cmd == FuzzyMatch)) { + if (fuzzyMatch) + *fuzzyMatch = (cmd == FuzzyMatch); + return true; + } + return false; } diff --git a/tests/baselineserver/shared/baselineprotocol.h b/tests/baselineserver/shared/baselineprotocol.h index 61feaa34a9..a5f873e3d5 100644 --- a/tests/baselineserver/shared/baselineprotocol.h +++ b/tests/baselineserver/shared/baselineprotocol.h @@ -55,6 +55,7 @@ #define FileFormat "png" +extern const QString PI_Project; extern const QString PI_TestCase; extern const QString PI_HostName; extern const QString PI_HostAddress; @@ -111,7 +112,9 @@ public: Ok = 0, BaselineNotFound = 1, IgnoreItem = 2, - Mismatch = 3 + Mismatch = 3, + FuzzyMatch = 4, + Error = 5 }; QString testFunction; @@ -155,21 +158,25 @@ public: // Queries AcceptPlatformInfo = 1, RequestBaselineChecksums = 2, + AcceptMatch = 3, AcceptNewBaseline = 4, AcceptMismatch = 5, // Responses Ack = 128, Abort = 129, - DoDryRun = 130 + DoDryRun = 130, + FuzzyMatch = 131 }; // For client: // For advanced client: bool connect(const QString &testCase, bool *dryrun = 0, const PlatformInfo& clientInfo = PlatformInfo()); + bool disconnect(); bool requestBaselineChecksums(const QString &testFunction, ImageItemList *itemList); + bool submitMatch(const ImageItem &item, QByteArray *serverMsg); bool submitNewBaseline(const ImageItem &item, QByteArray *serverMsg); - bool submitMismatch(const ImageItem &item, QByteArray *serverMsg); + bool submitMismatch(const ImageItem &item, QByteArray *serverMsg, bool *fuzzyMatch = 0); // For server: bool acceptConnection(PlatformInfo *pi); diff --git a/tests/baselineserver/shared/qbaselinetest.cpp b/tests/baselineserver/shared/qbaselinetest.cpp index 11de0421e9..0c28f6eb46 100644 --- a/tests/baselineserver/shared/qbaselinetest.cpp +++ b/tests/baselineserver/shared/qbaselinetest.cpp @@ -41,44 +41,233 @@ #include "qbaselinetest.h" #include "baselineprotocol.h" +#include <QtCore/QProcess> +#include <QtCore/QDir> + +#define MAXCMDLINEARGS 128 namespace QBaselineTest { -BaselineProtocol proto; -bool connected = false; -bool triedConnecting = false; +static char *fargv[MAXCMDLINEARGS]; +static bool simfail = false; +static PlatformInfo customInfo; -QByteArray curFunction; -ImageItemList itemList; -bool gotBaselines; +static BaselineProtocol proto; +static bool connected = false; +static bool triedConnecting = false; +static QByteArray curFunction; +static ImageItemList itemList; +static bool gotBaselines; -bool connect(QByteArray *msg, bool *error) +static QString definedTestProject; +static QString definedTestCase; + + +void handleCmdLineArgs(int *argcp, char ***argvp) { - if (!triedConnecting) { - triedConnecting = true; - if (!proto.connect(QTest::testObject()->metaObject()->className())) { - *msg += "Failed to connect to baseline server: " + proto.errorMessage().toLatin1(); - *error = true; - return false; + if (!argcp || !argvp) + return; + + bool showHelp = false; + + int fargc = 0; + int numArgs = *argcp; + + for (int i = 0; i < numArgs; i++) { + QByteArray arg = (*argvp)[i]; + QByteArray nextArg = (i+1 < numArgs) ? (*argvp)[i+1] : 0; + + if (arg == "-simfail") { + simfail = true; + } else if (arg == "-auto") { + customInfo.setAdHocRun(false); + } else if (arg == "-adhoc") { + customInfo.setAdHocRun(true); + } else if (arg == "-compareto") { + i++; + int split = qMax(0, nextArg.indexOf('=')); + QByteArray key = nextArg.left(split).trimmed(); + QByteArray value = nextArg.mid(split+1).trimmed(); + if (key.isEmpty() || value.isEmpty()) { + qWarning() << "-compareto requires parameter of the form <key>=<value>"; + showHelp = true; + break; + } + customInfo.addOverride(key, value); + } else { + if ( (arg == "-help") || (arg == "--help") ) + showHelp = true; + if (fargc >= MAXCMDLINEARGS) { + qWarning() << "Too many command line arguments!"; + break; + } + fargv[fargc++] = (*argvp)[i]; + } + } + *argcp = fargc; + *argvp = fargv; + + if (showHelp) { + // TBD: arrange for this to be printed *after* QTest's help + QTextStream out(stdout); + out << "\n Baseline testing (lancelot) options:\n"; + out << " -simfail : Force an image comparison mismatch. For testing purposes.\n"; + out << " -auto : Inform server that this run is done by a daemon, CI system or similar.\n"; + out << " -adhoc (default) : The inverse of -auto; this run is done by human, e.g. for testing.\n"; + out << " -compareto KEY=VAL : Force comparison to baselines from a different client,\n"; + out << " for example: -compareto QtVersion=4.8.0\n"; + out << " Multiple -compareto client specifications may be given.\n"; + out << "\n"; + } +} + + +void addClientProperty(const QString& key, const QString& value) +{ + customInfo.insert(key, value); +} + + +/* + If a client property script is present, run it and accept its output + in the form of one 'key: value' property per line +*/ +void fetchCustomClientProperties() +{ + QString script = "hostinfo.sh"; //### TBD: better name + + QProcess runScript; + runScript.setWorkingDirectory(QCoreApplication::applicationDirPath()); + runScript.start("sh", QStringList() << script, QIODevice::ReadOnly); + if (!runScript.waitForFinished(5000) || runScript.error() != QProcess::UnknownError) { + qWarning() << "QBaselineTest: Error running script" << runScript.workingDirectory() + QDir::separator() + script << ":" << runScript.errorString(); + qDebug() << " stderr:" << runScript.readAllStandardError().trimmed(); + } + while (!runScript.atEnd()) { + QByteArray line = runScript.readLine().trimmed(); // ###local8bit? utf8? + QString key, val; + int colonPos = line.indexOf(':'); + if (colonPos > 0) { + key = line.left(colonPos).simplified().replace(' ', '_'); + val = line.mid(colonPos+1).trimmed(); } - connected = true; + if (!key.isEmpty() && key.length() < 64 && val.length() < 256) // ###TBD: maximum 256 chars in value? + addClientProperty(key, val); + else + qDebug() << "Unparseable script output ignored:" << line; + } +} + + +bool connect(QByteArray *msg, bool *error) +{ + if (connected) { + return true; } - if (!connected) { + else if (triedConnecting) { + // Avoid repeated connection attempts, to avoid the program using Timeout * #testItems seconds before giving up *msg = "Not connected to baseline server."; *error = true; return false; } + + triedConnecting = true; + fetchCustomClientProperties(); + // Merge the platform info set by the program with the protocols default info + PlatformInfo clientInfo = customInfo; + PlatformInfo defaultInfo = PlatformInfo::localHostInfo(); + foreach (QString key, defaultInfo.keys()) { + if (!clientInfo.contains(key)) + clientInfo.insert(key, defaultInfo.value(key)); + } + + if (!definedTestProject.isEmpty()) + clientInfo.insert(PI_Project, definedTestProject); + + QString testCase = definedTestCase; + if (testCase.isEmpty() && QTest::testObject() && QTest::testObject()->metaObject()) { + //qDebug() << "Trying to Read TestCaseName from Testlib!"; + testCase = QTest::testObject()->metaObject()->className(); + } + if (testCase.isEmpty()) { + qWarning("QBaselineTest::connect: No test case name specified, cannot connect."); + return false; + } + + bool dummy; // ### TBD: dryrun handling + if (!proto.connect(testCase, &dummy, clientInfo)) { + *msg += "Failed to connect to baseline server: " + proto.errorMessage().toLatin1(); + *error = true; + return false; + } + connected = true; return true; } +bool disconnectFromBaselineServer() +{ + if (proto.disconnect()) { + connected = false; + triedConnecting = false; + return true; + } + + return false; +} + +bool connectToBaselineServer(QByteArray *msg, const QString &testProject, const QString &testCase) +{ + bool dummy; + QByteArray dummyMsg; + + definedTestProject = testProject; + definedTestCase = testCase; + + return connect(msg ? msg : &dummyMsg, &dummy); +} + +void setAutoMode(bool mode) +{ + customInfo.setAdHocRun(!mode); +} + +void setSimFail(bool fail) +{ + simfail = fail; +} + + +void modifyImage(QImage *img) +{ + uint c0 = 0x0000ff00; + uint c1 = 0x0080ff00; + img->setPixel(1,1,c0); + img->setPixel(2,1,c1); + img->setPixel(3,1,c0); + img->setPixel(1,2,c1); + img->setPixel(1,3,c0); + img->setPixel(2,3,c1); + img->setPixel(3,3,c0); + img->setPixel(1,4,c1); + img->setPixel(1,5,c0); +} + bool compareItem(const ImageItem &baseline, const QImage &img, QByteArray *msg, bool *error) { ImageItem item = baseline; - item.image = img; + if (simfail) { + // Simulate test failure by forcing image mismatch; for testing purposes + QImage misImg = img; + modifyImage(&misImg); + item.image = misImg; + simfail = false; // One failure is typically enough + } else { + item.image = img; + } item.imageChecksums.clear(); - item.imageChecksums.prepend(ImageItem::computeChecksum(img)); + item.imageChecksums.prepend(ImageItem::computeChecksum(item.image)); QByteArray srvMsg; switch (baseline.status) { case ImageItem::Ok: @@ -88,6 +277,7 @@ bool compareItem(const ImageItem &baseline, const QImage &img, QByteArray *msg, return true; break; case ImageItem::BaselineNotFound: + // ### TBD: don't submit if have overrides; will be rejected anyway if (proto.submitNewBaseline(item, &srvMsg)) qDebug() << msg->constData() << "Baseline not found on server. New baseline uploaded."; else @@ -101,27 +291,43 @@ bool compareItem(const ImageItem &baseline, const QImage &img, QByteArray *msg, } *error = false; // The actual comparison of the given image with the baseline: - if (baseline.imageChecksums.contains(item.imageChecksums.at(0))) + if (baseline.imageChecksums.contains(item.imageChecksums.at(0))) { + if (!proto.submitMatch(item, &srvMsg)) + qWarning() << "Failed to report image match to server:" << srvMsg; return true; - proto.submitMismatch(item, &srvMsg); + } + bool fuzzyMatch = false; + bool res = proto.submitMismatch(item, &srvMsg, &fuzzyMatch); + if (res && fuzzyMatch) { + *error = true; // To force a QSKIP/debug output; somewhat kludgy + *msg += srvMsg; + return true; // The server decides: a fuzzy match means no mismatch + } *msg += "Mismatch. See report:\n " + srvMsg; return false; } -bool checkImage(const QImage &img, const char *name, quint16 checksum, QByteArray *msg, bool *error) +bool checkImage(const QImage &img, const char *name, quint16 checksum, QByteArray *msg, bool *error, int manualdatatag) { if (!connected && !connect(msg, error)) return true; QByteArray itemName; bool hasName = qstrlen(name); + const char *tag = QTest::currentDataTag(); if (qstrlen(tag)) { itemName = tag; if (hasName) itemName.append('_').append(name); } else { - itemName = hasName ? name : "default_name"; + itemName = hasName ? name : "default_name"; + } + + if (manualdatatag > 0) + { + itemName.prepend("_"); + itemName.prepend(QByteArray::number(manualdatatag)); } *msg = "Baseline check of image '" + itemName + "': "; diff --git a/tests/baselineserver/shared/qbaselinetest.h b/tests/baselineserver/shared/qbaselinetest.h index 40f4160e60..0bcfefa056 100644 --- a/tests/baselineserver/shared/qbaselinetest.h +++ b/tests/baselineserver/shared/qbaselinetest.h @@ -45,9 +45,15 @@ #include <QTest> namespace QBaselineTest { -bool checkImage(const QImage& img, const char *name, quint16 checksum, QByteArray *msg, bool *error); +void setAutoMode(bool mode); +void setSimFail(bool fail); +void handleCmdLineArgs(int *argcp, char ***argvp); +void addClientProperty(const QString& key, const QString& value); +bool connectToBaselineServer(QByteArray *msg = 0, const QString &testProject = QString(), const QString &testCase = QString()); +bool checkImage(const QImage& img, const char *name, quint16 checksum, QByteArray *msg, bool *error, int manualdatatag = 0); bool testImage(const QImage& img, QByteArray *msg, bool *error); QTestData &newRow(const char *dataTag, quint16 checksum = 0); +bool disconnectFromBaselineServer(); } #define QBASELINE_CHECK_SUM(image, name, checksum)\ |