diff options
| -rw-r--r-- | tests/auto/lancelot/paintcommands.cpp | 4 | ||||
| -rw-r--r-- | tests/auto/lancelot/tst_lancelot.cpp | 21 | ||||
| -rw-r--r-- | tests/baselineserver/shared/baselineprotocol.cpp | 46 | ||||
| -rw-r--r-- | tests/baselineserver/shared/baselineprotocol.h | 13 | ||||
| -rw-r--r-- | tests/baselineserver/src/baselineserver.cpp | 74 | ||||
| -rw-r--r-- | tests/baselineserver/src/baselineserver.h | 7 | ||||
| -rw-r--r-- | tests/baselineserver/src/report.cpp | 49 | ||||
| -rw-r--r-- | tests/baselineserver/src/report.h | 1 |
8 files changed, 141 insertions, 74 deletions
diff --git a/tests/auto/lancelot/paintcommands.cpp b/tests/auto/lancelot/paintcommands.cpp index 8735baa3f0..9659659973 100644 --- a/tests/auto/lancelot/paintcommands.cpp +++ b/tests/auto/lancelot/paintcommands.cpp @@ -2306,7 +2306,11 @@ void PaintCommands::command_gradient_setRadialExtended(QRegExp re) "focal radius=%.2f, spread=%d\n", cx, cy, rad, fx, fy, frad, m_gradientSpread); +#if QT_VERSION >= 0x040800 QRadialGradient rg(QPointF(cx, cy), rad, QPointF(fx, fy), frad); +#else + QRadialGradient rg(QPointF(cx, cy), rad, QPointF(fx, fy)); +#endif rg.setStops(m_gradientStops); rg.setSpread(m_gradientSpread); rg.setCoordinateMode(m_gradientCoordinate); diff --git a/tests/auto/lancelot/tst_lancelot.cpp b/tests/auto/lancelot/tst_lancelot.cpp index cba5fab4d1..256ff29889 100644 --- a/tests/auto/lancelot/tst_lancelot.cpp +++ b/tests/auto/lancelot/tst_lancelot.cpp @@ -62,6 +62,7 @@ public: tst_Lancelot(); static bool simfail; + static PlatformInfo clientInfo; private: enum GraphicsEngine { @@ -98,6 +99,7 @@ private slots: }; bool tst_Lancelot::simfail = false; +PlatformInfo tst_Lancelot::clientInfo; tst_Lancelot::tst_Lancelot() { @@ -112,7 +114,7 @@ void tst_Lancelot::initTestCase() #if defined(Q_OS_SOMEPLATFORM) QSKIP("This test is not supported on this platform.", SkipAll); #endif - if (!proto.connect(QLatin1String("tst_Lancelot"), &dryRunMode)) + if (!proto.connect(QLatin1String("tst_Lancelot"), &dryRunMode, clientInfo)) QSKIP(qPrintable(proto.errorMessage()), SkipAll); #if defined(USE_RUNTIME_DIR) @@ -329,13 +331,26 @@ QTEST_MAIN(tst_Lancelot) int main(int argc, char *argv[]) { + tst_Lancelot::clientInfo = PlatformInfo::localHostInfo(); + char *fargv[20]; int fargc = 0; for (int i = 0; i < qMin(argc, 19); i++) { - if (!qstrcmp(argv[i], "-simfail")) + if (!qstrcmp(argv[i], "-simfail")) { tst_Lancelot::simfail = true; - else + } else if (!qstrcmp(argv[i], "-compareto") && i < argc-1) { + QString arg = QString::fromLocal8Bit(argv[++i]); + int split = arg.indexOf(QLC('=')); + if (split < 0) + continue; + QString key = arg.left(split).trimmed(); + QString value = arg.mid(split+1).trimmed(); + if (key.isEmpty() || value.isEmpty()) + continue; + tst_Lancelot::clientInfo.addOverride(key, value); + } else { fargv[fargc++] = argv[i]; + } } fargv[fargc] = 0; return rmain(fargc, fargv); diff --git a/tests/baselineserver/shared/baselineprotocol.cpp b/tests/baselineserver/shared/baselineprotocol.cpp index 0fe3aa22a9..630ca88e5b 100644 --- a/tests/baselineserver/shared/baselineprotocol.cpp +++ b/tests/baselineserver/shared/baselineprotocol.cpp @@ -83,7 +83,7 @@ void BaselineProtocol::sysSleep(int ms) } PlatformInfo::PlatformInfo() - : QMap<QString, QString>(), replaceDefault(false) + : QMap<QString, QString>(), adHoc(true) { } @@ -134,11 +134,15 @@ PlatformInfo PlatformInfo::localHostInfo() pi.insert(PI_GitCommit, QLS("Unknown")); QByteArray gb = qgetenv("PULSE_GIT_BRANCH"); - if (!gb.isEmpty()) + if (!gb.isEmpty()) { pi.insert(PI_PulseGitBranch, QString::fromLatin1(gb)); + pi.setAdHocRun(false); + } QByteArray tb = qgetenv("PULSE_TESTR_BRANCH"); - if (!tb.isEmpty()) + if (!tb.isEmpty()) { pi.insert(PI_PulseTestrBranch, QString::fromLatin1(tb)); + pi.setAdHocRun(false); + } return pi; } @@ -147,43 +151,49 @@ PlatformInfo PlatformInfo::localHostInfo() PlatformInfo::PlatformInfo(const PlatformInfo &other) : QMap<QString, QString>(other) { - sigKeys = other.sigKeys; - replaceDefault = other.replaceDefault; + orides = other.orides; + adHoc = other.adHoc; } PlatformInfo &PlatformInfo::operator=(const PlatformInfo &other) { QMap<QString, QString>::operator=(other); - sigKeys = other.sigKeys; - replaceDefault = other.replaceDefault; + orides = other.orides; + adHoc = other.adHoc; return *this; } -void PlatformInfo::addSignificantKeys(const QStringList &keys, bool replaceDefaultKeys) +void PlatformInfo::addOverride(const QString& key, const QString& value) +{ + orides.append(key); + orides.append(value); +} + + +QStringList PlatformInfo::overrides() const { - sigKeys = keys; - replaceDefault = replaceDefaultKeys; + return orides; } -QStringList PlatformInfo::addedKeys() const +void PlatformInfo::setAdHocRun(bool isAdHoc) { - return sigKeys; + adHoc = isAdHoc; } -bool PlatformInfo::addedKeysReplaceDefault() const +bool PlatformInfo::isAdHocRun() const { - return replaceDefault; + return adHoc; } QDataStream & operator<< (QDataStream &stream, const PlatformInfo &pi) { stream << static_cast<const QMap<QString, QString>&>(pi); - stream << pi.sigKeys << pi.replaceDefault; + stream << pi.orides << pi.adHoc; return stream; } @@ -191,7 +201,7 @@ QDataStream & operator<< (QDataStream &stream, const PlatformInfo &pi) QDataStream & operator>> (QDataStream &stream, PlatformInfo &pi) { stream >> static_cast<QMap<QString, QString>&>(pi); - stream >> pi.sigKeys >> pi.replaceDefault; + stream >> pi.orides >> pi.adHoc; return stream; } @@ -346,7 +356,7 @@ BaselineProtocol::~BaselineProtocol() } -bool BaselineProtocol::connect(const QString &testCase, bool *dryrun) +bool BaselineProtocol::connect(const QString &testCase, bool *dryrun, const PlatformInfo& clientInfo) { errMsg.clear(); QByteArray serverName(qgetenv("QT_LANCELOT_SERVER")); @@ -362,7 +372,7 @@ bool BaselineProtocol::connect(const QString &testCase, bool *dryrun) } } - PlatformInfo pi = PlatformInfo::localHostInfo(); + PlatformInfo pi = clientInfo.isEmpty() ? PlatformInfo::localHostInfo() : clientInfo; pi.insert(PI_TestCase, testCase); QByteArray block; QDataStream ds(&block, QIODevice::ReadWrite); diff --git a/tests/baselineserver/shared/baselineprotocol.h b/tests/baselineserver/shared/baselineprotocol.h index 2d09e683d4..0b08e444fd 100644 --- a/tests/baselineserver/shared/baselineprotocol.h +++ b/tests/baselineserver/shared/baselineprotocol.h @@ -78,13 +78,14 @@ public: static PlatformInfo localHostInfo(); - void addSignificantKeys(const QStringList& keys, bool replaceDefaultKeys=false); - QStringList addedKeys() const; - bool addedKeysReplaceDefault() const; + void addOverride(const QString& key, const QString& value); + QStringList overrides() const; + bool isAdHocRun() const; + void setAdHocRun(bool isAdHoc); private: - QStringList sigKeys; - bool replaceDefault; + QStringList orides; + bool adHoc; friend QDataStream & operator<< (QDataStream &stream, const PlatformInfo &pi); friend QDataStream & operator>> (QDataStream &stream, PlatformInfo& pi); }; @@ -165,7 +166,7 @@ public: // For client: // For advanced client: - bool connect(const QString &testCase, bool *dryrun = 0); + bool connect(const QString &testCase, bool *dryrun = 0, const PlatformInfo& clientInfo = PlatformInfo()); bool requestBaselineChecksums(const QString &testFunction, ImageItemList *itemList); bool submitNewBaseline(const ImageItem &item, QByteArray *serverMsg); bool submitMismatch(const ImageItem &item, QByteArray *serverMsg); diff --git a/tests/baselineserver/src/baselineserver.cpp b/tests/baselineserver/src/baselineserver.cpp index 6ff0a0c72d..5ac78d4b32 100644 --- a/tests/baselineserver/src/baselineserver.cpp +++ b/tests/baselineserver/src/baselineserver.cpp @@ -172,27 +172,31 @@ const char *BaselineHandler::logtime() bool BaselineHandler::establishConnection() { - if (!proto.acceptConnection(&plat)) { + if (!proto.acceptConnection(&clientInfo)) { qWarning() << runId << logtime() << "Accepting new connection from" << proto.socket.peerAddress().toString() << "failed." << proto.errorMessage(); proto.sendBlock(BaselineProtocol::Abort, proto.errorMessage().toLatin1()); // In case the client can hear us, tell it what's wrong. proto.socket.disconnectFromHost(); return false; } QString logMsg; - foreach (QString key, plat.keys()) { + foreach (QString key, clientInfo.keys()) { if (key != PI_HostName && key != PI_HostAddress) - logMsg += key + QLS(": '") + plat.value(key) + QLS("', "); + logMsg += key + QLS(": '") + clientInfo.value(key) + QLS("', "); } - qDebug() << runId << logtime() << "Connection established with" << plat.value(PI_HostName) - << "[" << qPrintable(plat.value(PI_HostAddress)) << "]" << logMsg; + qDebug() << runId << logtime() << "Connection established with" << clientInfo.value(PI_HostName) + << "[" << qPrintable(clientInfo.value(PI_HostAddress)) << "]" << logMsg + << "Overrides:" << clientInfo.overrides() << "AdHoc-Run:" << clientInfo.isAdHocRun(); + + //### Temporarily override the client setting, for client compatibility: + if (!clientInfo.isAdHocRun()) + clientInfo.setAdHocRun(clientInfo.value(PI_PulseGitBranch).isEmpty() && clientInfo.value(PI_PulseTestrBranch).isEmpty()); settings->beginGroup("ClientFilters"); - if (!settings->childKeys().isEmpty() && !plat.value(PI_PulseGitBranch).isEmpty()) { // i.e. not adhoc client - // Abort if client does not match the filters + if (!clientInfo.isAdHocRun()) { // for CI runs, allow filtering of clients. TBD: different filters (settings file) per testCase foreach (QString filterKey, settings->childKeys()) { QString filter = settings->value(filterKey).toString(); - QString platVal = plat.value(filterKey); - if (filter.isEmpty() || platVal.isEmpty()) + QString platVal = clientInfo.value(filterKey); + if (filter.isEmpty()) continue; // tbd: add a syntax for specifying a "value-must-be-present" filter if (!platVal.contains(filter)) { qDebug() << runId << logtime() << "Did not pass client filter on" << filterKey << "; disconnecting."; @@ -206,7 +210,7 @@ bool BaselineHandler::establishConnection() proto.sendBlock(BaselineProtocol::Ack, QByteArray()); - report.init(this, runId, plat); + report.init(this, runId, clientInfo); return true; } @@ -297,6 +301,12 @@ void BaselineHandler::storeImage(const QByteArray &itemBlock, bool isBaseline) ImageItem item; ds >> item; + if (isBaseline && !clientInfo.overrides().isEmpty()) { + qDebug() << runId << logtime() << "Received baseline from client with override info, ignoring. Item:" << item.itemName; + proto.sendBlock(BaselineProtocol::UnknownError, "New baselines not accepted from client with override info."); + return; + } + QString prefix = pathForItem(item, isBaseline); qDebug() << runId << logtime() << "Received" << (isBaseline ? "baseline" : "mismatched") << "image for:" << item.itemName << "Storing in" << prefix; @@ -313,7 +323,7 @@ void BaselineHandler::storeImage(const QByteArray &itemBlock, bool isBaseline) cwd.mkpath(dir); item.image.save(prefix + QLS(FileFormat), FileFormat); - PlatformInfo itemData = plat; + PlatformInfo itemData = clientInfo; itemData.insert(PI_ImageChecksum, QString::number(item.imageChecksums.at(0), 16)); //# Only the first is stored. TBD: get rid of list itemData.insert(PI_RunId, runId); itemData.insert(PI_CreationDate, QDateTime::currentDateTime().toString()); @@ -366,17 +376,16 @@ void BaselineHandler::receiveDisconnect() } -void BaselineHandler::mapPlatformInfo() const +PlatformInfo BaselineHandler::mapPlatformInfo(const PlatformInfo& orig) const { - mapped = plat; + PlatformInfo mapped = orig; // Map hostname - QString host = plat.value(PI_HostName).section(QLC('.'), 0, 0); // Filter away domain, if any + QString host = orig.value(PI_HostName).section(QLC('.'), 0, 0); // Filter away domain, if any if (host.isEmpty() || host == QLS("localhost")) { - host = plat.value(PI_HostAddress); + host = orig.value(PI_HostAddress); } else { - if (!plat.value(PI_PulseGitBranch).isEmpty()) { - // i.e. pulse run, so remove index postfix typical of vm hostnames + if (!orig.isAdHocRun()) { // i.e. CI system run, so remove index postfix typical of vm hostnames host.remove(QRegExp(QLS("\\d+$"))); if (host.endsWith(QLC('-'))) host.chop(1); @@ -387,18 +396,28 @@ void BaselineHandler::mapPlatformInfo() const mapped.insert(PI_HostName, host); // Map qmakespec - QString mkspec = plat.value(PI_QMakeSpec); + QString mkspec = orig.value(PI_QMakeSpec); mapped.insert(PI_QMakeSpec, mkspec.replace(QLC('/'), QLC('_'))); // Map Qt version - QString ver = plat.value(PI_QtVersion); + QString ver = orig.value(PI_QtVersion); mapped.insert(PI_QtVersion, ver.prepend(QLS("Qt-"))); + + return mapped; } + QString BaselineHandler::pathForItem(const ImageItem &item, bool isBaseline, bool absolute) const { - if (mapped.isEmpty()) - mapPlatformInfo(); + if (mappedClientInfo.isEmpty()) { + mappedClientInfo = mapPlatformInfo(clientInfo); + PlatformInfo oraw = clientInfo; + for (int i = 0; i < clientInfo.overrides().size()-1; i+=2) + oraw.insert(clientInfo.overrides().at(i), clientInfo.overrides().at(i+1)); + overriddenMappedClientInfo = mapPlatformInfo(oraw); + } + + const PlatformInfo& mapped = isBaseline ? overriddenMappedClientInfo : mappedClientInfo; QString itemName = item.itemName.simplified(); itemName.replace(QLC(' '), QLC('_')); @@ -512,13 +531,14 @@ void BaselineHandler::testPathMapping() item.imageChecksums << 0x0123456789abcdefULL; item.itemChecksum = 0x0123; - plat.insert(PI_QtVersion, QLS("5.0.0")); - plat.insert(PI_BuildKey, QLS("(nobuildkey)")); - plat.insert(PI_QMakeSpec, QLS("linux-g++")); - plat.insert(PI_PulseGitBranch, QLS("somebranch")); + clientInfo.insert(PI_QtVersion, QLS("5.0.0")); + clientInfo.insert(PI_BuildKey, QLS("(nobuildkey)")); + clientInfo.insert(PI_QMakeSpec, QLS("linux-g++")); + clientInfo.insert(PI_PulseGitBranch, QLS("somebranch")); + foreach(const QString& host, hosts) { - mapped.clear(); - plat.insert(PI_HostName, host); + mappedClientInfo.clear(); + clientInfo.insert(PI_HostName, host); qDebug() << "Baseline from" << host << "->" << pathForItem(item, true); qDebug() << "Mismatch from" << host << "->" << pathForItem(item, false); } diff --git a/tests/baselineserver/src/baselineserver.h b/tests/baselineserver/src/baselineserver.h index d73bb974f9..333d9ed30b 100644 --- a/tests/baselineserver/src/baselineserver.h +++ b/tests/baselineserver/src/baselineserver.h @@ -125,13 +125,14 @@ private: void storeImage(const QByteArray &itemBlock, bool isBaseline); void storeItemMetadata(const PlatformInfo &metadata, const QString &path); PlatformInfo fetchItemMetadata(const QString &path); - void mapPlatformInfo() const; + PlatformInfo mapPlatformInfo(const PlatformInfo& orig) const; const char *logtime(); QString computeMismatchScore(const QImage& baseline, const QImage& rendered); BaselineProtocol proto; - PlatformInfo plat; - mutable PlatformInfo mapped; + PlatformInfo clientInfo; + mutable PlatformInfo mappedClientInfo; + mutable PlatformInfo overriddenMappedClientInfo; QString runId; bool connectionEstablished; Report report; diff --git a/tests/baselineserver/src/report.cpp b/tests/baselineserver/src/report.cpp index 7c2d6ac6df..16f061c7e7 100644 --- a/tests/baselineserver/src/report.cpp +++ b/tests/baselineserver/src/report.cpp @@ -66,12 +66,13 @@ void Report::init(const BaselineHandler *h, const QString &r, const PlatformInfo runId = r; plat = p; rootDir = BaselineServer::storagePath() + QLC('/'); - reportDir = plat.value(PI_TestCase) + QLC('/') + (plat.value(PI_PulseGitBranch).isEmpty() ? QLS("reports/adhoc/") : QLS("reports/pulse/")); + reportDir = plat.value(PI_TestCase) + QLC('/') + (plat.isAdHocRun() ? QLS("reports/adhoc/") : QLS("reports/pulse/")); QString dir = rootDir + reportDir; QDir cwd; if (!cwd.exists(dir)) cwd.mkpath(dir); path = reportDir + QLS("Report_") + runId + QLS(".html"); + hasOverride = !plat.overrides().isEmpty(); } void Report::addItems(const ImageItemList &items) @@ -141,11 +142,19 @@ void Report::writeHeader() << "<p>Note: This is a <i>static</i> page, generated at " << QDateTime::currentDateTime().toString() << " for the test run with id " << runId << "</p>\n" << "<p>Summary: <b><span style=\"color:red\">" << numMismatches << " of " << numItems << "</b></span> items reported mismatching</p>\n\n"; - out << "<h3>Platform Info:</h3>\n" + out << "<h3>Testing Client Platform Info:</h3>\n" << "<table>\n"; foreach (QString key, plat.keys()) - out << "<tr><td>" << key << "</td><td>" << plat.value(key) << "</td></tr>\n"; + out << "<tr><td>" << key << ":</td><td>" << plat.value(key) << "</td></tr>\n"; out << "</table>\n\n"; + if (hasOverride) { + out << "<span style=\"color:red\"><h4>Note! Platform Override Info:</h4></span>\n" + << "<p>The client's output has been compared to baselines created on a different platform. Differences:</p>\n" + << "<table>\n"; + for (int i = 0; i < plat.overrides().size()-1; i+=2) + out << "<tr><td>" << plat.overrides().at(i) << ":</td><td>" << plat.overrides().at(i+1) << "</td></tr>\n"; + out << "</table>\n\n"; + } } @@ -158,10 +167,12 @@ void Report::writeFunctionResults(const ImageItemList &list) out << "\n<p> </p><h3>Test function: " << testFunction << "</h3>\n"; - out << "<p><a href=\"/cgi-bin/server.cgi?cmd=clearAllBaselines&context=" << ctx << "&url=" << pageUrl - << "\"><b>Clear all baselines</b></a> for this testfunction (They will be recreated by the next run)</p>\n"; - out << "<p><a href=\"/cgi-bin/server.cgi?cmd=updateAllBaselines&context=" << ctx << "&mismatchContext=" << misCtx << "&url=" << pageUrl - << "\"><b>Let these mismatching images be the new baselines</b></a> for this testfunction</p>\n\n"; + if (!hasOverride) { + out << "<p><a href=\"/cgi-bin/server.cgi?cmd=clearAllBaselines&context=" << ctx << "&url=" << pageUrl + << "\"><b>Clear all baselines</b></a> for this testfunction (They will be recreated by the next run)</p>\n"; + out << "<p><a href=\"/cgi-bin/server.cgi?cmd=updateAllBaselines&context=" << ctx << "&mismatchContext=" << misCtx << "&url=" << pageUrl + << "\"><b>Let these mismatching images be the new baselines</b></a> for this testfunction</p>\n\n"; + } out << "<table border=\"2\">\n" "<tr>\n" @@ -192,10 +203,12 @@ void Report::writeFunctionResults(const ImageItemList &list) out << "Baseline not found/regenerated"; break; case ImageItem::IgnoreItem: - out << "<span style=\"background-color:yellow\">Blacklisted</span> " - << "<a href=\"/cgi-bin/server.cgi?cmd=whitelist&context=" << ctx - << "&itemId=" << item.itemName << "&url=" << pageUrl - << "\">Whitelist this item</a>"; + out << "<span style=\"background-color:yellow\">Blacklisted</span> "; + if (!hasOverride) { + out << "<a href=\"/cgi-bin/server.cgi?cmd=whitelist&context=" << ctx + << "&itemId=" << item.itemName << "&url=" << pageUrl + << "\">Whitelist this item</a>"; + } break; case ImageItem::Ok: out << "<span style=\"color:green\"><small>No mismatch reported</small></span>"; @@ -224,12 +237,14 @@ void Report::writeItem(const QString &baseline, const QString &rendered, const I out << "<td align=center>\n" << "<p><span style=\"color:red\">Mismatch reported</span></p>\n" - << "<p><a href=\"/" << metadata << "\">Baseline Info</a>\n" - << "<p><a href=\"/cgi-bin/server.cgi?cmd=updateSingleBaseline&context=" << ctx << "&mismatchContext=" << misCtx - << "&itemFile=" << itemFile << "&url=" << pageUrl << "\">Let this be the new baseline</a></p>\n" - << "<p><a href=\"/cgi-bin/server.cgi?cmd=blacklist&context=" << ctx - << "&itemId=" << item.itemName << "&url=" << pageUrl << "\">Blacklist this item</a></p>\n" - << "<p><a href=\"/cgi-bin/server.cgi?cmd=view&baseline=" << baseline << "&rendered=" << rendered + << "<p><a href=\"/" << metadata << "\">Baseline Info</a>\n"; + if (!hasOverride) { + out << "<p><a href=\"/cgi-bin/server.cgi?cmd=updateSingleBaseline&context=" << ctx << "&mismatchContext=" << misCtx + << "&itemFile=" << itemFile << "&url=" << pageUrl << "\">Let this be the new baseline</a></p>\n" + << "<p><a href=\"/cgi-bin/server.cgi?cmd=blacklist&context=" << ctx + << "&itemId=" << item.itemName << "&url=" << pageUrl << "\">Blacklist this item</a></p>\n"; + } + out << "<p><a href=\"/cgi-bin/server.cgi?cmd=view&baseline=" << baseline << "&rendered=" << rendered << "&compared=" << compared << "&url=" << pageUrl << "\">Inspect</a></p>\n" << "</td>\n"; } diff --git a/tests/baselineserver/src/report.h b/tests/baselineserver/src/report.h index d21102d32f..610497c84b 100644 --- a/tests/baselineserver/src/report.h +++ b/tests/baselineserver/src/report.h @@ -86,6 +86,7 @@ private: int numItems; int numMismatches; QTextStream out; + bool hasOverride; }; #endif // REPORT_H |