diff options
Diffstat (limited to 'tests/baselineserver/src')
| -rw-r--r-- | tests/baselineserver/src/baselineserver.cpp | 422 | ||||
| -rw-r--r-- | tests/baselineserver/src/baselineserver.h | 32 | ||||
| -rw-r--r-- | tests/baselineserver/src/report.cpp | 270 | ||||
| -rw-r--r-- | tests/baselineserver/src/report.h | 27 | ||||
| -rw-r--r-- | tests/baselineserver/src/templates/view.html | 9 |
5 files changed, 632 insertions, 128 deletions
diff --git a/tests/baselineserver/src/baselineserver.cpp b/tests/baselineserver/src/baselineserver.cpp index 1653754333..a85dca375a 100644 --- a/tests/baselineserver/src/baselineserver.cpp +++ b/tests/baselineserver/src/baselineserver.cpp @@ -52,6 +52,7 @@ #include <QTextStream> #include <QProcess> #include <QDirIterator> +#include <QUrl> // extra fields, for use in image metadata storage const QString PI_ImageChecksum(QLS("ImageChecksum")); @@ -60,7 +61,7 @@ const QString PI_CreationDate(QLS("CreationDate")); QString BaselineServer::storage; QString BaselineServer::url; -QString BaselineServer::settingsFile; +QStringList BaselineServer::pathKeys; BaselineServer::BaselineServer(QObject *parent) : QTcpServer(parent), lastRunIdIdx(0) @@ -92,13 +93,11 @@ QString BaselineServer::baseUrl() return url; } -QString BaselineServer::settingsFilePath() +QStringList BaselineServer::defaultPathKeys() { - if (settingsFile.isEmpty()) { - QString exeName = QCoreApplication::applicationFilePath().section(QLC('/'), -1); - settingsFile = storagePath() + QLC('/') + exeName + QLS(".ini"); - } - return settingsFile; + if (pathKeys.isEmpty()) + pathKeys << PI_QtVersion << PI_QMakeSpec << PI_HostName; + return pathKeys; } void BaselineServer::incomingConnection(qintptr socketDescriptor) @@ -152,9 +151,13 @@ void BaselineThread::run() BaselineHandler::BaselineHandler(const QString &runId, int socketDescriptor) - : QObject(), runId(runId), connectionEstablished(false) + : QObject(), runId(runId), connectionEstablished(false), settings(0), fuzzLevel(0) { - settings = new QSettings(BaselineServer::settingsFilePath(), QSettings::IniFormat, this); + idleTimer = new QTimer(this); + idleTimer->setSingleShot(true); + idleTimer->setInterval(IDLE_CLIENT_TIMEOUT * 1000); + connect(idleTimer, SIGNAL(timeout()), this, SLOT(idleClientTimeout())); + idleTimer->start(); if (socketDescriptor == -1) return; @@ -162,6 +165,7 @@ BaselineHandler::BaselineHandler(const QString &runId, int socketDescriptor) connect(&proto.socket, SIGNAL(readyRead()), this, SLOT(receiveRequest())); connect(&proto.socket, SIGNAL(disconnected()), this, SLOT(receiveDisconnect())); proto.socket.setSocketDescriptor(socketDescriptor); + proto.socket.setSocketOption(QAbstractSocket::KeepAliveOption, 1); } const char *BaselineHandler::logtime() @@ -170,6 +174,85 @@ const char *BaselineHandler::logtime() //return QTime::currentTime().toString(QLS("mm:ss.zzz")); } +QString BaselineHandler::projectPath(bool absolute) const +{ + QString p = clientInfo.value(PI_Project); + return absolute ? BaselineServer::storagePath() + QLC('/') + p : p; +} + +bool BaselineHandler::checkClient(QByteArray *errMsg, bool *dryRunMode) +{ + if (!errMsg) + return false; + if (clientInfo.value(PI_Project).isEmpty() || clientInfo.value(PI_TestCase).isEmpty()) { + *errMsg = "No Project and/or TestCase specified in client info."; + return false; + } + + // Determine ad-hoc state ### hardcoded for now + if (clientInfo.value(PI_TestCase) == QLS("tst_Lancelot")) { + //### Todo: push this stuff out in a script + if (!clientInfo.isAdHocRun()) { + // ### comp. with earlier versions still running (4.8) (?) + clientInfo.setAdHocRun(clientInfo.value(PI_PulseGitBranch).isEmpty() && clientInfo.value(PI_PulseTestrBranch).isEmpty()); + } + } + else { + // TBD + } + + if (clientInfo.isAdHocRun()) { + if (dryRunMode) + *dryRunMode = false; + return true; + } + + // Not ad hoc: filter the client + settings->beginGroup("ClientFilters"); + bool matched = false; + bool dryRunReq = false; + foreach (const QString &rule, settings->childKeys()) { + //qDebug() << " > RULE" << rule; + dryRunReq = false; + QString ruleMode = settings->value(rule).toString().toLower(); + if (ruleMode == QLS("dryrun")) + dryRunReq = true; + else if (ruleMode != QLS("enabled")) + continue; + settings->beginGroup(rule); + bool ruleMatched = true; + foreach (const QString &filterKey, settings->childKeys()) { + //qDebug() << " > FILTER" << filterKey; + QString filter = settings->value(filterKey).toString(); + if (filter.isEmpty()) + continue; + QString platVal = clientInfo.value(filterKey); + if (!platVal.contains(filter)) { + ruleMatched = false; + break; + } + } + if (ruleMatched) { + ruleName = rule; + matched = true; + break; + } + settings->endGroup(); + } + + if (!matched && errMsg) + *errMsg = "Non-adhoc client did not match any filter rule in " + settings->fileName().toLatin1(); + + if (matched && dryRunMode) + *dryRunMode = dryRunReq; + + // NB! Must reset the settings object before returning + while (!settings->group().isEmpty()) + settings->endGroup(); + + return matched; +} + bool BaselineHandler::establishConnection() { if (!proto.acceptConnection(&clientInfo)) { @@ -187,35 +270,46 @@ bool BaselineHandler::establishConnection() << "[" << qPrintable(clientInfo.value(PI_HostAddress)) << "]" << logMsg << "Overrides:" << clientInfo.overrides() << "AdHoc-Run:" << clientInfo.isAdHocRun(); - //### Temporarily override the client setting, for client compatibility: - if (!clientInfo.isAdHocRun()) - clientInfo.setAdHocRun(clientInfo.value(PI_PulseGitBranch).isEmpty() && clientInfo.value(PI_PulseTestrBranch).isEmpty()); + // ### Hardcoded backwards compatibility: add project field for certain existing clients that lack it + if (clientInfo.value(PI_Project).isEmpty()) { + QString tc = clientInfo.value(PI_TestCase); + if (tc == QLS("tst_Lancelot")) + clientInfo.insert(PI_Project, QLS("Raster")); + else if (tc == QLS("tst_Scenegraph")) + clientInfo.insert(PI_Project, QLS("SceneGraph")); + else + clientInfo.insert(PI_Project, QLS("Other")); + } - settings->beginGroup("ClientFilters"); - if (!clientInfo.isAdHocRun()) { // for CI runs, allow filtering of clients. TBD: different filters (settings file) per testCase - foreach (QString filterKey, settings->childKeys()) { - QString filter = settings->value(filterKey).toString(); - QString platVal = clientInfo.value(filterKey); - if (filter.isEmpty()) - continue; // tbd: add a syntax for specifying a "value-must-be-present" filter - if (!platVal.contains(filter)) { - qDebug() << runId << logtime() << "Did not pass client filter on" << filterKey << "; disconnecting."; - proto.sendBlock(BaselineProtocol::Abort, QByteArray("Configured to not do testing for this client or repo, ref. ") + BaselineServer::settingsFilePath().toLatin1()); - proto.socket.disconnectFromHost(); - return false; - } - } + QString settingsFile = projectPath() + QLS("/config.ini"); + settings = new QSettings(settingsFile, QSettings::IniFormat, this); + + QByteArray errMsg; + bool dryRunMode = false; + if (!checkClient(&errMsg, &dryRunMode)) { + qDebug() << runId << logtime() << "Rejecting connection:" << errMsg; + proto.sendBlock(BaselineProtocol::Abort, errMsg); + proto.socket.disconnectFromHost(); + return false; } - settings->endGroup(); - proto.sendBlock(BaselineProtocol::Ack, QByteArray()); + fuzzLevel = qBound(0, settings->value("FuzzLevel").toInt(), 100); + if (!clientInfo.isAdHocRun()) { + qDebug() << runId << logtime() << "Client matches filter rule" << ruleName + << "Dryrun:" << dryRunMode + << "FuzzLevel:" << fuzzLevel + << "ReportMissingResults:" << settings->value("ReportMissingResults").toBool(); + } - report.init(this, runId, clientInfo); + proto.sendBlock(dryRunMode ? BaselineProtocol::DoDryRun : BaselineProtocol::Ack, QByteArray()); + report.init(this, runId, clientInfo, settings); return true; } void BaselineHandler::receiveRequest() { + idleTimer->start(); // Restart idle client timeout + if (!connectionEstablished) { connectionEstablished = establishConnection(); return; @@ -233,6 +327,9 @@ void BaselineHandler::receiveRequest() case BaselineProtocol::RequestBaselineChecksums: provideBaselineChecksums(block); break; + case BaselineProtocol::AcceptMatch: + recordMatch(block); + break; case BaselineProtocol::AcceptNewBaseline: storeImage(block, true); break; @@ -295,6 +392,16 @@ void BaselineHandler::provideBaselineChecksums(const QByteArray &itemListBlock) } +void BaselineHandler::recordMatch(const QByteArray &itemBlock) +{ + QDataStream ds(itemBlock); + ImageItem item; + ds >> item; + report.addResult(item); + proto.sendBlock(BaselineProtocol::Ack, QByteArray()); +} + + void BaselineHandler::storeImage(const QByteArray &itemBlock, bool isBaseline) { QDataStream ds(itemBlock); @@ -307,16 +414,23 @@ void BaselineHandler::storeImage(const QByteArray &itemBlock, bool isBaseline) return; } - QString prefix = pathForItem(item, isBaseline); + QString blPrefix = pathForItem(item, true); + QString mmPrefix = pathForItem(item, false); + QString prefix = isBaseline ? blPrefix : mmPrefix; + qDebug() << runId << logtime() << "Received" << (isBaseline ? "baseline" : "mismatched") << "image for:" << item.itemName << "Storing in" << prefix; + // Reply to the client QString msg; if (isBaseline) - msg = QLS("New baseline image stored: ") + pathForItem(item, true, true) + QLS(FileFormat); + msg = QLS("New baseline image stored: ") + blPrefix + QLS(FileFormat); else msg = BaselineServer::baseUrl() + report.filePath(); - proto.sendBlock(BaselineProtocol::Ack, msg.toLatin1()); + if (isBaseline || !fuzzLevel) + proto.sendBlock(BaselineProtocol::Ack, msg.toLatin1()); // Do early reply if possible: don't make the client wait longer than necessary + + // Store the image QString dir = prefix.section(QLC('/'), 0, -2); QDir cwd; if (!cwd.exists(dir)) @@ -329,8 +443,23 @@ void BaselineHandler::storeImage(const QByteArray &itemBlock, bool isBaseline) itemData.insert(PI_CreationDate, QDateTime::currentDateTime().toString()); storeItemMetadata(itemData, prefix); - if (!isBaseline) - report.addMismatch(item); + if (!isBaseline) { + // Do fuzzy matching + bool fuzzyMatch = false; + if (fuzzLevel) { + BaselineProtocol::Command cmd = BaselineProtocol::Ack; + fuzzyMatch = fuzzyCompare(blPrefix, mmPrefix); + if (fuzzyMatch) { + msg.prepend(QString("Fuzzy match at fuzzlevel %1%. Report: ").arg(fuzzLevel)); + cmd = BaselineProtocol::FuzzyMatch; + } + proto.sendBlock(cmd, msg.toLatin1()); // We didn't reply earlier + } + + // Add to report + item.status = fuzzyMatch ? ImageItem::FuzzyMatch : ImageItem::Mismatch; + report.addResult(item); + } } @@ -355,7 +484,7 @@ PlatformInfo BaselineHandler::fetchItemMetadata(const QString &path) { PlatformInfo res; QFile file(path + QLS(MetadataFileExt)); - if (!file.open(QIODevice::ReadOnly)) + if (!file.open(QIODevice::ReadOnly) || !QFile::exists(path + QLS(FileFormat))) return res; QTextStream in(&file); do { @@ -368,20 +497,50 @@ PlatformInfo BaselineHandler::fetchItemMetadata(const QString &path) } +void BaselineHandler::idleClientTimeout() +{ + qWarning() << runId << logtime() << "Idle client timeout: no request received for" << IDLE_CLIENT_TIMEOUT << "seconds, terminating connection."; + proto.socket.disconnectFromHost(); +} + + void BaselineHandler::receiveDisconnect() { qDebug() << runId << logtime() << "Client disconnected."; report.end(); + if (report.reportProduced() && !clientInfo.isAdHocRun()) + issueMismatchNotification(); + if (settings && settings->value("ProcessXmlResults").toBool() && !clientInfo.isAdHocRun()) { + // ### TBD: actually execute the processing command. For now, just generate the xml files. + QString xmlDir = report.writeResultsXmlFiles(); + } QThread::currentThread()->exit(0); } PlatformInfo BaselineHandler::mapPlatformInfo(const PlatformInfo& orig) const { - PlatformInfo mapped = orig; + PlatformInfo mapped; + foreach (const QString &key, orig.uniqueKeys()) { + QString val = orig.value(key).simplified(); + val.replace(QLC('/'), QLC('_')); + val.replace(QLC(' '), QLC('_')); + mapped.insert(key, QUrl::toPercentEncoding(val, "+")); + //qDebug() << "MAPPED" << key << "FROM" << orig.value(key) << "TO" << mapped.value(key); + } + + // Special fixup for OS version + if (mapped.value(PI_OSName) == QLS("MacOS")) { + int ver = mapped.value(PI_OSVersion).toInt(); + if (ver > 1) + mapped.insert(PI_OSVersion, QString("MV_10_%1").arg(ver-2)); + } + else if (mapped.value(PI_OSName) == QLS("Windows")) { + // TBD: map windows version numbers to names + } - // Map hostname - QString host = orig.value(PI_HostName).section(QLC('.'), 0, 0); // Filter away domain, if any + // Special fixup for hostname + QString host = mapped.value(PI_HostName).section(QLC('.'), 0, 0); // Filter away domain, if any if (host.isEmpty() || host == QLS("localhost")) { host = orig.value(PI_HostAddress); } else { @@ -392,16 +551,15 @@ PlatformInfo BaselineHandler::mapPlatformInfo(const PlatformInfo& orig) const } } if (host.isEmpty()) - host = QLS("unknownhost"); + host = QLS("UNKNOWN-HOST"); + if (mapped.value(PI_OSName) == QLS("MacOS")) // handle multiple os versions on same host + host += QLC('-') + mapped.value(PI_OSVersion); mapped.insert(PI_HostName, host); - // Map qmakespec - QString mkspec = orig.value(PI_QMakeSpec); - mapped.insert(PI_QMakeSpec, mkspec.replace(QLC('/'), QLC('_'))); - - // Map Qt version - QString ver = orig.value(PI_QtVersion); - mapped.insert(PI_QtVersion, ver.prepend(QLS("Qt-"))); + // Special fixup for Qt version + QString ver = mapped.value(PI_QtVersion); + if (!ver.isEmpty()) + mapped.insert(PI_QtVersion, ver.prepend(QLS("Qt-"))); return mapped; } @@ -412,6 +570,7 @@ QString BaselineHandler::pathForItem(const ImageItem &item, bool isBaseline, boo if (mappedClientInfo.isEmpty()) { mappedClientInfo = mapPlatformInfo(clientInfo); PlatformInfo oraw = clientInfo; + // ### simplify: don't map if no overrides! for (int i = 0; i < clientInfo.overrides().size()-1; i+=2) oraw.insert(clientInfo.overrides().at(i), clientInfo.overrides().at(i+1)); overriddenMappedClientInfo = mapPlatformInfo(oraw); @@ -419,21 +578,21 @@ QString BaselineHandler::pathForItem(const ImageItem &item, bool isBaseline, boo const PlatformInfo& mapped = isBaseline ? overriddenMappedClientInfo : mappedClientInfo; - QString itemName = item.itemName.simplified(); - itemName.replace(QLC(' '), QLC('_')); - itemName.replace(QLC('.'), QLC('_')); - itemName.append(QLC('_')); - itemName.append(QString::number(item.itemChecksum, 16).rightJustified(4, QLC('0'))); + QString itemName = safeName(item.itemName); + itemName.append(QLC('_') + QString::number(item.itemChecksum, 16).rightJustified(4, QLC('0'))); QStringList path; - if (absolute) - path += BaselineServer::storagePath(); + path += projectPath(absolute); path += mapped.value(PI_TestCase); path += QLS(isBaseline ? "baselines" : "mismatches"); path += item.testFunction; - path += mapped.value(PI_QtVersion); - path += mapped.value(PI_QMakeSpec); - path += mapped.value(PI_HostName); + QStringList itemPathKeys; + if (settings) + itemPathKeys = settings->value("ItemPathKeys").toStringList(); + if (itemPathKeys.isEmpty()) + itemPathKeys = BaselineServer::defaultPathKeys(); + foreach (const QString &key, itemPathKeys) + path += mapped.value(key, QLS("UNSET-")+key); if (!isBaseline) path += runId; path += itemName + QLC('.'); @@ -446,19 +605,33 @@ QString BaselineHandler::view(const QString &baseline, const QString &rendered, { QFile f(":/templates/view.html"); f.open(QIODevice::ReadOnly); - return QString::fromLatin1(f.readAll()).arg('/'+baseline, '/'+rendered, '/'+compared); + return QString::fromLatin1(f.readAll()).arg('/'+baseline, '/'+rendered, '/'+compared, diffstats(baseline, rendered)); } +QString BaselineHandler::diffstats(const QString &baseline, const QString &rendered) +{ + QImage blImg(BaselineServer::storagePath() + QLC('/') + baseline); + QImage mmImg(BaselineServer::storagePath() + QLC('/') + rendered); + if (blImg.isNull() || mmImg.isNull()) + return QLS("Could not compute diffstats: image loading failed."); + + // ### TBD: cache the results + return computeMismatchScore(blImg, mmImg); +} QString BaselineHandler::clearAllBaselines(const QString &context) { int tot = 0; int failed = 0; QDirIterator it(BaselineServer::storagePath() + QLC('/') + context, - QStringList() << QLS("*.") + QLS(FileFormat) << QLS("*.") + QLS(MetadataFileExt)); + QStringList() << QLS("*.") + QLS(FileFormat) + << QLS("*.") + QLS(MetadataFileExt) + << QLS("*.") + QLS(ThumbnailExt)); while (it.hasNext()) { - tot++; - if (!QFile::remove(it.next())) + bool counting = !it.next().endsWith(QLS(ThumbnailExt)); + if (counting) + tot++; + if (!QFile::remove(it.filePath()) && counting) failed++; } return QString(QLS("%1 of %2 baselines cleared from context ")).arg((tot-failed)/2).arg(tot/2) + context; @@ -471,13 +644,17 @@ QString BaselineHandler::updateBaselines(const QString &context, const QString & QString storagePrefix = BaselineServer::storagePath() + QLC('/'); // If itemId is set, update just that one, otherwise, update all: QString filter = itemFile.isEmpty() ? QLS("*_????.") : itemFile; - QDirIterator it(storagePrefix + mismatchContext, QStringList() << filter + QLS(FileFormat) << filter + QLS(MetadataFileExt)); + QDirIterator it(storagePrefix + mismatchContext, + QStringList() << filter + QLS(FileFormat) + << filter + QLS(MetadataFileExt) + << filter + QLS(ThumbnailExt)); while (it.hasNext()) { - tot++; - it.next(); + bool counting = !it.next().endsWith(QLS(ThumbnailExt)); + if (counting) + tot++; QString oldFile = storagePrefix + context + QLC('/') + it.fileName(); - QFile::remove(oldFile); // Remove existing baseline file - if (!QFile::copy(it.filePath(), oldFile)) // and replace it with the mismatch + QFile::remove(oldFile); // Remove existing baseline file + if (!QFile::copy(it.filePath(), oldFile) && counting) // and replace it with the mismatch failed++; } return QString(QLS("%1 of %2 baselines updated in context %3 from context %4")).arg((tot-failed)/2).arg(tot/2).arg(context, mismatchContext); @@ -534,7 +711,7 @@ void BaselineHandler::testPathMapping() clientInfo.insert(PI_QtVersion, QLS("5.0.0")); clientInfo.insert(PI_QMakeSpec, QLS("linux-g++")); clientInfo.insert(PI_PulseGitBranch, QLS("somebranch")); - + clientInfo.setAdHocRun(false); foreach(const QString& host, hosts) { mappedClientInfo.clear(); clientInfo.insert(PI_HostName, host); @@ -547,9 +724,9 @@ void BaselineHandler::testPathMapping() QString BaselineHandler::computeMismatchScore(const QImage &baseline, const QImage &rendered) { if (baseline.size() != rendered.size() || baseline.format() != rendered.format()) - return QLS("[No score, incomparable images.]"); + return QLS("[No diffstats, incomparable images.]"); if (baseline.depth() != 32) - return QLS("[Score computation not implemented for format.]"); + return QLS("[Diffstats computation not implemented for format.]"); int w = baseline.width(); int h = baseline.height(); @@ -558,6 +735,8 @@ QString BaselineHandler::computeMismatchScore(const QImage &baseline, const QIma uint nad = 0; // number of differing alpha pixels uint scd = 0; // sum of color pixel difference uint sad = 0; // sum of alpha pixel difference + uint mind = 0; // minimum difference + uint maxd = 0; // maximum difference for (int y=0; y<h; ++y) { const QRgb *bl = (const QRgb *) baseline.constScanLine(y); @@ -566,14 +745,18 @@ QString BaselineHandler::computeMismatchScore(const QImage &baseline, const QIma QRgb b = bl[x]; QRgb r = rl[x]; if (r != b) { - int dr = qAbs(qRed(b) - qRed(r)); - int dg = qAbs(qGreen(b) - qGreen(r)); - int db = qAbs(qBlue(b) - qBlue(r)); - int ds = dr + dg + db; - int da = qAbs(qAlpha(b) - qAlpha(r)); + uint dr = qAbs(qRed(b) - qRed(r)); + uint dg = qAbs(qGreen(b) - qGreen(r)); + uint db = qAbs(qBlue(b) - qBlue(r)); + uint ds = (dr + dg + db) / 3; + uint da = qAbs(qAlpha(b) - qAlpha(r)); if (ds) { ncd++; scd += ds; + if (!mind || ds < mind) + mind = ds; + if (ds > maxd) + maxd = ds; } if (da) { nad++; @@ -583,13 +766,100 @@ QString BaselineHandler::computeMismatchScore(const QImage &baseline, const QIma } } + double pcd = 100.0 * ncd / (w*h); // percent of pixels that differ - double acd = ncd ? double(scd) / (3*ncd) : 0; // avg. difference - QString res = QString(QLS("Diffscore: %1% (Num:%2 Avg:%3)")).arg(pcd, 0, 'g', 2).arg(ncd).arg(acd, 0, 'g', 2); + double acd = ncd ? double(scd) / (ncd) : 0; // avg. difference +/* if (baseline.hasAlphaChannel()) { double pad = 100.0 * nad / (w*h); // percent of pixels that differ double aad = nad ? double(sad) / (3*nad) : 0; // avg. difference - res += QString(QLS(" Alpha-diffscore: %1% (Num:%2 Avg:%3)")).arg(pad, 0, 'g', 2).arg(nad).arg(aad, 0, 'g', 2); } +*/ + QString res = "<table>\n"; + QString item = "<tr><td>%1</td><td align=right>%2</td></tr>\n"; + res += item.arg("Number of mismatching pixels").arg(ncd); + res += item.arg("Percentage mismatching pixels").arg(pcd, 0, 'g', 2); + res += item.arg("Minimum pixel distance").arg(mind); + res += item.arg("Maximum pixel distance").arg(maxd); + if (acd >= 10.0) + res += item.arg("Average pixel distance").arg(qRound(acd)); + else + res += item.arg("Average pixel distance").arg(acd, 0, 'g', 2); + + if (baseline.hasAlphaChannel()) + res += item.arg("Number of mismatching alpha values").arg(nad); + + res += "</table>\n"; + res += "<p>(Distances are normalized to the range 0-255)</p>\n"; + return res; +} + + +bool BaselineHandler::fuzzyCompare(const QString &baselinePath, const QString &mismatchPath) +{ + QProcess compareProc; + QStringList args; + args << "-fuzz" << QString("%1%").arg(fuzzLevel) << "-metric" << "AE"; + args << baselinePath + QLS(FileFormat) << mismatchPath + QLS(FileFormat) << "/dev/null"; // TBD: Should save output image, so report won't have to regenerate it + + compareProc.setProcessChannelMode(QProcess::MergedChannels); + compareProc.start("compare", args, QIODevice::ReadOnly); + if (compareProc.waitForFinished(3000) && compareProc.error() == QProcess::UnknownError) { + bool ok = false; + int metric = compareProc.readAll().trimmed().toInt(&ok); + if (ok && metric == 0) + return true; + } + return false; +} + + +void BaselineHandler::issueMismatchNotification() +{ + // KISS: hardcoded use of the "sendemail" utility. Make this configurable if and when demand arises. + if (!settings) + return; + + settings->beginGroup("Notification"); + QStringList receivers = settings->value("Receivers").toStringList(); + QString sender = settings->value("Sender").toString(); + QString server = settings->value("SMTPserver").toString(); + settings->endGroup(); + if (receivers.isEmpty() || sender.isEmpty() || server.isEmpty()) + return; + + QString msg = QString("\nResult summary for test run %1:\n").arg(runId); + msg += report.summary(); + msg += "\nReport: " + BaselineServer::baseUrl() + report.filePath() + "\n"; + + msg += "\nTest run platform properties:\n------------------\n"; + foreach (const QString &key, clientInfo.keys()) + msg += key + ": " + clientInfo.value(key) + '\n'; + msg += "\nCheers,\n- Your friendly Lancelot Baseline Server\n"; + + QProcess proc; + QString cmd = "sendemail"; + QStringList args; + args << "-s" << server << "-f" << sender << "-t" << receivers; + args << "-u" << "[Lancelot] Mismatch report for project " + clientInfo.value(PI_Project) + ", test case " + clientInfo.value(PI_TestCase); + args << "-m" << msg; + + //proc.setProcessChannelMode(QProcess::MergedChannels); + proc.start(cmd, args); + if (!proc.waitForFinished(10 * 1000) || (proc.exitStatus() != QProcess::NormalExit) || proc.exitCode()) { + qWarning() << "FAILED to issue notification. Command:" << cmd << args.mid(0, args.size()-2); + qWarning() << " Command standard output:" << proc.readAllStandardOutput(); + qWarning() << " Command error output:" << proc.readAllStandardError(); + } +} + + +// Make an identifer safer for use as filename and URL +QString safeName(const QString& name) +{ + QString res = name.simplified(); + res.replace(QLC(' '), QLC('_')); + res.replace(QLC('.'), QLC('_')); + res.replace(QLC('/'), QLC('^')); return res; } diff --git a/tests/baselineserver/src/baselineserver.h b/tests/baselineserver/src/baselineserver.h index c8ce3860fb..d30a74195d 100644 --- a/tests/baselineserver/src/baselineserver.h +++ b/tests/baselineserver/src/baselineserver.h @@ -53,9 +53,14 @@ #include "baselineprotocol.h" #include "report.h" -// #seconds between update checks +// #seconds between checks for update of the executable #define HEARTBEAT 10 +// Timeout if no activity received from client, #seconds +#define IDLE_CLIENT_TIMEOUT 3*60 + #define MetadataFileExt "metadata" +#define ThumbnailExt "thumbnail.jpg" + class BaselineServer : public QTcpServer { @@ -66,7 +71,7 @@ public: static QString storagePath(); static QString baseUrl(); - static QString settingsFilePath(); + static QStringList defaultPathKeys(); protected: void incomingConnection(qintptr socketDescriptor); @@ -81,7 +86,7 @@ private: int lastRunIdIdx; static QString storage; static QString url; - static QString settingsFile; + static QStringList pathKeys; }; @@ -106,28 +111,38 @@ class BaselineHandler : public QObject public: BaselineHandler(const QString &runId, int socketDescriptor = -1); - void testPathMapping(); + QString projectPath(bool absolute = true) const; QString pathForItem(const ImageItem &item, bool isBaseline = true, bool absolute = true) const; // CGI callbacks: static QString view(const QString &baseline, const QString &rendered, const QString &compared); + static QString diffstats(const QString &baseline, const QString &rendered); static QString clearAllBaselines(const QString &context); static QString updateBaselines(const QString &context, const QString &mismatchContext, const QString &itemFile); static QString blacklistTest(const QString &context, const QString &itemId, bool removeFromBlacklist = false); + // for debugging + void testPathMapping(); + private slots: void receiveRequest(); void receiveDisconnect(); + void idleClientTimeout(); private: + bool checkClient(QByteArray *errMsg, bool *dryRunMode = 0); bool establishConnection(); void provideBaselineChecksums(const QByteArray &itemListBlock); + void recordMatch(const QByteArray &itemBlock); void storeImage(const QByteArray &itemBlock, bool isBaseline); void storeItemMetadata(const PlatformInfo &metadata, const QString &path); PlatformInfo fetchItemMetadata(const QString &path); PlatformInfo mapPlatformInfo(const PlatformInfo& orig) const; const char *logtime(); - QString computeMismatchScore(const QImage& baseline, const QImage& rendered); + void issueMismatchNotification(); + bool fuzzyCompare(const QString& baselinePath, const QString& mismatchPath); + + static QString computeMismatchScore(const QImage& baseline, const QImage& rendered); BaselineProtocol proto; PlatformInfo clientInfo; @@ -137,6 +152,13 @@ private: bool connectionEstablished; Report report; QSettings *settings; + QString ruleName; + int fuzzLevel; + QTimer *idleTimer; }; + +// Make an identifer safer for use as filename and URL +QString safeName(const QString& name); + #endif // BASELINESERVER_H diff --git a/tests/baselineserver/src/report.cpp b/tests/baselineserver/src/report.cpp index d29f22e2a3..cd544a6c78 100644 --- a/tests/baselineserver/src/report.cpp +++ b/tests/baselineserver/src/report.cpp @@ -44,9 +44,11 @@ #include <QDir> #include <QProcess> #include <QUrl> +#include <QXmlStreamWriter> Report::Report() - : written(false), numItems(0), numMismatches(0) + : initialized(false), handler(0), written(false), numItems(0), numMismatches(0), settings(0), + hasStats(false) { } @@ -60,19 +62,31 @@ QString Report::filePath() return path; } -void Report::init(const BaselineHandler *h, const QString &r, const PlatformInfo &p) +int Report::numberOfMismatches() +{ + return numMismatches; +} + +bool Report::reportProduced() +{ + return written; +} + +void Report::init(const BaselineHandler *h, const QString &r, const PlatformInfo &p, const QSettings *s) { handler = h; runId = r; plat = p; + settings = s; rootDir = BaselineServer::storagePath() + QLC('/'); - reportDir = plat.value(PI_TestCase) + QLC('/') + (plat.isAdHocRun() ? QLS("reports/adhoc/") : QLS("reports/pulse/")); - QString dir = rootDir + reportDir; + baseDir = handler->pathForItem(ImageItem(), true, false).remove(QRegExp("/baselines/.*$")); + QString dir = baseDir + (plat.isAdHocRun() ? QLS("/adhoc-reports") : QLS("/auto-reports")); QDir cwd; - if (!cwd.exists(dir)) - cwd.mkpath(dir); - path = reportDir + QLS("Report_") + runId + QLS(".html"); + if (!cwd.exists(rootDir + dir)) + cwd.mkpath(rootDir + dir); + path = dir + QLS("/Report_") + runId + QLS(".html"); hasOverride = !plat.overrides().isEmpty(); + initialized = true; } void Report::addItems(const ImageItemList &items) @@ -83,38 +97,110 @@ void Report::addItems(const ImageItemList &items) QString func = items.at(0).testFunction; if (!testFunctions.contains(func)) testFunctions.append(func); - itemLists[func] += items; + ImageItemList list = items; + if (settings->value("ReportMissingResults").toBool()) { + for (ImageItemList::iterator it = list.begin(); it != list.end(); ++it) { + if (it->status == ImageItem::Ok) + it->status = ImageItem::Error; // Status should be set by report from client, else report as error + } + } + itemLists[func] += list; } -void Report::addMismatch(const ImageItem &item) +void Report::addResult(const ImageItem &item) { if (!testFunctions.contains(item.testFunction)) { - qWarning() << "Report::addMismatch: unknown testfunction" << item.testFunction; + qWarning() << "Report::addResult: unknown testfunction" << item.testFunction; return; } bool found = false; ImageItemList &list = itemLists[item.testFunction]; for (ImageItemList::iterator it = list.begin(); it != list.end(); ++it) { if (it->itemName == item.itemName && it->itemChecksum == item.itemChecksum) { - it->status = ImageItem::Mismatch; + it->status = item.status; found = true; break; } } - if (found) - numMismatches++; - else - qWarning() << "Report::addMismatch: unknown item" << item.itemName << "in testfunction" << item.testFunction; + if (found) { + if (item.status == ImageItem::Mismatch) + numMismatches++; + } else { + qWarning() << "Report::addResult: unknown item" << item.itemName << "in testfunction" << item.testFunction; + } } void Report::end() { - if (written || !numMismatches) + if (!initialized || written) + return; + // Make report iff (#mismatches>0) || (#fuzzymatches>0) || (#errors>0 && settings say report errors) + bool doReport = (numMismatches > 0); + if (!doReport) { + bool reportErrors = settings->value("ReportMissingResults").toBool(); + computeStats(); + foreach (const QString &func, itemLists.keys()) { + FuncStats stat = stats.value(func); + if (stat.value(ImageItem::FuzzyMatch) > 0) { + doReport = true; + break; + } + foreach (const ImageItem &item, itemLists.value(func)) { + if (reportErrors && item.status == ImageItem::Error) { + doReport = true; + break; + } + } + if (doReport) + break; + } + } + if (!doReport) return; write(); written = true; } +void Report::computeStats() +{ + if (hasStats) + return; + foreach (const QString &func, itemLists.keys()) { + FuncStats funcStat; + funcStat[ImageItem::Ok] = 0; + funcStat[ImageItem::BaselineNotFound] = 0; + funcStat[ImageItem::IgnoreItem] = 0; + funcStat[ImageItem::Mismatch] = 0; + funcStat[ImageItem::FuzzyMatch] = 0; + funcStat[ImageItem::Error] = 0; + foreach (const ImageItem &item, itemLists.value(func)) { + funcStat[item.status]++; + } + stats[func] = funcStat; + } + hasStats = true; +} + +QString Report::summary() +{ + computeStats(); + QString res; + foreach (const QString &func, itemLists.keys()) { + FuncStats stat = stats.value(func); + QString s = QString("%1 %3 mismatch(es), %4 error(s), %5 fuzzy match(es)\n"); + s = s.arg(QString("%1() [%2 items]:").arg(func).arg(itemLists.value(func).size()).leftJustified(40)); + s = s.arg(stat.value(ImageItem::Mismatch)); + s = s.arg(stat.value(ImageItem::Error)); + s = s.arg(stat.value(ImageItem::FuzzyMatch)); + res += s; + } +#if 0 + qDebug() << "***************************** Summary *************************"; + qDebug() << res; + qDebug() << "***************************************************************"; +#endif + return res; +} void Report::write() { @@ -131,24 +217,27 @@ void Report::write() } writeFooter(); file.close(); + updateLatestPointer(); } void Report::writeHeader() { - QString title = plat.value(PI_TestCase) + QLS(" Qt Baseline Test Report"); - out << "<head><title>" << title << "</title></head>\n" - << "<html><body><h1>" << title << "</h1>\n" + QString title = plat.value(PI_Project) + QLC(':') + plat.value(PI_TestCase) + QLS(" Lancelot Test Report"); + out << "<!DOCTYPE html>\n" + << "<html><head><title>" << title << "</title></head>\n" + << "<body bgcolor=""#ddeeff""><h1>" << title << "</h1>\n" << "<p>Note: This is a <i>static</i> page, generated at " << QDateTime::currentDateTime().toString() << " for the test run with id " << runId << "</p>\n" - << "<p>Summary: <b><span style=\"color:red\">" << numMismatches << " of " << numItems << "</b></span> items reported mismatching</p>\n\n"; + << "<p>Summary: <b><span style=\"color:red\">" << numMismatches << " of " << numItems << "</span></b> items reported mismatching</p>\n"; + out << "<pre>\n" << summary() << "</pre>\n\n"; out << "<h3>Testing Client Platform Info:</h3>\n" << "<table>\n"; foreach (QString key, plat.keys()) out << "<tr><td>" << key << ":</td><td>" << plat.value(key) << "</td></tr>\n"; out << "</table>\n\n"; if (hasOverride) { - out << "<span style=\"color:red\"><h4>Note! Platform Override Info:</h4></span>\n" + out << "<span style=\"color:red\"><h4>Note! Override Platform Info:</h4></span>\n" << "<p>The client's output has been compared to baselines created on a different platform. Differences:</p>\n" << "<table>\n"; for (int i = 0; i < plat.overrides().size()-1; i+=2) @@ -184,14 +273,25 @@ void Report::writeFunctionResults(const ImageItemList &list) "</tr>\n\n"; foreach (const ImageItem &item, list) { + QString mmPrefix = handler->pathForItem(item, false, false); + QString blPrefix = handler->pathForItem(item, true, false); + + // Make hard links to the current baseline, so that the report is static even if the baseline changes + generateThumbnail(blPrefix + QLS(FileFormat), rootDir); // Make sure baseline thumbnail is up to date + QString lnPrefix = mmPrefix + QLS("baseline."); + QByteArray blPrefixBa = (rootDir + blPrefix).toLatin1(); + QByteArray lnPrefixBa = (rootDir + lnPrefix).toLatin1(); + ::link((blPrefixBa + FileFormat).constData(), (lnPrefixBa + FileFormat).constData()); + ::link((blPrefixBa + MetadataFileExt).constData(), (lnPrefixBa + MetadataFileExt).constData()); + ::link((blPrefixBa + ThumbnailExt).constData(), (lnPrefixBa + ThumbnailExt).constData()); + + QString baseline = lnPrefix + QLS(FileFormat); + QString metadata = lnPrefix + QLS(MetadataFileExt); out << "<tr>\n"; out << "<td>" << item.itemName << "</td>\n"; - QString prefix = handler->pathForItem(item, true, false); - QString baseline = prefix + QLS(FileFormat); - QString metadata = prefix + QLS(MetadataFileExt); - if (item.status == ImageItem::Mismatch) { - QString rendered = handler->pathForItem(item, false, false) + QLS(FileFormat); - QString itemFile = prefix.section(QLC('/'), -1); + if (item.status == ImageItem::Mismatch || item.status == ImageItem::FuzzyMatch) { + QString rendered = mmPrefix + QLS(FileFormat); + QString itemFile = mmPrefix.section(QLC('/'), -1); writeItem(baseline, rendered, item, itemFile, ctx, misCtx, metadata); } else { @@ -210,6 +310,9 @@ void Report::writeFunctionResults(const ImageItemList &list) << "\">Whitelist this item</a>"; } break; + case ImageItem::Error: + out << "<span style=\"background-color:red\">Error: No result reported!</span>"; + break; case ImageItem::Ok: out << "<span style=\"color:green\"><small>No mismatch reported</small></span>"; break; @@ -233,11 +336,14 @@ void Report::writeItem(const QString &baseline, const QString &rendered, const I QStringList images = QStringList() << baseline << rendered << compared; foreach (const QString& img, images) - out << "<td height=246 align=center><a href=\"/" << img << "\"><img src=\"/" << generateThumbnail(img) << "\"></a></td>\n"; + out << "<td height=246 align=center><a href=\"/" << img << "\"><img src=\"/" << generateThumbnail(img, rootDir) << "\"></a></td>\n"; - out << "<td align=center>\n" - << "<p><span style=\"color:red\">Mismatch reported</span></p>\n" - << "<p><a href=\"/" << metadata << "\">Baseline Info</a>\n"; + out << "<td align=center>\n"; + if (item.status == ImageItem::FuzzyMatch) + out << "<p><span style=\"color:orange\">Fuzzy match</span></p>\n"; + else + out << "<p><span style=\"color:red\">Mismatch reported</span></p>\n"; + out << "<p><a href=\"/" << metadata << "\">Baseline Info</a>\n"; if (!hasOverride) { out << "<p><a href=\"/cgi-bin/server.cgi?cmd=updateSingleBaseline&context=" << ctx << "&mismatchContext=" << misCtx << "&itemFile=" << itemFile << "&url=" << pageUrl << "\">Let this be the new baseline</a></p>\n" @@ -245,8 +351,14 @@ void Report::writeItem(const QString &baseline, const QString &rendered, const I << "&itemId=" << item.itemName << "&url=" << pageUrl << "\">Blacklist this item</a></p>\n"; } out << "<p><a href=\"/cgi-bin/server.cgi?cmd=view&baseline=" << baseline << "&rendered=" << rendered - << "&compared=" << compared << "&url=" << pageUrl << "\">Inspect</a></p>\n" - << "</td>\n"; + << "&compared=" << compared << "&url=" << pageUrl << "\">Inspect</a></p>\n"; + +#if 0 + out << "<p><a href=\"/cgi-bin/server.cgi?cmd=diffstats&baseline=" << baseline << "&rendered=" << rendered + << "&url=" << pageUrl << "\">Diffstats</a></p>\n"; +#endif + + out << "</td>\n"; } void Report::writeFooter() @@ -259,8 +371,8 @@ QString Report::generateCompared(const QString &baseline, const QString &rendere { QString res = rendered; QFileInfo fi(res); - res.chop(fi.suffix().length() + 1); - res += QLS(fuzzy ? "_fuzzycompared.png" : "_compared.png"); + res.chop(fi.suffix().length()); + res += QLS(fuzzy ? "fuzzycompared.png" : "compared.png"); QStringList args; if (fuzzy) args << QLS("-fuzz") << QLS("5%"); @@ -270,12 +382,14 @@ QString Report::generateCompared(const QString &baseline, const QString &rendere } -QString Report::generateThumbnail(const QString &image) +QString Report::generateThumbnail(const QString &image, const QString &rootDir) { QString res = image; QFileInfo imgFI(rootDir+image); - res.chop(imgFI.suffix().length() + 1); - res += QLS("_thumbnail.jpg"); + if (!imgFI.exists()) + return res; + res.chop(imgFI.suffix().length()); + res += ThumbnailExt; QFileInfo resFI(rootDir+res); if (resFI.exists() && resFI.lastModified() > imgFI.lastModified()) return res; @@ -286,12 +400,80 @@ QString Report::generateThumbnail(const QString &image) } +QString Report::writeResultsXmlFiles() +{ + if (!itemLists.size()) + return QString(); + QString dir = rootDir + baseDir + QLS("/xml-reports/") + runId; + QDir cwd; + if (!cwd.exists(dir)) + cwd.mkpath(dir); + foreach (const QString &func, itemLists.keys()) { + QFile f(dir + "/" + func + "-results.xml"); + if (!f.open(QIODevice::WriteOnly)) + continue; + QXmlStreamWriter s(&f); + s.setAutoFormatting(true); + s.writeStartDocument(); + foreach (QString key, plat.keys()) { + QString cmt = " " + key + "=\"" + plat.value(key) +"\" "; + s.writeComment(cmt.replace("--", "[-]")); + } + s.writeStartElement("testsuite"); + s.writeAttribute("name", func); + foreach (const ImageItem &item, itemLists.value(func)) { + QString res; + switch (item.status) { + case ImageItem::Ok: + case ImageItem::FuzzyMatch: + res = "pass"; + break; + case ImageItem::Mismatch: + case ImageItem::Error: + res = "fail"; + break; + case ImageItem::BaselineNotFound: + case ImageItem::IgnoreItem: + default: + res = "skip"; + } + s.writeStartElement("testcase"); + s.writeAttribute("name", item.itemName); + s.writeAttribute("result", res); + s.writeEndElement(); + } + s.writeEndElement(); + s.writeEndDocument(); + } + return dir; +} + + + +void Report::updateLatestPointer() +{ + QString linkPath = rootDir + baseDir + QLS("/latest_report.html"); + QString reportPath = path.mid(baseDir.size()+1); + QFile::remove(linkPath); // possible race with another thread, yada yada yada + QFile::link(reportPath, linkPath); + +#if 0 + QByteArray fwd = "<!DOCTYPE html><html><head><meta HTTP-EQUIV=\"refresh\" CONTENT=\"0;URL=%1\"></meta></head><body></body></html>\n"; + fwd.replace("%1", filePath().prepend(QLC('/')).toLatin1()); + + QFile file(rootDir + baseDir + "/latest_report.html"); + if (file.open(QIODevice::WriteOnly | QIODevice::Truncate)) + file.write(fwd); +#endif +} + + void Report::handleCGIQuery(const QString &query) { QUrl cgiUrl(QLS("http://dummy/cgi-bin/dummy.cgi?") + query); QTextStream s(stdout); s << "Content-Type: text/html\r\n\r\n" - << "<HTML>"; + << "<!DOCTYPE html>\n<HTML>\n<body bgcolor=""#ddeeff"">\n"; // Lancelot blue QString command(cgiUrl.queryItemValue("cmd")); @@ -300,6 +482,12 @@ void Report::handleCGIQuery(const QString &query) cgiUrl.queryItemValue(QLS("rendered")), cgiUrl.queryItemValue(QLS("compared"))); } +#if 0 + else if (command == QLS("diffstats")) { + s << BaselineHandler::diffstats(cgiUrl.queryItemValue(QLS("baseline")), + cgiUrl.queryItemValue(QLS("rendered"))); + } +#endif else if (command == QLS("updateSingleBaseline")) { s << BaselineHandler::updateBaselines(cgiUrl.queryItemValue(QLS("context")), cgiUrl.queryItemValue(QLS("mismatchContext")), @@ -321,6 +509,6 @@ void Report::handleCGIQuery(const QString &query) } else { s << "Unknown query:<br>" << query << "<br>"; } - s << "<p><a href=\"" << cgiUrl.queryItemValue(QLS("url")) << "\">Back to report</a>"; - s << "</HTML>"; + s << "<p><a href=\"" << cgiUrl.queryItemValue(QLS("url")) << "\">Back to report</a>\n"; + s << "</body>\n</HTML>"; } diff --git a/tests/baselineserver/src/report.h b/tests/baselineserver/src/report.h index 660593784e..918d09d420 100644 --- a/tests/baselineserver/src/report.h +++ b/tests/baselineserver/src/report.h @@ -46,6 +46,7 @@ #include <QTextStream> #include <QMap> #include <QStringList> +#include <QSettings> class BaselineHandler; @@ -55,15 +56,24 @@ public: Report(); ~Report(); - void init(const BaselineHandler *h, const QString &r, const PlatformInfo &p); + void init(const BaselineHandler *h, const QString &r, const PlatformInfo &p, const QSettings *s); void addItems(const ImageItemList& items); - void addMismatch(const ImageItem& item); + void addResult(const ImageItem& item); void end(); + bool reportProduced(); + + int numberOfMismatches(); + QString summary(); + QString filePath(); + QString writeResultsXmlFiles(); + static void handleCGIQuery(const QString &query); + static QString generateThumbnail(const QString &image, const QString &rootDir = QString()); + private: void write(); void writeFunctionResults(const ImageItemList &list); @@ -72,13 +82,17 @@ private: void writeHeader(); void writeFooter(); QString generateCompared(const QString &baseline, const QString &rendered, bool fuzzy = false); - QString generateThumbnail(const QString &image); + void updateLatestPointer(); + + void computeStats(); + + bool initialized; const BaselineHandler *handler; QString runId; PlatformInfo plat; QString rootDir; - QString reportDir; + QString baseDir; QString path; QStringList testFunctions; QMap<QString, ImageItemList> itemLists; @@ -87,6 +101,11 @@ private: int numMismatches; QTextStream out; bool hasOverride; + const QSettings *settings; + + typedef QMap<ImageItem::ItemStatus, int> FuncStats; + QMap<QString, FuncStats> stats; + bool hasStats; }; #endif // REPORT_H diff --git a/tests/baselineserver/src/templates/view.html b/tests/baselineserver/src/templates/view.html index c048f4781c..f0971010f2 100644 --- a/tests/baselineserver/src/templates/view.html +++ b/tests/baselineserver/src/templates/view.html @@ -23,9 +23,14 @@ Zoom: </table></p> -<p> +<p><table cellspacing="25"><tr> +<td valign="top"> <canvas id="c" width="800" height="800"></canvas> -</p> +</td> +<td valign="top"> +%4 +</td> +</tr></table></p> <script> var canvas = document.getElementById("c"); |