diff options
Diffstat (limited to 'tests/baselineserver/src')
-rw-r--r-- | tests/baselineserver/src/baselineserver.cpp | 853 | ||||
-rw-r--r-- | tests/baselineserver/src/baselineserver.h | 151 | ||||
-rw-r--r-- | tests/baselineserver/src/baselineserver.pro | 24 | ||||
-rw-r--r-- | tests/baselineserver/src/baselineserver.qrc | 5 | ||||
-rw-r--r-- | tests/baselineserver/src/main.cpp | 57 | ||||
-rw-r--r-- | tests/baselineserver/src/report.cpp | 503 | ||||
-rw-r--r-- | tests/baselineserver/src/report.h | 98 | ||||
-rw-r--r-- | tests/baselineserver/src/templates/view.html | 84 |
8 files changed, 0 insertions, 1775 deletions
diff --git a/tests/baselineserver/src/baselineserver.cpp b/tests/baselineserver/src/baselineserver.cpp deleted file mode 100644 index c6e12dc09c..0000000000 --- a/tests/baselineserver/src/baselineserver.cpp +++ /dev/null @@ -1,853 +0,0 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the test suite of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:GPL-EXCEPT$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 3 as published by the Free Software -** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ - -#define QT_USE_FAST_CONCATENATION -#define QT_USE_FAST_OPERATOR_PLUS - -#include "baselineserver.h" -#include <QBuffer> -#include <QFile> -#include <QDir> -#include <QCoreApplication> -#include <QFileInfo> -#include <QHostInfo> -#include <QTextStream> -#include <QProcess> -#include <QDirIterator> -#include <QUrl> -#include <QRegularExpression> - -// extra fields, for use in image metadata storage -const QString PI_ImageChecksum(QLS("ImageChecksum")); -const QString PI_RunId(QLS("RunId")); -const QString PI_CreationDate(QLS("CreationDate")); - -QString BaselineServer::storage; -QString BaselineServer::url; -QStringList BaselineServer::pathKeys; - -BaselineServer::BaselineServer(QObject *parent) - : QTcpServer(parent), lastRunIdIdx(0) -{ - QFileInfo me(QCoreApplication::applicationFilePath()); - meLastMod = me.lastModified(); - heartbeatTimer = new QTimer(this); - connect(heartbeatTimer, SIGNAL(timeout()), this, SLOT(heartbeat())); - heartbeatTimer->start(HEARTBEAT*1000); -} - -QString BaselineServer::storagePath() -{ - if (storage.isEmpty()) { - storage = QLS(qgetenv("QT_LANCELOT_DIR")); - if (storage.isEmpty()) - storage = QLS("/var/www"); - } - return storage; -} - -QString BaselineServer::baseUrl() -{ - if (url.isEmpty()) { - url = QLS("http://") - + QHostInfo::localHostName().toLatin1() + '.' - + QHostInfo::localDomainName().toLatin1() + '/'; - } - return url; -} - -QStringList BaselineServer::defaultPathKeys() -{ - if (pathKeys.isEmpty()) - pathKeys << PI_QtVersion << PI_QMakeSpec << PI_HostName; - return pathKeys; -} - -void BaselineServer::incomingConnection(qintptr socketDescriptor) -{ - QString runId = QDateTime::currentDateTime().toString(QLS("MMMdd-hhmmss")); - if (runId == lastRunId) { - runId += QLC('-') + QString::number(++lastRunIdIdx); - } else { - lastRunId = runId; - lastRunIdIdx = 0; - } - qDebug() << "Server: New connection! RunId:" << runId; - BaselineThread *thread = new BaselineThread(runId, socketDescriptor, this); - connect(thread, SIGNAL(finished()), thread, SLOT(deleteLater())); - thread->start(); -} - -void BaselineServer::heartbeat() -{ - // The idea is to exit to be restarted when modified, as soon as not actually serving - QFileInfo me(QCoreApplication::applicationFilePath()); - if (me.lastModified() == meLastMod) - return; - if (!me.exists() || !me.isExecutable()) - return; - - //# (could close() here to avoid accepting new connections, to avoid livelock) - //# also, could check for a timeout to force exit, to avoid hung threads blocking - bool isServing = false; - foreach(BaselineThread *thread, findChildren<BaselineThread *>()) { - if (thread->isRunning()) { - isServing = true; - break; - } - } - - if (!isServing) - QCoreApplication::exit(); -} - -BaselineThread::BaselineThread(const QString &runId, int socketDescriptor, QObject *parent) - : QThread(parent), runId(runId), socketDescriptor(socketDescriptor) -{ -} - -void BaselineThread::run() -{ - BaselineHandler handler(runId, socketDescriptor); - exec(); -} - - -BaselineHandler::BaselineHandler(const QString &runId, int socketDescriptor) - : QObject(), runId(runId), connectionEstablished(false), settings(0), fuzzLevel(0) -{ - idleTimer = new QTimer(this); - idleTimer->setSingleShot(true); - idleTimer->setInterval(IDLE_CLIENT_TIMEOUT * 1000); - connect(idleTimer, SIGNAL(timeout()), this, SLOT(idleClientTimeout())); - idleTimer->start(); - - if (socketDescriptor == -1) - return; - - connect(&proto.socket, SIGNAL(readyRead()), this, SLOT(receiveRequest())); - connect(&proto.socket, SIGNAL(disconnected()), this, SLOT(receiveDisconnect())); - proto.socket.setSocketDescriptor(socketDescriptor); - proto.socket.setSocketOption(QAbstractSocket::KeepAliveOption, 1); -} - -const char *BaselineHandler::logtime() -{ - return 0; - //return QTime::currentTime().toString(QLS("mm:ss.zzz")); -} - -QString BaselineHandler::projectPath(bool absolute) const -{ - QString p = clientInfo.value(PI_Project); - return absolute ? BaselineServer::storagePath() + QLC('/') + p : p; -} - -bool BaselineHandler::checkClient(QByteArray *errMsg, bool *dryRunMode) -{ - if (!errMsg) - return false; - if (clientInfo.value(PI_Project).isEmpty() || clientInfo.value(PI_TestCase).isEmpty()) { - *errMsg = "No Project and/or TestCase specified in client info."; - return false; - } - - // Determine ad-hoc state ### hardcoded for now - if (clientInfo.value(PI_TestCase) == QLS("tst_Lancelot")) { - //### Todo: push this stuff out in a script - if (!clientInfo.isAdHocRun()) { - // ### comp. with earlier versions still running (4.8) (?) - clientInfo.setAdHocRun(clientInfo.value(PI_PulseGitBranch).isEmpty() && clientInfo.value(PI_PulseTestrBranch).isEmpty()); - } - } - else { - // TBD - } - - if (clientInfo.isAdHocRun()) { - if (dryRunMode) - *dryRunMode = false; - return true; - } - - // Not ad hoc: filter the client - settings->beginGroup("ClientFilters"); - bool matched = false; - bool dryRunReq = false; - foreach (const QString &rule, settings->childKeys()) { - //qDebug() << " > RULE" << rule; - dryRunReq = false; - QString ruleMode = settings->value(rule).toString().toLower(); - if (ruleMode == QLS("dryrun")) - dryRunReq = true; - else if (ruleMode != QLS("enabled")) - continue; - settings->beginGroup(rule); - bool ruleMatched = true; - foreach (const QString &filterKey, settings->childKeys()) { - //qDebug() << " > FILTER" << filterKey; - QString filter = settings->value(filterKey).toString(); - if (filter.isEmpty()) - continue; - QString platVal = clientInfo.value(filterKey); - if (!platVal.contains(filter)) { - ruleMatched = false; - break; - } - } - if (ruleMatched) { - ruleName = rule; - matched = true; - break; - } - settings->endGroup(); - } - - if (!matched && errMsg) - *errMsg = "Non-adhoc client did not match any filter rule in " + settings->fileName().toLatin1(); - - if (matched && dryRunMode) - *dryRunMode = dryRunReq; - - // NB! Must reset the settings object before returning - while (!settings->group().isEmpty()) - settings->endGroup(); - - return matched; -} - -bool BaselineHandler::establishConnection() -{ - if (!proto.acceptConnection(&clientInfo)) { - qWarning() << runId << logtime() << "Accepting new connection from" << proto.socket.peerAddress().toString() << "failed." << proto.errorMessage(); - proto.sendBlock(BaselineProtocol::Abort, proto.errorMessage().toLatin1()); // In case the client can hear us, tell it what's wrong. - proto.socket.disconnectFromHost(); - return false; - } - QString logMsg; - foreach (QString key, clientInfo.keys()) { - if (key != PI_HostName && key != PI_HostAddress) - logMsg += key + QLS(": '") + clientInfo.value(key) + QLS("', "); - } - qDebug() << runId << logtime() << "Connection established with" << clientInfo.value(PI_HostName) - << '[' << qPrintable(clientInfo.value(PI_HostAddress)) << ']' << logMsg - << "Overrides:" << clientInfo.overrides() << "AdHoc-Run:" << clientInfo.isAdHocRun(); - - // ### Hardcoded backwards compatibility: add project field for certain existing clients that lack it - if (clientInfo.value(PI_Project).isEmpty()) { - QString tc = clientInfo.value(PI_TestCase); - if (tc == QLS("tst_Lancelot")) - clientInfo.insert(PI_Project, QLS("Raster")); - else if (tc == QLS("tst_Scenegraph")) - clientInfo.insert(PI_Project, QLS("SceneGraph")); - else - clientInfo.insert(PI_Project, QLS("Other")); - } - - QString settingsFile = projectPath() + QLS("/config.ini"); - settings = new QSettings(settingsFile, QSettings::IniFormat, this); - - QByteArray errMsg; - bool dryRunMode = false; - if (!checkClient(&errMsg, &dryRunMode)) { - qDebug() << runId << logtime() << "Rejecting connection:" << errMsg; - proto.sendBlock(BaselineProtocol::Abort, errMsg); - proto.socket.disconnectFromHost(); - return false; - } - - fuzzLevel = qBound(0, settings->value("FuzzLevel").toInt(), 100); - if (!clientInfo.isAdHocRun()) { - qDebug() << runId << logtime() << "Client matches filter rule" << ruleName - << "Dryrun:" << dryRunMode - << "FuzzLevel:" << fuzzLevel - << "ReportMissingResults:" << settings->value("ReportMissingResults").toBool(); - } - - proto.sendBlock(dryRunMode ? BaselineProtocol::DoDryRun : BaselineProtocol::Ack, QByteArray()); - report.init(this, runId, clientInfo, settings); - return true; -} - -void BaselineHandler::receiveRequest() -{ - idleTimer->start(); // Restart idle client timeout - - if (!connectionEstablished) { - connectionEstablished = establishConnection(); - return; - } - - QByteArray block; - BaselineProtocol::Command cmd; - if (!proto.receiveBlock(&cmd, &block)) { - qWarning() << runId << logtime() << "Command reception failed. "<< proto.errorMessage(); - QThread::currentThread()->exit(1); - return; - } - - switch(cmd) { - case BaselineProtocol::RequestBaselineChecksums: - provideBaselineChecksums(block); - break; - case BaselineProtocol::AcceptMatch: - recordMatch(block); - break; - case BaselineProtocol::AcceptNewBaseline: - storeImage(block, true); - break; - case BaselineProtocol::AcceptMismatch: - storeImage(block, false); - break; - default: - qWarning() << runId << logtime() << "Unknown command received. " << proto.errorMessage(); - proto.sendBlock(BaselineProtocol::UnknownError, QByteArray()); - } -} - - -void BaselineHandler::provideBaselineChecksums(const QByteArray &itemListBlock) -{ - ImageItemList itemList; - QDataStream ds(itemListBlock); - ds >> itemList; - qDebug() << runId << logtime() << "Received request for checksums for" << itemList.count() - << "items in test function" << itemList.at(0).testFunction; - - for (ImageItemList::iterator i = itemList.begin(); i != itemList.end(); ++i) { - i->imageChecksums.clear(); - i->status = ImageItem::BaselineNotFound; - QString prefix = pathForItem(*i, true); - PlatformInfo itemData = fetchItemMetadata(prefix); - if (itemData.contains(PI_ImageChecksum)) { - bool ok = false; - quint64 checksum = itemData.value(PI_ImageChecksum).toULongLong(&ok, 16); - if (ok) { - i->imageChecksums.prepend(checksum); - i->status = ImageItem::Ok; - } - } - } - - // Find and mark blacklisted items - QString context = pathForItem(itemList.at(0), true, false).section(QLC('/'), 0, -2); - if (itemList.count() > 0) { - QFile file(BaselineServer::storagePath() + QLC('/') + context + QLS("/BLACKLIST")); - if (file.open(QIODevice::ReadOnly)) { - QTextStream in(&file); - do { - QString itemName = in.readLine(); - if (!itemName.isNull()) { - for (ImageItemList::iterator i = itemList.begin(); i != itemList.end(); ++i) { - if (i->itemName == itemName) - i->status = ImageItem::IgnoreItem; - } - } - } while (!in.atEnd()); - } - } - - QByteArray block; - QDataStream ods(&block, QIODevice::WriteOnly); - ods << itemList; - proto.sendBlock(BaselineProtocol::Ack, block); - report.addItems(itemList); -} - - -void BaselineHandler::recordMatch(const QByteArray &itemBlock) -{ - QDataStream ds(itemBlock); - ImageItem item; - ds >> item; - report.addResult(item); - proto.sendBlock(BaselineProtocol::Ack, QByteArray()); -} - - -void BaselineHandler::storeImage(const QByteArray &itemBlock, bool isBaseline) -{ - QDataStream ds(itemBlock); - ImageItem item; - ds >> item; - - if (isBaseline && !clientInfo.overrides().isEmpty()) { - qDebug() << runId << logtime() << "Received baseline from client with override info, ignoring. Item:" << item.itemName; - proto.sendBlock(BaselineProtocol::UnknownError, "New baselines not accepted from client with override info."); - return; - } - - QString blPrefix = pathForItem(item, true); - QString mmPrefix = pathForItem(item, false); - QString prefix = isBaseline ? blPrefix : mmPrefix; - - qDebug() << runId << logtime() << "Received" << (isBaseline ? "baseline" : "mismatched") << "image for:" << item.itemName << "Storing in" << prefix; - - // Reply to the client - QString msg; - if (isBaseline) - msg = QLS("New baseline image stored: ") + blPrefix + QLS(FileFormat); - else - msg = BaselineServer::baseUrl() + report.filePath(); - - if (isBaseline || !fuzzLevel) - proto.sendBlock(BaselineProtocol::Ack, msg.toLatin1()); // Do early reply if possible: don't make the client wait longer than necessary - - // Store the image - QString dir = prefix.section(QLC('/'), 0, -2); - QDir cwd; - if (!cwd.exists(dir)) - cwd.mkpath(dir); - item.image.save(prefix + QLS(FileFormat), FileFormat); - - PlatformInfo itemData = clientInfo; - itemData.insert(PI_ImageChecksum, QString::number(item.imageChecksums.at(0), 16)); //# Only the first is stored. TBD: get rid of list - itemData.insert(PI_RunId, runId); - itemData.insert(PI_CreationDate, QDateTime::currentDateTime().toString()); - storeItemMetadata(itemData, prefix); - - if (!isBaseline) { - // Do fuzzy matching - bool fuzzyMatch = false; - if (fuzzLevel) { - BaselineProtocol::Command cmd = BaselineProtocol::Ack; - fuzzyMatch = fuzzyCompare(blPrefix, mmPrefix); - if (fuzzyMatch) { - msg.prepend(QString("Fuzzy match at fuzzlevel %1%. Report: ").arg(fuzzLevel)); - cmd = BaselineProtocol::FuzzyMatch; - } - proto.sendBlock(cmd, msg.toLatin1()); // We didn't reply earlier - } - - // Add to report - item.status = fuzzyMatch ? ImageItem::FuzzyMatch : ImageItem::Mismatch; - report.addResult(item); - } -} - - -void BaselineHandler::storeItemMetadata(const PlatformInfo &metadata, const QString &path) -{ - QFile file(path + QLS(MetadataFileExt)); - if (!file.open(QIODevice::WriteOnly | QIODevice::Truncate)) { - qWarning() << runId << logtime() << "ERROR: could not write to file" << file.fileName(); - return; - } - QTextStream out(&file); - PlatformInfo::const_iterator it = metadata.constBegin(); - while (it != metadata.constEnd()) { - out << it.key() << ": " << it.value() << endl; - ++it; - } - file.close(); -} - - -PlatformInfo BaselineHandler::fetchItemMetadata(const QString &path) -{ - PlatformInfo res; - QFile file(path + QLS(MetadataFileExt)); - if (!file.open(QIODevice::ReadOnly) || !QFile::exists(path + QLS(FileFormat))) - return res; - QTextStream in(&file); - do { - QString line = in.readLine(); - int idx = line.indexOf(QLS(": ")); - if (idx > 0) - res.insert(line.left(idx), line.mid(idx+2)); - } while (!in.atEnd()); - return res; -} - - -void BaselineHandler::idleClientTimeout() -{ - qWarning() << runId << logtime() << "Idle client timeout: no request received for" << IDLE_CLIENT_TIMEOUT << "seconds, terminating connection."; - proto.socket.disconnectFromHost(); -} - - -void BaselineHandler::receiveDisconnect() -{ - qDebug() << runId << logtime() << "Client disconnected."; - report.end(); - if (report.reportProduced() && !clientInfo.isAdHocRun()) - issueMismatchNotification(); - if (settings && settings->value("ProcessXmlResults").toBool() && !clientInfo.isAdHocRun()) { - // ### TBD: actually execute the processing command. For now, just generate the xml files. - QString xmlDir = report.writeResultsXmlFiles(); - } - QThread::currentThread()->exit(0); -} - - -PlatformInfo BaselineHandler::mapPlatformInfo(const PlatformInfo& orig) const -{ - PlatformInfo mapped; - foreach (const QString &key, orig.uniqueKeys()) { - QString val = orig.value(key).simplified(); - val.replace(QLC('/'), QLC('_')); - val.replace(QLC(' '), QLC('_')); - mapped.insert(key, QUrl::toPercentEncoding(val, "+")); - //qDebug() << "MAPPED" << key << "FROM" << orig.value(key) << "TO" << mapped.value(key); - } - - // Special fixup for OS version - if (mapped.value(PI_OSName) == QLS("MacOS")) { - int ver = mapped.value(PI_OSVersion).toInt(); - if (ver > 1) - mapped.insert(PI_OSVersion, QString("MV_10_%1").arg(ver-2)); - } - else if (mapped.value(PI_OSName) == QLS("Windows")) { - // TBD: map windows version numbers to names - } - - // Special fixup for hostname - QString host = mapped.value(PI_HostName).section(QLC('.'), 0, 0); // Filter away domain, if any - if (host.isEmpty() || host == QLS("localhost")) { - host = orig.value(PI_HostAddress); - } else { - if (!orig.isAdHocRun()) { // i.e. CI system run, so remove index postfix typical of vm hostnames - host.remove(QRegularExpression(QLS("\\d+$"))); - if (host.endsWith(QLC('-'))) - host.chop(1); - } - } - if (host.isEmpty()) - host = QLS("UNKNOWN-HOST"); - if (mapped.value(PI_OSName) == QLS("MacOS")) // handle multiple os versions on same host - host += QLC('-') + mapped.value(PI_OSVersion); - mapped.insert(PI_HostName, host); - - // Special fixup for Qt version - QString ver = mapped.value(PI_QtVersion); - if (!ver.isEmpty()) - mapped.insert(PI_QtVersion, ver.prepend(QLS("Qt-"))); - - return mapped; -} - - -QString BaselineHandler::pathForItem(const ImageItem &item, bool isBaseline, bool absolute) const -{ - if (mappedClientInfo.isEmpty()) { - mappedClientInfo = mapPlatformInfo(clientInfo); - PlatformInfo oraw = clientInfo; - // ### simplify: don't map if no overrides! - for (int i = 0; i < clientInfo.overrides().size()-1; i+=2) - oraw.insert(clientInfo.overrides().at(i), clientInfo.overrides().at(i+1)); - overriddenMappedClientInfo = mapPlatformInfo(oraw); - } - - const PlatformInfo& mapped = isBaseline ? overriddenMappedClientInfo : mappedClientInfo; - - QString itemName = safeName(item.itemName); - itemName.append(QLC('_') + QString::number(item.itemChecksum, 16).rightJustified(4, QLC('0'))); - - QStringList path; - path += projectPath(absolute); - path += mapped.value(PI_TestCase); - path += QLS(isBaseline ? "baselines" : "mismatches"); - path += item.testFunction; - QStringList itemPathKeys; - if (settings) - itemPathKeys = settings->value("ItemPathKeys").toStringList(); - if (itemPathKeys.isEmpty()) - itemPathKeys = BaselineServer::defaultPathKeys(); - foreach (const QString &key, itemPathKeys) - path += mapped.value(key, QLS("UNSET-")+key); - if (!isBaseline) - path += runId; - path += itemName + QLC('.'); - - return path.join(QLS("/")); -} - - -QString BaselineHandler::view(const QString &baseline, const QString &rendered, const QString &compared) -{ - QFile f(":/templates/view.html"); - f.open(QIODevice::ReadOnly); - return QString::fromLatin1(f.readAll()).arg('/'+baseline, '/'+rendered, '/'+compared, diffstats(baseline, rendered)); -} - -QString BaselineHandler::diffstats(const QString &baseline, const QString &rendered) -{ - QImage blImg(BaselineServer::storagePath() + QLC('/') + baseline); - QImage mmImg(BaselineServer::storagePath() + QLC('/') + rendered); - if (blImg.isNull() || mmImg.isNull()) - return QLS("Could not compute diffstats: image loading failed."); - - // ### TBD: cache the results - return computeMismatchScore(blImg, mmImg); -} - -QString BaselineHandler::clearAllBaselines(const QString &context) -{ - int tot = 0; - int failed = 0; - QDirIterator it(BaselineServer::storagePath() + QLC('/') + context, - QStringList() << QLS("*.") + QLS(FileFormat) - << QLS("*.") + QLS(MetadataFileExt) - << QLS("*.") + QLS(ThumbnailExt)); - while (it.hasNext()) { - bool counting = !it.next().endsWith(QLS(ThumbnailExt)); - if (counting) - tot++; - if (!QFile::remove(it.filePath()) && counting) - failed++; - } - return QString(QLS("%1 of %2 baselines cleared from context ")).arg((tot-failed)/2).arg(tot/2) + context; -} - -QString BaselineHandler::updateBaselines(const QString &context, const QString &mismatchContext, const QString &itemFile) -{ - int tot = 0; - int failed = 0; - QString storagePrefix = BaselineServer::storagePath() + QLC('/'); - // If itemId is set, update just that one, otherwise, update all: - QString filter = itemFile.isEmpty() ? QLS("*_????.") : itemFile; - QDirIterator it(storagePrefix + mismatchContext, - QStringList() << filter + QLS(FileFormat) - << filter + QLS(MetadataFileExt) - << filter + QLS(ThumbnailExt)); - while (it.hasNext()) { - bool counting = !it.next().endsWith(QLS(ThumbnailExt)); - if (counting) - tot++; - QString oldFile = storagePrefix + context + QLC('/') + it.fileName(); - QFile::remove(oldFile); // Remove existing baseline file - if (!QFile::copy(it.filePath(), oldFile) && counting) // and replace it with the mismatch - failed++; - } - return QString(QLS("%1 of %2 baselines updated in context %3 from context %4")).arg((tot-failed)/2).arg(tot/2).arg(context, mismatchContext); -} - -QString BaselineHandler::blacklistTest(const QString &context, const QString &itemId, bool removeFromBlacklist) -{ - QFile file(BaselineServer::storagePath() + QLC('/') + context + QLS("/BLACKLIST")); - QStringList blackList; - if (file.open(QIODevice::ReadWrite)) { - while (!file.atEnd()) - blackList.append(file.readLine().trimmed()); - - if (removeFromBlacklist) - blackList.removeAll(itemId); - else if (!blackList.contains(itemId)) - blackList.append(itemId); - - file.resize(0); - foreach (QString id, blackList) - file.write(id.toLatin1() + '\n'); - file.close(); - return QLS(removeFromBlacklist ? "Whitelisted " : "Blacklisted ") + itemId + QLS(" in context ") + context; - } else { - return QLS("Unable to update blacklisted tests, failed to open ") + file.fileName(); - } -} - - -void BaselineHandler::testPathMapping() -{ - qDebug() << "Storage prefix:" << BaselineServer::storagePath(); - - QStringList hosts; - hosts << QLS("bq-ubuntu910-x86-01") - << QLS("bq-ubuntu910-x86-15") - << QLS("osl-mac-master-5.test.qt-project.org") - << QLS("osl-mac-master-6.test.qt-project.org") - << QLS("sv-xp-vs-010") - << QLS("sv-xp-vs-011") - << QLS("sv-solaris-sparc-008") - << QLS("macbuilder-02.test.troll.no") - << QLS("bqvm1164") - << QLS("chimera") - << QLS("localhost") - << QLS(""); - - ImageItem item; - item.testFunction = QLS("testPathMapping"); - item.itemName = QLS("arcs.qps"); - item.imageChecksums << 0x0123456789abcdefULL; - item.itemChecksum = 0x0123; - - clientInfo.insert(PI_QtVersion, QLS("5.0.0")); - clientInfo.insert(PI_QMakeSpec, QLS("linux-g++")); - clientInfo.insert(PI_PulseGitBranch, QLS("somebranch")); - clientInfo.setAdHocRun(false); - foreach(const QString& host, hosts) { - mappedClientInfo.clear(); - clientInfo.insert(PI_HostName, host); - qDebug() << "Baseline from" << host << "->" << pathForItem(item, true); - qDebug() << "Mismatch from" << host << "->" << pathForItem(item, false); - } -} - - -QString BaselineHandler::computeMismatchScore(const QImage &baseline, const QImage &rendered) -{ - if (baseline.size() != rendered.size() || baseline.format() != rendered.format()) - return QLS("[No diffstats, incomparable images.]"); - if (baseline.depth() != 32) - return QLS("[Diffstats computation not implemented for format.]"); - - int w = baseline.width(); - int h = baseline.height(); - - uint ncd = 0; // number of differing color pixels - uint nad = 0; // number of differing alpha pixels - uint scd = 0; // sum of color pixel difference - uint sad = 0; // sum of alpha pixel difference - uint mind = 0; // minimum difference - uint maxd = 0; // maximum difference - - for (int y=0; y<h; ++y) { - const QRgb *bl = (const QRgb *) baseline.constScanLine(y); - const QRgb *rl = (const QRgb *) rendered.constScanLine(y); - for (int x=0; x<w; ++x) { - QRgb b = bl[x]; - QRgb r = rl[x]; - if (r != b) { - uint dr = qAbs(qRed(b) - qRed(r)); - uint dg = qAbs(qGreen(b) - qGreen(r)); - uint db = qAbs(qBlue(b) - qBlue(r)); - uint ds = (dr + dg + db) / 3; - uint da = qAbs(qAlpha(b) - qAlpha(r)); - if (ds) { - ncd++; - scd += ds; - if (!mind || ds < mind) - mind = ds; - if (ds > maxd) - maxd = ds; - } - if (da) { - nad++; - sad += da; - } - } - } - } - - - double pcd = 100.0 * ncd / (w*h); // percent of pixels that differ - double acd = ncd ? double(scd) / (ncd) : 0; // avg. difference -/* - if (baseline.hasAlphaChannel()) { - double pad = 100.0 * nad / (w*h); // percent of pixels that differ - double aad = nad ? double(sad) / (3*nad) : 0; // avg. difference - } -*/ - QString res = "<table>\n"; - QString item = "<tr><td>%1</td><td align=right>%2</td></tr>\n"; - res += item.arg("Number of mismatching pixels").arg(ncd); - res += item.arg("Percentage mismatching pixels").arg(pcd, 0, 'g', 2); - res += item.arg("Minimum pixel distance").arg(mind); - res += item.arg("Maximum pixel distance").arg(maxd); - if (acd >= 10.0) - res += item.arg("Average pixel distance").arg(qRound(acd)); - else - res += item.arg("Average pixel distance").arg(acd, 0, 'g', 2); - - if (baseline.hasAlphaChannel()) - res += item.arg("Number of mismatching alpha values").arg(nad); - - res += "</table>\n"; - res += "<p>(Distances are normalized to the range 0-255)</p>\n"; - return res; -} - - -bool BaselineHandler::fuzzyCompare(const QString &baselinePath, const QString &mismatchPath) -{ - QProcess compareProc; - QStringList args; - args << "-fuzz" << QString("%1%").arg(fuzzLevel) << "-metric" << "AE"; - args << baselinePath + QLS(FileFormat) << mismatchPath + QLS(FileFormat) << "/dev/null"; // TBD: Should save output image, so report won't have to regenerate it - - compareProc.setProcessChannelMode(QProcess::MergedChannels); - compareProc.start("compare", args, QIODevice::ReadOnly); - if (compareProc.waitForFinished(3000) && compareProc.error() == QProcess::UnknownError) { - bool ok = false; - int metric = compareProc.readAll().trimmed().toInt(&ok); - if (ok && metric == 0) - return true; - } - return false; -} - - -void BaselineHandler::issueMismatchNotification() -{ - // KISS: hardcoded use of the "sendemail" utility. Make this configurable if and when demand arises. - if (!settings) - return; - - settings->beginGroup("Notification"); - QStringList receivers = settings->value("Receivers").toStringList(); - QString sender = settings->value("Sender").toString(); - QString server = settings->value("SMTPserver").toString(); - settings->endGroup(); - if (receivers.isEmpty() || sender.isEmpty() || server.isEmpty()) - return; - - QString msg = QString("\nResult summary for test run %1:\n").arg(runId); - msg += report.summary(); - msg += "\nReport: " + BaselineServer::baseUrl() + report.filePath() + "\n"; - - msg += "\nTest run platform properties:\n------------------\n"; - foreach (const QString &key, clientInfo.keys()) - msg += key + ": " + clientInfo.value(key) + '\n'; - msg += "\nCheers,\n- Your friendly Lancelot Baseline Server\n"; - - QProcess proc; - QString cmd = "sendemail"; - QStringList args; - args << "-s" << server << "-f" << sender << "-t" << receivers; - args << "-u" << "[Lancelot] Mismatch report for project " + clientInfo.value(PI_Project) + ", test case " + clientInfo.value(PI_TestCase); - args << "-m" << msg; - - //proc.setProcessChannelMode(QProcess::MergedChannels); - proc.start(cmd, args); - if (!proc.waitForFinished(10 * 1000) || (proc.exitStatus() != QProcess::NormalExit) || proc.exitCode()) { - qWarning() << "FAILED to issue notification. Command:" << cmd << args.mid(0, args.size()-2); - qWarning() << " Command standard output:" << proc.readAllStandardOutput(); - qWarning() << " Command error output:" << proc.readAllStandardError(); - } -} - - -// Make an identifer safer for use as filename and URL -QString safeName(const QString& name) -{ - QString res = name.simplified(); - res.replace(QLC(' '), QLC('_')); - res.replace(QLC('.'), QLC('_')); - res.replace(QLC('/'), QLC('^')); - return res; -} diff --git a/tests/baselineserver/src/baselineserver.h b/tests/baselineserver/src/baselineserver.h deleted file mode 100644 index 25ef17f023..0000000000 --- a/tests/baselineserver/src/baselineserver.h +++ /dev/null @@ -1,151 +0,0 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the test suite of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:GPL-EXCEPT$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 3 as published by the Free Software -** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ -#ifndef BASELINESERVER_H -#define BASELINESERVER_H - -#include <QStringList> -#include <QTcpServer> -#include <QThread> -#include <QTcpSocket> -#include <QScopedPointer> -#include <QTimer> -#include <QDateTime> -#include <QSettings> - -#include "baselineprotocol.h" -#include "report.h" - -// #seconds between checks for update of the executable -#define HEARTBEAT 10 -// Timeout if no activity received from client, #seconds -#define IDLE_CLIENT_TIMEOUT 3*60 - -#define MetadataFileExt "metadata" -#define ThumbnailExt "thumbnail.jpg" - - -class BaselineServer : public QTcpServer -{ - Q_OBJECT - -public: - BaselineServer(QObject *parent = nullptr); - - static QString storagePath(); - static QString baseUrl(); - static QStringList defaultPathKeys(); - -protected: - void incomingConnection(qintptr socketDescriptor); - -private slots: - void heartbeat(); - -private: - QTimer *heartbeatTimer; - QDateTime meLastMod; - QString lastRunId; - int lastRunIdIdx; - static QString storage; - static QString url; - static QStringList pathKeys; -}; - - - -class BaselineThread : public QThread -{ - Q_OBJECT - -public: - BaselineThread(const QString &runId, int socketDescriptor, QObject *parent); - void run(); - -private: - QString runId; - int socketDescriptor; -}; - - -class BaselineHandler : public QObject -{ - Q_OBJECT - -public: - BaselineHandler(const QString &runId, int socketDescriptor = -1); - QString projectPath(bool absolute = true) const; - QString pathForItem(const ImageItem &item, bool isBaseline = true, bool absolute = true) const; - - // CGI callbacks: - static QString view(const QString &baseline, const QString &rendered, const QString &compared); - static QString diffstats(const QString &baseline, const QString &rendered); - static QString clearAllBaselines(const QString &context); - static QString updateBaselines(const QString &context, const QString &mismatchContext, const QString &itemFile); - static QString blacklistTest(const QString &context, const QString &itemId, bool removeFromBlacklist = false); - - // for debugging - void testPathMapping(); - -private slots: - void receiveRequest(); - void receiveDisconnect(); - void idleClientTimeout(); - -private: - bool checkClient(QByteArray *errMsg, bool *dryRunMode = nullptr); - bool establishConnection(); - void provideBaselineChecksums(const QByteArray &itemListBlock); - void recordMatch(const QByteArray &itemBlock); - void storeImage(const QByteArray &itemBlock, bool isBaseline); - void storeItemMetadata(const PlatformInfo &metadata, const QString &path); - PlatformInfo fetchItemMetadata(const QString &path); - PlatformInfo mapPlatformInfo(const PlatformInfo& orig) const; - const char *logtime(); - void issueMismatchNotification(); - bool fuzzyCompare(const QString& baselinePath, const QString& mismatchPath); - - static QString computeMismatchScore(const QImage& baseline, const QImage& rendered); - - BaselineProtocol proto; - PlatformInfo clientInfo; - mutable PlatformInfo mappedClientInfo; - mutable PlatformInfo overriddenMappedClientInfo; - QString runId; - bool connectionEstablished; - Report report; - QSettings *settings; - QString ruleName; - int fuzzLevel; - QTimer *idleTimer; -}; - - -// Make an identifer safer for use as filename and URL -QString safeName(const QString& name); - -#endif // BASELINESERVER_H diff --git a/tests/baselineserver/src/baselineserver.pro b/tests/baselineserver/src/baselineserver.pro deleted file mode 100644 index 2d8438cb51..0000000000 --- a/tests/baselineserver/src/baselineserver.pro +++ /dev/null @@ -1,24 +0,0 @@ -QT += core network - -# gui needed for QImage -# QT -= gui - -TARGET = baselineserver -DESTDIR = ../bin -CONFIG += cmdline - -TEMPLATE = app - -include(../shared/baselineprotocol.pri) - -SOURCES += main.cpp \ - baselineserver.cpp \ - report.cpp - -HEADERS += \ - baselineserver.h \ - report.h - -RESOURCES += \ - baselineserver.qrc -DEFINES += QT_DISABLE_DEPRECATED_BEFORE=0 diff --git a/tests/baselineserver/src/baselineserver.qrc b/tests/baselineserver/src/baselineserver.qrc deleted file mode 100644 index b5cd6afadb..0000000000 --- a/tests/baselineserver/src/baselineserver.qrc +++ /dev/null @@ -1,5 +0,0 @@ -<RCC> - <qresource prefix="/"> - <file>templates/view.html</file> - </qresource> -</RCC> diff --git a/tests/baselineserver/src/main.cpp b/tests/baselineserver/src/main.cpp deleted file mode 100644 index dfc9b83da8..0000000000 --- a/tests/baselineserver/src/main.cpp +++ /dev/null @@ -1,57 +0,0 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the test suite of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:GPL-EXCEPT$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 3 as published by the Free Software -** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ -#include <QtCore/QCoreApplication> -#include "baselineserver.h" - -int main(int argc, char *argv[]) -{ - QCoreApplication a(argc, argv); - - QString queryString(qgetenv("QUERY_STRING")); - if (!queryString.isEmpty()) { - // run as CGI script - Report::handleCGIQuery(queryString); - return 0; - } - - if (a.arguments().contains(QLatin1String("-testmapping"))) { - BaselineHandler h(QLS("SomeRunId")); - h.testPathMapping(); - return 0; - } - - BaselineServer server; - if (!server.listen(QHostAddress::Any, BaselineProtocol::ServerPort)) { - qWarning("Failed to listen!"); - return 1; - } - - qDebug() << "\n*****" << argv[0] << "started, ready to serve on port" << BaselineProtocol::ServerPort - << "with baseline protocol version" << BaselineProtocol::ProtocolVersion << "*****\n"; - return a.exec(); -} diff --git a/tests/baselineserver/src/report.cpp b/tests/baselineserver/src/report.cpp deleted file mode 100644 index 748d76ebfe..0000000000 --- a/tests/baselineserver/src/report.cpp +++ /dev/null @@ -1,503 +0,0 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the test suite of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:GPL-EXCEPT$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 3 as published by the Free Software -** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ -#include "report.h" -#include "baselineprotocol.h" -#include "baselineserver.h" -#include <QDir> -#include <QProcess> -#include <QUrl> -#include <QXmlStreamWriter> -#include <QRegularExpression> -#include <unistd.h> - -Report::Report() - : initialized(false), handler(0), written(false), numItems(0), numMismatches(0), settings(0), - hasStats(false) -{ -} - -Report::~Report() -{ - end(); -} - -QString Report::filePath() -{ - return path; -} - -int Report::numberOfMismatches() -{ - return numMismatches; -} - -bool Report::reportProduced() -{ - return written; -} - -void Report::init(const BaselineHandler *h, const QString &r, const PlatformInfo &p, const QSettings *s) -{ - handler = h; - runId = r; - plat = p; - settings = s; - rootDir = BaselineServer::storagePath() + QLC('/'); - baseDir = handler->pathForItem(ImageItem(), true, false).remove(QRegularExpression("/baselines/.*$")); - QString dir = baseDir + (plat.isAdHocRun() ? QLS("/adhoc-reports") : QLS("/auto-reports")); - QDir cwd; - if (!cwd.exists(rootDir + dir)) - cwd.mkpath(rootDir + dir); - path = dir + QLS("/Report_") + runId + QLS(".html"); - hasOverride = !plat.overrides().isEmpty(); - initialized = true; -} - -void Report::addItems(const ImageItemList &items) -{ - if (items.isEmpty()) - return; - numItems += items.size(); - QString func = items.at(0).testFunction; - if (!testFunctions.contains(func)) - testFunctions.append(func); - ImageItemList list = items; - if (settings->value("ReportMissingResults").toBool()) { - for (ImageItemList::iterator it = list.begin(); it != list.end(); ++it) { - if (it->status == ImageItem::Ok) - it->status = ImageItem::Error; // Status should be set by report from client, else report as error - } - } - itemLists[func] += list; -} - -void Report::addResult(const ImageItem &item) -{ - if (!testFunctions.contains(item.testFunction)) { - qWarning() << "Report::addResult: unknown testfunction" << item.testFunction; - return; - } - bool found = false; - ImageItemList &list = itemLists[item.testFunction]; - for (ImageItemList::iterator it = list.begin(); it != list.end(); ++it) { - if (it->itemName == item.itemName && it->itemChecksum == item.itemChecksum) { - it->status = item.status; - found = true; - break; - } - } - if (found) { - if (item.status == ImageItem::Mismatch) - numMismatches++; - } else { - qWarning() << "Report::addResult: unknown item" << item.itemName << "in testfunction" << item.testFunction; - } -} - -void Report::end() -{ - if (!initialized || written) - return; - // Make report iff (#mismatches>0) || (#fuzzymatches>0) || (#errors>0 && settings say report errors) - bool doReport = (numMismatches > 0); - if (!doReport) { - bool reportErrors = settings->value("ReportMissingResults").toBool(); - computeStats(); - foreach (const QString &func, itemLists.keys()) { - FuncStats stat = stats.value(func); - if (stat.value(ImageItem::FuzzyMatch) > 0) { - doReport = true; - break; - } - foreach (const ImageItem &item, itemLists.value(func)) { - if (reportErrors && item.status == ImageItem::Error) { - doReport = true; - break; - } - } - if (doReport) - break; - } - } - if (!doReport) - return; - write(); - written = true; -} - -void Report::computeStats() -{ - if (hasStats) - return; - foreach (const QString &func, itemLists.keys()) { - FuncStats funcStat; - funcStat[ImageItem::Ok] = 0; - funcStat[ImageItem::BaselineNotFound] = 0; - funcStat[ImageItem::IgnoreItem] = 0; - funcStat[ImageItem::Mismatch] = 0; - funcStat[ImageItem::FuzzyMatch] = 0; - funcStat[ImageItem::Error] = 0; - foreach (const ImageItem &item, itemLists.value(func)) { - funcStat[item.status]++; - } - stats[func] = funcStat; - } - hasStats = true; -} - -QString Report::summary() -{ - computeStats(); - QString res; - foreach (const QString &func, itemLists.keys()) { - FuncStats stat = stats.value(func); - QString s = QString("%1 %3 mismatch(es), %4 error(s), %5 fuzzy match(es)\n"); - s = s.arg(QString("%1() [%2 items]:").arg(func).arg(itemLists.value(func).size()).leftJustified(40)); - s = s.arg(stat.value(ImageItem::Mismatch)); - s = s.arg(stat.value(ImageItem::Error)); - s = s.arg(stat.value(ImageItem::FuzzyMatch)); - res += s; - } -#if 0 - qDebug() << "***************************** Summary *************************"; - qDebug() << res; - qDebug() << "***************************************************************"; -#endif - return res; -} - -void Report::write() -{ - QFile file(rootDir + path); - if (!file.open(QIODevice::WriteOnly | QIODevice::Truncate)) { - qWarning() << "Failed to open report file" << file.fileName(); - return; - } - out.setDevice(&file); - - writeHeader(); - foreach(const QString &func, testFunctions) { - writeFunctionResults(itemLists.value(func)); - } - writeFooter(); - file.close(); - updateLatestPointer(); -} - - -void Report::writeHeader() -{ - QString title = plat.value(PI_Project) + QLC(':') + plat.value(PI_TestCase) + QLS(" Lancelot Test Report"); - out << "<!DOCTYPE html>\n" - << "<html><head><title>" << title << "</title></head>\n" - << "<body bgcolor=""#ddeeff""><h1>" << title << "</h1>\n" - << "<p>Note: This is a <i>static</i> page, generated at " << QDateTime::currentDateTime().toString() - << " for the test run with id " << runId << "</p>\n" - << "<p>Summary: <b><span style=\"color:red\">" << numMismatches << " of " << numItems << "</span></b> items reported mismatching</p>\n"; - out << "<pre>\n" << summary() << "</pre>\n\n"; - out << "<h3>Testing Client Platform Info:</h3>\n" - << "<table>\n"; - foreach (QString key, plat.keys()) - out << "<tr><td>" << key << ":</td><td>" << plat.value(key) << "</td></tr>\n"; - out << "</table>\n\n"; - if (hasOverride) { - out << "<span style=\"color:red\"><h4>Note! Override Platform Info:</h4></span>\n" - << "<p>The client's output has been compared to baselines created on a different platform. Differences:</p>\n" - << "<table>\n"; - for (int i = 0; i < plat.overrides().size()-1; i+=2) - out << "<tr><td>" << plat.overrides().at(i) << ":</td><td>" << plat.overrides().at(i+1) << "</td></tr>\n"; - out << "</table>\n\n"; - } -} - - -void Report::writeFunctionResults(const ImageItemList &list) -{ - QString testFunction = list.at(0).testFunction; - QString pageUrl = BaselineServer::baseUrl() + path; - QString ctx = handler->pathForItem(list.at(0), true, false).section(QLC('/'), 0, -2); - QString misCtx = handler->pathForItem(list.at(0), false, false).section(QLC('/'), 0, -2); - - - out << "\n<p> </p><h3>Test function: " << testFunction << "</h3>\n"; - if (!hasOverride) { - out << "<p><a href=\"/cgi-bin/server.cgi?cmd=clearAllBaselines&context=" << ctx << "&url=" << pageUrl - << "\"><b>Clear all baselines</b></a> for this testfunction (They will be recreated by the next run)</p>\n"; - out << "<p><a href=\"/cgi-bin/server.cgi?cmd=updateAllBaselines&context=" << ctx << "&mismatchContext=" << misCtx << "&url=" << pageUrl - << "\"><b>Let these mismatching images be the new baselines</b></a> for this testfunction</p>\n\n"; - } - - out << "<table border=\"2\">\n" - "<tr>\n" - "<th width=123>Item</th>\n" - "<th width=246>Baseline</th>\n" - "<th width=246>Rendered</th>\n" - "<th width=246>Comparison (diffs are <span style=\"color:red\">RED</span>)</th>\n" - "<th width=246>Info/Action</th>\n" - "</tr>\n\n"; - - foreach (const ImageItem &item, list) { - QString mmPrefix = handler->pathForItem(item, false, false); - QString blPrefix = handler->pathForItem(item, true, false); - - // Make hard links to the current baseline, so that the report is static even if the baseline changes - generateThumbnail(blPrefix + QLS(FileFormat), rootDir); // Make sure baseline thumbnail is up to date - QString lnPrefix = mmPrefix + QLS("baseline."); - QByteArray blPrefixBa = (rootDir + blPrefix).toLatin1(); - QByteArray lnPrefixBa = (rootDir + lnPrefix).toLatin1(); - ::link((blPrefixBa + FileFormat).constData(), (lnPrefixBa + FileFormat).constData()); - ::link((blPrefixBa + MetadataFileExt).constData(), (lnPrefixBa + MetadataFileExt).constData()); - ::link((blPrefixBa + ThumbnailExt).constData(), (lnPrefixBa + ThumbnailExt).constData()); - - QString baseline = lnPrefix + QLS(FileFormat); - QString metadata = lnPrefix + QLS(MetadataFileExt); - out << "<tr>\n"; - out << "<td>" << item.itemName << "</td>\n"; - if (item.status == ImageItem::Mismatch || item.status == ImageItem::FuzzyMatch) { - QString rendered = mmPrefix + QLS(FileFormat); - QString itemFile = mmPrefix.section(QLC('/'), -1); - writeItem(baseline, rendered, item, itemFile, ctx, misCtx, metadata); - } - else { - out << "<td align=center><a href=\"/" << baseline << "\">image</a> <a href=\"/" << metadata << "\">info</a></td>\n" - << "<td align=center colspan=2><small>n/a</small></td>\n" - << "<td align=center>"; - switch (item.status) { - case ImageItem::BaselineNotFound: - out << "Baseline not found/regenerated"; - break; - case ImageItem::IgnoreItem: - out << "<span style=\"background-color:yellow\">Blacklisted</span> "; - if (!hasOverride) { - out << "<a href=\"/cgi-bin/server.cgi?cmd=whitelist&context=" << ctx - << "&itemId=" << item.itemName << "&url=" << pageUrl - << "\">Whitelist this item</a>"; - } - break; - case ImageItem::Error: - out << "<span style=\"background-color:red\">Error: No result reported!</span>"; - break; - case ImageItem::Ok: - out << "<span style=\"color:green\"><small>No mismatch reported</small></span>"; - break; - default: - out << '?'; - break; - } - out << "</td>\n"; - } - out << "</tr>\n\n"; - } - - out << "</table>\n"; -} - -void Report::writeItem(const QString &baseline, const QString &rendered, const ImageItem &item, - const QString &itemFile, const QString &ctx, const QString &misCtx, const QString &metadata) -{ - QString compared = generateCompared(baseline, rendered); - QString pageUrl = BaselineServer::baseUrl() + path; - - QStringList images = QStringList() << baseline << rendered << compared; - foreach (const QString& img, images) - out << "<td height=246 align=center><a href=\"/" << img << "\"><img src=\"/" << generateThumbnail(img, rootDir) << "\"></a></td>\n"; - - out << "<td align=center>\n"; - if (item.status == ImageItem::FuzzyMatch) - out << "<p><span style=\"color:orange\">Fuzzy match</span></p>\n"; - else - out << "<p><span style=\"color:red\">Mismatch reported</span></p>\n"; - out << "<p><a href=\"/" << metadata << "\">Baseline Info</a>\n"; - if (!hasOverride) { - out << "<p><a href=\"/cgi-bin/server.cgi?cmd=updateSingleBaseline&context=" << ctx << "&mismatchContext=" << misCtx - << "&itemFile=" << itemFile << "&url=" << pageUrl << "\">Let this be the new baseline</a></p>\n" - << "<p><a href=\"/cgi-bin/server.cgi?cmd=blacklist&context=" << ctx - << "&itemId=" << item.itemName << "&url=" << pageUrl << "\">Blacklist this item</a></p>\n"; - } - out << "<p><a href=\"/cgi-bin/server.cgi?cmd=view&baseline=" << baseline << "&rendered=" << rendered - << "&compared=" << compared << "&url=" << pageUrl << "\">Inspect</a></p>\n"; - -#if 0 - out << "<p><a href=\"/cgi-bin/server.cgi?cmd=diffstats&baseline=" << baseline << "&rendered=" << rendered - << "&url=" << pageUrl << "\">Diffstats</a></p>\n"; -#endif - - out << "</td>\n"; -} - -void Report::writeFooter() -{ - out << "\n</body></html>\n"; -} - - -QString Report::generateCompared(const QString &baseline, const QString &rendered, bool fuzzy) -{ - QString res = rendered; - QFileInfo fi(res); - res.chop(fi.suffix().length()); - res += QLS(fuzzy ? "fuzzycompared.png" : "compared.png"); - QStringList args; - if (fuzzy) - args << QLS("-fuzz") << QLS("5%"); - args << rootDir+baseline << rootDir+rendered << rootDir+res; - QProcess::execute(QLS("compare"), args); - return res; -} - - -QString Report::generateThumbnail(const QString &image, const QString &rootDir) -{ - QString res = image; - QFileInfo imgFI(rootDir+image); - if (!imgFI.exists()) - return res; - res.chop(imgFI.suffix().length()); - res += ThumbnailExt; - QFileInfo resFI(rootDir+res); - if (resFI.exists() && resFI.lastModified() > imgFI.lastModified()) - return res; - QStringList args; - args << rootDir+image << QLS("-resize") << QLS("240x240>") << QLS("-quality") << QLS("50") << rootDir+res; - QProcess::execute(QLS("convert"), args); - return res; -} - - -QString Report::writeResultsXmlFiles() -{ - if (!itemLists.size()) - return QString(); - QString dir = rootDir + baseDir + QLS("/xml-reports/") + runId; - QDir cwd; - if (!cwd.exists(dir)) - cwd.mkpath(dir); - foreach (const QString &func, itemLists.keys()) { - QFile f(dir + QLatin1Char('/') + func + "-results.xml"); - if (!f.open(QIODevice::WriteOnly)) - continue; - QXmlStreamWriter s(&f); - s.setAutoFormatting(true); - s.writeStartDocument(); - foreach (QString key, plat.keys()) { - QString cmt = QLatin1Char(' ') + key + "=\"" + plat.value(key) +"\" "; - s.writeComment(cmt.replace("--", "[-]")); - } - s.writeStartElement("testsuite"); - s.writeAttribute("name", func); - foreach (const ImageItem &item, itemLists.value(func)) { - QString res; - switch (item.status) { - case ImageItem::Ok: - case ImageItem::FuzzyMatch: - res = "pass"; - break; - case ImageItem::Mismatch: - case ImageItem::Error: - res = "fail"; - break; - case ImageItem::BaselineNotFound: - case ImageItem::IgnoreItem: - default: - res = "skip"; - } - s.writeStartElement("testcase"); - s.writeAttribute("name", item.itemName); - s.writeAttribute("result", res); - s.writeEndElement(); - } - s.writeEndElement(); - s.writeEndDocument(); - } - return dir; -} - - - -void Report::updateLatestPointer() -{ - QString linkPath = rootDir + baseDir + QLS("/latest_report.html"); - QString reportPath = path.mid(baseDir.size()+1); - QFile::remove(linkPath); // possible race with another thread, yada yada yada - QFile::link(reportPath, linkPath); - -#if 0 - QByteArray fwd = "<!DOCTYPE html><html><head><meta HTTP-EQUIV=\"refresh\" CONTENT=\"0;URL=%1\"></meta></head><body></body></html>\n"; - fwd.replace("%1", filePath().prepend(QLC('/')).toLatin1()); - - QFile file(rootDir + baseDir + "/latest_report.html"); - if (file.open(QIODevice::WriteOnly | QIODevice::Truncate)) - file.write(fwd); -#endif -} - - -void Report::handleCGIQuery(const QString &query) -{ - QUrl cgiUrl(QLS("http://dummy/cgi-bin/dummy.cgi?") + query); - QTextStream s(stdout); - s << "Content-Type: text/html\r\n\r\n" - << "<!DOCTYPE html>\n<HTML>\n<body bgcolor=""#ddeeff"">\n"; // Lancelot blue - - QString command(cgiUrl.queryItemValue("cmd")); - - if (command == QLS("view")) { - s << BaselineHandler::view(cgiUrl.queryItemValue(QLS("baseline")), - cgiUrl.queryItemValue(QLS("rendered")), - cgiUrl.queryItemValue(QLS("compared"))); - } -#if 0 - else if (command == QLS("diffstats")) { - s << BaselineHandler::diffstats(cgiUrl.queryItemValue(QLS("baseline")), - cgiUrl.queryItemValue(QLS("rendered"))); - } -#endif - else if (command == QLS("updateSingleBaseline")) { - s << BaselineHandler::updateBaselines(cgiUrl.queryItemValue(QLS("context")), - cgiUrl.queryItemValue(QLS("mismatchContext")), - cgiUrl.queryItemValue(QLS("itemFile"))); - } else if (command == QLS("updateAllBaselines")) { - s << BaselineHandler::updateBaselines(cgiUrl.queryItemValue(QLS("context")), - cgiUrl.queryItemValue(QLS("mismatchContext")), - QString()); - } else if (command == QLS("clearAllBaselines")) { - s << BaselineHandler::clearAllBaselines(cgiUrl.queryItemValue(QLS("context"))); - } else if (command == QLS("blacklist")) { - // blacklist a test - s << BaselineHandler::blacklistTest(cgiUrl.queryItemValue(QLS("context")), - cgiUrl.queryItemValue(QLS("itemId"))); - } else if (command == QLS("whitelist")) { - // whitelist a test - s << BaselineHandler::blacklistTest(cgiUrl.queryItemValue(QLS("context")), - cgiUrl.queryItemValue(QLS("itemId")), true); - } else { - s << "Unknown query:<br>" << query << "<br>"; - } - s << "<p><a href=\"" << cgiUrl.queryItemValue(QLS("url")) << "\">Back to report</a>\n"; - s << "</body>\n</HTML>"; -} diff --git a/tests/baselineserver/src/report.h b/tests/baselineserver/src/report.h deleted file mode 100644 index c568e7ab8d..0000000000 --- a/tests/baselineserver/src/report.h +++ /dev/null @@ -1,98 +0,0 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the test suite of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:GPL-EXCEPT$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 3 as published by the Free Software -** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ -#ifndef REPORT_H -#define REPORT_H - -#include "baselineprotocol.h" -#include <QFile> -#include <QTextStream> -#include <QMap> -#include <QStringList> -#include <QSettings> - -class BaselineHandler; - -class Report -{ -public: - Report(); - ~Report(); - - void init(const BaselineHandler *h, const QString &r, const PlatformInfo &p, const QSettings *s); - void addItems(const ImageItemList& items); - void addResult(const ImageItem& item); - void end(); - - bool reportProduced(); - - int numberOfMismatches(); - QString summary(); - - QString filePath(); - - QString writeResultsXmlFiles(); - - static void handleCGIQuery(const QString &query); - - static QString generateThumbnail(const QString &image, const QString &rootDir = QString()); - -private: - void write(); - void writeFunctionResults(const ImageItemList &list); - void writeItem(const QString &baseline, const QString &rendered, const ImageItem &item, - const QString &itemFile, const QString &ctx, const QString &misCtx, const QString &metadata); - void writeHeader(); - void writeFooter(); - QString generateCompared(const QString &baseline, const QString &rendered, bool fuzzy = false); - - void updateLatestPointer(); - - void computeStats(); - - bool initialized; - const BaselineHandler *handler; - QString runId; - PlatformInfo plat; - QString rootDir; - QString baseDir; - QString path; - QStringList testFunctions; - QMap<QString, ImageItemList> itemLists; - bool written; - int numItems; - int numMismatches; - QTextStream out; - bool hasOverride; - const QSettings *settings; - - typedef QMap<ImageItem::ItemStatus, int> FuncStats; - QMap<QString, FuncStats> stats; - bool hasStats; -}; - -#endif // REPORT_H diff --git a/tests/baselineserver/src/templates/view.html b/tests/baselineserver/src/templates/view.html deleted file mode 100644 index f0971010f2..0000000000 --- a/tests/baselineserver/src/templates/view.html +++ /dev/null @@ -1,84 +0,0 @@ -<h3>Lancelot Viewer</h3> - -<p> -Zoom: -<input name="zoom" id="z1" type="radio" checked="Checked">1x</input> -<input name="zoom" id="z2" type="radio">2x</input> -<input name="zoom" id="z4" type="radio">4x</input> -</p> - -<p><table> -<tr> -<td><input name="imgselect" id="baseline" type="radio" checked="Checked">Baseline</input></td> -<td>%1</td> -</tr> -<tr> -<td><input name="imgselect" id="rendered" type="radio">Rendered</input></td> -<td>%2</td> -</tr> -<tr> -<td><input name="imgselect" id="compared" type="radio">Differences</input></td> -<td></td> -</tr> -</table></p> - - -<p><table cellspacing="25"><tr> -<td valign="top"> -<canvas id="c" width="800" height="800"></canvas> -</td> -<td valign="top"> -%4 -</td> -</tr></table></p> - -<script> - var canvas = document.getElementById("c"); - var context = canvas.getContext("2d"); - var cat = new Image(); - cat.src = "%1"; - var z = 1; - cat.onload = function() { - context.mozImageSmoothingEnabled = false; - context.drawImage(cat, 0, 0, z*cat.width, z*cat.height); - }; - - var bbut = document.getElementById("baseline"); - bbut.onclick = function() { - cat.src = "%1"; - }; - - var rbut = document.getElementById("rendered"); - rbut.onclick = function() { - cat.src = "%2"; - }; - - var cbut = document.getElementById("compared"); - cbut.onclick = function() { - cat.src = "%3"; - }; - - function setZoom(zoom) - { - z = zoom; - canvas.width = z*800; - canvas.height = z*800; - context.mozImageSmoothingEnabled = false; - context.drawImage(cat, 0, 0, z*cat.width, z*cat.height); - } - - var z1but = document.getElementById("z1"); - z1but.onclick = function() { - setZoom(1); - }; - - var z2but = document.getElementById("z2"); - z2but.onclick = function() { - setZoom(2); - }; - - var z4but = document.getElementById("z4"); - z4but.onclick = function() { - setZoom(4); - }; -</script> |