summaryrefslogtreecommitdiffstats
path: root/tests/baselineserver/src
diff options
context:
space:
mode:
Diffstat (limited to 'tests/baselineserver/src')
-rw-r--r--tests/baselineserver/src/baselineserver.cpp853
-rw-r--r--tests/baselineserver/src/baselineserver.h151
-rw-r--r--tests/baselineserver/src/baselineserver.pro24
-rw-r--r--tests/baselineserver/src/baselineserver.qrc5
-rw-r--r--tests/baselineserver/src/main.cpp57
-rw-r--r--tests/baselineserver/src/report.cpp503
-rw-r--r--tests/baselineserver/src/report.h98
-rw-r--r--tests/baselineserver/src/templates/view.html84
8 files changed, 0 insertions, 1775 deletions
diff --git a/tests/baselineserver/src/baselineserver.cpp b/tests/baselineserver/src/baselineserver.cpp
deleted file mode 100644
index c6e12dc09c..0000000000
--- a/tests/baselineserver/src/baselineserver.cpp
+++ /dev/null
@@ -1,853 +0,0 @@
-/****************************************************************************
-**
-** Copyright (C) 2016 The Qt Company Ltd.
-** Contact: https://www.qt.io/licensing/
-**
-** This file is part of the test suite of the Qt Toolkit.
-**
-** $QT_BEGIN_LICENSE:GPL-EXCEPT$
-** Commercial License Usage
-** Licensees holding valid commercial Qt licenses may use this file in
-** accordance with the commercial license agreement provided with the
-** Software or, alternatively, in accordance with the terms contained in
-** a written agreement between you and The Qt Company. For licensing terms
-** and conditions see https://www.qt.io/terms-conditions. For further
-** information use the contact form at https://www.qt.io/contact-us.
-**
-** GNU General Public License Usage
-** Alternatively, this file may be used under the terms of the GNU
-** General Public License version 3 as published by the Free Software
-** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
-** included in the packaging of this file. Please review the following
-** information to ensure the GNU General Public License requirements will
-** be met: https://www.gnu.org/licenses/gpl-3.0.html.
-**
-** $QT_END_LICENSE$
-**
-****************************************************************************/
-
-#define QT_USE_FAST_CONCATENATION
-#define QT_USE_FAST_OPERATOR_PLUS
-
-#include "baselineserver.h"
-#include <QBuffer>
-#include <QFile>
-#include <QDir>
-#include <QCoreApplication>
-#include <QFileInfo>
-#include <QHostInfo>
-#include <QTextStream>
-#include <QProcess>
-#include <QDirIterator>
-#include <QUrl>
-#include <QRegularExpression>
-
-// extra fields, for use in image metadata storage
-const QString PI_ImageChecksum(QLS("ImageChecksum"));
-const QString PI_RunId(QLS("RunId"));
-const QString PI_CreationDate(QLS("CreationDate"));
-
-QString BaselineServer::storage;
-QString BaselineServer::url;
-QStringList BaselineServer::pathKeys;
-
-BaselineServer::BaselineServer(QObject *parent)
- : QTcpServer(parent), lastRunIdIdx(0)
-{
- QFileInfo me(QCoreApplication::applicationFilePath());
- meLastMod = me.lastModified();
- heartbeatTimer = new QTimer(this);
- connect(heartbeatTimer, SIGNAL(timeout()), this, SLOT(heartbeat()));
- heartbeatTimer->start(HEARTBEAT*1000);
-}
-
-QString BaselineServer::storagePath()
-{
- if (storage.isEmpty()) {
- storage = QLS(qgetenv("QT_LANCELOT_DIR"));
- if (storage.isEmpty())
- storage = QLS("/var/www");
- }
- return storage;
-}
-
-QString BaselineServer::baseUrl()
-{
- if (url.isEmpty()) {
- url = QLS("http://")
- + QHostInfo::localHostName().toLatin1() + '.'
- + QHostInfo::localDomainName().toLatin1() + '/';
- }
- return url;
-}
-
-QStringList BaselineServer::defaultPathKeys()
-{
- if (pathKeys.isEmpty())
- pathKeys << PI_QtVersion << PI_QMakeSpec << PI_HostName;
- return pathKeys;
-}
-
-void BaselineServer::incomingConnection(qintptr socketDescriptor)
-{
- QString runId = QDateTime::currentDateTime().toString(QLS("MMMdd-hhmmss"));
- if (runId == lastRunId) {
- runId += QLC('-') + QString::number(++lastRunIdIdx);
- } else {
- lastRunId = runId;
- lastRunIdIdx = 0;
- }
- qDebug() << "Server: New connection! RunId:" << runId;
- BaselineThread *thread = new BaselineThread(runId, socketDescriptor, this);
- connect(thread, SIGNAL(finished()), thread, SLOT(deleteLater()));
- thread->start();
-}
-
-void BaselineServer::heartbeat()
-{
- // The idea is to exit to be restarted when modified, as soon as not actually serving
- QFileInfo me(QCoreApplication::applicationFilePath());
- if (me.lastModified() == meLastMod)
- return;
- if (!me.exists() || !me.isExecutable())
- return;
-
- //# (could close() here to avoid accepting new connections, to avoid livelock)
- //# also, could check for a timeout to force exit, to avoid hung threads blocking
- bool isServing = false;
- foreach(BaselineThread *thread, findChildren<BaselineThread *>()) {
- if (thread->isRunning()) {
- isServing = true;
- break;
- }
- }
-
- if (!isServing)
- QCoreApplication::exit();
-}
-
-BaselineThread::BaselineThread(const QString &runId, int socketDescriptor, QObject *parent)
- : QThread(parent), runId(runId), socketDescriptor(socketDescriptor)
-{
-}
-
-void BaselineThread::run()
-{
- BaselineHandler handler(runId, socketDescriptor);
- exec();
-}
-
-
-BaselineHandler::BaselineHandler(const QString &runId, int socketDescriptor)
- : QObject(), runId(runId), connectionEstablished(false), settings(0), fuzzLevel(0)
-{
- idleTimer = new QTimer(this);
- idleTimer->setSingleShot(true);
- idleTimer->setInterval(IDLE_CLIENT_TIMEOUT * 1000);
- connect(idleTimer, SIGNAL(timeout()), this, SLOT(idleClientTimeout()));
- idleTimer->start();
-
- if (socketDescriptor == -1)
- return;
-
- connect(&proto.socket, SIGNAL(readyRead()), this, SLOT(receiveRequest()));
- connect(&proto.socket, SIGNAL(disconnected()), this, SLOT(receiveDisconnect()));
- proto.socket.setSocketDescriptor(socketDescriptor);
- proto.socket.setSocketOption(QAbstractSocket::KeepAliveOption, 1);
-}
-
-const char *BaselineHandler::logtime()
-{
- return 0;
- //return QTime::currentTime().toString(QLS("mm:ss.zzz"));
-}
-
-QString BaselineHandler::projectPath(bool absolute) const
-{
- QString p = clientInfo.value(PI_Project);
- return absolute ? BaselineServer::storagePath() + QLC('/') + p : p;
-}
-
-bool BaselineHandler::checkClient(QByteArray *errMsg, bool *dryRunMode)
-{
- if (!errMsg)
- return false;
- if (clientInfo.value(PI_Project).isEmpty() || clientInfo.value(PI_TestCase).isEmpty()) {
- *errMsg = "No Project and/or TestCase specified in client info.";
- return false;
- }
-
- // Determine ad-hoc state ### hardcoded for now
- if (clientInfo.value(PI_TestCase) == QLS("tst_Lancelot")) {
- //### Todo: push this stuff out in a script
- if (!clientInfo.isAdHocRun()) {
- // ### comp. with earlier versions still running (4.8) (?)
- clientInfo.setAdHocRun(clientInfo.value(PI_PulseGitBranch).isEmpty() && clientInfo.value(PI_PulseTestrBranch).isEmpty());
- }
- }
- else {
- // TBD
- }
-
- if (clientInfo.isAdHocRun()) {
- if (dryRunMode)
- *dryRunMode = false;
- return true;
- }
-
- // Not ad hoc: filter the client
- settings->beginGroup("ClientFilters");
- bool matched = false;
- bool dryRunReq = false;
- foreach (const QString &rule, settings->childKeys()) {
- //qDebug() << " > RULE" << rule;
- dryRunReq = false;
- QString ruleMode = settings->value(rule).toString().toLower();
- if (ruleMode == QLS("dryrun"))
- dryRunReq = true;
- else if (ruleMode != QLS("enabled"))
- continue;
- settings->beginGroup(rule);
- bool ruleMatched = true;
- foreach (const QString &filterKey, settings->childKeys()) {
- //qDebug() << " > FILTER" << filterKey;
- QString filter = settings->value(filterKey).toString();
- if (filter.isEmpty())
- continue;
- QString platVal = clientInfo.value(filterKey);
- if (!platVal.contains(filter)) {
- ruleMatched = false;
- break;
- }
- }
- if (ruleMatched) {
- ruleName = rule;
- matched = true;
- break;
- }
- settings->endGroup();
- }
-
- if (!matched && errMsg)
- *errMsg = "Non-adhoc client did not match any filter rule in " + settings->fileName().toLatin1();
-
- if (matched && dryRunMode)
- *dryRunMode = dryRunReq;
-
- // NB! Must reset the settings object before returning
- while (!settings->group().isEmpty())
- settings->endGroup();
-
- return matched;
-}
-
-bool BaselineHandler::establishConnection()
-{
- if (!proto.acceptConnection(&clientInfo)) {
- qWarning() << runId << logtime() << "Accepting new connection from" << proto.socket.peerAddress().toString() << "failed." << proto.errorMessage();
- proto.sendBlock(BaselineProtocol::Abort, proto.errorMessage().toLatin1()); // In case the client can hear us, tell it what's wrong.
- proto.socket.disconnectFromHost();
- return false;
- }
- QString logMsg;
- foreach (QString key, clientInfo.keys()) {
- if (key != PI_HostName && key != PI_HostAddress)
- logMsg += key + QLS(": '") + clientInfo.value(key) + QLS("', ");
- }
- qDebug() << runId << logtime() << "Connection established with" << clientInfo.value(PI_HostName)
- << '[' << qPrintable(clientInfo.value(PI_HostAddress)) << ']' << logMsg
- << "Overrides:" << clientInfo.overrides() << "AdHoc-Run:" << clientInfo.isAdHocRun();
-
- // ### Hardcoded backwards compatibility: add project field for certain existing clients that lack it
- if (clientInfo.value(PI_Project).isEmpty()) {
- QString tc = clientInfo.value(PI_TestCase);
- if (tc == QLS("tst_Lancelot"))
- clientInfo.insert(PI_Project, QLS("Raster"));
- else if (tc == QLS("tst_Scenegraph"))
- clientInfo.insert(PI_Project, QLS("SceneGraph"));
- else
- clientInfo.insert(PI_Project, QLS("Other"));
- }
-
- QString settingsFile = projectPath() + QLS("/config.ini");
- settings = new QSettings(settingsFile, QSettings::IniFormat, this);
-
- QByteArray errMsg;
- bool dryRunMode = false;
- if (!checkClient(&errMsg, &dryRunMode)) {
- qDebug() << runId << logtime() << "Rejecting connection:" << errMsg;
- proto.sendBlock(BaselineProtocol::Abort, errMsg);
- proto.socket.disconnectFromHost();
- return false;
- }
-
- fuzzLevel = qBound(0, settings->value("FuzzLevel").toInt(), 100);
- if (!clientInfo.isAdHocRun()) {
- qDebug() << runId << logtime() << "Client matches filter rule" << ruleName
- << "Dryrun:" << dryRunMode
- << "FuzzLevel:" << fuzzLevel
- << "ReportMissingResults:" << settings->value("ReportMissingResults").toBool();
- }
-
- proto.sendBlock(dryRunMode ? BaselineProtocol::DoDryRun : BaselineProtocol::Ack, QByteArray());
- report.init(this, runId, clientInfo, settings);
- return true;
-}
-
-void BaselineHandler::receiveRequest()
-{
- idleTimer->start(); // Restart idle client timeout
-
- if (!connectionEstablished) {
- connectionEstablished = establishConnection();
- return;
- }
-
- QByteArray block;
- BaselineProtocol::Command cmd;
- if (!proto.receiveBlock(&cmd, &block)) {
- qWarning() << runId << logtime() << "Command reception failed. "<< proto.errorMessage();
- QThread::currentThread()->exit(1);
- return;
- }
-
- switch(cmd) {
- case BaselineProtocol::RequestBaselineChecksums:
- provideBaselineChecksums(block);
- break;
- case BaselineProtocol::AcceptMatch:
- recordMatch(block);
- break;
- case BaselineProtocol::AcceptNewBaseline:
- storeImage(block, true);
- break;
- case BaselineProtocol::AcceptMismatch:
- storeImage(block, false);
- break;
- default:
- qWarning() << runId << logtime() << "Unknown command received. " << proto.errorMessage();
- proto.sendBlock(BaselineProtocol::UnknownError, QByteArray());
- }
-}
-
-
-void BaselineHandler::provideBaselineChecksums(const QByteArray &itemListBlock)
-{
- ImageItemList itemList;
- QDataStream ds(itemListBlock);
- ds >> itemList;
- qDebug() << runId << logtime() << "Received request for checksums for" << itemList.count()
- << "items in test function" << itemList.at(0).testFunction;
-
- for (ImageItemList::iterator i = itemList.begin(); i != itemList.end(); ++i) {
- i->imageChecksums.clear();
- i->status = ImageItem::BaselineNotFound;
- QString prefix = pathForItem(*i, true);
- PlatformInfo itemData = fetchItemMetadata(prefix);
- if (itemData.contains(PI_ImageChecksum)) {
- bool ok = false;
- quint64 checksum = itemData.value(PI_ImageChecksum).toULongLong(&ok, 16);
- if (ok) {
- i->imageChecksums.prepend(checksum);
- i->status = ImageItem::Ok;
- }
- }
- }
-
- // Find and mark blacklisted items
- QString context = pathForItem(itemList.at(0), true, false).section(QLC('/'), 0, -2);
- if (itemList.count() > 0) {
- QFile file(BaselineServer::storagePath() + QLC('/') + context + QLS("/BLACKLIST"));
- if (file.open(QIODevice::ReadOnly)) {
- QTextStream in(&file);
- do {
- QString itemName = in.readLine();
- if (!itemName.isNull()) {
- for (ImageItemList::iterator i = itemList.begin(); i != itemList.end(); ++i) {
- if (i->itemName == itemName)
- i->status = ImageItem::IgnoreItem;
- }
- }
- } while (!in.atEnd());
- }
- }
-
- QByteArray block;
- QDataStream ods(&block, QIODevice::WriteOnly);
- ods << itemList;
- proto.sendBlock(BaselineProtocol::Ack, block);
- report.addItems(itemList);
-}
-
-
-void BaselineHandler::recordMatch(const QByteArray &itemBlock)
-{
- QDataStream ds(itemBlock);
- ImageItem item;
- ds >> item;
- report.addResult(item);
- proto.sendBlock(BaselineProtocol::Ack, QByteArray());
-}
-
-
-void BaselineHandler::storeImage(const QByteArray &itemBlock, bool isBaseline)
-{
- QDataStream ds(itemBlock);
- ImageItem item;
- ds >> item;
-
- if (isBaseline && !clientInfo.overrides().isEmpty()) {
- qDebug() << runId << logtime() << "Received baseline from client with override info, ignoring. Item:" << item.itemName;
- proto.sendBlock(BaselineProtocol::UnknownError, "New baselines not accepted from client with override info.");
- return;
- }
-
- QString blPrefix = pathForItem(item, true);
- QString mmPrefix = pathForItem(item, false);
- QString prefix = isBaseline ? blPrefix : mmPrefix;
-
- qDebug() << runId << logtime() << "Received" << (isBaseline ? "baseline" : "mismatched") << "image for:" << item.itemName << "Storing in" << prefix;
-
- // Reply to the client
- QString msg;
- if (isBaseline)
- msg = QLS("New baseline image stored: ") + blPrefix + QLS(FileFormat);
- else
- msg = BaselineServer::baseUrl() + report.filePath();
-
- if (isBaseline || !fuzzLevel)
- proto.sendBlock(BaselineProtocol::Ack, msg.toLatin1()); // Do early reply if possible: don't make the client wait longer than necessary
-
- // Store the image
- QString dir = prefix.section(QLC('/'), 0, -2);
- QDir cwd;
- if (!cwd.exists(dir))
- cwd.mkpath(dir);
- item.image.save(prefix + QLS(FileFormat), FileFormat);
-
- PlatformInfo itemData = clientInfo;
- itemData.insert(PI_ImageChecksum, QString::number(item.imageChecksums.at(0), 16)); //# Only the first is stored. TBD: get rid of list
- itemData.insert(PI_RunId, runId);
- itemData.insert(PI_CreationDate, QDateTime::currentDateTime().toString());
- storeItemMetadata(itemData, prefix);
-
- if (!isBaseline) {
- // Do fuzzy matching
- bool fuzzyMatch = false;
- if (fuzzLevel) {
- BaselineProtocol::Command cmd = BaselineProtocol::Ack;
- fuzzyMatch = fuzzyCompare(blPrefix, mmPrefix);
- if (fuzzyMatch) {
- msg.prepend(QString("Fuzzy match at fuzzlevel %1%. Report: ").arg(fuzzLevel));
- cmd = BaselineProtocol::FuzzyMatch;
- }
- proto.sendBlock(cmd, msg.toLatin1()); // We didn't reply earlier
- }
-
- // Add to report
- item.status = fuzzyMatch ? ImageItem::FuzzyMatch : ImageItem::Mismatch;
- report.addResult(item);
- }
-}
-
-
-void BaselineHandler::storeItemMetadata(const PlatformInfo &metadata, const QString &path)
-{
- QFile file(path + QLS(MetadataFileExt));
- if (!file.open(QIODevice::WriteOnly | QIODevice::Truncate)) {
- qWarning() << runId << logtime() << "ERROR: could not write to file" << file.fileName();
- return;
- }
- QTextStream out(&file);
- PlatformInfo::const_iterator it = metadata.constBegin();
- while (it != metadata.constEnd()) {
- out << it.key() << ": " << it.value() << endl;
- ++it;
- }
- file.close();
-}
-
-
-PlatformInfo BaselineHandler::fetchItemMetadata(const QString &path)
-{
- PlatformInfo res;
- QFile file(path + QLS(MetadataFileExt));
- if (!file.open(QIODevice::ReadOnly) || !QFile::exists(path + QLS(FileFormat)))
- return res;
- QTextStream in(&file);
- do {
- QString line = in.readLine();
- int idx = line.indexOf(QLS(": "));
- if (idx > 0)
- res.insert(line.left(idx), line.mid(idx+2));
- } while (!in.atEnd());
- return res;
-}
-
-
-void BaselineHandler::idleClientTimeout()
-{
- qWarning() << runId << logtime() << "Idle client timeout: no request received for" << IDLE_CLIENT_TIMEOUT << "seconds, terminating connection.";
- proto.socket.disconnectFromHost();
-}
-
-
-void BaselineHandler::receiveDisconnect()
-{
- qDebug() << runId << logtime() << "Client disconnected.";
- report.end();
- if (report.reportProduced() && !clientInfo.isAdHocRun())
- issueMismatchNotification();
- if (settings && settings->value("ProcessXmlResults").toBool() && !clientInfo.isAdHocRun()) {
- // ### TBD: actually execute the processing command. For now, just generate the xml files.
- QString xmlDir = report.writeResultsXmlFiles();
- }
- QThread::currentThread()->exit(0);
-}
-
-
-PlatformInfo BaselineHandler::mapPlatformInfo(const PlatformInfo& orig) const
-{
- PlatformInfo mapped;
- foreach (const QString &key, orig.uniqueKeys()) {
- QString val = orig.value(key).simplified();
- val.replace(QLC('/'), QLC('_'));
- val.replace(QLC(' '), QLC('_'));
- mapped.insert(key, QUrl::toPercentEncoding(val, "+"));
- //qDebug() << "MAPPED" << key << "FROM" << orig.value(key) << "TO" << mapped.value(key);
- }
-
- // Special fixup for OS version
- if (mapped.value(PI_OSName) == QLS("MacOS")) {
- int ver = mapped.value(PI_OSVersion).toInt();
- if (ver > 1)
- mapped.insert(PI_OSVersion, QString("MV_10_%1").arg(ver-2));
- }
- else if (mapped.value(PI_OSName) == QLS("Windows")) {
- // TBD: map windows version numbers to names
- }
-
- // Special fixup for hostname
- QString host = mapped.value(PI_HostName).section(QLC('.'), 0, 0); // Filter away domain, if any
- if (host.isEmpty() || host == QLS("localhost")) {
- host = orig.value(PI_HostAddress);
- } else {
- if (!orig.isAdHocRun()) { // i.e. CI system run, so remove index postfix typical of vm hostnames
- host.remove(QRegularExpression(QLS("\\d+$")));
- if (host.endsWith(QLC('-')))
- host.chop(1);
- }
- }
- if (host.isEmpty())
- host = QLS("UNKNOWN-HOST");
- if (mapped.value(PI_OSName) == QLS("MacOS")) // handle multiple os versions on same host
- host += QLC('-') + mapped.value(PI_OSVersion);
- mapped.insert(PI_HostName, host);
-
- // Special fixup for Qt version
- QString ver = mapped.value(PI_QtVersion);
- if (!ver.isEmpty())
- mapped.insert(PI_QtVersion, ver.prepend(QLS("Qt-")));
-
- return mapped;
-}
-
-
-QString BaselineHandler::pathForItem(const ImageItem &item, bool isBaseline, bool absolute) const
-{
- if (mappedClientInfo.isEmpty()) {
- mappedClientInfo = mapPlatformInfo(clientInfo);
- PlatformInfo oraw = clientInfo;
- // ### simplify: don't map if no overrides!
- for (int i = 0; i < clientInfo.overrides().size()-1; i+=2)
- oraw.insert(clientInfo.overrides().at(i), clientInfo.overrides().at(i+1));
- overriddenMappedClientInfo = mapPlatformInfo(oraw);
- }
-
- const PlatformInfo& mapped = isBaseline ? overriddenMappedClientInfo : mappedClientInfo;
-
- QString itemName = safeName(item.itemName);
- itemName.append(QLC('_') + QString::number(item.itemChecksum, 16).rightJustified(4, QLC('0')));
-
- QStringList path;
- path += projectPath(absolute);
- path += mapped.value(PI_TestCase);
- path += QLS(isBaseline ? "baselines" : "mismatches");
- path += item.testFunction;
- QStringList itemPathKeys;
- if (settings)
- itemPathKeys = settings->value("ItemPathKeys").toStringList();
- if (itemPathKeys.isEmpty())
- itemPathKeys = BaselineServer::defaultPathKeys();
- foreach (const QString &key, itemPathKeys)
- path += mapped.value(key, QLS("UNSET-")+key);
- if (!isBaseline)
- path += runId;
- path += itemName + QLC('.');
-
- return path.join(QLS("/"));
-}
-
-
-QString BaselineHandler::view(const QString &baseline, const QString &rendered, const QString &compared)
-{
- QFile f(":/templates/view.html");
- f.open(QIODevice::ReadOnly);
- return QString::fromLatin1(f.readAll()).arg('/'+baseline, '/'+rendered, '/'+compared, diffstats(baseline, rendered));
-}
-
-QString BaselineHandler::diffstats(const QString &baseline, const QString &rendered)
-{
- QImage blImg(BaselineServer::storagePath() + QLC('/') + baseline);
- QImage mmImg(BaselineServer::storagePath() + QLC('/') + rendered);
- if (blImg.isNull() || mmImg.isNull())
- return QLS("Could not compute diffstats: image loading failed.");
-
- // ### TBD: cache the results
- return computeMismatchScore(blImg, mmImg);
-}
-
-QString BaselineHandler::clearAllBaselines(const QString &context)
-{
- int tot = 0;
- int failed = 0;
- QDirIterator it(BaselineServer::storagePath() + QLC('/') + context,
- QStringList() << QLS("*.") + QLS(FileFormat)
- << QLS("*.") + QLS(MetadataFileExt)
- << QLS("*.") + QLS(ThumbnailExt));
- while (it.hasNext()) {
- bool counting = !it.next().endsWith(QLS(ThumbnailExt));
- if (counting)
- tot++;
- if (!QFile::remove(it.filePath()) && counting)
- failed++;
- }
- return QString(QLS("%1 of %2 baselines cleared from context ")).arg((tot-failed)/2).arg(tot/2) + context;
-}
-
-QString BaselineHandler::updateBaselines(const QString &context, const QString &mismatchContext, const QString &itemFile)
-{
- int tot = 0;
- int failed = 0;
- QString storagePrefix = BaselineServer::storagePath() + QLC('/');
- // If itemId is set, update just that one, otherwise, update all:
- QString filter = itemFile.isEmpty() ? QLS("*_????.") : itemFile;
- QDirIterator it(storagePrefix + mismatchContext,
- QStringList() << filter + QLS(FileFormat)
- << filter + QLS(MetadataFileExt)
- << filter + QLS(ThumbnailExt));
- while (it.hasNext()) {
- bool counting = !it.next().endsWith(QLS(ThumbnailExt));
- if (counting)
- tot++;
- QString oldFile = storagePrefix + context + QLC('/') + it.fileName();
- QFile::remove(oldFile); // Remove existing baseline file
- if (!QFile::copy(it.filePath(), oldFile) && counting) // and replace it with the mismatch
- failed++;
- }
- return QString(QLS("%1 of %2 baselines updated in context %3 from context %4")).arg((tot-failed)/2).arg(tot/2).arg(context, mismatchContext);
-}
-
-QString BaselineHandler::blacklistTest(const QString &context, const QString &itemId, bool removeFromBlacklist)
-{
- QFile file(BaselineServer::storagePath() + QLC('/') + context + QLS("/BLACKLIST"));
- QStringList blackList;
- if (file.open(QIODevice::ReadWrite)) {
- while (!file.atEnd())
- blackList.append(file.readLine().trimmed());
-
- if (removeFromBlacklist)
- blackList.removeAll(itemId);
- else if (!blackList.contains(itemId))
- blackList.append(itemId);
-
- file.resize(0);
- foreach (QString id, blackList)
- file.write(id.toLatin1() + '\n');
- file.close();
- return QLS(removeFromBlacklist ? "Whitelisted " : "Blacklisted ") + itemId + QLS(" in context ") + context;
- } else {
- return QLS("Unable to update blacklisted tests, failed to open ") + file.fileName();
- }
-}
-
-
-void BaselineHandler::testPathMapping()
-{
- qDebug() << "Storage prefix:" << BaselineServer::storagePath();
-
- QStringList hosts;
- hosts << QLS("bq-ubuntu910-x86-01")
- << QLS("bq-ubuntu910-x86-15")
- << QLS("osl-mac-master-5.test.qt-project.org")
- << QLS("osl-mac-master-6.test.qt-project.org")
- << QLS("sv-xp-vs-010")
- << QLS("sv-xp-vs-011")
- << QLS("sv-solaris-sparc-008")
- << QLS("macbuilder-02.test.troll.no")
- << QLS("bqvm1164")
- << QLS("chimera")
- << QLS("localhost")
- << QLS("");
-
- ImageItem item;
- item.testFunction = QLS("testPathMapping");
- item.itemName = QLS("arcs.qps");
- item.imageChecksums << 0x0123456789abcdefULL;
- item.itemChecksum = 0x0123;
-
- clientInfo.insert(PI_QtVersion, QLS("5.0.0"));
- clientInfo.insert(PI_QMakeSpec, QLS("linux-g++"));
- clientInfo.insert(PI_PulseGitBranch, QLS("somebranch"));
- clientInfo.setAdHocRun(false);
- foreach(const QString& host, hosts) {
- mappedClientInfo.clear();
- clientInfo.insert(PI_HostName, host);
- qDebug() << "Baseline from" << host << "->" << pathForItem(item, true);
- qDebug() << "Mismatch from" << host << "->" << pathForItem(item, false);
- }
-}
-
-
-QString BaselineHandler::computeMismatchScore(const QImage &baseline, const QImage &rendered)
-{
- if (baseline.size() != rendered.size() || baseline.format() != rendered.format())
- return QLS("[No diffstats, incomparable images.]");
- if (baseline.depth() != 32)
- return QLS("[Diffstats computation not implemented for format.]");
-
- int w = baseline.width();
- int h = baseline.height();
-
- uint ncd = 0; // number of differing color pixels
- uint nad = 0; // number of differing alpha pixels
- uint scd = 0; // sum of color pixel difference
- uint sad = 0; // sum of alpha pixel difference
- uint mind = 0; // minimum difference
- uint maxd = 0; // maximum difference
-
- for (int y=0; y<h; ++y) {
- const QRgb *bl = (const QRgb *) baseline.constScanLine(y);
- const QRgb *rl = (const QRgb *) rendered.constScanLine(y);
- for (int x=0; x<w; ++x) {
- QRgb b = bl[x];
- QRgb r = rl[x];
- if (r != b) {
- uint dr = qAbs(qRed(b) - qRed(r));
- uint dg = qAbs(qGreen(b) - qGreen(r));
- uint db = qAbs(qBlue(b) - qBlue(r));
- uint ds = (dr + dg + db) / 3;
- uint da = qAbs(qAlpha(b) - qAlpha(r));
- if (ds) {
- ncd++;
- scd += ds;
- if (!mind || ds < mind)
- mind = ds;
- if (ds > maxd)
- maxd = ds;
- }
- if (da) {
- nad++;
- sad += da;
- }
- }
- }
- }
-
-
- double pcd = 100.0 * ncd / (w*h); // percent of pixels that differ
- double acd = ncd ? double(scd) / (ncd) : 0; // avg. difference
-/*
- if (baseline.hasAlphaChannel()) {
- double pad = 100.0 * nad / (w*h); // percent of pixels that differ
- double aad = nad ? double(sad) / (3*nad) : 0; // avg. difference
- }
-*/
- QString res = "<table>\n";
- QString item = "<tr><td>%1</td><td align=right>%2</td></tr>\n";
- res += item.arg("Number of mismatching pixels").arg(ncd);
- res += item.arg("Percentage mismatching pixels").arg(pcd, 0, 'g', 2);
- res += item.arg("Minimum pixel distance").arg(mind);
- res += item.arg("Maximum pixel distance").arg(maxd);
- if (acd >= 10.0)
- res += item.arg("Average pixel distance").arg(qRound(acd));
- else
- res += item.arg("Average pixel distance").arg(acd, 0, 'g', 2);
-
- if (baseline.hasAlphaChannel())
- res += item.arg("Number of mismatching alpha values").arg(nad);
-
- res += "</table>\n";
- res += "<p>(Distances are normalized to the range 0-255)</p>\n";
- return res;
-}
-
-
-bool BaselineHandler::fuzzyCompare(const QString &baselinePath, const QString &mismatchPath)
-{
- QProcess compareProc;
- QStringList args;
- args << "-fuzz" << QString("%1%").arg(fuzzLevel) << "-metric" << "AE";
- args << baselinePath + QLS(FileFormat) << mismatchPath + QLS(FileFormat) << "/dev/null"; // TBD: Should save output image, so report won't have to regenerate it
-
- compareProc.setProcessChannelMode(QProcess::MergedChannels);
- compareProc.start("compare", args, QIODevice::ReadOnly);
- if (compareProc.waitForFinished(3000) && compareProc.error() == QProcess::UnknownError) {
- bool ok = false;
- int metric = compareProc.readAll().trimmed().toInt(&ok);
- if (ok && metric == 0)
- return true;
- }
- return false;
-}
-
-
-void BaselineHandler::issueMismatchNotification()
-{
- // KISS: hardcoded use of the "sendemail" utility. Make this configurable if and when demand arises.
- if (!settings)
- return;
-
- settings->beginGroup("Notification");
- QStringList receivers = settings->value("Receivers").toStringList();
- QString sender = settings->value("Sender").toString();
- QString server = settings->value("SMTPserver").toString();
- settings->endGroup();
- if (receivers.isEmpty() || sender.isEmpty() || server.isEmpty())
- return;
-
- QString msg = QString("\nResult summary for test run %1:\n").arg(runId);
- msg += report.summary();
- msg += "\nReport: " + BaselineServer::baseUrl() + report.filePath() + "\n";
-
- msg += "\nTest run platform properties:\n------------------\n";
- foreach (const QString &key, clientInfo.keys())
- msg += key + ": " + clientInfo.value(key) + '\n';
- msg += "\nCheers,\n- Your friendly Lancelot Baseline Server\n";
-
- QProcess proc;
- QString cmd = "sendemail";
- QStringList args;
- args << "-s" << server << "-f" << sender << "-t" << receivers;
- args << "-u" << "[Lancelot] Mismatch report for project " + clientInfo.value(PI_Project) + ", test case " + clientInfo.value(PI_TestCase);
- args << "-m" << msg;
-
- //proc.setProcessChannelMode(QProcess::MergedChannels);
- proc.start(cmd, args);
- if (!proc.waitForFinished(10 * 1000) || (proc.exitStatus() != QProcess::NormalExit) || proc.exitCode()) {
- qWarning() << "FAILED to issue notification. Command:" << cmd << args.mid(0, args.size()-2);
- qWarning() << " Command standard output:" << proc.readAllStandardOutput();
- qWarning() << " Command error output:" << proc.readAllStandardError();
- }
-}
-
-
-// Make an identifer safer for use as filename and URL
-QString safeName(const QString& name)
-{
- QString res = name.simplified();
- res.replace(QLC(' '), QLC('_'));
- res.replace(QLC('.'), QLC('_'));
- res.replace(QLC('/'), QLC('^'));
- return res;
-}
diff --git a/tests/baselineserver/src/baselineserver.h b/tests/baselineserver/src/baselineserver.h
deleted file mode 100644
index 25ef17f023..0000000000
--- a/tests/baselineserver/src/baselineserver.h
+++ /dev/null
@@ -1,151 +0,0 @@
-/****************************************************************************
-**
-** Copyright (C) 2016 The Qt Company Ltd.
-** Contact: https://www.qt.io/licensing/
-**
-** This file is part of the test suite of the Qt Toolkit.
-**
-** $QT_BEGIN_LICENSE:GPL-EXCEPT$
-** Commercial License Usage
-** Licensees holding valid commercial Qt licenses may use this file in
-** accordance with the commercial license agreement provided with the
-** Software or, alternatively, in accordance with the terms contained in
-** a written agreement between you and The Qt Company. For licensing terms
-** and conditions see https://www.qt.io/terms-conditions. For further
-** information use the contact form at https://www.qt.io/contact-us.
-**
-** GNU General Public License Usage
-** Alternatively, this file may be used under the terms of the GNU
-** General Public License version 3 as published by the Free Software
-** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
-** included in the packaging of this file. Please review the following
-** information to ensure the GNU General Public License requirements will
-** be met: https://www.gnu.org/licenses/gpl-3.0.html.
-**
-** $QT_END_LICENSE$
-**
-****************************************************************************/
-#ifndef BASELINESERVER_H
-#define BASELINESERVER_H
-
-#include <QStringList>
-#include <QTcpServer>
-#include <QThread>
-#include <QTcpSocket>
-#include <QScopedPointer>
-#include <QTimer>
-#include <QDateTime>
-#include <QSettings>
-
-#include "baselineprotocol.h"
-#include "report.h"
-
-// #seconds between checks for update of the executable
-#define HEARTBEAT 10
-// Timeout if no activity received from client, #seconds
-#define IDLE_CLIENT_TIMEOUT 3*60
-
-#define MetadataFileExt "metadata"
-#define ThumbnailExt "thumbnail.jpg"
-
-
-class BaselineServer : public QTcpServer
-{
- Q_OBJECT
-
-public:
- BaselineServer(QObject *parent = nullptr);
-
- static QString storagePath();
- static QString baseUrl();
- static QStringList defaultPathKeys();
-
-protected:
- void incomingConnection(qintptr socketDescriptor);
-
-private slots:
- void heartbeat();
-
-private:
- QTimer *heartbeatTimer;
- QDateTime meLastMod;
- QString lastRunId;
- int lastRunIdIdx;
- static QString storage;
- static QString url;
- static QStringList pathKeys;
-};
-
-
-
-class BaselineThread : public QThread
-{
- Q_OBJECT
-
-public:
- BaselineThread(const QString &runId, int socketDescriptor, QObject *parent);
- void run();
-
-private:
- QString runId;
- int socketDescriptor;
-};
-
-
-class BaselineHandler : public QObject
-{
- Q_OBJECT
-
-public:
- BaselineHandler(const QString &runId, int socketDescriptor = -1);
- QString projectPath(bool absolute = true) const;
- QString pathForItem(const ImageItem &item, bool isBaseline = true, bool absolute = true) const;
-
- // CGI callbacks:
- static QString view(const QString &baseline, const QString &rendered, const QString &compared);
- static QString diffstats(const QString &baseline, const QString &rendered);
- static QString clearAllBaselines(const QString &context);
- static QString updateBaselines(const QString &context, const QString &mismatchContext, const QString &itemFile);
- static QString blacklistTest(const QString &context, const QString &itemId, bool removeFromBlacklist = false);
-
- // for debugging
- void testPathMapping();
-
-private slots:
- void receiveRequest();
- void receiveDisconnect();
- void idleClientTimeout();
-
-private:
- bool checkClient(QByteArray *errMsg, bool *dryRunMode = nullptr);
- bool establishConnection();
- void provideBaselineChecksums(const QByteArray &itemListBlock);
- void recordMatch(const QByteArray &itemBlock);
- void storeImage(const QByteArray &itemBlock, bool isBaseline);
- void storeItemMetadata(const PlatformInfo &metadata, const QString &path);
- PlatformInfo fetchItemMetadata(const QString &path);
- PlatformInfo mapPlatformInfo(const PlatformInfo& orig) const;
- const char *logtime();
- void issueMismatchNotification();
- bool fuzzyCompare(const QString& baselinePath, const QString& mismatchPath);
-
- static QString computeMismatchScore(const QImage& baseline, const QImage& rendered);
-
- BaselineProtocol proto;
- PlatformInfo clientInfo;
- mutable PlatformInfo mappedClientInfo;
- mutable PlatformInfo overriddenMappedClientInfo;
- QString runId;
- bool connectionEstablished;
- Report report;
- QSettings *settings;
- QString ruleName;
- int fuzzLevel;
- QTimer *idleTimer;
-};
-
-
-// Make an identifer safer for use as filename and URL
-QString safeName(const QString& name);
-
-#endif // BASELINESERVER_H
diff --git a/tests/baselineserver/src/baselineserver.pro b/tests/baselineserver/src/baselineserver.pro
deleted file mode 100644
index 2d8438cb51..0000000000
--- a/tests/baselineserver/src/baselineserver.pro
+++ /dev/null
@@ -1,24 +0,0 @@
-QT += core network
-
-# gui needed for QImage
-# QT -= gui
-
-TARGET = baselineserver
-DESTDIR = ../bin
-CONFIG += cmdline
-
-TEMPLATE = app
-
-include(../shared/baselineprotocol.pri)
-
-SOURCES += main.cpp \
- baselineserver.cpp \
- report.cpp
-
-HEADERS += \
- baselineserver.h \
- report.h
-
-RESOURCES += \
- baselineserver.qrc
-DEFINES += QT_DISABLE_DEPRECATED_BEFORE=0
diff --git a/tests/baselineserver/src/baselineserver.qrc b/tests/baselineserver/src/baselineserver.qrc
deleted file mode 100644
index b5cd6afadb..0000000000
--- a/tests/baselineserver/src/baselineserver.qrc
+++ /dev/null
@@ -1,5 +0,0 @@
-<RCC>
- <qresource prefix="/">
- <file>templates/view.html</file>
- </qresource>
-</RCC>
diff --git a/tests/baselineserver/src/main.cpp b/tests/baselineserver/src/main.cpp
deleted file mode 100644
index dfc9b83da8..0000000000
--- a/tests/baselineserver/src/main.cpp
+++ /dev/null
@@ -1,57 +0,0 @@
-/****************************************************************************
-**
-** Copyright (C) 2016 The Qt Company Ltd.
-** Contact: https://www.qt.io/licensing/
-**
-** This file is part of the test suite of the Qt Toolkit.
-**
-** $QT_BEGIN_LICENSE:GPL-EXCEPT$
-** Commercial License Usage
-** Licensees holding valid commercial Qt licenses may use this file in
-** accordance with the commercial license agreement provided with the
-** Software or, alternatively, in accordance with the terms contained in
-** a written agreement between you and The Qt Company. For licensing terms
-** and conditions see https://www.qt.io/terms-conditions. For further
-** information use the contact form at https://www.qt.io/contact-us.
-**
-** GNU General Public License Usage
-** Alternatively, this file may be used under the terms of the GNU
-** General Public License version 3 as published by the Free Software
-** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
-** included in the packaging of this file. Please review the following
-** information to ensure the GNU General Public License requirements will
-** be met: https://www.gnu.org/licenses/gpl-3.0.html.
-**
-** $QT_END_LICENSE$
-**
-****************************************************************************/
-#include <QtCore/QCoreApplication>
-#include "baselineserver.h"
-
-int main(int argc, char *argv[])
-{
- QCoreApplication a(argc, argv);
-
- QString queryString(qgetenv("QUERY_STRING"));
- if (!queryString.isEmpty()) {
- // run as CGI script
- Report::handleCGIQuery(queryString);
- return 0;
- }
-
- if (a.arguments().contains(QLatin1String("-testmapping"))) {
- BaselineHandler h(QLS("SomeRunId"));
- h.testPathMapping();
- return 0;
- }
-
- BaselineServer server;
- if (!server.listen(QHostAddress::Any, BaselineProtocol::ServerPort)) {
- qWarning("Failed to listen!");
- return 1;
- }
-
- qDebug() << "\n*****" << argv[0] << "started, ready to serve on port" << BaselineProtocol::ServerPort
- << "with baseline protocol version" << BaselineProtocol::ProtocolVersion << "*****\n";
- return a.exec();
-}
diff --git a/tests/baselineserver/src/report.cpp b/tests/baselineserver/src/report.cpp
deleted file mode 100644
index 748d76ebfe..0000000000
--- a/tests/baselineserver/src/report.cpp
+++ /dev/null
@@ -1,503 +0,0 @@
-/****************************************************************************
-**
-** Copyright (C) 2016 The Qt Company Ltd.
-** Contact: https://www.qt.io/licensing/
-**
-** This file is part of the test suite of the Qt Toolkit.
-**
-** $QT_BEGIN_LICENSE:GPL-EXCEPT$
-** Commercial License Usage
-** Licensees holding valid commercial Qt licenses may use this file in
-** accordance with the commercial license agreement provided with the
-** Software or, alternatively, in accordance with the terms contained in
-** a written agreement between you and The Qt Company. For licensing terms
-** and conditions see https://www.qt.io/terms-conditions. For further
-** information use the contact form at https://www.qt.io/contact-us.
-**
-** GNU General Public License Usage
-** Alternatively, this file may be used under the terms of the GNU
-** General Public License version 3 as published by the Free Software
-** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
-** included in the packaging of this file. Please review the following
-** information to ensure the GNU General Public License requirements will
-** be met: https://www.gnu.org/licenses/gpl-3.0.html.
-**
-** $QT_END_LICENSE$
-**
-****************************************************************************/
-#include "report.h"
-#include "baselineprotocol.h"
-#include "baselineserver.h"
-#include <QDir>
-#include <QProcess>
-#include <QUrl>
-#include <QXmlStreamWriter>
-#include <QRegularExpression>
-#include <unistd.h>
-
-Report::Report()
- : initialized(false), handler(0), written(false), numItems(0), numMismatches(0), settings(0),
- hasStats(false)
-{
-}
-
-Report::~Report()
-{
- end();
-}
-
-QString Report::filePath()
-{
- return path;
-}
-
-int Report::numberOfMismatches()
-{
- return numMismatches;
-}
-
-bool Report::reportProduced()
-{
- return written;
-}
-
-void Report::init(const BaselineHandler *h, const QString &r, const PlatformInfo &p, const QSettings *s)
-{
- handler = h;
- runId = r;
- plat = p;
- settings = s;
- rootDir = BaselineServer::storagePath() + QLC('/');
- baseDir = handler->pathForItem(ImageItem(), true, false).remove(QRegularExpression("/baselines/.*$"));
- QString dir = baseDir + (plat.isAdHocRun() ? QLS("/adhoc-reports") : QLS("/auto-reports"));
- QDir cwd;
- if (!cwd.exists(rootDir + dir))
- cwd.mkpath(rootDir + dir);
- path = dir + QLS("/Report_") + runId + QLS(".html");
- hasOverride = !plat.overrides().isEmpty();
- initialized = true;
-}
-
-void Report::addItems(const ImageItemList &items)
-{
- if (items.isEmpty())
- return;
- numItems += items.size();
- QString func = items.at(0).testFunction;
- if (!testFunctions.contains(func))
- testFunctions.append(func);
- ImageItemList list = items;
- if (settings->value("ReportMissingResults").toBool()) {
- for (ImageItemList::iterator it = list.begin(); it != list.end(); ++it) {
- if (it->status == ImageItem::Ok)
- it->status = ImageItem::Error; // Status should be set by report from client, else report as error
- }
- }
- itemLists[func] += list;
-}
-
-void Report::addResult(const ImageItem &item)
-{
- if (!testFunctions.contains(item.testFunction)) {
- qWarning() << "Report::addResult: unknown testfunction" << item.testFunction;
- return;
- }
- bool found = false;
- ImageItemList &list = itemLists[item.testFunction];
- for (ImageItemList::iterator it = list.begin(); it != list.end(); ++it) {
- if (it->itemName == item.itemName && it->itemChecksum == item.itemChecksum) {
- it->status = item.status;
- found = true;
- break;
- }
- }
- if (found) {
- if (item.status == ImageItem::Mismatch)
- numMismatches++;
- } else {
- qWarning() << "Report::addResult: unknown item" << item.itemName << "in testfunction" << item.testFunction;
- }
-}
-
-void Report::end()
-{
- if (!initialized || written)
- return;
- // Make report iff (#mismatches>0) || (#fuzzymatches>0) || (#errors>0 && settings say report errors)
- bool doReport = (numMismatches > 0);
- if (!doReport) {
- bool reportErrors = settings->value("ReportMissingResults").toBool();
- computeStats();
- foreach (const QString &func, itemLists.keys()) {
- FuncStats stat = stats.value(func);
- if (stat.value(ImageItem::FuzzyMatch) > 0) {
- doReport = true;
- break;
- }
- foreach (const ImageItem &item, itemLists.value(func)) {
- if (reportErrors && item.status == ImageItem::Error) {
- doReport = true;
- break;
- }
- }
- if (doReport)
- break;
- }
- }
- if (!doReport)
- return;
- write();
- written = true;
-}
-
-void Report::computeStats()
-{
- if (hasStats)
- return;
- foreach (const QString &func, itemLists.keys()) {
- FuncStats funcStat;
- funcStat[ImageItem::Ok] = 0;
- funcStat[ImageItem::BaselineNotFound] = 0;
- funcStat[ImageItem::IgnoreItem] = 0;
- funcStat[ImageItem::Mismatch] = 0;
- funcStat[ImageItem::FuzzyMatch] = 0;
- funcStat[ImageItem::Error] = 0;
- foreach (const ImageItem &item, itemLists.value(func)) {
- funcStat[item.status]++;
- }
- stats[func] = funcStat;
- }
- hasStats = true;
-}
-
-QString Report::summary()
-{
- computeStats();
- QString res;
- foreach (const QString &func, itemLists.keys()) {
- FuncStats stat = stats.value(func);
- QString s = QString("%1 %3 mismatch(es), %4 error(s), %5 fuzzy match(es)\n");
- s = s.arg(QString("%1() [%2 items]:").arg(func).arg(itemLists.value(func).size()).leftJustified(40));
- s = s.arg(stat.value(ImageItem::Mismatch));
- s = s.arg(stat.value(ImageItem::Error));
- s = s.arg(stat.value(ImageItem::FuzzyMatch));
- res += s;
- }
-#if 0
- qDebug() << "***************************** Summary *************************";
- qDebug() << res;
- qDebug() << "***************************************************************";
-#endif
- return res;
-}
-
-void Report::write()
-{
- QFile file(rootDir + path);
- if (!file.open(QIODevice::WriteOnly | QIODevice::Truncate)) {
- qWarning() << "Failed to open report file" << file.fileName();
- return;
- }
- out.setDevice(&file);
-
- writeHeader();
- foreach(const QString &func, testFunctions) {
- writeFunctionResults(itemLists.value(func));
- }
- writeFooter();
- file.close();
- updateLatestPointer();
-}
-
-
-void Report::writeHeader()
-{
- QString title = plat.value(PI_Project) + QLC(':') + plat.value(PI_TestCase) + QLS(" Lancelot Test Report");
- out << "<!DOCTYPE html>\n"
- << "<html><head><title>" << title << "</title></head>\n"
- << "<body bgcolor=""#ddeeff""><h1>" << title << "</h1>\n"
- << "<p>Note: This is a <i>static</i> page, generated at " << QDateTime::currentDateTime().toString()
- << " for the test run with id " << runId << "</p>\n"
- << "<p>Summary: <b><span style=\"color:red\">" << numMismatches << " of " << numItems << "</span></b> items reported mismatching</p>\n";
- out << "<pre>\n" << summary() << "</pre>\n\n";
- out << "<h3>Testing Client Platform Info:</h3>\n"
- << "<table>\n";
- foreach (QString key, plat.keys())
- out << "<tr><td>" << key << ":</td><td>" << plat.value(key) << "</td></tr>\n";
- out << "</table>\n\n";
- if (hasOverride) {
- out << "<span style=\"color:red\"><h4>Note! Override Platform Info:</h4></span>\n"
- << "<p>The client's output has been compared to baselines created on a different platform. Differences:</p>\n"
- << "<table>\n";
- for (int i = 0; i < plat.overrides().size()-1; i+=2)
- out << "<tr><td>" << plat.overrides().at(i) << ":</td><td>" << plat.overrides().at(i+1) << "</td></tr>\n";
- out << "</table>\n\n";
- }
-}
-
-
-void Report::writeFunctionResults(const ImageItemList &list)
-{
- QString testFunction = list.at(0).testFunction;
- QString pageUrl = BaselineServer::baseUrl() + path;
- QString ctx = handler->pathForItem(list.at(0), true, false).section(QLC('/'), 0, -2);
- QString misCtx = handler->pathForItem(list.at(0), false, false).section(QLC('/'), 0, -2);
-
-
- out << "\n<p>&nbsp;</p><h3>Test function: " << testFunction << "</h3>\n";
- if (!hasOverride) {
- out << "<p><a href=\"/cgi-bin/server.cgi?cmd=clearAllBaselines&context=" << ctx << "&url=" << pageUrl
- << "\"><b>Clear all baselines</b></a> for this testfunction (They will be recreated by the next run)</p>\n";
- out << "<p><a href=\"/cgi-bin/server.cgi?cmd=updateAllBaselines&context=" << ctx << "&mismatchContext=" << misCtx << "&url=" << pageUrl
- << "\"><b>Let these mismatching images be the new baselines</b></a> for this testfunction</p>\n\n";
- }
-
- out << "<table border=\"2\">\n"
- "<tr>\n"
- "<th width=123>Item</th>\n"
- "<th width=246>Baseline</th>\n"
- "<th width=246>Rendered</th>\n"
- "<th width=246>Comparison (diffs are <span style=\"color:red\">RED</span>)</th>\n"
- "<th width=246>Info/Action</th>\n"
- "</tr>\n\n";
-
- foreach (const ImageItem &item, list) {
- QString mmPrefix = handler->pathForItem(item, false, false);
- QString blPrefix = handler->pathForItem(item, true, false);
-
- // Make hard links to the current baseline, so that the report is static even if the baseline changes
- generateThumbnail(blPrefix + QLS(FileFormat), rootDir); // Make sure baseline thumbnail is up to date
- QString lnPrefix = mmPrefix + QLS("baseline.");
- QByteArray blPrefixBa = (rootDir + blPrefix).toLatin1();
- QByteArray lnPrefixBa = (rootDir + lnPrefix).toLatin1();
- ::link((blPrefixBa + FileFormat).constData(), (lnPrefixBa + FileFormat).constData());
- ::link((blPrefixBa + MetadataFileExt).constData(), (lnPrefixBa + MetadataFileExt).constData());
- ::link((blPrefixBa + ThumbnailExt).constData(), (lnPrefixBa + ThumbnailExt).constData());
-
- QString baseline = lnPrefix + QLS(FileFormat);
- QString metadata = lnPrefix + QLS(MetadataFileExt);
- out << "<tr>\n";
- out << "<td>" << item.itemName << "</td>\n";
- if (item.status == ImageItem::Mismatch || item.status == ImageItem::FuzzyMatch) {
- QString rendered = mmPrefix + QLS(FileFormat);
- QString itemFile = mmPrefix.section(QLC('/'), -1);
- writeItem(baseline, rendered, item, itemFile, ctx, misCtx, metadata);
- }
- else {
- out << "<td align=center><a href=\"/" << baseline << "\">image</a> <a href=\"/" << metadata << "\">info</a></td>\n"
- << "<td align=center colspan=2><small>n/a</small></td>\n"
- << "<td align=center>";
- switch (item.status) {
- case ImageItem::BaselineNotFound:
- out << "Baseline not found/regenerated";
- break;
- case ImageItem::IgnoreItem:
- out << "<span style=\"background-color:yellow\">Blacklisted</span> ";
- if (!hasOverride) {
- out << "<a href=\"/cgi-bin/server.cgi?cmd=whitelist&context=" << ctx
- << "&itemId=" << item.itemName << "&url=" << pageUrl
- << "\">Whitelist this item</a>";
- }
- break;
- case ImageItem::Error:
- out << "<span style=\"background-color:red\">Error: No result reported!</span>";
- break;
- case ImageItem::Ok:
- out << "<span style=\"color:green\"><small>No mismatch reported</small></span>";
- break;
- default:
- out << '?';
- break;
- }
- out << "</td>\n";
- }
- out << "</tr>\n\n";
- }
-
- out << "</table>\n";
-}
-
-void Report::writeItem(const QString &baseline, const QString &rendered, const ImageItem &item,
- const QString &itemFile, const QString &ctx, const QString &misCtx, const QString &metadata)
-{
- QString compared = generateCompared(baseline, rendered);
- QString pageUrl = BaselineServer::baseUrl() + path;
-
- QStringList images = QStringList() << baseline << rendered << compared;
- foreach (const QString& img, images)
- out << "<td height=246 align=center><a href=\"/" << img << "\"><img src=\"/" << generateThumbnail(img, rootDir) << "\"></a></td>\n";
-
- out << "<td align=center>\n";
- if (item.status == ImageItem::FuzzyMatch)
- out << "<p><span style=\"color:orange\">Fuzzy match</span></p>\n";
- else
- out << "<p><span style=\"color:red\">Mismatch reported</span></p>\n";
- out << "<p><a href=\"/" << metadata << "\">Baseline Info</a>\n";
- if (!hasOverride) {
- out << "<p><a href=\"/cgi-bin/server.cgi?cmd=updateSingleBaseline&context=" << ctx << "&mismatchContext=" << misCtx
- << "&itemFile=" << itemFile << "&url=" << pageUrl << "\">Let this be the new baseline</a></p>\n"
- << "<p><a href=\"/cgi-bin/server.cgi?cmd=blacklist&context=" << ctx
- << "&itemId=" << item.itemName << "&url=" << pageUrl << "\">Blacklist this item</a></p>\n";
- }
- out << "<p><a href=\"/cgi-bin/server.cgi?cmd=view&baseline=" << baseline << "&rendered=" << rendered
- << "&compared=" << compared << "&url=" << pageUrl << "\">Inspect</a></p>\n";
-
-#if 0
- out << "<p><a href=\"/cgi-bin/server.cgi?cmd=diffstats&baseline=" << baseline << "&rendered=" << rendered
- << "&url=" << pageUrl << "\">Diffstats</a></p>\n";
-#endif
-
- out << "</td>\n";
-}
-
-void Report::writeFooter()
-{
- out << "\n</body></html>\n";
-}
-
-
-QString Report::generateCompared(const QString &baseline, const QString &rendered, bool fuzzy)
-{
- QString res = rendered;
- QFileInfo fi(res);
- res.chop(fi.suffix().length());
- res += QLS(fuzzy ? "fuzzycompared.png" : "compared.png");
- QStringList args;
- if (fuzzy)
- args << QLS("-fuzz") << QLS("5%");
- args << rootDir+baseline << rootDir+rendered << rootDir+res;
- QProcess::execute(QLS("compare"), args);
- return res;
-}
-
-
-QString Report::generateThumbnail(const QString &image, const QString &rootDir)
-{
- QString res = image;
- QFileInfo imgFI(rootDir+image);
- if (!imgFI.exists())
- return res;
- res.chop(imgFI.suffix().length());
- res += ThumbnailExt;
- QFileInfo resFI(rootDir+res);
- if (resFI.exists() && resFI.lastModified() > imgFI.lastModified())
- return res;
- QStringList args;
- args << rootDir+image << QLS("-resize") << QLS("240x240>") << QLS("-quality") << QLS("50") << rootDir+res;
- QProcess::execute(QLS("convert"), args);
- return res;
-}
-
-
-QString Report::writeResultsXmlFiles()
-{
- if (!itemLists.size())
- return QString();
- QString dir = rootDir + baseDir + QLS("/xml-reports/") + runId;
- QDir cwd;
- if (!cwd.exists(dir))
- cwd.mkpath(dir);
- foreach (const QString &func, itemLists.keys()) {
- QFile f(dir + QLatin1Char('/') + func + "-results.xml");
- if (!f.open(QIODevice::WriteOnly))
- continue;
- QXmlStreamWriter s(&f);
- s.setAutoFormatting(true);
- s.writeStartDocument();
- foreach (QString key, plat.keys()) {
- QString cmt = QLatin1Char(' ') + key + "=\"" + plat.value(key) +"\" ";
- s.writeComment(cmt.replace("--", "[-]"));
- }
- s.writeStartElement("testsuite");
- s.writeAttribute("name", func);
- foreach (const ImageItem &item, itemLists.value(func)) {
- QString res;
- switch (item.status) {
- case ImageItem::Ok:
- case ImageItem::FuzzyMatch:
- res = "pass";
- break;
- case ImageItem::Mismatch:
- case ImageItem::Error:
- res = "fail";
- break;
- case ImageItem::BaselineNotFound:
- case ImageItem::IgnoreItem:
- default:
- res = "skip";
- }
- s.writeStartElement("testcase");
- s.writeAttribute("name", item.itemName);
- s.writeAttribute("result", res);
- s.writeEndElement();
- }
- s.writeEndElement();
- s.writeEndDocument();
- }
- return dir;
-}
-
-
-
-void Report::updateLatestPointer()
-{
- QString linkPath = rootDir + baseDir + QLS("/latest_report.html");
- QString reportPath = path.mid(baseDir.size()+1);
- QFile::remove(linkPath); // possible race with another thread, yada yada yada
- QFile::link(reportPath, linkPath);
-
-#if 0
- QByteArray fwd = "<!DOCTYPE html><html><head><meta HTTP-EQUIV=\"refresh\" CONTENT=\"0;URL=%1\"></meta></head><body></body></html>\n";
- fwd.replace("%1", filePath().prepend(QLC('/')).toLatin1());
-
- QFile file(rootDir + baseDir + "/latest_report.html");
- if (file.open(QIODevice::WriteOnly | QIODevice::Truncate))
- file.write(fwd);
-#endif
-}
-
-
-void Report::handleCGIQuery(const QString &query)
-{
- QUrl cgiUrl(QLS("http://dummy/cgi-bin/dummy.cgi?") + query);
- QTextStream s(stdout);
- s << "Content-Type: text/html\r\n\r\n"
- << "<!DOCTYPE html>\n<HTML>\n<body bgcolor=""#ddeeff"">\n"; // Lancelot blue
-
- QString command(cgiUrl.queryItemValue("cmd"));
-
- if (command == QLS("view")) {
- s << BaselineHandler::view(cgiUrl.queryItemValue(QLS("baseline")),
- cgiUrl.queryItemValue(QLS("rendered")),
- cgiUrl.queryItemValue(QLS("compared")));
- }
-#if 0
- else if (command == QLS("diffstats")) {
- s << BaselineHandler::diffstats(cgiUrl.queryItemValue(QLS("baseline")),
- cgiUrl.queryItemValue(QLS("rendered")));
- }
-#endif
- else if (command == QLS("updateSingleBaseline")) {
- s << BaselineHandler::updateBaselines(cgiUrl.queryItemValue(QLS("context")),
- cgiUrl.queryItemValue(QLS("mismatchContext")),
- cgiUrl.queryItemValue(QLS("itemFile")));
- } else if (command == QLS("updateAllBaselines")) {
- s << BaselineHandler::updateBaselines(cgiUrl.queryItemValue(QLS("context")),
- cgiUrl.queryItemValue(QLS("mismatchContext")),
- QString());
- } else if (command == QLS("clearAllBaselines")) {
- s << BaselineHandler::clearAllBaselines(cgiUrl.queryItemValue(QLS("context")));
- } else if (command == QLS("blacklist")) {
- // blacklist a test
- s << BaselineHandler::blacklistTest(cgiUrl.queryItemValue(QLS("context")),
- cgiUrl.queryItemValue(QLS("itemId")));
- } else if (command == QLS("whitelist")) {
- // whitelist a test
- s << BaselineHandler::blacklistTest(cgiUrl.queryItemValue(QLS("context")),
- cgiUrl.queryItemValue(QLS("itemId")), true);
- } else {
- s << "Unknown query:<br>" << query << "<br>";
- }
- s << "<p><a href=\"" << cgiUrl.queryItemValue(QLS("url")) << "\">Back to report</a>\n";
- s << "</body>\n</HTML>";
-}
diff --git a/tests/baselineserver/src/report.h b/tests/baselineserver/src/report.h
deleted file mode 100644
index c568e7ab8d..0000000000
--- a/tests/baselineserver/src/report.h
+++ /dev/null
@@ -1,98 +0,0 @@
-/****************************************************************************
-**
-** Copyright (C) 2016 The Qt Company Ltd.
-** Contact: https://www.qt.io/licensing/
-**
-** This file is part of the test suite of the Qt Toolkit.
-**
-** $QT_BEGIN_LICENSE:GPL-EXCEPT$
-** Commercial License Usage
-** Licensees holding valid commercial Qt licenses may use this file in
-** accordance with the commercial license agreement provided with the
-** Software or, alternatively, in accordance with the terms contained in
-** a written agreement between you and The Qt Company. For licensing terms
-** and conditions see https://www.qt.io/terms-conditions. For further
-** information use the contact form at https://www.qt.io/contact-us.
-**
-** GNU General Public License Usage
-** Alternatively, this file may be used under the terms of the GNU
-** General Public License version 3 as published by the Free Software
-** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
-** included in the packaging of this file. Please review the following
-** information to ensure the GNU General Public License requirements will
-** be met: https://www.gnu.org/licenses/gpl-3.0.html.
-**
-** $QT_END_LICENSE$
-**
-****************************************************************************/
-#ifndef REPORT_H
-#define REPORT_H
-
-#include "baselineprotocol.h"
-#include <QFile>
-#include <QTextStream>
-#include <QMap>
-#include <QStringList>
-#include <QSettings>
-
-class BaselineHandler;
-
-class Report
-{
-public:
- Report();
- ~Report();
-
- void init(const BaselineHandler *h, const QString &r, const PlatformInfo &p, const QSettings *s);
- void addItems(const ImageItemList& items);
- void addResult(const ImageItem& item);
- void end();
-
- bool reportProduced();
-
- int numberOfMismatches();
- QString summary();
-
- QString filePath();
-
- QString writeResultsXmlFiles();
-
- static void handleCGIQuery(const QString &query);
-
- static QString generateThumbnail(const QString &image, const QString &rootDir = QString());
-
-private:
- void write();
- void writeFunctionResults(const ImageItemList &list);
- void writeItem(const QString &baseline, const QString &rendered, const ImageItem &item,
- const QString &itemFile, const QString &ctx, const QString &misCtx, const QString &metadata);
- void writeHeader();
- void writeFooter();
- QString generateCompared(const QString &baseline, const QString &rendered, bool fuzzy = false);
-
- void updateLatestPointer();
-
- void computeStats();
-
- bool initialized;
- const BaselineHandler *handler;
- QString runId;
- PlatformInfo plat;
- QString rootDir;
- QString baseDir;
- QString path;
- QStringList testFunctions;
- QMap<QString, ImageItemList> itemLists;
- bool written;
- int numItems;
- int numMismatches;
- QTextStream out;
- bool hasOverride;
- const QSettings *settings;
-
- typedef QMap<ImageItem::ItemStatus, int> FuncStats;
- QMap<QString, FuncStats> stats;
- bool hasStats;
-};
-
-#endif // REPORT_H
diff --git a/tests/baselineserver/src/templates/view.html b/tests/baselineserver/src/templates/view.html
deleted file mode 100644
index f0971010f2..0000000000
--- a/tests/baselineserver/src/templates/view.html
+++ /dev/null
@@ -1,84 +0,0 @@
-<h3>Lancelot Viewer</h3>
-
-<p>
-Zoom:
-<input name="zoom" id="z1" type="radio" checked="Checked">1x</input>
-<input name="zoom" id="z2" type="radio">2x</input>
-<input name="zoom" id="z4" type="radio">4x</input>
-</p>
-
-<p><table>
-<tr>
-<td><input name="imgselect" id="baseline" type="radio" checked="Checked">Baseline</input></td>
-<td>%1</td>
-</tr>
-<tr>
-<td><input name="imgselect" id="rendered" type="radio">Rendered</input></td>
-<td>%2</td>
-</tr>
-<tr>
-<td><input name="imgselect" id="compared" type="radio">Differences</input></td>
-<td></td>
-</tr>
-</table></p>
-
-
-<p><table cellspacing="25"><tr>
-<td valign="top">
-<canvas id="c" width="800" height="800"></canvas>
-</td>
-<td valign="top">
-%4
-</td>
-</tr></table></p>
-
-<script>
- var canvas = document.getElementById("c");
- var context = canvas.getContext("2d");
- var cat = new Image();
- cat.src = "%1";
- var z = 1;
- cat.onload = function() {
- context.mozImageSmoothingEnabled = false;
- context.drawImage(cat, 0, 0, z*cat.width, z*cat.height);
- };
-
- var bbut = document.getElementById("baseline");
- bbut.onclick = function() {
- cat.src = "%1";
- };
-
- var rbut = document.getElementById("rendered");
- rbut.onclick = function() {
- cat.src = "%2";
- };
-
- var cbut = document.getElementById("compared");
- cbut.onclick = function() {
- cat.src = "%3";
- };
-
- function setZoom(zoom)
- {
- z = zoom;
- canvas.width = z*800;
- canvas.height = z*800;
- context.mozImageSmoothingEnabled = false;
- context.drawImage(cat, 0, 0, z*cat.width, z*cat.height);
- }
-
- var z1but = document.getElementById("z1");
- z1but.onclick = function() {
- setZoom(1);
- };
-
- var z2but = document.getElementById("z2");
- z2but.onclick = function() {
- setZoom(2);
- };
-
- var z4but = document.getElementById("z4");
- z4but.onclick = function() {
- setZoom(4);
- };
-</script>
generated by cgit v1.2.3 (git 2.25.1) at 2024-05-29 09:53:12 +0000