From bf3c84f718454f84b2ffbfa7fd1c7998bb5b01c2 Mon Sep 17 00:00:00 2001 From: aavit Date: Wed, 29 Jun 2011 13:24:37 +0200 Subject: Cleaning up the QPainter/arthur testing stuff This removes various remains of historical test tools, and the entire tests/arthur directory. The living parts are now: tests/auto/lancelot - including the suite of qps scripts. The script engine now lives here. tests/baselineserver - moved to toplevel since not arthur-specific. tests/manual/lance - for manual running and editing of qps scripts. Change-Id: I7c7f5df9197f4984a918dd1f9b31f42ee80d6152 Reviewed-on: http://codereview.qt.nokia.com/895 Reviewed-by: Qt Sanity Bot Reviewed-by: Gunnar Sletta --- tests/baselineserver/src/baselineserver.cpp | 576 +++++++++++++++++++++++++++ tests/baselineserver/src/baselineserver.h | 141 +++++++ tests/baselineserver/src/baselineserver.pro | 30 ++ tests/baselineserver/src/baselineserver.qrc | 5 + tests/baselineserver/src/main.cpp | 70 ++++ tests/baselineserver/src/report.cpp | 311 +++++++++++++++ tests/baselineserver/src/report.h | 91 +++++ tests/baselineserver/src/templates/view.html | 79 ++++ 8 files changed, 1303 insertions(+) create mode 100644 tests/baselineserver/src/baselineserver.cpp create mode 100644 tests/baselineserver/src/baselineserver.h create mode 100644 tests/baselineserver/src/baselineserver.pro create mode 100644 tests/baselineserver/src/baselineserver.qrc create mode 100644 tests/baselineserver/src/main.cpp create mode 100644 tests/baselineserver/src/report.cpp create mode 100644 tests/baselineserver/src/report.h create mode 100644 tests/baselineserver/src/templates/view.html (limited to 'tests/baselineserver/src') diff --git a/tests/baselineserver/src/baselineserver.cpp b/tests/baselineserver/src/baselineserver.cpp new file mode 100644 index 0000000000..6ff0a0c72d --- /dev/null +++ b/tests/baselineserver/src/baselineserver.cpp @@ -0,0 +1,576 @@ +/**************************************************************************** +** +** Copyright (C) 2011 Nokia Corporation and/or its subsidiary(-ies). +** All rights reserved. +** Contact: Nokia Corporation (qt-info@nokia.com) +** +** This file is part of the test suite of the Qt Toolkit. +** +** $QT_BEGIN_LICENSE:LGPL$ +** GNU Lesser General Public License Usage +** This file may be used under the terms of the GNU Lesser General Public +** License version 2.1 as published by the Free Software Foundation and +** appearing in the file LICENSE.LGPL included in the packaging of this +** file. Please review the following information to ensure the GNU Lesser +** General Public License version 2.1 requirements will be met: +** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. +** +** In addition, as a special exception, Nokia gives you certain additional +** rights. These rights are described in the Nokia Qt LGPL Exception +** version 1.1, included in the file LGPL_EXCEPTION.txt in this package. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU General +** Public License version 3.0 as published by the Free Software Foundation +** and appearing in the file LICENSE.GPL included in the packaging of this +** file. Please review the following information to ensure the GNU General +** Public License version 3.0 requirements will be met: +** http://www.gnu.org/copyleft/gpl.html. +** +** Other Usage +** Alternatively, this file may be used in accordance with the terms and +** conditions contained in a signed written agreement between you and Nokia. +** +** +** +** +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#define QT_USE_FAST_CONCATENATION +#define QT_USE_FAST_OPERATOR_PLUS + +#include "baselineserver.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// extra fields, for use in image metadata storage +const QString PI_ImageChecksum(QLS("ImageChecksum")); +const QString PI_RunId(QLS("RunId")); +const QString PI_CreationDate(QLS("CreationDate")); + +QString BaselineServer::storage; +QString BaselineServer::url; +QString BaselineServer::settingsFile; + +BaselineServer::BaselineServer(QObject *parent) + : QTcpServer(parent), lastRunIdIdx(0) +{ + QFileInfo me(QCoreApplication::applicationFilePath()); + meLastMod = me.lastModified(); + heartbeatTimer = new QTimer(this); + connect(heartbeatTimer, SIGNAL(timeout()), this, SLOT(heartbeat())); + heartbeatTimer->start(HEARTBEAT*1000); +} + +QString BaselineServer::storagePath() +{ + if (storage.isEmpty()) { + storage = QLS(qgetenv("QT_LANCELOT_DIR")); + if (storage.isEmpty()) + storage = QLS("/var/www"); + } + return storage; +} + +QString BaselineServer::baseUrl() +{ + if (url.isEmpty()) { + url = QLS("http://") + + QHostInfo::localHostName().toLatin1() + '.' + + QHostInfo::localDomainName().toLatin1() + '/'; + } + return url; +} + +QString BaselineServer::settingsFilePath() +{ + if (settingsFile.isEmpty()) { + QString exeName = QCoreApplication::applicationFilePath().section(QLC('/'), -1); + settingsFile = storagePath() + QLC('/') + exeName + QLS(".ini"); + } + return settingsFile; +} + +void BaselineServer::incomingConnection(int socketDescriptor) +{ + QString runId = QDateTime::currentDateTime().toString(QLS("MMMdd-hhmmss")); + if (runId == lastRunId) { + runId += QLC('-') + QString::number(++lastRunIdIdx); + } else { + lastRunId = runId; + lastRunIdIdx = 0; + } + qDebug() << "Server: New connection! RunId:" << runId; + BaselineThread *thread = new BaselineThread(runId, socketDescriptor, this); + connect(thread, SIGNAL(finished()), thread, SLOT(deleteLater())); + thread->start(); +} + +void BaselineServer::heartbeat() +{ + // The idea is to exit to be restarted when modified, as soon as not actually serving + QFileInfo me(QCoreApplication::applicationFilePath()); + if (me.lastModified() == meLastMod) + return; + if (!me.exists() || !me.isExecutable()) + return; + + //# (could close() here to avoid accepting new connections, to avoid livelock) + //# also, could check for a timeout to force exit, to avoid hung threads blocking + bool isServing = false; + foreach(BaselineThread *thread, findChildren()) { + if (thread->isRunning()) { + isServing = true; + break; + } + } + + if (!isServing) + QCoreApplication::exit(); +} + +BaselineThread::BaselineThread(const QString &runId, int socketDescriptor, QObject *parent) + : QThread(parent), runId(runId), socketDescriptor(socketDescriptor) +{ +} + +void BaselineThread::run() +{ + BaselineHandler handler(runId, socketDescriptor); + exec(); +} + + +BaselineHandler::BaselineHandler(const QString &runId, int socketDescriptor) + : QObject(), runId(runId), connectionEstablished(false) +{ + settings = new QSettings(BaselineServer::settingsFilePath(), QSettings::IniFormat, this); + + if (socketDescriptor == -1) + return; + + connect(&proto.socket, SIGNAL(readyRead()), this, SLOT(receiveRequest())); + connect(&proto.socket, SIGNAL(disconnected()), this, SLOT(receiveDisconnect())); + proto.socket.setSocketDescriptor(socketDescriptor); +} + +const char *BaselineHandler::logtime() +{ + return 0; + //return QTime::currentTime().toString(QLS("mm:ss.zzz")); +} + +bool BaselineHandler::establishConnection() +{ + if (!proto.acceptConnection(&plat)) { + qWarning() << runId << logtime() << "Accepting new connection from" << proto.socket.peerAddress().toString() << "failed." << proto.errorMessage(); + proto.sendBlock(BaselineProtocol::Abort, proto.errorMessage().toLatin1()); // In case the client can hear us, tell it what's wrong. + proto.socket.disconnectFromHost(); + return false; + } + QString logMsg; + foreach (QString key, plat.keys()) { + if (key != PI_HostName && key != PI_HostAddress) + logMsg += key + QLS(": '") + plat.value(key) + QLS("', "); + } + qDebug() << runId << logtime() << "Connection established with" << plat.value(PI_HostName) + << "[" << qPrintable(plat.value(PI_HostAddress)) << "]" << logMsg; + + settings->beginGroup("ClientFilters"); + if (!settings->childKeys().isEmpty() && !plat.value(PI_PulseGitBranch).isEmpty()) { // i.e. not adhoc client + // Abort if client does not match the filters + foreach (QString filterKey, settings->childKeys()) { + QString filter = settings->value(filterKey).toString(); + QString platVal = plat.value(filterKey); + if (filter.isEmpty() || platVal.isEmpty()) + continue; // tbd: add a syntax for specifying a "value-must-be-present" filter + if (!platVal.contains(filter)) { + qDebug() << runId << logtime() << "Did not pass client filter on" << filterKey << "; disconnecting."; + proto.sendBlock(BaselineProtocol::Abort, QByteArray("Configured to not do testing for this client or repo, ref. ") + BaselineServer::settingsFilePath().toLatin1()); + proto.socket.disconnectFromHost(); + return false; + } + } + } + settings->endGroup(); + + proto.sendBlock(BaselineProtocol::Ack, QByteArray()); + + report.init(this, runId, plat); + return true; +} + +void BaselineHandler::receiveRequest() +{ + if (!connectionEstablished) { + connectionEstablished = establishConnection(); + return; + } + + QByteArray block; + BaselineProtocol::Command cmd; + if (!proto.receiveBlock(&cmd, &block)) { + qWarning() << runId << logtime() << "Command reception failed. "<< proto.errorMessage(); + QThread::currentThread()->exit(1); + return; + } + + switch(cmd) { + case BaselineProtocol::RequestBaselineChecksums: + provideBaselineChecksums(block); + break; + case BaselineProtocol::AcceptNewBaseline: + storeImage(block, true); + break; + case BaselineProtocol::AcceptMismatch: + storeImage(block, false); + break; + default: + qWarning() << runId << logtime() << "Unknown command received. " << proto.errorMessage(); + proto.sendBlock(BaselineProtocol::UnknownError, QByteArray()); + } +} + + +void BaselineHandler::provideBaselineChecksums(const QByteArray &itemListBlock) +{ + ImageItemList itemList; + QDataStream ds(itemListBlock); + ds >> itemList; + qDebug() << runId << logtime() << "Received request for checksums for" << itemList.count() + << "items in test function" << itemList.at(0).testFunction; + + for (ImageItemList::iterator i = itemList.begin(); i != itemList.end(); ++i) { + i->imageChecksums.clear(); + i->status = ImageItem::BaselineNotFound; + QString prefix = pathForItem(*i, true); + PlatformInfo itemData = fetchItemMetadata(prefix); + if (itemData.contains(PI_ImageChecksum)) { + bool ok = false; + quint64 checksum = itemData.value(PI_ImageChecksum).toULongLong(&ok, 16); + if (ok) { + i->imageChecksums.prepend(checksum); + i->status = ImageItem::Ok; + } + } + } + + // Find and mark blacklisted items + QString context = pathForItem(itemList.at(0), true, false).section(QLC('/'), 0, -2); + if (itemList.count() > 0) { + QFile file(BaselineServer::storagePath() + QLC('/') + context + QLS("/BLACKLIST")); + if (file.open(QIODevice::ReadOnly)) { + QTextStream in(&file); + do { + QString itemName = in.readLine(); + if (!itemName.isNull()) { + for (ImageItemList::iterator i = itemList.begin(); i != itemList.end(); ++i) { + if (i->itemName == itemName) + i->status = ImageItem::IgnoreItem; + } + } + } while (!in.atEnd()); + } + } + + QByteArray block; + QDataStream ods(&block, QIODevice::WriteOnly); + ods << itemList; + proto.sendBlock(BaselineProtocol::Ack, block); + report.addItems(itemList); +} + + +void BaselineHandler::storeImage(const QByteArray &itemBlock, bool isBaseline) +{ + QDataStream ds(itemBlock); + ImageItem item; + ds >> item; + + QString prefix = pathForItem(item, isBaseline); + qDebug() << runId << logtime() << "Received" << (isBaseline ? "baseline" : "mismatched") << "image for:" << item.itemName << "Storing in" << prefix; + + QString msg; + if (isBaseline) + msg = QLS("New baseline image stored: ") + pathForItem(item, true, true) + QLS(FileFormat); + else + msg = BaselineServer::baseUrl() + report.filePath(); + proto.sendBlock(BaselineProtocol::Ack, msg.toLatin1()); + + QString dir = prefix.section(QLC('/'), 0, -2); + QDir cwd; + if (!cwd.exists(dir)) + cwd.mkpath(dir); + item.image.save(prefix + QLS(FileFormat), FileFormat); + + PlatformInfo itemData = plat; + itemData.insert(PI_ImageChecksum, QString::number(item.imageChecksums.at(0), 16)); //# Only the first is stored. TBD: get rid of list + itemData.insert(PI_RunId, runId); + itemData.insert(PI_CreationDate, QDateTime::currentDateTime().toString()); + storeItemMetadata(itemData, prefix); + + if (!isBaseline) + report.addMismatch(item); +} + + +void BaselineHandler::storeItemMetadata(const PlatformInfo &metadata, const QString &path) +{ + QFile file(path + QLS(MetadataFileExt)); + if (!file.open(QIODevice::WriteOnly | QIODevice::Truncate)) { + qWarning() << runId << logtime() << "ERROR: could not write to file" << file.fileName(); + return; + } + QTextStream out(&file); + PlatformInfo::const_iterator it = metadata.constBegin(); + while (it != metadata.constEnd()) { + out << it.key() << ": " << it.value() << endl; + ++it; + } + file.close(); +} + + +PlatformInfo BaselineHandler::fetchItemMetadata(const QString &path) +{ + PlatformInfo res; + QFile file(path + QLS(MetadataFileExt)); + if (!file.open(QIODevice::ReadOnly)) + return res; + QTextStream in(&file); + do { + QString line = in.readLine(); + int idx = line.indexOf(QLS(": ")); + if (idx > 0) + res.insert(line.left(idx), line.mid(idx+2)); + } while (!in.atEnd()); + return res; +} + + +void BaselineHandler::receiveDisconnect() +{ + qDebug() << runId << logtime() << "Client disconnected."; + report.end(); + QThread::currentThread()->exit(0); +} + + +void BaselineHandler::mapPlatformInfo() const +{ + mapped = plat; + + // Map hostname + QString host = plat.value(PI_HostName).section(QLC('.'), 0, 0); // Filter away domain, if any + if (host.isEmpty() || host == QLS("localhost")) { + host = plat.value(PI_HostAddress); + } else { + if (!plat.value(PI_PulseGitBranch).isEmpty()) { + // i.e. pulse run, so remove index postfix typical of vm hostnames + host.remove(QRegExp(QLS("\\d+$"))); + if (host.endsWith(QLC('-'))) + host.chop(1); + } + } + if (host.isEmpty()) + host = QLS("unknownhost"); + mapped.insert(PI_HostName, host); + + // Map qmakespec + QString mkspec = plat.value(PI_QMakeSpec); + mapped.insert(PI_QMakeSpec, mkspec.replace(QLC('/'), QLC('_'))); + + // Map Qt version + QString ver = plat.value(PI_QtVersion); + mapped.insert(PI_QtVersion, ver.prepend(QLS("Qt-"))); +} + +QString BaselineHandler::pathForItem(const ImageItem &item, bool isBaseline, bool absolute) const +{ + if (mapped.isEmpty()) + mapPlatformInfo(); + + QString itemName = item.itemName.simplified(); + itemName.replace(QLC(' '), QLC('_')); + itemName.replace(QLC('.'), QLC('_')); + itemName.append(QLC('_')); + itemName.append(QString::number(item.itemChecksum, 16).rightJustified(4, QLC('0'))); + + QStringList path; + if (absolute) + path += BaselineServer::storagePath(); + path += mapped.value(PI_TestCase); + path += QLS(isBaseline ? "baselines" : "mismatches"); + path += item.testFunction; + path += mapped.value(PI_QtVersion); + path += mapped.value(PI_QMakeSpec); + path += mapped.value(PI_HostName); + if (!isBaseline) + path += runId; + path += itemName + QLC('.'); + + return path.join(QLS("/")); +} + + +QString BaselineHandler::view(const QString &baseline, const QString &rendered, const QString &compared) +{ + QFile f(":/templates/view.html"); + f.open(QIODevice::ReadOnly); + return QString::fromLatin1(f.readAll()).arg('/'+baseline, '/'+rendered, '/'+compared); +} + + +QString BaselineHandler::clearAllBaselines(const QString &context) +{ + int tot = 0; + int failed = 0; + QDirIterator it(BaselineServer::storagePath() + QLC('/') + context, + QStringList() << QLS("*.") + QLS(FileFormat) << QLS("*.") + QLS(MetadataFileExt)); + while (it.hasNext()) { + tot++; + if (!QFile::remove(it.next())) + failed++; + } + return QString(QLS("%1 of %2 baselines cleared from context ")).arg((tot-failed)/2).arg(tot/2) + context; +} + +QString BaselineHandler::updateBaselines(const QString &context, const QString &mismatchContext, const QString &itemFile) +{ + int tot = 0; + int failed = 0; + QString storagePrefix = BaselineServer::storagePath() + QLC('/'); + // If itemId is set, update just that one, otherwise, update all: + QString filter = itemFile.isEmpty() ? QLS("*_????.") : itemFile; + QDirIterator it(storagePrefix + mismatchContext, QStringList() << filter + QLS(FileFormat) << filter + QLS(MetadataFileExt)); + while (it.hasNext()) { + tot++; + it.next(); + QString oldFile = storagePrefix + context + QLC('/') + it.fileName(); + QFile::remove(oldFile); // Remove existing baseline file + if (!QFile::copy(it.filePath(), oldFile)) // and replace it with the mismatch + failed++; + } + return QString(QLS("%1 of %2 baselines updated in context %3 from context %4")).arg((tot-failed)/2).arg(tot/2).arg(context, mismatchContext); +} + +QString BaselineHandler::blacklistTest(const QString &context, const QString &itemId, bool removeFromBlacklist) +{ + QFile file(BaselineServer::storagePath() + QLC('/') + context + QLS("/BLACKLIST")); + QStringList blackList; + if (file.open(QIODevice::ReadWrite)) { + while (!file.atEnd()) + blackList.append(file.readLine().trimmed()); + + if (removeFromBlacklist) + blackList.removeAll(itemId); + else if (!blackList.contains(itemId)) + blackList.append(itemId); + + file.resize(0); + foreach (QString id, blackList) + file.write(id.toLatin1() + '\n'); + file.close(); + return QLS(removeFromBlacklist ? "Whitelisted " : "Blacklisted ") + itemId + QLS(" in context ") + context; + } else { + return QLS("Unable to update blacklisted tests, failed to open ") + file.fileName(); + } +} + + +void BaselineHandler::testPathMapping() +{ + qDebug() << "Storage prefix:" << BaselineServer::storagePath(); + + QStringList hosts; + hosts << QLS("bq-ubuntu910-x86-01") + << QLS("bq-ubuntu910-x86-15") + << QLS("osl-mac-master-5.test.qt.nokia.com") + << QLS("osl-mac-master-6.test.qt.nokia.com") + << QLS("sv-xp-vs-010") + << QLS("sv-xp-vs-011") + << QLS("sv-solaris-sparc-008") + << QLS("macbuilder-02.test.troll.no") + << QLS("bqvm1164") + << QLS("chimera") + << QLS("localhost") + << QLS(""); + + ImageItem item; + item.testFunction = QLS("testPathMapping"); + item.itemName = QLS("arcs.qps"); + item.imageChecksums << 0x0123456789abcdefULL; + item.itemChecksum = 0x0123; + + plat.insert(PI_QtVersion, QLS("5.0.0")); + plat.insert(PI_BuildKey, QLS("(nobuildkey)")); + plat.insert(PI_QMakeSpec, QLS("linux-g++")); + plat.insert(PI_PulseGitBranch, QLS("somebranch")); + foreach(const QString& host, hosts) { + mapped.clear(); + plat.insert(PI_HostName, host); + qDebug() << "Baseline from" << host << "->" << pathForItem(item, true); + qDebug() << "Mismatch from" << host << "->" << pathForItem(item, false); + } +} + + +QString BaselineHandler::computeMismatchScore(const QImage &baseline, const QImage &rendered) +{ + if (baseline.size() != rendered.size() || baseline.format() != rendered.format()) + return QLS("[No score, incomparable images.]"); + if (baseline.depth() != 32) + return QLS("[Score computation not implemented for format.]"); + + int w = baseline.width(); + int h = baseline.height(); + + uint ncd = 0; // number of differing color pixels + uint nad = 0; // number of differing alpha pixels + uint scd = 0; // sum of color pixel difference + uint sad = 0; // sum of alpha pixel difference + + for (int y=0; y +#include +#include +#include +#include +#include +#include +#include + +#include "baselineprotocol.h" +#include "report.h" + +// #seconds between update checks +#define HEARTBEAT 10 +#define MetadataFileExt "metadata" + +class BaselineServer : public QTcpServer +{ + Q_OBJECT + +public: + BaselineServer(QObject *parent = 0); + + static QString storagePath(); + static QString baseUrl(); + static QString settingsFilePath(); + +protected: + void incomingConnection(int socketDescriptor); + +private slots: + void heartbeat(); + +private: + QTimer *heartbeatTimer; + QDateTime meLastMod; + QString lastRunId; + int lastRunIdIdx; + static QString storage; + static QString url; + static QString settingsFile; +}; + + + +class BaselineThread : public QThread +{ + Q_OBJECT + +public: + BaselineThread(const QString &runId, int socketDescriptor, QObject *parent); + void run(); + +private: + QString runId; + int socketDescriptor; +}; + + +class BaselineHandler : public QObject +{ + Q_OBJECT + +public: + BaselineHandler(const QString &runId, int socketDescriptor = -1); + void testPathMapping(); + QString pathForItem(const ImageItem &item, bool isBaseline = true, bool absolute = true) const; + + // CGI callbacks: + static QString view(const QString &baseline, const QString &rendered, const QString &compared); + static QString clearAllBaselines(const QString &context); + static QString updateBaselines(const QString &context, const QString &mismatchContext, const QString &itemFile); + static QString blacklistTest(const QString &context, const QString &itemId, bool removeFromBlacklist = false); + +private slots: + void receiveRequest(); + void receiveDisconnect(); + +private: + bool establishConnection(); + void provideBaselineChecksums(const QByteArray &itemListBlock); + void storeImage(const QByteArray &itemBlock, bool isBaseline); + void storeItemMetadata(const PlatformInfo &metadata, const QString &path); + PlatformInfo fetchItemMetadata(const QString &path); + void mapPlatformInfo() const; + const char *logtime(); + QString computeMismatchScore(const QImage& baseline, const QImage& rendered); + + BaselineProtocol proto; + PlatformInfo plat; + mutable PlatformInfo mapped; + QString runId; + bool connectionEstablished; + Report report; + QSettings *settings; +}; + +#endif // BASELINESERVER_H diff --git a/tests/baselineserver/src/baselineserver.pro b/tests/baselineserver/src/baselineserver.pro new file mode 100644 index 0000000000..b59d59d1ed --- /dev/null +++ b/tests/baselineserver/src/baselineserver.pro @@ -0,0 +1,30 @@ +#------------------------------------------------- +# +# Project created by QtCreator 2010-08-11T11:51:09 +# +#------------------------------------------------- + +QT += core network + +# gui needed for QImage +# QT -= gui + +TARGET = baselineserver +DESTDIR = ../bin +CONFIG += console +CONFIG -= app_bundle + +TEMPLATE = app + +include(../shared/baselineprotocol.pri) + +SOURCES += main.cpp \ + baselineserver.cpp \ + report.cpp + +HEADERS += \ + baselineserver.h \ + report.h + +RESOURCES += \ + baselineserver.qrc diff --git a/tests/baselineserver/src/baselineserver.qrc b/tests/baselineserver/src/baselineserver.qrc new file mode 100644 index 0000000000..b5cd6afadb --- /dev/null +++ b/tests/baselineserver/src/baselineserver.qrc @@ -0,0 +1,5 @@ + + + templates/view.html + + diff --git a/tests/baselineserver/src/main.cpp b/tests/baselineserver/src/main.cpp new file mode 100644 index 0000000000..8e5fa4e669 --- /dev/null +++ b/tests/baselineserver/src/main.cpp @@ -0,0 +1,70 @@ +/**************************************************************************** +** +** Copyright (C) 2011 Nokia Corporation and/or its subsidiary(-ies). +** All rights reserved. +** Contact: Nokia Corporation (qt-info@nokia.com) +** +** This file is part of the test suite of the Qt Toolkit. +** +** $QT_BEGIN_LICENSE:LGPL$ +** GNU Lesser General Public License Usage +** This file may be used under the terms of the GNU Lesser General Public +** License version 2.1 as published by the Free Software Foundation and +** appearing in the file LICENSE.LGPL included in the packaging of this +** file. Please review the following information to ensure the GNU Lesser +** General Public License version 2.1 requirements will be met: +** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. +** +** In addition, as a special exception, Nokia gives you certain additional +** rights. These rights are described in the Nokia Qt LGPL Exception +** version 1.1, included in the file LGPL_EXCEPTION.txt in this package. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU General +** Public License version 3.0 as published by the Free Software Foundation +** and appearing in the file LICENSE.GPL included in the packaging of this +** file. Please review the following information to ensure the GNU General +** Public License version 3.0 requirements will be met: +** http://www.gnu.org/copyleft/gpl.html. +** +** Other Usage +** Alternatively, this file may be used in accordance with the terms and +** conditions contained in a signed written agreement between you and Nokia. +** +** +** +** +** +** $QT_END_LICENSE$ +** +****************************************************************************/ +#include +#include "baselineserver.h" + +int main(int argc, char *argv[]) +{ + QCoreApplication a(argc, argv); + + QString queryString(qgetenv("QUERY_STRING")); + if (!queryString.isEmpty()) { + // run as CGI script + Report::handleCGIQuery(queryString); + return 0; + } + + if (a.arguments().contains(QLatin1String("-testmapping"))) { + BaselineHandler h(QLS("SomeRunId")); + h.testPathMapping(); + return 0; + } + + BaselineServer server; + if (!server.listen(QHostAddress::Any, BaselineProtocol::ServerPort)) { + qWarning("Failed to listen!"); + return 1; + } + + qDebug() << "\n*****" << argv[0] << "started, ready to serve on port" << BaselineProtocol::ServerPort + << "with baseline protocol version" << BaselineProtocol::ProtocolVersion << "*****\n"; + return a.exec(); +} diff --git a/tests/baselineserver/src/report.cpp b/tests/baselineserver/src/report.cpp new file mode 100644 index 0000000000..7c2d6ac6df --- /dev/null +++ b/tests/baselineserver/src/report.cpp @@ -0,0 +1,311 @@ +/**************************************************************************** +** +** Copyright (C) 2011 Nokia Corporation and/or its subsidiary(-ies). +** All rights reserved. +** Contact: Nokia Corporation (qt-info@nokia.com) +** +** This file is part of the test suite of the Qt Toolkit. +** +** $QT_BEGIN_LICENSE:LGPL$ +** GNU Lesser General Public License Usage +** This file may be used under the terms of the GNU Lesser General Public +** License version 2.1 as published by the Free Software Foundation and +** appearing in the file LICENSE.LGPL included in the packaging of this +** file. Please review the following information to ensure the GNU Lesser +** General Public License version 2.1 requirements will be met: +** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. +** +** In addition, as a special exception, Nokia gives you certain additional +** rights. These rights are described in the Nokia Qt LGPL Exception +** version 1.1, included in the file LGPL_EXCEPTION.txt in this package. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU General +** Public License version 3.0 as published by the Free Software Foundation +** and appearing in the file LICENSE.GPL included in the packaging of this +** file. Please review the following information to ensure the GNU General +** Public License version 3.0 requirements will be met: +** http://www.gnu.org/copyleft/gpl.html. +** +** Other Usage +** Alternatively, this file may be used in accordance with the terms and +** conditions contained in a signed written agreement between you and Nokia. +** +** +** +** +** +** $QT_END_LICENSE$ +** +****************************************************************************/ +#include "report.h" +#include "baselineprotocol.h" +#include "baselineserver.h" +#include +#include +#include + +Report::Report() + : written(false), numItems(0), numMismatches(0) +{ +} + +Report::~Report() +{ + end(); +} + +QString Report::filePath() +{ + return path; +} + +void Report::init(const BaselineHandler *h, const QString &r, const PlatformInfo &p) +{ + handler = h; + runId = r; + plat = p; + rootDir = BaselineServer::storagePath() + QLC('/'); + reportDir = plat.value(PI_TestCase) + QLC('/') + (plat.value(PI_PulseGitBranch).isEmpty() ? QLS("reports/adhoc/") : QLS("reports/pulse/")); + QString dir = rootDir + reportDir; + QDir cwd; + if (!cwd.exists(dir)) + cwd.mkpath(dir); + path = reportDir + QLS("Report_") + runId + QLS(".html"); +} + +void Report::addItems(const ImageItemList &items) +{ + if (items.isEmpty()) + return; + numItems += items.size(); + QString func = items.at(0).testFunction; + if (!testFunctions.contains(func)) + testFunctions.append(func); + itemLists[func] += items; +} + +void Report::addMismatch(const ImageItem &item) +{ + if (!testFunctions.contains(item.testFunction)) { + qWarning() << "Report::addMismatch: unknown testfunction" << item.testFunction; + return; + } + bool found = false; + ImageItemList &list = itemLists[item.testFunction]; + for (ImageItemList::iterator it = list.begin(); it != list.end(); ++it) { + if (it->itemName == item.itemName && it->itemChecksum == item.itemChecksum) { + it->status = ImageItem::Mismatch; + found = true; + break; + } + } + if (found) + numMismatches++; + else + qWarning() << "Report::addMismatch: unknown item" << item.itemName << "in testfunction" << item.testFunction; +} + +void Report::end() +{ + if (written || !numMismatches) + return; + write(); + written = true; +} + + +void Report::write() +{ + QFile file(rootDir + path); + if (!file.open(QIODevice::WriteOnly | QIODevice::Truncate)) { + qWarning() << "Failed to open report file" << file.fileName(); + return; + } + out.setDevice(&file); + + writeHeader(); + foreach(const QString &func, testFunctions) { + writeFunctionResults(itemLists.value(func)); + } + writeFooter(); + file.close(); +} + + +void Report::writeHeader() +{ + QString title = plat.value(PI_TestCase) + QLS(" Qt Baseline Test Report"); + out << "" << title << "\n" + << "

" << title << "

\n" + << "

Note: This is a static page, generated at " << QDateTime::currentDateTime().toString() + << " for the test run with id " << runId << "

\n" + << "

Summary: " << numMismatches << " of " << numItems << " items reported mismatching

\n\n"; + out << "

Platform Info:

\n" + << "\n"; + foreach (QString key, plat.keys()) + out << "\n"; + out << "
" << key << "" << plat.value(key) << "
\n\n"; +} + + +void Report::writeFunctionResults(const ImageItemList &list) +{ + QString testFunction = list.at(0).testFunction; + QString pageUrl = BaselineServer::baseUrl() + path; + QString ctx = handler->pathForItem(list.at(0), true, false).section(QLC('/'), 0, -2); + QString misCtx = handler->pathForItem(list.at(0), false, false).section(QLC('/'), 0, -2); + + + out << "\n

 

Test function: " << testFunction << "

\n"; + out << "

Clear all baselines for this testfunction (They will be recreated by the next run)

\n"; + out << "

Let these mismatching images be the new baselines for this testfunction

\n\n"; + + out << "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n\n"; + + foreach (const ImageItem &item, list) { + out << "\n"; + out << "\n"; + QString prefix = handler->pathForItem(item, true, false); + QString baseline = prefix + QLS(FileFormat); + QString metadata = prefix + QLS(MetadataFileExt); + if (item.status == ImageItem::Mismatch) { + QString rendered = handler->pathForItem(item, false, false) + QLS(FileFormat); + QString itemFile = prefix.section(QLC('/'), -1); + writeItem(baseline, rendered, item, itemFile, ctx, misCtx, metadata); + } + else { + out << "\n" + << "\n" + << "\n"; + } + out << "\n\n"; + } + + out << "
ItemBaselineRenderedComparison (diffs are RED)Info/Action
" << item.itemName << "image infon/a"; + switch (item.status) { + case ImageItem::BaselineNotFound: + out << "Baseline not found/regenerated"; + break; + case ImageItem::IgnoreItem: + out << "Blacklisted " + << "Whitelist this item"; + break; + case ImageItem::Ok: + out << "No mismatch reported"; + break; + default: + out << "?"; + break; + } + out << "
\n"; +} + +void Report::writeItem(const QString &baseline, const QString &rendered, const ImageItem &item, + const QString &itemFile, const QString &ctx, const QString &misCtx, const QString &metadata) +{ + QString compared = generateCompared(baseline, rendered); + QString pageUrl = BaselineServer::baseUrl() + path; + + QStringList images = QStringList() << baseline << rendered << compared; + foreach (const QString& img, images) + out << "\n"; + + out << "\n" + << "

Mismatch reported

\n" + << "

Baseline Info\n" + << "

Let this be the new baseline

\n" + << "

Blacklist this item

\n" + << "

Inspect

\n" + << "\n"; +} + +void Report::writeFooter() +{ + out << "\n\n"; +} + + +QString Report::generateCompared(const QString &baseline, const QString &rendered, bool fuzzy) +{ + QString res = rendered; + QFileInfo fi(res); + res.chop(fi.suffix().length() + 1); + res += QLS(fuzzy ? "_fuzzycompared.png" : "_compared.png"); + QStringList args; + if (fuzzy) + args << QLS("-fuzz") << QLS("5%"); + args << rootDir+baseline << rootDir+rendered << rootDir+res; + QProcess::execute(QLS("compare"), args); + return res; +} + + +QString Report::generateThumbnail(const QString &image) +{ + QString res = image; + QFileInfo imgFI(rootDir+image); + res.chop(imgFI.suffix().length() + 1); + res += QLS("_thumbnail.jpg"); + QFileInfo resFI(rootDir+res); + if (resFI.exists() && resFI.lastModified() > imgFI.lastModified()) + return res; + QStringList args; + args << rootDir+image << QLS("-resize") << QLS("240x240>") << QLS("-quality") << QLS("50") << rootDir+res; + QProcess::execute(QLS("convert"), args); + return res; +} + + +void Report::handleCGIQuery(const QString &query) +{ + QUrl cgiUrl(QLS("http://dummy/cgi-bin/dummy.cgi?") + query); + QTextStream s(stdout); + s << "Content-Type: text/html\r\n\r\n" + << ""; + + QString command(cgiUrl.queryItemValue("cmd")); + + if (command == QLS("view")) { + s << BaselineHandler::view(cgiUrl.queryItemValue(QLS("baseline")), + cgiUrl.queryItemValue(QLS("rendered")), + cgiUrl.queryItemValue(QLS("compared"))); + } + else if (command == QLS("updateSingleBaseline")) { + s << BaselineHandler::updateBaselines(cgiUrl.queryItemValue(QLS("context")), + cgiUrl.queryItemValue(QLS("mismatchContext")), + cgiUrl.queryItemValue(QLS("itemFile"))); + } else if (command == QLS("updateAllBaselines")) { + s << BaselineHandler::updateBaselines(cgiUrl.queryItemValue(QLS("context")), + cgiUrl.queryItemValue(QLS("mismatchContext")), + QString()); + } else if (command == QLS("clearAllBaselines")) { + s << BaselineHandler::clearAllBaselines(cgiUrl.queryItemValue(QLS("context"))); + } else if (command == QLS("blacklist")) { + // blacklist a test + s << BaselineHandler::blacklistTest(cgiUrl.queryItemValue(QLS("context")), + cgiUrl.queryItemValue(QLS("itemId"))); + } else if (command == QLS("whitelist")) { + // whitelist a test + s << BaselineHandler::blacklistTest(cgiUrl.queryItemValue(QLS("context")), + cgiUrl.queryItemValue(QLS("itemId")), true); + } else { + s << "Unknown query:
" << query << "
"; + } + s << "

Back to report"; + s << ""; +} diff --git a/tests/baselineserver/src/report.h b/tests/baselineserver/src/report.h new file mode 100644 index 0000000000..d21102d32f --- /dev/null +++ b/tests/baselineserver/src/report.h @@ -0,0 +1,91 @@ +/**************************************************************************** +** +** Copyright (C) 2011 Nokia Corporation and/or its subsidiary(-ies). +** All rights reserved. +** Contact: Nokia Corporation (qt-info@nokia.com) +** +** This file is part of the test suite of the Qt Toolkit. +** +** $QT_BEGIN_LICENSE:LGPL$ +** GNU Lesser General Public License Usage +** This file may be used under the terms of the GNU Lesser General Public +** License version 2.1 as published by the Free Software Foundation and +** appearing in the file LICENSE.LGPL included in the packaging of this +** file. Please review the following information to ensure the GNU Lesser +** General Public License version 2.1 requirements will be met: +** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. +** +** In addition, as a special exception, Nokia gives you certain additional +** rights. These rights are described in the Nokia Qt LGPL Exception +** version 1.1, included in the file LGPL_EXCEPTION.txt in this package. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU General +** Public License version 3.0 as published by the Free Software Foundation +** and appearing in the file LICENSE.GPL included in the packaging of this +** file. Please review the following information to ensure the GNU General +** Public License version 3.0 requirements will be met: +** http://www.gnu.org/copyleft/gpl.html. +** +** Other Usage +** Alternatively, this file may be used in accordance with the terms and +** conditions contained in a signed written agreement between you and Nokia. +** +** +** +** +** +** $QT_END_LICENSE$ +** +****************************************************************************/ +#ifndef REPORT_H +#define REPORT_H + +#include "baselineprotocol.h" +#include +#include +#include +#include + +class BaselineHandler; + +class Report +{ +public: + Report(); + ~Report(); + + void init(const BaselineHandler *h, const QString &r, const PlatformInfo &p); + void addItems(const ImageItemList& items); + void addMismatch(const ImageItem& item); + void end(); + + QString filePath(); + + static void handleCGIQuery(const QString &query); + +private: + void write(); + void writeFunctionResults(const ImageItemList &list); + void writeItem(const QString &baseline, const QString &rendered, const ImageItem &item, + const QString &itemFile, const QString &ctx, const QString &misCtx, const QString &metadata); + void writeHeader(); + void writeFooter(); + QString generateCompared(const QString &baseline, const QString &rendered, bool fuzzy = false); + QString generateThumbnail(const QString &image); + + const BaselineHandler *handler; + QString runId; + PlatformInfo plat; + QString rootDir; + QString reportDir; + QString path; + QStringList testFunctions; + QMap itemLists; + bool written; + int numItems; + int numMismatches; + QTextStream out; +}; + +#endif // REPORT_H diff --git a/tests/baselineserver/src/templates/view.html b/tests/baselineserver/src/templates/view.html new file mode 100644 index 0000000000..c048f4781c --- /dev/null +++ b/tests/baselineserver/src/templates/view.html @@ -0,0 +1,79 @@ +

Lancelot Viewer

+ +

+Zoom: +1x +2x +4x +

+ +

+ + + + + + + + + + + + +
Baseline%1
Rendered%2
Differences

+ + +

+ +

+ + -- cgit v1.2.3