diff options
Diffstat (limited to 'tests/baselineserver/src/report.cpp')
| -rw-r--r-- | tests/baselineserver/src/report.cpp | 503 |
1 files changed, 0 insertions, 503 deletions
diff --git a/tests/baselineserver/src/report.cpp b/tests/baselineserver/src/report.cpp deleted file mode 100644 index 748d76ebfe..0000000000 --- a/tests/baselineserver/src/report.cpp +++ /dev/null @@ -1,503 +0,0 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the test suite of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:GPL-EXCEPT$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 3 as published by the Free Software -** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ -#include "report.h" -#include "baselineprotocol.h" -#include "baselineserver.h" -#include <QDir> -#include <QProcess> -#include <QUrl> -#include <QXmlStreamWriter> -#include <QRegularExpression> -#include <unistd.h> - -Report::Report() - : initialized(false), handler(0), written(false), numItems(0), numMismatches(0), settings(0), - hasStats(false) -{ -} - -Report::~Report() -{ - end(); -} - -QString Report::filePath() -{ - return path; -} - -int Report::numberOfMismatches() -{ - return numMismatches; -} - -bool Report::reportProduced() -{ - return written; -} - -void Report::init(const BaselineHandler *h, const QString &r, const PlatformInfo &p, const QSettings *s) -{ - handler = h; - runId = r; - plat = p; - settings = s; - rootDir = BaselineServer::storagePath() + QLC('/'); - baseDir = handler->pathForItem(ImageItem(), true, false).remove(QRegularExpression("/baselines/.*$")); - QString dir = baseDir + (plat.isAdHocRun() ? QLS("/adhoc-reports") : QLS("/auto-reports")); - QDir cwd; - if (!cwd.exists(rootDir + dir)) - cwd.mkpath(rootDir + dir); - path = dir + QLS("/Report_") + runId + QLS(".html"); - hasOverride = !plat.overrides().isEmpty(); - initialized = true; -} - -void Report::addItems(const ImageItemList &items) -{ - if (items.isEmpty()) - return; - numItems += items.size(); - QString func = items.at(0).testFunction; - if (!testFunctions.contains(func)) - testFunctions.append(func); - ImageItemList list = items; - if (settings->value("ReportMissingResults").toBool()) { - for (ImageItemList::iterator it = list.begin(); it != list.end(); ++it) { - if (it->status == ImageItem::Ok) - it->status = ImageItem::Error; // Status should be set by report from client, else report as error - } - } - itemLists[func] += list; -} - -void Report::addResult(const ImageItem &item) -{ - if (!testFunctions.contains(item.testFunction)) { - qWarning() << "Report::addResult: unknown testfunction" << item.testFunction; - return; - } - bool found = false; - ImageItemList &list = itemLists[item.testFunction]; - for (ImageItemList::iterator it = list.begin(); it != list.end(); ++it) { - if (it->itemName == item.itemName && it->itemChecksum == item.itemChecksum) { - it->status = item.status; - found = true; - break; - } - } - if (found) { - if (item.status == ImageItem::Mismatch) - numMismatches++; - } else { - qWarning() << "Report::addResult: unknown item" << item.itemName << "in testfunction" << item.testFunction; - } -} - -void Report::end() -{ - if (!initialized || written) - return; - // Make report iff (#mismatches>0) || (#fuzzymatches>0) || (#errors>0 && settings say report errors) - bool doReport = (numMismatches > 0); - if (!doReport) { - bool reportErrors = settings->value("ReportMissingResults").toBool(); - computeStats(); - foreach (const QString &func, itemLists.keys()) { - FuncStats stat = stats.value(func); - if (stat.value(ImageItem::FuzzyMatch) > 0) { - doReport = true; - break; - } - foreach (const ImageItem &item, itemLists.value(func)) { - if (reportErrors && item.status == ImageItem::Error) { - doReport = true; - break; - } - } - if (doReport) - break; - } - } - if (!doReport) - return; - write(); - written = true; -} - -void Report::computeStats() -{ - if (hasStats) - return; - foreach (const QString &func, itemLists.keys()) { - FuncStats funcStat; - funcStat[ImageItem::Ok] = 0; - funcStat[ImageItem::BaselineNotFound] = 0; - funcStat[ImageItem::IgnoreItem] = 0; - funcStat[ImageItem::Mismatch] = 0; - funcStat[ImageItem::FuzzyMatch] = 0; - funcStat[ImageItem::Error] = 0; - foreach (const ImageItem &item, itemLists.value(func)) { - funcStat[item.status]++; - } - stats[func] = funcStat; - } - hasStats = true; -} - -QString Report::summary() -{ - computeStats(); - QString res; - foreach (const QString &func, itemLists.keys()) { - FuncStats stat = stats.value(func); - QString s = QString("%1 %3 mismatch(es), %4 error(s), %5 fuzzy match(es)\n"); - s = s.arg(QString("%1() [%2 items]:").arg(func).arg(itemLists.value(func).size()).leftJustified(40)); - s = s.arg(stat.value(ImageItem::Mismatch)); - s = s.arg(stat.value(ImageItem::Error)); - s = s.arg(stat.value(ImageItem::FuzzyMatch)); - res += s; - } -#if 0 - qDebug() << "***************************** Summary *************************"; - qDebug() << res; - qDebug() << "***************************************************************"; -#endif - return res; -} - -void Report::write() -{ - QFile file(rootDir + path); - if (!file.open(QIODevice::WriteOnly | QIODevice::Truncate)) { - qWarning() << "Failed to open report file" << file.fileName(); - return; - } - out.setDevice(&file); - - writeHeader(); - foreach(const QString &func, testFunctions) { - writeFunctionResults(itemLists.value(func)); - } - writeFooter(); - file.close(); - updateLatestPointer(); -} - - -void Report::writeHeader() -{ - QString title = plat.value(PI_Project) + QLC(':') + plat.value(PI_TestCase) + QLS(" Lancelot Test Report"); - out << "<!DOCTYPE html>\n" - << "<html><head><title>" << title << "</title></head>\n" - << "<body bgcolor=""#ddeeff""><h1>" << title << "</h1>\n" - << "<p>Note: This is a <i>static</i> page, generated at " << QDateTime::currentDateTime().toString() - << " for the test run with id " << runId << "</p>\n" - << "<p>Summary: <b><span style=\"color:red\">" << numMismatches << " of " << numItems << "</span></b> items reported mismatching</p>\n"; - out << "<pre>\n" << summary() << "</pre>\n\n"; - out << "<h3>Testing Client Platform Info:</h3>\n" - << "<table>\n"; - foreach (QString key, plat.keys()) - out << "<tr><td>" << key << ":</td><td>" << plat.value(key) << "</td></tr>\n"; - out << "</table>\n\n"; - if (hasOverride) { - out << "<span style=\"color:red\"><h4>Note! Override Platform Info:</h4></span>\n" - << "<p>The client's output has been compared to baselines created on a different platform. Differences:</p>\n" - << "<table>\n"; - for (int i = 0; i < plat.overrides().size()-1; i+=2) - out << "<tr><td>" << plat.overrides().at(i) << ":</td><td>" << plat.overrides().at(i+1) << "</td></tr>\n"; - out << "</table>\n\n"; - } -} - - -void Report::writeFunctionResults(const ImageItemList &list) -{ - QString testFunction = list.at(0).testFunction; - QString pageUrl = BaselineServer::baseUrl() + path; - QString ctx = handler->pathForItem(list.at(0), true, false).section(QLC('/'), 0, -2); - QString misCtx = handler->pathForItem(list.at(0), false, false).section(QLC('/'), 0, -2); - - - out << "\n<p> </p><h3>Test function: " << testFunction << "</h3>\n"; - if (!hasOverride) { - out << "<p><a href=\"/cgi-bin/server.cgi?cmd=clearAllBaselines&context=" << ctx << "&url=" << pageUrl - << "\"><b>Clear all baselines</b></a> for this testfunction (They will be recreated by the next run)</p>\n"; - out << "<p><a href=\"/cgi-bin/server.cgi?cmd=updateAllBaselines&context=" << ctx << "&mismatchContext=" << misCtx << "&url=" << pageUrl - << "\"><b>Let these mismatching images be the new baselines</b></a> for this testfunction</p>\n\n"; - } - - out << "<table border=\"2\">\n" - "<tr>\n" - "<th width=123>Item</th>\n" - "<th width=246>Baseline</th>\n" - "<th width=246>Rendered</th>\n" - "<th width=246>Comparison (diffs are <span style=\"color:red\">RED</span>)</th>\n" - "<th width=246>Info/Action</th>\n" - "</tr>\n\n"; - - foreach (const ImageItem &item, list) { - QString mmPrefix = handler->pathForItem(item, false, false); - QString blPrefix = handler->pathForItem(item, true, false); - - // Make hard links to the current baseline, so that the report is static even if the baseline changes - generateThumbnail(blPrefix + QLS(FileFormat), rootDir); // Make sure baseline thumbnail is up to date - QString lnPrefix = mmPrefix + QLS("baseline."); - QByteArray blPrefixBa = (rootDir + blPrefix).toLatin1(); - QByteArray lnPrefixBa = (rootDir + lnPrefix).toLatin1(); - ::link((blPrefixBa + FileFormat).constData(), (lnPrefixBa + FileFormat).constData()); - ::link((blPrefixBa + MetadataFileExt).constData(), (lnPrefixBa + MetadataFileExt).constData()); - ::link((blPrefixBa + ThumbnailExt).constData(), (lnPrefixBa + ThumbnailExt).constData()); - - QString baseline = lnPrefix + QLS(FileFormat); - QString metadata = lnPrefix + QLS(MetadataFileExt); - out << "<tr>\n"; - out << "<td>" << item.itemName << "</td>\n"; - if (item.status == ImageItem::Mismatch || item.status == ImageItem::FuzzyMatch) { - QString rendered = mmPrefix + QLS(FileFormat); - QString itemFile = mmPrefix.section(QLC('/'), -1); - writeItem(baseline, rendered, item, itemFile, ctx, misCtx, metadata); - } - else { - out << "<td align=center><a href=\"/" << baseline << "\">image</a> <a href=\"/" << metadata << "\">info</a></td>\n" - << "<td align=center colspan=2><small>n/a</small></td>\n" - << "<td align=center>"; - switch (item.status) { - case ImageItem::BaselineNotFound: - out << "Baseline not found/regenerated"; - break; - case ImageItem::IgnoreItem: - out << "<span style=\"background-color:yellow\">Blacklisted</span> "; - if (!hasOverride) { - out << "<a href=\"/cgi-bin/server.cgi?cmd=whitelist&context=" << ctx - << "&itemId=" << item.itemName << "&url=" << pageUrl - << "\">Whitelist this item</a>"; - } - break; - case ImageItem::Error: - out << "<span style=\"background-color:red\">Error: No result reported!</span>"; - break; - case ImageItem::Ok: - out << "<span style=\"color:green\"><small>No mismatch reported</small></span>"; - break; - default: - out << '?'; - break; - } - out << "</td>\n"; - } - out << "</tr>\n\n"; - } - - out << "</table>\n"; -} - -void Report::writeItem(const QString &baseline, const QString &rendered, const ImageItem &item, - const QString &itemFile, const QString &ctx, const QString &misCtx, const QString &metadata) -{ - QString compared = generateCompared(baseline, rendered); - QString pageUrl = BaselineServer::baseUrl() + path; - - QStringList images = QStringList() << baseline << rendered << compared; - foreach (const QString& img, images) - out << "<td height=246 align=center><a href=\"/" << img << "\"><img src=\"/" << generateThumbnail(img, rootDir) << "\"></a></td>\n"; - - out << "<td align=center>\n"; - if (item.status == ImageItem::FuzzyMatch) - out << "<p><span style=\"color:orange\">Fuzzy match</span></p>\n"; - else - out << "<p><span style=\"color:red\">Mismatch reported</span></p>\n"; - out << "<p><a href=\"/" << metadata << "\">Baseline Info</a>\n"; - if (!hasOverride) { - out << "<p><a href=\"/cgi-bin/server.cgi?cmd=updateSingleBaseline&context=" << ctx << "&mismatchContext=" << misCtx - << "&itemFile=" << itemFile << "&url=" << pageUrl << "\">Let this be the new baseline</a></p>\n" - << "<p><a href=\"/cgi-bin/server.cgi?cmd=blacklist&context=" << ctx - << "&itemId=" << item.itemName << "&url=" << pageUrl << "\">Blacklist this item</a></p>\n"; - } - out << "<p><a href=\"/cgi-bin/server.cgi?cmd=view&baseline=" << baseline << "&rendered=" << rendered - << "&compared=" << compared << "&url=" << pageUrl << "\">Inspect</a></p>\n"; - -#if 0 - out << "<p><a href=\"/cgi-bin/server.cgi?cmd=diffstats&baseline=" << baseline << "&rendered=" << rendered - << "&url=" << pageUrl << "\">Diffstats</a></p>\n"; -#endif - - out << "</td>\n"; -} - -void Report::writeFooter() -{ - out << "\n</body></html>\n"; -} - - -QString Report::generateCompared(const QString &baseline, const QString &rendered, bool fuzzy) -{ - QString res = rendered; - QFileInfo fi(res); - res.chop(fi.suffix().length()); - res += QLS(fuzzy ? "fuzzycompared.png" : "compared.png"); - QStringList args; - if (fuzzy) - args << QLS("-fuzz") << QLS("5%"); - args << rootDir+baseline << rootDir+rendered << rootDir+res; - QProcess::execute(QLS("compare"), args); - return res; -} - - -QString Report::generateThumbnail(const QString &image, const QString &rootDir) -{ - QString res = image; - QFileInfo imgFI(rootDir+image); - if (!imgFI.exists()) - return res; - res.chop(imgFI.suffix().length()); - res += ThumbnailExt; - QFileInfo resFI(rootDir+res); - if (resFI.exists() && resFI.lastModified() > imgFI.lastModified()) - return res; - QStringList args; - args << rootDir+image << QLS("-resize") << QLS("240x240>") << QLS("-quality") << QLS("50") << rootDir+res; - QProcess::execute(QLS("convert"), args); - return res; -} - - -QString Report::writeResultsXmlFiles() -{ - if (!itemLists.size()) - return QString(); - QString dir = rootDir + baseDir + QLS("/xml-reports/") + runId; - QDir cwd; - if (!cwd.exists(dir)) - cwd.mkpath(dir); - foreach (const QString &func, itemLists.keys()) { - QFile f(dir + QLatin1Char('/') + func + "-results.xml"); - if (!f.open(QIODevice::WriteOnly)) - continue; - QXmlStreamWriter s(&f); - s.setAutoFormatting(true); - s.writeStartDocument(); - foreach (QString key, plat.keys()) { - QString cmt = QLatin1Char(' ') + key + "=\"" + plat.value(key) +"\" "; - s.writeComment(cmt.replace("--", "[-]")); - } - s.writeStartElement("testsuite"); - s.writeAttribute("name", func); - foreach (const ImageItem &item, itemLists.value(func)) { - QString res; - switch (item.status) { - case ImageItem::Ok: - case ImageItem::FuzzyMatch: - res = "pass"; - break; - case ImageItem::Mismatch: - case ImageItem::Error: - res = "fail"; - break; - case ImageItem::BaselineNotFound: - case ImageItem::IgnoreItem: - default: - res = "skip"; - } - s.writeStartElement("testcase"); - s.writeAttribute("name", item.itemName); - s.writeAttribute("result", res); - s.writeEndElement(); - } - s.writeEndElement(); - s.writeEndDocument(); - } - return dir; -} - - - -void Report::updateLatestPointer() -{ - QString linkPath = rootDir + baseDir + QLS("/latest_report.html"); - QString reportPath = path.mid(baseDir.size()+1); - QFile::remove(linkPath); // possible race with another thread, yada yada yada - QFile::link(reportPath, linkPath); - -#if 0 - QByteArray fwd = "<!DOCTYPE html><html><head><meta HTTP-EQUIV=\"refresh\" CONTENT=\"0;URL=%1\"></meta></head><body></body></html>\n"; - fwd.replace("%1", filePath().prepend(QLC('/')).toLatin1()); - - QFile file(rootDir + baseDir + "/latest_report.html"); - if (file.open(QIODevice::WriteOnly | QIODevice::Truncate)) - file.write(fwd); -#endif -} - - -void Report::handleCGIQuery(const QString &query) -{ - QUrl cgiUrl(QLS("http://dummy/cgi-bin/dummy.cgi?") + query); - QTextStream s(stdout); - s << "Content-Type: text/html\r\n\r\n" - << "<!DOCTYPE html>\n<HTML>\n<body bgcolor=""#ddeeff"">\n"; // Lancelot blue - - QString command(cgiUrl.queryItemValue("cmd")); - - if (command == QLS("view")) { - s << BaselineHandler::view(cgiUrl.queryItemValue(QLS("baseline")), - cgiUrl.queryItemValue(QLS("rendered")), - cgiUrl.queryItemValue(QLS("compared"))); - } -#if 0 - else if (command == QLS("diffstats")) { - s << BaselineHandler::diffstats(cgiUrl.queryItemValue(QLS("baseline")), - cgiUrl.queryItemValue(QLS("rendered"))); - } -#endif - else if (command == QLS("updateSingleBaseline")) { - s << BaselineHandler::updateBaselines(cgiUrl.queryItemValue(QLS("context")), - cgiUrl.queryItemValue(QLS("mismatchContext")), - cgiUrl.queryItemValue(QLS("itemFile"))); - } else if (command == QLS("updateAllBaselines")) { - s << BaselineHandler::updateBaselines(cgiUrl.queryItemValue(QLS("context")), - cgiUrl.queryItemValue(QLS("mismatchContext")), - QString()); - } else if (command == QLS("clearAllBaselines")) { - s << BaselineHandler::clearAllBaselines(cgiUrl.queryItemValue(QLS("context"))); - } else if (command == QLS("blacklist")) { - // blacklist a test - s << BaselineHandler::blacklistTest(cgiUrl.queryItemValue(QLS("context")), - cgiUrl.queryItemValue(QLS("itemId"))); - } else if (command == QLS("whitelist")) { - // whitelist a test - s << BaselineHandler::blacklistTest(cgiUrl.queryItemValue(QLS("context")), - cgiUrl.queryItemValue(QLS("itemId")), true); - } else { - s << "Unknown query:<br>" << query << "<br>"; - } - s << "<p><a href=\"" << cgiUrl.queryItemValue(QLS("url")) << "\">Back to report</a>\n"; - s << "</body>\n</HTML>"; -} |