/**************************************************************************** ** ** Copyright (C) 2013 Digia Plc and/or its subsidiary(-ies). ** Contact: http://www.qt-project.org/legal ** ** This file is part of the test suite of the Qt Toolkit. ** ** $QT_BEGIN_LICENSE:LGPL$ ** Commercial License Usage ** Licensees holding valid commercial Qt licenses may use this file in ** accordance with the commercial license agreement provided with the ** Software or, alternatively, in accordance with the terms contained in ** a written agreement between you and Digia. For licensing terms and ** conditions see http://qt.digia.com/licensing. For further information ** use the contact form at http://qt.digia.com/contact-us. ** ** GNU Lesser General Public License Usage ** Alternatively, this file may be used under the terms of the GNU Lesser ** General Public License version 2.1 as published by the Free Software ** Foundation and appearing in the file LICENSE.LGPL included in the ** packaging of this file. Please review the following information to ** ensure the GNU Lesser General Public License version 2.1 requirements ** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. ** ** In addition, as a special exception, Digia gives you certain additional ** rights. These rights are described in the Digia Qt LGPL Exception ** version 1.1, included in the file LGPL_EXCEPTION.txt in this package. ** ** GNU General Public License Usage ** Alternatively, this file may be used under the terms of the GNU ** General Public License version 3.0 as published by the Free Software ** Foundation and appearing in the file LICENSE.GPL included in the ** packaging of this file. Please review the following information to ** ensure the GNU General Public License version 3.0 requirements will be ** met: http://www.gnu.org/copyleft/gpl.html. ** ** ** $QT_END_LICENSE$ ** ****************************************************************************/ #include "report.h" #include "baselineprotocol.h" #include "baselineserver.h" #include #include #include #include Report::Report() : initialized(false), handler(0), written(false), numItems(0), numMismatches(0), settings(0), hasStats(false) { } Report::~Report() { end(); } QString Report::filePath() { return path; } int Report::numberOfMismatches() { return numMismatches; } bool Report::reportProduced() { return written; } void Report::init(const BaselineHandler *h, const QString &r, const PlatformInfo &p, const QSettings *s) { handler = h; runId = r; plat = p; settings = s; rootDir = BaselineServer::storagePath() + QLC('/'); baseDir = handler->pathForItem(ImageItem(), true, false).remove(QRegExp("/baselines/.*$")); QString dir = baseDir + (plat.isAdHocRun() ? QLS("/adhoc-reports") : QLS("/auto-reports")); QDir cwd; if (!cwd.exists(rootDir + dir)) cwd.mkpath(rootDir + dir); path = dir + QLS("/Report_") + runId + QLS(".html"); hasOverride = !plat.overrides().isEmpty(); initialized = true; } void Report::addItems(const ImageItemList &items) { if (items.isEmpty()) return; numItems += items.size(); QString func = items.at(0).testFunction; if (!testFunctions.contains(func)) testFunctions.append(func); ImageItemList list = items; if (settings->value("ReportMissingResults").toBool()) { for (ImageItemList::iterator it = list.begin(); it != list.end(); ++it) { if (it->status == ImageItem::Ok) it->status = ImageItem::Error; // Status should be set by report from client, else report as error } } itemLists[func] += list; } void Report::addResult(const ImageItem &item) { if (!testFunctions.contains(item.testFunction)) { qWarning() << "Report::addResult: unknown testfunction" << item.testFunction; return; } bool found = false; ImageItemList &list = itemLists[item.testFunction]; for (ImageItemList::iterator it = list.begin(); it != list.end(); ++it) { if (it->itemName == item.itemName && it->itemChecksum == item.itemChecksum) { it->status = item.status; found = true; break; } } if (found) { if (item.status == ImageItem::Mismatch) numMismatches++; } else { qWarning() << "Report::addResult: unknown item" << item.itemName << "in testfunction" << item.testFunction; } } void Report::end() { if (!initialized || written) return; // Make report iff (#mismatches>0) || (#fuzzymatches>0) || (#errors>0 && settings say report errors) bool doReport = (numMismatches > 0); if (!doReport) { bool reportErrors = settings->value("ReportMissingResults").toBool(); computeStats(); foreach (const QString &func, itemLists.keys()) { FuncStats stat = stats.value(func); if (stat.value(ImageItem::FuzzyMatch) > 0) { doReport = true; break; } foreach (const ImageItem &item, itemLists.value(func)) { if (reportErrors && item.status == ImageItem::Error) { doReport = true; break; } } if (doReport) break; } } if (!doReport) return; write(); written = true; } void Report::computeStats() { if (hasStats) return; foreach (const QString &func, itemLists.keys()) { FuncStats funcStat; funcStat[ImageItem::Ok] = 0; funcStat[ImageItem::BaselineNotFound] = 0; funcStat[ImageItem::IgnoreItem] = 0; funcStat[ImageItem::Mismatch] = 0; funcStat[ImageItem::FuzzyMatch] = 0; funcStat[ImageItem::Error] = 0; foreach (const ImageItem &item, itemLists.value(func)) { funcStat[item.status]++; } stats[func] = funcStat; } hasStats = true; } QString Report::summary() { computeStats(); QString res; foreach (const QString &func, itemLists.keys()) { FuncStats stat = stats.value(func); QString s = QString("%1 %3 mismatch(es), %4 error(s), %5 fuzzy match(es)\n"); s = s.arg(QString("%1() [%2 items]:").arg(func).arg(itemLists.value(func).size()).leftJustified(40)); s = s.arg(stat.value(ImageItem::Mismatch)); s = s.arg(stat.value(ImageItem::Error)); s = s.arg(stat.value(ImageItem::FuzzyMatch)); res += s; } #if 0 qDebug() << "***************************** Summary *************************"; qDebug() << res; qDebug() << "***************************************************************"; #endif return res; } void Report::write() { QFile file(rootDir + path); if (!file.open(QIODevice::WriteOnly | QIODevice::Truncate)) { qWarning() << "Failed to open report file" << file.fileName(); return; } out.setDevice(&file); writeHeader(); foreach(const QString &func, testFunctions) { writeFunctionResults(itemLists.value(func)); } writeFooter(); file.close(); updateLatestPointer(); } void Report::writeHeader() { QString title = plat.value(PI_Project) + QLC(':') + plat.value(PI_TestCase) + QLS(" Lancelot Test Report"); out << "\n" << "" << title << "\n" << "

" << title << "

\n" << "

Note: This is a static page, generated at " << QDateTime::currentDateTime().toString() << " for the test run with id " << runId << "

\n" << "

Summary: " << numMismatches << " of " << numItems << " items reported mismatching

\n"; out << "
\n" << summary() << "
\n\n"; out << "

Testing Client Platform Info:

\n" << "\n"; foreach (QString key, plat.keys()) out << "\n"; out << "
" << key << ":" << plat.value(key) << "
\n\n"; if (hasOverride) { out << "

Note! Override Platform Info:

\n" << "

The client's output has been compared to baselines created on a different platform. Differences:

\n" << "\n"; for (int i = 0; i < plat.overrides().size()-1; i+=2) out << "\n"; out << "
" << plat.overrides().at(i) << ":" << plat.overrides().at(i+1) << "
\n\n"; } } void Report::writeFunctionResults(const ImageItemList &list) { QString testFunction = list.at(0).testFunction; QString pageUrl = BaselineServer::baseUrl() + path; QString ctx = handler->pathForItem(list.at(0), true, false).section(QLC('/'), 0, -2); QString misCtx = handler->pathForItem(list.at(0), false, false).section(QLC('/'), 0, -2); out << "\n

 

Test function: " << testFunction << "

\n"; if (!hasOverride) { out << "

Clear all baselines for this testfunction (They will be recreated by the next run)

\n"; out << "

Let these mismatching images be the new baselines for this testfunction

\n\n"; } out << "\n" "\n" "\n" "\n" "\n" "\n" "\n" "\n\n"; foreach (const ImageItem &item, list) { QString mmPrefix = handler->pathForItem(item, false, false); QString blPrefix = handler->pathForItem(item, true, false); // Make hard links to the current baseline, so that the report is static even if the baseline changes generateThumbnail(blPrefix + QLS(FileFormat), rootDir); // Make sure baseline thumbnail is up to date QString lnPrefix = mmPrefix + QLS("baseline."); QByteArray blPrefixBa = (rootDir + blPrefix).toLatin1(); QByteArray lnPrefixBa = (rootDir + lnPrefix).toLatin1(); ::link((blPrefixBa + FileFormat).constData(), (lnPrefixBa + FileFormat).constData()); ::link((blPrefixBa + MetadataFileExt).constData(), (lnPrefixBa + MetadataFileExt).constData()); ::link((blPrefixBa + ThumbnailExt).constData(), (lnPrefixBa + ThumbnailExt).constData()); QString baseline = lnPrefix + QLS(FileFormat); QString metadata = lnPrefix + QLS(MetadataFileExt); out << "\n"; out << "\n"; if (item.status == ImageItem::Mismatch || item.status == ImageItem::FuzzyMatch) { QString rendered = mmPrefix + QLS(FileFormat); QString itemFile = mmPrefix.section(QLC('/'), -1); writeItem(baseline, rendered, item, itemFile, ctx, misCtx, metadata); } else { out << "\n" << "\n" << "\n"; } out << "\n\n"; } out << "
ItemBaselineRenderedComparison (diffs are RED)Info/Action
" << item.itemName << "image infon/a"; switch (item.status) { case ImageItem::BaselineNotFound: out << "Baseline not found/regenerated"; break; case ImageItem::IgnoreItem: out << "Blacklisted "; if (!hasOverride) { out << "Whitelist this item"; } break; case ImageItem::Error: out << "Error: No result reported!"; break; case ImageItem::Ok: out << "No mismatch reported"; break; default: out << "?"; break; } out << "
\n"; } void Report::writeItem(const QString &baseline, const QString &rendered, const ImageItem &item, const QString &itemFile, const QString &ctx, const QString &misCtx, const QString &metadata) { QString compared = generateCompared(baseline, rendered); QString pageUrl = BaselineServer::baseUrl() + path; QStringList images = QStringList() << baseline << rendered << compared; foreach (const QString& img, images) out << "\n"; out << "\n"; if (item.status == ImageItem::FuzzyMatch) out << "

Fuzzy match

\n"; else out << "

Mismatch reported

\n"; out << "

Baseline Info\n"; if (!hasOverride) { out << "

Let this be the new baseline

\n" << "

Blacklist this item

\n"; } out << "

Inspect

\n"; #if 0 out << "

Diffstats

\n"; #endif out << "\n"; } void Report::writeFooter() { out << "\n\n"; } QString Report::generateCompared(const QString &baseline, const QString &rendered, bool fuzzy) { QString res = rendered; QFileInfo fi(res); res.chop(fi.suffix().length()); res += QLS(fuzzy ? "fuzzycompared.png" : "compared.png"); QStringList args; if (fuzzy) args << QLS("-fuzz") << QLS("5%"); args << rootDir+baseline << rootDir+rendered << rootDir+res; QProcess::execute(QLS("compare"), args); return res; } QString Report::generateThumbnail(const QString &image, const QString &rootDir) { QString res = image; QFileInfo imgFI(rootDir+image); if (!imgFI.exists()) return res; res.chop(imgFI.suffix().length()); res += ThumbnailExt; QFileInfo resFI(rootDir+res); if (resFI.exists() && resFI.lastModified() > imgFI.lastModified()) return res; QStringList args; args << rootDir+image << QLS("-resize") << QLS("240x240>") << QLS("-quality") << QLS("50") << rootDir+res; QProcess::execute(QLS("convert"), args); return res; } QString Report::writeResultsXmlFiles() { if (!itemLists.size()) return QString(); QString dir = rootDir + baseDir + QLS("/xml-reports/") + runId; QDir cwd; if (!cwd.exists(dir)) cwd.mkpath(dir); foreach (const QString &func, itemLists.keys()) { QFile f(dir + "/" + func + "-results.xml"); if (!f.open(QIODevice::WriteOnly)) continue; QXmlStreamWriter s(&f); s.setAutoFormatting(true); s.writeStartDocument(); foreach (QString key, plat.keys()) { QString cmt = " " + key + "=\"" + plat.value(key) +"\" "; s.writeComment(cmt.replace("--", "[-]")); } s.writeStartElement("testsuite"); s.writeAttribute("name", func); foreach (const ImageItem &item, itemLists.value(func)) { QString res; switch (item.status) { case ImageItem::Ok: case ImageItem::FuzzyMatch: res = "pass"; break; case ImageItem::Mismatch: case ImageItem::Error: res = "fail"; break; case ImageItem::BaselineNotFound: case ImageItem::IgnoreItem: default: res = "skip"; } s.writeStartElement("testcase"); s.writeAttribute("name", item.itemName); s.writeAttribute("result", res); s.writeEndElement(); } s.writeEndElement(); s.writeEndDocument(); } return dir; } void Report::updateLatestPointer() { QString linkPath = rootDir + baseDir + QLS("/latest_report.html"); QString reportPath = path.mid(baseDir.size()+1); QFile::remove(linkPath); // possible race with another thread, yada yada yada QFile::link(reportPath, linkPath); #if 0 QByteArray fwd = "\n"; fwd.replace("%1", filePath().prepend(QLC('/')).toLatin1()); QFile file(rootDir + baseDir + "/latest_report.html"); if (file.open(QIODevice::WriteOnly | QIODevice::Truncate)) file.write(fwd); #endif } void Report::handleCGIQuery(const QString &query) { QUrl cgiUrl(QLS("http://dummy/cgi-bin/dummy.cgi?") + query); QTextStream s(stdout); s << "Content-Type: text/html\r\n\r\n" << "\n\n\n"; // Lancelot blue QString command(cgiUrl.queryItemValue("cmd")); if (command == QLS("view")) { s << BaselineHandler::view(cgiUrl.queryItemValue(QLS("baseline")), cgiUrl.queryItemValue(QLS("rendered")), cgiUrl.queryItemValue(QLS("compared"))); } #if 0 else if (command == QLS("diffstats")) { s << BaselineHandler::diffstats(cgiUrl.queryItemValue(QLS("baseline")), cgiUrl.queryItemValue(QLS("rendered"))); } #endif else if (command == QLS("updateSingleBaseline")) { s << BaselineHandler::updateBaselines(cgiUrl.queryItemValue(QLS("context")), cgiUrl.queryItemValue(QLS("mismatchContext")), cgiUrl.queryItemValue(QLS("itemFile"))); } else if (command == QLS("updateAllBaselines")) { s << BaselineHandler::updateBaselines(cgiUrl.queryItemValue(QLS("context")), cgiUrl.queryItemValue(QLS("mismatchContext")), QString()); } else if (command == QLS("clearAllBaselines")) { s << BaselineHandler::clearAllBaselines(cgiUrl.queryItemValue(QLS("context"))); } else if (command == QLS("blacklist")) { // blacklist a test s << BaselineHandler::blacklistTest(cgiUrl.queryItemValue(QLS("context")), cgiUrl.queryItemValue(QLS("itemId"))); } else if (command == QLS("whitelist")) { // whitelist a test s << BaselineHandler::blacklistTest(cgiUrl.queryItemValue(QLS("context")), cgiUrl.queryItemValue(QLS("itemId")), true); } else { s << "Unknown query:
" << query << "
"; } s << "

Back to report\n"; s << "\n"; }