diff options
Diffstat (limited to 'tests/baselineserver/src/report.cpp')
| -rw-r--r-- | tests/baselineserver/src/report.cpp | 270 |
1 files changed, 229 insertions, 41 deletions
diff --git a/tests/baselineserver/src/report.cpp b/tests/baselineserver/src/report.cpp index d29f22e2a3..cd544a6c78 100644 --- a/tests/baselineserver/src/report.cpp +++ b/tests/baselineserver/src/report.cpp @@ -44,9 +44,11 @@ #include <QDir> #include <QProcess> #include <QUrl> +#include <QXmlStreamWriter> Report::Report() - : written(false), numItems(0), numMismatches(0) + : initialized(false), handler(0), written(false), numItems(0), numMismatches(0), settings(0), + hasStats(false) { } @@ -60,19 +62,31 @@ QString Report::filePath() return path; } -void Report::init(const BaselineHandler *h, const QString &r, const PlatformInfo &p) +int Report::numberOfMismatches() +{ + return numMismatches; +} + +bool Report::reportProduced() +{ + return written; +} + +void Report::init(const BaselineHandler *h, const QString &r, const PlatformInfo &p, const QSettings *s) { handler = h; runId = r; plat = p; + settings = s; rootDir = BaselineServer::storagePath() + QLC('/'); - reportDir = plat.value(PI_TestCase) + QLC('/') + (plat.isAdHocRun() ? QLS("reports/adhoc/") : QLS("reports/pulse/")); - QString dir = rootDir + reportDir; + baseDir = handler->pathForItem(ImageItem(), true, false).remove(QRegExp("/baselines/.*$")); + QString dir = baseDir + (plat.isAdHocRun() ? QLS("/adhoc-reports") : QLS("/auto-reports")); QDir cwd; - if (!cwd.exists(dir)) - cwd.mkpath(dir); - path = reportDir + QLS("Report_") + runId + QLS(".html"); + if (!cwd.exists(rootDir + dir)) + cwd.mkpath(rootDir + dir); + path = dir + QLS("/Report_") + runId + QLS(".html"); hasOverride = !plat.overrides().isEmpty(); + initialized = true; } void Report::addItems(const ImageItemList &items) @@ -83,38 +97,110 @@ void Report::addItems(const ImageItemList &items) QString func = items.at(0).testFunction; if (!testFunctions.contains(func)) testFunctions.append(func); - itemLists[func] += items; + ImageItemList list = items; + if (settings->value("ReportMissingResults").toBool()) { + for (ImageItemList::iterator it = list.begin(); it != list.end(); ++it) { + if (it->status == ImageItem::Ok) + it->status = ImageItem::Error; // Status should be set by report from client, else report as error + } + } + itemLists[func] += list; } -void Report::addMismatch(const ImageItem &item) +void Report::addResult(const ImageItem &item) { if (!testFunctions.contains(item.testFunction)) { - qWarning() << "Report::addMismatch: unknown testfunction" << item.testFunction; + qWarning() << "Report::addResult: unknown testfunction" << item.testFunction; return; } bool found = false; ImageItemList &list = itemLists[item.testFunction]; for (ImageItemList::iterator it = list.begin(); it != list.end(); ++it) { if (it->itemName == item.itemName && it->itemChecksum == item.itemChecksum) { - it->status = ImageItem::Mismatch; + it->status = item.status; found = true; break; } } - if (found) - numMismatches++; - else - qWarning() << "Report::addMismatch: unknown item" << item.itemName << "in testfunction" << item.testFunction; + if (found) { + if (item.status == ImageItem::Mismatch) + numMismatches++; + } else { + qWarning() << "Report::addResult: unknown item" << item.itemName << "in testfunction" << item.testFunction; + } } void Report::end() { - if (written || !numMismatches) + if (!initialized || written) + return; + // Make report iff (#mismatches>0) || (#fuzzymatches>0) || (#errors>0 && settings say report errors) + bool doReport = (numMismatches > 0); + if (!doReport) { + bool reportErrors = settings->value("ReportMissingResults").toBool(); + computeStats(); + foreach (const QString &func, itemLists.keys()) { + FuncStats stat = stats.value(func); + if (stat.value(ImageItem::FuzzyMatch) > 0) { + doReport = true; + break; + } + foreach (const ImageItem &item, itemLists.value(func)) { + if (reportErrors && item.status == ImageItem::Error) { + doReport = true; + break; + } + } + if (doReport) + break; + } + } + if (!doReport) return; write(); written = true; } +void Report::computeStats() +{ + if (hasStats) + return; + foreach (const QString &func, itemLists.keys()) { + FuncStats funcStat; + funcStat[ImageItem::Ok] = 0; + funcStat[ImageItem::BaselineNotFound] = 0; + funcStat[ImageItem::IgnoreItem] = 0; + funcStat[ImageItem::Mismatch] = 0; + funcStat[ImageItem::FuzzyMatch] = 0; + funcStat[ImageItem::Error] = 0; + foreach (const ImageItem &item, itemLists.value(func)) { + funcStat[item.status]++; + } + stats[func] = funcStat; + } + hasStats = true; +} + +QString Report::summary() +{ + computeStats(); + QString res; + foreach (const QString &func, itemLists.keys()) { + FuncStats stat = stats.value(func); + QString s = QString("%1 %3 mismatch(es), %4 error(s), %5 fuzzy match(es)\n"); + s = s.arg(QString("%1() [%2 items]:").arg(func).arg(itemLists.value(func).size()).leftJustified(40)); + s = s.arg(stat.value(ImageItem::Mismatch)); + s = s.arg(stat.value(ImageItem::Error)); + s = s.arg(stat.value(ImageItem::FuzzyMatch)); + res += s; + } +#if 0 + qDebug() << "***************************** Summary *************************"; + qDebug() << res; + qDebug() << "***************************************************************"; +#endif + return res; +} void Report::write() { @@ -131,24 +217,27 @@ void Report::write() } writeFooter(); file.close(); + updateLatestPointer(); } void Report::writeHeader() { - QString title = plat.value(PI_TestCase) + QLS(" Qt Baseline Test Report"); - out << "<head><title>" << title << "</title></head>\n" - << "<html><body><h1>" << title << "</h1>\n" + QString title = plat.value(PI_Project) + QLC(':') + plat.value(PI_TestCase) + QLS(" Lancelot Test Report"); + out << "<!DOCTYPE html>\n" + << "<html><head><title>" << title << "</title></head>\n" + << "<body bgcolor=""#ddeeff""><h1>" << title << "</h1>\n" << "<p>Note: This is a <i>static</i> page, generated at " << QDateTime::currentDateTime().toString() << " for the test run with id " << runId << "</p>\n" - << "<p>Summary: <b><span style=\"color:red\">" << numMismatches << " of " << numItems << "</b></span> items reported mismatching</p>\n\n"; + << "<p>Summary: <b><span style=\"color:red\">" << numMismatches << " of " << numItems << "</span></b> items reported mismatching</p>\n"; + out << "<pre>\n" << summary() << "</pre>\n\n"; out << "<h3>Testing Client Platform Info:</h3>\n" << "<table>\n"; foreach (QString key, plat.keys()) out << "<tr><td>" << key << ":</td><td>" << plat.value(key) << "</td></tr>\n"; out << "</table>\n\n"; if (hasOverride) { - out << "<span style=\"color:red\"><h4>Note! Platform Override Info:</h4></span>\n" + out << "<span style=\"color:red\"><h4>Note! Override Platform Info:</h4></span>\n" << "<p>The client's output has been compared to baselines created on a different platform. Differences:</p>\n" << "<table>\n"; for (int i = 0; i < plat.overrides().size()-1; i+=2) @@ -184,14 +273,25 @@ void Report::writeFunctionResults(const ImageItemList &list) "</tr>\n\n"; foreach (const ImageItem &item, list) { + QString mmPrefix = handler->pathForItem(item, false, false); + QString blPrefix = handler->pathForItem(item, true, false); + + // Make hard links to the current baseline, so that the report is static even if the baseline changes + generateThumbnail(blPrefix + QLS(FileFormat), rootDir); // Make sure baseline thumbnail is up to date + QString lnPrefix = mmPrefix + QLS("baseline."); + QByteArray blPrefixBa = (rootDir + blPrefix).toLatin1(); + QByteArray lnPrefixBa = (rootDir + lnPrefix).toLatin1(); + ::link((blPrefixBa + FileFormat).constData(), (lnPrefixBa + FileFormat).constData()); + ::link((blPrefixBa + MetadataFileExt).constData(), (lnPrefixBa + MetadataFileExt).constData()); + ::link((blPrefixBa + ThumbnailExt).constData(), (lnPrefixBa + ThumbnailExt).constData()); + + QString baseline = lnPrefix + QLS(FileFormat); + QString metadata = lnPrefix + QLS(MetadataFileExt); out << "<tr>\n"; out << "<td>" << item.itemName << "</td>\n"; - QString prefix = handler->pathForItem(item, true, false); - QString baseline = prefix + QLS(FileFormat); - QString metadata = prefix + QLS(MetadataFileExt); - if (item.status == ImageItem::Mismatch) { - QString rendered = handler->pathForItem(item, false, false) + QLS(FileFormat); - QString itemFile = prefix.section(QLC('/'), -1); + if (item.status == ImageItem::Mismatch || item.status == ImageItem::FuzzyMatch) { + QString rendered = mmPrefix + QLS(FileFormat); + QString itemFile = mmPrefix.section(QLC('/'), -1); writeItem(baseline, rendered, item, itemFile, ctx, misCtx, metadata); } else { @@ -210,6 +310,9 @@ void Report::writeFunctionResults(const ImageItemList &list) << "\">Whitelist this item</a>"; } break; + case ImageItem::Error: + out << "<span style=\"background-color:red\">Error: No result reported!</span>"; + break; case ImageItem::Ok: out << "<span style=\"color:green\"><small>No mismatch reported</small></span>"; break; @@ -233,11 +336,14 @@ void Report::writeItem(const QString &baseline, const QString &rendered, const I QStringList images = QStringList() << baseline << rendered << compared; foreach (const QString& img, images) - out << "<td height=246 align=center><a href=\"/" << img << "\"><img src=\"/" << generateThumbnail(img) << "\"></a></td>\n"; + out << "<td height=246 align=center><a href=\"/" << img << "\"><img src=\"/" << generateThumbnail(img, rootDir) << "\"></a></td>\n"; - out << "<td align=center>\n" - << "<p><span style=\"color:red\">Mismatch reported</span></p>\n" - << "<p><a href=\"/" << metadata << "\">Baseline Info</a>\n"; + out << "<td align=center>\n"; + if (item.status == ImageItem::FuzzyMatch) + out << "<p><span style=\"color:orange\">Fuzzy match</span></p>\n"; + else + out << "<p><span style=\"color:red\">Mismatch reported</span></p>\n"; + out << "<p><a href=\"/" << metadata << "\">Baseline Info</a>\n"; if (!hasOverride) { out << "<p><a href=\"/cgi-bin/server.cgi?cmd=updateSingleBaseline&context=" << ctx << "&mismatchContext=" << misCtx << "&itemFile=" << itemFile << "&url=" << pageUrl << "\">Let this be the new baseline</a></p>\n" @@ -245,8 +351,14 @@ void Report::writeItem(const QString &baseline, const QString &rendered, const I << "&itemId=" << item.itemName << "&url=" << pageUrl << "\">Blacklist this item</a></p>\n"; } out << "<p><a href=\"/cgi-bin/server.cgi?cmd=view&baseline=" << baseline << "&rendered=" << rendered - << "&compared=" << compared << "&url=" << pageUrl << "\">Inspect</a></p>\n" - << "</td>\n"; + << "&compared=" << compared << "&url=" << pageUrl << "\">Inspect</a></p>\n"; + +#if 0 + out << "<p><a href=\"/cgi-bin/server.cgi?cmd=diffstats&baseline=" << baseline << "&rendered=" << rendered + << "&url=" << pageUrl << "\">Diffstats</a></p>\n"; +#endif + + out << "</td>\n"; } void Report::writeFooter() @@ -259,8 +371,8 @@ QString Report::generateCompared(const QString &baseline, const QString &rendere { QString res = rendered; QFileInfo fi(res); - res.chop(fi.suffix().length() + 1); - res += QLS(fuzzy ? "_fuzzycompared.png" : "_compared.png"); + res.chop(fi.suffix().length()); + res += QLS(fuzzy ? "fuzzycompared.png" : "compared.png"); QStringList args; if (fuzzy) args << QLS("-fuzz") << QLS("5%"); @@ -270,12 +382,14 @@ QString Report::generateCompared(const QString &baseline, const QString &rendere } -QString Report::generateThumbnail(const QString &image) +QString Report::generateThumbnail(const QString &image, const QString &rootDir) { QString res = image; QFileInfo imgFI(rootDir+image); - res.chop(imgFI.suffix().length() + 1); - res += QLS("_thumbnail.jpg"); + if (!imgFI.exists()) + return res; + res.chop(imgFI.suffix().length()); + res += ThumbnailExt; QFileInfo resFI(rootDir+res); if (resFI.exists() && resFI.lastModified() > imgFI.lastModified()) return res; @@ -286,12 +400,80 @@ QString Report::generateThumbnail(const QString &image) } +QString Report::writeResultsXmlFiles() +{ + if (!itemLists.size()) + return QString(); + QString dir = rootDir + baseDir + QLS("/xml-reports/") + runId; + QDir cwd; + if (!cwd.exists(dir)) + cwd.mkpath(dir); + foreach (const QString &func, itemLists.keys()) { + QFile f(dir + "/" + func + "-results.xml"); + if (!f.open(QIODevice::WriteOnly)) + continue; + QXmlStreamWriter s(&f); + s.setAutoFormatting(true); + s.writeStartDocument(); + foreach (QString key, plat.keys()) { + QString cmt = " " + key + "=\"" + plat.value(key) +"\" "; + s.writeComment(cmt.replace("--", "[-]")); + } + s.writeStartElement("testsuite"); + s.writeAttribute("name", func); + foreach (const ImageItem &item, itemLists.value(func)) { + QString res; + switch (item.status) { + case ImageItem::Ok: + case ImageItem::FuzzyMatch: + res = "pass"; + break; + case ImageItem::Mismatch: + case ImageItem::Error: + res = "fail"; + break; + case ImageItem::BaselineNotFound: + case ImageItem::IgnoreItem: + default: + res = "skip"; + } + s.writeStartElement("testcase"); + s.writeAttribute("name", item.itemName); + s.writeAttribute("result", res); + s.writeEndElement(); + } + s.writeEndElement(); + s.writeEndDocument(); + } + return dir; +} + + + +void Report::updateLatestPointer() +{ + QString linkPath = rootDir + baseDir + QLS("/latest_report.html"); + QString reportPath = path.mid(baseDir.size()+1); + QFile::remove(linkPath); // possible race with another thread, yada yada yada + QFile::link(reportPath, linkPath); + +#if 0 + QByteArray fwd = "<!DOCTYPE html><html><head><meta HTTP-EQUIV=\"refresh\" CONTENT=\"0;URL=%1\"></meta></head><body></body></html>\n"; + fwd.replace("%1", filePath().prepend(QLC('/')).toLatin1()); + + QFile file(rootDir + baseDir + "/latest_report.html"); + if (file.open(QIODevice::WriteOnly | QIODevice::Truncate)) + file.write(fwd); +#endif +} + + void Report::handleCGIQuery(const QString &query) { QUrl cgiUrl(QLS("http://dummy/cgi-bin/dummy.cgi?") + query); QTextStream s(stdout); s << "Content-Type: text/html\r\n\r\n" - << "<HTML>"; + << "<!DOCTYPE html>\n<HTML>\n<body bgcolor=""#ddeeff"">\n"; // Lancelot blue QString command(cgiUrl.queryItemValue("cmd")); @@ -300,6 +482,12 @@ void Report::handleCGIQuery(const QString &query) cgiUrl.queryItemValue(QLS("rendered")), cgiUrl.queryItemValue(QLS("compared"))); } +#if 0 + else if (command == QLS("diffstats")) { + s << BaselineHandler::diffstats(cgiUrl.queryItemValue(QLS("baseline")), + cgiUrl.queryItemValue(QLS("rendered"))); + } +#endif else if (command == QLS("updateSingleBaseline")) { s << BaselineHandler::updateBaselines(cgiUrl.queryItemValue(QLS("context")), cgiUrl.queryItemValue(QLS("mismatchContext")), @@ -321,6 +509,6 @@ void Report::handleCGIQuery(const QString &query) } else { s << "Unknown query:<br>" << query << "<br>"; } - s << "<p><a href=\"" << cgiUrl.queryItemValue(QLS("url")) << "\">Back to report</a>"; - s << "</HTML>"; + s << "<p><a href=\"" << cgiUrl.queryItemValue(QLS("url")) << "\">Back to report</a>\n"; + s << "</body>\n</HTML>"; } |