Misc. updates to the lancelot autotest framework

Moving more logic into the protocol and framework, easening
the burden on the autotest implementation.
Implementing several new features in the server and
report, like fuzzy matching and static baselines.

Change-Id: Iaf070918195ae05767808a548f019d09d9d5f8c0
Reviewed-by: Paul Olav Tvete <paul.tvete@digia.com>

1 files changed, 229 insertions, 41 deletions

diff --git a/tests/baselineserver/src/report.cpp b/tests/baselineserver/src/report.cpp
index d29f22e2a3..cd544a6c78 100644
--- a/tests/baselineserver/src/report.cpp
+++ b/tests/baselineserver/src/report.cpp
@@ -44,9 +44,11 @@
 #include <QDir>
 #include <QProcess>
 #include <QUrl>
+#include <QXmlStreamWriter>
 
 Report::Report()
-    : written(false), numItems(0), numMismatches(0)
+    : initialized(false), handler(0), written(false), numItems(0), numMismatches(0), settings(0),
+      hasStats(false)
 {
 }
 
@@ -60,19 +62,31 @@ QString Report::filePath()
     return path;
 }
 
-void Report::init(const BaselineHandler *h, const QString &r, const PlatformInfo &p)
+int Report::numberOfMismatches()
+{
+    return numMismatches;
+}
+
+bool Report::reportProduced()
+{
+    return written;
+}
+
+void Report::init(const BaselineHandler *h, const QString &r, const PlatformInfo &p, const QSettings *s)
 {
     handler = h;
     runId = r;
     plat = p;
+    settings = s;
     rootDir = BaselineServer::storagePath() + QLC('/');
-    reportDir = plat.value(PI_TestCase) + QLC('/') + (plat.isAdHocRun() ? QLS("reports/adhoc/") : QLS("reports/pulse/"));
-    QString dir = rootDir + reportDir;
+    baseDir = handler->pathForItem(ImageItem(), true, false).remove(QRegExp("/baselines/.*$"));
+    QString dir = baseDir + (plat.isAdHocRun() ? QLS("/adhoc-reports") : QLS("/auto-reports"));
     QDir cwd;
-    if (!cwd.exists(dir))
-        cwd.mkpath(dir);
-    path = reportDir + QLS("Report_") + runId + QLS(".html");
+    if (!cwd.exists(rootDir + dir))
+        cwd.mkpath(rootDir + dir);
+    path = dir + QLS("/Report_") + runId + QLS(".html");
     hasOverride = !plat.overrides().isEmpty();
+    initialized = true;
 }
 
 void Report::addItems(const ImageItemList &items)
@@ -83,38 +97,110 @@ void Report::addItems(const ImageItemList &items)
     QString func = items.at(0).testFunction;
     if (!testFunctions.contains(func))
         testFunctions.append(func);
-    itemLists[func] += items;
+    ImageItemList list = items;
+    if (settings->value("ReportMissingResults").toBool()) {
+        for (ImageItemList::iterator it = list.begin(); it != list.end(); ++it) {
+            if (it->status == ImageItem::Ok)
+                it->status = ImageItem::Error;   // Status should be set by report from client, else report as error
+        }
+    }
+    itemLists[func] += list;
 }
 
-void Report::addMismatch(const ImageItem &item)
+void Report::addResult(const ImageItem &item)
 {
     if (!testFunctions.contains(item.testFunction)) {
-        qWarning() << "Report::addMismatch: unknown testfunction" << item.testFunction;
+        qWarning() << "Report::addResult: unknown testfunction" << item.testFunction;
         return;
     }
     bool found = false;
     ImageItemList &list = itemLists[item.testFunction];
     for (ImageItemList::iterator it = list.begin(); it != list.end(); ++it) {
         if (it->itemName == item.itemName && it->itemChecksum == item.itemChecksum) {
-            it->status = ImageItem::Mismatch;
+            it->status = item.status;
             found = true;
             break;
         }
     }
-    if (found)
-        numMismatches++;
-    else
-        qWarning() << "Report::addMismatch: unknown item" << item.itemName << "in testfunction" << item.testFunction;
+    if (found) {
+        if (item.status == ImageItem::Mismatch)
+            numMismatches++;
+    } else {
+        qWarning() << "Report::addResult: unknown item" << item.itemName << "in testfunction" << item.testFunction;
+    }
 }
 
 void Report::end()
 {
-    if (written || !numMismatches)
+    if (!initialized || written)
+        return;
+    // Make report iff (#mismatches>0) || (#fuzzymatches>0) || (#errors>0 && settings say report errors)
+    bool doReport = (numMismatches > 0);
+    if (!doReport) {
+        bool reportErrors = settings->value("ReportMissingResults").toBool();
+        computeStats();
+        foreach (const QString &func, itemLists.keys()) {
+            FuncStats stat = stats.value(func);
+            if (stat.value(ImageItem::FuzzyMatch) > 0) {
+                doReport = true;
+                break;
+            }
+            foreach (const ImageItem &item, itemLists.value(func)) {
+                if (reportErrors && item.status == ImageItem::Error) {
+                    doReport = true;
+                    break;
+                }
+            }
+            if (doReport)
+                break;
+        }
+    }
+    if (!doReport)
         return;
     write();
     written = true;
 }
 
+void Report::computeStats()
+{
+    if (hasStats)
+        return;
+    foreach (const QString &func, itemLists.keys()) {
+        FuncStats funcStat;
+        funcStat[ImageItem::Ok] = 0;
+        funcStat[ImageItem::BaselineNotFound] = 0;
+        funcStat[ImageItem::IgnoreItem] = 0;
+        funcStat[ImageItem::Mismatch] = 0;
+        funcStat[ImageItem::FuzzyMatch] = 0;
+        funcStat[ImageItem::Error] = 0;
+        foreach (const ImageItem &item, itemLists.value(func)) {
+            funcStat[item.status]++;
+        }
+        stats[func] = funcStat;
+    }
+    hasStats = true;
+}
+
+QString Report::summary()
+{
+    computeStats();
+    QString res;
+    foreach (const QString &func, itemLists.keys()) {
+        FuncStats stat = stats.value(func);
+        QString s = QString("%1 %3 mismatch(es), %4 error(s), %5 fuzzy match(es)\n");
+        s = s.arg(QString("%1() [%2 items]:").arg(func).arg(itemLists.value(func).size()).leftJustified(40));
+        s = s.arg(stat.value(ImageItem::Mismatch));
+        s = s.arg(stat.value(ImageItem::Error));
+        s = s.arg(stat.value(ImageItem::FuzzyMatch));
+        res += s;
+    }
+#if 0
+    qDebug() << "***************************** Summary *************************";
+    qDebug() << res;
+    qDebug() << "***************************************************************";
+#endif
+    return res;
+}
 
 void Report::write()
 {
@@ -131,24 +217,27 @@ void Report::write()
     }
     writeFooter();
     file.close();
+    updateLatestPointer();
 }
 
 
 void Report::writeHeader()
 {
-    QString title = plat.value(PI_TestCase) + QLS(" Qt Baseline Test Report");
-    out << "<head><title>" << title << "</title></head>\n"
-        << "<html><body><h1>" << title << "</h1>\n"
+    QString title = plat.value(PI_Project) + QLC(':') + plat.value(PI_TestCase) + QLS(" Lancelot Test Report");
+    out << "<!DOCTYPE html>\n"
+        << "<html><head><title>" << title << "</title></head>\n"
+        << "<body bgcolor=""#ddeeff""><h1>" << title << "</h1>\n"
         << "<p>Note: This is a <i>static</i> page, generated at " << QDateTime::currentDateTime().toString()
         << " for the test run with id " << runId << "</p>\n"
-        << "<p>Summary: <b><span style=\"color:red\">" << numMismatches << " of " << numItems << "</b></span> items reported mismatching</p>\n\n";
+        << "<p>Summary: <b><span style=\"color:red\">" << numMismatches << " of " << numItems << "</span></b> items reported mismatching</p>\n";
+    out << "<pre>\n" << summary() << "</pre>\n\n";
     out << "<h3>Testing Client Platform Info:</h3>\n"
         << "<table>\n";
     foreach (QString key, plat.keys())
         out << "<tr><td>" << key << ":</td><td>" << plat.value(key) << "</td></tr>\n";
     out << "</table>\n\n";
     if (hasOverride) {
-        out << "<span style=\"color:red\"><h4>Note! Platform Override Info:</h4></span>\n"
+        out << "<span style=\"color:red\"><h4>Note! Override Platform Info:</h4></span>\n"
             << "<p>The client's output has been compared to baselines created on a different platform. Differences:</p>\n"
             << "<table>\n";
         for (int i = 0; i < plat.overrides().size()-1; i+=2)
@@ -184,14 +273,25 @@ void Report::writeFunctionResults(const ImageItemList &list)
            "</tr>\n\n";
 
     foreach (const ImageItem &item, list) {
+        QString mmPrefix = handler->pathForItem(item, false, false);
+        QString blPrefix = handler->pathForItem(item, true, false);
+
+        // Make hard links to the current baseline, so that the report is static even if the baseline changes
+        generateThumbnail(blPrefix + QLS(FileFormat), rootDir);   // Make sure baseline thumbnail is up to date
+        QString lnPrefix = mmPrefix + QLS("baseline.");
+        QByteArray blPrefixBa = (rootDir + blPrefix).toLatin1();
+        QByteArray lnPrefixBa = (rootDir + lnPrefix).toLatin1();
+        ::link((blPrefixBa + FileFormat).constData(), (lnPrefixBa + FileFormat).constData());
+        ::link((blPrefixBa + MetadataFileExt).constData(), (lnPrefixBa + MetadataFileExt).constData());
+        ::link((blPrefixBa + ThumbnailExt).constData(), (lnPrefixBa + ThumbnailExt).constData());
+
+        QString baseline = lnPrefix + QLS(FileFormat);
+        QString metadata = lnPrefix + QLS(MetadataFileExt);
         out << "<tr>\n";
         out << "<td>" << item.itemName << "</td>\n";
-        QString prefix = handler->pathForItem(item, true, false);
-        QString baseline = prefix + QLS(FileFormat);
-        QString metadata = prefix + QLS(MetadataFileExt);
-        if (item.status == ImageItem::Mismatch) {
-            QString rendered = handler->pathForItem(item, false, false) + QLS(FileFormat);
-            QString itemFile = prefix.section(QLC('/'), -1);
+        if (item.status == ImageItem::Mismatch || item.status == ImageItem::FuzzyMatch) {
+            QString rendered = mmPrefix + QLS(FileFormat);
+            QString itemFile = mmPrefix.section(QLC('/'), -1);
             writeItem(baseline, rendered, item, itemFile, ctx, misCtx, metadata);
         }
         else {
@@ -210,6 +310,9 @@ void Report::writeFunctionResults(const ImageItemList &list)
                         << "\">Whitelist this item</a>";
                 }
                 break;
+            case ImageItem::Error:
+                out << "<span style=\"background-color:red\">Error: No result reported!</span>";
+                break;
             case ImageItem::Ok:
                 out << "<span style=\"color:green\"><small>No mismatch reported</small></span>";
                 break;
@@ -233,11 +336,14 @@ void Report::writeItem(const QString &baseline, const QString &rendered, const I
 
     QStringList images = QStringList() << baseline << rendered << compared;
     foreach (const QString& img, images)
-        out << "<td height=246 align=center><a href=\"/" << img << "\"><img src=\"/" << generateThumbnail(img) << "\"></a></td>\n";
+        out << "<td height=246 align=center><a href=\"/" << img << "\"><img src=\"/" << generateThumbnail(img, rootDir) << "\"></a></td>\n";
 
-    out << "<td align=center>\n"
-        << "<p><span style=\"color:red\">Mismatch reported</span></p>\n"
-        << "<p><a href=\"/" << metadata << "\">Baseline Info</a>\n";
+    out << "<td align=center>\n";
+    if (item.status == ImageItem::FuzzyMatch)
+        out << "<p><span style=\"color:orange\">Fuzzy match</span></p>\n";
+    else
+        out << "<p><span style=\"color:red\">Mismatch reported</span></p>\n";
+    out << "<p><a href=\"/" << metadata << "\">Baseline Info</a>\n";
     if (!hasOverride) {
         out << "<p><a href=\"/cgi-bin/server.cgi?cmd=updateSingleBaseline&context=" << ctx << "&mismatchContext=" << misCtx
             << "&itemFile=" << itemFile << "&url=" << pageUrl << "\">Let this be the new baseline</a></p>\n"
@@ -245,8 +351,14 @@ void Report::writeItem(const QString &baseline, const QString &rendered, const I
             << "&itemId=" << item.itemName << "&url=" << pageUrl << "\">Blacklist this item</a></p>\n";
     }
     out << "<p><a href=\"/cgi-bin/server.cgi?cmd=view&baseline=" << baseline << "&rendered=" << rendered
-        << "&compared=" << compared << "&url=" << pageUrl << "\">Inspect</a></p>\n"
-        << "</td>\n";
+        << "&compared=" << compared << "&url=" << pageUrl << "\">Inspect</a></p>\n";
+
+#if 0
+    out << "<p><a href=\"/cgi-bin/server.cgi?cmd=diffstats&baseline=" << baseline << "&rendered=" << rendered
+        << "&url=" << pageUrl << "\">Diffstats</a></p>\n";
+#endif
+
+    out << "</td>\n";
 }
 
 void Report::writeFooter()
@@ -259,8 +371,8 @@ QString Report::generateCompared(const QString &baseline, const QString &rendere
 {
     QString res = rendered;
     QFileInfo fi(res);
-    res.chop(fi.suffix().length() + 1);
-    res += QLS(fuzzy ? "_fuzzycompared.png" : "_compared.png");
+    res.chop(fi.suffix().length());
+    res += QLS(fuzzy ? "fuzzycompared.png" : "compared.png");
     QStringList args;
     if (fuzzy)
         args << QLS("-fuzz") << QLS("5%");
@@ -270,12 +382,14 @@ QString Report::generateCompared(const QString &baseline, const QString &rendere
 }
 
 
-QString Report::generateThumbnail(const QString &image)
+QString Report::generateThumbnail(const QString &image, const QString &rootDir)
 {
     QString res = image;
     QFileInfo imgFI(rootDir+image);
-    res.chop(imgFI.suffix().length() + 1);
-    res += QLS("_thumbnail.jpg");
+    if (!imgFI.exists())
+        return res;
+    res.chop(imgFI.suffix().length());
+    res += ThumbnailExt;
     QFileInfo resFI(rootDir+res);
     if (resFI.exists() && resFI.lastModified() > imgFI.lastModified())
         return res;
@@ -286,12 +400,80 @@ QString Report::generateThumbnail(const QString &image)
 }
 
 
+QString Report::writeResultsXmlFiles()
+{
+    if (!itemLists.size())
+        return QString();
+    QString dir = rootDir + baseDir + QLS("/xml-reports/") + runId;
+    QDir cwd;
+    if (!cwd.exists(dir))
+        cwd.mkpath(dir);
+    foreach (const QString &func, itemLists.keys()) {
+        QFile f(dir + "/" + func + "-results.xml");
+        if (!f.open(QIODevice::WriteOnly))
+            continue;
+        QXmlStreamWriter s(&f);
+        s.setAutoFormatting(true);
+        s.writeStartDocument();
+        foreach (QString key, plat.keys()) {
+            QString cmt = " " + key + "=\"" + plat.value(key) +"\" ";
+            s.writeComment(cmt.replace("--", "[-]"));
+        }
+        s.writeStartElement("testsuite");
+        s.writeAttribute("name", func);
+        foreach (const ImageItem &item, itemLists.value(func)) {
+            QString res;
+            switch (item.status) {
+            case ImageItem::Ok:
+            case ImageItem::FuzzyMatch:
+                res = "pass";
+                break;
+            case ImageItem::Mismatch:
+            case ImageItem::Error:
+                res = "fail";
+                break;
+            case ImageItem::BaselineNotFound:
+            case ImageItem::IgnoreItem:
+            default:
+                res = "skip";
+            }
+            s.writeStartElement("testcase");
+            s.writeAttribute("name", item.itemName);
+            s.writeAttribute("result", res);
+            s.writeEndElement();
+        }
+        s.writeEndElement();
+        s.writeEndDocument();
+    }
+    return dir;
+}
+
+
+
+void Report::updateLatestPointer()
+{
+    QString linkPath = rootDir + baseDir + QLS("/latest_report.html");
+    QString reportPath = path.mid(baseDir.size()+1);
+    QFile::remove(linkPath);             // possible race with another thread, yada yada yada
+    QFile::link(reportPath, linkPath);
+
+#if 0
+    QByteArray fwd = "<!DOCTYPE html><html><head><meta HTTP-EQUIV=\"refresh\" CONTENT=\"0;URL=%1\"></meta></head><body></body></html>\n";
+    fwd.replace("%1", filePath().prepend(QLC('/')).toLatin1());
+
+    QFile file(rootDir + baseDir + "/latest_report.html");
+    if (file.open(QIODevice::WriteOnly | QIODevice::Truncate))
+        file.write(fwd);
+#endif
+}
+
+
 void Report::handleCGIQuery(const QString &query)
 {
     QUrl cgiUrl(QLS("http://dummy/cgi-bin/dummy.cgi?") + query);
     QTextStream s(stdout);
     s << "Content-Type: text/html\r\n\r\n"
-      << "<HTML>";
+      << "<!DOCTYPE html>\n<HTML>\n<body bgcolor=""#ddeeff"">\n";          // Lancelot blue
 
     QString command(cgiUrl.queryItemValue("cmd"));
 
@@ -300,6 +482,12 @@ void Report::handleCGIQuery(const QString &query)
                                    cgiUrl.queryItemValue(QLS("rendered")),
                                    cgiUrl.queryItemValue(QLS("compared")));
     }
+#if 0
+    else if (command == QLS("diffstats")) {
+        s << BaselineHandler::diffstats(cgiUrl.queryItemValue(QLS("baseline")),
+                                        cgiUrl.queryItemValue(QLS("rendered")));
+    }
+#endif
     else if (command == QLS("updateSingleBaseline")) {
         s << BaselineHandler::updateBaselines(cgiUrl.queryItemValue(QLS("context")),
                                               cgiUrl.queryItemValue(QLS("mismatchContext")),
@@ -321,6 +509,6 @@ void Report::handleCGIQuery(const QString &query)
     } else {
         s << "Unknown query:<br>" << query << "<br>";
     }
-    s << "<p><a href=\"" << cgiUrl.queryItemValue(QLS("url")) << "\">Back to report</a>";
-    s << "</HTML>";
+    s << "<p><a href=\"" << cgiUrl.queryItemValue(QLS("url")) << "\">Back to report</a>\n";
+    s << "</body>\n</HTML>";
 }