Update baseline testing framework

Merge in various minor changes and fixes that have been done to the
branched copy in qtquick3d, and clean out some outdated code.

Mostly just coding style fixes and cleanups, but also:

- adds -keeprunning command line parameter, intended for tests that by
  default exits early if it seems the platform is too unstable
  (e.g. crashing) for a meaningful testrun.

- Changes behaviour for fuzzy matches, from SKIP to PASS. The (mis)use
  of QSKIP was done to force log output; now the output is just
  printed by qInfo() instead.

Pick-to: 6.2
Change-Id: I46e77a94cc5b1980ac420086c2ae88dc9b84ef12
Reviewed-by: Volker Hilsheimer <volker.hilsheimer@qt.io>

5 files changed, 40 insertions, 68 deletions

diff --git a/tests/baseline/shared/baselineprotocol.cpp b/tests/baseline/shared/baselineprotocol.cpp
index 63cce0bd80..33620fbe5c 100644
--- a/tests/baseline/shared/baselineprotocol.cpp
+++ b/tests/baseline/shared/baselineprotocol.cpp
@@ -36,6 +36,7 @@
 #endif
 #include <QFileInfo>
 #include <QDir>
+#include <QThread>
 #include <QTime>
 #include <QPointer>
 #include <QRegularExpression>
@@ -49,29 +50,7 @@ const QString PI_OSVersion(QLS("OSVersion"));
 const QString PI_QtVersion(QLS("QtVersion"));
 const QString PI_QtBuildMode(QLS("QtBuildMode"));
 const QString PI_GitCommit(QLS("GitCommit"));
-const QString PI_QMakeSpec(QLS("QMakeSpec"));
-const QString PI_PulseGitBranch(QLS("PulseGitBranch"));
-const QString PI_PulseTestrBranch(QLS("PulseTestrBranch"));
-
-#ifndef QMAKESPEC
-#define QMAKESPEC "Unknown"
-#endif
-
-#if defined(Q_OS_WIN)
-#include <QtCore/qt_windows.h>
-#endif
-#if defined(Q_OS_UNIX)
-#include <time.h>
-#endif
-void BaselineProtocol::sysSleep(int ms)
-{
-#if defined(Q_OS_WIN)
-    Sleep(DWORD(ms));
-#else
-    struct timespec ts = { ms / 1000, (ms % 1000) * 1000 * 1000 };
-    nanosleep(&ts, NULL);
-#endif
-}
+const QString PI_GitBranch(QLS("GitBranch"));
 
 PlatformInfo::PlatformInfo()
     : QMap<QString, QString>(), adHoc(true)
@@ -83,7 +62,6 @@ PlatformInfo PlatformInfo::localHostInfo()
     PlatformInfo pi;
     pi.insert(PI_HostName, QHostInfo::localHostName());
     pi.insert(PI_QtVersion, QLS(qVersion()));
-    pi.insert(PI_QMakeSpec, QString(QLS(QMAKESPEC)).remove(QRegularExpression(QLS("^.*mkspecs/"))));
     pi.insert(PI_QtBuildMode, QLibraryInfo::isDebugBuild() ? QLS("QtDebug") : QLS("QtRelease"));
 #if defined(Q_OS_LINUX) && QT_CONFIG(process)
     pi.insert(PI_OSName, QLS("Linux"));
@@ -113,26 +91,14 @@ PlatformInfo PlatformInfo::localHostInfo()
         pi.insert(PI_GitCommit, QString::fromLocal8Bit(git.readAllStandardOutput().constData()).simplified());
     else
         pi.insert(PI_GitCommit, QLS("Unknown"));
+#endif // QT_CONFIG(process)
 
-    QByteArray gb = qgetenv("PULSE_GIT_BRANCH");
-    if (!gb.isEmpty()) {
-        pi.insert(PI_PulseGitBranch, QString::fromLatin1(gb));
-        pi.setAdHocRun(false);
-    }
-    QByteArray tb = qgetenv("PULSE_TESTR_BRANCH");
-    if (!tb.isEmpty()) {
-        pi.insert(PI_PulseTestrBranch, QString::fromLatin1(tb));
-        pi.setAdHocRun(false);
-    }
-    if (!qgetenv("JENKINS_HOME").isEmpty()) {
+    if (qEnvironmentVariableIsSet("JENKINS_HOME"))
         pi.setAdHocRun(false);
-        gb = qgetenv("GIT_BRANCH");
-        if (!gb.isEmpty()) {
-            // FIXME: the string "Pulse" should be eliminated, since that is not the used tool.
-            pi.insert(PI_PulseGitBranch, QString::fromLatin1(gb));
-        }
-    }
-#endif // QT_CONFIG(process)
+
+    QString gb = qEnvironmentVariable("GIT_BRANCH");
+    if (!gb.isEmpty())
+        pi.insert(PI_GitBranch, gb);
 
     return pi;
 }
@@ -286,8 +252,7 @@ void ImageItem::writeImageToStream(QDataStream &out) const
     out << quint8('Q') << quint8(image.format());
     out << quint8(QSysInfo::ByteOrder) << quint8(0);       // pad to multiple of 4 bytes
     out << quint32(image.width()) << quint32(image.height()) << quint32(image.bytesPerLine());
-    out << qCompress(reinterpret_cast<const uchar *>(image.constBits()),
-                     int(image.sizeInBytes()));
+    out << qCompress(reinterpret_cast<const uchar *>(image.constBits()), image.sizeInBytes());
     //# can be followed by colormap for formats that use it
 }
 
@@ -360,7 +325,7 @@ bool BaselineProtocol::connect(const QString &testCase, bool *dryrun, const Plat
 
     socket.connectToHost(serverName, ServerPort);
     if (!socket.waitForConnected(Timeout)) {
-        sysSleep(3000);  // Wait a bit and try again, the server might just be restarting
+        QThread::msleep(3000);  // Wait a bit and try again, the server might just be restarting
         if (!socket.waitForConnected(Timeout)) {
             errMsg += QLS("TCP connectToHost failed. Host:") + QLS(serverName) + QLS(" port:") + QString::number(ServerPort);
             return false;
@@ -425,7 +390,7 @@ bool BaselineProtocol::requestBaselineChecksums(const QString &testFunction, Ima
     if (!itemList)
         return false;
 
-    for(ImageItemList::iterator it = itemList->begin(); it != itemList->end(); it++)
+    for (ImageItemList::iterator it = itemList->begin(); it != itemList->end(); it++)
         it->testFunction = testFunction;
 
     QByteArray block;
@@ -448,7 +413,7 @@ bool BaselineProtocol::submitMatch(const ImageItem &item, QByteArray *serverMsg)
 {
     Command cmd;
     ImageItem smallItem = item;
-    smallItem.image = QImage();  // No need to waste bandwith sending image (identical to baseline) to server
+    smallItem.image = QImage();  // No need to waste bandwidth sending image (identical to baseline) to server
     return (sendItem(AcceptMatch, smallItem) && receiveBlock(&cmd, serverMsg) && cmd == Ack);
 }
 
diff --git a/tests/baseline/shared/baselineprotocol.h b/tests/baseline/shared/baselineprotocol.h
index 510762e304..4151d30c9c 100644
--- a/tests/baseline/shared/baselineprotocol.h
+++ b/tests/baseline/shared/baselineprotocol.h
@@ -51,9 +51,7 @@ extern const QString PI_OSVersion;
 extern const QString PI_QtVersion;
 extern const QString PI_QtBuildMode;
 extern const QString PI_GitCommit;
-extern const QString PI_QMakeSpec;
-extern const QString PI_PulseGitBranch;
-extern const QString PI_PulseTestrBranch;
+extern const QString PI_GitBranch;
 
 class PlatformInfo : public QMap<QString, QString>
 {
@@ -174,7 +172,6 @@ private:
 
     bool sendBlock(Command cmd, const QByteArray &block);
     bool receiveBlock(Command *cmd, QByteArray *block);
-    void sysSleep(int ms);
 
     QString errMsg;
     QTcpSocket socket;
diff --git a/tests/baseline/shared/lookup3.cpp b/tests/baseline/shared/lookup3.cpp
index 3d8d763bb7..1fbe6b499e 100644
--- a/tests/baseline/shared/lookup3.cpp
+++ b/tests/baseline/shared/lookup3.cpp
@@ -204,7 +204,7 @@ quint32        initval)         /* the previous hash, or an arbitrary value */
   }
 
   /*------------------------------------------- handle the last 3 quint32's */
-  switch(length)                     /* all the case statements fall through */
+  switch (length)                     /* all the case statements fall through */
   {
   case 3 : c+=k[2];
            Q_FALLTHROUGH();
@@ -253,7 +253,7 @@ quint32       *pb)               /* IN: more seed OUT: secondary hash value */
   }
 
   /*------------------------------------------- handle the last 3 quint32's */
-  switch(length)                     /* all the case statements fall through */
+  switch (length)                     /* all the case statements fall through */
   {
   case 3 : c+=k[2];
            Q_FALLTHROUGH();
@@ -332,7 +332,7 @@ quint32 hashlittle( const void *key, size_t length, quint32 initval)
      */
 #ifndef VALGRIND
 
-    switch(length)
+    switch (length)
     {
     case 12: c+=k[2]; b+=k[1]; a+=k[0]; break;
     case 11: c+=k[2]&0xffffff; b+=k[1]; a+=k[0]; break;
@@ -352,7 +352,7 @@ quint32 hashlittle( const void *key, size_t length, quint32 initval)
 #else /* make valgrind happy */
 
     const quint8  *k8 = (const quint8 *)k;
-    switch(length)
+    switch (length)
     {
     case 12: c+=k[2]; b+=k[1]; a+=k[0]; break;
     case 11: c+=((quint32)k8[10])<<16;
@@ -396,7 +396,7 @@ quint32 hashlittle( const void *key, size_t length, quint32 initval)
 
     /*----------------------------- handle the last (probably partial) block */
     k8 = (const quint8 *)k;
-    switch(length)
+    switch (length)
     {
     case 12: c+=k[4]+(((quint32)k[5])<<16);
              b+=k[2]+(((quint32)k[3])<<16);
@@ -455,7 +455,7 @@ quint32 hashlittle( const void *key, size_t length, quint32 initval)
     }
 
     /*-------------------------------- last block: affect all 32 bits of (c) */
-    switch(length)                   /* all the case statements fall through */
+    switch (length)                   /* all the case statements fall through */
     {
     case 12: c+=((quint32)k[11])<<24;
              Q_FALLTHROUGH();
@@ -540,7 +540,7 @@ void hashlittle2(
      */
 #ifndef VALGRIND
 
-    switch(length)
+    switch (length)
     {
     case 12: c+=k[2]; b+=k[1]; a+=k[0]; break;
     case 11: c+=k[2]&0xffffff; b+=k[1]; a+=k[0]; break;
@@ -560,7 +560,7 @@ void hashlittle2(
 #else /* make valgrind happy */
 
     const quint8  *k8 = (const quint8 *)k;
-    switch(length)
+    switch (length)
     {
     case 12: c+=k[2]; b+=k[1]; a+=k[0]; break;
     case 11: c+=((quint32)k8[10])<<16;
@@ -604,7 +604,7 @@ void hashlittle2(
 
     /*----------------------------- handle the last (probably partial) block */
     k8 = (const quint8 *)k;
-    switch(length)
+    switch (length)
     {
     case 12: c+=k[4]+(((quint32)k[5])<<16);
              b+=k[2]+(((quint32)k[3])<<16);
@@ -663,7 +663,7 @@ void hashlittle2(
     }
 
     /*-------------------------------- last block: affect all 32 bits of (c) */
-    switch(length)                   /* all the case statements fall through */
+    switch (length)                   /* all the case statements fall through */
     {
     case 12: c+=((quint32)k[11])<<24;
              Q_FALLTHROUGH();
@@ -740,7 +740,7 @@ quint32 hashbig( const void *key, size_t length, quint32 initval)
      */
 #ifndef VALGRIND
 
-    switch(length)
+    switch (length)
     {
     case 12: c+=k[2]; b+=k[1]; a+=k[0]; break;
     case 11: c+=k[2]&0xffffff00; b+=k[1]; a+=k[0]; break;
@@ -760,7 +760,7 @@ quint32 hashbig( const void *key, size_t length, quint32 initval)
 #else  /* make valgrind happy */
 
     const quint8 *k8 = (const quint8 *)k;
-    switch(length)                   /* all the case statements fall through */
+    switch (length)                   /* all the case statements fall through */
     {
     case 12: c+=k[2]; b+=k[1]; a+=k[0]; break;
     case 11: c+=((quint32)k8[10])<<8;
@@ -811,7 +811,7 @@ quint32 hashbig( const void *key, size_t length, quint32 initval)
     }
 
     /*-------------------------------- last block: affect all 32 bits of (c) */
-    switch(length)                   /* all the case statements fall through */
+    switch (length)                   /* all the case statements fall through */
     {
     case 12: c+=k[11];
              Q_FALLTHROUGH();
diff --git a/tests/baseline/shared/qbaselinetest.cpp b/tests/baseline/shared/qbaselinetest.cpp
index 3587cd01ea..004d8013f1 100644
--- a/tests/baseline/shared/qbaselinetest.cpp
+++ b/tests/baseline/shared/qbaselinetest.cpp
@@ -45,6 +45,7 @@ static bool connected = false;
 static bool triedConnecting = false;
 static bool dryRunMode = false;
 static enum { UploadMissing, UploadAll, UploadNone } baselinePolicy = UploadMissing;
+static bool abortIfUnstable = true;
 
 static QByteArray curFunction;
 static ImageItemList itemList;
@@ -66,7 +67,7 @@ void handleCmdLineArgs(int *argcp, char ***argvp)
 
     for (int i = 0; i < numArgs; i++) {
         QByteArray arg = (*argvp)[i];
-        QByteArray nextArg = (i+1 < numArgs) ? (*argvp)[i+1] : 0;
+        QByteArray nextArg = (i+1 < numArgs) ? (*argvp)[i+1] : nullptr;
 
         if (arg == "-simfail") {
             simfail = true;
@@ -88,6 +89,8 @@ void handleCmdLineArgs(int *argcp, char ***argvp)
             customInfo.setAdHocRun(true);
         } else if (arg == "-setbaselines") {
             baselinePolicy = UploadAll;
+        } else if (arg == "-keeprunning") {
+            abortIfUnstable = false;
         } else if (arg == "-nosetbaselines") {
             baselinePolicy = UploadNone;
         } else if (arg == "-compareto") {
@@ -122,6 +125,7 @@ void handleCmdLineArgs(int *argcp, char ***argvp)
         out << " -fuzzlevel <int>    : Specify the percentage of fuzziness in comparison. Overrides server default. 0 means exact match.\n";
         out << " -auto               : Inform server that this run is done by a daemon, CI system or similar.\n";
         out << " -adhoc (default)    : The inverse of -auto; this run is done by human, e.g. for testing.\n";
+        out << " -keeprunning        : Run all tests even if the system is unstable \n";
         out << " -setbaselines       : Store ALL rendered images as new baselines. Forces replacement of previous baselines.\n";
         out << " -nosetbaselines     : Do not store rendered images as new baselines when previous baselines are missing.\n";
         out << " -compareto KEY=VAL  : Force comparison to baselines from a different client,\n";
@@ -131,6 +135,10 @@ void handleCmdLineArgs(int *argcp, char ***argvp)
     }
 }
 
+bool shouldAbortIfUnstable()
+{
+    return abortIfUnstable;
+}
 
 void addClientProperty(const QString& key, const QString& value)
 {
@@ -184,7 +192,8 @@ bool connect(QByteArray *msg, bool *error)
     // Merge the platform info set by the program with the protocols default info
     PlatformInfo clientInfo = customInfo;
     PlatformInfo defaultInfo = PlatformInfo::localHostInfo();
-    foreach (QString key, defaultInfo.keys()) {
+    const auto &defaultInfoKeys = defaultInfo.keys();
+    for (const QString &key : defaultInfoKeys) {
         if (!clientInfo.contains(key))
             clientInfo.insert(key, defaultInfo.value(key));
     }
@@ -319,8 +328,7 @@ bool compareItem(const ImageItem &baseline, const QImage &img, QByteArray *msg,
     bool fuzzyMatch = false;
     bool res = proto.submitMismatch(item, &srvMsg, &fuzzyMatch);
     if (res && fuzzyMatch) {
-        *error = true;          // To force a QSKIP/debug output; somewhat kludgy
-        *msg += srvMsg;
+        qInfo() << "Baseline server reports:" << srvMsg;
         return true;            // The server decides: a fuzzy match means no mismatch
     }
     *msg += "Mismatch. See report:\n   " + srvMsg;
diff --git a/tests/baseline/shared/qbaselinetest.h b/tests/baseline/shared/qbaselinetest.h
index ede0fe42e3..c525fc8466 100644
--- a/tests/baseline/shared/qbaselinetest.h
+++ b/tests/baseline/shared/qbaselinetest.h
@@ -30,6 +30,7 @@
 #define BASELINETEST_H
 
 #include <QTest>
+#include <QString>
 
 namespace QBaselineTest {
 void setAutoMode(bool mode);
@@ -41,6 +42,7 @@ bool checkImage(const QImage& img, const char *name, quint16 checksum, QByteArra
 bool testImage(const QImage& img, QByteArray *msg, bool *error);
 QTestData &newRow(const char *dataTag, quint16 checksum = 0);
 bool disconnectFromBaselineServer();
+bool shouldAbortIfUnstable();
 }
 
 #define QBASELINE_CHECK_SUM(image, name, checksum)\