Add GPU_BLACKLIST support to QTestLib

In addition to BLACKLIST, Qt will now look for GPU_BLACKLIST too.
Test cases that are specified as disabled in the GPU blacklist
will be skipped. This is particularly relevant when running tests
on Embedded Linux devices.

For example, the following JSON would configure the test case
glxContextWrap to be skipped on drivers where GL_VENDOR contains
UnstableDriverVendor:

{
    "entries": [ {
            "gl_vendor": "UnstableDriverVendor",
            "features": [ "disable_glxContextWrap" ]
    } ]
}

In contrast to the regular blacklist, GPU-blacklisted test cases are
not run at all. This is because driver problems and instabilities
often lead to crashes.

Change-Id: I340cf5c0261a206109b78409774408981bba5c68
Reviewed-by: Friedemann Kleint <Friedemann.Kleint@theqtcompany.com>
Reviewed-by: Simon Hausmann <simon.hausmann@theqtcompany.com>

4 files changed, 50 insertions, 3 deletions

diff --git a/src/testlib/qtest.h b/src/testlib/qtest.h
index 70e923927b..994179958b 100644
--- a/src/testlib/qtest.h
+++ b/src/testlib/qtest.h
@@ -289,6 +289,18 @@ int main(int argc, char *argv[]) \
 }
 
 #include <QtTest/qtestsystem.h>
+#include <set>
+
+#ifndef QT_NO_OPENGL
+#  define QTEST_ADD_GPU_BLACKLIST_SUPPORT_DEFS \
+    extern Q_TESTLIB_EXPORT std::set<QByteArray> *(*qgpu_features_ptr)(const QString &); \
+    extern Q_GUI_EXPORT std::set<QByteArray> *qgpu_features(const QString &);
+#  define QTEST_ADD_GPU_BLACKLIST_SUPPORT \
+    qgpu_features_ptr = qgpu_features;
+#else
+#  define QTEST_ADD_GPU_BLACKLIST_SUPPORT_DEFS
+#  define QTEST_ADD_GPU_BLACKLIST_SUPPORT
+#endif
 
 #if defined(QT_WIDGETS_LIB)
 
@@ -301,11 +313,15 @@ int main(int argc, char *argv[]) \
 #endif
 
 #define QTEST_MAIN(TestObject) \
+QT_BEGIN_NAMESPACE \
+QTEST_ADD_GPU_BLACKLIST_SUPPORT_DEFS \
+QT_END_NAMESPACE \
 int main(int argc, char *argv[]) \
 { \
     QApplication app(argc, argv); \
     app.setAttribute(Qt::AA_Use96Dpi, true); \
     QTEST_DISABLE_KEYPAD_NAVIGATION \
+    QTEST_ADD_GPU_BLACKLIST_SUPPORT \
     TestObject tc; \
     QTEST_SET_MAIN_SOURCE_PATH \
     return QTest::qExec(&tc, argc, argv); \
@@ -316,10 +332,14 @@ int main(int argc, char *argv[]) \
 #include <QtTest/qtest_gui.h>
 
 #define QTEST_MAIN(TestObject) \
+QT_BEGIN_NAMESPACE \
+QTEST_ADD_GPU_BLACKLIST_SUPPORT_DEFS \
+QT_END_NAMESPACE \
 int main(int argc, char *argv[]) \
 { \
     QGuiApplication app(argc, argv); \
     app.setAttribute(Qt::AA_Use96Dpi, true); \
+    QTEST_ADD_GPU_BLACKLIST_SUPPORT \
     TestObject tc; \
     QTEST_SET_MAIN_SOURCE_PATH \
     return QTest::qExec(&tc, argc, argv); \
diff --git a/src/testlib/qtestblacklist.cpp b/src/testlib/qtestblacklist.cpp
index f8285c3ea9..28a2878b32 100644
--- a/src/testlib/qtestblacklist.cpp
+++ b/src/testlib/qtestblacklist.cpp
@@ -34,6 +34,7 @@
 #include "qtestresult_p.h"
 
 #include <QtTest/qtestcase.h>
+#include <QtTest/qtest.h>
 #include <QtCore/qbytearray.h>
 #include <QtCore/qfile.h>
 #include <QtCore/QSysInfo>
@@ -154,6 +155,9 @@ static bool checkCondition(const QByteArray &condition)
 
 static bool ignoreAll = false;
 static std::set<QByteArray> *ignoredTests = 0;
+static std::set<QByteArray> *gpuFeatures = 0;
+
+Q_TESTLIB_EXPORT std::set<QByteArray> *(*qgpu_features_ptr)(const QString &) = 0;
 
 namespace QTestPrivate {
 
@@ -189,7 +193,18 @@ void parseBlackList()
     }
 }
 
-void checkBlackList(const char *slot, const char *data)
+void parseGpuBlackList()
+{
+    if (!qgpu_features_ptr)
+        return;
+    QString filename = QTest::qFindTestData(QStringLiteral("GPU_BLACKLIST"));
+    if (filename.isEmpty())
+        return;
+    if (!gpuFeatures)
+        gpuFeatures = qgpu_features_ptr(filename);
+}
+
+void checkBlackLists(const char *slot, const char *data)
 {
     bool ignore = ignoreAll;
 
@@ -204,6 +219,16 @@ void checkBlackList(const char *slot, const char *data)
     }
 
     QTestResult::setBlacklistCurrentTest(ignore);
+
+    // Tests blacklisted in GPU_BLACKLIST are to be skipped. Just ignoring the result is
+    // not sufficient since these are expected to crash or behave in undefined ways.
+    if (!ignore && gpuFeatures) {
+        const QByteArray disableKey = QByteArrayLiteral("disable_") + QByteArray(slot);
+        if (gpuFeatures->find(disableKey) != gpuFeatures->end()) {
+            const QByteArray msg = QByteArrayLiteral("Skipped due to GPU blacklist: ") + disableKey;
+            QTest::qSkip(msg.constData(), __FILE__, __LINE__);
+        }
+    }
 }
 
 }
diff --git a/src/testlib/qtestblacklist_p.h b/src/testlib/qtestblacklist_p.h
index 158d99593e..0107e5d282 100644
--- a/src/testlib/qtestblacklist_p.h
+++ b/src/testlib/qtestblacklist_p.h
@@ -51,7 +51,8 @@ QT_BEGIN_NAMESPACE
 
 namespace QTestPrivate {
     void parseBlackList();
-    void checkBlackList(const char *slot, const char *data);
+    void parseGpuBlackList();
+    void checkBlackLists(const char *slot, const char *data);
 }
 
 QT_END_NAMESPACE
diff --git a/src/testlib/qtestcase.cpp b/src/testlib/qtestcase.cpp
index b76e5544ba..e3c543671b 100644
--- a/src/testlib/qtestcase.cpp
+++ b/src/testlib/qtestcase.cpp
@@ -2074,7 +2074,7 @@ static bool qInvokeTestMethod(const char *slotName, const char *data=0)
                 if (!data || !qstrcmp(data, table.testData(curDataIndex)->dataTag())) {
                     foundFunction = true;
 
-                    QTestPrivate::checkBlackList(slot, dataCount ? table.testData(curDataIndex)->dataTag() : 0);
+                    QTestPrivate::checkBlackLists(slot, dataCount ? table.testData(curDataIndex)->dataTag() : 0);
 
                     QTestDataSetter s(curDataIndex >= dataCount ? static_cast<QTestData *>(0)
                                                       : table.testData(curDataIndex));
@@ -2583,6 +2583,7 @@ int QTest::qExec(QObject *testObject, int argc, char **argv)
 #endif
 
     QTestPrivate::parseBlackList();
+    QTestPrivate::parseGpuBlackList();
 
     QTestResult::reset();