Add support for blacklisting test functions

We need to have a finer grained control over the tests
we skip in our CI system. This adds a blacklisting
mechanism that allows blacklisting individual test
functions (or even test data) using a set of predefined
matching keys for the operating system and some other
relevant variables.

QTestlib will search for a file called BLACKLIST in the test
directory and parse it if found. The file contains a simple
ini style list of functions to blacklist. For details see
qtestblacklist.cpp.

Change-Id: Id3fae4b264ca99970cbf9f45bfb85fa75c1fd823
Reviewed-by: Lars Knoll <lars.knoll@digia.com>

12 files changed, 353 insertions, 9 deletions

diff --git a/src/testlib/qabstracttestlogger_p.h b/src/testlib/qabstracttestlogger_p.h
index c549233ddb..6bb54d1af8 100644
--- a/src/testlib/qabstracttestlogger_p.h
+++ b/src/testlib/qabstracttestlogger_p.h
@@ -68,7 +68,9 @@ public:
         Pass,
         XFail,
         Fail,
-        XPass
+        XPass,
+        BlacklistedPass,
+        BlacklistedFail
     };
 
     enum MessageTypes {
diff --git a/src/testlib/qplaintestlogger.cpp b/src/testlib/qplaintestlogger.cpp
index 774f2cc4fb..74f0290f38 100644
--- a/src/testlib/qplaintestlogger.cpp
+++ b/src/testlib/qplaintestlogger.cpp
@@ -89,6 +89,10 @@ namespace QTest {
             return "FAIL!  ";
         case QAbstractTestLogger::XPass:
             return "XPASS  ";
+        case QAbstractTestLogger::BlacklistedPass:
+            return "BPASS  ";
+        case QAbstractTestLogger::BlacklistedFail:
+            return "BFAIL  ";
         }
         return "??????";
     }
@@ -351,15 +355,16 @@ void QPlainTestLogger::stopLogging()
 {
     char buf[1024];
     if (QTestLog::verboseLevel() < 0) {
-        qsnprintf(buf, sizeof(buf), "Totals: %d passed, %d failed, %d skipped\n",
+        qsnprintf(buf, sizeof(buf), "Totals: %d passed, %d failed, %d skipped, %d blacklisted\n",
                   QTestLog::passCount(), QTestLog::failCount(),
-                  QTestLog::skipCount());
+                  QTestLog::skipCount(), QTestLog::blacklistCount());
     } else {
         qsnprintf(buf, sizeof(buf),
-                  "Totals: %d passed, %d failed, %d skipped\n"
+                  "Totals: %d passed, %d failed, %d skipped, %d blacklisted\n"
                   "********* Finished testing of %s *********\n",
                   QTestLog::passCount(), QTestLog::failCount(),
-                  QTestLog::skipCount(), QTestResult::currentTestObjectName());
+                  QTestLog::skipCount(), QTestLog::blacklistCount(),
+                  QTestResult::currentTestObjectName());
     }
     outputMessage(buf);
 
diff --git a/src/testlib/qtestblacklist.cpp b/src/testlib/qtestblacklist.cpp
new file mode 100644
index 0000000000..7921b350b5
--- /dev/null
+++ b/src/testlib/qtestblacklist.cpp
@@ -0,0 +1,201 @@
+/****************************************************************************
+**
+** Copyright (C) 2014 Digia Plc and/or its subsidiary(-ies).
+** Contact: http://www.qt-project.org/legal
+**
+** This file is part of the QtTest module of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:LGPL$
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and Digia.  For licensing terms and
+** conditions see http://qt.digia.com/licensing.  For further information
+** use the contact form at http://qt.digia.com/contact-us.
+**
+** GNU Lesser General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU Lesser
+** General Public License version 2.1 as published by the Free Software
+** Foundation and appearing in the file LICENSE.LGPL included in the
+** packaging of this file.  Please review the following information to
+** ensure the GNU Lesser General Public License version 2.1 requirements
+** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
+**
+** In addition, as a special exception, Digia gives you certain additional
+** rights.  These rights are described in the Digia Qt LGPL Exception
+** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3.0 as published by the Free Software
+** Foundation and appearing in the file LICENSE.GPL included in the
+** packaging of this file.  Please review the following information to
+** ensure the GNU General Public License version 3.0 requirements will be
+** met: http://www.gnu.org/copyleft/gpl.html.
+**
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+#include "qtestblacklist_p.h"
+#include "qtestresult_p.h"
+
+#include <QtTest/qtestcase.h>
+#include <QtCore/qbytearray.h>
+#include <QtCore/qfile.h>
+
+#include <set>
+
+QT_BEGIN_NAMESPACE
+
+/*
+ The file format is simply a grouped listing of keywords
+ Ungrouped entries at the beginning apply to the whole testcase
+ Groups define testfunctions or specific test data to ignore.
+ After the groups come a list of entries (one per line) that define
+ for which platform/os combination to ignore the test result.
+ All keys in a single line have to match to blacklist the test.
+
+ mac
+ [testFunction]
+ linux
+ windows 64bit
+ [testfunction2:testData]
+ msvc
+
+ The known keys are listed below:
+*/
+
+// this table can be extended with new keywords as required
+const char *matchedConditions[] =
+{
+#ifdef Q_OS_LINUX
+    "linux",
+#endif
+#ifdef Q_OS_OSX
+    "osx",
+#endif
+#ifdef Q_OS_WIN
+    "windows",
+#endif
+#ifdef Q_OS_IOS
+    "ios",
+#endif
+#ifdef Q_OS_ANDROID
+    "android",
+#endif
+#ifdef Q_OS_QNX
+    "qnx",
+#endif
+#ifdef Q_OS_WINRT
+    "winrt",
+#endif
+#ifdef Q_OS_WINCE
+    "wince",
+#endif
+
+#if QT_POINTER_SIZE == 8
+    "64bit",
+#else
+    "32bit",
+#endif
+
+#ifdef Q_CC_GNU
+    "gcc",
+#endif
+#ifdef Q_CC_CLANG
+    "clang",
+#endif
+#ifdef Q_CC_MSVC
+    "msvc",
+#endif
+
+#ifdef Q_AUTOTEST_EXPORT
+    "developer-build",
+#endif
+    0
+};
+
+
+static bool checkCondition(const QByteArray &condition)
+{
+    QList<QByteArray> conds = condition.split(' ');
+    std::set<QByteArray> matches;
+    const char **m = matchedConditions;
+    while (*m) {
+        matches.insert(*m);
+        ++m;
+    }
+
+    for (int i = 0; i < conds.size(); ++i) {
+        QByteArray c = conds.at(i);
+        bool result = c.startsWith('!');
+        if (result)
+            c = c.mid(1);
+
+        result ^= (matches.find(c) != matches.end());
+        if (!result)
+            return false;
+    }
+    return true;
+}
+
+static bool ignoreAll = false;
+static std::set<QByteArray> *ignoredTests = 0;
+
+namespace QTestPrivate {
+
+void parseBlackList()
+{
+    QString filename = QTest::qFindTestData(QStringLiteral("BLACKLIST"));
+    if (filename.isEmpty())
+        return;
+    QFile ignored(filename);
+    if (!ignored.open(QIODevice::ReadOnly))
+        return;
+
+    QByteArray function;
+
+    while (!ignored.atEnd()) {
+        QByteArray line = ignored.readLine().simplified();
+        if (line.isEmpty() || line.startsWith('#'))
+            continue;
+        if (line.startsWith('[')) {
+            function = line.mid(1, line.length() - 2);
+            continue;
+        }
+        bool condition = checkCondition(line);
+        if (condition) {
+            if (!function.size()) {
+                ignoreAll = true;
+            } else {
+                if (!ignoredTests)
+                    ignoredTests = new std::set<QByteArray>;
+                ignoredTests->insert(function);
+            }
+        }
+    }
+}
+
+void checkBlackList(const char *slot, const char *data)
+{
+    bool ignore = ignoreAll;
+
+    if (!ignore && ignoredTests) {
+        QByteArray s = slot;
+        ignore = (ignoredTests->find(s) != ignoredTests->end());
+        if (!ignore && data) {
+            s += ':';
+            s += data;
+            ignore = (ignoredTests->find(s) != ignoredTests->end());
+        }
+    }
+
+    QTestResult::setBlacklistCurrentTest(ignore);
+}
+
+}
+
+
+QT_END_NAMESPACE
diff --git a/src/testlib/qtestblacklist_p.h b/src/testlib/qtestblacklist_p.h
new file mode 100644
index 0000000000..a9d07ae024
--- /dev/null
+++ b/src/testlib/qtestblacklist_p.h
@@ -0,0 +1,67 @@
+/****************************************************************************
+**
+** Copyright (C) 2014 Digia Plc and/or its subsidiary(-ies).
+** Contact: http://www.qt-project.org/legal
+**
+** This file is part of the QtTest module of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:LGPL$
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and Digia.  For licensing terms and
+** conditions see http://qt.digia.com/licensing.  For further information
+** use the contact form at http://qt.digia.com/contact-us.
+**
+** GNU Lesser General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU Lesser
+** General Public License version 2.1 as published by the Free Software
+** Foundation and appearing in the file LICENSE.LGPL included in the
+** packaging of this file.  Please review the following information to
+** ensure the GNU Lesser General Public License version 2.1 requirements
+** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
+**
+** In addition, as a special exception, Digia gives you certain additional
+** rights.  These rights are described in the Digia Qt LGPL Exception
+** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3.0 as published by the Free Software
+** Foundation and appearing in the file LICENSE.GPL included in the
+** packaging of this file.  Please review the following information to
+** ensure the GNU General Public License version 3.0 requirements will be
+** met: http://www.gnu.org/copyleft/gpl.html.
+**
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+
+#ifndef QTESTBLACKLIST_P_H
+#define QTESTBLACKLIST_P_H
+
+//
+//  W A R N I N G
+//  -------------
+//
+// This file is not part of the Qt API.  It exists purely as an
+// implementation detail.  This header file may change from version to
+// version without notice, or even be removed.
+//
+// We mean it.
+//
+
+#include <QtTest/qtest_global.h>
+
+QT_BEGIN_NAMESPACE
+
+namespace QTestPrivate {
+    void parseBlackList();
+    void checkBlackList(const char *slot, const char *data);
+}
+
+QT_END_NAMESPACE
+
+#endif
diff --git a/src/testlib/qtestcase.cpp b/src/testlib/qtestcase.cpp
index 739d341233..5d5b298740 100644
--- a/src/testlib/qtestcase.cpp
+++ b/src/testlib/qtestcase.cpp
@@ -63,6 +63,7 @@
 #include <QtTest/private/qsignaldumper_p.h>
 #include <QtTest/private/qbenchmark_p.h>
 #include <QtTest/private/cycle_p.h>
+#include <QtTest/private/qtestblacklist_p.h>
 
 #include <numeric>
 #include <algorithm>
@@ -2016,6 +2017,9 @@ static bool qInvokeTestMethod(const char *slotName, const char *data=0)
                 QTestResult::setSkipCurrentTest(false);
                 if (!data || !qstrcmp(data, table.testData(curDataIndex)->dataTag())) {
                     foundFunction = true;
+
+                    QTestPrivate::checkBlackList(slot, dataCount ? table.testData(curDataIndex)->dataTag() : 0);
+
                     QTestDataSetter s(curDataIndex >= dataCount ? static_cast<QTestData *>(0)
                                                       : table.testData(curDataIndex));
 
@@ -2433,6 +2437,8 @@ int QTest::qExec(QObject *testObject, int argc, char **argv)
     }
 #endif
 
+    QTestPrivate::parseBlackList();
+
     QTestResult::reset();
 
     QTEST_ASSERT(testObject);
diff --git a/src/testlib/qtestlog.cpp b/src/testlib/qtestlog.cpp
index 708a91b557..b7ed3d0763 100644
--- a/src/testlib/qtestlog.cpp
+++ b/src/testlib/qtestlog.cpp
@@ -84,6 +84,7 @@ namespace QTest {
     int fails = 0;
     int passes = 0;
     int skips = 0;
+    int blacklists = 0;
 
     struct IgnoreResultList
     {
@@ -415,6 +416,25 @@ void QTestLog::addXPass(const char *msg, const char *file, int line)
     QTest::TestLoggers::addIncident(QAbstractTestLogger::XPass, msg, file, line);
 }
 
+void QTestLog::addBPass(const char *msg)
+{
+    QTEST_ASSERT(msg);
+
+    ++QTest::blacklists;
+
+    QTest::TestLoggers::addIncident(QAbstractTestLogger::BlacklistedPass, msg);
+}
+
+void QTestLog::addBFail(const char *msg, const char *file, int line)
+{
+    QTEST_ASSERT(msg);
+    QTEST_ASSERT(file);
+
+    ++QTest::blacklists;
+
+    QTest::TestLoggers::addIncident(QAbstractTestLogger::BlacklistedFail, msg, file, line);
+}
+
 void QTestLog::addSkip(const char *msg, const char *file, int line)
 {
     QTEST_ASSERT(msg);
@@ -552,6 +572,11 @@ int QTestLog::skipCount()
     return QTest::skips;
 }
 
+int QTestLog::blacklistCount()
+{
+    return QTest::blacklists;
+}
+
 void QTestLog::resetCounters()
 {
     QTest::passes = 0;
diff --git a/src/testlib/qtestlog_p.h b/src/testlib/qtestlog_p.h
index a987c45806..540ffe75eb 100644
--- a/src/testlib/qtestlog_p.h
+++ b/src/testlib/qtestlog_p.h
@@ -72,6 +72,8 @@ public:
     static void addFail(const char *msg, const char *file, int line);
     static void addXFail(const char *msg, const char *file, int line);
     static void addXPass(const char *msg, const char *file, int line);
+    static void addBPass(const char *msg);
+    static void addBFail(const char *msg, const char *file, int line);
     static void addSkip(const char *msg, const char *file, int line);
     static void addBenchmarkResult(const QBenchmarkResult &result);
 
@@ -102,6 +104,7 @@ public:
     static int passCount();
     static int failCount();
     static int skipCount();
+    static int blacklistCount();
 
     static void resetCounters();
 
diff --git a/src/testlib/qtestresult.cpp b/src/testlib/qtestresult.cpp
index 8eb1fa2d9c..8d1a942d8b 100644
--- a/src/testlib/qtestresult.cpp
+++ b/src/testlib/qtestresult.cpp
@@ -62,6 +62,7 @@ namespace QTest
     static const char *currentTestObjectName = 0;
     static bool failed = false;
     static bool skipCurrentTest = false;
+    static bool blacklistCurrentTest = false;
 
     static const char *expectFailComment = 0;
     static int expectFailMode = 0;
@@ -77,10 +78,16 @@ void QTestResult::reset()
 
     QTest::expectFailComment = 0;
     QTest::expectFailMode = 0;
+    QTest::blacklistCurrentTest = false;
 
     QTestLog::resetCounters();
 }
 
+void QTestResult::setBlacklistCurrentTest(bool b)
+{
+    QTest::blacklistCurrentTest = b;
+}
+
 bool QTestResult::currentTestFailed()
 {
     return QTest::failed;
@@ -139,7 +146,10 @@ void QTestResult::finishedCurrentTestDataCleanup()
 {
     // If the current test hasn't failed or been skipped, then it passes.
     if (!QTest::failed && !QTest::skipCurrentTest) {
-        QTestLog::addPass("");
+        if (QTest::blacklistCurrentTest)
+            QTestLog::addBPass("");
+        else
+            QTestLog::addPass("");
     }
 
     QTest::failed = false;
@@ -290,7 +300,10 @@ void QTestResult::addFailure(const char *message, const char *file, int line)
 {
     clearExpectFail();
 
-    QTestLog::addFail(message, file, line);
+    if (QTest::blacklistCurrentTest)
+        QTestLog::addBFail(message, file, line);
+    else
+        QTestLog::addFail(message, file, line);
     QTest::failed = true;
 }
 
diff --git a/src/testlib/qtestresult_p.h b/src/testlib/qtestresult_p.h
index ea8173b169..8454acd5f8 100644
--- a/src/testlib/qtestresult_p.h
+++ b/src/testlib/qtestresult_p.h
@@ -74,6 +74,7 @@ public:
     static void finishedCurrentTestDataCleanup();
     static void finishedCurrentTestFunction();
     static void reset();
+    static void setBlacklistCurrentTest(bool b);
 
     static void addFailure(const char *message, const char *file, int line);
     static bool compare(bool success, const char *failureMsg,
diff --git a/src/testlib/qxmltestlogger.cpp b/src/testlib/qxmltestlogger.cpp
index 5ea28bd46d..52dc5db69f 100644
--- a/src/testlib/qxmltestlogger.cpp
+++ b/src/testlib/qxmltestlogger.cpp
@@ -86,6 +86,10 @@ namespace QTest {
             return "fail";
         case QAbstractTestLogger::XPass:
             return "xpass";
+        case QAbstractTestLogger::BlacklistedPass:
+            return "bpass";
+        case QAbstractTestLogger::BlacklistedFail:
+            return "bfail";
         }
         return "??????";
     }
diff --git a/src/testlib/qxunittestlogger.cpp b/src/testlib/qxunittestlogger.cpp
index fe5b13eeeb..5d860cf1c9 100644
--- a/src/testlib/qxunittestlogger.cpp
+++ b/src/testlib/qxunittestlogger.cpp
@@ -175,6 +175,13 @@ void QXunitTestLogger::addIncident(IncidentTypes type, const char *description,
         ++failureCounter;
         typeBuf = "fail";
         break;
+    case QAbstractTestLogger::BlacklistedPass:
+        typeBuf = "bpass";
+        break;
+    case QAbstractTestLogger::BlacklistedFail:
+        ++failureCounter;
+        typeBuf = "bfail";
+        break;
     default:
         typeBuf = "??????";
         break;
@@ -207,6 +214,13 @@ void QXunitTestLogger::addIncident(IncidentTypes type, const char *description,
         if (!strcmp(oldResult, "pass")) {
             overwrite = true;
         }
+        else if (!strcmp(oldResult, "bpass")) {
+            overwrite = (type == QAbstractTestLogger::XPass || type == QAbstractTestLogger::Fail) || (type == QAbstractTestLogger::XFail)
+                    || (type == QAbstractTestLogger::BlacklistedFail);
+        }
+        else if (!strcmp(oldResult, "bfail")) {
+            overwrite = (type == QAbstractTestLogger::XPass || type == QAbstractTestLogger::Fail) || (type == QAbstractTestLogger::XFail);
+        }
         else if (!strcmp(oldResult, "xfail")) {
             overwrite = (type == QAbstractTestLogger::XPass || type == QAbstractTestLogger::Fail);
         }
diff --git a/src/testlib/testlib.pro b/src/testlib/testlib.pro
index 1fb6d6df18..17a03781c2 100644
--- a/src/testlib/testlib.pro
+++ b/src/testlib/testlib.pro
@@ -33,7 +33,8 @@ HEADERS = qbenchmark.h \
     qtestmouse.h \
     qtestspontaneevent.h \
     qtestsystem.h \
-    qtesttouch.h
+    qtesttouch.h \
+    qtestblacklist_p.h
 
 SOURCES = qtestcase.cpp \
     qtestlog.cpp \
@@ -55,7 +56,9 @@ SOURCES = qtestcase.cpp \
     qtestelement.cpp \
     qtestelementattribute.cpp \
     qtestxunitstreamer.cpp \
-    qxunittestlogger.cpp
+    qxunittestlogger.cpp \
+    qtestblacklist.cpp
+
 DEFINES *= QT_NO_CAST_TO_ASCII \
     QT_NO_CAST_FROM_ASCII \
     QT_NO_DATASTREAM