diff options
author | Lars Knoll <lars.knoll@digia.com> | 2014-09-15 15:39:36 +0200 |
---|---|---|
committer | Lars Knoll <lars.knoll@digia.com> | 2014-09-21 20:58:41 +0200 |
commit | 10414444e1fdc3b91328aba19fb10bd92f6dc899 (patch) | |
tree | 382677984b22b3d96254373926859b6a7f73803e /src/testlib | |
parent | 23a03ebcd17be328508e749afeb2cd962296be3b (diff) |
Add support for blacklisting test functions
We need to have a finer grained control over the tests
we skip in our CI system. This adds a blacklisting
mechanism that allows blacklisting individual test
functions (or even test data) using a set of predefined
matching keys for the operating system and some other
relevant variables.
QTestlib will search for a file called BLACKLIST in the test
directory and parse it if found. The file contains a simple
ini style list of functions to blacklist. For details see
qtestblacklist.cpp.
Change-Id: Id3fae4b264ca99970cbf9f45bfb85fa75c1fd823
Reviewed-by: Lars Knoll <lars.knoll@digia.com>
Diffstat (limited to 'src/testlib')
-rw-r--r-- | src/testlib/qabstracttestlogger_p.h | 4 | ||||
-rw-r--r-- | src/testlib/qplaintestlogger.cpp | 13 | ||||
-rw-r--r-- | src/testlib/qtestblacklist.cpp | 201 | ||||
-rw-r--r-- | src/testlib/qtestblacklist_p.h | 67 | ||||
-rw-r--r-- | src/testlib/qtestcase.cpp | 6 | ||||
-rw-r--r-- | src/testlib/qtestlog.cpp | 25 | ||||
-rw-r--r-- | src/testlib/qtestlog_p.h | 3 | ||||
-rw-r--r-- | src/testlib/qtestresult.cpp | 17 | ||||
-rw-r--r-- | src/testlib/qtestresult_p.h | 1 | ||||
-rw-r--r-- | src/testlib/qxmltestlogger.cpp | 4 | ||||
-rw-r--r-- | src/testlib/qxunittestlogger.cpp | 14 | ||||
-rw-r--r-- | src/testlib/testlib.pro | 7 |
12 files changed, 353 insertions, 9 deletions
diff --git a/src/testlib/qabstracttestlogger_p.h b/src/testlib/qabstracttestlogger_p.h index c549233ddb..6bb54d1af8 100644 --- a/src/testlib/qabstracttestlogger_p.h +++ b/src/testlib/qabstracttestlogger_p.h @@ -68,7 +68,9 @@ public: Pass, XFail, Fail, - XPass + XPass, + BlacklistedPass, + BlacklistedFail }; enum MessageTypes { diff --git a/src/testlib/qplaintestlogger.cpp b/src/testlib/qplaintestlogger.cpp index 774f2cc4fb..74f0290f38 100644 --- a/src/testlib/qplaintestlogger.cpp +++ b/src/testlib/qplaintestlogger.cpp @@ -89,6 +89,10 @@ namespace QTest { return "FAIL! "; case QAbstractTestLogger::XPass: return "XPASS "; + case QAbstractTestLogger::BlacklistedPass: + return "BPASS "; + case QAbstractTestLogger::BlacklistedFail: + return "BFAIL "; } return "??????"; } @@ -351,15 +355,16 @@ void QPlainTestLogger::stopLogging() { char buf[1024]; if (QTestLog::verboseLevel() < 0) { - qsnprintf(buf, sizeof(buf), "Totals: %d passed, %d failed, %d skipped\n", + qsnprintf(buf, sizeof(buf), "Totals: %d passed, %d failed, %d skipped, %d blacklisted\n", QTestLog::passCount(), QTestLog::failCount(), - QTestLog::skipCount()); + QTestLog::skipCount(), QTestLog::blacklistCount()); } else { qsnprintf(buf, sizeof(buf), - "Totals: %d passed, %d failed, %d skipped\n" + "Totals: %d passed, %d failed, %d skipped, %d blacklisted\n" "********* Finished testing of %s *********\n", QTestLog::passCount(), QTestLog::failCount(), - QTestLog::skipCount(), QTestResult::currentTestObjectName()); + QTestLog::skipCount(), QTestLog::blacklistCount(), + QTestResult::currentTestObjectName()); } outputMessage(buf); diff --git a/src/testlib/qtestblacklist.cpp b/src/testlib/qtestblacklist.cpp new file mode 100644 index 0000000000..7921b350b5 --- /dev/null +++ b/src/testlib/qtestblacklist.cpp @@ -0,0 +1,201 @@ +/**************************************************************************** +** +** Copyright (C) 2014 Digia Plc and/or its subsidiary(-ies). +** Contact: http://www.qt-project.org/legal +** +** This file is part of the QtTest module of the Qt Toolkit. +** +** $QT_BEGIN_LICENSE:LGPL$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and Digia. For licensing terms and +** conditions see http://qt.digia.com/licensing. For further information +** use the contact form at http://qt.digia.com/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 2.1 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 2.1 requirements +** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. +** +** In addition, as a special exception, Digia gives you certain additional +** rights. These rights are described in the Digia Qt LGPL Exception +** version 1.1, included in the file LGPL_EXCEPTION.txt in this package. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 3.0 as published by the Free Software +** Foundation and appearing in the file LICENSE.GPL included in the +** packaging of this file. Please review the following information to +** ensure the GNU General Public License version 3.0 requirements will be +** met: http://www.gnu.org/copyleft/gpl.html. +** +** +** $QT_END_LICENSE$ +** +****************************************************************************/ +#include "qtestblacklist_p.h" +#include "qtestresult_p.h" + +#include <QtTest/qtestcase.h> +#include <QtCore/qbytearray.h> +#include <QtCore/qfile.h> + +#include <set> + +QT_BEGIN_NAMESPACE + +/* + The file format is simply a grouped listing of keywords + Ungrouped entries at the beginning apply to the whole testcase + Groups define testfunctions or specific test data to ignore. + After the groups come a list of entries (one per line) that define + for which platform/os combination to ignore the test result. + All keys in a single line have to match to blacklist the test. + + mac + [testFunction] + linux + windows 64bit + [testfunction2:testData] + msvc + + The known keys are listed below: +*/ + +// this table can be extended with new keywords as required +const char *matchedConditions[] = +{ +#ifdef Q_OS_LINUX + "linux", +#endif +#ifdef Q_OS_OSX + "osx", +#endif +#ifdef Q_OS_WIN + "windows", +#endif +#ifdef Q_OS_IOS + "ios", +#endif +#ifdef Q_OS_ANDROID + "android", +#endif +#ifdef Q_OS_QNX + "qnx", +#endif +#ifdef Q_OS_WINRT + "winrt", +#endif +#ifdef Q_OS_WINCE + "wince", +#endif + +#if QT_POINTER_SIZE == 8 + "64bit", +#else + "32bit", +#endif + +#ifdef Q_CC_GNU + "gcc", +#endif +#ifdef Q_CC_CLANG + "clang", +#endif +#ifdef Q_CC_MSVC + "msvc", +#endif + +#ifdef Q_AUTOTEST_EXPORT + "developer-build", +#endif + 0 +}; + + +static bool checkCondition(const QByteArray &condition) +{ + QList<QByteArray> conds = condition.split(' '); + std::set<QByteArray> matches; + const char **m = matchedConditions; + while (*m) { + matches.insert(*m); + ++m; + } + + for (int i = 0; i < conds.size(); ++i) { + QByteArray c = conds.at(i); + bool result = c.startsWith('!'); + if (result) + c = c.mid(1); + + result ^= (matches.find(c) != matches.end()); + if (!result) + return false; + } + return true; +} + +static bool ignoreAll = false; +static std::set<QByteArray> *ignoredTests = 0; + +namespace QTestPrivate { + +void parseBlackList() +{ + QString filename = QTest::qFindTestData(QStringLiteral("BLACKLIST")); + if (filename.isEmpty()) + return; + QFile ignored(filename); + if (!ignored.open(QIODevice::ReadOnly)) + return; + + QByteArray function; + + while (!ignored.atEnd()) { + QByteArray line = ignored.readLine().simplified(); + if (line.isEmpty() || line.startsWith('#')) + continue; + if (line.startsWith('[')) { + function = line.mid(1, line.length() - 2); + continue; + } + bool condition = checkCondition(line); + if (condition) { + if (!function.size()) { + ignoreAll = true; + } else { + if (!ignoredTests) + ignoredTests = new std::set<QByteArray>; + ignoredTests->insert(function); + } + } + } +} + +void checkBlackList(const char *slot, const char *data) +{ + bool ignore = ignoreAll; + + if (!ignore && ignoredTests) { + QByteArray s = slot; + ignore = (ignoredTests->find(s) != ignoredTests->end()); + if (!ignore && data) { + s += ':'; + s += data; + ignore = (ignoredTests->find(s) != ignoredTests->end()); + } + } + + QTestResult::setBlacklistCurrentTest(ignore); +} + +} + + +QT_END_NAMESPACE diff --git a/src/testlib/qtestblacklist_p.h b/src/testlib/qtestblacklist_p.h new file mode 100644 index 0000000000..a9d07ae024 --- /dev/null +++ b/src/testlib/qtestblacklist_p.h @@ -0,0 +1,67 @@ +/**************************************************************************** +** +** Copyright (C) 2014 Digia Plc and/or its subsidiary(-ies). +** Contact: http://www.qt-project.org/legal +** +** This file is part of the QtTest module of the Qt Toolkit. +** +** $QT_BEGIN_LICENSE:LGPL$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and Digia. For licensing terms and +** conditions see http://qt.digia.com/licensing. For further information +** use the contact form at http://qt.digia.com/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 2.1 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 2.1 requirements +** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. +** +** In addition, as a special exception, Digia gives you certain additional +** rights. These rights are described in the Digia Qt LGPL Exception +** version 1.1, included in the file LGPL_EXCEPTION.txt in this package. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 3.0 as published by the Free Software +** Foundation and appearing in the file LICENSE.GPL included in the +** packaging of this file. Please review the following information to +** ensure the GNU General Public License version 3.0 requirements will be +** met: http://www.gnu.org/copyleft/gpl.html. +** +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#ifndef QTESTBLACKLIST_P_H +#define QTESTBLACKLIST_P_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include <QtTest/qtest_global.h> + +QT_BEGIN_NAMESPACE + +namespace QTestPrivate { + void parseBlackList(); + void checkBlackList(const char *slot, const char *data); +} + +QT_END_NAMESPACE + +#endif diff --git a/src/testlib/qtestcase.cpp b/src/testlib/qtestcase.cpp index 739d341233..5d5b298740 100644 --- a/src/testlib/qtestcase.cpp +++ b/src/testlib/qtestcase.cpp @@ -63,6 +63,7 @@ #include <QtTest/private/qsignaldumper_p.h> #include <QtTest/private/qbenchmark_p.h> #include <QtTest/private/cycle_p.h> +#include <QtTest/private/qtestblacklist_p.h> #include <numeric> #include <algorithm> @@ -2016,6 +2017,9 @@ static bool qInvokeTestMethod(const char *slotName, const char *data=0) QTestResult::setSkipCurrentTest(false); if (!data || !qstrcmp(data, table.testData(curDataIndex)->dataTag())) { foundFunction = true; + + QTestPrivate::checkBlackList(slot, dataCount ? table.testData(curDataIndex)->dataTag() : 0); + QTestDataSetter s(curDataIndex >= dataCount ? static_cast<QTestData *>(0) : table.testData(curDataIndex)); @@ -2433,6 +2437,8 @@ int QTest::qExec(QObject *testObject, int argc, char **argv) } #endif + QTestPrivate::parseBlackList(); + QTestResult::reset(); QTEST_ASSERT(testObject); diff --git a/src/testlib/qtestlog.cpp b/src/testlib/qtestlog.cpp index 708a91b557..b7ed3d0763 100644 --- a/src/testlib/qtestlog.cpp +++ b/src/testlib/qtestlog.cpp @@ -84,6 +84,7 @@ namespace QTest { int fails = 0; int passes = 0; int skips = 0; + int blacklists = 0; struct IgnoreResultList { @@ -415,6 +416,25 @@ void QTestLog::addXPass(const char *msg, const char *file, int line) QTest::TestLoggers::addIncident(QAbstractTestLogger::XPass, msg, file, line); } +void QTestLog::addBPass(const char *msg) +{ + QTEST_ASSERT(msg); + + ++QTest::blacklists; + + QTest::TestLoggers::addIncident(QAbstractTestLogger::BlacklistedPass, msg); +} + +void QTestLog::addBFail(const char *msg, const char *file, int line) +{ + QTEST_ASSERT(msg); + QTEST_ASSERT(file); + + ++QTest::blacklists; + + QTest::TestLoggers::addIncident(QAbstractTestLogger::BlacklistedFail, msg, file, line); +} + void QTestLog::addSkip(const char *msg, const char *file, int line) { QTEST_ASSERT(msg); @@ -552,6 +572,11 @@ int QTestLog::skipCount() return QTest::skips; } +int QTestLog::blacklistCount() +{ + return QTest::blacklists; +} + void QTestLog::resetCounters() { QTest::passes = 0; diff --git a/src/testlib/qtestlog_p.h b/src/testlib/qtestlog_p.h index a987c45806..540ffe75eb 100644 --- a/src/testlib/qtestlog_p.h +++ b/src/testlib/qtestlog_p.h @@ -72,6 +72,8 @@ public: static void addFail(const char *msg, const char *file, int line); static void addXFail(const char *msg, const char *file, int line); static void addXPass(const char *msg, const char *file, int line); + static void addBPass(const char *msg); + static void addBFail(const char *msg, const char *file, int line); static void addSkip(const char *msg, const char *file, int line); static void addBenchmarkResult(const QBenchmarkResult &result); @@ -102,6 +104,7 @@ public: static int passCount(); static int failCount(); static int skipCount(); + static int blacklistCount(); static void resetCounters(); diff --git a/src/testlib/qtestresult.cpp b/src/testlib/qtestresult.cpp index 8eb1fa2d9c..8d1a942d8b 100644 --- a/src/testlib/qtestresult.cpp +++ b/src/testlib/qtestresult.cpp @@ -62,6 +62,7 @@ namespace QTest static const char *currentTestObjectName = 0; static bool failed = false; static bool skipCurrentTest = false; + static bool blacklistCurrentTest = false; static const char *expectFailComment = 0; static int expectFailMode = 0; @@ -77,10 +78,16 @@ void QTestResult::reset() QTest::expectFailComment = 0; QTest::expectFailMode = 0; + QTest::blacklistCurrentTest = false; QTestLog::resetCounters(); } +void QTestResult::setBlacklistCurrentTest(bool b) +{ + QTest::blacklistCurrentTest = b; +} + bool QTestResult::currentTestFailed() { return QTest::failed; @@ -139,7 +146,10 @@ void QTestResult::finishedCurrentTestDataCleanup() { // If the current test hasn't failed or been skipped, then it passes. if (!QTest::failed && !QTest::skipCurrentTest) { - QTestLog::addPass(""); + if (QTest::blacklistCurrentTest) + QTestLog::addBPass(""); + else + QTestLog::addPass(""); } QTest::failed = false; @@ -290,7 +300,10 @@ void QTestResult::addFailure(const char *message, const char *file, int line) { clearExpectFail(); - QTestLog::addFail(message, file, line); + if (QTest::blacklistCurrentTest) + QTestLog::addBFail(message, file, line); + else + QTestLog::addFail(message, file, line); QTest::failed = true; } diff --git a/src/testlib/qtestresult_p.h b/src/testlib/qtestresult_p.h index ea8173b169..8454acd5f8 100644 --- a/src/testlib/qtestresult_p.h +++ b/src/testlib/qtestresult_p.h @@ -74,6 +74,7 @@ public: static void finishedCurrentTestDataCleanup(); static void finishedCurrentTestFunction(); static void reset(); + static void setBlacklistCurrentTest(bool b); static void addFailure(const char *message, const char *file, int line); static bool compare(bool success, const char *failureMsg, diff --git a/src/testlib/qxmltestlogger.cpp b/src/testlib/qxmltestlogger.cpp index 5ea28bd46d..52dc5db69f 100644 --- a/src/testlib/qxmltestlogger.cpp +++ b/src/testlib/qxmltestlogger.cpp @@ -86,6 +86,10 @@ namespace QTest { return "fail"; case QAbstractTestLogger::XPass: return "xpass"; + case QAbstractTestLogger::BlacklistedPass: + return "bpass"; + case QAbstractTestLogger::BlacklistedFail: + return "bfail"; } return "??????"; } diff --git a/src/testlib/qxunittestlogger.cpp b/src/testlib/qxunittestlogger.cpp index fe5b13eeeb..5d860cf1c9 100644 --- a/src/testlib/qxunittestlogger.cpp +++ b/src/testlib/qxunittestlogger.cpp @@ -175,6 +175,13 @@ void QXunitTestLogger::addIncident(IncidentTypes type, const char *description, ++failureCounter; typeBuf = "fail"; break; + case QAbstractTestLogger::BlacklistedPass: + typeBuf = "bpass"; + break; + case QAbstractTestLogger::BlacklistedFail: + ++failureCounter; + typeBuf = "bfail"; + break; default: typeBuf = "??????"; break; @@ -207,6 +214,13 @@ void QXunitTestLogger::addIncident(IncidentTypes type, const char *description, if (!strcmp(oldResult, "pass")) { overwrite = true; } + else if (!strcmp(oldResult, "bpass")) { + overwrite = (type == QAbstractTestLogger::XPass || type == QAbstractTestLogger::Fail) || (type == QAbstractTestLogger::XFail) + || (type == QAbstractTestLogger::BlacklistedFail); + } + else if (!strcmp(oldResult, "bfail")) { + overwrite = (type == QAbstractTestLogger::XPass || type == QAbstractTestLogger::Fail) || (type == QAbstractTestLogger::XFail); + } else if (!strcmp(oldResult, "xfail")) { overwrite = (type == QAbstractTestLogger::XPass || type == QAbstractTestLogger::Fail); } diff --git a/src/testlib/testlib.pro b/src/testlib/testlib.pro index 1fb6d6df18..17a03781c2 100644 --- a/src/testlib/testlib.pro +++ b/src/testlib/testlib.pro @@ -33,7 +33,8 @@ HEADERS = qbenchmark.h \ qtestmouse.h \ qtestspontaneevent.h \ qtestsystem.h \ - qtesttouch.h + qtesttouch.h \ + qtestblacklist_p.h SOURCES = qtestcase.cpp \ qtestlog.cpp \ @@ -55,7 +56,9 @@ SOURCES = qtestcase.cpp \ qtestelement.cpp \ qtestelementattribute.cpp \ qtestxunitstreamer.cpp \ - qxunittestlogger.cpp + qxunittestlogger.cpp \ + qtestblacklist.cpp + DEFINES *= QT_NO_CAST_TO_ASCII \ QT_NO_CAST_FROM_ASCII \ QT_NO_DATASTREAM |