diff options
Diffstat (limited to 'tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp')
| -rw-r--r-- | tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp | 364 |
1 files changed, 248 insertions, 116 deletions
diff --git a/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp b/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp index fb6300b4a8..2e67012cc8 100644 --- a/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp +++ b/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp @@ -1,30 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the test suite of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:GPL-EXCEPT$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 3 as published by the Free Software -** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only #include <QtCore/QCoreApplication> @@ -37,29 +12,66 @@ class tst_ExpectFail: public QObject Q_OBJECT private slots: + void cleanupTestCase() const; + void init() const; void xfailAndContinue() const; void xfailAndAbort() const; + void xfailContinueSkip() const; + void xfailAbortSkip() const; void xfailTwice() const; + void xfailDataDrivenTwice() const; + void xfailDataDrivenTwice_data() const { xfailDataDriven_data(false); } void xfailWithQString() const; - void xfailDataDrivenWithQVerify_data() const; + void xfailDataDrivenWithQString_data() const { xfailDataDriven_data(false); } + void xfailDataDrivenWithQString() const; + void xfailDataDrivenWithQVerify_data() const { xfailDataDriven_data(false); } void xfailDataDrivenWithQVerify() const; - void xfailDataDrivenWithQCompare_data() const; + void xfailDataDrivenWithQCompare_data() const { xfailDataDriven_data(false); } void xfailDataDrivenWithQCompare() const; - void xfailOnWrongRow_data() const; + void xfailOnWrongRow_data() const { xfailDataDriven_data(true); } void xfailOnWrongRow() const; - void xfailOnAnyRow_data() const; + void xfailOnAnyRow_data() const { xfailDataDriven_data(true); } void xfailOnAnyRow() const; - void xfailWithoutVerify_data() const; - void xfailWithoutVerify() const; - void xpass() const; - void xpassDataDrivenWithQVerify_data() const; - void xpassDataDrivenWithQVerify() const; - void xpassDataDrivenWithQCompare_data() const; - void xpassDataDrivenWithQCompare() const; + void xfailWithoutCheck_data() const { xfailDataDriven_data(true); } + void xfailWithoutCheck() const; + void xpassAbort() const; + void xpassAbortSkip() const; + void xpassAbortXfailContinue() const; + void xpassContinue() const; + void xpassContinueSkip() const; + void xpassContinueXfailAbort() const; + void xpassAbortDataDrivenWithQVerify_data() const { xpassDataDriven_data(); } + void xpassAbortDataDrivenWithQVerify() const; + void xpassContinueDataDrivenWithQVerify_data() const { xpassDataDriven_data(); } + void xpassContinueDataDrivenWithQVerify() const; + void xpassAbortDataDrivenWithQCompare_data() const { xpassDataDriven_data(); } + void xpassAbortDataDrivenWithQCompare() const; + void xpassContinueDataDrivenWithQCompare_data() const { xpassDataDriven_data(); } + void xpassContinueDataDrivenWithQCompare() const; + +private: + void xfailDataDriven_data(bool failOnly) const; + void xpassDataDriven_data() const; }; +static int casesTested = 2; +// What the totals line's numbers *should* be: +static int passed = 2, failed = 0, skipped = 0; +// Total and passed get {init,cleanup}TestCase() in addition to the actual tests. + +void tst_ExpectFail::init() const +{ + ++casesTested; +} + +void tst_ExpectFail::cleanupTestCase() const +{ + qDebug("Totals should add up to %d: %d passed, %d failed, %d skipped", + casesTested, passed, failed, skipped); +} void tst_ExpectFail::xfailAndContinue() const { + ++passed; qDebug("begin"); QEXPECT_FAIL("", "This should xfail", Continue); QVERIFY(false); @@ -68,6 +80,7 @@ void tst_ExpectFail::xfailAndContinue() const void tst_ExpectFail::xfailAndAbort() const { + ++passed; qDebug("begin"); QEXPECT_FAIL("", "This should xfail", Abort); QVERIFY(false); @@ -76,11 +89,52 @@ void tst_ExpectFail::xfailAndAbort() const QVERIFY2(false, "This should not be reached"); } +void tst_ExpectFail::xfailContinueSkip() const +{ + ++skipped; + QEXPECT_FAIL("", "This should xfail then skip", Continue); + QVERIFY(false); + QSKIP("This skip should be reported and counted"); +} + +void tst_ExpectFail::xfailAbortSkip() const +{ + ++passed; + QEXPECT_FAIL("", "This should xfail", Abort); + QVERIFY(false); + + // If we get here the test did not correctly abort on the previous QVERIFY. + QSKIP("This skip should not be reached"); +} + void tst_ExpectFail::xfailTwice() const { - QEXPECT_FAIL("", "Calling QEXPECT_FAIL once is fine", Abort); + ++failed; + QEXPECT_FAIL("", "Calling QEXPECT_FAIL once is fine", Continue); QEXPECT_FAIL("", "Calling QEXPECT_FAIL when already expecting a failure is " - "an error and should abort this test function", Abort); + "an error and should abort this test function", Continue); + + // If we get here the test did not correctly abort on the double call to QEXPECT_FAIL. + QVERIFY2(false, "This should not be reached"); +} + +void tst_ExpectFail::xfailDataDrivenTwice() const +{ + ++failed; + // Same with data-driven cases (twist semantics of unused shouldPass; we + // have four combinations to test): + QEXPECT_FAIL("Pass Abort", "Calling QEXPECT_FAIL once on a test-case is fine", Abort); + QEXPECT_FAIL("Pass Abort", "Calling QEXPECT_FAIL when already expecting a failure is " + "an error and should abort this test function", Continue); + QEXPECT_FAIL("Fail Abort", "Calling QEXPECT_FAIL once on a test-case is fine", Abort); + QEXPECT_FAIL("Fail Abort", "Calling QEXPECT_FAIL when already expecting a failure is " + "an error and should abort this test function", Abort); + QEXPECT_FAIL("Pass Continue", "Calling QEXPECT_FAIL once on a test-case is fine", Continue); + QEXPECT_FAIL("Pass Continue", "Calling QEXPECT_FAIL when already expecting a failure is " + "an error and should abort this test function", Abort); + QEXPECT_FAIL("Fail Continue", "Calling QEXPECT_FAIL once on a test-case is fine", Continue); + QEXPECT_FAIL("Fail Continue", "Calling QEXPECT_FAIL when already expecting a failure is " + "an error and should abort this test function", Continue); // If we get here the test did not correctly abort on the double call to QEXPECT_FAIL. QVERIFY2(false, "This should not be reached"); @@ -88,138 +142,174 @@ void tst_ExpectFail::xfailTwice() const void tst_ExpectFail::xfailWithQString() const { + ++passed; QEXPECT_FAIL("", QString("A string").toLatin1().constData(), Continue); QVERIFY(false); int bugNo = 5; - QString msg("The message"); - QEXPECT_FAIL( "", QString("Bug %1 (%2)").arg(bugNo).arg(msg).toLatin1().constData(), Continue); + QLatin1String msg("The message"); + QEXPECT_FAIL("", QString("Bug %1 (%2)").arg(bugNo).arg(msg).toLatin1().constData(), Continue); QVERIFY(false); } -void tst_ExpectFail::xfailDataDrivenWithQVerify_data() const +void tst_ExpectFail::xfailDataDrivenWithQString() const { - QTest::addColumn<bool>("shouldPass"); - QTest::addColumn<QTest::TestFailMode>("failMode"); + // This test does not (yet) distinguish the two Pass cases. + QFETCH(bool, shouldPass); + QFETCH(QTest::TestFailMode, failMode); + if (shouldPass || failMode == QTest::Continue) + ++skipped; + else + ++passed; - QTest::newRow("Pass 1") << true << QTest::Abort; - QTest::newRow("Pass 2") << true << QTest::Continue; - QTest::newRow("Abort") << false << QTest::Abort; - QTest::newRow("Continue") << false << QTest::Continue; + QEXPECT_FAIL("Fail Abort", QString("A string").toLatin1().constData(), Abort); + QEXPECT_FAIL("Fail Continue", QString("A string").toLatin1().constData(), Continue); + // TODO: why aren't QVERIFY2()'s smessages seen ? + QVERIFY2(shouldPass, "Both Fail cases should XFAIL here"); + // Fail Abort should be gone now. + + int bugNo = 5; + QLatin1String msg("The message"); + QEXPECT_FAIL("Fail Continue", qPrintable(QString("Bug %1 (%2)").arg(bugNo).arg(msg)), Continue); + QVERIFY2(shouldPass, "Only Fail Continue should see this"); + + // FAIL is a pass, and SKIP trumps pass. + QSKIP("Each Continue or Pass reports this and increments skip-count"); } void tst_ExpectFail::xfailDataDrivenWithQVerify() const { + // This test does not (yet) distinguish the two Pass cases. + ++passed; QFETCH(bool, shouldPass); QFETCH(QTest::TestFailMode, failMode); - // You can't pass a variable as the last parameter of QEXPECT_FAIL, - // because the macro adds "QTest::" in front of the last parameter. - // That is why the following code appears to be a little strange. - if (!shouldPass) { - if (failMode == QTest::Abort) - QEXPECT_FAIL(QTest::currentDataTag(), "This test should xfail", Abort); - else - QEXPECT_FAIL(QTest::currentDataTag(), "This test should xfail", Continue); - } - - QVERIFY(shouldPass); + QEXPECT_FAIL("Fail Abort", "This test should xfail", Abort); + QEXPECT_FAIL("Fail Continue", "This test should xfail", Continue); + QVERIFY2(shouldPass, "Both Fail cases should XFAIL here"); - // If we get here, we either expected to pass or we expected to + // If we get here, either we expected to pass or we expected to // fail and the failure mode was Continue. if (!shouldPass) QCOMPARE(failMode, QTest::Continue); } -void tst_ExpectFail::xfailDataDrivenWithQCompare_data() const +void tst_ExpectFail::xfailDataDriven_data(bool failOnly) const { QTest::addColumn<bool>("shouldPass"); QTest::addColumn<QTest::TestFailMode>("failMode"); - QTest::newRow("Pass 1") << true << QTest::Abort; - QTest::newRow("Pass 2") << true << QTest::Continue; - QTest::newRow("Abort") << false << QTest::Abort; - QTest::newRow("Continue") << false << QTest::Continue; + if (!failOnly) { + QTest::newRow("Pass Abort") << true << QTest::Abort; + QTest::newRow("Pass Continue") << true << QTest::Continue; + } + QTest::newRow("Fail Abort") << false << QTest::Abort; + QTest::newRow("Fail Continue") << false << QTest::Continue; } void tst_ExpectFail::xfailDataDrivenWithQCompare() const { + // This test does not (yet) distinguish the two Pass cases. + ++passed; QFETCH(bool, shouldPass); QFETCH(QTest::TestFailMode, failMode); - // You can't pass a variable as the last parameter of QEXPECT_FAIL, - // because the macro adds "QTest::" in front of the last parameter. - // That is why the following code appears to be a little strange. - if (!shouldPass) { - if (failMode == QTest::Abort) - QEXPECT_FAIL(QTest::currentDataTag(), "This test should xfail", Abort); - else - QEXPECT_FAIL(QTest::currentDataTag(), "This test should xfail", Continue); - } + QEXPECT_FAIL("Fail Abort", "This test should xfail", Abort); + QEXPECT_FAIL("Fail Continue", "This test should xfail", Continue); QCOMPARE(1, shouldPass ? 1 : 2); - // If we get here, we either expected to pass or we expected to + // If we get here, either we expected to pass or we expected to // fail and the failure mode was Continue. if (!shouldPass) QCOMPARE(failMode, QTest::Continue); } -void tst_ExpectFail::xfailOnWrongRow_data() const -{ - QTest::addColumn<int>("dummy"); - - QTest::newRow("right row") << 0; -} - void tst_ExpectFail::xfailOnWrongRow() const { - // QEXPECT_FAIL for a row that is not the current row should be ignored. - QEXPECT_FAIL("wrong row", "This xfail should be ignored", Abort); - QVERIFY(true); -} - -void tst_ExpectFail::xfailOnAnyRow_data() const -{ - QTest::addColumn<int>("dummy"); - - QTest::newRow("first row") << 0; - QTest::newRow("second row") << 1; + ++passed; + qDebug("Should pass (*not* xpass), despite test-case name"); + // QEXPECT_FAIL for a row that does not exist should be ignored. + // (It might be conditional data(), so exist in other circumstances.) + QFETCH(QTest::TestFailMode, failMode); + // You can't pass a variable as the last parameter of QEXPECT_FAIL, + // because the macro adds "QTest::" in front of the last parameter. + // That is why the following code appears to be a little strange. + if (failMode == QTest::Abort) + QEXPECT_FAIL("wrong row", "This xfail should be ignored", Abort); + else + QEXPECT_FAIL("wrong row", "This xfail should be ignored", Continue); + QTEST(false, "shouldPass"); // _data skips the passing tests as pointless } void tst_ExpectFail::xfailOnAnyRow() const { + ++passed; // In a data-driven test, passing an empty first parameter to QEXPECT_FAIL // should mean that the failure is expected for all data rows. - QEXPECT_FAIL("", "This test should xfail", Abort); - QVERIFY(false); + QFETCH(QTest::TestFailMode, failMode); + // You can't pass a variable as the last parameter of QEXPECT_FAIL, + // because the macro adds "QTest::" in front of the last parameter. + // That is why the following code appears to be a little strange. + if (failMode == QTest::Abort) + QEXPECT_FAIL("", "This test should xfail", Abort); + else + QEXPECT_FAIL("", "This test should xfail", Continue); + QTEST(true, "shouldPass"); // _data skips the passing tests as pointless +} + +void tst_ExpectFail::xfailWithoutCheck() const +{ + ++failed; + qDebug("Should fail (*not* xfail), despite test-case name"); + QTEST(false, "shouldPass"); // _data skips the passing tests as pass/fail is irrelevant + QEXPECT_FAIL("Fail Abort", "Calling QEXPECT_FAIL without any subsequent check is an error", + Abort); + QEXPECT_FAIL("Fail Continue", "Calling QEXPECT_FAIL without any subsequent check is an error", + Continue); } -void tst_ExpectFail::xfailWithoutVerify_data() const +void tst_ExpectFail::xpassAbort() const { - QTest::addColumn<int>("dummy"); + ++failed; + QEXPECT_FAIL("", "This test should xpass", Abort); + QVERIFY(true); - QTest::newRow("first row") << 0; - QTest::newRow("second row") << 1; + // If we get here the test did not correctly abort on the previous + // unexpected pass. + QVERIFY2(false, "This should not be reached"); } -void tst_ExpectFail::xfailWithoutVerify() const +void tst_ExpectFail::xpassAbortSkip() const { + ++failed; + QEXPECT_FAIL("", "This test should xpass", Abort); QVERIFY(true); - QEXPECT_FAIL("", "This expected failure should be ignored", Abort); + + QSKIP("This should not be reached (and not add to skip-count)"); } -void tst_ExpectFail::xpass() const +void tst_ExpectFail::xpassAbortXfailContinue() const { + ++failed; QEXPECT_FAIL("", "This test should xpass", Abort); QVERIFY(true); // If we get here the test did not correctly abort on the previous // unexpected pass. + QEXPECT_FAIL("", "This should not be reached", Continue); QVERIFY2(false, "This should not be reached"); } -void tst_ExpectFail::xpassDataDrivenWithQVerify_data() const +void tst_ExpectFail::xpassContinue() const +{ + ++failed; + QEXPECT_FAIL("", "This test should xpass", Continue); + QVERIFY(true); + qDebug("This should be reached"); +} + +void tst_ExpectFail::xpassDataDriven_data() const { QTest::addColumn<bool>("shouldXPass"); @@ -227,38 +317,80 @@ void tst_ExpectFail::xpassDataDrivenWithQVerify_data() const QTest::newRow("Pass") << false; } -void tst_ExpectFail::xpassDataDrivenWithQVerify() const +void tst_ExpectFail::xpassContinueSkip() const { - QFETCH(bool, shouldXPass); + ++failed; // and *not* ++skipped + QEXPECT_FAIL("", "This test should xpass", Continue); + QVERIFY(true); + QSKIP("This should be reached but not increment skip-count"); +} +void tst_ExpectFail::xpassContinueXfailAbort() const +{ + ++failed; + QEXPECT_FAIL("", "This test should xpass", Continue); + QVERIFY(true); + QEXPECT_FAIL("", "This test should xfail but not add to totals", Abort); + QVERIFY(false); + QVERIFY2(false, "This should not be reached"); +} + +void tst_ExpectFail::xpassAbortDataDrivenWithQVerify() const +{ + QFETCH(bool, shouldXPass); if (shouldXPass) - QEXPECT_FAIL(QTest::currentDataTag(), "This test should xpass", Abort); + ++failed; + else + ++passed; + QEXPECT_FAIL("XPass", "This test-row should xpass", Abort); QVERIFY(true); // We should only get here if the test wasn't supposed to xpass. - QVERIFY2(!shouldXPass, "Test failed to terminate on XPASS"); + QVERIFY2(!shouldXPass, "Test failed to Abort on XPASS"); } -void tst_ExpectFail::xpassDataDrivenWithQCompare_data() const +void tst_ExpectFail::xpassContinueDataDrivenWithQVerify() const { - QTest::addColumn<bool>("shouldXPass"); + QFETCH(bool, shouldXPass); + if (shouldXPass) + ++failed; + else + ++passed; - QTest::newRow("XPass") << true; - QTest::newRow("Pass") << false; + QEXPECT_FAIL("XPass", "This test-row should xpass", Continue); + QVERIFY(true); + + qDebug(shouldXPass ? "Test should Continue past XPASS" : "Test should simply PASS"); } -void tst_ExpectFail::xpassDataDrivenWithQCompare() const +void tst_ExpectFail::xpassAbortDataDrivenWithQCompare() const { QFETCH(bool, shouldXPass); - if (shouldXPass) - QEXPECT_FAIL(QTest::currentDataTag(), "This test should xpass", Abort); + ++failed; + else + ++passed; + QEXPECT_FAIL("XPass", "This test should xpass", Abort); QCOMPARE(1, 1); // We should only get here if the test wasn't supposed to xpass. - QVERIFY2(!shouldXPass, "Test failed to terminate on XPASS"); + QVERIFY2(!shouldXPass, "Test failed to Abort on XPASS"); +} + +void tst_ExpectFail::xpassContinueDataDrivenWithQCompare() const +{ + QFETCH(bool, shouldXPass); + if (shouldXPass) + ++failed; + else + ++passed; + + QEXPECT_FAIL("XPass", "This test should xpass", Continue); + QCOMPARE(1, 1); + + qDebug(shouldXPass ? "Test should Continue past XPASS" : "Test should simply PASS"); } QTEST_MAIN(tst_ExpectFail) |