From 27fe56d3e7eedb2f8c45746880689c1e9e9b56b4 Mon Sep 17 00:00:00 2001 From: Kari Oikarinen Date: Wed, 9 May 2018 14:58:36 +0300 Subject: Exit test executable with failure if specified test functions are not found If QtQuickTest test executable was given filters that didn't match in any of the qml files run, it exited successfully. In combination with a Coin bug when the test function name contains a space, this allowed failing tests to pass the CI. If the test function fails, the repeat attempts would pass two arguments to the executable. Neither of those hit, but that wasn't considered a problem. Check that all of the test functions named on the command line are actually executed during the whole run and otherwise exit with a non-zero exit code. I assume there's no duplicates in the whole names of test functions scoped with testcase names. Task-number: QTBUG-68197 Change-Id: Icf7fe263945403f02920522dfd187aeb76b7cb3c Reviewed-by: Qt CI Bot Reviewed-by: Kalle Viironen Reviewed-by: Ville Voutilainen --- tests/auto/quicktest/quicktest.pro | 3 +- .../testfiltering/quicktestmain/quicktestmain.cpp | 29 +++++ .../testfiltering/quicktestmain/quicktestmain.pro | 11 ++ .../testfiltering/quicktestmain/tst_first.qml | 38 ++++++ .../testfiltering/quicktestmain/tst_second.qml | 38 ++++++ tests/auto/quicktest/testfiltering/test/test.pro | 5 + .../auto/quicktest/testfiltering/testfiltering.pro | 4 + .../quicktest/testfiltering/tst_testfiltering.cpp | 135 +++++++++++++++++++++ 8 files changed, 262 insertions(+), 1 deletion(-) create mode 100644 tests/auto/quicktest/testfiltering/quicktestmain/quicktestmain.cpp create mode 100644 tests/auto/quicktest/testfiltering/quicktestmain/quicktestmain.pro create mode 100644 tests/auto/quicktest/testfiltering/quicktestmain/tst_first.qml create mode 100644 tests/auto/quicktest/testfiltering/quicktestmain/tst_second.qml create mode 100644 tests/auto/quicktest/testfiltering/test/test.pro create mode 100644 tests/auto/quicktest/testfiltering/testfiltering.pro create mode 100644 tests/auto/quicktest/testfiltering/tst_testfiltering.cpp (limited to 'tests/auto/quicktest') diff --git a/tests/auto/quicktest/quicktest.pro b/tests/auto/quicktest/quicktest.pro index 9d909f1d16..0e3f257e33 100644 --- a/tests/auto/quicktest/quicktest.pro +++ b/tests/auto/quicktest/quicktest.pro @@ -1,4 +1,5 @@ TEMPLATE = subdirs SUBDIRS = \ signalspy \ - quicktestmainwithsetup + quicktestmainwithsetup \ + testfiltering diff --git a/tests/auto/quicktest/testfiltering/quicktestmain/quicktestmain.cpp b/tests/auto/quicktest/testfiltering/quicktestmain/quicktestmain.cpp new file mode 100644 index 0000000000..656911f842 --- /dev/null +++ b/tests/auto/quicktest/testfiltering/quicktestmain/quicktestmain.cpp @@ -0,0 +1,29 @@ +/**************************************************************************** +** +** Copyright (C) 2018 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of the test suite of the Qt Toolkit. +** +** $QT_BEGIN_LICENSE:GPL-EXCEPT$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 3 as published by the Free Software +** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ +#include +QUICK_TEST_MAIN(quicktestmain) diff --git a/tests/auto/quicktest/testfiltering/quicktestmain/quicktestmain.pro b/tests/auto/quicktest/testfiltering/quicktestmain/quicktestmain.pro new file mode 100644 index 0000000000..7b3e734cb4 --- /dev/null +++ b/tests/auto/quicktest/testfiltering/quicktestmain/quicktestmain.pro @@ -0,0 +1,11 @@ +CONFIG += qmltestcase +macos:CONFIG -= app_bundle +TARGET = quicktestmain + +DEFINES += QT_QMLTEST_DATADIR=\\\"$${PWD}\\\" + +SOURCES += quicktestmain.cpp + +TESTDATA += $$PWD/*.qml + +DESTDIR = ./ diff --git a/tests/auto/quicktest/testfiltering/quicktestmain/tst_first.qml b/tests/auto/quicktest/testfiltering/quicktestmain/tst_first.qml new file mode 100644 index 0000000000..55c9612b78 --- /dev/null +++ b/tests/auto/quicktest/testfiltering/quicktestmain/tst_first.qml @@ -0,0 +1,38 @@ +/**************************************************************************** +** +** Copyright (C) 2018 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of the test suite of the Qt Toolkit. +** +** $QT_BEGIN_LICENSE:GPL-EXCEPT$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 3 as published by the Free Software +** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +import QtQuick 2.3 +import QtTest 1.1 + +TestCase { + name: "First" + + function test_foo() { } + function test_bar() { } + function test_baz() { } +} diff --git a/tests/auto/quicktest/testfiltering/quicktestmain/tst_second.qml b/tests/auto/quicktest/testfiltering/quicktestmain/tst_second.qml new file mode 100644 index 0000000000..2143d93e12 --- /dev/null +++ b/tests/auto/quicktest/testfiltering/quicktestmain/tst_second.qml @@ -0,0 +1,38 @@ +/**************************************************************************** +** +** Copyright (C) 2018 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of the test suite of the Qt Toolkit. +** +** $QT_BEGIN_LICENSE:GPL-EXCEPT$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 3 as published by the Free Software +** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +import QtQuick 2.3 +import QtTest 1.1 + +TestCase { + name: "Second" + + function test_dupfoo() { } + function test_dupbar() { } + function test_dupbaz() { } +} diff --git a/tests/auto/quicktest/testfiltering/test/test.pro b/tests/auto/quicktest/testfiltering/test/test.pro new file mode 100644 index 0000000000..cecbdca725 --- /dev/null +++ b/tests/auto/quicktest/testfiltering/test/test.pro @@ -0,0 +1,5 @@ +CONFIG += testcase +QT += testlib +TARGET = ../tst_testfiltering + +SOURCES = ../tst_testfiltering.cpp diff --git a/tests/auto/quicktest/testfiltering/testfiltering.pro b/tests/auto/quicktest/testfiltering/testfiltering.pro new file mode 100644 index 0000000000..ae7a039b8b --- /dev/null +++ b/tests/auto/quicktest/testfiltering/testfiltering.pro @@ -0,0 +1,4 @@ +TEMPLATE = subdirs +SUBDIRS = \ + test \ + quicktestmain diff --git a/tests/auto/quicktest/testfiltering/tst_testfiltering.cpp b/tests/auto/quicktest/testfiltering/tst_testfiltering.cpp new file mode 100644 index 0000000000..72bb8f02b7 --- /dev/null +++ b/tests/auto/quicktest/testfiltering/tst_testfiltering.cpp @@ -0,0 +1,135 @@ +/**************************************************************************** +** +** Copyright (C) 2018 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of the test suite of the Qt Toolkit. +** +** $QT_BEGIN_LICENSE:GPL-EXCEPT$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 3 as published by the Free Software +** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ +#include +#include + +class tst_TestFiltering : public QObject +{ + Q_OBJECT +private slots: + void noFilters(); + void oneMatchingFilter(); + void filterThatDoesntMatch(); + void twoFilters(); + void twoFiltersWithOneMatch(); + void manyFilters(); +}; + + +const QString testExe = +#if defined(Q_OS_WIN) + QFINDTESTDATA("quicktestmain/quicktestmain.exe"); +#else + QFINDTESTDATA("quicktestmain/quicktestmain"); +#endif + +void tst_TestFiltering::noFilters() +{ + QProcess process; + process.start(testExe); + + QVERIFY(process.waitForFinished()); + + const QString output = process.readAll(); + QVERIFY(output.contains(QLatin1String("Totals: 10 passed"))); + QCOMPARE(process.exitStatus(), QProcess::NormalExit); + QCOMPARE(process.exitCode(), 0); +} + +void tst_TestFiltering::oneMatchingFilter() +{ + QProcess process; + process.start(testExe, {QLatin1String("First::test_bar")}); + + QVERIFY(process.waitForFinished()); + + const QString output = process.readAll(); + QVERIFY(output.contains(QLatin1String("Totals: 3 passed"))); + QCOMPARE(process.exitStatus(), QProcess::NormalExit); + QCOMPARE(process.exitCode(), 0); +} + +void tst_TestFiltering::filterThatDoesntMatch() +{ + QProcess process; + process.start(testExe, {QLatin1String("First::test_nonexisting")}); + + QVERIFY(process.waitForFinished()); + + QCOMPARE(process.exitStatus(), QProcess::NormalExit); + QCOMPARE(process.exitCode(), 1); +} + +void tst_TestFiltering::twoFilters() +{ + QProcess process; + process.start(testExe, + {QLatin1String("Second::test_dupfoo"), QLatin1String("Second::test_dupbaz")}); + + QVERIFY(process.waitForFinished()); + + const QString output = process.readAll(); + QVERIFY(output.contains(QLatin1String("Totals: 4 passed"))); + QCOMPARE(process.exitStatus(), QProcess::NormalExit); + QCOMPARE(process.exitCode(), 0); +} + +void tst_TestFiltering::twoFiltersWithOneMatch() +{ + QProcess process; + process.start(testExe, + {QLatin1String("First::test_foo"), QLatin1String("Second::test_nonexisting")}); + + QVERIFY(process.waitForFinished()); + + const QString output = process.readAll(); + QVERIFY(output.contains(QLatin1String("Totals: 3 passed"))); + QCOMPARE(process.exitStatus(), QProcess::NormalExit); + QCOMPARE(process.exitCode(), 1); +} + +void tst_TestFiltering::manyFilters() +{ + QProcess process; + process.start(testExe, + {QLatin1String("First::test_foo"), + QLatin1String("First::test_baz"), + QLatin1String("Second::test_dupfoo"), + QLatin1String("Second::test_dupbaz")}); + + QVERIFY(process.waitForFinished()); + + const QString output = process.readAll(); + QVERIFY(output.contains(QLatin1String("Totals: 8 passed"))); + QCOMPARE(process.exitStatus(), QProcess::NormalExit); + QCOMPARE(process.exitCode(), 0); +} + +QTEST_MAIN(tst_TestFiltering); + +#include "tst_testfiltering.moc" -- cgit v1.2.3