Exit test executable with failure if specified test functions are not found

If QtQuickTest test executable was given filters that didn't match in any of the
qml files run, it exited successfully.

In combination with a Coin bug when the test function name contains a space,
this allowed failing tests to pass the CI. If the test function fails, the
repeat attempts would pass two arguments to the executable. Neither of those
hit, but that wasn't considered a problem.

Check that all of the test functions named on the command line are actually
executed during the whole run and otherwise exit with a non-zero exit code.

I assume there's no duplicates in the whole names of test functions scoped with
testcase names.

Task-number: QTBUG-68197
Change-Id: Icf7fe263945403f02920522dfd187aeb76b7cb3c
Reviewed-by: Qt CI Bot <qt_ci_bot@qt-project.org>
Reviewed-by: Kalle Viironen <kalle.viironen@qt.io>
Reviewed-by: Ville Voutilainen <ville.voutilainen@qt.io>

8 files changed, 262 insertions, 1 deletions

diff --git a/tests/auto/quicktest/quicktest.pro b/tests/auto/quicktest/quicktest.pro
index 9d909f1d16..0e3f257e33 100644
--- a/tests/auto/quicktest/quicktest.pro
+++ b/tests/auto/quicktest/quicktest.pro
@@ -1,4 +1,5 @@
 TEMPLATE = subdirs
 SUBDIRS = \
     signalspy \
-    quicktestmainwithsetup
+    quicktestmainwithsetup \
+    testfiltering
diff --git a/tests/auto/quicktest/testfiltering/quicktestmain/quicktestmain.cpp b/tests/auto/quicktest/testfiltering/quicktestmain/quicktestmain.cpp
new file mode 100644
index 0000000000..656911f842
--- /dev/null
+++ b/tests/auto/quicktest/testfiltering/quicktestmain/quicktestmain.cpp
@@ -0,0 +1,29 @@
+/****************************************************************************
+**
+** Copyright (C) 2018 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of the test suite of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:GPL-EXCEPT$
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3 as published by the Free Software
+** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-3.0.html.
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+#include <QtQuickTest/quicktest.h>
+QUICK_TEST_MAIN(quicktestmain)
diff --git a/tests/auto/quicktest/testfiltering/quicktestmain/quicktestmain.pro b/tests/auto/quicktest/testfiltering/quicktestmain/quicktestmain.pro
new file mode 100644
index 0000000000..7b3e734cb4
--- /dev/null
+++ b/tests/auto/quicktest/testfiltering/quicktestmain/quicktestmain.pro
@@ -0,0 +1,11 @@
+CONFIG += qmltestcase
+macos:CONFIG -= app_bundle
+TARGET = quicktestmain
+
+DEFINES += QT_QMLTEST_DATADIR=\\\"$${PWD}\\\"
+
+SOURCES += quicktestmain.cpp
+
+TESTDATA += $$PWD/*.qml
+
+DESTDIR = ./
diff --git a/tests/auto/quicktest/testfiltering/quicktestmain/tst_first.qml b/tests/auto/quicktest/testfiltering/quicktestmain/tst_first.qml
new file mode 100644
index 0000000000..55c9612b78
--- /dev/null
+++ b/tests/auto/quicktest/testfiltering/quicktestmain/tst_first.qml
@@ -0,0 +1,38 @@
+/****************************************************************************
+**
+** Copyright (C) 2018 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of the test suite of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:GPL-EXCEPT$
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3 as published by the Free Software
+** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-3.0.html.
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+
+import QtQuick 2.3
+import QtTest 1.1
+
+TestCase {
+    name: "First"
+
+    function test_foo() { }
+    function test_bar() { }
+    function test_baz() { }
+}
diff --git a/tests/auto/quicktest/testfiltering/quicktestmain/tst_second.qml b/tests/auto/quicktest/testfiltering/quicktestmain/tst_second.qml
new file mode 100644
index 0000000000..2143d93e12
--- /dev/null
+++ b/tests/auto/quicktest/testfiltering/quicktestmain/tst_second.qml
@@ -0,0 +1,38 @@
+/****************************************************************************
+**
+** Copyright (C) 2018 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of the test suite of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:GPL-EXCEPT$
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3 as published by the Free Software
+** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-3.0.html.
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+
+import QtQuick 2.3
+import QtTest 1.1
+
+TestCase {
+    name: "Second"
+
+    function test_dupfoo() { }
+    function test_dupbar() { }
+    function test_dupbaz() { }
+}
diff --git a/tests/auto/quicktest/testfiltering/test/test.pro b/tests/auto/quicktest/testfiltering/test/test.pro
new file mode 100644
index 0000000000..cecbdca725
--- /dev/null
+++ b/tests/auto/quicktest/testfiltering/test/test.pro
@@ -0,0 +1,5 @@
+CONFIG += testcase
+QT += testlib
+TARGET = ../tst_testfiltering
+
+SOURCES = ../tst_testfiltering.cpp
diff --git a/tests/auto/quicktest/testfiltering/testfiltering.pro b/tests/auto/quicktest/testfiltering/testfiltering.pro
new file mode 100644
index 0000000000..ae7a039b8b
--- /dev/null
+++ b/tests/auto/quicktest/testfiltering/testfiltering.pro
@@ -0,0 +1,4 @@
+TEMPLATE = subdirs
+SUBDIRS = \
+    test \
+    quicktestmain
diff --git a/tests/auto/quicktest/testfiltering/tst_testfiltering.cpp b/tests/auto/quicktest/testfiltering/tst_testfiltering.cpp
new file mode 100644
index 0000000000..72bb8f02b7
--- /dev/null
+++ b/tests/auto/quicktest/testfiltering/tst_testfiltering.cpp
@@ -0,0 +1,135 @@
+/****************************************************************************
+**
+** Copyright (C) 2018 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of the test suite of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:GPL-EXCEPT$
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3 as published by the Free Software
+** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-3.0.html.
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+#include <QProcess>
+#include <QtTest>
+
+class tst_TestFiltering : public QObject
+{
+    Q_OBJECT
+private slots:
+    void noFilters();
+    void oneMatchingFilter();
+    void filterThatDoesntMatch();
+    void twoFilters();
+    void twoFiltersWithOneMatch();
+    void manyFilters();
+};
+
+
+const QString testExe =
+#if defined(Q_OS_WIN)
+    QFINDTESTDATA("quicktestmain/quicktestmain.exe");
+#else
+    QFINDTESTDATA("quicktestmain/quicktestmain");
+#endif
+
+void tst_TestFiltering::noFilters()
+{
+    QProcess process;
+    process.start(testExe);
+
+    QVERIFY(process.waitForFinished());
+
+    const QString output = process.readAll();
+    QVERIFY(output.contains(QLatin1String("Totals: 10 passed")));
+    QCOMPARE(process.exitStatus(), QProcess::NormalExit);
+    QCOMPARE(process.exitCode(), 0);
+}
+
+void tst_TestFiltering::oneMatchingFilter()
+{
+    QProcess process;
+    process.start(testExe, {QLatin1String("First::test_bar")});
+
+    QVERIFY(process.waitForFinished());
+
+    const QString output = process.readAll();
+    QVERIFY(output.contains(QLatin1String("Totals: 3 passed")));
+    QCOMPARE(process.exitStatus(), QProcess::NormalExit);
+    QCOMPARE(process.exitCode(), 0);
+}
+
+void tst_TestFiltering::filterThatDoesntMatch()
+{
+    QProcess process;
+    process.start(testExe, {QLatin1String("First::test_nonexisting")});
+
+    QVERIFY(process.waitForFinished());
+
+    QCOMPARE(process.exitStatus(), QProcess::NormalExit);
+    QCOMPARE(process.exitCode(), 1);
+}
+
+void tst_TestFiltering::twoFilters()
+{
+    QProcess process;
+    process.start(testExe,
+                  {QLatin1String("Second::test_dupfoo"), QLatin1String("Second::test_dupbaz")});
+
+    QVERIFY(process.waitForFinished());
+
+    const QString output = process.readAll();
+    QVERIFY(output.contains(QLatin1String("Totals: 4 passed")));
+    QCOMPARE(process.exitStatus(), QProcess::NormalExit);
+    QCOMPARE(process.exitCode(), 0);
+}
+
+void tst_TestFiltering::twoFiltersWithOneMatch()
+{
+    QProcess process;
+    process.start(testExe,
+                  {QLatin1String("First::test_foo"), QLatin1String("Second::test_nonexisting")});
+
+    QVERIFY(process.waitForFinished());
+
+    const QString output = process.readAll();
+    QVERIFY(output.contains(QLatin1String("Totals: 3 passed")));
+    QCOMPARE(process.exitStatus(), QProcess::NormalExit);
+    QCOMPARE(process.exitCode(), 1);
+}
+
+void tst_TestFiltering::manyFilters()
+{
+    QProcess process;
+    process.start(testExe,
+                  {QLatin1String("First::test_foo"),
+                   QLatin1String("First::test_baz"),
+                   QLatin1String("Second::test_dupfoo"),
+                   QLatin1String("Second::test_dupbaz")});
+
+    QVERIFY(process.waitForFinished());
+
+    const QString output = process.readAll();
+    QVERIFY(output.contains(QLatin1String("Totals: 8 passed")));
+    QCOMPARE(process.exitStatus(), QProcess::NormalExit);
+    QCOMPARE(process.exitCode(), 0);
+}
+
+QTEST_MAIN(tst_TestFiltering);
+
+#include "tst_testfiltering.moc"