qtestlib: exit with error if a test is invoked with unknown data tag

Previously trying to execute a test function with an unknown data tag
would print an error message but exit with 0.

This patch stores a test failure, and continues trying to execute the
rest of the command line arguments, if any. In the end the process exits
with the usual exit code (number of failed tests) which is now !=0.

Fixes: QTBUG-24240
Change-Id: Id4d422035f173e01e77ca88028dfd94dc0f9085c
Reviewed-by: Edward Welbourne <edward.welbourne@qt.io>
(cherry picked from commit d23dcc05425dcd2ab0eb621f5f2ea793a78fcc9f)
Reviewed-by: Qt Cherry-pick Bot <cherrypick_bot@qt-project.org>

2 files changed, 74 insertions, 21 deletions

diff --git a/src/testlib/qtestcase.cpp b/src/testlib/qtestcase.cpp
index f55d4ba38f..242246d279 100644
--- a/src/testlib/qtestcase.cpp
+++ b/src/testlib/qtestcase.cpp
@@ -1292,6 +1292,26 @@ public:
 #endif  // QT_CONFIG(thread)
 
 
+static void printUnknownDataTagError(QLatin1StringView name, QLatin1StringView tag,
+                                     const QTestTable &lTable, const QTestTable &gTable)
+{
+    fprintf(stderr, "Unknown testdata for function %s(): '%s'\n", name.constData(), tag.data());
+    const int localDataCount = lTable.dataCount();
+    if (localDataCount) {
+        fputs("Available test-specific data tags:\n", stderr);
+        for (int i = 0; i < localDataCount; ++i)
+            fprintf(stderr, "\t%s\n", lTable.testData(i)->dataTag());
+    }
+    const int globalDataCount = gTable.dataCount();
+    if (globalDataCount) {
+        fputs("Available global data tags:\n", stderr);
+        for (int i = 0; i < globalDataCount; ++i)
+            fprintf(stderr, "\t%s\n", gTable.testData(i)->dataTag());
+    }
+    if (localDataCount == 0 && globalDataCount == 0)
+        fputs("Function has no data tags\n", stderr);
+}
+
 /*!
     \internal
 
@@ -1352,13 +1372,6 @@ bool TestMethods::invokeTest(int index, QLatin1StringView tag, WatchDog *watchDo
             return dataCount ? table.testData(index)->dataTag() : nullptr;
         };
 
-        // Data tag requested but none available?
-        if (!tag.isEmpty() && !dataCount && !globalDataCount) {
-            fprintf(stderr, "Unknown test data tag for function %s(): '%s'\n"
-                    "Function has no testdata.\n", name.constData(), tag.data());
-            return false;
-        }
-
         /* For each entry in this test's data table, do: */
         do {
             QTestResult::setSkipCurrentTest(false);
@@ -1366,7 +1379,6 @@ bool TestMethods::invokeTest(int index, QLatin1StringView tag, WatchDog *watchDo
             if (dataTagMatches(tag, QLatin1StringView(dataTag(curDataIndex)),
                                QLatin1StringView(globalDataTag(curGlobalDataIndex)))) {
                 foundFunction = true;
-
                 QTestPrivate::checkBlackLists(name.constData(), dataTag(curDataIndex),
                                               globalDataTag(curGlobalDataIndex));
 
@@ -1391,20 +1403,9 @@ bool TestMethods::invokeTest(int index, QLatin1StringView tag, WatchDog *watchDo
     } while (curGlobalDataIndex < globalDataCount);
 
     if (!tag.isEmpty() && !foundFunction) {
-        fprintf(stderr, "Unknown testdata for function %s(): '%s'\n", name.constData(), tag.data());
-        if (table.dataCount()) {
-            fputs("Available test-specific data tags:\n", stderr);
-            for (int i = 0; i < table.dataCount(); ++i)
-                fprintf(stderr, "\t%s\n", table.testData(i)->dataTag());
-        }
-        if (globalDataCount) {
-            fputs("Available global data tags:\n", stderr);
-            for (int i = 0; i < globalDataCount; ++i)
-                fprintf(stderr, "\t%s\n", gTable->testData(i)->dataTag());
-        }
-        return false;
+        printUnknownDataTagError(QLatin1StringView(name), tag, table, *gTable);
+        QTestResult::addFailure(qPrintable("Data tag not found: %1"_L1.arg(tag)));
     }
-
     QTestResult::finishedCurrentTestFunction();
     QTestResult::setSkipCurrentTest(false);
     QTestResult::setBlacklistCurrentTest(false);
diff --git a/tests/auto/testlib/selftests/tst_selftests.cpp b/tests/auto/testlib/selftests/tst_selftests.cpp
index 5f67d0e25b..b70aec2c85 100644
--- a/tests/auto/testlib/selftests/tst_selftests.cpp
+++ b/tests/auto/testlib/selftests/tst_selftests.cpp
@@ -1195,6 +1195,58 @@ SCENARIO("Test output of the loggers is as expected")
     }
 }
 
+struct TestCase {
+    int expectedExitCode;
+    const char *cmdline;
+};
+
+SCENARIO("Exit code is as expected")
+{
+    // Listing of test command lines and expected exit codes
+    // NOTE: Use at least 2 spaces to separate arguments because some contain a space themselves.
+    const struct TestCase testCases[] = {
+    // 'pass' is a test with no data tags at all
+        { 0, "pass  testNumber1" },
+        { 1, "pass  unknownFunction" },
+        { 1, "pass  testNumber1:blah" },
+        { 1, "pass  testNumber1:blah:blue" },
+    // 'counting' is a test that has only local data tags
+        { 0, "counting  testPassPass" },
+        { 0, "counting  testPassPass:row 1" },
+        { 1, "counting  testPassPass:blah" },
+        { 1, "counting  testPassPass:blah:row 1" },
+        { 1, "counting  testPassPass:blah:blue" },
+    // 'globaldata' is a test with global and local data tags
+        { 0, "globaldata  testGlobal" },
+        { 0, "globaldata  testGlobal:global=true" },
+        { 0, "globaldata  testGlobal:local=true" },
+        { 0, "globaldata  testGlobal:global=true:local=true" },
+        { 1, "globaldata  testGlobal:local=true:global=true" },
+        { 1, "globaldata  testGlobal:global=true:blah" },
+        { 1, "globaldata  testGlobal:blah:local=true" },
+        { 1, "globaldata  testGlobal:blah:global=true" },
+        { 1, "globaldata  testGlobal:blah" },
+        { 1, "globaldata  testGlobal:blah:blue" },
+    // Passing multiple testcase:data on the command line
+        { 0, "globaldata  testGlobal:global=true  skipSingle:global=true:local=true" },
+        { 1, "globaldata  testGlobal:blah         skipSingle:global=true:local=true" },
+        { 1, "globaldata  testGlobal:global=true  skipSingle:blah" },
+        { 2, "globaldata  testGlobal:blah         skipSingle:blue" },
+    };
+
+    size_t n_testCases = sizeof(testCases) / sizeof(*testCases);
+    for (size_t i = 0; i < n_testCases; i++) {
+        GIVEN("The command line: " << testCases[i].cmdline) {
+            const QStringList cmdSplit = QString(testCases[i].cmdline)
+                    .split(QRegularExpression("  +"));    // at least 2 spaces
+            const QString test     = cmdSplit[0];
+            const QStringList args = cmdSplit.sliced(1);
+            auto runResult = runTestProcess(test, args);
+            REQUIRE(runResult.exitCode == testCases[i].expectedExitCode);
+        }
+    }
+}
+
 // ----------------------- Entrypoint -----------------------
 
 int main(int argc, char **argv)