summaryrefslogtreecommitdiffstats
path: root/tests/auto/testlib/selftests/tst_selftests.cpp
diff options
context:
space:
mode:
authorDimitrios Apostolou <jimis@qt.io>2022-05-10 16:32:29 +0200
committerDimitrios Apostolou <jimis@qt.io>2022-06-28 18:06:33 +0200
commitd23dcc05425dcd2ab0eb621f5f2ea793a78fcc9f (patch)
tree36eafb912e0b2f30002ba0f7d0a6b7975a3d3aa4 /tests/auto/testlib/selftests/tst_selftests.cpp
parent8a96c8a22ca2c81969bb5e06249c06143b7f8249 (diff)
qtestlib: exit with error if a test is invoked with unknown data tag
Previously trying to execute a test function with an unknown data tag would print an error message but exit with 0. This patch stores a test failure, and continues trying to execute the rest of the command line arguments, if any. In the end the process exits with the usual exit code (number of failed tests) which is now !=0. Pick-to: 6.4 6.3 6.2 Fixes: QTBUG-24240 Change-Id: Id4d422035f173e01e77ca88028dfd94dc0f9085c Reviewed-by: Edward Welbourne <edward.welbourne@qt.io>
Diffstat (limited to 'tests/auto/testlib/selftests/tst_selftests.cpp')
-rw-r--r--tests/auto/testlib/selftests/tst_selftests.cpp52
1 files changed, 52 insertions, 0 deletions
diff --git a/tests/auto/testlib/selftests/tst_selftests.cpp b/tests/auto/testlib/selftests/tst_selftests.cpp
index 5f67d0e25b..b70aec2c85 100644
--- a/tests/auto/testlib/selftests/tst_selftests.cpp
+++ b/tests/auto/testlib/selftests/tst_selftests.cpp
@@ -1195,6 +1195,58 @@ SCENARIO("Test output of the loggers is as expected")
}
}
+struct TestCase {
+ int expectedExitCode;
+ const char *cmdline;
+};
+
+SCENARIO("Exit code is as expected")
+{
+ // Listing of test command lines and expected exit codes
+ // NOTE: Use at least 2 spaces to separate arguments because some contain a space themselves.
+ const struct TestCase testCases[] = {
+ // 'pass' is a test with no data tags at all
+ { 0, "pass testNumber1" },
+ { 1, "pass unknownFunction" },
+ { 1, "pass testNumber1:blah" },
+ { 1, "pass testNumber1:blah:blue" },
+ // 'counting' is a test that has only local data tags
+ { 0, "counting testPassPass" },
+ { 0, "counting testPassPass:row 1" },
+ { 1, "counting testPassPass:blah" },
+ { 1, "counting testPassPass:blah:row 1" },
+ { 1, "counting testPassPass:blah:blue" },
+ // 'globaldata' is a test with global and local data tags
+ { 0, "globaldata testGlobal" },
+ { 0, "globaldata testGlobal:global=true" },
+ { 0, "globaldata testGlobal:local=true" },
+ { 0, "globaldata testGlobal:global=true:local=true" },
+ { 1, "globaldata testGlobal:local=true:global=true" },
+ { 1, "globaldata testGlobal:global=true:blah" },
+ { 1, "globaldata testGlobal:blah:local=true" },
+ { 1, "globaldata testGlobal:blah:global=true" },
+ { 1, "globaldata testGlobal:blah" },
+ { 1, "globaldata testGlobal:blah:blue" },
+ // Passing multiple testcase:data on the command line
+ { 0, "globaldata testGlobal:global=true skipSingle:global=true:local=true" },
+ { 1, "globaldata testGlobal:blah skipSingle:global=true:local=true" },
+ { 1, "globaldata testGlobal:global=true skipSingle:blah" },
+ { 2, "globaldata testGlobal:blah skipSingle:blue" },
+ };
+
+ size_t n_testCases = sizeof(testCases) / sizeof(*testCases);
+ for (size_t i = 0; i < n_testCases; i++) {
+ GIVEN("The command line: " << testCases[i].cmdline) {
+ const QStringList cmdSplit = QString(testCases[i].cmdline)
+ .split(QRegularExpression(" +")); // at least 2 spaces
+ const QString test = cmdSplit[0];
+ const QStringList args = cmdSplit.sliced(1);
+ auto runResult = runTestProcess(test, args);
+ REQUIRE(runResult.exitCode == testCases[i].expectedExitCode);
+ }
+ }
+}
+
// ----------------------- Entrypoint -----------------------
int main(int argc, char **argv)