From 8237e39f5dfa5e6e43aca7d7b9107c666d40de7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tor=20Arne=20Vestb=C3=B8?= Date: Thu, 6 Feb 2020 12:07:04 +0100 Subject: testlib: Move subtest argument handling to individual tests The test orchestrator shouldn't have to deal with the individual options needed for each test. Change-Id: I78bbf4850cc649e625bd08a7aedf02267ba1314d Reviewed-by: Edward Welbourne --- tests/auto/testlib/selftests/badxml/tst_badxml.cpp | 3 ++ .../benchlibcallgrind/tst_benchlibcallgrind.cpp | 10 ++++- .../benchlibcounting/tst_benchlibcounting.cpp | 11 ++++- .../tst_benchlibeventcounter.cpp | 5 +++ .../benchliboptions/tst_benchliboptions.cpp | 23 ++++++----- .../tst_benchlibtickcounter.cpp | 10 ++++- .../commandlinedata/tst_commandlinedata.cpp | 12 +++++- .../testlib/selftests/counting/tst_counting.cpp | 12 +++++- .../testlib/selftests/generate_expected_output.py | 21 +--------- .../selftests/printdatatags/tst_printdatatags.cpp | 10 ++++- .../tst_printdatatagswithglobaltags.cpp | 10 ++++- .../selftests/signaldumper/tst_signaldumper.cpp | 11 ++++- tests/auto/testlib/selftests/silent/tst_silent.cpp | 11 ++++- tests/auto/testlib/selftests/tst_selftests.cpp | 47 +--------------------- tests/auto/testlib/selftests/verbose1/verbose1.pro | 2 + tests/auto/testlib/selftests/verbose2/verbose2.pro | 2 + 16 files changed, 115 insertions(+), 85 deletions(-) diff --git a/tests/auto/testlib/selftests/badxml/tst_badxml.cpp b/tests/auto/testlib/selftests/badxml/tst_badxml.cpp index 164dc7eaa9..d82765311b 100644 --- a/tests/auto/testlib/selftests/badxml/tst_badxml.cpp +++ b/tests/auto/testlib/selftests/badxml/tst_badxml.cpp @@ -194,6 +194,9 @@ int main(int argc, char** argv) args << argv[i]; } } + + args << "-eventcounter"; + /* We just want testlib to output a benchmark result, we don't actually care about the value, so just do one iteration to save time. diff --git a/tests/auto/testlib/selftests/benchlibcallgrind/tst_benchlibcallgrind.cpp b/tests/auto/testlib/selftests/benchlibcallgrind/tst_benchlibcallgrind.cpp index fe83ee6608..f2176612ad 100644 --- a/tests/auto/testlib/selftests/benchlibcallgrind/tst_benchlibcallgrind.cpp +++ b/tests/auto/testlib/selftests/benchlibcallgrind/tst_benchlibcallgrind.cpp @@ -79,6 +79,14 @@ void tst_BenchlibCallgrind::twoHundredMillionInstructions() #endif } -QTEST_MAIN(tst_BenchlibCallgrind) +int main(int argc, char *argv[]) +{ + std::vector args(argv, argv + argc); + args.push_back("-callgrind"); + argc = args.size(); + argv = const_cast(&args[0]); + + QTEST_MAIN_IMPL(tst_BenchlibCallgrind) +} #include "tst_benchlibcallgrind.moc" diff --git a/tests/auto/testlib/selftests/benchlibcounting/tst_benchlibcounting.cpp b/tests/auto/testlib/selftests/benchlibcounting/tst_benchlibcounting.cpp index 58df127f3a..56b6d92f67 100644 --- a/tests/auto/testlib/selftests/benchlibcounting/tst_benchlibcounting.cpp +++ b/tests/auto/testlib/selftests/benchlibcounting/tst_benchlibcounting.cpp @@ -59,5 +59,14 @@ void tst_BenchlibCounting::failingBenchmark() }; } -QTEST_MAIN(tst_BenchlibCounting) +int main(int argc, char *argv[]) +{ + std::vector args(argv, argv + argc); + args.push_back("-eventcounter"); + argc = args.size(); + argv = const_cast(&args[0]); + + QTEST_MAIN_IMPL(tst_BenchlibCounting) +} + #include "tst_benchlibcounting.moc" diff --git a/tests/auto/testlib/selftests/benchlibeventcounter/tst_benchlibeventcounter.cpp b/tests/auto/testlib/selftests/benchlibeventcounter/tst_benchlibeventcounter.cpp index e8015acb95..e3f4720a5e 100644 --- a/tests/auto/testlib/selftests/benchlibeventcounter/tst_benchlibeventcounter.cpp +++ b/tests/auto/testlib/selftests/benchlibeventcounter/tst_benchlibeventcounter.cpp @@ -94,6 +94,11 @@ void tst_BenchlibEventCounter::events_data() int main(int argc, char** argv) { + std::vector args(argv, argv + argc); + args.push_back("-eventcounter"); + argc = args.size(); + argv = const_cast(&args[0]); + TestEventDispatcher dispatcher; QCoreApplication app(argc, argv); tst_BenchlibEventCounter test; diff --git a/tests/auto/testlib/selftests/benchliboptions/tst_benchliboptions.cpp b/tests/auto/testlib/selftests/benchliboptions/tst_benchliboptions.cpp index c1acf5a6e8..b89ca28ff2 100644 --- a/tests/auto/testlib/selftests/benchliboptions/tst_benchliboptions.cpp +++ b/tests/auto/testlib/selftests/benchliboptions/tst_benchliboptions.cpp @@ -83,6 +83,9 @@ void tst_BenchlibOptions::threeEvents() int main(int argc, char** argv) { + std::vector args(argv, argv + argc); + args.push_back("-eventcounter"); + int ret = 0; TestEventDispatcher dispatcher; @@ -91,29 +94,27 @@ int main(int argc, char** argv) /* Run with no special arguments. */ { tst_BenchlibOptions test; - ret += QTest::qExec(&test, argc, argv); + ret += QTest::qExec(&test, args.size(), const_cast(&args[0])); } /* Run with an exact number of iterations. */ { - QVector args; - for (int i = 0; i < argc; ++i) args << argv[i]; - args << "-iterations"; - args << "15"; + auto extraArgs = args; + extraArgs.push_back("-iterations"); + extraArgs.push_back("15"); tst_BenchlibFifteenIterations test; - ret += QTest::qExec(&test, args.count(), const_cast(args.data())); + ret += QTest::qExec(&test, extraArgs.size(), const_cast(&extraArgs[0])); } /* Run until getting a value of at least 100. */ { - QVector args; - for (int i = 0; i < argc; ++i) args << argv[i]; - args << "-minimumvalue"; - args << "100"; + auto extraArgs = args; + extraArgs.push_back("-minimumvalue"); + extraArgs.push_back("100"); tst_BenchlibOneHundredMinimum test; - ret += QTest::qExec(&test, args.count(), const_cast(args.data())); + ret += QTest::qExec(&test, extraArgs.size(), const_cast(&extraArgs[0])); } return ret; diff --git a/tests/auto/testlib/selftests/benchlibtickcounter/tst_benchlibtickcounter.cpp b/tests/auto/testlib/selftests/benchlibtickcounter/tst_benchlibtickcounter.cpp index b37f63f20a..2cb7dfd77f 100644 --- a/tests/auto/testlib/selftests/benchlibtickcounter/tst_benchlibtickcounter.cpp +++ b/tests/auto/testlib/selftests/benchlibtickcounter/tst_benchlibtickcounter.cpp @@ -62,6 +62,14 @@ void tst_BenchlibTickCounter::threeBillionTicks() #endif } -QTEST_MAIN(tst_BenchlibTickCounter) +int main(int argc, char *argv[]) +{ + std::vector args(argv, argv + argc); + args.push_back("-tickcounter"); + argc = args.size(); + argv = const_cast(&args[0]); + + QTEST_MAIN_IMPL(tst_BenchlibTickCounter) +} #include "tst_benchlibtickcounter.moc" diff --git a/tests/auto/testlib/selftests/commandlinedata/tst_commandlinedata.cpp b/tests/auto/testlib/selftests/commandlinedata/tst_commandlinedata.cpp index 8cb1192ea2..a916fd7f1e 100644 --- a/tests/auto/testlib/selftests/commandlinedata/tst_commandlinedata.cpp +++ b/tests/auto/testlib/selftests/commandlinedata/tst_commandlinedata.cpp @@ -63,6 +63,16 @@ void tst_DataTable::fiveTablePasses_data() const QTest::newRow("fiveTablePasses_data5") << true; } -QTEST_MAIN(tst_DataTable) +int main(int argc, char *argv[]) +{ + std::vector args(argv, argv + argc); + args.push_back("fiveTablePasses"); + args.push_back("fiveTablePasses:fiveTablePasses_data1"); + args.push_back("-v2"); + argc = args.size(); + argv = const_cast(&args[0]); + + QTEST_MAIN_IMPL(tst_DataTable) +} #include "tst_commandlinedata.moc" diff --git a/tests/auto/testlib/selftests/counting/tst_counting.cpp b/tests/auto/testlib/selftests/counting/tst_counting.cpp index da5595b04f..56933c78b7 100644 --- a/tests/auto/testlib/selftests/counting/tst_counting.cpp +++ b/tests/auto/testlib/selftests/counting/tst_counting.cpp @@ -287,5 +287,15 @@ void tst_Counting::testSkipInCleanup() qDebug() << "This test function should execute and then QSKIP in cleanup()"; } -QTEST_MAIN(tst_Counting) +int main(int argc, char *argv[]) +{ +#ifdef TESTLIB_VERBOSITY_ARG + std::vector args(argv, argv + argc); + args.push_back(QT_STRINGIFY(TESTLIB_VERBOSITY_ARG)); + argc = args.size(); + argv = const_cast(&args[0]); +#endif + + QTEST_MAIN_IMPL(tst_Counting) +} #include "tst_counting.moc" diff --git a/tests/auto/testlib/selftests/generate_expected_output.py b/tests/auto/testlib/selftests/generate_expected_output.py index 1c0beff7fb..96fab44408 100755 --- a/tests/auto/testlib/selftests/generate_expected_output.py +++ b/tests/auto/testlib/selftests/generate_expected_output.py @@ -295,23 +295,7 @@ def testEnv(testname, return data def generateTestData(testname, clean, - formats = ('xml', 'txt', 'junitxml', 'lightxml', 'teamcity', 'tap'), - # Make sure this matches tst_Selftests::runSubTest_data(): - extraArgs = { - "commandlinedata": "fiveTablePasses fiveTablePasses:fiveTablePasses_data1 -v2", - "benchlibcallgrind": "-callgrind", - "benchlibeventcounter": "-eventcounter", - "benchliboptions": "-eventcounter", - "benchlibtickcounter": "-tickcounter", - "badxml": "-eventcounter", - "benchlibcounting": "-eventcounter", - "printdatatags": "-datatags", - "printdatatagswithglobaltags": "-datatags", - "signaldumper": "-vs", - "silent": "-silent", - "verbose1": "-v1", - "verbose2": "-v2", - }): + formats = ('xml', 'txt', 'junitxml', 'lightxml', 'teamcity', 'tap')): """Run one test and save its cleaned results. Required arguments are the name of the test directory (the binary @@ -330,9 +314,6 @@ def generateTestData(testname, clean, print(" running", testname) for format in formats: cmd = [path, '-' + format] - if testname in extraArgs: - cmd += extraArgs[testname].split() - data = subprocess.Popen(cmd, stdout=subprocess.PIPE, env=env, universal_newlines=True).communicate()[0] with open('expected_' + testname + '.' + format, 'w') as out: diff --git a/tests/auto/testlib/selftests/printdatatags/tst_printdatatags.cpp b/tests/auto/testlib/selftests/printdatatags/tst_printdatatags.cpp index 5c9f5c5898..60a295c0f7 100644 --- a/tests/auto/testlib/selftests/printdatatags/tst_printdatatags.cpp +++ b/tests/auto/testlib/selftests/printdatatags/tst_printdatatags.cpp @@ -72,6 +72,14 @@ void tst_PrintDataTags::c() const { } -QTEST_MAIN(tst_PrintDataTags) +int main(int argc, char *argv[]) +{ + std::vector args(argv, argv + argc); + args.push_back("-datatags"); + argc = args.size(); + argv = const_cast(&args[0]); + + QTEST_MAIN_IMPL(tst_PrintDataTags) +} #include "tst_printdatatags.moc" diff --git a/tests/auto/testlib/selftests/printdatatagswithglobaltags/tst_printdatatagswithglobaltags.cpp b/tests/auto/testlib/selftests/printdatatagswithglobaltags/tst_printdatatagswithglobaltags.cpp index b68daae538..6fea34467a 100644 --- a/tests/auto/testlib/selftests/printdatatagswithglobaltags/tst_printdatatagswithglobaltags.cpp +++ b/tests/auto/testlib/selftests/printdatatagswithglobaltags/tst_printdatatagswithglobaltags.cpp @@ -88,6 +88,14 @@ void tst_PrintDataTagsWithGlobalTags::c() const { } -QTEST_MAIN(tst_PrintDataTagsWithGlobalTags) +int main(int argc, char *argv[]) +{ + std::vector args(argv, argv + argc); + args.push_back("-datatags"); + argc = args.size(); + argv = const_cast(&args[0]); + + QTEST_MAIN_IMPL(tst_PrintDataTagsWithGlobalTags) +} #include "tst_printdatatagswithglobaltags.moc" diff --git a/tests/auto/testlib/selftests/signaldumper/tst_signaldumper.cpp b/tests/auto/testlib/selftests/signaldumper/tst_signaldumper.cpp index 08592e222d..c60260f109 100644 --- a/tests/auto/testlib/selftests/signaldumper/tst_signaldumper.cpp +++ b/tests/auto/testlib/selftests/signaldumper/tst_signaldumper.cpp @@ -424,5 +424,14 @@ void tst_Signaldumper::deletingSender() emit signalSlotOwner->signalWithoutParameters(); } -QTEST_MAIN(tst_Signaldumper) +int main(int argc, char *argv[]) +{ + std::vector args(argv, argv + argc); + args.push_back("-vs"); + argc = args.size(); + argv = const_cast(&args[0]); + + QTEST_MAIN_IMPL(tst_Signaldumper) +} + #include "tst_signaldumper.moc" diff --git a/tests/auto/testlib/selftests/silent/tst_silent.cpp b/tests/auto/testlib/selftests/silent/tst_silent.cpp index 999d9acf07..7d64efffa2 100644 --- a/tests/auto/testlib/selftests/silent/tst_silent.cpp +++ b/tests/auto/testlib/selftests/silent/tst_silent.cpp @@ -102,5 +102,14 @@ void tst_Silent::messages() qFatal("This is a fatal error message that should still appear in silent test output"); } -QTEST_MAIN(tst_Silent) +int main(int argc, char *argv[]) +{ + std::vector args(argv, argv + argc); + args.push_back("-silent"); + argc = args.size(); + argv = const_cast(&args[0]); + + QTEST_MAIN_IMPL(tst_Silent) +} + #include "tst_silent.moc" diff --git a/tests/auto/testlib/selftests/tst_selftests.cpp b/tests/auto/testlib/selftests/tst_selftests.cpp index 5c50fc6548..e2abf6869b 100644 --- a/tests/auto/testlib/selftests/tst_selftests.cpp +++ b/tests/auto/testlib/selftests/tst_selftests.cpp @@ -528,49 +528,6 @@ void tst_Selftests::runSubTest_data() QStringList loggers = loggerSet.loggers; foreach (QString const& subtest, tests) { - QStringList arguments = loggerSet.arguments; - // Keep in sync with generateTestData()'s extraArgs in generate_expected_output.py: - if (subtest == "commandlinedata") { - arguments << QString("fiveTablePasses fiveTablePasses:fiveTablePasses_data1 -v2").split(' '); - } - else if (subtest == "benchlibcallgrind") { - arguments << "-callgrind"; - } - else if (subtest == "benchlibeventcounter") { - arguments << "-eventcounter"; - } - else if (subtest == "benchliboptions") { - arguments << "-eventcounter"; - } - else if (subtest == "benchlibtickcounter") { - arguments << "-tickcounter"; - } - else if (subtest == "badxml") { - arguments << "-eventcounter"; - } - else if (subtest == "benchlibcounting") { - arguments << "-eventcounter"; - } - else if (subtest == "printdatatags") { - arguments << "-datatags"; - } - else if (subtest == "printdatatagswithglobaltags") { - arguments << "-datatags"; - } - else if (subtest == "signaldumper") { - arguments << "-vs"; - } - else if (subtest == "silent") { - arguments << "-silent"; - } - else if (subtest == "verbose1") { - arguments << "-v1"; - } - else if (subtest == "verbose2") { - arguments << "-v2"; - } - - // These tests don't work right unless logging plain text to // standard output, either because they execute multiple test // objects or because they internally supply arguments to @@ -634,7 +591,7 @@ void tst_Selftests::runSubTest_data() QTest::newRow(qPrintable(QString("%1 %2").arg(subtest).arg(loggerSet.name))) << subtest << loggers - << arguments + << loggerSet.arguments << crashes ; } @@ -690,7 +647,7 @@ static inline QByteArray msgProcessError(const QString &binary, const QStringLis void tst_Selftests::doRunSubTest(QString const& subdir, QStringList const& loggers, QStringList const& arguments, bool crashes) { #if defined(__GNUC__) && defined(__i386) && defined(Q_OS_LINUX) - if (arguments.contains("-callgrind")) { + if (subdir == "benchlibcallgrind") { QProcess checkProcess; QStringList args; args << QLatin1String("--version"); diff --git a/tests/auto/testlib/selftests/verbose1/verbose1.pro b/tests/auto/testlib/selftests/verbose1/verbose1.pro index 1f16d70d66..f50ce83ea6 100644 --- a/tests/auto/testlib/selftests/verbose1/verbose1.pro +++ b/tests/auto/testlib/selftests/verbose1/verbose1.pro @@ -4,6 +4,8 @@ SOURCES += ../counting/tst_counting.cpp QT = core testlib +DEFINES += TESTLIB_VERBOSITY_ARG="-v1" + mac:CONFIG -= app_bundle CONFIG -= debug_and_release_target diff --git a/tests/auto/testlib/selftests/verbose2/verbose2.pro b/tests/auto/testlib/selftests/verbose2/verbose2.pro index bf54904488..d4765f01b5 100644 --- a/tests/auto/testlib/selftests/verbose2/verbose2.pro +++ b/tests/auto/testlib/selftests/verbose2/verbose2.pro @@ -4,6 +4,8 @@ SOURCES += ../counting/tst_counting.cpp QT = core testlib +DEFINES += TESTLIB_VERBOSITY_ARG="-v2" + mac:CONFIG -= app_bundle CONFIG -= debug_and_release_target -- cgit v1.2.3