summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorTor Arne Vestbø <tor.arne.vestbo@qt.io>2020-02-06 12:07:04 +0100
committerTor Arne Vestbø <tor.arne.vestbo@qt.io>2020-02-06 23:43:05 +0100
commit8237e39f5dfa5e6e43aca7d7b9107c666d40de7b (patch)
treedd490b65d6ba1567cede59822ee5f1e0cda9f1c8
parentdbdb3cbc907fed4055adaa848dd5e36fb67b2263 (diff)
testlib: Move subtest argument handling to individual tests
The test orchestrator shouldn't have to deal with the individual options needed for each test. Change-Id: I78bbf4850cc649e625bd08a7aedf02267ba1314d Reviewed-by: Edward Welbourne <edward.welbourne@qt.io>
-rw-r--r--tests/auto/testlib/selftests/badxml/tst_badxml.cpp3
-rw-r--r--tests/auto/testlib/selftests/benchlibcallgrind/tst_benchlibcallgrind.cpp10
-rw-r--r--tests/auto/testlib/selftests/benchlibcounting/tst_benchlibcounting.cpp11
-rw-r--r--tests/auto/testlib/selftests/benchlibeventcounter/tst_benchlibeventcounter.cpp5
-rw-r--r--tests/auto/testlib/selftests/benchliboptions/tst_benchliboptions.cpp23
-rw-r--r--tests/auto/testlib/selftests/benchlibtickcounter/tst_benchlibtickcounter.cpp10
-rw-r--r--tests/auto/testlib/selftests/commandlinedata/tst_commandlinedata.cpp12
-rw-r--r--tests/auto/testlib/selftests/counting/tst_counting.cpp12
-rwxr-xr-xtests/auto/testlib/selftests/generate_expected_output.py21
-rw-r--r--tests/auto/testlib/selftests/printdatatags/tst_printdatatags.cpp10
-rw-r--r--tests/auto/testlib/selftests/printdatatagswithglobaltags/tst_printdatatagswithglobaltags.cpp10
-rw-r--r--tests/auto/testlib/selftests/signaldumper/tst_signaldumper.cpp11
-rw-r--r--tests/auto/testlib/selftests/silent/tst_silent.cpp11
-rw-r--r--tests/auto/testlib/selftests/tst_selftests.cpp47
-rw-r--r--tests/auto/testlib/selftests/verbose1/verbose1.pro2
-rw-r--r--tests/auto/testlib/selftests/verbose2/verbose2.pro2
16 files changed, 115 insertions, 85 deletions
diff --git a/tests/auto/testlib/selftests/badxml/tst_badxml.cpp b/tests/auto/testlib/selftests/badxml/tst_badxml.cpp
index 164dc7eaa9..d82765311b 100644
--- a/tests/auto/testlib/selftests/badxml/tst_badxml.cpp
+++ b/tests/auto/testlib/selftests/badxml/tst_badxml.cpp
@@ -194,6 +194,9 @@ int main(int argc, char** argv)
args << argv[i];
}
}
+
+ args << "-eventcounter";
+
/*
We just want testlib to output a benchmark result, we don't actually care about the value,
so just do one iteration to save time.
diff --git a/tests/auto/testlib/selftests/benchlibcallgrind/tst_benchlibcallgrind.cpp b/tests/auto/testlib/selftests/benchlibcallgrind/tst_benchlibcallgrind.cpp
index fe83ee6608..f2176612ad 100644
--- a/tests/auto/testlib/selftests/benchlibcallgrind/tst_benchlibcallgrind.cpp
+++ b/tests/auto/testlib/selftests/benchlibcallgrind/tst_benchlibcallgrind.cpp
@@ -79,6 +79,14 @@ void tst_BenchlibCallgrind::twoHundredMillionInstructions()
#endif
}
-QTEST_MAIN(tst_BenchlibCallgrind)
+int main(int argc, char *argv[])
+{
+ std::vector<const char*> args(argv, argv + argc);
+ args.push_back("-callgrind");
+ argc = args.size();
+ argv = const_cast<char**>(&args[0]);
+
+ QTEST_MAIN_IMPL(tst_BenchlibCallgrind)
+}
#include "tst_benchlibcallgrind.moc"
diff --git a/tests/auto/testlib/selftests/benchlibcounting/tst_benchlibcounting.cpp b/tests/auto/testlib/selftests/benchlibcounting/tst_benchlibcounting.cpp
index 58df127f3a..56b6d92f67 100644
--- a/tests/auto/testlib/selftests/benchlibcounting/tst_benchlibcounting.cpp
+++ b/tests/auto/testlib/selftests/benchlibcounting/tst_benchlibcounting.cpp
@@ -59,5 +59,14 @@ void tst_BenchlibCounting::failingBenchmark()
};
}
-QTEST_MAIN(tst_BenchlibCounting)
+int main(int argc, char *argv[])
+{
+ std::vector<const char*> args(argv, argv + argc);
+ args.push_back("-eventcounter");
+ argc = args.size();
+ argv = const_cast<char**>(&args[0]);
+
+ QTEST_MAIN_IMPL(tst_BenchlibCounting)
+}
+
#include "tst_benchlibcounting.moc"
diff --git a/tests/auto/testlib/selftests/benchlibeventcounter/tst_benchlibeventcounter.cpp b/tests/auto/testlib/selftests/benchlibeventcounter/tst_benchlibeventcounter.cpp
index e8015acb95..e3f4720a5e 100644
--- a/tests/auto/testlib/selftests/benchlibeventcounter/tst_benchlibeventcounter.cpp
+++ b/tests/auto/testlib/selftests/benchlibeventcounter/tst_benchlibeventcounter.cpp
@@ -94,6 +94,11 @@ void tst_BenchlibEventCounter::events_data()
int main(int argc, char** argv)
{
+ std::vector<const char*> args(argv, argv + argc);
+ args.push_back("-eventcounter");
+ argc = args.size();
+ argv = const_cast<char**>(&args[0]);
+
TestEventDispatcher dispatcher;
QCoreApplication app(argc, argv);
tst_BenchlibEventCounter test;
diff --git a/tests/auto/testlib/selftests/benchliboptions/tst_benchliboptions.cpp b/tests/auto/testlib/selftests/benchliboptions/tst_benchliboptions.cpp
index c1acf5a6e8..b89ca28ff2 100644
--- a/tests/auto/testlib/selftests/benchliboptions/tst_benchliboptions.cpp
+++ b/tests/auto/testlib/selftests/benchliboptions/tst_benchliboptions.cpp
@@ -83,6 +83,9 @@ void tst_BenchlibOptions::threeEvents()
int main(int argc, char** argv)
{
+ std::vector<const char*> args(argv, argv + argc);
+ args.push_back("-eventcounter");
+
int ret = 0;
TestEventDispatcher dispatcher;
@@ -91,29 +94,27 @@ int main(int argc, char** argv)
/* Run with no special arguments. */
{
tst_BenchlibOptions test;
- ret += QTest::qExec(&test, argc, argv);
+ ret += QTest::qExec(&test, args.size(), const_cast<char**>(&args[0]));
}
/* Run with an exact number of iterations. */
{
- QVector<char const*> args;
- for (int i = 0; i < argc; ++i) args << argv[i];
- args << "-iterations";
- args << "15";
+ auto extraArgs = args;
+ extraArgs.push_back("-iterations");
+ extraArgs.push_back("15");
tst_BenchlibFifteenIterations test;
- ret += QTest::qExec(&test, args.count(), const_cast<char**>(args.data()));
+ ret += QTest::qExec(&test, extraArgs.size(), const_cast<char**>(&extraArgs[0]));
}
/*
Run until getting a value of at least 100.
*/
{
- QVector<char const*> args;
- for (int i = 0; i < argc; ++i) args << argv[i];
- args << "-minimumvalue";
- args << "100";
+ auto extraArgs = args;
+ extraArgs.push_back("-minimumvalue");
+ extraArgs.push_back("100");
tst_BenchlibOneHundredMinimum test;
- ret += QTest::qExec(&test, args.count(), const_cast<char**>(args.data()));
+ ret += QTest::qExec(&test, extraArgs.size(), const_cast<char**>(&extraArgs[0]));
}
return ret;
diff --git a/tests/auto/testlib/selftests/benchlibtickcounter/tst_benchlibtickcounter.cpp b/tests/auto/testlib/selftests/benchlibtickcounter/tst_benchlibtickcounter.cpp
index b37f63f20a..2cb7dfd77f 100644
--- a/tests/auto/testlib/selftests/benchlibtickcounter/tst_benchlibtickcounter.cpp
+++ b/tests/auto/testlib/selftests/benchlibtickcounter/tst_benchlibtickcounter.cpp
@@ -62,6 +62,14 @@ void tst_BenchlibTickCounter::threeBillionTicks()
#endif
}
-QTEST_MAIN(tst_BenchlibTickCounter)
+int main(int argc, char *argv[])
+{
+ std::vector<const char*> args(argv, argv + argc);
+ args.push_back("-tickcounter");
+ argc = args.size();
+ argv = const_cast<char**>(&args[0]);
+
+ QTEST_MAIN_IMPL(tst_BenchlibTickCounter)
+}
#include "tst_benchlibtickcounter.moc"
diff --git a/tests/auto/testlib/selftests/commandlinedata/tst_commandlinedata.cpp b/tests/auto/testlib/selftests/commandlinedata/tst_commandlinedata.cpp
index 8cb1192ea2..a916fd7f1e 100644
--- a/tests/auto/testlib/selftests/commandlinedata/tst_commandlinedata.cpp
+++ b/tests/auto/testlib/selftests/commandlinedata/tst_commandlinedata.cpp
@@ -63,6 +63,16 @@ void tst_DataTable::fiveTablePasses_data() const
QTest::newRow("fiveTablePasses_data5") << true;
}
-QTEST_MAIN(tst_DataTable)
+int main(int argc, char *argv[])
+{
+ std::vector<const char*> args(argv, argv + argc);
+ args.push_back("fiveTablePasses");
+ args.push_back("fiveTablePasses:fiveTablePasses_data1");
+ args.push_back("-v2");
+ argc = args.size();
+ argv = const_cast<char**>(&args[0]);
+
+ QTEST_MAIN_IMPL(tst_DataTable)
+}
#include "tst_commandlinedata.moc"
diff --git a/tests/auto/testlib/selftests/counting/tst_counting.cpp b/tests/auto/testlib/selftests/counting/tst_counting.cpp
index da5595b04f..56933c78b7 100644
--- a/tests/auto/testlib/selftests/counting/tst_counting.cpp
+++ b/tests/auto/testlib/selftests/counting/tst_counting.cpp
@@ -287,5 +287,15 @@ void tst_Counting::testSkipInCleanup()
qDebug() << "This test function should execute and then QSKIP in cleanup()";
}
-QTEST_MAIN(tst_Counting)
+int main(int argc, char *argv[])
+{
+#ifdef TESTLIB_VERBOSITY_ARG
+ std::vector<const char*> args(argv, argv + argc);
+ args.push_back(QT_STRINGIFY(TESTLIB_VERBOSITY_ARG));
+ argc = args.size();
+ argv = const_cast<char**>(&args[0]);
+#endif
+
+ QTEST_MAIN_IMPL(tst_Counting)
+}
#include "tst_counting.moc"
diff --git a/tests/auto/testlib/selftests/generate_expected_output.py b/tests/auto/testlib/selftests/generate_expected_output.py
index 1c0beff7fb..96fab44408 100755
--- a/tests/auto/testlib/selftests/generate_expected_output.py
+++ b/tests/auto/testlib/selftests/generate_expected_output.py
@@ -295,23 +295,7 @@ def testEnv(testname,
return data
def generateTestData(testname, clean,
- formats = ('xml', 'txt', 'junitxml', 'lightxml', 'teamcity', 'tap'),
- # Make sure this matches tst_Selftests::runSubTest_data():
- extraArgs = {
- "commandlinedata": "fiveTablePasses fiveTablePasses:fiveTablePasses_data1 -v2",
- "benchlibcallgrind": "-callgrind",
- "benchlibeventcounter": "-eventcounter",
- "benchliboptions": "-eventcounter",
- "benchlibtickcounter": "-tickcounter",
- "badxml": "-eventcounter",
- "benchlibcounting": "-eventcounter",
- "printdatatags": "-datatags",
- "printdatatagswithglobaltags": "-datatags",
- "signaldumper": "-vs",
- "silent": "-silent",
- "verbose1": "-v1",
- "verbose2": "-v2",
- }):
+ formats = ('xml', 'txt', 'junitxml', 'lightxml', 'teamcity', 'tap')):
"""Run one test and save its cleaned results.
Required arguments are the name of the test directory (the binary
@@ -330,9 +314,6 @@ def generateTestData(testname, clean,
print(" running", testname)
for format in formats:
cmd = [path, '-' + format]
- if testname in extraArgs:
- cmd += extraArgs[testname].split()
-
data = subprocess.Popen(cmd, stdout=subprocess.PIPE, env=env,
universal_newlines=True).communicate()[0]
with open('expected_' + testname + '.' + format, 'w') as out:
diff --git a/tests/auto/testlib/selftests/printdatatags/tst_printdatatags.cpp b/tests/auto/testlib/selftests/printdatatags/tst_printdatatags.cpp
index 5c9f5c5898..60a295c0f7 100644
--- a/tests/auto/testlib/selftests/printdatatags/tst_printdatatags.cpp
+++ b/tests/auto/testlib/selftests/printdatatags/tst_printdatatags.cpp
@@ -72,6 +72,14 @@ void tst_PrintDataTags::c() const
{
}
-QTEST_MAIN(tst_PrintDataTags)
+int main(int argc, char *argv[])
+{
+ std::vector<const char*> args(argv, argv + argc);
+ args.push_back("-datatags");
+ argc = args.size();
+ argv = const_cast<char**>(&args[0]);
+
+ QTEST_MAIN_IMPL(tst_PrintDataTags)
+}
#include "tst_printdatatags.moc"
diff --git a/tests/auto/testlib/selftests/printdatatagswithglobaltags/tst_printdatatagswithglobaltags.cpp b/tests/auto/testlib/selftests/printdatatagswithglobaltags/tst_printdatatagswithglobaltags.cpp
index b68daae538..6fea34467a 100644
--- a/tests/auto/testlib/selftests/printdatatagswithglobaltags/tst_printdatatagswithglobaltags.cpp
+++ b/tests/auto/testlib/selftests/printdatatagswithglobaltags/tst_printdatatagswithglobaltags.cpp
@@ -88,6 +88,14 @@ void tst_PrintDataTagsWithGlobalTags::c() const
{
}
-QTEST_MAIN(tst_PrintDataTagsWithGlobalTags)
+int main(int argc, char *argv[])
+{
+ std::vector<const char*> args(argv, argv + argc);
+ args.push_back("-datatags");
+ argc = args.size();
+ argv = const_cast<char**>(&args[0]);
+
+ QTEST_MAIN_IMPL(tst_PrintDataTagsWithGlobalTags)
+}
#include "tst_printdatatagswithglobaltags.moc"
diff --git a/tests/auto/testlib/selftests/signaldumper/tst_signaldumper.cpp b/tests/auto/testlib/selftests/signaldumper/tst_signaldumper.cpp
index 08592e222d..c60260f109 100644
--- a/tests/auto/testlib/selftests/signaldumper/tst_signaldumper.cpp
+++ b/tests/auto/testlib/selftests/signaldumper/tst_signaldumper.cpp
@@ -424,5 +424,14 @@ void tst_Signaldumper::deletingSender()
emit signalSlotOwner->signalWithoutParameters();
}
-QTEST_MAIN(tst_Signaldumper)
+int main(int argc, char *argv[])
+{
+ std::vector<const char*> args(argv, argv + argc);
+ args.push_back("-vs");
+ argc = args.size();
+ argv = const_cast<char**>(&args[0]);
+
+ QTEST_MAIN_IMPL(tst_Signaldumper)
+}
+
#include "tst_signaldumper.moc"
diff --git a/tests/auto/testlib/selftests/silent/tst_silent.cpp b/tests/auto/testlib/selftests/silent/tst_silent.cpp
index 999d9acf07..7d64efffa2 100644
--- a/tests/auto/testlib/selftests/silent/tst_silent.cpp
+++ b/tests/auto/testlib/selftests/silent/tst_silent.cpp
@@ -102,5 +102,14 @@ void tst_Silent::messages()
qFatal("This is a fatal error message that should still appear in silent test output");
}
-QTEST_MAIN(tst_Silent)
+int main(int argc, char *argv[])
+{
+ std::vector<const char*> args(argv, argv + argc);
+ args.push_back("-silent");
+ argc = args.size();
+ argv = const_cast<char**>(&args[0]);
+
+ QTEST_MAIN_IMPL(tst_Silent)
+}
+
#include "tst_silent.moc"
diff --git a/tests/auto/testlib/selftests/tst_selftests.cpp b/tests/auto/testlib/selftests/tst_selftests.cpp
index 5c50fc6548..e2abf6869b 100644
--- a/tests/auto/testlib/selftests/tst_selftests.cpp
+++ b/tests/auto/testlib/selftests/tst_selftests.cpp
@@ -528,49 +528,6 @@ void tst_Selftests::runSubTest_data()
QStringList loggers = loggerSet.loggers;
foreach (QString const& subtest, tests) {
- QStringList arguments = loggerSet.arguments;
- // Keep in sync with generateTestData()'s extraArgs in generate_expected_output.py:
- if (subtest == "commandlinedata") {
- arguments << QString("fiveTablePasses fiveTablePasses:fiveTablePasses_data1 -v2").split(' ');
- }
- else if (subtest == "benchlibcallgrind") {
- arguments << "-callgrind";
- }
- else if (subtest == "benchlibeventcounter") {
- arguments << "-eventcounter";
- }
- else if (subtest == "benchliboptions") {
- arguments << "-eventcounter";
- }
- else if (subtest == "benchlibtickcounter") {
- arguments << "-tickcounter";
- }
- else if (subtest == "badxml") {
- arguments << "-eventcounter";
- }
- else if (subtest == "benchlibcounting") {
- arguments << "-eventcounter";
- }
- else if (subtest == "printdatatags") {
- arguments << "-datatags";
- }
- else if (subtest == "printdatatagswithglobaltags") {
- arguments << "-datatags";
- }
- else if (subtest == "signaldumper") {
- arguments << "-vs";
- }
- else if (subtest == "silent") {
- arguments << "-silent";
- }
- else if (subtest == "verbose1") {
- arguments << "-v1";
- }
- else if (subtest == "verbose2") {
- arguments << "-v2";
- }
-
-
// These tests don't work right unless logging plain text to
// standard output, either because they execute multiple test
// objects or because they internally supply arguments to
@@ -634,7 +591,7 @@ void tst_Selftests::runSubTest_data()
QTest::newRow(qPrintable(QString("%1 %2").arg(subtest).arg(loggerSet.name)))
<< subtest
<< loggers
- << arguments
+ << loggerSet.arguments
<< crashes
;
}
@@ -690,7 +647,7 @@ static inline QByteArray msgProcessError(const QString &binary, const QStringLis
void tst_Selftests::doRunSubTest(QString const& subdir, QStringList const& loggers, QStringList const& arguments, bool crashes)
{
#if defined(__GNUC__) && defined(__i386) && defined(Q_OS_LINUX)
- if (arguments.contains("-callgrind")) {
+ if (subdir == "benchlibcallgrind") {
QProcess checkProcess;
QStringList args;
args << QLatin1String("--version");
diff --git a/tests/auto/testlib/selftests/verbose1/verbose1.pro b/tests/auto/testlib/selftests/verbose1/verbose1.pro
index 1f16d70d66..f50ce83ea6 100644
--- a/tests/auto/testlib/selftests/verbose1/verbose1.pro
+++ b/tests/auto/testlib/selftests/verbose1/verbose1.pro
@@ -4,6 +4,8 @@
SOURCES += ../counting/tst_counting.cpp
QT = core testlib
+DEFINES += TESTLIB_VERBOSITY_ARG="-v1"
+
mac:CONFIG -= app_bundle
CONFIG -= debug_and_release_target
diff --git a/tests/auto/testlib/selftests/verbose2/verbose2.pro b/tests/auto/testlib/selftests/verbose2/verbose2.pro
index bf54904488..d4765f01b5 100644
--- a/tests/auto/testlib/selftests/verbose2/verbose2.pro
+++ b/tests/auto/testlib/selftests/verbose2/verbose2.pro
@@ -4,6 +4,8 @@
SOURCES += ../counting/tst_counting.cpp
QT = core testlib
+DEFINES += TESTLIB_VERBOSITY_ARG="-v2"
+
mac:CONFIG -= app_bundle
CONFIG -= debug_and_release_target