Implement the batch_tests feature

An approach of test batching (joining multiple tests into a single
binary) has been taken, due to long linking times/binary size on certain
platforms, including WASM. This change adds a new feature
'batch_test_support' in Qt testlib. Based on the value of the feature,
test batching may become enabled with the -batch-tests switch.

Batching works for every target added via qt_internal_add_test. When
first such target is being processed, a new combined target for all of
the future test sources is created under the name of 'test_batch'.
CMake attempts to merge the parameters of each of the tests, and some
basic checks are run for parameter differences that are impossible to
reconcile.

On the C++ level, convenience macros instantiating the tests are
redefined when batch_tests is on. The new, changed behavior triggered
by the changes in the macros registers the tests in a central test
registry, where they are available for execution based solely on their
test name. The test name is interoperable with the names CMake is aware
of, so CTest is able to run the tests one by one in the combined binary.

Task-number: QTBUG-105273
Change-Id: I2b6071d58be16979bd967eab2d405249f5a4e658
Reviewed-by: Topi Reiniö <topi.reinio@qt.io>

10 files changed, 185 insertions, 1 deletions

diff --git a/src/testlib/CMakeLists.txt b/src/testlib/CMakeLists.txt
index 3211961541..1d5dc71460 100644
--- a/src/testlib/CMakeLists.txt
+++ b/src/testlib/CMakeLists.txt
@@ -41,7 +41,7 @@ qt_internal_add_module(Test
         qtestaccessible.h
         qtestassert.h
         qtestblacklist.cpp qtestblacklist_p.h
-        qtestcase.cpp qtestcase.h
+        qtestcase.cpp qtestcase.h qtestcase_p.h
         qtestcoreelement_p.h
         qtestdata.cpp qtestdata.h
         qtestelement.cpp qtestelement_p.h
@@ -74,6 +74,7 @@ qt_internal_add_module(Test
     PRIVATE_MODULE_INTERFACE
         Qt::CorePrivate
     GENERATE_CPP_EXPORTS
+    GENERATE_PRIVATE_CPP_EXPORTS
 )
 
 #### Keys ignored in scope 1:.:.:testlib.pro:<TRUE>:
@@ -90,6 +91,11 @@ qt_internal_extend_target(Test CONDITION QT_FEATURE_itemmodeltester
         qabstractitemmodeltester.cpp qabstractitemmodeltester.h
 )
 
+qt_internal_extend_target(Test CONDITION QT_FEATURE_batch_test_support
+    SOURCES
+        qtestregistry.cpp qtestregistry_p.h
+)
+
 qt_internal_extend_target(Test CONDITION QT_FEATURE_valgrind
     SOURCES
         3rdparty/callgrind_p.h
diff --git a/src/testlib/configure.cmake b/src/testlib/configure.cmake
index d8ede189f4..3490c64874 100644
--- a/src/testlib/configure.cmake
+++ b/src/testlib/configure.cmake
@@ -32,6 +32,13 @@ qt_feature("valgrind" PUBLIC
     PURPOSE "Profiling support with callgrind."
     CONDITION ( LINUX OR APPLE ) AND QT_FEATURE_process AND QT_FEATURE_regularexpression
 )
+qt_feature("batch_test_support" PUBLIC
+    LABEL "Batch tests"
+    PURPOSE "Allows merging of all tests into a single executable on demand"
+    AUTODETECT QT_BUILD_TESTS_BATCHED
+    ENABLE INPUT_batch_tests STREQUAL 'yes'
+)
 qt_configure_add_summary_section(NAME "Qt Testlib")
 qt_configure_add_summary_entry(ARGS "itemmodeltester")
+qt_configure_add_summary_entry(ARGS "batch_test_support")
 qt_configure_end_summary_section() # end of "Qt Testlib" section
diff --git a/src/testlib/doc/qttestlib.qdocconf b/src/testlib/doc/qttestlib.qdocconf
index fe80362cca..4330fe197a 100644
--- a/src/testlib/doc/qttestlib.qdocconf
+++ b/src/testlib/doc/qttestlib.qdocconf
@@ -48,6 +48,8 @@ excludedirs += ../../../examples/widgets/doc
 
 imagedirs   += images
 
+defines     += QT_FEATURE_batch_test_support
+
 # Add a thumbnail for examples that do not have images
 manifestmeta.thumbnail.names = "QtTestLib/Chapter *"
 
diff --git a/src/testlib/qt_cmdline.cmake b/src/testlib/qt_cmdline.cmake
index e69de29bb2..d3601c1e39 100644
--- a/src/testlib/qt_cmdline.cmake
+++ b/src/testlib/qt_cmdline.cmake
@@ -0,0 +1 @@
+qt_commandline_option(batch-tests TYPE boolean NAME batch_tests)
diff --git a/src/testlib/qtest.h b/src/testlib/qtest.h
index f4f0ff5dfd..841751fac1 100644
--- a/src/testlib/qtest.h
+++ b/src/testlib/qtest.h
@@ -582,6 +582,7 @@ struct QtCoverageScanner
 #define TESTLIB_SELFCOVERAGE_START(name)
 #endif
 
+#if !defined(QTEST_BATCH_TESTS)
 // Internal (but used by some testlib selftests to hack argc and argv).
 // Tests should normally implement initMain() if they have set-up to do before
 // instantiating the test class.
@@ -595,6 +596,30 @@ int main(int argc, char *argv[]) \
     QTEST_SET_MAIN_SOURCE_PATH \
     return QTest::qExec(&tc, argc, argv); \
 }
+#else
+// BATCHED_TEST_NAME is defined for each test in a batch in cmake. Some odd
+// targets, like snippets, don't define it though. Play safe by providing a
+// default value.
+#if !defined(BATCHED_TEST_NAME)
+#define BATCHED_TEST_NAME "other"
+#endif
+#define QTEST_MAIN_WRAPPER(TestObject, ...) \
+\
+void qRegister##TestObject() \
+{ \
+    auto runTest = [](int argc, char** argv) -> int { \
+        TESTLIB_SELFCOVERAGE_START(TestObject) \
+        QT_PREPEND_NAMESPACE(QTest::Internal::callInitMain)<TestObject>(); \
+        __VA_ARGS__ \
+        TestObject tc; \
+        QTEST_SET_MAIN_SOURCE_PATH \
+        return QTest::qExec(&tc, argc, argv); \
+    }; \
+    QTest::qRegisterTestCase(BATCHED_TEST_NAME, runTest); \
+} \
+\
+Q_CONSTRUCTOR_FUNCTION(qRegister##TestObject)
+#endif
 
 // For when you don't even want a QApplication:
 #define QTEST_APPLESS_MAIN(TestObject) QTEST_MAIN_WRAPPER(TestObject)
diff --git a/src/testlib/qtestcase.cpp b/src/testlib/qtestcase.cpp
index 89530f823f..99be5afaef 100644
--- a/src/testlib/qtestcase.cpp
+++ b/src/testlib/qtestcase.cpp
@@ -3,6 +3,7 @@
 // SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
 
 #include <QtTest/qtestcase.h>
+#include <QtTest/private/qtestcase_p.h>
 #include <QtTest/qtestassert.h>
 
 #include <QtCore/qbytearray.h>
@@ -33,6 +34,9 @@
 #include <QtTest/private/qtestresult_p.h>
 #include <QtTest/private/qsignaldumper_p.h>
 #include <QtTest/private/qbenchmark_p.h>
+#if QT_CONFIG(batch_test_support)
+#include <QtTest/private/qtestregistry_p.h>
+#endif  // QT_CONFIG(batch_test_support)
 #include <QtTest/private/cycle_p.h>
 #include <QtTest/private/qtestblacklist_p.h>
 #if defined(HAVE_XCTEST)
@@ -2388,6 +2392,32 @@ void QTest::qCleanup()
 #endif
 }
 
+#if QT_CONFIG(batch_test_support) || defined(Q_QDOC)
+/*!
+    Registers the test \a name, with entry function \a entryFunction, in a
+    central test case registry for the current binary.
+
+    The \a name will be listed when running the batch test binary with no
+    parameters. Running the test binary with the argv[1] of \a name will result
+    in \a entryFunction being called.
+*/
+void QTest::qRegisterTestCase(const QString &name, TestEntryFunction entryFunction)
+{
+    QTest::TestRegistry::instance()->registerTest(name, entryFunction);
+}
+
+QList<QString> QTest::qGetTestCaseNames()
+{
+    return QTest::TestRegistry::instance()->getAllTestNames();
+}
+
+QTest::TestEntryFunction QTest::qGetTestCaseEntryFunction(const QString& name)
+{
+    return QTest::TestRegistry::instance()->getTestEntryFunction(name);
+}
+
+#endif  // QT_CONFIG(batch_test_support)
+
 /*!
   \overload
   \since 4.4
diff --git a/src/testlib/qtestcase.h b/src/testlib/qtestcase.h
index a0df8dd305..cd713ceebf 100644
--- a/src/testlib/qtestcase.h
+++ b/src/testlib/qtestcase.h
@@ -374,6 +374,11 @@ namespace QTest
     Q_TESTLIB_EXPORT int qExec(QObject *testObject, int argc = 0, char **argv = nullptr);
     Q_TESTLIB_EXPORT int qExec(QObject *testObject, const QStringList &arguments);
 
+#if QT_CONFIG(batch_test_support) || defined(Q_QDOC)
+    using TestEntryFunction = int (*)(int, char **);
+    Q_TESTLIB_EXPORT void qRegisterTestCase(const QString &name, TestEntryFunction entryFunction);
+#endif  // QT_CONFIG(batch_test_support)
+
     Q_TESTLIB_EXPORT void setMainSourcePath(const char *file, const char *builddir = nullptr);
 
     Q_TESTLIB_EXPORT bool qVerify(bool statement, const char *statementStr, const char *description,
diff --git a/src/testlib/qtestcase_p.h b/src/testlib/qtestcase_p.h
new file mode 100644
index 0000000000..91a5314f97
--- /dev/null
+++ b/src/testlib/qtestcase_p.h
@@ -0,0 +1,36 @@
+// Copyright (C) 2022 The Qt Company Ltd.
+// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
+
+#ifndef QTESTCASE_P_H
+#define QTESTCASE_P_H
+
+//
+//  W A R N I N G
+//  -------------
+//
+// This file is not part of the Qt API.  It exists purely as an
+// implementation detail.  This header file may change from version to
+// version without notice, or even be removed.
+//
+// We mean it.
+//
+
+#include <QtTest/qtestcase.h>
+#include <QtTest/private/qttestexports_p.h>
+#include <QtTest/qttestglobal.h>
+
+#include <QtCore/qstring.h>
+#include <QtCore/qnamespace.h>
+
+QT_BEGIN_NAMESPACE
+
+namespace QTest {
+#if QT_CONFIG(batch_test_support)
+    Q_TESTLIB_PRIVATE_EXPORT QList<QString> qGetTestCaseNames();
+    Q_TESTLIB_PRIVATE_EXPORT TestEntryFunction qGetTestCaseEntryFunction(const QString &name);
+#endif  // QT_CONFIG(batch_test_support)
+} // namespace QTest
+
+QT_END_NAMESPACE
+
+#endif  // QTESTCASE_P_H
diff --git a/src/testlib/qtestregistry.cpp b/src/testlib/qtestregistry.cpp
new file mode 100644
index 0000000000..ff1f8a57e6
--- /dev/null
+++ b/src/testlib/qtestregistry.cpp
@@ -0,0 +1,36 @@
+// Copyright (C) 2022 The Qt Company Ltd.
+// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
+
+#include <QtTest/private/qtestregistry_p.h>
+
+QT_REQUIRE_CONFIG(batch_test_support);
+
+QT_BEGIN_NAMESPACE
+
+namespace QTest {
+Q_GLOBAL_STATIC(TestRegistry, g_registry);
+
+TestRegistry *TestRegistry::instance()
+{
+    return g_registry;
+}
+
+void TestRegistry::registerTest(const QString& name, TestEntryFunction entry)
+{
+    m_tests.emplace(name, std::move(entry));
+}
+
+TestRegistry::TestEntryFunction
+TestRegistry::getTestEntryFunction(const QString& name) const
+{
+    const auto it = m_tests.find(name);
+    return it != m_tests.end() ? it.value() : nullptr;
+}
+
+QStringList TestRegistry::getAllTestNames() const
+{
+    return m_tests.keys();
+}
+}
+
+QT_END_NAMESPACE
diff --git a/src/testlib/qtestregistry_p.h b/src/testlib/qtestregistry_p.h
new file mode 100644
index 0000000000..eab6d4356c
--- /dev/null
+++ b/src/testlib/qtestregistry_p.h
@@ -0,0 +1,36 @@
+// Copyright (C) 2022 The Qt Company Ltd.
+// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
+
+#ifndef QTESTREGISTRY_P_H
+#define QTESTREGISTRY_P_H
+
+#include <QString>
+#include <QtCore/qhash.h>
+#include <QtTest/qttestglobal.h>
+
+QT_REQUIRE_CONFIG(batch_test_support);
+
+QT_BEGIN_NAMESPACE
+
+namespace QTest {
+class TestRegistry {
+public:
+    using TestEntryFunction = int(*)(int argv, char** argc);
+
+    static TestRegistry* instance();
+
+    void registerTest(const QString& name, TestEntryFunction data);
+    size_t total() const {
+        return m_tests.size();
+    }
+    TestEntryFunction getTestEntryFunction(const QString& name) const;
+    QStringList getAllTestNames() const;
+
+private:
+    QHash<QString, TestEntryFunction> m_tests;
+};
+}  // namespace QTest
+
+QT_END_NAMESPACE
+
+#endif  // QTESTREGISTRY_P_H