qmltest: Enumerate test cases / functions without evaluating QML

Most, if not all, QML tests are written without any sort of dynamic
instantiation of the test data, so doing view.setSource() will evaluate
the whole source file, compute bindings, create items, windows, etc.

This is less then ideal when all you want is to list the test functions
using -functions, or when running a single test from the command line,
as in both cases we'll still actually evaluate every single QML file.

This makes it really hard to evaluate test output, e.g. from the CI,
especially with logging enabled, as even if a single test is requested,
the logs are filled with results from the loading of the other tests.

To improve the situation we use a non-instantiated QML component that
we then inspect its compilation data, looking for test cases and
functions.

In the future the implementation of TestCase's qtest_run* machinery
should be built on top of QTestLib instead of being reimplemented in
JavaScript, but this is left for later.

Change-Id: Ie5448208daf786e335583ab6bdfbc195891ec1f5
Reviewed-by: Simon Hausmann <simon.hausmann@qt.io>

3 files changed, 152 insertions, 29 deletions

diff --git a/src/imports/testlib/TestCase.qml b/src/imports/testlib/TestCase.qml
index 18af7e0ab0..0e7e09c65c 100644
--- a/src/imports/testlib/TestCase.qml
+++ b/src/imports/testlib/TestCase.qml
@@ -1739,11 +1739,6 @@ Item {
 
     /*! \internal */
     function qtest_run() {
-        if (util.printAvailableFunctions) {
-            completed = true
-            return
-        }
-
         if (TestLogger.log_start_test()) {
             qtest_results.reset()
             qtest_results.testCaseName = name
@@ -1894,29 +1889,9 @@ Item {
         }
     }
 
-
     Component.onCompleted: {
         QTestRootObject.hasTestCase = true;
         qtest_componentCompleted = true;
-
-        if (util.printAvailableFunctions) {
-            var testList = []
-            for (var prop in testCase) {
-                if (prop.indexOf("test_") != 0 && prop.indexOf("benchmark_") != 0)
-                    continue
-                var tail = prop.lastIndexOf("_data");
-                if (tail != -1 && tail == (prop.length - 5))
-                    continue
-                // Note: cannot run functions in TestCase elements
-                // that lack a name.
-                if (name.length > 0)
-                    testList.push(name + "::" + prop + "()")
-            }
-            testList.sort()
-            for (var index in testList)
-                console.log(testList[index])
-            return
-        }
         qtest_testId = TestLogger.log_register_test(name)
         if (optional)
             TestLogger.log_optional_test(qtest_testId)
diff --git a/src/qml/qml/qqmltypenamecache_p.h b/src/qml/qml/qqmltypenamecache_p.h
index 6e5cd0fb54..8ac25c4fbe 100644
--- a/src/qml/qml/qqmltypenamecache_p.h
+++ b/src/qml/qml/qqmltypenamecache_p.h
@@ -81,7 +81,7 @@ struct QQmlImportRef {
 
 class QQmlType;
 class QQmlEngine;
-class QQmlTypeNameCache : public QQmlRefCount
+class Q_QML_PRIVATE_EXPORT QQmlTypeNameCache : public QQmlRefCount
 {
 public:
     QQmlTypeNameCache(const QQmlImports &imports);
diff --git a/src/qmltest/quicktest.cpp b/src/qmltest/quicktest.cpp
index 993a411013..56f76aa880 100644
--- a/src/qmltest/quicktest.cpp
+++ b/src/qmltest/quicktest.cpp
@@ -63,6 +63,8 @@
 #include <QtCore/QTranslator>
 #include <QtTest/QSignalSpy>
 
+#include <private/qqmlcomponent_p.h>
+
 #ifdef QT_QMLTEST_WITH_WIDGETS
 #include <QtWidgets/QApplication>
 #endif
@@ -195,6 +197,133 @@ bool qWaitForSignal(QObject *obj, const char* signal, int timeout = 5000)
     return spy.size();
 }
 
+using namespace QV4::CompiledData;
+
+class TestCaseCollector
+{
+public:
+    typedef QList<QString> TestCaseList;
+
+    TestCaseCollector(const QFileInfo &fileInfo, QQmlEngine *engine)
+    {
+        QQmlComponent component(engine, fileInfo.absoluteFilePath());
+        m_errors += component.errors();
+
+        if (component.isReady()) {
+            CompilationUnit *rootCompilationUnit = QQmlComponentPrivate::get(&component)->compilationUnit;
+            TestCaseEnumerationResult result = enumerateTestCases(rootCompilationUnit);
+            m_testCases = result.testCases + result.finalizedPartialTestCases();
+            m_errors += result.errors;
+        }
+    }
+
+    TestCaseList testCases() const { return m_testCases; }
+    QList<QQmlError> errors() const { return m_errors; }
+
+private:
+    TestCaseList m_testCases;
+    QList<QQmlError> m_errors;
+
+    struct TestCaseEnumerationResult
+    {
+        TestCaseList testCases;
+        QList<QQmlError> errors;
+
+        // Partially constructed test cases
+        bool isTestCase = false;
+        TestCaseList testFunctions;
+        QString testCaseName;
+
+        TestCaseList finalizedPartialTestCases() const
+        {
+            TestCaseList result;
+            for (const QString &function : testFunctions)
+                result << QString(QStringLiteral("%1::%2")).arg(testCaseName).arg(function);
+            return result;
+        }
+
+        TestCaseEnumerationResult &operator<<(const TestCaseEnumerationResult &other)
+        {
+            testCases += other.testCases + other.finalizedPartialTestCases();
+            errors += other.errors;
+            return *this;
+        }
+    };
+
+    TestCaseEnumerationResult enumerateTestCases(CompilationUnit *compilationUnit, const Object *object = nullptr)
+    {
+        QQmlType testCaseType;
+        for (quint32 i = 0; i < compilationUnit->data->nImports; ++i) {
+            const Import *import = compilationUnit->data->importAt(i);
+            if (compilationUnit->stringAt(import->uriIndex) != QLatin1Literal("QtTest"))
+                continue;
+
+            QString testCaseTypeName(QStringLiteral("TestCase"));
+            QString typeQualifier = compilationUnit->stringAt(import->qualifierIndex);
+            if (!typeQualifier.isEmpty())
+                testCaseTypeName = typeQualifier % QLatin1Char('.') % testCaseTypeName;
+
+            testCaseType = compilationUnit->typeNameCache->query(testCaseTypeName).type;
+            if (testCaseType.isValid())
+                break;
+        }
+
+        TestCaseEnumerationResult result;
+
+        if (!object) // Start at root of compilation unit if not enumerating a specific child
+            object = compilationUnit->objectAt(compilationUnit->rootObjectIndex());
+
+        if (CompilationUnit *superTypeUnit = compilationUnit->resolvedTypes.value(object->inheritedTypeNameIndex)->compilationUnit) {
+            // We have a non-C++ super type, which could indicate we're a subtype of a TestCase
+            if (testCaseType.isValid() && superTypeUnit->url() == testCaseType.sourceUrl())
+                result.isTestCase = true;
+            else
+                result = enumerateTestCases(superTypeUnit);
+
+            if (result.isTestCase) {
+                // Look for override of name in this type
+                for (auto binding = object->bindingsBegin(); binding != object->bindingsEnd(); ++binding) {
+                    if (compilationUnit->stringAt(binding->propertyNameIndex) == QLatin1Literal("name")) {
+                        if (binding->type == QV4::CompiledData::Binding::Type_String) {
+                            result.testCaseName = compilationUnit->stringAt(binding->stringIndex);
+                        } else {
+                            QQmlError error;
+                            error.setUrl(compilationUnit->url());
+                            error.setLine(binding->location.line);
+                            error.setColumn(binding->location.column);
+                            error.setDescription(QStringLiteral("the 'name' property of a TestCase must be a literal string"));
+                            result.errors << error;
+                        }
+                        break;
+                    }
+                }
+
+                // Look for additional functions in this type
+                auto functionsEnd = compilationUnit->objectFunctionsEnd(object);
+                for (auto function = compilationUnit->objectFunctionsBegin(object); function != functionsEnd; ++function) {
+                    QString functionName = compilationUnit->stringAt(function->nameIndex);
+                    if (!(functionName.startsWith(QLatin1Literal("test_")) || functionName.startsWith(QLatin1Literal("benchmark_"))))
+                        continue;
+
+                    if (functionName.endsWith(QLatin1Literal("_data")))
+                        continue;
+
+                    result.testFunctions << functionName;
+                }
+            }
+        }
+
+        for (auto binding = object->bindingsBegin(); binding != object->bindingsEnd(); ++binding) {
+            if (binding->type == QV4::CompiledData::Binding::Type_Object) {
+                const Object *child = compilationUnit->objectAt(binding->value.objectIndex);
+                result << enumerateTestCases(compilationUnit, child);
+            }
+        }
+
+        return result;
+    }
+};
+
 int quick_test_main(int argc, char **argv, const char *name, const char *sourceDir)
 {
     // Peek at arguments to check for '-widgets' argument
@@ -339,7 +468,28 @@ int quick_test_main(int argc, char **argv, const char *name, const char *sourceD
         if (!fi.exists())
             continue;
 
-        QQuickView view ;
+        QQmlEngine engine;
+
+        TestCaseCollector testCaseCollector(fi, &engine);
+        if (!testCaseCollector.errors().isEmpty()) {
+            for (const QQmlError &error : testCaseCollector.errors())
+                qWarning() << error;
+            exit(1);
+        }
+
+        TestCaseCollector::TestCaseList availableTestFunctions = testCaseCollector.testCases();
+        if (QTest::printAvailableFunctions) {
+            for (const QString &function : availableTestFunctions)
+                qDebug("%s()", qPrintable(function));
+            continue;
+        }
+
+        static const QSet<QString> commandLineTestFunctions = QTest::testFunctions.toSet();
+        if (!commandLineTestFunctions.isEmpty() &&
+            !availableTestFunctions.toSet().intersects(commandLineTestFunctions))
+            continue;
+
+        QQuickView view(&engine, nullptr);
         view.setFlags(Qt::Window | Qt::WindowSystemMenuHint
                          | Qt::WindowTitleHint | Qt::WindowMinMaxButtonsHint
                          | Qt::WindowCloseButtonHint);
@@ -364,8 +514,6 @@ int quick_test_main(int argc, char **argv, const char *name, const char *sourceD
         else
             view.setSource(QUrl::fromLocalFile(path));
 
-        if (QTest::printAvailableFunctions)
-            continue;
         while (view.status() == QQuickView::Loading)
             QTest::qWait(10);
         if (view.status() == QQuickView::Error) {