wasm: add qtwasmtestlib

qtwasmtestlib supports writing asynchronous tests for the web platform. Asynchronous test functions differ from normal test functions in that they allow returning from the test function before the test has completed: void TestObject::testTimer() { QTimer::singleShot(100, [](){ completeTestFunction(); // Test pass if we get here }); } Currently one logging backend is supported which writes the results to an html element. See the README file for further documentation. Change-Id: Ia633ad3f41a653e40d6bf35dd09d62a97c608f84 Reviewed-by: Mikołaj Boc <Mikolaj.Boc@qt.io> Reviewed-by: Tor Arne Vestbø <tor.arne.vestbo@qt.io>
author: Morten Johan Sørvig <morten.sorvig@qt.io> 2022-01-12 10:40:08 +0100
committer: Morten Sørvig <morten.sorvig@qt.io> 2022-06-29 15:09:02 +0200
commit: 122aa530d6b335d582f9809d53614ac48211f086 (patch)
tree: a4353f1b9f9be23ec692ab44a088f11207513767 /tests
parent: c27cca5c3421b08253535cfcfb9dd414986c7653 (diff)
5 files changed, 416 insertions, 0 deletions
diff --git a/tests/manual/wasm/README.md b/tests/manual/wasm/README.md
index 5117e2f70b..9266f38cc6 100644
--- a/tests/manual/wasm/README.md
+++ b/tests/manual/wasm/README.md
@@ -12,3 +12,4 @@ Content
     eventloop       Event loops, application startup, dialog exec()
     localfiles      Local file download and upload
     rasterwindow    Basic GUI app, event handling
+    qtwasmtestlib   native auto test framework
diff --git a/tests/manual/wasm/qtwasmtestlib/README.md b/tests/manual/wasm/qtwasmtestlib/README.md
new file mode 100644
index 0000000000..e122fb26bc
--- /dev/null
+++ b/tests/manual/wasm/qtwasmtestlib/README.md
@@ -0,0 +1,72 @@
+QtWasmTestLib - async auto tests for WebAssembly
+================================================
+
+QtWasmTestLib supports auto-test cases in the web browser. Like QTestLib, each
+test case is defined by a QObject subclass with one or more test functions. The
+test functions may be asynchronous, where they return early and then complete
+at some later point.
+
+The test lib is implemented as a C++ and JavaScript library, where the test is written
+using C++ and a hosting html page calls JavaScript API to run the test.
+
+Implementing a basic test case
+------------------------------
+
+In the test cpp file, define the test functions as private slots. All test
+functions must call completeTestFunction() exactly once, or will time out
+otherwise. The can can be made after the test function itself has returned.
+
+    class TestTest: public QObject
+    {
+        Q_OBJECT
+    private slots:
+        void timerTest() {
+            QTimer::singleShot(timeout, [](){
+                completeTestFunction();
+             });
+        }
+    };
+
+Then define a main() function which calls initTestCase(). The main()
+function is async too, as per Emscripten default. Build the .cpp file
+as a normal Qt for WebAssembly app.
+
+    int main(int argc, char **argv)
+    {
+        auto testObject = std::make_shared<TestTest>();
+        initTestCase<QCoreApplication>(argc, argv, testObject);
+        return 0;
+    }
+
+Finally provide an html file which hosts the test runner and calls runTestCase()
+
+    <!doctype html>
+    <script type="text/javascript" src="qtwasmtestlib.js"></script>
+    <script type="text/javascript" src="test_case.js"></script>
+    <script>
+        window.onload = async () => {
+            runTestCase(document.getElementById("log"));
+        };
+    </script>
+    <p>Running Foo auto test.</p>
+    <div id="log"></div>
+
+Implementing a GUI test case
+----------------------------
+
+This is similar to implementing a basic test case, with the difference that the hosting
+html file provides container elements which becomes QScreens for the test code.
+
+    <!doctype html>
+    <script type="text/javascript" src="qtwasmtestlib.js"></script>
+    <script type="text/javascript" src="test_case.js"></script>
+    <script>
+        window.onload = async () => {
+            let log = document.getElementById("log")
+            let containers = [document.getElementById("container")];
+            runTestCase(log, containers);
+        };
+    </script>
+    <p>Running Foo auto test.</p>
+    <div id="container"></div>
+    <div id="log"></div>
diff --git a/tests/manual/wasm/qtwasmtestlib/qtwasmtestlib.cpp b/tests/manual/wasm/qtwasmtestlib/qtwasmtestlib.cpp
new file mode 100644
index 0000000000..eeac82a266
--- /dev/null
+++ b/tests/manual/wasm/qtwasmtestlib/qtwasmtestlib.cpp
@@ -0,0 +1,135 @@
+// Copyright (C) 2022 The Qt Company Ltd.
+// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0
+
+#include "qtwasmtestlib.h"
+
+#include <QtCore/qmetaobject.h>
+
+#include <emscripten/bind.h>
+#include <emscripten.h>
+#include <emscripten/threading.h>
+
+namespace QtWasmTest {
+
+QObject *g_testObject = nullptr;
+std::function<void ()> g_cleanup;
+
+void runOnMainThread(std::function<void(void)> fn);
+static bool isValidSlot(const QMetaMethod &sl);
+
+
+//
+// Public API
+//
+
+// Initializes the test case with a test object and cleanup function. The
+// cleanup function is called when all test functions have completed.
+void initTestCase(QObject *testObject, std::function<void ()> cleanup)
+{
+    g_testObject = testObject;
+    g_cleanup = cleanup;
+}
+
+// Completes the currently running test function with a result. This function is
+// thread-safe and call be called from any thread.
+void completeTestFunction(TestResult result)
+{
+    // Report test result to JavaScript test runner, on the main thread
+    runOnMainThread([result](){
+        const char *resultString;
+        switch (result) {
+            case TestResult::Pass:
+                resultString = "PASS";
+            break;
+            case TestResult::Fail:
+                resultString = "FAIL";
+            break;
+        };
+        EM_ASM({
+            completeTestFunction(UTF8ToString($0));
+        }, resultString);
+    });
+}
+
+//
+// Private API for the Javascript test runnner
+//
+
+void cleanupTestCase()
+{
+    g_testObject = nullptr;
+    g_cleanup();
+}
+
+std::string getTestFunctions()
+{
+    std::string testFunctions;
+
+    // Duplicate qPrintTestSlots (private QTestLib function) logic.
+    for (int i = 0; i < g_testObject->metaObject()->methodCount(); ++i) {
+        QMetaMethod sl = g_testObject->metaObject()->method(i);
+        if (!isValidSlot(sl))
+            continue;
+        QByteArray signature = sl.methodSignature();
+        Q_ASSERT(signature.endsWith("()"));
+        signature.chop(2);
+        if (!testFunctions.empty())
+            testFunctions += " ";
+        testFunctions += std::string(signature.constData());
+    }
+
+    return testFunctions;
+}
+
+void runTestFunction(std::string name)
+{
+    QMetaObject::invokeMethod(g_testObject, name.c_str());
+}
+
+EMSCRIPTEN_BINDINGS(qtwebtestrunner) {
+    emscripten::function("cleanupTestCase", &cleanupTestCase);
+    emscripten::function("getTestFunctions", &getTestFunctions);
+    emscripten::function("runTestFunction", &runTestFunction);
+}
+
+//
+// Test lib implementation
+//
+
+static bool isValidSlot(const QMetaMethod &sl)
+{
+    if (sl.access() != QMetaMethod::Private || sl.parameterCount() != 0
+        || sl.returnType() != QMetaType::Void || sl.methodType() != QMetaMethod::Slot)
+        return false;
+    const QByteArray name = sl.name();
+    return !(name.isEmpty() || name.endsWith("_data")
+        || name == "initTestCase" || name == "cleanupTestCase"
+        || name == "init" || name == "cleanup");
+}
+
+void trampoline(void *context)
+{
+    Q_ASSERT(emscripten_is_main_runtime_thread());
+
+    emscripten_async_call([](void *context) {
+        std::function<void(void)> *fn = reinterpret_cast<std::function<void(void)> *>(context);
+        (*fn)();
+        delete fn;
+    }, context, 0);
+}
+
+// Runs the given function on the main thread, asynchronously
+void runOnMainThread(std::function<void(void)> fn)
+{
+    void *context = new std::function<void(void)>(fn);
+    if (emscripten_is_main_runtime_thread()) {
+        trampoline(context);
+    } else {
+#if QT_CONFIG(thread)
+        emscripten_async_run_in_main_runtime_thread_(EM_FUNC_SIG_VI, reinterpret_cast<void *>(trampoline), context);
+#endif
+    }
+}
+
+} // namespace QtWasmTest
+
diff --git a/tests/manual/wasm/qtwasmtestlib/qtwasmtestlib.h b/tests/manual/wasm/qtwasmtestlib/qtwasmtestlib.h
new file mode 100644
index 0000000000..8bf0c20259
--- /dev/null
+++ b/tests/manual/wasm/qtwasmtestlib/qtwasmtestlib.h
@@ -0,0 +1,35 @@
+// Copyright (C) 2022 The Qt Company Ltd.
+// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0
+
+#ifndef QT_WASM_TESTRUNNER_H
+#define QT_WASM_TESTRUNNER_H
+
+#include <QtCore/qobject.h>
+
+#include <functional>
+
+namespace QtWasmTest {
+
+enum TestResult {
+    Pass,
+    Fail,
+};
+void completeTestFunction(TestResult result = TestResult::Pass);
+void initTestCase(QObject *testObject, std::function<void ()> cleanup);
+template <typename App>
+void initTestCase(int argc, char **argv, std::shared_ptr<QObject> testObject)
+{
+    auto app = std::make_shared<App>(argc, argv);
+    auto cleanup = [testObject, app]() mutable {
+        // C++ lambda capture destruction order is unspecified;
+        // delete test before app by calling reset().
+        testObject.reset();
+        app.reset();
+    };
+    initTestCase(testObject.get(), cleanup);
+}
+
+}
+
+#endif
+
diff --git a/tests/manual/wasm/qtwasmtestlib/qtwasmtestlib.js b/tests/manual/wasm/qtwasmtestlib/qtwasmtestlib.js
new file mode 100644
index 0000000000..add3c46619
--- /dev/null
+++ b/tests/manual/wasm/qtwasmtestlib/qtwasmtestlib.js
@@ -0,0 +1,173 @@
+/****************************************************************************
+**
+** Copyright (C) 2022 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of the examples of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:BSD$
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** BSD License Usage
+** Alternatively, you may use this file under the terms of the BSD license
+** as follows:
+**
+** "Redistribution and use in source and binary forms, with or without
+** modification, are permitted provided that the following conditions are
+** met:
+**   * Redistributions of source code must retain the above copyright
+**     notice, this list of conditions and the following disclaimer.
+**   * Redistributions in binary form must reproduce the above copyright
+**     notice, this list of conditions and the following disclaimer in
+**     the documentation and/or other materials provided with the
+**     distribution.
+**   * Neither the name of The Qt Company Ltd nor the names of its
+**     contributors may be used to endorse or promote products derived
+**     from this software without specific prior written permission.
+**
+**
+** THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+** "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+** LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+** A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+** OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+** SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+** LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+** DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+** THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+** (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+** OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE."
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+
+// A minimal async test runner for Qt async auto tests.
+//
+// Usage: Call runTest(name, testFunctionCompleted), where "name" is the name of the app
+// (the .wasm file name), and testFunctionCompleted is a test-function-complete
+// callback. The test runner will then instantiate the app and run tests.
+//
+// The test runner expects that the app instance defines the following
+// functions:
+//
+//  void cleanupTestCase()
+//  string getTestFunctions()
+//  runTestFunction(string)
+//
+// Further, the test runner expects that the app instance calls
+// completeTestFunction() (below - note that both the instance and this
+// file have a function with that name) when a test function finishes. This
+// can be done during runTestFunction(), or after it has returned (this
+// is the part which enables async testing). Test functions which fail
+// to call completeTestFunction() will time out after 2000ms.
+//
+let g_maxTime = 2000;
+var g_timeoutId = undefined;
+var g_testResolve = undefined;
+var g_testResult = undefined;
+
+function completeTestFunction(result)
+{
+    // Reset timeout
+    if (g_timeoutId !== undefined) {
+        clearTimeout(g_timeoutId);
+        g_timeoutId = undefined;
+    }
+
+    // Set test result directy, or resolve the pending promise
+    if (g_testResolve === undefined) {
+        g_testResult = result
+    } else {
+        g_testResolve(result);
+        g_testResolve = undefined;
+    }
+}
+
+function runTestFunction(instance, name)
+{
+    if (g_timeoutId !== undefined)
+        console.log("existing timer found");
+
+    // Set timer which will catch test functions
+    // which fail to call completeTestFunction()
+    g_timeoutId = setTimeout( () => {
+        if (g_timeoutId === undefined)
+            return;
+        g_timeoutId = undefined;
+        completeTestFunction("FAIL")
+    }, g_maxTime);
+
+    instance.runTestFunction(name);
+
+    // If the test function completed with a result immediately then return
+    // the result directly, otherwise return a Promise to the result.
+    if (g_testResult !== undefined) {
+        let result = g_testResult;
+        g_testResult = undefined;
+        return result;
+    } else {
+        return new Promise((resolve) => {
+            g_testResolve = resolve;
+        });
+    }
+}
+
+async function runTestCaseImpl(testFunctionStarted, testFunctionCompleted, qtContainers)
+{
+    // Create test case instance
+    let config = {
+        qtContainerElements : qtContainers || []
+    }
+    let instance = await createQtAppInstance(config);
+
+    // Run all test functions
+    let functionsString = instance.getTestFunctions();
+    let functions = functionsString.split(" ").filter(Boolean);
+    for (name of functions) {
+        testFunctionStarted(name);
+        let result = await runTestFunction(instance, name);
+        testFunctionCompleted(name, result);
+    }
+
+    // Cleanup
+    instance.cleanupTestCase();
+}
+
+var g_htmlLogElement = undefined;
+
+function testFunctionStarted(name) {
+    let line = name + ": ";
+    g_htmlLogElement.innerHTML += line;
+}
+
+function testFunctionCompleted(name, status) {
+    var color = "black";
+    switch (status) {
+        case "PASS":
+            color = "green";
+        break;
+        case "FAIL":
+            color = "red";
+        break;
+    }
+    let line = "<text style='color:" + color + ";'>" + status + "</text><br>";
+    g_htmlLogElement.innerHTML += line;
+}
+
+async function runTestCase(htmlLogElement, qtContainers)
+{
+    g_htmlLogElement = htmlLogElement;
+    try {
+        await runTestCaseImpl(testFunctionStarted, testFunctionCompleted, qtContainers);
+        g_htmlLogElement.innerHTML += "<br> DONE"
+    } catch (err) {
+        g_htmlLogElement.innerHTML += err
+    }
+}
author	Morten Johan Sørvig <morten.sorvig@qt.io>	2022-01-12 10:40:08 +0100
committer	Morten Sørvig <morten.sorvig@qt.io>	2022-06-29 15:09:02 +0200
commit	122aa530d6b335d582f9809d53614ac48211f086 (patch)
tree	a4353f1b9f9be23ec692ab44a088f11207513767 /tests
parent	c27cca5c3421b08253535cfcfb9dd414986c7653 (diff)