Enhance the qtwasmtestlib with comparison functions and status reporting

Added the functionality to report text statuses from tests, reporting
file and line of assertion failures. Refactored the qtwasmtestlib.js
for improved stability.

Task-number: QTBUG-99611
Change-Id: I717e0cc38ac7f155fe870710f6b5e4bfb81b9317
Reviewed-by: Morten Johan Sørvig <morten.sorvig@qt.io>

4 files changed, 131 insertions, 83 deletions

diff --git a/tests/manual/wasm/qtwasmtestlib/README.md b/tests/manual/wasm/qtwasmtestlib/README.md
index e122fb26bc..515c33ae6a 100644
--- a/tests/manual/wasm/qtwasmtestlib/README.md
+++ b/tests/manual/wasm/qtwasmtestlib/README.md
@@ -14,7 +14,10 @@ Implementing a basic test case
 
 In the test cpp file, define the test functions as private slots. All test
 functions must call completeTestFunction() exactly once, or will time out
-otherwise. The can can be made after the test function itself has returned.
+otherwise. Subsequent calls to completeTestFunction will be disregarded.
+It is advised to use QWASMSUCCESS/QWASMFAIL for reporting the test execution
+status and QWASMCOMPARE/QWASMVERIFY to assert on test conditions. The call can
+be made after the test function itself has returned.
 
     class TestTest: public QObject
     {
diff --git a/tests/manual/wasm/qtwasmtestlib/qtwasmtestlib.cpp b/tests/manual/wasm/qtwasmtestlib/qtwasmtestlib.cpp
index eeac82a266..dd7d11c398 100644
--- a/tests/manual/wasm/qtwasmtestlib/qtwasmtestlib.cpp
+++ b/tests/manual/wasm/qtwasmtestlib/qtwasmtestlib.cpp
@@ -10,9 +10,11 @@
 #include <emscripten/threading.h>
 
 namespace QtWasmTest {
-
+namespace {
 QObject *g_testObject = nullptr;
+std::string g_currentTestName;
 std::function<void ()> g_cleanup;
+}
 
 void runOnMainThread(std::function<void(void)> fn);
 static bool isValidSlot(const QMetaMethod &sl);
@@ -30,24 +32,25 @@ void initTestCase(QObject *testObject, std::function<void ()> cleanup)
     g_cleanup = cleanup;
 }
 
+void verify(bool condition, std::string_view conditionString, std::string_view file, int line)
+{
+    if (!condition) {
+        completeTestFunction(
+            TestResult::Fail,
+            formatMessage(file, line, "Condition failed: " + std::string(conditionString)));
+    }
+}
+
 // Completes the currently running test function with a result. This function is
 // thread-safe and call be called from any thread.
-void completeTestFunction(TestResult result)
+void completeTestFunction(TestResult result, std::string message)
 {
+
     // Report test result to JavaScript test runner, on the main thread
-    runOnMainThread([result](){
-        const char *resultString;
-        switch (result) {
-            case TestResult::Pass:
-                resultString = "PASS";
-            break;
-            case TestResult::Fail:
-                resultString = "FAIL";
-            break;
-        };
+    runOnMainThread([resultString = result == TestResult::Pass ? "PASS" : "FAIL", message](){
         EM_ASM({
-            completeTestFunction(UTF8ToString($0));
-        }, resultString);
+            completeTestFunction(UTF8ToString($0), UTF8ToString($1), UTF8ToString($2));
+        }, g_currentTestName.c_str(), resultString, message.c_str());
     });
 }
 
@@ -55,6 +58,11 @@ void completeTestFunction(TestResult result)
 // Private API for the Javascript test runnner
 //
 
+std::string formatMessage(std::string_view file, int line, std::string_view message)
+{
+    return "[" + std::string(file) + ":" + QString::number(line).toStdString() + "] " + std::string(message);
+}
+
 void cleanupTestCase()
 {
     g_testObject = nullptr;
@@ -83,6 +91,8 @@ std::string getTestFunctions()
 
 void runTestFunction(std::string name)
 {
+    g_currentTestName = name;
+    QMetaObject::invokeMethod(g_testObject, "init");
     QMetaObject::invokeMethod(g_testObject, name.c_str());
 }
 
diff --git a/tests/manual/wasm/qtwasmtestlib/qtwasmtestlib.h b/tests/manual/wasm/qtwasmtestlib/qtwasmtestlib.h
index 8bf0c20259..f4b0f75241 100644
--- a/tests/manual/wasm/qtwasmtestlib/qtwasmtestlib.h
+++ b/tests/manual/wasm/qtwasmtestlib/qtwasmtestlib.h
@@ -14,10 +14,17 @@ enum TestResult {
     Pass,
     Fail,
 };
-void completeTestFunction(TestResult result = TestResult::Pass);
+
+std::string formatMessage(std::string_view file,
+                          int line,
+                          std::string_view message);
+
+void completeTestFunction(TestResult result, std::string message);
 void initTestCase(QObject *testObject, std::function<void ()> cleanup);
 template <typename App>
-void initTestCase(int argc, char **argv, std::shared_ptr<QObject> testObject)
+void initTestCase(int argc,
+                  char **argv,
+                std::shared_ptr<QObject> testObject)
 {
     auto app = std::make_shared<App>(argc, argv);
     auto cleanup = [testObject, app]() mutable {
@@ -28,8 +35,37 @@ void initTestCase(int argc, char **argv, std::shared_ptr<QObject> testObject)
     };
     initTestCase(testObject.get(), cleanup);
 }
+void verify(bool condition,
+            std::string_view conditionString,
+            std::string_view file,
+            int line);
 
+template<class L, class R>
+void compare(const L& lhs,
+             const R& rhs,
+             std::string_view lhsString,
+             std::string_view rhsString,
+             std::string_view file,
+             int line) {
+    if (lhs != rhs) {
+        completeTestFunction(
+            TestResult::Fail,
+            formatMessage(file, line, "Comparison failed: " + std::string(lhsString) + " == " + std::string(rhsString)));
+    }
 }
 
-#endif
+}  // namespace QtWasmTest
+
+#define QWASMVERIFY(condition) \
+    QtWasmTest::verify((condition), #condition, __FILE__, __LINE__);
 
+#define QWASMCOMPARE(left, right) \
+    QtWasmTest::compare((left), (right), #left, #right, __FILE__, __LINE__);
+
+#define QWASMSUCCESS() \
+    QtWasmTest::completeTestFunction(QtWasmTest::Pass, "")
+
+#define QWASMFAIL(message) \
+    QtWasmTest::completeTestFunction(QtWasmTest::Fail, QtWasmTest::formatMessage(__FILE__, __LINE__, message))
+
+#endif
diff --git a/tests/manual/wasm/qtwasmtestlib/qtwasmtestlib.js b/tests/manual/wasm/qtwasmtestlib/qtwasmtestlib.js
index add3c46619..96ff3d81a7 100644
--- a/tests/manual/wasm/qtwasmtestlib/qtwasmtestlib.js
+++ b/tests/manual/wasm/qtwasmtestlib/qtwasmtestlib.js
@@ -68,72 +68,80 @@
 // is the part which enables async testing). Test functions which fail
 // to call completeTestFunction() will time out after 2000ms.
 //
-let g_maxTime = 2000;
-var g_timeoutId = undefined;
-var g_testResolve = undefined;
-var g_testResult = undefined;
-
-function completeTestFunction(result)
-{
-    // Reset timeout
-    if (g_timeoutId !== undefined) {
-        clearTimeout(g_timeoutId);
-        g_timeoutId = undefined;
+const g_maxTime = 2000;
+
+class TestFunction {
+    constructor(instance, name) {
+        this.instance = instance;
+        this.name = name;
+        this.resolve = undefined;
+        this.reject = undefined;
+        this.timeoutId = undefined;
+    }
+
+    complete(result, details) {
+        // Reset timeout
+        clearTimeout(this.timeoutId);
+        this.timeoutId = undefined;
+
+        const callback = result.startsWith('FAIL') ? this.reject : this.resolve;
+        callback(`${result}${details ? ': ' + details : ''}`);
     }
 
-    // Set test result directy, or resolve the pending promise
-    if (g_testResolve === undefined) {
-        g_testResult = result
-    } else {
-        g_testResolve(result);
-        g_testResolve = undefined;
+    run() {
+        // Set timer which will catch test functions
+        // which fail to call completeTestFunction()
+        this.timeoutId = setTimeout(() => {
+            completeTestFunction(this.name, 'FAIL', `Timeout after ${g_maxTime} ms`)
+        }, g_maxTime);
+
+        return new Promise((resolve, reject) => {
+            this.resolve = resolve;
+            this.reject = reject;
+
+            this.instance.runTestFunction(this.name);
+        });
     }
+};
+
+function completeTestFunction(testFunctionName, result, details) {
+    if (!window.currentTestFunction || testFunctionName !== window.currentTestFunction.name)
+        return;
+
+    window.currentTestFunction.complete(result, details);
 }
 
-function runTestFunction(instance, name)
-{
-    if (g_timeoutId !== undefined)
-        console.log("existing timer found");
-
-    // Set timer which will catch test functions
-    // which fail to call completeTestFunction()
-    g_timeoutId = setTimeout( () => {
-        if (g_timeoutId === undefined)
-            return;
-        g_timeoutId = undefined;
-        completeTestFunction("FAIL")
-    }, g_maxTime);
-
-    instance.runTestFunction(name);
-
-    // If the test function completed with a result immediately then return
-    // the result directly, otherwise return a Promise to the result.
-    if (g_testResult !== undefined) {
-        let result = g_testResult;
-        g_testResult = undefined;
+async function runTestFunction(instance, name) {
+    if (window.currentTestFunction) {
+        throw new Error(`While trying to run ${name}: Last function hasn't yet finished`);
+    }
+    window.currentTestFunction = new TestFunction(instance, name);
+    try {
+        const result = await window.currentTestFunction.run();
         return result;
-    } else {
-        return new Promise((resolve) => {
-            g_testResolve = resolve;
-        });
+    } finally {
+        delete window.currentTestFunction;
     }
 }
 
-async function runTestCaseImpl(testFunctionStarted, testFunctionCompleted, qtContainers)
-{
+async function runTestCaseImpl(testFunctionStarted, testFunctionCompleted, qtContainers) {
     // Create test case instance
-    let config = {
-        qtContainerElements : qtContainers || []
+    const config = {
+        qtContainerElements: qtContainers || []
     }
-    let instance = await createQtAppInstance(config);
+    const instance = await createQtAppInstance(config);
 
     // Run all test functions
-    let functionsString = instance.getTestFunctions();
-    let functions = functionsString.split(" ").filter(Boolean);
-    for (name of functions) {
+    const functionsString = instance.getTestFunctions();
+    const functions = functionsString.split(" ").filter(Boolean);
+    for (const name of functions) {
         testFunctionStarted(name);
-        let result = await runTestFunction(instance, name);
-        testFunctionCompleted(name, result);
+        try {
+            const result = await runTestFunction(instance, name);
+            testFunctionCompleted(result);
+        } catch (err) {
+            testFunctionCompleted(err.message ?? err);
+        }
     }
 
     // Cleanup
@@ -147,22 +155,13 @@ function testFunctionStarted(name) {
     g_htmlLogElement.innerHTML += line;
 }
 
-function testFunctionCompleted(name, status) {
-    var color = "black";
-    switch (status) {
-        case "PASS":
-            color = "green";
-        break;
-        case "FAIL":
-            color = "red";
-        break;
-    }
-    let line = "<text style='color:" + color + ";'>" + status + "</text><br>";
+function testFunctionCompleted(status) {
+    const color = status.startsWith("PASS") ? "green" : status.startsWith("FAIL") ? "red" : "black";
+    let line = `<span style='color: ${color};'>${status}</text><br>`;
     g_htmlLogElement.innerHTML += line;
 }
 
-async function runTestCase(htmlLogElement, qtContainers)
-{
+async function runTestCase(htmlLogElement, qtContainers) {
     g_htmlLogElement = htmlLogElement;
     try {
         await runTestCaseImpl(testFunctionStarted, testFunctionCompleted, qtContainers);