Adapt the js batched test runner to emrun interface

This makes the js batched test runner cooperate with emrun. The runner sends back the output and exit messages to emrun to inform it about the test execution state. The code is based on emrun_postjs.js from the emsdk. Change-Id: I758f2c185797f4000810eb4314423eebc1c5d457 Reviewed-by: David Skoland <david.skoland@qt.io>
author: Mikolaj Boc <mikolaj.boc@qt.io> 2022-09-07 15:38:08 +0200
committer: Mikolaj Boc <mikolaj.boc@qt.io> 2022-09-12 22:28:10 +0200
commit: 7dbbe0a22251085b00706c4eb997ac69148c4d70 (patch)
tree: 49f8a6e86653bda4bbeaa1963c63ca91731ac546 /util
parent: 84c494d0f27b37c59132151b924fdf5adbc542e5 (diff)
6 files changed, 302 insertions, 108 deletions
diff --git a/util/wasm/batchedtestrunner/README.md b/util/wasm/batchedtestrunner/README.md
index 5098cd405d..a5d165c630 100644
--- a/util/wasm/batchedtestrunner/README.md
+++ b/util/wasm/batchedtestrunner/README.md
@@ -5,8 +5,24 @@ prints out a list of test classes inside its module. Then, when run with the fir
 equal to the name of one of the test classes, the test program will execute all tests within
 that single class.
 
-The scripts in the page will load the wasm file called 'test_batch.wasm' with its corresponding
-js script 'test_batch.js'.
+The following query parameters are recognized by the webpage:
+
+qtestname=testname - the test case to run. When batched test module is used, the test is assumed to
+    be a part of the batch. If a standalone test module is used, this is assumed to be the name of
+    the wasm module.
+
+quseemrun - if specified, the test communicates with the emrun instance via the protocol expected
+    by emrun.
+
+qtestoutputformat=txt|xml|lightxml|junitxml|tap - specifies the output format for the test case.
+
+qbatchedtest - if specified, the script will load the test_batch.wasm module and either run all
+    testcases in it or a specific test case, depending on the existence of the qtestname parameter.
+    Otherwise, the test is assumed to be a standalone binary whose name is determined by the
+    qtestname parameter.
+
+The scripts in the page will load the wasm file as specified by a combination of qbatchedtest and
+qtestname.
 
 Public interface for querying the test execution status is accessible via the global object
 'qtTestRunner':
@@ -15,8 +31,8 @@ qtTestRunner.status - this contains the status of the test runner itself, of the
 RunnerStatus.
 
 qtTestRunner.results - a map of test class name to test result. The result contains a test status
-(status, of the enumeration TestStatus), and in case of a terminal status, also the test's exit code
-(exitCode) and xml text output (textOutput), if available.
+(status, of the enumeration TestStatus), text output chunks (output), and in case of a terminal
+status, also the test's exit code (exitCode)
 
 qtTestRunner.onStatusChanged - an event for changes in state of the runner itself. The possible
 values are those of the enumeration RunnerStatus.
@@ -39,3 +55,6 @@ Query for test execution state:
     - qtTestRunner.status === (...)
     - qtTestRunner.results['tst_mytest'].status === (...)
     - qtTestRunner.results['tst_mytest'].textOutput
+
+When queseemrun is specified, the built-in emrun support module will POST the test output to the
+emrun instance and will report ^exit^ with a suitable exit code to it when testing is finished.
diff --git a/util/wasm/batchedtestrunner/batchedtestrunner.html b/util/wasm/batchedtestrunner/batchedtestrunner.html
index 123c24890b..14a9fa1807 100644
--- a/util/wasm/batchedtestrunner/batchedtestrunner.html
+++ b/util/wasm/batchedtestrunner/batchedtestrunner.html
@@ -7,8 +7,8 @@ SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only
 <html>
 <head>
     <meta charset="utf-8">
-    <title>WASM batched test runner</title>
-    <script type="module" defer="defer" src="batchedtestrunner.js"></script>
+    <title>WASM batched test runner (emrun-enabled)</title>
+    <script type="module" defer="defer" src="qwasmtestmain.js"></script>
 </head>
 <body></body>
 </html>
diff --git a/util/wasm/batchedtestrunner/batchedtestrunner.js b/util/wasm/batchedtestrunner/batchedtestrunner.js
index c2fa0e83d7..e26e561b5a 100644
--- a/util/wasm/batchedtestrunner/batchedtestrunner.js
+++ b/util/wasm/batchedtestrunner/batchedtestrunner.js
@@ -3,12 +3,9 @@
 
 import {
     AbortedError,
-    ModuleLoader,
-    ResourceFetcher,
-    ResourceLocator,
 } from './qwasmjsruntime.js';
 
-import { parseQuery, EventSource } from './util.js';
+import { EventSource } from './util.js';
 
 class ProgramError extends Error {
     constructor(exitCode) {
@@ -16,26 +13,34 @@ class ProgramError extends Error {
     }
 }
 
-class RunnerStatus {
+export class RunnerStatus {
     static Running = 'Running';
-    static Completed = 'Completed';
+    static Passed = 'Passed';
     static Error = 'Error';
+    static TestCrashed = 'TestCrashed';
+    static TestsFailed = 'TestsFailed';
 }
 
-class TestStatus {
+export class TestStatus {
     static Pending = 'Pending';
     static Running = 'Running';
     static Completed = 'Completed';
     static Error = 'Error';
+    static Failed = 'Failed';
     static Crashed = 'Crashed';
 }
 
-// Represents the public API of the runner.
-class WebApi {
+export class BatchedTestRunner {
+    static #TestBatchModuleName = 'test_batch';
+
+    #loader;
+
     #results = new Map();
     #status = RunnerStatus.Running;
+    #numberOfFailed = 0;
     #statusChangedEventPrivate;
     #testStatusChangedEventPrivate;
+    #testOutputChangedEventPrivate;
     #errorDetails;
 
     onStatusChanged =
@@ -43,137 +48,130 @@ class WebApi {
     onTestStatusChanged =
         new EventSource((privateInterface) =>
             this.#testStatusChangedEventPrivate = privateInterface);
+    onTestOutputChanged =
+        new EventSource(
+            (privateInterface) => this.#testOutputChangedEventPrivate = privateInterface);
 
-    // The callback receives the private interface of this object, meant not to be used by the
-    // end user on the web side.
-    constructor(receivePrivateInterface) {
-        receivePrivateInterface({
-            registerTest: testName => this.#registerTest(testName),
-            setTestStatus: (testName, status) => this.#setTestStatus(testName, status),
-            setTestResultData: (testName, testStatus, exitCode, textOutput) =>
-                this.#setTestResultData(testName, testStatus, exitCode, textOutput),
-            setTestRunnerStatus: status => this.#setTestRunnerStatus(status),
-            setTestRunnerError: details => this.#setTestRunnerError(details),
-        });
+    constructor(loader) {
+        this.#loader = loader;
     }
 
     get results() { return this.#results; }
-    get status() { return this.#status; }
-    get errorDetails() { return this.#errorDetails; }
 
-    #registerTest(testName) { this.#results.set(testName, { status: TestStatus.Pending }); }
+    get status() { return this.#status; }
 
-    #setTestStatus(testName, status) {
-        const testData = this.#results.get(testName);
-        if (testData.status === status)
-            return;
-        this.#results.get(testName).status = status;
-        this.#testStatusChangedEventPrivate.fireEvent(testName, status);
+    get numberOfFailed() {
+        if (this.#status !== RunnerStatus.TestsFailed)
+            throw new Error(`numberOfFailed called with status=${this.#status}`);
+        return this.#numberOfFailed;
     }
 
-    #setTestResultData(testName, testStatus, exitCode, textOutput) {
-        const testData = this.#results.get(testName);
-        const statusChanged = testStatus !== testData.status;
-        testData.status = testStatus;
-        testData.exitCode = exitCode;
-        testData.textOutput = textOutput;
-        if (statusChanged)
-            this.#testStatusChangedEventPrivate.fireEvent(testName, testStatus);
-    }
+    get errorDetails() { return this.#errorDetails; }
 
-    #setTestRunnerStatus(status) {
-        if (status === this.#status)
+    async run(targetIsBatch, testName, testOutputFormat) {
+        try {
+            await this.#doRun(targetIsBatch, testName, testOutputFormat);
+        } catch (e) {
+            this.#setTestRunnerError(e.message);
             return;
-        this.#status = status;
-        this.#statusChangedEventPrivate.fireEvent(status);
-    }
-
-    #setTestRunnerError(details) {
-        this.#status = RunnerStatus.Error;
-        this.#errorDetails = details;
-        this.#statusChangedEventPrivate.fireEvent(RunnerStatus.Error);
-    }
-}
-
-class BatchedTestRunner {
-    static #TestBatchModuleName = 'test_batch';
+        }
 
-    #loader;
-    #privateWebApi;
+        const status = (() => {
+            const hasAnyCrashedTest =
+                !![...window.qtTestRunner.results.values()].find(
+                    result => result.status === TestStatus.Crashed);
+            if (hasAnyCrashedTest)
+                return { code: RunnerStatus.TestCrashed };
+            const numberOfFailed = [...window.qtTestRunner.results.values()].reduce(
+                (previous, current) => previous + current.exitCode, 0);
+            return {
+                code: (numberOfFailed ? RunnerStatus.TestsFailed : RunnerStatus.Passed),
+                numberOfFailed
+            };
+        })();
 
-    constructor(loader, privateWebApi) {
-        this.#loader = loader;
-        this.#privateWebApi = privateWebApi;
+        this.#setTestRunnerStatus(status.code, status.numberOfFailed);
     }
 
-    async #doRun(testName, testOutputFormat) {
+    async #doRun(targetIsBatch, testName, testOutputFormat) {
         const module = await this.#loader.loadEmscriptenModule(
-            BatchedTestRunner.#TestBatchModuleName,
+            targetIsBatch ? BatchedTestRunner.#TestBatchModuleName : testName,
             () => { }
         );
 
-        const testsToExecute = testName ? [testName] : await this.#getTestClassNames(module);
-        testsToExecute.forEach(testClassName => this.#privateWebApi.registerTest(testClassName));
+        const testsToExecute = (testName || !targetIsBatch)
+            ? [testName] : await this.#getTestClassNames(module);
+        testsToExecute.forEach(testClassName => this.#registerTest(testClassName));
+
         for (const testClassName of testsToExecute) {
             let result = {};
-            this.#privateWebApi.setTestStatus(testClassName, TestStatus.Running);
+            this.#setTestStatus(testClassName, TestStatus.Running);
 
             try {
                 const LogToStdoutSpecialFilename = '-';
                 result = await module.exec({
-                    args: [testClassName, '-o', `${LogToStdoutSpecialFilename},${testOutputFormat}`],
+                    args: [...(targetIsBatch ? [testClassName] : []),
+                           '-o', `${LogToStdoutSpecialFilename},${testOutputFormat}`],
+                    onStdout: (output) => {
+                        this.#addTestOutput(testClassName, output);
+                    }
                 });
 
                 if (result.exitCode < 0)
                     throw new ProgramError(result.exitCode);
-                result.status = TestStatus.Completed;
+                result.status = result.exitCode > 0 ? TestStatus.Failed : TestStatus.Completed;
+                // Yield to other tasks on the main thread.
+                await new Promise(resolve => window.setTimeout(resolve, 0));
             } catch (e) {
                 result.status = e instanceof ProgramError ? TestStatus.Error : TestStatus.Crashed;
                 result.stdout = e instanceof AbortedError ? e.stdout : result.stdout;
             }
-            this.#privateWebApi.setTestResultData(
-                testClassName, result.status, result.exitCode, result.stdout);
-            }
+            this.#setTestResultData(testClassName, result.status, result.exitCode);
         }
+    }
 
-    async run(testName, testOutputFormat) {
+    async #getTestClassNames(module) {
+        return (await module.exec()).stdout.trim().split(' ');
+    }
 
-        await this.#doRun(testName, testOutputFormat);
-            this.#privateWebApi.setTestRunnerStatus(RunnerStatus.Completed);
+    #registerTest(testName) {
+        this.#results.set(testName, { status: TestStatus.Pending, output: [] });
+    }
 
+    #setTestStatus(testName, status) {
+        const testData = this.#results.get(testName);
+        if (testData.status === status)
+            return;
+        this.#results.get(testName).status = status;
+        this.#testStatusChangedEventPrivate.fireEvent(testName, status);
     }
 
-    async #getTestClassNames(module) {
-        return (await module.exec()).stdout.trim().split(' ');
+    #setTestResultData(testName, testStatus, exitCode) {
+        const testData = this.#results.get(testName);
+        const statusChanged = testStatus !== testData.status;
+        testData.status = testStatus;
+        testData.exitCode = exitCode;
+        if (statusChanged)
+            this.#testStatusChangedEventPrivate.fireEvent(testName, testStatus);
     }
-}
 
-(() => {
-    let privateWebApi;
-    window.qtTestRunner = new WebApi(privateApi => privateWebApi = privateApi);
-
-    const parsed = parseQuery(location.search);
-    const testName = parsed.get('qtestname');
-    try {
-        if (typeof testName !== 'undefined' && (typeof testName !== 'string' || testName === ''))
-            throw new Error('The testName parameter is incorrect');
-
-        const testOutputFormat = (() => {
-            const format = parsed.get('qtestoutputformat') ?? 'txt';
-            console.log(format);
-            if (-1 === ['txt', 'xml', 'lightxml', 'junitxml', 'tap'].indexOf(format))
-                throw new Error(`Bad file format: ${format}`);
-            return format;
-        })();
+    #setTestRunnerStatus(status, numberOfFailed) {
+        if (status === this.#status)
+            return;
+        this.#status = status;
+        this.#numberOfFailed = numberOfFailed;
+        this.#statusChangedEventPrivate.fireEvent(status);
+    }
 
-        const resourceLocator = new ResourceLocator('');
-        const testRunner = new BatchedTestRunner(
-            new ModuleLoader(new ResourceFetcher(resourceLocator), resourceLocator),
-            privateWebApi
-        );
+    #setTestRunnerError(details) {
+        this.#status = RunnerStatus.Error;
+        this.#errorDetails = details;
+        this.#statusChangedEventPrivate.fireEvent(this.#status);
+    }
 
-        testRunner.run(testName, testOutputFormat);
-    } catch (e) {
-        privateWebApi.setTestRunnerError(e.message);
+    #addTestOutput(testName, output) {
+        const testData = this.#results.get(testName);
+        testData.output.push(output);
+        this.#testOutputChangedEventPrivate.fireEvent(testName, testData.output);
     }
-})();
+}
diff --git a/util/wasm/batchedtestrunner/emrunadapter.js b/util/wasm/batchedtestrunner/emrunadapter.js
new file mode 100644
index 0000000000..cd793a38f2
--- /dev/null
+++ b/util/wasm/batchedtestrunner/emrunadapter.js
@@ -0,0 +1,119 @@
+// Copyright (C) 2022 The Qt Company Ltd.
+// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only
+
+import { RunnerStatus, TestStatus } from './batchedtestrunner.js';
+
+// Sends messages to the running emrun instance via POST requests.
+export class EmrunCommunication {
+    #indexOfMessage = 0;
+    #postOutputPromises = [];
+
+    #post(body) {
+        return fetch('stdio.html', {
+            method: 'POST',
+            body
+        });
+    }
+
+    // Returns a promise whose resolution signals that all outstanding traffic to the emrun instance
+    // has been completed.
+    waitUntilAllSent() {
+        return Promise.all(this.#postOutputPromises);
+    }
+
+    // Posts the exit status to the running emrun instance. Emrun will drop connection unless it is
+    // run with --serve_after_exit, therefore this method will throw most of the times.
+    postExit(status) {
+        return this.#post(`^exit^${status}`);
+    }
+
+    // Posts an indexed output chunk to the running emrun instance. Each consecutive call to this
+    // method increments the output index by 1.
+    postOutput(output) {
+        const newPromise = this.#post(`^out^${this.#indexOfMessage++}^${output}`);
+        this.#postOutputPromises.push(newPromise);
+        newPromise.finally(() => {
+            this.#postOutputPromises.splice(this.#postOutputPromises.indexOf(newPromise), 1);
+        });
+        return newPromise;
+    }
+}
+
+// Wraps a test module runner; forwards its output and resolution state to the running emrun
+// instance.
+export class EmrunAdapter {
+    #communication;
+    #batchedTestRunner;
+    #sentLines = 0;
+    #onExitSent;
+
+    constructor(communication, batchedTestRunner, onExitSent) {
+        this.#communication = communication;
+        this.#batchedTestRunner = batchedTestRunner;
+        this.#onExitSent = onExitSent;
+    }
+
+    // Starts listening to test module runner's state changes. When the test module runner finishes
+    // or reports output, sends suitable messages to the emrun instance.
+    run() {
+        this.#batchedTestRunner.onStatusChanged.addEventListener(
+            status => this.#onRunnerStatusChanged(status));
+        this.#batchedTestRunner.onTestStatusChanged.addEventListener(
+            (test, status) => this.#onTestStatusChanged(test, status));
+        this.#batchedTestRunner.onTestOutputChanged.addEventListener(
+            (test, output) => this.#onTestOutputChanged(test, output));
+
+        const currentTest = [...this.#batchedTestRunner.results.entries()].find(
+            entry => entry[1].status === TestStatus.Running)?.[0];
+
+        const output = this.#batchedTestRunner.results.get(currentTest)?.output;
+        if (output)
+            this.#onTestOutputChanged(testName, output);
+        this.#onRunnerStatusChanged(this.#batchedTestRunner.status);
+    }
+
+    #toExitCode(status) {
+        switch (status) {
+            case RunnerStatus.Error:
+                return -1;
+            case RunnerStatus.Passed:
+                return 0;
+            case RunnerStatus.Running:
+                throw new Error('No exit code when still running');
+            case RunnerStatus.TestCrashed:
+                return -2;
+            case RunnerStatus.TestsFailed:
+                return this.#batchedTestRunner.numberOfFailed;
+        }
+    }
+
+    async #onRunnerStatusChanged(status) {
+        if (RunnerStatus.Running === status)
+            return;
+
+        const exit = this.#toExitCode(status);
+        if (RunnerStatus.Error === status)
+            this.#communication.postOutput(this.#batchedTestRunner.errorDetails);
+
+        await this.#communication.waitUntilAllSent();
+        try {
+            await this.#communication.postExit(exit);
+        } catch {
+            // no-op: The remote end will drop connection on exit.
+        } finally {
+            this.#onExitSent?.();
+        }
+    }
+
+    async #onTestOutputChanged(_, output) {
+        const notSent = output.slice(this.#sentLines);
+        for (const out of notSent)
+            this.#communication.postOutput(out);
+        this.#sentLines = output.length;
+    }
+
+    async #onTestStatusChanged(_, status) {
+        if (status === TestStatus.Running)
+            this.#sentLines = 0;
+    }
+}
diff --git a/util/wasm/batchedtestrunner/qwasmjsruntime.js b/util/wasm/batchedtestrunner/qwasmjsruntime.js
index e167c87d4a..34a46c6e3f 100644
--- a/util/wasm/batchedtestrunner/qwasmjsruntime.js
+++ b/util/wasm/batchedtestrunner/qwasmjsruntime.js
@@ -145,8 +145,7 @@ export class CompiledModule {
                 params.arguments = parameters?.args;
                 let data = '';
                 params.print = (out) => {
-                    if (parameters?.printStdout === true)
-                        console.log(out);
+                    parameters?.onStdout?.(out);
                     data += `${out}\n`;
                 };
                 params.printErr = () => { };
diff --git a/util/wasm/batchedtestrunner/qwasmtestmain.js b/util/wasm/batchedtestrunner/qwasmtestmain.js
new file mode 100644
index 0000000000..e91ff6799d
--- /dev/null
+++ b/util/wasm/batchedtestrunner/qwasmtestmain.js
@@ -0,0 +1,59 @@
+// Copyright (C) 2022 The Qt Company Ltd.
+// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only
+
+import { BatchedTestRunner } from './batchedtestrunner.js'
+import { EmrunAdapter, EmrunCommunication } from './emrunadapter.js'
+import {
+    ModuleLoader,
+    ResourceFetcher,
+    ResourceLocator,
+} from './qwasmjsruntime.js';
+import { parseQuery } from './util.js';
+
+(() => {
+    const setPageTitle = (useEmrun, testName, isBatch) => {
+        document.title = 'Qt WASM test runner';
+        if (useEmrun || testName || isBatch) {
+            document.title += `(${[
+                    ...[useEmrun ? ['emrun'] : []],
+                    ...[testName ? ['test=' + testName] : []],
+                    ...[isBatch ? ['batch'] : []]
+                ].flat().join(", ")})`;
+        }
+    }
+
+    const parsed = parseQuery(location.search);
+    const testName = parsed.get('qtestname');
+    const isBatch = parsed.get('qbatchedtest') !== undefined;
+    const useEmrun = parsed.get('quseemrun') !== undefined;
+
+    if (testName === undefined) {
+        if (!isBatch)
+            throw new Error('The qtestname parameter is required if not running a batch');
+    } else if (testName === '') {
+        throw new Error(`The qtestname=${testName} parameter is incorrect`);
+    }
+
+    const testOutputFormat = (() => {
+        const format = parsed.get('qtestoutputformat') ?? 'txt';
+        if (-1 === ['txt', 'xml', 'lightxml', 'junitxml', 'tap'].indexOf(format))
+            throw new Error(`Bad file format: ${format}`);
+        return format;
+    })();
+
+    const resourceLocator = new ResourceLocator('');
+    const testRunner = new BatchedTestRunner(
+        new ModuleLoader(new ResourceFetcher(resourceLocator), resourceLocator),
+    );
+    window.qtTestRunner = testRunner;
+
+    if (useEmrun) {
+        const adapter = new EmrunAdapter(new EmrunCommunication(), testRunner, () => {
+            window.close();
+        });
+        adapter.run();
+    }
+    setPageTitle(useEmrun, testName, isBatch);
+
+    testRunner.run(isBatch, testName, testOutputFormat);
+})();
author	Mikolaj Boc <mikolaj.boc@qt.io>	2022-09-07 15:38:08 +0200
committer	Mikolaj Boc <mikolaj.boc@qt.io>	2022-09-12 22:28:10 +0200
commit	7dbbe0a22251085b00706c4eb997ac69148c4d70 (patch)
tree	49f8a6e86653bda4bbeaa1963c63ca91731ac546 /util
parent	84c494d0f27b37c59132151b924fdf5adbc542e5 (diff)