Create a driver for running batched tests on WASM

A driver application has been prepared in js for running batched tests. There is a convenient public API defined for reading the current test status & subscribing to changes thereof. The solution is modular - the module qwasmjsruntime can be used for any wasm instantiation, e.g. in the next iteration of qtloader. Change-Id: I00df88188c46a42f86d431285ca96d60d89b3f05 Pick-to: 6.4 Reviewed-by: David Skoland <david.skoland@qt.io>
author: Mikolaj Boc <mikolaj.boc@qt.io> 2022-07-08 12:40:49 +0200
committer: Mikolaj Boc <mikolaj.boc@qt.io> 2022-08-24 19:08:58 +0200
commit: ad1980cd4326acca891ed0fa4326ed1b22828324 (patch)
tree: 97c533043bee1a7e18c66cdc4c540d3bd4a84763 /util
parent: a021b5e09fed671195c0a8faf3b13d83a699dc9f (diff)
5 files changed, 478 insertions, 0 deletions
diff --git a/util/wasm/batchedtestrunner/README.md b/util/wasm/batchedtestrunner/README.md
new file mode 100644
index 0000000000..5098cd405d
--- /dev/null
+++ b/util/wasm/batchedtestrunner/README.md
@@ -0,0 +1,41 @@
+This package contains sources for a webpage whose scripts run batched WASM tests - a single
+executable with a number of linked test classes.
+The webpage operates on an assumption that the test program, when run without arguments,
+prints out a list of test classes inside its module. Then, when run with the first argument
+equal to the name of one of the test classes, the test program will execute all tests within
+that single class.
+
+The scripts in the page will load the wasm file called 'test_batch.wasm' with its corresponding
+js script 'test_batch.js'.
+
+Public interface for querying the test execution status is accessible via the global object
+'qtTestRunner':
+
+qtTestRunner.status - this contains the status of the test runner itself, of the enumeration type
+RunnerStatus.
+
+qtTestRunner.results - a map of test class name to test result. The result contains a test status
+(status, of the enumeration TestStatus), and in case of a terminal status, also the test's exit code
+(exitCode) and xml text output (textOutput), if available.
+
+qtTestRunner.onStatusChanged - an event for changes in state of the runner itself. The possible
+values are those of the enumeration RunnerStatus.
+
+qtTestRunner.onTestStatusChanged - an event for changes in state of a single tests class. The
+possible values are those of the enumeration TestStatus. When a terminal state is reached
+(Completed, Error, Crashed), the text results and exit code are filled in, if available, and
+will not change.
+
+Typical usage:
+Run all tests in a batch:
+    - load the webpage batchedtestrunner.html
+
+Run a single test in a batch:
+    - load the webpage batchedtestrunner.html?qtestname=tst_mytest
+
+Query for test execution state:
+    - qtTestRunner.onStatusChanged.addEventListener((runnerStatus) => (...)))
+    - qtTestRunner.onTestStatusChanged.addEventListener((testName, status) => (...))
+    - qtTestRunner.status === (...)
+    - qtTestRunner.results['tst_mytest'].status === (...)
+    - qtTestRunner.results['tst_mytest'].textOutput
diff --git a/util/wasm/batchedtestrunner/batchedtestrunner.html b/util/wasm/batchedtestrunner/batchedtestrunner.html
new file mode 100644
index 0000000000..123c24890b
--- /dev/null
+++ b/util/wasm/batchedtestrunner/batchedtestrunner.html
@@ -0,0 +1,14 @@
+<!--
+Copyright (C) 2022 The Qt Company Ltd.
+SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only
+-->
+
+<!doctype html>
+<html>
+<head>
+    <meta charset="utf-8">
+    <title>WASM batched test runner</title>
+    <script type="module" defer="defer" src="batchedtestrunner.js"></script>
+</head>
+<body></body>
+</html>
diff --git a/util/wasm/batchedtestrunner/batchedtestrunner.js b/util/wasm/batchedtestrunner/batchedtestrunner.js
new file mode 100644
index 0000000000..9a7597b7b8
--- /dev/null
+++ b/util/wasm/batchedtestrunner/batchedtestrunner.js
@@ -0,0 +1,162 @@
+// Copyright (C) 2022 The Qt Company Ltd.
+// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only
+
+import {
+    AbortedError,
+    ModuleLoader,
+    ResourceFetcher,
+    ResourceLocator,
+} from './qwasmjsruntime.js';
+
+import { parseQuery, EventSource } from './util.js';
+
+class ProgramError extends Error {
+    constructor(exitCode) {
+        super(`The program reported an exit code of ${exitCode}`)
+    }
+}
+
+class RunnerStatus {
+    static Running = 'Running';
+    static Completed = 'Completed';
+    static Error = 'Error';
+}
+
+class TestStatus {
+    static Pending = 'Pending';
+    static Running = 'Running';
+    static Completed = 'Completed';
+    static Error = 'Error';
+    static Crashed = 'Crashed';
+}
+
+// Represents the public API of the runner.
+class WebApi {
+    #results = new Map();
+    #status = RunnerStatus.Running;
+    #statusChangedEventPrivate;
+    #testStatusChangedEventPrivate;
+
+    onStatusChanged =
+        new EventSource((privateInterface) => this.#statusChangedEventPrivate = privateInterface);
+    onTestStatusChanged =
+        new EventSource((privateInterface) =>
+            this.#testStatusChangedEventPrivate = privateInterface);
+
+    // The callback receives the private interface of this object, meant not to be used by the
+    // end user on the web side.
+    constructor(receivePrivateInterface) {
+        receivePrivateInterface({
+            registerTest: testName => this.#registerTest(testName),
+            setTestStatus: (testName, status) => this.#setTestStatus(testName, status),
+            setTestResultData: (testName, testStatus, exitCode, textOutput) =>
+                this.#setTestResultData(testName, testStatus, exitCode, textOutput),
+            setTestRunnerStatus: status => this.#setTestRunnerStatus(status),
+        });
+    }
+
+    get results() { return this.#results; }
+    get status() { return this.#status; }
+
+    #registerTest(testName) { this.#results.set(testName, { status: TestStatus.Pending }); }
+
+    #setTestStatus(testName, status) {
+        const testData = this.#results.get(testName);
+        if (testData.status === status)
+            return;
+        this.#results.get(testName).status = status;
+        this.#testStatusChangedEventPrivate.fireEvent(testName, status);
+    }
+
+    #setTestResultData(testName, testStatus, exitCode, textOutput) {
+        const testData = this.#results.get(testName);
+        const statusChanged = testStatus !== testData.status;
+        testData.status = testStatus;
+        testData.exitCode = exitCode;
+        testData.textOutput = textOutput;
+        if (statusChanged)
+            this.#testStatusChangedEventPrivate.fireEvent(testName, testStatus);
+    }
+
+    #setTestRunnerStatus(status) {
+        if (status === this.#status)
+            return;
+        this.#status = status;
+        this.#statusChangedEventPrivate.fireEvent(status);
+    }
+}
+
+class BatchedTestRunner {
+    static #TestBatchModuleName = 'test_batch';
+
+    #loader;
+    #privateWebApi;
+
+    constructor(loader, privateWebApi) {
+        this.#loader = loader;
+        this.#privateWebApi = privateWebApi;
+    }
+
+    async #doRun(testName) {
+        const module = await this.#loader.loadEmscriptenModule(
+            BatchedTestRunner.#TestBatchModuleName,
+            () => { }
+        );
+
+        const testsToExecute = testName ? [testName] : await this.#getTestClassNames(module);
+        testsToExecute.forEach(testClassName => this.#privateWebApi.registerTest(testClassName));
+        for (const testClassName of testsToExecute) {
+            let result = {};
+            this.#privateWebApi.setTestStatus(testClassName, TestStatus.Running);
+
+            try {
+                const LogToStdoutSpecialFilename = '-';
+                result = await module.exec({
+                    args: [testClassName, '-o', `${LogToStdoutSpecialFilename},xml`],
+                });
+
+                if (result.exitCode < 0)
+                    throw new ProgramError(result.exitCode);
+                result.status = TestStatus.Completed;
+            } catch (e) {
+                result.status = e instanceof ProgramError ? TestStatus.Error : TestStatus.Crashed;
+                result.stdout = e instanceof AbortedError ? e.stdout : result.stdout;
+            }
+            this.#privateWebApi.setTestResultData(
+                testClassName, result.status, result.exitCode, result.stdout);
+            }
+        }
+
+    async run(testName) {
+        try {
+            await this.#doRun(testName);
+            this.#privateWebApi.setTestRunnerStatus(RunnerStatus.Completed);
+        } catch (e) {
+            this.#privateWebApi.setTestRunnerStatus(RunnerStatus.Error);
+        }
+    }
+
+    async #getTestClassNames(module) {
+        return (await module.exec()).stdout.trim().split(' ');
+    }
+}
+
+(() => {
+    let privateWebApi;
+    window.qtTestRunner = new WebApi(privateApi => privateWebApi = privateApi);
+
+    const parsed = parseQuery(location.search);
+    const testName = parsed['qtestname'];
+    if (typeof testName !== 'undefined' && (typeof testName !== 'string' || testName === '')) {
+        console.error('The testName parameter is incorrect');
+        return;
+    }
+
+    const resourceLocator = new ResourceLocator('');
+    const testRunner = new BatchedTestRunner(
+        new ModuleLoader(new ResourceFetcher(resourceLocator), resourceLocator),
+        privateWebApi
+    );
+
+    testRunner.run(testName);
+})();
diff --git a/util/wasm/batchedtestrunner/qwasmjsruntime.js b/util/wasm/batchedtestrunner/qwasmjsruntime.js
new file mode 100644
index 0000000000..e167c87d4a
--- /dev/null
+++ b/util/wasm/batchedtestrunner/qwasmjsruntime.js
@@ -0,0 +1,230 @@
+// Copyright (C) 2022 The Qt Company Ltd.
+// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only
+
+// Exposes platform capabilities as static properties
+
+export class AbortedError extends Error {
+    constructor(stdout) {
+        super(`The program has been aborted`)
+
+        this.stdout = stdout;
+    }
+}
+export class Platform {
+    static #webAssemblySupported = typeof WebAssembly !== 'undefined';
+
+    static #canCompileStreaming = WebAssembly.compileStreaming !== 'undefined';
+
+    static #webGLSupported = (() => {
+        // We expect that WebGL is supported if WebAssembly is; however
+        // the GPU may be blacklisted.
+        try {
+            const canvas = document.createElement('canvas');
+            return !!(
+                window.WebGLRenderingContext &&
+                (canvas.getContext('webgl') || canvas.getContext('experimental-webgl'))
+            );
+        } catch (e) {
+            return false;
+        }
+    })();
+
+    static #canLoadQt = Platform.#webAssemblySupported && Platform.#webGLSupported;
+
+    static get webAssemblySupported() {
+        return this.#webAssemblySupported;
+    }
+    static get canCompileStreaming() {
+        return this.#canCompileStreaming;
+    }
+    static get webGLSupported() {
+        return this.#webGLSupported;
+    }
+    static get canLoadQt() {
+        return this.#canLoadQt;
+    }
+}
+
+// Locates a resource, based on its relative path
+export class ResourceLocator {
+    #rootPath;
+
+    constructor(rootPath) {
+        this.#rootPath = rootPath;
+        if (rootPath.length > 0 && !rootPath.endsWith('/')) rootPath += '/';
+    }
+
+    locate(relativePath) {
+        return this.#rootPath + relativePath;
+    }
+}
+
+// Allows fetching of resources, such as text resources or wasm modules.
+export class ResourceFetcher {
+    #locator;
+
+    constructor(locator) {
+        this.#locator = locator;
+    }
+
+    async fetchText(filePath) {
+        return (await this.#fetchRawResource(filePath)).text();
+    }
+
+    async fetchCompileWasm(filePath, onFetched) {
+        const fetchResponse = await this.#fetchRawResource(filePath);
+        onFetched?.();
+
+        if (Platform.canCompileStreaming) {
+            try {
+                return await WebAssembly.compileStreaming(fetchResponse);
+            } catch {
+                // NOOP - fallback to sequential fetching below
+            }
+        }
+        return WebAssembly.compile(await fetchResponse.arrayBuffer());
+    }
+
+    async #fetchRawResource(filePath) {
+        const response = await fetch(this.#locator.locate(filePath));
+        if (!response.ok)
+            throw new Error(
+                `${response.status} ${response.statusText} ${response.url}`
+            );
+        return response;
+    }
+}
+
+// Represents a WASM module, wrapping the instantiation and execution thereof.
+export class CompiledModule {
+    #createQtAppInstanceFn;
+    #js;
+    #wasm;
+    #resourceLocator;
+
+    constructor(createQtAppInstanceFn, js, wasm, resourceLocator) {
+        this.#createQtAppInstanceFn = createQtAppInstanceFn;
+        this.#js = js;
+        this.#wasm = wasm;
+        this.#resourceLocator = resourceLocator;
+    }
+
+    static make(js, wasm, resourceLocator
+    ) {
+        const exports = {};
+        eval(js);
+        if (!exports.createQtAppInstance) {
+            throw new Error(
+                'createQtAppInstance has not been exported by the main script'
+            );
+        }
+
+        return new CompiledModule(
+            exports.createQtAppInstance, js, wasm, resourceLocator
+        );
+    }
+
+    async exec(parameters) {
+        return await new Promise(async (resolve, reject) => {
+            let instance = undefined;
+            let result = undefined;
+            const continuation = () => {
+                if (!(instance && result))
+                    return;
+                resolve({
+                    stdout: result.stdout,
+                    exitCode: result.exitCode,
+                    instance,
+                });
+            };
+
+            instance = await this.#createQtAppInstanceFn((() => {
+                const params = this.#makeDefaultExecParams({
+                    onInstantiationError: (error) => { reject(error); },
+                });
+                params.arguments = parameters?.args;
+                let data = '';
+                params.print = (out) => {
+                    if (parameters?.printStdout === true)
+                        console.log(out);
+                    data += `${out}\n`;
+                };
+                params.printErr = () => { };
+                params.onAbort = () => reject(new AbortedError(data));
+                params.quit = (code, exception) => {
+                    if (exception && exception.name !== 'ExitStatus')
+                        reject(exception);
+                    result = { stdout: data, exitCode: code };
+                    continuation();
+                };
+                return params;
+            })());
+            continuation();
+        });
+    }
+
+    #makeDefaultExecParams(params) {
+        const instanceParams = {};
+        instanceParams.instantiateWasm = async (imports, onDone) => {
+            try {
+                onDone(await WebAssembly.instantiate(this.#wasm, imports));
+            } catch (e) {
+                params?.onInstantiationError?.(e);
+            }
+        };
+        instanceParams.locateFile = (filename) =>
+            this.#resourceLocator.locate(filename);
+        instanceParams.monitorRunDependencies = (name) => { };
+        instanceParams.print = (text) => true && console.log(text);
+        instanceParams.printErr = (text) => true && console.warn(text);
+        instanceParams.preRun = [
+            (instance) => {
+                const env = {};
+                instance.ENV = env;
+            },
+        ];
+
+        instanceParams.mainScriptUrlOrBlob = new Blob([this.#js], {
+            type: 'text/javascript',
+        });
+        return instanceParams;
+    }
+}
+
+// Streamlines loading of WASM modules.
+export class ModuleLoader {
+    #fetcher;
+    #resourceLocator;
+
+    constructor(
+        fetcher,
+        resourceLocator
+    ) {
+        this.#fetcher = fetcher;
+        this.#resourceLocator = resourceLocator;
+    }
+
+    // Loads an emscripten module named |moduleName| from the main resource path. Provides
+    // progress of 'downloading' and 'compiling' to the caller using the |onProgress| callback.
+    async loadEmscriptenModule(
+        moduleName, onProgress
+    ) {
+        if (!Platform.webAssemblySupported)
+            throw new Error('Web assembly not supported');
+        if (!Platform.webGLSupported)
+            throw new Error('WebGL is not supported');
+
+        onProgress('downloading');
+
+        const jsLoadPromise = this.#fetcher.fetchText(`${moduleName}.js`);
+        const wasmLoadPromise = this.#fetcher.fetchCompileWasm(
+            `${moduleName}.wasm`,
+            () => {
+                onProgress('compiling');
+            }
+        );
+
+        const [js, wasm] = await Promise.all([jsLoadPromise, wasmLoadPromise]);
+        return CompiledModule.make(js, wasm, this.#resourceLocator);
+    }
+}
diff --git a/util/wasm/batchedtestrunner/util.js b/util/wasm/batchedtestrunner/util.js
new file mode 100644
index 0000000000..07a0e73e1a
--- /dev/null
+++ b/util/wasm/batchedtestrunner/util.js
@@ -0,0 +1,31 @@
+// Copyright (C) 2022 The Qt Company Ltd.
+// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only
+
+export function parseQuery() {
+    const trimmed = window.location.search.substring(1);
+    return new Map(
+        trimmed.length === 0 ?
+            [] :
+            trimmed.split('&').map(paramNameAndValue => {
+                const [name, value] = paramNameAndValue.split('=');
+                return [decodeURIComponent(name), value ? decodeURIComponent(value) : ''];
+            }));
+}
+
+export class EventSource {
+    #listeners = [];
+
+    constructor(receivePrivateInterface) {
+        receivePrivateInterface({
+            fireEvent: (arg0, arg1) => this.#fireEvent(arg0, arg1)
+        });
+    }
+
+    addEventListener(listener) {
+        this.#listeners.push(listener);
+    }
+
+    #fireEvent(arg0, arg1) {
+        this.#listeners.forEach(listener => listener(arg0, arg1));
+    }
+}
author	Mikolaj Boc <mikolaj.boc@qt.io>	2022-07-08 12:40:49 +0200
committer	Mikolaj Boc <mikolaj.boc@qt.io>	2022-08-24 19:08:58 +0200
commit	ad1980cd4326acca891ed0fa4326ed1b22828324 (patch)
tree	97c533043bee1a7e18c66cdc4c540d3bd4a84763 /util
parent	a021b5e09fed671195c0a8faf3b13d83a699dc9f (diff)