diff options
Diffstat (limited to 'util/wasm')
| -rw-r--r-- | util/wasm/batchedtestrunner/batchedtestrunner.html | 2 | ||||
| -rw-r--r-- | util/wasm/batchedtestrunner/batchedtestrunner.js | 9 | ||||
| -rw-r--r-- | util/wasm/batchedtestrunner/emrunadapter.js | 44 | ||||
| -rw-r--r-- | util/wasm/batchedtestrunner/qtestoutputreporter.css | 2 | ||||
| -rw-r--r-- | util/wasm/batchedtestrunner/qtestoutputreporter.js | 2 | ||||
| -rw-r--r-- | util/wasm/batchedtestrunner/qwasmjsruntime.js | 50 | ||||
| -rw-r--r-- | util/wasm/batchedtestrunner/qwasmtestmain.js | 31 | ||||
| -rw-r--r-- | util/wasm/batchedtestrunner/util.js | 2 | ||||
| -rwxr-xr-x | util/wasm/preload/preload_qml_imports.py | 101 | ||||
| -rwxr-xr-x | util/wasm/preload/preload_qt_plugins.py | 54 | ||||
| -rw-r--r-- | util/wasm/preload/wasm_binary_tools.py | 78 |
11 files changed, 323 insertions, 52 deletions
diff --git a/util/wasm/batchedtestrunner/batchedtestrunner.html b/util/wasm/batchedtestrunner/batchedtestrunner.html index 0b85e48691..147cf34376 100644 --- a/util/wasm/batchedtestrunner/batchedtestrunner.html +++ b/util/wasm/batchedtestrunner/batchedtestrunner.html @@ -1,6 +1,6 @@ <!-- Copyright (C) 2022 The Qt Company Ltd. -SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only +SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0 --> <!doctype html> diff --git a/util/wasm/batchedtestrunner/batchedtestrunner.js b/util/wasm/batchedtestrunner/batchedtestrunner.js index e26e561b5a..453865a935 100644 --- a/util/wasm/batchedtestrunner/batchedtestrunner.js +++ b/util/wasm/batchedtestrunner/batchedtestrunner.js @@ -1,5 +1,5 @@ // Copyright (C) 2022 The Qt Company Ltd. -// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0 import { AbortedError, @@ -68,9 +68,9 @@ export class BatchedTestRunner { get errorDetails() { return this.#errorDetails; } - async run(targetIsBatch, testName, testOutputFormat) { + async run(targetIsBatch, testName, functions, testOutputFormat) { try { - await this.#doRun(targetIsBatch, testName, testOutputFormat); + await this.#doRun(targetIsBatch, testName, functions, testOutputFormat); } catch (e) { this.#setTestRunnerError(e.message); return; @@ -93,7 +93,7 @@ export class BatchedTestRunner { this.#setTestRunnerStatus(status.code, status.numberOfFailed); } - async #doRun(targetIsBatch, testName, testOutputFormat) { + async #doRun(targetIsBatch, testName, functions, testOutputFormat) { const module = await this.#loader.loadEmscriptenModule( targetIsBatch ? BatchedTestRunner.#TestBatchModuleName : testName, () => { } @@ -111,6 +111,7 @@ export class BatchedTestRunner { const LogToStdoutSpecialFilename = '-'; result = await module.exec({ args: [...(targetIsBatch ? [testClassName] : []), + ...(functions ?? []), '-o', `${LogToStdoutSpecialFilename},${testOutputFormat}`], onStdout: (output) => { this.#addTestOutput(testClassName, output); diff --git a/util/wasm/batchedtestrunner/emrunadapter.js b/util/wasm/batchedtestrunner/emrunadapter.js index cd793a38f2..5b4284e18f 100644 --- a/util/wasm/batchedtestrunner/emrunadapter.js +++ b/util/wasm/batchedtestrunner/emrunadapter.js @@ -1,12 +1,17 @@ // Copyright (C) 2022 The Qt Company Ltd. -// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0 import { RunnerStatus, TestStatus } from './batchedtestrunner.js'; // Sends messages to the running emrun instance via POST requests. export class EmrunCommunication { + static #BATCHING_DELAY = 300; + #indexOfMessage = 0; - #postOutputPromises = []; + #postOutputPromise; + // Accumulate output in a batch that gets sent with a delay so that the emrun http server + // does not get pounded with requests. + #nextOutputBatch = null; #post(body) { return fetch('stdio.html', { @@ -15,10 +20,11 @@ export class EmrunCommunication { }); } - // Returns a promise whose resolution signals that all outstanding traffic to the emrun instance - // has been completed. - waitUntilAllSent() { - return Promise.all(this.#postOutputPromises); + // Waits for the output sending to finish, if any output transfer is still in progress. + async waitUntilAllSent() + { + if (this.#postOutputPromise) + await this.#postOutputPromise; } // Posts the exit status to the running emrun instance. Emrun will drop connection unless it is @@ -29,13 +35,25 @@ export class EmrunCommunication { // Posts an indexed output chunk to the running emrun instance. Each consecutive call to this // method increments the output index by 1. - postOutput(output) { - const newPromise = this.#post(`^out^${this.#indexOfMessage++}^${output}`); - this.#postOutputPromises.push(newPromise); - newPromise.finally(() => { - this.#postOutputPromises.splice(this.#postOutputPromises.indexOf(newPromise), 1); - }); - return newPromise; + postOutput(output) + { + if (this.#nextOutputBatch) { + this.#nextOutputBatch += output; + } else { + this.#nextOutputBatch = output; + this.#postOutputPromise = new Promise(resolve => + { + window.setTimeout(() => + { + const toSend = this.#nextOutputBatch; + this.#nextOutputBatch = null; + this.#post(`^out^${this.#indexOfMessage++}^${toSend}$`) + .finally(resolve); + }, EmrunCommunication.#BATCHING_DELAY); + }); + } + + return this.#postOutputPromise; } } diff --git a/util/wasm/batchedtestrunner/qtestoutputreporter.css b/util/wasm/batchedtestrunner/qtestoutputreporter.css index 3cf312b11a..aefb867b81 100644 --- a/util/wasm/batchedtestrunner/qtestoutputreporter.css +++ b/util/wasm/batchedtestrunner/qtestoutputreporter.css @@ -1,6 +1,6 @@ /* Copyright (C) 2022 The Qt Company Ltd. - SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only + SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0 */ :root { diff --git a/util/wasm/batchedtestrunner/qtestoutputreporter.js b/util/wasm/batchedtestrunner/qtestoutputreporter.js index ad8a373540..7af288b8f0 100644 --- a/util/wasm/batchedtestrunner/qtestoutputreporter.js +++ b/util/wasm/batchedtestrunner/qtestoutputreporter.js @@ -1,5 +1,5 @@ // Copyright (C) 2022 The Qt Company Ltd. -// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0 import { RunnerStatus, TestStatus } from './batchedtestrunner.js' diff --git a/util/wasm/batchedtestrunner/qwasmjsruntime.js b/util/wasm/batchedtestrunner/qwasmjsruntime.js index c560f38ea4..3f2d421181 100644 --- a/util/wasm/batchedtestrunner/qwasmjsruntime.js +++ b/util/wasm/batchedtestrunner/qwasmjsruntime.js @@ -1,5 +1,5 @@ // Copyright (C) 2022 The Qt Company Ltd. -// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0 // Exposes platform capabilities as static properties @@ -109,18 +109,19 @@ export class CompiledModule { this.#resourceLocator = resourceLocator; } - static make(js, wasm, resourceLocator - ) { + static make(js, wasm, entryFunctionName, resourceLocator) + { const exports = {}; + const module = {}; eval(js); - if (!exports.createQtAppInstance) { + if (!module.exports) { throw new Error( - 'createQtAppInstance has not been exported by the main script' + '${entryFunctionName} has not been exported by the main script' ); } return new CompiledModule( - exports.createQtAppInstance, js, wasm, resourceLocator + module.exports, js, wasm, resourceLocator ); } @@ -128,16 +129,9 @@ export class CompiledModule { return await new Promise(async (resolve, reject) => { let instance = undefined; let result = undefined; - const continuation = () => { - if (!(instance && result)) - return; - resolve({ - stdout: result.stdout, - exitCode: result.exitCode, - instance, - }); - }; + let testFinished = false; + const testFinishedEvent = new CustomEvent('testFinished'); instance = await this.#createQtAppInstanceFn((() => { const params = this.#makeDefaultExecParams({ onInstantiationError: (error) => { reject(error); }, @@ -153,12 +147,26 @@ export class CompiledModule { params.quit = (code, exception) => { if (exception && exception.name !== 'ExitStatus') reject(exception); + }; + params.notifyTestFinished = (code) => { result = { stdout: data, exitCode: code }; - continuation(); + testFinished = true; + window.dispatchEvent(testFinishedEvent); }; return params; })()); - continuation(); + if (!testFinished) { + await new Promise((resolve) => { + window.addEventListener('testFinished', () => { + resolve(); + }); + }); + } + resolve({ + stdout: result.stdout, + exitCode: result.exitCode, + instance, + }); }); } @@ -176,12 +184,6 @@ export class CompiledModule { instanceParams.monitorRunDependencies = (name) => { }; instanceParams.print = (text) => true && console.log(text); instanceParams.printErr = (text) => true && console.warn(text); - instanceParams.preRun = [ - (instance) => { - const env = {}; - instance.ENV = env; - }, - ]; instanceParams.mainScriptUrlOrBlob = new Blob([this.#js], { type: 'text/javascript', @@ -224,6 +226,6 @@ export class ModuleLoader { ); const [js, wasm] = await Promise.all([jsLoadPromise, wasmLoadPromise]); - return CompiledModule.make(js, wasm, this.#resourceLocator); + return CompiledModule.make(js, wasm, `${moduleName}_entry`, this.#resourceLocator); } } diff --git a/util/wasm/batchedtestrunner/qwasmtestmain.js b/util/wasm/batchedtestrunner/qwasmtestmain.js index 644455907f..a92a3a4b30 100644 --- a/util/wasm/batchedtestrunner/qwasmtestmain.js +++ b/util/wasm/batchedtestrunner/qwasmtestmain.js @@ -1,5 +1,5 @@ // Copyright (C) 2022 The Qt Company Ltd. -// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0 import { BatchedTestRunner } from './batchedtestrunner.js' import { EmrunAdapter, EmrunCommunication } from './emrunadapter.js' @@ -11,6 +11,22 @@ import { import { parseQuery } from './util.js'; import { VisualOutputProducer, UI, ScannerFactory } from './qtestoutputreporter.js' +const StandardArg = { + qVisualOutput: 'qvisualoutput', + qTestName: 'qtestname', + qBatchedTest: 'qbatchedtest', + qUseEmrun: 'quseemrun', + qTestOutputFormat: 'qtestoutputformat', +} + +const allArgs = new Set(Object.getOwnPropertyNames(StandardArg).map(arg => StandardArg[arg])); +Object.defineProperty(StandardArg, 'isKnown', { + get() + { + return name => allArgs.has(name); + }, +}); + (() => { const setPageTitle = (useEmrun, testName, isBatch) => { document.title = 'Qt WASM test runner'; @@ -24,10 +40,11 @@ import { VisualOutputProducer, UI, ScannerFactory } from './qtestoutputreporter. } const parsed = parseQuery(location.search); - const outputInPage = parsed.get('qvisualoutput') !== undefined; - const testName = parsed.get('qtestname'); - const isBatch = parsed.get('qbatchedtest') !== undefined; - const useEmrun = parsed.get('quseemrun') !== undefined; + const outputInPage = parsed.has(StandardArg.qVisualOutput); + const testName = parsed.get(StandardArg.qTestName); + const isBatch = parsed.has(StandardArg.qBatchedTest); + const useEmrun = parsed.has(StandardArg.qUseEmrun); + const functions = [...parsed.keys()].filter(arg => !StandardArg.isKnown(arg)); if (testName === undefined) { if (!isBatch) @@ -37,7 +54,7 @@ import { VisualOutputProducer, UI, ScannerFactory } from './qtestoutputreporter. } const testOutputFormat = (() => { - const format = parsed.get('qtestoutputformat') ?? 'txt'; + const format = parsed.get(StandardArg.qTestOutputFormat) ?? 'txt'; if (-1 === ['txt', 'xml', 'lightxml', 'junitxml', 'tap'].indexOf(format)) throw new Error(`Bad file format: ${format}`); return format; @@ -65,5 +82,5 @@ import { VisualOutputProducer, UI, ScannerFactory } from './qtestoutputreporter. } setPageTitle(useEmrun, testName, isBatch); - testRunner.run(isBatch, testName, testOutputFormat); + testRunner.run(isBatch, testName, functions, testOutputFormat); })(); diff --git a/util/wasm/batchedtestrunner/util.js b/util/wasm/batchedtestrunner/util.js index 07a0e73e1a..a297baf6b2 100644 --- a/util/wasm/batchedtestrunner/util.js +++ b/util/wasm/batchedtestrunner/util.js @@ -1,5 +1,5 @@ // Copyright (C) 2022 The Qt Company Ltd. -// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0 export function parseQuery() { const trimmed = window.location.search.substring(1); diff --git a/util/wasm/preload/preload_qml_imports.py b/util/wasm/preload/preload_qml_imports.py new file mode 100755 index 0000000000..9af4fa2a28 --- /dev/null +++ b/util/wasm/preload/preload_qml_imports.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +# Copyright (C) 2023 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0 + +import os +import sys +import subprocess +import json + +# Paths to shared libraries and qml imports on the Qt installation on the web server. +# "$QTDIR" is replaced by qtloader.js at load time (defaults to "qt"), and makes +# possible to relocate the application build relative to the Qt build on the web server. +qt_lib_path = "$QTDIR/lib" +qt_qml_path = "$QTDIR/qml" + +# Path to QML imports on the in-memory file system provided by Emscripten. This script emits +# preload commands which copies QML imports to this directory. In addition, preload_qt_plugins.py +# creates (and preloads) a qt.conf file which makes Qt load QML plugins from this location. +qt_deploy_qml_path = "/qt/qml" + + +def eprint(*args, **kwargs): + print(*args, file=sys.stderr, **kwargs) + + +def preload_file(source, destination): + preload_files.append({"source": source, "destination": destination}) + + +def extract_preload_files_from_imports(imports): + libraries = [] + for qml_import in imports: + try: + relative_path = qml_import["relativePath"] + plugin = qml_import["plugin"] + + # plugin .so + plugin_filename = "lib" + plugin + ".so" + so_plugin_source_path = os.path.join( + qt_qml_path, relative_path, plugin_filename + ) + so_plugin_destination_path = os.path.join( + qt_deploy_qml_path, relative_path, plugin_filename + ) + + preload_file(so_plugin_source_path, so_plugin_destination_path) + so_plugin_qt_install_path = os.path.join( + qt_wasm_path, "qml", relative_path, plugin_filename + ) + + # qmldir file + qmldir_source_path = os.path.join(qt_qml_path, relative_path, "qmldir") + qmldir_destination_path = os.path.join( + qt_deploy_qml_path, relative_path, "qmldir" + ) + preload_file(qmldir_source_path, qmldir_destination_path) + except Exception as e: + eprint(e) + continue + return libraries + + +if __name__ == "__main__": + if len(sys.argv) != 4: + print("Usage: python preload_qml_imports.py <qml-source-path> <qt-host-path> <qt-wasm-path>") + sys.exit(1) + + qml_source_path = sys.argv[1] + qt_host_path = sys.argv[2] + qt_wasm_path = sys.argv[3] + + qml_import_path = os.path.join(qt_wasm_path, "qml") + qmlimportsscanner_path = os.path.join(qt_host_path, "libexec/qmlimportscanner") + + eprint("runing qmlimportsscanner") + command = [qmlimportsscanner_path, "-rootPath", qml_source_path, "-importPath", qml_import_path] + result = subprocess.run(command, stdout=subprocess.PIPE) + imports = json.loads(result.stdout) + + preload_files = [] + libraries = extract_preload_files_from_imports(imports) + + # Deploy plugin dependencies, that is, shared libraries used by the plugins. + # Skip some of the obvious libraries which will be + skip_libraries = [ + "libQt6Core.so", + "libQt6Gui.so", + "libQt6Quick.so", + "libQt6Qml.so" "libQt6Network.so", + "libQt6OpenGL.so", + ] + + libraries = set(libraries) - set(skip_libraries) + for library in libraries: + source = os.path.join(qt_lib_path, library) + # Emscripten looks for shared libraries on "/", shared libraries + # most be deployed there instead of at /qt/lib + destination = os.path.join("/", library) + preload_file(source, destination) + + print(json.dumps(preload_files, indent=2)) diff --git a/util/wasm/preload/preload_qt_plugins.py b/util/wasm/preload/preload_qt_plugins.py new file mode 100755 index 0000000000..362d129732 --- /dev/null +++ b/util/wasm/preload/preload_qt_plugins.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 +# Copyright (C) 2023 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0 + +import os +import sys +import json + +# Path to plugins on the Qt installation on the web server. "$QTPATH" is replaced by qtloader.js +# at load time (defaults to "qt"), which makes it possible to relocate the application build relative +# to the Qt build on the web server. +qt_plugins_path = "$QTDIR/plugins" + +# Path to plugins on the in-memory file system provided by Emscripten. This script emits +# preload commands which copies plugins to this directory. +qt_deploy_plugins_path = "/qt/plugins" + + +def find_so_files(directory): + so_files = [] + for root, dirs, files in os.walk(directory): + for file in files: + if file.endswith(".so"): + relative_path = os.path.relpath(os.path.join(root, file), directory) + so_files.append(relative_path) + return so_files + + +if __name__ == "__main__": + if len(sys.argv) != 2: + print("Usage: python make_qt_symlinks.py <qt-wasm-path>") + sys.exit(1) + + qt_wasm_path = sys.argv[1] + + # preload all plugins + plugins = find_so_files(os.path.join(qt_wasm_path, "plugins")) + preload = [ + { + "source": os.path.join(qt_plugins_path, plugin), + "destination": os.path.join(qt_deploy_plugins_path, plugin), + } + for plugin in plugins + ] + + # Create and preload qt.conf which will tell Qt to look for plugins + # and QML imports in /qt/plugins and /qt/qml. The qt.conf file is + # written to the current directory. + qtconf = "[Paths]\nPrefix = /qt\n" + with open("qt.conf", "w") as f: + f.write(qtconf) + preload.append({"source": "qt.conf", "destination": "/qt.conf"}) + + print(json.dumps(preload, indent=2)) diff --git a/util/wasm/preload/wasm_binary_tools.py b/util/wasm/preload/wasm_binary_tools.py new file mode 100644 index 0000000000..9a2150b964 --- /dev/null +++ b/util/wasm/preload/wasm_binary_tools.py @@ -0,0 +1,78 @@ + +#!/usr/bin/env python3 +# Copyright (C) 2024 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0 + +import sys +import struct + + +class WasmBinary: + """For reference of binary format see Emscripten source code, especially library_dylink.js.""" + + def __init__(self, filepath): + self._offset = 0 + self._end = 0 + self._dependencies = [] + with open(filepath, 'rb') as file: + self._binary = file.read() + self._check_preamble() + self._parse_subsections() + + def get_dependencies(self): + return self._dependencies + + def _get_leb(self): + ret = 0 + mul = 1 + while True: + byte = self._binary[self._offset] + self._offset += 1 + ret += (byte & 0x7f) * mul + mul *= 0x80 + if not (byte & 0x80): + break + return ret + + def _get_string(self): + length = self._get_leb() + self._offset += length + return self._binary[self._offset - length:self._offset].decode('utf-8') + + def _check_preamble(self): + preamble = memoryview(self._binary)[:24] + int32View = struct.unpack('<6I', preamble) + assert int32View[0] == 0x6d736100, "magic number not found" + assert self._binary[8] == 0, "dynlink section needs to be first" + self._offset = 9 + section_size = self._get_leb() + self._end = self._offset + section_size + name = self._get_string() + assert name == "dylink.0", "section dylink.0 not found" + + def _parse_subsections(self): + WASM_DYLINK_NEEDED = 0x2 + + while self._offset < self._end: + subsection_type = self._binary[self._offset] + self._offset += 1 + subsection_size = self._get_leb() + + if subsection_type == WASM_DYLINK_NEEDED: + needed_dynlibs_count = self._get_leb() + for _ in range(needed_dynlibs_count): + self._dependencies.append(self._get_string()) + else: + self._offset += subsection_size # we don't care about other sections for now + + +if __name__ == "__main__": + if len(sys.argv) != 2: + print("Usage: python wasm_binary_tools.py <shared_object>") + sys.exit(1) + + file_path = sys.argv[1] + binary = WasmBinary(file_path) + dependencies = binary.get_dependencies() + for d in dependencies: + print(d) |