tst_ecmascript: Run tests on separate processes instead of threads

Previously, tests were run in parallel on separate threads. This was
faster than running them on only one but was still significantly slower
than it could be, on Windows. This is due to them sharing the same heap
and the fact that each memory allocation and free would temporarilly
lock the heap for all other threads making the tests run much slower
than on other platforms.

This patch changes the way the test is run so that each js test file is
run on a separate process. This ensures that the heap is no longer being
shared by all test runners and reduces overhead significantly.

The test runner processes listen for test data in JSON format over their
standard input, run the test, return the results over their standard
output and then wait for the next test data.

tst_ecmascripttests on 13900k with 32 threads
             Debug MSVC Windows    Debug GCC Linux
threads:          569s               105s
processes:         89s (~ -84%)       52s (~ -50%)

On platforms where QT_CONFIG(process) returns false, the tests fallback
to running on threads as before.

Change-Id: Id51fc9d6e0d5ef0ae5c88f96b0119aa99e57f0fe
Reviewed-by: Qt CI Bot <qt_ci_bot@qt-project.org>
Reviewed-by: Fabian Kosmale <fabian.kosmale@qt.io>

6 files changed, 450 insertions, 319 deletions

diff --git a/tests/auto/qml/ecmascripttests/CMakeLists.txt b/tests/auto/qml/ecmascripttests/CMakeLists.txt
index e03f5ffa82..1ee70cb101 100644
--- a/tests/auto/qml/ecmascripttests/CMakeLists.txt
+++ b/tests/auto/qml/ecmascripttests/CMakeLists.txt
@@ -20,7 +20,7 @@ list(FILTER test_data EXCLUDE REGEX ".git")
 
 qt_internal_add_test(tst_ecmascripttests
     SOURCES
-        qjstest/test262runner.cpp qjstest/test262runner.h
+        test262runner.cpp test262runner.h
         tst_ecmascripttests.cpp
     LIBRARIES
         Qt::QmlPrivate
@@ -47,7 +47,3 @@ else()
             QT_QMLTEST_DATADIR="${CMAKE_CURRENT_SOURCE_DIR}/test262"
     )
 endif()
-
-if(NOT CMAKE_CROSSCOMPILING)
-    add_subdirectory(qjstest)
-endif()
diff --git a/tests/auto/qml/ecmascripttests/qjstest/CMakeLists.txt b/tests/auto/qml/ecmascripttests/qjstest/CMakeLists.txt
deleted file mode 100644
index 86ca5f97a3..0000000000
--- a/tests/auto/qml/ecmascripttests/qjstest/CMakeLists.txt
+++ /dev/null
@@ -1,28 +0,0 @@
-# Copyright (C) 2022 The Qt Company Ltd.
-# SPDX-License-Identifier: BSD-3-Clause
-
-# Generated from qjstest.pro.
-
-#####################################################################
-## qjstest Tool:
-#####################################################################
-
-qt_get_tool_target_name(target_name qjstest)
-qt_internal_add_tool(${target_name}
-    TARGET_DESCRIPTION "Javascript test runner"
-    SOURCES
-        main.cpp
-        test262runner.cpp test262runner.h
-    DEFINES
-        QT_DEPRECATED_WARNINGS
-    INCLUDE_DIRECTORIES
-        .
-    LIBRARIES
-        Qt::Gui
-        Qt::QmlPrivate
-)
-qt_internal_return_unless_building_tools()
-
-#### Keys ignored in scope 1:.:.:qjstest.pro:<TRUE>:
-# QMAKE_TARGET_DESCRIPTION = "Javascript" "test" "runner"
-# TEMPLATE = "app"
diff --git a/tests/auto/qml/ecmascripttests/qjstest/main.cpp b/tests/auto/qml/ecmascripttests/qjstest/main.cpp
deleted file mode 100644
index 7bffedae81..0000000000
--- a/tests/auto/qml/ecmascripttests/qjstest/main.cpp
+++ /dev/null
@@ -1,90 +0,0 @@
-// Copyright (C) 2016 The Qt Company Ltd.
-// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0
-#include <QJSEngine>
-#include <QCoreApplication>
-#include <QCommandLineParser>
-#include <qdebug.h>
-#include <stdlib.h>
-
-#include "test262runner.h"
-
-int main(int argc, char **argv)
-{
-    QCoreApplication app(argc, argv);
-
-
-    QCommandLineParser parser;
-    parser.addHelpOption();
-    parser.addVersionOption();
-    QCommandLineOption verbose("verbose", "Verbose output");
-    parser.addOption(verbose);
-    QCommandLineOption commandOption("command", "Javascript command line interpreter", "command");
-    parser.addOption(commandOption);
-    QCommandLineOption testDir("tests", "path to the tests", "tests", "test262");
-    parser.addOption(testDir);
-    QCommandLineOption cat("cat", "Print packaged test code that would be run");
-    parser.addOption(cat);
-    QCommandLineOption parallel("parallel", "Run tests in parallel");
-    parser.addOption(parallel);
-    QCommandLineOption jit("jit", "JIT all code");
-    parser.addOption(jit);
-    QCommandLineOption bytecode("interpret", "Run using the bytecode interpreter");
-    parser.addOption(bytecode);
-    QCommandLineOption withExpectations("with-test-expectations", "Parse TestExpectations to deal with known failures");
-    parser.addOption(withExpectations);
-    QCommandLineOption updateExpectations("update-expectations", "Update TestExpectations to remove unexepected passes");
-    parser.addOption(updateExpectations);
-    QCommandLineOption writeExpectations("write-expectations", "Generate a new TestExpectations file based on the results of the run");
-    parser.addOption(writeExpectations);
-    parser.addPositionalArgument("[filter]", "Only run tests that contain filter in their name");
-
-    parser.process(app);
-
-    Test262Runner testRunner(parser.value(commandOption), parser.value(testDir), QStringLiteral("TestExpectations"));
-
-    QStringList otherArgs = parser.positionalArguments();
-    if (otherArgs.size() > 1) {
-        qWarning() << "too many arguments";
-        return 1;
-    } else if (otherArgs.size()) {
-        testRunner.setFilter(otherArgs.at(0));
-    }
-
-    if (parser.isSet(cat)) {
-        testRunner.cat();
-        return 0;
-    }
-
-    if (parser.isSet(updateExpectations) && parser.isSet(writeExpectations)) {
-        qWarning() << "Can only specify one of --update-expectations and --write-expectations.";
-        exit(1);
-    }
-
-    if (parser.isSet(jit) && parser.isSet(bytecode)) {
-        qWarning() << "Can only specify one of --jit and --interpret.";
-        exit(1);
-    }
-
-    int flags = 0;
-    if (parser.isSet(verbose))
-
-        flags |= Test262Runner::Verbose;
-    if (parser.isSet(parallel))
-        flags |= Test262Runner::Parallel;
-    if (parser.isSet(jit))
-        flags |= Test262Runner::ForceJIT;
-    if (parser.isSet(bytecode))
-        flags |= Test262Runner::ForceBytecode;
-    if (parser.isSet(withExpectations))
-        flags |= Test262Runner::WithTestExpectations;
-    if (parser.isSet(updateExpectations))
-        flags |= Test262Runner::UpdateTestExpectations;
-    if (parser.isSet(writeExpectations))
-        flags |= Test262Runner::WriteTestExpectations;
-    testRunner.setFlags(flags);
-
-    if (testRunner.run())
-        return EXIT_SUCCESS;
-    else
-        return EXIT_FAILURE;
-}
diff --git a/tests/auto/qml/ecmascripttests/qjstest/test262runner.cpp b/tests/auto/qml/ecmascripttests/test262runner.cpp
index f7c1a21c74..4c39dd661f 100644
--- a/tests/auto/qml/ecmascripttests/qjstest/test262runner.cpp
+++ b/tests/auto/qml/ecmascripttests/test262runner.cpp
@@ -3,20 +3,24 @@
 
 #include "test262runner.h"
 
-#include <qfile.h>
+#include <qdebug.h>
 #include <qdir.h>
 #include <qdiriterator.h>
-#include <qdebug.h>
+#include <qfile.h>
+#include <qjsondocument.h>
+#include <qjsonobject.h>
+#include <qlibraryinfo.h>
 #include <qprocess.h>
 #include <qtemporaryfile.h>
+#include <qthread.h>
 
-#include <private/qv4script_p.h>
-#include "private/qv4globalobject_p.h"
 #include "private/qqmlbuiltinfunctions_p.h"
 #include "private/qv4arraybuffer_p.h"
+#include "private/qv4globalobject_p.h"
 #include <QtCore/QLoggingCategory>
+#include <private/qv4script_p.h>
 
-#include "qrunnable.h"
+using namespace Qt::StringLiterals;
 
 static const char *excludedFeatures[] = {
     "BigInt",
@@ -72,7 +76,7 @@ static ReturnedValue method_detachArrayBuffer(const FunctionObject *f, const Val
     return Encode::null();
 }
 
-static void initD262(ExecutionEngine *e)
+void initD262(ExecutionEngine *e)
 {
     Scope scope(e);
     ScopedObject d262(scope, e->newObject());
@@ -85,7 +89,6 @@ static void initD262(ExecutionEngine *e)
 
 }
 
-QT_END_NAMESPACE
 
 Q_DECLARE_LOGGING_CATEGORY(lcJsTest);
 Q_LOGGING_CATEGORY(lcJsTest, "qt.v4.ecma262.tests", QtWarningMsg);
@@ -99,7 +102,8 @@ Test262Runner::Test262Runner(const QString &command, const QString &dir, const Q
 
 Test262Runner::~Test262Runner()
 {
-    delete threadPool;
+    if (threadPool)
+        delete threadPool;
 }
 
 void Test262Runner::cat()
@@ -113,18 +117,272 @@ void Test262Runner::cat()
     printf("%s", data.content.constData());
 }
 
+void Test262Runner::assignTaskOrTerminate(int processIndex)
+{
+    if (tasks.isEmpty()) {
+        sendDone(processIndex);
+        return;
+    }
+
+    currentTasks[processIndex] = tasks.dequeue();
+    TestData &task = currentTasks[processIndex];
+
+    // Sloppy run + maybe strict run later
+    if (task.runInSloppyMode) {
+        if (task.runInStrictMode)
+            task.stillNeedStrictRun = true;
+        assignSloppy(processIndex);
+        return;
+    }
+
+    // Only strict run
+    if (task.runInStrictMode) {
+        assignStrict(processIndex);
+        return;
+    }
+
+    // TODO: Start a timer for timeouts?
+}
+
+void Test262Runner::assignSloppy(int processIndex)
+{
+    QProcess &p = *processes[processIndex];
+    TestData &task = currentTasks[processIndex];
+
+    QJsonObject json;
+    json.insert("mode", "sloppy");
+    json.insert("testData", QString::fromUtf8(task.content));
+    json.insert("runAsModule", false);
+    json.insert("testCasePath", "");
+    json.insert("harnessForModules", "");
+    p.write(QJsonDocument(json).toJson(QJsonDocument::Compact));
+    p.write("\r\n");
+}
+
+void Test262Runner::assignStrict(int processIndex)
+{
+    QProcess &p = *processes[processIndex];
+    TestData &task = currentTasks[processIndex];
+
+    QJsonObject json;
+    json.insert("mode", "strict");
+    QString strictContent = "'use strict';\n" + QString::fromUtf8(task.content);
+    json.insert("testData", strictContent);
+    json.insert("runAsModule", task.runAsModuleCode);
+    json.insert("testCasePath", QFileInfo(testDir + "/test/" + task.test).absoluteFilePath());
+    json.insert("harnessForModules", QString::fromUtf8(task.harness));
+    p.write(QJsonDocument(json).toJson(QJsonDocument::Compact));
+    p.write("\r\n");
+}
+
+void Test262Runner::sendDone(int processIndex)
+{
+    QProcess &p = *processes[processIndex];
+
+    QJsonObject json;
+    json.insert("done", true);
+    p.write(QJsonDocument(json).toJson(QJsonDocument::Compact));
+    p.write("\r\n");
+}
+
+void Test262Runner::createProcesses()
+{
+    const int processCount = QThread::idealThreadCount();
+    qDebug() << "Running in parallel with" << processCount << "processes";
+    for (int i = 0; i < processCount; ++i) {
+        processes.emplace_back(std::make_unique<QProcess>());
+        QProcess &p = *processes[i];
+        QProcess::connect(&p, &QProcess::started, this, [&, i]() {
+            assignTaskOrTerminate(i);
+        });
+
+        QProcess::connect(&p, &QIODevice::readyRead, this, [&, i]() {
+            QProcess &p = *processes[i];
+            QString output;
+            while (output.isEmpty())
+                output = p.readLine();
+            QJsonDocument response = QJsonDocument::fromJson(output.toUtf8());
+
+            TestData &testData(currentTasks[i]);
+            auto mode = response["mode"].toString();
+            auto state = TestCase::State(response["resultState"].toInt(int(TestCase::State::Fails)));
+            auto errorMessage = response["resultErrorMessage"].toString();
+
+            auto &result = mode == "strict" ? testData.strictResult : testData.sloppyResult;
+            result = TestCase::Result(state, errorMessage);
+            if (testData.negative)
+                result.negateResult();
+
+            if (testData.stillNeedStrictRun) {
+                testData.stillNeedStrictRun = false;
+                assignStrict(i);
+            } else {
+                addResult(testData);
+                assignTaskOrTerminate(i);
+            }
+        });
+
+        QObject::connect(&p, &QProcess::finished, this, [&, i](int, QProcess::ExitStatus status) {
+            if (status != QProcess::NormalExit) {
+                qDebug() << QStringLiteral("Process %1 of %2 exited with a non-normal status")
+                                    .arg(i).arg(processCount - 1);
+            }
+
+            --runningCount;
+            if (runningCount == 0)
+                loop.exit();
+        });
+
+        p.setProgram(QCoreApplication::applicationFilePath());
+        QProcessEnvironment env = QProcessEnvironment::systemEnvironment();
+        env.insert(u"runnerProcess"_s, u"1"_s);
+        p.setProcessEnvironment(env);
+        ++runningCount;
+        p.start();
+    }
+}
+
+class SingleTest : public QRunnable
+{
+public:
+    SingleTest(Test262Runner *runner, const TestData &data)
+        : runner(runner), data(data)
+    {}
+    void run() override;
+
+    Test262Runner *runner;
+    TestData data;
+};
+
+TestCase::Result getTestExecutionResult(QV4::ExecutionEngine &vm)
+{
+    TestCase::State state;
+    QString errorMessage;
+    if (vm.hasException) {
+        state = TestCase::State::Fails;
+        QV4::Scope scope(&vm);
+        QV4::ScopedValue val(scope, vm.catchException());
+        errorMessage = val->toQString();
+    } else {
+        state = TestCase::State::Passes;
+    }
+    return TestCase::Result(state, errorMessage);
+}
+
+void SingleTest::run()
+{
+    if (data.runInSloppyMode) {
+        QV4::ExecutionEngine vm;
+        Test262Runner::executeTest(vm, data.content);
+        TestCase::Result ok = getTestExecutionResult(vm);
+
+        if (data.negative)
+            ok.negateResult();
+
+        data.sloppyResult = ok;
+    } else {
+        data.sloppyResult = TestCase::Result(TestCase::Skipped);
+    }
+    if (data.runInStrictMode) {
+        QString testCasePath = QFileInfo(runner->testDirectory() + "/test/" + data.test).absoluteFilePath();
+        QByteArray c = "'use strict';\n" + data.content;
+
+        QV4::ExecutionEngine vm;
+        Test262Runner::executeTest(vm, c, testCasePath, data.harness, data.runAsModuleCode);
+        TestCase::Result ok = getTestExecutionResult(vm);
+
+        if (data.negative)
+            ok.negateResult();
+
+        data.strictResult = ok;
+    } else {
+        data.strictResult = TestCase::Result(TestCase::Skipped);
+    }
+    runner->addResult(data);
+}
+
+void Test262Runner::executeTest(QV4::ExecutionEngine &vm, const QString &testData,
+                             const QString &testCasePath, const QString &harnessForModules,
+                             bool runAsModule)
+{
+    QV4::Scope scope(&vm);
+    QV4::GlobalExtensions::init(vm.globalObject,
+                                QJSEngine::ConsoleExtension | QJSEngine::GarbageCollectionExtension);
+    QV4::initD262(&vm);
+
+    if (runAsModule) {
+        const QUrl rootModuleUrl = QUrl::fromLocalFile(testCasePath);
+        // inject all modules with the harness
+        QVector<QUrl> modulesToLoad = { rootModuleUrl };
+        while (!modulesToLoad.isEmpty()) {
+            QUrl url = modulesToLoad.takeFirst();
+            QQmlRefPointer<QV4::ExecutableCompilationUnit> module;
+
+            QFile f(url.toLocalFile());
+            if (f.open(QIODevice::ReadOnly)) {
+                QByteArray content = harnessForModules.toLocal8Bit() + f.readAll();
+                module = vm.compileModule(url.toString(),
+                                          QString::fromUtf8(content.constData(),content.size()),
+                                          QFileInfo(f).lastModified());
+                if (vm.hasException)
+                    break;
+                vm.injectCompiledModule(module);
+            } else {
+                vm.throwError(QStringLiteral("Could not load module"));
+                break;
+            }
+
+            for (const QString &request: module->moduleRequests()) {
+                const QUrl absoluteRequest = module->finalUrl().resolved(QUrl(request));
+                const auto module = vm.moduleForUrl(absoluteRequest);
+                if (module.native == nullptr && module.compiled == nullptr)
+                    modulesToLoad << absoluteRequest;
+            }
+        }
+
+        if (!vm.hasException) {
+            const auto rootModule = vm.loadModule(rootModuleUrl);
+            if (rootModule.compiled && rootModule.compiled->instantiate(&vm))
+                rootModule.compiled->evaluate();
+        }
+    } else {
+        QV4::ScopedContext ctx(scope, vm.rootContext());
+
+        QV4::Script script(ctx, QV4::Compiler::ContextType::Global, testData);
+        script.parse();
+
+        if (!vm.hasException)
+            script.run();
+    }
+}
+
+void Test262Runner::runWithThreadPool()
+{
+    threadPool = new QThreadPool();
+    threadPool->setStackSize(16*1024*1024);
+    qDebug() << "Running in parallel with" << QThread::idealThreadCount() << "threads";
+
+    for (const TestCase &testCase : std::as_const(testCases)) {
+        TestData testData = getTestData(testCase);
+        if (testData.isExcluded || testData.async)
+            continue;
+        SingleTest *test = new SingleTest(this, testData);
+        threadPool->start(test);
+    }
+
+    while (!threadPool->waitForDone(10'000)) {
+        if (lcJsTest().isEnabled(QtDebugMsg)) {
+            // heartbeat, only needed when there is no other debug output
+            qDebug("test262: in progress...");
+        }
+    }
+}
+
 bool Test262Runner::run()
 {
     if (!loadTests())
         return false;
 
-    if (flags & Parallel) {
-        threadPool = new QThreadPool;
-        threadPool->setStackSize(16*1024*1024);
-        if (flags & Verbose)
-            qDebug() << "Running in parallel with" << QThread::idealThreadCount() << "threads.";
-    }
-
     if (flags & ForceJIT)
         qputenv("QV4_JIT_CALL_THRESHOLD", QByteArray("0"));
     else if (flags & ForceBytecode)
@@ -136,19 +394,24 @@ bool Test262Runner::run()
     for (auto it = testCases.constBegin(); it != testCases.constEnd(); ++it) {
         auto c = it.value();
         if (!c.skipTestCase) {
-            int result = runSingleTest(c);
-            if (result == -2)
-                return false;
+            TestData data = getTestData(c);
+            if (data.isExcluded || data.async)
+                continue;
+
+            tasks.append(data);
         }
     }
 
-    if (threadPool)
-        while (!threadPool->waitForDone(10000)) {
-            if (!lcJsTest().isEnabled(QtDebugMsg)) {
-                // heartbeat, only needed when there is no other debug output
-                qDebug("test262: in progress...");
-            }
-        }
+    if (command.isEmpty()) {
+#if QT_CONFIG(process)
+        createProcesses();
+        loop.exec();
+#else
+        runWithThreadPool();
+#endif
+    } else {
+        runAsExternalTests();
+    }
 
     const bool testsOk = report();
 
@@ -172,7 +435,7 @@ bool Test262Runner::report()
         if (c.strictResult.state == c.strictExpectation.state
             && c.sloppyResult.state == c.sloppyExpectation.state)
             continue;
-        auto report = [&](TestCase::Result expected, TestCase::Result result, const char *s) {
+        auto report = [&](const TestCase::Result &expected, const TestCase::Result &result, const char *s) {
             if (result.state == TestCase::Crashes)
                 crashes << (it.key() + " crashed in " + s + " mode");
             if (result.state == TestCase::Fails && expected.state == TestCase::Passes)
@@ -465,7 +728,7 @@ void Test262Runner::writeTestExpectations()
     QTemporaryFile expectations;
     expectations.open();
 
-    for (auto c : std::as_const(testCases)) {
+    for (const auto &c : std::as_const(testCases)) {
         TestExpectationLine line = TestExpectationLine::fromTestCase(c);
         expectations.write(line.toLine());
     }
@@ -479,175 +742,47 @@ void Test262Runner::writeTestExpectations()
         qWarning() << "Could not write new TestExpectations file at" << expectationsFile;
 }
 
-static TestCase::Result executeTest(const QByteArray &data, bool runAsModule = false,
-                                    const QString &testCasePath = QString(),
-                                    const QByteArray &harnessForModules = QByteArray())
+void Test262Runner::runAsExternalTests()
 {
-    QString testData = QString::fromUtf8(data.constData(), data.size());
-
-    QV4::ExecutionEngine vm;
-
-    QV4::Scope scope(&vm);
-
-    QV4::GlobalExtensions::init(vm.globalObject, QJSEngine::ConsoleExtension | QJSEngine::GarbageCollectionExtension);
-    QV4::initD262(&vm);
-
-    if (runAsModule) {
-        const QUrl rootModuleUrl = QUrl::fromLocalFile(testCasePath);
-        // inject all modules with the harness
-        QVector<QUrl> modulesToLoad = { rootModuleUrl };
-        while (!modulesToLoad.isEmpty()) {
-            QUrl url = modulesToLoad.takeFirst();
-            QQmlRefPointer<QV4::ExecutableCompilationUnit> module;
-
-            QFile f(url.toLocalFile());
-            if (f.open(QIODevice::ReadOnly)) {
-                QByteArray content = harnessForModules + f.readAll();
-                module = vm.compileModule(url.toString(), QString::fromUtf8(content.constData(), content.size()), QFileInfo(f).lastModified());
-                if (vm.hasException)
-                    break;
-                vm.injectCompiledModule(module);
-            } else {
-                vm.throwError(QStringLiteral("Could not load module"));
-                break;
+    for (TestData &testData : tasks) {
+        auto runTest = [&] (const char *header, TestCase::Result *result) {
+            QTemporaryFile tempFile;
+            tempFile.open();
+            tempFile.write(header);
+            tempFile.write(testData.content);
+            tempFile.close();
+
+            QProcess process;
+            process.start(command, QStringList(tempFile.fileName()));
+            if (!process.waitForFinished(-1) || process.error() == QProcess::FailedToStart) {
+                qWarning() << "Could not execute" << command;
+                *result = TestCase::Result(TestCase::Crashes);
             }
-
-            for (const QString &request: module->moduleRequests()) {
-                const QUrl absoluteRequest = module->finalUrl().resolved(QUrl(request));
-                const auto module = vm.moduleForUrl(absoluteRequest);
-                if (module.native == nullptr && module.compiled == nullptr)
-                    modulesToLoad << absoluteRequest;
+            if (process.exitStatus() != QProcess::NormalExit) {
+                *result = TestCase::Result(TestCase::Crashes);
             }
-        }
-
-        if (!vm.hasException) {
-            const auto rootModule = vm.loadModule(rootModuleUrl);
-            if (rootModule.compiled && rootModule.compiled->instantiate(&vm))
-                rootModule.compiled->evaluate();
-        }
-    } else {
-        QV4::ScopedContext ctx(scope, vm.rootContext());
-
-        QV4::Script script(ctx, QV4::Compiler::ContextType::Global, testData);
-        script.parse();
-
-        if (!vm.hasException)
-            script.run();
-    }
-
-    if (vm.hasException) {
-        QV4::Scope scope(&vm);
-        QV4::ScopedValue val(scope, vm.catchException());
-        return TestCase::Result(TestCase::Fails, val->toQString());
-    }
-    return TestCase::Result(TestCase::Passes);
-}
-
-class SingleTest : public QRunnable
-{
-public:
-    SingleTest(Test262Runner *runner, const TestData &data)
-        : runner(runner), data(data)
-    {
-        command = runner->command;
-    }
-    void run() override;
-
-    void runExternalTest();
-
-    QString command;
-    Test262Runner *runner;
-    TestData data;
-};
-
-void SingleTest::run()
-{
-    if (!command.isEmpty()) {
-        runExternalTest();
-        return;
-    }
-
-    if (data.runInSloppyMode) {
-        TestCase::Result ok = ::executeTest(data.content);
-        if (data.negative)
-            ok.negateResult();
-
-        data.sloppyResult = ok;
-    } else {
-        data.sloppyResult = TestCase::Result(TestCase::Skipped);
-    }
-    if (data.runInStrictMode) {
-        const QString testCasePath = QFileInfo(runner->testDir + "/test/" + data.test).absoluteFilePath();
-        QByteArray c = "'use strict';\n" + data.content;
-        TestCase::Result ok = ::executeTest(c, data.runAsModuleCode, testCasePath, data.harness);
-        if (data.negative)
-            ok.negateResult();
-
-        data.strictResult = ok;
-    } else {
-        data.strictResult = TestCase::Result(TestCase::Skipped);
-    }
-    runner->addResult(data);
-}
-
-void SingleTest::runExternalTest()
-{
-    auto runTest = [this] (const char *header, TestCase::Result *result) {
-        QTemporaryFile tempFile;
-        tempFile.open();
-        tempFile.write(header);
-        tempFile.write(data.content);
-        tempFile.close();
-
-        QProcess process;
-//        if (flags & Verbose)
-//            process.setReadChannelMode(QProcess::ForwardedChannels);
-
-        process.start(command, QStringList(tempFile.fileName()));
-        if (!process.waitForFinished(-1) || process.error() == QProcess::FailedToStart) {
-            qWarning() << "Could not execute" << command;
-            *result = TestCase::Result(TestCase::Crashes);
-        }
-        if (process.exitStatus() != QProcess::NormalExit) {
-            *result = TestCase::Result(TestCase::Crashes);
-        }
-        bool ok = (process.exitCode() == EXIT_SUCCESS);
-        if (data.negative)
-            ok = !ok;
-        *result = ok ? TestCase::Result(TestCase::Passes)
-                     : TestCase::Result(TestCase::Fails, process.readAllStandardError());
-    };
-
-    if (data.runInSloppyMode)
-        runTest("", &data.sloppyResult);
-    if (data.runInStrictMode)
-        runTest("'use strict';\n", &data.strictResult);
-
-    runner->addResult(data);
-}
-
-int Test262Runner::runSingleTest(TestCase testCase)
-{
-    TestData data = getTestData(testCase);
-//    qDebug() << "starting test" << data.test;
+            bool ok = (process.exitCode() == EXIT_SUCCESS);
+            if (testData.negative)
+                ok = !ok;
+            *result = ok ? TestCase::Result(TestCase::Passes)
+                         : TestCase::Result(TestCase::Fails, process.readAllStandardError());
+        };
 
-    if (data.isExcluded || data.async)
-        return 0;
+        if (testData.runInSloppyMode)
+            runTest("", &testData.sloppyResult);
+        if (testData.runInStrictMode)
+            runTest("'use strict';\n", &testData.strictResult);
 
-    if (threadPool) {
-        SingleTest *test = new SingleTest(this, data);
-        threadPool->start(test);
-        return 0;
+        addResult(testData);
     }
-    SingleTest test(this, data);
-    test.run();
-    return 0;
 }
 
 void Test262Runner::addResult(TestCase result)
 {
     {
+#if !QT_CONFIG(process)
         QMutexLocker locker(&mutex);
+#endif
         Q_ASSERT(result.strictExpectation.state == testCases[result.test].strictExpectation.state);
         Q_ASSERT(result.sloppyExpectation.state == testCases[result.test].sloppyExpectation.state);
         testCases[result.test] = result;
@@ -852,3 +987,5 @@ QByteArray Test262Runner::harness(const QByteArray &name)
     harnessFiles.insert(name, content);
     return content;
 }
+
+QT_END_NAMESPACE
diff --git a/tests/auto/qml/ecmascripttests/qjstest/test262runner.h b/tests/auto/qml/ecmascripttests/test262runner.h
index e2bf26296f..53c66618f0 100644
--- a/tests/auto/qml/ecmascripttests/qjstest/test262runner.h
+++ b/tests/auto/qml/ecmascripttests/test262runner.h
@@ -1,14 +1,24 @@
 // Copyright (C) 2016 The Qt Company Ltd.
 // SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0
+
 #ifndef TEST262RUNNER_H
 #define TEST262RUNNER_H
-#include <qstring.h>
-#include <qstringlist.h>
-#include <qset.h>
+
+#include <qeventloop.h>
 #include <qmap.h>
 #include <qmutex.h>
+#include <qprocess.h>
+#include <qqueue.h>
+#include <qset.h>
 #include <qthreadpool.h>
 
+QT_BEGIN_NAMESPACE
+
+namespace QV4 {
+struct ExecutionEngine;
+void initD262(ExecutionEngine *e);
+}
+
 struct TestCase {
     TestCase() = default;
     TestCase(const QString &test)
@@ -40,16 +50,18 @@ struct TestCase {
         }
     };
 
-    bool skipTestCase = false;
     Result strictExpectation = Result(Passes);
     Result sloppyExpectation = Result(Passes);
     Result strictResult = Result(Skipped);
     Result sloppyResult = Result(Skipped);
+    bool skipTestCase = false;
+    bool stillNeedStrictRun = false;
 
     QString test;
 };
 
 struct TestData : TestCase {
+    TestData() = default;
     TestData(const TestCase &testCase)
         : TestCase(testCase) {}
     // flags
@@ -67,8 +79,12 @@ struct TestData : TestCase {
     QByteArray content;
 };
 
-class Test262Runner
+class SingleTest;
+
+class Test262Runner : public QObject
 {
+    Q_OBJECT
+
 public:
     Test262Runner(const QString &command, const QString &testDir, const QString &expectationsFile);
     ~Test262Runner();
@@ -95,6 +111,12 @@ public:
     bool run();
 
     bool report();
+    QString testDirectory() const { return testDir; }
+
+    static void executeTest(QV4::ExecutionEngine &vm, const QString &testData,
+                            const QString &testCasePath = QString(),
+                            const QString &harnessForModules = QString(),
+                            bool runAsModule = false);
 
 private:
     friend class SingleTest;
@@ -102,7 +124,16 @@ private:
     void loadTestExpectations();
     void updateTestExpectations();
     void writeTestExpectations();
-    int runSingleTest(TestCase testCase);
+
+    void runWithThreadPool();
+
+    void runAsExternalTests();
+    void createProcesses();
+    void assignTaskOrTerminate(int processIndex);
+    void assignSloppy(int processIndex);
+    void assignStrict(int processIndex);
+    void sendDone(int processIndex);
+    QString readUntilNull(QProcess &p);
 
     TestData getTestData(const TestCase &testCase);
     void parseYaml(const QByteArray &content, TestData *data);
@@ -116,14 +147,21 @@ private:
     QString expectationsFile;
     int flags = 0;
 
-    QMutex mutex;
     QString filter;
 
     QMap<QString, TestCase> testCases;
     QHash<QByteArray, QByteArray> harnessFiles;
 
     QThreadPool *threadPool = nullptr;
+    QMutex mutex;
+
+    QEventLoop loop;
+    std::vector<std::unique_ptr<QProcess>> processes;
+    int runningCount = 0;
+    QQueue<TestData> tasks;
+    QHash<int, TestData> currentTasks;
 };
 
+QT_END_NAMESPACE
 
 #endif
diff --git a/tests/auto/qml/ecmascripttests/tst_ecmascripttests.cpp b/tests/auto/qml/ecmascripttests/tst_ecmascripttests.cpp
index e4a108a2e8..fc20c80a1a 100644
--- a/tests/auto/qml/ecmascripttests/tst_ecmascripttests.cpp
+++ b/tests/auto/qml/ecmascripttests/tst_ecmascripttests.cpp
@@ -1,11 +1,22 @@
 // Copyright (C) 2017 The Qt Company Ltd.
 // SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0
 
-#include <QtTest/QtTest>
-#include <QProcess>
+#include <QFileInfo>
+#include <QJSEngine>
+#include <QJsonDocument>
+#include <QJsonObject>
 #include <QLibraryInfo>
-#include <qjstest/test262runner.h>
+#include <QProcess>
 #include <QtQuickTestUtils/private/qmlutils_p.h>
+#include <QtTest/QtTest>
+
+#include "test262runner.h"
+#include "private/qqmlbuiltinfunctions_p.h"
+#include "private/qv4arraybuffer_p.h"
+#include "private/qv4globalobject_p.h"
+#include "private/qv4script_p.h"
+
+#include <stdio.h>
 
 class tst_EcmaScriptTests : public QQmlDataTest
 {
@@ -94,7 +105,74 @@ void tst_EcmaScriptTests::runJitted()
     QVERIFY(result);
 }
 
-QTEST_GUILESS_MAIN(tst_EcmaScriptTests)
+//// v RUNNER PROCESS MODE v ////
 
-#include "tst_ecmascripttests.moc"
+void readInput(bool &done, QString &mode, QString &testData, QString &testCasePath,
+               QString &harnessForModules, bool &runAsModule)
+{
+    QTextStream in(stdin);
+    QString input;
+    while (input.isEmpty())
+        input = in.readLine();
+
+    QJsonDocument json = QJsonDocument::fromJson(input.toUtf8());
+    done = json["done"].toBool(false);
+    mode = json["mode"].toString();
+    testData = json["testData"].toString();
+    testCasePath = json["testCasePath"].toString();
+    harnessForModules = json["harnessForModules"].toString();
+    runAsModule = json["runAsModule"].toBool(false);
+}
+
+void printResult(QV4::ExecutionEngine &vm, const QString &mode)
+{
+    QJsonObject result;
+    result.insert("mode", mode);
+    if (vm.hasException) {
+        QV4::Scope scope(&vm);
+        QV4::ScopedValue val(scope, vm.catchException());
+
+        result.insert("resultState", int(TestCase::State::Fails));
+        result.insert("resultErrorMessage", val->toQString());
+    } else {
+        result.insert("resultState", int(TestCase::State::Passes));
+    }
 
+    QTextStream(stdout) << QJsonDocument(result).toJson(QJsonDocument::Compact) << "\r\n";
+}
+
+void doRunnerProcess()
+{
+    bool done = false;
+    QString mode;
+    QString testData;
+    QString testCasePath;
+    QString harnessForModules;
+    bool runAsModule = false;
+
+    while (!done) {
+        QV4::ExecutionEngine vm;
+        readInput(done, mode, testData, testCasePath, harnessForModules, runAsModule);
+        if (done)
+            break;
+        Test262Runner::executeTest(vm, testData, testCasePath, harnessForModules, runAsModule);
+        printResult(vm, mode);
+    }
+}
+
+//// ^ RUNNER PROCESS MODE ^ ////
+
+int main(int argc, char *argv[])
+{
+    QCoreApplication app(argc, argv);
+
+    if (qEnvironmentVariableIntValue("runnerProcess") == 1) {
+        doRunnerProcess();
+    } else {
+        tst_EcmaScriptTests tc;
+        QTEST_SET_MAIN_SOURCE_PATH
+        return QTest::qExec(&tc, argc, argv);
+    }
+}
+
+#include "tst_ecmascripttests.moc"