diff options
| -rw-r--r-- | tests/auto/qml/ecmascripttests/CMakeLists.txt | 6 | ||||
| -rw-r--r-- | tests/auto/qml/ecmascripttests/qjstest/CMakeLists.txt | 28 | ||||
| -rw-r--r-- | tests/auto/qml/ecmascripttests/qjstest/main.cpp | 90 | ||||
| -rw-r--r-- | tests/auto/qml/ecmascripttests/test262runner.cpp (renamed from tests/auto/qml/ecmascripttests/qjstest/test262runner.cpp) | 505 | ||||
| -rw-r--r-- | tests/auto/qml/ecmascripttests/test262runner.h (renamed from tests/auto/qml/ecmascripttests/qjstest/test262runner.h) | 52 | ||||
| -rw-r--r-- | tests/auto/qml/ecmascripttests/tst_ecmascripttests.cpp | 88 |
6 files changed, 450 insertions, 319 deletions
diff --git a/tests/auto/qml/ecmascripttests/CMakeLists.txt b/tests/auto/qml/ecmascripttests/CMakeLists.txt index e03f5ffa82..1ee70cb101 100644 --- a/tests/auto/qml/ecmascripttests/CMakeLists.txt +++ b/tests/auto/qml/ecmascripttests/CMakeLists.txt @@ -20,7 +20,7 @@ list(FILTER test_data EXCLUDE REGEX ".git") qt_internal_add_test(tst_ecmascripttests SOURCES - qjstest/test262runner.cpp qjstest/test262runner.h + test262runner.cpp test262runner.h tst_ecmascripttests.cpp LIBRARIES Qt::QmlPrivate @@ -47,7 +47,3 @@ else() QT_QMLTEST_DATADIR="${CMAKE_CURRENT_SOURCE_DIR}/test262" ) endif() - -if(NOT CMAKE_CROSSCOMPILING) - add_subdirectory(qjstest) -endif() diff --git a/tests/auto/qml/ecmascripttests/qjstest/CMakeLists.txt b/tests/auto/qml/ecmascripttests/qjstest/CMakeLists.txt deleted file mode 100644 index 86ca5f97a3..0000000000 --- a/tests/auto/qml/ecmascripttests/qjstest/CMakeLists.txt +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright (C) 2022 The Qt Company Ltd. -# SPDX-License-Identifier: BSD-3-Clause - -# Generated from qjstest.pro. - -##################################################################### -## qjstest Tool: -##################################################################### - -qt_get_tool_target_name(target_name qjstest) -qt_internal_add_tool(${target_name} - TARGET_DESCRIPTION "Javascript test runner" - SOURCES - main.cpp - test262runner.cpp test262runner.h - DEFINES - QT_DEPRECATED_WARNINGS - INCLUDE_DIRECTORIES - . - LIBRARIES - Qt::Gui - Qt::QmlPrivate -) -qt_internal_return_unless_building_tools() - -#### Keys ignored in scope 1:.:.:qjstest.pro:<TRUE>: -# QMAKE_TARGET_DESCRIPTION = "Javascript" "test" "runner" -# TEMPLATE = "app" diff --git a/tests/auto/qml/ecmascripttests/qjstest/main.cpp b/tests/auto/qml/ecmascripttests/qjstest/main.cpp deleted file mode 100644 index 7bffedae81..0000000000 --- a/tests/auto/qml/ecmascripttests/qjstest/main.cpp +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright (C) 2016 The Qt Company Ltd. -// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0 -#include <QJSEngine> -#include <QCoreApplication> -#include <QCommandLineParser> -#include <qdebug.h> -#include <stdlib.h> - -#include "test262runner.h" - -int main(int argc, char **argv) -{ - QCoreApplication app(argc, argv); - - - QCommandLineParser parser; - parser.addHelpOption(); - parser.addVersionOption(); - QCommandLineOption verbose("verbose", "Verbose output"); - parser.addOption(verbose); - QCommandLineOption commandOption("command", "Javascript command line interpreter", "command"); - parser.addOption(commandOption); - QCommandLineOption testDir("tests", "path to the tests", "tests", "test262"); - parser.addOption(testDir); - QCommandLineOption cat("cat", "Print packaged test code that would be run"); - parser.addOption(cat); - QCommandLineOption parallel("parallel", "Run tests in parallel"); - parser.addOption(parallel); - QCommandLineOption jit("jit", "JIT all code"); - parser.addOption(jit); - QCommandLineOption bytecode("interpret", "Run using the bytecode interpreter"); - parser.addOption(bytecode); - QCommandLineOption withExpectations("with-test-expectations", "Parse TestExpectations to deal with known failures"); - parser.addOption(withExpectations); - QCommandLineOption updateExpectations("update-expectations", "Update TestExpectations to remove unexepected passes"); - parser.addOption(updateExpectations); - QCommandLineOption writeExpectations("write-expectations", "Generate a new TestExpectations file based on the results of the run"); - parser.addOption(writeExpectations); - parser.addPositionalArgument("[filter]", "Only run tests that contain filter in their name"); - - parser.process(app); - - Test262Runner testRunner(parser.value(commandOption), parser.value(testDir), QStringLiteral("TestExpectations")); - - QStringList otherArgs = parser.positionalArguments(); - if (otherArgs.size() > 1) { - qWarning() << "too many arguments"; - return 1; - } else if (otherArgs.size()) { - testRunner.setFilter(otherArgs.at(0)); - } - - if (parser.isSet(cat)) { - testRunner.cat(); - return 0; - } - - if (parser.isSet(updateExpectations) && parser.isSet(writeExpectations)) { - qWarning() << "Can only specify one of --update-expectations and --write-expectations."; - exit(1); - } - - if (parser.isSet(jit) && parser.isSet(bytecode)) { - qWarning() << "Can only specify one of --jit and --interpret."; - exit(1); - } - - int flags = 0; - if (parser.isSet(verbose)) - - flags |= Test262Runner::Verbose; - if (parser.isSet(parallel)) - flags |= Test262Runner::Parallel; - if (parser.isSet(jit)) - flags |= Test262Runner::ForceJIT; - if (parser.isSet(bytecode)) - flags |= Test262Runner::ForceBytecode; - if (parser.isSet(withExpectations)) - flags |= Test262Runner::WithTestExpectations; - if (parser.isSet(updateExpectations)) - flags |= Test262Runner::UpdateTestExpectations; - if (parser.isSet(writeExpectations)) - flags |= Test262Runner::WriteTestExpectations; - testRunner.setFlags(flags); - - if (testRunner.run()) - return EXIT_SUCCESS; - else - return EXIT_FAILURE; -} diff --git a/tests/auto/qml/ecmascripttests/qjstest/test262runner.cpp b/tests/auto/qml/ecmascripttests/test262runner.cpp index f7c1a21c74..4c39dd661f 100644 --- a/tests/auto/qml/ecmascripttests/qjstest/test262runner.cpp +++ b/tests/auto/qml/ecmascripttests/test262runner.cpp @@ -3,20 +3,24 @@ #include "test262runner.h" -#include <qfile.h> +#include <qdebug.h> #include <qdir.h> #include <qdiriterator.h> -#include <qdebug.h> +#include <qfile.h> +#include <qjsondocument.h> +#include <qjsonobject.h> +#include <qlibraryinfo.h> #include <qprocess.h> #include <qtemporaryfile.h> +#include <qthread.h> -#include <private/qv4script_p.h> -#include "private/qv4globalobject_p.h" #include "private/qqmlbuiltinfunctions_p.h" #include "private/qv4arraybuffer_p.h" +#include "private/qv4globalobject_p.h" #include <QtCore/QLoggingCategory> +#include <private/qv4script_p.h> -#include "qrunnable.h" +using namespace Qt::StringLiterals; static const char *excludedFeatures[] = { "BigInt", @@ -72,7 +76,7 @@ static ReturnedValue method_detachArrayBuffer(const FunctionObject *f, const Val return Encode::null(); } -static void initD262(ExecutionEngine *e) +void initD262(ExecutionEngine *e) { Scope scope(e); ScopedObject d262(scope, e->newObject()); @@ -85,7 +89,6 @@ static void initD262(ExecutionEngine *e) } -QT_END_NAMESPACE Q_DECLARE_LOGGING_CATEGORY(lcJsTest); Q_LOGGING_CATEGORY(lcJsTest, "qt.v4.ecma262.tests", QtWarningMsg); @@ -99,7 +102,8 @@ Test262Runner::Test262Runner(const QString &command, const QString &dir, const Q Test262Runner::~Test262Runner() { - delete threadPool; + if (threadPool) + delete threadPool; } void Test262Runner::cat() @@ -113,18 +117,272 @@ void Test262Runner::cat() printf("%s", data.content.constData()); } +void Test262Runner::assignTaskOrTerminate(int processIndex) +{ + if (tasks.isEmpty()) { + sendDone(processIndex); + return; + } + + currentTasks[processIndex] = tasks.dequeue(); + TestData &task = currentTasks[processIndex]; + + // Sloppy run + maybe strict run later + if (task.runInSloppyMode) { + if (task.runInStrictMode) + task.stillNeedStrictRun = true; + assignSloppy(processIndex); + return; + } + + // Only strict run + if (task.runInStrictMode) { + assignStrict(processIndex); + return; + } + + // TODO: Start a timer for timeouts? +} + +void Test262Runner::assignSloppy(int processIndex) +{ + QProcess &p = *processes[processIndex]; + TestData &task = currentTasks[processIndex]; + + QJsonObject json; + json.insert("mode", "sloppy"); + json.insert("testData", QString::fromUtf8(task.content)); + json.insert("runAsModule", false); + json.insert("testCasePath", ""); + json.insert("harnessForModules", ""); + p.write(QJsonDocument(json).toJson(QJsonDocument::Compact)); + p.write("\r\n"); +} + +void Test262Runner::assignStrict(int processIndex) +{ + QProcess &p = *processes[processIndex]; + TestData &task = currentTasks[processIndex]; + + QJsonObject json; + json.insert("mode", "strict"); + QString strictContent = "'use strict';\n" + QString::fromUtf8(task.content); + json.insert("testData", strictContent); + json.insert("runAsModule", task.runAsModuleCode); + json.insert("testCasePath", QFileInfo(testDir + "/test/" + task.test).absoluteFilePath()); + json.insert("harnessForModules", QString::fromUtf8(task.harness)); + p.write(QJsonDocument(json).toJson(QJsonDocument::Compact)); + p.write("\r\n"); +} + +void Test262Runner::sendDone(int processIndex) +{ + QProcess &p = *processes[processIndex]; + + QJsonObject json; + json.insert("done", true); + p.write(QJsonDocument(json).toJson(QJsonDocument::Compact)); + p.write("\r\n"); +} + +void Test262Runner::createProcesses() +{ + const int processCount = QThread::idealThreadCount(); + qDebug() << "Running in parallel with" << processCount << "processes"; + for (int i = 0; i < processCount; ++i) { + processes.emplace_back(std::make_unique<QProcess>()); + QProcess &p = *processes[i]; + QProcess::connect(&p, &QProcess::started, this, [&, i]() { + assignTaskOrTerminate(i); + }); + + QProcess::connect(&p, &QIODevice::readyRead, this, [&, i]() { + QProcess &p = *processes[i]; + QString output; + while (output.isEmpty()) + output = p.readLine(); + QJsonDocument response = QJsonDocument::fromJson(output.toUtf8()); + + TestData &testData(currentTasks[i]); + auto mode = response["mode"].toString(); + auto state = TestCase::State(response["resultState"].toInt(int(TestCase::State::Fails))); + auto errorMessage = response["resultErrorMessage"].toString(); + + auto &result = mode == "strict" ? testData.strictResult : testData.sloppyResult; + result = TestCase::Result(state, errorMessage); + if (testData.negative) + result.negateResult(); + + if (testData.stillNeedStrictRun) { + testData.stillNeedStrictRun = false; + assignStrict(i); + } else { + addResult(testData); + assignTaskOrTerminate(i); + } + }); + + QObject::connect(&p, &QProcess::finished, this, [&, i](int, QProcess::ExitStatus status) { + if (status != QProcess::NormalExit) { + qDebug() << QStringLiteral("Process %1 of %2 exited with a non-normal status") + .arg(i).arg(processCount - 1); + } + + --runningCount; + if (runningCount == 0) + loop.exit(); + }); + + p.setProgram(QCoreApplication::applicationFilePath()); + QProcessEnvironment env = QProcessEnvironment::systemEnvironment(); + env.insert(u"runnerProcess"_s, u"1"_s); + p.setProcessEnvironment(env); + ++runningCount; + p.start(); + } +} + +class SingleTest : public QRunnable +{ +public: + SingleTest(Test262Runner *runner, const TestData &data) + : runner(runner), data(data) + {} + void run() override; + + Test262Runner *runner; + TestData data; +}; + +TestCase::Result getTestExecutionResult(QV4::ExecutionEngine &vm) +{ + TestCase::State state; + QString errorMessage; + if (vm.hasException) { + state = TestCase::State::Fails; + QV4::Scope scope(&vm); + QV4::ScopedValue val(scope, vm.catchException()); + errorMessage = val->toQString(); + } else { + state = TestCase::State::Passes; + } + return TestCase::Result(state, errorMessage); +} + +void SingleTest::run() +{ + if (data.runInSloppyMode) { + QV4::ExecutionEngine vm; + Test262Runner::executeTest(vm, data.content); + TestCase::Result ok = getTestExecutionResult(vm); + + if (data.negative) + ok.negateResult(); + + data.sloppyResult = ok; + } else { + data.sloppyResult = TestCase::Result(TestCase::Skipped); + } + if (data.runInStrictMode) { + QString testCasePath = QFileInfo(runner->testDirectory() + "/test/" + data.test).absoluteFilePath(); + QByteArray c = "'use strict';\n" + data.content; + + QV4::ExecutionEngine vm; + Test262Runner::executeTest(vm, c, testCasePath, data.harness, data.runAsModuleCode); + TestCase::Result ok = getTestExecutionResult(vm); + + if (data.negative) + ok.negateResult(); + + data.strictResult = ok; + } else { + data.strictResult = TestCase::Result(TestCase::Skipped); + } + runner->addResult(data); +} + +void Test262Runner::executeTest(QV4::ExecutionEngine &vm, const QString &testData, + const QString &testCasePath, const QString &harnessForModules, + bool runAsModule) +{ + QV4::Scope scope(&vm); + QV4::GlobalExtensions::init(vm.globalObject, + QJSEngine::ConsoleExtension | QJSEngine::GarbageCollectionExtension); + QV4::initD262(&vm); + + if (runAsModule) { + const QUrl rootModuleUrl = QUrl::fromLocalFile(testCasePath); + // inject all modules with the harness + QVector<QUrl> modulesToLoad = { rootModuleUrl }; + while (!modulesToLoad.isEmpty()) { + QUrl url = modulesToLoad.takeFirst(); + QQmlRefPointer<QV4::ExecutableCompilationUnit> module; + + QFile f(url.toLocalFile()); + if (f.open(QIODevice::ReadOnly)) { + QByteArray content = harnessForModules.toLocal8Bit() + f.readAll(); + module = vm.compileModule(url.toString(), + QString::fromUtf8(content.constData(),content.size()), + QFileInfo(f).lastModified()); + if (vm.hasException) + break; + vm.injectCompiledModule(module); + } else { + vm.throwError(QStringLiteral("Could not load module")); + break; + } + + for (const QString &request: module->moduleRequests()) { + const QUrl absoluteRequest = module->finalUrl().resolved(QUrl(request)); + const auto module = vm.moduleForUrl(absoluteRequest); + if (module.native == nullptr && module.compiled == nullptr) + modulesToLoad << absoluteRequest; + } + } + + if (!vm.hasException) { + const auto rootModule = vm.loadModule(rootModuleUrl); + if (rootModule.compiled && rootModule.compiled->instantiate(&vm)) + rootModule.compiled->evaluate(); + } + } else { + QV4::ScopedContext ctx(scope, vm.rootContext()); + + QV4::Script script(ctx, QV4::Compiler::ContextType::Global, testData); + script.parse(); + + if (!vm.hasException) + script.run(); + } +} + +void Test262Runner::runWithThreadPool() +{ + threadPool = new QThreadPool(); + threadPool->setStackSize(16*1024*1024); + qDebug() << "Running in parallel with" << QThread::idealThreadCount() << "threads"; + + for (const TestCase &testCase : std::as_const(testCases)) { + TestData testData = getTestData(testCase); + if (testData.isExcluded || testData.async) + continue; + SingleTest *test = new SingleTest(this, testData); + threadPool->start(test); + } + + while (!threadPool->waitForDone(10'000)) { + if (lcJsTest().isEnabled(QtDebugMsg)) { + // heartbeat, only needed when there is no other debug output + qDebug("test262: in progress..."); + } + } +} + bool Test262Runner::run() { if (!loadTests()) return false; - if (flags & Parallel) { - threadPool = new QThreadPool; - threadPool->setStackSize(16*1024*1024); - if (flags & Verbose) - qDebug() << "Running in parallel with" << QThread::idealThreadCount() << "threads."; - } - if (flags & ForceJIT) qputenv("QV4_JIT_CALL_THRESHOLD", QByteArray("0")); else if (flags & ForceBytecode) @@ -136,19 +394,24 @@ bool Test262Runner::run() for (auto it = testCases.constBegin(); it != testCases.constEnd(); ++it) { auto c = it.value(); if (!c.skipTestCase) { - int result = runSingleTest(c); - if (result == -2) - return false; + TestData data = getTestData(c); + if (data.isExcluded || data.async) + continue; + + tasks.append(data); } } - if (threadPool) - while (!threadPool->waitForDone(10000)) { - if (!lcJsTest().isEnabled(QtDebugMsg)) { - // heartbeat, only needed when there is no other debug output - qDebug("test262: in progress..."); - } - } + if (command.isEmpty()) { +#if QT_CONFIG(process) + createProcesses(); + loop.exec(); +#else + runWithThreadPool(); +#endif + } else { + runAsExternalTests(); + } const bool testsOk = report(); @@ -172,7 +435,7 @@ bool Test262Runner::report() if (c.strictResult.state == c.strictExpectation.state && c.sloppyResult.state == c.sloppyExpectation.state) continue; - auto report = [&](TestCase::Result expected, TestCase::Result result, const char *s) { + auto report = [&](const TestCase::Result &expected, const TestCase::Result &result, const char *s) { if (result.state == TestCase::Crashes) crashes << (it.key() + " crashed in " + s + " mode"); if (result.state == TestCase::Fails && expected.state == TestCase::Passes) @@ -465,7 +728,7 @@ void Test262Runner::writeTestExpectations() QTemporaryFile expectations; expectations.open(); - for (auto c : std::as_const(testCases)) { + for (const auto &c : std::as_const(testCases)) { TestExpectationLine line = TestExpectationLine::fromTestCase(c); expectations.write(line.toLine()); } @@ -479,175 +742,47 @@ void Test262Runner::writeTestExpectations() qWarning() << "Could not write new TestExpectations file at" << expectationsFile; } -static TestCase::Result executeTest(const QByteArray &data, bool runAsModule = false, - const QString &testCasePath = QString(), - const QByteArray &harnessForModules = QByteArray()) +void Test262Runner::runAsExternalTests() { - QString testData = QString::fromUtf8(data.constData(), data.size()); - - QV4::ExecutionEngine vm; - - QV4::Scope scope(&vm); - - QV4::GlobalExtensions::init(vm.globalObject, QJSEngine::ConsoleExtension | QJSEngine::GarbageCollectionExtension); - QV4::initD262(&vm); - - if (runAsModule) { - const QUrl rootModuleUrl = QUrl::fromLocalFile(testCasePath); - // inject all modules with the harness - QVector<QUrl> modulesToLoad = { rootModuleUrl }; - while (!modulesToLoad.isEmpty()) { - QUrl url = modulesToLoad.takeFirst(); - QQmlRefPointer<QV4::ExecutableCompilationUnit> module; - - QFile f(url.toLocalFile()); - if (f.open(QIODevice::ReadOnly)) { - QByteArray content = harnessForModules + f.readAll(); - module = vm.compileModule(url.toString(), QString::fromUtf8(content.constData(), content.size()), QFileInfo(f).lastModified()); - if (vm.hasException) - break; - vm.injectCompiledModule(module); - } else { - vm.throwError(QStringLiteral("Could not load module")); - break; + for (TestData &testData : tasks) { + auto runTest = [&] (const char *header, TestCase::Result *result) { + QTemporaryFile tempFile; + tempFile.open(); + tempFile.write(header); + tempFile.write(testData.content); + tempFile.close(); + + QProcess process; + process.start(command, QStringList(tempFile.fileName())); + if (!process.waitForFinished(-1) || process.error() == QProcess::FailedToStart) { + qWarning() << "Could not execute" << command; + *result = TestCase::Result(TestCase::Crashes); } - - for (const QString &request: module->moduleRequests()) { - const QUrl absoluteRequest = module->finalUrl().resolved(QUrl(request)); - const auto module = vm.moduleForUrl(absoluteRequest); - if (module.native == nullptr && module.compiled == nullptr) - modulesToLoad << absoluteRequest; + if (process.exitStatus() != QProcess::NormalExit) { + *result = TestCase::Result(TestCase::Crashes); } - } - - if (!vm.hasException) { - const auto rootModule = vm.loadModule(rootModuleUrl); - if (rootModule.compiled && rootModule.compiled->instantiate(&vm)) - rootModule.compiled->evaluate(); - } - } else { - QV4::ScopedContext ctx(scope, vm.rootContext()); - - QV4::Script script(ctx, QV4::Compiler::ContextType::Global, testData); - script.parse(); - - if (!vm.hasException) - script.run(); - } - - if (vm.hasException) { - QV4::Scope scope(&vm); - QV4::ScopedValue val(scope, vm.catchException()); - return TestCase::Result(TestCase::Fails, val->toQString()); - } - return TestCase::Result(TestCase::Passes); -} - -class SingleTest : public QRunnable -{ -public: - SingleTest(Test262Runner *runner, const TestData &data) - : runner(runner), data(data) - { - command = runner->command; - } - void run() override; - - void runExternalTest(); - - QString command; - Test262Runner *runner; - TestData data; -}; - -void SingleTest::run() -{ - if (!command.isEmpty()) { - runExternalTest(); - return; - } - - if (data.runInSloppyMode) { - TestCase::Result ok = ::executeTest(data.content); - if (data.negative) - ok.negateResult(); - - data.sloppyResult = ok; - } else { - data.sloppyResult = TestCase::Result(TestCase::Skipped); - } - if (data.runInStrictMode) { - const QString testCasePath = QFileInfo(runner->testDir + "/test/" + data.test).absoluteFilePath(); - QByteArray c = "'use strict';\n" + data.content; - TestCase::Result ok = ::executeTest(c, data.runAsModuleCode, testCasePath, data.harness); - if (data.negative) - ok.negateResult(); - - data.strictResult = ok; - } else { - data.strictResult = TestCase::Result(TestCase::Skipped); - } - runner->addResult(data); -} - -void SingleTest::runExternalTest() -{ - auto runTest = [this] (const char *header, TestCase::Result *result) { - QTemporaryFile tempFile; - tempFile.open(); - tempFile.write(header); - tempFile.write(data.content); - tempFile.close(); - - QProcess process; -// if (flags & Verbose) -// process.setReadChannelMode(QProcess::ForwardedChannels); - - process.start(command, QStringList(tempFile.fileName())); - if (!process.waitForFinished(-1) || process.error() == QProcess::FailedToStart) { - qWarning() << "Could not execute" << command; - *result = TestCase::Result(TestCase::Crashes); - } - if (process.exitStatus() != QProcess::NormalExit) { - *result = TestCase::Result(TestCase::Crashes); - } - bool ok = (process.exitCode() == EXIT_SUCCESS); - if (data.negative) - ok = !ok; - *result = ok ? TestCase::Result(TestCase::Passes) - : TestCase::Result(TestCase::Fails, process.readAllStandardError()); - }; - - if (data.runInSloppyMode) - runTest("", &data.sloppyResult); - if (data.runInStrictMode) - runTest("'use strict';\n", &data.strictResult); - - runner->addResult(data); -} - -int Test262Runner::runSingleTest(TestCase testCase) -{ - TestData data = getTestData(testCase); -// qDebug() << "starting test" << data.test; + bool ok = (process.exitCode() == EXIT_SUCCESS); + if (testData.negative) + ok = !ok; + *result = ok ? TestCase::Result(TestCase::Passes) + : TestCase::Result(TestCase::Fails, process.readAllStandardError()); + }; - if (data.isExcluded || data.async) - return 0; + if (testData.runInSloppyMode) + runTest("", &testData.sloppyResult); + if (testData.runInStrictMode) + runTest("'use strict';\n", &testData.strictResult); - if (threadPool) { - SingleTest *test = new SingleTest(this, data); - threadPool->start(test); - return 0; + addResult(testData); } - SingleTest test(this, data); - test.run(); - return 0; } void Test262Runner::addResult(TestCase result) { { +#if !QT_CONFIG(process) QMutexLocker locker(&mutex); +#endif Q_ASSERT(result.strictExpectation.state == testCases[result.test].strictExpectation.state); Q_ASSERT(result.sloppyExpectation.state == testCases[result.test].sloppyExpectation.state); testCases[result.test] = result; @@ -852,3 +987,5 @@ QByteArray Test262Runner::harness(const QByteArray &name) harnessFiles.insert(name, content); return content; } + +QT_END_NAMESPACE diff --git a/tests/auto/qml/ecmascripttests/qjstest/test262runner.h b/tests/auto/qml/ecmascripttests/test262runner.h index e2bf26296f..53c66618f0 100644 --- a/tests/auto/qml/ecmascripttests/qjstest/test262runner.h +++ b/tests/auto/qml/ecmascripttests/test262runner.h @@ -1,14 +1,24 @@ // Copyright (C) 2016 The Qt Company Ltd. // SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0 + #ifndef TEST262RUNNER_H #define TEST262RUNNER_H -#include <qstring.h> -#include <qstringlist.h> -#include <qset.h> + +#include <qeventloop.h> #include <qmap.h> #include <qmutex.h> +#include <qprocess.h> +#include <qqueue.h> +#include <qset.h> #include <qthreadpool.h> +QT_BEGIN_NAMESPACE + +namespace QV4 { +struct ExecutionEngine; +void initD262(ExecutionEngine *e); +} + struct TestCase { TestCase() = default; TestCase(const QString &test) @@ -40,16 +50,18 @@ struct TestCase { } }; - bool skipTestCase = false; Result strictExpectation = Result(Passes); Result sloppyExpectation = Result(Passes); Result strictResult = Result(Skipped); Result sloppyResult = Result(Skipped); + bool skipTestCase = false; + bool stillNeedStrictRun = false; QString test; }; struct TestData : TestCase { + TestData() = default; TestData(const TestCase &testCase) : TestCase(testCase) {} // flags @@ -67,8 +79,12 @@ struct TestData : TestCase { QByteArray content; }; -class Test262Runner +class SingleTest; + +class Test262Runner : public QObject { + Q_OBJECT + public: Test262Runner(const QString &command, const QString &testDir, const QString &expectationsFile); ~Test262Runner(); @@ -95,6 +111,12 @@ public: bool run(); bool report(); + QString testDirectory() const { return testDir; } + + static void executeTest(QV4::ExecutionEngine &vm, const QString &testData, + const QString &testCasePath = QString(), + const QString &harnessForModules = QString(), + bool runAsModule = false); private: friend class SingleTest; @@ -102,7 +124,16 @@ private: void loadTestExpectations(); void updateTestExpectations(); void writeTestExpectations(); - int runSingleTest(TestCase testCase); + + void runWithThreadPool(); + + void runAsExternalTests(); + void createProcesses(); + void assignTaskOrTerminate(int processIndex); + void assignSloppy(int processIndex); + void assignStrict(int processIndex); + void sendDone(int processIndex); + QString readUntilNull(QProcess &p); TestData getTestData(const TestCase &testCase); void parseYaml(const QByteArray &content, TestData *data); @@ -116,14 +147,21 @@ private: QString expectationsFile; int flags = 0; - QMutex mutex; QString filter; QMap<QString, TestCase> testCases; QHash<QByteArray, QByteArray> harnessFiles; QThreadPool *threadPool = nullptr; + QMutex mutex; + + QEventLoop loop; + std::vector<std::unique_ptr<QProcess>> processes; + int runningCount = 0; + QQueue<TestData> tasks; + QHash<int, TestData> currentTasks; }; +QT_END_NAMESPACE #endif diff --git a/tests/auto/qml/ecmascripttests/tst_ecmascripttests.cpp b/tests/auto/qml/ecmascripttests/tst_ecmascripttests.cpp index e4a108a2e8..fc20c80a1a 100644 --- a/tests/auto/qml/ecmascripttests/tst_ecmascripttests.cpp +++ b/tests/auto/qml/ecmascripttests/tst_ecmascripttests.cpp @@ -1,11 +1,22 @@ // Copyright (C) 2017 The Qt Company Ltd. // SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0 -#include <QtTest/QtTest> -#include <QProcess> +#include <QFileInfo> +#include <QJSEngine> +#include <QJsonDocument> +#include <QJsonObject> #include <QLibraryInfo> -#include <qjstest/test262runner.h> +#include <QProcess> #include <QtQuickTestUtils/private/qmlutils_p.h> +#include <QtTest/QtTest> + +#include "test262runner.h" +#include "private/qqmlbuiltinfunctions_p.h" +#include "private/qv4arraybuffer_p.h" +#include "private/qv4globalobject_p.h" +#include "private/qv4script_p.h" + +#include <stdio.h> class tst_EcmaScriptTests : public QQmlDataTest { @@ -94,7 +105,74 @@ void tst_EcmaScriptTests::runJitted() QVERIFY(result); } -QTEST_GUILESS_MAIN(tst_EcmaScriptTests) +//// v RUNNER PROCESS MODE v //// -#include "tst_ecmascripttests.moc" +void readInput(bool &done, QString &mode, QString &testData, QString &testCasePath, + QString &harnessForModules, bool &runAsModule) +{ + QTextStream in(stdin); + QString input; + while (input.isEmpty()) + input = in.readLine(); + + QJsonDocument json = QJsonDocument::fromJson(input.toUtf8()); + done = json["done"].toBool(false); + mode = json["mode"].toString(); + testData = json["testData"].toString(); + testCasePath = json["testCasePath"].toString(); + harnessForModules = json["harnessForModules"].toString(); + runAsModule = json["runAsModule"].toBool(false); +} + +void printResult(QV4::ExecutionEngine &vm, const QString &mode) +{ + QJsonObject result; + result.insert("mode", mode); + if (vm.hasException) { + QV4::Scope scope(&vm); + QV4::ScopedValue val(scope, vm.catchException()); + + result.insert("resultState", int(TestCase::State::Fails)); + result.insert("resultErrorMessage", val->toQString()); + } else { + result.insert("resultState", int(TestCase::State::Passes)); + } + QTextStream(stdout) << QJsonDocument(result).toJson(QJsonDocument::Compact) << "\r\n"; +} + +void doRunnerProcess() +{ + bool done = false; + QString mode; + QString testData; + QString testCasePath; + QString harnessForModules; + bool runAsModule = false; + + while (!done) { + QV4::ExecutionEngine vm; + readInput(done, mode, testData, testCasePath, harnessForModules, runAsModule); + if (done) + break; + Test262Runner::executeTest(vm, testData, testCasePath, harnessForModules, runAsModule); + printResult(vm, mode); + } +} + +//// ^ RUNNER PROCESS MODE ^ //// + +int main(int argc, char *argv[]) +{ + QCoreApplication app(argc, argv); + + if (qEnvironmentVariableIntValue("runnerProcess") == 1) { + doRunnerProcess(); + } else { + tst_EcmaScriptTests tc; + QTEST_SET_MAIN_SOURCE_PATH + return QTest::qExec(&tc, argc, argv); + } +} + +#include "tst_ecmascripttests.moc" |