From d08ede57dd530a67c3420b3858fe39bf1e5eb598 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Olivier=20De=20Canni=C3=A8re?= Date: Mon, 11 Dec 2023 14:08:25 +0100 Subject: tst_ecmascript: Run tests on separate processes instead of threads Previously, tests were run in parallel on separate threads. This was faster than running them on only one but was still significantly slower than it could be, on Windows. This is due to them sharing the same heap and the fact that each memory allocation and free would temporarilly lock the heap for all other threads making the tests run much slower than on other platforms. This patch changes the way the test is run so that each js test file is run on a separate process. This ensures that the heap is no longer being shared by all test runners and reduces overhead significantly. The test runner processes listen for test data in JSON format over their standard input, run the test, return the results over their standard output and then wait for the next test data. tst_ecmascripttests on 13900k with 32 threads Debug MSVC Windows Debug GCC Linux threads: 569s 105s processes: 89s (~ -84%) 52s (~ -50%) On platforms where QT_CONFIG(process) returns false, the tests fallback to running on threads as before. Change-Id: Id51fc9d6e0d5ef0ae5c88f96b0119aa99e57f0fe Reviewed-by: Qt CI Bot Reviewed-by: Fabian Kosmale --- tests/auto/qml/ecmascripttests/CMakeLists.txt | 6 +- .../qml/ecmascripttests/qjstest/CMakeLists.txt | 28 - tests/auto/qml/ecmascripttests/qjstest/main.cpp | 90 -- .../qml/ecmascripttests/qjstest/test262runner.cpp | 854 ------------------ .../qml/ecmascripttests/qjstest/test262runner.h | 129 --- tests/auto/qml/ecmascripttests/test262runner.cpp | 991 +++++++++++++++++++++ tests/auto/qml/ecmascripttests/test262runner.h | 167 ++++ .../qml/ecmascripttests/tst_ecmascripttests.cpp | 88 +- 8 files changed, 1242 insertions(+), 1111 deletions(-) delete mode 100644 tests/auto/qml/ecmascripttests/qjstest/CMakeLists.txt delete mode 100644 tests/auto/qml/ecmascripttests/qjstest/main.cpp delete mode 100644 tests/auto/qml/ecmascripttests/qjstest/test262runner.cpp delete mode 100644 tests/auto/qml/ecmascripttests/qjstest/test262runner.h create mode 100644 tests/auto/qml/ecmascripttests/test262runner.cpp create mode 100644 tests/auto/qml/ecmascripttests/test262runner.h (limited to 'tests/auto/qml/ecmascripttests') diff --git a/tests/auto/qml/ecmascripttests/CMakeLists.txt b/tests/auto/qml/ecmascripttests/CMakeLists.txt index e03f5ffa82..1ee70cb101 100644 --- a/tests/auto/qml/ecmascripttests/CMakeLists.txt +++ b/tests/auto/qml/ecmascripttests/CMakeLists.txt @@ -20,7 +20,7 @@ list(FILTER test_data EXCLUDE REGEX ".git") qt_internal_add_test(tst_ecmascripttests SOURCES - qjstest/test262runner.cpp qjstest/test262runner.h + test262runner.cpp test262runner.h tst_ecmascripttests.cpp LIBRARIES Qt::QmlPrivate @@ -47,7 +47,3 @@ else() QT_QMLTEST_DATADIR="${CMAKE_CURRENT_SOURCE_DIR}/test262" ) endif() - -if(NOT CMAKE_CROSSCOMPILING) - add_subdirectory(qjstest) -endif() diff --git a/tests/auto/qml/ecmascripttests/qjstest/CMakeLists.txt b/tests/auto/qml/ecmascripttests/qjstest/CMakeLists.txt deleted file mode 100644 index 86ca5f97a3..0000000000 --- a/tests/auto/qml/ecmascripttests/qjstest/CMakeLists.txt +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright (C) 2022 The Qt Company Ltd. -# SPDX-License-Identifier: BSD-3-Clause - -# Generated from qjstest.pro. - -##################################################################### -## qjstest Tool: -##################################################################### - -qt_get_tool_target_name(target_name qjstest) -qt_internal_add_tool(${target_name} - TARGET_DESCRIPTION "Javascript test runner" - SOURCES - main.cpp - test262runner.cpp test262runner.h - DEFINES - QT_DEPRECATED_WARNINGS - INCLUDE_DIRECTORIES - . - LIBRARIES - Qt::Gui - Qt::QmlPrivate -) -qt_internal_return_unless_building_tools() - -#### Keys ignored in scope 1:.:.:qjstest.pro:: -# QMAKE_TARGET_DESCRIPTION = "Javascript" "test" "runner" -# TEMPLATE = "app" diff --git a/tests/auto/qml/ecmascripttests/qjstest/main.cpp b/tests/auto/qml/ecmascripttests/qjstest/main.cpp deleted file mode 100644 index 7bffedae81..0000000000 --- a/tests/auto/qml/ecmascripttests/qjstest/main.cpp +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright (C) 2016 The Qt Company Ltd. -// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0 -#include -#include -#include -#include -#include - -#include "test262runner.h" - -int main(int argc, char **argv) -{ - QCoreApplication app(argc, argv); - - - QCommandLineParser parser; - parser.addHelpOption(); - parser.addVersionOption(); - QCommandLineOption verbose("verbose", "Verbose output"); - parser.addOption(verbose); - QCommandLineOption commandOption("command", "Javascript command line interpreter", "command"); - parser.addOption(commandOption); - QCommandLineOption testDir("tests", "path to the tests", "tests", "test262"); - parser.addOption(testDir); - QCommandLineOption cat("cat", "Print packaged test code that would be run"); - parser.addOption(cat); - QCommandLineOption parallel("parallel", "Run tests in parallel"); - parser.addOption(parallel); - QCommandLineOption jit("jit", "JIT all code"); - parser.addOption(jit); - QCommandLineOption bytecode("interpret", "Run using the bytecode interpreter"); - parser.addOption(bytecode); - QCommandLineOption withExpectations("with-test-expectations", "Parse TestExpectations to deal with known failures"); - parser.addOption(withExpectations); - QCommandLineOption updateExpectations("update-expectations", "Update TestExpectations to remove unexepected passes"); - parser.addOption(updateExpectations); - QCommandLineOption writeExpectations("write-expectations", "Generate a new TestExpectations file based on the results of the run"); - parser.addOption(writeExpectations); - parser.addPositionalArgument("[filter]", "Only run tests that contain filter in their name"); - - parser.process(app); - - Test262Runner testRunner(parser.value(commandOption), parser.value(testDir), QStringLiteral("TestExpectations")); - - QStringList otherArgs = parser.positionalArguments(); - if (otherArgs.size() > 1) { - qWarning() << "too many arguments"; - return 1; - } else if (otherArgs.size()) { - testRunner.setFilter(otherArgs.at(0)); - } - - if (parser.isSet(cat)) { - testRunner.cat(); - return 0; - } - - if (parser.isSet(updateExpectations) && parser.isSet(writeExpectations)) { - qWarning() << "Can only specify one of --update-expectations and --write-expectations."; - exit(1); - } - - if (parser.isSet(jit) && parser.isSet(bytecode)) { - qWarning() << "Can only specify one of --jit and --interpret."; - exit(1); - } - - int flags = 0; - if (parser.isSet(verbose)) - - flags |= Test262Runner::Verbose; - if (parser.isSet(parallel)) - flags |= Test262Runner::Parallel; - if (parser.isSet(jit)) - flags |= Test262Runner::ForceJIT; - if (parser.isSet(bytecode)) - flags |= Test262Runner::ForceBytecode; - if (parser.isSet(withExpectations)) - flags |= Test262Runner::WithTestExpectations; - if (parser.isSet(updateExpectations)) - flags |= Test262Runner::UpdateTestExpectations; - if (parser.isSet(writeExpectations)) - flags |= Test262Runner::WriteTestExpectations; - testRunner.setFlags(flags); - - if (testRunner.run()) - return EXIT_SUCCESS; - else - return EXIT_FAILURE; -} diff --git a/tests/auto/qml/ecmascripttests/qjstest/test262runner.cpp b/tests/auto/qml/ecmascripttests/qjstest/test262runner.cpp deleted file mode 100644 index f7c1a21c74..0000000000 --- a/tests/auto/qml/ecmascripttests/qjstest/test262runner.cpp +++ /dev/null @@ -1,854 +0,0 @@ -// Copyright (C) 2016 The Qt Company Ltd. -// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0 - -#include "test262runner.h" - -#include -#include -#include -#include -#include -#include - -#include -#include "private/qv4globalobject_p.h" -#include "private/qqmlbuiltinfunctions_p.h" -#include "private/qv4arraybuffer_p.h" -#include - -#include "qrunnable.h" - -static const char *excludedFeatures[] = { - "BigInt", - "class-fields-public", - "class-fields-private", - "Promise.prototype.finally", - "async-iteration", - "Symbol.asyncIterator", - "object-rest", - "object-spread", - "optional-catch-binding", - "regexp-dotall", - "regexp-lookbehind", - "regexp-named-groups", - "regexp-unicode-property-escapes", - "Atomics", - "SharedArrayBuffer", - "Array.prototype.flatten", - "Array.prototype.flatMap", - "string-trimming", - "String.prototype.trimEnd", - "String.prototype.trimStart", - "numeric-separator-literal", - - // optional features, not supported by us - "caller", - nullptr -}; - -static const char *excludedFilePatterns[] = { - "realm", - nullptr -}; - -QT_BEGIN_NAMESPACE - -namespace QV4 { - -static ReturnedValue method_detachArrayBuffer(const FunctionObject *f, const Value *, const Value *argv, int argc) -{ - Scope scope(f); - if (!argc) - return scope.engine->throwTypeError(); - Scoped a(scope, argv[0]); - if (!a) - return scope.engine->throwTypeError(); - - if (a->hasSharedArrayData()) - return scope.engine->throwTypeError(); - - a->d()->detachArrayData(); - - return Encode::null(); -} - -static void initD262(ExecutionEngine *e) -{ - Scope scope(e); - ScopedObject d262(scope, e->newObject()); - - d262->defineDefaultProperty(QStringLiteral("detachArrayBuffer"), method_detachArrayBuffer, 1); - - ScopedString s(scope, e->newString(QStringLiteral("$262"))); - e->globalObject->put(s, d262); -} - -} - -QT_END_NAMESPACE - -Q_DECLARE_LOGGING_CATEGORY(lcJsTest); -Q_LOGGING_CATEGORY(lcJsTest, "qt.v4.ecma262.tests", QtWarningMsg); - -Test262Runner::Test262Runner(const QString &command, const QString &dir, const QString &expectationsFile) - : command(command), testDir(dir), expectationsFile(expectationsFile) -{ - if (testDir.endsWith(QLatin1Char('/'))) - testDir = testDir.chopped(1); -} - -Test262Runner::~Test262Runner() -{ - delete threadPool; -} - -void Test262Runner::cat() -{ - if (!loadTests()) - return; - - if (testCases.size() != 1) - qWarning() << "test262 --cat: Ambiguous test case, using" << testCases.begin().key(); - TestData data = getTestData(testCases.begin().value()); - printf("%s", data.content.constData()); -} - -bool Test262Runner::run() -{ - if (!loadTests()) - return false; - - if (flags & Parallel) { - threadPool = new QThreadPool; - threadPool->setStackSize(16*1024*1024); - if (flags & Verbose) - qDebug() << "Running in parallel with" << QThread::idealThreadCount() << "threads."; - } - - if (flags & ForceJIT) - qputenv("QV4_JIT_CALL_THRESHOLD", QByteArray("0")); - else if (flags & ForceBytecode) - qputenv("QV4_FORCE_INTERPRETER", QByteArray("1")); - - if (flags & WithTestExpectations) - loadTestExpectations(); - - for (auto it = testCases.constBegin(); it != testCases.constEnd(); ++it) { - auto c = it.value(); - if (!c.skipTestCase) { - int result = runSingleTest(c); - if (result == -2) - return false; - } - } - - if (threadPool) - while (!threadPool->waitForDone(10000)) { - if (!lcJsTest().isEnabled(QtDebugMsg)) { - // heartbeat, only needed when there is no other debug output - qDebug("test262: in progress..."); - } - } - - const bool testsOk = report(); - - if (flags & WriteTestExpectations) - writeTestExpectations(); - else if (flags & UpdateTestExpectations) - updateTestExpectations(); - - return testsOk; -} - -bool Test262Runner::report() -{ - qDebug() << "Test execution summary:"; - qDebug() << " Executed" << testCases.size() << "test cases."; - QStringList crashes; - QStringList unexpectedFailures; - QStringList unexpectedPasses; - for (auto it = testCases.constBegin(); it != testCases.constEnd(); ++it) { - const auto c = it.value(); - if (c.strictResult.state == c.strictExpectation.state - && c.sloppyResult.state == c.sloppyExpectation.state) - continue; - auto report = [&](TestCase::Result expected, TestCase::Result result, const char *s) { - if (result.state == TestCase::Crashes) - crashes << (it.key() + " crashed in " + s + " mode"); - if (result.state == TestCase::Fails && expected.state == TestCase::Passes) - unexpectedFailures << (it.key() + " failed in " + s - + " mode with error message: " + result.errorMessage); - if (result.state == TestCase::Passes && expected.state == TestCase::Fails) - unexpectedPasses << (it.key() + " unexpectedly passed in " + s + " mode"); - }; - report(c.strictExpectation, c.strictResult, "strict"); - report(c.sloppyExpectation, c.sloppyResult, "sloppy"); - } - if (!crashes.isEmpty()) { - qDebug() << " Encountered" << crashes.size() << "crashes in the following files:"; - for (const QString &f : std::as_const(crashes)) - qDebug() << " " << f; - } - if (!unexpectedFailures.isEmpty()) { - qDebug() << " Encountered" << unexpectedFailures.size() << "unexpected failures in the following files:"; - for (const QString &f : std::as_const(unexpectedFailures)) - qDebug() << " " << f; - } - if (!unexpectedPasses.isEmpty()) { - qDebug() << " Encountered" << unexpectedPasses.size() << "unexpected passes in the following files:"; - for (const QString &f : std::as_const(unexpectedPasses)) - qDebug() << " " << f; - } - return crashes.isEmpty() && unexpectedFailures.isEmpty() && unexpectedPasses.isEmpty(); -} - -bool Test262Runner::loadTests() -{ - QDir dir(testDir + "/test"); - if (!dir.exists()) { - qWarning() << "Could not load tests," << dir.path() << "does not exist."; - return false; - } - - QString annexB = "annexB"; - QString harness = "harness"; - QString intl402 = "intl402"; - - int pathlen = dir.path().size() + 1; - QDirIterator it(dir, QDirIterator::Subdirectories); - while (it.hasNext()) { - QString file = it.next().mid(pathlen); - if (!file.endsWith(".js")) - continue; - if (file.endsWith("_FIXTURE.js")) - continue; - if (!filter.isEmpty() && !file.contains(filter)) - continue; - if (file.startsWith(annexB) || file.startsWith(harness) || file.startsWith(intl402)) - continue; - const char **excluded = excludedFilePatterns; - bool skip = false; - while (*excluded) { - if (file.contains(QLatin1String(*excluded))) - skip = true; - ++excluded; - } - if (skip) - continue; - - testCases.insert(file, TestCase{ file }); - } - if (testCases.isEmpty()) { - qWarning() << "No tests to run."; - return false; - } - - return true; -} - - -struct TestExpectationLine { - TestExpectationLine(const QByteArray &line); - enum State { - Fails, - SloppyFails, - StrictFails, - Skip, - Passes - } state; - QString testCase; - - QByteArray toLine() const; - void update(const TestCase &testCase); - - static TestExpectationLine fromTestCase(const TestCase &testCase); -private: - TestExpectationLine() = default; - static State stateFromTestCase(const TestCase &testCase); -}; - -TestExpectationLine::TestExpectationLine(const QByteArray &line) -{ - int space = line.indexOf(' '); - - testCase = QString::fromUtf8(space > 0 ? line.left(space) : line); - if (!testCase.endsWith(".js")) - testCase += ".js"; - - state = Fails; - if (space < 0) - return; - QByteArray qualifier = line.mid(space + 1); - if (qualifier == "skip") - state = Skip; - else if (qualifier == "strictFails") - state = StrictFails; - else if (qualifier == "sloppyFails") - state = SloppyFails; - else if (qualifier == "fails") - state = Fails; - else - qWarning() << "illegal format in TestExpectations, line" << line; -} - -QByteArray TestExpectationLine::toLine() const { - const char *res = nullptr; - switch (state) { - case Fails: - res = " fails\n"; - break; - case SloppyFails: - res = " sloppyFails\n"; - break; - case StrictFails: - res = " strictFails\n"; - break; - case Skip: - res = " skip\n"; - break; - case Passes: - // no need for an entry - return QByteArray(); - } - QByteArray result = testCase.toUtf8() + res; - return result; -} - -void TestExpectationLine::update(const TestCase &testCase) -{ - Q_ASSERT(testCase.test == this->testCase); - - State resultState = stateFromTestCase(testCase); - switch (resultState) { - case Fails: - // no improvement, don't update - break; - case SloppyFails: - if (state == Fails) - state = SloppyFails; - else if (state == StrictFails) - // we have a regression in sloppy mode, but strict now passes - state = Passes; - break; - case StrictFails: - if (state == Fails) - state = StrictFails; - else if (state == SloppyFails) - // we have a regression in strict mode, but sloppy now passes - state = Passes; - break; - case Skip: - Q_ASSERT(state == Skip); - // nothing to do - break; - case Passes: - state = Passes; - } -} - -TestExpectationLine TestExpectationLine::fromTestCase(const TestCase &testCase) -{ - TestExpectationLine l; - l.testCase = testCase.test; - l.state = stateFromTestCase(testCase); - return l; -} - -TestExpectationLine::State TestExpectationLine::stateFromTestCase(const TestCase &testCase) -{ - // keep skipped tests - if (testCase.skipTestCase) - return Skip; - - bool strictFails = (testCase.strictResult.state == TestCase::Crashes - || testCase.strictResult.state == TestCase::Fails); - bool sloppyFails = (testCase.sloppyResult.state == TestCase::Crashes - || testCase.sloppyResult.state == TestCase::Fails); - if (strictFails && sloppyFails) - return Fails; - if (strictFails) - return StrictFails; - if (sloppyFails) - return SloppyFails; - return Passes; -} - - -void Test262Runner::loadTestExpectations() -{ - QFile file(expectationsFile); - if (!file.open(QFile::ReadOnly)) { - qWarning() << "Could not open TestExpectations file at" << expectationsFile; - return; - } - - while (!file.atEnd()) { - QByteArray line = file.readLine().trimmed(); - if (line.startsWith('#') || line.isEmpty()) - continue; - TestExpectationLine expectation(line); - if (!filter.isEmpty() && !expectation.testCase.contains(filter)) - continue; - - if (!testCases.contains(expectation.testCase)) - qWarning() << "Unknown test case" << expectation.testCase << "in TestExpectations file."; - //qDebug() << "TestExpectations:" << expectation.testCase << expectation.state; - TestCase &s = testCases[expectation.testCase]; - switch (expectation.state) { - case TestExpectationLine::Fails: - s.strictExpectation.state = TestCase::Fails; - s.sloppyExpectation.state = TestCase::Fails; - break; - case TestExpectationLine::SloppyFails: - s.strictExpectation.state = TestCase::Passes; - s.sloppyExpectation.state = TestCase::Fails; - break; - case TestExpectationLine::StrictFails: - s.strictExpectation.state = TestCase::Fails; - s.sloppyExpectation.state = TestCase::Passes; - break; - case TestExpectationLine::Skip: - s.skipTestCase = true; - break; - case TestExpectationLine::Passes: - Q_UNREACHABLE(); - } - } -} - -void Test262Runner::updateTestExpectations() -{ - QFile file(expectationsFile); - if (!file.open(QFile::ReadOnly)) { - qWarning() << "Could not open TestExpectations file at" << expectationsFile; - return; - } - - QTemporaryFile updatedExpectations; - updatedExpectations.open(); - - while (!file.atEnd()) { - QByteArray originalLine = file.readLine(); - QByteArray line = originalLine.trimmed(); - if (line.startsWith('#') || line.isEmpty()) { - updatedExpectations.write(originalLine); - continue; - } - - TestExpectationLine expectation(line); -// qDebug() << "checking: " << expectation.testCase; - if (!testCases.contains(expectation.testCase)) { - updatedExpectations.write(originalLine); - continue; - } - const TestCase &testcase = testCases.value(expectation.testCase); - expectation.update(testcase); - - line = expectation.toLine(); -// qDebug() << "updated line:" << line; - updatedExpectations.write(line); - } - file.close(); - updatedExpectations.close(); - if (!file.remove()) - qWarning() << "Could not remove old TestExpectations file at" << expectationsFile; - if (updatedExpectations.copy(file.fileName())) - qDebug() << "Updated TestExpectations file written!"; - else - qWarning() << "Could not write new TestExpectations file at" << expectationsFile; -} - -void Test262Runner::writeTestExpectations() -{ - QFile file(expectationsFile); - - QTemporaryFile expectations; - expectations.open(); - - for (auto c : std::as_const(testCases)) { - TestExpectationLine line = TestExpectationLine::fromTestCase(c); - expectations.write(line.toLine()); - } - - expectations.close(); - if (file.exists() && !file.remove()) - qWarning() << "Could not remove old TestExpectations file at" << expectationsFile; - if (expectations.copy(file.fileName())) - qDebug() << "new TestExpectations file written!"; - else - qWarning() << "Could not write new TestExpectations file at" << expectationsFile; -} - -static TestCase::Result executeTest(const QByteArray &data, bool runAsModule = false, - const QString &testCasePath = QString(), - const QByteArray &harnessForModules = QByteArray()) -{ - QString testData = QString::fromUtf8(data.constData(), data.size()); - - QV4::ExecutionEngine vm; - - QV4::Scope scope(&vm); - - QV4::GlobalExtensions::init(vm.globalObject, QJSEngine::ConsoleExtension | QJSEngine::GarbageCollectionExtension); - QV4::initD262(&vm); - - if (runAsModule) { - const QUrl rootModuleUrl = QUrl::fromLocalFile(testCasePath); - // inject all modules with the harness - QVector modulesToLoad = { rootModuleUrl }; - while (!modulesToLoad.isEmpty()) { - QUrl url = modulesToLoad.takeFirst(); - QQmlRefPointer module; - - QFile f(url.toLocalFile()); - if (f.open(QIODevice::ReadOnly)) { - QByteArray content = harnessForModules + f.readAll(); - module = vm.compileModule(url.toString(), QString::fromUtf8(content.constData(), content.size()), QFileInfo(f).lastModified()); - if (vm.hasException) - break; - vm.injectCompiledModule(module); - } else { - vm.throwError(QStringLiteral("Could not load module")); - break; - } - - for (const QString &request: module->moduleRequests()) { - const QUrl absoluteRequest = module->finalUrl().resolved(QUrl(request)); - const auto module = vm.moduleForUrl(absoluteRequest); - if (module.native == nullptr && module.compiled == nullptr) - modulesToLoad << absoluteRequest; - } - } - - if (!vm.hasException) { - const auto rootModule = vm.loadModule(rootModuleUrl); - if (rootModule.compiled && rootModule.compiled->instantiate(&vm)) - rootModule.compiled->evaluate(); - } - } else { - QV4::ScopedContext ctx(scope, vm.rootContext()); - - QV4::Script script(ctx, QV4::Compiler::ContextType::Global, testData); - script.parse(); - - if (!vm.hasException) - script.run(); - } - - if (vm.hasException) { - QV4::Scope scope(&vm); - QV4::ScopedValue val(scope, vm.catchException()); - return TestCase::Result(TestCase::Fails, val->toQString()); - } - return TestCase::Result(TestCase::Passes); -} - -class SingleTest : public QRunnable -{ -public: - SingleTest(Test262Runner *runner, const TestData &data) - : runner(runner), data(data) - { - command = runner->command; - } - void run() override; - - void runExternalTest(); - - QString command; - Test262Runner *runner; - TestData data; -}; - -void SingleTest::run() -{ - if (!command.isEmpty()) { - runExternalTest(); - return; - } - - if (data.runInSloppyMode) { - TestCase::Result ok = ::executeTest(data.content); - if (data.negative) - ok.negateResult(); - - data.sloppyResult = ok; - } else { - data.sloppyResult = TestCase::Result(TestCase::Skipped); - } - if (data.runInStrictMode) { - const QString testCasePath = QFileInfo(runner->testDir + "/test/" + data.test).absoluteFilePath(); - QByteArray c = "'use strict';\n" + data.content; - TestCase::Result ok = ::executeTest(c, data.runAsModuleCode, testCasePath, data.harness); - if (data.negative) - ok.negateResult(); - - data.strictResult = ok; - } else { - data.strictResult = TestCase::Result(TestCase::Skipped); - } - runner->addResult(data); -} - -void SingleTest::runExternalTest() -{ - auto runTest = [this] (const char *header, TestCase::Result *result) { - QTemporaryFile tempFile; - tempFile.open(); - tempFile.write(header); - tempFile.write(data.content); - tempFile.close(); - - QProcess process; -// if (flags & Verbose) -// process.setReadChannelMode(QProcess::ForwardedChannels); - - process.start(command, QStringList(tempFile.fileName())); - if (!process.waitForFinished(-1) || process.error() == QProcess::FailedToStart) { - qWarning() << "Could not execute" << command; - *result = TestCase::Result(TestCase::Crashes); - } - if (process.exitStatus() != QProcess::NormalExit) { - *result = TestCase::Result(TestCase::Crashes); - } - bool ok = (process.exitCode() == EXIT_SUCCESS); - if (data.negative) - ok = !ok; - *result = ok ? TestCase::Result(TestCase::Passes) - : TestCase::Result(TestCase::Fails, process.readAllStandardError()); - }; - - if (data.runInSloppyMode) - runTest("", &data.sloppyResult); - if (data.runInStrictMode) - runTest("'use strict';\n", &data.strictResult); - - runner->addResult(data); -} - -int Test262Runner::runSingleTest(TestCase testCase) -{ - TestData data = getTestData(testCase); -// qDebug() << "starting test" << data.test; - - if (data.isExcluded || data.async) - return 0; - - if (threadPool) { - SingleTest *test = new SingleTest(this, data); - threadPool->start(test); - return 0; - } - SingleTest test(this, data); - test.run(); - return 0; -} - -void Test262Runner::addResult(TestCase result) -{ - { - QMutexLocker locker(&mutex); - Q_ASSERT(result.strictExpectation.state == testCases[result.test].strictExpectation.state); - Q_ASSERT(result.sloppyExpectation.state == testCases[result.test].sloppyExpectation.state); - testCases[result.test] = result; - } - - if (!(flags & Verbose)) - return; - - QString test = result.test; - if (result.strictResult.state == TestCase::Skipped) { - ; - } else if (result.strictResult.state == TestCase::Crashes) { - qDebug() << "FAIL:" << test << "crashed in strict mode!"; - } else if (result.strictResult.state == TestCase::Fails - && result.strictExpectation.state == TestCase::Fails) { - qCDebug(lcJsTest) << "PASS:" << test << "failed in strict mode as expected"; - } else if ((result.strictResult.state == TestCase::Passes) - == (result.strictExpectation.state == TestCase::Passes)) { - qCDebug(lcJsTest) << "PASS:" << test << "passed in strict mode"; - } else if (!(result.strictExpectation.state == TestCase::Fails)) { - qDebug() << "FAIL:" << test << "failed in strict mode with error message:\n" - << result.strictResult.errorMessage; - } else { - qDebug() << "XPASS:" << test << "unexpectedly passed in strict mode"; - } - - if (result.sloppyResult.state == TestCase::Skipped) { - ; - } else if (result.sloppyResult.state == TestCase::Crashes) { - qDebug() << "FAIL:" << test << "crashed in sloppy mode!"; - } else if (result.sloppyResult.state == TestCase::Fails - && result.sloppyExpectation.state == TestCase::Fails) { - qCDebug(lcJsTest) << "PASS:" << test << "failed in sloppy mode as expected"; - } else if ((result.sloppyResult.state == TestCase::Passes) - == (result.sloppyExpectation.state == TestCase::Passes)) { - qCDebug(lcJsTest) << "PASS:" << test << "passed in sloppy mode"; - } else if (!(result.sloppyExpectation.state == TestCase::Fails)) { - qDebug() << "FAIL:" << test << "failed in sloppy mode with error message:\n" - << result.sloppyResult.errorMessage; - } else { - qDebug() << "XPASS:" << test << "unexpectedly passed in sloppy mode"; - } -} - -TestData Test262Runner::getTestData(const TestCase &testCase) -{ - QFile testFile(testDir + "/test/" + testCase.test); - if (!testFile.open(QFile::ReadOnly)) { - qWarning() << "wrong test file" << testCase.test; - exit(1); - } - QByteArray content = testFile.readAll(); - content.replace(QByteArrayLiteral("\r\n"), "\n"); - - qCDebug(lcJsTest) << "parsing test file" << testCase.test; - - TestData data(testCase); - parseYaml(content, &data); - - data.harness += harness("assert.js"); - data.harness += harness("sta.js"); - - for (QByteArray inc : std::as_const(data.includes)) { - inc = inc.trimmed(); - data.harness += harness(inc); - } - - if (data.async) - data.harness += harness("doneprintHandle.js"); - - data.content = data.harness + content; - - return data; -} - -struct YamlSection { - YamlSection(const QByteArray &yaml, const char *sectionName); - - bool contains(const char *keyword) const; - QList keywords() const; - - QByteArray yaml; - int start = -1; - int length = 0; - bool shortSection = false; -}; - -YamlSection::YamlSection(const QByteArray &yaml, const char *sectionName) - : yaml(yaml) -{ - start = yaml.indexOf(sectionName); - if (start < 0) - return; - start += static_cast(strlen(sectionName)); - int end = yaml.indexOf('\n', start + 1); - if (end < 0) - end = yaml.size(); - - int s = yaml.indexOf('[', start); - if (s > 0 && s < end) { - shortSection = true; - start = s + 1; - end = yaml.indexOf(']', s); - } else { - while (end < yaml.size() - 1 && yaml.at(end + 1) == ' ') - end = yaml.indexOf('\n', end + 1); - } - length = end - start; -} - -bool YamlSection::contains(const char *keyword) const -{ - if (start < 0) - return false; - int idx = yaml.indexOf(keyword, start); - if (idx >= start && idx < start + length) - return true; - return false; -} - -QList YamlSection::keywords() const -{ - if (start < 0) - return QList(); - - QByteArray content = yaml.mid(start, length); - QList keywords; - if (shortSection) { - keywords = content.split(','); - } else { - const QList list = content.split('\n'); - for (const QByteArray &l : list) { - int i = 0; - while (i < l.size() && (l.at(i) == ' ' || l.at(i) == '-')) - ++i; - QByteArray entry = l.mid(i); - if (!entry.isEmpty()) - keywords.append(entry); - } - } -// qDebug() << "keywords:" << keywords; - return keywords; -} - - -void Test262Runner::parseYaml(const QByteArray &content, TestData *data) -{ - int start = content.indexOf("/*---"); - if (start < 0) - return; - start += sizeof("/*---"); - - int end = content.indexOf("---*/"); - if (end < 0) - return; - - QByteArray yaml = content.mid(start, end - start); - - if (yaml.contains("negative:")) - data->negative = true; - - YamlSection flags(yaml, "flags:"); - data->runInSloppyMode = !flags.contains("onlyStrict"); - data->runInStrictMode = !flags.contains("noStrict") && !flags.contains("raw"); - data->runAsModuleCode = flags.contains("module"); - data->async = flags.contains("async"); - - if (data->runAsModuleCode) { - data->runInStrictMode = true; - data->runInSloppyMode = false; - } - - YamlSection includes(yaml, "includes:"); - data->includes = includes.keywords(); - - YamlSection features = YamlSection(yaml, "features:"); - - const char **f = excludedFeatures; - while (*f) { - if (features.contains(*f)) { - data->isExcluded = true; - break; - } - ++f; - } - -// qDebug() << "Yaml:\n" << yaml; -} - -QByteArray Test262Runner::harness(const QByteArray &name) -{ - if (harnessFiles.contains(name)) - return harnessFiles.value(name); - - QFile h(testDir + QLatin1String("/harness/") + name); - if (!h.open(QFile::ReadOnly)) { - qWarning() << "Illegal test harness file" << name; - exit(1); - } - - QByteArray content = h.readAll(); - harnessFiles.insert(name, content); - return content; -} diff --git a/tests/auto/qml/ecmascripttests/qjstest/test262runner.h b/tests/auto/qml/ecmascripttests/qjstest/test262runner.h deleted file mode 100644 index e2bf26296f..0000000000 --- a/tests/auto/qml/ecmascripttests/qjstest/test262runner.h +++ /dev/null @@ -1,129 +0,0 @@ -// Copyright (C) 2016 The Qt Company Ltd. -// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0 -#ifndef TEST262RUNNER_H -#define TEST262RUNNER_H -#include -#include -#include -#include -#include -#include - -struct TestCase { - TestCase() = default; - TestCase(const QString &test) - : test(test) {} - - enum State { Skipped, Passes, Fails, Crashes }; - - struct Result - { - State state; - QString errorMessage; - - Result(State state, QString errorMessage = "") - : state(state), errorMessage(errorMessage) { } - - void negateResult() - { - switch (state) { - case TestCase::Passes: - state = TestCase::Fails; - break; - case TestCase::Fails: - state = TestCase::Passes; - break; - case TestCase::Skipped: - case TestCase::Crashes: - break; - } - } - }; - - bool skipTestCase = false; - Result strictExpectation = Result(Passes); - Result sloppyExpectation = Result(Passes); - Result strictResult = Result(Skipped); - Result sloppyResult = Result(Skipped); - - QString test; -}; - -struct TestData : TestCase { - TestData(const TestCase &testCase) - : TestCase(testCase) {} - // flags - bool negative = false; - bool runInStrictMode = true; - bool runInSloppyMode = true; - bool runAsModuleCode = false; - bool async = false; - - bool isExcluded = false; - - QList includes; - - QByteArray harness; - QByteArray content; -}; - -class Test262Runner -{ -public: - Test262Runner(const QString &command, const QString &testDir, const QString &expectationsFile); - ~Test262Runner(); - - enum Mode { - Sloppy = 0, - Strict = 1 - }; - - enum Flags { - Verbose = 0x1, - Parallel = 0x2, - ForceBytecode = 0x4, - ForceJIT = 0x8, - WithTestExpectations = 0x10, - UpdateTestExpectations = 0x20, - WriteTestExpectations = 0x40, - }; - void setFlags(int f) { flags = f; } - - void setFilter(const QString &f) { filter = f; } - - void cat(); - bool run(); - - bool report(); - -private: - friend class SingleTest; - bool loadTests(); - void loadTestExpectations(); - void updateTestExpectations(); - void writeTestExpectations(); - int runSingleTest(TestCase testCase); - - TestData getTestData(const TestCase &testCase); - void parseYaml(const QByteArray &content, TestData *data); - - QByteArray harness(const QByteArray &name); - - void addResult(TestCase result); - - QString command; - QString testDir; - QString expectationsFile; - int flags = 0; - - QMutex mutex; - QString filter; - - QMap testCases; - QHash harnessFiles; - - QThreadPool *threadPool = nullptr; -}; - - -#endif diff --git a/tests/auto/qml/ecmascripttests/test262runner.cpp b/tests/auto/qml/ecmascripttests/test262runner.cpp new file mode 100644 index 0000000000..4c39dd661f --- /dev/null +++ b/tests/auto/qml/ecmascripttests/test262runner.cpp @@ -0,0 +1,991 @@ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0 + +#include "test262runner.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "private/qqmlbuiltinfunctions_p.h" +#include "private/qv4arraybuffer_p.h" +#include "private/qv4globalobject_p.h" +#include +#include + +using namespace Qt::StringLiterals; + +static const char *excludedFeatures[] = { + "BigInt", + "class-fields-public", + "class-fields-private", + "Promise.prototype.finally", + "async-iteration", + "Symbol.asyncIterator", + "object-rest", + "object-spread", + "optional-catch-binding", + "regexp-dotall", + "regexp-lookbehind", + "regexp-named-groups", + "regexp-unicode-property-escapes", + "Atomics", + "SharedArrayBuffer", + "Array.prototype.flatten", + "Array.prototype.flatMap", + "string-trimming", + "String.prototype.trimEnd", + "String.prototype.trimStart", + "numeric-separator-literal", + + // optional features, not supported by us + "caller", + nullptr +}; + +static const char *excludedFilePatterns[] = { + "realm", + nullptr +}; + +QT_BEGIN_NAMESPACE + +namespace QV4 { + +static ReturnedValue method_detachArrayBuffer(const FunctionObject *f, const Value *, const Value *argv, int argc) +{ + Scope scope(f); + if (!argc) + return scope.engine->throwTypeError(); + Scoped a(scope, argv[0]); + if (!a) + return scope.engine->throwTypeError(); + + if (a->hasSharedArrayData()) + return scope.engine->throwTypeError(); + + a->d()->detachArrayData(); + + return Encode::null(); +} + +void initD262(ExecutionEngine *e) +{ + Scope scope(e); + ScopedObject d262(scope, e->newObject()); + + d262->defineDefaultProperty(QStringLiteral("detachArrayBuffer"), method_detachArrayBuffer, 1); + + ScopedString s(scope, e->newString(QStringLiteral("$262"))); + e->globalObject->put(s, d262); +} + +} + + +Q_DECLARE_LOGGING_CATEGORY(lcJsTest); +Q_LOGGING_CATEGORY(lcJsTest, "qt.v4.ecma262.tests", QtWarningMsg); + +Test262Runner::Test262Runner(const QString &command, const QString &dir, const QString &expectationsFile) + : command(command), testDir(dir), expectationsFile(expectationsFile) +{ + if (testDir.endsWith(QLatin1Char('/'))) + testDir = testDir.chopped(1); +} + +Test262Runner::~Test262Runner() +{ + if (threadPool) + delete threadPool; +} + +void Test262Runner::cat() +{ + if (!loadTests()) + return; + + if (testCases.size() != 1) + qWarning() << "test262 --cat: Ambiguous test case, using" << testCases.begin().key(); + TestData data = getTestData(testCases.begin().value()); + printf("%s", data.content.constData()); +} + +void Test262Runner::assignTaskOrTerminate(int processIndex) +{ + if (tasks.isEmpty()) { + sendDone(processIndex); + return; + } + + currentTasks[processIndex] = tasks.dequeue(); + TestData &task = currentTasks[processIndex]; + + // Sloppy run + maybe strict run later + if (task.runInSloppyMode) { + if (task.runInStrictMode) + task.stillNeedStrictRun = true; + assignSloppy(processIndex); + return; + } + + // Only strict run + if (task.runInStrictMode) { + assignStrict(processIndex); + return; + } + + // TODO: Start a timer for timeouts? +} + +void Test262Runner::assignSloppy(int processIndex) +{ + QProcess &p = *processes[processIndex]; + TestData &task = currentTasks[processIndex]; + + QJsonObject json; + json.insert("mode", "sloppy"); + json.insert("testData", QString::fromUtf8(task.content)); + json.insert("runAsModule", false); + json.insert("testCasePath", ""); + json.insert("harnessForModules", ""); + p.write(QJsonDocument(json).toJson(QJsonDocument::Compact)); + p.write("\r\n"); +} + +void Test262Runner::assignStrict(int processIndex) +{ + QProcess &p = *processes[processIndex]; + TestData &task = currentTasks[processIndex]; + + QJsonObject json; + json.insert("mode", "strict"); + QString strictContent = "'use strict';\n" + QString::fromUtf8(task.content); + json.insert("testData", strictContent); + json.insert("runAsModule", task.runAsModuleCode); + json.insert("testCasePath", QFileInfo(testDir + "/test/" + task.test).absoluteFilePath()); + json.insert("harnessForModules", QString::fromUtf8(task.harness)); + p.write(QJsonDocument(json).toJson(QJsonDocument::Compact)); + p.write("\r\n"); +} + +void Test262Runner::sendDone(int processIndex) +{ + QProcess &p = *processes[processIndex]; + + QJsonObject json; + json.insert("done", true); + p.write(QJsonDocument(json).toJson(QJsonDocument::Compact)); + p.write("\r\n"); +} + +void Test262Runner::createProcesses() +{ + const int processCount = QThread::idealThreadCount(); + qDebug() << "Running in parallel with" << processCount << "processes"; + for (int i = 0; i < processCount; ++i) { + processes.emplace_back(std::make_unique()); + QProcess &p = *processes[i]; + QProcess::connect(&p, &QProcess::started, this, [&, i]() { + assignTaskOrTerminate(i); + }); + + QProcess::connect(&p, &QIODevice::readyRead, this, [&, i]() { + QProcess &p = *processes[i]; + QString output; + while (output.isEmpty()) + output = p.readLine(); + QJsonDocument response = QJsonDocument::fromJson(output.toUtf8()); + + TestData &testData(currentTasks[i]); + auto mode = response["mode"].toString(); + auto state = TestCase::State(response["resultState"].toInt(int(TestCase::State::Fails))); + auto errorMessage = response["resultErrorMessage"].toString(); + + auto &result = mode == "strict" ? testData.strictResult : testData.sloppyResult; + result = TestCase::Result(state, errorMessage); + if (testData.negative) + result.negateResult(); + + if (testData.stillNeedStrictRun) { + testData.stillNeedStrictRun = false; + assignStrict(i); + } else { + addResult(testData); + assignTaskOrTerminate(i); + } + }); + + QObject::connect(&p, &QProcess::finished, this, [&, i](int, QProcess::ExitStatus status) { + if (status != QProcess::NormalExit) { + qDebug() << QStringLiteral("Process %1 of %2 exited with a non-normal status") + .arg(i).arg(processCount - 1); + } + + --runningCount; + if (runningCount == 0) + loop.exit(); + }); + + p.setProgram(QCoreApplication::applicationFilePath()); + QProcessEnvironment env = QProcessEnvironment::systemEnvironment(); + env.insert(u"runnerProcess"_s, u"1"_s); + p.setProcessEnvironment(env); + ++runningCount; + p.start(); + } +} + +class SingleTest : public QRunnable +{ +public: + SingleTest(Test262Runner *runner, const TestData &data) + : runner(runner), data(data) + {} + void run() override; + + Test262Runner *runner; + TestData data; +}; + +TestCase::Result getTestExecutionResult(QV4::ExecutionEngine &vm) +{ + TestCase::State state; + QString errorMessage; + if (vm.hasException) { + state = TestCase::State::Fails; + QV4::Scope scope(&vm); + QV4::ScopedValue val(scope, vm.catchException()); + errorMessage = val->toQString(); + } else { + state = TestCase::State::Passes; + } + return TestCase::Result(state, errorMessage); +} + +void SingleTest::run() +{ + if (data.runInSloppyMode) { + QV4::ExecutionEngine vm; + Test262Runner::executeTest(vm, data.content); + TestCase::Result ok = getTestExecutionResult(vm); + + if (data.negative) + ok.negateResult(); + + data.sloppyResult = ok; + } else { + data.sloppyResult = TestCase::Result(TestCase::Skipped); + } + if (data.runInStrictMode) { + QString testCasePath = QFileInfo(runner->testDirectory() + "/test/" + data.test).absoluteFilePath(); + QByteArray c = "'use strict';\n" + data.content; + + QV4::ExecutionEngine vm; + Test262Runner::executeTest(vm, c, testCasePath, data.harness, data.runAsModuleCode); + TestCase::Result ok = getTestExecutionResult(vm); + + if (data.negative) + ok.negateResult(); + + data.strictResult = ok; + } else { + data.strictResult = TestCase::Result(TestCase::Skipped); + } + runner->addResult(data); +} + +void Test262Runner::executeTest(QV4::ExecutionEngine &vm, const QString &testData, + const QString &testCasePath, const QString &harnessForModules, + bool runAsModule) +{ + QV4::Scope scope(&vm); + QV4::GlobalExtensions::init(vm.globalObject, + QJSEngine::ConsoleExtension | QJSEngine::GarbageCollectionExtension); + QV4::initD262(&vm); + + if (runAsModule) { + const QUrl rootModuleUrl = QUrl::fromLocalFile(testCasePath); + // inject all modules with the harness + QVector modulesToLoad = { rootModuleUrl }; + while (!modulesToLoad.isEmpty()) { + QUrl url = modulesToLoad.takeFirst(); + QQmlRefPointer module; + + QFile f(url.toLocalFile()); + if (f.open(QIODevice::ReadOnly)) { + QByteArray content = harnessForModules.toLocal8Bit() + f.readAll(); + module = vm.compileModule(url.toString(), + QString::fromUtf8(content.constData(),content.size()), + QFileInfo(f).lastModified()); + if (vm.hasException) + break; + vm.injectCompiledModule(module); + } else { + vm.throwError(QStringLiteral("Could not load module")); + break; + } + + for (const QString &request: module->moduleRequests()) { + const QUrl absoluteRequest = module->finalUrl().resolved(QUrl(request)); + const auto module = vm.moduleForUrl(absoluteRequest); + if (module.native == nullptr && module.compiled == nullptr) + modulesToLoad << absoluteRequest; + } + } + + if (!vm.hasException) { + const auto rootModule = vm.loadModule(rootModuleUrl); + if (rootModule.compiled && rootModule.compiled->instantiate(&vm)) + rootModule.compiled->evaluate(); + } + } else { + QV4::ScopedContext ctx(scope, vm.rootContext()); + + QV4::Script script(ctx, QV4::Compiler::ContextType::Global, testData); + script.parse(); + + if (!vm.hasException) + script.run(); + } +} + +void Test262Runner::runWithThreadPool() +{ + threadPool = new QThreadPool(); + threadPool->setStackSize(16*1024*1024); + qDebug() << "Running in parallel with" << QThread::idealThreadCount() << "threads"; + + for (const TestCase &testCase : std::as_const(testCases)) { + TestData testData = getTestData(testCase); + if (testData.isExcluded || testData.async) + continue; + SingleTest *test = new SingleTest(this, testData); + threadPool->start(test); + } + + while (!threadPool->waitForDone(10'000)) { + if (lcJsTest().isEnabled(QtDebugMsg)) { + // heartbeat, only needed when there is no other debug output + qDebug("test262: in progress..."); + } + } +} + +bool Test262Runner::run() +{ + if (!loadTests()) + return false; + + if (flags & ForceJIT) + qputenv("QV4_JIT_CALL_THRESHOLD", QByteArray("0")); + else if (flags & ForceBytecode) + qputenv("QV4_FORCE_INTERPRETER", QByteArray("1")); + + if (flags & WithTestExpectations) + loadTestExpectations(); + + for (auto it = testCases.constBegin(); it != testCases.constEnd(); ++it) { + auto c = it.value(); + if (!c.skipTestCase) { + TestData data = getTestData(c); + if (data.isExcluded || data.async) + continue; + + tasks.append(data); + } + } + + if (command.isEmpty()) { +#if QT_CONFIG(process) + createProcesses(); + loop.exec(); +#else + runWithThreadPool(); +#endif + } else { + runAsExternalTests(); + } + + const bool testsOk = report(); + + if (flags & WriteTestExpectations) + writeTestExpectations(); + else if (flags & UpdateTestExpectations) + updateTestExpectations(); + + return testsOk; +} + +bool Test262Runner::report() +{ + qDebug() << "Test execution summary:"; + qDebug() << " Executed" << testCases.size() << "test cases."; + QStringList crashes; + QStringList unexpectedFailures; + QStringList unexpectedPasses; + for (auto it = testCases.constBegin(); it != testCases.constEnd(); ++it) { + const auto c = it.value(); + if (c.strictResult.state == c.strictExpectation.state + && c.sloppyResult.state == c.sloppyExpectation.state) + continue; + auto report = [&](const TestCase::Result &expected, const TestCase::Result &result, const char *s) { + if (result.state == TestCase::Crashes) + crashes << (it.key() + " crashed in " + s + " mode"); + if (result.state == TestCase::Fails && expected.state == TestCase::Passes) + unexpectedFailures << (it.key() + " failed in " + s + + " mode with error message: " + result.errorMessage); + if (result.state == TestCase::Passes && expected.state == TestCase::Fails) + unexpectedPasses << (it.key() + " unexpectedly passed in " + s + " mode"); + }; + report(c.strictExpectation, c.strictResult, "strict"); + report(c.sloppyExpectation, c.sloppyResult, "sloppy"); + } + if (!crashes.isEmpty()) { + qDebug() << " Encountered" << crashes.size() << "crashes in the following files:"; + for (const QString &f : std::as_const(crashes)) + qDebug() << " " << f; + } + if (!unexpectedFailures.isEmpty()) { + qDebug() << " Encountered" << unexpectedFailures.size() << "unexpected failures in the following files:"; + for (const QString &f : std::as_const(unexpectedFailures)) + qDebug() << " " << f; + } + if (!unexpectedPasses.isEmpty()) { + qDebug() << " Encountered" << unexpectedPasses.size() << "unexpected passes in the following files:"; + for (const QString &f : std::as_const(unexpectedPasses)) + qDebug() << " " << f; + } + return crashes.isEmpty() && unexpectedFailures.isEmpty() && unexpectedPasses.isEmpty(); +} + +bool Test262Runner::loadTests() +{ + QDir dir(testDir + "/test"); + if (!dir.exists()) { + qWarning() << "Could not load tests," << dir.path() << "does not exist."; + return false; + } + + QString annexB = "annexB"; + QString harness = "harness"; + QString intl402 = "intl402"; + + int pathlen = dir.path().size() + 1; + QDirIterator it(dir, QDirIterator::Subdirectories); + while (it.hasNext()) { + QString file = it.next().mid(pathlen); + if (!file.endsWith(".js")) + continue; + if (file.endsWith("_FIXTURE.js")) + continue; + if (!filter.isEmpty() && !file.contains(filter)) + continue; + if (file.startsWith(annexB) || file.startsWith(harness) || file.startsWith(intl402)) + continue; + const char **excluded = excludedFilePatterns; + bool skip = false; + while (*excluded) { + if (file.contains(QLatin1String(*excluded))) + skip = true; + ++excluded; + } + if (skip) + continue; + + testCases.insert(file, TestCase{ file }); + } + if (testCases.isEmpty()) { + qWarning() << "No tests to run."; + return false; + } + + return true; +} + + +struct TestExpectationLine { + TestExpectationLine(const QByteArray &line); + enum State { + Fails, + SloppyFails, + StrictFails, + Skip, + Passes + } state; + QString testCase; + + QByteArray toLine() const; + void update(const TestCase &testCase); + + static TestExpectationLine fromTestCase(const TestCase &testCase); +private: + TestExpectationLine() = default; + static State stateFromTestCase(const TestCase &testCase); +}; + +TestExpectationLine::TestExpectationLine(const QByteArray &line) +{ + int space = line.indexOf(' '); + + testCase = QString::fromUtf8(space > 0 ? line.left(space) : line); + if (!testCase.endsWith(".js")) + testCase += ".js"; + + state = Fails; + if (space < 0) + return; + QByteArray qualifier = line.mid(space + 1); + if (qualifier == "skip") + state = Skip; + else if (qualifier == "strictFails") + state = StrictFails; + else if (qualifier == "sloppyFails") + state = SloppyFails; + else if (qualifier == "fails") + state = Fails; + else + qWarning() << "illegal format in TestExpectations, line" << line; +} + +QByteArray TestExpectationLine::toLine() const { + const char *res = nullptr; + switch (state) { + case Fails: + res = " fails\n"; + break; + case SloppyFails: + res = " sloppyFails\n"; + break; + case StrictFails: + res = " strictFails\n"; + break; + case Skip: + res = " skip\n"; + break; + case Passes: + // no need for an entry + return QByteArray(); + } + QByteArray result = testCase.toUtf8() + res; + return result; +} + +void TestExpectationLine::update(const TestCase &testCase) +{ + Q_ASSERT(testCase.test == this->testCase); + + State resultState = stateFromTestCase(testCase); + switch (resultState) { + case Fails: + // no improvement, don't update + break; + case SloppyFails: + if (state == Fails) + state = SloppyFails; + else if (state == StrictFails) + // we have a regression in sloppy mode, but strict now passes + state = Passes; + break; + case StrictFails: + if (state == Fails) + state = StrictFails; + else if (state == SloppyFails) + // we have a regression in strict mode, but sloppy now passes + state = Passes; + break; + case Skip: + Q_ASSERT(state == Skip); + // nothing to do + break; + case Passes: + state = Passes; + } +} + +TestExpectationLine TestExpectationLine::fromTestCase(const TestCase &testCase) +{ + TestExpectationLine l; + l.testCase = testCase.test; + l.state = stateFromTestCase(testCase); + return l; +} + +TestExpectationLine::State TestExpectationLine::stateFromTestCase(const TestCase &testCase) +{ + // keep skipped tests + if (testCase.skipTestCase) + return Skip; + + bool strictFails = (testCase.strictResult.state == TestCase::Crashes + || testCase.strictResult.state == TestCase::Fails); + bool sloppyFails = (testCase.sloppyResult.state == TestCase::Crashes + || testCase.sloppyResult.state == TestCase::Fails); + if (strictFails && sloppyFails) + return Fails; + if (strictFails) + return StrictFails; + if (sloppyFails) + return SloppyFails; + return Passes; +} + + +void Test262Runner::loadTestExpectations() +{ + QFile file(expectationsFile); + if (!file.open(QFile::ReadOnly)) { + qWarning() << "Could not open TestExpectations file at" << expectationsFile; + return; + } + + while (!file.atEnd()) { + QByteArray line = file.readLine().trimmed(); + if (line.startsWith('#') || line.isEmpty()) + continue; + TestExpectationLine expectation(line); + if (!filter.isEmpty() && !expectation.testCase.contains(filter)) + continue; + + if (!testCases.contains(expectation.testCase)) + qWarning() << "Unknown test case" << expectation.testCase << "in TestExpectations file."; + //qDebug() << "TestExpectations:" << expectation.testCase << expectation.state; + TestCase &s = testCases[expectation.testCase]; + switch (expectation.state) { + case TestExpectationLine::Fails: + s.strictExpectation.state = TestCase::Fails; + s.sloppyExpectation.state = TestCase::Fails; + break; + case TestExpectationLine::SloppyFails: + s.strictExpectation.state = TestCase::Passes; + s.sloppyExpectation.state = TestCase::Fails; + break; + case TestExpectationLine::StrictFails: + s.strictExpectation.state = TestCase::Fails; + s.sloppyExpectation.state = TestCase::Passes; + break; + case TestExpectationLine::Skip: + s.skipTestCase = true; + break; + case TestExpectationLine::Passes: + Q_UNREACHABLE(); + } + } +} + +void Test262Runner::updateTestExpectations() +{ + QFile file(expectationsFile); + if (!file.open(QFile::ReadOnly)) { + qWarning() << "Could not open TestExpectations file at" << expectationsFile; + return; + } + + QTemporaryFile updatedExpectations; + updatedExpectations.open(); + + while (!file.atEnd()) { + QByteArray originalLine = file.readLine(); + QByteArray line = originalLine.trimmed(); + if (line.startsWith('#') || line.isEmpty()) { + updatedExpectations.write(originalLine); + continue; + } + + TestExpectationLine expectation(line); +// qDebug() << "checking: " << expectation.testCase; + if (!testCases.contains(expectation.testCase)) { + updatedExpectations.write(originalLine); + continue; + } + const TestCase &testcase = testCases.value(expectation.testCase); + expectation.update(testcase); + + line = expectation.toLine(); +// qDebug() << "updated line:" << line; + updatedExpectations.write(line); + } + file.close(); + updatedExpectations.close(); + if (!file.remove()) + qWarning() << "Could not remove old TestExpectations file at" << expectationsFile; + if (updatedExpectations.copy(file.fileName())) + qDebug() << "Updated TestExpectations file written!"; + else + qWarning() << "Could not write new TestExpectations file at" << expectationsFile; +} + +void Test262Runner::writeTestExpectations() +{ + QFile file(expectationsFile); + + QTemporaryFile expectations; + expectations.open(); + + for (const auto &c : std::as_const(testCases)) { + TestExpectationLine line = TestExpectationLine::fromTestCase(c); + expectations.write(line.toLine()); + } + + expectations.close(); + if (file.exists() && !file.remove()) + qWarning() << "Could not remove old TestExpectations file at" << expectationsFile; + if (expectations.copy(file.fileName())) + qDebug() << "new TestExpectations file written!"; + else + qWarning() << "Could not write new TestExpectations file at" << expectationsFile; +} + +void Test262Runner::runAsExternalTests() +{ + for (TestData &testData : tasks) { + auto runTest = [&] (const char *header, TestCase::Result *result) { + QTemporaryFile tempFile; + tempFile.open(); + tempFile.write(header); + tempFile.write(testData.content); + tempFile.close(); + + QProcess process; + process.start(command, QStringList(tempFile.fileName())); + if (!process.waitForFinished(-1) || process.error() == QProcess::FailedToStart) { + qWarning() << "Could not execute" << command; + *result = TestCase::Result(TestCase::Crashes); + } + if (process.exitStatus() != QProcess::NormalExit) { + *result = TestCase::Result(TestCase::Crashes); + } + bool ok = (process.exitCode() == EXIT_SUCCESS); + if (testData.negative) + ok = !ok; + *result = ok ? TestCase::Result(TestCase::Passes) + : TestCase::Result(TestCase::Fails, process.readAllStandardError()); + }; + + if (testData.runInSloppyMode) + runTest("", &testData.sloppyResult); + if (testData.runInStrictMode) + runTest("'use strict';\n", &testData.strictResult); + + addResult(testData); + } +} + +void Test262Runner::addResult(TestCase result) +{ + { +#if !QT_CONFIG(process) + QMutexLocker locker(&mutex); +#endif + Q_ASSERT(result.strictExpectation.state == testCases[result.test].strictExpectation.state); + Q_ASSERT(result.sloppyExpectation.state == testCases[result.test].sloppyExpectation.state); + testCases[result.test] = result; + } + + if (!(flags & Verbose)) + return; + + QString test = result.test; + if (result.strictResult.state == TestCase::Skipped) { + ; + } else if (result.strictResult.state == TestCase::Crashes) { + qDebug() << "FAIL:" << test << "crashed in strict mode!"; + } else if (result.strictResult.state == TestCase::Fails + && result.strictExpectation.state == TestCase::Fails) { + qCDebug(lcJsTest) << "PASS:" << test << "failed in strict mode as expected"; + } else if ((result.strictResult.state == TestCase::Passes) + == (result.strictExpectation.state == TestCase::Passes)) { + qCDebug(lcJsTest) << "PASS:" << test << "passed in strict mode"; + } else if (!(result.strictExpectation.state == TestCase::Fails)) { + qDebug() << "FAIL:" << test << "failed in strict mode with error message:\n" + << result.strictResult.errorMessage; + } else { + qDebug() << "XPASS:" << test << "unexpectedly passed in strict mode"; + } + + if (result.sloppyResult.state == TestCase::Skipped) { + ; + } else if (result.sloppyResult.state == TestCase::Crashes) { + qDebug() << "FAIL:" << test << "crashed in sloppy mode!"; + } else if (result.sloppyResult.state == TestCase::Fails + && result.sloppyExpectation.state == TestCase::Fails) { + qCDebug(lcJsTest) << "PASS:" << test << "failed in sloppy mode as expected"; + } else if ((result.sloppyResult.state == TestCase::Passes) + == (result.sloppyExpectation.state == TestCase::Passes)) { + qCDebug(lcJsTest) << "PASS:" << test << "passed in sloppy mode"; + } else if (!(result.sloppyExpectation.state == TestCase::Fails)) { + qDebug() << "FAIL:" << test << "failed in sloppy mode with error message:\n" + << result.sloppyResult.errorMessage; + } else { + qDebug() << "XPASS:" << test << "unexpectedly passed in sloppy mode"; + } +} + +TestData Test262Runner::getTestData(const TestCase &testCase) +{ + QFile testFile(testDir + "/test/" + testCase.test); + if (!testFile.open(QFile::ReadOnly)) { + qWarning() << "wrong test file" << testCase.test; + exit(1); + } + QByteArray content = testFile.readAll(); + content.replace(QByteArrayLiteral("\r\n"), "\n"); + + qCDebug(lcJsTest) << "parsing test file" << testCase.test; + + TestData data(testCase); + parseYaml(content, &data); + + data.harness += harness("assert.js"); + data.harness += harness("sta.js"); + + for (QByteArray inc : std::as_const(data.includes)) { + inc = inc.trimmed(); + data.harness += harness(inc); + } + + if (data.async) + data.harness += harness("doneprintHandle.js"); + + data.content = data.harness + content; + + return data; +} + +struct YamlSection { + YamlSection(const QByteArray &yaml, const char *sectionName); + + bool contains(const char *keyword) const; + QList keywords() const; + + QByteArray yaml; + int start = -1; + int length = 0; + bool shortSection = false; +}; + +YamlSection::YamlSection(const QByteArray &yaml, const char *sectionName) + : yaml(yaml) +{ + start = yaml.indexOf(sectionName); + if (start < 0) + return; + start += static_cast(strlen(sectionName)); + int end = yaml.indexOf('\n', start + 1); + if (end < 0) + end = yaml.size(); + + int s = yaml.indexOf('[', start); + if (s > 0 && s < end) { + shortSection = true; + start = s + 1; + end = yaml.indexOf(']', s); + } else { + while (end < yaml.size() - 1 && yaml.at(end + 1) == ' ') + end = yaml.indexOf('\n', end + 1); + } + length = end - start; +} + +bool YamlSection::contains(const char *keyword) const +{ + if (start < 0) + return false; + int idx = yaml.indexOf(keyword, start); + if (idx >= start && idx < start + length) + return true; + return false; +} + +QList YamlSection::keywords() const +{ + if (start < 0) + return QList(); + + QByteArray content = yaml.mid(start, length); + QList keywords; + if (shortSection) { + keywords = content.split(','); + } else { + const QList list = content.split('\n'); + for (const QByteArray &l : list) { + int i = 0; + while (i < l.size() && (l.at(i) == ' ' || l.at(i) == '-')) + ++i; + QByteArray entry = l.mid(i); + if (!entry.isEmpty()) + keywords.append(entry); + } + } +// qDebug() << "keywords:" << keywords; + return keywords; +} + + +void Test262Runner::parseYaml(const QByteArray &content, TestData *data) +{ + int start = content.indexOf("/*---"); + if (start < 0) + return; + start += sizeof("/*---"); + + int end = content.indexOf("---*/"); + if (end < 0) + return; + + QByteArray yaml = content.mid(start, end - start); + + if (yaml.contains("negative:")) + data->negative = true; + + YamlSection flags(yaml, "flags:"); + data->runInSloppyMode = !flags.contains("onlyStrict"); + data->runInStrictMode = !flags.contains("noStrict") && !flags.contains("raw"); + data->runAsModuleCode = flags.contains("module"); + data->async = flags.contains("async"); + + if (data->runAsModuleCode) { + data->runInStrictMode = true; + data->runInSloppyMode = false; + } + + YamlSection includes(yaml, "includes:"); + data->includes = includes.keywords(); + + YamlSection features = YamlSection(yaml, "features:"); + + const char **f = excludedFeatures; + while (*f) { + if (features.contains(*f)) { + data->isExcluded = true; + break; + } + ++f; + } + +// qDebug() << "Yaml:\n" << yaml; +} + +QByteArray Test262Runner::harness(const QByteArray &name) +{ + if (harnessFiles.contains(name)) + return harnessFiles.value(name); + + QFile h(testDir + QLatin1String("/harness/") + name); + if (!h.open(QFile::ReadOnly)) { + qWarning() << "Illegal test harness file" << name; + exit(1); + } + + QByteArray content = h.readAll(); + harnessFiles.insert(name, content); + return content; +} + +QT_END_NAMESPACE diff --git a/tests/auto/qml/ecmascripttests/test262runner.h b/tests/auto/qml/ecmascripttests/test262runner.h new file mode 100644 index 0000000000..53c66618f0 --- /dev/null +++ b/tests/auto/qml/ecmascripttests/test262runner.h @@ -0,0 +1,167 @@ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0 + +#ifndef TEST262RUNNER_H +#define TEST262RUNNER_H + +#include +#include +#include +#include +#include +#include +#include + +QT_BEGIN_NAMESPACE + +namespace QV4 { +struct ExecutionEngine; +void initD262(ExecutionEngine *e); +} + +struct TestCase { + TestCase() = default; + TestCase(const QString &test) + : test(test) {} + + enum State { Skipped, Passes, Fails, Crashes }; + + struct Result + { + State state; + QString errorMessage; + + Result(State state, QString errorMessage = "") + : state(state), errorMessage(errorMessage) { } + + void negateResult() + { + switch (state) { + case TestCase::Passes: + state = TestCase::Fails; + break; + case TestCase::Fails: + state = TestCase::Passes; + break; + case TestCase::Skipped: + case TestCase::Crashes: + break; + } + } + }; + + Result strictExpectation = Result(Passes); + Result sloppyExpectation = Result(Passes); + Result strictResult = Result(Skipped); + Result sloppyResult = Result(Skipped); + bool skipTestCase = false; + bool stillNeedStrictRun = false; + + QString test; +}; + +struct TestData : TestCase { + TestData() = default; + TestData(const TestCase &testCase) + : TestCase(testCase) {} + // flags + bool negative = false; + bool runInStrictMode = true; + bool runInSloppyMode = true; + bool runAsModuleCode = false; + bool async = false; + + bool isExcluded = false; + + QList includes; + + QByteArray harness; + QByteArray content; +}; + +class SingleTest; + +class Test262Runner : public QObject +{ + Q_OBJECT + +public: + Test262Runner(const QString &command, const QString &testDir, const QString &expectationsFile); + ~Test262Runner(); + + enum Mode { + Sloppy = 0, + Strict = 1 + }; + + enum Flags { + Verbose = 0x1, + Parallel = 0x2, + ForceBytecode = 0x4, + ForceJIT = 0x8, + WithTestExpectations = 0x10, + UpdateTestExpectations = 0x20, + WriteTestExpectations = 0x40, + }; + void setFlags(int f) { flags = f; } + + void setFilter(const QString &f) { filter = f; } + + void cat(); + bool run(); + + bool report(); + QString testDirectory() const { return testDir; } + + static void executeTest(QV4::ExecutionEngine &vm, const QString &testData, + const QString &testCasePath = QString(), + const QString &harnessForModules = QString(), + bool runAsModule = false); + +private: + friend class SingleTest; + bool loadTests(); + void loadTestExpectations(); + void updateTestExpectations(); + void writeTestExpectations(); + + void runWithThreadPool(); + + void runAsExternalTests(); + void createProcesses(); + void assignTaskOrTerminate(int processIndex); + void assignSloppy(int processIndex); + void assignStrict(int processIndex); + void sendDone(int processIndex); + QString readUntilNull(QProcess &p); + + TestData getTestData(const TestCase &testCase); + void parseYaml(const QByteArray &content, TestData *data); + + QByteArray harness(const QByteArray &name); + + void addResult(TestCase result); + + QString command; + QString testDir; + QString expectationsFile; + int flags = 0; + + QString filter; + + QMap testCases; + QHash harnessFiles; + + QThreadPool *threadPool = nullptr; + QMutex mutex; + + QEventLoop loop; + std::vector> processes; + int runningCount = 0; + QQueue tasks; + QHash currentTasks; +}; + +QT_END_NAMESPACE + +#endif diff --git a/tests/auto/qml/ecmascripttests/tst_ecmascripttests.cpp b/tests/auto/qml/ecmascripttests/tst_ecmascripttests.cpp index e4a108a2e8..fc20c80a1a 100644 --- a/tests/auto/qml/ecmascripttests/tst_ecmascripttests.cpp +++ b/tests/auto/qml/ecmascripttests/tst_ecmascripttests.cpp @@ -1,11 +1,22 @@ // Copyright (C) 2017 The Qt Company Ltd. // SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0 -#include -#include +#include +#include +#include +#include #include -#include +#include #include +#include + +#include "test262runner.h" +#include "private/qqmlbuiltinfunctions_p.h" +#include "private/qv4arraybuffer_p.h" +#include "private/qv4globalobject_p.h" +#include "private/qv4script_p.h" + +#include class tst_EcmaScriptTests : public QQmlDataTest { @@ -94,7 +105,74 @@ void tst_EcmaScriptTests::runJitted() QVERIFY(result); } -QTEST_GUILESS_MAIN(tst_EcmaScriptTests) +//// v RUNNER PROCESS MODE v //// -#include "tst_ecmascripttests.moc" +void readInput(bool &done, QString &mode, QString &testData, QString &testCasePath, + QString &harnessForModules, bool &runAsModule) +{ + QTextStream in(stdin); + QString input; + while (input.isEmpty()) + input = in.readLine(); + + QJsonDocument json = QJsonDocument::fromJson(input.toUtf8()); + done = json["done"].toBool(false); + mode = json["mode"].toString(); + testData = json["testData"].toString(); + testCasePath = json["testCasePath"].toString(); + harnessForModules = json["harnessForModules"].toString(); + runAsModule = json["runAsModule"].toBool(false); +} + +void printResult(QV4::ExecutionEngine &vm, const QString &mode) +{ + QJsonObject result; + result.insert("mode", mode); + if (vm.hasException) { + QV4::Scope scope(&vm); + QV4::ScopedValue val(scope, vm.catchException()); + + result.insert("resultState", int(TestCase::State::Fails)); + result.insert("resultErrorMessage", val->toQString()); + } else { + result.insert("resultState", int(TestCase::State::Passes)); + } + QTextStream(stdout) << QJsonDocument(result).toJson(QJsonDocument::Compact) << "\r\n"; +} + +void doRunnerProcess() +{ + bool done = false; + QString mode; + QString testData; + QString testCasePath; + QString harnessForModules; + bool runAsModule = false; + + while (!done) { + QV4::ExecutionEngine vm; + readInput(done, mode, testData, testCasePath, harnessForModules, runAsModule); + if (done) + break; + Test262Runner::executeTest(vm, testData, testCasePath, harnessForModules, runAsModule); + printResult(vm, mode); + } +} + +//// ^ RUNNER PROCESS MODE ^ //// + +int main(int argc, char *argv[]) +{ + QCoreApplication app(argc, argv); + + if (qEnvironmentVariableIntValue("runnerProcess") == 1) { + doRunnerProcess(); + } else { + tst_EcmaScriptTests tc; + QTEST_SET_MAIN_SOURCE_PATH + return QTest::qExec(&tc, argc, argv); + } +} + +#include "tst_ecmascripttests.moc" -- cgit v1.2.3