Run ES5 tests on Linux/x86-64

This requires including the test suite as a submodule by default and
bumping the module to a new sha1 that removes some particularly long
paths.

Also force the test suite to run under a US locale, as tests like
ch15/15.5/15.5.4/15.5.4.7/S15.5.4.7_A1_T11 cannot deal with locale
dependent output for date formatting.

The test harness now returns a non-zero exit code when a test fails.
This is used by the QTestLib wrapper to determine success or failure.

The tests with JIT are run, but the tests with the interpreter are
omitted at the moment until the last failure is fixed.

The tests add about 10-15 minutes extra time to the total time it takes
to run tests on Linux in the CI.

Change-Id: Id01fd3b41350f9c9a6ce9e43236f51f7f0fb71c8
Reviewed-by: Robin Burchell <robin.burchell@crimson.no>

10 files changed, 114 insertions, 26 deletions

diff --git a/.gitmodules b/.gitmodules
index 881629497e..c9561f396d 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -1,4 +1,3 @@
-[submodule "tests/manual/v4/test262"]
-	path = tests/manual/v4/test262
+[submodule "tests/auto/qml/ecmascripttests/test262"]
+	path = tests/auto/qml/ecmascripttests/test262
 	url = ../qtdeclarative-testsuites.git
-	update = none
diff --git a/tests/manual/v4/TestExpectations b/tests/auto/qml/ecmascripttests/TestExpectations
index 49f107452a..49f107452a 100644
--- a/tests/manual/v4/TestExpectations
+++ b/tests/auto/qml/ecmascripttests/TestExpectations
diff --git a/tests/auto/qml/ecmascripttests/ecmascripttests.pro b/tests/auto/qml/ecmascripttests/ecmascripttests.pro
new file mode 100644
index 0000000000..6d3ee12307
--- /dev/null
+++ b/tests/auto/qml/ecmascripttests/ecmascripttests.pro
@@ -0,0 +1,20 @@
+CONFIG += testcase
+TARGET = tst_ecmascripttests
+QT += testlib
+macos:CONFIG -= app_bundle
+SOURCES += tst_ecmascripttests.cpp
+DEFINES += SRCDIR=\\\"$$PWD\\\"
+
+TESTSCRIPT=$$PWD/test262.py
+isEmpty(V4CMD): V4CMD = qmljs
+
+checkjittarget.target = check-jit
+checkjittarget.commands = python $$TESTSCRIPT --command=$$V4CMD --parallel --with-test-expectations --update-expectations
+checkjittarget.depends = all
+QMAKE_EXTRA_TARGETS += checkjittarget
+
+checkmothtarget.target = check-interpreter
+checkmothtarget.commands = python $$TESTSCRIPT --command=\"$$V4CMD --interpret\" --parallel --with-test-expectations
+checkmothtarget.depends = all
+QMAKE_EXTRA_TARGETS += checkmothtarget
+
diff --git a/tests/auto/qml/ecmascripttests/test262 b/tests/auto/qml/ecmascripttests/test262
new file mode 160000
+Subproject d60c4ed97e69639bc5bc1db43a98828debf80c8
diff --git a/tests/manual/v4/test262.py b/tests/auto/qml/ecmascripttests/test262.py
index 3b5bfa119a..99f029cffd 100755
--- a/tests/manual/v4/test262.py
+++ b/tests/auto/qml/ecmascripttests/test262.py
@@ -555,6 +555,7 @@ class TestSuite(object):
     print
     if update_expectations:
         self.expectations.update(progress)
+    return progress.failed == 0
 
   def Print(self, tests):
     cases = self.EnumerateTests(tests)
@@ -567,6 +568,7 @@ def Main():
   # Uncomment the next line for more logging info.
   #logging.basicConfig(level=logging.DEBUG)
   os.environ["TZ"] = "PST8PDT"
+  os.environ["LANG"] = "en_US.UTF-8"
   parser = BuildOptions()
   (options, args) = parser.parse_args()
   ValidateOptions(options)
@@ -578,18 +580,21 @@ def Main():
   test_suite.Validate()
   if options.cat:
     test_suite.Print(args)
+    return 0
   else:
-    test_suite.Run(options.command, args,
-                   options.summary or options.full_summary,
-                   options.full_summary,
-                   options.parallel,
-                   options.update_expectations)
+    if test_suite.Run(options.command, args,
+                      options.summary or options.full_summary,
+                      options.full_summary,
+                      options.parallel,
+                      options.update_expectations):
+      return 0
+    else:
+      return 1
 
 
 if __name__ == '__main__':
   try:
-    Main()
-    sys.exit(0)
+    sys.exit(Main())
   except Test262Error, e:
     print "Error: %s" % e.message
     sys.exit(1)
diff --git a/tests/auto/qml/ecmascripttests/tst_ecmascripttests.cpp b/tests/auto/qml/ecmascripttests/tst_ecmascripttests.cpp
new file mode 100644
index 0000000000..2ce0cfe210
--- /dev/null
+++ b/tests/auto/qml/ecmascripttests/tst_ecmascripttests.cpp
@@ -0,0 +1,77 @@
+/****************************************************************************
+**
+** Copyright (C) 2017 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of the test suite of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:GPL-EXCEPT$
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3 as published by the Free Software
+** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-3.0.html.
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+
+
+#include <QtTest/QtTest>
+#include <QProcess>
+#include <QLibraryInfo>
+
+class tst_EcmaScriptTests : public QObject
+{
+    Q_OBJECT
+private slots:
+    void runTests_data();
+    void runTests();
+};
+
+void tst_EcmaScriptTests::runTests_data()
+{
+    QTest::addColumn<QString>("qmljsParameter");
+
+    QTest::newRow("jit") << QStringLiteral("--jit");
+    // Not passing yet: QTest::newRow("interpreter") << QStringLiteral("--interpret");
+}
+
+void tst_EcmaScriptTests::runTests()
+{
+#if defined(Q_OS_LINUX) && defined(Q_PROCESSOR_X86_64)
+    QFETCH(QString, qmljsParameter);
+
+    QProcess process;
+    process.setProcessChannelMode(QProcess::ForwardedChannels);
+    process.setWorkingDirectory(QLatin1String(SRCDIR));
+    process.setProgram("python");
+    process.setArguments(QStringList() << "test262.py" << "--command=" + QLibraryInfo::location(QLibraryInfo::BinariesPath) + "/qmljs " + qmljsParameter << "--parallel" << "--with-test-expectations");
+
+    qDebug() << "Going to run" << process.program() << process.arguments() << "in" << process.workingDirectory();
+
+    process.start();
+    QVERIFY(process.waitForStarted());
+    const int timeoutInMSecs = 20 * 60 * 1000;
+    QVERIFY2(process.waitForFinished(timeoutInMSecs), "Tests did not terminate in time -- see output above for details");
+    QVERIFY2(process.exitStatus() == QProcess::NormalExit, "Running the test harness failed -- see output above for details");
+    QVERIFY2(process.exitCode() == 0, "Tests failed -- see output above for details");
+#else
+    QSKIP("Currently the ecmascript tests are only run on Linux/x86-64");
+#endif
+}
+
+QTEST_MAIN(tst_EcmaScriptTests)
+
+#include "tst_ecmascripttests.moc"
+
diff --git a/tests/auto/qml/qml.pro b/tests/auto/qml/qml.pro
index b347b603bb..a9b18e9f32 100644
--- a/tests/auto/qml/qml.pro
+++ b/tests/auto/qml/qml.pro
@@ -62,7 +62,8 @@ PRIVATETESTS += \
     qqmlimport \
     qqmlobjectmodel \
     qmldiskcache \
-    qv4mm
+    qv4mm \
+    ecmascripttests
 
 qtHaveModule(widgets) {
     PUBLICTESTS += \
diff --git a/tests/auto/qml/qmlmin/tst_qmlmin.cpp b/tests/auto/qml/qmlmin/tst_qmlmin.cpp
index 90868d2ead..171c2bda8a 100644
--- a/tests/auto/qml/qmlmin/tst_qmlmin.cpp
+++ b/tests/auto/qml/qmlmin/tst_qmlmin.cpp
@@ -84,6 +84,7 @@ void tst_qmlmin::initTestCase()
     excludedDirs << "doc/src/snippets/qtquick1/qtbinding";
     excludedDirs << "doc/src/snippets/qtquick1/imports";
     excludedDirs << "tests/manual/v4";
+    excludedDirs << "tests/auto/qml/ecmascripttests";
     excludedDirs << "tests/auto/qml/qmllint";
 
     // Add invalid files (i.e. files with syntax errors)
diff --git a/tests/manual/v4/test262 b/tests/manual/v4/test262
deleted file mode 160000
-Subproject 9741ac4655808ac46c127e3d1d8ba3d27ada618
diff --git a/tests/manual/v4/tests.pro b/tests/manual/v4/tests.pro
deleted file mode 100644
index ce4a34f7a0..0000000000
--- a/tests/manual/v4/tests.pro
+++ /dev/null
@@ -1,15 +0,0 @@
-TEMPLATE = aux
-
-TESTSCRIPT=$$PWD/test262.py
-isEmpty(V4CMD): V4CMD = qmljs
-
-checktarget.target = check
-checktarget.commands = python $$TESTSCRIPT --command=$$V4CMD --parallel --with-test-expectations --update-expectations
-checktarget.depends = all
-QMAKE_EXTRA_TARGETS += checktarget
-
-checkmothtarget.target = check-interpreter
-checkmothtarget.commands = python $$TESTSCRIPT --command=\"$$V4CMD --interpret\" --parallel --with-test-expectations
-checkmothtarget.depends = all
-QMAKE_EXTRA_TARGETS += checkmothtarget
-