Fix running the ES test suite

When we have one or more failures, make sure that the auto-test fails.
This produces a few additional failures but oddly also "fixes" some
tests that are marked as expected failure.

Change-Id: I0bbbed463b326336ab9caa33f434520c90ed36de
Reviewed-by: Lars Knoll <lars.knoll@qt.io>

3 files changed, 7 insertions, 6 deletions

diff --git a/tests/auto/qml/ecmascripttests/qjstest/test262runner.cpp b/tests/auto/qml/ecmascripttests/qjstest/test262runner.cpp
index 2a14bdb463..bd945047c1 100644
--- a/tests/auto/qml/ecmascripttests/qjstest/test262runner.cpp
+++ b/tests/auto/qml/ecmascripttests/qjstest/test262runner.cpp
@@ -123,17 +123,17 @@ bool Test262Runner::run()
     if (threadPool)
         threadPool->waitForDone();
 
-    report();
+    const bool testsOk = report();
 
     if (flags & WriteTestExpectations)
         writeTestExpectations();
     else if (flags & UpdateTestExpectations)
         updateTestExpectations();
 
-    return true;
+    return testsOk;
 }
 
-void Test262Runner::report()
+bool Test262Runner::report()
 {
     qDebug() << "Test execution summary:";
     qDebug() << "    Executed" << testCases.size() << "test cases.";
@@ -170,6 +170,7 @@ void Test262Runner::report()
         for (const QString &f : qAsConst(unexpectedPasses))
             qDebug() << "        " << f;
     }
+    return crashes.isEmpty() && unexpectedFailures.isEmpty() && unexpectedPasses.isEmpty();
 }
 
 bool Test262Runner::loadTests()
diff --git a/tests/auto/qml/ecmascripttests/qjstest/test262runner.h b/tests/auto/qml/ecmascripttests/qjstest/test262runner.h
index a4035e0b72..6b64298f42 100644
--- a/tests/auto/qml/ecmascripttests/qjstest/test262runner.h
+++ b/tests/auto/qml/ecmascripttests/qjstest/test262runner.h
@@ -97,7 +97,7 @@ public:
     void cat();
     bool run();
 
-    void report();
+    bool report();
 
 private:
     friend class SingleTest;
diff --git a/tests/auto/qml/ecmascripttests/tst_ecmascripttests.cpp b/tests/auto/qml/ecmascripttests/tst_ecmascripttests.cpp
index 27d2822762..418dd4dc53 100644
--- a/tests/auto/qml/ecmascripttests/tst_ecmascripttests.cpp
+++ b/tests/auto/qml/ecmascripttests/tst_ecmascripttests.cpp
@@ -43,7 +43,7 @@ private slots:
 
 void tst_EcmaScriptTests::runInterpreted()
 {
-#if defined(Q_PROCESSOR_X86_64)
+#if defined(Q_PROCESSOR_X86_64) && !defined(Q_OS_DARWIN)
     QDir::setCurrent(QLatin1String(SRCDIR));
     Test262Runner runner(QString(), "test262");
     runner.setFlags(Test262Runner::ForceBytecode|Test262Runner::WithTestExpectations|Test262Runner::Parallel|Test262Runner::Verbose);
@@ -54,7 +54,7 @@ void tst_EcmaScriptTests::runInterpreted()
 
 void tst_EcmaScriptTests::runJitted()
 {
-#if defined(Q_PROCESSOR_X86_64)
+#if defined(Q_PROCESSOR_X86_64) && !defined(Q_OS_DARWIN)
     QDir::setCurrent(QLatin1String(SRCDIR));
     Test262Runner runner(QString(), "test262");
     runner.setFlags(Test262Runner::ForceJIT|Test262Runner::WithTestExpectations|Test262Runner::Parallel|Test262Runner::Verbose);