Fix running the ES test suite

When we have one or more failures, make sure that the auto-test fails.
This produces a few additional failures but oddly also "fixes" some
tests that are marked as expected failure.

Change-Id: I0bbbed463b326336ab9caa33f434520c90ed36de
Reviewed-by: Lars Knoll <lars.knoll@qt.io>

1 files changed, 4 insertions, 3 deletions

diff --git a/tests/auto/qml/ecmascripttests/qjstest/test262runner.cpp b/tests/auto/qml/ecmascripttests/qjstest/test262runner.cpp
index 2a14bdb463..bd945047c1 100644
--- a/tests/auto/qml/ecmascripttests/qjstest/test262runner.cpp
+++ b/tests/auto/qml/ecmascripttests/qjstest/test262runner.cpp
@@ -123,17 +123,17 @@ bool Test262Runner::run()
     if (threadPool)
         threadPool->waitForDone();
 
-    report();
+    const bool testsOk = report();
 
     if (flags & WriteTestExpectations)
         writeTestExpectations();
     else if (flags & UpdateTestExpectations)
         updateTestExpectations();
 
-    return true;
+    return testsOk;
 }
 
-void Test262Runner::report()
+bool Test262Runner::report()
 {
     qDebug() << "Test execution summary:";
     qDebug() << "    Executed" << testCases.size() << "test cases.";
@@ -170,6 +170,7 @@ void Test262Runner::report()
         for (const QString &f : qAsConst(unexpectedPasses))
             qDebug() << "        " << f;
     }
+    return crashes.isEmpty() && unexpectedFailures.isEmpty() && unexpectedPasses.isEmpty();
 }
 
 bool Test262Runner::loadTests()