From 825e4291cd44b82c0566e86e60bb77f434196a44 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tor=20Arne=20Vestb=C3=B8?= <tor.arne.vestbo@qt.io>
Date: Mon, 2 Aug 2021 15:14:53 +0200
Subject: testlib: Produce <error> elements on fatal errors in JUnit reporter
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Test errors represents unanticipated problems, e.g., an unhandled
exception, or a problem with the implementation of the test.

Pick-to: 6.2
Change-Id: I87219e7ffdea56862278f005de44526ad97545f0
Reviewed-by: Mårten Nordheim <marten.nordheim@qt.io>
---
 tests/auto/testlib/selftests/expected_assert.junitxml        | 9 +++------
 tests/auto/testlib/selftests/expected_faildatatype.junitxml  | 9 +++------
 tests/auto/testlib/selftests/expected_failfetchtype.junitxml | 9 +++------
 tests/auto/testlib/selftests/expected_fetchbogus.junitxml    | 9 +++------
 tests/auto/testlib/selftests/expected_silent.junitxml        | 6 ++----
 tests/auto/testlib/selftests/expected_watchdog.junitxml      | 9 +++------
 6 files changed, 17 insertions(+), 34 deletions(-)

(limited to 'tests/auto/testlib')

diff --git a/tests/auto/testlib/selftests/expected_assert.junitxml b/tests/auto/testlib/selftests/expected_assert.junitxml
index 5c854832f2..b000a0decc 100644
--- a/tests/auto/testlib/selftests/expected_assert.junitxml
+++ b/tests/auto/testlib/selftests/expected_assert.junitxml
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" ?>
-<testsuite name="tst_Assert" timestamp="@TEST_START_TIME@" hostname="@HOSTNAME@" tests="3" failures="1" errors="0" skipped="0" time="@TEST_DURATION@">
+<testsuite name="tst_Assert" timestamp="@TEST_START_TIME@" hostname="@HOSTNAME@" tests="3" failures="0" errors="1" skipped="0" time="@TEST_DURATION@">
   <properties>
     <property name="QTestVersion" value="@INSERT_QT_VERSION_HERE@"/>
     <property name="QtVersion" value="@INSERT_QT_VERSION_HERE@"/>
@@ -8,11 +8,8 @@
   <testcase name="initTestCase" classname="tst_Assert" time="@TEST_DURATION@"/>
   <testcase name="testNumber1" classname="tst_Assert" time="@TEST_DURATION@"/>
   <testcase name="testNumber2" classname="tst_Assert" time="@TEST_DURATION@">
-    <!-- type="qfatal" message="ASSERT: &quot;false&quot; in file qtbase/tests/auto/testlib/selftests/assert/tst_assert.cpp, line 0" -->
-    <failure type="fail" message="Received a fatal error."/>
+    <error type="qfatal" message="ASSERT: &quot;false&quot; in file qtbase/tests/auto/testlib/selftests/assert/tst_assert.cpp, line 0"/>
   </testcase>
   <system-out/>
-  <system-err>
-    <![CDATA[ASSERT: "false" in file qtbase/tests/auto/testlib/selftests/assert/tst_assert.cpp, line 0]]>
-  </system-err>
+  <system-err/>
 </testsuite>
diff --git a/tests/auto/testlib/selftests/expected_faildatatype.junitxml b/tests/auto/testlib/selftests/expected_faildatatype.junitxml
index 3a2c8bf19b..697fe1e759 100644
--- a/tests/auto/testlib/selftests/expected_faildatatype.junitxml
+++ b/tests/auto/testlib/selftests/expected_faildatatype.junitxml
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" ?>
-<testsuite name="tst_FailDataType" timestamp="@TEST_START_TIME@" hostname="@HOSTNAME@" tests="2" failures="1" errors="0" skipped="0" time="@TEST_DURATION@">
+<testsuite name="tst_FailDataType" timestamp="@TEST_START_TIME@" hostname="@HOSTNAME@" tests="2" failures="0" errors="1" skipped="0" time="@TEST_DURATION@">
   <properties>
     <property name="QTestVersion" value="@INSERT_QT_VERSION_HERE@"/>
     <property name="QtVersion" value="@INSERT_QT_VERSION_HERE@"/>
@@ -8,13 +8,10 @@
   <testcase name="initTestCase" classname="tst_FailDataType" time="@TEST_DURATION@"/>
   <testcase name="value" classname="tst_FailDataType" time="@TEST_DURATION@">
     <!-- type="qdebug" message="expected data of type &apos;QString&apos;, got &apos;bool&apos; for element 0 of data with tag &apos;bool&#x002D;as&#x002D;string&apos;" -->
-    <!-- type="qfatal" message="ASSERT: &quot;false&quot; in file qtbase/src/testlib/qtestdata.cpp, line 0" -->
-    <failure type="fail" message="Received a fatal error."/>
+    <error type="qfatal" message="ASSERT: &quot;false&quot; in file qtbase/src/testlib/qtestdata.cpp, line 0"/>
   </testcase>
   <system-out>
     <![CDATA[expected data of type 'QString', got 'bool' for element 0 of data with tag 'bool-as-string']]>
   </system-out>
-  <system-err>
-    <![CDATA[ASSERT: "false" in file qtbase/src/testlib/qtestdata.cpp, line 0]]>
-  </system-err>
+  <system-err/>
 </testsuite>
diff --git a/tests/auto/testlib/selftests/expected_failfetchtype.junitxml b/tests/auto/testlib/selftests/expected_failfetchtype.junitxml
index 58ad4e725f..f7c0bb72cb 100644
--- a/tests/auto/testlib/selftests/expected_failfetchtype.junitxml
+++ b/tests/auto/testlib/selftests/expected_failfetchtype.junitxml
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" ?>
-<testsuite name="tst_FailFetchType" timestamp="@TEST_START_TIME@" hostname="@HOSTNAME@" tests="2" failures="1" errors="0" skipped="0" time="@TEST_DURATION@">
+<testsuite name="tst_FailFetchType" timestamp="@TEST_START_TIME@" hostname="@HOSTNAME@" tests="2" failures="0" errors="1" skipped="0" time="@TEST_DURATION@">
   <properties>
     <property name="QTestVersion" value="@INSERT_QT_VERSION_HERE@"/>
     <property name="QtVersion" value="@INSERT_QT_VERSION_HERE@"/>
@@ -7,11 +7,8 @@
   </properties>
   <testcase name="initTestCase" classname="tst_FailFetchType" time="@TEST_DURATION@"/>
   <testcase name="fetch(bool)" classname="tst_FailFetchType" time="@TEST_DURATION@">
-    <!-- type="qfatal" message="Requested type &apos;QString&apos; does not match available type &apos;bool&apos;." -->
-    <failure type="fail" message="Received a fatal error."/>
+    <error type="qfatal" message="Requested type &apos;QString&apos; does not match available type &apos;bool&apos;."/>
   </testcase>
   <system-out/>
-  <system-err>
-    <![CDATA[Requested type 'QString' does not match available type 'bool'.]]>
-  </system-err>
+  <system-err/>
 </testsuite>
diff --git a/tests/auto/testlib/selftests/expected_fetchbogus.junitxml b/tests/auto/testlib/selftests/expected_fetchbogus.junitxml
index ef9ac181ac..96f8df5fbd 100644
--- a/tests/auto/testlib/selftests/expected_fetchbogus.junitxml
+++ b/tests/auto/testlib/selftests/expected_fetchbogus.junitxml
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" ?>
-<testsuite name="tst_FetchBogus" timestamp="@TEST_START_TIME@" hostname="@HOSTNAME@" tests="2" failures="1" errors="0" skipped="0" time="@TEST_DURATION@">
+<testsuite name="tst_FetchBogus" timestamp="@TEST_START_TIME@" hostname="@HOSTNAME@" tests="2" failures="0" errors="1" skipped="0" time="@TEST_DURATION@">
   <properties>
     <property name="QTestVersion" value="@INSERT_QT_VERSION_HERE@"/>
     <property name="QtVersion" value="@INSERT_QT_VERSION_HERE@"/>
@@ -7,11 +7,8 @@
   </properties>
   <testcase name="initTestCase" classname="tst_FetchBogus" time="@TEST_DURATION@"/>
   <testcase name="fetchBogus(foo)" classname="tst_FetchBogus" time="@TEST_DURATION@">
-    <!-- type="qfatal" message="QFETCH: Requested testdata &apos;bubu&apos; not available, check your _data function." -->
-    <failure type="fail" message="Received a fatal error."/>
+    <error type="qfatal" message="QFETCH: Requested testdata &apos;bubu&apos; not available, check your _data function."/>
   </testcase>
   <system-out/>
-  <system-err>
-    <![CDATA[QFETCH: Requested testdata 'bubu' not available, check your _data function.]]>
-  </system-err>
+  <system-err/>
 </testsuite>
diff --git a/tests/auto/testlib/selftests/expected_silent.junitxml b/tests/auto/testlib/selftests/expected_silent.junitxml
index f1f0c70599..7293e30170 100644
--- a/tests/auto/testlib/selftests/expected_silent.junitxml
+++ b/tests/auto/testlib/selftests/expected_silent.junitxml
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" ?>
-<testsuite name="tst_Silent" timestamp="@TEST_START_TIME@" hostname="@HOSTNAME@" tests="7" failures="3" errors="0" skipped="1" time="@TEST_DURATION@">
+<testsuite name="tst_Silent" timestamp="@TEST_START_TIME@" hostname="@HOSTNAME@" tests="7" failures="2" errors="1" skipped="1" time="@TEST_DURATION@">
   <properties>
     <property name="QTestVersion" value="@INSERT_QT_VERSION_HERE@"/>
     <property name="QtVersion" value="@INSERT_QT_VERSION_HERE@"/>
@@ -26,8 +26,7 @@
     <!-- type="system" message="This is a critical message that should not appear in silent test output" -->
     <!-- type="qinfo" message="This is an info message that should not appear in silent test output" -->
     <!-- type="info" message="This is an internal testlib info message that should not appear in silent test output" -->
-    <!-- type="qfatal" message="This is a fatal error message that should still appear in silent test output" -->
-    <failure type="fail" message="Received a fatal error."/>
+    <error type="qfatal" message="This is a fatal error message that should still appear in silent test output"/>
   </testcase>
   <system-out>
     <![CDATA[This test should XFAIL]]>
@@ -39,6 +38,5 @@
   <system-err>
     <![CDATA[This is a warning that should not appear in silent test output]]>
     <![CDATA[This is an internal testlib warning that should not appear in silent test output]]>
-    <![CDATA[This is a fatal error message that should still appear in silent test output]]>
   </system-err>
 </testsuite>
diff --git a/tests/auto/testlib/selftests/expected_watchdog.junitxml b/tests/auto/testlib/selftests/expected_watchdog.junitxml
index ba2269ca4e..f9b7e6fd9a 100644
--- a/tests/auto/testlib/selftests/expected_watchdog.junitxml
+++ b/tests/auto/testlib/selftests/expected_watchdog.junitxml
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" ?>
-<testsuite name="tst_Watchdog" timestamp="@TEST_START_TIME@" hostname="@HOSTNAME@" tests="2" failures="1" errors="0" skipped="0" time="@TEST_DURATION@">
+<testsuite name="tst_Watchdog" timestamp="@TEST_START_TIME@" hostname="@HOSTNAME@" tests="2" failures="0" errors="1" skipped="0" time="@TEST_DURATION@">
   <properties>
     <property name="QTestVersion" value="@INSERT_QT_VERSION_HERE@"/>
     <property name="QtVersion" value="@INSERT_QT_VERSION_HERE@"/>
@@ -7,11 +7,8 @@
   </properties>
   <testcase name="initTestCase" classname="tst_Watchdog" time="@TEST_DURATION@"/>
   <testcase name="delay" classname="tst_Watchdog" time="@TEST_DURATION@">
-    <!-- type="qfatal" message="Test function timed out" -->
-    <failure type="fail" message="Received a fatal error."/>
+    <error type="qfatal" message="Test function timed out"/>
   </testcase>
   <system-out/>
-  <system-err>
-    <![CDATA[Test function timed out]]>
-  </system-err>
+  <system-err/>
 </testsuite>
-- 
cgit v1.2.3