testlib: Only generate expecations for silent test with plain logger

The -silent option to tests is only supported with the plain text
logger, so we don't need to maintain expectation files for the
others.

Pick-to: 6.2
Change-Id: I0f42bfe90d82b7ce04f550c747d4a80e99621e74
Reviewed-by: Edward Welbourne <edward.welbourne@qt.io>
Reviewed-by: Mårten Nordheim <marten.nordheim@qt.io>

6 files changed, 10 insertions, 239 deletions

diff --git a/tests/auto/testlib/selftests/expected_silent.junitxml b/tests/auto/testlib/selftests/expected_silent.junitxml
deleted file mode 100644
index 7293e30170..0000000000
--- a/tests/auto/testlib/selftests/expected_silent.junitxml
+++ /dev/null
@@ -1,42 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" ?>
-<testsuite name="tst_Silent" timestamp="@TEST_START_TIME@" hostname="@HOSTNAME@" tests="7" failures="2" errors="1" skipped="1" time="@TEST_DURATION@">
-  <properties>
-    <property name="QTestVersion" value="@INSERT_QT_VERSION_HERE@"/>
-    <property name="QtVersion" value="@INSERT_QT_VERSION_HERE@"/>
-    <property name="QtBuild" value=""/>
-  </properties>
-  <testcase name="initTestCase" classname="tst_Silent" time="@TEST_DURATION@"/>
-  <testcase name="pass" classname="tst_Silent" time="@TEST_DURATION@"/>
-  <testcase name="skip" classname="tst_Silent" time="@TEST_DURATION@">
-    <skipped message="This test should skip"/>
-  </testcase>
-  <testcase name="fail" classname="tst_Silent" time="@TEST_DURATION@">
-    <failure type="fail" message="&apos;false&apos; returned FALSE. (This test should fail)"/>
-  </testcase>
-  <testcase name="xfail" classname="tst_Silent" time="@TEST_DURATION@">
-    <!-- type="info" message="This test should XFAIL" -->
-  </testcase>
-  <testcase name="xpass" classname="tst_Silent" time="@TEST_DURATION@">
-    <failure type="xpass" message="&apos;true&apos; returned TRUE unexpectedly. (This test should XPASS)"/>
-  </testcase>
-  <testcase name="messages" classname="tst_Silent" time="@TEST_DURATION@">
-    <!-- type="qwarn" message="This is a warning that should not appear in silent test output" -->
-    <!-- type="warn" message="This is an internal testlib warning that should not appear in silent test output" -->
-    <!-- type="qdebug" message="This is a debug message that should not appear in silent test output" -->
-    <!-- type="system" message="This is a critical message that should not appear in silent test output" -->
-    <!-- type="qinfo" message="This is an info message that should not appear in silent test output" -->
-    <!-- type="info" message="This is an internal testlib info message that should not appear in silent test output" -->
-    <error type="qfatal" message="This is a fatal error message that should still appear in silent test output"/>
-  </testcase>
-  <system-out>
-    <![CDATA[This test should XFAIL]]>
-    <![CDATA[This is a debug message that should not appear in silent test output]]>
-    <![CDATA[This is a critical message that should not appear in silent test output]]>
-    <![CDATA[This is an info message that should not appear in silent test output]]>
-    <![CDATA[This is an internal testlib info message that should not appear in silent test output]]>
-  </system-out>
-  <system-err>
-    <![CDATA[This is a warning that should not appear in silent test output]]>
-    <![CDATA[This is an internal testlib warning that should not appear in silent test output]]>
-  </system-err>
-</testsuite>
diff --git a/tests/auto/testlib/selftests/expected_silent.lightxml b/tests/auto/testlib/selftests/expected_silent.lightxml
deleted file mode 100644
index ad07180bd6..0000000000
--- a/tests/auto/testlib/selftests/expected_silent.lightxml
+++ /dev/null
@@ -1,66 +0,0 @@
-<Environment>
-    <QtVersion>@INSERT_QT_VERSION_HERE@</QtVersion>
-    <QtBuild/>
-    <QTestVersion>@INSERT_QT_VERSION_HERE@</QTestVersion>
-</Environment>
-<TestFunction name="initTestCase">
-<Incident type="pass" file="" line="0" />
-    <Duration msecs="0"/>
-</TestFunction>
-<TestFunction name="pass">
-<Incident type="pass" file="" line="0" />
-    <Duration msecs="0"/>
-</TestFunction>
-<TestFunction name="skip">
-<Message type="skip" file="qtbase/tests/auto/testlib/selftests/silent/tst_silent.cpp" line="0">
-    <Description><![CDATA[This test should skip]]></Description>
-</Message>
-    <Duration msecs="0"/>
-</TestFunction>
-<TestFunction name="fail">
-<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/silent/tst_silent.cpp" line="0">
-    <Description><![CDATA['false' returned FALSE. (This test should fail)]]></Description>
-</Incident>
-    <Duration msecs="0"/>
-</TestFunction>
-<TestFunction name="xfail">
-<Incident type="xfail" file="qtbase/tests/auto/testlib/selftests/silent/tst_silent.cpp" line="0">
-    <Description><![CDATA[This test should XFAIL]]></Description>
-</Incident>
-<Incident type="pass" file="" line="0" />
-    <Duration msecs="0"/>
-</TestFunction>
-<TestFunction name="xpass">
-<Incident type="xpass" file="qtbase/tests/auto/testlib/selftests/silent/tst_silent.cpp" line="0">
-    <Description><![CDATA['true' returned TRUE unexpectedly. (This test should XPASS)]]></Description>
-</Incident>
-    <Duration msecs="0"/>
-</TestFunction>
-<TestFunction name="messages">
-<Message type="qwarn" file="" line="0">
-    <Description><![CDATA[This is a warning that should not appear in silent test output]]></Description>
-</Message>
-<Message type="warn" file="qtbase/tests/auto/testlib/selftests/silent/tst_silent.cpp" line="0">
-    <Description><![CDATA[This is an internal testlib warning that should not appear in silent test output]]></Description>
-</Message>
-<Message type="qdebug" file="" line="0">
-    <Description><![CDATA[This is a debug message that should not appear in silent test output]]></Description>
-</Message>
-<Message type="system" file="" line="0">
-    <Description><![CDATA[This is a critical message that should not appear in silent test output]]></Description>
-</Message>
-<Message type="qinfo" file="" line="0">
-    <Description><![CDATA[This is an info message that should not appear in silent test output]]></Description>
-</Message>
-<Message type="info" file="qtbase/tests/auto/testlib/selftests/silent/tst_silent.cpp" line="0">
-    <Description><![CDATA[This is an internal testlib info message that should not appear in silent test output]]></Description>
-</Message>
-<Message type="qfatal" file="" line="0">
-    <Description><![CDATA[This is a fatal error message that should still appear in silent test output]]></Description>
-</Message>
-<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/silent/tst_silent.cpp" line="0">
-    <Description><![CDATA[Received a fatal error.]]></Description>
-</Incident>
-    <Duration msecs="0"/>
-</TestFunction>
-<Duration msecs="0"/>
diff --git a/tests/auto/testlib/selftests/expected_silent.tap b/tests/auto/testlib/selftests/expected_silent.tap
deleted file mode 100644
index 5970f4a058..0000000000
--- a/tests/auto/testlib/selftests/expected_silent.tap
+++ /dev/null
@@ -1,42 +0,0 @@
-TAP version 13
-# tst_Silent
-ok 1 - initTestCase()
-ok 2 - pass()
-ok 3 - skip() # SKIP This test should skip
-not ok 4 - fail()
-  ---
-  type: QVERIFY
-  message: This test should fail
-  wanted: true (false)
-  found: false (false)
-  expected: true (false)
-  actual: false (false)
-  at: tst_Silent::fail() (qtbase/tests/auto/testlib/selftests/silent/tst_silent.cpp:0)
-  file: qtbase/tests/auto/testlib/selftests/silent/tst_silent.cpp
-  line: 0
-  ...
-not ok 5 - xfail() # TODO This test should XFAIL
-  ---
-  at: tst_Silent::xfail() (qtbase/tests/auto/testlib/selftests/silent/tst_silent.cpp:0)
-  file: qtbase/tests/auto/testlib/selftests/silent/tst_silent.cpp
-  line: 0
-  ...
-ok 6 - xpass() # TODO 'true' returned TRUE unexpectedly. (This test should XPASS)
-# This is a warning that should not appear in silent test output
-# This is an internal testlib warning that should not appear in silent test output
-# This is a debug message that should not appear in silent test output
-# This is a critical message that should not appear in silent test output
-# This is an info message that should not appear in silent test output
-# This is an internal testlib info message that should not appear in silent test output
-# This is a fatal error message that should still appear in silent test output
-not ok 7 - messages()
-  ---
-  # Received a fatal error.
-  at: tst_Silent::messages() (qtbase/tests/auto/testlib/selftests/silent/tst_silent.cpp:0)
-  file: qtbase/tests/auto/testlib/selftests/silent/tst_silent.cpp
-  line: 0
-  ...
-1..7
-# tests 7
-# pass 3
-# fail 3
diff --git a/tests/auto/testlib/selftests/expected_silent.teamcity b/tests/auto/testlib/selftests/expected_silent.teamcity
deleted file mode 100644
index a8b90b296a..0000000000
--- a/tests/auto/testlib/selftests/expected_silent.teamcity
+++ /dev/null
@@ -1,12 +0,0 @@
-##teamcity[testSuiteStarted name='tst_Silent' flowId='tst_Silent']
-##teamcity[testStarted name='fail()' flowId='tst_Silent']
-##teamcity[testFailed name='fail()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/silent/tst_silent.cpp(0)|]' details='|'false|' returned FALSE. (This test should fail)' flowId='tst_Silent']
-##teamcity[testFinished name='fail()' flowId='tst_Silent']
-##teamcity[testStarted name='xpass()' flowId='tst_Silent']
-##teamcity[testFailed name='xpass()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/silent/tst_silent.cpp(0)|]' details='|'true|' returned TRUE unexpectedly. (This test should XPASS)' flowId='tst_Silent']
-##teamcity[testFinished name='xpass()' flowId='tst_Silent']
-##teamcity[testStarted name='messages()' flowId='tst_Silent']
-##teamcity[testFailed name='messages()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/silent/tst_silent.cpp(0)|]' details='Received a fatal error.' flowId='tst_Silent']
-##teamcity[testStdOut name='messages()' out='QFATAL: This is a fatal error message that should still appear in silent test output' flowId='tst_Silent']
-##teamcity[testFinished name='messages()' flowId='tst_Silent']
-##teamcity[testSuiteFinished name='tst_Silent' flowId='tst_Silent']
diff --git a/tests/auto/testlib/selftests/expected_silent.xml b/tests/auto/testlib/selftests/expected_silent.xml
deleted file mode 100644
index 6c3ffc3e62..0000000000
--- a/tests/auto/testlib/selftests/expected_silent.xml
+++ /dev/null
@@ -1,69 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<TestCase name="tst_Silent">
-<Environment>
-    <QtVersion>@INSERT_QT_VERSION_HERE@</QtVersion>
-    <QtBuild/>
-    <QTestVersion>@INSERT_QT_VERSION_HERE@</QTestVersion>
-</Environment>
-<TestFunction name="initTestCase">
-<Incident type="pass" file="" line="0" />
-    <Duration msecs="0"/>
-</TestFunction>
-<TestFunction name="pass">
-<Incident type="pass" file="" line="0" />
-    <Duration msecs="0"/>
-</TestFunction>
-<TestFunction name="skip">
-<Message type="skip" file="qtbase/tests/auto/testlib/selftests/silent/tst_silent.cpp" line="0">
-    <Description><![CDATA[This test should skip]]></Description>
-</Message>
-    <Duration msecs="0"/>
-</TestFunction>
-<TestFunction name="fail">
-<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/silent/tst_silent.cpp" line="0">
-    <Description><![CDATA['false' returned FALSE. (This test should fail)]]></Description>
-</Incident>
-    <Duration msecs="0"/>
-</TestFunction>
-<TestFunction name="xfail">
-<Incident type="xfail" file="qtbase/tests/auto/testlib/selftests/silent/tst_silent.cpp" line="0">
-    <Description><![CDATA[This test should XFAIL]]></Description>
-</Incident>
-<Incident type="pass" file="" line="0" />
-    <Duration msecs="0"/>
-</TestFunction>
-<TestFunction name="xpass">
-<Incident type="xpass" file="qtbase/tests/auto/testlib/selftests/silent/tst_silent.cpp" line="0">
-    <Description><![CDATA['true' returned TRUE unexpectedly. (This test should XPASS)]]></Description>
-</Incident>
-    <Duration msecs="0"/>
-</TestFunction>
-<TestFunction name="messages">
-<Message type="qwarn" file="" line="0">
-    <Description><![CDATA[This is a warning that should not appear in silent test output]]></Description>
-</Message>
-<Message type="warn" file="qtbase/tests/auto/testlib/selftests/silent/tst_silent.cpp" line="0">
-    <Description><![CDATA[This is an internal testlib warning that should not appear in silent test output]]></Description>
-</Message>
-<Message type="qdebug" file="" line="0">
-    <Description><![CDATA[This is a debug message that should not appear in silent test output]]></Description>
-</Message>
-<Message type="system" file="" line="0">
-    <Description><![CDATA[This is a critical message that should not appear in silent test output]]></Description>
-</Message>
-<Message type="qinfo" file="" line="0">
-    <Description><![CDATA[This is an info message that should not appear in silent test output]]></Description>
-</Message>
-<Message type="info" file="qtbase/tests/auto/testlib/selftests/silent/tst_silent.cpp" line="0">
-    <Description><![CDATA[This is an internal testlib info message that should not appear in silent test output]]></Description>
-</Message>
-<Message type="qfatal" file="" line="0">
-    <Description><![CDATA[This is a fatal error message that should still appear in silent test output]]></Description>
-</Message>
-<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/silent/tst_silent.cpp" line="0">
-    <Description><![CDATA[Received a fatal error.]]></Description>
-</Incident>
-    <Duration msecs="0"/>
-</TestFunction>
-<Duration msecs="0"/>
-</TestCase>
diff --git a/tests/auto/testlib/selftests/generate_expected_output.py b/tests/auto/testlib/selftests/generate_expected_output.py
index 7dd9aa4576..5695f03df9 100755
--- a/tests/auto/testlib/selftests/generate_expected_output.py
+++ b/tests/auto/testlib/selftests/generate_expected_output.py
@@ -281,6 +281,15 @@ def testEnv(testname,
         data.update(extraEnv[testname])
     return data
 
+# See TestLogger::shouldIgnoreTest() in tst_selftest.cpp
+def shouldIgnoreTest(testname, format):
+    if testname == "junit" and not format == "junitxml":
+        return True
+    if testname in ["float", "silent"] and not format == "txt":
+        return True
+
+    return False
+
 def generateTestData(test_path, expected_path, clean, formats):
     """Run one test and save its cleaned results.
 
@@ -296,18 +305,11 @@ def generateTestData(test_path, expected_path, clean, formats):
         print("Warning: directory", testname, "contains no test executable")
         return
 
-    # See TestLogger::shouldIgnoreTest() in tst_selftest.cpp for these
-    # single-format tests:
-    if testname == 'junit':
-        formats = ( 'junitxml', ) if 'junitxml' in formats else ()
-    elif testname == 'float':
-        formats = ( 'txt', ) if 'txt' in formats else ()
-
     # Prepare environment in which to run tests:
     env = testEnv(testname)
 
     for format in formats:
-        if testname == "junit" and not format == "junitxml":
+        if shouldIgnoreTest(testname, format):
             continue
         print(f'  running {testname}/{format}')
         cmd = [path, f'-{format}']